const void *getResult(const void *firstRow) { IHThorAggregateArg *helper = (IHThorAggregateArg *)baseHelper.get(); unsigned numPartialResults = container.queryJob().querySlaves(); if (1 == numPartialResults) return firstRow; CThorExpandingRowArray partialResults(*this, this, true, stableSort_none, true, numPartialResults); if (hadElement) partialResults.setRow(0, firstRow); --numPartialResults; size32_t sz; while (numPartialResults--) { CMessageBuffer msg; rank_t sender; if (!receiveMsg(msg, RANK_ALL, mpTag, &sender)) return NULL; if (abortSoon) return NULL; msg.read(sz); if (sz) { assertex(NULL == partialResults.query(sender-1)); CThorStreamDeserializerSource mds(sz, msg.readDirect(sz)); RtlDynamicRowBuilder rowBuilder(queryRowAllocator()); size32_t sz = queryRowDeserializer()->deserialize(rowBuilder, mds); partialResults.setRow(sender-1, rowBuilder.finalizeRowClear(sz)); } } RtlDynamicRowBuilder rowBuilder(queryRowAllocator(), false); bool first = true; numPartialResults = container.queryJob().querySlaves(); unsigned p=0; for (;p<numPartialResults; p++) { const void *row = partialResults.query(p); if (row) { if (first) { first = false; sz = cloneRow(rowBuilder, row, queryRowMetaData()); } else sz = helper->mergeAggregate(rowBuilder, row); } } if (first) sz = helper->clearAggregate(rowBuilder); return rowBuilder.finalizeRowClear(sz); }
CATCH_NEXTROW() { ActivityTimer t(totalCycles, timeActivities); if (abortSoon || eof) return NULL; RtlDynamicRowBuilder out(queryRowAllocator()); size32_t sz = helper->clearAggregate(out); OwnedConstThorRow row = input->nextRow(); if (row) { sz = helper->processFirst(out, row); // NB: if ungrouped existsAggregate, no need to look at rest of input if (!ungroupedExistsAggregate) { while (!abortSoon) { row.setown(input->nextRow()); if (!row) break; sz = helper->processNext(out, row); } } if (!input->isGrouped()) eof = true; } else { eof = true; if (input->isGrouped()) return NULL; } dataLinkIncrement(); return out.finalizeRowClear(sz); }
void start() { ActivityTimer s(totalCycles, timeActivities); anyThisGroup = false; eogNext = false; count = 0; input = inputs.item(0); startInput(input); dataLinkStart(); RtlDynamicRowBuilder r(queryRowAllocator()); size32_t sz = helper->createDefault(r); defaultLeft.setown(r.finalizeRowClear(sz)); }
CATCH_NEXTROW() { ActivityTimer t(totalCycles, timeActivities); loop { if (eof || abortSoon) break; if (!prev) { if (!global || firstNode()) { // construct first row RtlDynamicRowBuilder r(queryRowAllocator()); size32_t sz = helper->createDefault(r); prev.setown(r.finalizeRowClear(sz)); } else { prev.setown(getFirst()); if (!prev) { putNext(NULL); // send to next node (even though prev not got) eof = true; break; } } } OwnedConstThorRow next = inputStream->ungroupedNextRow(); if (!next) { putNext(prev); // send to next node if applicable eof = true; break; } RtlDynamicRowBuilder ret(queryRowAllocator()); size32_t sz = helper->transform(ret, prev, next, ++count); if (sz != 0) { dataLinkIncrement(); prev.setown(ret.finalizeRowClear(sz)); return prev.getLink(); } } return NULL; }
CATCH_NEXTROW() { ActivityTimer t(totalCycles, timeActivities, NULL); if (eof || abortSoon) return NULL; RtlDynamicRowBuilder row(queryRowAllocator()); size32_t sizeGot = helper->getRow(row, currentRow++); if (sizeGot) { dataLinkIncrement(); return row.finalizeRowClear(sizeGot); } eof = true; return NULL; }
const void * getResult() { CMessageBuffer mb; if (receiveMsg(mb, 0, mpTag)) { size32_t sz; mb.read(sz); if (sz) { CThorStreamDeserializerSource ds(sz,mb.readDirect(sz)); RtlDynamicRowBuilder rowBuilder(queryRowAllocator()); size32_t sz = queryRowDeserializer()->deserialize(rowBuilder,ds); return rowBuilder.finalizeRowClear(sz); } } return NULL; }
CATCH_NEXTROW() { ActivityTimer t(totalCycles, timeActivities, NULL); if (empty || abortSoon) return NULL; // Filtering empty rows, returns the next valid row while (currentRow < numRows) { RtlDynamicRowBuilder row(queryRowAllocator()); size32_t sizeGot = helper->getRow(row, currentRow++); if (sizeGot) { dataLinkIncrement(); return row.finalizeRowClear(sizeGot); } } return NULL; }
CATCH_NEXTROW() { ActivityTimer t(totalCycles, timeActivities, NULL); if (abortSoon || rowSource.eos()) return NULL; if (eogPending) { eogPending = false; return NULL; } RtlDynamicRowBuilder rowBuilder(queryRowAllocator()); size32_t sz = queryRowDeserializer()->deserialize(rowBuilder,rowSource); if (grouped) rowSource.read(sizeof(bool), &eogPending); dataLinkIncrement(); return rowBuilder.finalizeRowClear(sz); }
CATCH_NEXTROW() { ActivityTimer t(totalCycles, timeActivities); if (abortSoon || eof) return NULL; eof = true; OwnedConstThorRow next = input->ungroupedNextRow(); RtlDynamicRowBuilder resultcr(queryRowAllocator()); size32_t sz = helper->clearAggregate(resultcr); if (next) { hadElement = true; sz = helper->processFirst(resultcr, next); if (container.getKind() != TAKexistsaggregate) { while (!abortSoon) { next.setown(input->ungroupedNextRow()); if (!next) break; sz = helper->processNext(resultcr, next); } } } doStopInput(); if (!firstNode()) { OwnedConstThorRow result(resultcr.finalizeRowClear(sz)); sendResult(result.get(),queryRowSerializer(), 1); // send partial result return NULL; } OwnedConstThorRow ret = getResult(resultcr.finalizeRowClear(sz)); if (ret) { dataLinkIncrement(); return ret.getClear(); } sz = helper->clearAggregate(resultcr); return resultcr.finalizeRowClear(sz); }
CATCH_NEXTROW() { ActivityTimer t(totalCycles, timeActivities); loop { if(abortSoon) break; if(eogNext) { eogNext = false; count = 0; if (anyThisGroup) { // ignore eogNext if none in group anyThisGroup = false; break; } } OwnedConstThorRow row = input->nextRow(); if (!row) { count = 0; if (anyThisGroup) { anyThisGroup = false; break; } row.setown(input->nextRow()); if (!row) break; } RtlDynamicRowBuilder ret(queryRowAllocator()); size32_t thisSize = helper->transform(ret, anyThisGroup?prev.get():defaultLeft.get(), row, ++count); if (thisSize != 0) { const void *r = ret.finalizeRowClear(thisSize); prev.set(r); dataLinkIncrement(); anyThisGroup = true; return r; } } return NULL; }
// IThorDataLink virtual void start() { ActivityTimer s(totalCycles, timeActivities, NULL); input = inputs.item(0); startInput(input); dataLinkStart("SELFJOIN", container.queryId()); bool hintparallelmatch = container.queryXGMML().getPropInt("hint[@name=\"parallel_match\"]")!=0; bool hintunsortedoutput = container.queryXGMML().getPropInt("hint[@name=\"unsorted_output\"]")!=0; if (helper->getJoinFlags()&JFlimitedprefixjoin) { CriticalBlock b(joinHelperCrit); // use std join helper (less efficient but implements limited prefix) joinhelper.setown(createJoinHelper(helper,"SELFJOIN", container.queryId(), queryRowAllocator(),hintparallelmatch,hintunsortedoutput)); } else { CriticalBlock b(joinHelperCrit); joinhelper.setown(createSelfJoinHelper(helper,"SELFJOIN", container.queryId(), queryRowAllocator(),hintparallelmatch,hintunsortedoutput)); } strm.setown(isLightweight? doLightweightSelfJoin() : (isLocal ? doLocalSelfJoin() : doGlobalSelfJoin())); assertex(strm); joinhelper->init(strm, NULL, ::queryRowAllocator(inputs.item(0)), ::queryRowAllocator(inputs.item(0)), ::queryRowMetaData(inputs.item(0)), &abortSoon, this); }
// IThorSlaveActivity overloaded methods virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData) { allocator.set(queryRowAllocator()); }
// IThorSlaveActivity overloaded methods virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData) { appendOutputLinked(this); helper = static_cast <IHThorXmlParseArg *> (queryHelper()); allocator.set(queryRowAllocator()); }
CATCH_NEXTROW() { ActivityTimer t(totalCycles, timeActivities); OwnedConstThorRow ret; Owned<IException> exception; if (first) // only return 1! { try { first = false; initN(); if (RCMAX==N) // indicates before start of dataset e.g. ds[0] { RtlDynamicRowBuilder row(queryRowAllocator()); size32_t sz = helper->createDefault(row); ret.setown(row.finalizeRowClear(sz)); N = 0; // return that processed all } else if (N) { while (!abortSoon) { ret.setown(input->ungroupedNextRow()); if (!ret) break; N--; { SpinBlock block(spin); if (lookaheadN<startN) // will not reach N==0, so don't bother continuing to read { N = startN-lookaheadN; ret.clear(); break; } } if (0==N) break; } if ((N!=0)&&createDefaultIfFail) { N = 0; // return that processed all (i.e. none left) RtlDynamicRowBuilder row(queryRowAllocator()); size32_t sz = helper->createDefault(row); ret.setown(row.finalizeRowClear(sz)); } } if (startN && 0 == N) seenNth = true; } catch (IException *e) { N=0; exception.setown(e); } sendN(); if (exception.get()) throw exception.getClear(); } if (ret) dataLinkIncrement(); return ret.getClear(); }