virtual void start() override { ActivityTimer s(totalCycles, timeActivities); ActPrintLog(rolloverEnabled ? "GROUP: is global" : "GROUP: is local"); PARENT::start(); eogNext = prevEog = eof = false; if (rolloverEnabled) { useRollover = !lastNode(); #ifdef _TESTING ActPrintLog("Node number = %d, Total Nodes = %d", queryJobChannel().queryMyRank(), container.queryJob().querySlaves()); #endif } stream.set(inputStream); startLastGroup = getDataLinkGlobalCount(); next.setown(getNext()); if (rolloverEnabled && !firstNode()) // 1st node can have nothing to send { Owned<IThorRowCollector> collector = createThorRowCollector(*this, this, NULL, stableSort_none, rc_mixed, SPILL_PRIORITY_SPILLABLE_STREAM); Owned<IRowWriter> writer = collector->getWriter(); if (next) { ActPrintLog("GROUP: Sending first group to previous node(%d)", queryJobChannel().queryMyRank()-1); for (;;) { writer->putRow(next.getLink()); if (abortSoon) break; //always send group even when aborting OwnedConstThorRow next2 = getNext(); if (!next2) { eof = true; break; } else if (!helper->isSameGroup(next2, next)) { next.setown(next2.getClear()); break; } next.setown(next2.getClear()); } } writer.clear(); ActPrintLog("GROUP: %" RCPF "d records to send", collector->numRows()); Owned<IRowStream> strm = collector->getStream(); rowServer.setown(createRowServer(this, strm, queryJobChannel().queryJobComm(), mpTag)); } }
CATCH_NEXTROW() { ActivityTimer t(totalCycles, timeActivities); if (abortSoon || eof) return NULL; RtlDynamicRowBuilder out(queryRowAllocator()); size32_t sz = helper->clearAggregate(out); OwnedConstThorRow row = input->nextRow(); if (row) { sz = helper->processFirst(out, row); // NB: if ungrouped existsAggregate, no need to look at rest of input if (!ungroupedExistsAggregate) { while (!abortSoon) { row.setown(input->nextRow()); if (!row) break; sz = helper->processNext(out, row); } } if (!input->isGrouped()) eof = true; } else { eof = true; if (input->isGrouped()) return NULL; } dataLinkIncrement(); return out.finalizeRowClear(sz); }
CATCH_NEXTROW() { ActivityTimer t(totalCycles, timeActivities); while(!abortSoon) { OwnedConstThorRow row = input->nextRow(); if(!row) { numToSkip = whichSample ? whichSample - 1 : 0; if(anyThisGroup) { anyThisGroup = false; break; } row.setown(input->nextRow()); if(!row) break; } if(numToSkip == 0) { anyThisGroup = true; numToSkip = numSamples - 1; dataLinkIncrement(); return row.getClear(); } numToSkip--; } return NULL; }
CATCH_NEXTROW() { ActivityTimer t(totalCycles, timeActivities); if (eogNext || eof) { eogNext = false; return NULL; } OwnedConstThorRow prev = next.getClear(); next.setown(getNext()); if (next && !helper->isSameGroup(prev, next)) { noteEndOfGroup(); eogNext = true; } if (prev) { dataLinkIncrement(); return prev.getClear(); } if (prevEog) { noteEndOfGroup(); eof = true; } prevEog = true; return NULL; }
CATCH_NEXTROW() { ActivityTimer t(totalCycles, timeActivities, NULL); while(!abortSoon) { OwnedConstThorRow row = input->nextRow(); if (!row) { if(anyThisGroup) { anyThisGroup = false; break; } row.setown(input->nextRow()); if (!row) break; } if(helper->isValid(row)) { matched++; anyThisGroup = true; dataLinkIncrement(); return row.getClear(); } } return NULL; }
virtual void write() { ActPrintLog("%s",grouped?"Grouped":"Ungrouped"); while(!abortSoon) { OwnedConstThorRow r = input->nextRow(); if (!r.get()) { if (grouped) { if ((processed & THORDATALINK_COUNT_MASK)!=0) out->putRow(NULL); } r.setown(input->nextRow()); if (!r.get()) break; } out->putRow(r.getClear()); processed++; } }
CATCH_NEXTROW() { ActivityTimer t(totalCycles, timeActivities); if (abortSoon || eof) return NULL; eof = true; OwnedConstThorRow next = input->ungroupedNextRow(); RtlDynamicRowBuilder resultcr(queryRowAllocator()); size32_t sz = helper->clearAggregate(resultcr); if (next) { hadElement = true; sz = helper->processFirst(resultcr, next); if (container.getKind() != TAKexistsaggregate) { while (!abortSoon) { next.setown(input->ungroupedNextRow()); if (!next) break; sz = helper->processNext(resultcr, next); } } } doStopInput(); if (!firstNode()) { OwnedConstThorRow result(resultcr.finalizeRowClear(sz)); sendResult(result.get(),queryRowSerializer(), 1); // send partial result return NULL; } OwnedConstThorRow ret = getResult(resultcr.finalizeRowClear(sz)); if (ret) { dataLinkIncrement(); return ret.getClear(); } sz = helper->clearAggregate(resultcr); return resultcr.finalizeRowClear(sz); }
CATCH_NEXTROW() { ActivityTimer t(totalCycles, timeActivities); loop { if(abortSoon) break; if(eogNext) { eogNext = false; count = 0; if (anyThisGroup) { // ignore eogNext if none in group anyThisGroup = false; break; } } OwnedConstThorRow row = input->nextRow(); if (!row) { count = 0; if (anyThisGroup) { anyThisGroup = false; break; } row.setown(input->nextRow()); if (!row) break; } RtlDynamicRowBuilder ret(queryRowAllocator()); size32_t thisSize = helper->transform(ret, anyThisGroup?prev.get():defaultLeft.get(), row, ++count); if (thisSize != 0) { const void *r = ret.finalizeRowClear(thisSize); prev.set(r); dataLinkIncrement(); anyThisGroup = true; return r; } } return NULL; }
STRAND_CATCH_NEXTROW() { ActivityTimer t(totalCycles, timeActivities); loop { if (parent.queryAbortSoon()) return nullptr; OwnedConstThorRow in = inputStream->nextRow(); if (!in) { if (numProcessedLastGroup == rowsProcessed) in.setown(inputStream->nextRow()); if (!in) { numProcessedLastGroup = rowsProcessed; return nullptr; } } RtlDynamicRowBuilder rowBuilder(allocator); size32_t outSize; try { outSize = helper->transform(rowBuilder, in); } catch (IException *e) { parent.ActPrintLog(e, "In helper->transform()"); throw; } if (outSize) { rowsProcessed++; return rowBuilder.finalizeRowClear(outSize); } } }
int run() { if (!started) { try { in->start(); started = true; } catch(IException * e) { ActPrintLog(&activity, e, "ThorLookaheadCache starting input"); startexception.setown(e); if (asyncstart) notify->onInputStarted(startexception); running = false; stopped = true; startsem.signal(); return 0; } } try { StringBuffer temp; if (allowspill) GetTempName(temp,"lookahd",true); assertex(bufsize); if (allowspill) smartbuf.setown(createSmartBuffer(&activity, temp.toCharArray(), bufsize, queryRowInterfaces(in))); else smartbuf.setown(createSmartInMemoryBuffer(&activity, queryRowInterfaces(in), bufsize)); if (notify) notify->onInputStarted(NULL); startsem.signal(); Linked<IRowWriter> writer = smartbuf->queryWriter(); if (preserveLhsGrouping) { while (required&&running) { OwnedConstThorRow row = in->nextRow(); if (!row) { row.setown(in->nextRow()); if (!row) break; else writer->putRow(NULL); // eog } ++count; writer->putRow(row.getClear()); if (required!=RCUNBOUND) required--; } } else { while (required&&running) { OwnedConstThorRow row = in->ungroupedNextRow(); if (!row) break; ++count; writer->putRow(row.getClear()); if (required!=RCUNBOUND) required--; } } } catch(IException * e) { ActPrintLog(&activity, e, "ThorLookaheadCache get exception"); getexception.setown(e); } if (notify) notify->onInputFinished(count); if (smartbuf) smartbuf->queryWriter()->flush(); running = false; try { if (in) in->stop(); } catch(IException * e) { ActPrintLog(&activity, e, "ThorLookaheadCache stop exception"); if (!getexception.get()) getexception.setown(e); } return 0; }
CATCH_NEXTROW() { ActivityTimer t(totalCycles, timeActivities); OwnedConstThorRow ret; Owned<IException> exception; if (first) // only return 1! { try { first = false; initN(); if (RCMAX==N) // indicates before start of dataset e.g. ds[0] { RtlDynamicRowBuilder row(queryRowAllocator()); size32_t sz = helper->createDefault(row); ret.setown(row.finalizeRowClear(sz)); N = 0; // return that processed all } else if (N) { while (!abortSoon) { ret.setown(input->ungroupedNextRow()); if (!ret) break; N--; { SpinBlock block(spin); if (lookaheadN<startN) // will not reach N==0, so don't bother continuing to read { N = startN-lookaheadN; ret.clear(); break; } } if (0==N) break; } if ((N!=0)&&createDefaultIfFail) { N = 0; // return that processed all (i.e. none left) RtlDynamicRowBuilder row(queryRowAllocator()); size32_t sz = helper->createDefault(row); ret.setown(row.finalizeRowClear(sz)); } } if (startN && 0 == N) seenNth = true; } catch (IException *e) { N=0; exception.setown(e); } sendN(); if (exception.get()) throw exception.getClear(); } if (ret) dataLinkIncrement(); return ret.getClear(); }