virtual void start() override
{
ActivityTimer s(totalCycles, timeActivities);
ActPrintLog(rolloverEnabled ? "GROUP: is global" : "GROUP: is local");
PARENT::start();
eogNext = prevEog = eof = false;
if (rolloverEnabled)
{
useRollover = !lastNode();
#ifdef _TESTING
ActPrintLog("Node number = %d, Total Nodes = %d", queryJobChannel().queryMyRank(), container.queryJob().querySlaves());
#endif
}
stream.set(inputStream);
startLastGroup = getDataLinkGlobalCount();
next.setown(getNext());
if (rolloverEnabled && !firstNode()) // 1st node can have nothing to send
{
Owned<IThorRowCollector> collector = createThorRowCollector(*this, this, NULL, stableSort_none, rc_mixed, SPILL_PRIORITY_SPILLABLE_STREAM);
Owned<IRowWriter> writer = collector->getWriter();
if (next)
{
ActPrintLog("GROUP: Sending first group to previous node(%d)", queryJobChannel().queryMyRank()-1);
for (;;)
{
writer->putRow(next.getLink());
if (abortSoon)
break; //always send group even when aborting
OwnedConstThorRow next2 = getNext();
if (!next2)
{
eof = true;
break;
}
else if (!helper->isSameGroup(next2, next))
{
next.setown(next2.getClear());
break;
}
next.setown(next2.getClear());
}
}
writer.clear();
ActPrintLog("GROUP: %" RCPF "d records to send", collector->numRows());
Owned<IRowStream> strm = collector->getStream();
rowServer.setown(createRowServer(this, strm, queryJobChannel().queryJobComm(), mpTag));
}
}
CATCH_NEXTROW()
{
ActivityTimer t(totalCycles, timeActivities);
if (abortSoon || eof)
return NULL;
RtlDynamicRowBuilder out(queryRowAllocator());
size32_t sz = helper->clearAggregate(out);
OwnedConstThorRow row = input->nextRow();
if (row)
{
sz = helper->processFirst(out, row);
// NB: if ungrouped existsAggregate, no need to look at rest of input
if (!ungroupedExistsAggregate)
{
while (!abortSoon)
{
row.setown(input->nextRow());
if (!row)
break;
sz = helper->processNext(out, row);
}
}
if (!input->isGrouped())
eof = true;
}
else
{
eof = true;
if (input->isGrouped())
return NULL;
}
dataLinkIncrement();
return out.finalizeRowClear(sz);
}
CATCH_NEXTROW()
{
ActivityTimer t(totalCycles, timeActivities);
while(!abortSoon)
{
OwnedConstThorRow row = input->nextRow();
if(!row) {
numToSkip = whichSample ? whichSample - 1 : 0;
if(anyThisGroup) {
anyThisGroup = false;
break;
}
row.setown(input->nextRow());
if(!row)
break;
}
if(numToSkip == 0) {
anyThisGroup = true;
numToSkip = numSamples - 1;
dataLinkIncrement();
return row.getClear();
}
numToSkip--;
}
return NULL;
}
CATCH_NEXTROW()
{
ActivityTimer t(totalCycles, timeActivities);
if (eogNext || eof)
{
eogNext = false;
return NULL;
}
OwnedConstThorRow prev = next.getClear();
next.setown(getNext());
if (next && !helper->isSameGroup(prev, next))
{
noteEndOfGroup();
eogNext = true;
}
if (prev)
{
dataLinkIncrement();
return prev.getClear();
}
if (prevEog)
{
noteEndOfGroup();
eof = true;
}
prevEog = true;
return NULL;
}
CATCH_NEXTROW()
{
ActivityTimer t(totalCycles, timeActivities, NULL);
while(!abortSoon)
{
OwnedConstThorRow row = input->nextRow();
if (!row)
{
if(anyThisGroup)
{
anyThisGroup = false;
break;
}
row.setown(input->nextRow());
if (!row)
break;
}
if(helper->isValid(row))
{
matched++;
anyThisGroup = true;
dataLinkIncrement();
return row.getClear();
}
}
return NULL;
}
virtual void write()
{
ActPrintLog("%s",grouped?"Grouped":"Ungrouped");
while(!abortSoon)
{
OwnedConstThorRow r = input->nextRow();
if (!r.get()) {
if (grouped) {
if ((processed & THORDATALINK_COUNT_MASK)!=0)
out->putRow(NULL);
}
r.setown(input->nextRow());
if (!r.get())
break;
}
out->putRow(r.getClear());
processed++;
}
}
CATCH_NEXTROW()
{
ActivityTimer t(totalCycles, timeActivities);
if (abortSoon || eof)
return NULL;
eof = true;
OwnedConstThorRow next = input->ungroupedNextRow();
RtlDynamicRowBuilder resultcr(queryRowAllocator());
size32_t sz = helper->clearAggregate(resultcr);
if (next)
{
hadElement = true;
sz = helper->processFirst(resultcr, next);
if (container.getKind() != TAKexistsaggregate)
{
while (!abortSoon)
{
next.setown(input->ungroupedNextRow());
if (!next)
break;
sz = helper->processNext(resultcr, next);
}
}
}
doStopInput();
if (!firstNode())
{
OwnedConstThorRow result(resultcr.finalizeRowClear(sz));
sendResult(result.get(),queryRowSerializer(), 1); // send partial result
return NULL;
}
OwnedConstThorRow ret = getResult(resultcr.finalizeRowClear(sz));
if (ret)
{
dataLinkIncrement();
return ret.getClear();
}
sz = helper->clearAggregate(resultcr);
return resultcr.finalizeRowClear(sz);
}
CATCH_NEXTROW()
{
ActivityTimer t(totalCycles, timeActivities);
loop {
if(abortSoon)
break;
if(eogNext) {
eogNext = false;
count = 0;
if (anyThisGroup) { // ignore eogNext if none in group
anyThisGroup = false;
break;
}
}
OwnedConstThorRow row = input->nextRow();
if (!row) {
count = 0;
if (anyThisGroup) {
anyThisGroup = false;
break;
}
row.setown(input->nextRow());
if (!row)
break;
}
RtlDynamicRowBuilder ret(queryRowAllocator());
size32_t thisSize = helper->transform(ret, anyThisGroup?prev.get():defaultLeft.get(), row, ++count);
if (thisSize != 0) {
const void *r = ret.finalizeRowClear(thisSize);
prev.set(r);
dataLinkIncrement();
anyThisGroup = true;
return r;
}
}
return NULL;
}
STRAND_CATCH_NEXTROW()
{
ActivityTimer t(totalCycles, timeActivities);
loop
{
if (parent.queryAbortSoon())
return nullptr;
OwnedConstThorRow in = inputStream->nextRow();
if (!in)
{
if (numProcessedLastGroup == rowsProcessed)
in.setown(inputStream->nextRow());
if (!in)
{
numProcessedLastGroup = rowsProcessed;
return nullptr;
}
}
RtlDynamicRowBuilder rowBuilder(allocator);
size32_t outSize;
try
{
outSize = helper->transform(rowBuilder, in);
}
catch (IException *e)
{
parent.ActPrintLog(e, "In helper->transform()");
throw;
}
if (outSize)
{
rowsProcessed++;
return rowBuilder.finalizeRowClear(outSize);
}
}
}
int run()
{
if (!started) {
try {
in->start();
started = true;
}
catch(IException * e)
{
ActPrintLog(&activity, e, "ThorLookaheadCache starting input");
startexception.setown(e);
if (asyncstart)
notify->onInputStarted(startexception);
running = false;
stopped = true;
startsem.signal();
return 0;
}
}
try {
StringBuffer temp;
if (allowspill)
GetTempName(temp,"lookahd",true);
assertex(bufsize);
if (allowspill)
smartbuf.setown(createSmartBuffer(&activity, temp.toCharArray(), bufsize, queryRowInterfaces(in)));
else
smartbuf.setown(createSmartInMemoryBuffer(&activity, queryRowInterfaces(in), bufsize));
if (notify)
notify->onInputStarted(NULL);
startsem.signal();
Linked<IRowWriter> writer = smartbuf->queryWriter();
if (preserveLhsGrouping)
{
while (required&&running)
{
OwnedConstThorRow row = in->nextRow();
if (!row)
{
row.setown(in->nextRow());
if (!row)
break;
else
writer->putRow(NULL); // eog
}
++count;
writer->putRow(row.getClear());
if (required!=RCUNBOUND)
required--;
}
}
else
{
while (required&&running)
{
OwnedConstThorRow row = in->ungroupedNextRow();
if (!row)
break;
++count;
writer->putRow(row.getClear());
if (required!=RCUNBOUND)
required--;
}
}
}
catch(IException * e)
{
ActPrintLog(&activity, e, "ThorLookaheadCache get exception");
getexception.setown(e);
}
if (notify)
notify->onInputFinished(count);
if (smartbuf)
smartbuf->queryWriter()->flush();
running = false;
try {
if (in)
in->stop();
}
catch(IException * e)
{
ActPrintLog(&activity, e, "ThorLookaheadCache stop exception");
if (!getexception.get())
getexception.setown(e);
}
return 0;
}
CATCH_NEXTROW()
{
ActivityTimer t(totalCycles, timeActivities);
OwnedConstThorRow ret;
Owned<IException> exception;
if (first) // only return 1!
{
try
{
first = false;
initN();
if (RCMAX==N) // indicates before start of dataset e.g. ds[0]
{
RtlDynamicRowBuilder row(queryRowAllocator());
size32_t sz = helper->createDefault(row);
ret.setown(row.finalizeRowClear(sz));
N = 0; // return that processed all
}
else if (N)
{
while (!abortSoon)
{
ret.setown(input->ungroupedNextRow());
if (!ret)
break;
N--;
{
SpinBlock block(spin);
if (lookaheadN<startN) // will not reach N==0, so don't bother continuing to read
{
N = startN-lookaheadN;
ret.clear();
break;
}
}
if (0==N)
break;
}
if ((N!=0)&&createDefaultIfFail)
{
N = 0; // return that processed all (i.e. none left)
RtlDynamicRowBuilder row(queryRowAllocator());
size32_t sz = helper->createDefault(row);
ret.setown(row.finalizeRowClear(sz));
}
}
if (startN && 0 == N)
seenNth = true;
}
catch (IException *e)
{
N=0;
exception.setown(e);
}
sendN();
if (exception.get())
throw exception.getClear();
}
if (ret)
dataLinkIncrement();
return ret.getClear();
}
