const void *getResult(const void *firstRow)
{
IHThorAggregateArg *helper = (IHThorAggregateArg *)baseHelper.get();
unsigned numPartialResults = container.queryJob().querySlaves();
if (1 == numPartialResults)
return firstRow;
CThorExpandingRowArray partialResults(*this, this, true, stableSort_none, true, numPartialResults);
if (hadElement)
partialResults.setRow(0, firstRow);
--numPartialResults;
size32_t sz;
while (numPartialResults--)
{
CMessageBuffer msg;
rank_t sender;
if (!receiveMsg(msg, RANK_ALL, mpTag, &sender))
return NULL;
if (abortSoon)
return NULL;
msg.read(sz);
if (sz)
{
assertex(NULL == partialResults.query(sender-1));
CThorStreamDeserializerSource mds(sz, msg.readDirect(sz));
RtlDynamicRowBuilder rowBuilder(queryRowAllocator());
size32_t sz = queryRowDeserializer()->deserialize(rowBuilder, mds);
partialResults.setRow(sender-1, rowBuilder.finalizeRowClear(sz));
}
}
RtlDynamicRowBuilder rowBuilder(queryRowAllocator(), false);
bool first = true;
numPartialResults = container.queryJob().querySlaves();
unsigned p=0;
for (;p<numPartialResults; p++)
{
const void *row = partialResults.query(p);
if (row)
{
if (first)
{
first = false;
sz = cloneRow(rowBuilder, row, queryRowMetaData());
}
else
sz = helper->mergeAggregate(rowBuilder, row);
}
}
if (first)
sz = helper->clearAggregate(rowBuilder);
return rowBuilder.finalizeRowClear(sz);
}
CATCH_NEXTROW()
{
ActivityTimer t(totalCycles, timeActivities);
if (abortSoon || eof)
return NULL;
RtlDynamicRowBuilder out(queryRowAllocator());
size32_t sz = helper->clearAggregate(out);
OwnedConstThorRow row = input->nextRow();
if (row)
{
sz = helper->processFirst(out, row);
// NB: if ungrouped existsAggregate, no need to look at rest of input
if (!ungroupedExistsAggregate)
{
while (!abortSoon)
{
row.setown(input->nextRow());
if (!row)
break;
sz = helper->processNext(out, row);
}
}
if (!input->isGrouped())
eof = true;
}
else
{
eof = true;
if (input->isGrouped())
return NULL;
}
dataLinkIncrement();
return out.finalizeRowClear(sz);
}
void start()
{
ActivityTimer s(totalCycles, timeActivities);
anyThisGroup = false;
eogNext = false;
count = 0;
input = inputs.item(0);
startInput(input);
dataLinkStart();
RtlDynamicRowBuilder r(queryRowAllocator());
size32_t sz = helper->createDefault(r);
defaultLeft.setown(r.finalizeRowClear(sz));
}
CATCH_NEXTROW()
{
ActivityTimer t(totalCycles, timeActivities);
loop {
if (eof || abortSoon)
break;
if (!prev) {
if (!global || firstNode()) {
// construct first row
RtlDynamicRowBuilder r(queryRowAllocator());
size32_t sz = helper->createDefault(r);
prev.setown(r.finalizeRowClear(sz));
}
else {
prev.setown(getFirst());
if (!prev) {
putNext(NULL); // send to next node (even though prev not got)
eof = true;
break;
}
}
}
OwnedConstThorRow next = inputStream->ungroupedNextRow();
if (!next) {
putNext(prev); // send to next node if applicable
eof = true;
break;
}
RtlDynamicRowBuilder ret(queryRowAllocator());
size32_t sz = helper->transform(ret, prev, next, ++count);
if (sz != 0) {
dataLinkIncrement();
prev.setown(ret.finalizeRowClear(sz));
return prev.getLink();
}
}
return NULL;
}
CATCH_NEXTROW()
{
ActivityTimer t(totalCycles, timeActivities, NULL);
if (eof || abortSoon)
return NULL;
RtlDynamicRowBuilder row(queryRowAllocator());
size32_t sizeGot = helper->getRow(row, currentRow++);
if (sizeGot)
{
dataLinkIncrement();
return row.finalizeRowClear(sizeGot);
}
eof = true;
return NULL;
}
const void * getResult()
{
CMessageBuffer mb;
if (receiveMsg(mb, 0, mpTag)) {
size32_t sz;
mb.read(sz);
if (sz) {
CThorStreamDeserializerSource ds(sz,mb.readDirect(sz));
RtlDynamicRowBuilder rowBuilder(queryRowAllocator());
size32_t sz = queryRowDeserializer()->deserialize(rowBuilder,ds);
return rowBuilder.finalizeRowClear(sz);
}
}
return NULL;
}
CATCH_NEXTROW()
{
ActivityTimer t(totalCycles, timeActivities, NULL);
if (empty || abortSoon)
return NULL;
// Filtering empty rows, returns the next valid row
while (currentRow < numRows) {
RtlDynamicRowBuilder row(queryRowAllocator());
size32_t sizeGot = helper->getRow(row, currentRow++);
if (sizeGot)
{
dataLinkIncrement();
return row.finalizeRowClear(sizeGot);
}
}
return NULL;
}
CATCH_NEXTROW()
{
ActivityTimer t(totalCycles, timeActivities, NULL);
if (abortSoon || rowSource.eos())
return NULL;
if (eogPending)
{
eogPending = false;
return NULL;
}
RtlDynamicRowBuilder rowBuilder(queryRowAllocator());
size32_t sz = queryRowDeserializer()->deserialize(rowBuilder,rowSource);
if (grouped)
rowSource.read(sizeof(bool), &eogPending);
dataLinkIncrement();
return rowBuilder.finalizeRowClear(sz);
}
CATCH_NEXTROW()
{
ActivityTimer t(totalCycles, timeActivities);
if (abortSoon || eof)
return NULL;
eof = true;
OwnedConstThorRow next = input->ungroupedNextRow();
RtlDynamicRowBuilder resultcr(queryRowAllocator());
size32_t sz = helper->clearAggregate(resultcr);
if (next)
{
hadElement = true;
sz = helper->processFirst(resultcr, next);
if (container.getKind() != TAKexistsaggregate)
{
while (!abortSoon)
{
next.setown(input->ungroupedNextRow());
if (!next)
break;
sz = helper->processNext(resultcr, next);
}
}
}
doStopInput();
if (!firstNode())
{
OwnedConstThorRow result(resultcr.finalizeRowClear(sz));
sendResult(result.get(),queryRowSerializer(), 1); // send partial result
return NULL;
}
OwnedConstThorRow ret = getResult(resultcr.finalizeRowClear(sz));
if (ret)
{
dataLinkIncrement();
return ret.getClear();
}
sz = helper->clearAggregate(resultcr);
return resultcr.finalizeRowClear(sz);
}
CATCH_NEXTROW()
{
ActivityTimer t(totalCycles, timeActivities);
loop {
if(abortSoon)
break;
if(eogNext) {
eogNext = false;
count = 0;
if (anyThisGroup) { // ignore eogNext if none in group
anyThisGroup = false;
break;
}
}
OwnedConstThorRow row = input->nextRow();
if (!row) {
count = 0;
if (anyThisGroup) {
anyThisGroup = false;
break;
}
row.setown(input->nextRow());
if (!row)
break;
}
RtlDynamicRowBuilder ret(queryRowAllocator());
size32_t thisSize = helper->transform(ret, anyThisGroup?prev.get():defaultLeft.get(), row, ++count);
if (thisSize != 0) {
const void *r = ret.finalizeRowClear(thisSize);
prev.set(r);
dataLinkIncrement();
anyThisGroup = true;
return r;
}
}
return NULL;
}
// IThorDataLink
virtual void start()
{
ActivityTimer s(totalCycles, timeActivities, NULL);
input = inputs.item(0);
startInput(input);
dataLinkStart("SELFJOIN", container.queryId());
bool hintparallelmatch = container.queryXGMML().getPropInt("hint[@name=\"parallel_match\"]")!=0;
bool hintunsortedoutput = container.queryXGMML().getPropInt("hint[@name=\"unsorted_output\"]")!=0;
if (helper->getJoinFlags()&JFlimitedprefixjoin) {
CriticalBlock b(joinHelperCrit);
// use std join helper (less efficient but implements limited prefix)
joinhelper.setown(createJoinHelper(helper,"SELFJOIN", container.queryId(), queryRowAllocator(),hintparallelmatch,hintunsortedoutput));
}
else {
CriticalBlock b(joinHelperCrit);
joinhelper.setown(createSelfJoinHelper(helper,"SELFJOIN", container.queryId(), queryRowAllocator(),hintparallelmatch,hintunsortedoutput));
}
strm.setown(isLightweight? doLightweightSelfJoin() : (isLocal ? doLocalSelfJoin() : doGlobalSelfJoin()));
assertex(strm);
joinhelper->init(strm, NULL, ::queryRowAllocator(inputs.item(0)), ::queryRowAllocator(inputs.item(0)), ::queryRowMetaData(inputs.item(0)), &abortSoon, this);
}
// IThorSlaveActivity overloaded methods
virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData)
{
allocator.set(queryRowAllocator());
}
// IThorSlaveActivity overloaded methods
virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData)
{
appendOutputLinked(this);
helper = static_cast <IHThorXmlParseArg *> (queryHelper());
allocator.set(queryRowAllocator());
}
CATCH_NEXTROW()
{
ActivityTimer t(totalCycles, timeActivities);
OwnedConstThorRow ret;
Owned<IException> exception;
if (first) // only return 1!
{
try
{
first = false;
initN();
if (RCMAX==N) // indicates before start of dataset e.g. ds[0]
{
RtlDynamicRowBuilder row(queryRowAllocator());
size32_t sz = helper->createDefault(row);
ret.setown(row.finalizeRowClear(sz));
N = 0; // return that processed all
}
else if (N)
{
while (!abortSoon)
{
ret.setown(input->ungroupedNextRow());
if (!ret)
break;
N--;
{
SpinBlock block(spin);
if (lookaheadN<startN) // will not reach N==0, so don't bother continuing to read
{
N = startN-lookaheadN;
ret.clear();
break;
}
}
if (0==N)
break;
}
if ((N!=0)&&createDefaultIfFail)
{
N = 0; // return that processed all (i.e. none left)
RtlDynamicRowBuilder row(queryRowAllocator());
size32_t sz = helper->createDefault(row);
ret.setown(row.finalizeRowClear(sz));
}
}
if (startN && 0 == N)
seenNth = true;
}
catch (IException *e)
{
N=0;
exception.setown(e);
}
sendN();
if (exception.get())
throw exception.getClear();
}
if (ret)
dataLinkIncrement();
return ret.getClear();
}