CATCH_NEXTROW()
{
ActivityTimer t(totalCycles, timeActivities);
if (abortSoon || eof)
return NULL;
eof = true;
OwnedConstThorRow next = inputStream->ungroupedNextRow();
RtlDynamicRowBuilder resultcr(queryRowAllocator());
size32_t sz = helper->clearAggregate(resultcr);
if (next)
{
hadElement = true;
sz = helper->processFirst(resultcr, next);
if (container.getKind() != TAKexistsaggregate)
{
while (!abortSoon)
{
next.setown(inputStream->ungroupedNextRow());
if (!next)
break;
sz = helper->processNext(resultcr, next);
}
}
}
doStopInput();
if (!firstNode())
{
OwnedConstThorRow result(resultcr.finalizeRowClear(sz));
sendResult(result.get(),queryRowSerializer(), 1); // send partial result
return NULL;
}
OwnedConstThorRow ret = getResult(resultcr.finalizeRowClear(sz));
if (ret)
{
dataLinkIncrement();
return ret.getClear();
}
sz = helper->clearAggregate(resultcr);
return resultcr.finalizeRowClear(sz);
}
CATCH_NEXTROW()
{
ActivityTimer t(totalCycles, timeActivities);
loop
{
OwnedConstThorRow row = input->ungroupedNextRow();
if(!row || done || abortSoon)
break;
switch (helper->getRecordAction(row))
{
case 2:
done = true;
//fall through
case 1:
dataLinkIncrement();
return row.getClear();
}
}
return NULL;
}
CATCH_NEXTROW()
{
ActivityTimer t(totalCycles, timeActivities, NULL);
if (abortSoon || eoi)
return NULL;
OwnedConstThorRow row = out->nextRow();
if (!row)
{
if (!container.queryGrouped())
{
eoi = true;
return NULL;
}
out.setown(iLoader->loadGroup(input, abortSoon));
if (0 == iLoader->numRows())
eoi = true;
return NULL; // eog marker
}
dataLinkIncrement();
return row.getClear();
}
void process()
{
processed = 0;
input = inputs.item(0);
startInput(input);
processed = THORDATALINK_STARTED;
OwnedConstThorRow row = input->ungroupedNextRow();
CMessageBuffer mb;
size32_t lenpos = mb.length(); // its 0 really
mb.append((size32_t)0);
if (row) {
CMemoryRowSerializer msz(mb);
::queryRowSerializer(input)->serialize(msz,(const byte *)row.get());
size32_t sz = mb.length()-lenpos-sizeof(size32_t);
mb.writeDirect(lenpos,sizeof(size32_t),&sz);
processed++;
}
container.queryJob().queryJobComm().send(mb, 0, masterMpTag);
}
CATCH_NEXTROW()
{
ActivityTimer t(totalCycles, timeActivities);
loop {
if(abortSoon)
break;
if(eogNext) {
eogNext = false;
count = 0;
if (anyThisGroup) { // ignore eogNext if none in group
anyThisGroup = false;
break;
}
}
OwnedConstThorRow row = input->nextRow();
if (!row) {
count = 0;
if (anyThisGroup) {
anyThisGroup = false;
break;
}
row.setown(input->nextRow());
if (!row)
break;
}
RtlDynamicRowBuilder ret(queryRowAllocator());
size32_t thisSize = helper->transform(ret, anyThisGroup?prev.get():defaultLeft.get(), row, ++count);
if (thisSize != 0) {
const void *r = ret.finalizeRowClear(thisSize);
prev.set(r);
dataLinkIncrement();
anyThisGroup = true;
return r;
}
}
return NULL;
}
STRAND_CATCH_NEXTROW()
{
ActivityTimer t(totalCycles, timeActivities);
loop
{
if (parent.queryAbortSoon())
return nullptr;
OwnedConstThorRow in = inputStream->nextRow();
if (!in)
{
if (numProcessedLastGroup == rowsProcessed)
in.setown(inputStream->nextRow());
if (!in)
{
numProcessedLastGroup = rowsProcessed;
return nullptr;
}
}
RtlDynamicRowBuilder rowBuilder(allocator);
size32_t outSize;
try
{
outSize = helper->transform(rowBuilder, in);
}
catch (IException *e)
{
parent.ActPrintLog(e, "In helper->transform()");
throw;
}
if (outSize)
{
rowsProcessed++;
return rowBuilder.finalizeRowClear(outSize);
}
}
}
IRowStream *createFirstNReadSeqVar(IRowStream *input, unsigned limit)
{
class CFirstNReadSeqVar : public CSimpleInterface, implements IRowStream
{
IRowStream *input;
unsigned limit, c;
bool stopped;
public:
IMPLEMENT_IINTERFACE_USING(CSimpleInterface);
CFirstNReadSeqVar(IRowStream *_input, unsigned _limit) : input(_input), limit(_limit), c(0), stopped(false) { }
~CFirstNReadSeqVar() { ::Release(input); }
// IRowStream
const void *nextRow()
{
if (c<limit)
{
OwnedConstThorRow row = input->nextRow();
if (row)
{
c++;
return row.getClear();
}
}
stop();
return NULL;
}
virtual void stop()
{
if (!stopped)
{
stopped = true;
input->stop();
}
}
};
void operator = (const OwnedConstThorRow & other) { set(other.get()); }
inline OwnedConstThorRow(const OwnedConstThorRow & other) { ptr = other.getLink(); }
inline void set(const OwnedConstThorRow &other) { set(other.get()); }
virtual void process() override
{
ActPrintLog("INDEXWRITE: Start");
init();
IRowStream *stream = inputStream;
ThorDataLinkMetaInfo info;
input->getMetaInfo(info);
outRowAllocator.setown(getRowAllocator(helper->queryDiskRecordSize()));
start();
if (refactor)
{
assertex(isLocal);
if (active)
{
unsigned targetWidth = partDesc->queryOwner().numParts()-(buildTlk?1:0);
assertex(0 == container.queryJob().querySlaves() % targetWidth);
unsigned partsPerNode = container.queryJob().querySlaves() / targetWidth;
unsigned myPart = queryJobChannel().queryMyRank();
IArrayOf<IRowStream> streams;
streams.append(*LINK(stream));
--partsPerNode;
// Should this be merging 1,11,21,31 etc.
unsigned p=0;
unsigned fromPart = targetWidth+1 + (partsPerNode * (myPart-1));
for (; p<partsPerNode; p++)
{
streams.append(*createRowStreamFromNode(*this, fromPart++, queryJobChannel().queryJobComm(), mpTag, abortSoon));
}
ICompare *icompare = helper->queryCompare();
assertex(icompare);
Owned<IRowLinkCounter> linkCounter = new CThorRowLinkCounter;
myInputStream.setown(createRowStreamMerger(streams.ordinality(), streams.getArray(), icompare, false, linkCounter));
stream = myInputStream;
}
else // serve nodes, creating merged parts
rowServer.setown(createRowServer(this, stream, queryJobChannel().queryJobComm(), mpTag));
}
processed = THORDATALINK_STARTED;
// single part key support
// has to serially pull all data fron nodes 2-N
// nodes 2-N, could/should start pushing some data (as it's supposed to be small) to cut down on serial nature.
unsigned node = queryJobChannel().queryMyRank();
if (singlePartKey)
{
if (1 == node)
{
try
{
open(*partDesc, false, helper->queryDiskRecordSize()->isVariableSize());
loop
{
OwnedConstThorRow row = inputStream->ungroupedNextRow();
if (!row)
break;
if (abortSoon) return;
processRow(row);
}
unsigned node = 2;
while (node <= container.queryJob().querySlaves())
{
Linked<IOutputRowDeserializer> deserializer = ::queryRowDeserializer(input);
CMessageBuffer mb;
Owned<ISerialStream> stream = createMemoryBufferSerialStream(mb);
CThorStreamDeserializerSource rowSource;
rowSource.setStream(stream);
bool successSR;
loop
{
{
BooleanOnOff tf(receivingTag2);
successSR = queryJobChannel().queryJobComm().sendRecv(mb, node, mpTag2);
}
if (successSR)
{
if (rowSource.eos())
break;
Linked<IEngineRowAllocator> allocator = ::queryRowAllocator(input);
do
{
RtlDynamicRowBuilder rowBuilder(allocator);
size32_t sz = deserializer->deserialize(rowBuilder, rowSource);
OwnedConstThorRow fRow = rowBuilder.finalizeRowClear(sz);
processRow(fRow);
}
while (!rowSource.eos());
}
}
node++;
}
}
catch (CATCHALL)
{
close(*partDesc, partCrc, true);
throw;
}
close(*partDesc, partCrc, true);
doStopInput();
}
else
{
CMessageBuffer mb;
CMemoryRowSerializer mbs(mb);
Linked<IOutputRowSerializer> serializer = ::queryRowSerializer(input);
loop
{
BooleanOnOff tf(receivingTag2);
if (queryJobChannel().queryJobComm().recv(mb, 1, mpTag2)) // node 1 asking for more..
{
if (abortSoon) break;
mb.clear();
do
{
OwnedConstThorRow row = inputStream->ungroupedNextRow();
if (!row) break;
serializer->serialize(mbs, (const byte *)row.get());
} while (mb.length() < SINGLEPART_KEY_TRANSFER_SIZE); // NB: at least one row
if (!queryJobChannel().queryJobComm().reply(mb))
throw MakeThorException(0, "Failed to send index data to node 1, from node %d", node);
if (0 == mb.length())
break;
}
}
}
}
CATCH_NEXTROW()
{
ActivityTimer t(totalCycles, timeActivities);
if (abortSoon)
return NULL;
if (eogNext)
{
eogNext = false;
anyThisGroup = false;
return NULL;
}
try
{
for (;;)
{
if (xmlParser)
{
for (;;)
{
if (!xmlParser->next())
{
if (helper->searchTextNeedsFree())
{
rtlFree(searchStr);
searchStr = NULL;
}
xmlParser.clear();
break;
}
if (lastMatch)
{
RtlDynamicRowBuilder row(allocator);
size32_t sizeGot;
try { sizeGot = helper->transform(row, nxt, lastMatch); }
catch (IException *e)
{
ActPrintLog(e, "In helper->transform()");
throw;
}
lastMatch.clear();
if (sizeGot == 0)
continue; // not sure if this will ever be possible in this context.
dataLinkIncrement();
anyThisGroup = true;
OwnedConstThorRow ret = row.finalizeRowClear(sizeGot);
return ret.getClear();
}
}
}
nxt.setown(inputStream->nextRow());
if (!nxt && !anyThisGroup)
nxt.setown(inputStream->nextRow());
if (!nxt)
break;
unsigned len;
helper->getSearchText(len, searchStr, nxt);
OwnedRoxieString xmlIteratorPath(helper->getXmlIteratorPath());
xmlParser.setown(createXMLParse(searchStr, len, xmlIteratorPath, *this, ptr_noRoot, helper->requiresContents()));
}
}
catch (IOutOfMemException *e)
{
StringBuffer s("XMLParse actId(");
s.append(container.queryId()).append(") out of memory.").newline();
s.append("INTERNAL ERROR ").append(e->errorCode()).append(": ");
e->errorMessage(s);
e->Release();
throw MakeActivityException(this, 0, "%s", s.str());
}
catch (IException *e)
{
StringBuffer s("XMLParse actId(");
s.append(container.queryId());
s.append(") INTERNAL ERROR ").append(e->errorCode()).append(": ");
e->errorMessage(s);
e->Release();
throw MakeActivityException(this, 0, "%s", s.str());
}
eogNext = false;
anyThisGroup = false;
return NULL;
}
int run()
{
if (!started) {
try {
in->start();
started = true;
}
catch(IException * e)
{
ActPrintLog(&activity, e, "ThorLookaheadCache starting input");
startexception.setown(e);
if (asyncstart)
notify->onInputStarted(startexception);
running = false;
stopped = true;
startsem.signal();
return 0;
}
}
try {
StringBuffer temp;
if (allowspill)
GetTempName(temp,"lookahd",true);
assertex(bufsize);
if (allowspill)
smartbuf.setown(createSmartBuffer(&activity, temp.toCharArray(), bufsize, queryRowInterfaces(in)));
else
smartbuf.setown(createSmartInMemoryBuffer(&activity, queryRowInterfaces(in), bufsize));
if (notify)
notify->onInputStarted(NULL);
startsem.signal();
Linked<IRowWriter> writer = smartbuf->queryWriter();
if (preserveLhsGrouping)
{
while (required&&running)
{
OwnedConstThorRow row = in->nextRow();
if (!row)
{
row.setown(in->nextRow());
if (!row)
break;
else
writer->putRow(NULL); // eog
}
++count;
writer->putRow(row.getClear());
if (required!=RCUNBOUND)
required--;
}
}
else
{
while (required&&running)
{
OwnedConstThorRow row = in->ungroupedNextRow();
if (!row)
break;
++count;
writer->putRow(row.getClear());
if (required!=RCUNBOUND)
required--;
}
}
}
catch(IException * e)
{
ActPrintLog(&activity, e, "ThorLookaheadCache get exception");
getexception.setown(e);
}
if (notify)
notify->onInputFinished(count);
if (smartbuf)
smartbuf->queryWriter()->flush();
running = false;
try {
if (in)
in->stop();
}
catch(IException * e)
{
ActPrintLog(&activity, e, "ThorLookaheadCache stop exception");
if (!getexception.get())
getexception.setown(e);
}
return 0;
}
CATCH_NEXTROW()
{
ActivityTimer t(totalCycles, timeActivities);
OwnedConstThorRow ret;
Owned<IException> exception;
if (first) // only return 1!
{
try
{
first = false;
initN();
if (RCMAX==N) // indicates before start of dataset e.g. ds[0]
{
RtlDynamicRowBuilder row(queryRowAllocator());
size32_t sz = helper->createDefault(row);
ret.setown(row.finalizeRowClear(sz));
N = 0; // return that processed all
}
else if (N)
{
while (!abortSoon)
{
ret.setown(input->ungroupedNextRow());
if (!ret)
break;
N--;
{
SpinBlock block(spin);
if (lookaheadN<startN) // will not reach N==0, so don't bother continuing to read
{
N = startN-lookaheadN;
ret.clear();
break;
}
}
if (0==N)
break;
}
if ((N!=0)&&createDefaultIfFail)
{
N = 0; // return that processed all (i.e. none left)
RtlDynamicRowBuilder row(queryRowAllocator());
size32_t sz = helper->createDefault(row);
ret.setown(row.finalizeRowClear(sz));
}
}
if (startN && 0 == N)
seenNth = true;
}
catch (IException *e)
{
N=0;
exception.setown(e);
}
sendN();
if (exception.get())
throw exception.getClear();
}
if (ret)
dataLinkIncrement();
return ret.getClear();
}
virtual void process() override
{
ActPrintLog("INDEXWRITE: Start");
init();
IRowStream *stream = inputStream;
ThorDataLinkMetaInfo info;
input->getMetaInfo(info);
outRowAllocator.setown(getRowAllocator(helper->queryDiskRecordSize()));
start();
if (refactor)
{
assertex(isLocal);
if (active)
{
unsigned targetWidth = partDesc->queryOwner().numParts()-(buildTlk?1:0);
assertex(0 == container.queryJob().querySlaves() % targetWidth);
unsigned partsPerNode = container.queryJob().querySlaves() / targetWidth;
unsigned myPart = queryJobChannel().queryMyRank();
IArrayOf<IRowStream> streams;
streams.append(*LINK(stream));
--partsPerNode;
// Should this be merging 1,11,21,31 etc.
unsigned p=0;
unsigned fromPart = targetWidth+1 + (partsPerNode * (myPart-1));
for (; p<partsPerNode; p++)
{
streams.append(*createRowStreamFromNode(*this, fromPart++, queryJobChannel().queryJobComm(), mpTag, abortSoon));
}
ICompare *icompare = helper->queryCompare();
assertex(icompare);
Owned<IRowLinkCounter> linkCounter = new CThorRowLinkCounter;
myInputStream.setown(createRowStreamMerger(streams.ordinality(), streams.getArray(), icompare, false, linkCounter));
stream = myInputStream;
}
else // serve nodes, creating merged parts
rowServer.setown(createRowServer(this, stream, queryJobChannel().queryJobComm(), mpTag));
}
processed = THORDATALINK_STARTED;
// single part key support
// has to serially pull all data fron nodes 2-N
// nodes 2-N, could/should start pushing some data (as it's supposed to be small) to cut down on serial nature.
unsigned node = queryJobChannel().queryMyRank();
if (singlePartKey)
{
if (1 == node)
{
try
{
open(*partDesc, false, helper->queryDiskRecordSize()->isVariableSize());
for (;;)
{
OwnedConstThorRow row = inputStream->ungroupedNextRow();
if (!row)
break;
if (abortSoon) return;
processRow(row);
}
unsigned node = 2;
while (node <= container.queryJob().querySlaves())
{
Linked<IOutputRowDeserializer> deserializer = ::queryRowDeserializer(input);
CMessageBuffer mb;
Owned<ISerialStream> stream = createMemoryBufferSerialStream(mb);
CThorStreamDeserializerSource rowSource;
rowSource.setStream(stream);
bool successSR;
for (;;)
{
{
BooleanOnOff tf(receivingTag2);
successSR = queryJobChannel().queryJobComm().sendRecv(mb, node, mpTag2);
}
if (successSR)
{
if (rowSource.eos())
break;
Linked<IEngineRowAllocator> allocator = ::queryRowAllocator(input);
do
{
RtlDynamicRowBuilder rowBuilder(allocator);
size32_t sz = deserializer->deserialize(rowBuilder, rowSource);
OwnedConstThorRow fRow = rowBuilder.finalizeRowClear(sz);
processRow(fRow);
}
while (!rowSource.eos());
}
}
node++;
}
}
catch (CATCHALL)
{
close(*partDesc, partCrc, true);
throw;
}
close(*partDesc, partCrc, true);
stop();
}
else
{
CMessageBuffer mb;
CMemoryRowSerializer mbs(mb);
Linked<IOutputRowSerializer> serializer = ::queryRowSerializer(input);
for (;;)
{
BooleanOnOff tf(receivingTag2);
if (queryJobChannel().queryJobComm().recv(mb, 1, mpTag2)) // node 1 asking for more..
{
if (abortSoon) break;
mb.clear();
do
{
OwnedConstThorRow row = inputStream->ungroupedNextRow();
if (!row) break;
serializer->serialize(mbs, (const byte *)row.get());
} while (mb.length() < SINGLEPART_KEY_TRANSFER_SIZE); // NB: at least one row
if (!queryJobChannel().queryJobComm().reply(mb))
throw MakeThorException(0, "Failed to send index data to node 1, from node %d", node);
if (0 == mb.length())
break;
}
}
}
}
else
{
if (!refactor || active)
{
try
{
StringBuffer partFname;
getPartFilename(*partDesc, 0, partFname);
ActPrintLog("INDEXWRITE: process: handling fname : %s", partFname.str());
open(*partDesc, false, helper->queryDiskRecordSize()->isVariableSize());
ActPrintLog("INDEXWRITE: write");
BooleanOnOff tf(receiving);
if (!refactor || !active)
receiving = false;
do
{
OwnedConstThorRow row = inputStream->ungroupedNextRow();
if (!row)
break;
processRow(row);
} while (!abortSoon);
ActPrintLog("INDEXWRITE: write level 0 complete");
}
catch (CATCHALL)
{
close(*partDesc, partCrc, isLocal && !buildTlk && 1 == node);
throw;
}
close(*partDesc, partCrc, isLocal && !buildTlk && 1 == node);
stop();
ActPrintLog("INDEXWRITE: Wrote %" RCPF "d records", processed & THORDATALINK_COUNT_MASK);
if (buildTlk)
{
ActPrintLog("INDEXWRITE: sending rows");
NodeInfoArray tlkRows;
CMessageBuffer msg;
if (firstNode())
{
if (processed & THORDATALINK_COUNT_MASK)
{
if (enableTlkPart0)
tlkRows.append(* new CNodeInfo(0, firstRow.get(), firstRowSize, totalCount));
tlkRows.append(* new CNodeInfo(1, lastRow.get(), lastRowSize, totalCount));
}
}
else
{
if (processed & THORDATALINK_COUNT_MASK)
{
CNodeInfo row(queryJobChannel().queryMyRank(), lastRow.get(), lastRowSize, totalCount);
row.serialize(msg);
}
queryJobChannel().queryJobComm().send(msg, 1, mpTag);
}
if (firstNode())
{
ActPrintLog("INDEXWRITE: Waiting on tlk to complete");
// JCSMORE if refactor==true, is rowsToReceive here right??
unsigned rowsToReceive = (refactor ? (tlkDesc->queryOwner().numParts()-1) : container.queryJob().querySlaves()) -1; // -1 'cos got my own in array already
ActPrintLog("INDEXWRITE: will wait for info from %d slaves before writing TLK", rowsToReceive);
while (rowsToReceive--)
{
msg.clear();
receiveMsg(msg, RANK_ALL, mpTag); // NH->JCS RANK_ALL_OTHER not supported for recv
if (abortSoon)
return;
if (msg.length())
{
CNodeInfo *ni = new CNodeInfo();
ni->deserialize(msg);
tlkRows.append(*ni);
}
}
tlkRows.sort(CNodeInfo::compare);
StringBuffer path;
getPartFilename(*tlkDesc, 0, path);
ActPrintLog("INDEXWRITE: creating toplevel key file : %s", path.str());
try
{
open(*tlkDesc, true, helper->queryDiskRecordSize()->isVariableSize());
if (tlkRows.length())
{
CNodeInfo &lastNode = tlkRows.item(tlkRows.length()-1);
memset(lastNode.value, 0xff, lastNode.size);
}
ForEachItemIn(idx, tlkRows)
{
CNodeInfo &info = tlkRows.item(idx);
builder->processKeyData((char *)info.value, info.pos, info.size);
}
close(*tlkDesc, tlkCrc, true);
}
catch (CATCHALL)
{
abortSoon = true;
close(*tlkDesc, tlkCrc, true);
removeFiles(*partDesc);
throw;
}
}
}
else if (!isLocal && firstNode())
