CATCH_NEXTROW()
{
ActivityTimer t(totalCycles, timeActivities);
if (eogNext || eof)
{
eogNext = false;
return NULL;
}
OwnedConstThorRow prev = next.getClear();
next.setown(getNext());
if (next && !helper->isSameGroup(prev, next))
{
noteEndOfGroup();
eogNext = true;
}
if (prev)
{
dataLinkIncrement();
return prev.getClear();
}
if (prevEog)
{
noteEndOfGroup();
eof = true;
}
prevEog = true;
return NULL;
}
const void *nextRowGENoCatch(const void *seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra &stepExtra)
{
ActivityTimer t(totalCycles, timeActivities, NULL);
while (!abortSoon)
{
OwnedConstThorRow ret = input->nextRowGE(seek, numFields, wasCompleteMatch, stepExtra);
if (!ret)
{
abortSoon = true;
return NULL;
}
if (!wasCompleteMatch)
{
anyThisGroup = false; // RKC->GH - is this right??
return ret.getClear();
}
if (helper->isValid(ret))
{
anyThisGroup = true;
dataLinkIncrement();
return ret.getClear();
}
if (!stepExtra.returnMismatches())
return nextRow();
if (stepCompare->docompare(ret, seek, numFields) != 0)
{
wasCompleteMatch = false;
anyThisGroup = false; // WHY?
return ret.getClear();
}
}
return NULL;
}
CATCH_NEXTROW()
{
ActivityTimer t(totalCycles, timeActivities, NULL);
while (!abortSoon) {
if (rowIter->isValid()) {
anyThisGroup = true;
OwnedConstThorRow r = rowIter->getRow();
dataLinkIncrement();
rowIter->next();
return r.getClear();
}
curRow.setown(input->nextRow());
if (!curRow) {
if (anyThisGroup) {
anyThisGroup = false;
break;
}
curRow.setown(input->nextRow());
if (!curRow)
break;
}
processRecord(curRow.get());
rowIter->first();
}
return NULL;
}
CATCH_NEXTROW()
{
ActivityTimer t(totalCycles, timeActivities);
while(!abortSoon)
{
OwnedConstThorRow row = input->nextRow();
if(!row) {
numToSkip = whichSample ? whichSample - 1 : 0;
if(anyThisGroup) {
anyThisGroup = false;
break;
}
row.setown(input->nextRow());
if(!row)
break;
}
if(numToSkip == 0) {
anyThisGroup = true;
numToSkip = numSamples - 1;
dataLinkIncrement();
return row.getClear();
}
numToSkip--;
}
return NULL;
}
CATCH_NEXTROW()
{
ActivityTimer t(totalCycles, timeActivities, NULL);
while(!abortSoon)
{
OwnedConstThorRow row = input->nextRow();
if (!row)
{
if(anyThisGroup)
{
anyThisGroup = false;
break;
}
row.setown(input->nextRow());
if (!row)
break;
}
if(helper->isValid(row))
{
matched++;
anyThisGroup = true;
dataLinkIncrement();
return row.getClear();
}
}
return NULL;
}
bool processRows()
{
// NB sends a single buffer
CriticalBlock block(crit);
bool sent = false;
try {
if (!socket)
return false;
if (!started)
init();
loop {
OwnedConstThorRow row = iseq->nextRow();
if (!row) {
if (sent)
out->flush();
break;
}
out->putRow(row.getClear());
ndone ++;
sent = true;
if (out->bufferSent())
return true;
}
}
catch (IException *e) {
PrintExceptionLog(e,"CSortMergeBase processRows");
throw;
}
return sent;
}
const void *nextRowGENoCatch(const void * seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra &stepExtra)
{
if (!eos)
{
try
{
ActivityTimer t(totalCycles, timeActivities, NULL);
OwnedConstThorRow ret = input->nextRowGE(seek, numFields, wasCompleteMatch, stepExtra);
if (ret && wasCompleteMatch)
dataLinkIncrement();
return ret.getClear();
}
catch (IException *e)
{
eos = true;
ActPrintLog(e, NULL);
e->Release();
helper->onExceptionCaught();
}
catch (...)
{
eos = true;
helper->onExceptionCaught();
}
throwUnexpected(); // onExceptionCaught should have thrown something
}
return NULL;
}
CATCH_NEXTROW()
{
ActivityTimer t(totalCycles, timeActivities);
if (!abortSoon)
{
if (firstget)
{
firstget = false;
while (skipped<skipCount)
{
OwnedConstThorRow row = input->ungroupedNextRow();
if (!row)
{
stop();
return NULL;
}
skipped++;
}
}
if (getDataLinkCount() < limit)
{
OwnedConstThorRow row = input->ungroupedNextRow();
if (row)
{
dataLinkIncrement();
return row.getClear();
}
}
stop(); // NB: really whatever is pulling, should stop asap.
}
return NULL;
}
virtual const void *nextRow() override
{
OwnedConstThorRow row = smartbuf->nextRow();
if (exception)
throw exception.getClear();
return row.getClear();
}
CATCH_NEXTROW()
{
ActivityTimer t(totalCycles, timeActivities);
if (first)
{
first = false;
if (container.queryLocalOrGrouped() || firstNode())
memset(tallies, 0, sizeof(unsigned)*numSets);
else
getTallies();
done = helper->setCounts(tallies);
}
if (!done)
{
while (!abortSoon)
{
OwnedConstThorRow row = inputStream->ungroupedNextRow();
if (!row)
break;
switch (helper->getRecordAction(row))
{
case 2:
done = true;
//fall through
case 1:
dataLinkIncrement();
return row.getClear();
}
}
}
if (!container.queryLocalOrGrouped() && !lastNode())
sendTallies();
return NULL;
}
CATCH_NEXTROW()
{
ActivityTimer t(totalCycles, timeActivities);
if (!abortSoon)
{
for (;;)
{
if (0 == countThisGroup && skipCount)
{
unsigned skipped = 0;
do
{
OwnedConstThorRow row = inputStream->nextRow();
if (row)
skipped++;
else
{
if (0 == skipped)
{
doStopInput();
return NULL;
}
skipped = 0; // reset, skip group
}
}
while (skipped<skipCount);
}
if (countThisGroup < limit)
{
OwnedConstThorRow row = inputStream->nextRow();
if (row)
{
countThisGroup++;
dataLinkIncrement();
return row.getClear();
}
else if (0 == countThisGroup && 0==skipCount)
{
doStopInput();
return NULL;
}
}
else
{ // consume rest of group
for (;;)
{
OwnedConstThorRow row = inputStream->nextRow();
if (!row)
break;
}
}
if (countThisGroup)
break; // return eog and reset
}
}
countThisGroup = 0;
return NULL;
}
CATCH_NEXTROW()
{
ActivityTimer t(totalCycles, timeActivities, NULL);
if(abortSoon || eof || eogNext)
{
eogNext = false;
return NULL;
}
if (refill) {
refill=false;
index=0;
try
{
group.reset(false);
loop {
OwnedConstThorRow row = input->nextRow();
if (!row)
break;
group.append(row.getClear());
if (group.isFull()) {
StringBuffer errStr("GROUPSORT");
errStr.append("(").append(container.queryId()).append(") ");
errStr.append("exceeded available memory. records=").append(group.ordinality()).append(", memory usage=").append((unsigned)(group.totalSize()/1024)).append('k');
IException *e = MakeActivityException(this, TE_TooMuchData, "%s", errStr.str());
EXCLOG(e, NULL);
throw e;
}
}
if (group.ordinality()==0) {
eof = true;
return NULL;
}
group.sort(*icompare,!unstable);
}
catch (IOutOfMemException *e)
{
StringBuffer errStr("GROUPSORT");
errStr.append("(").append(container.queryId()).append(") ");
errStr.append("exceeded available memory. records=").append(group.ordinality()).append(", memory usage=").append((unsigned)(group.totalSize()/1024)).append('k');
errStr.append(": ").append(e->errorCode()).append(", ");
e->errorMessage(errStr);
e->Release();
IException *e2 = MakeActivityException(this, TE_TooMuchData, "%s", errStr.str());
EXCLOG(e2, NULL);
throw e2;
}
}
if(index >= group.ordinality())
{
refill = true;
return NULL; // eog
}
const void *row = group.itemClear(index++);
assertex(row);
dataLinkIncrement();
return row;
}
const void * nextRow()
{
ActivityTimer t(totalCycles, timeActivities);
OwnedConstThorRow row = input->nextRow();
if (!row)
return NULL;
dataLinkIncrement();
return row.getClear();
}
offset_t write(IRowStream *input)
{
StringBuffer tempname;
GetTempName(tempname,"srtmrg",false);
dataFile.setown(createIFile(tempname.str()));
Owned<IExtRowWriter> output = createRowWriter(dataFile, rowIf);
bool overflowed = false;
ActPrintLog(&activity, "Local Overflow Merge start");
unsigned ret=0;
loop
{
const void *_row = input->nextRow();
if (!_row)
break;
ret++;
OwnedConstThorRow row = _row;
offset_t start = output->getPosition();
output->putRow(row.getLink());
idx++;
if (idx==interval)
{
idx = 0;
if (!sampleRows.append(row.getClear()))
{
// JCSMORE used to check if 'isFull()' here, but only to warn
// I think this is bad news, if has run out of room here...
// should at least warn in workunit I suspect
overflowsize = output->getPosition();
if (!overflowed)
{
WARNLOG("Sample buffer full");
overflowed = true;
}
}
}
writeidxofs(start);
}
output->flush();
offset_t end = output->getPosition();
output.clear();
writeidxofs(end);
if (idxFileIO)
{
idxFileStream->flush();
idxFileStream.clear();
idxFileIO.clear();
}
if (overflowed)
WARNLOG("Overflowed by %"I64F"d", overflowsize);
ActPrintLog(&activity, "Local Overflow Merge done: overflow file '%s', size = %"I64F"d", dataFile->queryFilename(), dataFile->size());
return end;
}
CATCH_NEXTROW()
{
ActivityTimer t(totalCycles, timeActivities);
if (abortSoon)
return NULL;
OwnedConstThorRow row = output->nextRow();
if (!row)
return NULL;
dataLinkIncrement();
return row.getClear();
}
CATCH_NEXTROW()
{
ActivityTimer t(totalCycles, timeActivities, NULL);
if(joinhelper) {
OwnedConstThorRow row = joinhelper->nextRow();
if (row) {
dataLinkIncrement();
return row.getClear();
}
}
return NULL;
}
const void *nextRow()
{
OwnedConstThorRow row = smartbuf->nextRow();
if (getexception)
throw getexception.getClear();
if (!row) {
#ifdef _FULL_TRACE
ActPrintLog(&activity, "ThorLookaheadCache eos %x",(unsigned)(memsize_t)this);
#endif
}
return row.getClear();
}
const void *getFirst() // for global, not called on 1st slave
{
CMessageBuffer msg;
if (!queryJobChannel().queryJobComm().recv(msg, queryJobChannel().queryMyRank()-1, mpTag)) // previous node
return NULL;
msg.read(count);
size32_t r = msg.remaining();
OwnedConstThorRow firstRow;
if (r)
firstRow.deserialize(inrowif, r, msg.readDirect(r));
return firstRow.getClear();
}
// IEngineRowStream
virtual const void *nextRow() override
{
OwnedConstThorRow row = smartbuf->nextRow();
if (getexception)
throw getexception.getClear();
if (!row)
{
#ifdef _FULL_TRACE
ActPrintLog(&activity, "CRowStreamLookAhead eos %x",(unsigned)(memsize_t)this);
#endif
}
return row.getClear();
}
CATCH_NEXTROW()
{
ActivityTimer t(totalCycles, timeActivities, NULL);
OwnedConstThorRow ret = input->nextRow();
if (ret && prev && icompare->docompare(prev, ret) > 0)
{
// MORE - better to give mismatching rows than indexes?
throw MakeActivityException(this, TE_NotSorted, "detected incorrectly sorted rows (row %"RCPF"d, %"RCPF"d))", getDataLinkCount(), getDataLinkCount()+1);
}
prev.set(ret);
if (ret)
dataLinkIncrement();
return ret.getClear();
}
const void *nextRowGENoCatch(const void *seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra &stepExtra)
{
ActivityTimer t(totalCycles, timeActivities, NULL);
OwnedConstThorRow ret = input->nextRowGE(seek, numFields, wasCompleteMatch, stepExtra);
if (ret && prev && stepCompare->docompare(prev, ret, numFields) > 0)
{
// MORE - better to give mismatching rows than indexes?
throw MakeActivityException(this, TE_NotSorted, "detected incorrectly sorted rows (row %"RCPF"d, %"RCPF"d))", getDataLinkCount(), getDataLinkCount()+1);
}
prev.set(ret);
if (ret)
dataLinkIncrement();
return ret.getClear();
}
CATCH_NEXTROW()
{
ActivityTimer t(totalCycles, timeActivities, NULL);
if (!abortSoon)
{
OwnedConstThorRow row = input->ungroupedNextRow();
if (row)
{
dataLinkIncrement();
return row.getClear();
}
abortSoon = true;
}
return NULL;
}
const void *nextRow()
{
if (stream) {
OwnedConstThorRow row = stream->nextRow();
if (row)
return row.getClear();
#ifdef _FULL_TRACE
char url[100];
endpoint.getUrlStr(url,sizeof(url));
PrintLog("SORT Merge READ: EOS for %s",url);
#endif
eos();
}
return NULL;
}
CATCH_NEXTROW()
{
ActivityTimer t(totalCycles, timeActivities);
if (!eof && wscHelper.get())
{
OwnedConstThorRow row = wscHelper->getRow();
if (row)
{
dataLinkIncrement();
return row.getClear();
}
}
eof = true;
return NULL;
}
const void *nextRowGENoCatch(const void *seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra &stepExtra)
{
ActivityTimer t(totalCycles, timeActivities, NULL);
if (!abortSoon)
{
OwnedConstThorRow row = input->nextRowGE(seek, numFields, wasCompleteMatch, stepExtra);
if (row)
{
dataLinkIncrement();
return row.getClear();
}
abortSoon = true;
}
return NULL;
}
void putRow(const void *_row)
{
offset_t start = out->getPosition();
OwnedConstThorRow row = _row;
out->putRow(row.getLink());
idx++;
if (idx==interval) {
idx = 0;
if (overflowed||rowArray.isFull()) {
overflowsize = out->getPosition();
if (!overflowed) {
PROGLOG("Sample buffer full");
overflowed = true;
}
}
else
rowArray.append(row.getClear());
}
writeidxofs(start);
}
virtual void write()
{
ActPrintLog("%s",grouped?"Grouped":"Ungrouped");
while(!abortSoon)
{
OwnedConstThorRow r = input->nextRow();
if (!r.get()) {
if (grouped) {
if ((processed & THORDATALINK_COUNT_MASK)!=0)
out->putRow(NULL);
}
r.setown(input->nextRow());
if (!r.get())
break;
}
out->putRow(r.getClear());
processed++;
}
}
CATCH_NEXTROW()
{
ActivityTimer t(totalCycles, timeActivities);
if (abortSoon || eof)
return NULL;
eof = true;
OwnedConstThorRow next = input->ungroupedNextRow();
RtlDynamicRowBuilder resultcr(queryRowAllocator());
size32_t sz = helper->clearAggregate(resultcr);
if (next)
{
hadElement = true;
sz = helper->processFirst(resultcr, next);
if (container.getKind() != TAKexistsaggregate)
{
while (!abortSoon)
{
next.setown(input->ungroupedNextRow());
if (!next)
break;
sz = helper->processNext(resultcr, next);
}
}
}
doStopInput();
if (!firstNode())
{
OwnedConstThorRow result(resultcr.finalizeRowClear(sz));
sendResult(result.get(),queryRowSerializer(), 1); // send partial result
return NULL;
}
OwnedConstThorRow ret = getResult(resultcr.finalizeRowClear(sz));
if (ret)
{
dataLinkIncrement();
return ret.getClear();
}
sz = helper->clearAggregate(resultcr);
return resultcr.finalizeRowClear(sz);
}
CATCH_NEXTROW()
{
ActivityTimer t(totalCycles, timeActivities);
loop
{
OwnedConstThorRow row = input->ungroupedNextRow();
if(!row || done || abortSoon)
break;
switch (helper->getRecordAction(row))
{
case 2:
done = true;
//fall through
case 1:
dataLinkIncrement();
return row.getClear();
}
}
return NULL;
}
CATCH_NEXTROW()
{
ActivityTimer t(totalCycles, timeActivities, NULL);
if (abortSoon || eoi)
return NULL;
OwnedConstThorRow row = out->nextRow();
if (!row)
{
if (!container.queryGrouped())
{
eoi = true;
return NULL;
}
out.setown(iLoader->loadGroup(input, abortSoon));
if (0 == iLoader->numRows())
eoi = true;
return NULL; // eog marker
}
dataLinkIncrement();
return row.getClear();
}
