CATCH_NEXTROW() { ActivityTimer t(totalCycles, timeActivities); if (eogNext || eof) { eogNext = false; return NULL; } OwnedConstThorRow prev = next.getClear(); next.setown(getNext()); if (next && !helper->isSameGroup(prev, next)) { noteEndOfGroup(); eogNext = true; } if (prev) { dataLinkIncrement(); return prev.getClear(); } if (prevEog) { noteEndOfGroup(); eof = true; } prevEog = true; return NULL; }
const void *nextRowGENoCatch(const void *seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra &stepExtra) { ActivityTimer t(totalCycles, timeActivities, NULL); while (!abortSoon) { OwnedConstThorRow ret = input->nextRowGE(seek, numFields, wasCompleteMatch, stepExtra); if (!ret) { abortSoon = true; return NULL; } if (!wasCompleteMatch) { anyThisGroup = false; // RKC->GH - is this right?? return ret.getClear(); } if (helper->isValid(ret)) { anyThisGroup = true; dataLinkIncrement(); return ret.getClear(); } if (!stepExtra.returnMismatches()) return nextRow(); if (stepCompare->docompare(ret, seek, numFields) != 0) { wasCompleteMatch = false; anyThisGroup = false; // WHY? return ret.getClear(); } } return NULL; }
CATCH_NEXTROW() { ActivityTimer t(totalCycles, timeActivities, NULL); while (!abortSoon) { if (rowIter->isValid()) { anyThisGroup = true; OwnedConstThorRow r = rowIter->getRow(); dataLinkIncrement(); rowIter->next(); return r.getClear(); } curRow.setown(input->nextRow()); if (!curRow) { if (anyThisGroup) { anyThisGroup = false; break; } curRow.setown(input->nextRow()); if (!curRow) break; } processRecord(curRow.get()); rowIter->first(); } return NULL; }
CATCH_NEXTROW() { ActivityTimer t(totalCycles, timeActivities); while(!abortSoon) { OwnedConstThorRow row = input->nextRow(); if(!row) { numToSkip = whichSample ? whichSample - 1 : 0; if(anyThisGroup) { anyThisGroup = false; break; } row.setown(input->nextRow()); if(!row) break; } if(numToSkip == 0) { anyThisGroup = true; numToSkip = numSamples - 1; dataLinkIncrement(); return row.getClear(); } numToSkip--; } return NULL; }
CATCH_NEXTROW() { ActivityTimer t(totalCycles, timeActivities, NULL); while(!abortSoon) { OwnedConstThorRow row = input->nextRow(); if (!row) { if(anyThisGroup) { anyThisGroup = false; break; } row.setown(input->nextRow()); if (!row) break; } if(helper->isValid(row)) { matched++; anyThisGroup = true; dataLinkIncrement(); return row.getClear(); } } return NULL; }
bool processRows() { // NB sends a single buffer CriticalBlock block(crit); bool sent = false; try { if (!socket) return false; if (!started) init(); loop { OwnedConstThorRow row = iseq->nextRow(); if (!row) { if (sent) out->flush(); break; } out->putRow(row.getClear()); ndone ++; sent = true; if (out->bufferSent()) return true; } } catch (IException *e) { PrintExceptionLog(e,"CSortMergeBase processRows"); throw; } return sent; }
const void *nextRowGENoCatch(const void * seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra &stepExtra) { if (!eos) { try { ActivityTimer t(totalCycles, timeActivities, NULL); OwnedConstThorRow ret = input->nextRowGE(seek, numFields, wasCompleteMatch, stepExtra); if (ret && wasCompleteMatch) dataLinkIncrement(); return ret.getClear(); } catch (IException *e) { eos = true; ActPrintLog(e, NULL); e->Release(); helper->onExceptionCaught(); } catch (...) { eos = true; helper->onExceptionCaught(); } throwUnexpected(); // onExceptionCaught should have thrown something } return NULL; }
CATCH_NEXTROW() { ActivityTimer t(totalCycles, timeActivities); if (!abortSoon) { if (firstget) { firstget = false; while (skipped<skipCount) { OwnedConstThorRow row = input->ungroupedNextRow(); if (!row) { stop(); return NULL; } skipped++; } } if (getDataLinkCount() < limit) { OwnedConstThorRow row = input->ungroupedNextRow(); if (row) { dataLinkIncrement(); return row.getClear(); } } stop(); // NB: really whatever is pulling, should stop asap. } return NULL; }
virtual const void *nextRow() override { OwnedConstThorRow row = smartbuf->nextRow(); if (exception) throw exception.getClear(); return row.getClear(); }
CATCH_NEXTROW() { ActivityTimer t(totalCycles, timeActivities); if (first) { first = false; if (container.queryLocalOrGrouped() || firstNode()) memset(tallies, 0, sizeof(unsigned)*numSets); else getTallies(); done = helper->setCounts(tallies); } if (!done) { while (!abortSoon) { OwnedConstThorRow row = inputStream->ungroupedNextRow(); if (!row) break; switch (helper->getRecordAction(row)) { case 2: done = true; //fall through case 1: dataLinkIncrement(); return row.getClear(); } } } if (!container.queryLocalOrGrouped() && !lastNode()) sendTallies(); return NULL; }
CATCH_NEXTROW() { ActivityTimer t(totalCycles, timeActivities); if (!abortSoon) { for (;;) { if (0 == countThisGroup && skipCount) { unsigned skipped = 0; do { OwnedConstThorRow row = inputStream->nextRow(); if (row) skipped++; else { if (0 == skipped) { doStopInput(); return NULL; } skipped = 0; // reset, skip group } } while (skipped<skipCount); } if (countThisGroup < limit) { OwnedConstThorRow row = inputStream->nextRow(); if (row) { countThisGroup++; dataLinkIncrement(); return row.getClear(); } else if (0 == countThisGroup && 0==skipCount) { doStopInput(); return NULL; } } else { // consume rest of group for (;;) { OwnedConstThorRow row = inputStream->nextRow(); if (!row) break; } } if (countThisGroup) break; // return eog and reset } } countThisGroup = 0; return NULL; }
CATCH_NEXTROW() { ActivityTimer t(totalCycles, timeActivities, NULL); if(abortSoon || eof || eogNext) { eogNext = false; return NULL; } if (refill) { refill=false; index=0; try { group.reset(false); loop { OwnedConstThorRow row = input->nextRow(); if (!row) break; group.append(row.getClear()); if (group.isFull()) { StringBuffer errStr("GROUPSORT"); errStr.append("(").append(container.queryId()).append(") "); errStr.append("exceeded available memory. records=").append(group.ordinality()).append(", memory usage=").append((unsigned)(group.totalSize()/1024)).append('k'); IException *e = MakeActivityException(this, TE_TooMuchData, "%s", errStr.str()); EXCLOG(e, NULL); throw e; } } if (group.ordinality()==0) { eof = true; return NULL; } group.sort(*icompare,!unstable); } catch (IOutOfMemException *e) { StringBuffer errStr("GROUPSORT"); errStr.append("(").append(container.queryId()).append(") "); errStr.append("exceeded available memory. records=").append(group.ordinality()).append(", memory usage=").append((unsigned)(group.totalSize()/1024)).append('k'); errStr.append(": ").append(e->errorCode()).append(", "); e->errorMessage(errStr); e->Release(); IException *e2 = MakeActivityException(this, TE_TooMuchData, "%s", errStr.str()); EXCLOG(e2, NULL); throw e2; } } if(index >= group.ordinality()) { refill = true; return NULL; // eog } const void *row = group.itemClear(index++); assertex(row); dataLinkIncrement(); return row; }
const void * nextRow() { ActivityTimer t(totalCycles, timeActivities); OwnedConstThorRow row = input->nextRow(); if (!row) return NULL; dataLinkIncrement(); return row.getClear(); }
offset_t write(IRowStream *input) { StringBuffer tempname; GetTempName(tempname,"srtmrg",false); dataFile.setown(createIFile(tempname.str())); Owned<IExtRowWriter> output = createRowWriter(dataFile, rowIf); bool overflowed = false; ActPrintLog(&activity, "Local Overflow Merge start"); unsigned ret=0; loop { const void *_row = input->nextRow(); if (!_row) break; ret++; OwnedConstThorRow row = _row; offset_t start = output->getPosition(); output->putRow(row.getLink()); idx++; if (idx==interval) { idx = 0; if (!sampleRows.append(row.getClear())) { // JCSMORE used to check if 'isFull()' here, but only to warn // I think this is bad news, if has run out of room here... // should at least warn in workunit I suspect overflowsize = output->getPosition(); if (!overflowed) { WARNLOG("Sample buffer full"); overflowed = true; } } } writeidxofs(start); } output->flush(); offset_t end = output->getPosition(); output.clear(); writeidxofs(end); if (idxFileIO) { idxFileStream->flush(); idxFileStream.clear(); idxFileIO.clear(); } if (overflowed) WARNLOG("Overflowed by %"I64F"d", overflowsize); ActPrintLog(&activity, "Local Overflow Merge done: overflow file '%s', size = %"I64F"d", dataFile->queryFilename(), dataFile->size()); return end; }
CATCH_NEXTROW() { ActivityTimer t(totalCycles, timeActivities); if (abortSoon) return NULL; OwnedConstThorRow row = output->nextRow(); if (!row) return NULL; dataLinkIncrement(); return row.getClear(); }
CATCH_NEXTROW() { ActivityTimer t(totalCycles, timeActivities, NULL); if(joinhelper) { OwnedConstThorRow row = joinhelper->nextRow(); if (row) { dataLinkIncrement(); return row.getClear(); } } return NULL; }
const void *nextRow() { OwnedConstThorRow row = smartbuf->nextRow(); if (getexception) throw getexception.getClear(); if (!row) { #ifdef _FULL_TRACE ActPrintLog(&activity, "ThorLookaheadCache eos %x",(unsigned)(memsize_t)this); #endif } return row.getClear(); }
const void *getFirst() // for global, not called on 1st slave { CMessageBuffer msg; if (!queryJobChannel().queryJobComm().recv(msg, queryJobChannel().queryMyRank()-1, mpTag)) // previous node return NULL; msg.read(count); size32_t r = msg.remaining(); OwnedConstThorRow firstRow; if (r) firstRow.deserialize(inrowif, r, msg.readDirect(r)); return firstRow.getClear(); }
// IEngineRowStream virtual const void *nextRow() override { OwnedConstThorRow row = smartbuf->nextRow(); if (getexception) throw getexception.getClear(); if (!row) { #ifdef _FULL_TRACE ActPrintLog(&activity, "CRowStreamLookAhead eos %x",(unsigned)(memsize_t)this); #endif } return row.getClear(); }
CATCH_NEXTROW() { ActivityTimer t(totalCycles, timeActivities, NULL); OwnedConstThorRow ret = input->nextRow(); if (ret && prev && icompare->docompare(prev, ret) > 0) { // MORE - better to give mismatching rows than indexes? throw MakeActivityException(this, TE_NotSorted, "detected incorrectly sorted rows (row %"RCPF"d, %"RCPF"d))", getDataLinkCount(), getDataLinkCount()+1); } prev.set(ret); if (ret) dataLinkIncrement(); return ret.getClear(); }
const void *nextRowGENoCatch(const void *seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra &stepExtra) { ActivityTimer t(totalCycles, timeActivities, NULL); OwnedConstThorRow ret = input->nextRowGE(seek, numFields, wasCompleteMatch, stepExtra); if (ret && prev && stepCompare->docompare(prev, ret, numFields) > 0) { // MORE - better to give mismatching rows than indexes? throw MakeActivityException(this, TE_NotSorted, "detected incorrectly sorted rows (row %"RCPF"d, %"RCPF"d))", getDataLinkCount(), getDataLinkCount()+1); } prev.set(ret); if (ret) dataLinkIncrement(); return ret.getClear(); }
CATCH_NEXTROW() { ActivityTimer t(totalCycles, timeActivities, NULL); if (!abortSoon) { OwnedConstThorRow row = input->ungroupedNextRow(); if (row) { dataLinkIncrement(); return row.getClear(); } abortSoon = true; } return NULL; }
const void *nextRow() { if (stream) { OwnedConstThorRow row = stream->nextRow(); if (row) return row.getClear(); #ifdef _FULL_TRACE char url[100]; endpoint.getUrlStr(url,sizeof(url)); PrintLog("SORT Merge READ: EOS for %s",url); #endif eos(); } return NULL; }
CATCH_NEXTROW() { ActivityTimer t(totalCycles, timeActivities); if (!eof && wscHelper.get()) { OwnedConstThorRow row = wscHelper->getRow(); if (row) { dataLinkIncrement(); return row.getClear(); } } eof = true; return NULL; }
const void *nextRowGENoCatch(const void *seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra &stepExtra) { ActivityTimer t(totalCycles, timeActivities, NULL); if (!abortSoon) { OwnedConstThorRow row = input->nextRowGE(seek, numFields, wasCompleteMatch, stepExtra); if (row) { dataLinkIncrement(); return row.getClear(); } abortSoon = true; } return NULL; }
void putRow(const void *_row) { offset_t start = out->getPosition(); OwnedConstThorRow row = _row; out->putRow(row.getLink()); idx++; if (idx==interval) { idx = 0; if (overflowed||rowArray.isFull()) { overflowsize = out->getPosition(); if (!overflowed) { PROGLOG("Sample buffer full"); overflowed = true; } } else rowArray.append(row.getClear()); } writeidxofs(start); }
virtual void write() { ActPrintLog("%s",grouped?"Grouped":"Ungrouped"); while(!abortSoon) { OwnedConstThorRow r = input->nextRow(); if (!r.get()) { if (grouped) { if ((processed & THORDATALINK_COUNT_MASK)!=0) out->putRow(NULL); } r.setown(input->nextRow()); if (!r.get()) break; } out->putRow(r.getClear()); processed++; } }
CATCH_NEXTROW() { ActivityTimer t(totalCycles, timeActivities); if (abortSoon || eof) return NULL; eof = true; OwnedConstThorRow next = input->ungroupedNextRow(); RtlDynamicRowBuilder resultcr(queryRowAllocator()); size32_t sz = helper->clearAggregate(resultcr); if (next) { hadElement = true; sz = helper->processFirst(resultcr, next); if (container.getKind() != TAKexistsaggregate) { while (!abortSoon) { next.setown(input->ungroupedNextRow()); if (!next) break; sz = helper->processNext(resultcr, next); } } } doStopInput(); if (!firstNode()) { OwnedConstThorRow result(resultcr.finalizeRowClear(sz)); sendResult(result.get(),queryRowSerializer(), 1); // send partial result return NULL; } OwnedConstThorRow ret = getResult(resultcr.finalizeRowClear(sz)); if (ret) { dataLinkIncrement(); return ret.getClear(); } sz = helper->clearAggregate(resultcr); return resultcr.finalizeRowClear(sz); }
CATCH_NEXTROW() { ActivityTimer t(totalCycles, timeActivities); loop { OwnedConstThorRow row = input->ungroupedNextRow(); if(!row || done || abortSoon) break; switch (helper->getRecordAction(row)) { case 2: done = true; //fall through case 1: dataLinkIncrement(); return row.getClear(); } } return NULL; }
CATCH_NEXTROW() { ActivityTimer t(totalCycles, timeActivities, NULL); if (abortSoon || eoi) return NULL; OwnedConstThorRow row = out->nextRow(); if (!row) { if (!container.queryGrouped()) { eoi = true; return NULL; } out.setown(iLoader->loadGroup(input, abortSoon)); if (0 == iLoader->numRows()) eoi = true; return NULL; // eog marker } dataLinkIncrement(); return row.getClear(); }