offset_t write(IRowStream *input) { StringBuffer tempname; GetTempName(tempname,"srtmrg",false); dataFile.setown(createIFile(tempname.str())); Owned<IExtRowWriter> output = createRowWriter(dataFile, rowIf); bool overflowed = false; ActPrintLog(&activity, "Local Overflow Merge start"); unsigned ret=0; loop { const void *_row = input->nextRow(); if (!_row) break; ret++; OwnedConstThorRow row = _row; offset_t start = output->getPosition(); output->putRow(row.getLink()); idx++; if (idx==interval) { idx = 0; if (!sampleRows.append(row.getClear())) { // JCSMORE used to check if 'isFull()' here, but only to warn // I think this is bad news, if has run out of room here... // should at least warn in workunit I suspect overflowsize = output->getPosition(); if (!overflowed) { WARNLOG("Sample buffer full"); overflowed = true; } } } writeidxofs(start); } output->flush(); offset_t end = output->getPosition(); output.clear(); writeidxofs(end); if (idxFileIO) { idxFileStream->flush(); idxFileStream.clear(); idxFileIO.clear(); } if (overflowed) WARNLOG("Overflowed by %"I64F"d", overflowsize); ActPrintLog(&activity, "Local Overflow Merge done: overflow file '%s', size = %"I64F"d", dataFile->queryFilename(), dataFile->size()); return end; }
virtual void start() override { ActivityTimer s(totalCycles, timeActivities); ActPrintLog(rolloverEnabled ? "GROUP: is global" : "GROUP: is local"); PARENT::start(); eogNext = prevEog = eof = false; if (rolloverEnabled) { useRollover = !lastNode(); #ifdef _TESTING ActPrintLog("Node number = %d, Total Nodes = %d", queryJobChannel().queryMyRank(), container.queryJob().querySlaves()); #endif } stream.set(inputStream); startLastGroup = getDataLinkGlobalCount(); next.setown(getNext()); if (rolloverEnabled && !firstNode()) // 1st node can have nothing to send { Owned<IThorRowCollector> collector = createThorRowCollector(*this, this, NULL, stableSort_none, rc_mixed, SPILL_PRIORITY_SPILLABLE_STREAM); Owned<IRowWriter> writer = collector->getWriter(); if (next) { ActPrintLog("GROUP: Sending first group to previous node(%d)", queryJobChannel().queryMyRank()-1); for (;;) { writer->putRow(next.getLink()); if (abortSoon) break; //always send group even when aborting OwnedConstThorRow next2 = getNext(); if (!next2) { eof = true; break; } else if (!helper->isSameGroup(next2, next)) { next.setown(next2.getClear()); break; } next.setown(next2.getClear()); } } writer.clear(); ActPrintLog("GROUP: %" RCPF "d records to send", collector->numRows()); Owned<IRowStream> strm = collector->getStream(); rowServer.setown(createRowServer(this, strm, queryJobChannel().queryJobComm(), mpTag)); } }
void putRow(const void *_row) { offset_t start = out->getPosition(); OwnedConstThorRow row = _row; out->putRow(row.getLink()); idx++; if (idx==interval) { idx = 0; if (overflowed||rowArray.isFull()) { overflowsize = out->getPosition(); if (!overflowed) { PROGLOG("Sample buffer full"); overflowed = true; } } else rowArray.append(row.getClear()); } writeidxofs(start); }
inline OwnedConstThorRow(const OwnedConstThorRow & other) { ptr = other.getLink(); }