IRowStream *load( IRowStream *in, IRowInterfaces *rowif, ICompare *icompare, bool alldisk, bool &abort, bool &isempty, const char *tracename, bool isstable, unsigned maxcores) { overflowcount = 0; unsigned nrecs; serializer.set(rowif->queryRowSerializer()); rows.setSizing(true,false); #ifdef _FULL_TRACE PROGLOG("CThorRowSortedLoader load start"); #endif isempty = true; while (!abort) { rows.clear(); bool hasoverflowed = false; nrecs = rows.load(*in,true,abort,&hasoverflowed); if (nrecs) isempty = false; if (hasoverflowed) overflowcount++; #ifdef _FULL_TRACE PROGLOG("rows loaded overflowed = %s",hasoverflowed?"true":"false"); #endif if (nrecs&&icompare) rows.sort(*icompare,isstable,maxcores); numrows += nrecs; totalsize += rows.totalSize(); if (!hasoverflowed&&!alldisk) break; #ifdef _FULL_TRACE PROGLOG("CThorRowSortedLoader spilling %u",(unsigned)rows.totalSize()); #endif alldisk = true; unsigned idx = newAuxFile(); Owned<IExtRowWriter> writer = createRowWriter(&auxfiles.item(idx),rowif->queryRowSerializer(),rowif->queryRowAllocator(),false,false,false); rows.save(writer); writer->flush(); #ifdef _FULL_TRACE PROGLOG("CThorRowSortedLoader spilt"); #endif if (!hasoverflowed) break; } ForEachItemIn(i,auxfiles) { #ifdef _FULL_TRACE PROGLOG("CThorRowSortedLoader spill file(%d) %s %"I64F"d",i,auxfiles.item(i).queryFilename(),auxfiles.item(i).size()); #endif instrms.append(*createSimpleRowStream(&auxfiles.item(i),rowif)); }
offset_t write(IRowStream *input) { StringBuffer tempname; GetTempName(tempname,"srtmrg",false); dataFile.setown(createIFile(tempname.str())); Owned<IExtRowWriter> output = createRowWriter(dataFile, rowIf); bool overflowed = false; ActPrintLog(&activity, "Local Overflow Merge start"); unsigned ret=0; loop { const void *_row = input->nextRow(); if (!_row) break; ret++; OwnedConstThorRow row = _row; offset_t start = output->getPosition(); output->putRow(row.getLink()); idx++; if (idx==interval) { idx = 0; if (!sampleRows.append(row.getClear())) { // JCSMORE used to check if 'isFull()' here, but only to warn // I think this is bad news, if has run out of room here... // should at least warn in workunit I suspect overflowsize = output->getPosition(); if (!overflowed) { WARNLOG("Sample buffer full"); overflowed = true; } } } writeidxofs(start); } output->flush(); offset_t end = output->getPosition(); output.clear(); writeidxofs(end); if (idxFileIO) { idxFileStream->flush(); idxFileStream.clear(); idxFileIO.clear(); } if (overflowed) WARNLOG("Overflowed by %"I64F"d", overflowsize); ActPrintLog(&activity, "Local Overflow Merge done: overflow file '%s', size = %"I64F"d", dataFile->queryFilename(), dataFile->size()); return end; }