IRowStream *load(
IRowStream *in,
IRowInterfaces *rowif,
ICompare *icompare,
bool alldisk,
bool &abort,
bool &isempty,
const char *tracename,
bool isstable, unsigned maxcores)
{
overflowcount = 0;
unsigned nrecs;
serializer.set(rowif->queryRowSerializer());
rows.setSizing(true,false);
#ifdef _FULL_TRACE
PROGLOG("CThorRowSortedLoader load start");
#endif
isempty = true;
while (!abort) {
rows.clear();
bool hasoverflowed = false;
nrecs = rows.load(*in,true,abort,&hasoverflowed);
if (nrecs)
isempty = false;
if (hasoverflowed)
overflowcount++;
#ifdef _FULL_TRACE
PROGLOG("rows loaded overflowed = %s",hasoverflowed?"true":"false");
#endif
if (nrecs&&icompare)
rows.sort(*icompare,isstable,maxcores);
numrows += nrecs;
totalsize += rows.totalSize();
if (!hasoverflowed&&!alldisk)
break;
#ifdef _FULL_TRACE
PROGLOG("CThorRowSortedLoader spilling %u",(unsigned)rows.totalSize());
#endif
alldisk = true;
unsigned idx = newAuxFile();
Owned<IExtRowWriter> writer = createRowWriter(&auxfiles.item(idx),rowif->queryRowSerializer(),rowif->queryRowAllocator(),false,false,false);
rows.save(writer);
writer->flush();
#ifdef _FULL_TRACE
PROGLOG("CThorRowSortedLoader spilt");
#endif
if (!hasoverflowed)
break;
}
ForEachItemIn(i,auxfiles) {
#ifdef _FULL_TRACE
PROGLOG("CThorRowSortedLoader spill file(%d) %s %"I64F"d",i,auxfiles.item(i).queryFilename(),auxfiles.item(i).size());
#endif
instrms.append(*createSimpleRowStream(&auxfiles.item(i),rowif));
}
offset_t write(IRowStream *input)
{
StringBuffer tempname;
GetTempName(tempname,"srtmrg",false);
dataFile.setown(createIFile(tempname.str()));
Owned<IExtRowWriter> output = createRowWriter(dataFile, rowIf);
bool overflowed = false;
ActPrintLog(&activity, "Local Overflow Merge start");
unsigned ret=0;
loop
{
const void *_row = input->nextRow();
if (!_row)
break;
ret++;
OwnedConstThorRow row = _row;
offset_t start = output->getPosition();
output->putRow(row.getLink());
idx++;
if (idx==interval)
{
idx = 0;
if (!sampleRows.append(row.getClear()))
{
// JCSMORE used to check if 'isFull()' here, but only to warn
// I think this is bad news, if has run out of room here...
// should at least warn in workunit I suspect
overflowsize = output->getPosition();
if (!overflowed)
{
WARNLOG("Sample buffer full");
overflowed = true;
}
}
}
writeidxofs(start);
}
output->flush();
offset_t end = output->getPosition();
output.clear();
writeidxofs(end);
if (idxFileIO)
{
idxFileStream->flush();
idxFileStream.clear();
idxFileIO.clear();
}
if (overflowed)
WARNLOG("Overflowed by %"I64F"d", overflowsize);
ActPrintLog(&activity, "Local Overflow Merge done: overflow file '%s', size = %"I64F"d", dataFile->queryFilename(), dataFile->size());
return end;
}