offset_t write(IRowStream *input)
{
StringBuffer tempname;
GetTempName(tempname,"srtmrg",false);
dataFile.setown(createIFile(tempname.str()));
Owned<IExtRowWriter> output = createRowWriter(dataFile, rowIf);
bool overflowed = false;
ActPrintLog(&activity, "Local Overflow Merge start");
unsigned ret=0;
loop
{
const void *_row = input->nextRow();
if (!_row)
break;
ret++;
OwnedConstThorRow row = _row;
offset_t start = output->getPosition();
output->putRow(row.getLink());
idx++;
if (idx==interval)
{
idx = 0;
if (!sampleRows.append(row.getClear()))
{
// JCSMORE used to check if 'isFull()' here, but only to warn
// I think this is bad news, if has run out of room here...
// should at least warn in workunit I suspect
overflowsize = output->getPosition();
if (!overflowed)
{
WARNLOG("Sample buffer full");
overflowed = true;
}
}
}
writeidxofs(start);
}
output->flush();
offset_t end = output->getPosition();
output.clear();
writeidxofs(end);
if (idxFileIO)
{
idxFileStream->flush();
idxFileStream.clear();
idxFileIO.clear();
}
if (overflowed)
WARNLOG("Overflowed by %"I64F"d", overflowsize);
ActPrintLog(&activity, "Local Overflow Merge done: overflow file '%s', size = %"I64F"d", dataFile->queryFilename(), dataFile->size());
return end;
}
virtual void flush()
{
if (!flushdone) {
out->flush();
offset_t end = out->getPosition();
writeidxofs(end);
if (outidx)
outidx->flush();
flushdone = true;
}
}
void putRow(const void *_row)
{
offset_t start = out->getPosition();
OwnedConstThorRow row = _row;
out->putRow(row.getLink());
idx++;
if (idx==interval) {
idx = 0;
if (overflowed||rowArray.isFull()) {
overflowsize = out->getPosition();
if (!overflowed) {
PROGLOG("Sample buffer full");
overflowed = true;
}
}
else
rowArray.append(row.getClear());
}
writeidxofs(start);
}
void writeidxofs(offset_t o)
{
// lazy index write
if (outidx.get()) {
outidx->write(sizeof(o),&o);
return;
}
if (lastofs) {
if (fixedsize!=o-lastofs) {
// right create idx
StringBuffer tempname;
GetTempName(tempname.clear(),"srtidx",false);
outidxfile.setown(createIFile(tempname.str()));
Owned<IFileIO> fileioidx = outidxfile->open(IFOcreate);
outidx.setown(fileioidx?createBufferedIOStream(fileioidx,0x100000):NULL);
if (outidx.get()==NULL) {
StringBuffer err;
err.append("Cannot create ").append(outidxfile->queryFilename());
LOG(MCerror, thorJob, "%s", err.str());
throw MakeStringException(-1, "%s", err.str());
}
offset_t s = 0;
while (s<=lastofs) {
outidx->write(sizeof(s),&s);
s += fixedsize;
}
assertex(s==lastofs+fixedsize);
fixedsize = 0;
writeidxofs(o);
return;
}
}
else
fixedsize = (size32_t)(o-lastofs);
lastofs = o;
}
