virtual void setInputStream(unsigned index, CThorInput &_input, bool consumerOrdered) override
{
PARENT::setInputStream(index, _input, consumerOrdered);
// JCSMORE - not sure why you ever want a look ahead on a sink like this?
if (!isFastThrough(input))
setLookAhead(0, createRowStreamLookAhead(this, inputStream, queryRowInterfaces(input), INDEXWRITE_SMART_BUFFER_SIZE, true, false, RCUNBOUND, this, &container.queryJob().queryIDiskUsage()));
}
void start()
{
ActivityTimer s(totalCycles, timeActivities, NULL);
dataLinkStart();
input = inputs.item(0);
unsigned spillPriority = container.queryGrouped() ? SPILL_PRIORITY_GROUPSORT : SPILL_PRIORITY_LARGESORT;
iLoader.setown(createThorRowLoader(*this, queryRowInterfaces(input), iCompare, unstable ? stableSort_none : stableSort_earlyAlloc, rc_mixed, spillPriority));
startInput(input);
eoi = false;
if (container.queryGrouped())
out.setown(iLoader->loadGroup(input, abortSoon));
else
out.setown(iLoader->load(input, abortSoon));
if (0 == iLoader->numRows())
eoi = true;
}
virtual void setInputStream(unsigned index, CThorInput &_input, bool consumerOrdered) override
{
PARENT::setInputStream(index, _input, consumerOrdered);
setLookAhead(0, createRowStreamLookAhead(this, inputStream, queryRowInterfaces(input), INDEXWRITE_SMART_BUFFER_SIZE, true, false, RCUNBOUND, this, &container.queryJob().queryIDiskUsage()));
}
void start()
{
ActivityTimer s(totalCycles, timeActivities);
input = inputs.item(0);
try
{
try {
startInput(input);
}
catch (IException *e)
{
fireException(e);
barrier->cancel();
throw;
}
catch (CATCHALL)
{
Owned<IException> e = MakeActivityException(this, 0, "Unknown exception starting sort input");
fireException(e);
barrier->cancel();
throw;
}
dataLinkStart();
Linked<IRowInterfaces> rowif = queryRowInterfaces(input);
Owned<IRowInterfaces> auxrowif = createRowInterfaces(helper->querySortedRecordSize(),queryActivityId(),queryCodeContext());
sorter->Gather(
rowif,
input,
helper->queryCompare(),
helper->queryCompareLeftRight(),
NULL,helper->querySerialize(),
NULL,
false,
isUnstable(),
abortSoon,
auxrowif);
stopInput(input);
input = NULL;
if (abortSoon)
{
ActPrintLogEx(&queryContainer(), thorlog_null, MCwarning, "MSortSlaveActivity::start aborting");
barrier->cancel();
return;
}
}
catch (IException *e)
{
fireException(e);
barrier->cancel();
throw;
}
catch (CATCHALL)
{
Owned<IException> e = MakeActivityException(this, 0, "Unknown exception gathering sort input");
fireException(e);
barrier->cancel();
throw;
}
ActPrintLog("SORT waiting barrier.1");
if (!barrier->wait(false)) {
Sleep(1000); // let original error through
throw MakeThorException(TE_BarrierAborted,"SORT: Barrier Aborted");
}
ActPrintLog("SORT barrier.1 raised");
output.setown(sorter->startMerge(totalrows));
}
virtual void setInputStream(unsigned index, CThorInput &_input, bool consumerOrdered) override
{
PARENT::setInputStream(index, _input, consumerOrdered);
if (global) // only want lookahead if global (hence serial)
setLookAhead(0, createRowStreamLookAhead(this, inputStream, queryRowInterfaces(input), ENTH_SMART_BUFFER_SIZE, true, false, RCUNBOUND, NULL, &container.queryJob().queryIDiskUsage()));
}
int run()
{
if (!started) {
try {
in->start();
started = true;
}
catch(IException * e)
{
ActPrintLog(&activity, e, "ThorLookaheadCache starting input");
startexception.setown(e);
if (asyncstart)
notify->onInputStarted(startexception);
running = false;
stopped = true;
startsem.signal();
return 0;
}
}
try {
StringBuffer temp;
if (allowspill)
GetTempName(temp,"lookahd",true);
assertex(bufsize);
if (allowspill)
smartbuf.setown(createSmartBuffer(&activity, temp.toCharArray(), bufsize, queryRowInterfaces(in)));
else
smartbuf.setown(createSmartInMemoryBuffer(&activity, queryRowInterfaces(in), bufsize));
if (notify)
notify->onInputStarted(NULL);
startsem.signal();
Linked<IRowWriter> writer = smartbuf->queryWriter();
if (preserveLhsGrouping)
{
while (required&&running)
{
OwnedConstThorRow row = in->nextRow();
if (!row)
{
row.setown(in->nextRow());
if (!row)
break;
else
writer->putRow(NULL); // eog
}
++count;
writer->putRow(row.getClear());
if (required!=RCUNBOUND)
required--;
}
}
else
{
while (required&&running)
{
OwnedConstThorRow row = in->ungroupedNextRow();
if (!row)
break;
++count;
writer->putRow(row.getClear());
if (required!=RCUNBOUND)
required--;
}
}
}
catch(IException * e)
{
ActPrintLog(&activity, e, "ThorLookaheadCache get exception");
getexception.setown(e);
}
if (notify)
notify->onInputFinished(count);
if (smartbuf)
smartbuf->queryWriter()->flush();
running = false;
try {
if (in)
in->stop();
}
catch(IException * e)
{
ActPrintLog(&activity, e, "ThorLookaheadCache stop exception");
if (!getexception.get())
getexception.setown(e);
}
return 0;
}
virtual void setInputStream(unsigned index, CThorInput &_input, bool consumerOrdered) override
{
PARENT::setInputStream(index, _input, consumerOrdered);
inputCounter->setInputStream(inputStream);
setLookAhead(0, createRowStreamLookAhead(this, inputCounter.get(), queryRowInterfaces(input), CHOOSESETSPLUS_SMART_BUFFER_SIZE, true, false, RCUNBOUND, this, &container.queryJob().queryIDiskUsage())); // read all input
}
virtual void setInputStream(unsigned index, CThorInput &_input, bool consumerOrdered) override
{
PARENT::setInputStream(index, _input, consumerOrdered);
if (!isFastThrough(input))
setLookAhead(0, createRowStreamLookAhead(this, inputStream, queryRowInterfaces(input), CHOOSESETS_SMART_BUFFER_SIZE, isSmartBufferSpillNeeded(this), false, RCUNBOUND, NULL, &container.queryJob().queryIDiskUsage()));
}
void CDiskWriteSlaveActivityBase::setInputStream(unsigned index, CThorInput &_input, bool consumerOrdered)
{
PARENT::setInputStream(index, _input, consumerOrdered);
if (dlfn.isExternal() && !firstNode())
setLookAhead(0, createRowStreamLookAhead(this, inputStream, queryRowInterfaces(input), PROCESS_SMART_BUFFER_SIZE, isSmartBufferSpillNeeded(this), grouped, RCUNBOUND, NULL, &container.queryJob().queryIDiskUsage()));
}