virtual void process()
{
IRecordSize *recordSize = helper->queryOutputMeta();
Owned<IRowInterfaces> rowIf = createRowInterfaces(helper->queryOutputMeta(), queryActivityId(), queryCodeContext());
OwnedConstThorRow result = getAggregate(*this, container.queryJob().querySlaves(), *rowIf, *helper, mpTag);
if (!result)
return;
CMessageBuffer msg;
CMemoryRowSerializer mbs(msg);
rowIf->queryRowSerializer()->serialize(mbs, (const byte *)result.get());
if (!container.queryJob().queryJobComm().send(msg, 1, mpTag, 5000))
throw MakeThorException(0, "Failed to give result to slave");
}
virtual void process()
{
CMasterActivity::process();
bool results = false;
unsigned nslaves = container.queryJob().querySlaves();
while (nslaves--)
{
CMessageBuffer mb;
if (abortSoon || !receiveMsg(mb, RANK_ALL, replyTag, NULL)) break;
StringBuffer str;
mb.getSender().getUrlStr(str);
size32_t sz;
mb.read(sz);
if (sz)
{
if (results)
throw MakeThorException(TE_UnexpectedMultipleSlaveResults, "Received greater than one result from slaves");
IHThorRemoteResultArg *helper = (IHThorRemoteResultArg *)queryHelper();
Owned<IThorRowInterfaces> resultRowIf = createRowInterfaces(helper->queryOutputMeta());
CThorStreamDeserializerSource mds(sz, mb.readDirect(sz));
RtlDynamicRowBuilder rowBuilder(resultRowIf->queryRowAllocator());
size32_t sz = resultRowIf->queryRowDeserializer()->deserialize(rowBuilder, mds);
OwnedConstThorRow result = rowBuilder.finalizeRowClear(sz);
helper->sendResult(result);
results = true;
}
}
if (!results && !abortSoon)
{
ActPrintLog("WARNING: no results");
IHThorRemoteResultArg *helper = (IHThorRemoteResultArg *)queryHelper();
//helper->sendResult(NULL);
// Jake I think this always cores (so raise exception instead)
throw MakeThorException(TE_UnexpectedMultipleSlaveResults, "Received no results from slaves");
}
}
IRowInterfaces * CDiskReadSlaveActivityBase::queryDiskRowInterfaces()
{
if (!diskRowIf)
diskRowIf.setown(createRowInterfaces(helper->queryDiskRecordSize(),queryActivityId(),queryCodeContext()));
return diskRowIf;
}
void start()
{
ActivityTimer s(totalCycles, timeActivities);
input = inputs.item(0);
try
{
try {
startInput(input);
}
catch (IException *e)
{
fireException(e);
barrier->cancel();
throw;
}
catch (CATCHALL)
{
Owned<IException> e = MakeActivityException(this, 0, "Unknown exception starting sort input");
fireException(e);
barrier->cancel();
throw;
}
dataLinkStart();
Linked<IRowInterfaces> rowif = queryRowInterfaces(input);
Owned<IRowInterfaces> auxrowif = createRowInterfaces(helper->querySortedRecordSize(),queryActivityId(),queryCodeContext());
sorter->Gather(
rowif,
input,
helper->queryCompare(),
helper->queryCompareLeftRight(),
NULL,helper->querySerialize(),
NULL,
false,
isUnstable(),
abortSoon,
auxrowif);
stopInput(input);
input = NULL;
if (abortSoon)
{
ActPrintLogEx(&queryContainer(), thorlog_null, MCwarning, "MSortSlaveActivity::start aborting");
barrier->cancel();
return;
}
}
catch (IException *e)
{
fireException(e);
barrier->cancel();
throw;
}
catch (CATCHALL)
{
Owned<IException> e = MakeActivityException(this, 0, "Unknown exception gathering sort input");
fireException(e);
barrier->cancel();
throw;
}
ActPrintLog("SORT waiting barrier.1");
if (!barrier->wait(false)) {
Sleep(1000); // let original error through
throw MakeThorException(TE_BarrierAborted,"SORT: Barrier Aborted");
}
ActPrintLog("SORT barrier.1 raised");
output.setown(sorter->startMerge(totalrows));
}
virtual void process()
{
ActPrintLog("process");
CMasterActivity::process();
IHThorSortArg *helper = (IHThorSortArg *)queryHelper();
StringBuffer skewV;
double skewError;
container.queryJob().getWorkUnitValue("overrideSkewError", skewV);
if (skewV.length())
skewError = atof(skewV.str());
else
{
skewError = helper->getSkew();
if (!skewError)
{
container.queryJob().getWorkUnitValue("defaultSkewError", skewV.clear());
if (skewV.length())
skewError = atof(skewV.str());
}
}
container.queryJob().getWorkUnitValue("defaultSkewWarning", skewV.clear());
double defaultSkewWarning = skewV.length() ? atof(skewV.str()) : 0;
double skewWarning = defaultSkewWarning;
unsigned __int64 skewThreshold = container.queryJob().getWorkUnitValueInt("overrideSkewThreshold", 0);
if (!skewThreshold)
{
skewThreshold = helper->getThreshold();
if (!skewThreshold)
skewThreshold = container.queryJob().getWorkUnitValueInt("defaultSkewThreshold", 0);
}
StringBuffer cosortfilenames;
const char *cosortlogname = helper->getSortedFilename();
if (cosortlogname&&*cosortlogname) {
Owned<IDistributedFile> file = queryThorFileManager().lookup(container.queryJob(), cosortlogname);
Owned<IFileDescriptor> fileDesc = file->getFileDescriptor();
queryThorFileManager().noteFileRead(container.queryJob(), file);
unsigned o;
for (o=0; o<fileDesc->numParts(); o++)
{
Owned<IPartDescriptor> partDesc = fileDesc->getPart(o);
if (cosortfilenames.length())
cosortfilenames.append("|");
// JCSMORE - picking the primary here, means no automatic use of backup copy, could use RMF's possibly.
getPartFilename(*partDesc, 0, cosortfilenames);
}
}
Owned<IRowInterfaces> rowif = createRowInterfaces(container.queryInput(0)->queryHelper()->queryOutputMeta(),queryActivityId(),queryCodeContext());
Owned<IRowInterfaces> auxrowif = createRowInterfaces(helper->querySortedRecordSize(),queryActivityId(),queryCodeContext());
try {
imaster->SortSetup(rowif,helper->queryCompare(),helper->querySerialize(),cosortfilenames.length()!=0,true,cosortfilenames.toCharArray(),auxrowif);
if (barrier->wait(false)) { // local sort complete
size32_t maxdeviance=globals->getPropInt("@sort_max_deviance", 10*1024*1024);
if (!imaster->Sort(skewThreshold,skewWarning,skewError,maxdeviance,true,false,false,(unsigned)globals->getPropInt("@smallSortThreshold"))) {
Owned<IThorException> e = MakeActivityException(this, TE_SortFailedSkewExceeded,"SORT failed, skew exceeded");
fireException(e);
}
barrier->wait(false); // merge complete
}
imaster->SortDone();
}
catch (IException *e)
{
ActPrintLog(e, "WARNING: exception during sort");
throw;
}
::Release(imaster);
ActPrintLog("process exit");
}
virtual void process()
{
ActPrintLog("process");
CSortBaseActivityMaster::process();
IHThorSortArg *helper = (IHThorSortArg *)queryHelper();
StringBuffer skewV;
double skewError;
container.queryJob().getWorkUnitValue("overrideSkewError", skewV);
if (skewV.length())
skewError = atof(skewV.str());
else
{
skewError = helper->getSkew();
if (!skewError)
{
container.queryJob().getWorkUnitValue("defaultSkewError", skewV.clear());
if (skewV.length())
skewError = atof(skewV.str());
}
}
container.queryJob().getWorkUnitValue("defaultSkewWarning", skewV.clear());
double defaultSkewWarning = skewV.length() ? atof(skewV.str()) : 0;
double skewWarning = defaultSkewWarning;
unsigned __int64 skewThreshold = container.queryJob().getWorkUnitValueInt("overrideSkewThreshold", 0);
if (!skewThreshold)
{
skewThreshold = helper->getThreshold();
if (!skewThreshold)
skewThreshold = container.queryJob().getWorkUnitValueInt("defaultSkewThreshold", 0);
}
Owned<IThorRowInterfaces> rowif = createRowInterfaces(container.queryInput(0)->queryHelper()->queryOutputMeta());
Owned<IThorRowInterfaces> auxrowif = createRowInterfaces(helper->querySortedRecordSize());
try
{
imaster->SortSetup(rowif,helper->queryCompare(),helper->querySerialize(),cosortfilenames.length()!=0,true,cosortfilenames.str(),auxrowif);
if (barrier->wait(false)) // local sort complete
{
size32_t maxdeviance = getOptUInt(THOROPT_SORT_MAX_DEVIANCE, 10*1024*1024);
try
{
imaster->Sort(skewThreshold,skewWarning,skewError,maxdeviance,true,false,false,getOptUInt(THOROPT_SMALLSORT));
}
catch (IThorException *e)
{
if (TE_SkewError == e->errorCode())
{
StringBuffer s;
Owned<IThorException> e2 = MakeActivityException(this, TE_SortFailedSkewExceeded, "SORT failed. %s", e->errorMessage(s).str());
e->Release();
fireException(e2);
}
else
throw;
}
barrier->wait(false); // merge complete
}
imaster->SortDone();
}
catch (IException *e)
{
ActPrintLog(e, "WARNING: exception during sort");
throw;
}
::Release(imaster);
ActPrintLog("process exit");
}
virtual void process()
{
ActPrintLog("GlobalMergeActivityMaster::process");
CMasterActivity::process();
IHThorMergeArg *helper = (IHThorMergeArg *)queryHelper();
Owned<IThorRowInterfaces> rowif = createRowInterfaces(helper->queryOutputMeta());
CThorKeyArray sample(*this, rowif,helper->querySerialize(),helper->queryCompare(),helper->queryCompareKey(),helper->queryCompareRowKey());
unsigned n = container.queryJob().querySlaves();
mptag_t *replytags = new mptag_t[n];
mptag_t *intertags = new mptag_t[n];
unsigned i;
for (i=0;i<n;i++) {
replytags[i] = TAG_NULL;
intertags[i] = TAG_NULL;
}
try {
for (i=0;i<n;i++) {
if (abortSoon)
return;
CMessageBuffer mb;
#ifdef _TRACE
ActPrintLog("Merge process, Receiving on tag %d",replyTag);
#endif
rank_t sender;
if (!receiveMsg(mb, RANK_ALL, replyTag, &sender)||abortSoon)
return;
#ifdef _TRACE
ActPrintLog("Merge process, Received sample from %d",sender);
#endif
sender--;
assertex((unsigned)sender<n);
assertex(replytags[(unsigned)sender]==TAG_NULL);
deserializeMPtag(mb,replytags[(unsigned)sender]);
deserializeMPtag(mb,intertags[(unsigned)sender]);
sample.deserialize(mb,true);
}
ActPrintLog("GlobalMergeActivityMaster::process samples merged");
sample.createSortedPartition(n);
ActPrintLog("GlobalMergeActivityMaster::process partition generated");
for (i=0;i<n;i++) {
if (abortSoon)
break;
CMessageBuffer mb;
mb.append(n);
for (unsigned j = 0;j<n;j++)
serializeMPtag(mb,intertags[j]);
sample.serialize(mb);
#ifdef _TRACE
ActPrintLog("Merge process, Replying to node %d tag %d",i+1,replytags[i]);
#endif
if (!queryJobChannel().queryJobComm().send(mb, (rank_t)i+1, replytags[i]))
break;
}
}
catch (IException *e) {
delete [] replytags;
delete [] intertags;
ActPrintLog(e, "MERGE");
throw;
}
delete [] replytags;
delete [] intertags;
ActPrintLog("GlobalMergeActivityMaster::process exit");
}