StringBuffer &errorMessage(StringBuffer &str) const
{
if (!origin.length() || 0 != stricmp("user", origin.get())) // don't report slave in user message
{
if (graphId)
str.append("Graph[").append(graphId).append("], ");
if (kind)
str.append(activityKindStr(kind));
if (id)
{
if (kind) str.append('[');
str.append(id);
if (kind) str.append(']');
str.append(": ");
}
if (node)
{
str.appendf("SLAVE #%d [", node);
queryClusterGroup().queryNode(node).endpoint().getUrlStr(str);
str.append("]: ");
}
}
str.append(msg);
return str;
}
StringBuffer &errorMessage(StringBuffer &str) const
{
if (!origin.length() || 0 != stricmp("user", origin.get())) // don't report slave in user message
{
if (graphId)
str.append("Graph ").append(graphName).append("[").append(graphId).append("], ");
if (kind)
str.append(activityKindStr(kind));
if (id)
{
if (kind) str.append('[');
str.append(id);
if (kind) str.append(']');
str.append(": ");
}
if (slave)
{
str.appendf("SLAVE #%d [", slave);
queryClusterGroup().queryNode(slave).endpoint().getUrlStr(str);
str.append("]: ");
}
}
str.append(msg);
if (originalException)
{
if (msg.length())
str.append(" - ");
str.append("caused by (");
str.append(originalException->errorCode());
str.append(", ");
originalException->errorMessage(str);
str.append(")");
}
return str;
}
CDiskPartHandlerBase::CDiskPartHandlerBase(CDiskReadSlaveActivityBase &_activity)
: activity(_activity)
{
checkFileCrc = activity.checkFileCrc;
which = 0;
eoi = false;
kindStr = activityKindStr(activity.queryContainer().getKind());
}
virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData)
{
helper = static_cast <IHThorCatchArg *> (queryHelper());
kindStr.append(activityKindStr(container.getKind()));
kindStr.toUpperCase();
eos = false;
appendOutputLinked(this);
}
CDiskPartHandlerBase::CDiskPartHandlerBase(CDiskReadSlaveActivityBase &_activity)
: activity(_activity)
{
checkFileCrc = activity.checkFileCrc;
which = 0;
eoi = false;
kindStr = activityKindStr(activity.queryContainer().getKind());
compressed = blockCompressed = firstInGroup = checkFileCrc = false;
}
void start()
{
ActivityTimer s(totalCycles, timeActivities, NULL);
dataLinkStart(activityKindStr(queryContainer().getKind()), container.queryId());
input = inputs.item(0);
unsigned spillPriority = container.queryGrouped() ? SPILL_PRIORITY_GROUPSORT : SPILL_PRIORITY_LARGESORT;
iLoader.setown(createThorRowLoader(*this, queryRowInterfaces(input), iCompare, !unstable, rc_mixed, spillPriority));
startInput(input);
eoi = false;
if (container.queryGrouped())
out.setown(iLoader->loadGroup(input, abortSoon));
else
out.setown(iLoader->load(input, abortSoon));
if (0 == iLoader->numRows())
eoi = true;
}
StringBuffer &ActPrintLogArgsPrep(StringBuffer &res, const CGraphElementBase *container, const ActLogEnum flags, const char *format, va_list args)
{
if (format)
res.valist_appendf(format, args).append(" - ");
res.appendf("activity(%s, %" ACTPF "d)",activityKindStr(container->getKind()), container->queryId());
if (0 != (flags & thorlog_ecl))
{
StringBuffer ecltext;
container->getEclText(ecltext);
ecltext.trim();
if (ecltext.length() > 0)
res.append(" [ecl=").append(ecltext.str()).append(']');
}
#ifdef _WIN32
#ifdef MEMLOG
MEMORYSTATUS mS;
GlobalMemoryStatus(&mS);
res.appendf(", mem=%ld",mS.dwAvailPhys);
#endif
#endif
return res;
}
virtual CActivityBase *factory(ThorActivityKind kind)
{
CActivityBase *ret = NULL;
switch (kind)
{
case TAKfiltergroup:
case TAKlocalresultread:
case TAKchildif:
case TAKchildcase:
case TAKdegroup:
case TAKsplit:
case TAKproject:
case TAKprefetchproject:
case TAKprefetchcountproject:
case TAKxmlparse:
case TAKchilditerator:
case TAKlinkedrawiterator:
case TAKcatch:
case TAKsample:
case TAKnormalize:
case TAKnormalizechild:
case TAKnormalizelinkedchild:
case TAKinlinetable:
case TAKpull:
case TAKnull:
case TAKpiperead:
case TAKpipethrough:
case TAKparse:
case TAKchildaggregate:
case TAKchildgroupaggregate:
case TAKchildthroughnormalize:
case TAKchildnormalize:
case TAKapply:
case TAKfunnel:
case TAKcombine:
case TAKregroup:
case TAKsorted:
case TAKnwayinput:
case TAKnwayselect:
case TAKnwaymerge:
case TAKnwaymergejoin:
case TAKnwayjoin:
case TAKgraphloopresultread:
case TAKstreamediterator:
case TAKsoap_rowdataset:
case TAKsoap_rowaction:
case TAKsoap_datasetdataset:
case TAKsoap_datasetaction:
case TAKhttp_rowdataset:
case TAKdistributed:
ret = new CMasterActivity(this);
break;
case TAKskipcatch:
case TAKcreaterowcatch:
ret = createSkipCatchActivityMaster(this);
break;
case TAKdiskread:
case TAKdisknormalize:
ret = createDiskReadActivityMaster(this);
break;
case TAKdiskaggregate:
ret = createDiskAggregateActivityMaster(this);
break;
case TAKdiskcount:
ret = createDiskCountActivityMaster(this);
break;
case TAKdiskgroupaggregate:
ret = createDiskGroupAggregateActivityMaster(this);
break;
case TAKindexread:
ret = createIndexReadActivityMaster(this);
break;
case TAKindexcount:
ret = createIndexCountActivityMaster(this);
break;
case TAKindexnormalize:
ret = createIndexNormalizeActivityMaster(this);
break;
case TAKindexaggregate:
ret = createIndexAggregateActivityMaster(this);
break;
case TAKindexgroupaggregate:
case TAKindexgroupexists:
case TAKindexgroupcount:
ret = createIndexGroupAggregateActivityMaster(this);
break;
case TAKdiskwrite:
ret = createDiskWriteActivityMaster(this);
break;
case TAKcsvwrite:
ret = createCsvWriteActivityMaster(this);
break;
case TAKspill:
ret = createSpillActivityMaster(this);
break;
case TAKdedup:
case TAKrollup:
case TAKrollupgroup:
ret = createDedupRollupActivityMaster(this);
break;
case TAKfilter:
case TAKfilterproject:
ret = createFilterActivityMaster(this);
break;
case TAKnonempty:
ret = createNonEmptyActivityMaster(this);
break;
case TAKsort:
ret = createSortActivityMaster(this);
break;
case TAKgroup:
ret = createGroupActivityMaster(this);
break;
case TAKprocess:
case TAKiterate:
ret = createIterateActivityMaster(this);
break;
case TAKthroughaggregate:
ret = createThroughAggregateActivityMaster(this);
break;
case TAKaggregate:
case TAKexistsaggregate:
case TAKcountaggregate:
ret = createAggregateActivityMaster(this);
break;
case TAKhashdistribute:
case TAKpartition:
ret = createHashDistributeActivityMaster(this);
break;
case TAKhashaggregate:
ret = createHashAggregateActivityMaster(this);
break;
case TAKhashjoin:
case TAKhashdenormalize:
case TAKhashdenormalizegroup:
ret= createHashJoinActivityMaster(this);
break;
case TAKkeyeddistribute:
ret = createKeyedDistributeActivityMaster(this);
break;
case TAKhashdistributemerge:
ret = createDistributeMergeActivityMaster(this);
break;
case TAKhashdedup:
ret = createHashDedupMergeActivityMaster(this);
break;
case TAKfirstn:
ret = createFirstNActivityMaster(this);
break;
case TAKjoin:
case TAKselfjoin:
case TAKselfjoinlight:
case TAKdenormalize:
case TAKdenormalizegroup:
ret = createJoinActivityMaster(this);
break;
case TAKlookupjoin:
case TAKalljoin:
case TAKlookupdenormalize:
case TAKlookupdenormalizegroup:
case TAKsmartjoin:
case TAKsmartdenormalize:
case TAKsmartdenormalizegroup:
case TAKalldenormalize:
case TAKalldenormalizegroup:
ret = createLookupJoinActivityMaster(this);
break;
case TAKkeyedjoin:
case TAKkeyeddenormalize:
case TAKkeyeddenormalizegroup:
ret = createKeyedJoinActivityMaster(this);
break;
case TAKworkunitwrite:
ret = createWorkUnitWriteActivityMaster(this);
break;
case TAKdictionaryworkunitwrite:
ret = createDictionaryWorkunitWriteMaster(this);
break;
case TAKdictionaryresultwrite:
if (!queryOwner().queryOwner() || queryOwner().isGlobal()) // don't need dictionary in master if in local child query
ret = createDictionaryResultActivityMaster(this);
else
ret = new CMasterActivity(this);
break;
break;
case TAKremoteresult:
ret = createResultActivityMaster(this);
break;
case TAKselectn:
ret = createSelectNthActivityMaster(this);
break;
case TAKenth:
ret = createEnthActivityMaster(this);
break;
case TAKdistribution:
ret = createDistributionActivityMaster(this);
break;
case TAKcountproject:
ret = createCountProjectActivityMaster(this);
break;
case TAKchoosesets:
ret = createChooseSetsActivityMaster(this);
break;
case TAKchoosesetsenth:
case TAKchoosesetslast:
ret = createChooseSetsPlusActivityMaster(this);
break;
case TAKpipewrite:
ret = createPipeWriteActivityMaster(this);
break;
case TAKcsvread:
ret = createCCsvReadActivityMaster(this);
break;
case TAKindexwrite:
ret = createIndexWriteActivityMaster(this);
break;
case TAKfetch:
ret = createFetchActivityMaster(this);
break;
case TAKcsvfetch:
ret = createCsvFetchActivityMaster(this);
break;
case TAKxmlfetch:
ret = createXmlFetchActivityMaster(this);
break;
case TAKworkunitread:
ret = createWorkUnitActivityMaster(this);
break;
case TAKsideeffect:
ret = createNullActionActivityMaster(this);
break;
case TAKsimpleaction:
ret = createActionActivityMaster(this);
break;
case TAKtopn:
ret = createTopNActivityMaster(this);
break;
case TAKxmlread:
ret = createXmlReadActivityMaster(this);
break;
case TAKxmlwrite:
case TAKjsonwrite:
ret = createXmlWriteActivityMaster(this, kind);
break;
case TAKmerge:
ret = createMergeActivityMaster(this);
break;
case TAKkeydiff:
ret = createKeyDiffActivityMaster(this);
break;
case TAKkeypatch:
ret = createKeyPatchActivityMaster(this);
break;
case TAKlimit:
case TAKskiplimit:
case TAKcreaterowlimit:
ret = createLimitActivityMaster(this);
break;
case TAKlooprow:
case TAKloopcount:
case TAKloopdataset:
ret = createLoopActivityMaster(this);
break;
case TAKgraphloop:
case TAKparallelgraphloop:
ret = createGraphLoopActivityMaster(this);
break;
case TAKlocalresultspill:
case TAKlocalresultwrite:
if (!queryOwner().queryOwner() || queryOwner().isGlobal()) // don't need result in master if in local child query
ret = createLocalResultActivityMaster(this);
else
ret = new CMasterActivity(this);
break;
case TAKgraphloopresultwrite:
ret = createGraphLoopResultActivityMaster(this);
break;
case TAKchilddataset:
UNIMPLEMENTED;
case TAKcase: // gen. time.
case TAKif:
case TAKifaction:
throwUnexpected();
case TAKwhen_dataset:
ret = createWhenActivityMaster(this);
break;
default:
throw MakeActivityException(this, TE_UnsupportedActivityKind, "Unsupported activity kind: %s", activityKindStr(kind));
}
return ret;
}
virtual CActivityBase *factory(ThorActivityKind kind)
{
CActivityBase *ret = NULL;
switch (kind)
{
case TAKdiskread:
ret = createDiskReadSlave(this);
break;
case TAKdisknormalize:
ret = createDiskNormalizeSlave(this);
break;
case TAKdiskaggregate:
ret = createDiskAggregateSlave(this);
break;
case TAKdiskcount:
ret = createDiskCountSlave(this);
break;
case TAKdiskgroupaggregate:
ret = createDiskGroupAggregateSlave(this);
break;
case TAKindexread:
ret = createIndexReadSlave(this);
break;
case TAKindexcount:
ret = createIndexCountSlave(this);
break;
case TAKindexnormalize:
ret = createIndexNormalizeSlave(this);
break;
case TAKindexaggregate:
ret = createIndexAggregateSlave(this);
break;
case TAKindexgroupaggregate:
case TAKindexgroupexists:
case TAKindexgroupcount:
ret = createIndexGroupAggregateSlave(this);
break;
case TAKchildaggregate:
ret = createChildAggregateSlave(this);
break;
case TAKchildgroupaggregate:
ret = createChildGroupAggregateSlave(this);
break;
case TAKchildthroughnormalize:
ret = createChildThroughNormalizeSlave(this);
break;
case TAKchildnormalize:
ret = createChildNormalizeSlave(this);
break;
case TAKspill:
ret = createSpillSlave(this);
break;
case TAKdiskwrite:
ret = createDiskWriteSlave(this);
break;
case TAKsort:
if (queryGrouped())
ret = createGroupSortSlave(this);
else if (queryLocal())
ret = createLocalSortSlave(this);
else
ret = createMSortSlave(this);
break;
case TAKsorted:
ret = createSortedSlave(this);
break;
case TAKdedup:
if (queryGrouped())
ret = createGroupDedupSlave(this);
else if (queryLocal())
ret = createLocalDedupSlave(this);
else
ret = createDedupSlave(this);
break;
case TAKrollupgroup:
ret = createRollupGroupSlave(this);
break;
case TAKrollup:
if (queryGrouped())
ret = createGroupRollupSlave(this);
else if (queryLocal())
ret = createLocalRollupSlave(this);
else
ret = createRollupSlave(this);
break;
case TAKprocess:
if (queryGrouped())
ret = createGroupProcessSlave(this);
else if (queryLocal())
ret = createLocalProcessSlave(this);
else
ret = createProcessSlave(this);
break;
case TAKfilter:
ret = createFilterSlave(this);
break;
case TAKfilterproject:
ret = createFilterProjectSlave(this);
break;
case TAKfiltergroup:
ret = createFilterGroupSlave(this);
break;
case TAKsplit:
ret = createNSplitterSlave(this);
break;
case TAKproject:
ret = createProjectSlave(this);
break;
case TAKprefetchproject:
ret = createPrefetchProjectSlave(this);
break;
case TAKprefetchcountproject:
break;
case TAKiterate:
if (queryGrouped())
ret = createGroupIterateSlave(this);
else if (queryLocal())
ret = createLocalIterateSlave(this);
else
ret = createIterateSlave(this);
break;
case TAKaggregate:
case TAKexistsaggregate:
case TAKcountaggregate:
if (queryLocalOrGrouped())
ret = createGroupAggregateSlave(this);
else
ret = createAggregateSlave(this);
break;
case TAKhashaggregate:
ret = createHashAggregateSlave(this);
break;
case TAKfirstn:
ret = createFirstNSlave(this);
break;
case TAKsample:
ret = createSampleSlave(this);
break;
case TAKdegroup:
ret = createDegroupSlave(this);
break;
case TAKjoin:
if (queryLocalOrGrouped())
ret = createLocalJoinSlave(this);
else
ret = createJoinSlave(this);
break;
case TAKhashjoin:
case TAKhashdenormalize:
case TAKhashdenormalizegroup:
ret = createHashJoinSlave(this);
break;
case TAKlookupjoin:
ret = createLookupJoinSlave(this);
break;
case TAKalljoin:
ret = createAllJoinSlave(this);
break;
case TAKselfjoin:
if (queryLocalOrGrouped())
ret = createLocalSelfJoinSlave(this);
else
ret = createSelfJoinSlave(this);
break;
case TAKselfjoinlight:
ret = createLightweightSelfJoinSlave(this);
break;
case TAKkeyedjoin:
case TAKkeyeddenormalize:
case TAKkeyeddenormalizegroup:
ret = createKeyedJoinSlave(this);
break;
case TAKgroup:
if (queryLocalOrGrouped())
ret = createLocalGroupSlave(this);
else
ret = createGroupSlave(this);
break;
case TAKworkunitwrite:
ret = createWorkUnitWriteSlave(this);
break;
case TAKfunnel:
ret = createFunnelSlave(this);
break;
case TAKcombine:
ret = createCombineSlave(this);
break;
case TAKregroup:
ret = createRegroupSlave(this);
break;
case TAKapply:
ret = createApplySlave(this);
break;
case TAKtemptable:
case TAKtemprow:
ret = createTempTableSlave(this);
break;
case TAKkeyeddistribute:
ret = createIndexDistributeSlave(this);
break;
case TAKhashdistribute:
ret = createHashDistributeSlave(this);
break;
case TAKhashdistributemerge:
ret = createHashDistributeMergeSlave(this);
break;
case TAKhashdedup:
if (queryLocalOrGrouped())
ret = createHashLocalDedupSlave(this);
else
ret = createHashDedupSlave(this);
break;
case TAKnormalize:
ret = createNormalizeSlave(this);
break;
case TAKnormalizechild:
ret = createNormalizeChildSlave(this);
break;
case TAKnormalizelinkedchild:
ret = createNormalizeLinkedChildSlave(this);
break;
case TAKremoteresult:
ret = createResultSlave(this);
break;
case TAKpull:
ret = createPullSlave(this);
break;
case TAKdenormalize:
case TAKdenormalizegroup:
if (queryLocalOrGrouped())
ret = createLocalDenormalizeSlave(this);
else
ret = createDenormalizeSlave(this);
break;
case TAKnwayinput:
ret = createNWayInputSlave(this);
break;
case TAKnwayselect:
ret = createNWaySelectSlave(this);
break;
case TAKnwaymerge:
ret = createNWayMergeActivity(this);
break;
case TAKnwaymergejoin:
case TAKnwayjoin:
ret = createNWayMergeJoinActivity(this);
break;
case TAKalldenormalize:
case TAKalldenormalizegroup:
ret = createAllDenormalizeSlave(this);
break;
case TAKlookupdenormalize:
case TAKlookupdenormalizegroup:
ret = createLookupDenormalizeSlave(this);
break;
case TAKchilddataset:
UNIMPLEMENTED;
case TAKchilditerator:
ret = createChildIteratorSlave(this);
break;
case TAKrawiterator:
ret = createRawIteratorSlave(this);
break;
case TAKlinkedrawiterator:
ret = createLinkedRawIteratorSlave(this);
break;
case TAKselectn:
if (queryLocalOrGrouped())
ret = createLocalSelectNthSlave(this);
else
ret = createSelectNthSlave(this);
break;
case TAKenth:
if (queryLocalOrGrouped())
ret = createLocalEnthSlave(this);
else
ret = createEnthSlave(this);
break;
case TAKnull:
ret = createNullSlave(this);
break;
case TAKdistribution:
ret = createDistributionSlave(this);
break;
case TAKcountproject:
if (queryLocalOrGrouped())
ret = createLocalCountProjectSlave(this);
else
ret = createCountProjectSlave(this);
break;
case TAKchoosesets:
if (queryLocalOrGrouped())
ret = createLocalChooseSetsSlave(this);
else
ret = createChooseSetsSlave(this);
break;
case TAKpiperead:
ret = createPipeReadSlave(this);
break;
case TAKpipewrite:
ret = createPipeWriteSlave(this);
break;
case TAKcsvread:
ret = createCsvReadSlave(this);
break;
case TAKcsvwrite:
ret = createCsvWriteSlave(this);
break;
case TAKpipethrough:
ret = createPipeThroughSlave(this);
break;
case TAKindexwrite:
ret = createIndexWriteSlave(this);
break;
case TAKchoosesetsenth:
ret = createChooseSetsEnthSlave(this);
break;
case TAKchoosesetslast:
ret = createChooseSetsLastSlave(this);
break;
case TAKfetch:
ret = createFetchSlave(this);
break;
case TAKcsvfetch:
ret = createCsvFetchSlave(this);
break;
case TAKxmlfetch:
ret = createXmlFetchSlave(this);
break;
case TAKthroughaggregate:
ret = createThroughAggregateSlave(this);
break;
case TAKcase:
case TAKif:
throwUnexpected();
break;
case TAKwhen_dataset:
ret = createWhenSlave(this);
break;
case TAKworkunitread:
{
if (wuidread2diskread)
{
Owned<IHThorDiskReadArg> diskReadHelper = createWorkUnitReadArg(wuidreadFilename, (IHThorWorkunitReadArg *)LINK(baseHelper));
Owned<CActivityBase> retAct = createDiskReadSlave(this, diskReadHelper);
return retAct.getClear();
}
else
ret = createWuidReadSlave(this);
break;
}
case TAKparse:
ret = createParseSlave(this);
break;
case TAKsideeffect:
ret = createNullActionSlave(this);
break;
case TAKsimpleaction:
ret = createNullSlave(this);
break;
case TAKtopn:
if (queryGrouped())
ret = createGroupedTopNSlave(this);
else if (queryLocal())
ret = createLocalTopNSlave(this);
else
ret = createGlobalTopNSlave(this);
break;
case TAKxmlparse:
ret = createXmlParseSlave(this);
break;
case TAKxmlread:
ret = createXmlReadSlave(this);
break;
case TAKxmlwrite:
ret = createXmlWriteSlave(this);
break;
case TAKmerge:
if (queryLocalOrGrouped())
ret = createLocalMergeSlave(this);
else
ret = createGlobalMergeSlave(this);
break;
case TAKsoap_rowdataset:
ret = createSoapRowCallSlave(this);
break;
case TAKsoap_rowaction:
ret = createSoapRowActionSlave(this);
break;
case TAKsoap_datasetdataset:
ret = createSoapDatasetCallSlave(this);
break;
case TAKsoap_datasetaction:
ret = createSoapDatasetActionSlave(this);
break;
case TAKcountdisk:
return new CSlaveActivity(this);
case TAKkeydiff:
ret = createKeyDiffSlave(this);
break;
case TAKkeypatch:
ret = createKeyPatchSlave(this);
break;
case TAKlimit:
ret = createLimitSlave(this);
break;
case TAKskiplimit:
ret = createSkipLimitSlave(this);
break;
case TAKcreaterowlimit:
ret = createRowLimitSlave(this);
break;
case TAKnonempty:
ret = createNonEmptySlave(this);
break;
case TAKlocalresultread:
ret = createLocalResultReadSlave(this);
break;
case TAKlocalresultwrite:
ret = createLocalResultWriteSlave(this);
break;
case TAKlocalresultspill:
ret = createLocalResultSpillSlave(this);
break;
case TAKchildif:
ret = createIfSlave(this);
break;
case TAKcatch:
case TAKskipcatch:
case TAKcreaterowcatch:
ret = createCatchSlave(this);
break;
case TAKlooprow:
case TAKloopcount:
case TAKloopdataset:
ret = createLoopSlave(this);
break;
case TAKgraphloop:
case TAKparallelgraphloop:
ret = createGraphLoopSlave(this);
break;
case TAKgraphloopresultread:
ret = createGraphLoopResultReadSlave(this);
break;
case TAKgraphloopresultwrite:
ret = createGraphLoopResultWriteSlave(this);
break;
case TAKstreamediterator:
ret = createStreamedIteratorSlave(this);
break;
case TAKifaction:
ret = createIfActionSlave(this);
break;
default:
throw MakeStringException(TE_UnsupportedActivityKind, "Unsupported activity kind: %s", activityKindStr(kind));
}
return ret;
}
