virtual void init() { CSortBaseActivityMaster::init(); IHThorSortArg *helper = (IHThorSortArg *)queryHelper(); IHThorAlgorithm *algo = static_cast<IHThorAlgorithm *>(helper->selectInterface(TAIalgorithm_1)); OwnedRoxieString algoname(algo->getAlgorithm()); unsigned flags = algo->getAlgorithmFlags(); if (algoname && (0 != stricmp(algoname, "quicksort"))) { Owned<IException> e = MakeActivityException(this, 0, "Ignoring, unsupported sort order algorithm '%s'", algoname.get()); reportExceptionToWorkunit(container.queryJob().queryWorkUnit(), e); } OwnedRoxieString cosortlogname(helper->getSortedFilename()); if (cosortlogname&&*cosortlogname) { Owned<IDistributedFile> coSortFile = queryThorFileManager().lookup(container.queryJob(), cosortlogname); addReadFile(coSortFile); Owned<IFileDescriptor> fileDesc = coSortFile->getFileDescriptor(); unsigned o; for (o=0; o<fileDesc->numParts(); o++) { Owned<IPartDescriptor> partDesc = fileDesc->getPart(o); if (cosortfilenames.length()) cosortfilenames.append("|"); // JCSMORE - picking the primary here, means no automatic use of backup copy, could use RMF's possibly. getPartFilename(*partDesc, 0, cosortfilenames); } } }
virtual void process() { ActPrintLog("process"); CMasterActivity::process(); IHThorSortArg *helper = (IHThorSortArg *)queryHelper(); StringBuffer skewV; double skewError; container.queryJob().getWorkUnitValue("overrideSkewError", skewV); if (skewV.length()) skewError = atof(skewV.str()); else { skewError = helper->getSkew(); if (!skewError) { container.queryJob().getWorkUnitValue("defaultSkewError", skewV.clear()); if (skewV.length()) skewError = atof(skewV.str()); } } container.queryJob().getWorkUnitValue("defaultSkewWarning", skewV.clear()); double defaultSkewWarning = skewV.length() ? atof(skewV.str()) : 0; double skewWarning = defaultSkewWarning; unsigned __int64 skewThreshold = container.queryJob().getWorkUnitValueInt("overrideSkewThreshold", 0); if (!skewThreshold) { skewThreshold = helper->getThreshold(); if (!skewThreshold) skewThreshold = container.queryJob().getWorkUnitValueInt("defaultSkewThreshold", 0); } StringBuffer cosortfilenames; const char *cosortlogname = helper->getSortedFilename(); if (cosortlogname&&*cosortlogname) { Owned<IDistributedFile> file = queryThorFileManager().lookup(container.queryJob(), cosortlogname); Owned<IFileDescriptor> fileDesc = file->getFileDescriptor(); queryThorFileManager().noteFileRead(container.queryJob(), file); unsigned o; for (o=0; o<fileDesc->numParts(); o++) { Owned<IPartDescriptor> partDesc = fileDesc->getPart(o); if (cosortfilenames.length()) cosortfilenames.append("|"); // JCSMORE - picking the primary here, means no automatic use of backup copy, could use RMF's possibly. getPartFilename(*partDesc, 0, cosortfilenames); } } Owned<IRowInterfaces> rowif = createRowInterfaces(container.queryInput(0)->queryHelper()->queryOutputMeta(),queryActivityId(),queryCodeContext()); Owned<IRowInterfaces> auxrowif = createRowInterfaces(helper->querySortedRecordSize(),queryActivityId(),queryCodeContext()); try { imaster->SortSetup(rowif,helper->queryCompare(),helper->querySerialize(),cosortfilenames.length()!=0,true,cosortfilenames.toCharArray(),auxrowif); if (barrier->wait(false)) { // local sort complete size32_t maxdeviance=globals->getPropInt("@sort_max_deviance", 10*1024*1024); if (!imaster->Sort(skewThreshold,skewWarning,skewError,maxdeviance,true,false,false,(unsigned)globals->getPropInt("@smallSortThreshold"))) { Owned<IThorException> e = MakeActivityException(this, TE_SortFailedSkewExceeded,"SORT failed, skew exceeded"); fireException(e); } barrier->wait(false); // merge complete } imaster->SortDone(); } catch (IException *e) { ActPrintLog(e, "WARNING: exception during sort"); throw; } ::Release(imaster); ActPrintLog("process exit"); }