void ProcessSlaveActivity::main() { #ifdef TIME_ACTIVITIES if (timeActivities) lastCycles = get_cycles_now(); #endif try { process(); } catch (IException *_e) { IThorException *e = QUERYINTERFACE(_e, IThorException); if (e) { if (!e->queryActivityId()) { e->setGraphId(container.queryOwner().queryGraphId()); e->setActivityKind(container.getKind()); e->setActivityId(container.queryId()); } } else { e = MakeActivityException(this, _e); if (QUERYINTERFACE(_e, ISEH_Exception)) { IThorException *e2 = MakeThorFatal(e, TE_SEH, "FATAL: (SEH)"); e->Release(); e = e2; } _e->Release(); } ActPrintLog(e, NULL); exception.setown(e); } catch (std::exception & es) { StringBuffer m("FATAL std::exception "); if(dynamic_cast<std::bad_alloc *>(&es)) m.append("out of memory (std::bad_alloc)"); else m.append("standard library exception (std::exception ").append(es.what()).append(")"); m.appendf(" in %"ACTPF"d",container.queryId()); ActPrintLogEx(&queryContainer(), thorlog_null, MCerror, "%s", m.str()); exception.setown(MakeThorFatal(NULL, TE_UnknownException, "%s", m.str())); } catch (CATCHALL) { ActPrintLogEx(&queryContainer(), thorlog_null, MCerror, "Unknown exception thrown in process()"); exception.setown(MakeThorFatal(NULL, TE_UnknownException, "FATAL: Unknown exception thrown by ProcessThread")); } try { endProcess(); } catch (IException *_e) { ActPrintLog(_e, "Exception calling activity endProcess"); fireException(_e); exception.set(_e); _e->Release(); } }
void start() { ActivityTimer s(totalCycles, timeActivities); input = inputs.item(0); try { try { startInput(input); } catch (IException *e) { fireException(e); barrier->cancel(); throw; } catch (CATCHALL) { Owned<IException> e = MakeActivityException(this, 0, "Unknown exception starting sort input"); fireException(e); barrier->cancel(); throw; } dataLinkStart(); Linked<IRowInterfaces> rowif = queryRowInterfaces(input); Owned<IRowInterfaces> auxrowif = createRowInterfaces(helper->querySortedRecordSize(),queryActivityId(),queryCodeContext()); sorter->Gather( rowif, input, helper->queryCompare(), helper->queryCompareLeftRight(), NULL,helper->querySerialize(), NULL, false, isUnstable(), abortSoon, auxrowif); stopInput(input); input = NULL; if (abortSoon) { ActPrintLogEx(&queryContainer(), thorlog_null, MCwarning, "MSortSlaveActivity::start aborting"); barrier->cancel(); return; } } catch (IException *e) { fireException(e); barrier->cancel(); throw; } catch (CATCHALL) { Owned<IException> e = MakeActivityException(this, 0, "Unknown exception gathering sort input"); fireException(e); barrier->cancel(); throw; } ActPrintLog("SORT waiting barrier.1"); if (!barrier->wait(false)) { Sleep(1000); // let original error through throw MakeThorException(TE_BarrierAborted,"SORT: Barrier Aborted"); } ActPrintLog("SORT barrier.1 raised"); output.setown(sorter->startMerge(totalrows)); }
virtual void process() { ActPrintLog("process"); CMasterActivity::process(); IHThorSortArg *helper = (IHThorSortArg *)queryHelper(); StringBuffer skewV; double skewError; container.queryJob().getWorkUnitValue("overrideSkewError", skewV); if (skewV.length()) skewError = atof(skewV.str()); else { skewError = helper->getSkew(); if (!skewError) { container.queryJob().getWorkUnitValue("defaultSkewError", skewV.clear()); if (skewV.length()) skewError = atof(skewV.str()); } } container.queryJob().getWorkUnitValue("defaultSkewWarning", skewV.clear()); double defaultSkewWarning = skewV.length() ? atof(skewV.str()) : 0; double skewWarning = defaultSkewWarning; unsigned __int64 skewThreshold = container.queryJob().getWorkUnitValueInt("overrideSkewThreshold", 0); if (!skewThreshold) { skewThreshold = helper->getThreshold(); if (!skewThreshold) skewThreshold = container.queryJob().getWorkUnitValueInt("defaultSkewThreshold", 0); } StringBuffer cosortfilenames; const char *cosortlogname = helper->getSortedFilename(); if (cosortlogname&&*cosortlogname) { Owned<IDistributedFile> file = queryThorFileManager().lookup(container.queryJob(), cosortlogname); Owned<IFileDescriptor> fileDesc = file->getFileDescriptor(); queryThorFileManager().noteFileRead(container.queryJob(), file); unsigned o; for (o=0; o<fileDesc->numParts(); o++) { Owned<IPartDescriptor> partDesc = fileDesc->getPart(o); if (cosortfilenames.length()) cosortfilenames.append("|"); // JCSMORE - picking the primary here, means no automatic use of backup copy, could use RMF's possibly. getPartFilename(*partDesc, 0, cosortfilenames); } } Owned<IRowInterfaces> rowif = createRowInterfaces(container.queryInput(0)->queryHelper()->queryOutputMeta(),queryActivityId(),queryCodeContext()); Owned<IRowInterfaces> auxrowif = createRowInterfaces(helper->querySortedRecordSize(),queryActivityId(),queryCodeContext()); try { imaster->SortSetup(rowif,helper->queryCompare(),helper->querySerialize(),cosortfilenames.length()!=0,true,cosortfilenames.toCharArray(),auxrowif); if (barrier->wait(false)) { // local sort complete size32_t maxdeviance=globals->getPropInt("@sort_max_deviance", 10*1024*1024); if (!imaster->Sort(skewThreshold,skewWarning,skewError,maxdeviance,true,false,false,(unsigned)globals->getPropInt("@smallSortThreshold"))) { Owned<IThorException> e = MakeActivityException(this, TE_SortFailedSkewExceeded,"SORT failed, skew exceeded"); fireException(e); } barrier->wait(false); // merge complete } imaster->SortDone(); } catch (IException *e) { ActPrintLog(e, "WARNING: exception during sort"); throw; } ::Release(imaster); ActPrintLog("process exit"); }
virtual void process() { ActPrintLog("process"); CSortBaseActivityMaster::process(); IHThorSortArg *helper = (IHThorSortArg *)queryHelper(); StringBuffer skewV; double skewError; container.queryJob().getWorkUnitValue("overrideSkewError", skewV); if (skewV.length()) skewError = atof(skewV.str()); else { skewError = helper->getSkew(); if (!skewError) { container.queryJob().getWorkUnitValue("defaultSkewError", skewV.clear()); if (skewV.length()) skewError = atof(skewV.str()); } } container.queryJob().getWorkUnitValue("defaultSkewWarning", skewV.clear()); double defaultSkewWarning = skewV.length() ? atof(skewV.str()) : 0; double skewWarning = defaultSkewWarning; unsigned __int64 skewThreshold = container.queryJob().getWorkUnitValueInt("overrideSkewThreshold", 0); if (!skewThreshold) { skewThreshold = helper->getThreshold(); if (!skewThreshold) skewThreshold = container.queryJob().getWorkUnitValueInt("defaultSkewThreshold", 0); } Owned<IThorRowInterfaces> rowif = createRowInterfaces(container.queryInput(0)->queryHelper()->queryOutputMeta()); Owned<IThorRowInterfaces> auxrowif = createRowInterfaces(helper->querySortedRecordSize()); try { imaster->SortSetup(rowif,helper->queryCompare(),helper->querySerialize(),cosortfilenames.length()!=0,true,cosortfilenames.str(),auxrowif); if (barrier->wait(false)) // local sort complete { size32_t maxdeviance = getOptUInt(THOROPT_SORT_MAX_DEVIANCE, 10*1024*1024); try { imaster->Sort(skewThreshold,skewWarning,skewError,maxdeviance,true,false,false,getOptUInt(THOROPT_SMALLSORT)); } catch (IThorException *e) { if (TE_SkewError == e->errorCode()) { StringBuffer s; Owned<IThorException> e2 = MakeActivityException(this, TE_SortFailedSkewExceeded, "SORT failed. %s", e->errorMessage(s).str()); e->Release(); fireException(e2); } else throw; } barrier->wait(false); // merge complete } imaster->SortDone(); } catch (IException *e) { ActPrintLog(e, "WARNING: exception during sort"); throw; } ::Release(imaster); ActPrintLog("process exit"); }