void init(MemoryBuffer &data, MemoryBuffer &slaveData)
 {
     helper = (IHThorSortArg *)queryHelper();
     iCompare = helper->queryCompare();
     IHThorAlgorithm * algo = helper?(static_cast<IHThorAlgorithm *>(helper->selectInterface(TAIalgorithm_1))):NULL;
     unstable = (algo&&algo->getAlgorithmFlags()&TAFunstable);
     appendOutputLinked(this);
 }
 void start()
 {
     ActivityTimer s(totalCycles, timeActivities);
     input = inputs.item(0);
     try
     {
         try { 
             startInput(input); 
         }
         catch (IException *e)
         {
             fireException(e);
             barrier->cancel();
             throw;
         }
         catch (CATCHALL)
         {
             Owned<IException> e = MakeActivityException(this, 0, "Unknown exception starting sort input");
             fireException(e);
             barrier->cancel();
             throw;
         }
         dataLinkStart();
         
         Linked<IRowInterfaces> rowif = queryRowInterfaces(input);
         Owned<IRowInterfaces> auxrowif = createRowInterfaces(helper->querySortedRecordSize(),queryActivityId(),queryCodeContext());
         sorter->Gather(
             rowif,
             input,
             helper->queryCompare(),
             helper->queryCompareLeftRight(),
             NULL,helper->querySerialize(),
             NULL,
             false,
             isUnstable(),
             abortSoon,
             auxrowif);
         stopInput(input);
         input = NULL;
         if (abortSoon)
         {
             ActPrintLogEx(&queryContainer(), thorlog_null, MCwarning, "MSortSlaveActivity::start aborting");
             barrier->cancel();
             return;
         }
     }
     catch (IException *e)
     {
         fireException(e);
         barrier->cancel();
         throw;
     }
     catch (CATCHALL)
     {
         Owned<IException> e = MakeActivityException(this, 0, "Unknown exception gathering sort input");
         fireException(e);
         barrier->cancel();
         throw;
     }
     ActPrintLog("SORT waiting barrier.1");
     if (!barrier->wait(false)) {
         Sleep(1000); // let original error through
         throw MakeThorException(TE_BarrierAborted,"SORT: Barrier Aborted");
     }
     ActPrintLog("SORT barrier.1 raised");
     output.setown(sorter->startMerge(totalrows));
 }
Exemple #3
0
    virtual void process()
    {
        ActPrintLog("process");

        CMasterActivity::process();

        IHThorSortArg *helper = (IHThorSortArg *)queryHelper();
        StringBuffer skewV;
        double skewError;
        container.queryJob().getWorkUnitValue("overrideSkewError", skewV);
        if (skewV.length())
            skewError = atof(skewV.str());
        else
        {
            skewError = helper->getSkew();
            if (!skewError)
            {
                container.queryJob().getWorkUnitValue("defaultSkewError", skewV.clear());
                if (skewV.length())
                    skewError = atof(skewV.str());
            }
        }
        container.queryJob().getWorkUnitValue("defaultSkewWarning", skewV.clear());
        double defaultSkewWarning = skewV.length() ? atof(skewV.str()) : 0;
        double skewWarning = defaultSkewWarning;
        unsigned __int64 skewThreshold = container.queryJob().getWorkUnitValueInt("overrideSkewThreshold", 0);
        if (!skewThreshold)
        {
            skewThreshold = helper->getThreshold();         
            if (!skewThreshold)
                skewThreshold = container.queryJob().getWorkUnitValueInt("defaultSkewThreshold", 0);
        }
        StringBuffer cosortfilenames;
        const char *cosortlogname = helper->getSortedFilename();
        if (cosortlogname&&*cosortlogname) {

            Owned<IDistributedFile> file = queryThorFileManager().lookup(container.queryJob(), cosortlogname);
            Owned<IFileDescriptor> fileDesc = file->getFileDescriptor();
            queryThorFileManager().noteFileRead(container.queryJob(), file);
            unsigned o;
            for (o=0; o<fileDesc->numParts(); o++)
            {
                Owned<IPartDescriptor> partDesc = fileDesc->getPart(o);
                if (cosortfilenames.length())
                    cosortfilenames.append("|");

                // JCSMORE - picking the primary here, means no automatic use of backup copy, could use RMF's possibly.
                getPartFilename(*partDesc, 0, cosortfilenames);
            }
        }

        Owned<IRowInterfaces> rowif = createRowInterfaces(container.queryInput(0)->queryHelper()->queryOutputMeta(),queryActivityId(),queryCodeContext());
        Owned<IRowInterfaces> auxrowif = createRowInterfaces(helper->querySortedRecordSize(),queryActivityId(),queryCodeContext());
        try {   
            imaster->SortSetup(rowif,helper->queryCompare(),helper->querySerialize(),cosortfilenames.length()!=0,true,cosortfilenames.toCharArray(),auxrowif);
            if (barrier->wait(false)) { // local sort complete
                size32_t maxdeviance=globals->getPropInt("@sort_max_deviance", 10*1024*1024);
                if (!imaster->Sort(skewThreshold,skewWarning,skewError,maxdeviance,true,false,false,(unsigned)globals->getPropInt("@smallSortThreshold"))) {
                    Owned<IThorException> e = MakeActivityException(this, TE_SortFailedSkewExceeded,"SORT failed, skew exceeded");
                    fireException(e);
                }
                barrier->wait(false); // merge complete
            }
            imaster->SortDone();
        }
        catch (IException *e)
        {
            ActPrintLog(e, "WARNING: exception during sort");
            throw;
        }
        ::Release(imaster);
        ActPrintLog("process exit");
    }
Exemple #4
0
    virtual void process()
    {
        ActPrintLog("process");

        CSortBaseActivityMaster::process();

        IHThorSortArg *helper = (IHThorSortArg *)queryHelper();
        StringBuffer skewV;
        double skewError;
        container.queryJob().getWorkUnitValue("overrideSkewError", skewV);
        if (skewV.length())
            skewError = atof(skewV.str());
        else
        {
            skewError = helper->getSkew();
            if (!skewError)
            {
                container.queryJob().getWorkUnitValue("defaultSkewError", skewV.clear());
                if (skewV.length())
                    skewError = atof(skewV.str());
            }
        }
        container.queryJob().getWorkUnitValue("defaultSkewWarning", skewV.clear());
        double defaultSkewWarning = skewV.length() ? atof(skewV.str()) : 0;
        double skewWarning = defaultSkewWarning;
        unsigned __int64 skewThreshold = container.queryJob().getWorkUnitValueInt("overrideSkewThreshold", 0);
        if (!skewThreshold)
        {
            skewThreshold = helper->getThreshold();         
            if (!skewThreshold)
                skewThreshold = container.queryJob().getWorkUnitValueInt("defaultSkewThreshold", 0);
        }

        Owned<IThorRowInterfaces> rowif = createRowInterfaces(container.queryInput(0)->queryHelper()->queryOutputMeta());
        Owned<IThorRowInterfaces> auxrowif = createRowInterfaces(helper->querySortedRecordSize());
        try
        {
            imaster->SortSetup(rowif,helper->queryCompare(),helper->querySerialize(),cosortfilenames.length()!=0,true,cosortfilenames.str(),auxrowif);
            if (barrier->wait(false)) // local sort complete
            {
                size32_t maxdeviance = getOptUInt(THOROPT_SORT_MAX_DEVIANCE, 10*1024*1024);
                try
                {
                    imaster->Sort(skewThreshold,skewWarning,skewError,maxdeviance,true,false,false,getOptUInt(THOROPT_SMALLSORT));
                }
                catch (IThorException *e)
                {
                    if (TE_SkewError == e->errorCode())
                    {
                        StringBuffer s;
                        Owned<IThorException> e2 = MakeActivityException(this, TE_SortFailedSkewExceeded, "SORT failed. %s", e->errorMessage(s).str());
                        e->Release();
                        fireException(e2);
                    }
                    else
                        throw;
                }
                barrier->wait(false); // merge complete
            }
            imaster->SortDone();
        }
        catch (IException *e)
        {
            ActPrintLog(e, "WARNING: exception during sort");
            throw;
        }
        ::Release(imaster);
        ActPrintLog("process exit");
    }