Example #1
0
 virtual void process()
 {
     if (totalCountKnown) return;
     if (container.queryLocalOrGrouped())
         return;
     totalCount = ::getCount(*this, container.queryJob().querySlaves(), stopAfter, mpTag);
     if (totalCount > stopAfter)
         totalCount = stopAfter;
     CMessageBuffer msg;
     msg.append(totalCount);
     if (!container.queryJob().queryJobComm().send(msg, 1, mpTag, 5000))
         throw MakeThorException(0, "Failed to give result to slave");
 }
Example #2
0
void CLogMsgLinkToChild::disconnect()
{
    CMessageBuffer out;
    out.append('D').append(pid);
    try
    {
        queryWorldCommunicator().send(out, childNode, MPTAG_JLOG_CONNECT_TO_CHILD, MP_ASYNC_SEND);
    }
    catch(IException * e)
    {
        e->Release();
    }
    connected = false;
}
Example #3
0
 void sendResult(const void *row, IOutputRowSerializer *serializer)
 {
     CMessageBuffer mb;
     size32_t start = mb.length();
     size32_t sz = 0;
     mb.append(sz);
     if (row&&hadElement) {
         CMemoryRowSerializer mbs(mb);
         serializer->serialize(mbs,(const byte *)row);
         sz = mb.length()-start-sizeof(size32_t);
         mb.writeDirect(start,sizeof(size32_t),&sz);
     }
     container.queryJob().queryJobComm().send(mb, 0, masterMpTag);
 }
    void getTallies() // NB: not called on first node.
    {
        CMessageBuffer msg;
        if (!receiveMsg(msg, queryJobChannel().queryMyRank()-1, mpTag))
            return;
        memcpy(tallies, msg.readDirect(numSets*sizeof(unsigned)), numSets*sizeof(unsigned));
#if THOR_TRACE_LEVEL >= 5
        StringBuffer s;
        unsigned idx=0;
        for (; idx<numSets; idx++)
            s.append("[").append(tallies[idx]).append("]");
        ActPrintLog("CHOOSESETS: Incoming count = %s", s.str());
#endif
    }
void CDiskWriteSlaveActivityBase::close()
{
    try
    {
        if (out) {
            uncompressedBytesWritten = out->getPosition();
            if (calcFileCrc) {
                if (diskHelperBase->getFlags() & TDWextend) {
                    assertex(!"TBD need to merge CRC");
                }   
                else
                    out->flush(&fileCRC);
            }
            else if (!abortSoon)
                out->flush();
            out.clear();
        }
        else if (outraw) {
            outraw->flush();
            uncompressedBytesWritten = outraw->tell();
            outraw.clear();
        }

        {
            CriticalBlock block(statsCs);
            mergeStats(fileStats, outputIO);
            outputIO.clear();
        }

        if (!rfsQueryParallel && dlfn.isExternal() && !lastNode())
        {
            rowcount_t rows = processed & THORDATALINK_COUNT_MASK;
            ActPrintLog("External write done, signalling next (row count = %" RCPF "d)", rows);
            CMessageBuffer msg;
            msg.append(rows);
            msg.append(tempExternalName);
            queryJobChannel().queryJobComm().send(msg, queryJobChannel().queryMyRank()+1, mpTag);
        }
    }
    catch (IException *e)
    { 
        ActPrintLogEx(&queryContainer(), e, thorlog_null, MCwarning, "Error closing file: %s", fName.get());
        abortSoon = true;
        removeFiles();
        throw e;
    }
    if (abortSoon)
        removeFiles();
}
__int64 RemoteDataSource::numRows(bool force)
{
    if (!force)
        return cachedNumRows;
    CMessageBuffer msg;
    msg.setEndian(__BIG_ENDIAN);
    msg.append((byte)FVCMDnumrows);
    msg.append(id);

    sendReceive(msg);

    __int64 result;
    msg.read(result);
    return result;
}
Example #7
0
 const void * getResult()
 {
     CMessageBuffer mb;
     if (receiveMsg(mb, 0, mpTag)) {
         size32_t sz;
         mb.read(sz);
         if (sz) {
             CThorStreamDeserializerSource ds(sz,mb.readDirect(sz));
             RtlDynamicRowBuilder rowBuilder(queryRowAllocator());
             size32_t sz = queryRowDeserializer()->deserialize(rowBuilder,ds);
             return rowBuilder.finalizeRowClear(sz);
         }
     }
     return NULL;
 }
Example #8
0
void CLogMsgLinkToChild::connect()
{
    CMessageBuffer out;
    out.append('A').append(cid);
    try
    {
        queryWorldCommunicator().sendRecv(out, childNode, MPTAG_JLOG_CONNECT_TO_CHILD);
    }
    catch(IException * e)
    {
        e->Release();
    }
    out.read(pid);
    connected = true;
}
Example #9
0
const void *CPartialResultAggregator::getResult()
{
    CMessageBuffer mb;
    if (activity.receiveMsg(mb, 0, activity.queryMpTag()))
    {
        if (mb.length())
        {
            CThorStreamDeserializerSource ds(mb.length(), mb.readDirect(mb.length()));
            RtlDynamicRowBuilder rowBuilder(activity.queryRowAllocator());
            size32_t sz = activity.queryRowDeserializer()->deserialize(rowBuilder,ds);
            return rowBuilder.finalizeRowClear(sz);
        }
    }
    return NULL;
}
Example #10
0
 void sendHeaderLines(unsigned subFile, unsigned part)
 {
     if (0 == headerLinesRemaining[subFile])
     {
         if (sentHeaderLines->testSet(subFile))
             return;
         unsigned which = gotHeaderLines->scan(0, false);
         if (which == subFiles) // all received
         {
             bool someLeft=false;
             unsigned hL=0;
             for (; hL<subFiles; hL++)
             {
                 if (headerLinesRemaining[hL])
                 {
                     someLeft = true;
                     break;
                 }
             }
             if (!someLeft)
             {
                 sendAllDone();
                 return;
             }
         }
     }
     else
     {
         if (localLastPart[subFile] != part) // only ready to send if last local part
             return;
         if (sentHeaderLines->testSet(subFile))
             return;
     }
     CMessageBuffer msgMb;
     msgMb.append(subFile);
     msgMb.append(headerLinesRemaining[subFile]);
     // inform next slave about all subfiles I'm not dealing with.
     for (unsigned s=0; s<subFiles; s++)
     {
         if (NotFound == localLastPart[s])
         {
             sentHeaderLines->testSet(s);
             msgMb.append(s);
             msgMb.append(headerLinesRemaining[s]);
         }
     }
     queryJobChannel().queryJobComm().send(msgMb, queryJobChannel().queryMyRank()+1, mpTag);
 }
 void putNext(const void *prev)
 {
     if (nextPut) return;
     nextPut = true;
     if (global && !lastNode())
     {
         CMessageBuffer msg;
         msg.append(count);
         if (prev)
         {
             CMemoryRowSerializer msz(msg);
             ::queryRowSerializer(input)->serialize(msz, (const byte *)prev);
         }
         if (!queryJobChannel().queryJobComm().send(msg, queryJobChannel().queryMyRank()+1, mpTag)) // to next
             return;
     }
 }
Example #12
0
bool RemoteDataSource::getARow(MemoryBuffer & out, RowCache & cache, byte cmd, __int64 row)
{
    RowLocation location;

    if (cache.getCacheRow(row, location))
    {
        out.append(location.matchLength, location.matchRow);
        return true;
    }

    CMessageBuffer msg;
    msg.setEndian(__BIG_ENDIAN);
    msg.append(cmd);
    msg.append(id);
    msg.append(row);

    sendReceive(msg);

    bool ok;
    msg.read(ok);
    if (!ok) return false;

    __int64 start;
    msg.read(start);

    VariableRowBlock * next = new VariableRowBlock(msg, start);
    cache.addRowsOwn(next);

    if (!cache.getCacheRow(row, location))
        assertex(!"Internal Error!");
    out.append(location.matchLength, location.matchRow);
    return true;
}
Example #13
0
rowcount_t getCount(CActivityBase &activity, unsigned partialResults, rowcount_t limit, mptag_t mpTag)
{
    rowcount_t totalCount = 0;
    CMessageBuffer msg;
    while (partialResults--)
    {
        rank_t sender;
        msg.clear();
        if (!activity.receiveMsg(msg, RANK_ALL, mpTag, &sender)) return 0;
        if (activity.queryAbortSoon()) return 0;
        rowcount_t partialCount;
        msg.read(partialCount);
        totalCount += (rowcount_t)partialCount;
        if (totalCount > limit)
            break;
    }
    return totalCount;
}
Example #14
0
 void transferFrom(CMessageBuffer &mb)
 {
     // endian TBD
     swapWith(mb);
     tag = mb.tag;
     sender = mb.sender;
     replytag = mb.replytag;
     mb.clear();
 }
Example #15
0
static void sendReceive(INode * serverNode, CMessageBuffer & msg)
{
    if (!queryWorldCommunicator().sendRecv(msg, serverNode, MPTAG_FILEVIEW, TIMEOUT))
        throwError(FVERR_TimeoutRemoteFileView);

    msg.setEndian(__BIG_ENDIAN);
    IException * error = deserializeException(msg);
    if (error)
        throw error;
}
Example #16
0
const void *getAggregate(CActivityBase &activity, unsigned partialResults, IRowInterfaces &rowIf, IHThorCompoundAggregateExtra &aggHelper, mptag_t mpTag)
{
    // JCSMORE - pity this isn't common routine with similar one in aggregate, but helper is not common
    CThorRowArray slaveResults;
    slaveResults.ensure(partialResults);
    unsigned _partialResults = partialResults;
    while (_partialResults--)
    {
        CMessageBuffer mb;
        rank_t sender;
        if (!activity.receiveMsg(mb, RANK_ALL, mpTag, &sender)) return false;
        if (activity.queryAbortSoon()) return 0;
        if (mb.length())
        {
            CThorStreamDeserializerSource ds(mb.length(), mb.readDirect(mb.length()));
            RtlDynamicRowBuilder rowBuilder(rowIf.queryRowAllocator());
            size32_t sz = rowIf.queryRowDeserializer()->deserialize(rowBuilder, ds);
            slaveResults.setRow(sender-1, rowBuilder.finalizeRowClear(sz));
        }
    }
    RtlDynamicRowBuilder result(rowIf.queryRowAllocator(), false);
    size32_t sz;
    bool first = true;
    _partialResults = 0;
    for (;_partialResults<partialResults; _partialResults++)
    {
        const void *partialResult = slaveResults.item(_partialResults);
        if (partialResult)
        {
            if (first)
            {
                first = false;
                sz = cloneRow(result, slaveResults.item(_partialResults), rowIf.queryRowMetaData());
            }
            else
                sz = aggHelper.mergeAggregate(result, partialResult);
        }
    }
    if (first)
        sz = aggHelper.clearAggregate(result);
    return result.finalizeRowClear(sz);
}
Example #17
0
bool SortSlaveMP::sendRecv(CMessageBuffer &mb, unsigned timeout)
{
    if (!comm->sendRecv(mb,rank,tag,timeout))
        return false;
    byte ok = 255;
    if (mb.length()) {
        mb.read(ok);
        if (ok==1)
            return true;
        if (ok==0) {
            int err;
            mb.read(err);
            StringAttr errstr;
            mb.read(errstr);
            throw MakeStringException(err, "%s", errstr.get());
        }
    }
    throw MakeStringException(-1,"SortSlaveMP::sendRecv() protocol error %d",(int)ok);
    return false;
}
    void doBroadcast()
    {
        try
        {
            unsigned i = 0;
            unsigned n;
            if (1 == nodeindex)
                n = broadcastSlave-1;
            else if (broadcastSlave==nodeindex)
                n = 0;
            else
                n = nodeindex-1;
            loop {
                unsigned t = target(i++,n);
                if (t>numnodes)
                    break;
                if (t != broadcastSlave)
                {
#ifdef _TRACEBROADCAST
                    ActPrintLog(activity, "Broadcast node %d Sending to node %d size %d",nodeindex,t,broadcasting.length());
#endif
                    mptag_t rt = createReplyTag();
                    broadcasting.setReplyTag(rt); // simulate sendRecv
                    comm->send(broadcasting, t, mpTag);     
                    CMessageBuffer rMsg;
                    comm->recv(rMsg, t, rt);                    
#ifdef _TRACEBROADCAST
                    ActPrintLog(activity, "Broadcast node %d Sent to node %d size %d received back %d",nodeindex,t,broadcasting.length(),rMsg.length());
#endif
                }
            }
        }
        catch (IException *e)
        {
            ActPrintLog(activity, e, "CBroadcaster::broadcast exception");
            throw;
        }
#ifdef _TRACEBROADCAST
        ActPrintLog(activity, "do broadcast done done");
#endif
    }
Example #19
0
void UnregisterSelf()
{
    StringBuffer slfStr;
    slfEp.getUrlStr(slfStr);
    LOG(MCdebugProgress, thorJob, "Unregistering slave : %s", slfStr.toCharArray());
    try
    {
        CMessageBuffer msg;
        msg.append((int)rc_deregister);
        if (!queryWorldCommunicator().send(msg, masterNode, MPTAG_THORREGISTRATION, 60*1000))
        {
            LOG(MCerror, thorJob, "Failed to unregister slave : %s", slfStr.toCharArray());
            return;
        }
        LOG(MCdebugProgress, thorJob, "Unregistered slave : %s", slfStr.toCharArray());
    }
    catch (IException *e) {
        FLLOG(MCexception(e), thorJob, e,"slave unregistration error");
        e->Release();
    }
}
Example #20
0
    virtual void process()
    {
        CChooseSetsActivityMaster::process();

        IHThorChooseSetsArg *helper = (IHThorChooseSetsArg *)queryHelper();
        unsigned numSets = helper->getNumSets();
        unsigned nslaves = container.queryJob().querySlaves();

        MemoryBuffer countMb;
        rowcount_t *counts = (rowcount_t *)countMb.reserveTruncate((numSets*(nslaves+2)) * sizeof(rowcount_t));
        rowcount_t *totals = counts + nslaves*numSets;
        rowcount_t *tallies = totals + numSets;
        memset(counts, 0, countMb.length());

        unsigned s=nslaves;
        CMessageBuffer msg;
        while (s--)
        {
            msg.clear();
            rank_t sender;
            if (!receiveMsg(msg, RANK_ALL, mpTag, &sender))
                return;
            assertex(msg.length() == numSets*sizeof(rowcount_t));
            unsigned set = (unsigned)sender - 1;
            memcpy(&counts[set*numSets], msg.toByteArray(), numSets*sizeof(rowcount_t));
        }
        for (s=0; s<nslaves; s++)
        {
            unsigned i=0;
            for (; i<numSets; i++)
                totals[i] += counts[s * numSets + i];
        }
        msg.clear();
        msg.append(numSets*sizeof(rowcount_t), totals);
        unsigned endTotalsPos = msg.length();
        for (s=0; s<nslaves; s++)
        {
            msg.rewrite(endTotalsPos);
            msg.append(numSets*sizeof(rowcount_t), tallies);
            container.queryJob().queryJobComm().send(msg, s+1, mpTag);
            unsigned i=0;
            for (; i<numSets; i++)
                tallies[i] += counts[s * numSets + i];
        }
    }
Example #21
0
    void process()
    {
        processed = 0;

        input = inputs.item(0);
        startInput(input);

        processed = THORDATALINK_STARTED;

        OwnedConstThorRow row = input->ungroupedNextRow();
        CMessageBuffer mb;
        size32_t lenpos = mb.length(); // its 0 really
        mb.append((size32_t)0);
        if (row) {
            CMemoryRowSerializer msz(mb);
            ::queryRowSerializer(input)->serialize(msz,(const byte *)row.get());
            size32_t sz = mb.length()-lenpos-sizeof(size32_t);
            mb.writeDirect(lenpos,sizeof(size32_t),&sz);
            processed++;
        }
        container.queryJob().queryJobComm().send(mb, 0, masterMpTag);
    }
Example #22
0
    void process()
    {
        unsigned slaves = container.queryJob().querySlaves();
        IHThorLimitArg *helper = (IHThorLimitArg *)queryHelper();

        rowcount_t rowLimit = (rowcount_t)helper->getRowLimit();
        rowcount_t total = 0;
        while (slaves--)
        {
            CMessageBuffer mb;
            if (!receiveMsg(mb, RANK_ALL, mpTag, NULL))
                return;
            if (abortSoon)
                return;
            rowcount_t count;
            mb.read(count);
            total += count;
            if (total > rowLimit)
                break;
        }
        switch (container.getKind())
        {
            case TAKcreaterowlimit: 
            case TAKskiplimit: 
            {
                unsigned slaves = container.queryJob().querySlaves();
                CMessageBuffer mb;
                mb.append(total);
                queryJobChannel().queryJobComm().send(mb, RANK_ALL_OTHER, mpTag);
                break;
            }
            case TAKlimit:
            {
                if (total > rowLimit)
                    helper->onLimitExceeded();
                break;
            }
        }
    }
Example #23
0
 rowcount_t aggregateToLimit()
 {
     rowcount_t total = 0;
     ICommunicator &comm = queryJobChannel().queryJobComm();
     unsigned slaves = container.queryJob().querySlaves();
     unsigned s;
     for (s=0; s<slaves; s++)
     {
         CMessageBuffer msg;
         rank_t sender;
         if (!receiveMsg(msg, RANK_ALL, mpTag, &sender))
             return 0;
         if (abortSoon)
             return 0;
         rowcount_t count;
         msg.read(count);
         total += count;
         if (total > limit)
             break;
     }
     return total;
 }
    virtual void process()
    {
        CMasterActivity::process();

        IHThorDistributionArg * helper = (IHThorDistributionArg *)queryHelper();
        IOutputMetaData *rcSz = helper->queryInternalRecordSize();

        unsigned nslaves = container.queryJob().querySlaves();

        IDistributionTable * * result = (IDistributionTable * *)createThorRow(rcSz->getMinRecordSize()); // not a real row
        helper->clearAggregate(result);


        while (nslaves--)
        {
            rank_t sender;
            CMessageBuffer msg;
            if (!receiveMsg(msg, RANK_ALL, mpTag, &sender))
                return;
#if THOR_TRACE_LEVEL >= 5
            ActPrintLog("Received distribution result from node %d", (unsigned)sender);
#endif
            if (msg.length())
                helper->merge(result, msg);
        }

        StringBuffer tmp;
        tmp.append("<XML>");
        helper->gatherResult(result, tmp);
        tmp.append("</XML>");

#if THOR_TRACE_LEVEL >= 5
        ActPrintLog("Distribution result: %s", tmp.str());
#endif

        helper->sendResult(tmp.length(), tmp.str());

        destroyThorRow(result);
    }
    bool receive(MemoryBuffer &mb)
    {
#ifdef _TRACEBROADCAST
        ActPrintLog(activity, "Broadcast node %d Receiving on tag %d",nodeindex,(int)mpTag);
#endif
        CMessageBuffer msg;
        rank_t sender;
        BooleanOnOff onOff(receiving);
        if (comm->recv(msg, RANK_ALL, mpTag, &sender))
        {
#ifdef _TRACEBROADCAST
            ActPrintLog(activity, "Broadcast node %d Received %d from %d",nodeindex, msg.length(), sender);
#endif
            try
            {
                mb.swapWith(msg);
                msg.clear(); // send empty reply
#ifdef _TRACEBROADCAST
                ActPrintLog(activity, "Broadcast node %d reply to %d",nodeindex, sender);
#endif
                comm->reply(msg);
                if (aborted) 
                    return false;
#ifdef _TRACEBROADCAST
                ActPrintLog(activity, "Broadcast node %d Received %d",nodeindex, mb.length());
#endif
            }
            catch (IException *e)
            {
                ActPrintLog(activity, e, "CBroadcaster::recv(2): exception");
                throw;
            }
        }
#ifdef _TRACEBROADCAST
        ActPrintLog(activity, "receive done");
#endif
        return (0 != mb.length());
    }
Example #26
0
    virtual void process()
    {
        CMasterActivity::process();

        bool results = false;
        unsigned nslaves = container.queryJob().querySlaves();
        while (nslaves--)
        {
            CMessageBuffer mb;
            if (abortSoon || !receiveMsg(mb, RANK_ALL, replyTag, NULL)) break;
            StringBuffer str;
            mb.getSender().getUrlStr(str);
            size32_t sz;
            mb.read(sz);
            if (sz)
            {
                if (results)
                    throw MakeThorException(TE_UnexpectedMultipleSlaveResults, "Received greater than one result from slaves");
                IHThorRemoteResultArg *helper = (IHThorRemoteResultArg *)queryHelper();
                Owned<IThorRowInterfaces> resultRowIf = createRowInterfaces(helper->queryOutputMeta());
                CThorStreamDeserializerSource mds(sz, mb.readDirect(sz));
                RtlDynamicRowBuilder rowBuilder(resultRowIf->queryRowAllocator());
                size32_t sz = resultRowIf->queryRowDeserializer()->deserialize(rowBuilder, mds);
                OwnedConstThorRow result = rowBuilder.finalizeRowClear(sz);
                helper->sendResult(result);
                results = true;
            }
        }
        if (!results && !abortSoon)
        {
            ActPrintLog("WARNING: no results");
            IHThorRemoteResultArg *helper = (IHThorRemoteResultArg *)queryHelper();
            //helper->sendResult(NULL);
            // Jake I think this always cores (so raise exception instead)
            throw MakeThorException(TE_UnexpectedMultipleSlaveResults, "Received no results from slaves");

        }
    }
Example #27
0
    int run()
    {
        ICoven &coven=queryCoven();

        CMessageHandler<CSessionRequestServer> handler("CSessionRequestServer",this,&CSessionRequestServer::processMessage);
        stopped = false;
        CMessageBuffer mb;
        while (!stopped) {
            try {
                mb.clear();
                if (coven.recv(mb,RANK_ALL,MPTAG_DALI_SESSION_REQUEST,NULL))
                    handler.handleMessage(mb);
                else
                    stopped = true;
            }
            catch (IException *e)
            {
                EXCLOG(e, "CDaliPublisherServer");
                e->Release();
            }
        }
        return 0;
    }
Example #28
0
 int run()
 {
     ICoven &coven=queryCoven();
     CMessageBuffer mb;
     stopped = false;
     CMessageHandler<CDaliDiagnosticsServer> handler("CDaliDiagnosticsServer",this,&CDaliDiagnosticsServer::processMessage);
     while (!stopped) {
         try {
             mb.clear();
             if (coven.recv(mb,RANK_ALL,MPTAG_DALI_DIAGNOSTICS_REQUEST,NULL)) {
                 handler.handleMessage(mb);
             }   
             else
                 stopped = true;
         }
         catch (IException *e)
         {
             EXCLOG(e, "CDaliDiagnosticsServer");
             e->Release();
         }
     }
     return 0;
 }
Example #29
0
 bool sendLoopingCount(unsigned n, unsigned emptyIterations)
 {
     if (!container.queryLocalOrGrouped())
     {
         if (global || (lastMaxEmpty && (0 == emptyIterations)) || (!lastMaxEmpty && (emptyIterations>maxEmptyLoopIterations)) || ((0 == n) && (0 == emptyIterations)))
         {
             CMessageBuffer msg; // inform master starting
             msg.append(n);
             msg.append(emptyIterations);
             queryJobChannel().queryJobComm().send(msg, 0, mpTag);
             if (!global)
             {
                 lastMaxEmpty = emptyIterations>maxEmptyLoopIterations;
                 return true;
             }
             receiveMsg(msg, 0, mpTag);
             bool ok;
             msg.read(ok);
             return ok;
         }
     }
     return true;
 }
Example #30
0
 void main(CMessageBuffer &mb)
 {
     if (hasexceptionhandler)
         (parent->*handler)(mb);
     else {
         try {
             (parent->*handler)(mb);
         }
         catch (IException *e) {
             EXCLOG(e, name);
             e->Release();
         }
     }
     mb.resetBuffer();
 }