示例#1
0
IFvDataSource * createRemoteFileDataSource(const SocketEndpoint & server, const char * username, const char * password, const char * logicalName)
{
    Owned<INode> serverNode = createINode(server);

    CMessageBuffer msg;
    msg.setEndian(__BIG_ENDIAN);
    msg.append((byte)FVCMDcreatefile);
    msg.append(myProcessSession());
    msg.append(username);
    msg.append(password);
    msg.append(logicalName);

    sendReceive(serverNode, msg);

    unsigned short version;
    unique_id_t id;
    __int64 numRows;
    bool isIndex;
    msg.read(version);
    msg.read(id);
    msg.read(numRows);
    Owned<IFvDataSourceMetaData> meta = deserializeDataSourceMeta(msg);
    msg.read(isIndex);

    if (id)
        return new RemoteDataSource(server, id, meta, numRows, isIndex);
    return 0;
}
示例#2
0
void SortSlaveMP::StartMiniSort(rowcount_t _totalrows) /* async */
{
    CMessageBuffer mb;
    mb.append((byte)FN_StartMiniSort);
    mb.append(_totalrows);
    sendRecv(mb);
}
示例#3
0
 void sendRemainingHeaderLines()
 {
     if (!headerLines)
         return;
     unsigned which = sentHeaderLines->scanInvert(0, false);
     if (which < subFiles)
     {
         CMessageBuffer msgMb;
         bool someLeft=false;
         do
         {
             msgMb.append(which);
             unsigned &remaining = getHeaderLines(which);
             if (0 != remaining)
                 someLeft = true;
             msgMb.append(remaining);
             which = sentHeaderLines->scanInvert(which+1, false);
         }
         while (which < subFiles);
         if (someLeft)
             queryJobChannel().queryJobComm().send(msgMb, queryJobChannel().queryMyRank()+1, mpTag);
         else
             sendAllDone();
     }
 }
示例#4
0
bool RemoteDataSource::getARow(MemoryBuffer & out, RowCache & cache, byte cmd, __int64 row)
{
    RowLocation location;

    if (cache.getCacheRow(row, location))
    {
        out.append(location.matchLength, location.matchRow);
        return true;
    }

    CMessageBuffer msg;
    msg.setEndian(__BIG_ENDIAN);
    msg.append(cmd);
    msg.append(id);
    msg.append(row);

    sendReceive(msg);

    bool ok;
    msg.read(ok);
    if (!ok) return false;

    __int64 start;
    msg.read(start);

    VariableRowBlock * next = new VariableRowBlock(msg, start);
    cache.addRowsOwn(next);

    if (!cache.getCacheRow(row, location))
        assertex(!"Internal Error!");
    out.append(location.matchLength, location.matchRow);
    return true;
}
示例#5
0
void SortSlaveMP::MultiBinChopStop(unsigned num,rowcount_t *pos)
{
    CMessageBuffer mb;
    mb.append((byte)FN_MultiBinChopStop);
    mb.append(num);
    sendRecv(mb);
    mb.read(num*sizeof(rowcount_t),pos);
}
示例#6
0
void SortSlaveMP::GetGatherInfo(rowcount_t &numlocal, offset_t &totalsize, unsigned &overflowscale, bool hasserializer)
{
    CMessageBuffer mb;
    mb.append((byte)FN_GetGatherInfo);
    mb.append(hasserializer);
    sendRecv(mb);
    mb.read(numlocal).read(totalsize).read(overflowscale);
}
示例#7
0
void SortSlaveMP::GetMultiNthRow(unsigned numsplits,size32_t &mkeybuffsize, void * &mkeybuf)
{
    CMessageBuffer mb;
    mb.append((byte)FN_GetMultiNthRow);
    mb.append(numsplits);
    sendRecv(mb);
    deserializeblk(mb,mkeybuffsize,mkeybuf);

}
示例#8
0
void RemoteDataSource::beforeDispose()
{
    CMessageBuffer msg;
    msg.setEndian(__BIG_ENDIAN);
    msg.append((byte)FVCMDdestroy);
    msg.append(id);

    sendReceive(msg);
}
示例#9
0
void SortSlaveMP::OverflowAdjustMapStart(unsigned mapsize,rowcount_t *map,size32_t keybuffsize,const byte *keybuff, byte cmpfn,bool useaux) /* async */
{
    CMessageBuffer mb;
    mb.append((byte)FN_OverflowAdjustMapStart);
    mb.append(mapsize).append(mapsize*sizeof(rowcount_t),map);
    serializeblk(mb,keybuffsize,keybuff).append(cmpfn).append(useaux);
    sendRecv(mb);

}
示例#10
0
rowcount_t SortSlaveMP::OverflowAdjustMapStop( unsigned mapsize, rowcount_t *map)
{
    CMessageBuffer mb;
    mb.append((byte)FN_OverflowAdjustMapStop);
    mb.append(mapsize);
    sendRecv(mb);
    rowcount_t ret;
    mb.read(ret).read(mapsize*sizeof(rowcount_t),map);
    return ret;
}
示例#11
0
bool SortSlaveMP::Connect(unsigned _part, unsigned _numnodes)
{
    CMessageBuffer mb;
    mb.append((byte)FN_Connect);
    mb.append(_part).append(_numnodes);
    sendRecv(mb);
    bool ret;
    mb.read(ret);
    return ret;
}
示例#12
0
bool SortSlaveMP::FirstRowOfFile(const char *filename,size32_t &rowbuffsize, byte * &rowbuf)
{
    CMessageBuffer mb;
    mb.append((byte)FN_FirstRowOfFile);
    mb.append(filename);
    sendRecv(mb);
    deserializeblk(mb,rowbuffsize,rowbuf);
    bool ret;
    mb.read(ret);
    return ret;
}
示例#13
0
void SortSlaveMP::MultiMerge(unsigned mapsize,rowcount_t *map,unsigned num,SocketEndpoint* endpoints) /* async */
{
    CMessageBuffer mb;
    mb.append((byte)FN_MultiMerge);
    mb.append(mapsize).append(mapsize*sizeof(rowcount_t),map);
    mb.append(num);
    while (num--) {
        endpoints->serialize(mb);
        endpoints++;
    }
    sendRecv(mb);
}
示例#14
0
void CDiskWriteSlaveActivityBase::close()
{
    try
    {
        if (out) {
            uncompressedBytesWritten = out->getPosition();
            if (calcFileCrc) {
                if (diskHelperBase->getFlags() & TDWextend) {
                    assertex(!"TBD need to merge CRC");
                }   
                else
                    out->flush(&fileCRC);
            }
            else if (!abortSoon)
                out->flush();
            out.clear();
        }
        else if (outraw) {
            outraw->flush();
            uncompressedBytesWritten = outraw->tell();
            outraw.clear();
        }

        {
            CriticalBlock block(statsCs);
            mergeStats(fileStats, outputIO);
            outputIO.clear();
        }

        if (!rfsQueryParallel && dlfn.isExternal() && !lastNode())
        {
            rowcount_t rows = processed & THORDATALINK_COUNT_MASK;
            ActPrintLog("External write done, signalling next (row count = %" RCPF "d)", rows);
            CMessageBuffer msg;
            msg.append(rows);
            msg.append(tempExternalName);
            queryJobChannel().queryJobComm().send(msg, queryJobChannel().queryMyRank()+1, mpTag);
        }
    }
    catch (IException *e)
    { 
        ActPrintLogEx(&queryContainer(), e, thorlog_null, MCwarning, "Error closing file: %s", fName.get());
        abortSoon = true;
        removeFiles();
        throw e;
    }
    if (abortSoon)
        removeFiles();
}
示例#15
0
__int64 RemoteDataSource::numRows(bool force)
{
    if (!force)
        return cachedNumRows;
    CMessageBuffer msg;
    msg.setEndian(__BIG_ENDIAN);
    msg.append((byte)FVCMDnumrows);
    msg.append(id);

    sendReceive(msg);

    __int64 result;
    msg.read(result);
    return result;
}
示例#16
0
 void sendHeaderLines(unsigned subFile, unsigned part)
 {
     if (0 == headerLinesRemaining[subFile])
     {
         if (sentHeaderLines->testSet(subFile))
             return;
         unsigned which = gotHeaderLines->scan(0, false);
         if (which == subFiles) // all received
         {
             bool someLeft=false;
             unsigned hL=0;
             for (; hL<subFiles; hL++)
             {
                 if (headerLinesRemaining[hL])
                 {
                     someLeft = true;
                     break;
                 }
             }
             if (!someLeft)
             {
                 sendAllDone();
                 return;
             }
         }
     }
     else
     {
         if (localLastPart[subFile] != part) // only ready to send if last local part
             return;
         if (sentHeaderLines->testSet(subFile))
             return;
     }
     CMessageBuffer msgMb;
     msgMb.append(subFile);
     msgMb.append(headerLinesRemaining[subFile]);
     // inform next slave about all subfiles I'm not dealing with.
     for (unsigned s=0; s<subFiles; s++)
     {
         if (NotFound == localLastPart[s])
         {
             sentHeaderLines->testSet(s);
             msgMb.append(s);
             msgMb.append(headerLinesRemaining[s]);
         }
     }
     queryJobChannel().queryJobComm().send(msgMb, queryJobChannel().queryMyRank()+1, mpTag);
 }
示例#17
0
int LogMsgLogReceiverThread::run()
{
    while(!done)
    {
        try
        {
            if(queryWorldCommunicator().recv(in, childNode, MPTAG_JLOG_CHILD_TO_PARENT))
            {
                msgBuffer.deserialize(in, true);
                if(isListener)
                    listener->report(msgBuffer);
                else
                    queryLogMsgManager()->report(msgBuffer);
            }
        }
        catch(IException * e)
        {
            done = true;
            CMessageBuffer out;
            out.append('D').append(cid);
            try
            {
                queryWorldCommunicator().send(out, queryMyNode(), MPTAG_JLOG_CONNECT_TO_PARENT, MP_ASYNC_SEND);
            }
            catch(IException * ee)
            {
                ee->Release();
            }
            e->Release();
        }
    }
    return 0;
}
示例#18
0
void sendPartialCount(CSlaveActivity &activity, rowcount_t partialCount)
{
    CMessageBuffer msg;
    msg.append(partialCount);
    if (!activity.queryContainer().queryJob().queryJobComm().send(msg, 0, activity.queryMpTag(), 5000))
        throw MakeThorException(0, "Failed to give partial result to master");
}
示例#19
0
bool UnregisterSelf(IException *e)
{
    if (!hasMPServerStarted())
        return false;

    StringBuffer slfStr;
    slfEp.getUrlStr(slfStr);
    LOG(MCdebugProgress, thorJob, "Unregistering slave : %s", slfStr.str());
    try
    {
        CMessageBuffer msg;
        msg.append((int)rc_deregister);
        serializeException(e, msg); // NB: allows exception to be NULL
        if (!queryWorldCommunicator().send(msg, masterNode, MPTAG_THORREGISTRATION, 60*1000))
        {
            LOG(MCerror, thorJob, "Failed to unregister slave : %s", slfStr.str());
            return false;
        }
        LOG(MCdebugProgress, thorJob, "Unregistered slave : %s", slfStr.str());
        return true;
    }
    catch (IException *e) {
        if (!jobListenerStopped)
            FLLOG(MCexception(e), thorJob, e,"slave unregistration error");
        e->Release();
    }
    return false;
}
示例#20
0
    void notify(MemoryBuffer &returndata)   // if returns false should unsubscribe
    {
        if (hasaborted) {
            throw MakeStringException(-1,"Subscription notification aborted");
            return;
        }
        size32_t dlen = returndata.length();
        CMessageBuffer mb;
        mb.append(tag).append(sid).append(dlen).append(returndata);
        try {
            if (!queryWorldCommunicator().send(mb,dst,MPTAG_DALI_SUBSCRIPTION_FULFILL,1000*60*3))  {
                // Must reply in 3 Minutes
                // Kludge to avoid locking SDS on blocked client
                hasaborted = true;
                StringBuffer tmp;
                throw MakeStringException(-1,"Subscription notification to %s timed out",dst->endpoint().getUrlStr(tmp).str());
                return;
            }

        }
        catch (IMP_Exception *e) {
            PrintExceptionLog(e,"Dali CSubscriptionStub");

            hasaborted = true;
            throw;
        }
    }
示例#21
0
void SortSlaveMP::GetMultiMidPointStop(size32_t &mkeybuffsize, void * &mkeybuf)
{
    CMessageBuffer mb;
    mb.append((byte)FN_GetMultiMidPointStop);
    sendRecv(mb);
    deserializeblk(mb,mkeybuffsize,mkeybuf);
}
示例#22
0
void SortSlaveMP::MultiBinChopStart(size32_t keybuffsize,const byte *keybuff, byte cmpfn) /* async */
{
    CMessageBuffer mb;
    mb.append((byte)FN_MultiBinChopStart);
    serializeblk(mb,keybuffsize,keybuff).append(cmpfn);
    sendRecv(mb);
}
示例#23
0
    void process()
    {
        ActPrintLog("ReDistributeActivityMaster::process");
        HashDistributeMasterBase::process();        
        IHThorHashDistributeArg *helper = (IHThorHashDistributeArg *)queryHelper(); 
        unsigned n = container.queryJob().querySlaves();
        MemoryAttr ma;
        offset_t *sizes = (offset_t *)ma.allocate(sizeof(offset_t)*n);
        unsigned i;
        try {
            for (i=0;i<n;i++) {
                if (abortSoon)
                    return;
                CMessageBuffer mb;
#ifdef _TRACE
                ActPrintLog("ReDistribute process, Receiving on tag %d",statstag);
#endif
                rank_t sender;
                if (!receiveMsg(mb, RANK_ALL, statstag, &sender)||abortSoon) 
                    return;
#ifdef _TRACE
                ActPrintLog("ReDistribute process, Received size from %d",sender);
#endif
                sender--;
                assertex((unsigned)sender<n);
                mb.read(sizes[sender]);
            }
            ActPrintLog("ReDistributeActivityMaster::process sizes got");
            for (i=0;i<n;i++) {
                CMessageBuffer mb;
                mb.append(n*sizeof(offset_t),sizes);
#ifdef _TRACE
                ActPrintLog("ReDistribute process, Replying to node %d tag %d",i+1,statstag);
#endif
                if (!container.queryJob().queryJobComm().send(mb, (rank_t)i+1, statstag))
                    return;
            }
            // check if any max skew broken
            double maxskew = helper->getTargetSkew();
            if (maxskew>helper->getSkew()) {
                offset_t tot = 0;
                for (i=0;i<n;i++) 
                    tot += sizes[i];
                offset_t avg = tot/n;
                for (i=0;i<n;i++) {
                    double r = ((double)sizes[i]-(double)avg)/(double)avg;
                    if ((r>=maxskew)||(-r>maxskew)) {
                        throw MakeActivityException(this, TE_DistributeFailedSkewExceeded, "DISTRIBUTE maximum skew exceeded (node %d has %"I64F"d, average is %"I64F"d)",i+1,sizes[i],avg);
                    }               
                }
            }

        }
        catch (IException *e) {
            ActPrintLog(e,"ReDistribute");
            throw;
        }
        ActPrintLog("ReDistributeActivityMaster::process exit");
    }
示例#24
0
 void sendResult(rowcount_t r)
 {
     if (resultSent) return;
     resultSent = true;
     CMessageBuffer mb;
     mb.append(r);
     container.queryJob().queryJobComm().send(mb, 0, mpTag);
 }
示例#25
0
MemoryBuffer & getDaliDiagnosticValue(MemoryBuffer &m)
{
    CMessageBuffer mb;
    mb.append((int)MDR_GET_VALUE).append(m);
    queryCoven().sendRecv(mb,RANK_RANDOM,MPTAG_DALI_DIAGNOSTICS_REQUEST);
    m.swapWith(mb);
    return m;
}
示例#26
0
void SortSlaveMP::GetMultiMidPointStart(size32_t lkeybuffsize, const void * lkeybuff, size32_t hkeybuffsize, const void * hkeybuff) /* async */
{
    CMessageBuffer mb;
    mb.append((byte)FN_GetMultiMidPointStart);
    serializeblk(mb,lkeybuffsize,lkeybuff);
    serializeblk(mb,hkeybuffsize,hkeybuff);
    sendRecv(mb);
}
示例#27
0
void SortSlaveMP::MultiBinChop(size32_t keybuffsize,const byte *keybuff, unsigned num,rowcount_t *pos,byte cmpfn)
{
    CMessageBuffer mb;
    mb.append((byte)FN_MultiBinChop);
    serializeblk(mb,keybuffsize,keybuff).append(num).append(cmpfn);
    sendRecv(mb);
    mb.read(num*sizeof(rowcount_t),pos);
}
 virtual void onInputFinished(rowcount_t count) override
 {
     if (container.queryLocalOrGrouped())
         return;
     CMessageBuffer msg;
     msg.append(numSets*sizeof(rowcount_t), counts);
     queryJobChannel().queryJobComm().send(msg, 0, mpTag);
 }
示例#29
0
    virtual void process()
    {
        CChooseSetsActivityMaster::process();

        IHThorChooseSetsArg *helper = (IHThorChooseSetsArg *)queryHelper();
        unsigned numSets = helper->getNumSets();
        unsigned nslaves = container.queryJob().querySlaves();

        MemoryBuffer countMb;
        rowcount_t *counts = (rowcount_t *)countMb.reserveTruncate((numSets*(nslaves+2)) * sizeof(rowcount_t));
        rowcount_t *totals = counts + nslaves*numSets;
        rowcount_t *tallies = totals + numSets;
        memset(counts, 0, countMb.length());

        unsigned s=nslaves;
        CMessageBuffer msg;
        while (s--)
        {
            msg.clear();
            rank_t sender;
            if (!receiveMsg(msg, RANK_ALL, mpTag, &sender))
                return;
            assertex(msg.length() == numSets*sizeof(rowcount_t));
            unsigned set = (unsigned)sender - 1;
            memcpy(&counts[set*numSets], msg.toByteArray(), numSets*sizeof(rowcount_t));
        }
        for (s=0; s<nslaves; s++)
        {
            unsigned i=0;
            for (; i<numSets; i++)
                totals[i] += counts[s * numSets + i];
        }
        msg.clear();
        msg.append(numSets*sizeof(rowcount_t), totals);
        unsigned endTotalsPos = msg.length();
        for (s=0; s<nslaves; s++)
        {
            msg.rewrite(endTotalsPos);
            msg.append(numSets*sizeof(rowcount_t), tallies);
            container.queryJob().queryJobComm().send(msg, s+1, mpTag);
            unsigned i=0;
            for (; i<numSets; i++)
                tallies[i] += counts[s * numSets + i];
        }
    }
示例#30
0
bool RemoteDataSource::fetchRawRow(MemoryBuffer & out, __int64 offset)
{
    CMessageBuffer msg;
    msg.setEndian(__BIG_ENDIAN);
    msg.append(FVCMDfetchraw);
    msg.append(id);
    msg.append(offset);

    sendReceive(msg);

    bool ok;
    msg.read(ok);
    if (!ok) return false;
    size32_t len;
    msg.read(len);
    out.append(len, msg.readDirect(len));
    return true;
}