Пример #1
0
bool processPartitionCommand(ISocket * masterSocket, MemoryBuffer & msg, MemoryBuffer & results)
{
    FileFormat srcFormat;
    FileFormat tgtFormat;
    unsigned whichInput;
    RemoteFilename fullPath;
    offset_t totalSize;
    offset_t thisOffset;
    offset_t thisSize;
    unsigned thisHeaderSize;
    unsigned numParts;
    bool compressedInput = false;
    unsigned compatflags = 0;  

    srcFormat.deserialize(msg);
    tgtFormat.deserialize(msg);
    msg.read(whichInput);
    fullPath.deserialize(msg);
    msg.read(totalSize);
    msg.read(thisOffset);
    msg.read(thisSize);
    msg.read(thisHeaderSize);
    msg.read(numParts);
    if (msg.remaining())
        msg.read(compressedInput);
    if (msg.remaining())
        msg.read(compatflags); // not yet used
    StringAttr decryptkey;
    if (msg.remaining())
        msg.read(decryptkey);
    if (msg.remaining())
    {
        srcFormat.deserializeExtra(msg, 1);
        tgtFormat.deserializeExtra(msg, 1);
    }

    StringBuffer text;
    fullPath.getRemotePath(text);
    LOG(MCdebugProgress, unknownJob, "Process partition %d(%s)", whichInput, text.str());
    Owned<IFormatProcessor> processor = createFormatProcessor(srcFormat, tgtFormat, true);
    Owned<IOutputProcessor> target = createOutputProcessor(tgtFormat);
    processor->setTarget(target);

    processor->setPartitionRange(totalSize, thisOffset, thisSize, thisHeaderSize, numParts);
    processor->setSource(whichInput, fullPath, compressedInput, decryptkey);
    processor->calcPartitions(NULL);

    PartitionPointArray partition;
    processor->getResults(partition);

    serialize(partition, results);
    return true;
}
Пример #2
0
void TransferServer::deserializeAction(MemoryBuffer & msg, unsigned action)
{
    SocketEndpoint ep;
    ep.deserialize(msg);
    if (!ep.isLocal())
    {
        StringBuffer host, expected;
        queryHostIP().getIpText(host);
        ep.getIpText(expected);
        throwError2(DFTERR_WrongComputer, expected.str(), host.str());
    }

    srcFormat.deserialize(msg);
    tgtFormat.deserialize(msg);
    msg.read(calcInputCRC);
    msg.read(calcOutputCRC);
    deserialize(partition, msg);
    msg.read(numParallelSlaves);
    msg.read(updateFrequency);
    msg.read(replicate);
    msg.read(mirror);
    msg.read(isSafeMode);

    srand((unsigned)get_cycles_now());
    int adjust = (rand() * rand() * rand()) % updateFrequency - (updateFrequency/2);
    lastTick = msTick() + adjust;

    StringBuffer localFilename;
    if (action == FTactionpull)
    {
        partition.item(0).outputName.getPath(localFilename);
        LOG(MCdebugProgress, unknownJob, "Process Pull Command: %s", localFilename.str());
    }
    else
    {
        partition.item(0).inputName.getPath(localFilename);
        LOG(MCdebugProgress, unknownJob, "Process Push Command: %s", localFilename.str());
    }
    LOG(MCdebugProgress, unknownJob, "Num Parallel Slaves=%d Adjust=%d/%d", numParallelSlaves, adjust, updateFrequency);
    LOG(MCdebugProgress, unknownJob, "replicate(%d) mirror(%d) safe(%d) incrc(%d) outcrc(%d)", replicate, mirror, isSafeMode, calcInputCRC, calcOutputCRC);

    displayPartition(partition);

    unsigned numProgress;
    msg.read(numProgress);
    for (unsigned i = 0; i < numProgress; i++)
    {
        OutputProgress & next = *new OutputProgress;
        next.deserializeCore(msg);
        progress.append(next);
    }
    if (msg.remaining())
        msg.read(throttleNicSpeed);
    if (msg.remaining())
        msg.read(compressedInput).read(compressOutput);
    if (msg.remaining())
        msg.read(copyCompressed);
    if (msg.remaining())
        msg.read(transferBufferSize);
    if (msg.remaining()) 
        msg.read(encryptKey).read(decryptKey);
    if (msg.remaining())
    {
        srcFormat.deserializeExtra(msg, 1);
        tgtFormat.deserializeExtra(msg, 1);
    }

    ForEachItemIn(i1, progress)
        progress.item(i1).deserializeExtra(msg, 1);

    LOG(MCdebugProgress, unknownJob, "throttle(%d), transferBufferSize(%d)", throttleNicSpeed, transferBufferSize);
    PROGLOG("compressedInput(%d), compressedOutput(%d), copyCompressed(%d)", compressedInput?1:0, compressOutput?1:0, copyCompressed?1:0);
    PROGLOG("encrypt(%d), decrypt(%d)", encryptKey.isEmpty()?0:1, decryptKey.isEmpty()?0:1);

    //---Finished deserializing ---
    displayProgress(progress);

    totalLengthRead = 0;
    totalLengthToRead = 0;
    ForEachItemIn(idx, partition)
        totalLengthToRead += partition.item(idx).inputLength;
}
Пример #3
0
    void deserialize(MemoryBuffer &mb)
    {
        // clear optional
#ifndef NO_XLS
        xslproc.clear();
        xsltrans.clear();
        xslt.clear();
#endif
        resultoverflow = false;
        wuservices = false;
        unsigned version;
        mb.read(version);
        unsigned v;
        mb.read(v);
        action = (SashaCommandAction)v;
        unsigned n;
        mb.read(n);
        while (n--) {
            StringAttrItem *s = new StringAttrItem;
            mb.read(s->text);
            ids.append(*s);
        }
        mb.read(after);
        mb.read(before);
        mb.read(state);
        mb.read(owner);
        mb.read(cluster);
        mb.read(jobname);
        mb.read(outputformat);
        mb.read(online);
        mb.read(archived);
        mb.read(dfu);
        mb.read(start);
        mb.read(limit);
        if (mb.remaining()>sizeof(unsigned)) { // new format (trailing WUs)
            mb.read(resultoverflow);
            mb.read(n);
            while (n--) {
                StringAttrItem *s = new StringAttrItem;
                mb.read(s->text);
                results.append(*s);
            }
            bool isxslt;
            mb.read(isxslt);
            if (isxslt)
                mb.read(xslt);
            if (mb.remaining()>sizeof(bool)) { // new WorkUnitServices format (trailing WUs)
                mb.read(wuservices);
                if (wuservices) {
                    mb.read(priority);
                    mb.read(fileread);
                    mb.read(filewritten);
                    mb.read(roxiecluster);
                    mb.read(eclcontains);
                }
                if (mb.remaining()>=sizeof(numdts)) { // new listdts info
                    mb.read(numdts);
                    free(dts);
                    dts = NULL;
                    if (numdts) {
                        dts = (CDateTime *)calloc(numdts,sizeof(CDateTime));
                        for (unsigned i=0;i<numdts;i++)
                            dts[i].deserialize(mb);
                    }
                }
            }
        }
    }
Пример #4
0
size32_t ThorExpand(MemoryBuffer & src, MemoryBuffer & dest)
{
    size32_t len = src.remaining();
    const void *pSrc = src.readDirect(len);
    return ThorExpand(pSrc, len, dest);
}