Ejemplo n.º 1
0
    void slaveDone(size32_t slaveIdx, MemoryBuffer &mb)
    {
        if (mb.length()) // if 0 implies aborted out from this slave.
        {
            rowcount_t slaveProcessed;
            mb.read(slaveProcessed);
            recordsProcessed += slaveProcessed;

            offset_t size, physicalSize;
            mb.read(size);
            mb.read(physicalSize);

            unsigned fileCrc;
            mb.read(fileCrc);

            CDateTime modifiedTime(mb);
            StringBuffer timeStr;
            modifiedTime.getString(timeStr);

            IPartDescriptor *partDesc = fileDesc->queryPart(slaveIdx);
            IPropertyTree &props = partDesc->queryProperties();
            props.setPropInt64("@size", size);
            if (fileDesc->isCompressed())
                props.setPropInt64("@compressedSize", physicalSize);
            props.setPropInt("@fileCrc", fileCrc);
            props.setProp("@modified", timeStr.str());
        }
    }
Ejemplo n.º 2
0
// IThorSlaveActivity
void CDiskReadSlaveActivityBase::init(MemoryBuffer &data, MemoryBuffer &slaveData)
{
    subfileLogicalFilenames.kill();
    partDescs.kill();
    data.read(logicalFilename);
    unsigned subfiles;
    data.read(subfiles);
    unsigned s=0;
    for (; s<subfiles; s++)
    {
        StringAttr subfile;
        data.read(subfile);
        subfileLogicalFilenames.append(subfile);
    }
    unsigned parts;
    data.read(parts);
    if (parts)
    {
        deserializePartFileDescriptors(data, partDescs);

        // put temp files in individual slave temp dirs (incl port)
        if ((helper->getFlags() & TDXtemporary) && (!container.queryJob().queryUseCheckpoints()))
            partDescs.item(0).queryOwner().setDefaultDir(queryTempDir());
    }
}
Ejemplo n.º 3
0
void CDiskWriteSlaveActivityBase::init(MemoryBuffer &data, MemoryBuffer &slaveData)
{
    StringAttr logicalFilename;
    data.read(logicalFilename);
    dlfn.set(logicalFilename);
    if (diskHelperBase->getFlags() & TDXtemporary)
        data.read(usageCount);
    if (diskHelperBase->getFlags() & TDWextend)
    {
        assertex(!"TBD extended CRC broken");
        unsigned crc;
        if (partDesc->getCrc(crc))
            fileCRC.reset(~crc);
    }
    partDesc.setown(deserializePartFileDescriptor(data));

    // put temp files in individual slave temp dirs (incl port)
    if ((diskHelperBase->getFlags() & TDXtemporary) && (!container.queryJob().queryUseCheckpoints()))
        partDesc->queryOwner().setDefaultDir(queryTempDir());

    if (dlfn.isExternal())
    {
        mpTag = container.queryJobChannel().deserializeMPTag(data);
        if (dlfn.isQuery() && (0 != container.queryJob().getWorkUnitValueInt("rfsqueryparallel", 0)))
            rfsQueryParallel = true;
    }
    if (0 != (diskHelperBase->getFlags() & TDXgrouped))
        grouped = true;
}
Ejemplo n.º 4
0
void CDiskWriteSlaveActivityBase::init(MemoryBuffer &data, MemoryBuffer &slaveData)
{
    diskHelperBase = static_cast <IHThorDiskWriteArg *> (queryHelper());

    data.read(logicalFilename);
    dlfn.set(logicalFilename);
    if (diskHelperBase->getFlags() & TDXtemporary)
        data.read(usageCount);
    if (diskHelperBase->getFlags() & TDWextend)
    {
        assertex(!"TBD extended CRC broken");
        unsigned crc;
        if (partDesc->getCrc(crc))
            fileCRC.reset(~crc);
    }
    partDesc.setown(deserializePartFileDescriptor(data));
    if (dlfn.isExternal())
    {
        mpTag = container.queryJob().deserializeMPTag(data);
        if (dlfn.isQuery() && (0 != container.queryJob().getWorkUnitValueInt("rfsqueryparallel", 0)))
            rfsQueryParallel = true;
    }
    if (0 != (diskHelperBase->getFlags() & TDXgrouped))
        grouped = true;
    compress = partDesc->queryOwner().isCompressed();
    void *ekey;
    size32_t ekeylen;
    diskHelperBase->getEncryptKey(ekeylen,ekey);
    if (ekeylen!=0) {
        ecomp.setown(createAESCompressor256(ekeylen,ekey));
        memset(ekey,0,ekeylen);
        free(ekey);
        compress = true;
    }
}
Ejemplo n.º 5
0
void FileFormat::deserialize(MemoryBuffer & in)
{
    byte tempType;

    in.read(tempType); type = (FileFormatType)tempType;
    switch (type)
    {
    case FFTfixed:
    case FFTblocked:
        in.read(recordSize);
        break;
    case FFTcsv:
    case FFTutf:
    case FFTutf8: case FFTutf8n:
    case FFTutf16: case FFTutf16be: case FFTutf16le:
    case FFTutf32: case FFTutf32be: case FFTutf32le:
        in.read(maxRecordSize);
        ::deserialize(in, separate);
        ::deserialize(in, quote);
        ::deserialize(in, terminate);
        ::deserialize(in, escape);
        ::deserialize(in, rowTag);
        break;
    }
}
Ejemplo n.º 6
0
void RemoteDataSourceServer::doCmdCreateWorkunit(MemoryBuffer & in, MemoryBuffer & out)
{
    SessionId session;
    StringAttr wuid, username, password;
    unsigned sequence;
    StringAttr name;

    in.read(session);
    in.read(username).read(password);
    in.read(wuid);
    in.read(sequence);
    in.read(name);

    DBGLOG("RemoteFileView:CreateWorkunit('%s',%d,'%s') by[%s:%" I64F "d", wuid.get(), sequence, name ? name.get() : "", username.get(), session);
    Owned<IConstWUResult> wuResult = resolveResult(wuid, sequence, name);
    Owned<IFvDataSource> ds = createDataSource(wuResult, wuid, username, password);
    unique_id_t id = addDataSource(session, ds);

    out.append((unsigned short)CurRemoteVersion);
    out.append(id);
    out.append(ds->numRows(false));
    ds->queryMetaData()->serialize(out);
    out.append(ds->isIndex());

    DBGLOG("RemoteFileView:CreateWorkunit returns %" I64F "d", id);
}
Ejemplo n.º 7
0
DataSourceMetaItem::DataSourceMetaItem(unsigned _flags, MemoryBuffer & in)
{
    flags = _flags;
    in.read(name);
    in.read(xpath);
    type.setown(deserializeType(in));
}
Ejemplo n.º 8
0
void FileFormat::deserialize(MemoryBuffer & in)
{
    byte tempType;

    in.read(tempType); type = (FileFormatType)tempType;
    switch (type)
    {
    case FFTfixed:
    case FFTblocked:
        in.read(recordSize);
        break;
    case FFTcsv:
    case FFTutf:
    case FFTutf8: case FFTutf8n:
    case FFTutf16: case FFTutf16be: case FFTutf16le:
    case FFTutf32: case FFTutf32be: case FFTutf32le:
        in.read(maxRecordSize);
        ::deserialize(in, separate);
        ::deserialize(in, quote);
        ::deserialize(in, terminate);
        ::deserialize(in, rowTag);
        updateMarkupType(rowTag, NULL); //neither kind nor markup currently serialized.  may add later
        break;
    }
}
Ejemplo n.º 9
0
// IThorSlaveActivity
void CDiskReadSlaveActivityBase::init(MemoryBuffer &data, MemoryBuffer &slaveData)
{
    subfileLogicalFilenames.kill();
    partDescs.kill();
    data.read(logicalFilename);
    unsigned subfiles;
    data.read(subfiles);
    unsigned s=0;
    for (; s<subfiles; s++)
    {
        StringAttr subfile;
        data.read(subfile);
        subfileLogicalFilenames.append(subfile);
    }
    unsigned parts;
    data.read(parts);
    if (parts)
    {
        deserializePartFileDescriptors(data, partDescs);
        unsigned encryptedKeyLen;
        void *encryptedKey;
        helper->getEncryptKey(encryptedKeyLen, encryptedKey);
        if (0 != encryptedKeyLen) 
        {
            bool dfsEncrypted = partDescs.item(0).queryOwner().queryProperties().getPropBool("@encrypted");
            if (dfsEncrypted) // otherwise ignore (warning issued by master)
                eexp.setown(createAESExpander256(encryptedKeyLen, encryptedKey));
            memset(encryptedKey, 0, encryptedKeyLen);
            free(encryptedKey);
        }
    }
}
Ejemplo n.º 10
0
    virtual void slaveDone(size32_t slaveIdx, MemoryBuffer &mb)
    {
        if (mb.length()) // if 0 implies aborted out from this slave.
        {
            offset_t size;
            mb.read(size);
            CDateTime modifiedTime(mb);

            IPartDescriptor *partDesc = newIndexDesc->queryPart(slaveIdx);
            IPropertyTree &props = partDesc->queryProperties();
            props.setPropInt64("@size", size);
            StringBuffer timeStr;
            modifiedTime.getString(timeStr);
            props.setProp("@modified", timeStr.str());
            if (!local && 0 == slaveIdx)
            {
                mb.read(size);
                CDateTime modifiedTime(mb);
                IPartDescriptor *partDesc = newIndexDesc->queryPart(newIndexDesc->numParts()-1);
                IPropertyTree &props = partDesc->queryProperties();
                props.setPropInt64("@size", size);
                StringBuffer timeStr;
                modifiedTime.getString(timeStr);
                props.setProp("@modified", timeStr.str());
            }
        }
    }
Ejemplo n.º 11
0
DataSourceMetaItem::DataSourceMetaItem(unsigned _flags, MemoryBuffer & in)
{
    flags = _flags;
    in.read(name);
    in.read(xpath);
    type.setown(deserializeType(in));
    splitXmlTagNamesFromXPath(xpath.get(), tagname);
}
Ejemplo n.º 12
0
DataSourceMetaData::DataSourceMetaData(MemoryBuffer & buffer)
{
    numVirtualFields = 0;
    buffer.read(numFieldsToIgnore);
    buffer.read(randomIsOk);
    buffer.read(hasMixedContent);
    buffer.read(isStoredFixedWidth);
    buffer.read(minRecordSize);
    buffer.read(keyedSize);
    buffer.read(maxRecordSize);

    unsigned numFields;
    buffer.read(numFields);
    for (unsigned idx=0; idx < numFields; idx++)
    {
        byte flags;
        buffer.read(flags);
        if (flags == FVFFdataset)
            fields.append(*new DataSourceDatasetItem(flags, buffer));
        else if (flags == FVFFset)
            fields.append(*new DataSourceSetItem(flags, buffer));
        else
            fields.append(*new DataSourceMetaItem(flags, buffer));
        if (flags == FVFFvirtual)
            ++numVirtualFields;
    }
    gatherAttributes();
}
Ejemplo n.º 13
0
 virtual void deserializeCreateContext(MemoryBuffer &mb)
 {
     CSlaveGraphElement::deserializeCreateContext(mb);
     if (TAKworkunitread == kind)
     {
         mb.read(wuidread2diskread); // have I been converted
         if (wuidread2diskread)
             mb.read(wuidreadFilename);
     }
     haveCreateCtx = true;
 }
Ejemplo n.º 14
0
VariableRowBlock::VariableRowBlock(MemoryBuffer & inBuffer, __int64 _start) : RowBlock(_start, 0)
{
    inBuffer.read(numRows);
    for (unsigned row = 0; row < numRows; row++)
    {
        unsigned thisLength;
        rowIndex.append(buffer.length());
        inBuffer.read(thisLength);
        buffer.append(thisLength, inBuffer.readDirect(thisLength));
    }
    rowIndex.append(buffer.length());
}
Ejemplo n.º 15
0
void PartitionPoint::deserialize(MemoryBuffer & in)
{
    inputName.deserialize(in);
    outputName.deserialize(in);

    in.read(whichInput);
    in.read(inputOffset);
    in.read(inputLength);
    in.read(whichOutput);
    in.read(outputOffset);
    in.read(outputLength);
    in.read(whichSlave);
    modifiedTime.deserialize(in);
    ::deserialize(in, fixedText);
}
Ejemplo n.º 16
0
bool processPartitionCommand(ISocket * masterSocket, MemoryBuffer & msg, MemoryBuffer & results)
{
    FileFormat srcFormat;
    FileFormat tgtFormat;
    unsigned whichInput;
    RemoteFilename fullPath;
    offset_t totalSize;
    offset_t thisOffset;
    offset_t thisSize;
    unsigned thisHeaderSize;
    unsigned numParts;
    bool compressedInput = false;
    unsigned compatflags = 0;  

    srcFormat.deserialize(msg);
    tgtFormat.deserialize(msg);
    msg.read(whichInput);
    fullPath.deserialize(msg);
    msg.read(totalSize);
    msg.read(thisOffset);
    msg.read(thisSize);
    msg.read(thisHeaderSize);
    msg.read(numParts);
    if (msg.remaining())
        msg.read(compressedInput);
    if (msg.remaining())
        msg.read(compatflags); // not yet used
    StringAttr decryptkey;
    if (msg.remaining())
        msg.read(decryptkey);
    if (msg.remaining())
    {
        srcFormat.deserializeExtra(msg, 1);
        tgtFormat.deserializeExtra(msg, 1);
    }

    StringBuffer text;
    fullPath.getRemotePath(text);
    LOG(MCdebugProgress, unknownJob, "Process partition %d(%s)", whichInput, text.str());
    Owned<IFormatProcessor> processor = createFormatProcessor(srcFormat, tgtFormat, true);
    Owned<IOutputProcessor> target = createOutputProcessor(tgtFormat);
    processor->setTarget(target);

    processor->setPartitionRange(totalSize, thisOffset, thisSize, thisHeaderSize, numParts);
    processor->setSource(whichInput, fullPath, compressedInput, decryptkey);
    processor->calcPartitions(NULL);

    PartitionPointArray partition;
    processor->getResults(partition);

    serialize(partition, results);
    return true;
}
Ejemplo n.º 17
0
extern DLLSERVER_API bool decompressResource(size32_t len, const void *data, StringBuffer &result)
{
    bool hasVersion = len && (*(const byte *)data == 0x80);
    MemoryBuffer src;
    src.setBuffer(len, const_cast<void *>(data), false);
    byte version = 1;
    if (hasVersion)
    {
        src.skip(1);
        src.read(version);
    }

    MemoryBuffer tgt;
    switch (version)
    {
    case 1:
        decompressToBuffer(tgt, src);
        break;
    default:
        throwUnexpected();
    }

    tgt.append((char)0);
    unsigned expandedLen = tgt.length();
    result.setBuffer(expandedLen, reinterpret_cast<char *>(tgt.detach()), expandedLen-1);
    return true;
}
Ejemplo n.º 18
0
void CWriteMasterBase::deserializeStats(unsigned node, MemoryBuffer &mb)
{
    CMasterActivity::deserializeStats(node, mb);
    unsigned repPerc;
    mb.read(repPerc);
    replicateProgress->set(node, repPerc);
}
Ejemplo n.º 19
0
void CDiskReadMasterBase::deserializeStats(unsigned node, MemoryBuffer &mb)
{
    CMasterActivity::deserializeStats(node, mb);
    rowcount_t progress;
    mb.read(progress);
    inputProgress->set(node, progress);
}
Ejemplo n.º 20
0
 virtual void merge(MemoryBuffer &in)    
 { 
     unsigned __int64 c[2];
     in.read(sizeof(c), &c);
     counts[false] += c[false];
     counts[true] += c[true];
 }
Ejemplo n.º 21
0
 virtual void slaveDone(size32_t slaveIdx, MemoryBuffer &mb)
 {
     if (mb.length()) // if 0 implies aborted out from this slave.
     {
         rowcount_t rc;
         mb.read(rc);
         recordsProcessed += rc;
     }
 }
Ejemplo n.º 22
0
 virtual void deserializeStats(unsigned node, MemoryBuffer &mb)
 {
     HashDistributeActivityMaster::deserializeStats(node, mb);
     rowcount_t lhsProgressCount, rhsProgressCount;
     mb.read(lhsProgressCount);
     mb.read(rhsProgressCount);
     lhsProgress->set(node, lhsProgressCount);
     rhsProgress->set(node, rhsProgressCount);
 }
Ejemplo n.º 23
0
 byte getWUSresult(MemoryBuffer &mb)
 {
     byte ret = 0;
     wusbuf.swapWith(mb);    
     if (mb.length()==1) {
         mb.read(ret);
         mb.clear();
     }
     return ret;
 }
Ejemplo n.º 24
0
void RemoteDataSourceServer::doCmdRow(bool raw, MemoryBuffer & in, MemoryBuffer & out)
{
    Owned<IFvDataSource> ds = readDataSource(in);
    if (!ds)
    {
        out.append(false);
        return;
    }

    __int64 requestedRow;
    in.read(requestedRow);

    unsigned startPos = out.length();
    unsigned numRows = 0;
    out.append(true);                       // ok
    out.append(requestedRow);       // start 

    unsigned numRowsPos = out.length();
    out.append(numRows);                // total number of rows;
    loop
    {
        unsigned lengthPos = out.length();
        out.append((unsigned)0);                // size of this row.
        unsigned startRow = out.length();
        if (raw)
        {
            if (!ds->getRawRow(out, requestedRow+numRows))
                break;
        }
        else
        {
            if (!ds->getRow(out, requestedRow+numRows))
                break;
        }
        if ((numRows != 0) && (out.length() > REMOTE_DATA_SIZE))
            break;
        unsigned endRow = out.length();
        out.rewrite(lengthPos);
        out.append(endRow-startRow);
        out.rewrite(endRow);
        numRows++;
    }

    if (numRows == 0)
    {
        out.rewrite(startPos);
        out.append(false);
        return;
    }

    unsigned totalLength = out.length();
    out.rewrite(numRowsPos);
    out.append(numRows);
    out.rewrite(totalLength);
}
Ejemplo n.º 25
0
    void slaveDone(size32_t slaveIdx, MemoryBuffer &mb)
    {
        if (mb.length()) // if 0 implies aborted out from this slave.
        {
            offset_t size;
            mb.read(size);
            CDateTime modifiedTime(mb);

            IPartDescriptor *partDesc = patchDesc->queryPart(slaveIdx);
            IPropertyTree &props = partDesc->queryProperties();         
            StringBuffer timeStr;
            modifiedTime.getString(timeStr);
            props.setProp("@modified", timeStr.str());
            unsigned crc;
            mb.read(crc);
            props.setPropInt64("@fileCrc", crc);
            if (!local && 0 == slaveIdx)
            {
                IPartDescriptor *partDesc = patchDesc->queryPart(patchDesc->numParts()-1);
                IPropertyTree &props = partDesc->queryProperties();
                mb.read(size);
                props.setPropInt64("@size", size);
                CDateTime modifiedTime(mb);
                StringBuffer timeStr;
                modifiedTime.getString(timeStr);
                props.setProp("@modified", timeStr.str());
                if (copyTlk)
                {
                    Owned<IPartDescriptor> tlkDesc = newIndexDesc->getPart(newIndexDesc->numParts()-1);
                    if (partDesc->getCrc(crc))
                        props.setPropInt64("@fileCrc", crc);
                    props.setProp("@diffFormat", "copy");
                }
                else
                {
                    mb.read(crc);
                    props.setPropInt64("@fileCrc", crc);
                    props.setProp("@diffFormat", "diffV1");
                }
            }
        }
    }
Ejemplo n.º 26
0
size32_t aesDecryptWithRSAEncryptedKey(MemoryBuffer &out, size32_t inSz, const void *inBytes, const CLoadedKey &privateKey)
{
    MemoryBuffer in;
    in.setBuffer(inSz, (void *)inBytes, false);
    // read encrypted AES key
    size32_t encryptedAESKeySz;
    in.read(encryptedAESKeySz);
    MemoryBuffer aesKey;
    size32_t decryptedAesKeySz = privateKeyDecrypt(aesKey, encryptedAESKeySz, in.readDirect(encryptedAESKeySz), privateKey);
    if (decryptedAesKeySz != aesMaxKeySize)
        throw makeStringException(0, "aesDecryptWithRSAEncryptedKey - invalid input");

    unsigned iVPos = in.getPos(); // read directly further down
    in.skip(aesBlockSize);

    size32_t aesEncryptedSz;
    in.read(aesEncryptedSz);

    return aesDecrypt(out, aesEncryptedSz, in.readDirect(aesEncryptedSz), aesMaxKeySize, (const char *)aesKey.bytes(), (const char *)in.bytes()+iVPos);
}
Ejemplo n.º 27
0
void CachedPasswordProvider::deserialize(MemoryBuffer & in)
{
    unsigned num;
    in.read(num);
    while (num--)
    {
        CachedPassword & cur = * new CachedPassword;
        cur.deserialize(in);
        passwords.append(cur);
    }
}
Ejemplo n.º 28
0
//---------------------------------------------------------------------------
void deserializeBoolArray(unsigned len, bool * values, MemoryBuffer & in)
{
    for (unsigned i = 0; i < len; i+= 8)
    {
        unsigned char next;
        in.read(next);
        unsigned max = i+8 <= len ? 8 : len - i;
        for (unsigned j=0; j<max; j++)
            values[i+j] = (next & (1 << j)) != 0;
    }
}
Ejemplo n.º 29
0
void deserialize(PartitionPointArray & partition, MemoryBuffer & in)
{
    unsigned count;
    in.read(count);
    for (unsigned idx = 0; idx < count; idx++)
    {
        PartitionPoint & next = * new PartitionPoint;
        next.deserialize(in);
        partition.append(next);
    }
}
Ejemplo n.º 30
0
void RemoteDataSourceServer::doCmdCreateFile(MemoryBuffer & in, MemoryBuffer & out)
{
    SessionId session;
    StringAttr username, password, logicalName;

    in.read(session);
    in.read(username).read(password);
    in.read(logicalName);

    DBGLOG("RemoteFileView:CreateFile('%s') by[%s:%" I64F "d", logicalName.get(), username.get(), session);
    Owned<IFvDataSource> ds = createFileDataSource(logicalName, cluster, username, password);
    unique_id_t id = addDataSource(session, ds);

    out.append((unsigned short)CurRemoteVersion);
    out.append(id);
    out.append(ds->numRows(false));
    ds->queryMetaData()->serialize(out);
    out.append(ds->isIndex());

    DBGLOG("RemoteFileView:CreateFile returns %" I64F "d", id);
}