コード例 #1
0
size32_t ThorExpand(const void * src, size32_t srcSz, MemoryBuffer & dest)
{
    size32_t sz;
    memcpy(&sz, src, sizeof(size32_t));
    size32_t bufSz = (sz == 0) ? (srcSz-sizeof(size32_t)) : sz;
    void * buf = dest.reserve(bufSz);   
    return ThorExpand(src, srcSz, buf, bufSz); 
}
コード例 #2
0
size32_t ThorCompress(const void * src, size32_t srcSz, MemoryBuffer & dest, size32_t threshold)
{
    size32_t prev = dest.length();
    size32_t dSz = srcSz + sizeof(size32_t);
    void * d = dest.reserve(dSz);
    size32_t ret = ThorCompress(src, srcSz, d, dSz, threshold);
    dest.setLength(prev+ret);
    return ret;
}
コード例 #3
0
void CThorTransferGroup::_receive(ISocket * rcv)
{
    CThorRowArray *received = new CThorRowArray();
    rcv->set_block_mode(BF_SYNC_TRANSFER_PULL,0,TRANSFER_TIMEOUT);
    size32_t bufferSz = rcv->receive_block_size();
    if (bufferSz) {
        MemoryBuffer mb;
        void * receiveBlock = mb.reserve(bufferSz);
        rcv->receive_block(receiveBlock,bufferSz); 
        received->deserialize(*allocator,deserializer,bufferSz,receiveBlock,false); 
    }
    rcv->close();
    rcv->Release();
    receive(received); // frees receiveArray
}
コード例 #4
0
ファイル: jlzma.cpp プロジェクト: Josh-Googler/HPCC-Platform
void LZMACompressToBuffer(MemoryBuffer & out, size32_t len, const void * src)
{
    CLZMA lzma;
    size32_t outbase = out.length();
    size32_t *sz = (size32_t *)out.reserve(len+sizeof(size32_t)*2);
    *sz = len;
    sz++;
    *sz = lzma.compress(src,len,sz+1);
    if (*sz>len) {
        *sz = len;
        memcpy(sz+1,src,len);
    }
    else 
        out.setLength(outbase+sizeof(size32_t)*2+*sz);
}
コード例 #5
0
ファイル: jlzma.cpp プロジェクト: Josh-Googler/HPCC-Platform
void LZMADecompressToBuffer(MemoryBuffer & out, MemoryBuffer & in)
{
    size32_t expsz;
    size32_t cmpsz;
    in.read(expsz).read(cmpsz);
    void *o = out.reserve(expsz);
    if (cmpsz!=expsz) {
        CLZMA lzma;
        size32_t written = lzma.expand(in.readDirect(cmpsz),cmpsz,o,expsz);
        if (written!=expsz)
            throw MakeStringException(0, "fastLZDecompressToBuffer - corrupt data(3) %d %d",written,expsz);
    }
    else
        memcpy(o,in.readDirect(cmpsz),expsz);
}
コード例 #6
0
ファイル: jlzma.cpp プロジェクト: Josh-Googler/HPCC-Platform
void LZMADecompressToBuffer(MemoryBuffer & out, const void * src)
{
    size32_t *sz = (size32_t *)src;
    size32_t expsz = *(sz++);
    size32_t cmpsz = *(sz++);
    void *o = out.reserve(expsz);
    if (cmpsz!=expsz) {
        CLZMA lzma;
        size32_t written = lzma.expand(sz,cmpsz,o,expsz);
        if (written!=expsz)
            throw MakeStringException(0, "fastLZDecompressToBuffer - corrupt data(1) %d %d",written,expsz);
    }
    else
        memcpy(o,sz,expsz);
}
コード例 #7
0
ファイル: ctfile.cpp プロジェクト: kenrowland/HPCC-Platform
void CJHTreeNode::unpack(const void *node, bool needCopy)
{
    memcpy(&hdr, node, sizeof(hdr));
    SwapBigEndian(hdr);
    __int64 maxsib = keyHdr->getHdrStruct()->phyrec;
    if (!hdr.isValid(keyHdr->getNodeSize()))
    {
        PROGLOG("hdr.leafFlag=%d",(int)hdr.leafFlag);
        PROGLOG("hdr.rightSib=%" I64F "d",hdr.rightSib);
        PROGLOG("hdr.leftSib=%" I64F "d",hdr.leftSib);
        PROGLOG("maxsib=%" I64F "d",maxsib);
        PROGLOG("nodeSize=%d", keyHdr->getNodeSize());
        PROGLOG("keyBytes=%d",(int)hdr.keyBytes);
        PrintStackReport();
        throw MakeStringException(0, "Htree: Corrupt key node detected");
    }
    if (!hdr.leafFlag)
        keyLen = keyHdr->getNodeKeyLength();
    keyRecLen = keyLen + sizeof(offset_t);
    char *keys = ((char *) node) + sizeof(hdr);
    if (hdr.crc32)
    {
        unsigned crc = crc32(keys, hdr.keyBytes, 0);
        if (hdr.crc32 != crc)
            throw MakeStringException(0, "CRC error on key node");
    }
    if (hdr.leafFlag==1)
    {
        firstSequence = *(unsigned __int64 *) keys;
        keys += sizeof(unsigned __int64);
        _WINREV(firstSequence);
    }
    if(isMetadata())
    {
        unsigned short len = *reinterpret_cast<unsigned short *>(keys);
        _WINREV(len);
        expandedSize = len;
        keyBuf = (char *) allocMem(len);
        memcpy(keyBuf, keys+sizeof(unsigned short), len);
    }
    else if (isLeaf() && (keyType & HTREE_COMPRESSED_KEY))
    {
        {
            MTIME_SECTION(queryActiveTimer(), "Compressed node expand");
            expandedSize = keyHdr->getNodeSize();
            bool quick = (keyType&HTREE_QUICK_COMPRESSED_KEY)==HTREE_QUICK_COMPRESSED_KEY;
#ifndef _OLD_VERSION
            keyBuf = NULL;
            if (quick)
                rowexp.setown(expandQuickKeys(keys, needCopy));
            if (!quick||!rowexp.get())
#endif
            {
                keyBuf = expandKeys(keys,keyLen,expandedSize,quick);
            }
        }
        assertex(keyBuf||rowexp.get());
    }
    else
    {
        int i;
        if (keyType & COL_PREFIX)
        {
            MTIME_SECTION(queryActiveTimer(), "COL_PREFIX expand");
            
            if (hdr.numKeys) {
                bool handleVariable = isVariable && isLeaf();
                KEYRECSIZE_T workRecLen;
                MemoryBuffer keyBufMb;
                const char *source = keys;
                char *target;
                // do first row
                if (handleVariable) {
                    memcpy(&workRecLen, source, sizeof(workRecLen));
                    _WINREV(workRecLen);
                    size32_t tmpSz = sizeof(workRecLen) + sizeof(offset_t);
                    target = (char *)keyBufMb.reserve(tmpSz+workRecLen);
                    memcpy(target, source, tmpSz);
                    source += tmpSz;
                    target += tmpSz;
                }
                else {
                    target = (char *)keyBufMb.reserveTruncate(hdr.numKeys * keyRecLen);
                    workRecLen = keyRecLen - sizeof(offset_t);
                    memcpy(target, source, sizeof(offset_t));
                    source += sizeof(offset_t);
                    target += sizeof(offset_t);
                }

                // this is where next row gets data from
                const char *prev, *next = NULL;
                unsigned prevOffset = 0;
                if (handleVariable)
                    prevOffset = target-((char *)keyBufMb.bufferBase());
                else
                    next = target;

                unsigned char pack1 = *source++;
#ifdef _DEBUG
                assertex(0==pack1); // 1st time will be always be 0
#endif
                KEYRECSIZE_T left = workRecLen;
                while (left--) {
                    *target = *source;
                    source++;
                    target++;
                }
                // do subsequent rows
                for (i = 1; i < hdr.numKeys; i++) {
                    if (handleVariable) {
                        memcpy(&workRecLen, source, sizeof(workRecLen));
                        _WINREV(workRecLen);
                        target = (char *)keyBufMb.reserve(sizeof(workRecLen)+sizeof(offset_t)+workRecLen);
                        size32_t tmpSz = sizeof(workRecLen)+sizeof(offset_t);
                        memcpy(target, source, tmpSz);
                        target += tmpSz;
                        source += tmpSz;
                    }
                    else
                    {
                        memcpy(target, source, sizeof(offset_t));
                        source += sizeof(offset_t);
                        target += sizeof(offset_t);
                    }
                    pack1 = *source++;
#ifdef _DEBUG
                    assertex(pack1<=workRecLen);            
#endif
                    if (handleVariable) {
                        prev = ((char *)keyBufMb.bufferBase())+prevOffset;
                        // for next
                        prevOffset = target-((char *)keyBufMb.bufferBase());
                    }
                    else {
                        prev = next;
                        next = target;
                    }
                    left = workRecLen - pack1;
                    while (pack1--) {
                        *target = *prev;
                        prev++;
                        target++;
                    }
                    while (left--) {
                        *target = *source;
                        source++;
                        target++;
                    }
                }
                expandedSize = keyBufMb.length();
                keyBuf = (char *)keyBufMb.detach();
                assertex(keyBuf);
            }
            else {
                keyBuf = NULL;
                expandedSize = 0;
            }
        }
        else
        {
            MTIME_SECTION(queryActiveTimer(), "NO compression copy");
            expandedSize = hdr.keyBytes + sizeof( __int64 );  // MORE - why is the +sizeof() there?
            keyBuf = (char *) allocMem(expandedSize);
            memcpy(keyBuf, keys, hdr.keyBytes + sizeof( __int64 ));
        }
    }
}