Exemplo n.º 1
0
void CKeyHdr::load(KeyHdr &_hdr)
{
    memcpy(&hdr, &_hdr, sizeof(hdr));
    SwapBigEndian(hdr);

    if (0xffff != hdr.version && KEYBUILD_VERSION < hdr.version)
        throw MakeKeyException(KeyExcpt_IncompatVersion, "This build is compatible with key versions <= %u. Key is version %u", KEYBUILD_VERSION, (unsigned) hdr.version);
}
Exemplo n.º 2
0
void CKeyHdr::write(IFileIOStream *out, CRC32 *crc)
{
    unsigned nodeSize = hdr.nodeSize;
    MemoryAttr ma;
    byte *buf = (byte *) ma.allocate(nodeSize); 
    memcpy(buf, &hdr, sizeof(hdr));
    memset(buf+sizeof(hdr), 0xff, nodeSize-sizeof(hdr));
    SwapBigEndian(*(KeyHdr*) buf);
    out->write(nodeSize, buf);
    if (crc)
        crc->tally(nodeSize, buf);
}
Exemplo n.º 3
0
void CKeyHdr::write(IWriteSeq *out, CRC32 *crc)
{
    unsigned nodeSize = hdr.nodeSize;
    assertex(out->getRecordSize()==nodeSize);
    MemoryAttr ma;
    byte *buf = (byte *) ma.allocate(nodeSize); 
    memcpy(buf, &hdr, sizeof(hdr));
    memset(buf+sizeof(hdr), 0xff, nodeSize-sizeof(hdr));
    SwapBigEndian(*(KeyHdr*) buf);
    out->put(buf);
    if (crc)
        crc->tally(nodeSize, buf);
}
Exemplo n.º 4
0
extern bool isCompressedIndex(const char *filename)
{
    OwnedIFile file = createIFile(filename);
    OwnedIFileIO io = file->open(IFOread);
    unsigned __int64 size = file->size();
    if (size)
    {
        KeyHdr hdr;
        if (io->read(0, sizeof(hdr), &hdr) == sizeof(hdr))
        {
            SwapBigEndian(hdr);
            if (size % hdr.nodeSize == 0 && hdr.phyrec == size-1 && hdr.root && hdr.root % hdr.nodeSize == 0 && hdr.ktype & (HTREE_COMPRESSED_KEY|HTREE_QUICK_COMPRESSED_KEY))
            {
                NodeHdr root;
                if (io->read(hdr.root, sizeof(root), &root) == sizeof(root))
                {
                    SwapBigEndian(root);
                    return root.leftSib==0 && root.rightSib==0; 
                }
            }
        }
    }
    return false;
}
Exemplo n.º 5
0
int main(int argc, const char *argv[])
{
    if (argc<2) 
        usage(0);

    int arg = 1;
    while (arg < argc)
    {
        if (stricmp(argv[arg], "-crc") == 0)
        {
            checkCRC = true;
        }
        else if (stricmp(argv[arg], "-errorLimit") == 0)
        {
            ++arg;
            if (arg>=argc)
                usage(1);           
            errorLimit = strtoul(argv[arg], NULL, 10);
            if (!errorLimit)
                errorLimit = (unsigned) -1;
        }
        else if (stricmp(argv[arg], "-node") == 0)
        {
            ++arg;
            if (arg>=argc)
                usage(1);           
            nodeAddress = strtoul(argv[arg], NULL, 16);
            checkCRC = true;
        }
        else if (stricmp(argv[arg], "-noleaf") == 0)
        {
            skipLeafLevel = true;
        }
        else if (stricmp(argv[arg], "-quick") == 0)
        {
            quick = true;
        }
        else if (*argv[arg]=='-')
            usage(1);
        ++arg;
    }
    arg = 1;
    while (arg < argc)
    {
        if (*argv[arg]!='-')
        {
            curFileName = argv[arg];
            printf("Processing key file %s\n", curFileName);
            int f = _open(argv[arg], _O_RDONLY|_O_BINARY);
            if (f==-1)
            {
                noteError(0, "Could not open file\n");
            }
            else
            {
                KeyHdr h;
                if (_read(f, &h, sizeof(h)) != sizeof(h))
                {
                    noteError(0, "Could not read key header\n");
                }
                else
                {
                    SwapBigEndian(h);
                    if (nodeAddress)
                    {
                        checkNode(f, h, nodeAddress);
                    }
                    else if (quick)
                    {
                        int levels = countLevels(f, h, h.root);
                        printf("%d levels found\n", levels);
                    }
                    else
                    {
                        unsigned level = 0;
                        checkLevel(f, h, level, h.root);
                    }
                }
                _close(f);
            }
        }
        arg++;
    }
    return errors;
}
Exemplo n.º 6
0
extern jhtree_decl void validateKeyFile(const char *filename, offset_t nodePos)
{
    OwnedIFile file = createIFile(filename);
    OwnedIFileIO io = file->open(IFOread);
    if (!io)
        throw MakeStringException(1, "Invalid key %s: cannot open file", filename);
    unsigned __int64 size = file->size();
    if (!size)
        throw MakeStringException(2, "Invalid key %s: zero size", filename);
    KeyHdr hdr;
    if (io->read(0, sizeof(hdr), &hdr) != sizeof(hdr))
        throw MakeStringException(4, "Invalid key %s: failed to read key header", filename);
    CKeyHdr keyHdr;
    keyHdr.load(hdr);

    _WINREV(hdr.phyrec);
    _WINREV(hdr.root);
    _WINREV(hdr.nodeSize);
    if (hdr.phyrec != size-1)
        throw MakeStringException(5, "Invalid key %s: phyrec was %" I64F "d, expected %" I64F "d", filename, hdr.phyrec, size-1);
    if (size % hdr.nodeSize)
        throw MakeStringException(3, "Invalid key %s: size %" I64F "d is not a multiple of key node size (%d)", filename, size, hdr.nodeSize);
    if (!hdr.root || hdr.root % hdr.nodeSize !=0)
        throw MakeStringException(6, "Invalid key %s: invalid root pointer %" I64F "x", filename, hdr.root);
    NodeHdr root;
    if (io->read(hdr.root, sizeof(root), &root) != sizeof(root))
        throw MakeStringException(7, "Invalid key %s: failed to read root node", filename);
    _WINREV(root.rightSib);
    _WINREV(root.leftSib);
    if (root.leftSib || root.rightSib)
        throw MakeStringException(8, "Invalid key %s: invalid root node sibling pointers 0x%" I64F "x, 0x%" I64F "x (expected 0,0)", filename, root.leftSib, root.rightSib);

    for (offset_t nodeOffset = (nodePos ? nodePos : hdr.nodeSize); nodeOffset < (nodePos ? nodePos+1 : size); nodeOffset += hdr.nodeSize)
    {
        MemoryAttr ma;
        char *buffer = (char *) ma.allocate(hdr.nodeSize);
        {
            MTIME_SECTION(queryActiveTimer(), "JHTREE read index node");
            io->read(nodeOffset, hdr.nodeSize, buffer);
        }
        CJHTreeNode theNode;
        {
            MTIME_SECTION(queryActiveTimer(), "JHTREE load index node");
            theNode.load(&keyHdr, buffer, nodeOffset, true);
        }
        NodeHdr *nodeHdr = (NodeHdr *) buffer;
        SwapBigEndian(*nodeHdr);
        if (!nodeHdr->isValid(hdr.nodeSize))
            throw MakeStringException(9, "Invalid key %s: invalid node header at position 0x%" I64F "x", filename, nodeOffset);
        if (nodeHdr->leftSib >= size || nodeHdr->rightSib >= size)
            throw MakeStringException(9, "Invalid key %s: out of range sibling pointers 0x%" I64F "x, 0x%" I64F "x at position 0x%" I64F "x", filename, nodeHdr->leftSib, nodeHdr->rightSib, nodeOffset);
        if (nodeHdr->crc32)
        {
            unsigned crc = crc32(buffer + sizeof(NodeHdr), nodeHdr->keyBytes, 0);
            if (crc != nodeHdr->crc32)
                throw MakeStringException(9, "Invalid key %s: crc mismatch at position 0x%" I64F "x", filename, nodeOffset);
        }
        else
        {
            // MORE - if we felt so inclined, we could decode the node and check records were in ascending order
        }
    }
}
Exemplo n.º 7
0
void CJHTreeNode::unpack(const void *node, bool needCopy)
{
    memcpy(&hdr, node, sizeof(hdr));
    SwapBigEndian(hdr);
    __int64 maxsib = keyHdr->getHdrStruct()->phyrec;
    if (!hdr.isValid(keyHdr->getNodeSize()))
    {
        PROGLOG("hdr.leafFlag=%d",(int)hdr.leafFlag);
        PROGLOG("hdr.rightSib=%" I64F "d",hdr.rightSib);
        PROGLOG("hdr.leftSib=%" I64F "d",hdr.leftSib);
        PROGLOG("maxsib=%" I64F "d",maxsib);
        PROGLOG("nodeSize=%d", keyHdr->getNodeSize());
        PROGLOG("keyBytes=%d",(int)hdr.keyBytes);
        PrintStackReport();
        throw MakeStringException(0, "Htree: Corrupt key node detected");
    }
    if (!hdr.leafFlag)
        keyLen = keyHdr->getNodeKeyLength();
    keyRecLen = keyLen + sizeof(offset_t);
    char *keys = ((char *) node) + sizeof(hdr);
    if (hdr.crc32)
    {
        unsigned crc = crc32(keys, hdr.keyBytes, 0);
        if (hdr.crc32 != crc)
            throw MakeStringException(0, "CRC error on key node");
    }
    if (hdr.leafFlag==1)
    {
        firstSequence = *(unsigned __int64 *) keys;
        keys += sizeof(unsigned __int64);
        _WINREV(firstSequence);
    }
    if(isMetadata())
    {
        unsigned short len = *reinterpret_cast<unsigned short *>(keys);
        _WINREV(len);
        expandedSize = len;
        keyBuf = (char *) allocMem(len);
        memcpy(keyBuf, keys+sizeof(unsigned short), len);
    }
    else if (isLeaf() && (keyType & HTREE_COMPRESSED_KEY))
    {
        {
            MTIME_SECTION(queryActiveTimer(), "Compressed node expand");
            expandedSize = keyHdr->getNodeSize();
            bool quick = (keyType&HTREE_QUICK_COMPRESSED_KEY)==HTREE_QUICK_COMPRESSED_KEY;
#ifndef _OLD_VERSION
            keyBuf = NULL;
            if (quick)
                rowexp.setown(expandQuickKeys(keys, needCopy));
            if (!quick||!rowexp.get())
#endif
            {
                keyBuf = expandKeys(keys,keyLen,expandedSize,quick);
            }
        }
        assertex(keyBuf||rowexp.get());
    }
    else
    {
        int i;
        if (keyType & COL_PREFIX)
        {
            MTIME_SECTION(queryActiveTimer(), "COL_PREFIX expand");
            
            if (hdr.numKeys) {
                bool handleVariable = isVariable && isLeaf();
                KEYRECSIZE_T workRecLen;
                MemoryBuffer keyBufMb;
                const char *source = keys;
                char *target;
                // do first row
                if (handleVariable) {
                    memcpy(&workRecLen, source, sizeof(workRecLen));
                    _WINREV(workRecLen);
                    size32_t tmpSz = sizeof(workRecLen) + sizeof(offset_t);
                    target = (char *)keyBufMb.reserve(tmpSz+workRecLen);
                    memcpy(target, source, tmpSz);
                    source += tmpSz;
                    target += tmpSz;
                }
                else {
                    target = (char *)keyBufMb.reserveTruncate(hdr.numKeys * keyRecLen);
                    workRecLen = keyRecLen - sizeof(offset_t);
                    memcpy(target, source, sizeof(offset_t));
                    source += sizeof(offset_t);
                    target += sizeof(offset_t);
                }

                // this is where next row gets data from
                const char *prev, *next = NULL;
                unsigned prevOffset = 0;
                if (handleVariable)
                    prevOffset = target-((char *)keyBufMb.bufferBase());
                else
                    next = target;

                unsigned char pack1 = *source++;
#ifdef _DEBUG
                assertex(0==pack1); // 1st time will be always be 0
#endif
                KEYRECSIZE_T left = workRecLen;
                while (left--) {
                    *target = *source;
                    source++;
                    target++;
                }
                // do subsequent rows
                for (i = 1; i < hdr.numKeys; i++) {
                    if (handleVariable) {
                        memcpy(&workRecLen, source, sizeof(workRecLen));
                        _WINREV(workRecLen);
                        target = (char *)keyBufMb.reserve(sizeof(workRecLen)+sizeof(offset_t)+workRecLen);
                        size32_t tmpSz = sizeof(workRecLen)+sizeof(offset_t);
                        memcpy(target, source, tmpSz);
                        target += tmpSz;
                        source += tmpSz;
                    }
                    else
                    {
                        memcpy(target, source, sizeof(offset_t));
                        source += sizeof(offset_t);
                        target += sizeof(offset_t);
                    }
                    pack1 = *source++;
#ifdef _DEBUG
                    assertex(pack1<=workRecLen);            
#endif
                    if (handleVariable) {
                        prev = ((char *)keyBufMb.bufferBase())+prevOffset;
                        // for next
                        prevOffset = target-((char *)keyBufMb.bufferBase());
                    }
                    else {
                        prev = next;
                        next = target;
                    }
                    left = workRecLen - pack1;
                    while (pack1--) {
                        *target = *prev;
                        prev++;
                        target++;
                    }
                    while (left--) {
                        *target = *source;
                        source++;
                        target++;
                    }
                }
                expandedSize = keyBufMb.length();
                keyBuf = (char *)keyBufMb.detach();
                assertex(keyBuf);
            }
            else {
                keyBuf = NULL;
                expandedSize = 0;
            }
        }
        else
        {
            MTIME_SECTION(queryActiveTimer(), "NO compression copy");
            expandedSize = hdr.keyBytes + sizeof( __int64 );  // MORE - why is the +sizeof() there?
            keyBuf = (char *) allocMem(expandedSize);
            memcpy(keyBuf, keys, hdr.keyBytes + sizeof( __int64 ));
        }
    }
}
Exemplo n.º 8
0
void CNodeHeader::load(NodeHdr &_hdr)
{
    memcpy(&hdr, &_hdr, sizeof(hdr));
    SwapBigEndian(hdr);
}
Exemplo n.º 9
0
void CWriteNodeBase::writeHdr()
{
    hdr.crc32 = crc32(nodeBuf+sizeof(hdr), hdr.keyBytes, 0);
    memcpy(nodeBuf, &hdr, sizeof(hdr));
    SwapBigEndian(*(NodeHdr *) nodeBuf);
}