void CKeyHdr::load(KeyHdr &_hdr) { memcpy(&hdr, &_hdr, sizeof(hdr)); SwapBigEndian(hdr); if (0xffff != hdr.version && KEYBUILD_VERSION < hdr.version) throw MakeKeyException(KeyExcpt_IncompatVersion, "This build is compatible with key versions <= %u. Key is version %u", KEYBUILD_VERSION, (unsigned) hdr.version); }
void CKeyHdr::write(IFileIOStream *out, CRC32 *crc) { unsigned nodeSize = hdr.nodeSize; MemoryAttr ma; byte *buf = (byte *) ma.allocate(nodeSize); memcpy(buf, &hdr, sizeof(hdr)); memset(buf+sizeof(hdr), 0xff, nodeSize-sizeof(hdr)); SwapBigEndian(*(KeyHdr*) buf); out->write(nodeSize, buf); if (crc) crc->tally(nodeSize, buf); }
void CKeyHdr::write(IWriteSeq *out, CRC32 *crc) { unsigned nodeSize = hdr.nodeSize; assertex(out->getRecordSize()==nodeSize); MemoryAttr ma; byte *buf = (byte *) ma.allocate(nodeSize); memcpy(buf, &hdr, sizeof(hdr)); memset(buf+sizeof(hdr), 0xff, nodeSize-sizeof(hdr)); SwapBigEndian(*(KeyHdr*) buf); out->put(buf); if (crc) crc->tally(nodeSize, buf); }
extern bool isCompressedIndex(const char *filename) { OwnedIFile file = createIFile(filename); OwnedIFileIO io = file->open(IFOread); unsigned __int64 size = file->size(); if (size) { KeyHdr hdr; if (io->read(0, sizeof(hdr), &hdr) == sizeof(hdr)) { SwapBigEndian(hdr); if (size % hdr.nodeSize == 0 && hdr.phyrec == size-1 && hdr.root && hdr.root % hdr.nodeSize == 0 && hdr.ktype & (HTREE_COMPRESSED_KEY|HTREE_QUICK_COMPRESSED_KEY)) { NodeHdr root; if (io->read(hdr.root, sizeof(root), &root) == sizeof(root)) { SwapBigEndian(root); return root.leftSib==0 && root.rightSib==0; } } } } return false; }
int main(int argc, const char *argv[]) { if (argc<2) usage(0); int arg = 1; while (arg < argc) { if (stricmp(argv[arg], "-crc") == 0) { checkCRC = true; } else if (stricmp(argv[arg], "-errorLimit") == 0) { ++arg; if (arg>=argc) usage(1); errorLimit = strtoul(argv[arg], NULL, 10); if (!errorLimit) errorLimit = (unsigned) -1; } else if (stricmp(argv[arg], "-node") == 0) { ++arg; if (arg>=argc) usage(1); nodeAddress = strtoul(argv[arg], NULL, 16); checkCRC = true; } else if (stricmp(argv[arg], "-noleaf") == 0) { skipLeafLevel = true; } else if (stricmp(argv[arg], "-quick") == 0) { quick = true; } else if (*argv[arg]=='-') usage(1); ++arg; } arg = 1; while (arg < argc) { if (*argv[arg]!='-') { curFileName = argv[arg]; printf("Processing key file %s\n", curFileName); int f = _open(argv[arg], _O_RDONLY|_O_BINARY); if (f==-1) { noteError(0, "Could not open file\n"); } else { KeyHdr h; if (_read(f, &h, sizeof(h)) != sizeof(h)) { noteError(0, "Could not read key header\n"); } else { SwapBigEndian(h); if (nodeAddress) { checkNode(f, h, nodeAddress); } else if (quick) { int levels = countLevels(f, h, h.root); printf("%d levels found\n", levels); } else { unsigned level = 0; checkLevel(f, h, level, h.root); } } _close(f); } } arg++; } return errors; }
extern jhtree_decl void validateKeyFile(const char *filename, offset_t nodePos) { OwnedIFile file = createIFile(filename); OwnedIFileIO io = file->open(IFOread); if (!io) throw MakeStringException(1, "Invalid key %s: cannot open file", filename); unsigned __int64 size = file->size(); if (!size) throw MakeStringException(2, "Invalid key %s: zero size", filename); KeyHdr hdr; if (io->read(0, sizeof(hdr), &hdr) != sizeof(hdr)) throw MakeStringException(4, "Invalid key %s: failed to read key header", filename); CKeyHdr keyHdr; keyHdr.load(hdr); _WINREV(hdr.phyrec); _WINREV(hdr.root); _WINREV(hdr.nodeSize); if (hdr.phyrec != size-1) throw MakeStringException(5, "Invalid key %s: phyrec was %" I64F "d, expected %" I64F "d", filename, hdr.phyrec, size-1); if (size % hdr.nodeSize) throw MakeStringException(3, "Invalid key %s: size %" I64F "d is not a multiple of key node size (%d)", filename, size, hdr.nodeSize); if (!hdr.root || hdr.root % hdr.nodeSize !=0) throw MakeStringException(6, "Invalid key %s: invalid root pointer %" I64F "x", filename, hdr.root); NodeHdr root; if (io->read(hdr.root, sizeof(root), &root) != sizeof(root)) throw MakeStringException(7, "Invalid key %s: failed to read root node", filename); _WINREV(root.rightSib); _WINREV(root.leftSib); if (root.leftSib || root.rightSib) throw MakeStringException(8, "Invalid key %s: invalid root node sibling pointers 0x%" I64F "x, 0x%" I64F "x (expected 0,0)", filename, root.leftSib, root.rightSib); for (offset_t nodeOffset = (nodePos ? nodePos : hdr.nodeSize); nodeOffset < (nodePos ? nodePos+1 : size); nodeOffset += hdr.nodeSize) { MemoryAttr ma; char *buffer = (char *) ma.allocate(hdr.nodeSize); { MTIME_SECTION(queryActiveTimer(), "JHTREE read index node"); io->read(nodeOffset, hdr.nodeSize, buffer); } CJHTreeNode theNode; { MTIME_SECTION(queryActiveTimer(), "JHTREE load index node"); theNode.load(&keyHdr, buffer, nodeOffset, true); } NodeHdr *nodeHdr = (NodeHdr *) buffer; SwapBigEndian(*nodeHdr); if (!nodeHdr->isValid(hdr.nodeSize)) throw MakeStringException(9, "Invalid key %s: invalid node header at position 0x%" I64F "x", filename, nodeOffset); if (nodeHdr->leftSib >= size || nodeHdr->rightSib >= size) throw MakeStringException(9, "Invalid key %s: out of range sibling pointers 0x%" I64F "x, 0x%" I64F "x at position 0x%" I64F "x", filename, nodeHdr->leftSib, nodeHdr->rightSib, nodeOffset); if (nodeHdr->crc32) { unsigned crc = crc32(buffer + sizeof(NodeHdr), nodeHdr->keyBytes, 0); if (crc != nodeHdr->crc32) throw MakeStringException(9, "Invalid key %s: crc mismatch at position 0x%" I64F "x", filename, nodeOffset); } else { // MORE - if we felt so inclined, we could decode the node and check records were in ascending order } } }
void CJHTreeNode::unpack(const void *node, bool needCopy) { memcpy(&hdr, node, sizeof(hdr)); SwapBigEndian(hdr); __int64 maxsib = keyHdr->getHdrStruct()->phyrec; if (!hdr.isValid(keyHdr->getNodeSize())) { PROGLOG("hdr.leafFlag=%d",(int)hdr.leafFlag); PROGLOG("hdr.rightSib=%" I64F "d",hdr.rightSib); PROGLOG("hdr.leftSib=%" I64F "d",hdr.leftSib); PROGLOG("maxsib=%" I64F "d",maxsib); PROGLOG("nodeSize=%d", keyHdr->getNodeSize()); PROGLOG("keyBytes=%d",(int)hdr.keyBytes); PrintStackReport(); throw MakeStringException(0, "Htree: Corrupt key node detected"); } if (!hdr.leafFlag) keyLen = keyHdr->getNodeKeyLength(); keyRecLen = keyLen + sizeof(offset_t); char *keys = ((char *) node) + sizeof(hdr); if (hdr.crc32) { unsigned crc = crc32(keys, hdr.keyBytes, 0); if (hdr.crc32 != crc) throw MakeStringException(0, "CRC error on key node"); } if (hdr.leafFlag==1) { firstSequence = *(unsigned __int64 *) keys; keys += sizeof(unsigned __int64); _WINREV(firstSequence); } if(isMetadata()) { unsigned short len = *reinterpret_cast<unsigned short *>(keys); _WINREV(len); expandedSize = len; keyBuf = (char *) allocMem(len); memcpy(keyBuf, keys+sizeof(unsigned short), len); } else if (isLeaf() && (keyType & HTREE_COMPRESSED_KEY)) { { MTIME_SECTION(queryActiveTimer(), "Compressed node expand"); expandedSize = keyHdr->getNodeSize(); bool quick = (keyType&HTREE_QUICK_COMPRESSED_KEY)==HTREE_QUICK_COMPRESSED_KEY; #ifndef _OLD_VERSION keyBuf = NULL; if (quick) rowexp.setown(expandQuickKeys(keys, needCopy)); if (!quick||!rowexp.get()) #endif { keyBuf = expandKeys(keys,keyLen,expandedSize,quick); } } assertex(keyBuf||rowexp.get()); } else { int i; if (keyType & COL_PREFIX) { MTIME_SECTION(queryActiveTimer(), "COL_PREFIX expand"); if (hdr.numKeys) { bool handleVariable = isVariable && isLeaf(); KEYRECSIZE_T workRecLen; MemoryBuffer keyBufMb; const char *source = keys; char *target; // do first row if (handleVariable) { memcpy(&workRecLen, source, sizeof(workRecLen)); _WINREV(workRecLen); size32_t tmpSz = sizeof(workRecLen) + sizeof(offset_t); target = (char *)keyBufMb.reserve(tmpSz+workRecLen); memcpy(target, source, tmpSz); source += tmpSz; target += tmpSz; } else { target = (char *)keyBufMb.reserveTruncate(hdr.numKeys * keyRecLen); workRecLen = keyRecLen - sizeof(offset_t); memcpy(target, source, sizeof(offset_t)); source += sizeof(offset_t); target += sizeof(offset_t); } // this is where next row gets data from const char *prev, *next = NULL; unsigned prevOffset = 0; if (handleVariable) prevOffset = target-((char *)keyBufMb.bufferBase()); else next = target; unsigned char pack1 = *source++; #ifdef _DEBUG assertex(0==pack1); // 1st time will be always be 0 #endif KEYRECSIZE_T left = workRecLen; while (left--) { *target = *source; source++; target++; } // do subsequent rows for (i = 1; i < hdr.numKeys; i++) { if (handleVariable) { memcpy(&workRecLen, source, sizeof(workRecLen)); _WINREV(workRecLen); target = (char *)keyBufMb.reserve(sizeof(workRecLen)+sizeof(offset_t)+workRecLen); size32_t tmpSz = sizeof(workRecLen)+sizeof(offset_t); memcpy(target, source, tmpSz); target += tmpSz; source += tmpSz; } else { memcpy(target, source, sizeof(offset_t)); source += sizeof(offset_t); target += sizeof(offset_t); } pack1 = *source++; #ifdef _DEBUG assertex(pack1<=workRecLen); #endif if (handleVariable) { prev = ((char *)keyBufMb.bufferBase())+prevOffset; // for next prevOffset = target-((char *)keyBufMb.bufferBase()); } else { prev = next; next = target; } left = workRecLen - pack1; while (pack1--) { *target = *prev; prev++; target++; } while (left--) { *target = *source; source++; target++; } } expandedSize = keyBufMb.length(); keyBuf = (char *)keyBufMb.detach(); assertex(keyBuf); } else { keyBuf = NULL; expandedSize = 0; } } else { MTIME_SECTION(queryActiveTimer(), "NO compression copy"); expandedSize = hdr.keyBytes + sizeof( __int64 ); // MORE - why is the +sizeof() there? keyBuf = (char *) allocMem(expandedSize); memcpy(keyBuf, keys, hdr.keyBytes + sizeof( __int64 )); } } }
void CNodeHeader::load(NodeHdr &_hdr) { memcpy(&hdr, &_hdr, sizeof(hdr)); SwapBigEndian(hdr); }
void CWriteNodeBase::writeHdr() { hdr.crc32 = crc32(nodeBuf+sizeof(hdr), hdr.keyBytes, 0); memcpy(nodeBuf, &hdr, sizeof(hdr)); SwapBigEndian(*(NodeHdr *) nodeBuf); }