size32_t ThorExpand(const void * src, size32_t srcSz, MemoryBuffer & dest)
{
size32_t sz;
memcpy(&sz, src, sizeof(size32_t));
size32_t bufSz = (sz == 0) ? (srcSz-sizeof(size32_t)) : sz;
void * buf = dest.reserve(bufSz);
return ThorExpand(src, srcSz, buf, bufSz);
}
size32_t ThorCompress(const void * src, size32_t srcSz, MemoryBuffer & dest, size32_t threshold)
{
size32_t prev = dest.length();
size32_t dSz = srcSz + sizeof(size32_t);
void * d = dest.reserve(dSz);
size32_t ret = ThorCompress(src, srcSz, d, dSz, threshold);
dest.setLength(prev+ret);
return ret;
}
void CThorTransferGroup::_receive(ISocket * rcv)
{
CThorRowArray *received = new CThorRowArray();
rcv->set_block_mode(BF_SYNC_TRANSFER_PULL,0,TRANSFER_TIMEOUT);
size32_t bufferSz = rcv->receive_block_size();
if (bufferSz) {
MemoryBuffer mb;
void * receiveBlock = mb.reserve(bufferSz);
rcv->receive_block(receiveBlock,bufferSz);
received->deserialize(*allocator,deserializer,bufferSz,receiveBlock,false);
}
rcv->close();
rcv->Release();
receive(received); // frees receiveArray
}
void LZMACompressToBuffer(MemoryBuffer & out, size32_t len, const void * src)
{
CLZMA lzma;
size32_t outbase = out.length();
size32_t *sz = (size32_t *)out.reserve(len+sizeof(size32_t)*2);
*sz = len;
sz++;
*sz = lzma.compress(src,len,sz+1);
if (*sz>len) {
*sz = len;
memcpy(sz+1,src,len);
}
else
out.setLength(outbase+sizeof(size32_t)*2+*sz);
}
void LZMADecompressToBuffer(MemoryBuffer & out, MemoryBuffer & in)
{
size32_t expsz;
size32_t cmpsz;
in.read(expsz).read(cmpsz);
void *o = out.reserve(expsz);
if (cmpsz!=expsz) {
CLZMA lzma;
size32_t written = lzma.expand(in.readDirect(cmpsz),cmpsz,o,expsz);
if (written!=expsz)
throw MakeStringException(0, "fastLZDecompressToBuffer - corrupt data(3) %d %d",written,expsz);
}
else
memcpy(o,in.readDirect(cmpsz),expsz);
}
void LZMADecompressToBuffer(MemoryBuffer & out, const void * src)
{
size32_t *sz = (size32_t *)src;
size32_t expsz = *(sz++);
size32_t cmpsz = *(sz++);
void *o = out.reserve(expsz);
if (cmpsz!=expsz) {
CLZMA lzma;
size32_t written = lzma.expand(sz,cmpsz,o,expsz);
if (written!=expsz)
throw MakeStringException(0, "fastLZDecompressToBuffer - corrupt data(1) %d %d",written,expsz);
}
else
memcpy(o,sz,expsz);
}
void CJHTreeNode::unpack(const void *node, bool needCopy)
{
memcpy(&hdr, node, sizeof(hdr));
SwapBigEndian(hdr);
__int64 maxsib = keyHdr->getHdrStruct()->phyrec;
if (!hdr.isValid(keyHdr->getNodeSize()))
{
PROGLOG("hdr.leafFlag=%d",(int)hdr.leafFlag);
PROGLOG("hdr.rightSib=%" I64F "d",hdr.rightSib);
PROGLOG("hdr.leftSib=%" I64F "d",hdr.leftSib);
PROGLOG("maxsib=%" I64F "d",maxsib);
PROGLOG("nodeSize=%d", keyHdr->getNodeSize());
PROGLOG("keyBytes=%d",(int)hdr.keyBytes);
PrintStackReport();
throw MakeStringException(0, "Htree: Corrupt key node detected");
}
if (!hdr.leafFlag)
keyLen = keyHdr->getNodeKeyLength();
keyRecLen = keyLen + sizeof(offset_t);
char *keys = ((char *) node) + sizeof(hdr);
if (hdr.crc32)
{
unsigned crc = crc32(keys, hdr.keyBytes, 0);
if (hdr.crc32 != crc)
throw MakeStringException(0, "CRC error on key node");
}
if (hdr.leafFlag==1)
{
firstSequence = *(unsigned __int64 *) keys;
keys += sizeof(unsigned __int64);
_WINREV(firstSequence);
}
if(isMetadata())
{
unsigned short len = *reinterpret_cast<unsigned short *>(keys);
_WINREV(len);
expandedSize = len;
keyBuf = (char *) allocMem(len);
memcpy(keyBuf, keys+sizeof(unsigned short), len);
}
else if (isLeaf() && (keyType & HTREE_COMPRESSED_KEY))
{
{
MTIME_SECTION(queryActiveTimer(), "Compressed node expand");
expandedSize = keyHdr->getNodeSize();
bool quick = (keyType&HTREE_QUICK_COMPRESSED_KEY)==HTREE_QUICK_COMPRESSED_KEY;
#ifndef _OLD_VERSION
keyBuf = NULL;
if (quick)
rowexp.setown(expandQuickKeys(keys, needCopy));
if (!quick||!rowexp.get())
#endif
{
keyBuf = expandKeys(keys,keyLen,expandedSize,quick);
}
}
assertex(keyBuf||rowexp.get());
}
else
{
int i;
if (keyType & COL_PREFIX)
{
MTIME_SECTION(queryActiveTimer(), "COL_PREFIX expand");
if (hdr.numKeys) {
bool handleVariable = isVariable && isLeaf();
KEYRECSIZE_T workRecLen;
MemoryBuffer keyBufMb;
const char *source = keys;
char *target;
// do first row
if (handleVariable) {
memcpy(&workRecLen, source, sizeof(workRecLen));
_WINREV(workRecLen);
size32_t tmpSz = sizeof(workRecLen) + sizeof(offset_t);
target = (char *)keyBufMb.reserve(tmpSz+workRecLen);
memcpy(target, source, tmpSz);
source += tmpSz;
target += tmpSz;
}
else {
target = (char *)keyBufMb.reserveTruncate(hdr.numKeys * keyRecLen);
workRecLen = keyRecLen - sizeof(offset_t);
memcpy(target, source, sizeof(offset_t));
source += sizeof(offset_t);
target += sizeof(offset_t);
}
// this is where next row gets data from
const char *prev, *next = NULL;
unsigned prevOffset = 0;
if (handleVariable)
prevOffset = target-((char *)keyBufMb.bufferBase());
else
next = target;
unsigned char pack1 = *source++;
#ifdef _DEBUG
assertex(0==pack1); // 1st time will be always be 0
#endif
KEYRECSIZE_T left = workRecLen;
while (left--) {
*target = *source;
source++;
target++;
}
// do subsequent rows
for (i = 1; i < hdr.numKeys; i++) {
if (handleVariable) {
memcpy(&workRecLen, source, sizeof(workRecLen));
_WINREV(workRecLen);
target = (char *)keyBufMb.reserve(sizeof(workRecLen)+sizeof(offset_t)+workRecLen);
size32_t tmpSz = sizeof(workRecLen)+sizeof(offset_t);
memcpy(target, source, tmpSz);
target += tmpSz;
source += tmpSz;
}
else
{
memcpy(target, source, sizeof(offset_t));
source += sizeof(offset_t);
target += sizeof(offset_t);
}
pack1 = *source++;
#ifdef _DEBUG
assertex(pack1<=workRecLen);
#endif
if (handleVariable) {
prev = ((char *)keyBufMb.bufferBase())+prevOffset;
// for next
prevOffset = target-((char *)keyBufMb.bufferBase());
}
else {
prev = next;
next = target;
}
left = workRecLen - pack1;
while (pack1--) {
*target = *prev;
prev++;
target++;
}
while (left--) {
*target = *source;
source++;
target++;
}
}
expandedSize = keyBufMb.length();
keyBuf = (char *)keyBufMb.detach();
assertex(keyBuf);
}
else {
keyBuf = NULL;
expandedSize = 0;
}
}
else
{
MTIME_SECTION(queryActiveTimer(), "NO compression copy");
expandedSize = hdr.keyBytes + sizeof( __int64 ); // MORE - why is the +sizeof() there?
keyBuf = (char *) allocMem(expandedSize);
memcpy(keyBuf, keys, hdr.keyBytes + sizeof( __int64 ));
}
}
}