inline void swap(NodeHdr &hdr)
{
_WINREV(hdr.rightSib);
_WINREV(hdr.leftSib);
_WINREV(hdr.numKeys);
_WINREV(hdr.keyBytes);
_WINREV(hdr.crc32);
// _WINREV(hdr.memNumber);
// _WINREV(hdr.leafFlag);
}
void sendFile(const char * filename, ISocket * socket)
{
FILE *in = fopen(filename, "rb");
unsigned size = 0;
void * buff = NULL;
if (in)
{
fseek(in, 0, SEEK_END);
size = ftell(in);
fseek(in, 0, SEEK_SET);
buff = malloc(size);
size_t numRead = fread(buff, 1, size, in);
fclose(in);
if (numRead != size)
{
printf("read from file %s failed (%u/%u)\n", filename, (unsigned)numRead, size);
size = 0;
}
}
else
printf("read from file %s failed\n", filename);
unsigned dllLen = size;
_WINREV(dllLen);
socket->write(&dllLen, sizeof(dllLen));
socket->write(buff, size);
free(buff);
}
size32_t CJHVarTreeNode::getSizeAt(unsigned int num) const
{
const char * p = recArray[num];
KEYRECSIZE_T reclen = ((KEYRECSIZE_T *) p)[-1];
_WINREV(reclen);
return reclen;
}
size32_t CJHTreeBlobNode::getTotalBlobSize(unsigned offset)
{
assertex(offset < expandedSize);
unsigned datalen;
memcpy(&datalen, keyBuf+offset, sizeof(datalen));
_WINREV(datalen);
return datalen;
}
offset_t CJHVarTreeNode::getFPosAt(unsigned int num) const
{
if (num >= hdr.numKeys) return 0;
const char * p = recArray[num];
offset_t pos;
memcpy( &pos, p, sizeof(__int64) );
_WINREV(pos);
return pos;
}
bool CJHVarTreeNode::getValueAt(unsigned int num, char *dst) const
{
if (num >= hdr.numKeys) return false;
if (NULL != dst)
{
const char * p = recArray[num];
KEYRECSIZE_T reclen = ((KEYRECSIZE_T *) p)[-1];
_WINREV(reclen);
memcpy(dst, p + sizeof(offset_t), reclen);
}
return true;
}
void CJHVarTreeNode::load(CKeyHdr *_keyHdr, const void *rawData, offset_t _fpos, bool needCopy)
{
CJHTreeNode::load(_keyHdr, rawData, _fpos, needCopy);
unsigned n = getNumKeys();
recArray = new const char * [n];
const char *finger = keyBuf;
for (unsigned int i=0; i<getNumKeys(); i++)
{
recArray[i] = finger + sizeof(KEYRECSIZE_T);
KEYRECSIZE_T recsize = *(KEYRECSIZE_T *)finger;
_WINREV(recsize);
finger += recsize + sizeof(KEYRECSIZE_T) + sizeof(offset_t);
}
}
offset_t CJHTreeNode::getFPosAt(unsigned int index) const
{
if (index >= hdr.numKeys) return 0;
offset_t pos;
if (rowexp.get())
rowexp->expandRow(&pos,index,0,sizeof(pos));
else {
const char * p = keyBuf + index*keyRecLen;
memcpy( &pos, p, sizeof(__int64));
}
_WINREV(pos);
return pos;
}
void writestr(const char *s)
{
size32_t sz;
if (s) {
size32_t l=(size32_t)strlen(s)+1;
sz = l;
_WINREV(sz);
write(&sz,sizeof(sz));
write(s,l);
}
else {
sz = 0;
write(&sz,sizeof(sz));
}
}
char * readstrptr()
{
size32_t sz;
read(&sz,sizeof(sz));
if (sz==0)
return NULL;
_WINREV(sz);
char *s = buff+bpread;
#ifdef _DEBUG
if (bpread+sz>bpwrite) {
assertex(!"buffer overflow");
}
#endif
bpread = align(bpread+sz);
return s;
}
char * readstrdup()
{
size32_t sz;
read(&sz,sizeof(sz));
if (sz==0)
return NULL;
_WINREV(sz);
char *s = buff+bpread;
#ifdef _DEBUG
if (bpread+sz>bpwrite) {
assertex(!"buffer overflow");
}
#endif
bpread = align(bpread+sz);
char *ret = (char *)malloc(sz);
memcpy(ret,s,sz);
return ret;
}
void CJHVarTreeNode::dump()
{
for (unsigned int i=0; i<getNumKeys(); i++)
{
const void * p = recArray[i];
unsigned reclen = ((KEYRECSIZE_T *) p)[-1];
_WINREV(reclen);
unsigned char *dst = (unsigned char *) alloca(reclen);
getValueAt(i,(char *) dst);
offset_t pos = getFPosAt(i);
StringBuffer nodeval;
for (unsigned j = 0; j < reclen; j++)
nodeval.appendf("%02x", dst[j] & 0xff);
DBGLOG("keyVal %d [%" I64F "d] = %s", i, pos, nodeval.str());
}
DBGLOG("==========");
}
extern jhtree_decl void validateKeyFile(const char *filename, offset_t nodePos)
{
OwnedIFile file = createIFile(filename);
OwnedIFileIO io = file->open(IFOread);
if (!io)
throw MakeStringException(1, "Invalid key %s: cannot open file", filename);
unsigned __int64 size = file->size();
if (!size)
throw MakeStringException(2, "Invalid key %s: zero size", filename);
KeyHdr hdr;
if (io->read(0, sizeof(hdr), &hdr) != sizeof(hdr))
throw MakeStringException(4, "Invalid key %s: failed to read key header", filename);
CKeyHdr keyHdr;
keyHdr.load(hdr);
_WINREV(hdr.phyrec);
_WINREV(hdr.root);
_WINREV(hdr.nodeSize);
if (hdr.phyrec != size-1)
throw MakeStringException(5, "Invalid key %s: phyrec was %" I64F "d, expected %" I64F "d", filename, hdr.phyrec, size-1);
if (size % hdr.nodeSize)
throw MakeStringException(3, "Invalid key %s: size %" I64F "d is not a multiple of key node size (%d)", filename, size, hdr.nodeSize);
if (!hdr.root || hdr.root % hdr.nodeSize !=0)
throw MakeStringException(6, "Invalid key %s: invalid root pointer %" I64F "x", filename, hdr.root);
NodeHdr root;
if (io->read(hdr.root, sizeof(root), &root) != sizeof(root))
throw MakeStringException(7, "Invalid key %s: failed to read root node", filename);
_WINREV(root.rightSib);
_WINREV(root.leftSib);
if (root.leftSib || root.rightSib)
throw MakeStringException(8, "Invalid key %s: invalid root node sibling pointers 0x%" I64F "x, 0x%" I64F "x (expected 0,0)", filename, root.leftSib, root.rightSib);
for (offset_t nodeOffset = (nodePos ? nodePos : hdr.nodeSize); nodeOffset < (nodePos ? nodePos+1 : size); nodeOffset += hdr.nodeSize)
{
MemoryAttr ma;
char *buffer = (char *) ma.allocate(hdr.nodeSize);
{
MTIME_SECTION(queryActiveTimer(), "JHTREE read index node");
io->read(nodeOffset, hdr.nodeSize, buffer);
}
CJHTreeNode theNode;
{
MTIME_SECTION(queryActiveTimer(), "JHTREE load index node");
theNode.load(&keyHdr, buffer, nodeOffset, true);
}
NodeHdr *nodeHdr = (NodeHdr *) buffer;
SwapBigEndian(*nodeHdr);
if (!nodeHdr->isValid(hdr.nodeSize))
throw MakeStringException(9, "Invalid key %s: invalid node header at position 0x%" I64F "x", filename, nodeOffset);
if (nodeHdr->leftSib >= size || nodeHdr->rightSib >= size)
throw MakeStringException(9, "Invalid key %s: out of range sibling pointers 0x%" I64F "x, 0x%" I64F "x at position 0x%" I64F "x", filename, nodeHdr->leftSib, nodeHdr->rightSib, nodeOffset);
if (nodeHdr->crc32)
{
unsigned crc = crc32(buffer + sizeof(NodeHdr), nodeHdr->keyBytes, 0);
if (crc != nodeHdr->crc32)
throw MakeStringException(9, "Invalid key %s: crc mismatch at position 0x%" I64F "x", filename, nodeOffset);
}
else
{
// MORE - if we felt so inclined, we could decode the node and check records were in ascending order
}
}
}
inline void SwapBigEndian(KeyHdr &hdr)
{
_WINREV(hdr.phyrec);
_WINREV(hdr.delstk);
_WINREV(hdr.numrec);
_WINREV(hdr.reshdr);
_WINREV(hdr.lstmbr);
_WINREV(hdr.sernum);
_WINREV(hdr.nument);
_WINREV(hdr.root);
_WINREV(hdr.fileid);
_WINREV(hdr.servid);
_WINREV(hdr.verson);
_WINREV(hdr.nodeSize);
_WINREV(hdr.extsiz);
_WINREV(hdr.flmode);
_WINREV(hdr.maxkbl);
_WINREV(hdr.maxkbn);
// _WINREV(hdr.updflg);
// _WINREV(hdr.autodup);
// _WINREV(hdr.deltyp);
// _WINREV(hdr.keypad);
// _WINREV(hdr.flflvr);
// _WINREV(hdr.flalgn);
// _WINREV(hdr.flpntr);
_WINREV(hdr.clstyp);
_WINREV(hdr.length);
_WINREV(hdr.nmem);
_WINREV(hdr.kmem);
_WINREV(hdr.lanchr);
_WINREV(hdr.supid);
_WINREV(hdr.hdrpos);
_WINREV(hdr.sihdr);
_WINREV(hdr.timeid);
_WINREV(hdr.suptyp);
_WINREV(hdr.maxmrk);
_WINREV(hdr.namlen);
_WINREV(hdr.xflmod);
_WINREV(hdr.defrel);
_WINREV(hdr.hghtrn);
_WINREV(hdr.hdrseq);
_WINREV(hdr.tstamp);
_WINREV(hdr.rs3[0]);
_WINREV(hdr.rs3[1]);
_WINREV(hdr.rs3[2]);
_WINREV(hdr.fposOffset);
_WINREV(hdr.fileSize);
_WINREV(hdr.nodeKeyLength);
_WINREV(hdr.version);
_WINREV(hdr.blobHead);
_WINREV(hdr.metadataHead);
}
void checkLevel(int f, KeyHdr &h, unsigned &level, offset_t firstnode)
{
unsigned nodecount = 0;
unsigned maxExpandSize = 0;
unsigned __int64 totalExpandSize = 0;
unsigned __int64 reccount = 0;
unsigned __int64 maxcount = 0;
offset_t thisnode = firstnode;
while (thisnode)
{
_lseeki64(f, thisnode, SEEK_SET);
char *nodeData = (char *) malloc(h.nodeSize);
if (!nodeData || _read(f, nodeData, h.nodeSize) != h.nodeSize)
{
noteError(thisnode, "Could not read node (error %d)\n", errno);
free(nodeData);
return;
}
NodeHdr &nodeHdr = *(NodeHdr *) nodeData;
swap(nodeHdr);
if ( nodeHdr.rightSib > h.phyrec ||
nodeHdr.leftSib > h.phyrec ||
nodeHdr.rightSib % h.nodeSize ||
nodeHdr.leftSib % h.nodeSize ||
nodeHdr.keyBytes == 0 ||
nodeHdr.keyBytes > h.nodeSize )
{
noteError(thisnode, "Htree: Corrupt key node detected (keyBytes==%x)\n", nodeHdr.keyBytes );
}
else if (nodeHdr.crc32 && checkCRC)
{
unsigned crc = mcrc32(nodeData+sizeof(NodeHdr), nodeHdr.keyBytes, 0);
if (nodeHdr.crc32 != crc)
{
noteError(thisnode, "CRC error on key node (keyBytes==%x)\n", nodeHdr.keyBytes );
}
}
if (nodeHdr.leafFlag)
{
if (skipLeafLevel)
{
level++;
free(nodeData);
return;
}
if ((h.ktype & (HTREE_COMPRESSED_KEY|HTREE_QUICK_COMPRESSED_KEY))==HTREE_COMPRESSED_KEY)
{
unsigned expandSize;
memcpy(&expandSize, nodeData+sizeof(NodeHdr)+8, 4);
_WINREV(expandSize);
if (expandSize > maxExpandSize)
maxExpandSize = expandSize;
totalExpandSize += expandSize;
}
}
else
{
unsigned nodeKeyLength = h.nodeKeyLength;
if (nodeKeyLength==(unsigned)-1)
nodeKeyLength = h.length;
unsigned expandSize = nodeKeyLength * nodeHdr.numKeys;
if (expandSize > maxExpandSize)
maxExpandSize = expandSize;
totalExpandSize += expandSize;
if (!nodecount)
{
unsigned __int64 fpos = *(unsigned __int64 *) (nodeData + sizeof(nodeHdr));
_WINREV(fpos);
checkLevel(f, h, level, fpos);
}
}
nodecount++;
reccount+= nodeHdr.numKeys;
if (nodeHdr.numKeys > maxcount)
maxcount = nodeHdr.numKeys;
thisnode = nodeHdr.rightSib;
free(nodeData);
}
printf("%d nodes containing %"I64F"d records expanding to %"I64F"d bytes (average %.2f per node, maximum %"I64F"d, max expand %d) at level %d\n", nodecount, reccount, totalExpandSize, (float) reccount/nodecount, maxcount, maxExpandSize, level);
level++;
}
int readResults(ISocket * socket, bool readBlocked, bool useHTTP, StringBuffer &result)
{
if (readBlocked)
socket->set_block_mode(BF_SYNC_TRANSFER_PULL,0,60*1000);
unsigned len;
bool is_status;
bool isBlockedResult;
for (;;)
{
if (delay)
MilliSleep(delay);
is_status = false;
isBlockedResult = false;
try
{
if (useHTTP)
len = 0x10000;
else if (readBlocked)
len = socket->receive_block_size();
else
{
socket->read(&len, sizeof(len));
_WINREV(len);
}
}
catch(IException * e)
{
if (manyResults)
showMessage("End of result multiple set\n");
else
pexception("failed to read len data", e);
e->Release();
return 1;
}
if (len == 0)
{
if (manyResults)
{
showMessage("----End of result set----\n");
continue;
}
break;
}
bool isSpecial = false;
bool pluginRequest = false;
bool dataBlockRequest = false;
if (len & 0x80000000)
{
unsigned char flag;
isSpecial = true;
socket->read(&flag, sizeof(flag));
switch (flag)
{
case '-':
if (echoResults)
fputs("Error:", stdout);
if (saveResults && trace != NULL)
fputs("Error:", trace);
break;
case 'D':
showMessage("request for datablock\n");
dataBlockRequest = true;
break;
case 'P':
showMessage("request for plugin\n");
pluginRequest = true;
break;
case 'S':
if (showStatus)
showMessage("Status:");
is_status=true;
break;
case 'T':
showMessage("Timing:\n");
break;
case 'X':
showMessage("---Compound query finished---\n");
return 1;
case 'R':
isBlockedResult = true;
break;
}
len &= 0x7FFFFFFF;
len--; // flag already read
}
char * mem = (char*) malloc(len+1);
char * t = mem;
unsigned sendlen = len;
t[len]=0;
try
{
if (useHTTP)
{
try
{
socket->read(t, 0, len, sendlen);
}
catch (IException *E)
{
if (E->errorCode()!= JSOCKERR_graceful_close)
throw;
E->Release();
break;
}
if (!sendlen)
break;
}
else if (readBlocked)
socket->receive_block(t, len);
else
socket->read(t, len);
}
catch(IException * e)
{
pexception("failed to read data", e);
return 1;
}
if (pluginRequest)
{
//Not very robust! A poor man's implementation for testing...
StringBuffer dllname, libname;
const char * dot = strchr(t, '.');
dllname.append("\\edata\\bin\\debug\\").append(t);
libname.append("\\edata\\bin\\debug\\").append(dot-t,t).append(".lib");
sendFile(dllname.str(), socket);
sendFile(libname.str(), socket);
}
else if (dataBlockRequest)
{
//Not very robust! A poor man's implementation for testing...
offset_t offset;
memcpy(&offset, t, sizeof(offset));
_WINREV(offset);
sendFileChunk(t+sizeof(offset), offset, socket);
}
else
{
if (isBlockedResult)
{
t += 8;
t += strlen(t)+1;
sendlen -= (t - mem);
}
if (echoResults && (!is_status || showStatus))
{
fwrite(t, sendlen, 1, stdout);
fflush(stdout);
}
if (!is_status)
result.append(sendlen, t);
}
free(mem);
if (abortAfterFirst)
return 0;
}
return 0;
}
int readResults(ISocket * socket, bool readBlocked, bool useHTTP, StringBuffer &result, const char *query, size32_t queryLen)
{
if (readBlocked)
socket->set_block_mode(BF_SYNC_TRANSFER_PULL,0,60*1000);
MemoryBuffer remoteReadCursorMb;
unsigned len;
bool is_status;
bool isBlockedResult;
for (;;)
{
if (delay)
MilliSleep(delay);
is_status = false;
isBlockedResult = false;
try
{
if (useHTTP)
len = 0x10000;
else if (readBlocked)
len = socket->receive_block_size();
else
{
socket->read(&len, sizeof(len));
_WINREV(len);
}
}
catch(IException * e)
{
if (manyResults)
showMessage("End of result multiple set\n");
else
pexception("failed to read len data", e);
e->Release();
return 1;
}
if (len == 0)
{
if (manyResults)
{
showMessage("----End of result set----\n");
continue;
}
break;
}
bool isSpecial = false;
bool pluginRequest = false;
bool dataBlockRequest = false;
bool remoteReadRequest = false;
if (len & 0x80000000)
{
unsigned char flag;
isSpecial = true;
socket->read(&flag, sizeof(flag));
switch (flag)
{
case '-':
if (echoResults)
fputs("Error:", stdout);
if (saveResults && trace != NULL)
fputs("Error:", trace);
break;
case 'D':
showMessage("request for datablock\n");
dataBlockRequest = true;
break;
case 'P':
showMessage("request for plugin\n");
pluginRequest = true;
break;
case 'S':
if (showStatus)
showMessage("Status:");
is_status=true;
break;
case 'T':
showMessage("Timing:\n");
break;
case 'X':
showMessage("---Compound query finished---\n");
return 1;
case 'R':
isBlockedResult = true;
break;
case 'J':
remoteReadRequest = true;
break;
}
len &= 0x7FFFFFFF;
len--; // flag already read
}
MemoryBuffer mb;
mb.setEndian(__BIG_ENDIAN);
char *mem = (char *)mb.reserveTruncate(len+1);
char * t = mem;
size32_t sendlen = len;
t[len]=0;
try
{
if (useHTTP)
{
try
{
socket->read(t, 0, len, sendlen);
}
catch (IException *E)
{
if (E->errorCode()!= JSOCKERR_graceful_close)
throw;
E->Release();
break;
}
if (!sendlen)
break;
}
else if (readBlocked)
socket->receive_block(t, len);
else
socket->read(t, len);
}
catch(IException * e)
{
pexception("failed to read data", e);
e->Release();
return 1;
}
if (pluginRequest)
{
//Not very robust! A poor man's implementation for testing...
StringBuffer dllname, libname;
const char * dot = strchr(t, '.');
dllname.append("\\edata\\bin\\debug\\").append(t);
libname.append("\\edata\\bin\\debug\\").append(dot-t,t).append(".lib");
sendFile(dllname.str(), socket);
sendFile(libname.str(), socket);
}
else if (dataBlockRequest)
{
//Not very robust! A poor man's implementation for testing...
offset_t offset;
mb.read(offset);
sendFileChunk((const char *)mb.readDirect(offset), offset, socket);
}
else if (remoteReadRequest)
{
Owned<IPropertyTree> requestTree = createPTreeFromJSONString(queryLen, query);
Owned<IPropertyTree> responseTree; // used if response is xml or json
const char *outputFmtStr = requestTree->queryProp("format");
const char *response = nullptr;
if (!outputFmtStr || strieq("xml", outputFmtStr))
{
response = (const char *)mb.readDirect(len);
responseTree.setown(createPTreeFromXMLString(len, response));
}
else if (strieq("json", outputFmtStr))
{
response = (const char *)mb.readDirect(len);
responseTree.setown(createPTreeFromJSONString(len, response));
}
unsigned cursorHandle;
if (responseTree)
cursorHandle = responseTree->getPropInt("cursor");
else
mb.read(cursorHandle);
bool retrySend = false;
if (cursorHandle)
{
PROGLOG("Got handle back: %u; len=%u", cursorHandle, len);
StringBuffer xml;
if (responseTree)
{
if (echoResults && response)
{
fputs(response, stdout);
fflush(stdout);
}
if (!responseTree->getPropBin("cursorBin", remoteReadCursorMb.clear()))
break;
}
else
{
size32_t dataLen;
mb.read(dataLen);
if (!dataLen)
break;
const void *rowData = mb.readDirect(dataLen);
// JCSMORE - output binary row data?
// cursor
size32_t cursorLen;
mb.read(cursorLen);
if (!cursorLen)
break;
const void *cursor = mb.readDirect(cursorLen);
memcpy(remoteReadCursorMb.clear().reserveTruncate(cursorLen), cursor, cursorLen);
}
if (remoteStreamForceResend)
cursorHandle = NotFound; // fake that it's a handle dafilesrv doesn't know about
Owned<IPropertyTree> requestTree = createPTree();
requestTree->setPropInt("cursor", cursorHandle);
// Only the handle is needed for continuation, but this tests the behaviour of some clients which may send cursor per request (e.g. to refresh)
if (remoteStreamSendCursor)
requestTree->setPropBin("cursorBin", remoteReadCursorMb.length(), remoteReadCursorMb.toByteArray());
requestTree->setProp("format", outputFmtStr);
StringBuffer requestStr;
toJSON(requestTree, requestStr);
#ifdef _DEBUG
fputs(requestStr, stdout);
#endif
sendlen = requestStr.length();
_WINREV(sendlen);
try
{
if (!rawSend && !useHTTP)
socket->write(&sendlen, sizeof(sendlen));
socket->write(requestStr.str(), requestStr.length());
}
catch (IJSOCK_Exception *e)
{
retrySend = true;
EXCLOG(e, nullptr);
e->Release();
}
}
else // dafilesrv didn't know who I was, resent query + serialized cursor
retrySend = true;
if (retrySend)
{
PROGLOG("Retry send for handle: %u", cursorHandle);
requestTree->setPropBin("cursorBin", remoteReadCursorMb.length(), remoteReadCursorMb.toByteArray());
StringBuffer requestStr;
toJSON(requestTree, requestStr);
PROGLOG("requestStr = %s", requestStr.str());
sendlen = requestStr.length();
_WINREV(sendlen);
if (!rawSend && !useHTTP)
socket->write(&sendlen, sizeof(sendlen));
socket->write(requestStr.str(), requestStr.length());
}
}
else
{
if (isBlockedResult)
{
t += 8;
t += strlen(t)+1;
sendlen -= (t - mem);
}
if (echoResults && (!is_status || showStatus))
{
fwrite(t, sendlen, 1, stdout);
fflush(stdout);
}
if (!is_status)
result.append(sendlen, t);
}
if (abortAfterFirst)
return 0;
}
return 0;
}
void winrev() { _WINREV(pos); _WINREV(numrecs); _WINREV(recsize); _WINREV(id); }
bool CWriteNode::add(offset_t pos, const void *indata, size32_t insize, unsigned __int64 sequence)
{
if (isLeaf() && !hdr.numKeys)
{
unsigned __int64 rsequence = sequence;
_WINREV(rsequence);
memcpy(keyPtr, &rsequence, sizeof(rsequence));
keyPtr += sizeof(rsequence);
hdr.keyBytes += sizeof(rsequence);
}
#if 0
// This test is no longer valid if we don't treat all fields as keyed
if (hdr.numKeys)
{
if (memcmp(indata, lastKeyValue, keyedSize) < 0)
{
// dump out the rows in question
StringBuffer hex;
unsigned i;
for (i = 0; i < insize; i++)
{
hex.appendf("%02x ", ((unsigned char *) indata)[i]);
}
DBGLOG("this: %s", hex.str());
hex.clear();
for (i = 0; i < insize; i++)
{
hex.appendf("%02x ", ((unsigned char *) lastKeyValue)[i]);
}
DBGLOG("last: %s", hex.str());
hex.clear();
for (i = 0; i < insize; i++)
{
unsigned char c = ((unsigned char *) indata)[i];
hex.appendf("%c", isprint(c) ? c : '.');
}
DBGLOG("this: %s", hex.str());
hex.clear();
for (i = 0; i < insize; i++)
{
unsigned char c = ((unsigned char *) lastKeyValue)[i];
hex.appendf("%c", isprint(c) ? c : '.');
}
DBGLOG("last: %s", hex.str());
throw MakeStringException(0, "Data written to key must be in sorted order");
}
}
#endif
if (isLeaf() && keyType & HTREE_COMPRESSED_KEY)
{
if (0 == hdr.numKeys)
lzwcomp.open(keyPtr, maxBytes-hdr.keyBytes, isVariable, (keyType&HTREE_QUICK_COMPRESSED_KEY)==HTREE_QUICK_COMPRESSED_KEY);
if (0xffff == hdr.numKeys || 0 == lzwcomp.writekey(pos, (const char *)indata, insize, sequence))
{
lzwcomp.close();
return false;
}
hdr.keyBytes = lzwcomp.buflen() + sizeof(unsigned __int64); // rsequence added above
}
else
{
if (0xffff == hdr.numKeys)
return false;
bool lastnode = false;
if (!indata)
{
lastnode = true;
indata = alloca(insize);
memset((void *) indata, 0xff, insize);
}
//assertex(insize==keyLen);
const void *data;
int size;
if (keyType & COL_PREFIX)
{
char *result = (char *) alloca(insize+1); // Gets bigger if no leading common!
size = compressValue((const char *) indata, insize, result);
data = result;
}
else
{
size = insize;
data = indata;
}
int bytes = sizeof(pos) + size;
if (isVariable)
bytes += sizeof(KEYRECSIZE_T);
if (hdr.keyBytes + bytes >= maxBytes) // probably could be '>' (loses byte)
return false;
if (isVariable && isLeaf())
{
KEYRECSIZE_T _insize = insize;
_WINREV(_insize);
memcpy(keyPtr, &_insize, sizeof(_insize));
keyPtr += sizeof(_insize);
}
_WINREV(pos);
memcpy(keyPtr, &pos, sizeof(pos));
keyPtr += sizeof(pos);
memcpy(keyPtr, data, size);
keyPtr += size;
hdr.keyBytes += bytes;
}
if (insize>keyLen)
throw MakeStringException(0, "key+payload (%u) exceeds max length (%u)", insize, keyLen);
memcpy(lastKeyValue, indata, insize);
lastSequence = sequence;
hdr.numKeys++;
return true;
}
void CJHTreeNode::unpack(const void *node, bool needCopy)
{
memcpy(&hdr, node, sizeof(hdr));
SwapBigEndian(hdr);
__int64 maxsib = keyHdr->getHdrStruct()->phyrec;
if (!hdr.isValid(keyHdr->getNodeSize()))
{
PROGLOG("hdr.leafFlag=%d",(int)hdr.leafFlag);
PROGLOG("hdr.rightSib=%" I64F "d",hdr.rightSib);
PROGLOG("hdr.leftSib=%" I64F "d",hdr.leftSib);
PROGLOG("maxsib=%" I64F "d",maxsib);
PROGLOG("nodeSize=%d", keyHdr->getNodeSize());
PROGLOG("keyBytes=%d",(int)hdr.keyBytes);
PrintStackReport();
throw MakeStringException(0, "Htree: Corrupt key node detected");
}
if (!hdr.leafFlag)
keyLen = keyHdr->getNodeKeyLength();
keyRecLen = keyLen + sizeof(offset_t);
char *keys = ((char *) node) + sizeof(hdr);
if (hdr.crc32)
{
unsigned crc = crc32(keys, hdr.keyBytes, 0);
if (hdr.crc32 != crc)
throw MakeStringException(0, "CRC error on key node");
}
if (hdr.leafFlag==1)
{
firstSequence = *(unsigned __int64 *) keys;
keys += sizeof(unsigned __int64);
_WINREV(firstSequence);
}
if(isMetadata())
{
unsigned short len = *reinterpret_cast<unsigned short *>(keys);
_WINREV(len);
expandedSize = len;
keyBuf = (char *) allocMem(len);
memcpy(keyBuf, keys+sizeof(unsigned short), len);
}
else if (isLeaf() && (keyType & HTREE_COMPRESSED_KEY))
{
{
MTIME_SECTION(queryActiveTimer(), "Compressed node expand");
expandedSize = keyHdr->getNodeSize();
bool quick = (keyType&HTREE_QUICK_COMPRESSED_KEY)==HTREE_QUICK_COMPRESSED_KEY;
#ifndef _OLD_VERSION
keyBuf = NULL;
if (quick)
rowexp.setown(expandQuickKeys(keys, needCopy));
if (!quick||!rowexp.get())
#endif
{
keyBuf = expandKeys(keys,keyLen,expandedSize,quick);
}
}
assertex(keyBuf||rowexp.get());
}
else
{
int i;
if (keyType & COL_PREFIX)
{
MTIME_SECTION(queryActiveTimer(), "COL_PREFIX expand");
if (hdr.numKeys) {
bool handleVariable = isVariable && isLeaf();
KEYRECSIZE_T workRecLen;
MemoryBuffer keyBufMb;
const char *source = keys;
char *target;
// do first row
if (handleVariable) {
memcpy(&workRecLen, source, sizeof(workRecLen));
_WINREV(workRecLen);
size32_t tmpSz = sizeof(workRecLen) + sizeof(offset_t);
target = (char *)keyBufMb.reserve(tmpSz+workRecLen);
memcpy(target, source, tmpSz);
source += tmpSz;
target += tmpSz;
}
else {
target = (char *)keyBufMb.reserveTruncate(hdr.numKeys * keyRecLen);
workRecLen = keyRecLen - sizeof(offset_t);
memcpy(target, source, sizeof(offset_t));
source += sizeof(offset_t);
target += sizeof(offset_t);
}
// this is where next row gets data from
const char *prev, *next = NULL;
unsigned prevOffset = 0;
if (handleVariable)
prevOffset = target-((char *)keyBufMb.bufferBase());
else
next = target;
unsigned char pack1 = *source++;
#ifdef _DEBUG
assertex(0==pack1); // 1st time will be always be 0
#endif
KEYRECSIZE_T left = workRecLen;
while (left--) {
*target = *source;
source++;
target++;
}
// do subsequent rows
for (i = 1; i < hdr.numKeys; i++) {
if (handleVariable) {
memcpy(&workRecLen, source, sizeof(workRecLen));
_WINREV(workRecLen);
target = (char *)keyBufMb.reserve(sizeof(workRecLen)+sizeof(offset_t)+workRecLen);
size32_t tmpSz = sizeof(workRecLen)+sizeof(offset_t);
memcpy(target, source, tmpSz);
target += tmpSz;
source += tmpSz;
}
else
{
memcpy(target, source, sizeof(offset_t));
source += sizeof(offset_t);
target += sizeof(offset_t);
}
pack1 = *source++;
#ifdef _DEBUG
assertex(pack1<=workRecLen);
#endif
if (handleVariable) {
prev = ((char *)keyBufMb.bufferBase())+prevOffset;
// for next
prevOffset = target-((char *)keyBufMb.bufferBase());
}
else {
prev = next;
next = target;
}
left = workRecLen - pack1;
while (pack1--) {
*target = *prev;
prev++;
target++;
}
while (left--) {
*target = *source;
source++;
target++;
}
}
expandedSize = keyBufMb.length();
keyBuf = (char *)keyBufMb.detach();
assertex(keyBuf);
}
else {
keyBuf = NULL;
expandedSize = 0;
}
}
else
{
MTIME_SECTION(queryActiveTimer(), "NO compression copy");
expandedSize = hdr.keyBytes + sizeof( __int64 ); // MORE - why is the +sizeof() there?
keyBuf = (char *) allocMem(expandedSize);
memcpy(keyBuf, keys, hdr.keyBytes + sizeof( __int64 ));
}
}
}
unsigned countLevels(int f, KeyHdr &h, offset_t firstnode)
{
_lseeki64(f, firstnode, SEEK_SET);
char *nodeData = (char *) malloc(h.nodeSize);
if (!nodeData || _read(f, nodeData, h.nodeSize) != h.nodeSize)
{
noteError(firstnode, "Could not read node (error %d)\n", errno);
free(nodeData);
return 0;
}
else
{
NodeHdr &nodeHdr = *(NodeHdr *) nodeData;
swap(nodeHdr);
if (!nodeHdr.leafFlag)
{
unsigned __int64 fpos = *(unsigned __int64 *) (nodeData + sizeof(nodeHdr));
_WINREV(fpos);
free(nodeData);
return countLevels(f, h, fpos)+1;
}
else
{
free(nodeData);
return 1;
}
}
}
void sendFileChunk(const char * filename, offset_t offset, ISocket * socket)
{
FILE *in = fopen(filename, "rb");
unsigned size = 0;
void * buff = NULL;
if (in)
{
fseek(in, 0, SEEK_END);
offset_t endOffset = ftell(in);
fseek(in, offset, SEEK_SET);
if (endOffset < offset)
size = 0;
else
size = (unsigned)(endOffset - offset);
if (size > CHUNK_SIZE)
size = CHUNK_SIZE;
buff = malloc(size);
size_t numRead = fread(buff, 1, size, in);
fclose(in);
if (numRead != size)
{
printf("read from file %s failed (%u/%u)\n", filename, (unsigned)numRead, size);
size = 0;
}
}
else
printf("read from file %s failed\n", filename);
if (size > 0)
{
MemoryBuffer sendBuffer;
unsigned rev = size + strlen(filename) + 10;
rev |= 0x80000000;
_WINREV(rev);
sendBuffer.append(rev);
sendBuffer.append('R');
rev = 0; // should put the sequence number here
_WINREV(rev);
sendBuffer.append(rev);
rev = 0; // should put the # of recs in msg here
_WINREV(rev);
sendBuffer.append(rev);
sendBuffer.append(strlen(filename)+1, filename);
sendBuffer.append(size, buff);
socket->write(sendBuffer.toByteArray(), sendBuffer.length());
}
else
{
unsigned zeroLen = 0;
socket->write(&zeroLen, sizeof(zeroLen));
}
free(buff);
}
