void LZMALZDecompressToBuffer(MemoryAttr & out, MemoryBuffer & in) { size32_t expsz; size32_t cmpsz; in.read(expsz).read(cmpsz); void *o = out.allocate(expsz); if (cmpsz!=expsz) { CLZMA lzma; size32_t written = lzma.expand(in.readDirect(cmpsz),cmpsz,o,expsz); if (written!=expsz) throw MakeStringException(0, "fastLZDecompressToBuffer - corrupt data(4) %d %d",written,expsz); } else memcpy(o,in.readDirect(cmpsz),expsz); }
CThorException(MemoryBuffer &mb) { mb.read((unsigned &)action); mb.read(jobId); mb.read(graphName); mb.read(graphId); mb.read((unsigned &)kind); mb.read(id); mb.read(slave); mb.read((unsigned &)audience); mb.read(errorcode); mb.read(msg); mb.read(file); mb.read(line); mb.read(column); mb.read((int &)severity); mb.read(origin); if (0 == origin.length()) // simpler to clear serialized 0 length terminated string here than check on query origin.clear(); bool oe; mb.read(oe); if (oe) originalException.setown(deserializeThorException(mb)); size32_t sz; mb.read(sz); if (sz) data.append(sz, mb.readDirect(sz)); }
size32_t aesDecryptWithRSAEncryptedKey(MemoryBuffer &out, size32_t inSz, const void *inBytes, const CLoadedKey &privateKey) { MemoryBuffer in; in.setBuffer(inSz, (void *)inBytes, false); // read encrypted AES key size32_t encryptedAESKeySz; in.read(encryptedAESKeySz); MemoryBuffer aesKey; size32_t decryptedAesKeySz = privateKeyDecrypt(aesKey, encryptedAESKeySz, in.readDirect(encryptedAESKeySz), privateKey); if (decryptedAesKeySz != aesMaxKeySize) throw makeStringException(0, "aesDecryptWithRSAEncryptedKey - invalid input"); unsigned iVPos = in.getPos(); // read directly further down in.skip(aesBlockSize); size32_t aesEncryptedSz; in.read(aesEncryptedSz); return aesDecrypt(out, aesEncryptedSz, in.readDirect(aesEncryptedSz), aesMaxKeySize, (const char *)aesKey.bytes(), (const char *)in.bytes()+iVPos); }
VariableRowBlock::VariableRowBlock(MemoryBuffer & inBuffer, __int64 _start) : RowBlock(_start, 0) { inBuffer.read(numRows); for (unsigned row = 0; row < numRows; row++) { unsigned thisLength; rowIndex.append(buffer.length()); inBuffer.read(thisLength); buffer.append(thisLength, inBuffer.readDirect(thisLength)); } rowIndex.append(buffer.length()); }
CThorException(MemoryBuffer &mb) { mb.read((unsigned &)action); mb.read(jobId); mb.read(graphId); mb.read((unsigned &)kind); mb.read(id); mb.read((unsigned &)audience); mb.read(errorcode); mb.read(msg); mb.read(file); mb.read(line); mb.read(column); mb.read((int &)severity); mb.read(origin); size32_t sz; mb.read(sz); if (sz) data.append(sz, mb.readDirect(sz)); }
size32_t ThorExpand(MemoryBuffer & src, MemoryBuffer & dest) { size32_t len = src.remaining(); const void *pSrc = src.readDirect(len); return ThorExpand(pSrc, len, dest); }
int readResults(ISocket * socket, bool readBlocked, bool useHTTP, StringBuffer &result, const char *query, size32_t queryLen) { if (readBlocked) socket->set_block_mode(BF_SYNC_TRANSFER_PULL,0,60*1000); MemoryBuffer remoteReadCursorMb; unsigned len; bool is_status; bool isBlockedResult; for (;;) { if (delay) MilliSleep(delay); is_status = false; isBlockedResult = false; try { if (useHTTP) len = 0x10000; else if (readBlocked) len = socket->receive_block_size(); else { socket->read(&len, sizeof(len)); _WINREV(len); } } catch(IException * e) { if (manyResults) showMessage("End of result multiple set\n"); else pexception("failed to read len data", e); e->Release(); return 1; } if (len == 0) { if (manyResults) { showMessage("----End of result set----\n"); continue; } break; } bool isSpecial = false; bool pluginRequest = false; bool dataBlockRequest = false; bool remoteReadRequest = false; if (len & 0x80000000) { unsigned char flag; isSpecial = true; socket->read(&flag, sizeof(flag)); switch (flag) { case '-': if (echoResults) fputs("Error:", stdout); if (saveResults && trace != NULL) fputs("Error:", trace); break; case 'D': showMessage("request for datablock\n"); dataBlockRequest = true; break; case 'P': showMessage("request for plugin\n"); pluginRequest = true; break; case 'S': if (showStatus) showMessage("Status:"); is_status=true; break; case 'T': showMessage("Timing:\n"); break; case 'X': showMessage("---Compound query finished---\n"); return 1; case 'R': isBlockedResult = true; break; case 'J': remoteReadRequest = true; break; } len &= 0x7FFFFFFF; len--; // flag already read } MemoryBuffer mb; mb.setEndian(__BIG_ENDIAN); char *mem = (char *)mb.reserveTruncate(len+1); char * t = mem; size32_t sendlen = len; t[len]=0; try { if (useHTTP) { try { socket->read(t, 0, len, sendlen); } catch (IException *E) { if (E->errorCode()!= JSOCKERR_graceful_close) throw; E->Release(); break; } if (!sendlen) break; } else if (readBlocked) socket->receive_block(t, len); else socket->read(t, len); } catch(IException * e) { pexception("failed to read data", e); e->Release(); return 1; } if (pluginRequest) { //Not very robust! A poor man's implementation for testing... StringBuffer dllname, libname; const char * dot = strchr(t, '.'); dllname.append("\\edata\\bin\\debug\\").append(t); libname.append("\\edata\\bin\\debug\\").append(dot-t,t).append(".lib"); sendFile(dllname.str(), socket); sendFile(libname.str(), socket); } else if (dataBlockRequest) { //Not very robust! A poor man's implementation for testing... offset_t offset; mb.read(offset); sendFileChunk((const char *)mb.readDirect(offset), offset, socket); } else if (remoteReadRequest) { Owned<IPropertyTree> requestTree = createPTreeFromJSONString(queryLen, query); Owned<IPropertyTree> responseTree; // used if response is xml or json const char *outputFmtStr = requestTree->queryProp("format"); const char *response = nullptr; if (!outputFmtStr || strieq("xml", outputFmtStr)) { response = (const char *)mb.readDirect(len); responseTree.setown(createPTreeFromXMLString(len, response)); } else if (strieq("json", outputFmtStr)) { response = (const char *)mb.readDirect(len); responseTree.setown(createPTreeFromJSONString(len, response)); } unsigned cursorHandle; if (responseTree) cursorHandle = responseTree->getPropInt("cursor"); else mb.read(cursorHandle); bool retrySend = false; if (cursorHandle) { PROGLOG("Got handle back: %u; len=%u", cursorHandle, len); StringBuffer xml; if (responseTree) { if (echoResults && response) { fputs(response, stdout); fflush(stdout); } if (!responseTree->getPropBin("cursorBin", remoteReadCursorMb.clear())) break; } else { size32_t dataLen; mb.read(dataLen); if (!dataLen) break; const void *rowData = mb.readDirect(dataLen); // JCSMORE - output binary row data? // cursor size32_t cursorLen; mb.read(cursorLen); if (!cursorLen) break; const void *cursor = mb.readDirect(cursorLen); memcpy(remoteReadCursorMb.clear().reserveTruncate(cursorLen), cursor, cursorLen); } if (remoteStreamForceResend) cursorHandle = NotFound; // fake that it's a handle dafilesrv doesn't know about Owned<IPropertyTree> requestTree = createPTree(); requestTree->setPropInt("cursor", cursorHandle); // Only the handle is needed for continuation, but this tests the behaviour of some clients which may send cursor per request (e.g. to refresh) if (remoteStreamSendCursor) requestTree->setPropBin("cursorBin", remoteReadCursorMb.length(), remoteReadCursorMb.toByteArray()); requestTree->setProp("format", outputFmtStr); StringBuffer requestStr; toJSON(requestTree, requestStr); #ifdef _DEBUG fputs(requestStr, stdout); #endif sendlen = requestStr.length(); _WINREV(sendlen); try { if (!rawSend && !useHTTP) socket->write(&sendlen, sizeof(sendlen)); socket->write(requestStr.str(), requestStr.length()); } catch (IJSOCK_Exception *e) { retrySend = true; EXCLOG(e, nullptr); e->Release(); } } else // dafilesrv didn't know who I was, resent query + serialized cursor retrySend = true; if (retrySend) { PROGLOG("Retry send for handle: %u", cursorHandle); requestTree->setPropBin("cursorBin", remoteReadCursorMb.length(), remoteReadCursorMb.toByteArray()); StringBuffer requestStr; toJSON(requestTree, requestStr); PROGLOG("requestStr = %s", requestStr.str()); sendlen = requestStr.length(); _WINREV(sendlen); if (!rawSend && !useHTTP) socket->write(&sendlen, sizeof(sendlen)); socket->write(requestStr.str(), requestStr.length()); } } else { if (isBlockedResult) { t += 8; t += strlen(t)+1; sendlen -= (t - mem); } if (echoResults && (!is_status || showStatus)) { fwrite(t, sendlen, 1, stdout); fflush(stdout); } if (!is_status) result.append(sendlen, t); } if (abortAfterFirst) return 0; } return 0; }