void slaveDone(size32_t slaveIdx, MemoryBuffer &mb) { if (mb.length()) // if 0 implies aborted out from this slave. { rowcount_t slaveProcessed; mb.read(slaveProcessed); recordsProcessed += slaveProcessed; offset_t size, physicalSize; mb.read(size); mb.read(physicalSize); unsigned fileCrc; mb.read(fileCrc); CDateTime modifiedTime(mb); StringBuffer timeStr; modifiedTime.getString(timeStr); IPartDescriptor *partDesc = fileDesc->queryPart(slaveIdx); IPropertyTree &props = partDesc->queryProperties(); props.setPropInt64("@size", size); if (fileDesc->isCompressed()) props.setPropInt64("@compressedSize", physicalSize); props.setPropInt("@fileCrc", fileCrc); props.setProp("@modified", timeStr.str()); } }
// IThorSlaveActivity void CDiskReadSlaveActivityBase::init(MemoryBuffer &data, MemoryBuffer &slaveData) { subfileLogicalFilenames.kill(); partDescs.kill(); data.read(logicalFilename); unsigned subfiles; data.read(subfiles); unsigned s=0; for (; s<subfiles; s++) { StringAttr subfile; data.read(subfile); subfileLogicalFilenames.append(subfile); } unsigned parts; data.read(parts); if (parts) { deserializePartFileDescriptors(data, partDescs); // put temp files in individual slave temp dirs (incl port) if ((helper->getFlags() & TDXtemporary) && (!container.queryJob().queryUseCheckpoints())) partDescs.item(0).queryOwner().setDefaultDir(queryTempDir()); } }
void CDiskWriteSlaveActivityBase::init(MemoryBuffer &data, MemoryBuffer &slaveData) { StringAttr logicalFilename; data.read(logicalFilename); dlfn.set(logicalFilename); if (diskHelperBase->getFlags() & TDXtemporary) data.read(usageCount); if (diskHelperBase->getFlags() & TDWextend) { assertex(!"TBD extended CRC broken"); unsigned crc; if (partDesc->getCrc(crc)) fileCRC.reset(~crc); } partDesc.setown(deserializePartFileDescriptor(data)); // put temp files in individual slave temp dirs (incl port) if ((diskHelperBase->getFlags() & TDXtemporary) && (!container.queryJob().queryUseCheckpoints())) partDesc->queryOwner().setDefaultDir(queryTempDir()); if (dlfn.isExternal()) { mpTag = container.queryJobChannel().deserializeMPTag(data); if (dlfn.isQuery() && (0 != container.queryJob().getWorkUnitValueInt("rfsqueryparallel", 0))) rfsQueryParallel = true; } if (0 != (diskHelperBase->getFlags() & TDXgrouped)) grouped = true; }
void CDiskWriteSlaveActivityBase::init(MemoryBuffer &data, MemoryBuffer &slaveData) { diskHelperBase = static_cast <IHThorDiskWriteArg *> (queryHelper()); data.read(logicalFilename); dlfn.set(logicalFilename); if (diskHelperBase->getFlags() & TDXtemporary) data.read(usageCount); if (diskHelperBase->getFlags() & TDWextend) { assertex(!"TBD extended CRC broken"); unsigned crc; if (partDesc->getCrc(crc)) fileCRC.reset(~crc); } partDesc.setown(deserializePartFileDescriptor(data)); if (dlfn.isExternal()) { mpTag = container.queryJob().deserializeMPTag(data); if (dlfn.isQuery() && (0 != container.queryJob().getWorkUnitValueInt("rfsqueryparallel", 0))) rfsQueryParallel = true; } if (0 != (diskHelperBase->getFlags() & TDXgrouped)) grouped = true; compress = partDesc->queryOwner().isCompressed(); void *ekey; size32_t ekeylen; diskHelperBase->getEncryptKey(ekeylen,ekey); if (ekeylen!=0) { ecomp.setown(createAESCompressor256(ekeylen,ekey)); memset(ekey,0,ekeylen); free(ekey); compress = true; } }
void FileFormat::deserialize(MemoryBuffer & in) { byte tempType; in.read(tempType); type = (FileFormatType)tempType; switch (type) { case FFTfixed: case FFTblocked: in.read(recordSize); break; case FFTcsv: case FFTutf: case FFTutf8: case FFTutf8n: case FFTutf16: case FFTutf16be: case FFTutf16le: case FFTutf32: case FFTutf32be: case FFTutf32le: in.read(maxRecordSize); ::deserialize(in, separate); ::deserialize(in, quote); ::deserialize(in, terminate); ::deserialize(in, escape); ::deserialize(in, rowTag); break; } }
void RemoteDataSourceServer::doCmdCreateWorkunit(MemoryBuffer & in, MemoryBuffer & out) { SessionId session; StringAttr wuid, username, password; unsigned sequence; StringAttr name; in.read(session); in.read(username).read(password); in.read(wuid); in.read(sequence); in.read(name); DBGLOG("RemoteFileView:CreateWorkunit('%s',%d,'%s') by[%s:%" I64F "d", wuid.get(), sequence, name ? name.get() : "", username.get(), session); Owned<IConstWUResult> wuResult = resolveResult(wuid, sequence, name); Owned<IFvDataSource> ds = createDataSource(wuResult, wuid, username, password); unique_id_t id = addDataSource(session, ds); out.append((unsigned short)CurRemoteVersion); out.append(id); out.append(ds->numRows(false)); ds->queryMetaData()->serialize(out); out.append(ds->isIndex()); DBGLOG("RemoteFileView:CreateWorkunit returns %" I64F "d", id); }
DataSourceMetaItem::DataSourceMetaItem(unsigned _flags, MemoryBuffer & in) { flags = _flags; in.read(name); in.read(xpath); type.setown(deserializeType(in)); }
void FileFormat::deserialize(MemoryBuffer & in) { byte tempType; in.read(tempType); type = (FileFormatType)tempType; switch (type) { case FFTfixed: case FFTblocked: in.read(recordSize); break; case FFTcsv: case FFTutf: case FFTutf8: case FFTutf8n: case FFTutf16: case FFTutf16be: case FFTutf16le: case FFTutf32: case FFTutf32be: case FFTutf32le: in.read(maxRecordSize); ::deserialize(in, separate); ::deserialize(in, quote); ::deserialize(in, terminate); ::deserialize(in, rowTag); updateMarkupType(rowTag, NULL); //neither kind nor markup currently serialized. may add later break; } }
// IThorSlaveActivity void CDiskReadSlaveActivityBase::init(MemoryBuffer &data, MemoryBuffer &slaveData) { subfileLogicalFilenames.kill(); partDescs.kill(); data.read(logicalFilename); unsigned subfiles; data.read(subfiles); unsigned s=0; for (; s<subfiles; s++) { StringAttr subfile; data.read(subfile); subfileLogicalFilenames.append(subfile); } unsigned parts; data.read(parts); if (parts) { deserializePartFileDescriptors(data, partDescs); unsigned encryptedKeyLen; void *encryptedKey; helper->getEncryptKey(encryptedKeyLen, encryptedKey); if (0 != encryptedKeyLen) { bool dfsEncrypted = partDescs.item(0).queryOwner().queryProperties().getPropBool("@encrypted"); if (dfsEncrypted) // otherwise ignore (warning issued by master) eexp.setown(createAESExpander256(encryptedKeyLen, encryptedKey)); memset(encryptedKey, 0, encryptedKeyLen); free(encryptedKey); } } }
virtual void slaveDone(size32_t slaveIdx, MemoryBuffer &mb) { if (mb.length()) // if 0 implies aborted out from this slave. { offset_t size; mb.read(size); CDateTime modifiedTime(mb); IPartDescriptor *partDesc = newIndexDesc->queryPart(slaveIdx); IPropertyTree &props = partDesc->queryProperties(); props.setPropInt64("@size", size); StringBuffer timeStr; modifiedTime.getString(timeStr); props.setProp("@modified", timeStr.str()); if (!local && 0 == slaveIdx) { mb.read(size); CDateTime modifiedTime(mb); IPartDescriptor *partDesc = newIndexDesc->queryPart(newIndexDesc->numParts()-1); IPropertyTree &props = partDesc->queryProperties(); props.setPropInt64("@size", size); StringBuffer timeStr; modifiedTime.getString(timeStr); props.setProp("@modified", timeStr.str()); } } }
DataSourceMetaItem::DataSourceMetaItem(unsigned _flags, MemoryBuffer & in) { flags = _flags; in.read(name); in.read(xpath); type.setown(deserializeType(in)); splitXmlTagNamesFromXPath(xpath.get(), tagname); }
DataSourceMetaData::DataSourceMetaData(MemoryBuffer & buffer) { numVirtualFields = 0; buffer.read(numFieldsToIgnore); buffer.read(randomIsOk); buffer.read(hasMixedContent); buffer.read(isStoredFixedWidth); buffer.read(minRecordSize); buffer.read(keyedSize); buffer.read(maxRecordSize); unsigned numFields; buffer.read(numFields); for (unsigned idx=0; idx < numFields; idx++) { byte flags; buffer.read(flags); if (flags == FVFFdataset) fields.append(*new DataSourceDatasetItem(flags, buffer)); else if (flags == FVFFset) fields.append(*new DataSourceSetItem(flags, buffer)); else fields.append(*new DataSourceMetaItem(flags, buffer)); if (flags == FVFFvirtual) ++numVirtualFields; } gatherAttributes(); }
virtual void deserializeCreateContext(MemoryBuffer &mb) { CSlaveGraphElement::deserializeCreateContext(mb); if (TAKworkunitread == kind) { mb.read(wuidread2diskread); // have I been converted if (wuidread2diskread) mb.read(wuidreadFilename); } haveCreateCtx = true; }
VariableRowBlock::VariableRowBlock(MemoryBuffer & inBuffer, __int64 _start) : RowBlock(_start, 0) { inBuffer.read(numRows); for (unsigned row = 0; row < numRows; row++) { unsigned thisLength; rowIndex.append(buffer.length()); inBuffer.read(thisLength); buffer.append(thisLength, inBuffer.readDirect(thisLength)); } rowIndex.append(buffer.length()); }
void PartitionPoint::deserialize(MemoryBuffer & in) { inputName.deserialize(in); outputName.deserialize(in); in.read(whichInput); in.read(inputOffset); in.read(inputLength); in.read(whichOutput); in.read(outputOffset); in.read(outputLength); in.read(whichSlave); modifiedTime.deserialize(in); ::deserialize(in, fixedText); }
bool processPartitionCommand(ISocket * masterSocket, MemoryBuffer & msg, MemoryBuffer & results) { FileFormat srcFormat; FileFormat tgtFormat; unsigned whichInput; RemoteFilename fullPath; offset_t totalSize; offset_t thisOffset; offset_t thisSize; unsigned thisHeaderSize; unsigned numParts; bool compressedInput = false; unsigned compatflags = 0; srcFormat.deserialize(msg); tgtFormat.deserialize(msg); msg.read(whichInput); fullPath.deserialize(msg); msg.read(totalSize); msg.read(thisOffset); msg.read(thisSize); msg.read(thisHeaderSize); msg.read(numParts); if (msg.remaining()) msg.read(compressedInput); if (msg.remaining()) msg.read(compatflags); // not yet used StringAttr decryptkey; if (msg.remaining()) msg.read(decryptkey); if (msg.remaining()) { srcFormat.deserializeExtra(msg, 1); tgtFormat.deserializeExtra(msg, 1); } StringBuffer text; fullPath.getRemotePath(text); LOG(MCdebugProgress, unknownJob, "Process partition %d(%s)", whichInput, text.str()); Owned<IFormatProcessor> processor = createFormatProcessor(srcFormat, tgtFormat, true); Owned<IOutputProcessor> target = createOutputProcessor(tgtFormat); processor->setTarget(target); processor->setPartitionRange(totalSize, thisOffset, thisSize, thisHeaderSize, numParts); processor->setSource(whichInput, fullPath, compressedInput, decryptkey); processor->calcPartitions(NULL); PartitionPointArray partition; processor->getResults(partition); serialize(partition, results); return true; }
extern DLLSERVER_API bool decompressResource(size32_t len, const void *data, StringBuffer &result) { bool hasVersion = len && (*(const byte *)data == 0x80); MemoryBuffer src; src.setBuffer(len, const_cast<void *>(data), false); byte version = 1; if (hasVersion) { src.skip(1); src.read(version); } MemoryBuffer tgt; switch (version) { case 1: decompressToBuffer(tgt, src); break; default: throwUnexpected(); } tgt.append((char)0); unsigned expandedLen = tgt.length(); result.setBuffer(expandedLen, reinterpret_cast<char *>(tgt.detach()), expandedLen-1); return true; }
void CWriteMasterBase::deserializeStats(unsigned node, MemoryBuffer &mb) { CMasterActivity::deserializeStats(node, mb); unsigned repPerc; mb.read(repPerc); replicateProgress->set(node, repPerc); }
void CDiskReadMasterBase::deserializeStats(unsigned node, MemoryBuffer &mb) { CMasterActivity::deserializeStats(node, mb); rowcount_t progress; mb.read(progress); inputProgress->set(node, progress); }
virtual void merge(MemoryBuffer &in) { unsigned __int64 c[2]; in.read(sizeof(c), &c); counts[false] += c[false]; counts[true] += c[true]; }
virtual void slaveDone(size32_t slaveIdx, MemoryBuffer &mb) { if (mb.length()) // if 0 implies aborted out from this slave. { rowcount_t rc; mb.read(rc); recordsProcessed += rc; } }
virtual void deserializeStats(unsigned node, MemoryBuffer &mb) { HashDistributeActivityMaster::deserializeStats(node, mb); rowcount_t lhsProgressCount, rhsProgressCount; mb.read(lhsProgressCount); mb.read(rhsProgressCount); lhsProgress->set(node, lhsProgressCount); rhsProgress->set(node, rhsProgressCount); }
byte getWUSresult(MemoryBuffer &mb) { byte ret = 0; wusbuf.swapWith(mb); if (mb.length()==1) { mb.read(ret); mb.clear(); } return ret; }
void RemoteDataSourceServer::doCmdRow(bool raw, MemoryBuffer & in, MemoryBuffer & out) { Owned<IFvDataSource> ds = readDataSource(in); if (!ds) { out.append(false); return; } __int64 requestedRow; in.read(requestedRow); unsigned startPos = out.length(); unsigned numRows = 0; out.append(true); // ok out.append(requestedRow); // start unsigned numRowsPos = out.length(); out.append(numRows); // total number of rows; loop { unsigned lengthPos = out.length(); out.append((unsigned)0); // size of this row. unsigned startRow = out.length(); if (raw) { if (!ds->getRawRow(out, requestedRow+numRows)) break; } else { if (!ds->getRow(out, requestedRow+numRows)) break; } if ((numRows != 0) && (out.length() > REMOTE_DATA_SIZE)) break; unsigned endRow = out.length(); out.rewrite(lengthPos); out.append(endRow-startRow); out.rewrite(endRow); numRows++; } if (numRows == 0) { out.rewrite(startPos); out.append(false); return; } unsigned totalLength = out.length(); out.rewrite(numRowsPos); out.append(numRows); out.rewrite(totalLength); }
void slaveDone(size32_t slaveIdx, MemoryBuffer &mb) { if (mb.length()) // if 0 implies aborted out from this slave. { offset_t size; mb.read(size); CDateTime modifiedTime(mb); IPartDescriptor *partDesc = patchDesc->queryPart(slaveIdx); IPropertyTree &props = partDesc->queryProperties(); StringBuffer timeStr; modifiedTime.getString(timeStr); props.setProp("@modified", timeStr.str()); unsigned crc; mb.read(crc); props.setPropInt64("@fileCrc", crc); if (!local && 0 == slaveIdx) { IPartDescriptor *partDesc = patchDesc->queryPart(patchDesc->numParts()-1); IPropertyTree &props = partDesc->queryProperties(); mb.read(size); props.setPropInt64("@size", size); CDateTime modifiedTime(mb); StringBuffer timeStr; modifiedTime.getString(timeStr); props.setProp("@modified", timeStr.str()); if (copyTlk) { Owned<IPartDescriptor> tlkDesc = newIndexDesc->getPart(newIndexDesc->numParts()-1); if (partDesc->getCrc(crc)) props.setPropInt64("@fileCrc", crc); props.setProp("@diffFormat", "copy"); } else { mb.read(crc); props.setPropInt64("@fileCrc", crc); props.setProp("@diffFormat", "diffV1"); } } } }
size32_t aesDecryptWithRSAEncryptedKey(MemoryBuffer &out, size32_t inSz, const void *inBytes, const CLoadedKey &privateKey) { MemoryBuffer in; in.setBuffer(inSz, (void *)inBytes, false); // read encrypted AES key size32_t encryptedAESKeySz; in.read(encryptedAESKeySz); MemoryBuffer aesKey; size32_t decryptedAesKeySz = privateKeyDecrypt(aesKey, encryptedAESKeySz, in.readDirect(encryptedAESKeySz), privateKey); if (decryptedAesKeySz != aesMaxKeySize) throw makeStringException(0, "aesDecryptWithRSAEncryptedKey - invalid input"); unsigned iVPos = in.getPos(); // read directly further down in.skip(aesBlockSize); size32_t aesEncryptedSz; in.read(aesEncryptedSz); return aesDecrypt(out, aesEncryptedSz, in.readDirect(aesEncryptedSz), aesMaxKeySize, (const char *)aesKey.bytes(), (const char *)in.bytes()+iVPos); }
void CachedPasswordProvider::deserialize(MemoryBuffer & in) { unsigned num; in.read(num); while (num--) { CachedPassword & cur = * new CachedPassword; cur.deserialize(in); passwords.append(cur); } }
//--------------------------------------------------------------------------- void deserializeBoolArray(unsigned len, bool * values, MemoryBuffer & in) { for (unsigned i = 0; i < len; i+= 8) { unsigned char next; in.read(next); unsigned max = i+8 <= len ? 8 : len - i; for (unsigned j=0; j<max; j++) values[i+j] = (next & (1 << j)) != 0; } }
void deserialize(PartitionPointArray & partition, MemoryBuffer & in) { unsigned count; in.read(count); for (unsigned idx = 0; idx < count; idx++) { PartitionPoint & next = * new PartitionPoint; next.deserialize(in); partition.append(next); } }
void RemoteDataSourceServer::doCmdCreateFile(MemoryBuffer & in, MemoryBuffer & out) { SessionId session; StringAttr username, password, logicalName; in.read(session); in.read(username).read(password); in.read(logicalName); DBGLOG("RemoteFileView:CreateFile('%s') by[%s:%" I64F "d", logicalName.get(), username.get(), session); Owned<IFvDataSource> ds = createFileDataSource(logicalName, cluster, username, password); unique_id_t id = addDataSource(session, ds); out.append((unsigned short)CurRemoteVersion); out.append(id); out.append(ds->numRows(false)); ds->queryMetaData()->serialize(out); out.append(ds->isIndex()); DBGLOG("RemoteFileView:CreateFile returns %" I64F "d", id); }