virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData) override { isLocal = 0 != (TIWlocal & helper->getFlags()); mpTag = container.queryJobChannel().deserializeMPTag(data); mpTag2 = container.queryJobChannel().deserializeMPTag(data); data.read(active); if (active) { data.read(logicalFilename); partDesc.setown(deserializePartFileDescriptor(data)); } data.read(singlePartKey); data.read(refactor); if (singlePartKey) buildTlk = false; else { data.read(buildTlk); if (firstNode()) { if (buildTlk) tlkDesc.setown(deserializePartFileDescriptor(data)); else if (!isLocal) // existing tlk then.. { tlkDesc.setown(deserializePartFileDescriptor(data)); unsigned c; data.read(c); while (c--) { RemoteFilename rf; rf.deserialize(data); if (!existingTlkIFile) { Owned<IFile> iFile = createIFile(rf); if (iFile->exists()) existingTlkIFile.set(iFile); } } if (!existingTlkIFile) throw MakeActivityException(this, TE_FileNotFound, "Top level key part does not exist, for key"); } } } IOutputMetaData * diskSize = helper->queryDiskRecordSize(); assertex(!(diskSize->getMetaFlags() & MDFneedserializedisk)); if (diskSize->isVariableSize()) { if (TIWmaxlength & helper->getFlags()) maxDiskRecordSize = helper->getMaxKeySize(); else maxDiskRecordSize = KEYBUILD_MAXLENGTH; //Current default behaviour, could be improved in the future } else maxDiskRecordSize = diskSize->getFixedSize(); reportOverflow = false; }
bool processPartitionCommand(ISocket * masterSocket, MemoryBuffer & msg, MemoryBuffer & results) { FileFormat srcFormat; FileFormat tgtFormat; unsigned whichInput; RemoteFilename fullPath; offset_t totalSize; offset_t thisOffset; offset_t thisSize; unsigned thisHeaderSize; unsigned numParts; bool compressedInput = false; unsigned compatflags = 0; srcFormat.deserialize(msg); tgtFormat.deserialize(msg); msg.read(whichInput); fullPath.deserialize(msg); msg.read(totalSize); msg.read(thisOffset); msg.read(thisSize); msg.read(thisHeaderSize); msg.read(numParts); if (msg.remaining()) msg.read(compressedInput); if (msg.remaining()) msg.read(compatflags); // not yet used StringAttr decryptkey; if (msg.remaining()) msg.read(decryptkey); if (msg.remaining()) { srcFormat.deserializeExtra(msg, 1); tgtFormat.deserializeExtra(msg, 1); } StringBuffer text; fullPath.getRemotePath(text); LOG(MCdebugProgress, unknownJob, "Process partition %d(%s)", whichInput, text.str()); Owned<IFormatProcessor> processor = createFormatProcessor(srcFormat, tgtFormat, true); Owned<IOutputProcessor> target = createOutputProcessor(tgtFormat); processor->setTarget(target); processor->setPartitionRange(totalSize, thisOffset, thisSize, thisHeaderSize, numParts); processor->setSource(whichInput, fullPath, compressedInput, decryptkey); processor->calcPartitions(NULL); PartitionPointArray partition; processor->getResults(partition); serialize(partition, results); return true; }