virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData) override
{
isLocal = 0 != (TIWlocal & helper->getFlags());
mpTag = container.queryJobChannel().deserializeMPTag(data);
mpTag2 = container.queryJobChannel().deserializeMPTag(data);
data.read(active);
if (active)
{
data.read(logicalFilename);
partDesc.setown(deserializePartFileDescriptor(data));
}
data.read(singlePartKey);
data.read(refactor);
if (singlePartKey)
buildTlk = false;
else
{
data.read(buildTlk);
if (firstNode())
{
if (buildTlk)
tlkDesc.setown(deserializePartFileDescriptor(data));
else if (!isLocal) // existing tlk then..
{
tlkDesc.setown(deserializePartFileDescriptor(data));
unsigned c;
data.read(c);
while (c--)
{
RemoteFilename rf;
rf.deserialize(data);
if (!existingTlkIFile)
{
Owned<IFile> iFile = createIFile(rf);
if (iFile->exists())
existingTlkIFile.set(iFile);
}
}
if (!existingTlkIFile)
throw MakeActivityException(this, TE_FileNotFound, "Top level key part does not exist, for key");
}
}
}
IOutputMetaData * diskSize = helper->queryDiskRecordSize();
assertex(!(diskSize->getMetaFlags() & MDFneedserializedisk));
if (diskSize->isVariableSize())
{
if (TIWmaxlength & helper->getFlags())
maxDiskRecordSize = helper->getMaxKeySize();
else
maxDiskRecordSize = KEYBUILD_MAXLENGTH; //Current default behaviour, could be improved in the future
}
else
maxDiskRecordSize = diskSize->getFixedSize();
reportOverflow = false;
}
bool processPartitionCommand(ISocket * masterSocket, MemoryBuffer & msg, MemoryBuffer & results)
{
FileFormat srcFormat;
FileFormat tgtFormat;
unsigned whichInput;
RemoteFilename fullPath;
offset_t totalSize;
offset_t thisOffset;
offset_t thisSize;
unsigned thisHeaderSize;
unsigned numParts;
bool compressedInput = false;
unsigned compatflags = 0;
srcFormat.deserialize(msg);
tgtFormat.deserialize(msg);
msg.read(whichInput);
fullPath.deserialize(msg);
msg.read(totalSize);
msg.read(thisOffset);
msg.read(thisSize);
msg.read(thisHeaderSize);
msg.read(numParts);
if (msg.remaining())
msg.read(compressedInput);
if (msg.remaining())
msg.read(compatflags); // not yet used
StringAttr decryptkey;
if (msg.remaining())
msg.read(decryptkey);
if (msg.remaining())
{
srcFormat.deserializeExtra(msg, 1);
tgtFormat.deserializeExtra(msg, 1);
}
StringBuffer text;
fullPath.getRemotePath(text);
LOG(MCdebugProgress, unknownJob, "Process partition %d(%s)", whichInput, text.str());
Owned<IFormatProcessor> processor = createFormatProcessor(srcFormat, tgtFormat, true);
Owned<IOutputProcessor> target = createOutputProcessor(tgtFormat);
processor->setTarget(target);
processor->setPartitionRange(totalSize, thisOffset, thisSize, thisHeaderSize, numParts);
processor->setSource(whichInput, fullPath, compressedInput, decryptkey);
processor->calcPartitions(NULL);
PartitionPointArray partition;
processor->getResults(partition);
serialize(partition, results);
return true;
}
