bool processPartitionCommand(ISocket * masterSocket, MemoryBuffer & msg, MemoryBuffer & results)
{
FileFormat srcFormat;
FileFormat tgtFormat;
unsigned whichInput;
RemoteFilename fullPath;
offset_t totalSize;
offset_t thisOffset;
offset_t thisSize;
unsigned thisHeaderSize;
unsigned numParts;
bool compressedInput = false;
unsigned compatflags = 0;
srcFormat.deserialize(msg);
tgtFormat.deserialize(msg);
msg.read(whichInput);
fullPath.deserialize(msg);
msg.read(totalSize);
msg.read(thisOffset);
msg.read(thisSize);
msg.read(thisHeaderSize);
msg.read(numParts);
if (msg.remaining())
msg.read(compressedInput);
if (msg.remaining())
msg.read(compatflags); // not yet used
StringAttr decryptkey;
if (msg.remaining())
msg.read(decryptkey);
if (msg.remaining())
{
srcFormat.deserializeExtra(msg, 1);
tgtFormat.deserializeExtra(msg, 1);
}
StringBuffer text;
fullPath.getRemotePath(text);
LOG(MCdebugProgress, unknownJob, "Process partition %d(%s)", whichInput, text.str());
Owned<IFormatProcessor> processor = createFormatProcessor(srcFormat, tgtFormat, true);
Owned<IOutputProcessor> target = createOutputProcessor(tgtFormat);
processor->setTarget(target);
processor->setPartitionRange(totalSize, thisOffset, thisSize, thisHeaderSize, numParts);
processor->setSource(whichInput, fullPath, compressedInput, decryptkey);
processor->calcPartitions(NULL);
PartitionPointArray partition;
processor->getResults(partition);
serialize(partition, results);
return true;
}
void TransferServer::deserializeAction(MemoryBuffer & msg, unsigned action)
{
SocketEndpoint ep;
ep.deserialize(msg);
if (!ep.isLocal())
{
StringBuffer host, expected;
queryHostIP().getIpText(host);
ep.getIpText(expected);
throwError2(DFTERR_WrongComputer, expected.str(), host.str());
}
srcFormat.deserialize(msg);
tgtFormat.deserialize(msg);
msg.read(calcInputCRC);
msg.read(calcOutputCRC);
deserialize(partition, msg);
msg.read(numParallelSlaves);
msg.read(updateFrequency);
msg.read(replicate);
msg.read(mirror);
msg.read(isSafeMode);
srand((unsigned)get_cycles_now());
int adjust = (rand() * rand() * rand()) % updateFrequency - (updateFrequency/2);
lastTick = msTick() + adjust;
StringBuffer localFilename;
if (action == FTactionpull)
{
partition.item(0).outputName.getPath(localFilename);
LOG(MCdebugProgress, unknownJob, "Process Pull Command: %s", localFilename.str());
}
else
{
partition.item(0).inputName.getPath(localFilename);
LOG(MCdebugProgress, unknownJob, "Process Push Command: %s", localFilename.str());
}
LOG(MCdebugProgress, unknownJob, "Num Parallel Slaves=%d Adjust=%d/%d", numParallelSlaves, adjust, updateFrequency);
LOG(MCdebugProgress, unknownJob, "replicate(%d) mirror(%d) safe(%d) incrc(%d) outcrc(%d)", replicate, mirror, isSafeMode, calcInputCRC, calcOutputCRC);
displayPartition(partition);
unsigned numProgress;
msg.read(numProgress);
for (unsigned i = 0; i < numProgress; i++)
{
OutputProgress & next = *new OutputProgress;
next.deserializeCore(msg);
progress.append(next);
}
if (msg.remaining())
msg.read(throttleNicSpeed);
if (msg.remaining())
msg.read(compressedInput).read(compressOutput);
if (msg.remaining())
msg.read(copyCompressed);
if (msg.remaining())
msg.read(transferBufferSize);
if (msg.remaining())
msg.read(encryptKey).read(decryptKey);
if (msg.remaining())
{
srcFormat.deserializeExtra(msg, 1);
tgtFormat.deserializeExtra(msg, 1);
}
ForEachItemIn(i1, progress)
progress.item(i1).deserializeExtra(msg, 1);
LOG(MCdebugProgress, unknownJob, "throttle(%d), transferBufferSize(%d)", throttleNicSpeed, transferBufferSize);
PROGLOG("compressedInput(%d), compressedOutput(%d), copyCompressed(%d)", compressedInput?1:0, compressOutput?1:0, copyCompressed?1:0);
PROGLOG("encrypt(%d), decrypt(%d)", encryptKey.isEmpty()?0:1, decryptKey.isEmpty()?0:1);
//---Finished deserializing ---
displayProgress(progress);
totalLengthRead = 0;
totalLengthToRead = 0;
ForEachItemIn(idx, partition)
totalLengthToRead += partition.item(idx).inputLength;
}
void deserialize(MemoryBuffer &mb)
{
// clear optional
#ifndef NO_XLS
xslproc.clear();
xsltrans.clear();
xslt.clear();
#endif
resultoverflow = false;
wuservices = false;
unsigned version;
mb.read(version);
unsigned v;
mb.read(v);
action = (SashaCommandAction)v;
unsigned n;
mb.read(n);
while (n--) {
StringAttrItem *s = new StringAttrItem;
mb.read(s->text);
ids.append(*s);
}
mb.read(after);
mb.read(before);
mb.read(state);
mb.read(owner);
mb.read(cluster);
mb.read(jobname);
mb.read(outputformat);
mb.read(online);
mb.read(archived);
mb.read(dfu);
mb.read(start);
mb.read(limit);
if (mb.remaining()>sizeof(unsigned)) { // new format (trailing WUs)
mb.read(resultoverflow);
mb.read(n);
while (n--) {
StringAttrItem *s = new StringAttrItem;
mb.read(s->text);
results.append(*s);
}
bool isxslt;
mb.read(isxslt);
if (isxslt)
mb.read(xslt);
if (mb.remaining()>sizeof(bool)) { // new WorkUnitServices format (trailing WUs)
mb.read(wuservices);
if (wuservices) {
mb.read(priority);
mb.read(fileread);
mb.read(filewritten);
mb.read(roxiecluster);
mb.read(eclcontains);
}
if (mb.remaining()>=sizeof(numdts)) { // new listdts info
mb.read(numdts);
free(dts);
dts = NULL;
if (numdts) {
dts = (CDateTime *)calloc(numdts,sizeof(CDateTime));
for (unsigned i=0;i<numdts;i++)
dts[i].deserialize(mb);
}
}
}
}
}
size32_t ThorExpand(MemoryBuffer & src, MemoryBuffer & dest)
{
size32_t len = src.remaining();
const void *pSrc = src.readDirect(len);
return ThorExpand(pSrc, len, dest);
}