void slaveDone(size32_t slaveIdx, MemoryBuffer &mb)
{
if (mb.length()) // if 0 implies aborted out from this slave.
{
rowcount_t slaveProcessed;
mb.read(slaveProcessed);
recordsProcessed += slaveProcessed;
offset_t size, physicalSize;
mb.read(size);
mb.read(physicalSize);
unsigned fileCrc;
mb.read(fileCrc);
CDateTime modifiedTime(mb);
StringBuffer timeStr;
modifiedTime.getString(timeStr);
IPartDescriptor *partDesc = fileDesc->queryPart(slaveIdx);
IPropertyTree &props = partDesc->queryProperties();
props.setPropInt64("@size", size);
if (fileDesc->isCompressed())
props.setPropInt64("@compressedSize", physicalSize);
props.setPropInt("@fileCrc", fileCrc);
props.setProp("@modified", timeStr.str());
}
}
// IThorSlaveActivity
void CDiskReadSlaveActivityBase::init(MemoryBuffer &data, MemoryBuffer &slaveData)
{
subfileLogicalFilenames.kill();
partDescs.kill();
data.read(logicalFilename);
unsigned subfiles;
data.read(subfiles);
unsigned s=0;
for (; s<subfiles; s++)
{
StringAttr subfile;
data.read(subfile);
subfileLogicalFilenames.append(subfile);
}
unsigned parts;
data.read(parts);
if (parts)
{
deserializePartFileDescriptors(data, partDescs);
// put temp files in individual slave temp dirs (incl port)
if ((helper->getFlags() & TDXtemporary) && (!container.queryJob().queryUseCheckpoints()))
partDescs.item(0).queryOwner().setDefaultDir(queryTempDir());
}
}
void CDiskWriteSlaveActivityBase::init(MemoryBuffer &data, MemoryBuffer &slaveData)
{
StringAttr logicalFilename;
data.read(logicalFilename);
dlfn.set(logicalFilename);
if (diskHelperBase->getFlags() & TDXtemporary)
data.read(usageCount);
if (diskHelperBase->getFlags() & TDWextend)
{
assertex(!"TBD extended CRC broken");
unsigned crc;
if (partDesc->getCrc(crc))
fileCRC.reset(~crc);
}
partDesc.setown(deserializePartFileDescriptor(data));
// put temp files in individual slave temp dirs (incl port)
if ((diskHelperBase->getFlags() & TDXtemporary) && (!container.queryJob().queryUseCheckpoints()))
partDesc->queryOwner().setDefaultDir(queryTempDir());
if (dlfn.isExternal())
{
mpTag = container.queryJobChannel().deserializeMPTag(data);
if (dlfn.isQuery() && (0 != container.queryJob().getWorkUnitValueInt("rfsqueryparallel", 0)))
rfsQueryParallel = true;
}
if (0 != (diskHelperBase->getFlags() & TDXgrouped))
grouped = true;
}
void CDiskWriteSlaveActivityBase::init(MemoryBuffer &data, MemoryBuffer &slaveData)
{
diskHelperBase = static_cast <IHThorDiskWriteArg *> (queryHelper());
data.read(logicalFilename);
dlfn.set(logicalFilename);
if (diskHelperBase->getFlags() & TDXtemporary)
data.read(usageCount);
if (diskHelperBase->getFlags() & TDWextend)
{
assertex(!"TBD extended CRC broken");
unsigned crc;
if (partDesc->getCrc(crc))
fileCRC.reset(~crc);
}
partDesc.setown(deserializePartFileDescriptor(data));
if (dlfn.isExternal())
{
mpTag = container.queryJob().deserializeMPTag(data);
if (dlfn.isQuery() && (0 != container.queryJob().getWorkUnitValueInt("rfsqueryparallel", 0)))
rfsQueryParallel = true;
}
if (0 != (diskHelperBase->getFlags() & TDXgrouped))
grouped = true;
compress = partDesc->queryOwner().isCompressed();
void *ekey;
size32_t ekeylen;
diskHelperBase->getEncryptKey(ekeylen,ekey);
if (ekeylen!=0) {
ecomp.setown(createAESCompressor256(ekeylen,ekey));
memset(ekey,0,ekeylen);
free(ekey);
compress = true;
}
}
void FileFormat::deserialize(MemoryBuffer & in)
{
byte tempType;
in.read(tempType); type = (FileFormatType)tempType;
switch (type)
{
case FFTfixed:
case FFTblocked:
in.read(recordSize);
break;
case FFTcsv:
case FFTutf:
case FFTutf8: case FFTutf8n:
case FFTutf16: case FFTutf16be: case FFTutf16le:
case FFTutf32: case FFTutf32be: case FFTutf32le:
in.read(maxRecordSize);
::deserialize(in, separate);
::deserialize(in, quote);
::deserialize(in, terminate);
::deserialize(in, escape);
::deserialize(in, rowTag);
break;
}
}
void RemoteDataSourceServer::doCmdCreateWorkunit(MemoryBuffer & in, MemoryBuffer & out)
{
SessionId session;
StringAttr wuid, username, password;
unsigned sequence;
StringAttr name;
in.read(session);
in.read(username).read(password);
in.read(wuid);
in.read(sequence);
in.read(name);
DBGLOG("RemoteFileView:CreateWorkunit('%s',%d,'%s') by[%s:%" I64F "d", wuid.get(), sequence, name ? name.get() : "", username.get(), session);
Owned<IConstWUResult> wuResult = resolveResult(wuid, sequence, name);
Owned<IFvDataSource> ds = createDataSource(wuResult, wuid, username, password);
unique_id_t id = addDataSource(session, ds);
out.append((unsigned short)CurRemoteVersion);
out.append(id);
out.append(ds->numRows(false));
ds->queryMetaData()->serialize(out);
out.append(ds->isIndex());
DBGLOG("RemoteFileView:CreateWorkunit returns %" I64F "d", id);
}
DataSourceMetaItem::DataSourceMetaItem(unsigned _flags, MemoryBuffer & in)
{
flags = _flags;
in.read(name);
in.read(xpath);
type.setown(deserializeType(in));
}
void FileFormat::deserialize(MemoryBuffer & in)
{
byte tempType;
in.read(tempType); type = (FileFormatType)tempType;
switch (type)
{
case FFTfixed:
case FFTblocked:
in.read(recordSize);
break;
case FFTcsv:
case FFTutf:
case FFTutf8: case FFTutf8n:
case FFTutf16: case FFTutf16be: case FFTutf16le:
case FFTutf32: case FFTutf32be: case FFTutf32le:
in.read(maxRecordSize);
::deserialize(in, separate);
::deserialize(in, quote);
::deserialize(in, terminate);
::deserialize(in, rowTag);
updateMarkupType(rowTag, NULL); //neither kind nor markup currently serialized. may add later
break;
}
}
// IThorSlaveActivity
void CDiskReadSlaveActivityBase::init(MemoryBuffer &data, MemoryBuffer &slaveData)
{
subfileLogicalFilenames.kill();
partDescs.kill();
data.read(logicalFilename);
unsigned subfiles;
data.read(subfiles);
unsigned s=0;
for (; s<subfiles; s++)
{
StringAttr subfile;
data.read(subfile);
subfileLogicalFilenames.append(subfile);
}
unsigned parts;
data.read(parts);
if (parts)
{
deserializePartFileDescriptors(data, partDescs);
unsigned encryptedKeyLen;
void *encryptedKey;
helper->getEncryptKey(encryptedKeyLen, encryptedKey);
if (0 != encryptedKeyLen)
{
bool dfsEncrypted = partDescs.item(0).queryOwner().queryProperties().getPropBool("@encrypted");
if (dfsEncrypted) // otherwise ignore (warning issued by master)
eexp.setown(createAESExpander256(encryptedKeyLen, encryptedKey));
memset(encryptedKey, 0, encryptedKeyLen);
free(encryptedKey);
}
}
}
virtual void slaveDone(size32_t slaveIdx, MemoryBuffer &mb)
{
if (mb.length()) // if 0 implies aborted out from this slave.
{
offset_t size;
mb.read(size);
CDateTime modifiedTime(mb);
IPartDescriptor *partDesc = newIndexDesc->queryPart(slaveIdx);
IPropertyTree &props = partDesc->queryProperties();
props.setPropInt64("@size", size);
StringBuffer timeStr;
modifiedTime.getString(timeStr);
props.setProp("@modified", timeStr.str());
if (!local && 0 == slaveIdx)
{
mb.read(size);
CDateTime modifiedTime(mb);
IPartDescriptor *partDesc = newIndexDesc->queryPart(newIndexDesc->numParts()-1);
IPropertyTree &props = partDesc->queryProperties();
props.setPropInt64("@size", size);
StringBuffer timeStr;
modifiedTime.getString(timeStr);
props.setProp("@modified", timeStr.str());
}
}
}
DataSourceMetaItem::DataSourceMetaItem(unsigned _flags, MemoryBuffer & in)
{
flags = _flags;
in.read(name);
in.read(xpath);
type.setown(deserializeType(in));
splitXmlTagNamesFromXPath(xpath.get(), tagname);
}
DataSourceMetaData::DataSourceMetaData(MemoryBuffer & buffer)
{
numVirtualFields = 0;
buffer.read(numFieldsToIgnore);
buffer.read(randomIsOk);
buffer.read(hasMixedContent);
buffer.read(isStoredFixedWidth);
buffer.read(minRecordSize);
buffer.read(keyedSize);
buffer.read(maxRecordSize);
unsigned numFields;
buffer.read(numFields);
for (unsigned idx=0; idx < numFields; idx++)
{
byte flags;
buffer.read(flags);
if (flags == FVFFdataset)
fields.append(*new DataSourceDatasetItem(flags, buffer));
else if (flags == FVFFset)
fields.append(*new DataSourceSetItem(flags, buffer));
else
fields.append(*new DataSourceMetaItem(flags, buffer));
if (flags == FVFFvirtual)
++numVirtualFields;
}
gatherAttributes();
}
virtual void deserializeCreateContext(MemoryBuffer &mb)
{
CSlaveGraphElement::deserializeCreateContext(mb);
if (TAKworkunitread == kind)
{
mb.read(wuidread2diskread); // have I been converted
if (wuidread2diskread)
mb.read(wuidreadFilename);
}
haveCreateCtx = true;
}
VariableRowBlock::VariableRowBlock(MemoryBuffer & inBuffer, __int64 _start) : RowBlock(_start, 0)
{
inBuffer.read(numRows);
for (unsigned row = 0; row < numRows; row++)
{
unsigned thisLength;
rowIndex.append(buffer.length());
inBuffer.read(thisLength);
buffer.append(thisLength, inBuffer.readDirect(thisLength));
}
rowIndex.append(buffer.length());
}
void PartitionPoint::deserialize(MemoryBuffer & in)
{
inputName.deserialize(in);
outputName.deserialize(in);
in.read(whichInput);
in.read(inputOffset);
in.read(inputLength);
in.read(whichOutput);
in.read(outputOffset);
in.read(outputLength);
in.read(whichSlave);
modifiedTime.deserialize(in);
::deserialize(in, fixedText);
}
bool processPartitionCommand(ISocket * masterSocket, MemoryBuffer & msg, MemoryBuffer & results)
{
FileFormat srcFormat;
FileFormat tgtFormat;
unsigned whichInput;
RemoteFilename fullPath;
offset_t totalSize;
offset_t thisOffset;
offset_t thisSize;
unsigned thisHeaderSize;
unsigned numParts;
bool compressedInput = false;
unsigned compatflags = 0;
srcFormat.deserialize(msg);
tgtFormat.deserialize(msg);
msg.read(whichInput);
fullPath.deserialize(msg);
msg.read(totalSize);
msg.read(thisOffset);
msg.read(thisSize);
msg.read(thisHeaderSize);
msg.read(numParts);
if (msg.remaining())
msg.read(compressedInput);
if (msg.remaining())
msg.read(compatflags); // not yet used
StringAttr decryptkey;
if (msg.remaining())
msg.read(decryptkey);
if (msg.remaining())
{
srcFormat.deserializeExtra(msg, 1);
tgtFormat.deserializeExtra(msg, 1);
}
StringBuffer text;
fullPath.getRemotePath(text);
LOG(MCdebugProgress, unknownJob, "Process partition %d(%s)", whichInput, text.str());
Owned<IFormatProcessor> processor = createFormatProcessor(srcFormat, tgtFormat, true);
Owned<IOutputProcessor> target = createOutputProcessor(tgtFormat);
processor->setTarget(target);
processor->setPartitionRange(totalSize, thisOffset, thisSize, thisHeaderSize, numParts);
processor->setSource(whichInput, fullPath, compressedInput, decryptkey);
processor->calcPartitions(NULL);
PartitionPointArray partition;
processor->getResults(partition);
serialize(partition, results);
return true;
}
extern DLLSERVER_API bool decompressResource(size32_t len, const void *data, StringBuffer &result)
{
bool hasVersion = len && (*(const byte *)data == 0x80);
MemoryBuffer src;
src.setBuffer(len, const_cast<void *>(data), false);
byte version = 1;
if (hasVersion)
{
src.skip(1);
src.read(version);
}
MemoryBuffer tgt;
switch (version)
{
case 1:
decompressToBuffer(tgt, src);
break;
default:
throwUnexpected();
}
tgt.append((char)0);
unsigned expandedLen = tgt.length();
result.setBuffer(expandedLen, reinterpret_cast<char *>(tgt.detach()), expandedLen-1);
return true;
}
void CWriteMasterBase::deserializeStats(unsigned node, MemoryBuffer &mb)
{
CMasterActivity::deserializeStats(node, mb);
unsigned repPerc;
mb.read(repPerc);
replicateProgress->set(node, repPerc);
}
void CDiskReadMasterBase::deserializeStats(unsigned node, MemoryBuffer &mb)
{
CMasterActivity::deserializeStats(node, mb);
rowcount_t progress;
mb.read(progress);
inputProgress->set(node, progress);
}
virtual void merge(MemoryBuffer &in)
{
unsigned __int64 c[2];
in.read(sizeof(c), &c);
counts[false] += c[false];
counts[true] += c[true];
}
virtual void slaveDone(size32_t slaveIdx, MemoryBuffer &mb)
{
if (mb.length()) // if 0 implies aborted out from this slave.
{
rowcount_t rc;
mb.read(rc);
recordsProcessed += rc;
}
}
virtual void deserializeStats(unsigned node, MemoryBuffer &mb)
{
HashDistributeActivityMaster::deserializeStats(node, mb);
rowcount_t lhsProgressCount, rhsProgressCount;
mb.read(lhsProgressCount);
mb.read(rhsProgressCount);
lhsProgress->set(node, lhsProgressCount);
rhsProgress->set(node, rhsProgressCount);
}
byte getWUSresult(MemoryBuffer &mb)
{
byte ret = 0;
wusbuf.swapWith(mb);
if (mb.length()==1) {
mb.read(ret);
mb.clear();
}
return ret;
}
void RemoteDataSourceServer::doCmdRow(bool raw, MemoryBuffer & in, MemoryBuffer & out)
{
Owned<IFvDataSource> ds = readDataSource(in);
if (!ds)
{
out.append(false);
return;
}
__int64 requestedRow;
in.read(requestedRow);
unsigned startPos = out.length();
unsigned numRows = 0;
out.append(true); // ok
out.append(requestedRow); // start
unsigned numRowsPos = out.length();
out.append(numRows); // total number of rows;
loop
{
unsigned lengthPos = out.length();
out.append((unsigned)0); // size of this row.
unsigned startRow = out.length();
if (raw)
{
if (!ds->getRawRow(out, requestedRow+numRows))
break;
}
else
{
if (!ds->getRow(out, requestedRow+numRows))
break;
}
if ((numRows != 0) && (out.length() > REMOTE_DATA_SIZE))
break;
unsigned endRow = out.length();
out.rewrite(lengthPos);
out.append(endRow-startRow);
out.rewrite(endRow);
numRows++;
}
if (numRows == 0)
{
out.rewrite(startPos);
out.append(false);
return;
}
unsigned totalLength = out.length();
out.rewrite(numRowsPos);
out.append(numRows);
out.rewrite(totalLength);
}
void slaveDone(size32_t slaveIdx, MemoryBuffer &mb)
{
if (mb.length()) // if 0 implies aborted out from this slave.
{
offset_t size;
mb.read(size);
CDateTime modifiedTime(mb);
IPartDescriptor *partDesc = patchDesc->queryPart(slaveIdx);
IPropertyTree &props = partDesc->queryProperties();
StringBuffer timeStr;
modifiedTime.getString(timeStr);
props.setProp("@modified", timeStr.str());
unsigned crc;
mb.read(crc);
props.setPropInt64("@fileCrc", crc);
if (!local && 0 == slaveIdx)
{
IPartDescriptor *partDesc = patchDesc->queryPart(patchDesc->numParts()-1);
IPropertyTree &props = partDesc->queryProperties();
mb.read(size);
props.setPropInt64("@size", size);
CDateTime modifiedTime(mb);
StringBuffer timeStr;
modifiedTime.getString(timeStr);
props.setProp("@modified", timeStr.str());
if (copyTlk)
{
Owned<IPartDescriptor> tlkDesc = newIndexDesc->getPart(newIndexDesc->numParts()-1);
if (partDesc->getCrc(crc))
props.setPropInt64("@fileCrc", crc);
props.setProp("@diffFormat", "copy");
}
else
{
mb.read(crc);
props.setPropInt64("@fileCrc", crc);
props.setProp("@diffFormat", "diffV1");
}
}
}
}
size32_t aesDecryptWithRSAEncryptedKey(MemoryBuffer &out, size32_t inSz, const void *inBytes, const CLoadedKey &privateKey)
{
MemoryBuffer in;
in.setBuffer(inSz, (void *)inBytes, false);
// read encrypted AES key
size32_t encryptedAESKeySz;
in.read(encryptedAESKeySz);
MemoryBuffer aesKey;
size32_t decryptedAesKeySz = privateKeyDecrypt(aesKey, encryptedAESKeySz, in.readDirect(encryptedAESKeySz), privateKey);
if (decryptedAesKeySz != aesMaxKeySize)
throw makeStringException(0, "aesDecryptWithRSAEncryptedKey - invalid input");
unsigned iVPos = in.getPos(); // read directly further down
in.skip(aesBlockSize);
size32_t aesEncryptedSz;
in.read(aesEncryptedSz);
return aesDecrypt(out, aesEncryptedSz, in.readDirect(aesEncryptedSz), aesMaxKeySize, (const char *)aesKey.bytes(), (const char *)in.bytes()+iVPos);
}
void CachedPasswordProvider::deserialize(MemoryBuffer & in)
{
unsigned num;
in.read(num);
while (num--)
{
CachedPassword & cur = * new CachedPassword;
cur.deserialize(in);
passwords.append(cur);
}
}
//---------------------------------------------------------------------------
void deserializeBoolArray(unsigned len, bool * values, MemoryBuffer & in)
{
for (unsigned i = 0; i < len; i+= 8)
{
unsigned char next;
in.read(next);
unsigned max = i+8 <= len ? 8 : len - i;
for (unsigned j=0; j<max; j++)
values[i+j] = (next & (1 << j)) != 0;
}
}
void deserialize(PartitionPointArray & partition, MemoryBuffer & in)
{
unsigned count;
in.read(count);
for (unsigned idx = 0; idx < count; idx++)
{
PartitionPoint & next = * new PartitionPoint;
next.deserialize(in);
partition.append(next);
}
}
void RemoteDataSourceServer::doCmdCreateFile(MemoryBuffer & in, MemoryBuffer & out)
{
SessionId session;
StringAttr username, password, logicalName;
in.read(session);
in.read(username).read(password);
in.read(logicalName);
DBGLOG("RemoteFileView:CreateFile('%s') by[%s:%" I64F "d", logicalName.get(), username.get(), session);
Owned<IFvDataSource> ds = createFileDataSource(logicalName, cluster, username, password);
unique_id_t id = addDataSource(session, ds);
out.append((unsigned short)CurRemoteVersion);
out.append(id);
out.append(ds->numRows(false));
ds->queryMetaData()->serialize(out);
out.append(ds->isIndex());
DBGLOG("RemoteFileView:CreateFile returns %" I64F "d", id);
}