IFvDataSource * createRemoteFileDataSource(const SocketEndpoint & server, const char * username, const char * password, const char * logicalName)
{
Owned<INode> serverNode = createINode(server);
CMessageBuffer msg;
msg.setEndian(__BIG_ENDIAN);
msg.append((byte)FVCMDcreatefile);
msg.append(myProcessSession());
msg.append(username);
msg.append(password);
msg.append(logicalName);
sendReceive(serverNode, msg);
unsigned short version;
unique_id_t id;
__int64 numRows;
bool isIndex;
msg.read(version);
msg.read(id);
msg.read(numRows);
Owned<IFvDataSourceMetaData> meta = deserializeDataSourceMeta(msg);
msg.read(isIndex);
if (id)
return new RemoteDataSource(server, id, meta, numRows, isIndex);
return 0;
}
void SortSlaveMP::StartMiniSort(rowcount_t _totalrows) /* async */
{
CMessageBuffer mb;
mb.append((byte)FN_StartMiniSort);
mb.append(_totalrows);
sendRecv(mb);
}
void sendRemainingHeaderLines()
{
if (!headerLines)
return;
unsigned which = sentHeaderLines->scanInvert(0, false);
if (which < subFiles)
{
CMessageBuffer msgMb;
bool someLeft=false;
do
{
msgMb.append(which);
unsigned &remaining = getHeaderLines(which);
if (0 != remaining)
someLeft = true;
msgMb.append(remaining);
which = sentHeaderLines->scanInvert(which+1, false);
}
while (which < subFiles);
if (someLeft)
queryJobChannel().queryJobComm().send(msgMb, queryJobChannel().queryMyRank()+1, mpTag);
else
sendAllDone();
}
}
bool RemoteDataSource::getARow(MemoryBuffer & out, RowCache & cache, byte cmd, __int64 row)
{
RowLocation location;
if (cache.getCacheRow(row, location))
{
out.append(location.matchLength, location.matchRow);
return true;
}
CMessageBuffer msg;
msg.setEndian(__BIG_ENDIAN);
msg.append(cmd);
msg.append(id);
msg.append(row);
sendReceive(msg);
bool ok;
msg.read(ok);
if (!ok) return false;
__int64 start;
msg.read(start);
VariableRowBlock * next = new VariableRowBlock(msg, start);
cache.addRowsOwn(next);
if (!cache.getCacheRow(row, location))
assertex(!"Internal Error!");
out.append(location.matchLength, location.matchRow);
return true;
}
void SortSlaveMP::MultiBinChopStop(unsigned num,rowcount_t *pos)
{
CMessageBuffer mb;
mb.append((byte)FN_MultiBinChopStop);
mb.append(num);
sendRecv(mb);
mb.read(num*sizeof(rowcount_t),pos);
}
void SortSlaveMP::GetGatherInfo(rowcount_t &numlocal, offset_t &totalsize, unsigned &overflowscale, bool hasserializer)
{
CMessageBuffer mb;
mb.append((byte)FN_GetGatherInfo);
mb.append(hasserializer);
sendRecv(mb);
mb.read(numlocal).read(totalsize).read(overflowscale);
}
void SortSlaveMP::GetMultiNthRow(unsigned numsplits,size32_t &mkeybuffsize, void * &mkeybuf)
{
CMessageBuffer mb;
mb.append((byte)FN_GetMultiNthRow);
mb.append(numsplits);
sendRecv(mb);
deserializeblk(mb,mkeybuffsize,mkeybuf);
}
void RemoteDataSource::beforeDispose()
{
CMessageBuffer msg;
msg.setEndian(__BIG_ENDIAN);
msg.append((byte)FVCMDdestroy);
msg.append(id);
sendReceive(msg);
}
void SortSlaveMP::OverflowAdjustMapStart(unsigned mapsize,rowcount_t *map,size32_t keybuffsize,const byte *keybuff, byte cmpfn,bool useaux) /* async */
{
CMessageBuffer mb;
mb.append((byte)FN_OverflowAdjustMapStart);
mb.append(mapsize).append(mapsize*sizeof(rowcount_t),map);
serializeblk(mb,keybuffsize,keybuff).append(cmpfn).append(useaux);
sendRecv(mb);
}
rowcount_t SortSlaveMP::OverflowAdjustMapStop( unsigned mapsize, rowcount_t *map)
{
CMessageBuffer mb;
mb.append((byte)FN_OverflowAdjustMapStop);
mb.append(mapsize);
sendRecv(mb);
rowcount_t ret;
mb.read(ret).read(mapsize*sizeof(rowcount_t),map);
return ret;
}
bool SortSlaveMP::Connect(unsigned _part, unsigned _numnodes)
{
CMessageBuffer mb;
mb.append((byte)FN_Connect);
mb.append(_part).append(_numnodes);
sendRecv(mb);
bool ret;
mb.read(ret);
return ret;
}
bool SortSlaveMP::FirstRowOfFile(const char *filename,size32_t &rowbuffsize, byte * &rowbuf)
{
CMessageBuffer mb;
mb.append((byte)FN_FirstRowOfFile);
mb.append(filename);
sendRecv(mb);
deserializeblk(mb,rowbuffsize,rowbuf);
bool ret;
mb.read(ret);
return ret;
}
void SortSlaveMP::MultiMerge(unsigned mapsize,rowcount_t *map,unsigned num,SocketEndpoint* endpoints) /* async */
{
CMessageBuffer mb;
mb.append((byte)FN_MultiMerge);
mb.append(mapsize).append(mapsize*sizeof(rowcount_t),map);
mb.append(num);
while (num--) {
endpoints->serialize(mb);
endpoints++;
}
sendRecv(mb);
}
void CDiskWriteSlaveActivityBase::close()
{
try
{
if (out) {
uncompressedBytesWritten = out->getPosition();
if (calcFileCrc) {
if (diskHelperBase->getFlags() & TDWextend) {
assertex(!"TBD need to merge CRC");
}
else
out->flush(&fileCRC);
}
else if (!abortSoon)
out->flush();
out.clear();
}
else if (outraw) {
outraw->flush();
uncompressedBytesWritten = outraw->tell();
outraw.clear();
}
{
CriticalBlock block(statsCs);
mergeStats(fileStats, outputIO);
outputIO.clear();
}
if (!rfsQueryParallel && dlfn.isExternal() && !lastNode())
{
rowcount_t rows = processed & THORDATALINK_COUNT_MASK;
ActPrintLog("External write done, signalling next (row count = %" RCPF "d)", rows);
CMessageBuffer msg;
msg.append(rows);
msg.append(tempExternalName);
queryJobChannel().queryJobComm().send(msg, queryJobChannel().queryMyRank()+1, mpTag);
}
}
catch (IException *e)
{
ActPrintLogEx(&queryContainer(), e, thorlog_null, MCwarning, "Error closing file: %s", fName.get());
abortSoon = true;
removeFiles();
throw e;
}
if (abortSoon)
removeFiles();
}
__int64 RemoteDataSource::numRows(bool force)
{
if (!force)
return cachedNumRows;
CMessageBuffer msg;
msg.setEndian(__BIG_ENDIAN);
msg.append((byte)FVCMDnumrows);
msg.append(id);
sendReceive(msg);
__int64 result;
msg.read(result);
return result;
}
void sendHeaderLines(unsigned subFile, unsigned part)
{
if (0 == headerLinesRemaining[subFile])
{
if (sentHeaderLines->testSet(subFile))
return;
unsigned which = gotHeaderLines->scan(0, false);
if (which == subFiles) // all received
{
bool someLeft=false;
unsigned hL=0;
for (; hL<subFiles; hL++)
{
if (headerLinesRemaining[hL])
{
someLeft = true;
break;
}
}
if (!someLeft)
{
sendAllDone();
return;
}
}
}
else
{
if (localLastPart[subFile] != part) // only ready to send if last local part
return;
if (sentHeaderLines->testSet(subFile))
return;
}
CMessageBuffer msgMb;
msgMb.append(subFile);
msgMb.append(headerLinesRemaining[subFile]);
// inform next slave about all subfiles I'm not dealing with.
for (unsigned s=0; s<subFiles; s++)
{
if (NotFound == localLastPart[s])
{
sentHeaderLines->testSet(s);
msgMb.append(s);
msgMb.append(headerLinesRemaining[s]);
}
}
queryJobChannel().queryJobComm().send(msgMb, queryJobChannel().queryMyRank()+1, mpTag);
}
int LogMsgLogReceiverThread::run()
{
while(!done)
{
try
{
if(queryWorldCommunicator().recv(in, childNode, MPTAG_JLOG_CHILD_TO_PARENT))
{
msgBuffer.deserialize(in, true);
if(isListener)
listener->report(msgBuffer);
else
queryLogMsgManager()->report(msgBuffer);
}
}
catch(IException * e)
{
done = true;
CMessageBuffer out;
out.append('D').append(cid);
try
{
queryWorldCommunicator().send(out, queryMyNode(), MPTAG_JLOG_CONNECT_TO_PARENT, MP_ASYNC_SEND);
}
catch(IException * ee)
{
ee->Release();
}
e->Release();
}
}
return 0;
}
void sendPartialCount(CSlaveActivity &activity, rowcount_t partialCount)
{
CMessageBuffer msg;
msg.append(partialCount);
if (!activity.queryContainer().queryJob().queryJobComm().send(msg, 0, activity.queryMpTag(), 5000))
throw MakeThorException(0, "Failed to give partial result to master");
}
bool UnregisterSelf(IException *e)
{
if (!hasMPServerStarted())
return false;
StringBuffer slfStr;
slfEp.getUrlStr(slfStr);
LOG(MCdebugProgress, thorJob, "Unregistering slave : %s", slfStr.str());
try
{
CMessageBuffer msg;
msg.append((int)rc_deregister);
serializeException(e, msg); // NB: allows exception to be NULL
if (!queryWorldCommunicator().send(msg, masterNode, MPTAG_THORREGISTRATION, 60*1000))
{
LOG(MCerror, thorJob, "Failed to unregister slave : %s", slfStr.str());
return false;
}
LOG(MCdebugProgress, thorJob, "Unregistered slave : %s", slfStr.str());
return true;
}
catch (IException *e) {
if (!jobListenerStopped)
FLLOG(MCexception(e), thorJob, e,"slave unregistration error");
e->Release();
}
return false;
}
void notify(MemoryBuffer &returndata) // if returns false should unsubscribe
{
if (hasaborted) {
throw MakeStringException(-1,"Subscription notification aborted");
return;
}
size32_t dlen = returndata.length();
CMessageBuffer mb;
mb.append(tag).append(sid).append(dlen).append(returndata);
try {
if (!queryWorldCommunicator().send(mb,dst,MPTAG_DALI_SUBSCRIPTION_FULFILL,1000*60*3)) {
// Must reply in 3 Minutes
// Kludge to avoid locking SDS on blocked client
hasaborted = true;
StringBuffer tmp;
throw MakeStringException(-1,"Subscription notification to %s timed out",dst->endpoint().getUrlStr(tmp).str());
return;
}
}
catch (IMP_Exception *e) {
PrintExceptionLog(e,"Dali CSubscriptionStub");
hasaborted = true;
throw;
}
}
void SortSlaveMP::GetMultiMidPointStop(size32_t &mkeybuffsize, void * &mkeybuf)
{
CMessageBuffer mb;
mb.append((byte)FN_GetMultiMidPointStop);
sendRecv(mb);
deserializeblk(mb,mkeybuffsize,mkeybuf);
}
void SortSlaveMP::MultiBinChopStart(size32_t keybuffsize,const byte *keybuff, byte cmpfn) /* async */
{
CMessageBuffer mb;
mb.append((byte)FN_MultiBinChopStart);
serializeblk(mb,keybuffsize,keybuff).append(cmpfn);
sendRecv(mb);
}
void process()
{
ActPrintLog("ReDistributeActivityMaster::process");
HashDistributeMasterBase::process();
IHThorHashDistributeArg *helper = (IHThorHashDistributeArg *)queryHelper();
unsigned n = container.queryJob().querySlaves();
MemoryAttr ma;
offset_t *sizes = (offset_t *)ma.allocate(sizeof(offset_t)*n);
unsigned i;
try {
for (i=0;i<n;i++) {
if (abortSoon)
return;
CMessageBuffer mb;
#ifdef _TRACE
ActPrintLog("ReDistribute process, Receiving on tag %d",statstag);
#endif
rank_t sender;
if (!receiveMsg(mb, RANK_ALL, statstag, &sender)||abortSoon)
return;
#ifdef _TRACE
ActPrintLog("ReDistribute process, Received size from %d",sender);
#endif
sender--;
assertex((unsigned)sender<n);
mb.read(sizes[sender]);
}
ActPrintLog("ReDistributeActivityMaster::process sizes got");
for (i=0;i<n;i++) {
CMessageBuffer mb;
mb.append(n*sizeof(offset_t),sizes);
#ifdef _TRACE
ActPrintLog("ReDistribute process, Replying to node %d tag %d",i+1,statstag);
#endif
if (!container.queryJob().queryJobComm().send(mb, (rank_t)i+1, statstag))
return;
}
// check if any max skew broken
double maxskew = helper->getTargetSkew();
if (maxskew>helper->getSkew()) {
offset_t tot = 0;
for (i=0;i<n;i++)
tot += sizes[i];
offset_t avg = tot/n;
for (i=0;i<n;i++) {
double r = ((double)sizes[i]-(double)avg)/(double)avg;
if ((r>=maxskew)||(-r>maxskew)) {
throw MakeActivityException(this, TE_DistributeFailedSkewExceeded, "DISTRIBUTE maximum skew exceeded (node %d has %"I64F"d, average is %"I64F"d)",i+1,sizes[i],avg);
}
}
}
}
catch (IException *e) {
ActPrintLog(e,"ReDistribute");
throw;
}
ActPrintLog("ReDistributeActivityMaster::process exit");
}
void sendResult(rowcount_t r)
{
if (resultSent) return;
resultSent = true;
CMessageBuffer mb;
mb.append(r);
container.queryJob().queryJobComm().send(mb, 0, mpTag);
}
MemoryBuffer & getDaliDiagnosticValue(MemoryBuffer &m)
{
CMessageBuffer mb;
mb.append((int)MDR_GET_VALUE).append(m);
queryCoven().sendRecv(mb,RANK_RANDOM,MPTAG_DALI_DIAGNOSTICS_REQUEST);
m.swapWith(mb);
return m;
}
void SortSlaveMP::GetMultiMidPointStart(size32_t lkeybuffsize, const void * lkeybuff, size32_t hkeybuffsize, const void * hkeybuff) /* async */
{
CMessageBuffer mb;
mb.append((byte)FN_GetMultiMidPointStart);
serializeblk(mb,lkeybuffsize,lkeybuff);
serializeblk(mb,hkeybuffsize,hkeybuff);
sendRecv(mb);
}
void SortSlaveMP::MultiBinChop(size32_t keybuffsize,const byte *keybuff, unsigned num,rowcount_t *pos,byte cmpfn)
{
CMessageBuffer mb;
mb.append((byte)FN_MultiBinChop);
serializeblk(mb,keybuffsize,keybuff).append(num).append(cmpfn);
sendRecv(mb);
mb.read(num*sizeof(rowcount_t),pos);
}
virtual void onInputFinished(rowcount_t count) override
{
if (container.queryLocalOrGrouped())
return;
CMessageBuffer msg;
msg.append(numSets*sizeof(rowcount_t), counts);
queryJobChannel().queryJobComm().send(msg, 0, mpTag);
}
virtual void process()
{
CChooseSetsActivityMaster::process();
IHThorChooseSetsArg *helper = (IHThorChooseSetsArg *)queryHelper();
unsigned numSets = helper->getNumSets();
unsigned nslaves = container.queryJob().querySlaves();
MemoryBuffer countMb;
rowcount_t *counts = (rowcount_t *)countMb.reserveTruncate((numSets*(nslaves+2)) * sizeof(rowcount_t));
rowcount_t *totals = counts + nslaves*numSets;
rowcount_t *tallies = totals + numSets;
memset(counts, 0, countMb.length());
unsigned s=nslaves;
CMessageBuffer msg;
while (s--)
{
msg.clear();
rank_t sender;
if (!receiveMsg(msg, RANK_ALL, mpTag, &sender))
return;
assertex(msg.length() == numSets*sizeof(rowcount_t));
unsigned set = (unsigned)sender - 1;
memcpy(&counts[set*numSets], msg.toByteArray(), numSets*sizeof(rowcount_t));
}
for (s=0; s<nslaves; s++)
{
unsigned i=0;
for (; i<numSets; i++)
totals[i] += counts[s * numSets + i];
}
msg.clear();
msg.append(numSets*sizeof(rowcount_t), totals);
unsigned endTotalsPos = msg.length();
for (s=0; s<nslaves; s++)
{
msg.rewrite(endTotalsPos);
msg.append(numSets*sizeof(rowcount_t), tallies);
container.queryJob().queryJobComm().send(msg, s+1, mpTag);
unsigned i=0;
for (; i<numSets; i++)
tallies[i] += counts[s * numSets + i];
}
}
bool RemoteDataSource::fetchRawRow(MemoryBuffer & out, __int64 offset)
{
CMessageBuffer msg;
msg.setEndian(__BIG_ENDIAN);
msg.append(FVCMDfetchraw);
msg.append(id);
msg.append(offset);
sendReceive(msg);
bool ok;
msg.read(ok);
if (!ok) return false;
size32_t len;
msg.read(len);
out.append(len, msg.readDirect(len));
return true;
}