virtual void preStart(size32_t parentExtractSz, const byte *parentExtract)
{
CMasterActivity::preStart(parentExtractSz, parentExtract);
ActPrintLog("preStart");
imaster = CreateThorSorterMaster(this);
unsigned s=0;
for (; s<container.queryJob().querySlaves(); s++)
{
SocketEndpoint ep;
ep.deserialize(queryInitializationData(s)); // this is a bit of a Kludge until we get proper MP Thor
imaster->AddSlave(&container.queryJob().queryJobComm(), s+1, ep,mpTagRPC);
}
}
void TransferServer::deserializeAction(MemoryBuffer & msg, unsigned action)
{
SocketEndpoint ep;
ep.deserialize(msg);
if (!ep.isLocal())
{
StringBuffer host, expected;
queryHostIP().getIpText(host);
ep.getIpText(expected);
throwError2(DFTERR_WrongComputer, expected.str(), host.str());
}
srcFormat.deserialize(msg);
tgtFormat.deserialize(msg);
msg.read(calcInputCRC);
msg.read(calcOutputCRC);
deserialize(partition, msg);
msg.read(numParallelSlaves);
msg.read(updateFrequency);
msg.read(replicate);
msg.read(mirror);
msg.read(isSafeMode);
srand((unsigned)get_cycles_now());
int adjust = (rand() * rand() * rand()) % updateFrequency - (updateFrequency/2);
lastTick = msTick() + adjust;
StringBuffer localFilename;
if (action == FTactionpull)
{
partition.item(0).outputName.getPath(localFilename);
LOG(MCdebugProgress, unknownJob, "Process Pull Command: %s", localFilename.str());
}
else
{
partition.item(0).inputName.getPath(localFilename);
LOG(MCdebugProgress, unknownJob, "Process Push Command: %s", localFilename.str());
}
LOG(MCdebugProgress, unknownJob, "Num Parallel Slaves=%d Adjust=%d/%d", numParallelSlaves, adjust, updateFrequency);
LOG(MCdebugProgress, unknownJob, "replicate(%d) mirror(%d) safe(%d) incrc(%d) outcrc(%d)", replicate, mirror, isSafeMode, calcInputCRC, calcOutputCRC);
displayPartition(partition);
unsigned numProgress;
msg.read(numProgress);
for (unsigned i = 0; i < numProgress; i++)
{
OutputProgress & next = *new OutputProgress;
next.deserializeCore(msg);
progress.append(next);
}
if (msg.remaining())
msg.read(throttleNicSpeed);
if (msg.remaining())
msg.read(compressedInput).read(compressOutput);
if (msg.remaining())
msg.read(copyCompressed);
if (msg.remaining())
msg.read(transferBufferSize);
if (msg.remaining())
msg.read(encryptKey).read(decryptKey);
if (msg.remaining())
{
srcFormat.deserializeExtra(msg, 1);
tgtFormat.deserializeExtra(msg, 1);
}
ForEachItemIn(i1, progress)
progress.item(i1).deserializeExtra(msg, 1);
LOG(MCdebugProgress, unknownJob, "throttle(%d), transferBufferSize(%d)", throttleNicSpeed, transferBufferSize);
PROGLOG("compressedInput(%d), compressedOutput(%d), copyCompressed(%d)", compressedInput?1:0, compressOutput?1:0, copyCompressed?1:0);
PROGLOG("encrypt(%d), decrypt(%d)", encryptKey.isEmpty()?0:1, decryptKey.isEmpty()?0:1);
//---Finished deserializing ---
displayProgress(progress);
totalLengthRead = 0;
totalLengthToRead = 0;
ForEachItemIn(idx, partition)
totalLengthToRead += partition.item(idx).inputLength;
}
bool SortSlaveMP::marshall(ISortSlaveMP &slave, ICommunicator* comm, mptag_t tag)
{
CMessageBuffer mb;
rank_t sender;
comm->recv(mb,0,tag,&sender); // NB only recv from master
if (mb.length()==0) {
PROGLOG("Stopping SortSlaveMP::marshall");
return false;
}
byte fn;
mb.read(fn);
CMessageBuffer mbout;
mbout.init(mb.getSender(),tag,mb.getReplyTag());
byte okout=1;
mbout.append(okout);
#ifdef FULLTRACE
StringBuffer tmp1;
PROGLOG(">SortSlaveMP::marshall(%d) got %d from %s tag %d replytag %d",(int)fn, mb.length(), mb.getSender().getUrlStr(tmp1).str(),tag,mb.getReplyTag());
#endif
bool replydone = false;
Owned<IException> err;
try {
switch ((MPSlaveFunctions)(int)fn) {
case FN_Connect: {
unsigned _part;
unsigned _numnodes;
mb.read(_part).read(_numnodes);
bool ret = slave.Connect(_part,_numnodes);
mbout.append(ret);
}
break;
case FN_StartGather: {
slave.StartGather();
}
break;
case FN_GetGatherInfo: {
bool hasserializer;
mb.read(hasserializer);
rowcount_t numlocal;
unsigned overflowscale;
offset_t totalsize;
slave.GetGatherInfo(numlocal,totalsize,overflowscale,hasserializer);
mbout.append(numlocal).append(totalsize).append(overflowscale);
}
break;
case FN_GetMinMax: {
size32_t keybuffsize;
void *keybuff;
size32_t avrecsize;
rowcount_t ret = slave.GetMinMax(keybuffsize,keybuff,avrecsize);
serializeblk(mbout,keybuffsize,keybuff).append(avrecsize).append(ret);
free(keybuff);
}
break;
case FN_GetMultiMidPointStart: {
replydone = true;
comm->reply(mbout);
size32_t lkeybuffsize;
void * lkeybuff;
size32_t hkeybuffsize;
void * hkeybuff;
deserializeblk(mb,lkeybuffsize,lkeybuff);
deserializeblk(mb,hkeybuffsize,hkeybuff);
slave.GetMultiMidPointStart(lkeybuffsize,lkeybuff,hkeybuffsize,hkeybuff);
free(lkeybuff);
free(hkeybuff);
}
break;
case FN_MultiBinChopStop: {
unsigned num;
mb.read(num);
void *out = mbout.reserveTruncate(num*sizeof(rowcount_t));
slave.MultiBinChopStop(num,(rowcount_t *)out);
}
break;
case FN_GetMultiMidPointStop: {
size32_t mkeybuffsize=0;
void * mkeybuff = NULL;
slave.GetMultiMidPointStop(mkeybuffsize,mkeybuff);
serializeblk(mbout,mkeybuffsize,mkeybuff);
free(mkeybuff);
}
break;
case FN_MultiBinChopStart: {
replydone = true;
comm->reply(mbout);
size32_t keybuffsize;
void * keybuff;
deserializeblk(mb,keybuffsize,keybuff);
byte cmpfn;
mb.read(cmpfn);
slave.MultiBinChopStart(keybuffsize,(const byte *)keybuff,cmpfn);
free(keybuff);
}
break;
case FN_MultiBinChop: {
size32_t keybuffsize;
void * keybuff;
deserializeblk(mb,keybuffsize,keybuff);
unsigned num;
byte cmpfn;
mb.read(num).read(cmpfn);
void *out = mbout.reserveTruncate(num*sizeof(rowcount_t));
slave.MultiBinChop(keybuffsize,(const byte *)keybuff,num,(rowcount_t *)out,cmpfn);
free(keybuff);
}
break;
case FN_OverflowAdjustMapStart: {
replydone = true;
comm->reply(mbout);
unsigned mapsize;
mb.read(mapsize);
const void * map = mb.readDirect(mapsize*sizeof(rowcount_t));
size32_t keybuffsize;
void * keybuff;
deserializeblk(mb,keybuffsize,keybuff);
byte cmpfn;
mb.read(cmpfn);
bool useaux;
mb.read(useaux);
slave.OverflowAdjustMapStart(mapsize,(rowcount_t *)map,keybuffsize,(const byte *)keybuff,cmpfn,useaux);
free(keybuff);
}
break;
case FN_OverflowAdjustMapStop: {
unsigned mapsize;
mb.read(mapsize);
rowcount_t ret=0;
size32_t retofs = mbout.length();
mbout.append(ret);
void *map=mbout.reserveTruncate(mapsize*sizeof(rowcount_t));
ret = slave.OverflowAdjustMapStop(mapsize,(rowcount_t *)map); // could avoid copy here if passed mb
mbout.writeDirect(retofs,sizeof(ret),&ret);
}
break;
case FN_MultiMerge: {
replydone = true;
comm->reply(mbout);
unsigned mapsize;
mb.read(mapsize);
const void *map = mb.readDirect(mapsize*sizeof(rowcount_t));
unsigned num;
mb.read(num);
SocketEndpointArray epa;
for (unsigned i=0;i<num;i++) {
SocketEndpoint ep;
ep.deserialize(mb);
epa.append(ep);
}
slave.MultiMerge(mapsize,(rowcount_t *)map,num,epa.getArray());
}
break;
case FN_MultiMergeBetween: {
replydone = true;
comm->reply(mbout);
unsigned mapsize;
mb.read(mapsize);
const void *map = mb.readDirect(mapsize*sizeof(rowcount_t));
const void *mapupper = mb.readDirect(mapsize*sizeof(rowcount_t));
unsigned num;
mb.read(num);
SocketEndpointArray epa;
for (unsigned i=0;i<num;i++) {
SocketEndpoint ep;
ep.deserialize(mb);
epa.append(ep);
}
slave.MultiMergeBetween(mapsize,(rowcount_t *)map,(rowcount_t *)mapupper,num,epa.getArray());
}
break;
case FN_SingleMerge: {
replydone = true;
comm->reply(mbout); // async
slave.SingleMerge();
}
break;
case FN_FirstRowOfFile: {
StringAttr filename;
mb.read(filename);
size32_t rowbufsize = 0;
byte *rowbuf = NULL;
bool ret = slave.FirstRowOfFile(filename,rowbufsize,rowbuf);
serializeblk(mbout,rowbufsize,rowbuf);
free(rowbuf);
mbout.append(ret);
}
break;
case FN_GetMultiNthRow: {
unsigned numsplits;
mb.read(numsplits);
size32_t mkeybuffsize = 0;
void * mkeybuf = NULL;
slave.GetMultiNthRow(numsplits,mkeybuffsize,mkeybuf);
serializeblk(mbout,mkeybuffsize,mkeybuf);
free(mkeybuf);
}
break;
case FN_StartMiniSort: {
replydone = true;
rowcount_t totalrows;
mb.read(totalrows);
comm->reply(mbout); // async
slave.StartMiniSort(totalrows);
}
break;
case FN_Close: {
replydone = true;
comm->reply(mbout); // async
slave.Close();
}
break;
case FN_CloseWait: {
slave.CloseWait();
}
break;
case FN_Disconnect: {
comm->reply(mbout); // async
replydone = true;
slave.Disconnect();
}
// fall through
return false;
default:
throw MakeStringException(-1,"unknown function %d",(int)fn);
}
}
catch (IException *e) {
EXCLOG(e,"SortSlaveMP::marshall");
if (!replydone) {
mbout.clear();
okout = 0;
mbout.append(okout);
int err = e->errorCode();
mbout.append(err);
StringBuffer outs;
e->errorMessage(outs);
mbout.append(outs.str());
}
err.setown(e);
}
if (!replydone) {
#ifdef FULLTRACE
StringBuffer tmp1;
PROGLOG("<SortSlaveMP::marshall(%d) send %d to %s tag %d",(int)fn, mbout.length(), mbout.getSender().getUrlStr(tmp1).str(),mbout.getReplyTag());
#endif
comm->reply(mbout);
}
if (err.get())
throw err.getClear();
return true;
}
