// IThorSlaveProcess overloaded methods
virtual void process()
{
start();
processed = 0;
processed = THORDATALINK_STARTED;
try
{
if (container.queryLocalOrGrouped() || firstNode())
helper->start();
while(!abortSoon)
{
ActivityTimer t(totalCycles, timeActivities);
OwnedConstThorRow r = inputStream->ungroupedNextRow();
if (!r)
break;
helper->apply(r);
processed++;
}
if (container.queryLocalOrGrouped() || firstNode())
helper->end();
}
catch(CATCHALL)
{
ActPrintLog("APPLY: exception");
throw;
}
}
virtual void start() override
{
ActivityTimer s(totalCycles, timeActivities);
PARENT::start();
__uint64 numRows = helper->numRows();
// local when generated from a child query (the range is per node, don't split)
bool isLocal = container.queryLocalData() || container.queryOwner().isLocalChild();
if (!isLocal && ((helper->getFlags() & TTFdistributed) != 0))
{
__uint64 nodes = queryCodeContext()->getNodes();
__uint64 nodeid = queryCodeContext()->getNodeNum();
startRow = (nodeid * numRows) / nodes;
maxRow = ((nodeid + 1) * numRows) / nodes;
ActPrintLog("InlineSLAVE: numRows = %" I64F "d, nodes = %" I64F
"d, nodeid = %" I64F "d, start = %" I64F "d, max = %" I64F "d",
numRows, nodes, nodeid, startRow, maxRow);
}
else
{
startRow = 0;
// when not distributed, only first node compute, unless local
if (firstNode() || isLocal)
maxRow = numRows;
else
maxRow = 0;
}
currentRow = startRow;
}
CATCH_NEXTROW()
{
ActivityTimer t(totalCycles, timeActivities);
if (first)
{
first = false;
if (container.queryLocalOrGrouped() || firstNode())
memset(tallies, 0, sizeof(unsigned)*numSets);
else
getTallies();
done = helper->setCounts(tallies);
}
if (!done)
{
while (!abortSoon)
{
OwnedConstThorRow row = inputStream->ungroupedNextRow();
if (!row)
break;
switch (helper->getRecordAction(row))
{
case 2:
done = true;
//fall through
case 1:
dataLinkIncrement();
return row.getClear();
}
}
}
if (!container.queryLocalOrGrouped() && !lastNode())
sendTallies();
return NULL;
}
void PlanRep::removeDeg1Nodes(Stack<Deg1RestoreInfo> &S, const NodeArray<bool> &mark)
{
for(node v = firstNode(); v != nullptr; v = v->succ())
{
if(mark[v] || v->degree() == 0)
continue;
adjEntry adjRef;
for(adjRef = v->firstAdj();
adjRef != nullptr && mark[adjRef->twinNode()];
adjRef = adjRef->succ()) ;
if(adjRef == nullptr) {
// only marked nodes adjacent with v (need no reference entry)
for(adjEntry adj : v->adjEdges) {
node x = adj->twinNode();
S.push(Deg1RestoreInfo(m_eOrig[adj->theEdge()],m_vOrig[x],nullptr));
delNode(x);
}
} else {
adjEntry adj, adjNext, adjStart = adjRef;
for(adj = adjRef->cyclicSucc(); adj != adjStart; adj = adjNext)
{
adjNext = adj->cyclicSucc();
node x = adj->twinNode();
if(mark[x]) {
S.push(Deg1RestoreInfo(m_eOrig[adj->theEdge()],m_vOrig[x],adjRef));
delNode(x);
} else
adjRef = adj;
}
}
}
}
virtual void write()
{
StringBuffer rowTag;
OwnedRoxieString xmlpath(helper->getXmlIteratorPath());
if (!xmlpath)
{
rowTag.append(DEFAULTXMLROWTAG);
}
else
{
const char *path = xmlpath;
if (*path == '/') path++;
if (strchr(path, '/')) UNIMPLEMENTED;
rowTag.append(path);
}
StringBuffer out;
if (!dlfn.isExternal() || firstNode()) // if external, 1 header,footer
{
OwnedRoxieString suppliedHeader(helper->getHeader());
if (kind==TAKjsonwrite)
buildJsonHeader(out, suppliedHeader, rowTag);
else if (suppliedHeader)
out.set(suppliedHeader);
else
out.set(DEFAULTXMLHEADER).newline();
outraw->write(out.length(), out.str());
if (calcFileCrc)
fileCRC.tally(out.length(), out.str());
}
Owned<IXmlWriterExt> writer = createIXmlWriterExt(helper->getXmlFlags(), 0, NULL, (kind==TAKjsonwrite) ? WTJSON : WTStandard);
writer->outputBeginArray(rowTag); //need this to format rows, even if not outputting it below
while(!abortSoon)
{
OwnedConstThorRow row = input->ungroupedNextRow();
if (!row)
break;
writer->clear().outputBeginNested(rowTag, false);
helper->toXML((const byte *)row.get(), *writer);
writer->outputEndNested(rowTag);
outraw->write(writer->length(), writer->str());
if (calcFileCrc)
fileCRC.tally(writer->length(), writer->str());
processed++;
}
if (!dlfn.isExternal() || lastNode()) // if external, 1 header,footer
{
OwnedRoxieString suppliedFooter(helper->getFooter());
if (kind==TAKjsonwrite)
buildJsonFooter(out.clear().newline(), suppliedFooter, rowTag);
else if (suppliedFooter)
out.set(suppliedFooter);
else
out.set(DEFAULTXMLFOOTER).newline();
outraw->write(out.length(), out.str());
if (calcFileCrc)
fileCRC.tally(out.length(), out.str());
}
}
virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData) override
{
isLocal = 0 != (TIWlocal & helper->getFlags());
mpTag = container.queryJobChannel().deserializeMPTag(data);
mpTag2 = container.queryJobChannel().deserializeMPTag(data);
data.read(active);
if (active)
{
data.read(logicalFilename);
partDesc.setown(deserializePartFileDescriptor(data));
}
data.read(singlePartKey);
data.read(refactor);
if (singlePartKey)
buildTlk = false;
else
{
data.read(buildTlk);
if (firstNode())
{
if (buildTlk)
tlkDesc.setown(deserializePartFileDescriptor(data));
else if (!isLocal) // existing tlk then..
{
tlkDesc.setown(deserializePartFileDescriptor(data));
unsigned c;
data.read(c);
while (c--)
{
RemoteFilename rf;
rf.deserialize(data);
if (!existingTlkIFile)
{
Owned<IFile> iFile = createIFile(rf);
if (iFile->exists())
existingTlkIFile.set(iFile);
}
}
if (!existingTlkIFile)
throw MakeActivityException(this, TE_FileNotFound, "Top level key part does not exist, for key");
}
}
}
IOutputMetaData * diskSize = helper->queryDiskRecordSize();
assertex(!(diskSize->getMetaFlags() & MDFneedserializedisk));
if (diskSize->isVariableSize())
{
if (TIWmaxlength & helper->getFlags())
maxDiskRecordSize = helper->getMaxKeySize();
else
maxDiskRecordSize = KEYBUILD_MAXLENGTH; //Current default behaviour, could be improved in the future
}
else
maxDiskRecordSize = diskSize->getFixedSize();
reportOverflow = false;
}
void start()
{
ActivityTimer s(totalCycles, timeActivities, NULL);
dataLinkStart("TEMPTABLE", container.queryId());
currentRow = 0;
isLocal = container.queryOwnerId() && container.queryOwner().isLocalOnly();
eof = isLocal ? false : !firstNode();
}
virtual void abort() override
{
#if THOR_TRACE_LEVEL >= 5
ActPrintLog("CHOOSESETS: abort()");
#endif
CSlaveActivity::abort();
if (!container.queryLocalOrGrouped() && !firstNode())
cancelReceiveMsg(RANK_ALL, mpTag);
}
void getMetaInfo(ThorDataLinkMetaInfo &info)
{
initMetaInfo(info);
info.isSource = true;
info.unknownRowsOutput = false;
if (isLocal || firstNode())
info.totalRowsMin = helper->numRows();
else
info.totalRowsMin = 0;
info.totalRowsMax = info.totalRowsMin;
}
virtual void write()
{
StringBuffer rowTag;
OwnedRoxieString xmlpath(helper->getXmlIteratorPath());
if (!xmlpath)
{
rowTag.append("Row");
}
else
{
const char *path = xmlpath;
if (*path == '/') path++;
if (strchr(path, '/')) UNIMPLEMENTED;
rowTag.append(path);
}
StringBuffer xmlOutput;
CommonXmlWriter xmlWriter(helper->getXmlFlags());
if (!dlfn.isExternal() || firstNode()) // if external, 1 header,footer
{
OwnedRoxieString header(helper->getHeader());
if (header)
xmlOutput.clear().append(header);
else
xmlOutput.clear().append("<Dataset>").newline();
outraw->write(xmlOutput.length(), xmlOutput.toCharArray());
if (calcFileCrc)
fileCRC.tally(xmlOutput.length(), xmlOutput.toCharArray());
}
while(!abortSoon)
{
OwnedConstThorRow row = input->ungroupedNextRow();
if (!row)
break;
xmlWriter.clear().outputBeginNested(rowTag, false);
helper->toXML((const byte *)row.get(), xmlWriter);
xmlWriter.outputEndNested(rowTag);
outraw->write(xmlWriter.length(), xmlWriter.str());
if (calcFileCrc)
fileCRC.tally(xmlWriter.length(), xmlWriter.str());
processed++;
}
if (!dlfn.isExternal() || lastNode()) // if external, 1 header,footer
{
OwnedRoxieString footer(helper->getFooter());
if (footer)
xmlOutput.clear().append(footer);
else
xmlOutput.clear().append("</Dataset>").newline();
outraw->write(xmlOutput.length(), xmlOutput.toCharArray());
if (calcFileCrc)
fileCRC.tally(xmlOutput.length(), xmlOutput.toCharArray());
}
}
virtual void write()
{
if (!singleHF || firstNode())
{
const char * header = helper->queryCsvParameters()->queryHeader();
if (header)
{
csvOutput.beginLine();
csvOutput.writeHeaderLn(strlen(header),header);
const char * outText = csvOutput.str();
unsigned outLength = csvOutput.length();
outraw->write(outLength,outText);
if (calcFileCrc)
fileCRC.tally(outLength, outText);
}
}
while(!abortSoon)
{
OwnedConstThorRow r(input->ungroupedNextRow());
if (!r)
break;
csvOutput.beginLine();
helper->writeRow((const byte *)r.get(), &csvOutput);
csvOutput.endLine();
const char * outText = csvOutput.str();
unsigned outLength = csvOutput.length();
outraw->write(outLength,outText);
if (calcFileCrc)
fileCRC.tally(outLength, outText);
processed++;
}
if (!singleHF || lastNode())
{
const char * footer = helper->queryCsvParameters()->queryFooter();
if (footer)
{
csvOutput.beginLine();
csvOutput.writeHeaderLn(strlen(footer),footer);
const char * outText = csvOutput.str();
unsigned outLength = csvOutput.length();
outraw->write(outLength,outText);
if (calcFileCrc)
fileCRC.tally(outLength, outText);
}
}
}
void init(MemoryBuffer &data, MemoryBuffer &slaveData)
{
AggregateSlaveBase::init(data, slaveData);
appendOutputLinked(this);
helper = (IHThorAggregateArg *)queryHelper();
if (firstNode())
{
mpTag = createReplyTag();
slaveData.append((int &)mpTag);
}
}
void getMetaInfo(ThorDataLinkMetaInfo &info)
{
initMetaInfo(info);
info.isSource = true;
if (firstNode())
info.unknownRowsOutput = true;
else
{
info.unknownRowsOutput = false;
info.totalRowsMin = 0;
info.totalRowsMax = 0;
}
}
virtual void start() override
{
ActivityTimer s(totalCycles, timeActivities);
ActPrintLog(rolloverEnabled ? "GROUP: is global" : "GROUP: is local");
PARENT::start();
eogNext = prevEog = eof = false;
if (rolloverEnabled)
{
useRollover = !lastNode();
#ifdef _TESTING
ActPrintLog("Node number = %d, Total Nodes = %d", queryJobChannel().queryMyRank(), container.queryJob().querySlaves());
#endif
}
stream.set(inputStream);
startLastGroup = getDataLinkGlobalCount();
next.setown(getNext());
if (rolloverEnabled && !firstNode()) // 1st node can have nothing to send
{
Owned<IThorRowCollector> collector = createThorRowCollector(*this, this, NULL, stableSort_none, rc_mixed, SPILL_PRIORITY_SPILLABLE_STREAM);
Owned<IRowWriter> writer = collector->getWriter();
if (next)
{
ActPrintLog("GROUP: Sending first group to previous node(%d)", queryJobChannel().queryMyRank()-1);
for (;;)
{
writer->putRow(next.getLink());
if (abortSoon)
break; //always send group even when aborting
OwnedConstThorRow next2 = getNext();
if (!next2)
{
eof = true;
break;
}
else if (!helper->isSameGroup(next2, next))
{
next.setown(next2.getClear());
break;
}
next.setown(next2.getClear());
}
}
writer.clear();
ActPrintLog("GROUP: %" RCPF "d records to send", collector->numRows());
Owned<IRowStream> strm = collector->getStream();
rowServer.setown(createRowServer(this, strm, queryJobChannel().queryJobComm(), mpTag));
}
}
/** Destructor */
GGraph::~GGraph()
{
freeMarker( nodeTextIsShown);
for ( GNode *node = firstNode();
isNotNullP( node);
)
{
GNode* next = node->nextNode();
int ir_id = node->irId();
deleteNode( node);
node = next;
}
foreach ( GStyle *style, styles)
{
delete style;
}
}
void start()
{
ActivityTimer s(totalCycles, timeActivities, NULL);
dataLinkStart();
eogPending = false;
if (container.queryLocal() || firstNode())
{
CMessageBuffer reqMsg;
reqMsg.setReplyTag(replyTag);
reqMsg.append(smt_actMsg);
reqMsg.append(container.queryOwner().queryGraphId());
reqMsg.append(container.queryId());
if (!container.queryJob().queryJobComm().sendRecv(reqMsg, 0, container.queryJob().querySlaveMpTag(), LONGTIMEOUT))
throwUnexpected();
masterReplyMsg.swapWith(reqMsg);
}
}
CATCH_NEXTROW()
{
ActivityTimer t(totalCycles, timeActivities);
if (abortSoon || eof)
return NULL;
eof = true;
OwnedConstThorRow next = input->ungroupedNextRow();
RtlDynamicRowBuilder resultcr(queryRowAllocator());
size32_t sz = helper->clearAggregate(resultcr);
if (next)
{
hadElement = true;
sz = helper->processFirst(resultcr, next);
if (container.getKind() != TAKexistsaggregate)
{
while (!abortSoon)
{
next.setown(input->ungroupedNextRow());
if (!next)
break;
sz = helper->processNext(resultcr, next);
}
}
}
doStopInput();
if (!firstNode())
{
OwnedConstThorRow result(resultcr.finalizeRowClear(sz));
sendResult(result.get(),queryRowSerializer(), 1); // send partial result
return NULL;
}
OwnedConstThorRow ret = getResult(resultcr.finalizeRowClear(sz));
if (ret)
{
dataLinkIncrement();
return ret.getClear();
}
sz = helper->clearAggregate(resultcr);
return resultcr.finalizeRowClear(sz);
}
CATCH_NEXTROW()
{
ActivityTimer t(totalCycles, timeActivities);
loop {
if (eof || abortSoon)
break;
if (!prev) {
if (!global || firstNode()) {
// construct first row
RtlDynamicRowBuilder r(queryRowAllocator());
size32_t sz = helper->createDefault(r);
prev.setown(r.finalizeRowClear(sz));
}
else {
prev.setown(getFirst());
if (!prev) {
putNext(NULL); // send to next node (even though prev not got)
eof = true;
break;
}
}
}
OwnedConstThorRow next = inputStream->ungroupedNextRow();
if (!next) {
putNext(prev); // send to next node if applicable
eof = true;
break;
}
RtlDynamicRowBuilder ret(queryRowAllocator());
size32_t sz = helper->transform(ret, prev, next, ++count);
if (sz != 0) {
dataLinkIncrement();
prev.setown(ret.finalizeRowClear(sz));
return prev.getLink();
}
}
return NULL;
}
void QuadSynopsis::takeSnapshot(string text)
{
stringstream ss;
//ss<<"\n label=\"Trie "<<id<<"\";";
ss<<"\n label=\"example QUAD ARF\";";
string output=ss.str();
// output+=exportText(text);
// output+=exportText(getQueryText()+text);
output+=firstNode();
vector< vector<int> > low = getChildrenLowerBounds(lowp,highp);
vector< vector<int> > high = getChildrenUpperBounds(lowp,highp);
for(int i=0;i<low.size();i++)
buildGraph(0,i,low[i],high[i],output);
writeGraph(output);
}
DeprecatedValueListImplIterator DeprecatedValueListImpl::begin() const
{
return DeprecatedValueListImplIterator(firstNode());
}
void CDiskWriteSlaveActivityBase::setInputStream(unsigned index, CThorInput &_input, bool consumerOrdered)
{
PARENT::setInputStream(index, _input, consumerOrdered);
if (dlfn.isExternal() && !firstNode())
setLookAhead(0, createRowStreamLookAhead(this, inputStream, queryRowInterfaces(input), PROCESS_SMART_BUFFER_SIZE, isSmartBufferSpillNeeded(this), grouped, RCUNBOUND, NULL, &container.queryJob().queryIDiskUsage()));
}
virtual void abort()
{
AggregateSlaveBase::abort();
if (firstNode())
cancelReceiveMsg(1, mpTag);
}
void clear() {
while (firstNode()) remove(firstNode());
}
virtual void abort()
{
CSlaveActivity::abort();
if (!firstNode())
cancelReceiveMsg(RANK_ALL, mpTag);
}
void CDiskWriteSlaveActivityBase::abort()
{
ProcessSlaveActivity::abort();
if (!rfsQueryParallel && dlfn.isExternal() && !firstNode())
cancelReceiveMsg(container.queryJob().queryMyRank()-1, mpTag);
}
void CDiskWriteSlaveActivityBase::open()
{
if (dlfn.isExternal() && !firstNode())
{
input.setown(createDataLinkSmartBuffer(this, inputs.item(0), PROCESS_SMART_BUFFER_SIZE, isSmartBufferSpillNeeded(this), grouped, RCUNBOUND, NULL, false, &container.queryJob().queryIDiskUsage()));
startInput(input);
if (!rfsQueryParallel)
{
ActPrintLog("Blocked, waiting for previous part to complete write");
CMessageBuffer msg;
if (!receiveMsg(msg, container.queryJob().queryMyRank()-1, mpTag))
return;
rowcount_t prevRows;
msg.read(prevRows);
msg.read(tempExternalName); // reuse temp filename, last node will rename
ActPrintLog("Previous write row count = %"RCPF"d", prevRows);
}
}
else
{
input.set(inputs.item(0));
startInput(input);
}
processed = THORDATALINK_STARTED;
bool extend = 0 != (diskHelperBase->getFlags() & TDWextend);
if (extend)
ActPrintLog("Extending file %s", fName.get());
size32_t exclsz = 0;
calcFileCrc = true;
bool external = dlfn.isExternal();
bool query = dlfn.isQuery();
if (query && compress)
UNIMPLEMENTED;
bool direct = query || (external && !firstNode());
bool rename = !external || (!query && lastNode());
Owned<IFileIO> iFileIO = createMultipleWrite(this, *partDesc, exclsz, compress, extend||(external&&!query), ecomp, this, direct, rename, &abortSoon, (external&&!query) ? &tempExternalName : NULL);
if (compress)
{
ActPrintLog("Performing row compression on output file: %s", fName.get());
calcFileCrc = false;
}
Owned<IFileIOStream> stream;
if (wantRaw())
{
outraw.setown(createBufferedIOStream(iFileIO));
stream.set(outraw);
}
else
{
stream.setown(createIOStream(iFileIO));
out.setown(createRowWriter(stream,::queryRowSerializer(input),::queryRowAllocator(input),grouped,calcFileCrc,false)); // flushed by close
}
CDfsLogicalFileName dlfn;
dlfn.set(logicalFilename);
if (extend || (external && !query))
stream->seek(0,IFSend);
ActPrintLog("Created output stream for %s", fName.get());
}
void CDiskWriteSlaveActivityBase::open()
{
if (dlfn.isExternal() && !firstNode())
{
input.setown(createDataLinkSmartBuffer(this, inputs.item(0), PROCESS_SMART_BUFFER_SIZE, isSmartBufferSpillNeeded(this), grouped, RCUNBOUND, NULL, false, &container.queryJob().queryIDiskUsage()));
startInput(input);
if (!rfsQueryParallel)
{
ActPrintLog("Blocked, waiting for previous part to complete write");
CMessageBuffer msg;
if (!receiveMsg(msg, queryJobChannel().queryMyRank()-1, mpTag))
return;
rowcount_t prevRows;
msg.read(prevRows);
msg.read(tempExternalName); // reuse temp filename, last node will rename
ActPrintLog("Previous write row count = %" RCPF "d", prevRows);
}
}
else
{
input.set(inputs.item(0));
startInput(input);
}
processed = THORDATALINK_STARTED;
bool extend = 0 != (diskHelperBase->getFlags() & TDWextend);
if (extend)
ActPrintLog("Extending file %s", fName.get());
/* Fixed length record size is used when outputting compressed stream to determine run-length compression vs default LZW compression.
* NB: only for FLAT files, not CSV or XML
*/
size32_t diskRowMinSz = 0;
IOutputMetaData *diskRowMeta = diskHelperBase->queryDiskRecordSize()->querySerializedDiskMeta();
if (diskRowMeta->isFixedSize() && (TAKdiskwrite == container.getKind()))
{
diskRowMinSz = diskRowMeta->getMinRecordSize();
if (grouped)
diskRowMinSz += 1;
}
calcFileCrc = true;
bool external = dlfn.isExternal();
bool query = dlfn.isQuery();
if (query && compress)
UNIMPLEMENTED;
unsigned twFlags = external ? TW_External : 0;
if (query || (external && !firstNode()))
twFlags |= TW_Direct;
if (!external || (!query && lastNode()))
twFlags |= TW_RenameToPrimary;
if (extend||(external&&!query))
twFlags |= TW_Extend;
Owned<IFileIO> iFileIO = createMultipleWrite(this, *partDesc, diskRowMinSz, twFlags, compress, ecomp, this, &abortSoon, (external&&!query) ? &tempExternalName : NULL);
if (compress)
{
ActPrintLog("Performing row compression on output file: %s", fName.get());
// NB: block compressed output has implicit crc of 0, no need to calculate in row writer.
calcFileCrc = false;
}
Owned<IFileIOStream> stream;
if (wantRaw())
{
outraw.setown(createBufferedIOStream(iFileIO));
stream.set(outraw);
}
else
{
stream.setown(createIOStream(iFileIO));
unsigned rwFlags = 0;
if (grouped)
rwFlags |= rw_grouped;
if (calcFileCrc)
rwFlags |= rw_crc;
out.setown(createRowWriter(stream, ::queryRowInterfaces(input), rwFlags));
}
if (extend || (external && !query))
stream->seek(0,IFSend);
ActPrintLog("Created output stream for %s", fName.get());
}
void trainSequences(map<int,node> &nodesMap,
map<vector<int> ,int> &str2intnodemap,
map<int,vector<int> > &int2strnodemap,
map<vector<int> ,int> &str2intedgemap,
map<int,vector<int> > &int2stredgemap,
vector<vector<vector<int> > > &windows,
int n, int l,int *totaledges){
if(windows.size() == 0){
cerr<<"No windows. No training possible. Exiting ... \n";return;
}
int currEdgeID = 0;
int currNodeID = 0;
//find the largest id in int2strnodemap and int2stredgemap
map<int,vector<int> >::iterator it2;
for(it2 = int2strnodemap.begin();it2 != int2strnodemap.end();it2++){
if(it2->first > currNodeID) currNodeID = it2->first;
}
for(it2 = int2stredgemap.begin();it2 != int2stredgemap.end();it2++){
if(it2->first > currEdgeID) currEdgeID = it2->first;
}
for(unsigned int j = 0; j < windows.size(); j++){
map<int,node>::iterator it;
map<pair<int,int>,int>::iterator it1;
vector<int> buf,buf1,buf2;
int c_node,c_edge,n_node,n_edge;
//read the first window
buf = windows[j][0];
buf1 = vector<int>(buf.begin(),buf.begin()+n-1);
buf2 = vector<int>(buf.begin()+n-1,buf.end());
if(str2intnodemap.find(buf1) != str2intnodemap.end()){
c_node = str2intnodemap[buf1];
}else{
c_node = currNodeID;
currNodeID++;
str2intnodemap[buf1] = c_node;
int2strnodemap[c_node] = buf1;
}
if(str2intedgemap.find(buf2) != str2intedgemap.end()){
c_edge = str2intedgemap[buf2];
}else{
c_edge = currEdgeID;
currEdgeID++;
str2intedgemap[buf2] = c_edge;
//check this
int2stredgemap[c_edge] = buf2;
}
*totaledges += 1;
if(nodesMap.find(c_node) == nodesMap.end()){
//create a new node in the FSA
node firstNode(c_node);
nodesMap[c_node] = firstNode;
}
for(unsigned int i = 1; i < windows[j].size(); i++){
buf = windows[j][i];
buf1 = vector<int>(buf.begin(),buf.begin()+n-1);
buf2 = vector<int>(buf.begin()+n-1,buf.end());
if(str2intnodemap.find(buf1) != str2intnodemap.end()){
n_node = str2intnodemap[buf1];
}else{
n_node = currNodeID;
currNodeID++;
str2intnodemap[buf1] = n_node;
int2strnodemap[n_node] = buf1;
}
if(str2intedgemap.find(buf2) != str2intedgemap.end()){
n_edge = str2intedgemap[buf2];
}else{
n_edge = currEdgeID;
currEdgeID++;
str2intedgemap[buf2] = n_edge;
int2stredgemap[n_edge] = buf2;
}
*totaledges += 1;
it = nodesMap.find(c_node);
if(it == nodesMap.end()){cerr<<"Error.\n";exit(0);}
it->second.numedges += 1;
if(nodesMap.find(n_node) != nodesMap.end()){
it1 = (it->second).transitions.find(pair<int,int>(n_node,c_edge));
if(it1 != (it->second).transitions.end()){
//scenario 1: n_node exists and a transition from c_node to n_node
//with edge c_edge exists
it1->second += 1;//increment the count
}else{
//scenario 2: n_node exists and a transition from c_node to n_node
//with edge c_edge does not exist
(it->second).transitions[pair<int,int>(n_node,c_edge)] = 1;
}
}else{
//scenario 3: n_node does not exist
node one_node(n_node);
nodesMap[n_node] = one_node;
(it->second).transitions[pair<int,int>(n_node,c_edge)] = 1;
}
c_node = n_node;
c_edge = n_edge;
}
}
}
virtual void process() override
{
ActPrintLog("INDEXWRITE: Start");
init();
IRowStream *stream = inputStream;
ThorDataLinkMetaInfo info;
input->getMetaInfo(info);
outRowAllocator.setown(getRowAllocator(helper->queryDiskRecordSize()));
start();
if (refactor)
{
assertex(isLocal);
if (active)
{
unsigned targetWidth = partDesc->queryOwner().numParts()-(buildTlk?1:0);
assertex(0 == container.queryJob().querySlaves() % targetWidth);
unsigned partsPerNode = container.queryJob().querySlaves() / targetWidth;
unsigned myPart = queryJobChannel().queryMyRank();
IArrayOf<IRowStream> streams;
streams.append(*LINK(stream));
--partsPerNode;
// Should this be merging 1,11,21,31 etc.
unsigned p=0;
unsigned fromPart = targetWidth+1 + (partsPerNode * (myPart-1));
for (; p<partsPerNode; p++)
{
streams.append(*createRowStreamFromNode(*this, fromPart++, queryJobChannel().queryJobComm(), mpTag, abortSoon));
}
ICompare *icompare = helper->queryCompare();
assertex(icompare);
Owned<IRowLinkCounter> linkCounter = new CThorRowLinkCounter;
myInputStream.setown(createRowStreamMerger(streams.ordinality(), streams.getArray(), icompare, false, linkCounter));
stream = myInputStream;
}
else // serve nodes, creating merged parts
rowServer.setown(createRowServer(this, stream, queryJobChannel().queryJobComm(), mpTag));
}
processed = THORDATALINK_STARTED;
// single part key support
// has to serially pull all data fron nodes 2-N
// nodes 2-N, could/should start pushing some data (as it's supposed to be small) to cut down on serial nature.
unsigned node = queryJobChannel().queryMyRank();
if (singlePartKey)
{
if (1 == node)
{
try
{
open(*partDesc, false, helper->queryDiskRecordSize()->isVariableSize());
for (;;)
{
OwnedConstThorRow row = inputStream->ungroupedNextRow();
if (!row)
break;
if (abortSoon) return;
processRow(row);
}
unsigned node = 2;
while (node <= container.queryJob().querySlaves())
{
Linked<IOutputRowDeserializer> deserializer = ::queryRowDeserializer(input);
CMessageBuffer mb;
Owned<ISerialStream> stream = createMemoryBufferSerialStream(mb);
CThorStreamDeserializerSource rowSource;
rowSource.setStream(stream);
bool successSR;
for (;;)
{
{
BooleanOnOff tf(receivingTag2);
successSR = queryJobChannel().queryJobComm().sendRecv(mb, node, mpTag2);
}
if (successSR)
{
if (rowSource.eos())
break;
Linked<IEngineRowAllocator> allocator = ::queryRowAllocator(input);
do
{
RtlDynamicRowBuilder rowBuilder(allocator);
size32_t sz = deserializer->deserialize(rowBuilder, rowSource);
OwnedConstThorRow fRow = rowBuilder.finalizeRowClear(sz);
processRow(fRow);
}
while (!rowSource.eos());
}
}
node++;
}
}
catch (CATCHALL)
{
close(*partDesc, partCrc, true);
throw;
}
close(*partDesc, partCrc, true);
stop();
}
else
{
CMessageBuffer mb;
CMemoryRowSerializer mbs(mb);
Linked<IOutputRowSerializer> serializer = ::queryRowSerializer(input);
for (;;)
{
BooleanOnOff tf(receivingTag2);
if (queryJobChannel().queryJobComm().recv(mb, 1, mpTag2)) // node 1 asking for more..
{
if (abortSoon) break;
mb.clear();
do
{
OwnedConstThorRow row = inputStream->ungroupedNextRow();
if (!row) break;
serializer->serialize(mbs, (const byte *)row.get());
} while (mb.length() < SINGLEPART_KEY_TRANSFER_SIZE); // NB: at least one row
if (!queryJobChannel().queryJobComm().reply(mb))
throw MakeThorException(0, "Failed to send index data to node 1, from node %d", node);
if (0 == mb.length())
break;
}
}
}
}
else
{
if (!refactor || active)
{
try
{
StringBuffer partFname;
getPartFilename(*partDesc, 0, partFname);
ActPrintLog("INDEXWRITE: process: handling fname : %s", partFname.str());
open(*partDesc, false, helper->queryDiskRecordSize()->isVariableSize());
ActPrintLog("INDEXWRITE: write");
BooleanOnOff tf(receiving);
if (!refactor || !active)
receiving = false;
do
{
OwnedConstThorRow row = inputStream->ungroupedNextRow();
if (!row)
break;
processRow(row);
} while (!abortSoon);
ActPrintLog("INDEXWRITE: write level 0 complete");
}
catch (CATCHALL)
{
close(*partDesc, partCrc, isLocal && !buildTlk && 1 == node);
throw;
}
close(*partDesc, partCrc, isLocal && !buildTlk && 1 == node);
stop();
ActPrintLog("INDEXWRITE: Wrote %" RCPF "d records", processed & THORDATALINK_COUNT_MASK);
if (buildTlk)
{
ActPrintLog("INDEXWRITE: sending rows");
NodeInfoArray tlkRows;
CMessageBuffer msg;
if (firstNode())
{
if (processed & THORDATALINK_COUNT_MASK)
{
if (enableTlkPart0)
tlkRows.append(* new CNodeInfo(0, firstRow.get(), firstRowSize, totalCount));
tlkRows.append(* new CNodeInfo(1, lastRow.get(), lastRowSize, totalCount));
}
}
else
{
if (processed & THORDATALINK_COUNT_MASK)
{
CNodeInfo row(queryJobChannel().queryMyRank(), lastRow.get(), lastRowSize, totalCount);
row.serialize(msg);
}
queryJobChannel().queryJobComm().send(msg, 1, mpTag);
}
if (firstNode())
{
ActPrintLog("INDEXWRITE: Waiting on tlk to complete");
// JCSMORE if refactor==true, is rowsToReceive here right??
unsigned rowsToReceive = (refactor ? (tlkDesc->queryOwner().numParts()-1) : container.queryJob().querySlaves()) -1; // -1 'cos got my own in array already
ActPrintLog("INDEXWRITE: will wait for info from %d slaves before writing TLK", rowsToReceive);
while (rowsToReceive--)
{
msg.clear();
receiveMsg(msg, RANK_ALL, mpTag); // NH->JCS RANK_ALL_OTHER not supported for recv
if (abortSoon)
return;
if (msg.length())
{
CNodeInfo *ni = new CNodeInfo();
ni->deserialize(msg);
tlkRows.append(*ni);
}
}
tlkRows.sort(CNodeInfo::compare);
StringBuffer path;
getPartFilename(*tlkDesc, 0, path);
ActPrintLog("INDEXWRITE: creating toplevel key file : %s", path.str());
try
{
open(*tlkDesc, true, helper->queryDiskRecordSize()->isVariableSize());
if (tlkRows.length())
{
CNodeInfo &lastNode = tlkRows.item(tlkRows.length()-1);
memset(lastNode.value, 0xff, lastNode.size);
}
ForEachItemIn(idx, tlkRows)
{
CNodeInfo &info = tlkRows.item(idx);
builder->processKeyData((char *)info.value, info.pos, info.size);
}
close(*tlkDesc, tlkCrc, true);
}
catch (CATCHALL)
{
abortSoon = true;
close(*tlkDesc, tlkCrc, true);
removeFiles(*partDesc);
throw;
}
}
}
else if (!isLocal && firstNode())
