// IThorSlaveProcess overloaded methods virtual void process() { start(); processed = 0; processed = THORDATALINK_STARTED; try { if (container.queryLocalOrGrouped() || firstNode()) helper->start(); while(!abortSoon) { ActivityTimer t(totalCycles, timeActivities); OwnedConstThorRow r = inputStream->ungroupedNextRow(); if (!r) break; helper->apply(r); processed++; } if (container.queryLocalOrGrouped() || firstNode()) helper->end(); } catch(CATCHALL) { ActPrintLog("APPLY: exception"); throw; } }
virtual void start() override { ActivityTimer s(totalCycles, timeActivities); PARENT::start(); __uint64 numRows = helper->numRows(); // local when generated from a child query (the range is per node, don't split) bool isLocal = container.queryLocalData() || container.queryOwner().isLocalChild(); if (!isLocal && ((helper->getFlags() & TTFdistributed) != 0)) { __uint64 nodes = queryCodeContext()->getNodes(); __uint64 nodeid = queryCodeContext()->getNodeNum(); startRow = (nodeid * numRows) / nodes; maxRow = ((nodeid + 1) * numRows) / nodes; ActPrintLog("InlineSLAVE: numRows = %" I64F "d, nodes = %" I64F "d, nodeid = %" I64F "d, start = %" I64F "d, max = %" I64F "d", numRows, nodes, nodeid, startRow, maxRow); } else { startRow = 0; // when not distributed, only first node compute, unless local if (firstNode() || isLocal) maxRow = numRows; else maxRow = 0; } currentRow = startRow; }
CATCH_NEXTROW() { ActivityTimer t(totalCycles, timeActivities); if (first) { first = false; if (container.queryLocalOrGrouped() || firstNode()) memset(tallies, 0, sizeof(unsigned)*numSets); else getTallies(); done = helper->setCounts(tallies); } if (!done) { while (!abortSoon) { OwnedConstThorRow row = inputStream->ungroupedNextRow(); if (!row) break; switch (helper->getRecordAction(row)) { case 2: done = true; //fall through case 1: dataLinkIncrement(); return row.getClear(); } } } if (!container.queryLocalOrGrouped() && !lastNode()) sendTallies(); return NULL; }
void PlanRep::removeDeg1Nodes(Stack<Deg1RestoreInfo> &S, const NodeArray<bool> &mark) { for(node v = firstNode(); v != nullptr; v = v->succ()) { if(mark[v] || v->degree() == 0) continue; adjEntry adjRef; for(adjRef = v->firstAdj(); adjRef != nullptr && mark[adjRef->twinNode()]; adjRef = adjRef->succ()) ; if(adjRef == nullptr) { // only marked nodes adjacent with v (need no reference entry) for(adjEntry adj : v->adjEdges) { node x = adj->twinNode(); S.push(Deg1RestoreInfo(m_eOrig[adj->theEdge()],m_vOrig[x],nullptr)); delNode(x); } } else { adjEntry adj, adjNext, adjStart = adjRef; for(adj = adjRef->cyclicSucc(); adj != adjStart; adj = adjNext) { adjNext = adj->cyclicSucc(); node x = adj->twinNode(); if(mark[x]) { S.push(Deg1RestoreInfo(m_eOrig[adj->theEdge()],m_vOrig[x],adjRef)); delNode(x); } else adjRef = adj; } } } }
virtual void write() { StringBuffer rowTag; OwnedRoxieString xmlpath(helper->getXmlIteratorPath()); if (!xmlpath) { rowTag.append(DEFAULTXMLROWTAG); } else { const char *path = xmlpath; if (*path == '/') path++; if (strchr(path, '/')) UNIMPLEMENTED; rowTag.append(path); } StringBuffer out; if (!dlfn.isExternal() || firstNode()) // if external, 1 header,footer { OwnedRoxieString suppliedHeader(helper->getHeader()); if (kind==TAKjsonwrite) buildJsonHeader(out, suppliedHeader, rowTag); else if (suppliedHeader) out.set(suppliedHeader); else out.set(DEFAULTXMLHEADER).newline(); outraw->write(out.length(), out.str()); if (calcFileCrc) fileCRC.tally(out.length(), out.str()); } Owned<IXmlWriterExt> writer = createIXmlWriterExt(helper->getXmlFlags(), 0, NULL, (kind==TAKjsonwrite) ? WTJSON : WTStandard); writer->outputBeginArray(rowTag); //need this to format rows, even if not outputting it below while(!abortSoon) { OwnedConstThorRow row = input->ungroupedNextRow(); if (!row) break; writer->clear().outputBeginNested(rowTag, false); helper->toXML((const byte *)row.get(), *writer); writer->outputEndNested(rowTag); outraw->write(writer->length(), writer->str()); if (calcFileCrc) fileCRC.tally(writer->length(), writer->str()); processed++; } if (!dlfn.isExternal() || lastNode()) // if external, 1 header,footer { OwnedRoxieString suppliedFooter(helper->getFooter()); if (kind==TAKjsonwrite) buildJsonFooter(out.clear().newline(), suppliedFooter, rowTag); else if (suppliedFooter) out.set(suppliedFooter); else out.set(DEFAULTXMLFOOTER).newline(); outraw->write(out.length(), out.str()); if (calcFileCrc) fileCRC.tally(out.length(), out.str()); } }
virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData) override { isLocal = 0 != (TIWlocal & helper->getFlags()); mpTag = container.queryJobChannel().deserializeMPTag(data); mpTag2 = container.queryJobChannel().deserializeMPTag(data); data.read(active); if (active) { data.read(logicalFilename); partDesc.setown(deserializePartFileDescriptor(data)); } data.read(singlePartKey); data.read(refactor); if (singlePartKey) buildTlk = false; else { data.read(buildTlk); if (firstNode()) { if (buildTlk) tlkDesc.setown(deserializePartFileDescriptor(data)); else if (!isLocal) // existing tlk then.. { tlkDesc.setown(deserializePartFileDescriptor(data)); unsigned c; data.read(c); while (c--) { RemoteFilename rf; rf.deserialize(data); if (!existingTlkIFile) { Owned<IFile> iFile = createIFile(rf); if (iFile->exists()) existingTlkIFile.set(iFile); } } if (!existingTlkIFile) throw MakeActivityException(this, TE_FileNotFound, "Top level key part does not exist, for key"); } } } IOutputMetaData * diskSize = helper->queryDiskRecordSize(); assertex(!(diskSize->getMetaFlags() & MDFneedserializedisk)); if (diskSize->isVariableSize()) { if (TIWmaxlength & helper->getFlags()) maxDiskRecordSize = helper->getMaxKeySize(); else maxDiskRecordSize = KEYBUILD_MAXLENGTH; //Current default behaviour, could be improved in the future } else maxDiskRecordSize = diskSize->getFixedSize(); reportOverflow = false; }
void start() { ActivityTimer s(totalCycles, timeActivities, NULL); dataLinkStart("TEMPTABLE", container.queryId()); currentRow = 0; isLocal = container.queryOwnerId() && container.queryOwner().isLocalOnly(); eof = isLocal ? false : !firstNode(); }
virtual void abort() override { #if THOR_TRACE_LEVEL >= 5 ActPrintLog("CHOOSESETS: abort()"); #endif CSlaveActivity::abort(); if (!container.queryLocalOrGrouped() && !firstNode()) cancelReceiveMsg(RANK_ALL, mpTag); }
void getMetaInfo(ThorDataLinkMetaInfo &info) { initMetaInfo(info); info.isSource = true; info.unknownRowsOutput = false; if (isLocal || firstNode()) info.totalRowsMin = helper->numRows(); else info.totalRowsMin = 0; info.totalRowsMax = info.totalRowsMin; }
virtual void write() { StringBuffer rowTag; OwnedRoxieString xmlpath(helper->getXmlIteratorPath()); if (!xmlpath) { rowTag.append("Row"); } else { const char *path = xmlpath; if (*path == '/') path++; if (strchr(path, '/')) UNIMPLEMENTED; rowTag.append(path); } StringBuffer xmlOutput; CommonXmlWriter xmlWriter(helper->getXmlFlags()); if (!dlfn.isExternal() || firstNode()) // if external, 1 header,footer { OwnedRoxieString header(helper->getHeader()); if (header) xmlOutput.clear().append(header); else xmlOutput.clear().append("<Dataset>").newline(); outraw->write(xmlOutput.length(), xmlOutput.toCharArray()); if (calcFileCrc) fileCRC.tally(xmlOutput.length(), xmlOutput.toCharArray()); } while(!abortSoon) { OwnedConstThorRow row = input->ungroupedNextRow(); if (!row) break; xmlWriter.clear().outputBeginNested(rowTag, false); helper->toXML((const byte *)row.get(), xmlWriter); xmlWriter.outputEndNested(rowTag); outraw->write(xmlWriter.length(), xmlWriter.str()); if (calcFileCrc) fileCRC.tally(xmlWriter.length(), xmlWriter.str()); processed++; } if (!dlfn.isExternal() || lastNode()) // if external, 1 header,footer { OwnedRoxieString footer(helper->getFooter()); if (footer) xmlOutput.clear().append(footer); else xmlOutput.clear().append("</Dataset>").newline(); outraw->write(xmlOutput.length(), xmlOutput.toCharArray()); if (calcFileCrc) fileCRC.tally(xmlOutput.length(), xmlOutput.toCharArray()); } }
virtual void write() { if (!singleHF || firstNode()) { const char * header = helper->queryCsvParameters()->queryHeader(); if (header) { csvOutput.beginLine(); csvOutput.writeHeaderLn(strlen(header),header); const char * outText = csvOutput.str(); unsigned outLength = csvOutput.length(); outraw->write(outLength,outText); if (calcFileCrc) fileCRC.tally(outLength, outText); } } while(!abortSoon) { OwnedConstThorRow r(input->ungroupedNextRow()); if (!r) break; csvOutput.beginLine(); helper->writeRow((const byte *)r.get(), &csvOutput); csvOutput.endLine(); const char * outText = csvOutput.str(); unsigned outLength = csvOutput.length(); outraw->write(outLength,outText); if (calcFileCrc) fileCRC.tally(outLength, outText); processed++; } if (!singleHF || lastNode()) { const char * footer = helper->queryCsvParameters()->queryFooter(); if (footer) { csvOutput.beginLine(); csvOutput.writeHeaderLn(strlen(footer),footer); const char * outText = csvOutput.str(); unsigned outLength = csvOutput.length(); outraw->write(outLength,outText); if (calcFileCrc) fileCRC.tally(outLength, outText); } } }
void init(MemoryBuffer &data, MemoryBuffer &slaveData) { AggregateSlaveBase::init(data, slaveData); appendOutputLinked(this); helper = (IHThorAggregateArg *)queryHelper(); if (firstNode()) { mpTag = createReplyTag(); slaveData.append((int &)mpTag); } }
void getMetaInfo(ThorDataLinkMetaInfo &info) { initMetaInfo(info); info.isSource = true; if (firstNode()) info.unknownRowsOutput = true; else { info.unknownRowsOutput = false; info.totalRowsMin = 0; info.totalRowsMax = 0; } }
virtual void start() override { ActivityTimer s(totalCycles, timeActivities); ActPrintLog(rolloverEnabled ? "GROUP: is global" : "GROUP: is local"); PARENT::start(); eogNext = prevEog = eof = false; if (rolloverEnabled) { useRollover = !lastNode(); #ifdef _TESTING ActPrintLog("Node number = %d, Total Nodes = %d", queryJobChannel().queryMyRank(), container.queryJob().querySlaves()); #endif } stream.set(inputStream); startLastGroup = getDataLinkGlobalCount(); next.setown(getNext()); if (rolloverEnabled && !firstNode()) // 1st node can have nothing to send { Owned<IThorRowCollector> collector = createThorRowCollector(*this, this, NULL, stableSort_none, rc_mixed, SPILL_PRIORITY_SPILLABLE_STREAM); Owned<IRowWriter> writer = collector->getWriter(); if (next) { ActPrintLog("GROUP: Sending first group to previous node(%d)", queryJobChannel().queryMyRank()-1); for (;;) { writer->putRow(next.getLink()); if (abortSoon) break; //always send group even when aborting OwnedConstThorRow next2 = getNext(); if (!next2) { eof = true; break; } else if (!helper->isSameGroup(next2, next)) { next.setown(next2.getClear()); break; } next.setown(next2.getClear()); } } writer.clear(); ActPrintLog("GROUP: %" RCPF "d records to send", collector->numRows()); Owned<IRowStream> strm = collector->getStream(); rowServer.setown(createRowServer(this, strm, queryJobChannel().queryJobComm(), mpTag)); } }
/** Destructor */ GGraph::~GGraph() { freeMarker( nodeTextIsShown); for ( GNode *node = firstNode(); isNotNullP( node); ) { GNode* next = node->nextNode(); int ir_id = node->irId(); deleteNode( node); node = next; } foreach ( GStyle *style, styles) { delete style; } }
void start() { ActivityTimer s(totalCycles, timeActivities, NULL); dataLinkStart(); eogPending = false; if (container.queryLocal() || firstNode()) { CMessageBuffer reqMsg; reqMsg.setReplyTag(replyTag); reqMsg.append(smt_actMsg); reqMsg.append(container.queryOwner().queryGraphId()); reqMsg.append(container.queryId()); if (!container.queryJob().queryJobComm().sendRecv(reqMsg, 0, container.queryJob().querySlaveMpTag(), LONGTIMEOUT)) throwUnexpected(); masterReplyMsg.swapWith(reqMsg); } }
CATCH_NEXTROW() { ActivityTimer t(totalCycles, timeActivities); if (abortSoon || eof) return NULL; eof = true; OwnedConstThorRow next = input->ungroupedNextRow(); RtlDynamicRowBuilder resultcr(queryRowAllocator()); size32_t sz = helper->clearAggregate(resultcr); if (next) { hadElement = true; sz = helper->processFirst(resultcr, next); if (container.getKind() != TAKexistsaggregate) { while (!abortSoon) { next.setown(input->ungroupedNextRow()); if (!next) break; sz = helper->processNext(resultcr, next); } } } doStopInput(); if (!firstNode()) { OwnedConstThorRow result(resultcr.finalizeRowClear(sz)); sendResult(result.get(),queryRowSerializer(), 1); // send partial result return NULL; } OwnedConstThorRow ret = getResult(resultcr.finalizeRowClear(sz)); if (ret) { dataLinkIncrement(); return ret.getClear(); } sz = helper->clearAggregate(resultcr); return resultcr.finalizeRowClear(sz); }
CATCH_NEXTROW() { ActivityTimer t(totalCycles, timeActivities); loop { if (eof || abortSoon) break; if (!prev) { if (!global || firstNode()) { // construct first row RtlDynamicRowBuilder r(queryRowAllocator()); size32_t sz = helper->createDefault(r); prev.setown(r.finalizeRowClear(sz)); } else { prev.setown(getFirst()); if (!prev) { putNext(NULL); // send to next node (even though prev not got) eof = true; break; } } } OwnedConstThorRow next = inputStream->ungroupedNextRow(); if (!next) { putNext(prev); // send to next node if applicable eof = true; break; } RtlDynamicRowBuilder ret(queryRowAllocator()); size32_t sz = helper->transform(ret, prev, next, ++count); if (sz != 0) { dataLinkIncrement(); prev.setown(ret.finalizeRowClear(sz)); return prev.getLink(); } } return NULL; }
void QuadSynopsis::takeSnapshot(string text) { stringstream ss; //ss<<"\n label=\"Trie "<<id<<"\";"; ss<<"\n label=\"example QUAD ARF\";"; string output=ss.str(); // output+=exportText(text); // output+=exportText(getQueryText()+text); output+=firstNode(); vector< vector<int> > low = getChildrenLowerBounds(lowp,highp); vector< vector<int> > high = getChildrenUpperBounds(lowp,highp); for(int i=0;i<low.size();i++) buildGraph(0,i,low[i],high[i],output); writeGraph(output); }
DeprecatedValueListImplIterator DeprecatedValueListImpl::begin() const { return DeprecatedValueListImplIterator(firstNode()); }
void CDiskWriteSlaveActivityBase::setInputStream(unsigned index, CThorInput &_input, bool consumerOrdered) { PARENT::setInputStream(index, _input, consumerOrdered); if (dlfn.isExternal() && !firstNode()) setLookAhead(0, createRowStreamLookAhead(this, inputStream, queryRowInterfaces(input), PROCESS_SMART_BUFFER_SIZE, isSmartBufferSpillNeeded(this), grouped, RCUNBOUND, NULL, &container.queryJob().queryIDiskUsage())); }
virtual void abort() { AggregateSlaveBase::abort(); if (firstNode()) cancelReceiveMsg(1, mpTag); }
void clear() { while (firstNode()) remove(firstNode()); }
virtual void abort() { CSlaveActivity::abort(); if (!firstNode()) cancelReceiveMsg(RANK_ALL, mpTag); }
void CDiskWriteSlaveActivityBase::abort() { ProcessSlaveActivity::abort(); if (!rfsQueryParallel && dlfn.isExternal() && !firstNode()) cancelReceiveMsg(container.queryJob().queryMyRank()-1, mpTag); }
void CDiskWriteSlaveActivityBase::open() { if (dlfn.isExternal() && !firstNode()) { input.setown(createDataLinkSmartBuffer(this, inputs.item(0), PROCESS_SMART_BUFFER_SIZE, isSmartBufferSpillNeeded(this), grouped, RCUNBOUND, NULL, false, &container.queryJob().queryIDiskUsage())); startInput(input); if (!rfsQueryParallel) { ActPrintLog("Blocked, waiting for previous part to complete write"); CMessageBuffer msg; if (!receiveMsg(msg, container.queryJob().queryMyRank()-1, mpTag)) return; rowcount_t prevRows; msg.read(prevRows); msg.read(tempExternalName); // reuse temp filename, last node will rename ActPrintLog("Previous write row count = %"RCPF"d", prevRows); } } else { input.set(inputs.item(0)); startInput(input); } processed = THORDATALINK_STARTED; bool extend = 0 != (diskHelperBase->getFlags() & TDWextend); if (extend) ActPrintLog("Extending file %s", fName.get()); size32_t exclsz = 0; calcFileCrc = true; bool external = dlfn.isExternal(); bool query = dlfn.isQuery(); if (query && compress) UNIMPLEMENTED; bool direct = query || (external && !firstNode()); bool rename = !external || (!query && lastNode()); Owned<IFileIO> iFileIO = createMultipleWrite(this, *partDesc, exclsz, compress, extend||(external&&!query), ecomp, this, direct, rename, &abortSoon, (external&&!query) ? &tempExternalName : NULL); if (compress) { ActPrintLog("Performing row compression on output file: %s", fName.get()); calcFileCrc = false; } Owned<IFileIOStream> stream; if (wantRaw()) { outraw.setown(createBufferedIOStream(iFileIO)); stream.set(outraw); } else { stream.setown(createIOStream(iFileIO)); out.setown(createRowWriter(stream,::queryRowSerializer(input),::queryRowAllocator(input),grouped,calcFileCrc,false)); // flushed by close } CDfsLogicalFileName dlfn; dlfn.set(logicalFilename); if (extend || (external && !query)) stream->seek(0,IFSend); ActPrintLog("Created output stream for %s", fName.get()); }
void CDiskWriteSlaveActivityBase::open() { if (dlfn.isExternal() && !firstNode()) { input.setown(createDataLinkSmartBuffer(this, inputs.item(0), PROCESS_SMART_BUFFER_SIZE, isSmartBufferSpillNeeded(this), grouped, RCUNBOUND, NULL, false, &container.queryJob().queryIDiskUsage())); startInput(input); if (!rfsQueryParallel) { ActPrintLog("Blocked, waiting for previous part to complete write"); CMessageBuffer msg; if (!receiveMsg(msg, queryJobChannel().queryMyRank()-1, mpTag)) return; rowcount_t prevRows; msg.read(prevRows); msg.read(tempExternalName); // reuse temp filename, last node will rename ActPrintLog("Previous write row count = %" RCPF "d", prevRows); } } else { input.set(inputs.item(0)); startInput(input); } processed = THORDATALINK_STARTED; bool extend = 0 != (diskHelperBase->getFlags() & TDWextend); if (extend) ActPrintLog("Extending file %s", fName.get()); /* Fixed length record size is used when outputting compressed stream to determine run-length compression vs default LZW compression. * NB: only for FLAT files, not CSV or XML */ size32_t diskRowMinSz = 0; IOutputMetaData *diskRowMeta = diskHelperBase->queryDiskRecordSize()->querySerializedDiskMeta(); if (diskRowMeta->isFixedSize() && (TAKdiskwrite == container.getKind())) { diskRowMinSz = diskRowMeta->getMinRecordSize(); if (grouped) diskRowMinSz += 1; } calcFileCrc = true; bool external = dlfn.isExternal(); bool query = dlfn.isQuery(); if (query && compress) UNIMPLEMENTED; unsigned twFlags = external ? TW_External : 0; if (query || (external && !firstNode())) twFlags |= TW_Direct; if (!external || (!query && lastNode())) twFlags |= TW_RenameToPrimary; if (extend||(external&&!query)) twFlags |= TW_Extend; Owned<IFileIO> iFileIO = createMultipleWrite(this, *partDesc, diskRowMinSz, twFlags, compress, ecomp, this, &abortSoon, (external&&!query) ? &tempExternalName : NULL); if (compress) { ActPrintLog("Performing row compression on output file: %s", fName.get()); // NB: block compressed output has implicit crc of 0, no need to calculate in row writer. calcFileCrc = false; } Owned<IFileIOStream> stream; if (wantRaw()) { outraw.setown(createBufferedIOStream(iFileIO)); stream.set(outraw); } else { stream.setown(createIOStream(iFileIO)); unsigned rwFlags = 0; if (grouped) rwFlags |= rw_grouped; if (calcFileCrc) rwFlags |= rw_crc; out.setown(createRowWriter(stream, ::queryRowInterfaces(input), rwFlags)); } if (extend || (external && !query)) stream->seek(0,IFSend); ActPrintLog("Created output stream for %s", fName.get()); }
void trainSequences(map<int,node> &nodesMap, map<vector<int> ,int> &str2intnodemap, map<int,vector<int> > &int2strnodemap, map<vector<int> ,int> &str2intedgemap, map<int,vector<int> > &int2stredgemap, vector<vector<vector<int> > > &windows, int n, int l,int *totaledges){ if(windows.size() == 0){ cerr<<"No windows. No training possible. Exiting ... \n";return; } int currEdgeID = 0; int currNodeID = 0; //find the largest id in int2strnodemap and int2stredgemap map<int,vector<int> >::iterator it2; for(it2 = int2strnodemap.begin();it2 != int2strnodemap.end();it2++){ if(it2->first > currNodeID) currNodeID = it2->first; } for(it2 = int2stredgemap.begin();it2 != int2stredgemap.end();it2++){ if(it2->first > currEdgeID) currEdgeID = it2->first; } for(unsigned int j = 0; j < windows.size(); j++){ map<int,node>::iterator it; map<pair<int,int>,int>::iterator it1; vector<int> buf,buf1,buf2; int c_node,c_edge,n_node,n_edge; //read the first window buf = windows[j][0]; buf1 = vector<int>(buf.begin(),buf.begin()+n-1); buf2 = vector<int>(buf.begin()+n-1,buf.end()); if(str2intnodemap.find(buf1) != str2intnodemap.end()){ c_node = str2intnodemap[buf1]; }else{ c_node = currNodeID; currNodeID++; str2intnodemap[buf1] = c_node; int2strnodemap[c_node] = buf1; } if(str2intedgemap.find(buf2) != str2intedgemap.end()){ c_edge = str2intedgemap[buf2]; }else{ c_edge = currEdgeID; currEdgeID++; str2intedgemap[buf2] = c_edge; //check this int2stredgemap[c_edge] = buf2; } *totaledges += 1; if(nodesMap.find(c_node) == nodesMap.end()){ //create a new node in the FSA node firstNode(c_node); nodesMap[c_node] = firstNode; } for(unsigned int i = 1; i < windows[j].size(); i++){ buf = windows[j][i]; buf1 = vector<int>(buf.begin(),buf.begin()+n-1); buf2 = vector<int>(buf.begin()+n-1,buf.end()); if(str2intnodemap.find(buf1) != str2intnodemap.end()){ n_node = str2intnodemap[buf1]; }else{ n_node = currNodeID; currNodeID++; str2intnodemap[buf1] = n_node; int2strnodemap[n_node] = buf1; } if(str2intedgemap.find(buf2) != str2intedgemap.end()){ n_edge = str2intedgemap[buf2]; }else{ n_edge = currEdgeID; currEdgeID++; str2intedgemap[buf2] = n_edge; int2stredgemap[n_edge] = buf2; } *totaledges += 1; it = nodesMap.find(c_node); if(it == nodesMap.end()){cerr<<"Error.\n";exit(0);} it->second.numedges += 1; if(nodesMap.find(n_node) != nodesMap.end()){ it1 = (it->second).transitions.find(pair<int,int>(n_node,c_edge)); if(it1 != (it->second).transitions.end()){ //scenario 1: n_node exists and a transition from c_node to n_node //with edge c_edge exists it1->second += 1;//increment the count }else{ //scenario 2: n_node exists and a transition from c_node to n_node //with edge c_edge does not exist (it->second).transitions[pair<int,int>(n_node,c_edge)] = 1; } }else{ //scenario 3: n_node does not exist node one_node(n_node); nodesMap[n_node] = one_node; (it->second).transitions[pair<int,int>(n_node,c_edge)] = 1; } c_node = n_node; c_edge = n_edge; } } }
virtual void process() override { ActPrintLog("INDEXWRITE: Start"); init(); IRowStream *stream = inputStream; ThorDataLinkMetaInfo info; input->getMetaInfo(info); outRowAllocator.setown(getRowAllocator(helper->queryDiskRecordSize())); start(); if (refactor) { assertex(isLocal); if (active) { unsigned targetWidth = partDesc->queryOwner().numParts()-(buildTlk?1:0); assertex(0 == container.queryJob().querySlaves() % targetWidth); unsigned partsPerNode = container.queryJob().querySlaves() / targetWidth; unsigned myPart = queryJobChannel().queryMyRank(); IArrayOf<IRowStream> streams; streams.append(*LINK(stream)); --partsPerNode; // Should this be merging 1,11,21,31 etc. unsigned p=0; unsigned fromPart = targetWidth+1 + (partsPerNode * (myPart-1)); for (; p<partsPerNode; p++) { streams.append(*createRowStreamFromNode(*this, fromPart++, queryJobChannel().queryJobComm(), mpTag, abortSoon)); } ICompare *icompare = helper->queryCompare(); assertex(icompare); Owned<IRowLinkCounter> linkCounter = new CThorRowLinkCounter; myInputStream.setown(createRowStreamMerger(streams.ordinality(), streams.getArray(), icompare, false, linkCounter)); stream = myInputStream; } else // serve nodes, creating merged parts rowServer.setown(createRowServer(this, stream, queryJobChannel().queryJobComm(), mpTag)); } processed = THORDATALINK_STARTED; // single part key support // has to serially pull all data fron nodes 2-N // nodes 2-N, could/should start pushing some data (as it's supposed to be small) to cut down on serial nature. unsigned node = queryJobChannel().queryMyRank(); if (singlePartKey) { if (1 == node) { try { open(*partDesc, false, helper->queryDiskRecordSize()->isVariableSize()); for (;;) { OwnedConstThorRow row = inputStream->ungroupedNextRow(); if (!row) break; if (abortSoon) return; processRow(row); } unsigned node = 2; while (node <= container.queryJob().querySlaves()) { Linked<IOutputRowDeserializer> deserializer = ::queryRowDeserializer(input); CMessageBuffer mb; Owned<ISerialStream> stream = createMemoryBufferSerialStream(mb); CThorStreamDeserializerSource rowSource; rowSource.setStream(stream); bool successSR; for (;;) { { BooleanOnOff tf(receivingTag2); successSR = queryJobChannel().queryJobComm().sendRecv(mb, node, mpTag2); } if (successSR) { if (rowSource.eos()) break; Linked<IEngineRowAllocator> allocator = ::queryRowAllocator(input); do { RtlDynamicRowBuilder rowBuilder(allocator); size32_t sz = deserializer->deserialize(rowBuilder, rowSource); OwnedConstThorRow fRow = rowBuilder.finalizeRowClear(sz); processRow(fRow); } while (!rowSource.eos()); } } node++; } } catch (CATCHALL) { close(*partDesc, partCrc, true); throw; } close(*partDesc, partCrc, true); stop(); } else { CMessageBuffer mb; CMemoryRowSerializer mbs(mb); Linked<IOutputRowSerializer> serializer = ::queryRowSerializer(input); for (;;) { BooleanOnOff tf(receivingTag2); if (queryJobChannel().queryJobComm().recv(mb, 1, mpTag2)) // node 1 asking for more.. { if (abortSoon) break; mb.clear(); do { OwnedConstThorRow row = inputStream->ungroupedNextRow(); if (!row) break; serializer->serialize(mbs, (const byte *)row.get()); } while (mb.length() < SINGLEPART_KEY_TRANSFER_SIZE); // NB: at least one row if (!queryJobChannel().queryJobComm().reply(mb)) throw MakeThorException(0, "Failed to send index data to node 1, from node %d", node); if (0 == mb.length()) break; } } } } else { if (!refactor || active) { try { StringBuffer partFname; getPartFilename(*partDesc, 0, partFname); ActPrintLog("INDEXWRITE: process: handling fname : %s", partFname.str()); open(*partDesc, false, helper->queryDiskRecordSize()->isVariableSize()); ActPrintLog("INDEXWRITE: write"); BooleanOnOff tf(receiving); if (!refactor || !active) receiving = false; do { OwnedConstThorRow row = inputStream->ungroupedNextRow(); if (!row) break; processRow(row); } while (!abortSoon); ActPrintLog("INDEXWRITE: write level 0 complete"); } catch (CATCHALL) { close(*partDesc, partCrc, isLocal && !buildTlk && 1 == node); throw; } close(*partDesc, partCrc, isLocal && !buildTlk && 1 == node); stop(); ActPrintLog("INDEXWRITE: Wrote %" RCPF "d records", processed & THORDATALINK_COUNT_MASK); if (buildTlk) { ActPrintLog("INDEXWRITE: sending rows"); NodeInfoArray tlkRows; CMessageBuffer msg; if (firstNode()) { if (processed & THORDATALINK_COUNT_MASK) { if (enableTlkPart0) tlkRows.append(* new CNodeInfo(0, firstRow.get(), firstRowSize, totalCount)); tlkRows.append(* new CNodeInfo(1, lastRow.get(), lastRowSize, totalCount)); } } else { if (processed & THORDATALINK_COUNT_MASK) { CNodeInfo row(queryJobChannel().queryMyRank(), lastRow.get(), lastRowSize, totalCount); row.serialize(msg); } queryJobChannel().queryJobComm().send(msg, 1, mpTag); } if (firstNode()) { ActPrintLog("INDEXWRITE: Waiting on tlk to complete"); // JCSMORE if refactor==true, is rowsToReceive here right?? unsigned rowsToReceive = (refactor ? (tlkDesc->queryOwner().numParts()-1) : container.queryJob().querySlaves()) -1; // -1 'cos got my own in array already ActPrintLog("INDEXWRITE: will wait for info from %d slaves before writing TLK", rowsToReceive); while (rowsToReceive--) { msg.clear(); receiveMsg(msg, RANK_ALL, mpTag); // NH->JCS RANK_ALL_OTHER not supported for recv if (abortSoon) return; if (msg.length()) { CNodeInfo *ni = new CNodeInfo(); ni->deserialize(msg); tlkRows.append(*ni); } } tlkRows.sort(CNodeInfo::compare); StringBuffer path; getPartFilename(*tlkDesc, 0, path); ActPrintLog("INDEXWRITE: creating toplevel key file : %s", path.str()); try { open(*tlkDesc, true, helper->queryDiskRecordSize()->isVariableSize()); if (tlkRows.length()) { CNodeInfo &lastNode = tlkRows.item(tlkRows.length()-1); memset(lastNode.value, 0xff, lastNode.size); } ForEachItemIn(idx, tlkRows) { CNodeInfo &info = tlkRows.item(idx); builder->processKeyData((char *)info.value, info.pos, info.size); } close(*tlkDesc, tlkCrc, true); } catch (CATCHALL) { abortSoon = true; close(*tlkDesc, tlkCrc, true); removeFiles(*partDesc); throw; } } } else if (!isLocal && firstNode())