CATCH_NEXTROW() { ActivityTimer t(totalCycles, timeActivities); if (abortSoon || eof) return NULL; eof = true; OwnedConstThorRow next = inputStream->ungroupedNextRow(); RtlDynamicRowBuilder resultcr(queryRowAllocator()); size32_t sz = helper->clearAggregate(resultcr); if (next) { hadElement = true; sz = helper->processFirst(resultcr, next); if (container.getKind() != TAKexistsaggregate) { while (!abortSoon) { next.setown(inputStream->ungroupedNextRow()); if (!next) break; sz = helper->processNext(resultcr, next); } } } doStopInput(); if (!firstNode()) { OwnedConstThorRow result(resultcr.finalizeRowClear(sz)); sendResult(result.get(),queryRowSerializer(), 1); // send partial result return NULL; } OwnedConstThorRow ret = getResult(resultcr.finalizeRowClear(sz)); if (ret) { dataLinkIncrement(); return ret.getClear(); } sz = helper->clearAggregate(resultcr); return resultcr.finalizeRowClear(sz); }
CATCH_NEXTROW() { ActivityTimer t(totalCycles, timeActivities); loop { OwnedConstThorRow row = input->ungroupedNextRow(); if(!row || done || abortSoon) break; switch (helper->getRecordAction(row)) { case 2: done = true; //fall through case 1: dataLinkIncrement(); return row.getClear(); } } return NULL; }
CATCH_NEXTROW() { ActivityTimer t(totalCycles, timeActivities, NULL); if (abortSoon || eoi) return NULL; OwnedConstThorRow row = out->nextRow(); if (!row) { if (!container.queryGrouped()) { eoi = true; return NULL; } out.setown(iLoader->loadGroup(input, abortSoon)); if (0 == iLoader->numRows()) eoi = true; return NULL; // eog marker } dataLinkIncrement(); return row.getClear(); }
void process() { processed = 0; input = inputs.item(0); startInput(input); processed = THORDATALINK_STARTED; OwnedConstThorRow row = input->ungroupedNextRow(); CMessageBuffer mb; size32_t lenpos = mb.length(); // its 0 really mb.append((size32_t)0); if (row) { CMemoryRowSerializer msz(mb); ::queryRowSerializer(input)->serialize(msz,(const byte *)row.get()); size32_t sz = mb.length()-lenpos-sizeof(size32_t); mb.writeDirect(lenpos,sizeof(size32_t),&sz); processed++; } container.queryJob().queryJobComm().send(mb, 0, masterMpTag); }
CATCH_NEXTROW() { ActivityTimer t(totalCycles, timeActivities); loop { if(abortSoon) break; if(eogNext) { eogNext = false; count = 0; if (anyThisGroup) { // ignore eogNext if none in group anyThisGroup = false; break; } } OwnedConstThorRow row = input->nextRow(); if (!row) { count = 0; if (anyThisGroup) { anyThisGroup = false; break; } row.setown(input->nextRow()); if (!row) break; } RtlDynamicRowBuilder ret(queryRowAllocator()); size32_t thisSize = helper->transform(ret, anyThisGroup?prev.get():defaultLeft.get(), row, ++count); if (thisSize != 0) { const void *r = ret.finalizeRowClear(thisSize); prev.set(r); dataLinkIncrement(); anyThisGroup = true; return r; } } return NULL; }
STRAND_CATCH_NEXTROW() { ActivityTimer t(totalCycles, timeActivities); loop { if (parent.queryAbortSoon()) return nullptr; OwnedConstThorRow in = inputStream->nextRow(); if (!in) { if (numProcessedLastGroup == rowsProcessed) in.setown(inputStream->nextRow()); if (!in) { numProcessedLastGroup = rowsProcessed; return nullptr; } } RtlDynamicRowBuilder rowBuilder(allocator); size32_t outSize; try { outSize = helper->transform(rowBuilder, in); } catch (IException *e) { parent.ActPrintLog(e, "In helper->transform()"); throw; } if (outSize) { rowsProcessed++; return rowBuilder.finalizeRowClear(outSize); } } }
IRowStream *createFirstNReadSeqVar(IRowStream *input, unsigned limit) { class CFirstNReadSeqVar : public CSimpleInterface, implements IRowStream { IRowStream *input; unsigned limit, c; bool stopped; public: IMPLEMENT_IINTERFACE_USING(CSimpleInterface); CFirstNReadSeqVar(IRowStream *_input, unsigned _limit) : input(_input), limit(_limit), c(0), stopped(false) { } ~CFirstNReadSeqVar() { ::Release(input); } // IRowStream const void *nextRow() { if (c<limit) { OwnedConstThorRow row = input->nextRow(); if (row) { c++; return row.getClear(); } } stop(); return NULL; } virtual void stop() { if (!stopped) { stopped = true; input->stop(); } } };
void operator = (const OwnedConstThorRow & other) { set(other.get()); }
inline OwnedConstThorRow(const OwnedConstThorRow & other) { ptr = other.getLink(); }
inline void set(const OwnedConstThorRow &other) { set(other.get()); }
virtual void process() override { ActPrintLog("INDEXWRITE: Start"); init(); IRowStream *stream = inputStream; ThorDataLinkMetaInfo info; input->getMetaInfo(info); outRowAllocator.setown(getRowAllocator(helper->queryDiskRecordSize())); start(); if (refactor) { assertex(isLocal); if (active) { unsigned targetWidth = partDesc->queryOwner().numParts()-(buildTlk?1:0); assertex(0 == container.queryJob().querySlaves() % targetWidth); unsigned partsPerNode = container.queryJob().querySlaves() / targetWidth; unsigned myPart = queryJobChannel().queryMyRank(); IArrayOf<IRowStream> streams; streams.append(*LINK(stream)); --partsPerNode; // Should this be merging 1,11,21,31 etc. unsigned p=0; unsigned fromPart = targetWidth+1 + (partsPerNode * (myPart-1)); for (; p<partsPerNode; p++) { streams.append(*createRowStreamFromNode(*this, fromPart++, queryJobChannel().queryJobComm(), mpTag, abortSoon)); } ICompare *icompare = helper->queryCompare(); assertex(icompare); Owned<IRowLinkCounter> linkCounter = new CThorRowLinkCounter; myInputStream.setown(createRowStreamMerger(streams.ordinality(), streams.getArray(), icompare, false, linkCounter)); stream = myInputStream; } else // serve nodes, creating merged parts rowServer.setown(createRowServer(this, stream, queryJobChannel().queryJobComm(), mpTag)); } processed = THORDATALINK_STARTED; // single part key support // has to serially pull all data fron nodes 2-N // nodes 2-N, could/should start pushing some data (as it's supposed to be small) to cut down on serial nature. unsigned node = queryJobChannel().queryMyRank(); if (singlePartKey) { if (1 == node) { try { open(*partDesc, false, helper->queryDiskRecordSize()->isVariableSize()); loop { OwnedConstThorRow row = inputStream->ungroupedNextRow(); if (!row) break; if (abortSoon) return; processRow(row); } unsigned node = 2; while (node <= container.queryJob().querySlaves()) { Linked<IOutputRowDeserializer> deserializer = ::queryRowDeserializer(input); CMessageBuffer mb; Owned<ISerialStream> stream = createMemoryBufferSerialStream(mb); CThorStreamDeserializerSource rowSource; rowSource.setStream(stream); bool successSR; loop { { BooleanOnOff tf(receivingTag2); successSR = queryJobChannel().queryJobComm().sendRecv(mb, node, mpTag2); } if (successSR) { if (rowSource.eos()) break; Linked<IEngineRowAllocator> allocator = ::queryRowAllocator(input); do { RtlDynamicRowBuilder rowBuilder(allocator); size32_t sz = deserializer->deserialize(rowBuilder, rowSource); OwnedConstThorRow fRow = rowBuilder.finalizeRowClear(sz); processRow(fRow); } while (!rowSource.eos()); } } node++; } } catch (CATCHALL) { close(*partDesc, partCrc, true); throw; } close(*partDesc, partCrc, true); doStopInput(); } else { CMessageBuffer mb; CMemoryRowSerializer mbs(mb); Linked<IOutputRowSerializer> serializer = ::queryRowSerializer(input); loop { BooleanOnOff tf(receivingTag2); if (queryJobChannel().queryJobComm().recv(mb, 1, mpTag2)) // node 1 asking for more.. { if (abortSoon) break; mb.clear(); do { OwnedConstThorRow row = inputStream->ungroupedNextRow(); if (!row) break; serializer->serialize(mbs, (const byte *)row.get()); } while (mb.length() < SINGLEPART_KEY_TRANSFER_SIZE); // NB: at least one row if (!queryJobChannel().queryJobComm().reply(mb)) throw MakeThorException(0, "Failed to send index data to node 1, from node %d", node); if (0 == mb.length()) break; } } } }
CATCH_NEXTROW() { ActivityTimer t(totalCycles, timeActivities); if (abortSoon) return NULL; if (eogNext) { eogNext = false; anyThisGroup = false; return NULL; } try { for (;;) { if (xmlParser) { for (;;) { if (!xmlParser->next()) { if (helper->searchTextNeedsFree()) { rtlFree(searchStr); searchStr = NULL; } xmlParser.clear(); break; } if (lastMatch) { RtlDynamicRowBuilder row(allocator); size32_t sizeGot; try { sizeGot = helper->transform(row, nxt, lastMatch); } catch (IException *e) { ActPrintLog(e, "In helper->transform()"); throw; } lastMatch.clear(); if (sizeGot == 0) continue; // not sure if this will ever be possible in this context. dataLinkIncrement(); anyThisGroup = true; OwnedConstThorRow ret = row.finalizeRowClear(sizeGot); return ret.getClear(); } } } nxt.setown(inputStream->nextRow()); if (!nxt && !anyThisGroup) nxt.setown(inputStream->nextRow()); if (!nxt) break; unsigned len; helper->getSearchText(len, searchStr, nxt); OwnedRoxieString xmlIteratorPath(helper->getXmlIteratorPath()); xmlParser.setown(createXMLParse(searchStr, len, xmlIteratorPath, *this, ptr_noRoot, helper->requiresContents())); } } catch (IOutOfMemException *e) { StringBuffer s("XMLParse actId("); s.append(container.queryId()).append(") out of memory.").newline(); s.append("INTERNAL ERROR ").append(e->errorCode()).append(": "); e->errorMessage(s); e->Release(); throw MakeActivityException(this, 0, "%s", s.str()); } catch (IException *e) { StringBuffer s("XMLParse actId("); s.append(container.queryId()); s.append(") INTERNAL ERROR ").append(e->errorCode()).append(": "); e->errorMessage(s); e->Release(); throw MakeActivityException(this, 0, "%s", s.str()); } eogNext = false; anyThisGroup = false; return NULL; }
int run() { if (!started) { try { in->start(); started = true; } catch(IException * e) { ActPrintLog(&activity, e, "ThorLookaheadCache starting input"); startexception.setown(e); if (asyncstart) notify->onInputStarted(startexception); running = false; stopped = true; startsem.signal(); return 0; } } try { StringBuffer temp; if (allowspill) GetTempName(temp,"lookahd",true); assertex(bufsize); if (allowspill) smartbuf.setown(createSmartBuffer(&activity, temp.toCharArray(), bufsize, queryRowInterfaces(in))); else smartbuf.setown(createSmartInMemoryBuffer(&activity, queryRowInterfaces(in), bufsize)); if (notify) notify->onInputStarted(NULL); startsem.signal(); Linked<IRowWriter> writer = smartbuf->queryWriter(); if (preserveLhsGrouping) { while (required&&running) { OwnedConstThorRow row = in->nextRow(); if (!row) { row.setown(in->nextRow()); if (!row) break; else writer->putRow(NULL); // eog } ++count; writer->putRow(row.getClear()); if (required!=RCUNBOUND) required--; } } else { while (required&&running) { OwnedConstThorRow row = in->ungroupedNextRow(); if (!row) break; ++count; writer->putRow(row.getClear()); if (required!=RCUNBOUND) required--; } } } catch(IException * e) { ActPrintLog(&activity, e, "ThorLookaheadCache get exception"); getexception.setown(e); } if (notify) notify->onInputFinished(count); if (smartbuf) smartbuf->queryWriter()->flush(); running = false; try { if (in) in->stop(); } catch(IException * e) { ActPrintLog(&activity, e, "ThorLookaheadCache stop exception"); if (!getexception.get()) getexception.setown(e); } return 0; }
CATCH_NEXTROW() { ActivityTimer t(totalCycles, timeActivities); OwnedConstThorRow ret; Owned<IException> exception; if (first) // only return 1! { try { first = false; initN(); if (RCMAX==N) // indicates before start of dataset e.g. ds[0] { RtlDynamicRowBuilder row(queryRowAllocator()); size32_t sz = helper->createDefault(row); ret.setown(row.finalizeRowClear(sz)); N = 0; // return that processed all } else if (N) { while (!abortSoon) { ret.setown(input->ungroupedNextRow()); if (!ret) break; N--; { SpinBlock block(spin); if (lookaheadN<startN) // will not reach N==0, so don't bother continuing to read { N = startN-lookaheadN; ret.clear(); break; } } if (0==N) break; } if ((N!=0)&&createDefaultIfFail) { N = 0; // return that processed all (i.e. none left) RtlDynamicRowBuilder row(queryRowAllocator()); size32_t sz = helper->createDefault(row); ret.setown(row.finalizeRowClear(sz)); } } if (startN && 0 == N) seenNth = true; } catch (IException *e) { N=0; exception.setown(e); } sendN(); if (exception.get()) throw exception.getClear(); } if (ret) dataLinkIncrement(); return ret.getClear(); }
virtual void process() override { ActPrintLog("INDEXWRITE: Start"); init(); IRowStream *stream = inputStream; ThorDataLinkMetaInfo info; input->getMetaInfo(info); outRowAllocator.setown(getRowAllocator(helper->queryDiskRecordSize())); start(); if (refactor) { assertex(isLocal); if (active) { unsigned targetWidth = partDesc->queryOwner().numParts()-(buildTlk?1:0); assertex(0 == container.queryJob().querySlaves() % targetWidth); unsigned partsPerNode = container.queryJob().querySlaves() / targetWidth; unsigned myPart = queryJobChannel().queryMyRank(); IArrayOf<IRowStream> streams; streams.append(*LINK(stream)); --partsPerNode; // Should this be merging 1,11,21,31 etc. unsigned p=0; unsigned fromPart = targetWidth+1 + (partsPerNode * (myPart-1)); for (; p<partsPerNode; p++) { streams.append(*createRowStreamFromNode(*this, fromPart++, queryJobChannel().queryJobComm(), mpTag, abortSoon)); } ICompare *icompare = helper->queryCompare(); assertex(icompare); Owned<IRowLinkCounter> linkCounter = new CThorRowLinkCounter; myInputStream.setown(createRowStreamMerger(streams.ordinality(), streams.getArray(), icompare, false, linkCounter)); stream = myInputStream; } else // serve nodes, creating merged parts rowServer.setown(createRowServer(this, stream, queryJobChannel().queryJobComm(), mpTag)); } processed = THORDATALINK_STARTED; // single part key support // has to serially pull all data fron nodes 2-N // nodes 2-N, could/should start pushing some data (as it's supposed to be small) to cut down on serial nature. unsigned node = queryJobChannel().queryMyRank(); if (singlePartKey) { if (1 == node) { try { open(*partDesc, false, helper->queryDiskRecordSize()->isVariableSize()); for (;;) { OwnedConstThorRow row = inputStream->ungroupedNextRow(); if (!row) break; if (abortSoon) return; processRow(row); } unsigned node = 2; while (node <= container.queryJob().querySlaves()) { Linked<IOutputRowDeserializer> deserializer = ::queryRowDeserializer(input); CMessageBuffer mb; Owned<ISerialStream> stream = createMemoryBufferSerialStream(mb); CThorStreamDeserializerSource rowSource; rowSource.setStream(stream); bool successSR; for (;;) { { BooleanOnOff tf(receivingTag2); successSR = queryJobChannel().queryJobComm().sendRecv(mb, node, mpTag2); } if (successSR) { if (rowSource.eos()) break; Linked<IEngineRowAllocator> allocator = ::queryRowAllocator(input); do { RtlDynamicRowBuilder rowBuilder(allocator); size32_t sz = deserializer->deserialize(rowBuilder, rowSource); OwnedConstThorRow fRow = rowBuilder.finalizeRowClear(sz); processRow(fRow); } while (!rowSource.eos()); } } node++; } } catch (CATCHALL) { close(*partDesc, partCrc, true); throw; } close(*partDesc, partCrc, true); stop(); } else { CMessageBuffer mb; CMemoryRowSerializer mbs(mb); Linked<IOutputRowSerializer> serializer = ::queryRowSerializer(input); for (;;) { BooleanOnOff tf(receivingTag2); if (queryJobChannel().queryJobComm().recv(mb, 1, mpTag2)) // node 1 asking for more.. { if (abortSoon) break; mb.clear(); do { OwnedConstThorRow row = inputStream->ungroupedNextRow(); if (!row) break; serializer->serialize(mbs, (const byte *)row.get()); } while (mb.length() < SINGLEPART_KEY_TRANSFER_SIZE); // NB: at least one row if (!queryJobChannel().queryJobComm().reply(mb)) throw MakeThorException(0, "Failed to send index data to node 1, from node %d", node); if (0 == mb.length()) break; } } } } else { if (!refactor || active) { try { StringBuffer partFname; getPartFilename(*partDesc, 0, partFname); ActPrintLog("INDEXWRITE: process: handling fname : %s", partFname.str()); open(*partDesc, false, helper->queryDiskRecordSize()->isVariableSize()); ActPrintLog("INDEXWRITE: write"); BooleanOnOff tf(receiving); if (!refactor || !active) receiving = false; do { OwnedConstThorRow row = inputStream->ungroupedNextRow(); if (!row) break; processRow(row); } while (!abortSoon); ActPrintLog("INDEXWRITE: write level 0 complete"); } catch (CATCHALL) { close(*partDesc, partCrc, isLocal && !buildTlk && 1 == node); throw; } close(*partDesc, partCrc, isLocal && !buildTlk && 1 == node); stop(); ActPrintLog("INDEXWRITE: Wrote %" RCPF "d records", processed & THORDATALINK_COUNT_MASK); if (buildTlk) { ActPrintLog("INDEXWRITE: sending rows"); NodeInfoArray tlkRows; CMessageBuffer msg; if (firstNode()) { if (processed & THORDATALINK_COUNT_MASK) { if (enableTlkPart0) tlkRows.append(* new CNodeInfo(0, firstRow.get(), firstRowSize, totalCount)); tlkRows.append(* new CNodeInfo(1, lastRow.get(), lastRowSize, totalCount)); } } else { if (processed & THORDATALINK_COUNT_MASK) { CNodeInfo row(queryJobChannel().queryMyRank(), lastRow.get(), lastRowSize, totalCount); row.serialize(msg); } queryJobChannel().queryJobComm().send(msg, 1, mpTag); } if (firstNode()) { ActPrintLog("INDEXWRITE: Waiting on tlk to complete"); // JCSMORE if refactor==true, is rowsToReceive here right?? unsigned rowsToReceive = (refactor ? (tlkDesc->queryOwner().numParts()-1) : container.queryJob().querySlaves()) -1; // -1 'cos got my own in array already ActPrintLog("INDEXWRITE: will wait for info from %d slaves before writing TLK", rowsToReceive); while (rowsToReceive--) { msg.clear(); receiveMsg(msg, RANK_ALL, mpTag); // NH->JCS RANK_ALL_OTHER not supported for recv if (abortSoon) return; if (msg.length()) { CNodeInfo *ni = new CNodeInfo(); ni->deserialize(msg); tlkRows.append(*ni); } } tlkRows.sort(CNodeInfo::compare); StringBuffer path; getPartFilename(*tlkDesc, 0, path); ActPrintLog("INDEXWRITE: creating toplevel key file : %s", path.str()); try { open(*tlkDesc, true, helper->queryDiskRecordSize()->isVariableSize()); if (tlkRows.length()) { CNodeInfo &lastNode = tlkRows.item(tlkRows.length()-1); memset(lastNode.value, 0xff, lastNode.size); } ForEachItemIn(idx, tlkRows) { CNodeInfo &info = tlkRows.item(idx); builder->processKeyData((char *)info.value, info.pos, info.size); } close(*tlkDesc, tlkCrc, true); } catch (CATCHALL) { abortSoon = true; close(*tlkDesc, tlkCrc, true); removeFiles(*partDesc); throw; } } } else if (!isLocal && firstNode())