CATCH_NEXTROW() { ActivityTimer t(totalCycles, timeActivities); if (!abortSoon) { if (firstget) { firstget = false; while (skipped<skipCount) { OwnedConstThorRow row = inputStream->ungroupedNextRow(); if (!row) { doStopInput(); return NULL; } skipped++; } } if (getDataLinkCount() < limit) { OwnedConstThorRow row = inputStream->ungroupedNextRow(); if (row) { dataLinkIncrement(); return row.getClear(); } } doStopInput(); // NB: really whatever is pulling, should stop asap. } return NULL; }
CATCH_NEXTROW() { ActivityTimer t(totalCycles, timeActivities); if (!abortSoon) { for (;;) { if (0 == countThisGroup && skipCount) { unsigned skipped = 0; do { OwnedConstThorRow row = inputStream->nextRow(); if (row) skipped++; else { if (0 == skipped) { doStopInput(); return NULL; } skipped = 0; // reset, skip group } } while (skipped<skipCount); } if (countThisGroup < limit) { OwnedConstThorRow row = inputStream->nextRow(); if (row) { countThisGroup++; dataLinkIncrement(); return row.getClear(); } else if (0 == countThisGroup && 0==skipCount) { doStopInput(); return NULL; } } else { // consume rest of group for (;;) { OwnedConstThorRow row = inputStream->nextRow(); if (!row) break; } } if (countThisGroup) break; // return eog and reset } } countThisGroup = 0; return NULL; }
void stop() { if (!inputStopped) { inputStopped = true; doStopInput(); } dataLinkStop(); }
CATCH_NEXTROW() { ActivityTimer t(totalCycles, timeActivities); if (abortSoon || eof) return NULL; eof = true; OwnedConstThorRow next = input->ungroupedNextRow(); RtlDynamicRowBuilder resultcr(queryRowAllocator()); size32_t sz = helper->clearAggregate(resultcr); if (next) { hadElement = true; sz = helper->processFirst(resultcr, next); if (container.getKind() != TAKexistsaggregate) { while (!abortSoon) { next.setown(input->ungroupedNextRow()); if (!next) break; sz = helper->processNext(resultcr, next); } } } doStopInput(); if (!firstNode()) { OwnedConstThorRow result(resultcr.finalizeRowClear(sz)); sendResult(result.get(),queryRowSerializer(), 1); // send partial result return NULL; } OwnedConstThorRow ret = getResult(resultcr.finalizeRowClear(sz)); if (ret) { dataLinkIncrement(); return ret.getClear(); } sz = helper->clearAggregate(resultcr); return resultcr.finalizeRowClear(sz); }
virtual void process() override { ActPrintLog("INDEXWRITE: Start"); init(); IRowStream *stream = inputStream; ThorDataLinkMetaInfo info; input->getMetaInfo(info); outRowAllocator.setown(getRowAllocator(helper->queryDiskRecordSize())); start(); if (refactor) { assertex(isLocal); if (active) { unsigned targetWidth = partDesc->queryOwner().numParts()-(buildTlk?1:0); assertex(0 == container.queryJob().querySlaves() % targetWidth); unsigned partsPerNode = container.queryJob().querySlaves() / targetWidth; unsigned myPart = queryJobChannel().queryMyRank(); IArrayOf<IRowStream> streams; streams.append(*LINK(stream)); --partsPerNode; // Should this be merging 1,11,21,31 etc. unsigned p=0; unsigned fromPart = targetWidth+1 + (partsPerNode * (myPart-1)); for (; p<partsPerNode; p++) { streams.append(*createRowStreamFromNode(*this, fromPart++, queryJobChannel().queryJobComm(), mpTag, abortSoon)); } ICompare *icompare = helper->queryCompare(); assertex(icompare); Owned<IRowLinkCounter> linkCounter = new CThorRowLinkCounter; myInputStream.setown(createRowStreamMerger(streams.ordinality(), streams.getArray(), icompare, false, linkCounter)); stream = myInputStream; } else // serve nodes, creating merged parts rowServer.setown(createRowServer(this, stream, queryJobChannel().queryJobComm(), mpTag)); } processed = THORDATALINK_STARTED; // single part key support // has to serially pull all data fron nodes 2-N // nodes 2-N, could/should start pushing some data (as it's supposed to be small) to cut down on serial nature. unsigned node = queryJobChannel().queryMyRank(); if (singlePartKey) { if (1 == node) { try { open(*partDesc, false, helper->queryDiskRecordSize()->isVariableSize()); loop { OwnedConstThorRow row = inputStream->ungroupedNextRow(); if (!row) break; if (abortSoon) return; processRow(row); } unsigned node = 2; while (node <= container.queryJob().querySlaves()) { Linked<IOutputRowDeserializer> deserializer = ::queryRowDeserializer(input); CMessageBuffer mb; Owned<ISerialStream> stream = createMemoryBufferSerialStream(mb); CThorStreamDeserializerSource rowSource; rowSource.setStream(stream); bool successSR; loop { { BooleanOnOff tf(receivingTag2); successSR = queryJobChannel().queryJobComm().sendRecv(mb, node, mpTag2); } if (successSR) { if (rowSource.eos()) break; Linked<IEngineRowAllocator> allocator = ::queryRowAllocator(input); do { RtlDynamicRowBuilder rowBuilder(allocator); size32_t sz = deserializer->deserialize(rowBuilder, rowSource); OwnedConstThorRow fRow = rowBuilder.finalizeRowClear(sz); processRow(fRow); } while (!rowSource.eos()); } } node++; } } catch (CATCHALL) { close(*partDesc, partCrc, true); throw; } close(*partDesc, partCrc, true); doStopInput(); } else { CMessageBuffer mb; CMemoryRowSerializer mbs(mb); Linked<IOutputRowSerializer> serializer = ::queryRowSerializer(input); loop { BooleanOnOff tf(receivingTag2); if (queryJobChannel().queryJobComm().recv(mb, 1, mpTag2)) // node 1 asking for more.. { if (abortSoon) break; mb.clear(); do { OwnedConstThorRow row = inputStream->ungroupedNextRow(); if (!row) break; serializer->serialize(mbs, (const byte *)row.get()); } while (mb.length() < SINGLEPART_KEY_TRANSFER_SIZE); // NB: at least one row if (!queryJobChannel().queryJobComm().reply(mb)) throw MakeThorException(0, "Failed to send index data to node 1, from node %d", node); if (0 == mb.length()) break; } } } }
virtual void stop() { doStopInput(); dataLinkStop(); }
virtual void stop() override { doStopInput(); PARENT::stop(); }
virtual void stop() override { doStopInput(); }