virtual void write()
{
StringBuffer rowTag;
OwnedRoxieString xmlpath(helper->getXmlIteratorPath());
if (!xmlpath)
{
rowTag.append(DEFAULTXMLROWTAG);
}
else
{
const char *path = xmlpath;
if (*path == '/') path++;
if (strchr(path, '/')) UNIMPLEMENTED;
rowTag.append(path);
}
StringBuffer out;
if (!dlfn.isExternal() || firstNode()) // if external, 1 header,footer
{
OwnedRoxieString suppliedHeader(helper->getHeader());
if (kind==TAKjsonwrite)
buildJsonHeader(out, suppliedHeader, rowTag);
else if (suppliedHeader)
out.set(suppliedHeader);
else
out.set(DEFAULTXMLHEADER).newline();
outraw->write(out.length(), out.str());
if (calcFileCrc)
fileCRC.tally(out.length(), out.str());
}
Owned<IXmlWriterExt> writer = createIXmlWriterExt(helper->getXmlFlags(), 0, NULL, (kind==TAKjsonwrite) ? WTJSON : WTStandard);
writer->outputBeginArray(rowTag); //need this to format rows, even if not outputting it below
while(!abortSoon)
{
OwnedConstThorRow row = input->ungroupedNextRow();
if (!row)
break;
writer->clear().outputBeginNested(rowTag, false);
helper->toXML((const byte *)row.get(), *writer);
writer->outputEndNested(rowTag);
outraw->write(writer->length(), writer->str());
if (calcFileCrc)
fileCRC.tally(writer->length(), writer->str());
processed++;
}
if (!dlfn.isExternal() || lastNode()) // if external, 1 header,footer
{
OwnedRoxieString suppliedFooter(helper->getFooter());
if (kind==TAKjsonwrite)
buildJsonFooter(out.clear().newline(), suppliedFooter, rowTag);
else if (suppliedFooter)
out.set(suppliedFooter);
else
out.set(DEFAULTXMLFOOTER).newline();
outraw->write(out.length(), out.str());
if (calcFileCrc)
fileCRC.tally(out.length(), out.str());
}
}
virtual void write()
{
ActPrintLog("%s",grouped?"Grouped":"Ungrouped");
while(!abortSoon)
{
OwnedConstThorRow r = input->nextRow();
if (!r.get()) {
if (grouped) {
if ((processed & THORDATALINK_COUNT_MASK)!=0)
out->putRow(NULL);
}
r.setown(input->nextRow());
if (!r.get())
break;
}
out->putRow(r.getClear());
processed++;
}
}
virtual void write()
{
StringBuffer rowTag;
OwnedRoxieString xmlpath(helper->getXmlIteratorPath());
if (!xmlpath)
{
rowTag.append("Row");
}
else
{
const char *path = xmlpath;
if (*path == '/') path++;
if (strchr(path, '/')) UNIMPLEMENTED;
rowTag.append(path);
}
StringBuffer xmlOutput;
CommonXmlWriter xmlWriter(helper->getXmlFlags());
if (!dlfn.isExternal() || firstNode()) // if external, 1 header,footer
{
OwnedRoxieString header(helper->getHeader());
if (header)
xmlOutput.clear().append(header);
else
xmlOutput.clear().append("<Dataset>").newline();
outraw->write(xmlOutput.length(), xmlOutput.toCharArray());
if (calcFileCrc)
fileCRC.tally(xmlOutput.length(), xmlOutput.toCharArray());
}
while(!abortSoon)
{
OwnedConstThorRow row = input->ungroupedNextRow();
if (!row)
break;
xmlWriter.clear().outputBeginNested(rowTag, false);
helper->toXML((const byte *)row.get(), xmlWriter);
xmlWriter.outputEndNested(rowTag);
outraw->write(xmlWriter.length(), xmlWriter.str());
if (calcFileCrc)
fileCRC.tally(xmlWriter.length(), xmlWriter.str());
processed++;
}
if (!dlfn.isExternal() || lastNode()) // if external, 1 header,footer
{
OwnedRoxieString footer(helper->getFooter());
if (footer)
xmlOutput.clear().append(footer);
else
xmlOutput.clear().append("</Dataset>").newline();
outraw->write(xmlOutput.length(), xmlOutput.toCharArray());
if (calcFileCrc)
fileCRC.tally(xmlOutput.length(), xmlOutput.toCharArray());
}
}
virtual void process()
{
IRecordSize *recordSize = helper->queryOutputMeta();
Owned<IThorRowInterfaces> rowIf = createThorRowInterfaces(queryRowManager(), helper->queryOutputMeta(), queryId(), queryCodeContext());
OwnedConstThorRow result = getAggregate(*this, container.queryJob().querySlaves(), *rowIf, *helper, mpTag);
if (!result)
return;
CMessageBuffer msg;
CMemoryRowSerializer mbs(msg);
rowIf->queryRowSerializer()->serialize(mbs, (const byte *)result.get());
if (!queryJobChannel().queryJobComm().send(msg, 1, mpTag, 5000))
throw MakeThorException(0, "Failed to give result to slave");
}
void process()
{
start();
processed = 0;
processed = THORDATALINK_STARTED;
OwnedConstThorRow row = inputStream->ungroupedNextRow();
CMessageBuffer mb;
DelayedSizeMarker sizeMark(mb);
if (row)
{
CMemoryRowSerializer msz(mb);
::queryRowSerializer(input)->serialize(msz,(const byte *)row.get());
sizeMark.write();
processed++;
}
queryJobChannel().queryJobComm().send(mb, 0, masterMpTag);
}
void process()
{
processed = 0;
input = inputs.item(0);
startInput(input);
processed = THORDATALINK_STARTED;
OwnedConstThorRow row = input->ungroupedNextRow();
CMessageBuffer mb;
size32_t lenpos = mb.length(); // its 0 really
mb.append((size32_t)0);
if (row) {
CMemoryRowSerializer msz(mb);
::queryRowSerializer(input)->serialize(msz,(const byte *)row.get());
size32_t sz = mb.length()-lenpos-sizeof(size32_t);
mb.writeDirect(lenpos,sizeof(size32_t),&sz);
processed++;
}
container.queryJob().queryJobComm().send(mb, 0, masterMpTag);
}
void operator = (const OwnedConstThorRow & other) { set(other.get()); }
inline void set(const OwnedConstThorRow &other) { set(other.get()); }
virtual void process() override
{
ActPrintLog("INDEXWRITE: Start");
init();
IRowStream *stream = inputStream;
ThorDataLinkMetaInfo info;
input->getMetaInfo(info);
outRowAllocator.setown(getRowAllocator(helper->queryDiskRecordSize()));
start();
if (refactor)
{
assertex(isLocal);
if (active)
{
unsigned targetWidth = partDesc->queryOwner().numParts()-(buildTlk?1:0);
assertex(0 == container.queryJob().querySlaves() % targetWidth);
unsigned partsPerNode = container.queryJob().querySlaves() / targetWidth;
unsigned myPart = queryJobChannel().queryMyRank();
IArrayOf<IRowStream> streams;
streams.append(*LINK(stream));
--partsPerNode;
// Should this be merging 1,11,21,31 etc.
unsigned p=0;
unsigned fromPart = targetWidth+1 + (partsPerNode * (myPart-1));
for (; p<partsPerNode; p++)
{
streams.append(*createRowStreamFromNode(*this, fromPart++, queryJobChannel().queryJobComm(), mpTag, abortSoon));
}
ICompare *icompare = helper->queryCompare();
assertex(icompare);
Owned<IRowLinkCounter> linkCounter = new CThorRowLinkCounter;
myInputStream.setown(createRowStreamMerger(streams.ordinality(), streams.getArray(), icompare, false, linkCounter));
stream = myInputStream;
}
else // serve nodes, creating merged parts
rowServer.setown(createRowServer(this, stream, queryJobChannel().queryJobComm(), mpTag));
}
processed = THORDATALINK_STARTED;
// single part key support
// has to serially pull all data fron nodes 2-N
// nodes 2-N, could/should start pushing some data (as it's supposed to be small) to cut down on serial nature.
unsigned node = queryJobChannel().queryMyRank();
if (singlePartKey)
{
if (1 == node)
{
try
{
open(*partDesc, false, helper->queryDiskRecordSize()->isVariableSize());
loop
{
OwnedConstThorRow row = inputStream->ungroupedNextRow();
if (!row)
break;
if (abortSoon) return;
processRow(row);
}
unsigned node = 2;
while (node <= container.queryJob().querySlaves())
{
Linked<IOutputRowDeserializer> deserializer = ::queryRowDeserializer(input);
CMessageBuffer mb;
Owned<ISerialStream> stream = createMemoryBufferSerialStream(mb);
CThorStreamDeserializerSource rowSource;
rowSource.setStream(stream);
bool successSR;
loop
{
{
BooleanOnOff tf(receivingTag2);
successSR = queryJobChannel().queryJobComm().sendRecv(mb, node, mpTag2);
}
if (successSR)
{
if (rowSource.eos())
break;
Linked<IEngineRowAllocator> allocator = ::queryRowAllocator(input);
do
{
RtlDynamicRowBuilder rowBuilder(allocator);
size32_t sz = deserializer->deserialize(rowBuilder, rowSource);
OwnedConstThorRow fRow = rowBuilder.finalizeRowClear(sz);
processRow(fRow);
}
while (!rowSource.eos());
}
}
node++;
}
}
catch (CATCHALL)
{
close(*partDesc, partCrc, true);
throw;
}
close(*partDesc, partCrc, true);
doStopInput();
}
else
{
CMessageBuffer mb;
CMemoryRowSerializer mbs(mb);
Linked<IOutputRowSerializer> serializer = ::queryRowSerializer(input);
loop
{
BooleanOnOff tf(receivingTag2);
if (queryJobChannel().queryJobComm().recv(mb, 1, mpTag2)) // node 1 asking for more..
{
if (abortSoon) break;
mb.clear();
do
{
OwnedConstThorRow row = inputStream->ungroupedNextRow();
if (!row) break;
serializer->serialize(mbs, (const byte *)row.get());
} while (mb.length() < SINGLEPART_KEY_TRANSFER_SIZE); // NB: at least one row
if (!queryJobChannel().queryJobComm().reply(mb))
throw MakeThorException(0, "Failed to send index data to node 1, from node %d", node);
if (0 == mb.length())
break;
}
}
}
}
virtual void process() override
{
ActPrintLog("INDEXWRITE: Start");
init();
IRowStream *stream = inputStream;
ThorDataLinkMetaInfo info;
input->getMetaInfo(info);
outRowAllocator.setown(getRowAllocator(helper->queryDiskRecordSize()));
start();
if (refactor)
{
assertex(isLocal);
if (active)
{
unsigned targetWidth = partDesc->queryOwner().numParts()-(buildTlk?1:0);
assertex(0 == container.queryJob().querySlaves() % targetWidth);
unsigned partsPerNode = container.queryJob().querySlaves() / targetWidth;
unsigned myPart = queryJobChannel().queryMyRank();
IArrayOf<IRowStream> streams;
streams.append(*LINK(stream));
--partsPerNode;
// Should this be merging 1,11,21,31 etc.
unsigned p=0;
unsigned fromPart = targetWidth+1 + (partsPerNode * (myPart-1));
for (; p<partsPerNode; p++)
{
streams.append(*createRowStreamFromNode(*this, fromPart++, queryJobChannel().queryJobComm(), mpTag, abortSoon));
}
ICompare *icompare = helper->queryCompare();
assertex(icompare);
Owned<IRowLinkCounter> linkCounter = new CThorRowLinkCounter;
myInputStream.setown(createRowStreamMerger(streams.ordinality(), streams.getArray(), icompare, false, linkCounter));
stream = myInputStream;
}
else // serve nodes, creating merged parts
rowServer.setown(createRowServer(this, stream, queryJobChannel().queryJobComm(), mpTag));
}
processed = THORDATALINK_STARTED;
// single part key support
// has to serially pull all data fron nodes 2-N
// nodes 2-N, could/should start pushing some data (as it's supposed to be small) to cut down on serial nature.
unsigned node = queryJobChannel().queryMyRank();
if (singlePartKey)
{
if (1 == node)
{
try
{
open(*partDesc, false, helper->queryDiskRecordSize()->isVariableSize());
for (;;)
{
OwnedConstThorRow row = inputStream->ungroupedNextRow();
if (!row)
break;
if (abortSoon) return;
processRow(row);
}
unsigned node = 2;
while (node <= container.queryJob().querySlaves())
{
Linked<IOutputRowDeserializer> deserializer = ::queryRowDeserializer(input);
CMessageBuffer mb;
Owned<ISerialStream> stream = createMemoryBufferSerialStream(mb);
CThorStreamDeserializerSource rowSource;
rowSource.setStream(stream);
bool successSR;
for (;;)
{
{
BooleanOnOff tf(receivingTag2);
successSR = queryJobChannel().queryJobComm().sendRecv(mb, node, mpTag2);
}
if (successSR)
{
if (rowSource.eos())
break;
Linked<IEngineRowAllocator> allocator = ::queryRowAllocator(input);
do
{
RtlDynamicRowBuilder rowBuilder(allocator);
size32_t sz = deserializer->deserialize(rowBuilder, rowSource);
OwnedConstThorRow fRow = rowBuilder.finalizeRowClear(sz);
processRow(fRow);
}
while (!rowSource.eos());
}
}
node++;
}
}
catch (CATCHALL)
{
close(*partDesc, partCrc, true);
throw;
}
close(*partDesc, partCrc, true);
stop();
}
else
{
CMessageBuffer mb;
CMemoryRowSerializer mbs(mb);
Linked<IOutputRowSerializer> serializer = ::queryRowSerializer(input);
for (;;)
{
BooleanOnOff tf(receivingTag2);
if (queryJobChannel().queryJobComm().recv(mb, 1, mpTag2)) // node 1 asking for more..
{
if (abortSoon) break;
mb.clear();
do
{
OwnedConstThorRow row = inputStream->ungroupedNextRow();
if (!row) break;
serializer->serialize(mbs, (const byte *)row.get());
} while (mb.length() < SINGLEPART_KEY_TRANSFER_SIZE); // NB: at least one row
if (!queryJobChannel().queryJobComm().reply(mb))
throw MakeThorException(0, "Failed to send index data to node 1, from node %d", node);
if (0 == mb.length())
break;
}
}
}
}
else
{
if (!refactor || active)
{
try
{
StringBuffer partFname;
getPartFilename(*partDesc, 0, partFname);
ActPrintLog("INDEXWRITE: process: handling fname : %s", partFname.str());
open(*partDesc, false, helper->queryDiskRecordSize()->isVariableSize());
ActPrintLog("INDEXWRITE: write");
BooleanOnOff tf(receiving);
if (!refactor || !active)
receiving = false;
do
{
OwnedConstThorRow row = inputStream->ungroupedNextRow();
if (!row)
break;
processRow(row);
} while (!abortSoon);
ActPrintLog("INDEXWRITE: write level 0 complete");
}
catch (CATCHALL)
{
close(*partDesc, partCrc, isLocal && !buildTlk && 1 == node);
throw;
}
close(*partDesc, partCrc, isLocal && !buildTlk && 1 == node);
stop();
ActPrintLog("INDEXWRITE: Wrote %" RCPF "d records", processed & THORDATALINK_COUNT_MASK);
if (buildTlk)
{
ActPrintLog("INDEXWRITE: sending rows");
NodeInfoArray tlkRows;
CMessageBuffer msg;
if (firstNode())
{
if (processed & THORDATALINK_COUNT_MASK)
{
if (enableTlkPart0)
tlkRows.append(* new CNodeInfo(0, firstRow.get(), firstRowSize, totalCount));
tlkRows.append(* new CNodeInfo(1, lastRow.get(), lastRowSize, totalCount));
}
}
else
{
if (processed & THORDATALINK_COUNT_MASK)
{
CNodeInfo row(queryJobChannel().queryMyRank(), lastRow.get(), lastRowSize, totalCount);
row.serialize(msg);
}
queryJobChannel().queryJobComm().send(msg, 1, mpTag);
}
if (firstNode())
{
ActPrintLog("INDEXWRITE: Waiting on tlk to complete");
// JCSMORE if refactor==true, is rowsToReceive here right??
unsigned rowsToReceive = (refactor ? (tlkDesc->queryOwner().numParts()-1) : container.queryJob().querySlaves()) -1; // -1 'cos got my own in array already
ActPrintLog("INDEXWRITE: will wait for info from %d slaves before writing TLK", rowsToReceive);
while (rowsToReceive--)
{
msg.clear();
receiveMsg(msg, RANK_ALL, mpTag); // NH->JCS RANK_ALL_OTHER not supported for recv
if (abortSoon)
return;
if (msg.length())
{
CNodeInfo *ni = new CNodeInfo();
ni->deserialize(msg);
tlkRows.append(*ni);
}
}
tlkRows.sort(CNodeInfo::compare);
StringBuffer path;
getPartFilename(*tlkDesc, 0, path);
ActPrintLog("INDEXWRITE: creating toplevel key file : %s", path.str());
try
{
open(*tlkDesc, true, helper->queryDiskRecordSize()->isVariableSize());
if (tlkRows.length())
{
CNodeInfo &lastNode = tlkRows.item(tlkRows.length()-1);
memset(lastNode.value, 0xff, lastNode.size);
}
ForEachItemIn(idx, tlkRows)
{
CNodeInfo &info = tlkRows.item(idx);
builder->processKeyData((char *)info.value, info.pos, info.size);
}
close(*tlkDesc, tlkCrc, true);
}
catch (CATCHALL)
{
abortSoon = true;
close(*tlkDesc, tlkCrc, true);
removeFiles(*partDesc);
throw;
}
}
}
else if (!isLocal && firstNode())
