node* node::lastNode(const node *endNode /* = NULL */) const
{
node *lastNode((node*)this);
while (lastNode && lastNode->nextNode() != endNode)
lastNode = lastNode->nextNode();
return lastNode;
}
CATCH_NEXTROW()
{
ActivityTimer t(totalCycles, timeActivities);
if (first)
{
first = false;
if (container.queryLocalOrGrouped() || firstNode())
memset(tallies, 0, sizeof(unsigned)*numSets);
else
getTallies();
done = helper->setCounts(tallies);
}
if (!done)
{
while (!abortSoon)
{
OwnedConstThorRow row = inputStream->ungroupedNextRow();
if (!row)
break;
switch (helper->getRecordAction(row))
{
case 2:
done = true;
//fall through
case 1:
dataLinkIncrement();
return row.getClear();
}
}
}
if (!container.queryLocalOrGrouped() && !lastNode())
sendTallies();
return NULL;
}
/*protected*/
std::auto_ptr<BoundableList>
AbstractSTRtree::createParentBoundables(BoundableList* childBoundables,
int newLevel)
{
assert(!childBoundables->empty());
std::auto_ptr< BoundableList > parentBoundables ( new BoundableList() );
parentBoundables->push_back(createNode(newLevel));
std::auto_ptr< BoundableList > sortedChildBoundables ( sortBoundables(childBoundables) );
for (BoundableList::iterator i=sortedChildBoundables->begin(),
e=sortedChildBoundables->end();
i!=e; i++)
//for(std::size_t i=0, scbsize=sortedChildBoundables->size(); i<scbsize; ++i)
{
Boundable *childBoundable=*i; // (*sortedChildBoundables)[i];
AbstractNode *last = lastNode(parentBoundables.get());
if (last->getChildBoundables()->size() == nodeCapacity)
{
last=createNode(newLevel);
parentBoundables->push_back(last);
}
last->addChildBoundable(childBoundable);
}
return parentBoundables;
}
/*protected*/
std::auto_ptr<BoundableList>
SIRtree::createParentBoundables(BoundableList *childBoundables,int newLevel)
{
assert(!childBoundables->empty());
std::auto_ptr<BoundableList> parentBoundables ( new BoundableList() );
parentBoundables->push_back(createNode(newLevel));
std::auto_ptr<BoundableList> sortedChildBoundables ( sortBoundables(childBoundables) );
//for(unsigned int i=0;i<sortedChildBoundables->size();i++)
for (BoundableList::iterator i=sortedChildBoundables->begin(),
e=sortedChildBoundables->end();
i!=e; ++i)
{
//Boundable *childBoundable=(AbstractNode*)(*sortedChildBoundables)[i];
Boundable *childBoundable=*i;
AbstractNode* lNode = lastNode(parentBoundables.get());
if (lNode->getChildBoundables()->size() == nodeCapacity)
{
parentBoundables->push_back(createNode(newLevel));
}
lNode->addChildBoundable(childBoundable);
}
return parentBoundables;
}
virtual void write()
{
StringBuffer rowTag;
OwnedRoxieString xmlpath(helper->getXmlIteratorPath());
if (!xmlpath)
{
rowTag.append(DEFAULTXMLROWTAG);
}
else
{
const char *path = xmlpath;
if (*path == '/') path++;
if (strchr(path, '/')) UNIMPLEMENTED;
rowTag.append(path);
}
StringBuffer out;
if (!dlfn.isExternal() || firstNode()) // if external, 1 header,footer
{
OwnedRoxieString suppliedHeader(helper->getHeader());
if (kind==TAKjsonwrite)
buildJsonHeader(out, suppliedHeader, rowTag);
else if (suppliedHeader)
out.set(suppliedHeader);
else
out.set(DEFAULTXMLHEADER).newline();
outraw->write(out.length(), out.str());
if (calcFileCrc)
fileCRC.tally(out.length(), out.str());
}
Owned<IXmlWriterExt> writer = createIXmlWriterExt(helper->getXmlFlags(), 0, NULL, (kind==TAKjsonwrite) ? WTJSON : WTStandard);
writer->outputBeginArray(rowTag); //need this to format rows, even if not outputting it below
while(!abortSoon)
{
OwnedConstThorRow row = input->ungroupedNextRow();
if (!row)
break;
writer->clear().outputBeginNested(rowTag, false);
helper->toXML((const byte *)row.get(), *writer);
writer->outputEndNested(rowTag);
outraw->write(writer->length(), writer->str());
if (calcFileCrc)
fileCRC.tally(writer->length(), writer->str());
processed++;
}
if (!dlfn.isExternal() || lastNode()) // if external, 1 header,footer
{
OwnedRoxieString suppliedFooter(helper->getFooter());
if (kind==TAKjsonwrite)
buildJsonFooter(out.clear().newline(), suppliedFooter, rowTag);
else if (suppliedFooter)
out.set(suppliedFooter);
else
out.set(DEFAULTXMLFOOTER).newline();
outraw->write(out.length(), out.str());
if (calcFileCrc)
fileCRC.tally(out.length(), out.str());
}
}
DataNode* DataNode::add(char ch)
{
DataNode *root((DataNode *)lastNode());
DataNode *newData = new DataNode(ch);
if (root)
root->next = newData;
return newData;
}
// The main function to sort a linked list. It mainly calls _quickSort()
void quickSort(struct Node *head)
{
// Find last node
struct Node *h = lastNode(head);
// Call the recursive QuickSort
_quickSort(head, h);
}
bool SuffixMachine::onMatch()
{
#if 0
for (node_info * part=lastNode();part!=NULL;part=previousNode(part))
out<<"\n"<<info.text->mid(part->start,part->finish-part->start+1)<<"\n";
#endif
//out<<"S:"<<info->word.mid(startingPos)<<"\n";
return controller->on_match_helper();
}
virtual void write()
{
StringBuffer rowTag;
OwnedRoxieString xmlpath(helper->getXmlIteratorPath());
if (!xmlpath)
{
rowTag.append("Row");
}
else
{
const char *path = xmlpath;
if (*path == '/') path++;
if (strchr(path, '/')) UNIMPLEMENTED;
rowTag.append(path);
}
StringBuffer xmlOutput;
CommonXmlWriter xmlWriter(helper->getXmlFlags());
if (!dlfn.isExternal() || firstNode()) // if external, 1 header,footer
{
OwnedRoxieString header(helper->getHeader());
if (header)
xmlOutput.clear().append(header);
else
xmlOutput.clear().append("<Dataset>").newline();
outraw->write(xmlOutput.length(), xmlOutput.toCharArray());
if (calcFileCrc)
fileCRC.tally(xmlOutput.length(), xmlOutput.toCharArray());
}
while(!abortSoon)
{
OwnedConstThorRow row = input->ungroupedNextRow();
if (!row)
break;
xmlWriter.clear().outputBeginNested(rowTag, false);
helper->toXML((const byte *)row.get(), xmlWriter);
xmlWriter.outputEndNested(rowTag);
outraw->write(xmlWriter.length(), xmlWriter.str());
if (calcFileCrc)
fileCRC.tally(xmlWriter.length(), xmlWriter.str());
processed++;
}
if (!dlfn.isExternal() || lastNode()) // if external, 1 header,footer
{
OwnedRoxieString footer(helper->getFooter());
if (footer)
xmlOutput.clear().append(footer);
else
xmlOutput.clear().append("</Dataset>").newline();
outraw->write(xmlOutput.length(), xmlOutput.toCharArray());
if (calcFileCrc)
fileCRC.tally(xmlOutput.length(), xmlOutput.toCharArray());
}
}
virtual void write()
{
if (!singleHF || firstNode())
{
const char * header = helper->queryCsvParameters()->queryHeader();
if (header)
{
csvOutput.beginLine();
csvOutput.writeHeaderLn(strlen(header),header);
const char * outText = csvOutput.str();
unsigned outLength = csvOutput.length();
outraw->write(outLength,outText);
if (calcFileCrc)
fileCRC.tally(outLength, outText);
}
}
while(!abortSoon)
{
OwnedConstThorRow r(input->ungroupedNextRow());
if (!r)
break;
csvOutput.beginLine();
helper->writeRow((const byte *)r.get(), &csvOutput);
csvOutput.endLine();
const char * outText = csvOutput.str();
unsigned outLength = csvOutput.length();
outraw->write(outLength,outText);
if (calcFileCrc)
fileCRC.tally(outLength, outText);
processed++;
}
if (!singleHF || lastNode())
{
const char * footer = helper->queryCsvParameters()->queryFooter();
if (footer)
{
csvOutput.beginLine();
csvOutput.writeHeaderLn(strlen(footer),footer);
const char * outText = csvOutput.str();
unsigned outLength = csvOutput.length();
outraw->write(outLength,outText);
if (calcFileCrc)
fileCRC.tally(outLength, outText);
}
}
}
bool PrefixMachine::onMatch()
{
#if 0
for (node_info * part=lastNode();part!=NULL;part=previousNode(part))
out<<"\n"<<info.text->mid(part->start,part->finish-part->start+1)<<"\n";
#endif
//out<<"p:"<<info->word.mid(0,position)<<"\n";
if (controller->Stem!=NULL)
delete controller->Stem;//TODO: change this allocation, deallocation to clear
controller->Stem=new StemMachine(controller,position,controller->Prefix->resulting_category_idOFCurrentMatch);
controller->Stem->setSolutionSettings(controller->multi_p);
return (*controller->Stem)();
}
virtual void start() override
{
ActivityTimer s(totalCycles, timeActivities);
ActPrintLog(rolloverEnabled ? "GROUP: is global" : "GROUP: is local");
PARENT::start();
eogNext = prevEog = eof = false;
if (rolloverEnabled)
{
useRollover = !lastNode();
#ifdef _TESTING
ActPrintLog("Node number = %d, Total Nodes = %d", queryJobChannel().queryMyRank(), container.queryJob().querySlaves());
#endif
}
stream.set(inputStream);
startLastGroup = getDataLinkGlobalCount();
next.setown(getNext());
if (rolloverEnabled && !firstNode()) // 1st node can have nothing to send
{
Owned<IThorRowCollector> collector = createThorRowCollector(*this, this, NULL, stableSort_none, rc_mixed, SPILL_PRIORITY_SPILLABLE_STREAM);
Owned<IRowWriter> writer = collector->getWriter();
if (next)
{
ActPrintLog("GROUP: Sending first group to previous node(%d)", queryJobChannel().queryMyRank()-1);
for (;;)
{
writer->putRow(next.getLink());
if (abortSoon)
break; //always send group even when aborting
OwnedConstThorRow next2 = getNext();
if (!next2)
{
eof = true;
break;
}
else if (!helper->isSameGroup(next2, next))
{
next.setown(next2.getClear());
break;
}
next.setown(next2.getClear());
}
}
writer.clear();
ActPrintLog("GROUP: %" RCPF "d records to send", collector->numRows());
Owned<IRowStream> strm = collector->getStream();
rowServer.setown(createRowServer(this, strm, queryJobChannel().queryJobComm(), mpTag));
}
}
void CDiskWriteSlaveActivityBase::close()
{
try
{
if (out) {
uncompressedBytesWritten = out->getPosition();
if (calcFileCrc) {
if (diskHelperBase->getFlags() & TDWextend) {
assertex(!"TBD need to merge CRC");
}
else
out->flush(&fileCRC);
}
else if (!abortSoon)
out->flush();
out.clear();
}
else if (outraw) {
outraw->flush();
uncompressedBytesWritten = outraw->tell();
outraw.clear();
}
{
CriticalBlock block(statsCs);
mergeStats(fileStats, outputIO);
outputIO.clear();
}
if (!rfsQueryParallel && dlfn.isExternal() && !lastNode())
{
rowcount_t rows = processed & THORDATALINK_COUNT_MASK;
ActPrintLog("External write done, signalling next (row count = %" RCPF "d)", rows);
CMessageBuffer msg;
msg.append(rows);
msg.append(tempExternalName);
queryJobChannel().queryJobComm().send(msg, queryJobChannel().queryMyRank()+1, mpTag);
}
}
catch (IException *e)
{
ActPrintLogEx(&queryContainer(), e, thorlog_null, MCwarning, "Error closing file: %s", fName.get());
abortSoon = true;
removeFiles();
throw e;
}
if (abortSoon)
removeFiles();
}
void putNext(const void *prev)
{
if (nextPut) return;
nextPut = true;
if (global && !lastNode())
{
CMessageBuffer msg;
msg.append(count);
if (prev)
{
CMemoryRowSerializer msz(msg);
::queryRowSerializer(input)->serialize(msz, (const byte *)prev);
}
if (!queryJobChannel().queryJobComm().send(msg, queryJobChannel().queryMyRank()+1, mpTag)) // to next
return;
}
}
void strNode::flatten(){
strNode *nodePos = this;
while(nodePos){
if(down){
strNode *oldRight = right;
strNode *lastDown = lastNode(down);
right = down;
down->left = this;
down = NULL;
if(oldRight){
lastDown->right = oldRight;
oldRight->left = lastDown;
}
nodePos = oldRight;
}
else
nodePos = nodePos->right;
}
}
void CDiskWriteSlaveActivityBase::open()
{
if (dlfn.isExternal() && !firstNode())
{
input.setown(createDataLinkSmartBuffer(this, inputs.item(0), PROCESS_SMART_BUFFER_SIZE, isSmartBufferSpillNeeded(this), grouped, RCUNBOUND, NULL, false, &container.queryJob().queryIDiskUsage()));
startInput(input);
if (!rfsQueryParallel)
{
ActPrintLog("Blocked, waiting for previous part to complete write");
CMessageBuffer msg;
if (!receiveMsg(msg, container.queryJob().queryMyRank()-1, mpTag))
return;
rowcount_t prevRows;
msg.read(prevRows);
msg.read(tempExternalName); // reuse temp filename, last node will rename
ActPrintLog("Previous write row count = %"RCPF"d", prevRows);
}
}
else
{
input.set(inputs.item(0));
startInput(input);
}
processed = THORDATALINK_STARTED;
bool extend = 0 != (diskHelperBase->getFlags() & TDWextend);
if (extend)
ActPrintLog("Extending file %s", fName.get());
size32_t exclsz = 0;
calcFileCrc = true;
bool external = dlfn.isExternal();
bool query = dlfn.isQuery();
if (query && compress)
UNIMPLEMENTED;
bool direct = query || (external && !firstNode());
bool rename = !external || (!query && lastNode());
Owned<IFileIO> iFileIO = createMultipleWrite(this, *partDesc, exclsz, compress, extend||(external&&!query), ecomp, this, direct, rename, &abortSoon, (external&&!query) ? &tempExternalName : NULL);
if (compress)
{
ActPrintLog("Performing row compression on output file: %s", fName.get());
calcFileCrc = false;
}
Owned<IFileIOStream> stream;
if (wantRaw())
{
outraw.setown(createBufferedIOStream(iFileIO));
stream.set(outraw);
}
else
{
stream.setown(createIOStream(iFileIO));
out.setown(createRowWriter(stream,::queryRowSerializer(input),::queryRowAllocator(input),grouped,calcFileCrc,false)); // flushed by close
}
CDfsLogicalFileName dlfn;
dlfn.set(logicalFilename);
if (extend || (external && !query))
stream->seek(0,IFSend);
ActPrintLog("Created output stream for %s", fName.get());
}
void CDiskWriteSlaveActivityBase::open()
{
if (dlfn.isExternal() && !firstNode())
{
input.setown(createDataLinkSmartBuffer(this, inputs.item(0), PROCESS_SMART_BUFFER_SIZE, isSmartBufferSpillNeeded(this), grouped, RCUNBOUND, NULL, false, &container.queryJob().queryIDiskUsage()));
startInput(input);
if (!rfsQueryParallel)
{
ActPrintLog("Blocked, waiting for previous part to complete write");
CMessageBuffer msg;
if (!receiveMsg(msg, queryJobChannel().queryMyRank()-1, mpTag))
return;
rowcount_t prevRows;
msg.read(prevRows);
msg.read(tempExternalName); // reuse temp filename, last node will rename
ActPrintLog("Previous write row count = %" RCPF "d", prevRows);
}
}
else
{
input.set(inputs.item(0));
startInput(input);
}
processed = THORDATALINK_STARTED;
bool extend = 0 != (diskHelperBase->getFlags() & TDWextend);
if (extend)
ActPrintLog("Extending file %s", fName.get());
/* Fixed length record size is used when outputting compressed stream to determine run-length compression vs default LZW compression.
* NB: only for FLAT files, not CSV or XML
*/
size32_t diskRowMinSz = 0;
IOutputMetaData *diskRowMeta = diskHelperBase->queryDiskRecordSize()->querySerializedDiskMeta();
if (diskRowMeta->isFixedSize() && (TAKdiskwrite == container.getKind()))
{
diskRowMinSz = diskRowMeta->getMinRecordSize();
if (grouped)
diskRowMinSz += 1;
}
calcFileCrc = true;
bool external = dlfn.isExternal();
bool query = dlfn.isQuery();
if (query && compress)
UNIMPLEMENTED;
unsigned twFlags = external ? TW_External : 0;
if (query || (external && !firstNode()))
twFlags |= TW_Direct;
if (!external || (!query && lastNode()))
twFlags |= TW_RenameToPrimary;
if (extend||(external&&!query))
twFlags |= TW_Extend;
Owned<IFileIO> iFileIO = createMultipleWrite(this, *partDesc, diskRowMinSz, twFlags, compress, ecomp, this, &abortSoon, (external&&!query) ? &tempExternalName : NULL);
if (compress)
{
ActPrintLog("Performing row compression on output file: %s", fName.get());
// NB: block compressed output has implicit crc of 0, no need to calculate in row writer.
calcFileCrc = false;
}
Owned<IFileIOStream> stream;
if (wantRaw())
{
outraw.setown(createBufferedIOStream(iFileIO));
stream.set(outraw);
}
else
{
stream.setown(createIOStream(iFileIO));
unsigned rwFlags = 0;
if (grouped)
rwFlags |= rw_grouped;
if (calcFileCrc)
rwFlags |= rw_crc;
out.setown(createRowWriter(stream, ::queryRowInterfaces(input), rwFlags));
}
if (extend || (external && !query))
stream->seek(0,IFSend);
ActPrintLog("Created output stream for %s", fName.get());
}
DeprecatedValueListImplIterator DeprecatedValueListImpl::fromLast() const
{
return DeprecatedValueListImplIterator(lastNode());
}
DeprecatedValueListImplIterator DeprecatedValueListImpl::fromLast()
{
copyOnWrite();
return DeprecatedValueListImplIterator(lastNode());
}
