//Outputs
void addOutputDatasetColumn(const char * datasetName, CColumn * _output)
{
if (0 == m_outputDatasets.ordinality())
m_outputDatasets.append(*(new CTable(datasetName,NULL,NULL)));//add new output dataset
else
{
CTable & dataset = m_outputDatasets.item(m_outputDatasets.ordinality() - 1);
if (0 != strcmp(datasetName, dataset.queryName()))
m_outputDatasets.append(*(new CTable(datasetName,NULL,NULL)));//add new output dataset
}
CTable & dataset = m_outputDatasets.item(m_outputDatasets.ordinality() - 1);
dataset.addColumn(_output);
}
virtual int processCMD()
{
StringBuffer s;
Owned<IClientWsWorkunits> client = createWsWorkunitsClient();
VStringBuffer url("http://%s:%s/WsWorkunits", optServer.sget(), optPort.sget());
client->addServiceUrl(url.str());
if (optUsername.length())
client->setUsernameToken(optUsername.get(), optPassword.sget(), NULL);
Owned<IClientWUQuerySetAliasActionRequest> req = client->createWUQuerysetAliasActionRequest();
IArrayOf<IEspQuerySetAliasActionItem> aliases;
Owned<IEspQuerySetAliasActionItem> item = createQuerySetAliasActionItem();
item->setName(optAlias.get());
aliases.append(*item.getClear());
req->setAliases(aliases);
req->setAction("Deactivate");
req->setQuerySetName(optQuerySet.get());
Owned<IClientWUQuerySetAliasActionResponse> resp = client->WUQuerysetAliasAction(req);
IArrayOf<IConstQuerySetAliasActionResult> &results = resp->getResults();
if (resp->getExceptions().ordinality())
outputMultiExceptions(resp->getExceptions());
else if (results.empty())
fprintf(stderr, "\nError Empty Result!\n");
else
{
IConstQuerySetAliasActionResult &item = results.item(0);
if (item.getSuccess())
fprintf(stdout, "Deactivated alias %s/%s\n", optQuerySet.sget(), optAlias.sget());
else if (item.getCode()|| item.getMessage())
fprintf(stderr, "Error (%d) %s\n", item.getCode(), item.getMessage());
}
return 0;
}
static bool deleteEmptyDir(IFile *dir)
{
// this is a bit odd - basically we already know no files but there may be empty sub-dirs
Owned<IDirectoryIterator> iter = dir->directoryFiles(NULL,false,true);
IArrayOf<IFile> subdirs;
bool candelete = true;
ForEach(*iter) {
if (iter->isDir())
subdirs.append(iter->get());
else
candelete = false;
}
if (!candelete)
return false;
try {
ForEachItemIn(i,subdirs) {
if (!deleteEmptyDir(&subdirs.item(i)))
candelete = false;
}
}
catch (IException *e) {
EXCLOG(e,"deleteEmptyDir");
candelete = false;
}
if (!candelete)
return false;
static CriticalSection sect;
CriticalBlock block(sect); // don't want to actually remove in parallel
dir->remove();
return !dir->exists();
}
int CFileSpraySoapBindingEx::onFinishUpload(IEspContext &ctx, CHttpRequest* request, CHttpResponse* response, const char *service, const char *method, StringArray& fileNames, StringArray& files, IMultiException *me)
{
if (!me || (me->ordinality()==0))
{
if (ctx.getResponseFormat()==ESPSerializationANY)
{
StringBuffer newUrl, netAddress, path;
request->getParameter("NetAddress", netAddress);
request->getParameter("Path", path);
newUrl.appendf("/FileSpray/DropZoneFiles?NetAddress=%s&Path=%s", netAddress.str(), path.str());
response->redirect(*request, newUrl.str());
}
else
{
IArrayOf<IEspDFUActionResult> results;
Owned<CUploadFilesResponse> esp_response = new CUploadFilesResponse("FileSpray");
ForEachItemIn(i, fileNames)
{
const char* fileName = fileNames.item(i);
Owned<IEspDFUActionResult> res = createDFUActionResult("", "");
res->setID(fileName);
res->setAction("Upload File");
res->setResult("Success");
results.append(*res.getLink());
}
if (!results.length())
{
Owned<IEspDFUActionResult> res = createDFUActionResult("", "");
res->setID("<N/A>");
res->setAction("Upload File");
res->setResult("No file uploaded");
results.append(*res.getLink());
}
esp_response->setUploadFileResults(results);
MemoryBuffer content;
StringBuffer mimetype;
esp_response->appendContent(&ctx,content, mimetype);
response->setContent(content.length(), content.toByteArray());
response->setContentType(mimetype.str());
response->send();
}
}
else
{
void CSlavePartMapping::getParts(unsigned i, IArrayOf<IPartDescriptor> &parts)
{
if (local)
i = 0;
if (i>=maps.ordinality()) return;
CSlaveMap &map = maps.item(i);
ForEachItemIn(m, map)
parts.append(*LINK(&map.item(m)));
}
aindex_t getAllCachedTables(IArrayOf<CTable> &_tables)
{
CriticalBlock b(m_crit);
HashIterator iter(m_tableCache);
ForEach(iter)
{
CTable *tbl = m_tableCache.mapToValue(&iter.query());
_tables.append(*(LINK(tbl)));
}
return _tables.ordinality();
}
aindex_t getMatchingTables(const char * tblFilter, IArrayOf<CTable> &_tables, bool bExact)
{
CriticalBlock b(m_crit);
HashIterator iter(m_tableCache);
if (bExact)
{
CTable *tbl = m_tableCache.getValue(tblFilter);
if (tbl)
_tables.append(*(LINK(tbl)));
}
else
{
unsigned filterLen = strlen(tblFilter);
ForEach(iter)
{
CTable *tbl = m_tableCache.mapToValue(&iter.query());
if (0 == strnicmp(tblFilter, tbl->queryName(), filterLen))
_tables.append(*(LINK(tbl)));
}
}
return _tables.ordinality();
}
void AddServers(const char *auditdir)
{
// order significant
servers.append(*createDaliSessionServer());
servers.append(*createDaliPublisherServer());
servers.append(*createDaliSDSServer(serverConfig));
servers.append(*createDaliNamedQueueServer());
servers.append(*createDaliDFSServer(serverConfig));
servers.append(*createDaliAuditServer(auditdir));
servers.append(*createDaliDiagnosticsServer());
// add new coven servers here
}
static void AddServers()
{
// order significant
servers.append(*createSashaArchiverServer());
// servers.append(*createSashaVerifierServer());
servers.append(*createSashaSDSCoalescingServer());
servers.append(*createSashaXrefServer());
servers.append(*createSashaDaFSMonitorServer());
servers.append(*createSashaQMonitorServer());
servers.append(*createSashaFileExpiryServer());
// add new servers here
}
void CWSESPControlEx::cleanSessions(bool allSessions, const char* _id, const char* _userID, const char* _fromIP)
{
StringBuffer searchPath;
setSessionXPath(allSessions, _id, _userID, _fromIP, searchPath);
Owned<IRemoteConnection> globalLock = querySDSConnectionForESPSession(RTM_LOCK_WRITE, SESSION_SDS_LOCK_TIMEOUT);
Owned<IPropertyTreeIterator> iter = globalLock->queryRoot()->getElements("*");
ForEach(*iter)
{
IArrayOf<IPropertyTree> toRemove;
Owned<IPropertyTreeIterator> iter1 = iter->query().getElements(searchPath.str());
ForEach(*iter1)
toRemove.append(*LINK(&iter1->query()));
ForEachItemIn(i, toRemove)
iter->query().removeTree(&toRemove.item(i));
}
}
void getPackageListInfo(IPropertyTree *mapTree, IEspPackageListMapData *pkgList)
{
pkgList->setId(mapTree->queryProp("@id"));
pkgList->setTarget(mapTree->queryProp("@querySet"));
Owned<IPropertyTreeIterator> iter = mapTree->getElements("Package");
IArrayOf<IConstPackageListData> results;
ForEach(*iter)
{
IPropertyTree &item = iter->query();
Owned<IEspPackageListData> res = createPackageListData("", "");
res->setId(item.queryProp("@id"));
if (item.hasProp("@queries"))
res->setQueries(item.queryProp("@queries"));
results.append(*res.getClear());
}
pkgList->setPkgListData(results);
}
bool CWSESPControlEx::onSessionQuery(IEspContext& context, IEspSessionQueryRequest& req, IEspSessionQueryResponse& resp)
{
try
{
#ifdef _USE_OPENLDAP
CLdapSecManager* secmgr = dynamic_cast<CLdapSecManager*>(context.querySecManager());
if(secmgr && !secmgr->isSuperUser(context.queryUser()))
{
context.setAuthStatus(AUTH_STATUS_NOACCESS);
throw MakeStringException(ECLWATCH_SUPER_USER_ACCESS_DENIED, "Failed to query session. Permission denied.");
}
#endif
StringBuffer xpath;
setSessionXPath(false, nullptr, req.getUserID(), req.getFromIP(), xpath);
IArrayOf<IEspSession> sessions;
Owned<IRemoteConnection> globalLock = querySDSConnectionForESPSession(RTM_LOCK_READ, SESSION_SDS_LOCK_TIMEOUT);
Owned<IPropertyTreeIterator> iter = globalLock->queryRoot()->getElements("*");
ForEach(*iter)
{
IPropertyTree& appSessionTree = iter->query();
unsigned port = appSessionTree.getPropInt("@port");
Owned<IPropertyTreeIterator> iter1 = appSessionTree.getElements(xpath.str());
ForEach(*iter1)
{
IPropertyTree& sessionTree = iter1->query();
Owned<IEspSession> s = createSession();
setSessionInfo(&sessionTree, port, s);
sessions.append(*s.getLink());
}
}
resp.setSessions(sessions);
}
catch(IException* e)
{
FORWARDEXCEPTION(context, e, ECLWATCH_INTERNAL_ERROR);
}
return true;
}
virtual void process() override
{
ActPrintLog("INDEXWRITE: Start");
init();
IRowStream *stream = inputStream;
ThorDataLinkMetaInfo info;
input->getMetaInfo(info);
outRowAllocator.setown(getRowAllocator(helper->queryDiskRecordSize()));
start();
if (refactor)
{
assertex(isLocal);
if (active)
{
unsigned targetWidth = partDesc->queryOwner().numParts()-(buildTlk?1:0);
assertex(0 == container.queryJob().querySlaves() % targetWidth);
unsigned partsPerNode = container.queryJob().querySlaves() / targetWidth;
unsigned myPart = queryJobChannel().queryMyRank();
IArrayOf<IRowStream> streams;
streams.append(*LINK(stream));
--partsPerNode;
// Should this be merging 1,11,21,31 etc.
unsigned p=0;
unsigned fromPart = targetWidth+1 + (partsPerNode * (myPart-1));
for (; p<partsPerNode; p++)
{
streams.append(*createRowStreamFromNode(*this, fromPart++, queryJobChannel().queryJobComm(), mpTag, abortSoon));
}
ICompare *icompare = helper->queryCompare();
assertex(icompare);
Owned<IRowLinkCounter> linkCounter = new CThorRowLinkCounter;
myInputStream.setown(createRowStreamMerger(streams.ordinality(), streams.getArray(), icompare, false, linkCounter));
stream = myInputStream;
}
else // serve nodes, creating merged parts
rowServer.setown(createRowServer(this, stream, queryJobChannel().queryJobComm(), mpTag));
}
processed = THORDATALINK_STARTED;
// single part key support
// has to serially pull all data fron nodes 2-N
// nodes 2-N, could/should start pushing some data (as it's supposed to be small) to cut down on serial nature.
unsigned node = queryJobChannel().queryMyRank();
if (singlePartKey)
{
if (1 == node)
{
try
{
open(*partDesc, false, helper->queryDiskRecordSize()->isVariableSize());
loop
{
OwnedConstThorRow row = inputStream->ungroupedNextRow();
if (!row)
break;
if (abortSoon) return;
processRow(row);
}
unsigned node = 2;
while (node <= container.queryJob().querySlaves())
{
Linked<IOutputRowDeserializer> deserializer = ::queryRowDeserializer(input);
CMessageBuffer mb;
Owned<ISerialStream> stream = createMemoryBufferSerialStream(mb);
CThorStreamDeserializerSource rowSource;
rowSource.setStream(stream);
bool successSR;
loop
{
{
BooleanOnOff tf(receivingTag2);
successSR = queryJobChannel().queryJobComm().sendRecv(mb, node, mpTag2);
}
if (successSR)
{
if (rowSource.eos())
break;
Linked<IEngineRowAllocator> allocator = ::queryRowAllocator(input);
do
{
RtlDynamicRowBuilder rowBuilder(allocator);
size32_t sz = deserializer->deserialize(rowBuilder, rowSource);
OwnedConstThorRow fRow = rowBuilder.finalizeRowClear(sz);
processRow(fRow);
}
while (!rowSource.eos());
}
}
node++;
}
}
catch (CATCHALL)
{
close(*partDesc, partCrc, true);
throw;
}
close(*partDesc, partCrc, true);
doStopInput();
}
else
{
CMessageBuffer mb;
CMemoryRowSerializer mbs(mb);
Linked<IOutputRowSerializer> serializer = ::queryRowSerializer(input);
loop
{
BooleanOnOff tf(receivingTag2);
if (queryJobChannel().queryJobComm().recv(mb, 1, mpTag2)) // node 1 asking for more..
{
if (abortSoon) break;
mb.clear();
do
{
OwnedConstThorRow row = inputStream->ungroupedNextRow();
if (!row) break;
serializer->serialize(mbs, (const byte *)row.get());
} while (mb.length() < SINGLEPART_KEY_TRANSFER_SIZE); // NB: at least one row
if (!queryJobChannel().queryJobComm().reply(mb))
throw MakeThorException(0, "Failed to send index data to node 1, from node %d", node);
if (0 == mb.length())
break;
}
}
}
}
void addColumn(CColumn * _column) { m_columns.append(*_column); }
bool QueryHelper::doit(FILE * fp)
{
Owned<IClientWUCreateRequest> creq = wuclient->createWUCreateRequest();
Owned<IClientWUCreateResponse> cresp = wuclient->WUCreate(creq);
const IMultiException* excep = &cresp->getExceptions();
if(excep != NULL && excep->ordinality() > 0)
{
StringBuffer msg;
excep->errorMessage(msg);
printf("%s\n", msg.str());
return false;
}
IConstECLWorkunit* wu = &cresp->getWorkunit();
if(!wu)
{
printf("can't create workunit\n");
return false;
}
Owned<IClientWUUpdateRequest> ureq = wuclient->createWUUpdateRequest();
ureq->setWuid(wu->getWuid());
// Make a workUnit
StringBuffer jobname;
if(globals->hasProp("jobname"))
jobname.append(globals->queryProp("jobname"));
StringBuffer ecl;
if (globals->getProp("ecl", ecl))
{
if (ecl.length() && ecl.charAt(0)=='@')
{
StringBuffer filename(ecl.str()+1);
ecl.clear().loadFile(filename);
if (jobname.length() == 0)
splitFilename(filename, NULL, NULL, &jobname, NULL);
}
ureq->setQueryText(ecl.str());
}
else if (globals->hasProp("main"))
ureq->setQueryMainDefinition(globals->queryProp("main"));
else if (globals->hasProp("attr"))
ureq->setQueryText(globals->queryProp("attr"));
if (globals->getPropInt("compileOnly", 0)!=0)
ureq->setAction(WUActionCompile);
if (jobname.length())
ureq->setJobname(jobname);
IArrayOf<IEspDebugValue> dvals;
IArrayOf<IEspApplicationValue> avals;
StringBuffer xmlParams;
Owned<IPropertyIterator> it = globals->getIterator();
bool xmlSeen = false;
ForEach(*it)
{
const char * key = it->getPropKey();
if (key && strlen(key)>1)
{
if(key[0] == '-')
{
if (key[1] == 'f')
{
Owned<IEspDebugValue> dval = createDebugValue();
dval->setName(&key[2]);
dval->setValue(globals->queryProp(key));
dvals.append(*dval.getLink());
}
//All other options are ignored.
}
else if(key[0] == '_')
{
Owned<IEspApplicationValue> aval = createApplicationValue();
aval->setApplication("eclplus");
aval->setName(&key[1]);
aval->setValue(globals->queryProp(key));
avals.append(*aval.getLink());
}
else if(key[0] == '/')
{
if (xmlSeen)
throw MakeStringException(0, "query option must not be used with stored or /, and cannot appear more than once");
// The / form is expected to be used for scalars, so xmlEncode is appropriate.
// To pass sets or datasets, use the xml= version
xmlParams.appendf("<%s>", &key[1]);
encodeXML(globals->queryProp(key), xmlParams);
xmlParams.appendf("</%s>", &key[1]);
}
else if(stricmp(key, "stored")==0)
{
if (xmlSeen)
throw MakeStringException(0, "query option must not be used with stored or /, and cannot appear more than once");
const char *xml = globals->queryProp(key);
try
{
Owned<IPropertyTree> checkValid = createPTreeFromXMLString(xml);
}
catch (IException *E)
{
StringBuffer msg;
E->errorMessage(msg);
E->Release();
throw MakeStringException(0, "Invalid xml: %s", msg.str());
}
xmlParams.append(xml);
}
else if(stricmp(key, "query")==0)
{
if (xmlSeen || xmlParams.length())
throw MakeStringException(0, "query option must not be used with stored or /, and cannot appear more than once");
xmlSeen = true;
StringBuffer xml;
if (!globals->getProp(key, xml))
throw MakeStringException(0, "Invalid value for query= parameter");
if (xml.length() && xml.charAt(0)=='@')
{
StringBuffer filename(xml.str()+1);
xml.clear().loadFile(filename);
}
try
{
Owned<IPropertyTree> checkValid = createPTreeFromXMLString(xml);
}
catch (IException *E)
{
StringBuffer msg;
E->errorMessage(msg);
E->Release();
throw MakeStringException(0, "Invalid xml: %s", msg.str());
}
xmlParams.append(xml);
}
}
}
if(dvals.length() > 0)
ureq->setDebugValues(dvals);
if(avals.length() > 0)
ureq->setApplicationValues(avals);
if (xmlParams.length())
{
if (!xmlSeen)
{
xmlParams.insert(0, "<Query>");
xmlParams.append("</Query>");
}
ureq->setXmlParams(xmlParams);
}
Owned<IClientWUUpdateResponse> uresp = wuclient->WUUpdate(ureq);
const IMultiException* uexcep = &uresp->getExceptions();
if(uexcep != NULL && uexcep->ordinality() > 0)
{
StringBuffer msg;
uexcep->errorMessage(msg);
printf("%s\n", msg.str());
return false;
}
// Execute it
return doSubmitWorkUnit(fp, wu->getWuid(), globals->queryProp("cluster"));
}
void addShutdownHook(IDaliClientShutdown &shutdown)
{
shutdownHooks.append(*LINK(&shutdown));
}
//Inputs
void addInput(CColumn * _input) { m_inputs.append(*_input); }
bool Cws_machineEx::doStartStop(IEspContext &context, StringArray& addresses, char* userName, char* password, bool bStop,
IEspStartStopResponse &resp)
{
bool containCluster = false;
double version = context.getClientVersion();
const int ordinality= addresses.ordinality();
UnsignedArray threadHandles;
IArrayOf<IEspStartStopResult> resultsArray;
for (int index=0; index<ordinality; index++)
{
const char* address0 = addresses.item(index);
//address passed in is of the form "192.168.1.4:EspProcess:2:path1"
StringArray sArray;
sArray.appendList(addresses.item(index), ":");
if (sArray.ordinality() < 4)
throw MakeStringException(ECLWATCH_MISSING_PARAMS, "Incomplete arguments");
Owned<IEspStartStopResult> pResult = static_cast<IEspStartStopResult*>(new CStartStopResult(""));
const char* address = sArray.item(0);
const char* compType= sArray.item(1);
const char* OS = sArray.item(3);//index 2 is component name
const char* path = sArray.item(4);
if (!(address && *address && compType && *compType && OS && *OS && path && *path))
throw MakeStringExceptionDirect(ECLWATCH_INVALID_INPUT, "Invalid input");
if (!stricmp(compType, "ThorCluster") || !stricmp(compType, "RoxieCluster"))
containCluster = true;
#ifndef OLD_START_STOP
{
char* configAddress = NULL;
char* props1 = (char*) strchr(address, '|');
if (props1)
{
configAddress = props1+1;
*props1 = '\0';
}
else
{
configAddress = (char*) address;
}
StringBuffer newAddress;
ConvertAddress(address0, newAddress);
pResult->setAddressOrig ( newAddress.str() );//can be either IP or name of component
pResult->setAddress ( address );//can be either IP or name of component
pResult->setCompType( compType );
if (version > 1.04)
{
pResult->setName( path );
const char* pStr2 = strstr(path, "LexisNexis");
if (pStr2)
{
char name[256];
const char* pStr1 = strchr(pStr2, '|');
if (!pStr1)
{
strcpy(name, pStr2+11);
}
else
{
strncpy(name, pStr2+11, pStr1 - pStr2 -11);
name[pStr1 - pStr2 -11] = 0;
}
pResult->setName( name );
}
}
pResult->setOS( atoi(OS) );
pResult->setPath( path );
resultsArray.append(*pResult.getLink());
CStartStopThreadParam* pThreadReq;
pThreadReq = new CStartStopThreadParam(address, configAddress, bStop, m_useDefaultHPCCInit, this, context);
pThreadReq->setResultObject( pResult );
if (userName && *userName)
pThreadReq->setUserID( userName );
if (password && *password)
pThreadReq->setPassword( password );
PooledThreadHandle handle = m_threadPool->start( pThreadReq );
threadHandles.append(handle);
}
#else
{
StringBuffer newAddress;
ConvertAddress(address0, newAddress);
char* pStr = (char*) strchr(address, '|');;
if (pStr)
pStr[0] = 0;
pResult->setAddressOrig ( newAddress.str() );//can be either IP or name of component
pResult->setAddress ( address );//can be either IP or name of component
pResult->setCompType( compType );
pResult->setOS( atoi(OS) );
pResult->setPath( path );
resultsArray.append(*pResult.getLink());
CStartStopThreadParam* pThreadReq;
pThreadReq = new CStartStopThreadParam(address, bStop, this, context);
pThreadReq->setResultObject( pResult );
if (userName && *userName)
pThreadReq->setUserID( userName );
if (password && *password)
pThreadReq->setPassword( password );
PooledThreadHandle handle = m_threadPool->start( pThreadReq );
threadHandles.append(handle);
}
#endif
}
//block for worker theads to finish, if necessary, and then collect results
//
PooledThreadHandle* pThreadHandle = threadHandles.getArray();
unsigned i=threadHandles.ordinality();
while (i--)
{
m_threadPool->join(*pThreadHandle, 30000);//abort after 30 secs in remote possibility that the command blocks
pThreadHandle++;
}
resp.setStartStopResults(resultsArray);
resp.setStop(bStop);
if (version > 1.08)
{
resp.setContainCluster(containCluster);
}
return true;
}
virtual void process() override
{
ActPrintLog("INDEXWRITE: Start");
init();
IRowStream *stream = inputStream;
ThorDataLinkMetaInfo info;
input->getMetaInfo(info);
outRowAllocator.setown(getRowAllocator(helper->queryDiskRecordSize()));
start();
if (refactor)
{
assertex(isLocal);
if (active)
{
unsigned targetWidth = partDesc->queryOwner().numParts()-(buildTlk?1:0);
assertex(0 == container.queryJob().querySlaves() % targetWidth);
unsigned partsPerNode = container.queryJob().querySlaves() / targetWidth;
unsigned myPart = queryJobChannel().queryMyRank();
IArrayOf<IRowStream> streams;
streams.append(*LINK(stream));
--partsPerNode;
// Should this be merging 1,11,21,31 etc.
unsigned p=0;
unsigned fromPart = targetWidth+1 + (partsPerNode * (myPart-1));
for (; p<partsPerNode; p++)
{
streams.append(*createRowStreamFromNode(*this, fromPart++, queryJobChannel().queryJobComm(), mpTag, abortSoon));
}
ICompare *icompare = helper->queryCompare();
assertex(icompare);
Owned<IRowLinkCounter> linkCounter = new CThorRowLinkCounter;
myInputStream.setown(createRowStreamMerger(streams.ordinality(), streams.getArray(), icompare, false, linkCounter));
stream = myInputStream;
}
else // serve nodes, creating merged parts
rowServer.setown(createRowServer(this, stream, queryJobChannel().queryJobComm(), mpTag));
}
processed = THORDATALINK_STARTED;
// single part key support
// has to serially pull all data fron nodes 2-N
// nodes 2-N, could/should start pushing some data (as it's supposed to be small) to cut down on serial nature.
unsigned node = queryJobChannel().queryMyRank();
if (singlePartKey)
{
if (1 == node)
{
try
{
open(*partDesc, false, helper->queryDiskRecordSize()->isVariableSize());
for (;;)
{
OwnedConstThorRow row = inputStream->ungroupedNextRow();
if (!row)
break;
if (abortSoon) return;
processRow(row);
}
unsigned node = 2;
while (node <= container.queryJob().querySlaves())
{
Linked<IOutputRowDeserializer> deserializer = ::queryRowDeserializer(input);
CMessageBuffer mb;
Owned<ISerialStream> stream = createMemoryBufferSerialStream(mb);
CThorStreamDeserializerSource rowSource;
rowSource.setStream(stream);
bool successSR;
for (;;)
{
{
BooleanOnOff tf(receivingTag2);
successSR = queryJobChannel().queryJobComm().sendRecv(mb, node, mpTag2);
}
if (successSR)
{
if (rowSource.eos())
break;
Linked<IEngineRowAllocator> allocator = ::queryRowAllocator(input);
do
{
RtlDynamicRowBuilder rowBuilder(allocator);
size32_t sz = deserializer->deserialize(rowBuilder, rowSource);
OwnedConstThorRow fRow = rowBuilder.finalizeRowClear(sz);
processRow(fRow);
}
while (!rowSource.eos());
}
}
node++;
}
}
catch (CATCHALL)
{
close(*partDesc, partCrc, true);
throw;
}
close(*partDesc, partCrc, true);
stop();
}
else
{
CMessageBuffer mb;
CMemoryRowSerializer mbs(mb);
Linked<IOutputRowSerializer> serializer = ::queryRowSerializer(input);
for (;;)
{
BooleanOnOff tf(receivingTag2);
if (queryJobChannel().queryJobComm().recv(mb, 1, mpTag2)) // node 1 asking for more..
{
if (abortSoon) break;
mb.clear();
do
{
OwnedConstThorRow row = inputStream->ungroupedNextRow();
if (!row) break;
serializer->serialize(mbs, (const byte *)row.get());
} while (mb.length() < SINGLEPART_KEY_TRANSFER_SIZE); // NB: at least one row
if (!queryJobChannel().queryJobComm().reply(mb))
throw MakeThorException(0, "Failed to send index data to node 1, from node %d", node);
if (0 == mb.length())
break;
}
}
}
}
else
{
if (!refactor || active)
{
try
{
StringBuffer partFname;
getPartFilename(*partDesc, 0, partFname);
ActPrintLog("INDEXWRITE: process: handling fname : %s", partFname.str());
open(*partDesc, false, helper->queryDiskRecordSize()->isVariableSize());
ActPrintLog("INDEXWRITE: write");
BooleanOnOff tf(receiving);
if (!refactor || !active)
receiving = false;
do
{
OwnedConstThorRow row = inputStream->ungroupedNextRow();
if (!row)
break;
processRow(row);
} while (!abortSoon);
ActPrintLog("INDEXWRITE: write level 0 complete");
}
catch (CATCHALL)
{
close(*partDesc, partCrc, isLocal && !buildTlk && 1 == node);
throw;
}
close(*partDesc, partCrc, isLocal && !buildTlk && 1 == node);
stop();
ActPrintLog("INDEXWRITE: Wrote %" RCPF "d records", processed & THORDATALINK_COUNT_MASK);
if (buildTlk)
{
ActPrintLog("INDEXWRITE: sending rows");
NodeInfoArray tlkRows;
CMessageBuffer msg;
if (firstNode())
{
if (processed & THORDATALINK_COUNT_MASK)
{
if (enableTlkPart0)
tlkRows.append(* new CNodeInfo(0, firstRow.get(), firstRowSize, totalCount));
tlkRows.append(* new CNodeInfo(1, lastRow.get(), lastRowSize, totalCount));
}
}
else
{
if (processed & THORDATALINK_COUNT_MASK)
{
CNodeInfo row(queryJobChannel().queryMyRank(), lastRow.get(), lastRowSize, totalCount);
row.serialize(msg);
}
queryJobChannel().queryJobComm().send(msg, 1, mpTag);
}
if (firstNode())
{
ActPrintLog("INDEXWRITE: Waiting on tlk to complete");
// JCSMORE if refactor==true, is rowsToReceive here right??
unsigned rowsToReceive = (refactor ? (tlkDesc->queryOwner().numParts()-1) : container.queryJob().querySlaves()) -1; // -1 'cos got my own in array already
ActPrintLog("INDEXWRITE: will wait for info from %d slaves before writing TLK", rowsToReceive);
while (rowsToReceive--)
{
msg.clear();
receiveMsg(msg, RANK_ALL, mpTag); // NH->JCS RANK_ALL_OTHER not supported for recv
if (abortSoon)
return;
if (msg.length())
{
CNodeInfo *ni = new CNodeInfo();
ni->deserialize(msg);
tlkRows.append(*ni);
}
}
tlkRows.sort(CNodeInfo::compare);
StringBuffer path;
getPartFilename(*tlkDesc, 0, path);
ActPrintLog("INDEXWRITE: creating toplevel key file : %s", path.str());
try
{
open(*tlkDesc, true, helper->queryDiskRecordSize()->isVariableSize());
if (tlkRows.length())
{
CNodeInfo &lastNode = tlkRows.item(tlkRows.length()-1);
memset(lastNode.value, 0xff, lastNode.size);
}
ForEachItemIn(idx, tlkRows)
{
CNodeInfo &info = tlkRows.item(idx);
builder->processKeyData((char *)info.value, info.pos, info.size);
}
close(*tlkDesc, tlkCrc, true);
}
catch (CATCHALL)
{
abortSoon = true;
close(*tlkDesc, tlkCrc, true);
removeFiles(*partDesc);
throw;
}
}
}
else if (!isLocal && firstNode())
