void cloneFileInfoToDali(StringArray ¬Found, IPropertyTree *packageMap, const char *lookupDaliIp, IConstWUClusterInfo *dstInfo, const char *srcCluster, const char *remotePrefix, bool overWrite, IUserDescriptor* userdesc, bool allowForeignFiles)
{
StringBuffer user;
StringBuffer password;
if (userdesc)
{
userdesc->getUserName(user);
userdesc->getPassword(password);
}
Owned<IReferencedFileList> wufiles = createReferencedFileList(user, password, allowForeignFiles, false);
wufiles->addFilesFromPackageMap(packageMap);
SCMStringBuffer processName;
dstInfo->getRoxieProcess(processName);
wufiles->resolveFiles(processName.str(), lookupDaliIp, remotePrefix, srcCluster, !overWrite, false);
StringBuffer defReplicateFolder;
getConfigurationDirectory(NULL, "data2", "roxie", processName.str(), defReplicateFolder);
Owned<IDFUhelper> helper = createIDFUhelper();
wufiles->cloneAllInfo(helper, overWrite, true, false, dstInfo->getRoxieRedundancy(), dstInfo->getChannelsPerNode(), dstInfo->getRoxieReplicateOffset(), defReplicateFolder);
Owned<IReferencedFileIterator> iter = wufiles->getFiles();
ForEach(*iter)
{
IReferencedFile &item = iter->query();
if (item.getFlags() & (RefFileNotFound | RefFileNotOnSource))
notFound.append(item.getLogicalName());
}
}
CAci(bool _isDeny, int _perm, ACT_TYPE _act_type, const char* _dn)
{
m_isDeny = _isDeny;
m_targetattr.append("*");
m_version.append("3.0");
Owned<IJlibDateTime> timeNow = createDateTimeNow();
SCMStringBuffer dateString;
timeNow->getDateString(dateString);
m_name.appendf("%s_%d", dateString.str(), getRandom());
if(_act_type == GROUP_ACT)
m_groupdns.append(_dn);
else
m_userdns.append(_dn);
if((_perm & NewSecAccess_Full) == NewSecAccess_Full)
{
m_perms.append("all");
}
else
{
if((_perm & NewSecAccess_Write) == NewSecAccess_Write)
m_perms.append("write");
if((_perm & NewSecAccess_Read) == NewSecAccess_Read)
m_perms.append("read");
if((_perm & NewSecAccess_Access) == NewSecAccess_Access)
{
m_perms.append("compare");
m_perms.append("search");
}
}
m_permission = _perm;
}
bool QueryDataSource::createBrowseWU()
{
StringAttr dataset, datasetDefs;
StringAttrAdaptor a1(dataset), a2(datasetDefs);
wuResult->getResultDataset(a1, a2);
if (!dataset || !datasetDefs)
return false;
StringBuffer fullText;
fullText.append(datasetDefs).append(dataset);
OwnedHqlExpr parsed = parseQuery(fullText.str());
if (!parsed)
return false;
HqlExprAttr selectFields = parsed.getLink();
if (selectFields->getOperator() == no_output)
selectFields.set(selectFields->queryChild(0));
OwnedHqlExpr browseWUcode = buildQueryViewerEcl(selectFields);
if (!browseWUcode)
return false;
returnedRecord.set(browseWUcode->queryChild(0)->queryRecord());
StringAttr tempAttr;
StringAttrAdaptor temp(tempAttr);
Owned<IWorkUnitFactory> factory = getWorkUnitFactory();
Owned<IConstWorkUnit> parent = factory->openWorkUnit(wuid, false);
SCMStringBuffer user;
StringAttrAdaptor acluster(cluster);
parent->getClusterName(acluster);
parent->getUser(user);
Owned<IWorkUnit> workunit = factory->createWorkUnit(NULL, "fileViewer", user.str());
workunit->setUser(user.str());
workunit->setClusterName(cluster);
workunit->setCustomerId(parent->getCustomerId(temp).str());
workunit->setCompareMode(CompareModeOff); // ? parent->getCompareMode()
StringAttrAdaptor bwa(browseWuid); workunit->getWuid(bwa);
workunit->setDebugValueInt("importImplicitModules", false, true);
workunit->setDebugValueInt("importAllModules", false, true);
workunit->setDebugValueInt("forceFakeThor", 1, true);
StringBuffer jobName;
jobName.append("FileView for ").append(wuid).append(":").append("x");
workunit->setJobName(jobName.str());
StringBuffer eclText;
toECL(browseWUcode, eclText, true);
Owned<IWUQuery> query = workunit->updateQuery();
query->setQueryText(eclText.str());
query->setQueryName(jobName.str());
return true;
}
bool isFileKnownOnCluster(const char *logicalname, IConstWUClusterInfo *clusterInfo, IUserDescriptor* userdesc)
{
Owned<IDistributedFile> dst = queryDistributedFileDirectory().lookup(logicalname, userdesc, true);
if (dst)
{
SCMStringBuffer processName;
clusterInfo->getRoxieProcess(processName);
if (dst->findCluster(processName.str()) != NotFound)
return true; // file already known for this cluster
}
return false;
}
void updateUsers()
{
Owned<IStringIterator> clusters = getTargetClusters("RoxieCluster", NULL);
ForEach(*clusters)
{
SCMStringBuffer target;
clusters->str(target);
Owned<IConstWUClusterInfo> info = getTargetClusterInfo(target.str());
Owned<IUserDescriptor> user = createUserDescriptor();
user->set(info->getLdapUser(), info->getLdapPassword());
roxieUserMap.setValue(target.str(), user);
roxieUsers.append(*user.getClear());
}
}
bool WuResubmit(const char *wuid)
{
Owned<IWorkUnitFactory> factory = getWorkUnitFactory();
Owned<IWorkUnit> wu = factory->updateWorkUnit(wuid);
if (!wu)
{
ERRLOG("WuResubmit(%s): could not find workunit",wuid);
return false;
}
if (wu->getState()!=WUStateFailed)
{
ERRLOG("WuResubmit(%s): could not resubmit as workunit state is '%s'", wuid, wu->queryStateDesc());
return false;
}
SCMStringBuffer token;
wu->getSecurityToken(token);
SCMStringBuffer user;
SCMStringBuffer password;
extractToken(token.str(), wuid, user, password);
wu->resetWorkflow();
wu->setState(WUStateSubmitted);
wu->commit();
wu.clear();
submitWorkUnit(wuid,user.str(),password.str());
PROGLOG("WuResubmit(%s): resubmitted",wuid);
return true;
}
void processWorkunit(IConstWorkUnit *workunit, SCMStringBuffer &queryName, IRoxieQueryProcessingInfo &processingInfo, WUQueryActivationOptions activateOption, const char *querySetName, bool notifyRoxie, SCMStringBuffer &status, SCMStringBuffer &roxieDeployStatus)
{
Owned<IWorkUnit> wu = &workunit->lock();
SCMStringBuffer jobName;
wu->getJobName(jobName);
if (stricmp(jobName.str(),queryName.str())!=0)
wu->setJobName(queryName.str());
try
{
// look up data file info
Owned<IRoxieWuProcessor> wuProcessor = createRoxieWuProcessor(roxieName, queryCommClient, logLevel);
wuProcessor->lookupFileNames(wu, processingInfo, status);
IPropertyTree *pkgInfo = wuProcessor->queryPackageInfo();
StringBuffer newQueryId;
const char *qsName = resolveQuerySetName(querySetName);
addQueryToQuerySet(wu, qsName, queryName.str(), pkgInfo, activateOption, newQueryId);
const char *queryComment = processingInfo.queryComment();
if (queryComment)
setQueryCommentForNamedQuery(qsName, newQueryId.str(), queryComment);
}
catch(IException *e)
{
int errCode = e->errorCode();
StringBuffer err;
e->errorMessage(err);
status.s.appendf("%d %s", errCode, err.str());
DBGLOG("ERROR updating query list %s", status.str());
e->Release();
}
if (notifyRoxie)
{
queryCommClient->sendRoxieReloadControlRequest();
}
}
bool WsEclWuInfo::getWsResource(const char *name, StringBuffer &out)
{
if (strieq(name, "SOAP"))
{
out.appendf("<message name=\"%s\">", queryname.sget());
IConstWUResultIterator &vars = ensureWorkUnit()->getVariables();
Owned<IResultSetFactory> resultSetFactory(getResultSetFactory(username, password));
ForEach(vars)
{
IConstWUResult &var = vars.query();
SCMStringBuffer varname;
var.getResultName(varname);
int seq = var.getResultSequence();
WUResultFormat fmt = var.getResultFormat();
SCMStringBuffer eclschema;
var.getResultEclSchema(eclschema);
SCMStringBuffer s;
Owned<IResultSetMetaData> meta = resultSetFactory->createResultSetMeta(&var);
if (!var.isResultScalar())
{
meta->getXmlSchema(s, false);
out.appendf("<part name=\"%s\" type=\"tns:XmlDataSet\" />", varname.str());
}
else
{
meta->getColumnEclType(s, 0);
DisplayType dt = meta->getColumnDisplayType(0);
StringAttr ptype;
switch (dt)
{
case TypeBoolean:
ptype.set("xsd:boolean");
break;
case TypeInteger:
ptype.set("xsd:integer");
break;
case TypeUnsignedInteger:
ptype.set("xsd:integer");
break;
case TypeReal:
ptype.set("xsd:real");
break;
case TypeSet:
ptype.set("tns:EspStringArray");
break;
case TypeDataset:
case TypeData:
ptype.set("tns:XmlDataSet");
break;
case TypeUnicode:
case TypeString:
ptype.set("xsd:string");
break;
case TypeUnknown:
case TypeBeginIfBlock:
case TypeEndIfBlock:
case TypeBeginRecord:
default:
ptype.set("xsd:string");
break;
}
out.appendf("<part name=\"%s\" type=\"%s\" />", varname.str(), ptype.sget());
}
}
out.append("</message>");
}
int CEclAgentExecutionServer::run()
{
SCMStringBuffer queueNames;
Owned<IFile> sentinelFile = createSentinelTarget();
removeSentinelFile(sentinelFile);
try
{
Owned<IGroup> serverGroup = createIGroup(daliServers, DALI_SERVER_PORT);
initClientProcess(serverGroup, DCR_AgentExec);
getAgentQueueNames(queueNames, agentName);
queue.setown(createJobQueue(queueNames.str()));
queue->connect();
}
catch (IException *e)
{
EXCLOG(e, "Server queue create/connect: ");
e->Release();
return -1;
}
catch(...)
{
ERRLOG("Terminating unexpectedly");
}
CSDSServerStatus serverStatus("HThorServer");
serverStatus.queryProperties()->setProp("@queue",queueNames.str());
serverStatus.queryProperties()->setProp("@cluster", agentName);
serverStatus.commitProperties();
writeSentinelFile(sentinelFile);
try
{
while (started)
{
PROGLOG("AgentExec: Waiting on queue(s) '%s'", queueNames.str());
Owned<IJobQueueItem> item = queue->dequeue();
if (item.get())
{
StringAttr wuid;
wuid.set(item->queryWUID());
PROGLOG("AgentExec: Dequeued workunit request '%s'", wuid.get());
try
{
executeWorkunit(wuid);
}
catch(IException *e)
{
EXCLOG(e, "CEclAgentExecutionServer::run: ");
}
catch(...)
{
ERRLOG("Unexpected exception in CEclAgentExecutionServer::run caught");
}
}
else
{
ERRLOG("Unexpected dequeue of bogus job queue item, exiting agentexec");
removeSentinelFile(sentinelFile);//no reason to restart
assert(!started);
break;
}
}
}
catch (IException *e)
{
EXCLOG(e, "Server Exception: ");
e->Release();
PROGLOG("Exiting");
}
try
{
queue->disconnect();
}
catch (IException *e)
{
EXCLOG(e, "Server queue disconnect: ");
e->Release();
}
PROGLOG("Exiting agentexec\n");
return 1;
}
bool compile(const char *wuid, const char *target, const char *targetCluster)
{
Owned<IConstWUQuery> query = workunit->getQuery();
if (!query)
{
reportError("Workunit does not contain a query", 2);
return false;
}
addTimeStamp(workunit, SSTglobal, NULL, StWhenCompiled);
SCMStringBuffer mainDefinition;
SCMStringBuffer eclQuery;
query->getQueryText(eclQuery);
query->getQueryMainDefinition(mainDefinition);
StringBuffer eclccProgName;
splitDirTail(queryCurrentProcessPath(), eclccProgName);
eclccProgName.append("eclcc");
StringBuffer eclccCmd(" -shared");
if (eclQuery.length())
eclccCmd.append(" -");
if (mainDefinition.length())
eclccCmd.append(" -main ").append(mainDefinition);
eclccCmd.append(" --timings --xml");
eclccCmd.append(" --nostdinc");
if (globals->getPropBool("@enableEclccDali", true))
{
const char *daliServers = globals->queryProp("@daliServers");
if (!daliServers)
daliServers = ".";
eclccCmd.appendf(" -dfs=%s", daliServers);
const char *wuScope = workunit->queryWuScope();
if (!isEmptyString(wuScope))
eclccCmd.appendf(" -scope=%s", wuScope);
eclccCmd.appendf(" -cluster=%s", targetCluster);
SCMStringBuffer token;
workunit->getSecurityToken(token);
if (token.length())
eclccCmd.appendf(" -wuid=%s -token=%s", workunit->queryWuid(), token.str());
}
Owned<IPipeProcess> pipe = createPipeProcess();
pipe->setenv("ECLCCSERVER_THREAD_INDEX", idxStr.str());
Owned<IPropertyTreeIterator> options = globals->getElements("./Option");
ForEach(*options)
{
IPropertyTree &option = options->query();
const char *name = option.queryProp("@name");
const char *value = option.queryProp("@value");
const char *cluster = option.queryProp("@cluster"); // if cluster is set it's specific to a particular target
if (name && (cluster==NULL || cluster[0]==0 || strcmp(cluster, targetCluster)==0))
processOption(name, value, eclccCmd, eclccProgName, *pipe, false);
}
eclccCmd.appendf(" -o%s", wuid);
eclccCmd.appendf(" -platform=%s", target);
eclccCmd.appendf(" --component=%s", queryStatisticsComponentName());
Owned<IStringIterator> debugValues = &workunit->getDebugValues();
ForEach (*debugValues)
{
SCMStringBuffer debugStr, valueStr;
debugValues->str(debugStr);
workunit->getDebugValue(debugStr.str(), valueStr);
processOption(debugStr.str(), valueStr.str(), eclccCmd, eclccProgName, *pipe, true);
}
if (workunit->getResultLimit())
{
eclccCmd.appendf(" -fapplyInstantEclTransformations=1 -fapplyInstantEclTransformationsLimit=%u", workunit->getResultLimit());
}
try
{
cycle_t startCycles = get_cycles_now();
Owned<ErrorReader> errorReader = new ErrorReader(pipe, this);
Owned<AbortWaiter> abortWaiter = new AbortWaiter(pipe, workunit);
eclccCmd.insert(0, eclccProgName);
if (!pipe->run(eclccProgName, eclccCmd, ".", true, false, true, 0, true))
throw makeStringExceptionV(999, "Failed to run eclcc command %s", eclccCmd.str());
errorReader->start();
abortWaiter->start();
try
{
pipe->write(eclQuery.s.length(), eclQuery.s.str());
pipe->closeInput();
}
catch (IException *e)
{
reportError(e);
e->Release();
}
unsigned retcode = pipe->wait();
errorReader->join();
abortWaiter->stop();
if (retcode == 0)
{
StringBuffer realdllname, dllurl;
realdllname.append(SharedObjectPrefix).append(wuid).append(SharedObjectExtension);
StringBuffer realdllfilename(dllPath);
realdllfilename.append(SharedObjectPrefix).append(wuid).append(SharedObjectExtension);
StringBuffer wuXML;
if (!getWorkunitXMLFromFile(realdllfilename, wuXML))
throw makeStringException(999, "Failed to extract workunit from query dll");
Owned<ILocalWorkUnit> embeddedWU = createLocalWorkUnit(wuXML);
queryExtendedWU(workunit)->copyWorkUnit(embeddedWU, false, true);
workunit->setIsClone(false);
const char *jobname = embeddedWU->queryJobName();
if (jobname && *jobname) //let ECL win naming job during initial compile
workunit->setJobName(jobname);
if (!workunit->getDebugValueBool("obfuscateOutput", false))
{
Owned<IWUQuery> query = workunit->updateQuery();
query->setQueryText(eclQuery.s.str());
}
createUNCFilename(realdllfilename.str(), dllurl);
unsigned crc = crc_file(realdllfilename.str());
Owned<IWUQuery> query = workunit->updateQuery();
associateLocalFile(query, FileTypeDll, realdllfilename, "Workunit DLL", crc);
queryDllServer().registerDll(realdllname.str(), "Workunit DLL", dllurl.str());
cycle_t elapsedCycles = get_cycles_now() - startCycles;
updateWorkunitTimeStat(workunit, SSTcompilestage, "compile", StTimeElapsed, NULL, cycle_to_nanosec(elapsedCycles));
workunit->commit();
return true;
}
}
catch (IException * e)
{
reportError(e);
e->Release();
}
return false;
}
bool DumpHelper::doit(FILE * fp)
{
SCMStringBuffer xml;
// If we were given a workunit dump that one workunit, otherwise moan
if (globals->hasProp("WUID"))
{
const char* wuid = globals->queryProp("WUID");
const char *whichPath = globals->queryProp("prop");
if (whichPath)
{
if (stricmp(whichPath, "ecl")==0)
{
Owned<IClientWUInfoRequest> inforeq = wuclient->createWUInfoRequest();
inforeq->setWuid(wuid);
Owned<IClientWUInfoResponse> inforesp = wuclient->WUInfo(inforeq);
if(!inforesp)
{
printf("Workunit %s not found\n", wuid);
return false;
}
IConstECLWorkunit* wu = &inforesp->getWorkunit();
IConstECLQuery* query = &wu->getQuery();
if(query)
xml.set(query->getText());
}
else
{
printf("Unrecognized parameter prop=%s", whichPath);
return false;
}
}
else
{
Owned<IClientWULogFileRequest> req = wuclient->createWUFileRequest();
req->setWuid(wuid);
req->setType("XML");
Owned<IClientWULogFileResponse> resp = wuclient->WUFile(req);
if(!resp)
{
printf("Workunit %s not found\n", wuid);
return false;
}
const IMultiException* excep = &resp->getExceptions();
if(excep != NULL && excep->ordinality())
{
unsigned i = 0;
while (i < excep->ordinality())
{
StringBuffer msg;
excep->item(i).errorMessage(msg);
unsigned code = excep->item(i).errorCode();
printf("<Error><code>%d</code><message>%s</message></Error>\n", code, msg.str());
}
return false;
}
const MemoryBuffer & xmlmem = resp->getThefile();
StringBuffer xmlbuf;
xmlbuf.append(xmlmem.length(), xmlmem.toByteArray());
xml.set(xmlbuf.str());
}
// Print the results
if (fp != NULL)
{
fprintf(fp, "%s", xml.str());
}
return true;
}
return false;
}
bool compile(const char *wuid, const char *target, const char *targetCluster)
{
Owned<IConstWUQuery> query = workunit->getQuery();
if (!query)
{
reportError("Workunit does not contain a query", 2);
return false;
}
SCMStringBuffer mainDefinition;
SCMStringBuffer eclQuery;
query->getQueryText(eclQuery);
query->getQueryMainDefinition(mainDefinition);
StringBuffer eclccCmd("eclcc -shared");
if (eclQuery.length())
eclccCmd.append(" -");
if (mainDefinition.length())
eclccCmd.append(" -main ").append(mainDefinition);
Owned<IPropertyTreeIterator> options = globals->getElements("./Option");
ForEach(*options)
{
IPropertyTree &option = options->query();
const char *name = option.queryProp("@name");
const char *value = option.queryProp("@value");
const char *cluster = option.queryProp("@cluster");
if (name && (cluster==NULL || cluster[0]==0 || strcmp(cluster, targetCluster)==0))
{
// options starting '-' are simply passed through to eclcc as name=value
// others are passed as -foption=value
// if cluster is set it's specific to a particular target
eclccCmd.append(" ");
if (name[0]!='-')
eclccCmd.append("-f");
eclccCmd.append(name);
if (value)
eclccCmd.append('=').append(value);
}
}
eclccCmd.appendf(" -o%s", wuid);
eclccCmd.appendf(" -target=%s", target);
Owned<IStringIterator> debugValues = &workunit->getDebugValues();
ForEach (*debugValues)
{
SCMStringBuffer debugStr, valueStr;
debugValues->str(debugStr);
workunit->getDebugValue(debugStr.str(), valueStr);
if (memicmp(debugStr.str(), "eclcc-", 6) == 0)
{
//Allow eclcc-xx-<n> so that multiple values can be passed through for the same named debug symbol
const char * start = debugStr.str() + 6;
const char * dash = strchr(start, '-');
StringAttr optName;
if (dash)
optName.set(start, dash-start);
else
optName.set(start);
if (stricmp(optName, "compileOption") == 0)
eclccCmd.appendf(" -Wc,%s", valueStr.str());
else if (stricmp(optName, "includeLibraryPath") == 0)
eclccCmd.appendf(" -I%s", valueStr.str());
else if (stricmp(optName, "libraryPath") == 0)
eclccCmd.appendf(" -L%s", valueStr.str());
else
eclccCmd.appendf(" -%s=%s", start, valueStr.str());
}
else
eclccCmd.appendf(" -f%s=%s", debugStr.str(), valueStr.str());
}
if (workunit->getResultLimit())
{
eclccCmd.appendf(" -fapplyInstantEclTransformations=1 -fapplyInstantEclTransformationsLimit=%u", workunit->getResultLimit());
}
try
{
Owned<IPipeProcess> pipe = createPipeProcess();
Owned<ErrorReader> errorReader = new ErrorReader(pipe, this);
pipe->run("eclcc", eclccCmd, ".", true, false, true, 0);
errorReader->start();
try
{
pipe->write(eclQuery.s.length(), eclQuery.s.str());
pipe->closeInput();
}
catch (IException *e)
{
reportError(e);
e->Release();
}
unsigned retcode = pipe->wait();
errorReader->join();
if (retcode == 0)
{
StringBuffer realdllname, dllurl;
realdllname.append(SharedObjectPrefix).append(wuid).append(SharedObjectExtension);
StringBuffer realdllfilename(dllPath);
realdllfilename.append(SharedObjectPrefix).append(wuid).append(SharedObjectExtension);
StringBuffer wuXML;
if (getWorkunitXMLFromFile(realdllfilename, wuXML))
{
Owned<ILocalWorkUnit> embeddedWU = createLocalWorkUnit();
embeddedWU->loadXML(wuXML);
queryExtendedWU(workunit)->copyWorkUnit(embeddedWU);
Owned<IWUQuery> query = workunit->updateQuery();
query->setQueryText(eclQuery.s.str());
}
createUNCFilename(realdllfilename.str(), dllurl);
unsigned crc = crc_file(realdllfilename.str());
Owned<IWUQuery> query = workunit->updateQuery();
associateLocalFile(query, FileTypeDll, realdllfilename, "Workunit DLL", crc);
queryDllServer().registerDll(realdllname.str(), "Workunit DLL", dllurl.str());
workunit->commit();
return true;
}
}
catch (IException * e)
{
reportError(e);
e->Release();
}
return false;
}
void appendVariableParmInfo(IArrayOf<IPropertyTree> &parts, IResultSetFactory *resultSetFactory, IConstWUResult &var, unsigned hashWebserviceSeq=0)
{
Owned<IResultSetMetaData> meta = resultSetFactory->createResultSetMeta(&var);
StringAttr noinput;
if (var.getResultFieldOpt("noinput", StringAttrAdaptor(noinput)).length() && strToBool(noinput.length(), noinput.get())) //developer specified not to show field on form
return;
SCMStringBuffer varname;
var.getResultName(varname);
int seq = var.getResultSequence();
WUResultFormat fmt = var.getResultFormat();
SCMStringBuffer eclschema;
var.getResultEclSchema(eclschema);
StringBuffer width, height, fieldSeq, isPassword, select;
var.getResultFieldOpt("fieldwidth", StringBufferAdaptor(width));
var.getResultFieldOpt("fieldheight", StringBufferAdaptor(height));
var.getResultFieldOpt("password", StringBufferAdaptor(isPassword));
var.getResultFieldOpt("select", StringBufferAdaptor(select));
if (hashWebserviceSeq)
fieldSeq.append(hashWebserviceSeq);
else
var.getResultFieldOpt("sequence", StringBufferAdaptor(fieldSeq));
SCMStringBuffer s;
Owned<IPropertyTree> part = createPTree("part");
if (!var.isResultScalar())
{
meta->getXmlSchema(s, false);
part->setProp("@name", varname.str());
part->setProp("@type", "tns:XmlDataset");
if (fieldSeq.length())
part->setProp("@sequence", fieldSeq);
}
else
{
meta->getColumnEclType(s, 0);
DisplayType dt = meta->getColumnDisplayType(0);
StringAttr ptype;
switch (dt)
{
case TypeBoolean:
ptype.set("xsd:boolean");
break;
case TypeInteger:
ptype.set("xsd:integer");
break;
case TypeUnsignedInteger:
ptype.set("xsd:integer");
break;
case TypeReal:
ptype.set("xsd:real");
break;
case TypeSet:
ptype.set("tns:EspStringArray");
break;
case TypeDataset:
case TypeData:
ptype.set("tns:XmlDataSet");
break;
case TypeUnicode:
case TypeString:
ptype.set("xsd:string");
break;
case TypeUnknown:
case TypeBeginIfBlock:
case TypeEndIfBlock:
case TypeBeginRecord:
default:
ptype.set("xsd:string");
break;
}
part->setProp("@name", varname.str());
part->setProp("@type", ptype.str());
if (width.length())
part->setProp("@width", width);
if (height.length())
part->setProp("@height", height);
if (fieldSeq.length())
part->setProp("@sequence", fieldSeq);
if (isPassword.length())
part->setProp("@password", isPassword);
if (select.length())
{
StringArray optionList;
optionList.appendList(select, ",");
IPropertyTree *selectTree = ensurePTree(part, "select");
ForEachItemIn(i, optionList)
{
const char *value = optionList.item(i);
bool selected = '*'==*value;
if (selected)
value++;
StringAttr name;
const char *eq = strchr(value, '=');
if (!eq)
name.set(value);
else
{
name.set(value, eq-value);
value = eq + 1;
}
Owned<IPropertyTree> optionTree = createPTree();
optionTree->setProp("@name", name);
optionTree->setProp("@value", value);
if (selected)
optionTree->setPropBool("@selected", true);
selectTree->addPropTree("option", optionTree.getClear());
}
}
}
bool CEclDirectEx::onRunEcl(IEspContext &context, IEspRunEclRequest & req, IEspRunEclResponse & resp)
{
if (!context.validateFeatureAccess(ECLDIRECT_ACCESS, SecAccess_Full, false))
throw MakeStringException(-1, "EclDirect access permission denied.");
StringBuffer user;
if (!context.getUserID(user).length())
user.append(req.getUserName());
Owned <IWorkUnitFactory> factory = getWorkUnitFactory(context.querySecManager(), context.queryUser());
Owned <IWorkUnit> workunit;
if (!user.length())
workunit.setown(factory->createWorkUnit(NULL, "ECL-Direct", ""));
else
{
workunit.setown(factory->createWorkUnit(NULL, "ECL-Direct", user.str()));
workunit->setUser(user.str());
}
Owned<IWUQuery> query = workunit->updateQuery();
query->setQueryText(req.getEclText());
query.clear();
const char* clustername = req.getCluster();
if (!clustername || !*clustername || strieq(clustername, "default"))
clustername = defaultCluster.str();
if (!clustername || !*clustername)
throw MakeStringException(-1, "No Cluster Specified");
if (!isValidCluster(clustername))
throw MakeStringException(-1, "Invalid TargetCluster %s Specified", clustername);
workunit->setClusterName(clustername);
if (req.getLimitResults())
workunit->setResultLimit(100);
const char* snapshot = req.getSnapshot();
if (snapshot && *snapshot)
workunit->setSnapshot(snapshot);
// Execute it
SCMStringBuffer wuid;
workunit->getWuid(wuid);
workunit->setAction(WUActionRun);
workunit->setState(WUStateSubmitted);
workunit.clear();
submitWorkUnit(wuid.str(), context.querySecManager(), context.queryUser());
if (waitForWorkUnitToComplete(wuid.str(), defaultWait))
{
Owned<IConstWorkUnit> cw = factory->openWorkUnit(wuid.str(), false);
SCMStringBuffer resultXML;
getFullWorkUnitResultsXML(context.queryUserId(), context.queryPassword(), cw.get(), resultXML);
resp.setResults(resultXML.str());
cw.clear();
if (deleteWorkunits)
deleteEclDirectWorkunit(factory, wuid.str());
}
else
{
// Don't delete these ones...
DBGLOG("WorkUnit %s timed out", wuid.str());
StringBuffer result;
result.appendf("<Exception><Source>ESP</Source><Message>Timed out waiting for job to complete: %s</Message></Exception>", wuid.str());
resp.setResults(result.str());
}
return true;
}
virtual void doWork()
{
if (m_useHPCCInit)
{
//address specified can be either IP or name of component
const char* address = m_sAddress.str();
const char* configAddress = m_sConfigAddress.str();
if (!address || !*address)
throw MakeStringException(ECLWATCH_INVALID_IP_OR_COMPONENT, "Invalid address or component name was specified!");
if (!strchr(address, '.')) //not an IP address
{
const char* compType = m_pResult->getCompType();
const char* compName = address;
if (!compType || !*compType)
throw MakeStringException(ECLWATCH_INVALID_COMPONENT_TYPE, "No component type specified!");
StringBuffer xpath;
if (!strcmp(compType, "RoxieCluster"))
{
xpath.append("RoxieServer");
}
else if (!strcmp(compType, "ThorCluster"))
{
xpath.append("ThorMaster");
}
else if (!strcmp(compType, "HoleCluster"))
{
xpath.append("HoleControl");
}
else
throw MakeStringException(ECLWATCH_INVALID_COMPONENT_TYPE, "Failed to resolve component type '%s'", compType);
Owned<IPropertyTree> pComponent = m_pService->getComponent(compType, compName);
xpath.append("Process[1]/@computer");
const char* computer = pComponent->queryProp(xpath.str());
if (!computer || !*computer)
throw MakeStringException(ECLWATCH_INVALID_COMPONENT_INFO, "Failed to resolve computer for %s '%s'!", compType, compName);
Owned<IConstEnvironment> pConstEnv = m_pService->getConstEnvironment();
Owned<IConstMachineInfo> pConstMc = pConstEnv->getMachine(computer);
SCMStringBuffer sAddress;
pConstMc->getNetAddress(sAddress);
if (!stricmp(m_sAddress.str(), m_sConfigAddress.str()))
{
m_sAddress.clear().append(sAddress.str());
m_sConfigAddress = m_sAddress;
m_pResult->setAddress( sAddress.str() );
}
else
{
m_sAddress.clear().append(sAddress.str());
m_pResult->setAddress( sAddress.str() );
if (configAddress && !strchr(configAddress, '.')) //not an IP address
{
Owned<IPropertyTree> pComponent = m_pService->getComponent(compType, configAddress);
xpath.append("Process[1]/@computer");
const char* computer = pComponent->queryProp(xpath.str());
if (!computer || !*computer)
throw MakeStringException(ECLWATCH_INVALID_COMPONENT_INFO, "Failed to resolve computer for %s '%s'!", compType, configAddress);
Owned<IConstEnvironment> pConstEnv = m_pService->getConstEnvironment();
Owned<IConstMachineInfo> pConstMc = pConstEnv->getMachine(computer);
SCMStringBuffer sAddress;
pConstMc->getNetAddress(sAddress);
m_sConfigAddress.clear().append(sAddress.str());
}
}
}
if ((m_sAddress.length() > 0) && !stricmp(m_sAddress.str(), "."))
{
StringBuffer ipStr;
IpAddress ipaddr = queryHostIP();
ipaddr.getIpText(ipStr);
if (ipStr.length() > 0)
{
#ifdef MACHINE_IP
m_sAddress.clear().append(MACHINE_IP);
#else
m_sAddress.clear().append(ipStr.str());
#endif
m_pResult->setAddress( m_sAddress.str() );
}
}
#ifdef OLD_START_STOP
int OS = m_pResult->getOS();
StringBuffer sPath( m_pResult->getPath() );
if (OS == 0)
sPath.replace('$', ':');
else
if (sPath.charAt(0) != '/')
sPath.insert(0, '/');
m_sCommand.clear().append(sPath).append( OS==0 ? '\\' : '/');
m_sCommand.append(m_bStop ? "stop" : "startup");
if (OS == 0)
m_sCommand.append(".bat");
m_sCommand.append(' ');
m_sCommand.append(sPath);
#else
StringBuffer sPath( m_pResult->getPath() );
if (sPath.charAt(sPath.length() - 1) == '/')
sPath.setLength(sPath.length() - 1);
if (sPath.length() > 0)
{
char* pStr = (char*) sPath.str();
char* ppStr = strchr(pStr, '/');
while (ppStr)
{
ppStr++;
pStr = ppStr;
ppStr = strchr(pStr, '/');
}
if (!m_bStop)
m_sCommand.appendf("sudo /etc/init.d/hpcc-init -c %s start", pStr);
else
m_sCommand.appendf("sudo /etc/init.d/hpcc-init -c %s stop", pStr);
}
#endif
m_pResult->setCommand( m_sCommand.str() );
}
else
{
//address specified can be either IP or name of component
const char* address = m_sAddress.str();
if (!address || !*address)
throw MakeStringException(ECLWATCH_INVALID_IP_OR_COMPONENT, "Invalid address or component name was specified!");
if (!strchr(address, '.')) //not an IP address
{
const char* compType = m_pResult->getCompType();
const char* compName = address;
if (!compType || !*compType)
throw MakeStringException(ECLWATCH_INVALID_COMPONENT_TYPE, "No component type specified!");
StringBuffer xpath;
if (!strcmp(compType, "RoxieCluster"))
{
xpath.append("RoxieServer");
}
else if (!strcmp(compType, "ThorCluster"))
{
xpath.append("ThorMaster");
}
else if (!strcmp(compType, "HoleCluster"))
{
xpath.append("HoleControl");
}
else
throw MakeStringException(ECLWATCH_INVALID_COMPONENT_TYPE, "Failed to resolve component type '%s'", compType);
Owned<IPropertyTree> pComponent = m_pService->getComponent(compType, compName);
xpath.append("Process[1]/@computer");
const char* computer = pComponent->queryProp(xpath.str());
if (!computer || !*computer)
throw MakeStringException(ECLWATCH_INVALID_COMPONENT_INFO, "Failed to resolve computer for %s '%s'!", compType, compName);
Owned<IConstEnvironment> pConstEnv = m_pService->getConstEnvironment();
Owned<IConstMachineInfo> pConstMc = pConstEnv->getMachine(computer);
SCMStringBuffer sAddress;
pConstMc->getNetAddress(sAddress);
m_sAddress.clear().append(sAddress.str());
m_pResult->setAddress( sAddress.str() );
}
int OS = m_pResult->getOS();
StringBuffer sPath( m_pResult->getPath() );
if (OS == 0)
sPath.replace('$', ':');
else
if (sPath.charAt(0) != '/')
sPath.insert(0, '/');
m_sCommand.clear().append(sPath).append( OS==0 ? '\\' : '/');
m_sCommand.append(m_bStop ? "stop" : "startup");
if (OS == 0)
m_sCommand.append(".bat");
m_sCommand.append(' ');
m_sCommand.append(sPath);
m_pResult->setCommand( m_sCommand.str() );
}
CRemoteExecThreadParam::doWork();
}
bool CEclDirectEx::onRunEclEx(IEspContext &context, IEspRunEclExRequest & req, IEspRunEclExResponse & resp)
{
if (!context.validateFeatureAccess(ECLDIRECT_ACCESS, SecAccess_Full, false))
throw MakeStringException(-1, "EclDirect access permission denied.");
const char* eclText = req.getEclText();
if (!eclText || !*eclText)
{
resp.setResults("<Exception><Source>ESP</Source><Message>No Ecl Text provided</Message></Exception>");
return true;
}
StringBuffer user;
if (!context.getUserID(user).length())
user.append(req.getUserName());
Owned <IWorkUnitFactory> factory = getWorkUnitFactory(context.querySecManager(), context.queryUser());
Owned <IWorkUnit> workunit;
if (!user.length())
workunit.setown(factory->createWorkUnit(NULL, "ECL-Direct", ""));
else
{
workunit.setown(factory->createWorkUnit(NULL, "ECL-Direct", user.str()));
workunit->setUser(user.str());
}
Owned<IWUQuery> query = workunit->updateQuery();
query->setQueryText(eclText);
query.clear();
const char* cluster = req.getCluster();
if (!cluster || !*cluster || !stricmp(cluster, "default"))
cluster = defaultCluster.str();
if (!cluster || !*cluster)
throw MakeStringException(-1, "No Cluster Specified");
if (!isValidCluster(cluster))
throw MakeStringException(-1, "Invalid TargetCluster %s Specified", cluster);
workunit->setClusterName(cluster);
const char* snapshot = req.getSnapshot();
if (snapshot && *snapshot)
workunit->setSnapshot(snapshot);
if (req.getResultLimit())
workunit->setResultLimit(req.getResultLimit());
// Execute it
SCMStringBuffer wuid;
workunit->getWuid(wuid);
workunit->setAction(WUActionRun);
workunit->setState(WUStateSubmitted);
workunit.clear();
resp.setWuid(wuid.str());
submitWorkUnit(wuid.str(), context.querySecManager(), context.queryUser());
if (!waitForWorkUnitToComplete(wuid.str(), (req.getWait_isNull()) ? defaultWait : req.getWait()))
{
StringBuffer result;
result.appendf("<Exception><Source>ESP</Source><Message>Timed out waiting for job to complete: %s</Message></Exception>", wuid.str());
resp.setResults(result.str());
return true;
}
if (!deleteWorkunits && context.queryRequestParameters()->hasProp("redirect"))
{
StringBuffer url("/WsWorkunits/WUInfo?Wuid=");
resp.setRedirectUrl(url.append(wuid).str());
return true;
}
Owned<IConstWorkUnit> cw = factory->openWorkUnit(wuid.str(), false);
EclDirectWUExceptions errors(*cw);
resp.setErrors(errors);
if (req.getIncludeResults())
{
StringBuffer results;
CRunEclExFormat outputFormat = req.getFormat();
Owned<IWuWebView> web = createWuWebView(wuid.str(), NULL, NULL, getCFD(), true);
if (!web)
results.appendf("<Exception><Source>ESP</Source><Message>Failed loading result workunit %s</Message></Exception>", wuid.str());
else if (outputFormat == CRunEclExFormat_Table)
{
StringBuffer xsltfile(getCFD());
web->applyResultsXSLT(xsltfile.append("xslt/wsecl3_result.xslt").str(), results);
}
else
{
unsigned xmlflags = 0;
if (outputFormat != CRunEclExFormat_ExtendedXml)
xmlflags |= WWV_OMIT_SCHEMAS;
if (context.queryRequestParameters()->hasProp("display_xslt"))
xmlflags |= WWV_USE_DISPLAY_XSLT;
else
xmlflags |= WWV_OMIT_XML_DECLARATION;
web->expandResults(results, xmlflags);
}
resp.setResults(results.str());
}
if (req.getIncludeGraphs())
{
Owned<IConstWUGraphIterator> it = &cw->getGraphs(GraphTypeAny);
StringBuffer xgmml("<Graphs>");
SCMStringBuffer s;
ForEach(*it)
xgmml.append(it->query().getXGMML(s, true).str());
xgmml.append("</Graphs>");
resp.setGraphsXGMML(xgmml.str());
}
if (deleteWorkunits)
deleteEclDirectWorkunit(factory, wuid.str());
return true;
}
void Cws_machineEx::ConvertAddress( const char* originalAddress, StringBuffer& newAddress)
{
if (!originalAddress || !*originalAddress)
throw MakeStringException(ECLWATCH_INVALID_IP_OR_COMPONENT, "No network address or computer name specified!");
StringArray sArray;
sArray.appendList(originalAddress, ":");
if (sArray.ordinality() < 4)
throw MakeStringException(ECLWATCH_MISSING_PARAMS, "Incomplete arguments");
const char* address = sArray.item(0);
const char* compType= sArray.item(1);
const char* compName= sArray.item(2);
const char* OS = sArray.item(3);
const char* path = sArray.item(4);
StringBuffer process;
if (sArray.ordinality() > 5)
{
const char* ClusterType = sArray.item(5);
if (ClusterType && *ClusterType)
{
if (strcmp("THORMACHINES",ClusterType) == 0)
{
process.append("ThorMasterProcess");
}
else if (strcmp("ROXIEMACHINES",ClusterType) == 0)
{
process.append("RoxieServerProcess");
}
}
}
if (strchr(address, '.')) //have an IP address
{
newAddress.clear().append(originalAddress);
return;
}
StringBuffer xpath;
if (!strcmp(compType, "RoxieCluster"))
{
xpath.append("RoxieServer");
}
else if (!strcmp(compType, "ThorCluster"))
{
xpath.append("ThorMaster");
}
else if (!strcmp(compType, "HoleCluster"))
{
xpath.append("HoleControl");
}
else
throw MakeStringException(ECLWATCH_INVALID_COMPONENT_TYPE, "Failed to resolve address for component type '%s'", compType);
Owned<IPropertyTree> pComponent = getComponent(compType, address);
xpath.append("Process[1]/@computer");
const char* computer = pComponent->queryProp(xpath.str());
if (!computer || !*computer)
throw MakeStringException(ECLWATCH_INVALID_COMPONENT_INFO, "Failed to resolve computer for %s '%s'!", compType, address);
Owned<IConstEnvironment> pConstEnv = getConstEnvironment();
Owned<IConstMachineInfo> pConstMc = pConstEnv->getMachine(computer);
SCMStringBuffer sAddress;
pConstMc->getNetAddress(sAddress);
#ifndef OLD_START_STOP
{
StringBuffer sConfigAddress;
sConfigAddress.append(sAddress.str());
if (!strcmp(sAddress.str(), "."))
{
StringBuffer ipStr;
IpAddress ipaddr = queryHostIP();
ipaddr.getIpText(ipStr);
if (ipStr.length() > 0)
{
#ifdef MACHINE_IP
sAddress.set(MACHINE_IP);
#else
sAddress.set(ipStr.str());
#endif
}
}
if (process.length() > 0)
newAddress.clear().appendf("%s|%s:%s:%s:%s:%s", sAddress.str(), sConfigAddress.str(), process.str(), compName, OS, path);
else
newAddress.clear().appendf("%s|%s:%s:%s:%s:%s", sAddress.str(), sConfigAddress.str(), compType, compName, OS, path);
}
#else
if (process.length() > 0)
newAddress.clear().appendf("%s:%s:%s:%s:%s", sAddress.str(), process.str(), compName, OS, path);
else
newAddress.clear().appendf("%s:%s:%s:%s:%s", sAddress.str(), compType, compName, OS, path);
//newAddress.clear().appendf("%s:ThorMasterProcess:%s:%s:%s", sAddress.str(), compName, OS, path);
#endif
return;
}
extern "C" HQL_API IEclRepository * attachLoggingServer(IEclUser * user, IXmlEclRepository & repository, IWorkUnit* workunit, const char * snapshot, bool sandbox4snapshot)
{
SCMStringBuffer cluster;
workunit->getClusterName(cluster);
return new LoggingDataServer(user, repository, cluster.str(), workunit, snapshot, sandbox4snapshot);
}
bool compile(const char *wuid, const char *target, const char *targetCluster)
{
Owned<IConstWUQuery> query = workunit->getQuery();
if (!query)
{
reportError("Workunit does not contain a query", 2);
return false;
}
SCMStringBuffer mainDefinition;
SCMStringBuffer eclQuery;
query->getQueryText(eclQuery);
query->getQueryMainDefinition(mainDefinition);
StringBuffer eclccProgName("eclcc");
StringBuffer eclccCmd(" -shared");
if (eclQuery.length())
eclccCmd.append(" -");
if (mainDefinition.length())
eclccCmd.append(" -main ").append(mainDefinition);
if (workunit->getDebugValueBool("addTimingToWorkunit", true))
eclccCmd.append(" --timings");
Owned<IPipeProcess> pipe = createPipeProcess();
pipe->setenv("ECLCCSERVER_THREAD_INDEX", idxStr.str());
Owned<IPropertyTreeIterator> options = globals->getElements("./Option");
ForEach(*options)
{
IPropertyTree &option = options->query();
const char *name = option.queryProp("@name");
const char *value = option.queryProp("@value");
const char *cluster = option.queryProp("@cluster"); // if cluster is set it's specific to a particular target
if (name && (cluster==NULL || cluster[0]==0 || strcmp(cluster, targetCluster)==0))
processOption(name, value, eclccCmd, eclccProgName, *pipe, false);
}
eclccCmd.appendf(" -o%s", wuid);
eclccCmd.appendf(" -platform=%s", target);
Owned<IStringIterator> debugValues = &workunit->getDebugValues();
ForEach (*debugValues)
{
SCMStringBuffer debugStr, valueStr;
debugValues->str(debugStr);
workunit->getDebugValue(debugStr.str(), valueStr);
processOption(debugStr.str(), valueStr.str(), eclccCmd, eclccProgName, *pipe, true);
}
if (workunit->getResultLimit())
{
eclccCmd.appendf(" -fapplyInstantEclTransformations=1 -fapplyInstantEclTransformationsLimit=%u", workunit->getResultLimit());
}
try
{
unsigned time = msTick();
Owned<ErrorReader> errorReader = new ErrorReader(pipe, this);
eclccCmd.insert(0, eclccProgName);
pipe->run(eclccProgName, eclccCmd, ".", true, false, true, 0);
errorReader->start();
try
{
pipe->write(eclQuery.s.length(), eclQuery.s.str());
pipe->closeInput();
}
catch (IException *e)
{
reportError(e);
e->Release();
}
unsigned retcode = pipe->wait();
errorReader->join();
if (retcode == 0)
{
StringBuffer realdllname, dllurl;
realdllname.append(SharedObjectPrefix).append(wuid).append(SharedObjectExtension);
StringBuffer realdllfilename(dllPath);
realdllfilename.append(SharedObjectPrefix).append(wuid).append(SharedObjectExtension);
StringBuffer wuXML;
if (getWorkunitXMLFromFile(realdllfilename, wuXML))
{
Owned<ILocalWorkUnit> embeddedWU = createLocalWorkUnit();
embeddedWU->loadXML(wuXML);
queryExtendedWU(workunit)->copyWorkUnit(embeddedWU, true);
workunit->setIsClone(false);
SCMStringBuffer jobname;
if (embeddedWU->getJobName(jobname).length()) //let ECL win naming job during initial compile
workunit->setJobName(jobname.str());
Owned<IWUQuery> query = workunit->updateQuery();
query->setQueryText(eclQuery.s.str());
}
createUNCFilename(realdllfilename.str(), dllurl);
unsigned crc = crc_file(realdllfilename.str());
Owned<IWUQuery> query = workunit->updateQuery();
associateLocalFile(query, FileTypeDll, realdllfilename, "Workunit DLL", crc);
queryDllServer().registerDll(realdllname.str(), "Workunit DLL", dllurl.str());
time = msTick()-time;
if (workunit->getDebugValueBool("addTimingToWorkunit", true))
workunit->setTimerInfo("eclccserver: create workunit", NULL, time, 1, 0);
workunit->commit();
return true;
}
}
catch (IException * e)
{
reportError(e);
e->Release();
}
return false;
}
