void cloneFileInfoToDali(StringArray ¬Found, IPropertyTree *packageMap, const char *lookupDaliIp, IConstWUClusterInfo *dstInfo, const char *srcCluster, const char *remotePrefix, bool overWrite, IUserDescriptor* userdesc, bool allowForeignFiles) { StringBuffer user; StringBuffer password; if (userdesc) { userdesc->getUserName(user); userdesc->getPassword(password); } Owned<IReferencedFileList> wufiles = createReferencedFileList(user, password, allowForeignFiles, false); wufiles->addFilesFromPackageMap(packageMap); SCMStringBuffer processName; dstInfo->getRoxieProcess(processName); wufiles->resolveFiles(processName.str(), lookupDaliIp, remotePrefix, srcCluster, !overWrite, false); StringBuffer defReplicateFolder; getConfigurationDirectory(NULL, "data2", "roxie", processName.str(), defReplicateFolder); Owned<IDFUhelper> helper = createIDFUhelper(); wufiles->cloneAllInfo(helper, overWrite, true, false, dstInfo->getRoxieRedundancy(), dstInfo->getChannelsPerNode(), dstInfo->getRoxieReplicateOffset(), defReplicateFolder); Owned<IReferencedFileIterator> iter = wufiles->getFiles(); ForEach(*iter) { IReferencedFile &item = iter->query(); if (item.getFlags() & (RefFileNotFound | RefFileNotOnSource)) notFound.append(item.getLogicalName()); } }
CAci(bool _isDeny, int _perm, ACT_TYPE _act_type, const char* _dn) { m_isDeny = _isDeny; m_targetattr.append("*"); m_version.append("3.0"); Owned<IJlibDateTime> timeNow = createDateTimeNow(); SCMStringBuffer dateString; timeNow->getDateString(dateString); m_name.appendf("%s_%d", dateString.str(), getRandom()); if(_act_type == GROUP_ACT) m_groupdns.append(_dn); else m_userdns.append(_dn); if((_perm & NewSecAccess_Full) == NewSecAccess_Full) { m_perms.append("all"); } else { if((_perm & NewSecAccess_Write) == NewSecAccess_Write) m_perms.append("write"); if((_perm & NewSecAccess_Read) == NewSecAccess_Read) m_perms.append("read"); if((_perm & NewSecAccess_Access) == NewSecAccess_Access) { m_perms.append("compare"); m_perms.append("search"); } } m_permission = _perm; }
bool QueryDataSource::createBrowseWU() { StringAttr dataset, datasetDefs; StringAttrAdaptor a1(dataset), a2(datasetDefs); wuResult->getResultDataset(a1, a2); if (!dataset || !datasetDefs) return false; StringBuffer fullText; fullText.append(datasetDefs).append(dataset); OwnedHqlExpr parsed = parseQuery(fullText.str()); if (!parsed) return false; HqlExprAttr selectFields = parsed.getLink(); if (selectFields->getOperator() == no_output) selectFields.set(selectFields->queryChild(0)); OwnedHqlExpr browseWUcode = buildQueryViewerEcl(selectFields); if (!browseWUcode) return false; returnedRecord.set(browseWUcode->queryChild(0)->queryRecord()); StringAttr tempAttr; StringAttrAdaptor temp(tempAttr); Owned<IWorkUnitFactory> factory = getWorkUnitFactory(); Owned<IConstWorkUnit> parent = factory->openWorkUnit(wuid, false); SCMStringBuffer user; StringAttrAdaptor acluster(cluster); parent->getClusterName(acluster); parent->getUser(user); Owned<IWorkUnit> workunit = factory->createWorkUnit(NULL, "fileViewer", user.str()); workunit->setUser(user.str()); workunit->setClusterName(cluster); workunit->setCustomerId(parent->getCustomerId(temp).str()); workunit->setCompareMode(CompareModeOff); // ? parent->getCompareMode() StringAttrAdaptor bwa(browseWuid); workunit->getWuid(bwa); workunit->setDebugValueInt("importImplicitModules", false, true); workunit->setDebugValueInt("importAllModules", false, true); workunit->setDebugValueInt("forceFakeThor", 1, true); StringBuffer jobName; jobName.append("FileView for ").append(wuid).append(":").append("x"); workunit->setJobName(jobName.str()); StringBuffer eclText; toECL(browseWUcode, eclText, true); Owned<IWUQuery> query = workunit->updateQuery(); query->setQueryText(eclText.str()); query->setQueryName(jobName.str()); return true; }
bool isFileKnownOnCluster(const char *logicalname, IConstWUClusterInfo *clusterInfo, IUserDescriptor* userdesc) { Owned<IDistributedFile> dst = queryDistributedFileDirectory().lookup(logicalname, userdesc, true); if (dst) { SCMStringBuffer processName; clusterInfo->getRoxieProcess(processName); if (dst->findCluster(processName.str()) != NotFound) return true; // file already known for this cluster } return false; }
void updateUsers() { Owned<IStringIterator> clusters = getTargetClusters("RoxieCluster", NULL); ForEach(*clusters) { SCMStringBuffer target; clusters->str(target); Owned<IConstWUClusterInfo> info = getTargetClusterInfo(target.str()); Owned<IUserDescriptor> user = createUserDescriptor(); user->set(info->getLdapUser(), info->getLdapPassword()); roxieUserMap.setValue(target.str(), user); roxieUsers.append(*user.getClear()); } }
bool WuResubmit(const char *wuid) { Owned<IWorkUnitFactory> factory = getWorkUnitFactory(); Owned<IWorkUnit> wu = factory->updateWorkUnit(wuid); if (!wu) { ERRLOG("WuResubmit(%s): could not find workunit",wuid); return false; } if (wu->getState()!=WUStateFailed) { ERRLOG("WuResubmit(%s): could not resubmit as workunit state is '%s'", wuid, wu->queryStateDesc()); return false; } SCMStringBuffer token; wu->getSecurityToken(token); SCMStringBuffer user; SCMStringBuffer password; extractToken(token.str(), wuid, user, password); wu->resetWorkflow(); wu->setState(WUStateSubmitted); wu->commit(); wu.clear(); submitWorkUnit(wuid,user.str(),password.str()); PROGLOG("WuResubmit(%s): resubmitted",wuid); return true; }
void processWorkunit(IConstWorkUnit *workunit, SCMStringBuffer &queryName, IRoxieQueryProcessingInfo &processingInfo, WUQueryActivationOptions activateOption, const char *querySetName, bool notifyRoxie, SCMStringBuffer &status, SCMStringBuffer &roxieDeployStatus) { Owned<IWorkUnit> wu = &workunit->lock(); SCMStringBuffer jobName; wu->getJobName(jobName); if (stricmp(jobName.str(),queryName.str())!=0) wu->setJobName(queryName.str()); try { // look up data file info Owned<IRoxieWuProcessor> wuProcessor = createRoxieWuProcessor(roxieName, queryCommClient, logLevel); wuProcessor->lookupFileNames(wu, processingInfo, status); IPropertyTree *pkgInfo = wuProcessor->queryPackageInfo(); StringBuffer newQueryId; const char *qsName = resolveQuerySetName(querySetName); addQueryToQuerySet(wu, qsName, queryName.str(), pkgInfo, activateOption, newQueryId); const char *queryComment = processingInfo.queryComment(); if (queryComment) setQueryCommentForNamedQuery(qsName, newQueryId.str(), queryComment); } catch(IException *e) { int errCode = e->errorCode(); StringBuffer err; e->errorMessage(err); status.s.appendf("%d %s", errCode, err.str()); DBGLOG("ERROR updating query list %s", status.str()); e->Release(); } if (notifyRoxie) { queryCommClient->sendRoxieReloadControlRequest(); } }
bool WsEclWuInfo::getWsResource(const char *name, StringBuffer &out) { if (strieq(name, "SOAP")) { out.appendf("<message name=\"%s\">", queryname.sget()); IConstWUResultIterator &vars = ensureWorkUnit()->getVariables(); Owned<IResultSetFactory> resultSetFactory(getResultSetFactory(username, password)); ForEach(vars) { IConstWUResult &var = vars.query(); SCMStringBuffer varname; var.getResultName(varname); int seq = var.getResultSequence(); WUResultFormat fmt = var.getResultFormat(); SCMStringBuffer eclschema; var.getResultEclSchema(eclschema); SCMStringBuffer s; Owned<IResultSetMetaData> meta = resultSetFactory->createResultSetMeta(&var); if (!var.isResultScalar()) { meta->getXmlSchema(s, false); out.appendf("<part name=\"%s\" type=\"tns:XmlDataSet\" />", varname.str()); } else { meta->getColumnEclType(s, 0); DisplayType dt = meta->getColumnDisplayType(0); StringAttr ptype; switch (dt) { case TypeBoolean: ptype.set("xsd:boolean"); break; case TypeInteger: ptype.set("xsd:integer"); break; case TypeUnsignedInteger: ptype.set("xsd:integer"); break; case TypeReal: ptype.set("xsd:real"); break; case TypeSet: ptype.set("tns:EspStringArray"); break; case TypeDataset: case TypeData: ptype.set("tns:XmlDataSet"); break; case TypeUnicode: case TypeString: ptype.set("xsd:string"); break; case TypeUnknown: case TypeBeginIfBlock: case TypeEndIfBlock: case TypeBeginRecord: default: ptype.set("xsd:string"); break; } out.appendf("<part name=\"%s\" type=\"%s\" />", varname.str(), ptype.sget()); } } out.append("</message>"); }
int CEclAgentExecutionServer::run() { SCMStringBuffer queueNames; Owned<IFile> sentinelFile = createSentinelTarget(); removeSentinelFile(sentinelFile); try { Owned<IGroup> serverGroup = createIGroup(daliServers, DALI_SERVER_PORT); initClientProcess(serverGroup, DCR_AgentExec); getAgentQueueNames(queueNames, agentName); queue.setown(createJobQueue(queueNames.str())); queue->connect(); } catch (IException *e) { EXCLOG(e, "Server queue create/connect: "); e->Release(); return -1; } catch(...) { ERRLOG("Terminating unexpectedly"); } CSDSServerStatus serverStatus("HThorServer"); serverStatus.queryProperties()->setProp("@queue",queueNames.str()); serverStatus.queryProperties()->setProp("@cluster", agentName); serverStatus.commitProperties(); writeSentinelFile(sentinelFile); try { while (started) { PROGLOG("AgentExec: Waiting on queue(s) '%s'", queueNames.str()); Owned<IJobQueueItem> item = queue->dequeue(); if (item.get()) { StringAttr wuid; wuid.set(item->queryWUID()); PROGLOG("AgentExec: Dequeued workunit request '%s'", wuid.get()); try { executeWorkunit(wuid); } catch(IException *e) { EXCLOG(e, "CEclAgentExecutionServer::run: "); } catch(...) { ERRLOG("Unexpected exception in CEclAgentExecutionServer::run caught"); } } else { ERRLOG("Unexpected dequeue of bogus job queue item, exiting agentexec"); removeSentinelFile(sentinelFile);//no reason to restart assert(!started); break; } } } catch (IException *e) { EXCLOG(e, "Server Exception: "); e->Release(); PROGLOG("Exiting"); } try { queue->disconnect(); } catch (IException *e) { EXCLOG(e, "Server queue disconnect: "); e->Release(); } PROGLOG("Exiting agentexec\n"); return 1; }
bool compile(const char *wuid, const char *target, const char *targetCluster) { Owned<IConstWUQuery> query = workunit->getQuery(); if (!query) { reportError("Workunit does not contain a query", 2); return false; } addTimeStamp(workunit, SSTglobal, NULL, StWhenCompiled); SCMStringBuffer mainDefinition; SCMStringBuffer eclQuery; query->getQueryText(eclQuery); query->getQueryMainDefinition(mainDefinition); StringBuffer eclccProgName; splitDirTail(queryCurrentProcessPath(), eclccProgName); eclccProgName.append("eclcc"); StringBuffer eclccCmd(" -shared"); if (eclQuery.length()) eclccCmd.append(" -"); if (mainDefinition.length()) eclccCmd.append(" -main ").append(mainDefinition); eclccCmd.append(" --timings --xml"); eclccCmd.append(" --nostdinc"); if (globals->getPropBool("@enableEclccDali", true)) { const char *daliServers = globals->queryProp("@daliServers"); if (!daliServers) daliServers = "."; eclccCmd.appendf(" -dfs=%s", daliServers); const char *wuScope = workunit->queryWuScope(); if (!isEmptyString(wuScope)) eclccCmd.appendf(" -scope=%s", wuScope); eclccCmd.appendf(" -cluster=%s", targetCluster); SCMStringBuffer token; workunit->getSecurityToken(token); if (token.length()) eclccCmd.appendf(" -wuid=%s -token=%s", workunit->queryWuid(), token.str()); } Owned<IPipeProcess> pipe = createPipeProcess(); pipe->setenv("ECLCCSERVER_THREAD_INDEX", idxStr.str()); Owned<IPropertyTreeIterator> options = globals->getElements("./Option"); ForEach(*options) { IPropertyTree &option = options->query(); const char *name = option.queryProp("@name"); const char *value = option.queryProp("@value"); const char *cluster = option.queryProp("@cluster"); // if cluster is set it's specific to a particular target if (name && (cluster==NULL || cluster[0]==0 || strcmp(cluster, targetCluster)==0)) processOption(name, value, eclccCmd, eclccProgName, *pipe, false); } eclccCmd.appendf(" -o%s", wuid); eclccCmd.appendf(" -platform=%s", target); eclccCmd.appendf(" --component=%s", queryStatisticsComponentName()); Owned<IStringIterator> debugValues = &workunit->getDebugValues(); ForEach (*debugValues) { SCMStringBuffer debugStr, valueStr; debugValues->str(debugStr); workunit->getDebugValue(debugStr.str(), valueStr); processOption(debugStr.str(), valueStr.str(), eclccCmd, eclccProgName, *pipe, true); } if (workunit->getResultLimit()) { eclccCmd.appendf(" -fapplyInstantEclTransformations=1 -fapplyInstantEclTransformationsLimit=%u", workunit->getResultLimit()); } try { cycle_t startCycles = get_cycles_now(); Owned<ErrorReader> errorReader = new ErrorReader(pipe, this); Owned<AbortWaiter> abortWaiter = new AbortWaiter(pipe, workunit); eclccCmd.insert(0, eclccProgName); if (!pipe->run(eclccProgName, eclccCmd, ".", true, false, true, 0, true)) throw makeStringExceptionV(999, "Failed to run eclcc command %s", eclccCmd.str()); errorReader->start(); abortWaiter->start(); try { pipe->write(eclQuery.s.length(), eclQuery.s.str()); pipe->closeInput(); } catch (IException *e) { reportError(e); e->Release(); } unsigned retcode = pipe->wait(); errorReader->join(); abortWaiter->stop(); if (retcode == 0) { StringBuffer realdllname, dllurl; realdllname.append(SharedObjectPrefix).append(wuid).append(SharedObjectExtension); StringBuffer realdllfilename(dllPath); realdllfilename.append(SharedObjectPrefix).append(wuid).append(SharedObjectExtension); StringBuffer wuXML; if (!getWorkunitXMLFromFile(realdllfilename, wuXML)) throw makeStringException(999, "Failed to extract workunit from query dll"); Owned<ILocalWorkUnit> embeddedWU = createLocalWorkUnit(wuXML); queryExtendedWU(workunit)->copyWorkUnit(embeddedWU, false, true); workunit->setIsClone(false); const char *jobname = embeddedWU->queryJobName(); if (jobname && *jobname) //let ECL win naming job during initial compile workunit->setJobName(jobname); if (!workunit->getDebugValueBool("obfuscateOutput", false)) { Owned<IWUQuery> query = workunit->updateQuery(); query->setQueryText(eclQuery.s.str()); } createUNCFilename(realdllfilename.str(), dllurl); unsigned crc = crc_file(realdllfilename.str()); Owned<IWUQuery> query = workunit->updateQuery(); associateLocalFile(query, FileTypeDll, realdllfilename, "Workunit DLL", crc); queryDllServer().registerDll(realdllname.str(), "Workunit DLL", dllurl.str()); cycle_t elapsedCycles = get_cycles_now() - startCycles; updateWorkunitTimeStat(workunit, SSTcompilestage, "compile", StTimeElapsed, NULL, cycle_to_nanosec(elapsedCycles)); workunit->commit(); return true; } } catch (IException * e) { reportError(e); e->Release(); } return false; }
bool DumpHelper::doit(FILE * fp) { SCMStringBuffer xml; // If we were given a workunit dump that one workunit, otherwise moan if (globals->hasProp("WUID")) { const char* wuid = globals->queryProp("WUID"); const char *whichPath = globals->queryProp("prop"); if (whichPath) { if (stricmp(whichPath, "ecl")==0) { Owned<IClientWUInfoRequest> inforeq = wuclient->createWUInfoRequest(); inforeq->setWuid(wuid); Owned<IClientWUInfoResponse> inforesp = wuclient->WUInfo(inforeq); if(!inforesp) { printf("Workunit %s not found\n", wuid); return false; } IConstECLWorkunit* wu = &inforesp->getWorkunit(); IConstECLQuery* query = &wu->getQuery(); if(query) xml.set(query->getText()); } else { printf("Unrecognized parameter prop=%s", whichPath); return false; } } else { Owned<IClientWULogFileRequest> req = wuclient->createWUFileRequest(); req->setWuid(wuid); req->setType("XML"); Owned<IClientWULogFileResponse> resp = wuclient->WUFile(req); if(!resp) { printf("Workunit %s not found\n", wuid); return false; } const IMultiException* excep = &resp->getExceptions(); if(excep != NULL && excep->ordinality()) { unsigned i = 0; while (i < excep->ordinality()) { StringBuffer msg; excep->item(i).errorMessage(msg); unsigned code = excep->item(i).errorCode(); printf("<Error><code>%d</code><message>%s</message></Error>\n", code, msg.str()); } return false; } const MemoryBuffer & xmlmem = resp->getThefile(); StringBuffer xmlbuf; xmlbuf.append(xmlmem.length(), xmlmem.toByteArray()); xml.set(xmlbuf.str()); } // Print the results if (fp != NULL) { fprintf(fp, "%s", xml.str()); } return true; } return false; }
bool compile(const char *wuid, const char *target, const char *targetCluster) { Owned<IConstWUQuery> query = workunit->getQuery(); if (!query) { reportError("Workunit does not contain a query", 2); return false; } SCMStringBuffer mainDefinition; SCMStringBuffer eclQuery; query->getQueryText(eclQuery); query->getQueryMainDefinition(mainDefinition); StringBuffer eclccCmd("eclcc -shared"); if (eclQuery.length()) eclccCmd.append(" -"); if (mainDefinition.length()) eclccCmd.append(" -main ").append(mainDefinition); Owned<IPropertyTreeIterator> options = globals->getElements("./Option"); ForEach(*options) { IPropertyTree &option = options->query(); const char *name = option.queryProp("@name"); const char *value = option.queryProp("@value"); const char *cluster = option.queryProp("@cluster"); if (name && (cluster==NULL || cluster[0]==0 || strcmp(cluster, targetCluster)==0)) { // options starting '-' are simply passed through to eclcc as name=value // others are passed as -foption=value // if cluster is set it's specific to a particular target eclccCmd.append(" "); if (name[0]!='-') eclccCmd.append("-f"); eclccCmd.append(name); if (value) eclccCmd.append('=').append(value); } } eclccCmd.appendf(" -o%s", wuid); eclccCmd.appendf(" -target=%s", target); Owned<IStringIterator> debugValues = &workunit->getDebugValues(); ForEach (*debugValues) { SCMStringBuffer debugStr, valueStr; debugValues->str(debugStr); workunit->getDebugValue(debugStr.str(), valueStr); if (memicmp(debugStr.str(), "eclcc-", 6) == 0) { //Allow eclcc-xx-<n> so that multiple values can be passed through for the same named debug symbol const char * start = debugStr.str() + 6; const char * dash = strchr(start, '-'); StringAttr optName; if (dash) optName.set(start, dash-start); else optName.set(start); if (stricmp(optName, "compileOption") == 0) eclccCmd.appendf(" -Wc,%s", valueStr.str()); else if (stricmp(optName, "includeLibraryPath") == 0) eclccCmd.appendf(" -I%s", valueStr.str()); else if (stricmp(optName, "libraryPath") == 0) eclccCmd.appendf(" -L%s", valueStr.str()); else eclccCmd.appendf(" -%s=%s", start, valueStr.str()); } else eclccCmd.appendf(" -f%s=%s", debugStr.str(), valueStr.str()); } if (workunit->getResultLimit()) { eclccCmd.appendf(" -fapplyInstantEclTransformations=1 -fapplyInstantEclTransformationsLimit=%u", workunit->getResultLimit()); } try { Owned<IPipeProcess> pipe = createPipeProcess(); Owned<ErrorReader> errorReader = new ErrorReader(pipe, this); pipe->run("eclcc", eclccCmd, ".", true, false, true, 0); errorReader->start(); try { pipe->write(eclQuery.s.length(), eclQuery.s.str()); pipe->closeInput(); } catch (IException *e) { reportError(e); e->Release(); } unsigned retcode = pipe->wait(); errorReader->join(); if (retcode == 0) { StringBuffer realdllname, dllurl; realdllname.append(SharedObjectPrefix).append(wuid).append(SharedObjectExtension); StringBuffer realdllfilename(dllPath); realdllfilename.append(SharedObjectPrefix).append(wuid).append(SharedObjectExtension); StringBuffer wuXML; if (getWorkunitXMLFromFile(realdllfilename, wuXML)) { Owned<ILocalWorkUnit> embeddedWU = createLocalWorkUnit(); embeddedWU->loadXML(wuXML); queryExtendedWU(workunit)->copyWorkUnit(embeddedWU); Owned<IWUQuery> query = workunit->updateQuery(); query->setQueryText(eclQuery.s.str()); } createUNCFilename(realdllfilename.str(), dllurl); unsigned crc = crc_file(realdllfilename.str()); Owned<IWUQuery> query = workunit->updateQuery(); associateLocalFile(query, FileTypeDll, realdllfilename, "Workunit DLL", crc); queryDllServer().registerDll(realdllname.str(), "Workunit DLL", dllurl.str()); workunit->commit(); return true; } } catch (IException * e) { reportError(e); e->Release(); } return false; }
void appendVariableParmInfo(IArrayOf<IPropertyTree> &parts, IResultSetFactory *resultSetFactory, IConstWUResult &var, unsigned hashWebserviceSeq=0) { Owned<IResultSetMetaData> meta = resultSetFactory->createResultSetMeta(&var); StringAttr noinput; if (var.getResultFieldOpt("noinput", StringAttrAdaptor(noinput)).length() && strToBool(noinput.length(), noinput.get())) //developer specified not to show field on form return; SCMStringBuffer varname; var.getResultName(varname); int seq = var.getResultSequence(); WUResultFormat fmt = var.getResultFormat(); SCMStringBuffer eclschema; var.getResultEclSchema(eclschema); StringBuffer width, height, fieldSeq, isPassword, select; var.getResultFieldOpt("fieldwidth", StringBufferAdaptor(width)); var.getResultFieldOpt("fieldheight", StringBufferAdaptor(height)); var.getResultFieldOpt("password", StringBufferAdaptor(isPassword)); var.getResultFieldOpt("select", StringBufferAdaptor(select)); if (hashWebserviceSeq) fieldSeq.append(hashWebserviceSeq); else var.getResultFieldOpt("sequence", StringBufferAdaptor(fieldSeq)); SCMStringBuffer s; Owned<IPropertyTree> part = createPTree("part"); if (!var.isResultScalar()) { meta->getXmlSchema(s, false); part->setProp("@name", varname.str()); part->setProp("@type", "tns:XmlDataset"); if (fieldSeq.length()) part->setProp("@sequence", fieldSeq); } else { meta->getColumnEclType(s, 0); DisplayType dt = meta->getColumnDisplayType(0); StringAttr ptype; switch (dt) { case TypeBoolean: ptype.set("xsd:boolean"); break; case TypeInteger: ptype.set("xsd:integer"); break; case TypeUnsignedInteger: ptype.set("xsd:integer"); break; case TypeReal: ptype.set("xsd:real"); break; case TypeSet: ptype.set("tns:EspStringArray"); break; case TypeDataset: case TypeData: ptype.set("tns:XmlDataSet"); break; case TypeUnicode: case TypeString: ptype.set("xsd:string"); break; case TypeUnknown: case TypeBeginIfBlock: case TypeEndIfBlock: case TypeBeginRecord: default: ptype.set("xsd:string"); break; } part->setProp("@name", varname.str()); part->setProp("@type", ptype.str()); if (width.length()) part->setProp("@width", width); if (height.length()) part->setProp("@height", height); if (fieldSeq.length()) part->setProp("@sequence", fieldSeq); if (isPassword.length()) part->setProp("@password", isPassword); if (select.length()) { StringArray optionList; optionList.appendList(select, ","); IPropertyTree *selectTree = ensurePTree(part, "select"); ForEachItemIn(i, optionList) { const char *value = optionList.item(i); bool selected = '*'==*value; if (selected) value++; StringAttr name; const char *eq = strchr(value, '='); if (!eq) name.set(value); else { name.set(value, eq-value); value = eq + 1; } Owned<IPropertyTree> optionTree = createPTree(); optionTree->setProp("@name", name); optionTree->setProp("@value", value); if (selected) optionTree->setPropBool("@selected", true); selectTree->addPropTree("option", optionTree.getClear()); } } }
bool CEclDirectEx::onRunEcl(IEspContext &context, IEspRunEclRequest & req, IEspRunEclResponse & resp) { if (!context.validateFeatureAccess(ECLDIRECT_ACCESS, SecAccess_Full, false)) throw MakeStringException(-1, "EclDirect access permission denied."); StringBuffer user; if (!context.getUserID(user).length()) user.append(req.getUserName()); Owned <IWorkUnitFactory> factory = getWorkUnitFactory(context.querySecManager(), context.queryUser()); Owned <IWorkUnit> workunit; if (!user.length()) workunit.setown(factory->createWorkUnit(NULL, "ECL-Direct", "")); else { workunit.setown(factory->createWorkUnit(NULL, "ECL-Direct", user.str())); workunit->setUser(user.str()); } Owned<IWUQuery> query = workunit->updateQuery(); query->setQueryText(req.getEclText()); query.clear(); const char* clustername = req.getCluster(); if (!clustername || !*clustername || strieq(clustername, "default")) clustername = defaultCluster.str(); if (!clustername || !*clustername) throw MakeStringException(-1, "No Cluster Specified"); if (!isValidCluster(clustername)) throw MakeStringException(-1, "Invalid TargetCluster %s Specified", clustername); workunit->setClusterName(clustername); if (req.getLimitResults()) workunit->setResultLimit(100); const char* snapshot = req.getSnapshot(); if (snapshot && *snapshot) workunit->setSnapshot(snapshot); // Execute it SCMStringBuffer wuid; workunit->getWuid(wuid); workunit->setAction(WUActionRun); workunit->setState(WUStateSubmitted); workunit.clear(); submitWorkUnit(wuid.str(), context.querySecManager(), context.queryUser()); if (waitForWorkUnitToComplete(wuid.str(), defaultWait)) { Owned<IConstWorkUnit> cw = factory->openWorkUnit(wuid.str(), false); SCMStringBuffer resultXML; getFullWorkUnitResultsXML(context.queryUserId(), context.queryPassword(), cw.get(), resultXML); resp.setResults(resultXML.str()); cw.clear(); if (deleteWorkunits) deleteEclDirectWorkunit(factory, wuid.str()); } else { // Don't delete these ones... DBGLOG("WorkUnit %s timed out", wuid.str()); StringBuffer result; result.appendf("<Exception><Source>ESP</Source><Message>Timed out waiting for job to complete: %s</Message></Exception>", wuid.str()); resp.setResults(result.str()); } return true; }
virtual void doWork() { if (m_useHPCCInit) { //address specified can be either IP or name of component const char* address = m_sAddress.str(); const char* configAddress = m_sConfigAddress.str(); if (!address || !*address) throw MakeStringException(ECLWATCH_INVALID_IP_OR_COMPONENT, "Invalid address or component name was specified!"); if (!strchr(address, '.')) //not an IP address { const char* compType = m_pResult->getCompType(); const char* compName = address; if (!compType || !*compType) throw MakeStringException(ECLWATCH_INVALID_COMPONENT_TYPE, "No component type specified!"); StringBuffer xpath; if (!strcmp(compType, "RoxieCluster")) { xpath.append("RoxieServer"); } else if (!strcmp(compType, "ThorCluster")) { xpath.append("ThorMaster"); } else if (!strcmp(compType, "HoleCluster")) { xpath.append("HoleControl"); } else throw MakeStringException(ECLWATCH_INVALID_COMPONENT_TYPE, "Failed to resolve component type '%s'", compType); Owned<IPropertyTree> pComponent = m_pService->getComponent(compType, compName); xpath.append("Process[1]/@computer"); const char* computer = pComponent->queryProp(xpath.str()); if (!computer || !*computer) throw MakeStringException(ECLWATCH_INVALID_COMPONENT_INFO, "Failed to resolve computer for %s '%s'!", compType, compName); Owned<IConstEnvironment> pConstEnv = m_pService->getConstEnvironment(); Owned<IConstMachineInfo> pConstMc = pConstEnv->getMachine(computer); SCMStringBuffer sAddress; pConstMc->getNetAddress(sAddress); if (!stricmp(m_sAddress.str(), m_sConfigAddress.str())) { m_sAddress.clear().append(sAddress.str()); m_sConfigAddress = m_sAddress; m_pResult->setAddress( sAddress.str() ); } else { m_sAddress.clear().append(sAddress.str()); m_pResult->setAddress( sAddress.str() ); if (configAddress && !strchr(configAddress, '.')) //not an IP address { Owned<IPropertyTree> pComponent = m_pService->getComponent(compType, configAddress); xpath.append("Process[1]/@computer"); const char* computer = pComponent->queryProp(xpath.str()); if (!computer || !*computer) throw MakeStringException(ECLWATCH_INVALID_COMPONENT_INFO, "Failed to resolve computer for %s '%s'!", compType, configAddress); Owned<IConstEnvironment> pConstEnv = m_pService->getConstEnvironment(); Owned<IConstMachineInfo> pConstMc = pConstEnv->getMachine(computer); SCMStringBuffer sAddress; pConstMc->getNetAddress(sAddress); m_sConfigAddress.clear().append(sAddress.str()); } } } if ((m_sAddress.length() > 0) && !stricmp(m_sAddress.str(), ".")) { StringBuffer ipStr; IpAddress ipaddr = queryHostIP(); ipaddr.getIpText(ipStr); if (ipStr.length() > 0) { #ifdef MACHINE_IP m_sAddress.clear().append(MACHINE_IP); #else m_sAddress.clear().append(ipStr.str()); #endif m_pResult->setAddress( m_sAddress.str() ); } } #ifdef OLD_START_STOP int OS = m_pResult->getOS(); StringBuffer sPath( m_pResult->getPath() ); if (OS == 0) sPath.replace('$', ':'); else if (sPath.charAt(0) != '/') sPath.insert(0, '/'); m_sCommand.clear().append(sPath).append( OS==0 ? '\\' : '/'); m_sCommand.append(m_bStop ? "stop" : "startup"); if (OS == 0) m_sCommand.append(".bat"); m_sCommand.append(' '); m_sCommand.append(sPath); #else StringBuffer sPath( m_pResult->getPath() ); if (sPath.charAt(sPath.length() - 1) == '/') sPath.setLength(sPath.length() - 1); if (sPath.length() > 0) { char* pStr = (char*) sPath.str(); char* ppStr = strchr(pStr, '/'); while (ppStr) { ppStr++; pStr = ppStr; ppStr = strchr(pStr, '/'); } if (!m_bStop) m_sCommand.appendf("sudo /etc/init.d/hpcc-init -c %s start", pStr); else m_sCommand.appendf("sudo /etc/init.d/hpcc-init -c %s stop", pStr); } #endif m_pResult->setCommand( m_sCommand.str() ); } else { //address specified can be either IP or name of component const char* address = m_sAddress.str(); if (!address || !*address) throw MakeStringException(ECLWATCH_INVALID_IP_OR_COMPONENT, "Invalid address or component name was specified!"); if (!strchr(address, '.')) //not an IP address { const char* compType = m_pResult->getCompType(); const char* compName = address; if (!compType || !*compType) throw MakeStringException(ECLWATCH_INVALID_COMPONENT_TYPE, "No component type specified!"); StringBuffer xpath; if (!strcmp(compType, "RoxieCluster")) { xpath.append("RoxieServer"); } else if (!strcmp(compType, "ThorCluster")) { xpath.append("ThorMaster"); } else if (!strcmp(compType, "HoleCluster")) { xpath.append("HoleControl"); } else throw MakeStringException(ECLWATCH_INVALID_COMPONENT_TYPE, "Failed to resolve component type '%s'", compType); Owned<IPropertyTree> pComponent = m_pService->getComponent(compType, compName); xpath.append("Process[1]/@computer"); const char* computer = pComponent->queryProp(xpath.str()); if (!computer || !*computer) throw MakeStringException(ECLWATCH_INVALID_COMPONENT_INFO, "Failed to resolve computer for %s '%s'!", compType, compName); Owned<IConstEnvironment> pConstEnv = m_pService->getConstEnvironment(); Owned<IConstMachineInfo> pConstMc = pConstEnv->getMachine(computer); SCMStringBuffer sAddress; pConstMc->getNetAddress(sAddress); m_sAddress.clear().append(sAddress.str()); m_pResult->setAddress( sAddress.str() ); } int OS = m_pResult->getOS(); StringBuffer sPath( m_pResult->getPath() ); if (OS == 0) sPath.replace('$', ':'); else if (sPath.charAt(0) != '/') sPath.insert(0, '/'); m_sCommand.clear().append(sPath).append( OS==0 ? '\\' : '/'); m_sCommand.append(m_bStop ? "stop" : "startup"); if (OS == 0) m_sCommand.append(".bat"); m_sCommand.append(' '); m_sCommand.append(sPath); m_pResult->setCommand( m_sCommand.str() ); } CRemoteExecThreadParam::doWork(); }
bool CEclDirectEx::onRunEclEx(IEspContext &context, IEspRunEclExRequest & req, IEspRunEclExResponse & resp) { if (!context.validateFeatureAccess(ECLDIRECT_ACCESS, SecAccess_Full, false)) throw MakeStringException(-1, "EclDirect access permission denied."); const char* eclText = req.getEclText(); if (!eclText || !*eclText) { resp.setResults("<Exception><Source>ESP</Source><Message>No Ecl Text provided</Message></Exception>"); return true; } StringBuffer user; if (!context.getUserID(user).length()) user.append(req.getUserName()); Owned <IWorkUnitFactory> factory = getWorkUnitFactory(context.querySecManager(), context.queryUser()); Owned <IWorkUnit> workunit; if (!user.length()) workunit.setown(factory->createWorkUnit(NULL, "ECL-Direct", "")); else { workunit.setown(factory->createWorkUnit(NULL, "ECL-Direct", user.str())); workunit->setUser(user.str()); } Owned<IWUQuery> query = workunit->updateQuery(); query->setQueryText(eclText); query.clear(); const char* cluster = req.getCluster(); if (!cluster || !*cluster || !stricmp(cluster, "default")) cluster = defaultCluster.str(); if (!cluster || !*cluster) throw MakeStringException(-1, "No Cluster Specified"); if (!isValidCluster(cluster)) throw MakeStringException(-1, "Invalid TargetCluster %s Specified", cluster); workunit->setClusterName(cluster); const char* snapshot = req.getSnapshot(); if (snapshot && *snapshot) workunit->setSnapshot(snapshot); if (req.getResultLimit()) workunit->setResultLimit(req.getResultLimit()); // Execute it SCMStringBuffer wuid; workunit->getWuid(wuid); workunit->setAction(WUActionRun); workunit->setState(WUStateSubmitted); workunit.clear(); resp.setWuid(wuid.str()); submitWorkUnit(wuid.str(), context.querySecManager(), context.queryUser()); if (!waitForWorkUnitToComplete(wuid.str(), (req.getWait_isNull()) ? defaultWait : req.getWait())) { StringBuffer result; result.appendf("<Exception><Source>ESP</Source><Message>Timed out waiting for job to complete: %s</Message></Exception>", wuid.str()); resp.setResults(result.str()); return true; } if (!deleteWorkunits && context.queryRequestParameters()->hasProp("redirect")) { StringBuffer url("/WsWorkunits/WUInfo?Wuid="); resp.setRedirectUrl(url.append(wuid).str()); return true; } Owned<IConstWorkUnit> cw = factory->openWorkUnit(wuid.str(), false); EclDirectWUExceptions errors(*cw); resp.setErrors(errors); if (req.getIncludeResults()) { StringBuffer results; CRunEclExFormat outputFormat = req.getFormat(); Owned<IWuWebView> web = createWuWebView(wuid.str(), NULL, NULL, getCFD(), true); if (!web) results.appendf("<Exception><Source>ESP</Source><Message>Failed loading result workunit %s</Message></Exception>", wuid.str()); else if (outputFormat == CRunEclExFormat_Table) { StringBuffer xsltfile(getCFD()); web->applyResultsXSLT(xsltfile.append("xslt/wsecl3_result.xslt").str(), results); } else { unsigned xmlflags = 0; if (outputFormat != CRunEclExFormat_ExtendedXml) xmlflags |= WWV_OMIT_SCHEMAS; if (context.queryRequestParameters()->hasProp("display_xslt")) xmlflags |= WWV_USE_DISPLAY_XSLT; else xmlflags |= WWV_OMIT_XML_DECLARATION; web->expandResults(results, xmlflags); } resp.setResults(results.str()); } if (req.getIncludeGraphs()) { Owned<IConstWUGraphIterator> it = &cw->getGraphs(GraphTypeAny); StringBuffer xgmml("<Graphs>"); SCMStringBuffer s; ForEach(*it) xgmml.append(it->query().getXGMML(s, true).str()); xgmml.append("</Graphs>"); resp.setGraphsXGMML(xgmml.str()); } if (deleteWorkunits) deleteEclDirectWorkunit(factory, wuid.str()); return true; }
void Cws_machineEx::ConvertAddress( const char* originalAddress, StringBuffer& newAddress) { if (!originalAddress || !*originalAddress) throw MakeStringException(ECLWATCH_INVALID_IP_OR_COMPONENT, "No network address or computer name specified!"); StringArray sArray; sArray.appendList(originalAddress, ":"); if (sArray.ordinality() < 4) throw MakeStringException(ECLWATCH_MISSING_PARAMS, "Incomplete arguments"); const char* address = sArray.item(0); const char* compType= sArray.item(1); const char* compName= sArray.item(2); const char* OS = sArray.item(3); const char* path = sArray.item(4); StringBuffer process; if (sArray.ordinality() > 5) { const char* ClusterType = sArray.item(5); if (ClusterType && *ClusterType) { if (strcmp("THORMACHINES",ClusterType) == 0) { process.append("ThorMasterProcess"); } else if (strcmp("ROXIEMACHINES",ClusterType) == 0) { process.append("RoxieServerProcess"); } } } if (strchr(address, '.')) //have an IP address { newAddress.clear().append(originalAddress); return; } StringBuffer xpath; if (!strcmp(compType, "RoxieCluster")) { xpath.append("RoxieServer"); } else if (!strcmp(compType, "ThorCluster")) { xpath.append("ThorMaster"); } else if (!strcmp(compType, "HoleCluster")) { xpath.append("HoleControl"); } else throw MakeStringException(ECLWATCH_INVALID_COMPONENT_TYPE, "Failed to resolve address for component type '%s'", compType); Owned<IPropertyTree> pComponent = getComponent(compType, address); xpath.append("Process[1]/@computer"); const char* computer = pComponent->queryProp(xpath.str()); if (!computer || !*computer) throw MakeStringException(ECLWATCH_INVALID_COMPONENT_INFO, "Failed to resolve computer for %s '%s'!", compType, address); Owned<IConstEnvironment> pConstEnv = getConstEnvironment(); Owned<IConstMachineInfo> pConstMc = pConstEnv->getMachine(computer); SCMStringBuffer sAddress; pConstMc->getNetAddress(sAddress); #ifndef OLD_START_STOP { StringBuffer sConfigAddress; sConfigAddress.append(sAddress.str()); if (!strcmp(sAddress.str(), ".")) { StringBuffer ipStr; IpAddress ipaddr = queryHostIP(); ipaddr.getIpText(ipStr); if (ipStr.length() > 0) { #ifdef MACHINE_IP sAddress.set(MACHINE_IP); #else sAddress.set(ipStr.str()); #endif } } if (process.length() > 0) newAddress.clear().appendf("%s|%s:%s:%s:%s:%s", sAddress.str(), sConfigAddress.str(), process.str(), compName, OS, path); else newAddress.clear().appendf("%s|%s:%s:%s:%s:%s", sAddress.str(), sConfigAddress.str(), compType, compName, OS, path); } #else if (process.length() > 0) newAddress.clear().appendf("%s:%s:%s:%s:%s", sAddress.str(), process.str(), compName, OS, path); else newAddress.clear().appendf("%s:%s:%s:%s:%s", sAddress.str(), compType, compName, OS, path); //newAddress.clear().appendf("%s:ThorMasterProcess:%s:%s:%s", sAddress.str(), compName, OS, path); #endif return; }
extern "C" HQL_API IEclRepository * attachLoggingServer(IEclUser * user, IXmlEclRepository & repository, IWorkUnit* workunit, const char * snapshot, bool sandbox4snapshot) { SCMStringBuffer cluster; workunit->getClusterName(cluster); return new LoggingDataServer(user, repository, cluster.str(), workunit, snapshot, sandbox4snapshot); }
bool compile(const char *wuid, const char *target, const char *targetCluster) { Owned<IConstWUQuery> query = workunit->getQuery(); if (!query) { reportError("Workunit does not contain a query", 2); return false; } SCMStringBuffer mainDefinition; SCMStringBuffer eclQuery; query->getQueryText(eclQuery); query->getQueryMainDefinition(mainDefinition); StringBuffer eclccProgName("eclcc"); StringBuffer eclccCmd(" -shared"); if (eclQuery.length()) eclccCmd.append(" -"); if (mainDefinition.length()) eclccCmd.append(" -main ").append(mainDefinition); if (workunit->getDebugValueBool("addTimingToWorkunit", true)) eclccCmd.append(" --timings"); Owned<IPipeProcess> pipe = createPipeProcess(); pipe->setenv("ECLCCSERVER_THREAD_INDEX", idxStr.str()); Owned<IPropertyTreeIterator> options = globals->getElements("./Option"); ForEach(*options) { IPropertyTree &option = options->query(); const char *name = option.queryProp("@name"); const char *value = option.queryProp("@value"); const char *cluster = option.queryProp("@cluster"); // if cluster is set it's specific to a particular target if (name && (cluster==NULL || cluster[0]==0 || strcmp(cluster, targetCluster)==0)) processOption(name, value, eclccCmd, eclccProgName, *pipe, false); } eclccCmd.appendf(" -o%s", wuid); eclccCmd.appendf(" -platform=%s", target); Owned<IStringIterator> debugValues = &workunit->getDebugValues(); ForEach (*debugValues) { SCMStringBuffer debugStr, valueStr; debugValues->str(debugStr); workunit->getDebugValue(debugStr.str(), valueStr); processOption(debugStr.str(), valueStr.str(), eclccCmd, eclccProgName, *pipe, true); } if (workunit->getResultLimit()) { eclccCmd.appendf(" -fapplyInstantEclTransformations=1 -fapplyInstantEclTransformationsLimit=%u", workunit->getResultLimit()); } try { unsigned time = msTick(); Owned<ErrorReader> errorReader = new ErrorReader(pipe, this); eclccCmd.insert(0, eclccProgName); pipe->run(eclccProgName, eclccCmd, ".", true, false, true, 0); errorReader->start(); try { pipe->write(eclQuery.s.length(), eclQuery.s.str()); pipe->closeInput(); } catch (IException *e) { reportError(e); e->Release(); } unsigned retcode = pipe->wait(); errorReader->join(); if (retcode == 0) { StringBuffer realdllname, dllurl; realdllname.append(SharedObjectPrefix).append(wuid).append(SharedObjectExtension); StringBuffer realdllfilename(dllPath); realdllfilename.append(SharedObjectPrefix).append(wuid).append(SharedObjectExtension); StringBuffer wuXML; if (getWorkunitXMLFromFile(realdllfilename, wuXML)) { Owned<ILocalWorkUnit> embeddedWU = createLocalWorkUnit(); embeddedWU->loadXML(wuXML); queryExtendedWU(workunit)->copyWorkUnit(embeddedWU, true); workunit->setIsClone(false); SCMStringBuffer jobname; if (embeddedWU->getJobName(jobname).length()) //let ECL win naming job during initial compile workunit->setJobName(jobname.str()); Owned<IWUQuery> query = workunit->updateQuery(); query->setQueryText(eclQuery.s.str()); } createUNCFilename(realdllfilename.str(), dllurl); unsigned crc = crc_file(realdllfilename.str()); Owned<IWUQuery> query = workunit->updateQuery(); associateLocalFile(query, FileTypeDll, realdllfilename, "Workunit DLL", crc); queryDllServer().registerDll(realdllname.str(), "Workunit DLL", dllurl.str()); time = msTick()-time; if (workunit->getDebugValueBool("addTimingToWorkunit", true)) workunit->setTimerInfo("eclccserver: create workunit", NULL, time, 1, 0); workunit->commit(); return true; } } catch (IException * e) { reportError(e); e->Release(); } return false; }