bool ResourceManifest::loadInclude(IPropertyTree &include, const char *dir)
{
const char *filename = include.queryProp("@filename");
StringBuffer includePath;
makeAbsolutePath(filename, dir, includePath);
VStringBuffer xpath("Include[@originalFilename='%s']", includePath.str());
if (manifest->hasProp(xpath.str()))
return false;
include.setProp("@originalFilename", includePath.str());
StringBuffer includeDir;
splitDirTail(includePath, includeDir);
Owned<IPropertyTree> manifestInclude = createPTreeFromXMLFile(includePath.str());
Owned<IPropertyTreeIterator> it = manifestInclude->getElements("*");
ForEach(*it)
{
IPropertyTree &item = it->query();
if (streq(item.queryName(), "Resource"))
updateResourcePaths(item, includeDir.str());
else if (streq(item.queryName(), "Include"))
{
if (!loadInclude(item, includeDir.str()))
continue;
}
manifest->addPropTree(item.queryName(), LINK(&item));
}
return true;
}
ResourceManifest(const char *filename)
: manifest(createPTreeFromXMLFile(filename))
{
makeAbsolutePath(filename, absFilename);
splitDirTail(absFilename, dir);
expand();
}
void ResourceManager::addManifestFromArchive(IPropertyTree *archive)
{
if (!archive)
return;
if (finalized)
throwError1(HQLERR_ResourceAddAfterFinalManifest, "MANIFEST");
ensureManifestInfo();
Owned<IPropertyTreeIterator> manifests = archive->getElements("AdditionalFiles/Manifest");
ForEach(*manifests)
{
const char *xml = manifests->query().queryProp(NULL);
Owned<IPropertyTree> manifestSrc = createPTreeFromXMLString(xml);
Owned<IAttributeIterator> aiter = manifestSrc->getAttributes();
ForEach (*aiter)
manifest->setProp(aiter->queryName(), aiter->queryValue());
StringBuffer manifestDir;
if (manifestSrc->hasProp("@originalFilename"))
splitDirTail(manifestSrc->queryProp("@originalFilename"), manifestDir);
Owned<IPropertyTreeIterator> iter = manifestSrc->getElements("*");
ForEach(*iter)
{
IPropertyTree &item = iter->query();
if (streq(item.queryName(), "Resource") && item.hasProp("@filename"))
{
if (!item.hasProp("@type"))
item.setProp("@type", "UNKNOWN");
const char *filename;
if (item.hasProp("@originalFilename"))
filename = item.queryProp("@originalFilename");
else
filename = item.queryProp("@filename");
int id;
if (getDuplicateResourceId(item.queryProp("@type"), NULL, filename, id))
{
item.setPropInt("@id", (int)id);
manifest->addPropTree("Resource", LINK(&item));
}
else
{
VStringBuffer xpath("AdditionalFiles/Resource[@originalFilename=\"%s\"]", filename);
MemoryBuffer content;
archive->getPropBin(xpath.str(), content);
addCompress(item.queryProp("@type"), content.length(), content.toByteArray(), &item);
}
}
else
manifest->addPropTree(item.queryName(), LINK(&item));
}
}
}
bool addFilename(StringAttrArray &names,const char *name,bool sub)
{
if (isWild(name)) {
StringBuffer dir;
const char *tail = splitDirTail(name,dir);
if (isWild(dir.str())) {
printf("Directory %s - cannot be wild!\n",dir.str());
return false;
}
Owned<IFile> dirf = createIFile(dir.str());
Owned<IDirectoryIterator> iter = dirf->directoryFiles(tail,sub,false);
StringBuffer subname;
ForEach(*iter) {
subname.clear().append(iter->query().queryFilename());
names.append(*new StringAttrItem(subname.str()));
}
}
else
void ResourceManager::addManifest(const char *filename)
{
if (finalized)
throwError1(HQLERR_ResourceAddAfterFinalManifest, "MANIFEST");
Owned<IPropertyTree> manifestSrc = createPTreeFromXMLFile(filename);
StringBuffer dir;
splitDirTail(filename, dir);
ensureManifestInfo();
Owned<IAttributeIterator> aiter = manifestSrc->getAttributes();
ForEach (*aiter)
manifest->setProp(aiter->queryName(), aiter->queryValue());
Owned<IPropertyTreeIterator> iter = manifestSrc->getElements("*");
ForEach(*iter)
{
IPropertyTree &item = iter->query();
if (streq(item.queryName(), "Resource") && item.hasProp("@filename"))
{
if (!item.hasProp("@type"))
item.setProp("@type", "UNKNOWN");
const char *filename = item.queryProp("@filename");
int id;
if (getDuplicateResourceId(item.queryProp("@type"), filename, id))
{
item.setPropInt("@id", id);
manifest->addPropTree("Resource", LINK(&item));
}
else
{
StringBuffer fullpath;
if (!isAbsolutePath(filename))
fullpath.append(dir);
fullpath.append(filename);
MemoryBuffer content;
loadResource(fullpath.str(), content);
addCompress(item.queryProp("@type"), content.length(), content.toByteArray(), &item);
}
}
else
manifest->addPropTree(item.queryName(), LINK(&item));
}
}
void ResourceManifest::expand()
{
manifest->setProp("@manifestDir", dir.str());
Owned<IPropertyTreeIterator> resources = manifest->getElements("Resource[@filename]");
ForEach(*resources)
updateResourcePaths(resources->query(), dir.str());
Owned<IPropertyTreeIterator> includes = manifest->getElements("Include[@filename]");
ForEach(*includes)
loadInclude(includes->query(), dir.str());
resources.setown(manifest->getElements("Resource[@filename]"));
ForEach(*resources)
{
IPropertyTree &res = resources->query();
const char *name = res.queryProp("@originalFilename");
if (containsFileWildcard(name))
{
StringBuffer wildpath;
const char *tail = splitDirTail(name, wildpath);
expandDirectory(res, wildpath, tail, res.getPropBool("@recursive"));
manifest->removeTree(&res);
}
}
}
bool compile(const char *wuid, const char *target, const char *targetCluster)
{
Owned<IConstWUQuery> query = workunit->getQuery();
if (!query)
{
reportError("Workunit does not contain a query", 2);
return false;
}
addTimeStamp(workunit, SSTglobal, NULL, StWhenCompiled);
SCMStringBuffer mainDefinition;
SCMStringBuffer eclQuery;
query->getQueryText(eclQuery);
query->getQueryMainDefinition(mainDefinition);
StringBuffer eclccProgName;
splitDirTail(queryCurrentProcessPath(), eclccProgName);
eclccProgName.append("eclcc");
StringBuffer eclccCmd(" -shared");
if (eclQuery.length())
eclccCmd.append(" -");
if (mainDefinition.length())
eclccCmd.append(" -main ").append(mainDefinition);
eclccCmd.append(" --timings --xml");
eclccCmd.append(" --nostdinc");
if (globals->getPropBool("@enableEclccDali", true))
{
const char *daliServers = globals->queryProp("@daliServers");
if (!daliServers)
daliServers = ".";
eclccCmd.appendf(" -dfs=%s", daliServers);
const char *wuScope = workunit->queryWuScope();
if (!isEmptyString(wuScope))
eclccCmd.appendf(" -scope=%s", wuScope);
eclccCmd.appendf(" -cluster=%s", targetCluster);
SCMStringBuffer token;
workunit->getSecurityToken(token);
if (token.length())
eclccCmd.appendf(" -wuid=%s -token=%s", workunit->queryWuid(), token.str());
}
Owned<IPipeProcess> pipe = createPipeProcess();
pipe->setenv("ECLCCSERVER_THREAD_INDEX", idxStr.str());
Owned<IPropertyTreeIterator> options = globals->getElements("./Option");
ForEach(*options)
{
IPropertyTree &option = options->query();
const char *name = option.queryProp("@name");
const char *value = option.queryProp("@value");
const char *cluster = option.queryProp("@cluster"); // if cluster is set it's specific to a particular target
if (name && (cluster==NULL || cluster[0]==0 || strcmp(cluster, targetCluster)==0))
processOption(name, value, eclccCmd, eclccProgName, *pipe, false);
}
eclccCmd.appendf(" -o%s", wuid);
eclccCmd.appendf(" -platform=%s", target);
eclccCmd.appendf(" --component=%s", queryStatisticsComponentName());
Owned<IStringIterator> debugValues = &workunit->getDebugValues();
ForEach (*debugValues)
{
SCMStringBuffer debugStr, valueStr;
debugValues->str(debugStr);
workunit->getDebugValue(debugStr.str(), valueStr);
processOption(debugStr.str(), valueStr.str(), eclccCmd, eclccProgName, *pipe, true);
}
if (workunit->getResultLimit())
{
eclccCmd.appendf(" -fapplyInstantEclTransformations=1 -fapplyInstantEclTransformationsLimit=%u", workunit->getResultLimit());
}
try
{
cycle_t startCycles = get_cycles_now();
Owned<ErrorReader> errorReader = new ErrorReader(pipe, this);
Owned<AbortWaiter> abortWaiter = new AbortWaiter(pipe, workunit);
eclccCmd.insert(0, eclccProgName);
if (!pipe->run(eclccProgName, eclccCmd, ".", true, false, true, 0, true))
throw makeStringExceptionV(999, "Failed to run eclcc command %s", eclccCmd.str());
errorReader->start();
abortWaiter->start();
try
{
pipe->write(eclQuery.s.length(), eclQuery.s.str());
pipe->closeInput();
}
catch (IException *e)
{
reportError(e);
e->Release();
}
unsigned retcode = pipe->wait();
errorReader->join();
abortWaiter->stop();
if (retcode == 0)
{
StringBuffer realdllname, dllurl;
realdllname.append(SharedObjectPrefix).append(wuid).append(SharedObjectExtension);
StringBuffer realdllfilename(dllPath);
realdllfilename.append(SharedObjectPrefix).append(wuid).append(SharedObjectExtension);
StringBuffer wuXML;
if (!getWorkunitXMLFromFile(realdllfilename, wuXML))
throw makeStringException(999, "Failed to extract workunit from query dll");
Owned<ILocalWorkUnit> embeddedWU = createLocalWorkUnit(wuXML);
queryExtendedWU(workunit)->copyWorkUnit(embeddedWU, false, true);
workunit->setIsClone(false);
const char *jobname = embeddedWU->queryJobName();
if (jobname && *jobname) //let ECL win naming job during initial compile
workunit->setJobName(jobname);
if (!workunit->getDebugValueBool("obfuscateOutput", false))
{
Owned<IWUQuery> query = workunit->updateQuery();
query->setQueryText(eclQuery.s.str());
}
createUNCFilename(realdllfilename.str(), dllurl);
unsigned crc = crc_file(realdllfilename.str());
Owned<IWUQuery> query = workunit->updateQuery();
associateLocalFile(query, FileTypeDll, realdllfilename, "Workunit DLL", crc);
queryDllServer().registerDll(realdllname.str(), "Workunit DLL", dllurl.str());
cycle_t elapsedCycles = get_cycles_now() - startCycles;
updateWorkunitTimeStat(workunit, SSTcompilestage, "compile", StTimeElapsed, NULL, cycle_to_nanosec(elapsedCycles));
workunit->commit();
return true;
}
}
catch (IException * e)
{
reportError(e);
e->Release();
}
return false;
}
void ResourceManager::addManifestFile(const char *filename)
{
Owned<IPropertyTree> manifestSrc = createPTreeFromXMLFile(filename);
StringBuffer dir;
splitDirTail(filename, dir);
ensureManifestInfo();
Owned<IAttributeIterator> aiter = manifestSrc->getAttributes();
ForEach (*aiter)
manifest->setProp(aiter->queryName(), aiter->queryValue());
Owned<IPropertyTreeIterator> iter = manifestSrc->getElements("*");
ForEach(*iter)
{
IPropertyTree &item = iter->query();
if (streq(item.queryName(), "Include") && item.hasProp("@filename"))
addManifestInclude(item, dir.str());
else if (streq(item.queryName(), "Resource") && item.hasProp("@filename"))
{
StringBuffer filepath;
StringBuffer respath;
makeAbsolutePath(item.queryProp("@filename"), dir.str(), filepath);
makePathUniversal(filepath.str(), respath);
item.setProp("@originalFilename", filepath.str());
item.setProp("@resourcePath", respath.str());
if (containsFileWildcard(filepath))
{
StringBuffer wildpath;
const char *tail = splitDirTail(filepath, wildpath);
expandManifestDirectory(manifestSrc, item, dir, wildpath, tail, item.getPropBool("@recursive"));
manifestSrc->removeTree(&item);
}
}
else
manifest->addPropTree(item.queryName(), LINK(&item));
}
Owned<IPropertyTreeIterator> resources = manifestSrc->getElements("Resource[@filename]");
ForEach(*resources)
{
IPropertyTree &item = resources->query();
if (!item.hasProp("@type"))
item.setProp("@type", "UNKNOWN");
int id;
if (getDuplicateResourceId(item.queryProp("@type"), item.queryProp("@resourcePath"), NULL, id))
{
item.setPropInt("@id", id);
manifest->addPropTree("Resource", LINK(&item));
}
else
{
MemoryBuffer content;
loadResource(item.queryProp("@originalFilename"), content);
addCompress(item.queryProp("@type"), content.length(), content.toByteArray(), &item);
}
}
}
//---------------------------------------------------------------------------
// determineInstallFiles
//---------------------------------------------------------------------------
int CConfigGenEngine::determineInstallFiles(IPropertyTree& processNode, CInstallFiles& installFiles) const
{
try
{
m_pCallback->printStatus(STATUS_NORMAL, NULL, NULL, NULL,
"Determining files to install for %s", processNode.queryProp("@name"));
StringBuffer compListPath(CONFIGGEN_COMP_LIST);
if (m_inDir.length())
compListPath.clear().append(m_inDir).append(PATHSEPCHAR).append(CONFIGGEN_COMP_LIST);
Owned<IPropertyTree> deployNode = createPTreeFromXMLFile(compListPath.str(), ipt_caseInsensitive);
StringBuffer srcFilePath;
srcFilePath.ensureCapacity(_MAX_PATH);
const bool bFindStartable = &m_process == &processNode && m_startable == unknown;
const bool bFindStoppable = &m_process == &processNode && m_stoppable == unknown;
StringBuffer xpath;
xpath.appendf("Component[@name=\"%s\"]",processNode.queryProp("@buildSet"));
IPropertyTree* pComponent = deployNode->queryPropTree(xpath.str());
if (!pComponent)
{
m_pCallback->printStatus(STATUS_NORMAL, NULL, NULL, NULL,
"Cannot find files to install for %s", processNode.queryProp("@buildSet"));
return 0;
}
Owned<IPropertyTreeIterator> iter = pComponent->getElements("File");
ForEach(*iter)
{
IPropertyTree* pFile = &iter->query();
const char* name = pFile->queryProp("@name");
if (!stricmp(name, "deploy_map.xml"))
continue;
if (bFindStartable && !strnicmp(name, "startup", sizeof("startup")-1))
m_startable = yes;
if (bFindStoppable && !strnicmp(name, "stop", sizeof("stop")-1))
m_stoppable = yes;
const char* method = pFile->queryProp("@method");
if (method && !stricmp(method, "schema"))
continue;
//if we are not deploying build files and method is copy then ignore this file
if (!(m_deployFlags & DEFLAGS_BUILDFILES) && (!method || !stricmp(method, "copy")))
continue;
const char* srcPath = pFile->queryProp("@srcPath");
const char* destPath= pFile->queryProp("@destPath");
const char* destName= pFile->queryProp("@destName");
bool bCacheable = pFile->getPropBool("@cache", false);
// Get source filespec
if (srcPath && !strcmp(srcPath, "@temp"))
{
char tempfile[_MAX_PATH];
getTempPath(tempfile, sizeof(tempfile), m_name);
srcFilePath.clear().append(tempfile).append(name);
}
else
{
srcFilePath.clear().append(m_inDir);
//adjust source paths
if (srcPath && 0!=strcmp(srcPath, "."))
{
if (!strncmp(srcPath, "..", 2) && (*(srcPath+2)=='/' || *(srcPath+2)=='\\'))
{
StringBuffer reldir(srcPath);
reldir.replace('/', '\\');
while (!strncmp(reldir.str(), "..\\", 3))
{
srcFilePath.setLength( srcFilePath.length() - 1 ); //remove last char PATHSEPCHAR
const char* tail = pathTail(srcFilePath.str());
srcFilePath.setLength( tail - srcFilePath.str() );
reldir.remove(0, 3);
}
srcFilePath.append(reldir).append(PATHSEPCHAR);
}
else
srcFilePath.append(srcPath).append(PATHSEPCHAR);
}
srcFilePath.append(name);
}
std::string sDestName;
if (method && (!stricmp(method, "esp_service_module") || !stricmp(method, "esp_plugin")))
{
//if this is xsl transformation and we are not generating config files then ignore
//
if (!(m_deployFlags & DEFLAGS_CONFIGFILES) && !stricmp(method, "esp_service_module"))
continue;
//if this file is an esp service module, encode name of service in the dest file name
//so the esp deployment can figure out which service this file belongs to
//
const char* serviceName = processNode.queryProp("@name");
//if destination name is specified then use it otherwise use <service-name>[index of module].xml
sDestName = serviceName;
if (destName)
{
sDestName += '_';
sDestName += destName;
}
else
{
int espServiceModules = m_envDepEngine.incrementEspModuleCount();
if (espServiceModules > 1)
{
char achNum[16];
itoa(espServiceModules, achNum, 10);
sDestName += achNum;
}
sDestName += ".xml";
}
//encode name of service herein - this is needed by and removed by CEspDeploymentEngine::processServiceModules()
sDestName += '+';
sDestName += processNode.queryProp("@name");//encode the name of service
}
else if (method && (!stricmp(method, "xsl") || !stricmp(method, "xslt")) && !(m_deployFlags & DEFLAGS_CONFIGFILES))
continue;//ignore xsl transformations if we are not generating config files
else
{
if (!method || !*method)
method = "copy";
// Get destination filespec
if (destName && *destName)
{
//we now support attribute names within the destination file names like delimted by @ and + (optional)
//for e.g. segment_@attrib1+_file_@attrib2 would produce segment_attribval1_file_attrib2value
//+ not necessary if the attribute name ends with the word, for e.g. file_@attrib1
//for instnace, suite_@eclServer+.bat would expand to suite_myeclserver.bat
//if this process has an @eclServer with value "myeclserver"
//
if (strchr(destName, '@') || strchr(destName, '+'))
{
char* pszParts = strdup(destName);
char *saveptr;
const char* pszPart = strtok_r(pszParts, "+", &saveptr);
while (pszPart)
{
const char* p = pszPart;
if (*p)
{
if (strchr(p, '@'))//xpath for an attribute?
{
// find name of attribute and replace it with its value
const char* value = m_process.queryProp( p );
if (value)
sDestName.append(value);
}
else
sDestName.append(p); //no attribute so copy verbatim
}
pszPart = strtok_r(NULL, "+", &saveptr);
}
free(pszParts);
}
else
sDestName = destName;
if (sDestName.empty())
throw MakeStringException(-1, "The destination file name '%s' for source file '%s' "
"translates to an empty string!", destName, name);
}
}
StringBuffer destFilePath;
destFilePath.ensureCapacity(_MAX_PATH);
bool bTempFile = (destPath && !stricmp(destPath, "@temp")) ||
!strnicmp(name, "@temp", 5); //@name starts with @temp or @tmp
if (bTempFile)
{
if (sDestName.empty())//dest name not specified
{
if (!strcmp(method, "copy"))
sDestName = name;
else
{
StringBuffer dir;
const char* pszFileName = splitDirTail(name, dir);
const char* pExt = findFileExtension(pszFileName);
if (pExt)
sDestName.append(pszFileName, pExt-pszFileName);
else
sDestName.append(pszFileName);
char index[16];
itoa(m_envDepEngine.incrementTempFileCount(), index, 10);
sDestName.append(index);
if (pExt)
sDestName.append(pExt);
}
}
destFilePath.append("@temp" PATHSEPSTR);
}
else
{
if (destPath && *destPath)
{
destFilePath.append(destPath);
if (destPath[strlen(destPath)-1] != PATHSEPCHAR)
destFilePath.append(PATHSEPCHAR);
}
if (sDestName.empty())
sDestName = name;
}
if (!bTempFile)
destFilePath.append(processNode.queryProp("@name")).append(PATHSEPCHAR);
destFilePath.append(sDestName.c_str());
//For oss, plugins to be handled globally, per Richard.
//esp plugins also end with plugins.xml but they should be handled above.
String destFilePathStr(destFilePath);
String* tmpstr = destFilePathStr.toLowerCase();
if (tmpstr->indexOf("plugins.xml") > 0)
{
delete tmpstr;
createFakePlugins(destFilePath);
continue;
}
delete tmpstr;
//find all occurrences of this destination file in the map and resove any conflicts
//like size mismatch etc.
bool bAddToFileMap = installFiles.resolveConflicts(processNode, method, srcFilePath.str(), destFilePath.str(),
m_name, m_curInstance, NULL);
//resolve conflicts if method is not schema or exec
if (0 != stricmp(method, "schema") && 0 != stricmp(method, "exec") && 0 != strnicmp(method, "del", 3))
{
}
else if (!strnicmp(method, "del", 3))//treat files to be deleted as temp files - to be deleted AFTER we are done!
{
bTempFile = true;
bAddToFileMap = false;
m_envDepEngine.addTempFile(destFilePath.str());
}
if (bAddToFileMap)
{
if (bTempFile)
m_envDepEngine.addTempFile(destFilePath.str());
//enable caching for files to be copied unless expressly asked not to do so
//
if (!bCacheable && !strcmp(method, "copy"))
bCacheable = pFile->getPropBool("@cache", true);
installFiles.addInstallFile(method, srcFilePath.str(), destFilePath.str(), bCacheable, NULL);
}
}
}
catch (IException* e)
{
StringBuffer msg;
e->errorMessage(msg);
e->Release();
throw MakeStringException(0, "Error creating file list for process %s: %s", m_name.get(), msg.str());
}
catch (...)
{
throw MakeErrnoException("Error creating file list for process %s", m_name.get());
}
m_pCallback->printStatus(STATUS_NORMAL, NULL, NULL, NULL, NULL);
return installFiles.getInstallFileList().size();
}
