int main( int argc, char *argv[] ) { #if defined(WIN32) && defined(_DEBUG) int tmpFlag = _CrtSetDbgFlag( _CRTDBG_REPORT_FLAG ); tmpFlag |= _CRTDBG_LEAK_CHECK_DF; _CrtSetDbgFlag( tmpFlag ); #endif InitModuleObjects(); addAbortHandler(ControlHandler); EnableSEHtoExceptionMapping(); dummyProc(); #ifndef __64BIT__ Thread::setDefaultStackSize(0x10000); // NB under windows requires linker setting (/stack:) #endif #ifdef _WIN32 Owned<CReleaseMutex> globalNamedMutex; #endif if (globals) globals->Release(); { Owned<IFile> iFile = createIFile("thor.xml"); globals = iFile->exists() ? createPTree(*iFile, ipt_caseInsensitive) : createPTree("Thor", ipt_caseInsensitive); } unsigned multiThorMemoryThreshold = 0; try { if (argc==1) { usage(); return 1; } cmdArgs = argv+1; mergeCmdParams(globals); cmdArgs = argv+1; const char *master = globals->queryProp("@MASTER"); if (!master) usage(); const char *slave = globals->queryProp("@SLAVE"); if (slave) { slfEp.set(slave); localHostToNIC(slfEp); } else slfEp.setLocalHost(0); if (globals->hasProp("@SLAVENUM")) mySlaveNum = atoi(globals->queryProp("@SLAVENUM")); else mySlaveNum = slfEp.port; // shouldn't happen, provided by script setMachinePortBase(slfEp.port); slfEp.port = getMachinePortBase(); startSlaveLog(); startMPServer(getFixedPort(TPORT_mp)); #ifdef USE_MP_LOG startLogMsgParentReceiver(); LOG(MCdebugProgress, thorJob, "MPServer started on port %d", getFixedPort(TPORT_mp)); #endif SocketEndpoint masterEp(master); localHostToNIC(masterEp); setMasterPortBase(masterEp.port); markNodeCentral(masterEp); if (RegisterSelf(masterEp)) { #define ISDALICLIENT // JCSMORE plugins *can* access dali - though I think we should probably prohibit somehow. #ifdef ISDALICLIENT const char *daliServers = globals->queryProp("@DALISERVERS"); if (!daliServers) { LOG(MCerror, thorJob, "No Dali server list specified\n"); return 1; } Owned<IGroup> serverGroup = createIGroup(daliServers, DALI_SERVER_PORT); unsigned retry = 0; loop { try { LOG(MCdebugProgress, thorJob, "calling initClientProcess"); initClientProcess(serverGroup,DCR_ThorSlave, getFixedPort(TPORT_mp)); break; } catch (IJSOCK_Exception *e) { if ((e->errorCode()!=JSOCKERR_port_in_use)) throw; FLLOG(MCexception(e), thorJob, e,"InitClientProcess"); if (retry++>10) throw; e->Release(); LOG(MCdebugProgress, thorJob, "Retrying"); Sleep(retry*2000); } } setPasswordsFromSDS(); #endif IDaFileSrvHook *daFileSrvHook = queryDaFileSrvHook(); if (daFileSrvHook) // probably always installed daFileSrvHook->addSubnetFilters(globals->queryPropTree("NAS"), NULL); StringBuffer thorPath; globals->getProp("@thorPath", thorPath); recursiveCreateDirectory(thorPath.str()); int err = _chdir(thorPath.str()); if (err) { IException *e = MakeErrnoException(-1, "Failed to change dir to '%s'",thorPath.str()); FLLOG(MCexception(e), thorJob, e); throw e; } // Initialization from globals setIORetryCount(globals->getPropInt("Debug/@ioRetries")); // default == 0 == off StringBuffer str; if (globals->getProp("@externalProgDir", str.clear())) _mkdir(str.str()); else globals->setProp("@externalProgDir", thorPath); const char * overrideBaseDirectory = globals->queryProp("@thorDataDirectory"); const char * overrideReplicateDirectory = globals->queryProp("@thorReplicateDirectory"); StringBuffer datadir; StringBuffer repdir; if (getConfigurationDirectory(globals->queryPropTree("Directories"),"data","thor",globals->queryProp("@name"),datadir)) overrideBaseDirectory = datadir.str(); if (getConfigurationDirectory(globals->queryPropTree("Directories"),"mirror","thor",globals->queryProp("@name"),repdir)) overrideReplicateDirectory = repdir.str(); if (overrideBaseDirectory&&*overrideBaseDirectory) setBaseDirectory(overrideBaseDirectory, false); if (overrideReplicateDirectory&&*overrideBaseDirectory) setBaseDirectory(overrideReplicateDirectory, true); StringBuffer tempdirstr; const char *tempdir = globals->queryProp("@thorTempDirectory"); if (getConfigurationDirectory(globals->queryPropTree("Directories"),"temp","thor",globals->queryProp("@name"),tempdirstr)) tempdir = tempdirstr.str(); SetTempDir(tempdir,true); useMemoryMappedRead(globals->getPropBool("@useMemoryMappedRead")); LOG(MCdebugProgress, thorJob, "ThorSlave Version LCR - %d.%d started",THOR_VERSION_MAJOR,THOR_VERSION_MINOR); StringBuffer url; LOG(MCdebugProgress, thorJob, "Slave %s - temporary dir set to : %s", slfEp.getUrlStr(url).toCharArray(), queryTempDir()); #ifdef _WIN32 ULARGE_INTEGER userfree; ULARGE_INTEGER total; ULARGE_INTEGER free; if (GetDiskFreeSpaceEx("c:\\",&userfree,&total,&free)&&total.QuadPart) { unsigned pc = (unsigned)(free.QuadPart*100/total.QuadPart); LOG(MCdebugProgress, thorJob, "Total disk space = %"I64F"d k", total.QuadPart/1000); LOG(MCdebugProgress, thorJob, "Free disk space = %"I64F"d k", free.QuadPart/1000); LOG(MCdebugProgress, thorJob, "%d%% disk free\n",pc); } #endif if (getConfigurationDirectory(globals->queryPropTree("Directories"),"query","thor",globals->queryProp("@name"),str.clear())) globals->setProp("@query_so_dir", str.str()); else globals->getProp("@query_so_dir", str.clear()); if (str.length()) { if (globals->getPropBool("Debug/@dllsToSlaves", true)) { StringBuffer uniqSoPath; if (PATHSEPCHAR == str.charAt(str.length()-1)) uniqSoPath.append(str.length()-1, str.str()); else uniqSoPath.append(str); uniqSoPath.append("_").append(getMachinePortBase()); str.swapWith(uniqSoPath); globals->setProp("@query_so_dir", str.str()); } PROGLOG("Using querySo directory: %s", str.str()); recursiveCreateDirectory(str.str()); } multiThorMemoryThreshold = globals->getPropInt("@multiThorMemoryThreshold")*0x100000; if (multiThorMemoryThreshold) { StringBuffer lgname; if (!globals->getProp("@multiThorResourceGroup",lgname)) globals->getProp("@nodeGroup",lgname); if (lgname.length()) { Owned<ILargeMemLimitNotify> notify = createMultiThorResourceMutex(lgname.str()); setMultiThorMemoryNotify(multiThorMemoryThreshold,notify); PROGLOG("Multi-Thor resource limit for %s set to %"I64F"d",lgname.str(),(__int64)multiThorMemoryThreshold); } else multiThorMemoryThreshold = 0; } slaveMain(); } LOG(MCdebugProgress, thorJob, "ThorSlave terminated OK"); } catch (IException *e) { FLLOG(MCexception(e), thorJob, e,"ThorSlave"); e->Release(); } catch (CATCHALL) { FLLOG(MCerror, thorJob, "ThorSlave exiting because of uncaught exception"); } ClearTempDirs(); if (multiThorMemoryThreshold) setMultiThorMemoryNotify(0,NULL); roxiemem::releaseRoxieHeap(); #ifdef ISDALICLIENT closeEnvironment(); closedownClientProcess(); // dali client closedown #endif #ifdef USE_MP_LOG stopLogMsgReceivers(); #endif stopMPServer(); ::Release(globals); releaseAtoms(); // don't know why we can't use a module_exit to destruct these... return 0; }
//--------------------------------------------------------------------------- // determineInstallFiles //--------------------------------------------------------------------------- int CConfigGenEngine::determineInstallFiles(IPropertyTree& processNode, CInstallFiles& installFiles) const { try { m_pCallback->printStatus(STATUS_NORMAL, NULL, NULL, NULL, "Determining files to install for %s", processNode.queryProp("@name")); StringBuffer compListPath(CONFIGGEN_COMP_LIST); if (m_inDir.length()) compListPath.clear().append(m_inDir).append(PATHSEPCHAR).append(CONFIGGEN_COMP_LIST); Owned<IPropertyTree> deployNode = createPTreeFromXMLFile(compListPath.str(), ipt_caseInsensitive); StringBuffer srcFilePath; srcFilePath.ensureCapacity(_MAX_PATH); const bool bFindStartable = &m_process == &processNode && m_startable == unknown; const bool bFindStoppable = &m_process == &processNode && m_stoppable == unknown; StringBuffer xpath; xpath.appendf("Component[@name=\"%s\"]",processNode.queryProp("@buildSet")); IPropertyTree* pComponent = deployNode->queryPropTree(xpath.str()); if (!pComponent) { m_pCallback->printStatus(STATUS_NORMAL, NULL, NULL, NULL, "Cannot find files to install for %s", processNode.queryProp("@buildSet")); return 0; } Owned<IPropertyTreeIterator> iter = pComponent->getElements("File"); ForEach(*iter) { IPropertyTree* pFile = &iter->query(); const char* name = pFile->queryProp("@name"); if (!stricmp(name, "deploy_map.xml")) continue; if (bFindStartable && !strnicmp(name, "startup", sizeof("startup")-1)) m_startable = yes; if (bFindStoppable && !strnicmp(name, "stop", sizeof("stop")-1)) m_stoppable = yes; const char* method = pFile->queryProp("@method"); if (method && !stricmp(method, "schema")) continue; //if we are not deploying build files and method is copy then ignore this file if (!(m_deployFlags & DEFLAGS_BUILDFILES) && (!method || !stricmp(method, "copy"))) continue; const char* srcPath = pFile->queryProp("@srcPath"); const char* destPath= pFile->queryProp("@destPath"); const char* destName= pFile->queryProp("@destName"); bool bCacheable = pFile->getPropBool("@cache", false); // Get source filespec if (srcPath && !strcmp(srcPath, "@temp")) { char tempfile[_MAX_PATH]; getTempPath(tempfile, sizeof(tempfile), m_name); srcFilePath.clear().append(tempfile).append(name); } else { srcFilePath.clear().append(m_inDir); //adjust source paths if (srcPath && 0!=strcmp(srcPath, ".")) { if (!strncmp(srcPath, "..", 2) && (*(srcPath+2)=='/' || *(srcPath+2)=='\\')) { StringBuffer reldir(srcPath); reldir.replace('/', '\\'); while (!strncmp(reldir.str(), "..\\", 3)) { srcFilePath.setLength( srcFilePath.length() - 1 ); //remove last char PATHSEPCHAR const char* tail = pathTail(srcFilePath.str()); srcFilePath.setLength( tail - srcFilePath.str() ); reldir.remove(0, 3); } srcFilePath.append(reldir).append(PATHSEPCHAR); } else srcFilePath.append(srcPath).append(PATHSEPCHAR); } srcFilePath.append(name); } std::string sDestName; if (method && (!stricmp(method, "esp_service_module") || !stricmp(method, "esp_plugin"))) { //if this is xsl transformation and we are not generating config files then ignore // if (!(m_deployFlags & DEFLAGS_CONFIGFILES) && !stricmp(method, "esp_service_module")) continue; //if this file is an esp service module, encode name of service in the dest file name //so the esp deployment can figure out which service this file belongs to // const char* serviceName = processNode.queryProp("@name"); //if destination name is specified then use it otherwise use <service-name>[index of module].xml sDestName = serviceName; if (destName) { sDestName += '_'; sDestName += destName; } else { int espServiceModules = m_envDepEngine.incrementEspModuleCount(); if (espServiceModules > 1) { char achNum[16]; itoa(espServiceModules, achNum, 10); sDestName += achNum; } sDestName += ".xml"; } //encode name of service herein - this is needed by and removed by CEspDeploymentEngine::processServiceModules() sDestName += '+'; sDestName += processNode.queryProp("@name");//encode the name of service } else if (method && (!stricmp(method, "xsl") || !stricmp(method, "xslt")) && !(m_deployFlags & DEFLAGS_CONFIGFILES)) continue;//ignore xsl transformations if we are not generating config files else { if (!method || !*method) method = "copy"; // Get destination filespec if (destName && *destName) { //we now support attribute names within the destination file names like delimted by @ and + (optional) //for e.g. segment_@attrib1+_file_@attrib2 would produce segment_attribval1_file_attrib2value //+ not necessary if the attribute name ends with the word, for e.g. file_@attrib1 //for instnace, suite_@eclServer+.bat would expand to suite_myeclserver.bat //if this process has an @eclServer with value "myeclserver" // if (strchr(destName, '@') || strchr(destName, '+')) { char* pszParts = strdup(destName); char *saveptr; const char* pszPart = strtok_r(pszParts, "+", &saveptr); while (pszPart) { const char* p = pszPart; if (*p) { if (strchr(p, '@'))//xpath for an attribute? { // find name of attribute and replace it with its value const char* value = m_process.queryProp( p ); if (value) sDestName.append(value); } else sDestName.append(p); //no attribute so copy verbatim } pszPart = strtok_r(NULL, "+", &saveptr); } free(pszParts); } else sDestName = destName; if (sDestName.empty()) throw MakeStringException(-1, "The destination file name '%s' for source file '%s' " "translates to an empty string!", destName, name); } } StringBuffer destFilePath; destFilePath.ensureCapacity(_MAX_PATH); bool bTempFile = (destPath && !stricmp(destPath, "@temp")) || !strnicmp(name, "@temp", 5); //@name starts with @temp or @tmp if (bTempFile) { if (sDestName.empty())//dest name not specified { if (!strcmp(method, "copy")) sDestName = name; else { StringBuffer dir; const char* pszFileName = splitDirTail(name, dir); const char* pExt = findFileExtension(pszFileName); if (pExt) sDestName.append(pszFileName, pExt-pszFileName); else sDestName.append(pszFileName); char index[16]; itoa(m_envDepEngine.incrementTempFileCount(), index, 10); sDestName.append(index); if (pExt) sDestName.append(pExt); } } destFilePath.append("@temp" PATHSEPSTR); } else { if (destPath && *destPath) { destFilePath.append(destPath); if (destPath[strlen(destPath)-1] != PATHSEPCHAR) destFilePath.append(PATHSEPCHAR); } if (sDestName.empty()) sDestName = name; } if (!bTempFile) destFilePath.append(processNode.queryProp("@name")).append(PATHSEPCHAR); destFilePath.append(sDestName.c_str()); //For oss, plugins to be handled globally, per Richard. //esp plugins also end with plugins.xml but they should be handled above. String destFilePathStr(destFilePath); String* tmpstr = destFilePathStr.toLowerCase(); if (tmpstr->indexOf("plugins.xml") > 0) { delete tmpstr; createFakePlugins(destFilePath); continue; } delete tmpstr; //find all occurrences of this destination file in the map and resove any conflicts //like size mismatch etc. bool bAddToFileMap = installFiles.resolveConflicts(processNode, method, srcFilePath.str(), destFilePath.str(), m_name, m_curInstance, NULL); //resolve conflicts if method is not schema or exec if (0 != stricmp(method, "schema") && 0 != stricmp(method, "exec") && 0 != strnicmp(method, "del", 3)) { } else if (!strnicmp(method, "del", 3))//treat files to be deleted as temp files - to be deleted AFTER we are done! { bTempFile = true; bAddToFileMap = false; m_envDepEngine.addTempFile(destFilePath.str()); } if (bAddToFileMap) { if (bTempFile) m_envDepEngine.addTempFile(destFilePath.str()); //enable caching for files to be copied unless expressly asked not to do so // if (!bCacheable && !strcmp(method, "copy")) bCacheable = pFile->getPropBool("@cache", true); installFiles.addInstallFile(method, srcFilePath.str(), destFilePath.str(), bCacheable, NULL); } } } catch (IException* e) { StringBuffer msg; e->errorMessage(msg); e->Release(); throw MakeStringException(0, "Error creating file list for process %s: %s", m_name.get(), msg.str()); } catch (...) { throw MakeErrnoException("Error creating file list for process %s", m_name.get()); } m_pCallback->printStatus(STATUS_NORMAL, NULL, NULL, NULL, NULL); return installFiles.getInstallFileList().size(); }