String AbstractContextI::loadValueFromURL(const String& urlstring) const throw (Exception)
{
String result;
StringBuffer buf;
buf->append(urlstring);
if (0 == buf->indexOf(L"http://"))
{
URL url = new URLI(urlstring);
HttpURLConnection httpurlconnection;
url->openConnection()->downcast(httpurlconnection);
if (verboseOutput()) cout << "JNDI context : Get response from url : " << urlstring << endl;
int rc = httpurlconnection->getResponseCode();
UTF8StringBuffer cbuf;
#if defined (_WINDOWS_SOURCE)
cbuf->append(getenv("TEMP"));
cbuf->append("\\");
cbuf->append(getenv("USERNAME"));
cbuf->append(".");
#else
cbuf->append(getenv("USER_HOME"));
cbuf->append("/.");
#endif
cbuf->append(urlstring->toMD5());
cbuf->append(".properties");
String cachefilepath = cbuf->toString()->toLowerCase();
if (200 != rc)
{
if (verboseOutput()) cout << "JNDI context : HTTP status = " << rc << endl;
if (verboseOutput()) cout << "JNDI context : Trying to load from cached file : " << cachefilepath << endl;
InputStream file = new FileInputStreamI(cachefilepath);
UTF8String content;
file->read(content, 0);
file->close();
result = content->toString();
}
else
{
String response = httpurlconnection->getResponseMessage();
Writer file = new FileWriterI(cachefilepath);
file->write(response);
file->close();
if (verboseOutput()) cout << "JNDI context : Caching to file : " << cachefilepath << endl;
result = response;
}
}
else if (0 == buf->indexOf(L"file://"))
{
result = loadValueFromFile(urlstring);
}
else
{
throw NamingException(WITHDETAILS(L"Unsupported URI format: " + urlstring));
}
StringBuffer b = result;
while (-1 < b->indexOf(L"${"))
{
InitialContext initialcontext = InitialContext::newInstance();
int bpos = 0;
int epos = 0;
while (bpos < b->length())
{
bpos = b->indexOf(L"${", bpos);
if (-1 == bpos) break;
epos = b->indexOf(L"}", bpos);
String variablename = b->substring(bpos + 2, epos);
StringAnything variablevalue ;
initialcontext->lookup(L"/" + variablename, variablevalue);
StringBuffer value = variablevalue->toString();
if (-1 < value->indexOf(L"${" + variablename + L"}") )
throw Exception(WITHDETAILS(L"Recursive reference found : ${" + variablename + L"}"));
b->replace(bpos, epos + 1, value->toString());
bpos = epos + 1;
}
}
return b->toString();
}
static void _doReplicate(CActivityBase *activity, IPartDescriptor &partDesc, ICopyFileProgress *iProgress)
{
StringBuffer primaryName;
getPartFilename(partDesc, 0, primaryName);;
RemoteFilename rfn;
IFileDescriptor &fileDesc = partDesc.queryOwner();
unsigned copies = partDesc.numCopies();
unsigned c=1;
for (; c<copies; c++)
{
unsigned replicateCopy;
unsigned clusterNum = partDesc.copyClusterNum(c, &replicateCopy);
rfn.clear();
partDesc.getFilename(c, rfn);
StringBuffer dstName;
rfn.getPath(dstName);
assertex(dstName.length());
if (replicateCopy>0 )
{
try
{
queryThor().queryBackup().backup(dstName.str(), primaryName.str());
}
catch (IException *e)
{
Owned<IThorException> re = MakeActivityWarning(activity, e, "Failed to create replicate file '%s'", dstName.str());
e->Release();
activity->fireException(re);
}
}
else // another primary
{
ActPrintLog(activity, "Copying to primary %s", dstName.str());
StringBuffer tmpName(dstName.str());
tmpName.append(".tmp");
OwnedIFile tmpIFile = createIFile(tmpName.str());
OwnedIFile srcFile = createIFile(primaryName.str());
CFIPScope fipScope(tmpName.str());
try
{
try
{
ensureDirectoryForFile(dstName.str());
::copyFile(tmpIFile, srcFile, 0x100000, iProgress);
}
catch (IException *e)
{
IThorException *re = MakeActivityException(activity, e, "Failed to copy to tmp file '%s' from source file '%s'", tmpIFile->queryFilename(), srcFile->queryFilename());
e->Release();
throw re;
}
try
{
OwnedIFile dstIFile = createIFile(dstName.str());
dstIFile->remove();
tmpIFile->rename(pathTail(dstName.str()));
}
catch (IException *e)
{
IThorException *re = ThorWrapException(e, "Failed to rename '%s' to '%s'", tmpName.str(), dstName.str());
e->Release();
throw re;
}
}
catch (IException *)
{
try { tmpIFile->remove(); }
catch (IException *e) { ActPrintLog(&activity->queryContainer(), e, NULL); e->Release(); }
throw;
}
}
}
}
int init_main(int argc, char* argv[])
{
InitModuleObjects();
Owned<IProperties> inputs = createProperties(true);
bool interactive = false;
for (int i = 1; i < argc; i++)
{
if (stricmp(argv[i], "-?")==0 || stricmp(argv[i], "-h")==0 || stricmp(argv[i], "-help")==0
|| stricmp(argv[i], "/?")==0 || stricmp(argv[i], "/h")==0)
usage();
else if(stricmp(argv[i], "interactive") == 0)
interactive = true;
else if (strchr(argv[i],'='))
{
inputs->loadProp(argv[i]);
}
else
{
fprintf(stderr, "Unknown option: %s", argv[i]);
return 0;
}
}
int result = -1;
#ifdef _WIN32
if (!interactive)
::SetErrorMode(SEM_NOGPFAULTERRORBOX|SEM_FAILCRITICALERRORS);
#endif
SET_ESP_SIGNAL_HANDLER(SIGPIPE, brokenpipe_handler);
bool SEHMappingEnabled = false;
CEspAbortHandler abortHandler;
Owned<IFile> sentinelFile = createSentinelTarget();
removeSentinelFile(sentinelFile);
Owned<CEspConfig> config;
Owned<CEspServer> server;
try
{
const char* cfgfile = NULL;
const char* procname = NULL;
if(inputs.get())
{
if(inputs->hasProp("config"))
cfgfile = inputs->queryProp("config");
if(inputs->hasProp("process"))
procname = inputs->queryProp("process");
}
if(!cfgfile || !*cfgfile)
cfgfile = "esp.xml";
Owned<IPropertyTree> envpt= createPTreeFromXMLFile(cfgfile, ipt_caseInsensitive);
Owned<IPropertyTree> procpt = NULL;
if (envpt)
{
envpt->addProp("@config", cfgfile);
StringBuffer xpath;
if (procname==NULL || strcmp(procname, ".")==0)
xpath.appendf("Software/EspProcess[1]");
else
xpath.appendf("Software/EspProcess[@name=\"%s\"]", procname);
DBGLOG("Using ESP configuration section [%s]", xpath.str());
procpt.set(envpt->queryPropTree(xpath.str()));
if (!procpt)
throw MakeStringException(-1, "Config section [%s] not found", xpath.str());
}
else
throw MakeStringException(-1, "Failed to load config file %s", cfgfile);
StringBuffer logdir;
if(procpt->hasProp("@name"))
{
StringBuffer espNameStr;
procpt->getProp("@name", espNameStr);
if (!getConfigurationDirectory(envpt->queryPropTree("Software/Directories"), "log", "esp", espNameStr.str(), logdir))
{
logdir.clear();
}
}
const char* build_ver = BUILD_TAG;
setBuildVersion(build_ver);
const char* build_level = BUILD_LEVEL;
setBuildLevel(build_level);
if(logdir.length() == 0)
{
if(procpt->hasProp("@logDir"))
procpt->getProp("@logDir", logdir);
}
if(logdir.length() == 0)
logdir.append(".");
if(stricmp(logdir.str(), ".") != 0)
{
recursiveCreateDirectory(logdir.str());
}
if(logdir.charAt(logdir.length() - 1) != PATHSEPCHAR)
logdir.append(PATHSEPCHAR);
openEspLogFile(logdir.str(), procpt.get());
StringBuffer componentfilesDir;
if(procpt->hasProp("@componentfilesDir"))
procpt->getProp("@componentfilesDir", componentfilesDir);
if(componentfilesDir.length() > 0 && strcmp(componentfilesDir.str(), ".") != 0)
{
DBGLOG("componentfiles are under %s", componentfilesDir.str());
setCFD(componentfilesDir.str());
}
StringBuffer sehsetting;
procpt->getProp("@enableSEHMapping", sehsetting);
if(!interactive && sehsetting.length() > 0 && (stricmp(sehsetting.str(), "true") == 0 || stricmp(sehsetting.str(), "1") == 0))
SEHMappingEnabled = true;
if(SEHMappingEnabled)
EnableSEHtoExceptionMapping();
CEspConfig* cfg = new CEspConfig(inputs.getLink(), envpt.getLink(), procpt.getLink(), false);
if(cfg && cfg->isValid())
{
config.setown(cfg);
abortHandler.setConfig(cfg);
}
}
catch(IException* e)
{
StringBuffer description;
ERRLOG("ESP Unhandled IException (%d -- %s)", e->errorCode(), e->errorMessage(description).str());
e->Release();
return -1;
}
catch (...)
{
ERRLOG("ESP Unhandled General Exception.");
return -1;
}
if (config && config->isValid())
{
PROGLOG("Configuring Esp Platform...");
try
{
CEspServer *srv = new CEspServer(config);
if(SEHMappingEnabled)
srv->setSavedSEHHandler(SEHMappingEnabled);
server.setown(srv);
abortHandler.setServer(srv);
setEspContainer(server.get());
config->loadAll();
config->bindServer(*server.get(), *server.get());
}
catch(IException* e)
{
StringBuffer description;
ERRLOG("ESP Unhandled IException (%d -- %s)", e->errorCode(), e->errorMessage(description).str());
e->Release();
return -1;
}
catch (...)
{
ERRLOG("ESP Unhandled General Exception.");
return -1;
}
writeSentinelFile(sentinelFile);
result = work_main(*config, *server.get());
}
else
{
ERRLOG("!!! Unable to load ESP configuration.");
}
return result;
}
void GermanStemmer::substitute(StringBuffer& buffer) {
substCount = 0;
for ( size_t c = 0; c < buffer.length(); c++ ) {
// Replace the second char of a pair of the equal characters with an asterisk
if ( c > 0 && buffer.charAt( c ) == buffer.charAt ( c - 1 ) ) {
buffer.setCharAt( c, _T('*') );
}
// Substitute Umlauts.
else if ( buffer.charAt( c ) == _T('ä') ) {
buffer.setCharAt( c, _T('a') );
}
else if ( buffer.charAt( c ) == _T('ö') ) {
buffer.setCharAt( c, _T('o') );
}
else if ( buffer.charAt( c ) == _T('ü') ) {
buffer.setCharAt( c, _T('u') );
}
// Fix bug so that 'ß' at the end of a word is replaced.
else if ( buffer.charAt( c ) == _T('ß') ) {
buffer.setCharAt( c, _T('s') );
buffer.insert( c + 1, _T('s') );
substCount++;
}
// Take care that at least one character is left left side from the current one
if ( c < buffer.length() - 1 ) {
// Masking several common character combinations with an token
if ( ( c < buffer.length() - 2 ) && buffer.charAt( c ) == _T('s') &&
buffer.charAt( c + 1 ) == _T('c') && buffer.charAt( c + 2 ) == _T('h') )
{
buffer.setCharAt( c, _T('$') );
buffer.deleteChars( c + 1, c + 3 );
substCount =+ 2;
}
else if ( buffer.charAt( c ) == _T('c') && buffer.charAt( c + 1 ) == _T('h') ) {
buffer.setCharAt( c, _T('§') );
buffer.deleteCharAt( c + 1 );
substCount++;
}
else if ( buffer.charAt( c ) == _T('e') && buffer.charAt( c + 1 ) == _T('i') ) {
buffer.setCharAt( c, _T('%') );
buffer.deleteCharAt( c + 1 );
substCount++;
}
else if ( buffer.charAt( c ) == _T('i') && buffer.charAt( c + 1 ) == _T('e') ) {
buffer.setCharAt( c, _T('&') );
buffer.deleteCharAt( c + 1 );
substCount++;
}
else if ( buffer.charAt( c ) == _T('i') && buffer.charAt( c + 1 ) == _T('g') ) {
buffer.setCharAt( c, _T('#') );
buffer.deleteCharAt( c + 1 );
substCount++;
}
else if ( buffer.charAt( c ) == _T('s') && buffer.charAt( c + 1 ) == _T('t') ) {
buffer.setCharAt( c, _T('!') );
buffer.deleteCharAt( c + 1 );
substCount++;
}
}
}
}
void processMessage(CMessageBuffer &mb)
{
ICoven &coven=queryCoven();
SessionId id;
int fn;
mb.read(fn);
switch (fn) {
case MSR_REGISTER_PROCESS_SESSION: {
acceptConnections.wait();
acceptConnections.signal();
Owned<INode> node(deserializeINode(mb));
Owned<INode> servernode(deserializeINode(mb)); // hopefully me, but not if forwarded
int role=0;
if (mb.length()-mb.getPos()>=sizeof(role)) { // a capability block present
mb.read(role);
if (!manager.authorizeConnection(role,false)) {
SocketEndpoint sender = mb.getSender();
mb.clear();
coven.reply(mb);
MilliSleep(100+getRandom()%1000); // Causes client to 'work' for a short time.
Owned<INode> node = createINode(sender);
coven.disconnect(node);
break;
}
#ifdef _DEBUG
StringBuffer eps;
PROGLOG("Connection to %s authorized",mb.getSender().getUrlStr(eps).str());
#endif
}
IGroup *covengrp;
id = manager.registerClientProcess(node.get(),covengrp,(DaliClientRole)role);
mb.clear().append(id);
if (covengrp->rank(servernode)==RANK_NULL) { // must have been redirected
covengrp->Release(); // no good, so just use one we know about (may use something more sophisticated later)
INode *na = servernode.get();
covengrp = createIGroup(1, &na);
}
covengrp->serialize(mb);
covengrp->Release();
coven.reply(mb);
}
break;
case MSR_SECONDARY_REGISTER_PROCESS_SESSION: {
mb.read(id);
Owned<INode> node (deserializeINode(mb));
int role;
mb.read(role);
manager.addProcessSession(id,node.get(),(DaliClientRole)role);
mb.clear();
coven.reply(mb);
}
break;
case MSR_REGISTER_SESSION: {
SecurityToken tok;
SessionId parentid;
mb.read(tok).read(parentid);
SessionId id = manager.registerSession(tok,parentid);
mb.clear().append(id);
coven.reply(mb);
}
break;
case MSR_SECONDARY_REGISTER_SESSION: {
mb.read(id);
manager.addSession(id);
mb.clear();
coven.reply(mb);
}
break;
case MSR_LOOKUP_PROCESS_SESSION: {
// looks up from node or from id
Owned<INode> node (deserializeINode(mb));
if (node->endpoint().isNull()&&(mb.length()-mb.getPos()>=sizeof(id))) {
mb.read(id);
INode *n = manager.getProcessSessionNode(id);
if (n)
node.setown(n);
node->serialize(mb.clear());
}
else {
id = manager.lookupProcessSession(node.get());
mb.clear().append(id);
}
coven.reply(mb);
}
break;
case MSR_STOP_SESSION: {
SessionId sessid;
bool failed;
mb.read(sessid).read(failed);
manager.stopSession(sessid,failed);
mb.clear();
coven.reply(mb);
}
break;
case MSR_LOOKUP_LDAP_PERMISSIONS: {
StringAttr key;
StringAttr obj;
Owned<IUserDescriptor> udesc=createUserDescriptor();
StringAttr username;
StringAttr passwordenc;
mb.read(key).read(obj);
udesc->deserialize(mb);
#ifndef _NO_DALIUSER_STACKTRACE
//following debug code to be removed
StringBuffer sb;
udesc->getUserName(sb);
if (0==sb.length())
{
DBGLOG("UNEXPECTED USER (NULL) in dasess.cpp CSessionRequestServer::processMessage() line %d", __LINE__);
}
#endif
unsigned auditflags = 0;
if (mb.length()-mb.getPos()>=sizeof(auditflags))
mb.read(auditflags);
int err = 0;
int ret=manager.getPermissionsLDAP(key,obj,udesc,auditflags,&err);
mb.clear().append(ret);
if (err)
mb.append(err);
coven.reply(mb);
}
break;
case MSR_CLEAR_PERMISSIONS_CACHE: {
Owned<IUserDescriptor> udesc=createUserDescriptor();
udesc->deserialize(mb);
bool ok = manager.clearPermissionsCache(udesc);
mb.append(ok);
coven.reply(mb);
}
break;
}
}
void GermanStemmer::substitute(StringBuffer& buffer) {
substCount = 0;
for ( size_t i = 0; i < buffer.length(); i++ ) {
#ifdef _UCS2
TCHAR c = buffer.charAt(i);
#else
unsigned char c = buffer.charAt(i);
#endif
// Replace the second char of a pair of the equal characters with an asterisk
if ( i > 0 && c == buffer.charAt ( i - 1 ) ) {
buffer.setCharAt( i, _T('*') );
}
// Substitute Umlauts.
else if ( c == 0xe4 ) {
buffer.setCharAt( i, _T('a') );
}
else if ( c == 0xf6 ) {
buffer.setCharAt( i, _T('o') );
}
else if ( c == 0xfc ) {
buffer.setCharAt( i, _T('u') );
}
// Fix bug so that 'ß' at the end of a word is replaced.
else if ( c == 0xdf ) {
buffer.setCharAt( i, _T('s') );
buffer.insert( i + 1, _T('s') );
substCount++;
}
// Take care that at least one character is left left side from the current one
if ( i < buffer.length() - 1 ) {
// Masking several common character combinations with an token
if ( ( i < buffer.length() - 2 ) && c == _T('s') &&
buffer.charAt( i + 1 ) == _T('c') && buffer.charAt( i + 2 ) == _T('h') )
{
buffer.setCharAt( i, _T('$') );
buffer.deleteChars( i + 1, i + 3 );
substCount =+ 2;
}
else if ( c == _T('c') && buffer.charAt( i + 1 ) == _T('h') ) {
buffer.setCharAt( i, 0xa7 ); // section sign in UTF-16
buffer.deleteCharAt( i + 1 );
substCount++;
}
else if ( c == _T('e') && buffer.charAt( i + 1 ) == _T('i') ) {
buffer.setCharAt( i, _T('%') );
buffer.deleteCharAt( i + 1 );
substCount++;
}
else if ( c == _T('i') && buffer.charAt( i + 1 ) == _T('e') ) {
buffer.setCharAt( i, _T('&') );
buffer.deleteCharAt( i + 1 );
substCount++;
}
else if ( c == _T('i') && buffer.charAt( i + 1 ) == _T('g') ) {
buffer.setCharAt( i, _T('#') );
buffer.deleteCharAt( i + 1 );
substCount++;
}
else if ( c == _T('s') && buffer.charAt( i + 1 ) == _T('t') ) {
buffer.setCharAt( i, _T('!') );
buffer.deleteCharAt( i + 1 );
substCount++;
}
}
}
}
//---------------------------------------------------------------------------
// determineInstallFiles
//---------------------------------------------------------------------------
int CConfigGenEngine::determineInstallFiles(IPropertyTree& processNode, CInstallFiles& installFiles) const
{
try
{
m_pCallback->printStatus(STATUS_NORMAL, NULL, NULL, NULL,
"Determining files to install for %s", processNode.queryProp("@name"));
StringBuffer compListPath(CONFIGGEN_COMP_LIST);
if (m_inDir.length())
compListPath.clear().append(m_inDir).append(PATHSEPCHAR).append(CONFIGGEN_COMP_LIST);
Owned<IPropertyTree> deployNode = createPTreeFromXMLFile(compListPath.str(), ipt_caseInsensitive);
StringBuffer srcFilePath;
srcFilePath.ensureCapacity(_MAX_PATH);
const bool bFindStartable = &m_process == &processNode && m_startable == unknown;
const bool bFindStoppable = &m_process == &processNode && m_stoppable == unknown;
StringBuffer xpath;
xpath.appendf("Component[@name=\"%s\"]",processNode.queryProp("@buildSet"));
IPropertyTree* pComponent = deployNode->queryPropTree(xpath.str());
if (!pComponent)
{
m_pCallback->printStatus(STATUS_NORMAL, NULL, NULL, NULL,
"Cannot find files to install for %s", processNode.queryProp("@buildSet"));
return 0;
}
Owned<IPropertyTreeIterator> iter = pComponent->getElements("File");
ForEach(*iter)
{
IPropertyTree* pFile = &iter->query();
const char* name = pFile->queryProp("@name");
if (!stricmp(name, "deploy_map.xml"))
continue;
if (bFindStartable && !strnicmp(name, "startup", sizeof("startup")-1))
m_startable = yes;
if (bFindStoppable && !strnicmp(name, "stop", sizeof("stop")-1))
m_stoppable = yes;
const char* method = pFile->queryProp("@method");
if (method && !stricmp(method, "schema"))
continue;
//if we are not deploying build files and method is copy then ignore this file
if (!(m_deployFlags & DEFLAGS_BUILDFILES) && (!method || !stricmp(method, "copy")))
continue;
const char* srcPath = pFile->queryProp("@srcPath");
const char* destPath= pFile->queryProp("@destPath");
const char* destName= pFile->queryProp("@destName");
bool bCacheable = pFile->getPropBool("@cache", false);
// Get source filespec
if (srcPath && !strcmp(srcPath, "@temp"))
{
char tempfile[_MAX_PATH];
getTempPath(tempfile, sizeof(tempfile), m_name);
srcFilePath.clear().append(tempfile).append(name);
}
else
{
srcFilePath.clear().append(m_inDir);
//adjust source paths
if (srcPath && 0!=strcmp(srcPath, "."))
{
if (!strncmp(srcPath, "..", 2) && (*(srcPath+2)=='/' || *(srcPath+2)=='\\'))
{
StringBuffer reldir(srcPath);
reldir.replace('/', '\\');
while (!strncmp(reldir.str(), "..\\", 3))
{
srcFilePath.setLength( srcFilePath.length() - 1 ); //remove last char PATHSEPCHAR
const char* tail = pathTail(srcFilePath.str());
srcFilePath.setLength( tail - srcFilePath.str() );
reldir.remove(0, 3);
}
srcFilePath.append(reldir).append(PATHSEPCHAR);
}
else
srcFilePath.append(srcPath).append(PATHSEPCHAR);
}
srcFilePath.append(name);
}
std::string sDestName;
if (method && (!stricmp(method, "esp_service_module") || !stricmp(method, "esp_plugin")))
{
//if this is xsl transformation and we are not generating config files then ignore
//
if (!(m_deployFlags & DEFLAGS_CONFIGFILES) && !stricmp(method, "esp_service_module"))
continue;
//if this file is an esp service module, encode name of service in the dest file name
//so the esp deployment can figure out which service this file belongs to
//
const char* serviceName = processNode.queryProp("@name");
//if destination name is specified then use it otherwise use <service-name>[index of module].xml
sDestName = serviceName;
if (destName)
{
sDestName += '_';
sDestName += destName;
}
else
{
int espServiceModules = m_envDepEngine.incrementEspModuleCount();
if (espServiceModules > 1)
{
char achNum[16];
itoa(espServiceModules, achNum, 10);
sDestName += achNum;
}
sDestName += ".xml";
}
//encode name of service herein - this is needed by and removed by CEspDeploymentEngine::processServiceModules()
sDestName += '+';
sDestName += processNode.queryProp("@name");//encode the name of service
}
else if (method && (!stricmp(method, "xsl") || !stricmp(method, "xslt")) && !(m_deployFlags & DEFLAGS_CONFIGFILES))
continue;//ignore xsl transformations if we are not generating config files
else
{
if (!method || !*method)
method = "copy";
// Get destination filespec
if (destName && *destName)
{
//we now support attribute names within the destination file names like delimted by @ and + (optional)
//for e.g. segment_@attrib1+_file_@attrib2 would produce segment_attribval1_file_attrib2value
//+ not necessary if the attribute name ends with the word, for e.g. file_@attrib1
//for instnace, suite_@eclServer+.bat would expand to suite_myeclserver.bat
//if this process has an @eclServer with value "myeclserver"
//
if (strchr(destName, '@') || strchr(destName, '+'))
{
char* pszParts = strdup(destName);
char *saveptr;
const char* pszPart = strtok_r(pszParts, "+", &saveptr);
while (pszPart)
{
const char* p = pszPart;
if (*p)
{
if (strchr(p, '@'))//xpath for an attribute?
{
// find name of attribute and replace it with its value
const char* value = m_process.queryProp( p );
if (value)
sDestName.append(value);
}
else
sDestName.append(p); //no attribute so copy verbatim
}
pszPart = strtok_r(NULL, "+", &saveptr);
}
free(pszParts);
}
else
sDestName = destName;
if (sDestName.empty())
throw MakeStringException(-1, "The destination file name '%s' for source file '%s' "
"translates to an empty string!", destName, name);
}
}
StringBuffer destFilePath;
destFilePath.ensureCapacity(_MAX_PATH);
bool bTempFile = (destPath && !stricmp(destPath, "@temp")) ||
!strnicmp(name, "@temp", 5); //@name starts with @temp or @tmp
if (bTempFile)
{
if (sDestName.empty())//dest name not specified
{
if (!strcmp(method, "copy"))
sDestName = name;
else
{
StringBuffer dir;
const char* pszFileName = splitDirTail(name, dir);
const char* pExt = findFileExtension(pszFileName);
if (pExt)
sDestName.append(pszFileName, pExt-pszFileName);
else
sDestName.append(pszFileName);
char index[16];
itoa(m_envDepEngine.incrementTempFileCount(), index, 10);
sDestName.append(index);
if (pExt)
sDestName.append(pExt);
}
}
destFilePath.append("@temp" PATHSEPSTR);
}
else
{
if (destPath && *destPath)
{
destFilePath.append(destPath);
if (destPath[strlen(destPath)-1] != PATHSEPCHAR)
destFilePath.append(PATHSEPCHAR);
}
if (sDestName.empty())
sDestName = name;
}
if (!bTempFile)
destFilePath.append(processNode.queryProp("@name")).append(PATHSEPCHAR);
destFilePath.append(sDestName.c_str());
//For oss, plugins to be handled globally, per Richard.
//esp plugins also end with plugins.xml but they should be handled above.
String destFilePathStr(destFilePath);
String* tmpstr = destFilePathStr.toLowerCase();
if (tmpstr->indexOf("plugins.xml") > 0)
{
delete tmpstr;
createFakePlugins(destFilePath);
continue;
}
delete tmpstr;
//find all occurrences of this destination file in the map and resove any conflicts
//like size mismatch etc.
bool bAddToFileMap = installFiles.resolveConflicts(processNode, method, srcFilePath.str(), destFilePath.str(),
m_name, m_curInstance, NULL);
//resolve conflicts if method is not schema or exec
if (0 != stricmp(method, "schema") && 0 != stricmp(method, "exec") && 0 != strnicmp(method, "del", 3))
{
}
else if (!strnicmp(method, "del", 3))//treat files to be deleted as temp files - to be deleted AFTER we are done!
{
bTempFile = true;
bAddToFileMap = false;
m_envDepEngine.addTempFile(destFilePath.str());
}
if (bAddToFileMap)
{
if (bTempFile)
m_envDepEngine.addTempFile(destFilePath.str());
//enable caching for files to be copied unless expressly asked not to do so
//
if (!bCacheable && !strcmp(method, "copy"))
bCacheable = pFile->getPropBool("@cache", true);
installFiles.addInstallFile(method, srcFilePath.str(), destFilePath.str(), bCacheable, NULL);
}
}
}
catch (IException* e)
{
StringBuffer msg;
e->errorMessage(msg);
e->Release();
throw MakeStringException(0, "Error creating file list for process %s: %s", m_name.get(), msg.str());
}
catch (...)
{
throw MakeErrnoException("Error creating file list for process %s", m_name.get());
}
m_pCallback->printStatus(STATUS_NORMAL, NULL, NULL, NULL, NULL);
return installFiles.getInstallFileList().size();
}
int main( int argc, char *argv[] )
{
int res=0;
if (argc < 3)
{
printf
("frunssh <nodelistfile> \"command\" [options] \n"
" options: -i:<identity-file> \n"
" -u:<user> \n"
" -n:<number_of_threads>\n"
" -t:<connect-timeout-secs>\n"
" -a:<connect-attempts>\n"
" -d:<working_directory>\n"
" -s -- strict, must match known_hosts\n"
" -b -- background\n"
" -pw:<password> -- INSECURE: requires pssh (NB identity file preferred)\n"
" -pe:<password> -- INSECURE: as -pw except encrypted password\n"
" -pl -- use plink (on windows)\n"
" -v -- verbose, lists commands run\n"
" -d -- dry run (for testing, enables verbose)\n"
);
return 255;
}
InitModuleObjects();
#ifndef __64BIT__
// Restrict stack sizes on 32-bit systems
Thread::setDefaultStackSize(0x10000); // NB under windows requires linker setting (/stack:)
#endif
try {
StringBuffer logname;
splitFilename(argv[0], NULL, NULL, &logname, NULL);
Owned<IComponentLogFileCreator> lf = createComponentLogFileCreator("frunssh");
lf->setCreateAliasFile(false);
lf->setMsgFields(MSGFIELD_prefix);
lf->beginLogging();
Owned<IFRunSSH> runssh = createFRunSSH();
runssh->init(argc,argv);
runssh->exec();
const StringArray & strArray = runssh->getReplyText();
const UnsignedArray & unsArray = runssh->getReply();
for(unsigned i = 0;i < unsArray.ordinality();i++) {
StringBuffer buf = strArray.item(i);
// strip newlines off end of string buf
if (buf.length() && (buf.charAt(buf.length()-1)) == '\n') {
buf.setLength(buf.length()-1);
buf.clip();
}
if (buf.length())
PROGLOG("%d: ssh(%d): %s",i+1,unsArray.item(i),buf.str());
}
}
catch(IException *e)
{
EXCLOG(e,"frunssh");
e->Release();
res=255;
}
releaseAtoms();
return res;
}
/**
* Run the built process.
* @return The built process.
* @throws SystemException Thrown if there is an error during the build.
*/
Process *ProcessBuilder::run(void) {
#if defined(__unix) || defined(__APPLE__)
int error = 0;
int old_in = -1, old_out = -1, old_err = -1;
Process *process = 0;
// Prepare the streams
if(in->fd() != io::in.fd()) {
old_in = dup(0);
if(old_in < 0)
error = errno;
else
dup2(in->fd(), 0);
}
if(!errno && out->fd() != io::out.fd()) {
old_out = dup(1);
if(old_out < 0)
error = errno;
else
dup2(out->fd(), 1);
}
if(!errno && err->fd() != io::err.fd()) {
old_err = dup(2);
if(old_err < 0)
error = errno;
else
dup2(err->fd(), 2);
}
// Create the process
if(!error) {
int pid = fork();
// error
if(pid < 0)
error = errno;
// father
else if(pid != 0)
process = new Process(pid);
// son
else {
// Build arguments
char *tab[args.count() + 1];
for(int i = 0; i < args.count(); i++)
tab[i] = (char *)&args[i];
tab[args.count()] = 0;
// Launch the command
execvp(tab[0], tab);
exit(1);
}
}
// Reset the streams
if(old_in >= 0) {
close(0);
dup2(old_in, 0);
close(old_in);
}
if(old_out >= 0) {
close(1);
dup2(old_out, 1);
close(old_out);
}
if(old_err >= 0) {
close(2);
dup2(old_err, 2);
close(old_err);
}
#elif defined(__WIN32) || defined(__WIN64)
// no need to redirect output, if bInheritHandles is set to false when creating process
// it uses standard input, output and error output
int error = 0;
Process *process;
PROCESS_INFORMATION *pi = new PROCESS_INFORMATION;
STARTUPINFO si = {sizeof(si)};
if (!error) {
// Build arguments
StringBuffer tab;
for(int i = 0; i < args.count() ; i++)
{
tab << args[i];
tab << " ";
}
char tabtemp[tab.length() +1];
strcpy(tabtemp,tab.toString().chars());
cout << tabtemp << " error1 = "<< GetLastError() << io::endl;
// Launch process
if(CreateProcess(NULL,tabtemp,0,0,TRUE,0,0,0,&si,pi) == 0)
error = GetLastError();
cout << "error2 = " << GetLastError() << "error value = " << error << io::endl;
process = new Process(pi);
cout << "erro3r = " << GetLastError() << io::endl;
}
// Return the result
if(error){
cout << "error detected " << GetLastError() << io::endl;
throw new SystemException(error, "process building");
}
else
return process;
#else
#error "System Unsupported"
#endif
// Return the result
if(error)
throw new SystemException(error, "process building");
else
return process;
}
bool CEclDirectEx::onRunEcl(IEspContext &context, IEspRunEclRequest & req, IEspRunEclResponse & resp)
{
if (!context.validateFeatureAccess(ECLDIRECT_ACCESS, SecAccess_Full, false))
throw MakeStringException(-1, "EclDirect access permission denied.");
StringBuffer user;
if (!context.getUserID(user).length())
user.append(req.getUserName());
Owned <IWorkUnitFactory> factory = getWorkUnitFactory(context.querySecManager(), context.queryUser());
Owned <IWorkUnit> workunit;
if (!user.length())
workunit.setown(factory->createWorkUnit("ECL-Direct", ""));
else
{
workunit.setown(factory->createWorkUnit("ECL-Direct", user.str()));
workunit->setUser(user.str());
}
Owned<IWUQuery> query = workunit->updateQuery();
query->setQueryText(req.getEclText());
query.clear();
const char* clustername = req.getCluster();
if (!clustername || !*clustername || strieq(clustername, "default"))
clustername = defaultCluster.str();
if (!clustername || !*clustername)
throw MakeStringException(-1, "No Cluster Specified");
if (!isValidCluster(clustername))
throw MakeStringException(-1, "Invalid TargetCluster %s Specified", clustername);
workunit->setClusterName(clustername);
if (req.getLimitResults())
workunit->setResultLimit(100);
const char* snapshot = req.getSnapshot();
if (snapshot && *snapshot)
workunit->setSnapshot(snapshot);
// Execute it
SCMStringBuffer wuid;
workunit->getWuid(wuid);
workunit->setAction(WUActionRun);
workunit->setState(WUStateSubmitted);
workunit.clear();
submitWorkUnit(wuid.str(), context.querySecManager(), context.queryUser());
if (waitForWorkUnitToComplete(wuid.str(), defaultWait))
{
Owned<IConstWorkUnit> cw = factory->openWorkUnit(wuid.str(), false);
SCMStringBuffer resultXML;
getFullWorkUnitResultsXML(context.queryUserId(), context.queryPassword(), cw.get(), resultXML);
resp.setResults(resultXML.str());
cw.clear();
if (deleteWorkunits)
deleteEclDirectWorkunit(factory, wuid.str());
}
else
{
// Don't delete these ones...
DBGLOG("WorkUnit %s timed out", wuid.str());
StringBuffer result;
result.appendf("<Exception><Source>ESP</Source><Message>Timed out waiting for job to complete: %s</Message></Exception>", wuid.str());
resp.setResults(result.str());
}
return true;
}
bool CEclDirectEx::onRunEclEx(IEspContext &context, IEspRunEclExRequest & req, IEspRunEclExResponse & resp)
{
if (!context.validateFeatureAccess(ECLDIRECT_ACCESS, SecAccess_Full, false))
throw MakeStringException(-1, "EclDirect access permission denied.");
const char* eclText = req.getEclText();
if (!eclText || !*eclText)
{
resp.setResults("<Exception><Source>ESP</Source><Message>No Ecl Text provided</Message></Exception>");
return true;
}
StringBuffer user;
if (!context.getUserID(user).length())
user.append(req.getUserName());
Owned <IWorkUnitFactory> factory = getWorkUnitFactory(context.querySecManager(), context.queryUser());
Owned <IWorkUnit> workunit;
if (!user.length())
workunit.setown(factory->createWorkUnit("ECL-Direct", ""));
else
{
workunit.setown(factory->createWorkUnit("ECL-Direct", user.str()));
workunit->setUser(user.str());
}
Owned<IWUQuery> query = workunit->updateQuery();
query->setQueryText(eclText);
query.clear();
const char* cluster = req.getCluster();
if (!cluster || !*cluster || !stricmp(cluster, "default"))
cluster = defaultCluster.str();
if (!cluster || !*cluster)
throw MakeStringException(-1, "No Cluster Specified");
if (!isValidCluster(cluster))
throw MakeStringException(-1, "Invalid TargetCluster %s Specified", cluster);
workunit->setClusterName(cluster);
const char* snapshot = req.getSnapshot();
if (snapshot && *snapshot)
workunit->setSnapshot(snapshot);
if (req.getResultLimit())
workunit->setResultLimit(req.getResultLimit());
// Execute it
SCMStringBuffer wuid;
workunit->getWuid(wuid);
workunit->setAction(WUActionRun);
workunit->setState(WUStateSubmitted);
workunit.clear();
resp.setWuid(wuid.str());
submitWorkUnit(wuid.str(), context.querySecManager(), context.queryUser());
if (!waitForWorkUnitToComplete(wuid.str(), (req.getWait_isNull()) ? defaultWait : req.getWait()))
{
StringBuffer result;
result.appendf("<Exception><Source>ESP</Source><Message>Timed out waiting for job to complete: %s</Message></Exception>", wuid.str());
resp.setResults(result.str());
return true;
}
if (!deleteWorkunits && context.queryRequestParameters()->hasProp("redirect"))
{
StringBuffer url("/WsWorkunits/WUInfo?Wuid=");
resp.setRedirectUrl(url.append(wuid).str());
return true;
}
Owned<IConstWorkUnit> cw = factory->openWorkUnit(wuid.str(), false);
EclDirectWUExceptions errors(*cw);
resp.setErrors(errors);
if (req.getIncludeResults())
{
StringBuffer results;
CRunEclExFormat outputFormat = req.getFormat();
Owned<IWuWebView> web = createWuWebView(wuid.str(), NULL, NULL, getCFD(), true);
if (!web)
results.appendf("<Exception><Source>ESP</Source><Message>Failed loading result workunit %s</Message></Exception>", wuid.str());
else if (outputFormat == CRunEclExFormat_Table)
{
StringBuffer xsltfile(getCFD());
web->applyResultsXSLT(xsltfile.append("xslt/wsecl3_result.xslt").str(), results);
}
else
{
unsigned xmlflags = 0;
if (outputFormat != CRunEclExFormat_ExtendedXml)
xmlflags |= WWV_OMIT_SCHEMAS;
if (context.queryRequestParameters()->hasProp("display_xslt"))
xmlflags |= WWV_USE_DISPLAY_XSLT;
else
xmlflags |= WWV_OMIT_XML_DECLARATION;
web->expandResults(results, xmlflags);
}
resp.setResults(results.str());
}
if (req.getIncludeGraphs())
{
Owned<IConstWUGraphIterator> it = &cw->getGraphs(GraphTypeAny);
StringBuffer xgmml("<Graphs>");
SCMStringBuffer s;
ForEach(*it)
xgmml.append(it->query().getXGMML(s, true).str());
xgmml.append("</Graphs>");
resp.setGraphsXGMML(xgmml.str());
}
if (deleteWorkunits)
deleteEclDirectWorkunit(factory, wuid.str());
return true;
}
int CFileSpraySoapBindingEx::onGetInstantQuery(IEspContext &context, CHttpRequest* request, CHttpResponse* response, const char *service, const char *method)
{
bool permission = true;
bool bDownloadFile = false;
bool bProcess;
StringBuffer sourceLogicalFile;
StringBuffer methodbuf;
StringBuffer submethod;
StringBuffer xsltFileName(getCFD());
xsltFileName.append("smc_xslt/");
if (stricmp(method, "SprayFixedInput")==0)
{
if (!context.validateFeatureAccess(FILE_SPRAY_URL, SecAccess_Write, false))
permission = false;
bProcess = true;
xsltFileName.append("fs_sprayForm.xslt");
methodbuf.append("SprayFixed");
}
else if(stricmp(method, "SprayVariableInput")==0)
{
if (!context.validateFeatureAccess(FILE_SPRAY_URL, SecAccess_Write, false))
permission = false;
bProcess = true;
xsltFileName.append("fs_sprayForm.xslt");
methodbuf.append("SprayVariable");
request->getParameter("submethod", submethod);
}
else if (stricmp(method, "DesprayInput")==0)
{
if (!context.validateFeatureAccess(FILE_DESPRAY_URL, SecAccess_Write, false))
permission = false;
request->getParameter("sourceLogicalName", sourceLogicalFile);
xsltFileName.append("fs_desprayCopyForm.xslt");
methodbuf.append("Despray");
bProcess = true;
}
else if (stricmp(method, "CopyInput") == 0)
{
if (!context.validateFeatureAccess(FILE_SPRAY_URL, SecAccess_Write, false))
permission = false;
request->getParameter("sourceLogicalName", sourceLogicalFile);
xsltFileName.append("fs_desprayCopyForm.xslt");
methodbuf.append("Copy");
bProcess = true;
}
else if (stricmp(method, "RenameInput") == 0)
{
if (!context.validateFeatureAccess(FILE_SPRAY_URL, SecAccess_Write, false))
permission = false;
request->getParameter("sourceLogicalName", sourceLogicalFile);
xsltFileName.append("fs_renameForm.xslt");
methodbuf.append("Rename");
bProcess = true;
}
else if (stricmp(method, "DownloadFile") == 0)
{
if (!context.validateFeatureAccess(FILE_SPRAY_URL, SecAccess_Full, false))
permission = false;
downloadFile(context, request, response);
bDownloadFile = true;
bProcess = true;
}
else
bProcess = false;
if (bProcess)
{
if (bDownloadFile)
return 0;
StringBuffer xml;
Owned<IProperties> params(createProperties());
if (!permission)
{
params->setProp("@method", methodbuf.str());
xml.append("<Environment><ErrorMessage>Permission denied.</ErrorMessage></Environment>");
}
else
{
if(submethod.length() > 0)
params->setProp("@submethod", submethod.str());
params->setProp("@method", methodbuf.str());
if (*sourceLogicalFile.str())
{
params->setProp("@sourceLogicalName", sourceLogicalFile.str());
Owned<IUserDescriptor> userdesc;
StringBuffer username;
context.getUserID(username);
if(username.length() > 0)
{
const char* passwd = context.queryPassword();
userdesc.setown(createUserDescriptor());
userdesc->set(username.str(), passwd);
try
{
if (stricmp(method, "CopyInput") == 0)
{
Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(sourceLogicalFile.str(), userdesc.get());
if(!df)
{
throw MakeStringException(ECLWATCH_FILE_NOT_EXIST,"Could not find file %s.",sourceLogicalFile.str());
}
const char *kind = df->queryProperties().queryProp("@kind");
if (kind && strcmp(kind,"key")==0)
{
params->setProp("@compressflag", 0);
}
else if(df->isCompressed())
{
params->setProp("@compressflag", 2);
}
else
{
params->setProp("@compressflag", 1);
}
}
params->setProp("@supercopyflag", 0); //super copy only for remote copy
}
catch (IException *E)
{
Owned<IXslProcessor> xslp = getXslProcessor();
if (!xslp)
throw E;
Owned<IMultiException> me = MakeMultiException();
me->append(*E);
response->handleExceptions(xslp, me, "FileSpray", method, StringBuffer(getCFD()).append("./smc_xslt/exceptions.xslt").str());
return 0;
}
}
}
else
{
params->setProp("@compressflag", 1);
params->setProp("@supercopyflag", 2);
}
StringBuffer wuid;
request->getParameter("wuid", wuid);
Owned<IPropertyTree> pTree = createPTreeForXslt(method, wuid.str());
toXML(pTree, xml, false);
}
IProperties* requestparams = request->queryParameters();
if(requestparams && requestparams->hasProp("rawxml_"))
{
response->setContent(xml.str());
response->setContentType(HTTP_TYPE_TEXT_XML_UTF8);
}
else{
StringBuffer htmlbuf;
xsltTransform(xml.str(), xsltFileName.str(), params, htmlbuf);
response->setContent(htmlbuf.str());
response->setContentType(HTTP_TYPE_TEXT_HTML_UTF8);
}
response->send();
return 0;
}
else
return CFileSpraySoapBinding::onGetInstantQuery(context, request, response, service, method);
}
IPropertyTree* CFileSpraySoapBindingEx::createPTreeForXslt(const char* method, const char* dfuwuid)
{
Owned<IEnvironmentFactory> factory = getEnvironmentFactory();
#if 0
Owned<IConstEnvironment> m_constEnv = factory->openEnvironment();
#else
Owned<IConstEnvironment> m_constEnv = factory->openEnvironmentByFile();
#endif
Owned<IPropertyTree> pEnvRoot = &m_constEnv->getPTree();
IPropertyTree* pEnvSoftware = pEnvRoot->queryPropTree("Software");
Owned<IPropertyTree> pRoot = createPTreeFromXMLString("<Environment/>");
IPropertyTree* pSoftware = pRoot->addPropTree("Software", createPTree("Software"));
if (pEnvSoftware)
{
StringBuffer dfuwuidSourcePartIP, wuxml;
if(dfuwuid && *dfuwuid)
{
Owned<IDFUWorkUnitFactory> dfuwu_factory = getDFUWorkUnitFactory();
Owned<IConstDFUWorkUnit> dfuwu = dfuwu_factory->openWorkUnit(dfuwuid, false);
if(dfuwu)
{
dfuwu->toXML(wuxml);
Owned<IPropertyTree> wu = createPTreeFromXMLString(wuxml.str());
if (wu)
{
const char* ip = wu->queryProp("Source/Part/@node");
if (ip && *ip)
dfuwuidSourcePartIP.append(ip);
}
}
}
Owned<IPropertyTreeIterator> it = pEnvSoftware->getElements("DropZone");
ForEach(*it)
{
IPropertyTree* pDropZone = pSoftware->addPropTree("DropZone", &it->get());
//get IP Address of the computer associated with this drop zone
const char* pszComputer = it->query().queryProp("@computer");
if (!strcmp(pszComputer, "."))
pszComputer = "localhost";
StringBuffer xpath;
xpath.appendf("Hardware/Computer[@name='%s']/@netAddress", pszComputer);
StringBuffer sNetAddr;
const char* pszNetAddr = pEnvRoot->queryProp(xpath.str());
if (strcmp(pszNetAddr, "."))
{
sNetAddr.append(pszNetAddr);
}
else
{
StringBuffer ipStr;
IpAddress ipaddr = queryHostIP();
ipaddr.getIpText(ipStr);
if (ipStr.length() > 0)
{
#ifdef MACHINE_IP
sNetAddr.append(MACHINE_IP);
#else
sNetAddr.append(ipStr.str());
#endif
}
}
pDropZone->addProp("@netAddress", sNetAddr.str());
if ((dfuwuidSourcePartIP.length() > 0) && (sNetAddr.length() > 0))
{
IpAddress ip1(dfuwuidSourcePartIP.str()), ip2(sNetAddr.str());
if (ip1.ipequals(ip2))
pDropZone->addProp("@sourceNode", "1");
}
Owned<IConstMachineInfo> machine;
if (strcmp(pszNetAddr, "."))
machine.setown(m_constEnv->getMachineByAddress(sNetAddr.str()));
else
{
machine.setown(m_constEnv->getMachineByAddress(pszNetAddr));
if (!machine)
machine.setown(m_constEnv->getMachineByAddress(sNetAddr.str()));
}
if (machine)
{
//int os = machine->getOS();
StringBuffer dir;
pDropZone->getProp("@directory", dir);
if (machine->getOS() == MachineOsLinux || machine->getOS() == MachineOsSolaris)
{
dir.replace('\\', '/');//replace all '\\' by '/'
pDropZone->setProp("@linux", "true");
}
else
{
dir.replace('/', '\\');
dir.replace('$', ':');
}
pDropZone->setProp("@directory", dir);
}
}
it.setown(pEnvSoftware->getElements("ThorCluster"));
ForEach(*it)
pSoftware->addPropTree("ThorCluster", &it->get());
it.setown(pEnvSoftware->getElements("EclAgentProcess"));
ForEach(*it)
{
IPropertyTree &cluster = it->query();
const char* name = cluster.queryProp("@name");
if (!name||!*name)
continue;
unsigned ins = 0;
Owned<IPropertyTreeIterator> insts = cluster.getElements("Instance");
ForEach(*insts)
{
const char *na = insts->query().queryProp("@netAddress");
if (!na || !*na)
{
insts->query().setProp("@gname", name);
continue;
}
SocketEndpoint ep(na);
if (ep.isNull())
continue;
ins++;
StringBuffer gname("hthor__");
//StringBuffer gname;
gname.append(name);
if (ins>1)
gname.append('_').append(ins);
insts->query().setProp("@gname", gname.str());
}
pSoftware->addPropTree("EclAgentProcess", &it->get());
}
if (stricmp(method, "CopyInput") == 0) //Limit for this method only
{
it.setown(pEnvSoftware->getElements("RoxieCluster"));
ForEach(*it)
pSoftware->addPropTree("RoxieCluster", &it->get());
}
if (wuxml.length() > 0)
pSoftware->addPropTree("DfuWorkunit", createPTreeFromXMLString(wuxml.str()));
}
return pRoot.getClear();
}
virtual void process()
{
ActPrintLog("process");
CMasterActivity::process();
IHThorSortArg *helper = (IHThorSortArg *)queryHelper();
StringBuffer skewV;
double skewError;
container.queryJob().getWorkUnitValue("overrideSkewError", skewV);
if (skewV.length())
skewError = atof(skewV.str());
else
{
skewError = helper->getSkew();
if (!skewError)
{
container.queryJob().getWorkUnitValue("defaultSkewError", skewV.clear());
if (skewV.length())
skewError = atof(skewV.str());
}
}
container.queryJob().getWorkUnitValue("defaultSkewWarning", skewV.clear());
double defaultSkewWarning = skewV.length() ? atof(skewV.str()) : 0;
double skewWarning = defaultSkewWarning;
unsigned __int64 skewThreshold = container.queryJob().getWorkUnitValueInt("overrideSkewThreshold", 0);
if (!skewThreshold)
{
skewThreshold = helper->getThreshold();
if (!skewThreshold)
skewThreshold = container.queryJob().getWorkUnitValueInt("defaultSkewThreshold", 0);
}
StringBuffer cosortfilenames;
const char *cosortlogname = helper->getSortedFilename();
if (cosortlogname&&*cosortlogname) {
Owned<IDistributedFile> file = queryThorFileManager().lookup(container.queryJob(), cosortlogname);
Owned<IFileDescriptor> fileDesc = file->getFileDescriptor();
queryThorFileManager().noteFileRead(container.queryJob(), file);
unsigned o;
for (o=0; o<fileDesc->numParts(); o++)
{
Owned<IPartDescriptor> partDesc = fileDesc->getPart(o);
if (cosortfilenames.length())
cosortfilenames.append("|");
// JCSMORE - picking the primary here, means no automatic use of backup copy, could use RMF's possibly.
getPartFilename(*partDesc, 0, cosortfilenames);
}
}
Owned<IRowInterfaces> rowif = createRowInterfaces(container.queryInput(0)->queryHelper()->queryOutputMeta(),queryActivityId(),queryCodeContext());
Owned<IRowInterfaces> auxrowif = createRowInterfaces(helper->querySortedRecordSize(),queryActivityId(),queryCodeContext());
try {
imaster->SortSetup(rowif,helper->queryCompare(),helper->querySerialize(),cosortfilenames.length()!=0,true,cosortfilenames.toCharArray(),auxrowif);
if (barrier->wait(false)) { // local sort complete
size32_t maxdeviance=globals->getPropInt("@sort_max_deviance", 10*1024*1024);
try
{
imaster->Sort(skewThreshold,skewWarning,skewError,maxdeviance,true,false,false,(unsigned)globals->getPropInt("@smallSortThreshold"));
}
catch (IThorException *e)
{
if (TE_SkewError == e->errorCode())
{
StringBuffer s;
Owned<IThorException> e2 = MakeActivityException(this, TE_SortFailedSkewExceeded, "SORT failed. %s", e->errorMessage(s).str());
e->Release();
fireException(e2);
}
else
throw;
}
barrier->wait(false); // merge complete
}
imaster->SortDone();
}
catch (IException *e)
{
ActPrintLog(e, "WARNING: exception during sort");
throw;
}
::Release(imaster);
ActPrintLog("process exit");
}
void
SchemaTypeTuple::validate(
const SchemaValidator * sv,
const Configuration * cfg,
const char * scope,
const char * name,
const char * typeName,
const char * origTypeName,
const StringVector & typeArgs,
int indentLevel) const
throw(ConfigurationException)
{
StringBuffer msg;
StringBuffer errSuffix;
StringBuffer fullyScopedName;
const char ** list;
const char * elemValue;
const char * elemTypeName;
int i;
int listSize;
int typeArgsSize;
int elemNameIndex;
int typeIndex;
int rowNum;
int numElems;
SchemaType * elemTypeDef;
StringVector emptyArgs;
bool ok;
const char * sep;
//--------
// Check the length of the list matches the size of the tuple
//--------
typeArgsSize = typeArgs.length();
assert(typeArgsSize != 0);
assert(typeArgsSize % 2 == 0);
numElems = typeArgsSize / 2;
cfg->lookupList(scope, name, list, listSize);
if (listSize != numElems) {
cfg->mergeNames(scope, name, fullyScopedName);
msg << cfg->fileName() << ": there should be " << numElems
<< " entries in the '" << fullyScopedName << "' " << typeName
<< "; entries denote";
for (i = 0; i < numElems; i++) {
msg << " '" << typeArgs[i*2+0] << "'";
if (i < numElems-1) {
msg << ",";
}
}
throw ConfigurationException(msg.c_str());
}
//--------
// Check each item is of the type specified in the tuple
//--------
for (i = 0; i < listSize; i++) {
typeIndex = (i * 2 + 0) % typeArgsSize;
elemNameIndex = (i * 2 + 1) % typeArgsSize;
rowNum = (i / numElems) + 1;
elemValue = list[i];
elemTypeName = typeArgs[typeIndex];
elemTypeDef = findType(sv, elemTypeName);
ok = callIsA(elemTypeDef, sv, cfg, elemValue, elemTypeName, emptyArgs,
indentLevel + 1, errSuffix);
if (!ok) {
if (errSuffix.length() == 0) {
sep = "";
} else {
sep = "; ";
}
cfg->mergeNames(scope, name, fullyScopedName);
msg << cfg->fileName() << ": bad " << elemTypeName << " value ('"
<< elemValue << "') for element " << i+1 << " ('"
<< typeArgs[elemNameIndex] << "') of the '" << fullyScopedName
<< "' " << typeName << sep << errSuffix;
throw ConfigurationException(msg.c_str());
}
}
}
bool CWsDfuXRefEx::onDFUXRefArrayAction(IEspContext &context, IEspDFUXRefArrayActionRequest &req, IEspDFUXRefArrayActionResponse &resp)
{
try
{
StringBuffer username;
context.getUserID(username);
DBGLOG("CWsDfuXRefEx::onDFUXRefArrayAction User=%s",username.str());
Owned<IUserDescriptor> userdesc;
if(username.length() > 0)
{
const char* passwd = context.queryPassword();
userdesc.setown(createUserDescriptor());
userdesc->set(username.str(), passwd);
}
if(*req.getAction() == 0 || *req.getType() == 0 || *req.getCluster() == 0)
{
ERRLOG("Invalid Parameters into CWsDfuXRefEx::onDFUXRefArrayAction");
throw MakeStringExceptionDirect(ECLWATCH_INVALID_INPUT, "Action, cluster, or type not defined.");
}
Owned<IXRefNode> xRefNode = XRefNodeManager->getXRefNode(req.getCluster());
if (xRefNode.get() == 0)
{
ERRLOG("Unable to resolve XRef cluster name %s",req.getCluster());
throw MakeStringException(ECLWATCH_CANNOT_RESOLVE_CLUSTER_NAME, "Unable to resolve cluster name %s",req.getCluster());
}
Owned<IXRefFilesNode> _fileNode = getFileNodeInterface(*xRefNode.get(),req.getType());
if (_fileNode.get() == 0)
{
ERRLOG("Unable to find a suitable IXRefFilesNode interface for %s",req.getType());
throw MakeStringException(ECLWATCH_CANNOT_FIND_IXREFFILESNODE, "Unable to find a suitable IXRefFilesNode interface for %s",req.getType());
}
if (!context.validateFeatureAccess(FEATURE_URL, SecAccess_Full, false))
throw MakeStringException(ECLWATCH_DFU_XREF_ACCESS_DENIED, "Failed to run Xref command. Permission denied.");
StringBuffer returnStr,UserName;
const char* ActionType = req.getAction();
DBGLOG("Running Xref Command %s for user %s",ActionType,context.getUserID(UserName).str());
for(unsigned i = 0; i < req.getXRefFiles().length();i++)
{
DBGLOG("CWsDfuXRefEx::onDFUXRefArrayAction %s file %s for User=%s", ActionType, req.getXRefFiles().item(i), username.str());
StringBuffer errstr;
if (strcmp("Delete" ,ActionType) == 0)
{
if (_fileNode->RemovePhysical(req.getXRefFiles().item(i),userdesc,req.getCluster(),errstr))
appendReplyMessage(returnStr,NULL,"Removed Physical part %s",req.getXRefFiles().item(i));
else
appendReplyMessage(returnStr,NULL,"Error(s) removing physical part %s\n%s",req.getXRefFiles().item(i),errstr.str());
}
else if (strcmp("Attach" ,ActionType) == 0)
{
if(_fileNode->AttachPhysical(req.getXRefFiles().item(i),userdesc,req.getCluster(),errstr) )
appendReplyMessage(returnStr,NULL,"Reattached Physical part %s",req.getXRefFiles().item(i));
else
appendReplyMessage(returnStr,NULL,"Error(s) attaching physical part %s\n%s",req.getXRefFiles().item(i),errstr.str());
}
if (strcmp("DeleteLogical" ,ActionType) == 0)
{
// Note we don't want to physically delete 'lost' files - this will end up with orphans on next time round but that is safer
if (_fileNode->RemoveLogical(req.getXRefFiles().item(i),userdesc,req.getCluster(),errstr)) {
appendReplyMessage(returnStr,NULL,"Removed Logical File %s",req.getXRefFiles().item(i));
}
else
appendReplyMessage(returnStr,NULL,"Error(s) removing File %s\n%s",req.getXRefFiles().item(i),errstr.str());
}
}
xRefNode->commit();
resp.setDFUXRefArrayActionResult(returnStr.str());
}
catch(IException* e)
{
FORWARDEXCEPTION(context, e, ECLWATCH_INTERNAL_ERROR);
}
return true;
}
int LdapUtils::getServerInfo(const char* ldapserver, int ldapport, StringBuffer& domainDN, LdapServerType& stype, const char* domainname)
{
LdapServerType deducedSType = LDAPSERVER_UNKNOWN;
LDAP* ld = LdapInit("ldap", ldapserver, ldapport, 636);
if(ld == NULL)
{
ERRLOG("ldap init error");
return false;
}
int err = LdapSimpleBind(ld, NULL, NULL);
if(err != LDAP_SUCCESS)
{
DBGLOG("ldap anonymous bind error (%d) - %s", err, ldap_err2string(err));
// for new versions of openldap, version 2.2.*
if(err == LDAP_PROTOCOL_ERROR)
DBGLOG("If you're trying to connect to an OpenLdap server, make sure you have \"allow bind_v2\" enabled in slapd.conf");
return err;
}
LDAPMessage* msg = NULL;
char* attrs[] = {"namingContexts", NULL};
TIMEVAL timeOut = {LDAPTIMEOUT,0};
err = ldap_search_ext_s(ld, NULL, LDAP_SCOPE_BASE, "objectClass=*", attrs, false, NULL, NULL, &timeOut, LDAP_NO_LIMIT, &msg);
if(err != LDAP_SUCCESS)
{
DBGLOG("ldap_search_ext_s error: %s", ldap_err2string( err ));
if (msg)
ldap_msgfree(msg);
return err;
}
LDAPMessage* entry = ldap_first_entry(ld, msg);
if(entry != NULL)
{
CLDAPGetValuesLenWrapper vals(ld, entry, "namingContexts");
if(vals.hasValues())
{
int i = 0;
const char* curdn;
StringBuffer onedn;
while((curdn = vals.queryCharValue(i)) != NULL)
{
if(*curdn != '\0' && (strncmp(curdn, "dc=", 3) == 0 || strncmp(curdn, "DC=", 3) == 0) && strstr(curdn,"DC=ForestDnsZones")==0 && strstr(curdn,"DC=DomainDnsZones")==0 )
{
if(domainDN.length() == 0)
{
StringBuffer curdomain;
getName(curdn, curdomain);
if(onedn.length() == 0)
{
DBGLOG("Queried '%s', selected basedn '%s'",curdn, curdomain.str());
onedn.append(curdomain.str());
}
else
DBGLOG("Ignoring %s", curdn);
if(!domainname || !*domainname || stricmp(curdomain.str(), domainname) == 0)
domainDN.append(curdn);
}
}
else if(*curdn != '\0' && strcmp(curdn, "o=NetscapeRoot") == 0)
{
PROGLOG("Deduced LDAP Server Type 'iPlanet'");
deducedSType = IPLANET;
}
i++;
}
if(domainDN.length() == 0)
domainDN.append(onedn.str());
if (deducedSType == LDAPSERVER_UNKNOWN)
{
if(i <= 1)
{
PROGLOG("Deduced LDAP Server Type 'OpenLDAP'");
deducedSType = OPEN_LDAP;
}
else
{
PROGLOG("Deduced LDAP Server Type 'Active Directory'");
deducedSType = ACTIVE_DIRECTORY;
}
}
}
}
ldap_msgfree(msg);
LDAP_UNBIND(ld);
if (stype == LDAPSERVER_UNKNOWN)
stype = deducedSType;
else if (deducedSType != stype)
WARNLOG("Ignoring deduced LDAP Server Type, does not match config LDAPServerType");
return err;
}
int main(int argc,char **argv)
{
InitModuleObjects();
EnableSEHtoExceptionMapping();
#ifndef __64BIT__
// Restrict stack sizes on 32-bit systems
Thread::setDefaultStackSize(0x10000); // 64K stack (also set in windows DSP)
#endif
Owned<IFile> sentinelFile = createSentinelTarget();
removeSentinelFile(sentinelFile);
SocketEndpoint listenep;
unsigned sendbufsize = 0;
unsigned recvbufsize = 0;
int i = 1;
bool isdaemon = (memicmp(argv[0]+strlen(argv[0])-4,".exe",4)==0);
// bit of a kludge for windows - if .exe not specified then not daemon
bool locallisten = false;
const char *logdir=NULL;
bool requireauthenticate = false;
StringBuffer logDir;
StringBuffer instanceName;
//Get SSL Settings
const char * sslCertFile;
bool useSSL;
unsigned short dafsPort;//DAFILESRV_PORT or SECURE_DAFILESRV_PORT
querySecuritySettings(&useSSL, &dafsPort, &sslCertFile, NULL);
unsigned maxThreads = DEFAULT_THREADLIMIT;
unsigned maxThreadsDelayMs = DEFAULT_THREADLIMITDELAYMS;
unsigned maxAsyncCopy = DEFAULT_ASYNCCOPYMAX;
unsigned parallelRequestLimit = DEFAULT_STDCMD_PARALLELREQUESTLIMIT;
unsigned throttleDelayMs = DEFAULT_STDCMD_THROTTLEDELAYMS;
unsigned throttleCPULimit = DEFAULT_STDCMD_THROTTLECPULIMIT;
unsigned throttleQueueLimit = DEFAULT_STDCMD_THROTTLEQUEUELIMIT;
unsigned parallelSlowRequestLimit = DEFAULT_SLOWCMD_PARALLELREQUESTLIMIT;
unsigned throttleSlowDelayMs = DEFAULT_SLOWCMD_THROTTLEDELAYMS;
unsigned throttleSlowCPULimit = DEFAULT_SLOWCMD_THROTTLECPULIMIT;
unsigned throttleSlowQueueLimit = DEFAULT_SLOWCMD_THROTTLEQUEUELIMIT;
Owned<IPropertyTree> env = getHPCCEnvironment();
if (env)
{
StringBuffer dafilesrvPath("Software/DafilesrvProcess");
if (instanceName.length())
dafilesrvPath.appendf("[@name=\"%s\"]", instanceName.str());
IPropertyTree *daFileSrv = env->queryPropTree(dafilesrvPath);
if (daFileSrv)
{
// global DaFileSrv settings:
maxThreads = daFileSrv->getPropInt("@maxThreads", DEFAULT_THREADLIMIT);
maxThreadsDelayMs = daFileSrv->getPropInt("@maxThreadsDelayMs", DEFAULT_THREADLIMITDELAYMS);
maxAsyncCopy = daFileSrv->getPropInt("@maxAsyncCopy", DEFAULT_ASYNCCOPYMAX);
parallelRequestLimit = daFileSrv->getPropInt("@parallelRequestLimit", DEFAULT_STDCMD_PARALLELREQUESTLIMIT);
throttleDelayMs = daFileSrv->getPropInt("@throttleDelayMs", DEFAULT_STDCMD_THROTTLEDELAYMS);
throttleCPULimit = daFileSrv->getPropInt("@throttleCPULimit", DEFAULT_STDCMD_THROTTLECPULIMIT);
throttleQueueLimit = daFileSrv->getPropInt("@throttleQueueLimit", DEFAULT_STDCMD_THROTTLEQUEUELIMIT);
parallelSlowRequestLimit = daFileSrv->getPropInt("@parallelSlowRequestLimit", DEFAULT_SLOWCMD_PARALLELREQUESTLIMIT);
throttleSlowDelayMs = daFileSrv->getPropInt("@throttleSlowDelayMs", DEFAULT_SLOWCMD_THROTTLEDELAYMS);
throttleSlowCPULimit = daFileSrv->getPropInt("@throttleSlowCPULimit", DEFAULT_SLOWCMD_THROTTLECPULIMIT);
throttleSlowQueueLimit = daFileSrv->getPropInt("@throttleSlowQueueLimit", DEFAULT_SLOWCMD_THROTTLEQUEUELIMIT);
// any overrides by Instance definitions?
// NB: This won't work if netAddress is "." or if we start supporting hostnames there
StringBuffer ipStr;
queryHostIP().getIpText(ipStr);
VStringBuffer daFileSrvPath("Instance[@netAddress=\"%s\"]", ipStr.str());
IPropertyTree *dafileSrvInstance = daFileSrv->queryPropTree(daFileSrvPath);
if (dafileSrvInstance)
{
maxThreads = dafileSrvInstance->getPropInt("@maxThreads", maxThreads);
maxThreadsDelayMs = dafileSrvInstance->getPropInt("@maxThreadsDelayMs", maxThreadsDelayMs);
maxAsyncCopy = dafileSrvInstance->getPropInt("@maxAsyncCopy", maxAsyncCopy);
parallelRequestLimit = dafileSrvInstance->getPropInt("@parallelRequestLimit", parallelRequestLimit);
throttleDelayMs = dafileSrvInstance->getPropInt("@throttleDelayMs", throttleDelayMs);
throttleCPULimit = dafileSrvInstance->getPropInt("@throttleCPULimit", throttleCPULimit);
throttleQueueLimit = dafileSrvInstance->getPropInt("@throttleQueueLimit", throttleQueueLimit);
parallelSlowRequestLimit = dafileSrvInstance->getPropInt("@parallelSlowRequestLimit", parallelSlowRequestLimit);
throttleSlowDelayMs = dafileSrvInstance->getPropInt("@throttleSlowDelayMs", throttleSlowDelayMs);
throttleSlowCPULimit = dafileSrvInstance->getPropInt("@throttleSlowCPULimit", throttleSlowCPULimit);
throttleSlowQueueLimit = dafileSrvInstance->getPropInt("@throttleSlowQueueLimit", throttleSlowQueueLimit);
}
}
}
while (argc>i) {
if (stricmp(argv[i],"-D")==0) {
i++;
isdaemon = true;
}
else if (stricmp(argv[i],"-R")==0) { // for remote run
i++;
#ifdef _WIN32
isdaemon = false;
#else
isdaemon = true;
#endif
}
else if (stricmp(argv[i],"-A")==0) {
i++;
requireauthenticate = true;
}
else if ((argv[i][0]=='-')&&(toupper(argv[i][1])=='T')&&(!argv[i][2]||isdigit(argv[i][2]))) {
if (argv[i][2])
setDafsTrace(NULL,(byte)atoi(argv[i]+2));
i++;
isdaemon = false;
}
else if ((argc>i+1)&&(stricmp(argv[i],"-L")==0)) {
i++;
logDir.clear().append(argv[i++]);
}
else if ((argc>i+1)&&(stricmp(argv[i],"-I")==0)) {
i++;
instanceName.clear().append(argv[i++]);
}
else if (stricmp(argv[i],"-LOCAL")==0) {
i++;
locallisten = true;
}
else if (stricmp(argv[i],"-NOSSL")==0) {//overrides config setting
i++;
if (useSSL)
{
PROGLOG("DaFileSrv SSL specified in config but overridden by -NOSSL in command line");
useSSL = false;
dafsPort = DAFILESRV_PORT;
}
}
else
break;
}
if (useSSL && !sslCertFile)
{
ERRLOG("DaFileSrv SSL specified but certificate file information missing from environment.conf");
exit(-1);
}
if (0 == logDir.length())
{
getConfigurationDirectory(NULL,"log","dafilesrv",instanceName.str(),logDir);
if (0 == logDir.length())
logDir.append(".");
}
if (instanceName.length())
{
addPathSepChar(logDir);
logDir.append(instanceName.str());
}
#ifdef _WIN32
if ((argc>i)&&(stricmp(argv[i],"-install")==0)) {
if (installService(DAFS_SERVICE_NAME,DAFS_SERVICE_DISPLAY_NAME,NULL)) {
PROGLOG(DAFS_SERVICE_DISPLAY_NAME " Installed");
return 0;
}
return 1;
}
if ((argc>i)&&(stricmp(argv[i],"-remove")==0)) {
if (uninstallService(DAFS_SERVICE_NAME,DAFS_SERVICE_DISPLAY_NAME)) {
PROGLOG(DAFS_SERVICE_DISPLAY_NAME " Uninstalled");
return 0;
}
return 1;
}
#endif
if (argc == i)
listenep.port = dafsPort;
else {
if (strchr(argv[i],'.')||!isdigit(argv[i][0]))
listenep.set(argv[i], dafsPort);
else
listenep.port = atoi(argv[i]);
if (listenep.port==0) {
usage();
exit(-1);
}
sendbufsize = (argc>i+1)?(atoi(argv[i+1])*1024):0;
recvbufsize = (argc>i+2)?(atoi(argv[i+2])*1024):0;
}
if (isdaemon) {
#ifdef _WIN32
class cserv: public CService
{
bool stopped;
bool started;
SocketEndpoint listenep;
bool useSSL;
bool requireauthenticate;
unsigned parallelRequestLimit;
unsigned throttleDelayMs;
unsigned throttleCPULimit;
class cpollthread: public Thread
{
cserv *parent;
public:
cpollthread( cserv *_parent )
: Thread("CService::cpollthread"), parent(_parent)
{
}
int run()
{
while (parent->poll())
Sleep(1000);
return 1;
}
} pollthread;
Owned<IRemoteFileServer> server;
public:
cserv(SocketEndpoint _listenep, bool _useSSL, unsigned _parallelRequestLimit, unsigned _throttleDelayMs, unsigned _throttleCPULimit)
: listenep(_listenep),useSSL(_useSSL),pollthread(this),
parallelRequestLimit(_parallelRequestLimit), throttleDelayMs(_throttleDelayMs), throttleCPULimit(_throttleCPULimit)
{
stopped = false;
started = false;
}
virtual ~cserv()
{
stopped = true;
if (started)
pollthread.join();
}
bool init()
{
PROGLOG(DAFS_SERVICE_DISPLAY_NAME " Initialized");
started = true;
pollthread.start();
return true;
}
bool poll()
{
if (stopped||!running()) {
PROGLOG(DAFS_SERVICE_DISPLAY_NAME " Stopping");
if (server) {
server->stop();
server.clear();
}
return false;
}
return true;
}
void run()
{
// Get params from HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Services\DaFileSrv\Parameters
int requireauthenticate=0;
HKEY hkey;
if (RegOpenKeyEx(HKEY_LOCAL_MACHINE,
"SYSTEM\\CurrentControlSet\\Services\\DaFileSrv\\Parameters",
0,
KEY_QUERY_VALUE,
&hkey) == ERROR_SUCCESS) {
DWORD dwType = 0;
DWORD dwSize = sizeof(requireauthenticate);
RegQueryValueEx(hkey,
"RequireAuthentication",
NULL,
&dwType,
(BYTE*)&requireauthenticate,
&dwSize);
RegCloseKey(hkey);
}
StringBuffer eps;
if (listenep.isNull())
eps.append(listenep.port);
else
listenep.getUrlStr(eps);
enableDafsAuthentication(requireauthenticate!=0);
PROGLOG("Opening " DAFS_SERVICE_DISPLAY_NAME " on %s%s", useSSL?"SECURE ":"",eps.str());
const char * verstring = remoteServerVersionString();
PROGLOG("Version: %s", verstring);
PROGLOG("Authentication:%s required",requireauthenticate?"":" not");
PROGLOG(DAFS_SERVICE_DISPLAY_NAME " Running");
server.setown(createRemoteFileServer(maxThreads, maxThreadsDelayMs, maxAsyncCopy));
server->setThrottle(ThrottleStd, parallelRequestLimit, throttleDelayMs, throttleCPULimit);
server->setThrottle(ThrottleSlow, parallelSlowRequestLimit, throttleSlowDelayMs, throttleSlowCPULimit);
try {
server->run(listenep, useSSL);
}
catch (IException *e) {
EXCLOG(e,DAFS_SERVICE_NAME);
e->Release();
}
PROGLOG(DAFS_SERVICE_DISPLAY_NAME " Stopped");
stopped = true;
}
} service(listenep, useSSL, parallelRequestLimit, throttleDelayMs, throttleCPULimit);
service.start();
return 0;
#else
int ret = initDaemon();
if (ret)
return ret;
#endif
}
{
Owned<IComponentLogFileCreator> lf = createComponentLogFileCreator(logDir.str(), "DAFILESRV");
lf->setCreateAliasFile(false);
lf->setMaxDetail(TopDetail);
lf->beginLogging();
}
PROGLOG("Parallel request limit = %d, throttleDelayMs = %d, throttleCPULimit = %d", parallelRequestLimit, throttleDelayMs, throttleCPULimit);
const char * verstring = remoteServerVersionString();
StringBuffer eps;
if (listenep.isNull())
eps.append(listenep.port);
else
listenep.getUrlStr(eps);
enableDafsAuthentication(requireauthenticate);
PROGLOG("Opening Dali File Server on %s%s", useSSL?"SECURE ":"",eps.str());
PROGLOG("Version: %s", verstring);
PROGLOG("Authentication:%s required",requireauthenticate?"":" not");
server.setown(createRemoteFileServer(maxThreads, maxThreadsDelayMs, maxAsyncCopy));
server->setThrottle(ThrottleStd, parallelRequestLimit, throttleDelayMs, throttleCPULimit);
server->setThrottle(ThrottleSlow, parallelSlowRequestLimit, throttleSlowDelayMs, throttleSlowCPULimit);
class CPerfHook : public CSimpleInterfaceOf<IPerfMonHook>
{
public:
virtual void processPerfStats(unsigned processorUsage, unsigned memoryUsage, unsigned memoryTotal, unsigned __int64 fistDiskUsage, unsigned __int64 firstDiskTotal, unsigned __int64 secondDiskUsage, unsigned __int64 secondDiskTotal, unsigned threadCount)
{
}
virtual StringBuffer &extraLogging(StringBuffer &extra)
{
return server->getStats(extra.newline(), true);
}
virtual void log(int level, const char *msg)
{
PROGLOG("%s", msg);
}
} perfHook;
startPerformanceMonitor(10*60*1000, PerfMonStandard, &perfHook);
writeSentinelFile(sentinelFile);
try
{
server->run(listenep, useSSL);
}
catch (IException *e)
{
EXCLOG(e,"DAFILESRV");
e->Release();
}
stopPerformanceMonitor();
if (server)
server->stop();
server.clear();
PROGLOG("Stopped Dali File Server");
return 0;
}
void GermanStemmer::strip(StringBuffer& buffer)
{
bool doMore = true;
while ( doMore && buffer.length() > 3 ) {
if ( ( buffer.length() + substCount > 5 ) &&
buffer.substringEquals( buffer.length() - 2, buffer.length(), _T("nd"), 2 ) )
{
buffer.deleteChars( buffer.length() - 2, buffer.length() );
}
else if ( ( buffer.length() + substCount > 4 ) &&
buffer.substringEquals( buffer.length() - 2, buffer.length(), _T("em"), 2 ) ) {
buffer.deleteChars( buffer.length() - 2, buffer.length() );
}
else if ( ( buffer.length() + substCount > 4 ) &&
buffer.substringEquals( buffer.length() - 2, buffer.length(), _T("er"), 2 ) ) {
buffer.deleteChars( buffer.length() - 2, buffer.length() );
}
else if ( buffer.charAt( buffer.length() - 1 ) == _T('e') ) {
buffer.deleteCharAt( buffer.length() - 1 );
}
else if ( buffer.charAt( buffer.length() - 1 ) == _T('s') ) {
buffer.deleteCharAt( buffer.length() - 1 );
}
else if ( buffer.charAt( buffer.length() - 1 ) == _T('n') ) {
buffer.deleteCharAt( buffer.length() - 1 );
}
// "t" occurs only as suffix of verbs.
else if ( buffer.charAt( buffer.length() - 1 ) == _T('t') ) {
buffer.deleteCharAt( buffer.length() - 1 );
}
else {
doMore = false;
}
}
}
int main( int argc, char *argv[] )
{
#if defined(WIN32) && defined(_DEBUG)
int tmpFlag = _CrtSetDbgFlag( _CRTDBG_REPORT_FLAG );
tmpFlag |= _CRTDBG_LEAK_CHECK_DF;
_CrtSetDbgFlag( tmpFlag );
#endif
InitModuleObjects();
addAbortHandler(ControlHandler);
EnableSEHtoExceptionMapping();
dummyProc();
#ifndef __64BIT__
Thread::setDefaultStackSize(0x10000); // NB under windows requires linker setting (/stack:)
#endif
#ifdef _WIN32
Owned<CReleaseMutex> globalNamedMutex;
#endif
if (globals)
globals->Release();
{
Owned<IFile> iFile = createIFile("thor.xml");
globals = iFile->exists() ? createPTree(*iFile, ipt_caseInsensitive) : createPTree("Thor", ipt_caseInsensitive);
}
unsigned multiThorMemoryThreshold = 0;
try {
if (argc==1)
{
usage();
return 1;
}
cmdArgs = argv+1;
mergeCmdParams(globals);
cmdArgs = argv+1;
const char *master = globals->queryProp("@MASTER");
if (!master)
usage();
const char *slave = globals->queryProp("@SLAVE");
if (slave)
{
slfEp.set(slave);
localHostToNIC(slfEp);
}
else
slfEp.setLocalHost(0);
if (globals->hasProp("@SLAVENUM"))
mySlaveNum = atoi(globals->queryProp("@SLAVENUM"));
else
mySlaveNum = slfEp.port; // shouldn't happen, provided by script
setMachinePortBase(slfEp.port);
slfEp.port = getMachinePortBase();
startSlaveLog();
startMPServer(getFixedPort(TPORT_mp));
#ifdef USE_MP_LOG
startLogMsgParentReceiver();
LOG(MCdebugProgress, thorJob, "MPServer started on port %d", getFixedPort(TPORT_mp));
#endif
SocketEndpoint masterEp(master);
localHostToNIC(masterEp);
setMasterPortBase(masterEp.port);
markNodeCentral(masterEp);
if (RegisterSelf(masterEp))
{
#define ISDALICLIENT // JCSMORE plugins *can* access dali - though I think we should probably prohibit somehow.
#ifdef ISDALICLIENT
const char *daliServers = globals->queryProp("@DALISERVERS");
if (!daliServers)
{
LOG(MCerror, thorJob, "No Dali server list specified\n");
return 1;
}
Owned<IGroup> serverGroup = createIGroup(daliServers, DALI_SERVER_PORT);
unsigned retry = 0;
loop {
try {
LOG(MCdebugProgress, thorJob, "calling initClientProcess");
initClientProcess(serverGroup,DCR_ThorSlave, getFixedPort(TPORT_mp));
break;
}
catch (IJSOCK_Exception *e) {
if ((e->errorCode()!=JSOCKERR_port_in_use))
throw;
FLLOG(MCexception(e), thorJob, e,"InitClientProcess");
if (retry++>10)
throw;
e->Release();
LOG(MCdebugProgress, thorJob, "Retrying");
Sleep(retry*2000);
}
}
setPasswordsFromSDS();
#endif
StringBuffer thorPath;
globals->getProp("@thorPath", thorPath);
recursiveCreateDirectory(thorPath.str());
int err = _chdir(thorPath.str());
if (err)
{
IException *e = MakeErrnoException(-1, "Failed to change dir to '%s'",thorPath.str());
FLLOG(MCexception(e), thorJob, e);
throw e;
}
// Initialization from globals
setIORetryCount(globals->getPropInt("Debug/@ioRetries")); // default == 0 == off
StringBuffer str;
if (globals->getProp("@externalProgDir", str.clear()))
_mkdir(str.str());
else
globals->setProp("@externalProgDir", thorPath);
const char * overrideBaseDirectory = globals->queryProp("@thorDataDirectory");
const char * overrideReplicateDirectory = globals->queryProp("@thorReplicateDirectory");
StringBuffer datadir;
StringBuffer repdir;
if (getConfigurationDirectory(globals->queryPropTree("Directories"),"data","thor",globals->queryProp("@name"),datadir))
overrideBaseDirectory = datadir.str();
if (getConfigurationDirectory(globals->queryPropTree("Directories"),"mirror","thor",globals->queryProp("@name"),repdir))
overrideReplicateDirectory = repdir.str();
if (overrideBaseDirectory&&*overrideBaseDirectory)
setBaseDirectory(overrideBaseDirectory, false);
if (overrideReplicateDirectory&&*overrideBaseDirectory)
setBaseDirectory(overrideReplicateDirectory, true);
StringBuffer tempdirstr;
const char *tempdir = globals->queryProp("@thorTempDirectory");
if (getConfigurationDirectory(globals->queryPropTree("Directories"),"temp","thor",globals->queryProp("@name"),tempdirstr))
tempdir = tempdirstr.str();
SetTempDir(tempdir,true);
useMemoryMappedRead(globals->getPropBool("@useMemoryMappedRead"));
LOG(MCdebugProgress, thorJob, "ThorSlave Version LCR - %d.%d started",THOR_VERSION_MAJOR,THOR_VERSION_MINOR);
StringBuffer url;
LOG(MCdebugProgress, thorJob, "Slave %s - temporary dir set to : %s", slfEp.getUrlStr(url).toCharArray(), queryTempDir());
#ifdef _WIN32
ULARGE_INTEGER userfree;
ULARGE_INTEGER total;
ULARGE_INTEGER free;
if (GetDiskFreeSpaceEx("c:\\",&userfree,&total,&free)&&total.QuadPart) {
unsigned pc = (unsigned)(free.QuadPart*100/total.QuadPart);
LOG(MCdebugProgress, thorJob, "Total disk space = %"I64F"d k", total.QuadPart/1000);
LOG(MCdebugProgress, thorJob, "Free disk space = %"I64F"d k", free.QuadPart/1000);
LOG(MCdebugProgress, thorJob, "%d%% disk free\n",pc);
}
#endif
if (getConfigurationDirectory(globals->queryPropTree("Directories"),"query","thor",globals->queryProp("@name"),str.clear()))
globals->setProp("@query_so_dir", str.str());
else
globals->getProp("@query_so_dir", str.clear());
if (str.length())
{
if (globals->getPropBool("Debug/@dllsToSlaves", true))
{
StringBuffer uniqSoPath;
if (PATHSEPCHAR == str.charAt(str.length()-1))
uniqSoPath.append(str.length()-1, str.str());
else
uniqSoPath.append(str);
uniqSoPath.append("_").append(getMachinePortBase());
str.swapWith(uniqSoPath);
globals->setProp("@query_so_dir", str.str());
}
PROGLOG("Using querySo directory: %s", str.str());
recursiveCreateDirectory(str.str());
}
multiThorMemoryThreshold = globals->getPropInt("@multiThorMemoryThreshold")*0x100000;
if (multiThorMemoryThreshold) {
StringBuffer lgname;
if (!globals->getProp("@multiThorResourceGroup",lgname))
globals->getProp("@nodeGroup",lgname);
if (lgname.length()) {
Owned<ILargeMemLimitNotify> notify = createMultiThorResourceMutex(lgname.str());
setMultiThorMemoryNotify(multiThorMemoryThreshold,notify);
PROGLOG("Multi-Thor resource limit for %s set to %"I64F"d",lgname.str(),(__int64)multiThorMemoryThreshold);
}
else
multiThorMemoryThreshold = 0;
}
slaveMain();
}
LOG(MCdebugProgress, thorJob, "ThorSlave terminated OK");
}
catch (IException *e)
{
FLLOG(MCexception(e), thorJob, e,"ThorSlave");
e->Release();
}
catch (CATCHALL)
{
FLLOG(MCerror, thorJob, "ThorSlave exiting because of uncaught exception");
}
ClearTempDirs();
if (multiThorMemoryThreshold)
setMultiThorMemoryNotify(0,NULL);
roxiemem::releaseRoxieHeap();
#ifdef ISDALICLIENT
closeEnvironment();
closedownClientProcess(); // dali client closedown
#endif
#ifdef USE_MP_LOG
stopLogMsgReceivers();
#endif
stopMPServer();
::Release(globals);
releaseAtoms(); // don't know why we can't use a module_exit to destruct these...
return 0;
}