void SQLParameterPlaceHolderExpression::toString(StringBuffer & targetstr, bool fullOutput)
{
targetstr.append(value.str());
}
void SQLBinaryExpression::toECLStringTranslateSource(
StringBuffer & eclStr,
IProperties * map,
bool ignoreMisTranslations,
bool forHaving,
bool funcParam,
bool countFuncParam)
{
StringBuffer translation1;
StringBuffer translation2;
operand1->toECLStringTranslateSource(translation1, map, ignoreMisTranslations, forHaving, false, false);
operand2->toECLStringTranslateSource(translation2, map, ignoreMisTranslations, forHaving, false, false);
if (translation1.length()>0 && translation2.length()>0)
{
eclStr.append(translation1);
eclStr.append(getOpStr());
eclStr.append(translation2);
}
else if (translation1.length()<0 && translation2.length()<0)
{
return;
}
else if (ignoreMisTranslations)
{
/*
* If operand1 or operand2 could not be translated using the translation map,
* and ignoreMisTranslations = true, we're going to attempt to return an valid
* ECL translation of this binary expression. IF the binary expression is of type
* OR | AND, we can substitute the mistranslated operand with the appropriate boolean value
* to complete the expression with out changing the gist of the expression.
*
* This is typically done when converting an SQL 'WHERE' clause or 'ON' clause to ECL to
* be used in an ECL JOIN function. In any one particular ECL Join funtion only two datasets
* are joined, therefore not all portions of the SQL logic clause might be relevant.
*
*/
if (op == OR_SYM || op == AND_SYM)
{
StringBuffer convert( op == OR_SYM ? "FALSE" : "TRUE");
if (translation1.length()>0)
{
WARNLOG("Operand 1 of binary expression could not be translated.");
eclStr.append(translation1);
eclStr.append(getOpStr());
eclStr.append(convert);
}
else
{
WARNLOG("Operand 2 of binary expression could not be translated.");
eclStr.append(convert);
eclStr.append(getOpStr());
eclStr.append(translation2);
}
}
else
{
WARNLOG("Binary expression could not be translated.");
return;
}
}
else
{
WARNLOG("Binary expression could not be translated.");
return;
}
}
void SQLBinaryExpression::toString(StringBuffer & targetstr, bool fullOutput)
{
operand1->toString(targetstr, fullOutput);
targetstr.append(getOpStr());
operand2->toString(targetstr, fullOutput);
}
void SQLParenthesisExpression::toString(StringBuffer & targetstr, bool fullOutput)
{
targetstr.append("( ");
innerexpression->toString(targetstr, fullOutput);
targetstr.append(" )");
}
void SQLValueExpression::toString(StringBuffer & targetstr, bool fullOutput)
{
targetstr.append(value.str());
}
void reserialize(VariableUnserializer *uns, StringBuffer &buf) {
char type = uns->readChar();
char sep = uns->readChar();
if (type == 'N') {
buf.append(type);
buf.append(sep);
return;
}
switch (type) {
case 'r':
case 'R':
case 'b':
case 'i':
case 'd':
{
buf.append(type);
buf.append(sep);
while (uns->peek() != ';') {
char ch;
ch = uns->readChar();
buf.append(ch);
}
}
break;
case 'S':
case 'A':
{
// shouldn't happen, but keep the code here anyway.
buf.append(type);
buf.append(sep);
char pointer[8];
uns->read(pointer, 8);
buf.append(pointer, 8);
}
break;
case 's':
{
String v;
v.unserialize(uns);
assert(!v.isNull());
if (v->isStatic()) {
union {
char pointer[8];
StringData *sd;
} u;
u.sd = v.get();
buf.append("S:");
buf.append(u.pointer, 8);
buf.append(';');
} else {
buf.append("s:");
buf.append(v.size());
buf.append(":\"");
buf.append(v.data(), v.size());
buf.append("\";");
}
sep = uns->readChar();
return;
}
break;
case 'a':
{
buf.append("a:");
int64_t size = uns->readInt();
char sep2 = uns->readChar();
buf.append(size);
buf.append(sep2);
sep2 = uns->readChar();
buf.append(sep2);
for (int64_t i = 0; i < size; i++) {
reserialize(uns, buf); // key
reserialize(uns, buf); // value
}
sep2 = uns->readChar(); // '}'
buf.append(sep2);
return;
}
break;
case 'o':
case 'O':
{
buf.append(type);
buf.append(sep);
String clsName;
clsName.unserialize(uns);
buf.append(clsName.size());
buf.append(":\"");
buf.append(clsName.data(), clsName.size());
buf.append("\":");
uns->readChar();
int64_t size = uns->readInt();
char sep2 = uns->readChar();
buf.append(size);
buf.append(sep2);
sep2 = uns->readChar(); // '{'
buf.append(sep2);
for (int64_t i = 0; i < size; i++) {
reserialize(uns, buf); // property name
reserialize(uns, buf); // property value
}
sep2 = uns->readChar(); // '}'
buf.append(sep2);
return;
}
break;
case 'C':
{
buf.append(type);
buf.append(sep);
String clsName;
clsName.unserialize(uns);
buf.append(clsName.size());
buf.append(":\"");
buf.append(clsName.data(), clsName.size());
buf.append("\":");
sep = uns->readChar(); // ':'
String serialized;
serialized.unserialize(uns, '{', '}');
buf.append(serialized.size());
buf.append(":{");
buf.append(serialized.data(), serialized.size());
buf.append('}');
return;
}
break;
default:
throw Exception("Unknown type '%c'", type);
}
sep = uns->readChar(); // the last ';'
buf.append(sep);
}
bool VirtualHost::rewriteURL(CStrRef host, String &url, bool &qsa,
int &redirect) const {
String normalized = url;
if (normalized.empty() || normalized.charAt(0) != '/') {
normalized = String("/") + normalized;
}
for (unsigned int i = 0; i < m_rewriteRules.size(); i++) {
const RewriteRule &rule = m_rewriteRules[i];
bool passed = true;
for (vector<RewriteCond>::const_iterator it = rule.rewriteConds.begin();
it != rule.rewriteConds.end(); ++it) {
String subject;
if (it->type == RewriteCond::Request) {
subject = normalized;
} else {
subject = host;
}
Variant ret = preg_match(String(it->pattern.c_str(), it->pattern.size(),
AttachLiteral), subject);
if (!ret.same(it->negate ? 0 : 1)) {
passed = false;
break;
}
}
if (!passed) continue;
Variant matches;
int count = preg_match(rule.pattern.c_str(), normalized, matches);
if (count > 0) {
const char *s = rule.to.c_str();
StringBuffer ret;
while (*s) {
int backref = -1;
if (*s == '\\') {
if ('0' <= s[1] && s[1] <= '9') {
s++;
backref = get_backref(&s);
} else if (s[1] == '\\') {
s++;
}
} else if (*s == '$') {
if (s[1] == '{') {
const char *t = s+2;
if ('0' <= *t && *t <= '9') {
backref = get_backref(&t);
if (*t != '}') {
backref = -1;
} else {
s = t+1;
}
}
} else if ('0' <= s[1] && s[1] <= '9') {
s++;
backref = get_backref(&s);
}
}
if (backref >= 0) {
String br = matches[backref].toString();
if (rule.encode_backrefs) {
br = StringUtil::UrlEncode(br);
}
ret.append(br);
} else {
ret.append(s, 1);
s++;
}
}
url = ret.detach();
qsa = rule.qsa;
redirect = rule.redirect;
return true;
}
}
return false;
}
virtual void process()
{
ActPrintLog("process");
CMasterActivity::process();
IHThorSortArg *helper = (IHThorSortArg *)queryHelper();
StringBuffer skewV;
double skewError;
container.queryJob().getWorkUnitValue("overrideSkewError", skewV);
if (skewV.length())
skewError = atof(skewV.str());
else
{
skewError = helper->getSkew();
if (!skewError)
{
container.queryJob().getWorkUnitValue("defaultSkewError", skewV.clear());
if (skewV.length())
skewError = atof(skewV.str());
}
}
container.queryJob().getWorkUnitValue("defaultSkewWarning", skewV.clear());
double defaultSkewWarning = skewV.length() ? atof(skewV.str()) : 0;
double skewWarning = defaultSkewWarning;
unsigned __int64 skewThreshold = container.queryJob().getWorkUnitValueInt("overrideSkewThreshold", 0);
if (!skewThreshold)
{
skewThreshold = helper->getThreshold();
if (!skewThreshold)
skewThreshold = container.queryJob().getWorkUnitValueInt("defaultSkewThreshold", 0);
}
StringBuffer cosortfilenames;
const char *cosortlogname = helper->getSortedFilename();
if (cosortlogname&&*cosortlogname) {
Owned<IDistributedFile> file = queryThorFileManager().lookup(container.queryJob(), cosortlogname);
Owned<IFileDescriptor> fileDesc = file->getFileDescriptor();
queryThorFileManager().noteFileRead(container.queryJob(), file);
unsigned o;
for (o=0; o<fileDesc->numParts(); o++)
{
Owned<IPartDescriptor> partDesc = fileDesc->getPart(o);
if (cosortfilenames.length())
cosortfilenames.append("|");
// JCSMORE - picking the primary here, means no automatic use of backup copy, could use RMF's possibly.
getPartFilename(*partDesc, 0, cosortfilenames);
}
}
Owned<IRowInterfaces> rowif = createRowInterfaces(container.queryInput(0)->queryHelper()->queryOutputMeta(),queryActivityId(),queryCodeContext());
Owned<IRowInterfaces> auxrowif = createRowInterfaces(helper->querySortedRecordSize(),queryActivityId(),queryCodeContext());
try {
imaster->SortSetup(rowif,helper->queryCompare(),helper->querySerialize(),cosortfilenames.length()!=0,true,cosortfilenames.toCharArray(),auxrowif);
if (barrier->wait(false)) { // local sort complete
size32_t maxdeviance=globals->getPropInt("@sort_max_deviance", 10*1024*1024);
try
{
imaster->Sort(skewThreshold,skewWarning,skewError,maxdeviance,true,false,false,(unsigned)globals->getPropInt("@smallSortThreshold"));
}
catch (IThorException *e)
{
if (TE_SkewError == e->errorCode())
{
StringBuffer s;
Owned<IThorException> e2 = MakeActivityException(this, TE_SortFailedSkewExceeded, "SORT failed. %s", e->errorMessage(s).str());
e->Release();
fireException(e2);
}
else
throw;
}
barrier->wait(false); // merge complete
}
imaster->SortDone();
}
catch (IException *e)
{
ActPrintLog(e, "WARNING: exception during sort");
throw;
}
::Release(imaster);
ActPrintLog("process exit");
}
//---------------------------------------------------------------------------
// determineInstallFiles
//---------------------------------------------------------------------------
int CConfigGenEngine::determineInstallFiles(IPropertyTree& processNode, CInstallFiles& installFiles) const
{
try
{
m_pCallback->printStatus(STATUS_NORMAL, NULL, NULL, NULL,
"Determining files to install for %s", processNode.queryProp("@name"));
StringBuffer compListPath(CONFIGGEN_COMP_LIST);
if (m_inDir.length())
compListPath.clear().append(m_inDir).append(PATHSEPCHAR).append(CONFIGGEN_COMP_LIST);
Owned<IPropertyTree> deployNode = createPTreeFromXMLFile(compListPath.str(), ipt_caseInsensitive);
StringBuffer srcFilePath;
srcFilePath.ensureCapacity(_MAX_PATH);
const bool bFindStartable = &m_process == &processNode && m_startable == unknown;
const bool bFindStoppable = &m_process == &processNode && m_stoppable == unknown;
StringBuffer xpath;
xpath.appendf("Component[@name=\"%s\"]",processNode.queryProp("@buildSet"));
IPropertyTree* pComponent = deployNode->queryPropTree(xpath.str());
if (!pComponent)
{
m_pCallback->printStatus(STATUS_NORMAL, NULL, NULL, NULL,
"Cannot find files to install for %s", processNode.queryProp("@buildSet"));
return 0;
}
Owned<IPropertyTreeIterator> iter = pComponent->getElements("File");
ForEach(*iter)
{
IPropertyTree* pFile = &iter->query();
const char* name = pFile->queryProp("@name");
if (!stricmp(name, "deploy_map.xml"))
continue;
if (bFindStartable && !strnicmp(name, "startup", sizeof("startup")-1))
m_startable = yes;
if (bFindStoppable && !strnicmp(name, "stop", sizeof("stop")-1))
m_stoppable = yes;
const char* method = pFile->queryProp("@method");
if (method && !stricmp(method, "schema"))
continue;
//if we are not deploying build files and method is copy then ignore this file
if (!(m_deployFlags & DEFLAGS_BUILDFILES) && (!method || !stricmp(method, "copy")))
continue;
const char* srcPath = pFile->queryProp("@srcPath");
const char* destPath= pFile->queryProp("@destPath");
const char* destName= pFile->queryProp("@destName");
bool bCacheable = pFile->getPropBool("@cache", false);
// Get source filespec
if (srcPath && !strcmp(srcPath, "@temp"))
{
char tempfile[_MAX_PATH];
getTempPath(tempfile, sizeof(tempfile), m_name);
srcFilePath.clear().append(tempfile).append(name);
}
else
{
srcFilePath.clear().append(m_inDir);
//adjust source paths
if (srcPath && 0!=strcmp(srcPath, "."))
{
if (!strncmp(srcPath, "..", 2) && (*(srcPath+2)=='/' || *(srcPath+2)=='\\'))
{
StringBuffer reldir(srcPath);
reldir.replace('/', '\\');
while (!strncmp(reldir.str(), "..\\", 3))
{
srcFilePath.setLength( srcFilePath.length() - 1 ); //remove last char PATHSEPCHAR
const char* tail = pathTail(srcFilePath.str());
srcFilePath.setLength( tail - srcFilePath.str() );
reldir.remove(0, 3);
}
srcFilePath.append(reldir).append(PATHSEPCHAR);
}
else
srcFilePath.append(srcPath).append(PATHSEPCHAR);
}
srcFilePath.append(name);
}
std::string sDestName;
if (method && (!stricmp(method, "esp_service_module") || !stricmp(method, "esp_plugin")))
{
//if this is xsl transformation and we are not generating config files then ignore
//
if (!(m_deployFlags & DEFLAGS_CONFIGFILES) && !stricmp(method, "esp_service_module"))
continue;
//if this file is an esp service module, encode name of service in the dest file name
//so the esp deployment can figure out which service this file belongs to
//
const char* serviceName = processNode.queryProp("@name");
//if destination name is specified then use it otherwise use <service-name>[index of module].xml
sDestName = serviceName;
if (destName)
{
sDestName += '_';
sDestName += destName;
}
else
{
int espServiceModules = m_envDepEngine.incrementEspModuleCount();
if (espServiceModules > 1)
{
char achNum[16];
itoa(espServiceModules, achNum, 10);
sDestName += achNum;
}
sDestName += ".xml";
}
//encode name of service herein - this is needed by and removed by CEspDeploymentEngine::processServiceModules()
sDestName += '+';
sDestName += processNode.queryProp("@name");//encode the name of service
}
else if (method && (!stricmp(method, "xsl") || !stricmp(method, "xslt")) && !(m_deployFlags & DEFLAGS_CONFIGFILES))
continue;//ignore xsl transformations if we are not generating config files
else
{
if (!method || !*method)
method = "copy";
// Get destination filespec
if (destName && *destName)
{
//we now support attribute names within the destination file names like delimted by @ and + (optional)
//for e.g. segment_@attrib1+_file_@attrib2 would produce segment_attribval1_file_attrib2value
//+ not necessary if the attribute name ends with the word, for e.g. file_@attrib1
//for instnace, suite_@eclServer+.bat would expand to suite_myeclserver.bat
//if this process has an @eclServer with value "myeclserver"
//
if (strchr(destName, '@') || strchr(destName, '+'))
{
char* pszParts = strdup(destName);
char *saveptr;
const char* pszPart = strtok_r(pszParts, "+", &saveptr);
while (pszPart)
{
const char* p = pszPart;
if (*p)
{
if (strchr(p, '@'))//xpath for an attribute?
{
// find name of attribute and replace it with its value
const char* value = m_process.queryProp( p );
if (value)
sDestName.append(value);
}
else
sDestName.append(p); //no attribute so copy verbatim
}
pszPart = strtok_r(NULL, "+", &saveptr);
}
free(pszParts);
}
else
sDestName = destName;
if (sDestName.empty())
throw MakeStringException(-1, "The destination file name '%s' for source file '%s' "
"translates to an empty string!", destName, name);
}
}
StringBuffer destFilePath;
destFilePath.ensureCapacity(_MAX_PATH);
bool bTempFile = (destPath && !stricmp(destPath, "@temp")) ||
!strnicmp(name, "@temp", 5); //@name starts with @temp or @tmp
if (bTempFile)
{
if (sDestName.empty())//dest name not specified
{
if (!strcmp(method, "copy"))
sDestName = name;
else
{
StringBuffer dir;
const char* pszFileName = splitDirTail(name, dir);
const char* pExt = findFileExtension(pszFileName);
if (pExt)
sDestName.append(pszFileName, pExt-pszFileName);
else
sDestName.append(pszFileName);
char index[16];
itoa(m_envDepEngine.incrementTempFileCount(), index, 10);
sDestName.append(index);
if (pExt)
sDestName.append(pExt);
}
}
destFilePath.append("@temp" PATHSEPSTR);
}
else
{
if (destPath && *destPath)
{
destFilePath.append(destPath);
if (destPath[strlen(destPath)-1] != PATHSEPCHAR)
destFilePath.append(PATHSEPCHAR);
}
if (sDestName.empty())
sDestName = name;
}
if (!bTempFile)
destFilePath.append(processNode.queryProp("@name")).append(PATHSEPCHAR);
destFilePath.append(sDestName.c_str());
//For oss, plugins to be handled globally, per Richard.
//esp plugins also end with plugins.xml but they should be handled above.
String destFilePathStr(destFilePath);
String* tmpstr = destFilePathStr.toLowerCase();
if (tmpstr->indexOf("plugins.xml") > 0)
{
delete tmpstr;
createFakePlugins(destFilePath);
continue;
}
delete tmpstr;
//find all occurrences of this destination file in the map and resove any conflicts
//like size mismatch etc.
bool bAddToFileMap = installFiles.resolveConflicts(processNode, method, srcFilePath.str(), destFilePath.str(),
m_name, m_curInstance, NULL);
//resolve conflicts if method is not schema or exec
if (0 != stricmp(method, "schema") && 0 != stricmp(method, "exec") && 0 != strnicmp(method, "del", 3))
{
}
else if (!strnicmp(method, "del", 3))//treat files to be deleted as temp files - to be deleted AFTER we are done!
{
bTempFile = true;
bAddToFileMap = false;
m_envDepEngine.addTempFile(destFilePath.str());
}
if (bAddToFileMap)
{
if (bTempFile)
m_envDepEngine.addTempFile(destFilePath.str());
//enable caching for files to be copied unless expressly asked not to do so
//
if (!bCacheable && !strcmp(method, "copy"))
bCacheable = pFile->getPropBool("@cache", true);
installFiles.addInstallFile(method, srcFilePath.str(), destFilePath.str(), bCacheable, NULL);
}
}
}
catch (IException* e)
{
StringBuffer msg;
e->errorMessage(msg);
e->Release();
throw MakeStringException(0, "Error creating file list for process %s: %s", m_name.get(), msg.str());
}
catch (...)
{
throw MakeErrnoException("Error creating file list for process %s", m_name.get());
}
m_pCallback->printStatus(STATUS_NORMAL, NULL, NULL, NULL, NULL);
return installFiles.getInstallFileList().size();
}
int main( int argc, char *argv[] )
{
#if defined(WIN32) && defined(_DEBUG)
int tmpFlag = _CrtSetDbgFlag( _CRTDBG_REPORT_FLAG );
tmpFlag |= _CRTDBG_LEAK_CHECK_DF;
_CrtSetDbgFlag( tmpFlag );
#endif
InitModuleObjects();
addAbortHandler(ControlHandler);
EnableSEHtoExceptionMapping();
dummyProc();
#ifndef __64BIT__
Thread::setDefaultStackSize(0x10000); // NB under windows requires linker setting (/stack:)
#endif
#ifdef _WIN32
Owned<CReleaseMutex> globalNamedMutex;
#endif
if (globals)
globals->Release();
{
Owned<IFile> iFile = createIFile("thor.xml");
globals = iFile->exists() ? createPTree(*iFile, ipt_caseInsensitive) : createPTree("Thor", ipt_caseInsensitive);
}
unsigned multiThorMemoryThreshold = 0;
try {
if (argc==1)
{
usage();
return 1;
}
cmdArgs = argv+1;
mergeCmdParams(globals);
cmdArgs = argv+1;
const char *master = globals->queryProp("@MASTER");
if (!master)
usage();
const char *slave = globals->queryProp("@SLAVE");
if (slave)
{
slfEp.set(slave);
localHostToNIC(slfEp);
}
else
slfEp.setLocalHost(0);
if (globals->hasProp("@SLAVENUM"))
mySlaveNum = atoi(globals->queryProp("@SLAVENUM"));
else
mySlaveNum = slfEp.port; // shouldn't happen, provided by script
setMachinePortBase(slfEp.port);
slfEp.port = getMachinePortBase();
startSlaveLog();
startMPServer(getFixedPort(TPORT_mp));
#ifdef USE_MP_LOG
startLogMsgParentReceiver();
LOG(MCdebugProgress, thorJob, "MPServer started on port %d", getFixedPort(TPORT_mp));
#endif
SocketEndpoint masterEp(master);
localHostToNIC(masterEp);
setMasterPortBase(masterEp.port);
markNodeCentral(masterEp);
if (RegisterSelf(masterEp))
{
#define ISDALICLIENT // JCSMORE plugins *can* access dali - though I think we should probably prohibit somehow.
#ifdef ISDALICLIENT
const char *daliServers = globals->queryProp("@DALISERVERS");
if (!daliServers)
{
LOG(MCerror, thorJob, "No Dali server list specified\n");
return 1;
}
Owned<IGroup> serverGroup = createIGroup(daliServers, DALI_SERVER_PORT);
unsigned retry = 0;
loop {
try {
LOG(MCdebugProgress, thorJob, "calling initClientProcess");
initClientProcess(serverGroup,DCR_ThorSlave, getFixedPort(TPORT_mp));
break;
}
catch (IJSOCK_Exception *e) {
if ((e->errorCode()!=JSOCKERR_port_in_use))
throw;
FLLOG(MCexception(e), thorJob, e,"InitClientProcess");
if (retry++>10)
throw;
e->Release();
LOG(MCdebugProgress, thorJob, "Retrying");
Sleep(retry*2000);
}
}
setPasswordsFromSDS();
#endif
StringBuffer thorPath;
globals->getProp("@thorPath", thorPath);
recursiveCreateDirectory(thorPath.str());
int err = _chdir(thorPath.str());
if (err)
{
IException *e = MakeErrnoException(-1, "Failed to change dir to '%s'",thorPath.str());
FLLOG(MCexception(e), thorJob, e);
throw e;
}
// Initialization from globals
setIORetryCount(globals->getPropInt("Debug/@ioRetries")); // default == 0 == off
StringBuffer str;
if (globals->getProp("@externalProgDir", str.clear()))
_mkdir(str.str());
else
globals->setProp("@externalProgDir", thorPath);
const char * overrideBaseDirectory = globals->queryProp("@thorDataDirectory");
const char * overrideReplicateDirectory = globals->queryProp("@thorReplicateDirectory");
StringBuffer datadir;
StringBuffer repdir;
if (getConfigurationDirectory(globals->queryPropTree("Directories"),"data","thor",globals->queryProp("@name"),datadir))
overrideBaseDirectory = datadir.str();
if (getConfigurationDirectory(globals->queryPropTree("Directories"),"mirror","thor",globals->queryProp("@name"),repdir))
overrideReplicateDirectory = repdir.str();
if (overrideBaseDirectory&&*overrideBaseDirectory)
setBaseDirectory(overrideBaseDirectory, false);
if (overrideReplicateDirectory&&*overrideBaseDirectory)
setBaseDirectory(overrideReplicateDirectory, true);
StringBuffer tempdirstr;
const char *tempdir = globals->queryProp("@thorTempDirectory");
if (getConfigurationDirectory(globals->queryPropTree("Directories"),"temp","thor",globals->queryProp("@name"),tempdirstr))
tempdir = tempdirstr.str();
SetTempDir(tempdir,true);
useMemoryMappedRead(globals->getPropBool("@useMemoryMappedRead"));
LOG(MCdebugProgress, thorJob, "ThorSlave Version LCR - %d.%d started",THOR_VERSION_MAJOR,THOR_VERSION_MINOR);
StringBuffer url;
LOG(MCdebugProgress, thorJob, "Slave %s - temporary dir set to : %s", slfEp.getUrlStr(url).toCharArray(), queryTempDir());
#ifdef _WIN32
ULARGE_INTEGER userfree;
ULARGE_INTEGER total;
ULARGE_INTEGER free;
if (GetDiskFreeSpaceEx("c:\\",&userfree,&total,&free)&&total.QuadPart) {
unsigned pc = (unsigned)(free.QuadPart*100/total.QuadPart);
LOG(MCdebugProgress, thorJob, "Total disk space = %"I64F"d k", total.QuadPart/1000);
LOG(MCdebugProgress, thorJob, "Free disk space = %"I64F"d k", free.QuadPart/1000);
LOG(MCdebugProgress, thorJob, "%d%% disk free\n",pc);
}
#endif
if (getConfigurationDirectory(globals->queryPropTree("Directories"),"query","thor",globals->queryProp("@name"),str.clear()))
globals->setProp("@query_so_dir", str.str());
else
globals->getProp("@query_so_dir", str.clear());
if (str.length())
{
if (globals->getPropBool("Debug/@dllsToSlaves", true))
{
StringBuffer uniqSoPath;
if (PATHSEPCHAR == str.charAt(str.length()-1))
uniqSoPath.append(str.length()-1, str.str());
else
uniqSoPath.append(str);
uniqSoPath.append("_").append(getMachinePortBase());
str.swapWith(uniqSoPath);
globals->setProp("@query_so_dir", str.str());
}
PROGLOG("Using querySo directory: %s", str.str());
recursiveCreateDirectory(str.str());
}
multiThorMemoryThreshold = globals->getPropInt("@multiThorMemoryThreshold")*0x100000;
if (multiThorMemoryThreshold) {
StringBuffer lgname;
if (!globals->getProp("@multiThorResourceGroup",lgname))
globals->getProp("@nodeGroup",lgname);
if (lgname.length()) {
Owned<ILargeMemLimitNotify> notify = createMultiThorResourceMutex(lgname.str());
setMultiThorMemoryNotify(multiThorMemoryThreshold,notify);
PROGLOG("Multi-Thor resource limit for %s set to %"I64F"d",lgname.str(),(__int64)multiThorMemoryThreshold);
}
else
multiThorMemoryThreshold = 0;
}
slaveMain();
}
LOG(MCdebugProgress, thorJob, "ThorSlave terminated OK");
}
catch (IException *e)
{
FLLOG(MCexception(e), thorJob, e,"ThorSlave");
e->Release();
}
catch (CATCHALL)
{
FLLOG(MCerror, thorJob, "ThorSlave exiting because of uncaught exception");
}
ClearTempDirs();
if (multiThorMemoryThreshold)
setMultiThorMemoryNotify(0,NULL);
roxiemem::releaseRoxieHeap();
#ifdef ISDALICLIENT
closeEnvironment();
closedownClientProcess(); // dali client closedown
#endif
#ifdef USE_MP_LOG
stopLogMsgReceivers();
#endif
stopMPServer();
::Release(globals);
releaseAtoms(); // don't know why we can't use a module_exit to destruct these...
return 0;
}
int doSendQuery(const char * ip, unsigned port, const char * base)
{
ISocket * socket;
__int64 starttime, endtime;
StringBuffer ipstr;
try
{
if (strcmp(ip, ".")==0)
ip = GetCachedHostName();
else
{
const char *dash = strchr(ip, '-');
if (dash && isdigit(dash[1]) && dash>ip && isdigit(dash[-1]))
{
if (persistConnections)
UNIMPLEMENTED;
const char *startrange = dash-1;
while (isdigit(startrange[-1]))
startrange--;
char *endptr;
unsigned firstnum = atoi(startrange);
unsigned lastnum = strtol(dash+1, &endptr, 10);
if (lastnum > firstnum)
{
static unsigned counter;
static CriticalSection counterCrit;
CriticalBlock b(counterCrit);
ipstr.append(startrange - ip, ip).append((counter++ % (lastnum+1-firstnum)) + firstnum).append(endptr);
ip = ipstr.str();
printf("Sending to %s\n", ip);
}
}
}
starttime= get_cycles_now();
if (persistConnections)
{
if (!persistSocket) {
SocketEndpoint ep(ip,port);
persistSocket = ISocket::connect_timeout(ep, 1000);
}
socket = persistSocket;
}
else {
SocketEndpoint ep(ip,port);
socket = ISocket::connect_timeout(ep,1000);
}
}
catch(IException * e)
{
pexception("failed to connect to server", e);
return 1;
}
StringBuffer fullQuery;
bool useHTTP = forceHTTP || strstr(base, "<soap:Envelope") != NULL;
if (useHTTP)
{
StringBuffer newQuery;
Owned<IPTree> p = createPTreeFromXMLString(base, ipt_none, ptr_none);
const char *queryName = p->queryName();
if ((stricmp(queryName, "envelope") != 0) && (stricmp(queryName, "envelope") != 0))
{
if (queryNameOverride.length())
queryName = queryNameOverride;
newQuery.appendf("<Envelope xmlns:soap=\"http://schemas.xmlsoap.org/soap/envelope/\"><Body><%sRequest>", queryName);
Owned<IPTreeIterator> elements = p->getElements("./*");
ForEach(*elements)
{
IPTree &elem = elements->query();
toXML(&elem, newQuery, 0, XML_SingleQuoteAttributeValues);
}
newQuery.appendf("</%sRequest></Body></Envelope>", queryName);
base = newQuery.str();
}
// note - don't support queryname override unless original query is xml
fullQuery.appendf("POST /doc HTTP/1.0\r\nContent-Type: application/x-www-form-urlencoded\r\nContent-Length: %d\r\n\r\n", (int) strlen(base)).append(base);
}
else
{
if (sendToSocket)
int readResults(ISocket * socket, bool readBlocked, bool useHTTP, StringBuffer &result)
{
if (readBlocked)
socket->set_block_mode(BF_SYNC_TRANSFER_PULL,0,60*1000);
unsigned len;
bool is_status;
bool isBlockedResult;
for (;;)
{
if (delay)
MilliSleep(delay);
is_status = false;
isBlockedResult = false;
try
{
if (useHTTP)
len = 0x10000;
else if (readBlocked)
len = socket->receive_block_size();
else
{
socket->read(&len, sizeof(len));
_WINREV(len);
}
}
catch(IException * e)
{
if (manyResults)
showMessage("End of result multiple set\n");
else
pexception("failed to read len data", e);
e->Release();
return 1;
}
if (len == 0)
{
if (manyResults)
{
showMessage("----End of result set----\n");
continue;
}
break;
}
bool isSpecial = false;
bool pluginRequest = false;
bool dataBlockRequest = false;
if (len & 0x80000000)
{
unsigned char flag;
isSpecial = true;
socket->read(&flag, sizeof(flag));
switch (flag)
{
case '-':
if (echoResults)
fputs("Error:", stdout);
if (saveResults && trace != NULL)
fputs("Error:", trace);
break;
case 'D':
showMessage("request for datablock\n");
dataBlockRequest = true;
break;
case 'P':
showMessage("request for plugin\n");
pluginRequest = true;
break;
case 'S':
if (showStatus)
showMessage("Status:");
is_status=true;
break;
case 'T':
showMessage("Timing:\n");
break;
case 'X':
showMessage("---Compound query finished---\n");
return 1;
case 'R':
isBlockedResult = true;
break;
}
len &= 0x7FFFFFFF;
len--; // flag already read
}
char * mem = (char*) malloc(len+1);
char * t = mem;
unsigned sendlen = len;
t[len]=0;
try
{
if (useHTTP)
{
try
{
socket->read(t, 0, len, sendlen);
}
catch (IException *E)
{
if (E->errorCode()!= JSOCKERR_graceful_close)
throw;
E->Release();
break;
}
if (!sendlen)
break;
}
else if (readBlocked)
socket->receive_block(t, len);
else
socket->read(t, len);
}
catch(IException * e)
{
pexception("failed to read data", e);
return 1;
}
if (pluginRequest)
{
//Not very robust! A poor man's implementation for testing...
StringBuffer dllname, libname;
const char * dot = strchr(t, '.');
dllname.append("\\edata\\bin\\debug\\").append(t);
libname.append("\\edata\\bin\\debug\\").append(dot-t,t).append(".lib");
sendFile(dllname.str(), socket);
sendFile(libname.str(), socket);
}
else if (dataBlockRequest)
{
//Not very robust! A poor man's implementation for testing...
offset_t offset;
memcpy(&offset, t, sizeof(offset));
_WINREV(offset);
sendFileChunk(t+sizeof(offset), offset, socket);
}
else
{
if (isBlockedResult)
{
t += 8;
t += strlen(t)+1;
sendlen -= (t - mem);
}
if (echoResults && (!is_status || showStatus))
{
fwrite(t, sendlen, 1, stdout);
fflush(stdout);
}
if (!is_status)
result.append(sendlen, t);
}
free(mem);
if (abortAfterFirst)
return 0;
}
return 0;
}
