void CConfigGenEngine::createFakePlugins(StringBuffer& destFilePath) const
{
String destFilePathStr(destFilePath);
String* tmpstr = destFilePathStr.toLowerCase();
if (!tmpstr->endsWith("plugins.xml"))
{
int index = tmpstr->indexOf("plugins.xml");
destFilePath.remove(index + 11, destFilePath.length() - (index + 11));
}
delete tmpstr;
StringBuffer tmpoutbuf("<?xml version=\"1.0\" encoding=\"UTF-8\"?><Plugins/>");
if (m_instances.ordinality() > 1 && strcmp(m_process.queryName(), XML_TAG_ESPPROCESS))
destFilePath.replaceString("@temp"PATHSEPSTR, m_cachePath);
else
{
char tempPath[_MAX_PATH];
getTempPath(tempPath, sizeof(tempPath), m_name);
ensurePath(tempPath);
destFilePath.replaceString("@temp"PATHSEPSTR, tempPath);
}
Owned<IFile> pTargetFile = createIFile(destFilePath.str());
if (pTargetFile->exists() && pTargetFile->isReadOnly())
pTargetFile->setReadOnly(false);
Owned<IFileIO> pTargetFileIO = pTargetFile->open(IFOcreate);
pTargetFileIO->write( 0, tmpoutbuf.length(), tmpoutbuf.str());
m_envDepEngine.addTempFile(destFilePath.str());
}
void CLogSerializer::Remove()
{
Close();
Owned<IFile> file = createIFile(m_FilePath.str());
if(file.get() && file->exists() == true)
file->remove();
}
virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData) override
{
isLocal = 0 != (TIWlocal & helper->getFlags());
mpTag = container.queryJobChannel().deserializeMPTag(data);
mpTag2 = container.queryJobChannel().deserializeMPTag(data);
data.read(active);
if (active)
{
data.read(logicalFilename);
partDesc.setown(deserializePartFileDescriptor(data));
}
data.read(singlePartKey);
data.read(refactor);
if (singlePartKey)
buildTlk = false;
else
{
data.read(buildTlk);
if (firstNode())
{
if (buildTlk)
tlkDesc.setown(deserializePartFileDescriptor(data));
else if (!isLocal) // existing tlk then..
{
tlkDesc.setown(deserializePartFileDescriptor(data));
unsigned c;
data.read(c);
while (c--)
{
RemoteFilename rf;
rf.deserialize(data);
if (!existingTlkIFile)
{
Owned<IFile> iFile = createIFile(rf);
if (iFile->exists())
existingTlkIFile.set(iFile);
}
}
if (!existingTlkIFile)
throw MakeActivityException(this, TE_FileNotFound, "Top level key part does not exist, for key");
}
}
}
IOutputMetaData * diskSize = helper->queryDiskRecordSize();
assertex(!(diskSize->getMetaFlags() & MDFneedserializedisk));
if (diskSize->isVariableSize())
{
if (TIWmaxlength & helper->getFlags())
maxDiskRecordSize = helper->getMaxKeySize();
else
maxDiskRecordSize = KEYBUILD_MAXLENGTH; //Current default behaviour, could be improved in the future
}
else
maxDiskRecordSize = diskSize->getFixedSize();
reportOverflow = false;
}
void CLogSerializer::Rollover(const char* ClosedPrefix)
{
Close();
Owned<IFile> file = createIFile(m_FilePath.str());
if(file.get() && file->exists() == true)
{
StringBuffer newFileName;
GetRolloverFileName(m_FileName,newFileName,ClosedPrefix);
file->rename(newFileName.str());
}
}
void CppCompiler::writeLogFile(const char* filepath, StringBuffer& log)
{
if(!filepath || !*filepath || !log.length())
return;
Owned <IFile> f = createIFile(filepath);
if(f->exists())
f->remove();
Owned <IFileIO> fio = f->open(IFOcreaterw);
if(fio.get())
fio->write(0, log.length(), log.str());
}
bool CLogSerializer::EnsureDirectory(StringBuffer& Dir)
{
try{
Owned<IFile> pDirectory = createIFile(Dir.str());
if(pDirectory->exists() == true)
return true;
return pDirectory->createDirectory();
}
catch(IException *ex)
{
ex->Release();
}
return false;
}
void CLocalDataLogger::Init(const char* logDirectory,const char* ext,const char* UrlRoot, unsigned TimerPeriod,unsigned CacheTimeoutPeriod)
{
if(!logDirectory || !ext)
throw MakeStringException(-1,"Invalid parameters passed to cLocalDataLogger::Init");
StringBuffer pathtodir;
pathtodir.appendf("%s",logDirectory);
Owned<IFile> pDirectory = createIFile(pathtodir.str());
if(pDirectory->exists() == false)
pDirectory->createDirectory();
m_UrlRoot.appendf("%s",UrlRoot);
m_logDirectory.append(logDirectory);
StringBuffer dirpath;
dirpath.appendf("%s",logDirectory);
m_CleanupThread.setown(new CFileCleanupThread(dirpath,ext,TimerPeriod,CacheTimeoutPeriod));
m_CleanupThread->start();
}
//---------------------------------------------------------------------------
// beforeDeploy
//---------------------------------------------------------------------------
void CConfigGenEngine::beforeDeploy()
{
m_installFiles.clear();
char tempPath[_MAX_PATH];
getTempPath(tempPath, sizeof(tempPath), m_name);
m_envDepEngine.addTempDirectory( tempPath );
if (m_instances.ordinality() > 1)
{
strcat(tempPath, "Cache");
char* pszEnd = tempPath + strlen(tempPath);
Owned<IFile> pFile = createIFile(tempPath);
int i = 1;
while (pFile->exists()) { //dir/file exists
itoa(++i, pszEnd, 10);
pFile.setown( createIFile(tempPath) );
}
strcat(tempPath, PATHSEPSTR);
m_cachePath.set( tempPath );
}
else
m_cachePath.set( tempPath );
ensurePath(tempPath);
determineInstallFiles(m_process, m_installFiles);
getCallback().installFileListChanged();
if (m_instances.ordinality() > 1)
{
EnvMachineOS os = m_envDepEngine.lookupMachineOS( m_instances.item(0) );
m_curInstance = "Cache";
copyInstallFiles("Cache", -1, tempPath, os);
}
}
int main(int argc, const char *argv[])
{
InitModuleObjects();
EnableSEHtoExceptionMapping();
NoQuickEditSection xxx;
Owned<IFile> file = createIFile("dfuserver.xml");
if (file->exists())
globals.setown(createPTreeFromXMLFile("dfuserver.xml", ipt_caseInsensitive));
else
globals.setown(readOldIni());
for (unsigned i=1;i<(unsigned)argc;i++) {
const char *arg = argv[i];
StringBuffer prop("@");
StringBuffer val;
while (*arg && *arg != '=')
prop.append(*arg++);
if (*arg) {
arg++;
while (isspace(*arg))
arg++;
val.append(arg);
prop.clip();
val.clip();
if (prop.length()>1)
globals->setProp(prop.str(), val.str());
}
}
StringBuffer daliServer;
StringBuffer queue;
if (!globals->getProp("@DALISERVERS", daliServer)||!globals->getProp("@QUEUE", queue)) {
usage();
globals.clear();
releaseAtoms();
return 1;
}
Owned<IFile> sentinelFile;
bool stop = globals->getPropInt("@STOP",0)!=0;
if (!stop) {
sentinelFile.setown(createSentinelTarget());
removeSentinelFile(sentinelFile);
StringBuffer logname;
StringBuffer logdir;
if (!getConfigurationDirectory(globals->queryPropTree("Directories"),"log","dfuserver",globals->queryProp("@name"),logdir))
globals->getProp("@LOG_DIR", logdir);
if (logdir.length() && recursiveCreateDirectory(logdir.str()))
logname.append(logdir);
else
appendCurrentDirectory(logname, true);
if (logname.length() && logname.charAt(logname.length()-1) != PATHSEPCHAR)
logname.append(PATHSEPCHAR);
logname.append("dfuserver");
StringBuffer aliasLogName(logname);
aliasLogName.append(".log");
fileMsgHandler = getRollingFileLogMsgHandler(logname.str(), ".log", MSGFIELD_STANDARD, false, true, NULL, aliasLogName.str());
queryLogMsgManager()->addMonitorOwn(fileMsgHandler, getCategoryLogMsgFilter(MSGAUD_all, MSGCLS_all, 1000));
}
StringBuffer ftslogdir;
if (getConfigurationDirectory(globals->queryPropTree("Directories"),"log","ftslave",globals->queryProp("@name"),ftslogdir)) // NB instance deliberately dfuserver's
setFtSlaveLogDir(ftslogdir.str());
setRemoteSpawnSSH(
globals->queryProp("SSH/@SSHidentityfile"),
globals->queryProp("SSH/@SSHusername"),
globals->queryProp("SSH/@SSHpassword"),
globals->getPropInt("SSH/@SSHtimeout",0),
globals->getPropInt("SSH/@SSHretries",3),
"run_");
bool enableSNMP = globals->getPropInt("@enableSNMP")!=0;
CSDSServerStatus *serverstatus=NULL;
Owned<IReplicateServer> replserver;
try {
Owned<IGroup> serverGroup = createIGroup(daliServer.str(),DALI_SERVER_PORT);
initClientProcess(serverGroup, DCR_DfuServer, 0, NULL, NULL, stop?(1000*30):MP_WAIT_FOREVER);
setPasswordsFromSDS();
if(!stop)
{
if (globals->getPropBool("@enableSysLog",true))
UseSysLogForOperatorMessages();
serverstatus = new CSDSServerStatus("DFUserver");
setDaliServixSocketCaching(true); // speeds up lixux operations
startLogMsgParentReceiver(); // for auditing
connectLogMsgManagerToDali();
engine.setown(createDFUengine());
addAbortHandler(exitDFUserver);
}
const char *q = queue.str();
loop {
StringBuffer subq;
const char *comma = strchr(q,',');
if (comma)
subq.append(comma-q,q);
else
subq.append(q);
if (stop) {
stopDFUserver(subq.str());
}
else {
StringBuffer mask;
mask.appendf("Queue[@name=\"%s\"][1]",subq.str());
IPropertyTree *t=serverstatus->queryProperties()->queryPropTree(mask.str());
if (t)
t->setPropInt("@num",t->getPropInt("@num",0)+1);
else {
t = createPTree();
t->setProp("@name",subq.str());
t->setPropInt("@num",1);
serverstatus->queryProperties()->addPropTree("Queue",t);
}
serverstatus->commitProperties();
engine->setDefaultTransferBufferSize((size32_t)globals->getPropInt("@transferBufferSize"));
engine->startListener(subq.str(),serverstatus);
}
if (!comma)
break;
q = comma+1;
if (!*q)
break;
}
q = globals->queryProp("@MONITORQUEUE");
if (q&&*q) {
if (stop) {
stopDFUserver(q);
}
else {
IPropertyTree *t=serverstatus->queryProperties()->addPropTree("MonitorQueue",createPTree());
t->setProp("@name",q);
engine->startMonitor(q,serverstatus,globals->getPropInt("@MONITORINTERVAL",60)*1000);
}
}
q = globals->queryProp("@REPLICATEQUEUE");
if (q&&*q) {
if (stop) {
// TBD?
}
else {
replserver.setown(createReplicateServer(q));
replserver->runServer();
}
}
if (!stop) {
serverstatus->commitProperties();
writeSentinelFile(sentinelFile);
engine->joinListeners();
if (replserver.get())
replserver->stopServer();
LOG(MCprogress, unknownJob, "Exiting");
}
}
catch(IException *e){
EXCLOG(e, "DFU Server Exception: ");
e->Release();
}
catch (const char *s) {
WARNLOG("DFU: %s",s);
}
delete serverstatus;
if (stop)
Sleep(2000); // give time to stop
engine.clear();
globals.clear();
closeEnvironment();
closedownClientProcess();
UseSysLogForOperatorMessages(false);
setDaliServixSocketCaching(false);
releaseAtoms();
return 0;
}
int main( int argc, char *argv[] )
{
#if defined(WIN32) && defined(_DEBUG)
int tmpFlag = _CrtSetDbgFlag( _CRTDBG_REPORT_FLAG );
tmpFlag |= _CRTDBG_LEAK_CHECK_DF;
_CrtSetDbgFlag( tmpFlag );
#endif
InitModuleObjects();
addAbortHandler(ControlHandler);
EnableSEHtoExceptionMapping();
dummyProc();
#ifndef __64BIT__
// Restrict stack sizes on 32-bit systems
Thread::setDefaultStackSize(0x10000); // NB under windows requires linker setting (/stack:)
#endif
#ifdef _WIN32
Owned<CReleaseMutex> globalNamedMutex;
#endif
if (globals)
globals->Release();
{
Owned<IFile> iFile = createIFile("thor.xml");
globals = iFile->exists() ? createPTree(*iFile, ipt_caseInsensitive) : createPTree("Thor", ipt_caseInsensitive);
}
unsigned multiThorMemoryThreshold = 0;
Owned<IException> unregisterException;
try
{
if (argc==1)
{
usage();
return 1;
}
cmdArgs = argv+1;
mergeCmdParams(globals);
cmdArgs = argv+1;
const char *master = globals->queryProp("@MASTER");
if (!master)
usage();
const char *slave = globals->queryProp("@SLAVE");
if (slave)
{
slfEp.set(slave);
localHostToNIC(slfEp);
}
else
slfEp.setLocalHost(0);
mySlaveNum = globals->getPropInt("@SLAVENUM");
setMachinePortBase(slfEp.port);
slfEp.port = getMachinePortBase();
startSlaveLog();
setSlaveAffinity(globals->getPropInt("@SLAVEPROCESSNUM"));
startMPServer(getFixedPort(TPORT_mp));
#ifdef USE_MP_LOG
startLogMsgParentReceiver();
LOG(MCdebugProgress, thorJob, "MPServer started on port %d", getFixedPort(TPORT_mp));
#endif
SocketEndpoint masterEp(master);
localHostToNIC(masterEp);
setMasterPortBase(masterEp.port);
markNodeCentral(masterEp);
if (RegisterSelf(masterEp))
{
if (globals->getPropBool("Debug/@slaveDaliClient"))
enableThorSlaveAsDaliClient();
IDaFileSrvHook *daFileSrvHook = queryDaFileSrvHook();
if (daFileSrvHook) // probably always installed
daFileSrvHook->addFilters(globals->queryPropTree("NAS"), &slfEp);
StringBuffer thorPath;
globals->getProp("@thorPath", thorPath);
recursiveCreateDirectory(thorPath.str());
int err = _chdir(thorPath.str());
if (err)
{
IException *e = makeErrnoExceptionV(-1, "Failed to change dir to '%s'", thorPath.str());
FLLOG(MCexception(e), thorJob, e);
throw e;
}
// Initialization from globals
setIORetryCount(globals->getPropInt("Debug/@ioRetries")); // default == 0 == off
StringBuffer str;
if (globals->getProp("@externalProgDir", str.clear()))
_mkdir(str.str());
else
globals->setProp("@externalProgDir", thorPath);
const char * overrideBaseDirectory = globals->queryProp("@thorDataDirectory");
const char * overrideReplicateDirectory = globals->queryProp("@thorReplicateDirectory");
StringBuffer datadir;
StringBuffer repdir;
if (getConfigurationDirectory(globals->queryPropTree("Directories"),"data","thor",globals->queryProp("@name"),datadir))
overrideBaseDirectory = datadir.str();
if (getConfigurationDirectory(globals->queryPropTree("Directories"),"mirror","thor",globals->queryProp("@name"),repdir))
overrideReplicateDirectory = repdir.str();
if (overrideBaseDirectory&&*overrideBaseDirectory)
setBaseDirectory(overrideBaseDirectory, false);
if (overrideReplicateDirectory&&*overrideBaseDirectory)
setBaseDirectory(overrideReplicateDirectory, true);
StringBuffer tempDirStr;
if (getConfigurationDirectory(globals->queryPropTree("Directories"),"temp","thor",globals->queryProp("@name"), tempDirStr))
globals->setProp("@thorTempDirectory", tempDirStr.str());
else
tempDirStr.append(globals->queryProp("@thorTempDirectory"));
addPathSepChar(tempDirStr).append(getMachinePortBase());
logDiskSpace(); // Log before temp space is cleared
SetTempDir(tempDirStr.str(), "thtmp", true);
useMemoryMappedRead(globals->getPropBool("@useMemoryMappedRead"));
LOG(MCdebugProgress, thorJob, "ThorSlave Version LCR - %d.%d started",THOR_VERSION_MAJOR,THOR_VERSION_MINOR);
StringBuffer url;
LOG(MCdebugProgress, thorJob, "Slave %s - temporary dir set to : %s", slfEp.getUrlStr(url).str(), queryTempDir());
#ifdef _WIN32
ULARGE_INTEGER userfree;
ULARGE_INTEGER total;
ULARGE_INTEGER free;
if (GetDiskFreeSpaceEx("c:\\",&userfree,&total,&free)&&total.QuadPart) {
unsigned pc = (unsigned)(free.QuadPart*100/total.QuadPart);
LOG(MCdebugProgress, thorJob, "Total disk space = %" I64F "d k", total.QuadPart/1000);
LOG(MCdebugProgress, thorJob, "Free disk space = %" I64F "d k", free.QuadPart/1000);
LOG(MCdebugProgress, thorJob, "%d%% disk free\n",pc);
}
#endif
if (getConfigurationDirectory(globals->queryPropTree("Directories"),"query","thor",globals->queryProp("@name"),str.clear()))
globals->setProp("@query_so_dir", str.str());
else
globals->getProp("@query_so_dir", str.clear());
if (str.length())
{
if (globals->getPropBool("Debug/@dllsToSlaves", true))
{
StringBuffer uniqSoPath;
if (PATHSEPCHAR == str.charAt(str.length()-1))
uniqSoPath.append(str.length()-1, str.str());
else
uniqSoPath.append(str);
uniqSoPath.append("_").append(getMachinePortBase());
str.swapWith(uniqSoPath);
globals->setProp("@query_so_dir", str.str());
}
PROGLOG("Using querySo directory: %s", str.str());
recursiveCreateDirectory(str.str());
}
multiThorMemoryThreshold = globals->getPropInt("@multiThorMemoryThreshold")*0x100000;
if (multiThorMemoryThreshold) {
StringBuffer lgname;
if (!globals->getProp("@multiThorResourceGroup",lgname))
globals->getProp("@nodeGroup",lgname);
if (lgname.length()) {
Owned<ILargeMemLimitNotify> notify = createMultiThorResourceMutex(lgname.str());
setMultiThorMemoryNotify(multiThorMemoryThreshold,notify);
PROGLOG("Multi-Thor resource limit for %s set to %" I64F "d",lgname.str(),(__int64)multiThorMemoryThreshold);
}
else
multiThorMemoryThreshold = 0;
}
slaveMain(jobListenerStopped);
}
LOG(MCdebugProgress, thorJob, "ThorSlave terminated OK");
}
catch (IException *e)
{
if (!jobListenerStopped)
FLLOG(MCexception(e), thorJob, e,"ThorSlave");
unregisterException.setown(e);
}
ClearTempDirs();
if (multiThorMemoryThreshold)
setMultiThorMemoryNotify(0,NULL);
roxiemem::releaseRoxieHeap();
if (unregisterException.get())
UnregisterSelf(unregisterException);
if (globals->getPropBool("Debug/@slaveDaliClient"))
disableThorSlaveAsDaliClient();
#ifdef USE_MP_LOG
stopLogMsgReceivers();
#endif
stopMPServer();
::Release(globals);
releaseAtoms(); // don't know why we can't use a module_exit to destruct these...
ExitModuleObjects(); // not necessary, atexit will call, but good for leak checking
return 0;
}
bool loadPwds()
{
try
{
if (!pwFile.length())
throw MakeStringException(-1, "htpasswd Password file not specified");
Owned<IFile> file = createIFile(pwFile.str());
if (!file->exists())
{
userMap.kill();
throw MakeStringException(-1, "htpasswd Password file does not exist");
}
bool isDir;
offset_t size;
CDateTime whenChanged;
file->getInfo(isDir,size,whenChanged);
if (isDir)
{
userMap.kill();
throw MakeStringException(-1, "htpasswd Password file specifies a directory");
}
if (0 == whenChanged.compare(pwFileLastMod))
return true;//Don't reload if file unchanged
userMap.kill();
OwnedIFileIO io = file->open(IFOread);
if (!io)
throw MakeStringException(-1, "htpasswd Unable to open Password file");
MemoryBuffer mb;
size32_t count = read(io, 0, (size32_t)-1, mb);
if (0 == count)
throw MakeStringException(-1, "htpasswd Password file is empty");
mb.append((char)NULL);
char * p = (char*)mb.toByteArray();
char *saveptr;
const char * seps = "\f\r\n";
char * next = strtok_r(p, seps, &saveptr);
if (next)
{
do
{
char * colon = strchr(next,':');
if (NULL == colon)
throw MakeStringException(-1, "htpasswd Password file appears malformed");
*colon = (char)NULL;
userMap.setValue(next, colon+1);//username, enctypted password
} while (next = strtok_r(NULL, seps, &saveptr));
}
io->close();
pwFileLastMod = whenChanged;//remember when last changed
}
catch(IException*)
{
throw MakeStringException(-1, "htpasswd Exception accessing Password file");
}
return true;
}
int main( int argc, char *argv[] )
{
#if defined(WIN32) && defined(_DEBUG)
int tmpFlag = _CrtSetDbgFlag( _CRTDBG_REPORT_FLAG );
tmpFlag |= _CRTDBG_LEAK_CHECK_DF;
_CrtSetDbgFlag( tmpFlag );
#endif
InitModuleObjects();
addAbortHandler(ControlHandler);
EnableSEHtoExceptionMapping();
dummyProc();
#ifndef __64BIT__
Thread::setDefaultStackSize(0x10000); // NB under windows requires linker setting (/stack:)
#endif
#ifdef _WIN32
Owned<CReleaseMutex> globalNamedMutex;
#endif
if (globals)
globals->Release();
{
Owned<IFile> iFile = createIFile("thor.xml");
globals = iFile->exists() ? createPTree(*iFile, ipt_caseInsensitive) : createPTree("Thor", ipt_caseInsensitive);
}
unsigned multiThorMemoryThreshold = 0;
try {
if (argc==1)
{
usage();
return 1;
}
cmdArgs = argv+1;
mergeCmdParams(globals);
cmdArgs = argv+1;
const char *master = globals->queryProp("@MASTER");
if (!master)
usage();
const char *slave = globals->queryProp("@SLAVE");
if (slave)
{
slfEp.set(slave);
localHostToNIC(slfEp);
}
else
slfEp.setLocalHost(0);
if (globals->hasProp("@SLAVENUM"))
mySlaveNum = atoi(globals->queryProp("@SLAVENUM"));
else
mySlaveNum = slfEp.port; // shouldn't happen, provided by script
setMachinePortBase(slfEp.port);
slfEp.port = getMachinePortBase();
startSlaveLog();
startMPServer(getFixedPort(TPORT_mp));
#ifdef USE_MP_LOG
startLogMsgParentReceiver();
LOG(MCdebugProgress, thorJob, "MPServer started on port %d", getFixedPort(TPORT_mp));
#endif
SocketEndpoint masterEp(master);
localHostToNIC(masterEp);
setMasterPortBase(masterEp.port);
markNodeCentral(masterEp);
if (RegisterSelf(masterEp))
{
#define ISDALICLIENT // JCSMORE plugins *can* access dali - though I think we should probably prohibit somehow.
#ifdef ISDALICLIENT
const char *daliServers = globals->queryProp("@DALISERVERS");
if (!daliServers)
{
LOG(MCerror, thorJob, "No Dali server list specified\n");
return 1;
}
Owned<IGroup> serverGroup = createIGroup(daliServers, DALI_SERVER_PORT);
unsigned retry = 0;
loop {
try {
LOG(MCdebugProgress, thorJob, "calling initClientProcess");
initClientProcess(serverGroup,DCR_ThorSlave, getFixedPort(TPORT_mp));
break;
}
catch (IJSOCK_Exception *e) {
if ((e->errorCode()!=JSOCKERR_port_in_use))
throw;
FLLOG(MCexception(e), thorJob, e,"InitClientProcess");
if (retry++>10)
throw;
e->Release();
LOG(MCdebugProgress, thorJob, "Retrying");
Sleep(retry*2000);
}
}
setPasswordsFromSDS();
#endif
IDaFileSrvHook *daFileSrvHook = queryDaFileSrvHook();
if (daFileSrvHook) // probably always installed
daFileSrvHook->addSubnetFilters(globals->queryPropTree("NAS"), NULL);
StringBuffer thorPath;
globals->getProp("@thorPath", thorPath);
recursiveCreateDirectory(thorPath.str());
int err = _chdir(thorPath.str());
if (err)
{
IException *e = MakeErrnoException(-1, "Failed to change dir to '%s'",thorPath.str());
FLLOG(MCexception(e), thorJob, e);
throw e;
}
// Initialization from globals
setIORetryCount(globals->getPropInt("Debug/@ioRetries")); // default == 0 == off
StringBuffer str;
if (globals->getProp("@externalProgDir", str.clear()))
_mkdir(str.str());
else
globals->setProp("@externalProgDir", thorPath);
const char * overrideBaseDirectory = globals->queryProp("@thorDataDirectory");
const char * overrideReplicateDirectory = globals->queryProp("@thorReplicateDirectory");
StringBuffer datadir;
StringBuffer repdir;
if (getConfigurationDirectory(globals->queryPropTree("Directories"),"data","thor",globals->queryProp("@name"),datadir))
overrideBaseDirectory = datadir.str();
if (getConfigurationDirectory(globals->queryPropTree("Directories"),"mirror","thor",globals->queryProp("@name"),repdir))
overrideReplicateDirectory = repdir.str();
if (overrideBaseDirectory&&*overrideBaseDirectory)
setBaseDirectory(overrideBaseDirectory, false);
if (overrideReplicateDirectory&&*overrideBaseDirectory)
setBaseDirectory(overrideReplicateDirectory, true);
StringBuffer tempdirstr;
const char *tempdir = globals->queryProp("@thorTempDirectory");
if (getConfigurationDirectory(globals->queryPropTree("Directories"),"temp","thor",globals->queryProp("@name"),tempdirstr))
tempdir = tempdirstr.str();
SetTempDir(tempdir,true);
useMemoryMappedRead(globals->getPropBool("@useMemoryMappedRead"));
LOG(MCdebugProgress, thorJob, "ThorSlave Version LCR - %d.%d started",THOR_VERSION_MAJOR,THOR_VERSION_MINOR);
StringBuffer url;
LOG(MCdebugProgress, thorJob, "Slave %s - temporary dir set to : %s", slfEp.getUrlStr(url).toCharArray(), queryTempDir());
#ifdef _WIN32
ULARGE_INTEGER userfree;
ULARGE_INTEGER total;
ULARGE_INTEGER free;
if (GetDiskFreeSpaceEx("c:\\",&userfree,&total,&free)&&total.QuadPart) {
unsigned pc = (unsigned)(free.QuadPart*100/total.QuadPart);
LOG(MCdebugProgress, thorJob, "Total disk space = %"I64F"d k", total.QuadPart/1000);
LOG(MCdebugProgress, thorJob, "Free disk space = %"I64F"d k", free.QuadPart/1000);
LOG(MCdebugProgress, thorJob, "%d%% disk free\n",pc);
}
#endif
if (getConfigurationDirectory(globals->queryPropTree("Directories"),"query","thor",globals->queryProp("@name"),str.clear()))
globals->setProp("@query_so_dir", str.str());
else
globals->getProp("@query_so_dir", str.clear());
if (str.length())
{
if (globals->getPropBool("Debug/@dllsToSlaves", true))
{
StringBuffer uniqSoPath;
if (PATHSEPCHAR == str.charAt(str.length()-1))
uniqSoPath.append(str.length()-1, str.str());
else
uniqSoPath.append(str);
uniqSoPath.append("_").append(getMachinePortBase());
str.swapWith(uniqSoPath);
globals->setProp("@query_so_dir", str.str());
}
PROGLOG("Using querySo directory: %s", str.str());
recursiveCreateDirectory(str.str());
}
multiThorMemoryThreshold = globals->getPropInt("@multiThorMemoryThreshold")*0x100000;
if (multiThorMemoryThreshold) {
StringBuffer lgname;
if (!globals->getProp("@multiThorResourceGroup",lgname))
globals->getProp("@nodeGroup",lgname);
if (lgname.length()) {
Owned<ILargeMemLimitNotify> notify = createMultiThorResourceMutex(lgname.str());
setMultiThorMemoryNotify(multiThorMemoryThreshold,notify);
PROGLOG("Multi-Thor resource limit for %s set to %"I64F"d",lgname.str(),(__int64)multiThorMemoryThreshold);
}
else
multiThorMemoryThreshold = 0;
}
slaveMain();
}
LOG(MCdebugProgress, thorJob, "ThorSlave terminated OK");
}
catch (IException *e)
{
FLLOG(MCexception(e), thorJob, e,"ThorSlave");
e->Release();
}
catch (CATCHALL)
{
FLLOG(MCerror, thorJob, "ThorSlave exiting because of uncaught exception");
}
ClearTempDirs();
if (multiThorMemoryThreshold)
setMultiThorMemoryNotify(0,NULL);
roxiemem::releaseRoxieHeap();
#ifdef ISDALICLIENT
closeEnvironment();
closedownClientProcess(); // dali client closedown
#endif
#ifdef USE_MP_LOG
stopLogMsgReceivers();
#endif
stopMPServer();
::Release(globals);
releaseAtoms(); // don't know why we can't use a module_exit to destruct these...
return 0;
}
int main(int argc, const char* argv[])
{
InitModuleObjects();
if ((argc >= 2) && ((stricmp(argv[1], "/version") == 0) || (stricmp(argv[1], "-v") == 0)
|| (stricmp(argv[1], "--version") == 0)))
{
printVersion();
return 0;
}
Owned<IFile> inifile = createIFile("dfuplus.ini");
if(argc < 2 && !(inifile->exists() && inifile->size() > 0))
{
handleSyntax();
return 0;
}
if ((argc >= 2) && ((argv[1][0]=='/' || argv[1][0]=='-') && (argv[1][1]=='?' || argv[1][1]=='h')))
{
handleSyntax();
return 0;
}
//queryLogMsgManager()->changeMonitorFilterOwn(queryStderrLogMsgHandler(), getPassNoneLogMsgFilter());
Owned<IProperties> globals = createProperties("dfuplus.ini", true);
if(!build_globals(argc, argv, globals))
{
fprintf(stderr, "ERROR: Invalid command syntax.\n");
releaseAtoms();
return DFUERR_InvalidCommandSyntax;
}
const char* action = globals->queryProp("action");
if(!action || !*action)
{
handleSyntax();
fprintf(stderr, "\nERROR: please specify one action");
releaseAtoms();
return DFUERR_TooFewArguments;
}
const char* server = globals->queryProp("server");
if (!server || !*server) {
if (stricmp(action,"dafilesrv")==0)
globals->setProp("server","127.0.0.1"); // dummy
else {
fprintf(stderr, "ERROR: Esp server url not specified.\n");
releaseAtoms();
return DFUERR_TooFewArguments;
}
}
try
{
Owned<CDfuPlusHelper> helper = new CDfuPlusHelper(LINK(globals.get()));
helper->doit();
}
catch(IException* e)
{
StringBuffer errmsg;
e->errorMessage(errmsg);
fprintf(stderr, "%s\n", errmsg.str());
}
releaseAtoms();
return 0;
}
void copyCompress(const char *from, const char *to, size32_t rowsize, bool fast, bool flzstrm, bool stats)
{
Owned<IFile> srcfile = createIFile(from);
Owned<IFileIO> baseio = srcfile->open(IFOread);
if (!baseio) {
printf("ERROR: could not open '%s' for read\n",from);
doexit(3);
}
Owned<ICompressedFileIO> cmpio = createCompressedFileReader(baseio);
Owned<IFileIOStream> flzstrmsrc = cmpio?NULL:createFastLZStreamRead(baseio);
bool plaincopy = false;
IFileIO *srcio = NULL;
if (cmpio) {
srcio = cmpio;
if (rowsize&&(cmpio->recordSize()==rowsize))
plaincopy = true;
else if (!rowsize) {
if (fast&&(cmpio->method()==COMPRESS_METHOD_FASTLZ))
plaincopy = true;
else if (!fast&&(cmpio->method()==COMPRESS_METHOD_LZW))
plaincopy = true;
}
}
else if (flzstrmsrc) {
if (flzstrm)
plaincopy = true;
}
else
srcio = baseio;
if (plaincopy) {
cmpio.clear();
srcio = baseio.get();
}
Owned<IFile> dstfile = createIFile(to);
StringBuffer fulldst;
if (dstfile->isDirectory()==foundYes) {
dstfile.clear();
addPathSepChar(fulldst.append(to)).append(pathTail(from));
to = fulldst.str();
dstfile.setown(createIFile(to));
}
if (dstfile->exists()) {
printf("ERROR: file '%s' already exists\n",to);
doexit(4);
}
unsigned start;
unsigned startu;
if (stats) {
start = msTick();
startu = usTick();
}
Owned<IFileIO> dstio;
Owned<IFileIOStream> flzstrmdst;
if (plaincopy||flzstrm) {
dstio.setown(dstfile->open(IFOcreate));
if (dstio&&!plaincopy)
flzstrmdst.setown(createFastLZStreamWrite(dstio));
}
else
dstio.setown(createCompressedFileWriter(dstfile,rowsize,false,true,NULL,fast));
if (!dstio) {
printf("ERROR: could not open '%s' for write\n",to);
doexit(5);
}
#ifdef __linux__
// this is not really needed in windows - if it is we will have to
// test the file extension - .exe, .bat
struct stat info;
if (stat(from, &info) == 0) // cannot fail - exception would have been thrown above
dstfile->setCreateFlags(info.st_mode&(S_IRUSR|S_IRGRP|S_IROTH|S_IWUSR|S_IWGRP|S_IWOTH|S_IXUSR|S_IXGRP|S_IXOTH));
#endif
MemoryAttr mb;
void * buffer = mb.allocate(BUFFERSIZE);
offset_t offset = 0;
try
{
loop {
size32_t got = cmpio.get()?cmpio->read(offset, BUFFERSIZE, buffer):srcio->read(offset, BUFFERSIZE, buffer);
if (got == 0)
break;
if (flzstrmdst)
flzstrmdst->write(got,buffer);
else
dstio->write(offset, got, buffer);
offset += got;
}
}
catch (IException *e)
{
// try to delete partial copy
dstio.clear();
try {
dstfile->remove();
}
catch (IException *e2) {
StringBuffer s;
pexception(s.clear().append("Removing partial copy file: ").append(to).str(),e2);
e2->Release();
}
throw e;
}
flzstrmdst.clear();
dstio.clear();
if (stats)
printStats(offset,start,startu);
CDateTime createTime, modifiedTime;
if (srcfile->getTime(&createTime, &modifiedTime, NULL))
dstfile->setTime(&createTime, &modifiedTime, NULL);
printf("copied %s to %s%s\n",from,to,plaincopy?"":" compressing");
{ // print details
dstio.setown(dstfile->open(IFOread));
if (dstio) {
Owned<ICompressedFileIO> cmpio = createCompressedFileReader(dstio);
Owned<IFileIOStream> flzstrm = cmpio?NULL:createFastLZStreamRead(dstio);
if (cmpio||flzstrm)
printCompDetails(to,dstio,cmpio,flzstrm);
else
printf("destination %s not compressed\n",to);
}
else
printf("destination %s could not be read\n",to);
}
}
int copyExpanded(const char *from, const char *to, bool stats)
{
Owned<IFile> srcfile = createIFile(from);
Owned<IFileIO> srcio = srcfile->open(IFOread);
if (!srcio) {
printf("ERROR: could not open '%s' for read\n",from);
doexit(3);
}
Owned<ICompressedFileIO> cmpio = createCompressedFileReader(srcio);
Owned<IFileIOStream> flzstrm = cmpio?NULL:createFastLZStreamRead(srcio);
int ret = 0;
if (cmpio||flzstrm)
printCompDetails(from,srcio,cmpio,flzstrm);
else {
ret = 1;
printf("%s is not compressed, size= %"I64F"d\n",from,srcio->size());
}
if (!to||!*to)
return ret;
Owned<IFile> dstfile = createIFile(to);
StringBuffer fulldst;
if (dstfile->isDirectory()==foundYes) {
dstfile.clear();
addPathSepChar(fulldst.append(to)).append(pathTail(from));
to = fulldst.str();
dstfile.setown(createIFile(to));
}
if (dstfile->exists()) {
printf("ERROR: file '%s' already exists\n",to);
doexit(4);
}
unsigned start;
unsigned startu;
if (stats) {
start = msTick();
startu = usTick();
}
Owned<IFileIO> dstio = dstfile->open(IFOcreate);
if (!dstio) {
printf("ERROR: could not open '%s' for write\n",to);
doexit(5);
}
#ifdef __linux__
// this is not really needed in windows - if it is we will have to
// test the file extension - .exe, .bat
struct stat info;
if (stat(from, &info) == 0) // cannot fail - exception would have been thrown above
dstfile->setCreateFlags(info.st_mode&(S_IRUSR|S_IRGRP|S_IROTH|S_IWUSR|S_IWGRP|S_IWOTH|S_IXUSR|S_IXGRP|S_IXOTH));
#endif
MemoryAttr mb;
void * buffer = mb.allocate(BUFFERSIZE);
offset_t offset = 0;
try
{
loop {
size32_t got = cmpio.get()?cmpio->read(offset,BUFFERSIZE, buffer):
(flzstrm?flzstrm->read(BUFFERSIZE, buffer):
srcio->read(offset, BUFFERSIZE, buffer));
if (got == 0)
break;
dstio->write(offset, got, buffer);
offset += got;
}
}
catch (IException *e)
{
// try to delete partial copy
dstio.clear();
try {
dstfile->remove();
}
catch (IException *e2) {
StringBuffer s;
pexception(s.clear().append("Removing partial copy file: ").append(to).str(),e2);
e2->Release();
}
throw e;
}
dstio.clear();
if (stats)
printStats(offset,start,startu);
CDateTime createTime, modifiedTime;
if (srcfile->getTime(&createTime, &modifiedTime, NULL))
dstfile->setTime(&createTime, &modifiedTime, NULL);
printf("copied %s to %s%s\n",from,to,cmpio.get()?" expanding":"");
return 0;
}