void CConfigGenEngine::createFakePlugins(StringBuffer& destFilePath) const
{
    String destFilePathStr(destFilePath);
    String* tmpstr = destFilePathStr.toLowerCase();
    if (!tmpstr->endsWith("plugins.xml"))
    {
        int index = tmpstr->indexOf("plugins.xml");
        destFilePath.remove(index + 11, destFilePath.length() - (index + 11));
    }

    delete tmpstr;

    StringBuffer tmpoutbuf("<?xml version=\"1.0\" encoding=\"UTF-8\"?><Plugins/>");

    if (m_instances.ordinality() > 1 && strcmp(m_process.queryName(), XML_TAG_ESPPROCESS))
        destFilePath.replaceString("@temp"PATHSEPSTR, m_cachePath);
    else
    {
        char tempPath[_MAX_PATH];
        getTempPath(tempPath, sizeof(tempPath), m_name);
        ensurePath(tempPath);
        destFilePath.replaceString("@temp"PATHSEPSTR, tempPath);
    }

    Owned<IFile> pTargetFile = createIFile(destFilePath.str());
    if (pTargetFile->exists() && pTargetFile->isReadOnly())
        pTargetFile->setReadOnly(false);
    Owned<IFileIO> pTargetFileIO = pTargetFile->open(IFOcreate);
    pTargetFileIO->write( 0, tmpoutbuf.length(), tmpoutbuf.str());
    m_envDepEngine.addTempFile(destFilePath.str());
}
void CLogSerializer::Remove()
{
    Close();
    Owned<IFile> file = createIFile(m_FilePath.str()); 
    if(file.get() && file->exists() == true)
        file->remove();
}
    virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData) override
    {
        isLocal = 0 != (TIWlocal & helper->getFlags());

        mpTag = container.queryJobChannel().deserializeMPTag(data);
        mpTag2 = container.queryJobChannel().deserializeMPTag(data);
        data.read(active);
        if (active)
        {
            data.read(logicalFilename);
            partDesc.setown(deserializePartFileDescriptor(data));
        }

        data.read(singlePartKey);
        data.read(refactor);
        if (singlePartKey)
            buildTlk = false;
        else
        {
            data.read(buildTlk);
            if (firstNode())
            {
                if (buildTlk)
                    tlkDesc.setown(deserializePartFileDescriptor(data));
                else if (!isLocal) // existing tlk then..
                {
                    tlkDesc.setown(deserializePartFileDescriptor(data));
                    unsigned c;
                    data.read(c);
                    while (c--)
                    {
                        RemoteFilename rf;
                        rf.deserialize(data);
                        if (!existingTlkIFile)
                        {
                            Owned<IFile> iFile = createIFile(rf);
                            if (iFile->exists())
                                existingTlkIFile.set(iFile);
                        }
                    }
                    if (!existingTlkIFile)
                        throw MakeActivityException(this, TE_FileNotFound, "Top level key part does not exist, for key");
                }
            }
        }

        IOutputMetaData * diskSize = helper->queryDiskRecordSize();
        assertex(!(diskSize->getMetaFlags() & MDFneedserializedisk));
        if (diskSize->isVariableSize())
        {
            if (TIWmaxlength & helper->getFlags())
                maxDiskRecordSize = helper->getMaxKeySize();
            else
                maxDiskRecordSize = KEYBUILD_MAXLENGTH; //Current default behaviour, could be improved in the future
        }
        else
            maxDiskRecordSize = diskSize->getFixedSize();
        reportOverflow = false;
    }
Esempio n. 4
0
void CLogSerializer::Rollover(const char* ClosedPrefix)
{
    Close();
    Owned<IFile> file = createIFile(m_FilePath.str());
    if(file.get() && file->exists() == true)
    {
        StringBuffer newFileName;
        GetRolloverFileName(m_FileName,newFileName,ClosedPrefix);
        file->rename(newFileName.str());
    }
}
Esempio n. 5
0
void CppCompiler::writeLogFile(const char* filepath, StringBuffer& log)
{
    if(!filepath || !*filepath || !log.length())
        return;

    Owned <IFile> f = createIFile(filepath);
    if(f->exists())
        f->remove();

    Owned <IFileIO> fio = f->open(IFOcreaterw);
    if(fio.get())
        fio->write(0, log.length(), log.str());
}
bool CLogSerializer::EnsureDirectory(StringBuffer& Dir)
{
    try{
        Owned<IFile> pDirectory = createIFile(Dir.str());
        if(pDirectory->exists() == true)
            return true;
        return pDirectory->createDirectory();
    }
    catch(IException *ex)
    {
        ex->Release();
    }
    return false;
}
void CLocalDataLogger::Init(const char* logDirectory,const char* ext,const char* UrlRoot, unsigned TimerPeriod,unsigned CacheTimeoutPeriod)
{
    if(!logDirectory || !ext)
        throw MakeStringException(-1,"Invalid parameters passed to cLocalDataLogger::Init");
    StringBuffer pathtodir;
    pathtodir.appendf("%s",logDirectory);

    Owned<IFile> pDirectory = createIFile(pathtodir.str());
    if(pDirectory->exists() == false)
        pDirectory->createDirectory();

    m_UrlRoot.appendf("%s",UrlRoot);

    m_logDirectory.append(logDirectory);
    StringBuffer dirpath;
    dirpath.appendf("%s",logDirectory);
    m_CleanupThread.setown(new CFileCleanupThread(dirpath,ext,TimerPeriod,CacheTimeoutPeriod));
    m_CleanupThread->start();
}
//---------------------------------------------------------------------------
//  beforeDeploy
//---------------------------------------------------------------------------
void CConfigGenEngine::beforeDeploy()
{
    m_installFiles.clear();

    char tempPath[_MAX_PATH];
    getTempPath(tempPath, sizeof(tempPath), m_name);
    m_envDepEngine.addTempDirectory( tempPath );

    if (m_instances.ordinality() > 1)
    {
        strcat(tempPath, "Cache");
        char* pszEnd = tempPath + strlen(tempPath);

        Owned<IFile> pFile = createIFile(tempPath);
        int i = 1;

        while (pFile->exists()) { //dir/file exists
            itoa(++i, pszEnd, 10);
            pFile.setown( createIFile(tempPath) );
        }

        strcat(tempPath, PATHSEPSTR);
        m_cachePath.set( tempPath );
    }
    else
        m_cachePath.set( tempPath );

    ensurePath(tempPath);
    determineInstallFiles(m_process, m_installFiles);
    getCallback().installFileListChanged();

    if (m_instances.ordinality() > 1)
    {
        EnvMachineOS os = m_envDepEngine.lookupMachineOS( m_instances.item(0) );
        m_curInstance = "Cache";
        copyInstallFiles("Cache", -1, tempPath, os);
    }
}
Esempio n. 9
0
int main(int argc, const char *argv[])
{
    InitModuleObjects();
    EnableSEHtoExceptionMapping();

    NoQuickEditSection xxx;

    Owned<IFile> file = createIFile("dfuserver.xml");
    if (file->exists())
        globals.setown(createPTreeFromXMLFile("dfuserver.xml", ipt_caseInsensitive));
    else
        globals.setown(readOldIni());

    for (unsigned i=1;i<(unsigned)argc;i++) {
        const char *arg = argv[i];
        StringBuffer prop("@");
        StringBuffer val;
        while (*arg && *arg != '=')
            prop.append(*arg++);
        if (*arg) {
            arg++;
            while (isspace(*arg))
                arg++;
            val.append(arg);
            prop.clip();
            val.clip();
            if (prop.length()>1)
                globals->setProp(prop.str(), val.str());
        }
    }
    StringBuffer daliServer;
    StringBuffer queue;
    if (!globals->getProp("@DALISERVERS", daliServer)||!globals->getProp("@QUEUE", queue)) {
        usage();
        globals.clear();
        releaseAtoms();
        return 1;
    }
    Owned<IFile> sentinelFile;
    bool stop = globals->getPropInt("@STOP",0)!=0;
    if (!stop) {
        sentinelFile.setown(createSentinelTarget());
        removeSentinelFile(sentinelFile);

        StringBuffer logname;
        StringBuffer logdir;
        if (!getConfigurationDirectory(globals->queryPropTree("Directories"),"log","dfuserver",globals->queryProp("@name"),logdir))
            globals->getProp("@LOG_DIR", logdir);
        if (logdir.length() && recursiveCreateDirectory(logdir.str()))
            logname.append(logdir);
        else
            appendCurrentDirectory(logname, true);

        if (logname.length() && logname.charAt(logname.length()-1) != PATHSEPCHAR)
            logname.append(PATHSEPCHAR);
        logname.append("dfuserver");
        StringBuffer aliasLogName(logname);
        aliasLogName.append(".log");
        fileMsgHandler = getRollingFileLogMsgHandler(logname.str(), ".log", MSGFIELD_STANDARD, false, true, NULL, aliasLogName.str());
        queryLogMsgManager()->addMonitorOwn(fileMsgHandler, getCategoryLogMsgFilter(MSGAUD_all, MSGCLS_all, 1000));
    }
    StringBuffer ftslogdir;
    if (getConfigurationDirectory(globals->queryPropTree("Directories"),"log","ftslave",globals->queryProp("@name"),ftslogdir)) // NB instance deliberately dfuserver's
        setFtSlaveLogDir(ftslogdir.str());
    setRemoteSpawnSSH(
        globals->queryProp("SSH/@SSHidentityfile"),
        globals->queryProp("SSH/@SSHusername"),
        globals->queryProp("SSH/@SSHpassword"),
        globals->getPropInt("SSH/@SSHtimeout",0),
        globals->getPropInt("SSH/@SSHretries",3),
        "run_");
    bool enableSNMP = globals->getPropInt("@enableSNMP")!=0;
    CSDSServerStatus *serverstatus=NULL;
    Owned<IReplicateServer> replserver;
    try {
        Owned<IGroup> serverGroup = createIGroup(daliServer.str(),DALI_SERVER_PORT);
        initClientProcess(serverGroup, DCR_DfuServer, 0, NULL, NULL, stop?(1000*30):MP_WAIT_FOREVER);
        setPasswordsFromSDS();

        if(!stop)
        {
            if (globals->getPropBool("@enableSysLog",true))
                UseSysLogForOperatorMessages();

            serverstatus = new CSDSServerStatus("DFUserver");
            setDaliServixSocketCaching(true); // speeds up lixux operations

            startLogMsgParentReceiver();    // for auditing
            connectLogMsgManagerToDali();

            engine.setown(createDFUengine());
            addAbortHandler(exitDFUserver);
        }
        const char *q = queue.str();
        loop {
            StringBuffer subq;
            const char *comma = strchr(q,',');
            if (comma)
                subq.append(comma-q,q);
            else
                subq.append(q);
            if (stop) {
                stopDFUserver(subq.str());
            }
            else {
                StringBuffer mask;
                mask.appendf("Queue[@name=\"%s\"][1]",subq.str());
                IPropertyTree *t=serverstatus->queryProperties()->queryPropTree(mask.str());
                if (t)
                    t->setPropInt("@num",t->getPropInt("@num",0)+1);
                else {
                    t = createPTree();
                    t->setProp("@name",subq.str());
                    t->setPropInt("@num",1);
                    serverstatus->queryProperties()->addPropTree("Queue",t);
                }
                serverstatus->commitProperties();
                engine->setDefaultTransferBufferSize((size32_t)globals->getPropInt("@transferBufferSize"));
                engine->startListener(subq.str(),serverstatus);
            }
            if (!comma)
                break;
            q = comma+1;
            if (!*q)
                break;
        }
        q = globals->queryProp("@MONITORQUEUE");
        if (q&&*q) {
            if (stop) {
                stopDFUserver(q);
            }
            else {
                IPropertyTree *t=serverstatus->queryProperties()->addPropTree("MonitorQueue",createPTree());
                t->setProp("@name",q);
                engine->startMonitor(q,serverstatus,globals->getPropInt("@MONITORINTERVAL",60)*1000);
            }
        }
        q = globals->queryProp("@REPLICATEQUEUE");
        if (q&&*q) {
            if (stop) {
                // TBD?
            }
            else {
                replserver.setown(createReplicateServer(q));
                replserver->runServer();
            }
        }
        if (!stop) {
            serverstatus->commitProperties();

            writeSentinelFile(sentinelFile);

            engine->joinListeners();
            if (replserver.get())
                replserver->stopServer();
            LOG(MCprogress, unknownJob, "Exiting");
        }

    }
    catch(IException *e){
        EXCLOG(e, "DFU Server Exception: ");
        e->Release();
    }
    catch (const char *s) {
        WARNLOG("DFU: %s",s);
    }

    delete serverstatus;
    if (stop)
        Sleep(2000);    // give time to stop
    engine.clear();
    globals.clear();
    closeEnvironment();
    closedownClientProcess();
    UseSysLogForOperatorMessages(false);
    setDaliServixSocketCaching(false);
    releaseAtoms();
    return 0;
}
Esempio n. 10
0
int main( int argc, char *argv[]  )
{
#if defined(WIN32) && defined(_DEBUG)
    int tmpFlag = _CrtSetDbgFlag( _CRTDBG_REPORT_FLAG );
    tmpFlag |= _CRTDBG_LEAK_CHECK_DF;
    _CrtSetDbgFlag( tmpFlag );
#endif

    InitModuleObjects();

    addAbortHandler(ControlHandler);
    EnableSEHtoExceptionMapping();

    dummyProc();
#ifndef __64BIT__
    // Restrict stack sizes on 32-bit systems
    Thread::setDefaultStackSize(0x10000);   // NB under windows requires linker setting (/stack:)
#endif

#ifdef _WIN32
    Owned<CReleaseMutex> globalNamedMutex;
#endif 

    if (globals)
        globals->Release();

    {
        Owned<IFile> iFile = createIFile("thor.xml");
        globals = iFile->exists() ? createPTree(*iFile, ipt_caseInsensitive) : createPTree("Thor", ipt_caseInsensitive);
    }
    unsigned multiThorMemoryThreshold = 0;

    Owned<IException> unregisterException;
    try
    {
        if (argc==1)
        {
            usage();
            return 1;
        }
        cmdArgs = argv+1;
        mergeCmdParams(globals);
        cmdArgs = argv+1;

        const char *master = globals->queryProp("@MASTER");
        if (!master)
            usage();

        const char *slave = globals->queryProp("@SLAVE");
        if (slave)
        {
            slfEp.set(slave);
            localHostToNIC(slfEp);
        }
        else 
            slfEp.setLocalHost(0);

        mySlaveNum = globals->getPropInt("@SLAVENUM");

        setMachinePortBase(slfEp.port);
        slfEp.port = getMachinePortBase();
        startSlaveLog();

        setSlaveAffinity(globals->getPropInt("@SLAVEPROCESSNUM"));

        startMPServer(getFixedPort(TPORT_mp));
#ifdef USE_MP_LOG
        startLogMsgParentReceiver();
        LOG(MCdebugProgress, thorJob, "MPServer started on port %d", getFixedPort(TPORT_mp));
#endif

        SocketEndpoint masterEp(master);
        localHostToNIC(masterEp);
        setMasterPortBase(masterEp.port);
        markNodeCentral(masterEp);
        if (RegisterSelf(masterEp))
        {
            if (globals->getPropBool("Debug/@slaveDaliClient"))
                enableThorSlaveAsDaliClient();

            IDaFileSrvHook *daFileSrvHook = queryDaFileSrvHook();
            if (daFileSrvHook) // probably always installed
                daFileSrvHook->addFilters(globals->queryPropTree("NAS"), &slfEp);

            StringBuffer thorPath;
            globals->getProp("@thorPath", thorPath);
            recursiveCreateDirectory(thorPath.str());
            int err = _chdir(thorPath.str());
            if (err)
            {
                IException *e = makeErrnoExceptionV(-1, "Failed to change dir to '%s'", thorPath.str());
                FLLOG(MCexception(e), thorJob, e);
                throw e;
            }

// Initialization from globals
            setIORetryCount(globals->getPropInt("Debug/@ioRetries")); // default == 0 == off

            StringBuffer str;
            if (globals->getProp("@externalProgDir", str.clear()))
                _mkdir(str.str());
            else
                globals->setProp("@externalProgDir", thorPath);

            const char * overrideBaseDirectory = globals->queryProp("@thorDataDirectory");
            const char * overrideReplicateDirectory = globals->queryProp("@thorReplicateDirectory");
            StringBuffer datadir;
            StringBuffer repdir;
            if (getConfigurationDirectory(globals->queryPropTree("Directories"),"data","thor",globals->queryProp("@name"),datadir))
                overrideBaseDirectory = datadir.str();
            if (getConfigurationDirectory(globals->queryPropTree("Directories"),"mirror","thor",globals->queryProp("@name"),repdir))
                overrideReplicateDirectory = repdir.str();
            if (overrideBaseDirectory&&*overrideBaseDirectory)
                setBaseDirectory(overrideBaseDirectory, false);
            if (overrideReplicateDirectory&&*overrideBaseDirectory)
                setBaseDirectory(overrideReplicateDirectory, true);
            StringBuffer tempDirStr;
            if (getConfigurationDirectory(globals->queryPropTree("Directories"),"temp","thor",globals->queryProp("@name"), tempDirStr))
                globals->setProp("@thorTempDirectory", tempDirStr.str());
            else
                tempDirStr.append(globals->queryProp("@thorTempDirectory"));
            addPathSepChar(tempDirStr).append(getMachinePortBase());

            logDiskSpace(); // Log before temp space is cleared
            SetTempDir(tempDirStr.str(), "thtmp", true);

            useMemoryMappedRead(globals->getPropBool("@useMemoryMappedRead"));

            LOG(MCdebugProgress, thorJob, "ThorSlave Version LCR - %d.%d started",THOR_VERSION_MAJOR,THOR_VERSION_MINOR);
            StringBuffer url;
            LOG(MCdebugProgress, thorJob, "Slave %s - temporary dir set to : %s", slfEp.getUrlStr(url).str(), queryTempDir());
#ifdef _WIN32
            ULARGE_INTEGER userfree;
            ULARGE_INTEGER total;
            ULARGE_INTEGER free;
            if (GetDiskFreeSpaceEx("c:\\",&userfree,&total,&free)&&total.QuadPart) {
                unsigned pc = (unsigned)(free.QuadPart*100/total.QuadPart);
                LOG(MCdebugProgress, thorJob, "Total disk space = %" I64F "d k", total.QuadPart/1000);
                LOG(MCdebugProgress, thorJob, "Free  disk space = %" I64F "d k", free.QuadPart/1000);
                LOG(MCdebugProgress, thorJob, "%d%% disk free\n",pc);
            }
#endif
            if (getConfigurationDirectory(globals->queryPropTree("Directories"),"query","thor",globals->queryProp("@name"),str.clear()))
                globals->setProp("@query_so_dir", str.str());
            else
                globals->getProp("@query_so_dir", str.clear());
            if (str.length())
            {
                if (globals->getPropBool("Debug/@dllsToSlaves", true))
                {
                    StringBuffer uniqSoPath;
                    if (PATHSEPCHAR == str.charAt(str.length()-1))
                        uniqSoPath.append(str.length()-1, str.str());
                    else
                        uniqSoPath.append(str);
                    uniqSoPath.append("_").append(getMachinePortBase());
                    str.swapWith(uniqSoPath);
                    globals->setProp("@query_so_dir", str.str());
                }
                PROGLOG("Using querySo directory: %s", str.str());
                recursiveCreateDirectory(str.str());
            }
     
            multiThorMemoryThreshold = globals->getPropInt("@multiThorMemoryThreshold")*0x100000;
            if (multiThorMemoryThreshold) {
                StringBuffer lgname;
                if (!globals->getProp("@multiThorResourceGroup",lgname))
                    globals->getProp("@nodeGroup",lgname);
                if (lgname.length()) {
                    Owned<ILargeMemLimitNotify> notify = createMultiThorResourceMutex(lgname.str());
                    setMultiThorMemoryNotify(multiThorMemoryThreshold,notify);
                    PROGLOG("Multi-Thor resource limit for %s set to %" I64F "d",lgname.str(),(__int64)multiThorMemoryThreshold);
                }   
                else
                    multiThorMemoryThreshold = 0;
            }
            slaveMain(jobListenerStopped);
        }

        LOG(MCdebugProgress, thorJob, "ThorSlave terminated OK");
    }
    catch (IException *e) 
    {
        if (!jobListenerStopped)
            FLLOG(MCexception(e), thorJob, e,"ThorSlave");
        unregisterException.setown(e);
    }
    ClearTempDirs();

    if (multiThorMemoryThreshold)
        setMultiThorMemoryNotify(0,NULL);
    roxiemem::releaseRoxieHeap();

    if (unregisterException.get())
        UnregisterSelf(unregisterException);

    if (globals->getPropBool("Debug/@slaveDaliClient"))
        disableThorSlaveAsDaliClient();

#ifdef USE_MP_LOG
    stopLogMsgReceivers();
#endif
    stopMPServer();
    ::Release(globals);
    releaseAtoms(); // don't know why we can't use a module_exit to destruct these...

    ExitModuleObjects(); // not necessary, atexit will call, but good for leak checking
    return 0;
}
Esempio n. 11
0
	bool loadPwds()
	{
		try
		{
			if (!pwFile.length())
				throw MakeStringException(-1, "htpasswd Password file not specified");

			Owned<IFile> file = createIFile(pwFile.str());
			if (!file->exists())
			{
				userMap.kill();
				throw MakeStringException(-1, "htpasswd Password file does not exist");
			}

			bool isDir;
			offset_t size;
			CDateTime whenChanged;
			file->getInfo(isDir,size,whenChanged);
			if (isDir)
			{
				userMap.kill();
				throw MakeStringException(-1, "htpasswd Password file specifies a directory");
			}
			if (0 == whenChanged.compare(pwFileLastMod))
				return true;//Don't reload if file unchanged
			userMap.kill();
			OwnedIFileIO io = file->open(IFOread);
			if (!io)
				throw MakeStringException(-1, "htpasswd Unable to open Password file");

			MemoryBuffer mb;
			size32_t count = read(io, 0, (size32_t)-1, mb);
			if (0 == count)
				throw MakeStringException(-1, "htpasswd Password file is empty");

			mb.append((char)NULL);
			char * p = (char*)mb.toByteArray();
			char *saveptr;
			const char * seps = "\f\r\n";
			char * next = strtok_r(p, seps, &saveptr);
			if (next)
			{
				do
				{
					char * colon = strchr(next,':');
					if (NULL == colon)
						throw MakeStringException(-1, "htpasswd Password file appears malformed");
					*colon = (char)NULL;
					userMap.setValue(next, colon+1);//username, enctypted password
				} while (next = strtok_r(NULL, seps, &saveptr));
			}

			io->close();
			pwFileLastMod = whenChanged;//remember when last changed
		}
		catch(IException*)
		{
			throw MakeStringException(-1, "htpasswd Exception accessing Password file");
		}
		return true;
	}
Esempio n. 12
0
int main( int argc, char *argv[]  )
{
#if defined(WIN32) && defined(_DEBUG)
    int tmpFlag = _CrtSetDbgFlag( _CRTDBG_REPORT_FLAG );
    tmpFlag |= _CRTDBG_LEAK_CHECK_DF;
    _CrtSetDbgFlag( tmpFlag );
#endif

    InitModuleObjects();

    addAbortHandler(ControlHandler);
    EnableSEHtoExceptionMapping();

    dummyProc();
#ifndef __64BIT__
    Thread::setDefaultStackSize(0x10000);   // NB under windows requires linker setting (/stack:)
#endif

#ifdef _WIN32
    Owned<CReleaseMutex> globalNamedMutex;
#endif 

    if (globals)
        globals->Release();

    {
        Owned<IFile> iFile = createIFile("thor.xml");
        globals = iFile->exists() ? createPTree(*iFile, ipt_caseInsensitive) : createPTree("Thor", ipt_caseInsensitive);
    }
    unsigned multiThorMemoryThreshold = 0;
    try {
        if (argc==1)
        {
            usage();
            return 1;
        }
        cmdArgs = argv+1;
        mergeCmdParams(globals);
        cmdArgs = argv+1;

        const char *master = globals->queryProp("@MASTER");
        if (!master)
            usage();

        const char *slave = globals->queryProp("@SLAVE");
        if (slave)
        {
            slfEp.set(slave);
            localHostToNIC(slfEp);
        }
        else 
            slfEp.setLocalHost(0);

        if (globals->hasProp("@SLAVENUM"))
            mySlaveNum = atoi(globals->queryProp("@SLAVENUM"));
        else
            mySlaveNum = slfEp.port; // shouldn't happen, provided by script

        setMachinePortBase(slfEp.port);
        slfEp.port = getMachinePortBase();
        startSlaveLog();

        startMPServer(getFixedPort(TPORT_mp));
#ifdef USE_MP_LOG
        startLogMsgParentReceiver();
        LOG(MCdebugProgress, thorJob, "MPServer started on port %d", getFixedPort(TPORT_mp));
#endif

        SocketEndpoint masterEp(master);
        localHostToNIC(masterEp);
        setMasterPortBase(masterEp.port);
        markNodeCentral(masterEp);
        if (RegisterSelf(masterEp))
        {
#define ISDALICLIENT // JCSMORE plugins *can* access dali - though I think we should probably prohibit somehow.
#ifdef ISDALICLIENT
            const char *daliServers = globals->queryProp("@DALISERVERS");
            if (!daliServers)
            {
                LOG(MCerror, thorJob, "No Dali server list specified\n");
                return 1;
            }
            Owned<IGroup> serverGroup = createIGroup(daliServers, DALI_SERVER_PORT);
            unsigned retry = 0;
            loop {
                try {
                    LOG(MCdebugProgress, thorJob, "calling initClientProcess");
                    initClientProcess(serverGroup,DCR_ThorSlave, getFixedPort(TPORT_mp));
                    break;
                }
                catch (IJSOCK_Exception *e) {
                    if ((e->errorCode()!=JSOCKERR_port_in_use))
                        throw;
                    FLLOG(MCexception(e), thorJob, e,"InitClientProcess");
                    if (retry++>10)
                        throw;
                    e->Release();
                    LOG(MCdebugProgress, thorJob, "Retrying");
                    Sleep(retry*2000);
                }
            }
            setPasswordsFromSDS();
#endif
            IDaFileSrvHook *daFileSrvHook = queryDaFileSrvHook();
            if (daFileSrvHook) // probably always installed
                daFileSrvHook->addSubnetFilters(globals->queryPropTree("NAS"), NULL);

            StringBuffer thorPath;
            globals->getProp("@thorPath", thorPath);
            recursiveCreateDirectory(thorPath.str());
            int err = _chdir(thorPath.str());
            if (err)
            {
                IException *e = MakeErrnoException(-1, "Failed to change dir to '%s'",thorPath.str());
                FLLOG(MCexception(e), thorJob, e);
                throw e;
            }

// Initialization from globals
            setIORetryCount(globals->getPropInt("Debug/@ioRetries")); // default == 0 == off

            StringBuffer str;
            if (globals->getProp("@externalProgDir", str.clear()))
                _mkdir(str.str());
            else
                globals->setProp("@externalProgDir", thorPath);

            const char * overrideBaseDirectory = globals->queryProp("@thorDataDirectory");
            const char * overrideReplicateDirectory = globals->queryProp("@thorReplicateDirectory");
            StringBuffer datadir;
            StringBuffer repdir;
            if (getConfigurationDirectory(globals->queryPropTree("Directories"),"data","thor",globals->queryProp("@name"),datadir))
                overrideBaseDirectory = datadir.str();
            if (getConfigurationDirectory(globals->queryPropTree("Directories"),"mirror","thor",globals->queryProp("@name"),repdir))
                overrideReplicateDirectory = repdir.str();
            if (overrideBaseDirectory&&*overrideBaseDirectory)
                setBaseDirectory(overrideBaseDirectory, false);
            if (overrideReplicateDirectory&&*overrideBaseDirectory)
                setBaseDirectory(overrideReplicateDirectory, true);
            StringBuffer tempdirstr;
            const char *tempdir = globals->queryProp("@thorTempDirectory");
            if (getConfigurationDirectory(globals->queryPropTree("Directories"),"temp","thor",globals->queryProp("@name"),tempdirstr))
                tempdir = tempdirstr.str();
            SetTempDir(tempdir,true);

            useMemoryMappedRead(globals->getPropBool("@useMemoryMappedRead"));

            LOG(MCdebugProgress, thorJob, "ThorSlave Version LCR - %d.%d started",THOR_VERSION_MAJOR,THOR_VERSION_MINOR);
            StringBuffer url;
            LOG(MCdebugProgress, thorJob, "Slave %s - temporary dir set to : %s", slfEp.getUrlStr(url).toCharArray(), queryTempDir());
#ifdef _WIN32
            ULARGE_INTEGER userfree;
            ULARGE_INTEGER total;
            ULARGE_INTEGER free;
            if (GetDiskFreeSpaceEx("c:\\",&userfree,&total,&free)&&total.QuadPart) {
                unsigned pc = (unsigned)(free.QuadPart*100/total.QuadPart);
                LOG(MCdebugProgress, thorJob, "Total disk space = %"I64F"d k", total.QuadPart/1000);
                LOG(MCdebugProgress, thorJob, "Free  disk space = %"I64F"d k", free.QuadPart/1000);
                LOG(MCdebugProgress, thorJob, "%d%% disk free\n",pc);
            }
#endif
            if (getConfigurationDirectory(globals->queryPropTree("Directories"),"query","thor",globals->queryProp("@name"),str.clear()))
                globals->setProp("@query_so_dir", str.str());
            else
                globals->getProp("@query_so_dir", str.clear());
            if (str.length())
            {
                if (globals->getPropBool("Debug/@dllsToSlaves", true))
                {
                    StringBuffer uniqSoPath;
                    if (PATHSEPCHAR == str.charAt(str.length()-1))
                        uniqSoPath.append(str.length()-1, str.str());
                    else
                        uniqSoPath.append(str);
                    uniqSoPath.append("_").append(getMachinePortBase());
                    str.swapWith(uniqSoPath);
                    globals->setProp("@query_so_dir", str.str());
                }
                PROGLOG("Using querySo directory: %s", str.str());
                recursiveCreateDirectory(str.str());
            }
     
            multiThorMemoryThreshold = globals->getPropInt("@multiThorMemoryThreshold")*0x100000;
            if (multiThorMemoryThreshold) {
                StringBuffer lgname;
                if (!globals->getProp("@multiThorResourceGroup",lgname))
                    globals->getProp("@nodeGroup",lgname);
                if (lgname.length()) {
                    Owned<ILargeMemLimitNotify> notify = createMultiThorResourceMutex(lgname.str());
                    setMultiThorMemoryNotify(multiThorMemoryThreshold,notify);
                    PROGLOG("Multi-Thor resource limit for %s set to %"I64F"d",lgname.str(),(__int64)multiThorMemoryThreshold);
                }   
                else
                    multiThorMemoryThreshold = 0;
            }
            slaveMain();
        }

        LOG(MCdebugProgress, thorJob, "ThorSlave terminated OK");
    }
    catch (IException *e) 
    {
        FLLOG(MCexception(e), thorJob, e,"ThorSlave");
        e->Release();
    }
    catch (CATCHALL)
    {
        FLLOG(MCerror, thorJob, "ThorSlave exiting because of uncaught exception");
    }
    ClearTempDirs();

    if (multiThorMemoryThreshold)
        setMultiThorMemoryNotify(0,NULL);
    roxiemem::releaseRoxieHeap();

#ifdef ISDALICLIENT
    closeEnvironment();
    closedownClientProcess();   // dali client closedown
#endif

#ifdef USE_MP_LOG
    stopLogMsgReceivers();
#endif
    stopMPServer();
    ::Release(globals);
    releaseAtoms(); // don't know why we can't use a module_exit to destruct these...

    return 0;
}
Esempio n. 13
0
int main(int argc, const char* argv[])
{
    InitModuleObjects();

    if ((argc >= 2) && ((stricmp(argv[1], "/version") == 0) || (stricmp(argv[1], "-v") == 0)
        || (stricmp(argv[1], "--version") == 0)))
    {
        printVersion();
        return 0;
    }

    Owned<IFile> inifile = createIFile("dfuplus.ini");
    if(argc < 2 && !(inifile->exists() && inifile->size() > 0))
    {
        handleSyntax();
        return 0;
    }

    if ((argc >= 2) && ((argv[1][0]=='/' || argv[1][0]=='-') && (argv[1][1]=='?' || argv[1][1]=='h'))) 
    {
        handleSyntax();
        return 0;
    }

    //queryLogMsgManager()->changeMonitorFilterOwn(queryStderrLogMsgHandler(), getPassNoneLogMsgFilter());

    Owned<IProperties> globals = createProperties("dfuplus.ini", true);

    if(!build_globals(argc, argv, globals))
    {
        fprintf(stderr, "ERROR: Invalid command syntax.\n");
        releaseAtoms();
        return DFUERR_InvalidCommandSyntax;
    }



    
    const char* action = globals->queryProp("action");
    if(!action || !*action)
    {
        handleSyntax();
        fprintf(stderr, "\nERROR: please specify one action");
        releaseAtoms();
        return DFUERR_TooFewArguments;
    }

    const char* server = globals->queryProp("server");
    if (!server || !*server) {
        if (stricmp(action,"dafilesrv")==0)
            globals->setProp("server","127.0.0.1"); // dummy
        else { 
            fprintf(stderr, "ERROR: Esp server url not specified.\n");
            releaseAtoms();
            return DFUERR_TooFewArguments;
        }
    }

    try
    {
        Owned<CDfuPlusHelper> helper = new CDfuPlusHelper(LINK(globals.get()));
        helper->doit();
    }
    catch(IException* e)
    {
        StringBuffer errmsg;
        e->errorMessage(errmsg);
        fprintf(stderr, "%s\n", errmsg.str());
    }
    
    releaseAtoms();
    return 0;
}
Esempio n. 14
0
void copyCompress(const char *from, const char *to, size32_t rowsize, bool fast, bool flzstrm, bool stats)
{
    Owned<IFile> srcfile = createIFile(from);
    Owned<IFileIO> baseio = srcfile->open(IFOread);
    if (!baseio) {
        printf("ERROR: could not open '%s' for read\n",from);
        doexit(3);
    }
    Owned<ICompressedFileIO> cmpio = createCompressedFileReader(baseio);
    Owned<IFileIOStream>  flzstrmsrc = cmpio?NULL:createFastLZStreamRead(baseio);
    bool plaincopy = false;
    IFileIO *srcio = NULL;
    if (cmpio) {
        srcio = cmpio;
        if (rowsize&&(cmpio->recordSize()==rowsize))
            plaincopy = true;
        else if (!rowsize) {
            if (fast&&(cmpio->method()==COMPRESS_METHOD_FASTLZ))
                plaincopy = true;
            else if (!fast&&(cmpio->method()==COMPRESS_METHOD_LZW))
                plaincopy = true;
        }
    }
    else if (flzstrmsrc) {
        if (flzstrm)
            plaincopy = true;
    }
    else
        srcio = baseio; 
    if (plaincopy) {
        cmpio.clear();
        srcio = baseio.get(); 
    }
    Owned<IFile> dstfile = createIFile(to);
    StringBuffer fulldst;
    if (dstfile->isDirectory()==foundYes) {
        dstfile.clear();
        addPathSepChar(fulldst.append(to)).append(pathTail(from));
        to = fulldst.str();
        dstfile.setown(createIFile(to));
    }

    if (dstfile->exists()) {
        printf("ERROR: file '%s' already exists\n",to);
        doexit(4);
    }
    unsigned start;
    unsigned startu;
    if (stats) {
         start = msTick();
         startu = usTick();
    }
    Owned<IFileIO> dstio;
    Owned<IFileIOStream>  flzstrmdst;
    if (plaincopy||flzstrm) {
        dstio.setown(dstfile->open(IFOcreate));
        if (dstio&&!plaincopy)
            flzstrmdst.setown(createFastLZStreamWrite(dstio));
    }
    else 
        dstio.setown(createCompressedFileWriter(dstfile,rowsize,false,true,NULL,fast));

    if (!dstio) {
        printf("ERROR: could not open '%s' for write\n",to);
        doexit(5);
    }
#ifdef __linux__
    // this is not really needed in windows - if it is we will have to
    // test the file extension - .exe, .bat

    struct stat info;
    if (stat(from, &info) == 0)  // cannot fail - exception would have been thrown above
        dstfile->setCreateFlags(info.st_mode&(S_IRUSR|S_IRGRP|S_IROTH|S_IWUSR|S_IWGRP|S_IWOTH|S_IXUSR|S_IXGRP|S_IXOTH));
#endif
    MemoryAttr mb;
    void * buffer = mb.allocate(BUFFERSIZE);

    offset_t offset = 0;
    try
    {
        loop {
            size32_t got = cmpio.get()?cmpio->read(offset, BUFFERSIZE, buffer):srcio->read(offset, BUFFERSIZE, buffer);
            if (got == 0)
                break;
            if (flzstrmdst)
                flzstrmdst->write(got,buffer);
            else
                dstio->write(offset, got, buffer);
            offset += got;
        }
    }
    catch (IException *e)
    {
        // try to delete partial copy
        dstio.clear();
        try {
            dstfile->remove();
        }
        catch (IException *e2) {
            StringBuffer s;
            pexception(s.clear().append("Removing partial copy file: ").append(to).str(),e2);
            e2->Release();
        }
        throw e;
    }
    flzstrmdst.clear();
    dstio.clear();
    if (stats) 
        printStats(offset,start,startu);
    CDateTime createTime, modifiedTime;
    if (srcfile->getTime(&createTime, &modifiedTime, NULL))
        dstfile->setTime(&createTime, &modifiedTime, NULL);
    printf("copied %s to %s%s\n",from,to,plaincopy?"":" compressing");
    { // print details 
        dstio.setown(dstfile->open(IFOread));
        if (dstio) {
            Owned<ICompressedFileIO> cmpio = createCompressedFileReader(dstio);
            Owned<IFileIOStream>  flzstrm = cmpio?NULL:createFastLZStreamRead(dstio);
            if (cmpio||flzstrm) 
                printCompDetails(to,dstio,cmpio,flzstrm);
            else 
                printf("destination %s not compressed\n",to);
        }
        else
            printf("destination %s could not be read\n",to);
    }
}
Esempio n. 15
0
int copyExpanded(const char *from, const char *to, bool stats)
{
    Owned<IFile> srcfile = createIFile(from);
    Owned<IFileIO> srcio = srcfile->open(IFOread);
    if (!srcio) {
        printf("ERROR: could not open '%s' for read\n",from);
        doexit(3);
    }
    Owned<ICompressedFileIO> cmpio = createCompressedFileReader(srcio);
    Owned<IFileIOStream>  flzstrm = cmpio?NULL:createFastLZStreamRead(srcio);
    int ret = 0;
    if (cmpio||flzstrm) 
        printCompDetails(from,srcio,cmpio,flzstrm);
    else {
        ret = 1;
        printf("%s is not compressed, size= %"I64F"d\n",from,srcio->size());
    }
    if (!to||!*to)
        return ret;
    Owned<IFile> dstfile = createIFile(to);
    StringBuffer fulldst;
    if (dstfile->isDirectory()==foundYes) {
        dstfile.clear();
        addPathSepChar(fulldst.append(to)).append(pathTail(from));
        to = fulldst.str();
        dstfile.setown(createIFile(to));
    }

    if (dstfile->exists()) {
        printf("ERROR: file '%s' already exists\n",to);
        doexit(4);
    }
    unsigned start;
    unsigned startu;
    if (stats) {
         start = msTick();
         startu = usTick();
    }
    Owned<IFileIO> dstio = dstfile->open(IFOcreate);
    if (!dstio) {
        printf("ERROR: could not open '%s' for write\n",to);
        doexit(5);
    }
#ifdef __linux__
    // this is not really needed in windows - if it is we will have to
    // test the file extension - .exe, .bat

    struct stat info;
    if (stat(from, &info) == 0)  // cannot fail - exception would have been thrown above
        dstfile->setCreateFlags(info.st_mode&(S_IRUSR|S_IRGRP|S_IROTH|S_IWUSR|S_IWGRP|S_IWOTH|S_IXUSR|S_IXGRP|S_IXOTH));
#endif
    MemoryAttr mb;
    void * buffer = mb.allocate(BUFFERSIZE);

    offset_t offset = 0;
    try
    {
        loop {
            size32_t got = cmpio.get()?cmpio->read(offset,BUFFERSIZE, buffer):
                (flzstrm?flzstrm->read(BUFFERSIZE, buffer):
                    srcio->read(offset, BUFFERSIZE, buffer));
            if (got == 0)
                break;
            dstio->write(offset, got, buffer);
            offset += got;
        }
    }
    catch (IException *e)
    {
        // try to delete partial copy
        dstio.clear();
        try {
            dstfile->remove();
        }
        catch (IException *e2) {
            StringBuffer s;
            pexception(s.clear().append("Removing partial copy file: ").append(to).str(),e2);
            e2->Release();
        }
        throw e;
    }
    dstio.clear();
    if (stats) 
        printStats(offset,start,startu);
    CDateTime createTime, modifiedTime;
    if (srcfile->getTime(&createTime, &modifiedTime, NULL))
        dstfile->setTime(&createTime, &modifiedTime, NULL);
    printf("copied %s to %s%s\n",from,to,cmpio.get()?" expanding":"");
    return 0;
}