Example #1
0
    bool eclcc(StringBuffer &out)
    {
        StringBuffer cmdLine;
        buildCmd(cmdLine);

        Owned<IPipeProcess> pipe = createPipeProcess();
        bool hasInput = streq(cmd.optObj.value.sget(), "stdin");
        pipe->run(cmd.optVerbose ? "EXEC" : NULL, cmdLine.str(), NULL, hasInput, true, true);

        StringBuffer errors;
        Owned<EclCmdErrorReader> errorReader = new EclCmdErrorReader(pipe, errors);
        errorReader->start();

        if (pipe->hasInput())
        {
            pipe->write(cmd.optObj.mb.length(), cmd.optObj.mb.toByteArray());
            pipe->closeInput();
        }
        if (pipe->hasOutput())
        {
           byte buf[4096];
           loop
           {
                size32_t read = pipe->read(sizeof(buf),buf);
                if (!read)
                    break;
                out.append(read, (const char *) buf);
            }
        }
Example #2
0
    bool compile(const char *wuid, const char *target, const char *targetCluster)
    {
        Owned<IConstWUQuery> query = workunit->getQuery();
        if (!query)
        {
            reportError("Workunit does not contain a query", 2);
            return false;
        }

        SCMStringBuffer mainDefinition;
        SCMStringBuffer eclQuery;
        query->getQueryText(eclQuery);
        query->getQueryMainDefinition(mainDefinition);

        StringBuffer eclccCmd("eclcc -shared");
        if (eclQuery.length())
            eclccCmd.append(" -");
        if (mainDefinition.length())
            eclccCmd.append(" -main ").append(mainDefinition);

        Owned<IPropertyTreeIterator> options = globals->getElements("./Option");
        ForEach(*options)
        {
            IPropertyTree &option = options->query();
            const char *name = option.queryProp("@name");
            const char *value = option.queryProp("@value");
            const char *cluster = option.queryProp("@cluster");
            if (name && (cluster==NULL || cluster[0]==0 || strcmp(cluster, targetCluster)==0))
            {
                // options starting '-' are simply passed through to eclcc as name=value
                // others are passed as -foption=value
                // if cluster is set it's specific to a particular target
                eclccCmd.append(" ");
                if (name[0]!='-')
                    eclccCmd.append("-f");
                eclccCmd.append(name);
                if (value)
                    eclccCmd.append('=').append(value);
            }
        }
        eclccCmd.appendf(" -o%s", wuid);
        eclccCmd.appendf(" -target=%s", target);

        Owned<IStringIterator> debugValues = &workunit->getDebugValues();
        ForEach (*debugValues)
        {
            SCMStringBuffer debugStr, valueStr;
            debugValues->str(debugStr);
            workunit->getDebugValue(debugStr.str(), valueStr);
            if (memicmp(debugStr.str(), "eclcc-", 6) == 0)
            {
                //Allow eclcc-xx-<n> so that multiple values can be passed through for the same named debug symbol
                const char * start = debugStr.str() + 6;
                const char * dash = strchr(start, '-');
                StringAttr optName;
                if (dash)
                    optName.set(start, dash-start);
                else
                    optName.set(start);

                if (stricmp(optName, "compileOption") == 0)
                    eclccCmd.appendf(" -Wc,%s", valueStr.str());
                else if (stricmp(optName, "includeLibraryPath") == 0)
                    eclccCmd.appendf(" -I%s", valueStr.str());
                else if (stricmp(optName, "libraryPath") == 0)
                    eclccCmd.appendf(" -L%s", valueStr.str());
                else
                    eclccCmd.appendf(" -%s=%s", start, valueStr.str());
            }
            else
                eclccCmd.appendf(" -f%s=%s", debugStr.str(), valueStr.str());
        }
        if (workunit->getResultLimit())
        {
            eclccCmd.appendf(" -fapplyInstantEclTransformations=1 -fapplyInstantEclTransformationsLimit=%u", workunit->getResultLimit());
        }
        try
        {
            Owned<IPipeProcess> pipe = createPipeProcess();
            Owned<ErrorReader> errorReader = new ErrorReader(pipe, this);
            pipe->run("eclcc", eclccCmd, ".", true, false, true, 0);
            errorReader->start();
            try
            {
                pipe->write(eclQuery.s.length(), eclQuery.s.str());
                pipe->closeInput();
            }
            catch (IException *e)
            {
                reportError(e);
                e->Release();
            }
            unsigned retcode = pipe->wait();
            errorReader->join();
            if (retcode == 0)
            {
                StringBuffer realdllname, dllurl;
                realdllname.append(SharedObjectPrefix).append(wuid).append(SharedObjectExtension);

                StringBuffer realdllfilename(dllPath);
                realdllfilename.append(SharedObjectPrefix).append(wuid).append(SharedObjectExtension);

                StringBuffer wuXML;
                if (getWorkunitXMLFromFile(realdllfilename, wuXML))
                {
                    Owned<ILocalWorkUnit> embeddedWU = createLocalWorkUnit();
                    embeddedWU->loadXML(wuXML);
                    queryExtendedWU(workunit)->copyWorkUnit(embeddedWU);
                    Owned<IWUQuery> query = workunit->updateQuery();
                    query->setQueryText(eclQuery.s.str());
                }

                createUNCFilename(realdllfilename.str(), dllurl);
                unsigned crc = crc_file(realdllfilename.str());

                Owned<IWUQuery> query = workunit->updateQuery();
                associateLocalFile(query, FileTypeDll, realdllfilename, "Workunit DLL", crc);
                queryDllServer().registerDll(realdllname.str(), "Workunit DLL", dllurl.str());

                workunit->commit();
                return true;
            }
        }
        catch (IException * e)
        {
            reportError(e);
            e->Release();
        }
        return false;
    }
    bool compile(const char *wuid, const char *target, const char *targetCluster)
    {
        Owned<IConstWUQuery> query = workunit->getQuery();
        if (!query)
        {
            reportError("Workunit does not contain a query", 2);
            return false;
        }

        addTimeStamp(workunit, SSTglobal, NULL, StWhenCompiled);

        SCMStringBuffer mainDefinition;
        SCMStringBuffer eclQuery;
        query->getQueryText(eclQuery);
        query->getQueryMainDefinition(mainDefinition);

        StringBuffer eclccProgName;
        splitDirTail(queryCurrentProcessPath(), eclccProgName);
        eclccProgName.append("eclcc");
        StringBuffer eclccCmd(" -shared");
        if (eclQuery.length())
            eclccCmd.append(" -");
        if (mainDefinition.length())
            eclccCmd.append(" -main ").append(mainDefinition);
        eclccCmd.append(" --timings --xml");
        eclccCmd.append(" --nostdinc");
        if (globals->getPropBool("@enableEclccDali", true))
        {
            const char *daliServers = globals->queryProp("@daliServers");
            if (!daliServers)
                daliServers = ".";
            eclccCmd.appendf(" -dfs=%s", daliServers);
            const char *wuScope = workunit->queryWuScope();
            if (!isEmptyString(wuScope))
                eclccCmd.appendf(" -scope=%s", wuScope);
            eclccCmd.appendf(" -cluster=%s", targetCluster);
            SCMStringBuffer token;
            workunit->getSecurityToken(token);
            if (token.length())
                eclccCmd.appendf(" -wuid=%s -token=%s", workunit->queryWuid(), token.str());
        }
        Owned<IPipeProcess> pipe = createPipeProcess();
        pipe->setenv("ECLCCSERVER_THREAD_INDEX", idxStr.str());
        Owned<IPropertyTreeIterator> options = globals->getElements("./Option");
        ForEach(*options)
        {
            IPropertyTree &option = options->query();
            const char *name = option.queryProp("@name");
            const char *value = option.queryProp("@value");
            const char *cluster = option.queryProp("@cluster");                // if cluster is set it's specific to a particular target
            if (name && (cluster==NULL || cluster[0]==0 || strcmp(cluster, targetCluster)==0))
                processOption(name, value, eclccCmd, eclccProgName, *pipe, false);
        }
        eclccCmd.appendf(" -o%s", wuid);
        eclccCmd.appendf(" -platform=%s", target);
        eclccCmd.appendf(" --component=%s", queryStatisticsComponentName());

        Owned<IStringIterator> debugValues = &workunit->getDebugValues();
        ForEach (*debugValues)
        {
            SCMStringBuffer debugStr, valueStr;
            debugValues->str(debugStr);
            workunit->getDebugValue(debugStr.str(), valueStr);
            processOption(debugStr.str(), valueStr.str(), eclccCmd, eclccProgName, *pipe, true);
        }
        if (workunit->getResultLimit())
        {
            eclccCmd.appendf(" -fapplyInstantEclTransformations=1 -fapplyInstantEclTransformationsLimit=%u", workunit->getResultLimit());
        }
        try
        {
            cycle_t startCycles = get_cycles_now();
            Owned<ErrorReader> errorReader = new ErrorReader(pipe, this);
            Owned<AbortWaiter> abortWaiter = new AbortWaiter(pipe, workunit);
            eclccCmd.insert(0, eclccProgName);
            if (!pipe->run(eclccProgName, eclccCmd, ".", true, false, true, 0, true))
                throw makeStringExceptionV(999, "Failed to run eclcc command %s", eclccCmd.str());
            errorReader->start();
            abortWaiter->start();
            try
            {
                pipe->write(eclQuery.s.length(), eclQuery.s.str());
                pipe->closeInput();
            }
            catch (IException *e)
            {
                reportError(e);
                e->Release();
            }
            unsigned retcode = pipe->wait();
            errorReader->join();
            abortWaiter->stop();
            if (retcode == 0)
            {
                StringBuffer realdllname, dllurl;
                realdllname.append(SharedObjectPrefix).append(wuid).append(SharedObjectExtension);

                StringBuffer realdllfilename(dllPath);
                realdllfilename.append(SharedObjectPrefix).append(wuid).append(SharedObjectExtension);

                StringBuffer wuXML;
                if (!getWorkunitXMLFromFile(realdllfilename, wuXML))
                    throw makeStringException(999, "Failed to extract workunit from query dll");

                Owned<ILocalWorkUnit> embeddedWU = createLocalWorkUnit(wuXML);
                queryExtendedWU(workunit)->copyWorkUnit(embeddedWU, false, true);
                workunit->setIsClone(false);
                const char *jobname = embeddedWU->queryJobName();
                if (jobname && *jobname) //let ECL win naming job during initial compile
                    workunit->setJobName(jobname);
                if (!workunit->getDebugValueBool("obfuscateOutput", false))
                {
                    Owned<IWUQuery> query = workunit->updateQuery();
                    query->setQueryText(eclQuery.s.str());
                }

                createUNCFilename(realdllfilename.str(), dllurl);
                unsigned crc = crc_file(realdllfilename.str());

                Owned<IWUQuery> query = workunit->updateQuery();
                associateLocalFile(query, FileTypeDll, realdllfilename, "Workunit DLL", crc);
                queryDllServer().registerDll(realdllname.str(), "Workunit DLL", dllurl.str());

                cycle_t elapsedCycles = get_cycles_now() - startCycles;
                updateWorkunitTimeStat(workunit, SSTcompilestage, "compile", StTimeElapsed, NULL, cycle_to_nanosec(elapsedCycles));

                workunit->commit();
                return true;
            }
        }
        catch (IException * e)
        {
            reportError(e);
            e->Release();
        }
        return false;
    }
Example #4
0
    bool compile(const char *wuid, const char *target, const char *targetCluster)
    {
        Owned<IConstWUQuery> query = workunit->getQuery();
        if (!query)
        {
            reportError("Workunit does not contain a query", 2);
            return false;
        }

        SCMStringBuffer mainDefinition;
        SCMStringBuffer eclQuery;
        query->getQueryText(eclQuery);
        query->getQueryMainDefinition(mainDefinition);

        StringBuffer eclccProgName("eclcc");
        StringBuffer eclccCmd(" -shared");
        if (eclQuery.length())
            eclccCmd.append(" -");
        if (mainDefinition.length())
            eclccCmd.append(" -main ").append(mainDefinition);
        if (workunit->getDebugValueBool("addTimingToWorkunit", true))
            eclccCmd.append(" --timings");

        Owned<IPipeProcess> pipe = createPipeProcess();
        pipe->setenv("ECLCCSERVER_THREAD_INDEX", idxStr.str());
        Owned<IPropertyTreeIterator> options = globals->getElements("./Option");
        ForEach(*options)
        {
            IPropertyTree &option = options->query();
            const char *name = option.queryProp("@name");
            const char *value = option.queryProp("@value");
            const char *cluster = option.queryProp("@cluster");                // if cluster is set it's specific to a particular target
            if (name && (cluster==NULL || cluster[0]==0 || strcmp(cluster, targetCluster)==0))
                processOption(name, value, eclccCmd, eclccProgName, *pipe, false);
        }
        eclccCmd.appendf(" -o%s", wuid);
        eclccCmd.appendf(" -platform=%s", target);

        Owned<IStringIterator> debugValues = &workunit->getDebugValues();
        ForEach (*debugValues)
        {
            SCMStringBuffer debugStr, valueStr;
            debugValues->str(debugStr);
            workunit->getDebugValue(debugStr.str(), valueStr);
            processOption(debugStr.str(), valueStr.str(), eclccCmd, eclccProgName, *pipe, true);
        }
        if (workunit->getResultLimit())
        {
            eclccCmd.appendf(" -fapplyInstantEclTransformations=1 -fapplyInstantEclTransformationsLimit=%u", workunit->getResultLimit());
        }
        try
        {
            unsigned time = msTick();
            Owned<ErrorReader> errorReader = new ErrorReader(pipe, this);
            eclccCmd.insert(0, eclccProgName);
            pipe->run(eclccProgName, eclccCmd, ".", true, false, true, 0);
            errorReader->start();
            try
            {
                pipe->write(eclQuery.s.length(), eclQuery.s.str());
                pipe->closeInput();
            }
            catch (IException *e)
            {
                reportError(e);
                e->Release();
            }
            unsigned retcode = pipe->wait();
            errorReader->join();
            if (retcode == 0)
            {
                StringBuffer realdllname, dllurl;
                realdllname.append(SharedObjectPrefix).append(wuid).append(SharedObjectExtension);

                StringBuffer realdllfilename(dllPath);
                realdllfilename.append(SharedObjectPrefix).append(wuid).append(SharedObjectExtension);

                StringBuffer wuXML;
                if (getWorkunitXMLFromFile(realdllfilename, wuXML))
                {
                    Owned<ILocalWorkUnit> embeddedWU = createLocalWorkUnit();
                    embeddedWU->loadXML(wuXML);
                    queryExtendedWU(workunit)->copyWorkUnit(embeddedWU, true);
                    workunit->setIsClone(false);
                    SCMStringBuffer jobname;
                    if (embeddedWU->getJobName(jobname).length()) //let ECL win naming job during initial compile
                        workunit->setJobName(jobname.str());
                    Owned<IWUQuery> query = workunit->updateQuery();
                    query->setQueryText(eclQuery.s.str());
                }

                createUNCFilename(realdllfilename.str(), dllurl);
                unsigned crc = crc_file(realdllfilename.str());

                Owned<IWUQuery> query = workunit->updateQuery();
                associateLocalFile(query, FileTypeDll, realdllfilename, "Workunit DLL", crc);
                queryDllServer().registerDll(realdllname.str(), "Workunit DLL", dllurl.str());
                time = msTick()-time;
                if (workunit->getDebugValueBool("addTimingToWorkunit", true))
                    workunit->setTimerInfo("eclccserver: create workunit", NULL, time, 1, 0);

                workunit->commit();
                return true;
            }
        }
        catch (IException * e)
        {
            reportError(e);
            e->Release();
        }
        return false;
    }