bool WuResubmit(const char *wuid)
{
    Owned<IWorkUnitFactory> factory = getWorkUnitFactory();
    Owned<IWorkUnit> wu = factory->updateWorkUnit(wuid);
    if (!wu)
    {
        ERRLOG("WuResubmit(%s): could not find workunit",wuid);
        return false;
    }
    if (wu->getState()!=WUStateFailed)
    {
        ERRLOG("WuResubmit(%s): could not resubmit as workunit state is '%s'", wuid, wu->queryStateDesc());
        return false;
    }
    SCMStringBuffer token;
    wu->getSecurityToken(token);
    SCMStringBuffer user;
    SCMStringBuffer password;
    extractToken(token.str(), wuid, user, password);
    wu->resetWorkflow();
    wu->setState(WUStateSubmitted);
    wu->commit();
    wu.clear();
    submitWorkUnit(wuid,user.str(),password.str());

    PROGLOG("WuResubmit(%s): resubmitted",wuid);
    return true;
}
    virtual void write()
    {
        StringBuffer rowTag;
        OwnedRoxieString xmlpath(helper->getXmlIteratorPath());
        if (!xmlpath)
        {
            rowTag.append(DEFAULTXMLROWTAG);
        }
        else
        {
            const char *path = xmlpath;
            if (*path == '/') path++;
            if (strchr(path, '/')) UNIMPLEMENTED;
            rowTag.append(path);
        }

        StringBuffer out;
        if (!dlfn.isExternal() || firstNode()) // if external, 1 header,footer
        {
            OwnedRoxieString suppliedHeader(helper->getHeader());
            if (kind==TAKjsonwrite)
                buildJsonHeader(out, suppliedHeader, rowTag);
            else if (suppliedHeader)
                out.set(suppliedHeader);
            else
                out.set(DEFAULTXMLHEADER).newline();
            outraw->write(out.length(), out.str());
            if (calcFileCrc)
                fileCRC.tally(out.length(), out.str());
        }
        Owned<IXmlWriterExt> writer = createIXmlWriterExt(helper->getXmlFlags(), 0, NULL, (kind==TAKjsonwrite) ? WTJSON : WTStandard);
        writer->outputBeginArray(rowTag); //need this to format rows, even if not outputting it below
        while(!abortSoon)
        {
            OwnedConstThorRow row = input->ungroupedNextRow();
            if (!row)
                break;
            writer->clear().outputBeginNested(rowTag, false);
            helper->toXML((const byte *)row.get(), *writer);
            writer->outputEndNested(rowTag);
            outraw->write(writer->length(), writer->str());
            if (calcFileCrc)
                fileCRC.tally(writer->length(), writer->str());
            processed++;
        }
        if (!dlfn.isExternal() || lastNode()) // if external, 1 header,footer
        {
            OwnedRoxieString suppliedFooter(helper->getFooter());
            if (kind==TAKjsonwrite)
                buildJsonFooter(out.clear().newline(), suppliedFooter, rowTag);
            else if (suppliedFooter)
                out.set(suppliedFooter);
            else
                out.set(DEFAULTXMLFOOTER).newline();
            outraw->write(out.length(), out.str());
            if (calcFileCrc)
                fileCRC.tally(out.length(), out.str());
        }
    }
Example #3
0
static void test1()
{
    printf("Test SDS read/write\n");
    Owned<IPropertyTree> ref = createPTree("DAREGRESS");
    fn(1,2,3,0,ref);
    StringBuffer refstr;
    toXML(ref,refstr,0,XML_SortTags|XML_Format);
    printf("Created reference size %d\n",refstr.length());
    Owned<IRemoteConnection> conn = querySDS().connect("/DAREGRESS",myProcessSession(), RTM_CREATE, 1000000);
    Rconn = conn;
    IPropertyTree *root = conn->queryRoot();
    fn(1,2,3,0,root);
    conn.clear();
    printf("Created test branch 1\n");
    conn.setown(querySDS().connect("/DAREGRESS",myProcessSession(), RTM_DELETE_ON_DISCONNECT, 1000000));
    root = conn->queryRoot();
    StringBuffer s;
    toXML(root,s,0,XML_SortTags|XML_Format);
    if (strcmp(s.str(),refstr.str())!=0) {
        ERROR("Branch 1 does not match");
    }
    else
        printf("Branch 1 matches\n");
    conn.clear();
    conn.setown(querySDS().connect("/DAREGRESS",myProcessSession(), 0, 1000000));
    if (conn)
        ERROR("RTM_DELETE_ON_DISCONNECT failed");
    Rconn = querySDS().connect("/DAREGRESS",myProcessSession(), RTM_CREATE, 1000000);
    StringBuffer pn;
    fn2(1,2,3,0,pn);
    ::Release(Rconn);
    printf("Created test branch 2\n");
    Rconn = NULL;
    conn.setown(querySDS().connect("/DAREGRESS",myProcessSession(), RTM_DELETE_ON_DISCONNECT, 1000000));
    root = conn->queryRoot();
    toXML(root,s.clear(),0,XML_SortTags|XML_Format);
    if (strcmp(s.str(),refstr.str())!=0) {
        ERROR("Branch 2 does not match");
    }
    else
        printf("Branch 2 matches\n");
    conn.clear();
    conn.setown(querySDS().connect("/DAREGRESS",myProcessSession(), 0, 1000000));
    if (conn)
        ERROR("RTM_DELETE_ON_DISCONNECT failed");
}
 void stop()
 {
     if (global)
         putNext(NULL);
     stopInput(input);
     input.clear();
     dataLinkStop();
 }
 virtual void stop() override
 {
     if (!isLocal)
     {
         barrier->wait(false);
         sorter->stopMerge();
     }
     {
         CriticalBlock b(joinHelperCrit);
         joinhelper.clear();
     }
     if (strm)
     {
         strm->stop();
         strm.clear();
     }
     PARENT::stop();
 }
 void stop()
 {
     if (!stopped)
     {
         abortSoon = true;
         stopped = true;
         doStop();
     }
     input.clear();
 }
 virtual void stop()
 {
     if (input)
     {
         stopInput(input);
         input = NULL;
     }
     if(!isLocal)
     {
         barrier->wait(false);
         sorter->stopMerge();
     }
     {
         CriticalBlock b(joinHelperCrit);
         joinhelper.clear();
     }
     strm->stop();
     strm.clear();
     dataLinkStop();
 }
Example #8
0
    offset_t write(IRowStream *input)
    {
        StringBuffer tempname;
        GetTempName(tempname,"srtmrg",false);
        dataFile.setown(createIFile(tempname.str()));
        Owned<IExtRowWriter> output = createRowWriter(dataFile, rowIf);

        bool overflowed = false;
        ActPrintLog(&activity, "Local Overflow Merge start");
        unsigned ret=0;
        loop
        {
            const void *_row = input->nextRow();
            if (!_row)
                break;
            ret++;

            OwnedConstThorRow row = _row;
            offset_t start = output->getPosition();
            output->putRow(row.getLink());
            idx++;
            if (idx==interval)
            {
                idx = 0;
                if (!sampleRows.append(row.getClear()))
                {
                    // JCSMORE used to check if 'isFull()' here, but only to warn
                    // I think this is bad news, if has run out of room here...
                    // should at least warn in workunit I suspect
                    overflowsize = output->getPosition();
                    if (!overflowed)
                    {
                        WARNLOG("Sample buffer full");
                        overflowed = true;
                    }
                }
            }
            writeidxofs(start);
        }
        output->flush();
        offset_t end = output->getPosition();
        output.clear();
        writeidxofs(end);
        if (idxFileIO)
        {
            idxFileStream->flush();
            idxFileStream.clear();
            idxFileIO.clear();
        }
        if (overflowed)
            WARNLOG("Overflowed by %"I64F"d", overflowsize);
        ActPrintLog(&activity, "Local Overflow Merge done: overflow file '%s', size = %"I64F"d", dataFile->queryFilename(), dataFile->size());
        return end;
    }
Example #9
0
    void kill()
    {
        ActPrintLog("MSortSlaveActivity::kill");

        {
            CriticalBlock block(statsCs);
            mergeStats(spillStats, sorter);
            sorter.clear();
        }

        CSlaveActivity::kill();
    }
Example #10
0
 bool poll()
 {
     if (stopped||!running()) {
         PROGLOG(DAFS_SERVICE_DISPLAY_NAME " Stopping");
         if (server) {
             server->stop();
             server.clear();
         }
         return false;
     }
     return true;
 }
Example #11
0
void testMultiCluster()
{
    Owned<IGroup> grp1 = createIGroup("192.168.51.1-5");
    Owned<IGroup> grp2 = createIGroup("192.168.16.1-5");
    Owned<IGroup> grp3 = createIGroup("192.168.53.1-5");
    queryNamedGroupStore().add("testgrp1",grp1);
    queryNamedGroupStore().add("testgrp2",grp2);
    queryNamedGroupStore().add("testgrp3",grp3);

    Owned<IFileDescriptor> fdesc = createFileDescriptor();
    fdesc->setDefaultDir("/c$/thordata/test");
    fdesc->setPartMask("testfile1._$P$_of_$N$");
    fdesc->setNumParts(5);
    ClusterPartDiskMapSpec mapping;
    fdesc->addCluster(grp1,mapping);
    fdesc->addCluster(grp2,mapping);
    fdesc->addCluster(grp3,mapping);
    queryDistributedFileDirectory().removeEntry("test::testfile1",UNKNOWN_USER);
    Owned<IDistributedFile> file = queryDistributedFileDirectory().createNew(fdesc);
    queryDistributedFileDirectory().removeEntry("test::testfile1",UNKNOWN_USER);
    file->attach("test::testfile1",UNKNOWN_USER);
    StringBuffer name;
    unsigned i;
    for (i=0;i<file->numClusters();i++)
        PROGLOG("cluster[%d] = %s",i,file->getClusterName(i,name.clear()).str());
    file.clear();
    file.setown(queryDistributedFileDirectory().lookup("test::testfile1",UNKNOWN_USER));
    for (i=0;i<file->numClusters();i++)
        PROGLOG("cluster[%d] = %s",i,file->getClusterName(i,name.clear()).str());
    file.clear();
    file.setown(queryDistributedFileDirectory().lookup("test::testfile1@testgrp1",UNKNOWN_USER));
    for (i=0;i<file->numClusters();i++)
        PROGLOG("cluster[%d] = %s",i,file->getClusterName(i,name.clear()).str());
    file.clear();
    queryDistributedFileDirectory().removePhysical("test::testfile1@testgrp2",UNKNOWN_USER);
    file.setown(queryDistributedFileDirectory().lookup("test::testfile1",UNKNOWN_USER));
    for (i=0;i<file->numClusters();i++)
        PROGLOG("cluster[%d] = %s",i,file->getClusterName(i,name.clear()).str());
}
Example #12
0
    void stop()
    {
        try {
            if (server) 
                server->stop();
        }
        catch (IException *e) {
            EXCLOG(e,"dfuplus(dafilesrvstop)");
            e->Release();
        }
        server.clear();

    }
    virtual void start() override
    {
        ActivityTimer s(totalCycles, timeActivities);
        ActPrintLog(rolloverEnabled ? "GROUP: is global" : "GROUP: is local");
        PARENT::start();
        eogNext = prevEog = eof = false;
        if (rolloverEnabled)
        {
            useRollover = !lastNode();
#ifdef _TESTING
            ActPrintLog("Node number = %d, Total Nodes = %d", queryJobChannel().queryMyRank(), container.queryJob().querySlaves());
#endif
        }

        stream.set(inputStream);
        startLastGroup = getDataLinkGlobalCount();
        next.setown(getNext());

        if (rolloverEnabled && !firstNode())  // 1st node can have nothing to send
        {
            Owned<IThorRowCollector> collector = createThorRowCollector(*this, this, NULL, stableSort_none, rc_mixed, SPILL_PRIORITY_SPILLABLE_STREAM);
            Owned<IRowWriter> writer = collector->getWriter();
            if (next)
            {
                ActPrintLog("GROUP: Sending first group to previous node(%d)", queryJobChannel().queryMyRank()-1);
                for (;;)
                {
                    writer->putRow(next.getLink());
                    if (abortSoon)
                        break; //always send group even when aborting
                    OwnedConstThorRow next2 = getNext();
                    if (!next2)
                    {
                        eof = true;
                        break;
                    }
                    else if (!helper->isSameGroup(next2, next))
                    {
                        next.setown(next2.getClear());
                        break;
                    }
                    next.setown(next2.getClear());
                }
            }
            writer.clear();
            ActPrintLog("GROUP: %" RCPF "d records to send", collector->numRows());
            Owned<IRowStream> strm = collector->getStream();
            rowServer.setown(createRowServer(this, strm, queryJobChannel().queryJobComm(), mpTag));
        }
    }
static IConstWorkUnit * getWorkunit(ICodeContext * ctx, const char * wuid)
{
    StringBuffer _wuid(wuid);
    if (!_wuid.length())
        return NULL;
    wuid = _wuid.toUpperCase().str();
    Owned<IWorkUnitFactory> wuFactory = getWorkunitFactory(ctx);
    Owned<IConstWorkUnit> wu = wuFactory->openWorkUnit(wuid);
    if (wu)
    {
        if (!checkScopeAuthorized(ctx->queryUserDescriptor(), wu->queryWuScope()))
            wu.clear();
    }
    return wu.getClear();
}
Example #15
0
 void stop()
 {
     if (output) {
         output->stop();
         output.clear();
     }
     ActPrintLog("SORT waiting barrier.2");
     barrier->wait(false);
     ActPrintLog("SORT barrier.2 raised");
     if (input)
         stopInput(input);
     sorter->stopMerge();
     ActPrintLog("SORT waiting for merge");
     dataLinkStop();
 }
Example #16
0
 virtual void stop() override
 {
     if (output)
     {
         output->stop();
         output.clear();
     }
     if (hasStarted())
     {
         ActPrintLog("SORT waiting barrier.2");
         barrier->wait(false);
         ActPrintLog("SORT barrier.2 raised");
         ActPrintLog("SORT waiting for merge");
         sorter->stopMerge();
     }
     PARENT::stop();
 }
Example #17
0
int main(int argc, const char *argv[])
{
    InitModuleObjects();
    EnableSEHtoExceptionMapping();

    NoQuickEditSection xxx;

    Owned<IFile> file = createIFile("dfuserver.xml");
    if (file->exists())
        globals.setown(createPTreeFromXMLFile("dfuserver.xml", ipt_caseInsensitive));
    else
        globals.setown(readOldIni());

    for (unsigned i=1;i<(unsigned)argc;i++) {
        const char *arg = argv[i];
        StringBuffer prop("@");
        StringBuffer val;
        while (*arg && *arg != '=')
            prop.append(*arg++);
        if (*arg) {
            arg++;
            while (isspace(*arg))
                arg++;
            val.append(arg);
            prop.clip();
            val.clip();
            if (prop.length()>1)
                globals->setProp(prop.str(), val.str());
        }
    }
    StringBuffer daliServer;
    StringBuffer queue;
    if (!globals->getProp("@DALISERVERS", daliServer)||!globals->getProp("@QUEUE", queue)) {
        usage();
        globals.clear();
        releaseAtoms();
        return 1;
    }
    Owned<IFile> sentinelFile;
    bool stop = globals->getPropInt("@STOP",0)!=0;
    if (!stop) {
        sentinelFile.setown(createSentinelTarget());
        removeSentinelFile(sentinelFile);

        StringBuffer logname;
        StringBuffer logdir;
        if (!getConfigurationDirectory(globals->queryPropTree("Directories"),"log","dfuserver",globals->queryProp("@name"),logdir))
            globals->getProp("@LOG_DIR", logdir);
        if (logdir.length() && recursiveCreateDirectory(logdir.str()))
            logname.append(logdir);
        else
            appendCurrentDirectory(logname, true);

        if (logname.length() && logname.charAt(logname.length()-1) != PATHSEPCHAR)
            logname.append(PATHSEPCHAR);
        logname.append("dfuserver");
        StringBuffer aliasLogName(logname);
        aliasLogName.append(".log");
        fileMsgHandler = getRollingFileLogMsgHandler(logname.str(), ".log", MSGFIELD_STANDARD, false, true, NULL, aliasLogName.str());
        queryLogMsgManager()->addMonitorOwn(fileMsgHandler, getCategoryLogMsgFilter(MSGAUD_all, MSGCLS_all, 1000));
    }
    StringBuffer ftslogdir;
    if (getConfigurationDirectory(globals->queryPropTree("Directories"),"log","ftslave",globals->queryProp("@name"),ftslogdir)) // NB instance deliberately dfuserver's
        setFtSlaveLogDir(ftslogdir.str());
    setRemoteSpawnSSH(
        globals->queryProp("SSH/@SSHidentityfile"),
        globals->queryProp("SSH/@SSHusername"),
        globals->queryProp("SSH/@SSHpassword"),
        globals->getPropInt("SSH/@SSHtimeout",0),
        globals->getPropInt("SSH/@SSHretries",3),
        "run_");
    bool enableSNMP = globals->getPropInt("@enableSNMP")!=0;
    CSDSServerStatus *serverstatus=NULL;
    Owned<IReplicateServer> replserver;
    try {
        Owned<IGroup> serverGroup = createIGroup(daliServer.str(),DALI_SERVER_PORT);
        initClientProcess(serverGroup, DCR_DfuServer, 0, NULL, NULL, stop?(1000*30):MP_WAIT_FOREVER);
        setPasswordsFromSDS();

        if(!stop)
        {
            if (globals->getPropBool("@enableSysLog",true))
                UseSysLogForOperatorMessages();

            serverstatus = new CSDSServerStatus("DFUserver");
            setDaliServixSocketCaching(true); // speeds up lixux operations

            startLogMsgParentReceiver();    // for auditing
            connectLogMsgManagerToDali();

            engine.setown(createDFUengine());
            addAbortHandler(exitDFUserver);
        }
        const char *q = queue.str();
        loop {
            StringBuffer subq;
            const char *comma = strchr(q,',');
            if (comma)
                subq.append(comma-q,q);
            else
                subq.append(q);
            if (stop) {
                stopDFUserver(subq.str());
            }
            else {
                StringBuffer mask;
                mask.appendf("Queue[@name=\"%s\"][1]",subq.str());
                IPropertyTree *t=serverstatus->queryProperties()->queryPropTree(mask.str());
                if (t)
                    t->setPropInt("@num",t->getPropInt("@num",0)+1);
                else {
                    t = createPTree();
                    t->setProp("@name",subq.str());
                    t->setPropInt("@num",1);
                    serverstatus->queryProperties()->addPropTree("Queue",t);
                }
                serverstatus->commitProperties();
                engine->setDefaultTransferBufferSize((size32_t)globals->getPropInt("@transferBufferSize"));
                engine->startListener(subq.str(),serverstatus);
            }
            if (!comma)
                break;
            q = comma+1;
            if (!*q)
                break;
        }
        q = globals->queryProp("@MONITORQUEUE");
        if (q&&*q) {
            if (stop) {
                stopDFUserver(q);
            }
            else {
                IPropertyTree *t=serverstatus->queryProperties()->addPropTree("MonitorQueue",createPTree());
                t->setProp("@name",q);
                engine->startMonitor(q,serverstatus,globals->getPropInt("@MONITORINTERVAL",60)*1000);
            }
        }
        q = globals->queryProp("@REPLICATEQUEUE");
        if (q&&*q) {
            if (stop) {
                // TBD?
            }
            else {
                replserver.setown(createReplicateServer(q));
                replserver->runServer();
            }
        }
        if (!stop) {
            serverstatus->commitProperties();

            writeSentinelFile(sentinelFile);

            engine->joinListeners();
            if (replserver.get())
                replserver->stopServer();
            LOG(MCprogress, unknownJob, "Exiting");
        }

    }
    catch(IException *e){
        EXCLOG(e, "DFU Server Exception: ");
        e->Release();
    }
    catch (const char *s) {
        WARNLOG("DFU: %s",s);
    }

    delete serverstatus;
    if (stop)
        Sleep(2000);    // give time to stop
    engine.clear();
    globals.clear();
    closeEnvironment();
    closedownClientProcess();
    UseSysLogForOperatorMessages(false);
    setDaliServixSocketCaching(false);
    releaseAtoms();
    return 0;
}
Example #18
0
static void test2() 
{
    const size32_t recsize = 17;
    printf("Test DFS\n");
    StringBuffer s;
    unsigned i;
    unsigned n;
    unsigned t;
    queryNamedGroupStore().remove("daregress_group");
    queryDistributedFileDirectory().removeEntry("daregress::superfile1");
    SocketEndpointArray epa;
    for (n=0;n<400;n++) {
        s.clear().append("192.168.").append(n/256).append('.').append(n%256);
        SocketEndpoint ep(s.str());
        epa.append(ep);
    }
    Owned<IGroup> group = createIGroup(epa); 
    queryNamedGroupStore().add("daregress_group",group,true);
    if (!queryNamedGroupStore().find(group,s.clear()))
        ERROR("Created logical group not found");
    if (stricmp(s.str(),"daregress_group")!=0)
        ERROR("Created logical group found with wrong name");
    group.setown(queryNamedGroupStore().lookup("daregress_group"));
    if (!group)
        ERROR("named group lookup failed");
    printf("Named group created    - 400 nodes\n");
    for (i=0;i<100;i++) {
        Owned<IPropertyTree> pp = createPTree("Part");
        Owned<IFileDescriptor>fdesc = createFileDescriptor();
        fdesc->setDefaultDir("c:\\thordata\\regress");
        n = 9;
        for (unsigned k=0;k<400;k++) {
            s.clear().append("192.168.").append(n/256).append('.').append(n%256);
            Owned<INode> node = createINode(s.str());
            pp->setPropInt64("@size",(n*777+i)*recsize);
            s.clear().append("daregress_test").append(i).append("._").append(n+1).append("_of_400");
            fdesc->setPart(n,node,s.str(),pp);
            n = (n+9)%400;
        }
        fdesc->queryProperties().setPropInt("@recordSize",17);
        s.clear().append("daregress::test").append(i);
        queryDistributedFileDirectory().removeEntry(s.str());
        StringBuffer cname;
        Owned<IDistributedFile> dfile = queryDistributedFileDirectory().createNew(fdesc);
        if (stricmp(dfile->getClusterName(0,cname),"daregress_group")!=0)
            ERROR1("Cluster name wrong %d",i);
        s.clear().append("daregress::test").append(i);
        dfile->attach(s.str());
    }
    printf("DFile create done      - 100 files\n");
    unsigned samples = 5;
    t = 33;
    for (i=0;i<100;i++) {
        s.clear().append("daregress::test").append(t);
        if (!queryDistributedFileDirectory().exists(s.str())) 
            ERROR1("Could not find %s",s.str());
        Owned<IDistributedFile> dfile = queryDistributedFileDirectory().lookup(s.str());
        if (!dfile) {
            ERROR1("Could not find %s",s.str());
            continue;
        }
        offset_t totsz = 0;
        n = 11;
        for (unsigned k=0;k<400;k++) {
            Owned<IDistributedFilePart> part = dfile->getPart(n);
            if (!part) {
                ERROR2("part not found %d %d",t,n);
                continue;
            }
            s.clear().append("192.168.").append(n/256).append('.').append(n%256);
            Owned<INode> node = createINode(s.str());
            if (!node->equals(part->queryNode()))
                ERROR2("part node mismatch %d, %d",t,n);
            if (part->getFileSize(false,false)!=(n*777+t)*recsize)
                ERROR4("size node mismatch %d, %d, %d, %d",t,n,(unsigned)part->getFileSize(false,false),(n*777+t)*recsize);
            s.clear().append("daregress_test").append(t).append("._").append(n+1).append("_of_400");
/* ** TBD
            if (stricmp(s.str(),part->queryPartName())!=0)
                ERROR4("part name mismatch %d, %d '%s' '%s'",t,n,s.str(),part->queryPartName());
*/
            totsz += (n*777+t)*recsize;
            if ((samples>0)&&(i+n+t==k)) {
                samples--;
                RemoteFilename rfn;
                part->getFilename(rfn,samples%2);
                StringBuffer fn;
                rfn.getRemotePath(fn);
                printf("SAMPLE: %d,%d %s\n",t,n,fn.str());
            }
            n = (n+11)%400;
        }
        if (totsz!=dfile->getFileSize(false,false))
            ERROR1("total size mismatch %d",t);
        t = (t+33)%100; 
    }
    printf("DFile lookup done      - 100 files\n");

    // check iteration
    __int64 crctot = 0;
    unsigned np = 0;
    unsigned totrows = 0;
    Owned<IDistributedFileIterator> fiter = queryDistributedFileDirectory().getIterator("daregress::*",false); 
    Owned<IDistributedFilePartIterator> piter;
    ForEach(*fiter) {
        piter.setown(fiter->query().getIterator()); 
        ForEach(*piter) {
            RemoteFilename rfn;
            StringBuffer s;
            piter->query().getFilename(rfn,0);
            rfn.getRemotePath(s);
            piter->query().getFilename(rfn,1);
            rfn.getRemotePath(s);
            crctot += crc32(s.str(),s.length(),0);
            np++;
            totrows += (unsigned)(piter->query().getFileSize(false,false)/fiter->query().queryProperties().getPropInt("@recordSize",-1));
        }
    }
    piter.clear();
    fiter.clear();
    printf("DFile iterate done     - %d parts, %d rows, CRC sum %"I64F"d\n",np,totrows,crctot);
    Owned<IDistributedSuperFile> sfile;
    sfile.setown(queryDistributedFileDirectory().createSuperFile("daregress::superfile1",true));
    for (i = 0;i<100;i++) {
        s.clear().append("daregress::test").append(i);
        sfile->addSubFile(s.str());
    }
    sfile.clear();
    sfile.setown(queryDistributedFileDirectory().lookupSuperFile("daregress::superfile1"));
    if (!sfile) {
        ERROR("Could not find added superfile");
        return;
    }
    __int64 savcrc = crctot;
    crctot = 0;
    np = 0;
    totrows = 0;
    size32_t srs = (size32_t)sfile->queryProperties().getPropInt("@recordSize",-1);
    if (srs!=17)
        ERROR1("Superfile does not match subfile row size %d",srs);
    piter.setown(sfile->getIterator()); 
    ForEach(*piter) {
        RemoteFilename rfn;
        StringBuffer s;
        piter->query().getFilename(rfn,0);
        rfn.getRemotePath(s);
        piter->query().getFilename(rfn,1);
        rfn.getRemotePath(s);
        crctot += crc32(s.str(),s.length(),0);
        np++;
        totrows += (unsigned)(piter->query().getFileSize(false,false)/srs);
    }
    piter.clear();
    printf("Superfile iterate done - %d parts, %d rows, CRC sum %"I64F"d\n",np,totrows,crctot);
    if (crctot!=savcrc)
        ERROR("SuperFile does not match sub files");
    unsigned tr = (unsigned)(sfile->getFileSize(false,false)/srs); 
    if (totrows!=tr)
        ERROR1("Superfile size does not match part sum %d",tr);
    sfile->detach();
    sfile.clear();
    sfile.setown(queryDistributedFileDirectory().lookupSuperFile("daregress::superfile1"));
    if (sfile)
        ERROR("Superfile deletion failed");
    t = 37;
    for (i=0;i<100;i++) {
        s.clear().append("daregress::test").append(t);
        if (i%1) {
            Owned<IDistributedFile> dfile = queryDistributedFileDirectory().lookup(s.str());
            if (!dfile)
                ERROR1("Could not find %s",s.str());
            dfile->detach();
        }
        else 
            queryDistributedFileDirectory().removeEntry(s.str());
        t = (t+37)%100; 
    }
    printf("DFile removal complete\n");
    t = 39;
    for (i=0;i<100;i++) {
        if (queryDistributedFileDirectory().exists(s.str()))
            ERROR1("Found %s after deletion",s.str());
        Owned<IDistributedFile> dfile = queryDistributedFileDirectory().lookup(s.str());
        if (dfile)
            ERROR1("Found %s after deletion",s.str());
        t = (t+39)%100; 
    }
    printf("DFile removal check complete\n");
    queryNamedGroupStore().remove("daregress_group");
    if (queryNamedGroupStore().lookup("daregress_group"))
        ERROR("Named group not removed");
}
Example #19
0
MODULE_EXIT()
{
    servers.kill(); // should already be clear when stopped
    serverConfig.clear();
    delete stopServerCrit;
}
bool CEclDirectEx::onRunEclEx(IEspContext &context, IEspRunEclExRequest & req, IEspRunEclExResponse & resp)
{
    if (!context.validateFeatureAccess(ECLDIRECT_ACCESS, SecAccess_Full, false))
        throw MakeStringException(-1, "EclDirect access permission denied.");

    const char* eclText = req.getEclText();
    if (!eclText || !*eclText)
    {
        resp.setResults("<Exception><Source>ESP</Source><Message>No Ecl Text provided</Message></Exception>");
        return true;
    }

    StringBuffer user;
    if (!context.getUserID(user).length())
        user.append(req.getUserName());

    Owned <IWorkUnitFactory> factory = getWorkUnitFactory(context.querySecManager(), context.queryUser());
    Owned <IWorkUnit> workunit;
    if (!user.length())
        workunit.setown(factory->createWorkUnit(NULL, "ECL-Direct", ""));
    else
    {
        workunit.setown(factory->createWorkUnit(NULL, "ECL-Direct", user.str()));
        workunit->setUser(user.str());
    }

    Owned<IWUQuery> query = workunit->updateQuery();
    query->setQueryText(eclText);
    query.clear();

    const char* cluster = req.getCluster();
    if (!cluster || !*cluster || !stricmp(cluster, "default"))
        cluster = defaultCluster.str();

    if (!cluster || !*cluster)
        throw MakeStringException(-1, "No Cluster Specified");

    if (!isValidCluster(cluster))
        throw MakeStringException(-1, "Invalid TargetCluster %s Specified", cluster);

    workunit->setClusterName(cluster);

    const char* snapshot = req.getSnapshot();
    if (snapshot && *snapshot)
        workunit->setSnapshot(snapshot);

    if (req.getResultLimit())
        workunit->setResultLimit(req.getResultLimit());

    // Execute it
    SCMStringBuffer wuid;
    workunit->getWuid(wuid);
    workunit->setAction(WUActionRun);
    workunit->setState(WUStateSubmitted);
    workunit.clear();

    resp.setWuid(wuid.str());

    submitWorkUnit(wuid.str(), context.querySecManager(), context.queryUser());

    if (!waitForWorkUnitToComplete(wuid.str(), (req.getWait_isNull()) ? defaultWait : req.getWait()))
    {
        StringBuffer result;
        result.appendf("<Exception><Source>ESP</Source><Message>Timed out waiting for job to complete: %s</Message></Exception>", wuid.str());
        resp.setResults(result.str());
        return true;
    }

    if (!deleteWorkunits && context.queryRequestParameters()->hasProp("redirect"))
    {
        StringBuffer url("/WsWorkunits/WUInfo?Wuid=");
        resp.setRedirectUrl(url.append(wuid).str());
        return true;
    }

    Owned<IConstWorkUnit> cw = factory->openWorkUnit(wuid.str(), false);
    EclDirectWUExceptions errors(*cw);
    resp.setErrors(errors);

    if (req.getIncludeResults())
    {
        StringBuffer results;
        CRunEclExFormat outputFormat = req.getFormat();
        Owned<IWuWebView> web = createWuWebView(wuid.str(), NULL, NULL, getCFD(), true);
        if (!web)
            results.appendf("<Exception><Source>ESP</Source><Message>Failed loading result workunit %s</Message></Exception>", wuid.str());
        else if (outputFormat == CRunEclExFormat_Table)
        {
            StringBuffer xsltfile(getCFD());
            web->applyResultsXSLT(xsltfile.append("xslt/wsecl3_result.xslt").str(), results);
        }
        else
        {
            unsigned xmlflags = 0;
            if (outputFormat != CRunEclExFormat_ExtendedXml)
                xmlflags |= WWV_OMIT_SCHEMAS;
            if (context.queryRequestParameters()->hasProp("display_xslt"))
                xmlflags |= WWV_USE_DISPLAY_XSLT;
            else
                xmlflags |= WWV_OMIT_XML_DECLARATION;
            web->expandResults(results, xmlflags);
        }
        resp.setResults(results.str());
    }

    if (req.getIncludeGraphs())
    {
        Owned<IConstWUGraphIterator> it = &cw->getGraphs(GraphTypeAny);
        StringBuffer xgmml("<Graphs>");
        SCMStringBuffer s;
        ForEach(*it)
            xgmml.append(it->query().getXGMML(s, true).str());
        xgmml.append("</Graphs>");
        resp.setGraphsXGMML(xgmml.str());
    }

    if (deleteWorkunits)
        deleteEclDirectWorkunit(factory, wuid.str());

    return true;
}
bool CEclDirectEx::onRunEcl(IEspContext &context, IEspRunEclRequest & req, IEspRunEclResponse & resp)
{
    if (!context.validateFeatureAccess(ECLDIRECT_ACCESS, SecAccess_Full, false))
        throw MakeStringException(-1, "EclDirect access permission denied.");

    StringBuffer user;
    if (!context.getUserID(user).length())
        user.append(req.getUserName());

    Owned <IWorkUnitFactory> factory = getWorkUnitFactory(context.querySecManager(), context.queryUser());
    Owned <IWorkUnit> workunit;
    if (!user.length())
        workunit.setown(factory->createWorkUnit(NULL, "ECL-Direct", ""));
    else
    {
        workunit.setown(factory->createWorkUnit(NULL, "ECL-Direct", user.str()));
        workunit->setUser(user.str());
    }

    Owned<IWUQuery> query = workunit->updateQuery();
    query->setQueryText(req.getEclText());
    query.clear();

    const char* clustername = req.getCluster();
    if (!clustername || !*clustername || strieq(clustername, "default"))
        clustername = defaultCluster.str();

    if (!clustername || !*clustername)
        throw MakeStringException(-1, "No Cluster Specified");

    if (!isValidCluster(clustername))
        throw MakeStringException(-1, "Invalid TargetCluster %s Specified", clustername);

    workunit->setClusterName(clustername);
    if (req.getLimitResults())
        workunit->setResultLimit(100);

    const char* snapshot = req.getSnapshot();
    if (snapshot && *snapshot)
        workunit->setSnapshot(snapshot);

    // Execute it
    SCMStringBuffer wuid;
    
    workunit->getWuid(wuid);
    workunit->setAction(WUActionRun);
    workunit->setState(WUStateSubmitted);
    workunit.clear();

    submitWorkUnit(wuid.str(), context.querySecManager(), context.queryUser());

    if (waitForWorkUnitToComplete(wuid.str(), defaultWait))
    {
        Owned<IConstWorkUnit> cw = factory->openWorkUnit(wuid.str(), false);

        SCMStringBuffer resultXML;
        getFullWorkUnitResultsXML(context.queryUserId(), context.queryPassword(), cw.get(), resultXML);
        resp.setResults(resultXML.str());

        cw.clear();

        if (deleteWorkunits)
            deleteEclDirectWorkunit(factory, wuid.str());
    }
    else
    {
        // Don't delete these ones...
        DBGLOG("WorkUnit %s timed out", wuid.str());
        
        StringBuffer result;
        result.appendf("<Exception><Source>ESP</Source><Message>Timed out waiting for job to complete: %s</Message></Exception>", wuid.str());
        resp.setResults(result.str());
    }

    return true;
}
Example #22
0
 void kill()
 {
     ActPrintLog("MSortSlaveActivity::kill");
     sorter.clear();
     CSlaveActivity::kill();
 }
 virtual void kill()
 {
     sorter.clear();
     CSlaveActivity::kill();
 }
Example #24
0
int main(int argc,char **argv) 
{
    InitModuleObjects();
    EnableSEHtoExceptionMapping();
#ifndef __64BIT__
    // Restrict stack sizes on 32-bit systems
    Thread::setDefaultStackSize(0x10000);   // 64K stack (also set in windows DSP)
#endif
    Owned<IFile> sentinelFile = createSentinelTarget();
    removeSentinelFile(sentinelFile);

    SocketEndpoint listenep;
    unsigned sendbufsize = 0;
    unsigned recvbufsize = 0;
    int i = 1;
    bool isdaemon = (memicmp(argv[0]+strlen(argv[0])-4,".exe",4)==0);
    // bit of a kludge for windows - if .exe not specified then not daemon
    bool locallisten = false;
    const char *logdir=NULL;
    bool requireauthenticate = false;
    StringBuffer logDir;
    StringBuffer instanceName;

   //Get SSL Settings
    const char *    sslCertFile;
    bool            useSSL;
    unsigned short  dafsPort;//DAFILESRV_PORT or SECURE_DAFILESRV_PORT
    querySecuritySettings(&useSSL, &dafsPort, &sslCertFile, NULL);

    unsigned parallelRequestLimit = DEFAULT_PARALLELREQUESTLIMIT;
    unsigned throttleDelayMs = DEFAULT_THROTTLEDELAYMS;
    unsigned throttleCPULimit = DEFAULT_THROTTLECPULIMIT;

    Owned<IPropertyTree> env = getHPCCEnvironment();
    if (env)
    {
        StringBuffer dafilesrvPath("Software/DafilesrvProcess");
        if (instanceName.length())
            dafilesrvPath.appendf("[@name=\"%s\"]", instanceName.str());
        IPropertyTree *daFileSrv = env->queryPropTree(dafilesrvPath);
        if (daFileSrv)
        {
            // global DaFileSrv settings:
            parallelRequestLimit = daFileSrv->getPropInt("@parallelRequestLimit", DEFAULT_PARALLELREQUESTLIMIT);
            throttleDelayMs = daFileSrv->getPropInt("@throttleDelayMs", DEFAULT_THROTTLEDELAYMS);
            throttleCPULimit = daFileSrv->getPropInt("@throttleCPULimit", DEFAULT_THROTTLECPULIMIT);

            // any overrides by Instance definitions?
            // NB: This won't work if netAddress is "." or if we start supporting hostnames there
            StringBuffer ipStr;
            queryHostIP().getIpText(ipStr);
            VStringBuffer daFileSrvPath("Instance[@netAddress=\"%s\"]", ipStr.str());
            IPropertyTree *dafileSrvInstance = daFileSrv->queryPropTree(daFileSrvPath);
            if (dafileSrvInstance)
            {
                parallelRequestLimit = dafileSrvInstance->getPropInt("@parallelRequestLimit", parallelRequestLimit);
                throttleDelayMs = dafileSrvInstance->getPropInt("@throttleDelayMs", throttleDelayMs);
                throttleCPULimit = dafileSrvInstance->getPropInt("@throttleCPULimit", throttleCPULimit);
            }
        }
    }

    while (argc>i) {
        if (stricmp(argv[i],"-D")==0) {
            i++;
            isdaemon = true;
        }
        else if (stricmp(argv[i],"-R")==0) { // for remote run
            i++;
#ifdef _WIN32
            isdaemon = false;
#else
            isdaemon = true;
#endif
        }
        else if (stricmp(argv[i],"-A")==0) { 
            i++;
            requireauthenticate = true;
        }
        else if ((argv[i][0]=='-')&&(toupper(argv[i][1])=='T')&&(!argv[i][2]||isdigit(argv[i][2]))) {
            if (argv[i][2])
                setDafsTrace(NULL,(byte)atoi(argv[i]+2));
            i++;
            isdaemon = false;
        }
        else if ((argc>i+1)&&(stricmp(argv[i],"-L")==0)) { 
            i++;
            logDir.clear().append(argv[i++]);
        }
        else if ((argc>i+1)&&(stricmp(argv[i],"-I")==0)) {
            i++;
            instanceName.clear().append(argv[i++]);
        }
        else if (stricmp(argv[i],"-LOCAL")==0) { 
            i++;
            locallisten = true;
        }
        else if (stricmp(argv[i],"-NOSSL")==0) {//overrides config setting
            i++;
            if (useSSL)
            {
                PROGLOG("DaFileSrv SSL specified in config but overridden by -NOSSL in command line");
                useSSL = false;
                dafsPort = DAFILESRV_PORT;
            }
        }
        else
            break;
    }

    if (useSSL && !sslCertFile)
    {
        ERRLOG("DaFileSrv SSL specified but certificate file information missing from environment.conf");
        exit(-1);
    }

    if (0 == logDir.length())
    {
        getConfigurationDirectory(NULL,"log","dafilesrv",instanceName.str(),logDir);
        if (0 == logDir.length())
            logDir.append(".");
    }
    if (instanceName.length())
    {
        addPathSepChar(logDir);
        logDir.append(instanceName.str());
    }

#ifdef _WIN32
    if ((argc>i)&&(stricmp(argv[i],"-install")==0)) {
        if (installService(DAFS_SERVICE_NAME,DAFS_SERVICE_DISPLAY_NAME,NULL)) {
            PROGLOG(DAFS_SERVICE_DISPLAY_NAME " Installed");
            return 0;
        }
        return 1;
    }
    if ((argc>i)&&(stricmp(argv[i],"-remove")==0)) {
        if (uninstallService(DAFS_SERVICE_NAME,DAFS_SERVICE_DISPLAY_NAME)) {
            PROGLOG(DAFS_SERVICE_DISPLAY_NAME " Uninstalled");
            return 0;
        }
        return 1;
    }
#endif
    if (argc == i)
        listenep.port = dafsPort;
    else {
        if (strchr(argv[i],'.')||!isdigit(argv[i][0]))
            listenep.set(argv[i], dafsPort);
        else
            listenep.port = atoi(argv[i]);
        if (listenep.port==0) {
            usage();
            exit(-1);
        }
        sendbufsize = (argc>i+1)?(atoi(argv[i+1])*1024):0;
        recvbufsize = (argc>i+2)?(atoi(argv[i+2])*1024):0;
    }
    if (isdaemon) {
#ifdef _WIN32
        class cserv: public CService
        {
            bool stopped;
            bool started;
            SocketEndpoint listenep;
            bool useSSL;
            bool requireauthenticate;

            
            class cpollthread: public Thread
                
            {
                cserv *parent;
            public:
                cpollthread( cserv *_parent ) 
                : Thread("CService::cpollthread"), parent(_parent) 
                {
                }
                int run() 
                { 
                    while (parent->poll())
                        Sleep(1000);
                    return 1;
                }
            } pollthread;
            Owned<IRemoteFileServer> server;

        public:

            cserv(SocketEndpoint _listenep, bool _useSSL)
                : listenep(_listenep),useSSL(_useSSL),pollthread(this)
            {
                stopped = false;
                started = false;
            }

            virtual ~cserv()
            {
                stopped = true;
                if (started)
                    pollthread.join();
            }

            bool init()
            {
                PROGLOG(DAFS_SERVICE_DISPLAY_NAME " Initialized");
                started = true;
                pollthread.start();
                return true;
            }

            bool poll()
            {
                if (stopped||!running()) {
                    PROGLOG(DAFS_SERVICE_DISPLAY_NAME " Stopping");
                    if (server) {
                        server->stop();
                        server.clear();
                    }
                    return false;
                }
                return true;
            }

            void run()
            {
                // Get params from HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Services\DaFileSrv\Parameters
                
                int requireauthenticate=0;
                HKEY hkey;
                if (RegOpenKeyEx(HKEY_LOCAL_MACHINE,
                                 "SYSTEM\\CurrentControlSet\\Services\\DaFileSrv\\Parameters",
                                 0,
                                 KEY_QUERY_VALUE,
                                 &hkey) == ERROR_SUCCESS) {
                    DWORD dwType = 0;
                    DWORD dwSize = sizeof(requireauthenticate);
                    RegQueryValueEx(hkey,
                                    "RequireAuthentication",
                                    NULL,
                                    &dwType,
                                    (BYTE*)&requireauthenticate,
                                    &dwSize);
                    RegCloseKey(hkey);
                }
                StringBuffer eps;
                if (listenep.isNull())
                    eps.append(listenep.port);
                else
                    listenep.getUrlStr(eps);
                enableDafsAuthentication(requireauthenticate!=0);
                PROGLOG("Opening " DAFS_SERVICE_DISPLAY_NAME " on %s%s", useSSL?"SECURE ":"",eps.str());
                const char * verstring = remoteServerVersionString();
                PROGLOG("Version: %s", verstring);
                PROGLOG("Authentication:%s required",requireauthenticate?"":" not");
                PROGLOG(DAFS_SERVICE_DISPLAY_NAME " Running");
                server.setown(createRemoteFileServer(parallelRequestLimit, throttleDelayMs, throttleCPULimit));
                try {
                    server->run(listenep, useSSL);
                }
                catch (IException *e) {
                    EXCLOG(e,DAFS_SERVICE_NAME);
                    e->Release();
                }
                PROGLOG(DAFS_SERVICE_DISPLAY_NAME " Stopped");
                stopped = true;
            }
        } service(listenep, useSSL);
        service.start();
        return 0;
#else
        int ret = initDaemon();
        if (ret)
            return ret;
#endif
    }
    {
        Owned<IComponentLogFileCreator> lf = createComponentLogFileCreator(logDir.str(), "DAFILESRV");
        lf->setCreateAliasFile(false);
        lf->setMaxDetail(TopDetail);
        lf->beginLogging();
    }

    PROGLOG("Parallel request limit = %d, throttleDelayMs = %d, throttleCPULimit = %d", parallelRequestLimit, throttleDelayMs, throttleCPULimit);

    const char * verstring = remoteServerVersionString();
    StringBuffer eps;
    if (listenep.isNull())
        eps.append(listenep.port);
    else
        listenep.getUrlStr(eps);
    enableDafsAuthentication(requireauthenticate);
    PROGLOG("Opening Dali File Server on %s%s", useSSL?"SECURE ":"",eps.str());
    PROGLOG("Version: %s", verstring);
    PROGLOG("Authentication:%s required",requireauthenticate?"":" not");
    startPerformanceMonitor(10*60*1000, PerfMonStandard);
    server.setown(createRemoteFileServer(parallelRequestLimit, throttleDelayMs, throttleCPULimit));
    writeSentinelFile(sentinelFile);
    try {
        server->run(listenep, useSSL);
    }
    catch (IException *e) {
        EXCLOG(e,"DAFILESRV");
        e->Release();
    }
    if (server)
        server->stop();
    server.clear();
    PROGLOG("Stopped Dali File Server");

    return 0;
}
Example #25
0
 void kill()
 {
     CMasterActivity::kill();
     originalIndexFile.clear();
     newIndexFile.clear();
 }
Example #26
0
 virtual ~CEspBindingEntry()
 {
     sock_.clear();
     binding_.clear();
 }
Example #27
0
    virtual void init()
    {
        CMasterActivity::init();
        OwnedRoxieString indexFileName(helper->getIndexFileName());
        Owned<IDistributedFile> dataFile;
        Owned<IDistributedFile> indexFile = queryThorFileManager().lookup(container.queryJob(), indexFileName, false, 0 != (helper->getJoinFlags() & JFindexoptional), true);

        unsigned keyReadWidth = (unsigned)container.queryJob().getWorkUnitValueInt("KJKRR", 0);
        if (!keyReadWidth || keyReadWidth>container.queryJob().querySlaves())
            keyReadWidth = container.queryJob().querySlaves();
        

        initMb.clear();
        initMb.append(indexFileName.get());
        if (helper->diskAccessRequired())
            numTags += 2;
        initMb.append(numTags);
        unsigned t=0;
        for (; t<numTags; t++)
        {
            tags[t] = container.queryJob().allocateMPTag();
            initMb.append(tags[t]);
        }
        bool keyHasTlk = false;
        if (indexFile)
        {
            unsigned numParts = 0;
            localKey = indexFile->queryAttributes().getPropBool("@local");

            if (container.queryLocalData() && !localKey)
                throw MakeActivityException(this, 0, "Keyed Join cannot be LOCAL unless supplied index is local");

            checkFormatCrc(this, indexFile, helper->getIndexFormatCrc(), true);
            Owned<IFileDescriptor> indexFileDesc = indexFile->getFileDescriptor();
            IDistributedSuperFile *superIndex = indexFile->querySuperFile();
            unsigned superIndexWidth = 0;
            unsigned numSuperIndexSubs = 0;
            if (superIndex)
            {
                numSuperIndexSubs = superIndex->numSubFiles(true);
                bool first=true;
                // consistency check
                Owned<IDistributedFileIterator> iter = superIndex->getSubFileIterator(true);
                ForEach(*iter)
                {
                    IDistributedFile &f = iter->query();
                    unsigned np = f.numParts()-1;
                    IDistributedFilePart &part = f.queryPart(np);
                    const char *kind = part.queryAttributes().queryProp("@kind");
                    bool hasTlk = NULL != kind && 0 == stricmp("topLevelKey", kind); // if last part not tlk, then deemed local (might be singlePartKey)
                    if (first)
                    {
                        first = false;
                        keyHasTlk = hasTlk;
                        superIndexWidth = f.numParts();
                        if (keyHasTlk)
                            --superIndexWidth;
                    }
                    else
                    {
                        if (hasTlk != keyHasTlk)
                            throw MakeActivityException(this, 0, "Local/Single part keys cannot be mixed with distributed(tlk) keys in keyedjoin");
                        if (keyHasTlk && superIndexWidth != f.numParts()-1)
                            throw MakeActivityException(this, 0, "Super sub keys of different width cannot be mixed with distributed(tlk) keys in keyedjoin");
                    }
                }
                if (keyHasTlk)
                    numParts = superIndexWidth * numSuperIndexSubs;
                else
                    numParts = superIndex->numParts();
            }
            else
            {
                numParts = indexFile->numParts();
                if (numParts)
                {
                    const char *kind = indexFile->queryPart(indexFile->numParts()-1).queryAttributes().queryProp("@kind");
                    keyHasTlk = NULL != kind && 0 == stricmp("topLevelKey", kind);
                    if (keyHasTlk)
                        --numParts;
                }
            }
            if (numParts)
            {
                initMb.append(numParts);
                initMb.append(superIndexWidth); // 0 if not superIndex
                initMb.append((superIndex && superIndex->isInterleaved()) ? numSuperIndexSubs : 0);
                unsigned p=0;
                UnsignedArray parts;
                for (; p<numParts; p++)
                    parts.append(p);
                indexFileDesc->serializeParts(initMb, parts);
                if (localKey)
                    keyHasTlk = false; // not used
                initMb.append(keyHasTlk);
                if (keyHasTlk)
                {
                    if (numSuperIndexSubs)
                        initMb.append(numSuperIndexSubs);
                    else
                        initMb.append((unsigned)1);

                    Owned<IDistributedFileIterator> iter;
                    IDistributedFile *f;
                    if (superIndex)
                    {
                        iter.setown(superIndex->getSubFileIterator(true));
                        f = &iter->query();
                    }
                    else
                        f = indexFile;
                    loop
                    {
                        unsigned location;
                        OwnedIFile iFile;
                        StringBuffer filePath;
                        Owned<IFileDescriptor> fileDesc = f->getFileDescriptor();
                        Owned<IPartDescriptor> tlkDesc = fileDesc->getPart(fileDesc->numParts()-1);
                        if (!getBestFilePart(this, *tlkDesc, iFile, location, filePath))
                            throw MakeThorException(TE_FileNotFound, "Top level key part does not exist, for key: %s", f->queryLogicalName());
                        OwnedIFileIO iFileIO = iFile->open(IFOread);
                        assertex(iFileIO);

                        size32_t tlkSz = (size32_t)iFileIO->size();
                        initMb.append(tlkSz);
                        ::read(iFileIO, 0, tlkSz, initMb);

                        if (!iter || !iter->next())
                            break;
                        f = &iter->query();
                    }
                }
                if (helper->diskAccessRequired())
                {
                    OwnedRoxieString fetchFilename(helper->getFileName());
                    if (fetchFilename)
                    {
                        dataFile.setown(queryThorFileManager().lookup(container.queryJob(), fetchFilename, false, 0 != (helper->getFetchFlags() & FFdatafileoptional), true));
                        if (dataFile)
                        {
                            if (superIndex)
                                throw MakeActivityException(this, 0, "Superkeys and full keyed joins are not supported");
                            Owned<IFileDescriptor> dataFileDesc = getConfiguredFileDescriptor(*dataFile);
                            void *ekey;
                            size32_t ekeylen;
                            helper->getFileEncryptKey(ekeylen,ekey);
                            bool encrypted = dataFileDesc->queryProperties().getPropBool("@encrypted");
                            if (0 != ekeylen)
                            {
                                memset(ekey,0,ekeylen);
                                free(ekey);
                                if (!encrypted)
                                {
                                    Owned<IException> e = MakeActivityWarning(&container, TE_EncryptionMismatch, "Ignoring encryption key provided as file '%s' was not published as encrypted", helper->getFileName());
                                    container.queryJob().fireException(e);
                                }
                            }
                            else if (encrypted)
                                throw MakeActivityException(this, 0, "File '%s' was published as encrypted but no encryption key provided", fetchFilename.get());
                            unsigned dataReadWidth = (unsigned)container.queryJob().getWorkUnitValueInt("KJDRR", 0);
                            if (!dataReadWidth || dataReadWidth>container.queryJob().querySlaves())
                                dataReadWidth = container.queryJob().querySlaves();
                            Owned<IGroup> grp = container.queryJob().querySlaveGroup().subset((unsigned)0, dataReadWidth);
                            dataFileMapping.setown(getFileSlaveMaps(dataFile->queryLogicalName(), *dataFileDesc, container.queryJob().queryUserDescriptor(), *grp, false, false, NULL));
                            dataFileMapping->serializeFileOffsetMap(offsetMapMb.clear());
                        }
                        else
                            indexFile.clear();
                    }
                }
            }
            else
Example #28
0
void CWriteMasterBase::publish()
{
    if (published) return;
    published = true;
    if (!(diskHelperBase->getFlags() & (TDXtemporary|TDXjobtemp)))
        updateActivityResult(container.queryJob().queryWorkUnit(), diskHelperBase->getFlags(), diskHelperBase->getSequence(), fileName, recordsProcessed);

    IPropertyTree &props = fileDesc->queryProperties();
    props.setPropInt64("@recordCount", recordsProcessed);
    if (0 == (diskHelperBase->getFlags() & TDXtemporary) || container.queryJob().queryUseCheckpoints())
    {
        if (0 != (diskHelperBase->getFlags() & TDWexpires))
            setExpiryTime(props, diskHelperBase->getExpiryDays());
        if (TDWupdate & diskHelperBase->getFlags())
        {
            unsigned eclCRC;
            unsigned __int64 totalCRC;
            diskHelperBase->getUpdateCRCs(eclCRC, totalCRC);
            props.setPropInt("@eclCRC", eclCRC);
            props.setPropInt64("@totalCRC", totalCRC);
        }
    }
    container.queryTempHandler()->registerFile(fileName, container.queryOwner().queryGraphId(), diskHelperBase->getTempUsageCount(), TDXtemporary & diskHelperBase->getFlags(), getDiskOutputKind(diskHelperBase->getFlags()), &clusters);
    if (!dlfn.isExternal())
    {
        bool temporary = 0 != (diskHelperBase->getFlags()&TDXtemporary);
        if (!temporary && (queryJob().querySlaves() < fileDesc->numParts()))
        {
            // create empty parts for a fileDesc being published that is larger than this clusters
            size32_t recordSize = 0;
            IOutputMetaData *diskRowMeta = diskHelperBase->queryDiskRecordSize()->querySerializedDiskMeta();
            if (diskRowMeta->isFixedSize() && (TAKdiskwrite == container.getKind()))
            {
                recordSize = diskRowMeta->getMinRecordSize();
                if (0 != (diskHelperBase->getFlags() & TDXgrouped))
                    recordSize += 1;
            }
            unsigned compMethod = COMPRESS_METHOD_LZW;
            // rowdiff used if recordSize > 0, else fallback to compMethod
            if (getOptBool(THOROPT_COMP_FORCELZW, false))
            {
                recordSize = 0; // by default if fixed length (recordSize set), row diff compression is used. This forces compMethod.
                compMethod = COMPRESS_METHOD_LZW;
            }
            else if (getOptBool(THOROPT_COMP_FORCEFLZ, false))
                compMethod = COMPRESS_METHOD_FASTLZ;
            else if (getOptBool(THOROPT_COMP_FORCELZ4, false))
                compMethod = COMPRESS_METHOD_LZ4;
            bool blockCompressed;
            bool compressed = fileDesc->isCompressed(&blockCompressed);
            for (unsigned clusterIdx=0; clusterIdx<fileDesc->numClusters(); clusterIdx++)
            {
                StringBuffer clusterName;
                fileDesc->getClusterGroupName(clusterIdx, clusterName, &queryNamedGroupStore());
                PROGLOG("Creating blank parts for file '%s', cluster '%s'", fileName.get(), clusterName.str());
                unsigned p=0;
                while (p<fileDesc->numParts())
                {
                    if (p == targetOffset)
                        p += queryJob().querySlaves();
                    IPartDescriptor *partDesc = fileDesc->queryPart(p);
                    CDateTime createTime, modifiedTime;
                    for (unsigned c=0; c<partDesc->numCopies(); c++)
                    {
                        RemoteFilename rfn;
                        partDesc->getFilename(c, rfn);
                        StringBuffer path;
                        rfn.getPath(path);
                        try
                        {
                            ensureDirectoryForFile(path.str());
                            OwnedIFile iFile = createIFile(path.str());
                            Owned<IFileIO> iFileIO;
                            if (compressed)
                                iFileIO.setown(createCompressedFileWriter(iFile, recordSize, false, true, NULL, compMethod));
                            else
                                iFileIO.setown(iFile->open(IFOcreate));
                            dbgassertex(iFileIO.get());
                            iFileIO.clear();
                            // ensure copies have matching datestamps, as they would do normally (backupnode expects it)
                            if (partDesc->numCopies() > 1)
                            {
                                if (0 == c)
                                    iFile->getTime(&createTime, &modifiedTime, NULL);
                                else
                                    iFile->setTime(&createTime, &modifiedTime, NULL);
                            }
                        }
                        catch (IException *e)
                        {
                            if (0 == c)
                                throw;
                            Owned<IThorException> e2 = MakeThorException(e);
                            e->Release();
                            e2->setAction(tea_warning);
                            queryJob().fireException(e2);
                        }
                    }
                    partDesc->queryProperties().setPropInt64("@size", 0);
                    p++;
                }
                clusterIdx++;
            }
        }
        queryThorFileManager().publish(container.queryJob(), fileName, *fileDesc, NULL);
    }
}
void ParseErrorHandler::resetErrors()
{
    m_errors.clear();
}
Example #30
0
//-----------------------------------------------------
//
//-----------------------------------------------------
int main(int argc, char* argv[])
{
#ifdef _NO_LDAP
    fprintf(stderr, "System was built with _NO_LDAP\n");
    return -1;
#endif

    for (int x = 1; x < argc; x++)
    {
        if (0==strncmp("-h", argv[x], 2))
        {
            usage();
            exit(0);
        }
        else
        {
            fprintf(stderr, "\nERROR: Unrecognized parameter : '%s', enter 'initldap -h' for help\n", argv[x]);
            exit(1);
        }
    }

    InitModuleObjects();

    //execute configgen to query the LDAP Server configuration(s)
    StringBuffer cmd;
    cmd.appendf("%s%cconfiggen -env %s%c%s -listldapservers", ADMIN_DIR,PATHSEPCHAR,CONFIG_DIR, PATHSEPCHAR, ENV_XML_FILE);

    char * configBuffer = NULL;

    //acquire LDAP configuration by executing configgen and capturing output
    {
        StringBuffer configBuff;
        Owned<IPipeProcess> pipe = createPipeProcess();
        if (pipe->run("configgen", cmd.str(), ".", false, true, true, 0))
        {
            Owned<ISimpleReadStream> pipeReader = pipe->getOutputStream();
            const size32_t chunkSize = 8192;
            for (;;)
            {
                size32_t sizeRead = pipeReader->read(chunkSize, configBuff.reserve(chunkSize));
                if (sizeRead < chunkSize)
                {
                    configBuff.setLength(configBuff.length() - (chunkSize - sizeRead));
                    break;
                }
            }
            pipe->closeOutput();
        }
        int retcode = pipe->wait();
        if (retcode)
        {
            fprintf(stderr, "\nERROR %d: unable to execute %s", retcode, cmd.str());
            exit(1);
        }
        configBuffer = strdup(configBuff.str());
    }

    //Using the LDAP Server parms queried from configgen, build an
    //LDAPSecurity property tree for each LDAP Server and call the LDAP
    //Security Manager to create the needed entries
    Owned<IPropertyTree> ldapProps;
    char *saveptr;
    char * pLine = strtok_r(configBuffer, "\n", &saveptr);
    while (pLine)
    {
        if (pLine && 0==strcmp(pLine, "LDAPServerProcess"))
        {
            if (ldapProps)
                initLDAP(ldapProps);
            ldapProps.clear();
            ldapProps.setown(createPTree("ldapSecurity"));
        }
        else
        {
            char * sep = strchr(pLine, ',');
            if (sep)
            {
                *sep = (char)NULL;
                ldapProps->addProp(pLine, sep+1);
            }
        }
        pLine = strtok_r(NULL, "\n", &saveptr);
    }
    if (ldapProps)
        initLDAP(ldapProps);
    if (configBuffer)
        free(configBuffer);
    ldapProps.clear();

    releaseAtoms();
    return 0;
}