// This test verifies that the environment secrets are resolved when launching a
// task.
TEST_F(EnvironmentSecretIsolatorTest, ResolveSecret)
{
  Try<Owned<cluster::Master>> master = StartMaster();
  ASSERT_SOME(master);

  mesos::internal::slave::Flags flags = CreateSlaveFlags();

  Fetcher fetcher(flags);
  Try<SecretResolver*> secretResolver = SecretResolver::create();
  EXPECT_SOME(secretResolver);

  Try<MesosContainerizer*> containerizer =
    MesosContainerizer::create(flags, false, &fetcher, secretResolver.get());
  EXPECT_SOME(containerizer);

  Owned<MasterDetector> detector = master.get()->createDetector();
  Try<Owned<cluster::Slave>> slave =
    StartSlave(detector.get(), containerizer.get());
  ASSERT_SOME(slave);

  MockScheduler sched;
  MesosSchedulerDriver driver(
      &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL);

  EXPECT_CALL(sched, registered(&driver, _, _));

  Future<std::vector<Offer>> offers;
  EXPECT_CALL(sched, resourceOffers(&driver, _))
    .WillOnce(FutureArg<1>(&offers))
    .WillRepeatedly(Return()); // Ignore subsequent offers.

  driver.start();

  AWAIT_READY(offers);
  EXPECT_FALSE(offers->empty());

  const string commandString = strings::format(
      "env; test \"$%s\" = \"%s\"",
      SECRET_ENV_NAME,
      SECRET_VALUE).get();

  CommandInfo command;
  command.set_value(commandString);

  // Request a secret.
  // TODO(kapil): Update createEnvironment() to support secrets.
  mesos::Environment::Variable *env =
    command.mutable_environment()->add_variables();
  env->set_name(SECRET_ENV_NAME);
  env->set_type(mesos::Environment::Variable::SECRET);

  mesos::Secret* secret = env->mutable_secret();
  secret->set_type(Secret::VALUE);
  secret->mutable_value()->set_data(SECRET_VALUE);

  TaskInfo task = createTask(
      offers.get()[0].slave_id(),
      Resources::parse("cpus:0.1;mem:32").get(),
      command);

  // NOTE: Successful tasks will output two status updates.
  Future<TaskStatus> statusRunning;
  Future<TaskStatus> statusFinished;
  EXPECT_CALL(sched, statusUpdate(&driver, _))
    .WillOnce(FutureArg<1>(&statusRunning))
    .WillOnce(FutureArg<1>(&statusFinished));

  driver.launchTasks(offers.get()[0].id(), {task});

  AWAIT_READY(statusRunning);
  EXPECT_EQ(TASK_RUNNING, statusRunning.get().state());
  AWAIT_READY(statusFinished);
  EXPECT_EQ(TASK_FINISHED, statusFinished.get().state());

  driver.stop();
  driver.join();
}
unsigned applyNodes(const char *grpip, ApplyMode mode, unsigned ver, bool isdali, bool quiet)
{
    SocketEndpointArray eps;
    if (isdali&&(stricmp(grpip,"all")==0)) {
        Owned<IRemoteConnection> conn = querySDS().connect("/Environment/Software", myProcessSession(), RTM_LOCK_READ, SDS_CONNECT_TIMEOUT);
        if (!conn) 
            return 0;
        IPropertyTree* root = conn->queryRoot();
        Owned<IPropertyTreeIterator> clusters= root->getElements("ThorCluster");
        unsigned ret = 0;
        if (clusters->first()) {
            do {
                IPropertyTree &cluster = clusters->query();
                ret += applyNodes(cluster.queryProp("@name"),mode,ver,true,quiet);
            } while (clusters->next());
        }
        return ret;
    }
    SocketEndpointArray result;
    StringAttrArray resultstr;
    if (!isdali||!getCluster(grpip,eps)) {
        SocketEndpoint ep(grpip);
        if (ep.isNull()) {
            ERRLOG("%s is not a group name or ip",grpip);
            return 0;
        }
        if (ep.port==0)
            ep.port = getDaliServixPort();
        eps.append(ep);
    }
    PointerIArrayOf<ISocket> sockets;
    unsigned to=10*1000;
    unsigned n=eps.ordinality();    // use approx log scale (timeout is long but only for failure situation)
    while (n>1) {
        n/=2;
        to+=10*1000;
    }
    if (!quiet&&(n>1))
        PROGLOG("Scanning %s...",grpip);
    multiConnect(eps,sockets,to);
    CriticalSection sect;
    class casyncfor: public CAsyncFor
    {
        SocketEndpointArray &eps;
        PointerIArrayOf<ISocket> &sockets;
        ApplyMode mode;
        unsigned ver;
        SocketEndpointArray &result;
        StringAttrArray &resultstr;
        CriticalSection &sect;
    public:
        casyncfor(ApplyMode _mode, unsigned _ver,SocketEndpointArray &_eps,PointerIArrayOf<ISocket> &_sockets,SocketEndpointArray &_result, StringAttrArray &_resultstr,CriticalSection &_sect) 
            : eps(_eps), sockets(_sockets), result(_result), resultstr(_resultstr), sect(_sect)
        { 
            mode = _mode;
            ver = _ver;
        }
        void Do(unsigned i)
        {
            ISocket *sock = sockets.item(i);
            StringBuffer epstr;
            SocketEndpoint ep = eps.item(i);
            ep.getUrlStr(epstr);
//          PROGLOG("T.1 %s %x",epstr.str(),(unsigned)sock);
            StringBuffer verstr;
            unsigned rver=0;
            if (sock) {
                rver = getRemoteVersion(sock, verstr);
                switch (mode) {
                case AMcheck:
                    if (rver!=0)
                        return;
                case AMver: {
                        CriticalBlock block(sect);
                        result.append(ep);
                        StringBuffer ln;
                        ln.append(rver).append(",\"").append(verstr).append('"');
                        resultstr.append(* new StringAttrItem(ln.str()));
                    }
                    return;
                case AMstopver:
                case AMcheckver: 
                case AMcheckvermajor: {
                    // compares versions up to the '-'
                        const char *rv = verstr.str();
                        const char *v = remoteServerVersionString();
                        if (mode!=AMcheckvermajor) {
                            while (*v&&(*v!='-')&&(*v==*rv)) {
                                v++;
                                rv++;
                            }
                        }
                        if ((*rv==*v)&&(rver==ver))
                            return;
                        while (*rv&&(*rv!='-'))
                            rv++;
                        verstr.setLength(rv-verstr.str());
                        if ((mode==AMcheckver)||(mode==AMcheckvermajor))
                            break;

                    }
                    // fall through
                case AMstop:
                    {
                        unsigned err = stopRemoteServer(sock);
                        if (err!=0) {
                            ERRLOG("Could not stop server on %s, %d returned",epstr.str(),err);
                            if (mode!=AMstopver)
                                return;     // even though failed to stop - still return code
                        }
                        else 
                            Sleep(1000); // let stop
                    }
                    break;
                default:
                    return;
                }
            }
            CriticalBlock block(sect);
            result.append(ep);
            if ((mode!=AMver)&&(mode!=AMcheckver)&&(mode!=AMcheckvermajor)&&(mode!=AMstopver))
                resultstr.append(* new StringAttrItem(""));
            else
                resultstr.append(* new StringAttrItem(verstr.str()));
        }
    } afor(mode,ver,eps,sockets,result,resultstr,sect);
    afor.For(eps.ordinality(), 10, false, true);
    if (result.ordinality()==0)
        return 0;
    switch (mode) {
    case AMstopver: 
    case AMcheckver: 
    case AMcheckvermajor: 
        if (!quiet) {
            StringBuffer epstr;
            ForEachItemIn(i,result) {
                result.item(i).getUrlStr(epstr.clear());
                StringAttrItem &attr = resultstr.item(i);
                if (attr.text.length()==0) 
                    ERRLOG("%s: %s not running DAFILESRV",grpip,epstr.str());
                else 
                    ERRLOG("%s: %s %s running DAFILESRV version %s",grpip,(mode==AMstopver)?"was":"is",epstr.str(),attr.text.get());
            }
            unsigned numok = eps.ordinality()-result.ordinality();
            if (mode==AMcheckvermajor)
                PROGLOG("%s: %d node%s running version %.1f of DAFILESRV",grpip,numok,(numok!=1)?"s":"",((double)FILESRV_VERSION)/10.0);
            else {
                StringBuffer vs;
                const char *v = remoteServerVersionString();
                while (*v&&(*v!='-'))
                    vs.append(*(v++));
                PROGLOG("%s: %d node%s running version %s of DAFILESRV",grpip,numok,(numok!=1)?"s":"",vs.str());
            }
        }
void sendRequest()
{
    if (!loggingManager)
    {
        printf("No logging manager.\n");
        return;
    }

    StringBuffer action, option, status;
    testData->getProp("action", action);
    testData->getProp("option", option);
    if (action.length() && strieq(action.str(), "getTransactionSeed"))
    {
        StringBuffer transactionSeed;
        loggingManager->getTransactionSeed(transactionSeed, status);
        if (transactionSeed.length())
            printf("Got transactionSeed: <%s>\n", transactionSeed.str());
    }
    else if (action.length() && strieq(action.str(), "UpdateLog"))
    {
        IPropertyTree* logContentTree = testData->queryPropTree("LogContent");
        if (!logContentTree)
        {
            printf("can't read log content.\n");
            return;
        }
        StringBuffer logContentXML;
        toXML(logContentTree, logContentXML);
        printf("log content: <%s>.\n", logContentXML.str());

        Owned<IEspContext> espContext =  createEspContext();
        const char* userName = logContentTree->queryProp("ESPContext/UserName");
        const char* sourceIP = logContentTree->queryProp("ESPContext/SourceIP");
        short servPort = logContentTree->getPropInt("ESPContext/Port");
        espContext->setUserID(userName);
        espContext->setServAddress(sourceIP, servPort);

        const char* backEndResp = logContentTree->queryProp("BackEndResponse");
        IPropertyTree* userContextTree = logContentTree->queryPropTree("MyUserContext");
        IPropertyTree* userRequestTree = logContentTree->queryPropTree("MyUserRequest");
        IPropertyTree* userRespTree = logContentTree->queryPropTree("MyUserResponseEx");
        IPropertyTree* logDatasetsTree = logContentTree->queryPropTree("LogDatasets");
        StringBuffer userContextXML, userRequestXML, userRespXML, logDatasetsXML;
        toXML(logDatasetsTree, logDatasetsXML);
        toXML(userRespTree, userRespXML);

        toXML(userContextTree, userContextXML);
        toXML(userRequestTree, userRequestXML);
        printf("userContextXML: <%s>.\n", userContextXML.str());
        printf("userRequestXML: <%s>.\n", userRequestXML.str());
        printf("userRespXML: <%s>.\n", userRespXML.str());
        printf("backEndResp: <%s>.\n", backEndResp);

        //Sleep(5000); //Waiting for loggingManager to start
        loggingManager->updateLog(option.str(), *espContext, userContextTree, userRequestTree, backEndResp, userRespXML.str(), logDatasetsXML.str(), status);
    }
    else if (action.length() && strieq(action.str(), "UpdateLog1"))
    {
        IPropertyTree* logContentTree = testData->queryPropTree("LogContent");
        if (!logContentTree)
        {
            printf("can't read log content.\n");
            return;
        }
        StringBuffer logContentXML;
        toXML(logContentTree, logContentXML);
        printf("log content: <%s>.\n", logContentXML.str());
        //Sleep(5000); //Waiting for loggingManager to start
        loggingManager->updateLog(option.str(), logContentXML.str(), status);
    }
    else
        printf("Invalid action.\n");
}
Example #4
0
 unsigned idleTime() { return server.get()?server->idleTime():0; }
TEST_P(MemoryIsolatorTest, ROOT_MemUsage)
{
  Try<Owned<cluster::Master>> master = StartMaster();
  ASSERT_SOME(master);

  slave::Flags flags = CreateSlaveFlags();
  flags.isolation = GetParam();

  Fetcher fetcher(flags);

  Try<MesosContainerizer*> _containerizer =
    MesosContainerizer::create(flags, true, &fetcher);

  ASSERT_SOME(_containerizer);

  Owned<MesosContainerizer> containerizer(_containerizer.get());

  Owned<MasterDetector> detector = master.get()->createDetector();

  Try<Owned<cluster::Slave>> slave = StartSlave(
      detector.get(),
      containerizer.get());

  ASSERT_SOME(slave);

  MockScheduler sched;
  MesosSchedulerDriver driver(
      &sched,
      DEFAULT_FRAMEWORK_INFO,
      master.get()->pid,
      DEFAULT_CREDENTIAL);

  EXPECT_CALL(sched, registered(&driver, _, _));

  Future<vector<Offer>> offers;
  EXPECT_CALL(sched, resourceOffers(&driver, _))
    .WillOnce(FutureArg<1>(&offers))
    .WillRepeatedly(Return()); // Ignore subsequent offers.

  driver.start();

  AWAIT_READY(offers);
  ASSERT_FALSE(offers->empty());

  TaskInfo task = createTask(offers.get()[0], "sleep 120");

  Future<TaskStatus> statusRunning;
  EXPECT_CALL(sched, statusUpdate(&driver, _))
    .WillOnce(FutureArg<1>(&statusRunning));

  driver.launchTasks(offers.get()[0].id(), {task});

  AWAIT_READY(statusRunning);
  EXPECT_EQ(TASK_RUNNING, statusRunning->state());

  Future<hashset<ContainerID>> containers = containerizer->containers();
  AWAIT_READY(containers);
  ASSERT_EQ(1u, containers->size());

  ContainerID containerId = *(containers->begin());

  Future<ResourceStatistics> usage = containerizer->usage(containerId);
  AWAIT_READY(usage);

  // TODO(jieyu): Consider using a program that predictably increases
  // RSS so that we can set more meaningful expectation here.
  EXPECT_LT(0u, usage->mem_rss_bytes());

  driver.stop();
  driver.join();
}
Example #6
0
int CDfuPlusHelper::despray()
{
    const char* srcname = globals->queryProp("srcname");
    if(srcname == NULL)
        throw MakeStringException(-1, "srcname not specified");
    
    const char* dstxml = globals->queryProp("dstxml");
    const char* dstip = globals->queryProp("dstip");
    const char* dstfile = globals->queryProp("dstfile");

    bool nowait = globals->getPropBool("nowait", false);

    MemoryBuffer xmlbuf;
    if(dstxml == NULL)
    {
        if(dstfile == NULL)
            throw MakeStringException(-1, "dstfile not specified");
        if(dstip == NULL) {
#ifdef DAFILESRV_LOCAL
            progress("dstip not specified - assuming spray from local machine\n");
            dstip = ".";
#else
            throw MakeStringException(-1, "dstip not specified");
#endif
        }
    }
    else
    {
        if(dstip != NULL || dstfile != NULL)
            throw MakeStringException(-1, "dstip/dstfile and dstxml can't be used at the same time");
        StringBuffer buf;
        buf.loadFile(dstxml);
        int len = buf.length();
        xmlbuf.setBuffer(len, buf.detach(), true);
    }

    if(dstxml == NULL)
        info("\nDespraying %s to host %s file %s\n", srcname, dstip, dstfile);
    else
        info("\nDespraying %s\n", srcname);

    Owned<IClientDespray> req = sprayclient->createDesprayRequest();
    req->setSourceLogicalName(srcname);
    if(dstxml == NULL)
    {
        req->setDestIP(dstip);
        req->setDestPath(dstfile);
    }
    else
        req->setDstxml(xmlbuf);

    req->setOverwrite(globals->getPropBool("overwrite", false));
    if(globals->hasProp("connect"))
        req->setMaxConnections(globals->getPropInt("connect"));
    if(globals->hasProp("throttle"))
        req->setThrottle(globals->getPropInt("throttle"));
    if(globals->hasProp("transferbuffersize"))
        req->setTransferBufferSize(globals->getPropInt("transferbuffersize"));

    if(globals->hasProp("norecover"))
        req->setNorecover(globals->getPropBool("norecover", false));
    else if(globals->hasProp("noRecover"))
        req->setNorecover(globals->getPropBool("noRecover", false));

    if(globals->hasProp("splitprefix"))
        req->setSplitprefix(globals->queryProp("splitprefix"));
    else if(globals->hasProp("splitPrefix"))
        req->setSplitprefix(globals->queryProp("splitPrefix"));

    if(globals->hasProp("wrap"))
        req->setWrap(globals->getPropBool("wrap", false));
    if(globals->hasProp("multicopy"))
        req->setMultiCopy(globals->getPropBool("multicopy", false));
    else if(globals->hasProp("multiCopy"))
        req->setMultiCopy(globals->getPropBool("multiCopy", false));

    if(globals->hasProp("compress"))
        req->setCompress(globals->getPropBool("compress", false));
    if(globals->hasProp("encrypt"))
        req->setEncrypt(globals->queryProp("encrypt"));
    if(globals->hasProp("decrypt"))
        req->setDecrypt(globals->queryProp("decrypt"));


    SocketEndpoint localep;
    StringBuffer localeps;
    if (checkLocalDaFileSvr(dstip,localep))
        dstip = localep.getUrlStr(localeps).str();
    Owned<IClientDesprayResponse> result = sprayclient->Despray(req);
    const char* wuid = result->getWuid();
    if(wuid == NULL || *wuid == '\0')
        exc(result->getExceptions(),"despraying");
    else
    {
        const char* jobname = globals->queryProp("jobname");
        if(jobname && *jobname)
            updatejobname(wuid, jobname);

        info("Submitted WUID %s\n", wuid);
        if(!nowait)
            waitToFinish(wuid);
    }

    return 0;
}
Example #7
0
int CDfuPlusHelper::copysuper()
{
    const char* srcname = globals->queryProp("srcname");
    if(srcname == NULL)
        throw MakeStringException(-1, "srcname not specified");
    const char* dstname = globals->queryProp("dstname");
    if(dstname == NULL)
        throw MakeStringException(-1, "dstname not specified");
    const char* dstcluster = globals->queryProp("dstcluster");
    if(dstcluster == NULL)
        throw MakeStringException(-1, "dstcluster not specified");
    const char* srcdali = globals->queryProp("srcdali");
    if(srcdali == NULL)
        throw MakeStringException(-1, "srcdali not specified");
    const char* dstclusterroxie = globals->queryProp("dstclusterroxie"); // not sure if this applicable
    const char* srcusername = globals->queryProp("srcusername");
    const char* srcpassword = globals->queryProp("srcpassword");

    bool nowait = globals->getPropBool("nowait", false);
    
    info("\nCopying superfile from %s to %s\n", srcname, dstname);

    Owned<IClientCopy> req = sprayclient->createCopyRequest();
    req->setSuperCopy(true);
    req->setSourceLogicalName(srcname);
    req->setDestLogicalName(dstname);
    req->setDestGroup(dstcluster);
    if(dstclusterroxie && !stricmp(dstclusterroxie, "Yes")) // not sure if this applicable
        req->setDestGroupRoxie("Yes");
    req->setSourceDali(srcdali);
    if(srcusername && *srcusername)
        req->setSrcusername(srcusername);
    if(srcpassword && *srcpassword)
        req->setSrcpassword(srcpassword);
    req->setOverwrite(globals->getPropBool("overwrite", false));
/* the foolowing are not currently supported serverside */
    req->setReplicate(globals->getPropBool("replicate", true));
    req->setReplicateOffset(globals->getPropInt("replicateoffset",1));
    if(globals->hasProp("nosplit"))
    req->setNosplit(globals->getPropBool("nosplit", false));
    if(globals->hasProp("push")) {
        if (globals->getPropBool("push"))
            req->setPush(true);
        else 
            req->setPull(true);
    }
    if(globals->hasProp("compress"))
        req->setCompress(globals->getPropBool("compress", false));
    if(globals->hasProp("encrypt"))
        req->setEncrypt(globals->queryProp("encrypt"));
    if(globals->hasProp("decrypt"))
        req->setDecrypt(globals->queryProp("decrypt"));
    if(globals->hasProp("connect"))
        req->setMaxConnections(globals->getPropInt("connect"));
    if(globals->hasProp("throttle"))
        req->setThrottle(globals->getPropInt("throttle"));
    if(globals->hasProp("transferbuffersize"))
        req->setTransferBufferSize(globals->getPropInt("transferbuffersize"));
    if(globals->hasProp("wrap"))
        req->setWrap(globals->getPropBool("wrap", false));
    if(globals->hasProp("norecover"))
        req->setNorecover(globals->getPropBool("norecover", false));
    else if(globals->hasProp("noRecover"))
        req->setNorecover(globals->getPropBool("noRecover", false));

    Owned<IClientCopyResponse> result = sprayclient->Copy(req);
    const char* ret = result->getResult();  
    if(ret == NULL || *ret == '\0')
        exc(result->getExceptions(),"copying");
    else if (stricmp(ret,"OK")==0)
        info("Superfile copy completed\n");
    else
        info("Superfile copy failed: %s\n",ret);

    return 0;
}
Example #8
0
 void kill()
 {
     CMasterActivity::kill();
     originalIndexFile.clear();
     newIndexFile.clear();
 }
Example #9
0
    void init()
    {
        helper = (IHThorKeyDiffArg *)queryHelper();

        originalIndexFile.setown(queryThorFileManager().lookup(container.queryJob(), helper->queryOriginalName()));
        newIndexFile.setown(queryThorFileManager().lookup(container.queryJob(), helper->queryUpdatedName()));
        if (originalIndexFile->numParts() != newIndexFile->numParts())
            throw MakeActivityException(this, TE_KeyDiffIndexSizeMismatch, "Index %s and %s differ in width", helper->queryOriginalName(), helper->queryUpdatedName());
        if (originalIndexFile->querySuperFile() || newIndexFile->querySuperFile())
            throw MakeActivityException(this, 0, "Diffing super files not supported");  

        width = originalIndexFile->numParts();

        originalDesc.setown(originalIndexFile->getFileDescriptor());
        newIndexDesc.setown(newIndexFile->getFileDescriptor());
        Owned<IPartDescriptor> tlkDesc = originalDesc->getPart(originalDesc->numParts()-1);
        const char *kind = tlkDesc->queryProperties().queryProp("@kind");
        local = NULL == kind || 0 != stricmp("topLevelKey", kind);

        if (!local)
            width--; // 1 part == No n distributed / Monolithic key
        if (width > container.queryJob().querySlaves())
            throw MakeActivityException(this, 0, "Unsupported: keydiff(%s, %s) - Cannot diff a key that's wider(%d) than the target cluster size(%d)", originalIndexFile->queryLogicalName(), newIndexFile->queryLogicalName(), width, container.queryJob().querySlaves());

        queryThorFileManager().noteFileRead(container.queryJob(), originalIndexFile);
        queryThorFileManager().noteFileRead(container.queryJob(), newIndexFile);

        IArrayOf<IGroup> groups;
        fillClusterArray(container.queryJob(), helper->queryOutputName(), clusters, groups);
        patchDesc.setown(queryThorFileManager().create(container.queryJob(), helper->queryOutputName(), clusters, groups, 0 != (KDPoverwrite & helper->getFlags()), 0, !local, width));
    }
// This test ensures that when explicit acknowledgements are enabled,
// acknowledgements for master-generated updates are dropped by the
// driver. We test this by creating an invalid task that uses no
// resources.
TEST_F(MesosSchedulerDriverTest, ExplicitAcknowledgementsMasterGeneratedUpdate)
{
  Try<Owned<cluster::Master>> master = StartMaster();
  ASSERT_SOME(master);

  Owned<MasterDetector> detector = master.get()->createDetector();
  Try<Owned<cluster::Slave>> slave = StartSlave(detector.get());
  ASSERT_SOME(slave);

  MockScheduler sched;
  MesosSchedulerDriver driver(
      &sched,
      DEFAULT_FRAMEWORK_INFO,
      master.get()->pid,
      false,
      DEFAULT_CREDENTIAL);

  EXPECT_CALL(sched, registered(&driver, _, _));

  Future<vector<Offer>> offers;
  EXPECT_CALL(sched, resourceOffers(&driver, _))
    .WillOnce(FutureArg<1>(&offers))
    .WillRepeatedly(Return()); // Ignore subsequent offers.

  // Ensure no status update acknowledgements are sent to the master.
  EXPECT_NO_FUTURE_CALLS(
      mesos::scheduler::Call(),
      mesos::scheduler::Call::ACKNOWLEDGE,
      _ ,
      master.get()->pid);

  driver.start();

  AWAIT_READY(offers);
  EXPECT_NE(0u, offers->size());

  // Launch a task using no resources.
  TaskInfo task;
  task.set_name("");
  task.mutable_task_id()->set_value("1");
  task.mutable_slave_id()->MergeFrom(offers.get()[0].slave_id());
  task.mutable_executor()->MergeFrom(DEFAULT_EXECUTOR_INFO);

  vector<TaskInfo> tasks;
  tasks.push_back(task);

  Future<TaskStatus> status;
  EXPECT_CALL(sched, statusUpdate(&driver, _))
    .WillOnce(FutureArg<1>(&status));

  driver.launchTasks(offers.get()[0].id(), tasks);

  AWAIT_READY(status);
  ASSERT_EQ(TASK_ERROR, status->state());
  ASSERT_EQ(TaskStatus::SOURCE_MASTER, status->source());
  ASSERT_EQ(TaskStatus::REASON_TASK_INVALID, status->reason());

  // Now send the acknowledgement.
  driver.acknowledgeStatusUpdate(status.get());

  // Settle the clock to ensure driver processes the acknowledgement,
  // which should get dropped due to having come from the master.
  Clock::pause();
  Clock::settle();

  driver.stop();
  driver.join();
}
Example #11
0
    void done()
    {
        StringBuffer scopedName;
        queryThorFileManager().addScope(container.queryJob(), helper->queryOutputName(), scopedName);
        Owned<IWorkUnit> wu = &container.queryJob().queryWorkUnit().lock();
        Owned<IWUResult> r = wu->updateResultBySequence(helper->getSequence());
        r->setResultStatus(ResultStatusCalculated);
        r->setResultLogicalName(scopedName.str());
        r.clear();
        wu.clear();

        IPropertyTree &patchProps = patchDesc->queryProperties();
        setExpiryTime(patchProps, helper->getExpiryDays());
        IPropertyTree &originalProps = originalDesc->queryProperties();;
        if (originalProps.queryProp("ECL"))
            patchProps.setProp("ECL", originalProps.queryProp("ECL"));
        if (originalProps.getPropBool("@local"))
            patchProps.setPropBool("@local", true);
        container.queryTempHandler()->registerFile(helper->queryOutputName(), container.queryOwner().queryGraphId(), 0, false, WUFileStandard, &clusters);
        Owned<IDistributedFile> patchFile;
        // set part sizes etc
        queryThorFileManager().publish(container.queryJob(), helper->queryOutputName(), false, *patchDesc, &patchFile, 0, false);
        try { // set file size
            if (patchFile) {
                __int64 fs = patchFile->getFileSize(true,false);
                if (fs!=-1)
                    patchFile->queryAttributes().setPropInt64("@size",fs);
            }
        }
        catch (IException *e) {
            EXCLOG(e,"keydiff setting file size");
            e->Release();
        }
        // Add a new 'Patch' description to the secondary key.
        DistributedFilePropertyLock lock(newIndexFile);
        IPropertyTree &fileProps = lock.queryAttributes();
        StringBuffer path("Patch[@name=\"");
        path.append(scopedName.str()).append("\"]");
        IPropertyTree *patch = fileProps.queryPropTree(path.str());
        if (!patch) patch = fileProps.addPropTree("Patch", createPTree());
        patch->setProp("@name", scopedName.str());
        unsigned checkSum;
        if (patchFile->getFileCheckSum(checkSum))
            patch->setPropInt64("@checkSum", checkSum);

        IPropertyTree *index = patch->setPropTree("Index", createPTree());
        index->setProp("@name", originalIndexFile->queryLogicalName());
        if (originalIndexFile->getFileCheckSum(checkSum))
            index->setPropInt64("@checkSum", checkSum);
    }
// Ensures that when a scheduler enables explicit acknowledgements
// on the driver, there are no implicit acknowledgements sent, and
// the call to 'acknowledgeStatusUpdate' sends the ack to the master.
TEST_F(MesosSchedulerDriverTest, ExplicitAcknowledgements)
{
  Try<Owned<cluster::Master>> master = StartMaster();
  ASSERT_SOME(master);

  MockExecutor exec(DEFAULT_EXECUTOR_ID);
  TestContainerizer containerizer(&exec);

  Owned<MasterDetector> detector = master.get()->createDetector();

  Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), &containerizer);
  ASSERT_SOME(slave);

  MockScheduler sched;
  MesosSchedulerDriver driver(
      &sched,
      DEFAULT_FRAMEWORK_INFO,
      master.get()->pid,
      false,
      DEFAULT_CREDENTIAL);

  EXPECT_CALL(sched, registered(&driver, _, _));

  EXPECT_CALL(sched, resourceOffers(&driver, _))
    .WillOnce(LaunchTasks(DEFAULT_EXECUTOR_INFO, 1, 1, 16, "*"))
    .WillRepeatedly(Return()); // Ignore subsequent offers.

  Future<TaskStatus> status;
  EXPECT_CALL(sched, statusUpdate(&driver, _))
    .WillOnce(FutureArg<1>(&status));

  // Ensure no status update acknowledgements are sent from the driver
  // to the master until the explicit acknowledgement is sent.
  EXPECT_NO_FUTURE_CALLS(
      mesos::scheduler::Call(),
      mesos::scheduler::Call::ACKNOWLEDGE,
      _ ,
      master.get()->pid);

  EXPECT_CALL(exec, registered(_, _, _, _));

  EXPECT_CALL(exec, launchTask(_, _))
    .WillOnce(SendStatusUpdateFromTask(TASK_RUNNING));

  EXPECT_CALL(exec, shutdown(_))
    .Times(AtMost(1));

  driver.start();

  AWAIT_READY(status);

  // Settle the clock to ensure driver finishes processing the status
  // update, we want to ensure that no implicit acknowledgement gets
  // sent.
  Clock::pause();
  Clock::settle();

  // Now send the acknowledgement.
  Future<mesos::scheduler::Call> acknowledgement = FUTURE_CALL(
      mesos::scheduler::Call(),
      mesos::scheduler::Call::ACKNOWLEDGE,
      _,
      master.get()->pid);

  driver.acknowledgeStatusUpdate(status.get());

  AWAIT_READY(acknowledgement);

  driver.stop();
  driver.join();
}
Example #13
0
void CDiskWriteSlaveActivityBase::open()
{
    if (dlfn.isExternal() && !firstNode())
    {
        input.setown(createDataLinkSmartBuffer(this, inputs.item(0), PROCESS_SMART_BUFFER_SIZE, isSmartBufferSpillNeeded(this), grouped, RCUNBOUND, NULL, false, &container.queryJob().queryIDiskUsage()));
        startInput(input);
        if (!rfsQueryParallel)
        {
            ActPrintLog("Blocked, waiting for previous part to complete write");
            CMessageBuffer msg;
            if (!receiveMsg(msg, container.queryJob().queryMyRank()-1, mpTag))
                return;
            rowcount_t prevRows;
            msg.read(prevRows);
            msg.read(tempExternalName); // reuse temp filename, last node will rename
            ActPrintLog("Previous write row count = %"RCPF"d", prevRows);
        }
    }
    else
    {
        input.set(inputs.item(0));
        startInput(input);
    }
    processed = THORDATALINK_STARTED;

    bool extend = 0 != (diskHelperBase->getFlags() & TDWextend);
    if (extend)
        ActPrintLog("Extending file %s", fName.get());
    size32_t exclsz = 0;
    calcFileCrc = true;

    bool external = dlfn.isExternal();
    bool query = dlfn.isQuery();
    if (query && compress)
        UNIMPLEMENTED;

    bool direct = query || (external && !firstNode());
    bool rename = !external || (!query && lastNode());
    Owned<IFileIO> iFileIO = createMultipleWrite(this, *partDesc, exclsz, compress, extend||(external&&!query), ecomp, this, direct, rename, &abortSoon, (external&&!query) ? &tempExternalName : NULL);

    if (compress)
    {
        ActPrintLog("Performing row compression on output file: %s", fName.get());
        calcFileCrc = false;
    }
    Owned<IFileIOStream> stream;
    if (wantRaw())
    {
        outraw.setown(createBufferedIOStream(iFileIO));
        stream.set(outraw);
    }
    else
    {
        stream.setown(createIOStream(iFileIO));
        out.setown(createRowWriter(stream,::queryRowSerializer(input),::queryRowAllocator(input),grouped,calcFileCrc,false)); // flushed by close
    }
    CDfsLogicalFileName dlfn;
    dlfn.set(logicalFilename);
    if (extend || (external && !query))
        stream->seek(0,IFSend);
    ActPrintLog("Created output stream for %s", fName.get());
}
Example #14
0
 foreachvalue (const Owned<Store>& store, stores) {
   recovers.push_back(store->recover());
 }
Example #15
0
bool CDfuPlusHelper::fixedSpray(const char* srcxml,const char* srcip,const char* srcfile,const MemoryBuffer &xmlbuf,const char* dstcluster,const char* dstname,const char *format, StringBuffer &retwuid, StringBuffer &except)
{
    int recordsize;
    if(stricmp(format, "recfmvb") == 0) {
        recordsize = RECFMVB_RECSIZE_ESCAPE;  // special value for recfmvb 
    }
    else if(stricmp(format, "recfmv") == 0) {
        recordsize = RECFMV_RECSIZE_ESCAPE;  // special value for recfmv
    }
    else if(stricmp(format, "variable") == 0) {
        recordsize = PREFIX_VARIABLE_RECSIZE_ESCAPE;  // special value for variable 
    }
    else if(stricmp(format, "variablebigendian") == 0) {
        recordsize = PREFIX_VARIABLE_BIGENDIAN_RECSIZE_ESCAPE;  // special value for variable bigendian
    }
    else {
        const char* rsstr = globals->queryProp("recordsize");
        recordsize = rsstr?atoi(rsstr):0;
        if(!recordsize && !globals->hasProp("nosplit"))
            throw MakeStringException(-1, "recordsize not specified for fixed");
    }

    Owned<IClientSprayFixed> req = sprayclient->createSprayFixedRequest();
    if(srcxml == NULL)
    {
        req->setSourceIP(srcip);
        req->setSourcePath(srcfile);
    }
    else
        req->setSrcxml(xmlbuf);

    if(recordsize != 0)
        req->setSourceRecordSize(recordsize);

    if(dstcluster != NULL)
        req->setDestGroup(dstcluster);
    req->setDestLogicalName(dstname);
    req->setOverwrite(globals->getPropBool("overwrite", false));
    req->setReplicate(globals->getPropBool("replicate", true));
    req->setReplicateOffset(globals->getPropInt("replicateoffset",1));
    if(globals->hasProp("prefix"))
        req->setPrefix(globals->queryProp("prefix"));
    if(globals->hasProp("nosplit"))
        req->setNosplit(globals->getPropBool("nosplit", false));
    if(globals->hasProp("wrap"))
        req->setWrap(globals->getPropBool("wrap", false));
    if(globals->hasProp("compress"))
        req->setCompress(globals->getPropBool("compress", false));
    if(globals->hasProp("encrypt"))
        req->setEncrypt(globals->queryProp("encrypt"));
    if(globals->hasProp("decrypt"))
        req->setDecrypt(globals->queryProp("decrypt"));
    if(globals->hasProp("push")) {
        if (globals->getPropBool("push"))
            req->setPush(true);
        else 
            req->setPull(true);
    }
    if(globals->hasProp("norecover"))
        req->setNorecover(globals->getPropBool("norecover", false));
    else if(globals->hasProp("noRecover"))
        req->setNorecover(globals->getPropBool("noRecover", false));
    
    if(globals->hasProp("connect"))
        req->setMaxConnections(globals->getPropInt("connect"));
    if(globals->hasProp("throttle"))
        req->setThrottle(globals->getPropInt("throttle"));
    if(globals->hasProp("transferbuffersize"))
        req->setTransferBufferSize(globals->getPropInt("transferbuffersize"));

    if(srcxml == NULL)
        info("\nFixed spraying from %s on %s to %s\n", srcfile, srcip, dstname);
    else
        info("\nSpraying to %s\n", dstname);

    Owned<IClientSprayFixedResponse> result = sprayclient->SprayFixed(req);
    const char *wuid = result->getWuid();
    if (!wuid||!*wuid) {
        result->getExceptions().errorMessage(except);
        return false;
    }
    retwuid.append(wuid);
    return true;
}
Example #16
0
 void serializeSlaveData(MemoryBuffer &dst, unsigned slave)
 {
     if (slave < width) // if false - due to mismatch width fitting - fill in with a blank entry
     {
         dst.append(true);
         Owned<IPartDescriptor> originalPartDesc = originalDesc->getPart(slave);
         originalPartDesc->serialize(dst);
         Owned<IPartDescriptor> newIndexPartDesc = newIndexDesc->getPart(slave);
         newIndexPartDesc->serialize(dst);
         patchDesc->queryPart(slave)->serialize(dst);
         
         if (0 == slave)
         {
             if (!local)
             {
                 dst.append(true);
                 Owned<IPartDescriptor> originalTlkPartDesc = originalDesc->getPart(originalDesc->numParts()-1);
                 originalTlkPartDesc->serialize(dst);                                        
                 Owned<IPartDescriptor> newIndexTlkPartDesc = newIndexDesc->getPart(newIndexDesc->numParts()-1);
                 newIndexTlkPartDesc->serialize(dst);
                 patchDesc->queryPart(patchDesc->numParts()-1)->serialize(dst);
             }
             else
                 dst.append(false);
         }
     }
     else
         dst.append(false); // no part
 }
Example #17
0
bool CDfuPlusHelper::variableSpray(const char* srcxml,const char* srcip,const char* srcfile,const MemoryBuffer &xmlbuf,const char* dstcluster,const char* dstname,const char *format,StringBuffer &retwuid, StringBuffer &except)
{
    Owned<IClientSprayVariable> req = sprayclient->createSprayVariableRequest();
    if(srcxml == NULL)
    {
        req->setSourceIP(srcip);
        req->setSourcePath(srcfile);
    }
    else
    {
        req->setSrcxml(xmlbuf);
    }

    const char* mrsstr = globals->queryProp("maxRecordSize");
    if(mrsstr != NULL)
        req->setSourceMaxRecordSize(atoi(mrsstr));
    else
        req->setSourceMaxRecordSize(8192);

    const char* encoding = globals->queryProp("encoding");
    const char* rowtag = globals->queryProp("rowtag");
    if(stricmp(format, "xml") == 0)
    {
        if(encoding == NULL)
            encoding = "utf8";
        else if(stricmp(encoding, "ascii") == 0)
            throw MakeStringException(-1, "xml format only accepts utf encodings");
        
        if(rowtag == NULL || *rowtag == '\0')
            throw MakeStringException(-1, "rowtag not specified.");
    }
    else if(stricmp(format, "csv") == 0)
    {
        if(encoding == NULL)
            encoding = "ascii";

        if(rowtag != NULL && *rowtag != '\0')
            throw MakeStringException(-1, "You can't use rowtag option with csv/delimited format");

        const char* separator = globals->queryProp("separator");
        if(separator && *separator)
            req->setSourceCsvSeparate(separator);
        const char* terminator = globals->queryProp("terminator");
        if(terminator && *terminator)
            req->setSourceCsvTerminate(terminator);
        const char* quote = globals->queryProp("quote");
        if(quote)
        {
            // Pass quote definition string from command line if defined
            // even it is empty to override default value
            req->setSourceCsvQuote(quote);
        }
    }
    else 
        encoding = format; // may need extra later

    req->setSourceFormat(CDFUfileformat::decode(encoding));
    if(rowtag != NULL)
        req->setSourceRowTag(rowtag);

    if(dstcluster != NULL)
        req->setDestGroup(dstcluster);
    req->setDestLogicalName(dstname);
    req->setOverwrite(globals->getPropBool("overwrite", false));
    req->setReplicate(globals->getPropBool("replicate", true));
    req->setReplicateOffset(globals->getPropInt("replicateoffset",1));
    if(globals->hasProp("prefix"))
        req->setPrefix(globals->queryProp("prefix"));
    if(globals->hasProp("nosplit"))
        req->setNosplit(globals->getPropBool("nosplit", false));
    if(globals->hasProp("push")) {
        if (globals->getPropBool("push"))
            req->setPush(true);
        else 
            req->setPull(true);
    }
    if(globals->hasProp("compress"))
        req->setCompress(globals->getPropBool("compress", false));
    if(globals->hasProp("encrypt"))
        req->setEncrypt(globals->queryProp("encrypt"));
    if(globals->hasProp("decrypt"))
        req->setDecrypt(globals->queryProp("decrypt"));

    if(globals->hasProp("norecover"))
        req->setNorecover(globals->getPropBool("norecover", false));
    else if(globals->hasProp("noRecover"))
        req->setNorecover(globals->getPropBool("noRecover", false));

    if(globals->hasProp("connect"))
        req->setMaxConnections(globals->getPropInt("connect"));
    if(globals->hasProp("throttle"))
        req->setThrottle(globals->getPropInt("throttle"));
    if(globals->hasProp("transferbuffersize"))
        req->setTransferBufferSize(globals->getPropInt("transferbuffersize"));

    if(globals->hasProp("failIfNoSourceFile"))
        req->setFailIfNoSourceFile(globals->getPropBool("failIfNoSourceFile",false));

    if(srcxml == NULL)
        info("\nVariable spraying from %s on %s to %s\n", srcfile, srcip, dstname);
    else
        info("\nSpraying to %s\n", dstname);
    Owned<IClientSprayResponse> result = sprayclient->SprayVariable(req);
    const char *wuid = result->getWuid();
    if (!wuid||!*wuid) {
        result->getExceptions().errorMessage(except);
        return false;
    }
    retwuid.append(wuid);
    return true;
}
Example #18
0
void RemoteDataSourceServer::doCmdNumRows(MemoryBuffer & in, MemoryBuffer & out)
{
    Owned<IFvDataSource> ds = readDataSource(in);
    __int64 numRows = ds ? ds->numRows(true) : 0;
    out.append(numRows);
}
Example #19
0
int CDfuPlusHelper::copy()
{
    const char* srcname = globals->queryProp("srcname");
    if(srcname == NULL)
        throw MakeStringException(-1, "srcname not specified");
    const char* dstname = globals->queryProp("dstname");
    if(dstname == NULL)
        throw MakeStringException(-1, "dstname not specified");
    const char* dstcluster = globals->queryProp("dstcluster");
    const char* dstclusterroxie = globals->queryProp("dstclusterroxie");
    const char* srcdali = globals->queryProp("srcdali");
    const char* srcusername = globals->queryProp("srcusername");
    const char* srcpassword = globals->queryProp("srcpassword");
    const char* diffkeydst = globals->queryProp("diffkeysrc");
    const char* diffkeysrc = globals->queryProp("diffkeydst");

    bool nowait = globals->getPropBool("nowait", false);
    
    info("\nCopying from %s to %s\n", srcname, dstname);

    Owned<IClientCopy> req = sprayclient->createCopyRequest();
    req->setSourceLogicalName(srcname);
    req->setDestLogicalName(dstname);
    if(dstcluster != NULL)
        req->setDestGroup(dstcluster);
    if(dstclusterroxie && !stricmp(dstclusterroxie, "Yes"))
        req->setDestGroupRoxie("Yes");
    if(srcdali !=  NULL)
        req->setSourceDali(srcdali);
    if(diffkeysrc !=  NULL)
        req->setSourceDiffKeyName(diffkeysrc);
    if(diffkeydst !=  NULL)
        req->setDestDiffKeyName(diffkeydst);
    if(srcusername && *srcusername)
        req->setSrcusername(srcusername);
    if(srcpassword && *srcpassword)
        req->setSrcpassword(srcpassword);
    req->setOverwrite(globals->getPropBool("overwrite", false));
    req->setReplicate(globals->getPropBool("replicate", true));
    req->setReplicateOffset(globals->getPropInt("replicateoffset",1));
    if(globals->hasProp("nosplit"))
        req->setNosplit(globals->getPropBool("nosplit", false));
    if(globals->hasProp("push")) {
        if (globals->getPropBool("push"))
            req->setPush(true);
        else 
            req->setPull(true);
    }
    if(globals->hasProp("compress"))
        req->setCompress(globals->getPropBool("compress", false));
    if(globals->hasProp("encrypt"))
        req->setEncrypt(globals->queryProp("encrypt"));
    if(globals->hasProp("decrypt"))
        req->setDecrypt(globals->queryProp("decrypt"));
    if(globals->hasProp("connect"))
        req->setMaxConnections(globals->getPropInt("connect"));
    if(globals->hasProp("throttle"))
        req->setThrottle(globals->getPropInt("throttle"));
    if(globals->hasProp("transferbuffersize"))
        req->setTransferBufferSize(globals->getPropInt("transferbuffersize"));
    if(globals->hasProp("wrap"))
        req->setWrap(globals->getPropBool("wrap", false));
    if(globals->hasProp("multicopy"))
        req->setMulticopy(globals->getPropBool("multicopy", false));
    else if(globals->hasProp("multiCopy"))
        req->setMulticopy(globals->getPropBool("multiCopy", false));

    if(globals->hasProp("norecover"))
        req->setNorecover(globals->getPropBool("norecover", false));
    else if(globals->hasProp("noRecover"))
        req->setNorecover(globals->getPropBool("noRecover", false));

    Owned<IClientCopyResponse> result = sprayclient->Copy(req);
    const char* wuid = result->getResult();
    if(wuid == NULL || *wuid == '\0')
        exc(result->getExceptions(),"copying");
    else
    {
        const char* jobname = globals->queryProp("jobname");
        if(jobname && *jobname)
            updatejobname(wuid, jobname);

        info("Submitted WUID %s\n", wuid);
        if(!nowait)
            waitToFinish(wuid);
    }

    return 0;
}
void CWriteMasterBase::init()
{
    published = false;
    recordsProcessed = 0;
    OwnedRoxieString fname(diskHelperBase->getFileName());
    dlfn.set(fname);
    if (diskHelperBase->getFlags() & TDWextend)
    {
        assertex(0 == (diskHelperBase->getFlags() & (TDXtemporary|TDXjobtemp)));
        Owned<IDistributedFile> file = queryThorFileManager().lookup(container.queryJob(), fname, false, true);
        if (file.get())
        {
            fileDesc.setown(file->getFileDescriptor());
            queryThorFileManager().noteFileRead(container.queryJob(), file, true);
        }
    }
    if (dlfn.isExternal())
        mpTag = container.queryJob().allocateMPTag(); // used 
    if (NULL == fileDesc.get())
    {
        bool overwriteok = 0!=(TDWoverwrite & diskHelperBase->getFlags());
        
        unsigned idx=0;
        while (true)
        {
            OwnedRoxieString cluster(diskHelperBase->getCluster(idx));
            if(!cluster)
                break;
            clusters.append(cluster);
            idx++;
        }
        IArrayOf<IGroup> groups;
        fillClusterArray(container.queryJob(), fname, clusters, groups);
        fileDesc.setown(queryThorFileManager().create(container.queryJob(), fname, clusters, groups, overwriteok, diskHelperBase->getFlags()));
        if (1 == groups.ordinality())
            targetOffset = getGroupOffset(groups.item(0), container.queryJob().querySlaveGroup());
        IPropertyTree &props = fileDesc->queryProperties();
        if (diskHelperBase->getFlags() & (TDWowned|TDXjobtemp|TDXtemporary))
            props.setPropBool("@owned", true);
        if (diskHelperBase->getFlags() & TDWresult)
            props.setPropBool("@result", true);
        const char *rececl= diskHelperBase->queryRecordECL();
        if (rececl&&*rececl)
            props.setProp("ECL", rececl);
        bool blockCompressed=false;
        void *ekey;
        size32_t ekeylen;
        diskHelperBase->getEncryptKey(ekeylen,ekey);
        if (ekeylen)
        {
            memset(ekey,0,ekeylen);
            free(ekey);
            props.setPropBool("@encrypted", true);      
            blockCompressed = true;
        }
        else if (0 != (diskHelperBase->getFlags() & TDWnewcompress) || 0 != (diskHelperBase->getFlags() & TDXcompress))
            blockCompressed = true;
        if (blockCompressed)
            props.setPropBool("@blockCompressed", true);
        if (TAKdiskwrite == container.getKind() && (0 != (diskHelperBase->getFlags() & TDXtemporary)) && container.queryOwner().queryOwner() && (!container.queryOwner().isGlobal())) // I am in a child query
        { // do early, because this will be local act. and will not come back to master until end of owning graph.
            publish();
        }
    }
}
Example #21
0
 bool ok() { return exc.get()==NULL; }
Example #22
0
static bool physicalPartCopy(IFile *from,const char *tofile, Owned<IException> &exc, StringBuffer *tmpname)
{
    StringBuffer tmpnamestr;
    if (!tmpname)
        tmpname = &tmpnamestr;
    tmpname->append(tofile).append("__");
    size32_t l = tmpname->length();
    genUUID(*tmpname,true); // true for windows
    StringAttr uuid(tmpname->str()+l);
    tmpname->append(".tmp");
    RemoteFilename tmpfn;
    tmpfn.setRemotePath(tmpname->str());
    //unsigned lastpc;
#ifdef LOG_PART_COPY
    PROGLOG("start physicalPartCopy(%s,%s)",from->queryFilename(),tmpname->str());
#endif
    try {
        recursiveCreateDirectoryForFile(tmpname->str());
        while(!asyncCopyFileSection(
                uuid,
                from,
                tmpfn,
                (offset_t)-1, // creates file
                0,
                (offset_t)-1, // all file
                NULL,
                PHYSICAL_COPY_POLL_TIME)) {
            // Abort check TBD
        }
    }
    catch (IException *e) {
        EXCLOG(e,"SingleFileCopy: File copy error");
        if (exc)
            exc.setown(e);
        else
            e->Release();
    }
    Owned<IFile> f = createIFile(tmpfn);
    if (!exc.get()&&(tmpnamestr.length()!=0)) {
        try {
#ifdef LOG_PART_COPY
            PROGLOG("physicalPartCopy rename(%s,%s)",tmpname->str(),pathTail(tofile));
#endif
            f->rename(pathTail(tofile));
        }
        catch (IException *e) {
            EXCLOG(e,"SingleFileCopy: File rename error");
            if (exc)
                exc.setown(e);
            else
                e->Release();
        }
    }
    if (exc.get()) {
        try {
            f->remove();
        }
        catch (IException *e) {
            // ignore
            e->Release();
        }
    }
#ifdef LOG_PART_COPY
    PROGLOG("done physicalPartCopy %s",(exc.get()==NULL)?"OK":"Failed");
#endif
    return exc.get()==NULL;
}
Example #23
0
int CDfuPlusHelper::remove()
{
    StringArray files;
    const char* name = globals->queryProp("name");
    const char* names = globals->queryProp("names");
    const char* namelist = globals->queryProp("namelist");
    if(name && *name)
    {
        files.append(name);
        info("\nRemoving %s\n", name);
    }
    else if(names && *names)
    {
        info("\nRemoving %s\n", names);

        const char* ptr = names;
        while(*ptr != '\0')
        {
            StringBuffer onesub;
            while(*ptr != '\0' && *ptr != ',')
            {
                onesub.append((char)(*ptr));
                ptr++;
            }
            if(onesub.length() > 0)
                files.append(onesub.str());
            if(*ptr != '\0')
                ptr++;
        }
    }
    else if(namelist && *namelist)
    {
        FILE* f = fopen(namelist, "r");
        if (f)
        {
            char buffer[1024*1];
            while(fgets(buffer,sizeof(buffer)-1,f))
            {
                if(strlen(buffer) == 0)
                    break;

                info("\nRemoving %s\n", buffer);

                const char* ptr = buffer;
                while(*ptr != '\0')
                {
                    StringBuffer onesub;
                    while(*ptr != '\0' && *ptr != ',')
                    {
                        onesub.append((char)(*ptr));
                        ptr++;
                    }
                    if(onesub.length() > 0)
                        files.append(onesub.str());
                    if(*ptr != '\0')
                        ptr++;
                }
            }

            fclose(f);
        }
    }

    if(files.length() < 1)
        throw MakeStringException(-1, "file name not specified");
    
    Owned<IClientDFUArrayActionRequest> req = dfuclient->createDFUArrayActionRequest();
    req->setType("Delete");
    req->setLogicalFiles(files);

    Owned<IClientDFUArrayActionResponse> resp = dfuclient->DFUArrayAction(req);

    const IMultiException* excep = &resp->getExceptions();
    if(excep != NULL && excep->ordinality() > 0)
    {
        StringBuffer errmsg;
        excep->errorMessage(errmsg);
        if(errmsg.length() > 0)
            error("%s\n", errmsg.str());
        return -1;
    }

    const char* result = resp->getDFUArrayActionResult();
    StringBuffer resultbuf;
    if(result != NULL && *result != '\0')
    {
        if(*result != '<')
            resultbuf.append(result);
        else
        {
            bool intag = false;
            int i = 0;
            char c;
            while((c = result[i]) != '\0')
            {
                if(c == '<')
                    intag = true;
                else if(c == '>')
                    intag = false;
                else if(!intag)
                    resultbuf.append(c);
                i++;
            }
        }
    }
    info("%s\n", resultbuf.str());

    return 0;
}
Example #24
0
int CDfuPlusHelper::superfile(const char* action)
{
    const char* superfile = globals->queryProp("superfile");
    if(superfile == NULL || *superfile == '\0')
        throw MakeStringException(-1, "superfile name is not specified");
    
    if(stricmp(action, "add") == 0 || stricmp(action, "remove") == 0)
    {
        const char* before = globals->queryProp("before");
        StringArray subfiles;
        const char* sfprop = globals->queryProp("subfiles");
        if(sfprop != NULL && *sfprop != '\0')
        {
            const char* ptr = sfprop;
            while(*ptr != '\0')
            {
                StringBuffer onesub;
                while(*ptr != '\0' && *ptr != ',')
                {
                    onesub.append((char)(*ptr));
                    ptr++;
                }
                if(onesub.length() > 0)
                    subfiles.append(onesub.str());
                if(*ptr != '\0')
                    ptr++;
            }
        }

        Owned<IClientSuperfileActionRequest> req = dfuclient->createSuperfileActionRequest();
        req->setAction(action);
        req->setSuperfile(superfile);
        req->setSubfiles(subfiles);
        req->setBefore(before);
        req->setDelete(globals->getPropBool("delete", false));

        Owned<IClientSuperfileActionResponse> resp = dfuclient->SuperfileAction(req);

        const IMultiException* excep = &resp->getExceptions();
        if(excep != NULL && excep->ordinality() > 0)
        {
            StringBuffer errmsg;
            excep->errorMessage(errmsg);
            info("%s\n", errmsg.str());
            return -1;
        }

        if(stricmp(action, "add") == 0)
        {
            info("Addsuper successfully finished");
        }
        else if(stricmp(action, "remove") == 0)
        {
            info("Removesuper successfully finished");
        }
    }
    else if(stricmp(action, "list") == 0)
    {
        Owned<IClientSuperfileListRequest> req = dfuclient->createSuperfileListRequest();
        req->setSuperfile(superfile);
        Owned<IClientSuperfileListResponse> resp = dfuclient->SuperfileList(req);
        
        const IMultiException* excep = &resp->getExceptions();
        if(excep != NULL && excep->ordinality() > 0)
        {
            StringBuffer errmsg;
            excep->errorMessage(errmsg);
            error("%s\n", errmsg.str());
            return -1;
        }
        StringArray& result = resp->getSubfiles();
        info("%s has %d subfiles:\n", superfile, result.length());
        for(unsigned i = 0; i < result.length(); i++)
        {
            info("\t%s\n", result.item(i));
        }
    }

    return 0;
}
Example #25
0
void testMultiCluster()
{
    Owned<IGroup> grp1 = createIGroup("192.168.51.1-5");
    Owned<IGroup> grp2 = createIGroup("192.168.16.1-5");
    Owned<IGroup> grp3 = createIGroup("192.168.53.1-5");
    queryNamedGroupStore().add("testgrp1",grp1);
    queryNamedGroupStore().add("testgrp2",grp2);
    queryNamedGroupStore().add("testgrp3",grp3);

    Owned<IFileDescriptor> fdesc = createFileDescriptor();
    fdesc->setDefaultDir("/c$/thordata/test");
    fdesc->setPartMask("testfile1._$P$_of_$N$");
    fdesc->setNumParts(5);
    ClusterPartDiskMapSpec mapping;
    fdesc->addCluster(grp1,mapping);
    fdesc->addCluster(grp2,mapping);
    fdesc->addCluster(grp3,mapping);
    queryDistributedFileDirectory().removeEntry("test::testfile1",UNKNOWN_USER);
    Owned<IDistributedFile> file = queryDistributedFileDirectory().createNew(fdesc);
    queryDistributedFileDirectory().removeEntry("test::testfile1",UNKNOWN_USER);
    file->attach("test::testfile1",UNKNOWN_USER);
    StringBuffer name;
    unsigned i;
    for (i=0;i<file->numClusters();i++)
        PROGLOG("cluster[%d] = %s",i,file->getClusterName(i,name.clear()).str());
    file.clear();
    file.setown(queryDistributedFileDirectory().lookup("test::testfile1",UNKNOWN_USER));
    for (i=0;i<file->numClusters();i++)
        PROGLOG("cluster[%d] = %s",i,file->getClusterName(i,name.clear()).str());
    file.clear();
    file.setown(queryDistributedFileDirectory().lookup("test::testfile1@testgrp1",UNKNOWN_USER));
    for (i=0;i<file->numClusters();i++)
        PROGLOG("cluster[%d] = %s",i,file->getClusterName(i,name.clear()).str());
    file.clear();
    queryDistributedFileDirectory().removePhysical("test::testfile1@testgrp2",UNKNOWN_USER);
    file.setown(queryDistributedFileDirectory().lookup("test::testfile1",UNKNOWN_USER));
    for (i=0;i<file->numClusters();i++)
        PROGLOG("cluster[%d] = %s",i,file->getClusterName(i,name.clear()).str());
}
Example #26
0
int CDfuPlusHelper::add()
{
    const char* lfn = globals->queryProp("dstname");
    if(lfn == NULL || *lfn == '\0')
        throw MakeStringException(-1, "dstname not specified");
    
    bool isRemote = false;
    const char* xmlfname = globals->queryProp("srcxml");
    const char* srcname = globals->queryProp("srcname");
    const char* srcdali = globals->queryProp("srcdali");
    const char* srcusername = globals->queryProp("srcusername");
    const char* srcpassword = globals->queryProp("srcpassword");
    if(xmlfname == NULL || *xmlfname == '\0')
    {
        if(srcname == NULL || *srcname == '\0')
            throw MakeStringException(-1, "Please specify srcxml for adding from xml, or srcname for adding from remote dali");
        else
        {
            isRemote = true;
            if(srcdali == NULL || *srcdali == '\0')
                throw MakeStringException(-1, "srcdali not specified for adding remote");
        }
    }

    if(!isRemote)
    {
        MemoryBuffer xmlbuf;
        StringBuffer buf;
        buf.loadFile(xmlfname);
        int len = buf.length();
        xmlbuf.setBuffer(len, buf.detach(), true);

        Owned<IClientAddRequest> req = dfuclient->createAddRequest();
        req->setDstname(lfn);
        req->setXmlmap(xmlbuf);
        
        Owned<IClientAddResponse> resp = dfuclient->Add(req);
        
        const IMultiException* excep = &resp->getExceptions();
        if(excep != NULL && excep->ordinality() > 0)
        {
            StringBuffer errmsg;
            excep->errorMessage(errmsg);
            error("%s\n", errmsg.str());
            return -1;
        }
    }
    else
    {
        Owned<IClientAddRemoteRequest> req = dfuclient->createAddRemoteRequest();
        req->setDstname(lfn);
        req->setSrcname(srcname);
        req->setSrcdali(srcdali);
        if(srcusername != NULL)
            req->setSrcusername(srcusername);
        if(srcpassword != NULL)
            req->setSrcpassword(srcpassword);

        Owned<IClientAddRemoteResponse> resp = dfuclient->AddRemote(req);
        
        const IMultiException* excep = &resp->getExceptions();
        if(excep != NULL && excep->ordinality() > 0)
        {
            StringBuffer errmsg;
            excep->errorMessage(errmsg);
            error("%s\n", errmsg.str());
            return -1;
        }
    }

    info("%s successfully added", lfn);
    return 0;
}
Example #27
0
    void processMessage(CMessageBuffer &mb)
    {
        ICoven &coven=queryCoven();
        MemoryBuffer params;
        params.swapWith(mb);
        int fn;
        params.read(fn);
        switch (fn) {
        case MDR_GET_VALUE: {
                StringAttr id;
                StringBuffer buf;
                params.read(id);
                if (0 == stricmp(id,"threads")) {
                    mb.append(getThreadList(buf).str());
                }
                else if (0 == stricmp(id, "mpqueue")) {
                    mb.append(getReceiveQueueDetails(buf).str());
                }
                else if (0 == stricmp(id, "locks")) { // Legacy - newer diag clients should use querySDS().getLocks() directly
                    Owned<ILockInfoCollection> lockInfoCollection = querySDS().getLocks();
                    mb.append(lockInfoCollection->toString(buf).str());
                }
                else if (0 == stricmp(id, "sdsstats")) { // Legacy - newer diag clients should use querySDS().getUsageStats() directly
                    mb.append(querySDS().getUsageStats(buf).str());
                }
                else if (0 == stricmp(id, "connections")) { // Legacy - newer diag clients should use querySDS().getConnections() directly
                    mb.append(querySDS().getConnections(buf).str());
                }
                else if (0 == stricmp(id, "sdssubscribers")) { // Legacy - newer diag clients should use querySDS().getSubscribers() directly
                    mb.append(querySDS().getSubscribers(buf).str());
                }
                else if (0 == stricmp(id, "clients")) {
                    mb.append(querySessionManager().getClientProcessList(buf).str());
                }
                else if (0 == stricmp(id, "subscriptions")) {
                    mb.append(getSubscriptionList(buf).str());
                }
                else if (0 == stricmp(id, "mpverify")) {
                    queryWorldCommunicator().verifyAll(buf);
                    mb.append(buf.str());
                }
                else if (0 == stricmp(id, "extconsistency")) {
                    mb.append(querySDS().getExternalReport(buf).str());
                }
                else if (0 == stricmp(id, "build")) {
                    mb.append("$Id: dadiags.cpp 62376 2011-02-04 21:59:58Z sort $");
                }
                else if (0 == stricmp(id, "sdsfetch")) {
                    StringAttr branchpath;
                    params.read(branchpath);
                    Linked<IPropertyTree> sroot = querySDSServer().lockStoreRead();
                    try { sroot->queryPropTree(branchpath)->serialize(mb); }
                    catch (...) { querySDSServer().unlockStoreRead(); throw; }
                    querySDSServer().unlockStoreRead();
                }
                else if (0 == stricmp(id, "perf")) {
                    getSystemTraceInfo(buf,PerfMonStandard);
                    mb.append(buf.str());
                }
                else if (0 == stricmp(id, "sdssize")) {
                    StringAttr branchpath;
                    params.read(branchpath);
                    Linked<IPropertyTree> sroot = querySDSServer().lockStoreRead();
                    StringBuffer sbuf;
                    try { 
                        toXML(sroot->queryPropTree(branchpath),sbuf); 
                        DBGLOG("sdssize '%s' = %d",branchpath.get(),sbuf.length());
                    }
                    catch (...) { 
                        querySDSServer().unlockStoreRead(); 
                        throw; 
                    }
                    querySDSServer().unlockStoreRead();
                    mb.append(sbuf.length());
                }
                else if (0 == stricmp(id, "disconnect")) {
                    StringAttr client;
                    params.read(client);
                    SocketEndpoint ep(client);
                    PROGLOG("Dalidiag request to close client connection: %s", client.get());
                    Owned<INode> node = createINode(ep);
                    queryCoven().disconnect(node);
                }
                else if (0 == stricmp(id, "unlock")) {
                    __int64 connectionId;
                    bool disconnect;
                    params.read(connectionId);
                    params.read(disconnect);
                    PROGLOG("Dalidiag request to unlock connection id: %" I64F "x", connectionId);
                    StringBuffer connectionInfo;
                    bool success = querySDSServer().unlock(connectionId, disconnect, connectionInfo);
                    mb.append(success);
                    if (success)
                        mb.append(connectionInfo);
                }
                else if (0 == stricmp(id, "save")) {
                    PROGLOG("Dalidiag requests SDS save");
                    querySDSServer().saveRequest();
                }
                else if (0 == stricmp(id, "settracetransactions")) {
                    PROGLOG("Dalidiag requests Trace Transactions");
                    if(traceAllTransactions(true))
                        mb.append("OK - no change");
                    else
                        mb.append("OK - transaction tracing enabled");
                }
                else if (0 == stricmp(id, "cleartracetransactions")) {
                    PROGLOG("Dalidiag requests Trace Transactions stopped");
                    if(traceAllTransactions(false))
                        mb.append("OK - transaction tracing disabled");
                    else
                        mb.append("OK - no change");
                }
                else if (0 == stricmp(id, "setldapflags")) {
                    unsigned f;
                    params.read(f);
                    PROGLOG("Dalidiag requests setldapflags %d",f);
                    querySessionManager().setLDAPflags(f);

                }
                else if (0 == stricmp(id, "getldapflags")) {
                    unsigned f=querySessionManager().getLDAPflags();;
                    mb.append(f);
                }
                else if (0 == stricmp(id, "setsdsdebug")) {
                    PROGLOG("Dalidiag setsdsdebug");
                    unsigned p;
                    params.read(p);
                    StringArray arr;
                    while (p--)
                    {
                        StringAttr s;
                        params.read(s);
                        arr.append(s);
                    }
                    StringBuffer reply;
                    bool success = querySDSServer().setSDSDebug(arr, reply);
                    mb.append(success).append(reply);
                }
                else
                    mb.append(StringBuffer("UNKNOWN OPTION: ").append(id).str());
            }
            break;
        }
        coven.reply(mb);            
    }   
Example #28
0
int CDfuPlusHelper::waitToFinish(const char* wuid)
{
    if(wuid == NULL || *wuid == '\0')
        return 0;

    Owned<IClientGetDFUWorkunit> req = sprayclient->createGetDFUWorkunitRequest();
    req->setWuid(wuid);

    int wait_cycle = 10;
    while(true)
    {
        Owned<IClientGetDFUWorkunitResponse> resp = sprayclient->GetDFUWorkunit(req);
        const IMultiException* excep = &resp->getExceptions();
        if(excep != NULL &&  excep->ordinality() > 0)
        {
            StringBuffer errmsg;
            excep->errorMessage(errmsg);
            error("%s\n", errmsg.str());
            break;
        }

        IConstDFUWorkunit & dfuwu = resp->getResult();

        
        switch(dfuwu.getState())
        {
            case DFUstate_unknown:
                info("%s status: unknown\n", wuid);
                break;
            case DFUstate_scheduled:
                info("%s status: scheduled\n", wuid);
                break;
            case DFUstate_queued:
                info("%s status: queued\n", wuid);
                break;
            case DFUstate_started:
                info("%s\n", dfuwu.getProgressMessage());
                break;

            case DFUstate_aborting:
                info("%s aborting\n", wuid);
                break;

            case DFUstate_aborted:
                info("%s aborted\n", wuid);
                return -1;

            case DFUstate_failed:
                {
                    const char* errmsg = dfuwu.getSummaryMessage();
                    if(errmsg && *errmsg)
                        info("%s\n", errmsg);
                    else
                        info("%s failed.\n", wuid);
                    return -1;
                }
            case DFUstate_finished:
                info("%s Finished\n", wuid);
                info("%s\n", dfuwu.getSummaryMessage());
                return 0;
        }

        sleep(wait_cycle);
        // make it wait shorter time at the beginning to avoid waiting too long for short jobs.
        if(wait_cycle < WAIT_SECONDS)
            wait_cycle = (wait_cycle+10<WAIT_SECONDS)?(wait_cycle+10):WAIT_SECONDS;
    }

    return 0;

}
IPropertyTree* CFileSpraySoapBindingEx::createPTreeForXslt(double clientVersion, const char* method, const char* dfuwuid)
{
    Owned<IEnvironmentFactory> factory = getEnvironmentFactory(true);
    Owned<IConstEnvironment> constEnv = factory->openEnvironment();
    Owned<IPropertyTree> pEnvRoot = &constEnv->getPTree();
    IPropertyTree* pEnvSoftware = pEnvRoot->queryPropTree("Software");

    Owned<IPropertyTree> pRoot = createPTreeFromXMLString("<Environment/>");
    IPropertyTree* pSoftware = pRoot->addPropTree("Software", createPTree("Software"));

    if (pEnvSoftware)
    {
        StringBuffer dfuwuidSourcePartIP, wuxml;
        if(dfuwuid && *dfuwuid)
        {
            Owned<IDFUWorkUnitFactory> dfuwu_factory = getDFUWorkUnitFactory();
            Owned<IConstDFUWorkUnit> dfuwu = dfuwu_factory->openWorkUnit(dfuwuid, false);
            if(dfuwu)
            {   
                dfuwu->toXML(wuxml);

                Owned<IPropertyTree> wu = createPTreeFromXMLString(wuxml.str());
                if (wu)
                {
                    const char* ip = wu->queryProp("Source/Part/@node");
                    if (ip && *ip)
                        dfuwuidSourcePartIP.append(ip);
                }
            }
        }

        appendDropZones(clientVersion, constEnv, dfuwuidSourcePartIP.str(), pSoftware);

        //For Spray files on Thor Cluster, fetch all the group names for all the thor instances (and dedup them)
        BoolHash uniqueThorClusterGroupNames;
        Owned<IPropertyTreeIterator> it =pEnvSoftware->getElements("ThorCluster");
        ForEach(*it)
        {
            StringBuffer thorClusterGroupName;
            IPropertyTree& cluster = it->query();
            getClusterGroupName(cluster, thorClusterGroupName);
            if (!thorClusterGroupName.length())
                continue;
            bool* found = uniqueThorClusterGroupNames.getValue(thorClusterGroupName.str());
            if (found && *found)
                continue;

            uniqueThorClusterGroupNames.setValue(thorClusterGroupName.str(), true);
            IPropertyTree* newClusterTree = pSoftware->addPropTree("ThorCluster", &it->get());
            newClusterTree->setProp("@name", thorClusterGroupName.str()); //set group name into @name for spray target
        }

        it.setown(pEnvSoftware->getElements("EclAgentProcess"));
        ForEach(*it)
        {
            IPropertyTree &cluster = it->query();
            const char* name = cluster.queryProp("@name");
            if (!name||!*name)
                continue;
            
            unsigned ins = 0;
            Owned<IPropertyTreeIterator> insts = cluster.getElements("Instance");
            ForEach(*insts) 
            {
                const char *na = insts->query().queryProp("@netAddress");
                if (!na || !*na) 
                {
                    insts->query().setProp("@gname", name);
                    continue;
                }
                
                SocketEndpoint ep(na);
                if (ep.isNull())
                    continue;

                ins++;
                StringBuffer gname("hthor__");
                //StringBuffer gname;
                gname.append(name);
                if (ins>1)
                    gname.append('_').append(ins);

                insts->query().setProp("@gname", gname.str());

            }

            pSoftware->addPropTree("EclAgentProcess", &it->get());
        }

        if (stricmp(method, "CopyInput") == 0) //Limit for this method only
        {
            it.setown(pEnvSoftware->getElements("RoxieCluster"));
            ForEach(*it)
                pSoftware->addPropTree("RoxieCluster", &it->get());
        }

        if (wuxml.length() > 0)
            pSoftware->addPropTree("DfuWorkunit", createPTreeFromXMLString(wuxml.str()));
    }
    return pRoot.getClear();
}
 IEspRpcBinding *queryBinding()
 {
     return binding_.get();
 }