void operator()() { TaskStatus status; status.mutable_task_id()->MergeFrom(task.task_id()); // Currently, just call the K3 executable with the generated command line from task.data() try { FILE* pipe = popen(k3_cmd.c_str(), "r"); if (!pipe) { status.set_state(TASK_FAILED); driver->sendStatusUpdate(status); cout << "Failed to open subprocess" << endl; } char buffer[256]; while (!feof(pipe)) { if (fgets(buffer, 256, pipe) != NULL) { std::string s = std::string(buffer); if (this->isMaster) { driver->sendFrameworkMessage(s); cout << s << endl; } else { cout << s << endl; } } } int k3 = pclose(pipe); if (k3 == 0) { status.set_state(TASK_FINISHED); cout << "Task " << task.task_id().value() << " Completed!" << endl; driver->sendStatusUpdate(status); } else { status.set_state(TASK_FAILED); cout << "K3 Task " << task.task_id().value() << " returned error code: " << k3 << endl; driver->sendStatusUpdate(status); } } catch (...) { status.set_state(TASK_FAILED); driver->sendStatusUpdate(status); } //------------- END OF TASK ------------------- }
TEST(MasterTest, FrameworkMessage) { ASSERT_TRUE(GTEST_IS_THREADSAFE); SimpleAllocator a; Master m(&a); PID<Master> master = process::spawn(&m); MockExecutor exec; ExecutorDriver* execDriver; string execData; trigger execFrameworkMessageCall, shutdownCall; EXPECT_CALL(exec, registered(_, _, _, _, _, _)) .WillOnce(SaveArg<0>(&execDriver)); EXPECT_CALL(exec, launchTask(_, _)) .WillOnce(SendStatusUpdate(TASK_RUNNING)); EXPECT_CALL(exec, frameworkMessage(_, _)) .WillOnce(DoAll(SaveArg<1>(&execData), Trigger(&execFrameworkMessageCall))); EXPECT_CALL(exec, shutdown(_)) .WillOnce(Trigger(&shutdownCall)); map<ExecutorID, Executor*> execs; execs[DEFAULT_EXECUTOR_ID] = &exec; TestingIsolationModule isolationModule(execs); Resources resources = Resources::parse("cpus:2;mem:1024"); Slave s(resources, true, &isolationModule); PID<Slave> slave = process::spawn(&s); BasicMasterDetector detector(master, slave, true); // Launch the first (i.e., failing) scheduler and wait until the // first status update message is sent to it (drop the message). MockScheduler sched; MesosSchedulerDriver schedDriver(&sched, "", DEFAULT_EXECUTOR_INFO, master); vector<Offer> offers; TaskStatus status; string schedData; trigger resourceOffersCall, statusUpdateCall, schedFrameworkMessageCall; EXPECT_CALL(sched, registered(&schedDriver, _)) .Times(1); EXPECT_CALL(sched, resourceOffers(&schedDriver, _)) .WillOnce(DoAll(SaveArg<1>(&offers), Trigger(&resourceOffersCall))) .WillRepeatedly(Return()); EXPECT_CALL(sched, statusUpdate(&schedDriver, _)) .WillOnce(DoAll(SaveArg<1>(&status), Trigger(&statusUpdateCall))); EXPECT_CALL(sched, frameworkMessage(&schedDriver, _, _, _)) .WillOnce(DoAll(SaveArg<3>(&schedData), Trigger(&schedFrameworkMessageCall))); schedDriver.start(); WAIT_UNTIL(resourceOffersCall); EXPECT_NE(0, offers.size()); TaskDescription task; task.set_name(""); task.mutable_task_id()->set_value("1"); task.mutable_slave_id()->MergeFrom(offers[0].slave_id()); task.mutable_resources()->MergeFrom(offers[0].resources()); vector<TaskDescription> tasks; tasks.push_back(task); schedDriver.launchTasks(offers[0].id(), tasks); WAIT_UNTIL(statusUpdateCall); EXPECT_EQ(TASK_RUNNING, status.state()); string hello = "hello"; schedDriver.sendFrameworkMessage(offers[0].slave_id(), DEFAULT_EXECUTOR_ID, hello); WAIT_UNTIL(execFrameworkMessageCall); EXPECT_EQ(hello, execData); string reply = "reply"; execDriver->sendFrameworkMessage(reply); WAIT_UNTIL(schedFrameworkMessageCall); EXPECT_EQ(reply, schedData); schedDriver.stop(); schedDriver.join(); WAIT_UNTIL(shutdownCall); // To ensure can deallocate MockExecutor. process::terminate(slave); process::wait(slave); process::terminate(master); process::wait(master); }
TEST_F(FaultToleranceTest, SchedulerFailoverFrameworkMessage) { Try<PID<Master> > master = StartMaster(); ASSERT_SOME(master); MockExecutor exec(DEFAULT_EXECUTOR_ID); Try<PID<Slave> > slave = StartSlave(&exec); ASSERT_SOME(slave); MockScheduler sched1; MesosSchedulerDriver driver1(&sched1, DEFAULT_FRAMEWORK_INFO, master.get()); FrameworkID frameworkId; EXPECT_CALL(sched1, registered(&driver1, _, _)) .WillOnce(SaveArg<1>(&frameworkId)); Future<vector<Offer> > offers; EXPECT_CALL(sched1, resourceOffers(&driver1, _)) .WillOnce(FutureArg<1>(&offers)) .WillRepeatedly(Return()); // Ignore subsequent offers. driver1.start(); AWAIT_READY(offers); EXPECT_NE(0u, offers.get().size()); TaskInfo task; task.set_name(""); task.mutable_task_id()->set_value("1"); task.mutable_slave_id()->MergeFrom(offers.get()[0].slave_id()); task.mutable_resources()->MergeFrom(offers.get()[0].resources()); task.mutable_executor()->MergeFrom(DEFAULT_EXECUTOR_INFO); vector<TaskInfo> tasks; tasks.push_back(task); Future<TaskStatus> status; EXPECT_CALL(sched1, statusUpdate(&driver1, _)) .WillOnce(FutureArg<1>(&status)); ExecutorDriver* execDriver; EXPECT_CALL(exec, registered(_, _, _, _)) .WillOnce(SaveArg<0>(&execDriver)); EXPECT_CALL(exec, launchTask(_, _)) .WillOnce(SendStatusUpdateFromTask(TASK_RUNNING)); driver1.launchTasks(offers.get()[0].id(), tasks); AWAIT_READY(status); EXPECT_EQ(TASK_RUNNING, status.get().state()); MockScheduler sched2; FrameworkInfo framework2; // Bug in gcc 4.1.*, must assign on next line. framework2 = DEFAULT_FRAMEWORK_INFO; framework2.mutable_id()->MergeFrom(frameworkId); MesosSchedulerDriver driver2(&sched2, framework2, master.get()); Future<Nothing> registered; EXPECT_CALL(sched2, registered(&driver2, frameworkId, _)) .WillOnce(FutureSatisfy(®istered)); Future<Nothing> frameworkMessage; EXPECT_CALL(sched2, frameworkMessage(&driver2, _, _, _)) .WillOnce(FutureSatisfy(&frameworkMessage)); EXPECT_CALL(sched1, error(&driver1, "Framework failed over")); driver2.start(); AWAIT_READY(registered); execDriver->sendFrameworkMessage("Executor to Framework message"); AWAIT_READY(frameworkMessage); EXPECT_CALL(exec, shutdown(_)) .Times(AtMost(1)); driver1.stop(); driver2.stop(); driver1.join(); driver2.join(); Shutdown(); }
TEST(FaultToleranceTest, SchedulerFailoverFrameworkMessage) { ASSERT_TRUE(GTEST_IS_THREADSAFE); SimpleAllocator a; Master m(&a); PID<Master> master = process::spawn(&m); MockExecutor exec; ExecutorDriver* execDriver; EXPECT_CALL(exec, registered(_, _, _, _)) .WillOnce(SaveArg<0>(&execDriver)); EXPECT_CALL(exec, launchTask(_, _)) .WillOnce(SendStatusUpdate(TASK_RUNNING)); EXPECT_CALL(exec, shutdown(_)) .Times(AtMost(1)); map<ExecutorID, Executor*> execs; execs[DEFAULT_EXECUTOR_ID] = &exec; TestingIsolationModule isolationModule(execs); Resources resources = Resources::parse("cpus:2;mem:1024"); Slave s(resources, true, &isolationModule); PID<Slave> slave = process::spawn(&s); BasicMasterDetector detector(master, slave, true); MockScheduler sched1; MesosSchedulerDriver driver1(&sched1, DEFAULT_FRAMEWORK_INFO, master); FrameworkID frameworkId; vector<Offer> offers; TaskStatus status; trigger sched1ResourceOfferCall, sched1StatusUpdateCall; EXPECT_CALL(sched1, registered(&driver1, _, _)) .WillOnce(SaveArg<1>(&frameworkId)); EXPECT_CALL(sched1, statusUpdate(&driver1, _)) .WillOnce(DoAll(SaveArg<1>(&status), Trigger(&sched1StatusUpdateCall))); EXPECT_CALL(sched1, resourceOffers(&driver1, _)) .WillOnce(DoAll(SaveArg<1>(&offers), Trigger(&sched1ResourceOfferCall))) .WillRepeatedly(Return()); EXPECT_CALL(sched1, error(&driver1, "Framework failed over")) .Times(1); driver1.start(); WAIT_UNTIL(sched1ResourceOfferCall); EXPECT_NE(0, offers.size()); TaskInfo task; task.set_name(""); task.mutable_task_id()->set_value("1"); task.mutable_slave_id()->MergeFrom(offers[0].slave_id()); task.mutable_resources()->MergeFrom(offers[0].resources()); task.mutable_executor()->MergeFrom(DEFAULT_EXECUTOR_INFO); vector<TaskInfo> tasks; tasks.push_back(task); driver1.launchTasks(offers[0].id(), tasks); WAIT_UNTIL(sched1StatusUpdateCall); EXPECT_EQ(TASK_RUNNING, status.state()); MockScheduler sched2; FrameworkInfo framework2; // Bug in gcc 4.1.*, must assign on next line. framework2 = DEFAULT_FRAMEWORK_INFO; framework2.mutable_id()->MergeFrom(frameworkId); MesosSchedulerDriver driver2(&sched2, framework2, master); trigger sched2RegisteredCall, sched2FrameworkMessageCall; EXPECT_CALL(sched2, registered(&driver2, frameworkId, _)) .WillOnce(Trigger(&sched2RegisteredCall)); EXPECT_CALL(sched2, frameworkMessage(&driver2, _, _, _)) .WillOnce(Trigger(&sched2FrameworkMessageCall)); driver2.start(); WAIT_UNTIL(sched2RegisteredCall); execDriver->sendFrameworkMessage(""); WAIT_UNTIL(sched2FrameworkMessageCall); driver1.stop(); driver2.stop(); driver1.join(); driver2.join(); process::terminate(slave); process::wait(slave); process::terminate(master); process::wait(master); }