void operator()() {
TaskStatus status;
status.mutable_task_id()->MergeFrom(task.task_id());
// Currently, just call the K3 executable with the generated command line from task.data()
try {
FILE* pipe = popen(k3_cmd.c_str(), "r");
if (!pipe) {
status.set_state(TASK_FAILED);
driver->sendStatusUpdate(status);
cout << "Failed to open subprocess" << endl;
}
char buffer[256];
while (!feof(pipe)) {
if (fgets(buffer, 256, pipe) != NULL) {
std::string s = std::string(buffer);
if (this->isMaster) {
driver->sendFrameworkMessage(s);
cout << s << endl;
}
else {
cout << s << endl;
}
}
}
int k3 = pclose(pipe);
if (k3 == 0) {
status.set_state(TASK_FINISHED);
cout << "Task " << task.task_id().value() << " Completed!" << endl;
driver->sendStatusUpdate(status);
}
else {
status.set_state(TASK_FAILED);
cout << "K3 Task " << task.task_id().value() << " returned error code: " << k3 << endl;
driver->sendStatusUpdate(status);
}
}
catch (...) {
status.set_state(TASK_FAILED);
driver->sendStatusUpdate(status);
}
//------------- END OF TASK -------------------
}
TEST(MasterTest, FrameworkMessage)
{
ASSERT_TRUE(GTEST_IS_THREADSAFE);
SimpleAllocator a;
Master m(&a);
PID<Master> master = process::spawn(&m);
MockExecutor exec;
ExecutorDriver* execDriver;
string execData;
trigger execFrameworkMessageCall, shutdownCall;
EXPECT_CALL(exec, registered(_, _, _, _, _, _))
.WillOnce(SaveArg<0>(&execDriver));
EXPECT_CALL(exec, launchTask(_, _))
.WillOnce(SendStatusUpdate(TASK_RUNNING));
EXPECT_CALL(exec, frameworkMessage(_, _))
.WillOnce(DoAll(SaveArg<1>(&execData),
Trigger(&execFrameworkMessageCall)));
EXPECT_CALL(exec, shutdown(_))
.WillOnce(Trigger(&shutdownCall));
map<ExecutorID, Executor*> execs;
execs[DEFAULT_EXECUTOR_ID] = &exec;
TestingIsolationModule isolationModule(execs);
Resources resources = Resources::parse("cpus:2;mem:1024");
Slave s(resources, true, &isolationModule);
PID<Slave> slave = process::spawn(&s);
BasicMasterDetector detector(master, slave, true);
// Launch the first (i.e., failing) scheduler and wait until the
// first status update message is sent to it (drop the message).
MockScheduler sched;
MesosSchedulerDriver schedDriver(&sched, "", DEFAULT_EXECUTOR_INFO, master);
vector<Offer> offers;
TaskStatus status;
string schedData;
trigger resourceOffersCall, statusUpdateCall, schedFrameworkMessageCall;
EXPECT_CALL(sched, registered(&schedDriver, _))
.Times(1);
EXPECT_CALL(sched, resourceOffers(&schedDriver, _))
.WillOnce(DoAll(SaveArg<1>(&offers),
Trigger(&resourceOffersCall)))
.WillRepeatedly(Return());
EXPECT_CALL(sched, statusUpdate(&schedDriver, _))
.WillOnce(DoAll(SaveArg<1>(&status), Trigger(&statusUpdateCall)));
EXPECT_CALL(sched, frameworkMessage(&schedDriver, _, _, _))
.WillOnce(DoAll(SaveArg<3>(&schedData),
Trigger(&schedFrameworkMessageCall)));
schedDriver.start();
WAIT_UNTIL(resourceOffersCall);
EXPECT_NE(0, offers.size());
TaskDescription task;
task.set_name("");
task.mutable_task_id()->set_value("1");
task.mutable_slave_id()->MergeFrom(offers[0].slave_id());
task.mutable_resources()->MergeFrom(offers[0].resources());
vector<TaskDescription> tasks;
tasks.push_back(task);
schedDriver.launchTasks(offers[0].id(), tasks);
WAIT_UNTIL(statusUpdateCall);
EXPECT_EQ(TASK_RUNNING, status.state());
string hello = "hello";
schedDriver.sendFrameworkMessage(offers[0].slave_id(),
DEFAULT_EXECUTOR_ID,
hello);
WAIT_UNTIL(execFrameworkMessageCall);
EXPECT_EQ(hello, execData);
string reply = "reply";
execDriver->sendFrameworkMessage(reply);
WAIT_UNTIL(schedFrameworkMessageCall);
EXPECT_EQ(reply, schedData);
schedDriver.stop();
schedDriver.join();
WAIT_UNTIL(shutdownCall); // To ensure can deallocate MockExecutor.
process::terminate(slave);
process::wait(slave);
process::terminate(master);
process::wait(master);
}
TEST_F(FaultToleranceTest, SchedulerFailoverFrameworkMessage)
{
Try<PID<Master> > master = StartMaster();
ASSERT_SOME(master);
MockExecutor exec(DEFAULT_EXECUTOR_ID);
Try<PID<Slave> > slave = StartSlave(&exec);
ASSERT_SOME(slave);
MockScheduler sched1;
MesosSchedulerDriver driver1(&sched1, DEFAULT_FRAMEWORK_INFO, master.get());
FrameworkID frameworkId;
EXPECT_CALL(sched1, registered(&driver1, _, _))
.WillOnce(SaveArg<1>(&frameworkId));
Future<vector<Offer> > offers;
EXPECT_CALL(sched1, resourceOffers(&driver1, _))
.WillOnce(FutureArg<1>(&offers))
.WillRepeatedly(Return()); // Ignore subsequent offers.
driver1.start();
AWAIT_READY(offers);
EXPECT_NE(0u, offers.get().size());
TaskInfo task;
task.set_name("");
task.mutable_task_id()->set_value("1");
task.mutable_slave_id()->MergeFrom(offers.get()[0].slave_id());
task.mutable_resources()->MergeFrom(offers.get()[0].resources());
task.mutable_executor()->MergeFrom(DEFAULT_EXECUTOR_INFO);
vector<TaskInfo> tasks;
tasks.push_back(task);
Future<TaskStatus> status;
EXPECT_CALL(sched1, statusUpdate(&driver1, _))
.WillOnce(FutureArg<1>(&status));
ExecutorDriver* execDriver;
EXPECT_CALL(exec, registered(_, _, _, _))
.WillOnce(SaveArg<0>(&execDriver));
EXPECT_CALL(exec, launchTask(_, _))
.WillOnce(SendStatusUpdateFromTask(TASK_RUNNING));
driver1.launchTasks(offers.get()[0].id(), tasks);
AWAIT_READY(status);
EXPECT_EQ(TASK_RUNNING, status.get().state());
MockScheduler sched2;
FrameworkInfo framework2; // Bug in gcc 4.1.*, must assign on next line.
framework2 = DEFAULT_FRAMEWORK_INFO;
framework2.mutable_id()->MergeFrom(frameworkId);
MesosSchedulerDriver driver2(&sched2, framework2, master.get());
Future<Nothing> registered;
EXPECT_CALL(sched2, registered(&driver2, frameworkId, _))
.WillOnce(FutureSatisfy(®istered));
Future<Nothing> frameworkMessage;
EXPECT_CALL(sched2, frameworkMessage(&driver2, _, _, _))
.WillOnce(FutureSatisfy(&frameworkMessage));
EXPECT_CALL(sched1, error(&driver1, "Framework failed over"));
driver2.start();
AWAIT_READY(registered);
execDriver->sendFrameworkMessage("Executor to Framework message");
AWAIT_READY(frameworkMessage);
EXPECT_CALL(exec, shutdown(_))
.Times(AtMost(1));
driver1.stop();
driver2.stop();
driver1.join();
driver2.join();
Shutdown();
}
TEST(FaultToleranceTest, SchedulerFailoverFrameworkMessage)
{
ASSERT_TRUE(GTEST_IS_THREADSAFE);
SimpleAllocator a;
Master m(&a);
PID<Master> master = process::spawn(&m);
MockExecutor exec;
ExecutorDriver* execDriver;
EXPECT_CALL(exec, registered(_, _, _, _))
.WillOnce(SaveArg<0>(&execDriver));
EXPECT_CALL(exec, launchTask(_, _))
.WillOnce(SendStatusUpdate(TASK_RUNNING));
EXPECT_CALL(exec, shutdown(_))
.Times(AtMost(1));
map<ExecutorID, Executor*> execs;
execs[DEFAULT_EXECUTOR_ID] = &exec;
TestingIsolationModule isolationModule(execs);
Resources resources = Resources::parse("cpus:2;mem:1024");
Slave s(resources, true, &isolationModule);
PID<Slave> slave = process::spawn(&s);
BasicMasterDetector detector(master, slave, true);
MockScheduler sched1;
MesosSchedulerDriver driver1(&sched1, DEFAULT_FRAMEWORK_INFO, master);
FrameworkID frameworkId;
vector<Offer> offers;
TaskStatus status;
trigger sched1ResourceOfferCall, sched1StatusUpdateCall;
EXPECT_CALL(sched1, registered(&driver1, _, _))
.WillOnce(SaveArg<1>(&frameworkId));
EXPECT_CALL(sched1, statusUpdate(&driver1, _))
.WillOnce(DoAll(SaveArg<1>(&status), Trigger(&sched1StatusUpdateCall)));
EXPECT_CALL(sched1, resourceOffers(&driver1, _))
.WillOnce(DoAll(SaveArg<1>(&offers),
Trigger(&sched1ResourceOfferCall)))
.WillRepeatedly(Return());
EXPECT_CALL(sched1, error(&driver1, "Framework failed over"))
.Times(1);
driver1.start();
WAIT_UNTIL(sched1ResourceOfferCall);
EXPECT_NE(0, offers.size());
TaskInfo task;
task.set_name("");
task.mutable_task_id()->set_value("1");
task.mutable_slave_id()->MergeFrom(offers[0].slave_id());
task.mutable_resources()->MergeFrom(offers[0].resources());
task.mutable_executor()->MergeFrom(DEFAULT_EXECUTOR_INFO);
vector<TaskInfo> tasks;
tasks.push_back(task);
driver1.launchTasks(offers[0].id(), tasks);
WAIT_UNTIL(sched1StatusUpdateCall);
EXPECT_EQ(TASK_RUNNING, status.state());
MockScheduler sched2;
FrameworkInfo framework2; // Bug in gcc 4.1.*, must assign on next line.
framework2 = DEFAULT_FRAMEWORK_INFO;
framework2.mutable_id()->MergeFrom(frameworkId);
MesosSchedulerDriver driver2(&sched2, framework2, master);
trigger sched2RegisteredCall, sched2FrameworkMessageCall;
EXPECT_CALL(sched2, registered(&driver2, frameworkId, _))
.WillOnce(Trigger(&sched2RegisteredCall));
EXPECT_CALL(sched2, frameworkMessage(&driver2, _, _, _))
.WillOnce(Trigger(&sched2FrameworkMessageCall));
driver2.start();
WAIT_UNTIL(sched2RegisteredCall);
execDriver->sendFrameworkMessage("");
WAIT_UNTIL(sched2FrameworkMessageCall);
driver1.stop();
driver2.stop();
driver1.join();
driver2.join();
process::terminate(slave);
process::wait(slave);
process::terminate(master);
process::wait(master);
}
