inline StatusUpdate createStatusUpdate( const FrameworkID& frameworkId, const SlaveID& slaveId, const TaskID& taskId, const TaskState& state, const std::string& message = "", const Option<ExecutorID>& executorId = None()) { StatusUpdate update; update.set_timestamp(process::Clock::now().secs()); update.set_uuid(UUID::random().toBytes()); update.mutable_framework_id()->MergeFrom(frameworkId); update.mutable_slave_id()->MergeFrom(slaveId); if (executorId.isSome()) { update.mutable_executor_id()->MergeFrom(executorId.get()); } TaskStatus* status = update.mutable_status(); status->mutable_task_id()->MergeFrom(taskId); status->mutable_slave_id()->MergeFrom(slaveId); status->set_state(state); status->set_message(message); status->set_timestamp(update.timestamp()); return update; }
// This test verifies that reconciliation of an unknown task that // belongs to a known slave results in TASK_LOST. TEST_F(ReconciliationTest, UnknownTask) { Try<PID<Master> > master = StartMaster(); ASSERT_SOME(master); Future<SlaveRegisteredMessage> slaveRegisteredMessage = FUTURE_PROTOBUF(SlaveRegisteredMessage(), _, _); Try<PID<Slave> > slave = StartSlave(); ASSERT_SOME(slave); // Wait for the slave to register and get the slave id. AWAIT_READY(slaveRegisteredMessage); const SlaveID slaveId = slaveRegisteredMessage.get().slave_id(); MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get(), DEFAULT_CREDENTIAL); Future<FrameworkID> frameworkId; EXPECT_CALL(sched, registered(&driver, _, _)) .WillOnce(FutureArg<1>(&frameworkId)); EXPECT_CALL(sched, resourceOffers(&driver, _)) .WillRepeatedly(Return()); // Ignore offers. driver.start(); // Wait until the framework is registered. AWAIT_READY(frameworkId); Future<TaskStatus> update; EXPECT_CALL(sched, statusUpdate(&driver, _)) .WillOnce(FutureArg<1>(&update)); vector<TaskStatus> statuses; // Create a task status with a random task id. TaskStatus status; status.mutable_task_id()->set_value(UUID::random().toString()); status.mutable_slave_id()->CopyFrom(slaveId); status.set_state(TASK_RUNNING); statuses.push_back(status); driver.reconcileTasks(statuses); // Framework should receive TASK_LOST for unknown task. AWAIT_READY(update); EXPECT_EQ(TASK_LOST, update.get().state()); driver.stop(); driver.join(); Shutdown(); // Must shutdown before 'containerizer' gets deallocated. }
// TODO(vinod): Make SlaveID optional because 'StatusUpdate.SlaveID' // is optional. StatusUpdate createStatusUpdate( const FrameworkID& frameworkId, const Option<SlaveID>& slaveId, const TaskID& taskId, const TaskState& state, const TaskStatus::Source& source, const string& message = "", const Option<TaskStatus::Reason>& reason = None(), const Option<ExecutorID>& executorId = None(), const Option<bool>& healthy = None()) { StatusUpdate update; update.set_timestamp(process::Clock::now().secs()); update.set_uuid(UUID::random().toBytes()); update.mutable_framework_id()->MergeFrom(frameworkId); if (slaveId.isSome()) { update.mutable_slave_id()->MergeFrom(slaveId.get()); } if (executorId.isSome()) { update.mutable_executor_id()->MergeFrom(executorId.get()); } TaskStatus* status = update.mutable_status(); status->mutable_task_id()->MergeFrom(taskId); if (slaveId.isSome()) { status->mutable_slave_id()->MergeFrom(slaveId.get()); } status->set_state(state); status->set_source(source); status->set_message(message); status->set_timestamp(update.timestamp()); if (reason.isSome()) { status->set_reason(reason.get()); } if (healthy.isSome()) { status->set_healthy(healthy.get()); } return update; }
// This test verifies that reconciliation of a task that belongs to an // unknown slave results in TASK_LOST. TEST_F(ReconciliationTest, UnknownSlave) { Try<PID<Master> > master = StartMaster(); ASSERT_SOME(master); MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get(), DEFAULT_CREDENTIAL); Future<FrameworkID> frameworkId; EXPECT_CALL(sched, registered(&driver, _, _)) .WillOnce(FutureArg<1>(&frameworkId)); driver.start(); // Wait until the framework is registered. AWAIT_READY(frameworkId); Future<TaskStatus> update; EXPECT_CALL(sched, statusUpdate(&driver, _)) .WillOnce(FutureArg<1>(&update)); vector<TaskStatus> statuses; // Create a task status with a random slave id (and task id). TaskStatus status; status.mutable_task_id()->set_value(UUID::random().toString()); status.mutable_slave_id()->set_value(UUID::random().toString()); status.set_state(TASK_RUNNING); statuses.push_back(status); driver.reconcileTasks(statuses); // Framework should receive TASK_LOST because the slave is unknown. AWAIT_READY(update); EXPECT_EQ(TASK_LOST, update.get().state()); driver.stop(); driver.join(); Shutdown(); }
// This test ensures we don't break the API when it comes to JSON // representation of tasks. Also, we want to ensure that tasks are // modeled the same way when using 'Task' vs. 'TaskInfo'. TEST(HTTP, ModelTask) { TaskID taskId; taskId.set_value("t"); SlaveID slaveId; slaveId.set_value("s"); ExecutorID executorId; executorId.set_value("t"); FrameworkID frameworkId; frameworkId.set_value("f"); TaskState state = TASK_RUNNING; vector<TaskStatus> statuses; TaskStatus status; status.mutable_task_id()->CopyFrom(taskId); status.set_state(state); status.mutable_slave_id()->CopyFrom(slaveId); status.mutable_executor_id()->CopyFrom(executorId); status.set_timestamp(0.0); statuses.push_back(status); TaskInfo task; task.set_name("task"); task.mutable_task_id()->CopyFrom(taskId); task.mutable_slave_id()->CopyFrom(slaveId); task.mutable_command()->set_value("echo hello"); Task task_ = protobuf::createTask(task, state, frameworkId); task_.add_statuses()->CopyFrom(statuses[0]); JSON::Value object = model(task, frameworkId, state, statuses); JSON::Value object_ = model(task_); Try<JSON::Value> expected = JSON::parse( "{" " \"executor_id\":\"\"," " \"framework_id\":\"f\"," " \"id\":\"t\"," " \"name\":\"task\"," " \"resources\":" " {" " \"cpus\":0," " \"disk\":0," " \"mem\":0" " }," " \"slave_id\":\"s\"," " \"state\":\"TASK_RUNNING\"," " \"statuses\":" " [" " {" " \"state\":\"TASK_RUNNING\"," " \"timestamp\":0" " }" " ]" "}"); ASSERT_SOME(expected); EXPECT_EQ(expected.get(), object); EXPECT_EQ(expected.get(), object_); // Ensure both are modeled the same. EXPECT_EQ(object, object_); }
// This test verifies that a reconciliation request that comes before // '_launchTasks()' is ignored. TEST_F(MasterAuthorizationTest, ReconcileTask) { MockAuthorizer authorizer; Try<PID<Master> > master = StartMaster(&authorizer); ASSERT_SOME(master); MockExecutor exec(DEFAULT_EXECUTOR_ID); Try<PID<Slave> > slave = StartSlave(&exec); ASSERT_SOME(slave); MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get(), DEFAULT_CREDENTIAL); EXPECT_CALL(sched, registered(&driver, _, _)) .Times(1); Future<vector<Offer> > offers; EXPECT_CALL(sched, resourceOffers(&driver, _)) .WillOnce(FutureArg<1>(&offers)) .WillRepeatedly(Return()); // Ignore subsequent offers. driver.start(); AWAIT_READY(offers); EXPECT_NE(0u, offers.get().size()); TaskInfo task = createTask(offers.get()[0], "", DEFAULT_EXECUTOR_ID); vector<TaskInfo> tasks; tasks.push_back(task); // Return a pending future from authorizer. Future<Nothing> future; Promise<bool> promise; EXPECT_CALL(authorizer, authorize(An<const mesos::ACL::RunTasks&>())) .WillOnce(DoAll(FutureSatisfy(&future), Return(promise.future()))); driver.launchTasks(offers.get()[0].id(), tasks); // Wait until authorization is in progress. AWAIT_READY(future); // Scheduler shouldn't get an update from reconciliation. EXPECT_CALL(sched, statusUpdate(&driver, _)) .Times(0); Future<ReconcileTasksMessage> reconcileTasksMessage = FUTURE_PROTOBUF(ReconcileTasksMessage(), _, _); vector<TaskStatus> statuses; TaskStatus status; status.mutable_task_id()->CopyFrom(task.task_id()); status.mutable_slave_id()->CopyFrom(offers.get()[0].slave_id()); status.set_state(TASK_STAGING); statuses.push_back(status); driver.reconcileTasks(statuses); AWAIT_READY(reconcileTasksMessage); // Make sure the framework doesn't receive any update. Clock::pause(); Clock::settle(); // Now stop the framework. driver.stop(); driver.join(); Shutdown(); // Must shutdown before 'containerizer' gets deallocated. }
// This test verifies that reconciliation sends the latest task // status, when the task state does not match between the framework // and the master. TEST_F(ReconciliationTest, TaskStateMismatch) { Try<PID<Master> > master = StartMaster(); ASSERT_SOME(master); MockExecutor exec(DEFAULT_EXECUTOR_ID); TestContainerizer containerizer(&exec); Try<PID<Slave> > slave = StartSlave(&containerizer); ASSERT_SOME(slave); MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get(), DEFAULT_CREDENTIAL); Future<FrameworkID> frameworkId; EXPECT_CALL(sched, registered(&driver, _, _)) .WillOnce(FutureArg<1>(&frameworkId)); EXPECT_CALL(sched, resourceOffers(&driver, _)) .WillOnce(LaunchTasks(DEFAULT_EXECUTOR_INFO, 1, 1, 512, "*")) .WillRepeatedly(Return()); // Ignore subsequent offers. EXPECT_CALL(exec, registered(_, _, _, _)); EXPECT_CALL(exec, launchTask(_, _)) .WillOnce(SendStatusUpdateFromTask(TASK_RUNNING)); Future<TaskStatus> update; EXPECT_CALL(sched, statusUpdate(&driver, _)) .WillOnce(FutureArg<1>(&update)); driver.start(); // Wait until the framework is registered. AWAIT_READY(frameworkId); AWAIT_READY(update); EXPECT_EQ(TASK_RUNNING, update.get().state()); EXPECT_EQ(true, update.get().has_slave_id()); const TaskID taskId = update.get().task_id(); const SlaveID slaveId = update.get().slave_id(); // If framework has different state, current state should be reported. Future<TaskStatus> update2; EXPECT_CALL(sched, statusUpdate(&driver, _)) .WillOnce(FutureArg<1>(&update2)); vector<TaskStatus> statuses; TaskStatus status; status.mutable_task_id()->CopyFrom(taskId); status.mutable_slave_id()->CopyFrom(slaveId); status.set_state(TASK_KILLED); statuses.push_back(status); driver.reconcileTasks(statuses); AWAIT_READY(update2); EXPECT_EQ(TASK_RUNNING, update2.get().state()); EXPECT_CALL(exec, shutdown(_)) .Times(AtMost(1)); driver.stop(); driver.join(); Shutdown(); // Must shutdown before 'containerizer' gets deallocated. }
// This test ensures that reconciliation requests for tasks that are // pending are exposed in reconciliation. TEST_F(ReconciliationTest, PendingTask) { MockAuthorizer authorizer; Try<PID<Master> > master = StartMaster(&authorizer); ASSERT_SOME(master); MockExecutor exec(DEFAULT_EXECUTOR_ID); Future<SlaveRegisteredMessage> slaveRegisteredMessage = FUTURE_PROTOBUF(SlaveRegisteredMessage(), _, _); Try<PID<Slave> > slave = StartSlave(); ASSERT_SOME(slave); // Wait for the slave to register and get the slave id. AWAIT_READY(slaveRegisteredMessage); const SlaveID slaveId = slaveRegisteredMessage.get().slave_id(); MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get(), DEFAULT_CREDENTIAL); EXPECT_CALL(sched, registered(&driver, _, _)) .Times(1); Future<vector<Offer> > offers; EXPECT_CALL(sched, resourceOffers(&driver, _)) .WillOnce(FutureArg<1>(&offers)) .WillRepeatedly(Return()); // Ignore subsequent offers. driver.start(); AWAIT_READY(offers); EXPECT_NE(0u, offers.get().size()); // Return a pending future from authorizer. Future<Nothing> authorize; Promise<bool> promise; EXPECT_CALL(authorizer, authorize(An<const mesos::ACL::RunTask&>())) .WillOnce(DoAll(FutureSatisfy(&authorize), Return(promise.future()))); TaskInfo task = createTask(offers.get()[0], "", DEFAULT_EXECUTOR_ID); vector<TaskInfo> tasks; tasks.push_back(task); driver.launchTasks(offers.get()[0].id(), tasks); // Wait until authorization is in progress. AWAIT_READY(authorize); // First send an implicit reconciliation request for this task. Future<TaskStatus> update; EXPECT_CALL(sched, statusUpdate(&driver, _)) .WillOnce(FutureArg<1>(&update)); vector<TaskStatus> statuses; driver.reconcileTasks(statuses); AWAIT_READY(update); EXPECT_EQ(TASK_STAGING, update.get().state()); EXPECT_TRUE(update.get().has_slave_id()); // Now send an explicit reconciliation request for this task. Future<TaskStatus> update2; EXPECT_CALL(sched, statusUpdate(&driver, _)) .WillOnce(FutureArg<1>(&update2)); TaskStatus status; status.mutable_task_id()->CopyFrom(task.task_id()); status.mutable_slave_id()->CopyFrom(slaveId); status.set_state(TASK_STAGING); statuses.push_back(status); driver.reconcileTasks(statuses); AWAIT_READY(update2); EXPECT_EQ(TASK_STAGING, update2.get().state()); EXPECT_TRUE(update2.get().has_slave_id()); driver.stop(); driver.join(); Shutdown(); // Must shutdown before 'containerizer' gets deallocated. }
// This test verifies that reconciliation of a task that belongs to a // slave that is a transitional state doesn't result in an update. TEST_F(ReconciliationTest, SlaveInTransition) { master::Flags masterFlags = CreateMasterFlags(); Try<PID<Master> > master = StartMaster(); ASSERT_SOME(master); // Start a checkpointing slave. slave::Flags slaveFlags = CreateSlaveFlags(); slaveFlags.checkpoint = true; Future<SlaveRegisteredMessage> slaveRegisteredMessage = FUTURE_PROTOBUF(SlaveRegisteredMessage(), _, _); Try<PID<Slave> > slave = StartSlave(slaveFlags); ASSERT_SOME(slave); // Wait for the slave to register and get the slave id. AWAIT_READY(slaveRegisteredMessage); const SlaveID slaveId = slaveRegisteredMessage.get().slave_id(); // Stop the master and slave. Stop(master.get()); Stop(slave.get()); MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get(), DEFAULT_CREDENTIAL); Future<FrameworkID> frameworkId; EXPECT_CALL(sched, registered(&driver, _, _)) .WillOnce(FutureArg<1>(&frameworkId)); EXPECT_CALL(sched, resourceOffers(&driver, _)) .WillRepeatedly(Return()); // Ignore offers. // Framework should not receive any update. EXPECT_CALL(sched, statusUpdate(&driver, _)) .Times(0); // Drop '&Master::_reregisterSlave' dispatch so that the slave is // in 'reregistering' state. Future<Nothing> _reregisterSlave = DROP_DISPATCH(_, &Master::_reregisterSlave); // Restart the master. master = StartMaster(masterFlags); ASSERT_SOME(master); driver.start(); // Wait for the framework to register. AWAIT_READY(frameworkId); // Restart the slave. slave = StartSlave(slaveFlags); ASSERT_SOME(slave); // Slave will be in 'reregistering' state here. AWAIT_READY(_reregisterSlave); vector<TaskStatus> statuses; // Create a task status with a random task id. TaskStatus status; status.mutable_task_id()->set_value(UUID::random().toString()); status.mutable_slave_id()->CopyFrom(slaveId); status.set_state(TASK_RUNNING); statuses.push_back(status); Future<ReconcileTasksMessage> reconcileTasksMessage = FUTURE_PROTOBUF(ReconcileTasksMessage(), _ , _); Clock::pause(); driver.reconcileTasks(statuses); // Make sure the master received the reconcile tasks message. AWAIT_READY(reconcileTasksMessage); // The Clock::settle() will ensure that framework would receive // a status update if it is sent by the master. In this test it // shouldn't receive any. Clock::settle(); driver.stop(); driver.join(); Shutdown(); }
StatusUpdate createStatusUpdate( const FrameworkID& frameworkId, const Option<SlaveID>& slaveId, const TaskID& taskId, const TaskState& state, const TaskStatus::Source& source, const Option<UUID>& uuid, const string& message = "", const Option<TaskStatus::Reason>& reason = None(), const Option<ExecutorID>& executorId = None(), const Option<bool>& healthy = None()) { StatusUpdate update; update.set_timestamp(process::Clock::now().secs()); update.mutable_framework_id()->MergeFrom(frameworkId); if (slaveId.isSome()) { update.mutable_slave_id()->MergeFrom(slaveId.get()); } if (executorId.isSome()) { update.mutable_executor_id()->MergeFrom(executorId.get()); } TaskStatus* status = update.mutable_status(); status->mutable_task_id()->MergeFrom(taskId); if (slaveId.isSome()) { status->mutable_slave_id()->MergeFrom(slaveId.get()); } status->set_state(state); status->set_source(source); status->set_message(message); status->set_timestamp(update.timestamp()); if (uuid.isSome()) { update.set_uuid(uuid.get().toBytes()); status->set_uuid(uuid.get().toBytes()); } else { // Note that in 0.22.x, the StatusUpdate.uuid was required // even though the scheduler driver ignores it for master // and scheduler driver generated updates. So we continue // to "set" it here so that updates coming from a 0.23.x // master can be parsed by a 0.22.x scheduler driver. // // TODO(bmahler): In 0.24.x, leave the uuid unset. update.set_uuid(""); } if (reason.isSome()) { status->set_reason(reason.get()); } if (healthy.isSome()) { status->set_healthy(healthy.get()); } return update; }
// This test ensures we don't break the API when it comes to JSON // representation of tasks. TEST(HTTPTest, ModelTask) { TaskID taskId; taskId.set_value("t"); SlaveID slaveId; slaveId.set_value("s"); ExecutorID executorId; executorId.set_value("t"); FrameworkID frameworkId; frameworkId.set_value("f"); TaskState state = TASK_RUNNING; vector<TaskStatus> statuses; TaskStatus status; status.mutable_task_id()->CopyFrom(taskId); status.set_state(state); status.mutable_slave_id()->CopyFrom(slaveId); status.mutable_executor_id()->CopyFrom(executorId); status.set_timestamp(0.0); statuses.push_back(status); Labels labels; labels.add_labels()->CopyFrom(createLabel("ACTION", "port:7987 DENY")); Ports ports; Port* port = ports.add_ports(); port->set_number(80); port->mutable_labels()->CopyFrom(labels); DiscoveryInfo discovery; discovery.set_visibility(DiscoveryInfo::CLUSTER); discovery.set_name("discover"); discovery.mutable_ports()->CopyFrom(ports); TaskInfo taskInfo; taskInfo.set_name("task"); taskInfo.mutable_task_id()->CopyFrom(taskId); taskInfo.mutable_slave_id()->CopyFrom(slaveId); taskInfo.mutable_command()->set_value("echo hello"); taskInfo.mutable_discovery()->CopyFrom(discovery); Task task = createTask(taskInfo, state, frameworkId); task.add_statuses()->CopyFrom(statuses[0]); JSON::Value object = model(task); Try<JSON::Value> expected = JSON::parse( "{" " \"executor_id\":\"\"," " \"framework_id\":\"f\"," " \"id\":\"t\"," " \"name\":\"task\"," " \"resources\":" " {" " \"cpus\":0," " \"disk\":0," " \"gpus\":0," " \"mem\":0" " }," " \"slave_id\":\"s\"," " \"state\":\"TASK_RUNNING\"," " \"statuses\":" " [" " {" " \"state\":\"TASK_RUNNING\"," " \"timestamp\":0" " }" " ]," " \"discovery\":" " {" " \"name\":\"discover\"," " \"ports\":" " {" " \"ports\":" " [" " {" " \"number\":80," " \"labels\":" " {" " \"labels\":" " [" " {" " \"key\":\"ACTION\"," " \"value\":\"port:7987 DENY\"" " }" " ]" " }" " }" " ]" " }," " \"visibility\":\"CLUSTER\"" " }" "}"); ASSERT_SOME(expected); EXPECT_EQ(expected.get(), object); }