// Ensures that the driver can handle the RESCIND event.
TEST_F(SchedulerDriverEventTest, Rescind)
{
Try<PID<Master>> master = StartMaster();
ASSERT_SOME(master);
MockScheduler sched;
MesosSchedulerDriver driver(
&sched, DEFAULT_FRAMEWORK_INFO, master.get(), DEFAULT_CREDENTIAL);
EXPECT_CALL(sched, registered(&driver, _, _));
Future<Message> frameworkRegisteredMessage =
FUTURE_MESSAGE(Eq(FrameworkRegisteredMessage().GetTypeName()), _, _);
driver.start();
AWAIT_READY(frameworkRegisteredMessage);
UPID frameworkPid = frameworkRegisteredMessage.get().to;
Event event;
event.set_type(Event::RESCIND);
event.mutable_rescind()->mutable_offer_id()->set_value("O");
Future<Nothing> offerRescinded;
EXPECT_CALL(sched, offerRescinded(&driver, event.rescind().offer_id()))
.WillOnce(FutureSatisfy(&offerRescinded));
process::post(master.get(), frameworkPid, event);
AWAIT_READY(offerRescinded);
}
// Ensures that the driver can handle the ERROR event.
TEST_F(SchedulerDriverEventTest, Error)
{
Try<PID<Master>> master = StartMaster();
ASSERT_SOME(master);
MockScheduler sched;
MesosSchedulerDriver driver(
&sched, DEFAULT_FRAMEWORK_INFO, master.get(), DEFAULT_CREDENTIAL);
EXPECT_CALL(sched, registered(&driver, _, _));
Future<Message> frameworkRegisteredMessage =
FUTURE_MESSAGE(Eq(FrameworkRegisteredMessage().GetTypeName()), _, _);
driver.start();
AWAIT_READY(frameworkRegisteredMessage);
UPID frameworkPid = frameworkRegisteredMessage.get().to;
Event event;
event.set_type(Event::ERROR);
event.mutable_error()->set_message("error message");
Future<Nothing> error;
EXPECT_CALL(sched, error(&driver, event.error().message()))
.WillOnce(FutureSatisfy(&error));
process::post(master.get(), frameworkPid, event);
AWAIT_READY(error);
}
// Ensures that the driver can handle the FAILURE event.
TEST_F(SchedulerDriverEventTest, Failure)
{
Try<Owned<cluster::Master>> master = StartMaster();
ASSERT_SOME(master);
MockScheduler sched;
MesosSchedulerDriver driver(
&sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL);
EXPECT_CALL(sched, registered(&driver, _, _));
Future<Message> frameworkRegisteredMessage =
FUTURE_MESSAGE(Eq(FrameworkRegisteredMessage().GetTypeName()), _, _);
driver.start();
AWAIT_READY(frameworkRegisteredMessage);
UPID frameworkPid = frameworkRegisteredMessage.get().to;
// Send a failure for an executor, which should trigger executorLost callback.
SlaveID slaveId;
slaveId.set_value("S");
ExecutorID executorId = DEFAULT_EXECUTOR_ID;
const int32_t status = 255;
Event event;
event.set_type(Event::FAILURE);
event.mutable_failure()->mutable_slave_id()->CopyFrom(slaveId);
event.mutable_failure()->mutable_executor_id()->CopyFrom(executorId);
event.mutable_failure()->set_status(status);
Future<Nothing> executorLost;
EXPECT_CALL(sched, executorLost(&driver, executorId, slaveId, status))
.WillOnce(FutureSatisfy(&executorLost));
process::post(master.get()->pid, frameworkPid, event);
AWAIT_READY(executorLost);
// Now, post a failure for a slave and expect a 'slaveLost'.
event.mutable_failure()->clear_executor_id();
Future<Nothing> slaveLost;
EXPECT_CALL(sched, slaveLost(&driver, slaveId))
.WillOnce(FutureSatisfy(&slaveLost));
process::post(master.get()->pid, frameworkPid, event);
AWAIT_READY(slaveLost);
driver.stop();
driver.join();
}
// Ensures that the driver can handle the SUBSCRIBED event.
TEST_F(SchedulerDriverEventTest, Subscribed)
{
Try<Owned<cluster::Master>> master = StartMaster();
ASSERT_SOME(master);
FrameworkInfo frameworkInfo = DEFAULT_FRAMEWORK_INFO;
frameworkInfo.set_failover_timeout(Weeks(2).secs());
// Make sure the initial registration calls 'registered'.
MockScheduler sched;
MesosSchedulerDriver driver(
&sched, frameworkInfo, master.get()->pid, DEFAULT_CREDENTIAL);
// Intercept the registration message, send a SUBSCRIBED instead.
Future<Message> frameworkRegisteredMessage =
DROP_MESSAGE(Eq(FrameworkRegisteredMessage().GetTypeName()), _, _);
// Ensure that there will be no (re-)registration retries
// from the scheduler driver.
Clock::pause();
driver.start();
AWAIT_READY(frameworkRegisteredMessage);
UPID frameworkPid = frameworkRegisteredMessage.get().to;
FrameworkRegisteredMessage message;
ASSERT_TRUE(message.ParseFromString(frameworkRegisteredMessage.get().body));
FrameworkID frameworkId = message.framework_id();
frameworkInfo.mutable_id()->CopyFrom(frameworkId);
Event event;
event.set_type(Event::SUBSCRIBED);
event.mutable_subscribed()->mutable_framework_id()->CopyFrom(frameworkId);
Future<Nothing> registered;
EXPECT_CALL(sched, registered(&driver, frameworkId, _))
.WillOnce(FutureSatisfy(®istered));
process::post(master.get()->pid, frameworkPid, event);
AWAIT_READY(registered);
driver.stop();
driver.join();
}
// Ensures that the driver can handle the FAILURE event.
TEST_F(SchedulerDriverEventTest, Failure)
{
Try<PID<Master>> master = StartMaster();
ASSERT_SOME(master);
MockScheduler sched;
MesosSchedulerDriver driver(
&sched, DEFAULT_FRAMEWORK_INFO, master.get(), DEFAULT_CREDENTIAL);
EXPECT_CALL(sched, registered(&driver, _, _));
Future<Message> frameworkRegisteredMessage =
FUTURE_MESSAGE(Eq(FrameworkRegisteredMessage().GetTypeName()), _, _);
driver.start();
AWAIT_READY(frameworkRegisteredMessage);
UPID frameworkPid = frameworkRegisteredMessage.get().to;
// Send a failure for an executor, this should be dropped
// to match the existing behavior of the scheduler driver.
SlaveID slaveId;
slaveId.set_value("S");
Event event;
event.set_type(Event::FAILURE);
event.mutable_failure()->mutable_slave_id()->CopyFrom(slaveId);
event.mutable_failure()->mutable_executor_id()->set_value("E");
process::post(master.get(), frameworkPid, event);
// Now, post a failure for a slave and expect a 'slaveLost'.
event.mutable_failure()->clear_executor_id();
Future<Nothing> slaveLost;
EXPECT_CALL(sched, slaveLost(&driver, slaveId))
.WillOnce(FutureSatisfy(&slaveLost));
process::post(master.get(), frameworkPid, event);
AWAIT_READY(slaveLost);
}
// Ensures that the driver can handle the MESSAGE event.
TEST_F(SchedulerDriverEventTest, Message)
{
Try<Owned<cluster::Master>> master = StartMaster();
ASSERT_SOME(master);
MockScheduler sched;
MesosSchedulerDriver driver(
&sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL);
EXPECT_CALL(sched, registered(&driver, _, _));
Future<Message> frameworkRegisteredMessage =
FUTURE_MESSAGE(Eq(FrameworkRegisteredMessage().GetTypeName()), _, _);
driver.start();
AWAIT_READY(frameworkRegisteredMessage);
UPID frameworkPid = frameworkRegisteredMessage.get().to;
Event event;
event.set_type(Event::MESSAGE);
event.mutable_message()->mutable_slave_id()->set_value("S");
event.mutable_message()->mutable_executor_id()->set_value("E");
event.mutable_message()->set_data("data");
Future<Nothing> frameworkMessage;
EXPECT_CALL(sched, frameworkMessage(
&driver,
event.message().executor_id(),
event.message().slave_id(),
event.message().data()))
.WillOnce(FutureSatisfy(&frameworkMessage));
process::post(master.get()->pid, frameworkPid, event);
AWAIT_READY(frameworkMessage);
driver.stop();
driver.join();
}
// This test verifies that if master --> framework socket closes and the
// framework is not aware of it (i.e., one way network partition), all
// subsequent calls from the framework after the master has marked it as
// disconnected would result in an error message causing the framework to abort.
TEST_F(PartitionTest, OneWayPartitionMasterToScheduler)
{
Try<Owned<cluster::Master>> master = StartMaster();
ASSERT_SOME(master);
FrameworkInfo frameworkInfo = DEFAULT_FRAMEWORK_INFO;
frameworkInfo.set_failover_timeout(Weeks(2).secs());
MockScheduler sched;
StandaloneMasterDetector detector(master.get()->pid);
TestingMesosSchedulerDriver driver(&sched, &detector, frameworkInfo);
Future<process::Message> frameworkRegisteredMessage =
FUTURE_MESSAGE(Eq(FrameworkRegisteredMessage().GetTypeName()), _, _);
Future<Nothing> registered;
EXPECT_CALL(sched, registered(&driver, _, _))
.WillOnce(FutureSatisfy(®istered));
driver.start();
AWAIT_READY(frameworkRegisteredMessage);
AWAIT_READY(registered);
Future<Nothing> error;
EXPECT_CALL(sched, error(&driver, _))
.WillOnce(FutureSatisfy(&error));
// Simulate framework disconnection. This should result in an error message.
ASSERT_TRUE(process::inject::exited(
frameworkRegisteredMessage.get().to, master.get()->pid));
AWAIT_READY(error);
driver.stop();
driver.join();
}
// Ensures the scheduler driver can handle the UPDATE event.
TEST_F(SchedulerDriverEventTest, Update)
{
Try<PID<Master>> master = StartMaster();
ASSERT_SOME(master);
MockScheduler sched;
MesosSchedulerDriver driver(
&sched, DEFAULT_FRAMEWORK_INFO, master.get(), DEFAULT_CREDENTIAL);
EXPECT_CALL(sched, registered(&driver, _, _));
Future<Message> frameworkRegisteredMessage =
FUTURE_MESSAGE(Eq(FrameworkRegisteredMessage().GetTypeName()), _, _);
driver.start();
AWAIT_READY(frameworkRegisteredMessage);
UPID frameworkPid = frameworkRegisteredMessage.get().to;
FrameworkRegisteredMessage message;
ASSERT_TRUE(message.ParseFromString(frameworkRegisteredMessage.get().body));
FrameworkID frameworkId = message.framework_id();
SlaveID slaveId;
slaveId.set_value("S");
TaskID taskId;
taskId.set_value("T");
ExecutorID executorId;
executorId.set_value("E");
// Generate an update that needs no acknowledgement.
Event event;
event.set_type(Event::UPDATE);
event.mutable_update()->mutable_status()->CopyFrom(
protobuf::createStatusUpdate(
frameworkId,
slaveId,
taskId,
TASK_RUNNING,
TaskStatus::SOURCE_MASTER,
None(),
"message",
None(),
executorId).status());
Future<Nothing> statusUpdate;
Future<Nothing> statusUpdate2;
EXPECT_CALL(sched, statusUpdate(&driver, event.update().status()))
.WillOnce(FutureSatisfy(&statusUpdate))
.WillOnce(FutureSatisfy(&statusUpdate2));
process::post(master.get(), frameworkPid, event);
AWAIT_READY(statusUpdate);
// Generate an update that requires acknowledgement.
event.mutable_update()->mutable_status()->set_uuid(UUID::random().toBytes());
Future<mesos::scheduler::Call> acknowledgement = DROP_CALL(
mesos::scheduler::Call(), mesos::scheduler::Call::ACKNOWLEDGE, _, _);
process::post(master.get(), frameworkPid, event);
AWAIT_READY(statusUpdate2);
AWAIT_READY(acknowledgement);
}
// Ensures that the driver can handle an OFFERS event.
// Note that this includes the ability to bypass the
// master when sending framework messages.
TEST_F(SchedulerDriverEventTest, Offers)
{
Try<PID<Master>> master = StartMaster();
ASSERT_SOME(master);
MockScheduler sched;
MesosSchedulerDriver schedDriver(
&sched, DEFAULT_FRAMEWORK_INFO, master.get(), DEFAULT_CREDENTIAL);
EXPECT_CALL(sched, registered(&schedDriver, _, _));
Future<Message> frameworkRegisteredMessage =
FUTURE_MESSAGE(Eq(FrameworkRegisteredMessage().GetTypeName()), _, _);
schedDriver.start();
AWAIT_READY(frameworkRegisteredMessage);
UPID frameworkPid = frameworkRegisteredMessage.get().to;
// Start a slave and capture the offers.
Future<ResourceOffersMessage> resourceOffersMessage =
DROP_PROTOBUF(ResourceOffersMessage(), _, _);
MockExecutor exec(DEFAULT_EXECUTOR_ID);
Try<PID<Slave>> slave = StartSlave(&exec);
ASSERT_SOME(slave);
AWAIT_READY(resourceOffersMessage);
google::protobuf::RepeatedPtrField<Offer> offers =
resourceOffersMessage.get().offers();
ASSERT_EQ(1, offers.size());
// Ignore future offer messages.
DROP_PROTOBUFS(ResourceOffersMessage(), _, _);
// Send the offers event and expect a 'resourceOffers' call.
Event event;
event.set_type(Event::OFFERS);
event.mutable_offers()->mutable_offers()->CopyFrom(offers);
Future<Nothing> resourceOffers;
EXPECT_CALL(sched, resourceOffers(&schedDriver, _))
.WillOnce(FutureSatisfy(&resourceOffers));
process::post(master.get(), frameworkPid, event);
AWAIT_READY(resourceOffers);
// To test that the framework -> executor messages are
// sent directly to the slave, launch a task and send
// the executor a message.
EXPECT_CALL(exec, registered(_, _, _, _));
EXPECT_CALL(exec, launchTask(_, _))
.WillOnce(SendStatusUpdateFromTask(TASK_RUNNING));
Future<TaskStatus> status;
EXPECT_CALL(sched, statusUpdate(&schedDriver, _))
.WillOnce(FutureArg<1>(&status));
TaskInfo task = createTask(offers.Get(0), "", DEFAULT_EXECUTOR_ID);
schedDriver.launchTasks(offers.Get(0).id(), {task});
AWAIT_READY(status);
EXPECT_EQ(TASK_RUNNING, status.get().state());
// This message should skip the master!
Future<FrameworkToExecutorMessage> frameworkToExecutorMessage =
FUTURE_PROTOBUF(FrameworkToExecutorMessage(), frameworkPid, slave.get());
Future<string> data;
EXPECT_CALL(exec, frameworkMessage(_, _))
.WillOnce(FutureArg<1>(&data));
schedDriver.sendFrameworkMessage(
DEFAULT_EXECUTOR_ID, offers.Get(0).slave_id(), "hello");
AWAIT_READY(frameworkToExecutorMessage);
AWAIT_EXPECT_EQ("hello", data);
EXPECT_CALL(exec, shutdown(_))
.Times(AtMost(1));
schedDriver.stop();
schedDriver.join();
Shutdown();
}
// Ensures that the driver can handle the SUBSCRIBED event
// after a master failover.
TEST_F(SchedulerDriverEventTest, SubscribedMasterFailover)
{
Try<PID<Master>> master = StartMaster();
ASSERT_SOME(master);
FrameworkInfo frameworkInfo = DEFAULT_FRAMEWORK_INFO;
frameworkInfo.set_failover_timeout(Weeks(2).secs());
// Make sure the initial registration calls 'registered'.
MockScheduler sched;
StandaloneMasterDetector detector(master.get());
TestingMesosSchedulerDriver driver(&sched, &detector, frameworkInfo);
// Intercept the registration message, send a SUBSCRIBED instead.
Future<Message> frameworkRegisteredMessage =
DROP_MESSAGE(Eq(FrameworkRegisteredMessage().GetTypeName()), _, _);
// Ensure that there will be no (re-)registration retries
// from the scheduler driver.
Clock::pause();
driver.start();
AWAIT_READY(frameworkRegisteredMessage);
UPID frameworkPid = frameworkRegisteredMessage.get().to;
FrameworkRegisteredMessage message;
ASSERT_TRUE(message.ParseFromString(frameworkRegisteredMessage.get().body));
FrameworkID frameworkId = message.framework_id();
frameworkInfo.mutable_id()->CopyFrom(frameworkId);
Event event;
event.set_type(Event::SUBSCRIBED);
event.mutable_subscribed()->mutable_framework_id()->CopyFrom(frameworkId);
Future<Nothing> registered;
EXPECT_CALL(sched, registered(&driver, frameworkId, _))
.WillOnce(FutureSatisfy(®istered));
process::post(master.get(), frameworkPid, event);
AWAIT_READY(registered);
// Fail over the master and expect a 'reregistered' call.
// Note that the master sends a registered message for
// this case (see MESOS-786).
Stop(master.get());
master = StartMaster();
ASSERT_SOME(master);
EXPECT_CALL(sched, disconnected(&driver));
frameworkRegisteredMessage =
DROP_MESSAGE(Eq(FrameworkRegisteredMessage().GetTypeName()), _, _);
detector.appoint(master.get());
AWAIT_READY(frameworkRegisteredMessage);
Future<Nothing> reregistered;
EXPECT_CALL(sched, reregistered(&driver, _))
.WillOnce(FutureSatisfy(&reregistered));
process::post(master.get(), frameworkPid, event);
AWAIT_READY(reregistered);
}
