// This is a simple end to end test that makes sure a master using log // storage with ZooKeeper can successfully launch a task. TEST_F(RegistrarZooKeeperTest, TaskRunning) { Try<Owned<cluster::Master>> master = StartMaster(); ASSERT_SOME(master); MockExecutor exec(DEFAULT_EXECUTOR_ID); TestContainerizer containerizer(&exec); Owned<MasterDetector> detector = master.get()->createDetector(); Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), &containerizer); ASSERT_SOME(slave); MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL); EXPECT_CALL(sched, registered(&driver, _, _)); Future<vector<Offer>> offers; EXPECT_CALL(sched, resourceOffers(&driver, _)) .WillOnce(FutureArg<1>(&offers)) .WillRepeatedly(Return()); // Ignore subsequent offers. driver.start(); AWAIT_READY(offers); EXPECT_NE(0u, offers->size()); TaskInfo task = createTask(offers.get()[0], "dummy", DEFAULT_EXECUTOR_ID); EXPECT_CALL(exec, registered(_, _, _, _)); EXPECT_CALL(exec, launchTask(_, _)) .WillOnce(SendStatusUpdateFromTask(TASK_RUNNING)); Future<Nothing> resourcesUpdated; EXPECT_CALL(containerizer, update(_, Resources(offers.get()[0].resources()))) .WillOnce(DoAll(FutureSatisfy(&resourcesUpdated), Return(Nothing()))); Future<TaskStatus> status; EXPECT_CALL(sched, statusUpdate(&driver, _)) .WillOnce(FutureArg<1>(&status)); driver.launchTasks(offers.get()[0].id(), {task}); AWAIT_READY(status); EXPECT_EQ(TASK_RUNNING, status->state()); AWAIT_READY(resourcesUpdated); EXPECT_CALL(exec, shutdown(_)) .Times(AtMost(1)); driver.stop(); driver.join(); }
// This test verifies that when the scheduler calls stop() before // abort(), no pending acknowledgements are sent. TEST_F(MesosSchedulerDriverTest, DropAckIfStopCalledBeforeAbort) { Try<PID<Master>> master = StartMaster(); ASSERT_SOME(master); MockExecutor exec(DEFAULT_EXECUTOR_ID); TestContainerizer containerizer(&exec); Try<PID<Slave>> slave = StartSlave(&containerizer); ASSERT_SOME(slave); MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get(), DEFAULT_CREDENTIAL); EXPECT_CALL(sched, registered(&driver, _, _)); EXPECT_CALL(sched, resourceOffers(&driver, _)) .WillOnce(LaunchTasks(DEFAULT_EXECUTOR_INFO, 1, 1, 16, "*")) .WillRepeatedly(Return()); // Ignore subsequent offers. // When an update is received, stop the driver and then abort it. Future<Nothing> statusUpdate; EXPECT_CALL(sched, statusUpdate(&driver, _)) .WillOnce(DoAll(StopAndAbort(), FutureSatisfy(&statusUpdate))); // Ensure no status update acknowledgements are sent from the driver // to the master. EXPECT_NO_FUTURE_CALLS( mesos::scheduler::Call(), mesos::scheduler::Call::ACKNOWLEDGE, _ , master.get()); EXPECT_CALL(exec, registered(_, _, _, _)); EXPECT_CALL(exec, launchTask(_, _)) .WillOnce(SendStatusUpdateFromTask(TASK_RUNNING)); EXPECT_CALL(exec, shutdown(_)) .Times(AtMost(1)); driver.start(); AWAIT_READY(statusUpdate); // Settle the clock to ensure driver finishes processing the status // update and sends acknowledgement if necessary. In this test it // shouldn't send an acknowledgement. Clock::pause(); Clock::settle(); driver.stop(); driver.join(); Shutdown(); }
// This test checks if destroy is called while container is being // launched, the composing containerizer still calls the underlying // containerizer's destroy and skip calling the rest of the // containerizers. TEST_F(ComposingContainerizerTest, DestroyWhileLaunching) { vector<Containerizer*> containerizers; MockContainerizer* mockContainerizer = new MockContainerizer(); MockContainerizer* mockContainerizer2 = new MockContainerizer(); containerizers.push_back(mockContainerizer); containerizers.push_back(mockContainerizer2); ComposingContainerizer containerizer(containerizers); ContainerID containerId; containerId.set_value("container"); TaskInfo taskInfo; ExecutorInfo executorInfo; SlaveID slaveId; std::map<std::string, std::string> environment; Promise<bool> launchPromise; EXPECT_CALL(*mockContainerizer, launch(_, _, _, _, _, _, _, _)) .WillOnce(Return(launchPromise.future())); Future<Nothing> destroy; EXPECT_CALL(*mockContainerizer, destroy(_)) .WillOnce(FutureSatisfy(&destroy)); Future<bool> launch = containerizer.launch( containerId, taskInfo, executorInfo, "dir", "user", slaveId, environment, false); Resources resources = Resources::parse("cpus:1;mem:256").get(); EXPECT_TRUE(launch.isPending()); containerizer.destroy(containerId); EXPECT_CALL(*mockContainerizer2, launch(_, _, _, _, _, _, _, _)) .Times(0); // We make sure the destroy is being called on the first containerizer. // The second containerizer shouldn't be called as well since the // container is already destroyed. AWAIT_READY(destroy); launchPromise.set(false); AWAIT_FAILED(launch); }
// This test ensures that destroy can be called at the end of the // launch loop. The composing containerizer still calls the // underlying containerizer's destroy (because it's not sure // if the containerizer can handle the type of container being // launched). If the launch is not supported by any containerizers // both the launch and destroy futures should be false. TEST_F(ComposingContainerizerTest, DestroyAfterLaunchLoop) { vector<Containerizer*> containerizers; MockContainerizer* mockContainerizer1 = new MockContainerizer(); containerizers.push_back(mockContainerizer1); ComposingContainerizer containerizer(containerizers); ContainerID containerId; containerId.set_value("container"); TaskInfo taskInfo; ExecutorInfo executorInfo; SlaveID slaveId; std::map<std::string, std::string> environment; Promise<bool> launchPromise; EXPECT_CALL(*mockContainerizer1, launch(_, _, _, _, _, _, _, _)) .WillOnce(Return(launchPromise.future())); Future<Nothing> destroy; Promise<bool> destroyPromise; EXPECT_CALL(*mockContainerizer1, destroy(_)) .WillOnce(DoAll(FutureSatisfy(&destroy), Return(destroyPromise.future()))); Future<bool> launched = containerizer.launch( containerId, taskInfo, executorInfo, "dir", "user", slaveId, environment, false); Resources resources = Resources::parse("cpus:1;mem:256").get(); EXPECT_TRUE(launched.isPending()); Future<bool> destroyed = containerizer.destroy(containerId); // We make sure the destroy is being called on the containerizer. AWAIT_READY(destroy); launchPromise.set(false); destroyPromise.set(false); // `launch` should return false and `destroyed` should return false // because none of the containerizers support the launch. AWAIT_EXPECT_EQ(false, launched); AWAIT_EXPECT_EQ(false, destroyed); }
// Ensures the containerizer responds correctly (false Future) to // a request to destroy an unknown container. TEST_F(ComposingContainerizerTest, DestroyUnknownContainer) { vector<Containerizer*> containerizers; MockContainerizer* mockContainerizer1 = new MockContainerizer(); MockContainerizer* mockContainerizer2 = new MockContainerizer(); containerizers.push_back(mockContainerizer1); containerizers.push_back(mockContainerizer2); ComposingContainerizer containerizer(containerizers); ContainerID containerId; containerId.set_value(UUID::random().toString()); AWAIT_EXPECT_FALSE(containerizer.destroy(containerId)); }
// Ensures the containerizer responds correctly (returns None) // to a request to wait on an unknown container. TEST_F(ComposingContainerizerTest, WaitUnknownContainer) { vector<Containerizer*> containerizers; MockContainerizer* mockContainerizer1 = new MockContainerizer(); MockContainerizer* mockContainerizer2 = new MockContainerizer(); containerizers.push_back(mockContainerizer1); containerizers.push_back(mockContainerizer2); ComposingContainerizer containerizer(containerizers); ContainerID containerId; containerId.set_value(UUID::random().toString()); Future<Option<ContainerTermination>> wait = containerizer.wait(containerId); AWAIT_READY(wait); EXPECT_NONE(wait.get()); }
// Test that memory pressure listening is restarted after recovery. TEST_F(MemoryPressureMesosTest, CGROUPS_ROOT_SlaveRecovery) { Try<Owned<cluster::Master>> master = StartMaster(); ASSERT_SOME(master); slave::Flags flags = CreateSlaveFlags(); // We only care about memory cgroup for this test. flags.isolation = "cgroups/mem"; Fetcher fetcher(flags); Try<MesosContainerizer*> _containerizer = MesosContainerizer::create(flags, true, &fetcher); ASSERT_SOME(_containerizer); Owned<MesosContainerizer> containerizer(_containerizer.get()); Owned<MasterDetector> detector = master.get()->createDetector(); Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), containerizer.get(), flags); ASSERT_SOME(slave); MockScheduler sched; // Enable checkpointing for the framework. FrameworkInfo frameworkInfo = DEFAULT_FRAMEWORK_INFO; frameworkInfo.set_checkpoint(true); MesosSchedulerDriver driver( &sched, frameworkInfo, master.get()->pid, DEFAULT_CREDENTIAL); EXPECT_CALL(sched, registered(_, _, _)); Future<vector<Offer>> offers; EXPECT_CALL(sched, resourceOffers(_, _)) .WillOnce(FutureArg<1>(&offers)) .WillRepeatedly(Return()); // Ignore subsequent offers. driver.start(); AWAIT_READY(offers); ASSERT_FALSE(offers->empty()); Offer offer = offers.get()[0]; // Run a task that triggers memory pressure event. We request 1G // disk because we are going to write a 512 MB file repeatedly. TaskInfo task = createTask( offer.slave_id(), Resources::parse("cpus:1;mem:256;disk:1024").get(), "while true; do dd count=512 bs=1M if=/dev/zero of=./temp; done"); Future<TaskStatus> starting; Future<TaskStatus> running; EXPECT_CALL(sched, statusUpdate(&driver, _)) .WillOnce(FutureArg<1>(&starting)) .WillOnce(FutureArg<1>(&running)) .WillRepeatedly(Return()); // Ignore subsequent updates. Future<Nothing> runningAck = FUTURE_DISPATCH(_, &Slave::_statusUpdateAcknowledgement); Future<Nothing> startingAck = FUTURE_DISPATCH(_, &Slave::_statusUpdateAcknowledgement); driver.launchTasks(offers.get()[0].id(), {task}); AWAIT_READY(starting); EXPECT_EQ(task.task_id(), starting->task_id()); EXPECT_EQ(TASK_STARTING, starting->state()); AWAIT_READY(startingAck); AWAIT_READY(running); EXPECT_EQ(task.task_id(), running->task_id()); EXPECT_EQ(TASK_RUNNING, running->state()); // Wait for the ACK to be checkpointed. AWAIT_READY_FOR(runningAck, Seconds(120)); // We restart the slave to let it recover. slave.get()->terminate(); // Set up so we can wait until the new slave updates the container's // resources (this occurs after the executor has re-registered). Future<Nothing> update = FUTURE_DISPATCH(_, &MesosContainerizerProcess::update); // Use the same flags. _containerizer = MesosContainerizer::create(flags, true, &fetcher); ASSERT_SOME(_containerizer); containerizer.reset(_containerizer.get()); Future<SlaveReregisteredMessage> reregistered = FUTURE_PROTOBUF(SlaveReregisteredMessage(), master.get()->pid, _); slave = StartSlave(detector.get(), containerizer.get(), flags); ASSERT_SOME(slave); AWAIT_READY(reregistered); // Wait until the containerizer is updated. AWAIT_READY(update); Future<hashset<ContainerID>> containers = containerizer->containers(); AWAIT_READY(containers); ASSERT_EQ(1u, containers->size()); ContainerID containerId = *(containers->begin()); // Wait a while for some memory pressure events to occur. Duration waited = Duration::zero(); do { Future<ResourceStatistics> usage = containerizer->usage(containerId); AWAIT_READY(usage); if (usage->mem_low_pressure_counter() > 0) { // We will check the correctness of the memory pressure counters // later, because the memory-hammering task is still active // and potentially incrementing these counters. break; } os::sleep(Milliseconds(100)); waited += Milliseconds(100); } while (waited < Seconds(5)); EXPECT_LE(waited, Seconds(5)); // Pause the clock to ensure that the reaper doesn't reap the exited // command executor and inform the containerizer/slave. Clock::pause(); Clock::settle(); Future<TaskStatus> killed; EXPECT_CALL(sched, statusUpdate(&driver, _)) .WillOnce(FutureArg<1>(&killed)); // Stop the memory-hammering task. driver.killTask(task.task_id()); AWAIT_READY_FOR(killed, Seconds(120)); EXPECT_EQ(task.task_id(), killed->task_id()); EXPECT_EQ(TASK_KILLED, killed->state()); // Now check the correctness of the memory pressure counters. Future<ResourceStatistics> usage = containerizer->usage(containerId); AWAIT_READY(usage); EXPECT_GE(usage->mem_low_pressure_counter(), usage->mem_medium_pressure_counter()); EXPECT_GE(usage->mem_medium_pressure_counter(), usage->mem_critical_pressure_counter()); Clock::resume(); driver.stop(); driver.join(); }
// This test checks that a scheduler exit shuts down the executor. TEST_F(HttpFaultToleranceTest, SchedulerExit) { master::Flags flags = CreateMasterFlags(); flags.authenticate_frameworks = false; Try<Owned<cluster::Master>> master = StartMaster(flags); ASSERT_SOME(master); auto scheduler = std::make_shared<v1::MockHTTPScheduler>(); auto executor = std::make_shared<v1::MockHTTPExecutor>(); ExecutorID executorId = DEFAULT_EXECUTOR_ID; TestContainerizer containerizer(executorId, executor); Owned<MasterDetector> detector = master.get()->createDetector(); Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), &containerizer); ASSERT_SOME(slave); Future<Nothing> connected; EXPECT_CALL(*scheduler, connected(_)) .WillOnce(FutureSatisfy(&connected)) .WillRepeatedly(Return()); // Ignore future invocations. ContentType contentType = ContentType::PROTOBUF; v1::scheduler::TestMesos schedulerLibrary( master.get()->pid, contentType, scheduler); AWAIT_READY(connected); Future<Event::Subscribed> subscribed; EXPECT_CALL(*scheduler, subscribed(_, _)) .WillOnce(FutureArg<1>(&subscribed)); EXPECT_CALL(*scheduler, heartbeat(_)) .WillRepeatedly(Return()); // Ignore heartbeats. Future<Event::Offers> offers; EXPECT_CALL(*scheduler, offers(_, _)) .WillOnce(FutureArg<1>(&offers)); { Call call; call.set_type(Call::SUBSCRIBE); Call::Subscribe* subscribe = call.mutable_subscribe(); subscribe->mutable_framework_info()->CopyFrom(v1::DEFAULT_FRAMEWORK_INFO); schedulerLibrary.send(call); } AWAIT_READY(subscribed); v1::FrameworkID frameworkId(subscribed->framework_id()); AWAIT_READY(offers); EXPECT_NE(0, offers->offers().size()); EXPECT_CALL(*executor, connected(_)) .WillOnce(v1::executor::SendSubscribe(frameworkId, evolve(executorId))); EXPECT_CALL(*executor, subscribed(_, _)); Future<Nothing> launch; EXPECT_CALL(*executor, launch(_, _)) .WillOnce(FutureSatisfy(&launch)); const v1::Offer& offer = offers->offers(0); v1::TaskInfo taskInfo = evolve(createTask(devolve(offer), "", DEFAULT_EXECUTOR_ID)); { Call call; call.mutable_framework_id()->CopyFrom(frameworkId); call.set_type(Call::ACCEPT); Call::Accept* accept = call.mutable_accept(); accept->add_offer_ids()->CopyFrom(offer.id()); v1::Offer::Operation* operation = accept->add_operations(); operation->set_type(v1::Offer::Operation::LAUNCH); operation->mutable_launch()->add_task_infos()->CopyFrom(taskInfo); schedulerLibrary.send(call); } AWAIT_READY(launch); EXPECT_CALL(*scheduler, disconnected(_)) .Times(AtMost(1)); Future<Nothing> shutdown; EXPECT_CALL(*executor, shutdown(_)) .WillOnce(FutureSatisfy(&shutdown)); { Call call; call.mutable_framework_id()->CopyFrom(frameworkId); call.set_type(Call::TEARDOWN); schedulerLibrary.send(call); } // Ensure that the executor receives a `Event::Shutdown` after the // scheduler exit. AWAIT_READY(shutdown); }
TEST_P(MemoryIsolatorTest, ROOT_MemUsage) { Try<Owned<cluster::Master>> master = StartMaster(); ASSERT_SOME(master); slave::Flags flags = CreateSlaveFlags(); flags.isolation = GetParam(); Fetcher fetcher(flags); Try<MesosContainerizer*> _containerizer = MesosContainerizer::create(flags, true, &fetcher); ASSERT_SOME(_containerizer); Owned<MesosContainerizer> containerizer(_containerizer.get()); Owned<MasterDetector> detector = master.get()->createDetector(); Try<Owned<cluster::Slave>> slave = StartSlave( detector.get(), containerizer.get()); ASSERT_SOME(slave); MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL); EXPECT_CALL(sched, registered(&driver, _, _)); Future<vector<Offer>> offers; EXPECT_CALL(sched, resourceOffers(&driver, _)) .WillOnce(FutureArg<1>(&offers)) .WillRepeatedly(Return()); // Ignore subsequent offers. driver.start(); AWAIT_READY(offers); ASSERT_FALSE(offers->empty()); TaskInfo task = createTask(offers.get()[0], "sleep 120"); Future<TaskStatus> statusRunning; EXPECT_CALL(sched, statusUpdate(&driver, _)) .WillOnce(FutureArg<1>(&statusRunning)); driver.launchTasks(offers.get()[0].id(), {task}); AWAIT_READY(statusRunning); EXPECT_EQ(TASK_RUNNING, statusRunning->state()); Future<hashset<ContainerID>> containers = containerizer->containers(); AWAIT_READY(containers); ASSERT_EQ(1u, containers->size()); ContainerID containerId = *(containers->begin()); Future<ResourceStatistics> usage = containerizer->usage(containerId); AWAIT_READY(usage); // TODO(jieyu): Consider using a program that predictably increases // RSS so that we can set more meaningful expectation here. EXPECT_LT(0u, usage->mem_rss_bytes()); driver.stop(); driver.join(); }
// This test ensures that the HTTP command executor can self terminate // after it gets the ACK for the terminal status update from agent. TEST_F_TEMP_DISABLED_ON_WINDOWS(HTTPCommandExecutorTest, TerminateWithACK) { Try<Owned<cluster::Master>> master = StartMaster(); ASSERT_SOME(master); slave::Flags flags = CreateSlaveFlags(); flags.http_command_executor = true; Fetcher fetcher; Try<MesosContainerizer*> _containerizer = MesosContainerizer::create(flags, false, &fetcher); CHECK_SOME(_containerizer); Owned<MesosContainerizer> containerizer(_containerizer.get()); StandaloneMasterDetector detector(master.get()->pid); MockSlave slave(flags, &detector, containerizer.get()); spawn(slave); MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL); EXPECT_CALL(sched, registered(&driver, _, _)); Future<vector<Offer>> offers; EXPECT_CALL(sched, resourceOffers(&driver, _)) .WillOnce(FutureArg<1>(&offers)) .WillRepeatedly(Return()); // Ignore subsequent offers. driver.start(); AWAIT_READY(offers); EXPECT_EQ(1u, offers->size()); // Launch a short lived task. TaskInfo task = createTask( offers->front().slave_id(), offers->front().resources(), "sleep 1"); Future<TaskStatus> statusRunning; Future<TaskStatus> statusFinished; EXPECT_CALL(sched, statusUpdate(_, _)) .WillOnce(FutureArg<1>(&statusRunning)) .WillOnce(FutureArg<1>(&statusFinished)); Future<Future<Option<ContainerTermination>>> termination; EXPECT_CALL(slave, executorTerminated(_, _, _)) .WillOnce(FutureArg<2>(&termination)); driver.launchTasks(offers->front().id(), {task}); // Scheduler should first receive TASK_RUNNING followed by TASK_FINISHED. AWAIT_READY(statusRunning); EXPECT_EQ(TASK_RUNNING, statusRunning->state()); AWAIT_READY(statusFinished); EXPECT_EQ(TASK_FINISHED, statusFinished->state()); // The executor should self terminate with 0 as exit status once // it gets the ACK for the terminal status update from agent. AWAIT_READY(termination); ASSERT_TRUE(termination.get().isReady()); EXPECT_EQ(0, termination.get().get().get().status()); driver.stop(); driver.join(); terminate(slave); wait(slave); }
// This test does not set any Accept header for the subscribe call. // The default response media type should be "application/json" in // this case. TEST_P(ExecutorHttpApiTest, NoAcceptHeader) { Try<Owned<cluster::Master>> master = StartMaster(); ASSERT_SOME(master); ExecutorID executorId = DEFAULT_EXECUTOR_ID; MockExecutor exec(executorId); TestContainerizer containerizer(&exec); Owned<MasterDetector> detector = master.get()->createDetector(); Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), &containerizer); ASSERT_SOME(slave); MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL); Future<FrameworkID> frameworkId; EXPECT_CALL(sched, registered(&driver, _, _)) .WillOnce(FutureArg<1>(&frameworkId)); Future<vector<Offer>> offers; EXPECT_CALL(sched, resourceOffers(&driver, _)) .WillOnce(FutureArg<1>(&offers)); Future<Nothing> statusUpdate; EXPECT_CALL(sched, statusUpdate(&driver, _)) .WillOnce(FutureSatisfy(&statusUpdate)); driver.start(); AWAIT_READY(frameworkId); AWAIT_READY(offers); ASSERT_EQ(1u, offers.get().size()); EXPECT_CALL(exec, registered(_, _, _, _)) .Times(1); EXPECT_CALL(exec, launchTask(_, _)) .WillOnce(SendStatusUpdateFromTask(TASK_RUNNING)); TaskInfo taskInfo = createTask(offers.get()[0], "", executorId); driver.launchTasks(offers.get()[0].id(), {taskInfo}); // Wait until status update is received on the scheduler before sending // an executor subscribe request. AWAIT_READY(statusUpdate); // Only subscribe needs to 'Accept' JSON or protobuf. Call call; call.mutable_framework_id()->CopyFrom(evolve(frameworkId.get())); call.mutable_executor_id()->CopyFrom(evolve(executorId)); call.set_type(Call::SUBSCRIBE); call.mutable_subscribe(); // Retrieve the parameter passed as content type to this test. const ContentType contentType = GetParam(); // No 'Accept' header leads to all media types considered // acceptable. JSON will be chosen by default. process::http::Headers headers; Future<Response> response = process::http::streaming::post( slave.get()->pid, "api/v1/executor", headers, serialize(contentType, call), stringify(contentType)); AWAIT_EXPECT_RESPONSE_STATUS_EQ(OK().status, response); AWAIT_EXPECT_RESPONSE_HEADER_EQ(APPLICATION_JSON, "Content-Type", response); driver.stop(); driver.join(); }
// The purpose of this test is to ensure that when slaves are removed // from the master, and then attempt to send exited executor messages, // we send a ShutdownMessage to the slave. Why? Because during a // network partition, the master will remove a partitioned slave, thus // sending its tasks to LOST. At this point, when the partition is // removed, the slave may attempt to send exited executor messages if // it was unaware that the master removed it. We've already // notified frameworks that the tasks under the executors were LOST, // so we have to have the slave shut down. TEST_F(PartitionTest, PartitionedSlaveExitedExecutor) { master::Flags masterFlags = CreateMasterFlags(); Try<Owned<cluster::Master>> master = StartMaster(masterFlags); ASSERT_SOME(master); // Allow the master to PING the slave, but drop all PONG messages // from the slave. Note that we don't match on the master / slave // PIDs because it's actually the SlaveObserver Process that sends // the pings. Future<Message> ping = FUTURE_MESSAGE( Eq(PingSlaveMessage().GetTypeName()), _, _); DROP_PROTOBUFS(PongSlaveMessage(), _, _); MockExecutor exec(DEFAULT_EXECUTOR_ID); TestContainerizer containerizer(&exec); Owned<MasterDetector> detector = master.get()->createDetector(); Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), &containerizer); ASSERT_SOME(slave); MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL); Future<FrameworkID> frameworkId; EXPECT_CALL(sched, registered(&driver, _, _)) .WillOnce(FutureArg<1>(&frameworkId));\ Future<vector<Offer>> offers; EXPECT_CALL(sched, resourceOffers(&driver, _)) .WillOnce(FutureArg<1>(&offers)) .WillRepeatedly(Return()); driver.start(); AWAIT_READY(frameworkId); AWAIT_READY(offers); ASSERT_NE(0u, offers.get().size()); // Launch a task. This allows us to have the slave send an // ExitedExecutorMessage. TaskID taskId; taskId.set_value("1"); TaskInfo task; task.set_name(""); task.mutable_task_id()->MergeFrom(taskId); task.mutable_slave_id()->MergeFrom(offers.get()[0].slave_id()); task.mutable_resources()->MergeFrom(offers.get()[0].resources()); task.mutable_executor()->MergeFrom(DEFAULT_EXECUTOR_INFO); task.mutable_executor()->mutable_command()->set_value("sleep 60"); // Set up the expectations for launching the task. EXPECT_CALL(exec, registered(_, _, _, _)); EXPECT_CALL(exec, launchTask(_, _)) .WillOnce(SendStatusUpdateFromTask(TASK_RUNNING)); // Drop all the status updates from the slave, so that we can // ensure the ExitedExecutorMessage is what triggers the slave // shutdown. DROP_PROTOBUFS(StatusUpdateMessage(), _, master.get()->pid); driver.launchTasks(offers.get()[0].id(), {task}); // Drop the first shutdown message from the master (simulated // partition) and allow the second shutdown message to pass when // triggered by the ExitedExecutorMessage. Future<ShutdownMessage> shutdownMessage = DROP_PROTOBUF(ShutdownMessage(), _, slave.get()->pid); Future<TaskStatus> lostStatus; EXPECT_CALL(sched, statusUpdate(&driver, _)) .WillOnce(FutureArg<1>(&lostStatus)); Future<Nothing> slaveLost; EXPECT_CALL(sched, slaveLost(&driver, _)) .WillOnce(FutureSatisfy(&slaveLost)); Clock::pause(); // Now, induce a partition of the slave by having the master // timeout the slave. size_t pings = 0; while (true) { AWAIT_READY(ping); pings++; if (pings == masterFlags.max_slave_ping_timeouts) { break; } ping = FUTURE_MESSAGE(Eq(PingSlaveMessage().GetTypeName()), _, _); Clock::advance(masterFlags.slave_ping_timeout); Clock::settle(); } Clock::advance(masterFlags.slave_ping_timeout); Clock::settle(); // The master will have notified the framework of the lost task. AWAIT_READY(lostStatus); EXPECT_EQ(TASK_LOST, lostStatus.get().state()); // Wait for the master to attempt to shut down the slave. AWAIT_READY(shutdownMessage); // The master will notify the framework that the slave was lost. AWAIT_READY(slaveLost); shutdownMessage = FUTURE_PROTOBUF(ShutdownMessage(), _, slave.get()->pid); // Induce an ExitedExecutorMessage from the slave. containerizer.destroy( frameworkId.get(), DEFAULT_EXECUTOR_INFO.executor_id()); // Upon receiving the message, the master will shutdown the slave. AWAIT_READY(shutdownMessage); Clock::resume(); driver.stop(); driver.join(); }
// The purpose of this test is to ensure that when slaves are removed // from the master, and then attempt to re-register, we deny the // re-registration by sending a ShutdownMessage to the slave. // Why? Because during a network partition, the master will remove a // partitioned slave, thus sending its tasks to LOST. At this point, // when the partition is removed, the slave will attempt to // re-register with its running tasks. We've already notified // frameworks that these tasks were LOST, so we have to have the slave // slave shut down. TEST_F(PartitionTest, PartitionedSlaveReregistration) { master::Flags masterFlags = CreateMasterFlags(); Try<Owned<cluster::Master>> master = StartMaster(masterFlags); ASSERT_SOME(master); // Allow the master to PING the slave, but drop all PONG messages // from the slave. Note that we don't match on the master / slave // PIDs because it's actually the SlaveObserver Process that sends // the pings. Future<Message> ping = FUTURE_MESSAGE( Eq(PingSlaveMessage().GetTypeName()), _, _); DROP_PROTOBUFS(PongSlaveMessage(), _, _); MockExecutor exec(DEFAULT_EXECUTOR_ID); TestContainerizer containerizer(&exec); StandaloneMasterDetector detector(master.get()->pid); Try<Owned<cluster::Slave>> slave = StartSlave(&detector, &containerizer); ASSERT_SOME(slave); MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL); EXPECT_CALL(sched, registered(&driver, _, _)); Future<vector<Offer>> offers; EXPECT_CALL(sched, resourceOffers(&driver, _)) .WillOnce(FutureArg<1>(&offers)) .WillRepeatedly(Return()); driver.start(); AWAIT_READY(offers); ASSERT_NE(0u, offers.get().size()); // Launch a task. This is to ensure the task is killed by the slave, // during shutdown. TaskID taskId; taskId.set_value("1"); TaskInfo task; task.set_name(""); task.mutable_task_id()->MergeFrom(taskId); task.mutable_slave_id()->MergeFrom(offers.get()[0].slave_id()); task.mutable_resources()->MergeFrom(offers.get()[0].resources()); task.mutable_executor()->MergeFrom(DEFAULT_EXECUTOR_INFO); task.mutable_executor()->mutable_command()->set_value("sleep 60"); // Set up the expectations for launching the task. EXPECT_CALL(exec, registered(_, _, _, _)); EXPECT_CALL(exec, launchTask(_, _)) .WillOnce(SendStatusUpdateFromTask(TASK_RUNNING)); Future<TaskStatus> runningStatus; EXPECT_CALL(sched, statusUpdate(&driver, _)) .WillOnce(FutureArg<1>(&runningStatus)); Future<Nothing> statusUpdateAck = FUTURE_DISPATCH( slave.get()->pid, &Slave::_statusUpdateAcknowledgement); driver.launchTasks(offers.get()[0].id(), {task}); AWAIT_READY(runningStatus); EXPECT_EQ(TASK_RUNNING, runningStatus.get().state()); // Wait for the slave to have handled the acknowledgment prior // to pausing the clock. AWAIT_READY(statusUpdateAck); // Drop the first shutdown message from the master (simulated // partition), allow the second shutdown message to pass when // the slave re-registers. Future<ShutdownMessage> shutdownMessage = DROP_PROTOBUF(ShutdownMessage(), _, slave.get()->pid); Future<TaskStatus> lostStatus; EXPECT_CALL(sched, statusUpdate(&driver, _)) .WillOnce(FutureArg<1>(&lostStatus)); Future<Nothing> slaveLost; EXPECT_CALL(sched, slaveLost(&driver, _)) .WillOnce(FutureSatisfy(&slaveLost)); Clock::pause(); // Now, induce a partition of the slave by having the master // timeout the slave. size_t pings = 0; while (true) { AWAIT_READY(ping); pings++; if (pings == masterFlags.max_slave_ping_timeouts) { break; } ping = FUTURE_MESSAGE(Eq(PingSlaveMessage().GetTypeName()), _, _); Clock::advance(masterFlags.slave_ping_timeout); Clock::settle(); } Clock::advance(masterFlags.slave_ping_timeout); Clock::settle(); // The master will have notified the framework of the lost task. AWAIT_READY(lostStatus); EXPECT_EQ(TASK_LOST, lostStatus.get().state()); // Wait for the master to attempt to shut down the slave. AWAIT_READY(shutdownMessage); // The master will notify the framework that the slave was lost. AWAIT_READY(slaveLost); Clock::resume(); // We now complete the partition on the slave side as well. This // is done by simulating a master loss event which would normally // occur during a network partition. detector.appoint(None()); Future<Nothing> shutdown; EXPECT_CALL(exec, shutdown(_)) .WillOnce(FutureSatisfy(&shutdown)); shutdownMessage = FUTURE_PROTOBUF(ShutdownMessage(), _, slave.get()->pid); // Have the slave re-register with the master. detector.appoint(master.get()->pid); // Upon re-registration, the master will shutdown the slave. // The slave will then shut down the executor. AWAIT_READY(shutdownMessage); AWAIT_READY(shutdown); driver.stop(); driver.join(); }
// This test verifies that the master reconciles tasks that are // missing from a re-registering slave. In this case, we trigger // a race between the slave re-registration message and the launch // message. There should be no TASK_LOST. // This was motivated by MESOS-1696. TEST_F(MasterSlaveReconciliationTest, ReconcileRace) { Try<Owned<cluster::Master>> master = StartMaster(); ASSERT_SOME(master); MockExecutor exec(DEFAULT_EXECUTOR_ID); TestContainerizer containerizer(&exec); StandaloneMasterDetector detector(master.get()->pid); Future<SlaveRegisteredMessage> slaveRegisteredMessage = FUTURE_PROTOBUF(SlaveRegisteredMessage(), master.get()->pid, _); Try<Owned<cluster::Slave>> slave = StartSlave(&detector, &containerizer); ASSERT_SOME(slave); AWAIT_READY(slaveRegisteredMessage); MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL); EXPECT_CALL(sched, registered(&driver, _, _)); Future<vector<Offer> > offers; EXPECT_CALL(sched, resourceOffers(&driver, _)) .WillOnce(FutureArg<1>(&offers)) .WillRepeatedly(Return()); // Ignore subsequent offers. driver.start(); // Since the agent may have retried registration, we want to // ensure that any duplicate registrations are flushed before // we appoint the master again. Otherwise, the agent may // receive a stale registration message. Clock::pause(); Clock::settle(); Clock::resume(); // Trigger a re-registration of the slave and capture the message // so that we can spoof a race with a launch task message. DROP_PROTOBUFS(ReregisterSlaveMessage(), slave.get()->pid, master.get()->pid); Future<ReregisterSlaveMessage> reregisterSlaveMessage = DROP_PROTOBUF( ReregisterSlaveMessage(), slave.get()->pid, master.get()->pid); detector.appoint(master.get()->pid); AWAIT_READY(reregisterSlaveMessage); AWAIT_READY(offers); EXPECT_NE(0u, offers.get().size()); TaskInfo task; task.set_name("test task"); task.mutable_task_id()->set_value("1"); task.mutable_slave_id()->MergeFrom(offers.get()[0].slave_id()); task.mutable_resources()->MergeFrom(offers.get()[0].resources()); task.mutable_executor()->MergeFrom(DEFAULT_EXECUTOR_INFO); ExecutorDriver* executorDriver; EXPECT_CALL(exec, registered(_, _, _, _)) .WillOnce(SaveArg<0>(&executorDriver)); // Leave the task in TASK_STAGING. Future<Nothing> launchTask; EXPECT_CALL(exec, launchTask(_, _)) .WillOnce(FutureSatisfy(&launchTask)); EXPECT_CALL(sched, statusUpdate(&driver, _)) .Times(0); driver.launchTasks(offers.get()[0].id(), {task}); AWAIT_READY(launchTask); // Send the stale re-registration message, which does not contain // the task we just launched. This will trigger a reconciliation // by the master. Future<SlaveReregisteredMessage> slaveReregisteredMessage = FUTURE_PROTOBUF(SlaveReregisteredMessage(), _, _); // Prevent this from being dropped per the DROP_PROTOBUFS above. FUTURE_PROTOBUF( ReregisterSlaveMessage(), slave.get()->pid, master.get()->pid); process::post( slave.get()->pid, master.get()->pid, reregisterSlaveMessage.get()); AWAIT_READY(slaveReregisteredMessage); // Neither the master nor the slave should send a TASK_LOST // as part of the reconciliation. We check this by calling // Clock::settle() to flush all pending events. Clock::pause(); Clock::settle(); Clock::resume(); // Now send TASK_FINISHED and make sure it's the only message // received by the scheduler. Future<TaskStatus> status; EXPECT_CALL(sched, statusUpdate(&driver, _)) .WillOnce(FutureArg<1>(&status)); TaskStatus taskStatus; taskStatus.mutable_task_id()->CopyFrom(task.task_id()); taskStatus.set_state(TASK_FINISHED); executorDriver->sendStatusUpdate(taskStatus); AWAIT_READY(status); ASSERT_EQ(TASK_FINISHED, status.get().state()); EXPECT_CALL(exec, shutdown(_)) .Times(AtMost(1)); driver.stop(); driver.join(); }
// This test verifies that when the slave re-registers, the master // does not send TASK_LOST update for a task that has reached terminal // state but is waiting for an acknowledgement. TEST_F(MasterSlaveReconciliationTest, SlaveReregisterTerminalTask) { Try<Owned<cluster::Master>> master = StartMaster(); ASSERT_SOME(master); MockExecutor exec(DEFAULT_EXECUTOR_ID); TestContainerizer containerizer(&exec); StandaloneMasterDetector detector(master.get()->pid); Try<Owned<cluster::Slave>> slave = StartSlave(&detector, &containerizer); ASSERT_SOME(slave); MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL); EXPECT_CALL(sched, registered(&driver, _, _)); Future<vector<Offer> > offers; EXPECT_CALL(sched, resourceOffers(&driver, _)) .WillOnce(FutureArg<1>(&offers)) .WillRepeatedly(Return()); // Ignore subsequent offers. driver.start(); AWAIT_READY(offers); EXPECT_NE(0u, offers.get().size()); TaskInfo task; task.set_name("test task"); task.mutable_task_id()->set_value("1"); task.mutable_slave_id()->MergeFrom(offers.get()[0].slave_id()); task.mutable_resources()->MergeFrom(offers.get()[0].resources()); task.mutable_executor()->MergeFrom(DEFAULT_EXECUTOR_INFO); EXPECT_CALL(exec, registered(_, _, _, _)); // Send a terminal update right away. EXPECT_CALL(exec, launchTask(_, _)) .WillOnce(SendStatusUpdateFromTask(TASK_FINISHED)); // Drop the status update from slave to the master, so that // the slave has a pending terminal update when it re-registers. DROP_PROTOBUF(StatusUpdateMessage(), _, master.get()->pid); Future<Nothing> _statusUpdate = FUTURE_DISPATCH(_, &Slave::_statusUpdate); Future<TaskStatus> status; EXPECT_CALL(sched, statusUpdate(&driver, _)) .WillOnce(FutureArg<1>(&status)) .WillRepeatedly(Return()); // Ignore retried update due to update framework. driver.launchTasks(offers.get()[0].id(), {task}); AWAIT_READY(_statusUpdate); Future<SlaveReregisteredMessage> slaveReregisteredMessage = FUTURE_PROTOBUF(SlaveReregisteredMessage(), _, _); // Simulate a spurious master change event (e.g., due to ZooKeeper // expiration) at the slave to force re-registration. detector.appoint(master.get()->pid); AWAIT_READY(slaveReregisteredMessage); // The master should not send a TASK_LOST after the slave // re-registers. We check this by calling Clock::settle() so that // the only update the scheduler receives is the retried // TASK_FINISHED update. // NOTE: The status update manager resends the status update when // it detects a new master. Clock::pause(); Clock::settle(); AWAIT_READY(status); ASSERT_EQ(TASK_FINISHED, status.get().state()); EXPECT_CALL(exec, shutdown(_)) .Times(AtMost(1)); driver.stop(); driver.join(); }
// Ensures that when a scheduler enables explicit acknowledgements // on the driver, there are no implicit acknowledgements sent, and // the call to 'acknowledgeStatusUpdate' sends the ack to the master. TEST_F(MesosSchedulerDriverTest, ExplicitAcknowledgements) { Try<PID<Master>> master = StartMaster(); ASSERT_SOME(master); MockExecutor exec(DEFAULT_EXECUTOR_ID); TestContainerizer containerizer(&exec); Try<PID<Slave>> slave = StartSlave(&containerizer); ASSERT_SOME(slave); MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get(), false, DEFAULT_CREDENTIAL); EXPECT_CALL(sched, registered(&driver, _, _)); EXPECT_CALL(sched, resourceOffers(&driver, _)) .WillOnce(LaunchTasks(DEFAULT_EXECUTOR_INFO, 1, 1, 16, "*")) .WillRepeatedly(Return()); // Ignore subsequent offers. Future<TaskStatus> status; EXPECT_CALL(sched, statusUpdate(&driver, _)) .WillOnce(FutureArg<1>(&status)); // Ensure no status update acknowledgements are sent from the driver // to the master until the explicit acknowledgement is sent. EXPECT_NO_FUTURE_CALLS( mesos::scheduler::Call(), mesos::scheduler::Call::ACKNOWLEDGE, _ , master.get()); EXPECT_CALL(exec, registered(_, _, _, _)); EXPECT_CALL(exec, launchTask(_, _)) .WillOnce(SendStatusUpdateFromTask(TASK_RUNNING)); EXPECT_CALL(exec, shutdown(_)) .Times(AtMost(1)); driver.start(); AWAIT_READY(status); // Settle the clock to ensure driver finishes processing the status // update, we want to ensure that no implicit acknowledgement gets // sent. Clock::pause(); Clock::settle(); // Now send the acknowledgement. Future<mesos::scheduler::Call> acknowledgement = FUTURE_CALL( mesos::scheduler::Call(), mesos::scheduler::Call::ACKNOWLEDGE, _, master.get()); driver.acknowledgeStatusUpdate(status.get()); AWAIT_READY(acknowledgement); driver.stop(); driver.join(); Shutdown(); }
// This test verifies that when the slave re-registers, we correctly // send the information about actively running frameworks. TEST_F(MasterSlaveReconciliationTest, SlaveReregisterFrameworks) { Try<Owned<cluster::Master>> master = StartMaster(); ASSERT_SOME(master); MockExecutor exec(DEFAULT_EXECUTOR_ID); TestContainerizer containerizer(&exec); StandaloneMasterDetector detector(master.get()->pid); Try<Owned<cluster::Slave>> slave = StartSlave(&detector, &containerizer); ASSERT_SOME(slave); MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL); EXPECT_CALL(sched, registered(&driver, _, _)); Future<vector<Offer> > offers; EXPECT_CALL(sched, resourceOffers(&driver, _)) .WillOnce(FutureArg<1>(&offers)) .WillRepeatedly(Return()); // Ignore subsequent offers. driver.start(); AWAIT_READY(offers); EXPECT_NE(0u, offers.get().size()); TaskInfo task; task.set_name("test task"); task.mutable_task_id()->set_value("1"); task.mutable_slave_id()->MergeFrom(offers.get()[0].slave_id()); task.mutable_resources()->MergeFrom(offers.get()[0].resources()); task.mutable_executor()->MergeFrom(DEFAULT_EXECUTOR_INFO); EXPECT_CALL(exec, registered(_, _, _, _)); // Send an update right away. EXPECT_CALL(exec, launchTask(_, _)) .WillOnce(SendStatusUpdateFromTask(TASK_RUNNING)); Future<Nothing> _statusUpdate = FUTURE_DISPATCH(_, &Slave::_statusUpdate); Future<TaskStatus> status; EXPECT_CALL(sched, statusUpdate(&driver, _)) .WillOnce(FutureArg<1>(&status)) .WillRepeatedly(Return()); // Ignore retried update due to update framework. driver.launchTasks(offers.get()[0].id(), {task}); AWAIT_READY(_statusUpdate); Future<ReregisterSlaveMessage> reregisterSlave = FUTURE_PROTOBUF(ReregisterSlaveMessage(), _, _); // Simulate a spurious master change event (e.g., due to ZooKeeper // expiration) at the slave to force re-registration. detector.appoint(master.get()->pid); // Expect to receive the 'ReregisterSlaveMessage' containing the // active frameworks. AWAIT_READY(reregisterSlave); EXPECT_EQ(1u, reregisterSlave.get().frameworks().size()); Clock::pause(); Clock::settle(); AWAIT_READY(status); EXPECT_CALL(exec, shutdown(_)) .Times(AtMost(1)); driver.stop(); driver.join(); }
// This test verifies that a re-registering slave sends the terminal // unacknowledged tasks for a terminal executor. This is required // for the master to correctly reconcile its view with the slave's // view of tasks. This test drops a terminal update to the master // and then forces the slave to re-register. TEST_F(MasterSlaveReconciliationTest, SlaveReregisterTerminatedExecutor) { Try<Owned<cluster::Master>> master = StartMaster(); ASSERT_SOME(master); MockExecutor exec(DEFAULT_EXECUTOR_ID); TestContainerizer containerizer(&exec); StandaloneMasterDetector detector(master.get()->pid); Try<Owned<cluster::Slave>> slave = StartSlave(&detector, &containerizer); ASSERT_SOME(slave); MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL); Future<FrameworkID> frameworkId; EXPECT_CALL(sched, registered(&driver, _, _)) .WillOnce(FutureArg<1>(&frameworkId)); EXPECT_CALL(sched, resourceOffers(&driver, _)) .WillOnce(LaunchTasks(DEFAULT_EXECUTOR_INFO, 1, 1, 512, "*")) .WillRepeatedly(Return()); // Ignore subsequent offers. ExecutorDriver* execDriver; EXPECT_CALL(exec, registered(_, _, _, _)) .WillOnce(SaveArg<0>(&execDriver)); EXPECT_CALL(exec, launchTask(_, _)) .WillOnce(SendStatusUpdateFromTask(TASK_RUNNING)); Future<TaskStatus> status; EXPECT_CALL(sched, statusUpdate(&driver, _)) .WillOnce(FutureArg<1>(&status)); Future<StatusUpdateAcknowledgementMessage> statusUpdateAcknowledgementMessage = FUTURE_PROTOBUF( StatusUpdateAcknowledgementMessage(), master.get()->pid, slave.get()->pid); driver.start(); AWAIT_READY(status); EXPECT_EQ(TASK_RUNNING, status.get().state()); // Make sure the acknowledgement reaches the slave. AWAIT_READY(statusUpdateAcknowledgementMessage); // Drop the TASK_FINISHED status update sent to the master. Future<StatusUpdateMessage> statusUpdateMessage = DROP_PROTOBUF(StatusUpdateMessage(), _, master.get()->pid); Future<ExitedExecutorMessage> executorExitedMessage = FUTURE_PROTOBUF(ExitedExecutorMessage(), _, _); TaskStatus finishedStatus; finishedStatus = status.get(); finishedStatus.set_state(TASK_FINISHED); execDriver->sendStatusUpdate(finishedStatus); // Ensure the update was sent. AWAIT_READY(statusUpdateMessage); EXPECT_CALL(sched, executorLost(&driver, DEFAULT_EXECUTOR_ID, _, _)); // Now kill the executor. containerizer.destroy(frameworkId.get(), DEFAULT_EXECUTOR_ID); Future<TaskStatus> status2; EXPECT_CALL(sched, statusUpdate(&driver, _)) .WillOnce(FutureArg<1>(&status2)); // We drop the 'UpdateFrameworkMessage' from the master to slave to // stop the status update manager from retrying the update that was // already sent due to the new master detection. DROP_PROTOBUFS(UpdateFrameworkMessage(), _, _); detector.appoint(master.get()->pid); AWAIT_READY(status2); EXPECT_EQ(TASK_FINISHED, status2.get().state()); driver.stop(); driver.join(); }
// Ensures that the driver can handle an OFFERS event. // Note that this includes the ability to bypass the // master when sending framework messages. TEST_F(SchedulerDriverEventTest, Offers) { Try<Owned<cluster::Master>> master = StartMaster(); ASSERT_SOME(master); MockScheduler sched; MesosSchedulerDriver schedDriver( &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL); EXPECT_CALL(sched, registered(&schedDriver, _, _)); Future<Message> frameworkRegisteredMessage = FUTURE_MESSAGE(Eq(FrameworkRegisteredMessage().GetTypeName()), _, _); schedDriver.start(); AWAIT_READY(frameworkRegisteredMessage); UPID frameworkPid = frameworkRegisteredMessage.get().to; // Start a slave and capture the offers. Future<ResourceOffersMessage> resourceOffersMessage = DROP_PROTOBUF(ResourceOffersMessage(), _, _); MockExecutor exec(DEFAULT_EXECUTOR_ID); TestContainerizer containerizer(&exec); Owned<MasterDetector> detector = master.get()->createDetector(); Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), &containerizer); ASSERT_SOME(slave); AWAIT_READY(resourceOffersMessage); google::protobuf::RepeatedPtrField<Offer> offers = resourceOffersMessage.get().offers(); ASSERT_EQ(1, offers.size()); // Ignore future offer messages. DROP_PROTOBUFS(ResourceOffersMessage(), _, _); // Send the offers event and expect a 'resourceOffers' call. Event event; event.set_type(Event::OFFERS); event.mutable_offers()->mutable_offers()->CopyFrom(offers); Future<Nothing> resourceOffers; EXPECT_CALL(sched, resourceOffers(&schedDriver, _)) .WillOnce(FutureSatisfy(&resourceOffers)); process::post(master.get()->pid, frameworkPid, event); AWAIT_READY(resourceOffers); // To test that the framework -> executor messages are // sent directly to the slave, launch a task and send // the executor a message. EXPECT_CALL(exec, registered(_, _, _, _)); EXPECT_CALL(exec, launchTask(_, _)) .WillOnce(SendStatusUpdateFromTask(TASK_RUNNING)); Future<TaskStatus> status; EXPECT_CALL(sched, statusUpdate(&schedDriver, _)) .WillOnce(FutureArg<1>(&status)); TaskInfo task = createTask(offers.Get(0), "", DEFAULT_EXECUTOR_ID); schedDriver.launchTasks(offers.Get(0).id(), {task}); AWAIT_READY(status); EXPECT_EQ(TASK_RUNNING, status.get().state()); // This message should skip the master! Future<FrameworkToExecutorMessage> frameworkToExecutorMessage = FUTURE_PROTOBUF( FrameworkToExecutorMessage(), frameworkPid, slave.get()->pid); Future<string> data; EXPECT_CALL(exec, frameworkMessage(_, _)) .WillOnce(FutureArg<1>(&data)); schedDriver.sendFrameworkMessage( DEFAULT_EXECUTOR_ID, offers.Get(0).slave_id(), "hello"); AWAIT_READY(frameworkToExecutorMessage); AWAIT_EXPECT_EQ("hello", data); EXPECT_CALL(exec, shutdown(_)) .Times(AtMost(1)); schedDriver.stop(); schedDriver.join(); }
TEST_P(CpuIsolatorTest, ROOT_UserCpuUsage) { Try<Owned<cluster::Master>> master = StartMaster(); ASSERT_SOME(master); slave::Flags flags = CreateSlaveFlags(); flags.isolation = GetParam(); Fetcher fetcher(flags); Try<MesosContainerizer*> _containerizer = MesosContainerizer::create(flags, true, &fetcher); ASSERT_SOME(_containerizer); Owned<MesosContainerizer> containerizer(_containerizer.get()); Owned<MasterDetector> detector = master.get()->createDetector(); Try<Owned<cluster::Slave>> slave = StartSlave( detector.get(), containerizer.get()); ASSERT_SOME(slave); MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL); EXPECT_CALL(sched, registered(&driver, _, _)); Future<vector<Offer>> offers; EXPECT_CALL(sched, resourceOffers(&driver, _)) .WillOnce(FutureArg<1>(&offers)) .WillRepeatedly(Return()); // Ignore subsequent offers. driver.start(); AWAIT_READY(offers); ASSERT_FALSE(offers->empty()); // Max out a single core in userspace. This will run for at most one // second. TaskInfo task = createTask( offers.get()[0], "while true ; do true ; done & sleep 60"); Future<TaskStatus> statusRunning; EXPECT_CALL(sched, statusUpdate(&driver, _)) .WillOnce(FutureArg<1>(&statusRunning)); driver.launchTasks(offers.get()[0].id(), {task}); AWAIT_READY(statusRunning); EXPECT_EQ(TASK_RUNNING, statusRunning->state()); Future<hashset<ContainerID>> containers = containerizer->containers(); AWAIT_READY(containers); ASSERT_EQ(1u, containers->size()); ContainerID containerId = *(containers->begin()); // Wait up to 1 second for the child process to induce 1/8 of a // second of user cpu time. ResourceStatistics statistics; Duration waited = Duration::zero(); do { Future<ResourceStatistics> usage = containerizer->usage(containerId); AWAIT_READY(usage); statistics = usage.get(); // If we meet our usage expectations, we're done! if (statistics.cpus_user_time_secs() >= 0.125) { break; } os::sleep(Milliseconds(200)); waited += Milliseconds(200); } while (waited < Seconds(1)); EXPECT_LE(0.125, statistics.cpus_user_time_secs()); driver.stop(); driver.join(); }
// This test ensures that destroy can be called while in the // launch loop. The composing containerizer still calls the // underlying containerizer's destroy (because it's not sure // if the containerizer can handle the type of container being // launched). If the launch is not supported by the 1st containerizer, // the composing containerizer should stop the launch loop and // set the value of destroy future to true. TEST_F(ComposingContainerizerTest, DestroyDuringUnsupportedLaunchLoop) { vector<Containerizer*> containerizers; MockContainerizer* mockContainerizer1 = new MockContainerizer(); MockContainerizer* mockContainerizer2 = new MockContainerizer(); containerizers.push_back(mockContainerizer1); containerizers.push_back(mockContainerizer2); ComposingContainerizer containerizer(containerizers); ContainerID containerId; containerId.set_value("container"); TaskInfo taskInfo; ExecutorInfo executorInfo; SlaveID slaveId; std::map<std::string, std::string> environment; Promise<bool> launchPromise; EXPECT_CALL(*mockContainerizer1, launch(_, _, _, _, _, _, _, _)) .WillOnce(Return(launchPromise.future())); Future<Nothing> destroy; Promise<bool> destroyPromise; EXPECT_CALL(*mockContainerizer1, destroy(_)) .WillOnce(DoAll(FutureSatisfy(&destroy), Return(destroyPromise.future()))); Future<bool> launched = containerizer.launch( containerId, taskInfo, executorInfo, "dir", "user", slaveId, environment, false); Resources resources = Resources::parse("cpus:1;mem:256").get(); EXPECT_TRUE(launched.isPending()); Future<bool> destroyed = containerizer.destroy(containerId); EXPECT_CALL(*mockContainerizer2, launch(_, _, _, _, _, _, _, _)) .Times(0); // We make sure the destroy is being called on the first containerizer. // The second containerizer shouldn't be called as well since the // container is already destroyed. AWAIT_READY(destroy); launchPromise.set(false); destroyPromise.set(false); // `launched` should be a failure and `destroyed` should be true // because the launch was stopped from being tried on the 2nd // containerizer because of the destroy. AWAIT_FAILED(launched); AWAIT_EXPECT_EQ(true, destroyed); }
// This test verifies that sandbox path volume allows two containers // nested under the same parent container to share data. // TODO(jieyu): Parameterize this test to test both linux and posix // launcher and filesystem isolator. TEST_F(VolumeSandboxPathIsolatorTest, SharedVolume) { slave::Flags flags = CreateSlaveFlags(); flags.isolation = "volume/sandbox_path"; Fetcher fetcher; Try<MesosContainerizer*> create = MesosContainerizer::create( flags, true, &fetcher); ASSERT_SOME(create); Owned<MesosContainerizer> containerizer(create.get()); SlaveState state; state.id = SlaveID(); AWAIT_READY(containerizer->recover(state)); ContainerID containerId; containerId.set_value(UUID::random().toString()); ExecutorInfo executor = createExecutorInfo("executor", "sleep 99", "cpus:1"); Try<string> directory = environment->mkdtemp(); ASSERT_SOME(directory); Future<bool> launch = containerizer->launch( containerId, None(), executor, directory.get(), None(), state.id, map<string, string>(), true); // TODO(benh): Ever want to check not-checkpointing? AWAIT_ASSERT_TRUE(launch); ContainerID nestedContainerId1; nestedContainerId1.mutable_parent()->CopyFrom(containerId); nestedContainerId1.set_value(UUID::random().toString()); ContainerInfo containerInfo; containerInfo.set_type(ContainerInfo::MESOS); Volume* volume = containerInfo.add_volumes(); volume->set_mode(Volume::RW); volume->set_container_path("parent"); Volume::Source* source = volume->mutable_source(); source->set_type(Volume::Source::SANDBOX_PATH); Volume::Source::SandboxPath* sandboxPath = source->mutable_sandbox_path(); sandboxPath->set_type(Volume::Source::SandboxPath::PARENT); sandboxPath->set_path("shared"); launch = containerizer->launch( nestedContainerId1, createCommandInfo("touch parent/file; sleep 1000"), containerInfo, None(), state.id); AWAIT_ASSERT_TRUE(launch); ContainerID nestedContainerId2; nestedContainerId2.mutable_parent()->CopyFrom(containerId); nestedContainerId2.set_value(UUID::random().toString()); launch = containerizer->launch( nestedContainerId2, createCommandInfo( "while true; do if [ -f parent/file ]; then exit 0; fi; done"), containerInfo, None(), state.id); AWAIT_ASSERT_TRUE(launch); Future<Option<ContainerTermination>> wait = containerizer->wait(nestedContainerId2); AWAIT_READY(wait); ASSERT_SOME(wait.get()); ASSERT_TRUE(wait.get()->has_status()); EXPECT_WEXITSTATUS_EQ(0, wait.get()->status()); wait = containerizer->wait(containerId); containerizer->destroy(containerId); AWAIT_READY(wait); ASSERT_SOME(wait.get()); ASSERT_TRUE(wait.get()->has_status()); EXPECT_WTERMSIG_EQ(SIGKILL, wait.get()->status()); }
// The purpose of this test is to ensure that when slaves are removed // from the master, and then attempt to send status updates, we send // a ShutdownMessage to the slave. Why? Because during a network // partition, the master will remove a partitioned slave, thus sending // its tasks to LOST. At this point, when the partition is removed, // the slave may attempt to send updates if it was unaware that the // master removed it. We've already notified frameworks that these // tasks were LOST, so we have to have the slave shut down. TEST_F(PartitionTest, PartitionedSlaveStatusUpdates) { master::Flags masterFlags = CreateMasterFlags(); Try<Owned<cluster::Master>> master = StartMaster(masterFlags); ASSERT_SOME(master); // Allow the master to PING the slave, but drop all PONG messages // from the slave. Note that we don't match on the master / slave // PIDs because it's actually the SlaveObserver Process that sends // the pings. Future<Message> ping = FUTURE_MESSAGE( Eq(PingSlaveMessage().GetTypeName()), _, _); DROP_PROTOBUFS(PongSlaveMessage(), _, _); Future<SlaveRegisteredMessage> slaveRegisteredMessage = FUTURE_PROTOBUF(SlaveRegisteredMessage(), _, _); MockExecutor exec(DEFAULT_EXECUTOR_ID); TestContainerizer containerizer(&exec); Owned<MasterDetector> detector = master.get()->createDetector(); Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), &containerizer); ASSERT_SOME(slave); AWAIT_READY(slaveRegisteredMessage); SlaveID slaveId = slaveRegisteredMessage.get().slave_id(); MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL); Future<FrameworkID> frameworkId; EXPECT_CALL(sched, registered(&driver, _, _)) .WillOnce(FutureArg<1>(&frameworkId)); EXPECT_CALL(sched, resourceOffers(&driver, _)) .WillRepeatedly(Return()); driver.start(); AWAIT_READY(frameworkId); // Drop the first shutdown message from the master (simulated // partition), allow the second shutdown message to pass when // the slave sends an update. Future<ShutdownMessage> shutdownMessage = DROP_PROTOBUF(ShutdownMessage(), _, slave.get()->pid); EXPECT_CALL(sched, offerRescinded(&driver, _)) .WillRepeatedly(Return()); Future<Nothing> slaveLost; EXPECT_CALL(sched, slaveLost(&driver, _)) .WillOnce(FutureSatisfy(&slaveLost)); Clock::pause(); // Now, induce a partition of the slave by having the master // timeout the slave. size_t pings = 0; while (true) { AWAIT_READY(ping); pings++; if (pings == masterFlags.max_slave_ping_timeouts) { break; } ping = FUTURE_MESSAGE(Eq(PingSlaveMessage().GetTypeName()), _, _); Clock::advance(masterFlags.slave_ping_timeout); Clock::settle(); } Clock::advance(masterFlags.slave_ping_timeout); Clock::settle(); // Wait for the master to attempt to shut down the slave. AWAIT_READY(shutdownMessage); // The master will notify the framework that the slave was lost. AWAIT_READY(slaveLost); shutdownMessage = FUTURE_PROTOBUF(ShutdownMessage(), _, slave.get()->pid); // At this point, the slave still thinks it's registered, so we // simulate a status update coming from the slave. TaskID taskId; taskId.set_value("task_id"); const StatusUpdate& update = protobuf::createStatusUpdate( frameworkId.get(), slaveId, taskId, TASK_RUNNING, TaskStatus::SOURCE_SLAVE, UUID::random()); StatusUpdateMessage message; message.mutable_update()->CopyFrom(update); message.set_pid(stringify(slave.get()->pid)); process::post(master.get()->pid, message); // The master should shutdown the slave upon receiving the update. AWAIT_READY(shutdownMessage); Clock::resume(); driver.stop(); driver.join(); }
TEST_F(MemoryPressureMesosTest, CGROUPS_ROOT_Statistics) { Try<Owned<cluster::Master>> master = StartMaster(); ASSERT_SOME(master); slave::Flags flags = CreateSlaveFlags(); // We only care about memory cgroup for this test. flags.isolation = "cgroups/mem"; flags.agent_subsystems = None(); Fetcher fetcher; Try<MesosContainerizer*> _containerizer = MesosContainerizer::create(flags, true, &fetcher); ASSERT_SOME(_containerizer); Owned<MesosContainerizer> containerizer(_containerizer.get()); Owned<MasterDetector> detector = master.get()->createDetector(); Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), containerizer.get(), flags); ASSERT_SOME(slave); MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL); EXPECT_CALL(sched, registered(_, _, _)); Future<vector<Offer>> offers; EXPECT_CALL(sched, resourceOffers(_, _)) .WillOnce(FutureArg<1>(&offers)) .WillRepeatedly(Return()); // Ignore subsequent offers. driver.start(); AWAIT_READY(offers); EXPECT_NE(0u, offers.get().size()); Offer offer = offers.get()[0]; // Run a task that triggers memory pressure event. We request 1G // disk because we are going to write a 512 MB file repeatedly. TaskInfo task = createTask( offer.slave_id(), Resources::parse("cpus:1;mem:256;disk:1024").get(), "while true; do dd count=512 bs=1M if=/dev/zero of=./temp; done"); Future<TaskStatus> running; Future<TaskStatus> killed; EXPECT_CALL(sched, statusUpdate(&driver, _)) .WillOnce(FutureArg<1>(&running)) .WillOnce(FutureArg<1>(&killed)) .WillRepeatedly(Return()); // Ignore subsequent updates. driver.launchTasks(offer.id(), {task}); AWAIT_READY(running); EXPECT_EQ(task.task_id(), running.get().task_id()); EXPECT_EQ(TASK_RUNNING, running.get().state()); Future<hashset<ContainerID>> containers = containerizer->containers(); AWAIT_READY(containers); ASSERT_EQ(1u, containers.get().size()); ContainerID containerId = *(containers.get().begin()); // Wait a while for some memory pressure events to occur. Duration waited = Duration::zero(); do { Future<ResourceStatistics> usage = containerizer->usage(containerId); AWAIT_READY(usage); if (usage.get().mem_low_pressure_counter() > 0) { // We will check the correctness of the memory pressure counters // later, because the memory-hammering task is still active // and potentially incrementing these counters. break; } os::sleep(Milliseconds(100)); waited += Milliseconds(100); } while (waited < Seconds(5)); EXPECT_LE(waited, Seconds(5)); // Pause the clock to ensure that the reaper doesn't reap the exited // command executor and inform the containerizer/slave. Clock::pause(); Clock::settle(); // Stop the memory-hammering task. driver.killTask(task.task_id()); AWAIT_READY_FOR(killed, Seconds(120)); EXPECT_EQ(task.task_id(), killed->task_id()); EXPECT_EQ(TASK_KILLED, killed->state()); // Now check the correctness of the memory pressure counters. Future<ResourceStatistics> usage = containerizer->usage(containerId); AWAIT_READY(usage); EXPECT_GE(usage.get().mem_low_pressure_counter(), usage.get().mem_medium_pressure_counter()); EXPECT_GE(usage.get().mem_medium_pressure_counter(), usage.get().mem_critical_pressure_counter()); Clock::resume(); driver.stop(); driver.join(); }
// This test verifies that the image specified in the volume will be // properly provisioned and mounted into the container if container // root filesystem is not specified. TEST_P(VolumeImageIsolatorTest, ROOT_ImageInVolumeWithoutRootFilesystem) { string registry = path::join(sandbox.get(), "registry"); AWAIT_READY(DockerArchive::create(registry, "test_image")); slave::Flags flags = CreateSlaveFlags(); flags.isolation = "filesystem/linux,volume/image,docker/runtime"; flags.docker_registry = registry; flags.docker_store_dir = path::join(sandbox.get(), "store"); flags.image_providers = "docker"; Fetcher fetcher(flags); Try<MesosContainerizer*> create = MesosContainerizer::create(flags, true, &fetcher); ASSERT_SOME(create); Owned<Containerizer> containerizer(create.get()); ContainerID containerId; containerId.set_value(id::UUID::random().toString()); ContainerInfo container = createContainerInfo( None(), {createVolumeFromDockerImage("rootfs", "test_image", Volume::RW)}); CommandInfo command = createCommandInfo("test -d rootfs/bin"); ExecutorInfo executor = createExecutorInfo( "test_executor", nesting ? createCommandInfo("sleep 1000") : command); if (!nesting) { executor.mutable_container()->CopyFrom(container); } string directory = path::join(flags.work_dir, "sandbox"); ASSERT_SOME(os::mkdir(directory)); Future<Containerizer::LaunchResult> launch = containerizer->launch( containerId, createContainerConfig(None(), executor, directory), map<string, string>(), None()); AWAIT_ASSERT_EQ(Containerizer::LaunchResult::SUCCESS, launch); Future<Option<ContainerTermination>> wait = containerizer->wait(containerId); if (nesting) { ContainerID nestedContainerId; nestedContainerId.mutable_parent()->CopyFrom(containerId); nestedContainerId.set_value(id::UUID::random().toString()); launch = containerizer->launch( nestedContainerId, createContainerConfig(command, container), map<string, string>(), None()); AWAIT_ASSERT_EQ(Containerizer::LaunchResult::SUCCESS, launch); wait = containerizer->wait(nestedContainerId); } AWAIT_READY(wait); ASSERT_SOME(wait.get()); ASSERT_TRUE(wait->get().has_status()); EXPECT_WEXITSTATUS_EQ(0, wait->get().status()); if (nesting) { Future<Option<ContainerTermination>> termination = containerizer->destroy(containerId); AWAIT_READY(termination); ASSERT_SOME(termination.get()); ASSERT_TRUE(termination->get().has_status()); EXPECT_WTERMSIG_EQ(SIGKILL, termination.get()->status()); } }
// This test checks that a failed over scheduler gets the retried status update // when the original instance dies without acknowledging the update. TEST_F(HttpFaultToleranceTest, SchedulerFailoverStatusUpdate) { master::Flags flags = CreateMasterFlags(); flags.authenticate_frameworks = false; Try<Owned<cluster::Master>> master = StartMaster(flags); ASSERT_SOME(master); auto scheduler = std::make_shared<v1::MockHTTPScheduler>(); auto executor = std::make_shared<v1::MockHTTPExecutor>(); ExecutorID executorId = DEFAULT_EXECUTOR_ID; TestContainerizer containerizer(executorId, executor); Owned<MasterDetector> detector = master.get()->createDetector(); Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), &containerizer); ASSERT_SOME(slave); Future<Nothing> connected; EXPECT_CALL(*scheduler, connected(_)) .WillOnce(FutureSatisfy(&connected)) .WillRepeatedly(Return()); // Ignore future invocations. ContentType contentType = ContentType::PROTOBUF; v1::scheduler::TestMesos schedulerLibrary( master.get()->pid, contentType, scheduler); AWAIT_READY(connected); Future<Event::Subscribed> subscribed; EXPECT_CALL(*scheduler, subscribed(_, _)) .WillOnce(FutureArg<1>(&subscribed)); EXPECT_CALL(*scheduler, heartbeat(_)) .WillRepeatedly(Return()); // Ignore heartbeats. Future<Event::Offers> offers; EXPECT_CALL(*scheduler, offers(_, _)) .WillOnce(FutureArg<1>(&offers)); { Call call; call.set_type(Call::SUBSCRIBE); Call::Subscribe* subscribe = call.mutable_subscribe(); subscribe->mutable_framework_info()->CopyFrom(v1::DEFAULT_FRAMEWORK_INFO); schedulerLibrary.send(call); } AWAIT_READY(subscribed); v1::FrameworkID frameworkId(subscribed->framework_id()); AWAIT_READY(offers); EXPECT_NE(0, offers->offers().size()); EXPECT_CALL(*executor, connected(_)) .WillOnce(v1::executor::SendSubscribe(frameworkId, evolve(executorId))); EXPECT_CALL(*executor, subscribed(_, _)); EXPECT_CALL(*executor, launch(_, _)) .WillOnce(v1::executor::SendUpdateFromTask( frameworkId, evolve(executorId), v1::TASK_RUNNING)); Future<Nothing> acknowledged; EXPECT_CALL(*executor, acknowledged(_, _)) .WillOnce(FutureSatisfy(&acknowledged)); Future<Event::Update> update; EXPECT_CALL(*scheduler, update(_, _)) .WillOnce(FutureArg<1>(&update)); const v1::Offer& offer = offers->offers(0); v1::TaskInfo taskInfo = evolve(createTask(devolve(offer), "", executorId)); { Call call; call.mutable_framework_id()->CopyFrom(frameworkId); call.set_type(Call::ACCEPT); Call::Accept* accept = call.mutable_accept(); accept->add_offer_ids()->CopyFrom(offer.id()); v1::Offer::Operation* operation = accept->add_operations(); operation->set_type(v1::Offer::Operation::LAUNCH); operation->mutable_launch()->add_task_infos()->CopyFrom(taskInfo); schedulerLibrary.send(call); } AWAIT_READY(acknowledged); AWAIT_READY(update); EXPECT_EQ(v1::TASK_RUNNING, update->status().state()); EXPECT_EQ(executorId, devolve(update->status().executor_id())); EXPECT_TRUE(update->status().has_executor_id()); EXPECT_TRUE(update->status().has_uuid()); // Failover the scheduler without acknowledging the status update. auto scheduler2 = std::make_shared<v1::MockHTTPScheduler>(); Future<Nothing> connected2; EXPECT_CALL(*scheduler2, connected(_)) .WillOnce(FutureSatisfy(&connected2)); // Failover to another scheduler instance. v1::scheduler::TestMesos schedulerLibrary2( master.get()->pid, contentType, scheduler2); AWAIT_READY(connected2); // The previously connected scheduler instance should receive an // error/disconnected event. Future<Nothing> error; EXPECT_CALL(*scheduler, error(_, _)) .WillOnce(FutureSatisfy(&error)); Future<Nothing> disconnected; EXPECT_CALL(*scheduler, disconnected(_)) .WillOnce(FutureSatisfy(&disconnected)); EXPECT_CALL(*scheduler2, subscribed(_, _)) .WillOnce(FutureArg<1>(&subscribed)); EXPECT_CALL(*scheduler2, heartbeat(_)) .WillRepeatedly(Return()); // Ignore heartbeats. // Scheduler2 should receive the retried status update. Future<Nothing> update2; EXPECT_CALL(*scheduler2, update(_, _)) .WillOnce(FutureSatisfy(&update2)) .WillRepeatedly(Return()); // Ignore subsequent updates. { Call call; call.mutable_framework_id()->CopyFrom(frameworkId); call.set_type(Call::SUBSCRIBE); Call::Subscribe* subscribe = call.mutable_subscribe(); subscribe->mutable_framework_info()->CopyFrom(v1::DEFAULT_FRAMEWORK_INFO); subscribe->mutable_framework_info()->mutable_id()->CopyFrom(frameworkId); schedulerLibrary2.send(call); } AWAIT_READY(error); AWAIT_READY(disconnected); AWAIT_READY(subscribed); EXPECT_EQ(frameworkId, subscribed->framework_id()); Clock::pause(); // Now advance time enough for the reliable timeout to kick in and // another status update to be sent. Clock::advance(slave::STATUS_UPDATE_RETRY_INTERVAL_MIN); AWAIT_READY(update2); EXPECT_CALL(*executor, shutdown(_)) .Times(AtMost(1)); EXPECT_CALL(*executor, disconnected(_)) .Times(AtMost(1)); }
// This test verifies if the executor is able to receive a Subscribed // event in response to a Subscribe call request. TEST_P(ExecutorHttpApiTest, Subscribe) { Try<Owned<cluster::Master>> master = StartMaster(); ASSERT_SOME(master); ExecutorID executorId = DEFAULT_EXECUTOR_ID; MockExecutor exec(executorId); TestContainerizer containerizer(&exec); Owned<MasterDetector> detector = master.get()->createDetector(); Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), &containerizer); ASSERT_SOME(slave); MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL); Future<FrameworkID> frameworkId; EXPECT_CALL(sched, registered(&driver, _, _)) .WillOnce(FutureArg<1>(&frameworkId)); Future<vector<Offer>> offers; EXPECT_CALL(sched, resourceOffers(&driver, _)) .WillOnce(FutureArg<1>(&offers)); driver.start(); AWAIT_READY(frameworkId); AWAIT_READY(offers); ASSERT_EQ(1u, offers.get().size()); Future<Message> registerExecutorMessage = DROP_MESSAGE(Eq(RegisterExecutorMessage().GetTypeName()), _, _); TaskInfo taskInfo = createTask(offers.get()[0], "", executorId); driver.launchTasks(offers.get()[0].id(), {taskInfo}); // Drop the `RegisterExecutorMessage` and then send a `Subscribe` request // from the HTTP based executor. AWAIT_READY(registerExecutorMessage); Call call; call.mutable_framework_id()->CopyFrom(evolve(frameworkId.get())); call.mutable_executor_id()->CopyFrom(evolve(executorId)); call.set_type(Call::SUBSCRIBE); call.mutable_subscribe(); // Retrieve the parameter passed as content type to this test. const ContentType contentType = GetParam(); const string contentTypeString = stringify(contentType); process::http::Headers headers; headers["Accept"] = contentTypeString; Future<Response> response = process::http::streaming::post( slave.get()->pid, "api/v1/executor", headers, serialize(contentType, call), contentTypeString); AWAIT_EXPECT_RESPONSE_STATUS_EQ(OK().status, response); AWAIT_EXPECT_RESPONSE_HEADER_EQ("chunked", "Transfer-Encoding", response); AWAIT_EXPECT_RESPONSE_HEADER_EQ(contentTypeString, "Content-Type", response); ASSERT_EQ(Response::PIPE, response.get().type); Option<Pipe::Reader> reader = response.get().reader; ASSERT_SOME(reader); auto deserializer = lambda::bind(deserialize<Event>, contentType, lambda::_1); Reader<Event> responseDecoder( Decoder<Event>(deserializer), reader.get()); Future<Result<Event>> event = responseDecoder.read(); AWAIT_READY(event); ASSERT_SOME(event.get()); // Check event type is subscribed and if the ExecutorID matches. ASSERT_EQ(Event::SUBSCRIBED, event.get().get().type()); ASSERT_EQ(event.get().get().subscribed().executor_info().executor_id(), call.executor_id()); reader.get().close(); driver.stop(); driver.join(); }
// This test ensures that the failed over scheduler is able to send a message // to the executor. TEST_F(HttpFaultToleranceTest, SchedulerFailoverFrameworkToExecutorMessage) { master::Flags flags = CreateMasterFlags(); flags.authenticate_frameworks = false; Try<Owned<cluster::Master>> master = StartMaster(flags); ASSERT_SOME(master); auto scheduler = std::make_shared<v1::MockHTTPScheduler>(); auto executor = std::make_shared<v1::MockHTTPExecutor>(); ExecutorID executorId = DEFAULT_EXECUTOR_ID; TestContainerizer containerizer(executorId, executor); Owned<MasterDetector> detector = master.get()->createDetector(); Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), &containerizer); ASSERT_SOME(slave); Future<Nothing> connected; EXPECT_CALL(*scheduler, connected(_)) .WillOnce(FutureSatisfy(&connected)) .WillRepeatedly(Return()); // Ignore future invocations. ContentType contentType = ContentType::PROTOBUF; v1::scheduler::TestMesos schedulerLibrary( master.get()->pid, contentType, scheduler); AWAIT_READY(connected); Future<Event::Subscribed> subscribed; EXPECT_CALL(*scheduler, subscribed(_, _)) .WillOnce(FutureArg<1>(&subscribed)); EXPECT_CALL(*scheduler, heartbeat(_)) .WillRepeatedly(Return()); // Ignore heartbeats. Future<Event::Offers> offers; EXPECT_CALL(*scheduler, offers(_, _)) .WillOnce(FutureArg<1>(&offers)); { Call call; call.set_type(Call::SUBSCRIBE); Call::Subscribe* subscribe = call.mutable_subscribe(); subscribe->mutable_framework_info()->CopyFrom(v1::DEFAULT_FRAMEWORK_INFO); schedulerLibrary.send(call); } AWAIT_READY(subscribed); v1::FrameworkID frameworkId(subscribed->framework_id()); AWAIT_READY(offers); EXPECT_NE(0, offers->offers().size()); EXPECT_CALL(*executor, connected(_)) .WillOnce(v1::executor::SendSubscribe(frameworkId, evolve(executorId))); EXPECT_CALL(*executor, subscribed(_, _)); Future<Nothing> launch; EXPECT_CALL(*executor, launch(_, _)) .WillOnce(FutureSatisfy(&launch)); const v1::Offer& offer = offers->offers(0); v1::TaskInfo taskInfo = evolve(createTask(devolve(offer), "", executorId)); { Call call; call.mutable_framework_id()->CopyFrom(frameworkId); call.set_type(Call::ACCEPT); Call::Accept* accept = call.mutable_accept(); accept->add_offer_ids()->CopyFrom(offer.id()); v1::Offer::Operation* operation = accept->add_operations(); operation->set_type(v1::Offer::Operation::LAUNCH); operation->mutable_launch()->add_task_infos()->CopyFrom(taskInfo); schedulerLibrary.send(call); } AWAIT_READY(launch); auto scheduler2 = std::make_shared<v1::MockHTTPScheduler>(); Future<Nothing> connected2; EXPECT_CALL(*scheduler2, connected(_)) .WillOnce(FutureSatisfy(&connected2)); // Failover to another scheduler instance. v1::scheduler::TestMesos schedulerLibrary2( master.get()->pid, contentType, scheduler2); AWAIT_READY(connected2); // The previously connected scheduler instance should receive an // error/disconnected event. Future<Nothing> error; EXPECT_CALL(*scheduler, error(_, _)) .WillOnce(FutureSatisfy(&error)); Future<Nothing> disconnected; EXPECT_CALL(*scheduler, disconnected(_)) .WillOnce(FutureSatisfy(&disconnected)); EXPECT_CALL(*scheduler2, subscribed(_, _)) .WillOnce(FutureArg<1>(&subscribed)); EXPECT_CALL(*scheduler2, heartbeat(_)) .WillRepeatedly(Return()); // Ignore heartbeats. { Call call; call.mutable_framework_id()->CopyFrom(frameworkId); call.set_type(Call::SUBSCRIBE); Call::Subscribe* subscribe = call.mutable_subscribe(); subscribe->mutable_framework_info()->CopyFrom(v1::DEFAULT_FRAMEWORK_INFO); subscribe->mutable_framework_info()->mutable_id()->CopyFrom(frameworkId); schedulerLibrary2.send(call); } AWAIT_READY(error); AWAIT_READY(disconnected); AWAIT_READY(subscribed); EXPECT_EQ(frameworkId, subscribed->framework_id()); Future<v1::executor::Event::Message> message; EXPECT_CALL(*executor, message(_, _)) .WillOnce(FutureArg<1>(&message)); { Call call; call.mutable_framework_id()->CopyFrom(frameworkId); call.set_type(Call::MESSAGE); Call::Message* message = call.mutable_message(); message->mutable_agent_id()->CopyFrom(offer.agent_id()); message->mutable_executor_id()->CopyFrom(v1::DEFAULT_EXECUTOR_ID); message->set_data("hello world"); schedulerLibrary2.send(call); } AWAIT_READY(message); ASSERT_EQ("hello world", message->data()); EXPECT_CALL(*executor, shutdown(_)) .Times(AtMost(1)); EXPECT_CALL(*executor, disconnected(_)) .Times(AtMost(1)); }
// This test has been temporarily disabled due to MESOS-1257. TEST_F(ExternalContainerizerTest, DISABLED_Launch) { Try<PID<Master> > master = this->StartMaster(); ASSERT_SOME(master); Flags testFlags; slave::Flags flags = this->CreateSlaveFlags(); flags.isolation = "external"; flags.containerizer_path = testFlags.build_dir + "/src/examples/python/test-containerizer"; MockExternalContainerizer containerizer(flags); Try<PID<Slave> > slave = this->StartSlave(&containerizer, flags); ASSERT_SOME(slave); MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get(), DEFAULT_CREDENTIAL); Future<FrameworkID> frameworkId; EXPECT_CALL(sched, registered(&driver, _, _)) .WillOnce(FutureArg<1>(&frameworkId)); Future<vector<Offer> > offers; EXPECT_CALL(sched, resourceOffers(&driver, _)) .WillOnce(FutureArg<1>(&offers)) .WillRepeatedly(Return()); // Ignore subsequent offers. driver.start(); AWAIT_READY(frameworkId); AWAIT_READY(offers); EXPECT_NE(0u, offers.get().size()); TaskInfo task; task.set_name("isolator_test"); task.mutable_task_id()->set_value("1"); task.mutable_slave_id()->CopyFrom(offers.get()[0].slave_id()); task.mutable_resources()->CopyFrom(offers.get()[0].resources()); Resources resources(offers.get()[0].resources()); Option<Bytes> mem = resources.mem(); ASSERT_SOME(mem); Option<double> cpus = resources.cpus(); ASSERT_SOME(cpus); const std::string& file = path::join(flags.work_dir, "ready"); // This task induces user/system load in a child process by // running top in a child process for ten seconds. task.mutable_command()->set_value( #ifdef __APPLE__ // Use logging mode with 30,000 samples with no interval. "top -l 30000 -s 0 2>&1 > /dev/null & " #else // Batch mode, with 30,000 samples with no interval. "top -b -d 0 -n 30000 2>&1 > /dev/null & " #endif "touch " + file + "; " // Signals that the top command is running. "sleep 60"); Future<TaskStatus> status; EXPECT_CALL(sched, statusUpdate(&driver, _)) .WillOnce(FutureArg<1>(&status)) .WillRepeatedly(Return()); // Ignore rest for now. Future<ContainerID> containerId; EXPECT_CALL(containerizer, launch(_, _, _, _, _, _, _, _)) .WillOnce(DoAll(FutureArg<0>(&containerId), Invoke(&containerizer, &MockExternalContainerizer::_launch))); driver.launchTasks(offers.get()[0].id(), {task}); AWAIT_READY(containerId); AWAIT_READY(status); EXPECT_EQ(TASK_RUNNING, status.get().state()); // Wait for the task to begin inducing cpu time. while (!os::exists(file)); ExecutorID executorId; executorId.set_value(task.task_id().value()); // We'll wait up to 10 seconds for the child process to induce // 1/8 of a second of user and system cpu time in total. // TODO(bmahler): Also induce rss memory consumption, by re-using // the balloon framework. ResourceStatistics statistics; Duration waited = Duration::zero(); do { Future<ResourceStatistics> usage = containerizer.usage(containerId.get()); AWAIT_READY(usage); statistics = usage.get(); // If we meet our usage expectations, we're done! // NOTE: We are currently getting dummy-data from the test- // containerizer python script matching these expectations. // TODO(tillt): Consider working with real data. if (statistics.cpus_user_time_secs() >= 0.120 && statistics.cpus_system_time_secs() >= 0.05 && statistics.mem_rss_bytes() >= 1024u) { break; } os::sleep(Milliseconds(100)); waited += Milliseconds(100); } while (waited < Seconds(10)); EXPECT_GE(statistics.cpus_user_time_secs(), 0.120); EXPECT_GE(statistics.cpus_system_time_secs(), 0.05); EXPECT_EQ(statistics.cpus_limit(), cpus.get()); EXPECT_GE(statistics.mem_rss_bytes(), 1024u); EXPECT_EQ(statistics.mem_limit_bytes(), mem.get().bytes()); EXPECT_CALL(sched, statusUpdate(&driver, _)) .WillOnce(FutureArg<1>(&status)); driver.killTask(task.task_id()); AWAIT_READY(status); EXPECT_EQ(TASK_KILLED, status.get().state()); driver.stop(); driver.join(); this->Shutdown(); }
// This test verifies that the slave run task label decorator can add // and remove labels from a task during the launch sequence. A task // with two labels ("foo":"bar" and "bar":"baz") is launched and will // get modified by the slave hook to strip the "foo":"bar" pair and // add a new "baz":"qux" pair. TEST_F(HookTest, VerifySlaveRunTaskHook) { Try<PID<Master>> master = StartMaster(); ASSERT_SOME(master); MockExecutor exec(DEFAULT_EXECUTOR_ID); TestContainerizer containerizer(&exec); Try<PID<Slave>> slave = StartSlave(&containerizer); ASSERT_SOME(slave); MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get(), DEFAULT_CREDENTIAL); EXPECT_CALL(sched, registered(&driver, _, _)); Future<vector<Offer>> offers; EXPECT_CALL(sched, resourceOffers(&driver, _)) .WillOnce(FutureArg<1>(&offers)) .WillRepeatedly(Return()); // Ignore subsequent offers. driver.start(); AWAIT_READY(offers); ASSERT_EQ(1u, offers.get().size()); TaskInfo task; task.set_name(""); task.mutable_task_id()->set_value("1"); task.mutable_slave_id()->CopyFrom(offers.get()[0].slave_id()); task.mutable_resources()->CopyFrom(offers.get()[0].resources()); task.mutable_executor()->CopyFrom(DEFAULT_EXECUTOR_INFO); // Add two labels: (1) will be removed by the hook to ensure that // runTaskHook can remove labels (2) will be preserved to ensure // that the framework can add labels to the task and have those be // available by the end of the launch task sequence when hooks are // used (to protect against hooks removing labels completely). Labels* labels = task.mutable_labels(); Label* label1 = labels->add_labels(); label1->set_key("foo"); label1->set_value("bar"); Label* label2 = labels->add_labels(); label2->set_key("bar"); label2->set_value("baz"); vector<TaskInfo> tasks; tasks.push_back(task); EXPECT_CALL(exec, registered(_, _, _, _)); Future<TaskInfo> taskInfo; EXPECT_CALL(exec, launchTask(_, _)) .WillOnce(DoAll( FutureArg<1>(&taskInfo), SendStatusUpdateFromTask(TASK_RUNNING))); driver.launchTasks(offers.get()[0].id(), tasks); AWAIT_READY(taskInfo); // The master hook will hang an extra label off. const Labels& labels_ = taskInfo.get().labels(); ASSERT_EQ(3, labels_.labels_size()); // The slave run task hook will prepend a new "baz":"qux" label. EXPECT_EQ(labels_.labels(0).key(), "baz"); EXPECT_EQ(labels_.labels(0).value(), "qux"); // Master launch task hook will still hang off test label. EXPECT_EQ(labels_.labels(1).key(), testLabelKey); EXPECT_EQ(labels_.labels(1).value(), testLabelValue); // And lastly, we only expect the "foo":"bar" pair to be stripped by // the module. The last pair should be the original "bar":"baz" // pair set by the test. EXPECT_EQ(labels_.labels(2).key(), "bar"); EXPECT_EQ(labels_.labels(2).value(), "baz"); driver.stop(); driver.join(); Shutdown(); // Must shutdown before 'containerizer' gets deallocated. }