// Using JSON base file for authentication without // protobuf tools assistance. TEST_F(CredentialsTest, AuthenticatedSlaveJSON) { string path = path::join(os::getcwd(), "credentials"); Try<int_fd> fd = os::open( path, O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, S_IRUSR | S_IWUSR | S_IRGRP); ASSERT_SOME(fd); // This unit tests our capacity to process JSON credentials without // using our protobuf tools. JSON::Object credential; credential.values["principal"] = DEFAULT_CREDENTIAL.principal(); credential.values["secret"] = DEFAULT_CREDENTIAL.secret(); JSON::Array array; array.values.push_back(credential); JSON::Object credentials; credentials.values["credentials"] = array; ASSERT_SOME(os::write(fd.get(), stringify(credentials))) << "Failed to write credentials to '" << path << "'"; ASSERT_SOME(os::close(fd.get())); map<string, Option<string>> values{ {"credentials", Some(uri::from_path(path))}}; master::Flags masterFlags = CreateMasterFlags(); masterFlags.load(values, true); Try<Owned<cluster::Master>> master = StartMaster(masterFlags); ASSERT_SOME(master); Future<SlaveRegisteredMessage> slaveRegisteredMessage = FUTURE_PROTOBUF(SlaveRegisteredMessage(), _, _); slave::Flags slaveFlags = CreateSlaveFlags(); slaveFlags.load(values, true); Owned<MasterDetector> detector = master.get()->createDetector(); Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), slaveFlags); ASSERT_SOME(slave); AWAIT_READY(slaveRegisteredMessage); ASSERT_NE("", slaveRegisteredMessage->slave_id().value()); }
TEST_F_TEMP_DISABLED_ON_WINDOWS( ResourceOffersTest, ResourceOfferWithMultipleSlaves) { Try<Owned<cluster::Master>> master = StartMaster(); ASSERT_SOME(master); Owned<MasterDetector> detector = master.get()->createDetector(); vector<Owned<cluster::Slave>> slaves; // Start 10 slaves. for (int i = 0; i < 10; i++) { slave::Flags flags = CreateSlaveFlags(); flags.launcher = "posix"; flags.resources = Option<std::string>("cpus:2;mem:1024"); Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), flags); ASSERT_SOME(slave); slaves.push_back(slave.get()); } MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL); EXPECT_CALL(sched, registered(&driver, _, _)); Future<vector<Offer>> offers; EXPECT_CALL(sched, resourceOffers(&driver, _)) .WillOnce(FutureArg<1>(&offers)) .WillRepeatedly(Return()); // All 10 slaves might not be in first offer. driver.start(); AWAIT_READY(offers); ASSERT_FALSE(offers->empty()); EXPECT_GE(10u, offers->size()); Resources resources(offers.get()[0].resources()); EXPECT_EQ(2, resources.get<Value::Scalar>("cpus")->value()); EXPECT_EQ(1024, resources.get<Value::Scalar>("mem")->value()); driver.stop(); driver.join(); }
// Tests that requests for an agent endpoint // always succeed if the authorizer is absent. TEST_P(SlaveEndpointTest, NoAuthorizer) { const string endpoint = GetParam(); StandaloneMasterDetector detector; Try<Owned<cluster::Slave>> agent = StartSlave(&detector, CreateSlaveFlags()); ASSERT_SOME(agent); Future<Response> response = http::get( agent.get()->pid, endpoint, None(), createBasicAuthHeaders(DEFAULT_CREDENTIAL)); AWAIT_EXPECT_RESPONSE_STATUS_EQ(OK().status, response) << response.get().body; }
TEST_F(ResourceOffersTest, ResourceOfferWithMultipleSlaves) { Try<PID<Master>> master = StartMaster(); ASSERT_SOME(master); // Start 10 slaves. for (int i = 0; i < 10; i++) { slave::Flags flags = CreateSlaveFlags(); flags.resources = Option<std::string>("cpus:2;mem:1024"); Try<PID<Slave>> slave = StartSlave(flags); ASSERT_SOME(slave); } MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get(), DEFAULT_CREDENTIAL); EXPECT_CALL(sched, registered(&driver, _, _)) .Times(1); Future<vector<Offer>> offers; EXPECT_CALL(sched, resourceOffers(&driver, _)) .WillOnce(FutureArg<1>(&offers)) .WillRepeatedly(Return()); // All 10 slaves might not be in first offer. driver.start(); AWAIT_READY(offers); EXPECT_NE(0u, offers.get().size()); EXPECT_GE(10u, offers.get().size()); Resources resources(offers.get()[0].resources()); EXPECT_EQ(2, resources.get<Value::Scalar>("cpus").get().value()); EXPECT_EQ(1024, resources.get<Value::Scalar>("mem").get().value()); driver.stop(); driver.join(); Shutdown(); }
// Test verifing well executed credential authentication // using text formatted credentials so as to test // backwards compatibility. TEST_F(CredentialsTest, AuthenticatedSlaveText) { string path = path::join(os::getcwd(), "credentials"); Try<int_fd> fd = os::open( path, O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, S_IRUSR | S_IWUSR | S_IRGRP); ASSERT_SOME(fd); string credentials = DEFAULT_CREDENTIAL.principal() + " " + DEFAULT_CREDENTIAL.secret(); ASSERT_SOME(os::write(fd.get(), credentials)) << "Failed to write credentials to '" << path << "'"; ASSERT_SOME(os::close(fd.get())); map<string, Option<string>> values{ {"credentials", Some(uri::from_path(path))}}; master::Flags masterFlags = CreateMasterFlags(); masterFlags.load(values, true); Try<Owned<cluster::Master>> master = StartMaster(masterFlags); ASSERT_SOME(master); Future<SlaveRegisteredMessage> slaveRegisteredMessage = FUTURE_PROTOBUF(SlaveRegisteredMessage(), _, _); slave::Flags slaveFlags = CreateSlaveFlags(); slaveFlags.load(values, true); Owned<MasterDetector> detector = master.get()->createDetector(); Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), slaveFlags); ASSERT_SOME(slave); AWAIT_READY(slaveRegisteredMessage); ASSERT_NE("", slaveRegisteredMessage->slave_id().value()); }
// Test verifing well executed credential authentication // using text formatted credentials so as to test // backwards compatibility. TEST_F(CredentialsTest, AuthenticatedSlaveText) { string path = path::join(os::getcwd(), "credentials"); Try<int> fd = os::open( path, O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, S_IRUSR | S_IWUSR | S_IRGRP); CHECK_SOME(fd); std::string credentials = DEFAULT_CREDENTIAL.principal() + " " + DEFAULT_CREDENTIAL.secret(); CHECK_SOME(os::write(fd.get(), credentials)) << "Failed to write credentials to '" << path << "'"; CHECK_SOME(os::close(fd.get())); map<string, Option<string>> values{{"credentials", Some("file://" + path)}}; master::Flags masterFlags = CreateMasterFlags(); masterFlags.load(values, true); Try<PID<Master>> master = StartMaster(masterFlags); ASSERT_SOME(master); Future<SlaveRegisteredMessage> slaveRegisteredMessage = FUTURE_PROTOBUF(SlaveRegisteredMessage(), _, _); slave::Flags slaveFlags = CreateSlaveFlags(); slaveFlags.load(values, true); Try<PID<Slave>> slave = StartSlave(slaveFlags); ASSERT_SOME(slave); AWAIT_READY(slaveRegisteredMessage); ASSERT_NE("", slaveRegisteredMessage.get().slave_id().value()); Shutdown(); }
// Test that the environment decorator hook adds a new environment // variable to the executor runtime. // Test hook adds a new environment variable "FOO" to the executor // with a value "bar". We validate the hook by verifying the value // of this environment variable. TEST_F(HookTest, VerifySlaveExecutorEnvironmentDecorator) { const string& directory = os::getcwd(); // We're inside a temporary sandbox. Fetcher fetcher; Try<MesosContainerizer*> containerizer = MesosContainerizer::create(CreateSlaveFlags(), false, &fetcher); ASSERT_SOME(containerizer); ContainerID containerId; containerId.set_value("test_container"); // Test hook adds a new environment variable "FOO" to the executor // with a value "bar". A '0' (success) exit status for the following // command validates the hook. process::Future<bool> launch = containerizer.get()->launch( containerId, CREATE_EXECUTOR_INFO("executor", "test $FOO = 'bar'"), directory, None(), SlaveID(), process::PID<Slave>(), false); AWAIT_READY(launch); ASSERT_TRUE(launch.get()); // Wait on the container. process::Future<containerizer::Termination> wait = containerizer.get()->wait(containerId); AWAIT_READY(wait); // Check the executor exited correctly. EXPECT_TRUE(wait.get().has_status()); EXPECT_EQ(0, wait.get().status()); delete containerizer.get(); }
TEST_F(MemoryPressureMesosTest, CGROUPS_ROOT_Statistics) { Try<Owned<cluster::Master>> master = StartMaster(); ASSERT_SOME(master); slave::Flags flags = CreateSlaveFlags(); // We only care about memory cgroup for this test. flags.isolation = "cgroups/mem"; flags.agent_subsystems = None(); Fetcher fetcher; Try<MesosContainerizer*> _containerizer = MesosContainerizer::create(flags, true, &fetcher); ASSERT_SOME(_containerizer); Owned<MesosContainerizer> containerizer(_containerizer.get()); Owned<MasterDetector> detector = master.get()->createDetector(); Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), containerizer.get(), flags); ASSERT_SOME(slave); MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL); EXPECT_CALL(sched, registered(_, _, _)); Future<vector<Offer>> offers; EXPECT_CALL(sched, resourceOffers(_, _)) .WillOnce(FutureArg<1>(&offers)) .WillRepeatedly(Return()); // Ignore subsequent offers. driver.start(); AWAIT_READY(offers); EXPECT_NE(0u, offers.get().size()); Offer offer = offers.get()[0]; // Run a task that triggers memory pressure event. We request 1G // disk because we are going to write a 512 MB file repeatedly. TaskInfo task = createTask( offer.slave_id(), Resources::parse("cpus:1;mem:256;disk:1024").get(), "while true; do dd count=512 bs=1M if=/dev/zero of=./temp; done"); Future<TaskStatus> running; Future<TaskStatus> killed; EXPECT_CALL(sched, statusUpdate(&driver, _)) .WillOnce(FutureArg<1>(&running)) .WillOnce(FutureArg<1>(&killed)) .WillRepeatedly(Return()); // Ignore subsequent updates. driver.launchTasks(offer.id(), {task}); AWAIT_READY(running); EXPECT_EQ(task.task_id(), running.get().task_id()); EXPECT_EQ(TASK_RUNNING, running.get().state()); Future<hashset<ContainerID>> containers = containerizer->containers(); AWAIT_READY(containers); ASSERT_EQ(1u, containers.get().size()); ContainerID containerId = *(containers.get().begin()); // Wait a while for some memory pressure events to occur. Duration waited = Duration::zero(); do { Future<ResourceStatistics> usage = containerizer->usage(containerId); AWAIT_READY(usage); if (usage.get().mem_low_pressure_counter() > 0) { // We will check the correctness of the memory pressure counters // later, because the memory-hammering task is still active // and potentially incrementing these counters. break; } os::sleep(Milliseconds(100)); waited += Milliseconds(100); } while (waited < Seconds(5)); EXPECT_LE(waited, Seconds(5)); // Pause the clock to ensure that the reaper doesn't reap the exited // command executor and inform the containerizer/slave. Clock::pause(); Clock::settle(); // Stop the memory-hammering task. driver.killTask(task.task_id()); AWAIT_READY_FOR(killed, Seconds(120)); EXPECT_EQ(task.task_id(), killed->task_id()); EXPECT_EQ(TASK_KILLED, killed->state()); // Now check the correctness of the memory pressure counters. Future<ResourceStatistics> usage = containerizer->usage(containerId); AWAIT_READY(usage); EXPECT_GE(usage.get().mem_low_pressure_counter(), usage.get().mem_medium_pressure_counter()); EXPECT_GE(usage.get().mem_medium_pressure_counter(), usage.get().mem_critical_pressure_counter()); Clock::resume(); driver.stop(); driver.join(); }
// This test confirms that setting no values for the soft and hard // limits implies an unlimited resource. TEST_F(PosixRLimitsIsolatorTest, UnsetLimits) { Try<Owned<cluster::Master>> master = StartMaster(); ASSERT_SOME(master); slave::Flags flags = CreateSlaveFlags(); flags.isolation = "posix/rlimits"; Owned<MasterDetector> detector = master.get()->createDetector(); Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), flags); ASSERT_SOME(slave); MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL); EXPECT_CALL(sched, registered(_, _, _)); Future<vector<Offer>> offers; EXPECT_CALL(sched, resourceOffers(_, _)) .WillOnce(FutureArg<1>(&offers)) .WillRepeatedly(Return()); // Ignore subsequent offers. driver.start(); AWAIT_READY(offers); ASSERT_NE(0u, offers->size()); TaskInfo task = createTask( offers.get()[0].slave_id(), offers.get()[0].resources(), "exit `ulimit -c | grep -q unlimited`"); // Force usage of C locale as we interpret a potentially translated // string in the task's command. mesos::Environment::Variable* locale = task.mutable_command()->mutable_environment()->add_variables(); locale->set_name("LC_ALL"); locale->set_value("C"); ContainerInfo* container = task.mutable_container(); container->set_type(ContainerInfo::MESOS); // Setting rlimit for core without soft or hard limit signifies // unlimited range. RLimitInfo rlimitInfo; RLimitInfo::RLimit* rlimit = rlimitInfo.add_rlimits(); rlimit->set_type(RLimitInfo::RLimit::RLMT_CORE); container->mutable_rlimit_info()->CopyFrom(rlimitInfo); Future<TaskStatus> statusRunning; Future<TaskStatus> statusFinal; EXPECT_CALL(sched, statusUpdate(&driver, _)) .WillOnce(FutureArg<1>(&statusRunning)) .WillOnce(FutureArg<1>(&statusFinal)); driver.launchTasks(offers.get()[0].id(), {task}); AWAIT_READY(statusRunning); EXPECT_EQ(task.task_id(), statusRunning->task_id()); EXPECT_EQ(TASK_RUNNING, statusRunning->state()); AWAIT_READY(statusFinal); EXPECT_EQ(task.task_id(), statusFinal->task_id()); EXPECT_EQ(TASK_FINISHED, statusFinal->state()); driver.stop(); driver.join(); }
// This test verifies that sandbox path volume allows two containers // nested under the same parent container to share data. // TODO(jieyu): Parameterize this test to test both linux and posix // launcher and filesystem isolator. TEST_F(VolumeSandboxPathIsolatorTest, SharedVolume) { slave::Flags flags = CreateSlaveFlags(); flags.isolation = "volume/sandbox_path"; Fetcher fetcher; Try<MesosContainerizer*> create = MesosContainerizer::create( flags, true, &fetcher); ASSERT_SOME(create); Owned<MesosContainerizer> containerizer(create.get()); SlaveState state; state.id = SlaveID(); AWAIT_READY(containerizer->recover(state)); ContainerID containerId; containerId.set_value(UUID::random().toString()); ExecutorInfo executor = createExecutorInfo("executor", "sleep 99", "cpus:1"); Try<string> directory = environment->mkdtemp(); ASSERT_SOME(directory); Future<bool> launch = containerizer->launch( containerId, None(), executor, directory.get(), None(), state.id, map<string, string>(), true); // TODO(benh): Ever want to check not-checkpointing? AWAIT_ASSERT_TRUE(launch); ContainerID nestedContainerId1; nestedContainerId1.mutable_parent()->CopyFrom(containerId); nestedContainerId1.set_value(UUID::random().toString()); ContainerInfo containerInfo; containerInfo.set_type(ContainerInfo::MESOS); Volume* volume = containerInfo.add_volumes(); volume->set_mode(Volume::RW); volume->set_container_path("parent"); Volume::Source* source = volume->mutable_source(); source->set_type(Volume::Source::SANDBOX_PATH); Volume::Source::SandboxPath* sandboxPath = source->mutable_sandbox_path(); sandboxPath->set_type(Volume::Source::SandboxPath::PARENT); sandboxPath->set_path("shared"); launch = containerizer->launch( nestedContainerId1, createCommandInfo("touch parent/file; sleep 1000"), containerInfo, None(), state.id); AWAIT_ASSERT_TRUE(launch); ContainerID nestedContainerId2; nestedContainerId2.mutable_parent()->CopyFrom(containerId); nestedContainerId2.set_value(UUID::random().toString()); launch = containerizer->launch( nestedContainerId2, createCommandInfo( "while true; do if [ -f parent/file ]; then exit 0; fi; done"), containerInfo, None(), state.id); AWAIT_ASSERT_TRUE(launch); Future<Option<ContainerTermination>> wait = containerizer->wait(nestedContainerId2); AWAIT_READY(wait); ASSERT_SOME(wait.get()); ASSERT_TRUE(wait.get()->has_status()); EXPECT_WEXITSTATUS_EQ(0, wait.get()->status()); wait = containerizer->wait(containerId); containerizer->destroy(containerId); AWAIT_READY(wait); ASSERT_SOME(wait.get()); ASSERT_TRUE(wait.get()->has_status()); EXPECT_WTERMSIG_EQ(SIGKILL, wait.get()->status()); }
TEST_F(StatusUpdateManagerTest, CheckpointStatusUpdate) { Try<PID<Master> > master = StartMaster(); ASSERT_SOME(master); MockExecutor exec(DEFAULT_EXECUTOR_ID); // Require flags to retrieve work_dir when recovering // the checkpointed data. slave::Flags flags = CreateSlaveFlags(); Try<PID<Slave> > slave = StartSlave(&exec, flags); ASSERT_SOME(slave); FrameworkInfo frameworkInfo = DEFAULT_FRAMEWORK_INFO; frameworkInfo.set_checkpoint(true); // Enable checkpointing. MockScheduler sched; MesosSchedulerDriver driver( &sched, frameworkInfo, master.get(), DEFAULT_CREDENTIAL); Future<FrameworkID> frameworkId; EXPECT_CALL(sched, registered(_, _, _)) .WillOnce(FutureArg<1>(&frameworkId)); Future<vector<Offer> > offers; EXPECT_CALL(sched, resourceOffers(_, _)) .WillOnce(FutureArg<1>(&offers)) .WillRepeatedly(Return()); // Ignore subsequent offers. driver.start(); AWAIT_READY(frameworkId); AWAIT_READY(offers); EXPECT_NE(0u, offers.get().size()); EXPECT_CALL(exec, registered(_, _, _, _)) .Times(1); EXPECT_CALL(exec, launchTask(_, _)) .WillOnce(SendStatusUpdateFromTask(TASK_RUNNING)); Future<TaskStatus> status; EXPECT_CALL(sched, statusUpdate(_, _)) .WillOnce(FutureArg<1>(&status)); Future<Nothing> _statusUpdateAcknowledgement = FUTURE_DISPATCH(slave.get(), &Slave::_statusUpdateAcknowledgement); driver.launchTasks(offers.get()[0].id(), createTasks(offers.get()[0])); AWAIT_READY(status); EXPECT_EQ(TASK_RUNNING, status.get().state()); AWAIT_READY(_statusUpdateAcknowledgement); // Ensure that both the status update and its acknowledgement are // correctly checkpointed. Result<slave::state::State> state = slave::state::recover(slave::paths::getMetaRootDir(flags.work_dir), true); ASSERT_SOME(state); ASSERT_SOME(state.get().slave); ASSERT_TRUE(state.get().slave.get().frameworks.contains(frameworkId.get())); slave::state::FrameworkState frameworkState = state.get().slave.get().frameworks.get(frameworkId.get()).get(); ASSERT_EQ(1u, frameworkState.executors.size()); slave::state::ExecutorState executorState = frameworkState.executors.begin()->second; ASSERT_EQ(1u, executorState.runs.size()); slave::state::RunState runState = executorState.runs.begin()->second; ASSERT_EQ(1u, runState.tasks.size()); slave::state::TaskState taskState = runState.tasks.begin()->second; EXPECT_EQ(1u, taskState.updates.size()); EXPECT_EQ(1u, taskState.acks.size()); EXPECT_CALL(exec, shutdown(_)) .Times(AtMost(1)); driver.stop(); driver.join(); Shutdown(); }
TEST_P(CpuIsolatorTest, ROOT_UserCpuUsage) { Try<Owned<cluster::Master>> master = StartMaster(); ASSERT_SOME(master); slave::Flags flags = CreateSlaveFlags(); flags.isolation = GetParam(); Fetcher fetcher(flags); Try<MesosContainerizer*> _containerizer = MesosContainerizer::create(flags, true, &fetcher); ASSERT_SOME(_containerizer); Owned<MesosContainerizer> containerizer(_containerizer.get()); Owned<MasterDetector> detector = master.get()->createDetector(); Try<Owned<cluster::Slave>> slave = StartSlave( detector.get(), containerizer.get()); ASSERT_SOME(slave); MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL); EXPECT_CALL(sched, registered(&driver, _, _)); Future<vector<Offer>> offers; EXPECT_CALL(sched, resourceOffers(&driver, _)) .WillOnce(FutureArg<1>(&offers)) .WillRepeatedly(Return()); // Ignore subsequent offers. driver.start(); AWAIT_READY(offers); ASSERT_FALSE(offers->empty()); // Max out a single core in userspace. This will run for at most one // second. TaskInfo task = createTask( offers.get()[0], "while true ; do true ; done & sleep 60"); Future<TaskStatus> statusRunning; EXPECT_CALL(sched, statusUpdate(&driver, _)) .WillOnce(FutureArg<1>(&statusRunning)); driver.launchTasks(offers.get()[0].id(), {task}); AWAIT_READY(statusRunning); EXPECT_EQ(TASK_RUNNING, statusRunning->state()); Future<hashset<ContainerID>> containers = containerizer->containers(); AWAIT_READY(containers); ASSERT_EQ(1u, containers->size()); ContainerID containerId = *(containers->begin()); // Wait up to 1 second for the child process to induce 1/8 of a // second of user cpu time. ResourceStatistics statistics; Duration waited = Duration::zero(); do { Future<ResourceStatistics> usage = containerizer->usage(containerId); AWAIT_READY(usage); statistics = usage.get(); // If we meet our usage expectations, we're done! if (statistics.cpus_user_time_secs() >= 0.125) { break; } os::sleep(Milliseconds(200)); waited += Milliseconds(200); } while (waited < Seconds(1)); EXPECT_LE(0.125, statistics.cpus_user_time_secs()); driver.stop(); driver.join(); }
// This test ensures that the command executor sends TASK_KILLING // to frameworks that support the capability. // TODO(hausdorff): Enable test. The executor tests use the replicated log // by default. This is not currently supported on Windows, so they will all // fail until that changes. TEST_P_TEMP_DISABLED_ON_WINDOWS(CommandExecutorTest, TaskKillingCapability) { Try<Owned<cluster::Master>> master = StartMaster(); ASSERT_SOME(master); Owned<MasterDetector> detector = master.get()->createDetector(); slave::Flags flags = CreateSlaveFlags(); flags.http_command_executor = GetParam(); Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), flags); ASSERT_SOME(slave); // Start the framework with the task killing capability. FrameworkInfo::Capability capability; capability.set_type(FrameworkInfo::Capability::TASK_KILLING_STATE); FrameworkInfo frameworkInfo = DEFAULT_FRAMEWORK_INFO; frameworkInfo.add_capabilities()->CopyFrom(capability); MockScheduler sched; MesosSchedulerDriver driver( &sched, frameworkInfo, master.get()->pid, DEFAULT_CREDENTIAL); EXPECT_CALL(sched, registered(&driver, _, _)); Future<vector<Offer>> offers; EXPECT_CALL(sched, resourceOffers(&driver, _)) .WillOnce(FutureArg<1>(&offers)) .WillRepeatedly(Return()); // Ignore subsequent offers. driver.start(); AWAIT_READY(offers); EXPECT_EQ(1u, offers->size()); // Launch a task with the command executor. TaskInfo task = createTask( offers->front().slave_id(), offers->front().resources(), "sleep 1000"); Future<TaskStatus> statusRunning; EXPECT_CALL(sched, statusUpdate(_, _)) .WillOnce(FutureArg<1>(&statusRunning)); driver.launchTasks(offers->front().id(), {task}); AWAIT_READY(statusRunning); EXPECT_EQ(TASK_RUNNING, statusRunning->state()); Future<TaskStatus> statusKilling, statusKilled; EXPECT_CALL(sched, statusUpdate(_, _)) .WillOnce(FutureArg<1>(&statusKilling)) .WillOnce(FutureArg<1>(&statusKilled)); driver.killTask(task.task_id()); AWAIT_READY(statusKilling); EXPECT_EQ(TASK_KILLING, statusKilling->state()); AWAIT_READY(statusKilled); EXPECT_EQ(TASK_KILLED, statusKilled->state()); driver.stop(); driver.join(); }
// This test ensures that the HTTP command executor can self terminate // after it gets the ACK for the terminal status update from agent. TEST_F_TEMP_DISABLED_ON_WINDOWS(HTTPCommandExecutorTest, TerminateWithACK) { Try<Owned<cluster::Master>> master = StartMaster(); ASSERT_SOME(master); slave::Flags flags = CreateSlaveFlags(); flags.http_command_executor = true; Fetcher fetcher; Try<MesosContainerizer*> _containerizer = MesosContainerizer::create(flags, false, &fetcher); CHECK_SOME(_containerizer); Owned<MesosContainerizer> containerizer(_containerizer.get()); StandaloneMasterDetector detector(master.get()->pid); MockSlave slave(flags, &detector, containerizer.get()); spawn(slave); MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL); EXPECT_CALL(sched, registered(&driver, _, _)); Future<vector<Offer>> offers; EXPECT_CALL(sched, resourceOffers(&driver, _)) .WillOnce(FutureArg<1>(&offers)) .WillRepeatedly(Return()); // Ignore subsequent offers. driver.start(); AWAIT_READY(offers); EXPECT_EQ(1u, offers->size()); // Launch a short lived task. TaskInfo task = createTask( offers->front().slave_id(), offers->front().resources(), "sleep 1"); Future<TaskStatus> statusRunning; Future<TaskStatus> statusFinished; EXPECT_CALL(sched, statusUpdate(_, _)) .WillOnce(FutureArg<1>(&statusRunning)) .WillOnce(FutureArg<1>(&statusFinished)); Future<Future<Option<ContainerTermination>>> termination; EXPECT_CALL(slave, executorTerminated(_, _, _)) .WillOnce(FutureArg<2>(&termination)); driver.launchTasks(offers->front().id(), {task}); // Scheduler should first receive TASK_RUNNING followed by TASK_FINISHED. AWAIT_READY(statusRunning); EXPECT_EQ(TASK_RUNNING, statusRunning->state()); AWAIT_READY(statusFinished); EXPECT_EQ(TASK_FINISHED, statusFinished->state()); // The executor should self terminate with 0 as exit status once // it gets the ACK for the terminal status update from agent. AWAIT_READY(termination); ASSERT_TRUE(termination.get().isReady()); EXPECT_EQ(0, termination.get().get().get().status()); driver.stop(); driver.join(); terminate(slave); wait(slave); }
// Test executor environment decorator hook and remove executor hook // for slave. We expect the environment-decorator hook to create a // temporary file and the remove-executor hook to delete that file. TEST_F(HookTest, DISABLED_VerifySlaveLaunchExecutorHook) { master::Flags masterFlags = CreateMasterFlags(); Try<PID<Master>> master = StartMaster(masterFlags); ASSERT_SOME(master); slave::Flags slaveFlags = CreateSlaveFlags(); MockExecutor exec(DEFAULT_EXECUTOR_ID); TestContainerizer containerizer(&exec); Try<PID<Slave>> slave = StartSlave(&containerizer); ASSERT_SOME(slave); MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get(), DEFAULT_CREDENTIAL); EXPECT_CALL(sched, registered(&driver, _, _)); Future<vector<Offer>> offers; EXPECT_CALL(sched, resourceOffers(&driver, _)) .WillOnce(FutureArg<1>(&offers)) .WillRepeatedly(Return()); // Ignore subsequent offers. driver.start(); AWAIT_READY(offers); EXPECT_NE(0u, offers.get().size()); // Launch a task with the command executor. TaskInfo task; task.set_name(""); task.mutable_task_id()->set_value("1"); task.mutable_slave_id()->CopyFrom(offers.get()[0].slave_id()); task.mutable_resources()->CopyFrom(offers.get()[0].resources()); task.mutable_executor()->CopyFrom(DEFAULT_EXECUTOR_INFO); vector<TaskInfo> tasks; tasks.push_back(task); EXPECT_CALL(exec, launchTask(_, _)); Future<ExecutorInfo> executorInfo; EXPECT_CALL(exec, registered(_, _, _, _)) .WillOnce(FutureArg<1>(&executorInfo)); // On successful completion of the "slaveLaunchExecutorHook", the // test hook will send a HookExecuted message to itself. We wait // until that message is intercepted by the testing infrastructure. Future<HookExecuted> hookFuture = FUTURE_PROTOBUF(HookExecuted(), _, _); driver.launchTasks(offers.get()[0].id(), tasks); AWAIT_READY(executorInfo); driver.stop(); driver.join(); Shutdown(); // Must shutdown before 'containerizer' gets deallocated. // Now wait for the hook to finish execution. AWAIT_READY(hookFuture); }
// This test confirms that if a task exceeds configured resource // limits it is forcibly terminated. TEST_F(PosixRLimitsIsolatorTest, TaskExceedingLimit) { Try<Owned<cluster::Master>> master = StartMaster(); ASSERT_SOME(master); slave::Flags flags = CreateSlaveFlags(); flags.isolation = "posix/rlimits"; Owned<MasterDetector> detector = master.get()->createDetector(); Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), flags); ASSERT_SOME(slave); MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL); EXPECT_CALL(sched, registered(_, _, _)); Future<vector<Offer>> offers; EXPECT_CALL(sched, resourceOffers(_, _)) .WillOnce(FutureArg<1>(&offers)) .WillRepeatedly(Return()); // Ignore subsequent offers. driver.start(); AWAIT_READY(offers); ASSERT_NE(0u, offers->size()); // The task attempts to use an infinite amount of CPU time. TaskInfo task = createTask( offers.get()[0].slave_id(), offers.get()[0].resources(), "while true; do true; done"); ContainerInfo* container = task.mutable_container(); container->set_type(ContainerInfo::MESOS); // Limit the process to use maximally 1 second of CPU time. RLimitInfo rlimitInfo; RLimitInfo::RLimit* cpuLimit = rlimitInfo.add_rlimits(); cpuLimit->set_type(RLimitInfo::RLimit::RLMT_CPU); cpuLimit->set_soft(1); cpuLimit->set_hard(1); container->mutable_rlimit_info()->CopyFrom(rlimitInfo); Future<TaskStatus> statusRunning; Future<TaskStatus> statusFailed; EXPECT_CALL(sched, statusUpdate(&driver, _)) .WillOnce(FutureArg<1>(&statusRunning)) .WillOnce(FutureArg<1>(&statusFailed)); driver.launchTasks(offers.get()[0].id(), {task}); AWAIT_READY(statusRunning); EXPECT_EQ(task.task_id(), statusRunning->task_id()); EXPECT_EQ(TASK_RUNNING, statusRunning->state()); AWAIT_READY(statusFailed); EXPECT_EQ(task.task_id(), statusFailed->task_id()); EXPECT_EQ(TASK_FAILED, statusFailed->state()); driver.stop(); driver.join(); }
// This test checks the behavior of passed invalid limits. TEST_F(PosixRLimitsIsolatorTest, InvalidLimits) { Try<Owned<cluster::Master>> master = StartMaster(); ASSERT_SOME(master); slave::Flags flags = CreateSlaveFlags(); flags.isolation = "posix/rlimits"; Owned<MasterDetector> detector = master.get()->createDetector(); Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), flags); ASSERT_SOME(slave); MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL); EXPECT_CALL(sched, registered(_, _, _)); Future<vector<Offer>> offers; EXPECT_CALL(sched, resourceOffers(_, _)) .WillOnce(FutureArg<1>(&offers)) .WillRepeatedly(Return()); // Ignore subsequent offers. driver.start(); AWAIT_READY(offers); ASSERT_NE(0u, offers->size()); TaskInfo task = createTask( offers.get()[0].slave_id(), offers.get()[0].resources(), "true"); ContainerInfo* container = task.mutable_container(); container->set_type(ContainerInfo::MESOS); // Set impossible limit soft > hard. RLimitInfo rlimitInfo; RLimitInfo::RLimit* rlimit = rlimitInfo.add_rlimits(); rlimit->set_type(RLimitInfo::RLimit::RLMT_CPU); rlimit->set_soft(100); rlimit->set_hard(1); container->mutable_rlimit_info()->CopyFrom(rlimitInfo); Future<TaskStatus> taskStatus; EXPECT_CALL(sched, statusUpdate(&driver, _)) .WillOnce(FutureArg<1>(&taskStatus)); driver.launchTasks(offers.get()[0].id(), {task}); AWAIT_READY(taskStatus); EXPECT_EQ(task.task_id(), taskStatus->task_id()); EXPECT_EQ(TASK_FAILED, taskStatus->state()); EXPECT_EQ(TaskStatus::REASON_EXECUTOR_TERMINATED, taskStatus->reason()); driver.stop(); driver.join(); }
// Test that memory pressure listening is restarted after recovery. TEST_F(MemoryPressureMesosTest, CGROUPS_ROOT_SlaveRecovery) { Try<Owned<cluster::Master>> master = StartMaster(); ASSERT_SOME(master); slave::Flags flags = CreateSlaveFlags(); // We only care about memory cgroup for this test. flags.isolation = "cgroups/mem"; Fetcher fetcher(flags); Try<MesosContainerizer*> _containerizer = MesosContainerizer::create(flags, true, &fetcher); ASSERT_SOME(_containerizer); Owned<MesosContainerizer> containerizer(_containerizer.get()); Owned<MasterDetector> detector = master.get()->createDetector(); Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), containerizer.get(), flags); ASSERT_SOME(slave); MockScheduler sched; // Enable checkpointing for the framework. FrameworkInfo frameworkInfo = DEFAULT_FRAMEWORK_INFO; frameworkInfo.set_checkpoint(true); MesosSchedulerDriver driver( &sched, frameworkInfo, master.get()->pid, DEFAULT_CREDENTIAL); EXPECT_CALL(sched, registered(_, _, _)); Future<vector<Offer>> offers; EXPECT_CALL(sched, resourceOffers(_, _)) .WillOnce(FutureArg<1>(&offers)) .WillRepeatedly(Return()); // Ignore subsequent offers. driver.start(); AWAIT_READY(offers); ASSERT_FALSE(offers->empty()); Offer offer = offers.get()[0]; // Run a task that triggers memory pressure event. We request 1G // disk because we are going to write a 512 MB file repeatedly. TaskInfo task = createTask( offer.slave_id(), Resources::parse("cpus:1;mem:256;disk:1024").get(), "while true; do dd count=512 bs=1M if=/dev/zero of=./temp; done"); Future<TaskStatus> starting; Future<TaskStatus> running; EXPECT_CALL(sched, statusUpdate(&driver, _)) .WillOnce(FutureArg<1>(&starting)) .WillOnce(FutureArg<1>(&running)) .WillRepeatedly(Return()); // Ignore subsequent updates. Future<Nothing> runningAck = FUTURE_DISPATCH(_, &Slave::_statusUpdateAcknowledgement); Future<Nothing> startingAck = FUTURE_DISPATCH(_, &Slave::_statusUpdateAcknowledgement); driver.launchTasks(offers.get()[0].id(), {task}); AWAIT_READY(starting); EXPECT_EQ(task.task_id(), starting->task_id()); EXPECT_EQ(TASK_STARTING, starting->state()); AWAIT_READY(startingAck); AWAIT_READY(running); EXPECT_EQ(task.task_id(), running->task_id()); EXPECT_EQ(TASK_RUNNING, running->state()); // Wait for the ACK to be checkpointed. AWAIT_READY_FOR(runningAck, Seconds(120)); // We restart the slave to let it recover. slave.get()->terminate(); // Set up so we can wait until the new slave updates the container's // resources (this occurs after the executor has re-registered). Future<Nothing> update = FUTURE_DISPATCH(_, &MesosContainerizerProcess::update); // Use the same flags. _containerizer = MesosContainerizer::create(flags, true, &fetcher); ASSERT_SOME(_containerizer); containerizer.reset(_containerizer.get()); Future<SlaveReregisteredMessage> reregistered = FUTURE_PROTOBUF(SlaveReregisteredMessage(), master.get()->pid, _); slave = StartSlave(detector.get(), containerizer.get(), flags); ASSERT_SOME(slave); AWAIT_READY(reregistered); // Wait until the containerizer is updated. AWAIT_READY(update); Future<hashset<ContainerID>> containers = containerizer->containers(); AWAIT_READY(containers); ASSERT_EQ(1u, containers->size()); ContainerID containerId = *(containers->begin()); // Wait a while for some memory pressure events to occur. Duration waited = Duration::zero(); do { Future<ResourceStatistics> usage = containerizer->usage(containerId); AWAIT_READY(usage); if (usage->mem_low_pressure_counter() > 0) { // We will check the correctness of the memory pressure counters // later, because the memory-hammering task is still active // and potentially incrementing these counters. break; } os::sleep(Milliseconds(100)); waited += Milliseconds(100); } while (waited < Seconds(5)); EXPECT_LE(waited, Seconds(5)); // Pause the clock to ensure that the reaper doesn't reap the exited // command executor and inform the containerizer/slave. Clock::pause(); Clock::settle(); Future<TaskStatus> killed; EXPECT_CALL(sched, statusUpdate(&driver, _)) .WillOnce(FutureArg<1>(&killed)); // Stop the memory-hammering task. driver.killTask(task.task_id()); AWAIT_READY_FOR(killed, Seconds(120)); EXPECT_EQ(task.task_id(), killed->task_id()); EXPECT_EQ(TASK_KILLED, killed->state()); // Now check the correctness of the memory pressure counters. Future<ResourceStatistics> usage = containerizer->usage(containerId); AWAIT_READY(usage); EXPECT_GE(usage->mem_low_pressure_counter(), usage->mem_medium_pressure_counter()); EXPECT_GE(usage->mem_medium_pressure_counter(), usage->mem_critical_pressure_counter()); Clock::resume(); driver.stop(); driver.join(); }
TEST_F(StatusUpdateManagerTest, RetryStatusUpdate) { Try<PID<Master> > master = StartMaster(); ASSERT_SOME(master); MockExecutor exec(DEFAULT_EXECUTOR_ID); slave::Flags flags = CreateSlaveFlags(); Try<PID<Slave> > slave = StartSlave(&exec, flags); ASSERT_SOME(slave); FrameworkInfo frameworkInfo = DEFAULT_FRAMEWORK_INFO; frameworkInfo.set_checkpoint(true); // Enable checkpointing. MockScheduler sched; MesosSchedulerDriver driver( &sched, frameworkInfo, master.get(), DEFAULT_CREDENTIAL); EXPECT_CALL(sched, registered(_, _, _)) .Times(1); Future<vector<Offer> > offers; EXPECT_CALL(sched, resourceOffers(_, _)) .WillOnce(FutureArg<1>(&offers)) .WillRepeatedly(Return()); // Ignore subsequent offers. driver.start(); AWAIT_READY(offers); EXPECT_NE(0u, offers.get().size()); EXPECT_CALL(exec, registered(_, _, _, _)) .Times(1); EXPECT_CALL(exec, launchTask(_, _)) .WillOnce(SendStatusUpdateFromTask(TASK_RUNNING)); Future<StatusUpdateMessage> statusUpdateMessage = DROP_PROTOBUF(StatusUpdateMessage(), master.get(), _); Clock::pause(); driver.launchTasks(offers.get()[0].id(), createTasks(offers.get()[0])); AWAIT_READY(statusUpdateMessage); Future<TaskStatus> status; EXPECT_CALL(sched, statusUpdate(_, _)) .WillOnce(FutureArg<1>(&status)); Clock::advance(slave::STATUS_UPDATE_RETRY_INTERVAL_MIN); AWAIT_READY(status); EXPECT_EQ(TASK_RUNNING, status.get().state()); Clock::resume(); EXPECT_CALL(exec, shutdown(_)) .Times(AtMost(1)); driver.stop(); driver.join(); Shutdown(); }
TEST_P(MemoryIsolatorTest, ROOT_MemUsage) { Try<Owned<cluster::Master>> master = StartMaster(); ASSERT_SOME(master); slave::Flags flags = CreateSlaveFlags(); flags.isolation = GetParam(); Fetcher fetcher(flags); Try<MesosContainerizer*> _containerizer = MesosContainerizer::create(flags, true, &fetcher); ASSERT_SOME(_containerizer); Owned<MesosContainerizer> containerizer(_containerizer.get()); Owned<MasterDetector> detector = master.get()->createDetector(); Try<Owned<cluster::Slave>> slave = StartSlave( detector.get(), containerizer.get()); ASSERT_SOME(slave); MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL); EXPECT_CALL(sched, registered(&driver, _, _)); Future<vector<Offer>> offers; EXPECT_CALL(sched, resourceOffers(&driver, _)) .WillOnce(FutureArg<1>(&offers)) .WillRepeatedly(Return()); // Ignore subsequent offers. driver.start(); AWAIT_READY(offers); ASSERT_FALSE(offers->empty()); TaskInfo task = createTask(offers.get()[0], "sleep 120"); Future<TaskStatus> statusRunning; EXPECT_CALL(sched, statusUpdate(&driver, _)) .WillOnce(FutureArg<1>(&statusRunning)); driver.launchTasks(offers.get()[0].id(), {task}); AWAIT_READY(statusRunning); EXPECT_EQ(TASK_RUNNING, statusRunning->state()); Future<hashset<ContainerID>> containers = containerizer->containers(); AWAIT_READY(containers); ASSERT_EQ(1u, containers->size()); ContainerID containerId = *(containers->begin()); Future<ResourceStatistics> usage = containerizer->usage(containerId); AWAIT_READY(usage); // TODO(jieyu): Consider using a program that predictably increases // RSS so that we can set more meaningful expectation here. EXPECT_LT(0u, usage->mem_rss_bytes()); driver.stop(); driver.join(); }
// This test verifies that the environment secrets are resolved when launching a // task. TEST_F(EnvironmentSecretIsolatorTest, ResolveSecret) { Try<Owned<cluster::Master>> master = StartMaster(); ASSERT_SOME(master); mesos::internal::slave::Flags flags = CreateSlaveFlags(); Fetcher fetcher(flags); Try<SecretResolver*> secretResolver = SecretResolver::create(); EXPECT_SOME(secretResolver); Try<MesosContainerizer*> containerizer = MesosContainerizer::create(flags, false, &fetcher, secretResolver.get()); EXPECT_SOME(containerizer); Owned<MasterDetector> detector = master.get()->createDetector(); Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), containerizer.get()); ASSERT_SOME(slave); MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL); EXPECT_CALL(sched, registered(&driver, _, _)); Future<std::vector<Offer>> offers; EXPECT_CALL(sched, resourceOffers(&driver, _)) .WillOnce(FutureArg<1>(&offers)) .WillRepeatedly(Return()); // Ignore subsequent offers. driver.start(); AWAIT_READY(offers); EXPECT_FALSE(offers->empty()); const string commandString = strings::format( "env; test \"$%s\" = \"%s\"", SECRET_ENV_NAME, SECRET_VALUE).get(); CommandInfo command; command.set_value(commandString); // Request a secret. // TODO(kapil): Update createEnvironment() to support secrets. mesos::Environment::Variable *env = command.mutable_environment()->add_variables(); env->set_name(SECRET_ENV_NAME); env->set_type(mesos::Environment::Variable::SECRET); mesos::Secret* secret = env->mutable_secret(); secret->set_type(Secret::VALUE); secret->mutable_value()->set_data(SECRET_VALUE); TaskInfo task = createTask( offers.get()[0].slave_id(), Resources::parse("cpus:0.1;mem:32").get(), command); // NOTE: Successful tasks will output two status updates. Future<TaskStatus> statusRunning; Future<TaskStatus> statusFinished; EXPECT_CALL(sched, statusUpdate(&driver, _)) .WillOnce(FutureArg<1>(&statusRunning)) .WillOnce(FutureArg<1>(&statusFinished)); driver.launchTasks(offers.get()[0].id(), {task}); AWAIT_READY(statusRunning); EXPECT_EQ(TASK_RUNNING, statusRunning.get().state()); AWAIT_READY(statusFinished); EXPECT_EQ(TASK_FINISHED, statusFinished.get().state()); driver.stop(); driver.join(); }
// Tests that the default container logger writes files into the sandbox. TEST_F(ContainerLoggerTest, DefaultToSandbox) { // Create a master, agent, and framework. Try<PID<Master>> master = StartMaster(); ASSERT_SOME(master); Future<SlaveRegisteredMessage> slaveRegisteredMessage = FUTURE_PROTOBUF(SlaveRegisteredMessage(), _, _); // We'll need access to these flags later. slave::Flags flags = CreateSlaveFlags(); Fetcher fetcher; // We use an actual containerizer + executor since we want something to run. Try<MesosContainerizer*> containerizer = MesosContainerizer::create(flags, false, &fetcher); CHECK_SOME(containerizer); Try<PID<Slave>> slave = StartSlave(containerizer.get(), flags); ASSERT_SOME(slave); AWAIT_READY(slaveRegisteredMessage); SlaveID slaveId = slaveRegisteredMessage.get().slave_id(); MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get(), DEFAULT_CREDENTIAL); Future<FrameworkID> frameworkId; EXPECT_CALL(sched, registered(&driver, _, _)) .WillOnce(FutureArg<1>(&frameworkId)); // Wait for an offer, and start a task. Future<vector<Offer>> offers; EXPECT_CALL(sched, resourceOffers(&driver, _)) .WillOnce(FutureArg<1>(&offers)) .WillRepeatedly(Return()); // Ignore subsequent offers. driver.start(); AWAIT_READY(frameworkId); AWAIT_READY(offers); EXPECT_NE(0u, offers.get().size()); // We'll start a task that outputs to stdout. TaskInfo task = createTask(offers.get()[0], "echo 'Hello World!'"); Future<TaskStatus> status; EXPECT_CALL(sched, statusUpdate(&driver, _)) .WillOnce(FutureArg<1>(&status)) .WillRepeatedly(Return()); // Ignore subsequent updates. driver.launchTasks(offers.get()[0].id(), {task}); AWAIT_READY(status); EXPECT_EQ(TASK_RUNNING, status.get().state()); // Check that the sandbox was written to. string sandboxDirectory = path::join( slave::paths::getExecutorPath( flags.work_dir, slaveId, frameworkId.get(), status->executor_id()), "runs", "latest"); ASSERT_TRUE(os::exists(sandboxDirectory)); string stdoutPath = path::join(sandboxDirectory, "stdout"); ASSERT_TRUE(os::exists(stdoutPath)); Result<string> stdout = os::read(stdoutPath); ASSERT_SOME(stdout); EXPECT_TRUE(strings::contains(stdout.get(), "Hello World!")); driver.stop(); driver.join(); Shutdown(); }
// Test that memory pressure listening is restarted after recovery. TEST_F(MemoryPressureMesosTest, CGROUPS_ROOT_SlaveRecovery) { Try<PID<Master>> master = StartMaster(); ASSERT_SOME(master); slave::Flags flags = CreateSlaveFlags(); // We only care about memory cgroup for this test. flags.isolation = "cgroups/mem"; flags.slave_subsystems = None(); Fetcher fetcher; Try<MesosContainerizer*> containerizer1 = MesosContainerizer::create(flags, true, &fetcher); ASSERT_SOME(containerizer1); Try<PID<Slave>> slave = StartSlave(containerizer1.get(), flags); ASSERT_SOME(slave); MockScheduler sched; // Enable checkpointing for the framework. FrameworkInfo frameworkInfo = DEFAULT_FRAMEWORK_INFO; frameworkInfo.set_checkpoint(true); MesosSchedulerDriver driver( &sched, frameworkInfo, master.get(), DEFAULT_CREDENTIAL); EXPECT_CALL(sched, registered(_, _, _)); Future<vector<Offer>> offers; EXPECT_CALL(sched, resourceOffers(_, _)) .WillOnce(FutureArg<1>(&offers)) .WillRepeatedly(Return()); // Ignore subsequent offers. driver.start(); AWAIT_READY(offers); EXPECT_NE(0u, offers.get().size()); Offer offer = offers.get()[0]; // Run a task that triggers memory pressure event. We request 1G // disk because we are going to write a 512 MB file repeatedly. TaskInfo task = createTask( offer.slave_id(), Resources::parse("cpus:1;mem:256;disk:1024").get(), "while true; do dd count=512 bs=1M if=/dev/zero of=./temp; done"); Future<TaskStatus> status; EXPECT_CALL(sched, statusUpdate(&driver, _)) .WillOnce(FutureArg<1>(&status)) .WillRepeatedly(Return()); // Ignore subsequent updates. driver.launchTasks(offers.get()[0].id(), {task}); AWAIT_READY(status); EXPECT_EQ(task.task_id(), status.get().task_id()); EXPECT_EQ(TASK_RUNNING, status.get().state()); // We restart the slave to let it recover. Stop(slave.get()); delete containerizer1.get(); // Set up so we can wait until the new slave updates the container's // resources (this occurs after the executor has re-registered). Future<Nothing> update = FUTURE_DISPATCH(_, &MesosContainerizerProcess::update); // Use the same flags. Try<MesosContainerizer*> containerizer2 = MesosContainerizer::create(flags, true, &fetcher); ASSERT_SOME(containerizer2); slave = StartSlave(containerizer2.get(), flags); ASSERT_SOME(slave); // Wait until the containerizer is updated. AWAIT_READY(update); Future<hashset<ContainerID>> containers = containerizer2.get()->containers(); AWAIT_READY(containers); ASSERT_EQ(1u, containers.get().size()); ContainerID containerId = *(containers.get().begin()); // Wait a while for some memory pressure events to occur. Duration waited = Duration::zero(); do { Future<ResourceStatistics> usage = containerizer2.get()->usage(containerId); AWAIT_READY(usage); if (usage.get().mem_low_pressure_counter() > 0) { // We will check the correctness of the memory pressure counters // later, because the memory-hammering task is still active // and potentially incrementing these counters. break; } os::sleep(Milliseconds(100)); waited += Milliseconds(100); } while (waited < Seconds(5)); EXPECT_LE(waited, Seconds(5)); // Stop the memory-hammering task. driver.killTask(task.task_id()); // Process any queued up events through before proceeding. process::Clock::pause(); process::Clock::settle(); process::Clock::resume(); // Now check the correctness of the memory pressure counters. Future<ResourceStatistics> usage = containerizer2.get()->usage(containerId); AWAIT_READY(usage); EXPECT_GE(usage.get().mem_low_pressure_counter(), usage.get().mem_medium_pressure_counter()); EXPECT_GE(usage.get().mem_medium_pressure_counter(), usage.get().mem_critical_pressure_counter()); driver.stop(); driver.join(); Shutdown(); delete containerizer2.get(); }
// This test verifies that the image specified in the volume will be // properly provisioned and mounted into the container if container // root filesystem is not specified. TEST_P(VolumeImageIsolatorTest, ROOT_ImageInVolumeWithoutRootFilesystem) { string registry = path::join(sandbox.get(), "registry"); AWAIT_READY(DockerArchive::create(registry, "test_image")); slave::Flags flags = CreateSlaveFlags(); flags.isolation = "filesystem/linux,volume/image,docker/runtime"; flags.docker_registry = registry; flags.docker_store_dir = path::join(sandbox.get(), "store"); flags.image_providers = "docker"; Fetcher fetcher(flags); Try<MesosContainerizer*> create = MesosContainerizer::create(flags, true, &fetcher); ASSERT_SOME(create); Owned<Containerizer> containerizer(create.get()); ContainerID containerId; containerId.set_value(id::UUID::random().toString()); ContainerInfo container = createContainerInfo( None(), {createVolumeFromDockerImage("rootfs", "test_image", Volume::RW)}); CommandInfo command = createCommandInfo("test -d rootfs/bin"); ExecutorInfo executor = createExecutorInfo( "test_executor", nesting ? createCommandInfo("sleep 1000") : command); if (!nesting) { executor.mutable_container()->CopyFrom(container); } string directory = path::join(flags.work_dir, "sandbox"); ASSERT_SOME(os::mkdir(directory)); Future<Containerizer::LaunchResult> launch = containerizer->launch( containerId, createContainerConfig(None(), executor, directory), map<string, string>(), None()); AWAIT_ASSERT_EQ(Containerizer::LaunchResult::SUCCESS, launch); Future<Option<ContainerTermination>> wait = containerizer->wait(containerId); if (nesting) { ContainerID nestedContainerId; nestedContainerId.mutable_parent()->CopyFrom(containerId); nestedContainerId.set_value(id::UUID::random().toString()); launch = containerizer->launch( nestedContainerId, createContainerConfig(command, container), map<string, string>(), None()); AWAIT_ASSERT_EQ(Containerizer::LaunchResult::SUCCESS, launch); wait = containerizer->wait(nestedContainerId); } AWAIT_READY(wait); ASSERT_SOME(wait.get()); ASSERT_TRUE(wait->get().has_status()); EXPECT_WEXITSTATUS_EQ(0, wait->get().status()); if (nesting) { Future<Option<ContainerTermination>> termination = containerizer->destroy(containerId); AWAIT_READY(termination); ASSERT_SOME(termination.get()); ASSERT_TRUE(termination->get().has_status()); EXPECT_WTERMSIG_EQ(SIGKILL, termination.get()->status()); } }
// This test verifies that docker image default entrypoint is executed // correctly using registry puller. This corresponds to the case in runtime // isolator logic table: sh=0, value=0, argv=1, entrypoint=1, cmd=0. TEST_F(DockerRuntimeIsolatorTest, ROOT_CURL_INTERNET_DockerDefaultEntryptRegistryPuller) { Try<Owned<cluster::Master>> master = StartMaster(); ASSERT_SOME(master); slave::Flags flags = CreateSlaveFlags(); flags.isolation = "docker/runtime,filesystem/linux"; flags.image_providers = "docker"; flags.docker_store_dir = path::join(os::getcwd(), "store"); Owned<MasterDetector> detector = master.get()->createDetector(); Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), flags); ASSERT_SOME(slave); MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL); EXPECT_CALL(sched, registered(&driver, _, _)); Future<vector<Offer>> offers; EXPECT_CALL(sched, resourceOffers(&driver, _)) .WillOnce(FutureArg<1>(&offers)) .WillRepeatedly(Return()); // Ignore subsequent offers. driver.start(); AWAIT_READY(offers); ASSERT_EQ(1u, offers->size()); const Offer& offer = offers.get()[0]; TaskInfo task; task.set_name("test-task"); task.mutable_task_id()->set_value(UUID::random().toString()); task.mutable_slave_id()->CopyFrom(offer.slave_id()); task.mutable_resources()->CopyFrom(Resources::parse("cpus:1;mem:128").get()); task.mutable_command()->set_shell(false); task.mutable_command()->add_arguments("hello world"); Image image; image.set_type(Image::DOCKER); // 'mesosphere/inky' image is used in docker containerizer test, which // contains entrypoint as 'echo' and cmd as null. image.mutable_docker()->set_name("mesosphere/inky"); ContainerInfo* container = task.mutable_container(); container->set_type(ContainerInfo::MESOS); container->mutable_mesos()->mutable_image()->CopyFrom(image); Future<TaskStatus> statusRunning; Future<TaskStatus> statusFinished; EXPECT_CALL(sched, statusUpdate(&driver, _)) .WillOnce(FutureArg<1>(&statusRunning)) .WillOnce(FutureArg<1>(&statusFinished)); driver.launchTasks(offer.id(), {task}); AWAIT_READY_FOR(statusRunning, Seconds(60)); EXPECT_EQ(task.task_id(), statusRunning->task_id()); EXPECT_EQ(TASK_RUNNING, statusRunning->state()); AWAIT_READY(statusFinished); EXPECT_EQ(task.task_id(), statusFinished->task_id()); EXPECT_EQ(TASK_FINISHED, statusFinished->state()); driver.stop(); driver.join(); }
// This test ensures that a task will transition straight from `TASK_KILLING` to // `TASK_KILLED`, even if the health check begins to fail during the kill policy // grace period. // // TODO(gkleiman): this test takes about 7 seconds to run, consider using mock // tasks and health checkers to speed it up. TEST_P(CommandExecutorTest, NoTransitionFromKillingToRunning) { Try<Owned<cluster::Master>> master = StartMaster(); ASSERT_SOME(master); Owned<MasterDetector> detector = master.get()->createDetector(); slave::Flags flags = CreateSlaveFlags(); flags.http_command_executor = GetParam(); Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), flags); ASSERT_SOME(slave); // Start the framework with the task killing capability. FrameworkInfo::Capability capability; capability.set_type(FrameworkInfo::Capability::TASK_KILLING_STATE); FrameworkInfo frameworkInfo = DEFAULT_FRAMEWORK_INFO; frameworkInfo.add_capabilities()->CopyFrom(capability); MockScheduler sched; MesosSchedulerDriver driver( &sched, frameworkInfo, master.get()->pid, DEFAULT_CREDENTIAL); EXPECT_CALL(sched, registered(&driver, _, _)); Future<vector<Offer>> offers; EXPECT_CALL(sched, resourceOffers(&driver, _)) .WillOnce(FutureArg<1>(&offers)) .WillRepeatedly(Return()); // Ignore subsequent offers. driver.start(); AWAIT_READY(offers); EXPECT_EQ(1u, offers->size()); const string command = strings::format( "%s %s --sleep_duration=15", getTestHelperPath("test-helper"), KillPolicyTestHelper::NAME).get(); TaskInfo task = createTask(offers->front(), command); // Create a health check that succeeds until a temporary file is removed. Try<string> temporaryPath = os::mktemp(path::join(os::getcwd(), "XXXXXX")); ASSERT_SOME(temporaryPath); const string tmpPath = temporaryPath.get(); HealthCheck healthCheck; healthCheck.set_type(HealthCheck::COMMAND); healthCheck.mutable_command()->set_value("ls " + tmpPath + " >/dev/null"); healthCheck.set_delay_seconds(0); healthCheck.set_grace_period_seconds(0); healthCheck.set_interval_seconds(0); task.mutable_health_check()->CopyFrom(healthCheck); // Set the kill policy grace period to 5 seconds. KillPolicy killPolicy; killPolicy.mutable_grace_period()->set_nanoseconds(Seconds(5).ns()); task.mutable_kill_policy()->CopyFrom(killPolicy); vector<TaskInfo> tasks; tasks.push_back(task); Future<TaskStatus> statusRunning; Future<TaskStatus> statusHealthy; Future<TaskStatus> statusKilling; Future<TaskStatus> statusKilled; EXPECT_CALL(sched, statusUpdate(&driver, _)) .WillOnce(FutureArg<1>(&statusRunning)) .WillOnce(FutureArg<1>(&statusHealthy)) .WillOnce(FutureArg<1>(&statusKilling)) .WillOnce(FutureArg<1>(&statusKilled)); driver.launchTasks(offers->front().id(), tasks); AWAIT_READY(statusRunning); EXPECT_EQ(TASK_RUNNING, statusRunning.get().state()); AWAIT_READY(statusHealthy); EXPECT_EQ(TASK_RUNNING, statusHealthy.get().state()); EXPECT_TRUE(statusHealthy.get().has_healthy()); EXPECT_TRUE(statusHealthy.get().healthy()); driver.killTask(task.task_id()); AWAIT_READY(statusKilling); EXPECT_EQ(TASK_KILLING, statusKilling->state()); EXPECT_FALSE(statusKilling.get().has_healthy()); // Remove the temporary file, so that the health check fails. os::rm(tmpPath); AWAIT_READY(statusKilled); EXPECT_EQ(TASK_KILLED, statusKilled->state()); EXPECT_FALSE(statusKilled.get().has_healthy()); driver.stop(); driver.join(); }
// This test verifies that docker image default cmd is executed correctly. // This corresponds to the case in runtime isolator logic table: sh=0, // value=0, argv=1, entrypoint=0, cmd=1. TEST_F(DockerRuntimeIsolatorTest, ROOT_DockerDefaultCmdLocalPuller) { Try<Owned<cluster::Master>> master = StartMaster(); ASSERT_SOME(master); const string directory = path::join(os::getcwd(), "archives"); Future<Nothing> testImage = DockerArchive::create(directory, "alpine", "null", "[\"sh\"]"); AWAIT_READY(testImage); ASSERT_TRUE(os::exists(path::join(directory, "alpine.tar"))); slave::Flags flags = CreateSlaveFlags(); flags.isolation = "docker/runtime,filesystem/linux"; flags.image_providers = "docker"; flags.docker_registry = directory; // Make docker store directory as a temparary directory. Because the // manifest of the test image is changeable, the image cached on // previous tests should never be used. flags.docker_store_dir = path::join(os::getcwd(), "store"); Owned<MasterDetector> detector = master.get()->createDetector(); Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), flags); ASSERT_SOME(slave); MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL); EXPECT_CALL(sched, registered(&driver, _, _)); Future<vector<Offer>> offers; EXPECT_CALL(sched, resourceOffers(&driver, _)) .WillOnce(FutureArg<1>(&offers)) .WillRepeatedly(Return()); // Ignore subsequent offers. driver.start(); AWAIT_READY(offers); ASSERT_EQ(1u, offers->size()); const Offer& offer = offers.get()[0]; TaskInfo task; task.set_name("test-task"); task.mutable_task_id()->set_value(UUID::random().toString()); task.mutable_slave_id()->CopyFrom(offer.slave_id()); task.mutable_resources()->CopyFrom(Resources::parse("cpus:1;mem:128").get()); task.mutable_command()->set_shell(false); task.mutable_command()->add_arguments("-c"); task.mutable_command()->add_arguments("echo 'hello world'"); Image image; image.set_type(Image::DOCKER); image.mutable_docker()->set_name("alpine"); ContainerInfo* container = task.mutable_container(); container->set_type(ContainerInfo::MESOS); container->mutable_mesos()->mutable_image()->CopyFrom(image); Future<TaskStatus> statusRunning; Future<TaskStatus> statusFinished; EXPECT_CALL(sched, statusUpdate(&driver, _)) .WillOnce(FutureArg<1>(&statusRunning)) .WillOnce(FutureArg<1>(&statusFinished)); driver.launchTasks(offer.id(), {task}); AWAIT_READY_FOR(statusRunning, Seconds(60)); EXPECT_EQ(task.task_id(), statusRunning->task_id()); EXPECT_EQ(TASK_RUNNING, statusRunning->state()); AWAIT_READY(statusFinished); EXPECT_EQ(task.task_id(), statusFinished->task_id()); EXPECT_EQ(TASK_FINISHED, statusFinished->state()); driver.stop(); driver.join(); }
// This test ensures that driver based schedulers using explicit // acknowledgements can acknowledge status updates sent from // HTTP based executors. TEST_F_TEMP_DISABLED_ON_WINDOWS( HTTPCommandExecutorTest, ExplicitAcknowledgements) { Try<Owned<cluster::Master>> master = StartMaster(); ASSERT_SOME(master); Owned<MasterDetector> detector = master.get()->createDetector(); slave::Flags flags = CreateSlaveFlags(); flags.http_command_executor = true; Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), flags); ASSERT_SOME(slave); MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, false, DEFAULT_CREDENTIAL); EXPECT_CALL(sched, registered(&driver, _, _)); Future<vector<Offer>> offers; EXPECT_CALL(sched, resourceOffers(&driver, _)) .WillOnce(FutureArg<1>(&offers)) .WillRepeatedly(Return()); // Ignore subsequent offers. driver.start(); AWAIT_READY(offers); EXPECT_EQ(1u, offers->size()); // Launch a task with the command executor. TaskInfo task = createTask( offers->front().slave_id(), offers->front().resources(), "sleep 1000"); Future<TaskStatus> statusRunning; EXPECT_CALL(sched, statusUpdate(_, _)) .WillOnce(FutureArg<1>(&statusRunning)); // Ensure no status update acknowledgements are sent from the driver // to the master until the explicit acknowledgement is sent. EXPECT_NO_FUTURE_CALLS( mesos::scheduler::Call(), mesos::scheduler::Call::ACKNOWLEDGE, _ , master.get()->pid); driver.launchTasks(offers->front().id(), {task}); AWAIT_READY(statusRunning); EXPECT_TRUE(statusRunning->has_slave_id()); EXPECT_EQ(TASK_RUNNING, statusRunning->state()); // Now send the acknowledgement. Future<mesos::scheduler::Call> acknowledgement = FUTURE_CALL( mesos::scheduler::Call(), mesos::scheduler::Call::ACKNOWLEDGE, _, master.get()->pid); driver.acknowledgeStatusUpdate(statusRunning.get()); AWAIT_READY(acknowledgement); driver.stop(); driver.join(); }
// This test verifies that the slave and status update manager // properly handle duplicate terminal status updates, when the // second update is received after the ACK for the first update. // The proper behavior here is for the status update manager to // forward the duplicate update to the scheduler. TEST_F(StatusUpdateManagerTest, DuplicateTerminalUpdateAfterAck) { Try<PID<Master> > master = StartMaster(); ASSERT_SOME(master); MockExecutor exec(DEFAULT_EXECUTOR_ID); slave::Flags flags = CreateSlaveFlags(); Try<PID<Slave> > slave = StartSlave(&exec, flags); ASSERT_SOME(slave); FrameworkInfo frameworkInfo = DEFAULT_FRAMEWORK_INFO; frameworkInfo.set_checkpoint(true); // Enable checkpointing. MockScheduler sched; MesosSchedulerDriver driver( &sched, frameworkInfo, master.get(), DEFAULT_CREDENTIAL); FrameworkID frameworkId; EXPECT_CALL(sched, registered(_, _, _)) .WillOnce(SaveArg<1>(&frameworkId)); Future<vector<Offer> > offers; EXPECT_CALL(sched, resourceOffers(_, _)) .WillOnce(FutureArg<1>(&offers)) .WillRepeatedly(Return()); // Ignore subsequent offers. driver.start(); AWAIT_READY(offers); EXPECT_NE(0u, offers.get().size()); ExecutorDriver* execDriver; EXPECT_CALL(exec, registered(_, _, _, _)) .WillOnce(SaveArg<0>(&execDriver)); // Send a terminal update right away. EXPECT_CALL(exec, launchTask(_, _)) .WillOnce(SendStatusUpdateFromTask(TASK_FINISHED)); Future<TaskStatus> status; EXPECT_CALL(sched, statusUpdate(_, _)) .WillOnce(FutureArg<1>(&status)); Future<Nothing> _statusUpdateAcknowledgement = FUTURE_DISPATCH(slave.get(), &Slave::_statusUpdateAcknowledgement); driver.launchTasks(offers.get()[0].id(), createTasks(offers.get()[0])); AWAIT_READY(status); EXPECT_EQ(TASK_FINISHED, status.get().state()); AWAIT_READY(_statusUpdateAcknowledgement); Future<TaskStatus> update; EXPECT_CALL(sched, statusUpdate(_, _)) .WillOnce(FutureArg<1>(&update)); Future<Nothing> _statusUpdateAcknowledgement2 = FUTURE_DISPATCH(slave.get(), &Slave::_statusUpdateAcknowledgement); Clock::pause(); // Now send a TASK_KILLED update for the same task. TaskStatus status2 = status.get(); status2.set_state(TASK_KILLED); execDriver->sendStatusUpdate(status2); // Ensure the scheduler receives TASK_KILLED. AWAIT_READY(update); EXPECT_EQ(TASK_KILLED, update.get().state()); // Ensure the slave properly handles the ACK. // Clock::settle() ensures that the slave successfully // executes Slave::_statusUpdateAcknowledgement(). AWAIT_READY(_statusUpdateAcknowledgement2); Clock::settle(); Clock::resume(); EXPECT_CALL(exec, shutdown(_)) .Times(AtMost(1)); driver.stop(); driver.join(); Shutdown(); }