virtual Result<TaskStatus> slaveTaskStatusDecorator( const FrameworkID& frameworkId, const TaskStatus& status) { LOG(INFO) << "Executing 'slaveTaskStatusDecorator' hook"; Labels labels; // Set one known label. Label* newLabel = labels.add_labels(); newLabel->set_key("bar"); newLabel->set_value("qux"); // Remove label which was set by test. foreach (const Label& oldLabel, status.labels().labels()) { if (oldLabel.key() != "foo") { labels.add_labels()->CopyFrom(oldLabel); } } TaskStatus result; result.mutable_labels()->CopyFrom(labels); // Set an IP address, a network isolation group, and a known label // in network info. This data is later validated by the // 'HookTest.VerifySlaveTaskStatusDecorator' test. NetworkInfo* networkInfo = result.mutable_container_status()->add_network_infos(); // TODO(CD): Deprecated -- remove after 0.27.0. networkInfo->set_ip_address("4.3.2.1"); NetworkInfo::IPAddress* ipAddress = networkInfo->add_ip_addresses(); ipAddress->set_ip_address("4.3.2.1"); networkInfo->add_groups("public"); Label* networkInfoLabel = networkInfo->mutable_labels()->add_labels(); networkInfoLabel->set_key("net_foo"); networkInfoLabel->set_value("net_bar"); return result; }
// This test verifies that the slave task status label decorator can // add and remove labels from a TaskStatus during the status update // sequence. A TaskStatus with two labels ("foo":"bar" and // "bar":"baz") is sent from the executor. The labels get modified by // the slave hook to strip the "foo":"bar" pair and/ add a new // "baz":"qux" pair. TEST_F(HookTest, VerifySlaveTaskStatusDecorator) { Try<Owned<cluster::Master>> master = StartMaster(); ASSERT_SOME(master); MockExecutor exec(DEFAULT_EXECUTOR_ID); TestContainerizer containerizer(&exec); Owned<MasterDetector> detector = master.get()->createDetector(); Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), &containerizer); ASSERT_SOME(slave); MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL); EXPECT_CALL(sched, registered(&driver, _, _)); Future<vector<Offer>> offers; EXPECT_CALL(sched, resourceOffers(&driver, _)) .WillOnce(FutureArg<1>(&offers)) .WillRepeatedly(Return()); // Ignore subsequent offers. driver.start(); AWAIT_READY(offers); ASSERT_EQ(1u, offers.get().size()); // Start a task. TaskInfo task = createTask(offers.get()[0], "", DEFAULT_EXECUTOR_ID); ExecutorDriver* execDriver; EXPECT_CALL(exec, registered(_, _, _, _)) .WillOnce(SaveArg<0>(&execDriver)); Future<TaskInfo> execTask; EXPECT_CALL(exec, launchTask(_, _)) .WillOnce(FutureArg<1>(&execTask)); Future<TaskStatus> status; EXPECT_CALL(sched, statusUpdate(&driver, _)) .WillOnce(FutureArg<1>(&status)); driver.launchTasks(offers.get()[0].id(), {task}); AWAIT_READY(execTask); // Now send TASK_RUNNING update with two labels. The first label // ("foo:bar") will be removed by the task status hook to ensure // that it can remove labels. The second label will be preserved // and forwarded to Master (and eventually to the framework). // The hook also adds a new label with the same key but a different // value ("bar:quz"). TaskStatus runningStatus; runningStatus.mutable_task_id()->MergeFrom(execTask.get().task_id()); runningStatus.set_state(TASK_RUNNING); // Add two labels to the TaskStatus Labels* labels = runningStatus.mutable_labels(); labels->add_labels()->CopyFrom(createLabel("foo", "bar")); labels->add_labels()->CopyFrom(createLabel("bar", "baz")); execDriver->sendStatusUpdate(runningStatus); AWAIT_READY(status); // The hook will hang an extra label off. const Labels& labels_ = status.get().labels(); EXPECT_EQ(2, labels_.labels_size()); // The test hook will prepend a new "baz":"qux" label. EXPECT_EQ("bar", labels_.labels(0).key()); EXPECT_EQ("qux", labels_.labels(0).value()); // And lastly, we only expect the "foo":"bar" pair to be stripped by // the module. The last pair should be the original "bar":"baz" // pair set by the test. EXPECT_EQ("bar", labels_.labels(1).key()); EXPECT_EQ("baz", labels_.labels(1).value()); // Now validate TaskInfo.container_status. We must have received a // container_status with one network_info set by the test hook module. EXPECT_TRUE(status.get().has_container_status()); EXPECT_EQ(1, status.get().container_status().network_infos().size()); const NetworkInfo networkInfo = status.get().container_status().network_infos(0); // The hook module sets up '4.3.2.1' as the IP address and 'public' as the // network isolation group. The `ip_address` field is deprecated, but the // hook module should continue to set it as well as the new `ip_addresses` // field for now. EXPECT_TRUE(networkInfo.has_ip_address()); EXPECT_EQ("4.3.2.1", networkInfo.ip_address()); EXPECT_EQ(1, networkInfo.ip_addresses().size()); EXPECT_TRUE(networkInfo.ip_addresses(0).has_ip_address()); EXPECT_EQ("4.3.2.1", networkInfo.ip_addresses(0).ip_address()); EXPECT_EQ(1, networkInfo.groups().size()); EXPECT_EQ("public", networkInfo.groups(0)); EXPECT_TRUE(networkInfo.has_labels()); EXPECT_EQ(1, networkInfo.labels().labels().size()); const Label networkInfoLabel = networkInfo.labels().labels(0); // Finally, the labels set inside NetworkInfo by the hook module. EXPECT_EQ("net_foo", networkInfoLabel.key()); EXPECT_EQ("net_bar", networkInfoLabel.value()); EXPECT_CALL(exec, shutdown(_)) .Times(AtMost(1)); driver.stop(); driver.join(); }
void launchTask(ExecutorDriver* driver, const TaskInfo& task) { if (run.isSome()) { // TODO(alexr): Use `protobuf::createTaskStatus()` // instead of manually setting fields. TaskStatus status; status.mutable_task_id()->CopyFrom(task.task_id()); status.set_state(TASK_FAILED); status.set_message( "Attempted to run multiple tasks using a \"docker\" executor"); driver->sendStatusUpdate(status); return; } // Capture the TaskID. taskId = task.task_id(); // Capture the kill policy. if (task.has_kill_policy()) { killPolicy = task.kill_policy(); } LOG(INFO) << "Starting task " << taskId.get(); CHECK(task.has_container()); CHECK(task.has_command()); CHECK(task.container().type() == ContainerInfo::DOCKER); Try<Docker::RunOptions> runOptions = Docker::RunOptions::create( task.container(), task.command(), containerName, sandboxDirectory, mappedDirectory, task.resources() + task.executor().resources(), cgroupsEnableCfs, taskEnvironment, None(), // No extra devices. defaultContainerDNS ); if (runOptions.isError()) { // TODO(alexr): Use `protobuf::createTaskStatus()` // instead of manually setting fields. TaskStatus status; status.mutable_task_id()->CopyFrom(task.task_id()); status.set_state(TASK_FAILED); status.set_message( "Failed to create docker run options: " + runOptions.error()); driver->sendStatusUpdate(status); _stop(); return; } // We're adding task and executor resources to launch docker since // the DockerContainerizer updates the container cgroup limits // directly and it expects it to be the sum of both task and // executor resources. This does leave to a bit of unaccounted // resources for running this executor, but we are assuming // this is just a very small amount of overcommit. run = docker->run( runOptions.get(), Subprocess::FD(STDOUT_FILENO), Subprocess::FD(STDERR_FILENO)); run->onAny(defer(self(), &Self::reaped, lambda::_1)); // Delay sending TASK_RUNNING status update until we receive // inspect output. Note that we store a future that completes // after the sending of the running update. This allows us to // ensure that the terminal update is sent after the running // update (see `reaped()`). inspect = docker->inspect(containerName, DOCKER_INSPECT_DELAY) .then(defer(self(), [=](const Docker::Container& container) { if (!killed) { containerPid = container.pid; // TODO(alexr): Use `protobuf::createTaskStatus()` // instead of manually setting fields. TaskStatus status; status.mutable_task_id()->CopyFrom(taskId.get()); status.set_state(TASK_RUNNING); status.set_data(container.output); if (container.ipAddress.isSome()) { // TODO(karya): Deprecated -- Remove after 0.25.0 has shipped. Label* label = status.mutable_labels()->add_labels(); label->set_key("Docker.NetworkSettings.IPAddress"); label->set_value(container.ipAddress.get()); NetworkInfo* networkInfo = status.mutable_container_status()->add_network_infos(); // Copy the NetworkInfo if it is specified in the // ContainerInfo. A Docker container has at most one // NetworkInfo, which is validated in containerizer. if (task.container().network_infos().size() > 0) { networkInfo->CopyFrom(task.container().network_infos(0)); networkInfo->clear_ip_addresses(); } NetworkInfo::IPAddress* ipAddress = networkInfo->add_ip_addresses(); ipAddress->set_ip_address(container.ipAddress.get()); containerNetworkInfo = *networkInfo; } driver->sendStatusUpdate(status); } return Nothing(); })); inspect.onFailed(defer(self(), [=](const string& failure) { LOG(ERROR) << "Failed to inspect container '" << containerName << "'" << ": " << failure; // TODO(bmahler): This is fatal, try to shut down cleanly. // Since we don't have a container id, we can only discard // the run future. })); inspect.onReady(defer(self(), &Self::launchCheck, task)); inspect.onReady( defer(self(), &Self::launchHealthCheck, containerName, task)); }