virtual Result<TaskStatus> slaveTaskStatusDecorator(
const FrameworkID& frameworkId,
const TaskStatus& status)
{
LOG(INFO) << "Executing 'slaveTaskStatusDecorator' hook";
Labels labels;
// Set one known label.
Label* newLabel = labels.add_labels();
newLabel->set_key("bar");
newLabel->set_value("qux");
// Remove label which was set by test.
foreach (const Label& oldLabel, status.labels().labels()) {
if (oldLabel.key() != "foo") {
labels.add_labels()->CopyFrom(oldLabel);
}
}
TaskStatus result;
result.mutable_labels()->CopyFrom(labels);
// Set an IP address, a network isolation group, and a known label
// in network info. This data is later validated by the
// 'HookTest.VerifySlaveTaskStatusDecorator' test.
NetworkInfo* networkInfo =
result.mutable_container_status()->add_network_infos();
// TODO(CD): Deprecated -- remove after 0.27.0.
networkInfo->set_ip_address("4.3.2.1");
NetworkInfo::IPAddress* ipAddress =
networkInfo->add_ip_addresses();
ipAddress->set_ip_address("4.3.2.1");
networkInfo->add_groups("public");
Label* networkInfoLabel = networkInfo->mutable_labels()->add_labels();
networkInfoLabel->set_key("net_foo");
networkInfoLabel->set_value("net_bar");
return result;
}
// This test verifies that the slave task status label decorator can
// add and remove labels from a TaskStatus during the status update
// sequence. A TaskStatus with two labels ("foo":"bar" and
// "bar":"baz") is sent from the executor. The labels get modified by
// the slave hook to strip the "foo":"bar" pair and/ add a new
// "baz":"qux" pair.
TEST_F(HookTest, VerifySlaveTaskStatusDecorator)
{
Try<Owned<cluster::Master>> master = StartMaster();
ASSERT_SOME(master);
MockExecutor exec(DEFAULT_EXECUTOR_ID);
TestContainerizer containerizer(&exec);
Owned<MasterDetector> detector = master.get()->createDetector();
Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), &containerizer);
ASSERT_SOME(slave);
MockScheduler sched;
MesosSchedulerDriver driver(
&sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL);
EXPECT_CALL(sched, registered(&driver, _, _));
Future<vector<Offer>> offers;
EXPECT_CALL(sched, resourceOffers(&driver, _))
.WillOnce(FutureArg<1>(&offers))
.WillRepeatedly(Return()); // Ignore subsequent offers.
driver.start();
AWAIT_READY(offers);
ASSERT_EQ(1u, offers.get().size());
// Start a task.
TaskInfo task = createTask(offers.get()[0], "", DEFAULT_EXECUTOR_ID);
ExecutorDriver* execDriver;
EXPECT_CALL(exec, registered(_, _, _, _))
.WillOnce(SaveArg<0>(&execDriver));
Future<TaskInfo> execTask;
EXPECT_CALL(exec, launchTask(_, _))
.WillOnce(FutureArg<1>(&execTask));
Future<TaskStatus> status;
EXPECT_CALL(sched, statusUpdate(&driver, _))
.WillOnce(FutureArg<1>(&status));
driver.launchTasks(offers.get()[0].id(), {task});
AWAIT_READY(execTask);
// Now send TASK_RUNNING update with two labels. The first label
// ("foo:bar") will be removed by the task status hook to ensure
// that it can remove labels. The second label will be preserved
// and forwarded to Master (and eventually to the framework).
// The hook also adds a new label with the same key but a different
// value ("bar:quz").
TaskStatus runningStatus;
runningStatus.mutable_task_id()->MergeFrom(execTask.get().task_id());
runningStatus.set_state(TASK_RUNNING);
// Add two labels to the TaskStatus
Labels* labels = runningStatus.mutable_labels();
labels->add_labels()->CopyFrom(createLabel("foo", "bar"));
labels->add_labels()->CopyFrom(createLabel("bar", "baz"));
execDriver->sendStatusUpdate(runningStatus);
AWAIT_READY(status);
// The hook will hang an extra label off.
const Labels& labels_ = status.get().labels();
EXPECT_EQ(2, labels_.labels_size());
// The test hook will prepend a new "baz":"qux" label.
EXPECT_EQ("bar", labels_.labels(0).key());
EXPECT_EQ("qux", labels_.labels(0).value());
// And lastly, we only expect the "foo":"bar" pair to be stripped by
// the module. The last pair should be the original "bar":"baz"
// pair set by the test.
EXPECT_EQ("bar", labels_.labels(1).key());
EXPECT_EQ("baz", labels_.labels(1).value());
// Now validate TaskInfo.container_status. We must have received a
// container_status with one network_info set by the test hook module.
EXPECT_TRUE(status.get().has_container_status());
EXPECT_EQ(1, status.get().container_status().network_infos().size());
const NetworkInfo networkInfo =
status.get().container_status().network_infos(0);
// The hook module sets up '4.3.2.1' as the IP address and 'public' as the
// network isolation group. The `ip_address` field is deprecated, but the
// hook module should continue to set it as well as the new `ip_addresses`
// field for now.
EXPECT_TRUE(networkInfo.has_ip_address());
EXPECT_EQ("4.3.2.1", networkInfo.ip_address());
EXPECT_EQ(1, networkInfo.ip_addresses().size());
EXPECT_TRUE(networkInfo.ip_addresses(0).has_ip_address());
EXPECT_EQ("4.3.2.1", networkInfo.ip_addresses(0).ip_address());
EXPECT_EQ(1, networkInfo.groups().size());
EXPECT_EQ("public", networkInfo.groups(0));
EXPECT_TRUE(networkInfo.has_labels());
EXPECT_EQ(1, networkInfo.labels().labels().size());
const Label networkInfoLabel = networkInfo.labels().labels(0);
// Finally, the labels set inside NetworkInfo by the hook module.
EXPECT_EQ("net_foo", networkInfoLabel.key());
EXPECT_EQ("net_bar", networkInfoLabel.value());
EXPECT_CALL(exec, shutdown(_))
.Times(AtMost(1));
driver.stop();
driver.join();
}
void launchTask(ExecutorDriver* driver, const TaskInfo& task)
{
if (run.isSome()) {
// TODO(alexr): Use `protobuf::createTaskStatus()`
// instead of manually setting fields.
TaskStatus status;
status.mutable_task_id()->CopyFrom(task.task_id());
status.set_state(TASK_FAILED);
status.set_message(
"Attempted to run multiple tasks using a \"docker\" executor");
driver->sendStatusUpdate(status);
return;
}
// Capture the TaskID.
taskId = task.task_id();
// Capture the kill policy.
if (task.has_kill_policy()) {
killPolicy = task.kill_policy();
}
LOG(INFO) << "Starting task " << taskId.get();
CHECK(task.has_container());
CHECK(task.has_command());
CHECK(task.container().type() == ContainerInfo::DOCKER);
Try<Docker::RunOptions> runOptions = Docker::RunOptions::create(
task.container(),
task.command(),
containerName,
sandboxDirectory,
mappedDirectory,
task.resources() + task.executor().resources(),
cgroupsEnableCfs,
taskEnvironment,
None(), // No extra devices.
defaultContainerDNS
);
if (runOptions.isError()) {
// TODO(alexr): Use `protobuf::createTaskStatus()`
// instead of manually setting fields.
TaskStatus status;
status.mutable_task_id()->CopyFrom(task.task_id());
status.set_state(TASK_FAILED);
status.set_message(
"Failed to create docker run options: " + runOptions.error());
driver->sendStatusUpdate(status);
_stop();
return;
}
// We're adding task and executor resources to launch docker since
// the DockerContainerizer updates the container cgroup limits
// directly and it expects it to be the sum of both task and
// executor resources. This does leave to a bit of unaccounted
// resources for running this executor, but we are assuming
// this is just a very small amount of overcommit.
run = docker->run(
runOptions.get(),
Subprocess::FD(STDOUT_FILENO),
Subprocess::FD(STDERR_FILENO));
run->onAny(defer(self(), &Self::reaped, lambda::_1));
// Delay sending TASK_RUNNING status update until we receive
// inspect output. Note that we store a future that completes
// after the sending of the running update. This allows us to
// ensure that the terminal update is sent after the running
// update (see `reaped()`).
inspect = docker->inspect(containerName, DOCKER_INSPECT_DELAY)
.then(defer(self(), [=](const Docker::Container& container) {
if (!killed) {
containerPid = container.pid;
// TODO(alexr): Use `protobuf::createTaskStatus()`
// instead of manually setting fields.
TaskStatus status;
status.mutable_task_id()->CopyFrom(taskId.get());
status.set_state(TASK_RUNNING);
status.set_data(container.output);
if (container.ipAddress.isSome()) {
// TODO(karya): Deprecated -- Remove after 0.25.0 has shipped.
Label* label = status.mutable_labels()->add_labels();
label->set_key("Docker.NetworkSettings.IPAddress");
label->set_value(container.ipAddress.get());
NetworkInfo* networkInfo =
status.mutable_container_status()->add_network_infos();
// Copy the NetworkInfo if it is specified in the
// ContainerInfo. A Docker container has at most one
// NetworkInfo, which is validated in containerizer.
if (task.container().network_infos().size() > 0) {
networkInfo->CopyFrom(task.container().network_infos(0));
networkInfo->clear_ip_addresses();
}
NetworkInfo::IPAddress* ipAddress = networkInfo->add_ip_addresses();
ipAddress->set_ip_address(container.ipAddress.get());
containerNetworkInfo = *networkInfo;
}
driver->sendStatusUpdate(status);
}
return Nothing();
}));
inspect.onFailed(defer(self(), [=](const string& failure) {
LOG(ERROR) << "Failed to inspect container '" << containerName << "'"
<< ": " << failure;
// TODO(bmahler): This is fatal, try to shut down cleanly.
// Since we don't have a container id, we can only discard
// the run future.
}));
inspect.onReady(defer(self(), &Self::launchCheck, task));
inspect.onReady(
defer(self(), &Self::launchHealthCheck, containerName, task));
}
