TEST_F(DockerTest, ROOT_DOCKER_CheckPortResource)
{
const string containerName = NAME_PREFIX + "-port-resource-test";
Owned<Docker> docker(Docker::create(tests::flags.docker,
tests::flags.docker_socket,
false).get());
// Make sure the container is removed.
Future<Nothing> remove = docker->rm(containerName, true);
ASSERT_TRUE(process::internal::await(remove, Seconds(10)));
ContainerInfo containerInfo;
containerInfo.set_type(ContainerInfo::DOCKER);
ContainerInfo::DockerInfo dockerInfo;
dockerInfo.set_image("busybox");
dockerInfo.set_network(ContainerInfo::DockerInfo::BRIDGE);
ContainerInfo::DockerInfo::PortMapping portMapping;
portMapping.set_host_port(10000);
portMapping.set_container_port(80);
dockerInfo.add_port_mappings()->CopyFrom(portMapping);
containerInfo.mutable_docker()->CopyFrom(dockerInfo);
CommandInfo commandInfo;
commandInfo.set_shell(false);
commandInfo.set_value("true");
Resources resources =
Resources::parse("ports:[9998-9999];ports:[10001-11000]").get();
Future<Nothing> run = docker->run(
containerInfo,
commandInfo,
containerName,
"dir",
"/mnt/mesos/sandbox",
resources);
// Port should be out side of the provided ranges.
AWAIT_EXPECT_FAILED(run);
resources = Resources::parse("ports:[9998-9999];ports:[10000-11000]").get();
Try<string> directory = environment->mkdtemp();
CHECK_SOME(directory) << "Failed to create temporary directory";
run = docker->run(
containerInfo,
commandInfo,
containerName,
directory.get(),
"/mnt/mesos/sandbox",
resources);
AWAIT_READY(run);
}
TaskInfo buildTask (string hostname, string id, const SlaveID& slave) {
hostProfile profile = hostList[hostname];
// Define the Docker container.
/* Since there is no "executor" to manage the tasks, the
container will be built and attached directly into the task below */
ContainerInfo container;
container.set_type(container.DOCKER);
ContainerInfo::DockerInfo docker;
docker.set_image(DOCKER_IMAGE);
container.mutable_docker()->MergeFrom(docker);
// Mount local volume inside Container
Volume * volume = container.add_volumes();
volume->set_container_path("/mnt");
volume->set_host_path("/local/mesos");
volume->set_mode(Volume_Mode_RW);
// Define the task
TaskInfo task;
task.set_name("K3-" + k3binary);
task.mutable_task_id()->set_value(id);
task.mutable_slave_id()->MergeFrom(slave);
task.mutable_container()->MergeFrom(container);
//task.set_data(stringify(localTasks));
// Define include files for the command
CommandInfo command;
CommandInfo_URI * k3_bin = command.add_uris();
k3_bin->set_value(fileServer + "/" + k3binary);
k3_bin->set_executable(true);
k3_bin->set_extract(false);
// CommandInfo_URI * k3_args = command.add_uris();
// k3_args->set_value(runpath + "/k3input.yaml");
// command.set_value("$MESOS_SANDBOX/" + k3binary + " -l INFO -p " +
// "$MESOS_SANDBOX/k3input.yaml");
task.mutable_command()->MergeFrom(command);
// Option A for doing resources management (see scheduler for option B)
Resource* resource;
resource = task.add_resources();
resource->set_name("cpus");
resource->set_type(Value::SCALAR);
resource->mutable_scalar()->set_value(profile.cpu);
resource = task.add_resources();
resource->set_name("mem");
resource->set_type(Value::SCALAR);
resource->mutable_scalar()->set_value(profile.mem);
return task;
}
virtual void resourceOffers(
SchedulerDriver* driver,
const vector<Offer>& offers)
{
static const Try<Resources> TASK_RESOURCES = Resources::parse(resources);
if (TASK_RESOURCES.isError()) {
cerr << "Failed to parse resources '" << resources
<< "': " << TASK_RESOURCES.error() << endl;
driver->abort();
return;
}
foreach (const Offer& offer, offers) {
if (!launched &&
Resources(offer.resources()).contains(TASK_RESOURCES.get())) {
TaskInfo task;
task.set_name(name);
task.mutable_task_id()->set_value(name);
task.mutable_slave_id()->MergeFrom(offer.slave_id());
task.mutable_resources()->CopyFrom(TASK_RESOURCES.get());
task.mutable_command()->set_value(command);
if (uri.isSome()) {
task.mutable_command()->add_uris()->set_value(uri.get());
}
if (dockerImage.isSome()) {
ContainerInfo containerInfo;
containerInfo.set_type(ContainerInfo::DOCKER);
ContainerInfo::DockerInfo dockerInfo;
dockerInfo.set_image(dockerImage.get());
containerInfo.mutable_docker()->CopyFrom(dockerInfo);
task.mutable_container()->CopyFrom(containerInfo);
}
vector<TaskInfo> tasks;
tasks.push_back(task);
driver->launchTasks(offer.id(), tasks);
cout << "task " << name << " submitted to slave "
<< offer.slave_id() << endl;
launched = true;
} else {
driver->declineOffer(offer.id());
}
}
}
// This test verifies mounting in an absolute path when running a
// docker container works.
TEST_F(DockerTest, ROOT_DOCKER_MountAbsolute)
{
Owned<Docker> docker = Docker::create(
tests::flags.docker,
tests::flags.docker_socket,
false).get();
ContainerInfo containerInfo;
containerInfo.set_type(ContainerInfo::DOCKER);
Try<string> directory = environment->mkdtemp();
CHECK_SOME(directory) << "Failed to create temporary directory";
const string testFile = path::join(directory.get(), "test_file");
EXPECT_SOME(os::write(testFile, "data"));
Volume* volume = containerInfo.add_volumes();
volume->set_host_path(testFile);
volume->set_container_path("/tmp/test_file");
volume->set_mode(Volume::RO);
ContainerInfo::DockerInfo dockerInfo;
dockerInfo.set_image("busybox");
containerInfo.mutable_docker()->CopyFrom(dockerInfo);
CommandInfo commandInfo;
commandInfo.set_shell(true);
commandInfo.set_value("ls /tmp/test_file");
Future<Nothing> run = docker->run(
containerInfo,
commandInfo,
NAME_PREFIX + "-mount-absolute-test",
directory.get(),
directory.get());
AWAIT_READY(run);
}
bool operator == (
const ContainerInfo::DockerInfo& left,
const ContainerInfo::DockerInfo& right)
{
// Order of port mappings is not important.
if (left.port_mappings().size() != right.port_mappings().size()) {
return false;
}
for (int i = 0; i < left.port_mappings().size(); i++) {
bool found = false;
for (int j = 0; j < right.port_mappings().size(); j++) {
if (left.port_mappings().Get(i) == right.port_mappings().Get(j)) {
found = true;
break;
}
}
if (!found) {
return false;
}
}
// Order of parameters is not important.
if (left.parameters().size() != right.parameters().size()) {
return false;
}
for (int i = 0; i < left.parameters().size(); i++) {
bool found = false;
for (int j = 0; j < right.parameters().size(); j++) {
if (left.parameters().Get(i) == right.parameters().Get(j)) {
found = true;
break;
}
}
if (!found) {
return false;
}
}
return left.image() == right.image() &&
left.network() == right.network() &&
left.privileged() == right.privileged() &&
left.force_pull_image() == right.force_pull_image();
}
// Test that the prepare launch docker hook execute before launch
// a docker container. Test hook create a file "foo" in the sandbox
// directory. When the docker container launched, the sandbox directory
// is mounted to the docker container. We validate the hook by verifying
// the "foo" file exists in the docker container or not.
TEST_F(HookTest, ROOT_DOCKER_VerifySlavePreLaunchDockerHook)
{
Try<Owned<cluster::Master>> master = StartMaster();
ASSERT_SOME(master);
MockDocker* mockDocker =
new MockDocker(tests::flags.docker, tests::flags.docker_socket);
Shared<Docker> docker(mockDocker);
slave::Flags flags = CreateSlaveFlags();
Fetcher fetcher;
Try<ContainerLogger*> logger =
ContainerLogger::create(flags.container_logger);
ASSERT_SOME(logger);
MockDockerContainerizer containerizer(
flags,
&fetcher,
Owned<ContainerLogger>(logger.get()),
docker);
Owned<MasterDetector> detector = master.get()->createDetector();
Try<Owned<cluster::Slave>> slave =
StartSlave(detector.get(), &containerizer, flags);
ASSERT_SOME(slave);
MockScheduler sched;
MesosSchedulerDriver driver(
&sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL);
Future<FrameworkID> frameworkId;
EXPECT_CALL(sched, registered(&driver, _, _))
.WillOnce(FutureArg<1>(&frameworkId));
Future<vector<Offer>> offers;
EXPECT_CALL(sched, resourceOffers(&driver, _))
.WillOnce(FutureArg<1>(&offers))
.WillRepeatedly(Return()); // Ignore subsequent offers.
driver.start();
AWAIT_READY(frameworkId);
AWAIT_READY(offers);
ASSERT_NE(0u, offers.get().size());
const Offer& offer = offers.get()[0];
SlaveID slaveId = offer.slave_id();
TaskInfo task;
task.set_name("");
task.mutable_task_id()->set_value("1");
task.mutable_slave_id()->CopyFrom(offer.slave_id());
task.mutable_resources()->CopyFrom(offer.resources());
CommandInfo command;
command.set_value("test -f " + path::join(flags.sandbox_directory, "foo"));
ContainerInfo containerInfo;
containerInfo.set_type(ContainerInfo::DOCKER);
// TODO(tnachen): Use local image to test if possible.
ContainerInfo::DockerInfo dockerInfo;
dockerInfo.set_image("alpine");
containerInfo.mutable_docker()->CopyFrom(dockerInfo);
task.mutable_command()->CopyFrom(command);
task.mutable_container()->CopyFrom(containerInfo);
vector<TaskInfo> tasks;
tasks.push_back(task);
Future<ContainerID> containerId;
EXPECT_CALL(containerizer, launch(_, _, _, _, _, _, _, _))
.WillOnce(DoAll(FutureArg<0>(&containerId),
Invoke(&containerizer,
&MockDockerContainerizer::_launch)));
Future<TaskStatus> statusRunning;
Future<TaskStatus> statusFinished;
EXPECT_CALL(sched, statusUpdate(&driver, _))
.WillOnce(FutureArg<1>(&statusRunning))
.WillOnce(FutureArg<1>(&statusFinished))
.WillRepeatedly(DoDefault());
driver.launchTasks(offers.get()[0].id(), tasks);
AWAIT_READY_FOR(containerId, Seconds(60));
AWAIT_READY_FOR(statusRunning, Seconds(60));
EXPECT_EQ(TASK_RUNNING, statusRunning.get().state());
AWAIT_READY_FOR(statusFinished, Seconds(60));
EXPECT_EQ(TASK_FINISHED, statusFinished.get().state());
Future<containerizer::Termination> termination =
containerizer.wait(containerId.get());
driver.stop();
driver.join();
AWAIT_READY(termination);
Future<list<Docker::Container>> containers =
docker.get()->ps(true, slave::DOCKER_NAME_PREFIX);
AWAIT_READY(containers);
// Cleanup all mesos launched containers.
foreach (const Docker::Container& container, containers.get()) {
AWAIT_READY_FOR(docker.get()->rm(container.id, true), Seconds(30));
}
}
virtual void resourceOffers(
SchedulerDriver* driver,
const vector<Offer>& offers)
{
static const Try<Resources> TASK_RESOURCES = Resources::parse(resources);
if (TASK_RESOURCES.isError()) {
cerr << "Failed to parse resources '" << resources
<< "': " << TASK_RESOURCES.error() << endl;
driver->abort();
return;
}
foreach (const Offer& offer, offers) {
if (!launched &&
Resources(offer.resources()).contains(TASK_RESOURCES.get())) {
TaskInfo task;
task.set_name(name);
task.mutable_task_id()->set_value(name);
task.mutable_slave_id()->MergeFrom(offer.slave_id());
task.mutable_resources()->CopyFrom(TASK_RESOURCES.get());
CommandInfo* commandInfo = task.mutable_command();
commandInfo->set_value(command);
if (environment.isSome()) {
Environment* environment_ = commandInfo->mutable_environment();
foreachpair (const std::string& name,
const std::string& value,
environment.get()) {
Environment_Variable* environmentVariable =
environment_->add_variables();
environmentVariable->set_name(name);
environmentVariable->set_value(value);
}
}
if (uri.isSome()) {
task.mutable_command()->add_uris()->set_value(uri.get());
}
if (dockerImage.isSome()) {
ContainerInfo containerInfo;
if (containerizer == "mesos") {
containerInfo.set_type(ContainerInfo::MESOS);
ContainerInfo::MesosInfo mesosInfo;
Image mesosImage;
mesosImage.set_type(Image::DOCKER);
mesosImage.mutable_docker()->set_name(dockerImage.get());
mesosInfo.mutable_image()->CopyFrom(mesosImage);
containerInfo.mutable_mesos()->CopyFrom(mesosInfo);
} else if (containerizer == "docker") {
containerInfo.set_type(ContainerInfo::DOCKER);
ContainerInfo::DockerInfo dockerInfo;
dockerInfo.set_image(dockerImage.get());
containerInfo.mutable_docker()->CopyFrom(dockerInfo);
} else {
cerr << "Unsupported containerizer: " << containerizer << endl;;
driver->abort();
return;
}
task.mutable_container()->CopyFrom(containerInfo);
}
vector<TaskInfo> tasks;
tasks.push_back(task);
driver->launchTasks(offer.id(), tasks);
cout << "task " << name << " submitted to slave "
<< offer.slave_id() << endl;
launched = true;
} else {
virtual void resourceOffers(SchedulerDriver* driver,
const vector<Offer>& offers)
{
cout << "." << flush;
for (size_t i = 0; i < offers.size(); i++) {
const Offer& offer = offers[i];
// Lookup resources we care about.
// TODO(benh): It would be nice to ultimately have some helper
// functions for looking up resources.
double cpus = 0;
double mem = 0;
for (int i = 0; i < offer.resources_size(); i++) {
const Resource& resource = offer.resources(i);
if (resource.name() == "cpus" &&
resource.type() == Value::SCALAR) {
cpus = resource.scalar().value();
} else if (resource.name() == "mem" &&
resource.type() == Value::SCALAR) {
mem = resource.scalar().value();
}
}
// Launch tasks.
vector<TaskInfo> tasks;
while (tasksLaunched < totalTasks &&
cpus >= CPUS_PER_TASK &&
mem >= MEM_PER_TASK) {
int taskId = tasksLaunched++;
cout << "Starting task " << taskId << " on "
<< offer.hostname() << endl;
TaskInfo task;
task.set_name("Task " + lexical_cast<string>(taskId));
task.mutable_task_id()->set_value(lexical_cast<string>(taskId));
task.mutable_slave_id()->MergeFrom(offer.slave_id());
task.mutable_command()->set_value("echo hello");
// Use Docker to run the task.
ContainerInfo containerInfo;
containerInfo.set_type(ContainerInfo::DOCKER);
ContainerInfo::DockerInfo dockerInfo;
dockerInfo.set_image("busybox");
containerInfo.mutable_docker()->CopyFrom(dockerInfo);
task.mutable_container()->CopyFrom(containerInfo);
Resource* resource;
resource = task.add_resources();
resource->set_name("cpus");
resource->set_type(Value::SCALAR);
resource->mutable_scalar()->set_value(CPUS_PER_TASK);
resource = task.add_resources();
resource->set_name("mem");
resource->set_type(Value::SCALAR);
resource->mutable_scalar()->set_value(MEM_PER_TASK);
tasks.push_back(task);
cpus -= CPUS_PER_TASK;
mem -= MEM_PER_TASK;
}
driver->launchTasks(offer.id(), tasks);
}
}
// This test tests the functionality of the docker's interfaces.
TEST(DockerTest, ROOT_DOCKER_interface)
{
string containerName = "mesos-docker-test";
Resources resources = Resources::parse("cpus:1;mem:512").get();
Docker docker = Docker::create(tests::flags.docker, false).get();
// Cleaning up the container first if it exists.
Future<Nothing> status = docker.rm(containerName, true);
ASSERT_TRUE(status.await(Seconds(10)));
// Verify that we do not see the container.
Future<list<Docker::Container> > containers = docker.ps(true, containerName);
AWAIT_READY(containers);
foreach (const Docker::Container& container, containers.get()) {
EXPECT_NE("/" + containerName, container.name);
}
Try<string> directory = environment->mkdtemp();
CHECK_SOME(directory) << "Failed to create temporary directory";
ContainerInfo containerInfo;
containerInfo.set_type(ContainerInfo::DOCKER);
ContainerInfo::DockerInfo dockerInfo;
dockerInfo.set_image("busybox");
containerInfo.mutable_docker()->CopyFrom(dockerInfo);
CommandInfo commandInfo;
commandInfo.set_value("sleep 120");
// Start the container.
status = docker.run(
containerInfo,
commandInfo,
containerName,
directory.get(),
"/mnt/mesos/sandbox",
resources);
AWAIT_READY(status);
// Should be able to see the container now.
containers = docker.ps();
AWAIT_READY(containers);
bool found = false;
foreach (const Docker::Container& container, containers.get()) {
if ("/" + containerName == container.name) {
found = true;
break;
}
}
EXPECT_TRUE(found);
Future<Docker::Container> container = docker.inspect(containerName);
AWAIT_READY(container);
// Test some fields of the container.
EXPECT_NE("", container.get().id);
EXPECT_EQ("/" + containerName, container.get().name);
EXPECT_SOME(container.get().pid);
// Kill the container.
status = docker.kill(containerName);
AWAIT_READY(status);
// Now, the container should not appear in the result of ps().
// But it should appear in the result of ps(true).
containers = docker.ps();
AWAIT_READY(containers);
foreach (const Docker::Container& container, containers.get()) {
EXPECT_NE("/" + containerName, container.name);
}
containers = docker.ps(true, containerName);
AWAIT_READY(containers);
found = false;
foreach (const Docker::Container& container, containers.get()) {
if ("/" + containerName == container.name) {
found = true;
break;
}
}
EXPECT_TRUE(found);
// Check the container's info, both id and name should remain the
// same since we haven't removed it, but the pid should be none
// since it's not running.
container = docker.inspect(containerName);
AWAIT_READY(container);
EXPECT_NE("", container.get().id);
EXPECT_EQ("/" + containerName, container.get().name);
EXPECT_NONE(container.get().pid);
// Remove the container.
status = docker.rm(containerName);
AWAIT_READY(status);
// Should not be able to inspect the container.
container = docker.inspect(containerName);
AWAIT_FAILED(container);
// Also, now we should not be able to see the container by invoking
// ps(true).
containers = docker.ps(true, containerName);
AWAIT_READY(containers);
foreach (const Docker::Container& container, containers.get()) {
EXPECT_NE("/" + containerName, container.name);
}
// Start the container again, this time we will do a "rm -f"
// directly, instead of killing and rm.
status = docker.run(
containerInfo,
commandInfo,
containerName,
directory.get(),
"/mnt/mesos/sandbox",
resources);
AWAIT_READY(status);
// Verify that the container is there.
containers = docker.ps();
AWAIT_READY(containers);
found = false;
foreach (const Docker::Container& container, containers.get()) {
if ("/" + containerName == container.name) {
found = true;
break;
}
}
EXPECT_TRUE(found);
// Then do a "rm -f".
status = docker.rm(containerName, true);
AWAIT_READY(status);
// Verify that the container is totally removed, that is we can't
// find it by ps() or ps(true).
containers = docker.ps();
AWAIT_READY(containers);
foreach (const Docker::Container& container, containers.get()) {
EXPECT_NE("/" + containerName, container.name);
}
containers = docker.ps(true, containerName);
AWAIT_READY(containers);
foreach (const Docker::Container& container, containers.get()) {
EXPECT_NE("/" + containerName, container.name);
}
}