TYPED_TEST(MemIsolatorTest, MemUsage)
{
slave::Flags flags;
Try<Isolator*> isolator = TypeParam::create(flags);
CHECK_SOME(isolator);
ExecutorInfo executorInfo;
executorInfo.mutable_resources()->CopyFrom(
Resources::parse("mem:1024").get());
ContainerID containerId;
containerId.set_value(UUID::random().toString());
// Use a relative temporary directory so it gets cleaned up
// automatically with the test.
Try<string> dir = os::mkdtemp(path::join(os::getcwd(), "XXXXXX"));
ASSERT_SOME(dir);
ContainerConfig containerConfig;
containerConfig.mutable_executor_info()->CopyFrom(executorInfo);
containerConfig.set_directory(dir.get());
AWAIT_READY(isolator.get()->prepare(
containerId,
containerConfig));
MemoryTestHelper helper;
ASSERT_SOME(helper.spawn());
ASSERT_SOME(helper.pid());
// Set up the reaper to wait on the subprocess.
Future<Option<int>> status = process::reap(helper.pid().get());
// Isolate the subprocess.
AWAIT_READY(isolator.get()->isolate(containerId, helper.pid().get()));
const Bytes allocation = Megabytes(128);
EXPECT_SOME(helper.increaseRSS(allocation));
Future<ResourceStatistics> usage = isolator.get()->usage(containerId);
AWAIT_READY(usage);
EXPECT_GE(usage.get().mem_rss_bytes(), allocation.bytes());
// Ensure the process is killed.
helper.cleanup();
// Make sure the subprocess was reaped.
AWAIT_READY(status);
// Let the isolator clean up.
AWAIT_READY(isolator.get()->cleanup(containerId));
delete isolator.get();
}
// A test to verify the number of processes and threads in a
// container.
TEST_F(LimitedCpuIsolatorTest, ROOT_CGROUPS_Pids_and_Tids)
{
slave::Flags flags;
flags.cgroups_cpu_enable_pids_and_tids_count = true;
Try<Isolator*> isolator = CgroupsCpushareIsolatorProcess::create(flags);
CHECK_SOME(isolator);
Try<Launcher*> launcher = LinuxLauncher::create(flags);
CHECK_SOME(launcher);
ExecutorInfo executorInfo;
executorInfo.mutable_resources()->CopyFrom(
Resources::parse("cpus:0.5;mem:512").get());
ContainerID containerId;
containerId.set_value(UUID::random().toString());
// Use a relative temporary directory so it gets cleaned up
// automatically with the test.
Try<string> dir = os::mkdtemp(path::join(os::getcwd(), "XXXXXX"));
ASSERT_SOME(dir);
ContainerConfig containerConfig;
containerConfig.mutable_executor_info()->CopyFrom(executorInfo);
containerConfig.set_directory(dir.get());
Future<Option<ContainerLaunchInfo>> prepare =
isolator.get()->prepare(
containerId,
containerConfig);
AWAIT_READY(prepare);
// Right after the creation of the cgroup, which happens in
// 'prepare', we check that it is empty.
Future<ResourceStatistics> usage = isolator.get()->usage(containerId);
AWAIT_READY(usage);
EXPECT_EQ(0U, usage.get().processes());
EXPECT_EQ(0U, usage.get().threads());
int pipes[2];
ASSERT_NE(-1, ::pipe(pipes));
vector<string> argv(1);
argv[0] = "cat";
Try<pid_t> pid = launcher.get()->fork(
containerId,
"cat",
argv,
Subprocess::FD(STDIN_FILENO),
Subprocess::FD(STDOUT_FILENO),
Subprocess::FD(STDERR_FILENO),
None(),
None(),
lambda::bind(&childSetup, pipes),
prepare.get().isSome() ? prepare.get().get().namespaces() : 0);
ASSERT_SOME(pid);
// Reap the forked child.
Future<Option<int>> status = process::reap(pid.get());
// Continue in the parent.
ASSERT_SOME(os::close(pipes[0]));
// Before isolation, the cgroup is empty.
usage = isolator.get()->usage(containerId);
AWAIT_READY(usage);
EXPECT_EQ(0U, usage.get().processes());
EXPECT_EQ(0U, usage.get().threads());
// Isolate the forked child.
AWAIT_READY(isolator.get()->isolate(containerId, pid.get()));
// After the isolation, the cgroup is not empty, even though the
// process hasn't exec'd yet.
usage = isolator.get()->usage(containerId);
AWAIT_READY(usage);
EXPECT_EQ(1U, usage.get().processes());
EXPECT_EQ(1U, usage.get().threads());
// Now signal the child to continue.
char dummy;
ASSERT_LT(0, ::write(pipes[1], &dummy, sizeof(dummy)));
ASSERT_SOME(os::close(pipes[1]));
// Process count should be 1 since 'sleep' is still sleeping.
usage = isolator.get()->usage(containerId);
AWAIT_READY(usage);
EXPECT_EQ(1U, usage.get().processes());
EXPECT_EQ(1U, usage.get().threads());
// Ensure all processes are killed.
AWAIT_READY(launcher.get()->destroy(containerId));
// Wait for the command to complete.
AWAIT_READY(status);
// After the process is killed, the cgroup should be empty again.
usage = isolator.get()->usage(containerId);
AWAIT_READY(usage);
EXPECT_EQ(0U, usage.get().processes());
EXPECT_EQ(0U, usage.get().threads());
// Let the isolator clean up.
AWAIT_READY(isolator.get()->cleanup(containerId));
delete isolator.get();
delete launcher.get();
}
// This test verifies that we can successfully launch a container with
// a big (>= 10 cpus) cpu quota. This is to catch the regression
// observed in MESOS-1049.
// TODO(vinod): Revisit this if/when the isolator restricts the number
// of cpus that an executor can use based on the slave cpus.
TEST_F(LimitedCpuIsolatorTest, ROOT_CGROUPS_CFS_Big_Quota)
{
slave::Flags flags;
// Enable CFS to cap CPU utilization.
flags.cgroups_enable_cfs = true;
Try<Isolator*> isolator = CgroupsCpushareIsolatorProcess::create(flags);
CHECK_SOME(isolator);
Try<Launcher*> launcher = LinuxLauncher::create(flags);
CHECK_SOME(launcher);
// Set the executor's resources to 100.5 cpu.
ExecutorInfo executorInfo;
executorInfo.mutable_resources()->CopyFrom(
Resources::parse("cpus:100.5").get());
ContainerID containerId;
containerId.set_value(UUID::random().toString());
// Use a relative temporary directory so it gets cleaned up
// automatically with the test.
Try<string> dir = os::mkdtemp(path::join(os::getcwd(), "XXXXXX"));
ASSERT_SOME(dir);
ContainerConfig containerConfig;
containerConfig.mutable_executor_info()->CopyFrom(executorInfo);
containerConfig.set_directory(dir.get());
Future<Option<ContainerLaunchInfo>> prepare =
isolator.get()->prepare(
containerId,
containerConfig);
AWAIT_READY(prepare);
int pipes[2];
ASSERT_NE(-1, ::pipe(pipes));
vector<string> argv(3);
argv[0] = "sh";
argv[1] = "-c";
argv[2] = "exit 0";
Try<pid_t> pid = launcher.get()->fork(
containerId,
"sh",
argv,
Subprocess::FD(STDIN_FILENO),
Subprocess::FD(STDOUT_FILENO),
Subprocess::FD(STDERR_FILENO),
None(),
None(),
lambda::bind(&childSetup, pipes),
prepare.get().isSome() ? prepare.get().get().namespaces() : 0);
ASSERT_SOME(pid);
// Reap the forked child.
Future<Option<int> > status = process::reap(pid.get());
// Continue in the parent.
ASSERT_SOME(os::close(pipes[0]));
// Isolate the forked child.
AWAIT_READY(isolator.get()->isolate(containerId, pid.get()));
// Now signal the child to continue.
char dummy;
ASSERT_LT(0, ::write(pipes[1], &dummy, sizeof(dummy)));
ASSERT_SOME(os::close(pipes[1]));
// Wait for the command to complete successfully.
AWAIT_READY(status);
ASSERT_SOME_EQ(0, status.get());
// Ensure all processes are killed.
AWAIT_READY(launcher.get()->destroy(containerId));
// Let the isolator clean up.
AWAIT_READY(isolator.get()->cleanup(containerId));
delete isolator.get();
delete launcher.get();
}
TEST_F(LimitedCpuIsolatorTest, ROOT_CGROUPS_CFS_Enable_Cfs)
{
slave::Flags flags;
// Enable CFS to cap CPU utilization.
flags.cgroups_enable_cfs = true;
Try<Isolator*> isolator = CgroupsCpushareIsolatorProcess::create(flags);
CHECK_SOME(isolator);
Try<Launcher*> launcher = LinuxLauncher::create(flags);
CHECK_SOME(launcher);
// Set the executor's resources to 0.5 cpu.
ExecutorInfo executorInfo;
executorInfo.mutable_resources()->CopyFrom(
Resources::parse("cpus:0.5").get());
ContainerID containerId;
containerId.set_value(UUID::random().toString());
// Use a relative temporary directory so it gets cleaned up
// automatically with the test.
Try<string> dir = os::mkdtemp(path::join(os::getcwd(), "XXXXXX"));
ASSERT_SOME(dir);
ContainerConfig containerConfig;
containerConfig.mutable_executor_info()->CopyFrom(executorInfo);
containerConfig.set_directory(dir.get());
Future<Option<ContainerLaunchInfo>> prepare =
isolator.get()->prepare(
containerId,
containerConfig);
AWAIT_READY(prepare);
// Generate random numbers to max out a single core. We'll run this for 0.5
// seconds of wall time so it should consume approximately 250 ms of total
// cpu time when limited to 0.5 cpu. We use /dev/urandom to prevent blocking
// on Linux when there's insufficient entropy.
string command = "cat /dev/urandom > /dev/null & "
"export MESOS_TEST_PID=$! && "
"sleep 0.5 && "
"kill $MESOS_TEST_PID";
int pipes[2];
ASSERT_NE(-1, ::pipe(pipes));
vector<string> argv(3);
argv[0] = "sh";
argv[1] = "-c";
argv[2] = command;
Try<pid_t> pid = launcher.get()->fork(
containerId,
"sh",
argv,
Subprocess::FD(STDIN_FILENO),
Subprocess::FD(STDOUT_FILENO),
Subprocess::FD(STDERR_FILENO),
None(),
None(),
lambda::bind(&childSetup, pipes),
prepare.get().isSome() ? prepare.get().get().namespaces() : 0);
ASSERT_SOME(pid);
// Reap the forked child.
Future<Option<int> > status = process::reap(pid.get());
// Continue in the parent.
ASSERT_SOME(os::close(pipes[0]));
// Isolate the forked child.
AWAIT_READY(isolator.get()->isolate(containerId, pid.get()));
// Now signal the child to continue.
char dummy;
ASSERT_LT(0, ::write(pipes[1], &dummy, sizeof(dummy)));
ASSERT_SOME(os::close(pipes[1]));
// Wait for the command to complete.
AWAIT_READY(status);
Future<ResourceStatistics> usage = isolator.get()->usage(containerId);
AWAIT_READY(usage);
// Expect that no more than 300 ms of cpu time has been consumed. We also
// check that at least 50 ms of cpu time has been consumed so this test will
// fail if the host system is very heavily loaded. This behavior is correct
// because under such conditions we aren't actually testing the CFS cpu
// limiter.
double cpuTime = usage.get().cpus_system_time_secs() +
usage.get().cpus_user_time_secs();
EXPECT_GE(0.30, cpuTime);
EXPECT_LE(0.05, cpuTime);
// Ensure all processes are killed.
AWAIT_READY(launcher.get()->destroy(containerId));
// Let the isolator clean up.
AWAIT_READY(isolator.get()->cleanup(containerId));
delete isolator.get();
delete launcher.get();
}
TEST_F(RevocableCpuIsolatorTest, ROOT_CGROUPS_RevocableCpu)
{
slave::Flags flags;
Try<Isolator*> isolator = CgroupsCpushareIsolatorProcess::create(flags);
CHECK_SOME(isolator);
Try<Launcher*> launcher = PosixLauncher::create(flags);
// Include revocable CPU in the executor's resources.
Resource cpu = Resources::parse("cpus", "1", "*").get();
cpu.mutable_revocable();
ExecutorInfo executorInfo;
executorInfo.add_resources()->CopyFrom(cpu);
ContainerID containerId;
containerId.set_value(UUID::random().toString());
ContainerConfig containerConfig;
containerConfig.mutable_executor_info()->CopyFrom(executorInfo);
containerConfig.set_directory(os::getcwd());
AWAIT_READY(isolator.get()->prepare(
containerId,
containerConfig));
vector<string> argv{"sleep", "100"};
Try<pid_t> pid = launcher.get()->fork(
containerId,
"/bin/sleep",
argv,
Subprocess::PATH("/dev/null"),
Subprocess::PATH("/dev/null"),
Subprocess::PATH("/dev/null"),
None(),
None(),
None(),
None());
ASSERT_SOME(pid);
AWAIT_READY(isolator.get()->isolate(containerId, pid.get()));
// Executor should have proper cpu.shares for revocable containers.
Result<string> cpuHierarchy = cgroups::hierarchy("cpu");
ASSERT_SOME(cpuHierarchy);
Result<string> cpuCgroup = cgroups::cpu::cgroup(pid.get());
ASSERT_SOME(cpuCgroup);
EXPECT_SOME_EQ(
CPU_SHARES_PER_CPU_REVOCABLE,
cgroups::cpu::shares(cpuHierarchy.get(), cpuCgroup.get()));
// Kill the container and clean up.
Future<Option<int>> status = process::reap(pid.get());
AWAIT_READY(launcher.get()->destroy(containerId));
AWAIT_READY(status);
AWAIT_READY(isolator.get()->cleanup(containerId));
delete isolator.get();
delete launcher.get();
}
TYPED_TEST(CpuIsolatorTest, SystemCpuUsage)
{
slave::Flags flags;
Try<Isolator*> isolator = TypeParam::create(flags);
CHECK_SOME(isolator);
// A PosixLauncher is sufficient even when testing a cgroups isolator.
Try<Launcher*> launcher = PosixLauncher::create(flags);
ExecutorInfo executorInfo;
executorInfo.mutable_resources()->CopyFrom(
Resources::parse("cpus:1.0").get());
ContainerID containerId;
containerId.set_value(UUID::random().toString());
// Use a relative temporary directory so it gets cleaned up
// automatically with the test.
Try<string> dir = os::mkdtemp(path::join(os::getcwd(), "XXXXXX"));
ASSERT_SOME(dir);
ContainerConfig containerConfig;
containerConfig.mutable_executor_info()->CopyFrom(executorInfo);
containerConfig.set_directory(dir.get());
AWAIT_READY(isolator.get()->prepare(
containerId,
containerConfig));
const string& file = path::join(dir.get(), "mesos_isolator_test_ready");
// Generating random numbers is done by the kernel and will max out a single
// core and run almost exclusively in the kernel, i.e., system time.
string command = "cat /dev/urandom > /dev/null & "
"touch " + file + "; " // Signals the command is running.
"sleep 60";
int pipes[2];
ASSERT_NE(-1, ::pipe(pipes));
vector<string> argv(3);
argv[0] = "sh";
argv[1] = "-c";
argv[2] = command;
Try<pid_t> pid = launcher.get()->fork(
containerId,
"sh",
argv,
Subprocess::FD(STDIN_FILENO),
Subprocess::FD(STDOUT_FILENO),
Subprocess::FD(STDERR_FILENO),
None(),
None(),
lambda::bind(&childSetup, pipes),
None());
ASSERT_SOME(pid);
// Reap the forked child.
Future<Option<int> > status = process::reap(pid.get());
// Continue in the parent.
ASSERT_SOME(os::close(pipes[0]));
// Isolate the forked child.
AWAIT_READY(isolator.get()->isolate(containerId, pid.get()));
// Now signal the child to continue.
char dummy;
ASSERT_LT(0, ::write(pipes[1], &dummy, sizeof(dummy)));
ASSERT_SOME(os::close(pipes[1]));
// Wait for the command to start.
while (!os::exists(file));
// Wait up to 1 second for the child process to induce 1/8 of a second of
// system cpu time.
ResourceStatistics statistics;
Duration waited = Duration::zero();
do {
Future<ResourceStatistics> usage = isolator.get()->usage(containerId);
AWAIT_READY(usage);
statistics = usage.get();
// If we meet our usage expectations, we're done!
if (statistics.cpus_system_time_secs() >= 0.125) {
break;
}
os::sleep(Milliseconds(200));
waited += Milliseconds(200);
} while (waited < Seconds(1));
EXPECT_LE(0.125, statistics.cpus_system_time_secs());
// Ensure all processes are killed.
AWAIT_READY(launcher.get()->destroy(containerId));
// Make sure the child was reaped.
AWAIT_READY(status);
// Let the isolator clean up.
AWAIT_READY(isolator.get()->cleanup(containerId));
delete isolator.get();
delete launcher.get();
}
