Future<Option<ContainerLaunchInfo>> PosixFilesystemIsolatorProcess::prepare(
const ContainerID& containerId,
const ContainerConfig& containerConfig)
{
if (infos.contains(containerId)) {
return Failure("Container has already been prepared");
}
const ExecutorInfo& executorInfo = containerConfig.executor_info();
if (executorInfo.has_container()) {
CHECK_EQ(executorInfo.container().type(), ContainerInfo::MESOS);
// Return failure if the container change the filesystem root
// because the symlinks will become invalid in the new root.
if (executorInfo.container().mesos().has_image()) {
return Failure("Container root filesystems not supported");
}
if (executorInfo.container().volumes().size() > 0) {
return Failure("Volumes in ContainerInfo is not supported");
}
}
infos.put(containerId, Owned<Info>(new Info(containerConfig.directory())));
return update(containerId, executorInfo.resources())
.then([]() -> Future<Option<ContainerLaunchInfo>> { return None(); });
}
Future<Option<ContainerLaunchInfo>> LinuxFilesystemIsolatorProcess::prepare(
const ContainerID& containerId,
const ContainerConfig& containerConfig)
{
const string& directory = containerConfig.directory();
Option<string> user;
if (containerConfig.has_user()) {
user = containerConfig.user();
}
if (infos.contains(containerId)) {
return Failure("Container has already been prepared");
}
Owned<Info> info(new Info(
directory,
containerConfig.executor_info()));
infos.put(containerId, info);
ContainerLaunchInfo launchInfo;
launchInfo.set_namespaces(CLONE_NEWNS);
// Prepare the commands that will be run in the container's mount
// namespace right after forking the executor process. We use these
// commands to mount those volumes specified in the container info
// so that they don't pollute the host mount namespace.
Try<string> _script = script(containerId, containerConfig);
if (_script.isError()) {
return Failure("Failed to generate isolation script: " + _script.error());
}
CommandInfo* command = launchInfo.add_commands();
command->set_value(_script.get());
return update(containerId, containerConfig.executor_info().resources())
.then([launchInfo]() -> Future<Option<ContainerLaunchInfo>> {
return launchInfo;
});
}
Try<string> LinuxFilesystemIsolatorProcess::script(
const ContainerID& containerId,
const ContainerConfig& containerConfig)
{
ostringstream out;
out << "#!/bin/sh\n";
out << "set -x -e\n";
// Make sure mounts in the container mount namespace do not
// propagate back to the host mount namespace.
// NOTE: We cannot simply run `mount --make-rslave /`, for more info
// please refer to comments in mount.hpp.
MesosContainerizerMount::Flags mountFlags;
mountFlags.operation = MesosContainerizerMount::MAKE_RSLAVE;
mountFlags.path = "/";
out << path::join(flags.launcher_dir, "mesos-containerizer") << " "
<< MesosContainerizerMount::NAME << " "
<< stringify(mountFlags) << "\n";
if (!containerConfig.executor_info().has_container()) {
return out.str();
}
// Bind mount the sandbox if the container specifies a rootfs.
if (containerConfig.has_rootfs()) {
string sandbox = path::join(
containerConfig.rootfs(),
flags.sandbox_directory);
Try<Nothing> mkdir = os::mkdir(sandbox);
if (mkdir.isError()) {
return Error(
"Failed to create sandbox mount point at '" +
sandbox + "': " + mkdir.error());
}
out << "mount -n --rbind '" << containerConfig.directory()
<< "' '" << sandbox << "'\n";
}
foreach (const Volume& volume,
containerConfig.executor_info().container().volumes()) {
// NOTE: Volumes with source will be handled by the corresponding
// isolators (e.g., docker/volume).
if (volume.has_source()) {
VLOG(1) << "Ignored a volume with source for container '"
<< containerId << "'";
continue;
}
if (!volume.has_host_path()) {
return Error("A volume misses 'host_path'");
}
// If both 'host_path' and 'container_path' are relative paths,
// return an error because the user can just directly access the
// volume in the work directory.
if (!strings::startsWith(volume.host_path(), "/") &&
!strings::startsWith(volume.container_path(), "/")) {
return Error(
"Both 'host_path' and 'container_path' of a volume are relative");
}
// Determine the source of the mount.
string source;
if (strings::startsWith(volume.host_path(), "/")) {
source = volume.host_path();
// An absolute path must already exist.
if (!os::exists(source)) {
return Error("Absolute host path does not exist");
}
} else {
// Path is interpreted as relative to the work directory.
source = path::join(containerConfig.directory(), volume.host_path());
// TODO(jieyu): We need to check that source resolves under the
// work directory because a user can potentially use a container
// path like '../../abc'.
Try<Nothing> mkdir = os::mkdir(source);
if (mkdir.isError()) {
return Error(
"Failed to create the source of the mount at '" +
source + "': " + mkdir.error());
}
// TODO(idownes): Consider setting ownership and mode.
}
// Determine the target of the mount.
string target;
if (strings::startsWith(volume.container_path(), "/")) {
if (containerConfig.has_rootfs()) {
target = path::join(
containerConfig.rootfs(),
volume.container_path());
Try<Nothing> mkdir = os::mkdir(target);
if (mkdir.isError()) {
return Error(
"Failed to create the target of the mount at '" +
target + "': " + mkdir.error());
}
} else {
target = volume.container_path();
// An absolute path must already exist. This is because we
// want to avoid creating mount points outside the work
// directory in the host filesystem.
if (!os::exists(target)) {
return Error("Absolute container path does not exist");
}
}
// TODO(jieyu): We need to check that target resolves under
// 'rootfs' because a user can potentially use a container path
// like '/../../abc'.
} else {
if (containerConfig.has_rootfs()) {
target = path::join(containerConfig.rootfs(),
flags.sandbox_directory,
volume.container_path());
} else {
target = path::join(containerConfig.directory(),
volume.container_path());
}
// TODO(jieyu): We need to check that target resolves under the
// sandbox because a user can potentially use a container path
// like '../../abc'.
// NOTE: We cannot create the mount point at 'target' if
// container has rootfs defined. The bind mount of the sandbox
// will hide what's inside 'target'. So we should always create
// the mount point in 'directory'.
string mountPoint = path::join(
containerConfig.directory(),
volume.container_path());
Try<Nothing> mkdir = os::mkdir(mountPoint);
if (mkdir.isError()) {
return Error(
"Failed to create the target of the mount at '" +
mountPoint + "': " + mkdir.error());
}
}
// TODO(jieyu): Consider the mode in the volume.
out << "mount -n --rbind '" << source << "' '" << target << "'\n";
}
return out.str();
}
Future<bool> launch(
const ContainerID& containerId,
const ContainerConfig& containerConfig,
const map<string, string>& environment,
const Option<string>& pidCheckpointPath)
{
CHECK(!terminatedContainers.contains(containerId))
<< "Failed to launch nested container " << containerId
<< " for executor '" << containerConfig.executor_info().executor_id()
<< "' of framework " << containerConfig.executor_info().framework_id()
<< " because this ContainerID is being re-used with"
<< " a previously terminated container";
CHECK(!containers_.contains(containerId))
<< "Failed to launch container " << containerId
<< " for executor '" << containerConfig.executor_info().executor_id()
<< "' of framework " << containerConfig.executor_info().framework_id()
<< " because it is already launched";
containers_[containerId] = Owned<ContainerData>(new ContainerData());
if (containerId.has_parent()) {
// Launching a nested container via the test containerizer is a
// no-op for now.
return true;
}
CHECK(executors.contains(containerConfig.executor_info().executor_id()))
<< "Failed to launch executor '"
<< containerConfig.executor_info().executor_id()
<< "' of framework " << containerConfig.executor_info().framework_id()
<< " because it is unknown to the containerizer";
containers_.at(containerId)->executorId =
containerConfig.executor_info().executor_id();
containers_.at(containerId)->frameworkId =
containerConfig.executor_info().framework_id();
// We need to synchronize all reads and writes to the environment
// as this is global state.
//
// TODO(jmlvanre): Even this is not sufficient, as other aspects
// of the code may read an environment variable while we are
// manipulating it. The better solution is to pass the environment
// variables into the fork, or to set them on the command line.
// See MESOS-3475.
static std::mutex mutex;
synchronized(mutex) {
// Since the constructor for `MesosExecutorDriver` reads
// environment variables to load flags, even it needs to
// be within this synchronization section.
//
// Prepare additional environment variables for the executor.
// TODO(benh): Need to get flags passed into the TestContainerizer
// in order to properly use here.
slave::Flags flags;
flags.recovery_timeout = Duration::zero();
// We need to save the original set of environment variables so we
// can reset the environment after calling 'driver->start()' below.
hashmap<string, string> original = os::environment();
foreachpair (const string& name, const string& variable, environment) {
os::setenv(name, variable);
}
// TODO(benh): Can this be removed and done exclusively in the
// 'executorEnvironment()' function? There are other places in the
// code where we do this as well and it's likely we can do this once
// in 'executorEnvironment()'.
foreach (const Environment::Variable& variable,
containerConfig.executor_info()
.command().environment().variables()) {
os::setenv(variable.name(), variable.value());
}
os::setenv("MESOS_LOCAL", "1");
const Owned<ExecutorData>& executorData =
executors.at(containerConfig.executor_info().executor_id());
if (executorData->executor != nullptr) {
executorData->driver = Owned<MesosExecutorDriver>(
new MesosExecutorDriver(executorData->executor));
executorData->driver->start();
} else {
shared_ptr<v1::MockHTTPExecutor> executor =
executorData->v1ExecutorMock;
executorData->v1Library = Owned<v1::executor::TestMesos>(
new v1::executor::TestMesos(ContentType::PROTOBUF, executor));
}
os::unsetenv("MESOS_LOCAL");
// Unset the environment variables we set by resetting them to their
// original values and also removing any that were not part of the
// original environment.
foreachpair (const string& name, const string& value, original) {
os::setenv(name, value);
}