bool operator == (const ExecutorInfo& left, const ExecutorInfo& right)
{
return left.executor_id() == right.executor_id() &&
left.data() == right.data() &&
Resources(left.resources()) == Resources(right.resources()) &&
left.command() == right.command() &&
left.framework_id() == right.framework_id() &&
left.name() == right.name() &&
left.source() == right.source() &&
left.container() == right.container() &&
left.discovery() == right.discovery();
}
inline bool operator == (const ExecutorInfo& left, const ExecutorInfo& right)
{
return left.executor_id() == right.executor_id() &&
left.has_framework_id() == right.has_framework_id() &&
(!left.has_framework_id() ||
(left.framework_id() == right.framework_id())) &&
left.command() == right.command() &&
Resources(left.resources()) == Resources(right.resources()) &&
left.has_name() == right.has_name() &&
(!left.has_name() || (left.name() == right.name())) &&
left.has_source() == right.has_source() &&
(!left.has_source() || (left.source() == right.source())) &&
left.has_data() == right.has_data() &&
(!left.has_data() || (left.data() == right.data()));
}
// In this hook, we create a new environment variable "FOO" and set
// it's value to "bar".
virtual Result<Environment> slaveExecutorEnvironmentDecorator(
const ExecutorInfo& executorInfo)
{
LOG(INFO) << "Executing 'slaveExecutorEnvironmentDecorator' hook";
Environment environment;
if (executorInfo.command().has_environment()) {
environment.CopyFrom(executorInfo.command().environment());
}
Environment::Variable* variable = environment.add_variables();
variable->set_name("FOO");
variable->set_value("bar");
return environment;
}
process::Future<Option<ContainerPrepareInfo>> CalicoIsolatorProcess::prepare(
const ContainerID& containerId,
const ExecutorInfo& executorInfo,
const std::string& directory,
const Option<std::string>& user)
{
LOG(INFO) << "CalicoIsolator::prepare";
std::string ipAddress = "auto";
std::string profile = "none";
foreach (const Environment_Variable& var,
executorInfo.command().environment().variables()) {
LOG(INFO) << "ENV: " << var.name() << "=" << var.value();
if (var.name() == "CALICO_IP") {
ipAddress = var.value();
}
else if (var.name() == "CALICO_PROFILE") {
profile = var.value();
}
}
if (ipAddress == "auto") {
std::vector<std::string> argv(3);
argv[0] = "python";
argv[1] = ipamPath;
argv[2] = "assign_ipv4";
Try<process::Subprocess> child = process::subprocess(
pythonPath,
argv,
process::Subprocess::PIPE(),
process::Subprocess::PIPE(),
process::Subprocess::PIPE());
CHECK_SOME(child);
waitpid(child.get().pid(), NULL, 0);
ipAddress = process::io::read(child.get().out().get()).get();
LOG(INFO) << "Got IP " << ipAddress << " from IPAM.";
}
LOG(INFO) << "LIBPROCESS_IP=" << ipAddress;
ContainerPrepareInfo prepareInfo;
Environment::Variable* variable =
prepareInfo.mutable_environment()->add_variables();
variable->set_name("LIBPROCESS_IP");
variable->set_value(ipAddress);
foreach (const Parameter& parameter, parameters.parameter()) {
if (parameter.key() == initializationKey) {
prepareInfo.add_commands()->set_value(parameter.value());
}
}
(*infos)[containerId] = new Info(ipAddress, profile);
(*executors)[executorInfo.executor_id()] = containerId;
return prepareInfo;
}
// This hook locates the file created by environment decorator hook
// and deletes it.
virtual Try<Nothing> slaveRemoveExecutorHook(
const FrameworkInfo& frameworkInfo,
const ExecutorInfo& executorInfo)
{
LOG(INFO) << "Executing 'slaveRemoveExecutorHook'";
foreach (const Environment::Variable& variable,
executorInfo.command().environment().variables()) {
if (variable.name() == testEnvironmentVariableName) {
string path = variable.value();
// The removeExecutor hook may be called multiple times; thus
// we ignore the subsequent calls.
if (os::isfile(path)) {
CHECK_SOME(os::rm(path));
}
break;
}
}
return Nothing();
}
Environment HookManager::slaveExecutorEnvironmentDecorator(
ExecutorInfo executorInfo)
{
Lock lock(&mutex);
foreachpair (const string& name, Hook* hook, availableHooks) {
const Result<Environment>& result =
hook->slaveExecutorEnvironmentDecorator(executorInfo);
if (result.isSome()) {
// Update executorInfo to include newer environment variables
// so that the next hook module can extend the environment
// variables instead of simply overwriting them.
executorInfo.mutable_command()->mutable_environment()->MergeFrom(
result.get());
} else if (result.isError()) {
LOG(WARNING) << "Slave environment decorator hook failed for module '"
<< name << "': " << result.error();
}
}
return executorInfo.command().environment();
}
Future<bool> launch(
const ContainerID& containerId,
const Option<TaskInfo>& taskInfo,
const ExecutorInfo& executorInfo,
const string& directory,
const Option<string>& user,
const SlaveID& slaveId,
const map<string, string>& environment,
bool checkpoint)
{
CHECK(!terminatedContainers.contains(containerId))
<< "Failed to launch nested container " << containerId
<< " for executor '" << executorInfo.executor_id() << "'"
<< " of framework " << executorInfo.framework_id()
<< " because this ContainerID is being re-used with"
<< " a previously terminated container";
CHECK(!containers_.contains(containerId))
<< "Failed to launch container " << containerId
<< " for executor '" << executorInfo.executor_id() << "'"
<< " of framework " << executorInfo.framework_id()
<< " because it is already launched";
CHECK(executors.contains(executorInfo.executor_id()))
<< "Failed to launch executor '" << executorInfo.executor_id() << "'"
<< " of framework " << executorInfo.framework_id()
<< " because it is unknown to the containerizer";
containers_[containerId] = Owned<ContainerData>(new ContainerData());
containers_.at(containerId)->executorId = executorInfo.executor_id();
containers_.at(containerId)->frameworkId = executorInfo.framework_id();
// We need to synchronize all reads and writes to the environment
// as this is global state.
//
// TODO(jmlvanre): Even this is not sufficient, as other aspects
// of the code may read an environment variable while we are
// manipulating it. The better solution is to pass the environment
// variables into the fork, or to set them on the command line.
// See MESOS-3475.
static std::mutex mutex;
synchronized(mutex) {
// Since the constructor for `MesosExecutorDriver` reads
// environment variables to load flags, even it needs to
// be within this synchronization section.
//
// Prepare additional environment variables for the executor.
// TODO(benh): Need to get flags passed into the TestContainerizer
// in order to properly use here.
slave::Flags flags;
flags.recovery_timeout = Duration::zero();
// We need to save the original set of environment variables so we
// can reset the environment after calling 'driver->start()' below.
hashmap<string, string> original = os::environment();
foreachpair (const string& name, const string variable, environment) {
os::setenv(name, variable);
}
// TODO(benh): Can this be removed and done exclusively in the
// 'executorEnvironment()' function? There are other places in the
// code where we do this as well and it's likely we can do this once
// in 'executorEnvironment()'.
foreach (const Environment::Variable& variable,
executorInfo.command().environment().variables()) {
os::setenv(variable.name(), variable.value());
}
os::setenv("MESOS_LOCAL", "1");
const Owned<ExecutorData>& executorData =
executors.at(executorInfo.executor_id());
if (executorData->executor != nullptr) {
executorData->driver = Owned<MesosExecutorDriver>(
new MesosExecutorDriver(executorData->executor));
executorData->driver->start();
} else {
shared_ptr<v1::MockHTTPExecutor> executor =
executorData->v1ExecutorMock;
executorData->v1Library = Owned<v1::executor::TestMesos>(
new v1::executor::TestMesos(ContentType::PROTOBUF, executor));
}
os::unsetenv("MESOS_LOCAL");
// Unset the environment variables we set by resetting them to their
// original values and also removing any that were not part of the
// original environment.
foreachpair (const string& name, const string& value, original) {
os::setenv(name, value);
}
// Prepare runs BEFORE a task is started
// will check if the volume is already mounted and if not,
// will mount the volume.
// A container can ask for multiple mounts, but if
// there are any problems parsing or mounting even one
// mount, we want to exit with an error and no new
// mounted volumes. Goal: make all mounts or none.
Future<Option<ContainerPrepareInfo>> DockerVolumeDriverIsolator::prepare(
const ContainerID& containerId,
const ExecutorInfo& executorInfo,
const string& directory,
const Option<string>& user)
{
LOG(INFO) << "Preparing external storage for container: "
<< stringify(containerId);
// Get things we need from task's environment in ExecutoInfo.
if (!executorInfo.command().has_environment()) {
// No environment means no external volume specification.
// Not an error, just nothing to do, so return None.
LOG(INFO) << "No environment specified for container ";
return None();
}
// In the future we aspire to accepting a json mount list
// some un-used "scaffolding" is in place now for this
JSON::Object environment;
JSON::Array jsonVariables;
// We accept <environment-var-name>#, where # can be 1-9, saved in array[#].
// We also accept <environment-var-name>, saved in array[0].
static constexpr size_t ARRAY_SIZE = 10;
std::array<std::string, ARRAY_SIZE> deviceDriverNames;
std::array<std::string, ARRAY_SIZE> volumeNames;
std::array<std::string, ARRAY_SIZE> mountOptions;
// Iterate through the environment variables,
// looking for the ones we need.
foreach (const auto &variable,
executorInfo.command().environment().variables()) {
JSON::Object variableObject;
variableObject.values["name"] = variable.name();
variableObject.values["value"] = variable.value();
jsonVariables.values.push_back(variableObject);
if (strings::startsWith(variable.name(), VOL_NAME_ENV_VAR_NAME)) {
if (containsProhibitedChars(variable.value())) {
LOG(ERROR) << "Environment variable " << variable.name()
<< " rejected because it's value contains "
<< "prohibited characters";
return Failure("prepare() failed due to illegal environment variable");
}
const size_t prefixLength = strlen(VOL_NAME_ENV_VAR_NAME);
if (variable.name().length() == prefixLength) {
volumeNames[0] = variable.value();
} else if (variable.name().length() == (prefixLength+1)) {
char digit = variable.name().data()[prefixLength];
if (isdigit(digit)) {
size_t index =
std::atoi(variable.name().substr(prefixLength).c_str());
if (index !=0) {
volumeNames[index] = variable.value();
}
}
}
LOG(INFO) << "External volume name (" << variable.value()
<< ") parsed from environment";
} else if (strings::startsWith(variable.name(), VOL_DRIVER_ENV_VAR_NAME)) {
if (containsProhibitedChars(variable.value())) {
LOG(ERROR) << "Environment variable " << variable.name()
<< " rejected because it's value contains prohibited characters";
return Failure("prepare() failed due to illegal environment variable");
}
const size_t prefixLength = strlen(VOL_DRIVER_ENV_VAR_NAME);
if (variable.name().length() == prefixLength) {
deviceDriverNames[0] = variable.value();
} else if (variable.name().length() == (prefixLength+1)) {
char digit = variable.name().data()[prefixLength];
if (isdigit(digit)) {
size_t index =
std::atoi(variable.name().substr(prefixLength).c_str());
if (index !=0) {
deviceDriverNames[index] = variable.value();
}
}
}
} else if (strings::startsWith(variable.name(), VOL_OPTS_ENV_VAR_NAME)) {
if (containsProhibitedChars(variable.value())) {
LOG(ERROR) << "Environment variable " << variable.name()
<< " rejected because it's value contains prohibited characters";
return Failure("prepare() failed due to illegal environment variable");
}
const size_t prefixLength = strlen(VOL_OPTS_ENV_VAR_NAME);
if (variable.name().length() == prefixLength) {
mountOptions[0] = variable.value();
} else if (variable.name().length() == (prefixLength+1)) {
char digit = variable.name().data()[prefixLength];
if (isdigit(digit)) {
size_t index =
std::atoi(variable.name().substr(prefixLength).c_str());
if (index !=0) {
mountOptions[index] = variable.value();
}
}
}
} else if (variable.name() == JSON_VOLS_ENV_VAR_NAME) {
//JSON::Value jsonVolArray = JSON::parse(variable.value());
}
}
// TODO: json environment is not used yet
environment.values["variables"] = jsonVariables;
// requestedExternalMounts is all mounts requested by container.
std::vector<process::Owned<ExternalMount>> requestedExternalMounts;
// unconnectedExternalMounts is the subset of those not already
// in use by another container.
std::vector<process::Owned<ExternalMount>> unconnectedExternalMounts;
// prevConnectedExternalMounts is the subset of those that are
// in use by another container.
std::vector<process::Owned<ExternalMount>> prevConnectedExternalMounts;
// Not using iterator because we access all 3 arrays using common index.
for (size_t i = 0; i < volumeNames.size(); i++) {
if (volumeNames[i].empty()) {
continue;
}
LOG(INFO) << "Validating mount name " << volumeNames[i];
if (deviceDriverNames[i].empty()) {
deviceDriverNames[i] = VOL_DRIVER_DEFAULT;
}
process::Owned<ExternalMount> mount(
new ExternalMount(deviceDriverNames[i],
volumeNames[i],
mountOptions[i]));
// Check for duplicates in environment.
bool duplicateInEnv = false;
for (const auto &ent : requestedExternalMounts) {
if (ent.get()->getExternalMountId() ==
mount.get()->getExternalMountId()) {
duplicateInEnv = true;
break;
}
}
if (duplicateInEnv) {
LOG(INFO) << "Duplicate mount request(" << *mount
<< ") in environment will be ignored";
continue;
}
requestedExternalMounts.push_back(mount);
// Now check if another container is already using this same mount.
bool mountInUse = false;
for (const auto &ent : infos) {
if (ent.second.get()->getExternalMountId() ==
mount.get()->getExternalMountId()) {
mountInUse = true;
prevConnectedExternalMounts.push_back(ent.second);
LOG(INFO) << "Requested mount(" << *mount
<< ") is already mounted by another container";
break;
}
}
if (!mountInUse) {
unconnectedExternalMounts.push_back(mount);
}
}
// As we connect mounts we will build a list of successful mounts.
// We need this because, if there is a failure, we need to unmount these.
// The goal is we mount either ALL or NONE.
std::vector<process::Owned<ExternalMount>> successfulExternalMounts;
for (const auto &iter : unconnectedExternalMounts) {
std::string mountpoint = mount(*iter, "prepare()");
if (!mountpoint.empty()) {
// Need to construct a newExternalMount because we just
// learned the mountpoint.
process::Owned<ExternalMount> newmount(
new ExternalMount(iter->deviceDriverName,
iter->volumeName,
iter->mountOptions,
mountpoint));
successfulExternalMounts.push_back(newmount);
} else {
// Once any mount attempt fails, give up on whole list
// and attempt to undo the mounts we already made.
LOG(ERROR) << "Mount failed during prepare()";
for (const auto &unmountme : successfulExternalMounts) {
if (unmount(*unmountme, "prepare()-reverting mounts after failure")) {
LOG(ERROR) << "During prepare() of a container requesting multiple "
<< "mounts, a mount failure occurred after making "
<< "at least one mount and a second failure occurred "
<< "while attempting to remove the earlier mount(s)";
break;
}
}
return Failure("prepare() failed during mount attempt");
}
}
// Note: infos has a record for each mount associated with this container
// even if the mount is also used by another container.
for (const auto &iter : prevConnectedExternalMounts) {
infos.put(containerId, iter);
}
for (const auto &iter : successfulExternalMounts) {
infos.put(containerId, iter);
}
//checkpoint the dvdi mounts for persistence
std::string myinfosout;
dumpInfos(myinfosout);
mesos::internal::slave::state::checkpoint(mountJsonFilename, myinfosout);
return None();
}
Future<bool> TestContainerizer::_launch(
const ContainerID& containerId,
const ExecutorInfo& executorInfo,
const string& directory,
const Option<string>& user,
const SlaveID& slaveId,
const PID<slave::Slave>& slavePid,
bool checkpoint)
{
CHECK(!drivers.contains(containerId))
<< "Failed to launch executor " << executorInfo.executor_id()
<< " of framework " << executorInfo.framework_id()
<< " because it is already launched";
CHECK(executors.contains(executorInfo.executor_id()))
<< "Failed to launch executor " << executorInfo.executor_id()
<< " of framework " << executorInfo.framework_id()
<< " because it is unknown to the containerizer";
// Store mapping from (frameworkId, executorId) -> containerId to facilitate
// easy destroy from tests.
std::pair<FrameworkID, ExecutorID> key(executorInfo.framework_id(),
executorInfo.executor_id());
containers_[key] = containerId;
Executor* executor = executors[executorInfo.executor_id()];
// We need to synchronize all reads and writes to the environment as this is
// global state.
// TODO(jmlvanre): Even this is not sufficient, as other aspects of the code
// may read an environment variable while we are manipulating it. The better
// solution is to pass the environment variables into the fork, or to set them
// on the command line. See MESOS-3475.
static std::mutex mutex;
synchronized(mutex) {
// Since the constructor for `MesosExecutorDriver` reads environment
// variables to load flags, even it needs to be within this synchronization
// section.
Owned<MesosExecutorDriver> driver(new MesosExecutorDriver(executor));
drivers[containerId] = driver;
// Prepare additional environment variables for the executor.
// TODO(benh): Need to get flags passed into the TestContainerizer
// in order to properly use here.
slave::Flags flags;
flags.recovery_timeout = Duration::zero();
// We need to save the original set of environment variables so we
// can reset the environment after calling 'driver->start()' below.
hashmap<string, string> original = os::environment();
const map<string, string> environment = executorEnvironment(
executorInfo,
directory,
slaveId,
slavePid,
checkpoint,
flags);
foreachpair (const string& name, const string variable, environment) {
os::setenv(name, variable);
}
// TODO(benh): Can this be removed and done exlusively in the
// 'executorEnvironment()' function? There are other places in the
// code where we do this as well and it's likely we can do this once
// in 'executorEnvironment()'.
foreach (const Environment::Variable& variable,
executorInfo.command().environment().variables()) {
os::setenv(variable.name(), variable.value());
}
os::setenv("MESOS_LOCAL", "1");
driver->start();
os::unsetenv("MESOS_LOCAL");
// Unset the environment variables we set by resetting them to their
// original values and also removing any that were not part of the
// original environment.
foreachpair (const string& name, const string& value, original) {
os::setenv(name, value);
}