// This test verifies that executor API and operator API calls receive an
// unsuccessful response if the request contains a properly-signed
// authentication token with invalid claims.
TEST_F(ExecutorAuthorizationTest, FailedApiCalls)
{
Try<Owned<cluster::Master>> master = StartMaster();
ASSERT_SOME(master);
// Start an agent with permissive ACLs so that a task can be launched and the
// local authorizer's implicit executor authorization will be performed.
ACLs acls;
acls.set_permissive(true);
slave::Flags flags = CreateSlaveFlags();
flags.acls = acls;
Owned<MasterDetector> detector = master.get()->createDetector();
v1::Resources resources =
v1::Resources::parse("cpus:0.1;mem:32;disk:32").get();
v1::ExecutorInfo executorInfo;
executorInfo.set_type(v1::ExecutorInfo::DEFAULT);
executorInfo.mutable_executor_id()->CopyFrom(v1::DEFAULT_EXECUTOR_ID);
executorInfo.mutable_resources()->CopyFrom(resources);
auto executor = std::make_shared<v1::MockHTTPExecutor>();
Owned<TestContainerizer> containerizer(new TestContainerizer(
devolve(executorInfo.executor_id()), executor));
Try<Owned<cluster::Slave>> slave =
this->StartSlave(detector.get(), containerizer.get(), flags);
ASSERT_SOME(slave);
auto scheduler = std::make_shared<v1::MockHTTPScheduler>();
Future<Nothing> connected;
EXPECT_CALL(*scheduler, connected(_))
.WillOnce(FutureSatisfy(&connected));
v1::scheduler::TestMesos mesos(
master.get()->pid,
ContentType::PROTOBUF,
scheduler);
AWAIT_READY(connected);
Future<v1::scheduler::Event::Subscribed> frameworkSubscribed;
EXPECT_CALL(*scheduler, subscribed(_, _))
.WillOnce(FutureArg<1>(&frameworkSubscribed));
Future<v1::scheduler::Event::Offers> offers;
EXPECT_CALL(*scheduler, offers(_, _))
.WillOnce(FutureArg<1>(&offers))
.WillRepeatedly(Return()); // Ignore subsequent offers.
EXPECT_CALL(*scheduler, heartbeat(_))
.WillRepeatedly(Return()); // Ignore heartbeats.
mesos.send(v1::createCallSubscribe(v1::DEFAULT_FRAMEWORK_INFO));
AWAIT_READY(frameworkSubscribed);
v1::FrameworkID frameworkId(frameworkSubscribed->framework_id());
executorInfo.mutable_framework_id()->CopyFrom(frameworkId);
AWAIT_READY(offers);
ASSERT_FALSE(offers->offers().empty());
Future<v1::executor::Mesos*> executorLib;
EXPECT_CALL(*executor, connected(_))
.WillOnce(FutureArg<0>(&executorLib));
const v1::Offer& offer = offers->offers(0);
const v1::AgentID& agentId = offer.agent_id();
{
v1::scheduler::Call call;
call.mutable_framework_id()->CopyFrom(frameworkId);
call.set_type(v1::scheduler::Call::ACCEPT);
v1::scheduler::Call::Accept* accept = call.mutable_accept();
accept->add_offer_ids()->CopyFrom(offer.id());
v1::Offer::Operation* operation = accept->add_operations();
operation->set_type(v1::Offer::Operation::LAUNCH_GROUP);
v1::TaskInfo taskInfo =
v1::createTask(agentId, resources, SLEEP_COMMAND(1000));
v1::TaskGroupInfo taskGroup;
taskGroup.add_tasks()->CopyFrom(taskInfo);
v1::Offer::Operation::LaunchGroup* launchGroup =
operation->mutable_launch_group();
launchGroup->mutable_executor()->CopyFrom(executorInfo);
launchGroup->mutable_task_group()->CopyFrom(taskGroup);
mesos.send(call);
}
AWAIT_READY(executorLib);
Future<v1::executor::Event::Subscribed> executorSubscribed;
EXPECT_CALL(*executor, subscribed(_, _))
.WillOnce(FutureArg<1>(&executorSubscribed));
Future<Nothing> launchGroup;
EXPECT_CALL(*executor, launchGroup(_, _))
.WillOnce(FutureSatisfy(&launchGroup));
{
v1::executor::Call call;
call.mutable_framework_id()->CopyFrom(frameworkId);
call.mutable_executor_id()->CopyFrom(v1::DEFAULT_EXECUTOR_ID);
call.set_type(v1::executor::Call::SUBSCRIBE);
call.mutable_subscribe();
executorLib.get()->send(call);
}
// Wait for the executor to subscribe. Once it is in the SUBSCRIBED state,
// the UPDATE and MESSAGE executor calls can be attempted.
AWAIT_READY(executorSubscribed);
AWAIT_READY(launchGroup);
// Create a principal which contains an incorrect ContainerID.
hashmap<string, string> claims;
claims["fid"] = frameworkId.value();
claims["eid"] = v1::DEFAULT_EXECUTOR_ID.value();
claims["cid"] = id::UUID::random().toString();
Principal incorrectPrincipal(None(), claims);
// Generate an authentication token which is signed using the correct key,
// but contains an invalid set of claims.
Owned<JWTSecretGenerator> jwtSecretGenerator(
new JWTSecretGenerator(DEFAULT_JWT_SECRET_KEY));
Future<Secret> authenticationToken =
jwtSecretGenerator->generate(incorrectPrincipal);
AWAIT_READY(authenticationToken);
v1::ContainerID containerId;
containerId.set_value(id::UUID::random().toString());
containerId.mutable_parent()->CopyFrom(executorSubscribed->container_id());
http::Headers headers;
headers["Authorization"] = "Bearer " + authenticationToken->value().data();
// Since the executor library has already been initialized with a valid
// authentication token, we use an HTTP helper function to send the
// executor API and operator API calls with an invalid token.
{
v1::agent::Call call;
call.set_type(v1::agent::Call::LAUNCH_NESTED_CONTAINER);
call.mutable_launch_nested_container()->mutable_container_id()
->CopyFrom(containerId);
Future<http::Response> response = http::post(
slave.get()->pid,
"api/v1",
headers,
serialize(ContentType::PROTOBUF, call),
stringify(ContentType::PROTOBUF));
AWAIT_EXPECT_RESPONSE_STATUS_EQ(http::Forbidden().status, response);
}
{
v1::agent::Call call;
call.set_type(v1::agent::Call::LAUNCH_NESTED_CONTAINER_SESSION);
call.mutable_launch_nested_container_session()->mutable_container_id()
->CopyFrom(containerId);
call.mutable_launch_nested_container_session()->mutable_command()
->set_value("sleep 120");
Future<http::Response> response = http::post(
slave.get()->pid,
"api/v1",
headers,
serialize(ContentType::PROTOBUF, call),
stringify(ContentType::PROTOBUF));
AWAIT_EXPECT_RESPONSE_STATUS_EQ(http::Forbidden().status, response);
}
{
v1::agent::Call call;
call.set_type(v1::agent::Call::WAIT_NESTED_CONTAINER);
call.mutable_wait_nested_container()->mutable_container_id()
->CopyFrom(containerId);
Future<http::Response> response = http::post(
slave.get()->pid,
"api/v1",
headers,
serialize(ContentType::PROTOBUF, call),
stringify(ContentType::PROTOBUF));
AWAIT_EXPECT_RESPONSE_STATUS_EQ(http::Forbidden().status, response);
}
{
v1::agent::Call call;
call.set_type(v1::agent::Call::KILL_NESTED_CONTAINER);
call.mutable_kill_nested_container()->mutable_container_id()
->CopyFrom(containerId);
Future<http::Response> response = http::post(
slave.get()->pid,
"api/v1",
headers,
serialize(ContentType::PROTOBUF, call),
stringify(ContentType::PROTOBUF));
AWAIT_EXPECT_RESPONSE_STATUS_EQ(http::Forbidden().status, response);
}
{
v1::agent::Call call;
call.set_type(v1::agent::Call::REMOVE_NESTED_CONTAINER);
call.mutable_remove_nested_container()->mutable_container_id()
->CopyFrom(containerId);
Future<http::Response> response = http::post(
slave.get()->pid,
"api/v1",
headers,
serialize(ContentType::PROTOBUF, call),
stringify(ContentType::PROTOBUF));
AWAIT_EXPECT_RESPONSE_STATUS_EQ(http::Forbidden().status, response);
}
{
v1::agent::Call call;
call.set_type(v1::agent::Call::ATTACH_CONTAINER_OUTPUT);
call.mutable_attach_container_output()->mutable_container_id()
->CopyFrom(containerId);
Future<http::Response> response = http::post(
slave.get()->pid,
"api/v1",
headers,
serialize(ContentType::PROTOBUF, call),
stringify(ContentType::PROTOBUF));
AWAIT_EXPECT_RESPONSE_STATUS_EQ(http::Forbidden().status, response);
}
const string failureMessage =
"does not contain a 'cid' claim with the correct active ContainerID";
{
v1::TaskStatus status;
status.mutable_task_id()->set_value(id::UUID::random().toString());
status.set_state(v1::TASK_RUNNING);
status.set_uuid(id::UUID::random().toBytes());
status.set_source(v1::TaskStatus::SOURCE_EXECUTOR);
v1::executor::Call call;
call.set_type(v1::executor::Call::UPDATE);
call.mutable_framework_id()->CopyFrom(frameworkId);
call.mutable_executor_id()->CopyFrom(v1::DEFAULT_EXECUTOR_ID);
call.mutable_update()->mutable_status()->CopyFrom(status);
Future<http::Response> response = http::post(
slave.get()->pid,
"api/v1/executor",
headers,
serialize(ContentType::PROTOBUF, call),
stringify(ContentType::PROTOBUF));
AWAIT_EXPECT_RESPONSE_STATUS_EQ(http::Forbidden().status, response);
EXPECT_TRUE(strings::contains(response->body, failureMessage));
}
{
v1::executor::Call call;
call.set_type(v1::executor::Call::MESSAGE);
call.mutable_framework_id()->CopyFrom(frameworkId);
call.mutable_executor_id()->CopyFrom(v1::DEFAULT_EXECUTOR_ID);
call.mutable_message()->set_data("executor message");
Future<http::Response> response = http::post(
slave.get()->pid,
"api/v1/executor",
headers,
serialize(ContentType::PROTOBUF, call),
stringify(ContentType::PROTOBUF));
AWAIT_EXPECT_RESPONSE_STATUS_EQ(http::Forbidden().status, response);
EXPECT_TRUE(strings::contains(response->body, failureMessage));
}
EXPECT_CALL(*executor, shutdown(_))
.Times(AtMost(1));
}
// This test verifies that the environment variables for sandbox
// (i.e., MESOS_SANDBOX) is set properly.
TEST_F(LinuxFilesystemIsolatorMesosTest, ROOT_SandboxEnvironmentVariable)
{
Try<Owned<cluster::Master>> master = StartMaster();
ASSERT_SOME(master);
string registry = path::join(sandbox.get(), "registry");
AWAIT_READY(DockerArchive::create(registry, "test_image"));
slave::Flags flags = CreateSlaveFlags();
flags.isolation = "filesystem/linux,docker/runtime";
flags.docker_registry = registry;
flags.docker_store_dir = path::join(sandbox.get(), "store");
flags.image_providers = "docker";
Owned<MasterDetector> detector = master.get()->createDetector();
Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), flags);
ASSERT_SOME(slave);
MockScheduler sched;
MesosSchedulerDriver driver(
&sched,
DEFAULT_FRAMEWORK_INFO,
master.get()->pid,
DEFAULT_CREDENTIAL);
EXPECT_CALL(sched, registered(&driver, _, _));
Future<vector<Offer>> offers;
EXPECT_CALL(sched, resourceOffers(&driver, _))
.WillOnce(FutureArg<1>(&offers))
.WillRepeatedly(Return()); // Ignore subsequent offers.
driver.start();
AWAIT_READY(offers);
ASSERT_FALSE(offers->empty());
const Offer& offer = offers.get()[0];
TaskInfo task = createTask(
offer.slave_id(),
offer.resources(),
strings::format(
"if [ \"$MESOS_SANDBOX\" != \"%s\" ]; then exit 1; fi &&"
"if [ ! -d \"$MESOS_SANDBOX\" ]; then exit 1; fi",
flags.sandbox_directory).get());
task.mutable_container()->CopyFrom(createContainerInfo("test_image"));
driver.launchTasks(offer.id(), {task});
Future<TaskStatus> statusStarting;
Future<TaskStatus> statusRunning;
Future<TaskStatus> statusFinished;
EXPECT_CALL(sched, statusUpdate(&driver, _))
.WillOnce(FutureArg<1>(&statusStarting))
.WillOnce(FutureArg<1>(&statusRunning))
.WillOnce(FutureArg<1>(&statusFinished));
AWAIT_READY(statusStarting);
EXPECT_EQ(TASK_STARTING, statusStarting->state());
AWAIT_READY(statusRunning);
EXPECT_EQ(TASK_RUNNING, statusRunning->state());
AWAIT_READY(statusFinished);
EXPECT_EQ(TASK_FINISHED, statusFinished->state());
driver.stop();
driver.join();
}
// Tests that the task fails when it attempts to write to a persistent volume
// mounted as read-only. Note that although we use a shared persistent volume,
// the behavior is the same for non-shared persistent volumes.
TEST_F(LinuxFilesystemIsolatorMesosTest,
ROOT_WriteAccessSharedPersistentVolumeReadOnlyMode)
{
Try<Owned<cluster::Master>> master = StartMaster();
ASSERT_SOME(master);
string registry = path::join(sandbox.get(), "registry");
AWAIT_READY(DockerArchive::create(registry, "test_image"));
slave::Flags flags = CreateSlaveFlags();
flags.resources = "cpus:2;mem:128;disk(role1):128";
flags.isolation = "filesystem/linux,docker/runtime";
flags.docker_registry = registry;
flags.docker_store_dir = path::join(sandbox.get(), "store");
flags.image_providers = "docker";
Owned<MasterDetector> detector = master.get()->createDetector();
Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), flags);
ASSERT_SOME(slave);
MockScheduler sched;
FrameworkInfo frameworkInfo = DEFAULT_FRAMEWORK_INFO;
frameworkInfo.set_roles(0, "role1");
frameworkInfo.add_capabilities()->set_type(
FrameworkInfo::Capability::SHARED_RESOURCES);
MesosSchedulerDriver driver(
&sched,
frameworkInfo,
master.get()->pid,
DEFAULT_CREDENTIAL);
EXPECT_CALL(sched, registered(&driver, _, _));
Future<vector<Offer>> offers;
EXPECT_CALL(sched, resourceOffers(&driver, _))
.WillOnce(FutureArg<1>(&offers))
.WillRepeatedly(Return()); // Ignore subsequent offers.
driver.start();
AWAIT_READY(offers);
ASSERT_FALSE(offers->empty());
// We create a shared volume which shall be used by the task to
// write to that volume.
Resource volume = createPersistentVolume(
Megabytes(4),
"role1",
"id1",
"volume_path",
None(),
None(),
frameworkInfo.principal(),
true); // Shared volume.
// The task uses the shared volume as read-only.
Resource roVolume = volume;
roVolume.mutable_disk()->mutable_volume()->set_mode(Volume::RO);
Resources taskResources =
Resources::parse("cpus:1;mem:64;disk(role1):1").get() + roVolume;
TaskInfo task = createTask(
offers.get()[0].slave_id(),
taskResources,
"echo hello > volume_path/file");
// The task fails to write to the volume since the task's resources
// intends to use the volume as read-only.
Future<TaskStatus> statusStarting;
Future<TaskStatus> statusRunning;
Future<TaskStatus> statusFailed;
EXPECT_CALL(sched, statusUpdate(&driver, _))
.WillOnce(FutureArg<1>(&statusStarting))
.WillOnce(FutureArg<1>(&statusRunning))
.WillOnce(FutureArg<1>(&statusFailed));
driver.acceptOffers(
{offers.get()[0].id()},
{CREATE(volume),
LAUNCH({task})});
AWAIT_READY(statusStarting);
EXPECT_EQ(task.task_id(), statusStarting->task_id());
EXPECT_EQ(TASK_STARTING, statusStarting->state());
AWAIT_READY(statusRunning);
EXPECT_EQ(task.task_id(), statusRunning->task_id());
EXPECT_EQ(TASK_RUNNING, statusRunning->state());
AWAIT_READY(statusFailed);
EXPECT_EQ(task.task_id(), statusFailed->task_id());
EXPECT_EQ(TASK_FAILED, statusFailed->state());
driver.stop();
driver.join();
}
Future<ExecutorInfo> ExternalContainerizerProcess::_launch(
const ContainerID& containerId,
const FrameworkID& frameworkId,
const ExecutorInfo executorInfo,
const SlaveID& slaveId,
bool checkpoint,
const Future<ResultFutures>& future)
{
VLOG(1) << "Launch callback triggered on container '" << containerId << "'";
if (!containers.contains(containerId)) {
return Failure("Container '" + containerId.value() + "' not running");
}
string result;
Try<bool> support = commandSupported(future, result);
if (support.isError()) {
terminate(containerId);
return Failure(support.error());
}
if (!support.get()) {
// We generally need to use an internal implementation in these
// cases.
// For the specific case of a launch however, there can not be an
// internal implementation for a external containerizer, hence
// we need to fail or even abort at this point.
// TODO(tillt): Consider using posix-isolator as a fall back.
terminate(containerId);
return Failure("External containerizer does not support launch");
}
VLOG(1) << "Launch supported by external containerizer";
ExternalStatus ps;
if (!ps.ParseFromString(result)) {
// TODO(tillt): Consider not terminating the containerizer due
// to protocol breach but only fail the operation.
terminate(containerId);
return Failure("Could not parse launch result protobuf (error: "
+ protobufError(ps) + ")");
}
VLOG(2) << "Launch result: '" << ps.message() << "'";
VLOG(2) << "Executor pid: " << ps.pid();
containers[containerId]->pid = ps.pid();
// Observe the executor process and install a callback for status
// changes.
process::reap(ps.pid())
.onAny(defer(
PID<ExternalContainerizerProcess>(this),
&ExternalContainerizerProcess::reaped,
containerId,
lambda::_1));
// Checkpoint the container's pid if requested.
if (checkpoint) {
const string& path = slave::paths::getForkedPidPath(
slave::paths::getMetaRootDir(flags.work_dir),
slaveId,
frameworkId,
executorInfo.executor_id(),
containerId);
LOG(INFO) << "Checkpointing containerized executor '" << containerId
<< "' pid " << ps.pid() << " to '" << path << "'";
Try<Nothing> checkpointed =
slave::state::checkpoint(path, stringify(ps.pid()));
if (checkpointed.isError()) {
terminate(containerId);
return Failure("Failed to checkpoint containerized executor '"
+ containerId.value() + "' pid " + stringify(ps.pid()) + " to '" + path
+ "'");
}
}
VLOG(1) << "Launch finishing up for container '" << containerId << "'";
return executorInfo;
}
TYPED_TEST(MemIsolatorTest, MemUsage)
{
Flags flags;
Try<Isolator*> isolator = TypeParam::create(flags);
CHECK_SOME(isolator);
// A PosixLauncher is sufficient even when testing a cgroups isolator.
Try<Launcher*> launcher = PosixLauncher::create(flags);
ExecutorInfo executorInfo;
executorInfo.mutable_resources()->CopyFrom(
Resources::parse("mem:1024").get());
ContainerID containerId;
containerId.set_value("memory_usage");
AWAIT_READY(isolator.get()->prepare(containerId, executorInfo));
int pipes[2];
ASSERT_NE(-1, ::pipe(pipes));
lambda::function<int()> inChild = lambda::bind(
&consumeMemory,
Megabytes(256),
Seconds(10),
pipes);
Try<pid_t> pid = launcher.get()->fork(containerId, inChild);
ASSERT_SOME(pid);
// Set up the reaper to wait on the forked child.
Future<Option<int> > status = process::reap(pid.get());
// Continue in the parent.
::close(pipes[0]);
// Isolate the forked child.
AWAIT_READY(isolator.get()->isolate(containerId, pid.get()));
// Now signal the child to continue.
int buf;
ASSERT_LT(0, ::write(pipes[1], &buf, sizeof(buf)));
::close(pipes[1]);
// Wait up to 5 seconds for the child process to consume 256 MB of memory;
ResourceStatistics statistics;
Bytes threshold = Megabytes(256);
Duration waited = Duration::zero();
do {
Future<ResourceStatistics> usage = isolator.get()->usage(containerId);
AWAIT_READY(usage);
statistics = usage.get();
// If we meet our usage expectations, we're done!
if (statistics.mem_rss_bytes() >= threshold.bytes()) {
break;
}
os::sleep(Seconds(1));
waited += Seconds(1);
} while (waited < Seconds(5));
EXPECT_LE(threshold.bytes(), statistics.mem_rss_bytes());
// Ensure all processes are killed.
AWAIT_READY(launcher.get()->destroy(containerId));
// Make sure the child was reaped.
AWAIT_READY(status);
// Let the isolator clean up.
AWAIT_READY(isolator.get()->cleanup(containerId));
delete isolator.get();
delete launcher.get();
}
TEST(Metrics, Snapshot)
{
ASSERT_TRUE(GTEST_IS_THREADSAFE);
UPID upid("metrics", process::address());
Clock::pause();
// Add a gauge and a counter.
GaugeProcess process;
PID<GaugeProcess> pid = spawn(&process);
ASSERT_TRUE(pid);
Gauge gauge("test/gauge", defer(pid, &GaugeProcess::get));
Gauge gaugeFail("test/gauge_fail", defer(pid, &GaugeProcess::fail));
Counter counter("test/counter");
AWAIT_READY(metrics::add(gauge));
AWAIT_READY(metrics::add(gaugeFail));
AWAIT_READY(metrics::add(counter));
// Advance the clock to avoid rate limit.
Clock::advance(Seconds(1));
// Get the snapshot.
Future<Response> response = http::get(upid, "snapshot");
AWAIT_EXPECT_RESPONSE_STATUS_EQ(OK().status, response);
// Parse the response.
Try<JSON::Object> responseJSON =
JSON::parse<JSON::Object>(response.get().body);
ASSERT_SOME(responseJSON);
map<string, JSON::Value> values = responseJSON.get().values;
EXPECT_EQ(1u, values.count("test/counter"));
EXPECT_FLOAT_EQ(0.0, values["test/counter"].as<JSON::Number>().value);
EXPECT_EQ(1u, values.count("test/gauge"));
EXPECT_FLOAT_EQ(42.0, values["test/gauge"].as<JSON::Number>().value);
EXPECT_EQ(0u, values.count("test/gauge_fail"));
// Remove the metrics and ensure they are no longer in the snapshot.
AWAIT_READY(metrics::remove(gauge));
AWAIT_READY(metrics::remove(gaugeFail));
AWAIT_READY(metrics::remove(counter));
// Advance the clock to avoid rate limit.
Clock::advance(Seconds(1));
// Ensure MetricsProcess has removed the metrics.
Clock::settle();
response = http::get(upid, "snapshot");
AWAIT_EXPECT_RESPONSE_STATUS_EQ(OK().status, response);
// Parse the response.
responseJSON = JSON::parse<JSON::Object>(response.get().body);
ASSERT_SOME(responseJSON);
values = responseJSON.get().values;
EXPECT_EQ(0u, values.count("test/counter"));
EXPECT_EQ(0u, values.count("test/gauge"));
EXPECT_EQ(0u, values.count("test/gauge_fail"));
terminate(process);
wait(process);
}
// Testing route with authorization header and good credentials.
TEST_F(TeardownTest, Success)
{
Try<Owned<cluster::Master>> master = StartMaster();
ASSERT_SOME(master);
MockScheduler sched;
MesosSchedulerDriver driver(
&sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL);
Future<FrameworkID> frameworkId;
EXPECT_CALL(sched, registered(&driver, _, _))
.WillOnce(FutureArg<1>(&frameworkId));
ASSERT_EQ(DRIVER_RUNNING, driver.start());
AWAIT_READY(frameworkId);
{
Future<Response> response = process::http::post(
master.get()->pid,
"teardown",
createBasicAuthHeaders(DEFAULT_CREDENTIAL),
"frameworkId=" + frameworkId.get().value());
AWAIT_READY(response);
AWAIT_EXPECT_RESPONSE_STATUS_EQ(OK().status, response);
}
// Check that the framework that was shutdown appears in the
// "completed_frameworks" list in the master's "/state" endpoint.
{
Future<Response> response = process::http::get(
master.get()->pid,
"state",
None(),
createBasicAuthHeaders(DEFAULT_CREDENTIAL));
AWAIT_EXPECT_RESPONSE_STATUS_EQ(OK().status, response);
AWAIT_EXPECT_RESPONSE_HEADER_EQ(APPLICATION_JSON, "Content-Type", response);
Try<JSON::Object> parse = JSON::parse<JSON::Object>(response.get().body);
ASSERT_SOME(parse);
JSON::Array frameworks = parse->values["frameworks"].as<JSON::Array>();
EXPECT_TRUE(frameworks.values.empty());
JSON::Array completedFrameworks =
parse->values["completed_frameworks"].as<JSON::Array>();
ASSERT_EQ(1u, completedFrameworks.values.size());
JSON::Object completedFramework =
completedFrameworks.values.front().as<JSON::Object>();
JSON::String completedFrameworkId =
completedFramework.values["id"].as<JSON::String>();
EXPECT_EQ(frameworkId.get(), completedFrameworkId.value);
}
driver.stop();
driver.join();
}
Try<RunState> RunState::recover(
const string& rootDir,
const SlaveID& slaveId,
const FrameworkID& frameworkId,
const ExecutorID& executorId,
const UUID& uuid,
bool strict)
{
RunState state;
state.id = uuid;
string message;
// Find the tasks.
const Try<list<string> >& tasks = os::glob(strings::format(
paths::TASK_PATH,
rootDir,
slaveId,
frameworkId,
executorId,
uuid.toString(),
"*").get());
if (tasks.isError()) {
return Error("Failed to find tasks for executor run " + uuid.toString() +
": " + tasks.error());
}
// Recover tasks.
foreach (const string& path, tasks.get()) {
TaskID taskId;
taskId.set_value(os::basename(path).get());
const Try<TaskState>& task = TaskState::recover(
rootDir, slaveId, frameworkId, executorId, uuid, taskId, strict);
if (task.isError()) {
return Error(
"Failed to recover task " + taskId.value() + ": " + task.error());
}
state.tasks[taskId] = task.get();
}
// Read the forked pid.
string path = paths::getForkedPidPath(
rootDir, slaveId, frameworkId, executorId, uuid);
Try<string> pid = os::read(path);
if (pid.isError()) {
message = "Failed to read executor's forked pid from '" + path +
"': " + pid.error();
if (strict) {
return Error(message);
} else {
LOG(WARNING) << message;
return state;
}
}
Try<pid_t> forkedPid = numify<pid_t>(pid.get());
if (forkedPid.isError()) {
return Error("Failed to parse forked pid " + pid.get() +
": " + forkedPid.error());
}
state.forkedPid = forkedPid.get();
// Read the libprocess pid.
path = paths::getLibprocessPidPath(
rootDir, slaveId, frameworkId, executorId, uuid);
pid = os::read(path);
if (pid.isError()) {
message = "Failed to read executor's libprocess pid from '" + path +
"': " + pid.error();
if (strict) {
return Error(message);
} else {
LOG(WARNING) << message;
return state;
}
}
state.libprocessPid = process::UPID(pid.get());
return state;
}
Try<Manifest> Manifest::create(const string& jsonString)
{
Try<JSON::Object> manifestJSON = JSON::parse<JSON::Object>(jsonString);
if (manifestJSON.isError()) {
return Error(manifestJSON.error());
}
Result<JSON::String> name = manifestJSON.get().find<JSON::String>("name");
if (name.isNone()) {
return Error("Failed to find \"name\" in manifest response");
}
Result<JSON::Array> fsLayersJSON =
manifestJSON.get().find<JSON::Array>("fsLayers");
if (fsLayersJSON.isNone()) {
return Error("Failed to find \"fsLayers\" in manifest response");
}
Result<JSON::Array> historyArray =
manifestJSON.get().find<JSON::Array>("history");
if (historyArray.isNone()) {
return Error("Failed to find \"history\" in manifest response");
}
if (historyArray.get().values.size() != fsLayersJSON.get().values.size()) {
return Error(
"\"history\" and \"fsLayers\" array count mismatch"
"in manifest response");
}
vector<FileSystemLayerInfo> fsLayers;
// We add layers in reverse order because 'fsLayers' in the manifest
// response is ordered with the latest layer on the top. When we apply the
// layer changes, we want the filesystem modification order to be the same
// as its history(oldest layer applied first).
for (size_t index = fsLayersJSON.get().values.size(); index-- > 0; ) {
const JSON::Value& layer = fsLayersJSON.get().values[index];
if (!layer.is<JSON::Object>()) {
return Error(
"Failed to parse layer as a JSON object for index: " +
stringify(index));
}
const JSON::Object& layerInfoJSON = layer.as<JSON::Object>();
// Get blobsum for layer.
const Result<JSON::String> blobSumInfo =
layerInfoJSON.find<JSON::String>("blobSum");
if (blobSumInfo.isNone()) {
return Error("Failed to find \"blobSum\" in manifest response");
}
// Get history for layer.
if (!historyArray.get().values[index].is<JSON::Object>()) {
return Error(
"Failed to parse history as a JSON object for index: " +
stringify(index));
}
const JSON::Object& historyObj =
historyArray.get().values[index].as<JSON::Object>();
// Get layer id.
const Result<JSON::String> v1CompatibilityJSON =
historyObj.find<JSON::String>("v1Compatibility");
if (!v1CompatibilityJSON.isSome()) {
return Error(
"Failed to obtain layer v1 compability json in manifest for layer: "
+ stringify(index));
}
Try<JSON::Object> v1CompatibilityObj =
JSON::parse<JSON::Object>(v1CompatibilityJSON.get().value);
if (!v1CompatibilityObj.isSome()) {
return Error(
"Failed to parse v1 compability json in manifest for layer: "
+ stringify(index));
}
const Result<JSON::String> id =
v1CompatibilityObj.get().find<JSON::String>("id");
if (!id.isSome()) {
return Error(
"Failed to find \"id\" in manifest for layer: " + stringify(index));
}
fsLayers.emplace_back(
FileSystemLayerInfo{
blobSumInfo.get().value,
id.get().value,
});
}
return Manifest{name.get().value, fsLayers};
}
TYPED_TEST(IsolatorTest, Usage)
{
Try<PID<Master> > master = this->StartMaster();
ASSERT_SOME(master);
TypeParam isolator;
slave::Flags flags = this->CreateSlaveFlags();
Try<PID<Slave> > slave = this->StartSlave(&isolator, flags);
ASSERT_SOME(slave);
MockScheduler sched;
MesosSchedulerDriver driver(
&sched, DEFAULT_FRAMEWORK_INFO, master.get(), DEFAULT_CREDENTIAL);
Future<FrameworkID> frameworkId;
EXPECT_CALL(sched, registered(&driver, _, _))
.WillOnce(FutureArg<1>(&frameworkId));
Future<vector<Offer> > offers;
EXPECT_CALL(sched, resourceOffers(&driver, _))
.WillOnce(FutureArg<1>(&offers))
.WillRepeatedly(Return()); // Ignore subsequent offers.
driver.start();
AWAIT_READY(frameworkId);
AWAIT_READY(offers);
EXPECT_NE(0u, offers.get().size());
TaskInfo task;
task.set_name("isolator_test");
task.mutable_task_id()->set_value("1");
task.mutable_slave_id()->MergeFrom(offers.get()[0].slave_id());
task.mutable_resources()->MergeFrom(offers.get()[0].resources());
Resources resources(offers.get()[0].resources());
Option<Bytes> mem = resources.mem();
ASSERT_SOME(mem);
Option<double> cpus = resources.cpus();
ASSERT_SOME(cpus);
const std::string& file = path::join(flags.work_dir, "ready");
// This task induces user/system load in a child process by
// running top in a child process for ten seconds.
task.mutable_command()->set_value(
#ifdef __APPLE__
// Use logging mode with 30,000 samples with no interval.
"top -l 30000 -s 0 2>&1 > /dev/null & "
#else
// Batch mode, with 30,000 samples with no interval.
"top -b -d 0 -n 30000 2>&1 > /dev/null & "
#endif
"touch " + file + "; " // Signals that the top command is running.
"sleep 60");
vector<TaskInfo> tasks;
tasks.push_back(task);
Future<TaskStatus> status;
EXPECT_CALL(sched, statusUpdate(&driver, _))
.WillOnce(FutureArg<1>(&status));
driver.launchTasks(offers.get()[0].id(), tasks);
AWAIT_READY(status);
EXPECT_EQ(TASK_RUNNING, status.get().state());
// Wait for the task to begin inducing cpu time.
while (!os::exists(file));
ExecutorID executorId;
executorId.set_value(task.task_id().value());
// We'll wait up to 10 seconds for the child process to induce
// 1/8 of a second of user and system cpu time in total.
// TODO(bmahler): Also induce rss memory consumption, by re-using
// the balloon framework.
ResourceStatistics statistics;
Duration waited = Duration::zero();
do {
Future<ResourceStatistics> usage =
process::dispatch(
(Isolator*) &isolator, // TODO(benh): Fix after reaper changes.
&Isolator::usage,
frameworkId.get(),
executorId);
AWAIT_READY(usage);
statistics = usage.get();
// If we meet our usage expectations, we're done!
if (statistics.cpus_user_time_secs() >= 0.125 &&
statistics.cpus_system_time_secs() >= 0.125 &&
statistics.mem_rss_bytes() >= 1024u) {
break;
}
os::sleep(Milliseconds(100));
waited += Milliseconds(100);
} while (waited < Seconds(10));
EXPECT_GE(statistics.cpus_user_time_secs(), 0.125);
EXPECT_GE(statistics.cpus_system_time_secs(), 0.125);
EXPECT_EQ(statistics.cpus_limit(), cpus.get());
EXPECT_GE(statistics.mem_rss_bytes(), 1024u);
EXPECT_EQ(statistics.mem_limit_bytes(), mem.get().bytes());
EXPECT_CALL(sched, statusUpdate(&driver, _))
.WillOnce(FutureArg<1>(&status));
driver.killTask(task.task_id());
AWAIT_READY(status);
EXPECT_EQ(TASK_KILLED, status.get().state());
driver.stop();
driver.join();
this->Shutdown(); // Must shutdown before 'isolator' gets deallocated.
}
TEST(IOTest, Redirect)
{
ASSERT_TRUE(GTEST_IS_THREADSAFE);
// Start by checking that using "invalid" file descriptors fails.
AWAIT_EXPECT_FAILED(io::redirect(-1, 0));
AWAIT_EXPECT_FAILED(io::redirect(0, -1));
// Create a temporary file for redirecting into.
Try<string> path = os::mktemp();
ASSERT_SOME(path);
Try<int> fd = os::open(
path.get(),
O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC,
S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
ASSERT_SOME(fd);
ASSERT_SOME(os::nonblock(fd.get()));
// Use a nonblocking pipe for doing the redirection.
int pipes[2];
ASSERT_NE(-1, ::pipe(pipes));
ASSERT_SOME(os::nonblock(pipes[0]));
ASSERT_SOME(os::nonblock(pipes[1]));
// Now write data to the pipe and splice to the file.
string data =
"Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do "
"eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim "
"ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut "
"aliquip ex ea commodo consequat. Duis aute irure dolor in "
"reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla "
"pariatur. Excepteur sint occaecat cupidatat non proident, sunt in "
"culpa qui officia deserunt mollit anim id est laborum.";
// Create more data!
while (Bytes(data.size()) < Megabytes(1)) {
data.append(data);
}
Future<Nothing> redirect = io::redirect(pipes[0], fd.get());
// Closing the read end of the pipe and the file should not have any
// impact as we dup the file descriptor.
ASSERT_SOME(os::close(pipes[0]));
ASSERT_SOME(os::close(fd.get()));
EXPECT_TRUE(redirect.isPending());
// Writing the data should keep the future pending as it hasn't seen
// EOF yet.
AWAIT_READY(io::write(pipes[1], data));
EXPECT_TRUE(redirect.isPending());
// Now closing the write pipe should cause an EOF on the read end,
// thus completing underlying splice in io::redirect.
ASSERT_SOME(os::close(pipes[1]));
AWAIT_READY(redirect);
// Now make sure all the data is there!
Try<string> read = os::read(path.get());
ASSERT_SOME(read);
EXPECT_EQ(data, read.get());
}
void FlagsBase::add(
Option<T> Flags::*option,
const std::string& name,
const std::string& help,
F validate)
{
// Don't bother adding anything if the pointer is NULL.
if (option == NULL) {
return;
}
Flags* flags = dynamic_cast<Flags*>(this);
if (flags == NULL) {
ABORT("Attempted to add flag '" + name + "' with incompatible type");
}
Flag flag;
flag.name = name;
flag.help = help;
flag.boolean = typeid(T) == typeid(bool);
// NOTE: See comment above in Flags::T* overload of FLagsBase::add
// for why we need to pass FlagsBase* (or const FlagsBase&) as a
// parameter.
flag.load =
[option](FlagsBase* base, const std::string& value) -> Try<Nothing> {
Flags* flags = dynamic_cast<Flags*>(base);
if (flags != NULL) {
// NOTE: 'fetch' "retrieves" the value if necessary and then
// invokes 'parse'. See 'fetch' for more details.
Try<T> t = fetch<T>(value);
if (t.isSome()) {
flags->*option = Some(t.get());
} else {
return Error("Failed to load value '" + value + "': " + t.error());
}
}
return Nothing();
};
flag.stringify = [option](const FlagsBase& base) -> Option<std::string> {
const Flags* flags = dynamic_cast<const Flags*>(&base);
if (flags != NULL) {
if ((flags->*option).isSome()) {
return stringify((flags->*option).get());
}
}
return None();
};
flag.validate = [option, validate](const FlagsBase& base) -> Option<Error> {
const Flags* flags = dynamic_cast<const Flags*>(&base);
if (flags != NULL) {
return validate(flags->*option);
}
return None();
};
add(flag);
}
void FlagsBase::add(
T1 Flags::*t1,
const std::string& name,
const std::string& help,
const T2& t2,
F validate)
{
// Don't bother adding anything if the pointer is NULL.
if (t1 == NULL) {
return;
}
Flags* flags = dynamic_cast<Flags*>(this);
if (flags == NULL) {
ABORT("Attempted to add flag '" + name + "' with incompatible type");
} else {
flags->*t1 = t2; // Set the default.
}
Flag flag;
flag.name = name;
flag.help = help;
flag.boolean = typeid(T1) == typeid(bool);
// NOTE: We need to take FlagsBase* (or const FlagsBase&) as the
// first argument to 'load', 'stringify', and 'validate' so that we
// use the correct instance of FlagsBase. In other words, we can't
// capture 'this' here because it's possible that the FlagsBase
// object that we're working with when we invoke FlagsBase::add is
// not the same instance as 'this' when the these lambdas get
// invoked.
flag.load = [t1](FlagsBase* base, const std::string& value) -> Try<Nothing> {
Flags* flags = dynamic_cast<Flags*>(base);
if (base != NULL) {
// NOTE: 'fetch' "retrieves" the value if necessary and then
// invokes 'parse'. See 'fetch' for more details.
Try<T1> t = fetch<T1>(value);
if (t.isSome()) {
flags->*t1 = t.get();
} else {
return Error("Failed to load value '" + value + "': " + t.error());
}
}
return Nothing();
};
flag.stringify = [t1](const FlagsBase& base) -> Option<std::string> {
const Flags* flags = dynamic_cast<const Flags*>(&base);
if (flags != NULL) {
return stringify(flags->*t1);
}
return None();
};
flag.validate = [t1, validate](const FlagsBase& base) -> Option<Error> {
const Flags* flags = dynamic_cast<const Flags*>(&base);
if (flags != NULL) {
return validate(flags->*t1);
}
return None();
};
// Update the help string to include the default value.
flag.help += help.size() > 0 && help.find_last_of("\n\r") != help.size() - 1
? " (default: " // On same line, add space.
: "(default: "; // On newline.
flag.help += stringify(t2);
flag.help += ")";
add(flag);
}
int main(int argc, char** argv)
{
GOOGLE_PROTOBUF_VERIFY_VERSION;
master::Flags flags;
// The following flags are executable specific (e.g., since we only
// have one instance of libprocess per execution, we only want to
// advertise the IP and port option once, here).
Option<string> ip;
flags.add(&ip,
"ip",
"IP address to listen on. This cannot be used in conjunction\n"
"with `--ip_discovery_command`.");
uint16_t port;
flags.add(&port,
"port",
"Port to listen on.",
MasterInfo().port());
Option<string> advertise_ip;
flags.add(&advertise_ip,
"advertise_ip",
"IP address advertised to reach this Mesos master.\n"
"The master does not bind using this IP address.\n"
"However, this IP address may be used to access this master.");
Option<string> advertise_port;
flags.add(&advertise_port,
"advertise_port",
"Port advertised to reach Mesos master (along with\n"
"`advertise_ip`). The master does not bind to this port.\n"
"However, this port (along with `advertise_ip`) may be used to\n"
"access this master.");
Option<string> zk;
flags.add(&zk,
"zk",
"ZooKeeper URL (used for leader election amongst masters)\n"
"May be one of:\n"
" `zk://host1:port1,host2:port2,.../path`\n"
" `zk://username:password@host1:port1,host2:port2,.../path`\n"
" `file:///path/to/file` (where file contains one of the above)\n"
"NOTE: Not required if master is run in standalone mode (non-HA).");
// Optional IP discover script that will set the Master IP.
// If set, its output is expected to be a valid parseable IP string.
Option<string> ip_discovery_command;
flags.add(&ip_discovery_command,
"ip_discovery_command",
"Optional IP discovery binary: if set, it is expected to emit\n"
"the IP address which the master will try to bind to.\n"
"Cannot be used in conjunction with `--ip`.");
Try<Nothing> load = flags.load("MESOS_", argc, argv);
if (load.isError()) {
cerr << flags.usage(load.error()) << endl;
return EXIT_FAILURE;
}
if (flags.version) {
version();
return EXIT_SUCCESS;
}
if (flags.help) {
cout << flags.usage() << endl;
return EXIT_SUCCESS;
}
// Initialize modules. Note that since other subsystems may depend
// upon modules, we should initialize modules before anything else.
if (flags.modules.isSome()) {
Try<Nothing> result = ModuleManager::load(flags.modules.get());
if (result.isError()) {
EXIT(EXIT_FAILURE) << "Error loading modules: " << result.error();
}
}
// Initialize hooks.
if (flags.hooks.isSome()) {
Try<Nothing> result = HookManager::initialize(flags.hooks.get());
if (result.isError()) {
EXIT(EXIT_FAILURE) << "Error installing hooks: " << result.error();
}
}
if (ip_discovery_command.isSome() && ip.isSome()) {
EXIT(EXIT_FAILURE) << flags.usage(
"Only one of `--ip` or `--ip_discovery_command` should be specified");
}
if (ip_discovery_command.isSome()) {
Try<string> ipAddress = os::shell(ip_discovery_command.get());
if (ipAddress.isError()) {
EXIT(EXIT_FAILURE) << ipAddress.error();
}
os::setenv("LIBPROCESS_IP", strings::trim(ipAddress.get()));
} else if (ip.isSome()) {
os::setenv("LIBPROCESS_IP", ip.get());
}
os::setenv("LIBPROCESS_PORT", stringify(port));
if (advertise_ip.isSome()) {
os::setenv("LIBPROCESS_ADVERTISE_IP", advertise_ip.get());
}
if (advertise_port.isSome()) {
os::setenv("LIBPROCESS_ADVERTISE_PORT", advertise_port.get());
}
// Initialize libprocess.
process::initialize("master");
logging::initialize(argv[0], flags, true); // Catch signals.
spawn(new VersionProcess(), true);
LOG(INFO) << "Build: " << build::DATE << " by " << build::USER;
LOG(INFO) << "Version: " << MESOS_VERSION;
if (build::GIT_TAG.isSome()) {
LOG(INFO) << "Git tag: " << build::GIT_TAG.get();
}
if (build::GIT_SHA.isSome()) {
LOG(INFO) << "Git SHA: " << build::GIT_SHA.get();
}
// Create an instance of allocator.
const string allocatorName = flags.allocator;
Try<Allocator*> allocator = Allocator::create(allocatorName);
if (allocator.isError()) {
EXIT(EXIT_FAILURE)
<< "Failed to create '" << allocatorName
<< "' allocator: " << allocator.error();
}
CHECK_NOTNULL(allocator.get());
LOG(INFO) << "Using '" << allocatorName << "' allocator";
state::Storage* storage = NULL;
Log* log = NULL;
if (flags.registry == "in_memory") {
if (flags.registry_strict) {
EXIT(EXIT_FAILURE)
<< "Cannot use '--registry_strict' when using in-memory storage"
<< " based registry";
}
storage = new state::InMemoryStorage();
} else if (flags.registry == "replicated_log" ||
flags.registry == "log_storage") {
// TODO(bmahler): "log_storage" is present for backwards
// compatibility, can be removed before 0.19.0.
if (flags.work_dir.isNone()) {
EXIT(EXIT_FAILURE)
<< "--work_dir needed for replicated log based registry";
}
Try<Nothing> mkdir = os::mkdir(flags.work_dir.get());
if (mkdir.isError()) {
EXIT(EXIT_FAILURE)
<< "Failed to create work directory '" << flags.work_dir.get()
<< "': " << mkdir.error();
}
if (zk.isSome()) {
// Use replicated log with ZooKeeper.
if (flags.quorum.isNone()) {
EXIT(EXIT_FAILURE)
<< "Need to specify --quorum for replicated log based"
<< " registry when using ZooKeeper";
}
Try<zookeeper::URL> url = zookeeper::URL::parse(zk.get());
if (url.isError()) {
EXIT(EXIT_FAILURE) << "Error parsing ZooKeeper URL: " << url.error();
}
log = new Log(
flags.quorum.get(),
path::join(flags.work_dir.get(), "replicated_log"),
url.get().servers,
flags.zk_session_timeout,
path::join(url.get().path, "log_replicas"),
url.get().authentication,
flags.log_auto_initialize);
} else {
// Use replicated log without ZooKeeper.
log = new Log(
1,
path::join(flags.work_dir.get(), "replicated_log"),
set<UPID>(),
flags.log_auto_initialize);
}
storage = new state::LogStorage(log);
} else {
EXIT(EXIT_FAILURE)
<< "'" << flags.registry << "' is not a supported"
<< " option for registry persistence";
}
CHECK_NOTNULL(storage);
state::protobuf::State* state = new state::protobuf::State(storage);
Registrar* registrar = new Registrar(flags, state);
Repairer* repairer = new Repairer();
Files files;
MasterContender* contender;
MasterDetector* detector;
Try<MasterContender*> contender_ = MasterContender::create(zk);
if (contender_.isError()) {
EXIT(EXIT_FAILURE)
<< "Failed to create a master contender: " << contender_.error();
}
contender = contender_.get();
Try<MasterDetector*> detector_ = MasterDetector::create(zk);
if (detector_.isError()) {
EXIT(EXIT_FAILURE)
<< "Failed to create a master detector: " << detector_.error();
}
detector = detector_.get();
Option<Authorizer*> authorizer_ = None();
auto authorizerNames = strings::split(flags.authorizers, ",");
if (authorizerNames.empty()) {
EXIT(EXIT_FAILURE) << "No authorizer specified";
}
if (authorizerNames.size() > 1) {
EXIT(EXIT_FAILURE) << "Multiple authorizers not supported";
}
string authorizerName = authorizerNames[0];
// NOTE: The flag --authorizers overrides the flag --acls, i.e. if
// a non default authorizer is requested, it will be used and
// the contents of --acls will be ignored.
// TODO(arojas): Consider adding support for multiple authorizers.
Result<Authorizer*> authorizer((None()));
if (authorizerName != master::DEFAULT_AUTHORIZER) {
LOG(INFO) << "Creating '" << authorizerName << "' authorizer";
authorizer = Authorizer::create(authorizerName);
} else {
// `authorizerName` is `DEFAULT_AUTHORIZER` at this point.
if (flags.acls.isSome()) {
LOG(INFO) << "Creating default '" << authorizerName << "' authorizer";
authorizer = Authorizer::create(flags.acls.get());
}
}
if (authorizer.isError()) {
EXIT(EXIT_FAILURE) << "Could not create '" << authorizerName
<< "' authorizer: " << authorizer.error();
} else if (authorizer.isSome()) {
authorizer_ = authorizer.get();
}
Option<shared_ptr<RateLimiter>> slaveRemovalLimiter = None();
if (flags.slave_removal_rate_limit.isSome()) {
// Parse the flag value.
// TODO(vinod): Move this parsing logic to flags once we have a
// 'Rate' abstraction in stout.
vector<string> tokens =
strings::tokenize(flags.slave_removal_rate_limit.get(), "/");
if (tokens.size() != 2) {
EXIT(EXIT_FAILURE)
<< "Invalid slave_removal_rate_limit: "
<< flags.slave_removal_rate_limit.get()
<< ". Format is <Number of slaves>/<Duration>";
}
Try<int> permits = numify<int>(tokens[0]);
if (permits.isError()) {
EXIT(EXIT_FAILURE)
<< "Invalid slave_removal_rate_limit: "
<< flags.slave_removal_rate_limit.get()
<< ". Format is <Number of slaves>/<Duration>"
<< ": " << permits.error();
}
Try<Duration> duration = Duration::parse(tokens[1]);
if (duration.isError()) {
EXIT(EXIT_FAILURE)
<< "Invalid slave_removal_rate_limit: "
<< flags.slave_removal_rate_limit.get()
<< ". Format is <Number of slaves>/<Duration>"
<< ": " << duration.error();
}
slaveRemovalLimiter = new RateLimiter(permits.get(), duration.get());
}
if (flags.firewall_rules.isSome()) {
vector<Owned<FirewallRule>> rules;
const Firewall firewall = flags.firewall_rules.get();
if (firewall.has_disabled_endpoints()) {
hashset<string> paths;
foreach (const string& path, firewall.disabled_endpoints().paths()) {
paths.insert(path);
}
rules.emplace_back(new DisabledEndpointsFirewallRule(paths));
}
// Tests whether a slave correctly detects the new master when its
// ZooKeeper session is expired and a new master is elected before the
// slave reconnects with ZooKeeper.
TEST_F(ZooKeeperMasterContenderDetectorTest,
MasterDetectorExpireSlaveZKSessionNewMaster)
{
Try<zookeeper::URL> url = zookeeper::URL::parse(
"zk://" + server->connectString() + "/mesos");
ASSERT_SOME(url);
// Simulate a leading master.
Owned<zookeeper::Group> leaderGroup(
new Group(url.get(), MASTER_CONTENDER_ZK_SESSION_TIMEOUT));
// 1. Simulate a leading contender.
ZooKeeperMasterContender leaderContender(leaderGroup);
ZooKeeperMasterDetector leaderDetector(leaderGroup);
PID<Master> leader;
leader.ip = 10000000;
leader.port = 10000;
leaderContender.initialize(leader);
Future<Future<Nothing> > contended = leaderContender.contend();
AWAIT_READY(contended);
Future<Option<UPID> > detected = leaderDetector.detect(None());
AWAIT_READY(detected);
EXPECT_SOME_EQ(leader, detected.get());
// 2. Simulate a non-leading contender.
Owned<zookeeper::Group> followerGroup(
new Group(url.get(), MASTER_CONTENDER_ZK_SESSION_TIMEOUT));
ZooKeeperMasterContender followerContender(followerGroup);
ZooKeeperMasterDetector followerDetector(followerGroup);
PID<Master> follower;
follower.ip = 10000001;
follower.port = 10001;
followerContender.initialize(follower);
contended = followerContender.contend();
AWAIT_READY(contended);
detected = followerDetector.detect(None());
EXPECT_SOME_EQ(leader, detected.get());
// 3. Simulate a non-contender.
Owned<zookeeper::Group> nonContenderGroup(
new Group(url.get(), MASTER_DETECTOR_ZK_SESSION_TIMEOUT));
ZooKeeperMasterDetector nonContenderDetector(nonContenderGroup);
detected = nonContenderDetector.detect();
EXPECT_SOME_EQ(leader, detected.get());
detected = nonContenderDetector.detect(leader);
// Now expire the slave's and leading master's zk sessions.
// NOTE: Here we assume that slave stays disconnected from the ZK
// when the leading master loses its session.
Future<Option<int64_t> > slaveSession = nonContenderGroup->session();
AWAIT_READY(slaveSession);
Future<Option<int64_t> > masterSession = leaderGroup->session();
AWAIT_READY(masterSession);
server->expireSession(slaveSession.get().get());
server->expireSession(masterSession.get().get());
// Wait for session expiration and ensure a new master is detected.
AWAIT_READY(detected);
EXPECT_SOME_EQ(follower, detected.get());
}
Future<size_t> RegistryClientProcess::getBlob(
const Image::Name& imageName,
const Option<string>& digest,
const Path& filePath)
{
Try<Nothing> mkdir = os::mkdir(filePath.dirname(), true);
if (mkdir.isError()) {
return Failure(
"Failed to create directory to download blob: " + mkdir.error());
}
const string blobURLPath = getRepositoryPath(imageName) + "/blobs/" +
digest.getOrElse("");
http::URL blobURL(registryServer_);
blobURL.path = blobURLPath;
return doHttpGet(blobURL, None(), true, true, None())
.then([this, blobURLPath, digest, filePath](
const http::Response& response) -> Future<size_t> {
Try<int> fd = os::open(
filePath.value,
O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC,
S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
if (fd.isError()) {
return Failure("Failed to open file '" + filePath.value + "': " +
fd.error());
}
Try<Nothing> nonblock = os::nonblock(fd.get());
if (nonblock.isError()) {
Try<Nothing> close = os::close(fd.get());
if (close.isError()) {
LOG(WARNING) << "Failed to close the file descriptor for file '"
<< stringify(filePath) << "': " << close.error();
}
return Failure(
"Failed to set non-blocking mode for file: " + filePath.value);
}
// TODO(jojy): Add blob validation.
// TODO(jojy): Add check for max size.
Option<Pipe::Reader> reader = response.reader;
if (reader.isNone()) {
Try<Nothing> close = os::close(fd.get());
if (close.isError()) {
LOG(WARNING) << "Failed to close the file descriptor for file '"
<< stringify(filePath) << "': " << close.error();
}
return Failure("Failed to get streaming reader from blob response");
}
return saveBlob(fd.get(), reader.get())
.onAny([blobURLPath, digest, filePath, fd](
const Future<size_t>& future) {
Try<Nothing> close = os::close(fd.get());
if (close.isError()) {
LOG(WARNING) << "Failed to close the file descriptor for blob '"
<< stringify(filePath) << "': " << close.error();
}
if (future.isFailed()) {
LOG(WARNING) << "Failed to save blob requested from '"
<< blobURLPath << "' to path '"
<< stringify(filePath) << "': " << future.failure();
}
if (future.isDiscarded()) {
LOG(WARNING) << "Failed to save blob requested from '"
<< blobURLPath << "' to path '" << stringify(filePath)
<< "': future discarded";
}
});
});
}
TEST(ProtobufTest, JSON)
{
tests::Message message;
message.set_b(true);
message.set_str("string");
message.set_bytes("bytes");
message.set_int32(-1);
message.set_int64(-1);
message.set_uint32(1);
message.set_uint64(1);
message.set_sint32(-1);
message.set_sint64(-1);
message.set_f(1.0);
message.set_d(1.0);
message.set_e(tests::ONE);
message.mutable_nested()->set_str("nested");
message.add_repeated_bool(true);
message.add_repeated_string("repeated_string");
message.add_repeated_bytes("repeated_bytes");
message.add_repeated_int32(-2);
message.add_repeated_int64(-2);
message.add_repeated_uint32(2);
message.add_repeated_uint64(2);
message.add_repeated_sint32(-2);
message.add_repeated_sint64(-2);
message.add_repeated_float(1.0);
message.add_repeated_double(1.0);
message.add_repeated_double(2.0);
message.add_repeated_enum(tests::TWO);
message.add_repeated_nested()->set_str("repeated_nested");
// TODO(bmahler): To dynamically generate a protobuf message,
// see the commented-out code below.
// DescriptorProto proto;
//
// proto.set_name("Message");
//
// FieldDescriptorProto* field = proto.add_field();
// field->set_name("str");
// field->set_type(FieldDescriptorProto::TYPE_STRING);
//
// const Descriptor* descriptor = proto.descriptor();
//
// DynamicMessageFactory factory;
// Message* message = factory.GetPrototype(descriptor);
//
// Reflection* message.getReflection();
// The keys are in alphabetical order.
string expected = strings::remove(
"{"
" \"b\": true,"
" \"bytes\": \"bytes\","
" \"d\": 1,"
" \"e\": \"ONE\","
" \"f\": 1,"
" \"int32\": -1,"
" \"int64\": -1,"
" \"nested\": { \"str\": \"nested\"},"
" \"optional_default\": 42,"
" \"repeated_bool\": [true],"
" \"repeated_bytes\": [\"repeated_bytes\"],"
" \"repeated_double\": [1, 2],"
" \"repeated_enum\": [\"TWO\"],"
" \"repeated_float\": [1],"
" \"repeated_int32\": [-2],"
" \"repeated_int64\": [-2],"
" \"repeated_nested\": [ { \"str\": \"repeated_nested\" } ],"
" \"repeated_sint32\": [-2],"
" \"repeated_sint64\": [-2],"
" \"repeated_string\": [\"repeated_string\"],"
" \"repeated_uint32\": [2],"
" \"repeated_uint64\": [2],"
" \"sint32\": -1,"
" \"sint64\": -1,"
" \"str\": \"string\","
" \"uint32\": 1,"
" \"uint64\": 1"
"}",
" ");
JSON::Object object = JSON::Protobuf(message);
EXPECT_EQ(expected, stringify(object));
// Test parsing too.
Try<tests::Message> parse = protobuf::parse<tests::Message>(object);
ASSERT_SOME(parse);
EXPECT_EQ(object, JSON::Protobuf(parse.get()));
// Modify the message to test (de-)serialization of random bytes generated
// by UUID.
message.set_bytes(UUID::random().toBytes());
object = JSON::Protobuf(message);
// Test parsing too.
parse = protobuf::parse<tests::Message>(object);
ASSERT_SOME(parse);
EXPECT_EQ(object, JSON::Protobuf(parse.get()));
// Now convert JSON to string and parse it back as JSON.
ASSERT_SOME_EQ(object, JSON::parse(stringify(object)));
}
int main(int argc, char** argv)
{
GOOGLE_PROTOBUF_VERIFY_VERSION;
master::Flags flags;
// The following flags are executable specific (e.g., since we only
// have one instance of libprocess per execution, we only want to
// advertise the IP and port option once, here).
Option<string> ip;
flags.add(&ip, "ip", "IP address to listen on");
uint16_t port;
flags.add(&port, "port", "Port to listen on", MasterInfo().port());
Option<string> zk;
flags.add(&zk,
"zk",
"ZooKeeper URL (used for leader election amongst masters)\n"
"May be one of:\n"
" zk://host1:port1,host2:port2,.../path\n"
" zk://username:password@host1:port1,host2:port2,.../path\n"
" file:///path/to/file (where file contains one of the above)");
bool help;
flags.add(&help,
"help",
"Prints this help message",
false);
Try<Nothing> load = flags.load("MESOS_", argc, argv);
if (load.isError()) {
cerr << load.error() << endl;
usage(argv[0], flags);
exit(1);
}
if (flags.version) {
version();
exit(0);
}
if (help) {
usage(argv[0], flags);
exit(1);
}
// Initialize libprocess.
if (ip.isSome()) {
os::setenv("LIBPROCESS_IP", ip.get());
}
os::setenv("LIBPROCESS_PORT", stringify(port));
process::initialize("master");
logging::initialize(argv[0], flags, true); // Catch signals.
LOG(INFO) << "Build: " << build::DATE << " by " << build::USER;
LOG(INFO) << "Version: " << MESOS_VERSION;
if (build::GIT_TAG.isSome()) {
LOG(INFO) << "Git tag: " << build::GIT_TAG.get();
}
if (build::GIT_SHA.isSome()) {
LOG(INFO) << "Git SHA: " << build::GIT_SHA.get();
}
allocator::AllocatorProcess* allocatorProcess =
new allocator::HierarchicalDRFAllocatorProcess();
allocator::Allocator* allocator =
new allocator::Allocator(allocatorProcess);
state::Storage* storage = NULL;
Log* log = NULL;
if (flags.registry == "in_memory") {
if (flags.registry_strict) {
EXIT(1) << "Cannot use '--registry_strict' when using in-memory storage"
<< " based registry";
}
storage = new state::InMemoryStorage();
} else if (flags.registry == "replicated_log" ||
flags.registry == "log_storage") {
// TODO(bmahler): "log_storage" is present for backwards
// compatibility, can be removed before 0.19.0.
if (flags.work_dir.isNone()) {
EXIT(1) << "--work_dir needed for replicated log based registry";
}
Try<Nothing> mkdir = os::mkdir(flags.work_dir.get());
if (mkdir.isError()) {
EXIT(1) << "Failed to create work directory '" << flags.work_dir.get()
<< "': " << mkdir.error();
}
if (zk.isSome()) {
// Use replicated log with ZooKeeper.
if (flags.quorum.isNone()) {
EXIT(1) << "Need to specify --quorum for replicated log based"
<< " registry when using ZooKeeper";
}
string zk_;
if (strings::startsWith(zk.get(), "file://")) {
const string& path = zk.get().substr(7);
const Try<string> read = os::read(path);
if (read.isError()) {
EXIT(1) << "Failed to read from file at '" + path + "': "
<< read.error();
}
zk_ = read.get();
} else {
zk_ = zk.get();
}
Try<URL> url = URL::parse(zk_);
if (url.isError()) {
EXIT(1) << "Error parsing ZooKeeper URL: " << url.error();
}
log = new Log(
flags.quorum.get(),
path::join(flags.work_dir.get(), "replicated_log"),
url.get().servers,
flags.zk_session_timeout,
path::join(url.get().path, "log_replicas"),
url.get().authentication,
flags.log_auto_initialize);
} else {
// Use replicated log without ZooKeeper.
log = new Log(
1,
path::join(flags.work_dir.get(), "replicated_log"),
set<UPID>(),
flags.log_auto_initialize);
}
storage = new state::LogStorage(log);
} else {
EXIT(1) << "'" << flags.registry << "' is not a supported"
<< " option for registry persistence";
}
CHECK_NOTNULL(storage);
state::protobuf::State* state = new state::protobuf::State(storage);
Registrar* registrar = new Registrar(flags, state);
Repairer* repairer = new Repairer();
Files files;
MasterContender* contender;
MasterDetector* detector;
// TODO(vinod): 'MasterContender::create()' should take
// Option<string>.
Try<MasterContender*> contender_ = MasterContender::create(zk.get(""));
if (contender_.isError()) {
EXIT(1) << "Failed to create a master contender: " << contender_.error();
}
contender = contender_.get();
// TODO(vinod): 'MasterDetector::create()' should take
// Option<string>.
Try<MasterDetector*> detector_ = MasterDetector::create(zk.get(""));
if (detector_.isError()) {
EXIT(1) << "Failed to create a master detector: " << detector_.error();
}
detector = detector_.get();
Option<Authorizer*> authorizer = None();
if (flags.acls.isSome()) {
Try<Owned<Authorizer> > authorizer_ = Authorizer::create(flags.acls.get());
if (authorizer_.isError()) {
EXIT(1) << "Failed to initialize the authorizer: "
<< authorizer_.error() << " (see --acls flag)";
}
Owned<Authorizer> authorizer__ = authorizer_.get();
authorizer = authorizer__.release();
}
LOG(INFO) << "Starting Mesos master";
Master* master =
new Master(
allocator,
registrar,
repairer,
&files,
contender,
detector,
authorizer,
flags);
if (zk.isNone()) {
// It means we are using the standalone detector so we need to
// appoint this Master as the leader.
dynamic_cast<StandaloneMasterDetector*>(detector)->appoint(master->info());
}
process::spawn(master);
process::wait(master->self());
delete master;
delete allocator;
delete allocatorProcess;
delete registrar;
delete repairer;
delete state;
delete storage;
delete log;
delete contender;
delete detector;
if (authorizer.isSome()) {
delete authorizer.get();
}
return 0;
}
TEST(Metrics, SnapshotTimeout)
{
ASSERT_TRUE(GTEST_IS_THREADSAFE);
UPID upid("metrics", process::address());
Clock::pause();
// Advance the clock to avoid rate limit.
Clock::advance(Seconds(1));
// Ensure the timeout parameter is validated.
AWAIT_EXPECT_RESPONSE_STATUS_EQ(
BadRequest().status,
http::get(upid, "snapshot", "timeout=foobar"));
// Advance the clock to avoid rate limit.
Clock::advance(Seconds(1));
// Add gauges and a counter.
GaugeProcess process;
PID<GaugeProcess> pid = spawn(&process);
ASSERT_TRUE(pid);
Gauge gauge("test/gauge", defer(pid, &GaugeProcess::get));
Gauge gaugeFail("test/gauge_fail", defer(pid, &GaugeProcess::fail));
Gauge gaugeTimeout("test/gauge_timeout", defer(pid, &GaugeProcess::pending));
Counter counter("test/counter");
AWAIT_READY(metrics::add(gauge));
AWAIT_READY(metrics::add(gaugeFail));
AWAIT_READY(metrics::add(gaugeTimeout));
AWAIT_READY(metrics::add(counter));
// Advance the clock to avoid rate limit.
Clock::advance(Seconds(1));
// Get the snapshot.
Future<Response> response = http::get(upid, "snapshot", "timeout=2secs");
// Make sure the request is pending before the timeout is exceeded.
Clock::settle();
ASSERT_TRUE(response.isPending());
// Advance the clock to trigger the timeout.
Clock::advance(Seconds(2));
AWAIT_EXPECT_RESPONSE_STATUS_EQ(OK().status, response);
// Parse the response.
Try<JSON::Object> responseJSON =
JSON::parse<JSON::Object>(response.get().body);
ASSERT_SOME(responseJSON);
// We can't use simple JSON equality testing here as initializing
// libprocess adds metrics to the system. We want to only check if
// the metrics from this test are correctly handled.
map<string, JSON::Value> values = responseJSON.get().values;
EXPECT_EQ(1u, values.count("test/counter"));
EXPECT_FLOAT_EQ(0.0, values["test/counter"].as<JSON::Number>().value);
EXPECT_EQ(1u, values.count("test/gauge"));
EXPECT_FLOAT_EQ(42.0, values["test/gauge"].as<JSON::Number>().value);
EXPECT_EQ(0u, values.count("test/gauge_fail"));
EXPECT_EQ(0u, values.count("test/gauge_timeout"));
// Remove the metrics and ensure they are no longer in the snapshot.
AWAIT_READY(metrics::remove(gauge));
AWAIT_READY(metrics::remove(gaugeFail));
AWAIT_READY(metrics::remove(gaugeTimeout));
AWAIT_READY(metrics::remove(counter));
// Advance the clock to avoid rate limit.
Clock::advance(Seconds(1));
// Ensure MetricsProcess has removed the metrics.
Clock::settle();
response = http::get(upid, "snapshot", "timeout=2secs");
AWAIT_EXPECT_RESPONSE_STATUS_EQ(OK().status, response);
// Parse the response.
responseJSON = JSON::parse<JSON::Object>(response.get().body);
ASSERT_SOME(responseJSON);
values = responseJSON.get().values;
ASSERT_SOME(responseJSON);
EXPECT_EQ(0u, values.count("test/counter"));
EXPECT_EQ(0u, values.count("test/gauge"));
EXPECT_EQ(0u, values.count("test/gauge_fail"));
EXPECT_EQ(0u, values.count("test/gauge_timeout"));
terminate(process);
wait(process);
}
Future<Response> RegistryClientProcess::doHttpGet(
const URL& url,
const Option<process::http::Headers>& headers,
const Duration& timeout,
bool resend,
const Option<string>& lastResponseStatus) const
{
return process::http::get(url, headers)
.after(timeout, [](
const Future<Response>& httpResponseFuture) -> Future<Response> {
return Failure("Response timeout");
})
.then(defer(self(), [=](
const Response& httpResponse) -> Future<Response> {
VLOG(1) << "Response status: " + httpResponse.status;
// Set the future if we get a OK response.
if (httpResponse.status == "200 OK") {
return httpResponse;
} else if (httpResponse.status == "400 Bad Request") {
Try<JSON::Object> errorResponse =
JSON::parse<JSON::Object>(httpResponse.body);
if (errorResponse.isError()) {
return Failure("Failed to parse bad request response JSON: " +
errorResponse.error());
}
std::ostringstream out;
bool first = true;
Result<JSON::Array> errorObjects =
errorResponse.get().find<JSON::Array>("errors");
if (errorObjects.isError()) {
return Failure("Failed to find 'errors' in bad request response: " +
errorObjects.error());
} else if (errorObjects.isNone()) {
return Failure("Errors not found in bad request response");
}
foreach (const JSON::Value& error, errorObjects.get().values) {
Result<JSON::String> message =
error.as<JSON::Object>().find<JSON::String>("message");
if (message.isError()) {
return Failure("Failed to parse bad request error message: " +
message.error());
} else if (message.isNone()) {
continue;
}
if (first) {
out << message.get().value;
first = false;
} else {
out << ", " << message.get().value;
}
}
return Failure("Received Bad request, errors: [" + out.str() + "]");
}
// Prevent infinite recursion.
if (lastResponseStatus.isSome() &&
(lastResponseStatus.get() == httpResponse.status)) {
return Failure("Invalid response: " + httpResponse.status);
}
// If resend is not set, we dont try again and stop here.
if (!resend) {
return Failure("Bad response: " + httpResponse.status);
}
// Handle 401 Unauthorized.
if (httpResponse.status == "401 Unauthorized") {
Try<process::http::Headers> authAttributes =
getAuthenticationAttributes(httpResponse);
if (authAttributes.isError()) {
return Failure(
"Failed to get authentication attributes: " +
authAttributes.error());
}
// TODO(jojy): Currently only handling TLS/cert authentication.
Future<Token> tokenResponse = tokenManager_->getToken(
authAttributes.get().at("service"),
authAttributes.get().at("scope"),
None());
return tokenResponse
.after(timeout, [=](
Future<Token> tokenResponse) -> Future<Token> {
tokenResponse.discard();
return Failure("Token response timeout");
})
.then(defer(self(), [=](
const Future<Token>& tokenResponse) {
// Send request with acquired token.
process::http::Headers authHeaders = {
{"Authorization", "Bearer " + tokenResponse.get().raw}
};
return doHttpGet(
url,
authHeaders,
timeout,
true,
httpResponse.status);
}));
} else if (httpResponse.status == "307 Temporary Redirect") {
// Handle redirect.
// TODO(jojy): Add redirect functionality in http::get.
auto toURL = [](
const string& urlString) -> Try<URL> {
// TODO(jojy): Need to add functionality to URL class that parses a
// string to its URL components. For now, assuming:
// - scheme is https
// - path always ends with /
static const string schemePrefix = "https://";
if (!strings::contains(urlString, schemePrefix)) {
return Error(
"Failed to find expected token '" + schemePrefix +
"' in redirect url");
}
const string schemeSuffix = urlString.substr(schemePrefix.length());
const vector<string> components =
strings::tokenize(schemeSuffix, "/");
const string path = schemeSuffix.substr(components[0].length());
const vector<string> addrComponents =
strings::tokenize(components[0], ":");
uint16_t port = DEFAULT_SSL_PORT;
string domain = components[0];
// Parse the port.
if (addrComponents.size() == 2) {
domain = addrComponents[0];
Try<uint16_t> tryPort = numify<uint16_t>(addrComponents[1]);
if (tryPort.isError()) {
return Error(
"Failed to parse location: " + urlString + " for port.");
}
port = tryPort.get();
}
return URL("https", domain, port, path);
};
if (httpResponse.headers.find("Location") ==
httpResponse.headers.end()) {
return Failure(
"Invalid redirect response: 'Location' not found in headers.");
}
const string& location = httpResponse.headers.at("Location");
Try<URL> tryUrl = toURL(location);
if (tryUrl.isError()) {
return Failure(
"Failed to parse '" + location + "': " + tryUrl.error());
}
return doHttpGet(
tryUrl.get(),
headers,
timeout,
false,
httpResponse.status);
} else {
return Failure("Invalid response: " + httpResponse.status);
}
}));
Future<ExecutorInfo> ExternalContainerizerProcess::launch(
const ContainerID& containerId,
const TaskInfo& taskInfo,
const FrameworkID& frameworkId,
const std::string& directory,
const Option<std::string>& user,
const SlaveID& slaveId,
const PID<Slave>& slavePid,
bool checkpoint)
{
LOG(INFO) << "Launching container '" << containerId << "'";
// Get the executor from our task. If no executor is associated with
// the given task, this function renders an ExecutorInfo using the
// mesos-executor as its command.
ExecutorInfo executor = containerExecutorInfo(flags, taskInfo, frameworkId);
executor.mutable_resources()->MergeFrom(taskInfo.resources());
if (containers.contains(containerId)) {
return Failure("Cannot start already running container '"
+ containerId.value() + "'");
}
sandboxes.put(containerId, Owned<Sandbox>(new Sandbox(directory, user)));
map<string, string> environment = executorEnvironment(
executor,
directory,
slaveId,
slavePid,
checkpoint,
flags.recovery_timeout);
if (!flags.hadoop_home.empty()) {
environment["HADOOP_HOME"] = flags.hadoop_home;
}
TaskInfo task;
task.CopyFrom(taskInfo);
CommandInfo* command = task.has_executor()
? task.mutable_executor()->mutable_command()
: task.mutable_command();
// When the selected command has no container attached, use the
// default from the slave startup flags, if available.
if (!command->has_container()) {
if (flags.default_container_image.isSome()) {
command->mutable_container()->set_image(
flags.default_container_image.get());
} else {
LOG(INFO) << "No container specified in task and no default given. "
<< "The external containerizer will have to fill in "
<< "defaults.";
}
}
ExternalTask external;
external.mutable_task()->CopyFrom(task);
external.set_mesos_executor_path(
path::join(flags.launcher_dir, "mesos-executor"));
stringstream output;
external.SerializeToOstream(&output);
Try<Subprocess> invoked = invoke(
"launch",
containerId,
output.str(),
environment);
if (invoked.isError()) {
return Failure("Launch of container '" + containerId.value()
+ "' failed (error: " + invoked.error() + ")");
}
// Record the process.
containers.put(
containerId,
Owned<Container>(new Container(invoked.get().pid())));
VLOG(2) << "Now awaiting data from pipe...";
// Read from the result-pipe and invoke callbacks when reaching EOF.
return await(read(invoked.get().out()), invoked.get().status())
.then(defer(
PID<ExternalContainerizerProcess>(this),
&ExternalContainerizerProcess::_launch,
containerId,
frameworkId,
executor,
slaveId,
checkpoint,
lambda::_1));
}
Future<Nothing> NetworkCniIsolatorProcess::_attach(
const ContainerID& containerId,
const string& networkName,
const string& plugin,
const tuple<Future<Option<int>>, Future<string>>& t)
{
CHECK(infos.contains(containerId));
CHECK(infos[containerId]->containerNetworks.contains(networkName));
Future<Option<int>> status = std::get<0>(t);
if (!status.isReady()) {
return Failure(
"Failed to get the exit status of the CNI plugin '" +
plugin + "' subprocess: " +
(status.isFailed() ? status.failure() : "discarded"));
}
if (status->isNone()) {
return Failure(
"Failed to reap the CNI plugin '" + plugin + "' subprocess");
}
// CNI plugin will print result (in case of success) or error (in
// case of failure) to stdout.
Future<string> output = std::get<1>(t);
if (!output.isReady()) {
return Failure(
"Failed to read stdout from the CNI plugin '" +
plugin + "' subprocess: " +
(output.isFailed() ? output.failure() : "discarded"));
}
if (status.get() != 0) {
return Failure(
"The CNI plugin '" + plugin + "' failed to attach container " +
containerId.value() + " to CNI network '" + networkName +
"': " + output.get());
}
// Parse the output of CNI plugin.
Try<spec::NetworkInfo> parse = spec::parseNetworkInfo(output.get());
if (parse.isError()) {
return Failure(
"Failed to parse the output of the CNI plugin '" +
plugin + "': " + parse.error());
}
if (parse.get().has_ip4()) {
LOG(INFO) << "Got assigned IPv4 address '" << parse.get().ip4().ip()
<< "' from CNI network '" << networkName
<< "' for container " << containerId;
}
if (parse.get().has_ip6()) {
LOG(INFO) << "Got assigned IPv6 address '" << parse.get().ip6().ip()
<< "' from CNI network '" << networkName
<< "' for container " << containerId;
}
// Checkpoint the output of CNI plugin.
// The destruction of the container cannot happen in the middle of
// 'attach()' and '_attach()' because the containerizer will wait
// for 'isolate()' to finish before destroying the container.
ContainerNetwork& containerNetwork =
infos[containerId]->containerNetworks[networkName];
const string networkInfoPath = paths::getNetworkInfoPath(
rootDir.get(),
containerId.value(),
networkName,
containerNetwork.ifName);
Try<Nothing> write = os::write(networkInfoPath, output.get());
if (write.isError()) {
return Failure(
"Failed to checkpoint the output of CNI plugin'" +
output.get() + "': " + write.error());
}
containerNetwork.cniNetworkInfo = parse.get();
return Nothing();
}
TYPED_TEST(CpuIsolatorTest, SystemCpuUsage)
{
Flags flags;
Try<Isolator*> isolator = TypeParam::create(flags);
CHECK_SOME(isolator);
// A PosixLauncher is sufficient even when testing a cgroups isolator.
Try<Launcher*> launcher = PosixLauncher::create(flags);
ExecutorInfo executorInfo;
executorInfo.mutable_resources()->CopyFrom(
Resources::parse("cpus:1.0").get());
ContainerID containerId;
containerId.set_value("system_cpu_usage");
AWAIT_READY(isolator.get()->prepare(containerId, executorInfo));
Try<string> dir = os::mkdtemp();
ASSERT_SOME(dir);
const string& file = path::join(dir.get(), "mesos_isolator_test_ready");
// Generating random numbers is done by the kernel and will max out a single
// core and run almost exclusively in the kernel, i.e., system time.
string command = "cat /dev/urandom > /dev/null & "
"touch " + file + "; " // Signals the command is running.
"sleep 60";
int pipes[2];
ASSERT_NE(-1, ::pipe(pipes));
lambda::function<int()> inChild = lambda::bind(&execute, command, pipes);
Try<pid_t> pid = launcher.get()->fork(containerId, inChild);
ASSERT_SOME(pid);
// Reap the forked child.
Future<Option<int> > status = process::reap(pid.get());
// Continue in the parent.
::close(pipes[0]);
// Isolate the forked child.
AWAIT_READY(isolator.get()->isolate(containerId, pid.get()));
// Now signal the child to continue.
int buf;
ASSERT_LT(0, ::write(pipes[1], &buf, sizeof(buf)));
::close(pipes[1]);
// Wait for the command to start.
while (!os::exists(file));
// Wait up to 1 second for the child process to induce 1/8 of a second of
// system cpu time.
ResourceStatistics statistics;
Duration waited = Duration::zero();
do {
Future<ResourceStatistics> usage = isolator.get()->usage(containerId);
AWAIT_READY(usage);
statistics = usage.get();
// If we meet our usage expectations, we're done!
if (statistics.cpus_system_time_secs() >= 0.125) {
break;
}
os::sleep(Milliseconds(200));
waited += Milliseconds(200);
} while (waited < Seconds(1));
EXPECT_LE(0.125, statistics.cpus_system_time_secs());
// Shouldn't be any appreciable user time.
EXPECT_GT(0.025, statistics.cpus_user_time_secs());
// Ensure all processes are killed.
AWAIT_READY(launcher.get()->destroy(containerId));
// Make sure the child was reaped.
AWAIT_READY(status);
// Let the isolator clean up.
AWAIT_READY(isolator.get()->cleanup(containerId));
delete isolator.get();
delete launcher.get();
CHECK_SOME(os::rmdir(dir.get()));
}
TEST_F(HealthTest, ObserveEndpoint)
{
Try<PID<Master> > master = StartMaster();
ASSERT_SOME(master);
// Empty get to the observe endpoint.
Future<Response> response = process::http::get(master.get(), "observe");
VALIDATE_BAD_RESPONSE(response, "Missing value for 'monitor'.");
// Empty post to the observe endpoint.
response = process::http::post(master.get(), "observe");
VALIDATE_BAD_RESPONSE(response, "Missing value for 'monitor'.");
// Query string is ignored.
response = process::http::post(master.get(), "observe?monitor=foo");
VALIDATE_BAD_RESPONSE(response, "Missing value for 'monitor'.");
// Malformed value causes error.
response = process::http::post(
master.get(),
"observe",
None(),
"monitor=foo%");
VALIDATE_BAD_RESPONSE(
response,
"Unable to decode query string: Malformed % escape in 'foo%': '%'");
// Empty value causes error.
response = process::http::post(
master.get(),
"observe",
None(),
"monitor=");
VALIDATE_BAD_RESPONSE(response, "Empty string for 'monitor'.");
// Missing hosts.
response = process::http::post(
master.get(),
"observe",
None(),
"monitor=a");
VALIDATE_BAD_RESPONSE(response, "Missing value for 'hosts'.");
// Missing level.
response = process::http::post(
master.get(),
"observe",
None(),
"monitor=a&hosts=b");
VALIDATE_BAD_RESPONSE(response, "Missing value for 'level'.");
// Good request is successful.
JsonResponse expected;
expected.monitor = "a";
expected.hosts.push_back("b");
expected.isHealthy = true;
response = process::http::post(
master.get(),
"observe",
None(),
"monitor=a&hosts=b&level=ok");
VALIDATE_GOOD_RESPONSE(response, stringify(expected) );
// ok is case-insensitive.
response = process::http::post(
master.get(),
"observe",
None(),
"monitor=a&hosts=b&level=Ok");
VALIDATE_GOOD_RESPONSE(response, stringify(expected) );
response = process::http::post(
master.get(),
"observe",
None(),
"monitor=a&hosts=b&level=oK");
VALIDATE_GOOD_RESPONSE(response, stringify(expected) );
response = process::http::post(
master.get(),
"observe",
None(),
"monitor=a&hosts=b&level=OK");
VALIDATE_GOOD_RESPONSE(response, stringify(expected) );
// level != OK is unhealthy.
expected.isHealthy = false;
response =
process::http::post(
master.get(),
"observe",
None(),
"monitor=a&hosts=b&level=true");
VALIDATE_GOOD_RESPONSE(response, stringify(expected) );
// Comma-separated hosts are parsed into an array.
expected.hosts.push_back("e");
response = process::http::post(
master.get(),
"observe",
None(),
"monitor=a&hosts=b,e&level=true");
VALIDATE_GOOD_RESPONSE(response, stringify(expected) );
Shutdown();
}
Try<pid_t> LinuxLauncher::fork(
const ContainerID& containerId,
const string& path,
const vector<string>& argv,
const process::Subprocess::IO& in,
const process::Subprocess::IO& out,
const process::Subprocess::IO& err,
const Option<flags::FlagsBase>& flags,
const Option<map<string, string>>& environment,
const Option<lambda::function<int()>>& setup,
const Option<int>& namespaces)
{
// Create a freezer cgroup for this container if necessary.
Try<bool> exists = cgroups::exists(hierarchy, cgroup(containerId));
if (exists.isError()) {
return Error("Failed to check existence of freezer cgroup: " +
exists.error());
}
if (!exists.get()) {
Try<Nothing> created = cgroups::create(hierarchy, cgroup(containerId));
if (created.isError()) {
return Error("Failed to create freezer cgroup: " + created.error());
}
}
// Use a pipe to block the child until it's been moved into the
// freezer cgroup.
int pipes[2];
// We assume this should not fail under reasonable conditions so we
// use CHECK.
CHECK_EQ(0, ::pipe(pipes));
Try<Subprocess> child = subprocess(
path,
argv,
in,
out,
err,
flags,
environment,
lambda::bind(&childSetup, pipes, setup),
lambda::bind(&clone, lambda::_1, namespaces));
if (child.isError()) {
return Error("Failed to clone child process: " + child.error());
}
// Parent.
os::close(pipes[0]);
// Move the child into the freezer cgroup. Any grandchildren will
// also be contained in the cgroup.
// TODO(jieyu): Move this logic to the subprocess (i.e.,
// mesos-containerizer launch).
Try<Nothing> assign = cgroups::assign(
hierarchy,
cgroup(containerId),
child.get().pid());
if (assign.isError()) {
LOG(ERROR) << "Failed to assign process " << child.get().pid()
<< " of container '" << containerId << "'"
<< " to its freezer cgroup: " << assign.error();
::kill(child.get().pid(), SIGKILL);
return Error("Failed to contain process");
}
// Now that we've contained the child we can signal it to continue
// by writing to the pipe.
char dummy;
ssize_t length;
while ((length = ::write(pipes[1], &dummy, sizeof(dummy))) == -1 &&
errno == EINTR);
os::close(pipes[1]);
if (length != sizeof(dummy)) {
// Ensure the child is killed.
::kill(child.get().pid(), SIGKILL);
return Error("Failed to synchronize child process");
}
if (!pids.contains(containerId)) {
pids.put(containerId, child.get().pid());
}
return child.get().pid();
}
// Tests that detectors do not fail when we reach our ZooKeeper
// session timeout.
TEST_F(ZooKeeperMasterContenderDetectorTest, MasterDetectorTimedoutSession)
{
// Use an arbitrary timeout value.
Duration sessionTimeout(Seconds(5));
Try<zookeeper::URL> url = zookeeper::URL::parse(
"zk://" + server->connectString() + "/mesos");
ASSERT_SOME(url);
Owned<zookeeper::Group> leaderGroup(new Group(url.get(), sessionTimeout));
// First we bring up three master contender/detector:
// 1. A leading contender.
// 2. A non-leading contender.
// 3. A non-contender (detector).
// 1. Simulate a leading contender.
ZooKeeperMasterContender leaderContender(leaderGroup);
PID<Master> leader;
leader.ip = 10000000;
leader.port = 10000;
leaderContender.initialize(leader);
Future<Future<Nothing> > contended = leaderContender.contend();
AWAIT_READY(contended);
Future<Nothing> leaderLostCandidacy = contended.get();
ZooKeeperMasterDetector leaderDetector(leaderGroup);
Future<Option<UPID> > detected = leaderDetector.detect();
AWAIT_READY(detected);
EXPECT_SOME_EQ(leader, detected.get());
// 2. Simulate a non-leading contender.
Owned<zookeeper::Group> followerGroup(new Group(url.get(), sessionTimeout));
ZooKeeperMasterContender followerContender(followerGroup);
PID<Master> follower;
follower.ip = 10000001;
follower.port = 10001;
followerContender.initialize(follower);
contended = followerContender.contend();
AWAIT_READY(contended);
Future<Nothing> followerLostCandidacy = contended.get();
ZooKeeperMasterDetector followerDetector(followerGroup);
detected = followerDetector.detect();
AWAIT_READY(detected);
EXPECT_SOME_EQ(leader, detected.get());
// 3. Simulate a non-contender.
Owned<zookeeper::Group> nonContenderGroup(
new Group(url.get(), sessionTimeout));
ZooKeeperMasterDetector nonContenderDetector(nonContenderGroup);
detected = nonContenderDetector.detect();
EXPECT_SOME_EQ(leader, detected.get());
// Expecting the reconnecting event after we shut down the ZK.
Future<Nothing> leaderReconnecting = FUTURE_DISPATCH(
leaderGroup->process->self(),
&GroupProcess::reconnecting);
Future<Nothing> followerReconnecting = FUTURE_DISPATCH(
followerGroup->process->self(),
&GroupProcess::reconnecting);
Future<Nothing> nonContenderReconnecting = FUTURE_DISPATCH(
nonContenderGroup->process->self(),
&GroupProcess::reconnecting);
server->shutdownNetwork();
AWAIT_READY(leaderReconnecting);
AWAIT_READY(followerReconnecting);
AWAIT_READY(nonContenderReconnecting);
// Now the detectors re-detect.
Future<Option<UPID> > leaderDetected = leaderDetector.detect(leader);
Future<Option<UPID> > followerDetected = followerDetector.detect(leader);
Future<Option<UPID> > nonContenderDetected =
nonContenderDetector.detect(leader);
Clock::pause();
// We know when the groups have timed out when the contenders are
// informed of the candidacy loss.
// We may need to advance multiple times because we could have
// advanced the clock before the timer in Group starts.
while (leaderLostCandidacy.isPending() || followerLostCandidacy.isPending()) {
Clock::advance(sessionTimeout);
Clock::settle();
}
// Detection is not interrupted.
EXPECT_TRUE(leaderDetected.isPending());
EXPECT_TRUE(followerDetected.isPending());
EXPECT_TRUE(nonContenderDetected.isPending());
Clock::resume();
}
// This test verifies that the volume usage accounting for sandboxes
// with bind-mounted volumes (while linux filesystem isolator is used)
// works correctly by creating a file within the volume the size of
// which exceeds the sandbox quota.
TEST_F(LinuxFilesystemIsolatorMesosTest,
ROOT_VolumeUsageExceedsSandboxQuota)
{
Try<Owned<cluster::Master>> master = StartMaster();
ASSERT_SOME(master);
string registry = path::join(sandbox.get(), "registry");
AWAIT_READY(DockerArchive::create(registry, "test_image"));
slave::Flags flags = CreateSlaveFlags();
flags.resources = "cpus:2;mem:128;disk(role1):128";
flags.isolation = "disk/du,filesystem/linux,docker/runtime";
flags.docker_registry = registry;
flags.docker_store_dir = path::join(sandbox.get(), "store");
flags.image_providers = "docker";
// NOTE: We can't pause the clock because we need the reaper to reap
// the 'du' subprocess.
flags.container_disk_watch_interval = Milliseconds(1);
flags.enforce_container_disk_quota = true;
Owned<MasterDetector> detector = master.get()->createDetector();
Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), flags);
ASSERT_SOME(slave);
MockScheduler sched;
FrameworkInfo frameworkInfo = DEFAULT_FRAMEWORK_INFO;
frameworkInfo.set_roles(0, "role1");
MesosSchedulerDriver driver(
&sched,
frameworkInfo,
master.get()->pid,
DEFAULT_CREDENTIAL);
EXPECT_CALL(sched, registered(&driver, _, _));
Future<vector<Offer>> offers;
EXPECT_CALL(sched, resourceOffers(&driver, _))
.WillOnce(FutureArg<1>(&offers))
.WillRepeatedly(Return()); // Ignore subsequent offers.
driver.start();
AWAIT_READY(offers);
ASSERT_FALSE(offers->empty());
// We request a sandbox (1MB) that is smaller than the persistent
// volume (4MB) and attempt to create a file in that volume that is
// twice the size of the sanbox (2MB).
Resources volume = createPersistentVolume(
Megabytes(4),
"role1",
"id1",
"volume_path",
None(),
None(),
frameworkInfo.principal());
Resources taskResources =
Resources::parse("cpus:1;mem:64;disk(role1):1").get() + volume;
// We sleep to give quota enforcement (du) a chance to kick in.
TaskInfo task = createTask(
offers.get()[0].slave_id(),
taskResources,
"dd if=/dev/zero of=volume_path/file bs=1048576 count=2 && sleep 1");
Future<TaskStatus> statusStarting;
Future<TaskStatus> statusRunning;
Future<TaskStatus> statusFinished;
EXPECT_CALL(sched, statusUpdate(&driver, _))
.WillOnce(FutureArg<1>(&statusStarting))
.WillOnce(FutureArg<1>(&statusRunning))
.WillOnce(FutureArg<1>(&statusFinished));
driver.acceptOffers(
{offers.get()[0].id()},
{CREATE(volume),
LAUNCH({task})});
AWAIT_READY(statusStarting);
EXPECT_EQ(task.task_id(), statusStarting->task_id());
EXPECT_EQ(TASK_STARTING, statusStarting->state());
AWAIT_READY(statusRunning);
EXPECT_EQ(task.task_id(), statusRunning->task_id());
EXPECT_EQ(TASK_RUNNING, statusRunning->state());
AWAIT_READY(statusFinished);
EXPECT_EQ(task.task_id(), statusFinished->task_id());
EXPECT_EQ(TASK_FINISHED, statusFinished->state());
driver.stop();
driver.join();
}
// Tests whether a leading master correctly detects a new master when
// its ZooKeeper session is expired (the follower becomes the new
// leader).
TEST_F(ZooKeeperMasterContenderDetectorTest,
MasterDetectorExpireMasterZKSession)
{
// Simulate a leading master.
Try<zookeeper::URL> url = zookeeper::URL::parse(
"zk://" + server->connectString() + "/mesos");
ASSERT_SOME(url);
PID<Master> leader;
leader.ip = 10000000;
leader.port = 10000;
// Create the group instance so we can expire its session.
Owned<zookeeper::Group> group(
new Group(url.get(), MASTER_CONTENDER_ZK_SESSION_TIMEOUT));
ZooKeeperMasterContender leaderContender(group);
leaderContender.initialize(leader);
Future<Future<Nothing> > leaderContended = leaderContender.contend();
AWAIT_READY(leaderContended);
Future<Nothing> leaderLostLeadership = leaderContended.get();
ZooKeeperMasterDetector leaderDetector(url.get());
Future<Option<UPID> > detected = leaderDetector.detect();
AWAIT_READY(detected);
EXPECT_SOME_EQ(leader, detected.get());
// Keep detecting.
Future<Option<UPID> > newLeaderDetected =
leaderDetector.detect(detected.get());
// Simulate a following master.
PID<Master> follower;
follower.ip = 10000001;
follower.port = 10001;
ZooKeeperMasterDetector followerDetector(url.get());
ZooKeeperMasterContender followerContender(url.get());
followerContender.initialize(follower);
Future<Future<Nothing> > followerContended = followerContender.contend();
AWAIT_READY(followerContended);
LOG(INFO) << "The follower now is detecting the leader";
detected = followerDetector.detect(None());
AWAIT_READY(detected);
EXPECT_SOME_EQ(leader, detected.get());
// Now expire the leader's zk session.
Future<Option<int64_t> > session = group->session();
AWAIT_READY(session);
EXPECT_SOME(session.get());
LOG(INFO) << "Now expire the ZK session: " << std::hex << session.get().get();
server->expireSession(session.get().get());
AWAIT_READY(leaderLostLeadership);
// Wait for session expiration and ensure the former leader detects
// a new leader.
AWAIT_READY(newLeaderDetected);
EXPECT_SOME(newLeaderDetected.get());
EXPECT_EQ(follower, newLeaderDetected.get().get());
}
// This test verifies that if a persistent volume and SANDBOX_PATH
// volume are both specified and the 'path' of the SANDBOX_PATH volume
// is the same relative path as the persistent volume's container
// path, the persistent volume will not be neglect and is mounted
// correctly. This is a regression test for MESOS-7770.
TEST_F(LinuxFilesystemIsolatorTest,
ROOT_PersistentVolumeAndHostVolumeWithRootFilesystem)
{
string registry = path::join(sandbox.get(), "registry");
AWAIT_READY(DockerArchive::create(registry, "test_image"));
slave::Flags flags = CreateSlaveFlags();
flags.isolation = "filesystem/linux,docker/runtime";
flags.docker_registry = registry;
flags.docker_store_dir = path::join(sandbox.get(), "store");
flags.image_providers = "docker";
Fetcher fetcher(flags);
Try<MesosContainerizer*> create =
MesosContainerizer::create(flags, true, &fetcher);
ASSERT_SOME(create);
Owned<Containerizer> containerizer(create.get());
ContainerID containerId;
containerId.set_value(id::UUID::random().toString());
// Write to an absolute path in the container's mount namespace to
// verify mounts of the SANDBOX_PATH volume and the persistent
// volume are done in the proper order.
ExecutorInfo executor = createExecutorInfo(
"test_executor",
"echo abc > /absolute_path/file");
executor.add_resources()->CopyFrom(createPersistentVolume(
Megabytes(32),
"test_role",
"persistent_volume_id",
"volume"));
executor.mutable_container()->CopyFrom(createContainerInfo(
"test_image",
{createVolumeSandboxPath("/absolute_path", "volume", Volume::RW)}));
// Create a persistent volume.
string volume = slave::paths::getPersistentVolumePath(
flags.work_dir,
"test_role",
"persistent_volume_id");
ASSERT_SOME(os::mkdir(volume));
string directory = path::join(flags.work_dir, "sandbox");
ASSERT_SOME(os::mkdir(directory));
Future<Containerizer::LaunchResult> launch = containerizer->launch(
containerId,
createContainerConfig(None(), executor, directory),
map<string, string>(),
None());
AWAIT_ASSERT_EQ(Containerizer::LaunchResult::SUCCESS, launch);
Future<Option<ContainerTermination>> wait = containerizer->wait(containerId);
AWAIT_READY(wait);
ASSERT_SOME(wait.get());
ASSERT_TRUE(wait->get().has_status());
EXPECT_WEXITSTATUS_EQ(0, wait->get().status());
EXPECT_SOME_EQ("abc\n", os::read(path::join(volume, "file")));
}
// This test verifies that default executor subscription fails if the executor
// provides a properly-signed authentication token with invalid claims.
TEST_F(ExecutorAuthorizationTest, FailedSubscribe)
{
Try<Owned<cluster::Master>> master = StartMaster();
ASSERT_SOME(master);
// Start an agent with permissive ACLs so that a task can be launched.
ACLs acls;
acls.set_permissive(true);
Result<Authorizer*> authorizer = Authorizer::create(acls);
ASSERT_SOME(authorizer);
slave::Flags flags = CreateSlaveFlags();
flags.acls = acls;
Owned<MasterDetector> detector = master.get()->createDetector();
auto executor = std::make_shared<v1::MockHTTPExecutor>();
v1::Resources resources =
v1::Resources::parse("cpus:0.1;mem:32;disk:32").get();
v1::ExecutorInfo executorInfo;
executorInfo.set_type(v1::ExecutorInfo::DEFAULT);
executorInfo.mutable_executor_id()->CopyFrom(v1::DEFAULT_EXECUTOR_ID);
executorInfo.mutable_resources()->CopyFrom(resources);
Owned<TestContainerizer> containerizer(
new TestContainerizer(devolve(executorInfo.executor_id()), executor));
// This pointer is passed to the agent, which will perform the cleanup.
Owned<MockSecretGenerator> mockSecretGenerator(new MockSecretGenerator());
Try<Owned<cluster::Slave>> slave = StartSlave(
detector.get(),
containerizer.get(),
mockSecretGenerator.get(),
authorizer.get(),
flags);
ASSERT_SOME(slave);
auto scheduler = std::make_shared<v1::MockHTTPScheduler>();
Future<Nothing> connected;
EXPECT_CALL(*scheduler, connected(_))
.WillOnce(FutureSatisfy(&connected));
v1::scheduler::TestMesos mesos(
master.get()->pid,
ContentType::PROTOBUF,
scheduler);
AWAIT_READY(connected);
Future<v1::scheduler::Event::Subscribed> subscribed;
EXPECT_CALL(*scheduler, subscribed(_, _))
.WillOnce(FutureArg<1>(&subscribed));
Future<v1::scheduler::Event::Offers> offers;
EXPECT_CALL(*scheduler, offers(_, _))
.WillOnce(FutureArg<1>(&offers))
.WillRepeatedly(Return()); // Ignore subsequent offers.
EXPECT_CALL(*scheduler, heartbeat(_))
.WillRepeatedly(Return()); // Ignore heartbeats.
mesos.send(v1::createCallSubscribe(v1::DEFAULT_FRAMEWORK_INFO));
AWAIT_READY(subscribed);
v1::FrameworkID frameworkId(subscribed->framework_id());
executorInfo.mutable_framework_id()->CopyFrom(frameworkId);
AWAIT_READY(offers);
ASSERT_FALSE(offers->offers().empty());
Future<v1::executor::Mesos*> executorLib;
EXPECT_CALL(*executor, connected(_))
.WillOnce(FutureArg<0>(&executorLib));
Owned<JWTSecretGenerator> jwtSecretGenerator(
new JWTSecretGenerator(DEFAULT_JWT_SECRET_KEY));
// Create a principal which contains an incorrect ContainerID.
hashmap<string, string> claims;
claims["fid"] = frameworkId.value();
claims["eid"] = v1::DEFAULT_EXECUTOR_ID.value();
claims["cid"] = id::UUID::random().toString();
Principal principal(None(), claims);
// Generate an authentication token which is signed using the correct key,
// but contains an invalid set of claims.
Future<Secret> authenticationToken =
jwtSecretGenerator->generate(principal);
AWAIT_READY(authenticationToken);
EXPECT_CALL(*mockSecretGenerator, generate(_))
.WillOnce(Return(authenticationToken.get()));
const v1::Offer& offer = offers->offers(0);
const v1::AgentID& agentId = offer.agent_id();
{
v1::TaskInfo taskInfo =
v1::createTask(agentId, resources, SLEEP_COMMAND(1000));
v1::TaskGroupInfo taskGroup;
taskGroup.add_tasks()->CopyFrom(taskInfo);
v1::scheduler::Call call;
call.mutable_framework_id()->CopyFrom(frameworkId);
call.set_type(v1::scheduler::Call::ACCEPT);
v1::scheduler::Call::Accept* accept = call.mutable_accept();
accept->add_offer_ids()->CopyFrom(offer.id());
v1::Offer::Operation* operation = accept->add_operations();
operation->set_type(v1::Offer::Operation::LAUNCH_GROUP);
v1::Offer::Operation::LaunchGroup* launchGroup =
operation->mutable_launch_group();
launchGroup->mutable_executor()->CopyFrom(executorInfo);
launchGroup->mutable_task_group()->CopyFrom(taskGroup);
mesos.send(call);
}
AWAIT_READY(executorLib);
{
v1::executor::Call call;
call.mutable_framework_id()->CopyFrom(frameworkId);
call.mutable_executor_id()->CopyFrom(v1::DEFAULT_EXECUTOR_ID);
call.set_type(v1::executor::Call::SUBSCRIBE);
call.mutable_subscribe();
executorLib.get()->send(call);
}
Future<v1::executor::Event::Error> error;
EXPECT_CALL(*executor, error(_, _))
.WillOnce(FutureArg<1>(&error));
AWAIT_READY(error);
EXPECT_EQ(
error->message(),
"Received unexpected '403 Forbidden' () for SUBSCRIBE");
}