// This test verifies that the scheduler will receive a `BadRequest` response
// when a teardown call is made with an incorrect stream ID header.
TEST_P(SchedulerHttpApiTest, TeardownWrongStreamId)
{
Try<Owned<cluster::Master>> master = StartMaster();
ASSERT_SOME(master);
// Retrieve the parameter passed as content type to this test.
const string contentType = GetParam();
process::http::Headers headers = createBasicAuthHeaders(DEFAULT_CREDENTIAL);
headers["Accept"] = contentType;
v1::FrameworkID frameworkId;
string streamId;
// Subscribe once to get a valid stream ID.
{
Call call;
call.set_type(Call::SUBSCRIBE);
Call::Subscribe* subscribe = call.mutable_subscribe();
subscribe->mutable_framework_info()->CopyFrom(v1::DEFAULT_FRAMEWORK_INFO);
Future<Response> response = process::http::streaming::post(
master.get()->pid,
"api/v1/scheduler",
headers,
serialize(call, contentType),
contentType);
AWAIT_EXPECT_RESPONSE_STATUS_EQ(OK().status, response);
AWAIT_EXPECT_RESPONSE_HEADER_EQ("chunked", "Transfer-Encoding", response);
ASSERT_EQ(Response::PIPE, response.get().type);
ASSERT_TRUE(response.get().headers.contains("Mesos-Stream-Id"));
streamId = response.get().headers.at("Mesos-Stream-Id");
Option<Pipe::Reader> reader = response.get().reader;
ASSERT_SOME(reader);
auto deserializer = lambda::bind(
&SchedulerHttpApiTest::deserialize, this, contentType, lambda::_1);
Reader<Event> responseDecoder(Decoder<Event>(deserializer), reader.get());
Future<Result<Event>> event = responseDecoder.read();
AWAIT_READY(event);
ASSERT_SOME(event.get());
// Check that the event type is subscribed and the framework ID is set.
ASSERT_EQ(Event::SUBSCRIBED, event.get().get().type());
EXPECT_NE("", event.get().get().subscribed().framework_id().value());
frameworkId = event.get().get().subscribed().framework_id();
}
// Subscribe again to invalidate the first stream ID and acquire another one.
{
Call call;
call.set_type(Call::SUBSCRIBE);
Call::Subscribe* subscribe = call.mutable_subscribe();
subscribe->mutable_framework_info()->CopyFrom(v1::DEFAULT_FRAMEWORK_INFO);
// Set the framework ID in the subscribe call.
call.mutable_framework_id()->CopyFrom(frameworkId);
subscribe->mutable_framework_info()->mutable_id()->CopyFrom(frameworkId);
Future<Response> response = process::http::streaming::post(
master.get()->pid,
"api/v1/scheduler",
headers,
serialize(call, contentType),
contentType);
AWAIT_EXPECT_RESPONSE_STATUS_EQ(OK().status, response);
AWAIT_EXPECT_RESPONSE_HEADER_EQ("chunked", "Transfer-Encoding", response);
ASSERT_EQ(Response::PIPE, response.get().type);
ASSERT_TRUE(response.get().headers.contains("Mesos-Stream-Id"));
// Make sure that the new stream ID is different.
ASSERT_NE(streamId, response.get().headers.at("Mesos-Stream-Id"));
Option<Pipe::Reader> reader = response.get().reader;
ASSERT_SOME(reader);
auto deserializer = lambda::bind(
&SchedulerHttpApiTest::deserialize, this, contentType, lambda::_1);
Reader<Event> responseDecoder(Decoder<Event>(deserializer), reader.get());
Future<Result<Event>> event = responseDecoder.read();
AWAIT_READY(event);
ASSERT_SOME(event.get());
ASSERT_EQ(Event::SUBSCRIBED, event.get().get().type());
EXPECT_NE("", event.get().get().subscribed().framework_id().value());
}
{
Call call;
call.set_type(Call::TEARDOWN);
call.mutable_framework_id()->CopyFrom(frameworkId);
// Send the first (now incorrect) stream ID with the teardown call.
headers["Mesos-Stream-Id"] = streamId;
Future<Response> response = process::http::streaming::post(
master.get()->pid,
"api/v1/scheduler",
headers,
serialize(call, contentType),
contentType);
AWAIT_EXPECT_RESPONSE_STATUS_EQ(BadRequest().status, response);
}
}
// This test verifies that a framework removal that comes before
// '_launchTasks()' is called results in recovery of resources.
TEST_F(MasterAuthorizationTest, FrameworkRemoved)
{
MockAuthorizer authorizer;
Try<PID<Master> > master = StartMaster(&authorizer);
ASSERT_SOME(master);
MockExecutor exec(DEFAULT_EXECUTOR_ID);
Try<PID<Slave> > slave = StartSlave(&exec);
ASSERT_SOME(slave);
MockScheduler sched;
MesosSchedulerDriver driver(
&sched, DEFAULT_FRAMEWORK_INFO, master.get(), DEFAULT_CREDENTIAL);
EXPECT_CALL(sched, registered(&driver, _, _))
.Times(1);
Future<vector<Offer> > offers;
EXPECT_CALL(sched, resourceOffers(&driver, _))
.WillOnce(FutureArg<1>(&offers))
.WillRepeatedly(Return()); // Ignore subsequent offers.
driver.start();
AWAIT_READY(offers);
EXPECT_NE(0u, offers.get().size());
TaskInfo task = createTask(offers.get()[0], "", DEFAULT_EXECUTOR_ID);
vector<TaskInfo> tasks;
tasks.push_back(task);
// Return a pending future from authorizer.
Future<Nothing> future;
Promise<bool> promise;
EXPECT_CALL(authorizer, authorize(An<const mesos::ACL::RunTasks&>()))
.WillOnce(DoAll(FutureSatisfy(&future),
Return(promise.future())));
driver.launchTasks(offers.get()[0].id(), tasks);
// Wait until authorization is in progress.
AWAIT_READY(future);
Future<Nothing> frameworkRemoved =
FUTURE_DISPATCH(_, &AllocatorProcess::frameworkRemoved);
// Now stop the framework.
driver.stop();
driver.join();
AWAIT_READY(frameworkRemoved);
Future<Nothing> resourcesRecovered =
FUTURE_DISPATCH(_, &AllocatorProcess::resourcesRecovered);
// Now complete authorization.
promise.set(true);
// No task launch should happen resulting in all resources being
// returned to the allocator.
AWAIT_READY(resourcesRecovered);
Shutdown(); // Must shutdown before 'containerizer' gets deallocated.
}
// This test verifies that an authorized task launch is successful.
TEST_F(MasterAuthorizationTest, AuthorizedTask)
{
// Setup ACLs so that the framework can launch tasks as "foo".
ACLs acls;
mesos::ACL::RunTasks* acl = acls.add_run_tasks();
acl->mutable_principals()->add_values(DEFAULT_FRAMEWORK_INFO.principal());
acl->mutable_users()->add_values("foo");
master::Flags flags = CreateMasterFlags();
flags.acls = acls;
Try<PID<Master> > master = StartMaster(flags);
ASSERT_SOME(master);
// Create an authorized executor.
ExecutorInfo executor; // Bug in gcc 4.1.*, must assign on next line.
executor = CREATE_EXECUTOR_INFO("test-executor", "exit 1");
executor.mutable_command()->set_user("foo");
MockExecutor exec(executor.executor_id());
Try<PID<Slave> > slave = StartSlave(&exec);
ASSERT_SOME(slave);
MockScheduler sched;
MesosSchedulerDriver driver(
&sched, DEFAULT_FRAMEWORK_INFO, master.get(), DEFAULT_CREDENTIAL);
EXPECT_CALL(sched, registered(&driver, _, _))
.Times(1);
Future<vector<Offer> > offers;
EXPECT_CALL(sched, resourceOffers(&driver, _))
.WillOnce(FutureArg<1>(&offers))
.WillRepeatedly(Return()); // Ignore subsequent offers.
driver.start();
AWAIT_READY(offers);
EXPECT_NE(0u, offers.get().size());
// Create an authorized task.
TaskInfo task;
task.set_name("test");
task.mutable_task_id()->set_value("1");
task.mutable_slave_id()->MergeFrom(offers.get()[0].slave_id());
task.mutable_resources()->MergeFrom(offers.get()[0].resources());
task.mutable_executor()->MergeFrom(executor);
vector<TaskInfo> tasks;
tasks.push_back(task);
EXPECT_CALL(exec, registered(_, _, _, _))
.Times(1);
EXPECT_CALL(exec, launchTask(_, _))
.WillOnce(SendStatusUpdateFromTask(TASK_RUNNING));
Future<TaskStatus> status;
EXPECT_CALL(sched, statusUpdate(&driver, _))
.WillOnce(FutureArg<1>(&status));
driver.launchTasks(offers.get()[0].id(), tasks);
AWAIT_READY(status);
EXPECT_EQ(TASK_RUNNING, status.get().state());
EXPECT_CALL(exec, shutdown(_))
.Times(AtMost(1));
driver.stop();
driver.join();
Shutdown(); // Must shutdown before 'containerizer' gets deallocated.
}
// This test ensures that an implicit reconciliation request results
// in updates for all non-terminal tasks known to the master.
TEST_F(ReconciliationTest, ImplicitNonTerminalTask)
{
Try<PID<Master> > master = StartMaster();
ASSERT_SOME(master);
MockExecutor exec(DEFAULT_EXECUTOR_ID);
TestContainerizer containerizer(&exec);
Try<PID<Slave> > slave = StartSlave(&containerizer);
ASSERT_SOME(slave);
// Launch a framework and get a task running.
MockScheduler sched;
MesosSchedulerDriver driver(
&sched, DEFAULT_FRAMEWORK_INFO, master.get(), DEFAULT_CREDENTIAL);
Future<FrameworkID> frameworkId;
EXPECT_CALL(sched, registered(&driver, _, _))
.WillOnce(FutureArg<1>(&frameworkId));
EXPECT_CALL(sched, resourceOffers(&driver, _))
.WillOnce(LaunchTasks(DEFAULT_EXECUTOR_INFO, 1, 1, 512, "*"))
.WillRepeatedly(Return()); // Ignore subsequent offers.
EXPECT_CALL(exec, registered(_, _, _, _));
EXPECT_CALL(exec, launchTask(_, _))
.WillOnce(SendStatusUpdateFromTask(TASK_RUNNING));
Future<TaskStatus> update;
EXPECT_CALL(sched, statusUpdate(&driver, _))
.WillOnce(FutureArg<1>(&update));
driver.start();
// Wait until the framework is registered.
AWAIT_READY(frameworkId);
AWAIT_READY(update);
EXPECT_EQ(TASK_RUNNING, update.get().state());
EXPECT_TRUE(update.get().has_slave_id());
// When making an implicit reconciliation request, the non-terminal
// task should be sent back.
Future<TaskStatus> update2;
EXPECT_CALL(sched, statusUpdate(&driver, _))
.WillOnce(FutureArg<1>(&update2));
vector<TaskStatus> statuses;
driver.reconcileTasks(statuses);
AWAIT_READY(update2);
EXPECT_EQ(TASK_RUNNING, update2.get().state());
EXPECT_CALL(exec, shutdown(_))
.Times(AtMost(1));
driver.stop();
driver.join();
Shutdown(); // Must shutdown before 'containerizer' gets deallocated.
}
// This test ensures that reconciliation requests for tasks that are
// pending are exposed in reconciliation.
TEST_F(ReconciliationTest, PendingTask)
{
MockAuthorizer authorizer;
Try<PID<Master> > master = StartMaster(&authorizer);
ASSERT_SOME(master);
MockExecutor exec(DEFAULT_EXECUTOR_ID);
Future<SlaveRegisteredMessage> slaveRegisteredMessage =
FUTURE_PROTOBUF(SlaveRegisteredMessage(), _, _);
Try<PID<Slave> > slave = StartSlave();
ASSERT_SOME(slave);
// Wait for the slave to register and get the slave id.
AWAIT_READY(slaveRegisteredMessage);
const SlaveID slaveId = slaveRegisteredMessage.get().slave_id();
MockScheduler sched;
MesosSchedulerDriver driver(
&sched, DEFAULT_FRAMEWORK_INFO, master.get(), DEFAULT_CREDENTIAL);
EXPECT_CALL(sched, registered(&driver, _, _))
.Times(1);
Future<vector<Offer> > offers;
EXPECT_CALL(sched, resourceOffers(&driver, _))
.WillOnce(FutureArg<1>(&offers))
.WillRepeatedly(Return()); // Ignore subsequent offers.
driver.start();
AWAIT_READY(offers);
EXPECT_NE(0u, offers.get().size());
// Return a pending future from authorizer.
Future<Nothing> authorize;
Promise<bool> promise;
EXPECT_CALL(authorizer, authorize(An<const mesos::ACL::RunTask&>()))
.WillOnce(DoAll(FutureSatisfy(&authorize),
Return(promise.future())));
TaskInfo task = createTask(offers.get()[0], "", DEFAULT_EXECUTOR_ID);
vector<TaskInfo> tasks;
tasks.push_back(task);
driver.launchTasks(offers.get()[0].id(), tasks);
// Wait until authorization is in progress.
AWAIT_READY(authorize);
// First send an implicit reconciliation request for this task.
Future<TaskStatus> update;
EXPECT_CALL(sched, statusUpdate(&driver, _))
.WillOnce(FutureArg<1>(&update));
vector<TaskStatus> statuses;
driver.reconcileTasks(statuses);
AWAIT_READY(update);
EXPECT_EQ(TASK_STAGING, update.get().state());
EXPECT_TRUE(update.get().has_slave_id());
// Now send an explicit reconciliation request for this task.
Future<TaskStatus> update2;
EXPECT_CALL(sched, statusUpdate(&driver, _))
.WillOnce(FutureArg<1>(&update2));
TaskStatus status;
status.mutable_task_id()->CopyFrom(task.task_id());
status.mutable_slave_id()->CopyFrom(slaveId);
status.set_state(TASK_STAGING);
statuses.push_back(status);
driver.reconcileTasks(statuses);
AWAIT_READY(update2);
EXPECT_EQ(TASK_STAGING, update2.get().state());
EXPECT_TRUE(update2.get().has_slave_id());
driver.stop();
driver.join();
Shutdown(); // Must shutdown before 'containerizer' gets deallocated.
}
// Ensures that the driver can handle the SUBSCRIBED event
// after a scheduler failover.
TEST_F(SchedulerDriverEventTest, SubscribedSchedulerFailover)
{
Try<PID<Master>> master = StartMaster();
ASSERT_SOME(master);
FrameworkInfo frameworkInfo = DEFAULT_FRAMEWORK_INFO;
frameworkInfo.set_failover_timeout(Weeks(2).secs());
// Make sure the initial registration calls 'registered'.
MockScheduler sched;
MesosSchedulerDriver driver(
&sched, frameworkInfo, master.get(), DEFAULT_CREDENTIAL);
// Intercept the registration message, send a SUBSCRIBED instead.
Future<Message> frameworkRegisteredMessage =
DROP_MESSAGE(Eq(FrameworkRegisteredMessage().GetTypeName()), _, _);
// Ensure that there will be no (re-)registration retries
// from the scheduler driver.
Clock::pause();
driver.start();
AWAIT_READY(frameworkRegisteredMessage);
UPID frameworkPid = frameworkRegisteredMessage.get().to;
FrameworkRegisteredMessage message;
ASSERT_TRUE(message.ParseFromString(frameworkRegisteredMessage.get().body));
FrameworkID frameworkId = message.framework_id();
frameworkInfo.mutable_id()->CopyFrom(frameworkId);
Event event;
event.set_type(Event::SUBSCRIBED);
event.mutable_subscribed()->mutable_framework_id()->CopyFrom(frameworkId);
Future<Nothing> registered;
EXPECT_CALL(sched, registered(&driver, frameworkId, _))
.WillOnce(FutureSatisfy(®istered));
process::post(master.get(), frameworkPid, event);
AWAIT_READY(registered);
// Fail over the scheduler and expect a 'registered' call.
driver.stop(true);
MockScheduler sched2;
MesosSchedulerDriver driver2(
&sched2, frameworkInfo, master.get(), DEFAULT_CREDENTIAL);
frameworkRegisteredMessage =
DROP_MESSAGE(Eq(FrameworkRegisteredMessage().GetTypeName()), _, _);
driver2.start();
AWAIT_READY(frameworkRegisteredMessage);
UPID frameworkPid2 = frameworkRegisteredMessage.get().to;
Future<Nothing> registered2;
EXPECT_CALL(sched2, registered(&driver2, frameworkId, _))
.WillOnce(FutureSatisfy(®istered2));
process::post(master.get(), frameworkPid2, event);
AWAIT_READY(registered2);
}
// Ensures the scheduler driver can handle the UPDATE event.
TEST_F(SchedulerDriverEventTest, Update)
{
Try<PID<Master>> master = StartMaster();
ASSERT_SOME(master);
MockScheduler sched;
MesosSchedulerDriver driver(
&sched, DEFAULT_FRAMEWORK_INFO, master.get(), DEFAULT_CREDENTIAL);
EXPECT_CALL(sched, registered(&driver, _, _));
Future<Message> frameworkRegisteredMessage =
FUTURE_MESSAGE(Eq(FrameworkRegisteredMessage().GetTypeName()), _, _);
driver.start();
AWAIT_READY(frameworkRegisteredMessage);
UPID frameworkPid = frameworkRegisteredMessage.get().to;
FrameworkRegisteredMessage message;
ASSERT_TRUE(message.ParseFromString(frameworkRegisteredMessage.get().body));
FrameworkID frameworkId = message.framework_id();
SlaveID slaveId;
slaveId.set_value("S");
TaskID taskId;
taskId.set_value("T");
ExecutorID executorId;
executorId.set_value("E");
// Generate an update that needs no acknowledgement.
Event event;
event.set_type(Event::UPDATE);
event.mutable_update()->mutable_status()->CopyFrom(
protobuf::createStatusUpdate(
frameworkId,
slaveId,
taskId,
TASK_RUNNING,
TaskStatus::SOURCE_MASTER,
None(),
"message",
None(),
executorId).status());
Future<Nothing> statusUpdate;
Future<Nothing> statusUpdate2;
EXPECT_CALL(sched, statusUpdate(&driver, event.update().status()))
.WillOnce(FutureSatisfy(&statusUpdate))
.WillOnce(FutureSatisfy(&statusUpdate2));
process::post(master.get(), frameworkPid, event);
AWAIT_READY(statusUpdate);
// Generate an update that requires acknowledgement.
event.mutable_update()->mutable_status()->set_uuid(UUID::random().toBytes());
Future<mesos::scheduler::Call> acknowledgement = DROP_CALL(
mesos::scheduler::Call(), mesos::scheduler::Call::ACKNOWLEDGE, _, _);
process::post(master.get(), frameworkPid, event);
AWAIT_READY(statusUpdate2);
AWAIT_READY(acknowledgement);
}
TEST_F(MemoryPressureMesosTest, CGROUPS_ROOT_Statistics)
{
Try<Owned<cluster::Master>> master = StartMaster();
ASSERT_SOME(master);
slave::Flags flags = CreateSlaveFlags();
// We only care about memory cgroup for this test.
flags.isolation = "cgroups/mem";
flags.agent_subsystems = None();
Fetcher fetcher;
Try<MesosContainerizer*> _containerizer =
MesosContainerizer::create(flags, true, &fetcher);
ASSERT_SOME(_containerizer);
Owned<MesosContainerizer> containerizer(_containerizer.get());
Owned<MasterDetector> detector = master.get()->createDetector();
Try<Owned<cluster::Slave>> slave =
StartSlave(detector.get(), containerizer.get(), flags);
ASSERT_SOME(slave);
MockScheduler sched;
MesosSchedulerDriver driver(
&sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL);
EXPECT_CALL(sched, registered(_, _, _));
Future<vector<Offer>> offers;
EXPECT_CALL(sched, resourceOffers(_, _))
.WillOnce(FutureArg<1>(&offers))
.WillRepeatedly(Return()); // Ignore subsequent offers.
driver.start();
AWAIT_READY(offers);
EXPECT_NE(0u, offers.get().size());
Offer offer = offers.get()[0];
// Run a task that triggers memory pressure event. We request 1G
// disk because we are going to write a 512 MB file repeatedly.
TaskInfo task = createTask(
offer.slave_id(),
Resources::parse("cpus:1;mem:256;disk:1024").get(),
"while true; do dd count=512 bs=1M if=/dev/zero of=./temp; done");
Future<TaskStatus> running;
Future<TaskStatus> killed;
EXPECT_CALL(sched, statusUpdate(&driver, _))
.WillOnce(FutureArg<1>(&running))
.WillOnce(FutureArg<1>(&killed))
.WillRepeatedly(Return()); // Ignore subsequent updates.
driver.launchTasks(offer.id(), {task});
AWAIT_READY(running);
EXPECT_EQ(task.task_id(), running.get().task_id());
EXPECT_EQ(TASK_RUNNING, running.get().state());
Future<hashset<ContainerID>> containers = containerizer->containers();
AWAIT_READY(containers);
ASSERT_EQ(1u, containers.get().size());
ContainerID containerId = *(containers.get().begin());
// Wait a while for some memory pressure events to occur.
Duration waited = Duration::zero();
do {
Future<ResourceStatistics> usage = containerizer->usage(containerId);
AWAIT_READY(usage);
if (usage.get().mem_low_pressure_counter() > 0) {
// We will check the correctness of the memory pressure counters
// later, because the memory-hammering task is still active
// and potentially incrementing these counters.
break;
}
os::sleep(Milliseconds(100));
waited += Milliseconds(100);
} while (waited < Seconds(5));
EXPECT_LE(waited, Seconds(5));
// Pause the clock to ensure that the reaper doesn't reap the exited
// command executor and inform the containerizer/slave.
Clock::pause();
Clock::settle();
// Stop the memory-hammering task.
driver.killTask(task.task_id());
AWAIT_READY_FOR(killed, Seconds(120));
EXPECT_EQ(task.task_id(), killed->task_id());
EXPECT_EQ(TASK_KILLED, killed->state());
// Now check the correctness of the memory pressure counters.
Future<ResourceStatistics> usage = containerizer->usage(containerId);
AWAIT_READY(usage);
EXPECT_GE(usage.get().mem_low_pressure_counter(),
usage.get().mem_medium_pressure_counter());
EXPECT_GE(usage.get().mem_medium_pressure_counter(),
usage.get().mem_critical_pressure_counter());
Clock::resume();
driver.stop();
driver.join();
}
TYPED_TEST(CpuIsolatorTest, SystemCpuUsage)
{
slave::Flags flags;
Try<Isolator*> isolator = TypeParam::create(flags);
CHECK_SOME(isolator);
// A PosixLauncher is sufficient even when testing a cgroups isolator.
Try<Launcher*> launcher = PosixLauncher::create(flags);
ExecutorInfo executorInfo;
executorInfo.mutable_resources()->CopyFrom(
Resources::parse("cpus:1.0").get());
ContainerID containerId;
containerId.set_value(UUID::random().toString());
// Use a relative temporary directory so it gets cleaned up
// automatically with the test.
Try<string> dir = os::mkdtemp(path::join(os::getcwd(), "XXXXXX"));
ASSERT_SOME(dir);
ContainerConfig containerConfig;
containerConfig.mutable_executor_info()->CopyFrom(executorInfo);
containerConfig.set_directory(dir.get());
AWAIT_READY(isolator.get()->prepare(
containerId,
containerConfig));
const string& file = path::join(dir.get(), "mesos_isolator_test_ready");
// Generating random numbers is done by the kernel and will max out a single
// core and run almost exclusively in the kernel, i.e., system time.
string command = "cat /dev/urandom > /dev/null & "
"touch " + file + "; " // Signals the command is running.
"sleep 60";
int pipes[2];
ASSERT_NE(-1, ::pipe(pipes));
vector<string> argv(3);
argv[0] = "sh";
argv[1] = "-c";
argv[2] = command;
Try<pid_t> pid = launcher.get()->fork(
containerId,
"sh",
argv,
Subprocess::FD(STDIN_FILENO),
Subprocess::FD(STDOUT_FILENO),
Subprocess::FD(STDERR_FILENO),
None(),
None(),
lambda::bind(&childSetup, pipes),
None());
ASSERT_SOME(pid);
// Reap the forked child.
Future<Option<int> > status = process::reap(pid.get());
// Continue in the parent.
ASSERT_SOME(os::close(pipes[0]));
// Isolate the forked child.
AWAIT_READY(isolator.get()->isolate(containerId, pid.get()));
// Now signal the child to continue.
char dummy;
ASSERT_LT(0, ::write(pipes[1], &dummy, sizeof(dummy)));
ASSERT_SOME(os::close(pipes[1]));
// Wait for the command to start.
while (!os::exists(file));
// Wait up to 1 second for the child process to induce 1/8 of a second of
// system cpu time.
ResourceStatistics statistics;
Duration waited = Duration::zero();
do {
Future<ResourceStatistics> usage = isolator.get()->usage(containerId);
AWAIT_READY(usage);
statistics = usage.get();
// If we meet our usage expectations, we're done!
if (statistics.cpus_system_time_secs() >= 0.125) {
break;
}
os::sleep(Milliseconds(200));
waited += Milliseconds(200);
} while (waited < Seconds(1));
EXPECT_LE(0.125, statistics.cpus_system_time_secs());
// Ensure all processes are killed.
AWAIT_READY(launcher.get()->destroy(containerId));
// Make sure the child was reaped.
AWAIT_READY(status);
// Let the isolator clean up.
AWAIT_READY(isolator.get()->cleanup(containerId));
delete isolator.get();
delete launcher.get();
}
// This test ensures that the command executor sends TASK_KILLING
// to frameworks that support the capability.
TEST_F(CommandExecutorTest, TaskKillingCapability)
{
Try<Owned<cluster::Master>> master = StartMaster();
ASSERT_SOME(master);
Owned<MasterDetector> detector = master.get()->createDetector();
Try<Owned<cluster::Slave>> slave = StartSlave(detector.get());
ASSERT_SOME(slave);
// Start the framework with the task killing capability.
FrameworkInfo::Capability capability;
capability.set_type(FrameworkInfo::Capability::TASK_KILLING_STATE);
FrameworkInfo frameworkInfo = DEFAULT_FRAMEWORK_INFO;
frameworkInfo.add_capabilities()->CopyFrom(capability);
MockScheduler sched;
MesosSchedulerDriver driver(
&sched, frameworkInfo, master.get()->pid, DEFAULT_CREDENTIAL);
EXPECT_CALL(sched, registered(&driver, _, _));
Future<vector<Offer>> offers;
EXPECT_CALL(sched, resourceOffers(&driver, _))
.WillOnce(FutureArg<1>(&offers))
.WillRepeatedly(Return()); // Ignore subsequent offers.
driver.start();
AWAIT_READY(offers);
EXPECT_EQ(1u, offers->size());
// Launch a task with the command executor.
TaskInfo task = createTask(
offers->front().slave_id(),
offers->front().resources(),
"sleep 1000");
Future<TaskStatus> statusRunning;
EXPECT_CALL(sched, statusUpdate(_, _))
.WillOnce(FutureArg<1>(&statusRunning));
driver.launchTasks(offers->front().id(), {task});
AWAIT_READY(statusRunning);
EXPECT_EQ(TASK_RUNNING, statusRunning->state());
Future<TaskStatus> statusKilling, statusKilled;
EXPECT_CALL(sched, statusUpdate(_, _))
.WillOnce(FutureArg<1>(&statusKilling))
.WillOnce(FutureArg<1>(&statusKilled));
driver.killTask(task.task_id());
AWAIT_READY(statusKilling);
EXPECT_EQ(TASK_KILLING, statusKilling->state());
AWAIT_READY(statusKilled);
EXPECT_EQ(TASK_KILLED, statusKilled->state());
driver.stop();
driver.join();
}
TEST(HTTP, Endpoints)
{
ASSERT_TRUE(GTEST_IS_THREADSAFE);
HttpProcess process;
spawn(process);
// First hit '/body' (using explicit sockets and HTTP/1.0).
Try<Socket> create = Socket::create();
ASSERT_SOME(create);
Socket socket = create.get();
AWAIT_READY(socket.connect(process.self().address));
std::ostringstream out;
out << "GET /" << process.self().id << "/body"
<< " HTTP/1.0\r\n"
<< "Connection: Keep-Alive\r\n"
<< "\r\n";
const string& data = out.str();
EXPECT_CALL(process, body(_))
.WillOnce(Return(http::OK()));
AWAIT_READY(socket.send(data));
string response = "HTTP/1.1 200 OK";
AWAIT_EXPECT_EQ(response, socket.recv(response.size()));
// Now hit '/pipe' (by using http::get).
int pipes[2];
ASSERT_NE(-1, ::pipe(pipes));
http::OK ok;
ok.type = http::Response::PIPE;
ok.pipe = pipes[0];
Future<Nothing> pipe;
EXPECT_CALL(process, pipe(_))
.WillOnce(DoAll(FutureSatisfy(&pipe),
Return(ok)));
Future<http::Response> future = http::get(process.self(), "pipe");
AWAIT_READY(pipe);
ASSERT_SOME(os::write(pipes[1], "Hello World\n"));
ASSERT_SOME(os::close(pipes[1]));
AWAIT_READY(future);
EXPECT_EQ(http::statuses[200], future.get().status);
EXPECT_SOME_EQ("chunked", future.get().headers.get("Transfer-Encoding"));
EXPECT_EQ("Hello World\n", future.get().body);
terminate(process);
wait(process);
}
TEST_F(ZooKeeperTest, LeaderContender)
{
Seconds timeout(10);
Group group(server->connectString(), timeout, "/test/");
Owned<LeaderContender> contender(
new LeaderContender(&group, "candidate 1", master::MASTER_INFO_LABEL));
// Calling withdraw before contending returns 'false' because there
// is nothing to withdraw.
Future<bool> withdrawn = contender->withdraw();
AWAIT_READY(withdrawn);
EXPECT_FALSE(withdrawn.get());
contender->contend();
// Immediately withdrawing after contending leads to delayed
// cancellation.
withdrawn = contender->withdraw();
AWAIT_READY(withdrawn);
EXPECT_TRUE(withdrawn.get());
// Normal workflow.
contender = Owned<LeaderContender>(
new LeaderContender(&group, "candidate 1", master::MASTER_INFO_LABEL));
Future<Future<Nothing> > candidated = contender->contend();
AWAIT_READY(candidated);
Future<Nothing> lostCandidacy = candidated.get();
EXPECT_TRUE(lostCandidacy.isPending());
// Expire the Group session while we are watching for updates from
// the contender and the candidacy will be lost.
Future<Option<int64_t> > session = group.session();
AWAIT_READY(session);
ASSERT_SOME(session.get());
Future<Nothing> connected = FUTURE_DISPATCH(
group.process->self(),
&GroupProcess::connected);
server->expireSession(session.get().get());
AWAIT_READY(lostCandidacy);
// Withdraw directly returns because candidacy is lost and there
// is nothing to cancel.
withdrawn = contender->withdraw();
AWAIT_READY(withdrawn);
EXPECT_FALSE(withdrawn.get());
// Contend again.
contender = Owned<LeaderContender>(
new LeaderContender(&group, "candidate 1", master::MASTER_INFO_LABEL));
candidated = contender->contend();
AWAIT_READY(connected);
session = group.session();
AWAIT_READY(session);
ASSERT_SOME(session.get());
server->expireSession(session.get().get());
Clock::pause();
// The retry timeout.
Clock::advance(GroupProcess::RETRY_INTERVAL);
Clock::settle();
Clock::resume();
// The contender weathered the expiration and succeeded in a retry.
AWAIT_READY(candidated);
withdrawn = contender->withdraw();
AWAIT_READY(withdrawn);
// Contend (3) and shutdown the network this time.
contender = Owned<LeaderContender>(
new LeaderContender(&group, "candidate 1", master::MASTER_INFO_LABEL));
candidated = contender->contend();
AWAIT_READY(candidated);
lostCandidacy = candidated.get();
Future<Nothing> reconnecting = FUTURE_DISPATCH(
group.process->self(),
&GroupProcess::reconnecting);
server->shutdownNetwork();
AWAIT_READY(reconnecting);
Clock::pause();
// Settle to make sure 'reconnecting()' schedules the timeout
// before we advance.
Clock::settle();
Clock::advance(timeout);
// Server failure results in candidacy loss.
AWAIT_READY(lostCandidacy);
Clock::resume();
server->startNetwork();
// Contend again (4).
contender = Owned<LeaderContender>(
new LeaderContender(&group, "candidate 1", master::MASTER_INFO_LABEL));
candidated = contender->contend();
AWAIT_READY(candidated);
}
TEST_F(ZooKeeperTest, LeaderDetector)
{
Group group(server->connectString(), NO_TIMEOUT, "/test/");
// Initialize two members.
Future<Group::Membership> membership1 =
group.join("member 1");
AWAIT_READY(membership1);
Future<Group::Membership> membership2 =
group.join("member 2");
AWAIT_READY(membership2);
LeaderDetector detector(&group);
// Detect the leader.
Future<Option<Group::Membership> > leader =
detector.detect(None());
AWAIT_READY(leader);
ASSERT_SOME_EQ(membership1.get(), leader.get());
// Detect next leader change.
leader = detector.detect(leader.get());
EXPECT_TRUE(leader.isPending());
// Leader doesn't change after cancelling the follower.
Future<bool> cancellation = group.cancel(membership2.get());
AWAIT_READY(cancellation);
EXPECT_TRUE(cancellation.get());
EXPECT_TRUE(leader.isPending());
// Join member 2 back.
membership2 = group.join("member 2");
AWAIT_READY(membership2);
EXPECT_TRUE(leader.isPending());
// Cancelling the incumbent leader allows member 2 to be elected.
cancellation = group.cancel(membership1.get());
AWAIT_READY(cancellation);
EXPECT_TRUE(cancellation.get());
AWAIT_READY(leader);
EXPECT_SOME_EQ(membership2.get(), leader.get());
// Cancelling the only member results in no leader elected.
leader = detector.detect(leader.get().get());
EXPECT_TRUE(leader.isPending());
cancellation = group.cancel(membership2.get());
AWAIT_READY(cancellation);
EXPECT_TRUE(cancellation.get());
AWAIT_READY(leader);
ASSERT_TRUE(leader.get().isNone());
}
// This test verifies that the scheduler will receive a `BadRequest` response
// when it tries to acknowledge a status update with a malformed UUID.
TEST_P(SchedulerHttpApiTest, MalformedUUID)
{
Try<Owned<cluster::Master>> master = StartMaster();
ASSERT_SOME(master);
// Retrieve the parameter passed as content type to this test.
const string contentType = GetParam();
process::http::Headers headers = createBasicAuthHeaders(DEFAULT_CREDENTIAL);
headers["Accept"] = contentType;
v1::FrameworkID frameworkId;
string streamId;
// Subscribe once to get a valid stream ID.
{
Call call;
call.set_type(Call::SUBSCRIBE);
Call::Subscribe* subscribe = call.mutable_subscribe();
subscribe->mutable_framework_info()->CopyFrom(v1::DEFAULT_FRAMEWORK_INFO);
Future<Response> response = process::http::streaming::post(
master.get()->pid,
"api/v1/scheduler",
headers,
serialize(call, contentType),
contentType);
AWAIT_EXPECT_RESPONSE_STATUS_EQ(OK().status, response);
ASSERT_EQ(Response::PIPE, response.get().type);
ASSERT_TRUE(response.get().headers.contains("Mesos-Stream-Id"));
streamId = response.get().headers.at("Mesos-Stream-Id");
Option<Pipe::Reader> reader = response.get().reader;
ASSERT_SOME(reader);
auto deserializer = lambda::bind(
&SchedulerHttpApiTest::deserialize, this, contentType, lambda::_1);
Reader<Event> responseDecoder(Decoder<Event>(deserializer), reader.get());
Future<Result<Event>> event = responseDecoder.read();
AWAIT_READY(event);
ASSERT_SOME(event.get());
// Check that the event type is subscribed and the framework ID is set.
ASSERT_EQ(Event::SUBSCRIBED, event.get().get().type());
frameworkId = event.get().get().subscribed().framework_id();
EXPECT_NE("", frameworkId.value());
}
// Make an acknowledge call with a malformed UUID. This should result in a
// `BadResponse`.
{
headers["Mesos-Stream-Id"] = streamId;
Call call;
call.set_type(Call::ACKNOWLEDGE);
// Set the framework ID in the subscribe call.
call.mutable_framework_id()->CopyFrom(frameworkId);
Call::Acknowledge* acknowledge = call.mutable_acknowledge();
acknowledge->mutable_task_id()->set_value("task-id");
acknowledge->mutable_agent_id()->set_value("agent-id");
// Set a malformed uuid.
acknowledge->set_uuid("bad-uuid");
Future<Response> response = process::http::post(
master.get()->pid,
"api/v1/scheduler",
headers,
serialize(call, contentType),
contentType);
AWAIT_EXPECT_RESPONSE_STATUS_EQ(BadRequest().status, response);
AWAIT_EXPECT_RESPONSE_BODY_EQ(
"Failed to validate scheduler::Call: Not a valid UUID", response);
}
}
// This test verifies that a framework attempting to subscribe
// after its failover timeout has elapsed is disallowed.
TEST_F(HttpFaultToleranceTest, SchedulerSubscribeAfterFailoverTimeout)
{
master::Flags flags = CreateMasterFlags();
flags.authenticate_frameworks = false;
v1::FrameworkInfo frameworkInfo = v1::DEFAULT_FRAMEWORK_INFO;
frameworkInfo.set_failover_timeout(Weeks(2).secs());
Try<Owned<cluster::Master>> master = StartMaster(flags);
ASSERT_SOME(master);
Future<Nothing> deactivateFramework = FUTURE_DISPATCH(
_, &master::allocator::MesosAllocatorProcess::deactivateFramework);
v1::FrameworkID frameworkId;
ContentType contentType = ContentType::PROTOBUF;
// Launch the first (i.e., failing) scheduler and wait until it receives
// a `SUBSCRIBED` event to launch the second (i.e., failover) scheduler.
{
auto scheduler = std::make_shared<v1::MockHTTPScheduler>();
Future<Nothing> connected;
EXPECT_CALL(*scheduler, connected(_))
.WillOnce(FutureSatisfy(&connected));
v1::scheduler::TestMesos schedulerLibrary(
master.get()->pid,
contentType,
scheduler);
AWAIT_READY(connected);
Future<Event::Subscribed> subscribed;
EXPECT_CALL(*scheduler, subscribed(_, _))
.WillOnce(FutureArg<1>(&subscribed));
EXPECT_CALL(*scheduler, heartbeat(_))
.WillRepeatedly(Return()); // Ignore heartbeats.
{
Call call;
call.set_type(Call::SUBSCRIBE);
Call::Subscribe* subscribe = call.mutable_subscribe();
subscribe->mutable_framework_info()->CopyFrom(frameworkInfo);
schedulerLibrary.send(call);
}
AWAIT_READY(subscribed);
frameworkId = subscribed->framework_id();
}
// Wait until master schedules the framework for removal.
AWAIT_READY(deactivateFramework);
// Simulate framework failover timeout.
Clock::pause();
Clock::settle();
Try<Duration> failoverTimeout =
Duration::create(frameworkInfo.failover_timeout());
ASSERT_SOME(failoverTimeout);
Future<Nothing> frameworkFailoverTimeout =
FUTURE_DISPATCH(_, &Master::frameworkFailoverTimeout);
Clock::advance(failoverTimeout.get());
Clock::resume();
// Wait until master actually marks the framework as completed.
AWAIT_READY(frameworkFailoverTimeout);
// Now launch the second (i.e., failover) scheduler using the
// framework id recorded from the first scheduler.
{
auto scheduler = std::make_shared<v1::MockHTTPScheduler>();
Future<Nothing> connected;
EXPECT_CALL(*scheduler, connected(_))
.WillOnce(FutureSatisfy(&connected))
.WillRepeatedly(Return()); // Ignore future invocations.
v1::scheduler::TestMesos schedulerLibrary(
master.get()->pid,
contentType,
scheduler);
AWAIT_READY(connected);
// Framework should get `Error` event because the framework with this id
// is marked as completed.
Future<Nothing> error;
EXPECT_CALL(*scheduler, error(_, _))
.WillOnce(FutureSatisfy(&error));
EXPECT_CALL(*scheduler, disconnected(_))
.Times(AtMost(1));
{
Call call;
call.mutable_framework_id()->CopyFrom(frameworkId);
call.set_type(Call::SUBSCRIBE);
Call::Subscribe* subscribe = call.mutable_subscribe();
subscribe->mutable_framework_info()->CopyFrom(v1::DEFAULT_FRAMEWORK_INFO);
subscribe->mutable_framework_info()->mutable_id()->CopyFrom(frameworkId);
schedulerLibrary.send(call);
}
AWAIT_READY(error);
}
}
TEST_F(LimitedCpuIsolatorTest, ROOT_CGROUPS_CFS_Enable_Cfs)
{
slave::Flags flags;
// Enable CFS to cap CPU utilization.
flags.cgroups_enable_cfs = true;
Try<Isolator*> isolator = CgroupsCpushareIsolatorProcess::create(flags);
CHECK_SOME(isolator);
Try<Launcher*> launcher = LinuxLauncher::create(flags);
CHECK_SOME(launcher);
// Set the executor's resources to 0.5 cpu.
ExecutorInfo executorInfo;
executorInfo.mutable_resources()->CopyFrom(
Resources::parse("cpus:0.5").get());
ContainerID containerId;
containerId.set_value(UUID::random().toString());
// Use a relative temporary directory so it gets cleaned up
// automatically with the test.
Try<string> dir = os::mkdtemp(path::join(os::getcwd(), "XXXXXX"));
ASSERT_SOME(dir);
ContainerConfig containerConfig;
containerConfig.mutable_executor_info()->CopyFrom(executorInfo);
containerConfig.set_directory(dir.get());
Future<Option<ContainerLaunchInfo>> prepare =
isolator.get()->prepare(
containerId,
containerConfig);
AWAIT_READY(prepare);
// Generate random numbers to max out a single core. We'll run this for 0.5
// seconds of wall time so it should consume approximately 250 ms of total
// cpu time when limited to 0.5 cpu. We use /dev/urandom to prevent blocking
// on Linux when there's insufficient entropy.
string command = "cat /dev/urandom > /dev/null & "
"export MESOS_TEST_PID=$! && "
"sleep 0.5 && "
"kill $MESOS_TEST_PID";
int pipes[2];
ASSERT_NE(-1, ::pipe(pipes));
vector<string> argv(3);
argv[0] = "sh";
argv[1] = "-c";
argv[2] = command;
Try<pid_t> pid = launcher.get()->fork(
containerId,
"sh",
argv,
Subprocess::FD(STDIN_FILENO),
Subprocess::FD(STDOUT_FILENO),
Subprocess::FD(STDERR_FILENO),
None(),
None(),
lambda::bind(&childSetup, pipes),
prepare.get().isSome() ? prepare.get().get().namespaces() : 0);
ASSERT_SOME(pid);
// Reap the forked child.
Future<Option<int> > status = process::reap(pid.get());
// Continue in the parent.
ASSERT_SOME(os::close(pipes[0]));
// Isolate the forked child.
AWAIT_READY(isolator.get()->isolate(containerId, pid.get()));
// Now signal the child to continue.
char dummy;
ASSERT_LT(0, ::write(pipes[1], &dummy, sizeof(dummy)));
ASSERT_SOME(os::close(pipes[1]));
// Wait for the command to complete.
AWAIT_READY(status);
Future<ResourceStatistics> usage = isolator.get()->usage(containerId);
AWAIT_READY(usage);
// Expect that no more than 300 ms of cpu time has been consumed. We also
// check that at least 50 ms of cpu time has been consumed so this test will
// fail if the host system is very heavily loaded. This behavior is correct
// because under such conditions we aren't actually testing the CFS cpu
// limiter.
double cpuTime = usage.get().cpus_system_time_secs() +
usage.get().cpus_user_time_secs();
EXPECT_GE(0.30, cpuTime);
EXPECT_LE(0.05, cpuTime);
// Ensure all processes are killed.
AWAIT_READY(launcher.get()->destroy(containerId));
// Let the isolator clean up.
AWAIT_READY(isolator.get()->cleanup(containerId));
delete isolator.get();
delete launcher.get();
}
TEST(Wait, waitWithDuration) {
{
Promise<int> p;
Future<int> f = p.getFuture();
f.wait(milliseconds(1));
EXPECT_FALSE(f.isReady());
p.setValue(1);
EXPECT_TRUE(f.isReady());
}
{
Promise<int> p;
Future<int> f = p.getFuture();
p.setValue(1);
f.wait(milliseconds(1));
EXPECT_TRUE(f.isReady());
}
{
vector<Future<bool>> v_fb;
v_fb.push_back(makeFuture(true));
v_fb.push_back(makeFuture(false));
auto f = collectAll(v_fb);
f.wait(milliseconds(1));
EXPECT_TRUE(f.isReady());
EXPECT_EQ(2, f.value().size());
}
{
vector<Future<bool>> v_fb;
Promise<bool> p1;
Promise<bool> p2;
v_fb.push_back(p1.getFuture());
v_fb.push_back(p2.getFuture());
auto f = collectAll(v_fb);
f.wait(milliseconds(1));
EXPECT_FALSE(f.isReady());
p1.setValue(true);
EXPECT_FALSE(f.isReady());
p2.setValue(true);
EXPECT_TRUE(f.isReady());
}
{
auto f = makeFuture().wait(milliseconds(1));
EXPECT_TRUE(f.isReady());
}
{
Promise<Unit> p;
auto start = std::chrono::steady_clock::now();
auto f = p.getFuture().wait(milliseconds(100));
auto elapsed = std::chrono::steady_clock::now() - start;
EXPECT_GE(elapsed, milliseconds(100));
EXPECT_FALSE(f.isReady());
p.setValue();
EXPECT_TRUE(f.isReady());
}
{
// Try to trigger the race where the resultant Future is not yet complete
// even if we didn't hit the timeout, and make sure we deal with it properly
Promise<Unit> p;
folly::Baton<> b;
auto t = std::thread([&]{
b.post();
/* sleep override */ std::this_thread::sleep_for(milliseconds(100));
p.setValue();
});
b.wait();
auto f = p.getFuture().wait(std::chrono::seconds(3600));
EXPECT_TRUE(f.isReady());
t.join();
}
}
// This test verifies that we can successfully launch a container with
// a big (>= 10 cpus) cpu quota. This is to catch the regression
// observed in MESOS-1049.
// TODO(vinod): Revisit this if/when the isolator restricts the number
// of cpus that an executor can use based on the slave cpus.
TEST_F(LimitedCpuIsolatorTest, ROOT_CGROUPS_CFS_Big_Quota)
{
slave::Flags flags;
// Enable CFS to cap CPU utilization.
flags.cgroups_enable_cfs = true;
Try<Isolator*> isolator = CgroupsCpushareIsolatorProcess::create(flags);
CHECK_SOME(isolator);
Try<Launcher*> launcher = LinuxLauncher::create(flags);
CHECK_SOME(launcher);
// Set the executor's resources to 100.5 cpu.
ExecutorInfo executorInfo;
executorInfo.mutable_resources()->CopyFrom(
Resources::parse("cpus:100.5").get());
ContainerID containerId;
containerId.set_value(UUID::random().toString());
// Use a relative temporary directory so it gets cleaned up
// automatically with the test.
Try<string> dir = os::mkdtemp(path::join(os::getcwd(), "XXXXXX"));
ASSERT_SOME(dir);
ContainerConfig containerConfig;
containerConfig.mutable_executor_info()->CopyFrom(executorInfo);
containerConfig.set_directory(dir.get());
Future<Option<ContainerLaunchInfo>> prepare =
isolator.get()->prepare(
containerId,
containerConfig);
AWAIT_READY(prepare);
int pipes[2];
ASSERT_NE(-1, ::pipe(pipes));
vector<string> argv(3);
argv[0] = "sh";
argv[1] = "-c";
argv[2] = "exit 0";
Try<pid_t> pid = launcher.get()->fork(
containerId,
"sh",
argv,
Subprocess::FD(STDIN_FILENO),
Subprocess::FD(STDOUT_FILENO),
Subprocess::FD(STDERR_FILENO),
None(),
None(),
lambda::bind(&childSetup, pipes),
prepare.get().isSome() ? prepare.get().get().namespaces() : 0);
ASSERT_SOME(pid);
// Reap the forked child.
Future<Option<int> > status = process::reap(pid.get());
// Continue in the parent.
ASSERT_SOME(os::close(pipes[0]));
// Isolate the forked child.
AWAIT_READY(isolator.get()->isolate(containerId, pid.get()));
// Now signal the child to continue.
char dummy;
ASSERT_LT(0, ::write(pipes[1], &dummy, sizeof(dummy)));
ASSERT_SOME(os::close(pipes[1]));
// Wait for the command to complete successfully.
AWAIT_READY(status);
ASSERT_SOME_EQ(0, status.get());
// Ensure all processes are killed.
AWAIT_READY(launcher.get()->destroy(containerId));
// Let the isolator clean up.
AWAIT_READY(isolator.get()->cleanup(containerId));
delete isolator.get();
delete launcher.get();
}
// Ensures that the driver can handle an OFFERS event.
// Note that this includes the ability to bypass the
// master when sending framework messages.
TEST_F(SchedulerDriverEventTest, Offers)
{
Try<PID<Master>> master = StartMaster();
ASSERT_SOME(master);
MockScheduler sched;
MesosSchedulerDriver schedDriver(
&sched, DEFAULT_FRAMEWORK_INFO, master.get(), DEFAULT_CREDENTIAL);
EXPECT_CALL(sched, registered(&schedDriver, _, _));
Future<Message> frameworkRegisteredMessage =
FUTURE_MESSAGE(Eq(FrameworkRegisteredMessage().GetTypeName()), _, _);
schedDriver.start();
AWAIT_READY(frameworkRegisteredMessage);
UPID frameworkPid = frameworkRegisteredMessage.get().to;
// Start a slave and capture the offers.
Future<ResourceOffersMessage> resourceOffersMessage =
DROP_PROTOBUF(ResourceOffersMessage(), _, _);
MockExecutor exec(DEFAULT_EXECUTOR_ID);
Try<PID<Slave>> slave = StartSlave(&exec);
ASSERT_SOME(slave);
AWAIT_READY(resourceOffersMessage);
google::protobuf::RepeatedPtrField<Offer> offers =
resourceOffersMessage.get().offers();
ASSERT_EQ(1, offers.size());
// Ignore future offer messages.
DROP_PROTOBUFS(ResourceOffersMessage(), _, _);
// Send the offers event and expect a 'resourceOffers' call.
Event event;
event.set_type(Event::OFFERS);
event.mutable_offers()->mutable_offers()->CopyFrom(offers);
Future<Nothing> resourceOffers;
EXPECT_CALL(sched, resourceOffers(&schedDriver, _))
.WillOnce(FutureSatisfy(&resourceOffers));
process::post(master.get(), frameworkPid, event);
AWAIT_READY(resourceOffers);
// To test that the framework -> executor messages are
// sent directly to the slave, launch a task and send
// the executor a message.
EXPECT_CALL(exec, registered(_, _, _, _));
EXPECT_CALL(exec, launchTask(_, _))
.WillOnce(SendStatusUpdateFromTask(TASK_RUNNING));
Future<TaskStatus> status;
EXPECT_CALL(sched, statusUpdate(&schedDriver, _))
.WillOnce(FutureArg<1>(&status));
TaskInfo task = createTask(offers.Get(0), "", DEFAULT_EXECUTOR_ID);
schedDriver.launchTasks(offers.Get(0).id(), {task});
AWAIT_READY(status);
EXPECT_EQ(TASK_RUNNING, status.get().state());
// This message should skip the master!
Future<FrameworkToExecutorMessage> frameworkToExecutorMessage =
FUTURE_PROTOBUF(FrameworkToExecutorMessage(), frameworkPid, slave.get());
Future<string> data;
EXPECT_CALL(exec, frameworkMessage(_, _))
.WillOnce(FutureArg<1>(&data));
schedDriver.sendFrameworkMessage(
DEFAULT_EXECUTOR_ID, offers.Get(0).slave_id(), "hello");
AWAIT_READY(frameworkToExecutorMessage);
AWAIT_EXPECT_EQ("hello", data);
EXPECT_CALL(exec, shutdown(_))
.Times(AtMost(1));
schedDriver.stop();
schedDriver.join();
Shutdown();
}
// A test to verify the number of processes and threads in a
// container.
TEST_F(LimitedCpuIsolatorTest, ROOT_CGROUPS_Pids_and_Tids)
{
slave::Flags flags;
flags.cgroups_cpu_enable_pids_and_tids_count = true;
Try<Isolator*> isolator = CgroupsCpushareIsolatorProcess::create(flags);
CHECK_SOME(isolator);
Try<Launcher*> launcher = LinuxLauncher::create(flags);
CHECK_SOME(launcher);
ExecutorInfo executorInfo;
executorInfo.mutable_resources()->CopyFrom(
Resources::parse("cpus:0.5;mem:512").get());
ContainerID containerId;
containerId.set_value(UUID::random().toString());
// Use a relative temporary directory so it gets cleaned up
// automatically with the test.
Try<string> dir = os::mkdtemp(path::join(os::getcwd(), "XXXXXX"));
ASSERT_SOME(dir);
ContainerConfig containerConfig;
containerConfig.mutable_executor_info()->CopyFrom(executorInfo);
containerConfig.set_directory(dir.get());
Future<Option<ContainerLaunchInfo>> prepare =
isolator.get()->prepare(
containerId,
containerConfig);
AWAIT_READY(prepare);
// Right after the creation of the cgroup, which happens in
// 'prepare', we check that it is empty.
Future<ResourceStatistics> usage = isolator.get()->usage(containerId);
AWAIT_READY(usage);
EXPECT_EQ(0U, usage.get().processes());
EXPECT_EQ(0U, usage.get().threads());
int pipes[2];
ASSERT_NE(-1, ::pipe(pipes));
vector<string> argv(1);
argv[0] = "cat";
Try<pid_t> pid = launcher.get()->fork(
containerId,
"cat",
argv,
Subprocess::FD(STDIN_FILENO),
Subprocess::FD(STDOUT_FILENO),
Subprocess::FD(STDERR_FILENO),
None(),
None(),
lambda::bind(&childSetup, pipes),
prepare.get().isSome() ? prepare.get().get().namespaces() : 0);
ASSERT_SOME(pid);
// Reap the forked child.
Future<Option<int>> status = process::reap(pid.get());
// Continue in the parent.
ASSERT_SOME(os::close(pipes[0]));
// Before isolation, the cgroup is empty.
usage = isolator.get()->usage(containerId);
AWAIT_READY(usage);
EXPECT_EQ(0U, usage.get().processes());
EXPECT_EQ(0U, usage.get().threads());
// Isolate the forked child.
AWAIT_READY(isolator.get()->isolate(containerId, pid.get()));
// After the isolation, the cgroup is not empty, even though the
// process hasn't exec'd yet.
usage = isolator.get()->usage(containerId);
AWAIT_READY(usage);
EXPECT_EQ(1U, usage.get().processes());
EXPECT_EQ(1U, usage.get().threads());
// Now signal the child to continue.
char dummy;
ASSERT_LT(0, ::write(pipes[1], &dummy, sizeof(dummy)));
ASSERT_SOME(os::close(pipes[1]));
// Process count should be 1 since 'sleep' is still sleeping.
usage = isolator.get()->usage(containerId);
AWAIT_READY(usage);
EXPECT_EQ(1U, usage.get().processes());
EXPECT_EQ(1U, usage.get().threads());
// Ensure all processes are killed.
AWAIT_READY(launcher.get()->destroy(containerId));
// Wait for the command to complete.
AWAIT_READY(status);
// After the process is killed, the cgroup should be empty again.
usage = isolator.get()->usage(containerId);
AWAIT_READY(usage);
EXPECT_EQ(0U, usage.get().processes());
EXPECT_EQ(0U, usage.get().threads());
// Let the isolator clean up.
AWAIT_READY(isolator.get()->cleanup(containerId));
delete isolator.get();
delete launcher.get();
}
// This test verifies that reconciliation of a task that belongs to a
// slave that is a transitional state doesn't result in an update.
TEST_F(ReconciliationTest, SlaveInTransition)
{
master::Flags masterFlags = CreateMasterFlags();
Try<PID<Master> > master = StartMaster();
ASSERT_SOME(master);
// Start a checkpointing slave.
slave::Flags slaveFlags = CreateSlaveFlags();
slaveFlags.checkpoint = true;
Future<SlaveRegisteredMessage> slaveRegisteredMessage =
FUTURE_PROTOBUF(SlaveRegisteredMessage(), _, _);
Try<PID<Slave> > slave = StartSlave(slaveFlags);
ASSERT_SOME(slave);
// Wait for the slave to register and get the slave id.
AWAIT_READY(slaveRegisteredMessage);
const SlaveID slaveId = slaveRegisteredMessage.get().slave_id();
// Stop the master and slave.
Stop(master.get());
Stop(slave.get());
MockScheduler sched;
MesosSchedulerDriver driver(
&sched, DEFAULT_FRAMEWORK_INFO, master.get(), DEFAULT_CREDENTIAL);
Future<FrameworkID> frameworkId;
EXPECT_CALL(sched, registered(&driver, _, _))
.WillOnce(FutureArg<1>(&frameworkId));
EXPECT_CALL(sched, resourceOffers(&driver, _))
.WillRepeatedly(Return()); // Ignore offers.
// Framework should not receive any update.
EXPECT_CALL(sched, statusUpdate(&driver, _))
.Times(0);
// Drop '&Master::_reregisterSlave' dispatch so that the slave is
// in 'reregistering' state.
Future<Nothing> _reregisterSlave =
DROP_DISPATCH(_, &Master::_reregisterSlave);
// Restart the master.
master = StartMaster(masterFlags);
ASSERT_SOME(master);
driver.start();
// Wait for the framework to register.
AWAIT_READY(frameworkId);
// Restart the slave.
slave = StartSlave(slaveFlags);
ASSERT_SOME(slave);
// Slave will be in 'reregistering' state here.
AWAIT_READY(_reregisterSlave);
vector<TaskStatus> statuses;
// Create a task status with a random task id.
TaskStatus status;
status.mutable_task_id()->set_value(UUID::random().toString());
status.mutable_slave_id()->CopyFrom(slaveId);
status.set_state(TASK_RUNNING);
statuses.push_back(status);
Future<ReconcileTasksMessage> reconcileTasksMessage =
FUTURE_PROTOBUF(ReconcileTasksMessage(), _ , _);
Clock::pause();
driver.reconcileTasks(statuses);
// Make sure the master received the reconcile tasks message.
AWAIT_READY(reconcileTasksMessage);
// The Clock::settle() will ensure that framework would receive
// a status update if it is sent by the master. In this test it
// shouldn't receive any.
Clock::settle();
driver.stop();
driver.join();
Shutdown();
}
// This tests the create, prepare, isolate and cleanup methods of the
// 'CgroupNetClsIsolatorProcess'. The test first creates a 'MesosContainerizer'
// with net_cls cgroup isolator enabled. The net_cls cgroup isolator is
// implemented in the 'CgroupNetClsIsolatorProcess' class. The test then
// launches a task in a mesos container and checks to see if the container has
// been added to the right net_cls cgroup. Finally, the test kills the task and
// makes sure that the 'CgroupNetClsIsolatorProcess' cleans up the net_cls
// cgroup created for the container.
TEST_F(NetClsIsolatorTest, ROOT_CGROUPS_NetClsIsolate)
{
Try<PID<Master>> master = StartMaster();
ASSERT_SOME(master);
uint16_t primary = 0x0012;
slave::Flags flags = CreateSlaveFlags();
flags.isolation = "cgroups/net_cls";
flags.cgroups_net_cls_primary_handle = stringify(primary);
Fetcher fetcher;
Try<MesosContainerizer*> containerizer =
MesosContainerizer::create(flags, true, &fetcher);
ASSERT_SOME(containerizer);
Try<PID<Slave>> slave = StartSlave(containerizer.get(), flags);
ASSERT_SOME(slave);
MockScheduler sched;
MesosSchedulerDriver driver(
&sched, DEFAULT_FRAMEWORK_INFO, master.get(), DEFAULT_CREDENTIAL);
Future<Nothing> schedRegistered;
EXPECT_CALL(sched, registered(_, _, _))
.WillOnce(FutureSatisfy(&schedRegistered));
Future<vector<Offer>> offers;
EXPECT_CALL(sched, resourceOffers(_, _))
.WillOnce(FutureArg<1>(&offers))
.WillRepeatedly(Return()); // Ignore subsequent offers.
driver.start();
AWAIT_READY(schedRegistered);
AWAIT_READY(offers);
EXPECT_EQ(1u, offers.get().size());
// Create a task to be launched in the mesos-container. We will be
// explicitly killing this task to perform the cleanup test.
TaskInfo task = createTask(offers.get()[0], "sleep 1000");
Future<TaskStatus> status;
EXPECT_CALL(sched, statusUpdate(_, _))
.WillOnce(FutureArg<1>(&status));
driver.launchTasks(offers.get()[0].id(), {task});
// Capture the update to verify that the task has been launched.
AWAIT_READY(status);
ASSERT_EQ(TASK_RUNNING, status.get().state());
// Task is ready. Make sure there is exactly 1 container in the hashset.
Future<hashset<ContainerID>> containers = containerizer.get()->containers();
AWAIT_READY(containers);
EXPECT_EQ(1u, containers.get().size());
const ContainerID& containerID = *(containers.get().begin());
Result<string> hierarchy = cgroups::hierarchy("net_cls");
ASSERT_SOME(hierarchy);
// Check if the net_cls cgroup for this container exists, by
// checking for the processes associated with this cgroup.
string cgroup = path::join(
flags.cgroups_root,
containerID.value());
Try<set<pid_t>> pids = cgroups::processes(hierarchy.get(), cgroup);
ASSERT_SOME(pids);
// There should be at least one TGID associated with this cgroup.
EXPECT_LE(1u, pids.get().size());
// Read the `net_cls.classid` to verify that the handle has been set.
Try<uint32_t> classid = cgroups::net_cls::classid(hierarchy.get(), cgroup);
EXPECT_SOME(classid);
if (classid.isSome()) {
// Make sure the primary handle is the same as the one set in
// `--cgroup_net_cls_primary_handle`.
EXPECT_EQ(primary, (classid.get() & 0xffff0000) >> 16);
// Make sure the secondary handle is non-zero.
EXPECT_NE(0, classid.get() & 0xffff);
}
// This test ensures that the master does not send updates for
// terminal tasks during an implicit reconciliation request.
// TODO(bmahler): Soon the master will keep non-acknowledged
// tasks, and this test may break.
TEST_F(ReconciliationTest, ImplicitTerminalTask)
{
Try<PID<Master> > master = StartMaster();
ASSERT_SOME(master);
MockExecutor exec(DEFAULT_EXECUTOR_ID);
TestContainerizer containerizer(&exec);
Try<PID<Slave> > slave = StartSlave(&containerizer);
ASSERT_SOME(slave);
// Launch a framework and get a task terminal.
MockScheduler sched;
MesosSchedulerDriver driver(
&sched, DEFAULT_FRAMEWORK_INFO, master.get(), DEFAULT_CREDENTIAL);
Future<FrameworkID> frameworkId;
EXPECT_CALL(sched, registered(&driver, _, _))
.WillOnce(FutureArg<1>(&frameworkId));
EXPECT_CALL(sched, resourceOffers(&driver, _))
.WillOnce(LaunchTasks(DEFAULT_EXECUTOR_INFO, 1, 1, 512, "*"))
.WillRepeatedly(Return()); // Ignore subsequent offers.
EXPECT_CALL(exec, registered(_, _, _, _));
EXPECT_CALL(exec, launchTask(_, _))
.WillOnce(SendStatusUpdateFromTask(TASK_FINISHED));
Future<TaskStatus> update;
EXPECT_CALL(sched, statusUpdate(&driver, _))
.WillOnce(FutureArg<1>(&update));
driver.start();
// Wait until the framework is registered.
AWAIT_READY(frameworkId);
AWAIT_READY(update);
EXPECT_EQ(TASK_FINISHED, update.get().state());
EXPECT_TRUE(update.get().has_slave_id());
// Framework should not receive any further updates.
EXPECT_CALL(sched, statusUpdate(&driver, _))
.Times(0);
EXPECT_CALL(exec, shutdown(_))
.Times(AtMost(1));
Future<ReconcileTasksMessage> reconcileTasksMessage =
FUTURE_PROTOBUF(ReconcileTasksMessage(), _ , _);
Clock::pause();
// When making an implicit reconciliation request, the master
// should not send back terminal tasks.
vector<TaskStatus> statuses;
driver.reconcileTasks(statuses);
// Make sure the master received the reconcile tasks message.
AWAIT_READY(reconcileTasksMessage);
// The Clock::settle() will ensure that framework would receive
// a status update if it is sent by the master. In this test it
// shouldn't receive any.
Clock::settle();
driver.stop();
driver.join();
Shutdown(); // Must shutdown before 'containerizer' gets deallocated.
}
// This test checks that a failed over scheduler gets the retried status update
// when the original instance dies without acknowledging the update.
TEST_F(HttpFaultToleranceTest, SchedulerFailoverStatusUpdate)
{
master::Flags flags = CreateMasterFlags();
flags.authenticate_frameworks = false;
Try<Owned<cluster::Master>> master = StartMaster(flags);
ASSERT_SOME(master);
auto scheduler = std::make_shared<v1::MockHTTPScheduler>();
auto executor = std::make_shared<v1::MockHTTPExecutor>();
ExecutorID executorId = DEFAULT_EXECUTOR_ID;
TestContainerizer containerizer(executorId, executor);
Owned<MasterDetector> detector = master.get()->createDetector();
Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), &containerizer);
ASSERT_SOME(slave);
Future<Nothing> connected;
EXPECT_CALL(*scheduler, connected(_))
.WillOnce(FutureSatisfy(&connected))
.WillRepeatedly(Return()); // Ignore future invocations.
ContentType contentType = ContentType::PROTOBUF;
v1::scheduler::TestMesos schedulerLibrary(
master.get()->pid,
contentType,
scheduler);
AWAIT_READY(connected);
Future<Event::Subscribed> subscribed;
EXPECT_CALL(*scheduler, subscribed(_, _))
.WillOnce(FutureArg<1>(&subscribed));
EXPECT_CALL(*scheduler, heartbeat(_))
.WillRepeatedly(Return()); // Ignore heartbeats.
Future<Event::Offers> offers;
EXPECT_CALL(*scheduler, offers(_, _))
.WillOnce(FutureArg<1>(&offers));
{
Call call;
call.set_type(Call::SUBSCRIBE);
Call::Subscribe* subscribe = call.mutable_subscribe();
subscribe->mutable_framework_info()->CopyFrom(v1::DEFAULT_FRAMEWORK_INFO);
schedulerLibrary.send(call);
}
AWAIT_READY(subscribed);
v1::FrameworkID frameworkId(subscribed->framework_id());
AWAIT_READY(offers);
EXPECT_NE(0, offers->offers().size());
EXPECT_CALL(*executor, connected(_))
.WillOnce(v1::executor::SendSubscribe(frameworkId, evolve(executorId)));
EXPECT_CALL(*executor, subscribed(_, _));
EXPECT_CALL(*executor, launch(_, _))
.WillOnce(v1::executor::SendUpdateFromTask(
frameworkId, evolve(executorId), v1::TASK_RUNNING));
Future<Nothing> acknowledged;
EXPECT_CALL(*executor, acknowledged(_, _))
.WillOnce(FutureSatisfy(&acknowledged));
Future<Event::Update> update;
EXPECT_CALL(*scheduler, update(_, _))
.WillOnce(FutureArg<1>(&update));
const v1::Offer& offer = offers->offers(0);
v1::TaskInfo taskInfo =
evolve(createTask(devolve(offer), "", executorId));
{
Call call;
call.mutable_framework_id()->CopyFrom(frameworkId);
call.set_type(Call::ACCEPT);
Call::Accept* accept = call.mutable_accept();
accept->add_offer_ids()->CopyFrom(offer.id());
v1::Offer::Operation* operation = accept->add_operations();
operation->set_type(v1::Offer::Operation::LAUNCH);
operation->mutable_launch()->add_task_infos()->CopyFrom(taskInfo);
schedulerLibrary.send(call);
}
AWAIT_READY(acknowledged);
AWAIT_READY(update);
EXPECT_EQ(v1::TASK_RUNNING, update->status().state());
EXPECT_EQ(executorId, devolve(update->status().executor_id()));
EXPECT_TRUE(update->status().has_executor_id());
EXPECT_TRUE(update->status().has_uuid());
// Failover the scheduler without acknowledging the status update.
auto scheduler2 = std::make_shared<v1::MockHTTPScheduler>();
Future<Nothing> connected2;
EXPECT_CALL(*scheduler2, connected(_))
.WillOnce(FutureSatisfy(&connected2));
// Failover to another scheduler instance.
v1::scheduler::TestMesos schedulerLibrary2(
master.get()->pid,
contentType,
scheduler2);
AWAIT_READY(connected2);
// The previously connected scheduler instance should receive an
// error/disconnected event.
Future<Nothing> error;
EXPECT_CALL(*scheduler, error(_, _))
.WillOnce(FutureSatisfy(&error));
Future<Nothing> disconnected;
EXPECT_CALL(*scheduler, disconnected(_))
.WillOnce(FutureSatisfy(&disconnected));
EXPECT_CALL(*scheduler2, subscribed(_, _))
.WillOnce(FutureArg<1>(&subscribed));
EXPECT_CALL(*scheduler2, heartbeat(_))
.WillRepeatedly(Return()); // Ignore heartbeats.
// Scheduler2 should receive the retried status update.
Future<Nothing> update2;
EXPECT_CALL(*scheduler2, update(_, _))
.WillOnce(FutureSatisfy(&update2))
.WillRepeatedly(Return()); // Ignore subsequent updates.
{
Call call;
call.mutable_framework_id()->CopyFrom(frameworkId);
call.set_type(Call::SUBSCRIBE);
Call::Subscribe* subscribe = call.mutable_subscribe();
subscribe->mutable_framework_info()->CopyFrom(v1::DEFAULT_FRAMEWORK_INFO);
subscribe->mutable_framework_info()->mutable_id()->CopyFrom(frameworkId);
schedulerLibrary2.send(call);
}
AWAIT_READY(error);
AWAIT_READY(disconnected);
AWAIT_READY(subscribed);
EXPECT_EQ(frameworkId, subscribed->framework_id());
Clock::pause();
// Now advance time enough for the reliable timeout to kick in and
// another status update to be sent.
Clock::advance(slave::STATUS_UPDATE_RETRY_INTERVAL_MIN);
AWAIT_READY(update2);
EXPECT_CALL(*executor, shutdown(_))
.Times(AtMost(1));
EXPECT_CALL(*executor, disconnected(_))
.Times(AtMost(1));
}
// This test verifies that reconciliation sends the latest task
// status, when the task state does not match between the framework
// and the master.
TEST_F(ReconciliationTest, TaskStateMismatch)
{
Try<PID<Master> > master = StartMaster();
ASSERT_SOME(master);
MockExecutor exec(DEFAULT_EXECUTOR_ID);
TestContainerizer containerizer(&exec);
Try<PID<Slave> > slave = StartSlave(&containerizer);
ASSERT_SOME(slave);
MockScheduler sched;
MesosSchedulerDriver driver(
&sched, DEFAULT_FRAMEWORK_INFO, master.get(), DEFAULT_CREDENTIAL);
Future<FrameworkID> frameworkId;
EXPECT_CALL(sched, registered(&driver, _, _))
.WillOnce(FutureArg<1>(&frameworkId));
EXPECT_CALL(sched, resourceOffers(&driver, _))
.WillOnce(LaunchTasks(DEFAULT_EXECUTOR_INFO, 1, 1, 512, "*"))
.WillRepeatedly(Return()); // Ignore subsequent offers.
EXPECT_CALL(exec, registered(_, _, _, _));
EXPECT_CALL(exec, launchTask(_, _))
.WillOnce(SendStatusUpdateFromTask(TASK_RUNNING));
Future<TaskStatus> update;
EXPECT_CALL(sched, statusUpdate(&driver, _))
.WillOnce(FutureArg<1>(&update));
driver.start();
// Wait until the framework is registered.
AWAIT_READY(frameworkId);
AWAIT_READY(update);
EXPECT_EQ(TASK_RUNNING, update.get().state());
EXPECT_EQ(true, update.get().has_slave_id());
const TaskID taskId = update.get().task_id();
const SlaveID slaveId = update.get().slave_id();
// If framework has different state, current state should be reported.
Future<TaskStatus> update2;
EXPECT_CALL(sched, statusUpdate(&driver, _))
.WillOnce(FutureArg<1>(&update2));
vector<TaskStatus> statuses;
TaskStatus status;
status.mutable_task_id()->CopyFrom(taskId);
status.mutable_slave_id()->CopyFrom(slaveId);
status.set_state(TASK_KILLED);
statuses.push_back(status);
driver.reconcileTasks(statuses);
AWAIT_READY(update2);
EXPECT_EQ(TASK_RUNNING, update2.get().state());
EXPECT_CALL(exec, shutdown(_))
.Times(AtMost(1));
driver.stop();
driver.join();
Shutdown(); // Must shutdown before 'containerizer' gets deallocated.
}
// This test ensures that the failed over scheduler is able to send a message
// to the executor.
TEST_F(HttpFaultToleranceTest, SchedulerFailoverFrameworkToExecutorMessage)
{
master::Flags flags = CreateMasterFlags();
flags.authenticate_frameworks = false;
Try<Owned<cluster::Master>> master = StartMaster(flags);
ASSERT_SOME(master);
auto scheduler = std::make_shared<v1::MockHTTPScheduler>();
auto executor = std::make_shared<v1::MockHTTPExecutor>();
ExecutorID executorId = DEFAULT_EXECUTOR_ID;
TestContainerizer containerizer(executorId, executor);
Owned<MasterDetector> detector = master.get()->createDetector();
Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), &containerizer);
ASSERT_SOME(slave);
Future<Nothing> connected;
EXPECT_CALL(*scheduler, connected(_))
.WillOnce(FutureSatisfy(&connected))
.WillRepeatedly(Return()); // Ignore future invocations.
ContentType contentType = ContentType::PROTOBUF;
v1::scheduler::TestMesos schedulerLibrary(
master.get()->pid,
contentType,
scheduler);
AWAIT_READY(connected);
Future<Event::Subscribed> subscribed;
EXPECT_CALL(*scheduler, subscribed(_, _))
.WillOnce(FutureArg<1>(&subscribed));
EXPECT_CALL(*scheduler, heartbeat(_))
.WillRepeatedly(Return()); // Ignore heartbeats.
Future<Event::Offers> offers;
EXPECT_CALL(*scheduler, offers(_, _))
.WillOnce(FutureArg<1>(&offers));
{
Call call;
call.set_type(Call::SUBSCRIBE);
Call::Subscribe* subscribe = call.mutable_subscribe();
subscribe->mutable_framework_info()->CopyFrom(v1::DEFAULT_FRAMEWORK_INFO);
schedulerLibrary.send(call);
}
AWAIT_READY(subscribed);
v1::FrameworkID frameworkId(subscribed->framework_id());
AWAIT_READY(offers);
EXPECT_NE(0, offers->offers().size());
EXPECT_CALL(*executor, connected(_))
.WillOnce(v1::executor::SendSubscribe(frameworkId, evolve(executorId)));
EXPECT_CALL(*executor, subscribed(_, _));
Future<Nothing> launch;
EXPECT_CALL(*executor, launch(_, _))
.WillOnce(FutureSatisfy(&launch));
const v1::Offer& offer = offers->offers(0);
v1::TaskInfo taskInfo =
evolve(createTask(devolve(offer), "", executorId));
{
Call call;
call.mutable_framework_id()->CopyFrom(frameworkId);
call.set_type(Call::ACCEPT);
Call::Accept* accept = call.mutable_accept();
accept->add_offer_ids()->CopyFrom(offer.id());
v1::Offer::Operation* operation = accept->add_operations();
operation->set_type(v1::Offer::Operation::LAUNCH);
operation->mutable_launch()->add_task_infos()->CopyFrom(taskInfo);
schedulerLibrary.send(call);
}
AWAIT_READY(launch);
auto scheduler2 = std::make_shared<v1::MockHTTPScheduler>();
Future<Nothing> connected2;
EXPECT_CALL(*scheduler2, connected(_))
.WillOnce(FutureSatisfy(&connected2));
// Failover to another scheduler instance.
v1::scheduler::TestMesos schedulerLibrary2(
master.get()->pid,
contentType,
scheduler2);
AWAIT_READY(connected2);
// The previously connected scheduler instance should receive an
// error/disconnected event.
Future<Nothing> error;
EXPECT_CALL(*scheduler, error(_, _))
.WillOnce(FutureSatisfy(&error));
Future<Nothing> disconnected;
EXPECT_CALL(*scheduler, disconnected(_))
.WillOnce(FutureSatisfy(&disconnected));
EXPECT_CALL(*scheduler2, subscribed(_, _))
.WillOnce(FutureArg<1>(&subscribed));
EXPECT_CALL(*scheduler2, heartbeat(_))
.WillRepeatedly(Return()); // Ignore heartbeats.
{
Call call;
call.mutable_framework_id()->CopyFrom(frameworkId);
call.set_type(Call::SUBSCRIBE);
Call::Subscribe* subscribe = call.mutable_subscribe();
subscribe->mutable_framework_info()->CopyFrom(v1::DEFAULT_FRAMEWORK_INFO);
subscribe->mutable_framework_info()->mutable_id()->CopyFrom(frameworkId);
schedulerLibrary2.send(call);
}
AWAIT_READY(error);
AWAIT_READY(disconnected);
AWAIT_READY(subscribed);
EXPECT_EQ(frameworkId, subscribed->framework_id());
Future<v1::executor::Event::Message> message;
EXPECT_CALL(*executor, message(_, _))
.WillOnce(FutureArg<1>(&message));
{
Call call;
call.mutable_framework_id()->CopyFrom(frameworkId);
call.set_type(Call::MESSAGE);
Call::Message* message = call.mutable_message();
message->mutable_agent_id()->CopyFrom(offer.agent_id());
message->mutable_executor_id()->CopyFrom(v1::DEFAULT_EXECUTOR_ID);
message->set_data("hello world");
schedulerLibrary2.send(call);
}
AWAIT_READY(message);
ASSERT_EQ("hello world", message->data());
EXPECT_CALL(*executor, shutdown(_))
.Times(AtMost(1));
EXPECT_CALL(*executor, disconnected(_))
.Times(AtMost(1));
}
// This test verifies that a reconciliation request that comes before
// '_launchTasks()' is ignored.
TEST_F(MasterAuthorizationTest, ReconcileTask)
{
MockAuthorizer authorizer;
Try<PID<Master> > master = StartMaster(&authorizer);
ASSERT_SOME(master);
MockExecutor exec(DEFAULT_EXECUTOR_ID);
Try<PID<Slave> > slave = StartSlave(&exec);
ASSERT_SOME(slave);
MockScheduler sched;
MesosSchedulerDriver driver(
&sched, DEFAULT_FRAMEWORK_INFO, master.get(), DEFAULT_CREDENTIAL);
EXPECT_CALL(sched, registered(&driver, _, _))
.Times(1);
Future<vector<Offer> > offers;
EXPECT_CALL(sched, resourceOffers(&driver, _))
.WillOnce(FutureArg<1>(&offers))
.WillRepeatedly(Return()); // Ignore subsequent offers.
driver.start();
AWAIT_READY(offers);
EXPECT_NE(0u, offers.get().size());
TaskInfo task = createTask(offers.get()[0], "", DEFAULT_EXECUTOR_ID);
vector<TaskInfo> tasks;
tasks.push_back(task);
// Return a pending future from authorizer.
Future<Nothing> future;
Promise<bool> promise;
EXPECT_CALL(authorizer, authorize(An<const mesos::ACL::RunTasks&>()))
.WillOnce(DoAll(FutureSatisfy(&future),
Return(promise.future())));
driver.launchTasks(offers.get()[0].id(), tasks);
// Wait until authorization is in progress.
AWAIT_READY(future);
// Scheduler shouldn't get an update from reconciliation.
EXPECT_CALL(sched, statusUpdate(&driver, _))
.Times(0);
Future<ReconcileTasksMessage> reconcileTasksMessage =
FUTURE_PROTOBUF(ReconcileTasksMessage(), _, _);
vector<TaskStatus> statuses;
TaskStatus status;
status.mutable_task_id()->CopyFrom(task.task_id());
status.mutable_slave_id()->CopyFrom(offers.get()[0].slave_id());
status.set_state(TASK_STAGING);
statuses.push_back(status);
driver.reconcileTasks(statuses);
AWAIT_READY(reconcileTasksMessage);
// Make sure the framework doesn't receive any update.
Clock::pause();
Clock::settle();
// Now stop the framework.
driver.stop();
driver.join();
Shutdown(); // Must shutdown before 'containerizer' gets deallocated.
}
// This test checks that a scheduler exit shuts down the executor.
TEST_F(HttpFaultToleranceTest, SchedulerExit)
{
master::Flags flags = CreateMasterFlags();
flags.authenticate_frameworks = false;
Try<Owned<cluster::Master>> master = StartMaster(flags);
ASSERT_SOME(master);
auto scheduler = std::make_shared<v1::MockHTTPScheduler>();
auto executor = std::make_shared<v1::MockHTTPExecutor>();
ExecutorID executorId = DEFAULT_EXECUTOR_ID;
TestContainerizer containerizer(executorId, executor);
Owned<MasterDetector> detector = master.get()->createDetector();
Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), &containerizer);
ASSERT_SOME(slave);
Future<Nothing> connected;
EXPECT_CALL(*scheduler, connected(_))
.WillOnce(FutureSatisfy(&connected))
.WillRepeatedly(Return()); // Ignore future invocations.
ContentType contentType = ContentType::PROTOBUF;
v1::scheduler::TestMesos schedulerLibrary(
master.get()->pid,
contentType,
scheduler);
AWAIT_READY(connected);
Future<Event::Subscribed> subscribed;
EXPECT_CALL(*scheduler, subscribed(_, _))
.WillOnce(FutureArg<1>(&subscribed));
EXPECT_CALL(*scheduler, heartbeat(_))
.WillRepeatedly(Return()); // Ignore heartbeats.
Future<Event::Offers> offers;
EXPECT_CALL(*scheduler, offers(_, _))
.WillOnce(FutureArg<1>(&offers));
{
Call call;
call.set_type(Call::SUBSCRIBE);
Call::Subscribe* subscribe = call.mutable_subscribe();
subscribe->mutable_framework_info()->CopyFrom(v1::DEFAULT_FRAMEWORK_INFO);
schedulerLibrary.send(call);
}
AWAIT_READY(subscribed);
v1::FrameworkID frameworkId(subscribed->framework_id());
AWAIT_READY(offers);
EXPECT_NE(0, offers->offers().size());
EXPECT_CALL(*executor, connected(_))
.WillOnce(v1::executor::SendSubscribe(frameworkId, evolve(executorId)));
EXPECT_CALL(*executor, subscribed(_, _));
Future<Nothing> launch;
EXPECT_CALL(*executor, launch(_, _))
.WillOnce(FutureSatisfy(&launch));
const v1::Offer& offer = offers->offers(0);
v1::TaskInfo taskInfo =
evolve(createTask(devolve(offer), "", DEFAULT_EXECUTOR_ID));
{
Call call;
call.mutable_framework_id()->CopyFrom(frameworkId);
call.set_type(Call::ACCEPT);
Call::Accept* accept = call.mutable_accept();
accept->add_offer_ids()->CopyFrom(offer.id());
v1::Offer::Operation* operation = accept->add_operations();
operation->set_type(v1::Offer::Operation::LAUNCH);
operation->mutable_launch()->add_task_infos()->CopyFrom(taskInfo);
schedulerLibrary.send(call);
}
AWAIT_READY(launch);
EXPECT_CALL(*scheduler, disconnected(_))
.Times(AtMost(1));
Future<Nothing> shutdown;
EXPECT_CALL(*executor, shutdown(_))
.WillOnce(FutureSatisfy(&shutdown));
{
Call call;
call.mutable_framework_id()->CopyFrom(frameworkId);
call.set_type(Call::TEARDOWN);
schedulerLibrary.send(call);
}
// Ensure that the executor receives a `Event::Shutdown` after the
// scheduler exit.
AWAIT_READY(shutdown);
}
protocol,
stringify(protocol == server_protocol));
}
// Set up the server.
Try<Socket> server = setup_server(server_environment);
ASSERT_SOME(server);
// Launch the client with a POLL socket.
Try<Subprocess> client = launch_client({
{"LIBPROCESS_SSL_ENABLED", "false"}},
server.get(),
false);
ASSERT_SOME(client);
Future<Socket> socket = server->accept();
AWAIT_ASSERT_READY(socket);
// TODO(jmlvanre): Remove const copy.
AWAIT_ASSERT_EQ(data, Socket(socket.get()).recv());
AWAIT_ASSERT_READY(Socket(socket.get()).send(data));
AWAIT_ASSERT_READY(await_subprocess(client.get(), 0));
}
}
// For each protocol: ensure we CANNOT communicate between a POLL
// based socket and an SSL socket if 'SSL_SUPPORT_DOWNGRADE' is not
// enabled.
TEST_F(SSLTest, NoValidDowngradeEachProtocol)
// This test verifies that we are able to downgrade from a HTTP based
// framework to PID.
TEST_P(SchedulerHttpApiTest, UpdateHttpToPidScheduler)
{
Try<Owned<cluster::Master>> master = StartMaster();
ASSERT_SOME(master);
v1::FrameworkInfo frameworkInfo = v1::DEFAULT_FRAMEWORK_INFO;
Call call;
call.set_type(Call::SUBSCRIBE);
Call::Subscribe* subscribe = call.mutable_subscribe();
subscribe->mutable_framework_info()->CopyFrom(frameworkInfo);
// Retrieve the parameter passed as content type to this test.
const string contentType = GetParam();
process::http::Headers headers = createBasicAuthHeaders(DEFAULT_CREDENTIAL);
headers["Accept"] = contentType;
Future<Response> response = process::http::streaming::post(
master.get()->pid,
"api/v1/scheduler",
headers,
serialize(call, contentType),
contentType);
AWAIT_EXPECT_RESPONSE_STATUS_EQ(OK().status, response);
AWAIT_EXPECT_RESPONSE_HEADER_EQ("chunked", "Transfer-Encoding", response);
ASSERT_EQ(Response::PIPE, response.get().type);
Option<Pipe::Reader> reader = response.get().reader;
ASSERT_SOME(reader);
auto deserializer = lambda::bind(
&SchedulerHttpApiTest::deserialize, this, contentType, lambda::_1);
Reader<Event> responseDecoder(Decoder<Event>(deserializer), reader.get());
Future<Result<Event>> event = responseDecoder.read();
AWAIT_READY(event);
ASSERT_SOME(event.get());
// Check event type is subscribed and the framework id is set.
ASSERT_EQ(Event::SUBSCRIBED, event.get().get().type());
frameworkInfo.mutable_id()->
CopyFrom(event.get().get().subscribed().framework_id());
// Make sure it receives a heartbeat.
event = responseDecoder.read();
AWAIT_READY(event);
ASSERT_SOME(event.get());
ASSERT_EQ(Event::HEARTBEAT, event.get().get().type());
MockScheduler sched;
MesosSchedulerDriver driver(
&sched, devolve(frameworkInfo), master.get()->pid, DEFAULT_CREDENTIAL);
Future<FrameworkID> frameworkId;
EXPECT_CALL(sched, registered(&driver, _, _))
.WillOnce(FutureArg<1>(&frameworkId));
driver.start();
AWAIT_READY(frameworkId);
ASSERT_EQ(evolve(frameworkId.get()), frameworkInfo.id());
driver.stop();
driver.join();
}