TEST_F(FrameworksManagerTestFixture, ResurrectFramework)
{
// Resurrect a non-existent framework.
FrameworkID id;
id.set_value("non-existent framework");
Future<Result<bool> > future1 =
process::dispatch(manager, &FrameworksManager::resurrect, id);
ASSERT_TRUE(future1.await(2.0));
EXPECT_FALSE(future1.get().get());
// Resurrect an existent framework that is NOT being removed.
// Add a dummy framework.
FrameworkID id2;
id2.set_value("id2");
FrameworkInfo info2;
info2.set_name("test name");
info2.set_user("test user");
// Add the framework.
Future<Result<bool> > future2 =
process::dispatch(manager, &FrameworksManager::add, id2, info2);
ASSERT_TRUE(future2.await(2.0));
EXPECT_TRUE(future2.get().get());
Future<Result<bool> > future3 =
process::dispatch(manager, &FrameworksManager::resurrect, id2);
ASSERT_TRUE(future3.await(2.0));
EXPECT_TRUE(future3.get().get());
}
// Not deriving from fixture...because we want to set specific expectations.
// Specifically we simulate caching failure in FrameworksManager.
TEST(FrameworksManagerTest, CacheFailure)
{
ASSERT_TRUE(GTEST_IS_THREADSAFE);
MockFrameworksStorage storage;
process::spawn(storage);
Result<map<FrameworkID, FrameworkInfo> > errMsg =
Result<map<FrameworkID, FrameworkInfo> >::error("Fake Caching Error.");
EXPECT_CALL(storage, list())
.Times(2)
.WillRepeatedly(Return(errMsg));
EXPECT_CALL(storage, add(_, _))
.WillOnce(Return(Result<bool>::some(true)));
EXPECT_CALL(storage, remove(_))
.Times(0);
FrameworksManager manager(&storage);
process::spawn(manager);
// Test if initially FrameworksManager returns error.
Future<Result<map<FrameworkID, FrameworkInfo> > > future1 =
process::dispatch(manager, &FrameworksManager::list);
ASSERT_TRUE(future1.await(2.0));
ASSERT_TRUE(future1.get().isError());
EXPECT_EQ(future1.get().error(), "Error caching framework infos.");
// Add framework should function normally despite caching failure.
FrameworkID id;
id.set_value("id");
FrameworkInfo info;
info.set_name("test name");
info.set_user("test user");
// Add the framework.
Future<Result<bool> > future2 =
process::dispatch(manager, &FrameworksManager::add, id, info);
ASSERT_TRUE(future2.await(2.0));
EXPECT_TRUE(future2.get().get());
// Remove framework should fail due to caching failure.
Future<Result<bool> > future3 =
process::dispatch(manager, &FrameworksManager::remove, id, seconds(0));
ASSERT_TRUE(future3.await(2.0));
ASSERT_TRUE(future3.get().isError());
EXPECT_EQ(future3.get().error(), "Error caching framework infos.");
process::terminate(manager);
process::wait(manager);
process::terminate(storage);
process::wait(storage);
}
TEST_F(ProtobufIOTest, RepeatedPtrField)
{
const string file = ".protobuf_io_test_repeated_ptr_field";
RepeatedPtrField<FrameworkID> expected;
const size_t size = 10;
for (size_t i = 0; i < size; i++) {
FrameworkID frameworkId;
frameworkId.set_value(stringify(i));
expected.Add()->CopyFrom(frameworkId);
}
Try<Nothing> write = ::protobuf::write(file, expected);
ASSERT_SOME(write);
Result<RepeatedPtrField<FrameworkID>> read =
::protobuf::read<RepeatedPtrField<FrameworkID>>(file);
ASSERT_SOME(read);
RepeatedPtrField<FrameworkID> actual = read.get();
ASSERT_EQ(expected.size(), actual.size());
for (size_t i = 0; i < size; i++) {
EXPECT_EQ(expected.Get(i), actual.Get(i));
}
}
TEST_F(FrameworksManagerTestFixture, RemoveFramework)
{
Clock::pause();
// Remove a non-existent framework.
FrameworkID id;
id.set_value("non-existent framework");
Future<Result<bool> > future1 =
process::dispatch(manager, &FrameworksManager::remove, id, seconds(0));
ASSERT_TRUE(future1.await(2.0));
EXPECT_TRUE(future1.get().isError());
// Remove an existing framework.
// First add a dummy framework.
FrameworkID id2;
id2.set_value("id2");
FrameworkInfo info2;
info2.set_name("test name");
info2.set_user("test user");
// Add the framework.
Future<Result<bool> > future2 =
process::dispatch(manager, &FrameworksManager::add, id2, info2);
ASSERT_TRUE(future2.await(2.0));
EXPECT_TRUE(future2.get().get());
// Now remove the added framework.
Future<Result<bool> > future3 =
process::dispatch(manager, &FrameworksManager::remove, id2, seconds(1.0));
Clock::update(Clock::now(manager) + 1.0);
ASSERT_TRUE(future3.await(2.0));
EXPECT_TRUE(future2.get().get());
// Now check if the removed framework exists...it shouldn't.
Future<Result<bool> > future4 =
process::dispatch(manager, &FrameworksManager::exists, id2);
ASSERT_TRUE(future4.await(2.0));
EXPECT_FALSE(future4.get().get());
Clock::resume();
}
// TODO(bmahler): Move this file into stout.
TEST_F(ProtobufIOTest, Basic)
{
const string file = ".protobuf_io_test_basic";
Try<int_fd> result = os::open(
file,
O_CREAT | O_WRONLY | O_SYNC | O_CLOEXEC,
S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
ASSERT_SOME(result);
int_fd fdw = result.get();
result = os::open(
file,
O_CREAT | O_RDONLY | O_CLOEXEC,
S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
ASSERT_SOME(result);
int_fd fdr = result.get();
const size_t writes = 10;
for (size_t i = 0; i < writes; i++) {
FrameworkID frameworkId;
frameworkId.set_value(stringify(i));
Try<Nothing> result = ::protobuf::write(fdw, frameworkId);
ASSERT_SOME(result);
}
Result<FrameworkID> read = None();
size_t reads = 0;
while (true) {
read = ::protobuf::read<FrameworkID>(fdr);
if (!read.isSome()) {
break;
}
EXPECT_EQ(read->value(), stringify(reads++));
}
// Ensure we've hit the end of the file without reading a partial
// protobuf.
ASSERT_TRUE(read.isNone());
ASSERT_EQ(writes, reads);
os::close(fdw);
os::close(fdr);
}
ExecutorInfo createExecutorInfo(
const string& _frameworkId,
const string& _executorId)
{
FrameworkID frameworkId;
frameworkId.set_value(_frameworkId);
ExecutorID executorId;
executorId.set_value(_executorId);
ExecutorInfo executorInfo;
executorInfo.mutable_executor_id()->CopyFrom(executorId);
executorInfo.mutable_framework_id()->CopyFrom(frameworkId);
return executorInfo;
}
TEST_F(FrameworksManagerTestFixture, ResurrectInterspersedExpiringFrameworks)
{
// This is another crucial test.
// Two remove messages are interspersed with a resurrect.
// Only the second remove should actually remove the framework.
// Add a dummy framework.
FrameworkID id;
id.set_value("id");
FrameworkInfo info;
info.set_name("test name");
info.set_user("test user");
// Add the framework.
process::dispatch(manager, &FrameworksManager::add, id, info);
Clock::pause();
Future<Result<bool> > future1 =
process::dispatch(manager, &FrameworksManager::remove, id, seconds(2.0));
// Resurrect in the meanwhile.
Future<Result<bool> > future2 =
process::dispatch(manager, &FrameworksManager::resurrect, id);
// Remove again.
Future<Result<bool> > future3 =
process::dispatch(manager, &FrameworksManager::remove, id, seconds(1.0));
ASSERT_TRUE(future2.await(2.0));
EXPECT_TRUE(future2.get().get());
Clock::update(Clock::now(manager) + 1.0);
ASSERT_TRUE(future3.await(2.0));
EXPECT_TRUE(future3.get().get());
Clock::update(Clock::now(manager) + 2.0);
ASSERT_TRUE(future1.await(2.0));
EXPECT_FALSE(future1.get().get());
Clock::resume();
}
TEST_F(FrameworksManagerTestFixture, AddFramework)
{
// Test if initially FM returns empty list.
Future<Result<map<FrameworkID, FrameworkInfo> > > future =
process::dispatch(manager, &FrameworksManager::list);
ASSERT_TRUE(future.await(2.0));
EXPECT_TRUE(future.get().get().empty());
// Add a dummy framework.
FrameworkID id;
id.set_value("id");
FrameworkInfo info;
info.set_name("test name");
info.set_user("test user");
// Add the framework.
Future<Result<bool> > future2 =
process::dispatch(manager, &FrameworksManager::add, id, info);
ASSERT_TRUE(future2.await(2.0));
EXPECT_TRUE(future2.get().get());
// Check if framework manager returns the added framework.
Future<Result<map<FrameworkID, FrameworkInfo> > > future3 =
process::dispatch(manager, &FrameworksManager::list);
ASSERT_TRUE(future3.await(2.0));
map<FrameworkID, FrameworkInfo> result = future3.get().get();
ASSERT_EQ(1, result.count(id));
EXPECT_EQ("test name", result[id].name());
EXPECT_EQ("test user", result[id].user());
// Check if the framework exists.
Future<Result<bool> > future4 =
process::dispatch(manager, &FrameworksManager::exists, id);
ASSERT_TRUE(future4.await(2.0));
EXPECT_TRUE(future4.get().get());
}
// This test verifies the correct handling of the statistics
// endpoint when statistics is missing in ResourceUsage.
TEST(MonitorTest, MissingStatistics)
{
ResourceMonitor monitor([]() -> Future<ResourceUsage> {
FrameworkID frameworkId;
frameworkId.set_value("framework");
ExecutorID executorId;
executorId.set_value("executor");
ExecutorInfo executorInfo;
executorInfo.mutable_executor_id()->CopyFrom(executorId);
executorInfo.mutable_framework_id()->CopyFrom(frameworkId);
executorInfo.set_name("name");
executorInfo.set_source("source");
Resources resources = Resources::parse("cpus:1;mem:2").get();
ResourceUsage usage;
ResourceUsage::Executor* executor = usage.add_executors();
executor->mutable_executor_info()->CopyFrom(executorInfo);
executor->mutable_allocated()->CopyFrom(resources);
return usage;
});
UPID upid("monitor", process::address());
Future<http::Response> response = http::get(upid, "statistics");
AWAIT_READY(response);
AWAIT_EXPECT_RESPONSE_STATUS_EQ(http::OK().status, response);
AWAIT_EXPECT_RESPONSE_HEADER_EQ(
"application/json",
"Content-Type",
response);
AWAIT_EXPECT_RESPONSE_BODY_EQ("[]", response);
}
// TODO(vinod): Using a paused clock in the tests means that
// future.await() may wait forever. This makes debugging hard.
TEST_F(FrameworksManagerTestFixture, ResurrectExpiringFramework)
{
// This is the crucial test.
// Resurrect an existing framework that is being removed,is being removed,
// which should cause the remove to be unsuccessful.
// Add a dummy framework.
FrameworkID id;
id.set_value("id");
FrameworkInfo info;
info.set_name("test name");
info.set_user("test user");
// Add the framework.
process::dispatch(manager, &FrameworksManager::add, id, info);
Clock::pause();
// Remove after 2 secs.
Future<Result<bool> > future1 =
process::dispatch(manager, &FrameworksManager::remove, id, seconds(2.0));
// Resurrect in the meanwhile.
Future<Result<bool> > future2 =
process::dispatch(manager, &FrameworksManager::resurrect, id);
ASSERT_TRUE(future2.await(2.0));
EXPECT_TRUE(future2.get().get());
Clock::update(Clock::now(manager) + 2.0);
ASSERT_TRUE(future1.await(2.0));
EXPECT_FALSE(future1.get().get());
Clock::resume();
}
// This test ensures we don't break the API when it comes to JSON
// representation of tasks. Also, we want to ensure that tasks are
// modeled the same way when using 'Task' vs. 'TaskInfo'.
TEST(HTTP, ModelTask)
{
TaskID taskId;
taskId.set_value("t");
SlaveID slaveId;
slaveId.set_value("s");
ExecutorID executorId;
executorId.set_value("t");
FrameworkID frameworkId;
frameworkId.set_value("f");
TaskState state = TASK_RUNNING;
vector<TaskStatus> statuses;
TaskStatus status;
status.mutable_task_id()->CopyFrom(taskId);
status.set_state(state);
status.mutable_slave_id()->CopyFrom(slaveId);
status.mutable_executor_id()->CopyFrom(executorId);
status.set_timestamp(0.0);
statuses.push_back(status);
TaskInfo task;
task.set_name("task");
task.mutable_task_id()->CopyFrom(taskId);
task.mutable_slave_id()->CopyFrom(slaveId);
task.mutable_command()->set_value("echo hello");
Task task_ = protobuf::createTask(task, state, frameworkId);
task_.add_statuses()->CopyFrom(statuses[0]);
JSON::Value object = model(task, frameworkId, state, statuses);
JSON::Value object_ = model(task_);
Try<JSON::Value> expected = JSON::parse(
"{"
" \"executor_id\":\"\","
" \"framework_id\":\"f\","
" \"id\":\"t\","
" \"name\":\"task\","
" \"resources\":"
" {"
" \"cpus\":0,"
" \"disk\":0,"
" \"mem\":0"
" },"
" \"slave_id\":\"s\","
" \"state\":\"TASK_RUNNING\","
" \"statuses\":"
" ["
" {"
" \"state\":\"TASK_RUNNING\","
" \"timestamp\":0"
" }"
" ]"
"}");
ASSERT_SOME(expected);
EXPECT_EQ(expected.get(), object);
EXPECT_EQ(expected.get(), object_);
// Ensure both are modeled the same.
EXPECT_EQ(object, object_);
}
TEST(MonitorTest, Statistics)
{
FrameworkID frameworkId;
frameworkId.set_value("framework");
ExecutorID executorId;
executorId.set_value("executor");
ExecutorInfo executorInfo;
executorInfo.mutable_executor_id()->CopyFrom(executorId);
executorInfo.mutable_framework_id()->CopyFrom(frameworkId);
executorInfo.set_name("name");
executorInfo.set_source("source");
ResourceStatistics statistics;
statistics.set_cpus_nr_periods(100);
statistics.set_cpus_nr_throttled(2);
statistics.set_cpus_user_time_secs(4);
statistics.set_cpus_system_time_secs(1);
statistics.set_cpus_throttled_time_secs(0.5);
statistics.set_cpus_limit(1.0);
statistics.set_mem_file_bytes(0);
statistics.set_mem_anon_bytes(0);
statistics.set_mem_mapped_file_bytes(0);
statistics.set_mem_rss_bytes(1024);
statistics.set_mem_limit_bytes(2048);
statistics.set_timestamp(0);
ResourceMonitor monitor([=]() -> Future<ResourceUsage> {
Resources resources = Resources::parse("cpus:1;mem:2").get();
ResourceUsage usage;
ResourceUsage::Executor* executor = usage.add_executors();
executor->mutable_executor_info()->CopyFrom(executorInfo);
executor->mutable_allocated()->CopyFrom(resources);
executor->mutable_statistics()->CopyFrom(statistics);
return usage;
});
UPID upid("monitor", process::address());
Future<http::Response> response = http::get(upid, "statistics");
AWAIT_READY(response);
AWAIT_EXPECT_RESPONSE_STATUS_EQ(http::OK().status, response);
AWAIT_EXPECT_RESPONSE_HEADER_EQ(
"application/json",
"Content-Type",
response);
JSON::Array expected;
JSON::Object usage;
usage.values["executor_id"] = "executor";
usage.values["executor_name"] = "name";
usage.values["framework_id"] = "framework";
usage.values["source"] = "source";
usage.values["statistics"] = JSON::Protobuf(statistics);
expected.values.push_back(usage);
Try<JSON::Array> result = JSON::parse<JSON::Array>(response.get().body);
ASSERT_SOME(result);
ASSERT_EQ(expected, result.get());
}
// TODO(bmahler): Add additional tests:
// 1. Check that the data has been published to statistics.
// 2. Check that metering is occurring on subsequent resource data.
TEST(MonitorTest, WatchUnwatch)
{
FrameworkID frameworkId;
frameworkId.set_value("framework");
ExecutorID executorId;
executorId.set_value("executor");
ExecutorInfo executorInfo;
executorInfo.mutable_executor_id()->CopyFrom(executorId);
executorInfo.mutable_framework_id()->CopyFrom(frameworkId);
executorInfo.set_name("name");
executorInfo.set_source("source");
ResourceStatistics initialStatistics;
initialStatistics.set_cpus_user_time_secs(0);
initialStatistics.set_cpus_system_time_secs(0);
initialStatistics.set_cpus_limit(2.5);
initialStatistics.set_mem_rss_bytes(0);
initialStatistics.set_mem_limit_bytes(2048);
initialStatistics.set_timestamp(Clock::now().secs());
ResourceStatistics statistics;
statistics.set_cpus_nr_periods(100);
statistics.set_cpus_nr_throttled(2);
statistics.set_cpus_user_time_secs(4);
statistics.set_cpus_system_time_secs(1);
statistics.set_cpus_throttled_time_secs(0.5);
statistics.set_cpus_limit(2.5);
statistics.set_mem_rss_bytes(1024);
statistics.set_mem_limit_bytes(2048);
statistics.set_timestamp(
initialStatistics.timestamp() +
slave::RESOURCE_MONITORING_INTERVAL.secs());
TestingIsolator isolator;
process::spawn(isolator);
Future<Nothing> usage1, usage2;
EXPECT_CALL(isolator, usage(frameworkId, executorId))
.WillOnce(DoAll(FutureSatisfy(&usage1),
Return(initialStatistics)))
.WillOnce(DoAll(FutureSatisfy(&usage2),
Return(statistics)));
slave::ResourceMonitor monitor(&isolator);
// We pause the clock first in order to make sure that we can
// advance time below to force the 'delay' in
// ResourceMonitorProcess::watch to execute.
process::Clock::pause();
monitor.watch(
frameworkId,
executorId,
executorInfo,
slave::RESOURCE_MONITORING_INTERVAL);
// Now wait for ResouorceMonitorProcess::watch to finish so we can
// advance time to cause collection to begin.
process::Clock::settle();
process::Clock::advance(slave::RESOURCE_MONITORING_INTERVAL);
process::Clock::settle();
AWAIT_READY(usage1);
// Wait until the isolator has finished returning the statistics.
process::Clock::settle();
// The second collection will populate the cpus_usage.
process::Clock::advance(slave::RESOURCE_MONITORING_INTERVAL);
process::Clock::settle();
AWAIT_READY(usage2);
// Wait until the isolator has finished returning the statistics.
process::Clock::settle();
process::UPID upid("monitor", process::ip(), process::port());
Future<Response> response = process::http::get(upid, "usage.json");
AWAIT_EXPECT_RESPONSE_STATUS_EQ(OK().status, response);
AWAIT_EXPECT_RESPONSE_HEADER_EQ(
"application/json",
"Content-Type",
response);
// TODO(bmahler): Verify metering directly through statistics.
AWAIT_EXPECT_RESPONSE_BODY_EQ(
strings::format(
"[{"
"\"executor_id\":\"executor\","
"\"executor_name\":\"name\","
"\"framework_id\":\"framework\","
"\"resource_usage\":{"
"\"cpu_time\":%g,"
"\"cpu_usage\":%g,"
"\"memory_rss\":%lu"
"},"
"\"source\":\"source\""
"}]",
statistics.cpus_system_time_secs() + statistics.cpus_user_time_secs(),
(statistics.cpus_system_time_secs() +
statistics.cpus_user_time_secs()) /
slave::RESOURCE_MONITORING_INTERVAL.secs(),
statistics.mem_rss_bytes()).get(),
response);
response = process::http::get(upid, "statistics.json");
AWAIT_EXPECT_RESPONSE_STATUS_EQ(OK().status, response);
AWAIT_EXPECT_RESPONSE_HEADER_EQ(
"application/json",
"Content-Type",
response);
// TODO(bmahler): Verify metering directly through statistics.
AWAIT_EXPECT_RESPONSE_BODY_EQ(
strings::format(
"[{"
"\"executor_id\":\"executor\","
"\"executor_name\":\"name\","
"\"framework_id\":\"framework\","
"\"source\":\"source\","
"\"statistics\":{"
"\"cpus_limit\":%g,"
"\"cpus_nr_periods\":%d,"
"\"cpus_nr_throttled\":%d,"
"\"cpus_system_time_secs\":%g,"
"\"cpus_throttled_time_secs\":%g,"
"\"cpus_user_time_secs\":%g,"
"\"mem_limit_bytes\":%lu,"
"\"mem_rss_bytes\":%lu"
"}"
"}]",
statistics.cpus_limit(),
statistics.cpus_nr_periods(),
statistics.cpus_nr_throttled(),
statistics.cpus_system_time_secs(),
statistics.cpus_throttled_time_secs(),
statistics.cpus_user_time_secs(),
statistics.mem_limit_bytes(),
statistics.mem_rss_bytes()).get(),
response);
// Ensure the monitor stops polling the isolator.
monitor.unwatch(frameworkId, executorId);
// Wait until ResourceMonitorProcess::unwatch has completed.
process::Clock::settle();
// This time, Isolator::usage should not get called.
EXPECT_CALL(isolator, usage(frameworkId, executorId))
.Times(0);
process::Clock::advance(slave::RESOURCE_MONITORING_INTERVAL);
process::Clock::settle();
response = process::http::get(upid, "usage.json");
AWAIT_EXPECT_RESPONSE_STATUS_EQ(OK().status, response);
AWAIT_EXPECT_RESPONSE_HEADER_EQ(
"application/json",
"Content-Type",
response);
AWAIT_EXPECT_RESPONSE_BODY_EQ("[]", response);
}
// This test ensures we don't break the API when it comes to JSON
// representation of tasks.
TEST(HTTPTest, ModelTask)
{
TaskID taskId;
taskId.set_value("t");
SlaveID slaveId;
slaveId.set_value("s");
ExecutorID executorId;
executorId.set_value("t");
FrameworkID frameworkId;
frameworkId.set_value("f");
TaskState state = TASK_RUNNING;
vector<TaskStatus> statuses;
TaskStatus status;
status.mutable_task_id()->CopyFrom(taskId);
status.set_state(state);
status.mutable_slave_id()->CopyFrom(slaveId);
status.mutable_executor_id()->CopyFrom(executorId);
status.set_timestamp(0.0);
statuses.push_back(status);
Labels labels;
labels.add_labels()->CopyFrom(createLabel("ACTION", "port:7987 DENY"));
Ports ports;
Port* port = ports.add_ports();
port->set_number(80);
port->mutable_labels()->CopyFrom(labels);
DiscoveryInfo discovery;
discovery.set_visibility(DiscoveryInfo::CLUSTER);
discovery.set_name("discover");
discovery.mutable_ports()->CopyFrom(ports);
TaskInfo taskInfo;
taskInfo.set_name("task");
taskInfo.mutable_task_id()->CopyFrom(taskId);
taskInfo.mutable_slave_id()->CopyFrom(slaveId);
taskInfo.mutable_command()->set_value("echo hello");
taskInfo.mutable_discovery()->CopyFrom(discovery);
Task task = createTask(taskInfo, state, frameworkId);
task.add_statuses()->CopyFrom(statuses[0]);
JSON::Value object = model(task);
Try<JSON::Value> expected = JSON::parse(
"{"
" \"executor_id\":\"\","
" \"framework_id\":\"f\","
" \"id\":\"t\","
" \"name\":\"task\","
" \"resources\":"
" {"
" \"cpus\":0,"
" \"disk\":0,"
" \"gpus\":0,"
" \"mem\":0"
" },"
" \"slave_id\":\"s\","
" \"state\":\"TASK_RUNNING\","
" \"statuses\":"
" ["
" {"
" \"state\":\"TASK_RUNNING\","
" \"timestamp\":0"
" }"
" ],"
" \"discovery\":"
" {"
" \"name\":\"discover\","
" \"ports\":"
" {"
" \"ports\":"
" ["
" {"
" \"number\":80,"
" \"labels\":"
" {"
" \"labels\":"
" ["
" {"
" \"key\":\"ACTION\","
" \"value\":\"port:7987 DENY\""
" }"
" ]"
" }"
" }"
" ]"
" },"
" \"visibility\":\"CLUSTER\""
" }"
"}");
ASSERT_SOME(expected);
EXPECT_EQ(expected.get(), object);
}