virtual void frameworkMessage(SchedulerDriver* driver,
const ExecutorID& executorId,
const SlaveID& slaveId,
const string& data)
{
vector<string> strVector = stringToVector(data);
string taskId = strVector[0];
string url = strVector[1];
if (executorId.value() == crawler.executor_id().value()) {
cout << "Crawler msg received: " << taskId << endl;
for (size_t i = 2; i < strVector.size(); i++) {
string& newURL = strVector[i];
crawlResults[url].push_back(newURL);
if (processed.find(newURL) == processed.end()) {
processed[newURL] = nextUrlId++;
if (newURL.substr(0, baseUrl.length()) == baseUrl) {
crawlQueue.push(newURL);
}
renderQueue.push(newURL);
}
}
} else {
if (access(strVector[2].c_str(), R_OK) == 0) {
renderResults[url] = strVector[2];
}
}
frameworkMessagesReceived++;
}
ExecutorInfo createExecutorInfo(
const string& _frameworkId,
const string& _executorId)
{
FrameworkID frameworkId;
frameworkId.set_value(_frameworkId);
ExecutorID executorId;
executorId.set_value(_executorId);
ExecutorInfo executorInfo;
executorInfo.mutable_executor_id()->CopyFrom(executorId);
executorInfo.mutable_framework_id()->CopyFrom(frameworkId);
return executorInfo;
}
void CephSchedulerAgent<T>::frameworkMessage(
T* driver,
const ExecutorID& executorId,
const SlaveID& slaveId,
const string& data)
{
ceph::TaskState initialMon = stateMachine->getInitialMon();
if (initialMon.executorId == executorId.value()) {
LOG(INFO) << "Got osd id from inital MON: " << data;
stateMachine->addPendingOSDID(lexical_cast<int>(data));
}
}
// This test verifies the correct handling of the statistics
// endpoint when statistics is missing in ResourceUsage.
TEST(MonitorTest, MissingStatistics)
{
ResourceMonitor monitor([]() -> Future<ResourceUsage> {
FrameworkID frameworkId;
frameworkId.set_value("framework");
ExecutorID executorId;
executorId.set_value("executor");
ExecutorInfo executorInfo;
executorInfo.mutable_executor_id()->CopyFrom(executorId);
executorInfo.mutable_framework_id()->CopyFrom(frameworkId);
executorInfo.set_name("name");
executorInfo.set_source("source");
Resources resources = Resources::parse("cpus:1;mem:2").get();
ResourceUsage usage;
ResourceUsage::Executor* executor = usage.add_executors();
executor->mutable_executor_info()->CopyFrom(executorInfo);
executor->mutable_allocated()->CopyFrom(resources);
return usage;
});
UPID upid("monitor", process::address());
Future<http::Response> response = http::get(upid, "statistics");
AWAIT_READY(response);
AWAIT_EXPECT_RESPONSE_STATUS_EQ(http::OK().status, response);
AWAIT_EXPECT_RESPONSE_HEADER_EQ(
"application/json",
"Content-Type",
response);
AWAIT_EXPECT_RESPONSE_BODY_EQ("[]", response);
}
void ChapelScheduler::frameworkMessage(SchedulerDriver* driver,
const ExecutorID& executorId,
const SlaveID& slaveId,
const string& data)
{
vector<string> strVector = stringToVector(data);
string taskId = strVector[0];
string url = strVector[1];
if(executorId.value().size() > 0) {
cout << "Framework message received: " << taskId << endl;
}
frameworkMessagesReceived+=1;
}
jobject convert(JNIEnv* env, const ExecutorID& executorId)
{
string data;
executorId.SerializeToString(&data);
// byte[] data = ..;
jbyteArray jdata = env->NewByteArray(data.size());
env->SetByteArrayRegion(jdata, 0, data.size(), (jbyte*) data.data());
// ExecutorID executorId = ExecutorID.parseFrom(data);
jclass clazz = FindMesosClass(env, "org/apache/mesos/Protos$ExecutorID");
jmethodID parseFrom =
env->GetStaticMethodID(clazz, "parseFrom",
"([B)Lorg/apache/mesos/Protos$ExecutorID;");
jobject jexecutorId = env->CallStaticObjectMethod(clazz, parseFrom, jdata);
return jexecutorId;
}
inline bool operator==(const ExecutorID& left, const std::string& right)
{
return left.value() == right;
}
inline std::size_t hash_value(const ExecutorID& executorId)
{
size_t seed = 0;
boost::hash_combine(seed, executorId.value());
return seed;
}
TEST(MonitorTest, Statistics)
{
FrameworkID frameworkId;
frameworkId.set_value("framework");
ExecutorID executorId;
executorId.set_value("executor");
ExecutorInfo executorInfo;
executorInfo.mutable_executor_id()->CopyFrom(executorId);
executorInfo.mutable_framework_id()->CopyFrom(frameworkId);
executorInfo.set_name("name");
executorInfo.set_source("source");
ResourceStatistics statistics;
statistics.set_cpus_nr_periods(100);
statistics.set_cpus_nr_throttled(2);
statistics.set_cpus_user_time_secs(4);
statistics.set_cpus_system_time_secs(1);
statistics.set_cpus_throttled_time_secs(0.5);
statistics.set_cpus_limit(1.0);
statistics.set_mem_file_bytes(0);
statistics.set_mem_anon_bytes(0);
statistics.set_mem_mapped_file_bytes(0);
statistics.set_mem_rss_bytes(1024);
statistics.set_mem_limit_bytes(2048);
statistics.set_timestamp(0);
ResourceMonitor monitor([=]() -> Future<ResourceUsage> {
Resources resources = Resources::parse("cpus:1;mem:2").get();
ResourceUsage usage;
ResourceUsage::Executor* executor = usage.add_executors();
executor->mutable_executor_info()->CopyFrom(executorInfo);
executor->mutable_allocated()->CopyFrom(resources);
executor->mutable_statistics()->CopyFrom(statistics);
return usage;
});
UPID upid("monitor", process::address());
Future<http::Response> response = http::get(upid, "statistics");
AWAIT_READY(response);
AWAIT_EXPECT_RESPONSE_STATUS_EQ(http::OK().status, response);
AWAIT_EXPECT_RESPONSE_HEADER_EQ(
"application/json",
"Content-Type",
response);
JSON::Array expected;
JSON::Object usage;
usage.values["executor_id"] = "executor";
usage.values["executor_name"] = "name";
usage.values["framework_id"] = "framework";
usage.values["source"] = "source";
usage.values["statistics"] = JSON::Protobuf(statistics);
expected.values.push_back(usage);
Try<JSON::Array> result = JSON::parse<JSON::Array>(response.get().body);
ASSERT_SOME(result);
ASSERT_EQ(expected, result.get());
}
void CephSchedulerAgent<T>::resourceOffers(
T* driver,
const vector<Offer>& offers)
{
LOG(INFO) << "Received " << offers.size() << " offers! ";
TaskType taskType;
int token;
int isInitialMonNode = 0;
//handle waiting OSD task, give them osdID to start docker
handleWaitingOSDTasks(driver);
Phase currentPhase = stateMachine->getCurrentPhase();
//try start new node
foreach (const Offer& offer, offers) {
//check offer with the correct role
LOG(INFO) << "Hostname: " << offer.hostname();
if (!hasRole(offer, config->role)) {
LOG(INFO) << "Decline this offer. Host " << offer.hostname() << " don't have correct role:"
<< config->role;
Filters refuse;
refuse.set_refuse_seconds(86400.0);
driver->declineOffer(offer.id(),refuse);
continue;
}
//reload or new hostconfig
stateMachine->addConfig(offer.hostname());
tryLaunchDiskTask(driver, offer, offer.hostname());
bool accept = stateMachine->nextMove(taskType,token,offer.hostname());
if (!accept) {
LOG(INFO) << "In the "
<< static_cast<int>(currentPhase)
<< " Staging Phase, cannot accept offer from "
<< offer.hostname()
<< " in this phase";
driver->declineOffer(offer.id());
continue;
}
LOG(INFO) << "Check offer's resources from " <<offer.hostname();
if (offerNotEnoughResources(offer,taskType)) {
LOG(INFO) << "Not enough, decline it from " << offer.hostname();
driver->declineOffer(offer.id());
continue;
}
if (currentPhase == Phase::WAINTING_REQUEST){
accept = fetchPendingRESTfulRequest();
if (!accept){
LOG(INFO) << "No pending OSD RESTful request.";
driver->declineOffer(offer.id());
stateMachine->decreaseOSDIndex();
continue;
}
}
LOG(INFO) << "Accepted offer from" << offer.hostname() << ", launch "
<< static_cast<int>(taskType) <<":" << token << " node";
if (taskType == TaskType::MON && token == 0) {
LOG(INFO) << "This is the initial MON";
isInitialMonNode = 1;
}
string taskId;
string executorId;
launchNode(
driver,
offer,
taskType,
token,
isInitialMonNode,
taskId,
executorId);
stateMachine->addStagingTask(
taskId,
executorId,
taskType,
offer.hostname(),
offer.slave_id().value());
if (!isInitialMonNode && taskType == TaskType::OSD) {
ceph::TaskState initialMon = stateMachine->getInitialMon();
const string m = lexical_cast<string>(static_cast<int>(MessageToExecutor::REGISTER_OSD));
ExecutorID eId;
eId.set_value(initialMon.executorId);
SlaveID sId;
sId.set_value(initialMon.slaveId);
driver->sendFrameworkMessage(
eId,
sId,
m);
}//end if
}//end foreach
// This test verifies that the framework can launch a command task
// that specifies both container image and persistent volumes.
TEST_F(LinuxFilesystemIsolatorMesosTest,
ROOT_ChangeRootFilesystemCommandExecutorPersistentVolume)
{
Try<Owned<cluster::Master>> master = StartMaster();
ASSERT_SOME(master);
string registry = path::join(sandbox.get(), "registry");
AWAIT_READY(DockerArchive::create(registry, "test_image"));
slave::Flags flags = CreateSlaveFlags();
flags.resources = "cpus:2;mem:1024;disk(role1):1024";
flags.isolation = "filesystem/linux,docker/runtime";
flags.docker_registry = registry;
flags.docker_store_dir = path::join(sandbox.get(), "store");
flags.image_providers = "docker";
Owned<MasterDetector> detector = master.get()->createDetector();
Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), flags);
ASSERT_SOME(slave);
MockScheduler sched;
FrameworkInfo frameworkInfo = DEFAULT_FRAMEWORK_INFO;
frameworkInfo.set_roles(0, "role1");
MesosSchedulerDriver driver(
&sched,
frameworkInfo,
master.get()->pid,
DEFAULT_CREDENTIAL);
Future<FrameworkID> frameworkId;
EXPECT_CALL(sched, registered(&driver, _, _))
.WillOnce(FutureArg<1>(&frameworkId));
Future<vector<Offer>> offers;
EXPECT_CALL(sched, resourceOffers(&driver, _))
.WillOnce(FutureArg<1>(&offers))
.WillRepeatedly(Return()); // Ignore subsequent offers.
driver.start();
AWAIT_READY(frameworkId);
AWAIT_READY(offers);
ASSERT_FALSE(offers->empty());
Offer offer = offers.get()[0];
string dir1 = path::join(sandbox.get(), "dir1");
ASSERT_SOME(os::mkdir(dir1));
Resource persistentVolume = createPersistentVolume(
Megabytes(64),
"role1",
"id1",
"path1",
None(),
None(),
frameworkInfo.principal());
// We use the filter explicitly here so that the resources will not
// be filtered for 5 seconds (the default).
Filters filters;
filters.set_refuse_seconds(0);
TaskInfo task = createTask(
offer.slave_id(),
Resources::parse("cpus:1;mem:512").get() + persistentVolume,
"echo abc > path1/file");
task.mutable_container()->CopyFrom(createContainerInfo(
"test_image",
{createVolumeHostPath("/tmp", dir1, Volume::RW)}));
// Create the persistent volumes and launch task via `acceptOffers`.
driver.acceptOffers(
{offer.id()},
{CREATE(persistentVolume), LAUNCH({task})},
filters);
Future<TaskStatus> statusStarting;
Future<TaskStatus> statusRunning;
Future<TaskStatus> statusFinished;
EXPECT_CALL(sched, statusUpdate(&driver, _))
.WillOnce(FutureArg<1>(&statusStarting))
.WillOnce(FutureArg<1>(&statusRunning))
.WillOnce(FutureArg<1>(&statusFinished));
AWAIT_READY(statusStarting);
EXPECT_EQ(TASK_STARTING, statusStarting->state());
AWAIT_READY(statusRunning);
EXPECT_EQ(TASK_RUNNING, statusRunning->state());
AWAIT_READY(statusFinished);
EXPECT_EQ(TASK_FINISHED, statusFinished->state());
// NOTE: The command executor's id is the same as the task id.
ExecutorID executorId;
executorId.set_value(task.task_id().value());
string directory = slave::paths::getExecutorLatestRunPath(
flags.work_dir,
offer.slave_id(),
frameworkId.get(),
executorId);
EXPECT_FALSE(os::exists(path::join(directory, "path1")));
string volumePath = slave::paths::getPersistentVolumePath(
flags.work_dir,
"role1",
"id1");
EXPECT_SOME_EQ("abc\n", os::read(path::join(volumePath, "file")));
driver.stop();
driver.join();
}
TYPED_TEST(IsolatorTest, Usage)
{
Try<PID<Master> > master = this->StartMaster();
ASSERT_SOME(master);
TypeParam isolator;
slave::Flags flags = this->CreateSlaveFlags();
Try<PID<Slave> > slave = this->StartSlave(&isolator, flags);
ASSERT_SOME(slave);
MockScheduler sched;
MesosSchedulerDriver driver(
&sched, DEFAULT_FRAMEWORK_INFO, master.get(), DEFAULT_CREDENTIAL);
Future<FrameworkID> frameworkId;
EXPECT_CALL(sched, registered(&driver, _, _))
.WillOnce(FutureArg<1>(&frameworkId));
Future<vector<Offer> > offers;
EXPECT_CALL(sched, resourceOffers(&driver, _))
.WillOnce(FutureArg<1>(&offers))
.WillRepeatedly(Return()); // Ignore subsequent offers.
driver.start();
AWAIT_READY(frameworkId);
AWAIT_READY(offers);
EXPECT_NE(0u, offers.get().size());
TaskInfo task;
task.set_name("isolator_test");
task.mutable_task_id()->set_value("1");
task.mutable_slave_id()->MergeFrom(offers.get()[0].slave_id());
task.mutable_resources()->MergeFrom(offers.get()[0].resources());
Resources resources(offers.get()[0].resources());
Option<Bytes> mem = resources.mem();
ASSERT_SOME(mem);
Option<double> cpus = resources.cpus();
ASSERT_SOME(cpus);
const std::string& file = path::join(flags.work_dir, "ready");
// This task induces user/system load in a child process by
// running top in a child process for ten seconds.
task.mutable_command()->set_value(
#ifdef __APPLE__
// Use logging mode with 30,000 samples with no interval.
"top -l 30000 -s 0 2>&1 > /dev/null & "
#else
// Batch mode, with 30,000 samples with no interval.
"top -b -d 0 -n 30000 2>&1 > /dev/null & "
#endif
"touch " + file + "; " // Signals that the top command is running.
"sleep 60");
vector<TaskInfo> tasks;
tasks.push_back(task);
Future<TaskStatus> status;
EXPECT_CALL(sched, statusUpdate(&driver, _))
.WillOnce(FutureArg<1>(&status));
driver.launchTasks(offers.get()[0].id(), tasks);
AWAIT_READY(status);
EXPECT_EQ(TASK_RUNNING, status.get().state());
// Wait for the task to begin inducing cpu time.
while (!os::exists(file));
ExecutorID executorId;
executorId.set_value(task.task_id().value());
// We'll wait up to 10 seconds for the child process to induce
// 1/8 of a second of user and system cpu time in total.
// TODO(bmahler): Also induce rss memory consumption, by re-using
// the balloon framework.
ResourceStatistics statistics;
Duration waited = Duration::zero();
do {
Future<ResourceStatistics> usage =
process::dispatch(
(Isolator*) &isolator, // TODO(benh): Fix after reaper changes.
&Isolator::usage,
frameworkId.get(),
executorId);
AWAIT_READY(usage);
statistics = usage.get();
// If we meet our usage expectations, we're done!
if (statistics.cpus_user_time_secs() >= 0.125 &&
statistics.cpus_system_time_secs() >= 0.125 &&
statistics.mem_rss_bytes() >= 1024u) {
break;
}
os::sleep(Milliseconds(100));
waited += Milliseconds(100);
} while (waited < Seconds(10));
EXPECT_GE(statistics.cpus_user_time_secs(), 0.125);
EXPECT_GE(statistics.cpus_system_time_secs(), 0.125);
EXPECT_EQ(statistics.cpus_limit(), cpus.get());
EXPECT_GE(statistics.mem_rss_bytes(), 1024u);
EXPECT_EQ(statistics.mem_limit_bytes(), mem.get().bytes());
EXPECT_CALL(sched, statusUpdate(&driver, _))
.WillOnce(FutureArg<1>(&status));
driver.killTask(task.task_id());
AWAIT_READY(status);
EXPECT_EQ(TASK_KILLED, status.get().state());
driver.stop();
driver.join();
this->Shutdown(); // Must shutdown before 'isolator' gets deallocated.
}
// TODO(bmahler): Add additional tests:
// 1. Check that the data has been published to statistics.
// 2. Check that metering is occurring on subsequent resource data.
TEST(MonitorTest, WatchUnwatch)
{
FrameworkID frameworkId;
frameworkId.set_value("framework");
ExecutorID executorId;
executorId.set_value("executor");
ExecutorInfo executorInfo;
executorInfo.mutable_executor_id()->CopyFrom(executorId);
executorInfo.mutable_framework_id()->CopyFrom(frameworkId);
executorInfo.set_name("name");
executorInfo.set_source("source");
ResourceStatistics initialStatistics;
initialStatistics.set_cpus_user_time_secs(0);
initialStatistics.set_cpus_system_time_secs(0);
initialStatistics.set_cpus_limit(2.5);
initialStatistics.set_mem_rss_bytes(0);
initialStatistics.set_mem_limit_bytes(2048);
initialStatistics.set_timestamp(Clock::now().secs());
ResourceStatistics statistics;
statistics.set_cpus_nr_periods(100);
statistics.set_cpus_nr_throttled(2);
statistics.set_cpus_user_time_secs(4);
statistics.set_cpus_system_time_secs(1);
statistics.set_cpus_throttled_time_secs(0.5);
statistics.set_cpus_limit(2.5);
statistics.set_mem_rss_bytes(1024);
statistics.set_mem_limit_bytes(2048);
statistics.set_timestamp(
initialStatistics.timestamp() +
slave::RESOURCE_MONITORING_INTERVAL.secs());
TestingIsolator isolator;
process::spawn(isolator);
Future<Nothing> usage1, usage2;
EXPECT_CALL(isolator, usage(frameworkId, executorId))
.WillOnce(DoAll(FutureSatisfy(&usage1),
Return(initialStatistics)))
.WillOnce(DoAll(FutureSatisfy(&usage2),
Return(statistics)));
slave::ResourceMonitor monitor(&isolator);
// We pause the clock first in order to make sure that we can
// advance time below to force the 'delay' in
// ResourceMonitorProcess::watch to execute.
process::Clock::pause();
monitor.watch(
frameworkId,
executorId,
executorInfo,
slave::RESOURCE_MONITORING_INTERVAL);
// Now wait for ResouorceMonitorProcess::watch to finish so we can
// advance time to cause collection to begin.
process::Clock::settle();
process::Clock::advance(slave::RESOURCE_MONITORING_INTERVAL);
process::Clock::settle();
AWAIT_READY(usage1);
// Wait until the isolator has finished returning the statistics.
process::Clock::settle();
// The second collection will populate the cpus_usage.
process::Clock::advance(slave::RESOURCE_MONITORING_INTERVAL);
process::Clock::settle();
AWAIT_READY(usage2);
// Wait until the isolator has finished returning the statistics.
process::Clock::settle();
process::UPID upid("monitor", process::ip(), process::port());
Future<Response> response = process::http::get(upid, "usage.json");
AWAIT_EXPECT_RESPONSE_STATUS_EQ(OK().status, response);
AWAIT_EXPECT_RESPONSE_HEADER_EQ(
"application/json",
"Content-Type",
response);
// TODO(bmahler): Verify metering directly through statistics.
AWAIT_EXPECT_RESPONSE_BODY_EQ(
strings::format(
"[{"
"\"executor_id\":\"executor\","
"\"executor_name\":\"name\","
"\"framework_id\":\"framework\","
"\"resource_usage\":{"
"\"cpu_time\":%g,"
"\"cpu_usage\":%g,"
"\"memory_rss\":%lu"
"},"
"\"source\":\"source\""
"}]",
statistics.cpus_system_time_secs() + statistics.cpus_user_time_secs(),
(statistics.cpus_system_time_secs() +
statistics.cpus_user_time_secs()) /
slave::RESOURCE_MONITORING_INTERVAL.secs(),
statistics.mem_rss_bytes()).get(),
response);
response = process::http::get(upid, "statistics.json");
AWAIT_EXPECT_RESPONSE_STATUS_EQ(OK().status, response);
AWAIT_EXPECT_RESPONSE_HEADER_EQ(
"application/json",
"Content-Type",
response);
// TODO(bmahler): Verify metering directly through statistics.
AWAIT_EXPECT_RESPONSE_BODY_EQ(
strings::format(
"[{"
"\"executor_id\":\"executor\","
"\"executor_name\":\"name\","
"\"framework_id\":\"framework\","
"\"source\":\"source\","
"\"statistics\":{"
"\"cpus_limit\":%g,"
"\"cpus_nr_periods\":%d,"
"\"cpus_nr_throttled\":%d,"
"\"cpus_system_time_secs\":%g,"
"\"cpus_throttled_time_secs\":%g,"
"\"cpus_user_time_secs\":%g,"
"\"mem_limit_bytes\":%lu,"
"\"mem_rss_bytes\":%lu"
"}"
"}]",
statistics.cpus_limit(),
statistics.cpus_nr_periods(),
statistics.cpus_nr_throttled(),
statistics.cpus_system_time_secs(),
statistics.cpus_throttled_time_secs(),
statistics.cpus_user_time_secs(),
statistics.mem_limit_bytes(),
statistics.mem_rss_bytes()).get(),
response);
// Ensure the monitor stops polling the isolator.
monitor.unwatch(frameworkId, executorId);
// Wait until ResourceMonitorProcess::unwatch has completed.
process::Clock::settle();
// This time, Isolator::usage should not get called.
EXPECT_CALL(isolator, usage(frameworkId, executorId))
.Times(0);
process::Clock::advance(slave::RESOURCE_MONITORING_INTERVAL);
process::Clock::settle();
response = process::http::get(upid, "usage.json");
AWAIT_EXPECT_RESPONSE_STATUS_EQ(OK().status, response);
AWAIT_EXPECT_RESPONSE_HEADER_EQ(
"application/json",
"Content-Type",
response);
AWAIT_EXPECT_RESPONSE_BODY_EQ("[]", response);
}
inline bool operator==(const ExecutorID& left, const ExecutorID& right)
{
return left.value() == right.value();
}
Try<ExecutorState> ExecutorState::recover(
const string& rootDir,
const SlaveID& slaveId,
const FrameworkID& frameworkId,
const ExecutorID& executorId,
bool strict,
bool rebooted)
{
ExecutorState state;
state.id = executorId;
string message;
// Find the runs.
Try<list<string>> runs = paths::getExecutorRunPaths(
rootDir,
slaveId,
frameworkId,
executorId);
if (runs.isError()) {
return Error("Failed to find runs for executor '" + executorId.value() +
"': " + runs.error());
}
// Recover the runs.
foreach (const string& path, runs.get()) {
if (Path(path).basename() == paths::LATEST_SYMLINK) {
const Result<string>& latest = os::realpath(path);
if (!latest.isSome()) {
return Error(
"Failed to find latest run of executor '" +
executorId.value() + "': " +
(latest.isError()
? latest.error()
: "No such file or directory"));
}
// Store the ContainerID of the latest executor run.
ContainerID containerId;
containerId.set_value(Path(latest.get()).basename());
state.latest = containerId;
} else {
ContainerID containerId;
containerId.set_value(Path(path).basename());
Try<RunState> run = RunState::recover(
rootDir,
slaveId,
frameworkId,
executorId,
containerId,
strict,
rebooted);
if (run.isError()) {
return Error(
"Failed to recover run " + containerId.value() +
" of executor '" + executorId.value() +
"': " + run.error());
}
state.runs[containerId] = run.get();
state.errors += run->errors;
}
}
// Find the latest executor.
// It is possible that we cannot find the "latest" executor if the
// slave died before it created the "latest" symlink.
if (state.latest.isNone()) {
LOG(WARNING) << "Failed to find the latest run of executor '"
<< executorId << "' of framework " << frameworkId;
return state;
}
// Read the executor info.
const string& path =
paths::getExecutorInfoPath(rootDir, slaveId, frameworkId, executorId);
if (!os::exists(path)) {
// This could happen if the slave died after creating the executor
// directory but before it checkpointed the executor info.
LOG(WARNING) << "Failed to find executor info file '" << path << "'";
return state;
}
Result<ExecutorInfo> executorInfo = state::read<ExecutorInfo>(path);
if (executorInfo.isError()) {
message = "Failed to read executor info from '" + path + "': " +
executorInfo.error();
if (strict) {
return Error(message);
} else {
LOG(WARNING) << message;
state.errors++;
return state;
}
}
if (executorInfo.isNone()) {
// This could happen if the slave is hard rebooted after the file is created
// but before the data is synced on disk.
LOG(WARNING) << "Found empty executor info file '" << path << "'";
return state;
}
state.info = executorInfo.get();
return state;
}
// This test has been temporarily disabled due to MESOS-1257.
TEST_F(ExternalContainerizerTest, DISABLED_Launch)
{
Try<PID<Master> > master = this->StartMaster();
ASSERT_SOME(master);
Flags testFlags;
slave::Flags flags = this->CreateSlaveFlags();
flags.isolation = "external";
flags.containerizer_path =
testFlags.build_dir + "/src/examples/python/test-containerizer";
MockExternalContainerizer containerizer(flags);
Try<PID<Slave> > slave = this->StartSlave(&containerizer, flags);
ASSERT_SOME(slave);
MockScheduler sched;
MesosSchedulerDriver driver(
&sched, DEFAULT_FRAMEWORK_INFO, master.get(), DEFAULT_CREDENTIAL);
Future<FrameworkID> frameworkId;
EXPECT_CALL(sched, registered(&driver, _, _))
.WillOnce(FutureArg<1>(&frameworkId));
Future<vector<Offer> > offers;
EXPECT_CALL(sched, resourceOffers(&driver, _))
.WillOnce(FutureArg<1>(&offers))
.WillRepeatedly(Return()); // Ignore subsequent offers.
driver.start();
AWAIT_READY(frameworkId);
AWAIT_READY(offers);
EXPECT_NE(0u, offers.get().size());
TaskInfo task;
task.set_name("isolator_test");
task.mutable_task_id()->set_value("1");
task.mutable_slave_id()->CopyFrom(offers.get()[0].slave_id());
task.mutable_resources()->CopyFrom(offers.get()[0].resources());
Resources resources(offers.get()[0].resources());
Option<Bytes> mem = resources.mem();
ASSERT_SOME(mem);
Option<double> cpus = resources.cpus();
ASSERT_SOME(cpus);
const std::string& file = path::join(flags.work_dir, "ready");
// This task induces user/system load in a child process by
// running top in a child process for ten seconds.
task.mutable_command()->set_value(
#ifdef __APPLE__
// Use logging mode with 30,000 samples with no interval.
"top -l 30000 -s 0 2>&1 > /dev/null & "
#else
// Batch mode, with 30,000 samples with no interval.
"top -b -d 0 -n 30000 2>&1 > /dev/null & "
#endif
"touch " + file + "; " // Signals that the top command is running.
"sleep 60");
Future<TaskStatus> status;
EXPECT_CALL(sched, statusUpdate(&driver, _))
.WillOnce(FutureArg<1>(&status))
.WillRepeatedly(Return()); // Ignore rest for now.
Future<ContainerID> containerId;
EXPECT_CALL(containerizer, launch(_, _, _, _, _, _, _, _))
.WillOnce(DoAll(FutureArg<0>(&containerId),
Invoke(&containerizer,
&MockExternalContainerizer::_launch)));
driver.launchTasks(offers.get()[0].id(), {task});
AWAIT_READY(containerId);
AWAIT_READY(status);
EXPECT_EQ(TASK_RUNNING, status.get().state());
// Wait for the task to begin inducing cpu time.
while (!os::exists(file));
ExecutorID executorId;
executorId.set_value(task.task_id().value());
// We'll wait up to 10 seconds for the child process to induce
// 1/8 of a second of user and system cpu time in total.
// TODO(bmahler): Also induce rss memory consumption, by re-using
// the balloon framework.
ResourceStatistics statistics;
Duration waited = Duration::zero();
do {
Future<ResourceStatistics> usage = containerizer.usage(containerId.get());
AWAIT_READY(usage);
statistics = usage.get();
// If we meet our usage expectations, we're done!
// NOTE: We are currently getting dummy-data from the test-
// containerizer python script matching these expectations.
// TODO(tillt): Consider working with real data.
if (statistics.cpus_user_time_secs() >= 0.120 &&
statistics.cpus_system_time_secs() >= 0.05 &&
statistics.mem_rss_bytes() >= 1024u) {
break;
}
os::sleep(Milliseconds(100));
waited += Milliseconds(100);
} while (waited < Seconds(10));
EXPECT_GE(statistics.cpus_user_time_secs(), 0.120);
EXPECT_GE(statistics.cpus_system_time_secs(), 0.05);
EXPECT_EQ(statistics.cpus_limit(), cpus.get());
EXPECT_GE(statistics.mem_rss_bytes(), 1024u);
EXPECT_EQ(statistics.mem_limit_bytes(), mem.get().bytes());
EXPECT_CALL(sched, statusUpdate(&driver, _))
.WillOnce(FutureArg<1>(&status));
driver.killTask(task.task_id());
AWAIT_READY(status);
EXPECT_EQ(TASK_KILLED, status.get().state());
driver.stop();
driver.join();
this->Shutdown();
}
// Ensures the scheduler driver can handle the UPDATE event.
TEST_F(SchedulerDriverEventTest, Update)
{
Try<PID<Master>> master = StartMaster();
ASSERT_SOME(master);
MockScheduler sched;
MesosSchedulerDriver driver(
&sched, DEFAULT_FRAMEWORK_INFO, master.get(), DEFAULT_CREDENTIAL);
EXPECT_CALL(sched, registered(&driver, _, _));
Future<Message> frameworkRegisteredMessage =
FUTURE_MESSAGE(Eq(FrameworkRegisteredMessage().GetTypeName()), _, _);
driver.start();
AWAIT_READY(frameworkRegisteredMessage);
UPID frameworkPid = frameworkRegisteredMessage.get().to;
FrameworkRegisteredMessage message;
ASSERT_TRUE(message.ParseFromString(frameworkRegisteredMessage.get().body));
FrameworkID frameworkId = message.framework_id();
SlaveID slaveId;
slaveId.set_value("S");
TaskID taskId;
taskId.set_value("T");
ExecutorID executorId;
executorId.set_value("E");
// Generate an update that needs no acknowledgement.
Event event;
event.set_type(Event::UPDATE);
event.mutable_update()->mutable_status()->CopyFrom(
protobuf::createStatusUpdate(
frameworkId,
slaveId,
taskId,
TASK_RUNNING,
TaskStatus::SOURCE_MASTER,
None(),
"message",
None(),
executorId).status());
Future<Nothing> statusUpdate;
Future<Nothing> statusUpdate2;
EXPECT_CALL(sched, statusUpdate(&driver, event.update().status()))
.WillOnce(FutureSatisfy(&statusUpdate))
.WillOnce(FutureSatisfy(&statusUpdate2));
process::post(master.get(), frameworkPid, event);
AWAIT_READY(statusUpdate);
// Generate an update that requires acknowledgement.
event.mutable_update()->mutable_status()->set_uuid(UUID::random().toBytes());
Future<mesos::scheduler::Call> acknowledgement = DROP_CALL(
mesos::scheduler::Call(), mesos::scheduler::Call::ACKNOWLEDGE, _, _);
process::post(master.get(), frameworkPid, event);
AWAIT_READY(statusUpdate2);
AWAIT_READY(acknowledgement);
}
// This test ensures we don't break the API when it comes to JSON
// representation of tasks. Also, we want to ensure that tasks are
// modeled the same way when using 'Task' vs. 'TaskInfo'.
TEST(HTTP, ModelTask)
{
TaskID taskId;
taskId.set_value("t");
SlaveID slaveId;
slaveId.set_value("s");
ExecutorID executorId;
executorId.set_value("t");
FrameworkID frameworkId;
frameworkId.set_value("f");
TaskState state = TASK_RUNNING;
vector<TaskStatus> statuses;
TaskStatus status;
status.mutable_task_id()->CopyFrom(taskId);
status.set_state(state);
status.mutable_slave_id()->CopyFrom(slaveId);
status.mutable_executor_id()->CopyFrom(executorId);
status.set_timestamp(0.0);
statuses.push_back(status);
TaskInfo task;
task.set_name("task");
task.mutable_task_id()->CopyFrom(taskId);
task.mutable_slave_id()->CopyFrom(slaveId);
task.mutable_command()->set_value("echo hello");
Task task_ = protobuf::createTask(task, state, frameworkId);
task_.add_statuses()->CopyFrom(statuses[0]);
JSON::Value object = model(task, frameworkId, state, statuses);
JSON::Value object_ = model(task_);
Try<JSON::Value> expected = JSON::parse(
"{"
" \"executor_id\":\"\","
" \"framework_id\":\"f\","
" \"id\":\"t\","
" \"name\":\"task\","
" \"resources\":"
" {"
" \"cpus\":0,"
" \"disk\":0,"
" \"mem\":0"
" },"
" \"slave_id\":\"s\","
" \"state\":\"TASK_RUNNING\","
" \"statuses\":"
" ["
" {"
" \"state\":\"TASK_RUNNING\","
" \"timestamp\":0"
" }"
" ]"
"}");
ASSERT_SOME(expected);
EXPECT_EQ(expected.get(), object);
EXPECT_EQ(expected.get(), object_);
// Ensure both are modeled the same.
EXPECT_EQ(object, object_);
}
// This test ensures we don't break the API when it comes to JSON
// representation of tasks.
TEST(HTTPTest, ModelTask)
{
TaskID taskId;
taskId.set_value("t");
SlaveID slaveId;
slaveId.set_value("s");
ExecutorID executorId;
executorId.set_value("t");
FrameworkID frameworkId;
frameworkId.set_value("f");
TaskState state = TASK_RUNNING;
vector<TaskStatus> statuses;
TaskStatus status;
status.mutable_task_id()->CopyFrom(taskId);
status.set_state(state);
status.mutable_slave_id()->CopyFrom(slaveId);
status.mutable_executor_id()->CopyFrom(executorId);
status.set_timestamp(0.0);
statuses.push_back(status);
Labels labels;
labels.add_labels()->CopyFrom(createLabel("ACTION", "port:7987 DENY"));
Ports ports;
Port* port = ports.add_ports();
port->set_number(80);
port->mutable_labels()->CopyFrom(labels);
DiscoveryInfo discovery;
discovery.set_visibility(DiscoveryInfo::CLUSTER);
discovery.set_name("discover");
discovery.mutable_ports()->CopyFrom(ports);
TaskInfo taskInfo;
taskInfo.set_name("task");
taskInfo.mutable_task_id()->CopyFrom(taskId);
taskInfo.mutable_slave_id()->CopyFrom(slaveId);
taskInfo.mutable_command()->set_value("echo hello");
taskInfo.mutable_discovery()->CopyFrom(discovery);
Task task = createTask(taskInfo, state, frameworkId);
task.add_statuses()->CopyFrom(statuses[0]);
JSON::Value object = model(task);
Try<JSON::Value> expected = JSON::parse(
"{"
" \"executor_id\":\"\","
" \"framework_id\":\"f\","
" \"id\":\"t\","
" \"name\":\"task\","
" \"resources\":"
" {"
" \"cpus\":0,"
" \"disk\":0,"
" \"gpus\":0,"
" \"mem\":0"
" },"
" \"slave_id\":\"s\","
" \"state\":\"TASK_RUNNING\","
" \"statuses\":"
" ["
" {"
" \"state\":\"TASK_RUNNING\","
" \"timestamp\":0"
" }"
" ],"
" \"discovery\":"
" {"
" \"name\":\"discover\","
" \"ports\":"
" {"
" \"ports\":"
" ["
" {"
" \"number\":80,"
" \"labels\":"
" {"
" \"labels\":"
" ["
" {"
" \"key\":\"ACTION\","
" \"value\":\"port:7987 DENY\""
" }"
" ]"
" }"
" }"
" ]"
" },"
" \"visibility\":\"CLUSTER\""
" }"
"}");
ASSERT_SOME(expected);
EXPECT_EQ(expected.get(), object);
}
