virtual void resourceOffers(SchedulerDriver* driver,
const vector<Offer>& offers)
{
cout << "." << flush;
vector<Offer>::const_iterator iterator = offers.begin();
for (; iterator != offers.end(); ++iterator) {
const Offer& offer = *iterator;
// Lookup resources we care about.
// TODO(benh): It would be nice to ultimately have some helper
// functions for looking up resources.
double cpus = 0;
double mem = 0;
for (int i = 0; i < offer.resources_size(); i++) {
const Resource& resource = offer.resources(i);
if (resource.name() == "cpus" &&
resource.type() == Value::SCALAR) {
cpus = resource.scalar().value();
} else if (resource.name() == "mem" &&
resource.type() == Value::SCALAR) {
mem = resource.scalar().value();
}
}
// Launch tasks (only one per offer).
vector<TaskDescription> tasks;
if (cpus >= CPUS_PER_TASK && mem >= MEM_PER_TASK) {
int taskId = tasksLaunched++;
cout << "Starting task " << taskId << " on "
<< offer.hostname() << endl;
TaskDescription task;
task.set_name("Task " + lexical_cast<string>(taskId));
task.mutable_task_id()->set_value(lexical_cast<string>(taskId));
task.mutable_slave_id()->MergeFrom(offer.slave_id());
Resource* resource;
resource = task.add_resources();
resource->set_name("cpus");
resource->set_type(Value::SCALAR);
resource->mutable_scalar()->set_value(CPUS_PER_TASK);
resource = task.add_resources();
resource->set_name("mem");
resource->set_type(Value::SCALAR);
resource->mutable_scalar()->set_value(MEM_PER_TASK);
tasks.push_back(task);
cpus -= CPUS_PER_TASK;
mem -= MEM_PER_TASK;
}
driver->launchTasks(offer.id(), tasks);
}
}
TEST(ResourceOffersTest, TaskUsesNoResources)
{
ASSERT_TRUE(GTEST_IS_THREADSAFE);
PID<Master> master = local::launch(1, 2, 1 * Gigabyte, false);
MockScheduler sched;
MesosSchedulerDriver driver(&sched, "", DEFAULT_EXECUTOR_INFO, master);
vector<Offer> offers;
trigger resourceOffersCall;
EXPECT_CALL(sched, registered(&driver, _))
.Times(1);
EXPECT_CALL(sched, resourceOffers(&driver, _))
.WillOnce(DoAll(SaveArg<1>(&offers),
Trigger(&resourceOffersCall)))
.WillRepeatedly(Return());
driver.start();
WAIT_UNTIL(resourceOffersCall);
EXPECT_NE(0, offers.size());
TaskDescription task;
task.set_name("");
task.mutable_task_id()->set_value("1");
task.mutable_slave_id()->MergeFrom(offers[0].slave_id());
vector<TaskDescription> tasks;
tasks.push_back(task);
TaskStatus status;
trigger statusUpdateCall;
EXPECT_CALL(sched, statusUpdate(&driver, _))
.WillOnce(DoAll(SaveArg<1>(&status),
Trigger(&statusUpdateCall)));
driver.launchTasks(offers[0].id(), tasks);
WAIT_UNTIL(statusUpdateCall);
EXPECT_EQ(task.task_id(), status.task_id());
EXPECT_EQ(TASK_LOST, status.state());
EXPECT_TRUE(status.has_message());
EXPECT_EQ("Task uses no resources", status.message());
driver.stop();
driver.join();
local::shutdown();
}
TEST(MasterTest, TaskRunning)
{
ASSERT_TRUE(GTEST_IS_THREADSAFE);
SimpleAllocator a;
Master m(&a);
PID<Master> master = process::spawn(&m);
MockExecutor exec;
trigger shutdownCall;
EXPECT_CALL(exec, registered(_, _, _, _, _, _))
.Times(1);
EXPECT_CALL(exec, launchTask(_, _))
.WillOnce(SendStatusUpdate(TASK_RUNNING));
EXPECT_CALL(exec, shutdown(_))
.WillOnce(Trigger(&shutdownCall));
map<ExecutorID, Executor*> execs;
execs[DEFAULT_EXECUTOR_ID] = &exec;
TestingIsolationModule isolationModule(execs);
Resources resources = Resources::parse("cpus:2;mem:1024");
Slave s(resources, true, &isolationModule);
PID<Slave> slave = process::spawn(&s);
BasicMasterDetector detector(master, slave, true);
MockScheduler sched;
MesosSchedulerDriver driver(&sched, "", DEFAULT_EXECUTOR_INFO, master);
vector<Offer> offers;
TaskStatus status;
trigger resourceOffersCall, statusUpdateCall, resourcesChangedCall;
EXPECT_CALL(sched, registered(&driver, _))
.Times(1);
EXPECT_CALL(sched, resourceOffers(&driver, _))
.WillOnce(DoAll(SaveArg<1>(&offers),
Trigger(&resourceOffersCall)))
.WillRepeatedly(Return());
EXPECT_CALL(sched, statusUpdate(&driver, _))
.WillOnce(DoAll(SaveArg<1>(&status), Trigger(&statusUpdateCall)));
driver.start();
WAIT_UNTIL(resourceOffersCall);
EXPECT_NE(0, offers.size());
TaskDescription task;
task.set_name("");
task.mutable_task_id()->set_value("1");
task.mutable_slave_id()->MergeFrom(offers[0].slave_id());
task.mutable_resources()->MergeFrom(offers[0].resources());
vector<TaskDescription> tasks;
tasks.push_back(task);
EXPECT_CALL(isolationModule,
resourcesChanged(_, _, Resources(offers[0].resources())))
.WillOnce(Trigger(&resourcesChangedCall));
driver.launchTasks(offers[0].id(), tasks);
WAIT_UNTIL(statusUpdateCall);
EXPECT_EQ(TASK_RUNNING, status.state());
WAIT_UNTIL(resourcesChangedCall);
driver.stop();
driver.join();
WAIT_UNTIL(shutdownCall); // To ensure can deallocate MockExecutor.
process::terminate(slave);
process::wait(slave);
process::terminate(master);
process::wait(master);
}
TEST(MasterTest, FrameworkMessage)
{
ASSERT_TRUE(GTEST_IS_THREADSAFE);
SimpleAllocator a;
Master m(&a);
PID<Master> master = process::spawn(&m);
MockExecutor exec;
ExecutorDriver* execDriver;
string execData;
trigger execFrameworkMessageCall, shutdownCall;
EXPECT_CALL(exec, registered(_, _, _, _, _, _))
.WillOnce(SaveArg<0>(&execDriver));
EXPECT_CALL(exec, launchTask(_, _))
.WillOnce(SendStatusUpdate(TASK_RUNNING));
EXPECT_CALL(exec, frameworkMessage(_, _))
.WillOnce(DoAll(SaveArg<1>(&execData),
Trigger(&execFrameworkMessageCall)));
EXPECT_CALL(exec, shutdown(_))
.WillOnce(Trigger(&shutdownCall));
map<ExecutorID, Executor*> execs;
execs[DEFAULT_EXECUTOR_ID] = &exec;
TestingIsolationModule isolationModule(execs);
Resources resources = Resources::parse("cpus:2;mem:1024");
Slave s(resources, true, &isolationModule);
PID<Slave> slave = process::spawn(&s);
BasicMasterDetector detector(master, slave, true);
// Launch the first (i.e., failing) scheduler and wait until the
// first status update message is sent to it (drop the message).
MockScheduler sched;
MesosSchedulerDriver schedDriver(&sched, "", DEFAULT_EXECUTOR_INFO, master);
vector<Offer> offers;
TaskStatus status;
string schedData;
trigger resourceOffersCall, statusUpdateCall, schedFrameworkMessageCall;
EXPECT_CALL(sched, registered(&schedDriver, _))
.Times(1);
EXPECT_CALL(sched, resourceOffers(&schedDriver, _))
.WillOnce(DoAll(SaveArg<1>(&offers),
Trigger(&resourceOffersCall)))
.WillRepeatedly(Return());
EXPECT_CALL(sched, statusUpdate(&schedDriver, _))
.WillOnce(DoAll(SaveArg<1>(&status), Trigger(&statusUpdateCall)));
EXPECT_CALL(sched, frameworkMessage(&schedDriver, _, _, _))
.WillOnce(DoAll(SaveArg<3>(&schedData),
Trigger(&schedFrameworkMessageCall)));
schedDriver.start();
WAIT_UNTIL(resourceOffersCall);
EXPECT_NE(0, offers.size());
TaskDescription task;
task.set_name("");
task.mutable_task_id()->set_value("1");
task.mutable_slave_id()->MergeFrom(offers[0].slave_id());
task.mutable_resources()->MergeFrom(offers[0].resources());
vector<TaskDescription> tasks;
tasks.push_back(task);
schedDriver.launchTasks(offers[0].id(), tasks);
WAIT_UNTIL(statusUpdateCall);
EXPECT_EQ(TASK_RUNNING, status.state());
string hello = "hello";
schedDriver.sendFrameworkMessage(offers[0].slave_id(),
DEFAULT_EXECUTOR_ID,
hello);
WAIT_UNTIL(execFrameworkMessageCall);
EXPECT_EQ(hello, execData);
string reply = "reply";
execDriver->sendFrameworkMessage(reply);
WAIT_UNTIL(schedFrameworkMessageCall);
EXPECT_EQ(reply, schedData);
schedDriver.stop();
schedDriver.join();
WAIT_UNTIL(shutdownCall); // To ensure can deallocate MockExecutor.
process::terminate(slave);
process::wait(slave);
process::terminate(master);
process::wait(master);
}
TEST(ResourceOffersTest, ResourcesGetReofferedAfterTaskDescriptionError)
{
ASSERT_TRUE(GTEST_IS_THREADSAFE);
PID<Master> master = local::launch(1, 2, 1 * Gigabyte, false);
MockScheduler sched1;
MesosSchedulerDriver driver1(&sched1, "", DEFAULT_EXECUTOR_INFO, master);
vector<Offer> offers;
trigger sched1ResourceOffersCall;
EXPECT_CALL(sched1, registered(&driver1, _))
.Times(1);
EXPECT_CALL(sched1, resourceOffers(&driver1, _))
.WillOnce(DoAll(SaveArg<1>(&offers),
Trigger(&sched1ResourceOffersCall)))
.WillRepeatedly(Return());
driver1.start();
WAIT_UNTIL(sched1ResourceOffersCall);
EXPECT_NE(0, offers.size());
TaskDescription task;
task.set_name("");
task.mutable_task_id()->set_value("1");
task.mutable_slave_id()->MergeFrom(offers[0].slave_id());
Resource* cpus = task.add_resources();
cpus->set_name("cpus");
cpus->set_type(Value::SCALAR);
cpus->mutable_scalar()->set_value(0);
Resource* mem = task.add_resources();
mem->set_name("mem");
mem->set_type(Value::SCALAR);
mem->mutable_scalar()->set_value(1 * Gigabyte);
vector<TaskDescription> tasks;
tasks.push_back(task);
TaskStatus status;
trigger sched1StatusUpdateCall;
EXPECT_CALL(sched1, statusUpdate(&driver1, _))
.WillOnce(DoAll(SaveArg<1>(&status),
Trigger(&sched1StatusUpdateCall)));
driver1.launchTasks(offers[0].id(), tasks);
WAIT_UNTIL(sched1StatusUpdateCall);
EXPECT_EQ(task.task_id(), status.task_id());
EXPECT_EQ(TASK_LOST, status.state());
EXPECT_TRUE(status.has_message());
EXPECT_EQ("Task uses invalid resources", status.message());
driver1.stop();
driver1.join();
MockScheduler sched2;
MesosSchedulerDriver driver2(&sched2, "", DEFAULT_EXECUTOR_INFO, master);
trigger sched2ResourceOffersCall;
EXPECT_CALL(sched2, registered(&driver2, _))
.Times(1);
EXPECT_CALL(sched2, resourceOffers(&driver2, _))
.WillOnce(Trigger(&sched2ResourceOffersCall))
.WillRepeatedly(Return());
EXPECT_CALL(sched2, offerRescinded(&driver2, _))
.Times(AtMost(1));
driver2.start();
WAIT_UNTIL(sched2ResourceOffersCall);
driver2.stop();
driver2.join();
local::shutdown();
}
virtual void resourceOffer(SchedulerDriver* driver,
const OfferID& offerId,
const vector<SlaveOffer>& offers)
{
time_t now = time(0);
double curTime = difftime(now, startTime);
vector<TaskDescription> toLaunch;
vector<SlaveOffer>::const_iterator iterator = offers.begin();
for (; iterator != offers.end(); ++iterator) {
const SlaveOffer& offer = *iterator;
// Lookup resources we care about.
// TODO(benh): It would be nice to ultimately have some helper
// functions for looking up resources.
double cpus = 0;
double mem = 0;
for (int i = 0; i < offer.resources_size(); i++) {
const Resource& resource = offer.resources(i);
if (resource.name() == "cpus" &&
resource.type() == Resource::SCALAR) {
cpus = resource.scalar().value();
} else if (resource.name() == "mem" &&
resource.type() == Resource::SCALAR) {
mem = resource.scalar().value();
}
}
// Launch tasks.
if (tasksLaunched < tasks.size() &&
cpus >= 1 &&
curTime >= tasks[tasksLaunched].launchTime &&
mem >= tasks[tasksLaunched].memToRequest)
{
int taskId = tasksLaunched++;
cout << "Starting task " << taskId << " on "
<< offer.hostname() << endl;
TaskDescription task;
task.set_name("Task " + lexical_cast<string>(taskId));
task.mutable_task_id()->set_value(lexical_cast<string>(taskId));
task.mutable_slave_id()->MergeFrom(offer.slave_id());
Resource* resource;
resource = task.add_resources();
resource->set_name("cpus");
resource->set_type(Resource::SCALAR);
resource->mutable_scalar()->set_value(1);
resource = task.add_resources();
resource->set_name("mem");
resource->set_type(Resource::SCALAR);
resource->mutable_scalar()->set_value(tasks[taskId].memToRequest);
ostringstream data;
data << tasks[taskId].memToHog << " " << tasks[taskId].duration
<< " " << threadsPerTask;
task.set_data(data.str());
toLaunch.push_back(task);
}
}
driver->replyToOffer(offerId, toLaunch);
}
TEST(MasterTest, ResourcesReofferedAfterBadResponse)
{
ASSERT_TRUE(GTEST_IS_THREADSAFE);
PID<Master> master = local::launch(1, 2, 1 * Gigabyte, false, false);
MockScheduler sched1;
MesosSchedulerDriver driver1(&sched1, master);
OfferID offerId;
vector<SlaveOffer> offers;
trigger sched1ResourceOfferCall;
EXPECT_CALL(sched1, getFrameworkName(&driver1))
.WillOnce(Return(""));
EXPECT_CALL(sched1, getExecutorInfo(&driver1))
.WillOnce(Return(DEFAULT_EXECUTOR_INFO));
EXPECT_CALL(sched1, registered(&driver1, _))
.Times(1);
EXPECT_CALL(sched1, resourceOffer(&driver1, _, ElementsAre(_)))
.WillOnce(DoAll(SaveArg<1>(&offerId), SaveArg<2>(&offers),
Trigger(&sched1ResourceOfferCall)))
.WillRepeatedly(Return());
driver1.start();
WAIT_UNTIL(sched1ResourceOfferCall);
EXPECT_NE(0, offers.size());
TaskDescription task;
task.set_name("");
task.mutable_task_id()->set_value("1");
task.mutable_slave_id()->MergeFrom(offers[0].slave_id());
Resource* cpus = task.add_resources();
cpus->set_name("cpus");
cpus->set_type(Resource::SCALAR);
cpus->mutable_scalar()->set_value(0);
Resource* mem = task.add_resources();
mem->set_name("mem");
mem->set_type(Resource::SCALAR);
mem->mutable_scalar()->set_value(1 * Gigabyte);
vector<TaskDescription> tasks;
tasks.push_back(task);
trigger sched1ErrorCall;
EXPECT_CALL(sched1,
error(&driver1, _, "Invalid resources for task"))
.WillOnce(Trigger(&sched1ErrorCall));
EXPECT_CALL(sched1, offerRescinded(&driver1, offerId))
.Times(AtMost(1));
driver1.replyToOffer(offerId, tasks);
WAIT_UNTIL(sched1ErrorCall);
driver1.stop();
driver1.join();
MockScheduler sched2;
MesosSchedulerDriver driver2(&sched2, master);
trigger sched2ResourceOfferCall;
EXPECT_CALL(sched2, getFrameworkName(&driver2))
.WillOnce(Return(""));
EXPECT_CALL(sched2, getExecutorInfo(&driver2))
.WillOnce(Return(DEFAULT_EXECUTOR_INFO));
EXPECT_CALL(sched2, registered(&driver2, _))
.Times(1);
EXPECT_CALL(sched2, resourceOffer(&driver2, _, _))
.WillOnce(Trigger(&sched2ResourceOfferCall))
.WillRepeatedly(Return());
EXPECT_CALL(sched2, offerRescinded(&driver2, _))
.Times(AtMost(1));
driver2.start();
WAIT_UNTIL(sched2ResourceOfferCall);
driver2.stop();
driver2.join();
local::shutdown();
}
