ACTION_P2(InvokeRecoverResourcesWithFilters, allocator, timeout)
{
Filters filters;
filters.set_refuse_seconds(timeout);
allocator->real->recoverResources(arg0, arg1, arg2, filters);
}
ACTION_P2(InvokeUnusedWithFilters, allocator, timeout)
{
Filters filters;
filters.set_refuse_seconds(timeout);
process::dispatch(
allocator->real,
&master::allocator::AllocatorProcess::resourcesUnused,
arg0,
arg1,
arg2,
filters);
}
void resourceOffers(
SchedulerDriver* driver,
const vector<Offer>& offers) override
{
LOG(INFO) << "Received " << offers.size()
<< " resource offers. Declining them";
Filters filters;
// Refuse for eternity so Master doesn't send us the same
// offers.
filters.set_refuse_seconds(Duration::max().secs());
for (size_t i = 0; i < offers.size(); i++) {
driver->declineOffer(offers[i].id(), filters);
}
}
void CephSchedulerAgent<T>::resourceOffers(
T* driver,
const vector<Offer>& offers)
{
LOG(INFO) << "Received " << offers.size() << " offers! ";
TaskType taskType;
int token;
int isInitialMonNode = 0;
//handle waiting OSD task, give them osdID to start docker
handleWaitingOSDTasks(driver);
Phase currentPhase = stateMachine->getCurrentPhase();
//try start new node
foreach (const Offer& offer, offers) {
//check offer with the correct role
LOG(INFO) << "Hostname: " << offer.hostname();
if (!hasRole(offer, config->role)) {
LOG(INFO) << "Decline this offer. Host " << offer.hostname() << " don't have correct role:"
<< config->role;
Filters refuse;
refuse.set_refuse_seconds(86400.0);
driver->declineOffer(offer.id(),refuse);
continue;
}
//reload or new hostconfig
stateMachine->addConfig(offer.hostname());
tryLaunchDiskTask(driver, offer, offer.hostname());
bool accept = stateMachine->nextMove(taskType,token,offer.hostname());
if (!accept) {
LOG(INFO) << "In the "
<< static_cast<int>(currentPhase)
<< " Staging Phase, cannot accept offer from "
<< offer.hostname()
<< " in this phase";
driver->declineOffer(offer.id());
continue;
}
LOG(INFO) << "Check offer's resources from " <<offer.hostname();
if (offerNotEnoughResources(offer,taskType)) {
LOG(INFO) << "Not enough, decline it from " << offer.hostname();
driver->declineOffer(offer.id());
continue;
}
if (currentPhase == Phase::WAINTING_REQUEST){
accept = fetchPendingRESTfulRequest();
if (!accept){
LOG(INFO) << "No pending OSD RESTful request.";
driver->declineOffer(offer.id());
stateMachine->decreaseOSDIndex();
continue;
}
}
LOG(INFO) << "Accepted offer from" << offer.hostname() << ", launch "
<< static_cast<int>(taskType) <<":" << token << " node";
if (taskType == TaskType::MON && token == 0) {
LOG(INFO) << "This is the initial MON";
isInitialMonNode = 1;
}
string taskId;
string executorId;
launchNode(
driver,
offer,
taskType,
token,
isInitialMonNode,
taskId,
executorId);
stateMachine->addStagingTask(
taskId,
executorId,
taskType,
offer.hostname(),
offer.slave_id().value());
if (!isInitialMonNode && taskType == TaskType::OSD) {
ceph::TaskState initialMon = stateMachine->getInitialMon();
const string m = lexical_cast<string>(static_cast<int>(MessageToExecutor::REGISTER_OSD));
ExecutorID eId;
eId.set_value(initialMon.executorId);
SlaveID sId;
sId.set_value(initialMon.slaveId);
driver->sendFrameworkMessage(
eId,
sId,
m);
}//end if
}//end foreach
// This test verifies that the framework can launch a command task
// that specifies both container image and persistent volumes.
TEST_F(LinuxFilesystemIsolatorMesosTest,
ROOT_ChangeRootFilesystemCommandExecutorPersistentVolume)
{
Try<Owned<cluster::Master>> master = StartMaster();
ASSERT_SOME(master);
string registry = path::join(sandbox.get(), "registry");
AWAIT_READY(DockerArchive::create(registry, "test_image"));
slave::Flags flags = CreateSlaveFlags();
flags.resources = "cpus:2;mem:1024;disk(role1):1024";
flags.isolation = "filesystem/linux,docker/runtime";
flags.docker_registry = registry;
flags.docker_store_dir = path::join(sandbox.get(), "store");
flags.image_providers = "docker";
Owned<MasterDetector> detector = master.get()->createDetector();
Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), flags);
ASSERT_SOME(slave);
MockScheduler sched;
FrameworkInfo frameworkInfo = DEFAULT_FRAMEWORK_INFO;
frameworkInfo.set_roles(0, "role1");
MesosSchedulerDriver driver(
&sched,
frameworkInfo,
master.get()->pid,
DEFAULT_CREDENTIAL);
Future<FrameworkID> frameworkId;
EXPECT_CALL(sched, registered(&driver, _, _))
.WillOnce(FutureArg<1>(&frameworkId));
Future<vector<Offer>> offers;
EXPECT_CALL(sched, resourceOffers(&driver, _))
.WillOnce(FutureArg<1>(&offers))
.WillRepeatedly(Return()); // Ignore subsequent offers.
driver.start();
AWAIT_READY(frameworkId);
AWAIT_READY(offers);
ASSERT_FALSE(offers->empty());
Offer offer = offers.get()[0];
string dir1 = path::join(sandbox.get(), "dir1");
ASSERT_SOME(os::mkdir(dir1));
Resource persistentVolume = createPersistentVolume(
Megabytes(64),
"role1",
"id1",
"path1",
None(),
None(),
frameworkInfo.principal());
// We use the filter explicitly here so that the resources will not
// be filtered for 5 seconds (the default).
Filters filters;
filters.set_refuse_seconds(0);
TaskInfo task = createTask(
offer.slave_id(),
Resources::parse("cpus:1;mem:512").get() + persistentVolume,
"echo abc > path1/file");
task.mutable_container()->CopyFrom(createContainerInfo(
"test_image",
{createVolumeHostPath("/tmp", dir1, Volume::RW)}));
// Create the persistent volumes and launch task via `acceptOffers`.
driver.acceptOffers(
{offer.id()},
{CREATE(persistentVolume), LAUNCH({task})},
filters);
Future<TaskStatus> statusStarting;
Future<TaskStatus> statusRunning;
Future<TaskStatus> statusFinished;
EXPECT_CALL(sched, statusUpdate(&driver, _))
.WillOnce(FutureArg<1>(&statusStarting))
.WillOnce(FutureArg<1>(&statusRunning))
.WillOnce(FutureArg<1>(&statusFinished));
AWAIT_READY(statusStarting);
EXPECT_EQ(TASK_STARTING, statusStarting->state());
AWAIT_READY(statusRunning);
EXPECT_EQ(TASK_RUNNING, statusRunning->state());
AWAIT_READY(statusFinished);
EXPECT_EQ(TASK_FINISHED, statusFinished->state());
// NOTE: The command executor's id is the same as the task id.
ExecutorID executorId;
executorId.set_value(task.task_id().value());
string directory = slave::paths::getExecutorLatestRunPath(
flags.work_dir,
offer.slave_id(),
frameworkId.get(),
executorId);
EXPECT_FALSE(os::exists(path::join(directory, "path1")));
string volumePath = slave::paths::getPersistentVolumePath(
flags.work_dir,
"role1",
"id1");
EXPECT_SOME_EQ("abc\n", os::read(path::join(volumePath, "file")));
driver.stop();
driver.join();
}
virtual void resourceOffers(SchedulerDriver* driver,
const vector<Offer>& offers)
{
foreach (const Offer& offer, offers) {
LOG(INFO) << "Received offer " << offer.id() << " with "
<< offer.resources();
// If the framework got this offer for the first time, the state is
// `State::INIT`; framework will reserve it (sending RESERVE operation
// to master) in this loop.
if (!states.contains(offer.slave_id())) {
// If all tasks were launched, do not reserve more resources; wait
// for them to finish and unreserve resources.
if (tasksLaunched == totalTasks) {
continue;
}
states[offer.slave_id()] = State::INIT;
}
const State state = states[offer.slave_id()];
Filters filters;
filters.set_refuse_seconds(0);
switch (state) {
case State::INIT: {
// Framework reserves resources from this offer for only one task;
// the task'll be dispatched when reserved resources are re-offered
// to this framework.
Resources resources = offer.resources();
Offer::Operation reserve = RESERVE(taskResources);
Try<Resources> apply = resources.apply(reserve);
if (apply.isError()) {
LOG(INFO) << "Failed to reserve resources for task in offer "
<< stringify(offer.id()) << ": " << apply.error();
break;
}
driver->acceptOffers({offer.id()}, {reserve}, filters);
states[offer.slave_id()] = State::RESERVING;
break;
}
case State::RESERVING: {
Resources resources = offer.resources();
Resources reserved = resources.reserved(role);
if (!reserved.contains(taskResources)) {
break;
}
states[offer.slave_id()] = State::RESERVED;
// We fallthrough here to save an offer cycle.
}
case State::RESERVED: {
Resources resources = offer.resources();
Resources reserved = resources.reserved(role);
CHECK(reserved.contains(taskResources));
// If all tasks were launched, unreserve those resources.
if (tasksLaunched == totalTasks) {
driver->acceptOffers(
{offer.id()}, {UNRESERVE(taskResources)}, filters);
states[offer.slave_id()] = State::UNRESERVING;
break;
}
// Framework dispatches task on the reserved resources.
CHECK(tasksLaunched < totalTasks);
// Launch tasks on reserved resources.
const string& taskId = stringify(tasksLaunched++);
LOG(INFO) << "Launching task " << taskId << " using offer "
<< offer.id();
TaskInfo task;
task.set_name("Task " + taskId + ": " + command);
task.mutable_task_id()->set_value(taskId);
task.mutable_slave_id()->MergeFrom(offer.slave_id());
task.mutable_command()->set_shell(true);
task.mutable_command()->set_value(command);
task.mutable_resources()->MergeFrom(taskResources);
driver->launchTasks(offer.id(), {task}, filters);
states[offer.slave_id()] = State::TASK_RUNNING;
break;
}
case State::TASK_RUNNING:
LOG(INFO) << "The task on " << offer.slave_id()
<< " is running, waiting for task done";
break;
case State::UNRESERVING: {
Resources resources = offer.resources();
Resources reserved = resources.reserved(role);
if (!reserved.contains(taskResources)) {
states[offer.slave_id()] = State::UNRESERVED;
}
break;
}
case State::UNRESERVED:
// If state of slave is UNRESERVED, ignore it. The driver is stopped
// when all tasks are done and all resources are unreserved.
break;
}
}