ACTION_P2(InvokeRecoverResourcesWithFilters, allocator, timeout) { Filters filters; filters.set_refuse_seconds(timeout); allocator->real->recoverResources(arg0, arg1, arg2, filters); }
ACTION_P2(InvokeUnusedWithFilters, allocator, timeout) { Filters filters; filters.set_refuse_seconds(timeout); process::dispatch( allocator->real, &master::allocator::AllocatorProcess::resourcesUnused, arg0, arg1, arg2, filters); }
void resourceOffers( SchedulerDriver* driver, const vector<Offer>& offers) override { LOG(INFO) << "Received " << offers.size() << " resource offers. Declining them"; Filters filters; // Refuse for eternity so Master doesn't send us the same // offers. filters.set_refuse_seconds(Duration::max().secs()); for (size_t i = 0; i < offers.size(); i++) { driver->declineOffer(offers[i].id(), filters); } }
void CephSchedulerAgent<T>::resourceOffers( T* driver, const vector<Offer>& offers) { LOG(INFO) << "Received " << offers.size() << " offers! "; TaskType taskType; int token; int isInitialMonNode = 0; //handle waiting OSD task, give them osdID to start docker handleWaitingOSDTasks(driver); Phase currentPhase = stateMachine->getCurrentPhase(); //try start new node foreach (const Offer& offer, offers) { //check offer with the correct role LOG(INFO) << "Hostname: " << offer.hostname(); if (!hasRole(offer, config->role)) { LOG(INFO) << "Decline this offer. Host " << offer.hostname() << " don't have correct role:" << config->role; Filters refuse; refuse.set_refuse_seconds(86400.0); driver->declineOffer(offer.id(),refuse); continue; } //reload or new hostconfig stateMachine->addConfig(offer.hostname()); tryLaunchDiskTask(driver, offer, offer.hostname()); bool accept = stateMachine->nextMove(taskType,token,offer.hostname()); if (!accept) { LOG(INFO) << "In the " << static_cast<int>(currentPhase) << " Staging Phase, cannot accept offer from " << offer.hostname() << " in this phase"; driver->declineOffer(offer.id()); continue; } LOG(INFO) << "Check offer's resources from " <<offer.hostname(); if (offerNotEnoughResources(offer,taskType)) { LOG(INFO) << "Not enough, decline it from " << offer.hostname(); driver->declineOffer(offer.id()); continue; } if (currentPhase == Phase::WAINTING_REQUEST){ accept = fetchPendingRESTfulRequest(); if (!accept){ LOG(INFO) << "No pending OSD RESTful request."; driver->declineOffer(offer.id()); stateMachine->decreaseOSDIndex(); continue; } } LOG(INFO) << "Accepted offer from" << offer.hostname() << ", launch " << static_cast<int>(taskType) <<":" << token << " node"; if (taskType == TaskType::MON && token == 0) { LOG(INFO) << "This is the initial MON"; isInitialMonNode = 1; } string taskId; string executorId; launchNode( driver, offer, taskType, token, isInitialMonNode, taskId, executorId); stateMachine->addStagingTask( taskId, executorId, taskType, offer.hostname(), offer.slave_id().value()); if (!isInitialMonNode && taskType == TaskType::OSD) { ceph::TaskState initialMon = stateMachine->getInitialMon(); const string m = lexical_cast<string>(static_cast<int>(MessageToExecutor::REGISTER_OSD)); ExecutorID eId; eId.set_value(initialMon.executorId); SlaveID sId; sId.set_value(initialMon.slaveId); driver->sendFrameworkMessage( eId, sId, m); }//end if }//end foreach
// This test verifies that the framework can launch a command task // that specifies both container image and persistent volumes. TEST_F(LinuxFilesystemIsolatorMesosTest, ROOT_ChangeRootFilesystemCommandExecutorPersistentVolume) { Try<Owned<cluster::Master>> master = StartMaster(); ASSERT_SOME(master); string registry = path::join(sandbox.get(), "registry"); AWAIT_READY(DockerArchive::create(registry, "test_image")); slave::Flags flags = CreateSlaveFlags(); flags.resources = "cpus:2;mem:1024;disk(role1):1024"; flags.isolation = "filesystem/linux,docker/runtime"; flags.docker_registry = registry; flags.docker_store_dir = path::join(sandbox.get(), "store"); flags.image_providers = "docker"; Owned<MasterDetector> detector = master.get()->createDetector(); Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), flags); ASSERT_SOME(slave); MockScheduler sched; FrameworkInfo frameworkInfo = DEFAULT_FRAMEWORK_INFO; frameworkInfo.set_roles(0, "role1"); MesosSchedulerDriver driver( &sched, frameworkInfo, master.get()->pid, DEFAULT_CREDENTIAL); Future<FrameworkID> frameworkId; EXPECT_CALL(sched, registered(&driver, _, _)) .WillOnce(FutureArg<1>(&frameworkId)); Future<vector<Offer>> offers; EXPECT_CALL(sched, resourceOffers(&driver, _)) .WillOnce(FutureArg<1>(&offers)) .WillRepeatedly(Return()); // Ignore subsequent offers. driver.start(); AWAIT_READY(frameworkId); AWAIT_READY(offers); ASSERT_FALSE(offers->empty()); Offer offer = offers.get()[0]; string dir1 = path::join(sandbox.get(), "dir1"); ASSERT_SOME(os::mkdir(dir1)); Resource persistentVolume = createPersistentVolume( Megabytes(64), "role1", "id1", "path1", None(), None(), frameworkInfo.principal()); // We use the filter explicitly here so that the resources will not // be filtered for 5 seconds (the default). Filters filters; filters.set_refuse_seconds(0); TaskInfo task = createTask( offer.slave_id(), Resources::parse("cpus:1;mem:512").get() + persistentVolume, "echo abc > path1/file"); task.mutable_container()->CopyFrom(createContainerInfo( "test_image", {createVolumeHostPath("/tmp", dir1, Volume::RW)})); // Create the persistent volumes and launch task via `acceptOffers`. driver.acceptOffers( {offer.id()}, {CREATE(persistentVolume), LAUNCH({task})}, filters); Future<TaskStatus> statusStarting; Future<TaskStatus> statusRunning; Future<TaskStatus> statusFinished; EXPECT_CALL(sched, statusUpdate(&driver, _)) .WillOnce(FutureArg<1>(&statusStarting)) .WillOnce(FutureArg<1>(&statusRunning)) .WillOnce(FutureArg<1>(&statusFinished)); AWAIT_READY(statusStarting); EXPECT_EQ(TASK_STARTING, statusStarting->state()); AWAIT_READY(statusRunning); EXPECT_EQ(TASK_RUNNING, statusRunning->state()); AWAIT_READY(statusFinished); EXPECT_EQ(TASK_FINISHED, statusFinished->state()); // NOTE: The command executor's id is the same as the task id. ExecutorID executorId; executorId.set_value(task.task_id().value()); string directory = slave::paths::getExecutorLatestRunPath( flags.work_dir, offer.slave_id(), frameworkId.get(), executorId); EXPECT_FALSE(os::exists(path::join(directory, "path1"))); string volumePath = slave::paths::getPersistentVolumePath( flags.work_dir, "role1", "id1"); EXPECT_SOME_EQ("abc\n", os::read(path::join(volumePath, "file"))); driver.stop(); driver.join(); }
virtual void resourceOffers(SchedulerDriver* driver, const vector<Offer>& offers) { foreach (const Offer& offer, offers) { LOG(INFO) << "Received offer " << offer.id() << " with " << offer.resources(); // If the framework got this offer for the first time, the state is // `State::INIT`; framework will reserve it (sending RESERVE operation // to master) in this loop. if (!states.contains(offer.slave_id())) { // If all tasks were launched, do not reserve more resources; wait // for them to finish and unreserve resources. if (tasksLaunched == totalTasks) { continue; } states[offer.slave_id()] = State::INIT; } const State state = states[offer.slave_id()]; Filters filters; filters.set_refuse_seconds(0); switch (state) { case State::INIT: { // Framework reserves resources from this offer for only one task; // the task'll be dispatched when reserved resources are re-offered // to this framework. Resources resources = offer.resources(); Offer::Operation reserve = RESERVE(taskResources); Try<Resources> apply = resources.apply(reserve); if (apply.isError()) { LOG(INFO) << "Failed to reserve resources for task in offer " << stringify(offer.id()) << ": " << apply.error(); break; } driver->acceptOffers({offer.id()}, {reserve}, filters); states[offer.slave_id()] = State::RESERVING; break; } case State::RESERVING: { Resources resources = offer.resources(); Resources reserved = resources.reserved(role); if (!reserved.contains(taskResources)) { break; } states[offer.slave_id()] = State::RESERVED; // We fallthrough here to save an offer cycle. } case State::RESERVED: { Resources resources = offer.resources(); Resources reserved = resources.reserved(role); CHECK(reserved.contains(taskResources)); // If all tasks were launched, unreserve those resources. if (tasksLaunched == totalTasks) { driver->acceptOffers( {offer.id()}, {UNRESERVE(taskResources)}, filters); states[offer.slave_id()] = State::UNRESERVING; break; } // Framework dispatches task on the reserved resources. CHECK(tasksLaunched < totalTasks); // Launch tasks on reserved resources. const string& taskId = stringify(tasksLaunched++); LOG(INFO) << "Launching task " << taskId << " using offer " << offer.id(); TaskInfo task; task.set_name("Task " + taskId + ": " + command); task.mutable_task_id()->set_value(taskId); task.mutable_slave_id()->MergeFrom(offer.slave_id()); task.mutable_command()->set_shell(true); task.mutable_command()->set_value(command); task.mutable_resources()->MergeFrom(taskResources); driver->launchTasks(offer.id(), {task}, filters); states[offer.slave_id()] = State::TASK_RUNNING; break; } case State::TASK_RUNNING: LOG(INFO) << "The task on " << offer.slave_id() << " is running, waiting for task done"; break; case State::UNRESERVING: { Resources resources = offer.resources(); Resources reserved = resources.reserved(role); if (!reserved.contains(taskResources)) { states[offer.slave_id()] = State::UNRESERVED; } break; } case State::UNRESERVED: // If state of slave is UNRESERVED, ignore it. The driver is stopped // when all tasks are done and all resources are unreserved. break; } }