inline StatusUpdate createStatusUpdate( const FrameworkID& frameworkId, const SlaveID& slaveId, const TaskID& taskId, const TaskState& state, const std::string& message = "", const Option<ExecutorID>& executorId = None()) { StatusUpdate update; update.set_timestamp(process::Clock::now().secs()); update.set_uuid(UUID::random().toBytes()); update.mutable_framework_id()->MergeFrom(frameworkId); update.mutable_slave_id()->MergeFrom(slaveId); if (executorId.isSome()) { update.mutable_executor_id()->MergeFrom(executorId.get()); } TaskStatus* status = update.mutable_status(); status->mutable_task_id()->MergeFrom(taskId); status->mutable_slave_id()->MergeFrom(slaveId); status->set_state(state); status->set_message(message); status->set_timestamp(update.timestamp()); return update; }
void update(const TaskInfo& task, const TaskState& state) { UUID uuid = UUID::random(); TaskStatus status; status.mutable_task_id()->CopyFrom(task.task_id()); status.mutable_executor_id()->CopyFrom(executorId); status.set_state(state); status.set_source(TaskStatus::SOURCE_EXECUTOR); status.set_timestamp(process::Clock::now().secs()); status.set_uuid(uuid.toBytes()); Call call; call.mutable_framework_id()->CopyFrom(frameworkId); call.mutable_executor_id()->CopyFrom(executorId); call.set_type(Call::UPDATE); call.mutable_update()->mutable_status()->CopyFrom(status); // Capture the status update. updates[uuid] = call.update(); mesos->send(call); }
// This test ensures that the driver handles an empty slave id // in an acknowledgement message by dropping it. The driver will // log an error in this case (but we don't test for that). We // generate a status with no slave id by performing reconciliation. TEST_F(MesosSchedulerDriverTest, ExplicitAcknowledgementsUnsetSlaveID) { Try<Owned<cluster::Master>> master = StartMaster(); ASSERT_SOME(master); MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, false, DEFAULT_CREDENTIAL); Future<Nothing> registered; EXPECT_CALL(sched, registered(&driver, _, _)) .WillOnce(FutureSatisfy(®istered)); // Ensure no status update acknowledgements are sent to the master. EXPECT_NO_FUTURE_CALLS( mesos::scheduler::Call(), mesos::scheduler::Call::ACKNOWLEDGE, _ , master.get()->pid); driver.start(); AWAIT_READY(registered); Future<TaskStatus> update; EXPECT_CALL(sched, statusUpdate(&driver, _)) .WillOnce(FutureArg<1>(&update)); // Peform reconciliation without using a slave id. vector<TaskStatus> statuses; TaskStatus status; status.mutable_task_id()->set_value("foo"); status.set_state(TASK_RUNNING); statuses.push_back(status); driver.reconcileTasks(statuses); AWAIT_READY(update); ASSERT_EQ(TASK_LOST, update.get().state()); ASSERT_EQ(TaskStatus::SOURCE_MASTER, update.get().source()); ASSERT_EQ(TaskStatus::REASON_RECONCILIATION, update.get().reason()); ASSERT_FALSE(update.get().has_slave_id()); // Now send the acknowledgement. driver.acknowledgeStatusUpdate(update.get()); // Settle the clock to ensure driver processes the acknowledgement, // which should get dropped due to the missing slave id. Clock::pause(); Clock::settle(); driver.stop(); driver.join(); }
// This test verifies that reconciliation of an unknown task that // belongs to a known slave results in TASK_LOST. TEST_F(ReconciliationTest, UnknownTask) { Try<PID<Master> > master = StartMaster(); ASSERT_SOME(master); Future<SlaveRegisteredMessage> slaveRegisteredMessage = FUTURE_PROTOBUF(SlaveRegisteredMessage(), _, _); Try<PID<Slave> > slave = StartSlave(); ASSERT_SOME(slave); // Wait for the slave to register and get the slave id. AWAIT_READY(slaveRegisteredMessage); const SlaveID slaveId = slaveRegisteredMessage.get().slave_id(); MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get(), DEFAULT_CREDENTIAL); Future<FrameworkID> frameworkId; EXPECT_CALL(sched, registered(&driver, _, _)) .WillOnce(FutureArg<1>(&frameworkId)); EXPECT_CALL(sched, resourceOffers(&driver, _)) .WillRepeatedly(Return()); // Ignore offers. driver.start(); // Wait until the framework is registered. AWAIT_READY(frameworkId); Future<TaskStatus> update; EXPECT_CALL(sched, statusUpdate(&driver, _)) .WillOnce(FutureArg<1>(&update)); vector<TaskStatus> statuses; // Create a task status with a random task id. TaskStatus status; status.mutable_task_id()->set_value(UUID::random().toString()); status.mutable_slave_id()->CopyFrom(slaveId); status.set_state(TASK_RUNNING); statuses.push_back(status); driver.reconcileTasks(statuses); // Framework should receive TASK_LOST for unknown task. AWAIT_READY(update); EXPECT_EQ(TASK_LOST, update.get().state()); driver.stop(); driver.join(); Shutdown(); // Must shutdown before 'containerizer' gets deallocated. }
void reaped( ExecutorDriver* driver, const TaskID& taskId, pid_t pid, const Future<Option<int> >& status_) { TaskState state; string message; Timer::cancel(escalationTimer); if (!status_.isReady()) { state = TASK_FAILED; message = "Failed to get exit status for Command: " + (status_.isFailed() ? status_.failure() : "future discarded"); } else if (status_.get().isNone()) { state = TASK_FAILED; message = "Failed to get exit status for Command"; } else { int status = status_.get().get(); CHECK(WIFEXITED(status) || WIFSIGNALED(status)) << status; if (WIFEXITED(status) && WEXITSTATUS(status) == 0) { state = TASK_FINISHED; } else if (killed) { // Send TASK_KILLED if the task was killed as a result of // killTask() or shutdown(). state = TASK_KILLED; } else { state = TASK_FAILED; } message = string("Command") + (WIFEXITED(status) ? " exited with status " : " terminated with signal ") + (WIFEXITED(status) ? stringify(WEXITSTATUS(status)) : strsignal(WTERMSIG(status))); } cout << message << " (pid: " << pid << ")" << endl; TaskStatus taskStatus; taskStatus.mutable_task_id()->MergeFrom(taskId); taskStatus.set_state(state); taskStatus.set_message(message); driver->sendStatusUpdate(taskStatus); // A hack for now ... but we need to wait until the status update // is sent to the slave before we shut ourselves down. os::sleep(Seconds(1)); driver->stop(); }
virtual void launchTask(ExecutorDriver* driver, const TaskInfo& task) { cout << "Starting task " << task.task_id().value() << endl; TaskStatus status; status.mutable_task_id()->MergeFrom(task.task_id()); status.set_state(TASK_RUNNING); driver->sendStatusUpdate(status); sleep(1); cout << "Finishing task " << task.task_id().value() << endl; status.mutable_task_id()->MergeFrom(task.task_id()); status.set_state(TASK_FINISHED); driver->sendStatusUpdate(status); }
virtual void launchTask(ExecutorDriver* driver, const TaskInfo& task) { cout << "Starting task " << task.task_id().value() << endl; TaskStatus status; status.mutable_task_id()->MergeFrom(task.task_id()); status.set_state(TASK_RUNNING); driver->sendStatusUpdate(status); // This is where one would perform the requested task. cout << "Finishing task " << task.task_id().value() << endl; status.mutable_task_id()->MergeFrom(task.task_id()); status.set_state(TASK_FINISHED); driver->sendStatusUpdate(status); }
void run(ExecutorDriver* driver, const TaskInfo& task) { os::sleep(Seconds(random() % 10)); TaskStatus status; status.mutable_task_id()->MergeFrom(task.task_id()); status.set_state(TASK_FINISHED); driver->sendStatusUpdate(status); }
void CephExecutor::shutdown(ExecutorDriver* driver) { LOG(INFO) << "Killing this container process"; LOG(INFO) << runShellCommand("docker rm -f " + containerName); TaskStatus status; status.mutable_task_id()->MergeFrom(myTaskId); status.set_state(TASK_KILLED); driver->sendStatusUpdate(status); }
// data format is: // <MessageToExecutor>.<OSDID> for OSD executor // or just <MessageToExecutor> for MON executor void CephExecutor::frameworkMessage(ExecutorDriver* driver, const string& data) { LOG(INFO) << "Got framework message: " << data; MessageToExecutor msg; vector<string> tokens = StringUtil::explode(data,'.'); msg = (MessageToExecutor)lexical_cast<int>(tokens[0]); switch (msg){ case MessageToExecutor::REGISTER_OSD: LOG(INFO) << "Will register an OSD, and return the OSD ID"; driver->sendFrameworkMessage(registerOSD()); break; case MessageToExecutor::LAUNCH_OSD: if (tokens.size() == 2){ LOG(INFO) << "Will launch OSD docker with OSD ID: " << tokens[1]; string dockerCommand = constructOSDCommand( localSharedConfigDirRoot + "/" + localConfigDirName, tokens[1], containerName); myPID = fork(); if (0 == myPID){ //child long running docker thread //TODO: we use fork here. Need to check why below line will hung the executor //thread(&CephExecutor::startLongRunning,*this,"docker", dockerCommand).detach(); startLongRunning("docker",dockerCommand); } else { bool started = block_until_started(containerName, "30"); TaskStatus status; status.mutable_task_id()->MergeFrom(myTaskId); if (started) { LOG(INFO) << "Starting OSD task " << myTaskId.value(); //send the OSD id back to let scheduler remove it //format: <MessageToScheduler::CONSUMED_OSD_ID>.OSDID string msg = lexical_cast<string>(static_cast<int>(MessageToScheduler::CONSUMED_OSD_ID)) + "." + tokens[1]; status.set_message(msg); status.set_state(TASK_RUNNING); } else { LOG(INFO) << "Failed to start OSD task " << myTaskId.value(); status.set_state(TASK_FAILED); } driver->sendStatusUpdate(status); }//end else "0==pid" } else { LOG(INFO) << "No OSD ID given!"; } break; default: LOG(INFO) << "unknown message from scheduler"; } }
TaskStatus createTaskStatus( const TaskID& taskId, const TaskState& state, const id::UUID& uuid, double timestamp) { TaskStatus status; status.set_uuid(uuid.toBytes()); status.set_timestamp(timestamp); status.mutable_task_id()->CopyFrom(taskId); status.set_state(state); return status; }
virtual void launchTask(ExecutorDriver* driver, const TaskInfo& task) { cout << "Starting task " << task.task_id().value() << endl; lambda::function<void(void)>* thunk = new lambda::function<void(void)>(lambda::bind(&run, driver, task)); pthread_t pthread; if (pthread_create(&pthread, NULL, &start, thunk) != 0) { TaskStatus status; status.mutable_task_id()->MergeFrom(task.task_id()); status.set_state(TASK_FAILED); driver->sendStatusUpdate(status); } else { pthread_detach(pthread); TaskStatus status; status.mutable_task_id()->MergeFrom(task.task_id()); status.set_state(TASK_RUNNING); driver->sendStatusUpdate(status); } }
// TODO(vinod): Make SlaveID optional because 'StatusUpdate.SlaveID' // is optional. StatusUpdate createStatusUpdate( const FrameworkID& frameworkId, const Option<SlaveID>& slaveId, const TaskID& taskId, const TaskState& state, const TaskStatus::Source& source, const string& message = "", const Option<TaskStatus::Reason>& reason = None(), const Option<ExecutorID>& executorId = None(), const Option<bool>& healthy = None()) { StatusUpdate update; update.set_timestamp(process::Clock::now().secs()); update.set_uuid(UUID::random().toBytes()); update.mutable_framework_id()->MergeFrom(frameworkId); if (slaveId.isSome()) { update.mutable_slave_id()->MergeFrom(slaveId.get()); } if (executorId.isSome()) { update.mutable_executor_id()->MergeFrom(executorId.get()); } TaskStatus* status = update.mutable_status(); status->mutable_task_id()->MergeFrom(taskId); if (slaveId.isSome()) { status->mutable_slave_id()->MergeFrom(slaveId.get()); } status->set_state(state); status->set_source(source); status->set_message(message); status->set_timestamp(update.timestamp()); if (reason.isSome()) { status->set_reason(reason.get()); } if (healthy.isSome()) { status->set_healthy(healthy.get()); } return update; }
void operator()() { TaskStatus status; status.mutable_task_id()->MergeFrom(task.task_id()); // Currently, just call the K3 executable with the generated command line from task.data() try { FILE* pipe = popen(k3_cmd.c_str(), "r"); if (!pipe) { status.set_state(TASK_FAILED); driver->sendStatusUpdate(status); cout << "Failed to open subprocess" << endl; } char buffer[256]; while (!feof(pipe)) { if (fgets(buffer, 256, pipe) != NULL) { std::string s = std::string(buffer); if (this->isMaster) { driver->sendFrameworkMessage(s); cout << s << endl; } else { cout << s << endl; } } } int k3 = pclose(pipe); if (k3 == 0) { status.set_state(TASK_FINISHED); cout << "Task " << task.task_id().value() << " Completed!" << endl; driver->sendStatusUpdate(status); } else { status.set_state(TASK_FAILED); cout << "K3 Task " << task.task_id().value() << " returned error code: " << k3 << endl; driver->sendStatusUpdate(status); } } catch (...) { status.set_state(TASK_FAILED); driver->sendStatusUpdate(status); } //------------- END OF TASK ------------------- }
// This test verifies that reconciliation of a task that belongs to an // unknown slave results in TASK_LOST. TEST_F(ReconciliationTest, UnknownSlave) { Try<PID<Master> > master = StartMaster(); ASSERT_SOME(master); MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get(), DEFAULT_CREDENTIAL); Future<FrameworkID> frameworkId; EXPECT_CALL(sched, registered(&driver, _, _)) .WillOnce(FutureArg<1>(&frameworkId)); driver.start(); // Wait until the framework is registered. AWAIT_READY(frameworkId); Future<TaskStatus> update; EXPECT_CALL(sched, statusUpdate(&driver, _)) .WillOnce(FutureArg<1>(&update)); vector<TaskStatus> statuses; // Create a task status with a random slave id (and task id). TaskStatus status; status.mutable_task_id()->set_value(UUID::random().toString()); status.mutable_slave_id()->set_value(UUID::random().toString()); status.set_state(TASK_RUNNING); statuses.push_back(status); driver.reconcileTasks(statuses); // Framework should receive TASK_LOST because the slave is unknown. AWAIT_READY(update); EXPECT_EQ(TASK_LOST, update.get().state()); driver.stop(); driver.join(); Shutdown(); }
virtual void launchTask(ExecutorDriver* driver, const TaskDescription& task) { cout << "Executor starting task " << task.task_id().value() << endl; int64_t memToHog; double duration; int numThreads; istringstream in(task.data()); in >> memToHog >> duration >> numThreads; memToHog *= 1024LL * 1024LL; // Convert from MB to bytes for (int i = 0; i < numThreads; i++) { ThreadArg* arg = new ThreadArg(this, task, i, memToHog, duration); pthread_t thread; pthread_create(&thread, 0, runTask, arg); pthread_detach(thread); TaskStatus status; status.mutable_task_id()->MergeFrom(task.task_id()); status.set_state(TASK_RUNNING); driver->sendStatusUpdate(status); } }
void launchTask (ExecutorDriver* driver, const TaskInfo& task) override { cout << "Starting task " << task.task_id().value() << endl; TaskStatus status; status.mutable_task_id()->MergeFrom(task.task_id()); StartInfo* info = new StartInfo(driver, task); pthread_t pthread; int res = pthread_create(&pthread, NULL, &RunProcess, info); if (res != 0) { status.set_state(TASK_FAILED); delete info; } else { pthread_detach(pthread); status.set_state(TASK_RUNNING); } driver->sendStatusUpdate(status); }
// This test ensures we don't break the API when it comes to JSON // representation of tasks. Also, we want to ensure that tasks are // modeled the same way when using 'Task' vs. 'TaskInfo'. TEST(HTTP, ModelTask) { TaskID taskId; taskId.set_value("t"); SlaveID slaveId; slaveId.set_value("s"); ExecutorID executorId; executorId.set_value("t"); FrameworkID frameworkId; frameworkId.set_value("f"); TaskState state = TASK_RUNNING; vector<TaskStatus> statuses; TaskStatus status; status.mutable_task_id()->CopyFrom(taskId); status.set_state(state); status.mutable_slave_id()->CopyFrom(slaveId); status.mutable_executor_id()->CopyFrom(executorId); status.set_timestamp(0.0); statuses.push_back(status); TaskInfo task; task.set_name("task"); task.mutable_task_id()->CopyFrom(taskId); task.mutable_slave_id()->CopyFrom(slaveId); task.mutable_command()->set_value("echo hello"); Task task_ = protobuf::createTask(task, state, frameworkId); task_.add_statuses()->CopyFrom(statuses[0]); JSON::Value object = model(task, frameworkId, state, statuses); JSON::Value object_ = model(task_); Try<JSON::Value> expected = JSON::parse( "{" " \"executor_id\":\"\"," " \"framework_id\":\"f\"," " \"id\":\"t\"," " \"name\":\"task\"," " \"resources\":" " {" " \"cpus\":0," " \"disk\":0," " \"mem\":0" " }," " \"slave_id\":\"s\"," " \"state\":\"TASK_RUNNING\"," " \"statuses\":" " [" " {" " \"state\":\"TASK_RUNNING\"," " \"timestamp\":0" " }" " ]" "}"); ASSERT_SOME(expected); EXPECT_EQ(expected.get(), object); EXPECT_EQ(expected.get(), object_); // Ensure both are modeled the same. EXPECT_EQ(object, object_); }
void launchTask(ExecutorDriver* driver, const TaskInfo& task) { CHECK_EQ(REGISTERED, state); if (launched) { TaskStatus status; status.mutable_task_id()->MergeFrom(task.task_id()); status.set_state(TASK_FAILED); status.set_message( "Attempted to run multiple tasks using a \"command\" executor"); driver->sendStatusUpdate(status); return; } // Capture the TaskID. taskId = task.task_id(); // Determine the command to launch the task. CommandInfo command; if (taskCommand.isSome()) { // Get CommandInfo from a JSON string. Try<JSON::Object> object = JSON::parse<JSON::Object>(taskCommand.get()); if (object.isError()) { cerr << "Failed to parse JSON: " << object.error() << endl; abort(); } Try<CommandInfo> parse = protobuf::parse<CommandInfo>(object.get()); if (parse.isError()) { cerr << "Failed to parse protobuf: " << parse.error() << endl; abort(); } command = parse.get(); } else if (task.has_command()) { command = task.command(); } else { CHECK_SOME(override) << "Expecting task '" << task.task_id() << "' to have a command!"; } if (override.isNone()) { // TODO(jieyu): For now, we just fail the executor if the task's // CommandInfo is not valid. The framework will receive // TASK_FAILED for the task, and will most likely find out the // cause with some debugging. This is a temporary solution. A more // correct solution is to perform this validation at master side. if (command.shell()) { CHECK(command.has_value()) << "Shell command of task '" << task.task_id() << "' is not specified!"; } else { CHECK(command.has_value()) << "Executable of task '" << task.task_id() << "' is not specified!"; } } cout << "Starting task " << task.task_id() << endl; // TODO(benh): Clean this up with the new 'Fork' abstraction. // Use pipes to determine which child has successfully changed // session. This is needed as the setsid call can fail from other // processes having the same group id. int pipes[2]; if (pipe(pipes) < 0) { perror("Failed to create a pipe"); abort(); } // Set the FD_CLOEXEC flags on these pipes. Try<Nothing> cloexec = os::cloexec(pipes[0]); if (cloexec.isError()) { cerr << "Failed to cloexec(pipe[0]): " << cloexec.error() << endl; abort(); } cloexec = os::cloexec(pipes[1]); if (cloexec.isError()) { cerr << "Failed to cloexec(pipe[1]): " << cloexec.error() << endl; abort(); } Option<string> rootfs; if (sandboxDirectory.isSome()) { // If 'sandbox_diretory' is specified, that means the user // task specifies a root filesystem, and that root filesystem has // already been prepared at COMMAND_EXECUTOR_ROOTFS_CONTAINER_PATH. // The command executor is responsible for mounting the sandbox // into the root filesystem, chrooting into it and changing the // user before exec-ing the user process. // // TODO(gilbert): Consider a better way to detect if a root // filesystem is specified for the command task. #ifdef __linux__ Result<string> user = os::user(); if (user.isError()) { cerr << "Failed to get current user: "******"Current username is not found" << endl; abort(); } else if (user.get() != "root") { cerr << "The command executor requires root with rootfs" << endl; abort(); } rootfs = path::join( os::getcwd(), COMMAND_EXECUTOR_ROOTFS_CONTAINER_PATH); string sandbox = path::join(rootfs.get(), sandboxDirectory.get()); if (!os::exists(sandbox)) { Try<Nothing> mkdir = os::mkdir(sandbox); if (mkdir.isError()) { cerr << "Failed to create sandbox mount point at '" << sandbox << "': " << mkdir.error() << endl; abort(); } } // Mount the sandbox into the container rootfs. // We need to perform a recursive mount because we want all the // volume mounts in the sandbox to be also mounted in the container // root filesystem. However, since the container root filesystem // is also mounted in the sandbox, after the recursive mount we // also need to unmount the root filesystem in the mounted sandbox. Try<Nothing> mount = fs::mount( os::getcwd(), sandbox, None(), MS_BIND | MS_REC, NULL); if (mount.isError()) { cerr << "Unable to mount the work directory into container " << "rootfs: " << mount.error() << endl;; abort(); } // Umount the root filesystem path in the mounted sandbox after // the recursive mount. Try<Nothing> unmountAll = fs::unmountAll(path::join( sandbox, COMMAND_EXECUTOR_ROOTFS_CONTAINER_PATH)); if (unmountAll.isError()) { cerr << "Unable to unmount rootfs under mounted sandbox: " << unmountAll.error() << endl; abort(); } #else cerr << "Not expecting root volume with non-linux platform." << endl; abort(); #endif // __linux__ } // Prepare the argv before fork as it's not async signal safe. char **argv = new char*[command.arguments().size() + 1]; for (int i = 0; i < command.arguments().size(); i++) { argv[i] = (char*) command.arguments(i).c_str(); } argv[command.arguments().size()] = NULL; // Prepare the command log message. string commandString; if (override.isSome()) { char** argv = override.get(); // argv is guaranteed to be NULL terminated and we rely on // that fact to print command to be executed. for (int i = 0; argv[i] != NULL; i++) { commandString += string(argv[i]) + " "; } } else if (command.shell()) {
// This test verifies that the master reconciles tasks that are // missing from a reregistering slave. In this case, we trigger // a race between the slave re-registration message and the launch // message. There should be no TASK_LOST / TASK_DROPPED. // This was motivated by MESOS-1696. TEST_F(MasterSlaveReconciliationTest, ReconcileRace) { Try<Owned<cluster::Master>> master = StartMaster(); ASSERT_SOME(master); MockExecutor exec(DEFAULT_EXECUTOR_ID); TestContainerizer containerizer(&exec); StandaloneMasterDetector detector(master.get()->pid); Future<SlaveRegisteredMessage> slaveRegisteredMessage = FUTURE_PROTOBUF(SlaveRegisteredMessage(), master.get()->pid, _); Try<Owned<cluster::Slave>> slave = StartSlave(&detector, &containerizer); ASSERT_SOME(slave); AWAIT_READY(slaveRegisteredMessage); MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL); EXPECT_CALL(sched, registered(&driver, _, _)); Future<vector<Offer>> offers; EXPECT_CALL(sched, resourceOffers(&driver, _)) .WillOnce(FutureArg<1>(&offers)) .WillRepeatedly(Return()); // Ignore subsequent offers. driver.start(); // Since the agent may have retried registration, we want to // ensure that any duplicate registrations are flushed before // we appoint the master again. Otherwise, the agent may // receive a stale registration message. Clock::pause(); Clock::settle(); Clock::resume(); // Trigger a re-registration of the slave and capture the message // so that we can spoof a race with a launch task message. DROP_PROTOBUFS(ReregisterSlaveMessage(), slave.get()->pid, master.get()->pid); Future<ReregisterSlaveMessage> reregisterSlaveMessage = DROP_PROTOBUF( ReregisterSlaveMessage(), slave.get()->pid, master.get()->pid); detector.appoint(master.get()->pid); AWAIT_READY(reregisterSlaveMessage); AWAIT_READY(offers); ASSERT_FALSE(offers->empty()); TaskInfo task; task.set_name("test task"); task.mutable_task_id()->set_value("1"); task.mutable_slave_id()->MergeFrom(offers.get()[0].slave_id()); task.mutable_resources()->MergeFrom(offers.get()[0].resources()); task.mutable_executor()->MergeFrom(DEFAULT_EXECUTOR_INFO); ExecutorDriver* executorDriver; EXPECT_CALL(exec, registered(_, _, _, _)) .WillOnce(SaveArg<0>(&executorDriver)); // Leave the task in TASK_STAGING. Future<Nothing> launchTask; EXPECT_CALL(exec, launchTask(_, _)) .WillOnce(FutureSatisfy(&launchTask)); EXPECT_CALL(sched, statusUpdate(&driver, _)) .Times(0); driver.launchTasks(offers.get()[0].id(), {task}); AWAIT_READY(launchTask); // Send the stale re-registration message, which does not contain // the task we just launched. This will trigger a reconciliation // by the master. Future<SlaveReregisteredMessage> slaveReregisteredMessage = FUTURE_PROTOBUF(SlaveReregisteredMessage(), _, _); // Prevent this from being dropped per the DROP_PROTOBUFS above. FUTURE_PROTOBUF( ReregisterSlaveMessage(), slave.get()->pid, master.get()->pid); process::post( slave.get()->pid, master.get()->pid, reregisterSlaveMessage.get()); AWAIT_READY(slaveReregisteredMessage); // Neither the master nor the slave should send a TASK_LOST // as part of the reconciliation. We check this by calling // Clock::settle() to flush all pending events. Clock::pause(); Clock::settle(); Clock::resume(); // Now send TASK_FINISHED and make sure it's the only message // received by the scheduler. Future<TaskStatus> status; EXPECT_CALL(sched, statusUpdate(&driver, _)) .WillOnce(FutureArg<1>(&status)); TaskStatus taskStatus; taskStatus.mutable_task_id()->CopyFrom(task.task_id()); taskStatus.set_state(TASK_FINISHED); executorDriver->sendStatusUpdate(taskStatus); AWAIT_READY(status); ASSERT_EQ(TASK_FINISHED, status->state()); EXPECT_CALL(exec, shutdown(_)) .Times(AtMost(1)); driver.stop(); driver.join(); }
// This test verifies that the slave task status label decorator can // add and remove labels from a TaskStatus during the status update // sequence. A TaskStatus with two labels ("foo":"bar" and // "bar":"baz") is sent from the executor. The labels get modified by // the slave hook to strip the "foo":"bar" pair and/ add a new // "baz":"qux" pair. TEST_F(HookTest, VerifySlaveTaskStatusDecorator) { Try<Owned<cluster::Master>> master = StartMaster(); ASSERT_SOME(master); MockExecutor exec(DEFAULT_EXECUTOR_ID); TestContainerizer containerizer(&exec); Owned<MasterDetector> detector = master.get()->createDetector(); Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), &containerizer); ASSERT_SOME(slave); MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL); EXPECT_CALL(sched, registered(&driver, _, _)); Future<vector<Offer>> offers; EXPECT_CALL(sched, resourceOffers(&driver, _)) .WillOnce(FutureArg<1>(&offers)) .WillRepeatedly(Return()); // Ignore subsequent offers. driver.start(); AWAIT_READY(offers); ASSERT_EQ(1u, offers.get().size()); // Start a task. TaskInfo task = createTask(offers.get()[0], "", DEFAULT_EXECUTOR_ID); ExecutorDriver* execDriver; EXPECT_CALL(exec, registered(_, _, _, _)) .WillOnce(SaveArg<0>(&execDriver)); Future<TaskInfo> execTask; EXPECT_CALL(exec, launchTask(_, _)) .WillOnce(FutureArg<1>(&execTask)); Future<TaskStatus> status; EXPECT_CALL(sched, statusUpdate(&driver, _)) .WillOnce(FutureArg<1>(&status)); driver.launchTasks(offers.get()[0].id(), {task}); AWAIT_READY(execTask); // Now send TASK_RUNNING update with two labels. The first label // ("foo:bar") will be removed by the task status hook to ensure // that it can remove labels. The second label will be preserved // and forwarded to Master (and eventually to the framework). // The hook also adds a new label with the same key but a different // value ("bar:quz"). TaskStatus runningStatus; runningStatus.mutable_task_id()->MergeFrom(execTask.get().task_id()); runningStatus.set_state(TASK_RUNNING); // Add two labels to the TaskStatus Labels* labels = runningStatus.mutable_labels(); labels->add_labels()->CopyFrom(createLabel("foo", "bar")); labels->add_labels()->CopyFrom(createLabel("bar", "baz")); execDriver->sendStatusUpdate(runningStatus); AWAIT_READY(status); // The hook will hang an extra label off. const Labels& labels_ = status.get().labels(); EXPECT_EQ(2, labels_.labels_size()); // The test hook will prepend a new "baz":"qux" label. EXPECT_EQ("bar", labels_.labels(0).key()); EXPECT_EQ("qux", labels_.labels(0).value()); // And lastly, we only expect the "foo":"bar" pair to be stripped by // the module. The last pair should be the original "bar":"baz" // pair set by the test. EXPECT_EQ("bar", labels_.labels(1).key()); EXPECT_EQ("baz", labels_.labels(1).value()); // Now validate TaskInfo.container_status. We must have received a // container_status with one network_info set by the test hook module. EXPECT_TRUE(status.get().has_container_status()); EXPECT_EQ(1, status.get().container_status().network_infos().size()); const NetworkInfo networkInfo = status.get().container_status().network_infos(0); // The hook module sets up '4.3.2.1' as the IP address and 'public' as the // network isolation group. The `ip_address` field is deprecated, but the // hook module should continue to set it as well as the new `ip_addresses` // field for now. EXPECT_TRUE(networkInfo.has_ip_address()); EXPECT_EQ("4.3.2.1", networkInfo.ip_address()); EXPECT_EQ(1, networkInfo.ip_addresses().size()); EXPECT_TRUE(networkInfo.ip_addresses(0).has_ip_address()); EXPECT_EQ("4.3.2.1", networkInfo.ip_addresses(0).ip_address()); EXPECT_EQ(1, networkInfo.groups().size()); EXPECT_EQ("public", networkInfo.groups(0)); EXPECT_TRUE(networkInfo.has_labels()); EXPECT_EQ(1, networkInfo.labels().labels().size()); const Label networkInfoLabel = networkInfo.labels().labels(0); // Finally, the labels set inside NetworkInfo by the hook module. EXPECT_EQ("net_foo", networkInfoLabel.key()); EXPECT_EQ("net_bar", networkInfoLabel.value()); EXPECT_CALL(exec, shutdown(_)) .Times(AtMost(1)); driver.stop(); driver.join(); }
// This test verifies that reconciliation sends the latest task // status, when the task state does not match between the framework // and the master. TEST_F(ReconciliationTest, TaskStateMismatch) { Try<PID<Master> > master = StartMaster(); ASSERT_SOME(master); MockExecutor exec(DEFAULT_EXECUTOR_ID); TestContainerizer containerizer(&exec); Try<PID<Slave> > slave = StartSlave(&containerizer); ASSERT_SOME(slave); MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get(), DEFAULT_CREDENTIAL); Future<FrameworkID> frameworkId; EXPECT_CALL(sched, registered(&driver, _, _)) .WillOnce(FutureArg<1>(&frameworkId)); EXPECT_CALL(sched, resourceOffers(&driver, _)) .WillOnce(LaunchTasks(DEFAULT_EXECUTOR_INFO, 1, 1, 512, "*")) .WillRepeatedly(Return()); // Ignore subsequent offers. EXPECT_CALL(exec, registered(_, _, _, _)); EXPECT_CALL(exec, launchTask(_, _)) .WillOnce(SendStatusUpdateFromTask(TASK_RUNNING)); Future<TaskStatus> update; EXPECT_CALL(sched, statusUpdate(&driver, _)) .WillOnce(FutureArg<1>(&update)); driver.start(); // Wait until the framework is registered. AWAIT_READY(frameworkId); AWAIT_READY(update); EXPECT_EQ(TASK_RUNNING, update.get().state()); EXPECT_EQ(true, update.get().has_slave_id()); const TaskID taskId = update.get().task_id(); const SlaveID slaveId = update.get().slave_id(); // If framework has different state, current state should be reported. Future<TaskStatus> update2; EXPECT_CALL(sched, statusUpdate(&driver, _)) .WillOnce(FutureArg<1>(&update2)); vector<TaskStatus> statuses; TaskStatus status; status.mutable_task_id()->CopyFrom(taskId); status.mutable_slave_id()->CopyFrom(slaveId); status.set_state(TASK_KILLED); statuses.push_back(status); driver.reconcileTasks(statuses); AWAIT_READY(update2); EXPECT_EQ(TASK_RUNNING, update2.get().state()); EXPECT_CALL(exec, shutdown(_)) .Times(AtMost(1)); driver.stop(); driver.join(); Shutdown(); // Must shutdown before 'containerizer' gets deallocated. }
// This test verifies that a reconciliation request that comes before // '_launchTasks()' is ignored. TEST_F(MasterAuthorizationTest, ReconcileTask) { MockAuthorizer authorizer; Try<PID<Master> > master = StartMaster(&authorizer); ASSERT_SOME(master); MockExecutor exec(DEFAULT_EXECUTOR_ID); Try<PID<Slave> > slave = StartSlave(&exec); ASSERT_SOME(slave); MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get(), DEFAULT_CREDENTIAL); EXPECT_CALL(sched, registered(&driver, _, _)) .Times(1); Future<vector<Offer> > offers; EXPECT_CALL(sched, resourceOffers(&driver, _)) .WillOnce(FutureArg<1>(&offers)) .WillRepeatedly(Return()); // Ignore subsequent offers. driver.start(); AWAIT_READY(offers); EXPECT_NE(0u, offers.get().size()); TaskInfo task = createTask(offers.get()[0], "", DEFAULT_EXECUTOR_ID); vector<TaskInfo> tasks; tasks.push_back(task); // Return a pending future from authorizer. Future<Nothing> future; Promise<bool> promise; EXPECT_CALL(authorizer, authorize(An<const mesos::ACL::RunTasks&>())) .WillOnce(DoAll(FutureSatisfy(&future), Return(promise.future()))); driver.launchTasks(offers.get()[0].id(), tasks); // Wait until authorization is in progress. AWAIT_READY(future); // Scheduler shouldn't get an update from reconciliation. EXPECT_CALL(sched, statusUpdate(&driver, _)) .Times(0); Future<ReconcileTasksMessage> reconcileTasksMessage = FUTURE_PROTOBUF(ReconcileTasksMessage(), _, _); vector<TaskStatus> statuses; TaskStatus status; status.mutable_task_id()->CopyFrom(task.task_id()); status.mutable_slave_id()->CopyFrom(offers.get()[0].slave_id()); status.set_state(TASK_STAGING); statuses.push_back(status); driver.reconcileTasks(statuses); AWAIT_READY(reconcileTasksMessage); // Make sure the framework doesn't receive any update. Clock::pause(); Clock::settle(); // Now stop the framework. driver.stop(); driver.join(); Shutdown(); // Must shutdown before 'containerizer' gets deallocated. }
// This test verifies that reconciliation of a task that belongs to a // slave that is a transitional state doesn't result in an update. TEST_F(ReconciliationTest, SlaveInTransition) { master::Flags masterFlags = CreateMasterFlags(); Try<PID<Master> > master = StartMaster(); ASSERT_SOME(master); // Start a checkpointing slave. slave::Flags slaveFlags = CreateSlaveFlags(); slaveFlags.checkpoint = true; Future<SlaveRegisteredMessage> slaveRegisteredMessage = FUTURE_PROTOBUF(SlaveRegisteredMessage(), _, _); Try<PID<Slave> > slave = StartSlave(slaveFlags); ASSERT_SOME(slave); // Wait for the slave to register and get the slave id. AWAIT_READY(slaveRegisteredMessage); const SlaveID slaveId = slaveRegisteredMessage.get().slave_id(); // Stop the master and slave. Stop(master.get()); Stop(slave.get()); MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get(), DEFAULT_CREDENTIAL); Future<FrameworkID> frameworkId; EXPECT_CALL(sched, registered(&driver, _, _)) .WillOnce(FutureArg<1>(&frameworkId)); EXPECT_CALL(sched, resourceOffers(&driver, _)) .WillRepeatedly(Return()); // Ignore offers. // Framework should not receive any update. EXPECT_CALL(sched, statusUpdate(&driver, _)) .Times(0); // Drop '&Master::_reregisterSlave' dispatch so that the slave is // in 'reregistering' state. Future<Nothing> _reregisterSlave = DROP_DISPATCH(_, &Master::_reregisterSlave); // Restart the master. master = StartMaster(masterFlags); ASSERT_SOME(master); driver.start(); // Wait for the framework to register. AWAIT_READY(frameworkId); // Restart the slave. slave = StartSlave(slaveFlags); ASSERT_SOME(slave); // Slave will be in 'reregistering' state here. AWAIT_READY(_reregisterSlave); vector<TaskStatus> statuses; // Create a task status with a random task id. TaskStatus status; status.mutable_task_id()->set_value(UUID::random().toString()); status.mutable_slave_id()->CopyFrom(slaveId); status.set_state(TASK_RUNNING); statuses.push_back(status); Future<ReconcileTasksMessage> reconcileTasksMessage = FUTURE_PROTOBUF(ReconcileTasksMessage(), _ , _); Clock::pause(); driver.reconcileTasks(statuses); // Make sure the master received the reconcile tasks message. AWAIT_READY(reconcileTasksMessage); // The Clock::settle() will ensure that framework would receive // a status update if it is sent by the master. In this test it // shouldn't receive any. Clock::settle(); driver.stop(); driver.join(); Shutdown(); }
// This test ensures that reconciliation requests for tasks that are // pending are exposed in reconciliation. TEST_F(ReconciliationTest, PendingTask) { MockAuthorizer authorizer; Try<PID<Master> > master = StartMaster(&authorizer); ASSERT_SOME(master); MockExecutor exec(DEFAULT_EXECUTOR_ID); Future<SlaveRegisteredMessage> slaveRegisteredMessage = FUTURE_PROTOBUF(SlaveRegisteredMessage(), _, _); Try<PID<Slave> > slave = StartSlave(); ASSERT_SOME(slave); // Wait for the slave to register and get the slave id. AWAIT_READY(slaveRegisteredMessage); const SlaveID slaveId = slaveRegisteredMessage.get().slave_id(); MockScheduler sched; MesosSchedulerDriver driver( &sched, DEFAULT_FRAMEWORK_INFO, master.get(), DEFAULT_CREDENTIAL); EXPECT_CALL(sched, registered(&driver, _, _)) .Times(1); Future<vector<Offer> > offers; EXPECT_CALL(sched, resourceOffers(&driver, _)) .WillOnce(FutureArg<1>(&offers)) .WillRepeatedly(Return()); // Ignore subsequent offers. driver.start(); AWAIT_READY(offers); EXPECT_NE(0u, offers.get().size()); // Return a pending future from authorizer. Future<Nothing> authorize; Promise<bool> promise; EXPECT_CALL(authorizer, authorize(An<const mesos::ACL::RunTask&>())) .WillOnce(DoAll(FutureSatisfy(&authorize), Return(promise.future()))); TaskInfo task = createTask(offers.get()[0], "", DEFAULT_EXECUTOR_ID); vector<TaskInfo> tasks; tasks.push_back(task); driver.launchTasks(offers.get()[0].id(), tasks); // Wait until authorization is in progress. AWAIT_READY(authorize); // First send an implicit reconciliation request for this task. Future<TaskStatus> update; EXPECT_CALL(sched, statusUpdate(&driver, _)) .WillOnce(FutureArg<1>(&update)); vector<TaskStatus> statuses; driver.reconcileTasks(statuses); AWAIT_READY(update); EXPECT_EQ(TASK_STAGING, update.get().state()); EXPECT_TRUE(update.get().has_slave_id()); // Now send an explicit reconciliation request for this task. Future<TaskStatus> update2; EXPECT_CALL(sched, statusUpdate(&driver, _)) .WillOnce(FutureArg<1>(&update2)); TaskStatus status; status.mutable_task_id()->CopyFrom(task.task_id()); status.mutable_slave_id()->CopyFrom(slaveId); status.set_state(TASK_STAGING); statuses.push_back(status); driver.reconcileTasks(statuses); AWAIT_READY(update2); EXPECT_EQ(TASK_STAGING, update2.get().state()); EXPECT_TRUE(update2.get().has_slave_id()); driver.stop(); driver.join(); Shutdown(); // Must shutdown before 'containerizer' gets deallocated. }
StatusUpdate createStatusUpdate( const FrameworkID& frameworkId, const Option<SlaveID>& slaveId, const TaskID& taskId, const TaskState& state, const TaskStatus::Source& source, const Option<UUID>& uuid, const string& message = "", const Option<TaskStatus::Reason>& reason = None(), const Option<ExecutorID>& executorId = None(), const Option<bool>& healthy = None()) { StatusUpdate update; update.set_timestamp(process::Clock::now().secs()); update.mutable_framework_id()->MergeFrom(frameworkId); if (slaveId.isSome()) { update.mutable_slave_id()->MergeFrom(slaveId.get()); } if (executorId.isSome()) { update.mutable_executor_id()->MergeFrom(executorId.get()); } TaskStatus* status = update.mutable_status(); status->mutable_task_id()->MergeFrom(taskId); if (slaveId.isSome()) { status->mutable_slave_id()->MergeFrom(slaveId.get()); } status->set_state(state); status->set_source(source); status->set_message(message); status->set_timestamp(update.timestamp()); if (uuid.isSome()) { update.set_uuid(uuid.get().toBytes()); status->set_uuid(uuid.get().toBytes()); } else { // Note that in 0.22.x, the StatusUpdate.uuid was required // even though the scheduler driver ignores it for master // and scheduler driver generated updates. So we continue // to "set" it here so that updates coming from a 0.23.x // master can be parsed by a 0.22.x scheduler driver. // // TODO(bmahler): In 0.24.x, leave the uuid unset. update.set_uuid(""); } if (reason.isSome()) { status->set_reason(reason.get()); } if (healthy.isSome()) { status->set_healthy(healthy.get()); } return update; }
void* RunProcess (void* args) { StartInfo* info = static_cast<StartInfo*>(args); ExecutorDriver* driver = info->driver; const TaskInfo& task = info->task; ExternalInfo external = StartExternalProcess(task); { TaskStatus status; status.mutable_task_id()->CopyFrom(task.task_id()); if (external.failed) { status.set_state(TASK_FAILED); driver->sendStatusUpdate(status); delete info; return nullptr; } cout << "PID " << external.pid << "\n"; status.set_state(TASK_RUNNING); driver->sendStatusUpdate(status); } { lock_guard<mutex> lock(TaskId2PidLock); const string& taskId = task.task_id().value(); TaskId2Pid[taskId] = external.pid; } TaskStatus status; status.mutable_task_id()->CopyFrom(task.task_id()); int s; waitpid(external.pid, &s, WUNTRACED); cout << "WAIT for pid " << external.pid << " returned\n"; if (WIFEXITED(s)) { int es = WEXITSTATUS(s); if (es == 0) { cout << "EXIT " << external.pid << ", status == 0\n"; status.set_state(TASK_FINISHED); } else { cout << "EXIT " << external.pid << ", status " << es << "\n"; status.set_state(TASK_FAILED); } } else if (WIFSIGNALED(s)) { cout << "EXIT " << external.pid << " signalled with " << WTERMSIG(s) << "\n"; status.set_state(TASK_FAILED); } else if (WIFSTOPPED(s)) { cout << "EXIT " << external.pid << " stopped\n"; // TODO(fc) deal with stopped, but how? kill(external.pid, 9); status.set_state(TASK_FAILED); } driver->sendStatusUpdate(status); return nullptr; }
//when the task before starting, //it should check task.data() to determin //what it will do, whether copy config? //whether start fileserver? // //task.data() here format is : //<isInitialMonNode>.<TaskType> void CephExecutor::launchTask(ExecutorDriver* driver, const TaskInfo& task) { //set class member localSharedConfDirRoot string cmd = "echo ~"; string r = runShellCommand(cmd); localSharedConfigDirRoot = r == " " ? r :"/root"; LOG(INFO) << "localSharedConfigDirRoot is " << localSharedConfigDirRoot; bool needCopyConfig = true; bool needStartFileServer = false; int taskType; if (task.has_data()){ LOG(INFO) << "Got TaskInfo data: " << task.data(); vector<string> tokens = StringUtil::explode(task.data(),'.'); //split by '.', the first part is isInitialMonNode, //second part is used for task type if (tokens[0] == "1"){ needCopyConfig = false; } taskType = lexical_cast<int>(tokens[1]); } string localMountDir = localSharedConfigDirRoot + "/" +localConfigDirName; TaskStatus status; status.mutable_task_id()->MergeFrom(task.task_id()); //make local shared dir, all type of task need this: //TODO: check if already exists valid dirctory tree if (!createLocalSharedConfigDir(localConfigDirName)) { LOG(INFO) << "created local shared directory failed!"; status.set_state(TASK_FAILED); driver->sendStatusUpdate(status); return; } LOG(INFO) << "Create directory tree done."; //mount shared local dir if (needCopyConfig) { string abPath = localMountDir + "/" + "/etc/ceph/"; if (!copySharedConfigDir(abPath)) { LOG(INFO) << "Copy shared config file failed!"; status.set_state(TASK_FAILED); driver->sendStatusUpdate(status); return; } LOG(INFO) << "Copy config files done."; } //run docker command for MON and RADOSGW string cName = getContainerName(task.task_id().value()); //set class member containerName, and myTaskId //TODO: see if put these in registed is more proper containerName = cName; myTaskId = task.task_id(); //TODO: kill existing container in case conflict runShellCommand("docker rm -f " + containerName); string dockerCommand; switch (taskType) { case static_cast<int>(TaskType::MON): needStartFileServer = true; dockerCommand = constructMonCommand( localMountDir, cName); downloadDockerImage("ceph/mon"); break; case static_cast<int>(TaskType::OSD): downloadDockerImage("ceph/osd"); //Will get osdId in FrameworkMessage dockerCommand = ""; status.set_state(TASK_STARTING); driver->sendStatusUpdate(status); return; case static_cast<int>(TaskType::RADOSGW): downloadDockerImage("ceph/radosgw"); dockerCommand = constructRADOSGWCommand( localMountDir, cName); break; } if (needStartFileServer) { thread fileServerThread(fileServer, 7777, localSharedConfigDirRoot + "/" + localConfigDirName + "/etc/ceph/"); fileServerThread.detach(); LOG(INFO) << "Mon fileserver started"; } LOG(INFO) << "Stating container with command: "; LOG(INFO) << dockerCommand; //fork a thread to enable docker long running. //TODO: <thread> here seems not working, figure it out //to find a better way myPID = fork(); if (0 == myPID){ //child long running docker thread //TODO: we use fork here. Need to check why below line will hung the executor //thread(&CephExecutor::startLongRunning,*this,"docker", dockerCommand).detach(); startLongRunning("docker",dockerCommand); } else { //parent thread //check if started normally bool started = block_until_started(cName, "30"); if (started) { LOG(INFO) << "Starting task " << task.task_id().value(); status.set_state(TASK_RUNNING); } else { LOG(INFO) << "Failed to start task " << task.task_id().value(); status.set_state(TASK_FAILED); } driver->sendStatusUpdate(status); } }
virtual void launchTask(ExecutorDriver* driver, const TaskInfo& task) { localPeerCount++; TaskStatus status; status.mutable_task_id()->MergeFrom(task.task_id()); status.set_state(TASK_RUNNING); driver->sendStatusUpdate(status); //------------- START TASK OPERATIONS ---------- cout << "Running K3 Program: " << task.name() << endl; string k3_cmd; using namespace YAML; Node hostParams = Load(task.data()); Node peerParams; Node peers; // vector<Node> peers; cout << "WHAT I RECEIVED\n----------------------\n"; cout << Dump(hostParams); cout << "\n---------------------------------\n"; k3_cmd = "cd $MESOS_SANDBOX && bash -c 'ulimit -c unlimited && ./" + hostParams["binary"].as<string>(); if (hostParams["logging"]) { k3_cmd += " -l INFO "; } if (hostParams["resultVar"]) { k3_cmd += " --result_path $MESOS_SANDBOX --result_var " + hostParams["resultVar"].as<string>(); } string datavar, datapath; string datapolicy = "default"; int peerStart = 0; int peerEnd = 0; for (const_iterator param=hostParams.begin(); param!=hostParams.end(); param++) { string key = param->first.as<string>(); // cout << " PROCESSING: " << key << endl; if (key == "logging" || key == "binary" || key == "server" || key == "server_group") { continue; } if (key == "roles") { continue; } else if (key == "peers") { peerParams["peers"] = hostParams["peers"]; } else if (key == "me") { Node meList = param->second; YAML::Emitter emit; emit << YAML::Flow << meList; for (std::size_t i=0; i<meList.size(); i++) { peers.push_back(meList[i]); } } else if (key == "data") { // TODO: Datafiles per group. This is a hack // that only includes the data files from the first peer group // and assigns them to any peer Node dataFilesNode = param->second[0]; for(YAML::const_iterator it=dataFilesNode.begin();it!=dataFilesNode.end();++it) { DataFile f; auto d = *it; f.path = d["path"].as<string>(); f.varName = d["var"].as<string>(); f.policy = d["policy"].as<string>(); dataFiles.push_back(f); } } //else if (key == "datavar") { // datavar = param->second.as<string>(); //} //else if (key == "datapath") { // datapath = "{path: " + param->second.as<string>() + "}"; //} //else if (key == "datapolicy") { // datapolicy = param->second.as<string>(); //} else if (key == "totalPeers") { totalPeerCount = param->second.as<int>(); } else if (key == "peerStart") { peerStart = param->second.as<int>(); } else if (key == "peerEnd") { peerEnd = param->second.as<int>(); } else if (key == "globals") { // handled per peer } else { // string value = i->second.as<string>(); //peerParams[key] = param->second; } } // DATA ALLOCATION * // TODO: Convert to multiple input dirs map<string, vector<string> > peerFiles[peers.size()]; for (auto dataFile : dataFiles) { cout << "Top of loop" << endl; vector<string> filePaths; // 1. GET DIR LIST IN datavar DIR *datadir = NULL; datadir = opendir(dataFile.path.c_str()); if (!datadir) { cout << "Failed to open data dir: " << dataFile.path << endl; TaskStatus status; status.mutable_task_id()->MergeFrom(task.task_id()); status.set_state(TASK_FAILED); driver->sendStatusUpdate(status); return; } else { cout << "Opened data dir: " << dataFile.path << endl; } struct dirent *srcfile = NULL; while (true) { srcfile = readdir(datadir); if (srcfile == NULL) { break; } cout << "FILE " << srcfile->d_name << ": "; if (srcfile->d_type == DT_REG) { string filename = srcfile->d_name; filePaths.push_back(dataFile.path + "/" + filename); cout << "Added -> " << filename; } cout << endl; } closedir(datadir); cout << "read directory" << endl; int numfiles = filePaths.size(); sort (filePaths.begin(), filePaths.end()); int p_start = 0; int p_end = numfiles; int p_total = peers.size(); int myfiles = 0; if (dataFile.policy == "global") { for (int i = 0; i < numfiles; i++) { int peer = i % totalPeerCount; if (peer >= peerStart && peer <= peerEnd) { myfiles++; peerFiles[peer-peerStart][dataFile.varName].push_back(filePaths[i]); } } } else if (dataFile.policy == "replicate") { for (int p = 0; p < peers.size(); p++) { for (int i =0; i < numfiles; i++) { myfiles++; peerFiles[p][dataFile.varName].push_back(filePaths[i]); } } } //if (dataFile.policy == "global") { // p_start = (numfiles / totalPeerCount) * peerStart; // p_end = (numfiles / totalPeerCount) * (peerEnd+1); // p_total = totalPeerCount; // cout << ("Global files s=" + stringify(p_start) + " e=" + stringify(p_end) + " t=" + stringify(p_total)) << endl; // for (int filenum = p_start; filenum < p_end; filenum++) { // int peer = floor((((p_total)*1.0*filenum) / numfiles)) - peerStart; // cout << " Peer # " << peer << " : [" << filenum << "] " << filePaths[filenum] << endl; // peerFiles[peer][dataFile.varName].push_back(filePaths[filenum]); // myfiles++; // } //} else if (dataFile.policy == "pinned") { for(int filenum = 0; filenum < numfiles; filenum++) { peerFiles[0][dataFile.varName].push_back(filePaths[filenum]); } } else if (dataFile.policy == "sharded") { for (int i =0; i < numfiles; i++) { myfiles++; int p = i % peers.size(); peerFiles[p][dataFile.varName].push_back(filePaths[i]); } } cout << "my files: " << myfiles << endl; } cout << "BUILDING PARAMS FOR PEERS" << endl; int pph = 0; if (peerParams["peers"].size() >= 1) { YAML::Node peer_masters; YAML::Node masters; YAML::Node curMaster = YAML::Load(YAML::Dump(peerParams["peers"][0])); masters.push_back(YAML::Load(YAML::Dump(curMaster))); std::cout << peerParams["peers"].size() << " peers to map" << endl; for (std::size_t i=0; i< peerParams["peers"].size(); i++) { YAML::Node kv; if (peerParams["peers"][i]["addr"][0].as<string>() != curMaster["addr"][0].as<string>()) { cout << "Host: " << curMaster["addr"][0].as<string>() << ". Peers: " << pph << endl; pph = 0; masters.push_back(YAML::Load(YAML::Dump(peerParams["peers"][i]))); curMaster = YAML::Load(YAML::Dump(peerParams["peers"][i])); } pph++; std::cout << "added one" << endl; kv["key"] = YAML::Load(YAML::Dump(peerParams["peers"][i]["addr"])); kv["value"] = YAML::Load(YAML::Dump(curMaster["addr"])); peer_masters.push_back(kv); } cout << "Host: " << curMaster["addr"][0].as<string>() << ". Peers: " << pph << endl; peerParams["peer_masters"] = YAML::Load(YAML::Dump(peer_masters)); peerParams["masters"] = YAML::Load(YAML::Dump(masters)); std::cout << "Masters: " << YAML::Dump(masters) << endl; } std::ostringstream oss; oss << "PEERS!!! (" << std::endl; for (std::size_t i=0; i<peers.size(); i++) { oss << "---" << std::endl; YAML::Node thispeer = peerParams; YAML::Node globals = hostParams["globals"][i]; for (const_iterator p=globals.begin(); p!=globals.end(); p++) { thispeer[p->first.as<string>()] = p->second; } YAML::Node me = peers[i]; thispeer["me"] = me; YAML::Node local_peers; std::cout << "start: " << peerStart << ". end: " << peerEnd << std::endl; for (int j=peerStart; j<= peerEnd; j++) { local_peers.push_back(YAML::Load(YAML::Dump(peerParams["peers"][j]))); } thispeer["local_peers"] = YAML::Load(YAML::Dump(local_peers)); for (auto it : peerFiles[i]) { auto datavar = it.first; if (thispeer[datavar]) { thispeer.remove(datavar); } for (auto &f : it.second) { Node src; src["path"] = f; thispeer[datavar].push_back(src); } } // ADD DATA SOURCE DIR HERE YAML::Emitter emit; emit << YAML::Flow << thispeer; string param = emit.c_str(); std::ofstream peerFile; string peerFileName = "/mnt/mesos/sandbox/peers" + std::to_string(i) + ".yaml"; peerFile.open(peerFileName, std::ofstream::out); peerFile << param; peerFile.close(); oss << param << std::endl; std::cout << param << std::endl; k3_cmd += " -p " + peerFileName; for (auto it : peerFiles[i]) { auto datavar = it.first; if (thispeer[datavar]) { thispeer.remove(datavar); } } } oss << ") END PEERS!!!" << std::endl; cout << oss.str() << std::endl; k3_cmd += "'"; cout << "FINAL COMMAND: " << k3_cmd << endl; if (thread) { driver->sendFrameworkMessage("Debug: thread already existed!"); thread->interrupt(); thread->join(); delete thread; thread = 0; } bool isMaster = false; cout << "Checking master" << endl; if (Dump(hostParams["me"][0]) == Dump(hostParams["master"])) { isMaster = true; cout << "I am master" << endl; } else { cout << "me: " << Dump(hostParams["me"][0]) << endl; cout << "master: " << Dump(hostParams["master"]) << endl; } cout << "Launching K3: " << endl; thread = new boost::thread(TaskThread(task, k3_cmd, driver, isMaster)); }
void launchTask(ExecutorDriver* driver, const TaskInfo& task) { if (launched) { TaskStatus status; status.mutable_task_id()->MergeFrom(task.task_id()); status.set_state(TASK_FAILED); status.set_message( "Attempted to run multiple tasks using a \"command\" executor"); driver->sendStatusUpdate(status); return; } CHECK(task.has_command()) << "Expecting task " << task.task_id() << " to have a command!"; std::cout << "Starting task " << task.task_id() << std::endl; // TODO(benh): Clean this up with the new 'Fork' abstraction. // Use pipes to determine which child has successfully changed // session. This is needed as the setsid call can fail from other // processes having the same group id. int pipes[2]; if (pipe(pipes) < 0) { perror("Failed to create a pipe"); abort(); } // Set the FD_CLOEXEC flags on these pipes Try<Nothing> cloexec = os::cloexec(pipes[0]); if (cloexec.isError()) { std::cerr << "Failed to cloexec(pipe[0]): " << cloexec.error() << std::endl; abort(); } cloexec = os::cloexec(pipes[1]); if (cloexec.isError()) { std::cerr << "Failed to cloexec(pipe[1]): " << cloexec.error() << std::endl; abort(); } if ((pid = fork()) == -1) { std::cerr << "Failed to fork to run '" << task.command().value() << "': " << strerror(errno) << std::endl; abort(); } if (pid == 0) { // In child process, we make cleanup easier by putting process // into it's own session. os::close(pipes[0]); // NOTE: We setsid() in a loop because setsid() might fail if another // process has the same process group id as the calling process. while ((pid = setsid()) == -1) { perror("Could not put command in its own session, setsid"); std::cout << "Forking another process and retrying" << std::endl; if ((pid = fork()) == -1) { perror("Failed to fork to launch command"); abort(); } if (pid > 0) { // In parent process. It is ok to suicide here, because // we're not watching this process. exit(0); } } if (write(pipes[1], &pid, sizeof(pid)) != sizeof(pid)) { perror("Failed to write PID on pipe"); abort(); } os::close(pipes[1]); // The child has successfully setsid, now run the command. std::cout << "sh -c '" << task.command().value() << "'" << std::endl; execl("/bin/sh", "sh", "-c", task.command().value().c_str(), (char*) NULL); perror("Failed to exec"); abort(); } // In parent process. os::close(pipes[1]); // Get the child's pid via the pipe. if (read(pipes[0], &pid, sizeof(pid)) == -1) { std::cerr << "Failed to get child PID from pipe, read: " << strerror(errno) << std::endl; abort(); } os::close(pipes[0]); std::cout << "Forked command at " << pid << std::endl; // Monitor this process. process::reap(pid) .onAny(defer(self(), &Self::reaped, driver, task.task_id(), pid, lambda::_1)); TaskStatus status; status.mutable_task_id()->MergeFrom(task.task_id()); status.set_state(TASK_RUNNING); driver->sendStatusUpdate(status); launched = true; }