// data format is:
// <MessageToExecutor>.<OSDID> for OSD executor
// or just <MessageToExecutor> for MON executor
void CephExecutor::frameworkMessage(ExecutorDriver* driver, const string& data)
{
LOG(INFO) << "Got framework message: " << data;
MessageToExecutor msg;
vector<string> tokens = StringUtil::explode(data,'.');
msg = (MessageToExecutor)lexical_cast<int>(tokens[0]);
switch (msg){
case MessageToExecutor::REGISTER_OSD:
LOG(INFO) << "Will register an OSD, and return the OSD ID";
driver->sendFrameworkMessage(registerOSD());
break;
case MessageToExecutor::LAUNCH_OSD:
if (tokens.size() == 2){
LOG(INFO) << "Will launch OSD docker with OSD ID: " << tokens[1];
string dockerCommand = constructOSDCommand(
localSharedConfigDirRoot + "/" + localConfigDirName,
tokens[1],
containerName);
myPID = fork();
if (0 == myPID){
//child long running docker thread
//TODO: we use fork here. Need to check why below line will hung the executor
//thread(&CephExecutor::startLongRunning,*this,"docker", dockerCommand).detach();
startLongRunning("docker",dockerCommand);
} else {
bool started = block_until_started(containerName, "30");
TaskStatus status;
status.mutable_task_id()->MergeFrom(myTaskId);
if (started) {
LOG(INFO) << "Starting OSD task " << myTaskId.value();
//send the OSD id back to let scheduler remove it
//format: <MessageToScheduler::CONSUMED_OSD_ID>.OSDID
string msg =
lexical_cast<string>(static_cast<int>(MessageToScheduler::CONSUMED_OSD_ID)) +
"." + tokens[1];
status.set_message(msg);
status.set_state(TASK_RUNNING);
} else {
LOG(INFO) << "Failed to start OSD task " << myTaskId.value();
status.set_state(TASK_FAILED);
}
driver->sendStatusUpdate(status);
}//end else "0==pid"
} else {
LOG(INFO) << "No OSD ID given!";
}
break;
default:
LOG(INFO) << "unknown message from scheduler";
}
}
inline StatusUpdate createStatusUpdate(
const FrameworkID& frameworkId,
const SlaveID& slaveId,
const TaskID& taskId,
const TaskState& state,
const std::string& message = "",
const Option<ExecutorID>& executorId = None())
{
StatusUpdate update;
update.set_timestamp(process::Clock::now().secs());
update.set_uuid(UUID::random().toBytes());
update.mutable_framework_id()->MergeFrom(frameworkId);
update.mutable_slave_id()->MergeFrom(slaveId);
if (executorId.isSome()) {
update.mutable_executor_id()->MergeFrom(executorId.get());
}
TaskStatus* status = update.mutable_status();
status->mutable_task_id()->MergeFrom(taskId);
status->mutable_slave_id()->MergeFrom(slaveId);
status->set_state(state);
status->set_message(message);
status->set_timestamp(update.timestamp());
return update;
}
void update(const TaskInfo& task, const TaskState& state)
{
UUID uuid = UUID::random();
TaskStatus status;
status.mutable_task_id()->CopyFrom(task.task_id());
status.mutable_executor_id()->CopyFrom(executorId);
status.set_state(state);
status.set_source(TaskStatus::SOURCE_EXECUTOR);
status.set_timestamp(process::Clock::now().secs());
status.set_uuid(uuid.toBytes());
Call call;
call.mutable_framework_id()->CopyFrom(frameworkId);
call.mutable_executor_id()->CopyFrom(executorId);
call.set_type(Call::UPDATE);
call.mutable_update()->mutable_status()->CopyFrom(status);
// Capture the status update.
updates[uuid] = call.update();
mesos->send(call);
}
TaskStatus createTaskStatus(
TaskStatus status,
const id::UUID& uuid,
double timestamp,
const Option<TaskState>& state,
const Option<string>& message,
const Option<TaskStatus::Source>& source,
const Option<TaskStatus::Reason>& reason,
const Option<string>& data,
const Option<bool>& healthy,
const Option<CheckStatusInfo>& checkStatus,
const Option<Labels>& labels,
const Option<ContainerStatus>& containerStatus,
const Option<TimeInfo>& unreachableTime)
{
status.set_uuid(uuid.toBytes());
status.set_timestamp(timestamp);
if (state.isSome()) {
status.set_state(state.get());
}
if (message.isSome()) {
status.set_message(message.get());
}
if (source.isSome()) {
status.set_source(source.get());
}
if (reason.isSome()) {
status.set_reason(reason.get());
}
if (data.isSome()) {
status.set_data(data.get());
}
if (healthy.isSome()) {
status.set_healthy(healthy.get());
}
if (checkStatus.isSome()) {
status.mutable_check_status()->CopyFrom(checkStatus.get());
}
if (labels.isSome()) {
status.mutable_labels()->CopyFrom(labels.get());
}
if (containerStatus.isSome()) {
status.mutable_container_status()->CopyFrom(containerStatus.get());
}
if (unreachableTime.isSome()) {
status.mutable_unreachable_time()->CopyFrom(unreachableTime.get());
}
return status;
}
// This test ensures that the driver handles an empty slave id
// in an acknowledgement message by dropping it. The driver will
// log an error in this case (but we don't test for that). We
// generate a status with no slave id by performing reconciliation.
TEST_F(MesosSchedulerDriverTest, ExplicitAcknowledgementsUnsetSlaveID)
{
Try<Owned<cluster::Master>> master = StartMaster();
ASSERT_SOME(master);
MockScheduler sched;
MesosSchedulerDriver driver(
&sched,
DEFAULT_FRAMEWORK_INFO,
master.get()->pid,
false,
DEFAULT_CREDENTIAL);
Future<Nothing> registered;
EXPECT_CALL(sched, registered(&driver, _, _))
.WillOnce(FutureSatisfy(®istered));
// Ensure no status update acknowledgements are sent to the master.
EXPECT_NO_FUTURE_CALLS(
mesos::scheduler::Call(),
mesos::scheduler::Call::ACKNOWLEDGE,
_ ,
master.get()->pid);
driver.start();
AWAIT_READY(registered);
Future<TaskStatus> update;
EXPECT_CALL(sched, statusUpdate(&driver, _))
.WillOnce(FutureArg<1>(&update));
// Peform reconciliation without using a slave id.
vector<TaskStatus> statuses;
TaskStatus status;
status.mutable_task_id()->set_value("foo");
status.set_state(TASK_RUNNING);
statuses.push_back(status);
driver.reconcileTasks(statuses);
AWAIT_READY(update);
ASSERT_EQ(TASK_LOST, update.get().state());
ASSERT_EQ(TaskStatus::SOURCE_MASTER, update.get().source());
ASSERT_EQ(TaskStatus::REASON_RECONCILIATION, update.get().reason());
ASSERT_FALSE(update.get().has_slave_id());
// Now send the acknowledgement.
driver.acknowledgeStatusUpdate(update.get());
// Settle the clock to ensure driver processes the acknowledgement,
// which should get dropped due to the missing slave id.
Clock::pause();
Clock::settle();
driver.stop();
driver.join();
}
// This test verifies that reconciliation of an unknown task that
// belongs to a known slave results in TASK_LOST.
TEST_F(ReconciliationTest, UnknownTask)
{
Try<PID<Master> > master = StartMaster();
ASSERT_SOME(master);
Future<SlaveRegisteredMessage> slaveRegisteredMessage =
FUTURE_PROTOBUF(SlaveRegisteredMessage(), _, _);
Try<PID<Slave> > slave = StartSlave();
ASSERT_SOME(slave);
// Wait for the slave to register and get the slave id.
AWAIT_READY(slaveRegisteredMessage);
const SlaveID slaveId = slaveRegisteredMessage.get().slave_id();
MockScheduler sched;
MesosSchedulerDriver driver(
&sched, DEFAULT_FRAMEWORK_INFO, master.get(), DEFAULT_CREDENTIAL);
Future<FrameworkID> frameworkId;
EXPECT_CALL(sched, registered(&driver, _, _))
.WillOnce(FutureArg<1>(&frameworkId));
EXPECT_CALL(sched, resourceOffers(&driver, _))
.WillRepeatedly(Return()); // Ignore offers.
driver.start();
// Wait until the framework is registered.
AWAIT_READY(frameworkId);
Future<TaskStatus> update;
EXPECT_CALL(sched, statusUpdate(&driver, _))
.WillOnce(FutureArg<1>(&update));
vector<TaskStatus> statuses;
// Create a task status with a random task id.
TaskStatus status;
status.mutable_task_id()->set_value(UUID::random().toString());
status.mutable_slave_id()->CopyFrom(slaveId);
status.set_state(TASK_RUNNING);
statuses.push_back(status);
driver.reconcileTasks(statuses);
// Framework should receive TASK_LOST for unknown task.
AWAIT_READY(update);
EXPECT_EQ(TASK_LOST, update.get().state());
driver.stop();
driver.join();
Shutdown(); // Must shutdown before 'containerizer' gets deallocated.
}
void reaped(
ExecutorDriver* driver,
const TaskID& taskId,
pid_t pid,
const Future<Option<int> >& status_)
{
TaskState state;
string message;
Timer::cancel(escalationTimer);
if (!status_.isReady()) {
state = TASK_FAILED;
message =
"Failed to get exit status for Command: " +
(status_.isFailed() ? status_.failure() : "future discarded");
} else if (status_.get().isNone()) {
state = TASK_FAILED;
message = "Failed to get exit status for Command";
} else {
int status = status_.get().get();
CHECK(WIFEXITED(status) || WIFSIGNALED(status)) << status;
if (WIFEXITED(status) && WEXITSTATUS(status) == 0) {
state = TASK_FINISHED;
} else if (killed) {
// Send TASK_KILLED if the task was killed as a result of
// killTask() or shutdown().
state = TASK_KILLED;
} else {
state = TASK_FAILED;
}
message = string("Command") +
(WIFEXITED(status)
? " exited with status "
: " terminated with signal ") +
(WIFEXITED(status)
? stringify(WEXITSTATUS(status))
: strsignal(WTERMSIG(status)));
}
cout << message << " (pid: " << pid << ")" << endl;
TaskStatus taskStatus;
taskStatus.mutable_task_id()->MergeFrom(taskId);
taskStatus.set_state(state);
taskStatus.set_message(message);
driver->sendStatusUpdate(taskStatus);
// A hack for now ... but we need to wait until the status update
// is sent to the slave before we shut ourselves down.
os::sleep(Seconds(1));
driver->stop();
}
void CephExecutor::shutdown(ExecutorDriver* driver)
{
LOG(INFO) << "Killing this container process";
LOG(INFO) << runShellCommand("docker rm -f " + containerName);
TaskStatus status;
status.mutable_task_id()->MergeFrom(myTaskId);
status.set_state(TASK_KILLED);
driver->sendStatusUpdate(status);
}
void run(ExecutorDriver* driver, const TaskInfo& task)
{
os::sleep(Seconds(random() % 10));
TaskStatus status;
status.mutable_task_id()->MergeFrom(task.task_id());
status.set_state(TASK_FINISHED);
driver->sendStatusUpdate(status);
}
void operator()() {
TaskStatus status;
status.mutable_task_id()->MergeFrom(task.task_id());
// Currently, just call the K3 executable with the generated command line from task.data()
try {
FILE* pipe = popen(k3_cmd.c_str(), "r");
if (!pipe) {
status.set_state(TASK_FAILED);
driver->sendStatusUpdate(status);
cout << "Failed to open subprocess" << endl;
}
char buffer[256];
while (!feof(pipe)) {
if (fgets(buffer, 256, pipe) != NULL) {
std::string s = std::string(buffer);
if (this->isMaster) {
driver->sendFrameworkMessage(s);
cout << s << endl;
}
else {
cout << s << endl;
}
}
}
int k3 = pclose(pipe);
if (k3 == 0) {
status.set_state(TASK_FINISHED);
cout << "Task " << task.task_id().value() << " Completed!" << endl;
driver->sendStatusUpdate(status);
}
else {
status.set_state(TASK_FAILED);
cout << "K3 Task " << task.task_id().value() << " returned error code: " << k3 << endl;
driver->sendStatusUpdate(status);
}
}
catch (...) {
status.set_state(TASK_FAILED);
driver->sendStatusUpdate(status);
}
//------------- END OF TASK -------------------
}
virtual void launchTask(ExecutorDriver* driver, const TaskInfo& task)
{
cout << "Starting task " << task.task_id().value() << endl;
TaskStatus status;
status.mutable_task_id()->MergeFrom(task.task_id());
status.set_state(TASK_RUNNING);
driver->sendStatusUpdate(status);
// This is where one would perform the requested task.
cout << "Finishing task " << task.task_id().value() << endl;
status.mutable_task_id()->MergeFrom(task.task_id());
status.set_state(TASK_FINISHED);
driver->sendStatusUpdate(status);
}
virtual void launchTask(ExecutorDriver* driver, const TaskInfo& task)
{
cout << "Starting task " << task.task_id().value() << endl;
TaskStatus status;
status.mutable_task_id()->MergeFrom(task.task_id());
status.set_state(TASK_RUNNING);
driver->sendStatusUpdate(status);
sleep(1);
cout << "Finishing task " << task.task_id().value() << endl;
status.mutable_task_id()->MergeFrom(task.task_id());
status.set_state(TASK_FINISHED);
driver->sendStatusUpdate(status);
}
void launchTask (ExecutorDriver* driver, const TaskInfo& task) override {
cout << "Starting task " << task.task_id().value() << endl;
TaskStatus status;
status.mutable_task_id()->MergeFrom(task.task_id());
StartInfo* info = new StartInfo(driver, task);
pthread_t pthread;
int res = pthread_create(&pthread, NULL, &RunProcess, info);
if (res != 0) {
status.set_state(TASK_FAILED);
delete info;
}
else {
pthread_detach(pthread);
status.set_state(TASK_RUNNING);
}
driver->sendStatusUpdate(status);
}
TaskStatus createTaskStatus(
const TaskID& taskId,
const TaskState& state,
const id::UUID& uuid,
double timestamp)
{
TaskStatus status;
status.set_uuid(uuid.toBytes());
status.set_timestamp(timestamp);
status.mutable_task_id()->CopyFrom(taskId);
status.set_state(state);
return status;
}
virtual void launchTask(ExecutorDriver* driver, const TaskInfo& task)
{
cout << "Starting task " << task.task_id().value() << endl;
lambda::function<void(void)>* thunk =
new lambda::function<void(void)>(lambda::bind(&run, driver, task));
pthread_t pthread;
if (pthread_create(&pthread, NULL, &start, thunk) != 0) {
TaskStatus status;
status.mutable_task_id()->MergeFrom(task.task_id());
status.set_state(TASK_FAILED);
driver->sendStatusUpdate(status);
} else {
pthread_detach(pthread);
TaskStatus status;
status.mutable_task_id()->MergeFrom(task.task_id());
status.set_state(TASK_RUNNING);
driver->sendStatusUpdate(status);
}
}
// TODO(vinod): Make SlaveID optional because 'StatusUpdate.SlaveID'
// is optional.
StatusUpdate createStatusUpdate(
const FrameworkID& frameworkId,
const Option<SlaveID>& slaveId,
const TaskID& taskId,
const TaskState& state,
const TaskStatus::Source& source,
const string& message = "",
const Option<TaskStatus::Reason>& reason = None(),
const Option<ExecutorID>& executorId = None(),
const Option<bool>& healthy = None())
{
StatusUpdate update;
update.set_timestamp(process::Clock::now().secs());
update.set_uuid(UUID::random().toBytes());
update.mutable_framework_id()->MergeFrom(frameworkId);
if (slaveId.isSome()) {
update.mutable_slave_id()->MergeFrom(slaveId.get());
}
if (executorId.isSome()) {
update.mutable_executor_id()->MergeFrom(executorId.get());
}
TaskStatus* status = update.mutable_status();
status->mutable_task_id()->MergeFrom(taskId);
if (slaveId.isSome()) {
status->mutable_slave_id()->MergeFrom(slaveId.get());
}
status->set_state(state);
status->set_source(source);
status->set_message(message);
status->set_timestamp(update.timestamp());
if (reason.isSome()) {
status->set_reason(reason.get());
}
if (healthy.isSome()) {
status->set_healthy(healthy.get());
}
return update;
}
// This test verifies that reconciliation of a task that belongs to an
// unknown slave results in TASK_LOST.
TEST_F(ReconciliationTest, UnknownSlave)
{
Try<PID<Master> > master = StartMaster();
ASSERT_SOME(master);
MockScheduler sched;
MesosSchedulerDriver driver(
&sched, DEFAULT_FRAMEWORK_INFO, master.get(), DEFAULT_CREDENTIAL);
Future<FrameworkID> frameworkId;
EXPECT_CALL(sched, registered(&driver, _, _))
.WillOnce(FutureArg<1>(&frameworkId));
driver.start();
// Wait until the framework is registered.
AWAIT_READY(frameworkId);
Future<TaskStatus> update;
EXPECT_CALL(sched, statusUpdate(&driver, _))
.WillOnce(FutureArg<1>(&update));
vector<TaskStatus> statuses;
// Create a task status with a random slave id (and task id).
TaskStatus status;
status.mutable_task_id()->set_value(UUID::random().toString());
status.mutable_slave_id()->set_value(UUID::random().toString());
status.set_state(TASK_RUNNING);
statuses.push_back(status);
driver.reconcileTasks(statuses);
// Framework should receive TASK_LOST because the slave is unknown.
AWAIT_READY(update);
EXPECT_EQ(TASK_LOST, update.get().state());
driver.stop();
driver.join();
Shutdown();
}
virtual void launchTask(ExecutorDriver* driver, const TaskDescription& task)
{
cout << "Executor starting task " << task.task_id().value() << endl;
int64_t memToHog;
double duration;
int numThreads;
istringstream in(task.data());
in >> memToHog >> duration >> numThreads;
memToHog *= 1024LL * 1024LL; // Convert from MB to bytes
for (int i = 0; i < numThreads; i++) {
ThreadArg* arg = new ThreadArg(this, task, i, memToHog, duration);
pthread_t thread;
pthread_create(&thread, 0, runTask, arg);
pthread_detach(thread);
TaskStatus status;
status.mutable_task_id()->MergeFrom(task.task_id());
status.set_state(TASK_RUNNING);
driver->sendStatusUpdate(status);
}
}
// This test verifies that the slave task status label decorator can
// add and remove labels from a TaskStatus during the status update
// sequence. A TaskStatus with two labels ("foo":"bar" and
// "bar":"baz") is sent from the executor. The labels get modified by
// the slave hook to strip the "foo":"bar" pair and/ add a new
// "baz":"qux" pair.
TEST_F(HookTest, VerifySlaveTaskStatusDecorator)
{
Try<Owned<cluster::Master>> master = StartMaster();
ASSERT_SOME(master);
MockExecutor exec(DEFAULT_EXECUTOR_ID);
TestContainerizer containerizer(&exec);
Owned<MasterDetector> detector = master.get()->createDetector();
Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), &containerizer);
ASSERT_SOME(slave);
MockScheduler sched;
MesosSchedulerDriver driver(
&sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL);
EXPECT_CALL(sched, registered(&driver, _, _));
Future<vector<Offer>> offers;
EXPECT_CALL(sched, resourceOffers(&driver, _))
.WillOnce(FutureArg<1>(&offers))
.WillRepeatedly(Return()); // Ignore subsequent offers.
driver.start();
AWAIT_READY(offers);
ASSERT_EQ(1u, offers.get().size());
// Start a task.
TaskInfo task = createTask(offers.get()[0], "", DEFAULT_EXECUTOR_ID);
ExecutorDriver* execDriver;
EXPECT_CALL(exec, registered(_, _, _, _))
.WillOnce(SaveArg<0>(&execDriver));
Future<TaskInfo> execTask;
EXPECT_CALL(exec, launchTask(_, _))
.WillOnce(FutureArg<1>(&execTask));
Future<TaskStatus> status;
EXPECT_CALL(sched, statusUpdate(&driver, _))
.WillOnce(FutureArg<1>(&status));
driver.launchTasks(offers.get()[0].id(), {task});
AWAIT_READY(execTask);
// Now send TASK_RUNNING update with two labels. The first label
// ("foo:bar") will be removed by the task status hook to ensure
// that it can remove labels. The second label will be preserved
// and forwarded to Master (and eventually to the framework).
// The hook also adds a new label with the same key but a different
// value ("bar:quz").
TaskStatus runningStatus;
runningStatus.mutable_task_id()->MergeFrom(execTask.get().task_id());
runningStatus.set_state(TASK_RUNNING);
// Add two labels to the TaskStatus
Labels* labels = runningStatus.mutable_labels();
labels->add_labels()->CopyFrom(createLabel("foo", "bar"));
labels->add_labels()->CopyFrom(createLabel("bar", "baz"));
execDriver->sendStatusUpdate(runningStatus);
AWAIT_READY(status);
// The hook will hang an extra label off.
const Labels& labels_ = status.get().labels();
EXPECT_EQ(2, labels_.labels_size());
// The test hook will prepend a new "baz":"qux" label.
EXPECT_EQ("bar", labels_.labels(0).key());
EXPECT_EQ("qux", labels_.labels(0).value());
// And lastly, we only expect the "foo":"bar" pair to be stripped by
// the module. The last pair should be the original "bar":"baz"
// pair set by the test.
EXPECT_EQ("bar", labels_.labels(1).key());
EXPECT_EQ("baz", labels_.labels(1).value());
// Now validate TaskInfo.container_status. We must have received a
// container_status with one network_info set by the test hook module.
EXPECT_TRUE(status.get().has_container_status());
EXPECT_EQ(1, status.get().container_status().network_infos().size());
const NetworkInfo networkInfo =
status.get().container_status().network_infos(0);
// The hook module sets up '4.3.2.1' as the IP address and 'public' as the
// network isolation group. The `ip_address` field is deprecated, but the
// hook module should continue to set it as well as the new `ip_addresses`
// field for now.
EXPECT_TRUE(networkInfo.has_ip_address());
EXPECT_EQ("4.3.2.1", networkInfo.ip_address());
EXPECT_EQ(1, networkInfo.ip_addresses().size());
EXPECT_TRUE(networkInfo.ip_addresses(0).has_ip_address());
EXPECT_EQ("4.3.2.1", networkInfo.ip_addresses(0).ip_address());
EXPECT_EQ(1, networkInfo.groups().size());
EXPECT_EQ("public", networkInfo.groups(0));
EXPECT_TRUE(networkInfo.has_labels());
EXPECT_EQ(1, networkInfo.labels().labels().size());
const Label networkInfoLabel = networkInfo.labels().labels(0);
// Finally, the labels set inside NetworkInfo by the hook module.
EXPECT_EQ("net_foo", networkInfoLabel.key());
EXPECT_EQ("net_bar", networkInfoLabel.value());
EXPECT_CALL(exec, shutdown(_))
.Times(AtMost(1));
driver.stop();
driver.join();
}
// This test verifies that a reconciliation request that comes before
// '_launchTasks()' is ignored.
TEST_F(MasterAuthorizationTest, ReconcileTask)
{
MockAuthorizer authorizer;
Try<PID<Master> > master = StartMaster(&authorizer);
ASSERT_SOME(master);
MockExecutor exec(DEFAULT_EXECUTOR_ID);
Try<PID<Slave> > slave = StartSlave(&exec);
ASSERT_SOME(slave);
MockScheduler sched;
MesosSchedulerDriver driver(
&sched, DEFAULT_FRAMEWORK_INFO, master.get(), DEFAULT_CREDENTIAL);
EXPECT_CALL(sched, registered(&driver, _, _))
.Times(1);
Future<vector<Offer> > offers;
EXPECT_CALL(sched, resourceOffers(&driver, _))
.WillOnce(FutureArg<1>(&offers))
.WillRepeatedly(Return()); // Ignore subsequent offers.
driver.start();
AWAIT_READY(offers);
EXPECT_NE(0u, offers.get().size());
TaskInfo task = createTask(offers.get()[0], "", DEFAULT_EXECUTOR_ID);
vector<TaskInfo> tasks;
tasks.push_back(task);
// Return a pending future from authorizer.
Future<Nothing> future;
Promise<bool> promise;
EXPECT_CALL(authorizer, authorize(An<const mesos::ACL::RunTasks&>()))
.WillOnce(DoAll(FutureSatisfy(&future),
Return(promise.future())));
driver.launchTasks(offers.get()[0].id(), tasks);
// Wait until authorization is in progress.
AWAIT_READY(future);
// Scheduler shouldn't get an update from reconciliation.
EXPECT_CALL(sched, statusUpdate(&driver, _))
.Times(0);
Future<ReconcileTasksMessage> reconcileTasksMessage =
FUTURE_PROTOBUF(ReconcileTasksMessage(), _, _);
vector<TaskStatus> statuses;
TaskStatus status;
status.mutable_task_id()->CopyFrom(task.task_id());
status.mutable_slave_id()->CopyFrom(offers.get()[0].slave_id());
status.set_state(TASK_STAGING);
statuses.push_back(status);
driver.reconcileTasks(statuses);
AWAIT_READY(reconcileTasksMessage);
// Make sure the framework doesn't receive any update.
Clock::pause();
Clock::settle();
// Now stop the framework.
driver.stop();
driver.join();
Shutdown(); // Must shutdown before 'containerizer' gets deallocated.
}
// This test verifies that a re-registering slave sends the terminal
// unacknowledged tasks for a terminal executor. This is required
// for the master to correctly reconcile its view with the slave's
// view of tasks. This test drops a terminal update to the master
// and then forces the slave to re-register.
TEST_F(MasterSlaveReconciliationTest, SlaveReregisterTerminatedExecutor)
{
Try<Owned<cluster::Master>> master = StartMaster();
ASSERT_SOME(master);
MockExecutor exec(DEFAULT_EXECUTOR_ID);
TestContainerizer containerizer(&exec);
StandaloneMasterDetector detector(master.get()->pid);
Try<Owned<cluster::Slave>> slave = StartSlave(&detector, &containerizer);
ASSERT_SOME(slave);
MockScheduler sched;
MesosSchedulerDriver driver(
&sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL);
Future<FrameworkID> frameworkId;
EXPECT_CALL(sched, registered(&driver, _, _))
.WillOnce(FutureArg<1>(&frameworkId));
EXPECT_CALL(sched, resourceOffers(&driver, _))
.WillOnce(LaunchTasks(DEFAULT_EXECUTOR_INFO, 1, 1, 512, "*"))
.WillRepeatedly(Return()); // Ignore subsequent offers.
ExecutorDriver* execDriver;
EXPECT_CALL(exec, registered(_, _, _, _))
.WillOnce(SaveArg<0>(&execDriver));
EXPECT_CALL(exec, launchTask(_, _))
.WillOnce(SendStatusUpdateFromTask(TASK_RUNNING));
Future<TaskStatus> status;
EXPECT_CALL(sched, statusUpdate(&driver, _))
.WillOnce(FutureArg<1>(&status));
Future<StatusUpdateAcknowledgementMessage> statusUpdateAcknowledgementMessage
= FUTURE_PROTOBUF(
StatusUpdateAcknowledgementMessage(),
master.get()->pid,
slave.get()->pid);
driver.start();
AWAIT_READY(status);
EXPECT_EQ(TASK_RUNNING, status.get().state());
// Make sure the acknowledgement reaches the slave.
AWAIT_READY(statusUpdateAcknowledgementMessage);
// Drop the TASK_FINISHED status update sent to the master.
Future<StatusUpdateMessage> statusUpdateMessage =
DROP_PROTOBUF(StatusUpdateMessage(), _, master.get()->pid);
Future<ExitedExecutorMessage> executorExitedMessage =
FUTURE_PROTOBUF(ExitedExecutorMessage(), _, _);
TaskStatus finishedStatus;
finishedStatus = status.get();
finishedStatus.set_state(TASK_FINISHED);
execDriver->sendStatusUpdate(finishedStatus);
// Ensure the update was sent.
AWAIT_READY(statusUpdateMessage);
EXPECT_CALL(sched, executorLost(&driver, DEFAULT_EXECUTOR_ID, _, _));
// Now kill the executor.
containerizer.destroy(frameworkId.get(), DEFAULT_EXECUTOR_ID);
Future<TaskStatus> status2;
EXPECT_CALL(sched, statusUpdate(&driver, _))
.WillOnce(FutureArg<1>(&status2));
// We drop the 'UpdateFrameworkMessage' from the master to slave to
// stop the status update manager from retrying the update that was
// already sent due to the new master detection.
DROP_PROTOBUFS(UpdateFrameworkMessage(), _, _);
detector.appoint(master.get()->pid);
AWAIT_READY(status2);
EXPECT_EQ(TASK_FINISHED, status2.get().state());
driver.stop();
driver.join();
}
// This test ensures that reconciliation requests for tasks that are
// pending are exposed in reconciliation.
TEST_F(ReconciliationTest, PendingTask)
{
MockAuthorizer authorizer;
Try<PID<Master> > master = StartMaster(&authorizer);
ASSERT_SOME(master);
MockExecutor exec(DEFAULT_EXECUTOR_ID);
Future<SlaveRegisteredMessage> slaveRegisteredMessage =
FUTURE_PROTOBUF(SlaveRegisteredMessage(), _, _);
Try<PID<Slave> > slave = StartSlave();
ASSERT_SOME(slave);
// Wait for the slave to register and get the slave id.
AWAIT_READY(slaveRegisteredMessage);
const SlaveID slaveId = slaveRegisteredMessage.get().slave_id();
MockScheduler sched;
MesosSchedulerDriver driver(
&sched, DEFAULT_FRAMEWORK_INFO, master.get(), DEFAULT_CREDENTIAL);
EXPECT_CALL(sched, registered(&driver, _, _))
.Times(1);
Future<vector<Offer> > offers;
EXPECT_CALL(sched, resourceOffers(&driver, _))
.WillOnce(FutureArg<1>(&offers))
.WillRepeatedly(Return()); // Ignore subsequent offers.
driver.start();
AWAIT_READY(offers);
EXPECT_NE(0u, offers.get().size());
// Return a pending future from authorizer.
Future<Nothing> authorize;
Promise<bool> promise;
EXPECT_CALL(authorizer, authorize(An<const mesos::ACL::RunTask&>()))
.WillOnce(DoAll(FutureSatisfy(&authorize),
Return(promise.future())));
TaskInfo task = createTask(offers.get()[0], "", DEFAULT_EXECUTOR_ID);
vector<TaskInfo> tasks;
tasks.push_back(task);
driver.launchTasks(offers.get()[0].id(), tasks);
// Wait until authorization is in progress.
AWAIT_READY(authorize);
// First send an implicit reconciliation request for this task.
Future<TaskStatus> update;
EXPECT_CALL(sched, statusUpdate(&driver, _))
.WillOnce(FutureArg<1>(&update));
vector<TaskStatus> statuses;
driver.reconcileTasks(statuses);
AWAIT_READY(update);
EXPECT_EQ(TASK_STAGING, update.get().state());
EXPECT_TRUE(update.get().has_slave_id());
// Now send an explicit reconciliation request for this task.
Future<TaskStatus> update2;
EXPECT_CALL(sched, statusUpdate(&driver, _))
.WillOnce(FutureArg<1>(&update2));
TaskStatus status;
status.mutable_task_id()->CopyFrom(task.task_id());
status.mutable_slave_id()->CopyFrom(slaveId);
status.set_state(TASK_STAGING);
statuses.push_back(status);
driver.reconcileTasks(statuses);
AWAIT_READY(update2);
EXPECT_EQ(TASK_STAGING, update2.get().state());
EXPECT_TRUE(update2.get().has_slave_id());
driver.stop();
driver.join();
Shutdown(); // Must shutdown before 'containerizer' gets deallocated.
}
// This test verifies that reconciliation sends the latest task
// status, when the task state does not match between the framework
// and the master.
TEST_F(ReconciliationTest, TaskStateMismatch)
{
Try<PID<Master> > master = StartMaster();
ASSERT_SOME(master);
MockExecutor exec(DEFAULT_EXECUTOR_ID);
TestContainerizer containerizer(&exec);
Try<PID<Slave> > slave = StartSlave(&containerizer);
ASSERT_SOME(slave);
MockScheduler sched;
MesosSchedulerDriver driver(
&sched, DEFAULT_FRAMEWORK_INFO, master.get(), DEFAULT_CREDENTIAL);
Future<FrameworkID> frameworkId;
EXPECT_CALL(sched, registered(&driver, _, _))
.WillOnce(FutureArg<1>(&frameworkId));
EXPECT_CALL(sched, resourceOffers(&driver, _))
.WillOnce(LaunchTasks(DEFAULT_EXECUTOR_INFO, 1, 1, 512, "*"))
.WillRepeatedly(Return()); // Ignore subsequent offers.
EXPECT_CALL(exec, registered(_, _, _, _));
EXPECT_CALL(exec, launchTask(_, _))
.WillOnce(SendStatusUpdateFromTask(TASK_RUNNING));
Future<TaskStatus> update;
EXPECT_CALL(sched, statusUpdate(&driver, _))
.WillOnce(FutureArg<1>(&update));
driver.start();
// Wait until the framework is registered.
AWAIT_READY(frameworkId);
AWAIT_READY(update);
EXPECT_EQ(TASK_RUNNING, update.get().state());
EXPECT_EQ(true, update.get().has_slave_id());
const TaskID taskId = update.get().task_id();
const SlaveID slaveId = update.get().slave_id();
// If framework has different state, current state should be reported.
Future<TaskStatus> update2;
EXPECT_CALL(sched, statusUpdate(&driver, _))
.WillOnce(FutureArg<1>(&update2));
vector<TaskStatus> statuses;
TaskStatus status;
status.mutable_task_id()->CopyFrom(taskId);
status.mutable_slave_id()->CopyFrom(slaveId);
status.set_state(TASK_KILLED);
statuses.push_back(status);
driver.reconcileTasks(statuses);
AWAIT_READY(update2);
EXPECT_EQ(TASK_RUNNING, update2.get().state());
EXPECT_CALL(exec, shutdown(_))
.Times(AtMost(1));
driver.stop();
driver.join();
Shutdown(); // Must shutdown before 'containerizer' gets deallocated.
}
// This test verifies that reconciliation of a task that belongs to a
// slave that is a transitional state doesn't result in an update.
TEST_F(ReconciliationTest, SlaveInTransition)
{
master::Flags masterFlags = CreateMasterFlags();
Try<PID<Master> > master = StartMaster();
ASSERT_SOME(master);
// Start a checkpointing slave.
slave::Flags slaveFlags = CreateSlaveFlags();
slaveFlags.checkpoint = true;
Future<SlaveRegisteredMessage> slaveRegisteredMessage =
FUTURE_PROTOBUF(SlaveRegisteredMessage(), _, _);
Try<PID<Slave> > slave = StartSlave(slaveFlags);
ASSERT_SOME(slave);
// Wait for the slave to register and get the slave id.
AWAIT_READY(slaveRegisteredMessage);
const SlaveID slaveId = slaveRegisteredMessage.get().slave_id();
// Stop the master and slave.
Stop(master.get());
Stop(slave.get());
MockScheduler sched;
MesosSchedulerDriver driver(
&sched, DEFAULT_FRAMEWORK_INFO, master.get(), DEFAULT_CREDENTIAL);
Future<FrameworkID> frameworkId;
EXPECT_CALL(sched, registered(&driver, _, _))
.WillOnce(FutureArg<1>(&frameworkId));
EXPECT_CALL(sched, resourceOffers(&driver, _))
.WillRepeatedly(Return()); // Ignore offers.
// Framework should not receive any update.
EXPECT_CALL(sched, statusUpdate(&driver, _))
.Times(0);
// Drop '&Master::_reregisterSlave' dispatch so that the slave is
// in 'reregistering' state.
Future<Nothing> _reregisterSlave =
DROP_DISPATCH(_, &Master::_reregisterSlave);
// Restart the master.
master = StartMaster(masterFlags);
ASSERT_SOME(master);
driver.start();
// Wait for the framework to register.
AWAIT_READY(frameworkId);
// Restart the slave.
slave = StartSlave(slaveFlags);
ASSERT_SOME(slave);
// Slave will be in 'reregistering' state here.
AWAIT_READY(_reregisterSlave);
vector<TaskStatus> statuses;
// Create a task status with a random task id.
TaskStatus status;
status.mutable_task_id()->set_value(UUID::random().toString());
status.mutable_slave_id()->CopyFrom(slaveId);
status.set_state(TASK_RUNNING);
statuses.push_back(status);
Future<ReconcileTasksMessage> reconcileTasksMessage =
FUTURE_PROTOBUF(ReconcileTasksMessage(), _ , _);
Clock::pause();
driver.reconcileTasks(statuses);
// Make sure the master received the reconcile tasks message.
AWAIT_READY(reconcileTasksMessage);
// The Clock::settle() will ensure that framework would receive
// a status update if it is sent by the master. In this test it
// shouldn't receive any.
Clock::settle();
driver.stop();
driver.join();
Shutdown();
}
// This test verifies that the master reconciles tasks that are
// missing from a reregistering slave. In this case, we trigger
// a race between the slave re-registration message and the launch
// message. There should be no TASK_LOST / TASK_DROPPED.
// This was motivated by MESOS-1696.
TEST_F(MasterSlaveReconciliationTest, ReconcileRace)
{
Try<Owned<cluster::Master>> master = StartMaster();
ASSERT_SOME(master);
MockExecutor exec(DEFAULT_EXECUTOR_ID);
TestContainerizer containerizer(&exec);
StandaloneMasterDetector detector(master.get()->pid);
Future<SlaveRegisteredMessage> slaveRegisteredMessage =
FUTURE_PROTOBUF(SlaveRegisteredMessage(), master.get()->pid, _);
Try<Owned<cluster::Slave>> slave = StartSlave(&detector, &containerizer);
ASSERT_SOME(slave);
AWAIT_READY(slaveRegisteredMessage);
MockScheduler sched;
MesosSchedulerDriver driver(
&sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL);
EXPECT_CALL(sched, registered(&driver, _, _));
Future<vector<Offer>> offers;
EXPECT_CALL(sched, resourceOffers(&driver, _))
.WillOnce(FutureArg<1>(&offers))
.WillRepeatedly(Return()); // Ignore subsequent offers.
driver.start();
// Since the agent may have retried registration, we want to
// ensure that any duplicate registrations are flushed before
// we appoint the master again. Otherwise, the agent may
// receive a stale registration message.
Clock::pause();
Clock::settle();
Clock::resume();
// Trigger a re-registration of the slave and capture the message
// so that we can spoof a race with a launch task message.
DROP_PROTOBUFS(ReregisterSlaveMessage(), slave.get()->pid, master.get()->pid);
Future<ReregisterSlaveMessage> reregisterSlaveMessage =
DROP_PROTOBUF(
ReregisterSlaveMessage(),
slave.get()->pid,
master.get()->pid);
detector.appoint(master.get()->pid);
AWAIT_READY(reregisterSlaveMessage);
AWAIT_READY(offers);
ASSERT_FALSE(offers->empty());
TaskInfo task;
task.set_name("test task");
task.mutable_task_id()->set_value("1");
task.mutable_slave_id()->MergeFrom(offers.get()[0].slave_id());
task.mutable_resources()->MergeFrom(offers.get()[0].resources());
task.mutable_executor()->MergeFrom(DEFAULT_EXECUTOR_INFO);
ExecutorDriver* executorDriver;
EXPECT_CALL(exec, registered(_, _, _, _))
.WillOnce(SaveArg<0>(&executorDriver));
// Leave the task in TASK_STAGING.
Future<Nothing> launchTask;
EXPECT_CALL(exec, launchTask(_, _))
.WillOnce(FutureSatisfy(&launchTask));
EXPECT_CALL(sched, statusUpdate(&driver, _))
.Times(0);
driver.launchTasks(offers.get()[0].id(), {task});
AWAIT_READY(launchTask);
// Send the stale re-registration message, which does not contain
// the task we just launched. This will trigger a reconciliation
// by the master.
Future<SlaveReregisteredMessage> slaveReregisteredMessage =
FUTURE_PROTOBUF(SlaveReregisteredMessage(), _, _);
// Prevent this from being dropped per the DROP_PROTOBUFS above.
FUTURE_PROTOBUF(
ReregisterSlaveMessage(),
slave.get()->pid,
master.get()->pid);
process::post(
slave.get()->pid,
master.get()->pid,
reregisterSlaveMessage.get());
AWAIT_READY(slaveReregisteredMessage);
// Neither the master nor the slave should send a TASK_LOST
// as part of the reconciliation. We check this by calling
// Clock::settle() to flush all pending events.
Clock::pause();
Clock::settle();
Clock::resume();
// Now send TASK_FINISHED and make sure it's the only message
// received by the scheduler.
Future<TaskStatus> status;
EXPECT_CALL(sched, statusUpdate(&driver, _))
.WillOnce(FutureArg<1>(&status));
TaskStatus taskStatus;
taskStatus.mutable_task_id()->CopyFrom(task.task_id());
taskStatus.set_state(TASK_FINISHED);
executorDriver->sendStatusUpdate(taskStatus);
AWAIT_READY(status);
ASSERT_EQ(TASK_FINISHED, status->state());
EXPECT_CALL(exec, shutdown(_))
.Times(AtMost(1));
driver.stop();
driver.join();
}
virtual void launchTask(ExecutorDriver* driver, const TaskInfo& task) {
localPeerCount++;
TaskStatus status;
status.mutable_task_id()->MergeFrom(task.task_id());
status.set_state(TASK_RUNNING);
driver->sendStatusUpdate(status);
//------------- START TASK OPERATIONS ----------
cout << "Running K3 Program: " << task.name() << endl;
string k3_cmd;
using namespace YAML;
Node hostParams = Load(task.data());
Node peerParams;
Node peers;
// vector<Node> peers;
cout << "WHAT I RECEIVED\n----------------------\n";
cout << Dump(hostParams);
cout << "\n---------------------------------\n";
k3_cmd = "cd $MESOS_SANDBOX && bash -c 'ulimit -c unlimited && ./" + hostParams["binary"].as<string>();
if (hostParams["logging"]) {
k3_cmd += " -l INFO ";
}
if (hostParams["resultVar"]) {
k3_cmd += " --result_path $MESOS_SANDBOX --result_var " + hostParams["resultVar"].as<string>();
}
string datavar, datapath;
string datapolicy = "default";
int peerStart = 0;
int peerEnd = 0;
for (const_iterator param=hostParams.begin(); param!=hostParams.end(); param++) {
string key = param->first.as<string>();
// cout << " PROCESSING: " << key << endl;
if (key == "logging" || key == "binary" ||
key == "server" || key == "server_group") {
continue;
}
if (key == "roles") {
continue;
}
else if (key == "peers") {
peerParams["peers"] = hostParams["peers"];
}
else if (key == "me") {
Node meList = param->second;
YAML::Emitter emit;
emit << YAML::Flow << meList;
for (std::size_t i=0; i<meList.size(); i++) {
peers.push_back(meList[i]);
}
}
else if (key == "data") {
// TODO: Datafiles per group. This is a hack
// that only includes the data files from the first peer group
// and assigns them to any peer
Node dataFilesNode = param->second[0];
for(YAML::const_iterator it=dataFilesNode.begin();it!=dataFilesNode.end();++it) {
DataFile f;
auto d = *it;
f.path = d["path"].as<string>();
f.varName = d["var"].as<string>();
f.policy = d["policy"].as<string>();
dataFiles.push_back(f);
}
}
//else if (key == "datavar") {
// datavar = param->second.as<string>();
//}
//else if (key == "datapath") {
// datapath = "{path: " + param->second.as<string>() + "}";
//}
//else if (key == "datapolicy") {
// datapolicy = param->second.as<string>();
//}
else if (key == "totalPeers") {
totalPeerCount = param->second.as<int>();
}
else if (key == "peerStart") {
peerStart = param->second.as<int>();
}
else if (key == "peerEnd") {
peerEnd = param->second.as<int>();
}
else if (key == "globals") {
// handled per peer
}
else {
// string value = i->second.as<string>();
//peerParams[key] = param->second;
}
}
// DATA ALLOCATION *
// TODO: Convert to multiple input dirs
map<string, vector<string> > peerFiles[peers.size()];
for (auto dataFile : dataFiles) {
cout << "Top of loop" << endl;
vector<string> filePaths;
// 1. GET DIR LIST IN datavar
DIR *datadir = NULL;
datadir = opendir(dataFile.path.c_str());
if (!datadir) {
cout << "Failed to open data dir: " << dataFile.path << endl;
TaskStatus status;
status.mutable_task_id()->MergeFrom(task.task_id());
status.set_state(TASK_FAILED);
driver->sendStatusUpdate(status);
return;
}
else {
cout << "Opened data dir: " << dataFile.path << endl;
}
struct dirent *srcfile = NULL;
while (true) {
srcfile = readdir(datadir);
if (srcfile == NULL) {
break;
}
cout << "FILE " << srcfile->d_name << ": ";
if (srcfile->d_type == DT_REG) {
string filename = srcfile->d_name;
filePaths.push_back(dataFile.path + "/" + filename);
cout << "Added -> " << filename;
}
cout << endl;
}
closedir(datadir);
cout << "read directory" << endl;
int numfiles = filePaths.size();
sort (filePaths.begin(), filePaths.end());
int p_start = 0;
int p_end = numfiles;
int p_total = peers.size();
int myfiles = 0;
if (dataFile.policy == "global") {
for (int i = 0; i < numfiles; i++) {
int peer = i % totalPeerCount;
if (peer >= peerStart && peer <= peerEnd) {
myfiles++;
peerFiles[peer-peerStart][dataFile.varName].push_back(filePaths[i]);
}
}
}
else if (dataFile.policy == "replicate") {
for (int p = 0; p < peers.size(); p++) {
for (int i =0; i < numfiles; i++) {
myfiles++;
peerFiles[p][dataFile.varName].push_back(filePaths[i]);
}
}
}
//if (dataFile.policy == "global") {
// p_start = (numfiles / totalPeerCount) * peerStart;
// p_end = (numfiles / totalPeerCount) * (peerEnd+1);
// p_total = totalPeerCount;
// cout << ("Global files s=" + stringify(p_start) + " e=" + stringify(p_end) + " t=" + stringify(p_total)) << endl;
// for (int filenum = p_start; filenum < p_end; filenum++) {
// int peer = floor((((p_total)*1.0*filenum) / numfiles)) - peerStart;
// cout << " Peer # " << peer << " : [" << filenum << "] " << filePaths[filenum] << endl;
// peerFiles[peer][dataFile.varName].push_back(filePaths[filenum]);
// myfiles++;
// }
//}
else if (dataFile.policy == "pinned") {
for(int filenum = 0; filenum < numfiles; filenum++) {
peerFiles[0][dataFile.varName].push_back(filePaths[filenum]);
}
}
else if (dataFile.policy == "sharded") {
for (int i =0; i < numfiles; i++) {
myfiles++;
int p = i % peers.size();
peerFiles[p][dataFile.varName].push_back(filePaths[i]);
}
}
cout << "my files: " << myfiles << endl;
}
cout << "BUILDING PARAMS FOR PEERS" << endl;
int pph = 0;
if (peerParams["peers"].size() >= 1) {
YAML::Node peer_masters;
YAML::Node masters;
YAML::Node curMaster = YAML::Load(YAML::Dump(peerParams["peers"][0]));
masters.push_back(YAML::Load(YAML::Dump(curMaster)));
std::cout << peerParams["peers"].size() << " peers to map" << endl;
for (std::size_t i=0; i< peerParams["peers"].size(); i++) {
YAML::Node kv;
if (peerParams["peers"][i]["addr"][0].as<string>() != curMaster["addr"][0].as<string>()) {
cout << "Host: " << curMaster["addr"][0].as<string>() << ". Peers: " << pph << endl;
pph = 0;
masters.push_back(YAML::Load(YAML::Dump(peerParams["peers"][i])));
curMaster = YAML::Load(YAML::Dump(peerParams["peers"][i]));
}
pph++;
std::cout << "added one" << endl;
kv["key"] = YAML::Load(YAML::Dump(peerParams["peers"][i]["addr"]));
kv["value"] = YAML::Load(YAML::Dump(curMaster["addr"]));
peer_masters.push_back(kv);
}
cout << "Host: " << curMaster["addr"][0].as<string>() << ". Peers: " << pph << endl;
peerParams["peer_masters"] = YAML::Load(YAML::Dump(peer_masters));
peerParams["masters"] = YAML::Load(YAML::Dump(masters));
std::cout << "Masters: " << YAML::Dump(masters) << endl;
}
std::ostringstream oss;
oss << "PEERS!!! (" << std::endl;
for (std::size_t i=0; i<peers.size(); i++) {
oss << "---" << std::endl;
YAML::Node thispeer = peerParams;
YAML::Node globals = hostParams["globals"][i];
for (const_iterator p=globals.begin(); p!=globals.end(); p++) {
thispeer[p->first.as<string>()] = p->second;
}
YAML::Node me = peers[i];
thispeer["me"] = me;
YAML::Node local_peers;
std::cout << "start: " << peerStart << ". end: " << peerEnd << std::endl;
for (int j=peerStart; j<= peerEnd; j++) {
local_peers.push_back(YAML::Load(YAML::Dump(peerParams["peers"][j])));
}
thispeer["local_peers"] = YAML::Load(YAML::Dump(local_peers));
for (auto it : peerFiles[i]) {
auto datavar = it.first;
if (thispeer[datavar]) {
thispeer.remove(datavar);
}
for (auto &f : it.second) {
Node src;
src["path"] = f;
thispeer[datavar].push_back(src);
}
}
// ADD DATA SOURCE DIR HERE
YAML::Emitter emit;
emit << YAML::Flow << thispeer;
string param = emit.c_str();
std::ofstream peerFile;
string peerFileName = "/mnt/mesos/sandbox/peers" + std::to_string(i) + ".yaml";
peerFile.open(peerFileName, std::ofstream::out);
peerFile << param;
peerFile.close();
oss << param << std::endl;
std::cout << param << std::endl;
k3_cmd += " -p " + peerFileName;
for (auto it : peerFiles[i]) {
auto datavar = it.first;
if (thispeer[datavar]) {
thispeer.remove(datavar);
}
}
}
oss << ") END PEERS!!!" << std::endl;
cout << oss.str() << std::endl;
k3_cmd += "'";
cout << "FINAL COMMAND: " << k3_cmd << endl;
if (thread) {
driver->sendFrameworkMessage("Debug: thread already existed!");
thread->interrupt();
thread->join();
delete thread;
thread = 0;
}
bool isMaster = false;
cout << "Checking master" << endl;
if (Dump(hostParams["me"][0]) == Dump(hostParams["master"])) {
isMaster = true;
cout << "I am master" << endl;
}
else {
cout << "me: " << Dump(hostParams["me"][0]) << endl;
cout << "master: " << Dump(hostParams["master"]) << endl;
}
cout << "Launching K3: " << endl;
thread = new boost::thread(TaskThread(task, k3_cmd, driver, isMaster));
}
//when the task before starting,
//it should check task.data() to determin
//what it will do, whether copy config?
//whether start fileserver?
//
//task.data() here format is :
//<isInitialMonNode>.<TaskType>
void CephExecutor::launchTask(ExecutorDriver* driver, const TaskInfo& task)
{
//set class member localSharedConfDirRoot
string cmd = "echo ~";
string r = runShellCommand(cmd);
localSharedConfigDirRoot = r == " " ? r :"/root";
LOG(INFO) << "localSharedConfigDirRoot is " << localSharedConfigDirRoot;
bool needCopyConfig = true;
bool needStartFileServer = false;
int taskType;
if (task.has_data()){
LOG(INFO) << "Got TaskInfo data: " << task.data();
vector<string> tokens = StringUtil::explode(task.data(),'.');
//split by '.', the first part is isInitialMonNode,
//second part is used for task type
if (tokens[0] == "1"){
needCopyConfig = false;
}
taskType = lexical_cast<int>(tokens[1]);
}
string localMountDir = localSharedConfigDirRoot +
"/" +localConfigDirName;
TaskStatus status;
status.mutable_task_id()->MergeFrom(task.task_id());
//make local shared dir, all type of task need this:
//TODO: check if already exists valid dirctory tree
if (!createLocalSharedConfigDir(localConfigDirName)) {
LOG(INFO) << "created local shared directory failed!";
status.set_state(TASK_FAILED);
driver->sendStatusUpdate(status);
return;
}
LOG(INFO) << "Create directory tree done.";
//mount shared local dir
if (needCopyConfig) {
string abPath = localMountDir + "/"
+ "/etc/ceph/";
if (!copySharedConfigDir(abPath)) {
LOG(INFO) << "Copy shared config file failed!";
status.set_state(TASK_FAILED);
driver->sendStatusUpdate(status);
return;
}
LOG(INFO) << "Copy config files done.";
}
//run docker command for MON and RADOSGW
string cName = getContainerName(task.task_id().value());
//set class member containerName, and myTaskId
//TODO: see if put these in registed is more proper
containerName = cName;
myTaskId = task.task_id();
//TODO: kill existing container in case conflict
runShellCommand("docker rm -f " + containerName);
string dockerCommand;
switch (taskType) {
case static_cast<int>(TaskType::MON):
needStartFileServer = true;
dockerCommand = constructMonCommand(
localMountDir,
cName);
downloadDockerImage("ceph/mon");
break;
case static_cast<int>(TaskType::OSD):
downloadDockerImage("ceph/osd");
//Will get osdId in FrameworkMessage
dockerCommand = "";
status.set_state(TASK_STARTING);
driver->sendStatusUpdate(status);
return;
case static_cast<int>(TaskType::RADOSGW):
downloadDockerImage("ceph/radosgw");
dockerCommand = constructRADOSGWCommand(
localMountDir,
cName);
break;
}
if (needStartFileServer) {
thread fileServerThread(fileServer,
7777,
localSharedConfigDirRoot + "/" + localConfigDirName + "/etc/ceph/");
fileServerThread.detach();
LOG(INFO) << "Mon fileserver started";
}
LOG(INFO) << "Stating container with command: ";
LOG(INFO) << dockerCommand;
//fork a thread to enable docker long running.
//TODO: <thread> here seems not working, figure it out
//to find a better way
myPID = fork();
if (0 == myPID){
//child long running docker thread
//TODO: we use fork here. Need to check why below line will hung the executor
//thread(&CephExecutor::startLongRunning,*this,"docker", dockerCommand).detach();
startLongRunning("docker",dockerCommand);
} else {
//parent thread
//check if started normally
bool started = block_until_started(cName, "30");
if (started) {
LOG(INFO) << "Starting task " << task.task_id().value();
status.set_state(TASK_RUNNING);
} else {
LOG(INFO) << "Failed to start task " << task.task_id().value();
status.set_state(TASK_FAILED);
}
driver->sendStatusUpdate(status);
}
}
// This test ensures we don't break the API when it comes to JSON
// representation of tasks. Also, we want to ensure that tasks are
// modeled the same way when using 'Task' vs. 'TaskInfo'.
TEST(HTTP, ModelTask)
{
TaskID taskId;
taskId.set_value("t");
SlaveID slaveId;
slaveId.set_value("s");
ExecutorID executorId;
executorId.set_value("t");
FrameworkID frameworkId;
frameworkId.set_value("f");
TaskState state = TASK_RUNNING;
vector<TaskStatus> statuses;
TaskStatus status;
status.mutable_task_id()->CopyFrom(taskId);
status.set_state(state);
status.mutable_slave_id()->CopyFrom(slaveId);
status.mutable_executor_id()->CopyFrom(executorId);
status.set_timestamp(0.0);
statuses.push_back(status);
TaskInfo task;
task.set_name("task");
task.mutable_task_id()->CopyFrom(taskId);
task.mutable_slave_id()->CopyFrom(slaveId);
task.mutable_command()->set_value("echo hello");
Task task_ = protobuf::createTask(task, state, frameworkId);
task_.add_statuses()->CopyFrom(statuses[0]);
JSON::Value object = model(task, frameworkId, state, statuses);
JSON::Value object_ = model(task_);
Try<JSON::Value> expected = JSON::parse(
"{"
" \"executor_id\":\"\","
" \"framework_id\":\"f\","
" \"id\":\"t\","
" \"name\":\"task\","
" \"resources\":"
" {"
" \"cpus\":0,"
" \"disk\":0,"
" \"mem\":0"
" },"
" \"slave_id\":\"s\","
" \"state\":\"TASK_RUNNING\","
" \"statuses\":"
" ["
" {"
" \"state\":\"TASK_RUNNING\","
" \"timestamp\":0"
" }"
" ]"
"}");
ASSERT_SOME(expected);
EXPECT_EQ(expected.get(), object);
EXPECT_EQ(expected.get(), object_);
// Ensure both are modeled the same.
EXPECT_EQ(object, object_);
}
void launchTask(ExecutorDriver* driver, const TaskInfo& task)
{
CHECK_EQ(REGISTERED, state);
if (launched) {
TaskStatus status;
status.mutable_task_id()->MergeFrom(task.task_id());
status.set_state(TASK_FAILED);
status.set_message(
"Attempted to run multiple tasks using a \"command\" executor");
driver->sendStatusUpdate(status);
return;
}
// Capture the TaskID.
taskId = task.task_id();
// Determine the command to launch the task.
CommandInfo command;
if (taskCommand.isSome()) {
// Get CommandInfo from a JSON string.
Try<JSON::Object> object = JSON::parse<JSON::Object>(taskCommand.get());
if (object.isError()) {
cerr << "Failed to parse JSON: " << object.error() << endl;
abort();
}
Try<CommandInfo> parse = protobuf::parse<CommandInfo>(object.get());
if (parse.isError()) {
cerr << "Failed to parse protobuf: " << parse.error() << endl;
abort();
}
command = parse.get();
} else if (task.has_command()) {
command = task.command();
} else {
CHECK_SOME(override)
<< "Expecting task '" << task.task_id()
<< "' to have a command!";
}
if (override.isNone()) {
// TODO(jieyu): For now, we just fail the executor if the task's
// CommandInfo is not valid. The framework will receive
// TASK_FAILED for the task, and will most likely find out the
// cause with some debugging. This is a temporary solution. A more
// correct solution is to perform this validation at master side.
if (command.shell()) {
CHECK(command.has_value())
<< "Shell command of task '" << task.task_id()
<< "' is not specified!";
} else {
CHECK(command.has_value())
<< "Executable of task '" << task.task_id()
<< "' is not specified!";
}
}
cout << "Starting task " << task.task_id() << endl;
// TODO(benh): Clean this up with the new 'Fork' abstraction.
// Use pipes to determine which child has successfully changed
// session. This is needed as the setsid call can fail from other
// processes having the same group id.
int pipes[2];
if (pipe(pipes) < 0) {
perror("Failed to create a pipe");
abort();
}
// Set the FD_CLOEXEC flags on these pipes.
Try<Nothing> cloexec = os::cloexec(pipes[0]);
if (cloexec.isError()) {
cerr << "Failed to cloexec(pipe[0]): " << cloexec.error() << endl;
abort();
}
cloexec = os::cloexec(pipes[1]);
if (cloexec.isError()) {
cerr << "Failed to cloexec(pipe[1]): " << cloexec.error() << endl;
abort();
}
Option<string> rootfs;
if (sandboxDirectory.isSome()) {
// If 'sandbox_diretory' is specified, that means the user
// task specifies a root filesystem, and that root filesystem has
// already been prepared at COMMAND_EXECUTOR_ROOTFS_CONTAINER_PATH.
// The command executor is responsible for mounting the sandbox
// into the root filesystem, chrooting into it and changing the
// user before exec-ing the user process.
//
// TODO(gilbert): Consider a better way to detect if a root
// filesystem is specified for the command task.
#ifdef __linux__
Result<string> user = os::user();
if (user.isError()) {
cerr << "Failed to get current user: "******"Current username is not found" << endl;
abort();
} else if (user.get() != "root") {
cerr << "The command executor requires root with rootfs" << endl;
abort();
}
rootfs = path::join(
os::getcwd(), COMMAND_EXECUTOR_ROOTFS_CONTAINER_PATH);
string sandbox = path::join(rootfs.get(), sandboxDirectory.get());
if (!os::exists(sandbox)) {
Try<Nothing> mkdir = os::mkdir(sandbox);
if (mkdir.isError()) {
cerr << "Failed to create sandbox mount point at '"
<< sandbox << "': " << mkdir.error() << endl;
abort();
}
}
// Mount the sandbox into the container rootfs.
// We need to perform a recursive mount because we want all the
// volume mounts in the sandbox to be also mounted in the container
// root filesystem. However, since the container root filesystem
// is also mounted in the sandbox, after the recursive mount we
// also need to unmount the root filesystem in the mounted sandbox.
Try<Nothing> mount = fs::mount(
os::getcwd(),
sandbox,
None(),
MS_BIND | MS_REC,
NULL);
if (mount.isError()) {
cerr << "Unable to mount the work directory into container "
<< "rootfs: " << mount.error() << endl;;
abort();
}
// Umount the root filesystem path in the mounted sandbox after
// the recursive mount.
Try<Nothing> unmountAll = fs::unmountAll(path::join(
sandbox,
COMMAND_EXECUTOR_ROOTFS_CONTAINER_PATH));
if (unmountAll.isError()) {
cerr << "Unable to unmount rootfs under mounted sandbox: "
<< unmountAll.error() << endl;
abort();
}
#else
cerr << "Not expecting root volume with non-linux platform." << endl;
abort();
#endif // __linux__
}
// Prepare the argv before fork as it's not async signal safe.
char **argv = new char*[command.arguments().size() + 1];
for (int i = 0; i < command.arguments().size(); i++) {
argv[i] = (char*) command.arguments(i).c_str();
}
argv[command.arguments().size()] = NULL;
// Prepare the command log message.
string commandString;
if (override.isSome()) {
char** argv = override.get();
// argv is guaranteed to be NULL terminated and we rely on
// that fact to print command to be executed.
for (int i = 0; argv[i] != NULL; i++) {
commandString += string(argv[i]) + " ";
}
} else if (command.shell()) {
void* RunProcess (void* args) {
StartInfo* info = static_cast<StartInfo*>(args);
ExecutorDriver* driver = info->driver;
const TaskInfo& task = info->task;
ExternalInfo external = StartExternalProcess(task);
{
TaskStatus status;
status.mutable_task_id()->CopyFrom(task.task_id());
if (external.failed) {
status.set_state(TASK_FAILED);
driver->sendStatusUpdate(status);
delete info;
return nullptr;
}
cout << "PID " << external.pid << "\n";
status.set_state(TASK_RUNNING);
driver->sendStatusUpdate(status);
}
{
lock_guard<mutex> lock(TaskId2PidLock);
const string& taskId = task.task_id().value();
TaskId2Pid[taskId] = external.pid;
}
TaskStatus status;
status.mutable_task_id()->CopyFrom(task.task_id());
int s;
waitpid(external.pid, &s, WUNTRACED);
cout << "WAIT for pid " << external.pid << " returned\n";
if (WIFEXITED(s)) {
int es = WEXITSTATUS(s);
if (es == 0) {
cout << "EXIT " << external.pid << ", status == 0\n";
status.set_state(TASK_FINISHED);
}
else {
cout << "EXIT " << external.pid << ", status " << es << "\n";
status.set_state(TASK_FAILED);
}
}
else if (WIFSIGNALED(s)) {
cout << "EXIT " << external.pid << " signalled with " << WTERMSIG(s) << "\n";
status.set_state(TASK_FAILED);
}
else if (WIFSTOPPED(s)) {
cout << "EXIT " << external.pid << " stopped\n";
// TODO(fc) deal with stopped, but how?
kill(external.pid, 9);
status.set_state(TASK_FAILED);
}
driver->sendStatusUpdate(status);
return nullptr;
}
