void ExecuteTask (const TaskInfo& task) {
const string& executorId = task.executor().executor_id().value();
const string& data = task.data();
vector<string> args = split(data, '\n');
Resources resources = task.resources();
LOG(INFO)
<< "executor " << executorId
<< " with '" << join(args, " ") << "'"
<< " and " << resources;
Resources persistent = filterIsPersistentVolume(resources);
for (const auto& volume : persistent) {
const string& path = volume.disk().volume().container_path();
LOG(INFO)
<< "using persistent volume " << path;
os::mkdir(path + "/data");
os::mkdir(path + "/logs");
os::mkdir(path + "/apps");
string cmd = "ls -l " + path;
int r = system(cmd.c_str());
if (r) {
LOG(WARNING)
<< "failed with " << r;
}
}
size_t len = args.size();
char** a = static_cast<char**>(malloc(sizeof(char*) * (len + 1)));
for (size_t i = 0; i < len; ++i) {
a[i] = const_cast<char*>(args[i].c_str());
LOG(INFO)
<< i << ". argument: " << a[i];
}
a[len] = nullptr;
execv(args[0].c_str(), a);
LOG(FATAL)
<< "PANIC execv failed";
// upps, exec failed
_exit(1);
}
//when the task before starting,
//it should check task.data() to determin
//what it will do, whether copy config?
//whether start fileserver?
//
//task.data() here format is :
//<isInitialMonNode>.<TaskType>
void CephExecutor::launchTask(ExecutorDriver* driver, const TaskInfo& task)
{
//set class member localSharedConfDirRoot
string cmd = "echo ~";
string r = runShellCommand(cmd);
localSharedConfigDirRoot = r == " " ? r :"/root";
LOG(INFO) << "localSharedConfigDirRoot is " << localSharedConfigDirRoot;
bool needCopyConfig = true;
bool needStartFileServer = false;
int taskType;
if (task.has_data()){
LOG(INFO) << "Got TaskInfo data: " << task.data();
vector<string> tokens = StringUtil::explode(task.data(),'.');
//split by '.', the first part is isInitialMonNode,
//second part is used for task type
if (tokens[0] == "1"){
needCopyConfig = false;
}
taskType = lexical_cast<int>(tokens[1]);
}
string localMountDir = localSharedConfigDirRoot +
"/" +localConfigDirName;
TaskStatus status;
status.mutable_task_id()->MergeFrom(task.task_id());
//make local shared dir, all type of task need this:
//TODO: check if already exists valid dirctory tree
if (!createLocalSharedConfigDir(localConfigDirName)) {
LOG(INFO) << "created local shared directory failed!";
status.set_state(TASK_FAILED);
driver->sendStatusUpdate(status);
return;
}
LOG(INFO) << "Create directory tree done.";
//mount shared local dir
if (needCopyConfig) {
string abPath = localMountDir + "/"
+ "/etc/ceph/";
if (!copySharedConfigDir(abPath)) {
LOG(INFO) << "Copy shared config file failed!";
status.set_state(TASK_FAILED);
driver->sendStatusUpdate(status);
return;
}
LOG(INFO) << "Copy config files done.";
}
//run docker command for MON and RADOSGW
string cName = getContainerName(task.task_id().value());
//set class member containerName, and myTaskId
//TODO: see if put these in registed is more proper
containerName = cName;
myTaskId = task.task_id();
//TODO: kill existing container in case conflict
runShellCommand("docker rm -f " + containerName);
string dockerCommand;
switch (taskType) {
case static_cast<int>(TaskType::MON):
needStartFileServer = true;
dockerCommand = constructMonCommand(
localMountDir,
cName);
downloadDockerImage("ceph/mon");
break;
case static_cast<int>(TaskType::OSD):
downloadDockerImage("ceph/osd");
//Will get osdId in FrameworkMessage
dockerCommand = "";
status.set_state(TASK_STARTING);
driver->sendStatusUpdate(status);
return;
case static_cast<int>(TaskType::RADOSGW):
downloadDockerImage("ceph/radosgw");
dockerCommand = constructRADOSGWCommand(
localMountDir,
cName);
break;
}
if (needStartFileServer) {
thread fileServerThread(fileServer,
7777,
localSharedConfigDirRoot + "/" + localConfigDirName + "/etc/ceph/");
fileServerThread.detach();
LOG(INFO) << "Mon fileserver started";
}
LOG(INFO) << "Stating container with command: ";
LOG(INFO) << dockerCommand;
//fork a thread to enable docker long running.
//TODO: <thread> here seems not working, figure it out
//to find a better way
myPID = fork();
if (0 == myPID){
//child long running docker thread
//TODO: we use fork here. Need to check why below line will hung the executor
//thread(&CephExecutor::startLongRunning,*this,"docker", dockerCommand).detach();
startLongRunning("docker",dockerCommand);
} else {
//parent thread
//check if started normally
bool started = block_until_started(cName, "30");
if (started) {
LOG(INFO) << "Starting task " << task.task_id().value();
status.set_state(TASK_RUNNING);
} else {
LOG(INFO) << "Failed to start task " << task.task_id().value();
status.set_state(TASK_FAILED);
}
driver->sendStatusUpdate(status);
}
}
virtual void launchTask(ExecutorDriver* driver, const TaskInfo& task) {
localPeerCount++;
TaskStatus status;
status.mutable_task_id()->MergeFrom(task.task_id());
status.set_state(TASK_RUNNING);
driver->sendStatusUpdate(status);
//------------- START TASK OPERATIONS ----------
cout << "Running K3 Program: " << task.name() << endl;
string k3_cmd;
using namespace YAML;
Node hostParams = Load(task.data());
Node peerParams;
Node peers;
// vector<Node> peers;
cout << "WHAT I RECEIVED\n----------------------\n";
cout << Dump(hostParams);
cout << "\n---------------------------------\n";
k3_cmd = "cd $MESOS_SANDBOX && bash -c 'ulimit -c unlimited && ./" + hostParams["binary"].as<string>();
if (hostParams["logging"]) {
k3_cmd += " -l INFO ";
}
if (hostParams["resultVar"]) {
k3_cmd += " --result_path $MESOS_SANDBOX --result_var " + hostParams["resultVar"].as<string>();
}
string datavar, datapath;
string datapolicy = "default";
int peerStart = 0;
int peerEnd = 0;
for (const_iterator param=hostParams.begin(); param!=hostParams.end(); param++) {
string key = param->first.as<string>();
// cout << " PROCESSING: " << key << endl;
if (key == "logging" || key == "binary" ||
key == "server" || key == "server_group") {
continue;
}
if (key == "roles") {
continue;
}
else if (key == "peers") {
peerParams["peers"] = hostParams["peers"];
}
else if (key == "me") {
Node meList = param->second;
YAML::Emitter emit;
emit << YAML::Flow << meList;
for (std::size_t i=0; i<meList.size(); i++) {
peers.push_back(meList[i]);
}
}
else if (key == "data") {
// TODO: Datafiles per group. This is a hack
// that only includes the data files from the first peer group
// and assigns them to any peer
Node dataFilesNode = param->second[0];
for(YAML::const_iterator it=dataFilesNode.begin();it!=dataFilesNode.end();++it) {
DataFile f;
auto d = *it;
f.path = d["path"].as<string>();
f.varName = d["var"].as<string>();
f.policy = d["policy"].as<string>();
dataFiles.push_back(f);
}
}
//else if (key == "datavar") {
// datavar = param->second.as<string>();
//}
//else if (key == "datapath") {
// datapath = "{path: " + param->second.as<string>() + "}";
//}
//else if (key == "datapolicy") {
// datapolicy = param->second.as<string>();
//}
else if (key == "totalPeers") {
totalPeerCount = param->second.as<int>();
}
else if (key == "peerStart") {
peerStart = param->second.as<int>();
}
else if (key == "peerEnd") {
peerEnd = param->second.as<int>();
}
else if (key == "globals") {
// handled per peer
}
else {
// string value = i->second.as<string>();
//peerParams[key] = param->second;
}
}
// DATA ALLOCATION *
// TODO: Convert to multiple input dirs
map<string, vector<string> > peerFiles[peers.size()];
for (auto dataFile : dataFiles) {
cout << "Top of loop" << endl;
vector<string> filePaths;
// 1. GET DIR LIST IN datavar
DIR *datadir = NULL;
datadir = opendir(dataFile.path.c_str());
if (!datadir) {
cout << "Failed to open data dir: " << dataFile.path << endl;
TaskStatus status;
status.mutable_task_id()->MergeFrom(task.task_id());
status.set_state(TASK_FAILED);
driver->sendStatusUpdate(status);
return;
}
else {
cout << "Opened data dir: " << dataFile.path << endl;
}
struct dirent *srcfile = NULL;
while (true) {
srcfile = readdir(datadir);
if (srcfile == NULL) {
break;
}
cout << "FILE " << srcfile->d_name << ": ";
if (srcfile->d_type == DT_REG) {
string filename = srcfile->d_name;
filePaths.push_back(dataFile.path + "/" + filename);
cout << "Added -> " << filename;
}
cout << endl;
}
closedir(datadir);
cout << "read directory" << endl;
int numfiles = filePaths.size();
sort (filePaths.begin(), filePaths.end());
int p_start = 0;
int p_end = numfiles;
int p_total = peers.size();
int myfiles = 0;
if (dataFile.policy == "global") {
for (int i = 0; i < numfiles; i++) {
int peer = i % totalPeerCount;
if (peer >= peerStart && peer <= peerEnd) {
myfiles++;
peerFiles[peer-peerStart][dataFile.varName].push_back(filePaths[i]);
}
}
}
else if (dataFile.policy == "replicate") {
for (int p = 0; p < peers.size(); p++) {
for (int i =0; i < numfiles; i++) {
myfiles++;
peerFiles[p][dataFile.varName].push_back(filePaths[i]);
}
}
}
//if (dataFile.policy == "global") {
// p_start = (numfiles / totalPeerCount) * peerStart;
// p_end = (numfiles / totalPeerCount) * (peerEnd+1);
// p_total = totalPeerCount;
// cout << ("Global files s=" + stringify(p_start) + " e=" + stringify(p_end) + " t=" + stringify(p_total)) << endl;
// for (int filenum = p_start; filenum < p_end; filenum++) {
// int peer = floor((((p_total)*1.0*filenum) / numfiles)) - peerStart;
// cout << " Peer # " << peer << " : [" << filenum << "] " << filePaths[filenum] << endl;
// peerFiles[peer][dataFile.varName].push_back(filePaths[filenum]);
// myfiles++;
// }
//}
else if (dataFile.policy == "pinned") {
for(int filenum = 0; filenum < numfiles; filenum++) {
peerFiles[0][dataFile.varName].push_back(filePaths[filenum]);
}
}
else if (dataFile.policy == "sharded") {
for (int i =0; i < numfiles; i++) {
myfiles++;
int p = i % peers.size();
peerFiles[p][dataFile.varName].push_back(filePaths[i]);
}
}
cout << "my files: " << myfiles << endl;
}
cout << "BUILDING PARAMS FOR PEERS" << endl;
int pph = 0;
if (peerParams["peers"].size() >= 1) {
YAML::Node peer_masters;
YAML::Node masters;
YAML::Node curMaster = YAML::Load(YAML::Dump(peerParams["peers"][0]));
masters.push_back(YAML::Load(YAML::Dump(curMaster)));
std::cout << peerParams["peers"].size() << " peers to map" << endl;
for (std::size_t i=0; i< peerParams["peers"].size(); i++) {
YAML::Node kv;
if (peerParams["peers"][i]["addr"][0].as<string>() != curMaster["addr"][0].as<string>()) {
cout << "Host: " << curMaster["addr"][0].as<string>() << ". Peers: " << pph << endl;
pph = 0;
masters.push_back(YAML::Load(YAML::Dump(peerParams["peers"][i])));
curMaster = YAML::Load(YAML::Dump(peerParams["peers"][i]));
}
pph++;
std::cout << "added one" << endl;
kv["key"] = YAML::Load(YAML::Dump(peerParams["peers"][i]["addr"]));
kv["value"] = YAML::Load(YAML::Dump(curMaster["addr"]));
peer_masters.push_back(kv);
}
cout << "Host: " << curMaster["addr"][0].as<string>() << ". Peers: " << pph << endl;
peerParams["peer_masters"] = YAML::Load(YAML::Dump(peer_masters));
peerParams["masters"] = YAML::Load(YAML::Dump(masters));
std::cout << "Masters: " << YAML::Dump(masters) << endl;
}
std::ostringstream oss;
oss << "PEERS!!! (" << std::endl;
for (std::size_t i=0; i<peers.size(); i++) {
oss << "---" << std::endl;
YAML::Node thispeer = peerParams;
YAML::Node globals = hostParams["globals"][i];
for (const_iterator p=globals.begin(); p!=globals.end(); p++) {
thispeer[p->first.as<string>()] = p->second;
}
YAML::Node me = peers[i];
thispeer["me"] = me;
YAML::Node local_peers;
std::cout << "start: " << peerStart << ". end: " << peerEnd << std::endl;
for (int j=peerStart; j<= peerEnd; j++) {
local_peers.push_back(YAML::Load(YAML::Dump(peerParams["peers"][j])));
}
thispeer["local_peers"] = YAML::Load(YAML::Dump(local_peers));
for (auto it : peerFiles[i]) {
auto datavar = it.first;
if (thispeer[datavar]) {
thispeer.remove(datavar);
}
for (auto &f : it.second) {
Node src;
src["path"] = f;
thispeer[datavar].push_back(src);
}
}
// ADD DATA SOURCE DIR HERE
YAML::Emitter emit;
emit << YAML::Flow << thispeer;
string param = emit.c_str();
std::ofstream peerFile;
string peerFileName = "/mnt/mesos/sandbox/peers" + std::to_string(i) + ".yaml";
peerFile.open(peerFileName, std::ofstream::out);
peerFile << param;
peerFile.close();
oss << param << std::endl;
std::cout << param << std::endl;
k3_cmd += " -p " + peerFileName;
for (auto it : peerFiles[i]) {
auto datavar = it.first;
if (thispeer[datavar]) {
thispeer.remove(datavar);
}
}
}
oss << ") END PEERS!!!" << std::endl;
cout << oss.str() << std::endl;
k3_cmd += "'";
cout << "FINAL COMMAND: " << k3_cmd << endl;
if (thread) {
driver->sendFrameworkMessage("Debug: thread already existed!");
thread->interrupt();
thread->join();
delete thread;
thread = 0;
}
bool isMaster = false;
cout << "Checking master" << endl;
if (Dump(hostParams["me"][0]) == Dump(hostParams["master"])) {
isMaster = true;
cout << "I am master" << endl;
}
else {
cout << "me: " << Dump(hostParams["me"][0]) << endl;
cout << "master: " << Dump(hostParams["master"]) << endl;
}
cout << "Launching K3: " << endl;
thread = new boost::thread(TaskThread(task, k3_cmd, driver, isMaster));
}