// TODO(bmahler): Expose the executor name / source.
JSON::Object model(
const TaskInfo& task,
const FrameworkID& frameworkId,
const TaskState& state,
const vector<TaskStatus>& statuses)
{
JSON::Object object;
object.values["id"] = task.task_id().value();
object.values["name"] = task.name();
object.values["framework_id"] = frameworkId.value();
if (task.has_executor()) {
object.values["executor_id"] = task.executor().executor_id().value();
} else {
object.values["executor_id"] = "";
}
object.values["slave_id"] = task.slave_id().value();
object.values["state"] = TaskState_Name(state);
object.values["resources"] = model(task.resources());
JSON::Array array;
foreach (const TaskStatus& status, statuses) {
array.values.push_back(model(status));
}
Task createTask(
const TaskInfo& task,
const TaskState& state,
const FrameworkID& frameworkId)
{
Task t;
t.mutable_framework_id()->MergeFrom(frameworkId);
t.set_state(state);
t.set_name(task.name());
t.mutable_task_id()->MergeFrom(task.task_id());
t.mutable_slave_id()->MergeFrom(task.slave_id());
t.mutable_resources()->MergeFrom(task.resources());
if (task.has_executor()) {
t.mutable_executor_id()->CopyFrom(task.executor().executor_id());
}
t.mutable_labels()->MergeFrom(task.labels());
if (task.has_discovery()) {
t.mutable_discovery()->MergeFrom(task.discovery());
}
return t;
}
inline Task createTask(const TaskInfo& task,
const TaskState& state,
const ExecutorID& executorId,
const FrameworkID& frameworkId)
{
Task t;
t.mutable_framework_id()->MergeFrom(frameworkId);
t.set_state(state);
t.set_name(task.name());
t.mutable_task_id()->MergeFrom(task.task_id());
t.mutable_slave_id()->MergeFrom(task.slave_id());
t.mutable_resources()->MergeFrom(task.resources());
if (!task.has_command()) {
t.mutable_executor_id()->MergeFrom(executorId);
}
return t;
}
Task createTask(
const TaskInfo& task,
const TaskState& state,
const FrameworkID& frameworkId)
{
Task t;
t.mutable_framework_id()->CopyFrom(frameworkId);
t.set_state(state);
t.set_name(task.name());
t.mutable_task_id()->CopyFrom(task.task_id());
t.mutable_slave_id()->CopyFrom(task.slave_id());
t.mutable_resources()->CopyFrom(task.resources());
if (task.has_executor()) {
t.mutable_executor_id()->CopyFrom(task.executor().executor_id());
}
if (task.has_labels()) {
t.mutable_labels()->CopyFrom(task.labels());
}
if (task.has_discovery()) {
t.mutable_discovery()->CopyFrom(task.discovery());
}
if (task.has_container()) {
t.mutable_container()->CopyFrom(task.container());
}
// Copy `user` if set.
if (task.has_command() && task.command().has_user()) {
t.set_user(task.command().user());
} else if (task.has_executor() && task.executor().command().has_user()) {
t.set_user(task.executor().command().user());
}
return t;
}
virtual void launchTask(ExecutorDriver* driver, const TaskInfo& task) {
localPeerCount++;
TaskStatus status;
status.mutable_task_id()->MergeFrom(task.task_id());
status.set_state(TASK_RUNNING);
driver->sendStatusUpdate(status);
//------------- START TASK OPERATIONS ----------
cout << "Running K3 Program: " << task.name() << endl;
string k3_cmd;
using namespace YAML;
Node hostParams = Load(task.data());
Node peerParams;
Node peers;
// vector<Node> peers;
cout << "WHAT I RECEIVED\n----------------------\n";
cout << Dump(hostParams);
cout << "\n---------------------------------\n";
k3_cmd = "cd $MESOS_SANDBOX && bash -c 'ulimit -c unlimited && ./" + hostParams["binary"].as<string>();
if (hostParams["logging"]) {
k3_cmd += " -l INFO ";
}
if (hostParams["resultVar"]) {
k3_cmd += " --result_path $MESOS_SANDBOX --result_var " + hostParams["resultVar"].as<string>();
}
string datavar, datapath;
string datapolicy = "default";
int peerStart = 0;
int peerEnd = 0;
for (const_iterator param=hostParams.begin(); param!=hostParams.end(); param++) {
string key = param->first.as<string>();
// cout << " PROCESSING: " << key << endl;
if (key == "logging" || key == "binary" ||
key == "server" || key == "server_group") {
continue;
}
if (key == "roles") {
continue;
}
else if (key == "peers") {
peerParams["peers"] = hostParams["peers"];
}
else if (key == "me") {
Node meList = param->second;
YAML::Emitter emit;
emit << YAML::Flow << meList;
for (std::size_t i=0; i<meList.size(); i++) {
peers.push_back(meList[i]);
}
}
else if (key == "data") {
// TODO: Datafiles per group. This is a hack
// that only includes the data files from the first peer group
// and assigns them to any peer
Node dataFilesNode = param->second[0];
for(YAML::const_iterator it=dataFilesNode.begin();it!=dataFilesNode.end();++it) {
DataFile f;
auto d = *it;
f.path = d["path"].as<string>();
f.varName = d["var"].as<string>();
f.policy = d["policy"].as<string>();
dataFiles.push_back(f);
}
}
//else if (key == "datavar") {
// datavar = param->second.as<string>();
//}
//else if (key == "datapath") {
// datapath = "{path: " + param->second.as<string>() + "}";
//}
//else if (key == "datapolicy") {
// datapolicy = param->second.as<string>();
//}
else if (key == "totalPeers") {
totalPeerCount = param->second.as<int>();
}
else if (key == "peerStart") {
peerStart = param->second.as<int>();
}
else if (key == "peerEnd") {
peerEnd = param->second.as<int>();
}
else if (key == "globals") {
// handled per peer
}
else {
// string value = i->second.as<string>();
//peerParams[key] = param->second;
}
}
// DATA ALLOCATION *
// TODO: Convert to multiple input dirs
map<string, vector<string> > peerFiles[peers.size()];
for (auto dataFile : dataFiles) {
cout << "Top of loop" << endl;
vector<string> filePaths;
// 1. GET DIR LIST IN datavar
DIR *datadir = NULL;
datadir = opendir(dataFile.path.c_str());
if (!datadir) {
cout << "Failed to open data dir: " << dataFile.path << endl;
TaskStatus status;
status.mutable_task_id()->MergeFrom(task.task_id());
status.set_state(TASK_FAILED);
driver->sendStatusUpdate(status);
return;
}
else {
cout << "Opened data dir: " << dataFile.path << endl;
}
struct dirent *srcfile = NULL;
while (true) {
srcfile = readdir(datadir);
if (srcfile == NULL) {
break;
}
cout << "FILE " << srcfile->d_name << ": ";
if (srcfile->d_type == DT_REG) {
string filename = srcfile->d_name;
filePaths.push_back(dataFile.path + "/" + filename);
cout << "Added -> " << filename;
}
cout << endl;
}
closedir(datadir);
cout << "read directory" << endl;
int numfiles = filePaths.size();
sort (filePaths.begin(), filePaths.end());
int p_start = 0;
int p_end = numfiles;
int p_total = peers.size();
int myfiles = 0;
if (dataFile.policy == "global") {
for (int i = 0; i < numfiles; i++) {
int peer = i % totalPeerCount;
if (peer >= peerStart && peer <= peerEnd) {
myfiles++;
peerFiles[peer-peerStart][dataFile.varName].push_back(filePaths[i]);
}
}
}
else if (dataFile.policy == "replicate") {
for (int p = 0; p < peers.size(); p++) {
for (int i =0; i < numfiles; i++) {
myfiles++;
peerFiles[p][dataFile.varName].push_back(filePaths[i]);
}
}
}
//if (dataFile.policy == "global") {
// p_start = (numfiles / totalPeerCount) * peerStart;
// p_end = (numfiles / totalPeerCount) * (peerEnd+1);
// p_total = totalPeerCount;
// cout << ("Global files s=" + stringify(p_start) + " e=" + stringify(p_end) + " t=" + stringify(p_total)) << endl;
// for (int filenum = p_start; filenum < p_end; filenum++) {
// int peer = floor((((p_total)*1.0*filenum) / numfiles)) - peerStart;
// cout << " Peer # " << peer << " : [" << filenum << "] " << filePaths[filenum] << endl;
// peerFiles[peer][dataFile.varName].push_back(filePaths[filenum]);
// myfiles++;
// }
//}
else if (dataFile.policy == "pinned") {
for(int filenum = 0; filenum < numfiles; filenum++) {
peerFiles[0][dataFile.varName].push_back(filePaths[filenum]);
}
}
else if (dataFile.policy == "sharded") {
for (int i =0; i < numfiles; i++) {
myfiles++;
int p = i % peers.size();
peerFiles[p][dataFile.varName].push_back(filePaths[i]);
}
}
cout << "my files: " << myfiles << endl;
}
cout << "BUILDING PARAMS FOR PEERS" << endl;
int pph = 0;
if (peerParams["peers"].size() >= 1) {
YAML::Node peer_masters;
YAML::Node masters;
YAML::Node curMaster = YAML::Load(YAML::Dump(peerParams["peers"][0]));
masters.push_back(YAML::Load(YAML::Dump(curMaster)));
std::cout << peerParams["peers"].size() << " peers to map" << endl;
for (std::size_t i=0; i< peerParams["peers"].size(); i++) {
YAML::Node kv;
if (peerParams["peers"][i]["addr"][0].as<string>() != curMaster["addr"][0].as<string>()) {
cout << "Host: " << curMaster["addr"][0].as<string>() << ". Peers: " << pph << endl;
pph = 0;
masters.push_back(YAML::Load(YAML::Dump(peerParams["peers"][i])));
curMaster = YAML::Load(YAML::Dump(peerParams["peers"][i]));
}
pph++;
std::cout << "added one" << endl;
kv["key"] = YAML::Load(YAML::Dump(peerParams["peers"][i]["addr"]));
kv["value"] = YAML::Load(YAML::Dump(curMaster["addr"]));
peer_masters.push_back(kv);
}
cout << "Host: " << curMaster["addr"][0].as<string>() << ". Peers: " << pph << endl;
peerParams["peer_masters"] = YAML::Load(YAML::Dump(peer_masters));
peerParams["masters"] = YAML::Load(YAML::Dump(masters));
std::cout << "Masters: " << YAML::Dump(masters) << endl;
}
std::ostringstream oss;
oss << "PEERS!!! (" << std::endl;
for (std::size_t i=0; i<peers.size(); i++) {
oss << "---" << std::endl;
YAML::Node thispeer = peerParams;
YAML::Node globals = hostParams["globals"][i];
for (const_iterator p=globals.begin(); p!=globals.end(); p++) {
thispeer[p->first.as<string>()] = p->second;
}
YAML::Node me = peers[i];
thispeer["me"] = me;
YAML::Node local_peers;
std::cout << "start: " << peerStart << ". end: " << peerEnd << std::endl;
for (int j=peerStart; j<= peerEnd; j++) {
local_peers.push_back(YAML::Load(YAML::Dump(peerParams["peers"][j])));
}
thispeer["local_peers"] = YAML::Load(YAML::Dump(local_peers));
for (auto it : peerFiles[i]) {
auto datavar = it.first;
if (thispeer[datavar]) {
thispeer.remove(datavar);
}
for (auto &f : it.second) {
Node src;
src["path"] = f;
thispeer[datavar].push_back(src);
}
}
// ADD DATA SOURCE DIR HERE
YAML::Emitter emit;
emit << YAML::Flow << thispeer;
string param = emit.c_str();
std::ofstream peerFile;
string peerFileName = "/mnt/mesos/sandbox/peers" + std::to_string(i) + ".yaml";
peerFile.open(peerFileName, std::ofstream::out);
peerFile << param;
peerFile.close();
oss << param << std::endl;
std::cout << param << std::endl;
k3_cmd += " -p " + peerFileName;
for (auto it : peerFiles[i]) {
auto datavar = it.first;
if (thispeer[datavar]) {
thispeer.remove(datavar);
}
}
}
oss << ") END PEERS!!!" << std::endl;
cout << oss.str() << std::endl;
k3_cmd += "'";
cout << "FINAL COMMAND: " << k3_cmd << endl;
if (thread) {
driver->sendFrameworkMessage("Debug: thread already existed!");
thread->interrupt();
thread->join();
delete thread;
thread = 0;
}
bool isMaster = false;
cout << "Checking master" << endl;
if (Dump(hostParams["me"][0]) == Dump(hostParams["master"])) {
isMaster = true;
cout << "I am master" << endl;
}
else {
cout << "me: " << Dump(hostParams["me"][0]) << endl;
cout << "master: " << Dump(hostParams["master"]) << endl;
}
cout << "Launching K3: " << endl;
thread = new boost::thread(TaskThread(task, k3_cmd, driver, isMaster));
}