void submittaskszht(ZHTClient &clientRet, string client_id)
{
int ret = 0;
Package package;
package.set_virtualpath(client_id);
package.set_operation(21);
cout << "Ok, what is the hell!\n" << endl;
for (map<uint32_t, NodeList>::iterator map_it = update_map_zht.begin(); map_it != update_map_zht.end(); ++map_it)
{
uint32_t index = map_it->first;
NodeList &update_list = map_it->second;
while (!update_list.empty())
{
int num_vector_count, per_vector_count;
string alltasks;
vector< vector<string> > tokenize_string = tokenize_1(update_list.front(), '$', '#', num_vector_count, per_vector_count);
//cout << " num_vector_count = " << num_vector_count << " per_vector_count = " << per_vector_count << endl;
for(int i = 0; i < per_vector_count; i++)
{
Package tmpPackage;
tmpPackage.ParseFromString(tokenize_string.at(0).at(i));
string key = tmpPackage.virtualpath();
alltasks.append(key); alltasks.append("\'\"");
} //cout << "Serve
package.set_realfullpath(alltasks);
string str = package.SerializeAsString();
ret += clientRet.svrtosvr(str, str.length(), index);
update_list.pop_front();
}
}
// Package package; string alltasks;
// package.set_virtualpath(client_id); // Here key is just the client ID
// package.set_operation(21);
// //package.set_operation(22);
// num_packages++;
//
// int num_tasks_this_package = max_tasks_per_package;
// int num_tasks_left = num_tasks - total_submitted1;
// if (num_tasks_left < max_tasks_per_package) {
// num_tasks_this_package = num_tasks_left;
// }
// for(int j = 0; j < num_tasks_this_package; j++) {
// int task_id = it->first; ++it;
//
// stringstream task_id_ss;
// task_id_ss << task_id << client_id; // Task ID + Client ID
// alltasks.append(task_id_ss.str()); alltasks.append("\'\""); // Task ID
// }
// total_submitted1 = total_submitted1 + num_tasks_this_package;
// package.set_realfullpath(alltasks);
// string str = package.SerializeAsString();
// //cout << "String size = " << str.size() << " str length = " << strlen(str.c_str());
// int32_t ret = clientRet.svrtosvr(str, str.length(), toserver);
}
void submittasks(ZHTClient &clientRet) {
/*uint32_t count = task_str_list.size();
for(uint32_t i = 0; i < count; i++) {
int32_t ret = clientRet.insert(task_str_list[i]); //cout << "task " << i << " sent" << endl;
}*/
int ret = 0;
Package package;
package.set_operation(20);
package.set_virtualpath("zht_insert");
for(map<uint32_t, NodeList>::iterator map_it = update_map.begin(); map_it != update_map.end(); ++map_it) {
uint32_t index = map_it->first;
NodeList &update_list = map_it->second;
while(!update_list.empty()) { //cout << "str = " << update_list.front() << endl;
package.set_realfullpath(update_list.front());
update_list.pop_front();
string update_str = package.SerializeAsString();
ret += clientRet.svrtosvr(update_str, update_str.size(), index);
}
}
cout << " no. tasks inserted = " << ret << endl;
}
//monitor the submitted tasks
// by polling each server for its load information. If a server is idle then it will return load = -4 (queue length - num of idle cores)
// so if every server returns load = -4 it implies that all submitted tasks are complete
void *monitor_function(void* args) {
ZHTClient *clientRet = (ZHTClient*)args;
Package loadPackage, shutdownPackage;
string loadmessage("Monitoring Information!");
loadPackage.set_virtualpath(loadmessage);
loadPackage.set_operation(15);
string loadstr = loadPackage.SerializeAsString();
string endmessage("Shutdown!");
loadPackage.set_virtualpath(endmessage);
loadPackage.set_operation(98);
string endstr = loadPackage.SerializeAsString();
//int num_worker = clientRet.memberList.size();
int num_worker = clientRet->memberList.size();
int num_cores = ncores;
int index = 0;
long termination_value = num_worker * num_cores * -1;
int total_avail_cores = num_cores * num_worker;
int32_t total_queued = 0;
int32_t total_idle = 0;
int32_t queued_busy = 0;
int32_t queued_idle = 0;
int32_t queued = 0;
int32_t num_idle = 0;
int32_t num_busy = 0;
int32_t load = 0;
int32_t total_busy = 0;
//int32_t status = 0;
int32_t finished = 0;
int32_t total_msg_count = 0;
int32_t ret = 0;
//sleep(60);
int min_lines = num_worker;
int num = num_worker - 1;
stringstream num_ss;
num_ss << num;
long num_monitor = 0;
//min_lines++;
// not sure why we need this
string filename(shared);
filename = filename + "startinfo" + num_ss.str();
string cmd("cat ");
cmd = cmd + filename + " | wc -l";
string result = executeShell(cmd);
cout << cmd << " " << result << endl;
//cout << "client: minlines = " << min_lines << " cmd = " << cmd << " result = " << result << endl;
/*string filename(shared);
filename = filename + "start_info";
string cmd("wc -l ");
cmd = cmd + filename + " | awk {\'print $1\'}";
string result = executeShell(cmd);*/
while(atoi(result.c_str()) < 1) {
usleep(minterval);
result = executeShell(cmd); cout << " temp result = " << result << endl;
}
cout << "client: minlines = 1 " << " cmd = " << cmd << " result = " << result << endl;
//cout << "starting to monitor" << endl;
cout << "TIME START: " << start_tasks.tv_sec << " SECONDS " << start_tasks.tv_nsec << " NANOSECONDS" << endl;
timespec local_start, local_diff;
clock_gettime(CLOCK_REALTIME, &local_start);
local_diff = timediff(start_tasks, local_start);
if (client_logfile.is_open() && cl_LOGGING) {
client_logfile << "Submission time: " << start_tasks.tv_sec << " SECONDS " << start_tasks.tv_nsec << " NANOSECONDS" << endl;
client_logfile << "Monitoring time: " << local_start.tv_sec << " SECONDS " << local_start.tv_nsec << " NANOSECONDS" << endl;
client_logfile << "TIME TAKEN: " << local_diff.tv_sec << " SECONDS " << local_diff.tv_nsec << " NANOSECONDS" << endl;
}
int total_fin = 0;
while(1) {
// total_queued = 0;
// total_idle = 0;
// queued_busy = 0;
total_fin = 0;
stringstream worker_load;
for(index = 0; index < num_worker; index++) {
//int32_t queued_idle = clientRet.svrtosvr(loadstr, loadstr.length(), index);
queued_idle = clientRet->svrtosvr(loadstr, loadstr.length(), index);
// queued = queued_idle/10; // summation of the lengths of the three queues
// num_idle = queued_idle%10; // number of idle cores
// total_queued = total_queued + queued;
// total_idle = total_idle + num_idle;
// num_busy = num_cores - num_idle;
// load = queued + num_busy;
total_fin += queued_idle;
// worker_load << load << " ";
worker_load << queued_idle << " ";
}
// loadfile << worker_load.str() << endl;
// total_busy = total_avail_cores - total_idle;
// queued_busy = total_queued + total_busy;
// finished = total_num_tasks - queued_busy;
num_monitor++;
clock_gettime(CLOCK_REALTIME, &end_tasks);
//cout << "Total busy cores " << total_busy << " Total Load on all workers = " << queued_busy << " No. of tasks finished = " << finished << " Total tasks submitted = " << total_num_tasks << endl;//" time = " << end_tasks.tv_sec << " " << end_tasks.tv_nsec << endl;
if (client_logfile.is_open() && cl_LOGGING) {
// client_logfile << "Total busy cores " << total_busy << " Total Load on all workers = " << queued_busy << " No. of tasks finished = " << finished << " Total tasks submitted = " << total_num_tasks << endl;
//client_logfile << "No. of tasks finished is:" << total_fin <<", No. of tasks submitted is:" << total_num_tasks << endl;
}
// if(finished == total_num_tasks)
if (total_fin == total_num_tasks)
{
clock_gettime(CLOCK_REALTIME, &end_tasks);
cout << "\n\n\n\n==============================All tasks finished===========================\n\n\n\n";
break;
}
usleep(minterval);
}
total_msg_count = 0;
for(index = 0; index < num_worker; index++) {
//clientRet.svrtosvr(endstr, endstr.length(), index);
ret = clientRet->svrtosvr(endstr, endstr.length(), index);
total_msg_count += ret;
}
cout << "TIME START: " << start_tasks.tv_sec << " SECONDS " << start_tasks.tv_nsec << " NANOSECONDS" << "\n";
cout << "TIME END: " << end_tasks.tv_sec << " SECONDS " << end_tasks.tv_nsec << " NANOSECONDS" << "\n";
timespec diff = timediff(start_tasks, end_tasks);
cout << "TIME TAKEN: " << diff.tv_sec << " SECONDS " << diff.tv_nsec << " NANOSECONDS" << "\n";
cout << "Total messages between all servers = " << total_msg_count << endl;
if (client_logfile.is_open() && cl_LOGGING) {
client_logfile << "\n\n\n\n==============================All tasks finished===========================\n\n\n\n";
client_logfile << "TIME START: " << start_tasks.tv_sec << " SECONDS " << start_tasks.tv_nsec << " NANOSECONDS" << "\n";
client_logfile << "TIME END: " << end_tasks.tv_sec << " SECONDS " << end_tasks.tv_nsec << " NANOSECONDS" << "\n";
client_logfile << "TIME TAKEN: " << diff.tv_sec << " SECONDS " << diff.tv_nsec << " NANOSECONDS" << endl;
client_logfile << "Total messages between all servers = " << total_msg_count << endl;
client_logfile << "Total monitoring times is = " << num_monitor << endl;
client_logfile.flush();
client_logfile.close();
if (loadfile.is_open())
{
loadfile.flush();
loadfile.close();
}
//return 1;
}
pthread_exit(NULL);
}
//monitor the submitted tasks
// by polling each server for its load information. If a server is idle then it will return load = -4 (queue length - num of idle cores)
// so if every server returns load = -4 it implies that all submitted tasks are complete
void *monitor_function(void* args) {
ZHTClient *clientRet = (ZHTClient*)args;
Package loadPackage, loadhpcPackage, shutdownPackage;
string loadmessage("Monitoring Information!");
loadPackage.set_virtualpath(loadmessage);
loadPackage.set_operation(15);
string loadstr = loadPackage.SerializeAsString();
string loadhpcmessage("Monitoring HPC Information!");
loadhpcPackage.set_virtualpath(loadhpcmessage);
loadhpcPackage.set_operation(24);
string loadhpcstr = loadhpcPackage.SerializeAsString();
string endmessage("Shutdown!");
loadPackage.set_virtualpath(endmessage);
loadPackage.set_operation(98);
string endstr = loadPackage.SerializeAsString();
//int num_worker = clientRet.memberList.size();
int num_worker = clientRet->memberList.size();
int num_cores = 2;
int index = 0;
long termination_value = num_worker * num_cores * -1;
int total_avail_cores = num_cores * num_worker;
int32_t total_queued = 0;
int32_t total_idle = 0;
int32_t queued_busy = 0;
int32_t queued_idle = 0;
int32_t queued = 0;
int32_t num_idle = 0;
int32_t num_busy = 0;
int32_t load = 0;
int32_t total_busy = 0;
//int32_t status = 0;
int32_t finished = 0;
int32_t total_msg_count = 0;
int32_t ret = 0;
//sleep(60);
int min_lines = num_worker;
int num = num_worker - 1;
cout << "The Number is " << num << endl;
stringstream num_ss;
num_ss << num;
//min_lines++;
string filename(shared);
filename = filename + "startinfo" + num_ss.str();
cout << "The filename is " << filename << endl;
string cmd("cat ");
cmd = cmd + filename + " | wc -l";
cout << "The command is " << cmd << endl;
string result = executeShell(cmd);
//cout << cmd << " " << result << endl;
//cout << "client: minlines = " << min_lines << " cmd = " << cmd << " result = " << result << endl;
/*string filename(shared);
filename = filename + "start_info";
string cmd("wc -l ");
cmd = cmd + filename + " | awk {\'print $1\'}";
string result = executeShell(cmd);*/
while(atoi(result.c_str()) < 1) {
sleep(5);
result = executeShell(cmd); cout << " temp result = " << result << endl;
}
cout << "client: minlines = 1 " << " cmd = " << cmd << " result = " << result << endl;
//cout << "starting to monitor" << endl;
cout << "TIME START: " << start_tasks.tv_sec << " SECONDS " << start_tasks.tv_nsec << " NANOSECONDS" << endl;
while(1) {
//If mtc task or only complete queue values are taken
if(NUM_OF_CORES == 1)
{
total_queued = 0;
total_idle = 0;
queued_busy = 0;
stringstream worker_load;
for(index = 0; index < num_worker; index++) {
//int32_t queued_idle = clientRet.svrtosvr(loadstr, loadstr.length(), index);
queued_idle = clientRet->svrtosvr(loadstr, loadstr.length(), index);
queued = queued_idle/10; // summation of the lengths of the three queues
num_idle = queued_idle%10; // number of idle cores
total_queued = total_queued + queued;
total_idle = total_idle + num_idle;
num_busy = num_cores - num_idle;
load = queued + num_busy;
worker_load << load << " ";
}
loadfile << worker_load.str() << endl;
total_busy = total_avail_cores - total_idle;
queued_busy = total_queued + total_busy;
finished = total_num_tasks - queued_busy;
clock_gettime(CLOCK_REALTIME, &end_tasks);
cout << "Total busy cores " << total_busy << " Total Load on all workers = " << queued_busy << " No. of tasks finished = " << finished << " Total tasks submitted = " << total_num_tasks << endl;//" time = " << end_tasks.tv_sec << " " << end_tasks.tv_nsec << endl;
if (client_logfile.is_open() && cl_LOGGING) {
client_logfile << "Total busy cores " << total_busy << " Total Load on all workers = " << queued_busy << " No. of tasks finished = " << finished << " Total tasks submitted = " << total_num_tasks << endl;
}
if(finished == total_num_tasks) {
clock_gettime(CLOCK_REALTIME, &end_tasks);
cout << "\n\n\n\n==============================All tasks finished===========================\n\n\n\n";
break;
}
usleep(200000);
}
else
{
finished = 0;
for(index = 0; index < num_worker; index++)
{
finished += clientRet->svrtosvr(loadhpcstr, loadhpcstr.length(), index);
}
cout << "The number of finished tasks = " << finished << endl;
if(finished >= total_num_tasks-50) {
clock_gettime(CLOCK_REALTIME, &end_tasks);
cout << "\n\n\n\n==============================All tasks finished===========================\n\n\n\n";
break;
}
usleep(200000);
}
}
total_msg_count = 0;
for(index = 0; index < num_worker; index++) {
//clientRet.svrtosvr(endstr, endstr.length(), index);
ret = clientRet->svrtosvr(endstr, endstr.length(), index);
total_msg_count += ret;
}
cout << "TIME START: " << start_tasks.tv_sec << " SECONDS " << start_tasks.tv_nsec << " NANOSECONDS" << "\n";
cout << "TIME END: " << end_tasks.tv_sec << " SECONDS " << end_tasks.tv_nsec << " NANOSECONDS" << "\n";
timespec diff = timediff(start_tasks, end_tasks);
cout << "TIME TAKEN: " << diff.tv_sec << " SECONDS " << diff.tv_nsec << " NANOSECONDS" << "\n";
cout << "Total messages between all servers = " << total_msg_count << endl;
if (client_logfile.is_open() && cl_LOGGING) {
client_logfile << "\n\n\n\n==============================All tasks finished===========================\n\n\n\n";
client_logfile << "TIME START: " << start_tasks.tv_sec << " SECONDS " << start_tasks.tv_nsec << " NANOSECONDS" << "\n";
client_logfile << "TIME END: " << end_tasks.tv_sec << " SECONDS " << end_tasks.tv_nsec << " NANOSECONDS" << "\n";
client_logfile << "TIME TAKEN: " << diff.tv_sec << " SECONDS " << diff.tv_nsec << " NANOSECONDS" << endl;
client_logfile << "Total messages between all servers = " << total_msg_count << endl;
client_logfile.close();
//return 1;
}
pthread_exit(NULL);
}
//initialize all tasks
int MATRIXClient::initializeTasks(int num_tasks_req, int numSleep, int mode, int max_tasks_per_package, ZHTClient &clientRet, int DAG_choice, int num_of_cores){
srand(time(NULL)); // Random seed for all time measurements
// Task description
char task[10];
memset(task, '\0', 10);
sprintf(task, "%d", numSleep); // sleep time - 1, 2, 4, 8, 16, 32
string task_desc(task);
int num_worker = clientRet.memberList.size();
toserver = (num_worker-1)/(selfindex+1); // Server index where the tasks will be initially submitted to the Wait queue
//cout << "to server = " << toserver << endl;
// Initialize a random DAG based on the number of tasks requested by client
// Note: The number of tasks in the generated DAG may not be actually equal to what is requested
// This is fine for now, as in real systems the actual DAG would be supplied rather than we generate one.
int num_tasks; // number of tasks actually generated
TaskDAG dag = generate_DAG(num_tasks_req, num_tasks, client_id, DAG_choice); //cout << "total tasks = " << num_tasks << endl;
//total_num_tasks = num_tasks * num_worker;
total_num_tasks = num_tasks;
print_DAG(dag);
// Submission time for the task; for simplicity it is kept same for all tasks
timespec sub_time;
clock_gettime(CLOCK_REALTIME, &sub_time);
uint64_t sub_time_ns;
sub_time_ns = (uint64_t)sub_time.tv_sec*1000000000 + (uint64_t)sub_time.tv_nsec;
// Arguments to be passed to submission thread:
// 1. Vector that holds serialized packages for each individual task,
// 2. ZHT Client for network communication, and
// 3. Number of tasks to be submitted
submit_args thread_args;
thread_args.task_str_list = &task_str_list;
thread_args.clientRet = clientRet;
thread_args.per_client_task = num_tasks;
cout << "THE NUMBER OF TASKS SUBMITTED IS " << num_tasks << endl;
// Spin the submission thread with the structure containing the above mentioned arguments
pthread_t submit_thread;
//pthread_create(&submit_thread, NULL, submit, &thread_args);
// Reserve space for the vector to hold serialized packages for each individual task
task_str_list.reserve(num_tasks);
// Measure the start time for task submission into NoVoHT
clock_gettime(CLOCK_REALTIME, &start_tasks);
// For all tasks in the DAG, package it and store it in NoVOHT
//for(i = 0; i < num_tasks; i++){
for(TaskDAG::iterator it = dag.begin(); it != dag.end(); ++it) {
int task_id = it->first;
TaskDAG_Value value(it->second);
stringstream task_id_ss;
task_id_ss << task_id << client_id; // Task ID + Client ID
//cout << "id = " << task_id_ss.str();
//cout << "numwait = " << value.first << " notlist = " << value.second << endl;
Package package;
package.set_virtualpath(task_id_ss.str()); // Key is task ID + client ID
package.set_operation(3); // Insert task and its decription into NoVoHT
// cout << "key = " << package.virtualpath();
// cout << " op = " << package.operation() << endl;
// string str1 = package.SerializeAsString(); // Serialize the package
//cout << " str1 = " << str1 << endl;
stringstream to_index_ss;
to_index_ss << toserver << "\'" << "\"";
package.set_nodehistory(to_index_ss.str()); // History of migrations delimited by \' with a final \"
// cout << "nodehistory = " << package.nodehistory();
package.set_currnode(toserver); // Current node
// cout << " currnode = " << package.currnode();
package.set_nummoves(0); // Number of migrations, initially it is zero
// cout << " nummoves = " << package.nummoves();
package.set_numwait(value.first); // Number of notifications to receive before it can be moved to ready queue
// cout << " numwait = " << package.numwait() << endl;
//package.set_notlist(value.second); // List of tasks to be notified after finishing execution
//cout << "notlist = " << package.notlist() << endl;
// string str2 = package.SerializeAsString(); // Serialize the package
//cout << " str2 = " << str2 << endl;
stringstream package_content_ss;
//package_content_ss << value.second; // List of tasks to be notified after finishing execution
package_content_ss << "NULL"; package_content_ss << "\'"; // Task completion status
package_content_ss << task_desc; package_content_ss << "\'"; // Task Description
package_content_ss << task_id_ss.str(); package_content_ss << "\'"; // Task ID
package_content_ss << sub_time_ns; package_content_ss << "\'"; // Task Submission Time
package_content_ss << num_of_cores;package_content_ss << "\'";
if(num_of_cores==1)
{
package_content_ss << -2;
package_content_ss << "\'";
}
else
{
package_content_ss << -1;
package_content_ss << "\'";
}
package_content_ss << "\"";
package_content_ss << value.second; // List of tasks to be notified after finishing execution
package.set_realfullpath(package_content_ss.str());
cout << "The package content at the client is " << package_content_ss.str() << endl;
string str = package.SerializeAsString(); // Serialize the package
//cout << " str = " << str << endl;
//pthread_mutex_lock(&submit_q);
// Push the serialized task into a vector which is shared by another thread that handles the submission over the network
task_str_list.push_back(str);
//pthread_mutex_unlock(&submit_q);
}
clock_gettime(CLOCK_REALTIME, &end_tasks);
timespec diff1 = timediff(start_tasks, end_tasks);
cout << "novoht jobs created. TIME TAKEN: " << diff1.tv_sec << " SECONDS " << diff1.tv_nsec << " NANOSECONDS" << endl;
get_map(task_str_list, num_worker);
clock_gettime(CLOCK_REALTIME, &start_tasks);
submittasks(clientRet);
//pthread_join(submit_thread, NULL); // Wait for the submission thread to finish sending the tasks over the network
clock_gettime(CLOCK_REALTIME, &end_tasks); // Measure the end time to insert all tasks into NoVoHT
timespec diff = timediff(start_tasks, end_tasks); // Measure the total time to insert all tasks into NoVoHT
cout << num_tasks << " tasks inserted into NoVoHT" << endl;
cout << "TIME TAKEN: " << diff.tv_sec << " SECONDS " << diff.tv_nsec << " NANOSECONDS" << endl;
if (client_logfile.is_open() && cl_LOGGING) {
client_logfile << num_tasks << "tasks inserted into NoVoHT" << endl;
client_logfile << "TIME TAKEN: " << diff.tv_sec << " SECONDS " << diff.tv_nsec << " NANOSECONDS" << endl;
}
//exit(1);
// Some temp parameters
int num_packages = 0;
int total_submitted1 = 0;
static int id = 0;
// Measure the start time for task submission into Wait queue
clock_gettime(CLOCK_REALTIME, &start_tasks);
TaskDAG::iterator it = dag.begin();
while (total_submitted1 != num_tasks) {
Package package; string alltasks;
package.set_virtualpath(client_id); // Here key is just the client ID
package.set_operation(21);
//package.set_operation(22);
num_packages++;
int num_tasks_this_package = max_tasks_per_package;
int num_tasks_left = num_tasks - total_submitted1;
if (num_tasks_left < max_tasks_per_package) {
num_tasks_this_package = num_tasks_left;
}
for(int j = 0; j < num_tasks_this_package; j++) {
int task_id = it->first; ++it;
stringstream task_id_ss;
task_id_ss << task_id << client_id; // Task ID + Client ID
alltasks.append(task_id_ss.str()); alltasks.append("\'\""); // Task ID
}
total_submitted1 = total_submitted1 + num_tasks_this_package;
package.set_realfullpath(alltasks);
string str = package.SerializeAsString();
//cout << "String size = " << str.size() << " str length = " << strlen(str.c_str());
int32_t ret = clientRet.svrtosvr(str, str.length(), toserver);
//int32_t ret = clientRet.svrtosvr(str, str.length(), selfindex);
}
//cout << "No. of packages = " << num_packages;
cout << "\tTotal Jobs submitted = " << total_submitted1 << endl;
clock_gettime(CLOCK_REALTIME, &end_tasks); // Measure the end time to insert all tasks into Wait queue
diff = timediff(start_tasks, end_tasks); // Measure the total time to insert all tasks into Wait queue
//cout << "TIME TAKEN: " << diff.tv_sec << " SECONDS " << diff.tv_nsec << " NANOSECONDS" << endl;
if (client_logfile.is_open() && cl_LOGGING) {
client_logfile << "to server = " << toserver << endl;
client_logfile << "Total Jobs submitted = " << total_submitted1 << endl;
client_logfile << "TIME TAKEN: " << diff.tv_sec << " SECONDS " << diff.tv_nsec << " NANOSECONDS" << endl;
}
//exit(1);
}