static void send_data (msg_task_t msg) { char mailbox[MAILBOX_ALIAS_SIZE]; double data_size; size_t my_id; task_info_t ti; my_id = get_worker_id (MSG_host_self ()); sprintf (mailbox, TASK_MAILBOX, get_worker_id (MSG_task_get_source (msg)), MSG_process_get_PID (MSG_task_get_sender (msg))); if (message_is (msg, SMS_GET_CHUNK)) { MSG_task_dsend (MSG_task_create ("DATA-C", 0.0, config.chunk_size, NULL), mailbox, NULL); } else if (message_is (msg, SMS_GET_INTER_PAIRS)) { ti = (task_info_t) MSG_task_get_data (msg); data_size = job.map_output[my_id][ti->id] - ti->map_output_copied[my_id]; MSG_task_dsend (MSG_task_create ("DATA-IP", 0.0, data_size, NULL), mailbox, NULL); } MSG_task_destroy (msg); }
int data_node (int argc, char* argv[]) { char mailbox[MAILBOX_ALIAS_SIZE]; msg_error_t status; msg_task_t msg = NULL; sprintf (mailbox, DATANODE_MAILBOX, get_worker_id (MSG_host_self ())); while (!job.finished) { msg = NULL; status = receive (&msg, mailbox); if (status == MSG_OK) { if (message_is (msg, SMS_FINISH)) { MSG_task_destroy (msg); break; } else { send_data (msg); } } } return 0; }
/** @brief Main master function. */ int master (int argc, char* argv[]) { heartbeat_t heartbeat; msg_error_t status; msg_host_t worker; msg_task_t msg = NULL; size_t wid; task_info_t ti; double total_cpu_time = 0.0; double total_task_time = 0.0; print_config (); XBT_INFO ("JOB BEGIN"); XBT_INFO (" "); tasks_log = fopen ("tasks.csv", "w"); fprintf (tasks_log, "task_id,phase,worker_id,time,action,shuffle_end\n"); while (job.tasks_pending[MAP] + job.tasks_pending[REDUCE] > 0) { msg = NULL; status = receive (&msg, MASTER_MAILBOX); if (status == MSG_OK) { worker = MSG_task_get_source (msg); wid = get_worker_id (worker); if (message_is (msg, SMS_HEARTBEAT)) { heartbeat = &job.heartbeats[wid]; if (is_straggler (worker)) { set_speculative_tasks (worker); } else { if (heartbeat->slots_av[MAP] > 0) send_scheduler_task(MAP, wid); if (heartbeat->slots_av[REDUCE] > 0) send_scheduler_task(REDUCE, wid); } } else if (message_is (msg, SMS_TASK_DONE)) { ti = (task_info_t) MSG_task_get_data (msg); if (job.task_status[ti->phase][ti->id] != T_STATUS_DONE) { job.task_status[ti->phase][ti->id] = T_STATUS_DONE; finish_all_task_copies (ti); job.tasks_pending[ti->phase]--; if (job.tasks_pending[ti->phase] <= 0) { XBT_INFO (" "); XBT_INFO ("%s PHASE DONE", (ti->phase==MAP?"MAP":"REDUCE")); XBT_INFO (" "); } ti->finished_time = MSG_get_clock(); ti->elapsed_time = ti->finished_time - ti->start_time; total_task_time += ti->elapsed_time; total_cpu_time += ti->cpu_time; } xbt_free_ref (&ti); } MSG_task_destroy (msg); } } fclose (tasks_log); job.finished = 1; print_config (); print_stats (); XBT_INFO ("JOB END"); XBT_INFO ("\tclock_time: %f", MSG_get_clock()); XBT_INFO ("\ttotal_task_time: %f(%f)", total_task_time, total_task_time / MSG_get_clock()); XBT_INFO ("\ttotal_cpu_time: %f(%f)", total_cpu_time, total_cpu_time / MSG_get_clock()); return 0; }