/**
* Reduce all server timers to server master.
* @param timer
*/
void mpi_reduce_timers(TIMER& timer){
sip::SIPMPIAttr &attr = sip::SIPMPIAttr::get_instance();
sip::check(attr.is_server(), "Trying to reduce timer on a non-server rank !");
long long * timers = timer.get_timers();
long long * timer_counts = timer.get_timer_count();
// Data to send to reduce
long long * sendbuf = new long long[2*timer.max_slots + 1];
sendbuf[0] = timer.max_slots;
// The data will be structured as
// Length of arrays 1 & 2
// Array1 -> timer_switched_ array
// Array2 -> timer_list_ array
std::copy(timer_counts + 0, timer_counts + timer.max_slots, sendbuf+1);
std::copy(timers + 0, timers + timer.max_slots, sendbuf+1+ timer.max_slots);
long long * recvbuf = new long long[2*timer.max_slots + 1]();
int server_master = attr.COMPANY_MASTER_RANK;
MPI_Comm server_company = attr.company_communicator();
MPI_Datatype server_timer_reduce_dt; // MPI Type for timer data to be reduced.
MPI_Op server_timer_reduce_op; // MPI OP to reduce timer data.
SIPMPIUtils::check_err(MPI_Type_contiguous(timer.max_slots*2+1, MPI_LONG_LONG, &server_timer_reduce_dt));
SIPMPIUtils::check_err(MPI_Type_commit(&server_timer_reduce_dt));
SIPMPIUtils::check_err(MPI_Op_create((MPI_User_function *)server_timer_reduce_op_function, 1, &server_timer_reduce_op));
SIPMPIUtils::check_err(MPI_Reduce(sendbuf, recvbuf, 1, server_timer_reduce_dt, server_timer_reduce_op, server_master, server_company));
if (attr.is_company_master()){
std::copy(recvbuf+1, recvbuf+1+timer.max_slots, timer_counts);
std::copy(recvbuf+1+timer.max_slots, recvbuf+1+2*timer.max_slots, timers);
}
// Cleanup
delete [] sendbuf;
delete [] recvbuf;
SIPMPIUtils::check_err(MPI_Type_free(&server_timer_reduce_dt));
SIPMPIUtils::check_err(MPI_Op_free(&server_timer_reduce_op));
}
virtual void execute(TIMER& timer){
mpi_reduce_timers(timer); // Reduce timers to server master.
// Print from the server master.
if (SIPMPIAttr::get_instance().is_company_master()){
std::cout << "Timers for Program " << GlobalState::get_program_name() << std::endl;
long long * timers = timer.get_timers();
long long * timer_counts = timer.get_timer_count();
const int LW = 10; // Line num
const int CW = 15; // Time
const int SW = 20; // String
assert(timer.check_timers_off());
std::cout<<"Timers"<<std::endl
<<std::setw(LW)<<std::left<<"Line"
<<std::setw(SW)<<std::left<<"Type"
<<std::setw(CW)<<std::left<<"Avg"
<<std::setw(CW)<<std::left<<"AvgBlkWait"
<<std::setw(CW)<<std::left<<"AvgDiskRead"
<<std::setw(CW)<<std::left<<"AvgDiskWrite"
<<std::setw(CW)<<std::left<<"Tot"
<<std::endl;
for (int i=1; i<sialx_lines_; i++){
int tot_time_offset = i + static_cast<int>(ServerTimer::TOTALTIME) * sialx_lines_;
if (timer_counts[tot_time_offset] > 0L){
double tot_time = timer.to_seconds(timers[tot_time_offset]); // Microsecond to second
double avg_time = tot_time / timer_counts[tot_time_offset];
double tot_blk_wait = 0;
double avg_blk_wait = 0;
double tot_disk_read = 0;
double avg_disk_read = 0;
double tot_disk_write = 0;
double avg_disk_write = 0;
int blk_wait_offset = i + static_cast<int>(ServerTimer::BLOCKWAITTIME) * sialx_lines_;
if (timer_counts[blk_wait_offset] > 0L){
tot_blk_wait = timer.to_seconds(timers[blk_wait_offset]);
avg_blk_wait = tot_blk_wait / timer_counts[blk_wait_offset];
}
int read_timer_offset = i + static_cast<int>(ServerTimer::READTIME) * sialx_lines_;
if (timer_counts[read_timer_offset] > 0L){
tot_disk_read = timer.to_seconds(timers[read_timer_offset]);
avg_disk_read = tot_disk_read / timer_counts[read_timer_offset];
}
int write_timer_offset = i + static_cast<int>(ServerTimer::WRITETIME) * sialx_lines_;
if (timer_counts[write_timer_offset] > 0L){
tot_disk_write = timer.to_seconds(timers[write_timer_offset]);
avg_disk_write = tot_disk_write / timer_counts[write_timer_offset];
}
std::cout<<std::setw(LW)<<std::left << i
<< std::setw(SW)<< std::left << line_to_str_.at(i)
<< std::setw(CW)<< std::left << avg_time
<< std::setw(CW)<< std::left << avg_blk_wait
<< std::setw(CW)<< std::left << avg_disk_read
<< std::setw(CW)<< std::left << avg_disk_write
<< std::setw(CW)<< std::left << tot_time
<< std::endl;
}
}
std::cout<<std::endl;
}
}
