void parallel_data_exchange_t(std::vector< std::vector<T> > &send_lists, std::vector< std::vector<T> > &recv_lists, MPI_Comm &mpi_communicator ) { // // Determine the number of processors involved in this communication // const int msg_tag = 10242; int num_procs; MPI_Comm_size(mpi_communicator, &num_procs); int my_proc; MPI_Comm_rank(mpi_communicator, &my_proc); ThrowRequire((unsigned int) num_procs == send_lists.size() && (unsigned int) num_procs == recv_lists.size()); int class_size = sizeof(T); // // Determine number of items each other processor will send to the current processor // std::vector<int> global_number_to_send(num_procs); for(int iproc=0; iproc<num_procs; ++iproc) { global_number_to_send[iproc] = send_lists[iproc].size(); } std::vector<int> numToRecvFrom = ComputeReceiveList(global_number_to_send, mpi_communicator); // // Send the actual messages as raw byte streams. // std::vector<MPI_Request> recv_handles(num_procs); for(int iproc = 0; iproc < num_procs; ++iproc) { recv_lists[iproc].resize(numToRecvFrom[iproc]); if(recv_lists[iproc].size() > 0) { char* recv_buffer = (char*)&recv_lists[iproc][0]; int recv_size = recv_lists[iproc].size()*class_size; MPI_Irecv(recv_buffer, recv_size, MPI_CHAR, iproc, msg_tag, mpi_communicator, &recv_handles[iproc]); } } MPI_Barrier(mpi_communicator); for(int iproc = 0; iproc < num_procs; ++iproc) { if(send_lists[iproc].size() > 0) { char* send_buffer = (char*)&send_lists[iproc][0]; int send_size = send_lists[iproc].size()*class_size; MPI_Send(send_buffer, send_size, MPI_CHAR, iproc, msg_tag, mpi_communicator); } } for(int iproc = 0; iproc < num_procs; ++iproc) { if(recv_lists[iproc].size() > 0) { MPI_Status status; MPI_Wait( &recv_handles[iproc], &status ); } } }
void parallel_data_exchange_sym_t(std::vector< std::vector<T> > &send_lists, std::vector< std::vector<T> > &recv_lists, MPI_Comm &mpi_communicator ) { // // Determine the number of processors involved in this communication // #if defined( STK_HAS_MPI) const int msg_tag = 10242; int num_procs = stk::parallel_machine_size(mpi_communicator); int class_size = sizeof(T); // // Send the actual messages as raw byte streams. // std::vector<MPI_Request> recv_handles(num_procs); for(int iproc = 0; iproc < num_procs; ++iproc) { recv_lists[iproc].resize(send_lists[iproc].size()); if(recv_lists[iproc].size() > 0) { char* recv_buffer = (char*)&recv_lists[iproc][0]; int recv_size = recv_lists[iproc].size()*class_size; MPI_Irecv(recv_buffer, recv_size, MPI_CHAR, iproc, msg_tag, mpi_communicator, &recv_handles[iproc]); } } MPI_Barrier(mpi_communicator); for(int iproc = 0; iproc < num_procs; ++iproc) { if(send_lists[iproc].size() > 0) { char* send_buffer = (char*)&send_lists[iproc][0]; int send_size = send_lists[iproc].size()*class_size; MPI_Send(send_buffer, send_size, MPI_CHAR, iproc, msg_tag, mpi_communicator); } } for(int iproc = 0; iproc < num_procs; ++iproc) { if(recv_lists[iproc].size() > 0) { MPI_Status status; MPI_Wait( &recv_handles[iproc], &status ); } } #endif }
inline void parallel_data_exchange_nonsym_known_sizes_t(std::vector< std::vector<T> > &send_lists, std::vector< std::vector<T> > &recv_lists, MPI_Comm mpi_communicator ) { #if defined( STK_HAS_MPI) const int msg_tag = 10243; //arbitrary tag value, anything less than 32768 is legal int num_procs = stk::parallel_machine_size(mpi_communicator); int class_size = sizeof(T); // // Send the actual messages as raw byte streams. // std::vector<MPI_Request> recv_handles(num_procs); for(int iproc = 0; iproc < num_procs; ++iproc) { if(recv_lists[iproc].size() > 0) { char* recv_buffer = (char*)&recv_lists[iproc][0]; int recv_size = recv_lists[iproc].size()*class_size; MPI_Irecv(recv_buffer, recv_size, MPI_CHAR, iproc, msg_tag, mpi_communicator, &recv_handles[iproc]); } } MPI_Barrier(mpi_communicator); for(int iproc = 0; iproc < num_procs; ++iproc) { if(send_lists[iproc].size() > 0) { char* send_buffer = (char*)&send_lists[iproc][0]; int send_size = send_lists[iproc].size()*class_size; MPI_Send(send_buffer, send_size, MPI_CHAR, iproc, msg_tag, mpi_communicator); } } for(int iproc = 0; iproc < num_procs; ++iproc) { if(recv_lists[iproc].size() > 0) { MPI_Status status; MPI_Wait( &recv_handles[iproc], &status ); } } #endif }
void parallel_data_exchange_sym_unknown_size_t(std::vector< std::vector<T> > &send_lists, std::vector< std::vector<T> > &recv_lists, MPI_Comm &mpi_communicator ) { #if defined( STK_HAS_MPI) const int msg_tag = 10242; int num_procs = stk::parallel_machine_size(mpi_communicator); int class_size = sizeof(T); // // Send the message sizes // std::vector<int> send_msg_sizes(num_procs); std::vector<int> recv_msg_sizes(num_procs); std::vector<MPI_Request> recv_handles(num_procs); for(int iproc = 0; iproc < num_procs; ++iproc) { send_msg_sizes[iproc] = send_lists[iproc].size(); } for(int iproc = 0; iproc < num_procs; ++iproc) { if(recv_lists[iproc].size()>0) { MPI_Irecv(&recv_msg_sizes[iproc], 1, MPI_INT, iproc, msg_tag, mpi_communicator, &recv_handles[iproc]); } } MPI_Barrier(mpi_communicator); for(int iproc = 0; iproc < num_procs; ++iproc) { if(send_lists[iproc].size()>0) { MPI_Send(&send_msg_sizes[iproc], 1, MPI_INT, iproc, msg_tag, mpi_communicator); } } for(int iproc = 0; iproc < num_procs; ++iproc) { if(recv_lists[iproc].size() > 0) { MPI_Status status; MPI_Wait( &recv_handles[iproc], &status ); recv_lists[iproc].resize(recv_msg_sizes[iproc]); } } // // Send the actual messages as raw byte streams. // for(int iproc = 0; iproc < num_procs; ++iproc) { if(recv_lists[iproc].size() > 0) { char* recv_buffer = (char*)&recv_lists[iproc][0]; int recv_size = recv_lists[iproc].size()*class_size; MPI_Irecv(recv_buffer, recv_size, MPI_CHAR, iproc, msg_tag, mpi_communicator, &recv_handles[iproc]); } } MPI_Barrier(mpi_communicator); for(int iproc = 0; iproc < num_procs; ++iproc) { if(send_lists[iproc].size() > 0) { char* send_buffer = (char*)&send_lists[iproc][0]; int send_size = send_lists[iproc].size()*class_size; MPI_Send(send_buffer, send_size, MPI_CHAR, iproc, msg_tag, mpi_communicator); } } for(int iproc = 0; iproc < num_procs; ++iproc) { if(recv_lists[iproc].size() > 0) { MPI_Status status; MPI_Wait( &recv_handles[iproc], &status ); } } #endif }