void parallel_data_exchange_t(std::vector< std::vector<T> > &send_lists,
std::vector< std::vector<T> > &recv_lists,
MPI_Comm &mpi_communicator ) {
//
// Determine the number of processors involved in this communication
//
const int msg_tag = 10242;
int num_procs;
MPI_Comm_size(mpi_communicator, &num_procs);
int my_proc;
MPI_Comm_rank(mpi_communicator, &my_proc);
ThrowRequire((unsigned int) num_procs == send_lists.size() && (unsigned int) num_procs == recv_lists.size());
int class_size = sizeof(T);
//
// Determine number of items each other processor will send to the current processor
//
std::vector<int> global_number_to_send(num_procs);
for(int iproc=0; iproc<num_procs; ++iproc) {
global_number_to_send[iproc] = send_lists[iproc].size();
}
std::vector<int> numToRecvFrom = ComputeReceiveList(global_number_to_send, mpi_communicator);
//
// Send the actual messages as raw byte streams.
//
std::vector<MPI_Request> recv_handles(num_procs);
for(int iproc = 0; iproc < num_procs; ++iproc) {
recv_lists[iproc].resize(numToRecvFrom[iproc]);
if(recv_lists[iproc].size() > 0) {
char* recv_buffer = (char*)&recv_lists[iproc][0];
int recv_size = recv_lists[iproc].size()*class_size;
MPI_Irecv(recv_buffer, recv_size, MPI_CHAR,
iproc, msg_tag, mpi_communicator, &recv_handles[iproc]);
}
}
MPI_Barrier(mpi_communicator);
for(int iproc = 0; iproc < num_procs; ++iproc) {
if(send_lists[iproc].size() > 0) {
char* send_buffer = (char*)&send_lists[iproc][0];
int send_size = send_lists[iproc].size()*class_size;
MPI_Send(send_buffer, send_size, MPI_CHAR,
iproc, msg_tag, mpi_communicator);
}
}
for(int iproc = 0; iproc < num_procs; ++iproc) {
if(recv_lists[iproc].size() > 0) {
MPI_Status status;
MPI_Wait( &recv_handles[iproc], &status );
}
}
}
void parallel_data_exchange_sym_t(std::vector< std::vector<T> > &send_lists,
std::vector< std::vector<T> > &recv_lists,
MPI_Comm &mpi_communicator )
{
//
// Determine the number of processors involved in this communication
//
#if defined( STK_HAS_MPI)
const int msg_tag = 10242;
int num_procs = stk::parallel_machine_size(mpi_communicator);
int class_size = sizeof(T);
//
// Send the actual messages as raw byte streams.
//
std::vector<MPI_Request> recv_handles(num_procs);
for(int iproc = 0; iproc < num_procs; ++iproc) {
recv_lists[iproc].resize(send_lists[iproc].size());
if(recv_lists[iproc].size() > 0) {
char* recv_buffer = (char*)&recv_lists[iproc][0];
int recv_size = recv_lists[iproc].size()*class_size;
MPI_Irecv(recv_buffer, recv_size, MPI_CHAR,
iproc, msg_tag, mpi_communicator, &recv_handles[iproc]);
}
}
MPI_Barrier(mpi_communicator);
for(int iproc = 0; iproc < num_procs; ++iproc) {
if(send_lists[iproc].size() > 0) {
char* send_buffer = (char*)&send_lists[iproc][0];
int send_size = send_lists[iproc].size()*class_size;
MPI_Send(send_buffer, send_size, MPI_CHAR,
iproc, msg_tag, mpi_communicator);
}
}
for(int iproc = 0; iproc < num_procs; ++iproc) {
if(recv_lists[iproc].size() > 0) {
MPI_Status status;
MPI_Wait( &recv_handles[iproc], &status );
}
}
#endif
}
inline void parallel_data_exchange_nonsym_known_sizes_t(std::vector< std::vector<T> > &send_lists,
std::vector< std::vector<T> > &recv_lists,
MPI_Comm mpi_communicator )
{
#if defined( STK_HAS_MPI)
const int msg_tag = 10243; //arbitrary tag value, anything less than 32768 is legal
int num_procs = stk::parallel_machine_size(mpi_communicator);
int class_size = sizeof(T);
//
// Send the actual messages as raw byte streams.
//
std::vector<MPI_Request> recv_handles(num_procs);
for(int iproc = 0; iproc < num_procs; ++iproc) {
if(recv_lists[iproc].size() > 0) {
char* recv_buffer = (char*)&recv_lists[iproc][0];
int recv_size = recv_lists[iproc].size()*class_size;
MPI_Irecv(recv_buffer, recv_size, MPI_CHAR, iproc, msg_tag, mpi_communicator, &recv_handles[iproc]);
}
}
MPI_Barrier(mpi_communicator);
for(int iproc = 0; iproc < num_procs; ++iproc) {
if(send_lists[iproc].size() > 0) {
char* send_buffer = (char*)&send_lists[iproc][0];
int send_size = send_lists[iproc].size()*class_size;
MPI_Send(send_buffer, send_size, MPI_CHAR, iproc, msg_tag, mpi_communicator);
}
}
for(int iproc = 0; iproc < num_procs; ++iproc) {
if(recv_lists[iproc].size() > 0) {
MPI_Status status;
MPI_Wait( &recv_handles[iproc], &status );
}
}
#endif
}
void parallel_data_exchange_sym_unknown_size_t(std::vector< std::vector<T> > &send_lists,
std::vector< std::vector<T> > &recv_lists,
MPI_Comm &mpi_communicator )
{
#if defined( STK_HAS_MPI)
const int msg_tag = 10242;
int num_procs = stk::parallel_machine_size(mpi_communicator);
int class_size = sizeof(T);
//
// Send the message sizes
//
std::vector<int> send_msg_sizes(num_procs);
std::vector<int> recv_msg_sizes(num_procs);
std::vector<MPI_Request> recv_handles(num_procs);
for(int iproc = 0; iproc < num_procs; ++iproc) {
send_msg_sizes[iproc] = send_lists[iproc].size();
}
for(int iproc = 0; iproc < num_procs; ++iproc) {
if(recv_lists[iproc].size()>0) {
MPI_Irecv(&recv_msg_sizes[iproc], 1, MPI_INT, iproc, msg_tag, mpi_communicator, &recv_handles[iproc]);
}
}
MPI_Barrier(mpi_communicator);
for(int iproc = 0; iproc < num_procs; ++iproc) {
if(send_lists[iproc].size()>0) {
MPI_Send(&send_msg_sizes[iproc], 1, MPI_INT, iproc, msg_tag, mpi_communicator);
}
}
for(int iproc = 0; iproc < num_procs; ++iproc) {
if(recv_lists[iproc].size() > 0) {
MPI_Status status;
MPI_Wait( &recv_handles[iproc], &status );
recv_lists[iproc].resize(recv_msg_sizes[iproc]);
}
}
//
// Send the actual messages as raw byte streams.
//
for(int iproc = 0; iproc < num_procs; ++iproc) {
if(recv_lists[iproc].size() > 0) {
char* recv_buffer = (char*)&recv_lists[iproc][0];
int recv_size = recv_lists[iproc].size()*class_size;
MPI_Irecv(recv_buffer, recv_size, MPI_CHAR,
iproc, msg_tag, mpi_communicator, &recv_handles[iproc]);
}
}
MPI_Barrier(mpi_communicator);
for(int iproc = 0; iproc < num_procs; ++iproc) {
if(send_lists[iproc].size() > 0) {
char* send_buffer = (char*)&send_lists[iproc][0];
int send_size = send_lists[iproc].size()*class_size;
MPI_Send(send_buffer, send_size, MPI_CHAR,
iproc, msg_tag, mpi_communicator);
}
}
for(int iproc = 0; iproc < num_procs; ++iproc) {
if(recv_lists[iproc].size() > 0) {
MPI_Status status;
MPI_Wait( &recv_handles[iproc], &status );
}
}
#endif
}
