void addItemsToCommMap(int proc, size_t numItems, const T* items,
typename CommMap<T>::Type& comm_map,
bool keep_sorted_and_unique = true)
{
typename CommMap<T>::Type::iterator iter = comm_map.find(proc);
if (iter == comm_map.end()) {
iter = comm_map.insert(std::make_pair(proc,std::vector<T>())).first;
}
std::vector<T>& comm_items = iter->second;
if (keep_sorted_and_unique) {
for(size_t i=0; i<numItems; ++i) {
fei::sortedListInsert(items[i], comm_items);
}
}
else {
for(size_t i=0; i<numItems; ++i) {
comm_items.push_back(items[i]);
}
}
}
int exchangeCommMapData(MPI_Comm comm,
const typename CommMap<T>::Type& sendCommMap,
typename CommMap<T>::Type& recvCommMap,
bool recvProcsKnownOnEntry = false,
bool recvLengthsKnownOnEntry = false)
{
if (!recvProcsKnownOnEntry) {
recvCommMap.clear();
}
#ifndef FEI_SER
int tag = 11120;
MPI_Datatype mpi_dtype = fei::mpiTraits<T>::mpi_type();
std::vector<int> sendProcs;
fei::copyKeysToVector(sendCommMap, sendProcs);
std::vector<int> recvProcs;
if (recvProcsKnownOnEntry) {
fei::copyKeysToVector(recvCommMap, recvProcs);
}
else {
mirrorProcs(comm, sendProcs, recvProcs);
for(size_t i=0; i<recvProcs.size(); ++i) {
addItemsToCommMap<T>(recvProcs[i], 0, NULL, recvCommMap);
}
}
if (!recvLengthsKnownOnEntry) {
std::vector<int> tmpIntData(sendProcs.size());
std::vector<int> recvLengths(recvProcs.size());
typename fei::CommMap<T>::Type::const_iterator
s_iter = sendCommMap.begin(), s_end = sendCommMap.end();
for(size_t i=0; s_iter != s_end; ++s_iter, ++i) {
tmpIntData[i] = s_iter->second.size();
}
if ( exchangeIntData(comm, sendProcs, tmpIntData, recvProcs, recvLengths) != 0) {
return(-1);
}
for(size_t i=0; i<recvProcs.size(); ++i) {
std::vector<T>& rdata = recvCommMap[recvProcs[i]];
rdata.resize(recvLengths[i]);
}
}
//launch Irecv's for recv-data:
std::vector<MPI_Request> mpiReqs;
mpiReqs.resize(recvProcs.size());
typename fei::CommMap<T>::Type::iterator
r_iter = recvCommMap.begin(), r_end = recvCommMap.end();
size_t req_offset = 0;
for(; r_iter != r_end; ++r_iter) {
int rproc = r_iter->first;
std::vector<T>& recv_vec = r_iter->second;
int len = recv_vec.size();
T* recv_buf = len > 0 ? &recv_vec[0] : NULL;
CHK_MPI( MPI_Irecv(recv_buf, len, mpi_dtype, rproc,
tag, comm, &mpiReqs[req_offset++]) );
}
//send the send-data:
typename fei::CommMap<T>::Type::const_iterator
s_iter = sendCommMap.begin(), s_end = sendCommMap.end();
for(; s_iter != s_end; ++s_iter) {
int sproc = s_iter->first;
const std::vector<T>& send_vec = s_iter->second;
int len = send_vec.size();
T* send_buf = len>0 ? const_cast<T*>(&send_vec[0]) : NULL;
CHK_MPI( MPI_Send(send_buf, len, mpi_dtype, sproc, tag, comm) );
}
//complete the Irecvs:
for(size_t i=0; i<mpiReqs.size(); ++i) {
int index;
MPI_Status status;
CHK_MPI( MPI_Waitany(mpiReqs.size(), &mpiReqs[0], &index, &status) );
}
#endif
return(0);
}
