void SpParHelper::BCastMatrix(MPI_Comm & comm1d, SpMat<IT,NT,DER> & Matrix, const vector<IT> & essentials, int root)
{
int myrank;
MPI_Comm_rank(comm1d, &myrank);
if(myrank != root)
{
Matrix.Create(essentials); // allocate memory for arrays
}
Arr<IT,NT> arrinfo = Matrix.GetArrays();
for(unsigned int i=0; i< arrinfo.indarrs.size(); ++i) // get index arrays
{
MPI_Bcast(arrinfo.indarrs[i].addr, arrinfo.indarrs[i].count, MPIType<IT>(), root, comm1d);
}
for(unsigned int i=0; i< arrinfo.numarrs.size(); ++i) // get numerical arrays
{
MPI_Bcast(arrinfo.numarrs[i].addr, arrinfo.numarrs[i].count, MPIType<NT>(), root, comm1d);
}
}
void SpParHelper::SetWindows(MPI_Comm & comm1d, const SpMat< IT,NT,DER > & Matrix, vector<MPI_Win> & arrwin)
{
Arr<IT,NT> arrs = Matrix.GetArrays();
// static MPI::Win MPI::Win::create(const void *base, MPI::Aint size, int disp_unit, MPI::Info info, const MPI_Comm & comm);
// The displacement unit argument is provided to facilitate address arithmetic in RMA operations
// *** COLLECTIVE OPERATION ***, everybody exposes its own array to everyone else in the communicator
for(int i=0; i< arrs.indarrs.size(); ++i)
{
MPI_Win nWin;
MPI_Win_create(arrs.indarrs[i].addr,
arrs.indarrs[i].count * sizeof(IT), sizeof(IT), MPI_INFO_NULL, comm1d, &nWin);
arrwin.push_back(nWin);
}
for(int i=0; i< arrs.numarrs.size(); ++i)
{
MPI_Win nWin;
MPI_Win_create(arrs.numarrs[i].addr,
arrs.numarrs[i].count * sizeof(NT), sizeof(NT), MPI_INFO_NULL, comm1d, &nWin);
arrwin.push_back(nWin);
}
}
void SpParHelper::FetchMatrix(SpMat<IT,NT,DER> & MRecv, const vector<IT> & essentials, vector<MPI_Win> & arrwin, int ownind)
{
MRecv.Create(essentials); // allocate memory for arrays
Arr<IT,NT> arrinfo = MRecv.GetArrays();
assert( (arrwin.size() == arrinfo.totalsize()));
// C-binding for MPI::Get
// int MPI_Get(void *origin_addr, int origin_count, MPI_Datatype origin_datatype, int target_rank, MPI_Aint target_disp,
// int target_count, MPI_Datatype target_datatype, MPI_Win win)
IT essk = 0;
for(int i=0; i< arrinfo.indarrs.size(); ++i) // get index arrays
{
//arrwin[essk].Lock(MPI::LOCK_SHARED, ownind, 0);
MPI_Get( arrinfo.indarrs[i].addr, arrinfo.indarrs[i].count, MPIType<IT>(), ownind, 0, arrinfo.indarrs[i].count, MPIType<IT>(), arrwin[essk++]);
}
for(int i=0; i< arrinfo.numarrs.size(); ++i) // get numerical arrays
{
//arrwin[essk].Lock(MPI::LOCK_SHARED, ownind, 0);
MPI_Get(arrinfo.numarrs[i].addr, arrinfo.numarrs[i].count, MPIType<NT>(), ownind, 0, arrinfo.numarrs[i].count, MPIType<NT>(), arrwin[essk++]);
}
}
