int MPIX_Neighbor_alltoall_x(const void *sendbuf, MPI_Count sendcount, MPI_Datatype sendtype,
void *recvbuf, MPI_Count recvcount, MPI_Datatype recvtype,
MPI_Comm comm)
{
int rc = MPI_SUCCESS;
if (likely (sendcount <= bigmpi_int_max && recvcount <= bigmpi_int_max )) {
rc = MPI_Neighbor_alltoall(sendbuf, (int)sendcount, sendtype, recvbuf, (int)recvcount, recvtype, comm);
} else if (sendcount > bigmpi_int_max && recvcount <= bigmpi_int_max ) {
MPI_Datatype newsendtype;
BigMPI_Type_contiguous(0,sendcount, sendtype, &newsendtype);
MPI_Type_commit(&newsendtype);
rc = MPI_Neighbor_alltoall(sendbuf, 1, newsendtype, recvbuf, (int)recvcount, recvtype, comm);
MPI_Type_free(&newsendtype);
} else if (sendcount <= bigmpi_int_max && recvcount > bigmpi_int_max ) {
MPI_Datatype newrecvtype;
BigMPI_Type_contiguous(0,recvcount, recvtype, &newrecvtype);
MPI_Type_commit(&newrecvtype);
rc = MPI_Neighbor_alltoall(sendbuf, (int)sendcount, sendtype, recvbuf, 1, newrecvtype, comm);
MPI_Type_free(&newrecvtype);
} else {
MPI_Datatype newsendtype, newrecvtype;
BigMPI_Type_contiguous(0,sendcount, sendtype, &newsendtype);
BigMPI_Type_contiguous(0,recvcount, recvtype, &newrecvtype);
MPI_Type_commit(&newsendtype);
MPI_Type_commit(&newrecvtype);
rc = MPI_Neighbor_alltoall(sendbuf, 1, newsendtype, recvbuf, 1, newrecvtype, comm);
MPI_Type_free(&newsendtype);
MPI_Type_free(&newrecvtype);
}
return rc;
}
static int Neighbor_alltoall(HostRead<I32> sources, HostRead<I32> destinations,
const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf,
int recvcount, MPI_Datatype recvtype, MPI_Comm comm) {
#if MPI_VERSION < 3
static int const tag = 42;
int indegree, outdegree;
indegree = sources.size();
outdegree = destinations.size();
int sendwidth;
CALL(MPI_Type_size(sendtype, &sendwidth));
int recvwidth;
CALL(MPI_Type_size(sendtype, &recvwidth));
MPI_Request* recvreqs = new MPI_Request[indegree];
MPI_Request* sendreqs = new MPI_Request[outdegree];
for (int i = 0; i < indegree; ++i)
CALL(MPI_Irecv(static_cast<char*>(recvbuf) + i * recvwidth, recvcount,
recvtype, sources[i], tag, comm, recvreqs + i));
CALL(MPI_Barrier(comm));
for (int i = 0; i < outdegree; ++i)
CALL(MPI_Isend(static_cast<char const*>(sendbuf) + i * sendwidth, sendcount,
sendtype, destinations[i], tag, comm, sendreqs + i));
CALL(MPI_Waitall(outdegree, sendreqs, MPI_STATUSES_IGNORE));
delete[] sendreqs;
CALL(MPI_Waitall(indegree, recvreqs, MPI_STATUSES_IGNORE));
delete[] recvreqs;
return MPI_SUCCESS;
#else
(void)sources;
(void)destinations;
return MPI_Neighbor_alltoall(
sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm);
#endif // end if MPI_VERSION < 3
}
void Halo::Halo_Init(matrix<long int>& A)
{
//post Irecv
//TODO: replace totalRecv with numNeighbors
requests = new MPI_Request[totalRecv];
status = new MPI_Status[totalRecv];
MPI_Neighbor_alltoall(elementsToSend, sendLength[0], MPI_LONG, elementsToRecv, recvLength[0], MPI_LONG, cart);
int recvIndex = 0;
for(int i=1; i<nix-1; i++, recvIndex++)
{
//send buffer
A[0][i] = elementsToRecv[recvIndex];
//A[0][i] = 96;
}
//totalSend scales with number of processes
for(int i=1; i<nix-1; i++, recvIndex++)
{
A[i][0] = elementsToRecv[recvIndex];
//A[i][0] = 69;
}
for(int i=1; i<niy-1; i++, recvIndex++)
{
A[local_rows+1][i] = elementsToRecv[recvIndex];
//A[local_rows+1][i] = 42;
}
for(int i=1; i<niy-1; i++, recvIndex++)
{
A[i][local_cols+1] = elementsToRecv[recvIndex];
//A[i][local_rows+1] = 21;
}
delete [] status;
delete [] requests;
}
int main(int argc, char *argv[])
{
int errs = 0;
int wrank, wsize;
int periods[1] = { 0 };
MPI_Comm cart, dgraph, graph;
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &wrank);
MPI_Comm_size(MPI_COMM_WORLD, &wsize);
#if defined(TEST_NEIGHB_COLL)
/* a basic test for the 10 (5 patterns x {blocking,nonblocking}) MPI-3
* neighborhood collective routines */
/* (wrap)--> 0 <--> 1 <--> ... <--> p-1 <--(wrap) */
MPI_Cart_create(MPI_COMM_WORLD, 1, &wsize, periods, /*reorder=*/0, &cart);
/* allgather */
{
int sendbuf[1] = { wrank };
int recvbuf[2] = { 0xdeadbeef, 0xdeadbeef };
/* should see one send to each neighbor (rank-1 and rank+1) and one receive
* each from same */
MPI_Neighbor_allgather(sendbuf, 1, MPI_INT, recvbuf, 1, MPI_INT, cart);
if (wrank == 0)
check(recvbuf[0] == 0xdeadbeef);
else
check(recvbuf[0] == wrank - 1);
if (wrank == wsize - 1)
check(recvbuf[1] == 0xdeadbeef);
else
check(recvbuf[1] == wrank + 1);
}
/* allgatherv */
{
int sendbuf[1] = { wrank };
int recvbuf[2] = { 0xdeadbeef, 0xdeadbeef };
int recvcounts[2] = { 1, 1 };
int displs[2] = { 1, 0};
/* should see one send to each neighbor (rank-1 and rank+1) and one receive
* each from same, but put them in opposite slots in the buffer */
MPI_Neighbor_allgatherv(sendbuf, 1, MPI_INT, recvbuf, recvcounts, displs, MPI_INT, cart);
if (wrank == 0)
check(recvbuf[1] == 0xdeadbeef);
else
check(recvbuf[1] == wrank - 1);
if (wrank == wsize - 1)
check(recvbuf[0] == 0xdeadbeef);
else
check(recvbuf[0] == wrank + 1);
}
/* alltoall */
{
int sendbuf[2] = { -(wrank+1), wrank+1 };
int recvbuf[2] = { 0xdeadbeef, 0xdeadbeef };
/* should see one send to each neighbor (rank-1 and rank+1) and one
* receive each from same */
MPI_Neighbor_alltoall(sendbuf, 1, MPI_INT, recvbuf, 1, MPI_INT, cart);
if (wrank == 0)
check(recvbuf[0] == 0xdeadbeef);
else
check(recvbuf[0] == wrank);
if (wrank == wsize - 1)
check(recvbuf[1] == 0xdeadbeef);
else
check(recvbuf[1] == -(wrank + 2));
}
/* alltoallv */
{
int sendbuf[2] = { -(wrank+1), wrank+1 };
int recvbuf[2] = { 0xdeadbeef, 0xdeadbeef };
int sendcounts[2] = { 1, 1 };
int recvcounts[2] = { 1, 1 };
int sdispls[2] = { 0, 1 };
int rdispls[2] = { 1, 0 };
/* should see one send to each neighbor (rank-1 and rank+1) and one receive
* each from same, but put them in opposite slots in the buffer */
MPI_Neighbor_alltoallv(sendbuf, sendcounts, sdispls, MPI_INT,
recvbuf, recvcounts, rdispls, MPI_INT,
cart);
if (wrank == 0)
check(recvbuf[1] == 0xdeadbeef);
else
check(recvbuf[1] == wrank);
if (wrank == wsize - 1)
check(recvbuf[0] == 0xdeadbeef);
else
check(recvbuf[0] == -(wrank + 2));
}
/* alltoallw */
{
int sendbuf[2] = { -(wrank+1), wrank+1 };
int recvbuf[2] = { 0xdeadbeef, 0xdeadbeef };
int sendcounts[2] = { 1, 1 };
int recvcounts[2] = { 1, 1 };
MPI_Aint sdispls[2] = { 0, sizeof(int) };
MPI_Aint rdispls[2] = { sizeof(int), 0 };
MPI_Datatype sendtypes[2] = { MPI_INT, MPI_INT };
MPI_Datatype recvtypes[2] = { MPI_INT, MPI_INT };
/* should see one send to each neighbor (rank-1 and rank+1) and one receive
* each from same, but put them in opposite slots in the buffer */
MPI_Neighbor_alltoallw(sendbuf, sendcounts, sdispls, sendtypes,
recvbuf, recvcounts, rdispls, recvtypes,
cart);
if (wrank == 0)
check(recvbuf[1] == 0xdeadbeef);
else
check(recvbuf[1] == wrank);
if (wrank == wsize - 1)
check(recvbuf[0] == 0xdeadbeef);
else
check(recvbuf[0] == -(wrank + 2));
}
MPI_Comm_free(&cart);
#endif /* defined(TEST_NEIGHB_COLL) */
MPI_Reduce((wrank == 0 ? MPI_IN_PLACE : &errs), &errs, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD);
if (wrank == 0) {
if (errs) {
printf("found %d errors\n", errs);
}
else {
printf(" No errors\n");
}
}
MPI_Finalize();
return 0;
}