int main( int argc, char *argv[] )
{
int errs = 0;
int topo_type, size, dims[1], periods[1];
MPI_Comm comm;
MTest_Init( &argc, &argv );
/* Check that topo test returns the correct type, including
MPI_UNDEFINED */
MPI_Topo_test( MPI_COMM_WORLD, &topo_type );
if (topo_type != MPI_UNDEFINED) {
errs++;
printf( "Topo type of comm world is not UNDEFINED\n" );
}
MPI_Comm_size( MPI_COMM_WORLD, &size );
dims[0] = size;
periods[0] = 0;
MPI_Cart_create( MPI_COMM_WORLD, 1, dims, periods, 0, &comm );
MPI_Topo_test( comm, &topo_type );
if (topo_type != MPI_CART) {
errs++;
printf( "Topo type of cart comm is not CART\n" );
}
MPI_Comm_free( &comm );
/* FIXME: still need graph example */
MTest_Finalize( errs );
MPI_Finalize();
return 0;
}
static MPI_Comm cart_comm_1d(MPI_Comm communicator) {
/* test whether the communicator is cartesian and correct dimensionality */
fcs_int status;
MPI_Topo_test(communicator, &status);
if (status == MPI_CART) {
/* Communicator is cartesian, so test dimensionality */
fcs_int ndims;
MPI_Cartdim_get(communicator, &ndims);
if (ndims == 3) {
/* Correct dimensionality, so get grid and test periodicity */
fcs_int node_grid[3], periodicity[3], node_pos[3];
MPI_Cart_get(communicator, 3, node_grid, periodicity, node_pos);
if (!periodicity[0] && !periodicity[1] && periodicity[2]) {
return communicator;
}
}
}
/* Otherwise, we have to set up the cartesian communicator */
fcs_int comm_size;
MPI_Comm_size(communicator, &comm_size);
fcs_int node_grid[3] = {0, 0, 0};
MPI_Dims_create(comm_size, 3, node_grid);
fcs_int periodicity[3] = {1, 1, 0};
MPI_Comm new_communicator;
MPI_Cart_create(communicator, 3, node_grid, periodicity, 1, &new_communicator);
return new_communicator;
}
int PX(is_cart_procmesh)(
MPI_Comm comm_cart
)
{
int status;
MPI_Topo_test(comm_cart, &status);
return (status == MPI_CART);
}
int PX(is_cart_procmesh_2d)(
MPI_Comm comm_cart_2d
)
{
int status, ndims;
MPI_Topo_test(comm_cart_2d, &status);
MPI_Cartdim_get(comm_cart_2d, &ndims);
return ( (status == MPI_CART) && (ndims == 2) );
}
int NBC_Comm_neighbors(MPI_Comm comm, int maxindegree, int sources[], int sourceweights[], int maxoutdegree, int destinations[], int destweights[]) {
int topo, res;
int index = 0;
int indeg, outdeg, wgtd;
res = NBC_Comm_neighbors_count(comm, &indeg, &outdeg, &wgtd);
if(indeg > maxindegree && outdeg > maxoutdegree) return NBC_INVALID_PARAM; /* we want to return *all* neighbors */
res = MPI_Topo_test(comm, &topo);
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Topo_test() (%i)\n", res); return res; }
switch(topo) {
case MPI_CART: /* cartesian */
{
int ndims, i, rpeer, speer;
res = MPI_Cartdim_get(comm, &ndims);
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Cartdim_get() (%i)\n", res); return res; }
for(i = 0; i<ndims; i++) {
res = MPI_Cart_shift(comm, i, 1, &rpeer, &speer);
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Cart_shift() (%i)\n", res); return res; }
sources[index] = destinations[index] = rpeer; index++;
sources[index] = destinations[index] = speer; index++;
}
}
break;
case MPI_GRAPH: /* graph */
{
int rank;
MPI_Comm_rank(comm, &rank);
res = MPI_Graph_neighbors(comm, rank, maxindegree, sources);
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Graph_neighbors_count() (%i)\n", res); return res; }
for(int i=0; i<maxindegree; i++) destinations[i] = sources[i];
}
break;
case MPI_DIST_GRAPH: /* dist graph */
{
res = MPI_Dist_graph_neighbors(comm, maxindegree, sources, sourceweights, maxoutdegree, destinations, destweights);
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Graph_neighbors_count() (%i)\n", res); return res; }
}
break;
case MPI_UNDEFINED:
return NBC_INVALID_TOPOLOGY_COMM;
break;
default:
return NBC_INVALID_PARAM;
break;
}
return NBC_OK;
}
void
mmm2d_comm_init(mmm2d_comm_struct *comm, MPI_Comm communicator) {
/* store the original communicator */
comm->mpicomm_orig = communicator;
MPI_Comm_size(communicator, &comm->size);
/* Test whether the communicator is cartesian and correct dimensionality */
fcs_int comm_is_cart = 0;
fcs_int status;
MPI_Topo_test(communicator, &status);
if (status == MPI_CART) {
/* Communicator is cartesian, so test dimensionality */
fcs_int ndims;
MPI_Cartdim_get(communicator, &ndims);
if (ndims == 3) {
/* Correct dimensionality, so get grid and test periodicity */
fcs_int periodicity[3];
MPI_Cart_get(communicator, 3, comm->node_grid, periodicity, comm->node_pos);
if (periodicity[0] && periodicity[1] && periodicity[2]) {
/* If periodicity is correct, we can just use this communicator */
comm->mpicomm = communicator;
/* get the rank */
MPI_Comm_rank(communicator, &comm->rank);
comm_is_cart = 1;
}
}
}
/* otherwise, we have to set up the cartesian communicator */
if (!comm_is_cart) {
comm->node_grid[0] = 0.0;
comm->node_grid[1] = 0.0;
comm->node_grid[2] = 0.0;
/* compute node grid */
MPI_Dims_create(comm->size, 3, comm->node_grid);
/* create communicator */
fcs_int periodicity[3] = {1, 1, 0};
MPI_Cart_create(comm->mpicomm_orig, 3, comm->node_grid, periodicity, 1, &comm->mpicomm);
/* get the rank */
MPI_Comm_rank(communicator, &comm->rank);
/* get node pos */
MPI_Cart_coords(comm->mpicomm, comm->rank, 3, comm->node_pos);
}
}
static int comm_is_cart_3d(
MPI_Comm comm
)
{
int ndims, status;
MPI_Topo_test(comm, &status);
if (status != MPI_CART)
return 0;
MPI_Cartdim_get(comm, &ndims);
if (ndims != 3)
return 0;
return 1;
}
Comm::Comm(MPI_Comm impl) : impl_(impl) {
int topo_type;
CALL(MPI_Topo_test(impl, &topo_type));
if (topo_type == MPI_DIST_GRAPH) {
int nin, nout, is_weighted;
CALL(MPI_Dist_graph_neighbors_count(impl, &nin, &nout, &is_weighted));
HostWrite<I32> sources(nin);
HostWrite<I32> destinations(nout);
CALL(MPI_Dist_graph_neighbors(impl, nin, sources.data(),
OMEGA_H_MPI_UNWEIGHTED, nout, destinations.data(),
OMEGA_H_MPI_UNWEIGHTED));
srcs_ = sources.write();
dsts_ = destinations.write();
host_srcs_ = HostRead<I32>(srcs_);
host_dsts_ = HostRead<I32>(dsts_);
}
}
int NBC_Comm_neighbors_count(MPI_Comm comm, int *indegree, int *outdegree, int *weighted) {
int topo, res;
res = MPI_Topo_test(comm, &topo);
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Topo_test() (%i)\n", res); return res; }
switch(topo) {
case MPI_CART: /* cartesian */
{
int ndims;
res = MPI_Cartdim_get(comm, &ndims) ;
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Cartdim_get() (%i)\n", res); return res; }
/* outdegree is always 2*ndims because we need to iterate over empty buffers for MPI_PROC_NULL */
*outdegree = *indegree = 2*ndims;
*weighted = 0;
}
break;
case MPI_GRAPH: /* graph */
{
int rank, nneighbors;
MPI_Comm_rank(comm, &rank);
res = MPI_Graph_neighbors_count(comm, rank, &nneighbors);
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Graph_neighbors_count() (%i)\n", res); return res; }
*outdegree = *indegree = nneighbors;
*weighted = 0;
}
break;
case MPI_DIST_GRAPH: /* graph */
{
res = MPI_Dist_graph_neighbors_count(comm, indegree, outdegree, weighted);
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Dist_graph_neighbors_count() (%i)\n", res); return res; }
}
break;
case MPI_UNDEFINED:
return NBC_INVALID_TOPOLOGY_COMM;
break;
default:
return NBC_INVALID_PARAM;
break;
}
return NBC_OK;
}
static int create_proclists_from_mpi_comm_topology(MPI_Comm comm, int rank, int *nsend_procs, int **send_procs, int *nrecv_procs, int **recv_procs)
{
int status;
int n, i;
MPI_Topo_test(comm, &status);
switch (status)
{
case MPI_CART:
MPI_Cartdim_get(comm, &n);
*nsend_procs = *nrecv_procs = 2 * n;
*send_procs = *recv_procs = z_alloc(2 * n, sizeof(int));
for (i = 0; i < n; ++i) MPI_Cart_shift(comm, i, -1, &(*send_procs)[2 * i + 1], &(*send_procs)[2 * i + 0]);
break;
case MPI_GRAPH:
MPI_Graph_neighbors_count(comm, rank, &n);
*nsend_procs = *nrecv_procs = n;
*send_procs = *recv_procs = z_alloc(n, sizeof(int));
MPI_Graph_neighbors(comm, rank, n, *send_procs);
break;
#if MPI_VERSION >= 2 && MPI_SUBVERSION >= 2
case MPI_DIST_GRAPH:
MPI_Dist_graph_neighbors_count(comm, nrecv_procs, nsend_procs, &n);
*send_procs = z_alloc(*nsend_procs + *nrecv_procs, sizeof(int));
*recv_procs = *send_procs + *nsend_procs;
MPI_Dist_graph_neighbors(comm, *nrecv_procs, *recv_procs, MPI_UNWEIGHTED, *nsend_procs, *send_procs, MPI_UNWEIGHTED);
break;
#endif
default:
return 0;
}
return 1;
}
void fcs_memd_setup_communicator(memd_struct* memd, MPI_Comm communicator)
{
/* store given communicator */
memd->mpiparams.original_comm = communicator;
MPI_Comm_size(communicator, &memd->mpiparams.size);
/* Test whether the communicator is cartesian and correct dimensionality */
int comm_is_cart = 0;
int status;
MPI_Topo_test(communicator, &status);
if (status == MPI_CART) {
/* Communicator is cartesian, so test dimensionality */
int ndims;
MPI_Cartdim_get(communicator, &ndims);
if (ndims == 3) {
/* Correct dimensionality, so get grid and test periodicity */
int periodicity[3];
MPI_Cart_get(communicator, 3, memd->mpiparams.node_grid, periodicity, memd->mpiparams.node_pos);
if (periodicity[0] && periodicity[1] && periodicity[2]) {
/* If periodicity is correct, we can just use this communicator */
memd->mpiparams.communicator = communicator;
/* get the rank */
MPI_Comm_rank(communicator, &memd->mpiparams.this_node);
comm_is_cart = 1;
}
}
}
/* otherwise, we have to set up the cartesian communicator */
if (!comm_is_cart) {
memd->mpiparams.node_grid[0] = 0.0;
memd->mpiparams.node_grid[1] = 0.0;
memd->mpiparams.node_grid[2] = 0.0;
/* compute node grid */
MPI_Dims_create(memd->mpiparams.size, 3, memd->mpiparams.node_grid);
/* swap first and last dimension, as MEMD currently wants to have them increasing */
fcs_int tmp = memd->mpiparams.node_grid[2];
memd->mpiparams.node_grid[2] = memd->mpiparams.node_grid[0];
memd->mpiparams.node_grid[0] = tmp;
/* create communicator */
int periodicity[3] = {1, 1, 1};
MPI_Cart_create(memd->mpiparams.original_comm, 3, memd->mpiparams.node_grid, periodicity, 1, &memd->mpiparams.communicator);
/* get the rank */
MPI_Comm_rank(communicator, &memd->mpiparams.this_node);
/* get node pos */
MPI_Cart_coords(memd->mpiparams.communicator, memd->mpiparams.this_node, 3, memd->mpiparams.node_pos);
}
/* fetch neighborhood info */
for (int dir = 0; dir<3; dir++) {
MPI_Cart_shift(memd->mpiparams.communicator, dir, 1,
&memd->mpiparams.node_neighbors[2*dir],
&memd->mpiparams.node_neighbors[2*dir+1]);
}
/* init local points */
for (int i=0; i< 3; i++) {
// memd->mpiparams.local_box_l[i] = 0.0;
memd->mpiparams.my_left[i] = 0.0;
memd->mpiparams.my_right[i] = 0.0;
}
/* compute box limits */
fcs_float local_box_length = 0.0;
for(fcs_int i = 0; i < 3; i++) {
local_box_length = memd->parameters.box_length[i] / (fcs_float)memd->mpiparams.node_grid[i];
memd->mpiparams.my_left[i] = memd->mpiparams.node_pos[i] * local_box_length;
memd->mpiparams.my_right[i] = (memd->mpiparams.node_pos[i]+1) * local_box_length;
}
}
void mpi_manager_3D::setup(NumArray<int> &nproc, NumArray<int> &mx) {
// Save number of processors in each dimension
for(int dir=0; dir<DIM; ++dir) {
this->nproc[dir] = nproc[dir];
}
// Determine the rank of the current task
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
// Get number of ranks from MPI
int ntasks;
MPI_Comm_size(MPI_COMM_WORLD, &ntasks);
this->ntasks = ntasks;
// Set the distribution of processes:
if(ntasks != nproc[0]*nproc[1]*nproc[2]){
std::cerr << " Wrong number of processes " << std::endl;
std::cout << ntasks << " " << nproc[0]*nproc[1]*nproc[2] << std::endl;
Finalise();
}
if(rank==0) {
std::cout << " Number of tasks: " << ntasks << std::endl;
}
// Check if grid can be subdevided as desired
for(int dir = 0; dir < DIM; ++dir) {
if(mx[dir] < nproc[dir] && nproc[dir] > 1) {
if(rank == 0) {
std::cerr << " Wrong grid topology for dimension ";
std::cerr << dir << std::endl;
std::cerr << " mx[" << dir << "]:" << mx[dir] << std::endl;
std::cerr << " nproc[" << dir << "]:" << nproc[dir] << std::endl;
}
Finalise();
}
}
// Check if grid is a power of 2:
double eps = 1.e-12;
for(int dir = 0; dir < DIM; ++dir) {
double exponent = log(mx[dir])/log(2.);
int i_exponent = static_cast<int>(exponent+eps);
if(exponent - i_exponent > 2.*eps) {
if(rank == 0) {
std::cerr << " Error: grid must be of the form mx = 2^n ";
std::cerr << std::endl;
std::cerr << " Exiting " << std::endl;
}
Finalise();
}
}
// Grid is not periodic
int periods[3] = {false, false, false};
int reorder = false;
// If all is okay: Create new communicator "comm3d"
MPI_Cart_create(MPI_COMM_WORLD, DIM, nproc, periods, reorder, &comm3d);
// Retrieve the cartesian topology
if (rank == 0) {
int TopoType;
std::cout << " Cart topology: ";
MPI_Topo_test(comm3d, &TopoType);
switch (TopoType) {
case MPI_UNDEFINED :
std::cout << " MPI_UNDEFINED " << std::endl;
break;
case MPI_GRAPH :
std::cout << "MPI_GRAPH" << std::endl;
break;
case MPI_CART :
std::cout << "MPI_CART" << std::endl;
break;
}
}
// Determine rank again for cartesian communicator -> overwrite rank
MPI_Comm_rank(comm3d, &rank);
// std::cout << " my rank: " << rank << std::endl;
// Translate rank to coordinates
MPI_Cart_coords(comm3d, rank, DIM, coords);
// // Backwards translation
// int TranslateRank;
// MPI_Cart_rank(comm3d, coords, &TranslateRank);
// Find neighbouring ranks
// Syntax: comm3d, shift direction, displacement, source, destination
MPI_Cart_shift(comm3d, 0, 1, &left , &right);
MPI_Cart_shift(comm3d, 1, 1, &front, &back);
MPI_Cart_shift(comm3d, 2, 1, &bottom, &top);
// std::cout << " My rank " << rank << " " << left << " " << right << " " << front << " " << back << " " << bottom << " " << top << std::endl;
if(rank==0) {
std::cout << " nearby " << right << " " << back << " " << top << std::endl;
}
// Determine ranks of neighbour processes:
int shiftcoord[DIM];
int lbound[DIM],ubound[DIM];
for(int dim=0;dim<DIM;dim++){
lbound[dim]=-1;
ubound[dim]= 1;
}
Neighbour.resize(lbound,ubound);
Neighbour.clear();
for(int dim0=-1; dim0<=1; dim0++){
shiftcoord[0] = (coords[0]+dim0)%nproc[0];
if(shiftcoord[0] < 0) shiftcoord[0]+=nproc[0];
for(int dim1=-1; dim1<=1; dim1++){
shiftcoord[1] = (coords[1]+dim1)%nproc[1];
if(shiftcoord[1] < 0) shiftcoord[1]+=nproc[1];
for(int dim2=-1; dim2<=1; dim2++){
shiftcoord[2] = (coords[2]+dim2)%nproc[2];
if(shiftcoord[2] < 0) shiftcoord[2]+=nproc[2];
MPI_Cart_rank(comm3d, shiftcoord,&Neighbour(dim0,dim1,dim2));
}
}
}
// if(rank==1) {
// for(int dim0=-1; dim0<=1; dim0++){
// for(int dim1=-1; dim1<=1; dim1++){
// for(int dim2=-1; dim2<=1; dim2++){
// std::cout << " neighbour " << dim0 << " " << dim1 << " ";
// std::cout << dim2 << " " << Neighbour(dim0, dim1, dim2);
// std::cout << std::endl;
// }
// }
// }
// }
// Determine absolute position of any rank:
AllRanks.resize(Index::set(0,0,0),
Index::set(nproc[0]-1,nproc[1]-1,nproc[2]-1));
for(int dim0=0; dim0<nproc[0]; ++dim0) {
for(int dim1=0; dim1<nproc[1]; ++dim1) {
for(int dim2=0; dim2<nproc[2]; ++dim2) {
int coord[3] = {dim0, dim1, dim2};
MPI_Cart_rank(comm3d, coord, &AllRanks(dim0, dim1, dim2));
}
}
}
// if(rank==2) {
// std::cout << " Neigh: " << rank << " "<<Neighbour(0,0,0) << " " << AllRanks(2,0,0) << std::endl;
// }
// Now make additional mpi groups relating to planes:
int count(0);
int num_xy = nproc[0]*nproc[1];
int num_xz = nproc[0]*nproc[2];
int num_yz = nproc[1]*nproc[2];
NumMatrix<int,1> x_ranks[nproc[0]];
NumMatrix<int,1> y_ranks[nproc[1]];
NumMatrix<int,1> z_ranks[nproc[2]];
// Walk trough z-axis -- xy plane
for(int irz=0; irz<nproc[2]; irz++) {
count = 0;
z_ranks[irz].resize(Index::set(0), Index::set(num_xy));
for(int irx=0; irx<nproc[0]; irx++) {
for(int iry=0; iry<nproc[1]; iry++) {
z_ranks[irz](count) = AllRanks(irx,iry,irz);
count++;
}
}
}
// Walk trough y-axis -- xz plane
for(int iry=0; iry<nproc[1]; iry++) {
count = 0;
y_ranks[iry].resize(Index::set(0), Index::set(num_xz));
for(int irx=0; irx<nproc[0]; irx++) {
for(int irz=0; irz<nproc[2]; irz++) {
y_ranks[iry](count) = AllRanks(irx,iry,irz);
count++;
}
}
}
// Walk trough x-axis -- yz plane
for(int irx=0; irx<nproc[0]; irx++) {
count = 0;
x_ranks[irx].resize(Index::set(0), Index::set(num_yz));
for(int iry=0; iry<nproc[1]; iry++) {
for(int irz=0; irz<nproc[2]; irz++) {
x_ranks[irx](count) = AllRanks(irx,iry,irz);
count++;
}
}
}
// Build local communicator:
MPI_Group group_all, group_constz, group_consty, group_constx;
// Get standard group handle:
MPI_Comm_group(comm3d, &group_all);
// Devide tasks into groups based on z-position
MPI_Group_incl(group_all, num_xy, z_ranks[coords[2]], &group_constz);
// Devide tasks into groups based on z-position
MPI_Group_incl(group_all, num_xz, y_ranks[coords[1]], &group_consty);
// Devide tasks into groups based on x-position
MPI_Group_incl(group_all, num_yz, x_ranks[coords[0]], &group_constx);
// // Make corresponding communicators:
// MPI_Comm_create(comm3d, group_constz, &comm_plane_xy); // const z
// MPI_Comm_create(comm3d, group_consty, &comm_plane_xz); // const x
// MPI_Comm_create(comm3d, group_constx, &comm_plane_yz); // const x
// // Get corresponding rank
// MPI_Group_rank (group_constz, &rank_plane_xy);
// MPI_Group_rank (group_consty, &rank_plane_xz);
// MPI_Group_rank (group_constx, &rank_plane_yz);
int remain_dims[3];
// x-y plane:
remain_dims[0] = 1;
remain_dims[1] = 1;
remain_dims[2] = 0;
MPI_Cart_sub(comm3d, remain_dims, &comm_plane_xy);
MPI_Comm_rank(comm_plane_xy, &rank_plane_xy);
// x-z plane
remain_dims[0] = 1;
remain_dims[1] = 0;
remain_dims[2] = 1;
MPI_Cart_sub(comm3d, remain_dims, &comm_plane_xz);
MPI_Comm_rank(comm_plane_xz, &rank_plane_xz);
// y-z plane
remain_dims[0] = 0;
remain_dims[1] = 1;
remain_dims[2] = 1;
MPI_Cart_sub(comm3d, remain_dims, &comm_plane_yz);
MPI_Comm_rank(comm_plane_yz, &rank_plane_yz);
}
int main( int argc, char **argv )
{
int rank, size, i;
int errors=0;
int dims[NUM_DIMS];
int periods[NUM_DIMS];
int coords[NUM_DIMS];
int new_coords[NUM_DIMS];
int reorder = 1;
MPI_Comm comm_temp, comm_cart, new_comm;
int topo_status;
int ndims;
int new_rank;
int remain_dims[NUM_DIMS];
int newnewrank;
MPI_Init( &argc, &argv );
MPI_Comm_rank( MPI_COMM_WORLD, &rank );
MPI_Comm_size( MPI_COMM_WORLD, &size );
/* Clear dims array and get dims for topology */
for(i=0;i<NUM_DIMS;i++) { dims[i] = 0; periods[i] = 0; }
MPI_Dims_create ( size, NUM_DIMS, dims );
/* Make a new communicator with a topology */
MPI_Cart_create ( MPI_COMM_WORLD, 2, dims, periods, reorder, &comm_temp );
MPI_Comm_dup ( comm_temp, &comm_cart );
/* Determine the status of the new communicator */
MPI_Topo_test ( comm_cart, &topo_status );
if (topo_status != MPI_CART) {
printf( "topo_status of duped comm is not MPI_CART\n" );
errors++;
}
/* How many dims do we have? */
MPI_Cartdim_get( comm_cart, &ndims );
if ( ndims != NUM_DIMS ) {
printf( "Number of dims of duped comm (%d) should be %d\n",
ndims, NUM_DIMS );
errors++;
}
/* Get the topology, does it agree with what we put in? */
for(i=0;i<NUM_DIMS;i++) { dims[i] = 0; periods[i] = 0; }
MPI_Cart_get ( comm_cart, NUM_DIMS, dims, periods, coords );
/* Does the mapping from coords to rank work? */
MPI_Cart_rank ( comm_cart, coords, &new_rank );
if ( new_rank != rank ) {
printf( "New rank of duped comm (%d) != old rank (%d)\n",
new_rank, rank );
errors++;
}
/* Does the mapping from rank to coords work */
MPI_Cart_coords ( comm_cart, rank, NUM_DIMS, new_coords );
for (i=0;i<NUM_DIMS;i++)
if ( coords[i] != new_coords[i] ) {
printf( "Old coords[%d] of duped comm (%d) != new_coords (%d)\n",
i, coords[i], new_coords[i] );
errors++;
}
/* Let's shift in each dimension and see how it works! */
/* Because it's late and I'm tired, I'm not making this */
/* automatically test itself. */
for (i=0;i<NUM_DIMS;i++) {
int source, dest;
MPI_Cart_shift(comm_cart, i, 1, &source, &dest);
#ifdef VERBOSE
printf ("[%d] Shifting %d in the %d dimension\n",rank,1,i);
printf ("[%d] source = %d dest = %d\n",rank,source,dest);
#endif
}
/* Subdivide */
remain_dims[0] = 0;
for (i=1; i<NUM_DIMS; i++) remain_dims[i] = 1;
MPI_Cart_sub ( comm_cart, remain_dims, &new_comm );
/* Determine the status of the new communicator */
MPI_Topo_test ( new_comm, &topo_status );
if (topo_status != MPI_CART) {
printf( "topo_status of cartsub comm is not MPI_CART\n" );
errors++;
}
/* How many dims do we have? */
MPI_Cartdim_get( new_comm, &ndims );
if ( ndims != NUM_DIMS-1 ) {
printf( "Number of dims of cartsub comm (%d) should be %d\n",
ndims, NUM_DIMS-1 );
errors++;
}
/* Get the topology, does it agree with what we put in? */
for(i=0;i<NUM_DIMS-1;i++) { dims[i] = 0; periods[i] = 0; }
MPI_Cart_get ( new_comm, ndims, dims, periods, coords );
/* Does the mapping from coords to rank work? */
MPI_Comm_rank ( new_comm, &newnewrank );
MPI_Cart_rank ( new_comm, coords, &new_rank );
if ( new_rank != newnewrank ) {
printf( "New rank of cartsub comm (%d) != old rank (%d)\n",
new_rank, newnewrank );
errors++;
}
/* Does the mapping from rank to coords work */
MPI_Cart_coords ( new_comm, new_rank, NUM_DIMS -1, new_coords );
for (i=0;i<NUM_DIMS-1;i++)
if ( coords[i] != new_coords[i] ) {
printf( "Old coords[%d] of cartsub comm (%d) != new_coords (%d)\n",
i, coords[i], new_coords[i] );
errors++;
}
/* We're at the end */
MPI_Comm_free( &new_comm );
MPI_Comm_free( &comm_temp );
MPI_Comm_free( &comm_cart );
Test_Waitforall( );
if (errors) printf( "[%d] done with %d ERRORS!\n", rank,errors );
MPI_Finalize();
return 0;
}
static int verify_comm(MPI_Comm comm)
{
int local_errs = 0;
int i, j;
int indegree, outdegree, weighted;
int *sources, *sweights, *destinations, *dweights;
int use_dup;
int topo_type = MPI_UNDEFINED;
MPI_Comm dupcomm = MPI_COMM_NULL;
sources = (int *) malloc(size * sizeof(int));
sweights = (int *) malloc(size * sizeof(int));
destinations = (int *) malloc(size * sizeof(int));
dweights = (int *) malloc(size * sizeof(int));
for (use_dup = 0; use_dup <= 1; ++use_dup) {
if (!use_dup) {
MPI_Dist_graph_neighbors_count(comm, &indegree, &outdegree, &weighted);
}
else {
MPI_Comm_dup(comm, &dupcomm);
comm = dupcomm; /* caller retains original comm value */
}
MPI_Topo_test(comm, &topo_type);
if (topo_type != MPI_DIST_GRAPH) {
fprintf(stderr, "topo_type != MPI_DIST_GRAPH\n");
++local_errs;
}
j = 0;
for (i = 0; i < size; i++)
if (layout[i][rank])
j++;
if (j != indegree) {
fprintf(stderr, "indegree does not match, expected=%d got=%d, layout='%s'\n", indegree, j, graph_layout_name);
++local_errs;
}
j = 0;
for (i = 0; i < size; i++)
if (layout[rank][i])
j++;
if (j != outdegree) {
fprintf(stderr, "outdegree does not match, expected=%d got=%d, layout='%s'\n", outdegree, j, graph_layout_name);
++local_errs;
}
if ((indegree || outdegree) && (weighted == 0)) {
fprintf(stderr, "MPI_Dist_graph_neighbors_count thinks the graph is not weighted\n");
++local_errs;
}
MPI_Dist_graph_neighbors(comm, indegree, sources, sweights, outdegree, destinations, dweights);
/* For each incoming and outgoing edge in the matrix, search if
* the query function listed it in the sources. */
for (i = 0; i < size; i++) {
if (layout[i][rank]) {
for (j = 0; j < indegree; j++) {
assert(sources[j] >= 0);
assert(sources[j] < size);
if (sources[j] == i)
break;
}
if (j == indegree) {
fprintf(stderr, "no edge from %d to %d specified\n", i, rank);
++local_errs;
}
else {
if (sweights[j] != layout[i][rank]) {
fprintf(stderr, "incorrect weight for edge (%d,%d): %d instead of %d\n",
i, rank, sweights[j], layout[i][rank]);
++local_errs;
}
}
}
if (layout[rank][i]) {
for (j = 0; j < outdegree; j++) {
assert(destinations[j] >= 0);
assert(destinations[j] < size);
if (destinations[j] == i)
break;
}
if (j == outdegree) {
fprintf(stderr, "no edge from %d to %d specified\n", rank, i);
++local_errs;
}
else {
if (dweights[j] != layout[rank][i]) {
fprintf(stderr, "incorrect weight for edge (%d,%d): %d instead of %d\n",
rank, i, dweights[j], layout[rank][i]);
++local_errs;
}
}
}
}
/* For each incoming and outgoing edge in the sources, we should
* have an entry in the matrix */
for (i = 0; i < indegree; i++) {
if (layout[sources[i]][rank] != sweights[i]) {
fprintf(stderr, "edge (%d,%d) has a weight %d instead of %d\n", i, rank,
sweights[i], layout[sources[i]][rank]);
++local_errs;
}
}
for (i = 0; i < outdegree; i++) {
if (layout[rank][destinations[i]] != dweights[i]) {
fprintf(stderr, "edge (%d,%d) has a weight %d instead of %d\n", rank, i,
dweights[i], layout[rank][destinations[i]]);
++local_errs;
}
}
}
if (dupcomm != MPI_COMM_NULL)
MPI_Comm_free(&dupcomm);
return local_errs;
}
Matrix createMatrixMPI(int n1, int n2, int N1, int N2, MPI_Comm* comm)
{
int i;
Matrix result = (Matrix)calloc(1, sizeof(matrix_t));
int topo_type;
MPI_Topo_test(*comm, &topo_type);
if (topo_type == MPI_CART) {
result->rows = n1;
result->cols = n2;
result->as_vec = malloc(sizeof(vector_t));
result->as_vec->comm = comm;
result->as_vec->len = n1*n2;
result->as_vec->stride = 1;
MPI_Comm_rank(*comm, &result->as_vec->comm_rank);
MPI_Comm_size(*comm, &result->as_vec->comm_size);
result->data = (double **)calloc(n2 ,sizeof(double *));
result->data[0] = (double *)calloc(n1*n2,sizeof(double));
result->as_vec->data = result->data[0];
for (i=1; i < n2; i++)
result->data[i] = result->data[i-1] + n1;
result->col = malloc(n2*sizeof(Vector));
for (int i=0;i<n2;++i) {
result->col[i] = malloc(sizeof(vector_t));
result->col[i]->data = result->data[i];
result->col[i]->stride = 1;
result->col[i]->len = n1;
}
result->row = malloc(n1*sizeof(Vector));
for (int i=0;i<n1;++i) {
result->row[i] = malloc(sizeof(vector_t));
result->row[i]->data = result->data[0]+i;
result->row[i]->stride = n1;
result->row[i]->len = n2;
}
return result;
}
result->as_vec = createVectorMPI(N1*N2, 0, comm);
int n12 = n1;
if (n1 == -1)
n1 = result->as_vec->len/N2;
else
n2 = result->as_vec->len/N1;
result->rows = n1;
result->cols = n2;
result->glob_rows = N1;
result->glob_cols = N2;
result->data = (double **)calloc(n2 ,sizeof(double *));
result->data[0] = (double *)calloc(n1*n2,sizeof(double));
result->as_vec->data = result->data[0];
for (i=1; i < n2; i++)
result->data[i] = result->data[i-1] + n1;
result->col = malloc(n2*sizeof(Vector));
for (int i=0;i<n2;++i) {
if (n12 == N1) {
result->col[i] = createVectorMPI(N1, 0, &SelfComm);
} else {
result->col[i] = createVectorMPI(N1, 0, comm);
}
result->col[i]->data = result->data[i];
}
result->row = malloc(n1*sizeof(Vector));
for (int i=0;i<n1;++i) {
if (n12 == N1)
result->row[i] = createVectorMPI(N2, 0, comm);
else
result->row[i] = createVectorMPI(N2, 0, &SelfComm);
result->row[i]->data = result->data[0]+i;
result->row[i]->stride = n1;
}
return result;
}
void mpi_manager_2D::setup(NumArray<int> &nproc, NumArray<int> &mx) {
// Determine the rank of the current task
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
// Get number of ranks from MPI
int ntasks;
MPI_Comm_size(MPI_COMM_WORLD, &ntasks);
this->ntasks = ntasks;
// Set the distribution of processes:
if(ntasks != nproc[0]*nproc[1]){
std::cerr << " Wrong number of processes " << std::endl;
std::cout << ntasks << " " << nproc[0]*nproc[1] << std::endl;
Finalise();
}
if(rank==0) {
std::cout << " Number of tasks: " << ntasks << " " << nproc[0] << " " << nproc[1] << std::endl;
}
// Check if grid can be subdevided as desired
for(int dir = 0; dir < DIM; ++dir) {
if(mx[dir] < nproc[dir] && nproc[dir] > 1) {
if(rank == 0) {
std::cerr << " Wrong grid topology for dimension ";
std::cerr << dir << std::endl;
std::cerr << " mx[" << dir << "]:" << mx[dir] << std::endl;
std::cerr << " nproc[" << dir << "]:" << nproc[dir] << std::endl;
}
Finalise();
}
}
// Check if grid is a power of 2:
double eps = 1.e-12;
for(int dir = 0; dir < DIM; ++dir) {
double exponent = log(mx[dir])/log(2.);
int i_exponent = static_cast<int>(exponent+eps);
if(exponent - i_exponent > 2.*eps) {
if(rank == 0) {
std::cerr << " Error: grid must be of the form mx = 2^n ";
std::cerr << std::endl;
std::cerr << " Exiting " << std::endl;
}
Finalise();
}
}
// Grid is not periodic
int periods[3] = {false, false, false};
int reorder = false;
// If all is okay: Create new communicator "comm3d"
MPI_Cart_create(MPI_COMM_WORLD, DIM, nproc, periods, reorder, &comm2d);
// Retrieve the cartesian topology
if (rank == 0) {
int TopoType;
std::cout << " Cart topology: ";
MPI_Topo_test(comm2d, &TopoType);
switch (TopoType) {
case MPI_UNDEFINED :
std::cout << " MPI_UNDEFINED " << std::endl;
break;
case MPI_GRAPH :
std::cout << "MPI_GRAPH" << std::endl;
break;
case MPI_CART :
std::cout << "MPI_CART" << std::endl;
break;
}
}
// Determine rank again for cartesian communicator -> overwrite rank
MPI_Comm_rank(comm2d, &rank);
// std::cout << " my rank: " << rank << std::endl;
// Translate rank to coordinates
MPI_Cart_coords(comm2d, rank, DIM, coords);
// // Backwards translation
// int TranslateRank;
// MPI_Cart_rank(comm3d, coords, &TranslateRank);
// Find neighbouring ranks
// Syntax: comm3d, shift direction, displacement, source, destination
MPI_Cart_shift(comm2d, 0, 1, &left , &right);
MPI_Cart_shift(comm2d, 1, 1, &front, &back);
// std::cout << " My rank " << rank << " " << left << " " << right << " " << front << " " << back << " " << bottom << " " << top << std::endl;
// Determine position of other ranks
determin_OtherRanks();
get_SubComms();
}
void Communication::prepare(p3m_float box_l[3]) {
P3M_DEBUG(printf( " P3M::Communication::prepare() started...\n"));
/* Test whether the communicator is cartesian and correct dimensionality */
bool comm_is_cart = false;
int status;
MPI_Topo_test(mpicomm_orig, &status);
if (status == MPI_CART) {
/* Communicator is cartesian, so test dimensionality */
int ndims;
MPI_Cartdim_get(mpicomm_orig, &ndims);
if (ndims == 3) {
/* Correct dimensionality, so get grid and test periodicity */
int periodicity[3];
MPI_Cart_get(mpicomm_orig, 3, node_grid,
periodicity, node_pos);
if (periodicity[0] && periodicity[1] && periodicity[2]) {
/* If periodicity is correct, we can just use this communicator */
mpicomm = mpicomm_orig;
/* get the rank */
MPI_Comm_rank(mpicomm, &rank);
comm_is_cart = true;
}
}
}
/* otherwise, we have to set up the cartesian communicator */
if (!comm_is_cart) {
P3M_DEBUG(printf( " Setting up cartesian communicator...\n"));
node_grid[0] = 0;
node_grid[1] = 0;
node_grid[2] = 0;
/* compute node grid */
MPI_Dims_create(size, 3, node_grid);
#ifdef P3M_ENABLE_INFO
if (onMaster())
printf(" node_grid=%dx%dx%d\n", node_grid[0], node_grid[1], node_grid[2]);
#endif
/* create communicator */
int periodicity[3] = {1, 1, 1};
MPI_Cart_create(mpicomm_orig, 3, node_grid,
periodicity, 1, &mpicomm);
/* get the rank */
MPI_Comm_rank(mpicomm, &rank);
/* get node pos */
MPI_Cart_coords(mpicomm, rank, 3, node_pos);
}
/* fetch neighborhood info */
for (int dir = 0; dir < 3; dir++) {
MPI_Cart_shift(mpicomm, dir, 1,
&node_neighbors[2*dir],
&node_neighbors[2*dir+1]);
P3M_DEBUG_LOCAL(printf( " %d: dir=%d: n1=%d n2=%d\n", rank, dir, \
node_neighbors[2*dir], \
node_neighbors[2*dir+1]));
}
/* init local points */
for (int i=0; i< 3; i++) {
local_box_l[i] = 0.0;
my_left[i] = 0.0;
my_right[i] = 0.0;
}
/* compute box limits */
for(p3m_int i = 0; i < 3; i++) {
local_box_l[i] = box_l[i]/(p3m_float)node_grid[i];
my_left[i] = node_pos[i] *local_box_l[i];
my_right[i] = (node_pos[i]+1)*local_box_l[i];
}
P3M_DEBUG(printf(" local_box_l=" F3FLOAT "\n" \
" my_left=" F3FLOAT "\n" \
" my_right=" F3FLOAT "\n", \
local_box_l[0], \
local_box_l[1], \
local_box_l[2], \
my_left[0], my_left[1], my_left[2], \
my_right[0], my_right[1], my_right[2] \
));
P3M_DEBUG(printf(" P3M::Communication::prepare() finished.\n"));
}
FORT_DLL_SPEC void FORT_CALL mpi_topo_test_ ( MPI_Fint *v1, MPI_Fint *v2, MPI_Fint *ierr ){
*ierr = MPI_Topo_test( (MPI_Comm)(*v1), v2 );
}
