int main( int argc, char *argv[] )
{
    int errs = 0;
    int topo_type, size, dims[1], periods[1];
    MPI_Comm comm;

    MTest_Init( &argc, &argv );

    /* Check that topo test returns the correct type, including 
       MPI_UNDEFINED */

    MPI_Topo_test( MPI_COMM_WORLD, &topo_type );
    if (topo_type != MPI_UNDEFINED) {
	errs++;
	printf( "Topo type of comm world is not UNDEFINED\n" );
    }

    MPI_Comm_size( MPI_COMM_WORLD, &size );
    dims[0] = size;
    periods[0] = 0;
    MPI_Cart_create( MPI_COMM_WORLD, 1, dims, periods, 0, &comm );
    MPI_Topo_test( comm, &topo_type );
    if (topo_type != MPI_CART) {
	errs++;
	printf( "Topo type of cart comm is not CART\n" );
    }

    MPI_Comm_free( &comm );
    /* FIXME: still need graph example */

    MTest_Finalize( errs );
    MPI_Finalize();
    return 0;
  
}
Exemple #2
0
static MPI_Comm cart_comm_1d(MPI_Comm communicator) {
  /* test whether the communicator is cartesian and correct dimensionality */
  fcs_int status;
  MPI_Topo_test(communicator, &status);
  if (status == MPI_CART) {
    /* Communicator is cartesian, so test dimensionality */
    fcs_int ndims;
    MPI_Cartdim_get(communicator, &ndims);
    if (ndims == 3) {
      /* Correct dimensionality, so get grid and test periodicity */
      fcs_int node_grid[3], periodicity[3], node_pos[3];
      MPI_Cart_get(communicator, 3, node_grid, periodicity, node_pos);
      if (!periodicity[0] && !periodicity[1] && periodicity[2]) {
        return communicator;
      }
    }
  }
  /* Otherwise, we have to set up the cartesian communicator */
  fcs_int comm_size;
  MPI_Comm_size(communicator, &comm_size);
  fcs_int node_grid[3] = {0, 0, 0};
  MPI_Dims_create(comm_size, 3, node_grid);
  fcs_int periodicity[3] = {1, 1, 0};

  MPI_Comm new_communicator;
  MPI_Cart_create(communicator, 3, node_grid, periodicity, 1, &new_communicator);

  return new_communicator;
}
Exemple #3
0
int PX(is_cart_procmesh)(
    MPI_Comm comm_cart
    )
{
  int status;

  MPI_Topo_test(comm_cart, &status);
  return (status == MPI_CART);
}
Exemple #4
0
int PX(is_cart_procmesh_2d)(
    MPI_Comm comm_cart_2d
    )
{
  int status, ndims;

  MPI_Topo_test(comm_cart_2d, &status);
  MPI_Cartdim_get(comm_cart_2d, &ndims);

  return ( (status == MPI_CART) && (ndims == 2) );
}
int NBC_Comm_neighbors(MPI_Comm comm, int maxindegree, int sources[], int sourceweights[], int maxoutdegree, int destinations[], int destweights[]) {
  int topo, res;
  int index = 0;

  int indeg, outdeg, wgtd;
  res = NBC_Comm_neighbors_count(comm, &indeg, &outdeg, &wgtd);
  if(indeg > maxindegree && outdeg > maxoutdegree) return NBC_INVALID_PARAM; /* we want to return *all* neighbors */

  res = MPI_Topo_test(comm, &topo);
  if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Topo_test() (%i)\n", res); return res; }

  switch(topo) {
    case MPI_CART: /* cartesian */
      {
        int ndims, i, rpeer, speer;
        res = MPI_Cartdim_get(comm, &ndims);
        if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Cartdim_get() (%i)\n", res); return res; }

        for(i = 0; i<ndims; i++) {
          res = MPI_Cart_shift(comm, i, 1, &rpeer, &speer);
          if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Cart_shift() (%i)\n", res); return res; }
          sources[index] = destinations[index] = rpeer; index++;
          sources[index] = destinations[index] = speer; index++;
        }
      }
      break;
    case MPI_GRAPH: /* graph */
      {
        int rank;
        MPI_Comm_rank(comm, &rank);
        res = MPI_Graph_neighbors(comm, rank, maxindegree, sources);
        if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Graph_neighbors_count() (%i)\n", res); return res; }
        for(int i=0; i<maxindegree; i++) destinations[i] = sources[i];
      }
      break;
    case MPI_DIST_GRAPH: /* dist graph */
      {
        res = MPI_Dist_graph_neighbors(comm, maxindegree, sources, sourceweights, maxoutdegree, destinations, destweights);
        if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Graph_neighbors_count() (%i)\n", res); return res; }
      }
      break;
    case MPI_UNDEFINED:
      return NBC_INVALID_TOPOLOGY_COMM;
      break;
    default:
      return NBC_INVALID_PARAM;
      break;
  }

  return NBC_OK;
}
Exemple #6
0
void
mmm2d_comm_init(mmm2d_comm_struct *comm, MPI_Comm communicator) {
  /* store the original communicator */
  comm->mpicomm_orig = communicator;
  MPI_Comm_size(communicator, &comm->size);

  /* Test whether the communicator is cartesian and correct dimensionality */
  fcs_int comm_is_cart = 0;
  fcs_int status;
  MPI_Topo_test(communicator, &status);
  if (status == MPI_CART) {
    /* Communicator is cartesian, so test dimensionality */
    fcs_int ndims;
    MPI_Cartdim_get(communicator, &ndims);
    if (ndims == 3) {
      /* Correct dimensionality, so get grid and test periodicity */
      fcs_int periodicity[3];
      MPI_Cart_get(communicator, 3, comm->node_grid, periodicity, comm->node_pos);
      if (periodicity[0] && periodicity[1] && periodicity[2]) {
        /* If periodicity is correct, we can just use this communicator */
        comm->mpicomm = communicator;
        /* get the rank */
        MPI_Comm_rank(communicator, &comm->rank);
        comm_is_cart = 1;
      }
    }
  }

  /* otherwise, we have to set up the cartesian communicator */
  if (!comm_is_cart) {

    comm->node_grid[0] = 0.0;
    comm->node_grid[1] = 0.0;
    comm->node_grid[2] = 0.0;

    /* compute node grid */
    MPI_Dims_create(comm->size, 3, comm->node_grid);

    /* create communicator */
    fcs_int periodicity[3] = {1, 1, 0};
    MPI_Cart_create(comm->mpicomm_orig, 3, comm->node_grid, periodicity, 1, &comm->mpicomm);

    /* get the rank */
    MPI_Comm_rank(communicator, &comm->rank);
    /* get node pos */
    MPI_Cart_coords(comm->mpicomm, comm->rank, 3, comm->node_pos);
  }
}
Exemple #7
0
static int comm_is_cart_3d(
    MPI_Comm comm
    )
{
  int ndims, status;

  MPI_Topo_test(comm, &status);
  if (status != MPI_CART)
    return 0;

  MPI_Cartdim_get(comm, &ndims);
  if (ndims != 3)
    return 0;

  return 1;
}
Exemple #8
0
Comm::Comm(MPI_Comm impl) : impl_(impl) {
  int topo_type;
  CALL(MPI_Topo_test(impl, &topo_type));
  if (topo_type == MPI_DIST_GRAPH) {
    int nin, nout, is_weighted;
    CALL(MPI_Dist_graph_neighbors_count(impl, &nin, &nout, &is_weighted));
    HostWrite<I32> sources(nin);
    HostWrite<I32> destinations(nout);
    CALL(MPI_Dist_graph_neighbors(impl, nin, sources.data(),
        OMEGA_H_MPI_UNWEIGHTED, nout, destinations.data(),
        OMEGA_H_MPI_UNWEIGHTED));
    srcs_ = sources.write();
    dsts_ = destinations.write();
    host_srcs_ = HostRead<I32>(srcs_);
    host_dsts_ = HostRead<I32>(dsts_);
  }
}
int NBC_Comm_neighbors_count(MPI_Comm comm, int *indegree, int *outdegree, int *weighted) {
  int topo, res;

  res = MPI_Topo_test(comm, &topo);
  if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Topo_test() (%i)\n", res); return res; }

  switch(topo) {
    case MPI_CART: /* cartesian */
      {
        int ndims;
        res = MPI_Cartdim_get(comm, &ndims)  ;
        if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Cartdim_get() (%i)\n", res); return res; }
        /* outdegree is always 2*ndims because we need to iterate over empty buffers for MPI_PROC_NULL */
        *outdegree = *indegree = 2*ndims;
        *weighted = 0;
      }
      break;
    case MPI_GRAPH: /* graph */
      {
        int rank, nneighbors;
        MPI_Comm_rank(comm, &rank);
        res = MPI_Graph_neighbors_count(comm, rank, &nneighbors);
        if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Graph_neighbors_count() (%i)\n", res); return res; }
        *outdegree = *indegree = nneighbors;
        *weighted = 0;
      }
      break;
    case MPI_DIST_GRAPH: /* graph */
      {
        res = MPI_Dist_graph_neighbors_count(comm, indegree, outdegree, weighted);
        if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Dist_graph_neighbors_count() (%i)\n", res); return res; }
      }
      break;
    case MPI_UNDEFINED:
      return NBC_INVALID_TOPOLOGY_COMM;
      break;
    default:
      return NBC_INVALID_PARAM;
      break;
  }
  return NBC_OK;
}
Exemple #10
0
static int create_proclists_from_mpi_comm_topology(MPI_Comm comm, int rank, int *nsend_procs, int **send_procs, int *nrecv_procs, int **recv_procs)
{
  int status;

  int n, i;


  MPI_Topo_test(comm, &status);

  switch (status)
  {
    case MPI_CART:
      MPI_Cartdim_get(comm, &n);
      *nsend_procs = *nrecv_procs = 2 * n;
      *send_procs = *recv_procs = z_alloc(2 * n, sizeof(int));
      for (i = 0; i < n; ++i) MPI_Cart_shift(comm, i, -1, &(*send_procs)[2 * i + 1], &(*send_procs)[2 * i + 0]);
      break;
    case MPI_GRAPH:
      MPI_Graph_neighbors_count(comm, rank, &n);
      *nsend_procs = *nrecv_procs = n;
      *send_procs = *recv_procs = z_alloc(n, sizeof(int));
      MPI_Graph_neighbors(comm, rank, n, *send_procs);
      break;
#if MPI_VERSION >= 2 && MPI_SUBVERSION >= 2
    case MPI_DIST_GRAPH:
      MPI_Dist_graph_neighbors_count(comm, nrecv_procs, nsend_procs, &n);
      *send_procs = z_alloc(*nsend_procs + *nrecv_procs, sizeof(int));
      *recv_procs = *send_procs + *nsend_procs;
      MPI_Dist_graph_neighbors(comm, *nrecv_procs, *recv_procs, MPI_UNWEIGHTED, *nsend_procs, *send_procs, MPI_UNWEIGHTED);
      break;
#endif
    default:
      return 0;
  }

  return 1;
}
Exemple #11
0
void fcs_memd_setup_communicator(memd_struct* memd, MPI_Comm communicator)
{
    /* store given communicator */
    memd->mpiparams.original_comm = communicator;
    MPI_Comm_size(communicator, &memd->mpiparams.size);

    /* Test whether the communicator is cartesian and correct dimensionality */
    int comm_is_cart = 0;
    int status;

    MPI_Topo_test(communicator, &status);
    if (status == MPI_CART) {
        /* Communicator is cartesian, so test dimensionality */
        int ndims;
        MPI_Cartdim_get(communicator, &ndims);
        if (ndims == 3) {
            /* Correct dimensionality, so get grid and test periodicity */
            int periodicity[3];
            MPI_Cart_get(communicator, 3, memd->mpiparams.node_grid, periodicity, memd->mpiparams.node_pos);
            if (periodicity[0] && periodicity[1] && periodicity[2]) {
                /* If periodicity is correct, we can just use this communicator */
                memd->mpiparams.communicator = communicator;
                /* get the rank */
                MPI_Comm_rank(communicator, &memd->mpiparams.this_node);
                comm_is_cart = 1;
            }
        }
    }
    
    /* otherwise, we have to set up the cartesian communicator */
    if (!comm_is_cart) {        
        memd->mpiparams.node_grid[0] = 0.0;
        memd->mpiparams.node_grid[1] = 0.0;
        memd->mpiparams.node_grid[2] = 0.0;
        
        /* compute node grid */
        MPI_Dims_create(memd->mpiparams.size, 3, memd->mpiparams.node_grid);
        /* swap first and last dimension, as MEMD currently wants to have them increasing */
        fcs_int tmp = memd->mpiparams.node_grid[2];
        memd->mpiparams.node_grid[2] = memd->mpiparams.node_grid[0];
        memd->mpiparams.node_grid[0] = tmp;
        
        /* create communicator */
        int periodicity[3] = {1, 1, 1};
        MPI_Cart_create(memd->mpiparams.original_comm, 3, memd->mpiparams.node_grid, periodicity, 1, &memd->mpiparams.communicator);
        
        /* get the rank */
        MPI_Comm_rank(communicator, &memd->mpiparams.this_node);
        /* get node pos */
        MPI_Cart_coords(memd->mpiparams.communicator, memd->mpiparams.this_node, 3, memd->mpiparams.node_pos);
    }
    
    /* fetch neighborhood info */
    for (int dir = 0; dir<3; dir++) {
        MPI_Cart_shift(memd->mpiparams.communicator, dir, 1, 
                       &memd->mpiparams.node_neighbors[2*dir], 
                       &memd->mpiparams.node_neighbors[2*dir+1]);
    }
    
    /* init local points */
    for (int i=0; i< 3; i++) {
//        memd->mpiparams.local_box_l[i] = 0.0;
        memd->mpiparams.my_left[i] = 0.0;
        memd->mpiparams.my_right[i] = 0.0;
    }
    
    /* compute box limits */
    fcs_float local_box_length = 0.0;
    for(fcs_int i = 0; i < 3; i++) {
        local_box_length = memd->parameters.box_length[i] / (fcs_float)memd->mpiparams.node_grid[i];
        memd->mpiparams.my_left[i]   = memd->mpiparams.node_pos[i] * local_box_length;
        memd->mpiparams.my_right[i]  = (memd->mpiparams.node_pos[i]+1) * local_box_length;
    }

}
Exemple #12
0
void mpi_manager_3D::setup(NumArray<int> &nproc, NumArray<int> &mx) {
	
	// Save number of processors in each dimension
	for(int dir=0; dir<DIM; ++dir) {
		this->nproc[dir] = nproc[dir];
	}

	// Determine the rank of the current task
	MPI_Comm_rank(MPI_COMM_WORLD, &rank);

	// Get number of ranks from MPI
	int ntasks;
	MPI_Comm_size(MPI_COMM_WORLD, &ntasks);
	this->ntasks = ntasks;

	// Set the distribution of processes:
	if(ntasks != nproc[0]*nproc[1]*nproc[2]){
		std::cerr << " Wrong number of processes " << std::endl;
		std::cout << ntasks << " " << nproc[0]*nproc[1]*nproc[2] << std::endl;
		Finalise();
	}

	if(rank==0) {
		std::cout << " Number of tasks: " << ntasks << std::endl;
	}

	// Check if grid can be subdevided as desired
	for(int dir = 0; dir < DIM; ++dir) {
		if(mx[dir] < nproc[dir] && nproc[dir] > 1) {
			if(rank == 0) {
				std::cerr << " Wrong grid topology for dimension ";
				std::cerr << dir << std::endl;
				std::cerr << "  mx[" << dir << "]:" << mx[dir] << std::endl;
				std::cerr << " nproc[" << dir << "]:" << nproc[dir] << std::endl;
			}
			Finalise();
		}
	}

	// Check if grid is a power of 2:
	double eps = 1.e-12;
	for(int dir = 0; dir < DIM; ++dir) {
		double exponent = log(mx[dir])/log(2.);
		int i_exponent = static_cast<int>(exponent+eps);

		if(exponent - i_exponent > 2.*eps) {
			if(rank == 0) {
				std::cerr << " Error: grid must be of the form mx = 2^n ";
				std::cerr << std::endl;
				std::cerr << " Exiting " << std::endl;
			}
			Finalise();
		}
	}

	// Grid is not periodic
	int periods[3] = {false, false, false};
	int reorder = false;
	// If all is okay: Create new communicator "comm3d"  
	MPI_Cart_create(MPI_COMM_WORLD, DIM, nproc, periods, reorder, &comm3d);

	// Retrieve the cartesian topology
	if (rank == 0) {
		int TopoType;
		std::cout << " Cart topology:  ";
		MPI_Topo_test(comm3d, &TopoType);
		switch (TopoType) {
		case MPI_UNDEFINED : 
			std::cout << " MPI_UNDEFINED " << std::endl;
			break;
		case MPI_GRAPH     :
			std::cout << "MPI_GRAPH" << std::endl;
			break;
		case MPI_CART      :
			std::cout << "MPI_CART" << std::endl;
			break;
		}
	}
	
	//   Determine rank again for cartesian communicator -> overwrite rank
	MPI_Comm_rank(comm3d, &rank);

	// std::cout << " my rank: " << rank << std::endl;

	// Translate rank to coordinates
	MPI_Cart_coords(comm3d, rank, DIM, coords);

	// // Backwards translation
	// int TranslateRank;
	// MPI_Cart_rank(comm3d, coords, &TranslateRank);

	// Find neighbouring ranks
	// Syntax: comm3d, shift direction, displacement, source, destination
	MPI_Cart_shift(comm3d, 0, 1, &left , &right);
	MPI_Cart_shift(comm3d, 1, 1, &front, &back);
	MPI_Cart_shift(comm3d, 2, 1, &bottom, &top);

	// std::cout << " My rank " << rank << " " << left << " " << right << " " << front << " " << back << " " << bottom << " " << top << std::endl;
	if(rank==0) {
		std::cout << " nearby " << right << " " << back << " " << top << std::endl;
	}

	// Determine ranks of neighbour processes:
	int shiftcoord[DIM];
	int lbound[DIM],ubound[DIM];
	for(int dim=0;dim<DIM;dim++){
		lbound[dim]=-1;
		ubound[dim]= 1;
	}
	Neighbour.resize(lbound,ubound);
	Neighbour.clear();

	for(int dim0=-1; dim0<=1; dim0++){
		shiftcoord[0] = (coords[0]+dim0)%nproc[0];
		if(shiftcoord[0] < 0) shiftcoord[0]+=nproc[0];
		for(int dim1=-1; dim1<=1; dim1++){
			shiftcoord[1] = (coords[1]+dim1)%nproc[1];
			if(shiftcoord[1] < 0) shiftcoord[1]+=nproc[1];
			for(int dim2=-1; dim2<=1; dim2++){
				shiftcoord[2] = (coords[2]+dim2)%nproc[2];
				if(shiftcoord[2] < 0) shiftcoord[2]+=nproc[2];
				MPI_Cart_rank(comm3d, shiftcoord,&Neighbour(dim0,dim1,dim2));
			}
		}
	}
	
	// if(rank==1) {
	// 	for(int dim0=-1; dim0<=1; dim0++){
	// 		for(int dim1=-1; dim1<=1; dim1++){
	// 			for(int dim2=-1; dim2<=1; dim2++){
	// 				std::cout << " neighbour " << dim0 << " " << dim1 << " ";
	// 				std::cout << dim2 << " " << Neighbour(dim0, dim1, dim2);
	// 				std::cout << std::endl;
	// 			}
	// 		}
	// 	}
	// }

	// Determine absolute position of any rank:
	AllRanks.resize(Index::set(0,0,0),
	               Index::set(nproc[0]-1,nproc[1]-1,nproc[2]-1));
	
	for(int dim0=0; dim0<nproc[0]; ++dim0) {
		for(int dim1=0; dim1<nproc[1]; ++dim1) {
			for(int dim2=0; dim2<nproc[2]; ++dim2) {
				int coord[3] = {dim0, dim1, dim2};
				MPI_Cart_rank(comm3d, coord, &AllRanks(dim0, dim1, dim2));
			}
		}
	}

	// if(rank==2) {
	// 	std::cout << " Neigh: " << rank << " "<<Neighbour(0,0,0) << " " << AllRanks(2,0,0) << std::endl;
	// }

	
	// Now make additional mpi groups relating to planes:

	int count(0);
	int num_xy = nproc[0]*nproc[1];
	int num_xz = nproc[0]*nproc[2];
	int num_yz = nproc[1]*nproc[2];
	
	NumMatrix<int,1> x_ranks[nproc[0]];
	NumMatrix<int,1> y_ranks[nproc[1]];
	NumMatrix<int,1> z_ranks[nproc[2]];

	// Walk trough z-axis -- xy plane
	for(int irz=0; irz<nproc[2]; irz++) {
		count = 0;
		z_ranks[irz].resize(Index::set(0), Index::set(num_xy));
		for(int irx=0; irx<nproc[0]; irx++) {
			for(int iry=0; iry<nproc[1]; iry++) {
				z_ranks[irz](count) = AllRanks(irx,iry,irz);
				count++;
			}
		}
	}

	// Walk trough y-axis -- xz plane
	for(int iry=0; iry<nproc[1]; iry++) {
		count = 0;
		y_ranks[iry].resize(Index::set(0), Index::set(num_xz));
		for(int irx=0; irx<nproc[0]; irx++) {
			for(int irz=0; irz<nproc[2]; irz++) {
				y_ranks[iry](count) = AllRanks(irx,iry,irz);
				count++;
			}
		}
	}

	// Walk trough x-axis -- yz plane
	for(int irx=0; irx<nproc[0]; irx++) {
		count = 0;
		x_ranks[irx].resize(Index::set(0), Index::set(num_yz));
		for(int iry=0; iry<nproc[1]; iry++) {
			for(int irz=0; irz<nproc[2]; irz++) {
				x_ranks[irx](count) = AllRanks(irx,iry,irz);
				count++;
			}
		}
	}

	// Build local communicator:
	MPI_Group group_all, group_constz, group_consty, group_constx;
	// Get standard group handle:
	MPI_Comm_group(comm3d, &group_all);


	// Devide tasks into groups based on z-position
	MPI_Group_incl(group_all, num_xy, z_ranks[coords[2]], &group_constz);

	// Devide tasks into groups based on z-position
	MPI_Group_incl(group_all, num_xz, y_ranks[coords[1]], &group_consty);

	// Devide tasks into groups based on x-position
	MPI_Group_incl(group_all, num_yz, x_ranks[coords[0]], &group_constx);

	// // Make corresponding communicators:
	// MPI_Comm_create(comm3d, group_constz, &comm_plane_xy); // const z
	// MPI_Comm_create(comm3d, group_consty, &comm_plane_xz); // const x
	// MPI_Comm_create(comm3d, group_constx, &comm_plane_yz); // const x
	// // Get corresponding rank
	// MPI_Group_rank (group_constz, &rank_plane_xy);
	// MPI_Group_rank (group_consty, &rank_plane_xz);
	// MPI_Group_rank (group_constx, &rank_plane_yz);

	int remain_dims[3];
	// x-y plane:
	remain_dims[0] = 1;
	remain_dims[1] = 1;
	remain_dims[2] = 0;
	MPI_Cart_sub(comm3d, remain_dims, &comm_plane_xy);
	MPI_Comm_rank(comm_plane_xy, &rank_plane_xy);

	// x-z plane
	remain_dims[0] = 1;
	remain_dims[1] = 0;
	remain_dims[2] = 1;
	MPI_Cart_sub(comm3d, remain_dims, &comm_plane_xz);
	MPI_Comm_rank(comm_plane_xz, &rank_plane_xz);

	// y-z plane
	remain_dims[0] = 0;
	remain_dims[1] = 1;
	remain_dims[2] = 1;
	MPI_Cart_sub(comm3d, remain_dims, &comm_plane_yz);
	MPI_Comm_rank(comm_plane_yz, &rank_plane_yz);

}
Exemple #13
0
int main( int argc, char **argv )
{
    int              rank, size, i;
    int              errors=0;
    int              dims[NUM_DIMS];
    int              periods[NUM_DIMS];
    int              coords[NUM_DIMS];
    int              new_coords[NUM_DIMS];
    int              reorder = 1;
    MPI_Comm         comm_temp, comm_cart, new_comm;
    int              topo_status;
    int              ndims;
    int              new_rank;
    int              remain_dims[NUM_DIMS];
    int              newnewrank;

    MPI_Init( &argc, &argv );

    MPI_Comm_rank( MPI_COMM_WORLD, &rank );
    MPI_Comm_size( MPI_COMM_WORLD, &size );

    /* Clear dims array and get dims for topology */
    for(i=0;i<NUM_DIMS;i++) { dims[i] = 0; periods[i] = 0; }
    MPI_Dims_create ( size, NUM_DIMS, dims );

    /* Make a new communicator with a topology */
    MPI_Cart_create ( MPI_COMM_WORLD, 2, dims, periods, reorder, &comm_temp );
    MPI_Comm_dup ( comm_temp, &comm_cart );

    /* Determine the status of the new communicator */
    MPI_Topo_test ( comm_cart, &topo_status );
    if (topo_status != MPI_CART) {
	printf( "topo_status of duped comm is not MPI_CART\n" );
	errors++;
    }

    /* How many dims do we have? */
    MPI_Cartdim_get( comm_cart, &ndims );
    if ( ndims != NUM_DIMS ) {
	printf( "Number of dims of duped comm (%d) should be %d\n", 
		ndims, NUM_DIMS );
	errors++;
    }

    /* Get the topology, does it agree with what we put in? */
    for(i=0;i<NUM_DIMS;i++) { dims[i] = 0; periods[i] = 0; }
    MPI_Cart_get ( comm_cart, NUM_DIMS, dims, periods, coords );

    /* Does the mapping from coords to rank work? */
    MPI_Cart_rank ( comm_cart, coords, &new_rank );
    if ( new_rank != rank ) {
	printf( "New rank of duped comm (%d) != old rank (%d)\n", 
		new_rank, rank );
	errors++;
    }

    /* Does the mapping from rank to coords work */
    MPI_Cart_coords ( comm_cart, rank, NUM_DIMS, new_coords );
    for (i=0;i<NUM_DIMS;i++) 
	if ( coords[i] != new_coords[i] ) {
	    printf( "Old coords[%d] of duped comm (%d) != new_coords (%d)\n", 
		    i, coords[i], new_coords[i] );
	    errors++;
	}

    /* Let's shift in each dimension and see how it works!   */
    /* Because it's late and I'm tired, I'm not making this  */
    /* automatically test itself.                            */
    for (i=0;i<NUM_DIMS;i++) {
      int source, dest;
      MPI_Cart_shift(comm_cart, i, 1, &source, &dest);
#ifdef VERBOSE      
      printf ("[%d] Shifting %d in the %d dimension\n",rank,1,i);
      printf ("[%d]    source = %d  dest = %d\n",rank,source,dest); 
#endif
    }

    /* Subdivide */
    remain_dims[0] = 0; 
    for (i=1; i<NUM_DIMS; i++) remain_dims[i] = 1;
    MPI_Cart_sub ( comm_cart, remain_dims, &new_comm );

    /* Determine the status of the new communicator */
    MPI_Topo_test ( new_comm, &topo_status );
    if (topo_status != MPI_CART) {
	printf( "topo_status of cartsub comm is not MPI_CART\n" );
	errors++;
    }

    /* How many dims do we have? */
    MPI_Cartdim_get( new_comm, &ndims );
    if ( ndims != NUM_DIMS-1 ) {
	printf( "Number of dims of cartsub comm (%d) should be %d\n", 
		ndims, NUM_DIMS-1 );
	errors++;
    }

    /* Get the topology, does it agree with what we put in? */
    for(i=0;i<NUM_DIMS-1;i++) { dims[i] = 0; periods[i] = 0; }
    MPI_Cart_get ( new_comm, ndims, dims, periods, coords );
    
    /* Does the mapping from coords to rank work? */
    MPI_Comm_rank ( new_comm, &newnewrank );
    MPI_Cart_rank ( new_comm, coords, &new_rank );
    if ( new_rank != newnewrank ) {
	printf( "New rank of cartsub comm (%d) != old rank (%d)\n", 
		new_rank, newnewrank );
	errors++;
    }

    /* Does the mapping from rank to coords work */
    MPI_Cart_coords ( new_comm, new_rank, NUM_DIMS -1, new_coords );
    for (i=0;i<NUM_DIMS-1;i++) 
	if ( coords[i] != new_coords[i] ) {
	    printf( "Old coords[%d] of cartsub comm (%d) != new_coords (%d)\n", 
		    i, coords[i], new_coords[i] );
	    errors++;
	}

    /* We're at the end */
    MPI_Comm_free( &new_comm );
    MPI_Comm_free( &comm_temp );
    MPI_Comm_free( &comm_cart );
    Test_Waitforall( );
    if (errors) printf( "[%d] done with %d ERRORS!\n", rank,errors );
    MPI_Finalize();
    return 0;
}
Exemple #14
0
static int verify_comm(MPI_Comm comm)
{
    int local_errs = 0;
    int i, j;
    int indegree, outdegree, weighted;
    int *sources, *sweights, *destinations, *dweights;
    int use_dup;
    int topo_type = MPI_UNDEFINED;
    MPI_Comm dupcomm = MPI_COMM_NULL;

    sources = (int *) malloc(size * sizeof(int));
    sweights = (int *) malloc(size * sizeof(int));
    destinations = (int *) malloc(size * sizeof(int));
    dweights = (int *) malloc(size * sizeof(int));

    for (use_dup = 0; use_dup <= 1; ++use_dup) {
        if (!use_dup) {
            MPI_Dist_graph_neighbors_count(comm, &indegree, &outdegree, &weighted);
        }
        else {
            MPI_Comm_dup(comm, &dupcomm);
            comm = dupcomm; /* caller retains original comm value */
        }

        MPI_Topo_test(comm, &topo_type);
        if (topo_type != MPI_DIST_GRAPH) {
            fprintf(stderr, "topo_type != MPI_DIST_GRAPH\n");
            ++local_errs;
        }

        j = 0;
        for (i = 0; i < size; i++)
            if (layout[i][rank])
                j++;
        if (j != indegree) {
            fprintf(stderr, "indegree does not match, expected=%d got=%d, layout='%s'\n", indegree, j, graph_layout_name);
            ++local_errs;
        }

        j = 0;
        for (i = 0; i < size; i++)
            if (layout[rank][i])
                j++;
        if (j != outdegree) {
            fprintf(stderr, "outdegree does not match, expected=%d got=%d, layout='%s'\n", outdegree, j, graph_layout_name);
            ++local_errs;
        }

        if ((indegree || outdegree) && (weighted == 0)) {
            fprintf(stderr, "MPI_Dist_graph_neighbors_count thinks the graph is not weighted\n");
            ++local_errs;
        }


        MPI_Dist_graph_neighbors(comm, indegree, sources, sweights, outdegree, destinations, dweights);

        /* For each incoming and outgoing edge in the matrix, search if
         * the query function listed it in the sources. */
        for (i = 0; i < size; i++) {
            if (layout[i][rank]) {
                for (j = 0; j < indegree; j++) {
                    assert(sources[j] >= 0);
                    assert(sources[j] < size);
                    if (sources[j] == i)
                        break;
                }
                if (j == indegree) {
                    fprintf(stderr, "no edge from %d to %d specified\n", i, rank);
                    ++local_errs;
                }
                else {
                    if (sweights[j] != layout[i][rank]) {
                        fprintf(stderr, "incorrect weight for edge (%d,%d): %d instead of %d\n",
                                i, rank, sweights[j], layout[i][rank]);
                        ++local_errs;
                    }
                }
            }
            if (layout[rank][i]) {
                for (j = 0; j < outdegree; j++) {
                    assert(destinations[j] >= 0);
                    assert(destinations[j] < size);
                    if (destinations[j] == i)
                        break;
                }
                if (j == outdegree) {
                    fprintf(stderr, "no edge from %d to %d specified\n", rank, i);
                    ++local_errs;
                }
                else {
                    if (dweights[j] != layout[rank][i]) {
                        fprintf(stderr, "incorrect weight for edge (%d,%d): %d instead of %d\n",
                                rank, i, dweights[j], layout[rank][i]);
                        ++local_errs;
                    }
                }
            }
        }

        /* For each incoming and outgoing edge in the sources, we should
         * have an entry in the matrix */
        for (i = 0; i < indegree; i++) {
            if (layout[sources[i]][rank] != sweights[i]) {
                fprintf(stderr, "edge (%d,%d) has a weight %d instead of %d\n", i, rank,
                        sweights[i], layout[sources[i]][rank]);
                ++local_errs;
            }
        }
        for (i = 0; i < outdegree; i++) {
            if (layout[rank][destinations[i]] != dweights[i]) {
                fprintf(stderr, "edge (%d,%d) has a weight %d instead of %d\n", rank, i,
                        dweights[i], layout[rank][destinations[i]]);
                ++local_errs;
            }
        }

    }

    if (dupcomm != MPI_COMM_NULL)
        MPI_Comm_free(&dupcomm);

    return local_errs;
}
Exemple #15
0
Matrix createMatrixMPI(int n1, int n2, int N1, int N2, MPI_Comm* comm)
{
  int i;
  Matrix result = (Matrix)calloc(1, sizeof(matrix_t));

  int topo_type;
  MPI_Topo_test(*comm, &topo_type);
  if (topo_type == MPI_CART) {
    result->rows = n1;
    result->cols = n2;
    result->as_vec = malloc(sizeof(vector_t));
    result->as_vec->comm = comm;
    result->as_vec->len = n1*n2;
    result->as_vec->stride = 1;
    MPI_Comm_rank(*comm, &result->as_vec->comm_rank);
    MPI_Comm_size(*comm, &result->as_vec->comm_size);
    result->data = (double **)calloc(n2   ,sizeof(double *));
    result->data[0] = (double  *)calloc(n1*n2,sizeof(double));
    result->as_vec->data = result->data[0];
    for (i=1; i < n2; i++)
      result->data[i] = result->data[i-1] + n1;
    result->col = malloc(n2*sizeof(Vector));
    for (int i=0;i<n2;++i) {
      result->col[i] = malloc(sizeof(vector_t));
      result->col[i]->data = result->data[i];
      result->col[i]->stride = 1;
      result->col[i]->len = n1;
    }
    result->row = malloc(n1*sizeof(Vector));
    for (int i=0;i<n1;++i) {
      result->row[i] = malloc(sizeof(vector_t));
      result->row[i]->data = result->data[0]+i;
      result->row[i]->stride = n1;
      result->row[i]->len = n2;
    }
    return result;
  } 
  result->as_vec = createVectorMPI(N1*N2, 0, comm);
  int n12 = n1;
  if (n1 == -1)
    n1 = result->as_vec->len/N2;
  else
    n2 = result->as_vec->len/N1;

  result->rows = n1;
  result->cols = n2;
  result->glob_rows = N1;
  result->glob_cols = N2;
  result->data = (double **)calloc(n2   ,sizeof(double *));
  result->data[0] = (double  *)calloc(n1*n2,sizeof(double));
  result->as_vec->data = result->data[0];
  for (i=1; i < n2; i++)
    result->data[i] = result->data[i-1] + n1;
  result->col = malloc(n2*sizeof(Vector));
  for (int i=0;i<n2;++i) {
    if (n12 == N1) {
      result->col[i] = createVectorMPI(N1, 0, &SelfComm);
    } else {
      result->col[i] = createVectorMPI(N1, 0, comm);
    }
    result->col[i]->data = result->data[i];
  }
  result->row = malloc(n1*sizeof(Vector));
  for (int i=0;i<n1;++i) {
    if (n12 == N1)
      result->row[i] = createVectorMPI(N2, 0, comm);
    else
      result->row[i] = createVectorMPI(N2, 0, &SelfComm);
    result->row[i]->data = result->data[0]+i;
    result->row[i]->stride = n1;
  }

  return result;
}
Exemple #16
0
void mpi_manager_2D::setup(NumArray<int> &nproc, NumArray<int> &mx) {
	
	// Determine the rank of the current task
	MPI_Comm_rank(MPI_COMM_WORLD, &rank);

	// Get number of ranks from MPI
	int ntasks;
	MPI_Comm_size(MPI_COMM_WORLD, &ntasks);
	this->ntasks = ntasks;

	// Set the distribution of processes:
	if(ntasks != nproc[0]*nproc[1]){
		std::cerr << " Wrong number of processes " << std::endl;
		std::cout << ntasks << " " << nproc[0]*nproc[1] << std::endl;
		Finalise();
	}

	if(rank==0) {
		std::cout << " Number of tasks: " << ntasks << " " << nproc[0] << " " << nproc[1] << std::endl;
	}

	// Check if grid can be subdevided as desired
	for(int dir = 0; dir < DIM; ++dir) {
		if(mx[dir] < nproc[dir] && nproc[dir] > 1) {
			if(rank == 0) {
				std::cerr << " Wrong grid topology for dimension ";
				std::cerr << dir << std::endl;
				std::cerr << "  mx[" << dir << "]:" << mx[dir] << std::endl;
				std::cerr << " nproc[" << dir << "]:" << nproc[dir] << std::endl;
			}
			Finalise();
		}
	}

	// Check if grid is a power of 2:
	double eps = 1.e-12;
	for(int dir = 0; dir < DIM; ++dir) {
		double exponent = log(mx[dir])/log(2.);
		int i_exponent = static_cast<int>(exponent+eps);

		if(exponent - i_exponent > 2.*eps) {
			if(rank == 0) {
				std::cerr << " Error: grid must be of the form mx = 2^n ";
				std::cerr << std::endl;
				std::cerr << " Exiting " << std::endl;
			}
			Finalise();
		}
	}

	// Grid is not periodic
	int periods[3] = {false, false, false};
	int reorder = false;
	// If all is okay: Create new communicator "comm3d"  
	MPI_Cart_create(MPI_COMM_WORLD, DIM, nproc, periods, reorder, &comm2d);

	// Retrieve the cartesian topology
	if (rank == 0) {
		int TopoType;
		std::cout << " Cart topology:  ";
		MPI_Topo_test(comm2d, &TopoType);
		switch (TopoType) {
		case MPI_UNDEFINED : 
			std::cout << " MPI_UNDEFINED " << std::endl;
			break;
		case MPI_GRAPH     :
			std::cout << "MPI_GRAPH" << std::endl;
			break;
		case MPI_CART      :
			std::cout << "MPI_CART" << std::endl;
			break;
		}
	}
	
	//   Determine rank again for cartesian communicator -> overwrite rank
	MPI_Comm_rank(comm2d, &rank);

	// std::cout << " my rank: " << rank << std::endl;

	// Translate rank to coordinates
	MPI_Cart_coords(comm2d, rank, DIM, coords);

	// // Backwards translation
	// int TranslateRank;
	// MPI_Cart_rank(comm3d, coords, &TranslateRank);

	// Find neighbouring ranks
	// Syntax: comm3d, shift direction, displacement, source, destination
	MPI_Cart_shift(comm2d, 0, 1, &left , &right);
	MPI_Cart_shift(comm2d, 1, 1, &front, &back);

	// std::cout << " My rank " << rank << " " << left << " " << right << " " << front << " " << back << " " << bottom << " " << top << std::endl;
	// Determine position of other ranks
	determin_OtherRanks();
	get_SubComms();

	

}
Exemple #17
0
  void Communication::prepare(p3m_float box_l[3]) {
    P3M_DEBUG(printf( "  P3M::Communication::prepare() started...\n"));

    /* Test whether the communicator is cartesian and correct dimensionality */
    bool comm_is_cart = false;
    int status;

    MPI_Topo_test(mpicomm_orig, &status);
    if (status == MPI_CART) {
      /* Communicator is cartesian, so test dimensionality */
      int ndims;
      MPI_Cartdim_get(mpicomm_orig, &ndims);
      if (ndims == 3) {
        /* Correct dimensionality, so get grid and test periodicity */
        int periodicity[3];
        MPI_Cart_get(mpicomm_orig, 3, node_grid,
                     periodicity, node_pos);
        if (periodicity[0] && periodicity[1] && periodicity[2]) {
          /* If periodicity is correct, we can just use this communicator */
          mpicomm = mpicomm_orig;
          /* get the rank */
          MPI_Comm_rank(mpicomm, &rank);
          comm_is_cart = true;
        }
      }
    }

    /* otherwise, we have to set up the cartesian communicator */
    if (!comm_is_cart) {
      P3M_DEBUG(printf( "    Setting up cartesian communicator...\n"));

      node_grid[0] = 0;
      node_grid[1] = 0;
      node_grid[2] = 0;

      /* compute node grid */
      MPI_Dims_create(size, 3, node_grid);

#ifdef P3M_ENABLE_INFO
      if (onMaster())
    	  printf("    node_grid=%dx%dx%d\n", node_grid[0], node_grid[1], node_grid[2]);
#endif

      /* create communicator */
      int periodicity[3] = {1, 1, 1};
      MPI_Cart_create(mpicomm_orig, 3, node_grid,
                      periodicity, 1, &mpicomm);

      /* get the rank */
      MPI_Comm_rank(mpicomm, &rank);
      /* get node pos */
      MPI_Cart_coords(mpicomm, rank, 3, node_pos);
    }
    
    /* fetch neighborhood info */
    for (int dir = 0; dir < 3; dir++) {
      MPI_Cart_shift(mpicomm, dir, 1,
                     &node_neighbors[2*dir],
                     &node_neighbors[2*dir+1]);
      P3M_DEBUG_LOCAL(printf( "    %d: dir=%d: n1=%d n2=%d\n", rank, dir, \
                              node_neighbors[2*dir],		\
                              node_neighbors[2*dir+1]));
    }

    /* init local points */
    for (int i=0; i< 3; i++) {
      local_box_l[i] = 0.0;
      my_left[i] = 0.0;
      my_right[i] = 0.0;
    }

    /* compute box limits */
    for(p3m_int i = 0; i < 3; i++) {
      local_box_l[i] = box_l[i]/(p3m_float)node_grid[i];
      my_left[i]   = node_pos[i]    *local_box_l[i];
      my_right[i]  = (node_pos[i]+1)*local_box_l[i];
    }
    P3M_DEBUG(printf("    local_box_l=" F3FLOAT "\n"                      \
                     "    my_left=" F3FLOAT "\n"                          \
                     "    my_right=" F3FLOAT "\n",                        \
                     local_box_l[0],                                \
                     local_box_l[1],                                \
                     local_box_l[2],                                \
                     my_left[0], my_left[1], my_left[2], \
                     my_right[0], my_right[1], my_right[2] \
                     ));

    P3M_DEBUG(printf("  P3M::Communication::prepare() finished.\n"));
  }
Exemple #18
0
FORT_DLL_SPEC void FORT_CALL mpi_topo_test_ ( MPI_Fint *v1, MPI_Fint *v2, MPI_Fint *ierr ){
    *ierr = MPI_Topo_test( (MPI_Comm)(*v1), v2 );
}