static int
mca_coll_basic_neighbor_alltoallv_graph(const void *sbuf, const int scounts[], const int sdisps[],
struct ompi_datatype_t *sdtype, void *rbuf, const int rcounts[],
const int rdisps[], struct ompi_datatype_t *rdtype,
struct ompi_communicator_t *comm, mca_coll_base_module_t *module)
{
mca_coll_basic_module_t *basic_module = (mca_coll_basic_module_t *) module;
const mca_topo_base_comm_graph_2_2_0_t *graph = comm->c_topo->mtc.graph;
int rc = MPI_SUCCESS, neighbor, degree;
const int rank = ompi_comm_rank (comm);
ptrdiff_t lb, rdextent, sdextent;
ompi_request_t **reqs;
const int *edges;
mca_topo_base_graph_neighbors_count (comm, rank, °ree);
/* ensure we have enough storage for requests */
rc = mca_coll_basic_check_for_requests (basic_module, 2 * degree);
if (OMPI_SUCCESS != rc) {
return rc;
}
edges = graph->edges;
if (rank > 0) {
edges += graph->index[rank - 1];
}
ompi_datatype_get_extent(rdtype, &lb, &rdextent);
ompi_datatype_get_extent(sdtype, &lb, &sdextent);
/* post all receives first */
for (neighbor = 0, reqs = basic_module->mccb_reqs ; neighbor < degree ; ++neighbor) {
rc = MCA_PML_CALL(irecv((char *) rbuf + rdisps[neighbor] * rdextent, rcounts[neighbor], rdtype,
edges[neighbor], MCA_COLL_BASE_TAG_ALLTOALL, comm, reqs++));
if (OMPI_SUCCESS != rc) break;
}
if (OMPI_SUCCESS != rc) {
/* should probably try to clean up here */
return rc;
}
for (neighbor = 0 ; neighbor < degree ; ++neighbor) {
/* remove cast from const when the pml layer is updated to take a const for the send buffer */
rc = MCA_PML_CALL(isend((char *) sbuf + sdisps[neighbor] * sdextent, scounts[neighbor], sdtype,
edges[neighbor], MCA_COLL_BASE_TAG_ALLTOALL, MCA_PML_BASE_SEND_STANDARD,
comm, reqs++));
if (OMPI_SUCCESS != rc) break;
}
if (OMPI_SUCCESS != rc) {
/* should probably try to clean up here */
return rc;
}
return ompi_request_wait_all (degree * 2, basic_module->mccb_reqs, MPI_STATUSES_IGNORE);
}
static int
mca_coll_basic_neighbor_alltoallv_graph(const void *sbuf, const int scounts[], const int sdisps[],
struct ompi_datatype_t *sdtype, void *rbuf, const int rcounts[],
const int rdisps[], struct ompi_datatype_t *rdtype,
struct ompi_communicator_t *comm, mca_coll_base_module_t *module)
{
const mca_topo_base_comm_graph_2_2_0_t *graph = comm->c_topo->mtc.graph;
int rc = MPI_SUCCESS, neighbor, degree;
const int rank = ompi_comm_rank (comm);
ptrdiff_t lb, rdextent, sdextent;
ompi_request_t **reqs, **preqs;
const int *edges;
mca_topo_base_graph_neighbors_count (comm, rank, °ree);
if( 0 == degree ) return OMPI_SUCCESS;
edges = graph->edges;
if (rank > 0) {
edges += graph->index[rank - 1];
}
ompi_datatype_get_extent(rdtype, &lb, &rdextent);
ompi_datatype_get_extent(sdtype, &lb, &sdextent);
reqs = preqs = ompi_coll_base_comm_get_reqs( module->base_data, 2 * degree );
if( NULL == reqs ) { return OMPI_ERR_OUT_OF_RESOURCE; }
/* post all receives first */
for (neighbor = 0; neighbor < degree ; ++neighbor) {
rc = MCA_PML_CALL(irecv((char *) rbuf + rdisps[neighbor] * rdextent, rcounts[neighbor], rdtype,
edges[neighbor], MCA_COLL_BASE_TAG_ALLTOALL, comm, preqs++));
if (OMPI_SUCCESS != rc) break;
}
if (OMPI_SUCCESS != rc) {
ompi_coll_base_free_reqs( reqs, neighbor + 1);
return rc;
}
for (neighbor = 0 ; neighbor < degree ; ++neighbor) {
/* remove cast from const when the pml layer is updated to take a const for the send buffer */
rc = MCA_PML_CALL(isend((char *) sbuf + sdisps[neighbor] * sdextent, scounts[neighbor], sdtype,
edges[neighbor], MCA_COLL_BASE_TAG_ALLTOALL, MCA_PML_BASE_SEND_STANDARD,
comm, preqs++));
if (OMPI_SUCCESS != rc) break;
}
if (OMPI_SUCCESS != rc) {
ompi_coll_base_free_reqs( reqs, degree + neighbor + 1);
return rc;
}
rc = ompi_request_wait_all (degree * 2, reqs, MPI_STATUSES_IGNORE);
if (OMPI_SUCCESS != rc) {
ompi_coll_base_free_reqs( reqs, degree * 2);
}
return rc;
}
static int
mca_coll_basic_neighbor_allgather_graph(const void *sbuf, int scount,
struct ompi_datatype_t *sdtype, void *rbuf,
int rcount, struct ompi_datatype_t *rdtype,
struct ompi_communicator_t *comm,
mca_coll_base_module_t *module)
{
const mca_topo_base_comm_graph_2_2_0_t *graph = comm->c_topo->mtc.graph;
const int rank = ompi_comm_rank (comm);
const int *edges;
int degree;
ompi_request_t **reqs, **preqs;
ptrdiff_t lb, extent;
int rc = MPI_SUCCESS, neighbor;
mca_topo_base_graph_neighbors_count (comm, rank, °ree);
edges = graph->edges;
if (rank > 0) {
edges += graph->index[rank - 1];
}
ompi_datatype_get_extent(rdtype, &lb, &extent);
reqs = preqs = coll_base_comm_get_reqs( module->base_data, 2 * degree);
if( NULL == reqs ) { return OMPI_ERR_OUT_OF_RESOURCE; }
for (neighbor = 0; neighbor < degree ; ++neighbor) {
rc = MCA_PML_CALL(irecv(rbuf, rcount, rdtype, edges[neighbor], MCA_COLL_BASE_TAG_ALLGATHER,
comm, preqs++));
if (OMPI_SUCCESS != rc) break;
rbuf = (char *) rbuf + extent * rcount;
/* remove cast from const when the pml layer is updated to take
* a const for the send buffer. */
rc = MCA_PML_CALL(isend((void *) sbuf, scount, sdtype, edges[neighbor],
MCA_COLL_BASE_TAG_ALLGATHER, MCA_PML_BASE_SEND_STANDARD,
comm, preqs++));
if (OMPI_SUCCESS != rc) break;
}
if (OMPI_SUCCESS != rc) {
ompi_coll_base_free_reqs( reqs, (2 * neighbor + 1));
return rc;
}
rc = ompi_request_wait_all (degree * 2, reqs, MPI_STATUSES_IGNORE);
if (OMPI_SUCCESS != rc) {
ompi_coll_base_free_reqs( reqs, degree * 2);
}
return rc;
}
int NBC_Comm_neighbors_count (ompi_communicator_t *comm, int *indegree, int *outdegree) {
if (OMPI_COMM_IS_CART(comm)) {
/* cartesian */
/* outdegree is always 2*ndims because we need to iterate over empty buffers for MPI_PROC_NULL */
*outdegree = *indegree = 2 * comm->c_topo->mtc.cart->ndims;
} else if (OMPI_COMM_IS_GRAPH(comm)) {
/* graph */
int rank, nneighbors;
rank = ompi_comm_rank (comm);
mca_topo_base_graph_neighbors_count (comm, rank, &nneighbors);
*outdegree = *indegree = nneighbors;
} else if (OMPI_COMM_IS_DIST_GRAPH(comm)) {
/* graph */
*indegree = comm->c_topo->mtc.dist_graph->indegree;
*outdegree = comm->c_topo->mtc.dist_graph->outdegree;
} else {
return OMPI_ERR_BAD_PARAM;
}
return OMPI_SUCCESS;
}
