int mca_coll_monitoring_ineighbor_alltoall(const void *sbuf, int scount, struct ompi_datatype_t *sdtype, void *rbuf, int rcount, struct ompi_datatype_t *rdtype, struct ompi_communicator_t *comm, ompi_request_t ** request, mca_coll_base_module_t *module) { mca_coll_monitoring_module_t*monitoring_module = (mca_coll_monitoring_module_t*) module; size_t type_size, data_size, data_size_aggreg = 0; const mca_topo_base_comm_cart_t *cart = comm->c_topo->mtc.cart; int dim, srank, drank, world_rank; ompi_datatype_type_size(sdtype, &type_size); data_size = scount * type_size; for( dim = 0; dim < cart->ndims; ++dim ) { srank = MPI_PROC_NULL, drank = MPI_PROC_NULL; if (cart->dims[dim] > 1) { mca_topo_base_cart_shift (comm, dim, 1, &srank, &drank); } else if (1 == cart->dims[dim] && cart->periods[dim]) { /* Don't record exchanges with self */ continue; } if (MPI_PROC_NULL != srank) { /** * If this fails the destination is not part of my MPI_COM_WORLD * Lookup its name in the rank hastable to get its MPI_COMM_WORLD rank */ if( OPAL_SUCCESS == mca_common_monitoring_get_world_rank(srank, comm, &world_rank) ) { mca_common_monitoring_record_coll(world_rank, data_size); data_size_aggreg += data_size; } } if (MPI_PROC_NULL != drank) { /** * If this fails the destination is not part of my MPI_COM_WORLD * Lookup its name in the rank hastable to get its MPI_COMM_WORLD rank */ if( OPAL_SUCCESS == mca_common_monitoring_get_world_rank(drank, comm, &world_rank) ) { mca_common_monitoring_record_coll(world_rank, data_size); data_size_aggreg += data_size; } } } mca_common_monitoring_coll_a2a(data_size_aggreg, monitoring_module->data); return monitoring_module->real.coll_ineighbor_alltoall(sbuf, scount, sdtype, rbuf, rcount, rdtype, comm, request, monitoring_module->real.coll_ineighbor_alltoall_module); }
static int mca_coll_basic_neighbor_allgather_cart(const void *sbuf, int scount, struct ompi_datatype_t *sdtype, void *rbuf, int rcount, struct ompi_datatype_t *rdtype, struct ompi_communicator_t *comm, mca_coll_base_module_t *module) { const mca_topo_base_comm_cart_2_2_0_t *cart = comm->c_topo->mtc.cart; const int rank = ompi_comm_rank (comm); ompi_request_t **reqs, **preqs; ptrdiff_t lb, extent; int rc = MPI_SUCCESS, dim, nreqs; if( 0 == cart->ndims ) return OMPI_SUCCESS; ompi_datatype_get_extent(rdtype, &lb, &extent); reqs = preqs = coll_base_comm_get_reqs( module->base_data, 4 * cart->ndims ); if( NULL == reqs ) { return OMPI_ERR_OUT_OF_RESOURCE; } /* The ordering is defined as -1 then +1 in each dimension in * order of dimension. */ for (dim = 0, nreqs = 0 ; dim < cart->ndims ; ++dim) { int srank = MPI_PROC_NULL, drank = MPI_PROC_NULL; if (cart->dims[dim] > 1) { mca_topo_base_cart_shift (comm, dim, 1, &srank, &drank); } else if (1 == cart->dims[dim] && cart->periods[dim]) { srank = drank = rank; } if (MPI_PROC_NULL != srank) { nreqs++; rc = MCA_PML_CALL(irecv(rbuf, rcount, rdtype, srank, MCA_COLL_BASE_TAG_ALLGATHER, comm, preqs++)); if (OMPI_SUCCESS != rc) break; nreqs++; /* remove cast from const when the pml layer is updated to take * a const for the send buffer. */ rc = MCA_PML_CALL(isend((void *) sbuf, scount, sdtype, srank, MCA_COLL_BASE_TAG_ALLGATHER, MCA_PML_BASE_SEND_STANDARD, comm, preqs++)); if (OMPI_SUCCESS != rc) break; } rbuf = (char *) rbuf + extent * rcount; if (MPI_PROC_NULL != drank) { nreqs++; rc = MCA_PML_CALL(irecv(rbuf, rcount, rdtype, drank, MCA_COLL_BASE_TAG_ALLGATHER, comm, preqs++)); if (OMPI_SUCCESS != rc) break; nreqs++; rc = MCA_PML_CALL(isend((void *) sbuf, scount, sdtype, drank, MCA_COLL_BASE_TAG_ALLGATHER, MCA_PML_BASE_SEND_STANDARD, comm, preqs++)); if (OMPI_SUCCESS != rc) break; } rbuf = (char *) rbuf + extent * rcount; } if (OMPI_SUCCESS != rc) { ompi_coll_base_free_reqs(reqs, nreqs); return rc; } rc = ompi_request_wait_all (nreqs, reqs, MPI_STATUSES_IGNORE); if (OMPI_SUCCESS != rc) { ompi_coll_base_free_reqs(reqs, nreqs); } return rc; }
static int mca_coll_basic_neighbor_alltoallv_cart(const void *sbuf, const int scounts[], const int sdisps[], struct ompi_datatype_t *sdtype, void *rbuf, const int rcounts[], const int rdisps[], struct ompi_datatype_t *rdtype, struct ompi_communicator_t *comm, mca_coll_base_module_t *module) { const mca_topo_base_comm_cart_2_2_0_t *cart = comm->c_topo->mtc.cart; const int rank = ompi_comm_rank (comm); int rc = MPI_SUCCESS, dim, i, nreqs; ptrdiff_t lb, rdextent, sdextent; ompi_request_t **reqs, **preqs; if( 0 == cart->ndims ) return OMPI_SUCCESS; ompi_datatype_get_extent(rdtype, &lb, &rdextent); ompi_datatype_get_extent(sdtype, &lb, &sdextent); reqs = preqs = ompi_coll_base_comm_get_reqs( module->base_data, 4 * cart->ndims ); if( NULL == reqs ) { return OMPI_ERR_OUT_OF_RESOURCE; } /* post receives first */ for (dim = 0, nreqs = 0, i = 0; dim < cart->ndims ; ++dim, i += 2) { int srank = MPI_PROC_NULL, drank = MPI_PROC_NULL; if (cart->dims[dim] > 1) { mca_topo_base_cart_shift (comm, dim, 1, &srank, &drank); } else if (1 == cart->dims[dim] && cart->periods[dim]) { srank = drank = rank; } if (MPI_PROC_NULL != srank) { nreqs++; rc = MCA_PML_CALL(irecv((char *) rbuf + rdisps[i] * rdextent, rcounts[i], rdtype, srank, MCA_COLL_BASE_TAG_ALLTOALL, comm, preqs++)); if (OMPI_SUCCESS != rc) break; } if (MPI_PROC_NULL != drank) { nreqs++; rc = MCA_PML_CALL(irecv((char *) rbuf + rdisps[i+1] * rdextent, rcounts[i+1], rdtype, drank, MCA_COLL_BASE_TAG_ALLTOALL, comm, preqs++)); if (OMPI_SUCCESS != rc) break; } } if (OMPI_SUCCESS != rc) { ompi_coll_base_free_reqs( reqs, nreqs ); return rc; } for (dim = 0, i = 0 ; dim < cart->ndims ; ++dim, i += 2) { int srank = MPI_PROC_NULL, drank = MPI_PROC_NULL; if (cart->dims[dim] > 1) { mca_topo_base_cart_shift (comm, dim, 1, &srank, &drank); } else if (1 == cart->dims[dim] && cart->periods[dim]) { srank = drank = rank; } if (MPI_PROC_NULL != srank) { nreqs++; /* remove cast from const when the pml layer is updated to take a const for the send buffer */ rc = MCA_PML_CALL(isend((char *) sbuf + sdisps[i] * sdextent, scounts[i], sdtype, srank, MCA_COLL_BASE_TAG_ALLTOALL, MCA_PML_BASE_SEND_STANDARD, comm, preqs++)); if (OMPI_SUCCESS != rc) break; } if (MPI_PROC_NULL != drank) { nreqs++; rc = MCA_PML_CALL(isend((char *) sbuf + sdisps[i+1] * sdextent, scounts[i+1], sdtype, drank, MCA_COLL_BASE_TAG_ALLTOALL, MCA_PML_BASE_SEND_STANDARD, comm, preqs++)); if (OMPI_SUCCESS != rc) break; } } if (OMPI_SUCCESS != rc) { ompi_coll_base_free_reqs( reqs, nreqs ); return rc; } rc = ompi_request_wait_all (nreqs, reqs, MPI_STATUSES_IGNORE); if (OMPI_SUCCESS != rc) { ompi_coll_base_free_reqs( reqs, nreqs ); } return rc; }
static int mca_coll_basic_neighbor_alltoallv_cart(const void *sbuf, const int scounts[], const int sdisps[], struct ompi_datatype_t *sdtype, void *rbuf, const int rcounts[], const int rdisps[], struct ompi_datatype_t *rdtype, struct ompi_communicator_t *comm, mca_coll_base_module_t *module) { mca_coll_basic_module_t *basic_module = (mca_coll_basic_module_t *) module; const mca_topo_base_comm_cart_2_2_0_t *cart = comm->c_topo->mtc.cart; const int rank = ompi_comm_rank (comm); int rc = MPI_SUCCESS, dim, i, nreqs; ptrdiff_t lb, rdextent, sdextent; ompi_request_t **reqs; /* ensure we have enough storage for requests */ rc = mca_coll_basic_check_for_requests (basic_module, cart->ndims * 4); if (OMPI_SUCCESS != rc) { return rc; } ompi_datatype_get_extent(rdtype, &lb, &rdextent); ompi_datatype_get_extent(sdtype, &lb, &sdextent); /* post receives first */ for (dim = 0, nreqs = 0, i = 0, reqs = basic_module->mccb_reqs ; dim < cart->ndims ; ++dim, i += 2) { int srank = MPI_PROC_NULL, drank = MPI_PROC_NULL; if (cart->dims[dim] > 1) { mca_topo_base_cart_shift (comm, dim, 1, &srank, &drank); } else if (1 == cart->dims[dim] && cart->periods[dim]) { srank = drank = rank; } if (MPI_PROC_NULL != srank) { rc = MCA_PML_CALL(irecv((char *) rbuf + rdisps[i] * rdextent, rcounts[i], rdtype, srank, MCA_COLL_BASE_TAG_ALLTOALL, comm, reqs++)); if (OMPI_SUCCESS != rc) break; nreqs++; } if (MPI_PROC_NULL != drank) { rc = MCA_PML_CALL(irecv((char *) rbuf + rdisps[i+1] * rdextent, rcounts[i+1], rdtype, drank, MCA_COLL_BASE_TAG_ALLTOALL, comm, reqs++)); if (OMPI_SUCCESS != rc) break; nreqs++; } } if (OMPI_SUCCESS != rc) { /* should probably try to clean up here */ return rc; } for (dim = 0, i = 0 ; dim < cart->ndims ; ++dim, i += 2) { int srank = MPI_PROC_NULL, drank = MPI_PROC_NULL; if (cart->dims[dim] > 1) { mca_topo_base_cart_shift (comm, dim, 1, &srank, &drank); } else if (1 == cart->dims[dim] && cart->periods[dim]) { srank = drank = rank; } if (MPI_PROC_NULL != srank) { /* remove cast from const when the pml layer is updated to take a const for the send buffer */ rc = MCA_PML_CALL(isend((char *) sbuf + sdisps[i] * sdextent, scounts[i], sdtype, srank, MCA_COLL_BASE_TAG_ALLTOALL, MCA_PML_BASE_SEND_STANDARD, comm, reqs++)); if (OMPI_SUCCESS != rc) break; nreqs++; } if (MPI_PROC_NULL != drank) { rc = MCA_PML_CALL(isend((char *) sbuf + sdisps[i+1] * sdextent, scounts[i+1], sdtype, drank, MCA_COLL_BASE_TAG_ALLTOALL, MCA_PML_BASE_SEND_STANDARD, comm, reqs++)); if (OMPI_SUCCESS != rc) break; nreqs++; } } if (OMPI_SUCCESS != rc) { /* should probably try to clean up here */ return rc; } return ompi_request_wait_all (nreqs, basic_module->mccb_reqs, MPI_STATUSES_IGNORE); }
int NBC_Comm_neighbors (ompi_communicator_t *comm, int **sources, int *source_count, int **destinations, int *dest_count) { int res, indeg, outdeg; *sources = *destinations = NULL; res = NBC_Comm_neighbors_count(comm, &indeg, &outdeg); if (OMPI_SUCCESS != res) { return res; } *source_count = indeg; *dest_count = outdeg; if (indeg) { *sources = malloc (sizeof (int) * indeg); if (OPAL_UNLIKELY(NULL == *sources)) { return OMPI_ERR_OUT_OF_RESOURCE; } } else { *sources = NULL; } if (outdeg) { *destinations = malloc (sizeof (int) * outdeg); if (OPAL_UNLIKELY(NULL == *destinations)) { free (*sources); *sources = NULL; return OMPI_ERR_OUT_OF_RESOURCE; } } else { *destinations = NULL; } /* silence clang static analyzer warning about NULL-dereference */ if (0 == indeg && 0 == outdeg) { return OMPI_SUCCESS; } if (OMPI_COMM_IS_CART(comm)) { /* cartesian */ int rpeer, speer; /* silence clang static analyzer warning */ assert (indeg == outdeg); for (int dim = 0, i = 0 ; dim < comm->c_topo->mtc.cart->ndims ; ++dim) { mca_topo_base_cart_shift (comm, dim, 1, &rpeer, &speer); sources[0][i] = destinations[0][i] = rpeer; i++; sources[0][i] = destinations[0][i] = speer; i++; } } else if (OMPI_COMM_IS_GRAPH(comm)) { /* graph */ mca_topo_base_graph_neighbors (comm, ompi_comm_rank (comm), indeg, sources[0]); memcpy (destinations[0], sources[0], indeg * sizeof (int)); } else if (OMPI_COMM_IS_DIST_GRAPH(comm)) { /* dist graph */ mca_topo_base_dist_graph_neighbors (comm, indeg, sources[0], MPI_UNWEIGHTED, outdeg, destinations[0], MPI_UNWEIGHTED); } return OMPI_SUCCESS; }