int NBC_Comm_neighbors(MPI_Comm comm, int maxindegree, int sources[], int sourceweights[], int maxoutdegree, int destinations[], int destweights[]) {
int topo, res;
int index = 0;
int indeg, outdeg, wgtd;
res = NBC_Comm_neighbors_count(comm, &indeg, &outdeg, &wgtd);
if(indeg > maxindegree && outdeg > maxoutdegree) return NBC_INVALID_PARAM; /* we want to return *all* neighbors */
res = MPI_Topo_test(comm, &topo);
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Topo_test() (%i)\n", res); return res; }
switch(topo) {
case MPI_CART: /* cartesian */
{
int ndims, i, rpeer, speer;
res = MPI_Cartdim_get(comm, &ndims);
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Cartdim_get() (%i)\n", res); return res; }
for(i = 0; i<ndims; i++) {
res = MPI_Cart_shift(comm, i, 1, &rpeer, &speer);
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Cart_shift() (%i)\n", res); return res; }
sources[index] = destinations[index] = rpeer; index++;
sources[index] = destinations[index] = speer; index++;
}
}
break;
case MPI_GRAPH: /* graph */
{
int rank;
MPI_Comm_rank(comm, &rank);
res = MPI_Graph_neighbors(comm, rank, maxindegree, sources);
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Graph_neighbors_count() (%i)\n", res); return res; }
for(int i=0; i<maxindegree; i++) destinations[i] = sources[i];
}
break;
case MPI_DIST_GRAPH: /* dist graph */
{
res = MPI_Dist_graph_neighbors(comm, maxindegree, sources, sourceweights, maxoutdegree, destinations, destweights);
if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Graph_neighbors_count() (%i)\n", res); return res; }
}
break;
case MPI_UNDEFINED:
return NBC_INVALID_TOPOLOGY_COMM;
break;
default:
return NBC_INVALID_PARAM;
break;
}
return NBC_OK;
}
void ompi_dist_graph_neighbors_f(MPI_Fint* comm, MPI_Fint* maxindegree,
MPI_Fint* sources, MPI_Fint* sourceweights,
MPI_Fint* maxoutdegree, MPI_Fint* destinations,
MPI_Fint* destweights,
MPI_Fint *ierr)
{
MPI_Comm c_comm;
OMPI_ARRAY_NAME_DECL(sources);
OMPI_ARRAY_NAME_DECL(sourceweights);
OMPI_ARRAY_NAME_DECL(destinations);
OMPI_ARRAY_NAME_DECL(destweights);
c_comm = MPI_Comm_f2c(*comm);
OMPI_ARRAY_FINT_2_INT_ALLOC(sources, *maxindegree);
if( !OMPI_IS_FORTRAN_UNWEIGHTED(sourceweights) ) {
OMPI_ARRAY_FINT_2_INT_ALLOC(sourceweights, *maxindegree);
}
OMPI_ARRAY_FINT_2_INT_ALLOC(destinations, *maxoutdegree);
if( !OMPI_IS_FORTRAN_UNWEIGHTED(destweights) ) {
OMPI_ARRAY_FINT_2_INT_ALLOC(destweights, *maxoutdegree);
}
*ierr = OMPI_INT_2_FINT(MPI_Dist_graph_neighbors(c_comm, OMPI_FINT_2_INT(*maxindegree),
OMPI_ARRAY_NAME_CONVERT(sources),
OMPI_IS_FORTRAN_UNWEIGHTED(sourceweights) ? MPI_UNWEIGHTED : OMPI_ARRAY_NAME_CONVERT(sourceweights),
OMPI_FINT_2_INT(*maxoutdegree), OMPI_ARRAY_NAME_CONVERT(destinations),
OMPI_IS_FORTRAN_UNWEIGHTED(destweights) ? MPI_UNWEIGHTED : OMPI_ARRAY_NAME_CONVERT(destweights)));
if (OMPI_SUCCESS == OMPI_FINT_2_INT(*ierr)) {
OMPI_ARRAY_INT_2_FINT(sources, *maxindegree);
if( !OMPI_IS_FORTRAN_UNWEIGHTED(sourceweights) ) {
OMPI_ARRAY_INT_2_FINT(sourceweights, *maxindegree);
}
OMPI_ARRAY_INT_2_FINT(destinations, *maxoutdegree);
if( !OMPI_IS_FORTRAN_UNWEIGHTED(destweights) ) {
OMPI_ARRAY_INT_2_FINT(destweights, *maxoutdegree);
}
} else {
OMPI_ARRAY_FINT_2_INT_CLEANUP(sources);
if( !OMPI_IS_FORTRAN_UNWEIGHTED(sourceweights) ) {
OMPI_ARRAY_FINT_2_INT_CLEANUP(sourceweights);
}
OMPI_ARRAY_FINT_2_INT_CLEANUP(destinations);
if( !OMPI_IS_FORTRAN_UNWEIGHTED(destweights) ) {
OMPI_ARRAY_FINT_2_INT_CLEANUP(destweights);
}
}
}
Comm::Comm(MPI_Comm impl) : impl_(impl) {
int topo_type;
CALL(MPI_Topo_test(impl, &topo_type));
if (topo_type == MPI_DIST_GRAPH) {
int nin, nout, is_weighted;
CALL(MPI_Dist_graph_neighbors_count(impl, &nin, &nout, &is_weighted));
HostWrite<I32> sources(nin);
HostWrite<I32> destinations(nout);
CALL(MPI_Dist_graph_neighbors(impl, nin, sources.data(),
OMEGA_H_MPI_UNWEIGHTED, nout, destinations.data(),
OMEGA_H_MPI_UNWEIGHTED));
srcs_ = sources.write();
dsts_ = destinations.write();
host_srcs_ = HostRead<I32>(srcs_);
host_dsts_ = HostRead<I32>(dsts_);
}
}
int ompi_comm_neighbors(MPI_Comm comm, int maxindegree, int sources[], int sourceweights[], int maxoutdegree, int destinations[], int destweights[]) {
int res;
int index = 0;
int indeg, outdeg, wgtd;
res = ompi_comm_neighbors_count(comm, &indeg, &outdeg, &wgtd);
if (MPI_SUCCESS != res) {
return res;
}
if(indeg > maxindegree && outdeg > maxoutdegree) return MPI_ERR_TRUNCATE; /* we want to return *all* neighbors */
if (OMPI_COMM_IS_CART(comm)) {
int ndims, i, rpeer, speer;
res = MPI_Cartdim_get(comm, &ndims);
if (MPI_SUCCESS != res) {
return res;
}
for(i = 0; i<ndims; i++) {
res = MPI_Cart_shift(comm, i, 1, &rpeer, &speer);
if (MPI_SUCCESS != res) {
return res;
}
sources[index] = destinations[index] = rpeer; index++;
sources[index] = destinations[index] = speer; index++;
}
} else if (OMPI_COMM_IS_GRAPH(comm)) {
int rank = ompi_comm_rank ((ompi_communicator_t *) comm);
res = MPI_Graph_neighbors(comm, rank, maxindegree, sources);
if (MPI_SUCCESS != res) {
return res;
}
for(int i=0; i<maxindegree; i++) destinations[i] = sources[i];
} else if (OMPI_COMM_IS_DIST_GRAPH(comm)) {
res = MPI_Dist_graph_neighbors(comm, maxindegree, sources, sourceweights, maxoutdegree, destinations, destweights);
if (MPI_SUCCESS != res) {
return res;
}
} else {
return MPI_ERR_ARG;
}
return MPI_SUCCESS;
}
JNIEXPORT jobject JNICALL Java_mpi_GraphComm_getDistGraphNeighbors(
JNIEnv *env, jobject jthis, jlong comm)
{
int inDegree, outDegree, weighted;
int rc = MPI_Dist_graph_neighbors_count(
(MPI_Comm)comm, &inDegree, &outDegree, &weighted);
if(ompi_java_exceptionCheck(env, rc))
return NULL;
jintArray sources = (*env)->NewIntArray(env, inDegree),
srcWeights = (*env)->NewIntArray(env, inDegree),
destinations = (*env)->NewIntArray(env, outDegree),
destWeights = (*env)->NewIntArray(env, outDegree);
jint *jSources, *jSrcWeights, *jDestinations, *jDestWeights;
int *cSources, *cSrcWeights, *cDestinations, *cDestWeights;
ompi_java_getIntArray(env, sources, &jSources, &cSources);
ompi_java_getIntArray(env, srcWeights, &jSrcWeights, &cSrcWeights);
ompi_java_getIntArray(env, destinations, &jDestinations, &cDestinations);
ompi_java_getIntArray(env, destWeights, &jDestWeights, &cDestWeights);
rc = MPI_Dist_graph_neighbors((MPI_Comm)comm,
inDegree, cSources, cSrcWeights,
outDegree, cDestinations, cDestWeights);
ompi_java_exceptionCheck(env, rc);
ompi_java_releaseIntArray(env, sources, jSources, cSources);
ompi_java_releaseIntArray(env, srcWeights, jSrcWeights, cSrcWeights);
ompi_java_releaseIntArray(env, destinations, jDestinations, cDestinations);
ompi_java_releaseIntArray(env, destWeights, jDestWeights, cDestWeights);
return (*env)->NewObject(env,
ompi_java.DistGraphNeighborsClass, ompi_java.DistGraphNeighborsInit,
sources, srcWeights, destinations, destWeights,
weighted ? JNI_TRUE : JNI_FALSE);
}
static int create_proclists_from_mpi_comm_topology(MPI_Comm comm, int rank, int *nsend_procs, int **send_procs, int *nrecv_procs, int **recv_procs)
{
int status;
int n, i;
MPI_Topo_test(comm, &status);
switch (status)
{
case MPI_CART:
MPI_Cartdim_get(comm, &n);
*nsend_procs = *nrecv_procs = 2 * n;
*send_procs = *recv_procs = z_alloc(2 * n, sizeof(int));
for (i = 0; i < n; ++i) MPI_Cart_shift(comm, i, -1, &(*send_procs)[2 * i + 1], &(*send_procs)[2 * i + 0]);
break;
case MPI_GRAPH:
MPI_Graph_neighbors_count(comm, rank, &n);
*nsend_procs = *nrecv_procs = n;
*send_procs = *recv_procs = z_alloc(n, sizeof(int));
MPI_Graph_neighbors(comm, rank, n, *send_procs);
break;
#if MPI_VERSION >= 2 && MPI_SUBVERSION >= 2
case MPI_DIST_GRAPH:
MPI_Dist_graph_neighbors_count(comm, nrecv_procs, nsend_procs, &n);
*send_procs = z_alloc(*nsend_procs + *nrecv_procs, sizeof(int));
*recv_procs = *send_procs + *nsend_procs;
MPI_Dist_graph_neighbors(comm, *nrecv_procs, *recv_procs, MPI_UNWEIGHTED, *nsend_procs, *send_procs, MPI_UNWEIGHTED);
break;
#endif
default:
return 0;
}
return 1;
}
static int verify_comm(MPI_Comm comm)
{
int local_errs = 0;
int i, j;
int indegree, outdegree, weighted;
int *sources, *sweights, *destinations, *dweights;
int use_dup;
int topo_type = MPI_UNDEFINED;
MPI_Comm dupcomm = MPI_COMM_NULL;
sources = (int *) malloc(size * sizeof(int));
sweights = (int *) malloc(size * sizeof(int));
destinations = (int *) malloc(size * sizeof(int));
dweights = (int *) malloc(size * sizeof(int));
for (use_dup = 0; use_dup <= 1; ++use_dup) {
if (!use_dup) {
MPI_Dist_graph_neighbors_count(comm, &indegree, &outdegree, &weighted);
}
else {
MPI_Comm_dup(comm, &dupcomm);
comm = dupcomm; /* caller retains original comm value */
}
MPI_Topo_test(comm, &topo_type);
if (topo_type != MPI_DIST_GRAPH) {
fprintf(stderr, "topo_type != MPI_DIST_GRAPH\n");
++local_errs;
}
j = 0;
for (i = 0; i < size; i++)
if (layout[i][rank])
j++;
if (j != indegree) {
fprintf(stderr, "indegree does not match, expected=%d got=%d, layout='%s'\n", indegree, j, graph_layout_name);
++local_errs;
}
j = 0;
for (i = 0; i < size; i++)
if (layout[rank][i])
j++;
if (j != outdegree) {
fprintf(stderr, "outdegree does not match, expected=%d got=%d, layout='%s'\n", outdegree, j, graph_layout_name);
++local_errs;
}
if ((indegree || outdegree) && (weighted == 0)) {
fprintf(stderr, "MPI_Dist_graph_neighbors_count thinks the graph is not weighted\n");
++local_errs;
}
MPI_Dist_graph_neighbors(comm, indegree, sources, sweights, outdegree, destinations, dweights);
/* For each incoming and outgoing edge in the matrix, search if
* the query function listed it in the sources. */
for (i = 0; i < size; i++) {
if (layout[i][rank]) {
for (j = 0; j < indegree; j++) {
assert(sources[j] >= 0);
assert(sources[j] < size);
if (sources[j] == i)
break;
}
if (j == indegree) {
fprintf(stderr, "no edge from %d to %d specified\n", i, rank);
++local_errs;
}
else {
if (sweights[j] != layout[i][rank]) {
fprintf(stderr, "incorrect weight for edge (%d,%d): %d instead of %d\n",
i, rank, sweights[j], layout[i][rank]);
++local_errs;
}
}
}
if (layout[rank][i]) {
for (j = 0; j < outdegree; j++) {
assert(destinations[j] >= 0);
assert(destinations[j] < size);
if (destinations[j] == i)
break;
}
if (j == outdegree) {
fprintf(stderr, "no edge from %d to %d specified\n", rank, i);
++local_errs;
}
else {
if (dweights[j] != layout[rank][i]) {
fprintf(stderr, "incorrect weight for edge (%d,%d): %d instead of %d\n",
rank, i, dweights[j], layout[rank][i]);
++local_errs;
}
}
}
}
/* For each incoming and outgoing edge in the sources, we should
* have an entry in the matrix */
for (i = 0; i < indegree; i++) {
if (layout[sources[i]][rank] != sweights[i]) {
fprintf(stderr, "edge (%d,%d) has a weight %d instead of %d\n", i, rank,
sweights[i], layout[sources[i]][rank]);
++local_errs;
}
}
for (i = 0; i < outdegree; i++) {
if (layout[rank][destinations[i]] != dweights[i]) {
fprintf(stderr, "edge (%d,%d) has a weight %d instead of %d\n", rank, i,
dweights[i], layout[rank][destinations[i]]);
++local_errs;
}
}
}
if (dupcomm != MPI_COMM_NULL)
MPI_Comm_free(&dupcomm);
return local_errs;
}
