int ompi_group_excl(ompi_group_t* group, int n, const int *ranks, ompi_group_t **new_group)
{
int i, j, k, result;
int *ranks_included=NULL;
/* determine the list of included processes for the excl-method */
k = 0;
if (0 < (group->grp_proc_count - n)) {
ranks_included = (int *)malloc( (group->grp_proc_count-n)*(sizeof(int)));
for (i=0 ; i<group->grp_proc_count ; i++) {
for(j=0 ; j<n ; j++) {
if(ranks[j] == i) {
break;
}
}
if (j==n) {
ranks_included[k] = i;
k++;
}
}
}
result = ompi_group_incl(group, k, ranks_included, new_group);
if (NULL != ranks_included) {
free(ranks_included);
}
return result;
}
int ompi_group_intersection(ompi_group_t* group1,ompi_group_t* group2,
ompi_group_t **new_group)
{
int proc1,proc2,k, result;
int *ranks_included=NULL;
ompi_group_t *group1_pointer, *group2_pointer;
ompi_proc_t *proc1_pointer, *proc2_pointer;
group1_pointer=(ompi_group_t *)group1;
group2_pointer=(ompi_group_t *)group2;
/* determine the number of included processes for the incl-method */
k = 0;
for (proc1 = 0; proc1 < group1_pointer->grp_proc_count; proc1++) {
proc1_pointer = ompi_group_peer_lookup (group1_pointer , proc1);
/* check to see if this proc is in group2 */
for (proc2 = 0; proc2 < group2_pointer->grp_proc_count; proc2++) {
proc2_pointer = ompi_group_peer_lookup (group2_pointer , proc2);
if( proc1_pointer == proc2_pointer ) {
k++;
break;
}
} /* end proc2 loop */
} /* end proc1 loop */
if (0 != k) {
ranks_included = (int *)malloc( k*(sizeof(int)));
}
/* determine the list of included processes for the incl-method */
k = 0;
for (proc1 = 0; proc1 < group1_pointer->grp_proc_count; proc1++) {
proc1_pointer = ompi_group_peer_lookup (group1_pointer , proc1);
/* check to see if this proc is in group2 */
for (proc2 = 0; proc2 < group2_pointer->grp_proc_count; proc2++) {
proc2_pointer = ompi_group_peer_lookup (group2_pointer ,proc2);
if( proc1_pointer == proc2_pointer ) {
ranks_included[k] = proc1;
k++;
break;
}
} /* end proc2 loop */
} /* end proc1 loop */
result = ompi_group_incl(group1, k, ranks_included, new_group);
if (NULL != ranks_included) {
free(ranks_included);
}
return result;
}
int MPI_Group_incl(MPI_Group group, int n, const int ranks[], MPI_Group *new_group)
{
int i, group_size, err;
ompi_group_t *group_pointer;
group_pointer = (ompi_group_t *)group;
group_size = ompi_group_size ( group_pointer );
if( MPI_PARAM_CHECK ) {
OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
/* verify that group is valid group */
if ( (MPI_GROUP_NULL == group) || ( NULL == group) ||
(NULL == new_group) ) {
return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, MPI_ERR_GROUP,
FUNC_NAME);
} else if (NULL == ranks && n > 0) {
return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, MPI_ERR_ARG,
FUNC_NAME);
}
/* check that new group is no larger than old group */
if ( n > group_size ) {
return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, MPI_ERR_RANK,
FUNC_NAME);
}
for (i = 0; i < n; i++) {
if ((ranks[i] < 0) || (ranks[i] >= group_size)){
return OMPI_ERRHANDLER_INVOKE (MPI_COMM_WORLD, MPI_ERR_RANK,
FUNC_NAME);
}
}
} /* end if( MPI_PARAM_CHECK ) */
if ( 0 == n ) {
*new_group = MPI_GROUP_EMPTY;
OBJ_RETAIN(MPI_GROUP_EMPTY);
return MPI_SUCCESS;
}
OPAL_CR_ENTER_LIBRARY();
err = ompi_group_incl(group,n,ranks,new_group);
OMPI_ERRHANDLER_RETURN(err, MPI_COMM_WORLD,err,FUNC_NAME);
}
int ompi_group_range_excl(ompi_group_t* group, int n_triplets, int ranges[][3],
ompi_group_t **new_group)
{
int j,k,i;
int *ranks_included=NULL, *ranks_excluded=NULL;
int index,first_rank,last_rank,stride,count,result;
count = 0;
/* determine the number of excluded processes for the range-excl-method */
k = 0;
for(j=0 ; j<n_triplets ; j++) {
first_rank = ranges[j][0];
last_rank = ranges[j][1];
stride = ranges[j][2];
if (first_rank < last_rank) {
/* positive stride */
index = first_rank;
while (index <= last_rank) {
count ++;
index += stride;
} /* end while loop */
}
else if (first_rank > last_rank) {
/* negative stride */
index = first_rank;
while (index >= last_rank) {
count ++;
index += stride;
} /* end while loop */
} else { /* first_rank == last_rank */
index = first_rank;
count ++;
}
}
if (0 != count) {
ranks_excluded = (int *)malloc( (count)*(sizeof(int)));
}
/* determine the list of included processes for the range-excl-method */
k = 0;
i = 0;
for(j=0 ; j<n_triplets ; j++) {
first_rank = ranges[j][0];
last_rank = ranges[j][1];
stride = ranges[j][2];
if (first_rank < last_rank) {
/* positive stride */
index = first_rank;
while (index <= last_rank) {
ranks_excluded[i] = index;
i++;
index += stride;
} /* end while loop */
}
else if (first_rank > last_rank) {
/* negative stride */
index = first_rank;
while (index >= last_rank) {
ranks_excluded[i] = index;
i++;
index += stride;
} /* end while loop */
} else { /* first_rank == last_rank */
index = first_rank;
ranks_excluded[i] = index;
i++;
}
}
if (0 != (group->grp_proc_count - count)) {
ranks_included = (int *)malloc( (group->grp_proc_count - count)*(sizeof(int)));
}
for (j=0 ; j<group->grp_proc_count ; j++) {
for(index=0 ; index<i ; index++) {
if(ranks_excluded[index] == j) break;
}
if (index == i) {
ranks_included[k] = j;
k++;
}
}
if (NULL != ranks_excluded) {
free(ranks_excluded);
}
result = ompi_group_incl(group, k, ranks_included, new_group);
if (NULL != ranks_included) {
free(ranks_included);
}
return result;
}
/*
* Invoked when there's a new communicator that has been created.
* Look at the communicator and decide which set of functions and
* priority we want to return.
*/
mca_scoll_base_module_t *
mca_scoll_mpi_comm_query(oshmem_group_t *osh_group, int *priority)
{
mca_scoll_base_module_t *module;
mca_scoll_mpi_module_t *mpi_module;
int err, i;
int tag;
ompi_group_t* parent_group, *new_group;
ompi_communicator_t* newcomm = NULL;
*priority = 0;
mca_scoll_mpi_component_t *cm;
cm = &mca_scoll_mpi_component;
int* ranks;
if (!cm->mpi_enable){
return NULL;
}
if ((osh_group->proc_count < 2) || (osh_group->proc_count < cm->mpi_np)) {
return NULL;
}
/* Create OMPI_Comm object and store ptr to it in group obj*/
if (NULL == oshmem_group_all) {
osh_group->ompi_comm = &(ompi_mpi_comm_world.comm);
} else {
int my_rank = MPI_UNDEFINED;
err = ompi_comm_group(&(ompi_mpi_comm_world.comm), &parent_group);
if (OPAL_UNLIKELY(OMPI_SUCCESS != err)) {
return NULL;
}
ranks = (int*) malloc(osh_group->proc_count * sizeof(int));
if (OPAL_UNLIKELY(NULL == ranks)) {
return NULL;
}
tag = 1;
for (i = 0; i < osh_group->proc_count; i++) {
ompi_proc_t* ompi_proc;
for( int j = 0; j < ompi_group_size(parent_group); j++ ) {
ompi_proc = ompi_group_peer_lookup(parent_group, j);
if( 0 == opal_compare_proc(ompi_proc->super.proc_name, osh_group->proc_array[i]->super.proc_name)) {
ranks[i] = j;
break;
}
}
/* NTH: keep track of my rank in the new group for the workaround below */
if (ranks[i] == ompi_comm_rank (&ompi_mpi_comm_world.comm)) {
my_rank = i;
}
}
err = ompi_group_incl(parent_group, osh_group->proc_count, ranks, &new_group);
if (OPAL_UNLIKELY(OMPI_SUCCESS != err)) {
free(ranks);
return NULL;
}
/* NTH: XXX -- WORKAROUND -- The oshmem code overwrites ompi_proc_local_proc with its
* own proc but does not update the proc list in comm world or comm self. This causes
* the code in ompi_group_incl that updates grp_my_rank to fail. This will cause failures
* here and when an application attempts to mix oshmem and mpi so it will really need to
* be fixed in oshmem/proc and not here. For now we need to work around a new jenkins
* failure so set my group ranking so we do not crash when running ompi_comm_create_group. */
new_group->grp_my_rank = my_rank;
err = ompi_comm_create_group(&(ompi_mpi_comm_world.comm), new_group, tag, &newcomm);
if (OPAL_UNLIKELY(OMPI_SUCCESS != err)) {
free(ranks);
return NULL;
}
err = ompi_group_free(&new_group);
if (OPAL_UNLIKELY(OMPI_SUCCESS != err)) {
free(ranks);
return NULL;
}
free(ranks);
osh_group->ompi_comm = newcomm;
}
mpi_module = OBJ_NEW(mca_scoll_mpi_module_t);
if (!mpi_module){
return NULL;
}
mpi_module->comm = osh_group->ompi_comm;
mpi_module->super.scoll_module_enable = mca_scoll_mpi_module_enable;
mpi_module->super.scoll_barrier = mca_scoll_mpi_barrier;
mpi_module->super.scoll_broadcast = mca_scoll_mpi_broadcast;
mpi_module->super.scoll_reduce = mca_scoll_mpi_reduce;
mpi_module->super.scoll_collect = mca_scoll_mpi_collect;
*priority = cm->mpi_priority;
module = &mpi_module->super;
return module;
}
/*
* Invoked when there's a new communicator that has been created.
* Look at the communicator and decide which set of functions and
* priority we want to return.
*/
mca_scoll_base_module_t *
mca_scoll_mpi_comm_query(oshmem_group_t *osh_group, int *priority)
{
mca_scoll_base_module_t *module;
mca_scoll_mpi_module_t *mpi_module;
int err, i;
int tag;
ompi_group_t* parent_group, *new_group;
ompi_communicator_t* newcomm = NULL;
*priority = 0;
mca_scoll_mpi_component_t *cm;
cm = &mca_scoll_mpi_component;
int* ranks;
if (!cm->mpi_enable){
return NULL;
}
if ((osh_group->proc_count < 2) || (osh_group->proc_count < cm->mpi_np)) {
return NULL;
}
/* Create OMPI_Comm object and store ptr to it in group obj*/
if (NULL == oshmem_group_all) {
osh_group->ompi_comm = &(ompi_mpi_comm_world.comm);
} else {
err = ompi_comm_group(&(ompi_mpi_comm_world.comm), &parent_group);
if (OPAL_UNLIKELY(OMPI_SUCCESS != err)) {
return NULL;
}
ranks = (int*) malloc(osh_group->proc_count * sizeof(int));
if (OPAL_UNLIKELY(NULL == ranks)) {
return NULL;
}
tag = 1;
for (i = 0; i < osh_group->proc_count; i++) {
ompi_proc_t* ompi_proc;
for( int j = 0; j < ompi_group_size(parent_group); j++ ) {
ompi_proc = ompi_group_peer_lookup(parent_group, j);
if( 0 == opal_compare_proc(ompi_proc->super.proc_name, osh_group->proc_array[i]->super.proc_name)) {
ranks[i] = j;
break;
}
}
}
err = ompi_group_incl(parent_group, osh_group->proc_count, ranks, &new_group);
if (OPAL_UNLIKELY(OMPI_SUCCESS != err)) {
free(ranks);
return NULL;
}
err = ompi_comm_create_group(&(ompi_mpi_comm_world.comm), new_group, tag, &newcomm);
if (OPAL_UNLIKELY(OMPI_SUCCESS != err)) {
free(ranks);
return NULL;
}
err = ompi_group_free(&new_group);
if (OPAL_UNLIKELY(OMPI_SUCCESS != err)) {
free(ranks);
return NULL;
}
free(ranks);
osh_group->ompi_comm = newcomm;
}
mpi_module = OBJ_NEW(mca_scoll_mpi_module_t);
if (!mpi_module){
return NULL;
}
mpi_module->comm = osh_group->ompi_comm;
mpi_module->super.scoll_module_enable = mca_scoll_mpi_module_enable;
mpi_module->super.scoll_barrier = mca_scoll_mpi_barrier;
mpi_module->super.scoll_broadcast = mca_scoll_mpi_broadcast;
mpi_module->super.scoll_reduce = mca_scoll_mpi_reduce;
mpi_module->super.scoll_collect = mca_scoll_mpi_collect;
mpi_module->super.scoll_alltoall = NULL;
*priority = cm->mpi_priority;
module = &mpi_module->super;
return module;
}
