int MPI_Group_translate_ranks(MPI_Group group1, int n_ranks, int *ranks1,
MPI_Group group2, int *ranks2)
{
int err;
/* check for errors */
if( MPI_PARAM_CHECK ) {
OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
if ((MPI_GROUP_NULL == group1) || (MPI_GROUP_NULL == group2) ||
(NULL == group1) || (NULL == group2)) {
return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, MPI_ERR_GROUP,
FUNC_NAME);
}
if (n_ranks < 0) {
return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, MPI_ERR_GROUP,
FUNC_NAME);
}
if (n_ranks > 0 && ((NULL == ranks1) || (NULL == ranks2 ))) {
return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, MPI_ERR_GROUP,
FUNC_NAME);
}
}
if (0 == n_ranks) {
return MPI_SUCCESS;
}
OPAL_CR_ENTER_LIBRARY();
err = ompi_group_translate_ranks ( group1, n_ranks, ranks1,
group2, ranks2 );
OMPI_ERRHANDLER_RETURN(err, MPI_COMM_WORLD, err, FUNC_NAME );
}
/**
* ompi_osc_pt2pt_get_comm_ranks:
*
* @param[in] module - OSC PT2PT module
* @param[in] sub_group - Group with ranks to translate
*
* @returns an array of translated ranks on success or NULL on failure
*
* Translate the ranks given in {sub_group} into ranks in the
* communicator used to create {module}.
*/
static int *ompi_osc_sm_group_ranks (ompi_group_t *group, ompi_group_t *sub_group)
{
int size = ompi_group_size(sub_group);
int *ranks1, *ranks2;
int ret;
ranks1 = calloc (size, sizeof(int));
ranks2 = calloc (size, sizeof(int));
if (NULL == ranks1 || NULL == ranks2) {
free (ranks1);
free (ranks2);
return NULL;
}
for (int i = 0 ; i < size ; ++i) {
ranks1[i] = i;
}
ret = ompi_group_translate_ranks (sub_group, size, ranks1, group, ranks2);
free (ranks1);
if (OMPI_SUCCESS != ret) {
free (ranks2);
return NULL;
}
qsort (ranks2, size, sizeof (int), compare_ranks);
return ranks2;
}
int ompi_group_dump (ompi_group_t* group)
{
int i;
int new_rank;
i=0;
printf("Group Proc Count: %d\n",group->grp_proc_count);
printf("Group My Rank: %d\n",group->grp_my_rank);
if (OMPI_GROUP_IS_SPORADIC(group)) {
ompi_group_translate_ranks( group,1,&group->grp_my_rank,
group->grp_parent_group_ptr,
&new_rank);
printf("Rank in the parent group: %d\n",new_rank);
printf("The Sporadic List Length: %d\n",
group->sparse_data.grp_sporadic.grp_sporadic_list_len);
printf("Rank First Length\n");
for(i=0 ; i<group->sparse_data.grp_sporadic.grp_sporadic_list_len ; i++) {
printf("%d %d\n",
group->sparse_data.grp_sporadic.grp_sporadic_list[i].rank_first,
group->sparse_data.grp_sporadic.grp_sporadic_list[i].length);
}
}
else if (OMPI_GROUP_IS_STRIDED(group)) {
ompi_group_translate_ranks( group,1,&group->grp_my_rank,
group->grp_parent_group_ptr,
&new_rank);
printf("Rank in the parent group: %d\n",new_rank);
printf("The Offset is: %d\n",group->sparse_data.grp_strided.grp_strided_offset);
printf("The Stride is: %d\n",group->sparse_data.grp_strided.grp_strided_stride);
printf("The Last Element is: %d\n",
group->sparse_data.grp_strided.grp_strided_last_element);
}
else if (OMPI_GROUP_IS_BITMAP(group)) {
ompi_group_translate_ranks( group,1,&group->grp_my_rank,
group->grp_parent_group_ptr,
&new_rank);
printf("Rank in the parent group: %d\n",new_rank);
printf("The length of the bitmap array is: %d\n",
group->sparse_data.grp_bitmap.grp_bitmap_array_len);
for (i=0 ; i<group->sparse_data.grp_bitmap.grp_bitmap_array_len ; i++) {
printf("%d\t",group->sparse_data.grp_bitmap.grp_bitmap_array[i]);
}
}
printf("*********************************************************\n");
return OMPI_SUCCESS;
}
/*
* This is the function that iterates through the sparse groups to the dense group
* to reach the process pointer
*/
ompi_proc_t* ompi_group_get_proc_ptr (ompi_group_t* group , int rank)
{
int ranks1,ranks2;
do {
if(OMPI_GROUP_IS_DENSE(group)) {
return group->grp_proc_pointers[rank];
}
ranks1 = rank;
ompi_group_translate_ranks( group, 1, &ranks1,
group->grp_parent_group_ptr,&ranks2);
rank = ranks2;
group = group->grp_parent_group_ptr;
} while (1);
}
ompi_proc_t *ompi_group_get_proc_ptr_raw (ompi_group_t *group, int rank)
{
#if OMPI_GROUP_SPARSE
do {
if (OMPI_GROUP_IS_DENSE(group)) {
return ompi_group_dense_lookup_raw (group, rank);
}
int ranks1 = rank;
ompi_group_translate_ranks (group, 1, &ranks1, group->grp_parent_group_ptr, &rank);
group = group->grp_parent_group_ptr;
} while (1);
#else
return ompi_group_dense_lookup_raw (group, rank);
#endif
}
int ompi_group_incl_bmap(ompi_group_t* group, int n, const int *ranks,
ompi_group_t **new_group)
{
/* local variables */
int my_group_rank,i,bit_set;
ompi_group_t *group_pointer, *new_group_pointer;
group_pointer = (ompi_group_t *)group;
if ( 0 == n ) {
*new_group = MPI_GROUP_EMPTY;
OBJ_RETAIN(MPI_GROUP_EMPTY);
return OMPI_SUCCESS;
}
new_group_pointer = ompi_group_allocate_bmap(group->grp_proc_count, n);
if( NULL == new_group_pointer ) {
return MPI_ERR_GROUP;
}
/* Initialize the bit array to zeros */
for (i=0 ; i<new_group_pointer->sparse_data.grp_bitmap.grp_bitmap_array_len ; i++) {
new_group_pointer->
sparse_data.grp_bitmap.grp_bitmap_array[i] = 0;
}
/* set the bits */
for (i=0 ; i<n ; i++) {
bit_set = ranks[i] % BSIZE;
new_group_pointer->
sparse_data.grp_bitmap.grp_bitmap_array[(int)(ranks[i]/BSIZE)] |= (1 << bit_set);
}
new_group_pointer -> grp_parent_group_ptr = group_pointer;
OBJ_RETAIN(new_group_pointer -> grp_parent_group_ptr);
ompi_group_increment_proc_count(new_group_pointer -> grp_parent_group_ptr);
ompi_group_increment_proc_count(new_group_pointer);
my_group_rank=group_pointer->grp_my_rank;
ompi_group_translate_ranks (group_pointer,1,&my_group_rank,
new_group_pointer,&new_group_pointer->grp_my_rank);
*new_group = (MPI_Group)new_group_pointer;
return OMPI_SUCCESS;
}
int ompi_group_incl_strided(ompi_group_t* group, int n, int *ranks,
ompi_group_t **new_group)
{
/* local variables */
int my_group_rank,stride;
ompi_group_t *group_pointer, *new_group_pointer;
group_pointer = (ompi_group_t *)group;
if ( 0 == n ) {
*new_group = MPI_GROUP_EMPTY;
OBJ_RETAIN(MPI_GROUP_EMPTY);
return OMPI_SUCCESS;
}
stride = check_stride(ranks,n);
new_group_pointer = ompi_group_allocate_strided();
if( NULL == new_group_pointer ) {
return MPI_ERR_GROUP;
}
new_group_pointer -> grp_parent_group_ptr = group_pointer;
OBJ_RETAIN(new_group_pointer -> grp_parent_group_ptr);
ompi_group_increment_proc_count(new_group_pointer -> grp_parent_group_ptr);
new_group_pointer -> sparse_data.grp_strided.grp_strided_stride = stride;
new_group_pointer -> sparse_data.grp_strided.grp_strided_offset = ranks[0];
new_group_pointer -> sparse_data.grp_strided.grp_strided_last_element = ranks[n-1];
new_group_pointer -> grp_proc_count = n;
ompi_group_increment_proc_count(new_group_pointer);
my_group_rank = group_pointer->grp_my_rank;
ompi_group_translate_ranks (new_group_pointer->grp_parent_group_ptr,1,&my_group_rank,
new_group_pointer,&new_group_pointer->grp_my_rank);
*new_group = (MPI_Group)new_group_pointer;
return OMPI_SUCCESS;
}
int ompi_group_incl_spor(ompi_group_t* group, int n, int *ranks,
ompi_group_t **new_group)
{
/* local variables */
int my_group_rank,l,i,j,proc_count;
ompi_group_t *group_pointer, *new_group_pointer;
group_pointer = (ompi_group_t *)group;
if (0 == n) {
*new_group = MPI_GROUP_EMPTY;
OBJ_RETAIN(MPI_GROUP_EMPTY);
return OMPI_SUCCESS;
}
l=0;
j=0;
proc_count = 0;
for(i=0 ; i<n ; i++){
if(ranks[i] == ranks[i-1]+1) {
if(l==0) l++;
}
else l++;
}
new_group_pointer = ompi_group_allocate_sporadic(l);
if( NULL == new_group_pointer ) {
return MPI_ERR_GROUP;
}
new_group_pointer ->
sparse_data.grp_sporadic.grp_sporadic_list[j].rank_first = ranks[0];
new_group_pointer ->
sparse_data.grp_sporadic.grp_sporadic_list[j].length = 1;
for(i=1 ; i<n ; i++){
if(ranks[i] == ranks[i-1]+1) {
new_group_pointer -> sparse_data.grp_sporadic.grp_sporadic_list[j].length ++;
}
else {
j++;
new_group_pointer ->
sparse_data.grp_sporadic.grp_sporadic_list[j].rank_first = ranks[i];
new_group_pointer ->
sparse_data.grp_sporadic.grp_sporadic_list[j].length = 1;
}
}
new_group_pointer->sparse_data.grp_sporadic.grp_sporadic_list_len = j+1;
new_group_pointer -> grp_parent_group_ptr = group_pointer;
OBJ_RETAIN(new_group_pointer -> grp_parent_group_ptr);
ompi_group_increment_proc_count(new_group_pointer -> grp_parent_group_ptr);
for(i=0 ; i<new_group_pointer->sparse_data.grp_sporadic.grp_sporadic_list_len ; i++) {
proc_count = proc_count + new_group_pointer ->
sparse_data.grp_sporadic.grp_sporadic_list[i].length;
}
new_group_pointer->grp_proc_count = proc_count;
ompi_group_increment_proc_count(new_group_pointer);
my_group_rank=group_pointer->grp_my_rank;
ompi_group_translate_ranks (group_pointer,1,&my_group_rank,
new_group_pointer,&new_group_pointer->grp_my_rank);
*new_group = (MPI_Group)new_group_pointer;
return OMPI_SUCCESS;
}
static int ompi_comm_allreduce_group_nb (int *inbuf, int *outbuf, int count,
struct ompi_op_t *op, ompi_comm_cid_context_t *cid_context,
ompi_request_t **req)
{
ompi_group_t *group = cid_context->newcomm->c_local_group;
const int group_size = ompi_group_size (group);
const int group_rank = ompi_group_rank (group);
ompi_communicator_t *comm = cid_context->comm;
int peers_group[3], *tmp, subreq_count = 0;
ompi_comm_allreduce_context_t *context;
ompi_comm_request_t *request;
ompi_request_t *subreq[3];
context = ompi_comm_allreduce_context_alloc (inbuf, outbuf, count, op, cid_context);
if (NULL == context) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
tmp = context->tmpbuf = calloc (sizeof (int), count * 3);
if (NULL == context->tmpbuf) {
OBJ_RELEASE(context);
return OMPI_ERR_OUT_OF_RESOURCE;
}
request = ompi_comm_request_get ();
if (NULL == request) {
OBJ_RELEASE(context);
return OMPI_ERR_OUT_OF_RESOURCE;
}
request->context = &context->super;
/* basic recursive doubling allreduce on the group */
peers_group[0] = group_rank ? ((group_rank - 1) >> 1) : MPI_PROC_NULL;
peers_group[1] = (group_rank * 2 + 1) < group_size ? group_rank * 2 + 1: MPI_PROC_NULL;
peers_group[2] = (group_rank * 2 + 2) < group_size ? group_rank * 2 + 2 : MPI_PROC_NULL;
/* translate the ranks into the ranks of the parent communicator */
ompi_group_translate_ranks (group, 3, peers_group, comm->c_local_group, context->peers_comm);
/* reduce */
memmove (outbuf, inbuf, sizeof (int) * count);
for (int i = 0 ; i < 2 ; ++i) {
if (MPI_PROC_NULL != context->peers_comm[i + 1]) {
int rc = MCA_PML_CALL(irecv(tmp, count, MPI_INT, context->peers_comm[i + 1],
cid_context->pml_tag, comm, subreq + subreq_count++));
if (OMPI_SUCCESS != rc) {
ompi_comm_request_return (request);
return rc;
}
tmp += count;
}
}
ompi_comm_request_schedule_append (request, ompi_comm_allreduce_group_recv_complete, subreq, subreq_count);
ompi_comm_request_start (request);
*req = &request->super;
return OMPI_SUCCESS;
}
struct ompi_communicator_t* mca_coll_hierarch_get_llcomm (int root,
mca_coll_hierarch_module_t *hierarch_module,
int* llroot,
int* lroot)
{
struct ompi_communicator_t *llcomm=NULL;
struct ompi_group_t *llgroup=NULL;
struct ompi_group_t *group=NULL;
struct mca_coll_hierarch_llead_t *llead=NULL;
int found, i, rc, num_llead, offset;
int rank = ompi_comm_rank (hierarch_module->hier_comm);
int color;
/* determine what our offset of root is in the colorarr */
offset = mca_coll_hierarch_get_offset ( root,
hierarch_module->hier_num_colorarr,
hierarch_module->hier_colorarr );
num_llead = opal_pointer_array_get_size ( &(hierarch_module->hier_llead) );
for ( found=0, i=0; i < num_llead; i++ ) {
llead = (struct mca_coll_hierarch_llead_t *) opal_pointer_array_get_item (
&(hierarch_module->hier_llead), i );
if ( NULL == llead ) {
continue;
}
if (llead->offset == offset ) {
found = 1;
break;
}
#if 0
else if () {
/* the offset of root = maxoffset of this color and
* the offset on llead is larger then offset of root.
* then we can also use this llead structure
*/
}
#endif
}
if ( !found ) {
/* allocate a new llead element */
llead = (struct mca_coll_hierarch_llead_t *) malloc (
sizeof(struct mca_coll_hierarch_llead_t));
if ( NULL == llead ) {
return NULL;
}
/* generate the list of lleaders with this offset */
mca_coll_hierarch_get_all_lleaders ( rank, hierarch_module, llead, offset );
color = MPI_UNDEFINED;
if ( llead->am_lleader ) {
color = 1;
}
/* create new lleader subcommunicator */
rc = ompi_comm_split ( hierarch_module->hier_comm, color, root, &llcomm, 0);
if ( OMPI_SUCCESS != rc ) {
return NULL;
}
if ( OMPI_COMM_CID_IS_LOWER ( llcomm, hierarch_module->hier_comm ) ) {
/* Mark the communicator as 'extra retain' and increase the
reference count by one more. See ompi_comm_activate
for detailed explanation. */
OMPI_COMM_SET_EXTRA_RETAIN (llcomm);
OBJ_RETAIN(llcomm);
}
llead->llcomm = llcomm;
/* Store the new element on the hierarch_module struct */
opal_pointer_array_add ( &(hierarch_module->hier_llead), llead);
}
llcomm = llead->llcomm;
*lroot = llead->my_lleader;
*llroot = MPI_UNDEFINED;
if ( MPI_COMM_NULL != llcomm ) {
group = hierarch_module->hier_comm->c_local_group;
llgroup = llcomm->c_local_group;
rc = ompi_group_translate_ranks ( group, 1, &root, llgroup, llroot);
if ( OMPI_SUCCESS != rc ) {
return NULL;
}
}
return llcomm;
}