void netpatterns_cleanup_narray_knomial_tree (netpatterns_narray_knomial_tree_node_t *my_node)
{
if (my_node->children_ranks) {
free (my_node->children_ranks);
my_node->children_ranks = NULL;
}
if (0 != my_node->my_rank) {
netpatterns_cleanup_recursive_knomial_tree_node (&my_node->k_node);
}
}
static void
mca_bcol_ptpcoll_module_destruct(mca_bcol_ptpcoll_module_t *ptpcoll_module)
{
int i;
mca_bcol_ptpcoll_local_mlmem_desc_t *ml_mem = &ptpcoll_module->ml_mem;
if (NULL != ml_mem->ml_buf_desc) {
/* Release the memory structs that were cache ML memory data */
uint32_t i, j, ci;
for (i = 0; i < ml_mem->num_banks; i++) {
for (j = 0; j < ml_mem->num_buffers_per_bank; j++) {
ci = i * ml_mem->num_buffers_per_bank + j;
if (NULL != ml_mem->ml_buf_desc[ci].requests) {
free(ml_mem->ml_buf_desc[ci].requests);
}
}
}
/* release the buffer descriptor */
free(ml_mem->ml_buf_desc);
ml_mem->ml_buf_desc = NULL;
}
if (NULL != ptpcoll_module->allgather_offsets) {
free_allreduce_offsets_array(ptpcoll_module);
}
if (NULL != ptpcoll_module->narray_node) {
for (i = 0; i < ptpcoll_module->group_size; i++) {
if (NULL != ptpcoll_module->narray_node[i].children_ranks) {
free(ptpcoll_module->narray_node[i].children_ranks);
}
}
free(ptpcoll_module->narray_node);
ptpcoll_module->narray_node = NULL;
}
OBJ_DESTRUCT(&ptpcoll_module->collreqs_free);
if (NULL != ptpcoll_module->super.list_n_connected) {
free(ptpcoll_module->super.list_n_connected);
ptpcoll_module->super.list_n_connected = NULL;
}
for (i = 0; i < BCOL_NUM_OF_FUNCTIONS; i++){
OPAL_LIST_DESTRUCT((&ptpcoll_module->super.bcol_fns_table[i]));
}
if (NULL != ptpcoll_module->kn_proxy_extra_index) {
free(ptpcoll_module->kn_proxy_extra_index);
ptpcoll_module->kn_proxy_extra_index = NULL;
}
if (NULL != ptpcoll_module->alltoall_iovec) {
free(ptpcoll_module->alltoall_iovec);
ptpcoll_module->alltoall_iovec = NULL;
}
if (NULL != ptpcoll_module->narray_knomial_proxy_extra_index) {
free(ptpcoll_module->narray_knomial_proxy_extra_index);
ptpcoll_module->narray_knomial_proxy_extra_index = NULL;
}
if (NULL != ptpcoll_module->narray_knomial_node) {
free(ptpcoll_module->narray_knomial_node);
ptpcoll_module->narray_knomial_node = NULL;
}
netpatterns_cleanup_recursive_knomial_allgather_tree_node(&ptpcoll_module->knomial_allgather_tree);
netpatterns_cleanup_recursive_knomial_tree_node(&ptpcoll_module->knomial_exchange_tree);
}
OMPI_DECLSPEC int netpatterns_setup_recursive_knomial_tree_node(
int num_nodes, int node_rank, int tree_order,
netpatterns_k_exchange_node_t *exchange_node)
{
/* local variables */
int i, j, tmp, cnt;
int n_levels;
int k_base, kpow_num, peer;
NETPATTERNS_VERBOSE(
("Enter netpatterns_setup_recursive_knomial_tree_node(num_nodes=%d, node_rank=%d, tree_order=%d)",
num_nodes, node_rank, tree_order));
assert(num_nodes > 1);
assert(tree_order > 1);
if (tree_order > num_nodes) {
tree_order = num_nodes;
}
exchange_node->tree_order = tree_order;
/* figure out number of levels in the tree */
n_levels = 0;
/* cnt - number of ranks in given level */
cnt=1;
while ( num_nodes > cnt ) {
cnt *= tree_order;
n_levels++;
};
/* figure out the largest power of tree_order that is less than or equal to
* num_nodes */
if ( cnt > num_nodes) {
cnt /= tree_order;
n_levels--;
}
exchange_node->log_tree_order = n_levels;
exchange_node->n_largest_pow_tree_order = cnt;
/* set node characteristics - node that is not within the largest
* power of tree_order will just send it's data to node that will participate
* in the recursive doubling, and get the result back at the end.
*/
if (node_rank + 1 > cnt) {
exchange_node->node_type = EXTRA_NODE;
} else {
exchange_node->node_type = EXCHANGE_NODE;
}
/* set the initial and final data exchanges - those that are not
* part of the recursive doubling.
*/
if (EXCHANGE_NODE == exchange_node->node_type) {
exchange_node->n_extra_sources = 0;
for (i = 0, tmp = node_rank * (tree_order - 1) + cnt + i;
tmp < num_nodes && i < tree_order - 1;
++i, ++tmp) {
++exchange_node->n_extra_sources;
}
assert(exchange_node->n_extra_sources < tree_order);
if (exchange_node->n_extra_sources > 0) {
exchange_node->rank_extra_sources_array = (int *) malloc
(exchange_node->n_extra_sources * sizeof(int));
if( NULL == exchange_node->rank_extra_sources_array ) {
goto Error;
}
for (i = 0, tmp = node_rank * (tree_order - 1) + cnt;
i < tree_order - 1 && tmp < num_nodes; ++i, ++tmp) {
NETPATTERNS_VERBOSE(("extra_source#%d = %d", i, tmp));
exchange_node->rank_extra_sources_array[i] = tmp;
}
} else {
exchange_node->rank_extra_sources_array = NULL;
}
} else {
exchange_node->n_extra_sources = 1;
exchange_node->rank_extra_sources_array = (int *) malloc (sizeof(int));
if( NULL == exchange_node->rank_extra_sources_array ) {
goto Error;
}
exchange_node->rank_extra_sources_array[0] = (node_rank - cnt) / (tree_order - 1);
NETPATTERNS_VERBOSE(("extra_source#%d = %d", 0,
exchange_node->rank_extra_sources_array[0] ));
}
/* set the exchange pattern */
if (EXCHANGE_NODE == exchange_node->node_type) {
exchange_node->n_exchanges = n_levels;
/* Allocate 2 dimension array thak keeps
rank exchange information for each step*/
exchange_node->rank_exchanges = (int **) malloc
(exchange_node->n_exchanges * sizeof(int *));
if(NULL == exchange_node->rank_exchanges) {
goto Error;
}
for (i = 0; i < exchange_node->n_exchanges; i++) {
exchange_node->rank_exchanges[i] = (int *) malloc
((tree_order - 1) * sizeof(int));
if( NULL == exchange_node->rank_exchanges ) {
goto Error;
}
}
/* fill in exchange partners */
for(i = 0, kpow_num = 1; i < exchange_node->n_exchanges;
i++, kpow_num *= tree_order) {
k_base = node_rank / (kpow_num * tree_order);
for(j = 1; j < tree_order; j++) {
peer = node_rank + kpow_num * j;
if (k_base != peer/(kpow_num * tree_order)) {
/* Wraparound the number */
peer = k_base * (kpow_num * tree_order) +
peer % (kpow_num * tree_order);
}
exchange_node->rank_exchanges[i][j - 1] = peer;
NETPATTERNS_VERBOSE(("rank_exchanges#(%d,%d)/%d = %d",
i, j, tree_order, peer));
}
}
} else {
exchange_node->n_exchanges=0;
exchange_node->rank_exchanges=NULL;
}
/* set the number of tags needed per stripe - this must be the
* same across all procs in the communicator.
*/
/* do we need this one */
exchange_node->n_tags = tree_order * n_levels + 1;
/* successful return */
return OMPI_SUCCESS;
Error:
netpatterns_cleanup_recursive_knomial_tree_node (exchange_node);
/* error return */
return OMPI_ERROR;
}
