// Dumb brute force function static int dumb_direct_children(int *children, int width, int id, int max_node_id) { int child; int count = 0; for(child = id+1; child < max_node_id; child++){ int parent_id, child_num, depth, max_depth; reverse_tree_info(child, max_node_id, width, &parent_id, &child_num, &depth, &max_depth); if( parent_id == id ){ children[count++] = child; } } return count; }
int main(int argc, char **argv) { int i, j; int n = 8192; int w = 5; int parent, children, depth, maxdepth; for (i = 0; i < n; i++) { int children1[w], children2[w]; int cnt1, cnt2; reverse_tree_info(i, n, w, &parent, &children, &depth, &maxdepth); printf("\ %d : par: %d nchild: %d depth: %d, maxdepth: %d\n", i, parent, children, depth, maxdepth); cnt1 = dumb_direct_children(children1, w, i, n); cnt2 = reverce_tree_direct_children(i, n, w, depth, children2); if (cnt1 != cnt2 ) { printf("\ Direct children sanity check error: cnt1 = %d, cnt2 = %d\n", cnt1, cnt2); return -1; } for(j = 0; j < cnt1; j++){ if (children1[j] != children2[j]) { printf("\ Direct children sanity check error: cnt1 = %d, cnt2 = %d\n", cnt1, cnt2); printf("\ Failed on %d'th element: children1[%d] = %d, children2[%d] = %d\n", j, j, children1[j], j, children2[j]); return -1; } }
static int _setup_stepd_tree_info(const stepd_step_rec_t *job, char ***env) { hostlist_t hl; char srun_host[64]; uint16_t port; char *p; int tree_width; /* job info available */ memset(&tree_info, 0, sizeof(tree_info)); hl = hostlist_create(job_info.step_nodelist); p = hostlist_nth(hl, job_info.nodeid); /* strdup-ed */ tree_info.this_node = xstrdup(p); free(p); /* this only controls the upward communication tree width */ p = getenvp(*env, PMI2_TREE_WIDTH_ENV); if (p) { tree_width = atoi(p); if (tree_width < 2) { info("invalid PMI2 tree width value (%d) detected. " "fallback to default value.", tree_width); tree_width = slurm_get_tree_width(); } } else { tree_width = slurm_get_tree_width(); } /* TODO: cannot launch 0 tasks on node */ /* * In tree position calculation, root of the tree is srun with id 0. * Stepd's id will be its nodeid plus 1. */ reverse_tree_info(job_info.nodeid + 1, job_info.nnodes + 1, tree_width, &tree_info.parent_id, &tree_info.num_children, &tree_info.depth, &tree_info.max_depth); tree_info.parent_id --; /* restore real nodeid */ if (tree_info.parent_id < 0) { /* parent is srun */ tree_info.parent_node = NULL; } else { p = hostlist_nth(hl, tree_info.parent_id); tree_info.parent_node = xstrdup(p); free(p); } hostlist_destroy(hl); tree_info.pmi_port = 0; /* not used */ p = getenvp(*env, "SLURM_SRUN_COMM_HOST"); if (!p) { error("mpi/pmi2: unable to find srun comm ifhn in env"); return SLURM_ERROR; } else { strncpy(srun_host, p, 64); } p = getenvp(*env, PMI2_SRUN_PORT_ENV); if (!p) { error("mpi/pmi2: unable to find srun pmi2 port in env"); return SLURM_ERROR; } else { port = atoi(p); unsetenvp(*env, PMI2_SRUN_PORT_ENV); } tree_info.srun_addr = xmalloc(sizeof(slurm_addr_t)); slurm_set_addr(tree_info.srun_addr, port, srun_host); /* init kvs seq to 0. TODO: reduce array size */ tree_info.children_kvs_seq = xmalloc(sizeof(uint32_t) * job_info.nnodes); return SLURM_SUCCESS; }
/* * Based on ideas provided by Hongjia Cao <*****@*****.**> in PMI2 plugin */ int pmixp_coll_init(pmixp_coll_t *coll, const pmix_proc_t *procs, size_t nprocs, pmixp_coll_type_t type) { hostlist_t hl; int max_depth, width, depth, i; char *p; #ifndef NDEBUG coll->magic = PMIXP_COLL_STATE_MAGIC; #endif coll->type = type; coll->state = PMIXP_COLL_SYNC; coll->pset.procs = xmalloc(sizeof(*procs) * nprocs); coll->pset.nprocs = nprocs; memcpy(coll->pset.procs, procs, sizeof(*procs) * nprocs); if (SLURM_SUCCESS != _hostset_from_ranges(procs, nprocs, &hl)) { /* TODO: provide ranges output routine */ PMIXP_ERROR("Bad ranges information"); goto err_exit; } #ifdef PMIXP_COLL_DEBUG /* if we debug collectives - store a copy of a full * hostlist to resolve participant id to the hostname */ coll->peers_hl = hostlist_copy(hl); #endif width = slurm_get_tree_width(); coll->peers_cnt = hostlist_count(hl); coll->my_peerid = hostlist_find(hl, pmixp_info_hostname()); reverse_tree_info(coll->my_peerid, coll->peers_cnt, width, &coll->prnt_peerid, &coll->chldrn_cnt, &depth, &max_depth); /* We interested in amount of direct childs */ coll->seq = 0; coll->contrib_children = 0; coll->contrib_local = false; coll->chldrn_ids = xmalloc(sizeof(int) * width); coll->contrib_chld = xmalloc(sizeof(int) * width); coll->chldrn_cnt = reverse_tree_direct_children(coll->my_peerid, coll->peers_cnt, width, depth, coll->chldrn_ids); if (coll->prnt_peerid == -1) { /* if we are the root of the tree: * - we don't have a parent; * - we have large list of all_childrens (we don't want * ourselfs there) */ coll->prnt_host = NULL; coll->all_chldrn_hl = hostlist_copy(hl); hostlist_delete_host(coll->all_chldrn_hl, pmixp_info_hostname()); coll->chldrn_str = hostlist_ranged_string_xmalloc(coll->all_chldrn_hl); } else { /* for all other nodes in the tree we need to know: * - nodename of our parent; * - we don't need a list of all_childrens and hl anymore */ /* * setup parent id's */ p = hostlist_nth(hl, coll->prnt_peerid); coll->prnt_host = xstrdup(p); free(p); /* reset prnt_peerid to the global peer */ coll->prnt_peerid = pmixp_info_job_hostid(coll->prnt_host); /* * setup root id's * (we need this for the SLURM API communication case) */ p = hostlist_nth(hl, 0); coll->root_host = xstrdup(p); free(p); /* reset prnt_peerid to the global peer */ coll->root_peerid = pmixp_info_job_hostid(coll->root_host); /* use empty hostlist here */ coll->all_chldrn_hl = hostlist_create(""); coll->chldrn_str = NULL; } /* fixup children peer ids to te global ones */ for(i=0; i<coll->chldrn_cnt; i++){ p = hostlist_nth(hl, coll->chldrn_ids[i]); coll->chldrn_ids[i] = pmixp_info_job_hostid(p); free(p); } hostlist_destroy(hl); /* Collective state */ coll->ufwd_buf = pmixp_server_buf_new(); coll->dfwd_buf = pmixp_server_buf_new(); _reset_coll_ufwd(coll); _reset_coll_dfwd(coll); coll->cbdata = NULL; coll->cbfunc = NULL; /* init fine grained lock */ slurm_mutex_init(&coll->lock); return SLURM_SUCCESS; err_exit: return SLURM_ERROR; }
/* * Based on ideas provided by Hongjia Cao <*****@*****.**> in PMI2 plugin */ int pmixp_coll_init(pmixp_coll_t *coll, const pmix_proc_t *procs, size_t nprocs, pmixp_coll_type_t type) { hostlist_t hl; uint32_t nodeid = 0, nodes = 0; int parent_id, depth, max_depth, tmp; int width, my_nspace = -1; char *p; int i, *ch_nodeids = NULL; #ifndef NDEBUG coll->magic = PMIXP_COLL_STATE_MAGIC; #endif coll->type = type; coll->state = PMIXP_COLL_SYNC; coll->procs = xmalloc(sizeof(*procs) * nprocs); memcpy(coll->procs, procs, sizeof(*procs) * nprocs); coll->nprocs = nprocs; coll->my_nspace = my_nspace; if (SLURM_SUCCESS != _hostset_from_ranges(procs, nprocs, &hl)) { /* TODO: provide ranges output routine */ PMIXP_ERROR("Bad ranges information"); goto err_exit; } width = slurm_get_tree_width(); nodes = hostlist_count(hl); nodeid = hostlist_find(hl, pmixp_info_hostname()); reverse_tree_info(nodeid, nodes, width, &parent_id, &tmp, &depth, &max_depth); coll->children_cnt = tmp; coll->nodeid = nodeid; /* We interested in amount of direct childs */ coll->seq = 0; coll->contrib_cntr = 0; coll->contrib_local = false; ch_nodeids = xmalloc(sizeof(int) * width); coll->ch_contribs = xmalloc(sizeof(int) * width); coll->children_cnt = reverse_tree_direct_children(nodeid, nodes, width, depth, ch_nodeids); /* create the hostlist with extract direct children's hostnames */ coll->ch_hosts = hostlist_create(""); for (i = 0; i < coll->children_cnt; i++) { char *hname = hostlist_nth(hl, ch_nodeids[i]); hostlist_push(coll->ch_hosts, hname); } /* just in case, shouldn't be needed */ hostlist_uniq(coll->ch_hosts); xfree(ch_nodeids); if (parent_id == -1) { /* if we are the root of the tree: * - we don't have a parent; * - we have large list of all_childrens (we don't want ourselfs there) */ coll->parent_host = NULL; hostlist_delete_host(hl, pmixp_info_hostname()); coll->all_children = hl; } else if (parent_id >= 0) { /* for all other nodes in the tree we need to know: * - nodename of our parent; * - we don't need a list of all_childrens and hl anymore */ p = hostlist_nth(hl, parent_id); coll->parent_host = xstrdup(p); /* use empty hostlist here */ coll->all_children = hostlist_create(""); free(p); hostlist_destroy(hl); } /* Collective data */ coll->buf = pmixp_server_new_buf(); coll->serv_offs = get_buf_offset(coll->buf); if (SLURM_SUCCESS != _pack_ranges(coll)) { PMIXP_ERROR("Cannot pack ranges to coll message header!"); goto err_exit; } /* Callback information */ coll->cbdata = NULL; coll->cbfunc = NULL; /* init fine grained lock */ slurm_mutex_init(&coll->lock); return SLURM_SUCCESS; err_exit: return SLURM_ERROR; }