int pmixp_coll_ring_init(pmixp_coll_t *coll, hostlist_t *hl) { #ifdef PMIXP_COLL_DEBUG PMIXP_DEBUG("called"); #endif int i; pmixp_coll_ring_ctx_t *coll_ctx = NULL; pmixp_coll_ring_t *ring = &coll->state.ring; char *p; int rel_id = hostlist_find(*hl, pmixp_info_hostname()); /* compute the next absolute id of the neighbor */ p = hostlist_nth(*hl, (rel_id + 1) % coll->peers_cnt); ring->next_peerid = pmixp_info_job_hostid(p); free(p); ring->fwrd_buf_pool = list_create(pmixp_free_buf); ring->ring_buf_pool = list_create(pmixp_free_buf); for (i = 0; i < PMIXP_COLL_RING_CTX_NUM; i++) { coll_ctx = &ring->ctx_array[i]; coll_ctx->coll = coll; coll_ctx->in_use = false; coll_ctx->seq = coll->seq; coll_ctx->contrib_local = false; coll_ctx->contrib_prev = 0; coll_ctx->state = PMIXP_COLL_RING_SYNC; // TODO bit vector coll_ctx->contrib_map = xmalloc(sizeof(bool) * coll->peers_cnt); } return SLURM_SUCCESS; }
char *find_hostname(uint32_t pos, char *hosts) { hostlist_t hostlist = NULL; char *temp = NULL, *host = NULL; if(!hosts || (pos == (uint32_t)NO_VAL)) return NULL; hostlist = hostlist_create(hosts); temp = hostlist_nth(hostlist, pos); if(temp) { host = xstrdup(temp); free(temp); } hostlist_destroy(hostlist); return host; }
/* send message defined by buf and size to given rank stepd */ static int pmix_stepd_send(const char* buf, uint32_t size, int rank) { int rc = SLURM_SUCCESS; /* map rank to host name */ char* host = hostlist_nth(pmix_stepd_hostlist, rank); /* strdup-ed */ /* delay to sleep between retries in seconds, * if there are multiple retires, we'll grow this delay * using exponential backoff, doubling it each time */ unsigned int delay = 1; /* we'll try multiple times to send message to stepd, * we retry in case stepd is just slow to get started */ int retries = 0; while (1) { /* attempt to send message */ rc = slurm_forward_data(&host, tree_sock_addr, size, buf); if (rc == SLURM_SUCCESS) { /* message sent successfully, we're done */ break; } /* check whether we've exceeded our retry count */ retries++; if (retries >= MAX_RETRIES) { /* cancel the step to avoid tasks hang */ slurm_kill_job_step(job_info.jobid, job_info.stepid, SIGKILL); } /* didn't succeeded, but we'll retry again, * sleep for a bit first */ sleep(delay); delay *= 2; } /* free host name */ free(host); /* strdup-ed */ return rc; }
/* * _set_collectors call the split_hostlist API on the all nodes hostlist * to set the node to be used as a collector for unsolicited node aggregation. * * If this node is a forwarding node (first node in any hostlist), * then its collector and backup are the ControlMachine and it's backup. * * Otherwise, we find the hostlist containing this node. * The forwarding node in that hostlist becomes a collector, the next node * which is not this node becomes the backup. * That list is split, we iterate through it and searching for a list in * which this node is a forwarding node. If found, we set the collector and * backup, else this process is repeated. */ static void _set_collectors(char *this_node_name) { slurm_ctl_conf_t *conf; hostlist_t nodes; hostlist_t* hll = NULL; char *parent = NULL, *backup = NULL; char addrbuf[32]; int i, j, f = -1; int hl_count = 0; uint16_t parent_port; uint16_t backup_port; bool found = false; bool ctldparent = true; #ifdef HAVE_FRONT_END return; /* on a FrontEnd system this would never be useful. */ #endif if (!run_in_daemon("slurmd")) return; /* Only compute nodes have collectors */ /* Set the initial iteration, collector is controller, * full list is split */ xassert(this_node_name); conf = slurm_conf_lock(); nodes = _get_all_nodes(); parent = strdup(conf->control_addr); if (conf->backup_addr) { backup = strdup(conf->backup_addr); } parent_port = conf->slurmctld_port; backup_port = parent_port; slurm_conf_unlock(); while (!found) { if ( route_g_split_hostlist(nodes, &hll, &hl_count) ) { error("unable to split forward hostlist"); goto clean; /* collector addrs remains null */ } /* Find which hostlist contains this node */ for (i=0; i < hl_count; i++) { f = hostlist_find(hll[i], this_node_name); if (f != -1) break; } if (i == hl_count) { fatal("ROUTE -- %s not found in node_record_table", this_node_name); } if (f == 0) { /* we are a forwarded to node, * so our parent is parent */ if (hostlist_count(hll[i]) > 1) this_is_collector = true; xfree(msg_collect_node); msg_collect_node = xmalloc(sizeof(slurm_addr_t)); if (ctldparent) slurm_set_addr(msg_collect_node, parent_port, parent); else { slurm_conf_get_addr(parent, msg_collect_node); msg_collect_node->sin_port = htons(parent_port); } if (debug_flags & DEBUG_FLAG_ROUTE) { slurm_print_slurm_addr(msg_collect_node, addrbuf, 32); info("ROUTE -- message collector address is %s", addrbuf); } xfree(msg_collect_backup); if (backup) { msg_collect_backup = xmalloc(sizeof(slurm_addr_t)); if (ctldparent) { slurm_set_addr(msg_collect_backup, backup_port, backup); } else { slurm_conf_get_addr(backup, msg_collect_backup); msg_collect_backup->sin_port = htons(backup_port); } if (debug_flags & DEBUG_FLAG_ROUTE) { slurm_print_slurm_addr( msg_collect_backup, addrbuf, 32); info("ROUTE -- message collector backup" " address is %s", addrbuf); } } else { if (debug_flags & DEBUG_FLAG_ROUTE) { info("ROUTE -- no message collector " "backup"); } } found = true; goto clean; } /* We are not a forwarding node, the first node in this list * will split the forward_list. * We also know that the forwarding node is not a controller. * * clean up parent context */ ctldparent = false; hostlist_destroy(nodes); if (parent) free(parent); if (backup) free(backup); nodes = hostlist_copy(hll[i]); for (j=0; j < hl_count; j++) { hostlist_destroy(hll[j]); } xfree(hll); /* set our parent, backup, and continue search */ parent = hostlist_shift(nodes); backup = hostlist_nth(nodes, 0); if (strcmp(backup, this_node_name) == 0) { free(backup); backup = NULL; if (hostlist_count(nodes) > 1) backup = hostlist_nth(nodes, 1); } parent_port = slurm_conf_get_port(parent); if (backup) { backup_port = slurm_conf_get_port(backup); } else backup_port = 0; } clean: if (debug_flags & DEBUG_FLAG_ROUTE) { if (this_is_collector) info("ROUTE -- %s is a collector node", this_node_name); else info("ROUTE -- %s is a leaf node", this_node_name); } hostlist_destroy(nodes); if (parent) free(parent); if (backup) free(backup); for (i=0; i < hl_count; i++) { hostlist_destroy(hll[i]); } xfree(hll); }
static int _setup_stepd_tree_info(const stepd_step_rec_t *job, char ***env) { hostlist_t hl; char srun_host[64]; uint16_t port; char *p; int tree_width; /* job info available */ memset(&tree_info, 0, sizeof(tree_info)); hl = hostlist_create(job_info.step_nodelist); p = hostlist_nth(hl, job_info.nodeid); /* strdup-ed */ tree_info.this_node = xstrdup(p); free(p); /* this only controls the upward communication tree width */ p = getenvp(*env, PMI2_TREE_WIDTH_ENV); if (p) { tree_width = atoi(p); if (tree_width < 2) { info("invalid PMI2 tree width value (%d) detected. " "fallback to default value.", tree_width); tree_width = slurm_get_tree_width(); } } else { tree_width = slurm_get_tree_width(); } /* TODO: cannot launch 0 tasks on node */ /* * In tree position calculation, root of the tree is srun with id 0. * Stepd's id will be its nodeid plus 1. */ reverse_tree_info(job_info.nodeid + 1, job_info.nnodes + 1, tree_width, &tree_info.parent_id, &tree_info.num_children, &tree_info.depth, &tree_info.max_depth); tree_info.parent_id --; /* restore real nodeid */ if (tree_info.parent_id < 0) { /* parent is srun */ tree_info.parent_node = NULL; } else { p = hostlist_nth(hl, tree_info.parent_id); tree_info.parent_node = xstrdup(p); free(p); } hostlist_destroy(hl); tree_info.pmi_port = 0; /* not used */ p = getenvp(*env, "SLURM_SRUN_COMM_HOST"); if (!p) { error("mpi/pmi2: unable to find srun comm ifhn in env"); return SLURM_ERROR; } else { strncpy(srun_host, p, 64); } p = getenvp(*env, PMI2_SRUN_PORT_ENV); if (!p) { error("mpi/pmi2: unable to find srun pmi2 port in env"); return SLURM_ERROR; } else { port = atoi(p); unsetenvp(*env, PMI2_SRUN_PORT_ENV); } tree_info.srun_addr = xmalloc(sizeof(slurm_addr_t)); slurm_set_addr(tree_info.srun_addr, port, srun_host); /* init kvs seq to 0. TODO: reduce array size */ tree_info.children_kvs_seq = xmalloc(sizeof(uint32_t) * job_info.nnodes); return SLURM_SUCCESS; }
static int _resources_set(char ***env) { char *p = NULL; /* Initialize all memory pointers that would be allocated to NULL * So in case of error exit we will know what to xfree */ _pmixp_job_info.job_hl = hostlist_create(""); _pmixp_job_info.step_hl = hostlist_create(""); _pmixp_job_info.hostname = NULL; /* Save step host list */ p = getenvp(*env, PMIXP_STEP_NODES_ENV); if (!p) { PMIXP_ERROR_NO(ENOENT, "Environment variable %s not found", PMIXP_STEP_NODES_ENV); goto err_exit; } hostlist_push(_pmixp_job_info.step_hl, p); /* Extract our node name */ p = hostlist_nth(_pmixp_job_info.step_hl, _pmixp_job_info.node_id); _pmixp_job_info.hostname = xstrdup(p); free(p); /* Determine job-wide node id and job-wide node count */ p = getenvp(*env, PMIXP_JOB_NODES_ENV); if (p == NULL) { p = getenvp(*env, PMIXP_JOB_NODES_ENV_DEP); if (p == NULL) { /* shouldn't happen if we are under SLURM! */ PMIXP_ERROR_NO(ENOENT, "Neither of nodelist environment variables: %s OR %s was found!", PMIXP_JOB_NODES_ENV, PMIXP_JOB_NODES_ENV_DEP); goto err_exit; } } hostlist_push(_pmixp_job_info.job_hl, p); _pmixp_job_info.nnodes_job = hostlist_count(_pmixp_job_info.job_hl); _pmixp_job_info.node_id_job = hostlist_find(_pmixp_job_info.job_hl, _pmixp_job_info.hostname); /* FIXME!! ------------------------------------------------------------- */ /* TODO: _get_task_count not always works well. if (_get_task_count(env, &_pmixp_job_info.ntasks_job, &_pmixp_job_info.ncpus_job) < 0) { _pmixp_job_info.ntasks_job = _pmixp_job_info.ntasks; _pmixp_job_info.ncpus_job = _pmixp_job_info.ntasks; } xassert(_pmixp_job_info.ntasks <= _pmixp_job_info.ntasks_job); */ _pmixp_job_info.ntasks_job = _pmixp_job_info.ntasks; _pmixp_job_info.ncpus_job = _pmixp_job_info.ntasks; /* Save task-to-node mapping */ p = getenvp(*env, PMIXP_SLURM_MAPPING_ENV); if (p == NULL) { /* Direct modex won't work */ PMIXP_ERROR_NO(ENOENT, "No %s environment variable found!", PMIXP_SLURM_MAPPING_ENV); goto err_exit; } _pmixp_job_info.task_map_packed = xstrdup(p); return SLURM_SUCCESS; err_exit: hostlist_destroy(_pmixp_job_info.job_hl); hostlist_destroy(_pmixp_job_info.step_hl); if (NULL != _pmixp_job_info.hostname) { xfree(_pmixp_job_info.hostname); } return SLURM_ERROR; }
/* * Based on ideas provided by Hongjia Cao <*****@*****.**> in PMI2 plugin */ int pmixp_coll_init(pmixp_coll_t *coll, const pmix_proc_t *procs, size_t nprocs, pmixp_coll_type_t type) { hostlist_t hl; int max_depth, width, depth, i; char *p; #ifndef NDEBUG coll->magic = PMIXP_COLL_STATE_MAGIC; #endif coll->type = type; coll->state = PMIXP_COLL_SYNC; coll->pset.procs = xmalloc(sizeof(*procs) * nprocs); coll->pset.nprocs = nprocs; memcpy(coll->pset.procs, procs, sizeof(*procs) * nprocs); if (SLURM_SUCCESS != _hostset_from_ranges(procs, nprocs, &hl)) { /* TODO: provide ranges output routine */ PMIXP_ERROR("Bad ranges information"); goto err_exit; } #ifdef PMIXP_COLL_DEBUG /* if we debug collectives - store a copy of a full * hostlist to resolve participant id to the hostname */ coll->peers_hl = hostlist_copy(hl); #endif width = slurm_get_tree_width(); coll->peers_cnt = hostlist_count(hl); coll->my_peerid = hostlist_find(hl, pmixp_info_hostname()); reverse_tree_info(coll->my_peerid, coll->peers_cnt, width, &coll->prnt_peerid, &coll->chldrn_cnt, &depth, &max_depth); /* We interested in amount of direct childs */ coll->seq = 0; coll->contrib_children = 0; coll->contrib_local = false; coll->chldrn_ids = xmalloc(sizeof(int) * width); coll->contrib_chld = xmalloc(sizeof(int) * width); coll->chldrn_cnt = reverse_tree_direct_children(coll->my_peerid, coll->peers_cnt, width, depth, coll->chldrn_ids); if (coll->prnt_peerid == -1) { /* if we are the root of the tree: * - we don't have a parent; * - we have large list of all_childrens (we don't want * ourselfs there) */ coll->prnt_host = NULL; coll->all_chldrn_hl = hostlist_copy(hl); hostlist_delete_host(coll->all_chldrn_hl, pmixp_info_hostname()); coll->chldrn_str = hostlist_ranged_string_xmalloc(coll->all_chldrn_hl); } else { /* for all other nodes in the tree we need to know: * - nodename of our parent; * - we don't need a list of all_childrens and hl anymore */ /* * setup parent id's */ p = hostlist_nth(hl, coll->prnt_peerid); coll->prnt_host = xstrdup(p); free(p); /* reset prnt_peerid to the global peer */ coll->prnt_peerid = pmixp_info_job_hostid(coll->prnt_host); /* * setup root id's * (we need this for the SLURM API communication case) */ p = hostlist_nth(hl, 0); coll->root_host = xstrdup(p); free(p); /* reset prnt_peerid to the global peer */ coll->root_peerid = pmixp_info_job_hostid(coll->root_host); /* use empty hostlist here */ coll->all_chldrn_hl = hostlist_create(""); coll->chldrn_str = NULL; } /* fixup children peer ids to te global ones */ for(i=0; i<coll->chldrn_cnt; i++){ p = hostlist_nth(hl, coll->chldrn_ids[i]); coll->chldrn_ids[i] = pmixp_info_job_hostid(p); free(p); } hostlist_destroy(hl); /* Collective state */ coll->ufwd_buf = pmixp_server_buf_new(); coll->dfwd_buf = pmixp_server_buf_new(); _reset_coll_ufwd(coll); _reset_coll_dfwd(coll); coll->cbdata = NULL; coll->cbfunc = NULL; /* init fine grained lock */ slurm_mutex_init(&coll->lock); return SLURM_SUCCESS; err_exit: return SLURM_ERROR; }
/* * information about relative ranks as assigned by the RM */ static void _set_procdatas(List lresp) { pmixp_namespace_t *nsptr = pmixp_nspaces_local(); pmix_info_t *kvp, *tkvp; char *p = NULL; int i; /* (char*) jobid assigned by scheduler */ xstrfmtcat(p, "%d.%d", pmixp_info_jobid(), pmixp_info_stepid()); PMIXP_ALLOC_KEY(kvp, PMIX_JOBID); PMIX_VAL_SET(&kvp->value, string, p); xfree(p); list_append(lresp, kvp); PMIXP_ALLOC_KEY(kvp, PMIX_NODEID); PMIX_VAL_SET(&kvp->value, uint32_t, nsptr->node_id); list_append(lresp, kvp); /* store information about local processes */ for (i = 0; i < pmixp_info_tasks(); i++) { List rankinfo; ListIterator it; int count, j, localid, nodeid; char *nodename; pmix_info_t *info; rankinfo = list_create(pmixp_xfree_xmalloced); PMIXP_ALLOC_KEY(kvp, PMIX_RANK); PMIX_VAL_SET(&kvp->value, int, i); list_append(rankinfo, kvp); /* TODO: always use 0 so far. this is not the general case though * (see SLURM MIMD: man srun, section MULTIPLE PROGRAM CONFIGURATION) */ PMIXP_ALLOC_KEY(kvp, PMIX_APPNUM); PMIX_VAL_SET(&kvp->value, int, 0); list_append(rankinfo, kvp); /* TODO: the same as for previous here */ PMIXP_ALLOC_KEY(kvp, PMIX_APPLDR); PMIX_VAL_SET(&kvp->value, int, 0); list_append(rankinfo, kvp); /* TODO: fix when several apps will appear */ PMIXP_ALLOC_KEY(kvp, PMIX_GLOBAL_RANK); PMIX_VAL_SET(&kvp->value, uint32_t, i); list_append(rankinfo, kvp); /* TODO: fix when several apps will appear */ PMIXP_ALLOC_KEY(kvp, PMIX_APP_RANK); PMIX_VAL_SET(&kvp->value, uint32_t, i); list_append(rankinfo, kvp); localid = pmixp_info_taskid2localid(i); /* this rank is local, store local info ab't it! */ if (0 <= localid) { PMIXP_ALLOC_KEY(kvp, PMIX_LOCAL_RANK); PMIX_VAL_SET(&kvp->value, uint16_t, localid); list_append(rankinfo, kvp); /* TODO: fix when several apps will appear */ PMIXP_ALLOC_KEY(kvp, PMIX_NODE_RANK); PMIX_VAL_SET(&kvp->value, uint16_t, localid); list_append(rankinfo, kvp); } nodeid = nsptr->task_map[i]; nodename = hostlist_nth(nsptr->hl, nodeid); PMIXP_ALLOC_KEY(kvp, PMIX_HOSTNAME); PMIX_VAL_SET(&kvp->value, string, nodename); list_append(rankinfo, kvp); free(nodename); /* merge rankinfo into one PMIX_PROC_DATA key */ count = list_count(rankinfo); PMIXP_ALLOC_KEY(kvp, PMIX_PROC_DATA); kvp->value.type = PMIX_INFO_ARRAY; kvp->value.data.array.size = count; PMIX_INFO_CREATE(info, count); it = list_iterator_create(rankinfo); j = 0; while (NULL != (tkvp = list_next(it))) { /* Just copy all the fields here. We will free original kvp's * using list_destroy without free'ing their fields so it is * safe to do so. */ info[j] = *tkvp; j++; } list_destroy(rankinfo); kvp->value.data.array.array = (pmix_info_t *)info; info = NULL; /* put the complex key to the list */ list_append(lresp, kvp); } }
/* * Based on ideas provided by Hongjia Cao <*****@*****.**> in PMI2 plugin */ int pmixp_coll_init(pmixp_coll_t *coll, const pmix_proc_t *procs, size_t nprocs, pmixp_coll_type_t type) { hostlist_t hl; uint32_t nodeid = 0, nodes = 0; int parent_id, depth, max_depth, tmp; int width, my_nspace = -1; char *p; int i, *ch_nodeids = NULL; #ifndef NDEBUG coll->magic = PMIXP_COLL_STATE_MAGIC; #endif coll->type = type; coll->state = PMIXP_COLL_SYNC; coll->procs = xmalloc(sizeof(*procs) * nprocs); memcpy(coll->procs, procs, sizeof(*procs) * nprocs); coll->nprocs = nprocs; coll->my_nspace = my_nspace; if (SLURM_SUCCESS != _hostset_from_ranges(procs, nprocs, &hl)) { /* TODO: provide ranges output routine */ PMIXP_ERROR("Bad ranges information"); goto err_exit; } width = slurm_get_tree_width(); nodes = hostlist_count(hl); nodeid = hostlist_find(hl, pmixp_info_hostname()); reverse_tree_info(nodeid, nodes, width, &parent_id, &tmp, &depth, &max_depth); coll->children_cnt = tmp; coll->nodeid = nodeid; /* We interested in amount of direct childs */ coll->seq = 0; coll->contrib_cntr = 0; coll->contrib_local = false; ch_nodeids = xmalloc(sizeof(int) * width); coll->ch_contribs = xmalloc(sizeof(int) * width); coll->children_cnt = reverse_tree_direct_children(nodeid, nodes, width, depth, ch_nodeids); /* create the hostlist with extract direct children's hostnames */ coll->ch_hosts = hostlist_create(""); for (i = 0; i < coll->children_cnt; i++) { char *hname = hostlist_nth(hl, ch_nodeids[i]); hostlist_push(coll->ch_hosts, hname); } /* just in case, shouldn't be needed */ hostlist_uniq(coll->ch_hosts); xfree(ch_nodeids); if (parent_id == -1) { /* if we are the root of the tree: * - we don't have a parent; * - we have large list of all_childrens (we don't want ourselfs there) */ coll->parent_host = NULL; hostlist_delete_host(hl, pmixp_info_hostname()); coll->all_children = hl; } else if (parent_id >= 0) { /* for all other nodes in the tree we need to know: * - nodename of our parent; * - we don't need a list of all_childrens and hl anymore */ p = hostlist_nth(hl, parent_id); coll->parent_host = xstrdup(p); /* use empty hostlist here */ coll->all_children = hostlist_create(""); free(p); hostlist_destroy(hl); } /* Collective data */ coll->buf = pmixp_server_new_buf(); coll->serv_offs = get_buf_offset(coll->buf); if (SLURM_SUCCESS != _pack_ranges(coll)) { PMIXP_ERROR("Cannot pack ranges to coll message header!"); goto err_exit; } /* Callback information */ coll->cbdata = NULL; coll->cbfunc = NULL; /* init fine grained lock */ slurm_mutex_init(&coll->lock); return SLURM_SUCCESS; err_exit: return SLURM_ERROR; }
/* * The pack_node_list may not be ordered across multiple components, which can * cause problems for some MPI implementations. Put the pack_node_list records * in alphabetic order and reorder pack_task_cnts pack_tids to match */ static void _reorder_pack_recs(char **in_node_list, uint16_t **in_task_cnts, uint32_t ***in_tids, int total_nnodes) { hostlist_t in_hl, out_hl; uint16_t *out_task_cnts = NULL; uint32_t **out_tids = NULL; char *hostname; int i, j; in_hl = hostlist_create(*in_node_list); if (!in_hl) { error("%s: Invalid hostlist(%s)", __func__, *in_node_list); return; } out_hl = hostlist_copy(in_hl); hostlist_sort(out_hl); hostlist_uniq(out_hl); i = hostlist_count(out_hl); if (i != total_nnodes) { error("%s: Invalid hostlist(%s) count(%d)", __func__, *in_node_list, total_nnodes); goto fini; } out_task_cnts = xmalloc(sizeof(uint16_t) * total_nnodes); out_tids = xmalloc(sizeof(uint32_t *) * total_nnodes); for (i = 0; i < total_nnodes; i++) { hostname = hostlist_nth(out_hl, i); if (!hostname) { error("%s: Invalid hostlist(%s) count(%d)", __func__, *in_node_list, total_nnodes); break; } j = hostlist_find(in_hl, hostname); if (j == -1) { error("%s: Invalid hostlist(%s) parsing", __func__, *in_node_list); free(hostname); break; } out_task_cnts[i] = in_task_cnts[0][j]; out_tids[i] = in_tids[0][j]; free(hostname); } if (i >= total_nnodes) { /* Success */ xfree(*in_node_list); *in_node_list = hostlist_ranged_string_xmalloc(out_hl); xfree(*in_task_cnts); *in_task_cnts = out_task_cnts; out_task_cnts = NULL; xfree(*in_tids); *in_tids = out_tids; out_tids = NULL; } #if 0 info("NODE_LIST[%d]:%s", total_nnodes, *in_node_list); for (i = 0; i < total_nnodes; i++) { info("TASK_CNT[%d]:%u", i, in_task_cnts[0][i]); for (j = 0; j < in_task_cnts[0][i]; j++) { info("TIDS[%d][%d]: %u", i, j, in_tids[0][i][j]); } } #endif fini: hostlist_destroy(in_hl); hostlist_destroy(out_hl); xfree(out_task_cnts); xfree(out_tids); }