static int
_delete_all (hostlist_t hl, hostlist_t dl)
{
int rc = 0;
char * host = NULL;
hostlist_iterator_t i = hostlist_iterator_create (dl);
while ((host = hostlist_next (i))) {
rc += hostlist_delete_host (hl, host);
free (host);
}
hostlist_iterator_destroy (i);
return (rc);
}
static int _send_to_stepds(hostlist_t hl, const char *addr, uint32_t len,
char *data)
{
List ret_list = NULL;
int temp_rc = 0, rc = 0;
ret_data_info_t *ret_data_info = NULL;
slurm_msg_t *msg = xmalloc(sizeof(slurm_msg_t));
forward_data_msg_t req;
char *nodelist = NULL;
slurm_msg_t_init(msg);
req.address = xstrdup(addr);
req.len = len;
req.data = data;
msg->msg_type = REQUEST_FORWARD_DATA;
msg->data = &req;
nodelist = hostlist_ranged_string_xmalloc(hl);
if ((ret_list = slurm_send_recv_msgs(nodelist, msg, 0, false))) {
while ((ret_data_info = list_pop(ret_list))) {
temp_rc = slurm_get_return_code(ret_data_info->type,
ret_data_info->data);
if (temp_rc) {
rc = temp_rc;
} else {
hostlist_delete_host(hl,
ret_data_info->node_name);
}
}
} else {
error("tree_msg_to_stepds: no list was returned");
rc = SLURM_ERROR;
}
slurm_free_msg(msg);
xfree(nodelist);
xfree(req.address);
return rc;
}
void *_fwd_tree_thread(void *arg)
{
fwd_tree_t *fwd_tree = (fwd_tree_t *)arg;
List ret_list = NULL;
char *name = NULL;
char *buf = NULL;
slurm_msg_t send_msg;
slurm_msg_t_init(&send_msg);
send_msg.msg_type = fwd_tree->orig_msg->msg_type;
send_msg.data = fwd_tree->orig_msg->data;
send_msg.protocol_version = fwd_tree->orig_msg->protocol_version;
/* repeat until we are sure the message was sent */
while ((name = hostlist_shift(fwd_tree->tree_hl))) {
if (slurm_conf_get_addr(name, &send_msg.address)
== SLURM_ERROR) {
error("fwd_tree_thread: can't find address for host "
"%s, check slurm.conf", name);
slurm_mutex_lock(fwd_tree->tree_mutex);
mark_as_failed_forward(&fwd_tree->ret_list, name,
SLURM_UNKNOWN_FORWARD_ADDR);
slurm_cond_signal(fwd_tree->notify);
slurm_mutex_unlock(fwd_tree->tree_mutex);
free(name);
continue;
}
send_msg.forward.timeout = fwd_tree->timeout;
if ((send_msg.forward.cnt = hostlist_count(fwd_tree->tree_hl))){
buf = hostlist_ranged_string_xmalloc(
fwd_tree->tree_hl);
send_msg.forward.nodelist = buf;
} else
send_msg.forward.nodelist = NULL;
if (send_msg.forward.nodelist && send_msg.forward.nodelist[0]) {
debug3("Tree sending to %s along with %s",
name, send_msg.forward.nodelist);
} else
debug3("Tree sending to %s", name);
ret_list = slurm_send_addr_recv_msgs(&send_msg, name,
fwd_tree->timeout);
xfree(send_msg.forward.nodelist);
if (ret_list) {
int ret_cnt = list_count(ret_list);
/* This is most common if a slurmd is running
an older version of Slurm than the
originator of the message.
*/
if ((ret_cnt <= send_msg.forward.cnt) &&
(errno != SLURM_COMMUNICATIONS_CONNECTION_ERROR)) {
error("fwd_tree_thread: %s failed to forward "
"the message, expecting %d ret got only "
"%d",
name, send_msg.forward.cnt + 1, ret_cnt);
if (ret_cnt > 1) { /* not likely */
ret_data_info_t *ret_data_info = NULL;
ListIterator itr =
list_iterator_create(ret_list);
while ((ret_data_info =
list_next(itr))) {
if (xstrcmp(ret_data_info->
node_name, name))
hostlist_delete_host(
fwd_tree->
tree_hl,
ret_data_info->
node_name);
}
list_iterator_destroy(itr);
}
}
slurm_mutex_lock(fwd_tree->tree_mutex);
list_transfer(fwd_tree->ret_list, ret_list);
slurm_cond_signal(fwd_tree->notify);
slurm_mutex_unlock(fwd_tree->tree_mutex);
FREE_NULL_LIST(ret_list);
/* try next node */
if (ret_cnt <= send_msg.forward.cnt) {
free(name);
/* Abandon tree. This way if all the
* nodes in the branch are down we
* don't have to time out for each
* node serially.
*/
_start_msg_tree_internal(
fwd_tree->tree_hl, NULL,
fwd_tree,
hostlist_count(fwd_tree->tree_hl));
continue;
}
} else {
/* This should never happen (when this was
* written slurm_send_addr_recv_msgs always
* returned a list */
error("fwd_tree_thread: no return list given from "
"slurm_send_addr_recv_msgs spawned for %s",
name);
slurm_mutex_lock(fwd_tree->tree_mutex);
mark_as_failed_forward(
&fwd_tree->ret_list, name,
SLURM_COMMUNICATIONS_CONNECTION_ERROR);
slurm_cond_signal(fwd_tree->notify);
slurm_mutex_unlock(fwd_tree->tree_mutex);
free(name);
continue;
}
free(name);
/* check for error and try again */
if (errno == SLURM_COMMUNICATIONS_CONNECTION_ERROR)
continue;
break;
}
_destroy_tree_fwd(fwd_tree);
return NULL;
}
/*
* Based on ideas provided by Hongjia Cao <*****@*****.**> in PMI2 plugin
*/
int pmixp_coll_init(pmixp_coll_t *coll, const pmix_proc_t *procs,
size_t nprocs, pmixp_coll_type_t type)
{
hostlist_t hl;
int max_depth, width, depth, i;
char *p;
#ifndef NDEBUG
coll->magic = PMIXP_COLL_STATE_MAGIC;
#endif
coll->type = type;
coll->state = PMIXP_COLL_SYNC;
coll->pset.procs = xmalloc(sizeof(*procs) * nprocs);
coll->pset.nprocs = nprocs;
memcpy(coll->pset.procs, procs, sizeof(*procs) * nprocs);
if (SLURM_SUCCESS != _hostset_from_ranges(procs, nprocs, &hl)) {
/* TODO: provide ranges output routine */
PMIXP_ERROR("Bad ranges information");
goto err_exit;
}
#ifdef PMIXP_COLL_DEBUG
/* if we debug collectives - store a copy of a full
* hostlist to resolve participant id to the hostname */
coll->peers_hl = hostlist_copy(hl);
#endif
width = slurm_get_tree_width();
coll->peers_cnt = hostlist_count(hl);
coll->my_peerid = hostlist_find(hl, pmixp_info_hostname());
reverse_tree_info(coll->my_peerid, coll->peers_cnt, width,
&coll->prnt_peerid, &coll->chldrn_cnt, &depth,
&max_depth);
/* We interested in amount of direct childs */
coll->seq = 0;
coll->contrib_children = 0;
coll->contrib_local = false;
coll->chldrn_ids = xmalloc(sizeof(int) * width);
coll->contrib_chld = xmalloc(sizeof(int) * width);
coll->chldrn_cnt = reverse_tree_direct_children(coll->my_peerid,
coll->peers_cnt,
width, depth,
coll->chldrn_ids);
if (coll->prnt_peerid == -1) {
/* if we are the root of the tree:
* - we don't have a parent;
* - we have large list of all_childrens (we don't want
* ourselfs there)
*/
coll->prnt_host = NULL;
coll->all_chldrn_hl = hostlist_copy(hl);
hostlist_delete_host(coll->all_chldrn_hl,
pmixp_info_hostname());
coll->chldrn_str =
hostlist_ranged_string_xmalloc(coll->all_chldrn_hl);
} else {
/* for all other nodes in the tree we need to know:
* - nodename of our parent;
* - we don't need a list of all_childrens and hl anymore
*/
/*
* setup parent id's
*/
p = hostlist_nth(hl, coll->prnt_peerid);
coll->prnt_host = xstrdup(p);
free(p);
/* reset prnt_peerid to the global peer */
coll->prnt_peerid = pmixp_info_job_hostid(coll->prnt_host);
/*
* setup root id's
* (we need this for the SLURM API communication case)
*/
p = hostlist_nth(hl, 0);
coll->root_host = xstrdup(p);
free(p);
/* reset prnt_peerid to the global peer */
coll->root_peerid = pmixp_info_job_hostid(coll->root_host);
/* use empty hostlist here */
coll->all_chldrn_hl = hostlist_create("");
coll->chldrn_str = NULL;
}
/* fixup children peer ids to te global ones */
for(i=0; i<coll->chldrn_cnt; i++){
p = hostlist_nth(hl, coll->chldrn_ids[i]);
coll->chldrn_ids[i] = pmixp_info_job_hostid(p);
free(p);
}
hostlist_destroy(hl);
/* Collective state */
coll->ufwd_buf = pmixp_server_buf_new();
coll->dfwd_buf = pmixp_server_buf_new();
_reset_coll_ufwd(coll);
_reset_coll_dfwd(coll);
coll->cbdata = NULL;
coll->cbfunc = NULL;
/* init fine grained lock */
slurm_mutex_init(&coll->lock);
return SLURM_SUCCESS;
err_exit:
return SLURM_ERROR;
}
void pmixp_coll_ring_log(pmixp_coll_t *coll)
{
int i;
pmixp_coll_ring_t *ring = &coll->state.ring;
char *nodename, *next, *prev;
char *out_str = NULL;
PMIXP_ERROR("%p: %s state seq=%d",
coll, pmixp_coll_type2str(coll->type), coll->seq);
nodename = pmixp_info_job_host(coll->my_peerid);
PMIXP_ERROR("my peerid: %d:%s", coll->my_peerid, nodename);
xfree(nodename);
next = pmixp_info_job_host(_ring_next_id(coll));
prev = pmixp_info_job_host(_ring_prev_id(coll));
xstrfmtcat(out_str,"neighbor id: next %d:%s, prev %d:%s",
_ring_next_id(coll), next, _ring_prev_id(coll), prev);
PMIXP_ERROR("%s", out_str);
xfree(next);
xfree(prev);
xfree(out_str);
for (i = 0; i < PMIXP_COLL_RING_CTX_NUM; i++) {
pmixp_coll_ring_ctx_t *coll_ctx = &ring->ctx_array[i];
PMIXP_ERROR("Context ptr=%p, #%d, in-use=%d",
coll_ctx, i, coll_ctx->in_use);
if (coll_ctx->in_use) {
int id;
char *done_contrib, *wait_contrib;
hostlist_t hl_done_contrib, hl_wait_contrib;
pmixp_hostset_from_ranges(coll->pset.procs,
coll->pset.nprocs,
&hl_done_contrib);
hl_wait_contrib = hostlist_copy(hl_done_contrib);
PMIXP_ERROR("\t seq=%d contribs: loc=%d/prev=%d/fwd=%d",
coll_ctx->seq, coll_ctx->contrib_local,
coll_ctx->contrib_prev,
coll_ctx->forward_cnt);
PMIXP_ERROR("\t neighbor contribs [%d]:",
coll->peers_cnt);
for (id = 0; id < coll->peers_cnt; id++) {
char *nodename = pmixp_info_job_host(id);
if(coll_ctx->contrib_map[id]) {
hostlist_delete_host(hl_wait_contrib,
nodename);
} else {
hostlist_delete_host(hl_done_contrib,
nodename);
}
xfree(nodename);
}
done_contrib = slurm_hostlist_ranged_string_xmalloc(
hl_done_contrib);
wait_contrib = slurm_hostlist_ranged_string_xmalloc(
hl_wait_contrib);
PMIXP_ERROR("\t done contrib: %s",
strlen(done_contrib) ? done_contrib : "-");
PMIXP_ERROR("\t wait contrib: %s",
strlen(wait_contrib) ? wait_contrib : "-");
PMIXP_ERROR("\t status=%s",
pmixp_coll_ring_state2str(coll_ctx->state));
PMIXP_ERROR("\t buf size=%u, remain=%u",
size_buf(coll_ctx->ring_buf),
remaining_buf(coll_ctx->ring_buf));
xfree(done_contrib);
xfree(wait_contrib);
hostlist_destroy(hl_done_contrib);
hostlist_destroy(hl_wait_contrib);
}
}
}
/*
* Based on ideas provided by Hongjia Cao <*****@*****.**> in PMI2 plugin
*/
int pmixp_coll_init(pmixp_coll_t *coll, const pmix_proc_t *procs,
size_t nprocs, pmixp_coll_type_t type)
{
hostlist_t hl;
uint32_t nodeid = 0, nodes = 0;
int parent_id, depth, max_depth, tmp;
int width, my_nspace = -1;
char *p;
int i, *ch_nodeids = NULL;
#ifndef NDEBUG
coll->magic = PMIXP_COLL_STATE_MAGIC;
#endif
coll->type = type;
coll->state = PMIXP_COLL_SYNC;
coll->procs = xmalloc(sizeof(*procs) * nprocs);
memcpy(coll->procs, procs, sizeof(*procs) * nprocs);
coll->nprocs = nprocs;
coll->my_nspace = my_nspace;
if (SLURM_SUCCESS != _hostset_from_ranges(procs, nprocs, &hl)) {
/* TODO: provide ranges output routine */
PMIXP_ERROR("Bad ranges information");
goto err_exit;
}
width = slurm_get_tree_width();
nodes = hostlist_count(hl);
nodeid = hostlist_find(hl, pmixp_info_hostname());
reverse_tree_info(nodeid, nodes, width, &parent_id, &tmp, &depth,
&max_depth);
coll->children_cnt = tmp;
coll->nodeid = nodeid;
/* We interested in amount of direct childs */
coll->seq = 0;
coll->contrib_cntr = 0;
coll->contrib_local = false;
ch_nodeids = xmalloc(sizeof(int) * width);
coll->ch_contribs = xmalloc(sizeof(int) * width);
coll->children_cnt = reverse_tree_direct_children(nodeid, nodes, width,
depth, ch_nodeids);
/* create the hostlist with extract direct children's hostnames */
coll->ch_hosts = hostlist_create("");
for (i = 0; i < coll->children_cnt; i++) {
char *hname = hostlist_nth(hl, ch_nodeids[i]);
hostlist_push(coll->ch_hosts, hname);
}
/* just in case, shouldn't be needed */
hostlist_uniq(coll->ch_hosts);
xfree(ch_nodeids);
if (parent_id == -1) {
/* if we are the root of the tree:
* - we don't have a parent;
* - we have large list of all_childrens (we don't want ourselfs there)
*/
coll->parent_host = NULL;
hostlist_delete_host(hl, pmixp_info_hostname());
coll->all_children = hl;
} else if (parent_id >= 0) {
/* for all other nodes in the tree we need to know:
* - nodename of our parent;
* - we don't need a list of all_childrens and hl anymore
*/
p = hostlist_nth(hl, parent_id);
coll->parent_host = xstrdup(p);
/* use empty hostlist here */
coll->all_children = hostlist_create("");
free(p);
hostlist_destroy(hl);
}
/* Collective data */
coll->buf = pmixp_server_new_buf();
coll->serv_offs = get_buf_offset(coll->buf);
if (SLURM_SUCCESS != _pack_ranges(coll)) {
PMIXP_ERROR("Cannot pack ranges to coll message header!");
goto err_exit;
}
/* Callback information */
coll->cbdata = NULL;
coll->cbfunc = NULL;
/* init fine grained lock */
slurm_mutex_init(&coll->lock);
return SLURM_SUCCESS;
err_exit:
return SLURM_ERROR;
}
