static int _progress_ufwd_sc(pmixp_coll_t *coll)
{
xassert(PMIXP_COLL_UPFWD_WSC == coll->state);
/* for some reasons doesnt switch to downfwd */
switch (coll->ufwd_status) {
case PMIXP_COLL_SND_FAILED:
/* something went wrong with upward send.
* notify libpmix about that and abort
* collective */
if (coll->cbfunc) {
coll->cbfunc(PMIX_ERROR, NULL, 0, coll->cbdata,
NULL, NULL);
}
_reset_coll(coll);
/* Don't need to do anything else */
return false;
case PMIXP_COLL_SND_ACTIVE:
/* still waiting for the send completion */
return false;
case PMIXP_COLL_SND_DONE:
/* move to the next step */
break;
default:
/* Should not happen, fatal error */
abort();
}
/* We now can upward part for the next collective */
_reset_coll_ufwd(coll);
/* move to the next state */
coll->state = PMIXP_COLL_UPFWD_WPC;
return true;
}
void pmixp_coll_reset_if_to(pmixp_coll_t *coll, time_t ts)
{
/* lock the */
slurm_mutex_lock(&coll->lock);
if (PMIXP_COLL_SYNC == coll->state) {
goto unlock;
}
if (ts - coll->ts > pmixp_info_timeout()) {
/* respond to the libpmix */
if (coll->contrib_local && coll->cbfunc) {
/* Call the callback only if:
* - we were asked to do that (coll->cbfunc != NULL);
* - local contribution was received.
* TODO: we may want to mark this event to respond with
* to the next local request immediately and with the
* proper (status == PMIX_ERR_TIMEOUT)
*/
coll->cbfunc(PMIX_ERR_TIMEOUT, NULL, 0, coll->cbdata,
NULL, NULL);
}
/* drop the collective */
_reset_coll(coll);
/* report the timeout event */
PMIXP_ERROR("Collective timeout!");
}
unlock:
/* unlock the structure */
slurm_mutex_unlock(&coll->lock);
}
static int _progress_dfwd(pmixp_coll_t *coll)
{
xassert(PMIXP_COLL_DOWNFWD == coll->state);
/* if all childrens + local callbacks was invoked */
if (coll->dfwd_cb_wait == coll->dfwd_cb_cnt) {
coll->dfwd_status = PMIXP_COLL_SND_DONE;
}
switch (coll->dfwd_status) {
case PMIXP_COLL_SND_ACTIVE:
return false;
case PMIXP_COLL_SND_FAILED:
/* something went wrong with upward send.
* notify libpmix about that and abort
* collective */
PMIXP_ERROR("%p: failed to send, abort collective", coll);
if (coll->cbfunc) {
coll->cbfunc(PMIX_ERROR, NULL, 0, coll->cbdata,
NULL, NULL);
}
_reset_coll(coll);
/* Don't need to do anything else */
return false;
case PMIXP_COLL_SND_DONE:
break;
default:
/* Should not happen, fatal error */
abort();
}
#ifdef PMIXP_COLL_DEBUG
PMIXP_DEBUG("%p: collective is DONE", coll);
#endif
_reset_coll(coll);
return true;
}
void pmixp_coll_reset_if_to(pmixp_coll_t *coll, time_t ts)
{
/* lock the */
slurm_mutex_lock(&coll->lock);
if (PMIXP_COLL_SYNC == coll->state) {
goto unlock;
}
if (ts - coll->ts > pmixp_info_timeout()) {
/* respond to the libpmix */
coll->cbfunc(PMIX_ERR_TIMEOUT, NULL, 0, coll->cbdata, NULL,
NULL);
/* drop the collective */
_reset_coll(coll);
/* report the timeout event */
PMIXP_ERROR("Collective timeout!");
}
unlock:
/* unlock the structure */
slurm_mutex_unlock(&coll->lock);
}
static int _progress_ufwd(pmixp_coll_t *coll)
{
pmixp_ep_t ep[coll->chldrn_cnt];
int ep_cnt = 0;
int rc, i;
char *nodename = NULL;
pmixp_coll_cbdata_t *cbdata = NULL;
xassert(PMIXP_COLL_UPFWD == coll->state);
/* for some reasons doesnt switch to downfwd */
switch (coll->ufwd_status) {
case PMIXP_COLL_SND_FAILED:
/* something went wrong with upward send.
* notify libpmix about that and abort
* collective */
if (coll->cbfunc) {
coll->cbfunc(PMIX_ERROR, NULL, 0, coll->cbdata,
NULL, NULL);
}
_reset_coll(coll);
/* Don't need to do anything else */
return false;
case PMIXP_COLL_SND_ACTIVE:
/* still waiting for the send completion */
return false;
case PMIXP_COLL_SND_DONE:
if (coll->contrib_prnt) {
/* all-set to go to the next stage */
break;
}
return false;
default:
/* Should not happen, fatal error */
abort();
}
/* We now can upward part for the next collective */
_reset_coll_ufwd(coll);
/* move to the next state */
coll->state = PMIXP_COLL_DOWNFWD;
coll->dfwd_status = PMIXP_COLL_SND_ACTIVE;
if (!pmixp_info_srv_direct_conn()) {
/* only root of the tree should get here */
xassert(0 > coll->prnt_peerid);
if (coll->chldrn_cnt) {
/* We can run on just one node */
ep[ep_cnt].type = PMIXP_EP_HLIST;
ep[ep_cnt].ep.hostlist = coll->chldrn_str;
ep_cnt++;
}
} else {
for(i=0; i<coll->chldrn_cnt; i++){
ep[i].type = PMIXP_EP_NOIDEID;
ep[i].ep.nodeid = coll->chldrn_ids[i];
ep_cnt++;
}
}
/* We need to wait for ep_cnt send completions + the local callback */
coll->dfwd_cb_wait = ep_cnt;
if (ep_cnt || coll->cbfunc) {
/* allocate the callback data */
cbdata = xmalloc(sizeof(pmixp_coll_cbdata_t));
cbdata->coll = coll;
cbdata->seq = coll->seq;
cbdata->refcntr = ep_cnt;
if (coll->cbfunc) {
cbdata->refcntr++;
}
}
for(i=0; i < ep_cnt; i++){
rc = pmixp_server_send_nb(&ep[i], PMIXP_MSG_FAN_OUT, coll->seq,
coll->dfwd_buf,
_dfwd_sent_cb, cbdata);
if (SLURM_SUCCESS != rc) {
if (PMIXP_EP_NOIDEID == ep[i].type){
nodename = pmixp_info_job_host(ep[i].ep.nodeid);
PMIXP_ERROR("Cannot send data (size = %lu), "
"to %s:%d",
(uint64_t) get_buf_offset(coll->dfwd_buf),
nodename, ep[i].ep.nodeid);
xfree(nodename);
} else {
PMIXP_ERROR("Cannot send data (size = %lu), "
"to %s",
(uint64_t) get_buf_offset(coll->dfwd_buf),
ep[i].ep.hostlist);
}
coll->dfwd_status = PMIXP_COLL_SND_FAILED;
}
#ifdef PMIXP_COLL_DEBUG
if (PMIXP_EP_NOIDEID == ep[i].type) {
nodename = pmixp_info_job_host(ep[i].ep.nodeid);
PMIXP_DEBUG("%p: fwd to %s:%d, size = %lu",
coll, nodename, ep[i].ep.nodeid,
(uint64_t) get_buf_offset(coll->dfwd_buf));
xfree(nodename);
} else {
PMIXP_DEBUG("%p: fwd to %s, size = %lu",
coll, ep[i].ep.hostlist,
(uint64_t) get_buf_offset(coll->dfwd_buf));
}
#endif
}
if (coll->cbfunc) {
char *data = get_buf_data(coll->dfwd_buf) + coll->dfwd_offset;
size_t size = get_buf_offset(coll->dfwd_buf) -
coll->dfwd_offset;
coll->dfwd_cb_wait++;
coll->cbfunc(PMIX_SUCCESS, data, size, coll->cbdata,
_libpmix_cb, (void *)cbdata);
#ifdef PMIXP_COLL_DEBUG
PMIXP_DEBUG("%p: local delivery, size = %lu",
coll, (uint64_t)size);
#endif
}
/* events observed - need another iteration */
return true;
}