static int _slurm_send(pmixp_ep_t *ep, pmixp_base_hdr_t bhdr, Buf buf)
{
const char *addr = NULL, *data = NULL, *hostlist = NULL;
char nhdr[PMIXP_BASE_HDR_MAX];
size_t hsize = 0, dsize = 0;
int rc;
/* setup the header */
addr = pmixp_info_srv_usock_path();
bhdr.ext_flag = 0;
if (pmixp_info_srv_direct_conn() && PMIXP_EP_NOIDEID == ep->type) {
bhdr.ext_flag = 1;
}
hsize = _slurm_pack_hdr(&bhdr, nhdr);
data = _buf_finalize(buf, nhdr, hsize, &dsize);
switch( ep->type ){
case PMIXP_EP_HLIST:
hostlist = ep->ep.hostlist;
rc = pmixp_stepd_send(ep->ep.hostlist, addr,
data, dsize, 500, 7, 0);
break;
case PMIXP_EP_NOIDEID: {
char *nodename = pmixp_info_job_host(ep->ep.nodeid);
rc = pmixp_p2p_send(nodename, addr, data, dsize,
500, 7, 0);
xfree(nodename);
break;
}
default:
PMIXP_ERROR("Bad value of the EP type: %d", (int)ep->type);
abort();
}
if (SLURM_SUCCESS != rc) {
PMIXP_ERROR("Cannot send message to %s, size = %u, "
"hostlist:\n%s",
addr, (uint32_t) dsize, hostlist);
}
return rc;
}
int pmixp_server_send_nb(pmixp_ep_t *ep, pmixp_srv_cmd_t type,
uint32_t seq, Buf buf,
pmixp_server_sent_cb_t complete_cb,
void *cb_data)
{
pmixp_base_hdr_t bhdr;
int rc = SLURM_ERROR;
pmixp_dconn_t *dconn = NULL;
PMIXP_BASE_HDR_SETUP(bhdr, type, seq, buf);
/* if direct connection is not enabled
* always use SLURM protocol
*/
if (!pmixp_info_srv_direct_conn()) {
goto send_slurm;
}
switch (ep->type) {
case PMIXP_EP_HLIST:
goto send_slurm;
case PMIXP_EP_NOIDEID:{
int hostid;
hostid = ep->ep.nodeid;
xassert(0 <= hostid);
dconn = pmixp_dconn_lock(hostid);
switch (pmixp_dconn_state(dconn)) {
case PMIXP_DIRECT_EP_SENT:
case PMIXP_DIRECT_CONNECTED:
/* keep the lock here and proceed
* to the direct send
*/
goto send_direct;
case PMIXP_DIRECT_INIT:
pmixp_dconn_req_sent(dconn);
pmixp_dconn_unlock(dconn);
goto send_slurm;
default:{
/* this is a bug! */
pmixp_dconn_state_t state = pmixp_dconn_state(dconn);
pmixp_dconn_unlock(dconn);
PMIXP_ERROR("Bad direct connection state: %d",
(int)state);
xassert( (state == PMIXP_DIRECT_INIT) ||
(state == PMIXP_DIRECT_EP_SENT) ||
(state == PMIXP_DIRECT_CONNECTED) );
abort();
}
}
}
default:
PMIXP_ERROR("Bad value of the endpoint type: %d",
(int)ep->type);
xassert( PMIXP_EP_HLIST == ep->type ||
PMIXP_EP_NOIDEID == ep->type);
abort();
}
return rc;
send_slurm:
rc = _slurm_send(ep, bhdr, buf);
complete_cb(rc, PMIXP_P2P_INLINE, cb_data);
return SLURM_SUCCESS;
send_direct:
xassert( NULL != dconn );
_direct_send(dconn, ep, bhdr, buf, complete_cb, cb_data);
pmixp_dconn_unlock(dconn);
return SLURM_SUCCESS;
}
static int _progress_collect(pmixp_coll_t *coll)
{
pmixp_ep_t ep = {0};
int rc;
xassert(PMIXP_COLL_COLLECT == coll->state);
ep.type = PMIXP_EP_NONE;
#ifdef PMIXP_COLL_DEBUG
PMIXP_DEBUG("%p: state=%s, local=%d, child_cntr=%d",
coll, pmixp_coll_state2str(coll->state),
(int)coll->contrib_local, coll->contrib_children);
#endif
/* lock the collective */
pmixp_coll_sanity_check(coll);
if (PMIXP_COLL_COLLECT != coll->state) {
/* In case of race condition between libpmix and
* slurm threads we can be called
* after we moved to the next step. */
return 0;
}
if (!coll->contrib_local ||
coll->contrib_children != coll->chldrn_cnt) {
/* Not yet ready to go to the next step */
return 0;
}
if (pmixp_info_srv_direct_conn()) {
/* We will need to forward aggregated
* message back to our children */
coll->state = PMIXP_COLL_UPFWD;
} else {
/* If we use SLURM API (SAPI) - intermediate nodes
* don't need to forward data as the root will do
* SAPI broadcast.
* So, only root has to go through the full UPFWD
* state and send the message back.
* Other procs have to go through other route. The reason for
* that is the fact that som of out children can receive bcast
* message early and initiate next collective. We need to handle
* that properly.
*/
if (0 > coll->prnt_peerid) {
coll->state = PMIXP_COLL_UPFWD;
} else {
coll->state = PMIXP_COLL_UPFWD_WSC;
}
}
/* The root of the collective will have parent_host == NULL */
if (NULL != coll->prnt_host) {
ep.type = PMIXP_EP_NOIDEID;
ep.ep.nodeid = coll->prnt_peerid;
coll->ufwd_status = PMIXP_COLL_SND_ACTIVE;
PMIXP_DEBUG("%p: send data to %s:%d",
coll, coll->prnt_host, coll->prnt_peerid);
} else {
/* move data from input buffer to the output */
char *dst, *src = get_buf_data(coll->ufwd_buf) +
coll->ufwd_offset;
size_t size = get_buf_offset(coll->ufwd_buf) -
coll->ufwd_offset;
pmixp_server_buf_reserve(coll->dfwd_buf, size);
dst = get_buf_data(coll->dfwd_buf) + coll->dfwd_offset;
memcpy(dst, src, size);
set_buf_offset(coll->dfwd_buf, coll->dfwd_offset + size);
/* no need to send */
coll->ufwd_status = PMIXP_COLL_SND_DONE;
/* this is root */
coll->contrib_prnt = true;
}
if (PMIXP_EP_NONE != ep.type) {
pmixp_coll_cbdata_t *cbdata;
cbdata = xmalloc(sizeof(pmixp_coll_cbdata_t));
cbdata->coll = coll;
cbdata->seq = coll->seq;
cbdata->refcntr = 1;
char *nodename = coll->prnt_host;
rc = pmixp_server_send_nb(&ep, PMIXP_MSG_FAN_IN, coll->seq,
coll->ufwd_buf,
_ufwd_sent_cb, cbdata);
if (SLURM_SUCCESS != rc) {
PMIXP_ERROR("Cannot send data (size = %lu), "
"to %s:%d",
(uint64_t) get_buf_offset(coll->ufwd_buf),
nodename, ep.ep.nodeid);
coll->ufwd_status = PMIXP_COLL_SND_FAILED;
}
#ifdef PMIXP_COLL_DEBUG
PMIXP_DEBUG("%p: fwd to %s:%d, size = %lu",
coll, nodename, ep.ep.nodeid,
(uint64_t) get_buf_offset(coll->dfwd_buf));
#endif
}
/* events observed - need another iteration */
return true;
}
static int _progress_ufwd(pmixp_coll_t *coll)
{
pmixp_ep_t ep[coll->chldrn_cnt];
int ep_cnt = 0;
int rc, i;
char *nodename = NULL;
pmixp_coll_cbdata_t *cbdata = NULL;
xassert(PMIXP_COLL_UPFWD == coll->state);
/* for some reasons doesnt switch to downfwd */
switch (coll->ufwd_status) {
case PMIXP_COLL_SND_FAILED:
/* something went wrong with upward send.
* notify libpmix about that and abort
* collective */
if (coll->cbfunc) {
coll->cbfunc(PMIX_ERROR, NULL, 0, coll->cbdata,
NULL, NULL);
}
_reset_coll(coll);
/* Don't need to do anything else */
return false;
case PMIXP_COLL_SND_ACTIVE:
/* still waiting for the send completion */
return false;
case PMIXP_COLL_SND_DONE:
if (coll->contrib_prnt) {
/* all-set to go to the next stage */
break;
}
return false;
default:
/* Should not happen, fatal error */
abort();
}
/* We now can upward part for the next collective */
_reset_coll_ufwd(coll);
/* move to the next state */
coll->state = PMIXP_COLL_DOWNFWD;
coll->dfwd_status = PMIXP_COLL_SND_ACTIVE;
if (!pmixp_info_srv_direct_conn()) {
/* only root of the tree should get here */
xassert(0 > coll->prnt_peerid);
if (coll->chldrn_cnt) {
/* We can run on just one node */
ep[ep_cnt].type = PMIXP_EP_HLIST;
ep[ep_cnt].ep.hostlist = coll->chldrn_str;
ep_cnt++;
}
} else {
for(i=0; i<coll->chldrn_cnt; i++){
ep[i].type = PMIXP_EP_NOIDEID;
ep[i].ep.nodeid = coll->chldrn_ids[i];
ep_cnt++;
}
}
/* We need to wait for ep_cnt send completions + the local callback */
coll->dfwd_cb_wait = ep_cnt;
if (ep_cnt || coll->cbfunc) {
/* allocate the callback data */
cbdata = xmalloc(sizeof(pmixp_coll_cbdata_t));
cbdata->coll = coll;
cbdata->seq = coll->seq;
cbdata->refcntr = ep_cnt;
if (coll->cbfunc) {
cbdata->refcntr++;
}
}
for(i=0; i < ep_cnt; i++){
rc = pmixp_server_send_nb(&ep[i], PMIXP_MSG_FAN_OUT, coll->seq,
coll->dfwd_buf,
_dfwd_sent_cb, cbdata);
if (SLURM_SUCCESS != rc) {
if (PMIXP_EP_NOIDEID == ep[i].type){
nodename = pmixp_info_job_host(ep[i].ep.nodeid);
PMIXP_ERROR("Cannot send data (size = %lu), "
"to %s:%d",
(uint64_t) get_buf_offset(coll->dfwd_buf),
nodename, ep[i].ep.nodeid);
xfree(nodename);
} else {
PMIXP_ERROR("Cannot send data (size = %lu), "
"to %s",
(uint64_t) get_buf_offset(coll->dfwd_buf),
ep[i].ep.hostlist);
}
coll->dfwd_status = PMIXP_COLL_SND_FAILED;
}
#ifdef PMIXP_COLL_DEBUG
if (PMIXP_EP_NOIDEID == ep[i].type) {
nodename = pmixp_info_job_host(ep[i].ep.nodeid);
PMIXP_DEBUG("%p: fwd to %s:%d, size = %lu",
coll, nodename, ep[i].ep.nodeid,
(uint64_t) get_buf_offset(coll->dfwd_buf));
xfree(nodename);
} else {
PMIXP_DEBUG("%p: fwd to %s, size = %lu",
coll, ep[i].ep.hostlist,
(uint64_t) get_buf_offset(coll->dfwd_buf));
}
#endif
}
if (coll->cbfunc) {
char *data = get_buf_data(coll->dfwd_buf) + coll->dfwd_offset;
size_t size = get_buf_offset(coll->dfwd_buf) -
coll->dfwd_offset;
coll->dfwd_cb_wait++;
coll->cbfunc(PMIX_SUCCESS, data, size, coll->cbdata,
_libpmix_cb, (void *)cbdata);
#ifdef PMIXP_COLL_DEBUG
PMIXP_DEBUG("%p: local delivery, size = %lu",
coll, (uint64_t)size);
#endif
}
/* events observed - need another iteration */
return true;
}
int pmixp_coll_contrib_parent(pmixp_coll_t *coll, uint32_t peerid,
uint32_t seq, Buf buf)
{
#ifdef PMIXP_COLL_DEBUG
char *nodename = NULL;
int lpeerid = -1;
#endif
char *data_src = NULL, *data_dst = NULL;
uint32_t size;
int expected_peerid;
/* lock the structure */
slurm_mutex_lock(&coll->lock);
if (pmixp_info_srv_direct_conn()) {
expected_peerid = coll->prnt_peerid;
} else {
expected_peerid = coll->root_peerid;
}
/* Sanity check */
pmixp_coll_sanity_check(coll);
if (expected_peerid != peerid) {
char *nodename = pmixp_info_job_host(peerid);
/* protect ourselfs if we are running with no asserts */
PMIXP_ERROR("%p: parent contrib from bad nodeid=%s:%u, "
"expect=%d",
coll, nodename, peerid, expected_peerid);
xfree(nodename);
goto proceed;
}
#ifdef PMIXP_COLL_DEBUG
nodename = pmixp_info_job_host(peerid);
lpeerid = hostlist_find(coll->peers_hl, nodename);
/* Mark this event */
PMIXP_DEBUG("%p: contrib/rem from %s:%d(%d): state=%s, size=%u",
coll, nodename, peerid, lpeerid,
pmixp_coll_state2str(coll->state), remaining_buf(buf));
#endif
switch (coll->state) {
case PMIXP_COLL_SYNC:
case PMIXP_COLL_COLLECT:
/* It looks like a retransmission attempt when remote side
* identified transmission failure, but we actually successfuly
* received the message */
#ifdef PMIXP_COLL_DEBUG
PMIXP_DEBUG("%p: prev contrib from %s:%d(%d): "
"seq=%u, cur_seq=%u, state=%s",
coll, nodename, peerid, lpeerid,
seq, coll->seq,
pmixp_coll_state2str(coll->state));
#endif
/* sanity check */
if ((coll->seq - 1) != seq) {
/* FATAL: should not happen in normal workflow */
char *nodename = pmixp_info_job_host(peerid);
PMIXP_ERROR("%p: unexpected contrib from %s:%d: "
"contrib_seq = %d, coll->seq = %d, "
"state=%s",
coll, nodename, peerid,
seq, coll->seq,
pmixp_coll_state2str(coll->state));
xfree(nodename);
xassert((coll->seq - 1) == seq);
abort();
}
goto proceed;
case PMIXP_COLL_UPFWD_WSC:{
/* we are not actually ready to receive this contribution as
* the upward portion of the collective wasn't received yet.
* This should not happen as SAPI (SLURM API) is blocking and
* we chould transit to PMIXP_COLL_UPFWD_WPC immediately */
/* FATAL: should not happen in normal workflow */
char *nodename = pmixp_info_job_host(peerid);
PMIXP_ERROR("%p: unexpected contrib from %s:%d: "
"contrib_seq = %d, coll->seq = %d, "
"state=%s",
coll, nodename, peerid,
seq, coll->seq,
pmixp_coll_state2str(coll->state));
xfree(nodename);
xassert((coll->seq - 1) == seq);
abort();
}
case PMIXP_COLL_UPFWD:
case PMIXP_COLL_UPFWD_WPC:
/* we were waiting for this */
break;
case PMIXP_COLL_DOWNFWD:
/* It looks like a retransmission attempt when remote side
* identified transmission failure, but we actually successfuly
* received the message */
#ifdef PMIXP_COLL_DEBUG
PMIXP_DEBUG("%p: double contrib from %s:%d(%d) "
"seq=%u, cur_seq=%u, state=%s",
coll, nodename, peerid, lpeerid,
seq, coll->seq, pmixp_coll_state2str(coll->state));
#endif
/* sanity check */
if (coll->seq != seq) {
char *nodename = pmixp_info_job_host(peerid);
/* FATAL: should not happen in normal workflow */
PMIXP_ERROR("%p: unexpected contrib from %s:%d: "
"seq = %d, coll->seq = %d, state=%s",
coll, nodename, peerid,
seq, coll->seq,
pmixp_coll_state2str(coll->state));
xassert((coll->seq - 1) == seq);
xfree(nodename);
abort();
}
goto proceed;
default:
/* should not happen in normal workflow */
PMIXP_ERROR("%p: unknown collective state %s",
coll, pmixp_coll_state2str(coll->state));
abort();
}
/* Because of possible timeouts/delays in transmission we
* can receive a contribution second time. Avoid duplications
* by checking our records. */
if (coll->contrib_prnt) {
char *nodename = pmixp_info_job_host(peerid);
/* May be 0 or 1. If grater - transmission skew, ignore.
* NOTE: this output is not on the critical path -
* don't preprocess it out */
PMIXP_DEBUG("%p: multiple contributions from parent %s:%d",
coll, nodename, peerid);
xfree(nodename);
/* this is duplication, skip. */
goto proceed;
}
coll->contrib_prnt = true;
data_src = get_buf_data(buf) + get_buf_offset(buf);
size = remaining_buf(buf);
pmixp_server_buf_reserve(coll->dfwd_buf, size);
data_dst = get_buf_data(coll->dfwd_buf) +
get_buf_offset(coll->dfwd_buf);
memcpy(data_dst, data_src, size);
set_buf_offset(coll->dfwd_buf,
get_buf_offset(coll->dfwd_buf) + size);
proceed:
_progress_coll(coll);
#ifdef PMIXP_COLL_DEBUG
if (nodename) {
PMIXP_DEBUG("%p: finish: node=%s:%d(%d), state=%s",
coll, nodename, peerid, lpeerid,
pmixp_coll_state2str(coll->state));
xfree(nodename);
}
#endif
/* unlock the structure */
slurm_mutex_unlock(&coll->lock);
return SLURM_SUCCESS;
}
