static int _serv_read(eio_obj_t *obj, List objs) { PMIXP_DEBUG("fd = %d", obj->fd); pmixp_io_engine_t *me = (pmixp_io_engine_t *)obj->arg; bool proceed = true; pmixp_debug_hang(0); /* Read and process all received messages */ while (proceed) { switch( _process_message(me) ){ case 2: obj->shutdown = true; PMIXP_DEBUG("Connection finalized fd = %d", obj->fd); /* cleanup after this connection */ eio_remove_obj(obj, objs); xfree(me); case 0: proceed = 0; case 1: break; } } return 0; }
static int _serv_read(eio_obj_t *obj, List objs) { /* sanity check */ xassert(NULL != obj ); if (obj->shutdown) { /* corresponding connection will be * cleaned up during plugin finalize */ return 0; } PMIXP_DEBUG("fd = %d", obj->fd); pmixp_conn_t *conn = (pmixp_conn_t *)obj->arg; bool proceed = true; /* debug stub */ pmixp_debug_hang(0); /* Read and process all received messages */ while (proceed) { if (!pmixp_conn_progress_rcv(conn)) { proceed = 0; } if (!pmixp_conn_is_alive(conn)) { obj->shutdown = true; PMIXP_DEBUG("Connection closed fd = %d", obj->fd); /* cleanup after this connection */ eio_remove_obj(obj, objs); pmixp_conn_return(conn); proceed = 0; } } return 0; }
static int _serv_write(eio_obj_t *obj, List objs) { /* sanity check */ xassert(NULL != obj ); if (obj->shutdown) { /* corresponding connection will be * cleaned up during plugin finalize */ return 0; } PMIXP_DEBUG("fd = %d", obj->fd); pmixp_conn_t *conn = (pmixp_conn_t *)obj->arg; /* debug stub */ pmixp_debug_hang(0); /* progress sends */ pmixp_conn_progress_snd(conn); /* if we are done with this connection - remove it */ if (!pmixp_conn_is_alive(conn)) { obj->shutdown = true; PMIXP_DEBUG("Connection finalized fd = %d", obj->fd); /* cleanup after this connection */ eio_remove_obj(obj, objs); pmixp_conn_return(conn); } return 0; }
static void _dfwd_sent_cb(int rc, pmixp_p2p_ctx_t ctx, void *_vcbdata) { pmixp_coll_cbdata_t *cbdata = (pmixp_coll_cbdata_t*)_vcbdata; pmixp_coll_t *coll = cbdata->coll; if( PMIXP_P2P_REGULAR == ctx ){ /* lock the collective */ slurm_mutex_lock(&coll->lock); } if (cbdata->seq != coll->seq) { /* it seems like this collective was reset since the time * we initiated this send. * Just exit to avoid data corruption. */ PMIXP_DEBUG("Collective was reset!"); goto exit; } xassert(PMIXP_COLL_DOWNFWD == coll->state); /* Change the status */ if( SLURM_SUCCESS == rc ){ coll->dfwd_cb_cnt++; } else { coll->dfwd_status = PMIXP_COLL_SND_FAILED; } #ifdef PMIXP_COLL_DEBUG PMIXP_DEBUG("%p: state: %s, snd_status=%s, compl_cnt=%d/%d", coll, pmixp_coll_state2str(coll->state), pmixp_coll_sndstatus2str(coll->dfwd_status), coll->dfwd_cb_cnt, coll->dfwd_cb_wait); #endif exit: xassert(0 < cbdata->refcntr); cbdata->refcntr--; if (!cbdata->refcntr) { xfree(cbdata); } if( PMIXP_P2P_REGULAR == ctx ){ /* progress, in the inline case progress * will be invoked by the caller */ _progress_coll(coll); /* unlock the collective */ slurm_mutex_unlock(&coll->lock); } }
static pmix_status_t lookup_fn(const pmix_proc_t *proc, char **keys, const pmix_info_t info[], size_t ninfo, pmix_lookup_cbfunc_t cbfunc, void *cbdata) { PMIXP_DEBUG("called"); return PMIX_ERR_NOT_IMPLEMENTED; }
static pmix_status_t _disconnect_fn(const pmix_proc_t procs[], size_t nprocs, const pmix_info_t info[], size_t ninfo, pmix_op_cbfunc_t cbfunc, void *cbdata) { PMIXP_DEBUG("called"); return PMIX_ERR_NOT_SUPPORTED; }
void pmixp_server_direct_conn(int fd) { eio_obj_t *obj; pmixp_conn_t *conn; PMIXP_DEBUG("Request from fd = %d", fd); /* Set nonblocking */ fd_set_nonblocking(fd); fd_set_close_on_exec(fd); pmixp_fd_set_nodelay(fd); conn = pmixp_conn_new_temp(PMIXP_PROTO_DIRECT, fd, _direct_conn_establish); /* try to process right here */ pmixp_conn_progress_rcv(conn); if (!pmixp_conn_is_alive(conn)) { /* success, don't need this connection anymore */ pmixp_conn_return(conn); return; } /* If it is a blocking operation: create AIO object to * handle it */ obj = eio_obj_create(fd, &direct_peer_ops, (void *)conn); eio_new_obj(pmixp_info_io(), obj); /* wakeup this connection to get processed */ eio_signal_wakeup(pmixp_info_io()); }
/* * TODO: we need to keep track of the "me" * structures created here, because we need to * free them in "pmixp_stepd_finalize" */ void pmixp_server_slurm_conn(int fd) { eio_obj_t *obj; pmixp_conn_t *conn = NULL; PMIXP_DEBUG("Request from fd = %d", fd); pmixp_debug_hang(0); /* Set nonblocking */ fd_set_nonblocking(fd); fd_set_close_on_exec(fd); conn = pmixp_conn_new_temp(PMIXP_PROTO_SLURM, fd, _slurm_new_msg); /* try to process right here */ pmixp_conn_progress_rcv(conn); if (!pmixp_conn_is_alive(conn)) { /* success, don't need this connection anymore */ pmixp_conn_return(conn); return; } /* If it is a blocking operation: create AIO object to * handle it */ obj = eio_obj_create(fd, &slurm_peer_ops, (void *)conn); eio_new_obj(pmixp_info_io(), obj); }
int pmixp_coll_ring_init(pmixp_coll_t *coll, hostlist_t *hl) { #ifdef PMIXP_COLL_DEBUG PMIXP_DEBUG("called"); #endif int i; pmixp_coll_ring_ctx_t *coll_ctx = NULL; pmixp_coll_ring_t *ring = &coll->state.ring; char *p; int rel_id = hostlist_find(*hl, pmixp_info_hostname()); /* compute the next absolute id of the neighbor */ p = hostlist_nth(*hl, (rel_id + 1) % coll->peers_cnt); ring->next_peerid = pmixp_info_job_hostid(p); free(p); ring->fwrd_buf_pool = list_create(pmixp_free_buf); ring->ring_buf_pool = list_create(pmixp_free_buf); for (i = 0; i < PMIXP_COLL_RING_CTX_NUM; i++) { coll_ctx = &ring->ctx_array[i]; coll_ctx->coll = coll; coll_ctx->in_use = false; coll_ctx->seq = coll->seq; coll_ctx->contrib_local = false; coll_ctx->contrib_prev = 0; coll_ctx->state = PMIXP_COLL_RING_SYNC; // TODO bit vector coll_ctx->contrib_map = xmalloc(sizeof(bool) * coll->peers_cnt); } return SLURM_SUCCESS; }
int pmixp_stepd_finalize(void) { char *path; if (!_was_initialized) { /* nothing to do */ return 0; } pmixp_libpmix_finalize(); pmixp_dmdx_finalize(); pmixp_conn_fini(); pmixp_dconn_fini(); pmixp_state_finalize(); pmixp_nspaces_finalize(); /* cleanup the UNIX socket */ PMIXP_DEBUG("Remove PMIx plugin usock"); close(pmixp_info_srv_usock_fd()); path = pmixp_info_nspace_usock(pmixp_info_namespace()); unlink(path); xfree(path); /* free the information */ pmixp_info_free(); return SLURM_SUCCESS; }
int pmixp_coll_contrib_local(pmixp_coll_t *coll, pmixp_coll_type_t type, char *data, size_t ndata, void *cbfunc, void *cbdata) { int ret = SLURM_SUCCESS; #ifdef PMIXP_COLL_DEBUG PMIXP_DEBUG("%p: %s seq=%d, size=%lu", coll, pmixp_coll_type2str(type), coll->seq, ndata); #endif switch (type) { case PMIXP_COLL_TYPE_FENCE_TREE: ret = pmixp_coll_tree_local(coll, data, ndata, cbfunc, cbdata); break; case PMIXP_COLL_TYPE_FENCE_RING: ret = pmixp_coll_ring_local(coll, data, ndata, cbfunc, cbdata); break; default: ret = SLURM_ERROR; break; } return ret; }
/* * TODO: we need to keep track of the "me" * structures created here, because we need to * free them in "pmixp_stepd_finalize" */ void pmix_server_new_conn(int fd) { eio_obj_t *obj; PMIXP_DEBUG("Request from fd = %d", fd); /* Set nonblocking */ fd_set_nonblocking(fd); fd_set_close_on_exec(fd); pmixp_io_engine_t *me = xmalloc(sizeof(pmixp_io_engine_t)); pmix_io_init(me, fd, srv_rcvd_header); /* We use slurm_forward_data to send message to stepd's * SLURM will put user ID there. We need to skip it. */ pmix_io_rcvd_padding(me, sizeof(uint32_t)); if( 2 == _process_message(me) ){ /* connection was fully processed here */ xfree(me); return; } /* If it is a blocking operation: create AIO object to * handle it */ obj = eio_obj_create(fd, &peer_ops, (void *)me); eio_new_obj(pmixp_info_io(), obj); }
int pmixp_p2p_send(const char *nodename, const char *address, const char *data, uint32_t len, unsigned int start_delay, unsigned int retry_cnt, int silent) { int retry = 0, rc; unsigned int delay = start_delay; /* in milliseconds */ pmixp_debug_hang(0); while (1) { if (!silent && retry >= 1) { PMIXP_DEBUG("send failed, rc=%d, try #%d", rc, retry); } rc = _pmix_p2p_send_core(nodename, address, data, len); if (rc == SLURM_SUCCESS) break; retry++; if (retry >= retry_cnt) { PMIXP_ERROR("send failed, rc=%d, exceeded the retry limit", rc); break; } /* wait with constantly increasing delay */ struct timespec ts = {(delay / 1000), ((delay % 1000) * 1000000)}; nanosleep(&ts, NULL); delay *= 2; } return rc; }
static pmix_status_t _unpublish_fn(const pmix_proc_t *proc, char **keys, const pmix_info_t info[], size_t ninfo, pmix_op_cbfunc_t cbfunc, void *cbdata) { PMIXP_DEBUG("called"); return PMIX_ERR_NOT_SUPPORTED; }
static pmix_status_t _fencenb_fn(const pmix_proc_t procs_v2[], size_t nprocs, const pmix_info_t info[], size_t ninfo, char *data, size_t ndata, pmix_modex_cbfunc_t cbfunc, void *cbdata) { PMIXP_DEBUG("called"); pmixp_coll_t *coll; pmixp_coll_type_t type = PMIXP_COLL_TYPE_FENCE; pmix_status_t status = PMIX_SUCCESS; int ret; size_t i; pmixp_proc_t *procs = xmalloc(sizeof(*procs) * nprocs); for (i = 0; i < nprocs; i++) { procs[i].rank = procs_v2[i].rank; strncpy(procs[i].nspace, procs_v2[i].nspace, PMIXP_MAX_NSLEN); } coll = pmixp_state_coll_get(type, procs, nprocs); ret = pmixp_coll_contrib_local(coll, data, ndata, cbfunc, cbdata); xfree(procs); if (SLURM_SUCCESS != ret) { status = PMIX_ERROR; goto error; } return PMIX_SUCCESS; error: cbfunc(status, NULL, 0, cbdata, NULL, NULL); return status; }
static int _pmix_p2p_send_core(const char *nodename, const char *address, const char *data, uint32_t len) { int rc, timeout; slurm_msg_t msg; forward_data_msg_t req; List ret_list; ret_data_info_t *ret_data_info = NULL; pmixp_debug_hang(0); slurm_msg_t_init(&msg); PMIXP_DEBUG("nodelist=%s, address=%s, len=%u", nodename, address, len); req.address = (char *)address; req.len = len; /* there is not much we can do - just cast) */ req.data = (char*)data; msg.msg_type = REQUEST_FORWARD_DATA; msg.data = &req; if (slurm_conf_get_addr(nodename, &msg.address) == SLURM_ERROR) { PMIXP_ERROR("Can't find address for host " "%s, check slurm.conf", nodename); return SLURM_ERROR; } timeout = slurm_get_msg_timeout() * 1000; msg.forward.timeout = timeout; msg.forward.cnt = 0; msg.forward.nodelist = NULL; ret_list = slurm_send_addr_recv_msgs(&msg, (char*)nodename, timeout); if (!ret_list) { /* This should never happen (when this was * written slurm_send_addr_recv_msgs always * returned a list */ PMIXP_ERROR("No return list given from " "slurm_send_addr_recv_msgs spawned for %s", nodename); return SLURM_ERROR; } else if ((errno != SLURM_COMMUNICATIONS_CONNECTION_ERROR) && !list_count(ret_list)) { PMIXP_ERROR("failed to send to %s, errno=%d", nodename, errno); return SLURM_ERROR; } rc = SLURM_SUCCESS; while ((ret_data_info = list_pop(ret_list))) { int temp_rc = slurm_get_return_code(ret_data_info->type, ret_data_info->data); if (temp_rc != SLURM_SUCCESS) rc = temp_rc; destroy_data_info(ret_data_info); } FREE_NULL_LIST(ret_list); return rc; }
static pmix_status_t _spawn_fn(const pmix_proc_t *proc, const pmix_info_t job_info[], size_t ninfo, const pmix_app_t apps[], size_t napps, pmix_spawn_cbfunc_t cbfunc, void *cbdata) { PMIXP_DEBUG("called"); return PMIX_ERR_NOT_SUPPORTED; }
static pmix_status_t _dmodex_fn(const pmix_proc_t *proc, const pmix_info_t info[], size_t ninfo, pmix_modex_cbfunc_t cbfunc, void *cbdata) { int rc; PMIXP_DEBUG("called"); rc = pmixp_dmdx_get(proc->nspace, proc->rank, cbfunc, cbdata); return (SLURM_SUCCESS == rc) ? PMIX_SUCCESS : PMIX_ERROR; }
static pmix_status_t abort_fn(const pmix_proc_t *proc, void *server_object, int status, const char msg[], pmix_proc_t procs[], size_t nprocs, pmix_op_cbfunc_t cbfunc, void *cbdata) { /* Just kill this stepid for now. Think what we can do for FT here? */ PMIXP_DEBUG("called: status = %d, msg = %s", status, msg); slurm_kill_job_step(pmixp_info_jobid(), pmixp_info_stepid(), SIGKILL); if (NULL != cbfunc) { cbfunc(PMIX_SUCCESS, cbdata); } return PMIX_SUCCESS; }
static int _ring_forward_data(pmixp_coll_ring_ctx_t *coll_ctx, uint32_t contrib_id, uint32_t hop_seq, void *data, size_t size) { pmixp_coll_ring_msg_hdr_t hdr; pmixp_coll_t *coll = _ctx_get_coll(coll_ctx); pmixp_coll_ring_t *ring = &coll->state.ring; hdr.nodeid = coll->my_peerid; hdr.msgsize = size; hdr.seq = coll_ctx->seq; hdr.hop_seq = hop_seq; hdr.contrib_id = contrib_id; pmixp_ep_t *ep = (pmixp_ep_t*)xmalloc(sizeof(*ep)); pmixp_coll_ring_cbdata_t *cbdata = NULL; uint32_t offset = 0; Buf buf = _get_fwd_buf(coll_ctx); int rc = SLURM_SUCCESS; pmixp_coll_ring_ctx_sanity_check(coll_ctx); #ifdef PMIXP_COLL_DEBUG PMIXP_DEBUG("%p: transit data to nodeid=%d, seq=%d, hop=%d, size=%lu, contrib=%d", coll_ctx, _ring_next_id(coll), hdr.seq, hdr.hop_seq, hdr.msgsize, hdr.contrib_id); #endif if (!buf) { rc = SLURM_ERROR; goto exit; } ep->type = PMIXP_EP_NOIDEID; ep->ep.nodeid = ring->next_peerid; /* pack ring info */ _pack_coll_ring_info(coll, &hdr, buf); /* insert payload to buf */ offset = get_buf_offset(buf); pmixp_server_buf_reserve(buf, size); memcpy(get_buf_data(buf) + offset, data, size); set_buf_offset(buf, offset + size); cbdata = xmalloc(sizeof(pmixp_coll_ring_cbdata_t)); cbdata->buf = buf; cbdata->coll = coll; cbdata->coll_ctx = coll_ctx; cbdata->seq = coll_ctx->seq; rc = pmixp_server_send_nb(ep, PMIXP_MSG_RING, coll_ctx->seq, buf, _ring_sent_cb, cbdata); exit: return rc; }
int pmixp_coll_contrib_local(pmixp_coll_t *coll, char *data, size_t size) { PMIXP_DEBUG("%s:%d: get local contribution", pmixp_info_namespace(), pmixp_info_nodeid()); /* sanity check */ pmixp_coll_sanity_check(coll); /* lock the structure */ slurm_mutex_lock(&coll->lock); /* change the collective state if need */ if (PMIXP_COLL_SYNC == coll->state) { PMIXP_DEBUG( "%s:%d: get local contribution: switch to PMIXP_COLL_FAN_IN", pmixp_info_namespace(), pmixp_info_nodeid()); coll->state = PMIXP_COLL_FAN_IN; coll->ts = time(NULL); } xassert(PMIXP_COLL_FAN_IN == coll->state); /* save & mark local contribution */ coll->contrib_local = true; grow_buf(coll->buf, size); memcpy(get_buf_data(coll->buf) + get_buf_offset(coll->buf), data, size); set_buf_offset(coll->buf, get_buf_offset(coll->buf) + size); /* unlock the structure */ slurm_mutex_unlock(&coll->lock); /* check if the collective is ready to progress */ _progress_fan_in(coll); PMIXP_DEBUG("%s:%d: get local contribution: finish", pmixp_info_namespace(), pmixp_info_nodeid()); return SLURM_SUCCESS; }
void _progres_fan_out(pmixp_coll_t *coll, Buf buf) { PMIXP_DEBUG("%s:%d: start", pmixp_info_namespace(), pmixp_info_nodeid()); pmixp_coll_sanity_check(coll); xassert(PMIXP_COLL_FAN_OUT == coll->state || PMIXP_COLL_FAN_OUT_IN == coll->state); /* update the database */ if (NULL != coll->cbfunc) { void *data = get_buf_data(buf) + get_buf_offset(buf); size_t size = remaining_buf(buf); PMIXP_DEBUG("%s:%d: use the callback", pmixp_info_namespace(), pmixp_info_nodeid()); coll->cbfunc(PMIX_SUCCESS, data, size, coll->cbdata, pmixp_free_Buf, (void *)buf); } /* Prepare for the next collective operation */ _fan_out_finished(coll); PMIXP_DEBUG("%s:%d: collective is prepared for the next use", pmixp_info_namespace(), pmixp_info_nodeid()); }
static void _ring_sent_cb(int rc, pmixp_p2p_ctx_t ctx, void *_cbdata) { pmixp_coll_ring_cbdata_t *cbdata = (pmixp_coll_ring_cbdata_t*)_cbdata; pmixp_coll_ring_ctx_t *coll_ctx = cbdata->coll_ctx; pmixp_coll_t *coll = cbdata->coll; Buf buf = cbdata->buf; pmixp_coll_sanity_check(coll); if (PMIXP_P2P_REGULAR == ctx) { /* lock the collective */ slurm_mutex_lock(&coll->lock); } #ifdef PMIXP_COLL_DEBUG PMIXP_DEBUG("%p: called %d", coll_ctx, coll_ctx->seq); #endif if (cbdata->seq != coll_ctx->seq) { /* it seems like this collective was reset since the time * we initiated this send. * Just exit to avoid data corruption. */ PMIXP_DEBUG("%p: collective was reset!", coll_ctx); goto exit; } coll_ctx->forward_cnt++; _progress_coll_ring(coll_ctx); exit: pmixp_server_buf_reset(buf); list_push(coll->state.ring.fwrd_buf_pool, buf); if (PMIXP_P2P_REGULAR == ctx) { /* unlock the collective */ slurm_mutex_unlock(&coll->lock); } xfree(cbdata); }
static void _reset_coll_ring(pmixp_coll_ring_ctx_t *coll_ctx) { pmixp_coll_t *coll = _ctx_get_coll(coll_ctx); #ifdef PMIXP_COLL_DEBUG PMIXP_DEBUG("%p: called", coll_ctx); #endif pmixp_coll_ring_ctx_sanity_check(coll_ctx); coll_ctx->in_use = false; coll_ctx->state = PMIXP_COLL_RING_SYNC; coll_ctx->contrib_local = false; coll_ctx->contrib_prev = 0; coll_ctx->forward_cnt = 0; coll->ts = time(NULL); memset(coll_ctx->contrib_map, 0, sizeof(bool) * coll->peers_cnt); coll_ctx->ring_buf = NULL; }
void pmixp_coll_bcast(pmixp_coll_t *coll, Buf buf) { PMIXP_DEBUG("%s:%d: start", pmixp_info_namespace(), pmixp_info_nodeid()); /* lock the structure */ slurm_mutex_lock(&coll->lock); _progres_fan_out(coll, buf); /* unlock the structure */ slurm_mutex_unlock(&coll->lock); /* We may already start next collective. Try to progress! * its OK if we in SYNC - there will be no-op */ _progress_fan_in(coll); }
int pmixp_coll_ring_local(pmixp_coll_t *coll, char *data, size_t size, void *cbfunc, void *cbdata) { int ret = SLURM_SUCCESS; pmixp_coll_ring_ctx_t *coll_ctx = NULL; /* lock the structure */ slurm_mutex_lock(&coll->lock); /* sanity check */ pmixp_coll_sanity_check(coll); /* setup callback info */ coll->cbfunc = cbfunc; coll->cbdata = cbdata; coll_ctx = pmixp_coll_ring_ctx_new(coll); if (!coll_ctx) { PMIXP_ERROR("Can not get new ring collective context, seq=%u", coll->seq); ret = SLURM_ERROR; goto exit; } #ifdef PMIXP_COLL_DEBUG PMIXP_DEBUG("%p: contrib/loc: seqnum=%u, state=%d, size=%lu", coll_ctx, coll_ctx->seq, coll_ctx->state, size); #endif if (_pmixp_coll_contrib(coll_ctx, coll->my_peerid, 0, data, size)) { goto exit; } /* mark local contribution */ coll_ctx->contrib_local = true; _progress_coll_ring(coll_ctx); exit: /* unlock the structure */ slurm_mutex_unlock(&coll->lock); return ret; }
static void _libpmix_cb(void *_vcbdata) { pmixp_coll_cbdata_t *cbdata = (pmixp_coll_cbdata_t*)_vcbdata; pmixp_coll_t *coll = cbdata->coll; /* lock the collective */ slurm_mutex_lock(&coll->lock); if (cbdata->seq != coll->seq) { /* it seems like this collective was reset since the time * we initiated this send. * Just exit to avoid data corruption. */ PMIXP_ERROR("%p: collective was reset: myseq=%u, curseq=%u", coll, cbdata->seq, coll->seq); goto exit; } xassert(PMIXP_COLL_DOWNFWD == coll->state); coll->dfwd_cb_cnt++; #ifdef PMIXP_COLL_DEBUG PMIXP_DEBUG("%p: state: %s, snd_status=%s, compl_cnt=%d/%d", coll, pmixp_coll_state2str(coll->state), pmixp_coll_sndstatus2str(coll->dfwd_status), coll->dfwd_cb_cnt, coll->dfwd_cb_wait); #endif _progress_coll(coll); exit: xassert(0 < cbdata->refcntr); cbdata->refcntr--; if (!cbdata->refcntr) { xfree(cbdata); } /* unlock the collective */ slurm_mutex_unlock(&coll->lock); }
pmix_status_t fencenb_fn(const pmix_proc_t procs[], size_t nprocs, const pmix_info_t info[], size_t ninfo, char *data, size_t ndata, pmix_modex_cbfunc_t cbfunc, void *cbdata) { PMIXP_DEBUG("called"); pmixp_coll_t *coll; pmixp_coll_type_t type = PMIXP_COLL_TYPE_FENCE; pmix_status_t status = PMIX_SUCCESS; pmixp_debug_hang(0); coll = pmixp_state_coll_get(type, procs, nprocs); pmixp_coll_set_callback(coll, cbfunc, cbdata); if (SLURM_SUCCESS != pmixp_coll_contrib_local(coll, data, ndata)) { goto error; } return PMIX_SUCCESS; error: cbfunc(status, NULL, 0, cbdata, NULL, NULL); return status; }
static int _progress_ufwd_wpc(pmixp_coll_t *coll) { xassert(PMIXP_COLL_UPFWD_WPC == coll->state); if (!coll->contrib_prnt) { return false; } /* Need to wait only for the local completion callback if installed*/ coll->dfwd_status = PMIXP_COLL_SND_ACTIVE; coll->dfwd_cb_wait = 0; /* move to the next state */ coll->state = PMIXP_COLL_DOWNFWD; /* local delivery */ if (coll->cbfunc) { pmixp_coll_cbdata_t *cbdata; cbdata = xmalloc(sizeof(pmixp_coll_cbdata_t)); cbdata->coll = coll; cbdata->seq = coll->seq; cbdata->refcntr = 1; char *data = get_buf_data(coll->dfwd_buf) + coll->dfwd_offset; size_t size = get_buf_offset(coll->dfwd_buf) - coll->dfwd_offset; coll->cbfunc(PMIX_SUCCESS, data, size, coll->cbdata, _libpmix_cb, (void *)cbdata); coll->dfwd_cb_wait++; #ifdef PMIXP_COLL_DEBUG PMIXP_DEBUG("%p: local delivery, size = %lu", coll, (uint64_t)size); #endif } /* events observed - need another iteration */ return true; }
int pmixp_stepd_send(const char *nodelist, const char *address, const char *data, uint32_t len, unsigned int start_delay, unsigned int retry_cnt, int silent) { int retry = 0, rc; unsigned int delay = start_delay; /* in milliseconds */ char *copy_of_nodelist = xstrdup(nodelist); while (1) { if (!silent && retry >= 1) { PMIXP_DEBUG("send failed, rc=%d, try #%d", rc, retry); } rc = slurm_forward_data(©_of_nodelist, (char *)address, len, data); if (rc == SLURM_SUCCESS) break; retry++; if (retry >= retry_cnt) { PMIXP_ERROR("send failed, rc=%d, exceeded the retry limit", rc); break; } /* wait with constantly increasing delay */ struct timespec ts = {(delay / 1000), ((delay % 1000) * 1000000)}; nanosleep(&ts, NULL); delay *= 2; } xfree(copy_of_nodelist); return rc; }