static void ucp_ep_flush_progress(ucp_request_t *req) { ucp_ep_h ep = req->send.ep; ucp_lane_index_t lane; ucs_status_t status; uct_ep_h uct_ep; ucs_trace("ep %p: progress flush req %p, lanes 0x%x count %d", ep, req, req->send.flush.lanes, req->send.uct_comp.count); while (req->send.flush.lanes) { /* Search for next lane to start flush */ lane = ucs_ffs64(req->send.flush.lanes); uct_ep = ep->uct_eps[lane]; if (uct_ep == NULL) { req->send.flush.lanes &= ~UCS_BIT(lane); --req->send.uct_comp.count; continue; } /* Start flush operation on UCT endpoint */ status = uct_ep_flush(uct_ep, 0, &req->send.uct_comp); ucs_trace("flushing ep %p lane[%d]: %s", ep, lane, ucs_status_string(status)); if (status == UCS_OK) { req->send.flush.lanes &= ~UCS_BIT(lane); --req->send.uct_comp.count; } else if (status == UCS_INPROGRESS) { req->send.flush.lanes &= ~UCS_BIT(lane); } else if (status == UCS_ERR_NO_RESOURCE) { if (req->send.lane != UCP_NULL_LANE) { ucs_trace("ep %p: not adding pending flush %p on lane %d, " "because it's already pending on lane %d", ep, req, lane, req->send.lane); break; } req->send.lane = lane; status = uct_ep_pending_add(uct_ep, &req->send.uct); ucs_trace("adding pending flush on ep %p lane[%d]: %s", ep, lane, ucs_status_string(status)); if (status == UCS_OK) { req->send.flush.lanes &= ~UCS_BIT(lane); } else if (status != UCS_ERR_BUSY) { ucp_ep_flush_error(req, status); } } else { ucp_ep_flush_error(req, status); } } }
void ucp_ep_add_pending(ucp_ep_h ep, uct_ep_h uct_ep, ucp_request_t *req) { ucs_status_t status; ucs_trace_data("add pending request %p uct %p to uct_ep %p", req, &req->send.uct, uct_ep); req->send.ep = ep; for (;;) { status = uct_ep_pending_add(uct_ep, &req->send.uct); if (status != UCS_ERR_BUSY) { ucs_assert(status == UCS_OK); return; /* Added to pending */ } /* Forced progress */ status = req->send.uct.func(&req->send.uct); if (status == UCS_OK) { return; /* Completed the operation */ } } }
ucs_status_t ucp_ep_add_pending_uct(ucp_ep_h ep, uct_ep_h uct_ep, uct_pending_req_t *req) { ucs_status_t status; ucs_assertv(req->func != NULL, "req=%p", req); status = uct_ep_pending_add(uct_ep, req); if (status != UCS_ERR_BUSY) { ucs_assert(status == UCS_OK); ucs_trace_data("ep %p: added pending uct request %p to uct_ep %p", ep, req, uct_ep); return UCS_OK; /* Added to pending */ } /* Forced progress */ status = req->func(req); if (status == UCS_OK) { return UCS_OK; /* Completed the operation */ } return UCS_ERR_NO_PROGRESS; }