static void ucp_ep_flush_progress(ucp_request_t *req)
{
ucp_ep_h ep = req->send.ep;
ucp_lane_index_t lane;
ucs_status_t status;
uct_ep_h uct_ep;
ucs_trace("ep %p: progress flush req %p, lanes 0x%x count %d", ep, req,
req->send.flush.lanes, req->send.uct_comp.count);
while (req->send.flush.lanes) {
/* Search for next lane to start flush */
lane = ucs_ffs64(req->send.flush.lanes);
uct_ep = ep->uct_eps[lane];
if (uct_ep == NULL) {
req->send.flush.lanes &= ~UCS_BIT(lane);
--req->send.uct_comp.count;
continue;
}
/* Start flush operation on UCT endpoint */
status = uct_ep_flush(uct_ep, 0, &req->send.uct_comp);
ucs_trace("flushing ep %p lane[%d]: %s", ep, lane,
ucs_status_string(status));
if (status == UCS_OK) {
req->send.flush.lanes &= ~UCS_BIT(lane);
--req->send.uct_comp.count;
} else if (status == UCS_INPROGRESS) {
req->send.flush.lanes &= ~UCS_BIT(lane);
} else if (status == UCS_ERR_NO_RESOURCE) {
if (req->send.lane != UCP_NULL_LANE) {
ucs_trace("ep %p: not adding pending flush %p on lane %d, "
"because it's already pending on lane %d",
ep, req, lane, req->send.lane);
break;
}
req->send.lane = lane;
status = uct_ep_pending_add(uct_ep, &req->send.uct);
ucs_trace("adding pending flush on ep %p lane[%d]: %s", ep, lane,
ucs_status_string(status));
if (status == UCS_OK) {
req->send.flush.lanes &= ~UCS_BIT(lane);
} else if (status != UCS_ERR_BUSY) {
ucp_ep_flush_error(req, status);
}
} else {
ucp_ep_flush_error(req, status);
}
}
}
void ucp_ep_add_pending(ucp_ep_h ep, uct_ep_h uct_ep, ucp_request_t *req)
{
ucs_status_t status;
ucs_trace_data("add pending request %p uct %p to uct_ep %p", req,
&req->send.uct, uct_ep);
req->send.ep = ep;
for (;;) {
status = uct_ep_pending_add(uct_ep, &req->send.uct);
if (status != UCS_ERR_BUSY) {
ucs_assert(status == UCS_OK);
return; /* Added to pending */
}
/* Forced progress */
status = req->send.uct.func(&req->send.uct);
if (status == UCS_OK) {
return; /* Completed the operation */
}
}
}
ucs_status_t ucp_ep_add_pending_uct(ucp_ep_h ep, uct_ep_h uct_ep,
uct_pending_req_t *req)
{
ucs_status_t status;
ucs_assertv(req->func != NULL, "req=%p", req);
status = uct_ep_pending_add(uct_ep, req);
if (status != UCS_ERR_BUSY) {
ucs_assert(status == UCS_OK);
ucs_trace_data("ep %p: added pending uct request %p to uct_ep %p", ep,
req, uct_ep);
return UCS_OK; /* Added to pending */
}
/* Forced progress */
status = req->func(req);
if (status == UCS_OK) {
return UCS_OK; /* Completed the operation */
}
return UCS_ERR_NO_PROGRESS;
}