ucs_arbiter_cb_result_t uct_mm_ep_process_pending(ucs_arbiter_t *arbiter,
ucs_arbiter_elem_t *elem,
void *arg)
{
uct_pending_req_t *req = ucs_container_of(elem, uct_pending_req_t, priv);
ucs_status_t status;
uct_mm_ep_t *ep = ucs_container_of(ucs_arbiter_elem_group(elem), uct_mm_ep_t, arb_group);
/* update the local tail with its actual value from the remote peer
* making sure that the pending sends would use the real tail value */
ucs_memory_cpu_load_fence();
ep->cached_tail = ep->fifo_ctl->tail;
if (!uct_mm_ep_has_tx_resources(ep)) {
return UCS_ARBITER_CB_RESULT_RESCHED_GROUP;
}
status = req->func(req);
ucs_trace_data("progress pending request %p returned %s", req,
ucs_status_string(status));
if (status == UCS_OK) {
/* sent successfully. remove from the arbiter */
return UCS_ARBITER_CB_RESULT_REMOVE_ELEM;
} else if (status == UCS_INPROGRESS) {
/* sent but not completed, keep in the arbiter */
return UCS_ARBITER_CB_RESULT_NEXT_GROUP;
} else {
/* couldn't send. keep this request in the arbiter until the next time
* this function is called */
return UCS_ARBITER_CB_RESULT_RESCHED_GROUP;
}
}
static ucs_arbiter_cb_result_t uct_dc_ep_abriter_purge_cb(ucs_arbiter_t *arbiter,
ucs_arbiter_elem_t *elem,
void *arg)
{
uct_purge_cb_args_t *cb_args = arg;
uct_pending_purge_callback_t cb = cb_args->cb;
uct_pending_req_t *req = ucs_container_of(elem, uct_pending_req_t, priv);
uct_rc_fc_request_t *freq = ucs_derived_of(req, uct_rc_fc_request_t);
uct_dc_ep_t *ep = ucs_container_of(ucs_arbiter_elem_group(elem),
uct_dc_ep_t, arb_group);
if (ucs_likely(req->func != uct_dc_iface_fc_grant)){
if (cb != NULL) {
cb(req, cb_args->arg);
} else {
ucs_warn("ep=%p cancelling user pending request %p", ep, req);
}
} else {
/* User callback should not be called for FC messages.
* Just return pending request memory to the pool */
ucs_mpool_put(freq);
}
return UCS_ARBITER_CB_RESULT_REMOVE_ELEM;
}
ucs_arbiter_cb_result_t uct_ugni_ep_process_pending(ucs_arbiter_t *arbiter,
ucs_arbiter_elem_t *elem,
void *arg){
uct_ugni_ep_t *ep = ucs_container_of(ucs_arbiter_elem_group(elem), uct_ugni_ep_t, arb_group);
uct_pending_req_t *req = ucs_container_of(elem, uct_pending_req_t, priv);
ucs_status_t rc;
ep->arb_sched = 1;
ucs_trace_data("progressing pending request %p", req);
rc = req->func(req);
ep->arb_sched = 0;
ucs_trace_data("status returned from progress pending: %s",
ucs_status_string(rc));
if (UCS_OK == rc) {
/* sent successfully. remove from the arbiter */
return UCS_ARBITER_CB_RESULT_REMOVE_ELEM;
} else if (UCS_INPROGRESS == rc) {
return UCS_ARBITER_CB_RESULT_NEXT_GROUP;
} else {
/* couldn't send. keep this request in the arbiter until the next time
* this function is called */
return UCS_ARBITER_CB_RESULT_RESCHED_GROUP;
}
}
/**
* dispatch requests waiting for tx resources
*/
ucs_arbiter_cb_result_t
uct_dc_mlx5_iface_dci_do_pending_tx(ucs_arbiter_t *arbiter,
ucs_arbiter_elem_t *elem,
void *arg)
{
uct_dc_mlx5_ep_t *ep = ucs_container_of(ucs_arbiter_elem_group(elem), uct_dc_mlx5_ep_t, arb_group);
uct_dc_mlx5_iface_t *iface = ucs_derived_of(ep->super.super.iface, uct_dc_mlx5_iface_t);
uct_pending_req_t *req = ucs_container_of(elem, uct_pending_req_t, priv);
ucs_status_t status;
if (!uct_rc_iface_has_tx_resources(&iface->super.super)) {
return UCS_ARBITER_CB_RESULT_STOP;
}
status = req->func(req);
ucs_trace_data("progress pending request %p returned: %s", req,
ucs_status_string(status));
if (status == UCS_OK) {
/* For dcs* policies release dci if this is the last elem in the group
* and the dci has no outstanding operations. For example pending
* callback did not send anything. (uct_ep_flush or just return ok)
*/
if (ucs_arbiter_elem_is_last(&ep->arb_group, elem)) {
uct_dc_mlx5_iface_dci_free(iface, ep);
}
return UCS_ARBITER_CB_RESULT_REMOVE_ELEM;
}
if (status == UCS_INPROGRESS) {
return UCS_ARBITER_CB_RESULT_NEXT_GROUP;
}
if (!uct_dc_mlx5_iface_dci_ep_can_send(ep)) {
/* Deschedule the group even if FC is the only resource, which
* is missing. It will be scheduled again when credits arrive.
* We can't desched group with rand policy if non FC resources are
* missing, since it's never scheduled again. */
if (uct_dc_mlx5_iface_is_dci_rand(iface) &&
uct_rc_fc_has_resources(&iface->super.super, &ep->fc)) {
return UCS_ARBITER_CB_RESULT_RESCHED_GROUP;
} else {
return UCS_ARBITER_CB_RESULT_DESCHED_GROUP;
}
}
ucs_assertv(!uct_rc_iface_has_tx_resources(&iface->super.super),
"pending callback returned error but send resources are available");
return UCS_ARBITER_CB_RESULT_STOP;
}
static ucs_arbiter_cb_result_t uct_mm_ep_abriter_purge_cb(ucs_arbiter_t *arbiter,
ucs_arbiter_elem_t *elem,
void *arg)
{
uct_pending_req_t *req = ucs_container_of(elem, uct_pending_req_t, priv);
uct_purge_cb_args_t *cb_args = arg;
uct_pending_purge_callback_t cb = cb_args->cb;
uct_mm_ep_t *ep = ucs_container_of(ucs_arbiter_elem_group(elem),
uct_mm_ep_t, arb_group);
if (cb != NULL) {
cb(req, cb_args->arg);
} else {
ucs_warn("ep=%p canceling user pending request %p", ep, req);
}
return UCS_ARBITER_CB_RESULT_REMOVE_ELEM;
}
/**
* dispatch requests waiting for dci allocation
*/
ucs_arbiter_cb_result_t
uct_dc_iface_dci_do_pending_wait(ucs_arbiter_t *arbiter,
ucs_arbiter_elem_t *elem,
void *arg)
{
uct_dc_ep_t *ep = ucs_container_of(ucs_arbiter_elem_group(elem), uct_dc_ep_t, arb_group);
uct_dc_iface_t *iface = ucs_derived_of(ep->super.super.iface, uct_dc_iface_t);
/**
* stop if dci can not be allocated
* else move group to the dci arbiter
*/
ucs_assert_always(ep->dci == UCT_DC_EP_NO_DCI);
if (!uct_dc_iface_dci_can_alloc(iface)) {
return UCS_ARBITER_CB_RESULT_STOP;
}
uct_dc_iface_dci_alloc(iface, ep);
ucs_assert_always(ep->dci != UCT_DC_EP_NO_DCI);
uct_dc_iface_dci_sched_tx(iface, ep);
return UCS_ARBITER_CB_RESULT_DESCHED_GROUP;
}
/**
* dispatch requests waiting for dci allocation
* Relevant for dcs and dcs_quota policies only.
*/
ucs_arbiter_cb_result_t
uct_dc_mlx5_iface_dci_do_pending_wait(ucs_arbiter_t *arbiter,
ucs_arbiter_elem_t *elem,
void *arg)
{
uct_dc_mlx5_ep_t *ep = ucs_container_of(ucs_arbiter_elem_group(elem), uct_dc_mlx5_ep_t, arb_group);
uct_dc_mlx5_iface_t *iface = ucs_derived_of(ep->super.super.iface, uct_dc_mlx5_iface_t);
ucs_assert(!uct_dc_mlx5_iface_is_dci_rand(iface));
if (ep->dci != UCT_DC_MLX5_EP_NO_DCI) {
return UCS_ARBITER_CB_RESULT_DESCHED_GROUP;
}
if (!uct_dc_mlx5_iface_dci_can_alloc(iface)) {
return UCS_ARBITER_CB_RESULT_STOP;
}
uct_dc_mlx5_iface_dci_alloc(iface, ep);
ucs_assert_always(ep->dci != UCT_DC_MLX5_EP_NO_DCI);
uct_dc_mlx5_iface_dci_sched_tx(iface, ep);
return UCS_ARBITER_CB_RESULT_DESCHED_GROUP;
}