int ucs_config_sscanf_ternary(const char *buf, void *dest, const void *arg)
{
UCS_STATIC_ASSERT(UCS_NO == 0);
UCS_STATIC_ASSERT(UCS_YES == 1);
if (!strcasecmp(buf, "try") || !strcasecmp(buf, "maybe")) {
*(int*)dest = UCS_TRY;
return 1;
} else {
return ucs_config_sscanf_bool(buf, dest, arg);
}
}
unsigned uct_rc_mlx5_iface_srq_post_recv(uct_rc_iface_t *iface, uct_ib_mlx5_srq_t *srq)
{
uct_ib_mlx5_srq_seg_t *seg;
uct_ib_iface_recv_desc_t *desc;
uint16_t count, index, next_index;
uct_rc_hdr_t *hdr;
/* Make sure the union is right */
UCS_STATIC_ASSERT(ucs_offsetof(uct_ib_mlx5_srq_seg_t, mlx5_srq.next_wqe_index) ==
ucs_offsetof(uct_ib_mlx5_srq_seg_t, srq.next_wqe_index));
UCS_STATIC_ASSERT(ucs_offsetof(uct_ib_mlx5_srq_seg_t, dptr) ==
sizeof(struct mlx5_wqe_srq_next_seg));
ucs_assert(UCS_CIRCULAR_COMPARE16(srq->ready_idx, <=, srq->free_idx));
index = srq->ready_idx;
for (;;) {
next_index = index + 1;
seg = uct_ib_mlx5_srq_get_wqe(srq, next_index & srq->mask);
if (UCS_CIRCULAR_COMPARE16(next_index, >, srq->free_idx)) {
if (!seg->srq.ooo) {
break;
}
ucs_assert(next_index == (uint16_t)(srq->free_idx + 1));
seg->srq.ooo = 0;
srq->free_idx = next_index;
}
if (seg->srq.desc == NULL) {
UCT_TL_IFACE_GET_RX_DESC(&iface->super.super, &iface->rx.mp,
desc, break);
/* Set receive data segment pointer. Length is pre-initialized. */
hdr = uct_ib_iface_recv_desc_hdr(&iface->super, desc);
seg->srq.desc = desc;
seg->dptr.lkey = htonl(desc->lkey);
seg->dptr.addr = htonll((uintptr_t)hdr);
VALGRIND_MAKE_MEM_NOACCESS(hdr, iface->super.config.seg_size);
}
index = next_index;
}
ucs_status_t uct_ugni_ep_pending_add(uct_ep_h tl_ep, uct_pending_req_t *n){
uct_ugni_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_ugni_iface_t);
uct_ugni_ep_t *ep = ucs_derived_of(tl_ep, uct_ugni_ep_t);
UCS_STATIC_ASSERT(sizeof(ucs_arbiter_elem_t) <= UCT_PENDING_REQ_PRIV_LEN);
ucs_arbiter_elem_init((ucs_arbiter_elem_t *)n->priv);
ucs_arbiter_group_push_elem(&ep->arb_group, (ucs_arbiter_elem_t*) n->priv);
ucs_arbiter_group_schedule(&iface->arbiter, &ep->arb_group);
return UCS_OK;
}
ucs_status_t uct_ugni_ep_pending_add(uct_ep_h tl_ep, uct_pending_req_t *n,
unsigned flags){
uct_ugni_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_ugni_iface_t);
uct_ugni_ep_t *ep = ucs_derived_of(tl_ep, uct_ugni_ep_t);
UCS_STATIC_ASSERT(sizeof(ucs_arbiter_elem_t) <= UCT_PENDING_REQ_PRIV_LEN);
uct_ugni_enter_async(iface);
uct_pending_req_arb_group_push(&ep->arb_group, n);
ucs_arbiter_group_schedule(&iface->arbiter, &ep->arb_group);
UCT_TL_EP_STAT_PEND(&ep->super);
uct_ugni_leave_async(iface);
return UCS_OK;
}
/* TODO:
currently pending code supports only dcs policy
support hash/random policies
*/
ucs_status_t uct_dc_mlx5_ep_pending_add(uct_ep_h tl_ep, uct_pending_req_t *r,
unsigned flags)
{
uct_dc_mlx5_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_dc_mlx5_iface_t);
uct_dc_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_dc_mlx5_ep_t);
/* ep can tx iff
* - iface has resources: cqe and tx skb
* - dci is either assigned or can be assigned
* - dci has resources
*/
if (uct_rc_iface_has_tx_resources(&iface->super.super)) {
if (ep->dci == UCT_DC_MLX5_EP_NO_DCI) {
if (uct_dc_mlx5_iface_dci_can_alloc(iface) && (ep->fc.fc_wnd > 0)) {
return UCS_ERR_BUSY;
}
} else {
if (uct_dc_mlx5_iface_dci_ep_can_send(ep)) {
return UCS_ERR_BUSY;
}
}
}
UCS_STATIC_ASSERT(sizeof(uct_pending_req_priv_arb_t) <=
UCT_PENDING_REQ_PRIV_LEN);
uct_pending_req_arb_group_push(&ep->arb_group, r);
/* no dci:
* Do not grab dci here. Instead put the group on dci allocation arbiter.
* This way we can assure fairness between all eps waiting for
* dci allocation. Relevant for dcs and dcs_quota policies.
*/
if (ep->dci == UCT_DC_MLX5_EP_NO_DCI) {
uct_dc_mlx5_iface_schedule_dci_alloc(iface, ep);
UCT_TL_EP_STAT_PEND(&ep->super);
return UCS_OK;
}
uct_dc_mlx5_iface_dci_sched_tx(iface, ep);
UCT_TL_EP_STAT_PEND(&ep->super);
return UCS_OK;
}
ucs_status_t ucp_tag_send(ucp_ep_h ep, const void *buffer, size_t length,
ucp_tag_t tag)
{
ucp_worker_h worker = ep->worker;
ucs_status_t status;
retry:
if (ucs_likely(length < ep->config.max_short_tag)) {
UCS_STATIC_ASSERT(sizeof(ucp_tag_t) == sizeof(uint64_t));
status = uct_ep_am_short(ep->uct_ep, UCP_AM_ID_EAGER_ONLY, tag,
buffer, length);
if (status == UCS_ERR_NO_RESOURCE) {
ucp_worker_progress(worker);
goto retry;
}
return status;
}
ucs_fatal("unsupported");
}
ucs_status_t uct_mm_ep_pending_add(uct_ep_h tl_ep, uct_pending_req_t *n)
{
uct_mm_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_mm_iface_t);
uct_mm_ep_t *ep = ucs_derived_of(tl_ep, uct_mm_ep_t);
/* check if resources became available */
if (uct_mm_ep_has_tx_resources(ep)) {
return UCS_ERR_BUSY;
}
UCS_STATIC_ASSERT(sizeof(ucs_arbiter_elem_t) <= UCT_PENDING_REQ_PRIV_LEN);
ucs_arbiter_elem_init((ucs_arbiter_elem_t *)n->priv);
/* add the request to the ep's arbiter_group (pending queue) */
ucs_arbiter_group_push_elem(&ep->arb_group, (ucs_arbiter_elem_t*) n->priv);
/* add the ep's group to the arbiter */
ucs_arbiter_group_schedule(&iface->arbiter, &ep->arb_group);
return UCS_OK;
}
