static ssize_t uct_ud_verbs_ep_am_bcopy(uct_ep_h tl_ep, uint8_t id, uct_pack_callback_t pack_cb, void *arg) { uct_ud_verbs_ep_t *ep = ucs_derived_of(tl_ep, uct_ud_verbs_ep_t); uct_ud_verbs_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_ud_verbs_iface_t); uct_ud_send_skb_t *skb; ucs_status_t status; size_t length; uct_ud_enter(&iface->super); uct_ud_iface_progress_pending_tx(&iface->super); status = uct_ud_am_common(&iface->super, &ep->super, id, &skb); if (status != UCS_OK) { uct_ud_leave(&iface->super); return status; } length = uct_ud_skb_bcopy(skb, pack_cb, arg); uct_ud_verbs_ep_tx_skb(iface, ep, skb, 0); ucs_trace_data("TX(iface=%p): AM_BCOPY [%d] skb=%p buf=%p len=%u", iface, id, skb, arg, skb->len); uct_ud_iface_complete_tx_skb(&iface->super, &ep->super, skb); uct_ud_leave(&iface->super); return length; }
static ssize_t uct_ud_mlx5_ep_am_bcopy(uct_ep_h tl_ep, uint8_t id, uct_pack_callback_t pack_cb, void *arg) { uct_ud_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_ud_mlx5_ep_t); uct_ud_mlx5_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_ud_mlx5_iface_t); uct_ud_send_skb_t *skb; ucs_status_t status; size_t length; uct_ud_enter(&iface->super); uct_ud_iface_progress_pending_tx(&iface->super); status = uct_ud_am_common(&iface->super, &ep->super, id, &skb); if (status != UCS_OK) { uct_ud_leave(&iface->super); return status; } length = uct_ud_skb_bcopy(skb, pack_cb, arg); uct_ud_mlx5_ep_tx_skb(iface, ep, skb); uct_ud_iface_complete_tx_skb(&iface->super, &ep->super, skb); UCT_TL_EP_STAT_OP(&ep->super.super, AM, BCOPY, length); uct_ud_leave(&iface->super); return length; }
static ssize_t uct_ud_mlx5_ep_am_bcopy(uct_ep_h tl_ep, uint8_t id, uct_pack_callback_t pack_cb, void *arg) { uct_ud_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_ud_mlx5_ep_t); uct_ud_mlx5_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_ud_mlx5_iface_t); uct_ud_send_skb_t *skb; ucs_status_t status; size_t length; status = uct_ud_am_common(&iface->super, &ep->super, id, &skb); if (status != UCS_OK) { return status; } length = uct_ud_skb_bcopy(skb, pack_cb, arg); uct_ud_mlx5_ep_tx_skb(iface, ep, skb); uct_ud_iface_complete_tx_skb(&iface->super, &ep->super, skb); return length; }
static ucs_status_t uct_ud_verbs_ep_am_short(uct_ep_h tl_ep, uint8_t id, uint64_t hdr, const void *buffer, unsigned length) { uct_ud_verbs_ep_t *ep = ucs_derived_of(tl_ep, uct_ud_verbs_ep_t); uct_ud_verbs_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_ud_verbs_iface_t); uct_ud_send_skb_t *skb; uct_ud_am_short_hdr_t *am_hdr; ucs_status_t status; UCT_CHECK_LENGTH(sizeof(uct_ud_neth_t) + sizeof(hdr) + length, iface->super.config.max_inline, "am_short"); uct_ud_enter(&iface->super); uct_ud_iface_progress_pending_tx(&iface->super); status = uct_ud_am_common(&iface->super, &ep->super, id, &skb); if (status != UCS_OK) { uct_ud_leave(&iface->super); return status; } am_hdr = (uct_ud_am_short_hdr_t *)(skb->neth+1); am_hdr->hdr = hdr; iface->tx.sge[0].length = sizeof(uct_ud_neth_t) + sizeof(*am_hdr); iface->tx.sge[0].addr = (uintptr_t)skb->neth; uct_ud_verbs_ep_tx_inlv(iface, ep, buffer, length); ucs_trace_data("TX: AM [%d] buf=%p len=%u", id, buffer, length); skb->len = iface->tx.sge[0].length; uct_ud_iface_complete_tx_inl(&iface->super, &ep->super, skb, am_hdr+1, buffer, length); uct_ud_leave(&iface->super); return UCS_OK; }