static ssize_t uct_ud_verbs_ep_am_bcopy(uct_ep_h tl_ep, uint8_t id,
uct_pack_callback_t pack_cb, void *arg)
{
uct_ud_verbs_ep_t *ep = ucs_derived_of(tl_ep, uct_ud_verbs_ep_t);
uct_ud_verbs_iface_t *iface = ucs_derived_of(tl_ep->iface,
uct_ud_verbs_iface_t);
uct_ud_send_skb_t *skb;
ucs_status_t status;
size_t length;
uct_ud_enter(&iface->super);
uct_ud_iface_progress_pending_tx(&iface->super);
status = uct_ud_am_common(&iface->super, &ep->super, id, &skb);
if (status != UCS_OK) {
uct_ud_leave(&iface->super);
return status;
}
length = uct_ud_skb_bcopy(skb, pack_cb, arg);
uct_ud_verbs_ep_tx_skb(iface, ep, skb, 0);
ucs_trace_data("TX(iface=%p): AM_BCOPY [%d] skb=%p buf=%p len=%u", iface, id,
skb, arg, skb->len);
uct_ud_iface_complete_tx_skb(&iface->super, &ep->super, skb);
uct_ud_leave(&iface->super);
return length;
}
static ssize_t uct_ud_mlx5_ep_am_bcopy(uct_ep_h tl_ep, uint8_t id,
uct_pack_callback_t pack_cb, void *arg)
{
uct_ud_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_ud_mlx5_ep_t);
uct_ud_mlx5_iface_t *iface = ucs_derived_of(tl_ep->iface,
uct_ud_mlx5_iface_t);
uct_ud_send_skb_t *skb;
ucs_status_t status;
size_t length;
uct_ud_enter(&iface->super);
uct_ud_iface_progress_pending_tx(&iface->super);
status = uct_ud_am_common(&iface->super, &ep->super, id, &skb);
if (status != UCS_OK) {
uct_ud_leave(&iface->super);
return status;
}
length = uct_ud_skb_bcopy(skb, pack_cb, arg);
uct_ud_mlx5_ep_tx_skb(iface, ep, skb);
uct_ud_iface_complete_tx_skb(&iface->super, &ep->super, skb);
UCT_TL_EP_STAT_OP(&ep->super.super, AM, BCOPY, length);
uct_ud_leave(&iface->super);
return length;
}
static ssize_t uct_ud_mlx5_ep_am_bcopy(uct_ep_h tl_ep, uint8_t id,
uct_pack_callback_t pack_cb, void *arg)
{
uct_ud_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_ud_mlx5_ep_t);
uct_ud_mlx5_iface_t *iface = ucs_derived_of(tl_ep->iface,
uct_ud_mlx5_iface_t);
uct_ud_send_skb_t *skb;
ucs_status_t status;
size_t length;
status = uct_ud_am_common(&iface->super, &ep->super, id, &skb);
if (status != UCS_OK) {
return status;
}
length = uct_ud_skb_bcopy(skb, pack_cb, arg);
uct_ud_mlx5_ep_tx_skb(iface, ep, skb);
uct_ud_iface_complete_tx_skb(&iface->super, &ep->super, skb);
return length;
}
static
ucs_status_t uct_ud_verbs_ep_am_short(uct_ep_h tl_ep, uint8_t id, uint64_t hdr,
const void *buffer, unsigned length)
{
uct_ud_verbs_ep_t *ep = ucs_derived_of(tl_ep, uct_ud_verbs_ep_t);
uct_ud_verbs_iface_t *iface = ucs_derived_of(tl_ep->iface,
uct_ud_verbs_iface_t);
uct_ud_send_skb_t *skb;
uct_ud_am_short_hdr_t *am_hdr;
ucs_status_t status;
UCT_CHECK_LENGTH(sizeof(uct_ud_neth_t) + sizeof(hdr) + length,
iface->super.config.max_inline, "am_short");
uct_ud_enter(&iface->super);
uct_ud_iface_progress_pending_tx(&iface->super);
status = uct_ud_am_common(&iface->super, &ep->super, id, &skb);
if (status != UCS_OK) {
uct_ud_leave(&iface->super);
return status;
}
am_hdr = (uct_ud_am_short_hdr_t *)(skb->neth+1);
am_hdr->hdr = hdr;
iface->tx.sge[0].length = sizeof(uct_ud_neth_t) + sizeof(*am_hdr);
iface->tx.sge[0].addr = (uintptr_t)skb->neth;
uct_ud_verbs_ep_tx_inlv(iface, ep, buffer, length);
ucs_trace_data("TX: AM [%d] buf=%p len=%u", id, buffer, length);
skb->len = iface->tx.sge[0].length;
uct_ud_iface_complete_tx_inl(&iface->super, &ep->super, skb,
am_hdr+1, buffer, length);
uct_ud_leave(&iface->super);
return UCS_OK;
}
