ucs_status_t uct_dc_mlx5_ep_am_zcopy(uct_ep_h tl_ep, uint8_t id, const void *header, unsigned header_length, const uct_iov_t *iov, size_t iovcnt, unsigned flags, uct_completion_t *comp) { uct_dc_mlx5_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_dc_mlx5_iface_t); uct_dc_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_dc_mlx5_ep_t); UCT_CHECK_IOV_SIZE(iovcnt, UCT_IB_MLX5_AM_ZCOPY_MAX_IOV, "uct_dc_mlx5_ep_am_zcopy"); UCT_RC_MLX5_CHECK_AM_ZCOPY(id, header_length, uct_iov_total_length(iov, iovcnt), iface->super.super.super.config.seg_size, UCT_IB_MLX5_AV_FULL_SIZE); UCT_DC_CHECK_RES_AND_FC(iface, ep); uct_dc_mlx5_iface_zcopy_post(iface, ep, MLX5_OPCODE_SEND, iov, iovcnt, id, header, header_length, 0, 0, 0ul, 0, 0, comp, MLX5_WQE_CTRL_SOLICITED); UCT_RC_UPDATE_FC_WND(&iface->super.super, &ep->fc); UCT_TL_EP_STAT_OP(&ep->super, AM, ZCOPY, header_length + uct_iov_total_length(iov, iovcnt)); return UCS_INPROGRESS; }
static ucs_status_t UCS_F_ALWAYS_INLINE uct_dc_mlx5_ep_am_short_inline(uct_ep_h tl_ep, uint8_t id, uint64_t hdr, const void *buffer, unsigned length) { uct_dc_mlx5_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_dc_mlx5_iface_t); uct_dc_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_dc_mlx5_ep_t); UCT_DC_MLX5_TXQP_DECL(txqp, txwq); UCT_RC_MLX5_CHECK_AM_SHORT(id, length, UCT_IB_MLX5_AV_FULL_SIZE); UCT_DC_CHECK_RES_AND_FC(iface, ep); UCT_DC_MLX5_IFACE_TXQP_GET(iface, ep, txqp, txwq); uct_rc_mlx5_txqp_inline_post(&iface->super, UCT_IB_QPT_DCI, txqp, txwq, MLX5_OPCODE_SEND, buffer, length, id, hdr, 0, 0, 0, &ep->av, uct_dc_mlx5_ep_get_grh(ep), uct_ib_mlx5_wqe_av_size(&ep->av), MLX5_WQE_CTRL_SOLICITED, INT_MAX); UCT_RC_UPDATE_FC_WND(&iface->super.super, &ep->fc); UCT_TL_EP_STAT_OP(&ep->super, AM, SHORT, sizeof(hdr) + length); return UCS_OK; }
ssize_t uct_dc_mlx5_ep_am_bcopy(uct_ep_h tl_ep, uint8_t id, uct_pack_callback_t pack_cb, void *arg) { uct_dc_mlx5_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_dc_mlx5_iface_t); uct_dc_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_dc_mlx5_ep_t); uct_rc_iface_send_desc_t *desc; size_t length; UCT_DC_CHECK_RES_AND_FC(&iface->super, &ep->super); UCT_RC_IFACE_GET_TX_AM_BCOPY_DESC(&iface->super.super, &iface->super.super.tx.mp, desc, id, pack_cb, arg, &length); uct_dc_mlx5_iface_bcopy_post(iface, ep, MLX5_OPCODE_SEND|UCT_RC_MLX5_OPCODE_FLAG_RAW, sizeof(uct_rc_hdr_t) + length, 0, NULL, 0, 0, 0, desc); UCT_RC_UPDATE_FC_WND(&iface->super.super, &ep->super.fc); UCT_TL_EP_STAT_OP(&ep->super.super, AM, BCOPY, length); return length; }
ssize_t uct_dc_mlx5_ep_am_bcopy(uct_ep_h tl_ep, uint8_t id, uct_pack_callback_t pack_cb, void *arg, unsigned flags) { uct_dc_mlx5_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_dc_mlx5_iface_t); uct_dc_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_dc_mlx5_ep_t); uct_rc_iface_send_desc_t *desc; size_t length; UCT_CHECK_AM_ID(id); UCT_DC_CHECK_RES_AND_FC(iface, ep); UCT_RC_IFACE_GET_TX_AM_BCOPY_DESC(&iface->super.super, &iface->super.super.tx.mp, desc, id, uct_rc_mlx5_am_hdr_fill, uct_rc_mlx5_hdr_t, pack_cb, arg, &length); uct_dc_mlx5_iface_bcopy_post(iface, ep, MLX5_OPCODE_SEND, sizeof(uct_rc_mlx5_hdr_t) + length, 0, 0, desc, MLX5_WQE_CTRL_SOLICITED, 0, desc + 1, NULL); UCT_RC_UPDATE_FC_WND(&iface->super.super, &ep->fc); UCT_TL_EP_STAT_OP(&ep->super, AM, BCOPY, length); return length; }
ucs_status_t uct_dc_mlx5_ep_am_short(uct_ep_h tl_ep, uint8_t id, uint64_t hdr, const void *buffer, unsigned length) { uct_dc_mlx5_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_dc_mlx5_iface_t); uct_dc_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_dc_mlx5_ep_t); UCT_DC_MLX5_TXQP_DECL(txqp, txwq); UCT_RC_MLX5_CHECK_AM_SHORT(id, length, UCT_IB_MLX5_AV_FULL_SIZE); UCT_DC_CHECK_RES_AND_FC(&iface->super, &ep->super); UCT_DC_MLX5_IFACE_TXQP_GET(iface, &ep->super, txqp, txwq); uct_rc_mlx5_txqp_inline_post(&iface->super.super, IBV_EXP_QPT_DC_INI, txqp, txwq, MLX5_OPCODE_SEND, buffer, length, id, hdr, 0, 0, 0, &ep->av, uct_ib_mlx5_wqe_av_size(&ep->av)); UCT_RC_UPDATE_FC_WND(&iface->super.super, &ep->super.fc); UCT_TL_EP_STAT_OP(&ep->super.super, AM, SHORT, sizeof(hdr) + length); return UCS_OK; }
ucs_status_t uct_dc_mlx5_ep_am_short(uct_ep_h tl_ep, uint8_t id, uint64_t hdr, const void *buffer, unsigned length) { #if HAVE_IBV_EXP_DM uct_dc_mlx5_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_dc_mlx5_iface_t); uct_dc_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_dc_mlx5_ep_t); ucs_status_t status; uct_rc_mlx5_dm_copy_data_t cache; if (ucs_likely((sizeof(uct_rc_mlx5_am_short_hdr_t) + length <= UCT_IB_MLX5_AM_MAX_SHORT(UCT_IB_MLX5_AV_FULL_SIZE)) || !iface->super.dm.dm)) { #endif return uct_dc_mlx5_ep_am_short_inline(tl_ep, id, hdr, buffer, length); #if HAVE_IBV_EXP_DM } UCT_CHECK_AM_ID(id); UCT_CHECK_LENGTH(length + sizeof(uct_rc_mlx5_am_short_hdr_t), 0, iface->super.dm.seg_len, "am_short"); UCT_DC_CHECK_RES_AND_FC(iface, ep); uct_rc_mlx5_am_hdr_fill(&cache.am_hdr.rc_hdr, id); cache.am_hdr.am_hdr = hdr; status = uct_dc_mlx5_ep_short_dm(ep, &cache, sizeof(cache.am_hdr), buffer, length, MLX5_OPCODE_SEND, MLX5_WQE_CTRL_SOLICITED | MLX5_WQE_CTRL_CQ_UPDATE, 0, 0); if (UCS_STATUS_IS_ERR(status)) { return status; } UCT_TL_EP_STAT_OP(&ep->super, AM, SHORT, sizeof(cache.am_hdr) + length); UCT_RC_UPDATE_FC_WND(&iface->super.super, &ep->fc); return UCS_OK; #endif }