ucs_status_t uct_rc_verbs_ep_flush(uct_ep_h tl_ep, unsigned flags, uct_completion_t *comp) { uct_rc_verbs_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_verbs_iface_t); uct_rc_verbs_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_verbs_ep_t); ucs_status_t status; if (!uct_rc_iface_has_tx_resources(&iface->super)) { return UCS_ERR_NO_RESOURCE; } if (uct_rc_txqp_available(&ep->super.txqp) == iface->config.tx_max_wr) { UCT_TL_EP_STAT_FLUSH(&ep->super.super); return UCS_OK; } if (uct_rc_txqp_unsignaled(&ep->super.txqp) != 0) { if (IBV_DEVICE_HAS_NOP(&uct_ib_iface_device(&iface->super.super)->dev_attr)) { status = uct_rc_verbs_ep_nop(ep); } else { status = uct_rc_verbs_ep_put_short(tl_ep, NULL, 0, 0, 0); } if (status != UCS_OK) { return status; } } else if (!uct_rc_ep_has_tx_resources(&ep->super)) { return UCS_ERR_NO_RESOURCE; } uct_rc_txqp_add_send_comp(&iface->super, &ep->super.txqp, comp, ep->txcnt.pi); UCT_TL_EP_STAT_FLUSH_WAIT(&ep->super.super); return UCS_INPROGRESS; }
static UCS_F_ALWAYS_INLINE void uct_dc_mlx5_iface_zcopy_post(uct_dc_mlx5_iface_t *iface, uct_dc_mlx5_ep_t *ep, unsigned opcode, const uct_iov_t *iov, size_t iovcnt, /* SEND */ uint8_t am_id, const void *am_hdr, unsigned am_hdr_len, /* RDMA */ uint64_t rdma_raddr, uct_rkey_t rdma_rkey, /* TAG */ uct_tag_t tag, uint32_t app_ctx, uint32_t ib_imm_be, uct_completion_t *comp, uint8_t send_flags) { uint16_t sn; UCT_DC_MLX5_TXQP_DECL(txqp, txwq); UCT_DC_MLX5_IFACE_TXQP_GET(iface, ep, txqp, txwq); sn = txwq->sw_pi; uct_rc_mlx5_txqp_dptr_post_iov(&iface->super, UCT_IB_QPT_DCI, txqp, txwq, opcode, iov, iovcnt, am_id, am_hdr, am_hdr_len, rdma_raddr, uct_ib_md_direct_rkey(rdma_rkey), tag, app_ctx, ib_imm_be, &ep->av, uct_dc_mlx5_ep_get_grh(ep), uct_ib_mlx5_wqe_av_size(&ep->av), MLX5_WQE_CTRL_CQ_UPDATE | send_flags, UCT_IB_MAX_ZCOPY_LOG_SGE(&iface->super.super.super)); uct_rc_txqp_add_send_comp(&iface->super.super, txqp, comp, sn, UCT_RC_IFACE_SEND_OP_FLAG_ZCOPY); }
/* * Helper function for zero-copy post. * Adds user completion to the callback queue. */ static UCS_F_ALWAYS_INLINE ucs_status_t uct_rc_mlx5_ep_zcopy_post(uct_rc_mlx5_ep_t *ep, unsigned opcode, const uct_iov_t *iov, size_t iovcnt, /* SEND */ uint8_t am_id, const void *am_hdr, unsigned am_hdr_len, /* RDMA */ uint64_t rdma_raddr, uct_rkey_t rdma_rkey, int force_sig, uct_completion_t *comp) { uct_rc_iface_t *iface = ucs_derived_of(ep->super.super.super.iface, uct_rc_iface_t); uint16_t sn; UCT_RC_CHECK_RES(iface, &ep->super); sn = ep->tx.wq.sw_pi; uct_rc_mlx5_txqp_dptr_post_iov(iface, IBV_QPT_RC, &ep->super.txqp, &ep->tx.wq, opcode, iov, iovcnt, am_id, am_hdr, am_hdr_len, rdma_raddr, uct_ib_md_direct_rkey(rdma_rkey), NULL, 0, (comp == NULL) ? force_sig : MLX5_WQE_CTRL_CQ_UPDATE); uct_rc_txqp_add_send_comp(iface, &ep->super.txqp, comp, sn); return UCS_INPROGRESS; }
ucs_status_t uct_rc_mlx5_ep_flush(uct_ep_h tl_ep, unsigned flags, uct_completion_t *comp) { uct_rc_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_mlx5_ep_t); uct_rc_mlx5_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_mlx5_iface_t); uint16_t sn; if (!uct_rc_iface_has_tx_resources(&iface->super)) { return UCS_ERR_NO_RESOURCE; } if (uct_rc_txqp_available(&ep->super.txqp) == ep->tx.wq.bb_max) { UCT_TL_EP_STAT_FLUSH(&ep->super.super); return UCS_OK; } if (uct_rc_txqp_unsignaled(&ep->super.txqp) != 0) { sn = ep->tx.wq.sw_pi; UCT_RC_CHECK_RES(&iface->super, &ep->super); uct_rc_mlx5_txqp_inline_post(&iface->super, IBV_QPT_RC, &ep->super.txqp, &ep->tx.wq, MLX5_OPCODE_NOP, NULL, 0, 0, 0, 0, 0, NULL, 0); } else if (!uct_rc_ep_has_tx_resources(&ep->super)) { return UCS_ERR_NO_RESOURCE; } else { sn = ep->tx.wq.sig_pi; } uct_rc_txqp_add_send_comp(&iface->super, &ep->super.txqp, comp, sn); UCT_TL_EP_STAT_FLUSH_WAIT(&ep->super.super); return UCS_INPROGRESS; }
static inline void uct_dc_mlx5_iface_add_send_comp(uct_dc_mlx5_iface_t *iface, uct_dc_mlx5_ep_t *ep, uct_completion_t *comp) { UCT_DC_MLX5_TXQP_DECL(txqp, txwq); UCT_DC_MLX5_IFACE_TXQP_GET(iface, ep, txqp, txwq); uct_rc_txqp_add_send_comp(&iface->super.super, txqp, comp, txwq->sig_pi); }
static inline ucs_status_t uct_rc_verbs_ep_rdma_zcopy(uct_rc_verbs_ep_t *ep, const uct_iov_t *iov, size_t iovcnt, uint64_t remote_addr, uct_rkey_t rkey, uct_completion_t *comp, int opcode) { uct_rc_verbs_iface_t *iface = ucs_derived_of(ep->super.super.super.iface, uct_rc_verbs_iface_t); struct ibv_sge sge[UCT_IB_MAX_IOV]; struct ibv_send_wr wr; size_t sge_cnt; UCT_RC_CHECK_RES(&iface->super, &ep->super); sge_cnt = uct_ib_verbs_sge_fill_iov(sge, iov, iovcnt); UCT_SKIP_ZERO_LENGTH(sge_cnt); UCT_RC_VERBS_FILL_RDMA_WR_IOV(wr, wr.opcode, opcode, sge, sge_cnt, remote_addr, rkey); wr.next = NULL; uct_rc_verbs_ep_post_send(iface, ep, &wr, IBV_SEND_SIGNALED); uct_rc_txqp_add_send_comp(&iface->super, &ep->super.txqp, comp, ep->txcnt.pi); return UCS_INPROGRESS; }
static UCS_F_ALWAYS_INLINE void uct_dc_mlx5_iface_zcopy_post(uct_dc_mlx5_iface_t *iface, uct_dc_mlx5_ep_t *ep, unsigned opcode, const uct_iov_t *iov, size_t iovcnt, /* SEND */ uint8_t am_id, const void *am_hdr, unsigned am_hdr_len, /* RDMA */ uint64_t rdma_raddr, uct_rkey_t rdma_rkey, uct_completion_t *comp) { uint16_t sn; UCT_DC_MLX5_TXQP_DECL(txqp, txwq); UCT_DC_MLX5_IFACE_TXQP_GET(iface, ep, txqp, txwq); sn = txwq->sw_pi; uct_rc_mlx5_txqp_dptr_post_iov(&iface->super.super, IBV_EXP_QPT_DC_INI, txqp, txwq, opcode, iov, iovcnt, am_id, am_hdr, am_hdr_len, rdma_raddr, uct_ib_md_direct_rkey(rdma_rkey), &ep->av, uct_ib_mlx5_wqe_av_size(&ep->av), MLX5_WQE_CTRL_CQ_UPDATE); uct_rc_txqp_add_send_comp(&iface->super.super, txqp, comp, sn); }