static UCS_F_ALWAYS_INLINE void
uct_dc_mlx5_iface_zcopy_post(uct_dc_mlx5_iface_t *iface, uct_dc_mlx5_ep_t *ep,
unsigned opcode, const uct_iov_t *iov, size_t iovcnt,
/* SEND */ uint8_t am_id, const void *am_hdr, unsigned am_hdr_len,
/* RDMA */ uint64_t rdma_raddr, uct_rkey_t rdma_rkey,
/* TAG */ uct_tag_t tag, uint32_t app_ctx, uint32_t ib_imm_be,
uct_completion_t *comp, uint8_t send_flags)
{
uint16_t sn;
UCT_DC_MLX5_TXQP_DECL(txqp, txwq);
UCT_DC_MLX5_IFACE_TXQP_GET(iface, ep, txqp, txwq);
sn = txwq->sw_pi;
uct_rc_mlx5_txqp_dptr_post_iov(&iface->super, UCT_IB_QPT_DCI, txqp,
txwq, opcode, iov, iovcnt,
am_id, am_hdr, am_hdr_len,
rdma_raddr, uct_ib_md_direct_rkey(rdma_rkey),
tag, app_ctx, ib_imm_be,
&ep->av, uct_dc_mlx5_ep_get_grh(ep),
uct_ib_mlx5_wqe_av_size(&ep->av),
MLX5_WQE_CTRL_CQ_UPDATE | send_flags,
UCT_IB_MAX_ZCOPY_LOG_SGE(&iface->super.super.super));
uct_rc_txqp_add_send_comp(&iface->super.super, txqp, comp, sn,
UCT_RC_IFACE_SEND_OP_FLAG_ZCOPY);
}
/*
* Helper function for zero-copy post.
* Adds user completion to the callback queue.
*/
static UCS_F_ALWAYS_INLINE ucs_status_t
uct_rc_mlx5_ep_zcopy_post(uct_rc_mlx5_ep_t *ep,
unsigned opcode, const uct_iov_t *iov, size_t iovcnt,
/* SEND */ uint8_t am_id, const void *am_hdr, unsigned am_hdr_len,
/* RDMA */ uint64_t rdma_raddr, uct_rkey_t rdma_rkey,
int force_sig, uct_completion_t *comp)
{
uct_rc_iface_t *iface = ucs_derived_of(ep->super.super.super.iface,
uct_rc_iface_t);
uint16_t sn;
UCT_RC_CHECK_RES(iface, &ep->super);
sn = ep->tx.wq.sw_pi;
uct_rc_mlx5_txqp_dptr_post_iov(iface, IBV_QPT_RC,
&ep->super.txqp, &ep->tx.wq,
opcode, iov, iovcnt,
am_id, am_hdr, am_hdr_len,
rdma_raddr, uct_ib_md_direct_rkey(rdma_rkey),
NULL, 0,
(comp == NULL) ? force_sig : MLX5_WQE_CTRL_CQ_UPDATE);
uct_rc_txqp_add_send_comp(iface, &ep->super.txqp, comp, sn);
return UCS_INPROGRESS;
}
static ucs_status_t UCS_F_ALWAYS_INLINE
uct_dc_mlx5_ep_put_short_inline(uct_ep_h tl_ep, const void *buffer,
unsigned length, uint64_t remote_addr,
uct_rkey_t rkey)
{
uct_dc_mlx5_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_dc_mlx5_iface_t);
uct_dc_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_dc_mlx5_ep_t);
UCT_DC_MLX5_TXQP_DECL(txqp, txwq);
UCT_RC_MLX5_CHECK_PUT_SHORT(length, UCT_IB_MLX5_AV_FULL_SIZE);
UCT_DC_MLX5_CHECK_RES(iface, ep);
UCT_DC_MLX5_IFACE_TXQP_GET(iface, ep, txqp, txwq);
uct_rc_mlx5_txqp_inline_post(&iface->super, UCT_IB_QPT_DCI,
txqp, txwq,
MLX5_OPCODE_RDMA_WRITE,
buffer, length, 0, 0, 0,
remote_addr, uct_ib_md_direct_rkey(rkey),
&ep->av, uct_dc_mlx5_ep_get_grh(ep),
uct_ib_mlx5_wqe_av_size(&ep->av), 0, INT_MAX);
UCT_TL_EP_STAT_OP(&ep->super, PUT, SHORT, length);
return UCS_OK;
}
/*
*
* Helper function for buffer-copy post.
* Adds the descriptor to the callback queue.
*/
static UCS_F_ALWAYS_INLINE void
uct_rc_mlx5_txqp_bcopy_post(uct_rc_iface_t *iface, uct_rc_txqp_t *txqp, uct_ib_mlx5_txwq_t *txwq,
unsigned opcode, unsigned length,
/* SEND */ uint8_t am_id, void *am_hdr, unsigned am_hdr_len,
/* RDMA */ uint64_t rdma_raddr, uct_rkey_t rdma_rkey,
int force_sig, uct_rc_iface_send_desc_t *desc)
{
desc->super.sn = txwq->sw_pi;
uct_rc_mlx5_txqp_dptr_post(iface, IBV_QPT_RC, txqp, txwq,
opcode, desc + 1, length, &desc->lkey,
am_id, am_hdr, am_hdr_len,
rdma_raddr, uct_ib_md_direct_rkey(rdma_rkey),
0, 0, 0,
NULL, 0, force_sig);
uct_rc_txqp_add_send_op(txqp, &desc->super);
}
ucs_status_t uct_rc_mlx5_ep_put_short(uct_ep_h tl_ep, const void *buffer, unsigned length,
uint64_t remote_addr, uct_rkey_t rkey)
{
uct_rc_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_iface_t);
uct_rc_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_mlx5_ep_t);
UCT_RC_MLX5_CHECK_PUT_SHORT(length, 0);
UCT_RC_CHECK_RES(iface, &ep->super);
uct_rc_mlx5_txqp_inline_post(iface, IBV_QPT_RC,
&ep->super.txqp, &ep->tx.wq,
MLX5_OPCODE_RDMA_WRITE,
buffer, length, 0, 0,
remote_addr, uct_ib_md_direct_rkey(rkey),
NULL, 0);
UCT_TL_EP_STAT_OP(&ep->super.super, PUT, SHORT, length);
return UCS_OK;
}
static UCS_F_ALWAYS_INLINE void
uct_dc_mlx5_iface_bcopy_post(uct_dc_mlx5_iface_t *iface, uct_dc_mlx5_ep_t *ep,
unsigned opcode, unsigned length,
/* SEND */ uint8_t am_id, void *am_hdr, unsigned am_hdr_len,
/* RDMA */ uint64_t rdma_raddr, uct_rkey_t rdma_rkey,
uct_rc_iface_send_desc_t *desc)
{
UCT_DC_MLX5_TXQP_DECL(txqp, txwq);
UCT_DC_MLX5_IFACE_TXQP_GET(iface, ep, txqp, txwq);
desc->super.sn = txwq->sw_pi;
uct_rc_mlx5_txqp_dptr_post(&iface->super.super, IBV_EXP_QPT_DC_INI, txqp, txwq,
opcode, desc + 1, length, &desc->lkey,
am_id, am_hdr, am_hdr_len,
rdma_raddr, uct_ib_md_direct_rkey(rdma_rkey),
0, 0, 0,
&ep->av, uct_ib_mlx5_wqe_av_size(&ep->av),
MLX5_WQE_CTRL_CQ_UPDATE);
uct_rc_txqp_add_send_op(txqp, &desc->super);
}
static UCS_F_ALWAYS_INLINE void
uct_dc_mlx5_iface_bcopy_post(uct_dc_mlx5_iface_t *iface, uct_dc_mlx5_ep_t *ep,
unsigned opcode, unsigned length,
/* RDMA */ uint64_t rdma_raddr, uct_rkey_t rdma_rkey,
uct_rc_iface_send_desc_t *desc, uint8_t send_flags,
uint32_t imm_val_be, const void *buffer,
uct_ib_log_sge_t *log_sge)
{
UCT_DC_MLX5_TXQP_DECL(txqp, txwq);
UCT_DC_MLX5_IFACE_TXQP_GET(iface, ep, txqp, txwq);
desc->super.sn = txwq->sw_pi;
uct_rc_mlx5_txqp_dptr_post(&iface->super, UCT_IB_QPT_DCI, txqp, txwq,
opcode, buffer, length, &desc->lkey,
rdma_raddr, uct_ib_md_direct_rkey(rdma_rkey),
0, 0, 0, 0,
&ep->av, uct_dc_mlx5_ep_get_grh(ep),
uct_ib_mlx5_wqe_av_size(&ep->av),
MLX5_WQE_CTRL_CQ_UPDATE | send_flags, imm_val_be, INT_MAX,
log_sge);
uct_rc_txqp_add_send_op(txqp, &desc->super);
}
static UCS_F_ALWAYS_INLINE void
uct_dc_mlx5_iface_zcopy_post(uct_dc_mlx5_iface_t *iface, uct_dc_mlx5_ep_t *ep,
unsigned opcode, const uct_iov_t *iov, size_t iovcnt,
/* SEND */ uint8_t am_id, const void *am_hdr, unsigned am_hdr_len,
/* RDMA */ uint64_t rdma_raddr, uct_rkey_t rdma_rkey,
uct_completion_t *comp)
{
uint16_t sn;
UCT_DC_MLX5_TXQP_DECL(txqp, txwq);
UCT_DC_MLX5_IFACE_TXQP_GET(iface, ep, txqp, txwq);
sn = txwq->sw_pi;
uct_rc_mlx5_txqp_dptr_post_iov(&iface->super.super, IBV_EXP_QPT_DC_INI, txqp,
txwq, opcode, iov, iovcnt,
am_id, am_hdr, am_hdr_len,
rdma_raddr, uct_ib_md_direct_rkey(rdma_rkey),
&ep->av, uct_ib_mlx5_wqe_av_size(&ep->av),
MLX5_WQE_CTRL_CQ_UPDATE);
uct_rc_txqp_add_send_comp(&iface->super.super, txqp, comp, sn);
}
