static UCS_F_ALWAYS_INLINE void
uct_dc_mlx5_iface_atomic_post(uct_dc_mlx5_iface_t *iface, uct_dc_mlx5_ep_t *ep,
unsigned opcode, uct_rc_iface_send_desc_t *desc, unsigned length,
uint64_t remote_addr, uct_rkey_t rkey,
uint64_t compare_mask, uint64_t compare,
uint64_t swap_mask, uint64_t swap_add)
{
uint32_t ib_rkey = uct_ib_resolve_atomic_rkey(rkey, ep->atomic_mr_offset,
&remote_addr);
UCT_DC_MLX5_TXQP_DECL(txqp, txwq);
UCT_DC_MLX5_IFACE_TXQP_GET(iface, ep, txqp, txwq);
desc->super.sn = txwq->sw_pi;
uct_rc_mlx5_txqp_dptr_post(&iface->super, UCT_IB_QPT_DCI, txqp, txwq,
opcode, desc + 1, length, &desc->lkey,
remote_addr, ib_rkey,
compare_mask, compare, swap_mask, swap_add,
&ep->av, uct_dc_mlx5_ep_get_grh(ep),
uct_ib_mlx5_wqe_av_size(&ep->av),
MLX5_WQE_CTRL_CQ_UPDATE, 0, INT_MAX, NULL);
UCT_TL_EP_STAT_ATOMIC(&ep->super);
uct_rc_txqp_add_send_op(txqp, &desc->super);
}
/*
*
* Helper function for buffer-copy post.
* Adds the descriptor to the callback queue.
*/
static UCS_F_ALWAYS_INLINE void
uct_rc_mlx5_txqp_bcopy_post(uct_rc_iface_t *iface, uct_rc_txqp_t *txqp, uct_ib_mlx5_txwq_t *txwq,
unsigned opcode, unsigned length,
/* SEND */ uint8_t am_id, void *am_hdr, unsigned am_hdr_len,
/* RDMA */ uint64_t rdma_raddr, uct_rkey_t rdma_rkey,
int force_sig, uct_rc_iface_send_desc_t *desc)
{
desc->super.sn = txwq->sw_pi;
uct_rc_mlx5_txqp_dptr_post(iface, IBV_QPT_RC, txqp, txwq,
opcode, desc + 1, length, &desc->lkey,
am_id, am_hdr, am_hdr_len,
rdma_raddr, uct_ib_md_direct_rkey(rdma_rkey),
0, 0, 0,
NULL, 0, force_sig);
uct_rc_txqp_add_send_op(txqp, &desc->super);
}
static UCS_F_ALWAYS_INLINE void
uct_dc_mlx5_iface_bcopy_post(uct_dc_mlx5_iface_t *iface, uct_dc_mlx5_ep_t *ep,
unsigned opcode, unsigned length,
/* SEND */ uint8_t am_id, void *am_hdr, unsigned am_hdr_len,
/* RDMA */ uint64_t rdma_raddr, uct_rkey_t rdma_rkey,
uct_rc_iface_send_desc_t *desc)
{
UCT_DC_MLX5_TXQP_DECL(txqp, txwq);
UCT_DC_MLX5_IFACE_TXQP_GET(iface, ep, txqp, txwq);
desc->super.sn = txwq->sw_pi;
uct_rc_mlx5_txqp_dptr_post(&iface->super.super, IBV_EXP_QPT_DC_INI, txqp, txwq,
opcode, desc + 1, length, &desc->lkey,
am_id, am_hdr, am_hdr_len,
rdma_raddr, uct_ib_md_direct_rkey(rdma_rkey),
0, 0, 0,
&ep->av, uct_ib_mlx5_wqe_av_size(&ep->av),
MLX5_WQE_CTRL_CQ_UPDATE);
uct_rc_txqp_add_send_op(txqp, &desc->super);
}
static UCS_F_ALWAYS_INLINE void
uct_dc_mlx5_iface_bcopy_post(uct_dc_mlx5_iface_t *iface, uct_dc_mlx5_ep_t *ep,
unsigned opcode, unsigned length,
/* RDMA */ uint64_t rdma_raddr, uct_rkey_t rdma_rkey,
uct_rc_iface_send_desc_t *desc, uint8_t send_flags,
uint32_t imm_val_be, const void *buffer,
uct_ib_log_sge_t *log_sge)
{
UCT_DC_MLX5_TXQP_DECL(txqp, txwq);
UCT_DC_MLX5_IFACE_TXQP_GET(iface, ep, txqp, txwq);
desc->super.sn = txwq->sw_pi;
uct_rc_mlx5_txqp_dptr_post(&iface->super, UCT_IB_QPT_DCI, txqp, txwq,
opcode, buffer, length, &desc->lkey,
rdma_raddr, uct_ib_md_direct_rkey(rdma_rkey),
0, 0, 0, 0,
&ep->av, uct_dc_mlx5_ep_get_grh(ep),
uct_ib_mlx5_wqe_av_size(&ep->av),
MLX5_WQE_CTRL_CQ_UPDATE | send_flags, imm_val_be, INT_MAX,
log_sge);
uct_rc_txqp_add_send_op(txqp, &desc->super);
}
static UCS_F_ALWAYS_INLINE void
uct_dc_mlx5_iface_atomic_post(uct_dc_mlx5_iface_t *iface, uct_dc_mlx5_ep_t *ep,
unsigned opcode, uct_rc_iface_send_desc_t *desc, unsigned length,
uint64_t remote_addr, uct_rkey_t rkey,
uint64_t compare_mask, uint64_t compare, uint64_t swap_add)
{
UCT_DC_MLX5_TXQP_DECL(txqp, txwq);
UCT_DC_MLX5_IFACE_TXQP_GET(iface, ep, txqp, txwq);
desc->super.sn = txwq->sw_pi;
uct_rc_mlx5_txqp_dptr_post(&iface->super.super, IBV_EXP_QPT_DC_INI, txqp, txwq,
opcode, desc + 1, length, &desc->lkey,
0, NULL, 0,
remote_addr + ep->super.umr_offset,
uct_ib_md_umr_rkey(rkey),
compare_mask, compare, swap_add,
&ep->av, uct_ib_mlx5_wqe_av_size(&ep->av),
MLX5_WQE_CTRL_CQ_UPDATE);
UCT_TL_EP_STAT_ATOMIC(&ep->super.super);
uct_rc_txqp_add_send_op(txqp, &desc->super);
}
static UCS_F_ALWAYS_INLINE void
uct_rc_mlx5_ep_atomic_post(uct_rc_mlx5_ep_t *ep, unsigned opcode,
uct_rc_iface_send_desc_t *desc, unsigned length,
uint64_t remote_addr, uct_rkey_t rkey,
uint64_t compare_mask, uint64_t compare,
uint64_t swap_add, int signal)
{
uct_rc_iface_t *iface = ucs_derived_of(ep->super.super.super.iface,
uct_rc_iface_t);
uint32_t ib_rkey = uct_ib_resolve_atomic_rkey(rkey, ep->super.atomic_mr_offset,
&remote_addr);
desc->super.sn = ep->tx.wq.sw_pi;
uct_rc_mlx5_txqp_dptr_post(iface, IBV_QPT_RC,
&ep->super.txqp, &ep->tx.wq,
opcode, desc + 1, length, &desc->lkey,
0, NULL, 0, remote_addr, ib_rkey,
compare_mask, compare, swap_add,
NULL, 0, signal);
UCT_TL_EP_STAT_ATOMIC(&ep->super.super);
uct_rc_txqp_add_send_op(&ep->super.txqp, &desc->super);
}
