static UCS_F_ALWAYS_INLINE void
uct_dc_mlx5_iface_atomic_post(uct_dc_mlx5_iface_t *iface, uct_dc_mlx5_ep_t *ep,
unsigned opcode, uct_rc_iface_send_desc_t *desc, unsigned length,
uint64_t remote_addr, uct_rkey_t rkey,
uint64_t compare_mask, uint64_t compare,
uint64_t swap_mask, uint64_t swap_add)
{
uint32_t ib_rkey = uct_ib_resolve_atomic_rkey(rkey, ep->atomic_mr_offset,
&remote_addr);
UCT_DC_MLX5_TXQP_DECL(txqp, txwq);
UCT_DC_MLX5_IFACE_TXQP_GET(iface, ep, txqp, txwq);
desc->super.sn = txwq->sw_pi;
uct_rc_mlx5_txqp_dptr_post(&iface->super, UCT_IB_QPT_DCI, txqp, txwq,
opcode, desc + 1, length, &desc->lkey,
remote_addr, ib_rkey,
compare_mask, compare, swap_mask, swap_add,
&ep->av, uct_dc_mlx5_ep_get_grh(ep),
uct_ib_mlx5_wqe_av_size(&ep->av),
MLX5_WQE_CTRL_CQ_UPDATE, 0, INT_MAX, NULL);
UCT_TL_EP_STAT_ATOMIC(&ep->super);
uct_rc_txqp_add_send_op(txqp, &desc->super);
}
static UCS_F_ALWAYS_INLINE void
uct_rc_verbs_ep_atomic_post(uct_rc_verbs_ep_t *ep, int opcode, uint64_t compare_add,
uint64_t swap, uint64_t remote_addr, uct_rkey_t rkey,
uct_rc_iface_send_desc_t *desc, int force_sig)
{
uint32_t ib_rkey = uct_ib_resolve_atomic_rkey(rkey, ep->super.atomic_mr_offset,
&remote_addr);
struct ibv_send_wr wr;
struct ibv_sge sge;
UCT_RC_VERBS_FILL_ATOMIC_WR(wr, wr.opcode, sge, opcode, compare_add, swap,
remote_addr, ib_rkey);
UCT_TL_EP_STAT_ATOMIC(&ep->super.super);
uct_rc_verbs_ep_post_send_desc(ep, &wr, desc, force_sig);
}
static UCS_F_ALWAYS_INLINE void
uct_rc_mlx5_ep_atomic_post(uct_rc_mlx5_ep_t *ep, unsigned opcode,
uct_rc_iface_send_desc_t *desc, unsigned length,
uint64_t remote_addr, uct_rkey_t rkey,
uint64_t compare_mask, uint64_t compare,
uint64_t swap_add, int signal)
{
uct_rc_iface_t *iface = ucs_derived_of(ep->super.super.super.iface,
uct_rc_iface_t);
uint32_t ib_rkey = uct_ib_resolve_atomic_rkey(rkey, ep->super.atomic_mr_offset,
&remote_addr);
desc->super.sn = ep->tx.wq.sw_pi;
uct_rc_mlx5_txqp_dptr_post(iface, IBV_QPT_RC,
&ep->super.txqp, &ep->tx.wq,
opcode, desc + 1, length, &desc->lkey,
0, NULL, 0, remote_addr, ib_rkey,
compare_mask, compare, swap_add,
NULL, 0, signal);
UCT_TL_EP_STAT_ATOMIC(&ep->super.super);
uct_rc_txqp_add_send_op(&ep->super.txqp, &desc->super);
}
