static UCS_F_ALWAYS_INLINE ucs_status_t uct_rc_mlx5_ep_atomic_add(uct_ep_h tl_ep, int opcode, unsigned length, uint64_t add, uint64_t remote_addr, uct_rkey_t rkey) { uct_rc_mlx5_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_mlx5_iface_t); uct_rc_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_mlx5_ep_t); uct_rc_iface_send_desc_t *desc; UCT_RC_MLX5_CHECK_RES(iface, ep); UCT_RC_IFACE_GET_TX_DESC(&iface->super, iface->tx.atomic_desc_mp, desc); desc->super.handler = (uct_rc_send_handler_t)ucs_mpool_put; return uct_rc_mlx5_ep_atomic_post(ep, opcode, desc, length, remote_addr, rkey, 0, 0, add, 0, UCS_OK); }
static UCS_F_ALWAYS_INLINE ucs_status_t uct_rc_mlx5_ep_atomic_add(uct_ep_h tl_ep, int opcode, unsigned length, uint64_t add, uint64_t remote_addr, uct_rkey_t rkey) { uct_rc_mlx5_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_mlx5_iface_t); uct_rc_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_mlx5_ep_t); uct_rc_iface_send_desc_t *desc; UCT_RC_CHECK_RES(&iface->super, &ep->super); UCT_RC_IFACE_GET_TX_ATOMIC_ADD_DESC(&iface->super, &iface->mlx5_common.tx.atomic_desc_mp, desc); uct_rc_mlx5_ep_atomic_post(ep, opcode, desc, length, remote_addr, rkey, 0, 0, add, 0); return UCS_OK; }
static UCS_F_ALWAYS_INLINE ucs_status_t uct_rc_mlx5_ep_atomic(uct_rc_mlx5_ep_t *ep, int opcode, void *result, int ext, unsigned length, uint64_t remote_addr, uct_rkey_t rkey, uint64_t compare_mask, uint64_t compare, uint64_t swap_add, uct_completion_t *comp) { uct_rc_mlx5_iface_t *iface = ucs_derived_of(ep->super.super.super.iface, uct_rc_mlx5_iface_t); uct_rc_iface_send_desc_t *desc; UCT_RC_CHECK_RES(&iface->super, &ep->super); UCT_RC_IFACE_GET_TX_ATOMIC_DESC(&iface->super, &iface->mlx5_common.tx.atomic_desc_mp, desc, uct_rc_iface_atomic_handler(&iface->super, ext, length), result, comp); uct_rc_mlx5_ep_atomic_post(ep, opcode, desc, length, remote_addr, rkey, compare_mask, compare, swap_add, MLX5_WQE_CTRL_CQ_UPDATE); return UCS_INPROGRESS; }
static UCS_F_ALWAYS_INLINE ucs_status_t uct_rc_mlx5_ep_atomic(uct_rc_mlx5_ep_t *ep, int opcode, void *result, unsigned length, uct_rc_send_handler_t handler, uint64_t remote_addr, uct_rkey_t rkey, uint64_t compare_mask, uint64_t compare, uint64_t swap_add, uct_completion_t *comp) { uct_rc_mlx5_iface_t *iface = ucs_derived_of(ep->super.super.super.iface, uct_rc_mlx5_iface_t); uct_rc_iface_send_desc_t *desc; UCT_CHECK_PARAM(comp != NULL, "completion must be non-NULL"); UCT_RC_MLX5_CHECK_RES(iface, ep); UCT_RC_IFACE_GET_TX_DESC(&iface->super, iface->tx.atomic_desc_mp, desc); desc->super.handler = handler; desc->super.result = result; desc->super.user_comp = comp; return uct_rc_mlx5_ep_atomic_post(ep, opcode, desc, length, remote_addr, rkey, compare_mask, compare, swap_add, MLX5_WQE_CTRL_CQ_UPDATE, UCS_INPROGRESS); }