static ucs_status_t UCS_F_ALWAYS_INLINE uct_dc_mlx5_ep_atomic_op_post(uct_ep_h tl_ep, unsigned opcode, unsigned size, uint64_t value, uint64_t remote_addr, uct_rkey_t rkey) { uct_dc_mlx5_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_dc_mlx5_iface_t); uct_dc_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_dc_mlx5_ep_t); uct_rc_iface_send_desc_t *desc; int op; uint64_t compare_mask; uint64_t compare; uint64_t swap_mask; uint64_t swap; int ext; /* not used here */ ucs_status_t status; UCT_DC_MLX5_CHECK_RES(iface, ep); UCT_RC_MLX5_CHECK_ATOMIC_OPS(opcode, size, UCT_RC_MLX5_ATOMIC_OPS); status = uct_rc_mlx5_iface_common_atomic_data(opcode, size, value, &op, &compare_mask, &compare, &swap_mask, &swap, &ext); if (ucs_unlikely(UCS_STATUS_IS_ERR(status))) { return status; } UCT_RC_IFACE_GET_TX_ATOMIC_DESC(&iface->super.super, &iface->super.tx.atomic_desc_mp, desc); uct_dc_mlx5_iface_atomic_post(iface, ep, op, desc, size, remote_addr, rkey, compare_mask, compare, swap_mask, swap); return UCS_OK; }
static UCS_F_ALWAYS_INLINE ucs_status_t uct_rc_verbs_ep_atomic(uct_rc_verbs_ep_t *ep, int opcode, void *result, uint64_t compare_add, uint64_t swap, uint64_t remote_addr, uct_rkey_t rkey, uct_completion_t *comp) { uct_rc_verbs_iface_t *iface = ucs_derived_of(ep->super.super.super.iface, uct_rc_verbs_iface_t); uct_rc_iface_send_desc_t *desc; UCT_RC_CHECK_RES(&iface->super, &ep->super); UCT_RC_IFACE_GET_TX_ATOMIC_DESC(&iface->super, &iface->verbs_common.short_desc_mp, desc, iface->super.config.atomic64_handler, result, comp); uct_rc_verbs_ep_atomic_post(ep, opcode, compare_add, swap, remote_addr, rkey, desc, IBV_SEND_SIGNALED); return UCS_INPROGRESS; }
static UCS_F_ALWAYS_INLINE ucs_status_t uct_dc_mlx5_ep_atomic(uct_dc_mlx5_ep_t *ep, int opcode, void *result, int ext, unsigned length, uint64_t remote_addr, uct_rkey_t rkey, uint64_t compare_mask, uint64_t compare, uint64_t swap_add, uct_completion_t *comp) { uct_dc_mlx5_iface_t *iface = ucs_derived_of(ep->super.super.super.iface, uct_dc_mlx5_iface_t); uct_rc_iface_send_desc_t *desc; UCT_DC_CHECK_RES(&iface->super, &ep->super); UCT_RC_IFACE_GET_TX_ATOMIC_DESC(&iface->super.super, &iface->mlx5_common.tx.atomic_desc_mp, desc, uct_rc_iface_atomic_handler(&iface->super.super, ext, length), result, comp); uct_dc_mlx5_iface_atomic_post(iface, ep, opcode, desc, length, remote_addr, rkey, compare_mask, compare, swap_add); return UCS_INPROGRESS; }
static inline ucs_status_t uct_rc_verbs_ep_ext_atomic(uct_rc_verbs_ep_t *ep, int opcode, void *result, uint32_t length, uint64_t compare_mask, uint64_t compare_add, uint64_t swap, uint64_t remote_addr, uct_rkey_t rkey, uct_completion_t *comp) { uct_rc_verbs_iface_t *iface = ucs_derived_of(ep->super.super.super.iface, uct_rc_verbs_iface_t); uct_rc_send_handler_t handler = uct_rc_iface_atomic_handler(&iface->super, 1, length); uct_rc_iface_send_desc_t *desc; UCT_RC_CHECK_RES(&iface->super, &ep->super); UCT_RC_IFACE_GET_TX_ATOMIC_DESC(&iface->super, &iface->verbs_common.short_desc_mp, desc, handler, result, comp); uct_rc_verbs_ep_ext_atomic_post(ep, opcode, length, compare_mask, compare_add, swap, remote_addr, rkey, desc, IBV_EXP_SEND_SIGNALED); return UCS_INPROGRESS; }
ucs_status_t uct_rc_verbs_ep_atomic64_post(uct_ep_h tl_ep, unsigned opcode, uint64_t value, uint64_t remote_addr, uct_rkey_t rkey) { uct_rc_verbs_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_verbs_iface_t); uct_rc_verbs_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_verbs_ep_t); uct_rc_iface_send_desc_t *desc; if (opcode != UCT_ATOMIC_OP_ADD) { return UCS_ERR_UNSUPPORTED; } /* TODO don't allocate descriptor - have dummy buffer */ UCT_RC_CHECK_RES(&iface->super, &ep->super); UCT_RC_IFACE_GET_TX_ATOMIC_DESC(&iface->super, &iface->short_desc_mp, desc); uct_rc_verbs_ep_atomic_post(ep, IBV_WR_ATOMIC_FETCH_AND_ADD, value, 0, remote_addr, rkey, desc, IBV_SEND_SIGNALED); return UCS_OK; }