static inline void
uct_rc_verbs_ep_ext_atomic_post(uct_rc_verbs_ep_t *ep, int opcode, uint32_t length,
uint64_t compare_mask, uint64_t compare_add,
uint64_t swap, uint64_t remote_addr, uct_rkey_t rkey,
uct_rc_iface_send_desc_t *desc, uint64_t force_sig)
{
struct ibv_exp_send_wr wr;
struct ibv_sge sge;
uct_rc_verbs_fill_ext_atomic_wr(&wr, &sge, opcode, length, compare_mask,
compare_add, swap, remote_addr, rkey, ep->super.atomic_mr_offset);
UCT_RC_VERBS_FILL_DESC_WR(&wr, desc);
UCT_TL_EP_STAT_ATOMIC(&ep->super.super);
uct_rc_verbs_exp_post_send(ep, &wr, force_sig|IBV_EXP_SEND_EXT_ATOMIC_INLINE);
uct_rc_txqp_add_send_op_sn(&ep->super.txqp, &desc->super, ep->txcnt.pi);
}
static ucs_status_t uct_rc_verbs_ep_nop(uct_rc_verbs_ep_t *ep)
{
#if HAVE_DECL_IBV_EXP_WR_NOP
uct_rc_verbs_iface_t *iface = ucs_derived_of(ep->super.super.super.iface,
uct_rc_verbs_iface_t);
struct ibv_exp_send_wr wr;
wr.next = NULL;
wr.num_sge = 0;
wr.exp_opcode = IBV_EXP_WR_NOP;
wr.exp_send_flags = IBV_EXP_SEND_FENCE;
wr.comp_mask = 0;
UCT_RC_CHECK_RES(&iface->super, &ep->super);
uct_rc_verbs_exp_post_send(ep, &wr, IBV_EXP_SEND_SIGNALED);
return UCS_OK;
#else
return UCS_ERR_UNSUPPORTED;
#endif
}
static inline ucs_status_t
uct_rc_verbs_ext_atomic_post(uct_rc_verbs_ep_t *ep, int opcode, uint32_t length,
uint64_t compare_mask, uint64_t compare_add,
uint64_t swap, uint64_t remote_addr, uct_rkey_t rkey,
uct_rc_iface_send_desc_t *desc, int force_sig,
ucs_status_t success)
{
struct ibv_exp_send_wr wr;
struct ibv_sge sge;
sge.addr = (uintptr_t)(desc + 1);
sge.lkey = desc->lkey;
sge.length = length;
wr.next = NULL;
wr.sg_list = &sge;
wr.num_sge = 1;
wr.exp_opcode = opcode;
wr.exp_send_flags = IBV_EXP_SEND_EXT_ATOMIC_INLINE;
wr.comp_mask = 0;
wr.ext_op.masked_atomics.log_arg_sz = ucs_ilog2(length);
wr.ext_op.masked_atomics.remote_addr = remote_addr;
wr.ext_op.masked_atomics.rkey = rkey;
switch (opcode) {
case IBV_EXP_WR_EXT_MASKED_ATOMIC_CMP_AND_SWP:
wr.ext_op.masked_atomics.wr_data.inline_data.op.cmp_swap.compare_mask = compare_mask;
wr.ext_op.masked_atomics.wr_data.inline_data.op.cmp_swap.compare_val = compare_add;
wr.ext_op.masked_atomics.wr_data.inline_data.op.cmp_swap.swap_mask = (uint64_t)(-1);
wr.ext_op.masked_atomics.wr_data.inline_data.op.cmp_swap.swap_val = swap;
break;
case IBV_EXP_WR_EXT_MASKED_ATOMIC_FETCH_AND_ADD:
wr.ext_op.masked_atomics.wr_data.inline_data.op.fetch_add.add_val = compare_add;
wr.ext_op.masked_atomics.wr_data.inline_data.op.fetch_add.field_boundary = 0;
break;
}
UCT_TL_EP_STAT_ATOMIC(&ep->super.super);
uct_rc_verbs_exp_post_send(ep, &wr, force_sig);
uct_rc_verbs_ep_push_desc(ep, desc);
return success;
}
