/*
* Helper function for zero-copy post.
* Adds user completion to the callback queue.
*/
static UCS_F_ALWAYS_INLINE ucs_status_t
uct_rc_mlx5_ep_zcopy_post(uct_rc_mlx5_ep_t *ep, unsigned opcode, const void *buffer,
unsigned length, struct ibv_mr *mr,
/* SEND */ uint8_t am_id, const void *am_hdr, unsigned am_hdr_len,
/* RDMA */ uint64_t rdma_raddr, uct_rkey_t rdma_rkey,
int force_sig, uct_completion_t *comp)
{
uct_rc_mlx5_iface_t *iface = ucs_derived_of(ep->super.super.super.iface,
uct_rc_mlx5_iface_t);
ucs_status_t status;
uint16_t sn;
UCT_RC_MLX5_CHECK_RES(iface, ep);
sn = ep->tx.sw_pi;
status = uct_rc_mlx5_ep_dptr_post(ep, opcode, buffer, length, &mr->lkey,
am_id, am_hdr, am_hdr_len, rdma_raddr, rdma_rkey,
0, 0, 0,
(comp == NULL) ? force_sig : MLX5_WQE_CTRL_CQ_UPDATE);
if (status != UCS_OK) {
return status;
}
uct_rc_ep_add_send_comp(&iface->super, &ep->super, comp, sn);
return UCS_INPROGRESS;
}
/*
*
* Helper function for buffer-copy post.
* Adds the descriptor to the callback queue.
*/
static UCS_F_ALWAYS_INLINE void
uct_rc_mlx5_ep_bcopy_post(uct_rc_mlx5_ep_t *ep, unsigned opcode, unsigned length,
/* SEND */ uint8_t am_id, void *am_hdr, unsigned am_hdr_len,
/* RDMA */ uint64_t rdma_raddr, uct_rkey_t rdma_rkey,
int force_sig, uct_rc_iface_send_desc_t *desc)
{
desc->super.sn = ep->tx.wq.sw_pi;
uct_rc_mlx5_ep_dptr_post(ep, opcode, desc + 1, length, &desc->lkey,
am_id, am_hdr, am_hdr_len, rdma_raddr, rdma_rkey,
0, 0, 0, force_sig);
ucs_queue_push(&ep->super.outstanding, &desc->super.queue);
}
static UCS_F_ALWAYS_INLINE void
uct_rc_mlx5_ep_atomic_post(uct_rc_mlx5_ep_t *ep, unsigned opcode,
uct_rc_iface_send_desc_t *desc, unsigned length,
uint64_t remote_addr, uct_rkey_t rkey,
uint64_t compare_mask, uint64_t compare,
uint64_t swap_add, int signal)
{
desc->super.sn = ep->tx.wq.sw_pi;
uct_rc_mlx5_ep_dptr_post(ep, opcode, desc + 1, length, &desc->lkey,
0, NULL, 0, remote_addr, rkey, compare_mask,
compare, swap_add, signal);
UCT_TL_EP_STAT_ATOMIC(&ep->super.super);
ucs_queue_push(&ep->super.outstanding, &desc->super.queue);
}
static UCS_F_ALWAYS_INLINE ucs_status_t
uct_rc_mlx5_ep_atomic_post(uct_rc_mlx5_ep_t *ep, unsigned opcode,
uct_rc_iface_send_desc_t *desc, unsigned length,
uint64_t remote_addr, uct_rkey_t rkey,
uint64_t compare_mask, uint64_t compare,
uint64_t swap_add, int signal, ucs_status_t success)
{
ucs_status_t status;
desc->super.sn = ep->tx.sw_pi;
status = uct_rc_mlx5_ep_dptr_post(ep, opcode, desc + 1, length, &desc->lkey,
0, NULL, 0, remote_addr, rkey, compare_mask,
compare, swap_add, signal);
if (status != UCS_OK) {
return status;
}
UCT_TL_EP_STAT_ATOMIC(&ep->super.super);
ucs_queue_push(&ep->super.outstanding, &desc->super.queue);
return success;
}
/*
*
* Helper function for buffer-copy post.
* Adds the descriptor to the callback queue.
*/
static UCS_F_ALWAYS_INLINE ucs_status_t
uct_rc_mlx5_ep_bcopy_post(uct_rc_mlx5_ep_t *ep, unsigned opcode, unsigned length,
/* SEND */ uint8_t am_id, void *am_hdr, unsigned am_hdr_len,
/* RDMA */ uint64_t rdma_raddr, uct_rkey_t rdma_rkey,
int force_sig, uct_rc_iface_send_desc_t *desc,
ucs_status_t success)
{
ucs_status_t status;
desc->super.sn = ep->tx.sw_pi;
status = uct_rc_mlx5_ep_dptr_post(ep, opcode, desc + 1, length, &desc->lkey,
am_id, am_hdr, am_hdr_len, rdma_raddr, rdma_rkey,
0, 0, 0, force_sig);
if (status != UCS_OK) {
ucs_mpool_put(desc);
return status;
}
ucs_queue_push(&ep->super.outstanding, &desc->super.queue);
return success;
}