ucs_status_t uct_dc_mlx5_ep_atomic_add(uct_ep_h tl_ep,
int opcode, unsigned length,
uint64_t add, uint64_t remote_addr, uct_rkey_t rkey)
{
uct_dc_mlx5_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_dc_mlx5_iface_t);
uct_dc_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_dc_mlx5_ep_t);
uct_rc_iface_send_desc_t *desc;
UCT_DC_CHECK_RES(&iface->super, &ep->super);
UCT_RC_IFACE_GET_TX_ATOMIC_ADD_DESC(&iface->super.super, &iface->mlx5_common.tx.atomic_desc_mp, desc);
uct_dc_mlx5_iface_atomic_post(iface, ep, opcode, desc, length,
remote_addr, rkey, 0, 0, add);
return UCS_OK;
}
ssize_t uct_dc_mlx5_ep_put_bcopy(uct_ep_h tl_ep, uct_pack_callback_t pack_cb,
void *arg, uint64_t remote_addr, uct_rkey_t rkey)
{
uct_dc_mlx5_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_dc_mlx5_iface_t);
uct_dc_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_dc_mlx5_ep_t);
uct_rc_iface_send_desc_t *desc;
size_t length;
UCT_DC_CHECK_RES(&iface->super, &ep->super);
UCT_RC_IFACE_GET_TX_PUT_BCOPY_DESC(&iface->super.super, &iface->super.super.tx.mp,
desc, pack_cb, arg, length);
uct_dc_mlx5_iface_bcopy_post(iface, ep,
MLX5_OPCODE_RDMA_WRITE, length, 0, NULL, 0,
remote_addr, rkey, desc);
UCT_TL_EP_STAT_OP(&ep->super.super, PUT, BCOPY, length);
return length;
}
static UCS_F_ALWAYS_INLINE ucs_status_t
uct_dc_mlx5_ep_atomic(uct_dc_mlx5_ep_t *ep, int opcode, void *result, int ext,
unsigned length, uint64_t remote_addr, uct_rkey_t rkey,
uint64_t compare_mask, uint64_t compare,
uint64_t swap_add, uct_completion_t *comp)
{
uct_dc_mlx5_iface_t *iface = ucs_derived_of(ep->super.super.super.iface, uct_dc_mlx5_iface_t);
uct_rc_iface_send_desc_t *desc;
UCT_DC_CHECK_RES(&iface->super, &ep->super);
UCT_RC_IFACE_GET_TX_ATOMIC_DESC(&iface->super.super, &iface->mlx5_common.tx.atomic_desc_mp, desc,
uct_rc_iface_atomic_handler(&iface->super.super, ext, length),
result, comp);
uct_dc_mlx5_iface_atomic_post(iface, ep, opcode, desc, length, remote_addr, rkey,
compare_mask, compare, swap_add);
return UCS_INPROGRESS;
}
ucs_status_t uct_dc_mlx5_ep_get_zcopy(uct_ep_h tl_ep, const uct_iov_t *iov, size_t iovcnt,
uint64_t remote_addr, uct_rkey_t rkey,
uct_completion_t *comp)
{
uct_dc_mlx5_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_dc_mlx5_iface_t);
uct_dc_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_dc_mlx5_ep_t);
UCT_CHECK_IOV_SIZE(iovcnt, uct_ib_iface_get_max_iov(&iface->super.super.super),
"uct_dc_mlx5_ep_get_zcopy");
UCT_CHECK_LENGTH(uct_iov_total_length(iov, iovcnt), UCT_IB_MAX_MESSAGE_SIZE,
"get_zcopy");
UCT_DC_CHECK_RES(&iface->super, &ep->super);
uct_dc_mlx5_iface_zcopy_post(iface, ep, MLX5_OPCODE_RDMA_READ, iov, iovcnt,
0, NULL, 0, remote_addr, rkey, comp);
UCT_TL_EP_STAT_OP(&ep->super.super, GET, ZCOPY,
uct_iov_total_length(iov, iovcnt));
return UCS_INPROGRESS;
}
ssize_t uct_dc_mlx5_ep_am_bcopy(uct_ep_h tl_ep, uint8_t id,
uct_pack_callback_t pack_cb, void *arg)
{
uct_dc_mlx5_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_dc_mlx5_iface_t);
uct_dc_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_dc_mlx5_ep_t);
uct_rc_iface_send_desc_t *desc;
size_t length;
UCT_DC_CHECK_RES(&iface->super, &ep->super);
UCT_RC_IFACE_GET_TX_AM_BCOPY_DESC(&iface->super.super, &iface->super.super.tx.mp, desc,
id, pack_cb, arg, &length);
uct_dc_mlx5_iface_bcopy_post(iface, ep,
MLX5_OPCODE_SEND|UCT_RC_MLX5_OPCODE_FLAG_RAW,
sizeof(uct_rc_hdr_t) + length, 0, NULL, 0, 0, 0, desc);
UCT_TL_EP_STAT_OP(&ep->super.super, AM, BCOPY, length);
return length;
}
ucs_status_t uct_dc_mlx5_ep_get_bcopy(uct_ep_h tl_ep,
uct_unpack_callback_t unpack_cb,
void *arg, size_t length,
uint64_t remote_addr, uct_rkey_t rkey,
uct_completion_t *comp)
{
uct_dc_mlx5_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_dc_mlx5_iface_t);
uct_dc_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_dc_mlx5_ep_t);
uct_rc_iface_send_desc_t *desc;
UCT_CHECK_LENGTH(length, iface->super.super.super.config.seg_size, "get_bcopy");
UCT_DC_CHECK_RES(&iface->super, &ep->super);
UCT_RC_IFACE_GET_TX_GET_BCOPY_DESC(&iface->super.super, &iface->super.super.tx.mp,
desc, unpack_cb, comp, arg, length);
uct_dc_mlx5_iface_bcopy_post(iface, ep,
MLX5_OPCODE_RDMA_READ, length, 0, NULL, 0,
remote_addr, rkey, desc);
UCT_TL_EP_STAT_OP(&ep->super.super, GET, BCOPY, length);
return UCS_INPROGRESS;
}
ucs_status_t uct_dc_mlx5_ep_am_zcopy(uct_ep_h tl_ep, uint8_t id, const void *header,
unsigned header_length, const uct_iov_t *iov,
size_t iovcnt, uct_completion_t *comp)
{
uct_dc_mlx5_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_dc_mlx5_iface_t);
uct_dc_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_dc_mlx5_ep_t);
UCT_CHECK_IOV_SIZE(iovcnt, UCT_IB_MLX5_AM_ZCOPY_MAX_IOV,
"uct_dc_mlx5_ep_am_zcopy");
UCT_RC_MLX5_CHECK_AM_ZCOPY(id, header_length, uct_iov_total_length(iov, iovcnt),
iface->super.super.super.config.seg_size,
UCT_IB_MLX5_AV_FULL_SIZE);
UCT_DC_CHECK_RES(&iface->super, &ep->super);
uct_dc_mlx5_iface_zcopy_post(iface, ep, MLX5_OPCODE_SEND, iov, iovcnt,
id, header, header_length, 0, 0, comp);
UCT_TL_EP_STAT_OP(&ep->super.super, AM, ZCOPY, header_length +
uct_iov_total_length(iov, iovcnt));
return UCS_INPROGRESS;
}
ucs_status_t uct_dc_mlx5_ep_am_short(uct_ep_h tl_ep, uint8_t id, uint64_t hdr,
const void *buffer, unsigned length)
{
uct_dc_mlx5_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_dc_mlx5_iface_t);
uct_dc_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_dc_mlx5_ep_t);
UCT_DC_MLX5_TXQP_DECL(txqp, txwq);
UCT_RC_MLX5_CHECK_AM_SHORT(id, length, UCT_IB_MLX5_AV_FULL_SIZE);
UCT_DC_CHECK_RES(&iface->super, &ep->super);
UCT_DC_MLX5_IFACE_TXQP_GET(iface, ep, txqp, txwq);
uct_rc_mlx5_txqp_inline_post(&iface->super.super, IBV_EXP_QPT_DC_INI,
txqp, txwq,
MLX5_OPCODE_SEND,
buffer, length, id, hdr,
0, 0,
&ep->av, uct_ib_mlx5_wqe_av_size(&ep->av));
UCT_TL_EP_STAT_OP(&ep->super.super, AM, SHORT, sizeof(hdr) + length);
return UCS_OK;
}
ucs_status_t uct_dc_mlx5_ep_put_short(uct_ep_h tl_ep, const void *buffer,
unsigned length, uint64_t remote_addr,
uct_rkey_t rkey)
{
uct_dc_mlx5_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_dc_mlx5_iface_t);
uct_dc_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_dc_mlx5_ep_t);
UCT_DC_MLX5_TXQP_DECL(txqp, txwq);
UCT_RC_MLX5_CHECK_PUT_SHORT(length, UCT_IB_MLX5_AV_FULL_SIZE);
UCT_DC_CHECK_RES(&iface->super, &ep->super);
UCT_DC_MLX5_IFACE_TXQP_GET(iface, ep, txqp, txwq);
uct_rc_mlx5_txqp_inline_post(&iface->super.super, IBV_EXP_QPT_DC_INI,
txqp, txwq,
MLX5_OPCODE_RDMA_WRITE,
buffer, length, 0, 0,
remote_addr, uct_ib_md_direct_rkey(rkey),
&ep->av, uct_ib_mlx5_wqe_av_size(&ep->av));
UCT_TL_EP_STAT_OP(&ep->super.super, PUT, SHORT, length);
return UCS_OK;
}
ucs_status_t uct_dc_mlx5_ep_fc_ctrl(uct_ep_t *tl_ep, unsigned op,
uct_rc_fc_request_t *req)
{
uintptr_t sender_ep;
uct_ib_iface_t *ib_iface;
uct_ib_mlx5_base_av_t av;
uct_dc_fc_request_t *dc_req;
uct_dc_mlx5_ep_t *dc_mlx5_ep;
uct_dc_ep_t *dc_ep = ucs_derived_of(tl_ep, uct_dc_ep_t);
uct_dc_mlx5_iface_t *iface = ucs_derived_of(tl_ep->iface,
uct_dc_mlx5_iface_t);
UCT_DC_MLX5_TXQP_DECL(txqp, txwq);
ucs_assert((sizeof(uint8_t) + sizeof(sender_ep)) <=
UCT_IB_MLX5_AV_FULL_SIZE);
UCT_DC_CHECK_RES(&iface->super, dc_ep);
UCT_DC_MLX5_IFACE_TXQP_GET(iface, dc_ep, txqp, txwq);
if (op == UCT_RC_EP_FC_PURE_GRANT) {
ucs_assert(req != NULL);
dc_req = ucs_derived_of(req, uct_dc_fc_request_t);
sender_ep = (uintptr_t)dc_req->sender_ep;
ib_iface = &iface->super.super.super;
/* Note av initialization is copied from exp verbs */
av.stat_rate_sl = ib_iface->config.sl; /* (attr->static_rate << 4) | attr->sl */
av.fl_mlid = ib_iface->path_bits[0] & 0x7f;
/* lid in dc_req is in BE already */
av.rlid = dc_req->lid | htons(ib_iface->path_bits[0]);
av.dqp_dct = htonl(dc_req->dct_num);
if (!iface->ud_common.config.compact_av) {
av.dqp_dct |= UCT_IB_MLX5_EXTENDED_UD_AV;
}
uct_rc_mlx5_txqp_inline_post(&iface->super.super, IBV_EXP_QPT_DC_INI,
txqp, txwq, MLX5_OPCODE_SEND,
NULL, 0, op, sender_ep, 0,
0, 0,
&av, uct_ib_mlx5_wqe_av_size(&av));
} else {
ucs_assert(op == UCT_RC_EP_FC_FLAG_HARD_REQ);
sender_ep = (uintptr_t)dc_ep;
dc_mlx5_ep = ucs_derived_of(tl_ep, uct_dc_mlx5_ep_t);
UCS_STATS_UPDATE_COUNTER(dc_ep->fc.stats,
UCT_RC_FC_STAT_TX_HARD_REQ, 1);
uct_rc_mlx5_txqp_inline_post(&iface->super.super, IBV_EXP_QPT_DC_INI,
txqp, txwq, MLX5_OPCODE_SEND_IMM,
NULL, 0, op, sender_ep,
iface->super.rx.dct->dct_num,
0, 0,
&dc_mlx5_ep->av,
uct_ib_mlx5_wqe_av_size(&dc_mlx5_ep->av));
}
return UCS_OK;
}