ucs_status_t uct_rc_verbs_ep_connect_to_ep(uct_ep_h tl_ep,
const uct_device_addr_t *dev_addr,
const uct_ep_addr_t *ep_addr)
{
uct_rc_verbs_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_verbs_ep_t);
uct_rc_verbs_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_verbs_iface_t);
const uct_ib_address_t *ib_addr = (const uct_ib_address_t *)dev_addr;
const uct_rc_ep_address_t *rc_addr = (const uct_rc_ep_address_t*)ep_addr;
struct ibv_ah_attr ah_attr;
ucs_status_t status;
uint32_t qp_num;
uct_ib_iface_fill_ah_attr(&iface->super.super, ib_addr, ep->super.path_bits,
&ah_attr);
#if HAVE_IBV_EX_HW_TM
if(UCT_RC_VERBS_TM_ENABLED(iface)) {
/* For HW TM we need 2 QPs, one of which will be used by the device for
* RNDV offload (for issuing RDMA reads and sending RNDV ACK). No WQEs
* should be posted to the send side of the QP which is owned by device. */
uct_rc_verbs_ep_tm_address_t *tm_addr = ucs_derived_of(rc_addr,
uct_rc_verbs_ep_tm_address_t);
ucs_assert_always(tm_addr->super.type == UCT_RC_EP_ADDR_TYPE_TM);
status = uct_rc_iface_qp_connect(&iface->super, ep->tm_qp,
uct_ib_unpack_uint24(rc_addr->qp_num),
&ah_attr);
if (status != UCS_OK) {
return status;
}
/* Need to connect local ep QP to the one owned by device
* (and bound to XRQ) on the peer. */
qp_num = uct_ib_unpack_uint24(tm_addr->tm_qp_num);
} else
#endif
{
ucs_assert_always(rc_addr->type == UCT_RC_EP_ADDR_TYPE_BASIC);
qp_num = uct_ib_unpack_uint24(rc_addr->qp_num);
}
status = uct_rc_iface_qp_connect(&iface->super, ep->super.txqp.qp,
qp_num, &ah_attr);
if (status != UCS_OK) {
return status;
}
ep->super.atomic_mr_offset = uct_ib_md_atomic_offset(rc_addr->atomic_mr_id);
return UCS_OK;
}
static UCS_CLASS_INIT_FUNC(uct_dc_mlx5_ep_t, uct_iface_t *tl_iface,
const uct_device_addr_t *dev_addr,
const uct_iface_addr_t *iface_addr)
{
uct_dc_mlx5_iface_t *iface = ucs_derived_of(tl_iface, uct_dc_mlx5_iface_t);
const uct_ib_address_t *ib_addr = (const uct_ib_address_t *)dev_addr;
const uct_dc_iface_addr_t *if_addr = (const uct_dc_iface_addr_t *)iface_addr;
ucs_status_t status;
int is_global;
ucs_trace_func("");
UCS_CLASS_CALL_SUPER_INIT(uct_dc_ep_t, &iface->super, if_addr);
status = uct_ud_mlx5_iface_get_av(&iface->super.super.super, &iface->ud_common,
ib_addr, iface->super.super.super.path_bits[0],
&self->av, NULL, &is_global);
if (status != UCS_OK) {
return UCS_ERR_INVALID_ADDR;
}
self->av.dqp_dct |= htonl(uct_ib_unpack_uint24(if_addr->qp_num));
ucs_debug("created ep %p on iface %p", self, iface);
return UCS_OK;
}
ucs_status_t uct_dc_mlx5_ep_fc_ctrl(uct_ep_t *tl_ep, unsigned op,
uct_rc_fc_request_t *req)
{
uct_dc_mlx5_ep_t *dc_ep = ucs_derived_of(tl_ep, uct_dc_mlx5_ep_t);
uct_dc_mlx5_iface_t *iface = ucs_derived_of(tl_ep->iface,
uct_dc_mlx5_iface_t);
uct_ib_iface_t *ib_iface = &iface->super.super.super;
struct ibv_ah_attr ah_attr = {.is_global = 0};
uct_dc_fc_sender_data_t sender;
uct_dc_fc_request_t *dc_req;
struct mlx5_wqe_av mlx5_av;
uct_ib_mlx5_base_av_t av;
ucs_status_t status;
uintptr_t sender_ep;
struct ibv_ah *ah;
UCT_DC_MLX5_TXQP_DECL(txqp, txwq);
ucs_assert((sizeof(uint8_t) + sizeof(sender_ep)) <=
UCT_IB_MLX5_AV_FULL_SIZE);
UCT_DC_MLX5_CHECK_RES(iface, dc_ep);
UCT_DC_MLX5_IFACE_TXQP_GET(iface, dc_ep, txqp, txwq);
dc_req = ucs_derived_of(req, uct_dc_fc_request_t);
if (op == UCT_RC_EP_FC_PURE_GRANT) {
ucs_assert(req != NULL);
sender_ep = (uintptr_t)dc_req->sender.ep;
/* TODO: look at common code with uct_ud_mlx5_iface_get_av */
if (dc_req->sender.global.is_global) {
uct_ib_iface_fill_ah_attr_from_gid_lid(ib_iface, dc_req->lid,
ucs_unaligned_ptr(&dc_req->sender.global.gid),
ib_iface->path_bits[0], &ah_attr);
status = uct_ib_iface_create_ah(ib_iface, &ah_attr, &ah);
if (status != UCS_OK) {
return status;
}
uct_ib_mlx5_get_av(ah, &mlx5_av);
}
/* Note av initialization is copied from exp verbs */
av.stat_rate_sl = ib_iface->config.sl; /* (attr->static_rate << 4) | attr->sl */
av.fl_mlid = ib_iface->path_bits[0] & 0x7f;
/* lid in dc_req is in BE already */
av.rlid = uct_ib_iface_is_roce(ib_iface) ? 0 :
(dc_req->lid | htons(ib_iface->path_bits[0]));
av.dqp_dct = htonl(dc_req->dct_num);
uct_dc_mlx5_iface_set_av_sport(iface, &av, dc_req->dct_num);
if (!iface->ud_common.config.compact_av || ah_attr.is_global) {
av.dqp_dct |= UCT_IB_MLX5_EXTENDED_UD_AV;
}
uct_rc_mlx5_txqp_inline_post(&iface->super, UCT_IB_QPT_DCI,
txqp, txwq, MLX5_OPCODE_SEND,
&av /*dummy*/, 0, op, sender_ep, 0,
0, 0,
&av, ah_attr.is_global ? mlx5_av_grh(&mlx5_av) : NULL,
uct_ib_mlx5_wqe_av_size(&av), 0, INT_MAX);
} else {
ucs_assert(op == UCT_RC_EP_FC_FLAG_HARD_REQ);
sender.ep = (uint64_t)dc_ep;
sender.global.gid = ib_iface->gid;
sender.global.is_global = dc_ep->flags & UCT_DC_MLX5_EP_FLAG_GRH;
UCS_STATS_UPDATE_COUNTER(dc_ep->fc.stats,
UCT_RC_FC_STAT_TX_HARD_REQ, 1);
uct_rc_mlx5_txqp_inline_post(&iface->super, UCT_IB_QPT_DCI,
txqp, txwq, MLX5_OPCODE_SEND_IMM,
&sender.global, sizeof(sender.global), op, sender.ep,
uct_dc_mlx5_get_dct_num(iface),
0, 0,
&dc_ep->av,
uct_dc_mlx5_ep_get_grh(dc_ep),
uct_ib_mlx5_wqe_av_size(&dc_ep->av),
MLX5_WQE_CTRL_SOLICITED, INT_MAX);
}
return UCS_OK;
}
UCS_CLASS_INIT_FUNC(uct_dc_mlx5_ep_t, uct_dc_mlx5_iface_t *iface, const uct_dc_mlx5_iface_addr_t *if_addr,
uct_ib_mlx5_base_av_t *av)
{
uint32_t remote_dctn;
ucs_trace_func("");
UCS_CLASS_CALL_SUPER_INIT(uct_base_ep_t, &iface->super.super.super.super);
self->atomic_mr_offset = uct_ib_md_atomic_offset(if_addr->atomic_mr_id);
remote_dctn = uct_ib_unpack_uint24(if_addr->qp_num);
memcpy(&self->av, av, sizeof(*av));
self->av.dqp_dct |= htonl(remote_dctn);
uct_dc_mlx5_iface_set_av_sport(iface, &self->av, remote_dctn);
return uct_dc_mlx5_ep_basic_init(iface, self);
}
