static ucs_status_t UCS_F_ALWAYS_INLINE uct_dc_mlx5_ep_put_short_inline(uct_ep_h tl_ep, const void *buffer, unsigned length, uint64_t remote_addr, uct_rkey_t rkey) { uct_dc_mlx5_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_dc_mlx5_iface_t); uct_dc_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_dc_mlx5_ep_t); UCT_DC_MLX5_TXQP_DECL(txqp, txwq); UCT_RC_MLX5_CHECK_PUT_SHORT(length, UCT_IB_MLX5_AV_FULL_SIZE); UCT_DC_MLX5_CHECK_RES(iface, ep); UCT_DC_MLX5_IFACE_TXQP_GET(iface, ep, txqp, txwq); uct_rc_mlx5_txqp_inline_post(&iface->super, UCT_IB_QPT_DCI, txqp, txwq, MLX5_OPCODE_RDMA_WRITE, buffer, length, 0, 0, 0, remote_addr, uct_ib_md_direct_rkey(rkey), &ep->av, uct_dc_mlx5_ep_get_grh(ep), uct_ib_mlx5_wqe_av_size(&ep->av), 0, INT_MAX); UCT_TL_EP_STAT_OP(&ep->super, PUT, SHORT, length); return UCS_OK; }
ucs_status_t uct_rc_mlx5_ep_flush(uct_ep_h tl_ep, unsigned flags, uct_completion_t *comp) { uct_rc_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_mlx5_ep_t); uct_rc_mlx5_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_mlx5_iface_t); uint16_t sn; if (!uct_rc_iface_has_tx_resources(&iface->super)) { return UCS_ERR_NO_RESOURCE; } if (uct_rc_txqp_available(&ep->super.txqp) == ep->tx.wq.bb_max) { UCT_TL_EP_STAT_FLUSH(&ep->super.super); return UCS_OK; } if (uct_rc_txqp_unsignaled(&ep->super.txqp) != 0) { sn = ep->tx.wq.sw_pi; UCT_RC_CHECK_RES(&iface->super, &ep->super); uct_rc_mlx5_txqp_inline_post(&iface->super, IBV_QPT_RC, &ep->super.txqp, &ep->tx.wq, MLX5_OPCODE_NOP, NULL, 0, 0, 0, 0, 0, NULL, 0); } else if (!uct_rc_ep_has_tx_resources(&ep->super)) { return UCS_ERR_NO_RESOURCE; } else { sn = ep->tx.wq.sig_pi; } uct_rc_txqp_add_send_comp(&iface->super, &ep->super.txqp, comp, sn); UCT_TL_EP_STAT_FLUSH_WAIT(&ep->super.super); return UCS_INPROGRESS; }
static ucs_status_t UCS_F_ALWAYS_INLINE uct_dc_mlx5_ep_am_short_inline(uct_ep_h tl_ep, uint8_t id, uint64_t hdr, const void *buffer, unsigned length) { uct_dc_mlx5_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_dc_mlx5_iface_t); uct_dc_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_dc_mlx5_ep_t); UCT_DC_MLX5_TXQP_DECL(txqp, txwq); UCT_RC_MLX5_CHECK_AM_SHORT(id, length, UCT_IB_MLX5_AV_FULL_SIZE); UCT_DC_CHECK_RES_AND_FC(iface, ep); UCT_DC_MLX5_IFACE_TXQP_GET(iface, ep, txqp, txwq); uct_rc_mlx5_txqp_inline_post(&iface->super, UCT_IB_QPT_DCI, txqp, txwq, MLX5_OPCODE_SEND, buffer, length, id, hdr, 0, 0, 0, &ep->av, uct_dc_mlx5_ep_get_grh(ep), uct_ib_mlx5_wqe_av_size(&ep->av), MLX5_WQE_CTRL_SOLICITED, INT_MAX); UCT_RC_UPDATE_FC_WND(&iface->super.super, &ep->fc); UCT_TL_EP_STAT_OP(&ep->super, AM, SHORT, sizeof(hdr) + length); return UCS_OK; }
ucs_status_t uct_rc_mlx5_ep_fc_ctrl(uct_rc_ep_t *rc_ep) { uct_rc_mlx5_ep_t *ep = ucs_derived_of(rc_ep, uct_rc_mlx5_ep_t); uct_rc_iface_t *iface = ucs_derived_of(rc_ep->super.super.iface, uct_rc_iface_t); UCT_RC_CHECK_RES(iface, &ep->super); uct_rc_mlx5_txqp_inline_post(iface, IBV_QPT_RC, &ep->super.txqp, &ep->tx.wq, MLX5_OPCODE_SEND|UCT_RC_MLX5_OPCODE_FLAG_RAW, NULL, 0, UCT_RC_EP_FC_PURE_GRANT, 0 , 0, 0, NULL, 0); UCT_TL_EP_STAT_OP(&ep->super.super, AM, SHORT, 0); return UCS_OK; }
ucs_status_t uct_rc_mlx5_ep_put_short(uct_ep_h tl_ep, const void *buffer, unsigned length, uint64_t remote_addr, uct_rkey_t rkey) { uct_rc_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_iface_t); uct_rc_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_mlx5_ep_t); UCT_RC_MLX5_CHECK_PUT_SHORT(length, 0); UCT_RC_CHECK_RES(iface, &ep->super); uct_rc_mlx5_txqp_inline_post(iface, IBV_QPT_RC, &ep->super.txqp, &ep->tx.wq, MLX5_OPCODE_RDMA_WRITE, buffer, length, 0, 0, remote_addr, uct_ib_md_direct_rkey(rkey), NULL, 0); UCT_TL_EP_STAT_OP(&ep->super.super, PUT, SHORT, length); return UCS_OK; }
ucs_status_t uct_dc_mlx5_ep_am_short(uct_ep_h tl_ep, uint8_t id, uint64_t hdr, const void *buffer, unsigned length) { uct_dc_mlx5_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_dc_mlx5_iface_t); uct_dc_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_dc_mlx5_ep_t); UCT_DC_MLX5_TXQP_DECL(txqp, txwq); UCT_RC_MLX5_CHECK_AM_SHORT(id, length, UCT_IB_MLX5_AV_FULL_SIZE); UCT_DC_CHECK_RES(&iface->super, &ep->super); UCT_DC_MLX5_IFACE_TXQP_GET(iface, ep, txqp, txwq); uct_rc_mlx5_txqp_inline_post(&iface->super.super, IBV_EXP_QPT_DC_INI, txqp, txwq, MLX5_OPCODE_SEND, buffer, length, id, hdr, 0, 0, &ep->av, uct_ib_mlx5_wqe_av_size(&ep->av)); UCT_TL_EP_STAT_OP(&ep->super.super, AM, SHORT, sizeof(hdr) + length); return UCS_OK; }
ucs_status_t uct_rc_mlx5_ep_am_short(uct_ep_h tl_ep, uint8_t id, uint64_t hdr, const void *payload, unsigned length) { uct_rc_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_iface_t); uct_rc_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_mlx5_ep_t); UCT_RC_MLX5_CHECK_AM_SHORT(id, length, 0); UCT_RC_CHECK_RES(iface, &ep->super); UCT_RC_CHECK_FC_WND(iface, &ep->super, id); uct_rc_mlx5_txqp_inline_post(iface, IBV_QPT_RC, &ep->super.txqp, &ep->tx.wq, MLX5_OPCODE_SEND, payload, length, id, hdr, 0, 0, NULL, 0); UCT_TL_EP_STAT_OP(&ep->super.super, AM, SHORT, sizeof(hdr) + length); UCT_RC_UPDATE_FC_WND(iface, &ep->super, id); return UCS_OK; }
ucs_status_t uct_dc_mlx5_ep_fc_ctrl(uct_ep_t *tl_ep, unsigned op, uct_rc_fc_request_t *req) { uct_dc_mlx5_ep_t *dc_ep = ucs_derived_of(tl_ep, uct_dc_mlx5_ep_t); uct_dc_mlx5_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_dc_mlx5_iface_t); uct_ib_iface_t *ib_iface = &iface->super.super.super; struct ibv_ah_attr ah_attr = {.is_global = 0}; uct_dc_fc_sender_data_t sender; uct_dc_fc_request_t *dc_req; struct mlx5_wqe_av mlx5_av; uct_ib_mlx5_base_av_t av; ucs_status_t status; uintptr_t sender_ep; struct ibv_ah *ah; UCT_DC_MLX5_TXQP_DECL(txqp, txwq); ucs_assert((sizeof(uint8_t) + sizeof(sender_ep)) <= UCT_IB_MLX5_AV_FULL_SIZE); UCT_DC_MLX5_CHECK_RES(iface, dc_ep); UCT_DC_MLX5_IFACE_TXQP_GET(iface, dc_ep, txqp, txwq); dc_req = ucs_derived_of(req, uct_dc_fc_request_t); if (op == UCT_RC_EP_FC_PURE_GRANT) { ucs_assert(req != NULL); sender_ep = (uintptr_t)dc_req->sender.ep; /* TODO: look at common code with uct_ud_mlx5_iface_get_av */ if (dc_req->sender.global.is_global) { uct_ib_iface_fill_ah_attr_from_gid_lid(ib_iface, dc_req->lid, ucs_unaligned_ptr(&dc_req->sender.global.gid), ib_iface->path_bits[0], &ah_attr); status = uct_ib_iface_create_ah(ib_iface, &ah_attr, &ah); if (status != UCS_OK) { return status; } uct_ib_mlx5_get_av(ah, &mlx5_av); } /* Note av initialization is copied from exp verbs */ av.stat_rate_sl = ib_iface->config.sl; /* (attr->static_rate << 4) | attr->sl */ av.fl_mlid = ib_iface->path_bits[0] & 0x7f; /* lid in dc_req is in BE already */ av.rlid = uct_ib_iface_is_roce(ib_iface) ? 0 : (dc_req->lid | htons(ib_iface->path_bits[0])); av.dqp_dct = htonl(dc_req->dct_num); uct_dc_mlx5_iface_set_av_sport(iface, &av, dc_req->dct_num); if (!iface->ud_common.config.compact_av || ah_attr.is_global) { av.dqp_dct |= UCT_IB_MLX5_EXTENDED_UD_AV; } uct_rc_mlx5_txqp_inline_post(&iface->super, UCT_IB_QPT_DCI, txqp, txwq, MLX5_OPCODE_SEND, &av /*dummy*/, 0, op, sender_ep, 0, 0, 0, &av, ah_attr.is_global ? mlx5_av_grh(&mlx5_av) : NULL, uct_ib_mlx5_wqe_av_size(&av), 0, INT_MAX); } else { ucs_assert(op == UCT_RC_EP_FC_FLAG_HARD_REQ); sender.ep = (uint64_t)dc_ep; sender.global.gid = ib_iface->gid; sender.global.is_global = dc_ep->flags & UCT_DC_MLX5_EP_FLAG_GRH; UCS_STATS_UPDATE_COUNTER(dc_ep->fc.stats, UCT_RC_FC_STAT_TX_HARD_REQ, 1); uct_rc_mlx5_txqp_inline_post(&iface->super, UCT_IB_QPT_DCI, txqp, txwq, MLX5_OPCODE_SEND_IMM, &sender.global, sizeof(sender.global), op, sender.ep, uct_dc_mlx5_get_dct_num(iface), 0, 0, &dc_ep->av, uct_dc_mlx5_ep_get_grh(dc_ep), uct_ib_mlx5_wqe_av_size(&dc_ep->av), MLX5_WQE_CTRL_SOLICITED, INT_MAX); } return UCS_OK; } UCS_CLASS_INIT_FUNC(uct_dc_mlx5_ep_t, uct_dc_mlx5_iface_t *iface, const uct_dc_mlx5_iface_addr_t *if_addr, uct_ib_mlx5_base_av_t *av) { uint32_t remote_dctn; ucs_trace_func(""); UCS_CLASS_CALL_SUPER_INIT(uct_base_ep_t, &iface->super.super.super.super); self->atomic_mr_offset = uct_ib_md_atomic_offset(if_addr->atomic_mr_id); remote_dctn = uct_ib_unpack_uint24(if_addr->qp_num); memcpy(&self->av, av, sizeof(*av)); self->av.dqp_dct |= htonl(remote_dctn); uct_dc_mlx5_iface_set_av_sport(iface, &self->av, remote_dctn); return uct_dc_mlx5_ep_basic_init(iface, self); }
ucs_status_t uct_dc_mlx5_ep_fc_ctrl(uct_ep_t *tl_ep, unsigned op, uct_rc_fc_request_t *req) { uintptr_t sender_ep; uct_ib_iface_t *ib_iface; uct_ib_mlx5_base_av_t av; uct_dc_fc_request_t *dc_req; uct_dc_mlx5_ep_t *dc_mlx5_ep; uct_dc_ep_t *dc_ep = ucs_derived_of(tl_ep, uct_dc_ep_t); uct_dc_mlx5_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_dc_mlx5_iface_t); UCT_DC_MLX5_TXQP_DECL(txqp, txwq); ucs_assert((sizeof(uint8_t) + sizeof(sender_ep)) <= UCT_IB_MLX5_AV_FULL_SIZE); UCT_DC_CHECK_RES(&iface->super, dc_ep); UCT_DC_MLX5_IFACE_TXQP_GET(iface, dc_ep, txqp, txwq); if (op == UCT_RC_EP_FC_PURE_GRANT) { ucs_assert(req != NULL); dc_req = ucs_derived_of(req, uct_dc_fc_request_t); sender_ep = (uintptr_t)dc_req->sender_ep; ib_iface = &iface->super.super.super; /* Note av initialization is copied from exp verbs */ av.stat_rate_sl = ib_iface->config.sl; /* (attr->static_rate << 4) | attr->sl */ av.fl_mlid = ib_iface->path_bits[0] & 0x7f; /* lid in dc_req is in BE already */ av.rlid = dc_req->lid | htons(ib_iface->path_bits[0]); av.dqp_dct = htonl(dc_req->dct_num); if (!iface->ud_common.config.compact_av) { av.dqp_dct |= UCT_IB_MLX5_EXTENDED_UD_AV; } uct_rc_mlx5_txqp_inline_post(&iface->super.super, IBV_EXP_QPT_DC_INI, txqp, txwq, MLX5_OPCODE_SEND, NULL, 0, op, sender_ep, 0, 0, 0, &av, uct_ib_mlx5_wqe_av_size(&av)); } else { ucs_assert(op == UCT_RC_EP_FC_FLAG_HARD_REQ); sender_ep = (uintptr_t)dc_ep; dc_mlx5_ep = ucs_derived_of(tl_ep, uct_dc_mlx5_ep_t); UCS_STATS_UPDATE_COUNTER(dc_ep->fc.stats, UCT_RC_FC_STAT_TX_HARD_REQ, 1); uct_rc_mlx5_txqp_inline_post(&iface->super.super, IBV_EXP_QPT_DC_INI, txqp, txwq, MLX5_OPCODE_SEND_IMM, NULL, 0, op, sender_ep, iface->super.rx.dct->dct_num, 0, 0, &dc_mlx5_ep->av, uct_ib_mlx5_wqe_av_size(&dc_mlx5_ep->av)); } return UCS_OK; }