UCS_CLASS_INIT_FUNC(uct_rc_verbs_ep_t, const uct_ep_params_t *params) { uct_rc_verbs_iface_t *iface = ucs_derived_of(params->iface, uct_rc_verbs_iface_t); UCS_CLASS_CALL_SUPER_INIT(uct_rc_ep_t, &iface->super); uct_rc_txqp_available_set(&self->super.txqp, iface->config.tx_max_wr); uct_rc_verbs_txcnt_init(&self->txcnt); uct_ib_fence_info_init(&self->fi); return UCS_OK; }
static UCS_CLASS_INIT_FUNC(uct_rc_verbs_ep_t, uct_iface_h tl_iface) { uct_rc_verbs_iface_t *iface = ucs_derived_of(tl_iface, uct_rc_verbs_iface_t); UCS_CLASS_CALL_SUPER_INIT(uct_rc_ep_t, &iface->super); uct_rc_txqp_available_set(&self->super.txqp, iface->config.tx_max_wr); uct_rc_verbs_txcnt_init(&self->txcnt); uct_worker_progress_register(iface->super.super.super.worker, uct_rc_verbs_iface_progress, iface); return UCS_OK; }
void uct_dc_ep_set_failed(ucs_class_t *ep_cls, uct_dc_iface_t *iface, uint32_t qp_num) { uint8_t dci = uct_dc_iface_dci_find(iface, qp_num); uct_dc_ep_t *ep = iface->tx.dcis[dci].ep; if (!ep) { return; } uct_rc_txqp_purge_outstanding(&iface->tx.dcis[dci].txqp, UCS_ERR_ENDPOINT_TIMEOUT, 0); uct_set_ep_failed(ep_cls, &ep->super.super, &iface->super.super.super.super); if (UCS_OK != uct_dc_iface_dci_reconnect(iface, &iface->tx.dcis[dci].txqp)) { ucs_fatal("Unsuccessful reconnect of DC QP #%u", qp_num); } uct_rc_txqp_available_set(&iface->tx.dcis[dci].txqp, iface->super.config.tx_qp_len); }
static UCS_F_ALWAYS_INLINE void uct_dc_mlx5_poll_tx(uct_dc_mlx5_iface_t *iface) { uint8_t dci; struct mlx5_cqe64 *cqe; uint32_t qp_num; uint16_t hw_ci; UCT_DC_MLX5_TXQP_DECL(txqp, txwq); cqe = uct_ib_mlx5_poll_cq(&iface->super.super.super, &iface->mlx5_common.tx.cq); if (cqe == NULL) { return; } UCS_STATS_UPDATE_COUNTER(iface->super.super.stats, UCT_RC_IFACE_STAT_TX_COMPLETION, 1); ucs_memory_cpu_load_fence(); qp_num = ntohl(cqe->sop_drop_qpn) & UCS_MASK(UCT_IB_QPN_ORDER); dci = uct_dc_iface_dci_find(&iface->super, qp_num); txqp = &iface->super.tx.dcis[dci].txqp; txwq = &iface->dci_wqs[dci]; hw_ci = ntohs(cqe->wqe_counter); ucs_trace_poll("dc_mlx5 iface %p tx_cqe: dci[%d] qpn 0x%x txqp %p hw_ci %d", iface, dci, qp_num, txqp, hw_ci); uct_rc_txqp_available_set(txqp, uct_ib_mlx5_txwq_update_bb(txwq, hw_ci)); uct_dc_iface_dci_put(&iface->super, dci); uct_rc_mlx5_txqp_process_tx_cqe(txqp, cqe, hw_ci); iface->super.super.tx.cq_available++; if (uct_dc_iface_dci_can_alloc(&iface->super)) { ucs_arbiter_dispatch(uct_dc_iface_dci_waitq(&iface->super), 1, uct_dc_iface_dci_do_pending_wait, NULL); } ucs_arbiter_dispatch(uct_dc_iface_tx_waitq(&iface->super), 1, uct_dc_iface_dci_do_pending_tx, NULL); }
static UCS_CLASS_INIT_FUNC(uct_rc_mlx5_ep_t, uct_iface_h tl_iface) { uct_rc_mlx5_iface_t *iface = ucs_derived_of(tl_iface, uct_rc_mlx5_iface_t); ucs_status_t status; UCS_CLASS_CALL_SUPER_INIT(uct_rc_ep_t, &iface->super); status = uct_ib_mlx5_txwq_init(iface->super.super.super.worker, &self->tx.wq, self->super.txqp.qp); if (status != UCS_OK) { ucs_error("Failed to get mlx5 QP information"); return status; } self->qp_num = self->super.txqp.qp->qp_num; self->tx.wq.bb_max = ucs_min(self->tx.wq.bb_max, iface->tx.bb_max); uct_rc_txqp_available_set(&self->super.txqp, self->tx.wq.bb_max); uct_worker_progress_register(iface->super.super.super.worker, uct_rc_mlx5_iface_progress, iface); return UCS_OK; }
static ucs_status_t uct_dc_mlx5_iface_init_dcis(uct_dc_mlx5_iface_t *iface) { ucs_status_t status; uint16_t bb_max; int i; bb_max = 0; for (i = 0; i < iface->super.tx.ndci; i++) { status = uct_ib_mlx5_get_txwq(iface->super.super.super.super.worker, iface->super.tx.dcis[i].txqp.qp, &iface->dci_wqs[i]); if (status != UCS_OK) { return status; } bb_max = iface->dci_wqs[i].bb_max; uct_rc_txqp_available_set(&iface->super.tx.dcis[i].txqp, bb_max); } iface->super.super.config.tx_qp_len = bb_max; return UCS_OK; }
static UCS_F_ALWAYS_INLINE void uct_dc_mlx5_poll_tx(uct_dc_mlx5_iface_t *iface) { uint8_t dci; struct mlx5_cqe64 *cqe; uint32_t qp_num; uint16_t hw_ci; UCT_DC_MLX5_TXQP_DECL(txqp, txwq); cqe = uct_ib_mlx5_get_cqe(&iface->super.super.super, &iface->mlx5_common.tx.cq, iface->mlx5_common.tx.cq.cqe_size_log); if (cqe == NULL) { return; } UCS_STATS_UPDATE_COUNTER(iface->super.super.stats, UCT_RC_IFACE_STAT_TX_COMPLETION, 1); ucs_memory_cpu_load_fence(); ucs_assertv(!(cqe->op_own & (MLX5_INLINE_SCATTER_32|MLX5_INLINE_SCATTER_64)), "tx inline scatter not supported"); qp_num = ntohl(cqe->sop_drop_qpn) & UCS_MASK(UCT_IB_QPN_ORDER); dci = uct_dc_iface_dci_find(&iface->super, qp_num); txqp = &iface->super.tx.dcis[dci].txqp; txwq = &iface->dci_wqs[dci]; hw_ci = ntohs(cqe->wqe_counter); uct_rc_txqp_available_set(txqp, uct_ib_mlx5_txwq_update_bb(txwq, hw_ci)); uct_rc_txqp_completion(txqp, hw_ci); iface->super.super.tx.cq_available++; uct_dc_iface_dci_put(&iface->super, dci); if (uct_dc_iface_dci_can_alloc(&iface->super)) { ucs_arbiter_dispatch(&iface->super.super.tx.arbiter, 1, uct_dc_iface_dci_do_pending_wait, NULL); } ucs_arbiter_dispatch(&iface->super.tx.dci_arbiter, 1, uct_dc_iface_dci_do_pending_tx, NULL); }