ucs_status_t uct_rc_mlx5_ep_am_zcopy(uct_ep_h tl_ep, uint8_t id, const void *header, unsigned header_length, const void *payload, size_t length, uct_mem_h memh, uct_completion_t *comp) { uct_rc_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_mlx5_ep_t); uct_rc_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_iface_t); ucs_status_t status; UCT_CHECK_AM_ID(id); UCT_RC_CHECK_FC_WND(iface, &ep->super, id); UCT_CHECK_LENGTH(sizeof(struct mlx5_wqe_ctrl_seg) + sizeof(struct mlx5_wqe_data_seg) + sizeof(struct mlx5_wqe_inl_data_seg) + sizeof(uct_rc_hdr_t) + header_length, UCT_RC_MLX5_MAX_BB * MLX5_SEND_WQE_BB, "am zcopy"); UCT_CHECK_LENGTH(header_length + length + sizeof(uct_rc_hdr_t), ucs_derived_of(tl_ep->iface, uct_ib_iface_t)->config.seg_size, "am_zcopy"); UCT_CHECK_LENGTH(header_length + length, UCT_IB_MAX_MESSAGE_SIZE, "am_zcopy"); status = uct_rc_mlx5_ep_zcopy_post(ep, MLX5_OPCODE_SEND, payload, length, memh, id, header, header_length, 0, 0, 0, comp); if (ucs_likely(status >= 0)) { UCT_TL_EP_STAT_OP(&ep->super.super, AM, ZCOPY, header_length + length); UCT_RC_UPDATE_FC_WND(&ep->super); } return status; }
ucs_status_t uct_rc_verbs_ep_am_zcopy(uct_ep_h tl_ep, uint8_t id, const void *header, unsigned header_length, const uct_iov_t *iov, size_t iovcnt, uct_completion_t *comp) { uct_rc_verbs_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_verbs_iface_t); uct_rc_verbs_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_verbs_ep_t); uct_rc_iface_send_desc_t *desc = NULL; struct ibv_sge sge[UCT_IB_MAX_IOV]; /* First sge is reserved for the header */ struct ibv_send_wr wr; int send_flags; size_t sge_cnt; UCT_CHECK_IOV_SIZE(iovcnt, uct_ib_iface_get_max_iov(&iface->super.super) - 1, "uct_rc_verbs_ep_am_zcopy"); UCT_RC_CHECK_AM_ZCOPY(id, header_length, uct_iov_total_length(iov, iovcnt), iface->verbs_common.config.short_desc_size, iface->super.super.config.seg_size); UCT_RC_CHECK_RES(&iface->super, &ep->super); UCT_RC_CHECK_FC_WND(&iface->super, &ep->super, id); UCT_RC_IFACE_GET_TX_AM_ZCOPY_DESC(&iface->super, &iface->verbs_common.short_desc_mp, desc, id, header, header_length, comp, &send_flags); sge[0].length = sizeof(uct_rc_hdr_t) + header_length; sge_cnt = uct_ib_verbs_sge_fill_iov(sge + 1, iov, iovcnt); UCT_RC_VERBS_FILL_AM_ZCOPY_WR_IOV(wr, sge, (sge_cnt + 1), wr.opcode); UCT_TL_EP_STAT_OP(&ep->super.super, AM, ZCOPY, (header_length + uct_iov_total_length(iov, iovcnt))); uct_rc_verbs_ep_post_send_desc(ep, &wr, desc, send_flags); UCT_RC_UPDATE_FC_WND(&iface->super, &ep->super, id); return UCS_INPROGRESS; }
ssize_t uct_rc_mlx5_ep_am_bcopy(uct_ep_h tl_ep, uint8_t id, uct_pack_callback_t pack_cb, void *arg) { uct_rc_mlx5_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_mlx5_iface_t); uct_rc_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_mlx5_ep_t); uct_rc_iface_send_desc_t *desc; uct_rc_hdr_t *rch; size_t length; UCT_CHECK_AM_ID(id); UCT_RC_CHECK_RES(&iface->super, &ep->super); UCT_RC_CHECK_FC_WND(&iface->super, &ep->super, id); UCT_RC_IFACE_GET_TX_DESC(&iface->super, &iface->super.tx.mp, desc); desc->super.handler = (uct_rc_send_handler_t)ucs_mpool_put; rch = (void*)(desc + 1); rch->am_id = id; length = pack_cb(rch + 1, arg); uct_rc_mlx5_ep_bcopy_post(ep, MLX5_OPCODE_SEND|UCT_RC_MLX5_OPCODE_FLAG_RAW, sizeof(*rch) + length, 0, NULL, 0, 0, 0, 0, desc); UCT_TL_EP_STAT_OP(&ep->super.super, AM, BCOPY, length); UCT_RC_UPDATE_FC_WND(&ep->super); return length; }
ucs_status_t uct_rc_verbs_ep_am_short(uct_ep_h tl_ep, uint8_t id, uint64_t hdr, const void *buffer, unsigned length) { uct_rc_verbs_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_verbs_iface_t); uct_rc_verbs_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_verbs_ep_t); uct_rc_am_short_hdr_t am; UCT_RC_CHECK_AM_SHORT(id, length, iface->verbs_common.config.max_inline); UCT_RC_CHECK_RES(&iface->super, &ep->super); UCT_RC_CHECK_FC_WND(&iface->super, &ep->super, id); uct_rc_verbs_iface_fill_inl_am_sge(&iface->verbs_common, &am, id, hdr, buffer, length); UCT_TL_EP_STAT_OP(&ep->super.super, AM, SHORT, sizeof(hdr) + length); uct_rc_verbs_ep_post_send(iface, ep, &iface->inl_am_wr, IBV_SEND_INLINE); UCT_RC_UPDATE_FC_WND(&iface->super, &ep->super, id); return UCS_OK; }
ucs_status_t uct_rc_mlx5_ep_am_short(uct_ep_h tl_ep, uint8_t id, uint64_t hdr, const void *payload, unsigned length) { uct_rc_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_iface_t); uct_rc_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_mlx5_ep_t); ucs_status_t status; UCT_CHECK_AM_ID(id); UCT_RC_CHECK_FC_WND(iface, &ep->super, id); status = uct_rc_mlx5_ep_inline_post(ep, MLX5_OPCODE_SEND, payload, length, id, hdr, 0, 0); if (ucs_likely(status >= 0)) { UCT_TL_EP_STAT_OP(&ep->super.super, AM, SHORT, sizeof(hdr) + length); UCT_RC_UPDATE_FC_WND(&ep->super); } return status; }
ucs_status_t uct_dc_ep_check_fc(uct_dc_iface_t *iface, uct_dc_ep_t *ep) { ucs_status_t status; if (iface->super.config.fc_enabled) { UCT_RC_CHECK_FC_WND(&ep->fc, ep->super.stats); if ((ep->fc.fc_wnd == iface->super.config.fc_hard_thresh) && !(ep->fc.flags & UCT_DC_EP_FC_FLAG_WAIT_FOR_GRANT)) { status = uct_rc_fc_ctrl(&ep->super.super, UCT_RC_EP_FC_FLAG_HARD_REQ, NULL); if (status != UCS_OK) { return status; } ep->fc.flags |= UCT_DC_EP_FC_FLAG_WAIT_FOR_GRANT; } } else { /* Set fc_wnd to max, to send as much as possible without checks */ ep->fc.fc_wnd = INT16_MAX; } return UCS_OK; }
ssize_t uct_rc_mlx5_ep_am_bcopy(uct_ep_h tl_ep, uint8_t id, uct_pack_callback_t pack_cb, void *arg) { uct_rc_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_iface_t); uct_rc_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_mlx5_ep_t); uct_rc_iface_send_desc_t *desc; size_t length; UCT_CHECK_AM_ID(id); UCT_RC_CHECK_RES(iface, &ep->super); UCT_RC_CHECK_FC_WND(iface, &ep->super, id); UCT_RC_IFACE_GET_TX_AM_BCOPY_DESC(iface, &iface->tx.mp, desc, id, pack_cb, arg, &length); uct_rc_mlx5_txqp_bcopy_post(iface, &ep->super.txqp, &ep->tx.wq, MLX5_OPCODE_SEND|UCT_RC_MLX5_OPCODE_FLAG_RAW, sizeof(uct_rc_hdr_t) + length, 0, NULL, 0, 0, 0, 0, desc); UCT_TL_EP_STAT_OP(&ep->super.super, AM, BCOPY, length); UCT_RC_UPDATE_FC_WND(iface, &ep->super, id); return length; }
ucs_status_t uct_rc_mlx5_ep_am_short(uct_ep_h tl_ep, uint8_t id, uint64_t hdr, const void *payload, unsigned length) { uct_rc_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_iface_t); uct_rc_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_mlx5_ep_t); UCT_RC_MLX5_CHECK_AM_SHORT(id, length, 0); UCT_RC_CHECK_RES(iface, &ep->super); UCT_RC_CHECK_FC_WND(iface, &ep->super, id); uct_rc_mlx5_txqp_inline_post(iface, IBV_QPT_RC, &ep->super.txqp, &ep->tx.wq, MLX5_OPCODE_SEND, payload, length, id, hdr, 0, 0, NULL, 0); UCT_TL_EP_STAT_OP(&ep->super.super, AM, SHORT, sizeof(hdr) + length); UCT_RC_UPDATE_FC_WND(iface, &ep->super, id); return UCS_OK; }
ssize_t uct_rc_verbs_ep_am_bcopy(uct_ep_h tl_ep, uint8_t id, uct_pack_callback_t pack_cb, void *arg) { uct_rc_verbs_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_verbs_iface_t); uct_rc_verbs_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_verbs_ep_t); uct_rc_iface_send_desc_t *desc; struct ibv_send_wr wr; struct ibv_sge sge; size_t length; UCT_CHECK_AM_ID(id); UCT_RC_CHECK_RES(&iface->super, &ep->super); UCT_RC_CHECK_FC_WND(&iface->super, &ep->super, id); UCT_RC_IFACE_GET_TX_AM_BCOPY_DESC(&iface->super, &iface->super.tx.mp, desc, id, pack_cb, arg, &length); UCT_RC_VERBS_FILL_AM_BCOPY_WR(wr, sge, length, wr.opcode); UCT_TL_EP_STAT_OP(&ep->super.super, AM, BCOPY, length); uct_rc_verbs_ep_post_send_desc(ep, &wr, desc, 0); UCT_RC_UPDATE_FC_WND(&iface->super, &ep->super, id); return length; }
ucs_status_t uct_rc_mlx5_ep_am_zcopy(uct_ep_h tl_ep, uint8_t id, const void *header, unsigned header_length, const uct_iov_t *iov, size_t iovcnt, uct_completion_t *comp) { uct_rc_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_mlx5_ep_t); uct_rc_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_iface_t); ucs_status_t status; UCT_CHECK_IOV_SIZE(iovcnt, UCT_IB_MLX5_AM_ZCOPY_MAX_IOV, "uct_rc_mlx5_ep_am_zcopy"); UCT_RC_MLX5_CHECK_AM_ZCOPY(id, header_length, uct_iov_total_length(iov, iovcnt), iface->super.config.seg_size, 0); UCT_RC_CHECK_FC_WND(iface, &ep->super, id); status = uct_rc_mlx5_ep_zcopy_post(ep, MLX5_OPCODE_SEND, iov, iovcnt, id, header, header_length, 0, 0, 0, comp); if (ucs_likely(status >= 0)) { UCT_TL_EP_STAT_OP(&ep->super.super, AM, ZCOPY, header_length + uct_iov_total_length(iov, iovcnt)); UCT_RC_UPDATE_FC_WND(iface, &ep->super, id); } return status; }