ucs_status_t uct_rc_mlx5_ep_get_bcopy(uct_ep_h tl_ep, uct_unpack_callback_t unpack_cb, void *arg, size_t length, uint64_t remote_addr, uct_rkey_t rkey, uct_completion_t *comp) { uct_rc_mlx5_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_mlx5_iface_t); uct_rc_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_mlx5_ep_t); uct_rc_iface_send_desc_t *desc; ucs_status_t status; UCT_CHECK_LENGTH(length, iface->super.super.config.seg_size, "get_bcopy"); UCT_RC_MLX5_CHECK_RES(iface, ep); UCT_RC_IFACE_GET_TX_DESC(&iface->super, iface->super.tx.mp, desc); desc->super.handler = (comp == NULL) ? uct_rc_ep_get_bcopy_handler_no_completion : uct_rc_ep_get_bcopy_handler; desc->super.unpack_arg = arg; desc->super.user_comp = comp; desc->super.length = length; desc->unpack_cb = unpack_cb; status = uct_rc_mlx5_ep_bcopy_post(ep, MLX5_OPCODE_RDMA_READ, length, 0, NULL, 0, remote_addr, rkey, MLX5_WQE_CTRL_CQ_UPDATE, desc, UCS_INPROGRESS); UCT_TL_EP_STAT_OP_IF_SUCCESS(status, &ep->super.super, GET, BCOPY, length); return status; }
ucs_status_t uct_rc_mlx5_ep_am_bcopy(uct_ep_h tl_ep, uint8_t id, uct_pack_callback_t pack_cb, void *arg, size_t length) { uct_rc_mlx5_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_mlx5_iface_t); uct_rc_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_mlx5_ep_t); uct_rc_iface_send_desc_t *desc; ucs_status_t status; uct_rc_hdr_t *rch; UCT_CHECK_AM_ID(id); UCT_CHECK_LENGTH(sizeof(*rch) + length, iface->super.super.config.seg_size, "am_bcopy"); UCT_RC_MLX5_CHECK_RES(iface, ep); UCT_RC_IFACE_GET_TX_DESC(&iface->super, iface->super.tx.mp, desc); desc->super.handler = (uct_rc_send_handler_t)ucs_mpool_put; rch = (void*)(desc + 1); rch->am_id = id; pack_cb(rch + 1, arg, length); status = uct_rc_mlx5_ep_bcopy_post(ep, MLX5_OPCODE_SEND|UCT_RC_MLX5_OPCODE_FLAG_RAW, sizeof(*rch) + length, 0, NULL, 0, 0, 0, 0, desc, UCS_OK); UCT_TL_EP_STAT_OP_IF_SUCCESS(status, &ep->super.super, AM, BCOPY, length); return status; }
ucs_status_t uct_rc_verbs_ep_get_bcopy(uct_ep_h tl_ep, uct_unpack_callback_t unpack_cb, void *arg, size_t length, uint64_t remote_addr, uct_rkey_t rkey, uct_completion_t *comp) { uct_rc_verbs_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_verbs_iface_t); uct_rc_verbs_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_verbs_ep_t); uct_rc_iface_send_desc_t *desc; struct ibv_send_wr wr; struct ibv_sge sge; UCT_CHECK_LENGTH(length, iface->super.super.config.seg_size, "get_bcopy"); UCT_RC_VERBS_CHECK_RES(iface, ep); UCT_RC_IFACE_GET_TX_DESC(&iface->super, iface->super.tx.mp, desc); ucs_assert(length <= iface->super.super.config.seg_size); desc->super.handler = (comp == NULL) ? uct_rc_ep_get_bcopy_handler_no_completion : uct_rc_ep_get_bcopy_handler; desc->super.unpack_arg = arg; desc->super.user_comp = comp; desc->super.length = length; desc->unpack_cb = unpack_cb; uct_rc_verbs_fill_rdma_wr(&wr, IBV_WR_RDMA_READ, &sge, length, remote_addr, rkey); UCT_TL_EP_STAT_OP(&ep->super.super, GET, BCOPY, length); uct_rc_verbs_ep_post_send_desc(ep, &wr, desc, IBV_SEND_SIGNALED); return UCS_INPROGRESS; }
ucs_status_t uct_rc_verbs_ep_am_bcopy(uct_ep_h tl_ep, uint8_t id, uct_pack_callback_t pack_cb, void *arg, size_t length) { uct_rc_verbs_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_verbs_iface_t); uct_rc_verbs_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_verbs_ep_t); uct_rc_iface_send_desc_t *desc; struct ibv_send_wr wr; struct ibv_sge sge; uct_rc_hdr_t *rch; UCT_CHECK_AM_ID(id); UCT_CHECK_LENGTH(sizeof(*rch) + length, iface->super.super.config.seg_size, "am_bcopy"); UCT_RC_VERBS_CHECK_RES(iface, ep); UCT_RC_IFACE_GET_TX_DESC(&iface->super, iface->super.tx.mp, desc); desc->super.handler = (uct_rc_send_handler_t)ucs_mpool_put; rch = (void*)(desc + 1); rch->am_id = id; pack_cb(rch + 1, arg, length); wr.sg_list = &sge; wr.num_sge = 1; wr.opcode = IBV_WR_SEND; sge.length = sizeof(*rch) + length; UCT_TL_EP_STAT_OP(&ep->super.super, AM, BCOPY, length); uct_rc_verbs_ep_post_send_desc(ep, &wr, desc, 0); return UCS_OK; }
static inline ucs_status_t uct_rc_verbs_ext_atomic(uct_rc_verbs_ep_t *ep, int opcode, void *result, uint32_t length, uint64_t compare_mask, uint64_t compare_add, uint64_t swap, uint64_t remote_addr, uct_rkey_t rkey, uct_completion_t *comp) { uct_rc_verbs_iface_t *iface = ucs_derived_of(ep->super.super.super.iface, uct_rc_verbs_iface_t); uct_rc_iface_send_desc_t *desc; UCT_CHECK_PARAM(comp != NULL, "completion must be non-NULL"); UCT_RC_VERBS_CHECK_RES(iface, ep); UCT_RC_IFACE_GET_TX_DESC(&iface->super, iface->short_desc_mp, desc); switch (length) { case sizeof(uint32_t): desc->super.handler = iface->config.atomic32_handler; break; case sizeof(uint64_t): desc->super.handler = iface->config.atomic64_handler; break; } desc->super.result = result; desc->super.user_comp = comp; return uct_rc_verbs_ext_atomic_post(ep, opcode, length, compare_mask, compare_add, swap, remote_addr, rkey, desc, IBV_EXP_SEND_SIGNALED, UCS_INPROGRESS); }
ssize_t uct_rc_mlx5_ep_am_bcopy(uct_ep_h tl_ep, uint8_t id, uct_pack_callback_t pack_cb, void *arg) { uct_rc_mlx5_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_mlx5_iface_t); uct_rc_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_mlx5_ep_t); uct_rc_iface_send_desc_t *desc; uct_rc_hdr_t *rch; size_t length; UCT_CHECK_AM_ID(id); UCT_RC_CHECK_RES(&iface->super, &ep->super); UCT_RC_CHECK_FC_WND(&iface->super, &ep->super, id); UCT_RC_IFACE_GET_TX_DESC(&iface->super, &iface->super.tx.mp, desc); desc->super.handler = (uct_rc_send_handler_t)ucs_mpool_put; rch = (void*)(desc + 1); rch->am_id = id; length = pack_cb(rch + 1, arg); uct_rc_mlx5_ep_bcopy_post(ep, MLX5_OPCODE_SEND|UCT_RC_MLX5_OPCODE_FLAG_RAW, sizeof(*rch) + length, 0, NULL, 0, 0, 0, 0, desc); UCT_TL_EP_STAT_OP(&ep->super.super, AM, BCOPY, length); UCT_RC_UPDATE_FC_WND(&ep->super); return length; }
ucs_status_t uct_rc_verbs_ep_am_zcopy(uct_ep_h tl_ep, uint8_t id, const void *header, unsigned header_length, const void *payload, size_t length, uct_mem_h memh, uct_completion_t *comp) { uct_rc_verbs_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_verbs_iface_t); uct_rc_verbs_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_verbs_ep_t); struct ibv_mr *mr = memh; uct_rc_iface_send_desc_t *desc; struct ibv_send_wr wr; struct ibv_sge sge[2]; uct_rc_hdr_t *rch; int send_flags; UCT_CHECK_AM_ID(id); UCT_CHECK_LENGTH(sizeof(*rch) + header_length, iface->config.short_desc_size, "am_zcopy header"); UCT_CHECK_LENGTH(header_length + length, iface->super.super.config.seg_size, "am_zcopy payload"); UCT_RC_VERBS_CHECK_RES(iface, ep); UCT_RC_IFACE_GET_TX_DESC(&iface->super, iface->short_desc_mp, desc); if (comp == NULL) { desc->super.handler = (uct_rc_send_handler_t)ucs_mpool_put; send_flags = 0; } else { desc->super.handler = uct_rc_verbs_ep_am_zcopy_handler; desc->super.user_comp = comp; send_flags = IBV_SEND_SIGNALED; } /* Header buffer: active message ID + user header */ rch = (void*)(desc + 1); rch->am_id = id; memcpy(rch + 1, header, header_length); wr.sg_list = sge; wr.opcode = IBV_WR_SEND; sge[0].length = sizeof(*rch) + header_length; if (ucs_unlikely(length == 0)) { wr.num_sge = 1; } else { wr.num_sge = 2; sge[1].addr = (uintptr_t)payload; sge[1].length = length; sge[1].lkey = (mr == UCT_INVALID_MEM_HANDLE) ? 0 : mr->lkey; } UCT_TL_EP_STAT_OP(&ep->super.super, AM, ZCOPY, header_length + length); uct_rc_verbs_ep_post_send_desc(ep, &wr, desc, send_flags); return UCS_INPROGRESS; }
static UCS_F_ALWAYS_INLINE ucs_status_t uct_rc_mlx5_ep_atomic_add(uct_ep_h tl_ep, int opcode, unsigned length, uint64_t add, uint64_t remote_addr, uct_rkey_t rkey) { uct_rc_mlx5_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_mlx5_iface_t); uct_rc_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_mlx5_ep_t); uct_rc_iface_send_desc_t *desc; UCT_RC_MLX5_CHECK_RES(iface, ep); UCT_RC_IFACE_GET_TX_DESC(&iface->super, iface->tx.atomic_desc_mp, desc); desc->super.handler = (uct_rc_send_handler_t)ucs_mpool_put; return uct_rc_mlx5_ep_atomic_post(ep, opcode, desc, length, remote_addr, rkey, 0, 0, add, 0, UCS_OK); }
ucs_status_t uct_rc_verbs_ep_atomic_add64(uct_ep_h tl_ep, uint64_t add, uint64_t remote_addr, uct_rkey_t rkey) { uct_rc_verbs_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_verbs_iface_t); uct_rc_verbs_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_verbs_ep_t); uct_rc_iface_send_desc_t *desc; /* TODO don't allocate descriptor - have dummy buffer */ UCT_RC_VERBS_CHECK_RES(iface, ep); UCT_RC_IFACE_GET_TX_DESC(&iface->super, iface->short_desc_mp, desc); desc->super.handler = (uct_rc_send_handler_t)ucs_mpool_put; uct_rc_verbs_ep_atomic_post(ucs_derived_of(tl_ep, uct_rc_verbs_ep_t), IBV_WR_ATOMIC_FETCH_AND_ADD, add, 0, remote_addr, rkey, desc, IBV_SEND_SIGNALED); return UCS_OK; }
ssize_t uct_rc_mlx5_ep_put_bcopy(uct_ep_h tl_ep, uct_pack_callback_t pack_cb, void *arg, uint64_t remote_addr, uct_rkey_t rkey) { uct_rc_mlx5_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_mlx5_iface_t); uct_rc_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_mlx5_ep_t); uct_rc_iface_send_desc_t *desc; size_t length; UCT_RC_CHECK_RES(&iface->super, &ep->super); UCT_RC_IFACE_GET_TX_DESC(&iface->super, &iface->super.tx.mp, desc); desc->super.handler = (uct_rc_send_handler_t)ucs_mpool_put; length = pack_cb(desc + 1, arg); uct_rc_mlx5_ep_bcopy_post(ep, MLX5_OPCODE_RDMA_WRITE, length, 0, NULL, 0, remote_addr, rkey, MLX5_WQE_CTRL_CQ_UPDATE, desc); UCT_TL_EP_STAT_OP(&ep->super.super, PUT, BCOPY, length); return length; }
ucs_status_t uct_rc_verbs_ep_atomic_add32(uct_ep_h tl_ep, uint32_t add, uint64_t remote_addr, uct_rkey_t rkey) { #if HAVE_IB_EXT_ATOMICS uct_rc_verbs_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_verbs_iface_t); uct_rc_verbs_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_verbs_ep_t); uct_rc_iface_send_desc_t *desc; UCT_RC_VERBS_CHECK_RES(iface, ep); UCT_RC_IFACE_GET_TX_DESC(&iface->super, iface->short_desc_mp, desc); /* TODO don't allocate descriptor - have dummy buffer */ desc->super.handler = (uct_rc_send_handler_t)ucs_mpool_put; return uct_rc_verbs_ext_atomic_post(ep, IBV_EXP_WR_EXT_MASKED_ATOMIC_FETCH_AND_ADD, sizeof(uint32_t), 0, add, 0, remote_addr, rkey, desc, IBV_EXP_SEND_SIGNALED, UCS_OK); #else return UCS_ERR_UNSUPPORTED; #endif }
ucs_status_t uct_rc_mlx5_ep_put_bcopy(uct_ep_h tl_ep, uct_pack_callback_t pack_cb, void *arg, size_t length, uint64_t remote_addr, uct_rkey_t rkey) { uct_rc_mlx5_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_mlx5_iface_t); uct_rc_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_mlx5_ep_t); uct_rc_iface_send_desc_t *desc; ucs_status_t status; UCT_CHECK_LENGTH(length, iface->super.super.config.seg_size, "put_bcopy"); UCT_RC_MLX5_CHECK_RES(iface, ep); UCT_RC_IFACE_GET_TX_DESC(&iface->super, iface->super.tx.mp, desc); desc->super.handler = (uct_rc_send_handler_t)ucs_mpool_put; pack_cb(desc + 1, arg, length); status = uct_rc_mlx5_ep_bcopy_post(ep, MLX5_OPCODE_RDMA_WRITE, length, 0, NULL, 0, remote_addr, rkey, MLX5_WQE_CTRL_CQ_UPDATE, desc, UCS_OK); UCT_TL_EP_STAT_OP_IF_SUCCESS(status, &ep->super.super, PUT, BCOPY, length); return status; }
static UCS_F_ALWAYS_INLINE ucs_status_t uct_rc_mlx5_ep_atomic(uct_rc_mlx5_ep_t *ep, int opcode, void *result, unsigned length, uct_rc_send_handler_t handler, uint64_t remote_addr, uct_rkey_t rkey, uint64_t compare_mask, uint64_t compare, uint64_t swap_add, uct_completion_t *comp) { uct_rc_mlx5_iface_t *iface = ucs_derived_of(ep->super.super.super.iface, uct_rc_mlx5_iface_t); uct_rc_iface_send_desc_t *desc; UCT_CHECK_PARAM(comp != NULL, "completion must be non-NULL"); UCT_RC_MLX5_CHECK_RES(iface, ep); UCT_RC_IFACE_GET_TX_DESC(&iface->super, iface->tx.atomic_desc_mp, desc); desc->super.handler = handler; desc->super.result = result; desc->super.user_comp = comp; return uct_rc_mlx5_ep_atomic_post(ep, opcode, desc, length, remote_addr, rkey, compare_mask, compare, swap_add, MLX5_WQE_CTRL_CQ_UPDATE, UCS_INPROGRESS); }
static UCS_F_ALWAYS_INLINE ucs_status_t uct_rc_verbs_ep_atomic(uct_rc_verbs_ep_t *ep, int opcode, void *result, uint64_t compare_add, uint64_t swap, uint64_t remote_addr, uct_rkey_t rkey, uct_completion_t *comp) { uct_rc_verbs_iface_t *iface = ucs_derived_of(ep->super.super.super.iface, uct_rc_verbs_iface_t); uct_rc_iface_send_desc_t *desc; UCT_CHECK_PARAM(comp != NULL, "completion must be non-NULL"); UCT_RC_VERBS_CHECK_RES(iface, ep); UCT_RC_IFACE_GET_TX_DESC(&iface->super, iface->short_desc_mp, desc); desc->super.handler = iface->config.atomic64_handler; desc->super.result = result; desc->super.user_comp = comp; uct_rc_verbs_ep_atomic_post(ep, opcode, compare_add, swap, remote_addr, rkey, desc, IBV_SEND_SIGNALED); return UCS_INPROGRESS; }
ucs_status_t uct_rc_verbs_ep_put_bcopy(uct_ep_h tl_ep, uct_pack_callback_t pack_cb, void *arg, size_t length, uint64_t remote_addr, uct_rkey_t rkey) { uct_rc_verbs_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_verbs_iface_t); uct_rc_verbs_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_verbs_ep_t); uct_rc_iface_send_desc_t *desc; struct ibv_send_wr wr; struct ibv_sge sge; UCT_CHECK_LENGTH(length, iface->super.super.config.seg_size, "put_bcopy"); UCT_RC_VERBS_ZERO_LENGTH_POST(length); UCT_RC_VERBS_CHECK_RES(iface, ep); UCT_RC_IFACE_GET_TX_DESC(&iface->super, iface->super.tx.mp, desc); desc->super.handler = (uct_rc_send_handler_t)ucs_mpool_put; pack_cb(desc + 1, arg, length); uct_rc_verbs_fill_rdma_wr(&wr, IBV_WR_RDMA_WRITE, &sge, length, remote_addr, rkey); UCT_TL_EP_STAT_OP(&ep->super.super, PUT, BCOPY, length); uct_rc_verbs_ep_post_send_desc(ep, &wr, desc, IBV_SEND_SIGNALED); return UCS_OK; }