ucs_status_t uct_rc_mlx5_ep_get_bcopy(uct_ep_h tl_ep,
uct_unpack_callback_t unpack_cb,
void *arg, size_t length,
uint64_t remote_addr, uct_rkey_t rkey,
uct_completion_t *comp)
{
uct_rc_mlx5_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_mlx5_iface_t);
uct_rc_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_mlx5_ep_t);
uct_rc_iface_send_desc_t *desc;
ucs_status_t status;
UCT_CHECK_LENGTH(length, iface->super.super.config.seg_size, "get_bcopy");
UCT_RC_MLX5_CHECK_RES(iface, ep);
UCT_RC_IFACE_GET_TX_DESC(&iface->super, iface->super.tx.mp, desc);
desc->super.handler = (comp == NULL) ?
uct_rc_ep_get_bcopy_handler_no_completion :
uct_rc_ep_get_bcopy_handler;
desc->super.unpack_arg = arg;
desc->super.user_comp = comp;
desc->super.length = length;
desc->unpack_cb = unpack_cb;
status = uct_rc_mlx5_ep_bcopy_post(ep, MLX5_OPCODE_RDMA_READ, length, 0, NULL,
0, remote_addr, rkey, MLX5_WQE_CTRL_CQ_UPDATE,
desc, UCS_INPROGRESS);
UCT_TL_EP_STAT_OP_IF_SUCCESS(status, &ep->super.super, GET, BCOPY, length);
return status;
}
ucs_status_t uct_rc_mlx5_ep_am_bcopy(uct_ep_h tl_ep, uint8_t id,
uct_pack_callback_t pack_cb, void *arg,
size_t length)
{
uct_rc_mlx5_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_mlx5_iface_t);
uct_rc_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_mlx5_ep_t);
uct_rc_iface_send_desc_t *desc;
ucs_status_t status;
uct_rc_hdr_t *rch;
UCT_CHECK_AM_ID(id);
UCT_CHECK_LENGTH(sizeof(*rch) + length, iface->super.super.config.seg_size,
"am_bcopy");
UCT_RC_MLX5_CHECK_RES(iface, ep);
UCT_RC_IFACE_GET_TX_DESC(&iface->super, iface->super.tx.mp, desc);
desc->super.handler = (uct_rc_send_handler_t)ucs_mpool_put;
rch = (void*)(desc + 1);
rch->am_id = id;
pack_cb(rch + 1, arg, length);
status = uct_rc_mlx5_ep_bcopy_post(ep, MLX5_OPCODE_SEND|UCT_RC_MLX5_OPCODE_FLAG_RAW,
sizeof(*rch) + length, 0, NULL, 0, 0, 0, 0, desc,
UCS_OK);
UCT_TL_EP_STAT_OP_IF_SUCCESS(status, &ep->super.super, AM, BCOPY, length);
return status;
}
ucs_status_t uct_rc_verbs_ep_get_bcopy(uct_ep_h tl_ep,
uct_unpack_callback_t unpack_cb,
void *arg, size_t length,
uint64_t remote_addr, uct_rkey_t rkey,
uct_completion_t *comp)
{
uct_rc_verbs_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_verbs_iface_t);
uct_rc_verbs_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_verbs_ep_t);
uct_rc_iface_send_desc_t *desc;
struct ibv_send_wr wr;
struct ibv_sge sge;
UCT_CHECK_LENGTH(length, iface->super.super.config.seg_size, "get_bcopy");
UCT_RC_VERBS_CHECK_RES(iface, ep);
UCT_RC_IFACE_GET_TX_DESC(&iface->super, iface->super.tx.mp, desc);
ucs_assert(length <= iface->super.super.config.seg_size);
desc->super.handler = (comp == NULL) ?
uct_rc_ep_get_bcopy_handler_no_completion :
uct_rc_ep_get_bcopy_handler;
desc->super.unpack_arg = arg;
desc->super.user_comp = comp;
desc->super.length = length;
desc->unpack_cb = unpack_cb;
uct_rc_verbs_fill_rdma_wr(&wr, IBV_WR_RDMA_READ, &sge, length, remote_addr,
rkey);
UCT_TL_EP_STAT_OP(&ep->super.super, GET, BCOPY, length);
uct_rc_verbs_ep_post_send_desc(ep, &wr, desc, IBV_SEND_SIGNALED);
return UCS_INPROGRESS;
}
ucs_status_t uct_rc_verbs_ep_am_bcopy(uct_ep_h tl_ep, uint8_t id,
uct_pack_callback_t pack_cb, void *arg,
size_t length)
{
uct_rc_verbs_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_verbs_iface_t);
uct_rc_verbs_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_verbs_ep_t);
uct_rc_iface_send_desc_t *desc;
struct ibv_send_wr wr;
struct ibv_sge sge;
uct_rc_hdr_t *rch;
UCT_CHECK_AM_ID(id);
UCT_CHECK_LENGTH(sizeof(*rch) + length, iface->super.super.config.seg_size,
"am_bcopy");
UCT_RC_VERBS_CHECK_RES(iface, ep);
UCT_RC_IFACE_GET_TX_DESC(&iface->super, iface->super.tx.mp, desc);
desc->super.handler = (uct_rc_send_handler_t)ucs_mpool_put;
rch = (void*)(desc + 1);
rch->am_id = id;
pack_cb(rch + 1, arg, length);
wr.sg_list = &sge;
wr.num_sge = 1;
wr.opcode = IBV_WR_SEND;
sge.length = sizeof(*rch) + length;
UCT_TL_EP_STAT_OP(&ep->super.super, AM, BCOPY, length);
uct_rc_verbs_ep_post_send_desc(ep, &wr, desc, 0);
return UCS_OK;
}
static inline ucs_status_t
uct_rc_verbs_ext_atomic(uct_rc_verbs_ep_t *ep, int opcode, void *result,
uint32_t length, uint64_t compare_mask,
uint64_t compare_add, uint64_t swap, uint64_t remote_addr,
uct_rkey_t rkey, uct_completion_t *comp)
{
uct_rc_verbs_iface_t *iface = ucs_derived_of(ep->super.super.super.iface,
uct_rc_verbs_iface_t);
uct_rc_iface_send_desc_t *desc;
UCT_CHECK_PARAM(comp != NULL, "completion must be non-NULL");
UCT_RC_VERBS_CHECK_RES(iface, ep);
UCT_RC_IFACE_GET_TX_DESC(&iface->super, iface->short_desc_mp, desc);
switch (length) {
case sizeof(uint32_t):
desc->super.handler = iface->config.atomic32_handler;
break;
case sizeof(uint64_t):
desc->super.handler = iface->config.atomic64_handler;
break;
}
desc->super.result = result;
desc->super.user_comp = comp;
return uct_rc_verbs_ext_atomic_post(ep, opcode, length, compare_mask, compare_add,
swap, remote_addr, rkey, desc,
IBV_EXP_SEND_SIGNALED, UCS_INPROGRESS);
}
ssize_t uct_rc_mlx5_ep_am_bcopy(uct_ep_h tl_ep, uint8_t id,
uct_pack_callback_t pack_cb, void *arg)
{
uct_rc_mlx5_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_mlx5_iface_t);
uct_rc_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_mlx5_ep_t);
uct_rc_iface_send_desc_t *desc;
uct_rc_hdr_t *rch;
size_t length;
UCT_CHECK_AM_ID(id);
UCT_RC_CHECK_RES(&iface->super, &ep->super);
UCT_RC_CHECK_FC_WND(&iface->super, &ep->super, id);
UCT_RC_IFACE_GET_TX_DESC(&iface->super, &iface->super.tx.mp, desc);
desc->super.handler = (uct_rc_send_handler_t)ucs_mpool_put;
rch = (void*)(desc + 1);
rch->am_id = id;
length = pack_cb(rch + 1, arg);
uct_rc_mlx5_ep_bcopy_post(ep, MLX5_OPCODE_SEND|UCT_RC_MLX5_OPCODE_FLAG_RAW,
sizeof(*rch) + length, 0, NULL, 0, 0, 0, 0, desc);
UCT_TL_EP_STAT_OP(&ep->super.super, AM, BCOPY, length);
UCT_RC_UPDATE_FC_WND(&ep->super);
return length;
}
ucs_status_t uct_rc_verbs_ep_am_zcopy(uct_ep_h tl_ep, uint8_t id, const void *header,
unsigned header_length, const void *payload,
size_t length, uct_mem_h memh,
uct_completion_t *comp)
{
uct_rc_verbs_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_verbs_iface_t);
uct_rc_verbs_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_verbs_ep_t);
struct ibv_mr *mr = memh;
uct_rc_iface_send_desc_t *desc;
struct ibv_send_wr wr;
struct ibv_sge sge[2];
uct_rc_hdr_t *rch;
int send_flags;
UCT_CHECK_AM_ID(id);
UCT_CHECK_LENGTH(sizeof(*rch) + header_length, iface->config.short_desc_size,
"am_zcopy header");
UCT_CHECK_LENGTH(header_length + length, iface->super.super.config.seg_size,
"am_zcopy payload");
UCT_RC_VERBS_CHECK_RES(iface, ep);
UCT_RC_IFACE_GET_TX_DESC(&iface->super, iface->short_desc_mp, desc);
if (comp == NULL) {
desc->super.handler = (uct_rc_send_handler_t)ucs_mpool_put;
send_flags = 0;
} else {
desc->super.handler = uct_rc_verbs_ep_am_zcopy_handler;
desc->super.user_comp = comp;
send_flags = IBV_SEND_SIGNALED;
}
/* Header buffer: active message ID + user header */
rch = (void*)(desc + 1);
rch->am_id = id;
memcpy(rch + 1, header, header_length);
wr.sg_list = sge;
wr.opcode = IBV_WR_SEND;
sge[0].length = sizeof(*rch) + header_length;
if (ucs_unlikely(length == 0)) {
wr.num_sge = 1;
} else {
wr.num_sge = 2;
sge[1].addr = (uintptr_t)payload;
sge[1].length = length;
sge[1].lkey = (mr == UCT_INVALID_MEM_HANDLE) ? 0 : mr->lkey;
}
UCT_TL_EP_STAT_OP(&ep->super.super, AM, ZCOPY, header_length + length);
uct_rc_verbs_ep_post_send_desc(ep, &wr, desc, send_flags);
return UCS_INPROGRESS;
}
static UCS_F_ALWAYS_INLINE ucs_status_t
uct_rc_mlx5_ep_atomic_add(uct_ep_h tl_ep, int opcode, unsigned length,
uint64_t add, uint64_t remote_addr, uct_rkey_t rkey)
{
uct_rc_mlx5_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_mlx5_iface_t);
uct_rc_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_mlx5_ep_t);
uct_rc_iface_send_desc_t *desc;
UCT_RC_MLX5_CHECK_RES(iface, ep);
UCT_RC_IFACE_GET_TX_DESC(&iface->super, iface->tx.atomic_desc_mp, desc);
desc->super.handler = (uct_rc_send_handler_t)ucs_mpool_put;
return uct_rc_mlx5_ep_atomic_post(ep, opcode, desc, length, remote_addr, rkey,
0, 0, add, 0, UCS_OK);
}
ucs_status_t uct_rc_verbs_ep_atomic_add64(uct_ep_h tl_ep, uint64_t add,
uint64_t remote_addr, uct_rkey_t rkey)
{
uct_rc_verbs_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_verbs_iface_t);
uct_rc_verbs_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_verbs_ep_t);
uct_rc_iface_send_desc_t *desc;
/* TODO don't allocate descriptor - have dummy buffer */
UCT_RC_VERBS_CHECK_RES(iface, ep);
UCT_RC_IFACE_GET_TX_DESC(&iface->super, iface->short_desc_mp, desc);
desc->super.handler = (uct_rc_send_handler_t)ucs_mpool_put;
uct_rc_verbs_ep_atomic_post(ucs_derived_of(tl_ep, uct_rc_verbs_ep_t),
IBV_WR_ATOMIC_FETCH_AND_ADD, add, 0,
remote_addr, rkey, desc,
IBV_SEND_SIGNALED);
return UCS_OK;
}
ssize_t uct_rc_mlx5_ep_put_bcopy(uct_ep_h tl_ep, uct_pack_callback_t pack_cb,
void *arg, uint64_t remote_addr, uct_rkey_t rkey)
{
uct_rc_mlx5_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_mlx5_iface_t);
uct_rc_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_mlx5_ep_t);
uct_rc_iface_send_desc_t *desc;
size_t length;
UCT_RC_CHECK_RES(&iface->super, &ep->super);
UCT_RC_IFACE_GET_TX_DESC(&iface->super, &iface->super.tx.mp, desc);
desc->super.handler = (uct_rc_send_handler_t)ucs_mpool_put;
length = pack_cb(desc + 1, arg);
uct_rc_mlx5_ep_bcopy_post(ep, MLX5_OPCODE_RDMA_WRITE, length, 0, NULL, 0,
remote_addr, rkey, MLX5_WQE_CTRL_CQ_UPDATE, desc);
UCT_TL_EP_STAT_OP(&ep->super.super, PUT, BCOPY, length);
return length;
}
ucs_status_t uct_rc_verbs_ep_atomic_add32(uct_ep_h tl_ep, uint32_t add,
uint64_t remote_addr, uct_rkey_t rkey)
{
#if HAVE_IB_EXT_ATOMICS
uct_rc_verbs_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_verbs_iface_t);
uct_rc_verbs_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_verbs_ep_t);
uct_rc_iface_send_desc_t *desc;
UCT_RC_VERBS_CHECK_RES(iface, ep);
UCT_RC_IFACE_GET_TX_DESC(&iface->super, iface->short_desc_mp, desc);
/* TODO don't allocate descriptor - have dummy buffer */
desc->super.handler = (uct_rc_send_handler_t)ucs_mpool_put;
return uct_rc_verbs_ext_atomic_post(ep, IBV_EXP_WR_EXT_MASKED_ATOMIC_FETCH_AND_ADD,
sizeof(uint32_t), 0, add, 0, remote_addr,
rkey, desc, IBV_EXP_SEND_SIGNALED, UCS_OK);
#else
return UCS_ERR_UNSUPPORTED;
#endif
}
ucs_status_t uct_rc_mlx5_ep_put_bcopy(uct_ep_h tl_ep, uct_pack_callback_t pack_cb,
void *arg, size_t length, uint64_t remote_addr,
uct_rkey_t rkey)
{
uct_rc_mlx5_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_mlx5_iface_t);
uct_rc_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_mlx5_ep_t);
uct_rc_iface_send_desc_t *desc;
ucs_status_t status;
UCT_CHECK_LENGTH(length, iface->super.super.config.seg_size, "put_bcopy");
UCT_RC_MLX5_CHECK_RES(iface, ep);
UCT_RC_IFACE_GET_TX_DESC(&iface->super, iface->super.tx.mp, desc);
desc->super.handler = (uct_rc_send_handler_t)ucs_mpool_put;
pack_cb(desc + 1, arg, length);
status = uct_rc_mlx5_ep_bcopy_post(ep, MLX5_OPCODE_RDMA_WRITE, length, 0, NULL,
0, remote_addr, rkey, MLX5_WQE_CTRL_CQ_UPDATE,
desc, UCS_OK);
UCT_TL_EP_STAT_OP_IF_SUCCESS(status, &ep->super.super, PUT, BCOPY, length);
return status;
}
static UCS_F_ALWAYS_INLINE ucs_status_t
uct_rc_mlx5_ep_atomic(uct_rc_mlx5_ep_t *ep, int opcode, void *result, unsigned length,
uct_rc_send_handler_t handler, uint64_t remote_addr,
uct_rkey_t rkey, uint64_t compare_mask, uint64_t compare,
uint64_t swap_add, uct_completion_t *comp)
{
uct_rc_mlx5_iface_t *iface = ucs_derived_of(ep->super.super.super.iface,
uct_rc_mlx5_iface_t);
uct_rc_iface_send_desc_t *desc;
UCT_CHECK_PARAM(comp != NULL, "completion must be non-NULL");
UCT_RC_MLX5_CHECK_RES(iface, ep);
UCT_RC_IFACE_GET_TX_DESC(&iface->super, iface->tx.atomic_desc_mp, desc);
desc->super.handler = handler;
desc->super.result = result;
desc->super.user_comp = comp;
return uct_rc_mlx5_ep_atomic_post(ep, opcode, desc, length, remote_addr,
rkey, compare_mask, compare, swap_add,
MLX5_WQE_CTRL_CQ_UPDATE, UCS_INPROGRESS);
}
static UCS_F_ALWAYS_INLINE ucs_status_t
uct_rc_verbs_ep_atomic(uct_rc_verbs_ep_t *ep, int opcode, void *result,
uint64_t compare_add, uint64_t swap, uint64_t remote_addr,
uct_rkey_t rkey, uct_completion_t *comp)
{
uct_rc_verbs_iface_t *iface = ucs_derived_of(ep->super.super.super.iface,
uct_rc_verbs_iface_t);
uct_rc_iface_send_desc_t *desc;
UCT_CHECK_PARAM(comp != NULL, "completion must be non-NULL");
UCT_RC_VERBS_CHECK_RES(iface, ep);
UCT_RC_IFACE_GET_TX_DESC(&iface->super, iface->short_desc_mp, desc);
desc->super.handler = iface->config.atomic64_handler;
desc->super.result = result;
desc->super.user_comp = comp;
uct_rc_verbs_ep_atomic_post(ep, opcode, compare_add, swap, remote_addr,
rkey, desc, IBV_SEND_SIGNALED);
return UCS_INPROGRESS;
}
ucs_status_t uct_rc_verbs_ep_put_bcopy(uct_ep_h tl_ep, uct_pack_callback_t pack_cb,
void *arg, size_t length, uint64_t remote_addr,
uct_rkey_t rkey)
{
uct_rc_verbs_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_verbs_iface_t);
uct_rc_verbs_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_verbs_ep_t);
uct_rc_iface_send_desc_t *desc;
struct ibv_send_wr wr;
struct ibv_sge sge;
UCT_CHECK_LENGTH(length, iface->super.super.config.seg_size, "put_bcopy");
UCT_RC_VERBS_ZERO_LENGTH_POST(length);
UCT_RC_VERBS_CHECK_RES(iface, ep);
UCT_RC_IFACE_GET_TX_DESC(&iface->super, iface->super.tx.mp, desc);
desc->super.handler = (uct_rc_send_handler_t)ucs_mpool_put;
pack_cb(desc + 1, arg, length);
uct_rc_verbs_fill_rdma_wr(&wr, IBV_WR_RDMA_WRITE, &sge, length, remote_addr,
rkey);
UCT_TL_EP_STAT_OP(&ep->super.super, PUT, BCOPY, length);
uct_rc_verbs_ep_post_send_desc(ep, &wr, desc, IBV_SEND_SIGNALED);
return UCS_OK;
}
