ucs_status_t uct_rc_verbs_ep_fc_ctrl(uct_rc_ep_t *rc_ep)
{
uct_rc_verbs_iface_t *iface = ucs_derived_of(rc_ep->super.super.iface,
uct_rc_verbs_iface_t);
uct_rc_verbs_ep_t *ep = ucs_derived_of(rc_ep, uct_rc_verbs_ep_t);
uct_rc_hdr_t hdr;
struct ibv_send_wr fc_wr;
/* Do not check FC WND here to avoid head-to-head deadlock.
* Credits grant should be sent regardless of FC wnd state. */
ucs_assert(sizeof(hdr) <= iface->verbs_common.config.max_inline);
UCT_RC_CHECK_RES(&iface->super, &ep->super);
hdr.am_id = UCT_RC_EP_FC_PURE_GRANT;
fc_wr.sg_list = iface->verbs_common.inl_sge;
fc_wr.num_sge = 1;
fc_wr.opcode = IBV_WR_SEND;
fc_wr.next = NULL;
iface->verbs_common.inl_sge[0].addr = (uintptr_t)&hdr;
iface->verbs_common.inl_sge[0].length = sizeof(hdr);
uct_rc_verbs_ep_post_send(iface, ep, &fc_wr, IBV_SEND_INLINE);
return UCS_OK;
}
/* For RNDV request send regular eager packet with IBV_SEND_WITH_IMM and
* imm_value = 0. Receiver will handle such message as rndv request. */
ucs_status_t uct_rc_verbs_ep_tag_rndv_request(uct_ep_h tl_ep, uct_tag_t tag,
const void* header,
unsigned header_length)
{
uct_rc_verbs_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_verbs_ep_t);
uct_rc_verbs_iface_t *iface = ucs_derived_of(tl_ep->iface,
uct_rc_verbs_iface_t);
void *tm_hdr = ucs_alloca(iface->tm.eager_hdr_size);
uint32_t app_ctx;
struct ibv_send_wr wr;
UCT_CHECK_LENGTH(header_length + iface->tm.eager_hdr_size, 0,
iface->verbs_common.config.max_inline, "tag_short");
UCT_RC_CHECK_RES(&iface->super, &ep->super);
wr.sg_list = iface->verbs_common.inl_sge;
wr.num_sge = 2;
wr.opcode = IBV_WR_SEND_WITH_IMM;
wr.next = NULL;
uct_rc_verbs_tag_imm_data_pack(&(wr.imm_data), &app_ctx, 0ul);
uct_rc_verbs_iface_fill_inl_tag_sge(iface, tm_hdr, tag,
header, header_length, app_ctx);
uct_rc_verbs_ep_post_send(iface, ep, &wr, IBV_SEND_INLINE);
return UCS_OK;
}
ucs_status_ptr_t uct_rc_verbs_ep_tag_rndv_zcopy(uct_ep_h tl_ep, uct_tag_t tag,
const void *header,
unsigned header_length,
const uct_iov_t *iov,
size_t iovcnt,
uct_completion_t *comp)
{
uct_rc_verbs_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_verbs_ep_t);
uct_rc_verbs_iface_t *iface = ucs_derived_of(tl_ep->iface,
uct_rc_verbs_iface_t);
void *hdr = ucs_alloca(iface->tm.rndv_hdr_size);
uct_ib_device_t *dev = uct_ib_iface_device(&iface->super.super);
uint32_t op_index;
UCT_CHECK_PARAM_PTR(iovcnt <= 1ul,
"Wrong iovcnt in uct_rc_verbs_ep_tag_rndv_zcopy %lu",
iovcnt);
UCT_CHECK_PARAM_PTR(header_length <= IBV_DEVICE_TM_CAPS(dev, max_rndv_priv_size),
"Invalid hdr len in uct_rc_verbs_ep_tag_rndv_zcopy %u",
header_length);
UCT_CHECK_PARAM_PTR((header_length + iface->tm.rndv_hdr_size) <=
iface->verbs_common.config.max_inline,
"Invalid RTS len in uct_rc_verbs_ep_tag_rndv_zcopy %u",
header_length + iface->tm.rndv_hdr_size);
op_index = uct_rc_verbs_iface_tag_get_op_id(iface, comp);
uct_rc_verbs_iface_fill_inl_rndv_sge(iface, hdr, tag, op_index,
((uct_ib_mem_t*)iov->memh)->mr->rkey,
iov->buffer, uct_iov_get_length(iov),
header, header_length);
uct_rc_verbs_ep_post_send(iface, ep, &iface->inl_am_wr, IBV_SEND_INLINE);
return (ucs_status_ptr_t)((uint64_t)op_index);
}
ucs_status_t uct_rc_verbs_ep_fc_ctrl(uct_ep_t *tl_ep, unsigned op,
uct_rc_fc_request_t *req)
{
struct ibv_send_wr fc_wr;
uct_rc_verbs_iface_t *iface = ucs_derived_of(tl_ep->iface,
uct_rc_verbs_iface_t);
uct_rc_verbs_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_verbs_ep_t);
size_t notag_hdr_size = iface->verbs_common.config.notag_hdr_size;
uct_rc_hdr_t *hdr = iface->verbs_common.am_inl_hdr + notag_hdr_size;
/* In RC only PURE grant is sent as a separate message. Other FC
* messages are bundled with AM. */
ucs_assert(op == UCT_RC_EP_FC_PURE_GRANT);
/* Do not check FC WND here to avoid head-to-head deadlock.
* Credits grant should be sent regardless of FC wnd state. */
ucs_assert(sizeof(*hdr) + notag_hdr_size <=
iface->verbs_common.config.max_inline);
UCT_RC_CHECK_RES(&iface->super, &ep->super);
hdr->am_id = UCT_RC_EP_FC_PURE_GRANT;
fc_wr.sg_list = iface->verbs_common.inl_sge;
fc_wr.opcode = IBV_WR_SEND;
fc_wr.next = NULL;
fc_wr.num_sge = 1;
iface->verbs_common.inl_sge[0].addr = (uintptr_t)iface->verbs_common.am_inl_hdr;
iface->verbs_common.inl_sge[0].length = sizeof(*hdr) + notag_hdr_size;
uct_rc_verbs_ep_post_send(iface, ep, &fc_wr, IBV_SEND_INLINE);
return UCS_OK;
}
/*
* Helper function for posting sends with a descriptor.
* User needs to fill: wr.opcode, wr.sg_list, wr.num_sge, first sge length, and
* operation-specific fields (e.g rdma).
*/
static UCS_F_ALWAYS_INLINE void
uct_rc_verbs_ep_post_send_desc(uct_rc_verbs_ep_t* ep, struct ibv_send_wr *wr,
uct_rc_iface_send_desc_t *desc, int send_flags)
{
uct_rc_verbs_iface_t *iface = ucs_derived_of(ep->super.super.super.iface,
uct_rc_verbs_iface_t);
UCT_RC_VERBS_FILL_DESC_WR(wr, desc);
uct_rc_verbs_ep_post_send(iface, ep, wr, send_flags);
uct_rc_txqp_add_send_op_sn(&ep->super.txqp, &desc->super, ep->txcnt.pi);
}
ucs_status_t uct_rc_verbs_ep_put_short(uct_ep_h tl_ep, const void *buffer,
unsigned length, uint64_t remote_addr,
uct_rkey_t rkey)
{
uct_rc_verbs_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_verbs_iface_t);
uct_rc_verbs_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_verbs_ep_t);
UCT_CHECK_LENGTH(length, iface->verbs_common.config.max_inline, "put_short");
UCT_RC_CHECK_RES(&iface->super, &ep->super);
UCT_RC_VERBS_FILL_INL_PUT_WR(iface, remote_addr, rkey, buffer, length);
UCT_TL_EP_STAT_OP(&ep->super.super, PUT, SHORT, length);
uct_rc_verbs_ep_post_send(iface, ep, &iface->inl_rwrite_wr,
IBV_SEND_INLINE | IBV_SEND_SIGNALED);
return UCS_OK;
}
/*
* Helper function for posting sends with a descriptor.
* User needs to fill: wr.opcode, wr.sg_list, wr.num_sge, first sge length, and
* operation-specific fields (e.g rdma).
*/
static UCS_F_ALWAYS_INLINE void
uct_rc_verbs_ep_post_send_desc(uct_rc_verbs_ep_t* ep, struct ibv_send_wr *wr,
uct_rc_iface_send_desc_t *desc, int send_flags)
{
uct_rc_verbs_iface_t *iface = ucs_derived_of(ep->super.super.super.iface,
uct_rc_verbs_iface_t);
struct ibv_sge *sge;
wr->next = NULL;
sge = wr->sg_list;
sge->addr = (uintptr_t)(desc + 1);
sge->lkey = desc->lkey;
uct_rc_verbs_ep_post_send(iface, ep, wr, send_flags);
uct_rc_verbs_ep_push_desc(ep, desc);
}
ucs_status_t uct_rc_verbs_ep_am_short(uct_ep_h tl_ep, uint8_t id, uint64_t hdr,
const void *buffer, unsigned length)
{
uct_rc_verbs_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_verbs_iface_t);
uct_rc_verbs_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_verbs_ep_t);
UCT_RC_CHECK_AM_SHORT(id, length, iface->config.max_inline);
UCT_RC_CHECK_RES(&iface->super, &ep->super);
UCT_RC_CHECK_FC(&iface->super, &ep->super, id);
uct_rc_verbs_iface_fill_inl_am_sge(iface, id, hdr, buffer, length);
UCT_TL_EP_STAT_OP(&ep->super.super, AM, SHORT, sizeof(hdr) + length);
uct_rc_verbs_ep_post_send(iface, ep, &iface->inl_am_wr,
IBV_SEND_INLINE | IBV_SEND_SOLICITED, INT_MAX);
UCT_RC_UPDATE_FC(&iface->super, &ep->super, id);
return UCS_OK;
}
ucs_status_t uct_rc_verbs_ep_tag_eager_short(uct_ep_h tl_ep, uct_tag_t tag,
const void *data, size_t length)
{
uct_rc_verbs_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_verbs_ep_t);
uct_rc_verbs_iface_t *iface = ucs_derived_of(tl_ep->iface,
uct_rc_verbs_iface_t);
void *tm_hdr = ucs_alloca(iface->tm.eager_hdr_size);
UCT_CHECK_LENGTH(length + iface->tm.eager_hdr_size, 0,
iface->verbs_common.config.max_inline, "tag_short");
UCT_RC_CHECK_RES(&iface->super, &ep->super);
uct_rc_verbs_iface_fill_inl_tag_sge(iface, tm_hdr, tag, data, length, 0);
uct_rc_verbs_ep_post_send(iface, ep, &iface->inl_am_wr, IBV_SEND_INLINE);
return UCS_OK;
}
ucs_status_t uct_rc_verbs_ep_put_short(uct_ep_h tl_ep, const void *buffer,
unsigned length, uint64_t remote_addr,
uct_rkey_t rkey)
{
uct_rc_verbs_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_verbs_iface_t);
uct_rc_verbs_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_verbs_ep_t);
UCT_CHECK_LENGTH(length, iface->config.max_inline, "put_short");
UCT_RC_VERBS_CHECK_RES(iface, ep);
iface->inl_rwrite_wr.wr.rdma.remote_addr = remote_addr;
iface->inl_rwrite_wr.wr.rdma.rkey = rkey;
iface->inl_sge[0].addr = (uintptr_t)buffer;
iface->inl_sge[0].length = length;
UCT_TL_EP_STAT_OP(&ep->super.super, PUT, SHORT, length);
uct_rc_verbs_ep_post_send(iface, ep, &iface->inl_rwrite_wr,
IBV_SEND_INLINE | IBV_SEND_SIGNALED);
return UCS_OK;
}
ucs_status_t uct_rc_verbs_ep_fc_ctrl(uct_ep_t *tl_ep, unsigned op,
uct_rc_fc_request_t *req)
{
struct ibv_send_wr fc_wr;
uct_rc_verbs_iface_t *iface = ucs_derived_of(tl_ep->iface,
uct_rc_verbs_iface_t);
uct_rc_verbs_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_verbs_ep_t);
uct_rc_hdr_t *hdr;
struct ibv_sge sge;
int flags;
if (!iface->fc_desc) {
hdr = &iface->am_inl_hdr.rc_hdr;
hdr->am_id = UCT_RC_EP_FC_PURE_GRANT;
fc_wr.sg_list = iface->inl_sge;
iface->inl_sge[0].addr = (uintptr_t)hdr;
iface->inl_sge[0].length = sizeof(*hdr);
flags = IBV_SEND_INLINE;
} else {
hdr = (uct_rc_hdr_t*)(iface->fc_desc + 1);
sge.addr = (uintptr_t)hdr;
sge.length = sizeof(*hdr);
sge.lkey = iface->fc_desc->lkey;
fc_wr.sg_list = &sge;
flags = 0;
}
/* In RC only PURE grant is sent as a separate message. Other FC
* messages are bundled with AM. */
ucs_assert(op == UCT_RC_EP_FC_PURE_GRANT);
/* Do not check FC WND here to avoid head-to-head deadlock.
* Credits grant should be sent regardless of FC wnd state. */
UCT_RC_CHECK_RES(&iface->super, &ep->super);
fc_wr.opcode = IBV_WR_SEND;
fc_wr.next = NULL;
fc_wr.num_sge = 1;
uct_rc_verbs_ep_post_send(iface, ep, &fc_wr, flags, INT_MAX);
return UCS_OK;
}
static inline ucs_status_t
uct_rc_verbs_ep_rdma_zcopy(uct_rc_verbs_ep_t *ep, const uct_iov_t *iov,
size_t iovcnt, uint64_t remote_addr, uct_rkey_t rkey,
uct_completion_t *comp, int opcode)
{
uct_rc_verbs_iface_t *iface = ucs_derived_of(ep->super.super.super.iface,
uct_rc_verbs_iface_t);
struct ibv_sge sge[UCT_IB_MAX_IOV];
struct ibv_send_wr wr;
size_t sge_cnt;
UCT_RC_CHECK_RES(&iface->super, &ep->super);
sge_cnt = uct_ib_verbs_sge_fill_iov(sge, iov, iovcnt);
UCT_SKIP_ZERO_LENGTH(sge_cnt);
UCT_RC_VERBS_FILL_RDMA_WR_IOV(wr, wr.opcode, opcode, sge, sge_cnt, remote_addr, rkey);
wr.next = NULL;
uct_rc_verbs_ep_post_send(iface, ep, &wr, IBV_SEND_SIGNALED);
uct_rc_txqp_add_send_comp(&iface->super, &ep->super.txqp, comp, ep->txcnt.pi);
return UCS_INPROGRESS;
}
ucs_status_t uct_rc_verbs_ep_am_short(uct_ep_h tl_ep, uint8_t id, uint64_t hdr,
const void *buffer, unsigned length)
{
uct_rc_verbs_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_verbs_iface_t);
uct_rc_verbs_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_verbs_ep_t);
uct_rc_am_short_hdr_t am;
UCT_CHECK_AM_ID(id);
UCT_CHECK_LENGTH(sizeof(am) + length, iface->config.max_inline, "am_short");
UCT_RC_VERBS_CHECK_RES(iface, ep);
am.rc_hdr.am_id = id;
am.am_hdr = hdr;
iface->inl_sge[0].addr = (uintptr_t)&am;
iface->inl_sge[0].length = sizeof(am);
iface->inl_sge[1].addr = (uintptr_t)buffer;
iface->inl_sge[1].length = length;
UCT_TL_EP_STAT_OP(&ep->super.super, AM, SHORT, sizeof(hdr) + length);
uct_rc_verbs_ep_post_send(iface, ep, &iface->inl_am_wr, IBV_SEND_INLINE);
return UCS_OK;
}
static inline ucs_status_t
uct_rc_verbs_ep_rdma_zcopy(uct_rc_verbs_ep_t *ep, const void *buffer, size_t length,
struct ibv_mr *mr, uint64_t remote_addr,
uct_rkey_t rkey, uct_completion_t *comp,
int opcode)
{
uct_rc_verbs_iface_t *iface = ucs_derived_of(ep->super.super.super.iface,
uct_rc_verbs_iface_t);
struct ibv_send_wr wr;
struct ibv_sge sge;
UCT_RC_VERBS_ZERO_LENGTH_POST(length);
UCT_RC_VERBS_CHECK_RES(iface, ep);
uct_rc_verbs_fill_rdma_wr(&wr, opcode, &sge, length, remote_addr, rkey);
wr.next = NULL;
sge.addr = (uintptr_t)buffer;
sge.lkey = (mr == UCT_INVALID_MEM_HANDLE) ? 0 : mr->lkey;
uct_rc_verbs_ep_post_send(iface, ep, &wr, IBV_SEND_SIGNALED);
uct_rc_ep_add_send_comp(&iface->super, &ep->super, comp, ep->tx.post_count);
return UCS_INPROGRESS;
}
