static inline ucs_status_t uct_ud_verbs_am_common(uct_ud_verbs_iface_t *iface,
uct_ud_verbs_ep_t *ep,
uint8_t id,
uct_ud_send_skb_t **skb_p)
{
uct_ud_send_skb_t *skb;
uct_ud_neth_t *neth;
UCT_CHECK_AM_ID(id);
if (!uct_ud_ep_is_connected(&ep->super)) {
return UCS_ERR_NO_RESOURCE;
}
skb = uct_ud_iface_get_tx_skb(&iface->super, &ep->super);
if (!skb) {
return UCS_ERR_NO_RESOURCE;
}
VALGRIND_MAKE_MEM_DEFINED(skb, sizeof *skb);
neth = skb->neth;
uct_ud_neth_init_data(&ep->super, neth);
uct_ud_neth_set_type_am(&ep->super, neth, id);
uct_ud_neth_ack_req(&ep->super, neth);
iface->tx.sge[0].addr = (uintptr_t)neth;
*skb_p = skb;
return UCS_OK;
}
ucs_status_t uct_rc_verbs_ep_am_bcopy(uct_ep_h tl_ep, uint8_t id,
uct_pack_callback_t pack_cb, void *arg,
size_t length)
{
uct_rc_verbs_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_verbs_iface_t);
uct_rc_verbs_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_verbs_ep_t);
uct_rc_iface_send_desc_t *desc;
struct ibv_send_wr wr;
struct ibv_sge sge;
uct_rc_hdr_t *rch;
UCT_CHECK_AM_ID(id);
UCT_CHECK_LENGTH(sizeof(*rch) + length, iface->super.super.config.seg_size,
"am_bcopy");
UCT_RC_VERBS_CHECK_RES(iface, ep);
UCT_RC_IFACE_GET_TX_DESC(&iface->super, iface->super.tx.mp, desc);
desc->super.handler = (uct_rc_send_handler_t)ucs_mpool_put;
rch = (void*)(desc + 1);
rch->am_id = id;
pack_cb(rch + 1, arg, length);
wr.sg_list = &sge;
wr.num_sge = 1;
wr.opcode = IBV_WR_SEND;
sge.length = sizeof(*rch) + length;
UCT_TL_EP_STAT_OP(&ep->super.super, AM, BCOPY, length);
uct_rc_verbs_ep_post_send_desc(ep, &wr, desc, 0);
return UCS_OK;
}
ssize_t uct_rc_mlx5_ep_am_bcopy(uct_ep_h tl_ep, uint8_t id,
uct_pack_callback_t pack_cb, void *arg)
{
uct_rc_mlx5_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_mlx5_iface_t);
uct_rc_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_mlx5_ep_t);
uct_rc_iface_send_desc_t *desc;
uct_rc_hdr_t *rch;
size_t length;
UCT_CHECK_AM_ID(id);
UCT_RC_CHECK_RES(&iface->super, &ep->super);
UCT_RC_CHECK_FC_WND(&iface->super, &ep->super, id);
UCT_RC_IFACE_GET_TX_DESC(&iface->super, &iface->super.tx.mp, desc);
desc->super.handler = (uct_rc_send_handler_t)ucs_mpool_put;
rch = (void*)(desc + 1);
rch->am_id = id;
length = pack_cb(rch + 1, arg);
uct_rc_mlx5_ep_bcopy_post(ep, MLX5_OPCODE_SEND|UCT_RC_MLX5_OPCODE_FLAG_RAW,
sizeof(*rch) + length, 0, NULL, 0, 0, 0, 0, desc);
UCT_TL_EP_STAT_OP(&ep->super.super, AM, BCOPY, length);
UCT_RC_UPDATE_FC_WND(&ep->super);
return length;
}
ucs_status_t uct_rc_mlx5_ep_am_zcopy(uct_ep_h tl_ep, uint8_t id, const void *header,
unsigned header_length, const void *payload,
size_t length, uct_mem_h memh,
uct_completion_t *comp)
{
uct_rc_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_mlx5_ep_t);
uct_rc_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_iface_t);
ucs_status_t status;
UCT_CHECK_AM_ID(id);
UCT_RC_CHECK_FC_WND(iface, &ep->super, id);
UCT_CHECK_LENGTH(sizeof(struct mlx5_wqe_ctrl_seg) +
sizeof(struct mlx5_wqe_data_seg) +
sizeof(struct mlx5_wqe_inl_data_seg) +
sizeof(uct_rc_hdr_t) + header_length,
UCT_RC_MLX5_MAX_BB * MLX5_SEND_WQE_BB,
"am zcopy");
UCT_CHECK_LENGTH(header_length + length + sizeof(uct_rc_hdr_t),
ucs_derived_of(tl_ep->iface, uct_ib_iface_t)->config.seg_size,
"am_zcopy");
UCT_CHECK_LENGTH(header_length + length, UCT_IB_MAX_MESSAGE_SIZE, "am_zcopy");
status = uct_rc_mlx5_ep_zcopy_post(ep, MLX5_OPCODE_SEND, payload, length, memh,
id, header, header_length, 0, 0, 0, comp);
if (ucs_likely(status >= 0)) {
UCT_TL_EP_STAT_OP(&ep->super.super, AM, ZCOPY, header_length + length);
UCT_RC_UPDATE_FC_WND(&ep->super);
}
return status;
}
ucs_status_t uct_ugni_smsg_ep_am_short(uct_ep_h tl_ep, uint8_t id, uint64_t header,
const void *payload, unsigned length)
{
uct_ugni_smsg_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_ugni_smsg_iface_t);
uct_ugni_smsg_ep_t *ep = ucs_derived_of(tl_ep, uct_ugni_smsg_ep_t);
uct_ugni_smsg_header_t *smsg_header;
uint64_t *header_data;
uct_ugni_smsg_desc_t *desc;
UCT_CHECK_AM_ID(id);
UCT_CHECK_LENGTH(length, iface->config.smsg_seg_size -
(sizeof(smsg_header) + sizeof(header)), "am_short");
UCT_TL_IFACE_GET_TX_DESC(&iface->super.super, &iface->free_desc,
desc, return UCS_ERR_NO_RESOURCE);
ucs_trace_data("AM_SHORT [%p] am_id: %d buf=%p length=%u",
iface, id, payload, length);
smsg_header = (uct_ugni_smsg_header_t *)(desc+1);
smsg_header->length = length + sizeof(header);
header_data = (uint64_t*)(smsg_header+1);
*header_data = header;
memcpy((void*)(header_data+1), payload, length);
uct_iface_trace_am(&iface->super.super, UCT_AM_TRACE_TYPE_SEND,
id, header_data, length, "TX: AM_SHORT");
return uct_ugni_smsg_ep_am_common_send(ep, iface, id, sizeof(uct_ugni_smsg_header_t),
smsg_header, smsg_header->length, (void*)header_data, desc);
}
ssize_t uct_ugni_smsg_ep_am_bcopy(uct_ep_h tl_ep, uint8_t id,
uct_pack_callback_t pack_cb,
void *arg)
{
uct_ugni_smsg_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_ugni_smsg_iface_t);
uct_ugni_smsg_ep_t *ep = ucs_derived_of(tl_ep, uct_ugni_smsg_ep_t);
ssize_t packed;
uct_ugni_smsg_desc_t *desc;
ucs_status_t rc;
void *smsg_data;
uct_ugni_smsg_header_t *smsg_header;
UCT_CHECK_AM_ID(id);
UCT_TL_IFACE_GET_TX_DESC(&iface->super.super, &iface->free_desc,
desc, return UCS_ERR_NO_RESOURCE);
ucs_trace_data("AM_BCOPY [%p] am_id: %d buf=%p",
iface, id, arg );
smsg_header = (uct_ugni_smsg_header_t *)(desc+1);
smsg_data = (void*)(smsg_header+1);
packed = pack_cb(smsg_data, arg);
smsg_header->length = packed;
uct_iface_trace_am(&iface->super.super, UCT_AM_TRACE_TYPE_SEND,
id, smsg_data, packed, "TX: AM_BCOPY");
rc = uct_ugni_smsg_ep_am_common_send(ep, iface, id, sizeof(uct_ugni_smsg_header_t),
smsg_header, packed, smsg_data, desc);
return (UCS_OK == rc) ? packed : rc;
}
ucs_status_t uct_rc_mlx5_ep_am_bcopy(uct_ep_h tl_ep, uint8_t id,
uct_pack_callback_t pack_cb, void *arg,
size_t length)
{
uct_rc_mlx5_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_mlx5_iface_t);
uct_rc_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_mlx5_ep_t);
uct_rc_iface_send_desc_t *desc;
ucs_status_t status;
uct_rc_hdr_t *rch;
UCT_CHECK_AM_ID(id);
UCT_CHECK_LENGTH(sizeof(*rch) + length, iface->super.super.config.seg_size,
"am_bcopy");
UCT_RC_MLX5_CHECK_RES(iface, ep);
UCT_RC_IFACE_GET_TX_DESC(&iface->super, iface->super.tx.mp, desc);
desc->super.handler = (uct_rc_send_handler_t)ucs_mpool_put;
rch = (void*)(desc + 1);
rch->am_id = id;
pack_cb(rch + 1, arg, length);
status = uct_rc_mlx5_ep_bcopy_post(ep, MLX5_OPCODE_SEND|UCT_RC_MLX5_OPCODE_FLAG_RAW,
sizeof(*rch) + length, 0, NULL, 0, 0, 0, 0, desc,
UCS_OK);
UCT_TL_EP_STAT_OP_IF_SUCCESS(status, &ep->super.super, AM, BCOPY, length);
return status;
}
ssize_t uct_rc_verbs_ep_am_bcopy(uct_ep_h tl_ep, uint8_t id,
uct_pack_callback_t pack_cb, void *arg,
unsigned flags)
{
uct_rc_verbs_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_verbs_iface_t);
uct_rc_verbs_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_verbs_ep_t);
uct_rc_iface_send_desc_t *desc;
struct ibv_send_wr wr;
struct ibv_sge sge;
size_t length;
UCT_CHECK_AM_ID(id);
UCT_RC_CHECK_RES(&iface->super, &ep->super);
UCT_RC_CHECK_FC(&iface->super, &ep->super, id);
UCT_RC_IFACE_GET_TX_AM_BCOPY_DESC(&iface->super, &iface->super.tx.mp, desc,
id, uct_rc_am_hdr_fill, uct_rc_hdr_t,
pack_cb, arg, &length);
UCT_RC_VERBS_FILL_AM_BCOPY_WR(wr, sge, length + sizeof(uct_rc_hdr_t),
wr.opcode);
UCT_TL_EP_STAT_OP(&ep->super.super, AM, BCOPY, length);
uct_rc_verbs_ep_post_send_desc(ep, &wr, desc, IBV_SEND_SOLICITED, INT_MAX);
UCT_RC_UPDATE_FC(&iface->super, &ep->super, id);
return length;
}
static ucs_status_t
uct_ud_mlx5_ep_am_short(uct_ep_h tl_ep, uint8_t id, uint64_t hdr,
const void *buffer, unsigned length)
{
uct_ud_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_ud_mlx5_ep_t);
uct_ud_mlx5_iface_t *iface = ucs_derived_of(tl_ep->iface,
uct_ud_mlx5_iface_t);
struct mlx5_wqe_ctrl_seg *ctrl;
struct mlx5_wqe_inl_data_seg *inl;
uct_ud_am_short_hdr_t *am;
uct_ud_neth_t *neth;
unsigned wqe_size;
uct_ud_send_skb_t *skb;
/* data a written directly into tx wqe, so it is impossible to use
* common ud am code
*/
UCT_CHECK_AM_ID(id);
UCT_CHECK_LENGTH(sizeof(uct_ud_neth_t) + sizeof(hdr) + length,
iface->super.config.max_inline, "am_short");
uct_ud_enter(&iface->super);
uct_ud_iface_progress_pending_tx(&iface->super);
skb = uct_ud_ep_get_tx_skb(&iface->super, &ep->super);
if (!skb) {
uct_ud_leave(&iface->super);
return UCS_ERR_NO_RESOURCE;
}
ctrl = iface->tx.wq.curr;
/* Set inline segment which has AM id, AM header, and AM payload */
inl = uct_ib_mlx5_get_next_seg(&iface->tx.wq, ctrl, UCT_UD_MLX5_WQE_SIZE);
wqe_size = length + sizeof(*am) + sizeof(*neth);
inl->byte_count = htonl(wqe_size | MLX5_INLINE_SEG);
/* assume that neth and am header fit into one bb */
ucs_assert(sizeof(*am) + sizeof(*neth) < MLX5_SEND_WQE_BB);
neth = (void*)(inl + 1);
uct_ud_am_set_neth(neth, &ep->super, id);
am = (void*)(neth + 1);
am->hdr = hdr;
uct_ib_mlx5_inline_copy(am + 1, buffer, length, &iface->tx.wq);
wqe_size += UCT_UD_MLX5_WQE_SIZE + sizeof(*inl);
UCT_CHECK_LENGTH(wqe_size, UCT_IB_MLX5_MAX_BB * MLX5_SEND_WQE_BB,
"am_short");
UCT_UD_EP_HOOK_CALL_TX(&ep->super, neth);
uct_ud_mlx5_post_send(iface, ep, ctrl, wqe_size);
skb->len = sizeof(*neth) + sizeof(*am);
memcpy(skb->neth, neth, skb->len);
uct_ud_iface_complete_tx_inl(&iface->super, &ep->super, skb,
(char *)skb->neth + skb->len, buffer, length);
UCT_TL_EP_STAT_OP(&ep->super.super, AM, SHORT, sizeof(hdr) + length);
uct_ud_leave(&iface->super);
return UCS_OK;
}
ucs_status_t uct_rc_verbs_ep_am_zcopy(uct_ep_h tl_ep, uint8_t id, const void *header,
unsigned header_length, const void *payload,
size_t length, uct_mem_h memh,
uct_completion_t *comp)
{
uct_rc_verbs_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_verbs_iface_t);
uct_rc_verbs_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_verbs_ep_t);
struct ibv_mr *mr = memh;
uct_rc_iface_send_desc_t *desc;
struct ibv_send_wr wr;
struct ibv_sge sge[2];
uct_rc_hdr_t *rch;
int send_flags;
UCT_CHECK_AM_ID(id);
UCT_CHECK_LENGTH(sizeof(*rch) + header_length, iface->config.short_desc_size,
"am_zcopy header");
UCT_CHECK_LENGTH(header_length + length, iface->super.super.config.seg_size,
"am_zcopy payload");
UCT_RC_VERBS_CHECK_RES(iface, ep);
UCT_RC_IFACE_GET_TX_DESC(&iface->super, iface->short_desc_mp, desc);
if (comp == NULL) {
desc->super.handler = (uct_rc_send_handler_t)ucs_mpool_put;
send_flags = 0;
} else {
desc->super.handler = uct_rc_verbs_ep_am_zcopy_handler;
desc->super.user_comp = comp;
send_flags = IBV_SEND_SIGNALED;
}
/* Header buffer: active message ID + user header */
rch = (void*)(desc + 1);
rch->am_id = id;
memcpy(rch + 1, header, header_length);
wr.sg_list = sge;
wr.opcode = IBV_WR_SEND;
sge[0].length = sizeof(*rch) + header_length;
if (ucs_unlikely(length == 0)) {
wr.num_sge = 1;
} else {
wr.num_sge = 2;
sge[1].addr = (uintptr_t)payload;
sge[1].length = length;
sge[1].lkey = (mr == UCT_INVALID_MEM_HANDLE) ? 0 : mr->lkey;
}
UCT_TL_EP_STAT_OP(&ep->super.super, AM, ZCOPY, header_length + length);
uct_rc_verbs_ep_post_send_desc(ep, &wr, desc, send_flags);
return UCS_INPROGRESS;
}
ucs_status_t uct_rc_mlx5_ep_am_short(uct_ep_h tl_ep, uint8_t id, uint64_t hdr,
const void *payload, unsigned length)
{
uct_rc_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_mlx5_ep_t);
ucs_status_t status;
UCT_CHECK_AM_ID(id);
status = uct_rc_mlx5_ep_inline_post(ep, MLX5_OPCODE_SEND, payload, length,
id, hdr, 0, 0);
UCT_TL_EP_STAT_OP_IF_SUCCESS(status, &ep->super.super, AM, SHORT,
sizeof(hdr) + length);
return status;
}
ucs_status_t uct_rc_mlx5_ep_am_zcopy(uct_ep_h tl_ep, uint8_t id, const void *header,
unsigned header_length, const void *payload,
size_t length, uct_mem_h memh,
uct_completion_t *comp)
{
uct_rc_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_mlx5_ep_t);
ucs_status_t status;
UCT_CHECK_AM_ID(id);
status = uct_rc_mlx5_ep_zcopy_post(ep, MLX5_OPCODE_SEND, payload, length, memh,
id, header, header_length, 0, 0, 0, comp);
UCT_TL_EP_STAT_OP_IF_SUCCESS(status, &ep->super.super, AM, ZCOPY,
header_length + length);
return status;
}
ucs_status_t uct_rc_mlx5_ep_am_short(uct_ep_h tl_ep, uint8_t id, uint64_t hdr,
const void *payload, unsigned length)
{
uct_rc_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_iface_t);
uct_rc_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_mlx5_ep_t);
ucs_status_t status;
UCT_CHECK_AM_ID(id);
UCT_RC_CHECK_FC_WND(iface, &ep->super, id);
status = uct_rc_mlx5_ep_inline_post(ep, MLX5_OPCODE_SEND, payload, length,
id, hdr, 0, 0);
if (ucs_likely(status >= 0)) {
UCT_TL_EP_STAT_OP(&ep->super.super, AM, SHORT, sizeof(hdr) + length);
UCT_RC_UPDATE_FC_WND(&ep->super);
}
return status;
}
ssize_t uct_ugni_smsg_ep_am_bcopy(uct_ep_h tl_ep, uint8_t id,
uct_pack_callback_t pack_cb,
void *arg, unsigned flags)
{
uct_ugni_smsg_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_ugni_smsg_iface_t);
uct_ugni_smsg_ep_t *ep = ucs_derived_of(tl_ep, uct_ugni_smsg_ep_t);
ssize_t packed;
uct_ugni_smsg_desc_t *desc;
ucs_status_t rc;
void *smsg_data;
uct_ugni_smsg_header_t *smsg_header;
UCT_CHECK_AM_ID(id);
UCT_TL_IFACE_GET_TX_DESC(&iface->super.super, &iface->free_desc,
desc, return UCS_ERR_NO_RESOURCE);
ucs_trace_data("AM_BCOPY [%p] am_id: %d send request %p",
iface, id, arg);
smsg_header = (uct_ugni_smsg_header_t *)(desc+1);
smsg_data = (void*)(smsg_header+1);
packed = pack_cb(smsg_data, arg);
smsg_header->length = packed;
UCT_CHECK_LENGTH(packed, 0, iface->config.smsg_seg_size -
0, "am_bcopy");
uct_iface_trace_am(&iface->super.super, UCT_AM_TRACE_TYPE_SEND,
id, smsg_data, packed, "TX: AM_BCOPY");
rc = uct_ugni_smsg_ep_am_common_send(ep, iface, id, sizeof(uct_ugni_smsg_header_t),
smsg_header, packed, smsg_data, desc);
UCT_TL_EP_STAT_OP_IF_SUCCESS(rc, ucs_derived_of(ep, uct_base_ep_t), AM, BCOPY, packed);
return (UCS_OK == rc) ? packed : rc;
}
ucs_status_t uct_rc_verbs_ep_am_short(uct_ep_h tl_ep, uint8_t id, uint64_t hdr,
const void *buffer, unsigned length)
{
uct_rc_verbs_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_verbs_iface_t);
uct_rc_verbs_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_verbs_ep_t);
uct_rc_am_short_hdr_t am;
UCT_CHECK_AM_ID(id);
UCT_CHECK_LENGTH(sizeof(am) + length, iface->config.max_inline, "am_short");
UCT_RC_VERBS_CHECK_RES(iface, ep);
am.rc_hdr.am_id = id;
am.am_hdr = hdr;
iface->inl_sge[0].addr = (uintptr_t)&am;
iface->inl_sge[0].length = sizeof(am);
iface->inl_sge[1].addr = (uintptr_t)buffer;
iface->inl_sge[1].length = length;
UCT_TL_EP_STAT_OP(&ep->super.super, AM, SHORT, sizeof(hdr) + length);
uct_rc_verbs_ep_post_send(iface, ep, &iface->inl_am_wr, IBV_SEND_INLINE);
return UCS_OK;
}
ssize_t uct_rc_mlx5_ep_am_bcopy(uct_ep_h tl_ep, uint8_t id,
uct_pack_callback_t pack_cb, void *arg)
{
uct_rc_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_iface_t);
uct_rc_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_mlx5_ep_t);
uct_rc_iface_send_desc_t *desc;
size_t length;
UCT_CHECK_AM_ID(id);
UCT_RC_CHECK_RES(iface, &ep->super);
UCT_RC_CHECK_FC_WND(iface, &ep->super, id);
UCT_RC_IFACE_GET_TX_AM_BCOPY_DESC(iface, &iface->tx.mp, desc,
id, pack_cb, arg, &length);
uct_rc_mlx5_txqp_bcopy_post(iface, &ep->super.txqp, &ep->tx.wq,
MLX5_OPCODE_SEND|UCT_RC_MLX5_OPCODE_FLAG_RAW,
sizeof(uct_rc_hdr_t) + length, 0, NULL, 0, 0, 0,
0, desc);
UCT_TL_EP_STAT_OP(&ep->super.super, AM, BCOPY, length);
UCT_RC_UPDATE_FC_WND(iface, &ep->super, id);
return length;
}
ssize_t uct_dc_mlx5_ep_am_bcopy(uct_ep_h tl_ep, uint8_t id,
uct_pack_callback_t pack_cb, void *arg,
unsigned flags)
{
uct_dc_mlx5_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_dc_mlx5_iface_t);
uct_dc_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_dc_mlx5_ep_t);
uct_rc_iface_send_desc_t *desc;
size_t length;
UCT_CHECK_AM_ID(id);
UCT_DC_CHECK_RES_AND_FC(iface, ep);
UCT_RC_IFACE_GET_TX_AM_BCOPY_DESC(&iface->super.super, &iface->super.super.tx.mp, desc,
id, uct_rc_mlx5_am_hdr_fill, uct_rc_mlx5_hdr_t,
pack_cb, arg, &length);
uct_dc_mlx5_iface_bcopy_post(iface, ep, MLX5_OPCODE_SEND,
sizeof(uct_rc_mlx5_hdr_t) + length, 0, 0, desc,
MLX5_WQE_CTRL_SOLICITED, 0, desc + 1, NULL);
UCT_RC_UPDATE_FC_WND(&iface->super.super, &ep->fc);
UCT_TL_EP_STAT_OP(&ep->super, AM, BCOPY, length);
return length;
}
ucs_status_t uct_dc_mlx5_ep_am_short(uct_ep_h tl_ep, uint8_t id, uint64_t hdr,
const void *buffer, unsigned length)
{
#if HAVE_IBV_EXP_DM
uct_dc_mlx5_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_dc_mlx5_iface_t);
uct_dc_mlx5_ep_t *ep = ucs_derived_of(tl_ep, uct_dc_mlx5_ep_t);
ucs_status_t status;
uct_rc_mlx5_dm_copy_data_t cache;
if (ucs_likely((sizeof(uct_rc_mlx5_am_short_hdr_t) + length <=
UCT_IB_MLX5_AM_MAX_SHORT(UCT_IB_MLX5_AV_FULL_SIZE)) ||
!iface->super.dm.dm)) {
#endif
return uct_dc_mlx5_ep_am_short_inline(tl_ep, id, hdr, buffer, length);
#if HAVE_IBV_EXP_DM
}
UCT_CHECK_AM_ID(id);
UCT_CHECK_LENGTH(length + sizeof(uct_rc_mlx5_am_short_hdr_t), 0,
iface->super.dm.seg_len, "am_short");
UCT_DC_CHECK_RES_AND_FC(iface, ep);
uct_rc_mlx5_am_hdr_fill(&cache.am_hdr.rc_hdr, id);
cache.am_hdr.am_hdr = hdr;
status = uct_dc_mlx5_ep_short_dm(ep, &cache, sizeof(cache.am_hdr), buffer, length,
MLX5_OPCODE_SEND,
MLX5_WQE_CTRL_SOLICITED | MLX5_WQE_CTRL_CQ_UPDATE,
0, 0);
if (UCS_STATUS_IS_ERR(status)) {
return status;
}
UCT_TL_EP_STAT_OP(&ep->super, AM, SHORT, sizeof(cache.am_hdr) + length);
UCT_RC_UPDATE_FC_WND(&iface->super.super, &ep->fc);
return UCS_OK;
#endif
}
ssize_t uct_rc_verbs_ep_am_bcopy(uct_ep_h tl_ep, uint8_t id,
uct_pack_callback_t pack_cb, void *arg)
{
uct_rc_verbs_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_verbs_iface_t);
uct_rc_verbs_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_verbs_ep_t);
uct_rc_iface_send_desc_t *desc;
struct ibv_send_wr wr;
struct ibv_sge sge;
size_t length;
size_t data_length;
UCT_CHECK_AM_ID(id);
UCT_RC_CHECK_RES(&iface->super, &ep->super);
UCT_RC_CHECK_FC(&iface->super, &ep->super, id);
UCT_RC_VERBS_GET_TX_AM_BCOPY_DESC(iface, &iface->super.tx.mp, desc, id,
pack_cb, arg, length, data_length);
UCT_RC_VERBS_FILL_AM_BCOPY_WR(wr, sge, length, wr.opcode);
UCT_TL_EP_STAT_OP(&ep->super.super, AM, BCOPY, data_length);
uct_rc_verbs_ep_post_send_desc(ep, &wr, desc, 0);
UCT_RC_UPDATE_FC(&iface->super, &ep->super, id);
return data_length;
}
ssize_t uct_cm_ep_am_bcopy(uct_ep_h tl_ep, uint8_t am_id,
uct_pack_callback_t pack_cb, void *arg)
{
uct_cm_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_cm_iface_t);
uct_cm_ep_t *ep = ucs_derived_of(tl_ep, uct_cm_ep_t);
struct ib_cm_sidr_req_param req;
struct ibv_sa_path_rec path;
struct ib_cm_id *id;
ucs_status_t status;
uct_cm_hdr_t *hdr;
size_t payload_len;
size_t total_len;
int ret;
UCT_CHECK_AM_ID(am_id);
uct_cm_enter(iface);
if (iface->num_outstanding >= iface->config.max_outstanding) {
status = UCS_ERR_NO_RESOURCE;
goto err;
}
/* Allocate temporary contiguous buffer */
hdr = ucs_malloc(IB_CM_SIDR_REQ_PRIVATE_DATA_SIZE, "cm_send_buf");
if (hdr == NULL) {
status = UCS_ERR_NO_MEMORY;
goto err;
}
payload_len = pack_cb(hdr + 1, arg);
hdr->am_id = am_id;
hdr->length = payload_len;
total_len = sizeof(*hdr) + payload_len;
status = uct_cm_ep_fill_path_rec(ep, &path);
if (status != UCS_OK) {
goto err_free;
}
/* Fill SIDR request */
memset(&req, 0, sizeof req);
req.path = &path;
req.service_id = ep->dest_addr.id;
req.timeout_ms = iface->config.timeout_ms;
req.private_data = hdr;
req.private_data_len = total_len;
req.max_cm_retries = iface->config.retry_count;
/* Create temporary ID for this message. Will be released when getting REP. */
ret = ib_cm_create_id(iface->cmdev, &id, NULL);
if (ret) {
ucs_error("ib_cm_create_id() failed: %m");
status = UCS_ERR_IO_ERROR;
goto err_free;
}
uct_cm_dump_path(&path);
ret = ib_cm_send_sidr_req(id, &req);
if (ret) {
ucs_error("ib_cm_send_sidr_req() failed: %m");
status = UCS_ERR_IO_ERROR;
goto err_destroy_id;
}
iface->outstanding[iface->num_outstanding++] = id;
UCT_TL_EP_STAT_OP(&ep->super, AM, BCOPY, payload_len);
uct_cm_leave(iface);
uct_cm_iface_trace_data(iface, UCT_AM_TRACE_TYPE_SEND, hdr,
"TX: SIDR_REQ [dlid %d svc 0x%"PRIx64"]",
ntohs(path.dlid), req.service_id);
ucs_free(hdr);
return payload_len;
err_destroy_id:
ib_cm_destroy_id(id);
err_free:
ucs_free(hdr);
err:
uct_cm_leave(iface);
return status;
}
static UCS_F_ALWAYS_INLINE ssize_t
uct_ugni_udt_ep_am_common_send(const unsigned is_short, uct_ugni_udt_ep_t *ep, uct_ugni_udt_iface_t *iface,
uint8_t am_id, unsigned length, uint64_t header,
const void *payload, uct_pack_callback_t pack_cb, void *arg)
{
gni_return_t ugni_rc;
uint16_t msg_length;
uct_ugni_udt_desc_t *desc;
uct_ugni_udt_header_t *sheader,
*rheader;
ssize_t packed_length;
UCT_CHECK_AM_ID(am_id);
if (ucs_unlikely(NULL != ep->posted_desc)) {
UCT_TL_IFACE_STAT_TX_NO_DESC(&iface->super.super);
return UCS_ERR_NO_RESOURCE;
}
UCT_TL_IFACE_GET_TX_DESC(&iface->super.super, &iface->free_desc,
desc, return UCS_ERR_NO_RESOURCE);
rheader = uct_ugni_udt_get_rheader(desc, iface);
rheader->type = UCT_UGNI_UDT_EMPTY;
sheader = uct_ugni_udt_get_sheader(desc, iface);
if (is_short) {
uint64_t *hdr = (uint64_t *)uct_ugni_udt_get_spayload(desc, iface);
*hdr = header;
memcpy((void*)(hdr + 1), payload, length);
sheader->length = length + sizeof(header);
msg_length = sheader->length + sizeof(*sheader);
UCT_TL_EP_STAT_OP(ucs_derived_of(ep, uct_base_ep_t), AM, SHORT, sizeof(header) + length);
} else {
packed_length = pack_cb((void *)uct_ugni_udt_get_spayload(desc, iface),
arg);
sheader->length = packed_length;
msg_length = sheader->length + sizeof(*sheader);
UCT_TL_EP_STAT_OP(ucs_derived_of(ep, uct_base_ep_t), AM, BCOPY, packed_length);
}
uct_iface_trace_am(&iface->super.super, UCT_AM_TRACE_TYPE_SEND, am_id,
uct_ugni_udt_get_spayload(desc, iface), length,
is_short ? "TX: AM_SHORT" : "TX: AM_BCOPY");
sheader->am_id = am_id;
sheader->type = UCT_UGNI_UDT_PAYLOAD;
ucs_assert_always(sheader->length <= GNI_DATAGRAM_MAXSIZE);
pthread_mutex_lock(&uct_ugni_global_lock);
ugni_rc = GNI_EpPostDataWId(ep->super.ep,
sheader, msg_length,
rheader, (uint16_t)iface->config.udt_seg_size,
ep->super.hash_key);
pthread_mutex_unlock(&uct_ugni_global_lock);
UCT_UGNI_UDT_CHECK_RC(ugni_rc);
ep->posted_desc = desc;
++ep->super.outstanding;
++iface->super.outstanding;
return is_short ? UCS_OK : packed_length;
}
/* A common mm active message sending function.
* The first parameter indicates the origin of the call.
* is_short = 1 - perform AM short sending
* is_short = 0 - perform AM bcopy sending
*/
static UCS_F_ALWAYS_INLINE ssize_t
uct_mm_ep_am_common_send(const unsigned is_short, uct_mm_ep_t *ep, uct_mm_iface_t *iface,
uint8_t am_id, size_t length, uint64_t header,
const void *payload, uct_pack_callback_t pack_cb, void *arg)
{
uct_mm_fifo_element_t *elem;
ucs_status_t status;
void *base_address;
uint64_t head;
UCT_CHECK_AM_ID(am_id);
head = ep->fifo_ctl->head;
/* check if there is room in the remote process's receive FIFO to write */
if (!UCT_MM_EP_IS_ABLE_TO_SEND(head, ep->cached_tail, iface->config.fifo_size)) {
if (!ucs_arbiter_group_is_empty(&ep->arb_group)) {
/* pending isn't empty. don't send now to prevent out-of-order sending */
UCS_STATS_UPDATE_COUNTER(ep->super.stats, UCT_EP_STAT_NO_RES, 1);
return UCS_ERR_NO_RESOURCE;
} else {
/* pending is empty */
/* update the local copy of the tail to its actual value on the remote peer */
uct_mm_ep_update_cached_tail(ep);
if (!UCT_MM_EP_IS_ABLE_TO_SEND(head, ep->cached_tail, iface->config.fifo_size)) {
UCS_STATS_UPDATE_COUNTER(ep->super.stats, UCT_EP_STAT_NO_RES, 1);
return UCS_ERR_NO_RESOURCE;
}
}
}
status = uct_mm_ep_get_remote_elem(ep, head, &elem);
if (status != UCS_OK) {
ucs_trace_poll("couldn't get an available FIFO element");
UCS_STATS_UPDATE_COUNTER(ep->super.stats, UCT_EP_STAT_NO_RES, 1);
return status;
}
if (is_short) {
/* AM_SHORT */
/* write to the remote FIFO */
*(uint64_t*) (elem + 1) = header;
memcpy((void*) (elem + 1) + sizeof(header), payload, length);
elem->flags |= UCT_MM_FIFO_ELEM_FLAG_INLINE;
elem->length = length + sizeof(header);
uct_iface_trace_am(&iface->super, UCT_AM_TRACE_TYPE_SEND, am_id,
elem + 1, length + sizeof(header), "TX: AM_SHORT");
UCT_TL_EP_STAT_OP(&ep->super, AM, SHORT, sizeof(header) + length);
} else {
/* AM_BCOPY */
/* write to the remote descriptor */
/* get the base_address: local ptr to remote memory chunk after attaching to it */
base_address = uct_mm_ep_attach_remote_seg(ep, iface, elem);
length = pack_cb(base_address + elem->desc_offset, arg);
elem->flags &= ~UCT_MM_FIFO_ELEM_FLAG_INLINE;
elem->length = length;
uct_iface_trace_am(&iface->super, UCT_AM_TRACE_TYPE_SEND, am_id,
base_address + elem->desc_offset, length, "TX: AM_BCOPY");
UCT_TL_EP_STAT_OP(&ep->super, AM, BCOPY, length);
}
elem->am_id = am_id;
/* memory barrier - make sure that the memory is flushed before setting the
* 'writing is complete' flag which the reader checks */
ucs_memory_cpu_store_fence();
/* change the owner bit to indicate that the writing is complete.
* the owner bit flips after every FIFO wraparound */
if (head & iface->config.fifo_size) {
elem->flags |= UCT_MM_FIFO_ELEM_FLAG_OWNER;
} else {
elem->flags &= ~UCT_MM_FIFO_ELEM_FLAG_OWNER;
}
if (is_short) {
return UCS_OK;
} else {
return length;
}
}
