ucs_status_t uct_rc_verbs_ep_connect_to_ep(uct_ep_h tl_ep,
const uct_device_addr_t *dev_addr,
const uct_ep_addr_t *ep_addr)
{
uct_rc_verbs_ep_t *ep = ucs_derived_of(tl_ep, uct_rc_verbs_ep_t);
uct_rc_verbs_iface_t *iface = ucs_derived_of(tl_ep->iface, uct_rc_verbs_iface_t);
const uct_ib_address_t *ib_addr = (const uct_ib_address_t *)dev_addr;
const uct_rc_ep_address_t *rc_addr = (const uct_rc_ep_address_t*)ep_addr;
struct ibv_ah_attr ah_attr;
ucs_status_t status;
uint32_t qp_num;
uct_ib_iface_fill_ah_attr(&iface->super.super, ib_addr, ep->super.path_bits,
&ah_attr);
#if HAVE_IBV_EX_HW_TM
if(UCT_RC_VERBS_TM_ENABLED(iface)) {
/* For HW TM we need 2 QPs, one of which will be used by the device for
* RNDV offload (for issuing RDMA reads and sending RNDV ACK). No WQEs
* should be posted to the send side of the QP which is owned by device. */
uct_rc_verbs_ep_tm_address_t *tm_addr = ucs_derived_of(rc_addr,
uct_rc_verbs_ep_tm_address_t);
ucs_assert_always(tm_addr->super.type == UCT_RC_EP_ADDR_TYPE_TM);
status = uct_rc_iface_qp_connect(&iface->super, ep->tm_qp,
uct_ib_unpack_uint24(rc_addr->qp_num),
&ah_attr);
if (status != UCS_OK) {
return status;
}
/* Need to connect local ep QP to the one owned by device
* (and bound to XRQ) on the peer. */
qp_num = uct_ib_unpack_uint24(tm_addr->tm_qp_num);
} else
#endif
{
ucs_assert_always(rc_addr->type == UCT_RC_EP_ADDR_TYPE_BASIC);
qp_num = uct_ib_unpack_uint24(rc_addr->qp_num);
}
status = uct_rc_iface_qp_connect(&iface->super, ep->super.txqp.qp,
qp_num, &ah_attr);
if (status != UCS_OK) {
return status;
}
ep->super.atomic_mr_offset = uct_ib_md_atomic_offset(rc_addr->atomic_mr_id);
return UCS_OK;
}
ucs_status_t uct_ib_iface_create_ah(uct_ib_iface_t *iface,
const uct_ib_address_t *ib_addr,
uint8_t path_bits,
struct ibv_ah **ah_p,
int *is_global_p)
{
struct ibv_ah_attr ah_attr;
struct ibv_ah *ah;
char buf[128];
char *p, *endp;
uct_ib_iface_fill_ah_attr(iface, ib_addr, path_bits, &ah_attr);
ah = ibv_create_ah(uct_ib_iface_md(iface)->pd, &ah_attr);
if (ah == NULL) {
p = buf;
endp = buf + sizeof(buf);
snprintf(p, endp - p, "dlid=%d sl=%d port=%d src_path_bits=%d",
ah_attr.dlid, ah_attr.sl, ah_attr.port_num, ah_attr.src_path_bits);
p += strlen(p);
if (ah_attr.is_global) {
snprintf(p, endp - p, " dgid=");
p += strlen(p);
inet_ntop(AF_INET6, &ah_attr.grh.dgid, p, endp - p);
p += strlen(p);
snprintf(p, endp - p, " sgid_index=%d", ah_attr.grh.sgid_index);
}
ucs_error("ibv_create_ah(%s) on "UCT_IB_IFACE_FMT" failed: %m", buf,
UCT_IB_IFACE_ARG(iface));
return UCS_ERR_INVALID_ADDR;
}
*ah_p = ah;
*is_global_p = ah_attr.is_global;
return UCS_OK;
}
