static UCS_CLASS_INIT_FUNC(uct_dc_mlx5_ep_t, uct_iface_t *tl_iface,
const uct_device_addr_t *dev_addr,
const uct_iface_addr_t *iface_addr)
{
uct_dc_mlx5_iface_t *iface = ucs_derived_of(tl_iface, uct_dc_mlx5_iface_t);
const uct_ib_address_t *ib_addr = (const uct_ib_address_t *)dev_addr;
const uct_dc_iface_addr_t *if_addr = (const uct_dc_iface_addr_t *)iface_addr;
ucs_status_t status;
int is_global;
ucs_trace_func("");
UCS_CLASS_CALL_SUPER_INIT(uct_dc_ep_t, &iface->super, if_addr);
status = uct_ud_mlx5_iface_get_av(&iface->super.super.super, &iface->ud_common,
ib_addr, iface->super.super.super.path_bits[0],
&self->av, NULL, &is_global);
if (status != UCS_OK) {
return UCS_ERR_INVALID_ADDR;
}
self->av.dqp_dct |= htonl(uct_ib_unpack_uint24(if_addr->qp_num));
ucs_debug("created ep %p on iface %p", self, iface);
return UCS_OK;
}
static UCS_CLASS_CLEANUP_FUNC(uct_ud_mlx5_iface_t)
{
ucs_trace_func("");
uct_worker_progress_unregister(self->super.super.super.worker,
uct_ud_mlx5_iface_progress, self);
uct_ib_mlx5_put_txwq(self->super.super.super.worker, &self->tx.wq);
}
static ucs_status_t recieve_datagram(uct_ugni_udt_iface_t *iface, uint64_t id, uct_ugni_udt_ep_t **ep_out)
{
uint32_t rem_addr, rem_id;
gni_post_state_t post_state;
gni_return_t ugni_rc;
uct_ugni_udt_ep_t *ep;
gni_ep_handle_t gni_ep;
uct_ugni_udt_desc_t *desc;
uct_ugni_udt_header_t *header;
ucs_trace_func("iface=%p, id=%lx", iface, id);
if (UCT_UGNI_UDT_ANY == id) {
ep = NULL;
gni_ep = iface->ep_any;
desc = iface->desc_any;
} else {
ep = ucs_derived_of(uct_ugni_iface_lookup_ep(&iface->super, id),
uct_ugni_udt_ep_t);
gni_ep = ep->super.ep;
desc = ep->posted_desc;
}
*ep_out = ep;
uct_ugni_device_lock(&iface->super.cdm);
ugni_rc = GNI_EpPostDataWaitById(gni_ep, id, -1, &post_state, &rem_addr, &rem_id);
uct_ugni_device_unlock(&iface->super.cdm);
if (ucs_unlikely(GNI_RC_SUCCESS != ugni_rc)) {
ucs_error("GNI_EpPostDataWaitById, id=%lu Error status: %s %d",
id, gni_err_str[ugni_rc], ugni_rc);
return UCS_ERR_IO_ERROR;
}
if (GNI_POST_TERMINATED == post_state) {
return UCS_ERR_CANCELED;
}
if (GNI_POST_COMPLETED != post_state) {
ucs_error("GNI_EpPostDataWaitById gave unexpected response: %u", post_state);
return UCS_ERR_IO_ERROR;
}
if (UCT_UGNI_UDT_ANY != id) {
--iface->super.outstanding;
}
header = uct_ugni_udt_get_rheader(desc, iface);
ucs_trace("Got datagram id: %lu type: %i len: %i am_id: %i", id, header->type, header->length, header->am_id);
if (UCT_UGNI_UDT_PAYLOAD != header->type) {
/* ack message, no data */
ucs_assert_always(NULL != ep);
ucs_mpool_put(ep->posted_desc);
uct_ugni_check_flush(ep->desc_flush_group);
ep->posted_desc = NULL;
return UCS_OK;
}
return UCS_INPROGRESS;
}
ucs_status_t uct_ud_ep_connect_to_ep(uct_ud_ep_t *ep,
const struct sockaddr *addr)
{
uct_ud_iface_t *iface = ucs_derived_of(ep->super.super.iface, uct_ud_iface_t);
uct_ib_device_t *dev = uct_ib_iface_device(&iface->super);
const uct_sockaddr_ib_t *ib_addr = (uct_sockaddr_ib_t *)addr;
ucs_assert_always(ep->dest_ep_id == UCT_UD_EP_NULL_ID);
ucs_trace_func("");
ep->dest_ep_id = ib_addr->id;
ep->dest_qpn = ib_addr->qp_num;
uct_ud_ep_reset(ep);
ucs_debug("%s:%d slid=%d qpn=%d ep=%u connected to dlid=%d qpn=%d ep=%u",
ibv_get_device_name(dev->ibv_context->device),
iface->super.port_num,
dev->port_attr[iface->super.port_num-dev->first_port].lid,
iface->qp->qp_num,
ep->ep_id,
ib_addr->lid, ep->dest_qpn, ep->dest_ep_id);
return UCS_OK;
}
ucs_status_t ucs_timerq_remove(ucs_timer_queue_t *timerq, int timer_id)
{
ucs_status_t status;
ucs_timer_t *ptr;
ucs_trace_func("timerq=%p timer_id=%d", timerq, timer_id);
status = UCS_ERR_NO_ELEM;
pthread_spin_lock(&timerq->lock);
timerq->min_interval = UCS_TIME_INFINITY;
ptr = timerq->timers;
while (ptr < timerq->timers + timerq->num_timers) {
if (ptr->id == timer_id) {
*ptr = timerq->timers[--timerq->num_timers];
status = UCS_OK;
} else {
timerq->min_interval = ucs_min(timerq->min_interval, ptr->interval);
++ptr;
}
}
/* TODO realloc - shrink */
if (timerq->num_timers == 0) {
ucs_assert(timerq->min_interval == UCS_TIME_INFINITY);
free(timerq->timers);
timerq->timers = NULL;
} else {
ucs_assert(timerq->min_interval != UCS_TIME_INFINITY);
}
pthread_spin_unlock(&timerq->lock);
return status;
}
static UCS_CLASS_INIT_FUNC(uct_self_iface_t, uct_md_h md, uct_worker_h worker,
const uct_iface_params_t *params,
const uct_iface_config_t *tl_config)
{
ucs_status_t status;
uct_self_iface_config_t *self_config = 0;
ucs_trace_func("Creating a loop-back transport self=%p rxh=%lu",
self, params->rx_headroom);
if (strcmp(params->dev_name, UCT_SELF_NAME) != 0) {
ucs_error("No device was found: %s", params->dev_name);
return UCS_ERR_NO_DEVICE;
}
UCS_CLASS_CALL_SUPER_INIT(uct_base_iface_t, &uct_self_iface_ops, md, worker,
tl_config UCS_STATS_ARG(params->stats_root)
UCS_STATS_ARG(UCT_SELF_NAME));
self_config = ucs_derived_of(tl_config, uct_self_iface_config_t);
self->id = ucs_generate_uuid((uintptr_t)self);
self->rx_headroom = params->rx_headroom;
self->data_length = self_config->super.max_bcopy;
self->release_desc.cb = uct_self_iface_release_desc;
/* create a memory pool for data transferred */
status = uct_iface_mpool_init(&self->super,
&self->msg_desc_mp,
sizeof(uct_recv_desc_t) + self->rx_headroom +
self->data_length,
sizeof(uct_recv_desc_t) + self->rx_headroom,
UCS_SYS_CACHE_LINE_SIZE,
&self_config->mp,
256,
ucs_empty_function,
"self_msg_desc");
if (UCS_OK != status) {
ucs_error("Failed to create a memory pool for the loop-back transport");
goto err;
}
/* set the message descriptor for the loop-back */
self->msg_cur_desc = ucs_mpool_get(&self->msg_desc_mp);
VALGRIND_MAKE_MEM_DEFINED(self->msg_cur_desc, sizeof(*(self->msg_cur_desc)));
if (NULL == self->msg_cur_desc) {
ucs_error("Failed to get the first descriptor in loop-back MP storage");
status = UCS_ERR_NO_RESOURCE;
goto destroy_mpool;
}
ucs_debug("Created a loop-back iface. id=0x%lx, desc=%p, len=%u, tx_hdr=%lu",
self->id, self->msg_cur_desc, self->data_length, self->rx_headroom);
return UCS_OK;
destroy_mpool:
ucs_mpool_cleanup(&self->msg_desc_mp, 1);
err:
return status;
}
static ucs_status_t uct_self_iface_query(uct_iface_h iface, uct_iface_attr_t *attr)
{
uct_self_iface_t *self_iface = ucs_derived_of(iface, uct_self_iface_t);
ucs_trace_func("iface=%p", iface);
memset(attr, 0, sizeof(*attr));
attr->iface_addr_len = sizeof(uct_self_iface_addr_t);
attr->device_addr_len = 0;
attr->ep_addr_len = 0; /* No UCT_IFACE_FLAG_CONNECT_TO_EP supported */
attr->cap.flags = UCT_IFACE_FLAG_CONNECT_TO_IFACE |
UCT_IFACE_FLAG_AM_SHORT |
UCT_IFACE_FLAG_AM_BCOPY |
UCT_IFACE_FLAG_PUT_SHORT |
UCT_IFACE_FLAG_PUT_BCOPY |
UCT_IFACE_FLAG_GET_BCOPY |
UCT_IFACE_FLAG_ATOMIC_ADD32 |
UCT_IFACE_FLAG_ATOMIC_ADD64 |
UCT_IFACE_FLAG_ATOMIC_FADD64 |
UCT_IFACE_FLAG_ATOMIC_FADD32 |
UCT_IFACE_FLAG_ATOMIC_SWAP64 |
UCT_IFACE_FLAG_ATOMIC_SWAP32 |
UCT_IFACE_FLAG_ATOMIC_CSWAP64 |
UCT_IFACE_FLAG_ATOMIC_CSWAP32 |
UCT_IFACE_FLAG_ATOMIC_CPU |
UCT_IFACE_FLAG_PENDING |
UCT_IFACE_FLAG_AM_CB_SYNC;
attr->cap.put.max_short = UINT_MAX;
attr->cap.put.max_bcopy = SIZE_MAX;
attr->cap.put.min_zcopy = 0;
attr->cap.put.max_zcopy = 0;
attr->cap.put.opt_zcopy_align = UCS_SYS_CACHE_LINE_SIZE;
attr->cap.put.align_mtu = attr->cap.put.opt_zcopy_align;
attr->cap.put.max_iov = 1;
attr->cap.get.max_bcopy = SIZE_MAX;
attr->cap.get.min_zcopy = 0;
attr->cap.get.max_zcopy = 0;
attr->cap.get.opt_zcopy_align = UCS_SYS_CACHE_LINE_SIZE;
attr->cap.get.align_mtu = attr->cap.get.opt_zcopy_align;
attr->cap.get.max_iov = 1;
attr->cap.am.max_short = self_iface->data_length;
attr->cap.am.max_bcopy = self_iface->data_length;
attr->cap.am.min_zcopy = 0;
attr->cap.am.max_zcopy = 0;
attr->cap.am.opt_zcopy_align = UCS_SYS_CACHE_LINE_SIZE;
attr->cap.am.align_mtu = attr->cap.am.opt_zcopy_align;
attr->cap.am.max_hdr = 0;
attr->cap.am.max_iov = 1;
attr->latency.overhead = 0;
attr->latency.growth = 0;
attr->bandwidth = 6911 * 1024.0 * 1024.0;
attr->overhead = 0;
attr->priority = 0;
return UCS_OK;
}
static UCS_CLASS_INIT_FUNC(uct_ud_mlx5_ep_t, uct_iface_h tl_iface)
{
uct_ud_mlx5_iface_t *iface = ucs_derived_of(tl_iface, uct_ud_mlx5_iface_t);
ucs_trace_func("");
UCS_CLASS_CALL_SUPER_INIT(uct_ud_ep_t, &iface->super);
return UCS_OK;
}
void uct_ugni_proccess_datagram_pipe(int event_id, void *arg) {
uct_ugni_udt_iface_t *iface = (uct_ugni_udt_iface_t *)arg;
uct_ugni_udt_ep_t *ep;
uct_ugni_udt_desc_t *datagram;
ucs_status_t status;
void *user_desc;
gni_return_t ugni_rc;
uint64_t id;
ucs_trace_func("");
uct_ugni_device_lock(&iface->super.cdm);
ugni_rc = GNI_PostDataProbeById(uct_ugni_udt_iface_nic_handle(iface), &id);
uct_ugni_device_unlock(&iface->super.cdm);
while (GNI_RC_SUCCESS == ugni_rc) {
status = recieve_datagram(iface, id, &ep);
if (UCS_INPROGRESS == status) {
if (ep != NULL){
ucs_trace_data("Processing reply");
datagram = ep->posted_desc;
status = processs_datagram(iface, datagram);
if (UCS_OK != status) {
user_desc = uct_ugni_udt_get_user_desc(datagram, iface);
uct_recv_desc(user_desc) = &iface->release_desc;
} else {
ucs_mpool_put(datagram);
}
ep->posted_desc = NULL;
uct_ugni_check_flush(ep->desc_flush_group);
} else {
ucs_trace_data("Processing wildcard");
datagram = iface->desc_any;
status = processs_datagram(iface, datagram);
if (UCS_OK != status) {
UCT_TL_IFACE_GET_TX_DESC(&iface->super.super, &iface->free_desc,
iface->desc_any, iface->desc_any=NULL);
user_desc = uct_ugni_udt_get_user_desc(datagram, iface);
uct_recv_desc(user_desc) = &iface->release_desc;
}
status = uct_ugni_udt_ep_any_post(iface);
if (UCS_OK != status) {
/* We can't continue if we can't post the first receive */
ucs_error("Failed to post wildcard request");
return;
}
}
}
uct_ugni_device_lock(&iface->super.cdm);
ugni_rc = GNI_PostDataProbeById(uct_ugni_udt_iface_nic_handle(iface), &id);
uct_ugni_device_unlock(&iface->super.cdm);
}
ucs_async_pipe_drain(&iface->event_pipe);
pthread_mutex_lock(&iface->device_lock);
iface->events_ready = 0;
pthread_mutex_unlock(&iface->device_lock);
ucs_trace("Signaling device thread to resume monitoring");
pthread_cond_signal(&iface->device_condition);
}
static UCS_CLASS_CLEANUP_FUNC(uct_dc_mlx5_iface_t)
{
ucs_trace_func("");
uct_worker_progress_unregister(self->super.super.super.super.worker,
uct_dc_mlx5_iface_progress, self);
uct_rc_mlx5_iface_common_cleanup(&self->mlx5_common);
}
static UCS_CLASS_CLEANUP_FUNC(uct_ud_verbs_ep_t)
{
ucs_trace_func("");
if (self->ah) {
ibv_destroy_ah(self->ah);
self->ah = NULL;
}
}
static void uct_self_iface_release_am_desc(uct_iface_t *tl_iface, void *desc)
{
uct_am_recv_desc_t *self_desc = 0;
self_desc = (uct_am_recv_desc_t *) desc - 1;
ucs_trace_func("iface=%p, desc=%p", tl_iface, self_desc);
ucs_mpool_put(self_desc);
}
static UCS_CLASS_CLEANUP_FUNC(uct_ud_ep_t)
{
uct_ud_iface_t *iface = ucs_derived_of(self->super.super.iface, uct_ud_iface_t);
ucs_trace_func("ep=%p id=%d conn_id=%d", self, self->ep_id, self->conn_id);
uct_ud_iface_remove_ep(iface, self);
uct_ud_iface_cep_remove(self);
/* TODO: in disconnect ucs_frag_list_cleanup(&self->rx.ooo_pkts); */
}
unsigned uct_tcp_ep_progress_rx(uct_tcp_ep_t *ep)
{
uct_tcp_iface_t *iface = ucs_derived_of(ep->super.super.iface,
uct_tcp_iface_t);
uct_tcp_am_hdr_t *hdr;
ucs_status_t status;
size_t recv_length;
ssize_t remainder;
ucs_trace_func("ep=%p", ep);
/* Receive next chunk of data */
recv_length = iface->config.buf_size - ep->length;
status = uct_tcp_recv(ep->fd, ep->buf + ep->length, &recv_length);
if (status != UCS_OK) {
if (status == UCS_ERR_CANCELED) {
ucs_debug("tcp_ep %p: remote disconnected", ep);
uct_tcp_ep_mod_events(ep, 0, EPOLLIN);
uct_tcp_ep_destroy(&ep->super.super);
}
return 0;
}
ep->length += recv_length;
ucs_trace_data("tcp_ep %p: recvd %zu bytes", ep, recv_length);
/* Parse received active messages */
while ((remainder = ep->length - ep->offset) >= sizeof(*hdr)) {
hdr = ep->buf + ep->offset;
if (remainder < sizeof(*hdr) + hdr->length) {
break;
}
/* Full message was received */
ep->offset += sizeof(*hdr) + hdr->length;
if (hdr->am_id >= UCT_AM_ID_MAX) {
ucs_error("invalid am id: %d", hdr->am_id);
continue;
}
uct_iface_trace_am(&iface->super, UCT_AM_TRACE_TYPE_RECV, hdr->am_id,
hdr + 1, hdr->length, "RECV fd %d", ep->fd);
uct_iface_invoke_am(&iface->super, hdr->am_id, hdr + 1,
hdr->length, 0);
}
/* Move the remaining data to the beginning of the buffer
* TODO avoid extra copy on partial receive
*/
ucs_assert(remainder >= 0);
memmove(ep->buf, ep->buf + ep->offset, remainder);
ep->offset = 0;
ep->length = remainder;
return recv_length > 0;
}
void ucs_timerq_cleanup(ucs_timer_queue_t *timerq)
{
ucs_trace_func("timerq=%p", timerq);
if (timerq->num_timers > 0) {
ucs_warn("timer queue with %d timers being destroyed", timerq->num_timers);
}
ucs_free(timerq->timers);
}
static ucs_status_t uct_ud_verbs_iface_query(uct_iface_h tl_iface, uct_iface_attr_t *iface_attr)
{
uct_ud_iface_t *iface = ucs_derived_of(tl_iface, uct_ud_iface_t);
ucs_trace_func("");
uct_ud_iface_query(iface, iface_attr);
return UCS_OK;
}
static UCS_CLASS_CLEANUP_FUNC(uct_self_iface_t)
{
ucs_trace_func("self=%p", self);
if (self->msg_cur_desc) {
ucs_mpool_put(self->msg_cur_desc);
}
ucs_mpool_cleanup(&self->msg_desc_mp, 1);
}
static unsigned uct_rdmacm_client_err_handle_progress(void *arg)
{
uct_rdmacm_ep_t *ep = arg;
ucs_trace_func("err_handle ep=%p",ep);
ep->slow_prog_id = UCS_CALLBACKQ_ID_NULL;
uct_set_ep_failed(&UCS_CLASS_NAME(uct_rdmacm_ep_t), &ep->super.super,
ep->super.super.iface, UCS_ERR_IO_ERROR);
return 0;
}
static ucs_status_t uct_self_iface_get_address(uct_iface_h iface,
uct_iface_addr_t *addr)
{
const uct_self_iface_t *self_iface = 0;
ucs_trace_func("iface=%p", iface);
self_iface = ucs_derived_of(iface, uct_self_iface_t);
*(uct_self_iface_addr_t*)addr = self_iface->id;
return UCS_OK;
}
static void uct_ud_ep_rx_ctl(uct_ud_iface_t *iface, uct_ud_ep_t *ep, uct_ud_ctl_hdr_t *ctl)
{
ucs_trace_func("");
ucs_assert_always(ctl->type == UCT_UD_PACKET_CREP);
/* note that duplicate creps are discared earlier */
ucs_assert_always(ep->dest_ep_id == UCT_UD_EP_NULL_ID ||
ep->dest_ep_id == ctl->conn_rep.src_ep_id);
ep->dest_ep_id = ctl->conn_rep.src_ep_id;
/* TODO notify */
}
static UCS_CLASS_CLEANUP_FUNC(uct_ud_mlx5_iface_t)
{
ucs_trace_func("");
uct_ud_enter(&self->super);
uct_worker_progress_unregister(self->super.super.super.worker,
uct_ud_mlx5_iface_progress, self);
uct_ib_mlx5_put_txwq(self->super.super.super.worker, &self->tx.wq);
UCT_UD_IFACE_DELETE_EPS(&self->super, uct_ud_mlx5_ep_t);
uct_ud_enter(&self->super);
}
static ucs_status_t
uct_ud_mlx5_query_resources(uct_pd_h pd,
uct_tl_resource_desc_t **resources_p,
unsigned *num_resources_p)
{
ucs_trace_func("");
/* TODO take transport overhead into account */
return uct_ib_device_query_tl_resources(&ucs_derived_of(pd, uct_ib_pd_t)->dev,
"ud_mlx5", UCT_IB_DEVICE_FLAG_MLX5_PRM,
resources_p, num_resources_p);
}
ucs_status_t ucs_timerq_init(ucs_timer_queue_t *timerq)
{
ucs_trace_func("timerq=%p", timerq);
pthread_spin_init(&timerq->lock, 0);
timerq->timers = NULL;
timerq->num_timers = 0;
/* coverity[missing_lock] */
timerq->min_interval = UCS_TIME_INFINITY;
return UCS_OK;
}
UCS_CLASS_INIT_FUNC(uct_dc_mlx5_grh_ep_t, uct_dc_mlx5_iface_t *iface,
const uct_dc_mlx5_iface_addr_t *if_addr,
uct_ib_mlx5_base_av_t *av,
struct mlx5_grh_av *grh_av)
{
ucs_trace_func("");
UCS_CLASS_CALL_SUPER_INIT(uct_dc_mlx5_ep_t, iface, if_addr, av);
self->super.flags |= UCT_DC_MLX5_EP_FLAG_GRH;
memcpy(&self->grh_av, grh_av, sizeof(*grh_av));
return UCS_OK;
}
static int uct_self_iface_is_reachable(const uct_iface_h iface, const uct_device_addr_t *dev_addr,
const uct_iface_addr_t *iface_addr)
{
const uct_self_iface_t *self_iface = NULL;
const uct_self_iface_addr_t *self_addr = NULL;
if (NULL == iface_addr) {
return 0;
}
self_iface = ucs_derived_of(iface, uct_self_iface_t);
self_addr = (const uct_self_iface_addr_t *) iface_addr;
ucs_trace_func("iface=%p id=%lx addr=%lx", iface, self_iface->id, *self_addr);
return self_iface->id == *self_addr;
}
UCS_CLASS_INIT_FUNC(uct_ud_ep_t, uct_ud_iface_t *iface)
{
ucs_trace_func("");
memset(self, 0, sizeof(*self));
UCS_CLASS_CALL_SUPER_INIT(uct_base_ep_t, &iface->super.super);
self->dest_ep_id = UCT_UD_EP_NULL_ID;
uct_ud_ep_reset(self);
ucs_list_head_init(&self->cep_list);
uct_ud_iface_add_ep(iface, self);
UCT_UD_EP_HOOK_INIT(self);
ucs_debug("NEW EP: iface=%p ep=%p id=%d", iface, self, self->ep_id);
return UCS_OK;
}
static UCS_CLASS_INIT_FUNC(uct_dc_mlx5_iface_t, uct_md_h md, uct_worker_h worker,
const uct_iface_params_t *params,
const uct_iface_config_t *tl_config)
{
uct_dc_mlx5_iface_config_t *config = ucs_derived_of(tl_config,
uct_dc_mlx5_iface_config_t);
ucs_status_t status;
ucs_trace_func("");
UCS_CLASS_CALL_SUPER_INIT(uct_dc_iface_t, &uct_dc_mlx5_iface_ops, md,
worker, params, 0, &config->super);
status = uct_rc_mlx5_iface_common_init(&self->mlx5_common, &self->super.super,
&config->super.super);
if (status != UCS_OK) {
goto err;
}
status = uct_ud_mlx5_iface_common_init(&self->super.super.super,
&self->ud_common, &config->ud_common);
if (status != UCS_OK) {
goto err_rc_mlx5_common_cleanup;
}
status = uct_dc_mlx5_iface_init_dcis(self);
if (status != UCS_OK) {
goto err_rc_mlx5_common_cleanup;
}
uct_dc_iface_set_quota(&self->super, &config->super);
/* Set max_iov for put_zcopy and get_zcopy */
uct_ib_iface_set_max_iov(&self->super.super.super,
((UCT_IB_MLX5_MAX_BB * MLX5_SEND_WQE_BB) -
sizeof(struct mlx5_wqe_raddr_seg) -
sizeof(struct mlx5_wqe_ctrl_seg) -
UCT_IB_MLX5_AV_FULL_SIZE) /
sizeof(struct mlx5_wqe_data_seg));
/* TODO: only register progress when we have a connection */
uct_worker_progress_register(worker, uct_dc_mlx5_iface_progress, self);
ucs_debug("created dc iface %p", self);
return UCS_OK;
err_rc_mlx5_common_cleanup:
uct_rc_mlx5_iface_common_cleanup(&self->mlx5_common);
err:
return status;
}
static UCS_CLASS_INIT_FUNC(uct_ud_verbs_iface_t, uct_pd_h pd, uct_worker_h worker,
const char *dev_name, size_t rx_headroom,
const uct_iface_config_t *tl_config)
{
uct_ud_iface_config_t *config = ucs_derived_of(tl_config,
uct_ud_iface_config_t);
ucs_trace_func("");
UCS_CLASS_CALL_SUPER_INIT(uct_ud_iface_t, &uct_ud_verbs_iface_ops, pd,
worker, dev_name, rx_headroom, 0, config);
self->super.ops.async_progress = uct_ud_verbs_iface_async_progress;
self->super.ops.tx_skb = uct_ud_verbs_ep_tx_ctl_skb;
if (self->super.config.rx_max_batch < UCT_IB_MAX_WC) {
ucs_warn("max batch is too low (%d < %d), performance may be impacted",
self->super.config.rx_max_batch,
UCT_IB_MAX_WC);
}
while (self->super.rx.available >= self->super.config.rx_max_batch) {
uct_ud_verbs_iface_post_recv(self);
}
memset(&self->tx.wr_inl, 0, sizeof(self->tx.wr_inl));
self->tx.wr_inl.opcode = IBV_WR_SEND;
self->tx.wr_inl.wr_id = 0xBEEBBEEB;
self->tx.wr_inl.wr.ud.remote_qkey = UCT_IB_QKEY;
self->tx.wr_inl.imm_data = 0;
self->tx.wr_inl.next = 0;
self->tx.wr_inl.sg_list = self->tx.sge;
self->tx.wr_inl.num_sge = UCT_UD_MAX_SGE;
memset(&self->tx.wr_skb, 0, sizeof(self->tx.wr_skb));
self->tx.wr_skb.opcode = IBV_WR_SEND;
self->tx.wr_skb.wr_id = 0xFAAFFAAF;
self->tx.wr_skb.wr.ud.remote_qkey = UCT_IB_QKEY;
self->tx.wr_skb.imm_data = 0;
self->tx.wr_skb.next = 0;
self->tx.wr_skb.sg_list = self->tx.sge;
self->tx.wr_skb.num_sge = 1;
/* TODO: add progress on first ep creation */
uct_worker_progress_register(worker, uct_ud_verbs_iface_progress, self);
uct_ud_leave(&self->super);
return UCS_OK;
}
static UCS_CLASS_CLEANUP_FUNC(uct_cm_iface_t)
{
ucs_trace_func("");
ucs_async_unset_event_handler(self->cmdev->fd);
uct_cm_enter(self);
while (self->num_outstanding > 0) {
ib_cm_destroy_id(uct_cm_iface_outstanding_pop(self));
}
ib_cm_destroy_id(self->listen_id);
ib_cm_close_device(self->cmdev);
uct_cm_leave(self);
ucs_free(self->outstanding);
}
static UCS_CLASS_INIT_FUNC(uct_ud_verbs_iface_t, uct_pd_h pd, uct_worker_h worker,
const char *dev_name, size_t rx_headroom,
const uct_iface_config_t *tl_config)
{
uct_ud_iface_config_t *config = ucs_derived_of(tl_config, uct_ud_iface_config_t);
ucs_trace_func("");
UCS_CLASS_CALL_SUPER_INIT(uct_ud_iface_t, &uct_ud_verbs_iface_ops, pd,
worker, dev_name, rx_headroom, 0, config);
while (self->super.rx.available >= self->super.config.rx_max_batch) {
uct_ud_verbs_iface_post_recv(self);
}
memset(&self->tx.wr_inl, 0, sizeof(self->tx.wr_inl));
self->tx.wr_inl.opcode = IBV_WR_SEND;
self->tx.wr_inl.wr_id = 0xBEEBBEEB;
self->tx.wr_inl.wr.ud.remote_qkey = UCT_UD_QKEY;
self->tx.wr_inl.imm_data = 0;
self->tx.wr_inl.next = 0;
self->tx.wr_inl.sg_list = self->tx.sge;
self->tx.wr_inl.num_sge = UCT_UD_MAX_SGE;
memset(&self->tx.wr_bcp, 0, sizeof(self->tx.wr_bcp));
self->tx.wr_bcp.opcode = IBV_WR_SEND;
self->tx.wr_bcp.wr_id = 0xFAAFFAAF;
self->tx.wr_bcp.wr.ud.remote_qkey = UCT_UD_QKEY;
self->tx.wr_bcp.imm_data = 0;
self->tx.wr_bcp.next = 0;
self->tx.wr_bcp.sg_list = self->tx.sge;
self->tx.wr_bcp.num_sge = 1;
memset(&self->tx.wr_ctl, 0, sizeof(self->tx.wr_ctl));
self->tx.wr_ctl.opcode = IBV_WR_SEND;
self->tx.wr_ctl.wr_id = 0xCCCCCCCC;
self->tx.wr_ctl.wr.ud.remote_qkey = UCT_UD_QKEY;
self->tx.wr_ctl.imm_data = 0;
self->tx.wr_ctl.next = 0;
self->tx.wr_ctl.sg_list = self->tx.sge;
self->tx.wr_ctl.num_sge = 1;
/* TODO: add progress on first ep creation */
ucs_notifier_chain_add(&worker->progress_chain, uct_ud_verbs_iface_progress,
self);
return UCS_OK;
}
