/* Endpoint definition */
UCS_CLASS_INIT_FUNC(uct_ugni_ep_t, const uct_ep_params_t *params)
{
uct_ugni_iface_t *iface = ucs_derived_of(params->iface, uct_ugni_iface_t);
ucs_status_t rc = UCS_OK;
gni_return_t ugni_rc;
uint32_t *big_hash;
self->arb_sched = 0;
UCS_CLASS_CALL_SUPER_INIT(uct_base_ep_t, &iface->super);
self->flush_group = uct_ugni_new_flush_group(iface);
#ifdef DEBUG
self->flush_group->flush_comp.func = NULL;
self->flush_group->parent = NULL;
#endif
uct_ugni_cdm_lock(&iface->cdm);
ugni_rc = GNI_EpCreate(uct_ugni_iface_nic_handle(iface), iface->local_cq, &self->ep);
uct_ugni_cdm_unlock(&iface->cdm);
if (GNI_RC_SUCCESS != ugni_rc) {
ucs_error("GNI_CdmCreate failed, Error status: %s %d",
gni_err_str[ugni_rc], ugni_rc);
return UCS_ERR_NO_DEVICE;
}
ucs_arbiter_group_init(&self->arb_group);
big_hash = (void *)&self->ep;
self->hash_key = big_hash[0];
if (uct_ugni_check_device_type(iface, GNI_DEVICE_ARIES)) {
self->hash_key &= 0x00FFFFFF;
}
ucs_debug("Adding ep hash %x to iface %p", self->hash_key, iface);
sglib_hashed_uct_ugni_ep_t_add(iface->eps, self);
return rc;
}
static UCS_CLASS_INIT_FUNC(uct_dc_mlx5_ep_t, uct_iface_t *tl_iface,
const uct_device_addr_t *dev_addr,
const uct_iface_addr_t *iface_addr)
{
uct_dc_mlx5_iface_t *iface = ucs_derived_of(tl_iface, uct_dc_mlx5_iface_t);
const uct_ib_address_t *ib_addr = (const uct_ib_address_t *)dev_addr;
const uct_dc_iface_addr_t *if_addr = (const uct_dc_iface_addr_t *)iface_addr;
ucs_status_t status;
int is_global;
ucs_trace_func("");
UCS_CLASS_CALL_SUPER_INIT(uct_dc_ep_t, &iface->super, if_addr);
status = uct_ud_mlx5_iface_get_av(&iface->super.super.super, &iface->ud_common,
ib_addr, iface->super.super.super.path_bits[0],
&self->av, NULL, &is_global);
if (status != UCS_OK) {
return UCS_ERR_INVALID_ADDR;
}
self->av.dqp_dct |= htonl(uct_ib_unpack_uint24(if_addr->qp_num));
ucs_debug("created ep %p on iface %p", self, iface);
return UCS_OK;
}
static UCS_CLASS_INIT_FUNC(uct_self_iface_t, uct_md_h md, uct_worker_h worker,
const uct_iface_params_t *params,
const uct_iface_config_t *tl_config)
{
ucs_status_t status;
uct_self_iface_config_t *self_config = 0;
ucs_trace_func("Creating a loop-back transport self=%p rxh=%lu",
self, params->rx_headroom);
if (strcmp(params->dev_name, UCT_SELF_NAME) != 0) {
ucs_error("No device was found: %s", params->dev_name);
return UCS_ERR_NO_DEVICE;
}
UCS_CLASS_CALL_SUPER_INIT(uct_base_iface_t, &uct_self_iface_ops, md, worker,
tl_config UCS_STATS_ARG(params->stats_root)
UCS_STATS_ARG(UCT_SELF_NAME));
self_config = ucs_derived_of(tl_config, uct_self_iface_config_t);
self->id = ucs_generate_uuid((uintptr_t)self);
self->rx_headroom = params->rx_headroom;
self->data_length = self_config->super.max_bcopy;
self->release_desc.cb = uct_self_iface_release_desc;
/* create a memory pool for data transferred */
status = uct_iface_mpool_init(&self->super,
&self->msg_desc_mp,
sizeof(uct_recv_desc_t) + self->rx_headroom +
self->data_length,
sizeof(uct_recv_desc_t) + self->rx_headroom,
UCS_SYS_CACHE_LINE_SIZE,
&self_config->mp,
256,
ucs_empty_function,
"self_msg_desc");
if (UCS_OK != status) {
ucs_error("Failed to create a memory pool for the loop-back transport");
goto err;
}
/* set the message descriptor for the loop-back */
self->msg_cur_desc = ucs_mpool_get(&self->msg_desc_mp);
VALGRIND_MAKE_MEM_DEFINED(self->msg_cur_desc, sizeof(*(self->msg_cur_desc)));
if (NULL == self->msg_cur_desc) {
ucs_error("Failed to get the first descriptor in loop-back MP storage");
status = UCS_ERR_NO_RESOURCE;
goto destroy_mpool;
}
ucs_debug("Created a loop-back iface. id=0x%lx, desc=%p, len=%u, tx_hdr=%lu",
self->id, self->msg_cur_desc, self->data_length, self->rx_headroom);
return UCS_OK;
destroy_mpool:
ucs_mpool_cleanup(&self->msg_desc_mp, 1);
err:
return status;
}
static UCS_CLASS_INIT_FUNC(uct_ud_mlx5_ep_t, uct_iface_h tl_iface)
{
uct_ud_mlx5_iface_t *iface = ucs_derived_of(tl_iface, uct_ud_mlx5_iface_t);
ucs_trace_func("");
UCS_CLASS_CALL_SUPER_INIT(uct_ud_ep_t, &iface->super);
return UCS_OK;
}
/* Endpoint definition */
UCS_CLASS_INIT_FUNC(uct_ugni_ep_t, uct_iface_t *tl_iface,
const struct sockaddr *addr)
{
uct_ugni_iface_t *iface = ucs_derived_of(tl_iface, uct_ugni_iface_t);
const uct_sockaddr_ugni_t *iface_addr = (const uct_sockaddr_ugni_t*)addr;
ucs_status_t rc = UCS_OK;
gni_return_t ugni_rc;
UCS_CLASS_CALL_SUPER_INIT(uct_base_ep_t, &iface->super);
ugni_rc = GNI_EpCreate(iface->nic_handle, iface->local_cq, &self->ep);
if (GNI_RC_SUCCESS != ugni_rc) {
ucs_error("GNI_CdmCreate failed, Error status: %s %d",
gni_err_str[ugni_rc], ugni_rc);
return UCS_ERR_NO_DEVICE;
}
if(NULL != addr){
rc = ugni_connect_ep(iface, iface_addr, self);
}
ucs_arbiter_group_init(&self->arb_group);
uint32_t *big_hash;
big_hash = (void *)&self->ep;
self->hash_key = big_hash[0];
sglib_hashed_uct_ugni_ep_t_add(iface->eps, self);
return rc;
}
static UCS_CLASS_INIT_FUNC(uct_mm_ep_t, uct_iface_t *tl_iface,
const struct sockaddr *addr)
{
uct_mm_iface_t *iface = ucs_derived_of(tl_iface, uct_mm_iface_t);
UCS_CLASS_CALL_SUPER_INIT(uct_base_ep_t, &iface->super);
return UCS_OK;
}
static UCS_CLASS_INIT_FUNC(uct_ugni_udt_ep_t, uct_iface_t *tl_iface,
const uct_device_addr_t *dev_addr,
const uct_iface_addr_t *iface_addr)
{
UCS_CLASS_CALL_SUPER_INIT(uct_ugni_ep_t, tl_iface, dev_addr, iface_addr);
self->posted_desc = NULL;
return UCS_OK;
}
static UCS_CLASS_INIT_FUNC(uct_cma_iface_t, uct_pd_h pd, uct_worker_h worker,
const char *dev_name, size_t rx_headroom,
const uct_iface_config_t *tl_config)
{
UCS_CLASS_CALL_SUPER_INIT(uct_base_iface_t, &uct_cma_iface_ops, pd, worker,
tl_config UCS_STATS_ARG(NULL));
return UCS_OK;
}
static UCS_CLASS_INIT_FUNC(uct_rocm_copy_ep_t, const uct_ep_params_t *params)
{
uct_rocm_copy_iface_t *iface = ucs_derived_of(params->iface, uct_rocm_copy_iface_t);
UCS_CLASS_CALL_SUPER_INIT(uct_base_ep_t, &iface->super);
return UCS_OK;
}
UCS_CLASS_INIT_FUNC(uct_dc_ep_t, uct_dc_iface_t *iface, const uct_dc_iface_addr_t *if_addr)
{
UCS_CLASS_CALL_SUPER_INIT(uct_base_ep_t, &iface->super.super.super);
self->atomic_mr_offset = uct_ib_md_atomic_offset(if_addr->atomic_mr_id);
return uct_dc_ep_basic_init(iface, self);
}
static UCS_CLASS_INIT_FUNC(uct_cm_ep_t, uct_iface_t *tl_iface, const struct sockaddr *addr)
{
uct_cm_iface_t *iface = ucs_derived_of(tl_iface, uct_cm_iface_t);
UCS_CLASS_CALL_SUPER_INIT(uct_base_ep_t, &iface->super.super);
self->dest_addr = *(const uct_sockaddr_ib_t*)addr;
return UCS_OK;
}
static UCS_CLASS_INIT_FUNC(uct_tcp_ep_t, uct_tcp_iface_t *iface,
int fd, const struct sockaddr_in *dest_addr)
{
ucs_status_t status;
UCS_CLASS_CALL_SUPER_INIT(uct_base_ep_t, &iface->super)
self->buf = ucs_malloc(iface->config.buf_size, "tcp_buf");
if (self->buf == NULL) {
return UCS_ERR_NO_MEMORY;
}
self->events = 0;
self->offset = 0;
self->length = 0;
ucs_queue_head_init(&self->pending_q);
if (fd == -1) {
status = ucs_tcpip_socket_create(&self->fd);
if (status != UCS_OK) {
goto err;
}
/* TODO use non-blocking connect */
status = uct_tcp_socket_connect(self->fd, dest_addr);
if (status != UCS_OK) {
goto err_close;
}
} else {
self->fd = fd;
}
status = ucs_sys_fcntl_modfl(self->fd, O_NONBLOCK, 0);
if (status != UCS_OK) {
goto err_close;
}
status = uct_tcp_iface_set_sockopt(iface, self->fd);
if (status != UCS_OK) {
goto err_close;
}
uct_tcp_ep_epoll_ctl(self, EPOLL_CTL_ADD);
UCS_ASYNC_BLOCK(iface->super.worker->async);
ucs_list_add_tail(&iface->ep_list, &self->list);
UCS_ASYNC_UNBLOCK(iface->super.worker->async);
ucs_debug("tcp_ep %p: created on iface %p, fd %d", self, iface, self->fd);
return UCS_OK;
err_close:
close(self->fd);
err:
return status;
}
static UCS_CLASS_INIT_FUNC(uct_cuda_copy_ep_t, uct_iface_t *tl_iface,
const uct_device_addr_t *dev_addr,
const uct_iface_addr_t *iface_addr)
{
uct_cuda_copy_iface_t *iface = ucs_derived_of(tl_iface, uct_cuda_copy_iface_t);
UCS_CLASS_CALL_SUPER_INIT(uct_base_ep_t, &iface->super);
return UCS_OK;
}
static UCS_CLASS_INIT_FUNC(uct_mm_ep_t, uct_iface_t *tl_iface,
const uct_device_addr_t *dev_addr,
const uct_iface_addr_t *iface_addr)
{
uct_mm_iface_t *iface = ucs_derived_of(tl_iface, uct_mm_iface_t);
const uct_mm_iface_addr_t *addr = (const void*)iface_addr;
ucs_status_t status;
size_t size_to_attach;
UCS_CLASS_CALL_SUPER_INIT(uct_base_ep_t, &iface->super);
/* Connect to the remote address (remote FIFO) */
/* Attach the address's memory */
size_to_attach = UCT_MM_GET_FIFO_SIZE(iface);
status =
uct_mm_md_mapper_ops(iface->super.md)->attach(addr->id,
size_to_attach,
(void *)addr->vaddr,
&self->mapped_desc.address,
&self->mapped_desc.cookie,
iface->path);
if (status != UCS_OK) {
ucs_error("failed to connect to remote peer with mm. remote mm_id: %zu",
addr->id);
return status;
}
self->mapped_desc.length = size_to_attach;
self->mapped_desc.mmid = addr->id;
uct_mm_set_fifo_ptrs(self->mapped_desc.address, &self->fifo_ctl, &self->fifo);
self->cached_tail = self->fifo_ctl->tail;
self->cached_signal_addrlen = self->fifo_ctl->signal_addrlen;
self->cached_signal_sockaddr = self->fifo_ctl->signal_sockaddr;
/* Send connect message to remote side so it will start polling */
status = uct_mm_ep_signal_remote(self, UCT_MM_IFACE_SIGNAL_CONNECT);
if (status != UCS_OK) {
uct_mm_md_mapper_ops(iface->super.md)->detach(&self->mapped_desc);
return status;
}
/* Initiate the hash which will keep the base_adresses of remote memory
* chunks that hold the descriptors for bcopy. */
sglib_hashed_uct_mm_remote_seg_t_init(self->remote_segments_hash);
ucs_arbiter_group_init(&self->arb_group);
/* Register for send side progress */
uct_worker_progress_register(iface->super.worker, uct_mm_iface_progress,
iface);
ucs_debug("mm: ep connected: %p, to remote_shmid: %zu", self, addr->id);
return UCS_OK;
}
static UCS_CLASS_INIT_FUNC(uct_rc_verbs_ep_t, uct_iface_h tl_iface)
{
uct_rc_verbs_iface_t *iface = ucs_derived_of(tl_iface, uct_rc_verbs_iface_t);
UCS_CLASS_CALL_SUPER_INIT(uct_rc_ep_t, &iface->super);
self->tx.available = iface->super.config.tx_qp_len;
self->tx.post_count = 0;
self->tx.completion_count = 0;
return UCS_OK;
}
UCS_CLASS_INIT_FUNC(uct_rc_verbs_ep_t, const uct_ep_params_t *params)
{
uct_rc_verbs_iface_t *iface = ucs_derived_of(params->iface, uct_rc_verbs_iface_t);
UCS_CLASS_CALL_SUPER_INIT(uct_rc_ep_t, &iface->super);
uct_rc_txqp_available_set(&self->super.txqp, iface->config.tx_max_wr);
uct_rc_verbs_txcnt_init(&self->txcnt);
uct_ib_fence_info_init(&self->fi);
return UCS_OK;
}
static UCS_CLASS_INIT_FUNC(uct_rc_verbs_ep_t, uct_iface_h tl_iface)
{
uct_rc_verbs_iface_t *iface = ucs_derived_of(tl_iface, uct_rc_verbs_iface_t);
UCS_CLASS_CALL_SUPER_INIT(uct_rc_ep_t, &iface->super);
uct_rc_txqp_available_set(&self->super.txqp, iface->config.tx_max_wr);
uct_rc_verbs_txcnt_init(&self->txcnt);
uct_worker_progress_register(iface->super.super.super.worker,
uct_rc_verbs_iface_progress, iface);
return UCS_OK;
}
UCS_CLASS_INIT_FUNC(uct_dc_mlx5_grh_ep_t, uct_dc_mlx5_iface_t *iface,
const uct_dc_mlx5_iface_addr_t *if_addr,
uct_ib_mlx5_base_av_t *av,
struct mlx5_grh_av *grh_av)
{
ucs_trace_func("");
UCS_CLASS_CALL_SUPER_INIT(uct_dc_mlx5_ep_t, iface, if_addr, av);
self->super.flags |= UCT_DC_MLX5_EP_FLAG_GRH;
memcpy(&self->grh_av, grh_av, sizeof(*grh_av));
return UCS_OK;
}
static UCS_CLASS_INIT_FUNC(uct_cuda_iface_t, uct_pd_h pd, uct_worker_h worker,
const char *dev_name, size_t rx_headroom,
const uct_iface_config_t *tl_config)
{
UCS_CLASS_CALL_SUPER_INIT(uct_base_iface_t, &uct_cuda_iface_ops, pd, worker,
tl_config UCS_STATS_ARG(NULL));
if (strcmp(dev_name, UCT_CUDA_DEV_NAME) != 0) {
ucs_error("No device was found: %s", dev_name);
return UCS_ERR_NO_DEVICE;
}
return UCS_OK;
}
static UCS_CLASS_INIT_FUNC(uct_knem_iface_t, uct_md_h md, uct_worker_h worker,
const uct_iface_params_t *params,
const uct_iface_config_t *tl_config)
{
ucs_assert(params->open_mode & UCT_IFACE_OPEN_MODE_DEVICE);
UCS_CLASS_CALL_SUPER_INIT(uct_base_iface_t, &uct_knem_iface_ops, md, worker,
params, tl_config UCS_STATS_ARG(params->stats_root)
UCS_STATS_ARG(UCT_KNEM_TL_NAME));
self->knem_md = (uct_knem_md_t *)md;
uct_sm_get_max_iov(); /* to initialize ucs_get_max_iov static variable */
return UCS_OK;
}
UCS_CLASS_INIT_FUNC(uct_ud_ep_t, uct_ud_iface_t *iface)
{
ucs_trace_func("");
memset(self, 0, sizeof(*self));
UCS_CLASS_CALL_SUPER_INIT(uct_base_ep_t, &iface->super.super);
self->dest_ep_id = UCT_UD_EP_NULL_ID;
uct_ud_ep_reset(self);
ucs_list_head_init(&self->cep_list);
uct_ud_iface_add_ep(iface, self);
UCT_UD_EP_HOOK_INIT(self);
ucs_debug("NEW EP: iface=%p ep=%p id=%d", iface, self, self->ep_id);
return UCS_OK;
}
static UCS_CLASS_INIT_FUNC(uct_dc_mlx5_iface_t, uct_md_h md, uct_worker_h worker,
const uct_iface_params_t *params,
const uct_iface_config_t *tl_config)
{
uct_dc_mlx5_iface_config_t *config = ucs_derived_of(tl_config,
uct_dc_mlx5_iface_config_t);
ucs_status_t status;
ucs_trace_func("");
UCS_CLASS_CALL_SUPER_INIT(uct_dc_iface_t, &uct_dc_mlx5_iface_ops, md,
worker, params, 0, &config->super);
status = uct_rc_mlx5_iface_common_init(&self->mlx5_common, &self->super.super,
&config->super.super);
if (status != UCS_OK) {
goto err;
}
status = uct_ud_mlx5_iface_common_init(&self->super.super.super,
&self->ud_common, &config->ud_common);
if (status != UCS_OK) {
goto err_rc_mlx5_common_cleanup;
}
status = uct_dc_mlx5_iface_init_dcis(self);
if (status != UCS_OK) {
goto err_rc_mlx5_common_cleanup;
}
uct_dc_iface_set_quota(&self->super, &config->super);
/* Set max_iov for put_zcopy and get_zcopy */
uct_ib_iface_set_max_iov(&self->super.super.super,
((UCT_IB_MLX5_MAX_BB * MLX5_SEND_WQE_BB) -
sizeof(struct mlx5_wqe_raddr_seg) -
sizeof(struct mlx5_wqe_ctrl_seg) -
UCT_IB_MLX5_AV_FULL_SIZE) /
sizeof(struct mlx5_wqe_data_seg));
/* TODO: only register progress when we have a connection */
uct_worker_progress_register(worker, uct_dc_mlx5_iface_progress, self);
ucs_debug("created dc iface %p", self);
return UCS_OK;
err_rc_mlx5_common_cleanup:
uct_rc_mlx5_iface_common_cleanup(&self->mlx5_common);
err:
return status;
}
static UCS_CLASS_INIT_FUNC(uct_ugni_smsg_ep_t, uct_iface_t *tl_iface)
{
UCS_CLASS_CALL_SUPER_INIT(uct_ugni_ep_t, tl_iface, NULL);
uct_ugni_smsg_iface_t *iface = ucs_derived_of(tl_iface, uct_ugni_smsg_iface_t);
void *mbox;
UCT_TL_IFACE_GET_TX_DESC(&iface->super.super, &iface->free_mbox,
mbox, return UCS_ERR_NO_RESOURCE);
self->smsg_attr = (uct_ugni_smsg_mbox_t *)mbox;
uct_ugni_smsg_mbox_reg(iface, self->smsg_attr);
uct_ugni_smsg_mbox_init(iface, self->smsg_attr);
return UCS_OK;
}
static UCS_CLASS_INIT_FUNC(uct_ud_verbs_iface_t, uct_pd_h pd, uct_worker_h worker,
const char *dev_name, size_t rx_headroom,
const uct_iface_config_t *tl_config)
{
uct_ud_iface_config_t *config = ucs_derived_of(tl_config,
uct_ud_iface_config_t);
ucs_trace_func("");
UCS_CLASS_CALL_SUPER_INIT(uct_ud_iface_t, &uct_ud_verbs_iface_ops, pd,
worker, dev_name, rx_headroom, 0, config);
self->super.ops.async_progress = uct_ud_verbs_iface_async_progress;
self->super.ops.tx_skb = uct_ud_verbs_ep_tx_ctl_skb;
if (self->super.config.rx_max_batch < UCT_IB_MAX_WC) {
ucs_warn("max batch is too low (%d < %d), performance may be impacted",
self->super.config.rx_max_batch,
UCT_IB_MAX_WC);
}
while (self->super.rx.available >= self->super.config.rx_max_batch) {
uct_ud_verbs_iface_post_recv(self);
}
memset(&self->tx.wr_inl, 0, sizeof(self->tx.wr_inl));
self->tx.wr_inl.opcode = IBV_WR_SEND;
self->tx.wr_inl.wr_id = 0xBEEBBEEB;
self->tx.wr_inl.wr.ud.remote_qkey = UCT_IB_QKEY;
self->tx.wr_inl.imm_data = 0;
self->tx.wr_inl.next = 0;
self->tx.wr_inl.sg_list = self->tx.sge;
self->tx.wr_inl.num_sge = UCT_UD_MAX_SGE;
memset(&self->tx.wr_skb, 0, sizeof(self->tx.wr_skb));
self->tx.wr_skb.opcode = IBV_WR_SEND;
self->tx.wr_skb.wr_id = 0xFAAFFAAF;
self->tx.wr_skb.wr.ud.remote_qkey = UCT_IB_QKEY;
self->tx.wr_skb.imm_data = 0;
self->tx.wr_skb.next = 0;
self->tx.wr_skb.sg_list = self->tx.sge;
self->tx.wr_skb.num_sge = 1;
/* TODO: add progress on first ep creation */
uct_worker_progress_register(worker, uct_ud_verbs_iface_progress, self);
uct_ud_leave(&self->super);
return UCS_OK;
}
static UCS_CLASS_INIT_FUNC(uct_ud_verbs_iface_t, uct_pd_h pd, uct_worker_h worker,
const char *dev_name, size_t rx_headroom,
const uct_iface_config_t *tl_config)
{
uct_ud_iface_config_t *config = ucs_derived_of(tl_config, uct_ud_iface_config_t);
ucs_trace_func("");
UCS_CLASS_CALL_SUPER_INIT(uct_ud_iface_t, &uct_ud_verbs_iface_ops, pd,
worker, dev_name, rx_headroom, 0, config);
while (self->super.rx.available >= self->super.config.rx_max_batch) {
uct_ud_verbs_iface_post_recv(self);
}
memset(&self->tx.wr_inl, 0, sizeof(self->tx.wr_inl));
self->tx.wr_inl.opcode = IBV_WR_SEND;
self->tx.wr_inl.wr_id = 0xBEEBBEEB;
self->tx.wr_inl.wr.ud.remote_qkey = UCT_UD_QKEY;
self->tx.wr_inl.imm_data = 0;
self->tx.wr_inl.next = 0;
self->tx.wr_inl.sg_list = self->tx.sge;
self->tx.wr_inl.num_sge = UCT_UD_MAX_SGE;
memset(&self->tx.wr_bcp, 0, sizeof(self->tx.wr_bcp));
self->tx.wr_bcp.opcode = IBV_WR_SEND;
self->tx.wr_bcp.wr_id = 0xFAAFFAAF;
self->tx.wr_bcp.wr.ud.remote_qkey = UCT_UD_QKEY;
self->tx.wr_bcp.imm_data = 0;
self->tx.wr_bcp.next = 0;
self->tx.wr_bcp.sg_list = self->tx.sge;
self->tx.wr_bcp.num_sge = 1;
memset(&self->tx.wr_ctl, 0, sizeof(self->tx.wr_ctl));
self->tx.wr_ctl.opcode = IBV_WR_SEND;
self->tx.wr_ctl.wr_id = 0xCCCCCCCC;
self->tx.wr_ctl.wr.ud.remote_qkey = UCT_UD_QKEY;
self->tx.wr_ctl.imm_data = 0;
self->tx.wr_ctl.next = 0;
self->tx.wr_ctl.sg_list = self->tx.sge;
self->tx.wr_ctl.num_sge = 1;
/* TODO: add progress on first ep creation */
ucs_notifier_chain_add(&worker->progress_chain, uct_ud_verbs_iface_progress,
self);
return UCS_OK;
}
static UCS_CLASS_INIT_FUNC(uct_gdr_copy_iface_t, uct_md_h md, uct_worker_h worker,
const uct_iface_params_t *params,
const uct_iface_config_t *tl_config)
{
UCS_CLASS_CALL_SUPER_INIT(uct_base_iface_t, &uct_gdr_copy_iface_ops, md, worker,
params, tl_config UCS_STATS_ARG(params->stats_root)
UCS_STATS_ARG(UCT_GDR_COPY_TL_NAME));
if (strncmp(params->mode.device.dev_name,
UCT_CUDA_DEV_NAME, strlen(UCT_CUDA_DEV_NAME)) != 0) {
ucs_error("No device was found: %s", params->mode.device.dev_name);
return UCS_ERR_NO_DEVICE;
}
self->id = ucs_generate_uuid((uintptr_t)self);
return UCS_OK;
}
static UCS_CLASS_INIT_FUNC(uct_rc_mlx5_ep_t, uct_iface_h tl_iface)
{
uct_rc_mlx5_iface_t *iface = ucs_derived_of(tl_iface, uct_rc_mlx5_iface_t);
ucs_status_t status;
UCS_CLASS_CALL_SUPER_INIT(uct_rc_ep_t, &iface->super);
status = uct_ib_mlx5_get_txwq(iface->super.super.super.worker, self->super.qp,
&self->tx.wq);
if (status != UCS_OK) {
ucs_error("Failed to get mlx5 QP information");
return status;
}
self->super.available = self->tx.wq.bb_max;
self->qp_num = self->super.qp->qp_num;
return UCS_OK;
}
static UCS_CLASS_INIT_FUNC(uct_rc_mlx5_ep_t, uct_iface_h tl_iface)
{
uct_rc_mlx5_iface_t *iface = ucs_derived_of(tl_iface, uct_rc_mlx5_iface_t);
ucs_status_t status;
UCS_CLASS_CALL_SUPER_INIT(uct_rc_ep_t, &iface->super);
status = uct_ib_mlx5_txwq_init(iface->super.super.super.worker, &self->tx.wq,
self->super.txqp.qp);
if (status != UCS_OK) {
ucs_error("Failed to get mlx5 QP information");
return status;
}
self->qp_num = self->super.txqp.qp->qp_num;
self->tx.wq.bb_max = ucs_min(self->tx.wq.bb_max, iface->tx.bb_max);
uct_rc_txqp_available_set(&self->super.txqp, self->tx.wq.bb_max);
uct_worker_progress_register(iface->super.super.super.worker,
uct_rc_mlx5_iface_progress, iface);
return UCS_OK;
}
static UCS_CLASS_INIT_FUNC(uct_tcp_ep_t, uct_iface_t *tl_iface,
const uct_device_addr_t *dev_addr,
const uct_iface_addr_t *iface_addr)
{
uct_tcp_iface_t *iface = ucs_derived_of(tl_iface, uct_tcp_iface_t);
struct sockaddr_in dest_addr;
ucs_status_t status;
UCS_CLASS_CALL_SUPER_INIT(uct_base_ep_t, &iface->super)
status = uct_tcp_socket_create(&self->fd);
if (status != UCS_OK) {
goto err;
}
status = uct_tcp_iface_set_sockopt(iface, self->fd);
if (status != UCS_OK) {
goto err_close;
}
memset(&dest_addr, 0, sizeof(dest_addr));
dest_addr.sin_family = AF_INET;
dest_addr.sin_port = *(in_port_t*)iface_addr;
dest_addr.sin_addr = *(struct in_addr*)dev_addr;
status = uct_tcp_socket_connect(self->fd, &dest_addr);
if (status != UCS_OK) {
goto err_close;
}
ucs_debug("connected to %s:%d", inet_ntoa(dest_addr.sin_addr),
ntohs(dest_addr.sin_port));
return UCS_OK;
err_close:
close(self->fd);
err:
return status;
}
static UCS_CLASS_INIT_FUNC(uct_rc_mlx5_ep_t, uct_iface_h tl_iface)
{
uct_rc_mlx5_iface_t *iface = ucs_derived_of(tl_iface, uct_rc_mlx5_iface_t);
uct_ib_mlx5_qp_info_t qp_info;
ucs_status_t status;
UCS_CLASS_CALL_SUPER_INIT(uct_rc_ep_t, &iface->super);
status = uct_ib_mlx5_get_qp_info(self->super.qp, &qp_info);
if (status != UCS_OK) {
ucs_error("Failed to get mlx5 QP information");
return status;
}
if ((qp_info.bf.size == 0) || !ucs_is_pow2(qp_info.bf.size) ||
(qp_info.sq.stride != MLX5_SEND_WQE_BB) ||
!ucs_is_pow2(qp_info.sq.wqe_cnt))
{
ucs_error("mlx5 device parameters not suitable for transport");
return UCS_ERR_IO_ERROR;
}
self->qp_num = self->super.qp->qp_num;
self->tx.qstart = qp_info.sq.buf;
self->tx.qend = qp_info.sq.buf + (MLX5_SEND_WQE_BB * qp_info.sq.wqe_cnt);
self->tx.seg = self->tx.qstart;
self->tx.sw_pi = 0;
self->tx.prev_sw_pi = -1;
self->tx.max_pi = uct_rc_mlx5_calc_max_pi(iface, self->tx.prev_sw_pi);
self->tx.bf_reg = qp_info.bf.reg;
self->tx.bf_size = qp_info.bf.size;
self->tx.dbrec = &qp_info.dbrec[MLX5_SND_DBR];
memset(self->tx.qstart, 0, self->tx.qend - self->tx.qstart);
return UCS_OK;
}