static UCS_CLASS_INIT_FUNC(uct_mm_ep_t, uct_iface_t *tl_iface,
const uct_device_addr_t *dev_addr,
const uct_iface_addr_t *iface_addr)
{
uct_mm_iface_t *iface = ucs_derived_of(tl_iface, uct_mm_iface_t);
const uct_mm_iface_addr_t *addr = (const void*)iface_addr;
ucs_status_t status;
size_t size_to_attach;
UCS_CLASS_CALL_SUPER_INIT(uct_base_ep_t, &iface->super);
/* Connect to the remote address (remote FIFO) */
/* Attach the address's memory */
size_to_attach = UCT_MM_GET_FIFO_SIZE(iface);
status =
uct_mm_md_mapper_ops(iface->super.md)->attach(addr->id,
size_to_attach,
(void *)addr->vaddr,
&self->mapped_desc.address,
&self->mapped_desc.cookie,
iface->path);
if (status != UCS_OK) {
ucs_error("failed to connect to remote peer with mm. remote mm_id: %zu",
addr->id);
return status;
}
self->mapped_desc.length = size_to_attach;
self->mapped_desc.mmid = addr->id;
uct_mm_set_fifo_ptrs(self->mapped_desc.address, &self->fifo_ctl, &self->fifo);
self->cached_tail = self->fifo_ctl->tail;
self->cached_signal_addrlen = self->fifo_ctl->signal_addrlen;
self->cached_signal_sockaddr = self->fifo_ctl->signal_sockaddr;
/* Send connect message to remote side so it will start polling */
status = uct_mm_ep_signal_remote(self, UCT_MM_IFACE_SIGNAL_CONNECT);
if (status != UCS_OK) {
uct_mm_md_mapper_ops(iface->super.md)->detach(&self->mapped_desc);
return status;
}
/* Initiate the hash which will keep the base_adresses of remote memory
* chunks that hold the descriptors for bcopy. */
sglib_hashed_uct_mm_remote_seg_t_init(self->remote_segments_hash);
ucs_arbiter_group_init(&self->arb_group);
/* Register for send side progress */
uct_worker_progress_register(iface->super.worker, uct_mm_iface_progress,
iface);
ucs_debug("mm: ep connected: %p, to remote_shmid: %zu", self, addr->id);
return UCS_OK;
}
static UCS_CLASS_INIT_FUNC(uct_rc_verbs_ep_t, uct_iface_h tl_iface)
{
uct_rc_verbs_iface_t *iface = ucs_derived_of(tl_iface, uct_rc_verbs_iface_t);
UCS_CLASS_CALL_SUPER_INIT(uct_rc_ep_t, &iface->super);
uct_rc_txqp_available_set(&self->super.txqp, iface->config.tx_max_wr);
uct_rc_verbs_txcnt_init(&self->txcnt);
uct_worker_progress_register(iface->super.super.super.worker,
uct_rc_verbs_iface_progress, iface);
return UCS_OK;
}
static ucs_status_t ucp_stub_pending_add(uct_ep_h uct_ep, uct_pending_req_t *req)
{
ucp_stub_ep_t *stub_ep = ucs_derived_of(uct_ep, ucp_stub_ep_t);
ucp_ep_h ep = stub_ep->ep;
ucs_queue_push(&stub_ep->pending_q, ucp_stub_ep_req_priv(req));
++ep->worker->stub_pend_count;
/* Add a reference to the dummy progress function. If we have a pending
* request and this endpoint is still doing wireup, we must make sure progress
* is made. */
uct_worker_progress_register(ep->worker->uct, ucp_stub_ep_progress, stub_ep);
return UCS_OK;
}
static UCS_CLASS_INIT_FUNC(uct_dc_mlx5_iface_t, uct_md_h md, uct_worker_h worker,
const uct_iface_params_t *params,
const uct_iface_config_t *tl_config)
{
uct_dc_mlx5_iface_config_t *config = ucs_derived_of(tl_config,
uct_dc_mlx5_iface_config_t);
ucs_status_t status;
ucs_trace_func("");
UCS_CLASS_CALL_SUPER_INIT(uct_dc_iface_t, &uct_dc_mlx5_iface_ops, md,
worker, params, 0, &config->super);
status = uct_rc_mlx5_iface_common_init(&self->mlx5_common, &self->super.super,
&config->super.super);
if (status != UCS_OK) {
goto err;
}
status = uct_ud_mlx5_iface_common_init(&self->super.super.super,
&self->ud_common, &config->ud_common);
if (status != UCS_OK) {
goto err_rc_mlx5_common_cleanup;
}
status = uct_dc_mlx5_iface_init_dcis(self);
if (status != UCS_OK) {
goto err_rc_mlx5_common_cleanup;
}
uct_dc_iface_set_quota(&self->super, &config->super);
/* Set max_iov for put_zcopy and get_zcopy */
uct_ib_iface_set_max_iov(&self->super.super.super,
((UCT_IB_MLX5_MAX_BB * MLX5_SEND_WQE_BB) -
sizeof(struct mlx5_wqe_raddr_seg) -
sizeof(struct mlx5_wqe_ctrl_seg) -
UCT_IB_MLX5_AV_FULL_SIZE) /
sizeof(struct mlx5_wqe_data_seg));
/* TODO: only register progress when we have a connection */
uct_worker_progress_register(worker, uct_dc_mlx5_iface_progress, self);
ucs_debug("created dc iface %p", self);
return UCS_OK;
err_rc_mlx5_common_cleanup:
uct_rc_mlx5_iface_common_cleanup(&self->mlx5_common);
err:
return status;
}
static UCS_CLASS_INIT_FUNC(uct_ud_verbs_iface_t, uct_pd_h pd, uct_worker_h worker,
const char *dev_name, size_t rx_headroom,
const uct_iface_config_t *tl_config)
{
uct_ud_iface_config_t *config = ucs_derived_of(tl_config,
uct_ud_iface_config_t);
ucs_trace_func("");
UCS_CLASS_CALL_SUPER_INIT(uct_ud_iface_t, &uct_ud_verbs_iface_ops, pd,
worker, dev_name, rx_headroom, 0, config);
self->super.ops.async_progress = uct_ud_verbs_iface_async_progress;
self->super.ops.tx_skb = uct_ud_verbs_ep_tx_ctl_skb;
if (self->super.config.rx_max_batch < UCT_IB_MAX_WC) {
ucs_warn("max batch is too low (%d < %d), performance may be impacted",
self->super.config.rx_max_batch,
UCT_IB_MAX_WC);
}
while (self->super.rx.available >= self->super.config.rx_max_batch) {
uct_ud_verbs_iface_post_recv(self);
}
memset(&self->tx.wr_inl, 0, sizeof(self->tx.wr_inl));
self->tx.wr_inl.opcode = IBV_WR_SEND;
self->tx.wr_inl.wr_id = 0xBEEBBEEB;
self->tx.wr_inl.wr.ud.remote_qkey = UCT_IB_QKEY;
self->tx.wr_inl.imm_data = 0;
self->tx.wr_inl.next = 0;
self->tx.wr_inl.sg_list = self->tx.sge;
self->tx.wr_inl.num_sge = UCT_UD_MAX_SGE;
memset(&self->tx.wr_skb, 0, sizeof(self->tx.wr_skb));
self->tx.wr_skb.opcode = IBV_WR_SEND;
self->tx.wr_skb.wr_id = 0xFAAFFAAF;
self->tx.wr_skb.wr.ud.remote_qkey = UCT_IB_QKEY;
self->tx.wr_skb.imm_data = 0;
self->tx.wr_skb.next = 0;
self->tx.wr_skb.sg_list = self->tx.sge;
self->tx.wr_skb.num_sge = 1;
/* TODO: add progress on first ep creation */
uct_worker_progress_register(worker, uct_ud_verbs_iface_progress, self);
uct_ud_leave(&self->super);
return UCS_OK;
}
static UCS_CLASS_INIT_FUNC(uct_rc_mlx5_ep_t, uct_iface_h tl_iface)
{
uct_rc_mlx5_iface_t *iface = ucs_derived_of(tl_iface, uct_rc_mlx5_iface_t);
ucs_status_t status;
UCS_CLASS_CALL_SUPER_INIT(uct_rc_ep_t, &iface->super);
status = uct_ib_mlx5_get_txwq(iface->super.super.super.worker, self->super.qp,
&self->tx.wq);
if (status != UCS_OK) {
ucs_error("Failed to get mlx5 QP information");
return status;
}
self->super.available = self->tx.wq.bb_max;
self->qp_num = self->super.qp->qp_num;
uct_worker_progress_register(iface->super.super.super.worker,
uct_rc_mlx5_iface_progress, iface);
return UCS_OK;
}
static UCS_CLASS_INIT_FUNC(uct_ugni_smsg_iface_t, uct_md_h md, uct_worker_h worker,
const uct_iface_params_t *params,
const uct_iface_config_t *tl_config)
{
uct_ugni_iface_config_t *config = ucs_derived_of(tl_config, uct_ugni_iface_config_t);
ucs_status_t status;
gni_return_t ugni_rc;
unsigned int bytes_per_mbox;
gni_smsg_attr_t smsg_attr;
pthread_mutex_lock(&uct_ugni_global_lock);
UCS_CLASS_CALL_SUPER_INIT(uct_ugni_iface_t, md, worker, params,
&uct_ugni_smsg_iface_ops,
&config->super UCS_STATS_ARG(NULL));
/* Setting initial configuration */
self->config.smsg_seg_size = 2048;
self->config.rx_headroom = params->rx_headroom;
self->config.smsg_max_retransmit = 16;
self->config.smsg_max_credit = 8;
self->smsg_id = 0;
smsg_attr.msg_type = GNI_SMSG_TYPE_MBOX_AUTO_RETRANSMIT;
smsg_attr.mbox_maxcredit = self->config.smsg_max_credit;
smsg_attr.msg_maxsize = self->config.smsg_seg_size;
ugni_rc = GNI_SmsgBufferSizeNeeded(&(smsg_attr), &bytes_per_mbox);
self->bytes_per_mbox = ucs_align_up_pow2(bytes_per_mbox, ucs_get_page_size());
if (ugni_rc != GNI_RC_SUCCESS) {
ucs_error("Smsg buffer size calculation failed");
status = UCS_ERR_INVALID_PARAM;
goto exit;
}
status = ucs_mpool_init(&self->free_desc,
0,
self->config.smsg_seg_size + sizeof(uct_ugni_smsg_desc_t),
0,
UCS_SYS_CACHE_LINE_SIZE, /* alignment */
128 , /* grow */
config->mpool.max_bufs, /* max buffers */
&uct_ugni_smsg_desc_mpool_ops,
"UGNI-SMSG-DESC");
if (UCS_OK != status) {
ucs_error("Desc Mpool creation failed");
goto exit;
}
status = ucs_mpool_init(&self->free_mbox,
0,
self->bytes_per_mbox + sizeof(uct_ugni_smsg_mbox_t),
sizeof(uct_ugni_smsg_mbox_t),
UCS_SYS_CACHE_LINE_SIZE, /* alignment */
128, /* grow */
config->mpool.max_bufs, /* max buffers */
&uct_ugni_smsg_mbox_mpool_ops,
"UGNI-SMSG-MBOX");
if (UCS_OK != status) {
ucs_error("Mbox Mpool creation failed");
goto clean_desc;
}
UCT_TL_IFACE_GET_TX_DESC(&self->super.super, &self->free_desc,
self->user_desc, self->user_desc = NULL);
status = ugni_smsg_activate_iface(self);
if (UCS_OK != status) {
ucs_error("Failed to activate the interface");
goto clean_mbox;
}
ugni_rc = GNI_SmsgSetMaxRetrans(self->super.nic_handle, self->config.smsg_max_retransmit);
if (ugni_rc != GNI_RC_SUCCESS) {
ucs_error("Smsg setting max retransmit count failed.");
status = UCS_ERR_INVALID_PARAM;
goto clean_iface;
}
/* TBD: eventually the uct_ugni_progress has to be moved to
* udt layer so each ugni layer will have own progress */
uct_worker_progress_register(worker, uct_ugni_smsg_progress, self);
pthread_mutex_unlock(&uct_ugni_global_lock);
return UCS_OK;
clean_iface:
ugni_smsg_deactivate_iface(self);
clean_desc:
ucs_mpool_put(self->user_desc);
ucs_mpool_cleanup(&self->free_desc, 1);
clean_mbox:
ucs_mpool_cleanup(&self->free_mbox, 1);
exit:
ucs_error("Failed to activate interface");
pthread_mutex_unlock(&uct_ugni_global_lock);
return status;
}
static UCS_CLASS_INIT_FUNC(uct_ugni_rdma_iface_t, uct_pd_h pd, uct_worker_h worker,
const char *dev_name, size_t rx_headroom,
const uct_iface_config_t *tl_config)
{
uct_ugni_rdma_iface_config_t *config = ucs_derived_of(tl_config, uct_ugni_rdma_iface_config_t);
ucs_status_t status;
pthread_mutex_lock(&uct_ugni_global_lock);
UCS_CLASS_CALL_SUPER_INIT(uct_ugni_iface_t, pd, worker, dev_name, &uct_ugni_rdma_iface_ops,
&config->super UCS_STATS_ARG(NULL));
/* Setting initial configuration */
self->config.fma_seg_size = UCT_UGNI_MAX_FMA;
self->config.rdma_max_size = UCT_UGNI_MAX_RDMA;
status = ucs_mpool_init(&self->free_desc,
0,
sizeof(uct_ugni_base_desc_t),
0, /* alignment offset */
UCS_SYS_CACHE_LINE_SIZE, /* alignment */
128, /* grow */
config->mpool.max_bufs, /* max buffers */
&uct_ugni_rdma_desc_mpool_ops,
"UGNI-DESC-ONLY");
if (UCS_OK != status) {
ucs_error("Mpool creation failed");
goto exit;
}
status = ucs_mpool_init(&self->free_desc_get,
0,
sizeof(uct_ugni_rdma_fetch_desc_t),
0, /* alignment offset */
UCS_SYS_CACHE_LINE_SIZE, /* alignment */
128 , /* grow */
config->mpool.max_bufs, /* max buffers */
&uct_ugni_rdma_desc_mpool_ops,
"UGNI-GET-DESC-ONLY");
if (UCS_OK != status) {
ucs_error("Mpool creation failed");
goto clean_desc;
}
status = ucs_mpool_init(&self->free_desc_buffer,
0,
sizeof(uct_ugni_base_desc_t) + self->config.fma_seg_size,
sizeof(uct_ugni_base_desc_t), /* alignment offset */
UCS_SYS_CACHE_LINE_SIZE, /* alignment */
128 , /* grow */
config->mpool.max_bufs, /* max buffers */
&uct_ugni_rdma_desc_mpool_ops,
"UGNI-DESC-BUFFER");
if (UCS_OK != status) {
ucs_error("Mpool creation failed");
goto clean_desc_get;
}
status = uct_iface_mpool_init(&self->super.super,
&self->free_desc_famo,
sizeof(uct_ugni_rdma_fetch_desc_t) + 8,
sizeof(uct_ugni_rdma_fetch_desc_t),/* alignment offset */
UCS_SYS_CACHE_LINE_SIZE, /* alignment */
&config->mpool, /* mpool config */
128 , /* grow */
uct_ugni_base_desc_key_init, /* memory/key init */
"UGNI-DESC-FAMO");
if (UCS_OK != status) {
ucs_error("Mpool creation failed");
goto clean_buffer;
}
status = uct_iface_mpool_init(&self->super.super,
&self->free_desc_get_buffer,
sizeof(uct_ugni_rdma_fetch_desc_t) +
self->config.fma_seg_size,
sizeof(uct_ugni_rdma_fetch_desc_t), /* alignment offset */
UCS_SYS_CACHE_LINE_SIZE, /* alignment */
&config->mpool, /* mpool config */
128 , /* grow */
uct_ugni_base_desc_key_init, /* memory/key init */
"UGNI-DESC-GET");
if (UCS_OK != status) {
ucs_error("Mpool creation failed");
goto clean_famo;
}
status = ugni_activate_iface(&self->super);
if (UCS_OK != status) {
ucs_error("Failed to activate the interface");
goto clean_get_buffer;
}
if(GNI_DEVICE_ARIES == self->super.dev->type) {
uct_ugni_rdma_iface_ops.ep_atomic_swap64 = uct_ugni_ep_atomic_swap64;
uct_ugni_rdma_iface_ops.ep_atomic_add32 = uct_ugni_ep_atomic_add32;
uct_ugni_rdma_iface_ops.ep_atomic_fadd32 = uct_ugni_ep_atomic_fadd32;
uct_ugni_rdma_iface_ops.ep_atomic_cswap32 = uct_ugni_ep_atomic_cswap32;
uct_ugni_rdma_iface_ops.ep_atomic_swap32 = uct_ugni_ep_atomic_swap32;
}
/* TBD: eventually the uct_ugni_progress has to be moved to
* rdma layer so each ugni layer will have own progress */
uct_worker_progress_register(worker, uct_ugni_progress, self);
pthread_mutex_unlock(&uct_ugni_global_lock);
return UCS_OK;
clean_get_buffer:
ucs_mpool_cleanup(&self->free_desc_get_buffer, 1);
clean_famo:
ucs_mpool_cleanup(&self->free_desc_famo, 1);
clean_buffer:
ucs_mpool_cleanup(&self->free_desc_buffer, 1);
clean_desc_get:
ucs_mpool_cleanup(&self->free_desc_get, 1);
clean_desc:
ucs_mpool_cleanup(&self->free_desc, 1);
exit:
ucs_error("Failed to activate interface");
pthread_mutex_unlock(&uct_ugni_global_lock);
return status;
}
static UCS_CLASS_INIT_FUNC(uct_ud_mlx5_iface_t,
uct_pd_h pd, uct_worker_h worker,
const char *dev_name, size_t rx_headroom,
const uct_iface_config_t *tl_config)
{
uct_ud_iface_config_t *config = ucs_derived_of(tl_config,
uct_ud_iface_config_t);
ucs_status_t status;
int i;
ucs_trace_func("");
UCS_CLASS_CALL_SUPER_INIT(uct_ud_iface_t, &uct_ud_mlx5_iface_ops,
pd, worker,
dev_name, rx_headroom, 0, config);
self->super.ops.async_progress = uct_ud_mlx5_iface_async_progress;
self->super.ops.tx_skb = uct_ud_mlx5_ep_tx_ctl_skb;
status = uct_ib_mlx5_get_cq(self->super.super.send_cq, &self->tx.cq);
if (status != UCS_OK) {
return status;
}
if (uct_ib_mlx5_cqe_size(&self->tx.cq) != sizeof(struct mlx5_cqe64)) {
ucs_error("TX CQE size (%d) is not %d",
uct_ib_mlx5_cqe_size(&self->tx.cq),
(int)sizeof(struct mlx5_cqe64));
return UCS_ERR_IO_ERROR;
}
status = uct_ib_mlx5_get_cq(self->super.super.recv_cq, &self->rx.cq);
if (status != UCS_OK) {
return UCS_ERR_IO_ERROR;
}
if (uct_ib_mlx5_cqe_size(&self->rx.cq) != sizeof(struct mlx5_cqe64)) {
ucs_error("RX CQE size (%d) is not %d",
uct_ib_mlx5_cqe_size(&self->rx.cq),
(int)sizeof(struct mlx5_cqe64));
return UCS_ERR_IO_ERROR;
}
status = uct_ib_mlx5_get_txwq(self->super.super.super.worker, self->super.qp,
&self->tx.wq);
if (status != UCS_OK) {
return UCS_ERR_IO_ERROR;
}
self->super.tx.available = self->tx.wq.bb_max;
status = uct_ib_mlx5_get_rxwq(self->super.qp, &self->rx.wq);
if (status != UCS_OK) {
return UCS_ERR_IO_ERROR;
}
/* write buffer sizes */
for (i = 0; i <= self->rx.wq.mask; i++) {
self->rx.wq.wqes[i].byte_count = htonl(self->super.super.config.rx_payload_offset +
self->super.super.config.seg_size);
}
while (self->super.rx.available >= self->super.config.rx_max_batch) {
uct_ud_mlx5_iface_post_recv(self);
}
/* TODO: add progress on first ep creation */
uct_worker_progress_register(worker, uct_ud_mlx5_iface_progress, self);
uct_ud_leave(&self->super);
return UCS_OK;
}
