static UCS_CLASS_INIT_FUNC(uct_ud_verbs_iface_t, uct_pd_h pd, uct_worker_h worker,
const char *dev_name, size_t rx_headroom,
const uct_iface_config_t *tl_config)
{
uct_ud_iface_config_t *config = ucs_derived_of(tl_config, uct_ud_iface_config_t);
ucs_trace_func("");
UCS_CLASS_CALL_SUPER_INIT(uct_ud_iface_t, &uct_ud_verbs_iface_ops, pd,
worker, dev_name, rx_headroom, 0, config);
while (self->super.rx.available >= self->super.config.rx_max_batch) {
uct_ud_verbs_iface_post_recv(self);
}
memset(&self->tx.wr_inl, 0, sizeof(self->tx.wr_inl));
self->tx.wr_inl.opcode = IBV_WR_SEND;
self->tx.wr_inl.wr_id = 0xBEEBBEEB;
self->tx.wr_inl.wr.ud.remote_qkey = UCT_UD_QKEY;
self->tx.wr_inl.imm_data = 0;
self->tx.wr_inl.next = 0;
self->tx.wr_inl.sg_list = self->tx.sge;
self->tx.wr_inl.num_sge = UCT_UD_MAX_SGE;
memset(&self->tx.wr_bcp, 0, sizeof(self->tx.wr_bcp));
self->tx.wr_bcp.opcode = IBV_WR_SEND;
self->tx.wr_bcp.wr_id = 0xFAAFFAAF;
self->tx.wr_bcp.wr.ud.remote_qkey = UCT_UD_QKEY;
self->tx.wr_bcp.imm_data = 0;
self->tx.wr_bcp.next = 0;
self->tx.wr_bcp.sg_list = self->tx.sge;
self->tx.wr_bcp.num_sge = 1;
memset(&self->tx.wr_ctl, 0, sizeof(self->tx.wr_ctl));
self->tx.wr_ctl.opcode = IBV_WR_SEND;
self->tx.wr_ctl.wr_id = 0xCCCCCCCC;
self->tx.wr_ctl.wr.ud.remote_qkey = UCT_UD_QKEY;
self->tx.wr_ctl.imm_data = 0;
self->tx.wr_ctl.next = 0;
self->tx.wr_ctl.sg_list = self->tx.sge;
self->tx.wr_ctl.num_sge = 1;
/* TODO: add progress on first ep creation */
ucs_notifier_chain_add(&worker->progress_chain, uct_ud_verbs_iface_progress,
self);
return UCS_OK;
}
static UCS_CLASS_INIT_FUNC(uct_ugni_rdma_iface_t, uct_pd_h pd, uct_worker_h worker,
const char *dev_name, size_t rx_headroom,
const uct_iface_config_t *tl_config)
{
uct_ugni_iface_config_t *config = ucs_derived_of(tl_config, uct_ugni_iface_config_t);
ucs_status_t rc;
pthread_mutex_lock(&uct_ugni_global_lock);
UCS_CLASS_CALL_SUPER_INIT(uct_ugni_iface_t, pd, worker, dev_name, &uct_ugni_iface_ops,
&config->super UCS_STATS_ARG(NULL));
/* Setting initial configuration */
self->config.fma_seg_size = UCT_UGNI_MAX_FMA;
self->config.rdma_max_size = UCT_UGNI_MAX_RDMA;
rc = ucs_mpool_create("UGNI-DESC-ONLY", sizeof(uct_ugni_base_desc_t),
0, /* alignment offset */
UCS_SYS_CACHE_LINE_SIZE, /* alignment */
128 , /* grow */
config->mpool.max_bufs, /* max buffers */
&self->super.super, /* iface */
ucs_mpool_hugetlb_malloc, /* allocation hooks */
ucs_mpool_hugetlb_free, /* free hook */
uct_ugni_base_desc_init, /* init func */
NULL , &self->free_desc);
if (UCS_OK != rc) {
ucs_error("Mpool creation failed");
goto exit;
}
rc = ucs_mpool_create("UGNI-GET-DESC-ONLY", sizeof(uct_ugni_fetch_desc_t),
0, /* alignment offset */
UCS_SYS_CACHE_LINE_SIZE, /* alignment */
128 , /* grow */
config->mpool.max_bufs, /* max buffers */
&self->super.super, /* iface */
ucs_mpool_hugetlb_malloc, /* allocation hooks */
ucs_mpool_hugetlb_free, /* free hook */
uct_ugni_base_desc_init, /* init func */
NULL , &self->free_desc_get);
if (UCS_OK != rc) {
ucs_error("Mpool creation failed");
goto clean_desc;
}
rc = ucs_mpool_create("UGNI-DESC-BUFFER",
sizeof(uct_ugni_base_desc_t) +
self->config.fma_seg_size,
sizeof(uct_ugni_base_desc_t), /* alignment offset */
UCS_SYS_CACHE_LINE_SIZE, /* alignment */
128 , /* grow */
config->mpool.max_bufs, /* max buffers */
&self->super.super, /* iface */
ucs_mpool_hugetlb_malloc, /* allocation hooks */
ucs_mpool_hugetlb_free, /* free hook */
uct_ugni_base_desc_init, /* init func */
NULL , &self->free_desc_buffer);
if (UCS_OK != rc) {
ucs_error("Mpool creation failed");
goto clean_desc_get;
}
rc = uct_iface_mpool_create(&self->super.super.super,
sizeof(uct_ugni_fetch_desc_t) + 8,
sizeof(uct_ugni_fetch_desc_t), /* alignment offset */
UCS_SYS_CACHE_LINE_SIZE, /* alignment */
&config->mpool, /* mpool config */
128 , /* grow */
uct_ugni_base_desc_key_init, /* memory/key init */
"UGNI-DESC-FAMO", /* name */
&self->free_desc_famo); /* mpool */
if (UCS_OK != rc) {
ucs_error("Mpool creation failed");
goto clean_buffer;
}
rc = uct_iface_mpool_create(&self->super.super.super,
sizeof(uct_ugni_fetch_desc_t) +
self->config.fma_seg_size,
sizeof(uct_ugni_fetch_desc_t), /* alignment offset */
UCS_SYS_CACHE_LINE_SIZE, /* alignment */
&config->mpool, /* mpool config */
128 , /* grow */
uct_ugni_base_desc_key_init, /* memory/key init */
"UGNI-DESC-GET", /* name */
&self->free_desc_get_buffer); /* mpool */
if (UCS_OK != rc) {
ucs_error("Mpool creation failed");
goto clean_famo;
}
rc = ugni_activate_iface(&self->super);
if (UCS_OK != rc) {
ucs_error("Failed to activate the interface");
goto clean_get_buffer;
}
/* TBD: eventually the uct_ugni_progress has to be moved to
* rdma layer so each ugni layer will have own progress */
ucs_notifier_chain_add(&worker->progress_chain, uct_ugni_progress, self);
pthread_mutex_unlock(&uct_ugni_global_lock);
return UCS_OK;
clean_get_buffer:
ucs_mpool_destroy(self->free_desc_get_buffer);
clean_famo:
ucs_mpool_destroy(self->free_desc_famo);
clean_buffer:
ucs_mpool_destroy(self->free_desc_buffer);
clean_desc_get:
ucs_mpool_destroy(self->free_desc_get);
clean_desc:
ucs_mpool_destroy(self->free_desc);
exit:
ucs_error("Failed to activate interface");
pthread_mutex_unlock(&uct_ugni_global_lock);
return rc;
}
static UCS_CLASS_INIT_FUNC(uct_rc_verbs_iface_t, uct_pd_h pd, uct_worker_h worker,
const char *dev_name, size_t rx_headroom,
const uct_iface_config_t *tl_config)
{
uct_rc_verbs_iface_config_t *config =
ucs_derived_of(tl_config, uct_rc_verbs_iface_config_t);
struct ibv_exp_device_attr *dev_attr;
size_t am_hdr_size;
ucs_status_t status;
struct ibv_qp_cap cap;
struct ibv_qp *qp;
extern uct_iface_ops_t uct_rc_verbs_iface_ops;
UCS_CLASS_CALL_SUPER_INIT(uct_rc_iface_t, &uct_rc_verbs_iface_ops, pd,
worker, dev_name, rx_headroom, 0, &config->super);
/* Initialize inline work request */
memset(&self->inl_am_wr, 0, sizeof(self->inl_am_wr));
self->inl_am_wr.sg_list = self->inl_sge;
self->inl_am_wr.num_sge = 2;
self->inl_am_wr.opcode = IBV_WR_SEND;
self->inl_am_wr.send_flags = IBV_SEND_INLINE;
memset(&self->inl_rwrite_wr, 0, sizeof(self->inl_rwrite_wr));
self->inl_rwrite_wr.sg_list = self->inl_sge;
self->inl_rwrite_wr.num_sge = 1;
self->inl_rwrite_wr.opcode = IBV_WR_RDMA_WRITE;
self->inl_rwrite_wr.send_flags = IBV_SEND_SIGNALED | IBV_SEND_INLINE;
memset(self->inl_sge, 0, sizeof(self->inl_sge));
/* Configuration */
am_hdr_size = ucs_max(config->max_am_hdr, sizeof(uct_rc_hdr_t));
self->config.short_desc_size = ucs_max(UCT_RC_MAX_ATOMIC_SIZE, am_hdr_size);
dev_attr = &uct_ib_iface_device(&self->super.super)->dev_attr;
if (IBV_EXP_HAVE_ATOMIC_HCA(dev_attr) || IBV_EXP_HAVE_ATOMIC_GLOB(dev_attr)) {
self->config.atomic32_handler = uct_rc_ep_atomic_handler_32_be0;
self->config.atomic64_handler = uct_rc_ep_atomic_handler_64_be0;
} else if (IBV_EXP_HAVE_ATOMIC_HCA_REPLY_BE(dev_attr)) {
self->config.atomic32_handler = uct_rc_ep_atomic_handler_32_be1;
self->config.atomic64_handler = uct_rc_ep_atomic_handler_64_be1;
}
/* Create a dummy QP in order to find out max_inline */
status = uct_rc_iface_qp_create(&self->super, &qp, &cap);
if (status != UCS_OK) {
goto err;
}
ibv_destroy_qp(qp);
self->config.max_inline = cap.max_inline_data;
/* Create AH headers and Atomic mempool */
status = uct_iface_mpool_create(&self->super.super.super.super,
sizeof(uct_rc_iface_send_desc_t) +
self->config.short_desc_size,
sizeof(uct_rc_iface_send_desc_t),
UCS_SYS_CACHE_LINE_SIZE,
&config->super.super.tx.mp,
self->super.config.tx_qp_len,
uct_rc_iface_send_desc_init,
"rc_verbs_short_desc", &self->short_desc_mp);
if (status != UCS_OK) {
goto err;
}
while (self->super.rx.available > 0) {
if (uct_rc_verbs_iface_post_recv(self, 1) == 0) {
ucs_error("failed to post receives");
status = UCS_ERR_NO_MEMORY;
goto err_destroy_short_desc_mp;
}
}
ucs_notifier_chain_add(&worker->progress_chain, uct_rc_verbs_iface_progress,
self);
return UCS_OK;
err_destroy_short_desc_mp:
ucs_mpool_destroy(self->short_desc_mp);
err:
return status;
}
