static UCS_CLASS_INIT_FUNC(uct_ugni_rdma_iface_t, uct_pd_h pd, uct_worker_h worker,
const char *dev_name, size_t rx_headroom,
const uct_iface_config_t *tl_config)
{
uct_ugni_rdma_iface_config_t *config = ucs_derived_of(tl_config, uct_ugni_rdma_iface_config_t);
ucs_status_t status;
pthread_mutex_lock(&uct_ugni_global_lock);
UCS_CLASS_CALL_SUPER_INIT(uct_ugni_iface_t, pd, worker, dev_name, &uct_ugni_rdma_iface_ops,
&config->super UCS_STATS_ARG(NULL));
/* Setting initial configuration */
self->config.fma_seg_size = UCT_UGNI_MAX_FMA;
self->config.rdma_max_size = UCT_UGNI_MAX_RDMA;
status = ucs_mpool_init(&self->free_desc,
0,
sizeof(uct_ugni_base_desc_t),
0, /* alignment offset */
UCS_SYS_CACHE_LINE_SIZE, /* alignment */
128, /* grow */
config->mpool.max_bufs, /* max buffers */
&uct_ugni_rdma_desc_mpool_ops,
"UGNI-DESC-ONLY");
if (UCS_OK != status) {
ucs_error("Mpool creation failed");
goto exit;
}
status = ucs_mpool_init(&self->free_desc_get,
0,
sizeof(uct_ugni_rdma_fetch_desc_t),
0, /* alignment offset */
UCS_SYS_CACHE_LINE_SIZE, /* alignment */
128 , /* grow */
config->mpool.max_bufs, /* max buffers */
&uct_ugni_rdma_desc_mpool_ops,
"UGNI-GET-DESC-ONLY");
if (UCS_OK != status) {
ucs_error("Mpool creation failed");
goto clean_desc;
}
status = ucs_mpool_init(&self->free_desc_buffer,
0,
sizeof(uct_ugni_base_desc_t) + self->config.fma_seg_size,
sizeof(uct_ugni_base_desc_t), /* alignment offset */
UCS_SYS_CACHE_LINE_SIZE, /* alignment */
128 , /* grow */
config->mpool.max_bufs, /* max buffers */
&uct_ugni_rdma_desc_mpool_ops,
"UGNI-DESC-BUFFER");
if (UCS_OK != status) {
ucs_error("Mpool creation failed");
goto clean_desc_get;
}
status = uct_iface_mpool_init(&self->super.super,
&self->free_desc_famo,
sizeof(uct_ugni_rdma_fetch_desc_t) + 8,
sizeof(uct_ugni_rdma_fetch_desc_t),/* alignment offset */
UCS_SYS_CACHE_LINE_SIZE, /* alignment */
&config->mpool, /* mpool config */
128 , /* grow */
uct_ugni_base_desc_key_init, /* memory/key init */
"UGNI-DESC-FAMO");
if (UCS_OK != status) {
ucs_error("Mpool creation failed");
goto clean_buffer;
}
status = uct_iface_mpool_init(&self->super.super,
&self->free_desc_get_buffer,
sizeof(uct_ugni_rdma_fetch_desc_t) +
self->config.fma_seg_size,
sizeof(uct_ugni_rdma_fetch_desc_t), /* alignment offset */
UCS_SYS_CACHE_LINE_SIZE, /* alignment */
&config->mpool, /* mpool config */
128 , /* grow */
uct_ugni_base_desc_key_init, /* memory/key init */
"UGNI-DESC-GET");
if (UCS_OK != status) {
ucs_error("Mpool creation failed");
goto clean_famo;
}
status = ugni_activate_iface(&self->super);
if (UCS_OK != status) {
ucs_error("Failed to activate the interface");
goto clean_get_buffer;
}
if(GNI_DEVICE_ARIES == self->super.dev->type) {
uct_ugni_rdma_iface_ops.ep_atomic_swap64 = uct_ugni_ep_atomic_swap64;
uct_ugni_rdma_iface_ops.ep_atomic_add32 = uct_ugni_ep_atomic_add32;
uct_ugni_rdma_iface_ops.ep_atomic_fadd32 = uct_ugni_ep_atomic_fadd32;
uct_ugni_rdma_iface_ops.ep_atomic_cswap32 = uct_ugni_ep_atomic_cswap32;
uct_ugni_rdma_iface_ops.ep_atomic_swap32 = uct_ugni_ep_atomic_swap32;
}
/* TBD: eventually the uct_ugni_progress has to be moved to
* rdma layer so each ugni layer will have own progress */
uct_worker_progress_register(worker, uct_ugni_progress, self);
pthread_mutex_unlock(&uct_ugni_global_lock);
return UCS_OK;
clean_get_buffer:
ucs_mpool_cleanup(&self->free_desc_get_buffer, 1);
clean_famo:
ucs_mpool_cleanup(&self->free_desc_famo, 1);
clean_buffer:
ucs_mpool_cleanup(&self->free_desc_buffer, 1);
clean_desc_get:
ucs_mpool_cleanup(&self->free_desc_get, 1);
clean_desc:
ucs_mpool_cleanup(&self->free_desc, 1);
exit:
ucs_error("Failed to activate interface");
pthread_mutex_unlock(&uct_ugni_global_lock);
return status;
}
static UCS_CLASS_INIT_FUNC(uct_ugni_rdma_iface_t, uct_pd_h pd, uct_worker_h worker,
const char *dev_name, size_t rx_headroom,
const uct_iface_config_t *tl_config)
{
uct_ugni_iface_config_t *config = ucs_derived_of(tl_config, uct_ugni_iface_config_t);
ucs_status_t rc;
pthread_mutex_lock(&uct_ugni_global_lock);
UCS_CLASS_CALL_SUPER_INIT(uct_ugni_iface_t, pd, worker, dev_name, &uct_ugni_iface_ops,
&config->super UCS_STATS_ARG(NULL));
/* Setting initial configuration */
self->config.fma_seg_size = UCT_UGNI_MAX_FMA;
self->config.rdma_max_size = UCT_UGNI_MAX_RDMA;
rc = ucs_mpool_create("UGNI-DESC-ONLY", sizeof(uct_ugni_base_desc_t),
0, /* alignment offset */
UCS_SYS_CACHE_LINE_SIZE, /* alignment */
128 , /* grow */
config->mpool.max_bufs, /* max buffers */
&self->super.super, /* iface */
ucs_mpool_hugetlb_malloc, /* allocation hooks */
ucs_mpool_hugetlb_free, /* free hook */
uct_ugni_base_desc_init, /* init func */
NULL , &self->free_desc);
if (UCS_OK != rc) {
ucs_error("Mpool creation failed");
goto exit;
}
rc = ucs_mpool_create("UGNI-GET-DESC-ONLY", sizeof(uct_ugni_fetch_desc_t),
0, /* alignment offset */
UCS_SYS_CACHE_LINE_SIZE, /* alignment */
128 , /* grow */
config->mpool.max_bufs, /* max buffers */
&self->super.super, /* iface */
ucs_mpool_hugetlb_malloc, /* allocation hooks */
ucs_mpool_hugetlb_free, /* free hook */
uct_ugni_base_desc_init, /* init func */
NULL , &self->free_desc_get);
if (UCS_OK != rc) {
ucs_error("Mpool creation failed");
goto clean_desc;
}
rc = ucs_mpool_create("UGNI-DESC-BUFFER",
sizeof(uct_ugni_base_desc_t) +
self->config.fma_seg_size,
sizeof(uct_ugni_base_desc_t), /* alignment offset */
UCS_SYS_CACHE_LINE_SIZE, /* alignment */
128 , /* grow */
config->mpool.max_bufs, /* max buffers */
&self->super.super, /* iface */
ucs_mpool_hugetlb_malloc, /* allocation hooks */
ucs_mpool_hugetlb_free, /* free hook */
uct_ugni_base_desc_init, /* init func */
NULL , &self->free_desc_buffer);
if (UCS_OK != rc) {
ucs_error("Mpool creation failed");
goto clean_desc_get;
}
rc = uct_iface_mpool_create(&self->super.super.super,
sizeof(uct_ugni_fetch_desc_t) + 8,
sizeof(uct_ugni_fetch_desc_t), /* alignment offset */
UCS_SYS_CACHE_LINE_SIZE, /* alignment */
&config->mpool, /* mpool config */
128 , /* grow */
uct_ugni_base_desc_key_init, /* memory/key init */
"UGNI-DESC-FAMO", /* name */
&self->free_desc_famo); /* mpool */
if (UCS_OK != rc) {
ucs_error("Mpool creation failed");
goto clean_buffer;
}
rc = uct_iface_mpool_create(&self->super.super.super,
sizeof(uct_ugni_fetch_desc_t) +
self->config.fma_seg_size,
sizeof(uct_ugni_fetch_desc_t), /* alignment offset */
UCS_SYS_CACHE_LINE_SIZE, /* alignment */
&config->mpool, /* mpool config */
128 , /* grow */
uct_ugni_base_desc_key_init, /* memory/key init */
"UGNI-DESC-GET", /* name */
&self->free_desc_get_buffer); /* mpool */
if (UCS_OK != rc) {
ucs_error("Mpool creation failed");
goto clean_famo;
}
rc = ugni_activate_iface(&self->super);
if (UCS_OK != rc) {
ucs_error("Failed to activate the interface");
goto clean_get_buffer;
}
/* TBD: eventually the uct_ugni_progress has to be moved to
* rdma layer so each ugni layer will have own progress */
ucs_notifier_chain_add(&worker->progress_chain, uct_ugni_progress, self);
pthread_mutex_unlock(&uct_ugni_global_lock);
return UCS_OK;
clean_get_buffer:
ucs_mpool_destroy(self->free_desc_get_buffer);
clean_famo:
ucs_mpool_destroy(self->free_desc_famo);
clean_buffer:
ucs_mpool_destroy(self->free_desc_buffer);
clean_desc_get:
ucs_mpool_destroy(self->free_desc_get);
clean_desc:
ucs_mpool_destroy(self->free_desc);
exit:
ucs_error("Failed to activate interface");
pthread_mutex_unlock(&uct_ugni_global_lock);
return rc;
}
