/*
* define methods needed for the GNI fabric provider
*/
static int gnix_fabric_open(struct fi_fabric_attr *attr,
struct fid_fabric **fabric,
void *context)
{
int ret;
struct gnix_fid_fabric *fab;
if (strcmp(attr->name, gnix_fab_name)) {
return -FI_ENODATA;
}
fab = calloc(1, sizeof(*fab));
if (!fab) {
return -FI_ENOMEM;
}
/*
* set defaults related to use of GNI datagrams
*/
fab->n_bnd_dgrams = gnix_def_gni_n_dgrams;
fab->n_wc_dgrams = gnix_def_gni_n_wc_dgrams;
fab->datagram_timeout = gnix_def_gni_datagram_timeouts;
fab->fab_fid.fid.fclass = FI_CLASS_FABRIC;
fab->fab_fid.fid.context = context;
fab->fab_fid.fid.ops = &gnix_fab_fi_ops;
fab->fab_fid.ops = &gnix_fab_ops;
_gnix_ref_init(&fab->ref_cnt, 1, __fabric_destruct);
dlist_init(&fab->domain_list);
ret = _gnix_notifier_init(&fab->mr_notifier);
if (ret != FI_SUCCESS) {
return ret;
}
// TODO: open dynamically as needed
ret = _gnix_notifier_open(&fab->mr_notifier);
if (ret != FI_SUCCESS && ret != -FI_EBUSY) {
return ret;
}
*fabric = &fab->fab_fid;
return FI_SUCCESS;
}
DIRECT_FN int gnix_domain_open(struct fid_fabric *fabric, struct fi_info *info,
struct fid_domain **dom, void *context)
{
struct gnix_fid_domain *domain = NULL;
int ret = FI_SUCCESS;
uint8_t ptag;
uint32_t cookie;
struct gnix_fid_fabric *fabric_priv;
GNIX_TRACE(FI_LOG_DOMAIN, "\n");
fabric_priv = container_of(fabric, struct gnix_fid_fabric, fab_fid);
/*
* check cookie/ptag credentials - for FI_EP_MSG we may be creating a
* domain
* using a cookie supplied being used by the server. Otherwise, we use
* use the cookie/ptag supplied by the job launch system.
*/
if (info->dest_addr) {
ret =
gnixu_get_rdma_credentials(info->dest_addr, &ptag, &cookie);
if (ret) {
GNIX_WARN(FI_LOG_DOMAIN,
"gnixu_get_rdma_credentials returned ptag %u cookie 0x%x\n",
ptag, cookie);
goto err;
}
} else {
ret = gnixu_get_rdma_credentials(NULL, &ptag, &cookie);
}
GNIX_INFO(FI_LOG_DOMAIN,
"gnix rdma credentials returned ptag %u cookie 0x%x\n",
ptag, cookie);
domain = calloc(1, sizeof *domain);
if (domain == NULL) {
ret = -FI_ENOMEM;
goto err;
}
domain->mr_cache_attr = _gnix_default_mr_cache_attr;
domain->mr_cache_attr.reg_context = (void *) domain;
domain->mr_cache_attr.dereg_context = NULL;
domain->mr_cache_attr.destruct_context = NULL;
ret = _gnix_notifier_open(&domain->mr_cache_attr.notifier);
if (ret != FI_SUCCESS)
goto err;
domain->mr_cache_ro = NULL;
domain->mr_cache_rw = NULL;
fastlock_init(&domain->mr_cache_lock);
domain->udreg_reg_limit = 4096;
dlist_init(&domain->nic_list);
dlist_init(&domain->list);
dlist_insert_after(&domain->list, &fabric_priv->domain_list);
domain->fabric = fabric_priv;
_gnix_ref_get(domain->fabric);
domain->ptag = ptag;
domain->cookie = cookie;
domain->cdm_id_seed = getpid(); /* TODO: direct syscall better */
/* user tunables */
domain->params.msg_rendezvous_thresh = default_msg_rendezvous_thresh;
domain->params.rma_rdma_thresh = default_rma_rdma_thresh;
domain->params.ct_init_size = default_ct_init_size;
domain->params.ct_max_size = default_ct_max_size;
domain->params.ct_step = default_ct_step;
domain->params.vc_id_table_capacity = default_vc_id_table_capacity;
domain->params.mbox_page_size = default_mbox_page_size;
domain->params.mbox_num_per_slab = default_mbox_num_per_slab;
domain->params.mbox_maxcredit = default_mbox_maxcredit;
domain->params.mbox_msg_maxsize = default_mbox_msg_maxsize;
domain->params.max_retransmits = default_max_retransmits;
domain->params.err_inject_count = default_err_inject_count;
#if HAVE_XPMEM
domain->params.xpmem_enabled = true;
#else
domain->params.xpmem_enabled = false;
#endif
domain->gni_tx_cq_size = default_tx_cq_size;
domain->gni_rx_cq_size = default_rx_cq_size;
domain->gni_cq_modes = gnix_def_gni_cq_modes;
_gnix_ref_init(&domain->ref_cnt, 1, __domain_destruct);
domain->domain_fid.fid.fclass = FI_CLASS_DOMAIN;
domain->domain_fid.fid.context = context;
domain->domain_fid.fid.ops = &gnix_domain_fi_ops;
domain->domain_fid.ops = &gnix_domain_ops;
domain->domain_fid.mr = &gnix_domain_mr_ops;
domain->control_progress = info->domain_attr->control_progress;
domain->data_progress = info->domain_attr->data_progress;
domain->thread_model = info->domain_attr->threading;
domain->mr_is_init = 0;
domain->mr_iov_limit = info->domain_attr->mr_iov_limit;
fastlock_init(&domain->cm_nic_lock);
_gnix_open_cache(domain, GNIX_DEFAULT_CACHE_TYPE);
*dom = &domain->domain_fid;
return FI_SUCCESS;
err:
if (domain != NULL) {
free(domain);
}
return ret;
}
