static int psmx2_domain_close(fid_t fid)
{
struct psmx2_fid_domain *domain;
domain = container_of(fid, struct psmx2_fid_domain,
util_domain.domain_fid.fid);
FI_INFO(&psmx2_prov, FI_LOG_DOMAIN, "refcnt=%d\n",
ofi_atomic_get32(&domain->util_domain.ref));
psmx2_domain_release(domain);
if (ofi_domain_close(&domain->util_domain))
return 0;
if (domain->progress_thread_enabled)
psmx2_domain_stop_progress(domain);
fastlock_destroy(&domain->sep_lock);
fastlock_destroy(&domain->vl_lock);
rbtDelete(domain->mr_map);
fastlock_destroy(&domain->mr_lock);
psmx2_trx_ctxt_free(domain->base_trx_ctxt);
domain->fabric->active_domain = NULL;
free(domain);
psmx2_atomic_global_fini();
psmx2_am_global_fini();
return 0;
}
static int psmx2_stx_close(fid_t fid)
{
struct psmx2_fid_stx *stx;
stx = container_of(fid, struct psmx2_fid_stx, stx.fid);
if (ofi_atomic_get32(&stx->ref))
return -FI_EBUSY;
psmx2_lock(&stx->domain->trx_ctxt_lock, 1);
dlist_remove(&stx->tx->entry);
psmx2_unlock(&stx->domain->trx_ctxt_lock, 1);
psmx2_trx_ctxt_free(stx->tx);
psmx2_domain_release(stx->domain);
free(stx);
return 0;
}
static int psmx2_sep_close(fid_t fid)
{
struct psmx2_fid_sep *sep;
struct psmx2_ep_name ep_name;
int i;
sep = container_of(fid, struct psmx2_fid_sep, ep.fid);
if (ofi_atomic_get32(&sep->ref))
return -FI_EBUSY;
for (i = 0; i < sep->ctxt_cnt; i++) {
if (sep->ctxts[i].ep && ofi_atomic_get32(&sep->ctxts[i].ep->ref))
return -FI_EBUSY;
}
ep_name.epid = sep->ctxts[0].trx_ctxt->psm2_epid;
ep_name.sep_id = sep->id;
ep_name.type = sep->type;
ofi_ns_del_local_name(&sep->domain->fabric->name_server,
&sep->service, &ep_name);
for (i = 0; i < sep->ctxt_cnt; i++) {
psmx2_lock(&sep->domain->trx_ctxt_lock, 1);
dlist_remove(&sep->ctxts[i].trx_ctxt->entry);
psmx2_unlock(&sep->domain->trx_ctxt_lock, 1);
if (sep->ctxts[i].ep)
psmx2_ep_close_internal(sep->ctxts[i].ep);
psmx2_trx_ctxt_free(sep->ctxts[i].trx_ctxt);
}
psmx2_lock(&sep->domain->sep_lock, 1);
dlist_remove(&sep->entry);
psmx2_unlock(&sep->domain->sep_lock, 1);
psmx2_domain_release(sep->domain);
free(sep);
return 0;
}
static int psmx2_ep_close(fid_t fid)
{
struct psmx2_fid_ep *ep;
struct psmx2_ep_name ep_name;
struct psmx2_trx_ctxt *trx_ctxt;
ep = container_of(fid, struct psmx2_fid_ep, ep.fid);
if (ep->base_ep) {
ofi_atomic_dec32(&ep->base_ep->ref);
return 0;
}
if (ofi_atomic_get32(&ep->ref))
return -FI_EBUSY;
if (ep->stx)
ofi_atomic_dec32(&ep->stx->ref);
if (ep->rx) {
ep_name.epid = ep->rx->psm2_epid;
ofi_ns_del_local_name(&ep->domain->fabric->name_server,
&ep->service, &ep_name);
}
if (ep->rx) {
psmx2_lock(&ep->domain->trx_ctxt_lock, 1);
dlist_remove(&ep->rx->entry);
psmx2_unlock(&ep->domain->trx_ctxt_lock, 1);
}
trx_ctxt = ep->rx;
psmx2_ep_close_internal(ep);
psmx2_trx_ctxt_free(trx_ctxt);
return 0;
}
int psmx2_sep_open(struct fid_domain *domain, struct fi_info *info,
struct fid_ep **sep, void *context)
{
struct psmx2_fid_domain *domain_priv;
struct psmx2_fid_ep *ep_priv;
struct psmx2_fid_sep *sep_priv;
struct psmx2_ep_name ep_name;
struct psmx2_ep_name *src_addr;
struct psmx2_trx_ctxt *trx_ctxt;
size_t ctxt_cnt = 1;
size_t ctxt_size;
int err = -FI_EINVAL;
int i;
domain_priv = container_of(domain, struct psmx2_fid_domain,
util_domain.domain_fid.fid);
if (!domain_priv)
goto errout;
if (info && info->ep_attr) {
if (info->ep_attr->tx_ctx_cnt > psmx2_env.sep_trx_ctxt) {
FI_WARN(&psmx2_prov, FI_LOG_EP_CTRL,
"tx_ctx_cnt %"PRIu64" exceed limit %d.\n",
info->ep_attr->tx_ctx_cnt,
psmx2_env.sep_trx_ctxt);
goto errout;
}
if (info->ep_attr->rx_ctx_cnt > psmx2_env.sep_trx_ctxt) {
FI_WARN(&psmx2_prov, FI_LOG_EP_CTRL,
"rx_ctx_cnt %"PRIu64" exceed limit %d.\n",
info->ep_attr->rx_ctx_cnt,
psmx2_env.sep_trx_ctxt);
goto errout;
}
ctxt_cnt = info->ep_attr->tx_ctx_cnt;
if (ctxt_cnt < info->ep_attr->rx_ctx_cnt)
ctxt_cnt = info->ep_attr->rx_ctx_cnt;
if (ctxt_cnt == 0) {
FI_INFO(&psmx2_prov, FI_LOG_EP_CTRL,
"tx_ctx_cnt and rx_ctx_cnt are 0, use 1.\n");
ctxt_cnt = 1;
}
}
ctxt_size = ctxt_cnt * sizeof(struct psmx2_sep_ctxt);
sep_priv = (struct psmx2_fid_sep *) calloc(1, sizeof(*sep_priv) + ctxt_size);
if (!sep_priv) {
err = -FI_ENOMEM;
goto errout;
}
sep_priv->ep.fid.fclass = FI_CLASS_SEP;
sep_priv->ep.fid.context = context;
sep_priv->ep.fid.ops = &psmx2_fi_ops_sep;
sep_priv->ep.ops = &psmx2_sep_ops;
sep_priv->ep.cm = &psmx2_cm_ops;
sep_priv->domain = domain_priv;
sep_priv->ctxt_cnt = ctxt_cnt;
ofi_atomic_initialize32(&sep_priv->ref, 0);
src_addr = NULL;
if (info && info->src_addr) {
if (info->addr_format == FI_ADDR_STR)
src_addr = psmx2_string_to_ep_name(info->src_addr);
else
src_addr = info->src_addr;
}
for (i = 0; i < ctxt_cnt; i++) {
trx_ctxt = psmx2_trx_ctxt_alloc(domain_priv, src_addr, i);
if (!trx_ctxt) {
err = -FI_ENOMEM;
goto errout_free_ctxt;
}
sep_priv->ctxts[i].trx_ctxt = trx_ctxt;
err = psmx2_ep_open_internal(domain_priv, info, &ep_priv, context,
trx_ctxt);
if (err)
goto errout_free_ctxt;
/* override the ops so the fid can't be closed individually */
ep_priv->ep.fid.ops = &psmx2_fi_ops_sep_ctxt;
trx_ctxt->ep = ep_priv;
sep_priv->ctxts[i].ep = ep_priv;
}
sep_priv->type = PSMX2_EP_SCALABLE;
sep_priv->service = PSMX2_ANY_SERVICE;
if (src_addr) {
sep_priv->service = src_addr->service;
if (info->addr_format == FI_ADDR_STR)
free(src_addr);
}
if (sep_priv->service == PSMX2_ANY_SERVICE)
sep_priv->service = ((getpid() & 0x7FFF) << 16) +
((uintptr_t)sep_priv & 0xFFFF);
sep_priv->id = ofi_atomic_inc32(&domain_priv->sep_cnt);
psmx2_lock(&domain_priv->sep_lock, 1);
dlist_insert_before(&sep_priv->entry, &domain_priv->sep_list);
psmx2_unlock(&domain_priv->sep_lock, 1);
psmx2_lock(&domain_priv->trx_ctxt_lock, 1);
for (i = 0; i< ctxt_cnt; i++) {
dlist_insert_before(&sep_priv->ctxts[i].trx_ctxt->entry,
&domain_priv->trx_ctxt_list);
}
psmx2_unlock(&domain_priv->trx_ctxt_lock, 1);
ep_name.epid = sep_priv->ctxts[0].trx_ctxt->psm2_epid;
ep_name.sep_id = sep_priv->id;
ep_name.type = sep_priv->type;
ofi_ns_add_local_name(&domain_priv->fabric->name_server,
&sep_priv->service, &ep_name);
psmx2_domain_acquire(domain_priv);
*sep = &sep_priv->ep;
/* Make sure the AM handler is installed to answer SEP query */
psmx2_am_init(sep_priv->ctxts[0].trx_ctxt);
return 0;
errout_free_ctxt:
while (i) {
if (sep_priv->ctxts[i].ep)
psmx2_ep_close_internal(sep_priv->ctxts[i].ep);
if (sep_priv->ctxts[i].trx_ctxt)
psmx2_trx_ctxt_free(sep_priv->ctxts[i].trx_ctxt);
i--;
}
free(sep_priv);
errout:
return err;
}
int psmx2_ep_open(struct fid_domain *domain, struct fi_info *info,
struct fid_ep **ep, void *context)
{
struct psmx2_fid_domain *domain_priv;
struct psmx2_fid_ep *ep_priv;
struct psmx2_ep_name ep_name;
struct psmx2_ep_name *src_addr;
struct psmx2_trx_ctxt *trx_ctxt = NULL;
int err = -FI_EINVAL;
int alloc_trx_ctxt = 1;
domain_priv = container_of(domain, struct psmx2_fid_domain,
util_domain.domain_fid.fid);
if (!domain_priv)
goto errout;
if (info && info->ep_attr && info->ep_attr->rx_ctx_cnt == FI_SHARED_CONTEXT)
return -FI_ENOSYS;
if (info && info->ep_attr && info->ep_attr->tx_ctx_cnt == FI_SHARED_CONTEXT &&
!ofi_recv_allowed(info->caps) && !ofi_rma_target_allowed(info->caps)) {
alloc_trx_ctxt = 0;
FI_INFO(&psmx2_prov, FI_LOG_EP_CTRL, "Tx only endpoint with STX context.\n");
}
src_addr = NULL;
if (info && info->src_addr) {
if (info->addr_format == FI_ADDR_STR)
src_addr = psmx2_string_to_ep_name(info->src_addr);
else
src_addr = info->src_addr;
}
if (alloc_trx_ctxt) {
trx_ctxt = psmx2_trx_ctxt_alloc(domain_priv, src_addr, 0);
if (!trx_ctxt)
goto errout;
}
err = psmx2_ep_open_internal(domain_priv, info, &ep_priv, context,
trx_ctxt);
if (err)
goto errout_free_ctxt;
ep_priv->type = PSMX2_EP_REGULAR;
ep_priv->service = PSMX2_ANY_SERVICE;
if (src_addr) {
ep_priv->service = src_addr->service;
if (info->addr_format == FI_ADDR_STR)
free(src_addr);
}
if (ep_priv->service == PSMX2_ANY_SERVICE)
ep_priv->service = ((getpid() & 0x7FFF) << 16) +
((uintptr_t)ep_priv & 0xFFFF);
if (alloc_trx_ctxt) {
trx_ctxt->ep = ep_priv;
psmx2_lock(&domain_priv->trx_ctxt_lock, 1);
dlist_insert_before(&trx_ctxt->entry,
&domain_priv->trx_ctxt_list);
psmx2_unlock(&domain_priv->trx_ctxt_lock, 1);
ep_name.epid = trx_ctxt->psm2_epid;
ep_name.type = ep_priv->type;
ofi_ns_add_local_name(&domain_priv->fabric->name_server,
&ep_priv->service, &ep_name);
}
*ep = &ep_priv->ep;
return 0;
errout_free_ctxt:
psmx2_trx_ctxt_free(trx_ctxt);
errout:
return err;
}
static int psmx2_domain_init(struct psmx2_fid_domain *domain,
struct psmx2_src_name *src_addr)
{
int err;
psmx2_am_global_init();
psmx2_atomic_global_init();
domain->base_trx_ctxt = psmx2_trx_ctxt_alloc(domain, src_addr, -1);
if (!domain->base_trx_ctxt)
return -FI_ENODEV;
err = fastlock_init(&domain->mr_lock);
if (err) {
FI_WARN(&psmx2_prov, FI_LOG_CORE,
"fastlock_init(mr_lock) returns %d\n", err);
goto err_out_free_trx_ctxt;
}
domain->mr_map = rbtNew(&psmx2_key_compare);
if (!domain->mr_map) {
FI_WARN(&psmx2_prov, FI_LOG_CORE,
"rbtNew failed\n");
goto err_out_destroy_mr_lock;
}
domain->mr_reserved_key = 1;
err = fastlock_init(&domain->vl_lock);
if (err) {
FI_WARN(&psmx2_prov, FI_LOG_CORE,
"fastlock_init(vl_lock) returns %d\n", err);
goto err_out_delete_mr_map;
}
memset(domain->vl_map, 0, sizeof(domain->vl_map));
domain->vl_alloc = 0;
ofi_atomic_initialize32(&domain->sep_cnt, 0);
fastlock_init(&domain->sep_lock);
dlist_init(&domain->sep_list);
dlist_init(&domain->trx_ctxt_list);
fastlock_init(&domain->trx_ctxt_lock);
dlist_insert_before(&domain->base_trx_ctxt->entry, &domain->trx_ctxt_list);
/* Set active domain before psmx2_domain_enable_ep() installs the
* AM handlers to ensure that psmx2_active_fabric->active_domain
* is always non-NULL inside the handlers. Notice that the vlaue
* active_domain becomes NULL again only when the domain is closed.
* At that time the AM handlers are gone with the PSM endpoint.
*/
domain->fabric->active_domain = domain;
if (psmx2_domain_enable_ep(domain, NULL) < 0)
goto err_out_reset_active_domain;
if (domain->progress_thread_enabled)
psmx2_domain_start_progress(domain);
psmx2_am_init(domain->base_trx_ctxt);
return 0;
err_out_reset_active_domain:
domain->fabric->active_domain = NULL;
fastlock_destroy(&domain->vl_lock);
err_out_delete_mr_map:
rbtDelete(domain->mr_map);
err_out_destroy_mr_lock:
fastlock_destroy(&domain->mr_lock);
err_out_free_trx_ctxt:
psmx2_trx_ctxt_free(domain->base_trx_ctxt);
return err;
}