static int psmx2_domain_close(fid_t fid)
{
struct psmx2_fid_domain *domain;
domain = container_of(fid, struct psmx2_fid_domain,
util_domain.domain_fid.fid);
FI_INFO(&psmx2_prov, FI_LOG_DOMAIN, "refcnt=%d\n",
ofi_atomic_get32(&domain->util_domain.ref));
if (ofi_domain_close(&domain->util_domain))
return 0;
if (domain->progress_thread_enabled)
psmx2_domain_stop_progress(domain);
fastlock_destroy(&domain->sep_lock);
fastlock_destroy(&domain->mr_lock);
rbtDelete(domain->mr_map);
psmx2_lock(&domain->fabric->domain_lock, 1);
dlist_remove(&domain->entry);
psmx2_unlock(&domain->fabric->domain_lock, 1);
psmx2_fabric_release(domain->fabric);
free(domain);
return 0;
}
static inline void psmx2_am_enqueue_rma(struct psmx2_trx_ctxt *trx_ctxt,
struct psmx2_am_request *req)
{
psmx2_lock(&trx_ctxt->rma_queue.lock, 2);
slist_insert_tail(&req->list_entry, &trx_ctxt->rma_queue.list);
psmx2_unlock(&trx_ctxt->rma_queue.lock, 2);
}
int psmx2_am_progress(struct psmx2_trx_ctxt *trx_ctxt)
{
struct slist_entry *item;
struct psmx2_am_request *req;
struct psmx2_trigger *trigger;
if (psmx2_env.tagged_rma) {
psmx2_lock(&trx_ctxt->rma_queue.lock, 2);
while (!slist_empty(&trx_ctxt->rma_queue.list)) {
item = slist_remove_head(&trx_ctxt->rma_queue.list);
req = container_of(item, struct psmx2_am_request, list_entry);
psmx2_unlock(&trx_ctxt->rma_queue.lock, 2);
psmx2_am_process_rma(trx_ctxt, req);
psmx2_lock(&trx_ctxt->rma_queue.lock, 2);
}
psmx2_unlock(&trx_ctxt->rma_queue.lock, 2);
}
static int psmx2_sep_close(fid_t fid)
{
struct psmx2_fid_sep *sep;
struct psmx2_ep_name ep_name;
int i;
sep = container_of(fid, struct psmx2_fid_sep, ep.fid);
if (ofi_atomic_get32(&sep->ref))
return -FI_EBUSY;
for (i = 0; i < sep->ctxt_cnt; i++) {
if (sep->ctxts[i].ep && ofi_atomic_get32(&sep->ctxts[i].ep->ref))
return -FI_EBUSY;
}
ep_name.epid = sep->ctxts[0].trx_ctxt->psm2_epid;
ep_name.sep_id = sep->id;
ep_name.type = sep->type;
ofi_ns_del_local_name(&sep->domain->fabric->name_server,
&sep->service, &ep_name);
for (i = 0; i < sep->ctxt_cnt; i++) {
psmx2_lock(&sep->domain->trx_ctxt_lock, 1);
dlist_remove(&sep->ctxts[i].trx_ctxt->entry);
psmx2_unlock(&sep->domain->trx_ctxt_lock, 1);
if (sep->ctxts[i].ep)
psmx2_ep_close_internal(sep->ctxts[i].ep);
psmx2_trx_ctxt_free(sep->ctxts[i].trx_ctxt);
}
psmx2_lock(&sep->domain->sep_lock, 1);
dlist_remove(&sep->entry);
psmx2_unlock(&sep->domain->sep_lock, 1);
psmx2_domain_release(sep->domain);
free(sep);
return 0;
}
static struct psmx2_cq_event *psmx2_cq_alloc_event(struct psmx2_fid_cq *cq)
{
struct psmx2_cq_event *event;
psmx2_lock(&cq->lock, 2);
if (!slist_empty(&cq->free_list)) {
event = container_of(slist_remove_head(&cq->free_list),
struct psmx2_cq_event, list_entry);
psmx2_unlock(&cq->lock, 2);
return event;
}
void psmx2_cq_enqueue_event(struct psmx2_fid_cq *cq,
struct psmx2_cq_event *event)
{
psmx2_lock(&cq->lock, 2);
slist_insert_tail(&event->list_entry, &cq->event_queue);
cq->event_count++;
psmx2_unlock(&cq->lock, 2);
if (cq->wait)
cq->wait->signal(cq->wait);
}
static struct psmx2_cq_event *psmx2_cq_dequeue_event(struct psmx2_fid_cq *cq)
{
struct slist_entry *entry;
if (slist_empty(&cq->event_queue))
return NULL;
psmx2_lock(&cq->lock, 2);
entry = slist_remove_head(&cq->event_queue);
cq->event_count--;
psmx2_unlock(&cq->lock, 2);
return container_of(entry, struct psmx2_cq_event, list_entry);
}
int psmx2_stx_ctx(struct fid_domain *domain, struct fi_tx_attr *attr,
struct fid_stx **stx, void *context)
{
struct psmx2_fid_domain *domain_priv;
struct psmx2_trx_ctxt *trx_ctxt;
struct psmx2_fid_stx *stx_priv;
int err = -FI_EINVAL;
domain_priv = container_of(domain, struct psmx2_fid_domain,
util_domain.domain_fid.fid);
if (!domain_priv)
goto errout;
stx_priv = (struct psmx2_fid_stx *) calloc(1, sizeof *stx_priv);
if (!stx_priv) {
err = -FI_ENOMEM;
goto errout;
}
trx_ctxt = psmx2_trx_ctxt_alloc(domain_priv, NULL/*src_addr*/, 0);
if (!trx_ctxt) {
err = -FI_ENOMEM;
goto errout_free_stx;
}
psmx2_domain_acquire(domain_priv);
stx_priv->stx.fid.fclass = FI_CLASS_STX_CTX;
stx_priv->stx.fid.context = context;
stx_priv->stx.fid.ops = &psmx2_fi_ops_stx;
stx_priv->stx.ops = &psmx2_stx_ops;
stx_priv->domain = domain_priv;
stx_priv->tx = trx_ctxt;
ofi_atomic_initialize32(&stx_priv->ref, 0);
psmx2_lock(&domain_priv->trx_ctxt_lock, 1);
dlist_insert_before(&trx_ctxt->entry,
&domain_priv->trx_ctxt_list);
psmx2_unlock(&domain_priv->trx_ctxt_lock, 1);
*stx = &stx_priv->stx;
return 0;
errout_free_stx:
free(stx_priv);
errout:
return err;
}
static int psmx2_stx_close(fid_t fid)
{
struct psmx2_fid_stx *stx;
stx = container_of(fid, struct psmx2_fid_stx, stx.fid);
if (ofi_atomic_get32(&stx->ref))
return -FI_EBUSY;
psmx2_lock(&stx->domain->trx_ctxt_lock, 1);
dlist_remove(&stx->tx->entry);
psmx2_unlock(&stx->domain->trx_ctxt_lock, 1);
psmx2_trx_ctxt_free(stx->tx);
psmx2_domain_release(stx->domain);
free(stx);
return 0;
}
static int psmx2_ep_close(fid_t fid)
{
struct psmx2_fid_ep *ep;
struct psmx2_ep_name ep_name;
struct psmx2_trx_ctxt *trx_ctxt;
ep = container_of(fid, struct psmx2_fid_ep, ep.fid);
if (ep->base_ep) {
ofi_atomic_dec32(&ep->base_ep->ref);
return 0;
}
if (ofi_atomic_get32(&ep->ref))
return -FI_EBUSY;
if (ep->stx)
ofi_atomic_dec32(&ep->stx->ref);
if (ep->rx) {
ep_name.epid = ep->rx->psm2_epid;
ofi_ns_del_local_name(&ep->domain->fabric->name_server,
&ep->service, &ep_name);
}
if (ep->rx) {
psmx2_lock(&ep->domain->trx_ctxt_lock, 1);
dlist_remove(&ep->rx->entry);
psmx2_unlock(&ep->domain->trx_ctxt_lock, 1);
}
trx_ctxt = ep->rx;
psmx2_ep_close_internal(ep);
psmx2_trx_ctxt_free(trx_ctxt);
return 0;
}
int psmx2_sep_open(struct fid_domain *domain, struct fi_info *info,
struct fid_ep **sep, void *context)
{
struct psmx2_fid_domain *domain_priv;
struct psmx2_fid_ep *ep_priv;
struct psmx2_fid_sep *sep_priv;
struct psmx2_ep_name ep_name;
struct psmx2_ep_name *src_addr;
struct psmx2_trx_ctxt *trx_ctxt;
size_t ctxt_cnt = 1;
size_t ctxt_size;
int err = -FI_EINVAL;
int i;
domain_priv = container_of(domain, struct psmx2_fid_domain,
util_domain.domain_fid.fid);
if (!domain_priv)
goto errout;
if (info && info->ep_attr) {
if (info->ep_attr->tx_ctx_cnt > psmx2_env.sep_trx_ctxt) {
FI_WARN(&psmx2_prov, FI_LOG_EP_CTRL,
"tx_ctx_cnt %"PRIu64" exceed limit %d.\n",
info->ep_attr->tx_ctx_cnt,
psmx2_env.sep_trx_ctxt);
goto errout;
}
if (info->ep_attr->rx_ctx_cnt > psmx2_env.sep_trx_ctxt) {
FI_WARN(&psmx2_prov, FI_LOG_EP_CTRL,
"rx_ctx_cnt %"PRIu64" exceed limit %d.\n",
info->ep_attr->rx_ctx_cnt,
psmx2_env.sep_trx_ctxt);
goto errout;
}
ctxt_cnt = info->ep_attr->tx_ctx_cnt;
if (ctxt_cnt < info->ep_attr->rx_ctx_cnt)
ctxt_cnt = info->ep_attr->rx_ctx_cnt;
if (ctxt_cnt == 0) {
FI_INFO(&psmx2_prov, FI_LOG_EP_CTRL,
"tx_ctx_cnt and rx_ctx_cnt are 0, use 1.\n");
ctxt_cnt = 1;
}
}
ctxt_size = ctxt_cnt * sizeof(struct psmx2_sep_ctxt);
sep_priv = (struct psmx2_fid_sep *) calloc(1, sizeof(*sep_priv) + ctxt_size);
if (!sep_priv) {
err = -FI_ENOMEM;
goto errout;
}
sep_priv->ep.fid.fclass = FI_CLASS_SEP;
sep_priv->ep.fid.context = context;
sep_priv->ep.fid.ops = &psmx2_fi_ops_sep;
sep_priv->ep.ops = &psmx2_sep_ops;
sep_priv->ep.cm = &psmx2_cm_ops;
sep_priv->domain = domain_priv;
sep_priv->ctxt_cnt = ctxt_cnt;
ofi_atomic_initialize32(&sep_priv->ref, 0);
src_addr = NULL;
if (info && info->src_addr) {
if (info->addr_format == FI_ADDR_STR)
src_addr = psmx2_string_to_ep_name(info->src_addr);
else
src_addr = info->src_addr;
}
for (i = 0; i < ctxt_cnt; i++) {
trx_ctxt = psmx2_trx_ctxt_alloc(domain_priv, src_addr, i);
if (!trx_ctxt) {
err = -FI_ENOMEM;
goto errout_free_ctxt;
}
sep_priv->ctxts[i].trx_ctxt = trx_ctxt;
err = psmx2_ep_open_internal(domain_priv, info, &ep_priv, context,
trx_ctxt);
if (err)
goto errout_free_ctxt;
/* override the ops so the fid can't be closed individually */
ep_priv->ep.fid.ops = &psmx2_fi_ops_sep_ctxt;
trx_ctxt->ep = ep_priv;
sep_priv->ctxts[i].ep = ep_priv;
}
sep_priv->type = PSMX2_EP_SCALABLE;
sep_priv->service = PSMX2_ANY_SERVICE;
if (src_addr) {
sep_priv->service = src_addr->service;
if (info->addr_format == FI_ADDR_STR)
free(src_addr);
}
if (sep_priv->service == PSMX2_ANY_SERVICE)
sep_priv->service = ((getpid() & 0x7FFF) << 16) +
((uintptr_t)sep_priv & 0xFFFF);
sep_priv->id = ofi_atomic_inc32(&domain_priv->sep_cnt);
psmx2_lock(&domain_priv->sep_lock, 1);
dlist_insert_before(&sep_priv->entry, &domain_priv->sep_list);
psmx2_unlock(&domain_priv->sep_lock, 1);
psmx2_lock(&domain_priv->trx_ctxt_lock, 1);
for (i = 0; i< ctxt_cnt; i++) {
dlist_insert_before(&sep_priv->ctxts[i].trx_ctxt->entry,
&domain_priv->trx_ctxt_list);
}
psmx2_unlock(&domain_priv->trx_ctxt_lock, 1);
ep_name.epid = sep_priv->ctxts[0].trx_ctxt->psm2_epid;
ep_name.sep_id = sep_priv->id;
ep_name.type = sep_priv->type;
ofi_ns_add_local_name(&domain_priv->fabric->name_server,
&sep_priv->service, &ep_name);
psmx2_domain_acquire(domain_priv);
*sep = &sep_priv->ep;
/* Make sure the AM handler is installed to answer SEP query */
psmx2_am_init(sep_priv->ctxts[0].trx_ctxt);
return 0;
errout_free_ctxt:
while (i) {
if (sep_priv->ctxts[i].ep)
psmx2_ep_close_internal(sep_priv->ctxts[i].ep);
if (sep_priv->ctxts[i].trx_ctxt)
psmx2_trx_ctxt_free(sep_priv->ctxts[i].trx_ctxt);
i--;
}
free(sep_priv);
errout:
return err;
}
int psmx2_ep_open(struct fid_domain *domain, struct fi_info *info,
struct fid_ep **ep, void *context)
{
struct psmx2_fid_domain *domain_priv;
struct psmx2_fid_ep *ep_priv;
struct psmx2_ep_name ep_name;
struct psmx2_ep_name *src_addr;
struct psmx2_trx_ctxt *trx_ctxt = NULL;
int err = -FI_EINVAL;
int alloc_trx_ctxt = 1;
domain_priv = container_of(domain, struct psmx2_fid_domain,
util_domain.domain_fid.fid);
if (!domain_priv)
goto errout;
if (info && info->ep_attr && info->ep_attr->rx_ctx_cnt == FI_SHARED_CONTEXT)
return -FI_ENOSYS;
if (info && info->ep_attr && info->ep_attr->tx_ctx_cnt == FI_SHARED_CONTEXT &&
!ofi_recv_allowed(info->caps) && !ofi_rma_target_allowed(info->caps)) {
alloc_trx_ctxt = 0;
FI_INFO(&psmx2_prov, FI_LOG_EP_CTRL, "Tx only endpoint with STX context.\n");
}
src_addr = NULL;
if (info && info->src_addr) {
if (info->addr_format == FI_ADDR_STR)
src_addr = psmx2_string_to_ep_name(info->src_addr);
else
src_addr = info->src_addr;
}
if (alloc_trx_ctxt) {
trx_ctxt = psmx2_trx_ctxt_alloc(domain_priv, src_addr, 0);
if (!trx_ctxt)
goto errout;
}
err = psmx2_ep_open_internal(domain_priv, info, &ep_priv, context,
trx_ctxt);
if (err)
goto errout_free_ctxt;
ep_priv->type = PSMX2_EP_REGULAR;
ep_priv->service = PSMX2_ANY_SERVICE;
if (src_addr) {
ep_priv->service = src_addr->service;
if (info->addr_format == FI_ADDR_STR)
free(src_addr);
}
if (ep_priv->service == PSMX2_ANY_SERVICE)
ep_priv->service = ((getpid() & 0x7FFF) << 16) +
((uintptr_t)ep_priv & 0xFFFF);
if (alloc_trx_ctxt) {
trx_ctxt->ep = ep_priv;
psmx2_lock(&domain_priv->trx_ctxt_lock, 1);
dlist_insert_before(&trx_ctxt->entry,
&domain_priv->trx_ctxt_list);
psmx2_unlock(&domain_priv->trx_ctxt_lock, 1);
ep_name.epid = trx_ctxt->psm2_epid;
ep_name.type = ep_priv->type;
ofi_ns_add_local_name(&domain_priv->fabric->name_server,
&ep_priv->service, &ep_name);
}
*ep = &ep_priv->ep;
return 0;
errout_free_ctxt:
psmx2_trx_ctxt_free(trx_ctxt);
errout:
return err;
}
int psmx2_domain_open(struct fid_fabric *fabric, struct fi_info *info,
struct fid_domain **domain, void *context)
{
struct psmx2_fid_fabric *fabric_priv;
struct psmx2_fid_domain *domain_priv;
struct psmx2_ep_name *src_addr = info->src_addr;
int mr_mode = (info->domain_attr->mr_mode & FI_MR_BASIC) ? FI_MR_BASIC : 0;
int err;
FI_INFO(&psmx2_prov, FI_LOG_DOMAIN, "\n");
fabric_priv = container_of(fabric, struct psmx2_fid_fabric,
util_fabric.fabric_fid);
if (!info->domain_attr->name ||
strcmp(info->domain_attr->name, PSMX2_DOMAIN_NAME)) {
err = -FI_EINVAL;
goto err_out;
}
domain_priv = (struct psmx2_fid_domain *) calloc(1, sizeof *domain_priv);
if (!domain_priv) {
err = -FI_ENOMEM;
goto err_out;
}
err = ofi_domain_init(fabric, info, &domain_priv->util_domain, context);
if (err)
goto err_out_free_domain;
/* fclass & context are set in ofi_domain_init */
domain_priv->util_domain.domain_fid.fid.ops = &psmx2_fi_ops;
domain_priv->util_domain.domain_fid.ops = &psmx2_domain_ops;
domain_priv->util_domain.domain_fid.mr = &psmx2_mr_ops;
domain_priv->mr_mode = mr_mode;
domain_priv->mode = info->mode;
domain_priv->caps = PSMX2_CAPS | PSMX2_DOM_CAPS;
domain_priv->fabric = fabric_priv;
domain_priv->progress_thread_enabled =
(info->domain_attr->data_progress == FI_PROGRESS_AUTO);
domain_priv->addr_format = info->addr_format;
if (info->addr_format == FI_ADDR_STR)
src_addr = psmx2_string_to_ep_name(info->src_addr);
err = psmx2_domain_init(domain_priv, src_addr);
if (info->addr_format == FI_ADDR_STR)
free(src_addr);
if (err)
goto err_out_close_domain;
psmx2_fabric_acquire(fabric_priv);
psmx2_lock(&fabric_priv->domain_lock, 1);
dlist_insert_before(&domain_priv->entry, &fabric_priv->domain_list);
psmx2_unlock(&fabric_priv->domain_lock, 1);
*domain = &domain_priv->util_domain.domain_fid;
return 0;
err_out_close_domain:
ofi_domain_close(&domain_priv->util_domain);
err_out_free_domain:
free(domain_priv);
err_out:
return err;
}
