static int psmx2_domain_init(struct psmx2_fid_domain *domain, struct psmx2_src_name *src_addr) { int err; psmx2_am_global_init(); psmx2_atomic_global_init(); domain->base_trx_ctxt = psmx2_trx_ctxt_alloc(domain, src_addr, -1); if (!domain->base_trx_ctxt) return -FI_ENODEV; err = fastlock_init(&domain->mr_lock); if (err) { FI_WARN(&psmx2_prov, FI_LOG_CORE, "fastlock_init(mr_lock) returns %d\n", err); goto err_out_free_trx_ctxt; } domain->mr_map = rbtNew(&psmx2_key_compare); if (!domain->mr_map) { FI_WARN(&psmx2_prov, FI_LOG_CORE, "rbtNew failed\n"); goto err_out_destroy_mr_lock; } domain->mr_reserved_key = 1; err = fastlock_init(&domain->vl_lock); if (err) { FI_WARN(&psmx2_prov, FI_LOG_CORE, "fastlock_init(vl_lock) returns %d\n", err); goto err_out_delete_mr_map; } memset(domain->vl_map, 0, sizeof(domain->vl_map)); domain->vl_alloc = 0; ofi_atomic_initialize32(&domain->sep_cnt, 0); fastlock_init(&domain->sep_lock); dlist_init(&domain->sep_list); dlist_init(&domain->trx_ctxt_list); fastlock_init(&domain->trx_ctxt_lock); dlist_insert_before(&domain->base_trx_ctxt->entry, &domain->trx_ctxt_list); /* Set active domain before psmx2_domain_enable_ep() installs the * AM handlers to ensure that psmx2_active_fabric->active_domain * is always non-NULL inside the handlers. Notice that the vlaue * active_domain becomes NULL again only when the domain is closed. * At that time the AM handlers are gone with the PSM endpoint. */ domain->fabric->active_domain = domain; if (psmx2_domain_enable_ep(domain, NULL) < 0) goto err_out_reset_active_domain; if (domain->progress_thread_enabled) psmx2_domain_start_progress(domain); psmx2_am_init(domain->base_trx_ctxt); return 0; err_out_reset_active_domain: domain->fabric->active_domain = NULL; fastlock_destroy(&domain->vl_lock); err_out_delete_mr_map: rbtDelete(domain->mr_map); err_out_destroy_mr_lock: fastlock_destroy(&domain->mr_lock); err_out_free_trx_ctxt: psmx2_trx_ctxt_free(domain->base_trx_ctxt); return err; }
int psmx2_fabric(struct fi_fabric_attr *attr, struct fid_fabric **fabric, void *context) { struct psmx2_fid_fabric *fabric_priv; int ret; FI_INFO(&psmx2_prov, FI_LOG_CORE, "\n"); if (strcmp(attr->name, PSMX2_FABRIC_NAME)) return -FI_ENODATA; if (psmx2_active_fabric) { psmx2_fabric_acquire(psmx2_active_fabric); *fabric = &psmx2_active_fabric->util_fabric.fabric_fid; return 0; } fabric_priv = calloc(1, sizeof(*fabric_priv)); if (!fabric_priv) return -FI_ENOMEM; fastlock_init(&fabric_priv->domain_lock); dlist_init(&fabric_priv->domain_list); psmx2_get_uuid(fabric_priv->uuid); if (psmx2_env.name_server) { fabric_priv->name_server.port = psmx2_uuid_to_port(fabric_priv->uuid); fabric_priv->name_server.name_len = sizeof(struct psmx2_ep_name); fabric_priv->name_server.service_len = sizeof(int); fabric_priv->name_server.service_cmp = psmx2_ns_service_cmp; fabric_priv->name_server.is_service_wildcard = psmx2_ns_is_service_wildcard; ofi_ns_init(&fabric_priv->name_server); ofi_ns_start_server(&fabric_priv->name_server); } ret = ofi_fabric_init(&psmx2_prov, &psmx2_fabric_attr, attr, &fabric_priv->util_fabric, context); if (ret) { FI_INFO(&psmx2_prov, FI_LOG_CORE, "ofi_fabric_init returns %d\n", ret); if (psmx2_env.name_server) ofi_ns_stop_server(&fabric_priv->name_server); free(fabric_priv); return ret; } /* fclass & context initialized in ofi_fabric_init */ fabric_priv->util_fabric.fabric_fid.fid.ops = &psmx2_fabric_fi_ops; fabric_priv->util_fabric.fabric_fid.ops = &psmx2_fabric_ops; psmx2_atomic_global_init(); psmx2_query_mpi(); /* take the reference to count for multiple fabric open calls */ psmx2_fabric_acquire(fabric_priv); *fabric = &fabric_priv->util_fabric.fabric_fid; psmx2_active_fabric = fabric_priv; return 0; }