int psmx2_stx_ctx(struct fid_domain *domain, struct fi_tx_attr *attr, struct fid_stx **stx, void *context) { struct psmx2_fid_domain *domain_priv; struct psmx2_fid_stx *stx_priv; FI_INFO(&psmx2_prov, FI_LOG_EP_DATA, "\n"); domain_priv = container_of(domain, struct psmx2_fid_domain, util_domain.domain_fid.fid); stx_priv = (struct psmx2_fid_stx *) calloc(1, sizeof *stx_priv); if (!stx_priv) return -FI_ENOMEM; psmx2_domain_acquire(domain_priv); stx_priv->stx.fid.fclass = FI_CLASS_STX_CTX; stx_priv->stx.fid.context = context; stx_priv->stx.fid.ops = &psmx2_fi_ops_stx; stx_priv->domain = domain_priv; *stx = &stx_priv->stx; return 0; }
int psmx2_av_open(struct fid_domain *domain, struct fi_av_attr *attr, struct fid_av **av, void *context) { struct psmx2_fid_domain *domain_priv; struct psmx2_fid_av *av_priv; int type = FI_AV_MAP; size_t count = 64; uint64_t flags = 0; domain_priv = container_of(domain, struct psmx2_fid_domain, util_domain.domain_fid); if (attr) { switch (attr->type) { case FI_AV_MAP: case FI_AV_TABLE: type = attr->type; break; default: FI_INFO(&psmx2_prov, FI_LOG_AV, "attr->type=%d, supported=%d %d\n", attr->type, FI_AV_MAP, FI_AV_TABLE); return -FI_EINVAL; } count = attr->count; flags = attr->flags; if (flags & (FI_READ | FI_SYMMETRIC)) { FI_INFO(&psmx2_prov, FI_LOG_AV, "attr->flags=%x, supported=%x\n", attr->flags, FI_EVENT); return -FI_EINVAL; } } av_priv = (struct psmx2_fid_av *) calloc(1, sizeof *av_priv); if (!av_priv) return -FI_ENOMEM; psmx2_domain_acquire(domain_priv); av_priv->domain = domain_priv; av_priv->type = type; av_priv->addrlen = sizeof(psm2_epaddr_t); av_priv->count = count; av_priv->flags = flags; av_priv->av.fid.fclass = FI_CLASS_AV; av_priv->av.fid.context = context; av_priv->av.fid.ops = &psmx2_fi_ops; av_priv->av.ops = &psmx2_av_ops; *av = &av_priv->av; return 0; }
static int psmx2_mr_regattr(struct fid *fid, const struct fi_mr_attr *attr, uint64_t flags, struct fid_mr **mr) { struct fid_domain *domain; struct psmx2_fid_domain *domain_priv; struct psmx2_fid_mr *mr_priv; int i, err; uint64_t key; if (fid->fclass != FI_CLASS_DOMAIN) { return -FI_EINVAL; } domain = container_of(fid, struct fid_domain, fid); domain_priv = container_of(domain, struct psmx2_fid_domain, util_domain.domain_fid); if (!attr) return -FI_EINVAL; if (attr->iov_count == 0 || attr->mr_iov == NULL) return -FI_EINVAL; mr_priv = (struct psmx2_fid_mr *) calloc(1, sizeof(*mr_priv) + sizeof(struct iovec) * attr->iov_count); if (!mr_priv) return -FI_ENOMEM; err = psmx2_mr_reserve_key(domain_priv, attr->requested_key, &key, mr_priv); if (err) { free(mr_priv); return err; } psmx2_domain_acquire(domain_priv); mr_priv->mr.fid.fclass = FI_CLASS_MR; mr_priv->mr.fid.context = attr->context; mr_priv->mr.fid.ops = &psmx2_fi_ops; mr_priv->mr.mem_desc = mr_priv; mr_priv->mr.key = key; mr_priv->domain = domain_priv; mr_priv->access = attr->access; mr_priv->flags = flags; mr_priv->iov_count = attr->iov_count; for (i=0; i<attr->iov_count; i++) mr_priv->iov[i] = attr->mr_iov[i]; psmx2_mr_normalize_iov(mr_priv->iov, &mr_priv->iov_count); mr_priv->offset = (domain_priv->mr_mode == FI_MR_SCALABLE) ? ((uint64_t)mr_priv->iov[0].iov_base - attr->offset) : 0; *mr = &mr_priv->mr; return 0; }
static int psmx2_mr_reg(struct fid *fid, const void *buf, size_t len, uint64_t access, uint64_t offset, uint64_t requested_key, uint64_t flags, struct fid_mr **mr, void *context) { struct fid_domain *domain; struct psmx2_fid_domain *domain_priv; struct psmx2_fid_mr *mr_priv; uint64_t key; int err; if (fid->fclass != FI_CLASS_DOMAIN) { return -FI_EINVAL; } domain = container_of(fid, struct fid_domain, fid); domain_priv = container_of(domain, struct psmx2_fid_domain, util_domain.domain_fid); mr_priv = (struct psmx2_fid_mr *) calloc(1, sizeof(*mr_priv) + sizeof(struct iovec)); if (!mr_priv) return -FI_ENOMEM; err = psmx2_mr_reserve_key(domain_priv, requested_key, &key, mr_priv); if (err) { free(mr_priv); return err; } psmx2_domain_acquire(domain_priv); mr_priv->mr.fid.fclass = FI_CLASS_MR; mr_priv->mr.fid.context = context; mr_priv->mr.fid.ops = &psmx2_fi_ops; mr_priv->mr.mem_desc = mr_priv; mr_priv->mr.fid.fclass = FI_CLASS_MR; mr_priv->mr.fid.context = context; mr_priv->mr.fid.ops = &psmx2_fi_ops; mr_priv->mr.mem_desc = mr_priv; mr_priv->mr.key = key; mr_priv->domain = domain_priv; mr_priv->access = access; mr_priv->flags = flags; mr_priv->iov_count = 1; mr_priv->iov[0].iov_base = (void *)buf; mr_priv->iov[0].iov_len = len; mr_priv->offset = (domain_priv->mr_mode == FI_MR_SCALABLE) ? ((uint64_t)mr_priv->iov[0].iov_base - offset) : 0; *mr = &mr_priv->mr; return 0; }
int psmx2_stx_ctx(struct fid_domain *domain, struct fi_tx_attr *attr, struct fid_stx **stx, void *context) { struct psmx2_fid_domain *domain_priv; struct psmx2_trx_ctxt *trx_ctxt; struct psmx2_fid_stx *stx_priv; int err = -FI_EINVAL; domain_priv = container_of(domain, struct psmx2_fid_domain, util_domain.domain_fid.fid); if (!domain_priv) goto errout; stx_priv = (struct psmx2_fid_stx *) calloc(1, sizeof *stx_priv); if (!stx_priv) { err = -FI_ENOMEM; goto errout; } trx_ctxt = psmx2_trx_ctxt_alloc(domain_priv, NULL/*src_addr*/, 0); if (!trx_ctxt) { err = -FI_ENOMEM; goto errout_free_stx; } psmx2_domain_acquire(domain_priv); stx_priv->stx.fid.fclass = FI_CLASS_STX_CTX; stx_priv->stx.fid.context = context; stx_priv->stx.fid.ops = &psmx2_fi_ops_stx; stx_priv->stx.ops = &psmx2_stx_ops; stx_priv->domain = domain_priv; stx_priv->tx = trx_ctxt; ofi_atomic_initialize32(&stx_priv->ref, 0); psmx2_lock(&domain_priv->trx_ctxt_lock, 1); dlist_insert_before(&trx_ctxt->entry, &domain_priv->trx_ctxt_list); psmx2_unlock(&domain_priv->trx_ctxt_lock, 1); *stx = &stx_priv->stx; return 0; errout_free_stx: free(stx_priv); errout: return err; }
int psmx2_sep_open(struct fid_domain *domain, struct fi_info *info, struct fid_ep **sep, void *context) { struct psmx2_fid_domain *domain_priv; struct psmx2_fid_ep *ep_priv; struct psmx2_fid_sep *sep_priv; struct psmx2_ep_name ep_name; struct psmx2_ep_name *src_addr; struct psmx2_trx_ctxt *trx_ctxt; size_t ctxt_cnt = 1; size_t ctxt_size; int err = -FI_EINVAL; int i; domain_priv = container_of(domain, struct psmx2_fid_domain, util_domain.domain_fid.fid); if (!domain_priv) goto errout; if (info && info->ep_attr) { if (info->ep_attr->tx_ctx_cnt > psmx2_env.sep_trx_ctxt) { FI_WARN(&psmx2_prov, FI_LOG_EP_CTRL, "tx_ctx_cnt %"PRIu64" exceed limit %d.\n", info->ep_attr->tx_ctx_cnt, psmx2_env.sep_trx_ctxt); goto errout; } if (info->ep_attr->rx_ctx_cnt > psmx2_env.sep_trx_ctxt) { FI_WARN(&psmx2_prov, FI_LOG_EP_CTRL, "rx_ctx_cnt %"PRIu64" exceed limit %d.\n", info->ep_attr->rx_ctx_cnt, psmx2_env.sep_trx_ctxt); goto errout; } ctxt_cnt = info->ep_attr->tx_ctx_cnt; if (ctxt_cnt < info->ep_attr->rx_ctx_cnt) ctxt_cnt = info->ep_attr->rx_ctx_cnt; if (ctxt_cnt == 0) { FI_INFO(&psmx2_prov, FI_LOG_EP_CTRL, "tx_ctx_cnt and rx_ctx_cnt are 0, use 1.\n"); ctxt_cnt = 1; } } ctxt_size = ctxt_cnt * sizeof(struct psmx2_sep_ctxt); sep_priv = (struct psmx2_fid_sep *) calloc(1, sizeof(*sep_priv) + ctxt_size); if (!sep_priv) { err = -FI_ENOMEM; goto errout; } sep_priv->ep.fid.fclass = FI_CLASS_SEP; sep_priv->ep.fid.context = context; sep_priv->ep.fid.ops = &psmx2_fi_ops_sep; sep_priv->ep.ops = &psmx2_sep_ops; sep_priv->ep.cm = &psmx2_cm_ops; sep_priv->domain = domain_priv; sep_priv->ctxt_cnt = ctxt_cnt; ofi_atomic_initialize32(&sep_priv->ref, 0); src_addr = NULL; if (info && info->src_addr) { if (info->addr_format == FI_ADDR_STR) src_addr = psmx2_string_to_ep_name(info->src_addr); else src_addr = info->src_addr; } for (i = 0; i < ctxt_cnt; i++) { trx_ctxt = psmx2_trx_ctxt_alloc(domain_priv, src_addr, i); if (!trx_ctxt) { err = -FI_ENOMEM; goto errout_free_ctxt; } sep_priv->ctxts[i].trx_ctxt = trx_ctxt; err = psmx2_ep_open_internal(domain_priv, info, &ep_priv, context, trx_ctxt); if (err) goto errout_free_ctxt; /* override the ops so the fid can't be closed individually */ ep_priv->ep.fid.ops = &psmx2_fi_ops_sep_ctxt; trx_ctxt->ep = ep_priv; sep_priv->ctxts[i].ep = ep_priv; } sep_priv->type = PSMX2_EP_SCALABLE; sep_priv->service = PSMX2_ANY_SERVICE; if (src_addr) { sep_priv->service = src_addr->service; if (info->addr_format == FI_ADDR_STR) free(src_addr); } if (sep_priv->service == PSMX2_ANY_SERVICE) sep_priv->service = ((getpid() & 0x7FFF) << 16) + ((uintptr_t)sep_priv & 0xFFFF); sep_priv->id = ofi_atomic_inc32(&domain_priv->sep_cnt); psmx2_lock(&domain_priv->sep_lock, 1); dlist_insert_before(&sep_priv->entry, &domain_priv->sep_list); psmx2_unlock(&domain_priv->sep_lock, 1); psmx2_lock(&domain_priv->trx_ctxt_lock, 1); for (i = 0; i< ctxt_cnt; i++) { dlist_insert_before(&sep_priv->ctxts[i].trx_ctxt->entry, &domain_priv->trx_ctxt_list); } psmx2_unlock(&domain_priv->trx_ctxt_lock, 1); ep_name.epid = sep_priv->ctxts[0].trx_ctxt->psm2_epid; ep_name.sep_id = sep_priv->id; ep_name.type = sep_priv->type; ofi_ns_add_local_name(&domain_priv->fabric->name_server, &sep_priv->service, &ep_name); psmx2_domain_acquire(domain_priv); *sep = &sep_priv->ep; /* Make sure the AM handler is installed to answer SEP query */ psmx2_am_init(sep_priv->ctxts[0].trx_ctxt); return 0; errout_free_ctxt: while (i) { if (sep_priv->ctxts[i].ep) psmx2_ep_close_internal(sep_priv->ctxts[i].ep); if (sep_priv->ctxts[i].trx_ctxt) psmx2_trx_ctxt_free(sep_priv->ctxts[i].trx_ctxt); i--; } free(sep_priv); errout: return err; }
int psmx2_ep_open_internal(struct psmx2_fid_domain *domain_priv, struct fi_info *info, struct psmx2_fid_ep **ep_out, void *context, struct psmx2_trx_ctxt *trx_ctxt) { struct psmx2_fid_ep *ep_priv; uint64_t ep_cap; int err = -FI_EINVAL; if (info) ep_cap = info->caps; else ep_cap = FI_TAGGED; if (info && info->ep_attr && info->ep_attr->auth_key) { if (info->ep_attr->auth_key_size != sizeof(psm2_uuid_t)) { FI_WARN(&psmx2_prov, FI_LOG_EP_CTRL, "Invalid auth_key_len %"PRIu64 ", should be %"PRIu64".\n", info->ep_attr->auth_key_size, sizeof(psm2_uuid_t)); goto errout; } if (memcmp(domain_priv->fabric->uuid, info->ep_attr->auth_key, sizeof(psm2_uuid_t))) { FI_WARN(&psmx2_prov, FI_LOG_EP_CTRL, "Invalid auth_key: %s\n", psmx2_uuid_to_string((void *)info->ep_attr->auth_key)); goto errout; } } ep_priv = (struct psmx2_fid_ep *) calloc(1, sizeof *ep_priv); if (!ep_priv) { err = -FI_ENOMEM; goto errout; } ep_priv->ep.fid.fclass = FI_CLASS_EP; ep_priv->ep.fid.context = context; ep_priv->ep.fid.ops = &psmx2_fi_ops; ep_priv->ep.ops = &psmx2_ep_ops; ep_priv->ep.cm = &psmx2_cm_ops; ep_priv->domain = domain_priv; ep_priv->rx = trx_ctxt; if (!(info && info->ep_attr && info->ep_attr->tx_ctx_cnt == FI_SHARED_CONTEXT)) ep_priv->tx = trx_ctxt; ofi_atomic_initialize32(&ep_priv->ref, 0); PSMX2_CTXT_TYPE(&ep_priv->nocomp_send_context) = PSMX2_NOCOMP_SEND_CONTEXT; PSMX2_CTXT_EP(&ep_priv->nocomp_send_context) = ep_priv; PSMX2_CTXT_TYPE(&ep_priv->nocomp_tsend_context) = PSMX2_NOCOMP_TSEND_CONTEXT; PSMX2_CTXT_EP(&ep_priv->nocomp_tsend_context) = ep_priv; if (ep_cap & FI_TAGGED) ep_priv->ep.tagged = &psmx2_tagged_ops; if (ep_cap & FI_MSG) ep_priv->ep.msg = &psmx2_msg_ops; if (ep_cap & FI_RMA) ep_priv->ep.rma = &psmx2_rma_ops; if (ep_cap & FI_ATOMICS) ep_priv->ep.atomic = &psmx2_atomic_ops; ep_priv->caps = ep_cap; err = psmx2_domain_enable_ep(domain_priv, ep_priv); if (err) goto errout_free_ep; psmx2_domain_acquire(domain_priv); if (info) { if (info->tx_attr) ep_priv->tx_flags = info->tx_attr->op_flags; if (info->rx_attr) ep_priv->rx_flags = info->rx_attr->op_flags; } psmx2_ep_optimize_ops(ep_priv); PSMX2_EP_INIT_OP_CONTEXT(ep_priv); *ep_out = ep_priv; return 0; errout_free_ep: free(ep_priv); errout: return err; }
int psmx2_domain_open(struct fid_fabric *fabric, struct fi_info *info, struct fid_domain **domain, void *context) { struct psmx2_fid_fabric *fabric_priv; struct psmx2_fid_domain *domain_priv; int mr_mode = (info->domain_attr->mr_mode & FI_MR_BASIC) ? FI_MR_BASIC : 0; int err; FI_INFO(&psmx2_prov, FI_LOG_DOMAIN, "\n"); if (!psmx2_env.sep) psmx2_domain_ops.scalable_ep = fi_no_scalable_ep; fabric_priv = container_of(fabric, struct psmx2_fid_fabric, util_fabric.fabric_fid); if (fabric_priv->active_domain) { if (mr_mode != fabric_priv->active_domain->mr_mode) { FI_INFO(&psmx2_prov, FI_LOG_DOMAIN, "mr_mode mismatch: expecting %s\n", mr_mode ? "FI_MR_SCALABLE" : "FI_MR_BASIC"); return -FI_EINVAL; } psmx2_domain_acquire(fabric_priv->active_domain); *domain = &fabric_priv->active_domain->util_domain.domain_fid; return 0; } if (!info->domain_attr->name || strcmp(info->domain_attr->name, PSMX2_DOMAIN_NAME)) { err = -FI_EINVAL; goto err_out; } domain_priv = (struct psmx2_fid_domain *) calloc(1, sizeof *domain_priv); if (!domain_priv) { err = -FI_ENOMEM; goto err_out; } err = ofi_domain_init(fabric, info, &domain_priv->util_domain, context); if (err) goto err_out_free_domain; /* fclass & context are set in ofi_domain_init */ domain_priv->util_domain.domain_fid.fid.ops = &psmx2_fi_ops; domain_priv->util_domain.domain_fid.ops = &psmx2_domain_ops; domain_priv->util_domain.domain_fid.mr = &psmx2_mr_ops; domain_priv->mr_mode = mr_mode; domain_priv->mode = info->mode; domain_priv->caps = PSMX2_CAPS | PSMX2_DOM_CAPS; domain_priv->fabric = fabric_priv; domain_priv->progress_thread_enabled = (info->domain_attr->data_progress == FI_PROGRESS_AUTO); err = psmx2_domain_init(domain_priv, info->src_addr); if (err) goto err_out_close_domain; /* take the reference to count for multiple domain open calls */ psmx2_domain_acquire(fabric_priv->active_domain); *domain = &domain_priv->util_domain.domain_fid; return 0; err_out_close_domain: ofi_domain_close(&domain_priv->util_domain); err_out_free_domain: free(domain_priv); err_out: return err; }
int psmx2_ep_open(struct fid_domain *domain, struct fi_info *info, struct fid_ep **ep, void *context) { struct psmx2_fid_domain *domain_priv; struct psmx2_fid_ep *ep_priv; struct psmx2_context *item; uint8_t vlane; uint64_t ep_cap; int err = -FI_EINVAL; int i; if (info) ep_cap = info->caps; else ep_cap = FI_TAGGED; domain_priv = container_of(domain, struct psmx2_fid_domain, domain.fid); if (!domain_priv) goto errout; err = psmx2_domain_check_features(domain_priv, ep_cap); if (err) goto errout; err = psmx2_alloc_vlane(domain_priv, &vlane); if (err) goto errout; ep_priv = (struct psmx2_fid_ep *) calloc(1, sizeof *ep_priv); if (!ep_priv) { err = -FI_ENOMEM; goto errout_free_vlane; } ep_priv->ep.fid.fclass = FI_CLASS_EP; ep_priv->ep.fid.context = context; ep_priv->ep.fid.ops = &psmx2_fi_ops; ep_priv->ep.ops = &psmx2_ep_ops; ep_priv->ep.cm = &psmx2_cm_ops; ep_priv->domain = domain_priv; ep_priv->vlane = vlane; PSMX2_CTXT_TYPE(&ep_priv->nocomp_send_context) = PSMX2_NOCOMP_SEND_CONTEXT; PSMX2_CTXT_EP(&ep_priv->nocomp_send_context) = ep_priv; PSMX2_CTXT_TYPE(&ep_priv->nocomp_recv_context) = PSMX2_NOCOMP_RECV_CONTEXT; PSMX2_CTXT_EP(&ep_priv->nocomp_recv_context) = ep_priv; if (ep_cap & FI_TAGGED) ep_priv->ep.tagged = &psmx2_tagged_ops; if (ep_cap & FI_MSG) ep_priv->ep.msg = &psmx2_msg_ops; if (ep_cap & FI_RMA) ep_priv->ep.rma = &psmx2_rma_ops; if (ep_cap & FI_ATOMICS) ep_priv->ep.atomic = &psmx2_atomic_ops; ep_priv->caps = ep_cap; err = psmx2_domain_enable_ep(domain_priv, ep_priv); if (err) goto errout_free_ep; psmx2_domain_acquire(domain_priv); domain_priv->eps[ep_priv->vlane] = ep_priv; if (info) { if (info->tx_attr) ep_priv->flags = info->tx_attr->op_flags; if (info->rx_attr) ep_priv->flags |= info->rx_attr->op_flags; } psmx2_ep_optimize_ops(ep_priv); slist_init(&ep_priv->free_context_list); fastlock_init(&ep_priv->context_lock); #define PSMX2_FREE_CONTEXT_LIST_SIZE 64 for (i=0; i<PSMX2_FREE_CONTEXT_LIST_SIZE; i++) { item = calloc(1, sizeof(*item)); if (!item) { FI_WARN(&psmx2_prov, FI_LOG_EP_CTRL, "out of memory.\n"); exit(-1); } slist_insert_tail(&item->list_entry, &ep_priv->free_context_list); } *ep = &ep_priv->ep; return 0; errout_free_ep: free(ep_priv); errout_free_vlane: psmx2_free_vlane(domain_priv, vlane); errout: return err; }
int psmx2_domain_open(struct fid_fabric *fabric, struct fi_info *info, struct fid_domain **domain, void *context) { struct psmx2_fid_fabric *fabric_priv; struct psmx2_fid_domain *domain_priv; struct psm2_ep_open_opts opts; int err; FI_INFO(&psmx2_prov, FI_LOG_DOMAIN, "\n"); fabric_priv = container_of(fabric, struct psmx2_fid_fabric, fabric); psmx2_fabric_acquire(fabric_priv); if (fabric_priv->active_domain) { psmx2_domain_acquire(fabric_priv->active_domain); *domain = &fabric_priv->active_domain->domain; return 0; } if (!info->domain_attr->name || strcmp(info->domain_attr->name, PSMX2_DOMAIN_NAME)) { err = -FI_EINVAL; goto err_out; } domain_priv = (struct psmx2_fid_domain *) calloc(1, sizeof *domain_priv); if (!domain_priv) { err = -FI_ENOMEM; goto err_out; } domain_priv->domain.fid.fclass = FI_CLASS_DOMAIN; domain_priv->domain.fid.context = context; domain_priv->domain.fid.ops = &psmx2_fi_ops; domain_priv->domain.ops = &psmx2_domain_ops; domain_priv->domain.mr = &psmx2_mr_ops; domain_priv->mr_mode = info->domain_attr->mr_mode; domain_priv->mode = info->mode; domain_priv->caps = info->caps; domain_priv->fabric = fabric_priv; domain_priv->progress_thread_enabled = (info->domain_attr->data_progress == FI_PROGRESS_AUTO); psm2_ep_open_opts_get_defaults(&opts); FI_INFO(&psmx2_prov, FI_LOG_CORE, "uuid: %s\n", psmx2_uuid_to_string(fabric_priv->uuid)); err = psm2_ep_open(fabric_priv->uuid, &opts, &domain_priv->psm2_ep, &domain_priv->psm2_epid); if (err != PSM2_OK) { FI_WARN(&psmx2_prov, FI_LOG_CORE, "psm2_ep_open returns %d, errno=%d\n", err, errno); err = psmx2_errno(err); goto err_out_free_domain; } FI_INFO(&psmx2_prov, FI_LOG_CORE, "epid: 0x%016lx\n", domain_priv->psm2_epid); err = psm2_mq_init(domain_priv->psm2_ep, PSM2_MQ_ORDERMASK_ALL, NULL, 0, &domain_priv->psm2_mq); if (err != PSM2_OK) { FI_WARN(&psmx2_prov, FI_LOG_CORE, "psm2_mq_init returns %d, errno=%d\n", err, errno); err = psmx2_errno(err); goto err_out_close_ep; } err = fastlock_init(&domain_priv->mr_lock); if (err) { FI_WARN(&psmx2_prov, FI_LOG_CORE, "fastlock_init(mr_lock) returns %d\n", err); goto err_out_finalize_mq; } domain_priv->mr_map = rbtNew(&psmx2_key_compare); if (!domain_priv->mr_map) { FI_WARN(&psmx2_prov, FI_LOG_CORE, "rbtNew failed\n"); goto err_out_destroy_mr_lock; } domain_priv->mr_reserved_key = 1; err = fastlock_init(&domain_priv->vl_lock); if (err) { FI_WARN(&psmx2_prov, FI_LOG_CORE, "fastlock_init(vl_lock) returns %d\n", err); goto err_out_delete_mr_map; } memset(domain_priv->vl_map, 0, sizeof(domain_priv->vl_map)); domain_priv->vl_alloc = 0; err = fastlock_init(&domain_priv->poll_lock); if (err) { FI_WARN(&psmx2_prov, FI_LOG_CORE, "fastlock_init(poll_lock) returns %d\n", err); goto err_out_destroy_vl_lock; } /* Set active domain before psmx2_domain_enable_ep() installs the * AM handlers to ensure that psmx2_active_fabric->active_domain * is always non-NULL inside the handlers. Notice that the vlaue * active_domain becomes NULL again only when the domain is closed. * At that time the AM handlers are gone with the PSM endpoint. */ fabric_priv->active_domain = domain_priv; if (psmx2_domain_enable_ep(domain_priv, NULL) < 0) goto err_out_reset_active_domain; if (domain_priv->progress_thread_enabled) psmx2_domain_start_progress(domain_priv); domain_priv->refcnt = 1; *domain = &domain_priv->domain; return 0; err_out_reset_active_domain: fabric_priv->active_domain = NULL; fastlock_destroy(&domain_priv->poll_lock); err_out_destroy_vl_lock: fastlock_destroy(&domain_priv->vl_lock); err_out_delete_mr_map: rbtDelete(domain_priv->mr_map); err_out_destroy_mr_lock: fastlock_destroy(&domain_priv->mr_lock); err_out_finalize_mq: psm2_mq_finalize(domain_priv->psm2_mq); err_out_close_ep: if (psm2_ep_close(domain_priv->psm2_ep, PSM2_EP_CLOSE_GRACEFUL, (int64_t) psmx2_env.timeout * 1000000000LL) != PSM2_OK) psm2_ep_close(domain_priv->psm2_ep, PSM2_EP_CLOSE_FORCE, 0); err_out_free_domain: free(domain_priv); err_out: psmx2_fabric_release(fabric_priv); return err; }
int psmx2_cntr_open(struct fid_domain *domain, struct fi_cntr_attr *attr, struct fid_cntr **cntr, void *context) { struct psmx2_fid_domain *domain_priv; struct psmx2_fid_cntr *cntr_priv; struct fid_wait *wait = NULL; struct fi_wait_attr wait_attr; int wait_is_local = 0; int events; uint64_t flags; int err; events = FI_CNTR_EVENTS_COMP; flags = 0; domain_priv = container_of(domain, struct psmx2_fid_domain, util_domain.domain_fid); switch (attr->events) { case FI_CNTR_EVENTS_COMP: events = attr->events; break; default: FI_INFO(&psmx2_prov, FI_LOG_CQ, "attr->events=%d, supported=%d\n", attr->events, FI_CNTR_EVENTS_COMP); return -FI_EINVAL; } switch (attr->wait_obj) { case FI_WAIT_NONE: break; case FI_WAIT_SET: if (!attr->wait_set) { FI_INFO(&psmx2_prov, FI_LOG_CQ, "FI_WAIT_SET is specified but attr->wait_set is NULL\n"); return -FI_EINVAL; } wait = attr->wait_set; break; case FI_WAIT_UNSPEC: case FI_WAIT_FD: case FI_WAIT_MUTEX_COND: wait_attr.wait_obj = attr->wait_obj; wait_attr.flags = 0; err = fi_wait_open(&domain_priv->fabric->util_fabric.fabric_fid, &wait_attr, (struct fid_wait **)&wait); if (err) return err; wait_is_local = 1; break; default: FI_INFO(&psmx2_prov, FI_LOG_CQ, "attr->wait_obj=%d, supported=%d...%d\n", attr->wait_obj, FI_WAIT_NONE, FI_WAIT_MUTEX_COND); return -FI_EINVAL; } cntr_priv = (struct psmx2_fid_cntr *) calloc(1, sizeof *cntr_priv); if (!cntr_priv) { err = -FI_ENOMEM; goto fail; } cntr_priv->domain = domain_priv; cntr_priv->events = events; if (wait) cntr_priv->wait = container_of(wait, struct util_wait, wait_fid); cntr_priv->wait_is_local = wait_is_local; cntr_priv->flags = flags; cntr_priv->cntr.fid.fclass = FI_CLASS_CNTR; cntr_priv->cntr.fid.context = context; cntr_priv->cntr.fid.ops = &psmx2_fi_ops; cntr_priv->cntr.ops = &psmx2_cntr_ops; atomic_initialize(&cntr_priv->counter, 0); atomic_initialize(&cntr_priv->error_counter, 0); pthread_mutex_init(&cntr_priv->trigger_lock, NULL); if (wait) fi_poll_add(&cntr_priv->wait->pollset->poll_fid, &cntr_priv->cntr.fid, 0); psmx2_domain_acquire(domain_priv); *cntr = &cntr_priv->cntr; return 0; fail: if (wait && wait_is_local) fi_close(&wait->fid); return err; }
int psmx2_ep_open(struct fid_domain *domain, struct fi_info *info, struct fid_ep **ep, void *context) { struct psmx2_fid_domain *domain_priv; struct psmx2_fid_ep *ep_priv; uint8_t vlane; uint64_t ep_cap; int err = -FI_EINVAL; if (info) ep_cap = info->caps; else ep_cap = FI_TAGGED; domain_priv = container_of(domain, struct psmx2_fid_domain, domain.fid); if (!domain_priv) goto errout; err = psmx2_domain_check_features(domain_priv, ep_cap); if (err) goto errout; err = psmx2_alloc_vlane(domain_priv, &vlane); if (err) goto errout; ep_priv = (struct psmx2_fid_ep *) calloc(1, sizeof *ep_priv); if (!ep_priv) { err = -FI_ENOMEM; goto errout_free_vlane; } ep_priv->ep.fid.fclass = FI_CLASS_EP; ep_priv->ep.fid.context = context; ep_priv->ep.fid.ops = &psmx2_fi_ops; ep_priv->ep.ops = &psmx2_ep_ops; ep_priv->ep.cm = &psmx2_cm_ops; ep_priv->domain = domain_priv; ep_priv->vlane = vlane; PSMX2_CTXT_TYPE(&ep_priv->nocomp_send_context) = PSMX2_NOCOMP_SEND_CONTEXT; PSMX2_CTXT_EP(&ep_priv->nocomp_send_context) = ep_priv; PSMX2_CTXT_TYPE(&ep_priv->nocomp_recv_context) = PSMX2_NOCOMP_RECV_CONTEXT; PSMX2_CTXT_EP(&ep_priv->nocomp_recv_context) = ep_priv; if (ep_cap & FI_TAGGED) ep_priv->ep.tagged = &psmx2_tagged_ops; if (ep_cap & FI_MSG) ep_priv->ep.msg = &psmx2_msg_ops; if (ep_cap & FI_RMA) ep_priv->ep.rma = &psmx2_rma_ops; if (ep_cap & FI_ATOMICS) ep_priv->ep.atomic = &psmx2_atomic_ops; ep_priv->caps = ep_cap; err = psmx2_domain_enable_ep(domain_priv, ep_priv); if (err) goto errout_free_ep; psmx2_domain_acquire(domain_priv); domain_priv->eps[ep_priv->vlane] = ep_priv; if (info) { if (info->tx_attr) ep_priv->flags = info->tx_attr->op_flags; if (info->rx_attr) ep_priv->flags |= info->rx_attr->op_flags; } psmx2_ep_optimize_ops(ep_priv); *ep = &ep_priv->ep; return 0; errout_free_ep: free(ep_priv); errout_free_vlane: psmx2_free_vlane(domain_priv, vlane); errout: return err; }