int psmx_domain_check_features(struct psmx_fid_domain *domain, int ep_cap)
{
if ((domain->caps & ep_cap & ~PSMX_SUB_CAPS) != (ep_cap & ~PSMX_SUB_CAPS)) {
FI_INFO(&psmx_prov, FI_LOG_CORE,
"caps mismatch: domain->caps=%llx, ep->caps=%llx, mask=%llx\n",
domain->caps, ep_cap, ~PSMX_SUB_CAPS);
return -FI_EOPNOTSUPP;
}
if ((ep_cap & FI_TAGGED) && domain->tagged_ep &&
ofi_recv_allowed(ep_cap))
return -FI_EBUSY;
if ((ep_cap & FI_MSG) && domain->msg_ep &&
ofi_recv_allowed(ep_cap))
return -FI_EBUSY;
if ((ep_cap & FI_RMA) && domain->rma_ep &&
ofi_rma_target_allowed(ep_cap))
return -FI_EBUSY;
if ((ep_cap & FI_ATOMICS) && domain->atomics_ep &&
ofi_rma_target_allowed(ep_cap))
return -FI_EBUSY;
return 0;
}
static uint64_t ofi_get_info_caps(const struct fi_info *prov_info,
const struct fi_info *user_info,
uint32_t api_version)
{
int prov_mode, user_mode;
uint64_t caps;
assert(user_info);
caps = ofi_get_caps(prov_info->caps, user_info->caps, prov_info->caps);
prov_mode = prov_info->domain_attr->mr_mode;
if (!ofi_rma_target_allowed(caps) ||
!(prov_mode & OFI_MR_MODE_RMA_TARGET))
return caps;
if (!user_info->domain_attr)
goto trim_caps;
user_mode = user_info->domain_attr->mr_mode;
if ((FI_VERSION_LT(api_version, FI_VERSION(1,5)) &&
(user_mode == FI_MR_UNSPEC)) ||
(user_mode == FI_MR_BASIC) ||
((user_mode & prov_mode & OFI_MR_MODE_RMA_TARGET) ==
(prov_mode & OFI_MR_MODE_RMA_TARGET)))
return caps;
trim_caps:
return caps & ~(FI_REMOTE_WRITE | FI_REMOTE_READ);
}
int psmx_domain_enable_ep(struct psmx_fid_domain *domain, struct psmx_fid_ep *ep)
{
uint64_t ep_cap = 0;
if (ep)
ep_cap = ep->caps;
if ((domain->caps & ep_cap & ~PSMX_SUB_CAPS) != (ep_cap & ~PSMX_SUB_CAPS)) {
FI_INFO(&psmx_prov, FI_LOG_CORE,
"caps mismatch: domain->caps=%llx, ep->caps=%llx, mask=%llx\n",
domain->caps, ep_cap, ~PSMX_SUB_CAPS);
return -FI_EOPNOTSUPP;
}
if (ep_cap & FI_MSG)
domain->reserved_tag_bits |= PSMX_MSG_BIT;
if (psmx_env.am_msg)
domain->reserved_tag_bits &= ~PSMX_MSG_BIT;
if ((ep_cap & FI_RMA) && psmx_env.tagged_rma)
domain->reserved_tag_bits |= PSMX_RMA_BIT;
if (((ep_cap & FI_RMA) || (ep_cap & FI_ATOMICS) || psmx_env.am_msg) &&
!domain->am_initialized) {
int err = psmx_am_init(domain);
if (err)
return err;
domain->am_initialized = 1;
}
if ((ep_cap & FI_RMA) && ofi_rma_target_allowed(ep_cap))
domain->rma_ep = ep;
if ((ep_cap & FI_ATOMICS) && ofi_rma_target_allowed(ep_cap))
domain->atomics_ep = ep;
if ((ep_cap & FI_TAGGED) && ofi_recv_allowed(ep_cap))
domain->tagged_ep = ep;
if ((ep_cap & FI_MSG) && ofi_recv_allowed(ep_cap))
domain->msg_ep = ep;
return 0;
}
/*
* Remove unneeded MR mode bits based on the requested capability bits.
*/
static int ofi_cap_mr_mode(uint64_t info_caps, int mr_mode)
{
if (!ofi_rma_target_allowed(info_caps)) {
if (!(mr_mode & FI_MR_LOCAL))
return 0;
mr_mode &= ~OFI_MR_MODE_RMA_TARGET;
}
return mr_mode & ~(FI_MR_BASIC | FI_MR_SCALABLE);
}
/*
* - Support FI_MR_LOCAL/FI_LOCAL_MR as ofi_rxm can handle it.
* - The RxM FI_RMA implementation is pass-through but the provider can handle
* FI_MR_PROV_KEY and FI_MR_VIRT_ADDR in its large message transfer rendezvous
* protocol.
* - fi_alter_domain_attr should correctly set the mr_mode in return fi_info
* based on hints.
*/
void rxm_info_to_core_mr_modes(uint32_t version, const struct fi_info *hints,
struct fi_info *core_info)
{
/* We handle FI_MR_BASIC and FI_MR_SCALABLE irrespective of version */
if (hints && hints->domain_attr &&
(hints->domain_attr->mr_mode & (FI_MR_SCALABLE | FI_MR_BASIC))) {
core_info->mode = FI_LOCAL_MR;
core_info->domain_attr->mr_mode = hints->domain_attr->mr_mode;
} else if (FI_VERSION_LT(version, FI_VERSION(1, 5))) {
core_info->mode |= FI_LOCAL_MR;
/* Specify FI_MR_UNSPEC (instead of FI_MR_BASIC) so that
* providers that support only FI_MR_SCALABLE aren't dropped */
core_info->domain_attr->mr_mode = FI_MR_UNSPEC;
} else {
core_info->domain_attr->mr_mode |= FI_MR_LOCAL;
if (!hints || !ofi_rma_target_allowed(hints->caps))
core_info->domain_attr->mr_mode |= OFI_MR_BASIC_MAP;
else if (hints->domain_attr)
core_info->domain_attr->mr_mode |=
hints->domain_attr->mr_mode & OFI_MR_BASIC_MAP;
}
}
int psmx2_ep_open(struct fid_domain *domain, struct fi_info *info,
struct fid_ep **ep, void *context)
{
struct psmx2_fid_domain *domain_priv;
struct psmx2_fid_ep *ep_priv;
struct psmx2_ep_name ep_name;
struct psmx2_ep_name *src_addr;
struct psmx2_trx_ctxt *trx_ctxt = NULL;
int err = -FI_EINVAL;
int alloc_trx_ctxt = 1;
domain_priv = container_of(domain, struct psmx2_fid_domain,
util_domain.domain_fid.fid);
if (!domain_priv)
goto errout;
if (info && info->ep_attr && info->ep_attr->rx_ctx_cnt == FI_SHARED_CONTEXT)
return -FI_ENOSYS;
if (info && info->ep_attr && info->ep_attr->tx_ctx_cnt == FI_SHARED_CONTEXT &&
!ofi_recv_allowed(info->caps) && !ofi_rma_target_allowed(info->caps)) {
alloc_trx_ctxt = 0;
FI_INFO(&psmx2_prov, FI_LOG_EP_CTRL, "Tx only endpoint with STX context.\n");
}
src_addr = NULL;
if (info && info->src_addr) {
if (info->addr_format == FI_ADDR_STR)
src_addr = psmx2_string_to_ep_name(info->src_addr);
else
src_addr = info->src_addr;
}
if (alloc_trx_ctxt) {
trx_ctxt = psmx2_trx_ctxt_alloc(domain_priv, src_addr, 0);
if (!trx_ctxt)
goto errout;
}
err = psmx2_ep_open_internal(domain_priv, info, &ep_priv, context,
trx_ctxt);
if (err)
goto errout_free_ctxt;
ep_priv->type = PSMX2_EP_REGULAR;
ep_priv->service = PSMX2_ANY_SERVICE;
if (src_addr) {
ep_priv->service = src_addr->service;
if (info->addr_format == FI_ADDR_STR)
free(src_addr);
}
if (ep_priv->service == PSMX2_ANY_SERVICE)
ep_priv->service = ((getpid() & 0x7FFF) << 16) +
((uintptr_t)ep_priv & 0xFFFF);
if (alloc_trx_ctxt) {
trx_ctxt->ep = ep_priv;
psmx2_lock(&domain_priv->trx_ctxt_lock, 1);
dlist_insert_before(&trx_ctxt->entry,
&domain_priv->trx_ctxt_list);
psmx2_unlock(&domain_priv->trx_ctxt_lock, 1);
ep_name.epid = trx_ctxt->psm2_epid;
ep_name.type = ep_priv->type;
ofi_ns_add_local_name(&domain_priv->fabric->name_server,
&ep_priv->service, &ep_name);
}
*ep = &ep_priv->ep;
return 0;
errout_free_ctxt:
psmx2_trx_ctxt_free(trx_ctxt);
errout:
return err;
}
static int fi_bgq_tx_ctx(struct fid_ep *sep, int index,
struct fi_tx_attr *attr, struct fid_ep **tx_ep,
void *context)
{
int ret;
struct fi_info info = {0};
struct fi_tx_attr tx_attr = {0};
struct fi_ep_attr ep_attr = {0};
struct fi_domain_attr dom_attr = {0};
struct fi_fabric_attr fab_attr = {0};
struct fi_bgq_sep *bgq_sep;
struct fi_bgq_ep *bgq_tx_ep;
if (!sep || !attr || !tx_ep) {
errno = FI_EINVAL;
return -errno;
}
bgq_sep = container_of(sep, struct fi_bgq_sep, ep_fid);
uint64_t caps = attr->caps; /* TODO - "By default, a transmit context inherits the properties of its associated endpoint. However, applications may request context specific attributes through the attr parameter." */
if ((caps & FI_MSG || caps & FI_TAGGED) && (caps & FI_RECV)) {
FI_LOG(fi_bgq_global.prov, FI_LOG_DEBUG, FI_LOG_DOMAIN,
"FI_MSG|FI_TAGGED with FI_RECV capability specified for a TX context\n");
caps &= ~FI_RECV;
}
if ((caps & FI_RMA || caps & FI_ATOMIC) && (caps & FI_REMOTE_READ || caps & FI_REMOTE_WRITE)) {
FI_LOG(fi_bgq_global.prov, FI_LOG_DEBUG, FI_LOG_DOMAIN,
"FI_RMA|FI_ATOMIC with FI_REMOTE_READ|FI_REMOTE_WRITE capability specified for a TX context\n");
caps &= ~FI_REMOTE_READ;
caps &= ~FI_REMOTE_WRITE;
}
if (caps & FI_MSG || caps & FI_TAGGED) {
caps |= FI_SEND;
}
if (caps & FI_RMA || caps & FI_ATOMIC) {
caps |= FI_READ;
caps |= FI_WRITE;
}
if (ofi_recv_allowed(caps) || ofi_rma_target_allowed(caps)) {
FI_LOG(fi_bgq_global.prov, FI_LOG_DEBUG, FI_LOG_DOMAIN,
"RX capabilities specified for TX context\n");
errno = FI_EINVAL;
return -errno;
}
if (!ofi_send_allowed(caps) && !ofi_rma_initiate_allowed(caps)) {
FI_LOG(fi_bgq_global.prov, FI_LOG_DEBUG, FI_LOG_DOMAIN,
"TX capabilities not specified for TX context\n");
errno = FI_EINVAL;
return -errno;
}
if (bgq_sep->domain->tx.count >= fi_bgq_domain_get_tx_max(bgq_sep->domain)) {
FI_LOG(fi_bgq_global.prov, FI_LOG_DEBUG, FI_LOG_DOMAIN,
"TX ctx count exceeded (max %lu, created %lu)\n",
fi_bgq_domain_get_tx_max(bgq_sep->domain), bgq_sep->domain->tx.count);
errno = FI_EINVAL;
return -errno;
}
info.caps = caps;
info.mode = attr->mode;
info.tx_attr = &tx_attr;
memcpy(info.tx_attr, attr, sizeof(*info.tx_attr));
info.ep_attr = &ep_attr;
memcpy(info.ep_attr, bgq_sep->info->ep_attr, sizeof(*info.ep_attr));
info.domain_attr = &dom_attr;
memcpy(info.domain_attr, bgq_sep->info->domain_attr, sizeof(*info.domain_attr));
info.fabric_attr = &fab_attr;
memcpy(info.fabric_attr, bgq_sep->info->fabric_attr, sizeof(*info.fabric_attr));
ret = fi_bgq_endpoint_rx_tx((struct fid_domain *)bgq_sep->domain,
&info, tx_ep, context, -1, index);
if (ret) {
goto err;
}
bgq_tx_ep = container_of(*tx_ep, struct fi_bgq_ep, ep_fid);
bgq_tx_ep->ep_fid.fid.fclass = FI_CLASS_TX_CTX;
bgq_tx_ep->av = bgq_sep->av;
fi_bgq_ref_inc(&bgq_tx_ep->av->ref_cnt, "address vector");
bgq_tx_ep->sep = container_of(sep, struct fi_bgq_sep, ep_fid);
++ bgq_sep->domain->tx.count;
fi_bgq_ref_inc(&bgq_sep->ref_cnt, "scalable endpoint");
attr->caps = caps;
return 0;
err:
return -errno;
}
static int fi_bgq_rx_ctx(struct fid_ep *sep, int index,
struct fi_rx_attr *attr, struct fid_ep **rx_ep,
void *context)
{
int ret;
struct fi_info info = {0};
struct fi_bgq_sep *bgq_sep;
struct fi_bgq_ep *bgq_rx_ep;
if (!sep || !attr || !rx_ep) {
errno = FI_EINVAL;
return -errno;
}
bgq_sep = container_of(sep, struct fi_bgq_sep, ep_fid);
uint64_t caps = attr->caps; /* TODO - "By default, a receive context inherits the properties of its associated endpoint. However, applications may request context specific attributes through the attr parameter." */
if ((caps & FI_MSG || caps & FI_TAGGED) && (caps & FI_SEND)) {
FI_LOG(fi_bgq_global.prov, FI_LOG_DEBUG, FI_LOG_DOMAIN,
"FI_MSG|FI_TAGGED with FI_SEND capability specified for a RX context\n");
caps &= ~FI_SEND;
}
if ((caps & FI_RMA || caps & FI_ATOMIC) && (caps & FI_READ || caps & FI_WRITE)) {
FI_LOG(fi_bgq_global.prov, FI_LOG_DEBUG, FI_LOG_DOMAIN,
"FI_RMA|FI_ATOMIC with FI_READ|FI_WRITE capability specified for a RX context\n");
caps &= ~FI_READ;
caps &= ~FI_WRITE;
}
if (caps & FI_MSG || caps & FI_TAGGED) {
caps |= FI_RECV;
}
if (caps & FI_RMA || caps & FI_ATOMIC) {
caps |= FI_REMOTE_READ;
caps |= FI_REMOTE_WRITE;
}
if (ofi_send_allowed(caps) || ofi_rma_initiate_allowed(caps)) {
FI_LOG(fi_bgq_global.prov, FI_LOG_DEBUG, FI_LOG_DOMAIN,
"TX capabilities specified for RX context\n");
errno = FI_EINVAL;
return -errno;
}
if (!ofi_recv_allowed(caps) && !ofi_rma_target_allowed(caps)) {
FI_LOG(fi_bgq_global.prov, FI_LOG_DEBUG, FI_LOG_DOMAIN,
"RX capabilities not specified for RX context\n");
errno = FI_EINVAL;
return -errno;
}
if (bgq_sep->domain->rx.count >= fi_bgq_domain_get_rx_max(bgq_sep->domain)) {
FI_LOG(fi_bgq_global.prov, FI_LOG_DEBUG, FI_LOG_DOMAIN,
"RX ctx count exceeded (max %lu, created %lu)\n",
fi_bgq_domain_get_rx_max(bgq_sep->domain), bgq_sep->domain->rx.count);
errno = FI_EINVAL;
return -errno;
}
info.caps = caps;
info.mode = attr->mode;
info.rx_attr = calloc(1, sizeof(*info.rx_attr));
if (!info.rx_attr) {
errno = FI_ENOMEM;
goto err;
}
info.rx_attr->caps = caps;
info.rx_attr->mode = attr->mode;
info.rx_attr->op_flags = attr->op_flags;
info.rx_attr->msg_order = attr->msg_order;
info.rx_attr->total_buffered_recv = attr->total_buffered_recv;
info.rx_attr->iov_limit = attr->iov_limit;
info.ep_attr = calloc(1, sizeof(*info.ep_attr));
if (!info.ep_attr) {
errno = FI_ENOMEM;
goto err;
}
memcpy(info.ep_attr, bgq_sep->info->ep_attr,
sizeof(*info.ep_attr));
info.domain_attr = calloc(1, sizeof(*info.domain_attr));
if (!info.domain_attr) {
errno = FI_ENOMEM;
goto err;
}
memcpy(info.domain_attr, bgq_sep->info->domain_attr,
sizeof(*info.domain_attr));
info.fabric_attr = calloc(1, sizeof(*info.fabric_attr));
if (!info.fabric_attr) {
errno = FI_ENOMEM;
goto err;
}
memcpy(info.fabric_attr, bgq_sep->info->fabric_attr,
sizeof(*info.fabric_attr));
ret = fi_bgq_endpoint_rx_tx(&bgq_sep->domain->domain_fid, &info,
rx_ep, context, index, -1);
if (ret) {
goto err;
}
bgq_rx_ep = container_of(*rx_ep, struct fi_bgq_ep, ep_fid);
bgq_rx_ep->ep_fid.fid.fclass = FI_CLASS_RX_CTX;
bgq_rx_ep->sep = container_of(sep, struct fi_bgq_sep, ep_fid);
bgq_rx_ep->av = bgq_sep->av;
fi_bgq_ref_inc(&bgq_rx_ep->av->ref_cnt, "address vector");
++ bgq_sep->domain->rx.count;
fi_bgq_ref_inc(&bgq_sep->ref_cnt, "scalable endpoint");
return 0;
err:
if (info.fabric_attr)
free(info.fabric_attr);
if (info.domain_attr)
free(info.domain_attr);
if (info.ep_attr)
free(info.ep_attr);
if (info.tx_attr)
free(info.tx_attr);
return -errno;
}
