int psmx_domain_check_features(struct psmx_fid_domain *domain, int ep_cap)
{
if ((domain->caps & ep_cap & ~PSMX_SUB_CAPS) != (ep_cap & ~PSMX_SUB_CAPS)) {
FI_INFO(&psmx_prov, FI_LOG_CORE,
"caps mismatch: domain->caps=%llx, ep->caps=%llx, mask=%llx\n",
domain->caps, ep_cap, ~PSMX_SUB_CAPS);
return -FI_EOPNOTSUPP;
}
if ((ep_cap & FI_TAGGED) && domain->tagged_ep &&
ofi_recv_allowed(ep_cap))
return -FI_EBUSY;
if ((ep_cap & FI_MSG) && domain->msg_ep &&
ofi_recv_allowed(ep_cap))
return -FI_EBUSY;
if ((ep_cap & FI_RMA) && domain->rma_ep &&
ofi_rma_target_allowed(ep_cap))
return -FI_EBUSY;
if ((ep_cap & FI_ATOMICS) && domain->atomics_ep &&
ofi_rma_target_allowed(ep_cap))
return -FI_EBUSY;
return 0;
}
int psmx_domain_enable_ep(struct psmx_fid_domain *domain, struct psmx_fid_ep *ep)
{
uint64_t ep_cap = 0;
if (ep)
ep_cap = ep->caps;
if ((domain->caps & ep_cap & ~PSMX_SUB_CAPS) != (ep_cap & ~PSMX_SUB_CAPS)) {
FI_INFO(&psmx_prov, FI_LOG_CORE,
"caps mismatch: domain->caps=%llx, ep->caps=%llx, mask=%llx\n",
domain->caps, ep_cap, ~PSMX_SUB_CAPS);
return -FI_EOPNOTSUPP;
}
if (ep_cap & FI_MSG)
domain->reserved_tag_bits |= PSMX_MSG_BIT;
if (psmx_env.am_msg)
domain->reserved_tag_bits &= ~PSMX_MSG_BIT;
if ((ep_cap & FI_RMA) && psmx_env.tagged_rma)
domain->reserved_tag_bits |= PSMX_RMA_BIT;
if (((ep_cap & FI_RMA) || (ep_cap & FI_ATOMICS) || psmx_env.am_msg) &&
!domain->am_initialized) {
int err = psmx_am_init(domain);
if (err)
return err;
domain->am_initialized = 1;
}
if ((ep_cap & FI_RMA) && ofi_rma_target_allowed(ep_cap))
domain->rma_ep = ep;
if ((ep_cap & FI_ATOMICS) && ofi_rma_target_allowed(ep_cap))
domain->atomics_ep = ep;
if ((ep_cap & FI_TAGGED) && ofi_recv_allowed(ep_cap))
domain->tagged_ep = ep;
if ((ep_cap & FI_MSG) && ofi_recv_allowed(ep_cap))
domain->msg_ep = ep;
return 0;
}
int psmx2_ep_open(struct fid_domain *domain, struct fi_info *info,
struct fid_ep **ep, void *context)
{
struct psmx2_fid_domain *domain_priv;
struct psmx2_fid_ep *ep_priv;
struct psmx2_ep_name ep_name;
struct psmx2_ep_name *src_addr;
struct psmx2_trx_ctxt *trx_ctxt = NULL;
int err = -FI_EINVAL;
int alloc_trx_ctxt = 1;
domain_priv = container_of(domain, struct psmx2_fid_domain,
util_domain.domain_fid.fid);
if (!domain_priv)
goto errout;
if (info && info->ep_attr && info->ep_attr->rx_ctx_cnt == FI_SHARED_CONTEXT)
return -FI_ENOSYS;
if (info && info->ep_attr && info->ep_attr->tx_ctx_cnt == FI_SHARED_CONTEXT &&
!ofi_recv_allowed(info->caps) && !ofi_rma_target_allowed(info->caps)) {
alloc_trx_ctxt = 0;
FI_INFO(&psmx2_prov, FI_LOG_EP_CTRL, "Tx only endpoint with STX context.\n");
}
src_addr = NULL;
if (info && info->src_addr) {
if (info->addr_format == FI_ADDR_STR)
src_addr = psmx2_string_to_ep_name(info->src_addr);
else
src_addr = info->src_addr;
}
if (alloc_trx_ctxt) {
trx_ctxt = psmx2_trx_ctxt_alloc(domain_priv, src_addr, 0);
if (!trx_ctxt)
goto errout;
}
err = psmx2_ep_open_internal(domain_priv, info, &ep_priv, context,
trx_ctxt);
if (err)
goto errout_free_ctxt;
ep_priv->type = PSMX2_EP_REGULAR;
ep_priv->service = PSMX2_ANY_SERVICE;
if (src_addr) {
ep_priv->service = src_addr->service;
if (info->addr_format == FI_ADDR_STR)
free(src_addr);
}
if (ep_priv->service == PSMX2_ANY_SERVICE)
ep_priv->service = ((getpid() & 0x7FFF) << 16) +
((uintptr_t)ep_priv & 0xFFFF);
if (alloc_trx_ctxt) {
trx_ctxt->ep = ep_priv;
psmx2_lock(&domain_priv->trx_ctxt_lock, 1);
dlist_insert_before(&trx_ctxt->entry,
&domain_priv->trx_ctxt_list);
psmx2_unlock(&domain_priv->trx_ctxt_lock, 1);
ep_name.epid = trx_ctxt->psm2_epid;
ep_name.type = ep_priv->type;
ofi_ns_add_local_name(&domain_priv->fabric->name_server,
&ep_priv->service, &ep_name);
}
*ep = &ep_priv->ep;
return 0;
errout_free_ctxt:
psmx2_trx_ctxt_free(trx_ctxt);
errout:
return err;
}
static int fi_ibv_msg_ep_enable(struct fid_ep *ep)
{
struct ibv_qp_init_attr attr;
struct fi_ibv_msg_ep *_ep;
struct ibv_pd *pd;
_ep = container_of(ep, struct fi_ibv_msg_ep, ep_fid);
if (!_ep->eq) {
VERBS_WARN(FI_LOG_EP_CTRL,
"Endpoint is not bound to an event queue\n");
return -FI_ENOEQ;
}
if (!_ep->scq && !_ep->rcq) {
VERBS_WARN(FI_LOG_EP_CTRL, "Endpoint is not bound to "
"a send or receive completion queue\n");
return -FI_ENOCQ;
}
if (!_ep->scq && (ofi_send_allowed(_ep->info->caps) ||
ofi_rma_initiate_allowed(_ep->info->caps))) {
VERBS_WARN(FI_LOG_EP_CTRL, "Endpoint is not bound to "
"a send completion queue when it has transmit "
"capabilities enabled (FI_SEND | FI_RMA).\n");
return -FI_ENOCQ;
}
if (!_ep->rcq && ofi_recv_allowed(_ep->info->caps)) {
VERBS_WARN(FI_LOG_EP_CTRL, "Endpoint is not bound to "
"a receive completion queue when it has receive "
"capabilities enabled. (FI_RECV)\n");
return -FI_ENOCQ;
}
memset(&attr, 0, sizeof attr);
if (_ep->scq) {
attr.cap.max_send_wr = _ep->info->tx_attr->size;
attr.cap.max_send_sge = _ep->info->tx_attr->iov_limit;
attr.send_cq = _ep->scq->cq;
pd = _ep->scq->domain->pd;
} else {
attr.send_cq = _ep->rcq->cq;
pd = _ep->rcq->domain->pd;
}
if (_ep->rcq) {
attr.cap.max_recv_wr = _ep->info->rx_attr->size;
attr.cap.max_recv_sge = _ep->info->rx_attr->iov_limit;
attr.recv_cq = _ep->rcq->cq;
} else {
attr.recv_cq = _ep->scq->cq;
}
attr.cap.max_inline_data = _ep->info->tx_attr->inject_size;
if (_ep->srq_ep) {
attr.srq =_ep->srq_ep->srq;
/* Use of SRQ, no need to allocate recv_wr entries in the QP */
attr.cap.max_recv_wr = 0;
/* Override the default ops to prevent the user from posting WRs to a
* QP where a SRQ is attached to */
_ep->ep_fid.msg = fi_ibv_msg_srq_ep_ops_msg(_ep);
}
attr.qp_type = IBV_QPT_RC;
attr.sq_sig_all = 0;
attr.qp_context = _ep;
return rdma_create_qp(_ep->id, pd, &attr) ?
-errno : 0;
}
static int fi_bgq_tx_ctx(struct fid_ep *sep, int index,
struct fi_tx_attr *attr, struct fid_ep **tx_ep,
void *context)
{
int ret;
struct fi_info info = {0};
struct fi_tx_attr tx_attr = {0};
struct fi_ep_attr ep_attr = {0};
struct fi_domain_attr dom_attr = {0};
struct fi_fabric_attr fab_attr = {0};
struct fi_bgq_sep *bgq_sep;
struct fi_bgq_ep *bgq_tx_ep;
if (!sep || !attr || !tx_ep) {
errno = FI_EINVAL;
return -errno;
}
bgq_sep = container_of(sep, struct fi_bgq_sep, ep_fid);
uint64_t caps = attr->caps; /* TODO - "By default, a transmit context inherits the properties of its associated endpoint. However, applications may request context specific attributes through the attr parameter." */
if ((caps & FI_MSG || caps & FI_TAGGED) && (caps & FI_RECV)) {
FI_LOG(fi_bgq_global.prov, FI_LOG_DEBUG, FI_LOG_DOMAIN,
"FI_MSG|FI_TAGGED with FI_RECV capability specified for a TX context\n");
caps &= ~FI_RECV;
}
if ((caps & FI_RMA || caps & FI_ATOMIC) && (caps & FI_REMOTE_READ || caps & FI_REMOTE_WRITE)) {
FI_LOG(fi_bgq_global.prov, FI_LOG_DEBUG, FI_LOG_DOMAIN,
"FI_RMA|FI_ATOMIC with FI_REMOTE_READ|FI_REMOTE_WRITE capability specified for a TX context\n");
caps &= ~FI_REMOTE_READ;
caps &= ~FI_REMOTE_WRITE;
}
if (caps & FI_MSG || caps & FI_TAGGED) {
caps |= FI_SEND;
}
if (caps & FI_RMA || caps & FI_ATOMIC) {
caps |= FI_READ;
caps |= FI_WRITE;
}
if (ofi_recv_allowed(caps) || ofi_rma_target_allowed(caps)) {
FI_LOG(fi_bgq_global.prov, FI_LOG_DEBUG, FI_LOG_DOMAIN,
"RX capabilities specified for TX context\n");
errno = FI_EINVAL;
return -errno;
}
if (!ofi_send_allowed(caps) && !ofi_rma_initiate_allowed(caps)) {
FI_LOG(fi_bgq_global.prov, FI_LOG_DEBUG, FI_LOG_DOMAIN,
"TX capabilities not specified for TX context\n");
errno = FI_EINVAL;
return -errno;
}
if (bgq_sep->domain->tx.count >= fi_bgq_domain_get_tx_max(bgq_sep->domain)) {
FI_LOG(fi_bgq_global.prov, FI_LOG_DEBUG, FI_LOG_DOMAIN,
"TX ctx count exceeded (max %lu, created %lu)\n",
fi_bgq_domain_get_tx_max(bgq_sep->domain), bgq_sep->domain->tx.count);
errno = FI_EINVAL;
return -errno;
}
info.caps = caps;
info.mode = attr->mode;
info.tx_attr = &tx_attr;
memcpy(info.tx_attr, attr, sizeof(*info.tx_attr));
info.ep_attr = &ep_attr;
memcpy(info.ep_attr, bgq_sep->info->ep_attr, sizeof(*info.ep_attr));
info.domain_attr = &dom_attr;
memcpy(info.domain_attr, bgq_sep->info->domain_attr, sizeof(*info.domain_attr));
info.fabric_attr = &fab_attr;
memcpy(info.fabric_attr, bgq_sep->info->fabric_attr, sizeof(*info.fabric_attr));
ret = fi_bgq_endpoint_rx_tx((struct fid_domain *)bgq_sep->domain,
&info, tx_ep, context, -1, index);
if (ret) {
goto err;
}
bgq_tx_ep = container_of(*tx_ep, struct fi_bgq_ep, ep_fid);
bgq_tx_ep->ep_fid.fid.fclass = FI_CLASS_TX_CTX;
bgq_tx_ep->av = bgq_sep->av;
fi_bgq_ref_inc(&bgq_tx_ep->av->ref_cnt, "address vector");
bgq_tx_ep->sep = container_of(sep, struct fi_bgq_sep, ep_fid);
++ bgq_sep->domain->tx.count;
fi_bgq_ref_inc(&bgq_sep->ref_cnt, "scalable endpoint");
attr->caps = caps;
return 0;
err:
return -errno;
}
static int fi_bgq_rx_ctx(struct fid_ep *sep, int index,
struct fi_rx_attr *attr, struct fid_ep **rx_ep,
void *context)
{
int ret;
struct fi_info info = {0};
struct fi_bgq_sep *bgq_sep;
struct fi_bgq_ep *bgq_rx_ep;
if (!sep || !attr || !rx_ep) {
errno = FI_EINVAL;
return -errno;
}
bgq_sep = container_of(sep, struct fi_bgq_sep, ep_fid);
uint64_t caps = attr->caps; /* TODO - "By default, a receive context inherits the properties of its associated endpoint. However, applications may request context specific attributes through the attr parameter." */
if ((caps & FI_MSG || caps & FI_TAGGED) && (caps & FI_SEND)) {
FI_LOG(fi_bgq_global.prov, FI_LOG_DEBUG, FI_LOG_DOMAIN,
"FI_MSG|FI_TAGGED with FI_SEND capability specified for a RX context\n");
caps &= ~FI_SEND;
}
if ((caps & FI_RMA || caps & FI_ATOMIC) && (caps & FI_READ || caps & FI_WRITE)) {
FI_LOG(fi_bgq_global.prov, FI_LOG_DEBUG, FI_LOG_DOMAIN,
"FI_RMA|FI_ATOMIC with FI_READ|FI_WRITE capability specified for a RX context\n");
caps &= ~FI_READ;
caps &= ~FI_WRITE;
}
if (caps & FI_MSG || caps & FI_TAGGED) {
caps |= FI_RECV;
}
if (caps & FI_RMA || caps & FI_ATOMIC) {
caps |= FI_REMOTE_READ;
caps |= FI_REMOTE_WRITE;
}
if (ofi_send_allowed(caps) || ofi_rma_initiate_allowed(caps)) {
FI_LOG(fi_bgq_global.prov, FI_LOG_DEBUG, FI_LOG_DOMAIN,
"TX capabilities specified for RX context\n");
errno = FI_EINVAL;
return -errno;
}
if (!ofi_recv_allowed(caps) && !ofi_rma_target_allowed(caps)) {
FI_LOG(fi_bgq_global.prov, FI_LOG_DEBUG, FI_LOG_DOMAIN,
"RX capabilities not specified for RX context\n");
errno = FI_EINVAL;
return -errno;
}
if (bgq_sep->domain->rx.count >= fi_bgq_domain_get_rx_max(bgq_sep->domain)) {
FI_LOG(fi_bgq_global.prov, FI_LOG_DEBUG, FI_LOG_DOMAIN,
"RX ctx count exceeded (max %lu, created %lu)\n",
fi_bgq_domain_get_rx_max(bgq_sep->domain), bgq_sep->domain->rx.count);
errno = FI_EINVAL;
return -errno;
}
info.caps = caps;
info.mode = attr->mode;
info.rx_attr = calloc(1, sizeof(*info.rx_attr));
if (!info.rx_attr) {
errno = FI_ENOMEM;
goto err;
}
info.rx_attr->caps = caps;
info.rx_attr->mode = attr->mode;
info.rx_attr->op_flags = attr->op_flags;
info.rx_attr->msg_order = attr->msg_order;
info.rx_attr->total_buffered_recv = attr->total_buffered_recv;
info.rx_attr->iov_limit = attr->iov_limit;
info.ep_attr = calloc(1, sizeof(*info.ep_attr));
if (!info.ep_attr) {
errno = FI_ENOMEM;
goto err;
}
memcpy(info.ep_attr, bgq_sep->info->ep_attr,
sizeof(*info.ep_attr));
info.domain_attr = calloc(1, sizeof(*info.domain_attr));
if (!info.domain_attr) {
errno = FI_ENOMEM;
goto err;
}
memcpy(info.domain_attr, bgq_sep->info->domain_attr,
sizeof(*info.domain_attr));
info.fabric_attr = calloc(1, sizeof(*info.fabric_attr));
if (!info.fabric_attr) {
errno = FI_ENOMEM;
goto err;
}
memcpy(info.fabric_attr, bgq_sep->info->fabric_attr,
sizeof(*info.fabric_attr));
ret = fi_bgq_endpoint_rx_tx(&bgq_sep->domain->domain_fid, &info,
rx_ep, context, index, -1);
if (ret) {
goto err;
}
bgq_rx_ep = container_of(*rx_ep, struct fi_bgq_ep, ep_fid);
bgq_rx_ep->ep_fid.fid.fclass = FI_CLASS_RX_CTX;
bgq_rx_ep->sep = container_of(sep, struct fi_bgq_sep, ep_fid);
bgq_rx_ep->av = bgq_sep->av;
fi_bgq_ref_inc(&bgq_rx_ep->av->ref_cnt, "address vector");
++ bgq_sep->domain->rx.count;
fi_bgq_ref_inc(&bgq_sep->ref_cnt, "scalable endpoint");
return 0;
err:
if (info.fabric_attr)
free(info.fabric_attr);
if (info.domain_attr)
free(info.domain_attr);
if (info.ep_attr)
free(info.ep_attr);
if (info.tx_attr)
free(info.tx_attr);
return -errno;
}
