int psmx_stx_ctx(struct fid_domain *domain, struct fi_tx_attr *attr,
struct fid_stx **stx, void *context)
{
struct psmx_fid_domain *domain_priv;
struct psmx_fid_stx *stx_priv;
FI_INFO(&psmx_prov, FI_LOG_EP_DATA, "\n");
domain_priv = container_of(domain, struct psmx_fid_domain,
util_domain.domain_fid.fid);
stx_priv = (struct psmx_fid_stx *) calloc(1, sizeof *stx_priv);
if (!stx_priv)
return -FI_ENOMEM;
psmx_domain_acquire(domain_priv);
stx_priv->stx.fid.fclass = FI_CLASS_STX_CTX;
stx_priv->stx.fid.context = context;
stx_priv->stx.fid.ops = &psmx_fi_ops_stx;
stx_priv->domain = domain_priv;
*stx = &stx_priv->stx;
return 0;
}
int psmx_av_open(struct fid_domain *domain, struct fi_av_attr *attr,
struct fid_av **av, void *context)
{
struct psmx_fid_domain *domain_priv;
struct psmx_fid_av *av_priv;
int type = FI_AV_MAP;
size_t count = 64;
uint64_t flags = 0;
domain_priv = container_of(domain, struct psmx_fid_domain, domain);
if (attr) {
switch (attr->type) {
case FI_AV_MAP:
case FI_AV_TABLE:
type = attr->type;
break;
default:
FI_INFO(&psmx_prov, FI_LOG_AV,
"attr->type=%d, supported=%d %d\n",
attr->type, FI_AV_MAP, FI_AV_TABLE);
return -FI_EINVAL;
}
count = attr->count;
flags = attr->flags;
if (flags & (FI_READ | FI_SYMMETRIC)) {
FI_INFO(&psmx_prov, FI_LOG_AV,
"attr->flags=%x, supported=%x\n",
attr->flags, FI_EVENT);
return -FI_EINVAL;
}
}
av_priv = (struct psmx_fid_av *) calloc(1, sizeof *av_priv);
if (!av_priv)
return -FI_ENOMEM;
psmx_domain_acquire(domain_priv);
av_priv->domain = domain_priv;
av_priv->type = type;
av_priv->addrlen = sizeof(psm_epaddr_t);
av_priv->count = count;
av_priv->flags = flags;
av_priv->av.fid.fclass = FI_CLASS_AV;
av_priv->av.fid.context = context;
av_priv->av.fid.ops = &psmx_fi_ops;
av_priv->av.ops = &psmx_av_ops;
*av = &av_priv->av;
return 0;
}
static int psmx_mr_regattr(struct fid *fid, const struct fi_mr_attr *attr,
uint64_t flags, struct fid_mr **mr)
{
struct fid_domain *domain;
struct psmx_fid_domain *domain_priv;
struct psmx_fid_mr *mr_priv;
int i, err;
uint64_t key;
if (fid->fclass != FI_CLASS_DOMAIN) {
return -FI_EINVAL;
}
domain = container_of(fid, struct fid_domain, fid);
domain_priv = container_of(domain, struct psmx_fid_domain, domain);
if (!attr)
return -FI_EINVAL;
if (attr->iov_count == 0 || attr->mr_iov == NULL)
return -FI_EINVAL;
mr_priv = (struct psmx_fid_mr *)
calloc(1, sizeof(*mr_priv) +
sizeof(struct iovec) * attr->iov_count);
if (!mr_priv)
return -FI_ENOMEM;
err = psmx_mr_reserve_key(domain_priv, attr->requested_key, &key, mr_priv);
if (err) {
free(mr_priv);
return err;
}
psmx_domain_acquire(domain_priv);
mr_priv->mr.fid.fclass = FI_CLASS_MR;
mr_priv->mr.fid.context = attr->context;
mr_priv->mr.fid.ops = &psmx_fi_ops;
mr_priv->mr.mem_desc = mr_priv;
mr_priv->mr.key = key;
mr_priv->domain = domain_priv;
mr_priv->access = attr->access;
mr_priv->flags = flags;
mr_priv->iov_count = attr->iov_count;
for (i=0; i<attr->iov_count; i++)
mr_priv->iov[i] = attr->mr_iov[i];
psmx_mr_normalize_iov(mr_priv->iov, &mr_priv->iov_count);
mr_priv->offset = (domain_priv->mr_mode == FI_MR_SCALABLE) ?
((uint64_t)mr_priv->iov[0].iov_base - attr->offset) :
0;
*mr = &mr_priv->mr;
return 0;
}
static int psmx_mr_reg(struct fid *fid, const void *buf, size_t len,
uint64_t access, uint64_t offset, uint64_t requested_key,
uint64_t flags, struct fid_mr **mr, void *context)
{
struct fid_domain *domain;
struct psmx_fid_domain *domain_priv;
struct psmx_fid_mr *mr_priv;
uint64_t key;
int err;
if (fid->fclass != FI_CLASS_DOMAIN) {
return -FI_EINVAL;
}
domain = container_of(fid, struct fid_domain, fid);
domain_priv = container_of(domain, struct psmx_fid_domain, domain);
mr_priv = (struct psmx_fid_mr *) calloc(1, sizeof(*mr_priv) + sizeof(struct iovec));
if (!mr_priv)
return -FI_ENOMEM;
err = psmx_mr_reserve_key(domain_priv, requested_key, &key, mr_priv);
if (err) {
free(mr_priv);
return err;
}
psmx_domain_acquire(domain_priv);
mr_priv->mr.fid.fclass = FI_CLASS_MR;
mr_priv->mr.fid.context = context;
mr_priv->mr.fid.ops = &psmx_fi_ops;
mr_priv->mr.mem_desc = mr_priv;
mr_priv->mr.fid.fclass = FI_CLASS_MR;
mr_priv->mr.fid.context = context;
mr_priv->mr.fid.ops = &psmx_fi_ops;
mr_priv->mr.mem_desc = mr_priv;
mr_priv->mr.key = key;
mr_priv->domain = domain_priv;
mr_priv->access = access;
mr_priv->flags = flags;
mr_priv->iov_count = 1;
mr_priv->iov[0].iov_base = (void *)buf;
mr_priv->iov[0].iov_len = len;
mr_priv->offset = (domain_priv->mr_mode == FI_MR_SCALABLE) ?
((uint64_t)mr_priv->iov[0].iov_base - offset) :
0;
*mr = &mr_priv->mr;
return 0;
}
int psmx_cntr_open(struct fid_domain *domain, struct fi_cntr_attr *attr,
struct fid_cntr **cntr, void *context)
{
struct psmx_fid_domain *domain_priv;
struct psmx_fid_cntr *cntr_priv;
struct fid_wait *wait = NULL;
struct fi_wait_attr wait_attr;
int wait_is_local = 0;
int events;
uint64_t flags;
int err;
events = FI_CNTR_EVENTS_COMP;
flags = 0;
domain_priv = container_of(domain, struct psmx_fid_domain,
util_domain.domain_fid);
switch (attr->events) {
case FI_CNTR_EVENTS_COMP:
events = attr->events;
break;
default:
FI_INFO(&psmx_prov, FI_LOG_CQ,
"attr->events=%d, supported=%d\n",
attr->events, FI_CNTR_EVENTS_COMP);
return -FI_EINVAL;
}
switch (attr->wait_obj) {
case FI_WAIT_NONE:
case FI_WAIT_UNSPEC:
break;
case FI_WAIT_SET:
if (!attr->wait_set) {
FI_INFO(&psmx_prov, FI_LOG_CQ,
"FI_WAIT_SET is specified but attr->wait_set is NULL\n");
return -FI_EINVAL;
}
wait = attr->wait_set;
break;
case FI_WAIT_FD:
case FI_WAIT_MUTEX_COND:
wait_attr.wait_obj = attr->wait_obj;
wait_attr.flags = 0;
err = fi_wait_open(&domain_priv->fabric->util_fabric.fabric_fid,
&wait_attr, (struct fid_wait **)&wait);
if (err)
return err;
wait_is_local = 1;
break;
default:
FI_INFO(&psmx_prov, FI_LOG_CQ,
"attr->wait_obj=%d, supported=%d...%d\n",
attr->wait_obj, FI_WAIT_NONE, FI_WAIT_MUTEX_COND);
return -FI_EINVAL;
}
cntr_priv = (struct psmx_fid_cntr *) calloc(1, sizeof *cntr_priv);
if (!cntr_priv) {
err = -FI_ENOMEM;
goto fail;
}
cntr_priv->domain = domain_priv;
cntr_priv->events = events;
if (wait)
cntr_priv->wait = container_of(wait, struct util_wait, wait_fid);
cntr_priv->wait_is_local = wait_is_local;
cntr_priv->flags = flags;
cntr_priv->cntr.fid.fclass = FI_CLASS_CNTR;
cntr_priv->cntr.fid.context = context;
cntr_priv->cntr.fid.ops = &psmx_fi_ops;
cntr_priv->cntr.ops = &psmx_cntr_ops;
ofi_atomic_initialize64(&cntr_priv->counter, 0);
ofi_atomic_initialize64(&cntr_priv->error_counter, 0);
pthread_mutex_init(&cntr_priv->trigger_lock, NULL);
if (wait)
fi_poll_add(&cntr_priv->wait->pollset->poll_fid,
&cntr_priv->cntr.fid, 0);
psmx_domain_acquire(domain_priv);
*cntr = &cntr_priv->cntr;
return 0;
fail:
if (wait && wait_is_local)
fi_close(&wait->fid);
return err;
}
int psmx_domain_open(struct fid_fabric *fabric, struct fi_info *info,
struct fid_domain **domain, void *context)
{
struct psmx_fid_fabric *fabric_priv;
struct psmx_fid_domain *domain_priv;
int err;
FI_INFO(&psmx_prov, FI_LOG_DOMAIN, "\n");
fabric_priv = container_of(fabric, struct psmx_fid_fabric,
util_fabric.fabric_fid);
if (fabric_priv->active_domain) {
psmx_domain_acquire(fabric_priv->active_domain);
*domain = &fabric_priv->active_domain->util_domain.domain_fid;
return 0;
}
if (!info->domain_attr->name ||
strcmp(info->domain_attr->name, PSMX_DOMAIN_NAME)) {
err = -FI_EINVAL;
goto err_out;
}
domain_priv = (struct psmx_fid_domain *) calloc(1, sizeof *domain_priv);
if (!domain_priv) {
err = -FI_ENOMEM;
goto err_out;
}
err = ofi_domain_init(fabric, info, &domain_priv->util_domain, context);
if (err)
goto err_out_free_domain;
/* fclass & context are set in ofi_domain_init */
domain_priv->util_domain.domain_fid.fid.ops = &psmx_fi_ops;
domain_priv->util_domain.domain_fid.ops = &psmx_domain_ops;
domain_priv->util_domain.domain_fid.mr = &psmx_mr_ops;
domain_priv->mr_mode = info->domain_attr->mr_mode;
domain_priv->mode = info->mode;
domain_priv->caps = info->caps;
domain_priv->fabric = fabric_priv;
domain_priv->progress_thread_enabled =
(info->domain_attr->data_progress == FI_PROGRESS_AUTO && psmx_env.prog_thread);
err = psmx_domain_init(domain_priv, info->src_addr);
if (err)
goto err_out_close_domain;
/* tale the reference to count for multiple domain open calls */
psmx_domain_acquire(fabric_priv->active_domain);
*domain = &domain_priv->util_domain.domain_fid;
return 0;
err_out_close_domain:
ofi_domain_close(&domain_priv->util_domain);
err_out_free_domain:
free(domain_priv);
err_out:
return err;
}
int psmx_ep_open(struct fid_domain *domain, struct fi_info *info,
struct fid_ep **ep, void *context)
{
struct psmx_fid_domain *domain_priv;
struct psmx_fid_ep *ep_priv;
int err;
uint64_t ep_cap;
if (info)
ep_cap = info->caps;
else
ep_cap = FI_TAGGED;
domain_priv = container_of(domain, struct psmx_fid_domain,
util_domain.domain_fid.fid);
if (!domain_priv)
return -FI_EINVAL;
if (info && info->ep_attr && info->ep_attr->auth_key) {
if (info->ep_attr->auth_keylen != sizeof(psm_uuid_t)) {
FI_WARN(&psmx_prov, FI_LOG_EP_CTRL,
"Invalid auth_key_len %d, should be %d.\n",
info->ep_attr->auth_keylen,
sizeof(psm_uuid_t));
return -FI_EINVAL;
}
if (memcmp(domain_priv->fabric->uuid, info->ep_attr->auth_key,
sizeof(psm_uuid_t))) {
FI_WARN(&psmx_prov, FI_LOG_EP_CTRL,
"Invalid auth_key: %s\n",
psmx_uuid_to_string((void *)info->ep_attr->auth_key));
return -FI_EINVAL;
}
}
err = psmx_domain_check_features(domain_priv, ep_cap);
if (err)
return err;
ep_priv = (struct psmx_fid_ep *) calloc(1, sizeof *ep_priv);
if (!ep_priv)
return -FI_ENOMEM;
ep_priv->ep.fid.fclass = FI_CLASS_EP;
ep_priv->ep.fid.context = context;
ep_priv->ep.fid.ops = &psmx_fi_ops;
ep_priv->ep.ops = &psmx_ep_ops;
ep_priv->ep.cm = &psmx_cm_ops;
ep_priv->domain = domain_priv;
atomic_initialize(&ep_priv->ref, 0);
PSMX_CTXT_TYPE(&ep_priv->nocomp_send_context) = PSMX_NOCOMP_SEND_CONTEXT;
PSMX_CTXT_EP(&ep_priv->nocomp_send_context) = ep_priv;
PSMX_CTXT_TYPE(&ep_priv->nocomp_recv_context) = PSMX_NOCOMP_RECV_CONTEXT;
PSMX_CTXT_EP(&ep_priv->nocomp_recv_context) = ep_priv;
if (ep_cap & FI_TAGGED)
ep_priv->ep.tagged = &psmx_tagged_ops;
if (ep_cap & FI_MSG)
ep_priv->ep.msg = &psmx_msg_ops;
if ((ep_cap & FI_MSG) && psmx_env.am_msg)
ep_priv->ep.msg = &psmx_msg2_ops;
if (ep_cap & FI_RMA)
ep_priv->ep.rma = &psmx_rma_ops;
if (ep_cap & FI_ATOMICS)
ep_priv->ep.atomic = &psmx_atomic_ops;
ep_priv->caps = ep_cap;
err = psmx_domain_enable_ep(domain_priv, ep_priv);
if (err) {
free(ep_priv);
return err;
}
psmx_domain_acquire(domain_priv);
if (info) {
if (info->tx_attr)
ep_priv->tx_flags = info->tx_attr->op_flags;
if (info->rx_attr)
ep_priv->rx_flags = info->rx_attr->op_flags;
}
psmx_ep_optimize_ops(ep_priv);
ep_priv->service = PSMX_ANY_SERVICE;
if (info && info->src_addr)
ep_priv->service = ((struct psmx_src_name *)info->src_addr)->service;
if (ep_priv->service == PSMX_ANY_SERVICE)
ep_priv->service = ((getpid() & 0x7FFF) << 16) +
((uintptr_t)ep_priv & 0xFFFF);
psmx_ns_add_local_name(ep_priv->service, domain_priv->psm_epid);
*ep = &ep_priv->ep;
return 0;
}
int psmx_ep_open(struct fid_domain *domain, struct fi_info *info,
struct fid_ep **ep, void *context)
{
struct psmx_fid_domain *domain_priv;
struct psmx_fid_ep *ep_priv;
int err;
uint64_t ep_cap;
if (info)
ep_cap = info->caps;
else
ep_cap = FI_TAGGED;
domain_priv = container_of(domain, struct psmx_fid_domain,
util_domain.domain_fid.fid);
if (!domain_priv)
return -FI_EINVAL;
err = psmx_domain_check_features(domain_priv, ep_cap);
if (err)
return err;
ep_priv = (struct psmx_fid_ep *) calloc(1, sizeof *ep_priv);
if (!ep_priv)
return -FI_ENOMEM;
ep_priv->ep.fid.fclass = FI_CLASS_EP;
ep_priv->ep.fid.context = context;
ep_priv->ep.fid.ops = &psmx_fi_ops;
ep_priv->ep.ops = &psmx_ep_ops;
ep_priv->ep.cm = &psmx_cm_ops;
ep_priv->domain = domain_priv;
atomic_initialize(&ep_priv->ref, 0);
PSMX_CTXT_TYPE(&ep_priv->nocomp_send_context) = PSMX_NOCOMP_SEND_CONTEXT;
PSMX_CTXT_EP(&ep_priv->nocomp_send_context) = ep_priv;
PSMX_CTXT_TYPE(&ep_priv->nocomp_recv_context) = PSMX_NOCOMP_RECV_CONTEXT;
PSMX_CTXT_EP(&ep_priv->nocomp_recv_context) = ep_priv;
if (ep_cap & FI_TAGGED)
ep_priv->ep.tagged = &psmx_tagged_ops;
if (ep_cap & FI_MSG)
ep_priv->ep.msg = &psmx_msg_ops;
if ((ep_cap & FI_MSG) && psmx_env.am_msg)
ep_priv->ep.msg = &psmx_msg2_ops;
if (ep_cap & FI_RMA)
ep_priv->ep.rma = &psmx_rma_ops;
if (ep_cap & FI_ATOMICS)
ep_priv->ep.atomic = &psmx_atomic_ops;
ep_priv->caps = ep_cap;
err = psmx_domain_enable_ep(domain_priv, ep_priv);
if (err) {
free(ep_priv);
return err;
}
psmx_domain_acquire(domain_priv);
if (info) {
if (info->tx_attr)
ep_priv->tx_flags = info->tx_attr->op_flags;
if (info->rx_attr)
ep_priv->rx_flags = info->rx_attr->op_flags;
}
psmx_ep_optimize_ops(ep_priv);
*ep = &ep_priv->ep;
return 0;
}
