/*
* helper function for initializing an ep of type
* GNIX_EPN_TYPE_BOUND
*/
static int __gnix_ep_bound_prep(struct gnix_fid_domain *domain,
struct fi_info *info,
struct gnix_fid_ep *ep)
{
int ret = FI_SUCCESS;
GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
assert(domain != NULL);
assert(info != NULL);
assert(ep != NULL);
ret = _gnix_cm_nic_alloc(domain,
info,
&ep->cm_nic);
if (ret != FI_SUCCESS) {
GNIX_WARN(FI_LOG_EP_CTRL,
"_gnix_cm_nic_alloc returned %s\n",
fi_strerror(-ret));
return ret;
}
ep->my_name = ep->cm_nic->my_name;
ep->nic = ep->cm_nic->nic;
_gnix_ref_get(ep->nic);
return ret;
}
Test(utils, references)
{
int refs;
struct gnix_reference_tester test;
/* initialize test structure */
_gnix_ref_init(&test.ref_cnt, 1, test_destruct);
test.destructed = 0;
/* check for validity */
cr_assert(atomic_get(&test.ref_cnt.references) == 1);
cr_assert(test.destructed == 0);
/* increment refs and check */
refs = _gnix_ref_get(&test);
cr_assert(refs == 2);
cr_assert(atomic_get(&test.ref_cnt.references) == 2);
cr_assert(test.destructed == 0);
/* decrement refs and check */
refs = _gnix_ref_put(&test);
cr_assert(refs == 1);
cr_assert(atomic_get(&test.ref_cnt.references) == 1);
cr_assert(test.destructed == 0);
/* decrement and destruct, check for validity */
refs = _gnix_ref_put(&test);
cr_assert(refs == 0);
cr_assert(atomic_get(&test.ref_cnt.references) == 0);
cr_assert(test.destructed == 1);
}
static int gnix_sep_tx_ctx(struct fid_ep *sep, int index,
struct fi_tx_attr *attr,
struct fid_ep **tx_ep, void *context)
{
int ret = FI_SUCCESS;
struct gnix_fid_sep *sep_priv;
struct gnix_fid_ep *ep_priv = NULL;
struct gnix_fid_trx *tx_priv = NULL;
struct fid_ep *ep_ptr;
struct gnix_ep_attr ep_attr = {0};
GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
sep_priv = container_of(sep, struct gnix_fid_sep, ep_fid);
if (!sep_priv) {
GNIX_WARN(FI_LOG_EP_CTRL, "endpoint is not initialized\n");
return -FI_EINVAL;
}
if ((sep_priv->ep_fid.fid.fclass != FI_CLASS_SEP) ||
(index >= sep_priv->info->ep_attr->tx_ctx_cnt))
return -FI_EINVAL;
/*
* check to see if the tx context was already
* allocated
*/
fastlock_acquire(&sep_priv->sep_lock);
if (sep_priv->tx_ep_table[index] != NULL) {
ret = -FI_EBUSY;
goto err;
}
tx_priv = calloc(1, sizeof(struct gnix_fid_trx));
if (!tx_priv) {
ret = -FI_ENOMEM;
goto err;
}
tx_priv->ep_fid.fid.fclass = FI_CLASS_TX_CTX;
tx_priv->ep_fid.fid.context = context;
tx_priv->ep_fid.fid.ops = &gnix_sep_fi_ops;
tx_priv->ep_fid.ops = &gnix_sep_ops;
tx_priv->ep_fid.msg = &gnix_sep_msg_ops;
tx_priv->ep_fid.rma = &gnix_sep_rma_ops;
tx_priv->ep_fid.tagged = &gnix_sep_tagged_ops;
tx_priv->ep_fid.atomic = &gnix_sep_atomic_ops;
tx_priv->ep_fid.cm = &gnix_sep_rxtx_cm_ops;
/* if an EP already allocated for this index, use it */
if (sep_priv->ep_table[index] != NULL) {
ep_priv = container_of(sep_priv->ep_table[index],
struct gnix_fid_ep, ep_fid);
sep_priv->tx_ep_table[index] = sep_priv->ep_table[index];
_gnix_ref_get(ep_priv);
} else {
/**
* Creates a shared transmit context.
*
* @param[in] val value to be sign extended
* @param[in] len length to sign extend the value
* @return FI_SUCCESS if shared tx context successfully created
* @return -FI_EINVAL if invalid arg(s) supplied
* @return -FI_ENOMEM insufficient memory
*/
DIRECT_FN STATIC int gnix_stx_open(struct fid_domain *dom,
struct fi_tx_attr *tx_attr,
struct fid_stx **stx, void *context)
{
int ret = FI_SUCCESS;
struct gnix_fid_domain *domain;
struct gnix_nic *nic;
struct gnix_fid_stx *stx_priv;
struct gnix_nic_attr nic_attr = {0};
GNIX_TRACE(FI_LOG_DOMAIN, "\n");
domain = container_of(dom, struct gnix_fid_domain, domain_fid.fid);
if (domain->domain_fid.fid.fclass != FI_CLASS_DOMAIN) {
ret = -FI_EINVAL;
goto err;
}
stx_priv = calloc(1, sizeof(*stx_priv));
if (!stx_priv) {
ret = -FI_ENOMEM;
goto err;
}
stx_priv->domain = domain;
/*
* we force allocation of a nic to make semantics
* match the intent fi_endpoint man page, provide
* a TX context (aka gnix nic) that can be shared
* explicitly amongst endpoints
*/
nic_attr.must_alloc = true;
ret = gnix_nic_alloc(domain, &nic_attr, &nic);
if (ret != FI_SUCCESS) {
GNIX_WARN(FI_LOG_EP_CTRL,
"_gnix_nic_alloc call returned %d\n",
ret);
goto err;
}
stx_priv->nic = nic;
_gnix_ref_init(&stx_priv->ref_cnt, 1, __stx_destruct);
_gnix_ref_get(stx_priv->domain);
stx_priv->stx_fid.fid.fclass = FI_CLASS_STX_CTX;
stx_priv->stx_fid.fid.context = context;
stx_priv->stx_fid.fid.ops = &gnix_stx_ops;
stx_priv->stx_fid.ops = NULL;
*stx = &stx_priv->stx_fid;
err:
return ret;
}
int gnix_wait_open(struct fid_fabric *fabric, struct fi_wait_attr *attr,
struct fid_wait **waitset)
{
struct gnix_fid_fabric *fab_priv;
struct gnix_fid_wait *wait_priv;
int ret = FI_SUCCESS;
GNIX_TRACE(WAIT_SUB, "\n");
ret = gnix_verify_wait_attr(attr);
if (ret)
goto err;
fab_priv = container_of(fabric, struct gnix_fid_fabric, fab_fid);
wait_priv = calloc(1, sizeof(*wait_priv));
if (!wait_priv) {
GNIX_ERR(WAIT_SUB,
"failed to allocate memory for wait set.\n");
ret = -FI_ENOMEM;
goto err;
}
ret = gnix_init_wait_obj(wait_priv, attr->wait_obj);
if (ret)
goto cleanup;
slist_init(&wait_priv->set);
wait_priv->wait.fid.fclass = FI_CLASS_WAIT;
wait_priv->wait.fid.ops = &gnix_fi_ops;
wait_priv->wait.ops = &gnix_wait_ops;
wait_priv->fabric = fab_priv;
_gnix_ref_get(fab_priv);
*waitset = &wait_priv->wait;
return ret;
cleanup:
free(wait_priv);
err:
return ret;
}
DIRECT_FN int gnix_pep_bind(struct fid *fid, struct fid *bfid, uint64_t flags)
{
int ret = FI_SUCCESS;
struct gnix_fid_pep *pep;
struct gnix_fid_eq *eq;
if (!fid || !bfid)
return -FI_EINVAL;
pep = container_of(fid, struct gnix_fid_pep, pep_fid.fid);
fastlock_acquire(&pep->lock);
switch (bfid->fclass) {
case FI_CLASS_EQ:
eq = container_of(bfid, struct gnix_fid_eq, eq_fid.fid);
if (pep->fabric != eq->fabric) {
ret = -FI_EINVAL;
break;
}
if (pep->eq) {
ret = -FI_EINVAL;
break;
}
pep->eq = eq;
_gnix_eq_poll_obj_add(eq, &pep->pep_fid.fid);
_gnix_ref_get(eq);
GNIX_DEBUG(FI_LOG_EP_CTRL, "Bound EQ to PEP: %p, %p\n",
eq, pep);
break;
default:
ret = -FI_ENOSYS;
break;
}
fastlock_release(&pep->lock);
return ret;
}
static int gnix_cq_set_wait(struct gnix_fid_cq *cq)
{
int ret = FI_SUCCESS;
GNIX_TRACE(FI_LOG_CQ, "\n");
struct fi_wait_attr requested = {
.wait_obj = cq->attr.wait_obj,
.flags = 0
};
switch (cq->attr.wait_obj) {
case FI_WAIT_UNSPEC:
case FI_WAIT_FD:
case FI_WAIT_MUTEX_COND:
ret = gnix_wait_open(&cq->domain->fabric->fab_fid,
&requested, &cq->wait);
break;
case FI_WAIT_SET:
ret = _gnix_wait_set_add(cq->attr.wait_set, &cq->cq_fid.fid);
if (!ret)
cq->wait = cq->attr.wait_set;
break;
default:
break;
}
return ret;
}
static void free_cq_entry(struct slist_entry *item)
{
struct gnix_cq_entry *entry;
entry = container_of(item, struct gnix_cq_entry, item);
free(entry->the_entry);
free(entry);
}
static struct slist_entry *alloc_cq_entry(size_t size)
{
struct gnix_cq_entry *entry = malloc(sizeof(*entry));
if (!entry) {
GNIX_DEBUG(FI_LOG_CQ, "out of memory\n");
goto err;
}
entry->the_entry = malloc(size);
if (!entry->the_entry) {
GNIX_DEBUG(FI_LOG_CQ, "out of memory\n");
goto cleanup;
}
return &entry->item;
cleanup:
free(entry);
err:
return NULL;
}
static int __gnix_cq_progress(struct gnix_fid_cq *cq)
{
return _gnix_prog_progress(&cq->pset);
}
/*******************************************************************************
* Exposed helper functions
******************************************************************************/
ssize_t _gnix_cq_add_event(struct gnix_fid_cq *cq, struct gnix_fid_ep *ep,
void *op_context, uint64_t flags, size_t len,
void *buf, uint64_t data, uint64_t tag,
fi_addr_t src_addr)
{
struct gnix_cq_entry *event;
struct slist_entry *item;
uint64_t mask;
ssize_t ret = FI_SUCCESS;
if (ep) {
if (ep->info && ep->info->mode & FI_NOTIFY_FLAGS_ONLY) {
mask = (FI_REMOTE_CQ_DATA | FI_MULTI_RECV);
if (flags & FI_RMA_EVENT) {
mask |= (FI_REMOTE_READ | FI_REMOTE_WRITE |
FI_RMA);
}
flags &= mask;
}
}
COND_ACQUIRE(cq->requires_lock, &cq->lock);
item = _gnix_queue_get_free(cq->events);
if (!item) {
GNIX_DEBUG(FI_LOG_CQ, "error creating cq_entry\n");
ret = -FI_ENOMEM;
goto err;
}
event = container_of(item, struct gnix_cq_entry, item);
assert(event->the_entry);
fill_function[cq->attr.format](event->the_entry, op_context, flags,
len, buf, data, tag);
event->src_addr = src_addr;
_gnix_queue_enqueue(cq->events, &event->item);
GNIX_DEBUG(FI_LOG_CQ, "Added event: %lx\n", op_context);
if (cq->wait)
_gnix_signal_wait_obj(cq->wait);
err:
COND_RELEASE(cq->requires_lock, &cq->lock);
return ret;
}
ssize_t _gnix_cq_add_error(struct gnix_fid_cq *cq, void *op_context,
uint64_t flags, size_t len, void *buf,
uint64_t data, uint64_t tag, size_t olen,
int err, int prov_errno, void *err_data,
size_t err_data_size)
{
struct fi_cq_err_entry *error;
struct gnix_cq_entry *event;
struct slist_entry *item;
ssize_t ret = FI_SUCCESS;
GNIX_INFO(FI_LOG_CQ, "creating error event entry\n");
COND_ACQUIRE(cq->requires_lock, &cq->lock);
item = _gnix_queue_get_free(cq->errors);
if (!item) {
GNIX_WARN(FI_LOG_CQ, "error creating error entry\n");
ret = -FI_ENOMEM;
goto err;
}
event = container_of(item, struct gnix_cq_entry, item);
error = event->the_entry;
error->op_context = op_context;
error->flags = flags;
error->len = len;
error->buf = buf;
error->data = data;
error->tag = tag;
error->olen = olen;
error->err = err;
error->prov_errno = prov_errno;
error->err_data = err_data;
error->err_data_size = err_data_size;
_gnix_queue_enqueue(cq->errors, &event->item);
if (cq->wait)
_gnix_signal_wait_obj(cq->wait);
err:
COND_RELEASE(cq->requires_lock, &cq->lock);
return ret;
}
int _gnix_cq_poll_obj_add(struct gnix_fid_cq *cq, void *obj,
int (*prog_fn)(void *data))
{
return _gnix_prog_obj_add(&cq->pset, obj, prog_fn);
}
int _gnix_cq_poll_obj_rem(struct gnix_fid_cq *cq, void *obj,
int (*prog_fn)(void *data))
{
return _gnix_prog_obj_rem(&cq->pset, obj, prog_fn);
}
static void __cq_destruct(void *obj)
{
struct gnix_fid_cq *cq = (struct gnix_fid_cq *) obj;
_gnix_ref_put(cq->domain);
switch (cq->attr.wait_obj) {
case FI_WAIT_NONE:
break;
case FI_WAIT_SET:
_gnix_wait_set_remove(cq->wait, &cq->cq_fid.fid);
break;
case FI_WAIT_UNSPEC:
case FI_WAIT_FD:
case FI_WAIT_MUTEX_COND:
assert(cq->wait);
gnix_wait_close(&cq->wait->fid);
break;
default:
GNIX_WARN(FI_LOG_CQ, "format: %d unsupported.\n",
cq->attr.wait_obj);
break;
}
_gnix_prog_fini(&cq->pset);
_gnix_queue_destroy(cq->events);
_gnix_queue_destroy(cq->errors);
fastlock_destroy(&cq->lock);
free(cq->cq_fid.ops);
free(cq->cq_fid.fid.ops);
free(cq);
}
/*******************************************************************************
* API functions.
******************************************************************************/
static int gnix_cq_close(fid_t fid)
{
struct gnix_fid_cq *cq;
int references_held;
GNIX_TRACE(FI_LOG_CQ, "\n");
cq = container_of(fid, struct gnix_fid_cq, cq_fid);
references_held = _gnix_ref_put(cq);
if (references_held) {
GNIX_INFO(FI_LOG_CQ, "failed to fully close cq due to lingering "
"references. references=%i cq=%p\n",
references_held, cq);
}
return FI_SUCCESS;
}
static ssize_t __gnix_cq_readfrom(struct fid_cq *cq, void *buf,
size_t count, fi_addr_t *src_addr)
{
struct gnix_fid_cq *cq_priv;
struct gnix_cq_entry *event;
struct slist_entry *temp;
ssize_t read_count = 0;
if (!cq || !buf || !count)
return -FI_EINVAL;
cq_priv = container_of(cq, struct gnix_fid_cq, cq_fid);
__gnix_cq_progress(cq_priv);
if (_gnix_queue_peek(cq_priv->errors))
return -FI_EAVAIL;
COND_ACQUIRE(cq_priv->requires_lock, &cq_priv->lock);
while (_gnix_queue_peek(cq_priv->events) && count--) {
temp = _gnix_queue_dequeue(cq_priv->events);
event = container_of(temp, struct gnix_cq_entry, item);
assert(event->the_entry);
memcpy(buf, event->the_entry, cq_priv->entry_size);
if (src_addr)
memcpy(&src_addr[read_count], &event->src_addr, sizeof(fi_addr_t));
_gnix_queue_enqueue_free(cq_priv->events, &event->item);
buf = (void *) ((uint8_t *) buf + cq_priv->entry_size);
read_count++;
}
COND_RELEASE(cq_priv->requires_lock, &cq_priv->lock);
return read_count ?: -FI_EAGAIN;
}
static ssize_t __gnix_cq_sreadfrom(int blocking, struct fid_cq *cq, void *buf,
size_t count, fi_addr_t *src_addr,
const void *cond, int timeout)
{
struct gnix_fid_cq *cq_priv;
cq_priv = container_of(cq, struct gnix_fid_cq, cq_fid);
if ((blocking && !cq_priv->wait) ||
(blocking && cq_priv->attr.wait_obj == FI_WAIT_SET))
return -FI_EINVAL;
if (_gnix_queue_peek(cq_priv->errors))
return -FI_EAVAIL;
if (cq_priv->wait)
gnix_wait_wait((struct fid_wait *)cq_priv->wait, timeout);
return __gnix_cq_readfrom(cq, buf, count, src_addr);
}
DIRECT_FN STATIC ssize_t gnix_cq_sreadfrom(struct fid_cq *cq, void *buf,
size_t count, fi_addr_t *src_addr,
const void *cond, int timeout)
{
return __gnix_cq_sreadfrom(1, cq, buf, count, src_addr, cond, timeout);
}
DIRECT_FN STATIC ssize_t gnix_cq_read(struct fid_cq *cq,
void *buf,
size_t count)
{
return __gnix_cq_sreadfrom(0, cq, buf, count, NULL, NULL, 0);
}
DIRECT_FN STATIC ssize_t gnix_cq_sread(struct fid_cq *cq, void *buf,
size_t count, const void *cond,
int timeout)
{
return __gnix_cq_sreadfrom(1, cq, buf, count, NULL, cond, timeout);
}
DIRECT_FN STATIC ssize_t gnix_cq_readfrom(struct fid_cq *cq, void *buf,
size_t count, fi_addr_t *src_addr)
{
return __gnix_cq_sreadfrom(0, cq, buf, count, src_addr, NULL, 0);
}
DIRECT_FN STATIC ssize_t gnix_cq_readerr(struct fid_cq *cq,
struct fi_cq_err_entry *buf,
uint64_t flags)
{
struct gnix_fid_cq *cq_priv;
struct gnix_cq_entry *event;
struct slist_entry *entry;
size_t err_data_cpylen;
struct fi_cq_err_entry *gnix_cq_err;
ssize_t read_count = 0;
if (!cq || !buf)
return -FI_EINVAL;
cq_priv = container_of(cq, struct gnix_fid_cq, cq_fid);
/*
* we need to progress cq. some apps may be only using
* cq to check for errors.
*/
_gnix_prog_progress(&cq_priv->pset);
COND_ACQUIRE(cq_priv->requires_lock, &cq_priv->lock);
entry = _gnix_queue_dequeue(cq_priv->errors);
if (!entry) {
read_count = -FI_EAGAIN;
goto err;
}
event = container_of(entry, struct gnix_cq_entry, item);
gnix_cq_err = event->the_entry;
buf->op_context = gnix_cq_err->op_context;
buf->flags = gnix_cq_err->flags;
buf->len = gnix_cq_err->len;
buf->buf = gnix_cq_err->buf;
buf->data = gnix_cq_err->data;
buf->tag = gnix_cq_err->tag;
buf->olen = gnix_cq_err->olen;
buf->err = gnix_cq_err->err;
buf->prov_errno = gnix_cq_err->prov_errno;
if (gnix_cq_err->err_data != NULL) {
/*
* Note: If the api version is >= 1.5 then copy err_data into
* buf->err_data and copy at most buf->err_data_size.
* If buf->err_data_size is zero or the api version is < 1.5,
* use the old method of allocating space in provider.
*/
if (FI_VERSION_LT(cq_priv->domain->fabric->fab_fid.api_version,
FI_VERSION(1, 5)) || buf->err_data_size == 0) {
err_data_cpylen = sizeof(cq_priv->err_data);
memcpy(cq_priv->err_data, gnix_cq_err->err_data,
err_data_cpylen);
buf->err_data = cq_priv->err_data;
} else {
if (buf->err_data == NULL)
return -FI_EINVAL;
err_data_cpylen = MIN(buf->err_data_size,
gnix_cq_err->err_data_size);
memcpy(buf->err_data, gnix_cq_err->err_data, err_data_cpylen);
buf->err_data_size = err_data_cpylen;
}
free(gnix_cq_err->err_data);
gnix_cq_err->err_data = NULL;
} else {
if (FI_VERSION_LT(cq_priv->domain->fabric->fab_fid.api_version,
FI_VERSION(1, 5))) {
buf->err_data = NULL;
} else {
buf->err_data_size = 0;
}
}
_gnix_queue_enqueue_free(cq_priv->errors, &event->item);
read_count++;
err:
COND_RELEASE(cq_priv->requires_lock, &cq_priv->lock);
return read_count;
}
DIRECT_FN STATIC const char *gnix_cq_strerror(struct fid_cq *cq, int prov_errno,
const void *prov_data, char *buf,
size_t len)
{
return NULL;
}
DIRECT_FN STATIC int gnix_cq_signal(struct fid_cq *cq)
{
struct gnix_fid_cq *cq_priv;
cq_priv = container_of(cq, struct gnix_fid_cq, cq_fid);
if (cq_priv->wait)
_gnix_signal_wait_obj(cq_priv->wait);
return FI_SUCCESS;
}
static int gnix_cq_control(struct fid *cq, int command, void *arg)
{
switch (command) {
case FI_GETWAIT:
return -FI_ENOSYS;
default:
return -FI_EINVAL;
}
}
DIRECT_FN int gnix_cq_open(struct fid_domain *domain, struct fi_cq_attr *attr,
struct fid_cq **cq, void *context)
{
struct gnix_fid_domain *domain_priv;
struct gnix_fid_cq *cq_priv;
struct fi_ops_cq *cq_ops;
struct fi_ops *fi_cq_ops;
int ret = FI_SUCCESS;
GNIX_TRACE(FI_LOG_CQ, "\n");
cq_ops = calloc(1, sizeof(*cq_ops));
if (!cq_ops) {
return -FI_ENOMEM;
}
fi_cq_ops = calloc(1, sizeof(*fi_cq_ops));
if (!fi_cq_ops) {
ret = -FI_ENOMEM;
goto free_cq_ops;
}
*cq_ops = gnix_cq_ops;
*fi_cq_ops = gnix_cq_fi_ops;
ret = verify_cq_attr(attr, cq_ops, fi_cq_ops);
if (ret)
goto free_fi_cq_ops;
domain_priv = container_of(domain, struct gnix_fid_domain, domain_fid);
if (!domain_priv) {
ret = -FI_EINVAL;
goto free_fi_cq_ops;
}
cq_priv = calloc(1, sizeof(*cq_priv));
if (!cq_priv) {
ret = -FI_ENOMEM;
goto free_fi_cq_ops;
}
cq_priv->requires_lock = (domain_priv->thread_model !=
FI_THREAD_COMPLETION);
cq_priv->domain = domain_priv;
cq_priv->attr = *attr;
_gnix_ref_init(&cq_priv->ref_cnt, 1, __cq_destruct);
_gnix_ref_get(cq_priv->domain);
_gnix_prog_init(&cq_priv->pset);
cq_priv->cq_fid.fid.fclass = FI_CLASS_CQ;
cq_priv->cq_fid.fid.context = context;
cq_priv->cq_fid.fid.ops = fi_cq_ops;
cq_priv->cq_fid.ops = cq_ops;
/*
* Although we don't need to store entry_size since we're already
* storing the format, this might provide a performance benefit
* when allocating storage.
*/
cq_priv->entry_size = format_sizes[cq_priv->attr.format];
fastlock_init(&cq_priv->lock);
ret = gnix_cq_set_wait(cq_priv);
if (ret)
goto free_cq_priv;
ret = _gnix_queue_create(&cq_priv->events, alloc_cq_entry,
free_cq_entry, cq_priv->entry_size,
cq_priv->attr.size);
if (ret)
goto free_cq_priv;
ret = _gnix_queue_create(&cq_priv->errors, alloc_cq_entry,
free_cq_entry, sizeof(struct fi_cq_err_entry),
0);
if (ret)
goto free_gnix_queue;
*cq = &cq_priv->cq_fid;
return ret;
free_gnix_queue:
_gnix_queue_destroy(cq_priv->events);
free_cq_priv:
_gnix_ref_put(cq_priv->domain);
fastlock_destroy(&cq_priv->lock);
free(cq_priv);
free_fi_cq_ops:
free(fi_cq_ops);
free_cq_ops:
free(cq_ops);
return ret;
}
/*******************************************************************************
* FI_OPS_* data structures.
******************************************************************************/
static const struct fi_ops gnix_cq_fi_ops = {
.size = sizeof(struct fi_ops),
.close = gnix_cq_close,
.bind = fi_no_bind,
.control = gnix_cq_control,
.ops_open = fi_no_ops_open
};
static const struct fi_ops_cq gnix_cq_ops = {
.size = sizeof(struct fi_ops_cq),
.read = gnix_cq_read,
.readfrom = gnix_cq_readfrom,
.readerr = gnix_cq_readerr,
.sread = gnix_cq_sread,
.sreadfrom = gnix_cq_sreadfrom,
.signal = gnix_cq_signal,
.strerror = gnix_cq_strerror
};
DIRECT_FN int gnix_domain_open(struct fid_fabric *fabric, struct fi_info *info,
struct fid_domain **dom, void *context)
{
struct gnix_fid_domain *domain = NULL;
int ret = FI_SUCCESS;
uint8_t ptag;
uint32_t cookie;
struct gnix_fid_fabric *fabric_priv;
GNIX_TRACE(FI_LOG_DOMAIN, "\n");
fabric_priv = container_of(fabric, struct gnix_fid_fabric, fab_fid);
/*
* check cookie/ptag credentials - for FI_EP_MSG we may be creating a
* domain
* using a cookie supplied being used by the server. Otherwise, we use
* use the cookie/ptag supplied by the job launch system.
*/
if (info->dest_addr) {
ret =
gnixu_get_rdma_credentials(info->dest_addr, &ptag, &cookie);
if (ret) {
GNIX_WARN(FI_LOG_DOMAIN,
"gnixu_get_rdma_credentials returned ptag %u cookie 0x%x\n",
ptag, cookie);
goto err;
}
} else {
ret = gnixu_get_rdma_credentials(NULL, &ptag, &cookie);
}
GNIX_INFO(FI_LOG_DOMAIN,
"gnix rdma credentials returned ptag %u cookie 0x%x\n",
ptag, cookie);
domain = calloc(1, sizeof *domain);
if (domain == NULL) {
ret = -FI_ENOMEM;
goto err;
}
domain->mr_cache_attr = _gnix_default_mr_cache_attr;
domain->mr_cache_attr.reg_context = (void *) domain;
domain->mr_cache_attr.dereg_context = NULL;
domain->mr_cache_attr.destruct_context = NULL;
ret = _gnix_notifier_open(&domain->mr_cache_attr.notifier);
if (ret != FI_SUCCESS)
goto err;
domain->mr_cache_ro = NULL;
domain->mr_cache_rw = NULL;
fastlock_init(&domain->mr_cache_lock);
domain->udreg_reg_limit = 4096;
dlist_init(&domain->nic_list);
dlist_init(&domain->list);
dlist_insert_after(&domain->list, &fabric_priv->domain_list);
domain->fabric = fabric_priv;
_gnix_ref_get(domain->fabric);
domain->ptag = ptag;
domain->cookie = cookie;
domain->cdm_id_seed = getpid(); /* TODO: direct syscall better */
/* user tunables */
domain->params.msg_rendezvous_thresh = default_msg_rendezvous_thresh;
domain->params.rma_rdma_thresh = default_rma_rdma_thresh;
domain->params.ct_init_size = default_ct_init_size;
domain->params.ct_max_size = default_ct_max_size;
domain->params.ct_step = default_ct_step;
domain->params.vc_id_table_capacity = default_vc_id_table_capacity;
domain->params.mbox_page_size = default_mbox_page_size;
domain->params.mbox_num_per_slab = default_mbox_num_per_slab;
domain->params.mbox_maxcredit = default_mbox_maxcredit;
domain->params.mbox_msg_maxsize = default_mbox_msg_maxsize;
domain->params.max_retransmits = default_max_retransmits;
domain->params.err_inject_count = default_err_inject_count;
#if HAVE_XPMEM
domain->params.xpmem_enabled = true;
#else
domain->params.xpmem_enabled = false;
#endif
domain->gni_tx_cq_size = default_tx_cq_size;
domain->gni_rx_cq_size = default_rx_cq_size;
domain->gni_cq_modes = gnix_def_gni_cq_modes;
_gnix_ref_init(&domain->ref_cnt, 1, __domain_destruct);
domain->domain_fid.fid.fclass = FI_CLASS_DOMAIN;
domain->domain_fid.fid.context = context;
domain->domain_fid.fid.ops = &gnix_domain_fi_ops;
domain->domain_fid.ops = &gnix_domain_ops;
domain->domain_fid.mr = &gnix_domain_mr_ops;
domain->control_progress = info->domain_attr->control_progress;
domain->data_progress = info->domain_attr->data_progress;
domain->thread_model = info->domain_attr->threading;
domain->mr_is_init = 0;
domain->mr_iov_limit = info->domain_attr->mr_iov_limit;
fastlock_init(&domain->cm_nic_lock);
_gnix_open_cache(domain, GNIX_DEFAULT_CACHE_TYPE);
*dom = &domain->domain_fid;
return FI_SUCCESS;
err:
if (domain != NULL) {
free(domain);
}
return ret;
}
int gnix_ep_open(struct fid_domain *domain, struct fi_info *info,
struct fid_ep **ep, void *context)
{
int ret = FI_SUCCESS;
int tsret = FI_SUCCESS;
uint32_t cdm_id;
struct gnix_fid_domain *domain_priv;
struct gnix_fid_ep *ep_priv;
gnix_hashtable_attr_t gnix_ht_attr;
gnix_ht_key_t *key_ptr;
struct gnix_tag_storage_attr untagged_attr = {
.type = GNIX_TAG_LIST,
.use_src_addr_matching = 1,
};
bool free_list_inited = false;
GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
if ((domain == NULL) || (info == NULL) || (ep == NULL) ||
(info->ep_attr == NULL))
return -FI_EINVAL;
/*
* TODO: need to implement other endpoint types
*/
if (info->ep_attr->type != FI_EP_RDM)
return -FI_ENOSYS;
domain_priv = container_of(domain, struct gnix_fid_domain, domain_fid);
ep_priv = calloc(1, sizeof *ep_priv);
if (!ep_priv)
return -FI_ENOMEM;
/* init untagged storages */
tsret = _gnix_posted_tag_storage_init(
&ep_priv->posted_recv_queue, &untagged_attr);
if (tsret)
return tsret;
tsret = _gnix_unexpected_tag_storage_init(
&ep_priv->unexp_recv_queue, &untagged_attr);
if (tsret)
return tsret;
/* init tagged storages */
tsret = _gnix_posted_tag_storage_init(
&ep_priv->tagged_posted_recv_queue, NULL);
if (tsret)
return tsret;
tsret = _gnix_unexpected_tag_storage_init(
&ep_priv->tagged_unexp_recv_queue, NULL);
if (tsret)
return tsret;
ep_priv->ep_fid.fid.fclass = FI_CLASS_EP;
ep_priv->ep_fid.fid.context = context;
ep_priv->ep_fid.fid.ops = &gnix_ep_fi_ops;
ep_priv->ep_fid.ops = &gnix_ep_ops;
ep_priv->domain = domain_priv;
ep_priv->type = info->ep_attr->type;
_gnix_ref_init(&ep_priv->ref_cnt, 1, __ep_destruct);
fastlock_init(&ep_priv->recv_comp_lock);
fastlock_init(&ep_priv->recv_queue_lock);
fastlock_init(&ep_priv->tagged_queue_lock);
slist_init(&ep_priv->pending_recv_comp_queue);
ep_priv->caps = info->caps & GNIX_EP_RDM_CAPS;
if (info->tx_attr)
ep_priv->op_flags = info->tx_attr->op_flags;
if (info->rx_attr)
ep_priv->op_flags |= info->rx_attr->op_flags;
ep_priv->op_flags &= GNIX_EP_OP_FLAGS;
ep_priv->min_multi_recv = GNIX_OPT_MIN_MULTI_RECV_DEFAULT;
ret = __fr_freelist_init(ep_priv);
if (ret != FI_SUCCESS) {
GNIX_WARN(FI_LOG_EP_CTRL,
"Error allocating gnix_fab_req freelist (%s)",
fi_strerror(-ret));
goto err;
} else
free_list_inited = true;
ep_priv->ep_fid.msg = &gnix_ep_msg_ops;
ep_priv->ep_fid.rma = &gnix_ep_rma_ops;
ep_priv->ep_fid.tagged = &gnix_ep_tagged_ops;
ep_priv->ep_fid.atomic = &gnix_ep_atomic_ops;
ep_priv->ep_fid.cm = &gnix_cm_ops;
if (ep_priv->type == FI_EP_RDM) {
if (info->src_addr != NULL) {
ret = __gnix_ep_bound_prep(domain_priv,
info,
ep_priv);
if (ret != FI_SUCCESS) {
GNIX_WARN(FI_LOG_EP_CTRL,
"__gnix_ep_bound_prep returned error (%s)",
fi_strerror(-ret));
goto err;
}
} else {
fastlock_acquire(&domain_priv->cm_nic_lock);
/*
* if a cm_nic has not yet been allocated for this
* domain, do it now. Reuse the embedded gnix_nic
* in the cm_nic as the nic for this endpoint
* to reduce demand on Aries hw resources.
*/
if (domain_priv->cm_nic == NULL) {
ret = _gnix_cm_nic_alloc(domain_priv,
info,
&domain_priv->cm_nic);
if (ret != FI_SUCCESS) {
GNIX_WARN(FI_LOG_EP_CTRL,
"_gnix_cm_nic_alloc returned %s\n",
fi_strerror(-ret));
fastlock_release(
&domain_priv->cm_nic_lock);
goto err;
}
ep_priv->cm_nic = domain_priv->cm_nic;
ep_priv->nic = ep_priv->cm_nic->nic;
_gnix_ref_get(ep_priv->nic);
} else {
ep_priv->cm_nic = domain_priv->cm_nic;
_gnix_ref_get(ep_priv->cm_nic);
}
fastlock_release(&domain_priv->cm_nic_lock);
ep_priv->my_name.gnix_addr.device_addr =
ep_priv->cm_nic->my_name.gnix_addr.device_addr;
ep_priv->my_name.cm_nic_cdm_id =
ep_priv->cm_nic->my_name.gnix_addr.cdm_id;
ret = _gnix_get_new_cdm_id(domain_priv, &cdm_id);
if (ret != FI_SUCCESS) {
GNIX_WARN(FI_LOG_EP_CTRL,
"gnix_get_new_cdm_id call returned %s\n",
fi_strerror(-ret));
goto err;
}
ep_priv->my_name.gnix_addr.cdm_id = cdm_id;
}
key_ptr = (gnix_ht_key_t *)&ep_priv->my_name.gnix_addr;
ret = _gnix_ht_insert(ep_priv->cm_nic->addr_to_ep_ht,
*key_ptr,
ep_priv);
if ((ret != FI_SUCCESS) && (ret != -FI_ENOSPC)) {
GNIX_WARN(FI_LOG_EP_CTRL,
"__gnix_ht_insert returned %d\n",
ret);
goto err;
}
gnix_ht_attr.ht_initial_size = domain_priv->params.ct_init_size;
gnix_ht_attr.ht_maximum_size = domain_priv->params.ct_max_size;
gnix_ht_attr.ht_increase_step = domain_priv->params.ct_step;
gnix_ht_attr.ht_increase_type = GNIX_HT_INCREASE_MULT;
gnix_ht_attr.ht_collision_thresh = 500;
gnix_ht_attr.ht_hash_seed = 0xdeadbeefbeefdead;
gnix_ht_attr.ht_internal_locking = 0;
gnix_ht_attr.destructor = __gnix_vc_destroy_ht_entry;
ep_priv->vc_ht = calloc(1, sizeof(struct gnix_hashtable));
if (ep_priv->vc_ht == NULL)
goto err;
ret = _gnix_ht_init(ep_priv->vc_ht, &gnix_ht_attr);
if (ret != FI_SUCCESS) {
GNIX_WARN(FI_LOG_EP_CTRL,
"gnix_ht_init call returned %d\n",
ret);
goto err;
}
fastlock_init(&ep_priv->vc_ht_lock);
} else {
ep_priv->cm_nic = NULL;
ep_priv->vc = NULL;
}
ep_priv->progress_fn = NULL;
ep_priv->rx_progress_fn = NULL;
if (ep_priv->nic == NULL) {
ret = gnix_nic_alloc(domain_priv, NULL, &ep_priv->nic);
if (ret != FI_SUCCESS) {
GNIX_WARN(FI_LOG_EP_CTRL,
"_gnix_nic_alloc call returned %d\n",
ret);
goto err;
}
}
/*
* if smsg callbacks not present hook them up now
*/
if (ep_priv->nic->smsg_callbacks == NULL)
ep_priv->nic->smsg_callbacks = gnix_ep_smsg_callbacks;
_gnix_ref_get(ep_priv->domain);
*ep = &ep_priv->ep_fid;
return ret;
err:
if (free_list_inited == true)
__fr_freelist_destroy(ep_priv);
if (ep_priv->vc_ht != NULL) {
_gnix_ht_destroy(ep_priv->vc_ht); /* may not be initialized but
okay */
free(ep_priv->vc_ht);
ep_priv->vc_ht = NULL;
}
if (ep_priv->cm_nic != NULL)
ret = _gnix_cm_nic_free(ep_priv->cm_nic);
if (ep_priv->nic != NULL)
ret = _gnix_nic_free(ep_priv->nic);
free(ep_priv);
return ret;
}
static int __match_context(struct slist_entry *item, const void *arg)
{
struct gnix_fab_req *req;
req = container_of(item, struct gnix_fab_req, slist);
return req->user_context == arg;
}
static inline struct gnix_fab_req *__find_tx_req(
struct gnix_fid_ep *ep,
void *context)
{
struct gnix_fab_req *req = NULL;
struct slist_entry *entry;
struct gnix_vc *vc;
GNIX_HASHTABLE_ITERATOR(ep->vc_ht, iter);
GNIX_DEBUG(FI_LOG_EP_CTRL, "searching VCs for the correct context to"
" cancel, context=%p", context);
fastlock_acquire(&ep->vc_ht_lock);
while ((vc = _gnix_ht_iterator_next(&iter))) {
fastlock_acquire(&vc->tx_queue_lock);
entry = slist_remove_first_match(&vc->tx_queue,
__match_context, context);
fastlock_release(&vc->tx_queue_lock);
if (entry) {
req = container_of(entry, struct gnix_fab_req, slist);
break;
}
}
fastlock_release(&ep->vc_ht_lock);
return req;
}
static inline struct gnix_fab_req *__find_rx_req(
struct gnix_fid_ep *ep,
void *context)
{
struct gnix_fab_req *req = NULL;
fastlock_acquire(&ep->recv_queue_lock);
req = _gnix_remove_req_by_context(&ep->posted_recv_queue, context);
fastlock_release(&ep->recv_queue_lock);
if (req)
return req;
fastlock_acquire(&ep->tagged_queue_lock);
req = _gnix_remove_req_by_context(&ep->tagged_posted_recv_queue,
context);
fastlock_release(&ep->tagged_queue_lock);
return req;
}
static ssize_t gnix_ep_cancel(fid_t fid, void *context)
{
int ret = FI_SUCCESS;
struct gnix_fid_ep *ep;
struct gnix_fab_req *req;
struct gnix_fid_cq *err_cq = NULL;
struct gnix_fid_cntr *err_cntr = NULL;
void *addr;
uint64_t tag, flags;
size_t len;
int is_send = 0;
GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
ep = container_of(fid, struct gnix_fid_ep, ep_fid.fid);
if (!ep->domain)
return -FI_EDOMAIN;
/* without context, we will have to find a request that matches
* a recv or send request. Try the send requests first.
*/
GNIX_INFO(FI_LOG_EP_CTRL, "looking for event to cancel\n");
req = __find_tx_req(ep, context);
if (!req) {
req = __find_rx_req(ep, context);
if (req) {
err_cq = ep->recv_cq;
err_cntr = ep->recv_cntr;
}
} else {
is_send = 1;
err_cq = ep->send_cq;
err_cntr = ep->send_cntr;
}
GNIX_INFO(FI_LOG_EP_CTRL, "finished searching\n");
if (!req)
return -FI_ENOENT;
if (err_cq) {
/* add canceled event */
if (!(req->type == GNIX_FAB_RQ_RDMA_READ ||
req->type == GNIX_FAB_RQ_RDMA_WRITE)) {
if (!is_send) {
addr = (void *) req->msg.recv_addr;
len = req->msg.recv_len;
} else {
addr = (void *) req->msg.send_addr;
len = req->msg.send_len;
}
tag = req->msg.tag;
} else {
/* rma information */
addr = (void *) req->rma.loc_addr;
len = req->rma.len;
tag = 0;
}
flags = req->flags;
_gnix_cq_add_error(err_cq, context, flags, len, addr, 0 /* data */,
tag, len, FI_ECANCELED, FI_ECANCELED, 0);
}
if (err_cntr) {
/* signal increase in cntr errs */
_gnix_cntr_inc_err(err_cntr);
}
_gnix_fr_free(ep, req);
return ret;
}
static int gnix_ep_bind(fid_t fid, struct fid *bfid, uint64_t flags)
{
int ret = FI_SUCCESS;
struct gnix_fid_ep *ep;
struct gnix_fid_av *av;
struct gnix_fid_cq *cq;
struct gnix_fid_stx *stx;
struct gnix_fid_cntr *cntr;
GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
ep = container_of(fid, struct gnix_fid_ep, ep_fid.fid);
if (!bfid)
return -FI_EINVAL;
switch (bfid->fclass) {
case FI_CLASS_EQ:
ret = -FI_ENOSYS;
goto err;
break;
case FI_CLASS_CQ:
cq = container_of(bfid, struct gnix_fid_cq, cq_fid.fid);
if (ep->domain != cq->domain) {
ret = -FI_EINVAL;
break;
}
if (flags & FI_TRANSMIT) {
/* don't allow rebinding */
if (ep->send_cq) {
ret = -FI_EINVAL;
break;
}
ep->send_cq = cq;
if (flags & FI_SELECTIVE_COMPLETION) {
ep->send_selective_completion = 1;
}
_gnix_cq_poll_nic_add(cq, ep->nic);
_gnix_ref_get(cq);
}
if (flags & FI_RECV) {
/* don't allow rebinding */
if (ep->recv_cq) {
ret = -FI_EINVAL;
break;
}
ep->recv_cq = cq;
if (flags & FI_SELECTIVE_COMPLETION) {
ep->recv_selective_completion = 1;
}
_gnix_cq_poll_nic_add(cq, ep->nic);
_gnix_ref_get(cq);
}
break;
case FI_CLASS_AV:
av = container_of(bfid, struct gnix_fid_av, av_fid.fid);
if (ep->domain != av->domain) {
ret = -FI_EINVAL;
break;
}
ep->av = av;
_gnix_ref_get(ep->av);
break;
case FI_CLASS_CNTR:
cntr = container_of(bfid, struct gnix_fid_cntr, cntr_fid.fid);
if (ep->domain != cntr->domain) {
ret = -FI_EINVAL;
break;
}
if (flags & FI_SEND) {
/* don't allow rebinding */
if (ep->send_cntr) {
GNIX_WARN(FI_LOG_EP_CTRL,
"cannot rebind send counter (%p)\n",
cntr);
ret = -FI_EINVAL;
break;
}
ep->send_cntr = cntr;
_gnix_cntr_poll_nic_add(cntr, ep->nic);
_gnix_ref_get(cntr);
}
if (flags & FI_RECV) {
/* don't allow rebinding */
if (ep->recv_cntr) {
GNIX_WARN(FI_LOG_EP_CTRL,
"cannot rebind recv counter (%p)\n",
cntr);
ret = -FI_EINVAL;
break;
}
ep->recv_cntr = cntr;
_gnix_cntr_poll_nic_add(cntr, ep->nic);
_gnix_ref_get(cntr);
}
if (flags & FI_READ) {
/* don't allow rebinding */
if (ep->read_cntr) {
GNIX_WARN(FI_LOG_EP_CTRL,
"cannot rebind read counter (%p)\n",
cntr);
ret = -FI_EINVAL;
break;
}
ep->read_cntr = cntr;
_gnix_cntr_poll_nic_add(cntr, ep->nic);
_gnix_ref_get(cntr);
}
if (flags & FI_WRITE) {
/* don't allow rebinding */
if (ep->write_cntr) {
GNIX_WARN(FI_LOG_EP_CTRL,
"cannot rebind write counter (%p)\n",
cntr);
ret = -FI_EINVAL;
break;
}
ep->write_cntr = cntr;
_gnix_cntr_poll_nic_add(cntr, ep->nic);
_gnix_ref_get(cntr);
}
/* TODO: don't support this option right now,
never should have gotten here since gni provider
doesn't claim cap FI_RMA_EVENT. This
option could be supported via Aries atomics
or using SMSG cntrl messages */
if ((flags & FI_REMOTE_WRITE) || (flags & FI_REMOTE_READ)) {
GNIX_WARN(FI_LOG_EP_CTRL,
"unsupported counter flags (%p)\n",
cntr);
ret = -FI_ENOSYS;
}
break;
case FI_CLASS_STX_CTX:
stx = container_of(bfid, struct gnix_fid_stx, stx_fid.fid);
if (ep->domain != stx->domain) {
ret = -FI_EINVAL;
break;
}
ep->stx_ctx = stx;
_gnix_ref_get(ep->stx_ctx);
break;
case FI_CLASS_MR:/*TODO: got to figure this one out */
default:
ret = -FI_ENOSYS;
break;
}
err:
return ret;
}
static inline void *__gnix_generic_register(
struct gnix_fid_domain *domain,
struct gnix_fid_mem_desc *md,
void *address,
size_t length,
gni_cq_handle_t dst_cq_hndl,
int flags,
int vmdh_index)
{
struct gnix_nic *nic;
gni_return_t grc = GNI_RC_SUCCESS;
int rc;
pthread_mutex_lock(&gnix_nic_list_lock);
/* If the nic list is empty, create a nic */
if (unlikely((dlist_empty(&gnix_nic_list_ptag[domain->ptag])))) {
/* release the lock because we are not checking the list after
this point. Additionally, gnix_nic_alloc takes the
lock to add the nic. */
pthread_mutex_unlock(&gnix_nic_list_lock);
rc = gnix_nic_alloc(domain, NULL, &nic);
if (rc) {
GNIX_INFO(FI_LOG_MR,
"could not allocate nic to do mr_reg,"
" ret=%i\n", rc);
return NULL;
}
} else {
nic = dlist_first_entry(&gnix_nic_list_ptag[domain->ptag],
struct gnix_nic, ptag_nic_list);
if (unlikely(nic == NULL)) {
GNIX_ERR(FI_LOG_MR, "Failed to find nic on "
"ptag list\n");
pthread_mutex_unlock(&gnix_nic_list_lock);
return NULL;
}
_gnix_ref_get(nic);
pthread_mutex_unlock(&gnix_nic_list_lock);
}
COND_ACQUIRE(nic->requires_lock, &nic->lock);
grc = GNI_MemRegister(nic->gni_nic_hndl, (uint64_t) address,
length, dst_cq_hndl, flags,
vmdh_index, &md->mem_hndl);
COND_RELEASE(nic->requires_lock, &nic->lock);
if (unlikely(grc != GNI_RC_SUCCESS)) {
GNIX_INFO(FI_LOG_MR, "failed to register memory with uGNI, "
"ret=%s\n", gni_err_str[grc]);
_gnix_ref_put(nic);
return NULL;
}
/* set up the mem desc */
md->nic = nic;
md->domain = domain;
/* take references on domain */
_gnix_ref_get(md->domain);
return md;
}
DIRECT_FN int gnix_mr_reg(struct fid *fid, const void *buf, size_t len,
uint64_t access, uint64_t offset,
uint64_t requested_key, uint64_t flags,
struct fid_mr **mr_o, void *context)
{
struct gnix_fid_mem_desc *mr = NULL;
struct gnix_fid_domain *domain;
int rc;
uint64_t reg_addr, reg_len;
struct _gnix_fi_reg_context fi_reg_context = {
.access = access,
.offset = offset,
.requested_key = requested_key,
.flags = flags,
.context = context,
};
GNIX_TRACE(FI_LOG_MR, "\n");
/* Flags are reserved for future use and must be 0. */
if (unlikely(flags))
return -FI_EBADFLAGS;
/* The offset parameter is reserved for future use and must be 0.
* Additionally, check for invalid pointers, bad access flags and the
* correct fclass on associated fid
*/
if (offset || !buf || !mr_o || !access ||
(access & ~(FI_READ | FI_WRITE | FI_RECV | FI_SEND |
FI_REMOTE_READ |
FI_REMOTE_WRITE)) ||
(fid->fclass != FI_CLASS_DOMAIN))
return -FI_EINVAL;
domain = container_of(fid, struct gnix_fid_domain, domain_fid.fid);
reg_addr = ((uint64_t) buf) & ~((1 << GNIX_MR_PAGE_SHIFT) - 1);
reg_len = __calculate_length((uint64_t) buf, len,
1 << GNIX_MR_PAGE_SHIFT);
/* call cache register op to retrieve the right entry */
fastlock_acquire(&domain->mr_cache_lock);
if (unlikely(!domain->mr_ops))
_gnix_open_cache(domain, GNIX_DEFAULT_CACHE_TYPE);
if (unlikely(!domain->mr_ops->is_init(domain))) {
rc = domain->mr_ops->init(domain);
if (rc != FI_SUCCESS) {
fastlock_release(&domain->mr_cache_lock);
goto err;
}
}
rc = domain->mr_ops->reg_mr(domain,
(uint64_t) reg_addr, reg_len, &fi_reg_context,
(void **) &mr);
fastlock_release(&domain->mr_cache_lock);
/* check retcode */
if (unlikely(rc != FI_SUCCESS))
goto err;
/* md.mr_fid */
mr->mr_fid.mem_desc = mr;
mr->mr_fid.fid.fclass = FI_CLASS_MR;
mr->mr_fid.fid.context = context;
mr->mr_fid.fid.ops = &fi_gnix_mr_ops;
/* setup internal key structure */
mr->mr_fid.key = _gnix_convert_mhdl_to_key(&mr->mem_hndl);
_gnix_ref_get(mr->domain);
/* set up mr_o out pointer */
*mr_o = &mr->mr_fid;
return FI_SUCCESS;
err:
return rc;
}
