int ofi_mr_cache_search(struct ofi_mr_cache *cache, const struct fi_mr_attr *attr,
struct ofi_mr_entry **entry)
{
util_mr_cache_process_events(cache);
assert(attr->iov_count == 1);
FI_DBG(cache->domain->prov, FI_LOG_MR, "search %p (len: %" PRIu64 ")\n",
attr->mr_iov->iov_base, attr->mr_iov->iov_len);
cache->search_cnt++;
while (((cache->cached_cnt >= cache->max_cached_cnt) ||
(cache->cached_size >= cache->max_cached_size)) &&
ofi_mr_cache_flush(cache))
;
*entry = cache->mr_storage.find(&cache->mr_storage, attr->mr_iov);
if (!*entry) {
return util_mr_cache_create(cache, attr->mr_iov,
attr->access, entry);
}
/* This branch is always false if the merging entries wasn't requested */
if (!ofi_iov_within(attr->mr_iov, &(*entry)->iov))
return util_mr_cache_merge(cache, attr, *entry, entry);
cache->hit_cnt++;
if ((*entry)->use_cnt++ == 0)
dlist_remove_init(&(*entry)->lru_entry);
return 0;
}
void ofi_mr_cache_cleanup(struct ofi_mr_cache *cache)
{
struct ofi_mr_entry *entry;
struct dlist_entry *tmp;
FI_INFO(cache->domain->prov, FI_LOG_MR, "MR cache stats: "
"searches %zu, deletes %zu, hits %zu\n",
cache->search_cnt, cache->delete_cnt, cache->hit_cnt);
util_mr_cache_process_events(cache);
dlist_foreach_container_safe(&cache->lru_list, struct ofi_mr_entry,
entry, lru_entry, tmp) {
assert(entry->use_cnt == 0);
util_mr_uncache_entry(cache, entry);
dlist_remove_init(&entry->lru_entry);
util_mr_free_entry(cache, entry);
}
cache->mr_storage.destroy(&cache->mr_storage);
ofi_monitor_del_queue(&cache->nq);
ofi_atomic_dec32(&cache->domain->ref);
util_buf_pool_destroy(cache->entry_pool);
assert(cache->cached_cnt == 0);
assert(cache->cached_size == 0);
}
int _gnix_dgram_alloc(struct gnix_dgram_hndl *hndl, enum gnix_dgram_type type,
struct gnix_datagram **d_ptr)
{
int ret = -FI_EAGAIN;
struct gnix_datagram *d = NULL;
struct dlist_entry *the_free_list;
struct dlist_entry *the_active_list;
GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
fastlock_acquire(&hndl->lock);
if (type == GNIX_DGRAM_WC) {
the_free_list = &hndl->wc_dgram_free_list;
the_active_list = &hndl->wc_dgram_active_list;
} else {
the_free_list = &hndl->bnd_dgram_free_list;
the_active_list = &hndl->bnd_dgram_active_list;
}
if (!dlist_empty(the_free_list)) {
d = dlist_first_entry(the_free_list, struct gnix_datagram,
list);
if (d != NULL) {
dlist_remove_init(&d->list);
dlist_insert_head(&d->list, the_active_list);
d->type = type;
ret = FI_SUCCESS;
}
}
static void __domain_destruct(void *obj)
{
int ret = FI_SUCCESS;
struct gnix_fid_domain *domain = (struct gnix_fid_domain *) obj;
GNIX_TRACE(FI_LOG_DOMAIN, "\n");
ret = _gnix_close_cache(domain);
if (ret != FI_SUCCESS)
GNIX_FATAL(FI_LOG_MR, "failed to close memory registration cache\n");
ret = _gnix_notifier_close(domain->mr_cache_attr.notifier);
if (ret != FI_SUCCESS)
GNIX_FATAL(FI_LOG_MR, "failed to close MR notifier\n");
/*
* remove from the list of cdms attached to fabric
*/
dlist_remove_init(&domain->list);
_gnix_ref_put(domain->fabric);
memset(domain, 0, sizeof *domain);
free(domain);
}
static struct gnix_vc *__gnix_nic_next_pending_rx_vc(struct gnix_nic *nic)
{
struct gnix_vc *vc = NULL;
fastlock_acquire(&nic->rx_vc_lock);
vc = dlist_first_entry(&nic->rx_vcs, struct gnix_vc, rx_list);
if (vc)
dlist_remove_init(&vc->rx_list);
fastlock_release(&nic->rx_vc_lock);
if (vc) {
GNIX_INFO(FI_LOG_EP_CTRL, "Dequeued RX VC (%p)\n", vc);
_gnix_clear_bit(&vc->flags, GNIX_VC_FLAG_RX_SCHEDULED);
}
return vc;
}
static void
util_mr_cache_process_events(struct ofi_mr_cache *cache)
{
struct ofi_subscription *subscription;
struct ofi_mr_entry *entry;
while ((subscription = ofi_monitor_get_event(&cache->nq))) {
entry = container_of(subscription, struct ofi_mr_entry,
subscription);
if (entry->cached)
util_mr_uncache_entry(cache, entry);
if (entry->use_cnt == 0) {
dlist_remove_init(&entry->lru_entry);
util_mr_free_entry(cache, entry);
}
}
}
static int handle_poll_list(struct poll_fd_mgr *poll_mgr)
{
struct poll_fd_info *poll_item;
int ret = FI_SUCCESS;
int id = 0;
fastlock_acquire(&poll_mgr->lock);
while (!dlist_empty(&poll_mgr->list)) {
poll_item = container_of(poll_mgr->list.next,
struct poll_fd_info, entry);
dlist_remove_init(&poll_item->entry);
if (poll_item->flags & POLL_MGR_DEL) {
id = poll_fds_find_dup(poll_mgr, poll_item);
assert(id > 0);
if (id <= 0) {
ret = -FI_EINVAL;
goto err;
}
poll_fds_swap_del_last(poll_mgr, id);
poll_item->flags |= POLL_MGR_ACK;
} else {
assert(poll_fds_find_dup(poll_mgr, poll_item) < 0);
ret = poll_fds_add_item(poll_mgr, poll_item);
if (ret) {
FI_WARN(&tcpx_prov, FI_LOG_EP_CTRL,
"Failed to add fd to event polling\n");
}
}
if (poll_item->flags & POLL_MGR_FREE)
free(poll_item);
else
poll_item->flags |= POLL_MGR_ACK;
}
err:
fastlock_release(&poll_mgr->lock);
return ret;
}
static int
util_mr_cache_merge(struct ofi_mr_cache *cache, const struct fi_mr_attr *attr,
struct ofi_mr_entry *old_entry, struct ofi_mr_entry **entry)
{
struct iovec iov, *old_iov;
iov = *attr->mr_iov;
do {
FI_DBG(cache->domain->prov, FI_LOG_MR,
"merging %p (len: %" PRIu64 ") with %p (len: %" PRIu64 ")\n",
iov.iov_base, iov.iov_len,
old_entry->iov.iov_base, old_entry->iov.iov_len);
old_iov = &old_entry->iov;
iov.iov_len = ((uintptr_t)
MAX(ofi_iov_end(&iov), ofi_iov_end(old_iov))) -
((uintptr_t) MIN(iov.iov_base, old_iov->iov_base));
iov.iov_base = MIN(iov.iov_base, old_iov->iov_base);
FI_DBG(cache->domain->prov, FI_LOG_MR, "merged %p (len: %" PRIu64 ")\n",
iov.iov_base, iov.iov_len);
if (old_entry->subscribed) {
/* old entry will be removed as soon as `use_cnt == 0`.
* unsubscribe from the entry */
ofi_monitor_unsubscribe(&old_entry->subscription);
old_entry->subscribed = 0;
}
cache->mr_storage.erase(&cache->mr_storage, old_entry);
old_entry->cached = 0;
if (old_entry->use_cnt == 0) {
dlist_remove_init(&old_entry->lru_entry);
util_mr_free_entry(cache, old_entry);
}
} while ((old_entry = cache->mr_storage.find(&cache->mr_storage, &iov)));
return util_mr_cache_create(cache, &iov, attr->access, entry);
}
int _gnix_cm_nic_progress(void *arg)
{
struct gnix_cm_nic *cm_nic = (struct gnix_cm_nic *)arg;
int ret = FI_SUCCESS;
int complete;
struct gnix_work_req *p = NULL;
/*
* if we're doing FI_PROGRESS_MANUAL,
* see what's going on inside kgni's datagram
* box...
*/
if (cm_nic->ctrl_progress == FI_PROGRESS_MANUAL) {
++cm_nic->poll_cnt;
if (((cm_nic->poll_cnt % 512) == 0) ||
!dlist_empty(&cm_nic->cm_nic_wq)) {
ret = _gnix_dgram_poll(cm_nic->dgram_hndl,
GNIX_DGRAM_NOBLOCK);
if (ret != FI_SUCCESS) {
GNIX_WARN(FI_LOG_EP_CTRL,
"_gnix_dgram_poll returned %s\n",
fi_strerror(-ret));
goto err;
}
}
}
/*
* do a quick check if queue doesn't have anything yet,
* don't need this to be atomic
*/
check_again:
if (dlist_empty(&cm_nic->cm_nic_wq))
return ret;
/*
* okay, stuff to do, lock work queue,
* dequeue head, unlock, process work element,
* if it doesn't compete, put back at the tail
* of the queue.
*/
fastlock_acquire(&cm_nic->wq_lock);
p = dlist_first_entry(&cm_nic->cm_nic_wq, struct gnix_work_req,
list);
if (p == NULL) {
fastlock_release(&cm_nic->wq_lock);
return ret;
}
dlist_remove_init(&p->list);
fastlock_release(&cm_nic->wq_lock);
assert(p->progress_fn);
ret = p->progress_fn(p->data, &complete);
if (ret != FI_SUCCESS) {
GNIX_WARN(FI_LOG_EP_CTRL,
"dgram prog fn returned %s\n",
fi_strerror(-ret));
}
if (complete == 1) {
if (p->completer_fn) {
ret = p->completer_fn(p->completer_data);
free(p);
if (ret != FI_SUCCESS) {
GNIX_WARN(FI_LOG_EP_CTRL,
"dgram completer fn returned %s\n",
fi_strerror(-ret));
goto err;
}
} else {
free(p);
}
goto check_again;
} else {
fastlock_acquire(&cm_nic->wq_lock);
dlist_insert_before(&p->list, &cm_nic->cm_nic_wq);
fastlock_release(&cm_nic->wq_lock);
}
err:
return ret;
}
