static int psmx2_domain_close(fid_t fid)
{
struct psmx2_fid_domain *domain;
domain = container_of(fid, struct psmx2_fid_domain,
util_domain.domain_fid.fid);
FI_INFO(&psmx2_prov, FI_LOG_DOMAIN, "refcnt=%d\n",
ofi_atomic_get32(&domain->util_domain.ref));
if (ofi_domain_close(&domain->util_domain))
return 0;
if (domain->progress_thread_enabled)
psmx2_domain_stop_progress(domain);
fastlock_destroy(&domain->sep_lock);
fastlock_destroy(&domain->mr_lock);
rbtDelete(domain->mr_map);
psmx2_lock(&domain->fabric->domain_lock, 1);
dlist_remove(&domain->entry);
psmx2_unlock(&domain->fabric->domain_lock, 1);
psmx2_fabric_release(domain->fabric);
free(domain);
return 0;
}
static int psmx2_domain_close(fid_t fid)
{
struct psmx2_fid_domain *domain;
domain = container_of(fid, struct psmx2_fid_domain,
util_domain.domain_fid.fid);
FI_INFO(&psmx2_prov, FI_LOG_DOMAIN, "refcnt=%d\n",
ofi_atomic_get32(&domain->util_domain.ref));
psmx2_domain_release(domain);
if (ofi_domain_close(&domain->util_domain))
return 0;
if (domain->progress_thread_enabled)
psmx2_domain_stop_progress(domain);
fastlock_destroy(&domain->sep_lock);
fastlock_destroy(&domain->vl_lock);
rbtDelete(domain->mr_map);
fastlock_destroy(&domain->mr_lock);
psmx2_trx_ctxt_free(domain->base_trx_ctxt);
domain->fabric->active_domain = NULL;
free(domain);
psmx2_atomic_global_fini();
psmx2_am_global_fini();
return 0;
}
static int psmx2_fabric_close(fid_t fid)
{
struct psmx2_fid_fabric *fabric;
fabric = container_of(fid, struct psmx2_fid_fabric,
util_fabric.fabric_fid.fid);
psmx2_fabric_release(fabric);
FI_INFO(&psmx2_prov, FI_LOG_CORE, "refcnt=%d\n",
ofi_atomic_get32(&fabric->util_fabric.ref));
if (ofi_fabric_close(&fabric->util_fabric))
return 0;
if (psmx2_env.name_server)
ofi_ns_stop_server(&fabric->name_server);
fastlock_destroy(&fabric->domain_lock);
assert(fabric == psmx2_active_fabric);
psmx2_active_fabric = NULL;
free(fabric);
psmx2_atomic_global_fini();
return 0;
}
static int util_eq_close(struct fid *fid)
{
struct util_eq *eq;
struct slist_entry *entry;
struct util_event *event;
eq = container_of(fid, struct util_eq, eq_fid.fid);
if (ofi_atomic_get32(&eq->ref))
return -FI_EBUSY;
while (!slist_empty(&eq->list)) {
entry = slist_remove_head(&eq->list);
event = container_of(entry, struct util_event, entry);
free(event);
}
if (eq->wait) {
fi_poll_del(&eq->wait->pollset->poll_fid,
&eq->eq_fid.fid, 0);
if (eq->internal_wait)
fi_close(&eq->wait->wait_fid.fid);
}
fastlock_destroy(&eq->lock);
ofi_atomic_dec32(&eq->fabric->ref);
free(eq);
return 0;
}
static int
usdf_domain_close(fid_t fid)
{
struct usdf_domain *udp;
int ret;
USDF_TRACE_SYS(DOMAIN, "\n");
udp = container_of(fid, struct usdf_domain, dom_fid.fid);
if (ofi_atomic_get32(&udp->dom_refcnt) > 0) {
return -FI_EBUSY;
}
if (udp->dom_dev != NULL) {
ret = usd_close(udp->dom_dev);
if (ret != 0) {
return ret;
}
}
usdf_dom_rdc_free_data(udp);
if (udp->dom_eq != NULL) {
ofi_atomic_dec32(&udp->dom_eq->eq_refcnt);
}
ofi_atomic_dec32(&udp->dom_fabric->fab_refcnt);
LIST_REMOVE(udp, dom_link);
fi_freeinfo(udp->dom_info);
free(udp);
return 0;
}
static int psmx2_sep_close(fid_t fid)
{
struct psmx2_fid_sep *sep;
struct psmx2_ep_name ep_name;
int i;
sep = container_of(fid, struct psmx2_fid_sep, ep.fid);
if (ofi_atomic_get32(&sep->ref))
return -FI_EBUSY;
for (i = 0; i < sep->ctxt_cnt; i++) {
if (sep->ctxts[i].ep && ofi_atomic_get32(&sep->ctxts[i].ep->ref))
return -FI_EBUSY;
}
ep_name.epid = sep->ctxts[0].trx_ctxt->psm2_epid;
ep_name.sep_id = sep->id;
ep_name.type = sep->type;
ofi_ns_del_local_name(&sep->domain->fabric->name_server,
&sep->service, &ep_name);
for (i = 0; i < sep->ctxt_cnt; i++) {
psmx2_lock(&sep->domain->trx_ctxt_lock, 1);
dlist_remove(&sep->ctxts[i].trx_ctxt->entry);
psmx2_unlock(&sep->domain->trx_ctxt_lock, 1);
if (sep->ctxts[i].ep)
psmx2_ep_close_internal(sep->ctxts[i].ep);
psmx2_trx_ctxt_free(sep->ctxts[i].trx_ctxt);
}
psmx2_lock(&sep->domain->sep_lock, 1);
dlist_remove(&sep->entry);
psmx2_unlock(&sep->domain->sep_lock, 1);
psmx2_domain_release(sep->domain);
free(sep);
return 0;
}
static int psmx_domain_close(fid_t fid)
{
struct psmx_fid_domain *domain;
int err;
domain = container_of(fid, struct psmx_fid_domain,
util_domain.domain_fid.fid);
FI_INFO(&psmx_prov, FI_LOG_DOMAIN, "refcnt=%d\n",
ofi_atomic_get32(&domain->util_domain.ref));
psmx_domain_release(domain);
if (ofi_domain_close(&domain->util_domain))
return 0;
if (domain->progress_thread_enabled)
psmx_domain_stop_progress(domain);
if (domain->am_initialized)
psmx_am_fini(domain);
fastlock_destroy(&domain->poll_lock);
rbtDelete(domain->mr_map);
fastlock_destroy(&domain->mr_lock);
#if 0
/* AM messages could arrive after MQ is finalized, causing segfault
* when trying to dereference the MQ pointer. There is no mechanism
* to properly shutdown AM. The workaround is to keep MQ valid.
*/
psm_mq_finalize(domain->psm_mq);
#endif
/* workaround for:
* Assertion failure at psm_ep.c:1059: ep->mctxt_master == ep
*/
sleep(psmx_env.delay);
if (psmx_env.timeout)
err = psm_ep_close(domain->psm_ep, PSM_EP_CLOSE_GRACEFUL,
(int64_t) psmx_env.timeout * 1000000000LL);
else
err = PSM_EP_CLOSE_TIMEOUT;
if (err != PSM_OK)
psm_ep_close(domain->psm_ep, PSM_EP_CLOSE_FORCE, 0);
domain->fabric->active_domain = NULL;
free(domain);
return 0;
}
int fi_wait_cleanup(struct util_wait *wait)
{
int ret;
if (ofi_atomic_get32(&wait->ref))
return -FI_EBUSY;
ret = fi_close(&wait->pollset->poll_fid.fid);
if (ret)
return ret;
ofi_atomic_dec32(&wait->fabric->ref);
return 0;
}
int ofi_domain_close(struct util_domain *domain)
{
if (ofi_atomic_get32(&domain->ref))
return -FI_EBUSY;
fastlock_acquire(&domain->fabric->lock);
dlist_remove(&domain->list_entry);
fastlock_release(&domain->fabric->lock);
free(domain->name);
fastlock_destroy(&domain->lock);
ofi_atomic_dec32(&domain->fabric->ref);
return 0;
}
static int psmx2_stx_close(fid_t fid)
{
struct psmx2_fid_stx *stx;
stx = container_of(fid, struct psmx2_fid_stx, stx.fid);
if (ofi_atomic_get32(&stx->ref))
return -FI_EBUSY;
psmx2_lock(&stx->domain->trx_ctxt_lock, 1);
dlist_remove(&stx->tx->entry);
psmx2_unlock(&stx->domain->trx_ctxt_lock, 1);
psmx2_trx_ctxt_free(stx->tx);
psmx2_domain_release(stx->domain);
free(stx);
return 0;
}
ssize_t fi_ibv_send(struct fi_ibv_msg_ep *ep, struct ibv_send_wr *wr, size_t len,
int count, void *context)
{
struct ibv_send_wr *bad_wr;
int ret;
assert(ep->scq);
wr->num_sge = count;
wr->wr_id = (uintptr_t) context;
if (wr->send_flags & IBV_SEND_SIGNALED) {
assert((wr->wr_id & ep->scq->wr_id_mask) != ep->scq->send_signal_wr_id);
ofi_atomic_set32(&ep->unsignaled_send_cnt, 0);
} else {
if (VERBS_SIGNAL_SEND(ep)) {
ret = fi_ibv_signal_send(ep, wr);
if (ret)
return ret;
} else {
ofi_atomic_inc32(&ep->unsignaled_send_cnt);
if (ofi_atomic_get32(&ep->unsignaled_send_cnt) >=
VERBS_SEND_COMP_THRESH(ep)) {
ret = fi_ibv_reap_comp(ep);
if (ret)
return ret;
}
}
}
ret = ibv_post_send(ep->id->qp, wr, &bad_wr);
switch (ret) {
case ENOMEM:
return -FI_EAGAIN;
case -1:
/* Deal with non-compliant libibverbs drivers which set errno
* instead of directly returning the error value */
return (errno == ENOMEM) ? -FI_EAGAIN : -errno;
default:
return -ret;
}
}
/**
* This function will return a block of id's starting at id through nids
*
* @param domain gnix domain
* @param nids number of id's
* @param id if -1 return an id based on the counter and seed
*/
int _gnix_get_new_cdm_id_set(struct gnix_fid_domain *domain, int nids,
uint32_t *id)
{
uint32_t cdm_id;
int v;
if (*id == -1) {
v = ofi_atomic_add32(&gnix_id_counter, nids);
cdm_id = ((domain->cdm_id_seed & 0xFFF) << 12) | v;
*id = cdm_id;
} else {
/*
* asking for a block starting at a chosen base
* TODO: sanity check that requested base is reasonable
*/
if (*id <= ofi_atomic_get32(&gnix_id_counter))
return -FI_ENOSPC;
ofi_atomic_set32(&gnix_id_counter, (*(int *)id + nids));
}
return FI_SUCCESS;
}
static int
usdf_fabric_close(fid_t fid)
{
struct usdf_fabric *fp;
int ret;
void *rv;
USDF_TRACE("\n");
fp = fab_fidtou(fid);
if (ofi_atomic_get32(&fp->fab_refcnt) > 0) {
return -FI_EBUSY;
}
/* Tell progression thread to exit */
fp->fab_exit = 1;
free(fp->fab_attr.name);
free(fp->fab_attr.prov_name);
if (fp->fab_thread) {
ret = usdf_fabric_wake_thread(fp);
if (ret != 0) {
return ret;
}
pthread_join(fp->fab_thread, &rv);
}
usdf_timer_deinit(fp);
if (fp->fab_epollfd != -1) {
close(fp->fab_epollfd);
}
if (fp->fab_eventfd != -1) {
close(fp->fab_eventfd);
}
if (fp->fab_arp_sockfd != -1) {
close(fp->fab_arp_sockfd);
}
free(fp);
return 0;
}
static int sock_cq_close(struct fid *fid)
{
struct sock_cq *cq;
cq = container_of(fid, struct sock_cq, cq_fid.fid);
if (ofi_atomic_get32(&cq->ref))
return -FI_EBUSY;
if (cq->signal && cq->attr.wait_obj == FI_WAIT_MUTEX_COND)
sock_wait_close(&cq->waitset->fid);
ofi_rbfree(&cq->addr_rb);
ofi_rbfree(&cq->cqerr_rb);
ofi_rbfdfree(&cq->cq_rbfd);
fastlock_destroy(&cq->lock);
fastlock_destroy(&cq->list_lock);
ofi_atomic_dec32(&cq->domain->ref);
free(cq);
return 0;
}
static int psmx_ep_close(fid_t fid)
{
struct psmx_fid_ep *ep;
ep = container_of(fid, struct psmx_fid_ep, ep.fid);
if (ep->base_ep) {
ofi_atomic_dec32(&ep->base_ep->ref);
return 0;
}
if (ofi_atomic_get32(&ep->ref))
return -FI_EBUSY;
ofi_ns_del_local_name(&ep->domain->fabric->name_server,
&ep->service, &ep->domain->psm_epid);
psmx_domain_disable_ep(ep->domain, ep);
psmx_domain_release(ep->domain);
free(ep);
return 0;
}
static int fi_ibv_reap_comp(struct fi_ibv_msg_ep *ep)
{
struct fi_ibv_wce *wce = NULL;
int got_wc = 0;
int ret = 0;
fastlock_acquire(&ep->scq->lock);
while (ofi_atomic_get32(&ep->comp_pending) > 0) {
if (!wce) {
wce = util_buf_alloc(ep->scq->wce_pool);
if (!wce) {
fastlock_release(&ep->scq->lock);
return -FI_ENOMEM;
}
memset(wce, 0, sizeof(*wce));
}
ret = fi_ibv_poll_cq(ep->scq, &wce->wc);
if (ret < 0) {
VERBS_WARN(FI_LOG_EP_DATA,
"Failed to read completion for signaled send\n");
util_buf_release(ep->scq->wce_pool, wce);
fastlock_release(&ep->scq->lock);
return ret;
} else if (ret > 0) {
slist_insert_tail(&wce->entry, &ep->scq->wcq);
got_wc = 1;
wce = NULL;
}
}
if (wce)
util_buf_release(ep->scq->wce_pool, wce);
if (got_wc && ep->scq->channel)
ret = fi_ibv_cq_signal(&ep->scq->cq_fid);
fastlock_release(&ep->scq->lock);
return ret;
}
static int psmx2_ep_close(fid_t fid)
{
struct psmx2_fid_ep *ep;
struct psmx2_ep_name ep_name;
struct psmx2_trx_ctxt *trx_ctxt;
ep = container_of(fid, struct psmx2_fid_ep, ep.fid);
if (ep->base_ep) {
ofi_atomic_dec32(&ep->base_ep->ref);
return 0;
}
if (ofi_atomic_get32(&ep->ref))
return -FI_EBUSY;
if (ep->stx)
ofi_atomic_dec32(&ep->stx->ref);
if (ep->rx) {
ep_name.epid = ep->rx->psm2_epid;
ofi_ns_del_local_name(&ep->domain->fabric->name_server,
&ep->service, &ep_name);
}
if (ep->rx) {
psmx2_lock(&ep->domain->trx_ctxt_lock, 1);
dlist_remove(&ep->rx->entry);
psmx2_unlock(&ep->domain->trx_ctxt_lock, 1);
}
trx_ctxt = ep->rx;
psmx2_ep_close_internal(ep);
psmx2_trx_ctxt_free(trx_ctxt);
return 0;
}
static void
usdf_dom_rdc_free_data(struct usdf_domain *udp)
{
struct usdf_rdm_connection *rdc;
int i;
if (udp->dom_rdc_hashtab != NULL) {
pthread_spin_lock(&udp->dom_progress_lock);
for (i = 0; i < USDF_RDM_HASH_SIZE; ++i) {
rdc = udp->dom_rdc_hashtab[i];
while (rdc != NULL) {
usdf_timer_reset(udp->dom_fabric,
rdc->dc_timer, 0);
rdc = rdc->dc_hash_next;
}
}
pthread_spin_unlock(&udp->dom_progress_lock);
/* XXX probably want a timeout here... */
while (ofi_atomic_get32(&udp->dom_rdc_free_cnt) <
(int)udp->dom_rdc_total) {
pthread_yield();
}
free(udp->dom_rdc_hashtab);
udp->dom_rdc_hashtab = NULL;
}
while (!SLIST_EMPTY(&udp->dom_rdc_free)) {
rdc = SLIST_FIRST(&udp->dom_rdc_free);
SLIST_REMOVE_HEAD(&udp->dom_rdc_free, dc_addr_link);
usdf_timer_free(udp->dom_fabric, rdc->dc_timer);
free(rdc);
}
}
static int sock_ep_close(struct fid *fid)
{
struct sock_ep *sock_ep;
char c = 0;
switch (fid->fclass) {
case FI_CLASS_EP:
sock_ep = container_of(fid, struct sock_ep, ep.fid);
break;
case FI_CLASS_SEP:
sock_ep = container_of(fid, struct sock_ep, ep.fid);
break;
default:
return -FI_EINVAL;
}
if (sock_ep->is_alias) {
ofi_atomic_dec32(&sock_ep->attr->ref);
return 0;
}
if (ofi_atomic_get32(&sock_ep->attr->ref) ||
ofi_atomic_get32(&sock_ep->attr->num_rx_ctx) ||
ofi_atomic_get32(&sock_ep->attr->num_tx_ctx))
return -FI_EBUSY;
if (sock_ep->attr->ep_type == FI_EP_MSG) {
sock_ep->attr->cm.do_listen = 0;
if (ofi_write_socket(sock_ep->attr->cm.signal_fds[0], &c, 1) != 1)
SOCK_LOG_DBG("Failed to signal\n");
if (sock_ep->attr->cm.listener_thread &&
pthread_join(sock_ep->attr->cm.listener_thread, NULL)) {
SOCK_LOG_ERROR("pthread join failed (%d)\n",
ofi_syserr());
}
ofi_close_socket(sock_ep->attr->cm.signal_fds[0]);
ofi_close_socket(sock_ep->attr->cm.signal_fds[1]);
} else {
if (sock_ep->attr->av)
ofi_atomic_dec32(&sock_ep->attr->av->ref);
}
if (sock_ep->attr->av) {
fastlock_acquire(&sock_ep->attr->av->list_lock);
fid_list_remove(&sock_ep->attr->av->ep_list,
&sock_ep->attr->lock, &sock_ep->ep.fid);
fastlock_release(&sock_ep->attr->av->list_lock);
}
pthread_mutex_lock(&sock_ep->attr->domain->pe->list_lock);
if (sock_ep->attr->tx_shared) {
fastlock_acquire(&sock_ep->attr->tx_ctx->lock);
dlist_remove(&sock_ep->attr->tx_ctx_entry);
fastlock_release(&sock_ep->attr->tx_ctx->lock);
}
if (sock_ep->attr->rx_shared) {
fastlock_acquire(&sock_ep->attr->rx_ctx->lock);
dlist_remove(&sock_ep->attr->rx_ctx_entry);
fastlock_release(&sock_ep->attr->rx_ctx->lock);
}
pthread_mutex_unlock(&sock_ep->attr->domain->pe->list_lock);
if (sock_ep->attr->conn_handle.do_listen) {
fastlock_acquire(&sock_ep->attr->domain->conn_listener.signal_lock);
fi_epoll_del(sock_ep->attr->domain->conn_listener.emap,
sock_ep->attr->conn_handle.sock);
fastlock_release(&sock_ep->attr->domain->conn_listener.signal_lock);
ofi_close_socket(sock_ep->attr->conn_handle.sock);
sock_ep->attr->conn_handle.do_listen = 0;
}
fastlock_destroy(&sock_ep->attr->cm.lock);
if (sock_ep->attr->eq) {
fastlock_acquire(&sock_ep->attr->eq->lock);
sock_ep_clear_eq_list(&sock_ep->attr->eq->list,
&sock_ep->ep);
/* Any err_data if present would be freed by
* sock_eq_clean_err_data_list when EQ is closed */
sock_ep_clear_eq_list(&sock_ep->attr->eq->err_list,
&sock_ep->ep);
fastlock_release(&sock_ep->attr->eq->lock);
}
if (sock_ep->attr->fclass != FI_CLASS_SEP) {
if (!sock_ep->attr->tx_shared)
sock_pe_remove_tx_ctx(sock_ep->attr->tx_array[0]);
sock_tx_ctx_close(sock_ep->attr->tx_array[0]);
sock_tx_ctx_free(sock_ep->attr->tx_array[0]);
}
if (sock_ep->attr->fclass != FI_CLASS_SEP) {
if (!sock_ep->attr->rx_shared)
sock_pe_remove_rx_ctx(sock_ep->attr->rx_array[0]);
sock_rx_ctx_close(sock_ep->attr->rx_array[0]);
sock_rx_ctx_free(sock_ep->attr->rx_array[0]);
}
free(sock_ep->attr->tx_array);
free(sock_ep->attr->rx_array);
if (sock_ep->attr->src_addr)
free(sock_ep->attr->src_addr);
if (sock_ep->attr->dest_addr)
free(sock_ep->attr->dest_addr);
fastlock_acquire(&sock_ep->attr->domain->pe->lock);
ofi_idm_reset(&sock_ep->attr->av_idm);
sock_conn_map_destroy(sock_ep->attr);
fastlock_release(&sock_ep->attr->domain->pe->lock);
ofi_atomic_dec32(&sock_ep->attr->domain->ref);
fastlock_destroy(&sock_ep->attr->lock);
free(sock_ep->attr);
free(sock_ep);
return 0;
}
void ofi_monitor_cleanup(struct ofi_mem_monitor *monitor)
{
assert(ofi_atomic_get32(&monitor->refcnt) == 0);
}
static ssize_t sock_cq_sreadfrom(struct fid_cq *cq, void *buf, size_t count,
fi_addr_t *src_addr, const void *cond, int timeout)
{
int ret = 0;
size_t threshold;
struct sock_cq *sock_cq;
uint64_t start_ms;
ssize_t cq_entry_len, avail;
sock_cq = container_of(cq, struct sock_cq, cq_fid);
if (ofi_rbused(&sock_cq->cqerr_rb))
return -FI_EAVAIL;
cq_entry_len = sock_cq->cq_entry_size;
if (sock_cq->attr.wait_cond == FI_CQ_COND_THRESHOLD)
threshold = MIN((uintptr_t) cond, count);
else
threshold = count;
start_ms = (timeout >= 0) ? fi_gettime_ms() : 0;
if (sock_cq->domain->progress_mode == FI_PROGRESS_MANUAL) {
while (1) {
sock_cq_progress(sock_cq);
fastlock_acquire(&sock_cq->lock);
avail = ofi_rbfdused(&sock_cq->cq_rbfd);
if (avail) {
ret = sock_cq_rbuf_read(sock_cq, buf,
MIN(threshold, (size_t)(avail / cq_entry_len)),
src_addr, cq_entry_len);
}
fastlock_release(&sock_cq->lock);
if (ret)
return ret;
if (timeout >= 0) {
timeout -= (int) (fi_gettime_ms() - start_ms);
if (timeout <= 0)
return -FI_EAGAIN;
}
if (ofi_atomic_get32(&sock_cq->signaled)) {
ofi_atomic_set32(&sock_cq->signaled, 0);
return -FI_ECANCELED;
}
};
} else {
do {
fastlock_acquire(&sock_cq->lock);
ret = 0;
avail = ofi_rbfdused(&sock_cq->cq_rbfd);
if (avail) {
ret = sock_cq_rbuf_read(sock_cq, buf,
MIN(threshold, (size_t)(avail / cq_entry_len)),
src_addr, cq_entry_len);
} else {
ofi_rbfdreset(&sock_cq->cq_rbfd);
}
fastlock_release(&sock_cq->lock);
if (ret && ret != -FI_EAGAIN)
return ret;
if (timeout >= 0) {
timeout -= (int) (fi_gettime_ms() - start_ms);
if (timeout <= 0)
return -FI_EAGAIN;
}
if (ofi_atomic_get32(&sock_cq->signaled)) {
ofi_atomic_set32(&sock_cq->signaled, 0);
return -FI_ECANCELED;
}
ret = ofi_rbfdwait(&sock_cq->cq_rbfd, timeout);
} while (ret > 0);
}
return (ret == 0 || ret == -FI_ETIMEDOUT) ? -FI_EAGAIN : ret;
}
