static ssize_t util_cq_readerr(struct fid_cq *cq_fid, struct fi_cq_err_entry *buf,
uint64_t flags)
{
struct util_cq *cq;
struct util_cq_err_entry *err;
struct slist_entry *entry;
ssize_t ret;
cq = container_of(cq_fid, struct util_cq, cq_fid);
fastlock_acquire(&cq->cq_lock);
if (!cirque_isempty(cq->cirq) &&
(cirque_head(cq->cirq)->flags & UTIL_FLAG_ERROR)) {
cirque_discard(cq->cirq);
entry = slist_remove_head(&cq->err_list);
err = container_of(entry, struct util_cq_err_entry, list_entry);
*buf = err->err_entry;
free(err);
ret = 0;
} else {
ssize_t rxm_send(struct fid_ep *ep_fid, const void *buf, size_t len, void *desc,
fi_addr_t dest_addr, void *context)
{
struct rxm_ep *rxm_ep;
struct fid_ep *msg_ep;
ssize_t ret;
rxm_ep = container_of(ep_fid, struct rxm_ep, util_ep.ep_fid.fid);
fastlock_acquire(&rxm_ep->cmap->lock);
ret = rxm_get_msg_ep(rxm_ep, dest_addr, &msg_ep);
if (ret)
goto unlock;
// TODO handle the case when send fails due to connection shutdown
ret = fi_send(msg_ep, buf, len, desc, 0, context);
unlock:
fastlock_release(&rxm_ep->cmap->lock);
return ret;
}
void mlx_send_callback( void *request,
ucs_status_t status)
{
struct util_cq *cq;
struct mlx_request *mlx_req = request;
struct fi_cq_tagged_entry *t_entry;
struct util_cq_err_entry *err;
cq = mlx_req->cq;
if (status == UCS_ERR_CANCELED) {
ucp_request_release(request);
return;
}
fastlock_acquire(&cq->cq_lock);
t_entry = cirque_tail(cq->cirq);
*t_entry = (mlx_req->completion.tagged);
cirque_commit(cq->cirq);
if (status != UCS_OK){
t_entry->flags |= UTIL_FLAG_ERROR;
err = calloc(1, sizeof(struct util_cq_err_entry));
if (!err) {
FI_WARN(&mlx_prov, FI_LOG_CQ,
"out of memory, cannot report CQ error\n");
return;
}
err->err_entry = (mlx_req->completion.error);
err->err_entry.prov_errno = (int)status;
err->err_entry.err = MLX_TRANSLATE_ERRCODE(status);
err->err_entry.olen = 0;
slist_insert_tail(&err->list_entry, &cq->err_list);
}
mlx_req->type = MLX_FI_REQ_UNINITIALIZED;
fastlock_release(&cq->cq_lock);
ucp_request_release(request);
}
int ofi_wait_fd_add(struct util_wait *wait, int fd, uint32_t events,
ofi_wait_fd_try_func wait_try, void *arg, void *context)
{
struct ofi_wait_fd_entry *fd_entry;
struct dlist_entry *entry;
struct util_wait_fd *wait_fd = container_of(wait, struct util_wait_fd,
util_wait);
int ret = 0;
fastlock_acquire(&wait_fd->lock);
entry = dlist_find_first_match(&wait_fd->fd_list, ofi_wait_fd_match, &fd);
if (entry) {
FI_DBG(wait->prov, FI_LOG_EP_CTRL,
"Given fd (%d) already added to wait list - %p \n",
fd, wait_fd);
fd_entry = container_of(entry, struct ofi_wait_fd_entry, entry);
ofi_atomic_inc32(&fd_entry->ref);
goto out;
}
ret = fi_epoll_add(wait_fd->epoll_fd, fd, events, context);
if (ret) {
FI_WARN(wait->prov, FI_LOG_FABRIC, "Unable to add fd to epoll\n");
goto out;
}
fd_entry = calloc(1, sizeof *fd_entry);
if (!fd_entry) {
ret = -FI_ENOMEM;
fi_epoll_del(wait_fd->epoll_fd, fd);
goto out;
}
fd_entry->fd = fd;
fd_entry->wait_try = wait_try;
fd_entry->arg = arg;
ofi_atomic_initialize32(&fd_entry->ref, 1);
dlist_insert_tail(&fd_entry->entry, &wait_fd->fd_list);
out:
fastlock_release(&wait_fd->lock);
return ret;
}
ssize_t sock_queue_rma_op(struct fid_ep *ep, const struct fi_msg_rma *msg,
uint64_t flags, uint8_t op_type)
{
struct sock_cntr *cntr;
struct sock_trigger *trigger;
struct fi_triggered_context *trigger_context;
struct fi_trigger_threshold *threshold;
trigger_context = (struct fi_triggered_context *) msg->context;
if ((flags & FI_INJECT) || !trigger_context ||
(trigger_context->event_type != FI_TRIGGER_THRESHOLD))
return -FI_EINVAL;
threshold = &trigger_context->trigger.threshold;
cntr = container_of(threshold->cntr, struct sock_cntr, cntr_fid);
if (atomic_get(&cntr->value) >= threshold->threshold)
return 1;
trigger = calloc(1, sizeof(*trigger));
if (!trigger)
return -FI_ENOMEM;
trigger->threshold = threshold->threshold;
memcpy(&trigger->op.rma.msg, msg, sizeof(*msg));
trigger->op.rma.msg.msg_iov = &trigger->op.rma.msg_iov[0];
trigger->op.rma.msg.rma_iov = &trigger->op.rma.rma_iov[0];
memcpy(&trigger->op.rma.msg_iov[0], &msg->msg_iov[0],
msg->iov_count * sizeof(struct iovec));
memcpy(&trigger->op.rma.rma_iov[0], &msg->rma_iov[0],
msg->rma_iov_count * sizeof(struct fi_rma_iov));
trigger->op_type = op_type;
trigger->ep = ep;
trigger->flags = flags;
fastlock_acquire(&cntr->trigger_lock);
dlist_insert_tail(&trigger->entry, &cntr->trigger_list);
fastlock_release(&cntr->trigger_lock);
sock_cntr_check_trigger_list(cntr);
return 0;
}
DIRECT_FN int gnix_pep_bind(struct fid *fid, struct fid *bfid, uint64_t flags)
{
int ret = FI_SUCCESS;
struct gnix_fid_pep *pep;
struct gnix_fid_eq *eq;
if (!fid || !bfid)
return -FI_EINVAL;
pep = container_of(fid, struct gnix_fid_pep, pep_fid.fid);
fastlock_acquire(&pep->lock);
switch (bfid->fclass) {
case FI_CLASS_EQ:
eq = container_of(bfid, struct gnix_fid_eq, eq_fid.fid);
if (pep->fabric != eq->fabric) {
ret = -FI_EINVAL;
break;
}
if (pep->eq) {
ret = -FI_EINVAL;
break;
}
pep->eq = eq;
_gnix_eq_poll_obj_add(eq, &pep->pep_fid.fid);
_gnix_ref_get(eq);
GNIX_DEBUG(FI_LOG_EP_CTRL, "Bound EQ to PEP: %p, %p\n",
eq, pep);
break;
default:
ret = -FI_ENOSYS;
break;
}
fastlock_release(&pep->lock);
return ret;
}
static int rxd_av_remove(struct fid_av *av_fid, fi_addr_t *fi_addr, size_t count,
uint64_t flags)
{
int ret = 0;
size_t i;
fi_addr_t dg_fiaddr;
struct rxd_av *av;
av = container_of(av_fid, struct rxd_av, util_av.av_fid);
fastlock_acquire(&av->util_av.lock);
for (i = 0; i < count; i++) {
dg_fiaddr = rxd_av_dg_addr(av, fi_addr[i]);
ret = fi_av_remove(av->dg_av, &dg_fiaddr, 1, flags);
if (ret)
break;
av->dg_av_used--;
}
fastlock_release(&av->util_av.lock);
return ret;
}
static ssize_t sock_cq_readerr(struct fid_cq *cq, struct fi_cq_err_entry *buf,
uint64_t flags)
{
struct sock_cq *sock_cq;
ssize_t ret;
sock_cq = container_of(cq, struct sock_cq, cq_fid);
if (sock_cq->domain->progress_mode == FI_PROGRESS_MANUAL)
sock_cq_progress(sock_cq);
fastlock_acquire(&sock_cq->lock);
if (rbused(&sock_cq->cqerr_rb) >= sizeof(struct fi_cq_err_entry)) {
rbread(&sock_cq->cqerr_rb, buf, sizeof(*buf));
ret = 1;
} else {
ret = -FI_EAGAIN;
}
fastlock_release(&sock_cq->lock);
return ret;
}
static ssize_t _sock_cq_write(struct sock_cq *cq, fi_addr_t addr,
const void *buf, size_t len)
{
ssize_t ret;
struct sock_cq_overflow_entry_t *overflow_entry;
fastlock_acquire(&cq->lock);
if (ofi_rbfdavail(&cq->cq_rbfd) < len) {
SOCK_LOG_ERROR("Not enough space in CQ\n");
overflow_entry = calloc(1, sizeof(*overflow_entry) + len);
if (!overflow_entry) {
ret = -FI_ENOSPC;
goto out;
}
memcpy(&overflow_entry->cq_entry[0], buf, len);
overflow_entry->len = len;
overflow_entry->addr = addr;
dlist_insert_tail(&overflow_entry->entry, &cq->overflow_list);
ret = len;
goto out;
}
ofi_rbwrite(&cq->addr_rb, &addr, sizeof(addr));
ofi_rbcommit(&cq->addr_rb);
ofi_rbfdwrite(&cq->cq_rbfd, buf, len);
if (cq->domain->progress_mode == FI_PROGRESS_MANUAL)
ofi_rbcommit(&cq->cq_rbfd.rb);
else
ofi_rbfdcommit(&cq->cq_rbfd);
ret = len;
if (cq->signal)
sock_wait_signal(cq->waitset);
out:
fastlock_release(&cq->lock);
return ret;
}
static int tcpx_ep_shutdown(struct fid_ep *ep, uint64_t flags)
{
struct tcpx_ep *tcpx_ep;
int ret;
tcpx_ep = container_of(ep, struct tcpx_ep, util_ep.ep_fid);
ret = ofi_shutdown(tcpx_ep->conn_fd, SHUT_RDWR);
if (ret && ofi_sockerr() != ENOTCONN) {
FI_WARN(&tcpx_prov, FI_LOG_EP_DATA, "ep shutdown unsuccessful\n");
}
fastlock_acquire(&tcpx_ep->lock);
ret = tcpx_ep_shutdown_report(tcpx_ep, &ep->fid);
fastlock_release(&tcpx_ep->lock);
if (ret) {
FI_WARN(&tcpx_prov, FI_LOG_EP_DATA, "Error writing to EQ\n");
}
return ret;
}
static int fi_ibv_signal_send(struct fi_ibv_msg_ep *ep, struct ibv_send_wr *wr)
{
struct fi_ibv_msg_epe *epe;
fastlock_acquire(&ep->scq->lock);
if (VERBS_SIGNAL_SEND(ep)) {
epe = util_buf_alloc(ep->scq->epe_pool);
if (!epe) {
fastlock_release(&ep->scq->lock);
return -FI_ENOMEM;
}
memset(epe, 0, sizeof(*epe));
wr->send_flags |= IBV_SEND_SIGNALED;
wr->wr_id = ep->ep_id;
epe->ep = ep;
slist_insert_tail(&epe->entry, &ep->scq->ep_list);
ofi_atomic_inc32(&ep->comp_pending);
}
fastlock_release(&ep->scq->lock);
return 0;
}
int ofi_monitor_add_cache(struct ofi_mem_monitor *monitor,
struct ofi_mr_cache *cache)
{
int ret = 0;
fastlock_acquire(&monitor->lock);
if (dlist_empty(&monitor->list)) {
if (monitor == uffd_monitor)
ret = ofi_uffd_init();
else
ret = -FI_ENOSYS;
if (ret)
goto out;
}
cache->monitor = monitor;
dlist_insert_tail(&cache->notify_entry, &monitor->list);
out:
fastlock_release(&monitor->lock);
return ret;
}
static ssize_t sock_cq_readerr(struct fid_cq *cq, struct fi_cq_err_entry *buf,
uint64_t flags)
{
struct sock_cq *sock_cq;
ssize_t ret;
struct fi_cq_err_entry entry;
uint32_t api_version;
size_t err_data_size = 0;
void *err_data = NULL;
sock_cq = container_of(cq, struct sock_cq, cq_fid);
if (sock_cq->domain->progress_mode == FI_PROGRESS_MANUAL)
sock_cq_progress(sock_cq);
fastlock_acquire(&sock_cq->lock);
if (ofi_rbused(&sock_cq->cqerr_rb) >= sizeof(struct fi_cq_err_entry)) {
api_version = sock_cq->domain->fab->fab_fid.api_version;
ofi_rbread(&sock_cq->cqerr_rb, &entry, sizeof(entry));
if ((FI_VERSION_GE(api_version, FI_VERSION(1, 5)))
&& buf->err_data && buf->err_data_size) {
err_data = buf->err_data;
err_data_size = buf->err_data_size;
*buf = entry;
buf->err_data = err_data;
/* Fill provided user's buffer */
buf->err_data_size = MIN(entry.err_data_size, err_data_size);
memcpy(buf->err_data, entry.err_data, buf->err_data_size);
} else {
*buf = entry;
}
ret = 1;
} else {
ret = -FI_EAGAIN;
}
fastlock_release(&sock_cq->lock);
return ret;
}
static int handle_poll_list(struct poll_fd_mgr *poll_mgr)
{
struct poll_fd_info *poll_item;
int ret = FI_SUCCESS;
int id = 0;
fastlock_acquire(&poll_mgr->lock);
while (!dlist_empty(&poll_mgr->list)) {
poll_item = container_of(poll_mgr->list.next,
struct poll_fd_info, entry);
dlist_remove_init(&poll_item->entry);
if (poll_item->flags & POLL_MGR_DEL) {
id = poll_fds_find_dup(poll_mgr, poll_item);
assert(id > 0);
if (id <= 0) {
ret = -FI_EINVAL;
goto err;
}
poll_fds_swap_del_last(poll_mgr, id);
poll_item->flags |= POLL_MGR_ACK;
} else {
assert(poll_fds_find_dup(poll_mgr, poll_item) < 0);
ret = poll_fds_add_item(poll_mgr, poll_item);
if (ret) {
FI_WARN(&tcpx_prov, FI_LOG_EP_CTRL,
"Failed to add fd to event polling\n");
}
}
if (poll_item->flags & POLL_MGR_FREE)
free(poll_item);
else
poll_item->flags |= POLL_MGR_ACK;
}
err:
fastlock_release(&poll_mgr->lock);
return ret;
}
/*
* All EPs use the same underlying datagram provider, so pick any and use its
* associated CQ.
*/
static const char *rxd_cq_strerror(struct fid_cq *cq_fid, int prov_errno,
const void *err_data, char *buf, size_t len)
{
struct fid_list_entry *fid_entry;
struct util_ep *util_ep;
struct rxd_cq *cq;
struct rxd_ep *ep;
const char *str;
cq = container_of(cq_fid, struct rxd_cq, util_cq.cq_fid);
fastlock_acquire(&cq->util_cq.ep_list_lock);
assert(!dlist_empty(&cq->util_cq.ep_list));
fid_entry = container_of(cq->util_cq.ep_list.next,
struct fid_list_entry, entry);
util_ep = container_of(fid_entry->fid, struct util_ep, ep_fid.fid);
ep = container_of(util_ep, struct rxd_ep, util_ep);
str = fi_cq_strerror(ep->dg_cq, prov_errno, err_data, buf, len);
fastlock_release(&cq->util_cq.ep_list_lock);
return str;
}
static int psmx2_alloc_vlane(struct psmx2_fid_domain *domain, uint8_t *vl)
{
int i;
int id;
fastlock_acquire(&domain->vl_lock);
for (i=0; i<BITMAP_SIZE; i++) {
id = (domain->vl_alloc + i) % BITMAP_SIZE;
if (bitmap_test(domain->vl_map, id) == 0) {
bitmap_set(domain->vl_map, id);
domain->vl_alloc = id + 1;
break;
}
}
fastlock_release(&domain->vl_lock);
if (i >= BITMAP_SIZE)
return -FI_ENOSPC;
*vl = (uint8_t)id;
return 0;
}
ssize_t sock_cq_sreadfrom(struct fid_cq *cq, void *buf, size_t count,
fi_addr_t *src_addr, const void *cond, int timeout)
{
int ret;
fi_addr_t addr;
int64_t threshold;
ssize_t i, bytes_read, num_read, cq_entry_len;
struct sock_cq *sock_cq;
sock_cq = container_of(cq, struct sock_cq, cq_fid);
cq_entry_len = sock_cq->cq_entry_size;
if (sock_cq->attr.wait_cond == FI_CQ_COND_THRESHOLD) {
threshold = MIN((int64_t)cond, count);
}else{
threshold = count;
}
fastlock_acquire(&sock_cq->lock);
bytes_read = rbfdsread(&sock_cq->cq_rbfd, buf,
cq_entry_len*threshold, timeout);
if(bytes_read == 0) {
ret = -FI_ETIMEDOUT;
goto out;
}
num_read = bytes_read/cq_entry_len;
for(i=0; i < num_read; i++) {
rbread(&sock_cq->addr_rb, &addr, sizeof(fi_addr_t));
if(src_addr)
src_addr[i] = addr;
}
ret = num_read;
out:
fastlock_release(&sock_cq->lock);
return ret;
}
static int sock_mr_close(struct fid *fid)
{
struct sock_domain *dom;
struct sock_mr *mr;
RbtIterator it;
RbtStatus res;
uint64_t mr_key;
mr = container_of(fid, struct sock_mr, mr_fid.fid);
dom = mr->domain;
mr_key = mr->key;
fastlock_acquire(&dom->lock);
it = rbtFind(dom->mr_heap, &mr_key);
if (!it || ((res = rbtErase(dom->mr_heap, it)) != RBT_STATUS_OK))
SOCK_LOG_ERROR("Invalid mr\n");
fastlock_release(&dom->lock);
atomic_dec(&dom->ref);
free(mr);
return 0;
}
static int util_wait_fd_try(struct util_wait *wait)
{
struct ofi_wait_fd_entry *fd_entry;
struct util_wait_fd *wait_fd;
void *context;
int ret;
wait_fd = container_of(wait, struct util_wait_fd, util_wait);
fd_signal_reset(&wait_fd->signal);
fastlock_acquire(&wait_fd->lock);
dlist_foreach_container(&wait_fd->fd_list, struct ofi_wait_fd_entry,
fd_entry, entry) {
ret = fd_entry->wait_try(fd_entry->arg);
if (ret != FI_SUCCESS) {
fastlock_release(&wait_fd->lock);
return ret;
}
}
fastlock_release(&wait_fd->lock);
ret = fi_poll(&wait->pollset->poll_fid, &context, 1);
return (ret > 0) ? -FI_EAGAIN : (ret == -FI_EAGAIN) ? FI_SUCCESS : ret;
}
int sock_cq_report_error(struct sock_cq *cq, struct sock_pe_entry *entry,
size_t olen, int err, int prov_errno, void *err_data,
size_t err_data_size)
{
int ret;
struct fi_cq_err_entry err_entry;
fastlock_acquire(&cq->lock);
if (ofi_rbavail(&cq->cqerr_rb) < sizeof(err_entry)) {
ret = -FI_ENOSPC;
goto out;
}
err_entry.err = err;
err_entry.olen = olen;
err_entry.err_data = err_data;
err_entry.err_data_size = err_data_size;
err_entry.len = entry->data_len;
err_entry.prov_errno = prov_errno;
err_entry.flags = entry->flags;
err_entry.data = entry->data;
err_entry.tag = entry->tag;
err_entry.op_context = (void *) (uintptr_t) entry->context;
if (entry->type == SOCK_PE_RX)
err_entry.buf = (void *) (uintptr_t) entry->pe.rx.rx_iov[0].iov.addr;
else
err_entry.buf = (void *) (uintptr_t) entry->pe.tx.tx_iov[0].src.iov.addr;
ofi_rbwrite(&cq->cqerr_rb, &err_entry, sizeof(err_entry));
ofi_rbcommit(&cq->cqerr_rb);
ret = 0;
ofi_rbfdsignal(&cq->cq_rbfd);
out:
fastlock_release(&cq->lock);
return ret;
}
void rxm_cq_progress(struct util_cq *util_cq)
{
ssize_t ret = 0;
struct rxm_cq *rxm_cq;
struct fi_cq_tagged_entry *comp;
rxm_cq = container_of(util_cq, struct rxm_cq, util_cq);
fastlock_acquire(&util_cq->cq_lock);
do {
if (cirque_isfull(util_cq->cirq))
goto out;
comp = cirque_tail(util_cq->cirq);
ret = rxm_msg_cq_read(util_cq, rxm_cq->msg_cq, comp);
if (ret < 0)
goto out;
cirque_commit(util_cq->cirq);
} while (ret > 0);
out:
fastlock_release(&util_cq->cq_lock);
}
static int fi_ibv_reap_comp(struct fi_ibv_msg_ep *ep)
{
struct fi_ibv_wce *wce = NULL;
int got_wc = 0;
int ret = 0;
fastlock_acquire(&ep->scq->lock);
while (ofi_atomic_get32(&ep->comp_pending) > 0) {
if (!wce) {
wce = util_buf_alloc(ep->scq->wce_pool);
if (!wce) {
fastlock_release(&ep->scq->lock);
return -FI_ENOMEM;
}
memset(wce, 0, sizeof(*wce));
}
ret = fi_ibv_poll_cq(ep->scq, &wce->wc);
if (ret < 0) {
VERBS_WARN(FI_LOG_EP_DATA,
"Failed to read completion for signaled send\n");
util_buf_release(ep->scq->wce_pool, wce);
fastlock_release(&ep->scq->lock);
return ret;
} else if (ret > 0) {
slist_insert_tail(&wce->entry, &ep->scq->wcq);
got_wc = 1;
wce = NULL;
}
}
if (wce)
util_buf_release(ep->scq->wce_pool, wce);
if (got_wc && ep->scq->channel)
ret = fi_ibv_cq_signal(&ep->scq->cq_fid);
fastlock_release(&ep->scq->lock);
return ret;
}
static int smr_ep_cancel_recv(struct smr_ep *ep, struct smr_queue *queue,
void *context)
{
struct smr_ep_entry *recv_entry;
struct dlist_entry *entry;
int ret = 0;
fastlock_acquire(&ep->util_ep.rx_cq->cq_lock);
entry = dlist_remove_first_match(&queue->list, smr_match_recv_ctx,
context);
if (entry) {
recv_entry = container_of(entry, struct smr_ep_entry, entry);
ret = ep->rx_comp(ep, (void *) recv_entry->context,
recv_entry->flags | FI_RECV, 0,
NULL, (void *) recv_entry->addr,
recv_entry->tag, 0, FI_ECANCELED);
freestack_push(ep->recv_fs, recv_entry);
ret = ret ? ret : 1;
}
fastlock_release(&ep->util_ep.rx_cq->cq_lock);
return ret;
}
static int tcpx_ep_msg_xfer_enable(struct tcpx_ep *ep)
{
int ret;
fastlock_acquire(&ep->lock);
if (ep->cm_state != TCPX_EP_CONNECTING) {
fastlock_release(&ep->lock);
return -FI_EINVAL;
}
ep->progress_func = tcpx_ep_progress;
ret = fi_fd_nonblock(ep->conn_fd);
if (ret)
goto err;
ret = tcpx_cq_wait_ep_add(ep);
if (ret)
goto err;
ep->cm_state = TCPX_EP_CONNECTED;
err:
fastlock_release(&ep->lock);
return ret;
}
ssize_t sock_cq_readerr(struct fid_cq *cq, struct fi_cq_err_entry *buf,
size_t len, uint64_t flags)
{
ssize_t num_read;
struct sock_cq *sock_cq;
sock_cq = container_of(cq, struct sock_cq, cq_fid);
if(len < sizeof(struct fi_cq_err_entry))
return -FI_ETOOSMALL;
num_read = 0;
fastlock_acquire(&sock_cq->lock);
while(rbused(&sock_cq->cqerr_rb) >= sizeof(struct fi_cq_err_entry)) {
rbread(&sock_cq->cqerr_rb,
(char*)buf +sizeof(struct fi_cq_err_entry) * num_read,
sizeof(struct fi_cq_err_entry));
num_read++;
}
fastlock_release(&sock_cq->lock);
return num_read;
}
static ssize_t _sock_cq_write(struct sock_cq *cq, fi_addr_t addr,
const void *buf, size_t len)
{
ssize_t ret;
fastlock_acquire(&cq->lock);
if(rbfdavail(&cq->cq_rbfd) < len) {
ret = -FI_ENOSPC;
goto out;
}
rbfdwrite(&cq->cq_rbfd, buf, len);
rbfdcommit(&cq->cq_rbfd);
ret = len;
rbwrite(&cq->addr_rb, &addr, sizeof(fi_addr_t));
rbcommit(&cq->addr_rb);
out:
fastlock_release(&cq->lock);
return ret;
}
static int rxd_cq_close(struct fid *fid)
{
int ret;
struct rxd_cq *cq;
cq = container_of(fid, struct rxd_cq, util_cq.cq_fid.fid);
fastlock_acquire(&cq->domain->lock);
dlist_remove(&cq->dom_entry);
fastlock_release(&cq->domain->lock);
fastlock_destroy(&cq->lock);
ret = fi_close(&cq->dg_cq->fid);
if (ret)
return ret;
ret = ofi_cq_cleanup(&cq->util_cq);
if (ret)
return ret;
util_buf_pool_destroy(cq->unexp_pool);
free(cq);
return 0;
}
int _gnix_buddy_alloc(gnix_buddy_alloc_handle_t *alloc_handle, void **ptr,
uint32_t len)
{
uint32_t block_size, i = 0;
GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
if (unlikely(!alloc_handle || !ptr || !len ||
len > alloc_handle->max)) {
GNIX_WARN(FI_LOG_EP_CTRL,
"Invalid parameter to _gnix_buddy_alloc.\n");
return -FI_EINVAL;
}
block_size = BLOCK_SIZE(len, MIN_BLOCK_SIZE);
i = (uint32_t) LIST_INDEX(block_size, MIN_BLOCK_SIZE);
fastlock_acquire(&alloc_handle->lock);
if (__gnix_buddy_find_block(alloc_handle, i, ptr)) {
fastlock_release(&alloc_handle->lock);
GNIX_WARN(FI_LOG_EP_CTRL,
"Could not allocate buddy block.\n");
return -FI_ENOMEM;
}
fastlock_release(&alloc_handle->lock);
_gnix_set_bit(&alloc_handle->bitmap,
__gnix_buddy_bitmap_index(*ptr, block_size,
alloc_handle->base,
alloc_handle->len,
MIN_BLOCK_SIZE));
return FI_SUCCESS;
}
static int process_srx_entry(struct tcpx_xfer_entry *rx_entry)
{
int ret;
ret = tcpx_recv_msg_data(rx_entry);
if (OFI_SOCK_TRY_SND_RCV_AGAIN(-ret))
return ret;
if (ret) {
FI_WARN(&tcpx_prov, FI_LOG_DOMAIN,
"msg recv Failed ret = %d\n", ret);
tcpx_ep_shutdown_report(rx_entry->ep,
&rx_entry->ep->util_ep.ep_fid.fid);
}
if ((ntohl(rx_entry->msg_hdr.hdr.flags) &
OFI_DELIVERY_COMPLETE) && !ret) {
if (tcpx_prepare_rx_entry_resp(rx_entry))
rx_entry->ep->cur_rx_proc_fn = tcpx_prepare_rx_entry_resp;
return FI_SUCCESS;
}
tcpx_cq_report_completion(rx_entry->ep->util_ep.rx_cq,
rx_entry, -ret);
/* release the shared entry */
if (rx_entry->ep->cur_rx_entry == rx_entry) {
rx_entry->ep->cur_rx_entry = NULL;
}
fastlock_acquire(&rx_entry->ep->srx_ctx->lock);
util_buf_release(rx_entry->ep->srx_ctx->buf_pool, rx_entry);
fastlock_release(&rx_entry->ep->srx_ctx->lock);
return FI_SUCCESS;
}
int sock_cq_progress(struct sock_cq *cq)
{
struct sock_tx_ctx *tx_ctx;
struct sock_rx_ctx *rx_ctx;
struct dlist_entry *entry;
if (cq->domain->progress_mode == FI_PROGRESS_AUTO)
return 0;
fastlock_acquire(&cq->list_lock);
for (entry = cq->tx_list.next; entry != &cq->tx_list;
entry = entry->next) {
tx_ctx = container_of(entry, struct sock_tx_ctx, cq_entry);
if (!tx_ctx->enabled)
continue;
if (tx_ctx->use_shared)
sock_pe_progress_tx_ctx(cq->domain->pe, tx_ctx->stx_ctx);
else
sock_pe_progress_tx_ctx(cq->domain->pe, tx_ctx);
}
for (entry = cq->rx_list.next; entry != &cq->rx_list;
entry = entry->next) {
rx_ctx = container_of(entry, struct sock_rx_ctx, cq_entry);
if (!rx_ctx->enabled)
continue;
if (rx_ctx->use_shared)
sock_pe_progress_rx_ctx(cq->domain->pe, rx_ctx->srx_ctx);
else
sock_pe_progress_rx_ctx(cq->domain->pe, rx_ctx);
}
fastlock_release(&cq->list_lock);
return 0;
}
