static ssize_t rxd_ep_cancel_recv(struct rxd_ep *ep, struct dlist_entry *list,
void *context)
{
struct dlist_entry *entry;
struct rxd_x_entry *rx_entry;
struct fi_cq_err_entry err_entry;
int ret = 0;
fastlock_acquire(&ep->util_ep.lock);
entry = dlist_find_first_match(list, &rxd_match_ctx, context);
if (!entry)
goto out;
rx_entry = container_of(entry, struct rxd_x_entry, entry);
memset(&err_entry, 0, sizeof(struct fi_cq_err_entry));
err_entry.op_context = rx_entry->cq_entry.op_context;
err_entry.flags = rx_entry->cq_entry.flags;
err_entry.err = FI_ECANCELED;
err_entry.prov_errno = 0;
ret = ofi_cq_write_error(&rxd_ep_rx_cq(ep)->util_cq, &err_entry);
if (ret) {
FI_WARN(&rxd_prov, FI_LOG_EP_CTRL, "could not write error entry\n");
goto out;
}
rx_entry->flags |= RXD_CANCELLED;
ret = 1;
out:
fastlock_release(&ep->util_ep.lock);
return ret;
}
int ofi_wait_fd_del(struct util_wait *wait, int fd)
{
int ret = 0;
struct ofi_wait_fd_entry *fd_entry;
struct dlist_entry *entry;
struct util_wait_fd *wait_fd = container_of(wait, struct util_wait_fd,
util_wait);
fastlock_acquire(&wait_fd->lock);
entry = dlist_find_first_match(&wait_fd->fd_list, ofi_wait_fd_match, &fd);
if (!entry) {
FI_INFO(wait->prov, FI_LOG_FABRIC,
"Given fd (%d) not found in wait list - %p\n",
fd, wait_fd);
ret = -FI_EINVAL;
goto out;
}
fd_entry = container_of(entry, struct ofi_wait_fd_entry, entry);
if (ofi_atomic_dec32(&fd_entry->ref))
goto out;
dlist_remove(&fd_entry->entry);
fi_epoll_del(wait_fd->epoll_fd, fd_entry->fd);
free(fd_entry);
out:
fastlock_release(&wait_fd->lock);
return ret;
}
void rxd_ep_check_unexp_tag_list(struct rxd_ep *ep, struct rxd_trecv_entry *trecv_entry)
{
struct dlist_entry *match;
struct rxd_rx_entry *rx_entry;
struct rxd_pkt_data_start *pkt_start;
FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "ep->num_unexp_msg: %d\n", ep->num_unexp_msg);
FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "ep->num_unexp_pkt: %d\n", ep->num_unexp_pkt);
match = dlist_find_first_match(&ep->unexp_tag_list, &rxd_match_unexp_tag,
(void *) trecv_entry);
if (match) {
dlist_remove(match);
dlist_remove(&trecv_entry->entry);
ep->num_unexp_msg--;
rx_entry = container_of(match, struct rxd_rx_entry, unexp_entry);
rx_entry->trecv = trecv_entry;
FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "progressing unexp tagged recv [%p]\n",
rx_entry->msg_id);
pkt_start = (struct rxd_pkt_data_start *) rx_entry->unexp_buf->buf;
rxd_ep_handle_data_msg(ep, rx_entry->peer_info, rx_entry, rx_entry->trecv->iov,
rx_entry->trecv->msg.iov_count, &pkt_start->ctrl,
pkt_start->data, rx_entry->unexp_buf);
rxd_ep_repost_buff(rx_entry->unexp_buf);
}
}
static void rxd_handle_connect_ack(struct rxd_ep *ep, struct ofi_ctrl_hdr *ctrl,
struct rxd_rx_buf *rx_buf)
{
struct rxd_peer *peer;
struct dlist_entry *match;
struct rxd_tx_entry *tx_entry;
FI_INFO(&rxd_prov, FI_LOG_EP_CTRL,
"connect ack- msg_id: %" PRIu64 ", segno: %d\n",
ctrl->msg_id, ctrl->seg_no);
match = dlist_find_first_match(&ep->tx_entry_list,
rxd_conn_msg_match, ctrl);
if (!match) {
FI_INFO(&rxd_prov, FI_LOG_EP_CTRL, "no matching connect\n");
goto out;
}
tx_entry = container_of(match, struct rxd_tx_entry, entry);
peer = rxd_ep_getpeer_info(ep, tx_entry->peer);
peer->state = CMAP_CONNECTED;
peer->conn_data = ctrl->conn_id;
dlist_remove(match);
rxd_tx_entry_done(ep, tx_entry);
out:
rxd_ep_repost_buff(rx_buf);
}
static void rxd_handle_dup_datastart(struct rxd_ep *ep, struct ofi_ctrl_hdr *ctrl,
struct rxd_rx_buf *rx_buf)
{
struct dlist_entry *item;
struct rxd_rx_entry *rx_entry;
struct rxd_peer *peer;
peer = rxd_ep_getpeer_info(ep, ctrl->conn_id);
item = dlist_find_first_match(&ep->rx_entry_list,
rxd_rx_entry_match, ctrl);
if (!item) {
/* for small (1-packet) messages we may have situation
* when receiver completed operation and destroyed
* rx_entry, but ack is lost (not delivered to sender).
* in this case just send ack with zero window to
* allow sender complete operation on sender side */
rxd_ep_reply_ack(ep, ctrl, ofi_ctrl_ack, 0, UINT64_MAX,
peer->conn_data, ctrl->conn_id);
return;
}
FI_INFO(&rxd_prov, FI_LOG_EP_CTRL,
"duplicate start-data: msg_id: %" PRIu64 ", seg_no: %d\n",
ctrl->msg_id, ctrl->seg_no);
rx_entry = container_of(item, struct rxd_rx_entry, entry);
rxd_ep_reply_ack(ep, ctrl, ofi_ctrl_ack, rx_entry->credits, rx_entry->key,
peer->conn_data, ctrl->conn_id);
return;
}
static ssize_t fi_ibv_rdm_tagged_ep_cancel(fid_t fid, void *ctx)
{
struct fi_context *context = (struct fi_context *)ctx;
struct fi_ibv_rdm_ep *ep_rdm =
container_of(fid, struct fi_ibv_rdm_ep, ep_fid);
int err = 1;
if (!ep_rdm->domain)
return -EBADF;
if (!context)
return -EINVAL;
if (context->internal[0] == NULL)
return 0;
struct fi_ibv_rdm_request *request = context->internal[0];
VERBS_DBG(FI_LOG_EP_DATA,
"ep_cancel, match %p, tag 0x%llx, len %d, ctx %p\n",
request, request->minfo.tag, request->len, request->context);
struct dlist_entry *found =
dlist_find_first_match(&fi_ibv_rdm_posted_queue,
fi_ibv_rdm_req_match, request);
if (found) {
assert(container_of(found, struct fi_ibv_rdm_request,
queue_entry) == request);
fi_ibv_rdm_remove_from_posted_queue(request, ep_rdm);
VERBS_DBG(FI_LOG_EP_DATA, "\t\t-> SUCCESS, post recv %d\n",
ep_rdm->posted_recvs);
err = 0;
} else {
struct util_fabric *fi_fabric_find(const char *name)
{
struct dlist_entry *item;
fastlock_acquire(&lock);
item = dlist_find_first_match(&fabric_list, fabric_match_name, name);
fastlock_release(&lock);
return item ? container_of(item, struct util_fabric, list_entry) : NULL;
}
static ssize_t fi_ibv_rdm_tagged_ep_cancel(fid_t fid, void *ctx)
{
struct fi_ibv_rdm_ep *fid_ep;
struct fi_context *context = (struct fi_context *)ctx;
int err = 1;
fid_ep = container_of(fid, struct fi_ibv_rdm_ep, ep_fid);
if (!fid_ep->domain)
return -EBADF;
if (!context)
return -EINVAL;
if (context->internal[0] == NULL)
return 0;
struct fi_ibv_rdm_tagged_request *request = context->internal[0];
VERBS_DBG(FI_LOG_EP_DATA,
"ep_cancel, match %p, tag 0x%llx, len %d, ctx %p\n",
request, (long long unsigned)request->tag,
request->len, request->context);
struct dlist_entry *found =
dlist_find_first_match(&fi_ibv_rdm_tagged_recv_posted_queue,
fi_ibv_rdm_tagged_match_requests, request);
if (found) {
assert(container_of(found, struct fi_ibv_rdm_tagged_request,
queue_entry) == request);
fi_ibv_rdm_tagged_remove_from_posted_queue(request, fid_ep);
assert(request->send_completions_wait == 0);
FI_IBV_RDM_TAGGED_DBG_REQUEST("to_pool: ", request,
FI_LOG_DEBUG);
fi_ibv_mem_pool_return(&request->mpe,
&fi_ibv_rdm_tagged_request_pool);
VERBS_DBG(FI_LOG_EP_DATA,
"\t\t-> SUCCESS, pend recv %d\n", fid_ep->pend_recv);
err = 0;
}
return err;
}
int rxd_handle_ack(struct rxd_ep *ep, struct ofi_ctrl_hdr *ctrl,
struct rxd_rx_buf *rx_buf)
{
int ret = 0;
uint64_t idx;
struct rxd_tx_entry *tx_entry;
struct dlist_entry *item;
struct rxd_pkt_meta *pkt;
rxd_ep_lock_if_required(ep);
FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "got ack: msg: %p - %d\n",
ctrl->msg_id, ctrl->seg_no);
idx = ctrl->msg_id & RXD_TX_IDX_BITS;
tx_entry = &ep->tx_entry_fs->buf[idx];
if (tx_entry->msg_id != ctrl->msg_id)
goto out;
item = dlist_find_first_match(&tx_entry->pkt_list, rxd_tx_pkt_match, ctrl);
if (!item)
goto out;
pkt = container_of(item, struct rxd_pkt_meta, entry);
switch (pkt->type) {
case RXD_PKT_STRT:
case RXD_PKT_DATA:
ret = rxd_tx_entry_progress(ep, tx_entry, ctrl);
break;
case RXD_PKT_LAST:
rxd_ep_free_acked_pkts(ep, tx_entry, ctrl->seg_no);
FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "reporting TX completion : %p\n", tx_entry);
if (tx_entry->op_type != RXD_TX_READ_REQ) {
rxd_cq_report_tx_comp(ep->tx_cq, tx_entry);
rxd_tx_entry_done(ep, tx_entry);
}
break;
default:
FI_WARN(&rxd_prov, FI_LOG_EP_CTRL, "invalid pkt type\n");
break;
}
out:
rxd_ep_repost_buff(rx_buf);
rxd_ep_unlock_if_required(ep);
return ret;
}
struct rxd_recv_entry *rxd_get_recv_entry(struct rxd_ep *ep, struct rxd_rx_entry *rx_entry)
{
struct dlist_entry *match;
struct rxd_recv_entry *recv_entry;
match = dlist_find_first_match(&ep->recv_list, &rxd_match_recv_entry,
(void *) rx_entry);
if (!match) {
/*todo: queue the pkt */
FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "no matching recv entry\n");
return NULL;
}
dlist_remove(match);
recv_entry = container_of(match, struct rxd_recv_entry, entry);
return recv_entry;
}
int ofi_wait_fd_add(struct util_wait *wait, int fd, uint32_t events,
ofi_wait_fd_try_func wait_try, void *arg, void *context)
{
struct ofi_wait_fd_entry *fd_entry;
struct dlist_entry *entry;
struct util_wait_fd *wait_fd = container_of(wait, struct util_wait_fd,
util_wait);
int ret = 0;
fastlock_acquire(&wait_fd->lock);
entry = dlist_find_first_match(&wait_fd->fd_list, ofi_wait_fd_match, &fd);
if (entry) {
FI_DBG(wait->prov, FI_LOG_EP_CTRL,
"Given fd (%d) already added to wait list - %p \n",
fd, wait_fd);
fd_entry = container_of(entry, struct ofi_wait_fd_entry, entry);
ofi_atomic_inc32(&fd_entry->ref);
goto out;
}
ret = fi_epoll_add(wait_fd->epoll_fd, fd, events, context);
if (ret) {
FI_WARN(wait->prov, FI_LOG_FABRIC, "Unable to add fd to epoll\n");
goto out;
}
fd_entry = calloc(1, sizeof *fd_entry);
if (!fd_entry) {
ret = -FI_ENOMEM;
fi_epoll_del(wait_fd->epoll_fd, fd);
goto out;
}
fd_entry->fd = fd;
fd_entry->wait_try = wait_try;
fd_entry->arg = arg;
ofi_atomic_initialize32(&fd_entry->ref, 1);
dlist_insert_tail(&fd_entry->entry, &wait_fd->fd_list);
out:
fastlock_release(&wait_fd->lock);
return ret;
}
struct rxd_trecv_entry *rxd_get_trecv_entry(struct rxd_ep *ep,
struct rxd_rx_entry *rx_entry)
{
struct dlist_entry *match;
struct rxd_trecv_entry *trecv_entry;
match = dlist_find_first_match(&ep->trecv_list, &rxd_match_trecv_entry,
(void *)rx_entry);
if (!match) {
/*todo: queue the pkt */
FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "no matching trecv entry, tag: %p\n",
rx_entry->op_hdr.tag);
return NULL;
}
FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "matched - tag: %p\n", rx_entry->op_hdr.tag);
dlist_remove(match);
trecv_entry = container_of(match, struct rxd_trecv_entry, entry);
trecv_entry->rx_entry = rx_entry;
return trecv_entry;
}
static void rxd_handle_dup_datastart(struct rxd_ep *ep, struct ofi_ctrl_hdr *ctrl,
struct rxd_rx_buf *rx_buf)
{
struct dlist_entry *item;
struct rxd_rx_entry *rx_entry;
struct rxd_peer *peer;
item = dlist_find_first_match(&ep->rx_entry_list,
rxd_rx_entry_match, ctrl);
if (!item)
return;
FI_INFO(&rxd_prov, FI_LOG_EP_CTRL,
"duplicate start-data: msg_id: %" PRIu64 ", seg_no: %d\n",
ctrl->msg_id, ctrl->seg_no);
rx_entry = container_of(item, struct rxd_rx_entry, entry);
peer = rxd_ep_getpeer_info(ep, ctrl->conn_id);
rxd_ep_reply_ack(ep, ctrl, ofi_ctrl_ack, rx_entry->window, rx_entry->key,
peer->conn_data, ctrl->conn_id);
return;
}
void rxd_handle_connect_ack(struct rxd_ep *ep, struct ofi_ctrl_hdr *ctrl,
struct rxd_rx_buf *rx_buf)
{
struct rxd_peer *peer;
struct dlist_entry *match;
struct rxd_tx_entry *tx_entry;
rxd_ep_lock_if_required(ep);
match = dlist_find_first_match(&ep->tx_entry_list, rxd_conn_msg_match, ctrl);
if (!match)
goto out;
tx_entry = container_of(match, struct rxd_tx_entry, entry);
peer = rxd_ep_getpeer_info(ep, tx_entry->peer);
peer->addr_published = 1;
peer->conn_data = ctrl->conn_id;
dlist_remove(match);
rxd_tx_entry_done(ep, tx_entry);
out:
rxd_ep_repost_buff(rx_buf);
rxd_ep_unlock_if_required(ep);
}
int fid_list_insert(struct dlist_entry *fid_list, fastlock_t *lock,
struct fid *fid)
{
int ret = 0;
struct dlist_entry *entry;
struct fid_list_entry *item;
fastlock_acquire(lock);
entry = dlist_find_first_match(fid_list, fi_fid_match, fid);
if (entry)
goto out;
item = calloc(1, sizeof(*item));
if (!item) {
ret = -FI_ENOMEM;
goto out;
}
item->fid = fid;
dlist_insert_tail(&item->entry, fid_list);
out:
fastlock_release(lock);
return ret;
}
int util_getinfo(const struct fi_provider *prov, uint32_t version,
const char *node, const char *service, uint64_t flags,
const struct fi_info *prov_info, struct fi_info *hints,
struct fi_info **info)
{
struct util_fabric *fabric;
struct util_domain *domain;
struct dlist_entry *item;
int ret, copy_dest;
FI_DBG(prov, FI_LOG_CORE, "checking info\n");
if ((flags & FI_SOURCE) && !node && !service) {
FI_INFO(prov, FI_LOG_CORE,
"FI_SOURCE set, but no node or service\n");
return -FI_EINVAL;
}
ret = fi_check_info(prov, prov_info, hints, FI_MATCH_EXACT);
if (ret)
return ret;
*info = fi_dupinfo(prov_info);
if (!*info) {
FI_INFO(prov, FI_LOG_CORE, "cannot copy info\n");
return -FI_ENOMEM;
}
ofi_alter_info(*info, hints);
fabric = fi_fabric_find((*info)->fabric_attr->name);
if (fabric) {
FI_DBG(prov, FI_LOG_CORE, "Found opened fabric\n");
(*info)->fabric_attr->fabric = &fabric->fabric_fid;
fastlock_acquire(&fabric->lock);
item = dlist_find_first_match(&fabric->domain_list,
util_find_domain, *info);
if (item) {
FI_DBG(prov, FI_LOG_CORE, "Found open domain\n");
domain = container_of(item, struct util_domain,
list_entry);
(*info)->domain_attr->domain = &domain->domain_fid;
}
fastlock_release(&fabric->lock);
}
if (flags & FI_SOURCE) {
ret = ofi_get_addr((*info)->addr_format, flags,
node, service, &(*info)->src_addr,
&(*info)->src_addrlen);
if (ret) {
FI_INFO(prov, FI_LOG_CORE,
"source address not available\n");
goto err;
}
copy_dest = (hints && hints->dest_addr);
} else {
if (node || service) {
copy_dest = 0;
ret = ofi_get_addr((*info)->addr_format, flags,
node, service, &(*info)->dest_addr,
&(*info)->dest_addrlen);
if (ret) {
FI_INFO(prov, FI_LOG_CORE,
"cannot resolve dest address\n");
goto err;
}
} else {
copy_dest = (hints && hints->dest_addr);
}
if (hints && hints->src_addr) {
(*info)->src_addr = mem_dup(hints->src_addr,
hints->src_addrlen);
if (!(*info)->src_addr) {
ret = -FI_ENOMEM;
goto err;
}
(*info)->src_addrlen = hints->src_addrlen;
}
}
if (copy_dest) {
(*info)->dest_addr = mem_dup(hints->dest_addr,
hints->dest_addrlen);
if (!(*info)->dest_addr) {
ret = -FI_ENOMEM;
goto err;
}
(*info)->dest_addrlen = hints->dest_addrlen;
}
if ((*info)->dest_addr && !(*info)->src_addr) {
ret = ofi_get_src_addr((*info)->addr_format, (*info)->dest_addr,
(*info)->dest_addrlen, &(*info)->src_addr,
&(*info)->src_addrlen);
if (ret) {
FI_INFO(prov, FI_LOG_CORE,
"cannot resolve source address\n");
}
}
return 0;
err:
fi_freeinfo(*info);
return ret;
}
