static ssize_t rxd_ep_cancel_recv(struct rxd_ep *ep, struct dlist_entry *list, void *context) { struct dlist_entry *entry; struct rxd_x_entry *rx_entry; struct fi_cq_err_entry err_entry; int ret = 0; fastlock_acquire(&ep->util_ep.lock); entry = dlist_find_first_match(list, &rxd_match_ctx, context); if (!entry) goto out; rx_entry = container_of(entry, struct rxd_x_entry, entry); memset(&err_entry, 0, sizeof(struct fi_cq_err_entry)); err_entry.op_context = rx_entry->cq_entry.op_context; err_entry.flags = rx_entry->cq_entry.flags; err_entry.err = FI_ECANCELED; err_entry.prov_errno = 0; ret = ofi_cq_write_error(&rxd_ep_rx_cq(ep)->util_cq, &err_entry); if (ret) { FI_WARN(&rxd_prov, FI_LOG_EP_CTRL, "could not write error entry\n"); goto out; } rx_entry->flags |= RXD_CANCELLED; ret = 1; out: fastlock_release(&ep->util_ep.lock); return ret; }
int ofi_wait_fd_del(struct util_wait *wait, int fd) { int ret = 0; struct ofi_wait_fd_entry *fd_entry; struct dlist_entry *entry; struct util_wait_fd *wait_fd = container_of(wait, struct util_wait_fd, util_wait); fastlock_acquire(&wait_fd->lock); entry = dlist_find_first_match(&wait_fd->fd_list, ofi_wait_fd_match, &fd); if (!entry) { FI_INFO(wait->prov, FI_LOG_FABRIC, "Given fd (%d) not found in wait list - %p\n", fd, wait_fd); ret = -FI_EINVAL; goto out; } fd_entry = container_of(entry, struct ofi_wait_fd_entry, entry); if (ofi_atomic_dec32(&fd_entry->ref)) goto out; dlist_remove(&fd_entry->entry); fi_epoll_del(wait_fd->epoll_fd, fd_entry->fd); free(fd_entry); out: fastlock_release(&wait_fd->lock); return ret; }
void rxd_ep_check_unexp_tag_list(struct rxd_ep *ep, struct rxd_trecv_entry *trecv_entry) { struct dlist_entry *match; struct rxd_rx_entry *rx_entry; struct rxd_pkt_data_start *pkt_start; FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "ep->num_unexp_msg: %d\n", ep->num_unexp_msg); FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "ep->num_unexp_pkt: %d\n", ep->num_unexp_pkt); match = dlist_find_first_match(&ep->unexp_tag_list, &rxd_match_unexp_tag, (void *) trecv_entry); if (match) { dlist_remove(match); dlist_remove(&trecv_entry->entry); ep->num_unexp_msg--; rx_entry = container_of(match, struct rxd_rx_entry, unexp_entry); rx_entry->trecv = trecv_entry; FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "progressing unexp tagged recv [%p]\n", rx_entry->msg_id); pkt_start = (struct rxd_pkt_data_start *) rx_entry->unexp_buf->buf; rxd_ep_handle_data_msg(ep, rx_entry->peer_info, rx_entry, rx_entry->trecv->iov, rx_entry->trecv->msg.iov_count, &pkt_start->ctrl, pkt_start->data, rx_entry->unexp_buf); rxd_ep_repost_buff(rx_entry->unexp_buf); } }
static void rxd_handle_connect_ack(struct rxd_ep *ep, struct ofi_ctrl_hdr *ctrl, struct rxd_rx_buf *rx_buf) { struct rxd_peer *peer; struct dlist_entry *match; struct rxd_tx_entry *tx_entry; FI_INFO(&rxd_prov, FI_LOG_EP_CTRL, "connect ack- msg_id: %" PRIu64 ", segno: %d\n", ctrl->msg_id, ctrl->seg_no); match = dlist_find_first_match(&ep->tx_entry_list, rxd_conn_msg_match, ctrl); if (!match) { FI_INFO(&rxd_prov, FI_LOG_EP_CTRL, "no matching connect\n"); goto out; } tx_entry = container_of(match, struct rxd_tx_entry, entry); peer = rxd_ep_getpeer_info(ep, tx_entry->peer); peer->state = CMAP_CONNECTED; peer->conn_data = ctrl->conn_id; dlist_remove(match); rxd_tx_entry_done(ep, tx_entry); out: rxd_ep_repost_buff(rx_buf); }
static void rxd_handle_dup_datastart(struct rxd_ep *ep, struct ofi_ctrl_hdr *ctrl, struct rxd_rx_buf *rx_buf) { struct dlist_entry *item; struct rxd_rx_entry *rx_entry; struct rxd_peer *peer; peer = rxd_ep_getpeer_info(ep, ctrl->conn_id); item = dlist_find_first_match(&ep->rx_entry_list, rxd_rx_entry_match, ctrl); if (!item) { /* for small (1-packet) messages we may have situation * when receiver completed operation and destroyed * rx_entry, but ack is lost (not delivered to sender). * in this case just send ack with zero window to * allow sender complete operation on sender side */ rxd_ep_reply_ack(ep, ctrl, ofi_ctrl_ack, 0, UINT64_MAX, peer->conn_data, ctrl->conn_id); return; } FI_INFO(&rxd_prov, FI_LOG_EP_CTRL, "duplicate start-data: msg_id: %" PRIu64 ", seg_no: %d\n", ctrl->msg_id, ctrl->seg_no); rx_entry = container_of(item, struct rxd_rx_entry, entry); rxd_ep_reply_ack(ep, ctrl, ofi_ctrl_ack, rx_entry->credits, rx_entry->key, peer->conn_data, ctrl->conn_id); return; }
static ssize_t fi_ibv_rdm_tagged_ep_cancel(fid_t fid, void *ctx) { struct fi_context *context = (struct fi_context *)ctx; struct fi_ibv_rdm_ep *ep_rdm = container_of(fid, struct fi_ibv_rdm_ep, ep_fid); int err = 1; if (!ep_rdm->domain) return -EBADF; if (!context) return -EINVAL; if (context->internal[0] == NULL) return 0; struct fi_ibv_rdm_request *request = context->internal[0]; VERBS_DBG(FI_LOG_EP_DATA, "ep_cancel, match %p, tag 0x%llx, len %d, ctx %p\n", request, request->minfo.tag, request->len, request->context); struct dlist_entry *found = dlist_find_first_match(&fi_ibv_rdm_posted_queue, fi_ibv_rdm_req_match, request); if (found) { assert(container_of(found, struct fi_ibv_rdm_request, queue_entry) == request); fi_ibv_rdm_remove_from_posted_queue(request, ep_rdm); VERBS_DBG(FI_LOG_EP_DATA, "\t\t-> SUCCESS, post recv %d\n", ep_rdm->posted_recvs); err = 0; } else {
struct util_fabric *fi_fabric_find(const char *name) { struct dlist_entry *item; fastlock_acquire(&lock); item = dlist_find_first_match(&fabric_list, fabric_match_name, name); fastlock_release(&lock); return item ? container_of(item, struct util_fabric, list_entry) : NULL; }
static ssize_t fi_ibv_rdm_tagged_ep_cancel(fid_t fid, void *ctx) { struct fi_ibv_rdm_ep *fid_ep; struct fi_context *context = (struct fi_context *)ctx; int err = 1; fid_ep = container_of(fid, struct fi_ibv_rdm_ep, ep_fid); if (!fid_ep->domain) return -EBADF; if (!context) return -EINVAL; if (context->internal[0] == NULL) return 0; struct fi_ibv_rdm_tagged_request *request = context->internal[0]; VERBS_DBG(FI_LOG_EP_DATA, "ep_cancel, match %p, tag 0x%llx, len %d, ctx %p\n", request, (long long unsigned)request->tag, request->len, request->context); struct dlist_entry *found = dlist_find_first_match(&fi_ibv_rdm_tagged_recv_posted_queue, fi_ibv_rdm_tagged_match_requests, request); if (found) { assert(container_of(found, struct fi_ibv_rdm_tagged_request, queue_entry) == request); fi_ibv_rdm_tagged_remove_from_posted_queue(request, fid_ep); assert(request->send_completions_wait == 0); FI_IBV_RDM_TAGGED_DBG_REQUEST("to_pool: ", request, FI_LOG_DEBUG); fi_ibv_mem_pool_return(&request->mpe, &fi_ibv_rdm_tagged_request_pool); VERBS_DBG(FI_LOG_EP_DATA, "\t\t-> SUCCESS, pend recv %d\n", fid_ep->pend_recv); err = 0; } return err; }
int rxd_handle_ack(struct rxd_ep *ep, struct ofi_ctrl_hdr *ctrl, struct rxd_rx_buf *rx_buf) { int ret = 0; uint64_t idx; struct rxd_tx_entry *tx_entry; struct dlist_entry *item; struct rxd_pkt_meta *pkt; rxd_ep_lock_if_required(ep); FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "got ack: msg: %p - %d\n", ctrl->msg_id, ctrl->seg_no); idx = ctrl->msg_id & RXD_TX_IDX_BITS; tx_entry = &ep->tx_entry_fs->buf[idx]; if (tx_entry->msg_id != ctrl->msg_id) goto out; item = dlist_find_first_match(&tx_entry->pkt_list, rxd_tx_pkt_match, ctrl); if (!item) goto out; pkt = container_of(item, struct rxd_pkt_meta, entry); switch (pkt->type) { case RXD_PKT_STRT: case RXD_PKT_DATA: ret = rxd_tx_entry_progress(ep, tx_entry, ctrl); break; case RXD_PKT_LAST: rxd_ep_free_acked_pkts(ep, tx_entry, ctrl->seg_no); FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "reporting TX completion : %p\n", tx_entry); if (tx_entry->op_type != RXD_TX_READ_REQ) { rxd_cq_report_tx_comp(ep->tx_cq, tx_entry); rxd_tx_entry_done(ep, tx_entry); } break; default: FI_WARN(&rxd_prov, FI_LOG_EP_CTRL, "invalid pkt type\n"); break; } out: rxd_ep_repost_buff(rx_buf); rxd_ep_unlock_if_required(ep); return ret; }
struct rxd_recv_entry *rxd_get_recv_entry(struct rxd_ep *ep, struct rxd_rx_entry *rx_entry) { struct dlist_entry *match; struct rxd_recv_entry *recv_entry; match = dlist_find_first_match(&ep->recv_list, &rxd_match_recv_entry, (void *) rx_entry); if (!match) { /*todo: queue the pkt */ FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "no matching recv entry\n"); return NULL; } dlist_remove(match); recv_entry = container_of(match, struct rxd_recv_entry, entry); return recv_entry; }
int ofi_wait_fd_add(struct util_wait *wait, int fd, uint32_t events, ofi_wait_fd_try_func wait_try, void *arg, void *context) { struct ofi_wait_fd_entry *fd_entry; struct dlist_entry *entry; struct util_wait_fd *wait_fd = container_of(wait, struct util_wait_fd, util_wait); int ret = 0; fastlock_acquire(&wait_fd->lock); entry = dlist_find_first_match(&wait_fd->fd_list, ofi_wait_fd_match, &fd); if (entry) { FI_DBG(wait->prov, FI_LOG_EP_CTRL, "Given fd (%d) already added to wait list - %p \n", fd, wait_fd); fd_entry = container_of(entry, struct ofi_wait_fd_entry, entry); ofi_atomic_inc32(&fd_entry->ref); goto out; } ret = fi_epoll_add(wait_fd->epoll_fd, fd, events, context); if (ret) { FI_WARN(wait->prov, FI_LOG_FABRIC, "Unable to add fd to epoll\n"); goto out; } fd_entry = calloc(1, sizeof *fd_entry); if (!fd_entry) { ret = -FI_ENOMEM; fi_epoll_del(wait_fd->epoll_fd, fd); goto out; } fd_entry->fd = fd; fd_entry->wait_try = wait_try; fd_entry->arg = arg; ofi_atomic_initialize32(&fd_entry->ref, 1); dlist_insert_tail(&fd_entry->entry, &wait_fd->fd_list); out: fastlock_release(&wait_fd->lock); return ret; }
struct rxd_trecv_entry *rxd_get_trecv_entry(struct rxd_ep *ep, struct rxd_rx_entry *rx_entry) { struct dlist_entry *match; struct rxd_trecv_entry *trecv_entry; match = dlist_find_first_match(&ep->trecv_list, &rxd_match_trecv_entry, (void *)rx_entry); if (!match) { /*todo: queue the pkt */ FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "no matching trecv entry, tag: %p\n", rx_entry->op_hdr.tag); return NULL; } FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "matched - tag: %p\n", rx_entry->op_hdr.tag); dlist_remove(match); trecv_entry = container_of(match, struct rxd_trecv_entry, entry); trecv_entry->rx_entry = rx_entry; return trecv_entry; }
static void rxd_handle_dup_datastart(struct rxd_ep *ep, struct ofi_ctrl_hdr *ctrl, struct rxd_rx_buf *rx_buf) { struct dlist_entry *item; struct rxd_rx_entry *rx_entry; struct rxd_peer *peer; item = dlist_find_first_match(&ep->rx_entry_list, rxd_rx_entry_match, ctrl); if (!item) return; FI_INFO(&rxd_prov, FI_LOG_EP_CTRL, "duplicate start-data: msg_id: %" PRIu64 ", seg_no: %d\n", ctrl->msg_id, ctrl->seg_no); rx_entry = container_of(item, struct rxd_rx_entry, entry); peer = rxd_ep_getpeer_info(ep, ctrl->conn_id); rxd_ep_reply_ack(ep, ctrl, ofi_ctrl_ack, rx_entry->window, rx_entry->key, peer->conn_data, ctrl->conn_id); return; }
void rxd_handle_connect_ack(struct rxd_ep *ep, struct ofi_ctrl_hdr *ctrl, struct rxd_rx_buf *rx_buf) { struct rxd_peer *peer; struct dlist_entry *match; struct rxd_tx_entry *tx_entry; rxd_ep_lock_if_required(ep); match = dlist_find_first_match(&ep->tx_entry_list, rxd_conn_msg_match, ctrl); if (!match) goto out; tx_entry = container_of(match, struct rxd_tx_entry, entry); peer = rxd_ep_getpeer_info(ep, tx_entry->peer); peer->addr_published = 1; peer->conn_data = ctrl->conn_id; dlist_remove(match); rxd_tx_entry_done(ep, tx_entry); out: rxd_ep_repost_buff(rx_buf); rxd_ep_unlock_if_required(ep); }
int fid_list_insert(struct dlist_entry *fid_list, fastlock_t *lock, struct fid *fid) { int ret = 0; struct dlist_entry *entry; struct fid_list_entry *item; fastlock_acquire(lock); entry = dlist_find_first_match(fid_list, fi_fid_match, fid); if (entry) goto out; item = calloc(1, sizeof(*item)); if (!item) { ret = -FI_ENOMEM; goto out; } item->fid = fid; dlist_insert_tail(&item->entry, fid_list); out: fastlock_release(lock); return ret; }
int util_getinfo(const struct fi_provider *prov, uint32_t version, const char *node, const char *service, uint64_t flags, const struct fi_info *prov_info, struct fi_info *hints, struct fi_info **info) { struct util_fabric *fabric; struct util_domain *domain; struct dlist_entry *item; int ret, copy_dest; FI_DBG(prov, FI_LOG_CORE, "checking info\n"); if ((flags & FI_SOURCE) && !node && !service) { FI_INFO(prov, FI_LOG_CORE, "FI_SOURCE set, but no node or service\n"); return -FI_EINVAL; } ret = fi_check_info(prov, prov_info, hints, FI_MATCH_EXACT); if (ret) return ret; *info = fi_dupinfo(prov_info); if (!*info) { FI_INFO(prov, FI_LOG_CORE, "cannot copy info\n"); return -FI_ENOMEM; } ofi_alter_info(*info, hints); fabric = fi_fabric_find((*info)->fabric_attr->name); if (fabric) { FI_DBG(prov, FI_LOG_CORE, "Found opened fabric\n"); (*info)->fabric_attr->fabric = &fabric->fabric_fid; fastlock_acquire(&fabric->lock); item = dlist_find_first_match(&fabric->domain_list, util_find_domain, *info); if (item) { FI_DBG(prov, FI_LOG_CORE, "Found open domain\n"); domain = container_of(item, struct util_domain, list_entry); (*info)->domain_attr->domain = &domain->domain_fid; } fastlock_release(&fabric->lock); } if (flags & FI_SOURCE) { ret = ofi_get_addr((*info)->addr_format, flags, node, service, &(*info)->src_addr, &(*info)->src_addrlen); if (ret) { FI_INFO(prov, FI_LOG_CORE, "source address not available\n"); goto err; } copy_dest = (hints && hints->dest_addr); } else { if (node || service) { copy_dest = 0; ret = ofi_get_addr((*info)->addr_format, flags, node, service, &(*info)->dest_addr, &(*info)->dest_addrlen); if (ret) { FI_INFO(prov, FI_LOG_CORE, "cannot resolve dest address\n"); goto err; } } else { copy_dest = (hints && hints->dest_addr); } if (hints && hints->src_addr) { (*info)->src_addr = mem_dup(hints->src_addr, hints->src_addrlen); if (!(*info)->src_addr) { ret = -FI_ENOMEM; goto err; } (*info)->src_addrlen = hints->src_addrlen; } } if (copy_dest) { (*info)->dest_addr = mem_dup(hints->dest_addr, hints->dest_addrlen); if (!(*info)->dest_addr) { ret = -FI_ENOMEM; goto err; } (*info)->dest_addrlen = hints->dest_addrlen; } if ((*info)->dest_addr && !(*info)->src_addr) { ret = ofi_get_src_addr((*info)->addr_format, (*info)->dest_addr, (*info)->dest_addrlen, &(*info)->src_addr, &(*info)->src_addrlen); if (ret) { FI_INFO(prov, FI_LOG_CORE, "cannot resolve source address\n"); } } return 0; err: fi_freeinfo(*info); return ret; }