static inline int rxm_finish_sar_segment_send(struct rxm_ep *rxm_ep, struct rxm_tx_sar_buf *tx_buf) { int ret = FI_SUCCESS; struct rxm_tx_sar_buf *first_tx_buf; switch (rxm_sar_get_seg_type(&tx_buf->pkt.ctrl_hdr)) { case RXM_SAR_SEG_FIRST: break; case RXM_SAR_SEG_MIDDLE: ofi_buf_free(tx_buf); break; case RXM_SAR_SEG_LAST: ret = rxm_cq_tx_comp_write(rxm_ep, ofi_tx_cq_flags(tx_buf->pkt.hdr.op), tx_buf->app_context, tx_buf->flags); assert(ofi_tx_cq_flags(tx_buf->pkt.hdr.op) & FI_SEND); ofi_ep_tx_cntr_inc(&rxm_ep->util_ep); first_tx_buf = ofi_bufpool_get_ibuf(rxm_ep-> buf_pools[RXM_BUF_POOL_TX_SAR].pool, tx_buf->pkt.ctrl_hdr.msg_id); ofi_buf_free(first_tx_buf); ofi_buf_free(tx_buf); break; } return ret; }
void rxd_ep_send_ack(struct rxd_ep *rxd_ep, fi_addr_t peer) { struct rxd_pkt_entry *pkt_entry; struct rxd_ack_pkt *ack; pkt_entry = rxd_get_tx_pkt(rxd_ep); if (!pkt_entry) { FI_WARN(&rxd_prov, FI_LOG_EP_CTRL, "Unable to send ack\n"); return; } ack = (struct rxd_ack_pkt *) (pkt_entry->pkt); pkt_entry->pkt_size = sizeof(*ack) + rxd_ep->tx_prefix_size; pkt_entry->peer = peer; ack->base_hdr.version = RXD_PROTOCOL_VERSION; ack->base_hdr.type = RXD_ACK; ack->base_hdr.peer = rxd_ep->peers[peer].peer_addr; ack->base_hdr.seq_no = rxd_ep->peers[peer].rx_seq_no; ack->ext_hdr.rx_id = rxd_ep->peers[peer].rx_window; rxd_ep->peers[peer].last_tx_ack = ack->base_hdr.seq_no; dlist_insert_tail(&pkt_entry->d_entry, &rxd_ep->ctrl_pkts); if (rxd_ep_send_pkt(rxd_ep, pkt_entry)) { dlist_remove(&pkt_entry->d_entry); ofi_buf_free(pkt_entry); } }
static void rxd_close_peer(struct rxd_ep *ep, struct rxd_peer *peer) { struct rxd_pkt_entry *pkt_entry; struct rxd_x_entry *x_entry; while (!dlist_empty(&peer->unacked)) { dlist_pop_front(&peer->unacked, struct rxd_pkt_entry, pkt_entry, d_entry); ofi_buf_free(pkt_entry); peer->unacked_cnt--; } while(!dlist_empty(&peer->tx_list)) { dlist_pop_front(&peer->tx_list, struct rxd_x_entry, x_entry, entry); rxd_tx_entry_free(ep, x_entry); } while(!dlist_empty(&peer->rx_list)) { dlist_pop_front(&peer->rx_list, struct rxd_x_entry, x_entry, entry); rxd_rx_entry_free(ep, x_entry); } while(!dlist_empty(&peer->rma_rx_list)) { dlist_pop_front(&peer->rma_rx_list, struct rxd_x_entry, x_entry, entry); rxd_tx_entry_free(ep, x_entry); } dlist_remove(&peer->entry); peer->active = 0; }
static ssize_t rxd_ep_send_rts(struct rxd_ep *rxd_ep, fi_addr_t rxd_addr) { struct rxd_pkt_entry *pkt_entry; struct rxd_rts_pkt *rts_pkt; ssize_t ret; size_t addrlen; pkt_entry = rxd_get_tx_pkt(rxd_ep); if (!pkt_entry) return -FI_ENOMEM; rts_pkt = (struct rxd_rts_pkt *) (pkt_entry->pkt); pkt_entry->pkt_size = sizeof(*rts_pkt) + rxd_ep->tx_prefix_size; pkt_entry->peer = rxd_addr; rts_pkt->base_hdr.version = RXD_PROTOCOL_VERSION; rts_pkt->base_hdr.type = RXD_RTS; rts_pkt->rts_addr = rxd_addr; addrlen = RXD_NAME_LENGTH; memset(rts_pkt->source, 0, RXD_NAME_LENGTH); ret = fi_getname(&rxd_ep->dg_ep->fid, (void *) rts_pkt->source, &addrlen); if (ret) { ofi_buf_free(pkt_entry); return ret; } rxd_ep_send_pkt(rxd_ep, pkt_entry); rxd_insert_unacked(rxd_ep, rxd_addr, pkt_entry); dlist_insert_tail(&rxd_ep->peers[rxd_addr].entry, &rxd_ep->rts_sent_list); return 0; }
static void rxd_peer_timeout(struct rxd_ep *rxd_ep, struct rxd_peer *peer) { struct fi_cq_err_entry err_entry; struct rxd_x_entry *tx_entry; struct rxd_pkt_entry *pkt_entry; int ret; while (!dlist_empty(&peer->tx_list)) { dlist_pop_front(&peer->tx_list, struct rxd_x_entry, tx_entry, entry); memset(&err_entry, 0, sizeof(struct fi_cq_err_entry)); rxd_tx_entry_free(rxd_ep, tx_entry); err_entry.op_context = tx_entry->cq_entry.op_context; err_entry.flags = tx_entry->cq_entry.flags; err_entry.err = FI_ECONNREFUSED; err_entry.prov_errno = 0; ret = ofi_cq_write_error(&rxd_ep_tx_cq(rxd_ep)->util_cq, &err_entry); if (ret) FI_WARN(&rxd_prov, FI_LOG_EP_CTRL, "could not write error entry\n"); } while (!dlist_empty(&peer->unacked)) { dlist_pop_front(&peer->unacked, struct rxd_pkt_entry, pkt_entry, d_entry); ofi_buf_free(pkt_entry); peer->unacked_cnt--; } dlist_remove(&peer->entry); }
static inline ssize_t rxm_ep_rma_common(struct rxm_ep *rxm_ep, const struct fi_msg_rma *msg, uint64_t flags, rxm_rma_msg_fn rma_msg, uint64_t comp_flags) { struct rxm_rma_buf *rma_buf; struct fi_msg_rma msg_rma = *msg; struct rxm_conn *rxm_conn; void *mr_desc[RXM_IOV_LIMIT] = { 0 }; int ret; assert(msg->rma_iov_count <= rxm_ep->rxm_info->tx_attr->rma_iov_limit); ret = rxm_ep_prepare_tx(rxm_ep, msg->addr, &rxm_conn); if (OFI_UNLIKELY(ret)) return ret; ofi_ep_lock_acquire(&rxm_ep->util_ep); rma_buf = rxm_rma_buf_alloc(rxm_ep); if (OFI_UNLIKELY(!rma_buf)) { FI_WARN(&rxm_prov, FI_LOG_EP_DATA, "Ran out of buffers from RMA buffer pool\n"); ret = -FI_ENOMEM; goto unlock; } rma_buf->app_context = msg->context; rma_buf->flags = flags; ret = rxm_ep_rma_reg_iov(rxm_ep, msg_rma.msg_iov, msg_rma.desc, mr_desc, msg_rma.iov_count, comp_flags & (FI_WRITE | FI_READ), rma_buf); if (OFI_UNLIKELY(ret)) goto release; msg_rma.desc = mr_desc; msg_rma.context = rma_buf; ret = rma_msg(rxm_conn->msg_ep, &msg_rma, flags); if (OFI_LIKELY(!ret)) goto unlock; if ((rxm_ep->msg_mr_local) && (!rxm_ep->rxm_mr_local)) rxm_ep_msg_mr_closev(rma_buf->mr.mr, rma_buf->mr.count); release: ofi_buf_free(rma_buf); unlock: ofi_ep_lock_release(&rxm_ep->util_ep); return ret; }
void tcpx_xfer_entry_release(struct tcpx_cq *tcpx_cq, struct tcpx_xfer_entry *xfer_entry) { if (xfer_entry->ep->cur_rx_entry == xfer_entry) { xfer_entry->ep->cur_rx_entry = NULL; } xfer_entry->hdr.base_hdr.flags = 0; xfer_entry->flags = 0; xfer_entry->context = 0; xfer_entry->rem_len = 0; tcpx_cq->util_cq.cq_fastlock_acquire(&tcpx_cq->util_cq.cq_lock); ofi_buf_free(xfer_entry); tcpx_cq->util_cq.cq_fastlock_release(&tcpx_cq->util_cq.cq_lock); }
static int rxm_rndv_tx_finish(struct rxm_ep *rxm_ep, struct rxm_tx_rndv_buf *tx_buf) { int ret; RXM_LOG_STATE_TX(FI_LOG_CQ, tx_buf, RXM_RNDV_FINISH); tx_buf->hdr.state = RXM_RNDV_FINISH; tx_buf->conn->rndv_tx_credits++; if (!rxm_ep->rxm_mr_local) rxm_ep_msg_mr_closev(tx_buf->mr, tx_buf->count); ret = rxm_cq_tx_comp_write(rxm_ep, ofi_tx_cq_flags(tx_buf->pkt.hdr.op), tx_buf->app_context, tx_buf->flags); assert(ofi_tx_cq_flags(tx_buf->pkt.hdr.op) & FI_SEND); ofi_ep_tx_cntr_inc(&rxm_ep->util_ep); ofi_buf_free(tx_buf); return ret; }
static inline int rxm_finish_rma(struct rxm_ep *rxm_ep, struct rxm_rma_buf *rma_buf, uint64_t comp_flags) { int ret = rxm_cq_tx_comp_write(rxm_ep, comp_flags, rma_buf->app_context, rma_buf->flags); assert(((comp_flags & FI_WRITE) && !(comp_flags & FI_READ)) || ((comp_flags & FI_READ) && !(comp_flags & FI_WRITE))); if (comp_flags & FI_WRITE) ofi_ep_wr_cntr_inc(&rxm_ep->util_ep); else ofi_ep_rd_cntr_inc(&rxm_ep->util_ep); if (!(rma_buf->flags & FI_INJECT) && !rxm_ep->rxm_mr_local && rxm_ep->msg_mr_local) { rxm_ep_msg_mr_closev(rma_buf->mr.mr, rma_buf->mr.count); } ofi_buf_free(rma_buf); return ret; }
int rxd_ep_post_buf(struct rxd_ep *ep) { struct rxd_pkt_entry *pkt_entry; ssize_t ret; pkt_entry = rxd_get_rx_pkt(ep); if (!pkt_entry) return -FI_ENOMEM; ret = fi_recv(ep->dg_ep, rxd_pkt_start(pkt_entry), rxd_ep_domain(ep)->max_mtu_sz, rxd_mr_desc(pkt_entry->mr, ep), FI_ADDR_UNSPEC, &pkt_entry->context); if (ret) { ofi_buf_free(pkt_entry); FI_WARN(&rxd_prov, FI_LOG_EP_CTRL, "failed to repost\n"); return ret; } ep->posted_bufs++; slist_insert_tail(&pkt_entry->s_entry, &ep->rx_pkt_list); return 0; }
static ssize_t rxm_ep_readv(struct fid_ep *ep_fid, const struct iovec *iov, void **desc, size_t count, fi_addr_t src_addr, uint64_t addr, uint64_t key, void *context) { struct rxm_ep *rxm_ep = container_of(ep_fid, struct rxm_ep, util_ep.ep_fid.fid); struct fi_rma_iov rma_iov = { .addr = addr, .len = ofi_total_iov_len(iov, count), .key = key, }; struct fi_msg_rma msg = { .msg_iov = iov, .desc = desc, .iov_count = count, .addr = src_addr, .rma_iov = &rma_iov, .rma_iov_count = 1, .context = context, .data = 0, }; return rxm_ep_rma_common(rxm_ep, &msg, rxm_ep_tx_flags(rxm_ep), fi_readmsg, FI_READ); } static ssize_t rxm_ep_read(struct fid_ep *ep_fid, void *buf, size_t len, void *desc, fi_addr_t src_addr, uint64_t addr, uint64_t key, void *context) { struct iovec iov = { .iov_base = (void*)buf, .iov_len = len, }; struct fi_rma_iov rma_iov = { .addr = addr, .len = len, .key = key, }; struct fi_msg_rma msg = { .msg_iov = &iov, .desc = &desc, .iov_count = 1, .addr = src_addr, .rma_iov = &rma_iov, .rma_iov_count = 1, .context = context, .data = 0, }; struct rxm_ep *rxm_ep = container_of(ep_fid, struct rxm_ep, util_ep.ep_fid.fid); return rxm_ep_rma_common(rxm_ep, &msg, rxm_ep_tx_flags(rxm_ep), fi_readmsg, FI_READ); } static inline void rxm_ep_format_rma_msg(struct rxm_rma_buf *rma_buf, const struct fi_msg_rma *orig_msg, struct iovec *rxm_iov, struct fi_msg_rma *rxm_msg) { rxm_msg->context = rma_buf; rxm_msg->addr = orig_msg->addr; rxm_msg->data = orig_msg->data; ofi_copy_from_iov(rma_buf->pkt.data, rma_buf->pkt.hdr.size, orig_msg->msg_iov, orig_msg->iov_count, 0); rxm_iov->iov_base = &rma_buf->pkt.data; rxm_iov->iov_len = rma_buf->pkt.hdr.size; rxm_msg->msg_iov = rxm_iov; rxm_msg->desc = &rma_buf->hdr.desc; rxm_msg->iov_count = 1; rxm_msg->rma_iov = orig_msg->rma_iov; rxm_msg->rma_iov_count = orig_msg->rma_iov_count; } static inline ssize_t rxm_ep_rma_emulate_inject_msg(struct rxm_ep *rxm_ep, struct rxm_conn *rxm_conn, size_t total_size, const struct fi_msg_rma *msg, uint64_t flags) { struct rxm_rma_buf *rma_buf; ssize_t ret; struct iovec rxm_msg_iov = { 0 }; struct fi_msg_rma rxm_rma_msg = { 0 }; assert(msg->rma_iov_count <= rxm_ep->rxm_info->tx_attr->rma_iov_limit); ofi_ep_lock_acquire(&rxm_ep->util_ep); rma_buf = rxm_rma_buf_alloc(rxm_ep); if (OFI_UNLIKELY(!rma_buf)) { FI_WARN(&rxm_prov, FI_LOG_EP_DATA, "Ran out of buffers from RMA buffer pool\n"); ret = -FI_ENOMEM; goto unlock; } rma_buf->pkt.hdr.size = total_size; rma_buf->app_context = msg->context; rma_buf->flags = flags; rxm_ep_format_rma_msg(rma_buf, msg, &rxm_msg_iov, &rxm_rma_msg); flags = (flags & ~FI_INJECT) | FI_COMPLETION; ret = fi_writemsg(rxm_conn->msg_ep, &rxm_rma_msg, flags); if (OFI_UNLIKELY(ret)) { if (ret == -FI_EAGAIN) rxm_ep_do_progress(&rxm_ep->util_ep); ofi_buf_free(rma_buf); } unlock: ofi_ep_lock_release(&rxm_ep->util_ep); return ret; } static inline ssize_t rxm_ep_rma_emulate_inject(struct rxm_ep *rxm_ep, struct rxm_conn *rxm_conn, const void *buf, size_t len, uint64_t data, fi_addr_t dest_addr, uint64_t addr, uint64_t key, uint64_t flags) { struct fi_rma_iov rma_iov = { .addr = addr, .len = len, .key = key, }; struct iovec iov = { .iov_base = (void*)buf, .iov_len = len, }; struct fi_msg_rma msg = { .msg_iov = &iov, .desc = NULL, .iov_count = 1, .addr = dest_addr, .rma_iov = &rma_iov, .rma_iov_count = 1, .context = NULL, .data = data, }; return rxm_ep_rma_emulate_inject_msg(rxm_ep, rxm_conn, len, &msg, flags); } static inline ssize_t rxm_ep_rma_inject_common(struct rxm_ep *rxm_ep, const struct fi_msg_rma *msg, uint64_t flags) { struct rxm_conn *rxm_conn; size_t total_size = ofi_total_iov_len(msg->msg_iov, msg->iov_count); ssize_t ret; assert(total_size <= rxm_ep->rxm_info->tx_attr->inject_size); ret = rxm_ep_prepare_tx(rxm_ep, msg->addr, &rxm_conn); if (OFI_UNLIKELY(ret)) return ret; if ((total_size <= rxm_ep->msg_info->tx_attr->inject_size) && !(flags & FI_COMPLETION) && (msg->iov_count == 1) && (msg->rma_iov_count == 1)) { if (flags & FI_REMOTE_CQ_DATA) { ret = fi_inject_writedata(rxm_conn->msg_ep, msg->msg_iov->iov_base, msg->msg_iov->iov_len, msg->data, msg->addr, msg->rma_iov->addr, msg->rma_iov->key); } else { ret = fi_inject_write(rxm_conn->msg_ep, msg->msg_iov->iov_base, msg->msg_iov->iov_len, msg->addr, msg->rma_iov->addr, msg->rma_iov->key); } if (OFI_LIKELY(!ret)) { ofi_ep_wr_cntr_inc(&rxm_ep->util_ep); } else { FI_DBG(&rxm_prov, FI_LOG_EP_DATA, "fi_inject_write* for MSG provider failed with ret - %" PRId64"\n", ret); if (OFI_LIKELY(ret == -FI_EAGAIN)) rxm_ep_progress(&rxm_ep->util_ep); } return ret; } else { return rxm_ep_rma_emulate_inject_msg(rxm_ep, rxm_conn, total_size, msg, flags); } } static inline ssize_t rxm_ep_generic_writemsg(struct fid_ep *ep_fid, const struct fi_msg_rma *msg, uint64_t flags) { struct rxm_ep *rxm_ep = container_of(ep_fid, struct rxm_ep, util_ep.ep_fid.fid); if (flags & FI_INJECT) return rxm_ep_rma_inject_common(rxm_ep, msg, flags); else return rxm_ep_rma_common(rxm_ep, msg, flags, fi_writemsg, FI_WRITE); } static inline ssize_t rxm_ep_writemsg(struct fid_ep *ep_fid, const struct fi_msg_rma *msg, uint64_t flags) { struct rxm_ep *rxm_ep = container_of(ep_fid, struct rxm_ep, util_ep.ep_fid.fid); return rxm_ep_generic_writemsg(ep_fid, msg, flags | rxm_ep->util_ep.tx_msg_flags); } static ssize_t rxm_ep_writev(struct fid_ep *ep_fid, const struct iovec *iov, void **desc, size_t count, fi_addr_t dest_addr, uint64_t addr, uint64_t key, void *context) { struct fi_rma_iov rma_iov = { .addr = addr, .len = ofi_total_iov_len(iov, count), .key = key, }; struct fi_msg_rma msg = { .msg_iov = iov, .desc = desc, .iov_count = count, .addr = dest_addr, .rma_iov = &rma_iov, .rma_iov_count = 1, .context = context, .data = 0, }; struct rxm_ep *rxm_ep = container_of(ep_fid, struct rxm_ep, util_ep.ep_fid.fid); return rxm_ep_generic_writemsg(ep_fid, &msg, rxm_ep_tx_flags(rxm_ep)); } static ssize_t rxm_ep_writedata(struct fid_ep *ep_fid, const void *buf, size_t len, void *desc, uint64_t data, fi_addr_t dest_addr, uint64_t addr, uint64_t key, void *context) { struct fi_rma_iov rma_iov = { .addr = addr, .len = len, .key = key, }; struct iovec iov = { .iov_base = (void*)buf, .iov_len = len, }; struct fi_msg_rma msg = { .msg_iov = &iov, .desc = &desc, .iov_count = 1, .addr = dest_addr, .rma_iov = &rma_iov, .rma_iov_count = 1, .context = context, .data = data, }; struct rxm_ep *rxm_ep = container_of(ep_fid, struct rxm_ep, util_ep.ep_fid.fid); return rxm_ep_generic_writemsg(ep_fid, &msg, rxm_ep_tx_flags(rxm_ep) | FI_REMOTE_CQ_DATA); } static ssize_t rxm_ep_write(struct fid_ep *ep_fid, const void *buf, size_t len, void *desc, fi_addr_t dest_addr, uint64_t addr, uint64_t key, void *context) { struct fi_rma_iov rma_iov = { .addr = addr, .len = len, .key = key, }; struct iovec iov = { .iov_base = (void*)buf, .iov_len = len, }; struct fi_msg_rma msg = { .msg_iov = &iov, .desc = &desc, .iov_count = 1, .addr = dest_addr, .rma_iov = &rma_iov, .rma_iov_count = 1, .context = context, .data = 0, }; struct rxm_ep *rxm_ep = container_of(ep_fid, struct rxm_ep, util_ep.ep_fid.fid); return rxm_ep_generic_writemsg(ep_fid, &msg, rxm_ep_tx_flags(rxm_ep)); } static ssize_t rxm_ep_inject_write(struct fid_ep *ep_fid, const void *buf, size_t len, fi_addr_t dest_addr, uint64_t addr, uint64_t key) { ssize_t ret; struct rxm_conn *rxm_conn; struct rxm_ep *rxm_ep = container_of(ep_fid, struct rxm_ep, util_ep.ep_fid.fid); ret = rxm_ep_prepare_tx(rxm_ep, dest_addr, &rxm_conn); if (OFI_UNLIKELY(ret)) return ret; if (len <= rxm_ep->msg_info->tx_attr->inject_size) { ret = fi_inject_write(rxm_conn->msg_ep, buf, len, dest_addr, addr, key); if (OFI_LIKELY(!ret)) { ofi_ep_wr_cntr_inc(&rxm_ep->util_ep); } else { FI_DBG(&rxm_prov, FI_LOG_EP_DATA, "fi_inject_write for MSG provider failed with ret - %" PRId64"\n", ret); if (OFI_LIKELY(ret == -FI_EAGAIN)) rxm_ep_progress(&rxm_ep->util_ep); } return ret; } else { return rxm_ep_rma_emulate_inject(rxm_ep, rxm_conn, buf, len, 0, dest_addr, addr, key, FI_INJECT); } } static ssize_t rxm_ep_inject_writedata(struct fid_ep *ep_fid, const void *buf, size_t len, uint64_t data, fi_addr_t dest_addr, uint64_t addr, uint64_t key) { ssize_t ret; struct rxm_conn *rxm_conn; struct rxm_ep *rxm_ep = container_of(ep_fid, struct rxm_ep, util_ep.ep_fid.fid); ret = rxm_ep_prepare_tx(rxm_ep, dest_addr, &rxm_conn); if (OFI_UNLIKELY(ret)) return ret; if (len <= rxm_ep->msg_info->tx_attr->inject_size) { ret = fi_inject_writedata(rxm_conn->msg_ep, buf, len, data, dest_addr, addr, key); if (OFI_LIKELY(!ret)) { ofi_ep_wr_cntr_inc(&rxm_ep->util_ep); } else { FI_DBG(&rxm_prov, FI_LOG_EP_DATA, "fi_inject_writedata for MSG provider failed with ret - %" PRId64"\n", ret); if (OFI_LIKELY(ret == -FI_EAGAIN)) rxm_ep_progress(&rxm_ep->util_ep); } return ret; } else { return rxm_ep_rma_emulate_inject(rxm_ep, rxm_conn, buf, len, data, dest_addr, addr, key, FI_REMOTE_CQ_DATA | FI_INJECT); } } struct fi_ops_rma rxm_ops_rma = { .size = sizeof (struct fi_ops_rma), .read = rxm_ep_read, .readv = rxm_ep_readv, .readmsg = rxm_ep_readmsg, .write = rxm_ep_write, .writev = rxm_ep_writev, .writemsg = rxm_ep_writemsg, .inject = rxm_ep_inject_write, .writedata = rxm_ep_writedata, .injectdata = rxm_ep_inject_writedata, };
static int rxd_ep_close(struct fid *fid) { int ret; struct rxd_ep *ep; struct rxd_pkt_entry *pkt_entry; struct slist_entry *entry; struct rxd_peer *peer; ep = container_of(fid, struct rxd_ep, util_ep.ep_fid.fid); dlist_foreach_container(&ep->active_peers, struct rxd_peer, peer, entry) rxd_close_peer(ep, peer); ret = fi_close(&ep->dg_ep->fid); if (ret) return ret; if (ep->dg_cq) { ret = fi_close(&ep->dg_cq->fid); if (ret) return ret; } while (!slist_empty(&ep->rx_pkt_list)) { entry = slist_remove_head(&ep->rx_pkt_list); pkt_entry = container_of(entry, struct rxd_pkt_entry, s_entry); ofi_buf_free(pkt_entry); } while (!dlist_empty(&ep->unexp_list)) { dlist_pop_front(&ep->unexp_list, struct rxd_pkt_entry, pkt_entry, d_entry); ofi_buf_free(pkt_entry); } while (!dlist_empty(&ep->unexp_tag_list)) { dlist_pop_front(&ep->unexp_tag_list, struct rxd_pkt_entry, pkt_entry, d_entry); ofi_buf_free(pkt_entry); } while (!dlist_empty(&ep->ctrl_pkts)) { dlist_pop_front(&ep->ctrl_pkts, struct rxd_pkt_entry, pkt_entry, d_entry); ofi_buf_free(pkt_entry); } if (ep->util_ep.tx_cq) { /* TODO: wait handling */ fid_list_remove(&ep->util_ep.tx_cq->ep_list, &ep->util_ep.tx_cq->ep_list_lock, &ep->util_ep.ep_fid.fid); } if (ep->util_ep.rx_cq) { if (ep->util_ep.rx_cq != ep->util_ep.tx_cq) { /* TODO: wait handling */ fid_list_remove(&ep->util_ep.rx_cq->ep_list, &ep->util_ep.rx_cq->ep_list_lock, &ep->util_ep.ep_fid.fid); } } rxd_ep_free_res(ep); ofi_endpoint_close(&ep->util_ep); free(ep); return 0; }
static ssize_t rxm_ep_atomic_common(struct rxm_ep *rxm_ep, struct rxm_conn *rxm_conn, const struct fi_msg_atomic *msg, const struct fi_ioc *comparev, void **compare_desc, size_t compare_iov_count, struct fi_ioc *resultv, void **result_desc, size_t result_iov_count, uint32_t op, uint64_t flags) { struct rxm_tx_atomic_buf *tx_buf; struct rxm_atomic_hdr *atomic_hdr; struct iovec buf_iov[RXM_IOV_LIMIT]; struct iovec cmp_iov[RXM_IOV_LIMIT]; size_t datatype_sz = ofi_datatype_size(msg->datatype); size_t buf_len = 0; size_t cmp_len = 0; size_t tot_len; ssize_t ret; assert(msg->iov_count <= RXM_IOV_LIMIT && msg->rma_iov_count <= RXM_IOV_LIMIT); if (flags & FI_REMOTE_CQ_DATA) { FI_WARN(&rxm_prov, FI_LOG_EP_DATA, "atomic with remote CQ data not supported\n"); return -FI_EINVAL; } if (msg->op != FI_ATOMIC_READ) { assert(msg->msg_iov); ofi_ioc_to_iov(msg->msg_iov, buf_iov, msg->iov_count, datatype_sz); buf_len = ofi_total_iov_len(buf_iov, msg->iov_count); } if (op == ofi_op_atomic_compare) { assert(comparev); ofi_ioc_to_iov(comparev, cmp_iov, compare_iov_count, datatype_sz); cmp_len = ofi_total_iov_len(cmp_iov, compare_iov_count); assert(buf_len == cmp_len); } tot_len = buf_len + cmp_len + sizeof(struct rxm_atomic_hdr) + sizeof(struct rxm_pkt); if (tot_len > rxm_eager_limit) { FI_WARN(&rxm_prov, FI_LOG_EP_DATA, "atomic data too large %zu\n", tot_len); return -FI_EINVAL; } ofi_ep_lock_acquire(&rxm_ep->util_ep); tx_buf = (struct rxm_tx_atomic_buf *) rxm_tx_buf_alloc(rxm_ep, RXM_BUF_POOL_TX_ATOMIC); if (OFI_UNLIKELY(!tx_buf)) { FI_WARN(&rxm_prov, FI_LOG_EP_DATA, "Ran out of buffers from Atomic buffer pool\n"); ret = -FI_EAGAIN; goto unlock; } rxm_ep_format_atomic_pkt_hdr(rxm_conn, tx_buf, tot_len, op, msg->datatype, msg->op, flags, msg->data, msg->rma_iov, msg->rma_iov_count); tx_buf->pkt.ctrl_hdr.msg_id = ofi_buf_index(tx_buf); tx_buf->app_context = msg->context; atomic_hdr = (struct rxm_atomic_hdr *) tx_buf->pkt.data; ofi_copy_from_iov(atomic_hdr->data, buf_len, buf_iov, msg->iov_count, 0); if (cmp_len) ofi_copy_from_iov(atomic_hdr->data + buf_len, cmp_len, cmp_iov, compare_iov_count, 0); tx_buf->result_iov_count = result_iov_count; if (resultv) ofi_ioc_to_iov(resultv, tx_buf->result_iov, result_iov_count, datatype_sz); ret = rxm_ep_send_atomic_req(rxm_ep, rxm_conn, tx_buf, tot_len); if (ret) ofi_buf_free(tx_buf); unlock: ofi_ep_lock_release(&rxm_ep->util_ep); return ret; }