static void rxd_handle_connect_ack(struct rxd_ep *ep, struct ofi_ctrl_hdr *ctrl, struct rxd_rx_buf *rx_buf) { struct rxd_peer *peer; struct dlist_entry *match; struct rxd_tx_entry *tx_entry; FI_INFO(&rxd_prov, FI_LOG_EP_CTRL, "connect ack- msg_id: %" PRIu64 ", segno: %d\n", ctrl->msg_id, ctrl->seg_no); match = dlist_find_first_match(&ep->tx_entry_list, rxd_conn_msg_match, ctrl); if (!match) { FI_INFO(&rxd_prov, FI_LOG_EP_CTRL, "no matching connect\n"); goto out; } tx_entry = container_of(match, struct rxd_tx_entry, entry); peer = rxd_ep_getpeer_info(ep, tx_entry->peer); peer->state = CMAP_CONNECTED; peer->conn_data = ctrl->conn_id; dlist_remove(match); rxd_tx_entry_done(ep, tx_entry); out: rxd_ep_repost_buff(rx_buf); }
int rxd_handle_ack(struct rxd_ep *ep, struct ofi_ctrl_hdr *ctrl, struct rxd_rx_buf *rx_buf) { int ret = 0; uint64_t idx; struct rxd_tx_entry *tx_entry; struct dlist_entry *item; struct rxd_pkt_meta *pkt; rxd_ep_lock_if_required(ep); FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "got ack: msg: %p - %d\n", ctrl->msg_id, ctrl->seg_no); idx = ctrl->msg_id & RXD_TX_IDX_BITS; tx_entry = &ep->tx_entry_fs->buf[idx]; if (tx_entry->msg_id != ctrl->msg_id) goto out; item = dlist_find_first_match(&tx_entry->pkt_list, rxd_tx_pkt_match, ctrl); if (!item) goto out; pkt = container_of(item, struct rxd_pkt_meta, entry); switch (pkt->type) { case RXD_PKT_STRT: case RXD_PKT_DATA: ret = rxd_tx_entry_progress(ep, tx_entry, ctrl); break; case RXD_PKT_LAST: rxd_ep_free_acked_pkts(ep, tx_entry, ctrl->seg_no); FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "reporting TX completion : %p\n", tx_entry); if (tx_entry->op_type != RXD_TX_READ_REQ) { rxd_cq_report_tx_comp(ep->tx_cq, tx_entry); rxd_tx_entry_done(ep, tx_entry); } break; default: FI_WARN(&rxd_prov, FI_LOG_EP_CTRL, "invalid pkt type\n"); break; } out: rxd_ep_repost_buff(rx_buf); rxd_ep_unlock_if_required(ep); return ret; }
/* * Discarded transfers were discarded by the receiving side, so we abort * transferring the rest of the data. However, the completion is still * reported to the sender as successful. This ensures that short and long * messages are treated the same, since short messages would be entirely * buffered at the receiver, with no notification that the application later * discarded the message. */ static void rxd_handle_discard(struct rxd_ep *ep, struct ofi_ctrl_hdr *ctrl, struct rxd_rx_buf *rx_buf) { struct rxd_tx_entry *tx_entry; uint64_t idx; FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "discard- msg_id: %" PRIu64 ", segno: %d\n", ctrl->msg_id, ctrl->seg_no); idx = ctrl->msg_id & RXD_TX_IDX_BITS; tx_entry = &ep->tx_entry_fs->buf[idx]; if (tx_entry->msg_id == ctrl->msg_id) { rxd_cq_report_tx_comp(rxd_ep_tx_cq(ep), tx_entry); rxd_cntr_report_tx_comp(ep, tx_entry); rxd_tx_entry_done(ep, tx_entry); } rxd_ep_repost_buff(rx_buf); }
void rxd_handle_connect_ack(struct rxd_ep *ep, struct ofi_ctrl_hdr *ctrl, struct rxd_rx_buf *rx_buf) { struct rxd_peer *peer; struct dlist_entry *match; struct rxd_tx_entry *tx_entry; rxd_ep_lock_if_required(ep); match = dlist_find_first_match(&ep->tx_entry_list, rxd_conn_msg_match, ctrl); if (!match) goto out; tx_entry = container_of(match, struct rxd_tx_entry, entry); peer = rxd_ep_getpeer_info(ep, tx_entry->peer); peer->addr_published = 1; peer->conn_data = ctrl->conn_id; dlist_remove(match); rxd_tx_entry_done(ep, tx_entry); out: rxd_ep_repost_buff(rx_buf); rxd_ep_unlock_if_required(ep); }
void rxd_ep_handle_data_msg(struct rxd_ep *ep, struct rxd_peer *peer, struct rxd_rx_entry *rx_entry, struct iovec *iov, size_t iov_count, struct ofi_ctrl_hdr *ctrl, void *data, struct rxd_rx_buf *rx_buf) { uint64_t done; ep->credits++; done = ofi_copy_to_iov(iov, iov_count, rx_entry->done, data, ctrl->seg_size); rx_entry->done += done; rx_entry->window--; rx_entry->exp_seg_no++; if (done != ctrl->seg_size) { /* todo: generate truncation error */ /* inform peer */ FI_WARN(&rxd_prov, FI_LOG_EP_CTRL, "TODO: message truncated\n"); } if (rx_entry->window == 0) { rx_entry->window = rxd_get_window_sz(ep, rx_entry->op_hdr.size - rx_entry->done); rx_entry->last_win_seg += rx_entry->window; ep->credits -= rx_entry->window; FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "replying ack [%p] - %d\n", ctrl->msg_id, ctrl->seg_no); rxd_ep_reply_ack(ep, ctrl, ofi_ctrl_ack, rx_entry->window, rx_entry->key, peer->conn_data, ctrl->conn_id); } if (rx_entry->op_hdr.size != rx_entry->done) { if (rx_entry->window == 0) { dlist_init(&rx_entry->wait_entry); dlist_insert_tail(&rx_entry->wait_entry, &ep->wait_rx_list); FI_WARN(&rxd_prov, FI_LOG_EP_CTRL, "rx-entry %p - %d enqueued\n", ctrl->msg_id, ctrl->seg_no); } else { FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "rx_entry->op_hdr.size: %d, rx_entry->done: %d\n", rx_entry->op_hdr.size, rx_entry->done); } return; } FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "reporting RX completion event\n"); rxd_report_rx_comp(ep->rx_cq, rx_entry); switch(rx_entry->op_hdr.op) { case ofi_op_msg: freestack_push(ep->recv_fs, rx_entry->recv); break; case ofi_op_tagged: freestack_push(ep->trecv_fs, rx_entry->trecv); break; case ofi_op_read_rsp: rxd_cq_report_tx_comp(ep->tx_cq, rx_entry->read_rsp.tx_entry); rxd_tx_entry_done(ep, rx_entry->read_rsp.tx_entry); break; default: break; } rxd_rx_entry_release(ep, rx_entry); }
void rxd_ep_handle_data_msg(struct rxd_ep *ep, struct rxd_peer *peer, struct rxd_rx_entry *rx_entry, struct iovec *iov, size_t iov_count, struct ofi_ctrl_hdr *ctrl, void *data, struct rxd_rx_buf *rx_buf) { struct fi_cq_tagged_entry cq_entry = {0}; struct util_cntr *cntr = NULL; uint64_t done; struct rxd_cq *rxd_rx_cq = rxd_ep_rx_cq(ep); ep->credits++; done = ofi_copy_to_iov(iov, iov_count, rx_entry->done, data, ctrl->seg_size); rx_entry->done += done; rx_entry->credits--; rx_entry->exp_seg_no++; if (done != ctrl->seg_size) { /* todo: generate truncation error */ /* inform peer */ FI_WARN(&rxd_prov, FI_LOG_EP_CTRL, "TODO: message truncated\n"); } if (rx_entry->credits == 0) { rxd_set_rx_credits(ep, rx_entry); FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "replying ack [%p] - %d\n", ctrl->msg_id, ctrl->seg_no); rxd_ep_reply_ack(ep, ctrl, ofi_ctrl_ack, rx_entry->credits, rx_entry->key, peer->conn_data, ctrl->conn_id); } if (rx_entry->op_hdr.size != rx_entry->done) { if (rx_entry->credits == 0) { dlist_init(&rx_entry->wait_entry); dlist_insert_tail(&rx_entry->wait_entry, &ep->wait_rx_list); FI_WARN(&rxd_prov, FI_LOG_EP_CTRL, "rx-entry %p - %d enqueued\n", ctrl->msg_id, ctrl->seg_no); } else { FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "rx_entry->op_hdr.size: %d, rx_entry->done: %d\n", rx_entry->op_hdr.size, rx_entry->done); } return; } /* todo: handle FI_COMPLETION for RX CQ comp */ switch(rx_entry->op_hdr.op) { case ofi_op_msg: freestack_push(ep->recv_fs, rx_entry->recv); /* Handle cntr */ cntr = ep->util_ep.rx_cntr; /* Handle CQ comp */ cq_entry.flags |= FI_RECV; cq_entry.op_context = rx_entry->recv->msg.context; cq_entry.len = rx_entry->done; cq_entry.buf = rx_entry->recv->iov[0].iov_base; cq_entry.data = rx_entry->op_hdr.data; rxd_rx_cq->write_fn(rxd_rx_cq, &cq_entry); break; case ofi_op_tagged: freestack_push(ep->trecv_fs, rx_entry->trecv); /* Handle cntr */ cntr = ep->util_ep.rx_cntr; /* Handle CQ comp */ cq_entry.flags |= (FI_RECV | FI_TAGGED); cq_entry.op_context = rx_entry->trecv->msg.context; cq_entry.len = rx_entry->done; cq_entry.buf = rx_entry->trecv->iov[0].iov_base; cq_entry.data = rx_entry->op_hdr.data; cq_entry.tag = rx_entry->trecv->msg.tag;\ rxd_rx_cq->write_fn(rxd_rx_cq, &cq_entry); break; case ofi_op_atomic: /* Handle cntr */ cntr = ep->util_ep.rem_wr_cntr; /* Handle CQ comp */ cq_entry.flags |= FI_ATOMIC; rxd_rx_cq->write_fn(rxd_rx_cq, &cq_entry); break; case ofi_op_write: /* Handle cntr */ cntr = ep->util_ep.rem_wr_cntr; /* Handle CQ comp */ if (rx_entry->op_hdr.flags & OFI_REMOTE_CQ_DATA) { cq_entry.flags |= (FI_RMA | FI_REMOTE_WRITE); cq_entry.op_context = rx_entry->trecv->msg.context; cq_entry.len = rx_entry->done; cq_entry.buf = rx_entry->write.iov[0].iov_base; cq_entry.data = rx_entry->op_hdr.data; rxd_rx_cq->write_fn(rxd_rx_cq, &cq_entry); } break; case ofi_op_read_rsp: rxd_cq_report_tx_comp(rxd_ep_tx_cq(ep), rx_entry->read_rsp.tx_entry); rxd_cntr_report_tx_comp(ep, rx_entry->read_rsp.tx_entry); rxd_tx_entry_done(ep, rx_entry->read_rsp.tx_entry); break; default: FI_WARN(&rxd_prov, FI_LOG_EP_CTRL, "invalid op type: %d\n", rx_entry->op_hdr.op); break; } if (cntr) cntr->cntr_fid.ops->add(&cntr->cntr_fid, 1); rxd_rx_entry_free(ep, rx_entry); }