void rxd_handle_data(struct rxd_ep *ep, struct rxd_peer *peer,
struct ofi_ctrl_hdr *ctrl, struct fi_cq_msg_entry *comp,
struct rxd_rx_buf *rx_buf)
{
int ret;
struct rxd_rx_entry *rx_entry;
struct rxd_tx_entry *tx_entry;
struct rxd_pkt_data *pkt_data = (struct rxd_pkt_data *) ctrl;
uint16_t win_sz;
uint64_t curr_stamp;
rxd_ep_lock_if_required(ep);
rx_entry = &ep->rx_entry_fs->buf[ctrl->rx_key];
ret = rxd_check_data_pkt_order(ep, peer, ctrl, rx_entry);
if (ret == RXD_PKT_ORDR_DUP) {
FI_DBG(&rxd_prov, FI_LOG_EP_CTRL,
"duplicate pkt: %d expected:%d, rx-key:%d, ctrl_msg_id: %p\n",
ctrl->seg_no, rx_entry->exp_seg_no, ctrl->rx_key, ctrl->msg_id);
win_sz = (rx_entry->msg_id == ctrl->msg_id &&
rx_entry->last_win_seg == ctrl->seg_no) ? rx_entry->window : 0;
rxd_ep_reply_ack(ep, ctrl, ofi_ctrl_ack, win_sz,
ctrl->rx_key, peer->conn_data, ctrl->conn_id);
goto repost;
} else if (ret == RXD_PKT_ORDR_UNEXP) {
if (!(comp->flags & RXD_UNEXP_ENTRY)) {
curr_stamp = fi_gettime_us();
if (rx_entry->nack_stamp == 0 ||
(curr_stamp > rx_entry->nack_stamp &&
curr_stamp - rx_entry->nack_stamp > RXD_RETRY_TIMEOUT)) {
FI_DBG(&rxd_prov, FI_LOG_EP_CTRL,
"unexpected pkt, sending NACK: %d\n", ctrl->seg_no);
rx_entry->nack_stamp = curr_stamp;
rxd_ep_reply_nack(ep, ctrl, rx_entry->exp_seg_no,
ctrl->rx_key, peer->conn_data,
ctrl->conn_id);
}
rxd_ep_enqueue_pkt(ep, ctrl, comp);
}
goto out;
}
rx_entry->nack_stamp = 0;
FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "expected pkt: %d\n", ctrl->seg_no);
switch (rx_entry->op_hdr.op) {
case ofi_op_msg:
rxd_ep_handle_data_msg(ep, peer, rx_entry, rx_entry->recv->iov,
rx_entry->recv->msg.iov_count, ctrl,
pkt_data->data, rx_buf);
break;
case ofi_op_tagged:
rxd_ep_handle_data_msg(ep, peer, rx_entry, rx_entry->trecv->iov,
rx_entry->trecv->msg.iov_count, ctrl,
pkt_data->data, rx_buf);
break;
case ofi_op_write:
rxd_ep_handle_data_msg(ep, peer, rx_entry, rx_entry->write.iov,
rx_entry->op_hdr.iov_count, ctrl,
pkt_data->data, rx_buf);
break;
case ofi_op_read_rsp:
tx_entry = rx_entry->read_rsp.tx_entry;
rxd_ep_handle_data_msg(ep, peer, rx_entry, tx_entry->read_req.dst_iov,
tx_entry->read_req.msg.iov_count, ctrl,
pkt_data->data, rx_buf);
break;
case ofi_op_atomic:
default:
FI_WARN(&rxd_prov, FI_LOG_EP_CTRL, "invalid op type\n");
}
repost:
if (comp->flags & RXD_UNEXP_ENTRY) {
rxd_release_unexp_entry(ep->rx_cq, comp);
ep->num_unexp_pkt--;
}
rxd_ep_repost_buff(rx_buf);
out:
rxd_ep_unlock_if_required(ep);
}
static void rxd_handle_data(struct rxd_ep *ep, struct rxd_peer *peer,
struct ofi_ctrl_hdr *ctrl, struct fi_cq_msg_entry *comp,
struct rxd_rx_buf *rx_buf)
{
struct rxd_rx_entry *rx_entry;
struct rxd_tx_entry *tx_entry;
struct rxd_pkt_data *pkt_data = (struct rxd_pkt_data *) ctrl;
uint16_t credits;
int ret;
FI_DBG(&rxd_prov, FI_LOG_EP_CTRL,
"data pkt- msg_id: %" PRIu64 ", segno: %d, buf: %p\n",
ctrl->msg_id, ctrl->seg_no, rx_buf);
rx_entry = &ep->rx_entry_fs->buf[ctrl->rx_key];
ret = rxd_check_data_pkt_order(ep, peer, ctrl, rx_entry);
if (ret) {
if (ret == -FI_EALREADY) {
FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "duplicate pkt: %d "
"expected:%d, rx-key:%d, ctrl_msg_id: %p\n",
ctrl->seg_no, rx_entry->exp_seg_no, ctrl->rx_key,
ctrl->msg_id);
credits = ((rx_entry->msg_id == ctrl->msg_id) &&
(rx_entry->last_win_seg == ctrl->seg_no)) ?
rx_entry->credits : 0;
rxd_ep_reply_ack(ep, ctrl, ofi_ctrl_ack, credits,
ctrl->rx_key, peer->conn_data,
ctrl->conn_id);
goto repost;
} else {
FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "invalid pkt: segno: %d "
"expected:%d, rx-key:%d, ctrl_msg_id: %ld, "
"rx_entry_msg_id: %ld\n",
ctrl->seg_no, rx_entry->exp_seg_no, ctrl->rx_key,
ctrl->msg_id, rx_entry->msg_id);
FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "invalid pkt: "
"credits: %d, last win: %d\n",
rx_entry->credits, rx_entry->last_win_seg);
credits = (rx_entry->msg_id == ctrl->msg_id) ?
rx_entry->last_win_seg - rx_entry->exp_seg_no : 0;
rxd_ep_reply_ack(ep, ctrl, ofi_ctrl_ack, credits,
ctrl->rx_key, peer->conn_data,
ctrl->conn_id);
goto repost;
}
}
rx_entry->nack_stamp = 0;
FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "expected pkt: %d\n", ctrl->seg_no);
switch (rx_entry->op_hdr.op) {
case ofi_op_msg:
rxd_ep_handle_data_msg(ep, peer, rx_entry, rx_entry->recv->iov,
rx_entry->recv->msg.iov_count, ctrl,
pkt_data->data, rx_buf);
break;
case ofi_op_tagged:
rxd_ep_handle_data_msg(ep, peer, rx_entry, rx_entry->trecv->iov,
rx_entry->trecv->msg.iov_count, ctrl,
pkt_data->data, rx_buf);
break;
case ofi_op_write:
rxd_ep_handle_data_msg(ep, peer, rx_entry, rx_entry->write.iov,
rx_entry->op_hdr.iov_count, ctrl,
pkt_data->data, rx_buf);
break;
case ofi_op_read_rsp:
tx_entry = rx_entry->read_rsp.tx_entry;
rxd_ep_handle_data_msg(ep, peer, rx_entry, tx_entry->read_req.dst_iov,
tx_entry->read_req.msg.iov_count, ctrl,
pkt_data->data, rx_buf);
break;
case ofi_op_atomic:
default:
FI_WARN(&rxd_prov, FI_LOG_EP_CTRL, "invalid op type\n");
}
repost:
rxd_ep_repost_buff(rx_buf);
}
