void rxd_ep_check_unexp_tag_list(struct rxd_ep *ep, struct rxd_trecv_entry *trecv_entry)
{
struct dlist_entry *match;
struct rxd_rx_entry *rx_entry;
struct rxd_pkt_data_start *pkt_start;
FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "ep->num_unexp_msg: %d\n", ep->num_unexp_msg);
FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "ep->num_unexp_pkt: %d\n", ep->num_unexp_pkt);
match = dlist_find_first_match(&ep->unexp_tag_list, &rxd_match_unexp_tag,
(void *) trecv_entry);
if (match) {
dlist_remove(match);
dlist_remove(&trecv_entry->entry);
ep->num_unexp_msg--;
rx_entry = container_of(match, struct rxd_rx_entry, unexp_entry);
rx_entry->trecv = trecv_entry;
FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "progressing unexp tagged recv [%p]\n",
rx_entry->msg_id);
pkt_start = (struct rxd_pkt_data_start *) rx_entry->unexp_buf->buf;
rxd_ep_handle_data_msg(ep, rx_entry->peer_info, rx_entry, rx_entry->trecv->iov,
rx_entry->trecv->msg.iov_count, &pkt_start->ctrl,
pkt_start->data, rx_entry->unexp_buf);
rxd_ep_repost_buff(rx_entry->unexp_buf);
}
}
static void rxd_handle_connect_ack(struct rxd_ep *ep, struct ofi_ctrl_hdr *ctrl,
struct rxd_rx_buf *rx_buf)
{
struct rxd_peer *peer;
struct dlist_entry *match;
struct rxd_tx_entry *tx_entry;
FI_INFO(&rxd_prov, FI_LOG_EP_CTRL,
"connect ack- msg_id: %" PRIu64 ", segno: %d\n",
ctrl->msg_id, ctrl->seg_no);
match = dlist_find_first_match(&ep->tx_entry_list,
rxd_conn_msg_match, ctrl);
if (!match) {
FI_INFO(&rxd_prov, FI_LOG_EP_CTRL, "no matching connect\n");
goto out;
}
tx_entry = container_of(match, struct rxd_tx_entry, entry);
peer = rxd_ep_getpeer_info(ep, tx_entry->peer);
peer->state = CMAP_CONNECTED;
peer->conn_data = ctrl->conn_id;
dlist_remove(match);
rxd_tx_entry_done(ep, tx_entry);
out:
rxd_ep_repost_buff(rx_buf);
}
int rxd_handle_ack(struct rxd_ep *ep, struct ofi_ctrl_hdr *ctrl,
struct rxd_rx_buf *rx_buf)
{
int ret = 0;
uint64_t idx;
struct rxd_tx_entry *tx_entry;
struct dlist_entry *item;
struct rxd_pkt_meta *pkt;
rxd_ep_lock_if_required(ep);
FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "got ack: msg: %p - %d\n",
ctrl->msg_id, ctrl->seg_no);
idx = ctrl->msg_id & RXD_TX_IDX_BITS;
tx_entry = &ep->tx_entry_fs->buf[idx];
if (tx_entry->msg_id != ctrl->msg_id)
goto out;
item = dlist_find_first_match(&tx_entry->pkt_list, rxd_tx_pkt_match, ctrl);
if (!item)
goto out;
pkt = container_of(item, struct rxd_pkt_meta, entry);
switch (pkt->type) {
case RXD_PKT_STRT:
case RXD_PKT_DATA:
ret = rxd_tx_entry_progress(ep, tx_entry, ctrl);
break;
case RXD_PKT_LAST:
rxd_ep_free_acked_pkts(ep, tx_entry, ctrl->seg_no);
FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "reporting TX completion : %p\n", tx_entry);
if (tx_entry->op_type != RXD_TX_READ_REQ) {
rxd_cq_report_tx_comp(ep->tx_cq, tx_entry);
rxd_tx_entry_done(ep, tx_entry);
}
break;
default:
FI_WARN(&rxd_prov, FI_LOG_EP_CTRL, "invalid pkt type\n");
break;
}
out:
rxd_ep_repost_buff(rx_buf);
rxd_ep_unlock_if_required(ep);
return ret;
}
void rxd_handle_recv_comp(struct rxd_ep *ep, struct fi_cq_msg_entry *comp)
{
struct ofi_ctrl_hdr *ctrl;
struct rxd_rx_buf *rx_buf;
struct rxd_peer *peer;
FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "got recv completion\n");
assert(rxd_reposted_bufs);
rxd_reposted_bufs--;
rx_buf = container_of(comp->op_context, struct rxd_rx_buf, context);
ctrl = (struct ofi_ctrl_hdr *) rx_buf->buf;
peer = rxd_ep_getpeer_info(ep, ctrl->conn_id);
if (ctrl->version != OFI_CTRL_VERSION) {
FI_WARN(&rxd_prov, FI_LOG_EP_CTRL, "ctrl version mismatch\n");
return;
}
switch (ctrl->type) {
case ofi_ctrl_connreq:
rxd_handle_conn_req(ep, ctrl, comp, rx_buf);
break;
case ofi_ctrl_ack:
rxd_handle_ack(ep, ctrl, rx_buf);
break;
case ofi_ctrl_discard:
rxd_handle_discard(ep, ctrl, rx_buf);
break;
case ofi_ctrl_connresp:
rxd_handle_connect_ack(ep, ctrl, rx_buf);
break;
case ofi_ctrl_start_data:
rxd_handle_start_data(ep, peer, ctrl, comp, rx_buf);
break;
case ofi_ctrl_data:
rxd_handle_data(ep, peer, ctrl, comp, rx_buf);
break;
default:
rxd_ep_repost_buff(rx_buf);
FI_WARN(&rxd_prov, FI_LOG_EP_CTRL,
"invalid ctrl type \n", ctrl->type);
}
rxd_check_waiting_rx(ep);
}
void rxd_handle_discard(struct rxd_ep *ep, struct ofi_ctrl_hdr *ctrl,
struct rxd_rx_buf *rx_buf)
{
uint64_t idx;
struct rxd_tx_entry *tx_entry;
rxd_ep_lock_if_required(ep);
FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "got Reject: msg: %p - %d\n",
ctrl->msg_id, ctrl->seg_no);
idx = ctrl->msg_id & RXD_TX_IDX_BITS;
tx_entry = &ep->tx_entry_fs->buf[idx];
if (tx_entry->msg_id != ctrl->msg_id)
goto out;
rxd_tx_entry_discard(ep, tx_entry);
out:
rxd_ep_repost_buff(rx_buf);
rxd_ep_unlock_if_required(ep);
}
/*
* Discarded transfers were discarded by the receiving side, so we abort
* transferring the rest of the data. However, the completion is still
* reported to the sender as successful. This ensures that short and long
* messages are treated the same, since short messages would be entirely
* buffered at the receiver, with no notification that the application later
* discarded the message.
*/
static void rxd_handle_discard(struct rxd_ep *ep, struct ofi_ctrl_hdr *ctrl,
struct rxd_rx_buf *rx_buf)
{
struct rxd_tx_entry *tx_entry;
uint64_t idx;
FI_DBG(&rxd_prov, FI_LOG_EP_CTRL,
"discard- msg_id: %" PRIu64 ", segno: %d\n",
ctrl->msg_id, ctrl->seg_no);
idx = ctrl->msg_id & RXD_TX_IDX_BITS;
tx_entry = &ep->tx_entry_fs->buf[idx];
if (tx_entry->msg_id == ctrl->msg_id) {
rxd_cq_report_tx_comp(rxd_ep_tx_cq(ep), tx_entry);
rxd_cntr_report_tx_comp(ep, tx_entry);
rxd_tx_entry_done(ep, tx_entry);
}
rxd_ep_repost_buff(rx_buf);
}
int rxd_handle_nack(struct rxd_ep *ep, struct ofi_ctrl_hdr *ctrl,
struct rxd_rx_buf *rx_buf)
{
int ret = 0;
uint64_t idx;
struct rxd_tx_entry *tx_entry;
rxd_ep_lock_if_required(ep);
FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "got NACK: msg: %p - %d\n",
ctrl->msg_id, ctrl->seg_no);
idx = ctrl->msg_id & RXD_TX_IDX_BITS;
tx_entry = &ep->tx_entry_fs->buf[idx];
if (tx_entry->msg_id != ctrl->msg_id)
goto out;
ret = rxd_tx_entry_progress(ep, tx_entry, ctrl);
out:
rxd_ep_repost_buff(rx_buf);
rxd_ep_unlock_if_required(ep);
return ret;
}
int rxd_handle_conn_req(struct rxd_ep *ep, struct ofi_ctrl_hdr *ctrl,
struct fi_cq_msg_entry *comp,
struct rxd_rx_buf *rx_buf)
{
int ret;
void *addr;
size_t addrlen;
uint64_t peer;
struct rxd_pkt_data *pkt_data;
struct rxd_peer *peer_info;
rxd_ep_lock_if_required(ep);
pkt_data = (struct rxd_pkt_data *) ctrl;
addr = pkt_data->data;
addrlen = ctrl->seg_size;
ret = rxd_av_dg_reverse_lookup(ep->av, ctrl->rx_key, addr, addrlen, &peer);
if (ret == -FI_ENODATA) {
ret = rxd_av_insert_dg_av(ep->av, addr);
assert(ret == 1);
ret = rxd_av_dg_reverse_lookup(ep->av, ctrl->rx_key, addr, addrlen, &peer);
assert(ret == 0);
}
peer_info = rxd_ep_getpeer_info(ep, peer);
if (!peer_info->addr_published) {
peer_info->addr_published = 1;
peer_info->conn_initiated = 1;
peer_info->conn_data = ctrl->conn_id;
peer_info->exp_msg_id++;
}
rxd_ep_reply_ack(ep, ctrl, ofi_ctrl_connresp, 0, ctrl->conn_id, peer, peer);
rxd_ep_repost_buff(rx_buf);
rxd_ep_unlock_if_required(ep);
return ret;
}
static void rxd_handle_conn_req(struct rxd_ep *ep, struct ofi_ctrl_hdr *ctrl,
struct fi_cq_msg_entry *comp,
struct rxd_rx_buf *rx_buf)
{
struct rxd_pkt_data *pkt_data;
struct rxd_peer *peer_info;
fi_addr_t dg_fiaddr;
void *addr;
int ret;
FI_INFO(&rxd_prov, FI_LOG_EP_DATA,
"conn req - rx_key: %" PRIu64 "\n", ctrl->rx_key);
pkt_data = (struct rxd_pkt_data *) ctrl;
addr = pkt_data->data;
if (ctrl->seg_size > RXD_MAX_DGRAM_ADDR) {
FI_WARN(&rxd_prov, FI_LOG_EP_DATA, "addr too large\n");
goto repost;
}
ret = rxd_av_insert_dg_addr(rxd_ep_av(ep), ctrl->rx_key, addr, &dg_fiaddr);
if (ret) {
FI_WARN(&rxd_prov, FI_LOG_EP_DATA, "failed to insert peer address\n");
goto repost;
}
peer_info = rxd_ep_getpeer_info(ep, dg_fiaddr);
if (peer_info->state != CMAP_CONNECTED) {
peer_info->state = CMAP_CONNECTED;
peer_info->conn_data = ctrl->conn_id;
peer_info->exp_msg_id++;
}
rxd_ep_reply_ack(ep, ctrl, ofi_ctrl_connresp, 0, ctrl->conn_id,
dg_fiaddr, dg_fiaddr);
repost:
rxd_ep_repost_buff(rx_buf);
}
void rxd_handle_connect_ack(struct rxd_ep *ep, struct ofi_ctrl_hdr *ctrl,
struct rxd_rx_buf *rx_buf)
{
struct rxd_peer *peer;
struct dlist_entry *match;
struct rxd_tx_entry *tx_entry;
rxd_ep_lock_if_required(ep);
match = dlist_find_first_match(&ep->tx_entry_list, rxd_conn_msg_match, ctrl);
if (!match)
goto out;
tx_entry = container_of(match, struct rxd_tx_entry, entry);
peer = rxd_ep_getpeer_info(ep, tx_entry->peer);
peer->addr_published = 1;
peer->conn_data = ctrl->conn_id;
dlist_remove(match);
rxd_tx_entry_done(ep, tx_entry);
out:
rxd_ep_repost_buff(rx_buf);
rxd_ep_unlock_if_required(ep);
}
static void rxd_handle_ack(struct rxd_ep *ep, struct ofi_ctrl_hdr *ctrl,
struct rxd_rx_buf *rx_buf)
{
struct rxd_tx_entry *tx_entry;
uint64_t idx;
FI_DBG(&rxd_prov, FI_LOG_EP_CTRL,
"ack- msg_id: %" PRIu64 ", segno: %d, segsz: %d, buf: %p\n",
ctrl->msg_id, ctrl->seg_no, ctrl->seg_size, rx_buf);
idx = ctrl->msg_id & RXD_TX_IDX_BITS;
tx_entry = &ep->tx_entry_fs->buf[idx];
if (tx_entry->msg_id != ctrl->msg_id)
goto out;
rxd_ep_free_acked_pkts(ep, tx_entry, ctrl->seg_no);
if ((tx_entry->bytes_sent == tx_entry->op_hdr.size) &&
dlist_empty(&tx_entry->pkt_list)) {
FI_DBG(&rxd_prov, FI_LOG_EP_CTRL,
"reporting TX completion : %p\n", tx_entry);
if (tx_entry->op_type != RXD_TX_READ_REQ) {
rxd_cq_report_tx_comp(rxd_ep_tx_cq(ep), tx_entry);
rxd_cntr_report_tx_comp(ep, tx_entry);
rxd_tx_entry_free(ep, tx_entry);
}
} else {
tx_entry->rx_key = ctrl->rx_key;
/* do not allow reduce window size (on duplicate acks) */
tx_entry->window = MAX(tx_entry->window, ctrl->seg_no + ctrl->seg_size);
FI_DBG(&rxd_prov, FI_LOG_EP_CTRL,
"ack- msg_id: %" PRIu64 ", window: %d\n",
ctrl->msg_id, tx_entry->window);
}
out:
rxd_ep_repost_buff(rx_buf);
}
void rxd_handle_data(struct rxd_ep *ep, struct rxd_peer *peer,
struct ofi_ctrl_hdr *ctrl, struct fi_cq_msg_entry *comp,
struct rxd_rx_buf *rx_buf)
{
int ret;
struct rxd_rx_entry *rx_entry;
struct rxd_tx_entry *tx_entry;
struct rxd_pkt_data *pkt_data = (struct rxd_pkt_data *) ctrl;
uint16_t win_sz;
uint64_t curr_stamp;
rxd_ep_lock_if_required(ep);
rx_entry = &ep->rx_entry_fs->buf[ctrl->rx_key];
ret = rxd_check_data_pkt_order(ep, peer, ctrl, rx_entry);
if (ret == RXD_PKT_ORDR_DUP) {
FI_DBG(&rxd_prov, FI_LOG_EP_CTRL,
"duplicate pkt: %d expected:%d, rx-key:%d, ctrl_msg_id: %p\n",
ctrl->seg_no, rx_entry->exp_seg_no, ctrl->rx_key, ctrl->msg_id);
win_sz = (rx_entry->msg_id == ctrl->msg_id &&
rx_entry->last_win_seg == ctrl->seg_no) ? rx_entry->window : 0;
rxd_ep_reply_ack(ep, ctrl, ofi_ctrl_ack, win_sz,
ctrl->rx_key, peer->conn_data, ctrl->conn_id);
goto repost;
} else if (ret == RXD_PKT_ORDR_UNEXP) {
if (!(comp->flags & RXD_UNEXP_ENTRY)) {
curr_stamp = fi_gettime_us();
if (rx_entry->nack_stamp == 0 ||
(curr_stamp > rx_entry->nack_stamp &&
curr_stamp - rx_entry->nack_stamp > RXD_RETRY_TIMEOUT)) {
FI_DBG(&rxd_prov, FI_LOG_EP_CTRL,
"unexpected pkt, sending NACK: %d\n", ctrl->seg_no);
rx_entry->nack_stamp = curr_stamp;
rxd_ep_reply_nack(ep, ctrl, rx_entry->exp_seg_no,
ctrl->rx_key, peer->conn_data,
ctrl->conn_id);
}
rxd_ep_enqueue_pkt(ep, ctrl, comp);
}
goto out;
}
rx_entry->nack_stamp = 0;
FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "expected pkt: %d\n", ctrl->seg_no);
switch (rx_entry->op_hdr.op) {
case ofi_op_msg:
rxd_ep_handle_data_msg(ep, peer, rx_entry, rx_entry->recv->iov,
rx_entry->recv->msg.iov_count, ctrl,
pkt_data->data, rx_buf);
break;
case ofi_op_tagged:
rxd_ep_handle_data_msg(ep, peer, rx_entry, rx_entry->trecv->iov,
rx_entry->trecv->msg.iov_count, ctrl,
pkt_data->data, rx_buf);
break;
case ofi_op_write:
rxd_ep_handle_data_msg(ep, peer, rx_entry, rx_entry->write.iov,
rx_entry->op_hdr.iov_count, ctrl,
pkt_data->data, rx_buf);
break;
case ofi_op_read_rsp:
tx_entry = rx_entry->read_rsp.tx_entry;
rxd_ep_handle_data_msg(ep, peer, rx_entry, tx_entry->read_req.dst_iov,
tx_entry->read_req.msg.iov_count, ctrl,
pkt_data->data, rx_buf);
break;
case ofi_op_atomic:
default:
FI_WARN(&rxd_prov, FI_LOG_EP_CTRL, "invalid op type\n");
}
repost:
if (comp->flags & RXD_UNEXP_ENTRY) {
rxd_release_unexp_entry(ep->rx_cq, comp);
ep->num_unexp_pkt--;
}
rxd_ep_repost_buff(rx_buf);
out:
rxd_ep_unlock_if_required(ep);
}
void rxd_handle_start_data(struct rxd_ep *ep, struct rxd_peer *peer,
struct ofi_ctrl_hdr *ctrl,
struct fi_cq_msg_entry *comp,
struct rxd_rx_buf *rx_buf)
{
int ret;
struct rxd_rx_entry *rx_entry;
struct rxd_pkt_data_start *pkt_start;
pkt_start = (struct rxd_pkt_data_start *) ctrl;
rxd_ep_lock_if_required(ep);
if (pkt_start->op.version != OFI_OP_VERSION) {
FI_WARN(&rxd_prov, FI_LOG_EP_CTRL, "op version mismatch\n");
goto repost;
}
ret = rxd_check_start_pkt_order(ep, peer, ctrl, comp);
if (ret == RXD_PKT_ORDR_DUP) {
FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "duplicate pkt: %d\n", ctrl->seg_no);
rxd_handle_dup_datastart(ep, ctrl, rx_buf);
goto repost;
} else if (ret == RXD_PKT_ORDR_UNEXP) {
FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "unexpected pkt: %d\n", ctrl->seg_no);
rxd_ep_enqueue_pkt(ep, ctrl, comp);
goto out;
}
rx_entry = rxd_get_rx_entry(ep);
if (!rx_entry)
goto repost;
rx_entry->peer_info = peer;
rx_entry->op_hdr = pkt_start->op;
rx_entry->exp_seg_no = 0;
rx_entry->msg_id = ctrl->msg_id;
rx_entry->done = 0;
rx_entry->peer = ctrl->conn_id;
rx_entry->source = (ep->caps & FI_DIRECTED_RECV) ?
rxd_av_get_fi_addr(ep->av, ctrl->conn_id) : FI_ADDR_UNSPEC;
rx_entry->window = 1;
rx_entry->last_win_seg = 1;
FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "Assign rx_entry :%d for %p\n",
rx_entry->key, rx_entry->msg_id);
ep->credits--;
ret = rxd_process_start_data(ep, rx_entry, peer, ctrl, comp, rx_buf);
if (ret == -FI_ENOMEM)
rxd_rx_entry_release(ep, rx_entry);
else if (ret == -FI_ENOENT) {
peer->exp_msg_id++;
/* reply ack, with win_sz = 0 */
FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "Sending wait-ACK [%p] - %d\n",
ctrl->msg_id, ctrl->seg_no);
goto out;
} else {
peer->exp_msg_id++;
}
repost:
if (comp->flags & RXD_UNEXP_ENTRY) {
rxd_release_unexp_entry(ep->rx_cq, comp);
ep->num_unexp_pkt--;
}
rxd_ep_repost_buff(rx_buf);
out:
rxd_ep_unlock_if_required(ep);
return;
}
static void rxd_handle_start_data(struct rxd_ep *ep, struct rxd_peer *peer,
struct ofi_ctrl_hdr *ctrl,
struct fi_cq_msg_entry *comp,
struct rxd_rx_buf *rx_buf)
{
struct rxd_rx_entry *rx_entry;
struct rxd_pkt_data_start *pkt_start;
int ret;
FI_DBG(&rxd_prov, FI_LOG_EP_CTRL,
"start data- msg_id: %" PRIu64 ", segno: %d, buf: %p\n",
ctrl->msg_id, ctrl->seg_no, rx_buf);
pkt_start = (struct rxd_pkt_data_start *) ctrl;
if (pkt_start->op.version != OFI_OP_VERSION) {
FI_WARN(&rxd_prov, FI_LOG_EP_CTRL, "op version mismatch\n");
goto repost;
}
ret = rxd_check_start_pkt_order(ep, peer, ctrl, comp);
if (ret) {
if (ret == -FI_EALREADY) {
FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "duplicate pkt: %d\n",
ctrl->seg_no);
rxd_handle_dup_datastart(ep, ctrl, rx_buf);
goto repost;
} else {
FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "unexpected pkt: %d\n",
ctrl->seg_no);
goto repost;
}
}
rx_entry = rxd_rx_entry_alloc(ep);
if (!rx_entry)
goto repost;
rx_entry->peer_info = peer;
rx_entry->op_hdr = pkt_start->op;
rx_entry->exp_seg_no = 0;
rx_entry->msg_id = ctrl->msg_id;
rx_entry->done = 0;
rx_entry->peer = ctrl->conn_id;
rx_entry->source = (ep->util_ep.caps & FI_DIRECTED_RECV) ?
rxd_av_fi_addr(rxd_ep_av(ep), ctrl->conn_id) : FI_ADDR_UNSPEC;
rx_entry->credits = 1;
rx_entry->last_win_seg = 1;
FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "Assign rx_entry :%d for %p\n",
rx_entry->key, rx_entry->msg_id);
ep->credits--;
ret = rxd_process_start_data(ep, rx_entry, peer, ctrl, comp, rx_buf);
if (ret == -FI_ENOMEM)
rxd_rx_entry_free(ep, rx_entry);
else if (ret == -FI_ENOENT) {
peer->exp_msg_id++;
/* reply ack, with no window = 0 */
FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "Sending wait-ACK [%p] - %d\n",
ctrl->msg_id, ctrl->seg_no);
goto out;
} else {
peer->exp_msg_id++;
}
repost:
rxd_ep_repost_buff(rx_buf);
out:
assert(rxd_reposted_bufs);
return;
}
static void rxd_handle_data(struct rxd_ep *ep, struct rxd_peer *peer,
struct ofi_ctrl_hdr *ctrl, struct fi_cq_msg_entry *comp,
struct rxd_rx_buf *rx_buf)
{
struct rxd_rx_entry *rx_entry;
struct rxd_tx_entry *tx_entry;
struct rxd_pkt_data *pkt_data = (struct rxd_pkt_data *) ctrl;
uint16_t credits;
int ret;
FI_DBG(&rxd_prov, FI_LOG_EP_CTRL,
"data pkt- msg_id: %" PRIu64 ", segno: %d, buf: %p\n",
ctrl->msg_id, ctrl->seg_no, rx_buf);
rx_entry = &ep->rx_entry_fs->buf[ctrl->rx_key];
ret = rxd_check_data_pkt_order(ep, peer, ctrl, rx_entry);
if (ret) {
if (ret == -FI_EALREADY) {
FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "duplicate pkt: %d "
"expected:%d, rx-key:%d, ctrl_msg_id: %p\n",
ctrl->seg_no, rx_entry->exp_seg_no, ctrl->rx_key,
ctrl->msg_id);
credits = ((rx_entry->msg_id == ctrl->msg_id) &&
(rx_entry->last_win_seg == ctrl->seg_no)) ?
rx_entry->credits : 0;
rxd_ep_reply_ack(ep, ctrl, ofi_ctrl_ack, credits,
ctrl->rx_key, peer->conn_data,
ctrl->conn_id);
goto repost;
} else {
FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "invalid pkt: segno: %d "
"expected:%d, rx-key:%d, ctrl_msg_id: %ld, "
"rx_entry_msg_id: %ld\n",
ctrl->seg_no, rx_entry->exp_seg_no, ctrl->rx_key,
ctrl->msg_id, rx_entry->msg_id);
FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "invalid pkt: "
"credits: %d, last win: %d\n",
rx_entry->credits, rx_entry->last_win_seg);
credits = (rx_entry->msg_id == ctrl->msg_id) ?
rx_entry->last_win_seg - rx_entry->exp_seg_no : 0;
rxd_ep_reply_ack(ep, ctrl, ofi_ctrl_ack, credits,
ctrl->rx_key, peer->conn_data,
ctrl->conn_id);
goto repost;
}
}
rx_entry->nack_stamp = 0;
FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "expected pkt: %d\n", ctrl->seg_no);
switch (rx_entry->op_hdr.op) {
case ofi_op_msg:
rxd_ep_handle_data_msg(ep, peer, rx_entry, rx_entry->recv->iov,
rx_entry->recv->msg.iov_count, ctrl,
pkt_data->data, rx_buf);
break;
case ofi_op_tagged:
rxd_ep_handle_data_msg(ep, peer, rx_entry, rx_entry->trecv->iov,
rx_entry->trecv->msg.iov_count, ctrl,
pkt_data->data, rx_buf);
break;
case ofi_op_write:
rxd_ep_handle_data_msg(ep, peer, rx_entry, rx_entry->write.iov,
rx_entry->op_hdr.iov_count, ctrl,
pkt_data->data, rx_buf);
break;
case ofi_op_read_rsp:
tx_entry = rx_entry->read_rsp.tx_entry;
rxd_ep_handle_data_msg(ep, peer, rx_entry, tx_entry->read_req.dst_iov,
tx_entry->read_req.msg.iov_count, ctrl,
pkt_data->data, rx_buf);
break;
case ofi_op_atomic:
default:
FI_WARN(&rxd_prov, FI_LOG_EP_CTRL, "invalid op type\n");
}
repost:
rxd_ep_repost_buff(rx_buf);
}
