static inline
ssize_t rxm_cq_copy_seg_data(struct rxm_rx_buf *rx_buf, int *done)
{
uint64_t done_len = ofi_copy_to_iov(rx_buf->recv_entry->rxm_iov.iov,
rx_buf->recv_entry->rxm_iov.count,
rx_buf->recv_entry->sar.total_recv_len,
rx_buf->pkt.data,
rx_buf->pkt.ctrl_hdr.seg_size);
rx_buf->recv_entry->sar.total_recv_len += done_len;
if ((rxm_sar_get_seg_type(&rx_buf->pkt.ctrl_hdr) == RXM_SAR_SEG_LAST) ||
(done_len != rx_buf->pkt.ctrl_hdr.seg_size)) {
dlist_remove(&rx_buf->recv_entry->sar.entry);
/* Mark rxm_recv_entry::msg_id as unknown for futher re-use */
rx_buf->recv_entry->sar.msg_id = RXM_SAR_RX_INIT;
done_len = rx_buf->recv_entry->sar.total_recv_len;
rx_buf->recv_entry->sar.total_recv_len = 0;
*done = 1;
return rxm_finish_recv(rx_buf, done_len);
} else {
if (rx_buf->recv_entry->sar.msg_id == RXM_SAR_RX_INIT) {
if (!rx_buf->conn) {
rx_buf->conn = rxm_key2conn(rx_buf->ep,
rx_buf->pkt.ctrl_hdr.conn_id);
}
rx_buf->recv_entry->sar.conn = rx_buf->conn;
rx_buf->recv_entry->sar.msg_id = rx_buf->pkt.ctrl_hdr.msg_id;
dlist_insert_tail(&rx_buf->recv_entry->sar.entry,
&rx_buf->conn->sar_rx_msg_list);
}
/* The RX buffer can be reposted for further re-use */
rx_buf->recv_entry = NULL;
rxm_rx_buf_release(rx_buf->ep, rx_buf);
*done = 0;
return FI_SUCCESS;
}
}
void rxd_ep_handle_data_msg(struct rxd_ep *ep, struct rxd_peer *peer,
struct rxd_rx_entry *rx_entry,
struct iovec *iov, size_t iov_count,
struct ofi_ctrl_hdr *ctrl, void *data,
struct rxd_rx_buf *rx_buf)
{
struct fi_cq_tagged_entry cq_entry = {0};
struct util_cntr *cntr = NULL;
uint64_t done;
struct rxd_cq *rxd_rx_cq = rxd_ep_rx_cq(ep);
ep->credits++;
done = ofi_copy_to_iov(iov, iov_count, rx_entry->done, data,
ctrl->seg_size);
rx_entry->done += done;
rx_entry->credits--;
rx_entry->exp_seg_no++;
if (done != ctrl->seg_size) {
/* todo: generate truncation error */
/* inform peer */
FI_WARN(&rxd_prov, FI_LOG_EP_CTRL, "TODO: message truncated\n");
}
if (rx_entry->credits == 0) {
rxd_set_rx_credits(ep, rx_entry);
FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "replying ack [%p] - %d\n",
ctrl->msg_id, ctrl->seg_no);
rxd_ep_reply_ack(ep, ctrl, ofi_ctrl_ack, rx_entry->credits,
rx_entry->key, peer->conn_data, ctrl->conn_id);
}
if (rx_entry->op_hdr.size != rx_entry->done) {
if (rx_entry->credits == 0) {
dlist_init(&rx_entry->wait_entry);
dlist_insert_tail(&rx_entry->wait_entry, &ep->wait_rx_list);
FI_WARN(&rxd_prov, FI_LOG_EP_CTRL, "rx-entry %p - %d enqueued\n",
ctrl->msg_id, ctrl->seg_no);
} else {
FI_DBG(&rxd_prov, FI_LOG_EP_CTRL,
"rx_entry->op_hdr.size: %d, rx_entry->done: %d\n",
rx_entry->op_hdr.size, rx_entry->done);
}
return;
}
/* todo: handle FI_COMPLETION for RX CQ comp */
switch(rx_entry->op_hdr.op) {
case ofi_op_msg:
freestack_push(ep->recv_fs, rx_entry->recv);
/* Handle cntr */
cntr = ep->util_ep.rx_cntr;
/* Handle CQ comp */
cq_entry.flags |= FI_RECV;
cq_entry.op_context = rx_entry->recv->msg.context;
cq_entry.len = rx_entry->done;
cq_entry.buf = rx_entry->recv->iov[0].iov_base;
cq_entry.data = rx_entry->op_hdr.data;
rxd_rx_cq->write_fn(rxd_rx_cq, &cq_entry);
break;
case ofi_op_tagged:
freestack_push(ep->trecv_fs, rx_entry->trecv);
/* Handle cntr */
cntr = ep->util_ep.rx_cntr;
/* Handle CQ comp */
cq_entry.flags |= (FI_RECV | FI_TAGGED);
cq_entry.op_context = rx_entry->trecv->msg.context;
cq_entry.len = rx_entry->done;
cq_entry.buf = rx_entry->trecv->iov[0].iov_base;
cq_entry.data = rx_entry->op_hdr.data;
cq_entry.tag = rx_entry->trecv->msg.tag;\
rxd_rx_cq->write_fn(rxd_rx_cq, &cq_entry);
break;
case ofi_op_atomic:
/* Handle cntr */
cntr = ep->util_ep.rem_wr_cntr;
/* Handle CQ comp */
cq_entry.flags |= FI_ATOMIC;
rxd_rx_cq->write_fn(rxd_rx_cq, &cq_entry);
break;
case ofi_op_write:
/* Handle cntr */
cntr = ep->util_ep.rem_wr_cntr;
/* Handle CQ comp */
if (rx_entry->op_hdr.flags & OFI_REMOTE_CQ_DATA) {
cq_entry.flags |= (FI_RMA | FI_REMOTE_WRITE);
cq_entry.op_context = rx_entry->trecv->msg.context;
cq_entry.len = rx_entry->done;
cq_entry.buf = rx_entry->write.iov[0].iov_base;
cq_entry.data = rx_entry->op_hdr.data;
rxd_rx_cq->write_fn(rxd_rx_cq, &cq_entry);
}
break;
case ofi_op_read_rsp:
rxd_cq_report_tx_comp(rxd_ep_tx_cq(ep), rx_entry->read_rsp.tx_entry);
rxd_cntr_report_tx_comp(ep, rx_entry->read_rsp.tx_entry);
rxd_tx_entry_done(ep, rx_entry->read_rsp.tx_entry);
break;
default:
FI_WARN(&rxd_prov, FI_LOG_EP_CTRL, "invalid op type: %d\n",
rx_entry->op_hdr.op);
break;
}
if (cntr)
cntr->cntr_fid.ops->add(&cntr->cntr_fid, 1);
rxd_rx_entry_free(ep, rx_entry);
}
void rxd_ep_handle_data_msg(struct rxd_ep *ep, struct rxd_peer *peer,
struct rxd_rx_entry *rx_entry,
struct iovec *iov, size_t iov_count,
struct ofi_ctrl_hdr *ctrl, void *data,
struct rxd_rx_buf *rx_buf)
{
uint64_t done;
ep->credits++;
done = ofi_copy_to_iov(iov, iov_count, rx_entry->done, data, ctrl->seg_size);
rx_entry->done += done;
rx_entry->window--;
rx_entry->exp_seg_no++;
if (done != ctrl->seg_size) {
/* todo: generate truncation error */
/* inform peer */
FI_WARN(&rxd_prov, FI_LOG_EP_CTRL, "TODO: message truncated\n");
}
if (rx_entry->window == 0) {
rx_entry->window = rxd_get_window_sz(ep, rx_entry->op_hdr.size - rx_entry->done);
rx_entry->last_win_seg += rx_entry->window;
ep->credits -= rx_entry->window;
FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "replying ack [%p] - %d\n",
ctrl->msg_id, ctrl->seg_no);
rxd_ep_reply_ack(ep, ctrl, ofi_ctrl_ack, rx_entry->window,
rx_entry->key, peer->conn_data, ctrl->conn_id);
}
if (rx_entry->op_hdr.size != rx_entry->done) {
if (rx_entry->window == 0) {
dlist_init(&rx_entry->wait_entry);
dlist_insert_tail(&rx_entry->wait_entry, &ep->wait_rx_list);
FI_WARN(&rxd_prov, FI_LOG_EP_CTRL, "rx-entry %p - %d enqueued\n",
ctrl->msg_id, ctrl->seg_no);
} else {
FI_DBG(&rxd_prov, FI_LOG_EP_CTRL,
"rx_entry->op_hdr.size: %d, rx_entry->done: %d\n",
rx_entry->op_hdr.size, rx_entry->done);
}
return;
}
FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "reporting RX completion event\n");
rxd_report_rx_comp(ep->rx_cq, rx_entry);
switch(rx_entry->op_hdr.op) {
case ofi_op_msg:
freestack_push(ep->recv_fs, rx_entry->recv);
break;
case ofi_op_tagged:
freestack_push(ep->trecv_fs, rx_entry->trecv);
break;
case ofi_op_read_rsp:
rxd_cq_report_tx_comp(ep->tx_cq, rx_entry->read_rsp.tx_entry);
rxd_tx_entry_done(ep, rx_entry->read_rsp.tx_entry);
break;
default:
break;
}
rxd_rx_entry_release(ep, rx_entry);
}
int mrail_cq_process_buf_recv(struct fi_cq_tagged_entry *comp,
struct mrail_recv *recv)
{
struct fi_recv_context *recv_ctx = comp->op_context;
struct fi_msg msg = {
.context = recv_ctx,
};
struct mrail_ep *mrail_ep;
struct mrail_pkt *mrail_pkt;
size_t size, len;
int ret, retv = 0;
if (comp->flags & FI_MORE) {
msg.msg_iov = recv->iov;
msg.iov_count = recv->count;
msg.addr = recv->addr;
recv_ctx->context = recv;
ret = fi_recvmsg(recv_ctx->ep, &msg, FI_CLAIM);
if (ret) {
FI_WARN(&mrail_prov, FI_LOG_CQ,
"Unable to claim buffered recv\n");
assert(0);
// TODO write cq error entry
}
return ret;
}
mrail_ep = recv_ctx->ep->fid.context;
mrail_pkt = (struct mrail_pkt *)comp->buf;
len = comp->len - sizeof(*mrail_pkt);
size = ofi_copy_to_iov(&recv->iov[1], recv->count - 1, 0,
mrail_pkt->data, len);
if (size < len) {
FI_WARN(&mrail_prov, FI_LOG_CQ, "Message truncated recv buf "
"size: %zu message length: %zu\n", size, len);
ret = ofi_cq_write_error_trunc(
mrail_ep->util_ep.rx_cq, recv->context,
recv->comp_flags | (comp->flags & FI_REMOTE_CQ_DATA),
0, NULL, comp->data, mrail_pkt->hdr.tag, comp->len - size);
if (ret) {
FI_WARN(&mrail_prov, FI_LOG_CQ,
"Unable to write truncation error to util cq\n");
retv = ret;
}
goto out;
}
ret = mrail_cq_write_recv_comp(mrail_ep, &mrail_pkt->hdr, comp, recv);
if (ret)
retv = ret;
out:
ret = fi_recvmsg(recv_ctx->ep, &msg, FI_DISCARD);
if (ret) {
FI_WARN(&mrail_prov, FI_LOG_CQ,
"Unable to discard buffered recv\n");
retv = ret;
}
mrail_push_recv(recv);
return retv;
}
