ssize_t rxd_ep_readmsg(struct fid_ep *ep, const struct fi_msg_rma *msg,
uint64_t flags)
{
struct rxd_ep *rxd_ep;
struct rxd_peer *peer;
struct rxd_tx_entry *tx_entry;
uint64_t peer_addr;
ssize_t ret;
rxd_ep = container_of(ep, struct rxd_ep, util_ep.ep_fid);
peer_addr = rxd_av_dg_addr(rxd_ep_av(rxd_ep), msg->addr);
peer = rxd_ep_getpeer_info(rxd_ep, peer_addr);
fastlock_acquire(&rxd_ep->lock);
if (peer->state != CMAP_CONNECTED) {
ret = rxd_ep_connect(rxd_ep, peer, peer_addr);
fastlock_release(&rxd_ep->lock);
if (ret == -FI_EALREADY) {
rxd_ep->util_ep.progress(&rxd_ep->util_ep);
ret = -FI_EAGAIN;
}
return ret ? ret : -FI_EAGAIN;
}
tx_entry = rxd_tx_entry_alloc(rxd_ep, peer, peer_addr, flags,
RXD_TX_READ_REQ);
if (!tx_entry) {
ret = -FI_EAGAIN;
goto out;
}
tx_entry->read_req.msg = *msg;
memcpy(&tx_entry->read_req.dst_iov[0], msg->msg_iov,
sizeof(*msg->msg_iov)* msg->iov_count);
memcpy(&tx_entry->read_req.src_iov[0], msg->rma_iov,
sizeof(*msg->rma_iov) * msg->rma_iov_count);
ret = rxd_ep_start_xfer(rxd_ep, peer, ofi_op_read_req, tx_entry);
if (ret)
rxd_tx_entry_free(rxd_ep, tx_entry);
out:
fastlock_release(&rxd_ep->lock);
return ret;
}
static ssize_t rxd_generic_write_inject(struct rxd_ep *rxd_ep,
const struct iovec *iov, size_t iov_count,
const struct fi_rma_iov *rma_iov, size_t rma_count,
fi_addr_t addr, void *context, uint32_t op, uint64_t data,
uint32_t rxd_flags)
{
struct rxd_x_entry *tx_entry;
fi_addr_t rxd_addr;
ssize_t ret = -FI_EAGAIN;
assert(iov_count <= RXD_IOV_LIMIT && rma_count <= RXD_IOV_LIMIT);
assert(ofi_total_iov_len(iov, iov_count) <= rxd_ep_domain(rxd_ep)->max_inline_rma);
fastlock_acquire(&rxd_ep->util_ep.lock);
fastlock_acquire(&rxd_ep->util_ep.tx_cq->cq_lock);
if (ofi_cirque_isfull(rxd_ep->util_ep.tx_cq->cirq))
goto out;
rxd_addr = rxd_ep_av(rxd_ep)->fi_addr_table[addr];
ret = rxd_send_rts_if_needed(rxd_ep, rxd_addr);
if (ret)
goto out;
tx_entry = rxd_tx_entry_init(rxd_ep, iov, iov_count, NULL, 0, rma_count, data,
0, context, rxd_addr, op, rxd_flags);
if (!tx_entry)
goto out;
ret = rxd_ep_send_op(rxd_ep, tx_entry, rma_iov, rma_count, NULL, 0, 0, 0);
if (ret) {
rxd_tx_entry_free(rxd_ep, tx_entry);
goto out;
}
if (tx_entry->op == RXD_READ_REQ)
goto out;
ret = 0;
out:
fastlock_release(&rxd_ep->util_ep.tx_cq->cq_lock);
fastlock_release(&rxd_ep->util_ep.lock);
return ret;
}
ssize_t rxd_generic_rma(struct rxd_ep *rxd_ep, const struct iovec *iov,
size_t iov_count, const struct fi_rma_iov *rma_iov, size_t rma_count,
void **desc, fi_addr_t addr, void *context, uint32_t op, uint64_t data,
uint32_t rxd_flags)
{
struct rxd_x_entry *tx_entry;
fi_addr_t rxd_addr;
ssize_t ret = -FI_EAGAIN;
if (rxd_flags & RXD_INJECT)
return rxd_generic_write_inject(rxd_ep, iov, iov_count, rma_iov,
rma_count, addr, context, op,
data, rxd_flags);
assert(iov_count <= RXD_IOV_LIMIT && rma_count <= RXD_IOV_LIMIT);
fastlock_acquire(&rxd_ep->util_ep.lock);
fastlock_acquire(&rxd_ep->util_ep.tx_cq->cq_lock);
if (ofi_cirque_isfull(rxd_ep->util_ep.tx_cq->cirq))
goto out;
rxd_addr = rxd_ep_av(rxd_ep)->fi_addr_table[addr];
ret = rxd_send_rts_if_needed(rxd_ep, rxd_addr);
if (ret)
goto out;
tx_entry = rxd_tx_entry_init(rxd_ep, iov, iov_count, NULL, 0, rma_count,
data, 0, context, rxd_addr, op, rxd_flags);
if (!tx_entry)
goto out;
ret = rxd_ep_send_op(rxd_ep, tx_entry, rma_iov, rma_count, NULL, 0, 0, 0);
if (ret)
rxd_tx_entry_free(rxd_ep, tx_entry);
out:
fastlock_release(&rxd_ep->util_ep.tx_cq->cq_lock);
fastlock_release(&rxd_ep->util_ep.lock);
return ret;
}
static void rxd_handle_conn_req(struct rxd_ep *ep, struct ofi_ctrl_hdr *ctrl,
struct fi_cq_msg_entry *comp,
struct rxd_rx_buf *rx_buf)
{
struct rxd_pkt_data *pkt_data;
struct rxd_peer *peer_info;
fi_addr_t dg_fiaddr;
void *addr;
int ret;
FI_INFO(&rxd_prov, FI_LOG_EP_DATA,
"conn req - rx_key: %" PRIu64 "\n", ctrl->rx_key);
pkt_data = (struct rxd_pkt_data *) ctrl;
addr = pkt_data->data;
if (ctrl->seg_size > RXD_MAX_DGRAM_ADDR) {
FI_WARN(&rxd_prov, FI_LOG_EP_DATA, "addr too large\n");
goto repost;
}
ret = rxd_av_insert_dg_addr(rxd_ep_av(ep), ctrl->rx_key, addr, &dg_fiaddr);
if (ret) {
FI_WARN(&rxd_prov, FI_LOG_EP_DATA, "failed to insert peer address\n");
goto repost;
}
peer_info = rxd_ep_getpeer_info(ep, dg_fiaddr);
if (peer_info->state != CMAP_CONNECTED) {
peer_info->state = CMAP_CONNECTED;
peer_info->conn_data = ctrl->conn_id;
peer_info->exp_msg_id++;
}
rxd_ep_reply_ack(ep, ctrl, ofi_ctrl_connresp, 0, ctrl->conn_id,
dg_fiaddr, dg_fiaddr);
repost:
rxd_ep_repost_buff(rx_buf);
}
int rxd_ep_send_pkt(struct rxd_ep *ep, struct rxd_pkt_entry *pkt_entry)
{
int ret;
if (ep->pending_cnt >= ep->tx_size)
return 1;
pkt_entry->timestamp = fi_gettime_ms();
ret = fi_send(ep->dg_ep, (const void *) rxd_pkt_start(pkt_entry),
pkt_entry->pkt_size, rxd_mr_desc(pkt_entry->mr, ep),
rxd_ep_av(ep)->rxd_addr_table[pkt_entry->peer].dg_addr,
&pkt_entry->context);
if (ret) {
FI_WARN(&rxd_prov, FI_LOG_EP_CTRL, "error sending packet: %d (%s)\n",
ret, fi_strerror(-ret));
} else {
pkt_entry->flags |= RXD_PKT_IN_USE;
ep->pending_cnt++;
}
return ret;
}
static void rxd_handle_start_data(struct rxd_ep *ep, struct rxd_peer *peer,
struct ofi_ctrl_hdr *ctrl,
struct fi_cq_msg_entry *comp,
struct rxd_rx_buf *rx_buf)
{
struct rxd_rx_entry *rx_entry;
struct rxd_pkt_data_start *pkt_start;
int ret;
FI_DBG(&rxd_prov, FI_LOG_EP_CTRL,
"start data- msg_id: %" PRIu64 ", segno: %d, buf: %p\n",
ctrl->msg_id, ctrl->seg_no, rx_buf);
pkt_start = (struct rxd_pkt_data_start *) ctrl;
if (pkt_start->op.version != OFI_OP_VERSION) {
FI_WARN(&rxd_prov, FI_LOG_EP_CTRL, "op version mismatch\n");
goto repost;
}
ret = rxd_check_start_pkt_order(ep, peer, ctrl, comp);
if (ret) {
if (ret == -FI_EALREADY) {
FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "duplicate pkt: %d\n",
ctrl->seg_no);
rxd_handle_dup_datastart(ep, ctrl, rx_buf);
goto repost;
} else {
FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "unexpected pkt: %d\n",
ctrl->seg_no);
goto repost;
}
}
rx_entry = rxd_rx_entry_alloc(ep);
if (!rx_entry)
goto repost;
rx_entry->peer_info = peer;
rx_entry->op_hdr = pkt_start->op;
rx_entry->exp_seg_no = 0;
rx_entry->msg_id = ctrl->msg_id;
rx_entry->done = 0;
rx_entry->peer = ctrl->conn_id;
rx_entry->source = (ep->util_ep.caps & FI_DIRECTED_RECV) ?
rxd_av_fi_addr(rxd_ep_av(ep), ctrl->conn_id) : FI_ADDR_UNSPEC;
rx_entry->credits = 1;
rx_entry->last_win_seg = 1;
FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "Assign rx_entry :%d for %p\n",
rx_entry->key, rx_entry->msg_id);
ep->credits--;
ret = rxd_process_start_data(ep, rx_entry, peer, ctrl, comp, rx_buf);
if (ret == -FI_ENOMEM)
rxd_rx_entry_free(ep, rx_entry);
else if (ret == -FI_ENOENT) {
peer->exp_msg_id++;
/* reply ack, with no window = 0 */
FI_DBG(&rxd_prov, FI_LOG_EP_CTRL, "Sending wait-ACK [%p] - %d\n",
ctrl->msg_id, ctrl->seg_no);
goto out;
} else {
peer->exp_msg_id++;
}
repost:
rxd_ep_repost_buff(rx_buf);
out:
assert(rxd_reposted_bufs);
return;
}
