static inline int rxm_finish_sar_segment_send(struct rxm_ep *rxm_ep, struct rxm_tx_sar_buf *tx_buf)
{
int ret = FI_SUCCESS;
struct rxm_tx_sar_buf *first_tx_buf;
switch (rxm_sar_get_seg_type(&tx_buf->pkt.ctrl_hdr)) {
case RXM_SAR_SEG_FIRST:
break;
case RXM_SAR_SEG_MIDDLE:
ofi_buf_free(tx_buf);
break;
case RXM_SAR_SEG_LAST:
ret = rxm_cq_tx_comp_write(rxm_ep, ofi_tx_cq_flags(tx_buf->pkt.hdr.op),
tx_buf->app_context, tx_buf->flags);
assert(ofi_tx_cq_flags(tx_buf->pkt.hdr.op) & FI_SEND);
ofi_ep_tx_cntr_inc(&rxm_ep->util_ep);
first_tx_buf = ofi_bufpool_get_ibuf(rxm_ep->
buf_pools[RXM_BUF_POOL_TX_SAR].pool,
tx_buf->pkt.ctrl_hdr.msg_id);
ofi_buf_free(first_tx_buf);
ofi_buf_free(tx_buf);
break;
}
return ret;
}
void rxd_ep_send_ack(struct rxd_ep *rxd_ep, fi_addr_t peer)
{
struct rxd_pkt_entry *pkt_entry;
struct rxd_ack_pkt *ack;
pkt_entry = rxd_get_tx_pkt(rxd_ep);
if (!pkt_entry) {
FI_WARN(&rxd_prov, FI_LOG_EP_CTRL, "Unable to send ack\n");
return;
}
ack = (struct rxd_ack_pkt *) (pkt_entry->pkt);
pkt_entry->pkt_size = sizeof(*ack) + rxd_ep->tx_prefix_size;
pkt_entry->peer = peer;
ack->base_hdr.version = RXD_PROTOCOL_VERSION;
ack->base_hdr.type = RXD_ACK;
ack->base_hdr.peer = rxd_ep->peers[peer].peer_addr;
ack->base_hdr.seq_no = rxd_ep->peers[peer].rx_seq_no;
ack->ext_hdr.rx_id = rxd_ep->peers[peer].rx_window;
rxd_ep->peers[peer].last_tx_ack = ack->base_hdr.seq_no;
dlist_insert_tail(&pkt_entry->d_entry, &rxd_ep->ctrl_pkts);
if (rxd_ep_send_pkt(rxd_ep, pkt_entry)) {
dlist_remove(&pkt_entry->d_entry);
ofi_buf_free(pkt_entry);
}
}
static void rxd_close_peer(struct rxd_ep *ep, struct rxd_peer *peer)
{
struct rxd_pkt_entry *pkt_entry;
struct rxd_x_entry *x_entry;
while (!dlist_empty(&peer->unacked)) {
dlist_pop_front(&peer->unacked, struct rxd_pkt_entry,
pkt_entry, d_entry);
ofi_buf_free(pkt_entry);
peer->unacked_cnt--;
}
while(!dlist_empty(&peer->tx_list)) {
dlist_pop_front(&peer->tx_list, struct rxd_x_entry,
x_entry, entry);
rxd_tx_entry_free(ep, x_entry);
}
while(!dlist_empty(&peer->rx_list)) {
dlist_pop_front(&peer->rx_list, struct rxd_x_entry,
x_entry, entry);
rxd_rx_entry_free(ep, x_entry);
}
while(!dlist_empty(&peer->rma_rx_list)) {
dlist_pop_front(&peer->rma_rx_list, struct rxd_x_entry,
x_entry, entry);
rxd_tx_entry_free(ep, x_entry);
}
dlist_remove(&peer->entry);
peer->active = 0;
}
static ssize_t rxd_ep_send_rts(struct rxd_ep *rxd_ep, fi_addr_t rxd_addr)
{
struct rxd_pkt_entry *pkt_entry;
struct rxd_rts_pkt *rts_pkt;
ssize_t ret;
size_t addrlen;
pkt_entry = rxd_get_tx_pkt(rxd_ep);
if (!pkt_entry)
return -FI_ENOMEM;
rts_pkt = (struct rxd_rts_pkt *) (pkt_entry->pkt);
pkt_entry->pkt_size = sizeof(*rts_pkt) + rxd_ep->tx_prefix_size;
pkt_entry->peer = rxd_addr;
rts_pkt->base_hdr.version = RXD_PROTOCOL_VERSION;
rts_pkt->base_hdr.type = RXD_RTS;
rts_pkt->rts_addr = rxd_addr;
addrlen = RXD_NAME_LENGTH;
memset(rts_pkt->source, 0, RXD_NAME_LENGTH);
ret = fi_getname(&rxd_ep->dg_ep->fid, (void *) rts_pkt->source,
&addrlen);
if (ret) {
ofi_buf_free(pkt_entry);
return ret;
}
rxd_ep_send_pkt(rxd_ep, pkt_entry);
rxd_insert_unacked(rxd_ep, rxd_addr, pkt_entry);
dlist_insert_tail(&rxd_ep->peers[rxd_addr].entry, &rxd_ep->rts_sent_list);
return 0;
}
static void rxd_peer_timeout(struct rxd_ep *rxd_ep, struct rxd_peer *peer)
{
struct fi_cq_err_entry err_entry;
struct rxd_x_entry *tx_entry;
struct rxd_pkt_entry *pkt_entry;
int ret;
while (!dlist_empty(&peer->tx_list)) {
dlist_pop_front(&peer->tx_list, struct rxd_x_entry, tx_entry, entry);
memset(&err_entry, 0, sizeof(struct fi_cq_err_entry));
rxd_tx_entry_free(rxd_ep, tx_entry);
err_entry.op_context = tx_entry->cq_entry.op_context;
err_entry.flags = tx_entry->cq_entry.flags;
err_entry.err = FI_ECONNREFUSED;
err_entry.prov_errno = 0;
ret = ofi_cq_write_error(&rxd_ep_tx_cq(rxd_ep)->util_cq, &err_entry);
if (ret)
FI_WARN(&rxd_prov, FI_LOG_EP_CTRL, "could not write error entry\n");
}
while (!dlist_empty(&peer->unacked)) {
dlist_pop_front(&peer->unacked, struct rxd_pkt_entry, pkt_entry,
d_entry);
ofi_buf_free(pkt_entry);
peer->unacked_cnt--;
}
dlist_remove(&peer->entry);
}
static inline ssize_t
rxm_ep_rma_common(struct rxm_ep *rxm_ep, const struct fi_msg_rma *msg, uint64_t flags,
rxm_rma_msg_fn rma_msg, uint64_t comp_flags)
{
struct rxm_rma_buf *rma_buf;
struct fi_msg_rma msg_rma = *msg;
struct rxm_conn *rxm_conn;
void *mr_desc[RXM_IOV_LIMIT] = { 0 };
int ret;
assert(msg->rma_iov_count <= rxm_ep->rxm_info->tx_attr->rma_iov_limit);
ret = rxm_ep_prepare_tx(rxm_ep, msg->addr, &rxm_conn);
if (OFI_UNLIKELY(ret))
return ret;
ofi_ep_lock_acquire(&rxm_ep->util_ep);
rma_buf = rxm_rma_buf_alloc(rxm_ep);
if (OFI_UNLIKELY(!rma_buf)) {
FI_WARN(&rxm_prov, FI_LOG_EP_DATA,
"Ran out of buffers from RMA buffer pool\n");
ret = -FI_ENOMEM;
goto unlock;
}
rma_buf->app_context = msg->context;
rma_buf->flags = flags;
ret = rxm_ep_rma_reg_iov(rxm_ep, msg_rma.msg_iov, msg_rma.desc, mr_desc,
msg_rma.iov_count, comp_flags & (FI_WRITE | FI_READ),
rma_buf);
if (OFI_UNLIKELY(ret))
goto release;
msg_rma.desc = mr_desc;
msg_rma.context = rma_buf;
ret = rma_msg(rxm_conn->msg_ep, &msg_rma, flags);
if (OFI_LIKELY(!ret))
goto unlock;
if ((rxm_ep->msg_mr_local) && (!rxm_ep->rxm_mr_local))
rxm_ep_msg_mr_closev(rma_buf->mr.mr, rma_buf->mr.count);
release:
ofi_buf_free(rma_buf);
unlock:
ofi_ep_lock_release(&rxm_ep->util_ep);
return ret;
}
void tcpx_xfer_entry_release(struct tcpx_cq *tcpx_cq,
struct tcpx_xfer_entry *xfer_entry)
{
if (xfer_entry->ep->cur_rx_entry == xfer_entry) {
xfer_entry->ep->cur_rx_entry = NULL;
}
xfer_entry->hdr.base_hdr.flags = 0;
xfer_entry->flags = 0;
xfer_entry->context = 0;
xfer_entry->rem_len = 0;
tcpx_cq->util_cq.cq_fastlock_acquire(&tcpx_cq->util_cq.cq_lock);
ofi_buf_free(xfer_entry);
tcpx_cq->util_cq.cq_fastlock_release(&tcpx_cq->util_cq.cq_lock);
}
static int rxm_rndv_tx_finish(struct rxm_ep *rxm_ep, struct rxm_tx_rndv_buf *tx_buf)
{
int ret;
RXM_LOG_STATE_TX(FI_LOG_CQ, tx_buf, RXM_RNDV_FINISH);
tx_buf->hdr.state = RXM_RNDV_FINISH;
tx_buf->conn->rndv_tx_credits++;
if (!rxm_ep->rxm_mr_local)
rxm_ep_msg_mr_closev(tx_buf->mr, tx_buf->count);
ret = rxm_cq_tx_comp_write(rxm_ep, ofi_tx_cq_flags(tx_buf->pkt.hdr.op),
tx_buf->app_context, tx_buf->flags);
assert(ofi_tx_cq_flags(tx_buf->pkt.hdr.op) & FI_SEND);
ofi_ep_tx_cntr_inc(&rxm_ep->util_ep);
ofi_buf_free(tx_buf);
return ret;
}
static inline int rxm_finish_rma(struct rxm_ep *rxm_ep, struct rxm_rma_buf *rma_buf,
uint64_t comp_flags)
{
int ret = rxm_cq_tx_comp_write(rxm_ep, comp_flags,
rma_buf->app_context, rma_buf->flags);
assert(((comp_flags & FI_WRITE) && !(comp_flags & FI_READ)) ||
((comp_flags & FI_READ) && !(comp_flags & FI_WRITE)));
if (comp_flags & FI_WRITE)
ofi_ep_wr_cntr_inc(&rxm_ep->util_ep);
else
ofi_ep_rd_cntr_inc(&rxm_ep->util_ep);
if (!(rma_buf->flags & FI_INJECT) && !rxm_ep->rxm_mr_local && rxm_ep->msg_mr_local) {
rxm_ep_msg_mr_closev(rma_buf->mr.mr, rma_buf->mr.count);
}
ofi_buf_free(rma_buf);
return ret;
}
int rxd_ep_post_buf(struct rxd_ep *ep)
{
struct rxd_pkt_entry *pkt_entry;
ssize_t ret;
pkt_entry = rxd_get_rx_pkt(ep);
if (!pkt_entry)
return -FI_ENOMEM;
ret = fi_recv(ep->dg_ep, rxd_pkt_start(pkt_entry),
rxd_ep_domain(ep)->max_mtu_sz,
rxd_mr_desc(pkt_entry->mr, ep),
FI_ADDR_UNSPEC, &pkt_entry->context);
if (ret) {
ofi_buf_free(pkt_entry);
FI_WARN(&rxd_prov, FI_LOG_EP_CTRL, "failed to repost\n");
return ret;
}
ep->posted_bufs++;
slist_insert_tail(&pkt_entry->s_entry, &ep->rx_pkt_list);
return 0;
}
static ssize_t rxm_ep_readv(struct fid_ep *ep_fid, const struct iovec *iov,
void **desc, size_t count, fi_addr_t src_addr,
uint64_t addr, uint64_t key, void *context)
{
struct rxm_ep *rxm_ep = container_of(ep_fid, struct rxm_ep,
util_ep.ep_fid.fid);
struct fi_rma_iov rma_iov = {
.addr = addr,
.len = ofi_total_iov_len(iov, count),
.key = key,
};
struct fi_msg_rma msg = {
.msg_iov = iov,
.desc = desc,
.iov_count = count,
.addr = src_addr,
.rma_iov = &rma_iov,
.rma_iov_count = 1,
.context = context,
.data = 0,
};
return rxm_ep_rma_common(rxm_ep, &msg, rxm_ep_tx_flags(rxm_ep), fi_readmsg, FI_READ);
}
static ssize_t rxm_ep_read(struct fid_ep *ep_fid, void *buf, size_t len,
void *desc, fi_addr_t src_addr, uint64_t addr,
uint64_t key, void *context)
{
struct iovec iov = {
.iov_base = (void*)buf,
.iov_len = len,
};
struct fi_rma_iov rma_iov = {
.addr = addr,
.len = len,
.key = key,
};
struct fi_msg_rma msg = {
.msg_iov = &iov,
.desc = &desc,
.iov_count = 1,
.addr = src_addr,
.rma_iov = &rma_iov,
.rma_iov_count = 1,
.context = context,
.data = 0,
};
struct rxm_ep *rxm_ep = container_of(ep_fid, struct rxm_ep,
util_ep.ep_fid.fid);
return rxm_ep_rma_common(rxm_ep, &msg, rxm_ep_tx_flags(rxm_ep), fi_readmsg, FI_READ);
}
static inline void
rxm_ep_format_rma_msg(struct rxm_rma_buf *rma_buf, const struct fi_msg_rma *orig_msg,
struct iovec *rxm_iov, struct fi_msg_rma *rxm_msg)
{
rxm_msg->context = rma_buf;
rxm_msg->addr = orig_msg->addr;
rxm_msg->data = orig_msg->data;
ofi_copy_from_iov(rma_buf->pkt.data, rma_buf->pkt.hdr.size,
orig_msg->msg_iov, orig_msg->iov_count, 0);
rxm_iov->iov_base = &rma_buf->pkt.data;
rxm_iov->iov_len = rma_buf->pkt.hdr.size;
rxm_msg->msg_iov = rxm_iov;
rxm_msg->desc = &rma_buf->hdr.desc;
rxm_msg->iov_count = 1;
rxm_msg->rma_iov = orig_msg->rma_iov;
rxm_msg->rma_iov_count = orig_msg->rma_iov_count;
}
static inline ssize_t
rxm_ep_rma_emulate_inject_msg(struct rxm_ep *rxm_ep, struct rxm_conn *rxm_conn, size_t total_size,
const struct fi_msg_rma *msg, uint64_t flags)
{
struct rxm_rma_buf *rma_buf;
ssize_t ret;
struct iovec rxm_msg_iov = { 0 };
struct fi_msg_rma rxm_rma_msg = { 0 };
assert(msg->rma_iov_count <= rxm_ep->rxm_info->tx_attr->rma_iov_limit);
ofi_ep_lock_acquire(&rxm_ep->util_ep);
rma_buf = rxm_rma_buf_alloc(rxm_ep);
if (OFI_UNLIKELY(!rma_buf)) {
FI_WARN(&rxm_prov, FI_LOG_EP_DATA,
"Ran out of buffers from RMA buffer pool\n");
ret = -FI_ENOMEM;
goto unlock;
}
rma_buf->pkt.hdr.size = total_size;
rma_buf->app_context = msg->context;
rma_buf->flags = flags;
rxm_ep_format_rma_msg(rma_buf, msg, &rxm_msg_iov, &rxm_rma_msg);
flags = (flags & ~FI_INJECT) | FI_COMPLETION;
ret = fi_writemsg(rxm_conn->msg_ep, &rxm_rma_msg, flags);
if (OFI_UNLIKELY(ret)) {
if (ret == -FI_EAGAIN)
rxm_ep_do_progress(&rxm_ep->util_ep);
ofi_buf_free(rma_buf);
}
unlock:
ofi_ep_lock_release(&rxm_ep->util_ep);
return ret;
}
static inline ssize_t
rxm_ep_rma_emulate_inject(struct rxm_ep *rxm_ep, struct rxm_conn *rxm_conn,
const void *buf, size_t len, uint64_t data,
fi_addr_t dest_addr, uint64_t addr, uint64_t key,
uint64_t flags)
{
struct fi_rma_iov rma_iov = {
.addr = addr,
.len = len,
.key = key,
};
struct iovec iov = {
.iov_base = (void*)buf,
.iov_len = len,
};
struct fi_msg_rma msg = {
.msg_iov = &iov,
.desc = NULL,
.iov_count = 1,
.addr = dest_addr,
.rma_iov = &rma_iov,
.rma_iov_count = 1,
.context = NULL,
.data = data,
};
return rxm_ep_rma_emulate_inject_msg(rxm_ep, rxm_conn, len, &msg, flags);
}
static inline ssize_t
rxm_ep_rma_inject_common(struct rxm_ep *rxm_ep, const struct fi_msg_rma *msg, uint64_t flags)
{
struct rxm_conn *rxm_conn;
size_t total_size = ofi_total_iov_len(msg->msg_iov, msg->iov_count);
ssize_t ret;
assert(total_size <= rxm_ep->rxm_info->tx_attr->inject_size);
ret = rxm_ep_prepare_tx(rxm_ep, msg->addr, &rxm_conn);
if (OFI_UNLIKELY(ret))
return ret;
if ((total_size <= rxm_ep->msg_info->tx_attr->inject_size) &&
!(flags & FI_COMPLETION) &&
(msg->iov_count == 1) && (msg->rma_iov_count == 1)) {
if (flags & FI_REMOTE_CQ_DATA) {
ret = fi_inject_writedata(rxm_conn->msg_ep,
msg->msg_iov->iov_base,
msg->msg_iov->iov_len, msg->data,
msg->addr, msg->rma_iov->addr,
msg->rma_iov->key);
} else {
ret = fi_inject_write(rxm_conn->msg_ep,
msg->msg_iov->iov_base,
msg->msg_iov->iov_len, msg->addr,
msg->rma_iov->addr,
msg->rma_iov->key);
}
if (OFI_LIKELY(!ret)) {
ofi_ep_wr_cntr_inc(&rxm_ep->util_ep);
} else {
FI_DBG(&rxm_prov, FI_LOG_EP_DATA,
"fi_inject_write* for MSG provider failed with ret - %"
PRId64"\n", ret);
if (OFI_LIKELY(ret == -FI_EAGAIN))
rxm_ep_progress(&rxm_ep->util_ep);
}
return ret;
} else {
return rxm_ep_rma_emulate_inject_msg(rxm_ep, rxm_conn, total_size, msg, flags);
}
}
static inline ssize_t
rxm_ep_generic_writemsg(struct fid_ep *ep_fid, const struct fi_msg_rma *msg,
uint64_t flags)
{
struct rxm_ep *rxm_ep =
container_of(ep_fid, struct rxm_ep, util_ep.ep_fid.fid);
if (flags & FI_INJECT)
return rxm_ep_rma_inject_common(rxm_ep, msg, flags);
else
return rxm_ep_rma_common(rxm_ep, msg, flags,
fi_writemsg, FI_WRITE);
}
static inline ssize_t
rxm_ep_writemsg(struct fid_ep *ep_fid, const struct fi_msg_rma *msg, uint64_t flags)
{
struct rxm_ep *rxm_ep =
container_of(ep_fid, struct rxm_ep, util_ep.ep_fid.fid);
return rxm_ep_generic_writemsg(ep_fid, msg, flags | rxm_ep->util_ep.tx_msg_flags);
}
static ssize_t rxm_ep_writev(struct fid_ep *ep_fid, const struct iovec *iov,
void **desc, size_t count, fi_addr_t dest_addr,
uint64_t addr, uint64_t key, void *context)
{
struct fi_rma_iov rma_iov = {
.addr = addr,
.len = ofi_total_iov_len(iov, count),
.key = key,
};
struct fi_msg_rma msg = {
.msg_iov = iov,
.desc = desc,
.iov_count = count,
.addr = dest_addr,
.rma_iov = &rma_iov,
.rma_iov_count = 1,
.context = context,
.data = 0,
};
struct rxm_ep *rxm_ep = container_of(ep_fid, struct rxm_ep,
util_ep.ep_fid.fid);
return rxm_ep_generic_writemsg(ep_fid, &msg, rxm_ep_tx_flags(rxm_ep));
}
static ssize_t rxm_ep_writedata(struct fid_ep *ep_fid, const void *buf,
size_t len, void *desc, uint64_t data,
fi_addr_t dest_addr, uint64_t addr,
uint64_t key, void *context)
{
struct fi_rma_iov rma_iov = {
.addr = addr,
.len = len,
.key = key,
};
struct iovec iov = {
.iov_base = (void*)buf,
.iov_len = len,
};
struct fi_msg_rma msg = {
.msg_iov = &iov,
.desc = &desc,
.iov_count = 1,
.addr = dest_addr,
.rma_iov = &rma_iov,
.rma_iov_count = 1,
.context = context,
.data = data,
};
struct rxm_ep *rxm_ep = container_of(ep_fid, struct rxm_ep,
util_ep.ep_fid.fid);
return rxm_ep_generic_writemsg(ep_fid, &msg, rxm_ep_tx_flags(rxm_ep) |
FI_REMOTE_CQ_DATA);
}
static ssize_t rxm_ep_write(struct fid_ep *ep_fid, const void *buf,
size_t len, void *desc, fi_addr_t dest_addr,
uint64_t addr, uint64_t key, void *context)
{
struct fi_rma_iov rma_iov = {
.addr = addr,
.len = len,
.key = key,
};
struct iovec iov = {
.iov_base = (void*)buf,
.iov_len = len,
};
struct fi_msg_rma msg = {
.msg_iov = &iov,
.desc = &desc,
.iov_count = 1,
.addr = dest_addr,
.rma_iov = &rma_iov,
.rma_iov_count = 1,
.context = context,
.data = 0,
};
struct rxm_ep *rxm_ep = container_of(ep_fid, struct rxm_ep,
util_ep.ep_fid.fid);
return rxm_ep_generic_writemsg(ep_fid, &msg, rxm_ep_tx_flags(rxm_ep));
}
static ssize_t rxm_ep_inject_write(struct fid_ep *ep_fid, const void *buf,
size_t len, fi_addr_t dest_addr,
uint64_t addr, uint64_t key)
{
ssize_t ret;
struct rxm_conn *rxm_conn;
struct rxm_ep *rxm_ep = container_of(ep_fid, struct rxm_ep,
util_ep.ep_fid.fid);
ret = rxm_ep_prepare_tx(rxm_ep, dest_addr, &rxm_conn);
if (OFI_UNLIKELY(ret))
return ret;
if (len <= rxm_ep->msg_info->tx_attr->inject_size) {
ret = fi_inject_write(rxm_conn->msg_ep, buf, len,
dest_addr, addr, key);
if (OFI_LIKELY(!ret)) {
ofi_ep_wr_cntr_inc(&rxm_ep->util_ep);
} else {
FI_DBG(&rxm_prov, FI_LOG_EP_DATA,
"fi_inject_write for MSG provider failed with ret - %"
PRId64"\n", ret);
if (OFI_LIKELY(ret == -FI_EAGAIN))
rxm_ep_progress(&rxm_ep->util_ep);
}
return ret;
} else {
return rxm_ep_rma_emulate_inject(rxm_ep, rxm_conn, buf, len,
0, dest_addr, addr, key, FI_INJECT);
}
}
static ssize_t rxm_ep_inject_writedata(struct fid_ep *ep_fid, const void *buf,
size_t len, uint64_t data,
fi_addr_t dest_addr, uint64_t addr,
uint64_t key)
{
ssize_t ret;
struct rxm_conn *rxm_conn;
struct rxm_ep *rxm_ep = container_of(ep_fid, struct rxm_ep,
util_ep.ep_fid.fid);
ret = rxm_ep_prepare_tx(rxm_ep, dest_addr, &rxm_conn);
if (OFI_UNLIKELY(ret))
return ret;
if (len <= rxm_ep->msg_info->tx_attr->inject_size) {
ret = fi_inject_writedata(rxm_conn->msg_ep, buf, len,
data, dest_addr, addr, key);
if (OFI_LIKELY(!ret)) {
ofi_ep_wr_cntr_inc(&rxm_ep->util_ep);
} else {
FI_DBG(&rxm_prov, FI_LOG_EP_DATA,
"fi_inject_writedata for MSG provider failed with ret - %"
PRId64"\n", ret);
if (OFI_LIKELY(ret == -FI_EAGAIN))
rxm_ep_progress(&rxm_ep->util_ep);
}
return ret;
} else {
return rxm_ep_rma_emulate_inject(rxm_ep, rxm_conn, buf, len,
data, dest_addr, addr, key,
FI_REMOTE_CQ_DATA | FI_INJECT);
}
}
struct fi_ops_rma rxm_ops_rma = {
.size = sizeof (struct fi_ops_rma),
.read = rxm_ep_read,
.readv = rxm_ep_readv,
.readmsg = rxm_ep_readmsg,
.write = rxm_ep_write,
.writev = rxm_ep_writev,
.writemsg = rxm_ep_writemsg,
.inject = rxm_ep_inject_write,
.writedata = rxm_ep_writedata,
.injectdata = rxm_ep_inject_writedata,
};
static int rxd_ep_close(struct fid *fid)
{
int ret;
struct rxd_ep *ep;
struct rxd_pkt_entry *pkt_entry;
struct slist_entry *entry;
struct rxd_peer *peer;
ep = container_of(fid, struct rxd_ep, util_ep.ep_fid.fid);
dlist_foreach_container(&ep->active_peers, struct rxd_peer, peer, entry)
rxd_close_peer(ep, peer);
ret = fi_close(&ep->dg_ep->fid);
if (ret)
return ret;
if (ep->dg_cq) {
ret = fi_close(&ep->dg_cq->fid);
if (ret)
return ret;
}
while (!slist_empty(&ep->rx_pkt_list)) {
entry = slist_remove_head(&ep->rx_pkt_list);
pkt_entry = container_of(entry, struct rxd_pkt_entry, s_entry);
ofi_buf_free(pkt_entry);
}
while (!dlist_empty(&ep->unexp_list)) {
dlist_pop_front(&ep->unexp_list, struct rxd_pkt_entry,
pkt_entry, d_entry);
ofi_buf_free(pkt_entry);
}
while (!dlist_empty(&ep->unexp_tag_list)) {
dlist_pop_front(&ep->unexp_tag_list, struct rxd_pkt_entry,
pkt_entry, d_entry);
ofi_buf_free(pkt_entry);
}
while (!dlist_empty(&ep->ctrl_pkts)) {
dlist_pop_front(&ep->ctrl_pkts, struct rxd_pkt_entry,
pkt_entry, d_entry);
ofi_buf_free(pkt_entry);
}
if (ep->util_ep.tx_cq) {
/* TODO: wait handling */
fid_list_remove(&ep->util_ep.tx_cq->ep_list,
&ep->util_ep.tx_cq->ep_list_lock,
&ep->util_ep.ep_fid.fid);
}
if (ep->util_ep.rx_cq) {
if (ep->util_ep.rx_cq != ep->util_ep.tx_cq) {
/* TODO: wait handling */
fid_list_remove(&ep->util_ep.rx_cq->ep_list,
&ep->util_ep.rx_cq->ep_list_lock,
&ep->util_ep.ep_fid.fid);
}
}
rxd_ep_free_res(ep);
ofi_endpoint_close(&ep->util_ep);
free(ep);
return 0;
}
static ssize_t
rxm_ep_atomic_common(struct rxm_ep *rxm_ep, struct rxm_conn *rxm_conn,
const struct fi_msg_atomic *msg, const struct fi_ioc *comparev,
void **compare_desc, size_t compare_iov_count,
struct fi_ioc *resultv, void **result_desc,
size_t result_iov_count, uint32_t op, uint64_t flags)
{
struct rxm_tx_atomic_buf *tx_buf;
struct rxm_atomic_hdr *atomic_hdr;
struct iovec buf_iov[RXM_IOV_LIMIT];
struct iovec cmp_iov[RXM_IOV_LIMIT];
size_t datatype_sz = ofi_datatype_size(msg->datatype);
size_t buf_len = 0;
size_t cmp_len = 0;
size_t tot_len;
ssize_t ret;
assert(msg->iov_count <= RXM_IOV_LIMIT &&
msg->rma_iov_count <= RXM_IOV_LIMIT);
if (flags & FI_REMOTE_CQ_DATA) {
FI_WARN(&rxm_prov, FI_LOG_EP_DATA,
"atomic with remote CQ data not supported\n");
return -FI_EINVAL;
}
if (msg->op != FI_ATOMIC_READ) {
assert(msg->msg_iov);
ofi_ioc_to_iov(msg->msg_iov, buf_iov, msg->iov_count,
datatype_sz);
buf_len = ofi_total_iov_len(buf_iov, msg->iov_count);
}
if (op == ofi_op_atomic_compare) {
assert(comparev);
ofi_ioc_to_iov(comparev, cmp_iov, compare_iov_count,
datatype_sz);
cmp_len = ofi_total_iov_len(cmp_iov, compare_iov_count);
assert(buf_len == cmp_len);
}
tot_len = buf_len + cmp_len + sizeof(struct rxm_atomic_hdr) +
sizeof(struct rxm_pkt);
if (tot_len > rxm_eager_limit) {
FI_WARN(&rxm_prov, FI_LOG_EP_DATA,
"atomic data too large %zu\n", tot_len);
return -FI_EINVAL;
}
ofi_ep_lock_acquire(&rxm_ep->util_ep);
tx_buf = (struct rxm_tx_atomic_buf *)
rxm_tx_buf_alloc(rxm_ep, RXM_BUF_POOL_TX_ATOMIC);
if (OFI_UNLIKELY(!tx_buf)) {
FI_WARN(&rxm_prov, FI_LOG_EP_DATA,
"Ran out of buffers from Atomic buffer pool\n");
ret = -FI_EAGAIN;
goto unlock;
}
rxm_ep_format_atomic_pkt_hdr(rxm_conn, tx_buf, tot_len, op,
msg->datatype, msg->op, flags, msg->data,
msg->rma_iov, msg->rma_iov_count);
tx_buf->pkt.ctrl_hdr.msg_id = ofi_buf_index(tx_buf);
tx_buf->app_context = msg->context;
atomic_hdr = (struct rxm_atomic_hdr *) tx_buf->pkt.data;
ofi_copy_from_iov(atomic_hdr->data, buf_len, buf_iov,
msg->iov_count, 0);
if (cmp_len)
ofi_copy_from_iov(atomic_hdr->data + buf_len, cmp_len,
cmp_iov, compare_iov_count, 0);
tx_buf->result_iov_count = result_iov_count;
if (resultv)
ofi_ioc_to_iov(resultv, tx_buf->result_iov, result_iov_count,
datatype_sz);
ret = rxm_ep_send_atomic_req(rxm_ep, rxm_conn, tx_buf, tot_len);
if (ret)
ofi_buf_free(tx_buf);
unlock:
ofi_ep_lock_release(&rxm_ep->util_ep);
return ret;
}
