int ft_finalize(struct fid_ep *tx_ep, struct fid_cq *scq, struct fid_cq *rcq,
fi_addr_t addr)
{
struct fi_msg msg;
struct iovec iov;
struct fi_context tx_ctx;
char buf[4] = "fin";
int ret;
iov.iov_base = buf;
iov.iov_len = sizeof buf;
msg.msg_iov = &iov;
msg.desc = NULL;
msg.iov_count = 1;
msg.addr = addr;
msg.context = &tx_ctx;
msg.data = 0;
ret = fi_sendmsg(tx_ep, &msg, FI_INJECT | FI_TRANSMIT_COMPLETE);
if (ret) {
FT_PRINTERR("fi_sendmsg", ret);
return ret;
}
wait_for_data_completion(scq, 1);
wait_for_data_completion(rcq, 1);
return 0;
}
static ssize_t
mrail_tsend_common(struct fid_ep *ep_fid, const struct iovec *iov, void **desc,
size_t count, size_t len, fi_addr_t dest_addr, uint64_t tag,
uint64_t data, void *context, uint64_t flags)
{
struct mrail_ep *mrail_ep = container_of(ep_fid, struct mrail_ep,
util_ep.ep_fid.fid);
struct mrail_peer_info *peer_info;
struct iovec *iov_dest = alloca(sizeof(*iov_dest) * (count + 1));
struct mrail_tx_buf *tx_buf;
uint32_t i = mrail_get_tx_rail(mrail_ep);
struct fi_msg msg;
ssize_t ret;
peer_info = ofi_av_get_addr(mrail_ep->util_ep.av, (int) dest_addr);
ofi_ep_lock_acquire(&mrail_ep->util_ep);
tx_buf = mrail_get_tx_buf(mrail_ep, context, peer_info->seq_no++,
ofi_op_tagged, flags | FI_TAGGED);
if (OFI_UNLIKELY(!tx_buf)) {
ret = -FI_ENOMEM;
goto err1;
}
tx_buf->hdr.tag = tag;
mrail_copy_iov_hdr(&tx_buf->hdr, iov_dest, iov, count);
msg.msg_iov = iov_dest;
msg.desc = desc;
msg.iov_count = count + 1;
msg.addr = dest_addr;
msg.context = tx_buf;
msg.data = data;
if (len < mrail_ep->rails[i].info->tx_attr->inject_size)
flags |= FI_INJECT;
FI_DBG(&mrail_prov, FI_LOG_EP_DATA, "Posting tsend of length: %" PRIu64
" dest_addr: 0x%" PRIx64 " tag: 0x%" PRIx64 " seq: %d"
" on rail: %d\n", len, dest_addr, tag, peer_info->seq_no - 1, i);
ret = fi_sendmsg(mrail_ep->rails[i].ep, &msg, flags);
if (ret) {
FI_WARN(&mrail_prov, FI_LOG_EP_DATA,
"Unable to fi_sendmsg on rail: %" PRIu32 "\n", i);
goto err2;
} else if (!(flags & FI_COMPLETION)) {
ofi_ep_tx_cntr_inc(&mrail_ep->util_ep);
}
ofi_ep_lock_release(&mrail_ep->util_ep);
return ret;
err2:
util_buf_release(mrail_ep->tx_buf_pool, tx_buf);
err1:
peer_info->seq_no--;
ofi_ep_lock_release(&mrail_ep->util_ep);
return ret;
}
void do_sendmsgdata(int len)
{
int ret;
ssize_t sz;
int source_done = 0, dest_done = 0;
struct fi_cq_tagged_entry s_cqe, d_cqe;
struct fi_msg msg;
struct iovec iov;
uint64_t s[NUMEPS] = {0}, r[NUMEPS] = {0}, s_e[NUMEPS] = {0};
uint64_t r_e[NUMEPS] = {0};
iov.iov_base = source;
iov.iov_len = len;
msg.msg_iov = &iov;
msg.desc = (void **)loc_mr;
msg.iov_count = 1;
msg.addr = gni_addr[1];
msg.context = target;
msg.data = (uint64_t)source;
rdm_sr_init_data(source, len, 0xef);
rdm_sr_init_data(target, len, 0);
sz = fi_sendmsg(ep[0], &msg, FI_REMOTE_CQ_DATA);
cr_assert_eq(sz, 0);
sz = fi_recv(ep[1], target, len, rem_mr[0], gni_addr[0], source);
cr_assert_eq(sz, 0);
/* need to progress both CQs simultaneously for rendezvous */
do {
ret = fi_cq_read(msg_cq[0], &s_cqe, 1);
if (ret == 1) {
source_done = 1;
}
ret = fi_cq_read(msg_cq[1], &d_cqe, 1);
if (ret == 1) {
dest_done = 1;
}
} while (!(source_done && dest_done));
rdm_sr_check_cqe(&s_cqe, target, (FI_MSG|FI_SEND), 0, 0, 0);
rdm_sr_check_cqe(&d_cqe, source, (FI_MSG|FI_RECV|FI_REMOTE_CQ_DATA),
target, len, (uint64_t)source);
s[0] = 1; r[1] = 1;
rdm_sr_check_cntrs(s, r, s_e, r_e);
dbg_printf("got context events!\n");
cr_assert(rdm_sr_check_data(source, target, len), "Data mismatch");
}
int ft_finalize(void)
{
struct iovec iov;
int ret;
struct fi_context ctx;
void *desc = fi_mr_desc(mr);
strcpy(tx_buf + ft_tx_prefix_size(), "fin");
iov.iov_base = tx_buf;
iov.iov_len = 4 + ft_tx_prefix_size();
if (hints->caps & FI_TAGGED) {
struct fi_msg_tagged tmsg;
memset(&tmsg, 0, sizeof tmsg);
tmsg.msg_iov = &iov;
tmsg.desc = &desc;
tmsg.iov_count = 1;
tmsg.addr = remote_fi_addr;
tmsg.tag = tx_seq;
tmsg.ignore = 0;
tmsg.context = &ctx;
ret = fi_tsendmsg(ep, &tmsg, FI_INJECT | FI_TRANSMIT_COMPLETE);
} else {
struct fi_msg msg;
memset(&msg, 0, sizeof msg);
msg.msg_iov = &iov;
msg.desc = &desc;
msg.iov_count = 1;
msg.addr = remote_fi_addr;
msg.context = &ctx;
ret = fi_sendmsg(ep, &msg, FI_INJECT | FI_TRANSMIT_COMPLETE);
}
if (ret) {
FT_PRINTERR("transmit", ret);
return ret;
}
ret = ft_get_tx_comp(++tx_seq);
if (ret)
return ret;
ret = ft_get_rx_comp(rx_seq);
if (ret)
return ret;
return 0;
}
void Connection::post_send_msg(struct fi_msg* wr) {
int err = fi_sendmsg(ep_, wr, FI_COMPLETION);
if (err) {
// dump_send_wr(wr);
L_(fatal) << "previous send requests: " << total_send_requests_;
L_(fatal) << "previous recv requests: " << total_recv_requests_;
L_(fatal) << "fi_sendmsg failed: " << err << "=" << fi_strerror(-err);
throw LibfabricException("fi_sendmsg failed");
}
++total_send_requests_;
for (size_t i = 0; i < wr->iov_count; ++i)
total_bytes_sent_ += wr->msg_iov[i].iov_len;
}
/*
ssize_t fi_sendmsg(struct fid_ep *ep, const struct fi_msg *msg,
uint64_t flags);
*/
void do_sendmsg(int len)
{
int ret;
ssize_t sz;
struct fi_cq_entry cqe;
struct fi_msg msg;
struct iovec iov;
iov.iov_base = source;
iov.iov_len = len;
msg.msg_iov = &iov;
msg.desc = (void **)&loc_mr;
msg.iov_count = 1;
msg.addr = gni_addr[1];
msg.context = target;
msg.data = (uint64_t)target;
rdm_sr_init_data(source, len, 0xef);
rdm_sr_init_data(target, len, 0);
sz = fi_sendmsg(ep[0], &msg, 0);
cr_assert_eq(sz, 0);
sz = fi_recv(ep[1], target, len, rem_mr, gni_addr[0], source);
cr_assert_eq(sz, 0);
while ((ret = fi_cq_read(msg_cq[0], &cqe, 1)) == -FI_EAGAIN) {
pthread_yield();
}
cr_assert_eq(ret, 1);
cr_assert_eq((uint64_t)cqe.op_context, (uint64_t)target);
dbg_printf("got send context event!\n");
while ((ret = fi_cq_read(msg_cq[1], &cqe, 1)) == -FI_EAGAIN) {
pthread_yield();
}
cr_assert_eq(ret, 1);
cr_assert_eq((uint64_t)cqe.op_context, (uint64_t)source);
dbg_printf("got recv context event!\n");
cr_assert(rdm_sr_check_data(source, target, len), "Data mismatch");
}
ssize_t rxm_sendmsg(struct fid_ep *ep_fid, const struct fi_msg *msg,
uint64_t flags)
{
struct rxm_ep *rxm_ep;
struct fid_ep *msg_ep;
ssize_t ret;
rxm_ep = container_of(ep_fid, struct rxm_ep, util_ep.ep_fid.fid);
fastlock_acquire(&rxm_ep->cmap->lock);
ret = rxm_get_msg_ep(rxm_ep, msg->addr, &msg_ep);
if (ret)
goto unlock;
ret = fi_sendmsg(msg_ep, msg, flags);
unlock:
fastlock_release(&rxm_ep->cmap->lock);
return ret;
}
void sep_sendmsgdata(int index, int len)
{
ssize_t sz;
struct fi_cq_tagged_entry s_cqe = { (void *) -1, UINT_MAX, UINT_MAX,
(void *) -1, UINT_MAX, UINT_MAX };
struct fi_cq_tagged_entry d_cqe = { (void *) -1, UINT_MAX, UINT_MAX,
(void *) -1, UINT_MAX, UINT_MAX };
struct fi_msg msg;
struct iovec iov;
uint64_t s[NUMEPS] = {0}, r[NUMEPS] = {0}, s_e[NUMEPS] = {0};
uint64_t r_e[NUMEPS] = {0};
iov.iov_base = source;
iov.iov_len = len;
msg.msg_iov = &iov;
msg.desc = (void **)loc_mr;
msg.iov_count = 1;
msg.addr = rx_addr[index];
msg.context = target;
msg.data = (uint64_t)source;
sep_init_data(source, len, 0xe0 + index);
sep_init_data(target, len, 0);
sz = fi_sendmsg(tx_ep[0][index], &msg, FI_REMOTE_CQ_DATA);
cr_assert_eq(sz, 0);
sz = fi_recv(rx_ep[1][index], target, len, rem_mr[0],
FI_ADDR_UNSPEC, source);
cr_assert_eq(sz, 0);
wait_for_cqs(tx_cq[0][index], rx_cq[1][index], &s_cqe, &d_cqe);
sep_check_cqe(&s_cqe, target, (FI_MSG|FI_SEND), 0, 0, 0, false);
sep_check_cqe(&d_cqe, source, (FI_MSG|FI_RECV|FI_REMOTE_CQ_DATA),
target, len, (uint64_t)source, false);
s[0] = 1; r[1] = 1;
sep_check_cntrs(s, r, s_e, r_e);
cr_assert(sep_check_data(source, target, len), "Data mismatch");
}