static int send_recv()
{
struct fi_cq_entry comp;
int ret;
ret = fi_recv(ep, rx_buf, rx_size + ft_rx_prefix_size(),
mr_desc, 0, &rx_ctx);
if (ret)
return ret;
ft_sync();
fprintf(stdout, "Posting a send...\n");
ret = ft_post_tx(ep, remote_fi_addr, tx_size, NO_CQ_DATA, &tx_ctx);
if (ret)
return ret;
while ((tx_cq_cntr < tx_seq) || (rx_cq_cntr < rx_seq)) {
/* Wait for completion events on CQs */
ret = fi_wait(waitset, -1);
if (ret < 0) {
FT_PRINTERR("fi_wait", ret);
return ret;
}
/* Read the send completion entry */
ret = fi_cq_read(txcq, &comp, 1);
if (ret > 0) {
tx_cq_cntr++;
fprintf(stdout, "Received send completion event!\n");
} else if (ret < 0 && ret != -FI_EAGAIN) {
if (ret == -FI_EAVAIL) {
ret = ft_cq_readerr(txcq);
} else {
FT_PRINTERR("fi_cq_read", ret);
}
return ret;
}
/* Read the recv completion entry */
ret = fi_cq_read(rxcq, &comp, 1);
if (ret > 0) {
rx_cq_cntr++;
fprintf(stdout, "Received recv completion event!\n");
} else if (ret < 0 && ret != -FI_EAGAIN) {
if (ret == -FI_EAVAIL) {
ret = ft_cq_readerr(rxcq);
} else {
FT_PRINTERR("fi_cq_read", ret);
}
return ret;
}
}
return 0;
}
static int send_recv()
{
struct fi_cq_entry comp;
int ret;
fprintf(stdout, "Posting a send...\n");
ret = ft_post_tx(tx_size);
if (ret)
return ret;
while ((tx_cq_cntr < tx_seq) || (rx_cq_cntr < rx_seq)) {
/* Wait for completion events on CQs */
ret = fi_wait(waitset, -1);
if (ret < 0) {
FT_PRINTERR("fi_wait", ret);
return ret;
}
/* Read the send completion entry */
ret = fi_cq_read(txcq, &comp, 1);
if (ret > 0) {
tx_cq_cntr++;
fprintf(stdout, "Received send completion event!\n");
} else if (ret < 0 && ret != -FI_EAGAIN) {
if (ret == -FI_EAVAIL) {
ret = ft_cq_readerr(txcq);
} else {
FT_PRINTERR("fi_cq_read", ret);
}
return ret;
}
/* Read the recv completion entry */
ret = fi_cq_read(rxcq, &comp, 1);
if (ret > 0) {
rx_cq_cntr++;
fprintf(stdout, "Received recv completion event!\n");
} else if (ret < 0 && ret != -FI_EAGAIN) {
if (ret == -FI_EAVAIL) {
ret = ft_cq_readerr(rxcq);
} else {
FT_PRINTERR("fi_cq_read", ret);
}
return ret;
}
}
return 0;
}
static int ft_get_cq_comp(struct fid_cq *cq, uint64_t *cur,
uint64_t total, int timeout)
{
int ret;
switch (opts.comp_method) {
case FT_COMP_SREAD:
ret = ft_wait_for_comp(cq, cur, total, timeout);
break;
case FT_COMP_WAIT_FD:
ret = ft_fdwait_for_comp(cq, cur, total, timeout);
break;
default:
ret = ft_spin_for_comp(cq, cur, total, timeout);
break;
}
if (ret) {
if (ret == -FI_EAVAIL) {
ret = ft_cq_readerr(cq);
(*cur)++;
} else {
FT_PRINTERR("ft_get_cq_comp", ret);
}
}
return ret;
}
static int tagged_peek(uint64_t tag)
{
struct fi_cq_tagged_entry comp;
struct fi_msg_tagged msg;
int ret;
memset(&msg, 0, sizeof msg);
msg.tag = tag;
msg.context = &rx_ctx;
ret = fi_trecvmsg(ep, &msg, FI_PEEK);
if (ret) {
FT_PRINTERR("FI_PEEK", ret);
return ret;
}
ret = fi_cq_sread(rxcq, &comp, 1, NULL, -1);
if (ret != 1) {
if (ret == -FI_EAVAIL)
ret = ft_cq_readerr(rxcq);
else
FT_PRINTERR("fi_cq_sread", ret);
}
return ret;
}
static int run_test()
{
int ret;
size_t size = 1000;
struct fi_cq_data_entry comp;
if (opts.dst_addr) {
fprintf(stdout,
"Posting send with CQ data: 0x%" PRIx64 "\n",
remote_cq_data);
ret = fi_senddata(ep, buf, size, fi_mr_desc(mr), remote_cq_data,
0, buf);
if (ret) {
FT_PRINTERR("fi_send", ret);
return ret;
}
ret = ft_get_tx_comp(++tx_seq);
fprintf(stdout, "Done\n");
} else {
fprintf(stdout, "Waiting for CQ data from client\n");
ret = fi_cq_sread(rxcq, &comp, 1, NULL, -1);
if (ret < 0) {
if (ret == -FI_EAVAIL) {
ret = ft_cq_readerr(rxcq);
} else {
FT_PRINTERR("fi_cq_sread", ret);
}
return ret;
}
if (comp.flags & FI_REMOTE_CQ_DATA) {
if (comp.data == remote_cq_data) {
fprintf(stdout, "remote_cq_data: success\n");
ret = 0;
} else {
fprintf(stdout, "error, Expected data:0x%" PRIx64
", Received data:0x%" PRIx64 "\n",
remote_cq_data, comp.data);
ret = -FI_EIO;
}
} else {
fprintf(stdout, "error, CQ data flag not set\n");
ret = -FI_EBADFLAGS;
}
}
return ret;
}
static int send_recv()
{
struct fi_cq_entry comp;
struct epoll_event event;
struct fid *fids[1];
int ret;
const char *message = "Hello from Client!";
size_t message_len = strlen(message) + 1;
if (opts.dst_addr) {
fprintf(stdout, "Posting a send...\n");
if (snprintf(tx_buf, tx_size, "%s", message) >= tx_size) {
fprintf(stderr, "Transmit buffer too small.\n");
return -FI_ETOOSMALL;
}
ret = ft_post_tx(ep, remote_fi_addr, message_len, NO_CQ_DATA, &tx_ctx);
if (ret)
return ret;
memset(&event, 0, sizeof event);
fids[0] = &txcq->fid;
do {
if (fi_trywait(fabric, fids, 1) == FI_SUCCESS) {
ret = TEMP_FAILURE_RETRY(epoll_wait(epfd, &event, 1, -1));
if (ret < 0) {
ret = -errno;
FT_PRINTERR("epoll_wait", ret);
return ret;
}
if (event.data.ptr != &txcq->fid)
fprintf(stdout, "unexpected event!\n");
}
ret = fi_cq_read(txcq, &comp, 1);
} while (ret == -FI_EAGAIN);
if (ret < 0) {
if (ret == -FI_EAVAIL)
ret = ft_cq_readerr(txcq);
return ret;
}
fprintf(stdout, "Send completion received\n");
} else {
fprintf(stdout, "Waiting for client...\n");
memset(&event, 0, sizeof event);
fids[0] = &rxcq->fid;
do {
if (fi_trywait(fabric, fids, 1) == FI_SUCCESS) {
ret = TEMP_FAILURE_RETRY(epoll_wait(epfd, &event, 1, -1));
if (ret < 0) {
ret = -errno;
FT_PRINTERR("epoll_wait", ret);
return ret;
}
if (event.data.ptr != &rxcq->fid) {
fprintf(stdout, "unexpected event!\n");
}
}
ret = fi_cq_read(rxcq, &comp, 1);
} while (ret == -FI_EAGAIN);
if (ret < 0) {
if (ret == -FI_EAVAIL)
ret = ft_cq_readerr(rxcq);
return ret;
}
ret = check_recv_msg(message);
if (ret)
return ret;
fprintf(stdout, "Received data from client: %s\n", (char *) rx_buf);
}
return 0;
}