void do_multirecv(int len)
{
int i, ret;
ssize_t sz;
struct fi_cq_tagged_entry s_cqe, d_cqe;
struct iovec iov;
struct fi_msg msg;
uint64_t s[NUMEPS] = {0}, r[NUMEPS] = {0}, s_e[NUMEPS] = {0};
uint64_t r_e[NUMEPS] = {0};
int nrecvs = 3;
rdm_sr_init_data(source, len, 0xab);
rdm_sr_init_data(target, len, 0);
/* Post receives first to force matching in SMSG callback. */
iov.iov_base = target;
iov.iov_len = len * nrecvs + 63;
msg.msg_iov = &iov;
msg.desc = (void **)rem_mr;
msg.iov_count = 1;
msg.addr = gni_addr[0];
msg.context = source;
msg.data = (uint64_t)source;
sz = fi_recvmsg(ep[1], &msg, FI_MULTI_RECV);
cr_assert_eq(sz, 0);
for (i = 0; i < nrecvs; i++) {
sz = fi_send(ep[0], source, len, loc_mr[0], gni_addr[1],
target);
cr_assert_eq(sz, 0);
}
/* need to progress both CQs simultaneously for rendezvous */
do {
ret = fi_cq_read(msg_cq[0], &s_cqe, 1);
if (ret == 1) {
rdm_sr_check_cqe(&s_cqe, target, (FI_MSG|FI_SEND),
0, 0, 0);
s[0]++;
}
ret = fi_cq_read(msg_cq[1], &d_cqe, 1);
if (ret == 1) {
rdm_sr_check_cqe(&d_cqe, source,
(FI_MSG|FI_RECV|FI_MULTI_RECV),
target + (r[1] * len), len, 0);
cr_assert(rdm_sr_check_data(source, d_cqe.buf, len),
"Data mismatch");
r[1]++;
}
} while (s[0] < nrecvs || r[1] < nrecvs);
rdm_sr_check_cntrs(s, r, s_e, r_e);
dbg_printf("got context events!\n");
}
void do_write_fence(int len)
{
int ret;
ssize_t sz;
struct fi_cq_tagged_entry cqe;
struct iovec iov;
struct fi_msg_rma msg;
struct fi_rma_iov rma_iov;
iov.iov_base = source;
iov.iov_len = len;
rma_iov.addr = (uint64_t)target;
rma_iov.len = sizeof(target);
rma_iov.key = mr_key;
msg.msg_iov = &iov;
msg.desc = (void **)&loc_mr;
msg.iov_count = 1;
msg.addr = gni_addr[1];
msg.rma_iov = &rma_iov;
msg.rma_iov_count = 1;
msg.context = target;
msg.data = (uint64_t)target;
init_data(source, len, 0xef);
init_data(target, len, 0);
/* write A */
sz = fi_writemsg(ep[0], &msg, 0);
cr_assert_eq(sz, 0);
/* write B */
sz = fi_writemsg(ep[0], &msg, FI_FENCE);
cr_assert_eq(sz, 0);
/* event A */
while ((ret = fi_cq_read(send_cq, &cqe, 1)) == -FI_EAGAIN) {
pthread_yield();
}
cr_assert_eq(ret, 1);
rdm_rma_check_tcqe(&cqe, target, FI_RMA | FI_WRITE, 0);
/* event B */
while ((ret = fi_cq_read(send_cq, &cqe, 1)) == -FI_EAGAIN) {
pthread_yield();
}
cr_assert_eq(ret, 1);
rdm_rma_check_tcqe(&cqe, target, FI_RMA | FI_WRITE, 0);
rdm_rma_check_cntrs(2, 0, 0, 0);
dbg_printf("got write context event!\n");
cr_assert(check_data(source, target, len), "Data mismatch");
}
/*
ssize_t (*recvv)(struct fid_ep *ep, const struct iovec *iov, void **desc,
size_t count, fi_addr_t src_addr, void *context);
*/
void do_recvv(int len)
{
int i, ret, iov_cnt;
ssize_t sz;
int source_done = 0, dest_done = 0;
struct fi_cq_tagged_entry s_cqe, d_cqe;
uint64_t s[NUMEPS] = {0}, r[NUMEPS] = {0}, s_e[NUMEPS] = {0};
uint64_t r_e[NUMEPS] = {0};
sz = fi_recvv(ep[1], NULL, NULL, IOV_CNT, gni_addr[0], iov_src_buf);
cr_assert_eq(sz, -FI_EINVAL);
sz = fi_recvv(ep[1], dest_iov, NULL, IOV_CNT + 1, gni_addr[0], iov_src_buf);
cr_assert_eq(sz, -FI_EINVAL);
for (iov_cnt = 1; iov_cnt <= IOV_CNT; iov_cnt++) {
rdm_sr_init_data(iov_src_buf, len * iov_cnt, 0xab);
for (i = 0; i < iov_cnt; i++) {
rdm_sr_init_data(dest_iov[i].iov_base, len, 0);
dest_iov[i].iov_len = len;
}
sz = fi_send(ep[0], iov_src_buf, len * iov_cnt, NULL, gni_addr[1],
dest_iov);
cr_assert_eq(sz, 0);
sz = fi_recvv(ep[1], dest_iov, NULL, iov_cnt, gni_addr[0], iov_src_buf);
cr_assert_eq(sz, 0);
/* need to progress both CQs simultaneously for rendezvous */
do {
ret = fi_cq_read(msg_cq[0], &s_cqe, 1);
if (ret == 1) {
source_done = 1;
}
ret = fi_cq_read(msg_cq[1], &d_cqe, 1);
if (ret == 1) {
dest_done = 1;
}
} while (!(source_done && dest_done));
rdm_sr_check_cqe(&s_cqe, dest_iov, (FI_MSG|FI_SEND), 0, 0, 0, false);
rdm_sr_check_cqe(&d_cqe, iov_src_buf, (FI_MSG|FI_RECV), dest_iov,
len * iov_cnt, 0, false);
s[0] = 1; r[1] = 1;
rdm_sr_check_cntrs(s, r, s_e, r_e);
dbg_printf("got context events!\n");
cr_assert(rdm_sr_check_iov_data(dest_iov, iov_src_buf, iov_cnt, len * iov_cnt),
"Data mismatch");
source_done = dest_done = 0;
}
}
void do_sendrecvv_alignment(int slen, int dlen, int offset)
{
int i, ret, iov_cnt;
ssize_t sz;
int source_done = 0, dest_done = 0;
struct fi_cq_tagged_entry s_cqe, d_cqe;
uint64_t s[NUMEPS] = {0}, r[NUMEPS] = {0}, s_e[NUMEPS] = {0};
uint64_t r_e[NUMEPS] = {0};
uint64_t iov_s_buf = (uint64_t) iov_src_buf;
iov_s_buf += offset;
for (iov_cnt = 1; iov_cnt <= IOV_CNT; iov_cnt++) {
for (i = 0; i < iov_cnt; i++) {
d_iov[i].iov_base = dest_iov[i].iov_base;
d_iov[i].iov_base = (void *) ((uint64_t)d_iov[i].iov_base + offset);
rdm_sr_init_data(d_iov[i].iov_base, dlen - offset, 0);
d_iov[i].iov_len = dlen - offset;
}
rdm_sr_init_data((void *) iov_s_buf, (slen - offset) * iov_cnt, 0xab);
sz = fi_send(ep[0], (void *) iov_s_buf, (slen - offset) * iov_cnt, NULL, gni_addr[1],
d_iov);
cr_assert_eq(sz, 0);
sz = fi_recvv(ep[1], d_iov, NULL, iov_cnt, gni_addr[0], (void *) iov_s_buf);
cr_assert_eq(sz, 0);
/* need to progress both CQs simultaneously for rendezvous */
do {
ret = fi_cq_read(msg_cq[0], &s_cqe, 1);
if (ret == 1) {
source_done = 1;
}
ret = fi_cq_read(msg_cq[1], &d_cqe, 1);
if (ret == 1) {
dest_done = 1;
}
} while (!(source_done && dest_done));
rdm_sr_check_cqe(&s_cqe, d_iov, (FI_MSG|FI_SEND), 0, 0, 0, false);
rdm_sr_check_cqe(&d_cqe, (void *) iov_s_buf, (FI_MSG|FI_RECV), d_iov,
MIN((slen - offset) * iov_cnt, (dlen - offset) * iov_cnt), 0, false);
s[0] = 1; r[1] = 1;
rdm_sr_check_cntrs(s, r, s_e, r_e);
dbg_printf("got context events!\n");
cr_assert(rdm_sr_check_iov_data(d_iov, (void *) iov_s_buf,
iov_cnt, (slen - offset) * iov_cnt),
"Data mismatch");
source_done = dest_done = 0;
}
}
static int send_recv()
{
struct fi_cq_entry comp;
int ret;
ret = fi_recv(ep, rx_buf, rx_size + ft_rx_prefix_size(),
mr_desc, 0, &rx_ctx);
if (ret)
return ret;
ft_sync();
fprintf(stdout, "Posting a send...\n");
ret = ft_post_tx(ep, remote_fi_addr, tx_size, NO_CQ_DATA, &tx_ctx);
if (ret)
return ret;
while ((tx_cq_cntr < tx_seq) || (rx_cq_cntr < rx_seq)) {
/* Wait for completion events on CQs */
ret = fi_wait(waitset, -1);
if (ret < 0) {
FT_PRINTERR("fi_wait", ret);
return ret;
}
/* Read the send completion entry */
ret = fi_cq_read(txcq, &comp, 1);
if (ret > 0) {
tx_cq_cntr++;
fprintf(stdout, "Received send completion event!\n");
} else if (ret < 0 && ret != -FI_EAGAIN) {
if (ret == -FI_EAVAIL) {
ret = ft_cq_readerr(txcq);
} else {
FT_PRINTERR("fi_cq_read", ret);
}
return ret;
}
/* Read the recv completion entry */
ret = fi_cq_read(rxcq, &comp, 1);
if (ret > 0) {
rx_cq_cntr++;
fprintf(stdout, "Received recv completion event!\n");
} else if (ret < 0 && ret != -FI_EAGAIN) {
if (ret == -FI_EAVAIL) {
ret = ft_cq_readerr(rxcq);
} else {
FT_PRINTERR("fi_cq_read", ret);
}
return ret;
}
}
return 0;
}
void do_sendmsgdata(int len)
{
int ret;
ssize_t sz;
int source_done = 0, dest_done = 0;
struct fi_cq_tagged_entry s_cqe, d_cqe;
struct fi_msg msg;
struct iovec iov;
uint64_t s[NUMEPS] = {0}, r[NUMEPS] = {0}, s_e[NUMEPS] = {0};
uint64_t r_e[NUMEPS] = {0};
iov.iov_base = source;
iov.iov_len = len;
msg.msg_iov = &iov;
msg.desc = (void **)loc_mr;
msg.iov_count = 1;
msg.addr = gni_addr[1];
msg.context = target;
msg.data = (uint64_t)source;
rdm_sr_init_data(source, len, 0xef);
rdm_sr_init_data(target, len, 0);
sz = fi_sendmsg(ep[0], &msg, FI_REMOTE_CQ_DATA);
cr_assert_eq(sz, 0);
sz = fi_recv(ep[1], target, len, rem_mr[0], gni_addr[0], source);
cr_assert_eq(sz, 0);
/* need to progress both CQs simultaneously for rendezvous */
do {
ret = fi_cq_read(msg_cq[0], &s_cqe, 1);
if (ret == 1) {
source_done = 1;
}
ret = fi_cq_read(msg_cq[1], &d_cqe, 1);
if (ret == 1) {
dest_done = 1;
}
} while (!(source_done && dest_done));
rdm_sr_check_cqe(&s_cqe, target, (FI_MSG|FI_SEND), 0, 0, 0);
rdm_sr_check_cqe(&d_cqe, source, (FI_MSG|FI_RECV|FI_REMOTE_CQ_DATA),
target, len, (uint64_t)source);
s[0] = 1; r[1] = 1;
rdm_sr_check_cntrs(s, r, s_e, r_e);
dbg_printf("got context events!\n");
cr_assert(rdm_sr_check_data(source, target, len), "Data mismatch");
}
void api_write_read(int len)
{
int ret;
struct fi_cq_tagged_entry cqe;
struct fi_cq_err_entry err_cqe = {0};
rdm_api_init_data(source, len, 0xab);
rdm_api_init_data(target, len, 0);
fi_write(ep[0], source, len,
loc_mr[0], gni_addr[1], (uint64_t)target, mr_key[1],
target);
while ((ret = fi_cq_read(msg_cq[0], &cqe, 1)) == -FI_EAGAIN)
pthread_yield();
if (ret == -FI_EAVAIL) {
fi_cq_readerr(msg_cq[0], &err_cqe, 0);
dbg_printf("fi_cq_readerr err:%d\n", err_cqe.err);
}
if (write_allowed(FI_RMA, fi[0]->caps, fi[1]->caps)) {
cr_assert(ret == 1,
"fi_write failed caps:0x%lx ret:%d",
fi[0]->caps, ret);
} else {
cr_assert(err_cqe.err == FI_EOPNOTSUPP,
"fi_write should fail caps:0x%lx err:%d",
fi[0]->caps, err_cqe.err);
}
fi_read(ep[0], source, len,
loc_mr[0], gni_addr[1], (uint64_t)target, mr_key[1],
(void *)READ_CTX);
while ((ret = fi_cq_read(msg_cq[0], &cqe, 1)) == -FI_EAGAIN)
pthread_yield();
if (ret == -FI_EAVAIL) {
fi_cq_readerr(msg_cq[0], &err_cqe, 0);
dbg_printf("fi_cq_readerr err:%d\n", err_cqe.err);
}
if (read_allowed(FI_RMA, fi[0]->caps, fi[1]->caps)) {
cr_assert(ret == 1,
"fi_read failed caps:0x%lx rcaps:0x%lx",
fi[0]->caps, fi[1]->caps);
} else {
cr_assert(err_cqe.err == FI_EOPNOTSUPP,
"fi_read should fail caps:0x%lx rcaps:0x%lx",
fi[0]->caps, fi[1]->caps);
}
}
/*
* ssize_t fi_send(struct fid_ep *ep, void *buf, size_t len,
* void *desc, fi_addr_t dest_addr, void *context);
*
* ssize_t fi_recv(struct fid_ep *ep, void * buf, size_t len,
* void *desc, fi_addr_t src_addr, void *context);
*/
void do_send(int len)
{
int ret;
int source_done = 0, dest_done = 0;
int scanceled = 0, dcanceled = 0;
struct fi_cq_tagged_entry s_cqe, d_cqe;
ssize_t sz;
uint64_t s[NUMEPS] = {0}, r[NUMEPS] = {0}, s_e[NUMEPS] = {0};
uint64_t r_e[NUMEPS] = {0};
rdm_sr_init_data(source, len, 0xab);
rdm_sr_init_data(target, len, 0);
sz = fi_send(ep[0], source, len, loc_mr[0], gni_addr[1], target);
cr_assert_eq(sz, 0);
sz = fi_recv(ep[1], target, len, rem_mr[1], gni_addr[0], source);
cr_assert_eq(sz, 0);
/* need to progress both CQs simultaneously for rendezvous */
do {
ret = fi_cq_read(msg_cq[0], &s_cqe, 1);
if (ret == 1) {
source_done = 1;
}
if (ret == -FI_EAVAIL) {
if (rdm_sr_check_canceled(msg_cq[0]))
scanceled = 1;
}
ret = fi_cq_read(msg_cq[1], &d_cqe, 1);
if (ret == 1) {
dest_done = 1;
}
if (ret == -FI_EAVAIL) {
if (rdm_sr_check_canceled(msg_cq[1]))
dcanceled = 1;
}
} while (!((source_done || scanceled) && (dest_done || dcanceled)));
/* no further checking needed */
if (dgram_should_fail && (scanceled || dcanceled))
return;
rdm_sr_check_cqe(&s_cqe, target, (FI_MSG|FI_SEND), 0, 0, 0);
rdm_sr_check_cqe(&d_cqe, source, (FI_MSG|FI_RECV), target, len, 0);
s[0] = 1; r[1] = 1;
rdm_sr_check_cntrs(s, r, s_e, r_e);
dbg_printf("got context events!\n");
cr_assert(rdm_sr_check_data(source, target, len), "Data mismatch");
}
void do_readmsgdata(int len)
{
int ret;
ssize_t sz;
struct fi_cq_tagged_entry cqe, dcqe;
struct iovec iov;
struct fi_msg_rma msg;
struct fi_rma_iov rma_iov;
iov.iov_base = source;
iov.iov_len = len;
rma_iov.addr = (uint64_t)target;
rma_iov.len = len;
rma_iov.key = mr_key;
msg.msg_iov = &iov;
msg.desc = (void **)&loc_mr;
msg.iov_count = 1;
msg.addr = gni_addr[1];
msg.rma_iov = &rma_iov;
msg.rma_iov_count = 1;
msg.context = target;
msg.data = (uint64_t)READ_DATA;
init_data(target, len, 0xef);
init_data(source, len, 0);
sz = fi_readmsg(ep[0], &msg, FI_REMOTE_CQ_DATA);
cr_assert_eq(sz, 0);
while ((ret = fi_cq_read(send_cq, &cqe, 1)) == -FI_EAGAIN) {
pthread_yield();
}
cr_assert_eq(ret, 1);
rdm_rma_check_tcqe(&cqe, target, FI_RMA | FI_READ, 0);
rdm_rma_check_cntrs(0, 1, 0, 0);
dbg_printf("got write context event!\n");
cr_assert(check_data(source, target, len), "Data mismatch");
while ((ret = fi_cq_read(recv_cq, &dcqe, 1)) == -FI_EAGAIN) {
ret = fi_cq_read(send_cq, &cqe, 1); /* for progress */
pthread_yield();
}
cr_assert(ret != FI_SUCCESS, "Missing remote data");
rdm_rma_check_tcqe(&dcqe, NULL,
(FI_RMA | FI_REMOTE_READ | FI_REMOTE_CQ_DATA),
READ_DATA);
}
static int send_recv()
{
struct fi_cq_entry comp;
int ret;
if (opts.dst_addr) {
/* Client */
fprintf(stdout, "Posting a send...\n");
sprintf(buf, "Hello from Client!");
ret = fi_send(ep, buf, sizeof("Hello from Client!"),
fi_mr_desc(mr), remote_fi_addr, &fi_ctx_send);
if (ret) {
FT_PRINTERR("fi_send", ret);
return ret;
}
/* Read send queue */
do {
ret = fi_cq_read(scq, &comp, 1);
if (ret < 0 && ret != -FI_EAGAIN) {
FT_PRINTERR("fi_cq_read", ret);
return ret;
}
} while (ret == -FI_EAGAIN);
fprintf(stdout, "Send completion received\n");
} else {
/* Server */
fprintf(stdout, "Posting a recv...\n");
ret = fi_recv(ep, buf, buffer_size, fi_mr_desc(mr), 0,
&fi_ctx_recv);
if (ret) {
FT_PRINTERR("fi_recv", ret);
return ret;
}
/* Read recv queue */
fprintf(stdout, "Waiting for client...\n");
do {
ret = fi_cq_read(rcq, &comp, 1);
if (ret < 0 && ret != -FI_EAGAIN) {
FT_PRINTERR("fi_cq_read", ret);
return ret;
}
} while (ret == -FI_EAGAIN);
fprintf(stdout, "Received data from client: %s\n", (char *)buf);
}
return 0;
}
static int send_recv()
{
struct fi_cq_entry comp;
int ret;
fprintf(stdout, "Posting a send...\n");
ret = ft_post_tx(tx_size);
if (ret)
return ret;
while ((tx_cq_cntr < tx_seq) || (rx_cq_cntr < rx_seq)) {
/* Wait for completion events on CQs */
ret = fi_wait(waitset, -1);
if (ret < 0) {
FT_PRINTERR("fi_wait", ret);
return ret;
}
/* Read the send completion entry */
ret = fi_cq_read(txcq, &comp, 1);
if (ret > 0) {
tx_cq_cntr++;
fprintf(stdout, "Received send completion event!\n");
} else if (ret < 0 && ret != -FI_EAGAIN) {
if (ret == -FI_EAVAIL) {
ret = ft_cq_readerr(txcq);
} else {
FT_PRINTERR("fi_cq_read", ret);
}
return ret;
}
/* Read the recv completion entry */
ret = fi_cq_read(rxcq, &comp, 1);
if (ret > 0) {
rx_cq_cntr++;
fprintf(stdout, "Received recv completion event!\n");
} else if (ret < 0 && ret != -FI_EAGAIN) {
if (ret == -FI_EAVAIL) {
ret = ft_cq_readerr(rxcq);
} else {
FT_PRINTERR("fi_cq_read", ret);
}
return ret;
}
}
return 0;
}
void do_write_autoreg_uncached(int len)
{
int ret;
ssize_t sz;
struct fi_cq_tagged_entry cqe;
init_data(uc_source, len, 0xab);
init_data(target, len, 0);
sz = fi_write(ep[0], uc_source, len,
NULL, gni_addr[1], (uint64_t)target, mr_key,
target);
cr_assert_eq(sz, 0);
while ((ret = fi_cq_read(send_cq, &cqe, 1)) == -FI_EAGAIN) {
pthread_yield();
}
cr_assert_eq(ret, 1);
rdm_rma_check_tcqe(&cqe, target, FI_RMA | FI_WRITE, 0);
rdm_rma_check_cntrs(1, 0, 0, 0);
dbg_printf("got write context event!\n");
cr_assert(check_data(uc_source, target, len), "Data mismatch");
}
void do_readv(int len)
{
int ret;
ssize_t sz;
struct fi_cq_tagged_entry cqe;
struct iovec iov;
iov.iov_base = source;
iov.iov_len = len;
init_data(target, len, 0x25);
init_data(source, len, 0);
sz = fi_readv(ep[0], &iov, (void **)&loc_mr, 1,
gni_addr[1], (uint64_t)target, mr_key,
target);
cr_assert_eq(sz, 0);
while ((ret = fi_cq_read(send_cq, &cqe, 1)) == -FI_EAGAIN) {
pthread_yield();
}
cr_assert_eq(ret, 1);
rdm_rma_check_tcqe(&cqe, target, FI_RMA | FI_READ, 0);
rdm_rma_check_cntrs(0, 1, 0, 0);
dbg_printf("got write context event!\n");
cr_assert(check_data(source, target, len), "Data mismatch");
}
static int recv_xfer(int size)
{
struct fi_cq_tagged_entry comp;
int ret;
do {
ret = fi_cq_read(rcq, &comp, 1);
if (ret < 0 && ret != -FI_EAGAIN) {
if (ret == -FI_EAVAIL) {
cq_readerr(rcq, "rcq");
} else {
FT_PRINTERR("fi_cq_read", ret);
}
return ret;
}
} while (ret == -FI_EAGAIN);
/* Posting recv for next send. Hence tag_data + 1 */
ret = fi_trecv(ep, buf, buffer_size, fi_mr_desc(mr), remote_fi_addr,
tag_data + 1, 0, &fi_ctx_trecv);
if (ret)
FT_PRINTERR("fi_trecv", ret);
return ret;
}
void do_tinject(int len)
{
int ret;
ssize_t sz;
struct fi_cq_tagged_entry cqe;
rdm_tagged_sr_init_data(source, len, 0x23);
rdm_tagged_sr_init_data(target, len, 0);
sz = fi_tinject(ep[0], source, len, gni_addr[1], len);
cr_assert_eq(sz, 0);
sz = fi_trecv(ep[1], target, len, rem_mr, gni_addr[0], len, 0, source);
cr_assert_eq(sz, 0);
while ((ret = fi_cq_read(msg_cq[1], &cqe, 1)) == -FI_EAGAIN) {
pthread_yield();
}
cr_assert_eq(ret, 1);
cr_assert_eq((uint64_t)cqe.op_context, (uint64_t)source);
dbg_printf("got recv context event!\n");
cr_assert(rdm_tagged_sr_check_data(source, target, len), "Data mismatch");
}
void api_do_read_buf(void)
{
int ret;
int len = 8*1024;
ssize_t sz;
struct fi_cq_tagged_entry cqe;
struct fi_cq_err_entry err_cqe;
rdm_api_init_data(source, BUF_SZ, 0);
rdm_api_init_data(target, BUF_SZ, 0xad);
/* cause a chained transaction */
sz = fi_read(ep[0], source+6, len,
loc_mr[0], gni_addr[1], (uint64_t)target+6, mr_key[1],
(void *)READ_CTX);
cr_assert_eq(sz, 0);
while ((ret = fi_cq_read(msg_cq[0], &cqe, 1)) == -FI_EAGAIN)
pthread_yield();
if (ret == -FI_EAVAIL) {
fi_cq_readerr(msg_cq[0], &err_cqe, 0);
dbg_printf("fi_cq_readerr err:%d\n", err_cqe.err);
}
if (read_allowed(FI_RMA, fi[0]->caps, fi[1]->caps)) {
cr_assert(ret == 1,
"fi_read failed caps:0x%lx rcaps:0x%lx",
fi[0]->caps, fi[1]->caps);
} else {
cr_assert(err_cqe.err == FI_EOPNOTSUPP,
"fi_read should fail caps:0x%lx rcaps:0x%lx",
fi[0]->caps, fi[1]->caps);
}
}
/*
* fi_cq_err_entry can be cast to any CQ entry format.
*/
static int ft_fdwait_for_comp(struct fid_cq *cq, uint64_t *cur,
uint64_t total, int timeout)
{
struct fi_cq_err_entry comp;
struct fid *fids[1];
int fd, ret;
fd = cq == txcq ? tx_fd : rx_fd;
fids[0] = &cq->fid;
while (total - *cur > 0) {
ret = fi_trywait(fabric, fids, 1);
if (ret == FI_SUCCESS) {
ret = ft_poll_fd(fd, timeout);
if (ret)
return ret;
}
ret = fi_cq_read(cq, &comp, 1);
if (ret > 0) {
(*cur)++;
} else if (ret < 0 && ret != -FI_EAGAIN) {
return ret;
}
}
return 0;
}
/*
* fi_cq_err_entry can be cast to any CQ entry format.
*/
static int ft_spin_for_comp(struct fid_cq *cq, uint64_t *cur,
uint64_t total, int timeout)
{
struct fi_cq_err_entry comp;
struct timespec a, b;
int ret;
if (timeout >= 0)
clock_gettime(CLOCK_MONOTONIC, &a);
while (total - *cur > 0) {
ret = fi_cq_read(cq, &comp, 1);
if (ret > 0) {
if (timeout >= 0)
clock_gettime(CLOCK_MONOTONIC, &a);
(*cur)++;
} else if (ret < 0 && ret != -FI_EAGAIN) {
return ret;
} else if (timeout >= 0) {
clock_gettime(CLOCK_MONOTONIC, &b);
if ((b.tv_sec - a.tv_sec) > timeout) {
fprintf(stderr, "%ds timeout expired\n", timeout);
return -FI_ENODATA;
}
}
}
return 0;
}
void rxd_cq_progress(struct util_cq *util_cq)
{
ssize_t ret = 0;
struct rxd_cq *cq;
struct fi_cq_msg_entry cq_entry;
struct dlist_entry *item, *next;
struct rxd_unexp_cq_entry *unexp;
cq = container_of(util_cq, struct rxd_cq, util_cq);
fastlock_acquire(&cq->lock);
do {
ret = fi_cq_read(cq->dg_cq, &cq_entry, 1);
if (ret == -FI_EAGAIN)
break;
if (cq_entry.flags & FI_SEND) {
rxd_handle_send_comp(&cq_entry);
} else if (cq_entry.flags & FI_RECV) {
rxd_handle_recv_comp(cq, &cq_entry, 0);
} else
assert (0);
} while (ret > 0);
for (item = cq->unexp_list.next; item != &cq->unexp_list;) {
unexp = container_of(item, struct rxd_unexp_cq_entry, entry);
next = item->next;
rxd_handle_recv_comp(cq, &unexp->cq_entry, 1);
item = next;
}
fastlock_release(&cq->lock);
}
static int wait_remote_writedata_completion(void)
{
struct fi_cq_data_entry comp;
int ret;
do {
ret = fi_cq_read(rcq, &comp, 1);
if (ret < 0 && ret != -FI_EAGAIN) {
if (ret == -FI_EAVAIL) {
cq_readerr(rcq, "rcq");
} else {
FT_PRINTERR("fi_cq_read", ret);
}
return ret;
}
} while (ret == -FI_EAGAIN);
ret = 0;
if (comp.data != cq_data) {
fprintf(stderr, "Got unexpected completion data %" PRIu64 "\n",
comp.data);
}
assert(comp.op_context == buf || comp.op_context == NULL);
if (comp.op_context == buf) {
/* We need to repost the receive */
ret = fi_recv(ep, buf, buffer_size, fi_mr_desc(mr), 0, buf);
if (ret)
FT_PRINTERR("fi_recv", ret);
}
return ret;
}
static int send_xfer(int size)
{
struct fi_cq_entry comp;
int ret;
while (!credits) {
ret = fi_cq_read(scq, &comp, 1);
if (ret > 0) {
goto post;
} else if (ret < 0) {
printf("RCQ read %d (%s)\n", ret, fi_strerror(-ret));
return ret;
}
}
credits--;
post:
ret = dst_addr ?
fi_send(ep, buf_ptr, (size_t) size, fi_mr_desc(mr), NULL) :
fi_sendto(ep, buf_ptr, (size_t) size, fi_mr_desc(mr),
client_addr, NULL);
if (ret)
printf("fi_send %d (%s)\n", ret, fi_strerror(-ret));
return ret;
}
static int wait_recvs()
{
struct fi_cq_tagged_entry entry;
int ret;
if (opts.comp_method == FT_COMP_SREAD) {
ret = fi_cq_sread(rxcq, &entry, 1, NULL, -1);
} else {
do {
ret = fi_cq_read(rxcq, &entry, 1);
} while (ret == -FI_EAGAIN);
}
if ((ret == 1) && send_data) {
if (entry.data != opts.transfer_size) {
printf("ERROR incorrect remote CQ data value. Got %lu, expected %d\n",
(unsigned long)entry.data, opts.transfer_size);
return -FI_EOTHER;
}
}
if (ret < 1)
printf("ERROR fi_cq_(s)read returned %d %s\n", ret, fi_strerror(-ret));
return ret;
}
static int send_xfer(int size)
{
struct fi_cq_entry comp;
int ret;
while (!credits) {
ret = fi_cq_read(scq, &comp, 1);
if (ret > 0) {
goto post;
} else if (ret < 0 && ret != -FI_EAGAIN) {
if (ret == -FI_EAVAIL) {
cq_readerr(scq, "scq");
} else {
FT_PRINTERR("fi_cq_read", ret);
}
return ret;
}
}
credits--;
post:
ret = fi_send(ep, buf, (size_t) size, fi_mr_desc(mr), remote_fi_addr,
&fi_ctx_send);
if (ret)
FT_PRINTERR("fi_send", ret);
return ret;
}
void do_read(int len)
{
int ret;
ssize_t sz;
struct fi_cq_tagged_entry cqe;
#define READ_CTX 0x4e3dda1aULL
init_data(source, len, 0);
init_data(target, len, 0xad);
sz = fi_read(ep[0], source, len,
loc_mr, gni_addr[1], (uint64_t)target, mr_key,
(void *)READ_CTX);
cr_assert_eq(sz, 0);
while ((ret = fi_cq_read(send_cq, &cqe, 1)) == -FI_EAGAIN) {
pthread_yield();
}
cr_assert_eq(ret, 1);
rdm_rma_check_tcqe(&cqe, (void *)READ_CTX, FI_RMA | FI_READ, 0);
rdm_rma_check_cntrs(0, 1, 0, 0);
dbg_printf("got read context event!\n");
cr_assert(check_data(source, target, len), "Data mismatch");
}
void sep_read(int index, int len)
{
int ret;
ssize_t sz;
struct fi_cq_tagged_entry cqe = { (void *) -1, UINT_MAX, UINT_MAX,
(void *) -1, UINT_MAX, UINT_MAX };
uint64_t w[2] = {0}, r[2] = {0}, w_e[2] = {0}, r_e[2] = {0};
#define READ_CTX 0x4e3dda1aULL
sep_init_data(source, len, 0);
sep_init_data(target, len, 0xad);
sz = fi_read(tx_ep[0][index], source, len,
loc_mr[0], rx_addr[index], (uint64_t)target, mr_key[1],
(void *)READ_CTX);
cr_assert_eq(sz, 0);
while ((ret = fi_cq_read(tx_cq[0][index], &cqe, 1)) == -FI_EAGAIN) {
pthread_yield();
}
cr_assert_eq(ret, 1);
sep_check_tcqe(&cqe, (void *)READ_CTX, FI_RMA | FI_READ, 0);
r[0] = 1;
sep_check_cntrs(w, r, w_e, r_e);
cr_assert(sep_check_data(source, target, len), "Data mismatch");
}
static int rxm_msg_cq_read(struct util_cq *util_cq, struct fid_cq *cq,
struct fi_cq_tagged_entry *comp)
{
struct util_cq_err_entry *entry;
int ret;
ret = fi_cq_read(cq, comp, 1);
if (ret == -FI_EAVAIL) {
entry = calloc(1, sizeof(*entry));
if (!entry) {
FI_WARN(&rxm_prov, FI_LOG_CQ,
"Unable to allocate util_cq_err_entry\n");
return -FI_ENOMEM;
}
OFI_CQ_READERR(&rxm_prov, FI_LOG_CQ, cq, ret, entry->err_entry);
if (ret < 0) {
free(entry);
return ret;
}
slist_insert_tail(&entry->list_entry, &util_cq->err_list);
comp->flags = UTIL_FLAG_ERROR;
}
return ret;
}
static int send_xfer(int size)
{
struct fi_cq_entry comp;
int ret;
while (!credits) {
ret = fi_cq_read(scq, &comp, 1);
if (ret > 0) {
goto post;
} else if (ret < 0) {
if (ret == -FI_EAVAIL) {
cq_readerr(scq, "scq");
} else {
printf("scq read %d (%s)\n", ret, fi_strerror(-ret));
}
return ret;
}
}
credits--;
post:
ret = fi_send(ep, buf, (size_t) size, fi_mr_desc(mr), NULL);
if (ret)
printf("fi_send %d (%s)\n", ret, fi_strerror(-ret));
return ret;
}
void sep_writev(int index, int len)
{
int ret;
ssize_t sz;
struct fi_cq_tagged_entry cqe = { (void *) -1, UINT_MAX, UINT_MAX,
(void *) -1, UINT_MAX, UINT_MAX };
struct iovec iov;
uint64_t w[2] = {0}, r[2] = {0}, w_e[2] = {0}, r_e[2] = {0};
iov.iov_base = source;
iov.iov_len = len;
sep_init_data(source, len, 0x25 + index);
sep_init_data(target, len, 0);
sz = fi_writev(tx_ep[0][index], &iov, (void **)loc_mr, 1,
gni_addr[1], (uint64_t)target, mr_key[1],
target);
cr_assert_eq(sz, 0);
while ((ret = fi_cq_read(tx_cq[0][index], &cqe, 1)) == -FI_EAGAIN) {
pthread_yield();
}
cr_assert_eq(ret, 1);
sep_check_tcqe(&cqe, target, FI_RMA | FI_WRITE, 0);
w[0] = 1;
sep_check_cntrs(w, r, w_e, r_e);
cr_assert(sep_check_data(source, target, len), "Data mismatch");
}
int wait_for_recv_completion(int num_completions)
{
int i, ret;
struct fi_cq_data_entry comp;
while (num_completions > 0) {
ret = fi_cq_read(rxcq, &comp, 1);
if (ret == -FI_EAGAIN)
continue;
if (ret < 0) {
FT_PRINTERR("fi_cq_read", ret);
return ret;
}
if (comp.len)
num_completions--;
if (comp.flags & FI_MULTI_RECV) {
i = (comp.op_context == &ctx_multi_recv[0]) ? 0 : 1;
ret = fi_recv(ep, rx_buf + (rx_size / 2) * i,
rx_size / 2, fi_mr_desc(mr_multi_recv),
0, &ctx_multi_recv[i]);
if (ret) {
FT_PRINTERR("fi_recv", ret);
return ret;
}
}
}
return 0;
}
void sep_atomic_rw(int index)
{
int ret;
ssize_t sz;
struct fi_cq_tagged_entry cqe = { (void *) -1, UINT_MAX, UINT_MAX,
(void *) -1, UINT_MAX, UINT_MAX };
uint64_t operand = SOURCE_DATA;
uint64_t w[NUMEPS] = {0}, r[NUMEPS] = {0}, w_e[NUMEPS] = {0};
uint64_t r_e[NUMEPS] = {0};
/* u64 */
*((uint64_t *)source) = FETCH_SOURCE_DATA;
*((uint64_t *)target) = TARGET_DATA;
sz = fi_fetch_atomic(tx_ep[0][index], &operand, 1, NULL, source,
loc_mr[0], rx_addr[index], (uint64_t)target,
mr_key[1], FI_UINT64, FI_SUM, target);
cr_assert_eq(sz, 0);
while ((ret = fi_cq_read(tx_cq[0][index], &cqe, 1)) == -FI_EAGAIN) {
pthread_yield();
}
cr_assert_eq(ret, 1);
sep_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
r[0] = 1;
sep_check_cntrs(w, r, w_e, r_e);
ret = *((uint64_t *)target) == (SOURCE_DATA + TARGET_DATA);
cr_assert(ret, "Data mismatch");
ret = *((uint64_t *)source) == TARGET_DATA;
cr_assert(ret, "Fetch data mismatch");
}
