void do_multirecv(int len)
{
int i, ret;
ssize_t sz;
struct fi_cq_tagged_entry s_cqe, d_cqe;
struct iovec iov;
struct fi_msg msg;
uint64_t s[NUMEPS] = {0}, r[NUMEPS] = {0}, s_e[NUMEPS] = {0};
uint64_t r_e[NUMEPS] = {0};
uint64_t flags;
int nrecvs = 3;
rdm_sr_init_data(source, len, 0xab);
rdm_sr_init_data(target, len, 0);
/* Post receives first to force matching in SMSG callback. */
iov.iov_base = target;
iov.iov_len = len * nrecvs + 63;
msg.msg_iov = &iov;
msg.desc = (void **)rem_mr;
msg.iov_count = 1;
msg.addr = gni_addr[0];
msg.context = source;
msg.data = (uint64_t)source;
sz = fi_recvmsg(ep[1], &msg, FI_MULTI_RECV);
cr_assert_eq(sz, 0);
for (i = 0; i < nrecvs; i++) {
sz = fi_send(ep[0], source, len, loc_mr[0], gni_addr[1],
target);
cr_assert_eq(sz, 0);
}
/* need to progress both CQs simultaneously for rendezvous */
do {
ret = fi_cq_read(msg_cq[0], &s_cqe, 1);
if (ret == 1) {
rdm_sr_check_cqe(&s_cqe, target, (FI_MSG|FI_SEND),
0, 0, 0, false);
s[0]++;
}
ret = fi_cq_read(msg_cq[1], &d_cqe, 1);
flags = (r[1] < (nrecvs -1 )) ? FI_MSG | FI_RECV :
FI_MSG | FI_RECV | FI_MULTI_RECV;
if (ret == 1) {
rdm_sr_check_cqe(&d_cqe, source,
flags,
target + (r[1] * len), len, 0, true);
cr_assert(rdm_sr_check_data(source, d_cqe.buf, len),
"Data mismatch");
r[1]++;
}
} while (s[0] < nrecvs || r[1] < nrecvs);
rdm_sr_check_cntrs(s, r, s_e, r_e);
dbg_printf("got context events!\n");
}
/*
ssize_t fi_sendv(struct fid_ep *ep, const struct iovec *iov, void **desc,
size_t count, fi_addr_t dest_addr, void *context);
*/
void do_sendv(int len)
{
int i, ret, iov_cnt;
int source_done = 0, dest_done = 0;
struct fi_cq_tagged_entry s_cqe, d_cqe;
ssize_t sz;
uint64_t s[NUMEPS] = {0}, r[NUMEPS] = {0}, s_e[NUMEPS] = {0};
uint64_t r_e[NUMEPS] = {0};
sz = fi_sendv(ep[0], src_iov, NULL, 0, gni_addr[1], iov_dest_buf);
cr_assert_eq(sz, -FI_EINVAL);
for (iov_cnt = 1; iov_cnt <= IOV_CNT; iov_cnt++) {
for (i = 0; i < iov_cnt; i++) {
rdm_sr_init_data(src_iov[i].iov_base, len, 0x25);
src_iov[i].iov_len = len;
}
rdm_sr_init_data(iov_dest_buf, len * iov_cnt, 0);
/*
* TODO: Register src_iov and dest_iov.
* Using NULL descriptor for now so that _gnix_send auto registers
* the buffers for rndzv messages.
*/
sz = fi_sendv(ep[0], src_iov, NULL, iov_cnt, gni_addr[1], iov_dest_buf);
cr_assert_eq(sz, 0);
sz = fi_recv(ep[1], iov_dest_buf, len * iov_cnt, iov_dest_buf_mr[1],
gni_addr[0], src_iov);
cr_assert_eq(sz, 0);
/* need to progress both CQs simultaneously for rendezvous */
do {
ret = fi_cq_read(msg_cq[0], &s_cqe, 1);
if (ret == 1) {
source_done = 1;
}
ret = fi_cq_read(msg_cq[1], &d_cqe, 1);
if (ret == 1) {
dest_done = 1;
}
} while (!(source_done && dest_done));
rdm_sr_check_cqe(&s_cqe, iov_dest_buf, (FI_MSG|FI_SEND), 0, 0, 0,
false);
rdm_sr_check_cqe(&d_cqe, src_iov, (FI_MSG|FI_RECV), iov_dest_buf,
len * iov_cnt, 0, false);
s[0] = 1; r[1] = 1;
rdm_sr_check_cntrs(s, r, s_e, r_e);
dbg_printf("got context events!\n");
cr_assert(rdm_sr_check_iov_data(src_iov, iov_dest_buf, iov_cnt, len * iov_cnt),
"Data mismatch");
source_done = dest_done = 0;
}
}
void do_sendrecvv_alignment(int slen, int dlen, int offset)
{
int i, ret, iov_cnt;
ssize_t sz;
int source_done = 0, dest_done = 0;
struct fi_cq_tagged_entry s_cqe, d_cqe;
uint64_t s[NUMEPS] = {0}, r[NUMEPS] = {0}, s_e[NUMEPS] = {0};
uint64_t r_e[NUMEPS] = {0};
uint64_t iov_s_buf = (uint64_t) iov_src_buf;
iov_s_buf += offset;
for (iov_cnt = 1; iov_cnt <= IOV_CNT; iov_cnt++) {
for (i = 0; i < iov_cnt; i++) {
d_iov[i].iov_base = dest_iov[i].iov_base;
d_iov[i].iov_base = (void *) ((uint64_t)d_iov[i].iov_base + offset);
rdm_sr_init_data(d_iov[i].iov_base, dlen - offset, 0);
d_iov[i].iov_len = dlen - offset;
}
rdm_sr_init_data((void *) iov_s_buf, (slen - offset) * iov_cnt, 0xab);
sz = fi_send(ep[0], (void *) iov_s_buf, (slen - offset) * iov_cnt, NULL, gni_addr[1],
d_iov);
cr_assert_eq(sz, 0);
sz = fi_recvv(ep[1], d_iov, NULL, iov_cnt, gni_addr[0], (void *) iov_s_buf);
cr_assert_eq(sz, 0);
/* need to progress both CQs simultaneously for rendezvous */
do {
ret = fi_cq_read(msg_cq[0], &s_cqe, 1);
if (ret == 1) {
source_done = 1;
}
ret = fi_cq_read(msg_cq[1], &d_cqe, 1);
if (ret == 1) {
dest_done = 1;
}
} while (!(source_done && dest_done));
rdm_sr_check_cqe(&s_cqe, d_iov, (FI_MSG|FI_SEND), 0, 0, 0, false);
rdm_sr_check_cqe(&d_cqe, (void *) iov_s_buf, (FI_MSG|FI_RECV), d_iov,
MIN((slen - offset) * iov_cnt, (dlen - offset) * iov_cnt), 0, false);
s[0] = 1; r[1] = 1;
rdm_sr_check_cntrs(s, r, s_e, r_e);
dbg_printf("got context events!\n");
cr_assert(rdm_sr_check_iov_data(d_iov, (void *) iov_s_buf,
iov_cnt, (slen - offset) * iov_cnt),
"Data mismatch");
source_done = dest_done = 0;
}
}
/*
ssize_t (*recvv)(struct fid_ep *ep, const struct iovec *iov, void **desc,
size_t count, fi_addr_t src_addr, void *context);
*/
void do_recvv(int len)
{
int i, ret, iov_cnt;
ssize_t sz;
int source_done = 0, dest_done = 0;
struct fi_cq_tagged_entry s_cqe, d_cqe;
uint64_t s[NUMEPS] = {0}, r[NUMEPS] = {0}, s_e[NUMEPS] = {0};
uint64_t r_e[NUMEPS] = {0};
sz = fi_recvv(ep[1], NULL, NULL, IOV_CNT, gni_addr[0], iov_src_buf);
cr_assert_eq(sz, -FI_EINVAL);
sz = fi_recvv(ep[1], dest_iov, NULL, IOV_CNT + 1, gni_addr[0], iov_src_buf);
cr_assert_eq(sz, -FI_EINVAL);
for (iov_cnt = 1; iov_cnt <= IOV_CNT; iov_cnt++) {
rdm_sr_init_data(iov_src_buf, len * iov_cnt, 0xab);
for (i = 0; i < iov_cnt; i++) {
rdm_sr_init_data(dest_iov[i].iov_base, len, 0);
dest_iov[i].iov_len = len;
}
sz = fi_send(ep[0], iov_src_buf, len * iov_cnt, NULL, gni_addr[1],
dest_iov);
cr_assert_eq(sz, 0);
sz = fi_recvv(ep[1], dest_iov, NULL, iov_cnt, gni_addr[0], iov_src_buf);
cr_assert_eq(sz, 0);
/* need to progress both CQs simultaneously for rendezvous */
do {
ret = fi_cq_read(msg_cq[0], &s_cqe, 1);
if (ret == 1) {
source_done = 1;
}
ret = fi_cq_read(msg_cq[1], &d_cqe, 1);
if (ret == 1) {
dest_done = 1;
}
} while (!(source_done && dest_done));
rdm_sr_check_cqe(&s_cqe, dest_iov, (FI_MSG|FI_SEND), 0, 0, 0, false);
rdm_sr_check_cqe(&d_cqe, iov_src_buf, (FI_MSG|FI_RECV), dest_iov,
len * iov_cnt, 0, false);
s[0] = 1; r[1] = 1;
rdm_sr_check_cntrs(s, r, s_e, r_e);
dbg_printf("got context events!\n");
cr_assert(rdm_sr_check_iov_data(dest_iov, iov_src_buf, iov_cnt, len * iov_cnt),
"Data mismatch");
source_done = dest_done = 0;
}
}
void do_sendmsgdata(int len)
{
int ret;
ssize_t sz;
int source_done = 0, dest_done = 0;
struct fi_cq_tagged_entry s_cqe, d_cqe;
struct fi_msg msg;
struct iovec iov;
uint64_t s[NUMEPS] = {0}, r[NUMEPS] = {0}, s_e[NUMEPS] = {0};
uint64_t r_e[NUMEPS] = {0};
iov.iov_base = source;
iov.iov_len = len;
msg.msg_iov = &iov;
msg.desc = (void **)loc_mr;
msg.iov_count = 1;
msg.addr = gni_addr[1];
msg.context = target;
msg.data = (uint64_t)source;
rdm_sr_init_data(source, len, 0xef);
rdm_sr_init_data(target, len, 0);
sz = fi_sendmsg(ep[0], &msg, FI_REMOTE_CQ_DATA);
cr_assert_eq(sz, 0);
sz = fi_recv(ep[1], target, len, rem_mr[0], gni_addr[0], source);
cr_assert_eq(sz, 0);
/* need to progress both CQs simultaneously for rendezvous */
do {
ret = fi_cq_read(msg_cq[0], &s_cqe, 1);
if (ret == 1) {
source_done = 1;
}
ret = fi_cq_read(msg_cq[1], &d_cqe, 1);
if (ret == 1) {
dest_done = 1;
}
} while (!(source_done && dest_done));
rdm_sr_check_cqe(&s_cqe, target, (FI_MSG|FI_SEND), 0, 0, 0);
rdm_sr_check_cqe(&d_cqe, source, (FI_MSG|FI_RECV|FI_REMOTE_CQ_DATA),
target, len, (uint64_t)source);
s[0] = 1; r[1] = 1;
rdm_sr_check_cntrs(s, r, s_e, r_e);
dbg_printf("got context events!\n");
cr_assert(rdm_sr_check_data(source, target, len), "Data mismatch");
}
/*
* ssize_t fi_send(struct fid_ep *ep, void *buf, size_t len,
* void *desc, fi_addr_t dest_addr, void *context);
*
* ssize_t fi_recv(struct fid_ep *ep, void * buf, size_t len,
* void *desc, fi_addr_t src_addr, void *context);
*/
void do_send(int len)
{
int ret;
int source_done = 0, dest_done = 0;
int scanceled = 0, dcanceled = 0;
struct fi_cq_tagged_entry s_cqe, d_cqe;
ssize_t sz;
uint64_t s[NUMEPS] = {0}, r[NUMEPS] = {0}, s_e[NUMEPS] = {0};
uint64_t r_e[NUMEPS] = {0};
rdm_sr_init_data(source, len, 0xab);
rdm_sr_init_data(target, len, 0);
sz = fi_send(ep[0], source, len, loc_mr[0], gni_addr[1], target);
cr_assert_eq(sz, 0);
sz = fi_recv(ep[1], target, len, rem_mr[1], gni_addr[0], source);
cr_assert_eq(sz, 0);
/* need to progress both CQs simultaneously for rendezvous */
do {
ret = fi_cq_read(msg_cq[0], &s_cqe, 1);
if (ret == 1) {
source_done = 1;
}
if (ret == -FI_EAVAIL) {
if (rdm_sr_check_canceled(msg_cq[0]))
scanceled = 1;
}
ret = fi_cq_read(msg_cq[1], &d_cqe, 1);
if (ret == 1) {
dest_done = 1;
}
if (ret == -FI_EAVAIL) {
if (rdm_sr_check_canceled(msg_cq[1]))
dcanceled = 1;
}
} while (!((source_done || scanceled) && (dest_done || dcanceled)));
/* no further checking needed */
if (dgram_should_fail && (scanceled || dcanceled))
return;
rdm_sr_check_cqe(&s_cqe, target, (FI_MSG|FI_SEND), 0, 0, 0);
rdm_sr_check_cqe(&d_cqe, source, (FI_MSG|FI_RECV), target, len, 0);
s[0] = 1; r[1] = 1;
rdm_sr_check_cntrs(s, r, s_e, r_e);
dbg_printf("got context events!\n");
cr_assert(rdm_sr_check_data(source, target, len), "Data mismatch");
}
/*
ssize_t (*recvv)(struct fid_ep *ep, const struct iovec *iov, void **desc,
size_t count, fi_addr_t src_addr, void *context);
*/
void do_recvv(int len)
{
int ret;
ssize_t sz;
int source_done = 0, dest_done = 0;
struct fi_cq_tagged_entry s_cqe, d_cqe;
struct iovec iov;
uint64_t s[NUMEPS] = {0}, r[NUMEPS] = {0}, s_e[NUMEPS] = {0};
uint64_t r_e[NUMEPS] = {0};
rdm_sr_init_data(source, len, 0xab);
rdm_sr_init_data(target, len, 0);
sz = fi_send(ep[0], source, len, loc_mr[0], gni_addr[1], target);
cr_assert_eq(sz, 0);
iov.iov_base = target;
iov.iov_len = len;
sz = fi_recvv(ep[1], &iov, (void **)rem_mr, 1, gni_addr[0], source);
cr_assert_eq(sz, 0);
/* need to progress both CQs simultaneously for rendezvous */
do {
ret = fi_cq_read(msg_cq[0], &s_cqe, 1);
if (ret == 1) {
source_done = 1;
}
ret = fi_cq_read(msg_cq[1], &d_cqe, 1);
if (ret == 1) {
dest_done = 1;
}
} while (!(source_done && dest_done));
rdm_sr_check_cqe(&s_cqe, target, (FI_MSG|FI_SEND), 0, 0, 0);
rdm_sr_check_cqe(&d_cqe, source, (FI_MSG|FI_RECV), target, len, 0);
s[0] = 1; r[1] = 1;
rdm_sr_check_cntrs(s, r, s_e, r_e);
dbg_printf("got context events!\n");
cr_assert(rdm_sr_check_data(source, target, len), "Data mismatch");
}
void do_sendrecv_buf(void *p, void *t, int send_len, int recv_len)
{
int ret;
int source_done = 0, dest_done = 0;
struct fi_cq_tagged_entry s_cqe, d_cqe;
ssize_t sz;
int xfer_len;
uint64_t s[NUMEPS] = {0}, r[NUMEPS] = {0}, s_e[NUMEPS] = {0};
uint64_t r_e[NUMEPS] = {0};
rdm_sr_init_data(p, send_len, 0xab);
rdm_sr_init_data(t, recv_len, 0);
sz = fi_send(ep[0], p, send_len, loc_mr[0], gni_addr[1], t);
cr_assert_eq(sz, 0);
sz = fi_recv(ep[1], t, recv_len, rem_mr[0], gni_addr[0], p);
cr_assert_eq(sz, 0);
/* need to progress both CQs simultaneously for rendezvous */
do {
ret = fi_cq_read(msg_cq[0], &s_cqe, 1);
if (ret == 1) {
source_done = 1;
}
ret = fi_cq_read(msg_cq[1], &d_cqe, 1);
if (ret == 1) {
dest_done = 1;
}
} while (!(source_done && dest_done));
xfer_len = MIN(send_len, recv_len);
rdm_sr_check_cqe(&s_cqe, t, (FI_MSG|FI_SEND), 0, 0, 0);
rdm_sr_check_cqe(&d_cqe, p, (FI_MSG|FI_RECV), t, xfer_len, 0);
s[0] = 1; r[1] = 1;
rdm_sr_check_cntrs(s, r, s_e, r_e);
dbg_printf("got context events!\n");
cr_assert(rdm_sr_check_data(p, t, xfer_len), "Data mismatch");
}
void do_send_autoreg_uncached_nolazydereg(int len)
{
int ret;
int source_done = 0, dest_done = 0;
struct fi_cq_tagged_entry s_cqe, d_cqe;
ssize_t sz;
uint64_t s[NUMEPS] = {0}, r[NUMEPS] = {0}, s_e[NUMEPS] = {0};
uint64_t r_e[NUMEPS] = {0};
rdm_sr_init_data(uc_source, len, 0xab);
rdm_sr_init_data(uc_target, len, 0);
sz = fi_send(ep[0], uc_source, len, NULL, gni_addr[1], uc_target);
cr_assert_eq(sz, 0);
sz = fi_recv(ep[1], uc_target, len, NULL, gni_addr[0], uc_source);
cr_assert_eq(sz, 0);
/* need to progress both CQs simultaneously for rendezvous */
do {
ret = fi_cq_read(msg_cq[0], &s_cqe, 1);
if (ret == 1) {
source_done = 1;
}
ret = fi_cq_read(msg_cq[1], &d_cqe, 1);
if (ret == 1) {
dest_done = 1;
}
} while (!(source_done && dest_done));
rdm_sr_check_cqe(&s_cqe, uc_target, (FI_MSG|FI_SEND), 0, 0, 0);
rdm_sr_check_cqe(&d_cqe, uc_source, (FI_MSG|FI_RECV),
uc_target, len, 0);
s[0] = 1; r[1] = 1;
rdm_sr_check_cntrs(s, r, s_e, r_e);
dbg_printf("got context events!\n");
cr_assert(rdm_sr_check_data(uc_source, uc_target, len),
"Data mismatch");
}
/*
* this test attempts to demonstrate issue ofi-cray/libfabric-cray#559.
* For domains with control_progress AUTO, this test should not hang.
*/
Test(rdm_sr, inject_progress)
{
int ret, len = 64;
ssize_t sz;
struct fi_cq_tagged_entry cqe;
uint64_t s[NUMEPS] = {0}, r[NUMEPS] = {0}, s_e[NUMEPS] = {0};
uint64_t r_e[NUMEPS] = {0};
rdm_sr_init_data(source, len, 0x23);
rdm_sr_init_data(target, len, 0);
sz = fi_inject(ep[0], source, len, gni_addr[1]);
cr_assert_eq(sz, 0);
sz = fi_recv(ep[1], target, len, rem_mr[1], gni_addr[0], source);
cr_assert_eq(sz, 0);
/*
* do progress until send counter is updated.
* This works because we have FI_PROGRESS_AUTO for control progress
*/
while (fi_cntr_read(send_cntr[0]) < 1) {
pthread_yield();
}
while ((ret = fi_cq_read(msg_cq[1], &cqe, 1)) == -FI_EAGAIN) {
pthread_yield();
}
cr_assert_eq(ret, 1);
rdm_sr_check_cqe(&cqe, source, (FI_MSG|FI_RECV),
target, len, (uint64_t)source);
dbg_printf("got recv context event!\n");
s[0] = 1; r[1] = 1;
rdm_sr_check_cntrs(s, r, s_e, r_e);
/* make sure inject does not generate a send competion */
cr_assert_eq(fi_cq_read(msg_cq[0], &cqe, 1), -FI_EAGAIN);
cr_assert(rdm_sr_check_data(source, target, len), "Data mismatch");
}
/*
ssize_t fi_injectdata(struct fid_ep *ep, const void *buf, size_t len,
uint64_t data, fi_addr_t dest_addr);
*/
void do_injectdata(int len)
{
int ret;
ssize_t sz;
struct fi_cq_tagged_entry cqe;
uint64_t s[NUMEPS] = {0}, r[NUMEPS] = {0}, s_e[NUMEPS] = {0};
uint64_t r_e[NUMEPS] = {0};
rdm_sr_init_data(source, len, 0xab);
rdm_sr_init_data(target, len, 0);
sz = fi_injectdata(ep[0], source, len, (uint64_t)source, gni_addr[1]);
cr_assert_eq(sz, 0);
sz = fi_recv(ep[1], target, len, rem_mr[0], gni_addr[0], source);
cr_assert_eq(sz, 0);
while ((ret = fi_cq_read(msg_cq[1], &cqe, 1)) == -FI_EAGAIN) {
pthread_yield();
/* Manually progress connection to domain 1 */
fi_cq_read(msg_cq[0], &cqe, 1);
}
rdm_sr_check_cqe(&cqe, source, (FI_MSG|FI_RECV|FI_REMOTE_CQ_DATA),
target, len, (uint64_t)source);
dbg_printf("got recv context event!\n");
/* don't progress until send counter is updated */
while (fi_cntr_read(send_cntr[0]) < 1) {
pthread_yield();
}
s[0] = 1; r[1] = 1;
rdm_sr_check_cntrs(s, r, s_e, r_e);
/* make sure inject does not generate a send competion */
cr_assert_eq(fi_cq_read(msg_cq[0], &cqe, 1), -FI_EAGAIN);
cr_assert(rdm_sr_check_data(source, target, len), "Data mismatch");
}
