void do_writedata(int len)
{
int ret;
ssize_t sz;
struct fi_cq_tagged_entry cqe, dcqe;
#define WRITE_DATA 0x5123da1a145
init_data(source, len, 0x23);
init_data(target, len, 0);
sz = fi_writedata(ep[0], source, len, loc_mr, WRITE_DATA,
gni_addr[1], (uint64_t)target, mr_key,
target);
cr_assert_eq(sz, 0);
while ((ret = fi_cq_read(send_cq, &cqe, 1)) == -FI_EAGAIN) {
pthread_yield();
}
cr_assert_eq(ret, 1);
rdm_rma_check_tcqe(&cqe, target, FI_RMA | FI_WRITE, 0);
rdm_rma_check_cntrs(1, 0, 0, 0);
dbg_printf("got write context event!\n");
cr_assert(check_data(source, target, len), "Data mismatch");
while ((ret = fi_cq_read(recv_cq, &dcqe, 1)) == -FI_EAGAIN) {
pthread_yield();
}
cr_assert(ret != FI_SUCCESS, "Missing remote data");
rdm_rma_check_tcqe(&dcqe, NULL,
(FI_RMA | FI_REMOTE_WRITE | FI_REMOTE_CQ_DATA),
WRITE_DATA);
}
static int write_data_with_cq_data(size_t size)
{
int ret;
ret = fi_writedata(ep, buf, size, fi_mr_desc(mr),
cq_data, 0, remote.addr, remote.key, ep);
if (ret) {
FT_PRINTERR("fi_writedata", ret);
return ret;
}
return 0;
}
void sep_writedata(int index, int len)
{
int ret;
ssize_t sz;
struct fi_cq_tagged_entry cqe = { (void *) -1, UINT_MAX, UINT_MAX,
(void *) -1, UINT_MAX, UINT_MAX };
struct fi_cq_tagged_entry dcqe = { (void *) -1, UINT_MAX, UINT_MAX,
(void *) -1, UINT_MAX, UINT_MAX };
uint64_t w[2] = {0}, r[2] = {0}, w_e[2] = {0}, r_e[2] = {0};
#define WRITE_DATA 0x5123da1a145
sep_init_data(source, len, 0x43 + index);
sep_init_data(target, len, 0);
sz = fi_writedata(tx_ep[0][index], source, len, loc_mr[0], WRITE_DATA,
rx_addr[index], (uint64_t)target, mr_key[1],
target);
cr_assert_eq(sz, 0);
while ((ret = fi_cq_read(tx_cq[0][index], &cqe, 1)) == -FI_EAGAIN) {
pthread_yield();
}
cr_assert_eq(ret, 1);
sep_check_tcqe(&cqe, target, FI_RMA | FI_WRITE, 0);
w[0] = 1;
sep_check_cntrs(w, r, w_e, r_e);
cr_assert(sep_check_data(source, target, len), "Data mismatch");
while ((ret = fi_cq_read(rx_cq[1][index], &dcqe, 1)) == -FI_EAGAIN) {
pthread_yield();
}
cr_assert(ret != FI_SUCCESS, "Missing remote data");
sep_check_tcqe(&dcqe, NULL,
(FI_RMA | FI_REMOTE_WRITE | FI_REMOTE_CQ_DATA),
WRITE_DATA);
}
void api_cq_send_recv(int len)
{
ssize_t sz;
struct iovec iov;
struct fi_msg_rma rma_msg;
struct fi_rma_iov rma_iov;
iov.iov_base = NULL;
iov.iov_len = 0;
api_cq_init_data(source, len, 0xab);
api_cq_init_data(target, len, 0);
sz = fi_send(ep[0], source, len, loc_mr[0], gni_addr[1], target);
api_cq_send_allowed(sz, cq_bind_flags, "fi_send");
sz = fi_recv(ep[1], target, len, rem_mr[1], gni_addr[0], source);
api_cq_recv_allowed(sz, cq_bind_flags, "fi_recv");
sz = fi_write(ep[0], source, len,
loc_mr[0], gni_addr[1], (uint64_t)target, mr_key[1],
target);
api_cq_send_allowed(sz, cq_bind_flags, "fi_write");
sz = fi_writev(ep[0], &iov, (void **)loc_mr, 1,
gni_addr[1], (uint64_t)target, mr_key[1],
target);
api_cq_send_allowed(sz, cq_bind_flags, "fi_writev");
iov.iov_len = len;
iov.iov_base = source;
rma_iov.addr = (uint64_t)target;
rma_iov.len = len;
rma_iov.key = mr_key[1];
rma_msg.msg_iov = &iov;
rma_msg.desc = (void **)loc_mr;
rma_msg.iov_count = 1;
rma_msg.addr = gni_addr[1];
rma_msg.rma_iov = &rma_iov;
rma_msg.rma_iov_count = 1;
rma_msg.context = target;
rma_msg.data = (uint64_t)target;
sz = fi_writemsg(ep[0], &rma_msg, 0);
api_cq_send_allowed(sz, cq_bind_flags, "fi_writemsg");
#define WRITE_DATA 0x5123da1a145
sz = fi_writedata(ep[0], source, len, loc_mr[0], WRITE_DATA,
gni_addr[1], (uint64_t)target, mr_key[1],
target);
api_cq_send_allowed(sz, cq_bind_flags, "fi_writedata");
#define READ_CTX 0x4e3dda1aULL
sz = fi_read(ep[0], source, len,
loc_mr[0], gni_addr[1], (uint64_t)target, mr_key[1],
(void *)READ_CTX);
api_cq_send_allowed(sz, cq_bind_flags, "fi_read");
sz = fi_readv(ep[0], &iov, (void **)loc_mr, 1,
gni_addr[1], (uint64_t)target, mr_key[1],
target);
api_cq_send_allowed(sz, cq_bind_flags, "fi_readv");
sz = fi_readmsg(ep[0], &rma_msg, 0);
api_cq_send_allowed(sz, cq_bind_flags, "fi_readmsg");
sz = fi_inject_write(ep[0], source, 64,
gni_addr[1], (uint64_t)target, mr_key[1]);
cr_assert_eq(sz, 0);
}
