void api_cq_send_recv(int len)
{
ssize_t sz;
struct iovec iov;
struct fi_msg_rma rma_msg;
struct fi_rma_iov rma_iov;
iov.iov_base = NULL;
iov.iov_len = 0;
api_cq_init_data(source, len, 0xab);
api_cq_init_data(target, len, 0);
sz = fi_send(ep[0], source, len, loc_mr[0], gni_addr[1], target);
api_cq_send_allowed(sz, cq_bind_flags, "fi_send");
sz = fi_recv(ep[1], target, len, rem_mr[1], gni_addr[0], source);
api_cq_recv_allowed(sz, cq_bind_flags, "fi_recv");
sz = fi_write(ep[0], source, len,
loc_mr[0], gni_addr[1], (uint64_t)target, mr_key[1],
target);
api_cq_send_allowed(sz, cq_bind_flags, "fi_write");
sz = fi_writev(ep[0], &iov, (void **)loc_mr, 1,
gni_addr[1], (uint64_t)target, mr_key[1],
target);
api_cq_send_allowed(sz, cq_bind_flags, "fi_writev");
iov.iov_len = len;
iov.iov_base = source;
rma_iov.addr = (uint64_t)target;
rma_iov.len = len;
rma_iov.key = mr_key[1];
rma_msg.msg_iov = &iov;
rma_msg.desc = (void **)loc_mr;
rma_msg.iov_count = 1;
rma_msg.addr = gni_addr[1];
rma_msg.rma_iov = &rma_iov;
rma_msg.rma_iov_count = 1;
rma_msg.context = target;
rma_msg.data = (uint64_t)target;
sz = fi_writemsg(ep[0], &rma_msg, 0);
api_cq_send_allowed(sz, cq_bind_flags, "fi_writemsg");
#define WRITE_DATA 0x5123da1a145
sz = fi_writedata(ep[0], source, len, loc_mr[0], WRITE_DATA,
gni_addr[1], (uint64_t)target, mr_key[1],
target);
api_cq_send_allowed(sz, cq_bind_flags, "fi_writedata");
#define READ_CTX 0x4e3dda1aULL
sz = fi_read(ep[0], source, len,
loc_mr[0], gni_addr[1], (uint64_t)target, mr_key[1],
(void *)READ_CTX);
api_cq_send_allowed(sz, cq_bind_flags, "fi_read");
sz = fi_readv(ep[0], &iov, (void **)loc_mr, 1,
gni_addr[1], (uint64_t)target, mr_key[1],
target);
api_cq_send_allowed(sz, cq_bind_flags, "fi_readv");
sz = fi_readmsg(ep[0], &rma_msg, 0);
api_cq_send_allowed(sz, cq_bind_flags, "fi_readmsg");
sz = fi_inject_write(ep[0], source, 64,
gni_addr[1], (uint64_t)target, mr_key[1]);
cr_assert_eq(sz, 0);
}
int main(int argc, char *argv[])
{
int i, j, peer;
int size, align_size;
char *s_buf, *r_buf;
uint64_t t_start = 0, t_end = 0, t = 0;
int op, ret;
buf_desc_t lbuf_desc;
ssize_t fi_rc;
FT_Init(&argc, &argv);
FT_Rank(&myid);
FT_Job_size(&numprocs);
hints = fi_allocinfo();
if (!hints)
return -1;
while ((op = getopt(argc, argv, "h" INFO_OPTS)) != -1) {
switch (op) {
default:
ft_parseinfo(op, optarg, hints);
break;
case '?':
case 'h':
print_usage();
return EXIT_FAILURE;
}
}
hints->ep_attr->type = FI_EP_RDM;
hints->caps = FI_MSG | FI_DIRECTED_RECV | FI_RMA;
hints->mode = FI_CONTEXT | FI_LOCAL_MR;
hints->domain_attr->mr_mode = FI_MR_BASIC;
if (numprocs != 2) {
if (myid == 0) {
fprintf(stderr, "This test requires exactly two processes\n");
}
FT_Finalize();
return -1;
}
/* Fabric initialization */
ret = init_fabric();
if (ret) {
fprintf(stderr, "Problem in fabric initialization\n");
return ret;
}
ret = init_av();
if (ret) {
fprintf(stderr, "Problem in AV initialization\n");
return ret;
}
/* Data initialization */
align_size = getpagesize();
assert(align_size <= MAX_ALIGNMENT);
s_buf = (char *) (((unsigned long) s_buf_original + (align_size - 1)) /
align_size * align_size);
r_buf = (char *) (((unsigned long) r_buf_original + (align_size - 1)) /
align_size * align_size);
ret = fi_mr_reg(dom, r_buf, MYBUFSIZE, FI_REMOTE_WRITE, 0, 0, 0, &r_mr, NULL);
if (ret) {
FT_PRINTERR("fi_mr_reg", ret);
return -1;
}
lbuf_desc.addr = (uint64_t)r_buf;
lbuf_desc.key = fi_mr_key(r_mr);
rbuf_descs = (buf_desc_t *)malloc(numprocs * sizeof(buf_desc_t));
/* Distribute memory keys */
FT_Allgather(&lbuf_desc, sizeof(lbuf_desc), rbuf_descs);
ret = fi_mr_reg(dom, s_buf, MYBUFSIZE, FI_WRITE, 0, 0, 0, &l_mr, NULL);
if (ret) {
FT_PRINTERR("fi_mr_reg", ret);
return -1;
}
if (myid == 0) {
fprintf(stdout, HEADER);
fprintf(stdout, "%-*s%*s%*s\n", 10, "# Size", FIELD_WIDTH,
"Bandwidth (MB/s)", FIELD_WIDTH, "latency");
fflush(stdout);
}
/* Bandwidth test */
for (size = 1; size <= MAX_MSG_SIZE; size *= 2) {
/* touch the data */
for (i = 0; i < size; i++) {
s_buf[i] = 'a';
r_buf[i] = 'b';
}
if (size > large_message_size) {
loop = loop_large;
skip = skip_large;
window_size = window_size_large;
}
FT_Barrier();
if (myid == 0) {
peer = 1;
for (i = 0; i < loop + skip; i++) {
if (i == skip) {
t_start = get_time_usec();
}
for (j = 0; j < window_size; j++) {
fi_rc = fi_write(ep, s_buf, size, l_mr,
fi_addrs[peer],
rbuf_descs[peer].addr,
rbuf_descs[peer].key,
(void *)(intptr_t)j);
if (fi_rc) {
FT_PRINTERR("fi_write", fi_rc);
return fi_rc;
}
}
ft_wait_for_comp(scq, window_size);
}
t_end = get_time_usec();
t = t_end - t_start;
} else if (myid == 1) {
peer = 0;
}
if (myid == 0) {
double latency = (t_end - t_start) /
(double)(loop * window_size);
double tmp = size / 1e6 * loop * window_size;
fprintf(stdout, "%-*d%*.*f%*.*f\n", 10, size,
FIELD_WIDTH, FLOAT_PRECISION,
tmp / (t / 1e6), FIELD_WIDTH,
FLOAT_PRECISION, latency);
fflush(stdout);
}
}
FT_Barrier();
fi_close(&l_mr->fid);
fi_close(&r_mr->fid);
free_ep_res();
fi_close(&ep->fid);
fi_close(&dom->fid);
fi_close(&fab->fid);
fi_freeinfo(hints);
fi_freeinfo(fi);
FT_Barrier();
FT_Finalize();
return 0;
}