/*
ssize_t (*recvv)(struct fid_ep *ep, const struct iovec *iov, void **desc,
size_t count, fi_addr_t src_addr, void *context);
*/
void do_recvv(int len)
{
int i, ret, iov_cnt;
ssize_t sz;
int source_done = 0, dest_done = 0;
struct fi_cq_tagged_entry s_cqe, d_cqe;
uint64_t s[NUMEPS] = {0}, r[NUMEPS] = {0}, s_e[NUMEPS] = {0};
uint64_t r_e[NUMEPS] = {0};
sz = fi_recvv(ep[1], NULL, NULL, IOV_CNT, gni_addr[0], iov_src_buf);
cr_assert_eq(sz, -FI_EINVAL);
sz = fi_recvv(ep[1], dest_iov, NULL, IOV_CNT + 1, gni_addr[0], iov_src_buf);
cr_assert_eq(sz, -FI_EINVAL);
for (iov_cnt = 1; iov_cnt <= IOV_CNT; iov_cnt++) {
rdm_sr_init_data(iov_src_buf, len * iov_cnt, 0xab);
for (i = 0; i < iov_cnt; i++) {
rdm_sr_init_data(dest_iov[i].iov_base, len, 0);
dest_iov[i].iov_len = len;
}
sz = fi_send(ep[0], iov_src_buf, len * iov_cnt, NULL, gni_addr[1],
dest_iov);
cr_assert_eq(sz, 0);
sz = fi_recvv(ep[1], dest_iov, NULL, iov_cnt, gni_addr[0], iov_src_buf);
cr_assert_eq(sz, 0);
/* need to progress both CQs simultaneously for rendezvous */
do {
ret = fi_cq_read(msg_cq[0], &s_cqe, 1);
if (ret == 1) {
source_done = 1;
}
ret = fi_cq_read(msg_cq[1], &d_cqe, 1);
if (ret == 1) {
dest_done = 1;
}
} while (!(source_done && dest_done));
rdm_sr_check_cqe(&s_cqe, dest_iov, (FI_MSG|FI_SEND), 0, 0, 0, false);
rdm_sr_check_cqe(&d_cqe, iov_src_buf, (FI_MSG|FI_RECV), dest_iov,
len * iov_cnt, 0, false);
s[0] = 1; r[1] = 1;
rdm_sr_check_cntrs(s, r, s_e, r_e);
dbg_printf("got context events!\n");
cr_assert(rdm_sr_check_iov_data(dest_iov, iov_src_buf, iov_cnt, len * iov_cnt),
"Data mismatch");
source_done = dest_done = 0;
}
}
ssize_t rxm_recvv(struct fid_ep *ep_fid, const struct iovec *iov, void **desc,
size_t count, fi_addr_t src_addr, void *context)
{
struct rxm_ep *rxm_ep;
rxm_ep = container_of(ep_fid, struct rxm_ep, util_ep.ep_fid.fid);
return fi_recvv(rxm_ep->srx_ctx, iov, desc, count, 0, context);
}
void do_sendrecvv_alignment(int slen, int dlen, int offset)
{
int i, ret, iov_cnt;
ssize_t sz;
int source_done = 0, dest_done = 0;
struct fi_cq_tagged_entry s_cqe, d_cqe;
uint64_t s[NUMEPS] = {0}, r[NUMEPS] = {0}, s_e[NUMEPS] = {0};
uint64_t r_e[NUMEPS] = {0};
uint64_t iov_s_buf = (uint64_t) iov_src_buf;
iov_s_buf += offset;
for (iov_cnt = 1; iov_cnt <= IOV_CNT; iov_cnt++) {
for (i = 0; i < iov_cnt; i++) {
d_iov[i].iov_base = dest_iov[i].iov_base;
d_iov[i].iov_base = (void *) ((uint64_t)d_iov[i].iov_base + offset);
rdm_sr_init_data(d_iov[i].iov_base, dlen - offset, 0);
d_iov[i].iov_len = dlen - offset;
}
rdm_sr_init_data((void *) iov_s_buf, (slen - offset) * iov_cnt, 0xab);
sz = fi_send(ep[0], (void *) iov_s_buf, (slen - offset) * iov_cnt, NULL, gni_addr[1],
d_iov);
cr_assert_eq(sz, 0);
sz = fi_recvv(ep[1], d_iov, NULL, iov_cnt, gni_addr[0], (void *) iov_s_buf);
cr_assert_eq(sz, 0);
/* need to progress both CQs simultaneously for rendezvous */
do {
ret = fi_cq_read(msg_cq[0], &s_cqe, 1);
if (ret == 1) {
source_done = 1;
}
ret = fi_cq_read(msg_cq[1], &d_cqe, 1);
if (ret == 1) {
dest_done = 1;
}
} while (!(source_done && dest_done));
rdm_sr_check_cqe(&s_cqe, d_iov, (FI_MSG|FI_SEND), 0, 0, 0, false);
rdm_sr_check_cqe(&d_cqe, (void *) iov_s_buf, (FI_MSG|FI_RECV), d_iov,
MIN((slen - offset) * iov_cnt, (dlen - offset) * iov_cnt), 0, false);
s[0] = 1; r[1] = 1;
rdm_sr_check_cntrs(s, r, s_e, r_e);
dbg_printf("got context events!\n");
cr_assert(rdm_sr_check_iov_data(d_iov, (void *) iov_s_buf,
iov_cnt, (slen - offset) * iov_cnt),
"Data mismatch");
source_done = dest_done = 0;
}
}
static void sep_recvv(int index, int len)
{
int i, iov_cnt;
ssize_t sz;
struct fi_cq_tagged_entry s_cqe, d_cqe;
uint64_t s[NUMEPS] = {0}, r[NUMEPS] = {0}, s_e[NUMEPS] = {0};
uint64_t r_e[NUMEPS] = {0};
for (iov_cnt = 1; iov_cnt <= IOV_CNT; iov_cnt++) {
for (i = 0; i < iov_cnt; i++) {
sep_init_data(src_iov[i].iov_base, len, 0x25 + index);
src_iov[i].iov_len = len;
}
for (i = 0; i < iov_cnt; i++) {
sep_init_data(dest_iov[i].iov_base, len, 0);
dest_iov[i].iov_len = len;
}
sz = fi_sendv(tx_ep[0][index], src_iov, NULL, iov_cnt,
rx_addr[index], iov_dest_buf);
cr_assert_eq(sz, 0);
sz = fi_recvv(rx_ep[1][index], dest_iov, NULL, iov_cnt,
FI_ADDR_UNSPEC, iov_src_buf);
cr_assert_eq(sz, 0);
/* reset cqe */
s_cqe.op_context = s_cqe.buf = (void *) -1;
s_cqe.flags = s_cqe.len = s_cqe.data = s_cqe.tag = UINT_MAX;
d_cqe.op_context = d_cqe.buf = (void *) -1;
d_cqe.flags = d_cqe.len = d_cqe.data = d_cqe.tag = UINT_MAX;
wait_for_cqs(tx_cq[0][index], rx_cq[1][index], &s_cqe, &d_cqe);
sep_check_cqe(&s_cqe, iov_dest_buf, (FI_MSG|FI_SEND),
0, 0, 0, false);
sep_check_cqe(&d_cqe, iov_src_buf, (FI_MSG|FI_RECV),
iov_dest_buf, len * iov_cnt, 0, false);
s[0] = 1; r[1] = 1;
sep_check_cntrs(s, r, s_e, r_e);
cr_assert(check_iov_data(src_iov, dest_iov, iov_cnt),
"Data mismatch");
}
}
/*
ssize_t (*recvv)(struct fid_ep *ep, const struct iovec *iov, void **desc,
size_t count, fi_addr_t src_addr, void *context);
*/
void do_recvv(int len)
{
int ret;
ssize_t sz;
int source_done = 0, dest_done = 0;
struct fi_cq_tagged_entry s_cqe, d_cqe;
struct iovec iov;
uint64_t s[NUMEPS] = {0}, r[NUMEPS] = {0}, s_e[NUMEPS] = {0};
uint64_t r_e[NUMEPS] = {0};
rdm_sr_init_data(source, len, 0xab);
rdm_sr_init_data(target, len, 0);
sz = fi_send(ep[0], source, len, loc_mr[0], gni_addr[1], target);
cr_assert_eq(sz, 0);
iov.iov_base = target;
iov.iov_len = len;
sz = fi_recvv(ep[1], &iov, (void **)rem_mr, 1, gni_addr[0], source);
cr_assert_eq(sz, 0);
/* need to progress both CQs simultaneously for rendezvous */
do {
ret = fi_cq_read(msg_cq[0], &s_cqe, 1);
if (ret == 1) {
source_done = 1;
}
ret = fi_cq_read(msg_cq[1], &d_cqe, 1);
if (ret == 1) {
dest_done = 1;
}
} while (!(source_done && dest_done));
rdm_sr_check_cqe(&s_cqe, target, (FI_MSG|FI_SEND), 0, 0, 0);
rdm_sr_check_cqe(&d_cqe, source, (FI_MSG|FI_RECV), target, len, 0);
s[0] = 1; r[1] = 1;
rdm_sr_check_cntrs(s, r, s_e, r_e);
dbg_printf("got context events!\n");
cr_assert(rdm_sr_check_data(source, target, len), "Data mismatch");
}
/*
ssize_t (*recvv)(struct fid_ep *ep, const struct iovec *iov, void **desc,
size_t count, fi_addr_t src_addr, void *context);
*/
void do_recvv(int len)
{
int ret;
ssize_t sz;
struct fi_cq_entry cqe;
struct iovec iov;
rdm_sr_init_data(source, len, 0xab);
rdm_sr_init_data(target, len, 0);
sz = fi_send(ep[0], source, len, loc_mr, gni_addr[1], target);
cr_assert_eq(sz, 0);
iov.iov_base = target;
iov.iov_len = len;
sz = fi_recvv(ep[1], &iov, (void **)&rem_mr, 1, gni_addr[0], source);
cr_assert_eq(sz, 0);
while ((ret = fi_cq_read(msg_cq[0], &cqe, 1)) == -FI_EAGAIN) {
pthread_yield();
}
cr_assert_eq(ret, 1);
cr_assert_eq((uint64_t)cqe.op_context, (uint64_t)target);
dbg_printf("got send context event!\n");
while ((ret = fi_cq_read(msg_cq[1], &cqe, 1)) == -FI_EAGAIN) {
pthread_yield();
}
cr_assert_eq(ret, 1);
cr_assert_eq((uint64_t)cqe.op_context, (uint64_t)source);
dbg_printf("got recv context event!\n");
cr_assert(rdm_sr_check_data(source, target, len), "Data mismatch");
}
double calc_bw(int rank, int num_pairs, int window_size, struct iovec *s_iov,
struct iovec *r_iov, int iov_cnt, char *s_buf, int s_buf_len,
char *r_buf, int r_buf_len, enum send_recv_type_e type)
{
uint64_t t_start = 0, t_end = 0, t = 0, maxtime = 0, *ts;
double bw = 0;
int c, i, j, target;
int loop, skip;
size_t cum_size = 0;
int mult = (DEFAULT_WINDOW / window_size) > 0 ? (DEFAULT_WINDOW /
window_size) : 1;
int __attribute__((unused)) fi_rc;
char r_fin[4] = {'b', 'b', 'b', 'b'};
char s_fin[4] = {'a', 'a', 'a', 'a'};
for (c = 0; c < iov_cnt; c++) {
for (i = 0; i < s_iov[c].iov_len; i++) {
((char *) s_iov[c].iov_base)[i] = 'a';
}
for (i = 0; i < r_iov[c].iov_len; i++) {
((char *) r_iov[c].iov_base)[i] = 'b';
}
/* Size will be all the receiver can hold */
cum_size += r_iov[c].iov_len;
}
for (c = 0; c < s_buf_len; c++) {
s_buf[c]= 'a';
}
for (c = 0; c < r_buf_len; c++) {
r_buf[c]= 'b';
}
if (cum_size > LARGE_THRESHOLD) {
loop = ITERS_LARGE * mult;
skip = WARMUP_ITERS_LARGE * mult;
} else {
loop = ITERS_SMALL * mult;
skip = WARMUP_ITERS_SMALL * mult;
}
ctpm_Barrier();
if (rank < num_pairs) {
target = rank + num_pairs;
for (i = 0; i < loop + skip; i++) {
if (i == skip) {
ctpm_Barrier();
t_start = get_time_usec();
}
for (j = 0; j < window_size; j++) {
switch (type) {
case SEND_RECV:
for (c = 0; c < iov_cnt; c++) {
fi_rc = fi_send(ep, s_iov[c].iov_base,
s_iov[c].iov_len, NULL,
fi_addrs[target],
NULL);
assert(!fi_rc);
}
break;
case SENDV_RECVV:
case SENDV_RECV:
fi_rc = fi_sendv(ep, s_iov, (void **) NULL,
iov_cnt, fi_addrs[target],
NULL);
assert(!fi_rc);
break;
case SEND_RECVV:
fi_rc = fi_send(ep, s_buf,
s_buf_len, NULL,
fi_addrs[target],
NULL);
assert(!fi_rc);
break;
default:
abort();
}
}
wait_for_comp(scq, type == SEND_RECV ? window_size * iov_cnt : window_size);
fi_rc = fi_recv(ep, r_fin, 4, NULL,
fi_addrs[target], NULL);
assert(!fi_rc);
wait_for_comp(rcq, 1);
}
t_end = get_time_usec();
t = t_end - t_start;
} else if (rank < num_pairs * 2) {
target = rank - num_pairs;
for (i = 0; i < loop + skip; i++) {
if (i == skip) {
ctpm_Barrier();
}
for (j = 0; j < window_size; j++) {
switch (type) {
case SEND_RECV:
for (c = 0; c < iov_cnt; c++) {
fi_rc = fi_recv(ep, r_iov[c].iov_base,
r_iov[c].iov_len, NULL,
fi_addrs[target],
NULL);
assert(!fi_rc);
}
break;
case SENDV_RECVV:
case SEND_RECVV:
fi_rc = fi_recvv(ep, r_iov, (void **) NULL, iov_cnt,
fi_addrs[target], NULL);
assert(!fi_rc);
break;
case SENDV_RECV:
fi_rc = fi_recv(ep, r_buf,
r_buf_len, NULL,
fi_addrs[target],
NULL);
assert(!fi_rc);
break;
default:
abort();
}
}
wait_for_comp(rcq, type == 0 ? window_size * iov_cnt : window_size);
fi_rc = fi_send(ep, s_fin, 4, NULL,
fi_addrs[target], NULL);
assert(!fi_rc);
wait_for_comp(scq, 1);
}
} else {
ctpm_Barrier();
}
ts = malloc(sizeof(t) * numprocs);
assert(ts);
ctpm_Allgather(&t, sizeof(t), ts);
if (!myid) {
for (i = 0; i < numprocs; i++) {
if (ts[i] > maxtime) {
maxtime = ts[i];
}
}
}
free(ts);
if (rank == 0) {
double tmp = num_pairs * cum_size / 1e6;
tmp = tmp * loop * window_size;
bw = tmp / (maxtime / 1e6);
return bw;
}
return 0;
}