static int run_test() { int ret = 0, i; if (opts.dst_addr) { for (i = 0; i < ctx_cnt && !ret; i++) { fprintf(stdout, "Posting send for ctx: %d\n", i); ret = fi_send(tx_ep[i], buf, tx_size, fi_mr_desc(mr), remote_rx_addr[i], NULL); if (ret) { FT_PRINTERR("fi_send", ret); return ret; } ret = wait_for_comp(txcq_array[i]); } } else { for (i = 0; i < ctx_cnt && !ret; i++) { fprintf(stdout, "wait for recv completion for ctx: %d\n", i); ret = wait_for_comp(rxcq_array[i]); } } return ret; }
static int run_test() { int ret = 0, i; uint32_t data; uint32_t *tb = (uint32_t *)tx_buf; uint32_t *rb = (uint32_t *)rx_buf; if (opts.dst_addr) { for (i = 0; i < ctx_cnt && !ret; i++) { fprintf(stdout, "Posting send for ctx: %d\n", i); tb[0] = DATA + i; ret = fi_send(tx_ep[i], tx_buf, tx_size, mr_desc, remote_rx_addr[i], NULL); if (ret) { FT_PRINTERR("fi_send", ret); return ret; } ret = wait_for_comp(txcq_array[i]); } } else { for (i = 0; i < ctx_cnt && !ret; i++) { fprintf(stdout, "wait for recv completion for ctx: %d\n", i); ret = wait_for_comp(rxcq_array[i]); data = DATA + i; if (memcmp(&data, rx_buf, 4) != 0) { fprintf(stdout, "failed compare expected 0x%x," " read 0x%x\n", data, rb[0]); } } } return ret; }
static int init_av(void) { size_t addrlen; int ret, i; if (opts.dst_addr) { ret = ft_av_insert(av, fi->dest_addr, 1, &remote_fi_addr, 0, NULL); if (ret) return ret; addrlen = FT_MAX_CTRL_MSG; ret = fi_getname(&sep->fid, tx_buf, &addrlen); if (ret) { FT_PRINTERR("fi_getname", ret); return ret; } ret = fi_send(tx_ep[0], tx_buf, addrlen, fi_mr_desc(mr), remote_fi_addr, NULL); if (ret) { FT_PRINTERR("fi_send", ret); return ret; } ret = wait_for_comp(rxcq_array[0]); if (ret) return ret; } else { ret = wait_for_comp(rxcq_array[0]); if (ret) return ret; ret = ft_av_insert(av, rx_buf, 1, &remote_fi_addr, 0, NULL); if (ret) return ret; ret = fi_send(tx_ep[0], tx_buf, 1, fi_mr_desc(mr), remote_fi_addr, NULL); if (ret) { FT_PRINTERR("fi_send", ret); return ret; } } for (i = 0; i < ctx_cnt; i++) remote_rx_addr[i] = fi_rx_addr(remote_fi_addr, i, rx_ctx_bits); ret = fi_recv(rx_ep[0], rx_buf, rx_size, fi_mr_desc(mr), 0, NULL); if (ret) { FT_PRINTERR("fi_recv", ret); return ret; } ret = wait_for_comp(txcq_array[0]); return ret; }
double calc_bw(int rank, int num_pairs, int window_size, struct iovec *s_iov, struct iovec *r_iov, int iov_cnt, char *s_buf, int s_buf_len, char *r_buf, int r_buf_len, enum send_recv_type_e type) { uint64_t t_start = 0, t_end = 0, t = 0, maxtime = 0, *ts; double bw = 0; int c, i, j, target; int loop, skip; size_t cum_size = 0; int mult = (DEFAULT_WINDOW / window_size) > 0 ? (DEFAULT_WINDOW / window_size) : 1; int __attribute__((unused)) fi_rc; char r_fin[4] = {'b', 'b', 'b', 'b'}; char s_fin[4] = {'a', 'a', 'a', 'a'}; for (c = 0; c < iov_cnt; c++) { for (i = 0; i < s_iov[c].iov_len; i++) { ((char *) s_iov[c].iov_base)[i] = 'a'; } for (i = 0; i < r_iov[c].iov_len; i++) { ((char *) r_iov[c].iov_base)[i] = 'b'; } /* Size will be all the receiver can hold */ cum_size += r_iov[c].iov_len; } for (c = 0; c < s_buf_len; c++) { s_buf[c]= 'a'; } for (c = 0; c < r_buf_len; c++) { r_buf[c]= 'b'; } if (cum_size > LARGE_THRESHOLD) { loop = ITERS_LARGE * mult; skip = WARMUP_ITERS_LARGE * mult; } else { loop = ITERS_SMALL * mult; skip = WARMUP_ITERS_SMALL * mult; } ctpm_Barrier(); if (rank < num_pairs) { target = rank + num_pairs; for (i = 0; i < loop + skip; i++) { if (i == skip) { ctpm_Barrier(); t_start = get_time_usec(); } for (j = 0; j < window_size; j++) { switch (type) { case SEND_RECV: for (c = 0; c < iov_cnt; c++) { fi_rc = fi_send(ep, s_iov[c].iov_base, s_iov[c].iov_len, NULL, fi_addrs[target], NULL); assert(!fi_rc); } break; case SENDV_RECVV: case SENDV_RECV: fi_rc = fi_sendv(ep, s_iov, (void **) NULL, iov_cnt, fi_addrs[target], NULL); assert(!fi_rc); break; case SEND_RECVV: fi_rc = fi_send(ep, s_buf, s_buf_len, NULL, fi_addrs[target], NULL); assert(!fi_rc); break; default: abort(); } } wait_for_comp(scq, type == SEND_RECV ? window_size * iov_cnt : window_size); fi_rc = fi_recv(ep, r_fin, 4, NULL, fi_addrs[target], NULL); assert(!fi_rc); wait_for_comp(rcq, 1); } t_end = get_time_usec(); t = t_end - t_start; } else if (rank < num_pairs * 2) { target = rank - num_pairs; for (i = 0; i < loop + skip; i++) { if (i == skip) { ctpm_Barrier(); } for (j = 0; j < window_size; j++) { switch (type) { case SEND_RECV: for (c = 0; c < iov_cnt; c++) { fi_rc = fi_recv(ep, r_iov[c].iov_base, r_iov[c].iov_len, NULL, fi_addrs[target], NULL); assert(!fi_rc); } break; case SENDV_RECVV: case SEND_RECVV: fi_rc = fi_recvv(ep, r_iov, (void **) NULL, iov_cnt, fi_addrs[target], NULL); assert(!fi_rc); break; case SENDV_RECV: fi_rc = fi_recv(ep, r_buf, r_buf_len, NULL, fi_addrs[target], NULL); assert(!fi_rc); break; default: abort(); } } wait_for_comp(rcq, type == 0 ? window_size * iov_cnt : window_size); fi_rc = fi_send(ep, s_fin, 4, NULL, fi_addrs[target], NULL); assert(!fi_rc); wait_for_comp(scq, 1); } } else { ctpm_Barrier(); } ts = malloc(sizeof(t) * numprocs); assert(ts); ctpm_Allgather(&t, sizeof(t), ts); if (!myid) { for (i = 0; i < numprocs; i++) { if (ts[i] > maxtime) { maxtime = ts[i]; } } } free(ts); if (rank == 0) { double tmp = num_pairs * cum_size / 1e6; tmp = tmp * loop * window_size; bw = tmp / (maxtime / 1e6); return bw; } return 0; }