static int run(void)
{
int i, ret = 0;
ret = init_fabric();
if (ret)
return ret;
ret = ft_init_av();
if (ret)
goto out;
if (!(opts.options & FT_OPT_SIZE)) {
for (i = 0; i < TEST_CNT; i++) {
if (test_size[i].option > opts.size_option)
continue;
opts.transfer_size = test_size[i].size;
init_test(&opts, test_name, sizeof(test_name));
ret = pingpong();
if (ret)
goto out;
}
} else {
init_test(&opts, test_name, sizeof(test_name));
ret = pingpong();
if (ret)
goto out;
}
ft_finalize();
out:
return ret;
}
static int run(void)
{
int i, ret = 0;
ret = init_fabric();
if (ret)
return ret;
ret = init_av();
if (ret)
goto out;
if (!(opts.options & FT_OPT_SIZE)) {
for (i = 0; i < TEST_CNT; i++) {
if (test_size[i].option > opts.size_option)
continue;
opts.transfer_size = test_size[i].size;
init_test(&opts, test_name, sizeof(test_name));
ret = run_test();
if (ret)
goto out;
}
} else {
init_test(&opts, test_name, sizeof(test_name));
ret = run_test();
if (ret)
goto out;
}
/* Finalize before closing ep */
ft_finalize(fi, ep, txcq, rxcq, remote_fi_addr);
out:
return ret;
}
static int run(void)
{
int ret = 0;
addr_array = calloc(ep_cnt, sizeof(*addr_array));
if (!addr_array) {
perror("malloc");
return -FI_ENOMEM;
}
if (hints->ep_attr->type == FI_EP_MSG) {
if (!opts.dst_addr) {
ret = ft_start_server();
if (ret)
return ret;
}
ret = opts.dst_addr ? client_connect() : server_connect();
} else {
ret = init_fabric();
}
if (ret)
return ret;
ret = run_test();
/* TODO: Add a local finalize applicable to shared ctx */
//ft_finalize(fi, ep_array[0], txcq, rxcq, addr_array[0]);
return ret;
}
static int run(void)
{
int ret;
ret = init_fabric();
if (ret)
return ret;
ret = ft_init_av();
if (ret)
return ret;
return send_recv();
}
static int run(void)
{
int i, ret = 0;
ret = init_fabric();
if (ret)
return ret;
ret = init_av();
if (ret)
goto out;
ret = exchange_addr_key();
if (ret)
goto out;
if (run_all_sizes) {
for (i = 0; i < TEST_CNT; i++) {
if (test_size[i].option > size_option)
continue;
init_test(test_size[i].size, test_name,
sizeof(test_name), &transfer_size,
&iterations);
ret = run_test();
if (ret) {
fprintf(stderr, "Test failed at iteration %d, "
"msg size %d\n", i,
transfer_size);
goto out;
}
}
} else {
ret = run_test();
if (ret)
goto out;
}
sync_test();
out:
fi_shutdown(ep, 0);
fi_close(&ep->fid);
free_ep_res();
fi_close(&dom->fid);
fi_close(&fab->fid);
return ret;
}
int main(int argc, char **argv)
{
int op, ret = 0;
opts = INIT_OPTS;
opts.options |= FT_OPT_SIZE;
hints = fi_allocinfo();
if (!hints)
return EXIT_FAILURE;
while ((op = getopt(argc, argv, "h" ADDR_OPTS INFO_OPTS)) != -1) {
switch (op) {
default:
ft_parse_addr_opts(op, optarg, &opts);
ft_parseinfo(op, optarg, hints);
break;
case '?':
case 'h':
ft_usage(argv[0], "A client-server example that uses poll.\n");
return EXIT_FAILURE;
}
}
if (optind < argc)
opts.dst_addr = argv[optind];
hints->ep_attr->type = FI_EP_RDM;
hints->caps = FI_MSG;
hints->mode = FI_CONTEXT | FI_LOCAL_MR;
ret = init_fabric();
if (ret)
return -ret;
ret = init_av();
if (ret)
return ret;
/* Exchange data */
ret = send_recv();
ft_free_res();
return ret;
}
static int run(void)
{
int ret = 0;
ret = init_fabric();
if (ret)
return ret;
ret = init_av();
if (ret)
goto out;
ret = run_test();
/* TODO: Add a local finalize applicable to shared ctx */
//ft_finalize(fi, ep_array[0], txcq, rxcq, remote_fi_addr[0]);
out:
return ret;
}
int main(int argc, char **argv)
{
int op, ret;
opts = INIT_OPTS;
hints = fi_allocinfo();
if (!hints)
return EXIT_FAILURE;
while ((op = getopt(argc, argv, "h" ADDR_OPTS INFO_OPTS)) != -1) {
switch (op) {
default:
ft_parse_addr_opts(op, optarg, &opts);
ft_parseinfo(op, optarg, hints);
break;
case '?':
case 'h':
ft_usage(argv[0], "A simple RDM client-sever example.");
return EXIT_FAILURE;
}
}
if (optind < argc)
opts.dst_addr = argv[optind];
hints->ep_attr->type = FI_EP_RDM;
hints->caps = FI_MSG;
hints->mode = FI_CONTEXT;
/* Fabric initialization */
ret = init_fabric();
if(ret)
return -ret;
/* Exchange data */
ret = send_recv();
free_ep_res();
fi_close(&dom->fid);
fi_close(&fab->fid);
fi_freeinfo(hints);
fi_freeinfo(fi);
return ret;
}
static int run(void)
{
int ret = 0;
ret = init_fabric();
if (ret)
goto out;
ret = init_av();
if (ret)
goto out;
ret = run_test();
rx_seq++;
ft_finalize();
out:
return ret;
}
static int run(void)
{
int ret = 0;
ret = init_fabric();
if (ret)
return ret;
ret = init_av();
if (ret)
return ret;
ret = run_test();
/*TODO: Add a local finalize applicable for scalable ep */
//ft_finalize(fi, tx_ep[0], txcq_array[0], rxcq_array[0], remote_rx_addr[0]);
return ret;
}
static int run(void)
{
int i, ret = 0;
ret = init_fabric();
if (ret)
return ret;
ret = init_av();
if (ret)
goto out;
ret = exchange_addr_key();
if (ret)
goto out;
if (!(opts.user_options & FT_OPT_SIZE)) {
for (i = 0; i < TEST_CNT; i++) {
if (test_size[i].option > opts.size_option)
continue;
opts.transfer_size = test_size[i].size;
init_test(&opts, test_name, sizeof(test_name));
ret = run_test();
if (ret)
goto out;
}
} else {
init_test(&opts, test_name, sizeof(test_name));
ret = run_test();
if (ret)
goto out;
}
/* Finalize before closing ep */
ft_finalize(ep, scq, rcq, remote_fi_addr);
out:
free_ep_res();
fi_close(&dom->fid);
fi_close(&fab->fid);
return ret;
}
static int run(void)
{
int ret = 0;
ret = init_fabric();
if (ret)
return ret;
ret = init_av();
if (ret)
goto out;
run_test();
/* TODO: Add a local finalize applicable to shared ctx */
//ft_finalize(ep[0], scq, rcq, remote_fi_addr[0]);
out:
free_ep_res();
fi_close(&dom->fid);
fi_close(&fab->fid);
return ret;
}
static int run_test(void)
{
int ret = 0;
ret = init_fabric();
if (ret)
return ret;
if (opts.dst_addr) {
/* Execute RMA write operation from Client */
fprintf(stdout, "RMA write to server\n");
sprintf(buf, "%s", welcome_text);
ret = write_data(sizeof(char *) * strlen(buf));
if (ret)
return ret;
ret = fi_cntr_wait(scntr, 1, -1);
if (ret < 0) {
FT_PRINTERR("fi_cntr_wait", ret);
return ret;
}
fprintf(stdout, "Received a completion event for RMA write\n");
} else {
/* Server waits for message from Client */
ret = fi_cntr_wait(rcntr, 1, -1);
if (ret < 0) {
FT_PRINTERR("fi_cntr_wait", ret);
return ret;
}
fprintf(stdout, "Received data from Client: %s\n", (char *)buf);
}
/* TODO: need support for finalize operation to sync test */
free_ep_res();
fi_close(&dom->fid);
fi_close(&fab->fid);
return 0;
}
static int run(void)
{
int ret = 0;
ret = init_fabric();
if (ret)
return ret;
ret = init_av();
if (ret)
goto out;
run_test();
/*TODO: Add a local finalize applicable for scalable ep */
//ft_finalize(tx_ep[0], scq[0], rcq[0], remote_rx_addr[0]);
out:
free_ep_res();
fi_close(&sep->fid);
fi_close(&dom->fid);
fi_close(&fab->fid);
return ret;
}
static int run(void)
{
int ret;
ret = init_fabric();
if (ret)
return ret;
ret = ft_init_av();
if (ret)
return ret;
if (opts.dst_addr) {
printf("Searching for a bad msg\n");
ret = tagged_peek(0xbad);
if (ret != -FI_ENOMSG) {
FT_PRINTERR("FI_PEEK", ret);
return ret;
}
printf("Synchronizing with sender..\n");
ret = ft_sync();
if (ret)
return ret;
printf("Searching for a good msg\n");
ret = tagged_peek(0x900d);
if (ret != 1) {
FT_PRINTERR("FI_PEEK", ret);
return ret;
}
printf("Receiving msg\n");
ret = fi_trecv(ep, buf, rx_size, fi_mr_desc(mr), remote_fi_addr,
0x900d, 0, &rx_ctx);
if (ret) {
FT_PRINTERR("fi_trecv", ret);
return ret;
}
printf("Completing recv\n");
ret = ft_get_rx_comp(++rx_seq);
if (ret)
return ret;
} else {
printf("Sending tagged message\n");
ret = fi_tsend(ep, tx_buf, tx_size, fi_mr_desc(mr),
remote_fi_addr, 0x900d, &tx_ctx);
if (ret)
return ret;
printf("Synchronizing with receiver..\n");
ret = ft_sync();
if (ret)
return ret;
printf("Getting send completion\n");
ret = ft_get_tx_comp(tx_seq + 1);
if (ret)
return ret;
}
ft_finalize();
return 0;
}
int main(int argc, char *argv[])
{
int i, j, peer;
int size, align_size;
char *s_buf, *r_buf;
uint64_t t_start = 0, t_end = 0, t = 0;
int op, ret;
buf_desc_t lbuf_desc;
ssize_t fi_rc;
FT_Init(&argc, &argv);
FT_Rank(&myid);
FT_Job_size(&numprocs);
hints = fi_allocinfo();
if (!hints)
return -1;
while ((op = getopt(argc, argv, "h" INFO_OPTS)) != -1) {
switch (op) {
default:
ft_parseinfo(op, optarg, hints);
break;
case '?':
case 'h':
print_usage();
return EXIT_FAILURE;
}
}
hints->ep_attr->type = FI_EP_RDM;
hints->caps = FI_MSG | FI_DIRECTED_RECV | FI_RMA;
hints->mode = FI_CONTEXT | FI_LOCAL_MR;
hints->domain_attr->mr_mode = FI_MR_BASIC;
if (numprocs != 2) {
if (myid == 0) {
fprintf(stderr, "This test requires exactly two processes\n");
}
FT_Finalize();
return -1;
}
/* Fabric initialization */
ret = init_fabric();
if (ret) {
fprintf(stderr, "Problem in fabric initialization\n");
return ret;
}
ret = init_av();
if (ret) {
fprintf(stderr, "Problem in AV initialization\n");
return ret;
}
/* Data initialization */
align_size = getpagesize();
assert(align_size <= MAX_ALIGNMENT);
s_buf = (char *) (((unsigned long) s_buf_original + (align_size - 1)) /
align_size * align_size);
r_buf = (char *) (((unsigned long) r_buf_original + (align_size - 1)) /
align_size * align_size);
ret = fi_mr_reg(dom, r_buf, MYBUFSIZE, FI_REMOTE_WRITE, 0, 0, 0, &r_mr, NULL);
if (ret) {
FT_PRINTERR("fi_mr_reg", ret);
return -1;
}
lbuf_desc.addr = (uint64_t)r_buf;
lbuf_desc.key = fi_mr_key(r_mr);
rbuf_descs = (buf_desc_t *)malloc(numprocs * sizeof(buf_desc_t));
/* Distribute memory keys */
FT_Allgather(&lbuf_desc, sizeof(lbuf_desc), rbuf_descs);
ret = fi_mr_reg(dom, s_buf, MYBUFSIZE, FI_WRITE, 0, 0, 0, &l_mr, NULL);
if (ret) {
FT_PRINTERR("fi_mr_reg", ret);
return -1;
}
if (myid == 0) {
fprintf(stdout, HEADER);
fprintf(stdout, "%-*s%*s%*s\n", 10, "# Size", FIELD_WIDTH,
"Bandwidth (MB/s)", FIELD_WIDTH, "latency");
fflush(stdout);
}
/* Bandwidth test */
for (size = 1; size <= MAX_MSG_SIZE; size *= 2) {
/* touch the data */
for (i = 0; i < size; i++) {
s_buf[i] = 'a';
r_buf[i] = 'b';
}
if (size > large_message_size) {
loop = loop_large;
skip = skip_large;
window_size = window_size_large;
}
FT_Barrier();
if (myid == 0) {
peer = 1;
for (i = 0; i < loop + skip; i++) {
if (i == skip) {
t_start = get_time_usec();
}
for (j = 0; j < window_size; j++) {
fi_rc = fi_write(ep, s_buf, size, l_mr,
fi_addrs[peer],
rbuf_descs[peer].addr,
rbuf_descs[peer].key,
(void *)(intptr_t)j);
if (fi_rc) {
FT_PRINTERR("fi_write", fi_rc);
return fi_rc;
}
}
ft_wait_for_comp(scq, window_size);
}
t_end = get_time_usec();
t = t_end - t_start;
} else if (myid == 1) {
peer = 0;
}
if (myid == 0) {
double latency = (t_end - t_start) /
(double)(loop * window_size);
double tmp = size / 1e6 * loop * window_size;
fprintf(stdout, "%-*d%*.*f%*.*f\n", 10, size,
FIELD_WIDTH, FLOAT_PRECISION,
tmp / (t / 1e6), FIELD_WIDTH,
FLOAT_PRECISION, latency);
fflush(stdout);
}
}
FT_Barrier();
fi_close(&l_mr->fid);
fi_close(&r_mr->fid);
free_ep_res();
fi_close(&ep->fid);
fi_close(&dom->fid);
fi_close(&fab->fid);
fi_freeinfo(hints);
fi_freeinfo(fi);
FT_Barrier();
FT_Finalize();
return 0;
}
static int run(void)
{
int ret = 0;
ret = init_fabric();
if (ret)
return ret;
ret = init_av();
if (ret)
goto out;
// Receiver
if (opts.dst_addr) {
// search for initial tag, it should fail since the sender
// hasn't sent anyting
fprintf(stdout, "Searching msg with tag [%" PRIu64 "]\n", tag_data);
tagged_search(tag_data);
fprintf(stdout, "Posting buffer for msg with tag [%" PRIu64 "]\n",
tag_data + 1);
ret = post_recv(tag_data + 1);
if (ret)
goto out;
// synchronize with sender
fprintf(stdout, "\nSynchronizing with sender..\n\n");
ret = sync_test();
if (ret)
goto out;
// wait for the completion event of the next tag
ret = wait_for_tagged_completion(rcq, 1);
if (ret)
goto out;
fprintf(stdout, "Received completion event for msg with tag "
"[%" PRIu64 "]\n", tag_data + 1);
// search again for the initial tag, it should be successful now
fprintf(stdout,
"Searching msg with initial tag [%" PRIu64 "]\n",
tag_data);
tagged_search(tag_data);
// wait for the completion event of the initial tag
ret = recv_msg(tag_data);
if (ret)
goto out;
fprintf(stdout, "Posted buffer and received completion event for"
" msg with tag [%" PRIu64 "]\n", tag_data);
} else {
// Sender
// synchronize with receiver
fprintf(stdout, "Synchronizing with receiver..\n\n");
ret = sync_test();
if (ret)
goto out;
fprintf(stdout, "Sending msg with tag [%" PRIu64 "]\n",
tag_data);
ret = send_msg(16, tag_data);
if (ret)
goto out;
fprintf(stdout, "Sending msg with tag [%" PRIu64 "]\n",
tag_data + 1);
ret = send_msg(16, tag_data + 1);
if (ret)
goto out;
}
/* Finalize before closing ep */
ft_finalize(ep, scq, rcq, remote_fi_addr);
out:
free_ep_res();
fi_close(&dom->fid);
fi_close(&fab->fid);
return ret;
}
int main(int argc, char *argv[])
{
int op, ret;
struct iovec s_iov[IOV_CNT], r_iov[IOV_CNT];
char *s_buf, *r_buf;
int align_size;
int pairs, print_rate;
int window_varied;
int c, j;
int curr_size;
enum send_recv_type_e type;
ctpm_Init(&argc, &argv);
ctpm_Rank(&myid);
ctpm_Job_size(&numprocs);
/* default values */
pairs = numprocs / 2;
window_size = DEFAULT_WINDOW;
window_varied = 0;
print_rate = 1;
hints = fi_allocinfo();
if (!hints)
return -1;
while ((op = getopt(argc, argv, "hp:w:vr:" CT_STD_OPTS)) != -1) {
switch (op) {
default:
ct_parse_std_opts(op, optarg, hints);
break;
case 'p':
pairs = atoi(optarg);
if (pairs > (numprocs / 2)) {
print_usage();
return EXIT_FAILURE;
}
break;
case 'w':
window_size = atoi(optarg);
break;
case 'v':
window_varied = 1;
break;
case 'r':
print_rate = atoi(optarg);
if (0 != print_rate && 1 != print_rate) {
print_usage();
return EXIT_FAILURE;
}
break;
case '?':
case 'h':
print_usage();
return EXIT_FAILURE;
}
}
hints->ep_attr->type = FI_EP_RDM;
hints->caps = FI_MSG | FI_DIRECTED_RECV;
hints->mode = FI_CONTEXT | FI_LOCAL_MR;
if (numprocs < 2) {
if (!myid) {
fprintf(stderr, "This test requires at least two processes\n");
}
ctpm_Finalize();
return -1;
}
/* Fabric initialization */
ret = init_fabric();
if (ret) {
fprintf(stderr, "Problem in fabric initialization\n");
return ret;
}
ret = init_av();
if (ret) {
fprintf(stderr, "Problem in AV initialization\n");
return ret;
}
/* Data initialization */
align_size = getpagesize();
assert(align_size <= MAX_ALIGNMENT);
/* Allocate page aligned buffers */
for (c = 0; c < IOV_CNT; c++) {
assert(!posix_memalign(&s_iov[c].iov_base, align_size, MAX_MSG_SIZE));
assert(!posix_memalign(&r_iov[c].iov_base, align_size, MAX_MSG_SIZE));
}
assert(!posix_memalign((void **)&s_buf, align_size, MAX_MSG_SIZE * IOV_CNT));
assert(!posix_memalign((void **)&r_buf, align_size, MAX_MSG_SIZE * IOV_CNT));
for (type = 0; type < FIN; type++) {
if (!myid) {
fprintf(stdout, HEADER);
switch (type) {
case SEND_RECV:
fprintf(stdout, SEND_RECV_DESC);
break;
case SENDV_RECVV:
fprintf(stdout, SENDV_RECVV_DESC);
break;
case SEND_RECVV:
fprintf(stdout, SEND_RECVV_DESC);
break;
case SENDV_RECV:
fprintf(stdout, SENDV_RECV_DESC);
break;
default:
abort();
}
if (window_varied) {
fprintf(stdout, "# [ pairs: %d ] [ window size: varied ]\n", pairs);
fprintf(stdout, "\n# Uni-directional Bandwidth (MB/sec)\n");
} else {
fprintf(stdout, "# [ pairs: %d ] [ window size: %d ]\n", pairs,
window_size);
if (print_rate) {
fprintf(stdout, "%-*s%*s%*s%*s\n", 10, "# Size", FIELD_WIDTH,
"Iov count", FIELD_WIDTH, "MB/s", FIELD_WIDTH, "Messages/s");
} else {
fprintf(stdout, "%-*s%*s%*s\n", 10, "# Size", FIELD_WIDTH,
"Iov count", FIELD_WIDTH, "MB/s");
}
}
fflush(stdout);
}
if (window_varied) {
int window_array[] = WINDOW_SIZES;
double **bandwidth_results;
int log_val = 1, tmp_message_size = MAX_MSG_SIZE;
int i, j;
for (i = 0; i < WINDOW_SIZES_COUNT; i++) {
if (window_array[i] > window_size) {
window_size = window_array[i];
}
}
while (tmp_message_size >>= 1) {
log_val++;
}
bandwidth_results = (double **)malloc(sizeof(double *) * log_val);
for (i = 0; i < log_val; i++) {
bandwidth_results[i] = (double *)malloc(sizeof(double) *
WINDOW_SIZES_COUNT);
}
if (!myid) {
fprintf(stdout, "# ");
for (i = 0; i < WINDOW_SIZES_COUNT; i++) {
fprintf(stdout, " %10d", window_array[i]);
}
fprintf(stdout, "\n");
fflush(stdout);
}
for (j = 0, curr_size = 1; curr_size <= MAX_MSG_SIZE; curr_size *= 2, j++) {
if (!myid) {
fprintf(stdout, "%-7d", curr_size);
}
for (i = 0; i < WINDOW_SIZES_COUNT; i++) {
for (c = 0; c < IOV_CNT; c++) {
r_iov[c].iov_len = s_iov[c].iov_len = curr_size;
bandwidth_results[j][i] = calc_bw(myid, pairs,
window_array[i], s_iov, r_iov, c + 1,
s_buf, (c + 1) * curr_size, r_buf,
(c + 1) * curr_size, type);
if (!myid) {
fprintf(stdout, "%*d %10.*f", FIELD_WIDTH, c + 1,
FLOAT_PRECISION,
bandwidth_results[j][i]);
}
fprintf(stdout, c == IOV_CNT - 1 ? "\n" : "");
}
}
if (!myid) {
fprintf(stdout, "\n");
fflush(stdout);
}
}
if (!myid && print_rate) {
fprintf(stdout, "\n# Message Rate Profile\n");
fprintf(stdout, "# ");
for (i = 0; i < WINDOW_SIZES_COUNT; i++) {
fprintf(stdout, " %10d", window_array[i]);
}
fprintf(stdout, "\n");
fflush(stdout);
for (c = 0; c < IOV_CNT; c++) {
for (j = 0, curr_size = 1; curr_size <= MAX_MSG_SIZE; curr_size *= 2) {
fprintf(stdout, "%-7d,%*d", curr_size * (c + 1), FIELD_WIDTH, c + 1);
for (i = 0; i < WINDOW_SIZES_COUNT; i++) {
double rate = 1e6 * bandwidth_results[j][i] / (curr_size * (c + 1));
fprintf(stdout, " %10.2f", rate);
}
fprintf(stdout, "\n");
fflush(stdout);
j++;
}
}
}
} else {
/* Just one window size */
for (curr_size = 1; curr_size <= MAX_MSG_SIZE; curr_size *= 2) {
double bw, rate;
for (c = 0; c < IOV_CNT; c++) {
r_iov[c].iov_len = s_iov[c].iov_len = curr_size;
bw = calc_bw(myid, pairs, window_size, s_iov, r_iov, c + 1,
s_buf, (c + 1) * curr_size, r_buf,
(c + 1) * curr_size, type);
if (!myid) {
rate = 1e6 * bw / (curr_size * (c + 1));
if (print_rate) {
fprintf(stdout, "%-*d%*d%*.*f%*.*f\n", 10, curr_size * (c + 1),
FIELD_WIDTH, c + 1, FIELD_WIDTH,
FLOAT_PRECISION, bw, FIELD_WIDTH,
FLOAT_PRECISION, rate);
fflush(stdout);
} else {
fprintf(stdout, "%-*d%*d%*.*f\n", 10, curr_size * (c + 1), FIELD_WIDTH,
FIELD_WIDTH, c + 1, FLOAT_PRECISION, bw);
fflush(stdout);
}
}
fprintf(stdout, c == IOV_CNT - 1 ? "\n" : "");
}
}
}
}
