static int run(void)
{
int ret = 0;
if (!dst_addr) {
ret = server_listen();
if (ret)
return ret;
}
ret = dst_addr ? client_connect() : server_connect();
if (ret) {
return ret;
}
run_test();
fi_shutdown(ep, 0);
fi_close(&ep->fid);
free_ep_res();
if (!dst_addr)
free_lres();
fi_close(&dom->fid);
fi_close(&fab->fid);
return ret;
}
static int init_fabric(void)
{
uint64_t flags = 0;
char *node, *service;
int ret;
ret = ft_read_addr_opts(&node, &service, hints, &flags, &opts);
if (ret)
return ret;
ret = fi_getinfo(FT_FIVERSION, node, service, flags, hints, &fi);
if (ret) {
FT_PRINTERR("fi_getinfo", ret);
return ret;
}
// we use provider MR attributes and direct address (no offsets)
// for RMA calls
if (!(fi->mode & FI_PROV_MR_ATTR))
fi->mode |= FI_PROV_MR_ATTR;
// get remote address
if (opts.dst_addr) {
addrlen = fi->dest_addrlen;
remote_addr = malloc(addrlen);
memcpy(remote_addr, fi->dest_addr, addrlen);
}
ret = fi_fabric(fi->fabric_attr, &fab, NULL);
if (ret) {
FT_PRINTERR("fi_fabric", ret);
goto err0;
}
ret = fi_domain(fab, fi, &dom, NULL);
if (ret) {
FT_PRINTERR("fi_domain", ret);
goto err1;
}
ret = alloc_ep_res(fi);
if (ret)
goto err3;
ret = bind_ep_res();
if (ret)
goto err4;
return 0;
err4:
free_ep_res();
err3:
fi_close(&dom->fid);
err1:
fi_close(&fab->fid);
err0:
return ret;
}
static int init_fabric(void)
{
int ret;
uint64_t flags = 0;
/* Get fabric info */
ret = fi_getinfo(CT_FIVERSION, NULL, NULL, flags, hints, &fi);
if (ret) {
ct_print_fi_error("fi_getinfo", ret);
return ret;
}
/* Open fabric */
ret = fi_fabric(fi->fabric_attr, &fab, NULL);
if (ret) {
ct_print_fi_error("fi_fabric", ret);
goto err1;
}
/* Open domain */
ret = fi_domain(fab, fi, &dom, NULL);
if (ret) {
ct_print_fi_error("fi_domain", ret);
goto err2;
}
/* Open endpoint */
ret = fi_endpoint(dom, fi, &ep, NULL);
if (ret) {
ct_print_fi_error("fi_endpoint", ret);
goto err3;
}
/* Allocate endpoint resources */
ret = alloc_ep_res();
if (ret)
goto err4;
/* Bind EQs and AVs with endpoint */
ret = bind_ep_res();
if (ret)
goto err5;
return 0;
err5:
free_ep_res();
err4:
fi_close(&ep->fid);
err3:
fi_close(&dom->fid);
err2:
fi_close(&fab->fid);
err1:
return ret;
}
int main(int argc, char **argv)
{
int op, ret;
opts = INIT_OPTS;
hints = fi_allocinfo();
if (!hints)
return EXIT_FAILURE;
while ((op = getopt(argc, argv, "h" ADDR_OPTS INFO_OPTS)) != -1) {
switch (op) {
default:
ft_parse_addr_opts(op, optarg, &opts);
ft_parseinfo(op, optarg, hints);
break;
case '?':
case 'h':
ft_usage(argv[0], "A simple MSG client-sever example that "
"demonstrates one possible usage of the underlying "
"cq wait objects.");
return EXIT_FAILURE;
}
}
if (optind < argc)
opts.dst_addr = argv[optind];
hints->ep_attr->type = FI_EP_MSG;
hints->caps = FI_MSG;
hints->mode = FI_LOCAL_MR;
hints->addr_format = FI_SOCKADDR;
/* Fabric and connection setup */
if (!opts.dst_addr) {
ret = server_listen();
if (ret)
return -ret;
}
ret = opts.dst_addr ? client_connect() : server_connect();
if (ret) {
return -ret;
}
/* Exchange data */
ret = send_recv();
fi_shutdown(ep, 0);
free_ep_res();
fi_close(&cmeq->fid);
fi_close(&dom->fid);
fi_close(&fab->fid);
return ret;
}
static int server_connect(void)
{
int ret;
struct fi_cq_entry comp;
ret = common_setup();
if (ret != 0)
goto err;
do {
ret = fi_cq_read(rcq, &comp, 1);
if (ret < 0 && ret != -FI_EAGAIN) {
FT_PRINTERR("fi_cq_read", ret);
return ret;
}
} while (ret == -FI_EAGAIN);
ret = fi_av_insert(av, buf_ptr, 1, &remote_fi_addr, 0, NULL);
if (ret != 1) {
if (ret == 0) {
fprintf(stderr, "Unable to resolve remote address 0x%x 0x%x\n",
((uint32_t *)buf)[0], ((uint32_t *)buf)[1]);
ret = -FI_EINVAL;
} else {
FT_PRINTERR("fi_av_insert", ret);
}
goto err;
}
ret = fi_recv(ep, buf, buffer_size, fi_mr_desc(mr), 0, buf);
if (ret != 0) {
FT_PRINTERR("fi_recv", ret);
goto err;
}
ret = send_xfer(4);
if (ret != 0)
goto err;
return 0;
err:
free_ep_res();
if (dom)
fi_close(&dom->fid);
if (fab)
fi_close(&fab->fid);
return ret;
}
static int run(void)
{
int i, ret = 0;
if (!opts.dst_addr) {
ret = server_listen();
if (ret)
return ret;
}
ret = opts.dst_addr ? client_connect() : server_connect();
if (ret)
return ret;
ret = exchange_addr_key();
if (ret)
return ret;
if (!(opts.user_options & FT_OPT_SIZE)) {
for (i = 0; i < TEST_CNT; i++) {
if (test_size[i].option > opts.size_option)
continue;
opts.transfer_size = test_size[i].size;
init_test(&opts, test_name, sizeof(test_name));
ret = run_test();
if (ret)
goto out;
}
} else {
init_test(&opts, test_name, sizeof(test_name));
ret = run_test();
if (ret)
goto out;
}
sync_test();
wait_for_data_completion(scq, max_credits - credits);
/* Finalize before closing ep */
ft_finalize(ep, scq, rcq, FI_ADDR_UNSPEC);
out:
fi_shutdown(ep, 0);
free_ep_res();
if (!opts.dst_addr)
free_lres();
fi_close(&dom->fid);
fi_close(&fab->fid);
return ret;
}
static int run(void)
{
int i, ret = 0;
ret = init_fabric();
if (ret)
return ret;
ret = init_av();
if (ret)
goto out;
ret = exchange_addr_key();
if (ret)
goto out;
if (run_all_sizes) {
for (i = 0; i < TEST_CNT; i++) {
if (test_size[i].option > size_option)
continue;
init_test(test_size[i].size, test_name,
sizeof(test_name), &transfer_size,
&iterations);
ret = run_test();
if (ret) {
fprintf(stderr, "Test failed at iteration %d, "
"msg size %d\n", i,
transfer_size);
goto out;
}
}
} else {
ret = run_test();
if (ret)
goto out;
}
sync_test();
out:
fi_shutdown(ep, 0);
fi_close(&ep->fid);
free_ep_res();
fi_close(&dom->fid);
fi_close(&fab->fid);
return ret;
}
int main(int argc, char **argv)
{
int op;
int ret = 0;
opts = INIT_OPTS;
opts.transfer_size = 256;
hints = fi_allocinfo();
if (!hints)
return EXIT_FAILURE;
while ((op = getopt(argc, argv, "c:vh" ADDR_OPTS INFO_OPTS)) != -1) {
switch (op) {
default:
ft_parse_addr_opts(op, optarg, &opts);
ft_parseinfo(op, optarg, hints);
break;
case 'c':
num_eps = atoi(optarg);
break;
case 'v':
opts.options |= FT_OPT_VERIFY_DATA;
break;
case '?':
case 'h':
ft_usage(argv[0], "Multi endpoint test");
FT_PRINT_OPTS_USAGE("-c <int>",
"number of endpoints to create and test (def 3)");
FT_PRINT_OPTS_USAGE("-v", "Enable DataCheck testing");
return EXIT_FAILURE;
}
}
if (optind < argc)
opts.dst_addr = argv[optind];
ft_skip_mr = 1;
hints->caps = FI_MSG;
hints->mode = FI_CONTEXT;
ret = run_test();
free_ep_res();
ft_free_res();
return ft_exit_code(ret);
}
static int run(void)
{
int i, ret = 0;
ret = opts.dst_addr ? client_connect() : server_connect();
if (ret)
return ret;
if (!(opts.user_options & FT_OPT_SIZE)) {
for (i = 0; i < TEST_CNT; i++) {
if (test_size[i].option > opts.size_option)
continue;
opts.transfer_size = test_size[i].size;
init_test(&opts, test_name, sizeof(test_name));
ret = run_test();
if (ret)
goto out;
}
} else {
init_test(&opts, test_name, sizeof(test_name));
ret = run_test();
if (ret)
goto out;
}
while (credits < max_credits)
poll_all_sends();
ft_finalize(ep, scq, rcq, remote_fi_addr);
out:
free_ep_res();
ret = fi_close(&av->fid);
if (ret != 0) {
FT_PRINTERR("fi_close", ret);
}
ret = fi_close(&dom->fid);
if (ret != 0) {
FT_PRINTERR("fi_close", ret);
}
ret = fi_close(&fab->fid);
if (ret != 0) {
FT_PRINTERR("fi_close", ret);
}
return ret;
}
int main(int argc, char **argv)
{
int op, ret;
opts = INIT_OPTS;
hints = fi_allocinfo();
if (!hints)
return EXIT_FAILURE;
while ((op = getopt(argc, argv, "h" ADDR_OPTS INFO_OPTS)) != -1) {
switch (op) {
default:
ft_parse_addr_opts(op, optarg, &opts);
ft_parseinfo(op, optarg, hints);
break;
case '?':
case 'h':
ft_usage(argv[0], "A simple RDM client-sever example.");
return EXIT_FAILURE;
}
}
if (optind < argc)
opts.dst_addr = argv[optind];
hints->ep_attr->type = FI_EP_RDM;
hints->caps = FI_MSG;
hints->mode = FI_CONTEXT;
/* Fabric initialization */
ret = init_fabric();
if(ret)
return -ret;
/* Exchange data */
ret = send_recv();
free_ep_res();
fi_close(&dom->fid);
fi_close(&fab->fid);
fi_freeinfo(hints);
fi_freeinfo(fi);
return ret;
}
static int run(void)
{
int i, ret = 0;
if (!dst_addr) {
ret = server_listen();
if (ret)
return ret;
}
print_test_hdr();
ret = dst_addr ? client_connect() : server_connect();
if (ret) {
return ret;
}
if (!custom) {
for (i = 0; i < TEST_CNT; i++) {
if (test_size[i].option > size_option)
continue;
init_test(test_size[i].size, test_name, &transfer_size, &iterations);
run_test();
}
} else {
ret = run_test();
}
ret = wait_for_completion(scq, max_credits - credits);
if (ret) {
return ret;
}
credits = max_credits;
fi_shutdown(ep, 0);
fi_close(&ep->fid);
free_ep_res();
if (!dst_addr)
free_lres();
fi_close(&dom->fid);
fi_close(&fab->fid);
return ret;
}
static int client_connect(void)
{
int ret;
socklen_t addrlen;
struct sockaddr *sin;
ret = common_setup();
if (ret != 0)
goto err;
ret = getaddr(dst_addr, port, (struct sockaddr **) &sin,
(socklen_t *) &addrlen);
if (ret != 0)
goto err;
ret = fi_connect(ep, sin, NULL, 0);
if (ret) {
printf("fi_connect %s\n", fi_strerror(-ret));
goto err;
}
// send initial message to server
ret = send_xfer(4);
if (ret != 0)
goto err;
// wait for reply to know server is ready
ret = recv_xfer(4);
if (ret != 0)
goto err;
return 0;
err:
free_ep_res();
fi_close(&av->fid);
fi_close(&ep->fid);
fi_close(&dom->fid);
fi_close(&fab->fid);
return ret;
}
static int server_connect(void)
{
int ret;
struct fi_cq_entry comp;
ret = common_setup();
if (ret != 0)
goto err;
do {
ret = fi_cq_readfrom(rcq, &comp, sizeof comp, &client_addr);
if (ret < 0) {
printf("RCQ readfrom %d (%s)\n", ret, fi_strerror(-ret));
return ret;
}
} while (ret == 0);
if (client_addr == FI_ADDR_NOTAVAIL) {
printf("Error getting address\n");
goto err;
}
ret = fi_recv(ep, buf, buffer_size, fi_mr_desc(mr), buf);
if (ret != 0) {
printf("fi_recv %d (%s)\n", ret, fi_strerror(-ret));
goto err;
}
ret = send_xfer(4);
if (ret != 0)
goto err;
return 0;
err:
free_ep_res();
fi_close(&ep->fid);
fi_close(&dom->fid);
fi_close(&fab->fid);
return ret;
}
static int run(void)
{
int i, ret = 0;
ret = init_fabric();
if (ret)
return ret;
ret = init_av();
if (ret)
goto out;
ret = exchange_addr_key();
if (ret)
goto out;
if (!(opts.user_options & FT_OPT_SIZE)) {
for (i = 0; i < TEST_CNT; i++) {
if (test_size[i].option > opts.size_option)
continue;
opts.transfer_size = test_size[i].size;
init_test(&opts, test_name, sizeof(test_name));
ret = run_test();
if (ret)
goto out;
}
} else {
init_test(&opts, test_name, sizeof(test_name));
ret = run_test();
if (ret)
goto out;
}
/* Finalize before closing ep */
ft_finalize(ep, scq, rcq, remote_fi_addr);
out:
free_ep_res();
fi_close(&dom->fid);
fi_close(&fab->fid);
return ret;
}
static int client_connect(void)
{
int ret;
ret = common_setup();
if (ret != 0)
return ret;
ret = ft_getdestaddr(opts.dst_addr, opts.dst_port, hints);
if (ret != 0)
goto err1;
ret = fi_av_insert(av, hints->dest_addr, 1, &remote_fi_addr, 0, NULL);
if (ret != 1) {
FT_PRINTERR("fi_av_insert", ret);
goto err2;
}
// send initial message to server with our local address
memcpy(buf_ptr, fi->src_addr, fi->src_addrlen);
ret = send_xfer(fi->src_addrlen);
if (ret != 0)
goto err2;
// wait for reply to know server is ready
ret = recv_xfer(4);
if (ret != 0)
goto err2;
return 0;
err2:
free(hints->dest_addr);
err1:
free_ep_res();
fi_close(&av->fid);
fi_close(&dom->fid);
fi_close(&fab->fid);
return ret;
}
static int run(void)
{
int ret = 0;
ret = init_fabric();
if (ret)
return ret;
ret = init_av();
if (ret)
goto out;
run_test();
/* TODO: Add a local finalize applicable to shared ctx */
//ft_finalize(ep[0], scq, rcq, remote_fi_addr[0]);
out:
free_ep_res();
fi_close(&dom->fid);
fi_close(&fab->fid);
return ret;
}
static int run_test(void)
{
int ret = 0;
ret = init_fabric();
if (ret)
return ret;
if (opts.dst_addr) {
/* Execute RMA write operation from Client */
fprintf(stdout, "RMA write to server\n");
sprintf(buf, "%s", welcome_text);
ret = write_data(sizeof(char *) * strlen(buf));
if (ret)
return ret;
ret = fi_cntr_wait(scntr, 1, -1);
if (ret < 0) {
FT_PRINTERR("fi_cntr_wait", ret);
return ret;
}
fprintf(stdout, "Received a completion event for RMA write\n");
} else {
/* Server waits for message from Client */
ret = fi_cntr_wait(rcntr, 1, -1);
if (ret < 0) {
FT_PRINTERR("fi_cntr_wait", ret);
return ret;
}
fprintf(stdout, "Received data from Client: %s\n", (char *)buf);
}
/* TODO: need support for finalize operation to sync test */
free_ep_res();
fi_close(&dom->fid);
fi_close(&fab->fid);
return 0;
}
static int run(void)
{
int ret = 0;
ret = init_fabric();
if (ret)
return ret;
ret = init_av();
if (ret)
goto out;
run_test();
/*TODO: Add a local finalize applicable for scalable ep */
//ft_finalize(tx_ep[0], scq[0], rcq[0], remote_rx_addr[0]);
out:
free_ep_res();
fi_close(&sep->fid);
fi_close(&dom->fid);
fi_close(&fab->fid);
return ret;
}
static int run(void)
{
int i, ret = 0;
if (!client) {
ret = server_listen();
if (ret)
return ret;
}
printf("%-8s%-8s%-8s%8s %10s%13s\n",
"bytes", "iters", "total", "time", "MB/sec", "usec/xfer");
ret = client ? client_connect() : server_connect();
if (ret)
return ret;
if (!custom) {
for (i = 0; i < TEST_CNT; i++) {
if (test_size[i].option > size_option)
continue;
init_test(test_size[i].size);
ret = run_test();
}
} else {
ret = run_test();
}
fi_shutdown(ep, 0);
fi_close(&ep->fid);
free_ep_res();
if (!client)
free_lres();
fi_close(&dom->fid);
fi_close(&fab->fid);
return ret;
}
static int run(void)
{
int i, ret = 0;
ret = dst_addr ? client_connect() : server_connect();
if (ret)
return ret;
printf("%-10s%-8s%-8s%-8s%-8s%8s %10s%13s\n",
"name", "bytes", "xfers", "iters", "total", "time",
"Gb/sec", "usec/xfer");
if (!custom) {
for (i = 0; i < TEST_CNT; i++) {
if (test_size[i].option > size_option ||
(max_msg_size && test_size[i].size > max_msg_size)) {
continue;
}
init_test(test_size[i].size);
run_test();
}
} else {
ret = run_test();
}
while (credits < max_credits)
poll_all_sends();
fi_shutdown(ep, 0);
fi_close(&ep->fid);
free_ep_res();
fi_close(&dom->fid);
fi_close(&fab->fid);
return ret;
}
static int init_fabric(void)
{
uint64_t flags = 0;
char *node, *service;
int ret;
if (opts.dst_addr) {
ret = ft_getsrcaddr(opts.src_addr, opts.src_port, hints);
if (ret)
return ret;
node = opts.dst_addr;
service = opts.dst_port;
} else {
node = opts.src_addr;
service = opts.src_port;
flags = FI_SOURCE;
}
ret = fi_getinfo(FT_FIVERSION, node, service, flags, hints, &fi);
if (ret) {
FT_PRINTERR("fi_getinfo", ret);
return ret;
}
// get remote address
if (opts.dst_addr) {
addrlen = fi->dest_addrlen;
remote_addr = malloc(addrlen);
memcpy(remote_addr, fi->dest_addr, addrlen);
}
ret = fi_fabric(fi->fabric_attr, &fab, NULL);
if (ret) {
FT_PRINTERR("fi_fabric", ret);
goto err0;
}
ret = fi_domain(fab, fi, &dom, NULL);
if (ret) {
FT_PRINTERR("fi_domain", ret);
goto err1;
}
ret = alloc_ep_res(fi);
if (ret)
goto err3;
ret = bind_ep_res();
if (ret)
goto err4;
return 0;
err4:
free_ep_res();
err3:
fi_close(&dom->fid);
err1:
fi_close(&fab->fid);
err0:
return ret;
}
static int common_setup(void)
{
int ret;
uint64_t flags = 0;
char *node, *service;
if (opts.dst_addr) {
ret = ft_getsrcaddr(opts.src_addr, opts.src_port, hints);
if (ret)
return ret;
node = opts.dst_addr;
service = opts.dst_port;
} else {
node = opts.src_addr;
service = opts.src_port;
flags = FI_SOURCE;
}
ret = fi_getinfo(FT_FIVERSION, node, service, flags, hints, &fi);
if (ret) {
FT_PRINTERR("fi_getinfo", ret);
goto err1;
}
if (fi->ep_attr->max_msg_size) {
max_msg_size = fi->ep_attr->max_msg_size;
}
ret = fi_fabric(fi->fabric_attr, &fab, NULL);
if (ret) {
FT_PRINTERR("fi_fabric", ret);
goto err1;
}
if (fi->mode & FI_MSG_PREFIX) {
prefix_len = fi->ep_attr->msg_prefix_size;
}
ret = fi_domain(fab, fi, &dom, NULL);
if (ret) {
FT_PRINTERR("fi_domain", ret);
goto err2;
}
ret = alloc_ep_res(fi);
if (ret) {
goto err4;
}
ret = bind_ep_res();
if (ret) {
goto err5;
}
free(hints->src_addr);
return 0;
err5:
free_ep_res();
err4:
fi_close(&dom->fid);
err2:
fi_close(&fab->fid);
err1:
free(hints->src_addr);
return ret;
}
static int init_fabric(void)
{
char *node, *service;
uint64_t flags = 0;
int ret;
ret = ft_read_addr_opts(&node, &service, hints, &flags, &opts);
if (ret)
return ret;
ret = fi_getinfo(FT_FIVERSION, node, service, flags, hints, &fi);
if (ret) {
FT_PRINTERR("fi_getinfo", ret);
return ret;
}
if (opts.dst_addr) {
addrlen = fi->dest_addrlen;
remote_addr = malloc(addrlen);
memcpy(remote_addr, fi->dest_addr, addrlen);
}
ret = fi_fabric(fi->fabric_attr, &fab, NULL);
if (ret) {
FT_PRINTERR("fi_fabric", ret);
goto err0;
}
ret = fi_domain(fab, fi, &dom, NULL);
if (ret) {
FT_PRINTERR("fi_domain", ret);
goto err1;
}
ret = alloc_ep_res(fi);
if (ret)
goto err3;
ret = bind_ep_res();
if (ret)
goto err4;
if(opts.dst_addr) {
ret = fi_av_insert(av, remote_addr, 1, &remote_fi_addr, 0,
&fi_ctx_av);
if (ret != 1) {
FT_PRINTERR("fi_av_insert", ret);
return ret;
}
}
return 0;
err4:
free_ep_res();
err3:
fi_close(&dom->fid);
err1:
fi_close(&fab->fid);
err0:
return ret;
}
static int init_fabric(void)
{
char *node, *service;
uint64_t flags = 0;
int ret;
ret = ft_read_addr_opts(&node, &service, hints, &flags, &opts);
if (ret)
return ret;
/* Get fabric info */
ret = fi_getinfo(FT_FIVERSION, node, service, flags, hints, &fi);
if (ret) {
FT_PRINTERR("fi_getinfo", ret);
return ret;
}
/* Get remote address of the server */
if (opts.dst_addr) {
addrlen = fi->dest_addrlen;
remote_addr = malloc(addrlen);
memcpy(remote_addr, fi->dest_addr, addrlen);
}
/* Open fabric */
ret = fi_fabric(fi->fabric_attr, &fab, NULL);
if (ret) {
FT_PRINTERR("fi_fabric", ret);
goto err1;
}
/* Open domain */
ret = fi_domain(fab, fi, &dom, NULL);
if (ret) {
FT_PRINTERR("fi_domain", ret);
goto err2;
}
ret = alloc_ep_res(fi);
if (ret)
goto err4;
ret = bind_ep_res();
if (ret)
goto err5;
if (opts.dst_addr) {
/* Insert address to the AV and get the fabric address back */
ret = fi_av_insert(av, remote_addr, 1, &remote_fi_addr, 0,
&fi_ctx_av);
if (ret != 1) {
FT_PRINTERR("fi_av_insert", ret);
return ret;
}
}
return 0;
err5:
free_ep_res();
err4:
fi_close(&dom->fid);
err2:
fi_close(&fab->fid);
err1:
return ret;
}
static int client_connect(void)
{
struct fi_eq_cm_entry entry;
uint32_t event;
struct fi_info *fi;
ssize_t rd;
int ret;
/* Get fabric info */
ret = fi_getinfo(FT_FIVERSION, opts.dst_addr, opts.dst_port, 0, hints, &fi);
if (ret) {
FT_PRINTERR("fi_getinfo", ret);
goto err0;
}
/* Open fabric */
ret = fi_fabric(fi->fabric_attr, &fab, NULL);
if (ret) {
FT_PRINTERR("fi_fabric", ret);
goto err1;
}
/* Open domain */
ret = fi_domain(fab, fi, &dom, NULL);
if (ret) {
FT_PRINTERR("fi_domain", ret);
goto err2;
}
ret = alloc_cm_res();
if (ret)
goto err4;
ret = alloc_ep_res(fi);
if (ret)
goto err5;
ret = bind_ep_res();
if (ret)
goto err6;
/* Connect to server */
ret = fi_connect(ep, fi->dest_addr, NULL, 0);
if (ret) {
FT_PRINTERR("fi_connect", ret);
goto err6;
}
/* Wait for the connection to be established */
rd = fi_eq_sread(cmeq, &event, &entry, sizeof entry, -1, 0);
if (rd != sizeof entry) {
FT_PROCESS_EQ_ERR(rd, cmeq, "fi_eq_sread", "connect");
ret = (int) rd;
goto err6;
}
if (event != FI_CONNECTED || entry.fid != &ep->fid) {
fprintf(stderr, "Unexpected CM event %d fid %p (ep %p)\n",
event, entry.fid, ep);
ret = -FI_EOTHER;
goto err6;
}
fi_freeinfo(fi);
return 0;
err6:
free_ep_res();
err5:
fi_close(&cmeq->fid);
err4:
fi_close(&dom->fid);
err2:
fi_close(&fab->fid);
err1:
fi_freeinfo(fi);
err0:
return ret;
}
static int server_connect(void)
{
struct fi_eq_cm_entry entry;
uint32_t event;
struct fi_info *info = NULL;
ssize_t rd;
int ret;
/* Wait for connection request from client */
rd = fi_eq_sread(cmeq, &event, &entry, sizeof entry, -1, 0);
if (rd != sizeof entry) {
FT_PROCESS_EQ_ERR(rd, cmeq, "fi_eq_sread", "listen");
return (int) rd;
}
info = entry.info;
if (event != FI_CONNREQ) {
fprintf(stderr, "Unexpected CM event %d\n", event);
ret = -FI_EOTHER;
goto err1;
}
ret = fi_domain(fab, info, &dom, NULL);
if (ret) {
FT_PRINTERR("fi_domain", ret);
goto err1;
}
ret = alloc_ep_res(info);
if (ret)
goto err1;
ret = bind_ep_res();
if (ret)
goto err3;
/* Accept the incoming connection. Also transitions endpoint to active state */
ret = fi_accept(ep, NULL, 0);
if (ret) {
FT_PRINTERR("fi_accept", ret);
goto err3;
}
/* Wait for the connection to be established */
rd = fi_eq_sread(cmeq, &event, &entry, sizeof entry, -1, 0);
if (rd != sizeof entry) {
FT_PROCESS_EQ_ERR(rd, cmeq, "fi_eq_sread", "accept");
ret = (int) rd;
goto err3;
}
if (event != FI_CONNECTED || entry.fid != &ep->fid) {
fprintf(stderr, "Unexpected CM event %d fid %p (ep %p)\n",
event, entry.fid, ep);
ret = -FI_EOTHER;
goto err3;
}
fi_freeinfo(info);
return 0;
err3:
free_ep_res();
err1:
fi_reject(pep, info->handle, NULL, 0);
fi_freeinfo(info);
return ret;
}
static int run(void)
{
int ret = 0;
ret = init_fabric();
if (ret)
return ret;
ret = init_av();
if (ret)
goto out;
// Receiver
if (opts.dst_addr) {
// search for initial tag, it should fail since the sender
// hasn't sent anyting
fprintf(stdout, "Searching msg with tag [%" PRIu64 "]\n", tag_data);
tagged_search(tag_data);
fprintf(stdout, "Posting buffer for msg with tag [%" PRIu64 "]\n",
tag_data + 1);
ret = post_recv(tag_data + 1);
if (ret)
goto out;
// synchronize with sender
fprintf(stdout, "\nSynchronizing with sender..\n\n");
ret = sync_test();
if (ret)
goto out;
// wait for the completion event of the next tag
ret = wait_for_tagged_completion(rcq, 1);
if (ret)
goto out;
fprintf(stdout, "Received completion event for msg with tag "
"[%" PRIu64 "]\n", tag_data + 1);
// search again for the initial tag, it should be successful now
fprintf(stdout,
"Searching msg with initial tag [%" PRIu64 "]\n",
tag_data);
tagged_search(tag_data);
// wait for the completion event of the initial tag
ret = recv_msg(tag_data);
if (ret)
goto out;
fprintf(stdout, "Posted buffer and received completion event for"
" msg with tag [%" PRIu64 "]\n", tag_data);
} else {
// Sender
// synchronize with receiver
fprintf(stdout, "Synchronizing with receiver..\n\n");
ret = sync_test();
if (ret)
goto out;
fprintf(stdout, "Sending msg with tag [%" PRIu64 "]\n",
tag_data);
ret = send_msg(16, tag_data);
if (ret)
goto out;
fprintf(stdout, "Sending msg with tag [%" PRIu64 "]\n",
tag_data + 1);
ret = send_msg(16, tag_data + 1);
if (ret)
goto out;
}
/* Finalize before closing ep */
ft_finalize(ep, scq, rcq, remote_fi_addr);
out:
free_ep_res();
fi_close(&dom->fid);
fi_close(&fab->fid);
return ret;
}
static int server_connect(void)
{
struct fi_eq_cm_entry entry;
uint32_t event;
struct fi_info *info = NULL;
ssize_t rd;
int ret;
rd = fi_eq_sread(cmeq, &event, &entry, sizeof entry, -1, 0);
if (rd != sizeof entry) {
printf("fi_eq_sread %zd %s\n", rd, fi_strerror((int) -rd));
return (int) rd;
}
if (event != FI_CONNREQ) {
printf("Unexpected CM event %d\n", event);
ret = -FI_EOTHER;
goto err1;
}
info = entry.info;
ret = fi_domain(fab, info, &dom, NULL);
if (ret) {
printf("fi_domain %s\n", fi_strerror(-ret));
goto err1;
}
ret = fi_endpoint(dom, info, &ep, NULL);
if (ret) {
printf("fi_endpoint for req %s\n", fi_strerror(-ret));
goto err1;
}
ret = alloc_ep_res(info);
if (ret)
goto err2;
ret = bind_ep_res();
if (ret)
goto err3;
ret = fi_accept(ep, NULL, 0);
if (ret) {
printf("fi_accept %s\n", fi_strerror(-ret));
goto err3;
}
rd = fi_eq_sread(cmeq, &event, &entry, sizeof entry, -1, 0);
if (rd != sizeof entry) {
printf("fi_eq_sread %zd %s\n", rd, fi_strerror((int) -rd));
goto err3;
}
if (event != FI_COMPLETE || entry.fid != &ep->fid) {
printf("Unexpected CM event %d fid %p (ep %p)\n",
event, entry.fid, ep);
ret = -FI_EOTHER;
goto err3;
}
fi_freeinfo(info);
return 0;
err3:
free_ep_res();
err2:
fi_close(&ep->fid);
err1:
fi_reject(pep, info->connreq, NULL, 0);
fi_freeinfo(info);
return ret;
}
static int client_connect(void)
{
struct fi_eq_cm_entry entry;
uint32_t event;
struct fi_info *fi;
ssize_t rd;
int ret;
if (src_addr) {
ret = getaddr(src_addr, NULL, (struct sockaddr **) &hints.src_addr,
(socklen_t *) &hints.src_addrlen);
if (ret)
printf("source address error %s\n", gai_strerror(ret));
}
ret = fi_getinfo(FI_VERSION(1, 0), dst_addr, port, 0, &hints, &fi);
if (ret) {
printf("fi_getinfo %s\n", strerror(-ret));
goto err0;
}
ret = fi_fabric(fi->fabric_attr, &fab, NULL);
if (ret) {
printf("fi_fabric %s\n", fi_strerror(-ret));
goto err1;
}
ret = fi_domain(fab, fi, &dom, NULL);
if (ret) {
printf("fi_domain %s %s\n", fi_strerror(-ret),
fi->domain_attr->name);
goto err2;
}
ret = fi_endpoint(dom, fi, &ep, NULL);
if (ret) {
printf("fi_endpoint %s\n", fi_strerror(-ret));
goto err3;
}
ret = alloc_ep_res(fi);
if (ret)
goto err4;
ret = bind_ep_res();
if (ret)
goto err5;
ret = fi_connect(ep, fi->dest_addr, NULL, 0);
if (ret) {
printf("fi_connect %s\n", fi_strerror(-ret));
goto err5;
}
rd = fi_eq_sread(cmeq, &event, &entry, sizeof entry, -1, 0);
if (rd != sizeof entry) {
printf("fi_eq_sread %zd %s\n", rd, fi_strerror((int) -rd));
return (int) rd;
}
if (event != FI_COMPLETE || entry.fid != &ep->fid) {
printf("Unexpected CM event %d fid %p (ep %p)\n",
event, entry.fid, ep);
ret = -FI_EOTHER;
goto err1;
}
if (hints.src_addr)
free(hints.src_addr);
fi_freeinfo(fi);
return 0;
err5:
free_ep_res();
err4:
fi_close(&ep->fid);
err3:
fi_close(&dom->fid);
err2:
fi_close(&fab->fid);
err1:
fi_freeinfo(fi);
err0:
if (hints.src_addr)
free(hints.src_addr);
return ret;
}
static int common_setup(void)
{
struct fi_info *fi;
int ret;
ret = getaddr(src_addr, port, (struct sockaddr **) &hints.src_addr,
(socklen_t *) &hints.src_addrlen);
if (ret)
printf("source address error %s\n", gai_strerror(ret));
ret = fi_getinfo(FI_VERSION(1, 0), dst_addr, port, 0, &hints, &fi);
if (ret) {
printf("fi_getinfo %s\n", strerror(-ret));
goto err0;
}
if (fi->ep_attr->max_msg_size) {
max_msg_size = fi->ep_attr->max_msg_size;
}
ret = fi_fabric(fi->fabric_attr, &fab, NULL);
if (ret) {
printf("fi_fabric %s\n", fi_strerror(-ret));
goto err1;
}
if (fi->mode & FI_MSG_PREFIX) {
prefix_len = fi->ep_attr->msg_prefix_size;
}
ret = fi_domain(fab, fi, &dom, NULL);
if (ret) {
printf("fi_fdomain %s %s\n", fi_strerror(-ret),
fi->domain_attr->name);
goto err2;
}
if (fi->src_addr != NULL) {
((struct sockaddr_in *)fi->src_addr)->sin_port =
((struct sockaddr_in *)hints.src_addr)->sin_port;
if (dst_addr == NULL) {
printf("Local address %s:%d\n",
inet_ntoa(((struct sockaddr_in *)fi->src_addr)->sin_addr),
ntohs(((struct sockaddr_in *)fi->src_addr)->sin_port));
}
}
ret = fi_endpoint(dom, fi, &ep, NULL);
if (ret) {
printf("fi_endpoint %s\n", fi_strerror(-ret));
goto err3;
}
ret = alloc_ep_res(fi);
if (ret) {
printf("alloc_ep_res %s\n", fi_strerror(-ret));
goto err4;
}
ret = bind_ep_res();
if (ret) {
printf("bind_ep_res %s\n", fi_strerror(-ret));
goto err5;
}
if (hints.src_addr)
free(hints.src_addr);
fi_freeinfo(fi);
return 0;
err5:
free_ep_res();
err4:
fi_close(&ep->fid);
err3:
fi_close(&dom->fid);
err2:
fi_close(&fab->fid);
err1:
fi_freeinfo(fi);
err0:
if (hints.src_addr)
free(hints.src_addr);
return ret;
}
