static int init_fabric(void)
{
uint64_t flags = 0;
char *node, *service;
int ret;
ret = ft_read_addr_opts(&node, &service, hints, &flags, &opts);
if (ret)
return ret;
ret = fi_getinfo(FT_FIVERSION, node, service, flags, hints, &fi);
if (ret) {
FT_PRINTERR("fi_getinfo", ret);
return ret;
}
ret = ft_open_fabric_res();
if (ret)
return ret;
ret = alloc_ep_res(fi);
if (ret)
return ret;
ret = ft_init_ep();
if (ret)
return ret;
return 0;
}
static int init_fabric(void)
{
int ret;
ret = ft_getinfo(hints, &fi);
if (ret)
return ret;
// set FI_MULTI_RECV flag for all recv operations
fi->rx_attr->op_flags = FI_MULTI_RECV;
ret = ft_open_fabric_res();
if (ret)
return ret;
ret = alloc_ep_res(fi);
if (ret)
return ret;
ret = ft_init_ep();
if (ret)
return ret;
ret = fi_setopt(&ep->fid, FI_OPT_ENDPOINT, FI_OPT_MIN_MULTI_RECV,
&tx_size, sizeof(tx_size));
if (ret)
return ret;
ret = post_multi_recv_buffer();
return ret;
}
static int init_fabric(void)
{
uint64_t flags = 0;
char *node, *service;
int ret;
ret = ft_read_addr_opts(&node, &service, hints, &flags, &opts);
if (ret)
return ret;
ret = fi_getinfo(FT_FIVERSION, node, service, flags, hints, &fi);
if (ret) {
FT_PRINTERR("fi_getinfo", ret);
return ret;
}
// we use provider MR attributes and direct address (no offsets)
// for RMA calls
if (!(fi->mode & FI_PROV_MR_ATTR))
fi->mode |= FI_PROV_MR_ATTR;
// get remote address
if (opts.dst_addr) {
addrlen = fi->dest_addrlen;
remote_addr = malloc(addrlen);
memcpy(remote_addr, fi->dest_addr, addrlen);
}
ret = fi_fabric(fi->fabric_attr, &fab, NULL);
if (ret) {
FT_PRINTERR("fi_fabric", ret);
goto err0;
}
ret = fi_domain(fab, fi, &dom, NULL);
if (ret) {
FT_PRINTERR("fi_domain", ret);
goto err1;
}
ret = alloc_ep_res(fi);
if (ret)
goto err3;
ret = bind_ep_res();
if (ret)
goto err4;
return 0;
err4:
free_ep_res();
err3:
fi_close(&dom->fid);
err1:
fi_close(&fab->fid);
err0:
return ret;
}
static int init_fabric(void)
{
int ret;
uint64_t flags = 0;
/* Get fabric info */
ret = fi_getinfo(CT_FIVERSION, NULL, NULL, flags, hints, &fi);
if (ret) {
ct_print_fi_error("fi_getinfo", ret);
return ret;
}
/* Open fabric */
ret = fi_fabric(fi->fabric_attr, &fab, NULL);
if (ret) {
ct_print_fi_error("fi_fabric", ret);
goto err1;
}
/* Open domain */
ret = fi_domain(fab, fi, &dom, NULL);
if (ret) {
ct_print_fi_error("fi_domain", ret);
goto err2;
}
/* Open endpoint */
ret = fi_endpoint(dom, fi, &ep, NULL);
if (ret) {
ct_print_fi_error("fi_endpoint", ret);
goto err3;
}
/* Allocate endpoint resources */
ret = alloc_ep_res();
if (ret)
goto err4;
/* Bind EQs and AVs with endpoint */
ret = bind_ep_res();
if (ret)
goto err5;
return 0;
err5:
free_ep_res();
err4:
fi_close(&ep->fid);
err3:
fi_close(&dom->fid);
err2:
fi_close(&fab->fid);
err1:
return ret;
}
static int client_connect(void)
{
struct fi_eq_cm_entry entry;
uint32_t event;
ssize_t rd;
int ret;
ret = ft_getsrcaddr(opts.src_addr, opts.src_port, hints);
if (ret)
return ret;
ret = fi_getinfo(FT_FIVERSION, opts.dst_addr, opts.dst_port, 0, hints, &fi);
if (ret) {
FT_PRINTERR("fi_getinfo", ret);
return ret;
}
ret = ft_open_fabric_res();
if (ret)
return ret;
ret = fi_domain(fabric, fi, &domain, NULL);
if (ret) {
FT_PRINTERR("fi_domain", ret);
return ret;
}
ret = alloc_ep_res(fi);
if (ret)
return ret;
ret = ft_init_ep(buf);
if (ret)
return ret;
ret = fi_connect(ep, fi->dest_addr, NULL, 0);
if (ret) {
FT_PRINTERR("fi_connect", ret);
return ret;
}
rd = fi_eq_sread(eq, &event, &entry, sizeof entry, -1, 0);
if (rd != sizeof entry) {
FT_PROCESS_EQ_ERR(rd, eq, "fi_eq_sread", "connect");
return (int) rd;
}
if (event != FI_CONNECTED || entry.fid != &ep->fid) {
fprintf(stderr, "Unexpected CM event %d fid %p (ep %p)\n",
event, entry.fid, ep);
return -FI_EOTHER;
}
return 0;
}
static int init_fabric(void)
{
char *node, *service;
uint64_t flags = 0;
int ret;
ret = ft_read_addr_opts(&node, &service, hints, &flags, &opts);
if (ret)
return ret;
/* Get fabric info */
ret = fi_getinfo(FT_FIVERSION, node, service, flags, hints, &fi);
if (ret) {
FT_PRINTERR("fi_getinfo", ret);
return ret;
}
/* Get remote address of the server */
if (opts.dst_addr) {
addrlen = fi->dest_addrlen;
remote_addr = malloc(addrlen);
memcpy(remote_addr, fi->dest_addr, addrlen);
}
ret = ft_open_fabric_res();
if (ret)
return ret;
/* Open domain */
ret = fi_domain(fabric, fi, &domain, NULL);
if (ret) {
FT_PRINTERR("fi_domain", ret);
return ret;
}
ret = alloc_ep_res(fi);
if (ret)
return ret;
ret = ft_init_ep(NULL);
if (ret)
return ret;
if (opts.dst_addr) {
/* Insert address to the AV and get the fabric address back */
ret = fi_av_insert(av, remote_addr, 1, &remote_fi_addr, 0,
&fi_ctx_av);
if (ret != 1) {
FT_PRINTERR("fi_av_insert", ret);
return ret;
}
}
return 0;
}
static int init_fabric(void)
{
uint64_t flags = 0;
char *node, *service;
int ret;
ret = ft_read_addr_opts(&node, &service, hints, &flags, &opts);
if (ret)
return ret;
ret = fi_getinfo(FT_FIVERSION, node, service, flags, hints, &fi);
if (ret) {
FT_PRINTERR("fi_getinfo", ret);
return ret;
}
/* Check the number of EPs supported by the provider */
if (ep_cnt > fi->domain_attr->ep_cnt) {
ep_cnt = fi->domain_attr->ep_cnt;
fprintf(stderr, "Provider can support only %d of EPs\n", ep_cnt);
}
/* Get remote address */
if (opts.dst_addr) {
addrlen = fi->dest_addrlen;
remote_addr = malloc(addrlen * ep_cnt);
memcpy(remote_addr, fi->dest_addr, addrlen);
}
ret = ft_open_fabric_res();
if (ret)
return ret;
ret = fi_domain(fabric, fi, &domain, NULL);
if (ret) {
FT_PRINTERR("fi_domain", ret);
return ret;
}
fi->ep_attr->tx_ctx_cnt = FI_SHARED_CONTEXT;
fi->ep_attr->rx_ctx_cnt = FI_SHARED_CONTEXT;
ret = alloc_ep_res(fi);
if (ret)
return ret;
ret = bind_ep_res();
if (ret)
return ret;
return 0;
}
static int init_fabric(void)
{
uint64_t flags = 0;
char *node, *service;
int ret;
ret = ft_read_addr_opts(&node, &service, hints, &flags, &opts);
if (ret)
return ret;
ret = fi_getinfo(FT_FIVERSION, node, service, flags, hints, &fi);
if (ret) {
FT_PRINTERR("fi_getinfo", ret);
return ret;
}
/* check max msg size */
max_inject_size = fi->tx_attr->inject_size;
if ((opts.options & FT_OPT_SIZE) &&
(opts.transfer_size > max_inject_size)) {
fprintf(stderr, "Msg size greater than max inject size\n");
return -FI_EINVAL;
}
/* Get remote address */
if (opts.dst_addr) {
addrlen = fi->dest_addrlen;
remote_addr = malloc(addrlen);
memcpy(remote_addr, fi->dest_addr, addrlen);
}
ret = ft_open_fabric_res();
if (ret)
return ret;
ret = fi_domain(fabric, fi, &domain, NULL);
if (ret) {
FT_PRINTERR("fi_domain", ret);
return ret;
}
ret = alloc_ep_res(fi);
if (ret)
return ret;
ret = ft_init_ep(NULL);
if (ret)
return ret;
return 0;
}
static int init_fabric(void)
{
uint64_t flags = 0;
char *node, *service;
int ret;
ret = ft_read_addr_opts(&node, &service, hints, &flags, &opts);
if (ret)
return ret;
ret = fi_getinfo(FT_FIVERSION, node, service, flags, hints, &fi);
if (ret) {
FT_PRINTERR("fi_getinfo", ret);
return ret;
}
/* Check the optimal number of TX and RX contexts supported by the provider */
ctx_cnt = MIN(ctx_cnt, fi->domain_attr->tx_ctx_cnt);
ctx_cnt = MIN(ctx_cnt, fi->domain_attr->rx_ctx_cnt);
if (!ctx_cnt) {
fprintf(stderr, "Provider doesn't support contexts\n");
return 1;
}
fi->ep_attr->tx_ctx_cnt = ctx_cnt;
fi->ep_attr->rx_ctx_cnt = ctx_cnt;
ret = ft_open_fabric_res();
if (ret)
return ret;
ret = fi_scalable_ep(domain, fi, &sep, NULL);
if (ret) {
FT_PRINTERR("fi_scalable_ep", ret);
return ret;
}
ret = alloc_ep_res(sep);
if (ret)
return ret;
ret = bind_ep_res();
return ret;
}
static int init_fabric(void)
{
uint64_t flags = 0;
char *node, *service;
int ret;
ret = ft_read_addr_opts(&node, &service, hints, &flags, &opts);
if (ret)
return ret;
ret = fi_getinfo(FT_FIVERSION, node, service, flags, hints, &fi);
if (ret) {
FT_PRINTERR("fi_getinfo", ret);
return ret;
}
/* Get remote address */
if (opts.dst_addr) {
addrlen = fi->dest_addrlen;
remote_addr = malloc(addrlen);
memcpy(remote_addr, fi->dest_addr, addrlen);
}
ret = ft_open_fabric_res();
if (ret)
return ret;
ret = fi_domain(fabric, fi, &domain, NULL);
if (ret) {
FT_PRINTERR("fi_domain", ret);
return ret;
}
ret = alloc_ep_res(fi);
if (ret)
return ret;
ret = ft_init_ep(NULL);
if (ret)
return ret;
return 0;
}
static int init_fabric(void)
{
int ret;
ret = ft_getinfo(hints, &fi);
if (ret)
return ret;
ret = get_dupinfo();
if (ret)
return ret;
ret = ft_open_fabric_res();
if (ret)
return ret;
av_attr.count = ep_cnt;
ret = alloc_ep_res(fi);
if (ret)
return ret;
ret = alloc_ep();
if (ret)
return ret;
ret = bind_ep_array_res();
if (ret)
return ret;
/* Post recv */
if (rx_shared_ctx)
ret = ft_post_rx(srx_ctx, MAX(rx_size, FT_MAX_CTRL_MSG), &rx_ctx);
else
ret = ft_post_rx(ep_array[0], MAX(rx_size, FT_MAX_CTRL_MSG), &rx_ctx);
if (ret)
return ret;
ret = init_av();
return ret;
}
static int init_fabric(void)
{
int ret;
ret = ft_getinfo(hints, &fi);
if (ret)
return ret;
ret = ft_open_fabric_res();
if (ret)
return ret;
ret = alloc_ep_res(fi);
if (ret)
return ret;
ret = ft_enable_ep_recv();
if (ret)
return ret;
return 0;
}
static int init_fabric(void)
{
int ret;
ret = ft_getinfo(hints, &fi);
if (ret)
return ret;
/* Check the optimal number of TX and RX contexts supported by the provider */
ctx_cnt = MIN(ctx_cnt, fi->domain_attr->tx_ctx_cnt);
ctx_cnt = MIN(ctx_cnt, fi->domain_attr->rx_ctx_cnt);
if (!ctx_cnt) {
fprintf(stderr, "Provider doesn't support contexts\n");
return 1;
}
fi->ep_attr->tx_ctx_cnt = ctx_cnt;
fi->ep_attr->rx_ctx_cnt = ctx_cnt;
ret = ft_open_fabric_res();
if (ret)
return ret;
ret = fi_scalable_ep(domain, fi, &sep, NULL);
if (ret) {
FT_PRINTERR("fi_scalable_ep", ret);
return ret;
}
ret = alloc_ep_res(sep);
if (ret)
return ret;
ret = bind_ep_res();
return ret;
}
static int server_connect(void)
{
struct fi_eq_cm_entry entry;
uint32_t event;
struct fi_info *info = NULL;
ssize_t rd;
int ret;
rd = fi_eq_sread(cmeq, &event, &entry, sizeof entry, -1, 0);
if (rd != sizeof entry) {
printf("fi_eq_sread %zd %s\n", rd, fi_strerror((int) -rd));
return (int) rd;
}
if (event != FI_CONNREQ) {
printf("Unexpected CM event %d\n", event);
ret = -FI_EOTHER;
goto err1;
}
info = entry.info;
ret = fi_domain(fab, info, &dom, NULL);
if (ret) {
printf("fi_domain %s\n", fi_strerror(-ret));
goto err1;
}
ret = fi_endpoint(dom, info, &ep, NULL);
if (ret) {
printf("fi_endpoint for req %s\n", fi_strerror(-ret));
goto err1;
}
ret = alloc_ep_res(info);
if (ret)
goto err2;
ret = bind_ep_res();
if (ret)
goto err3;
ret = fi_accept(ep, NULL, 0);
if (ret) {
printf("fi_accept %s\n", fi_strerror(-ret));
goto err3;
}
rd = fi_eq_sread(cmeq, &event, &entry, sizeof entry, -1, 0);
if (rd != sizeof entry) {
printf("fi_eq_sread %zd %s\n", rd, fi_strerror((int) -rd));
goto err3;
}
if (event != FI_COMPLETE || entry.fid != &ep->fid) {
printf("Unexpected CM event %d fid %p (ep %p)\n",
event, entry.fid, ep);
ret = -FI_EOTHER;
goto err3;
}
fi_freeinfo(info);
return 0;
err3:
free_ep_res();
err2:
fi_close(&ep->fid);
err1:
fi_reject(pep, info->connreq, NULL, 0);
fi_freeinfo(info);
return ret;
}
static int client_connect(void)
{
struct fi_eq_cm_entry entry;
uint32_t event;
struct fi_info *fi;
ssize_t rd;
int ret;
/* Get fabric info */
ret = fi_getinfo(FT_FIVERSION, opts.dst_addr, opts.dst_port, 0, hints, &fi);
if (ret) {
FT_PRINTERR("fi_getinfo", ret);
goto err0;
}
/* Open fabric */
ret = fi_fabric(fi->fabric_attr, &fab, NULL);
if (ret) {
FT_PRINTERR("fi_fabric", ret);
goto err1;
}
/* Open domain */
ret = fi_domain(fab, fi, &dom, NULL);
if (ret) {
FT_PRINTERR("fi_domain", ret);
goto err2;
}
ret = alloc_cm_res();
if (ret)
goto err4;
ret = alloc_ep_res(fi);
if (ret)
goto err5;
ret = bind_ep_res();
if (ret)
goto err6;
/* Connect to server */
ret = fi_connect(ep, fi->dest_addr, NULL, 0);
if (ret) {
FT_PRINTERR("fi_connect", ret);
goto err6;
}
/* Wait for the connection to be established */
rd = fi_eq_sread(cmeq, &event, &entry, sizeof entry, -1, 0);
if (rd != sizeof entry) {
FT_PROCESS_EQ_ERR(rd, cmeq, "fi_eq_sread", "connect");
ret = (int) rd;
goto err6;
}
if (event != FI_CONNECTED || entry.fid != &ep->fid) {
fprintf(stderr, "Unexpected CM event %d fid %p (ep %p)\n",
event, entry.fid, ep);
ret = -FI_EOTHER;
goto err6;
}
fi_freeinfo(fi);
return 0;
err6:
free_ep_res();
err5:
fi_close(&cmeq->fid);
err4:
fi_close(&dom->fid);
err2:
fi_close(&fab->fid);
err1:
fi_freeinfo(fi);
err0:
return ret;
}
static int server_connect(void)
{
struct fi_eq_cm_entry entry;
uint32_t event;
struct fi_info *info = NULL;
ssize_t rd;
int ret;
/* Wait for connection request from client */
rd = fi_eq_sread(cmeq, &event, &entry, sizeof entry, -1, 0);
if (rd != sizeof entry) {
FT_PROCESS_EQ_ERR(rd, cmeq, "fi_eq_sread", "listen");
return (int) rd;
}
info = entry.info;
if (event != FI_CONNREQ) {
fprintf(stderr, "Unexpected CM event %d\n", event);
ret = -FI_EOTHER;
goto err1;
}
ret = fi_domain(fab, info, &dom, NULL);
if (ret) {
FT_PRINTERR("fi_domain", ret);
goto err1;
}
ret = alloc_ep_res(info);
if (ret)
goto err1;
ret = bind_ep_res();
if (ret)
goto err3;
/* Accept the incoming connection. Also transitions endpoint to active state */
ret = fi_accept(ep, NULL, 0);
if (ret) {
FT_PRINTERR("fi_accept", ret);
goto err3;
}
/* Wait for the connection to be established */
rd = fi_eq_sread(cmeq, &event, &entry, sizeof entry, -1, 0);
if (rd != sizeof entry) {
FT_PROCESS_EQ_ERR(rd, cmeq, "fi_eq_sread", "accept");
ret = (int) rd;
goto err3;
}
if (event != FI_CONNECTED || entry.fid != &ep->fid) {
fprintf(stderr, "Unexpected CM event %d fid %p (ep %p)\n",
event, entry.fid, ep);
ret = -FI_EOTHER;
goto err3;
}
fi_freeinfo(info);
return 0;
err3:
free_ep_res();
err1:
fi_reject(pep, info->handle, NULL, 0);
fi_freeinfo(info);
return ret;
}
static int init_fabric(void)
{
struct fi_info *fi;
char *node;
uint64_t flags = 0;
int ret;
if (src_addr) {
ret = getaddr(src_addr, NULL,
(struct sockaddr **) &hints.src_addr,
(socklen_t *) &hints.src_addrlen);
if (ret) {
fprintf(stderr, "source address error %s\n",
gai_strerror(ret));
return ret;
}
}
if (dst_addr) {
node = dst_addr;
} else {
node = src_addr;
flags = FI_SOURCE;
}
ret = fi_getinfo(FI_VERSION(1, 0), node, port, flags, &hints, &fi);
if (ret) {
FI_PRINTERR("fi_getinfo", ret);
return ret;
}
// we use provider MR attributes and direct address (no offsets)
// for RMA calls
if (!(fi->mode & FI_PROV_MR_ATTR))
fi->mode |= FI_PROV_MR_ATTR;
// get remote address
if (dst_addr) {
addrlen = fi->dest_addrlen;
remote_addr = malloc(addrlen);
memcpy(remote_addr, fi->dest_addr, addrlen);
}
ret = fi_fabric(fi->fabric_attr, &fab, NULL);
if (ret) {
FI_PRINTERR("fi_fabric", ret);
goto err0;
}
ret = fi_domain(fab, fi, &dom, NULL);
if (ret) {
FI_PRINTERR("fi_domain", ret);
goto err1;
}
ret = fi_endpoint(dom, fi, &ep, NULL);
if (ret) {
FI_PRINTERR("fi_endpoint", ret);
goto err2;
}
if (dst_addr == NULL) {
printf("EP opened on fabric %s\n", fi->fabric_attr->name);
}
ret = alloc_ep_res(fi);
if (ret)
goto err3;
ret = bind_ep_res();
if (ret)
goto err4;
return 0;
err4:
free_ep_res();
err3:
fi_close(&ep->fid);
err2:
fi_close(&dom->fid);
err1:
fi_close(&fab->fid);
err0:
fi_freeinfo(fi);
return ret;
}
static int common_setup(void)
{
struct fi_info *fi;
int ret;
ret = getaddr(src_addr, port, (struct sockaddr **) &hints.src_addr,
(socklen_t *) &hints.src_addrlen);
if (ret)
printf("source address error %s\n", gai_strerror(ret));
ret = fi_getinfo(FI_VERSION(1, 0), dst_addr, port, 0, &hints, &fi);
if (ret) {
printf("fi_getinfo %s\n", strerror(-ret));
goto err0;
}
if (fi->ep_attr->max_msg_size) {
max_msg_size = fi->ep_attr->max_msg_size;
}
ret = fi_fabric(fi->fabric_attr, &fab, NULL);
if (ret) {
printf("fi_fabric %s\n", fi_strerror(-ret));
goto err1;
}
if (fi->mode & FI_MSG_PREFIX) {
prefix_len = fi->ep_attr->msg_prefix_size;
}
ret = fi_domain(fab, fi, &dom, NULL);
if (ret) {
printf("fi_fdomain %s %s\n", fi_strerror(-ret),
fi->domain_attr->name);
goto err2;
}
if (fi->src_addr != NULL) {
((struct sockaddr_in *)fi->src_addr)->sin_port =
((struct sockaddr_in *)hints.src_addr)->sin_port;
if (dst_addr == NULL) {
printf("Local address %s:%d\n",
inet_ntoa(((struct sockaddr_in *)fi->src_addr)->sin_addr),
ntohs(((struct sockaddr_in *)fi->src_addr)->sin_port));
}
}
ret = fi_endpoint(dom, fi, &ep, NULL);
if (ret) {
printf("fi_endpoint %s\n", fi_strerror(-ret));
goto err3;
}
ret = alloc_ep_res(fi);
if (ret) {
printf("alloc_ep_res %s\n", fi_strerror(-ret));
goto err4;
}
ret = bind_ep_res();
if (ret) {
printf("bind_ep_res %s\n", fi_strerror(-ret));
goto err5;
}
if (hints.src_addr)
free(hints.src_addr);
fi_freeinfo(fi);
return 0;
err5:
free_ep_res();
err4:
fi_close(&ep->fid);
err3:
fi_close(&dom->fid);
err2:
fi_close(&fab->fid);
err1:
fi_freeinfo(fi);
err0:
if (hints.src_addr)
free(hints.src_addr);
return ret;
}
static int server_connect(void)
{
struct fi_eq_cm_entry entry;
uint32_t event;
ssize_t rd;
int ret, k;
int num_conn_reqs = 0, num_connected = 0;
struct ep_info *ep_state_array = NULL;
ep_array = calloc(ep_cnt, sizeof(*ep_array));
if (!ep_array)
return -FI_ENOMEM;
ep_state_array = calloc(ep_cnt, sizeof(*ep_state_array));
if (!ep_state_array)
return -FI_ENOMEM;
while (num_connected != ep_cnt) {
rd = fi_eq_sread(eq, &event, &entry, sizeof entry, -1, 0);
if (rd != sizeof entry) {
FT_PROCESS_EQ_ERR(rd, eq, "fi_eq_sread", "cm-event");
ret = (int) rd;
goto err;
}
switch(event) {
case FI_CONNREQ:
if (num_conn_reqs == ep_cnt) {
fprintf(stderr, "Unexpected CM event %d\n", event);
ret = -FI_EOTHER;
goto err;
}
fi = ep_state_array[num_conn_reqs].fi = entry.info;
ep_state_array[num_conn_reqs].state = FT_EP_CONNECT_RCVD;
if (num_conn_reqs == 0) {
ret = fi_domain(fabric, fi, &domain, NULL);
if (ret) {
FT_PRINTERR("fi_domain", ret);
goto err;
}
ret = alloc_ep_res(fi);
if (ret)
goto err;
}
ret = fi_endpoint(domain, fi, &ep_array[num_conn_reqs], NULL);
if (ret) {
FT_PRINTERR("fi_endpoint", ret);
goto err;
}
ep_state_array[num_conn_reqs].ep = ep_array[num_conn_reqs];
ret = bind_ep_res(ep_array[num_conn_reqs]);
if (ret)
goto err;
ret = fi_accept(ep_array[num_conn_reqs], NULL, 0);
if (ret) {
FT_PRINTERR("fi_accept", ret);
goto err;
}
ep_state_array[num_conn_reqs].state = FT_EP_CONNECTING;
num_conn_reqs++;
break;
case FI_CONNECTED:
if (num_conn_reqs <= num_connected) {
ret = -FI_EOTHER;
goto err;
}
for (k = 0; k < num_conn_reqs; k++) {
if (ep_state_array[k].state != FT_EP_CONNECTING)
continue;
if (&ep_state_array[k].ep->fid == entry.fid) {
ep_state_array[k].state = FT_EP_CONNECTED;
num_connected++;
if (num_connected != ep_cnt)
fi_freeinfo(ep_state_array[k].fi);
break;
}
}
if (k == num_conn_reqs) {
fprintf(stderr, "Unexpected CM event %d fid %p (ep %p)\n",
event, entry.fid, ep);
ret = -FI_EOTHER;
goto err;
}
break;
default:
ret = -FI_EOTHER;
goto err;
}
}
/* Post recv */
if (rx_shared_ctx)
ret = ft_post_rx(srx_ctx, MAX(rx_size, FT_MAX_CTRL_MSG), &rx_ctx);
else
ret = ft_post_rx(ep_array[0], MAX(rx_size, FT_MAX_CTRL_MSG), &rx_ctx);
if (ret)
goto err;
free(ep_state_array);
return 0;
err:
for (k = 0; k < ep_cnt; k++) {
switch(ep_state_array[k].state) {
case FT_EP_CONNECT_RCVD:
fi_reject(pep, ep_state_array[k].fi->handle, NULL, 0);
break;
case FT_EP_CONNECTING:
case FT_EP_CONNECTED:
fi_shutdown(ep_state_array[k].ep, 0);
break;
case FT_EP_STATE_INIT:
default:
break;
}
}
free(ep_state_array);
return ret;
}
static int client_connect(void)
{
struct fi_eq_cm_entry entry;
uint32_t event;
ssize_t rd;
int i, ret;
ret = ft_getinfo(hints, &fi);
if (ret)
return ret;
ret = get_dupinfo();
if (ret)
return ret;
ret = ft_open_fabric_res();
if (ret)
return ret;
ret = alloc_ep_res(fi);
if (ret)
return ret;
ret = alloc_ep();
if (ret)
return ret;
ret = bind_ep_array_res();
if (ret)
return ret;
for (i = 0; i < ep_cnt; i++) {
ret = fi_connect(ep_array[i], fi->dest_addr, NULL, 0);
if (ret) {
FT_PRINTERR("fi_connect", ret);
return ret;
}
rd = fi_eq_sread(eq, &event, &entry, sizeof entry, -1, 0);
if (rd != sizeof entry) {
FT_PROCESS_EQ_ERR(rd, eq, "fi_eq_sread", "connect");
ret = (int) rd;
return ret;
}
if (event != FI_CONNECTED || entry.fid != &ep_array[i]->fid) {
fprintf(stderr, "Unexpected CM event %d fid %p (ep %p)\n",
event, entry.fid, ep);
ret = -FI_EOTHER;
return ret;
}
}
/* Post recv */
if (rx_shared_ctx)
ret = ft_post_rx(srx_ctx, MAX(rx_size, FT_MAX_CTRL_MSG), &rx_ctx);
else
ret = ft_post_rx(ep_array[0], MAX(rx_size, FT_MAX_CTRL_MSG), &rx_ctx);
if (ret)
return ret;
return 0;
}
static int init_fabric(void)
{
struct fi_info *fi;
uint64_t flags = 0;
char *node, *service;
int ret;
if (opts.dst_addr) {
ret = ft_getsrcaddr(opts.src_addr, opts.src_port, hints);
if (ret)
return ret;
node = opts.dst_addr;
service = opts.dst_port;
} else {
node = opts.src_addr;
service = opts.src_port;
flags = FI_SOURCE;
}
ret = fi_getinfo(FT_FIVERSION, node, service, flags, hints, &fi);
if (ret) {
FT_PRINTERR("fi_getinfo", ret);
return ret;
}
/* We use provider MR attributes and direct address (no offsets)
* for RMA calls */
if (!(fi->mode & FI_PROV_MR_ATTR))
fi->mode |= FI_PROV_MR_ATTR;
/* Get remote address */
if (opts.dst_addr) {
addrlen = fi->dest_addrlen;
remote_addr = malloc(addrlen);
memcpy(remote_addr, fi->dest_addr, addrlen);
}
ret = fi_fabric(fi->fabric_attr, &fab, NULL);
if (ret) {
FT_PRINTERR("fi_fabric", ret);
goto err0;
}
ret = fi_domain(fab, fi, &dom, NULL);
if (ret) {
FT_PRINTERR("fi_domain", ret);
goto err1;
}
ret = alloc_ep_res(fi);
if (ret)
goto err3;
ret = bind_ep_res();
if (ret)
goto err4;
return 0;
err4:
free_ep_res();
err3:
fi_close(&dom->fid);
err1:
fi_close(&fab->fid);
err0:
return ret;
}
static int server_connect(void)
{
struct fi_eq_cm_entry entry;
uint32_t event;
struct fi_info *info = NULL;
ssize_t rd;
int ret;
rd = fi_eq_sread(eq, &event, &entry, sizeof entry, -1, 0);
if (rd != sizeof entry) {
FT_PROCESS_EQ_ERR(rd, eq, "fi_eq_sread", "listen");
return (int) rd;
}
info = entry.info;
if (event != FI_CONNREQ) {
fprintf(stderr, "Unexpected CM event %d\n", event);
ret = -FI_EOTHER;
goto err;
}
ret = fi_domain(fabric, info, &domain, NULL);
if (ret) {
FT_PRINTERR("fi_domain", ret);
goto err;
}
ret = alloc_ep_res(info);
if (ret)
goto err;
ret = ft_init_ep(buf);
if (ret)
goto err;
ret = fi_accept(ep, NULL, 0);
if (ret) {
FT_PRINTERR("fi_accept", ret);
goto err;
}
rd = fi_eq_sread(eq, &event, &entry, sizeof entry, -1, 0);
if (rd != sizeof entry) {
FT_PROCESS_EQ_ERR(rd, eq, "fi_eq_sread", "accept");
ret = (int) rd;
goto err;
}
if (event != FI_CONNECTED || entry.fid != &ep->fid) {
fprintf(stderr, "Unexpected CM event %d fid %p (ep %p)\n",
event, entry.fid, ep);
ret = -FI_EOTHER;
goto err;
}
fi_freeinfo(info);
return 0;
err:
fi_reject(pep, info->handle, NULL, 0);
fi_freeinfo(info);
return ret;
}
static int init_fabric(void)
{
char *node, *service;
uint64_t flags = 0;
int ret;
ret = ft_read_addr_opts(&node, &service, hints, &flags, &opts);
if (ret)
return ret;
ret = fi_getinfo(FT_FIVERSION, node, service, flags, hints, &fi);
if (ret) {
FT_PRINTERR("fi_getinfo", ret);
return ret;
}
if (opts.dst_addr) {
addrlen = fi->dest_addrlen;
remote_addr = malloc(addrlen);
memcpy(remote_addr, fi->dest_addr, addrlen);
}
ret = fi_fabric(fi->fabric_attr, &fab, NULL);
if (ret) {
FT_PRINTERR("fi_fabric", ret);
goto err0;
}
ret = fi_domain(fab, fi, &dom, NULL);
if (ret) {
FT_PRINTERR("fi_domain", ret);
goto err1;
}
ret = alloc_ep_res(fi);
if (ret)
goto err3;
ret = bind_ep_res();
if (ret)
goto err4;
if(opts.dst_addr) {
ret = fi_av_insert(av, remote_addr, 1, &remote_fi_addr, 0,
&fi_ctx_av);
if (ret != 1) {
FT_PRINTERR("fi_av_insert", ret);
return ret;
}
}
return 0;
err4:
free_ep_res();
err3:
fi_close(&dom->fid);
err1:
fi_close(&fab->fid);
err0:
return ret;
}
static int server_connect(void)
{
struct fi_eq_cm_entry entry;
uint32_t event;
struct fi_info *info = NULL;
ssize_t rd;
int ret;
rd = fi_eq_sread(cmeq, &event, &entry, sizeof entry, -1, 0);
if (rd != sizeof entry) {
FT_PRINTERR("fi_eq_sread", rd);
return (int) rd;
}
info = entry.info;
if (event != FI_CONNREQ) {
fprintf(stderr, "Unexpected CM event %d\n", event);
ret = -FI_EOTHER;
goto err1;
}
ret = fi_domain(fab, info, &dom, NULL);
if (ret) {
FT_PRINTERR("fi_domain", ret);
goto err1;
}
ret = fi_endpoint(dom, info, &ep, NULL);
if (ret) {
FT_PRINTERR("fi_endpoint", -ret);
goto err1;
}
ret = alloc_ep_res(info);
if (ret)
goto err1;
ret = bind_ep_res();
if (ret)
goto err3;
ret = fi_accept(ep, NULL, 0);
if (ret) {
FT_PRINTERR("fi_accept", ret);
goto err3;
}
rd = fi_eq_sread(cmeq, &event, &entry, sizeof entry, -1, 0);
if (rd != sizeof entry) {
FT_PRINTERR("fi_eq_sread", rd);
goto err3;
}
if (event != FI_CONNECTED || entry.fid != &ep->fid) {
fprintf(stderr, "Unexpected CM event %d fid %p (ep %p)\n",
event, entry.fid, ep);
ret = -FI_EOTHER;
goto err3;
}
fi_freeinfo(info);
return 0;
err3:
free_ep_res();
err1:
fi_reject(pep, info->connreq, NULL, 0);
fi_freeinfo(info);
return ret;
}
static int init_fabric(void)
{
char *node, *service;
uint64_t flags = 0;
int ret;
ret = ft_read_addr_opts(&node, &service, hints, &flags, &opts);
if (ret)
return ret;
/* Get fabric info */
ret = fi_getinfo(FT_FIVERSION, node, service, flags, hints, &fi);
if (ret) {
FT_PRINTERR("fi_getinfo", ret);
return ret;
}
/* Get remote address of the server */
if (opts.dst_addr) {
addrlen = fi->dest_addrlen;
remote_addr = malloc(addrlen);
memcpy(remote_addr, fi->dest_addr, addrlen);
}
/* Open fabric */
ret = fi_fabric(fi->fabric_attr, &fab, NULL);
if (ret) {
FT_PRINTERR("fi_fabric", ret);
goto err1;
}
/* Open domain */
ret = fi_domain(fab, fi, &dom, NULL);
if (ret) {
FT_PRINTERR("fi_domain", ret);
goto err2;
}
ret = alloc_ep_res(fi);
if (ret)
goto err4;
ret = bind_ep_res();
if (ret)
goto err5;
if (opts.dst_addr) {
/* Insert address to the AV and get the fabric address back */
ret = fi_av_insert(av, remote_addr, 1, &remote_fi_addr, 0,
&fi_ctx_av);
if (ret != 1) {
FT_PRINTERR("fi_av_insert", ret);
return ret;
}
}
return 0;
err5:
free_ep_res();
err4:
fi_close(&dom->fid);
err2:
fi_close(&fab->fid);
err1:
return ret;
}
static int client_connect(void)
{
struct fi_eq_cm_entry entry;
uint32_t event;
struct fi_info *fi;
ssize_t rd;
int ret;
ret = ft_getsrcaddr(opts.src_addr, opts.src_port, hints);
if (ret)
return ret;
ret = fi_getinfo(FT_FIVERSION, opts.dst_addr, opts.dst_port, 0, hints, &fi);
if (ret) {
FT_PRINTERR("fi_getinfo", ret);
goto err0;
}
ret = fi_fabric(fi->fabric_attr, &fab, NULL);
if (ret) {
FT_PRINTERR("fi_fabric", ret);
goto err1;
}
ret = fi_domain(fab, fi, &dom, NULL);
if (ret) {
FT_PRINTERR("fi_domain", ret);
goto err2;
}
ret = fi_endpoint(dom, fi, &ep, NULL);
if (ret) {
FT_PRINTERR("fi_endpoint", ret);
goto err3;
}
ret = alloc_ep_res(fi);
if (ret)
goto err3;
ret = bind_ep_res();
if (ret)
goto err5;
ret = fi_connect(ep, fi->dest_addr, NULL, 0);
if (ret) {
FT_PRINTERR("fi_connect", ret);
goto err5;
}
rd = fi_eq_sread(cmeq, &event, &entry, sizeof entry, -1, 0);
if (rd != sizeof entry) {
FT_PRINTERR("fi_eq_sread", rd);
return (int) rd;
}
if (event != FI_CONNECTED || entry.fid != &ep->fid) {
fprintf(stderr, "Unexpected CM event %d fid %p (ep %p)\n",
event, entry.fid, ep);
ret = -FI_EOTHER;
goto err1;
}
fi_freeinfo(fi);
return 0;
err5:
free_ep_res();
err3:
fi_close(&dom->fid);
err2:
fi_close(&fab->fid);
err1:
fi_freeinfo(fi);
err0:
return ret;
}
static int common_setup(void)
{
int ret;
uint64_t flags = 0;
char *node, *service;
if (opts.dst_addr) {
ret = ft_getsrcaddr(opts.src_addr, opts.src_port, hints);
if (ret)
return ret;
node = opts.dst_addr;
service = opts.dst_port;
} else {
node = opts.src_addr;
service = opts.src_port;
flags = FI_SOURCE;
}
ret = fi_getinfo(FT_FIVERSION, node, service, flags, hints, &fi);
if (ret) {
FT_PRINTERR("fi_getinfo", ret);
goto err1;
}
if (fi->ep_attr->max_msg_size) {
max_msg_size = fi->ep_attr->max_msg_size;
}
ret = fi_fabric(fi->fabric_attr, &fab, NULL);
if (ret) {
FT_PRINTERR("fi_fabric", ret);
goto err1;
}
if (fi->mode & FI_MSG_PREFIX) {
prefix_len = fi->ep_attr->msg_prefix_size;
}
ret = fi_domain(fab, fi, &dom, NULL);
if (ret) {
FT_PRINTERR("fi_domain", ret);
goto err2;
}
ret = alloc_ep_res(fi);
if (ret) {
goto err4;
}
ret = bind_ep_res();
if (ret) {
goto err5;
}
free(hints->src_addr);
return 0;
err5:
free_ep_res();
err4:
fi_close(&dom->fid);
err2:
fi_close(&fab->fid);
err1:
free(hints->src_addr);
return ret;
}
static int client_connect(void)
{
struct fi_eq_cm_entry entry;
uint32_t event;
struct fi_info *fi;
ssize_t rd;
int ret;
if (src_addr) {
ret = getaddr(src_addr, NULL, (struct sockaddr **) &hints.src_addr,
(socklen_t *) &hints.src_addrlen);
if (ret)
printf("source address error %s\n", gai_strerror(ret));
}
ret = fi_getinfo(FI_VERSION(1, 0), dst_addr, port, 0, &hints, &fi);
if (ret) {
printf("fi_getinfo %s\n", strerror(-ret));
goto err0;
}
ret = fi_fabric(fi->fabric_attr, &fab, NULL);
if (ret) {
printf("fi_fabric %s\n", fi_strerror(-ret));
goto err1;
}
ret = fi_domain(fab, fi, &dom, NULL);
if (ret) {
printf("fi_domain %s %s\n", fi_strerror(-ret),
fi->domain_attr->name);
goto err2;
}
ret = fi_endpoint(dom, fi, &ep, NULL);
if (ret) {
printf("fi_endpoint %s\n", fi_strerror(-ret));
goto err3;
}
ret = alloc_ep_res(fi);
if (ret)
goto err4;
ret = bind_ep_res();
if (ret)
goto err5;
ret = fi_connect(ep, fi->dest_addr, NULL, 0);
if (ret) {
printf("fi_connect %s\n", fi_strerror(-ret));
goto err5;
}
rd = fi_eq_sread(cmeq, &event, &entry, sizeof entry, -1, 0);
if (rd != sizeof entry) {
printf("fi_eq_sread %zd %s\n", rd, fi_strerror((int) -rd));
return (int) rd;
}
if (event != FI_COMPLETE || entry.fid != &ep->fid) {
printf("Unexpected CM event %d fid %p (ep %p)\n",
event, entry.fid, ep);
ret = -FI_EOTHER;
goto err1;
}
if (hints.src_addr)
free(hints.src_addr);
fi_freeinfo(fi);
return 0;
err5:
free_ep_res();
err4:
fi_close(&ep->fid);
err3:
fi_close(&dom->fid);
err2:
fi_close(&fab->fid);
err1:
fi_freeinfo(fi);
err0:
if (hints.src_addr)
free(hints.src_addr);
return ret;
}
static int init_fabric(void)
{
uint64_t flags = 0;
char *node, *service;
int ret;
if (opts.dst_addr) {
ret = ft_getsrcaddr(opts.src_addr, opts.src_port, hints);
if (ret)
return ret;
node = opts.dst_addr;
service = opts.dst_port;
} else {
node = opts.src_addr;
service = opts.src_port;
flags = FI_SOURCE;
}
ret = fi_getinfo(FT_FIVERSION, node, service, flags, hints, &fi);
if (ret) {
FT_PRINTERR("fi_getinfo", ret);
return ret;
}
// get remote address
if (opts.dst_addr) {
addrlen = fi->dest_addrlen;
remote_addr = malloc(addrlen);
memcpy(remote_addr, fi->dest_addr, addrlen);
}
ret = fi_fabric(fi->fabric_attr, &fab, NULL);
if (ret) {
FT_PRINTERR("fi_fabric", ret);
goto err0;
}
ret = fi_domain(fab, fi, &dom, NULL);
if (ret) {
FT_PRINTERR("fi_domain", ret);
goto err1;
}
ret = alloc_ep_res(fi);
if (ret)
goto err3;
ret = bind_ep_res();
if (ret)
goto err4;
return 0;
err4:
free_ep_res();
err3:
fi_close(&dom->fid);
err1:
fi_close(&fab->fid);
err0:
return ret;
}
static int init_fabric(void)
{
uint64_t flags = 0;
char *node, *service;
int ret;
ret = ft_read_addr_opts(&node, &service, hints, &flags, &opts);
if (ret)
return ret;
ret = fi_getinfo(FT_FIVERSION, node, service, flags, hints, &fi);
if (ret) {
FT_PRINTERR("fi_getinfo", ret);
return ret;
}
/* Check the optimal number of TX and RX contexts supported by the provider */
ctx_cnt = MIN(ctx_cnt, fi->domain_attr->tx_ctx_cnt);
ctx_cnt = MIN(ctx_cnt, fi->domain_attr->rx_ctx_cnt);
if (!ctx_cnt) {
fprintf(stderr, "Provider doesn't support contexts\n");
return 1;
}
/* We use provider MR attributes and direct address (no offsets) for RMA calls */
if (!(fi->mode & FI_PROV_MR_ATTR))
fi->mode |= FI_PROV_MR_ATTR;
/* Get remote address */
if (opts.dst_addr) {
addrlen = fi->dest_addrlen;
remote_addr = malloc(addrlen);
memcpy(remote_addr, fi->dest_addr, addrlen);
}
ret = fi_fabric(fi->fabric_attr, &fab, NULL);
if (ret) {
FT_PRINTERR("fi_fabric", ret);
goto err0;
}
ret = fi_domain(fab, fi, &dom, NULL);
if (ret) {
FT_PRINTERR("fi_domain", ret);
goto err1;
}
/* Set the required number of TX and RX context counts */
fi->ep_attr->tx_ctx_cnt = ctx_cnt;
fi->ep_attr->rx_ctx_cnt = ctx_cnt;
ret = fi_scalable_ep(dom, fi, &sep, NULL);
if (ret) {
FT_PRINTERR("fi_scalable_ep", ret);
goto err2;
}
ret = alloc_ep_res(sep);
if (ret)
goto err3;
ret = bind_ep_res();
if (ret)
goto err4;
return 0;
err4:
free_ep_res();
err3:
fi_close(&sep->fid);
err2:
fi_close(&dom->fid);
err1:
fi_close(&fab->fid);
err0:
return ret;
}