Test(endpoint_info, info)
{
int ret;
hints = fi_allocinfo();
cr_assert(hints, "fi_allocinfo");
hints->fabric_attr->prov_name = strdup("gni");
ret = fi_getinfo(FI_VERSION(1, 0), NULL, 0, 0, hints, &fi);
cr_assert(!ret, "fi_getinfo");
cr_assert_eq(fi->ep_attr->type, FI_EP_RDM);
cr_assert_eq(fi->next->ep_attr->type, FI_EP_DGRAM);
cr_assert_eq(fi->next->next->ep_attr->type, FI_EP_MSG);
fi_freeinfo(fi);
hints->ep_attr->type = FI_EP_RDM;
ret = fi_getinfo(FI_VERSION(1, 0), NULL, 0, 0, hints, &fi);
cr_assert(!ret, "fi_getinfo");
cr_assert_eq(fi->ep_attr->type, FI_EP_RDM);
fi_freeinfo(fi);
hints->ep_attr->type = FI_EP_DGRAM;
ret = fi_getinfo(FI_VERSION(1, 0), NULL, 0, 0, hints, &fi);
cr_assert(!ret, "fi_getinfo");
cr_assert_eq(fi->ep_attr->type, FI_EP_DGRAM);
fi_freeinfo(fi);
fi_freeinfo(hints);
}
static int get_dupinfo(void)
{
struct fi_info *hints_dup;
int ret;
/* Get a fi_info corresponding to a wild card port. The first endpoint
* should use default/given port since that is what is known to both
* client and server. For other endpoints we should use addresses with
* random ports to avoid collision. fi_getinfo should return a random
* port if we don't specify it in the service arg or the hints. This
* is used only for non-MSG endpoints. */
hints_dup = fi_dupinfo(hints);
if (!hints_dup)
return -FI_ENOMEM;
free(hints_dup->src_addr);
free(hints_dup->dest_addr);
hints_dup->src_addr = NULL;
hints_dup->dest_addr = NULL;
hints_dup->src_addrlen = 0;
hints_dup->dest_addrlen = 0;
if (opts.dst_addr) {
ret = fi_getinfo(FT_FIVERSION, opts.dst_addr, NULL, 0,
hints_dup, &fi_dup);
} else {
ret = fi_getinfo(FT_FIVERSION, opts.src_addr, NULL, FI_SOURCE,
hints_dup, &fi_dup);
}
if (ret)
FT_PRINTERR("fi_getinfo", ret);
fi_freeinfo(hints_dup);
return ret;
}
/*
* Note: the default addr_format is FI_ADDR_STR unless use_str_fmt is otherwise
* set to false.
*/
void do_invalid_fi_getinfo(void)
{
int i, ret;
for (i = 0; i < NUMEPS; i++) {
/*
* This test is to ensure that gni provider fails to provide
* info if the FI_ADDR_STR format is being used and both the
* node and service parameters are non-NULL.
*
* See the fi_getinfo man page DESCRIPTION section.
*/
ret = fi_getinfo(FI_VERSION(1, 5), "this is a test", "testing",
0, hints, &fi[i]);
cr_assert(ret == -FI_ENODATA, "fi_getinfo returned: %s",
fi_strerror(-ret));
fi_freeinfo(fi[i]);
/*
* This test is to ensure that the gni provider does not allow
* FI_ADDR_STR to be used with api versions <= 1.5.
*/
ret = fi_getinfo(FI_VERSION(1, 0), NULL, NULL, 0, hints,
&fi[i]);
cr_assert(ret == -FI_ENODATA, "fi_getinfo returned: %s",
fi_strerror(-ret));
fi_freeinfo(fi[i]);
}
}
void do_valid_fi_getinfo_with_fas(void)
{
int i, ret;
char *fas = calloc(GNIX_FI_ADDR_STR_LEN, sizeof(char));
struct gnix_ep_name ep_name;
/*
* This test ensures the gni provider can set addresses properly with
* FI_ADDR_STR and no flags set.
*/
for (i = 0; i < NUMEPS; i++) {
generate_rand_fas(&fas);
ret = fi_getinfo(fi_version(), fas, NULL, 0, hints, &fi[i]);
cr_assert(ret == FI_SUCCESS, "fi_getinfo returned: %s",
fi_strerror(-ret));
dbg_printf(BLUE "fi[%d]->dest_addr = %s\n" COLOR_RESET,
i, (char *) fi[i]->dest_addr);
fas_to_ep_name(fas, &ep_name);
check_ep_name(((struct gnix_ep_name *) fi[i]->dest_addr)[0],
ep_name);
fi_freeinfo(fi[i]);
}
/*
* This test ensures the gni provider can set addresses properly with
* FI_ADDR_STR and the FI_SOURCE set.
*/
for (i = 0; i < NUMEPS; i++) {
generate_rand_fas(&fas);
ret = fi_getinfo(fi_version(), fas, NULL, FI_SOURCE, hints,
&fi[i]);
cr_assert(ret == FI_SUCCESS, "fi_getinfo returned: %s",
fi_strerror(-ret));
fas_to_ep_name(fas, &ep_name);
check_ep_name(((struct gnix_ep_name *) fi[i]->src_addr)[0],
ep_name);
fi_freeinfo(fi[i]);
}
free(fas);
}
void dg_setup_prog_manual(void)
{
int ret = 0;
hints = fi_allocinfo();
cr_assert(hints, "fi_allocinfo");
hints->domain_attr->cq_data_size = 4;
hints->domain_attr->control_progress = FI_PROGRESS_MANUAL;
hints->mode = ~0;
hints->fabric_attr->name = strdup("gni");
ret = fi_getinfo(FI_VERSION(1, 0), NULL, 0, 0, hints, &fi);
cr_assert(!ret, "fi_getinfo");
ret = fi_fabric(fi->fabric_attr, &fab, NULL);
cr_assert(!ret, "fi_fabric");
ret = fi_domain(fab, fi, &dom, NULL);
cr_assert(!ret, "fi_domain");
ret = fi_endpoint(dom, fi, &ep, NULL);
cr_assert(!ret, "fi_endpoint");
}
void dg_setup(void)
{
int ret = 0;
char my_hostname[HOST_NAME_MAX];
hints = fi_allocinfo();
cr_assert(hints, "fi_allocinfo");
hints->domain_attr->cq_data_size = 4;
hints->mode = ~0;
hints->fabric_attr->name = strdup("gni");
ret = gethostname(my_hostname, sizeof(my_hostname));
cr_assert(!ret, "gethostname");
ret = fi_getinfo(FI_VERSION(1, 0), my_hostname, my_cdm_id, FI_SOURCE,
hints, &fi);
cr_assert(!ret, "fi_getinfo");
ret = fi_fabric(fi->fabric_attr, &fab, NULL);
cr_assert(!ret, "fi_fabric");
ret = fi_domain(fab, fi, &dom, NULL);
cr_assert(!ret, "fi_domain");
ret = fi_endpoint(dom, fi, &ep, NULL);
cr_assert(!ret, "fi_endpoint");
}
static int setup_av_ep(struct fid_ep **ep, fi_addr_t *remote_addr)
{
int ret;
hints->src_addr = NULL;
fi_freeinfo(fi);
ret = fi_getinfo(FT_FIVERSION, NULL, NULL, 0, hints, &fi);
if (ret) {
FT_PRINTERR("fi_getinfo", ret);
return ret;
}
ret = fi_endpoint(domain, fi, ep, NULL);
if (ret) {
FT_PRINTERR("fi_endpoint", ret);
return ret;
}
ret = ft_setup_ep(*ep, eq, av, txcq, rxcq, txcntr, rxcntr, false);
if (ret)
return ret;
ret = ft_init_av_addr(av, *ep, remote_addr);
if (ret)
return ret;
return 0;
}
int ofi_get_core_info(uint32_t version, const char *node, const char *service,
uint64_t flags, const struct util_prov *util_prov,
const struct fi_info *util_hints, ofi_alter_info_t info_to_core,
struct fi_info **core_info)
{
struct fi_info *core_hints = NULL;
int ret;
ret = ofi_prov_check_info(util_prov, version, util_hints);
if (ret)
return ret;
ret = ofi_info_to_core(version, util_prov->prov, util_hints, info_to_core,
&core_hints);
if (ret)
return ret;
FI_DBG(util_prov->prov, FI_LOG_CORE, "--- Begin ofi_get_core_info ---\n");
ret = fi_getinfo(version, node, service, flags | OFI_CORE_PROV_ONLY,
core_hints, core_info);
FI_DBG(util_prov->prov, FI_LOG_CORE, "--- End ofi_get_core_info ---\n");
fi_freeinfo(core_hints);
return ret;
}
static int init_fabric(void)
{
uint64_t flags = 0;
char *node, *service;
int ret;
ret = ft_read_addr_opts(&node, &service, hints, &flags, &opts);
if (ret)
return ret;
ret = fi_getinfo(FT_FIVERSION, node, service, flags, hints, &fi);
if (ret) {
FT_PRINTERR("fi_getinfo", ret);
return ret;
}
ret = ft_open_fabric_res();
if (ret)
return ret;
ret = alloc_ep_res(fi);
if (ret)
return ret;
ret = ft_init_ep();
if (ret)
return ret;
return 0;
}
int cm_server_start(void)
{
int ret;
struct sockaddr_in loc_sa;
cm_local_ip(&loc_sa);
srv_hints = fi_allocinfo();
srv_hints->fabric_attr->name = strdup("gni");
srv_hints->ep_attr->type = FI_EP_MSG;
srv_hints->domain_attr->mr_mode = GNIX_DEFAULT_MR_MODE;
ret = fi_getinfo(fi_version(), inet_ntoa(loc_sa.sin_addr),
DEF_PORT, FI_SOURCE, srv_hints, &srv_fi);
cr_assert(!ret);
ret = fi_fabric(srv_fi->fabric_attr, &srv_fab, NULL);
cr_assert(!ret);
ret = fi_eq_open(srv_fab, &eq_attr, &srv_eq, NULL);
cr_assert(!ret);
ret = fi_passive_ep(srv_fab, srv_fi, &srv_pep, NULL);
cr_assert(!ret);
ret = fi_pep_bind(srv_pep, &srv_eq->fid, 0);
cr_assert(!ret);
ret = fi_listen(srv_pep);
cr_assert(!ret);
dbg_printf("Server start complete.\n");
return 0;
}
static int run(struct fi_info *hints, char *node, char *port)
{
struct fi_info *info;
int ret;
uint64_t flags;
flags = list_providers ? FI_PROV_ATTR_ONLY : 0;
ret = fi_getinfo(FI_VERSION(FI_MAJOR_VERSION, FI_MINOR_VERSION),
node, port, flags, hints, &info);
if (ret) {
fprintf(stderr, "fi_getinfo: %d\n", ret);
return ret;
}
if (env)
ret = print_vars();
else if (verbose)
ret = print_long_info(info);
else if (list_providers)
ret = print_providers(info);
else
ret = print_short_info(info);
fi_freeinfo(info);
return ret;
}
static int client_setup(void)
{
size_t opt_size;
int ret;
/* Get fabric info */
ret = fi_getinfo(FT_FIVERSION, opts.dst_addr, opts.dst_port, 0, hints,
&fi);
if (ret) {
FT_PRINTERR("fi_getinfo", ret);
return ret;
}
ret = ft_open_fabric_res();
if (ret)
return ret;
ret = ft_alloc_active_res(fi);
if (ret)
return ret;
ret = ft_init_ep();
if (ret)
return ret;
/* Get the maximum cm_size for this domain + endpoint combination */
opt_size = sizeof(opt_size);
return fi_getopt(&ep->fid, FI_OPT_ENDPOINT, FI_OPT_CM_DATA_SIZE,
&cm_data_size, &opt_size);
}
static inline void cntr_setup_eps(void)
{
int i, ret;
struct fi_av_attr attr;
hints = fi_allocinfo();
cr_assert(hints, "fi_allocinfo");
hints->domain_attr->cq_data_size = 4;
hints->mode = ~0;
hints->fabric_attr->name = strdup("gni");
ret = fi_getinfo(FI_VERSION(1, 0), NULL, 0, 0, hints, &fi);
cr_assert(!ret, "fi_getinfo");
ret = fi_fabric(fi->fabric_attr, &fab, NULL);
cr_assert(!ret, "fi_fabric");
ret = fi_domain(fab, fi, &dom, NULL);
cr_assert(!ret, "fi_domain");
attr.type = FI_AV_MAP;
attr.count = 16;
ret = fi_av_open(dom, &attr, &av, NULL);
cr_assert(!ret, "fi_av_open");
for (i = 0; i < NUM_EPS; i++) {
ret = fi_endpoint(dom, fi, &ep[i], NULL);
cr_assert(!ret, "fi_endpoint");
}
}
/*
* rpmem_fip_getinfo -- (internal) get fabric interface information
*/
static int
rpmem_fip_getinfo(struct rpmem_fip *fip, const char *node, const char *service,
enum rpmem_provider provider)
{
int ret = 0;
struct fi_info *hints = rpmem_fip_get_hints(provider);
if (!hints) {
RPMEM_LOG(ERR, "!getting fabric interface information hints");
goto err_hints;
}
ret = fi_getinfo(RPMEM_FIVERSION, node, service,
0, hints, &fip->fi);
if (ret) {
RPMEM_FI_ERR(ret, "getting fabric interface information");
goto err_fi_getinfo;
}
rpmem_fip_print_info(fip->fi);
/* fallback to free the hints */
err_fi_getinfo:
fi_freeinfo(hints);
err_hints:
return ret;
}
static int getaddr(char *node, char *service,
struct fi_info *hints, uint64_t flags)
{
int ret;
struct fi_info *fi;
if (!node && !service) {
if (flags & FI_SOURCE) {
hints->src_addr = NULL;
hints->src_addrlen = 0;
} else {
hints->dest_addr = NULL;
hints->dest_addrlen = 0;
}
return 0;
}
ret = fi_getinfo(FT_FIVERSION, node, service, flags, hints, &fi);
if (ret) {
FT_PRINTERR("fi_getinfo", ret);
return ret;
}
hints->addr_format = fi->addr_format;
if (flags & FI_SOURCE) {
ret = dupaddr(&hints->src_addr, &hints->src_addrlen,
fi->src_addr, fi->src_addrlen);
} else {
ret = dupaddr(&hints->dest_addr, &hints->dest_addrlen,
fi->dest_addr, fi->dest_addrlen);
}
fi_freeinfo(fi);
return ret;
}
/*
* rpmemd_fip_getinfo -- obtain fabric interface information
*/
static int
rpmemd_fip_getinfo(struct rpmemd_fip *fip, const char *service,
const char *node, enum rpmem_provider provider)
{
int ret;
struct fi_info *hints = rpmem_fip_get_hints(provider);
if (!hints) {
RPMEMD_LOG(ERR, "getting fabric interface hints");
ret = -1;
goto err_fi_get_hints;
}
ret = fi_getinfo(RPMEM_FIVERSION, node, service, 0,
hints, &fip->fi);
if (ret) {
RPMEMD_FI_ERR(ret, "getting fabric interface information");
goto err_fi_getinfo;
}
if (fip->fi->addr_format != FI_SOCKADDR_IN) {
RPMEMD_LOG(ERR, "unsupported address family -- %d",
fip->fi->addr_format);
goto err_addr_format;
}
fi_freeinfo(hints);
return 0;
err_addr_format:
fi_freeinfo(fip->fi);
err_fi_getinfo:
fi_freeinfo(hints);
err_fi_get_hints:
return ret;
}
void rdm_sr_bnd_ep_setup(void)
{
int ret = 0, i = 0;
char my_hostname[HOST_NAME_MAX];
hints = fi_allocinfo();
cr_assert(hints, "fi_allocinfo");
hints->domain_attr->cq_data_size = NUMEPS * 2;
hints->mode = ~0;
hints->fabric_attr->name = strdup("gni");
ret = gethostname(my_hostname, sizeof(my_hostname));
cr_assert(!ret, "gethostname");
for (; i < NUMEPS; i++) {
ret = fi_getinfo(FI_VERSION(1, 0), my_hostname,
cdm_id[i], 0, hints, fi + i);
cr_assert(!ret, "fi_getinfo");
}
using_bnd_ep = 1;
rdm_sr_setup_common();
}
void rdm_sr_setup(bool is_noreg, enum fi_progress pm)
{
int ret = 0, i = 0;
hints = fi_allocinfo();
cr_assert(hints, "fi_allocinfo");
hints->domain_attr->cq_data_size = NUMEPS * 2;
hints->domain_attr->control_progress = pm;
hints->domain_attr->data_progress = pm;
hints->mode = ~0;
hints->caps = is_noreg ? hints->caps : FI_SOURCE | FI_MSG;
hints->fabric_attr->name = strdup("gni");
/* Get info about fabric services with the provided hints */
for (; i < NUMEPS; i++) {
ret = fi_getinfo(FI_VERSION(1, 0), NULL, 0, 0, hints, &fi[i]);
cr_assert(!ret, "fi_getinfo");
}
if (is_noreg)
rdm_sr_setup_common_eps();
else
rdm_sr_setup_common();
}
static int init_fabric(void)
{
uint64_t flags = 0;
char *node, *service;
int ret;
ret = ft_read_addr_opts(&node, &service, hints, &flags, &opts);
if (ret)
return ret;
ret = fi_getinfo(FT_FIVERSION, node, service, flags, hints, &fi);
if (ret) {
FT_PRINTERR("fi_getinfo", ret);
return ret;
}
// we use provider MR attributes and direct address (no offsets)
// for RMA calls
if (!(fi->mode & FI_PROV_MR_ATTR))
fi->mode |= FI_PROV_MR_ATTR;
// get remote address
if (opts.dst_addr) {
addrlen = fi->dest_addrlen;
remote_addr = malloc(addrlen);
memcpy(remote_addr, fi->dest_addr, addrlen);
}
ret = fi_fabric(fi->fabric_attr, &fab, NULL);
if (ret) {
FT_PRINTERR("fi_fabric", ret);
goto err0;
}
ret = fi_domain(fab, fi, &dom, NULL);
if (ret) {
FT_PRINTERR("fi_domain", ret);
goto err1;
}
ret = alloc_ep_res(fi);
if (ret)
goto err3;
ret = bind_ep_res();
if (ret)
goto err4;
return 0;
err4:
free_ep_res();
err3:
fi_close(&dom->fid);
err1:
fi_close(&fab->fid);
err0:
return ret;
}
/******************************************************************************
* Begin setup routines
******************************************************************************/
static void fas_setup_common(uint32_t version)
{
int i, ret;
srand(time(NULL));
get_fid[EP] = get_ep_fid;
get_fid[PEP] = get_pep_fid;
get_fid[SEP] = get_sep_fid;
/* This is sufficient for verifying FI_ADDR_STR with seps */
ctx_cnt = 1;
hints = fi_allocinfo();
cr_assert(hints, "fi_allocinfo");
hints->domain_attr->mr_mode = GNIX_DEFAULT_MR_MODE;
hints->domain_attr->cq_data_size = NUMEPS * 2;
hints->domain_attr->data_progress = FI_PROGRESS_AUTO;
hints->domain_attr->control_progress = FI_PROGRESS_AUTO;
hints->mode = mode_bits;
hints->fabric_attr->prov_name = strdup("gni");
hints->addr_format = use_str_fmt ? FI_ADDR_STR : FI_ADDR_GNI;
if (ep_type == SEP) {
hints->ep_attr->tx_ctx_cnt = NUMCONTEXTS;
hints->ep_attr->rx_ctx_cnt = NUMCONTEXTS;
}
/* Get info about fabric services with the provided hints */
for (i = 0; i < NUMEPS; i++) {
ret = fi_getinfo(version, NULL, 0, 0, hints,
&fi[i]);
cr_assert(!ret, "fi_getinfo returned: %s", fi_strerror(-ret));
tx_cq[i] = calloc(ctx_cnt, sizeof(*tx_cq));
rx_cq[i] = calloc(ctx_cnt, sizeof(*rx_cq));
tx_ep[i] = calloc(ctx_cnt, sizeof(*tx_ep));
rx_ep[i] = calloc(ctx_cnt, sizeof(*rx_ep));
}
memset(&attr, 0, sizeof(attr));
attr.rx_ctx_bits = ctx_cnt;
attr.count = NUMEPS;
cq_attr.format = FI_CQ_FORMAT_TAGGED;
cq_attr.size = 1024;
cq_attr.wait_obj = 0;
for (i = 0; i < NUMEPS; i++) {
target[i] = malloc(BUF_SZ);
cr_assert(target[i], "malloc returned: %s", strerror(errno));
source[i] = malloc(BUF_SZ);
cr_assert(source[i], "malloc returned: %s", strerror(errno));
}
ret = fi_fabric(fi[0]->fabric_attr, &fab, NULL);
cr_assert(!ret, "fi_fabric returned: %s", fi_strerror(-ret));
}
FiInfo get_fi_info(const Interface& iface, fi_info& hints, uint64_t flags) {
fi_info* info{};
auto ret = fi_getinfo(FI_VER, iface.traddr.c_str(), iface.trsvcid.c_str(), flags, &hints, &info);
if (ret) {
throw FabricException("fi_getinfo error: " + fi_error_to_string(ret), ret);
}
return FiInfo(info);
}
static int ft_fw_server(void)
{
struct fi_info *hints, *info;
int ret;
hints = fi_allocinfo();
if (!hints)
return -FI_ENOMEM;
do {
ret = ft_fw_recv(sock, &test_info, sizeof test_info);
if (ret) {
if (ret == -FI_ENOTCONN)
ret = 0;
break;
}
ft_fw_convert_info(hints, &test_info);
printf("Starting test %d-%d: ", test_info.test_index,
test_info.test_subindex);
ft_show_test_info();
ret = fi_getinfo(FT_FIVERSION, ft_strptr(test_info.node),
ft_strptr(test_info.service), FI_SOURCE,
hints, &info);
if (ret) {
FT_PRINTERR("fi_getinfo", ret);
} else {
if (info->next) {
printf("fi_getinfo returned multiple matches\n");
ret = -FI_E2BIG;
} else {
/* fabric_info is replaced when connecting */
fabric_info = info;
ret = ft_run_test();
if (fabric_info != info)
fi_freeinfo(fabric_info);
}
fi_freeinfo(info);
}
if (ret) {
printf("Node: %s\nService: %s\n",
test_info.node, test_info.service);
printf("%s\n", fi_tostr(hints, FI_TYPE_INFO));
}
printf("Ending test %d-%d, result: %s\n", test_info.test_index,
test_info.test_subindex, fi_strerror(-ret));
results[ft_fw_result_index(-ret)]++;
ret = ft_fw_send(sock, &ret, sizeof ret);
} while (!ret);
fi_freeinfo(hints);
return ret;
}
int cm_client_start_connect(void)
{
int ret;
struct sockaddr_in loc_sa;
cm_local_ip(&loc_sa);
cli_hints = fi_allocinfo();
cli_hints->fabric_attr->name = strdup("gni");
cli_hints->caps = GNIX_EP_PRIMARY_CAPS;
cli_hints->ep_attr->type = FI_EP_MSG;
cli_hints->domain_attr->mr_mode = GNIX_DEFAULT_MR_MODE;
ret = fi_getinfo(fi_version(), inet_ntoa(loc_sa.sin_addr),
DEF_PORT, 0, cli_hints, &cli_fi);
cr_assert(!ret);
ret = fi_fabric(cli_fi->fabric_attr, &cli_fab, NULL);
cr_assert(!ret);
ret = fi_eq_open(cli_fab, &eq_attr, &cli_eq, NULL);
cr_assert(!ret);
ret = fi_domain(cli_fab, cli_fi, &cli_dom, NULL);
cr_assert(!ret);
ret = fi_endpoint(cli_dom, cli_fi, &cli_ep, NULL);
cr_assert(!ret, "fi_endpoint");
cq_attr.format = FI_CQ_FORMAT_TAGGED;
cq_attr.size = 1024;
cq_attr.wait_obj = 0;
ret = fi_cq_open(cli_dom, &cq_attr, &cli_cq, &cli_cq);
cr_assert(!ret);
ret = fi_ep_bind(cli_ep, &cli_eq->fid, 0);
cr_assert(!ret);
ret = fi_ep_bind(cli_ep, &cli_cq->fid, FI_SEND | FI_RECV);
cr_assert(!ret);
ret = fi_enable(cli_ep);
cr_assert(!ret);
ret = fi_connect(cli_ep, cli_fi->dest_addr, cli_cm_in_data,
GNIX_CM_DATA_MAX_SIZE+1);
cr_assert(ret == -FI_EINVAL);
ret = fi_connect(cli_ep, cli_fi->dest_addr, cli_cm_in_data,
strlen(cli_cm_in_data));
cr_assert(!ret);
dbg_printf("Client connect complete.\n");
return 0;
}
static int server_listen(void)
{
struct fi_info *fi;
int ret;
/* Get fabric info */
ret = fi_getinfo(FT_FIVERSION, NULL, opts.src_port, FI_SOURCE, hints, &fi);
if (ret) {
FT_PRINTERR("fi_getinfo", ret);
return ret;
}
/* Open the fabric */
ret = fi_fabric(fi->fabric_attr, &fab, NULL);
if (ret) {
FT_PRINTERR("fi_fabric", ret);
goto err0;
}
/* Open a passive endpoint */
ret = fi_passive_ep(fab, fi, &pep, NULL);
if (ret) {
FT_PRINTERR("fi_passive_ep", ret);
goto err1;
}
/* Allocate connection management resources */
ret = alloc_cm_res();
if (ret)
goto err2;
/* Bind EQ to passive endpoint */
ret = fi_pep_bind(pep, &cmeq->fid, 0);
if (ret) {
FT_PRINTERR("fi_pep_bind", ret);
goto err3;
}
/* Listen for incoming connections */
ret = fi_listen(pep);
if (ret) {
FT_PRINTERR("fi_listen", ret);
goto err3;
}
fi_freeinfo(fi);
return 0;
err3:
fi_close(&cmeq->fid);
err2:
fi_close(&pep->fid);
err1:
fi_close(&fab->fid);
err0:
fi_freeinfo(fi);
return ret;
}
static int init_fabric(void)
{
int ret;
uint64_t flags = 0;
/* Get fabric info */
ret = fi_getinfo(CT_FIVERSION, NULL, NULL, flags, hints, &fi);
if (ret) {
ct_print_fi_error("fi_getinfo", ret);
return ret;
}
/* Open fabric */
ret = fi_fabric(fi->fabric_attr, &fab, NULL);
if (ret) {
ct_print_fi_error("fi_fabric", ret);
goto err1;
}
/* Open domain */
ret = fi_domain(fab, fi, &dom, NULL);
if (ret) {
ct_print_fi_error("fi_domain", ret);
goto err2;
}
/* Open endpoint */
ret = fi_endpoint(dom, fi, &ep, NULL);
if (ret) {
ct_print_fi_error("fi_endpoint", ret);
goto err3;
}
/* Allocate endpoint resources */
ret = alloc_ep_res();
if (ret)
goto err4;
/* Bind EQs and AVs with endpoint */
ret = bind_ep_res();
if (ret)
goto err5;
return 0;
err5:
free_ep_res();
err4:
fi_close(&ep->fid);
err3:
fi_close(&dom->fid);
err2:
fi_close(&fab->fid);
err1:
return ret;
}
static int client_connect(void)
{
struct fi_eq_cm_entry entry;
uint32_t event;
ssize_t rd;
int ret;
ret = ft_getsrcaddr(opts.src_addr, opts.src_port, hints);
if (ret)
return ret;
ret = fi_getinfo(FT_FIVERSION, opts.dst_addr, opts.dst_port, 0, hints, &fi);
if (ret) {
FT_PRINTERR("fi_getinfo", ret);
return ret;
}
ret = ft_open_fabric_res();
if (ret)
return ret;
ret = fi_domain(fabric, fi, &domain, NULL);
if (ret) {
FT_PRINTERR("fi_domain", ret);
return ret;
}
ret = alloc_ep_res(fi);
if (ret)
return ret;
ret = ft_init_ep(buf);
if (ret)
return ret;
ret = fi_connect(ep, fi->dest_addr, NULL, 0);
if (ret) {
FT_PRINTERR("fi_connect", ret);
return ret;
}
rd = fi_eq_sread(eq, &event, &entry, sizeof entry, -1, 0);
if (rd != sizeof entry) {
FT_PROCESS_EQ_ERR(rd, eq, "fi_eq_sread", "connect");
return (int) rd;
}
if (event != FI_CONNECTED || entry.fid != &ep->fid) {
fprintf(stderr, "Unexpected CM event %d fid %p (ep %p)\n",
event, entry.fid, ep);
return -FI_EOTHER;
}
return 0;
}
static int init_fabric(void)
{
char *node, *service;
uint64_t flags = 0;
int ret;
ret = ft_read_addr_opts(&node, &service, hints, &flags, &opts);
if (ret)
return ret;
/* Get fabric info */
ret = fi_getinfo(FT_FIVERSION, node, service, flags, hints, &fi);
if (ret) {
FT_PRINTERR("fi_getinfo", ret);
return ret;
}
/* Get remote address of the server */
if (opts.dst_addr) {
addrlen = fi->dest_addrlen;
remote_addr = malloc(addrlen);
memcpy(remote_addr, fi->dest_addr, addrlen);
}
ret = ft_open_fabric_res();
if (ret)
return ret;
/* Open domain */
ret = fi_domain(fabric, fi, &domain, NULL);
if (ret) {
FT_PRINTERR("fi_domain", ret);
return ret;
}
ret = alloc_ep_res(fi);
if (ret)
return ret;
ret = ft_init_ep(NULL);
if (ret)
return ret;
if (opts.dst_addr) {
/* Insert address to the AV and get the fabric address back */
ret = fi_av_insert(av, remote_addr, 1, &remote_fi_addr, 0,
&fi_ctx_av);
if (ret != 1) {
FT_PRINTERR("fi_av_insert", ret);
return ret;
}
}
return 0;
}
static int server_listen(void)
{
struct fi_info *fi;
int ret;
ret = fi_getinfo(FI_VERSION(1, 0), src_addr, port, FI_SOURCE, &hints, &fi);
if (ret) {
printf("fi_getinfo %s\n", strerror(-ret));
return ret;
}
cq_data_size = fi->domain_attr->cq_data_size;
ret = fi_fabric(fi->fabric_attr, &fab, NULL);
if (ret) {
printf("fi_fabric %s\n", fi_strerror(-ret));
goto err0;
}
ret = fi_passive_ep(fab, fi, &pep, NULL);
if (ret) {
printf("fi_passive_ep %s\n", fi_strerror(-ret));
goto err1;
}
ret = alloc_cm_res();
if (ret)
goto err2;
ret = fi_bind(&pep->fid, &cmeq->fid, 0);
if (ret) {
printf("fi_bind %s\n", fi_strerror(-ret));
goto err3;
}
ret = fi_listen(pep);
if (ret) {
printf("fi_listen %s\n", fi_strerror(-ret));
goto err3;
}
fi_freeinfo(fi);
return 0;
err3:
free_lres();
err2:
fi_close(&pep->fid);
err1:
fi_close(&fab->fid);
err0:
fi_freeinfo(fi);
return ret;
}
static int init_fabric(void)
{
uint64_t flags = 0;
char *node, *service;
int ret;
ret = ft_read_addr_opts(&node, &service, hints, &flags, &opts);
if (ret)
return ret;
ret = fi_getinfo(FT_FIVERSION, node, service, flags, hints, &fi);
if (ret) {
FT_PRINTERR("fi_getinfo", ret);
return ret;
}
/* Check the number of EPs supported by the provider */
if (ep_cnt > fi->domain_attr->ep_cnt) {
ep_cnt = fi->domain_attr->ep_cnt;
fprintf(stderr, "Provider can support only %d of EPs\n", ep_cnt);
}
/* Get remote address */
if (opts.dst_addr) {
addrlen = fi->dest_addrlen;
remote_addr = malloc(addrlen * ep_cnt);
memcpy(remote_addr, fi->dest_addr, addrlen);
}
ret = ft_open_fabric_res();
if (ret)
return ret;
ret = fi_domain(fabric, fi, &domain, NULL);
if (ret) {
FT_PRINTERR("fi_domain", ret);
return ret;
}
fi->ep_attr->tx_ctx_cnt = FI_SHARED_CONTEXT;
fi->ep_attr->rx_ctx_cnt = FI_SHARED_CONTEXT;
ret = alloc_ep_res(fi);
if (ret)
return ret;
ret = bind_ep_res();
if (ret)
return ret;
return 0;
}
static int server_listen(void)
{
struct fi_info *fi;
int ret;
ret = fi_getinfo(FT_FIVERSION, opts.src_addr, opts.src_port, FI_SOURCE,
hints, &fi);
if (ret) {
FT_PRINTERR("fi_getinfo", ret);
return ret;
}
ret = fi_fabric(fi->fabric_attr, &fab, NULL);
if (ret) {
FT_PRINTERR("fi_fabric", ret);
goto err0;
}
ret = fi_passive_ep(fab, fi, &pep, NULL);
if (ret) {
FT_PRINTERR("fi_passive_ep", ret);
goto err1;
}
ret = alloc_cm_res();
if (ret)
goto err2;
ret = fi_pep_bind(pep, &cmeq->fid, 0);
if (ret) {
FT_PRINTERR("fi_pep_bind", ret);
goto err3;
}
ret = fi_listen(pep);
if (ret) {
FT_PRINTERR("fi_listen", ret);
goto err3;
}
fi_freeinfo(fi);
return 0;
err3:
free_lres();
err2:
fi_close(&pep->fid);
err1:
fi_close(&fab->fid);
err0:
fi_freeinfo(fi);
return ret;
}
