static int rxm_ep_msg_res_close(struct rxm_ep *rxm_ep)
{
int ret, retv = 0;
ret = fi_close(&rxm_ep->msg_cq->fid);
if (ret) {
FI_WARN(&rxm_prov, FI_LOG_EP_CTRL, "Unable to close msg CQ\n");
retv = ret;
}
ret = fi_close(&rxm_ep->srx_ctx->fid);
if (ret) {
FI_WARN(&rxm_prov, FI_LOG_EP_CTRL, "Unable to close msg shared ctx\n");
retv = ret;
}
ret = fi_close(&rxm_ep->msg_pep->fid);
if (ret) {
FI_WARN(&rxm_prov, FI_LOG_EP_CTRL, "Unable to close msg passive EP\n");
retv = ret;
}
fi_freeinfo(rxm_ep->msg_info);
return retv;
}
static void free_ep_res(void)
{
int ret;
if (ep) {
ret = fi_close(&ep->fid);
if (ret != 0) {
FT_PRINTERR("fi_close", ret);
}
}
if (mr) {
ret = fi_close(&mr->fid);
if (ret != 0) {
FT_PRINTERR("fi_close", ret);
}
}
if (rcq) {
ret = fi_close(&rcq->fid);
if (ret != 0) {
FT_PRINTERR("fi_close", ret);
}
}
if (scq) {
ret = fi_close(&scq->fid);
if (ret != 0) {
FT_PRINTERR("fi_close", ret);
}
}
free(buf);
}
Test(domain, cache_flush_op)
{
int i, ret;
const int num_doms = 11;
struct fid_domain *doms[num_doms];
struct fi_gni_ops_domain *gni_domain_ops;
struct fid_mr *mr;
char *buf = calloc(1024, sizeof(char));
cr_assert(buf);
memset(doms, 0, num_doms*sizeof(struct fid_domain *));
for (i = 0; i < num_doms; i++) {
ret = fi_domain(fabric, fi, &doms[i], NULL);
cr_assert(ret == FI_SUCCESS, "fi_domain");
ret = fi_open_ops(&doms[i]->fid, FI_GNI_DOMAIN_OPS_1,
0, (void **) &gni_domain_ops, NULL);
cr_assert(ret == FI_SUCCESS, "fi_open_ops");
ret = fi_mr_reg(doms[i], buf, 1024, FI_READ, 0, 0, 0, &mr, NULL);
cr_assert(ret == FI_SUCCESS, "fi_reg_mr");
ret = fi_close(&mr->fid);
cr_assert(ret == FI_SUCCESS, "fi_close mr");
ret = gni_domain_ops->flush_cache(&doms[i]->fid);
cr_assert(ret == FI_SUCCESS, "flush cache");
ret = fi_close(&doms[i]->fid);
cr_assert(ret == FI_SUCCESS, "fi_close domain");
}
free(buf);
}
void mca_btl_ofi_context_finalize(mca_btl_ofi_context_t *context, bool scalable_ep) {
/* if it is a scalable ep, we have to close all contexts. */
if (scalable_ep) {
if (NULL != context->tx_ctx) {
fi_close(&context->tx_ctx->fid);
}
if (NULL != context->rx_ctx) {
fi_close(&context->rx_ctx->fid);
}
}
if( NULL != context->cq) {
fi_close(&context->cq->fid);
}
/* Can we destruct the object that hasn't been constructed? */
OBJ_DESTRUCT(&context->rdma_comp_list);
if (TWO_SIDED_ENABLED) {
OBJ_DESTRUCT(&context->frag_comp_list);
OBJ_DESTRUCT(&context->frag_list);
}
}
static void free_ep_res(void)
{
fi_close(&mr->fid);
fi_close(&rcq->fid);
fi_close(&scq->fid);
free(buf);
}
static void rdm_str_addr_sr_teardown(void)
{
int ret = 0, i = 0;
for (; i < NUMEPS; i++) {
ret = fi_close(&ep[i]->fid);
cr_assert(!ret, "failure in closing ep.");
ret = fi_close(&msg_cq[i]->fid);
cr_assert(!ret, "failure in send cq.");
ret = fi_close(&av[i]->fid);
cr_assert(!ret, "failure in closing av.");
ret = fi_close(&dom[i]->fid);
cr_assert(!ret, "failure in closing domain.");
fi_freeinfo(fi[i]);
free(ep_name[i]);
}
free(target_base);
free(source_base);
ret = fi_close(&fab->fid);
cr_assert(!ret, "failure in closing fabric.");
fi_freeinfo(hints);
}
static int run(void)
{
int ret = 0;
if (!dst_addr) {
ret = server_listen();
if (ret)
return ret;
}
ret = dst_addr ? client_connect() : server_connect();
if (ret) {
return ret;
}
run_test();
fi_shutdown(ep, 0);
fi_close(&ep->fid);
free_ep_res();
if (!dst_addr)
free_lres();
fi_close(&dom->fid);
fi_close(&fab->fid);
return ret;
}
static int init_fabric(void)
{
uint64_t flags = 0;
char *node, *service;
int ret;
ret = ft_read_addr_opts(&node, &service, hints, &flags, &opts);
if (ret)
return ret;
ret = fi_getinfo(FT_FIVERSION, node, service, flags, hints, &fi);
if (ret) {
FT_PRINTERR("fi_getinfo", ret);
return ret;
}
// we use provider MR attributes and direct address (no offsets)
// for RMA calls
if (!(fi->mode & FI_PROV_MR_ATTR))
fi->mode |= FI_PROV_MR_ATTR;
// get remote address
if (opts.dst_addr) {
addrlen = fi->dest_addrlen;
remote_addr = malloc(addrlen);
memcpy(remote_addr, fi->dest_addr, addrlen);
}
ret = fi_fabric(fi->fabric_attr, &fab, NULL);
if (ret) {
FT_PRINTERR("fi_fabric", ret);
goto err0;
}
ret = fi_domain(fab, fi, &dom, NULL);
if (ret) {
FT_PRINTERR("fi_domain", ret);
goto err1;
}
ret = alloc_ep_res(fi);
if (ret)
goto err3;
ret = bind_ep_res();
if (ret)
goto err4;
return 0;
err4:
free_ep_res();
err3:
fi_close(&dom->fid);
err1:
fi_close(&fab->fid);
err0:
return ret;
}
static inline void cntr_teardown_mr(void)
{
fi_close(&loc_mr->fid);
fi_close(&rem_mr->fid);
free(target);
free(source);
}
static int alloc_ep_res(struct fi_info *fi)
{
struct fi_cq_attr cq_attr;
struct fi_av_attr av_attr;
int ret;
buffer_size = !custom ? test_size[TEST_CNT - 1].size : transfer_size;
buffer_size += prefix_len;
buf = malloc(buffer_size);
if (!buf) {
perror("malloc");
return -1;
}
buf_ptr = (char *)buf + prefix_len;
memset(&cq_attr, 0, sizeof cq_attr);
cq_attr.format = FI_CQ_FORMAT_CONTEXT;
cq_attr.wait_obj = FI_WAIT_NONE;
cq_attr.size = max_credits << 1;
ret = fi_cq_open(dom, &cq_attr, &scq, NULL);
if (ret) {
printf("fi_cq_open send comp %s\n", fi_strerror(-ret));
goto err1;
}
ret = fi_cq_open(dom, &cq_attr, &rcq, NULL);
if (ret) {
printf("fi_cq_open recv comp %s\n", fi_strerror(-ret));
goto err2;
}
ret = fi_mr_reg(dom, buf, buffer_size, 0, 0, 0, 0, &mr, NULL);
if (ret) {
printf("fi_mr_reg %s\n", fi_strerror(-ret));
goto err3;
}
av_attr.type = FI_AV_MAP;
av_attr.name = NULL;
av_attr.flags = 0;
ret = fi_av_open(dom, &av_attr, &av, NULL);
if (ret) {
printf("fi_av_open %s\n", fi_strerror(-ret));
goto err4;
}
return 0;
err4:
fi_close(&mr->fid);
err3:
fi_close(&rcq->fid);
err2:
fi_close(&scq->fid);
err1:
free(buf);
return ret;
}
static int server_listen(void)
{
struct fi_info *fi;
int ret;
/* Get fabric info */
ret = fi_getinfo(FT_FIVERSION, NULL, opts.src_port, FI_SOURCE, hints, &fi);
if (ret) {
FT_PRINTERR("fi_getinfo", ret);
return ret;
}
/* Open the fabric */
ret = fi_fabric(fi->fabric_attr, &fab, NULL);
if (ret) {
FT_PRINTERR("fi_fabric", ret);
goto err0;
}
/* Open a passive endpoint */
ret = fi_passive_ep(fab, fi, &pep, NULL);
if (ret) {
FT_PRINTERR("fi_passive_ep", ret);
goto err1;
}
/* Allocate connection management resources */
ret = alloc_cm_res();
if (ret)
goto err2;
/* Bind EQ to passive endpoint */
ret = fi_pep_bind(pep, &cmeq->fid, 0);
if (ret) {
FT_PRINTERR("fi_pep_bind", ret);
goto err3;
}
/* Listen for incoming connections */
ret = fi_listen(pep);
if (ret) {
FT_PRINTERR("fi_listen", ret);
goto err3;
}
fi_freeinfo(fi);
return 0;
err3:
fi_close(&cmeq->fid);
err2:
fi_close(&pep->fid);
err1:
fi_close(&fab->fid);
err0:
fi_freeinfo(fi);
return ret;
}
static void free_ep_res(void)
{
fi_close(&ep->fid);
fi_close(&av->fid);
fi_close(&mr->fid);
fi_close(&rcntr->fid);
fi_close(&scntr->fid);
free(buf);
}
static int init_fabric(void)
{
int ret;
uint64_t flags = 0;
/* Get fabric info */
ret = fi_getinfo(CT_FIVERSION, NULL, NULL, flags, hints, &fi);
if (ret) {
ct_print_fi_error("fi_getinfo", ret);
return ret;
}
/* Open fabric */
ret = fi_fabric(fi->fabric_attr, &fab, NULL);
if (ret) {
ct_print_fi_error("fi_fabric", ret);
goto err1;
}
/* Open domain */
ret = fi_domain(fab, fi, &dom, NULL);
if (ret) {
ct_print_fi_error("fi_domain", ret);
goto err2;
}
/* Open endpoint */
ret = fi_endpoint(dom, fi, &ep, NULL);
if (ret) {
ct_print_fi_error("fi_endpoint", ret);
goto err3;
}
/* Allocate endpoint resources */
ret = alloc_ep_res();
if (ret)
goto err4;
/* Bind EQs and AVs with endpoint */
ret = bind_ep_res();
if (ret)
goto err5;
return 0;
err5:
free_ep_res();
err4:
fi_close(&ep->fid);
err3:
fi_close(&dom->fid);
err2:
fi_close(&fab->fid);
err1:
return ret;
}
static inline void cntr_teardown_cqs(void)
{
int ret;
ret = fi_close(&send_cq->fid);
cr_assert(!ret, "failure in closing send cq.");
ret = fi_close(&recv_cq->fid);
cr_assert(!ret, "failure in closing recv cq.");
}
int main(int argc, char **argv)
{
int op, ret;
opts = INIT_OPTS;
hints = fi_allocinfo();
if (!hints)
return EXIT_FAILURE;
while ((op = getopt(argc, argv, "h" ADDR_OPTS INFO_OPTS)) != -1) {
switch (op) {
default:
ft_parse_addr_opts(op, optarg, &opts);
ft_parseinfo(op, optarg, hints);
break;
case '?':
case 'h':
ft_usage(argv[0], "A simple MSG client-sever example that "
"demonstrates one possible usage of the underlying "
"cq wait objects.");
return EXIT_FAILURE;
}
}
if (optind < argc)
opts.dst_addr = argv[optind];
hints->ep_attr->type = FI_EP_MSG;
hints->caps = FI_MSG;
hints->mode = FI_LOCAL_MR;
hints->addr_format = FI_SOCKADDR;
/* Fabric and connection setup */
if (!opts.dst_addr) {
ret = server_listen();
if (ret)
return -ret;
}
ret = opts.dst_addr ? client_connect() : server_connect();
if (ret) {
return -ret;
}
/* Exchange data */
ret = send_recv();
fi_shutdown(ep, 0);
free_ep_res();
fi_close(&cmeq->fid);
fi_close(&dom->fid);
fi_close(&fab->fid);
return ret;
}
static int alloc_ep_res(struct fi_info *fi)
{
struct fi_cq_attr cq_attr;
int ret;
buffer_size = !custom ? test_size[TEST_CNT - 1].size : transfer_size;
if (buffer_size < MIN_BUF_SIZE) {
buffer_size = MIN_BUF_SIZE;
}
buf = malloc(buffer_size);
if (!buf) {
perror("malloc");
return -1;
}
memset(&cq_attr, 0, sizeof cq_attr);
cq_attr.format = FI_CQ_FORMAT_CONTEXT;
cq_attr.wait_obj = FI_WAIT_NONE;
cq_attr.size = max_credits << 1;
ret = fi_cq_open(dom, &cq_attr, &scq, NULL);
if (ret) {
printf("fi_eq_open send comp %s\n", fi_strerror(-ret));
goto err1;
}
ret = fi_cq_open(dom, &cq_attr, &rcq, NULL);
if (ret) {
printf("fi_eq_open recv comp %s\n", fi_strerror(-ret));
goto err2;
}
ret = fi_mr_reg(dom, buf, buffer_size, FI_REMOTE_WRITE, 0, 0, 0, &mr, NULL);
if (ret) {
printf("fi_mr_reg %s\n", fi_strerror(-ret));
goto err3;
}
if (!cmeq) {
ret = alloc_cm_res();
if (ret)
goto err4;
}
return 0;
err4:
fi_close(&mr->fid);
err3:
fi_close(&rcq->fid);
err2:
fi_close(&scq->fid);
err1:
free(buf);
return ret;
}
static void av_teardown(void)
{
int ret = 0;
ret = fi_close(&dom->fid);
cr_assert_eq(ret, FI_SUCCESS, "failure in closing domain.");
ret = fi_close(&fab->fid);
cr_assert_eq(ret, FI_SUCCESS, "failure in closing fabric.");
fi_freeinfo(fi);
fi_freeinfo(hints);
}
void teardown(void)
{
int ret = 0;
ret = fi_close(&dom->fid);
cr_assert(!ret, "failure in closing domain.");
ret = fi_close(&fab->fid);
cr_assert(!ret, "failure in closing fabric.");
fi_freeinfo(fi);
fi_freeinfo(hints);
}
static int alloc_ep_res(struct fi_info *fi)
{
struct fi_cq_attr cq_attr;
int ret;
buffer_size = test_size[TEST_CNT - 1].size;
buf = malloc(buffer_size);
if (!buf) {
perror("malloc");
return -1;
}
memset(&cq_attr, 0, sizeof cq_attr);
cq_attr.format = FI_CQ_FORMAT_DATA;
cq_attr.wait_obj = FI_WAIT_NONE;
cq_attr.size = rx_depth;
ret = fi_cq_open(dom, &cq_attr, &rcq, NULL);
if (ret) {
printf("fi_cq_open send comp %s\n", fi_strerror(-ret));
goto err1;
}
cq_attr.format = FI_CQ_FORMAT_CONTEXT;
ret = fi_cq_open(dom, &cq_attr, &scq, NULL);
if (ret) {
printf("fi_cq_open recv comp %s\n", fi_strerror(-ret));
goto err2;
}
ret = fi_mr_reg(dom, buf, buffer_size, 0, 0, 0, 0, &mr, NULL);
if (ret) {
printf("fi_mr_reg %s\n", fi_strerror(-ret));
goto err3;
}
if (!cmeq) {
ret = alloc_cm_res();
if (ret)
goto err4;
}
return 0;
err4:
fi_close(&mr->fid);
err3:
fi_close(&rcq->fid);
err2:
fi_close(&scq->fid);
err1:
free(buf);
return ret;
}
static int server_listen(void)
{
struct fi_info *fi;
int ret;
ret = fi_getinfo(FI_VERSION(1, 0), src_addr, port, FI_SOURCE, &hints, &fi);
if (ret) {
printf("fi_getinfo %s\n", strerror(-ret));
return ret;
}
cq_data_size = fi->domain_attr->cq_data_size;
ret = fi_fabric(fi->fabric_attr, &fab, NULL);
if (ret) {
printf("fi_fabric %s\n", fi_strerror(-ret));
goto err0;
}
ret = fi_passive_ep(fab, fi, &pep, NULL);
if (ret) {
printf("fi_passive_ep %s\n", fi_strerror(-ret));
goto err1;
}
ret = alloc_cm_res();
if (ret)
goto err2;
ret = fi_bind(&pep->fid, &cmeq->fid, 0);
if (ret) {
printf("fi_bind %s\n", fi_strerror(-ret));
goto err3;
}
ret = fi_listen(pep);
if (ret) {
printf("fi_listen %s\n", fi_strerror(-ret));
goto err3;
}
fi_freeinfo(fi);
return 0;
err3:
free_lres();
err2:
fi_close(&pep->fid);
err1:
fi_close(&fab->fid);
err0:
fi_freeinfo(fi);
return ret;
}
static int server_listen(void)
{
struct fi_info *fi;
int ret;
ret = fi_getinfo(FT_FIVERSION, opts.src_addr, opts.src_port, FI_SOURCE,
hints, &fi);
if (ret) {
FT_PRINTERR("fi_getinfo", ret);
return ret;
}
ret = fi_fabric(fi->fabric_attr, &fab, NULL);
if (ret) {
FT_PRINTERR("fi_fabric", ret);
goto err0;
}
ret = fi_passive_ep(fab, fi, &pep, NULL);
if (ret) {
FT_PRINTERR("fi_passive_ep", ret);
goto err1;
}
ret = alloc_cm_res();
if (ret)
goto err2;
ret = fi_pep_bind(pep, &cmeq->fid, 0);
if (ret) {
FT_PRINTERR("fi_pep_bind", ret);
goto err3;
}
ret = fi_listen(pep);
if (ret) {
FT_PRINTERR("fi_listen", ret);
goto err3;
}
fi_freeinfo(fi);
return 0;
err3:
free_lres();
err2:
fi_close(&pep->fid);
err1:
fi_close(&fab->fid);
err0:
fi_freeinfo(fi);
return ret;
}
void wait_teardown(void)
{
int ret = 0;
ret = fi_close(&wait_set->fid);
cr_assert(!ret, "failure in closing wait set.");
ret = fi_close(&fab->fid);
cr_assert(!ret, "failure in closing fabric.");
fi_freeinfo(fi);
fi_freeinfo(hints);
}
static void teardown(void)
{
int ret;
ret = fi_close(&dom->fid);
cr_assert(!ret, "fi_close domain");
ret = fi_close(&fab->fid);
cr_assert(!ret, "fi_close fabric");
fi_freeinfo(fi);
fi_freeinfo(hints);
}
static int server_connect(void)
{
int ret;
struct fi_cq_entry comp;
ret = common_setup();
if (ret != 0)
goto err;
do {
ret = fi_cq_read(rcq, &comp, 1);
if (ret < 0 && ret != -FI_EAGAIN) {
FT_PRINTERR("fi_cq_read", ret);
return ret;
}
} while (ret == -FI_EAGAIN);
ret = fi_av_insert(av, buf_ptr, 1, &remote_fi_addr, 0, NULL);
if (ret != 1) {
if (ret == 0) {
fprintf(stderr, "Unable to resolve remote address 0x%x 0x%x\n",
((uint32_t *)buf)[0], ((uint32_t *)buf)[1]);
ret = -FI_EINVAL;
} else {
FT_PRINTERR("fi_av_insert", ret);
}
goto err;
}
ret = fi_recv(ep, buf, buffer_size, fi_mr_desc(mr), 0, buf);
if (ret != 0) {
FT_PRINTERR("fi_recv", ret);
goto err;
}
ret = send_xfer(4);
if (ret != 0)
goto err;
return 0;
err:
free_ep_res();
if (dom)
fi_close(&dom->fid);
if (fab)
fi_close(&fab->fid);
return ret;
}
static void free_ep_res(void)
{
fi_close(&av->fid);
fi_close(&mr->fid);
FT_CLOSEV(rx_ep, ctx_cnt);
FT_CLOSEV(tx_ep, ctx_cnt);
FT_CLOSEV(rcq, ctx_cnt);
FT_CLOSEV(scq, ctx_cnt);
free(buf);
free(rcq);
free(scq);
free(tx_ep);
free(rx_ep);
}
static inline void cntr_teardown_cntrs(void)
{
int ret;
ret = fi_close(&write_cntr->fid);
cr_assert(!ret, "failure in closing write_cntr.");
ret = fi_close(&read_cntr->fid);
cr_assert(!ret, "failure in closing read_cntr.");
ret = fi_close(&rcv_cntr->fid);
cr_assert(!ret, "failure in closing read_cntr.");
}
static int server_listen(void)
{
struct fi_info *fi;
int ret;
ret = fi_getinfo(FI_VERSION(1, 0), src_addr, port, 0, &hints, &fi);
if (ret) {
printf("fi_getinfo %s\n", strerror(-ret));
return ret;
}
ret = fi_fabric(fi->fabric_attr, &fab, NULL);
if (ret) {
printf("fi_fabric %s\n", fi_strerror(-ret));
goto err0;
}
ret = fi_pendpoint(fab, fi, &pep, NULL);
if (ret) {
printf("fi_endpoint %s\n", fi_strerror(-ret));
goto err1;
}
ret = alloc_cm_res();
if (ret)
goto err2;
ret = bind_fid(&pep->fid, &cmeq->fid, 0);
if (ret)
goto err3;
ret = fi_listen(pep);
if (ret) {
printf("fi_listen %s\n", fi_strerror(-ret));
goto err3;
}
fi_freeinfo(fi);
return 0;
err3:
free_lres();
err2:
fi_close(&pep->fid);
err1:
fi_close(&fab->fid);
err0:
fi_freeinfo(fi);
return ret;
}
static int run(void)
{
int i, ret = 0;
ret = init_fabric();
if (ret)
return ret;
ret = init_av();
if (ret)
goto out;
ret = exchange_addr_key();
if (ret)
goto out;
if (run_all_sizes) {
for (i = 0; i < TEST_CNT; i++) {
if (test_size[i].option > size_option)
continue;
init_test(test_size[i].size, test_name,
sizeof(test_name), &transfer_size,
&iterations);
ret = run_test();
if (ret) {
fprintf(stderr, "Test failed at iteration %d, "
"msg size %d\n", i,
transfer_size);
goto out;
}
}
} else {
ret = run_test();
if (ret)
goto out;
}
sync_test();
out:
fi_shutdown(ep, 0);
fi_close(&ep->fid);
free_ep_res();
fi_close(&dom->fid);
fi_close(&fab->fid);
return ret;
}
static void rdm_sr_teardown_common(bool unreg)
{
int ret = 0, i = 0;
for (; i < NUMEPS; i++) {
fi_close(&recv_cntr[i]->fid);
fi_close(&send_cntr[i]->fid);
if (unreg) {
fi_close(&loc_mr[i]->fid);
fi_close(&rem_mr[i]->fid);
}
ret = fi_close(&ep[i]->fid);
cr_assert(!ret, "failure in closing ep.");
ret = fi_close(&msg_cq[i]->fid);
cr_assert(!ret, "failure in send cq.");
ret = fi_close(&av[i]->fid);
cr_assert(!ret, "failure in closing av.");
ret = fi_close(&dom[i]->fid);
cr_assert(!ret, "failure in closing domain.");
fi_freeinfo(fi[i]);
free(ep_name[i]);
}
free(uc_source);
free(uc_target);
free(iov_src_buf);
free(iov_dest_buf);
free(target);
free(source);
for (i = 0; i < IOV_CNT; i++) {
free(src_iov[i].iov_base);
free(dest_iov[i].iov_base);
}
free(src_iov);
free(dest_iov);
free(s_iov);
free(d_iov);
ret = fi_close(&fab->fid);
cr_assert(!ret, "failure in closing fabric.");
fi_freeinfo(hints);
}
static int run(void)
{
int i, ret = 0;
if (!opts.dst_addr) {
ret = server_listen();
if (ret)
return ret;
}
ret = opts.dst_addr ? client_connect() : server_connect();
if (ret)
return ret;
ret = exchange_addr_key();
if (ret)
return ret;
if (!(opts.user_options & FT_OPT_SIZE)) {
for (i = 0; i < TEST_CNT; i++) {
if (test_size[i].option > opts.size_option)
continue;
opts.transfer_size = test_size[i].size;
init_test(&opts, test_name, sizeof(test_name));
ret = run_test();
if (ret)
goto out;
}
} else {
init_test(&opts, test_name, sizeof(test_name));
ret = run_test();
if (ret)
goto out;
}
sync_test();
wait_for_data_completion(scq, max_credits - credits);
/* Finalize before closing ep */
ft_finalize(ep, scq, rcq, FI_ADDR_UNSPEC);
out:
fi_shutdown(ep, 0);
free_ep_res();
if (!opts.dst_addr)
free_lres();
fi_close(&dom->fid);
fi_close(&fab->fid);
return ret;
}
