static int rxm_ep_msg_res_close(struct rxm_ep *rxm_ep) { int ret, retv = 0; ret = fi_close(&rxm_ep->msg_cq->fid); if (ret) { FI_WARN(&rxm_prov, FI_LOG_EP_CTRL, "Unable to close msg CQ\n"); retv = ret; } ret = fi_close(&rxm_ep->srx_ctx->fid); if (ret) { FI_WARN(&rxm_prov, FI_LOG_EP_CTRL, "Unable to close msg shared ctx\n"); retv = ret; } ret = fi_close(&rxm_ep->msg_pep->fid); if (ret) { FI_WARN(&rxm_prov, FI_LOG_EP_CTRL, "Unable to close msg passive EP\n"); retv = ret; } fi_freeinfo(rxm_ep->msg_info); return retv; }
static void free_ep_res(void) { int ret; if (ep) { ret = fi_close(&ep->fid); if (ret != 0) { FT_PRINTERR("fi_close", ret); } } if (mr) { ret = fi_close(&mr->fid); if (ret != 0) { FT_PRINTERR("fi_close", ret); } } if (rcq) { ret = fi_close(&rcq->fid); if (ret != 0) { FT_PRINTERR("fi_close", ret); } } if (scq) { ret = fi_close(&scq->fid); if (ret != 0) { FT_PRINTERR("fi_close", ret); } } free(buf); }
Test(domain, cache_flush_op) { int i, ret; const int num_doms = 11; struct fid_domain *doms[num_doms]; struct fi_gni_ops_domain *gni_domain_ops; struct fid_mr *mr; char *buf = calloc(1024, sizeof(char)); cr_assert(buf); memset(doms, 0, num_doms*sizeof(struct fid_domain *)); for (i = 0; i < num_doms; i++) { ret = fi_domain(fabric, fi, &doms[i], NULL); cr_assert(ret == FI_SUCCESS, "fi_domain"); ret = fi_open_ops(&doms[i]->fid, FI_GNI_DOMAIN_OPS_1, 0, (void **) &gni_domain_ops, NULL); cr_assert(ret == FI_SUCCESS, "fi_open_ops"); ret = fi_mr_reg(doms[i], buf, 1024, FI_READ, 0, 0, 0, &mr, NULL); cr_assert(ret == FI_SUCCESS, "fi_reg_mr"); ret = fi_close(&mr->fid); cr_assert(ret == FI_SUCCESS, "fi_close mr"); ret = gni_domain_ops->flush_cache(&doms[i]->fid); cr_assert(ret == FI_SUCCESS, "flush cache"); ret = fi_close(&doms[i]->fid); cr_assert(ret == FI_SUCCESS, "fi_close domain"); } free(buf); }
void mca_btl_ofi_context_finalize(mca_btl_ofi_context_t *context, bool scalable_ep) { /* if it is a scalable ep, we have to close all contexts. */ if (scalable_ep) { if (NULL != context->tx_ctx) { fi_close(&context->tx_ctx->fid); } if (NULL != context->rx_ctx) { fi_close(&context->rx_ctx->fid); } } if( NULL != context->cq) { fi_close(&context->cq->fid); } /* Can we destruct the object that hasn't been constructed? */ OBJ_DESTRUCT(&context->rdma_comp_list); if (TWO_SIDED_ENABLED) { OBJ_DESTRUCT(&context->frag_comp_list); OBJ_DESTRUCT(&context->frag_list); } }
static void free_ep_res(void) { fi_close(&mr->fid); fi_close(&rcq->fid); fi_close(&scq->fid); free(buf); }
static void rdm_str_addr_sr_teardown(void) { int ret = 0, i = 0; for (; i < NUMEPS; i++) { ret = fi_close(&ep[i]->fid); cr_assert(!ret, "failure in closing ep."); ret = fi_close(&msg_cq[i]->fid); cr_assert(!ret, "failure in send cq."); ret = fi_close(&av[i]->fid); cr_assert(!ret, "failure in closing av."); ret = fi_close(&dom[i]->fid); cr_assert(!ret, "failure in closing domain."); fi_freeinfo(fi[i]); free(ep_name[i]); } free(target_base); free(source_base); ret = fi_close(&fab->fid); cr_assert(!ret, "failure in closing fabric."); fi_freeinfo(hints); }
static int run(void) { int ret = 0; if (!dst_addr) { ret = server_listen(); if (ret) return ret; } ret = dst_addr ? client_connect() : server_connect(); if (ret) { return ret; } run_test(); fi_shutdown(ep, 0); fi_close(&ep->fid); free_ep_res(); if (!dst_addr) free_lres(); fi_close(&dom->fid); fi_close(&fab->fid); return ret; }
static int init_fabric(void) { uint64_t flags = 0; char *node, *service; int ret; ret = ft_read_addr_opts(&node, &service, hints, &flags, &opts); if (ret) return ret; ret = fi_getinfo(FT_FIVERSION, node, service, flags, hints, &fi); if (ret) { FT_PRINTERR("fi_getinfo", ret); return ret; } // we use provider MR attributes and direct address (no offsets) // for RMA calls if (!(fi->mode & FI_PROV_MR_ATTR)) fi->mode |= FI_PROV_MR_ATTR; // get remote address if (opts.dst_addr) { addrlen = fi->dest_addrlen; remote_addr = malloc(addrlen); memcpy(remote_addr, fi->dest_addr, addrlen); } ret = fi_fabric(fi->fabric_attr, &fab, NULL); if (ret) { FT_PRINTERR("fi_fabric", ret); goto err0; } ret = fi_domain(fab, fi, &dom, NULL); if (ret) { FT_PRINTERR("fi_domain", ret); goto err1; } ret = alloc_ep_res(fi); if (ret) goto err3; ret = bind_ep_res(); if (ret) goto err4; return 0; err4: free_ep_res(); err3: fi_close(&dom->fid); err1: fi_close(&fab->fid); err0: return ret; }
static inline void cntr_teardown_mr(void) { fi_close(&loc_mr->fid); fi_close(&rem_mr->fid); free(target); free(source); }
static int alloc_ep_res(struct fi_info *fi) { struct fi_cq_attr cq_attr; struct fi_av_attr av_attr; int ret; buffer_size = !custom ? test_size[TEST_CNT - 1].size : transfer_size; buffer_size += prefix_len; buf = malloc(buffer_size); if (!buf) { perror("malloc"); return -1; } buf_ptr = (char *)buf + prefix_len; memset(&cq_attr, 0, sizeof cq_attr); cq_attr.format = FI_CQ_FORMAT_CONTEXT; cq_attr.wait_obj = FI_WAIT_NONE; cq_attr.size = max_credits << 1; ret = fi_cq_open(dom, &cq_attr, &scq, NULL); if (ret) { printf("fi_cq_open send comp %s\n", fi_strerror(-ret)); goto err1; } ret = fi_cq_open(dom, &cq_attr, &rcq, NULL); if (ret) { printf("fi_cq_open recv comp %s\n", fi_strerror(-ret)); goto err2; } ret = fi_mr_reg(dom, buf, buffer_size, 0, 0, 0, 0, &mr, NULL); if (ret) { printf("fi_mr_reg %s\n", fi_strerror(-ret)); goto err3; } av_attr.type = FI_AV_MAP; av_attr.name = NULL; av_attr.flags = 0; ret = fi_av_open(dom, &av_attr, &av, NULL); if (ret) { printf("fi_av_open %s\n", fi_strerror(-ret)); goto err4; } return 0; err4: fi_close(&mr->fid); err3: fi_close(&rcq->fid); err2: fi_close(&scq->fid); err1: free(buf); return ret; }
static int server_listen(void) { struct fi_info *fi; int ret; /* Get fabric info */ ret = fi_getinfo(FT_FIVERSION, NULL, opts.src_port, FI_SOURCE, hints, &fi); if (ret) { FT_PRINTERR("fi_getinfo", ret); return ret; } /* Open the fabric */ ret = fi_fabric(fi->fabric_attr, &fab, NULL); if (ret) { FT_PRINTERR("fi_fabric", ret); goto err0; } /* Open a passive endpoint */ ret = fi_passive_ep(fab, fi, &pep, NULL); if (ret) { FT_PRINTERR("fi_passive_ep", ret); goto err1; } /* Allocate connection management resources */ ret = alloc_cm_res(); if (ret) goto err2; /* Bind EQ to passive endpoint */ ret = fi_pep_bind(pep, &cmeq->fid, 0); if (ret) { FT_PRINTERR("fi_pep_bind", ret); goto err3; } /* Listen for incoming connections */ ret = fi_listen(pep); if (ret) { FT_PRINTERR("fi_listen", ret); goto err3; } fi_freeinfo(fi); return 0; err3: fi_close(&cmeq->fid); err2: fi_close(&pep->fid); err1: fi_close(&fab->fid); err0: fi_freeinfo(fi); return ret; }
static void free_ep_res(void) { fi_close(&ep->fid); fi_close(&av->fid); fi_close(&mr->fid); fi_close(&rcntr->fid); fi_close(&scntr->fid); free(buf); }
static int init_fabric(void) { int ret; uint64_t flags = 0; /* Get fabric info */ ret = fi_getinfo(CT_FIVERSION, NULL, NULL, flags, hints, &fi); if (ret) { ct_print_fi_error("fi_getinfo", ret); return ret; } /* Open fabric */ ret = fi_fabric(fi->fabric_attr, &fab, NULL); if (ret) { ct_print_fi_error("fi_fabric", ret); goto err1; } /* Open domain */ ret = fi_domain(fab, fi, &dom, NULL); if (ret) { ct_print_fi_error("fi_domain", ret); goto err2; } /* Open endpoint */ ret = fi_endpoint(dom, fi, &ep, NULL); if (ret) { ct_print_fi_error("fi_endpoint", ret); goto err3; } /* Allocate endpoint resources */ ret = alloc_ep_res(); if (ret) goto err4; /* Bind EQs and AVs with endpoint */ ret = bind_ep_res(); if (ret) goto err5; return 0; err5: free_ep_res(); err4: fi_close(&ep->fid); err3: fi_close(&dom->fid); err2: fi_close(&fab->fid); err1: return ret; }
static inline void cntr_teardown_cqs(void) { int ret; ret = fi_close(&send_cq->fid); cr_assert(!ret, "failure in closing send cq."); ret = fi_close(&recv_cq->fid); cr_assert(!ret, "failure in closing recv cq."); }
int main(int argc, char **argv) { int op, ret; opts = INIT_OPTS; hints = fi_allocinfo(); if (!hints) return EXIT_FAILURE; while ((op = getopt(argc, argv, "h" ADDR_OPTS INFO_OPTS)) != -1) { switch (op) { default: ft_parse_addr_opts(op, optarg, &opts); ft_parseinfo(op, optarg, hints); break; case '?': case 'h': ft_usage(argv[0], "A simple MSG client-sever example that " "demonstrates one possible usage of the underlying " "cq wait objects."); return EXIT_FAILURE; } } if (optind < argc) opts.dst_addr = argv[optind]; hints->ep_attr->type = FI_EP_MSG; hints->caps = FI_MSG; hints->mode = FI_LOCAL_MR; hints->addr_format = FI_SOCKADDR; /* Fabric and connection setup */ if (!opts.dst_addr) { ret = server_listen(); if (ret) return -ret; } ret = opts.dst_addr ? client_connect() : server_connect(); if (ret) { return -ret; } /* Exchange data */ ret = send_recv(); fi_shutdown(ep, 0); free_ep_res(); fi_close(&cmeq->fid); fi_close(&dom->fid); fi_close(&fab->fid); return ret; }
static int alloc_ep_res(struct fi_info *fi) { struct fi_cq_attr cq_attr; int ret; buffer_size = !custom ? test_size[TEST_CNT - 1].size : transfer_size; if (buffer_size < MIN_BUF_SIZE) { buffer_size = MIN_BUF_SIZE; } buf = malloc(buffer_size); if (!buf) { perror("malloc"); return -1; } memset(&cq_attr, 0, sizeof cq_attr); cq_attr.format = FI_CQ_FORMAT_CONTEXT; cq_attr.wait_obj = FI_WAIT_NONE; cq_attr.size = max_credits << 1; ret = fi_cq_open(dom, &cq_attr, &scq, NULL); if (ret) { printf("fi_eq_open send comp %s\n", fi_strerror(-ret)); goto err1; } ret = fi_cq_open(dom, &cq_attr, &rcq, NULL); if (ret) { printf("fi_eq_open recv comp %s\n", fi_strerror(-ret)); goto err2; } ret = fi_mr_reg(dom, buf, buffer_size, FI_REMOTE_WRITE, 0, 0, 0, &mr, NULL); if (ret) { printf("fi_mr_reg %s\n", fi_strerror(-ret)); goto err3; } if (!cmeq) { ret = alloc_cm_res(); if (ret) goto err4; } return 0; err4: fi_close(&mr->fid); err3: fi_close(&rcq->fid); err2: fi_close(&scq->fid); err1: free(buf); return ret; }
static void av_teardown(void) { int ret = 0; ret = fi_close(&dom->fid); cr_assert_eq(ret, FI_SUCCESS, "failure in closing domain."); ret = fi_close(&fab->fid); cr_assert_eq(ret, FI_SUCCESS, "failure in closing fabric."); fi_freeinfo(fi); fi_freeinfo(hints); }
void teardown(void) { int ret = 0; ret = fi_close(&dom->fid); cr_assert(!ret, "failure in closing domain."); ret = fi_close(&fab->fid); cr_assert(!ret, "failure in closing fabric."); fi_freeinfo(fi); fi_freeinfo(hints); }
static int alloc_ep_res(struct fi_info *fi) { struct fi_cq_attr cq_attr; int ret; buffer_size = test_size[TEST_CNT - 1].size; buf = malloc(buffer_size); if (!buf) { perror("malloc"); return -1; } memset(&cq_attr, 0, sizeof cq_attr); cq_attr.format = FI_CQ_FORMAT_DATA; cq_attr.wait_obj = FI_WAIT_NONE; cq_attr.size = rx_depth; ret = fi_cq_open(dom, &cq_attr, &rcq, NULL); if (ret) { printf("fi_cq_open send comp %s\n", fi_strerror(-ret)); goto err1; } cq_attr.format = FI_CQ_FORMAT_CONTEXT; ret = fi_cq_open(dom, &cq_attr, &scq, NULL); if (ret) { printf("fi_cq_open recv comp %s\n", fi_strerror(-ret)); goto err2; } ret = fi_mr_reg(dom, buf, buffer_size, 0, 0, 0, 0, &mr, NULL); if (ret) { printf("fi_mr_reg %s\n", fi_strerror(-ret)); goto err3; } if (!cmeq) { ret = alloc_cm_res(); if (ret) goto err4; } return 0; err4: fi_close(&mr->fid); err3: fi_close(&rcq->fid); err2: fi_close(&scq->fid); err1: free(buf); return ret; }
static int server_listen(void) { struct fi_info *fi; int ret; ret = fi_getinfo(FI_VERSION(1, 0), src_addr, port, FI_SOURCE, &hints, &fi); if (ret) { printf("fi_getinfo %s\n", strerror(-ret)); return ret; } cq_data_size = fi->domain_attr->cq_data_size; ret = fi_fabric(fi->fabric_attr, &fab, NULL); if (ret) { printf("fi_fabric %s\n", fi_strerror(-ret)); goto err0; } ret = fi_passive_ep(fab, fi, &pep, NULL); if (ret) { printf("fi_passive_ep %s\n", fi_strerror(-ret)); goto err1; } ret = alloc_cm_res(); if (ret) goto err2; ret = fi_bind(&pep->fid, &cmeq->fid, 0); if (ret) { printf("fi_bind %s\n", fi_strerror(-ret)); goto err3; } ret = fi_listen(pep); if (ret) { printf("fi_listen %s\n", fi_strerror(-ret)); goto err3; } fi_freeinfo(fi); return 0; err3: free_lres(); err2: fi_close(&pep->fid); err1: fi_close(&fab->fid); err0: fi_freeinfo(fi); return ret; }
static int server_listen(void) { struct fi_info *fi; int ret; ret = fi_getinfo(FT_FIVERSION, opts.src_addr, opts.src_port, FI_SOURCE, hints, &fi); if (ret) { FT_PRINTERR("fi_getinfo", ret); return ret; } ret = fi_fabric(fi->fabric_attr, &fab, NULL); if (ret) { FT_PRINTERR("fi_fabric", ret); goto err0; } ret = fi_passive_ep(fab, fi, &pep, NULL); if (ret) { FT_PRINTERR("fi_passive_ep", ret); goto err1; } ret = alloc_cm_res(); if (ret) goto err2; ret = fi_pep_bind(pep, &cmeq->fid, 0); if (ret) { FT_PRINTERR("fi_pep_bind", ret); goto err3; } ret = fi_listen(pep); if (ret) { FT_PRINTERR("fi_listen", ret); goto err3; } fi_freeinfo(fi); return 0; err3: free_lres(); err2: fi_close(&pep->fid); err1: fi_close(&fab->fid); err0: fi_freeinfo(fi); return ret; }
void wait_teardown(void) { int ret = 0; ret = fi_close(&wait_set->fid); cr_assert(!ret, "failure in closing wait set."); ret = fi_close(&fab->fid); cr_assert(!ret, "failure in closing fabric."); fi_freeinfo(fi); fi_freeinfo(hints); }
static void teardown(void) { int ret; ret = fi_close(&dom->fid); cr_assert(!ret, "fi_close domain"); ret = fi_close(&fab->fid); cr_assert(!ret, "fi_close fabric"); fi_freeinfo(fi); fi_freeinfo(hints); }
static int server_connect(void) { int ret; struct fi_cq_entry comp; ret = common_setup(); if (ret != 0) goto err; do { ret = fi_cq_read(rcq, &comp, 1); if (ret < 0 && ret != -FI_EAGAIN) { FT_PRINTERR("fi_cq_read", ret); return ret; } } while (ret == -FI_EAGAIN); ret = fi_av_insert(av, buf_ptr, 1, &remote_fi_addr, 0, NULL); if (ret != 1) { if (ret == 0) { fprintf(stderr, "Unable to resolve remote address 0x%x 0x%x\n", ((uint32_t *)buf)[0], ((uint32_t *)buf)[1]); ret = -FI_EINVAL; } else { FT_PRINTERR("fi_av_insert", ret); } goto err; } ret = fi_recv(ep, buf, buffer_size, fi_mr_desc(mr), 0, buf); if (ret != 0) { FT_PRINTERR("fi_recv", ret); goto err; } ret = send_xfer(4); if (ret != 0) goto err; return 0; err: free_ep_res(); if (dom) fi_close(&dom->fid); if (fab) fi_close(&fab->fid); return ret; }
static void free_ep_res(void) { fi_close(&av->fid); fi_close(&mr->fid); FT_CLOSEV(rx_ep, ctx_cnt); FT_CLOSEV(tx_ep, ctx_cnt); FT_CLOSEV(rcq, ctx_cnt); FT_CLOSEV(scq, ctx_cnt); free(buf); free(rcq); free(scq); free(tx_ep); free(rx_ep); }
static inline void cntr_teardown_cntrs(void) { int ret; ret = fi_close(&write_cntr->fid); cr_assert(!ret, "failure in closing write_cntr."); ret = fi_close(&read_cntr->fid); cr_assert(!ret, "failure in closing read_cntr."); ret = fi_close(&rcv_cntr->fid); cr_assert(!ret, "failure in closing read_cntr."); }
static int server_listen(void) { struct fi_info *fi; int ret; ret = fi_getinfo(FI_VERSION(1, 0), src_addr, port, 0, &hints, &fi); if (ret) { printf("fi_getinfo %s\n", strerror(-ret)); return ret; } ret = fi_fabric(fi->fabric_attr, &fab, NULL); if (ret) { printf("fi_fabric %s\n", fi_strerror(-ret)); goto err0; } ret = fi_pendpoint(fab, fi, &pep, NULL); if (ret) { printf("fi_endpoint %s\n", fi_strerror(-ret)); goto err1; } ret = alloc_cm_res(); if (ret) goto err2; ret = bind_fid(&pep->fid, &cmeq->fid, 0); if (ret) goto err3; ret = fi_listen(pep); if (ret) { printf("fi_listen %s\n", fi_strerror(-ret)); goto err3; } fi_freeinfo(fi); return 0; err3: free_lres(); err2: fi_close(&pep->fid); err1: fi_close(&fab->fid); err0: fi_freeinfo(fi); return ret; }
static int run(void) { int i, ret = 0; ret = init_fabric(); if (ret) return ret; ret = init_av(); if (ret) goto out; ret = exchange_addr_key(); if (ret) goto out; if (run_all_sizes) { for (i = 0; i < TEST_CNT; i++) { if (test_size[i].option > size_option) continue; init_test(test_size[i].size, test_name, sizeof(test_name), &transfer_size, &iterations); ret = run_test(); if (ret) { fprintf(stderr, "Test failed at iteration %d, " "msg size %d\n", i, transfer_size); goto out; } } } else { ret = run_test(); if (ret) goto out; } sync_test(); out: fi_shutdown(ep, 0); fi_close(&ep->fid); free_ep_res(); fi_close(&dom->fid); fi_close(&fab->fid); return ret; }
static void rdm_sr_teardown_common(bool unreg) { int ret = 0, i = 0; for (; i < NUMEPS; i++) { fi_close(&recv_cntr[i]->fid); fi_close(&send_cntr[i]->fid); if (unreg) { fi_close(&loc_mr[i]->fid); fi_close(&rem_mr[i]->fid); } ret = fi_close(&ep[i]->fid); cr_assert(!ret, "failure in closing ep."); ret = fi_close(&msg_cq[i]->fid); cr_assert(!ret, "failure in send cq."); ret = fi_close(&av[i]->fid); cr_assert(!ret, "failure in closing av."); ret = fi_close(&dom[i]->fid); cr_assert(!ret, "failure in closing domain."); fi_freeinfo(fi[i]); free(ep_name[i]); } free(uc_source); free(uc_target); free(iov_src_buf); free(iov_dest_buf); free(target); free(source); for (i = 0; i < IOV_CNT; i++) { free(src_iov[i].iov_base); free(dest_iov[i].iov_base); } free(src_iov); free(dest_iov); free(s_iov); free(d_iov); ret = fi_close(&fab->fid); cr_assert(!ret, "failure in closing fabric."); fi_freeinfo(hints); }
static int run(void) { int i, ret = 0; if (!opts.dst_addr) { ret = server_listen(); if (ret) return ret; } ret = opts.dst_addr ? client_connect() : server_connect(); if (ret) return ret; ret = exchange_addr_key(); if (ret) return ret; if (!(opts.user_options & FT_OPT_SIZE)) { for (i = 0; i < TEST_CNT; i++) { if (test_size[i].option > opts.size_option) continue; opts.transfer_size = test_size[i].size; init_test(&opts, test_name, sizeof(test_name)); ret = run_test(); if (ret) goto out; } } else { init_test(&opts, test_name, sizeof(test_name)); ret = run_test(); if (ret) goto out; } sync_test(); wait_for_data_completion(scq, max_credits - credits); /* Finalize before closing ep */ ft_finalize(ep, scq, rcq, FI_ADDR_UNSPEC); out: fi_shutdown(ep, 0); free_ep_res(); if (!opts.dst_addr) free_lres(); fi_close(&dom->fid); fi_close(&fab->fid); return ret; }