/* * rpmem_fip_close -- close connection to remote peer */ int rpmem_fip_close(struct rpmem_fip *fip) { int ret; int lret = 0; ret = fi_shutdown(fip->ep, 0); if (ret) { RPMEM_FI_ERR(ret, "disconnecting endpoint"); lret = ret; } struct fi_eq_cm_entry entry; ret = rpmem_fip_read_eq(fip->eq, &entry, FI_SHUTDOWN, &fip->ep->fid, -1); if (ret) lret = ret; ret = rpmem_fip_fini_ep(fip); if (ret) lret = ret; ret = rpmem_fip_fini_cq(fip); if (ret) lret = ret; return lret; }
static int client_expect_accept(size_t paramlen) { size_t expected; uint32_t event; int ret; expected = paramlen + sizeof(*entry); ret = client_connect(paramlen); if (ret) { FT_PRINTERR("fi_connect", ret); return ret; } ret = fi_eq_sread(eq, &event, entry, expected, -1, 0); if (ret != expected) { FT_PROCESS_EQ_ERR(ret, eq, "fi_eq_sread", "connect"); return ret; } if (event != FI_CONNECTED || entry->fid != &ep->fid) { FT_ERR("Unexpected CM event %d fid %p (ep %p)", event, entry->fid, ep); return -FI_EOTHER; } /* Check data on FI_CONNECTED event. */ ret = ft_check_buf(entry->data, paramlen); if (ret) return ret; fi_shutdown(ep, 0); return read_shutdown_event(); }
static int run(void) { int ret = 0; if (!dst_addr) { ret = server_listen(); if (ret) return ret; } ret = dst_addr ? client_connect() : server_connect(); if (ret) { return ret; } run_test(); fi_shutdown(ep, 0); fi_close(&ep->fid); free_ep_res(); if (!dst_addr) free_lres(); fi_close(&dom->fid); fi_close(&fab->fid); return ret; }
static int cma_handler(uint32_t event, struct fi_eq_cm_entry *entry) { struct cma_node *node; int ret = 0; switch (event) { case FI_CONNREQ: ret = connreq_handler(entry->info); fi_freeinfo(entry->info); break; case FI_CONNECTED: node = entry->fid->context; node->connected = 1; connects_left--; disconnects_left++; break; case FI_SHUTDOWN: node = entry->fid->context; fi_shutdown(node->ep, 0); disconnects_left--; break; default: printf("unexpected event %d\n", event); break; } return ret; }
void Connection::disconnect() { L_(debug) << "[" << index_ << "] " << "disconnect"; int err = fi_shutdown(ep_, 0); if (err) { L_(fatal) << "fi_shutdown failed: " << err << "=" << fi_strerror(-err); throw LibfabricException("fi_shutdown failed"); } }
int main(int argc, char **argv) { int op, ret; opts = INIT_OPTS; hints = fi_allocinfo(); if (!hints) return EXIT_FAILURE; while ((op = getopt(argc, argv, "h" ADDR_OPTS INFO_OPTS)) != -1) { switch (op) { default: ft_parse_addr_opts(op, optarg, &opts); ft_parseinfo(op, optarg, hints); break; case '?': case 'h': ft_usage(argv[0], "A simple MSG client-sever example that " "demonstrates one possible usage of the underlying " "cq wait objects."); return EXIT_FAILURE; } } if (optind < argc) opts.dst_addr = argv[optind]; hints->ep_attr->type = FI_EP_MSG; hints->caps = FI_MSG; hints->mode = FI_LOCAL_MR; hints->addr_format = FI_SOCKADDR; /* Fabric and connection setup */ if (!opts.dst_addr) { ret = server_listen(); if (ret) return -ret; } ret = opts.dst_addr ? client_connect() : server_connect(); if (ret) { return -ret; } /* Exchange data */ ret = send_recv(); fi_shutdown(ep, 0); free_ep_res(); fi_close(&cmeq->fid); fi_close(&dom->fid); fi_close(&fab->fid); return ret; }
static int run(void) { int i, ret = 0; if (!opts.dst_addr) { ret = server_listen(); if (ret) return ret; } ret = opts.dst_addr ? client_connect() : server_connect(); if (ret) return ret; ret = exchange_addr_key(); if (ret) return ret; if (!(opts.user_options & FT_OPT_SIZE)) { for (i = 0; i < TEST_CNT; i++) { if (test_size[i].option > opts.size_option) continue; opts.transfer_size = test_size[i].size; init_test(&opts, test_name, sizeof(test_name)); ret = run_test(); if (ret) goto out; } } else { init_test(&opts, test_name, sizeof(test_name)); ret = run_test(); if (ret) goto out; } sync_test(); wait_for_data_completion(scq, max_credits - credits); /* Finalize before closing ep */ ft_finalize(ep, scq, rcq, FI_ADDR_UNSPEC); out: fi_shutdown(ep, 0); free_ep_res(); if (!opts.dst_addr) free_lres(); fi_close(&dom->fid); fi_close(&fab->fid); return ret; }
static int run(void) { int i, ret = 0; ret = init_fabric(); if (ret) return ret; ret = init_av(); if (ret) goto out; ret = exchange_addr_key(); if (ret) goto out; if (run_all_sizes) { for (i = 0; i < TEST_CNT; i++) { if (test_size[i].option > size_option) continue; init_test(test_size[i].size, test_name, sizeof(test_name), &transfer_size, &iterations); ret = run_test(); if (ret) { fprintf(stderr, "Test failed at iteration %d, " "msg size %d\n", i, transfer_size); goto out; } } } else { ret = run_test(); if (ret) goto out; } sync_test(); out: fi_shutdown(ep, 0); fi_close(&ep->fid); free_ep_res(); fi_close(&dom->fid); fi_close(&fab->fid); return ret; }
void client_disconnect(simple_context_t *ctx) { int ret; print_trace("in\n"); if (connected) { ret = fi_shutdown(ctx->ep, 0); if (ret) print_err("ERR: fi_shutdown() '%s'\n", fi_strerror(ret)); connected = 0; } if (ctx->mr) { fi_close((struct fid *) ctx->mr); ctx->mr = NULL; } if (!IS_ERR(ctx->buf)) { kfree(ctx->buf); ctx->buf = NULL; } if (ctx->rcq) { fi_close((struct fid *) ctx->rcq); ctx->rcq = NULL; } if (ctx->scq) { fi_close((struct fid *) ctx->scq); ctx->scq = NULL; } if (ctx->eq) { fi_close((struct fid *) ctx->eq); ctx->eq = NULL; } if (ctx->ep) { fi_close((struct fid *) ctx->ep); ctx->ep = NULL; } if (ctx->domain) { fi_close((struct fid *) ctx->domain); ctx->domain = NULL; } if (ctx->fabric) { fi_close((struct fid *) ctx->fabric); ctx->fabric = NULL; } }
static int run(void) { int i, ret = 0; if (!dst_addr) { ret = server_listen(); if (ret) return ret; } print_test_hdr(); ret = dst_addr ? client_connect() : server_connect(); if (ret) { return ret; } if (!custom) { for (i = 0; i < TEST_CNT; i++) { if (test_size[i].option > size_option) continue; init_test(test_size[i].size, test_name, &transfer_size, &iterations); run_test(); } } else { ret = run_test(); } ret = wait_for_completion(scq, max_credits - credits); if (ret) { return ret; } credits = max_credits; fi_shutdown(ep, 0); fi_close(&ep->fid); free_ep_res(); if (!dst_addr) free_lres(); fi_close(&dom->fid); fi_close(&fab->fid); return ret; }
static int run(void) { char *node, *service; uint64_t flags; int i, ret; ret = ft_read_addr_opts(&node, &service, hints, &flags, &opts); if (ret) return ret; if (!opts.dst_addr) { ret = ft_start_server(); if (ret) return ret; } ret = opts.dst_addr ? client_connect() : server_connect(); if (ret) { return ret; } if (!(opts.options & FT_OPT_SIZE)) { for (i = 0; i < TEST_CNT; i++) { if (!ft_use_size(i, opts.sizes_enabled)) continue; opts.transfer_size = test_size[i].size; init_test(&opts, test_name, sizeof(test_name)); ret = pingpong(); if (ret) goto out; } } else { init_test(&opts, test_name, sizeof(test_name)); ret = pingpong(); if (ret) goto out; } ret = ft_finalize(); out: fi_shutdown(ep, 0); return ret; }
static int run(void) { int ret = 0; if (!opts.dst_addr) { ret = ft_start_server(); if (ret) return ret; } ret = opts.dst_addr ? client_connect() : server_connect(); if (ret) { return ret; } ret = run_test(); fi_shutdown(ep, 0); return ret; }
static int run(void) { int ret; if (!opts.dst_addr) { ret = ft_start_server(); if (ret) return ret; } ret = opts.dst_addr ? ft_client_connect() : ft_server_connect(); if (ret) { return ret; } ret = ft_send_recv_greeting(ep); fi_shutdown(ep, 0); return ret; }
static int run(void) { int i, ret; if (!opts.dst_addr) { ret = ft_start_server(); if (ret) return ret; } ret = opts.dst_addr ? ft_client_connect() : ft_server_connect(); if (ret) return ret; ret = ft_exchange_keys(&remote); if (ret) return ret; if (!(opts.options & FT_OPT_SIZE)) { for (i = 0; i < TEST_CNT; i++) { if (!ft_use_size(i, opts.sizes_enabled)) continue; opts.transfer_size = test_size[i].size; init_test(&opts, test_name, sizeof(test_name)); ret = run_test(); if (ret) goto out; } } else { init_test(&opts, test_name, sizeof(test_name)); ret = run_test(); if (ret) goto out; } ft_sync(); ft_finalize(); out: fi_shutdown(ep, 0); return ret; }
/* * rpmem_fip_close -- close connection to remote peer */ int rpmem_fip_close(struct rpmem_fip *fip) { int ret; int lret = 0; ret = fi_shutdown(fip->ep, 0); if (ret) { RPMEM_FI_ERR(ret, "disconnecting endpoint"); lret = ret; } ret = rpmem_fip_fini_ep(fip); if (ret) lret = ret; ret = rpmem_fip_fini_cq(fip); if (ret) lret = ret; return lret; }
static int run(void) { char *node, *service; uint64_t flags; int ret; ret = ft_read_addr_opts(&node, &service, hints, &flags, &opts); if (ret) return ret; if (!opts.src_port) opts.src_port = "9229"; if (!opts.src_addr) { fprintf(stderr, "Source address (-s) is required for this test\n"); return -EXIT_FAILURE; } ret = setup_handle(); if (ret) return ret; if (!opts.dst_addr) { ret = server_listen(); if (ret) return ret; } ret = opts.dst_addr ? client_connect() : server_connect(); if (ret) { return ret; } ret = send_recv(); fi_shutdown(ep, 0); return ret; }
static int run(void) { int i, ret = 0; if (!client) { ret = server_listen(); if (ret) return ret; } printf("%-8s%-8s%-8s%8s %10s%13s\n", "bytes", "iters", "total", "time", "MB/sec", "usec/xfer"); ret = client ? client_connect() : server_connect(); if (ret) return ret; if (!custom) { for (i = 0; i < TEST_CNT; i++) { if (test_size[i].option > size_option) continue; init_test(test_size[i].size); ret = run_test(); } } else { ret = run_test(); } fi_shutdown(ep, 0); fi_close(&ep->fid); free_ep_res(); if (!client) free_lres(); fi_close(&dom->fid); fi_close(&fab->fid); return ret; }
static int run(void) { int ret; if (!opts.dst_addr) { ret = ft_start_server(); if (ret) return ret; } ret = opts.dst_addr ? ft_client_connect() : ft_server_connect(); if (ret) { return ret; } ret = alloc_epoll_res(); if (ret) return ret; ret = send_recv(); fi_shutdown(ep, 0); return ret; }
static int run(void) { int i, ret = 0; ret = dst_addr ? client_connect() : server_connect(); if (ret) return ret; printf("%-10s%-8s%-8s%-8s%-8s%8s %10s%13s\n", "name", "bytes", "xfers", "iters", "total", "time", "Gb/sec", "usec/xfer"); if (!custom) { for (i = 0; i < TEST_CNT; i++) { if (test_size[i].option > size_option || (max_msg_size && test_size[i].size > max_msg_size)) { continue; } init_test(test_size[i].size); run_test(); } } else { ret = run_test(); } while (credits < max_credits) poll_all_sends(); fi_shutdown(ep, 0); fi_close(&ep->fid); free_ep_res(); fi_close(&dom->fid); fi_close(&fab->fid); return ret; }
static int hook_shutdown(struct fid_ep *ep, uint64_t flags) { struct hook_ep *myep = container_of(ep, struct hook_ep, ep); return fi_shutdown(myep->hep, flags); }
int main(int argc, char *argv[]) { uint64_t flags = 0; char *service = NULL; char *node = NULL; struct pingpong_context *ctx; struct timeval start, end; unsigned long size = 4096; // No provider support yet //enum ibv_mtu mtu = IBV_MTU_1024; //size_t mtu = 1024; int rx_depth_default = 500; int rx_depth = 0; int iters = 1000; int use_event = 0; int rcnt, scnt; int ret, rc = 0; char * ptr; srand48(getpid() * time(NULL)); opts = INIT_OPTS; hints = fi_allocinfo(); if (!hints) return 1; while (1) { int c; c = getopt(argc, argv, "S:m:r:n:eh" ADDR_OPTS INFO_OPTS); if (c == -1) break; switch (c) { case 'S': errno = 0; size = strtol(optarg, &ptr, 10); if (ptr == optarg || *ptr != '\0' || ((size == LONG_MIN || size == LONG_MAX) && errno == ERANGE)) { fprintf(stderr, "Cannot convert from string to long\n"); rc = 1; goto err1; } break; // No provider support yet /*case 'm': mtu = strtol(optarg, NULL, 0); mtu = pp_mtu_to_enum(strtol(optarg, NULL, 0)); if (mtu < 0) { usage(argv[0]); return 1; } break; */ case 'r': rx_depth = strtol(optarg, NULL, 0); break; case 'n': iters = strtol(optarg, NULL, 0); break; case 'e': ++use_event; break; default: ft_parse_addr_opts(c, optarg, &opts); ft_parseinfo(c, optarg, hints); break; case '?': case 'h': usage(argv[0]); return 1; } } if (optind == argc - 1) opts.dst_addr = argv[optind]; else if (optind < argc) { usage(argv[0]); return 1; } page_size = sysconf(_SC_PAGESIZE); hints->ep_attr->type = FI_EP_MSG; hints->caps = FI_MSG; hints->mode = FI_LOCAL_MR; rc = ft_read_addr_opts(&node, &service, hints, &flags, &opts); if (rc) return -rc; rc = fi_getinfo(FT_FIVERSION, node, service, flags, hints, &fi); if (rc) { FT_PRINTERR("fi_getinfo", rc); return -rc; } fi_freeinfo(hints); if (rx_depth) { if (rx_depth > fi->rx_attr->size) { fprintf(stderr, "rx_depth requested: %d, " "rx_depth supported: %zd\n", rx_depth, fi->rx_attr->size); rc = 1; goto err1; } } else { rx_depth = (rx_depth_default > fi->rx_attr->size) ? fi->rx_attr->size : rx_depth_default; } ctx = pp_init_ctx(fi, size, rx_depth, use_event); if (!ctx) { rc = 1; goto err1; } if (opts.dst_addr) { /* client connect */ if (pp_connect_ctx(ctx)) { rc = 1; goto err2; } } else { /* server listen and accept */ pp_listen_ctx(ctx); pp_accept_ctx(ctx); } ctx->pending = PINGPONG_RECV_WCID; if (opts.dst_addr) { if (pp_post_send(ctx)) { fprintf(stderr, "Couldn't post send\n"); rc = 1; goto err3; } ctx->pending |= PINGPONG_SEND_WCID; } if (gettimeofday(&start, NULL)) { perror("gettimeofday"); rc = 1; goto err3; } rcnt = scnt = 0; while (rcnt < iters || scnt < iters) { struct fi_cq_entry wc; struct fi_cq_err_entry cq_err; int rd; if (use_event) { /* Blocking read */ rd = fi_cq_sread(ctx->cq, &wc, 1, NULL, -1); } else { do { rd = fi_cq_read(ctx->cq, &wc, 1); } while (rd == -FI_EAGAIN); } if (rd < 0) { fi_cq_readerr(ctx->cq, &cq_err, 0); fprintf(stderr, "cq fi_cq_readerr() %s (%d)\n", fi_cq_strerror(ctx->cq, cq_err.err, cq_err.err_data, NULL, 0), cq_err.err); rc = rd; goto err3; } switch ((int) (uintptr_t) wc.op_context) { case PINGPONG_SEND_WCID: ++scnt; break; case PINGPONG_RECV_WCID: if (--ctx->routs <= 1) { ctx->routs += pp_post_recv(ctx, ctx->rx_depth - ctx->routs); if (ctx->routs < ctx->rx_depth) { fprintf(stderr, "Couldn't post receive (%d)\n", ctx->routs); rc = 1; goto err3; } } ++rcnt; break; default: fprintf(stderr, "Completion for unknown wc_id %d\n", (int) (uintptr_t) wc.op_context); rc = 1; goto err3; } ctx->pending &= ~(int) (uintptr_t) wc.op_context; if (scnt < iters && !ctx->pending) { if (pp_post_send(ctx)) { fprintf(stderr, "Couldn't post send\n"); rc = 1; goto err3; } ctx->pending = PINGPONG_RECV_WCID | PINGPONG_SEND_WCID; } } if (gettimeofday(&end, NULL)) { perror("gettimeofday"); rc = 1; goto err3; } { float usec = (end.tv_sec - start.tv_sec) * 1000000 + (end.tv_usec - start.tv_usec); long long bytes = (long long) size * iters * 2; printf("%lld bytes in %.2f seconds = %.2f Mbit/sec\n", bytes, usec / 1000000., bytes * 8. / usec); printf("%d iters in %.2f seconds = %.2f usec/iter\n", iters, usec / 1000000., usec / iters); } err3: fi_shutdown(ctx->ep, 0); err2: ret = pp_close_ctx(ctx); if (!rc) rc = ret; err1: fi_freeinfo(fi); return rc; }
static int server_accept(size_t paramlen) { uint32_t event; int ret; ret = server_listen(paramlen); if (ret) return ret; ret = fi_domain(fabric, fi, &domain, NULL); if (ret) { FT_PRINTERR("fi_domain", ret); goto err; } ret = ft_alloc_active_res(fi); if (ret) { FT_PRINTERR("alloc_active_res", ret); goto err; } ret = ft_init_ep(); if (ret) { FT_PRINTERR("init_ep", ret); goto err; } /* Data will apppear on accept event on remote end. */ ft_fill_buf(cm_data, paramlen); /* Accept the incoming connection. Also transitions endpoint to active * state. */ ret = fi_accept(ep, cm_data, paramlen); if (ret) { FT_PRINTERR("fi_accept", ret); goto err; } /* Local FI_CONNECTED event does not have data associated. */ memset(entry, 0, sizeof(*entry)); ret = fi_eq_sread(eq, &event, entry, sizeof(*entry), -1, 0); if (ret != sizeof(*entry)) { FT_PROCESS_EQ_ERR(ret, eq, "fi_eq_sread", "accept"); goto err; } if (event != FI_CONNECTED || entry->fid != &ep->fid) { FT_ERR("Unexpected CM event %d fid %p (ep %p)", event, entry->fid, ep); ret = -FI_EOTHER; goto err; } fi_shutdown(ep, 0); ret = read_shutdown_event(); if (ret) goto err; FT_CLOSE_FID(ep); FT_CLOSE_FID(rxcq); FT_CLOSE_FID(txcq); FT_CLOSE_FID(rxcntr); FT_CLOSE_FID(txcntr); FT_CLOSE_FID(av); FT_CLOSE_FID(domain); return 0; err: fi_reject(pep, fi->handle, NULL, 0); return ret; }
static int server_connect(void) { struct fi_eq_cm_entry entry; uint32_t event; ssize_t rd; int ret, k; int num_conn_reqs = 0, num_connected = 0; struct ep_info *ep_state_array = NULL; ep_array = calloc(ep_cnt, sizeof(*ep_array)); if (!ep_array) return -FI_ENOMEM; ep_state_array = calloc(ep_cnt, sizeof(*ep_state_array)); if (!ep_state_array) return -FI_ENOMEM; while (num_connected != ep_cnt) { rd = fi_eq_sread(eq, &event, &entry, sizeof entry, -1, 0); if (rd != sizeof entry) { FT_PROCESS_EQ_ERR(rd, eq, "fi_eq_sread", "cm-event"); ret = (int) rd; goto err; } switch(event) { case FI_CONNREQ: if (num_conn_reqs == ep_cnt) { fprintf(stderr, "Unexpected CM event %d\n", event); ret = -FI_EOTHER; goto err; } fi = ep_state_array[num_conn_reqs].fi = entry.info; ep_state_array[num_conn_reqs].state = FT_EP_CONNECT_RCVD; if (num_conn_reqs == 0) { ret = fi_domain(fabric, fi, &domain, NULL); if (ret) { FT_PRINTERR("fi_domain", ret); goto err; } ret = alloc_ep_res(fi); if (ret) goto err; } ret = fi_endpoint(domain, fi, &ep_array[num_conn_reqs], NULL); if (ret) { FT_PRINTERR("fi_endpoint", ret); goto err; } ep_state_array[num_conn_reqs].ep = ep_array[num_conn_reqs]; ret = bind_ep_res(ep_array[num_conn_reqs]); if (ret) goto err; ret = fi_accept(ep_array[num_conn_reqs], NULL, 0); if (ret) { FT_PRINTERR("fi_accept", ret); goto err; } ep_state_array[num_conn_reqs].state = FT_EP_CONNECTING; num_conn_reqs++; break; case FI_CONNECTED: if (num_conn_reqs <= num_connected) { ret = -FI_EOTHER; goto err; } for (k = 0; k < num_conn_reqs; k++) { if (ep_state_array[k].state != FT_EP_CONNECTING) continue; if (&ep_state_array[k].ep->fid == entry.fid) { ep_state_array[k].state = FT_EP_CONNECTED; num_connected++; if (num_connected != ep_cnt) fi_freeinfo(ep_state_array[k].fi); break; } } if (k == num_conn_reqs) { fprintf(stderr, "Unexpected CM event %d fid %p (ep %p)\n", event, entry.fid, ep); ret = -FI_EOTHER; goto err; } break; default: ret = -FI_EOTHER; goto err; } } /* Post recv */ if (rx_shared_ctx) ret = ft_post_rx(srx_ctx, MAX(rx_size, FT_MAX_CTRL_MSG), &rx_ctx); else ret = ft_post_rx(ep_array[0], MAX(rx_size, FT_MAX_CTRL_MSG), &rx_ctx); if (ret) goto err; free(ep_state_array); return 0; err: for (k = 0; k < ep_cnt; k++) { switch(ep_state_array[k].state) { case FT_EP_CONNECT_RCVD: fi_reject(pep, ep_state_array[k].fi->handle, NULL, 0); break; case FT_EP_CONNECTING: case FT_EP_CONNECTED: fi_shutdown(ep_state_array[k].ep, 0); break; case FT_EP_STATE_INIT: default: break; } } free(ep_state_array); return ret; }
static int run_server(void) { int i, ret; printf("cmatose: starting server\n"); ret = fi_passive_ep(fabric, info, &pep, NULL); if (ret) { FT_PRINTERR("fi_passive_ep", ret); return ret; } ret = fi_pep_bind(pep, &eq->fid, 0); if (ret) { FT_PRINTERR("fi_ep_bind", ret); goto out; } ret = fi_listen(pep); if (ret) { FT_PRINTERR("fi_listen", ret); goto out; } ret = connect_events(); if (ret) goto out; if (hints->tx_attr->size) { printf("initiating data transfers\n"); for (i = 0; i < connections; i++) { ret = post_sends(&nodes[i]); if (ret) goto out; } printf("completing sends\n"); ret = poll_cqs(SEND_CQ_INDEX); if (ret) goto out; printf("receiving data transfers\n"); ret = poll_cqs(RECV_CQ_INDEX); if (ret) goto out; printf("data transfers complete\n"); } printf("cmatose: disconnecting\n"); for (i = 0; i < connections; i++) { if (!nodes[i].connected) continue; nodes[i].connected = 0; fi_shutdown(nodes[i].ep, 0); } ret = shutdown_events(); printf("disconnected\n"); out: fi_close(&pep->fid); return ret; }