static int pp_listen_ctx(struct pingpong_context *ctx) { int rc = 0; rc = fi_passive_ep(ctx->fabric, ctx->info, &ctx->lep, NULL); if (rc) { fprintf(stderr, "Unable to open listener endpoint\n"); return 1; } /* Create listener EQ */ rc = pp_eq_create(ctx); if (rc) { fprintf(stderr, "Unable to allocate listener resources\n"); return 1; } rc = fi_pep_bind(ctx->lep, &ctx->eq->fid, 0); if (rc) { FT_PRINTERR("fi_pep_bind", rc); return 1; } rc = fi_listen(ctx->lep); if (rc) { FT_PRINTERR("fi_listen", rc); return 1; } printf("Listening for incoming connections...\n"); return 0; }
int cm_server_start(void) { int ret; struct sockaddr_in loc_sa; cm_local_ip(&loc_sa); srv_hints = fi_allocinfo(); srv_hints->fabric_attr->name = strdup("gni"); srv_hints->ep_attr->type = FI_EP_MSG; srv_hints->domain_attr->mr_mode = GNIX_DEFAULT_MR_MODE; ret = fi_getinfo(fi_version(), inet_ntoa(loc_sa.sin_addr), DEF_PORT, FI_SOURCE, srv_hints, &srv_fi); cr_assert(!ret); ret = fi_fabric(srv_fi->fabric_attr, &srv_fab, NULL); cr_assert(!ret); ret = fi_eq_open(srv_fab, &eq_attr, &srv_eq, NULL); cr_assert(!ret); ret = fi_passive_ep(srv_fab, srv_fi, &srv_pep, NULL); cr_assert(!ret); ret = fi_pep_bind(srv_pep, &srv_eq->fid, 0); cr_assert(!ret); ret = fi_listen(srv_pep); cr_assert(!ret); dbg_printf("Server start complete.\n"); return 0; }
static int rxm_ep_msg_res_open(struct fi_info *rxm_info, struct util_domain *util_domain, struct rxm_ep *rxm_ep) { struct rxm_fabric *rxm_fabric; struct rxm_domain *rxm_domain; struct fi_cq_attr cq_attr; int ret; ret = ofix_getinfo(rxm_prov.version, NULL, NULL, 0, &rxm_util_prov, rxm_info, rxm_alter_layer_info, rxm_alter_base_info, 1, &rxm_ep->msg_info); if (ret) return ret; rxm_domain = container_of(util_domain, struct rxm_domain, util_domain); rxm_fabric = container_of(util_domain->fabric, struct rxm_fabric, util_fabric); ret = fi_passive_ep(rxm_fabric->msg_fabric, rxm_ep->msg_info, &rxm_ep->msg_pep, rxm_ep); if (ret) { FI_WARN(&rxm_prov, FI_LOG_FABRIC, "Unable to open msg PEP\n"); goto err1; } memset(&cq_attr, 0, sizeof(cq_attr)); cq_attr.size = rxm_info->tx_attr->size + rxm_info->rx_attr->size; cq_attr.format = FI_CQ_FORMAT_MSG; ret = fi_cq_open(rxm_domain->msg_domain, &cq_attr, &rxm_ep->msg_cq, NULL); if (ret) { FI_WARN(&rxm_prov, FI_LOG_CQ, "Unable to open MSG CQ\n"); goto err1; } ret = fi_srx_context(rxm_domain->msg_domain, rxm_ep->msg_info->rx_attr, &rxm_ep->srx_ctx, NULL); if (ret) { FI_WARN(&rxm_prov, FI_LOG_FABRIC, "Unable to open shared receive context\n"); goto err2; } /* We don't care what's in the dest_addr at this point. We go by AV. */ if (rxm_ep->msg_info->dest_addr) { free(rxm_ep->msg_info->dest_addr); rxm_ep->msg_info->dest_addr = NULL; rxm_ep->msg_info->dest_addrlen = 0; } /* Zero out the port as we would be creating multiple MSG EPs for a single * RXM EP and we don't want address conflicts. */ if (rxm_ep->msg_info->src_addr) ((struct sockaddr_in *)(rxm_ep->msg_info->src_addr))->sin_port = 0; return 0; err2: fi_close(&rxm_ep->msg_pep->fid); err1: fi_freeinfo(rxm_ep->msg_info); return ret; }
static int server_listen(void) { struct fi_info *fi; int ret; /* Get fabric info */ ret = fi_getinfo(FT_FIVERSION, NULL, opts.src_port, FI_SOURCE, hints, &fi); if (ret) { FT_PRINTERR("fi_getinfo", ret); return ret; } /* Open the fabric */ ret = fi_fabric(fi->fabric_attr, &fab, NULL); if (ret) { FT_PRINTERR("fi_fabric", ret); goto err0; } /* Open a passive endpoint */ ret = fi_passive_ep(fab, fi, &pep, NULL); if (ret) { FT_PRINTERR("fi_passive_ep", ret); goto err1; } /* Allocate connection management resources */ ret = alloc_cm_res(); if (ret) goto err2; /* Bind EQ to passive endpoint */ ret = fi_pep_bind(pep, &cmeq->fid, 0); if (ret) { FT_PRINTERR("fi_pep_bind", ret); goto err3; } /* Listen for incoming connections */ ret = fi_listen(pep); if (ret) { FT_PRINTERR("fi_listen", ret); goto err3; } fi_freeinfo(fi); return 0; err3: fi_close(&cmeq->fid); err2: fi_close(&pep->fid); err1: fi_close(&fab->fid); err0: fi_freeinfo(fi); return ret; }
static int server_listen(void) { struct fi_info *fi; int ret; ret = fi_getinfo(FI_VERSION(1, 0), src_addr, port, FI_SOURCE, &hints, &fi); if (ret) { printf("fi_getinfo %s\n", strerror(-ret)); return ret; } cq_data_size = fi->domain_attr->cq_data_size; ret = fi_fabric(fi->fabric_attr, &fab, NULL); if (ret) { printf("fi_fabric %s\n", fi_strerror(-ret)); goto err0; } ret = fi_passive_ep(fab, fi, &pep, NULL); if (ret) { printf("fi_passive_ep %s\n", fi_strerror(-ret)); goto err1; } ret = alloc_cm_res(); if (ret) goto err2; ret = fi_bind(&pep->fid, &cmeq->fid, 0); if (ret) { printf("fi_bind %s\n", fi_strerror(-ret)); goto err3; } ret = fi_listen(pep); if (ret) { printf("fi_listen %s\n", fi_strerror(-ret)); goto err3; } fi_freeinfo(fi); return 0; err3: free_lres(); err2: fi_close(&pep->fid); err1: fi_close(&fab->fid); err0: fi_freeinfo(fi); return ret; }
static int server_listen(void) { struct fi_info *fi; int ret; ret = fi_getinfo(FT_FIVERSION, opts.src_addr, opts.src_port, FI_SOURCE, hints, &fi); if (ret) { FT_PRINTERR("fi_getinfo", ret); return ret; } ret = fi_fabric(fi->fabric_attr, &fab, NULL); if (ret) { FT_PRINTERR("fi_fabric", ret); goto err0; } ret = fi_passive_ep(fab, fi, &pep, NULL); if (ret) { FT_PRINTERR("fi_passive_ep", ret); goto err1; } ret = alloc_cm_res(); if (ret) goto err2; ret = fi_pep_bind(pep, &cmeq->fid, 0); if (ret) { FT_PRINTERR("fi_pep_bind", ret); goto err3; } ret = fi_listen(pep); if (ret) { FT_PRINTERR("fi_listen", ret); goto err3; } fi_freeinfo(fi); return 0; err3: free_lres(); err2: fi_close(&pep->fid); err1: fi_close(&fab->fid); err0: fi_freeinfo(fi); return ret; }
int ft_start_server(void) { int ret; ret = fi_getinfo(FT_FIVERSION, opts.src_addr, opts.src_port, FI_SOURCE, hints, &fi_pep); if (ret) { FT_PRINTERR("fi_getinfo", ret); return ret; } ret = fi_fabric(fi_pep->fabric_attr, &fabric, NULL); if (ret) { FT_PRINTERR("fi_fabric", ret); return ret; } ret = fi_eq_open(fabric, &eq_attr, &eq, NULL); if (ret) { FT_PRINTERR("fi_eq_open", ret); return ret; } ret = fi_passive_ep(fabric, fi_pep, &pep, NULL); if (ret) { FT_PRINTERR("fi_passive_ep", ret); return ret; } ret = fi_pep_bind(pep, &eq->fid, 0); if (ret) { FT_PRINTERR("fi_pep_bind", ret); return ret; } ret = fi_listen(pep); if (ret) { FT_PRINTERR("fi_listen", ret); return ret; } return 0; }
int ft_start_server(void) { int ret; ret = ft_getinfo(hints, &fi_pep); if (ret) return ret; ret = fi_fabric(fi_pep->fabric_attr, &fabric, NULL); if (ret) { FT_PRINTERR("fi_fabric", ret); return ret; } ret = fi_eq_open(fabric, &eq_attr, &eq, NULL); if (ret) { FT_PRINTERR("fi_eq_open", ret); return ret; } ret = fi_passive_ep(fabric, fi_pep, &pep, NULL); if (ret) { FT_PRINTERR("fi_passive_ep", ret); return ret; } ret = fi_pep_bind(pep, &eq->fid, 0); if (ret) { FT_PRINTERR("fi_pep_bind", ret); return ret; } ret = fi_listen(pep); if (ret) { FT_PRINTERR("fi_listen", ret); return ret; } return 0; }
static void fas_ep_setup(void) { int ret, i, j; size_t addrlen = 0; fas_setup_common(fi_version()); ctx_cnt = MIN(ctx_cnt, fi[0]->domain_attr->rx_ctx_cnt); ctx_cnt = MIN(ctx_cnt, fi[0]->domain_attr->tx_ctx_cnt); for (i = 0; i < NUMEPS; i++) { fi[i]->ep_attr->tx_ctx_cnt = ctx_cnt; fi[i]->ep_attr->rx_ctx_cnt = ctx_cnt; ret = fi_domain(fab, fi[i], dom + i, NULL); cr_assert(!ret, "fi_domain returned: %s", fi_strerror(-ret)); ret = fi_cntr_open(dom[i], &cntr_attr, send_cntr + i, 0); cr_assert(!ret, "fi_cntr_open returned: %s", fi_strerror(-ret)); ret = fi_cntr_open(dom[i], &cntr_attr, recv_cntr + i, 0); cr_assert(!ret, "fi_cntr_open returned: %s", fi_strerror(-ret)); switch (ep_type) { case EP: ret = fi_endpoint(dom[i], fi[i], ep + i, NULL); cr_assert(!ret, "fi_endpoint returned: %s", fi_strerror(-ret)); break; case SEP: ret = fi_scalable_ep(dom[i], fi[i], ep + i, NULL); cr_assert(!ret, "fi_endpoint returned: %s", fi_strerror(-ret)); break; case PEP: ret = fi_passive_ep(fab, fi[i], pep + i, NULL); cr_assert(!ret, "fi_endpoint returned: %s", fi_strerror(-ret)); ret = fi_getname(get_fid[ep_type](i), NULL, &addrlen); if (use_str_fmt) { cr_assert(addrlen == GNIX_FI_ADDR_STR_LEN, "fi_getname returned: %s", fi_strerror(-ret)); } else { cr_assert(addrlen == sizeof(struct gnix_ep_name), "fi_getname returned: %s", fi_strerror(-ret)); } ep_name_len[i] = addrlen; continue; default: cr_assert_fail("Unknown endpoint type."); } ret = fi_av_open(dom[i], &attr, av + i, NULL); cr_assert(!ret, "fi_av_open returned: %s", fi_strerror(-ret)); switch (ep_type) { case EP: case PEP: ret = fi_cq_open(dom[i], &cq_attr, msg_cq + i, 0); cr_assert(!ret, "fi_cq_open returned: %s", fi_strerror(-ret)); ret = fi_ep_bind(ep[i], &msg_cq[i]->fid, FI_SEND | FI_RECV); cr_assert(!ret, "fi_ep_bind returned: %s", fi_strerror(-ret)); break; case SEP: dbg_printf(BLUE "ctx_cnt = %d\n" COLOR_RESET, ctx_cnt); for (j = 0; j < ctx_cnt; j++) { ret = fi_tx_context(ep[i], j, NULL, &tx_ep[i][j], NULL); cr_assert(!ret, "fi_tx_context returned: %s", fi_strerror(-ret)); ret = fi_cq_open(dom[i], &cq_attr, &tx_cq[i][j], NULL); cr_assert(!ret, "fi_cq_open returned: %s", fi_strerror(-ret)); ret = fi_rx_context(ep[i], j, NULL, &rx_ep[i][j], NULL); cr_assert(!ret, "fi_rx_context returned: %s", fi_strerror(-ret)); ret = fi_cq_open(dom[i], &cq_attr, &rx_cq[i][j], NULL); cr_assert(!ret, "fi_cq_open returned: %s", fi_strerror(-ret)); } break; default: cr_assert_fail("Unknown endpoint type."); } ret = fi_getname(get_fid[ep_type](i), NULL, &addrlen); if (use_str_fmt) { cr_assert(addrlen > sizeof(struct gnix_ep_name), "fi_getname returned: %s", fi_strerror(-ret)); } else { cr_assert(addrlen == sizeof(struct gnix_ep_name), "fi_getname returned: %s", fi_strerror(-ret)); } ep_name[i] = malloc(addrlen); ep_name_len[i] = addrlen; dbg_printf(BLUE "ep_name_len[%d] = %lu\n" COLOR_RESET, i, ep_name_len[i]); cr_assert(ep_name[i] != NULL, "malloc returned: %s", strerror(errno)); ret = fi_getname(get_fid[ep_type](i), ep_name[i], &addrlen); cr_assert(ret == FI_SUCCESS, "fi_getname returned: %s", fi_strerror(-ret)); } /* Just testing setname / getname for passive endpoints */ if (ep_type == PEP) return; for (i = 0; i < NUMEPS; i++) { /*Insert all gni addresses into each av*/ for (j = 0; j < NUMEPS; j++) { ret = fi_av_insert(av[i], ep_name[j], 1, &gni_addr[j], 0, NULL); cr_assert(ret == 1, "fi_av_insert returned: %s", fi_strerror(-ret)); } switch (ep_type) { case EP: ret = fi_ep_bind(ep[i], &av[i]->fid, 0); cr_assert(!ret, "fi_ep_bind returned: %s", fi_strerror(-ret)); ret = fi_ep_bind(ep[i], &send_cntr[i]->fid, FI_SEND); cr_assert(!ret, "fi_ep_bind returned: %s", fi_strerror(-ret)); ret = fi_ep_bind(ep[i], &recv_cntr[i]->fid, FI_RECV); cr_assert(!ret, "fi_ep_bind returned: %s", fi_strerror(-ret)); break; case SEP: ret = fi_scalable_ep_bind(ep[i], &av[i]->fid, 0); cr_assert(!ret, "fi_scalable_ep_bind returned: %s", fi_strerror(-ret)); dbg_printf(BLUE "ctx_cnt = %d\n" COLOR_RESET, ctx_cnt); for (j = 0; j < ctx_cnt; j++) { ret = fi_ep_bind(tx_ep[i][j], &tx_cq[i][j]->fid, FI_TRANSMIT); cr_assert(!ret, "fi_ep_bind returned: %s", fi_strerror(-ret)); ret = fi_ep_bind(tx_ep[i][j], &send_cntr[i]->fid, FI_SEND); cr_assert(!ret, "fi_ep_bind returned: %s", fi_strerror(-ret)); ret = fi_enable(tx_ep[i][j]); cr_assert(!ret, "fi_enable returned: %s", fi_strerror(-ret)); ret = fi_ep_bind(rx_ep[i][j], &rx_cq[i][j]->fid, FI_RECV); cr_assert(!ret, "fi_ep_bind returned: %s", fi_strerror(-ret)); ret = fi_ep_bind(rx_ep[i][j], &recv_cntr[i]->fid, FI_RECV); cr_assert(!ret, "fi_ep_bind returned: %s", fi_strerror(-ret)); ret = fi_enable(rx_ep[i][j]); cr_assert(!ret, "fi_enable returned: %s", fi_strerror(-ret)); } break; case PEP: break; default: cr_assert_fail("Unknown endpoint type."); } ret = fi_enable(ep[i]); cr_assert(!ret, "fi_ep_enable returned: %s", fi_strerror(-ret)); if (ep_type != SEP) { ret = fi_enable(ep[i]); cr_assert_eq(ret, -FI_EOPBADSTATE, "fi_enable returned: %s", fi_strerror(-ret)); } } }
static int setup_handle(void) { static char buf[BUFSIZ]; struct addrinfo *ai, aihints; const char *bound_addr_str; int ret; memset(&aihints, 0, sizeof aihints); aihints.ai_flags = AI_PASSIVE; ret = getaddrinfo(opts.src_addr, opts.src_port, &aihints, &ai); if (ret == EAI_SYSTEM) { FT_ERR("getaddrinfo for %s:%s: %s\n", opts.src_addr, opts.src_port, strerror(errno)); return -ret; } else if (ret) { FT_ERR("getaddrinfo: %s\n", gai_strerror(ret)); return -FI_ENODATA; } switch (ai->ai_family) { case AF_INET: hints->addr_format = FI_SOCKADDR_IN; break; case AF_INET6: hints->addr_format = FI_SOCKADDR_IN6; break; } /* Get fabric info */ ret = fi_getinfo(FT_FIVERSION, opts.src_addr, NULL, FI_SOURCE, hints, &fi); if (ret) { FT_PRINTERR("fi_getinfo", ret); goto out; } free(fi->src_addr); fi->src_addr = NULL; fi->src_addrlen = 0; ret = fi_fabric(fi->fabric_attr, &fabric, NULL); if (ret) { FT_PRINTERR("fi_fabric", ret); goto out; } ret = fi_eq_open(fabric, &eq_attr, &eq, NULL); if (ret) { FT_PRINTERR("fi_eq_open", ret); goto out; } /* Open a passive endpoint */ ret = fi_passive_ep(fabric, fi, &pep, NULL); if (ret) { FT_PRINTERR("fi_passive_ep", ret); goto out; } ret = fi_setname(&pep->fid, ai->ai_addr, ai->ai_addrlen); if (ret) { FT_PRINTERR("fi_setname", ret); goto out; } ret = fi_getname(&pep->fid, &bound_addr, &bound_addr_len); if (ret) { FT_PRINTERR("fi_getname", ret); goto out; } /* Verify port number */ switch (ai->ai_family) { case AF_INET: if (bound_addr.sin.sin_port == 0) { FT_ERR("port number is 0 after fi_setname()\n"); ret = -FI_EINVAL; goto out; } break; case AF_INET6: if (bound_addr.sin6.sin6_port == 0) { FT_ERR("port number is 0 after fi_setname()\n"); ret = -FI_EINVAL; goto out; } break; } bound_addr_str = sockaddrstr(&bound_addr, bound_addr_len, buf, BUFSIZ); if (!bound_addr_str) { FT_ERR("Unable to get bound_addr as string!\n"); ret = -FI_EINVAL; goto out; } printf("bound_addr: \"%s\"\n", bound_addr_str); hints->handle = &pep->fid; out: freeaddrinfo(ai); return ret; }
/* * rpmemd_fip_init_fabric_res -- initialize common fabric's resources */ static int rpmemd_fip_init_fabric_res(struct rpmemd_fip *fip) { int ret; ret = fi_fabric(fip->fi->fabric_attr, &fip->fabric, NULL); if (ret) { RPMEMD_FI_ERR(ret, "opening fabric domain"); goto err_fi_fabric; } ret = fi_domain(fip->fabric, fip->fi, &fip->domain, NULL); if (ret) { RPMEMD_FI_ERR(ret, "opening fabric access domain"); goto err_fi_domain; } struct fi_eq_attr eq_attr = { .size = 0, /* use default */ .flags = 0, .wait_obj = FI_WAIT_UNSPEC, .signaling_vector = 0, .wait_set = NULL, }; ret = fi_eq_open(fip->fabric, &eq_attr, &fip->eq, NULL); if (ret) { RPMEMD_FI_ERR(ret, "opening event queue"); goto err_eq_open; } ret = fi_passive_ep(fip->fabric, fip->fi, &fip->pep, NULL); if (ret) { RPMEMD_FI_ERR(ret, "allocating passive endpoint"); goto err_pep; } ret = fi_pep_bind(fip->pep, &fip->eq->fid, 0); if (ret) { RPMEMD_FI_ERR(ret, "binding event queue to passive endpoint"); goto err_pep_bind_eq; } return 0; err_pep_bind_eq: RPMEMD_FI_CLOSE(fip->pep, "closing passive endpoint"); err_pep: RPMEMD_FI_CLOSE(fip->eq, "closing event queue"); err_eq_open: RPMEMD_FI_CLOSE(fip->domain, "closing fabric access domain"); err_fi_domain: RPMEMD_FI_CLOSE(fip->fabric, "closing fabric domain"); err_fi_fabric: return ret; } /* * rpmemd_fip_fini_fabric_res -- deinitialize common fabric resources */ static void rpmemd_fip_fini_fabric_res(struct rpmemd_fip *fip) { RPMEMD_FI_CLOSE(fip->pep, "closing passive endpoint"); RPMEMD_FI_CLOSE(fip->eq, "closing event queue"); RPMEMD_FI_CLOSE(fip->domain, "closing fabric access domain"); RPMEMD_FI_CLOSE(fip->fabric, "closing fabric domain"); }
static void test_connect_with_accept_blocking_on_eq_fq_SERVER(void) { int ret; printf("SERVER running\n"); setup_ofi(NULL, NULL, FI_SOURCE); #if WANT_FDS // Add the EQ FD to the epoll fd static struct epoll_event edt; memset(&edt, 0, sizeof(edt)); edt.events = EPOLLIN; edt.data.u32 = 2222; ret = epoll_ctl(epoll_fd, EPOLL_CTL_ADD, fidev.eq_fd, &edt); if (ret < 0) { error("server epoll_ctl failed"); } #endif // Make a PEP ret = fi_passive_ep(fidev.fabric, fidev.info, &fidev.pep, NULL); if (0 != ret) { error("fi_passive_ep failed"); } #if WANT_FIXED_PORT size_t ss = sizeof(sin); ret = fi_getname(&(fidev.pep->fid), &sin, &ss); if (0 != ret) { error("fi_setname failed"); } sin.sin_port = htons(listen_port); // Bind the PEP to listen on a specific port ret = fi_setname(&(fidev.pep->fid), &sin, sizeof(sin)); if (0 != ret) { error("fi_setname failed"); } #endif // Bind the EQ to the PEP ret = fi_pep_bind(fidev.pep, &fidev.eq->fid, 0); if (0 != ret) { error("fi_pep_bind failed"); } // Listen ret = fi_listen(fidev.pep); if (0 != ret) { error("fi_listen failed"); } // Get the actual address of this PEP struct sockaddr_in sinout; size_t s = sizeof(sinout); ret = fi_getname(&(fidev.pep->fid), &sinout, &s); if (0 != ret) { error("fi_setname failed"); } sin.sin_family = sinout.sin_family; sin.sin_addr = sinout.sin_addr; sin.sin_port = sinout.sin_port; // Print server addr printf("SERVER listening on %s\n", addrstr(&sin)); // Send our node (IP addr) and service (port) to the client snprintf(ofi_node, sizeof(ofi_node) - 1, "%s", inet_ntoa(sin.sin_addr)); snprintf(ofi_service, sizeof(ofi_service) - 1, "%d", ntohs(sin.sin_port)); MPI_Send(ofi_node, sizeof(ofi_node) - 1, MPI_CHAR, 1, 101, MPI_COMM_WORLD); MPI_Send(ofi_service, sizeof(ofi_service) - 1, MPI_CHAR, 1, 102, MPI_COMM_WORLD); printf("SERVER sent via MPI to client: %s / %s\n", ofi_node, ofi_service); #if WANT_FDS // Now wait for the listen to complete int nevents; #define NEVENTS 32 struct epoll_event events[NEVENTS]; int timeout = 10000; while (1) { printf("SERVER blocking on epoll\n"); nevents = epoll_wait(epoll_fd, events, NEVENTS, timeout); if (nevents < 0) { if (errno != EINTR) { error("server epoll wait failed"); } else { continue; } } else { printf("SERVER successfully woke up from epoll! %d events\n", nevents); for (int i = 0; i < nevents; ++i) { if (events[i].data.u32 != 2222) { error("server unexpected epoll return type"); } } // If we got the expected event, then go read from the EQ break; } } #endif // Wait for the FI_CONNREQ event uint32_t event; uint8_t *entry_buffer; size_t expected_len = sizeof(struct fi_eq_cm_entry) + sizeof(client_data); entry_buffer = (uint8_t*) calloc(1, expected_len); if (NULL == entry_buffer) { error("calloc failed"); } struct fi_eq_cm_entry *entry = (struct fi_eq_cm_entry*) entry_buffer; while (1) { printf("SERVER waiting for FI_CONNREQ\n"); #if WANT_FDS ret = fi_eq_read(fidev.eq, &event, entry, expected_len, 0); #else ret = fi_eq_sread(fidev.eq, &event, entry, expected_len, -1, 0); #endif if (-FI_EAVAIL == ret) { printf("server fi_eq_sread failed because there's something in the error queue\n"); char buffer[2048]; struct fi_eq_err_entry *err_entry = (struct fi_eq_err_entry*) buffer; ret = fi_eq_readerr(fidev.eq, err_entry, 0); printf("error code: %d (%s), prov err code: %d (%s)\n", err_entry->err, fi_strerror(err_entry->err), err_entry->prov_errno, fi_strerror(err_entry->prov_errno)); error("sad panda"); } else if (-EAGAIN == ret) { fprintf(stderr, "SERVER fi_eq_sread fail got -EAGAIN... trying again...\n"); sleep(1); continue; } else if (ret < 0) { fprintf(stderr, "SERVER fi_eq_sread fail: %s (FI_EAVAIL = %d, -ret = %d)\n", fi_strerror(-ret), FI_EAVAIL, -ret); error("SERVER fi_eq_sread failed for some random reason"); } else if (event != FI_CONNREQ) { error("SERVER got some unexpected event"); } else if (ret != expected_len) { error("SERVER got wrong length back from fi_eq_sread"); } uint32_t *d = (uint32_t*) entry->data; for (int i = 0; i < (sizeof(client_data) / sizeof(uint32_t)); ++i) { if (d[i] != client_data[i]) { printf("SERVER got wrong CM client data: d[%d]=%d, should be %d\n", i, d[i], client_data[i]); } } printf("SERVER got FI_CONNREQ, correct size, and correct data -- yay!\n"); break; } // Silly logistics: setup_ofi_active adds the fd to the epoll set. // But we already added it. So for simplicity, just remove it // here so that setup_ofi_active() can re-add it. #if WANT_FDS // Remove the EQ FD from the epoll fd ret = epoll_ctl(epoll_fd, EPOLL_CTL_DEL, fidev.eq_fd, &edt); if (ret < 0) { error("server epoll_ctl DEL failed"); } #endif // Make an active endpoint setup_ofi_active(entry->info, &ficonn.ep); // Accept the incoming connection ret = fi_accept(ficonn.ep, (void*) server_data, sizeof(server_data)); if (ret != 0) { printf("fi_accept: ret=%d, %s\n", ret, fi_strerror(-ret)); error("SERVER fi_accept failed\n"); } // Need to read and get a FI_CONNECTED event while (1) { printf("SERVER waiting for FI_CONNECTED\n"); #if WANT_FDS ret = fi_eq_read(fidev.eq, &event, entry, expected_len, 0); #else ret = fi_eq_sread(fidev.eq, &event, entry, expected_len, -1, 0); #endif if (-FI_EAVAIL == ret) { printf("server fi_eq_sread failed because there's something in the error queue\n"); char buffer[2048]; struct fi_eq_err_entry *err_entry = (struct fi_eq_err_entry*) buffer; ret = fi_eq_readerr(fidev.eq, err_entry, 0); printf("error code: %d (%s), prov err code: %d (%s)\n", err_entry->err, fi_strerror(err_entry->err), err_entry->prov_errno, fi_strerror(err_entry->prov_errno)); error("sad panda"); } else if (-EAGAIN == ret) { fprintf(stderr, "SERVER fi_eq_sread fail got -EAGAIN... trying again...\n"); sleep(1); continue; } else if (ret < 0) { fprintf(stderr, "SERVER fi_eq_sread fail: %s (FI_EAVAIL = %d, -ret = %d)\n", fi_strerror(-ret), FI_EAVAIL, -ret); error("SERVER fi_eq_sread failed for some random reason"); } else if (event != FI_CONNECTED) { error("SERVER got some unexpected event"); } printf("SERVER got FI_CONNECTED -- yay!\n"); break; } // Post a recv buffer for the client to send int msg[4] = { 0 }; int len = sizeof(msg); printf("SERVER receiving len of %d\n", len); struct fid_mr no_mr; struct fid_mr *mr; void *recv_context = (void*) 0x17; #if 0 fi_mr_reg(fidev.domain, msg, len, FI_SEND | FI_RECV, 0, (uint64_t)(uintptr_t) msg, 0, &mr, NULL); #else // Try using no mr, like fi_msg_pingpong... memset(&no_mr, 0, sizeof(no_mr)); mr = &no_mr; #endif ret = fi_recv(ficonn.ep, msg, len, fi_mr_desc(mr), 0, recv_context); if (ret < 0) { printf("fi_recv failed! %d, %s\n", ret, fi_strerror(-ret)); MPI_Abort(MPI_COMM_WORLD, 37); } sleep(1); printf("SERVER posted receive -- waiting for client to send\n"); MPI_Barrier(MPI_COMM_WORLD); // Wait for receive completion struct fi_cq_entry cqe; while (1) { ret = fi_cq_sread(ficonn.cq, &cqe, 1, 0, -1); if (cqe.op_context == recv_context) { printf("SERVER receive completed\n"); break; } else { printf("SERVER got some other completion... continuing\n"); } } printf("SERVER finished -- waiting for client before teardown\n"); MPI_Barrier(MPI_COMM_WORLD); printf("SERVER tearing down\n"); fi_close(&(mr->fid)); teardown_ofi(); }
static int run_server(void) { int i, ret; printf("cmatose: starting server\n"); ret = fi_passive_ep(fabric, info, &pep, NULL); if (ret) { FT_PRINTERR("fi_passive_ep", ret); return ret; } ret = fi_pep_bind(pep, &eq->fid, 0); if (ret) { FT_PRINTERR("fi_ep_bind", ret); goto out; } ret = fi_listen(pep); if (ret) { FT_PRINTERR("fi_listen", ret); goto out; } ret = connect_events(); if (ret) goto out; if (hints->tx_attr->size) { printf("initiating data transfers\n"); for (i = 0; i < connections; i++) { ret = post_sends(&nodes[i]); if (ret) goto out; } printf("completing sends\n"); ret = poll_cqs(SEND_CQ_INDEX); if (ret) goto out; printf("receiving data transfers\n"); ret = poll_cqs(RECV_CQ_INDEX); if (ret) goto out; printf("data transfers complete\n"); } printf("cmatose: disconnecting\n"); for (i = 0; i < connections; i++) { if (!nodes[i].connected) continue; nodes[i].connected = 0; fi_shutdown(nodes[i].ep, 0); } ret = shutdown_events(); printf("disconnected\n"); out: fi_close(&pep->fid); return ret; }