static void libfabric_init_addrvec(int rx_ctx_cnt, int rx_ctx_bits) { struct gather_info* my_addr_info; void* addr_infos; char* addrs; char* tai; size_t my_addr_len; size_t addr_info_len; int i, j; // Assumes my_addr_len is the same on all nodes my_addr_len = 0; OFICHKRET(fi_getname(&ofi.ep->fid, NULL, &my_addr_len), -FI_ETOOSMALL); addr_info_len = sizeof(struct gather_info) + my_addr_len; my_addr_info = chpl_mem_alloc(addr_info_len, CHPL_RT_MD_COMM_UTIL, 0, 0); my_addr_info->node = chpl_nodeID; OFICHKERR(fi_getname(&ofi.ep->fid, &my_addr_info->info, &my_addr_len)); addr_infos = chpl_mem_allocMany(chpl_numNodes, addr_info_len, CHPL_RT_MD_COMM_PER_LOC_INFO, 0, 0); chpl_comm_ofi_oob_allgather(my_addr_info, addr_infos, addr_info_len); addrs = chpl_mem_allocMany(chpl_numNodes, my_addr_len, CHPL_RT_MD_COMM_PER_LOC_INFO, 0, 0); for (tai = addr_infos, i = 0; i < chpl_numNodes; i++) { struct gather_info* ai = (struct gather_info*) tai; assert(i >= 0); assert(i < chpl_numNodes); memcpy(addrs + ai->node * my_addr_len, ai->info, my_addr_len); tai += addr_info_len; } ofi.fi_addrs = chpl_mem_allocMany(chpl_numNodes, sizeof(ofi.fi_addrs[0]), CHPL_RT_MD_COMM_PER_LOC_INFO, 0, 0); OFICHKRET(fi_av_insert(ofi.av, addrs, chpl_numNodes, ofi.fi_addrs, 0, NULL), chpl_numNodes); ofi.rx_addrs = chpl_mem_allocMany(chpl_numNodes, sizeof(ofi.rx_addrs[0]), CHPL_RT_MD_COMM_PER_LOC_INFO, 0, 0); for (i = 0; i < chpl_numNodes; i++) { ofi.rx_addrs[i] = chpl_mem_allocMany(rx_ctx_cnt, sizeof(ofi.rx_addrs[i][0]), CHPL_RT_MD_COMM_PER_LOC_INFO, 0, 0); for (j = 0; j < rx_ctx_cnt; j++) { ofi.rx_addrs[i][j] = fi_rx_addr(ofi.fi_addrs[i], j, rx_ctx_bits); } } chpl_mem_free(my_addr_info, 0, 0); chpl_mem_free(addr_infos, 0, 0); chpl_mem_free(addrs, 0, 0); }
static inline void cntr_setup_av(void) { int i, ret; size_t addrlen = 0; ret = fi_getname(&ep[0]->fid, NULL, &addrlen); cr_assert(addrlen > 0); for (i = 0; i < NUM_EPS; i++) { ep_name[i] = malloc(addrlen); cr_assert(ep_name[i] != NULL); ret = fi_getname(&ep[i]->fid, ep_name[i], &addrlen); cr_assert(ret == FI_SUCCESS); ret = fi_av_insert(av, ep_name[i], 1, &gni_addr[i], 0, NULL); cr_assert(ret == 1); } for (i = 0; i < NUM_EPS; i++) { ret = fi_ep_bind(ep[i], &av->fid, 0); cr_assert(!ret, "fi_ep_bind av"); } }
static int init_av(void) { void *addr; size_t addrlen = 0; int ret; fi_getname(&ep->fid, NULL, &addrlen); addr = malloc(addrlen); assert(addr); ret = fi_getname(&ep->fid, addr, &addrlen); if (ret != 0) { ct_print_fi_error("fi_getname", ret); return ret; } addrs = malloc(numprocs * addrlen); assert(addrs); ctpm_Allgather(addr, addrlen, addrs); fi_addrs = malloc(numprocs * sizeof(fi_addr_t)); assert(fi_addrs); /* Insert address to the AV and get the fabric address back */ ret = fi_av_insert(av, addrs, numprocs, fi_addrs, 0, &fi_ctx_av); if (ret != numprocs) { ct_print_fi_error("fi_av_insert", ret); return ret; } free(addr); return 0; }
/****************************************************************************** * Begin test running routines ******************************************************************************/ void do_getname(void) { int i, ret; size_t addrlen; void *addr; struct gnix_ep_name *src_addr; ret = fi_getname(get_fid[ep_type](0), NULL, NULL); cr_assert(ret == -FI_EINVAL, "fi_getname returned: %s", fi_strerror(-ret)); for (i = 0; i < NUMEPS; i++) { ret = fi_getname(get_fid[ep_type](i), NULL, &addrlen); cr_assert(ret == -FI_ETOOSMALL, "fi_getname returned: %s", fi_strerror(-ret)); if (use_str_fmt) { cr_assert(addrlen == GNIX_FI_ADDR_STR_LEN, "addrlen: %lu does not match size for " "FI_ADDR_STR", addrlen); } else { cr_assert(addrlen == sizeof(struct gnix_ep_name), "addrlen: %lu does not match the size for" " FI_ADDR_GNI", addrlen); } addr = malloc(addrlen); ret = errno; cr_assert_not_null(addr, "malloc returned: %s", strerror(ret)); ret = fi_getname(get_fid[ep_type](i), addr, &addrlen); cr_assert(ret == FI_SUCCESS, "fi_getname returned: %s", fi_strerror(-ret)); if (use_str_fmt) { cr_assert(addrlen == GNIX_FI_ADDR_STR_LEN, "addrlen: %lu does not match size for " "FI_ADDR_STR", addrlen); } else { cr_assert(addrlen == sizeof(struct gnix_ep_name), "addrlen: %lu does not match the size for " "FI_ADDR_GNI", addrlen); } get_fid_ep(i, NULL, NULL, (void **) &src_addr); dbg_printf(BLUE "ep_name = %p\n" COLOR_RESET, src_addr); if (use_str_fmt) check_ep_name_str(*src_addr, addr, ep_name_len[i]); free(addr); } }
static int init_av(void) { size_t addrlen; int ret; if (opts.dst_addr) { ret = ft_av_insert(av, fi->dest_addr, 1, &remote_fi_addr, 0, NULL); if (ret) return ret; addrlen = 64; ret = fi_getname(&ep->fid, tx_buf, &addrlen); if (ret) { FT_PRINTERR("fi_getname", ret); return ret; } ret = ft_tx(ep, remote_fi_addr, addrlen, &tx_ctx); if (ret) return ret; } else { ret = wait_for_recv_completion(1); if (ret) return ret; ret = ft_av_insert(av, rx_buf, 1, &remote_fi_addr, 0, NULL); if (ret) return ret; } return 0; }
/* * rpmemd_fip_set_resp -- fill the response structure */ static int rpmemd_fip_set_resp(struct rpmemd_fip *fip, struct rpmem_resp_attr *resp) { int ret; struct sockaddr_in addr_in; size_t addrlen = sizeof(addr_in); ret = fi_getname(&fip->pep->fid, &addr_in, &addrlen); if (ret) { RPMEMD_FI_ERR(ret, "getting local endpoint address"); goto err_fi_getname; } if (!addr_in.sin_port) { RPMEMD_LOG(ERR, "dynamic allocation of port failed"); goto err_port; } resp->port = htons(addr_in.sin_port); resp->rkey = fi_mr_key(fip->mr); resp->persist_method = fip->persist_method; resp->raddr = (uint64_t)fip->addr; resp->nlanes = fip->nlanes; return 0; err_port: err_fi_getname: return -1; }
static int check_address(struct fid *fid, const char *message) { char buf1[BUFSIZ], buf2[BUFSIZ]; union sockaddr_any tmp; size_t tmplen; const char *ep_addr, *addr_expected; int ret; memset(&tmp, 0, sizeof tmp); tmplen = sizeof tmp; ret = fi_getname(fid, &tmp, &tmplen); if (ret) { FT_PRINTERR("fi_getname", ret); } if (sockaddrcmp(&tmp, tmplen, &bound_addr, bound_addr_len)) { ep_addr = sockaddrstr(&tmp, tmplen, buf1, BUFSIZ); if (!ep_addr) { FT_ERR("Unable to get ep_addr as string!\n"); return -FI_EINVAL; } addr_expected = sockaddrstr(&bound_addr, bound_addr_len, buf2, BUFSIZ); if (!addr_expected) { FT_ERR("Unable to get addr_expected as string!\n"); return -FI_EINVAL; } FT_ERR("address changed after %s: got %s expected %s\n", message, ep_addr, addr_expected); return -FI_EINVAL; } return 0; }
static ssize_t rxd_ep_send_rts(struct rxd_ep *rxd_ep, fi_addr_t rxd_addr) { struct rxd_pkt_entry *pkt_entry; struct rxd_rts_pkt *rts_pkt; ssize_t ret; size_t addrlen; pkt_entry = rxd_get_tx_pkt(rxd_ep); if (!pkt_entry) return -FI_ENOMEM; rts_pkt = (struct rxd_rts_pkt *) (pkt_entry->pkt); pkt_entry->pkt_size = sizeof(*rts_pkt) + rxd_ep->tx_prefix_size; pkt_entry->peer = rxd_addr; rts_pkt->base_hdr.version = RXD_PROTOCOL_VERSION; rts_pkt->base_hdr.type = RXD_RTS; rts_pkt->rts_addr = rxd_addr; addrlen = RXD_NAME_LENGTH; memset(rts_pkt->source, 0, RXD_NAME_LENGTH); ret = fi_getname(&rxd_ep->dg_ep->fid, (void *) rts_pkt->source, &addrlen); if (ret) { ofi_buf_free(pkt_entry); return ret; } rxd_ep_send_pkt(rxd_ep, pkt_entry); rxd_insert_unacked(rxd_ep, rxd_addr, pkt_entry); dlist_insert_tail(&rxd_ep->peers[rxd_addr].entry, &rxd_ep->rts_sent_list); return 0; }
static int rxd_ep_cm_getname(fid_t fid, void *addr, size_t *addrlen) { struct rxd_ep *ep; ep = container_of(fid, struct rxd_ep, util_ep.ep_fid.fid); return fi_getname(&ep->dg_ep->fid, addr, addrlen); }
int rxm_getname(fid_t fid, void *addr, size_t *addrlen) { struct rxm_ep *rxm_ep; rxm_ep = container_of(fid, struct rxm_ep, util_ep.ep_fid.fid); return fi_getname(&rxm_ep->msg_pep->fid, addr, addrlen); }
static int ft_load_av(void) { struct ft_msg msg; size_t len; int ret; memset(&msg, 0, sizeof(struct ft_msg)); len = sizeof(msg.data); ret = fi_getname(&ep->fid, msg.data, &len); if (ret) { FT_PRINTERR("fi_getname", ret); return ret; } msg.len = (uint32_t) len; ret = ft_fw_send(sock, &msg, sizeof msg); if (ret) return ret; ret = ft_fw_recv(sock, &msg, sizeof msg); if (ret) return ret; ret = ft_av_insert(av, msg.data, 1, &ft_tx_ctrl.addr, 0, NULL); if (ret) return ret; return 0; }
void vc_lookup_setup(int av_type, int av_size) { int ret = 0; struct fi_av_attr attr; hints = fi_allocinfo(); hints->mode = mode_bits; hints->domain_attr->mr_mode = GNIX_DEFAULT_MR_MODE; hints->fabric_attr->prov_name = strdup("gni"); /* Create endpoint */ ret = fi_getinfo(fi_version(), NULL, 0, 0, hints, &fi); cr_assert(!ret, "fi_getinfo"); ret = fi_fabric(fi->fabric_attr, &fab, NULL); cr_assert(!ret, "fi_fabric"); ret = fi_domain(fab, fi, &dom, NULL); cr_assert(!ret, "fi_domain"); attr.type = av_type; attr.count = av_size; ret = fi_av_open(dom, &attr, &av, NULL); cr_assert(!ret, "fi_av_open"); ret = fi_endpoint(dom, fi, &ep, NULL); cr_assert(!ret, "fi_endpoint"); gnix_ep = container_of(ep, struct gnix_fid_ep, ep_fid); ret = fi_getname(&ep->fid, NULL, &ep_name_len); ret = fi_getname(&ep->fid, &ep_name, &ep_name_len); cr_assert(ret == FI_SUCCESS); ret = fi_ep_bind(ep, &av->fid, 0); cr_assert(!ret, "fi_ep_bind"); ret = fi_enable(ep); cr_assert(!ret, "fi_ep_enable"); fi_freeinfo(hints); }
static int init_av(void) { size_t addrlen; int ret, i; if (opts.dst_addr) { ret = ft_av_insert(av, fi->dest_addr, 1, &remote_fi_addr, 0, NULL); if (ret) return ret; addrlen = FT_MAX_CTRL_MSG; ret = fi_getname(&sep->fid, tx_buf, &addrlen); if (ret) { FT_PRINTERR("fi_getname", ret); return ret; } ret = fi_send(tx_ep[0], tx_buf, addrlen, fi_mr_desc(mr), remote_fi_addr, NULL); if (ret) { FT_PRINTERR("fi_send", ret); return ret; } ret = wait_for_comp(rxcq_array[0]); if (ret) return ret; } else { ret = wait_for_comp(rxcq_array[0]); if (ret) return ret; ret = ft_av_insert(av, rx_buf, 1, &remote_fi_addr, 0, NULL); if (ret) return ret; ret = fi_send(tx_ep[0], tx_buf, 1, fi_mr_desc(mr), remote_fi_addr, NULL); if (ret) { FT_PRINTERR("fi_send", ret); return ret; } } for (i = 0; i < ctx_cnt; i++) remote_rx_addr[i] = fi_rx_addr(remote_fi_addr, i, rx_ctx_bits); ret = fi_recv(rx_ep[0], rx_buf, rx_size, fi_mr_desc(mr), 0, NULL); if (ret) { FT_PRINTERR("fi_recv", ret); return ret; } ret = wait_for_comp(txcq_array[0]); return ret; }
void do_getname_enosys(void) { int ret, i; for (i = 0; i < NUMEPS; i++) { ret = fi_getname(get_fid[ep_type](i), NULL, NULL); cr_assert_eq(ret, -FI_ENOSYS, "Invalid return value: %s", fi_strerror(-ret)); } }
/* TODO: retry send for unreliable endpoints */ int ft_init_av(void) { size_t addrlen; int ret; if (opts.dst_addr) { ret = ft_av_insert(av, fi->dest_addr, 1, &remote_fi_addr, 0, NULL); if (ret) return ret; addrlen = FT_MAX_CTRL_MSG; ret = fi_getname(&ep->fid, (char *) tx_buf + ft_tx_prefix_size(), &addrlen); if (ret) { FT_PRINTERR("fi_getname", ret); return ret; } ret = (int) ft_tx(addrlen); if (ret) return ret; ret = ft_rx(1); } else { ret = (int) ft_rx(FT_MAX_CTRL_MSG); if (ret) return ret; ret = ft_av_insert(av, (char *) rx_buf + ft_rx_prefix_size(), 1, &remote_fi_addr, 0, NULL); if (ret) return ret; ret = (int) ft_tx(1); } return ret; }
static inline int publish_av_info(struct fabric_info *info) { int ret = 0; char epname[128]; size_t epnamelen = sizeof(epname); #ifdef USE_ON_NODE_COMMS if(gethostname(myephostname, (EPHOSTNAMELEN - 1)) != 0) OFI_ERRMSG("gethostname error: %s \n", strerror(errno)); myephostname[EPHOSTNAMELEN-1] = '\0'; ret = shmem_runtime_put("fi_ephostname", myephostname, EPHOSTNAMELEN); if (ret != 0) { OFI_ERRMSG("shmem_runtime_put ephostname failed\n"); return ret; } #endif ret = fi_getname((fid_t)shmem_transport_ofi_epfd, epname, &epnamelen); if(ret!=0 || (epnamelen > sizeof(epname))){ OFI_ERRMSG("fi_getname failed\n"); return ret; } ret = shmem_runtime_put("fi_epname", epname, epnamelen); if (ret != 0) { OFI_ERRMSG("shmem_runtime_put epname failed\n"); return ret; } /* Note: we assume that the length of an address is the same for all * endpoints. This is safe for most HPC systems, but could be incorrect in * a heterogeneous context. */ shmem_transport_ofi_addrlen = epnamelen; return ret; }
void rdm_sr_setup_common_eps(void) { int ret = 0, i = 0, j = 0; struct fi_av_attr attr; size_t addrlen = 0; attr.type = FI_AV_MAP; attr.count = NUMEPS; cq_attr.format = FI_CQ_FORMAT_TAGGED; cq_attr.size = 1024; cq_attr.wait_obj = 0; target = malloc(BUF_SZ * 3); /* 3x BUF_SZ for multi recv testing */ assert(target); source = malloc(BUF_SZ); assert(source); uc_target = malloc(BUF_SZ); assert(uc_target); uc_source = malloc(BUF_SZ); assert(uc_source); ret = fi_fabric(fi[0]->fabric_attr, &fab, NULL); cr_assert(!ret, "fi_fabric"); for (; i < NUMEPS; i++) { ret = fi_domain(fab, fi[i], dom + i, NULL); cr_assert(!ret, "fi_domain"); ret = fi_open_ops(&dom[i]->fid, FI_GNI_DOMAIN_OPS_1, 0, (void **) (gni_domain_ops + i), NULL); ret = fi_av_open(dom[i], &attr, av + i, NULL); cr_assert(!ret, "fi_av_open"); ret = fi_endpoint(dom[i], fi[i], ep + i, NULL); cr_assert(!ret, "fi_endpoint"); ret = fi_cq_open(dom[i], &cq_attr, msg_cq + i, 0); cr_assert(!ret, "fi_cq_open"); ret = fi_ep_bind(ep[i], &msg_cq[i]->fid, FI_SEND | FI_RECV); cr_assert(!ret, "fi_ep_bind"); ret = fi_getname(&ep[i]->fid, NULL, &addrlen); cr_assert(addrlen > 0); ep_name[i] = malloc(addrlen); cr_assert(ep_name[i] != NULL); ret = fi_getname(&ep[i]->fid, ep_name[i], &addrlen); cr_assert(ret == FI_SUCCESS); } for (i = 0; i < NUMEPS; i++) { /* Insert all gni addresses into each av */ for (j = 0; j < NUMEPS; j++) { ret = fi_av_insert(av[i], ep_name[j], 1, &gni_addr[j], 0, NULL); cr_assert(ret == 1); } ret = fi_ep_bind(ep[i], &av[i]->fid, 0); cr_assert(!ret, "fi_ep_bind"); ret = fi_enable(ep[i]); cr_assert(!ret, "fi_ep_enable"); ret = fi_cntr_open(dom[i], &cntr_attr, send_cntr + i, 0); cr_assert(!ret, "fi_cntr_open"); ret = fi_ep_bind(ep[i], &send_cntr[i]->fid, FI_SEND); cr_assert(!ret, "fi_ep_bind"); ret = fi_cntr_open(dom[i], &cntr_attr, recv_cntr + i, 0); cr_assert(!ret, "fi_cntr_open"); ret = fi_ep_bind(ep[i], &recv_cntr[i]->fid, FI_RECV); cr_assert(!ret, "fi_ep_bind"); } }
static int init_av(void) { int ret; int i; if (opts.dst_addr) { ret = ft_av_insert(av, fi->dest_addr, 1, &addr_array[0], 0, NULL); if (ret) return ret; } for (i = 0; i < ep_cnt; i++) { addrlen = tx_size; ret = fi_getname(&ep_array[i]->fid, tx_buf + ft_tx_prefix_size(), &addrlen); if (ret) { FT_PRINTERR("fi_getname", ret); return ret; } if (opts.dst_addr) { ret = ft_tx(ep_array[0], addr_array[0], addrlen, &tx_ctx); if (ret) return ret; if (rx_shared_ctx) ret = ft_rx(srx_ctx, rx_size); else ret = ft_rx(ep_array[0], rx_size); if (ret) return ret; /* Skip the first address since we already have it in AV */ if (i) { ret = ft_av_insert(av, rx_buf + ft_rx_prefix_size(), 1, &addr_array[i], 0, NULL); if (ret) return ret; } } else { if (rx_shared_ctx) ret = ft_rx(srx_ctx, rx_size); else ret = ft_rx(ep_array[0], rx_size); if (ret) return ret; ret = ft_av_insert(av, rx_buf + ft_rx_prefix_size(), 1, &addr_array[i], 0, NULL); if (ret) return ret; ret = ft_tx(ep_array[0], addr_array[0], addrlen, &tx_ctx); if (ret) return ret; } } /* ACK */ if (opts.dst_addr) { ret = ft_tx(ep_array[0], addr_array[0], 1, &tx_ctx); } else { if (rx_shared_ctx) ret = ft_rx(srx_ctx, rx_size); else ret = ft_rx(ep_array[0], rx_size); } return ret; }
static void fas_ep_setup(void) { int ret, i, j; size_t addrlen = 0; fas_setup_common(fi_version()); ctx_cnt = MIN(ctx_cnt, fi[0]->domain_attr->rx_ctx_cnt); ctx_cnt = MIN(ctx_cnt, fi[0]->domain_attr->tx_ctx_cnt); for (i = 0; i < NUMEPS; i++) { fi[i]->ep_attr->tx_ctx_cnt = ctx_cnt; fi[i]->ep_attr->rx_ctx_cnt = ctx_cnt; ret = fi_domain(fab, fi[i], dom + i, NULL); cr_assert(!ret, "fi_domain returned: %s", fi_strerror(-ret)); ret = fi_cntr_open(dom[i], &cntr_attr, send_cntr + i, 0); cr_assert(!ret, "fi_cntr_open returned: %s", fi_strerror(-ret)); ret = fi_cntr_open(dom[i], &cntr_attr, recv_cntr + i, 0); cr_assert(!ret, "fi_cntr_open returned: %s", fi_strerror(-ret)); switch (ep_type) { case EP: ret = fi_endpoint(dom[i], fi[i], ep + i, NULL); cr_assert(!ret, "fi_endpoint returned: %s", fi_strerror(-ret)); break; case SEP: ret = fi_scalable_ep(dom[i], fi[i], ep + i, NULL); cr_assert(!ret, "fi_endpoint returned: %s", fi_strerror(-ret)); break; case PEP: ret = fi_passive_ep(fab, fi[i], pep + i, NULL); cr_assert(!ret, "fi_endpoint returned: %s", fi_strerror(-ret)); ret = fi_getname(get_fid[ep_type](i), NULL, &addrlen); if (use_str_fmt) { cr_assert(addrlen == GNIX_FI_ADDR_STR_LEN, "fi_getname returned: %s", fi_strerror(-ret)); } else { cr_assert(addrlen == sizeof(struct gnix_ep_name), "fi_getname returned: %s", fi_strerror(-ret)); } ep_name_len[i] = addrlen; continue; default: cr_assert_fail("Unknown endpoint type."); } ret = fi_av_open(dom[i], &attr, av + i, NULL); cr_assert(!ret, "fi_av_open returned: %s", fi_strerror(-ret)); switch (ep_type) { case EP: case PEP: ret = fi_cq_open(dom[i], &cq_attr, msg_cq + i, 0); cr_assert(!ret, "fi_cq_open returned: %s", fi_strerror(-ret)); ret = fi_ep_bind(ep[i], &msg_cq[i]->fid, FI_SEND | FI_RECV); cr_assert(!ret, "fi_ep_bind returned: %s", fi_strerror(-ret)); break; case SEP: dbg_printf(BLUE "ctx_cnt = %d\n" COLOR_RESET, ctx_cnt); for (j = 0; j < ctx_cnt; j++) { ret = fi_tx_context(ep[i], j, NULL, &tx_ep[i][j], NULL); cr_assert(!ret, "fi_tx_context returned: %s", fi_strerror(-ret)); ret = fi_cq_open(dom[i], &cq_attr, &tx_cq[i][j], NULL); cr_assert(!ret, "fi_cq_open returned: %s", fi_strerror(-ret)); ret = fi_rx_context(ep[i], j, NULL, &rx_ep[i][j], NULL); cr_assert(!ret, "fi_rx_context returned: %s", fi_strerror(-ret)); ret = fi_cq_open(dom[i], &cq_attr, &rx_cq[i][j], NULL); cr_assert(!ret, "fi_cq_open returned: %s", fi_strerror(-ret)); } break; default: cr_assert_fail("Unknown endpoint type."); } ret = fi_getname(get_fid[ep_type](i), NULL, &addrlen); if (use_str_fmt) { cr_assert(addrlen > sizeof(struct gnix_ep_name), "fi_getname returned: %s", fi_strerror(-ret)); } else { cr_assert(addrlen == sizeof(struct gnix_ep_name), "fi_getname returned: %s", fi_strerror(-ret)); } ep_name[i] = malloc(addrlen); ep_name_len[i] = addrlen; dbg_printf(BLUE "ep_name_len[%d] = %lu\n" COLOR_RESET, i, ep_name_len[i]); cr_assert(ep_name[i] != NULL, "malloc returned: %s", strerror(errno)); ret = fi_getname(get_fid[ep_type](i), ep_name[i], &addrlen); cr_assert(ret == FI_SUCCESS, "fi_getname returned: %s", fi_strerror(-ret)); } /* Just testing setname / getname for passive endpoints */ if (ep_type == PEP) return; for (i = 0; i < NUMEPS; i++) { /*Insert all gni addresses into each av*/ for (j = 0; j < NUMEPS; j++) { ret = fi_av_insert(av[i], ep_name[j], 1, &gni_addr[j], 0, NULL); cr_assert(ret == 1, "fi_av_insert returned: %s", fi_strerror(-ret)); } switch (ep_type) { case EP: ret = fi_ep_bind(ep[i], &av[i]->fid, 0); cr_assert(!ret, "fi_ep_bind returned: %s", fi_strerror(-ret)); ret = fi_ep_bind(ep[i], &send_cntr[i]->fid, FI_SEND); cr_assert(!ret, "fi_ep_bind returned: %s", fi_strerror(-ret)); ret = fi_ep_bind(ep[i], &recv_cntr[i]->fid, FI_RECV); cr_assert(!ret, "fi_ep_bind returned: %s", fi_strerror(-ret)); break; case SEP: ret = fi_scalable_ep_bind(ep[i], &av[i]->fid, 0); cr_assert(!ret, "fi_scalable_ep_bind returned: %s", fi_strerror(-ret)); dbg_printf(BLUE "ctx_cnt = %d\n" COLOR_RESET, ctx_cnt); for (j = 0; j < ctx_cnt; j++) { ret = fi_ep_bind(tx_ep[i][j], &tx_cq[i][j]->fid, FI_TRANSMIT); cr_assert(!ret, "fi_ep_bind returned: %s", fi_strerror(-ret)); ret = fi_ep_bind(tx_ep[i][j], &send_cntr[i]->fid, FI_SEND); cr_assert(!ret, "fi_ep_bind returned: %s", fi_strerror(-ret)); ret = fi_enable(tx_ep[i][j]); cr_assert(!ret, "fi_enable returned: %s", fi_strerror(-ret)); ret = fi_ep_bind(rx_ep[i][j], &rx_cq[i][j]->fid, FI_RECV); cr_assert(!ret, "fi_ep_bind returned: %s", fi_strerror(-ret)); ret = fi_ep_bind(rx_ep[i][j], &recv_cntr[i]->fid, FI_RECV); cr_assert(!ret, "fi_ep_bind returned: %s", fi_strerror(-ret)); ret = fi_enable(rx_ep[i][j]); cr_assert(!ret, "fi_enable returned: %s", fi_strerror(-ret)); } break; case PEP: break; default: cr_assert_fail("Unknown endpoint type."); } ret = fi_enable(ep[i]); cr_assert(!ret, "fi_ep_enable returned: %s", fi_strerror(-ret)); if (ep_type != SEP) { ret = fi_enable(ep[i]); cr_assert_eq(ret, -FI_EOPBADSTATE, "fi_enable returned: %s", fi_strerror(-ret)); } } }
static void setup(void) { int i, j; int ret = 0; struct fi_av_attr attr; size_t addrlen = 0; struct fi_gni_ops_domain *gni_domain_ops; uint32_t rx_cq_size; hints = fi_allocinfo(); cr_assert(hints, "fi_allocinfo"); hints->domain_attr->cq_data_size = 4; hints->domain_attr->data_progress = FI_PROGRESS_MANUAL; hints->mode = ~0; hints->fabric_attr->name = strdup("gni"); ret = fi_getinfo(FI_VERSION(1, 0), NULL, 0, 0, hints, &fi); cr_assert(!ret, "fi_getinfo"); ret = fi_fabric(fi->fabric_attr, &fab, NULL); cr_assert(!ret, "fi_fabric"); attr.type = FI_AV_TABLE; attr.count = NUM_EPS; cq_attr.format = FI_CQ_FORMAT_CONTEXT; cq_attr.size = 1024; cq_attr.wait_obj = 0; for (i = 0; i < NUM_EPS; i++) { ret = fi_domain(fab, fi, &dom[i], NULL); cr_assert(!ret, "fi_domain"); ret = fi_open_ops(&dom[i]->fid, FI_GNI_DOMAIN_OPS_1, 0, (void **) &gni_domain_ops, NULL); cr_assert(ret == FI_SUCCESS, "fi_open_ops"); rx_cq_size = min_rx_cq_size; ret = gni_domain_ops->set_val(&dom[i]->fid, GNI_RX_CQ_SIZE, &rx_cq_size); cr_assert(ret == FI_SUCCESS, "set_val"); ret = fi_av_open(dom[i], &attr, &av[i], NULL); cr_assert(!ret, "fi_av_open"); ret = fi_endpoint(dom[i], fi, &ep[i], NULL); cr_assert(!ret, "fi_endpoint"); cr_assert(ep[i]); ret = fi_cq_open(dom[i], &cq_attr, &msg_cq[i], 0); cr_assert(!ret, "fi_cq_open"); ret = fi_ep_bind(ep[i], &msg_cq[i]->fid, FI_SEND | FI_RECV); cr_assert(!ret, "fi_ep_bind"); } ret = fi_getname(&ep[0]->fid, NULL, &addrlen); cr_assert_eq(ret, -FI_ETOOSMALL); cr_assert(addrlen > 0); for (i = 0; i < NUM_EPS; i++) { ep_name[i] = malloc(addrlen); cr_assert(ep_name[i] != NULL); ret = fi_getname(&ep[i]->fid, ep_name[i], &addrlen); cr_assert(ret == FI_SUCCESS); for (j = 0; j < NUM_EPS; j++) { ret = fi_av_insert(av[j], ep_name[i], 1, &gni_addr[i], 0, NULL); cr_assert(ret == 1); } } for (i = 0; i < NUM_EPS; i++) { ret = fi_ep_bind(ep[i], &av[i]->fid, 0); cr_assert(!ret, "fi_ep_bind"); ret = fi_enable(ep[i]); cr_assert(!ret, "fi_ep_enable"); ret = fi_mr_reg(dom[i], target, NUM_EPS*sizeof(int), FI_RECV, 0, 0, 0, &rem_mr[i], &target); cr_assert_eq(ret, 0); ret = fi_mr_reg(dom[i], source, NUM_EPS*sizeof(int), FI_SEND, 0, 0, 0, &loc_mr[i], &source); cr_assert_eq(ret, 0); mr_key[i] = fi_mr_key(rem_mr[i]); } }
static int setup_handle(void) { static char buf[BUFSIZ]; struct addrinfo *ai, aihints; const char *bound_addr_str; int ret; memset(&aihints, 0, sizeof aihints); aihints.ai_flags = AI_PASSIVE; ret = getaddrinfo(opts.src_addr, opts.src_port, &aihints, &ai); if (ret == EAI_SYSTEM) { FT_ERR("getaddrinfo for %s:%s: %s\n", opts.src_addr, opts.src_port, strerror(errno)); return -ret; } else if (ret) { FT_ERR("getaddrinfo: %s\n", gai_strerror(ret)); return -FI_ENODATA; } switch (ai->ai_family) { case AF_INET: hints->addr_format = FI_SOCKADDR_IN; break; case AF_INET6: hints->addr_format = FI_SOCKADDR_IN6; break; } /* Get fabric info */ ret = fi_getinfo(FT_FIVERSION, opts.src_addr, NULL, FI_SOURCE, hints, &fi); if (ret) { FT_PRINTERR("fi_getinfo", ret); goto out; } free(fi->src_addr); fi->src_addr = NULL; fi->src_addrlen = 0; ret = fi_fabric(fi->fabric_attr, &fabric, NULL); if (ret) { FT_PRINTERR("fi_fabric", ret); goto out; } ret = fi_eq_open(fabric, &eq_attr, &eq, NULL); if (ret) { FT_PRINTERR("fi_eq_open", ret); goto out; } /* Open a passive endpoint */ ret = fi_passive_ep(fabric, fi, &pep, NULL); if (ret) { FT_PRINTERR("fi_passive_ep", ret); goto out; } ret = fi_setname(&pep->fid, ai->ai_addr, ai->ai_addrlen); if (ret) { FT_PRINTERR("fi_setname", ret); goto out; } ret = fi_getname(&pep->fid, &bound_addr, &bound_addr_len); if (ret) { FT_PRINTERR("fi_getname", ret); goto out; } /* Verify port number */ switch (ai->ai_family) { case AF_INET: if (bound_addr.sin.sin_port == 0) { FT_ERR("port number is 0 after fi_setname()\n"); ret = -FI_EINVAL; goto out; } break; case AF_INET6: if (bound_addr.sin6.sin6_port == 0) { FT_ERR("port number is 0 after fi_setname()\n"); ret = -FI_EINVAL; goto out; } break; } bound_addr_str = sockaddrstr(&bound_addr, bound_addr_len, buf, BUFSIZ); if (!bound_addr_str) { FT_ERR("Unable to get bound_addr as string!\n"); ret = -FI_EINVAL; goto out; } printf("bound_addr: \"%s\"\n", bound_addr_str); hints->handle = &pep->fid; out: freeaddrinfo(ai); return ret; }
static int init_av(void) { int ret; if (opts.dst_addr) { /* Get local address blob. Find the addrlen first. We set * addrlen as 0 and fi_getname will return the actual addrlen. */ addrlen = 0; ret = fi_getname(&ep->fid, local_addr, &addrlen); if (ret != -FI_ETOOSMALL) { FT_PRINTERR("fi_getname", ret); return ret; } local_addr = malloc(addrlen); ret = fi_getname(&ep->fid, local_addr, &addrlen); if (ret) { FT_PRINTERR("fi_getname", ret); return ret; } ret = fi_av_insert(av, remote_addr, 1, &remote_fi_addr, 0, &fi_ctx_av); if (ret != 1) { FT_PRINTERR("fi_av_insert", ret); return ret; } /* Send local addr size and local addr */ memcpy(buf, &addrlen, sizeof(size_t)); memcpy(buf + sizeof(size_t), local_addr, addrlen); ret = send_msg(sizeof(size_t) + addrlen); if (ret) return ret; /* Receive ACK from server */ ret = recv_msg(); if (ret) return ret; } else { /* Post a recv to get the remote address */ ret = recv_msg(); if (ret) return ret; memcpy(&addrlen, buf, sizeof(size_t)); remote_addr = malloc(addrlen); memcpy(remote_addr, buf + sizeof(size_t), addrlen); ret = fi_av_insert(av, remote_addr, 1, &remote_fi_addr, 0, &fi_ctx_av); if (ret != 1) { FT_PRINTERR("fi_av_insert", ret); return ret; } /* Send ACK */ ret = send_msg(16); if (ret) return ret; } /* Post first recv */ ret = fi_recv(ep, buf, buffer_size, fi_mr_desc(mr), remote_fi_addr, &fi_ctx_recv); if (ret) FT_PRINTERR("fi_recv", ret); return ret; }
static int init_av(void) { int ret; int i; /* Get local address blob. Find the addrlen first. We set addrlen * as 0 and fi_getname will return the actual addrlen. */ addrlen = 0; ret = fi_getname(&ep_array[0]->fid, local_addr, &addrlen); if (ret != -FI_ETOOSMALL) { FT_PRINTERR("fi_getname", ret); return ret; } if (ep_cnt <= 0) { fprintf(stderr, "ep_cnt needs to be greater than 0\n"); return -EXIT_FAILURE; } local_addr = malloc(addrlen * ep_cnt); /* Get local addresses for all EPs */ for (i = 0; i < ep_cnt; i++) { ret = fi_getname(&ep_array[i]->fid, local_addr + addrlen * i, &addrlen); if (ret) { FT_PRINTERR("fi_getname", ret); return ret; } } if (opts.dst_addr) { ret = ft_av_insert(av, remote_addr, 1, &addr_array[0], 0, NULL); if (ret) return ret; /* Send local EP addresses to one of the remote endpoints */ memcpy(buf, &addrlen, sizeof(size_t)); memcpy(buf + sizeof(size_t), local_addr, addrlen * ep_cnt); ret = send_msg(sizeof(size_t) + addrlen * ep_cnt); if (ret) return ret; /* Get remote EP addresses */ ret = recv_msg(); if (ret) return ret; memcpy(&addrlen, buf, sizeof(size_t)); memcpy(remote_addr, buf + sizeof(size_t), addrlen * ep_cnt); /* Insert remote addresses into AV * Skip the first address since we already have it in AV */ ret = ft_av_insert(av, remote_addr + addrlen, ep_cnt - 1, addr_array + 1, 0, NULL); if (ret) return ret; /* Send ACK */ ret = send_msg(16); if (ret) return ret; } else { /* Get remote EP addresses */ ret = recv_msg(); if (ret) return ret; memcpy(&addrlen, buf, sizeof(size_t)); remote_addr = malloc(addrlen * ep_cnt); memcpy(remote_addr, buf + sizeof(size_t), addrlen * ep_cnt); /* Insert remote addresses into AV */ ret = ft_av_insert(av, remote_addr, ep_cnt, addr_array, 0, NULL); if (ret) return ret; /* Send local EP addresses to one of the remote endpoints */ memcpy(buf, &addrlen, sizeof(size_t)); memcpy(buf + sizeof(size_t), local_addr, addrlen * ep_cnt); ret = send_msg(sizeof(size_t) + addrlen * ep_cnt); if (ret) return ret; /* Receive ACK from client */ ret = recv_msg(); if (ret) return ret; } free(local_addr); free(remote_addr); return 0; }
static mca_mtl_base_module_t* ompi_mtl_ofi_component_init(bool enable_progress_threads, bool enable_mpi_threads) { int ret, fi_version; struct fi_info *hints; struct fi_info *providers = NULL, *prov = NULL; struct fi_cq_attr cq_attr = {0}; struct fi_av_attr av_attr = {0}; char ep_name[FI_NAME_MAX] = {0}; size_t namelen; /** * Hints to filter providers * See man fi_getinfo for a list of all filters * mode: Select capabilities MTL is prepared to support. * In this case, MTL will pass in context into communication calls * ep_type: reliable datagram operation * caps: Capabilities required from the provider. * Tag matching is specified to implement MPI semantics. * msg_order: Guarantee that messages with same tag are ordered. */ hints = fi_allocinfo(); if (!hints) { opal_output_verbose(1, ompi_mtl_base_framework.framework_output, "%s:%d: Could not allocate fi_info\n", __FILE__, __LINE__); goto error; } hints->mode = FI_CONTEXT; hints->ep_attr->type = FI_EP_RDM; /* Reliable datagram */ hints->caps = FI_TAGGED; /* Tag matching interface */ hints->tx_attr->msg_order = FI_ORDER_SAS; hints->rx_attr->msg_order = FI_ORDER_SAS; hints->domain_attr->threading = FI_THREAD_UNSPEC; if (MTL_OFI_PROG_AUTO == control_progress) { hints->domain_attr->control_progress = FI_PROGRESS_AUTO; } else { hints->domain_attr->control_progress = FI_PROGRESS_MANUAL; } if (MTL_OFI_PROG_MANUAL == data_progress) { hints->domain_attr->data_progress = FI_PROGRESS_MANUAL; } else { hints->domain_attr->data_progress = FI_PROGRESS_AUTO; } if (MTL_OFI_AV_TABLE == av_type) { hints->domain_attr->av_type = FI_AV_TABLE; } else { hints->domain_attr->av_type = FI_AV_MAP; } hints->domain_attr->resource_mgmt = FI_RM_ENABLED; /** * FI_VERSION provides binary backward and forward compatibility support * Specify the version of OFI is coded to, the provider will select struct * layouts that are compatible with this version. */ fi_version = FI_VERSION(1, 0); /** * fi_getinfo: returns information about fabric services for reaching a * remote node or service. this does not necessarily allocate resources. * Pass NULL for name/service because we want a list of providers supported. */ ret = fi_getinfo(fi_version, /* OFI version requested */ NULL, /* Optional name or fabric to resolve */ NULL, /* Optional service name or port to request */ 0ULL, /* Optional flag */ hints, /* In: Hints to filter providers */ &providers); /* Out: List of matching providers */ if (0 != ret) { opal_output_verbose(1, ompi_mtl_base_framework.framework_output, "%s:%d: fi_getinfo failed: %s\n", __FILE__, __LINE__, fi_strerror(-ret)); goto error; } /** * Select a provider from the list returned by fi_getinfo(). */ prov = select_ofi_provider(providers); if (!prov) { opal_output_verbose(1, ompi_mtl_base_framework.framework_output, "%s:%d: select_ofi_provider: no provider found\n", __FILE__, __LINE__); goto error; } /** * Open fabric * The getinfo struct returns a fabric attribute struct that can be used to * instantiate the virtual or physical network. This opens a "fabric * provider". See man fi_fabric for details. */ ret = fi_fabric(prov->fabric_attr, /* In: Fabric attributes */ &ompi_mtl_ofi.fabric, /* Out: Fabric handle */ NULL); /* Optional context for fabric events */ if (0 != ret) { opal_output_verbose(1, ompi_mtl_base_framework.framework_output, "%s:%d: fi_fabric failed: %s\n", __FILE__, __LINE__, fi_strerror(-ret)); goto error; } /** * Create the access domain, which is the physical or virtual network or * hardware port/collection of ports. Returns a domain object that can be * used to create endpoints. See man fi_domain for details. */ ret = fi_domain(ompi_mtl_ofi.fabric, /* In: Fabric object */ prov, /* In: Provider */ &ompi_mtl_ofi.domain, /* Out: Domain oject */ NULL); /* Optional context for domain events */ if (0 != ret) { opal_output_verbose(1, ompi_mtl_base_framework.framework_output, "%s:%d: fi_domain failed: %s\n", __FILE__, __LINE__, fi_strerror(-ret)); goto error; } /** * Create a transport level communication endpoint. To use the endpoint, * it must be bound to completion counters or event queues and enabled, * and the resources consumed by it, such as address vectors, counters, * completion queues, etc. * see man fi_endpoint for more details. */ ret = fi_endpoint(ompi_mtl_ofi.domain, /* In: Domain object */ prov, /* In: Provider */ &ompi_mtl_ofi.ep, /* Out: Endpoint object */ NULL); /* Optional context */ if (0 != ret) { opal_output_verbose(1, ompi_mtl_base_framework.framework_output, "%s:%d: fi_endpoint failed: %s\n", __FILE__, __LINE__, fi_strerror(-ret)); goto error; } /** * Save the maximum inject size. */ ompi_mtl_ofi.max_inject_size = prov->tx_attr->inject_size; /** * Create the objects that will be bound to the endpoint. * The objects include: * - completion queue for events * - address vector of other endpoint addresses * - dynamic memory-spanning memory region */ cq_attr.format = FI_CQ_FORMAT_TAGGED; ret = fi_cq_open(ompi_mtl_ofi.domain, &cq_attr, &ompi_mtl_ofi.cq, NULL); if (ret) { opal_output_verbose(1, ompi_mtl_base_framework.framework_output, "%s:%d: fi_cq_open failed: %s\n", __FILE__, __LINE__, fi_strerror(-ret)); goto error; } /** * The remote fi_addr will be stored in the ofi_endpoint struct. */ av_attr.type = (MTL_OFI_AV_TABLE == av_type) ? FI_AV_TABLE: FI_AV_MAP; ret = fi_av_open(ompi_mtl_ofi.domain, &av_attr, &ompi_mtl_ofi.av, NULL); if (ret) { opal_output_verbose(1, ompi_mtl_base_framework.framework_output, "%s:%d: fi_av_open failed: %s\n", __FILE__, __LINE__, fi_strerror(-ret)); goto error; } /** * Bind the CQ and AV to the endpoint object. */ ret = fi_ep_bind(ompi_mtl_ofi.ep, (fid_t)ompi_mtl_ofi.cq, FI_SEND | FI_RECV); if (0 != ret) { opal_output_verbose(1, ompi_mtl_base_framework.framework_output, "%s:%d: fi_bind CQ-EP failed: %s\n", __FILE__, __LINE__, fi_strerror(-ret)); goto error; } ret = fi_ep_bind(ompi_mtl_ofi.ep, (fid_t)ompi_mtl_ofi.av, 0); if (0 != ret) { opal_output_verbose(1, ompi_mtl_base_framework.framework_output, "%s:%d: fi_bind AV-EP failed: %s\n", __FILE__, __LINE__, fi_strerror(-ret)); goto error; } /** * Enable the endpoint for communication * This commits the bind operations. */ ret = fi_enable(ompi_mtl_ofi.ep); if (0 != ret) { opal_output_verbose(1, ompi_mtl_base_framework.framework_output, "%s:%d: fi_enable failed: %s\n", __FILE__, __LINE__, fi_strerror(-ret)); goto error; } /** * Free providers info since it's not needed anymore. */ fi_freeinfo(hints); hints = NULL; fi_freeinfo(providers); providers = NULL; /** * Get our address and publish it with modex. */ namelen = sizeof(ep_name); ret = fi_getname((fid_t)ompi_mtl_ofi.ep, &ep_name[0], &namelen); if (ret) { opal_output_verbose(1, ompi_mtl_base_framework.framework_output, "%s:%d: fi_getname failed: %s\n", __FILE__, __LINE__, fi_strerror(-ret)); goto error; } OFI_COMPAT_MODEX_SEND(ret, &mca_mtl_ofi_component.super.mtl_version, &ep_name, namelen); if (OMPI_SUCCESS != ret) { opal_output_verbose(1, ompi_mtl_base_framework.framework_output, "%s:%d: modex_send failed: %d\n", __FILE__, __LINE__, ret); goto error; } ompi_mtl_ofi.epnamelen = namelen; /** * Set the ANY_SRC address. */ ompi_mtl_ofi.any_addr = FI_ADDR_UNSPEC; /** * Activate progress callback. */ ret = opal_progress_register(ompi_mtl_ofi_progress_no_inline); if (OMPI_SUCCESS != ret) { opal_output_verbose(1, ompi_mtl_base_framework.framework_output, "%s:%d: opal_progress_register failed: %d\n", __FILE__, __LINE__, ret); goto error; } return &ompi_mtl_ofi.base; error: if (providers) { (void) fi_freeinfo(providers); } if (hints) { (void) fi_freeinfo(hints); } if (ompi_mtl_ofi.av) { (void) fi_close((fid_t)ompi_mtl_ofi.av); } if (ompi_mtl_ofi.cq) { (void) fi_close((fid_t)ompi_mtl_ofi.cq); } if (ompi_mtl_ofi.ep) { (void) fi_close((fid_t)ompi_mtl_ofi.ep); } if (ompi_mtl_ofi.domain) { (void) fi_close((fid_t)ompi_mtl_ofi.domain); } if (ompi_mtl_ofi.fabric) { (void) fi_close((fid_t)ompi_mtl_ofi.fabric); } return NULL; }
void cancel_setup(void) { int ret = 0; struct fi_av_attr attr; size_t addrlen = 0; int rem_requested_key, loc_requested_key; hints = fi_allocinfo(); cr_assert(hints, "fi_allocinfo"); hints->domain_attr->mr_mode = GNIX_DEFAULT_MR_MODE; hints->domain_attr->cq_data_size = 4; hints->mode = mode_bits; hints->fabric_attr->prov_name = strdup("gni"); ret = fi_getinfo(fi_version(), NULL, 0, 0, hints, &fi); cr_assert(!ret, "fi_getinfo"); ret = fi_fabric(fi->fabric_attr, &fab, NULL); cr_assert(!ret, "fi_fabric"); ret = fi_domain(fab, fi, &dom, NULL); cr_assert(!ret, "fi_domain"); memset(&attr, 0, sizeof(attr)); attr.type = FI_AV_MAP; attr.count = 16; ret = fi_av_open(dom, &attr, &av, NULL); cr_assert(!ret, "fi_av_open"); ret = fi_endpoint(dom, fi, &ep[0], NULL); cr_assert(!ret, "fi_endpoint"); cq_attr.format = FI_CQ_FORMAT_CONTEXT; cq_attr.size = 1024; cq_attr.wait_obj = 0; ret = fi_cq_open(dom, &cq_attr, &msg_cq[0], 0); cr_assert(!ret, "fi_cq_open"); ret = fi_cq_open(dom, &cq_attr, &msg_cq[1], 0); cr_assert(!ret, "fi_cq_open"); ret = fi_ep_bind(ep[0], &msg_cq[0]->fid, FI_SEND | FI_RECV); cr_assert(!ret, "fi_ep_bind"); ret = fi_getname(&ep[0]->fid, NULL, &addrlen); cr_assert(addrlen > 0); ep_name[0] = malloc(addrlen); cr_assert(ep_name[0] != NULL); ret = fi_getname(&ep[0]->fid, ep_name[0], &addrlen); cr_assert(ret == FI_SUCCESS); ret = fi_endpoint(dom, fi, &ep[1], NULL); cr_assert(!ret, "fi_endpoint"); ret = fi_ep_bind(ep[1], &msg_cq[1]->fid, FI_SEND | FI_RECV); cr_assert(!ret, "fi_ep_bind"); ep_name[1] = malloc(addrlen); cr_assert(ep_name[1] != NULL); ret = fi_getname(&ep[1]->fid, ep_name[1], &addrlen); cr_assert(ret == FI_SUCCESS); ret = fi_av_insert(av, ep_name[0], 1, &gni_addr[0], 0, NULL); cr_assert(ret == 1); ret = fi_av_insert(av, ep_name[1], 1, &gni_addr[1], 0, NULL); cr_assert(ret == 1); ret = fi_ep_bind(ep[0], &av->fid, 0); cr_assert(!ret, "fi_ep_bind"); ret = fi_ep_bind(ep[1], &av->fid, 0); cr_assert(!ret, "fi_ep_bind"); ret = fi_enable(ep[0]); cr_assert(!ret, "fi_ep_enable"); ret = fi_enable(ep[1]); cr_assert(!ret, "fi_ep_enable"); target_base = malloc(GNIT_ALIGN_LEN(BUF_SZ)); assert(target_base); target = GNIT_ALIGN_BUFFER(char *, target_base); source_base = malloc(GNIT_ALIGN_LEN(BUF_SZ)); assert(source_base); source = GNIT_ALIGN_BUFFER(char *, source_base); rem_requested_key = USING_SCALABLE(fi) ? 1 : 0; loc_requested_key = USING_SCALABLE(fi) ? 2 : 0; ret = fi_mr_reg(dom, target, BUF_SZ, FI_REMOTE_WRITE, 0, rem_requested_key, 0, &rem_mr, &target); cr_assert_eq(ret, 0); ret = fi_mr_reg(dom, source, BUF_SZ, FI_REMOTE_WRITE, 0, loc_requested_key, 0, &loc_mr, &source); cr_assert_eq(ret, 0); if (USING_SCALABLE(fi)) { MR_ENABLE(rem_mr, target, BUF_SZ); MR_ENABLE(loc_mr, source, BUF_SZ); } mr_key = fi_mr_key(rem_mr); }
static int init_av(void) { int ret; if (opts.dst_addr) { // get local address blob. Find the addrlen first. We set // addrlen as 0 and fi_getname will return the actual addrlen addrlen = 0; ret = fi_getname(&ep->fid, local_addr, &addrlen); if (ret != -FI_ETOOSMALL) { FT_PRINTERR("fi_getname", ret); return ret; } local_addr = malloc(addrlen); ret = fi_getname(&ep->fid, local_addr, &addrlen); if (ret) { FT_PRINTERR("fi_getname", ret); return ret; } ret = fi_av_insert(av, remote_addr, 1, &remote_fi_addr, 0, &fi_ctx_av); if (ret != 1) { FT_PRINTERR("fi_av_insert", ret); return ret; } // send local addr size and local addr memcpy(buf, &addrlen, sizeof(size_t)); memcpy(buf + sizeof(size_t), local_addr, addrlen); ret = send_msg(sizeof(size_t) + addrlen); if (ret) return ret; // receive ACK from server ret = post_recv(); if (ret) return ret; } else { // post a recv to get the remote address ret = post_recv(); if (ret) return ret; memcpy(&addrlen, buf, sizeof(size_t)); remote_addr = malloc(addrlen); memcpy(remote_addr, buf + sizeof(size_t), addrlen); ret = fi_av_insert(av, remote_addr, 1, &remote_fi_addr, 0, &fi_ctx_av); if (ret != 1) { FT_PRINTERR("fi_av_insert", ret); return ret; } // send ACK ret = send_msg(16); if (ret) return ret; } return ret; }
static void setup_ep(void) { int ret; struct fi_av_attr attr; size_t addrlen = 0; attr.type = FI_AV_MAP; attr.count = 16; ret = fi_av_open(dom, &attr, &av, NULL); cr_assert(!ret, "fi_av_open"); ret = fi_endpoint(dom, fi, &ep[0], NULL); cr_assert(!ret, "fi_endpoint"); cq_attr.format = FI_CQ_FORMAT_TAGGED; cq_attr.size = 1024; cq_attr.wait_obj = 0; ret = fi_cq_open(dom, &cq_attr, &msg_cq[0], 0); cr_assert(!ret, "fi_cq_open"); ret = fi_cq_open(dom, &cq_attr, &msg_cq[1], 0); cr_assert(!ret, "fi_cq_open"); ret = fi_ep_bind(ep[0], &msg_cq[0]->fid, FI_SEND | FI_RECV); cr_assert(!ret, "fi_ep_bind"); ret = fi_getname(&ep[0]->fid, NULL, &addrlen); cr_assert(addrlen > 0); ep_name[0] = malloc(addrlen); cr_assert(ep_name[0] != NULL); ret = fi_getname(&ep[0]->fid, ep_name[0], &addrlen); cr_assert(ret == FI_SUCCESS); ret = fi_endpoint(dom, fi, &ep[1], NULL); cr_assert(!ret, "fi_endpoint"); ret = fi_ep_bind(ep[1], &msg_cq[1]->fid, FI_SEND | FI_RECV); cr_assert(!ret, "fi_ep_bind"); ep_name[1] = malloc(addrlen); cr_assert(ep_name[1] != NULL); ret = fi_getname(&ep[1]->fid, ep_name[1], &addrlen); cr_assert(ret == FI_SUCCESS); ret = fi_av_insert(av, ep_name[0], 1, &gni_addr[0], 0, NULL); cr_assert(ret == 1); ret = fi_av_insert(av, ep_name[1], 1, &gni_addr[1], 0, NULL); cr_assert(ret == 1); ret = fi_ep_bind(ep[0], &av->fid, 0); cr_assert(!ret, "fi_ep_bind"); ret = fi_ep_bind(ep[1], &av->fid, 0); cr_assert(!ret, "fi_ep_bind"); ret = fi_enable(ep[0]); cr_assert(!ret, "fi_ep_enable"); ret = fi_enable(ep[1]); cr_assert(!ret, "fi_ep_enable"); }
void rdm_str_addr_sr_setup_common(void) { int ret = 0, i = 0, j = 0; struct fi_av_attr attr; memset(&attr, 0, sizeof(attr)); attr.type = FI_AV_MAP; attr.count = NUMEPS; cq_attr.format = FI_CQ_FORMAT_TAGGED; cq_attr.size = 1024; cq_attr.wait_obj = 0; target_base = malloc(GNIT_ALIGN_LEN(BUF_SZ)); assert(target_base); target = GNIT_ALIGN_BUFFER(char *, target_base); source_base = malloc(GNIT_ALIGN_LEN(BUF_SZ)); assert(source_base); source = GNIT_ALIGN_BUFFER(char *, source_base); ret = fi_fabric(fi[0]->fabric_attr, &fab, NULL); cr_assert(!ret, "fi_fabric"); for (i = 0; i < NUMEPS; i++) { ret = fi_domain(fab, fi[i], dom + i, NULL); cr_assert(!ret, "fi_domain"); ret = fi_av_open(dom[i], &attr, av + i, NULL); cr_assert(!ret, "fi_av_open"); ret = fi_endpoint(dom[i], fi[i], ep + i, NULL); cr_assert(!ret, "fi_endpoint"); ret = fi_cq_open(dom[i], &cq_attr, msg_cq + i, 0); cr_assert(!ret, "fi_cq_open"); ret = fi_ep_bind(ep[i], &msg_cq[i]->fid, FI_SEND | FI_RECV); cr_assert(!ret, "fi_ep_bind"); ret = fi_getname(&ep[i]->fid, NULL, &addrlen); cr_assert(addrlen > 0); ep_name[i] = malloc(addrlen); cr_assert(ep_name[i] != NULL); ret = fi_getname(&ep[i]->fid, ep_name[i], &addrlen); cr_assert(ret == FI_SUCCESS); } for (i = 0; i < NUMEPS; i++) { /* * To test API-1.1: Reporting of unknown source addresses -- * only insert addresses into the sender's av */ if (i < (NUMEPS / 2)) { for (j = 0; j < NUMEPS; j++) { dbg_printf("Only does src EP insertions\n"); ret = fi_av_insert(av[i], ep_name[j], 1, &gni_addr[j], 0, NULL); cr_assert(ret == 1); } } ret = fi_ep_bind(ep[i], &av[i]->fid, 0); cr_assert(!ret, "fi_ep_bind"); ret = fi_enable(ep[i]); cr_assert(!ret, "fi_ep_enable"); } }
void rdm_api_setup_ep(void) { int ret, i, j; struct fi_av_attr attr; size_t addrlen = 0; /* Get info about fabric services with the provided hints */ for (i = 0; i < NUMEPS; i++) { ret = fi_getinfo(FI_VERSION(1, 0), NULL, 0, 0, hints[i], &fi[i]); cr_assert(!ret, "fi_getinfo"); } attr.type = FI_AV_MAP; attr.count = NUMEPS; cq_attr.format = FI_CQ_FORMAT_TAGGED; cq_attr.size = 1024; cq_attr.wait_obj = 0; target = malloc(BUF_SZ * 3); /* 3x BUF_SZ for multi recv testing */ assert(target); source = malloc(BUF_SZ); assert(source); uc_target = malloc(BUF_SZ); assert(uc_target); uc_source = malloc(BUF_SZ); assert(uc_source); ret = fi_fabric(fi[0]->fabric_attr, &fab, NULL); cr_assert(!ret, "fi_fabric"); for (i = 0; i < NUMEPS; i++) { ret = fi_domain(fab, fi[i], dom + i, NULL); cr_assert(!ret, "fi_domain"); ret = fi_open_ops(&dom[i]->fid, FI_GNI_DOMAIN_OPS_1, 0, (void **) (gni_domain_ops + i), NULL); ret = fi_av_open(dom[i], &attr, av + i, NULL); cr_assert(!ret, "fi_av_open"); ret = fi_endpoint(dom[i], fi[i], ep + i, NULL); cr_assert(!ret, "fi_endpoint"); ret = fi_cq_open(dom[i], &cq_attr, msg_cq + i, 0); cr_assert(!ret, "fi_cq_open"); ret = fi_ep_bind(ep[i], &msg_cq[i]->fid, FI_SEND | FI_RECV); cr_assert(!ret, "fi_ep_bind"); ret = fi_getname(&ep[i]->fid, NULL, &addrlen); cr_assert(addrlen > 0); ep_name[i] = malloc(addrlen); cr_assert(ep_name[i] != NULL); ret = fi_getname(&ep[i]->fid, ep_name[i], &addrlen); cr_assert(ret == FI_SUCCESS); } for (i = 0; i < NUMEPS; i++) { /* Insert all gni addresses into each av */ for (j = 0; j < NUMEPS; j++) { ret = fi_av_insert(av[i], ep_name[j], 1, &gni_addr[j], 0, NULL); cr_assert(ret == 1); } ret = fi_ep_bind(ep[i], &av[i]->fid, 0); cr_assert(!ret, "fi_ep_bind"); ret = fi_enable(ep[i]); cr_assert(!ret, "fi_ep_enable"); ret = fi_cntr_open(dom[i], &cntr_attr, send_cntr + i, 0); cr_assert(!ret, "fi_cntr_open"); ret = fi_ep_bind(ep[i], &send_cntr[i]->fid, FI_SEND); cr_assert(!ret, "fi_ep_bind"); ret = fi_cntr_open(dom[i], &cntr_attr, recv_cntr + i, 0); cr_assert(!ret, "fi_cntr_open"); ret = fi_ep_bind(ep[i], &recv_cntr[i]->fid, FI_RECV); cr_assert(!ret, "fi_ep_bind"); } for (i = 0; i < NUMEPS; i++) { ret = fi_mr_reg(dom[i], target, 3 * BUF_SZ, FI_REMOTE_WRITE, 0, 0, 0, rem_mr + i, &target); cr_assert_eq(ret, 0); ret = fi_mr_reg(dom[i], source, BUF_SZ, FI_REMOTE_WRITE, 0, 0, 0, loc_mr + i, &source); cr_assert_eq(ret, 0); mr_key[i] = fi_mr_key(rem_mr[i]); } }
static int hook_getname(fid_t fid, void *addr, size_t *addrlen) { return fi_getname(hook_to_hfid(fid), addr, addrlen); }