void fi_log_init(void) { struct fi_filter subsys_filter; int level, i; char *levelstr = NULL, *provstr = NULL, *subsysstr = NULL; fi_param_define(NULL, "log_level", FI_PARAM_STRING, "Specify logging level: warn, trace, info, debug (default: warn)"); fi_param_get_str(NULL, "log_level", &levelstr); level = fi_convert_log_str(levelstr); if (level >= 0) log_mask = ((1 << (level + 1)) - 1); fi_param_define(NULL, "log_prov", FI_PARAM_STRING, "Specify specific provider to log (default: all)"); fi_param_get_str(NULL, "log_prov", &provstr); ofi_create_filter(&prov_log_filter, provstr); fi_param_define(NULL, "log_subsys", FI_PARAM_STRING, "Specify specific subsystem to log (default: all)"); fi_param_get_str(NULL, "log_subsys", &subsysstr); ofi_create_filter(&subsys_filter, subsysstr); for (i = 0; i < FI_LOG_SUBSYS_MAX; i++) { if (!ofi_apply_filter(&subsys_filter, log_subsys[i])) log_mask |= (1 << (i + FI_LOG_SUBSYS_OFFSET)); } ofi_free_filter(&subsys_filter); }
static void psmx2_init_env(void) { if (getenv("OMPI_COMM_WORLD_RANK") || getenv("PMI_RANK")) psmx2_env.name_server = 0; fi_param_get_bool(&psmx2_prov, "name_server", &psmx2_env.name_server); fi_param_get_bool(&psmx2_prov, "tagged_rma", &psmx2_env.tagged_rma); fi_param_get_str(&psmx2_prov, "uuid", &psmx2_env.uuid); fi_param_get_int(&psmx2_prov, "delay", &psmx2_env.delay); fi_param_get_int(&psmx2_prov, "timeout", &psmx2_env.timeout); fi_param_get_int(&psmx2_prov, "prog_interval", &psmx2_env.prog_interval); fi_param_get_str(&psmx2_prov, "prog_affinity", &psmx2_env.prog_affinity); }
void ofi_hook_init(void) { char *param_val = NULL; fi_param_define(NULL, "hook", FI_PARAM_STRING, "Intercept calls to underlying provider and apply " "the specified functionality to them. Hook option: " "perf (gather performance data)"); fi_param_get_str(NULL, "hook", ¶m_val); hooks_enabled = 0; if (!param_val) return; if (!strcasecmp(param_val, "noop")) { FI_INFO(&core_prov, FI_LOG_CORE, "Noop hook requested\n"); hooks_enabled |= (1 << HOOK_NOOP); } if (!strcasecmp(param_val, "perf")) { FI_INFO(&core_prov, FI_LOG_CORE, "Perf hook requested\n"); hooks_enabled |= (1 << HOOK_PERF); } }
void fi_ini(void) { char *param_val = NULL; pthread_mutex_lock(&ofi_ini_lock); if (ofi_init) goto unlock; fi_param_init(); fi_log_init(); fi_util_init(); ofi_osd_init(); fi_param_define(NULL, "provider", FI_PARAM_STRING, "Only use specified provider (default: all available)"); fi_param_define(NULL, "fork_unsafe", FI_PARAM_BOOL, "Whether use of fork() may be unsafe for some providers" " (default: no). Setting this to yes could improve" " performance at the expense of making fork() potentially" " unsafe"); fi_param_get_str(NULL, "provider", ¶m_val); fi_create_filter(&prov_filter, param_val); #ifdef HAVE_LIBDL int n = 0; char **dirs; char *provdir = NULL; void *dlhandle; /* If dlopen fails, assume static linking and just return without error */ dlhandle = dlopen(NULL, RTLD_NOW); if (dlhandle == NULL) { goto libdl_done; } dlclose(dlhandle); fi_param_define(NULL, "provider_path", FI_PARAM_STRING, "Search for providers in specific path (default: " PROVDLDIR ")"); fi_param_get_str(NULL, "provider_path", &provdir); if (!provdir) provdir = PROVDLDIR; dirs = split_and_alloc(provdir, ":"); if (dirs) { for (n = 0; dirs[n]; ++n) { fi_ini_dir(dirs[n]); } free_string_array(dirs); } libdl_done: #endif fi_register_provider(PSM2_INIT, NULL); fi_register_provider(PSM_INIT, NULL); fi_register_provider(USNIC_INIT, NULL); fi_register_provider(MLX_INIT, NULL); fi_register_provider(VERBS_INIT, NULL); fi_register_provider(GNI_INIT, NULL); fi_register_provider(RXM_INIT, NULL); fi_register_provider(BGQ_INIT, NULL); /* Initialize the socket(s) provider last. This will result in it being the least preferred provider. */ fi_register_provider(UDP_INIT, NULL); fi_register_provider(SOCKETS_INIT, NULL); /* Before you add ANYTHING here, read the comment above!!! */ /* Seriously, read it! */ ofi_init = 1; unlock: pthread_mutex_unlock(&ofi_ini_lock); }
static int fi_ibv_rdm_find_sysaddrs(struct fi_ibv_rdm_sysaddr *iface_addr, struct fi_ibv_rdm_sysaddr *lo_addr) { struct ifaddrs *ifaddr, *ifa; char iface[IFNAMSIZ]; char *iface_tmp = "ib"; size_t iface_len = 2; int ret; if (!iface_addr || !lo_addr) { return -FI_EINVAL; } iface_addr->is_found = 0; lo_addr->is_found = 0; if (fi_param_get_str(&fi_ibv_prov, "iface", &iface_tmp) == FI_SUCCESS) { iface_len = strlen(iface_tmp); if (iface_len > IFNAMSIZ) { VERBS_INFO(FI_LOG_EP_CTRL, "Too long iface name: %s, max: %d\n", iface_tmp, IFNAMSIZ); return -FI_EINVAL; } } strncpy(iface, iface_tmp, iface_len); ret = getifaddrs(&ifaddr); if (ret) { FI_WARN(&fi_ibv_prov, FI_LOG_FABRIC, "Unable to get interface addresses\n"); return ret; } for (ifa = ifaddr; ifa; ifa = ifa->ifa_next) { if (!iface_addr->is_found && (ifa->ifa_addr->sa_family == AF_INET) && !strncmp(ifa->ifa_name, iface, iface_len)) { memcpy(&iface_addr->addr, ifa->ifa_addr, sizeof(iface_addr->addr)); iface_addr->is_found = 1; FI_INFO(&fi_ibv_prov, FI_LOG_FABRIC, "iface addr %s:%u\n", inet_ntoa(iface_addr->addr.sin_addr), ntohs(iface_addr->addr.sin_port)); } if (!lo_addr->is_found && (ifa->ifa_addr->sa_family == AF_INET) && !strncmp(ifa->ifa_name, "lo", strlen(ifa->ifa_name))) { memcpy(&lo_addr->addr, ifa->ifa_addr, sizeof(lo_addr->addr)); lo_addr->is_found = 1; FI_INFO(&fi_ibv_prov, FI_LOG_FABRIC, "lo addr %s:%u\n", inet_ntoa(lo_addr->addr.sin_addr), ntohs(lo_addr->addr.sin_port)); } if (iface_addr->is_found && lo_addr->is_found) { break; } } freeifaddrs(ifaddr); return 0; }
void fi_ini(void) { char *param_val = NULL; pthread_mutex_lock(&common_locks.ini_lock); if (ofi_init) goto unlock; ofi_ordered_provs_init(); fi_param_init(); fi_log_init(); ofi_osd_init(); ofi_pmem_init(); ofi_perf_init(); ofi_hook_init(); fi_param_define(NULL, "provider", FI_PARAM_STRING, "Only use specified provider (default: all available)"); fi_param_define(NULL, "fork_unsafe", FI_PARAM_BOOL, "Whether use of fork() may be unsafe for some providers" " (default: no). Setting this to yes could improve" " performance at the expense of making fork() potentially" " unsafe"); fi_param_define(NULL, "universe_size", FI_PARAM_SIZE_T, "Defines the maximum number of processes that will be" " used by distribute OFI application. The provider uses" " this to optimize resource allocations" " (default: OFI service specific)"); fi_param_get_str(NULL, "provider", ¶m_val); ofi_create_filter(&prov_filter, param_val); #ifdef HAVE_LIBDL int n = 0; char **dirs; char *provdir = NULL; void *dlhandle; /* If dlopen fails, assume static linking and just return without error */ dlhandle = dlopen(NULL, RTLD_NOW); if (dlhandle == NULL) { goto libdl_done; } dlclose(dlhandle); fi_param_define(NULL, "provider_path", FI_PARAM_STRING, "Search for providers in specific path (default: " PROVDLDIR ")"); fi_param_get_str(NULL, "provider_path", &provdir); if (!provdir) provdir = PROVDLDIR; dirs = ofi_split_and_alloc(provdir, ":", NULL); if (dirs) { for (n = 0; dirs[n]; ++n) { ofi_ini_dir(dirs[n]); } ofi_free_string_array(dirs); } libdl_done: #endif ofi_register_provider(PSM2_INIT, NULL); ofi_register_provider(PSM_INIT, NULL); ofi_register_provider(USNIC_INIT, NULL); ofi_register_provider(MLX_INIT, NULL); ofi_register_provider(GNI_INIT, NULL); ofi_register_provider(BGQ_INIT, NULL); ofi_register_provider(NETDIR_INIT, NULL); ofi_register_provider(SHM_INIT, NULL); ofi_register_provider(RXM_INIT, NULL); ofi_register_provider(VERBS_INIT, NULL); //ofi_register_provider(MRAIL_INIT, NULL); ofi_register_provider(RSTREAM_INIT, NULL); { /* TODO: RXD is not stable for now. Disable it by default */ int enable_rxd = 0; fi_param_define(NULL, "rxd_enable", FI_PARAM_BOOL, "Enable RXD provider (default: no)"); fi_param_get_bool(NULL, "rxd_enable", &enable_rxd); if (enable_rxd) ofi_register_provider(RXD_INIT, NULL); } ofi_register_provider(UDP_INIT, NULL); ofi_register_provider(SOCKETS_INIT, NULL); ofi_register_provider(TCP_INIT, NULL); ofi_init = 1; unlock: pthread_mutex_unlock(&common_locks.ini_lock); }
/* Builds a list of interfaces that correspond to active verbs devices */ static int fi_ibv_getifaddrs(struct dlist_entry *verbs_devs) { struct ifaddrs *ifaddr, *ifa; char name[INET6_ADDRSTRLEN]; struct rdma_addrinfo *rai; struct rdma_cm_id *id; const char *ret_ptr; int ret, num_verbs_ifs = 0; char *iface = NULL; size_t iface_len = 0; int exact_match = 0; ret = getifaddrs(&ifaddr); if (ret) { VERBS_WARN(FI_LOG_FABRIC, "Unable to get interface addresses\n"); return ret; } /* select best iface name based on user's input */ if (fi_param_get_str(&fi_ibv_prov, "iface", &iface) == FI_SUCCESS) { iface_len = strlen(iface); if (iface_len > IFNAMSIZ) { VERBS_INFO(FI_LOG_EP_CTRL, "Too long iface name: %s, max: %d\n", iface, IFNAMSIZ); return -FI_EINVAL; } for (ifa = ifaddr; ifa && !exact_match; ifa = ifa->ifa_next) exact_match = !strcmp(ifa->ifa_name, iface); } for (ifa = ifaddr; ifa; ifa = ifa->ifa_next) { if (!ifa->ifa_addr || !(ifa->ifa_flags & IFF_UP) || !strcmp(ifa->ifa_name, "lo")) continue; if(iface) { if(exact_match) { if(strcmp(ifa->ifa_name, iface)) continue; } else { if(strncmp(ifa->ifa_name, iface, iface_len)) continue; } } switch (ifa->ifa_addr->sa_family) { case AF_INET: ret_ptr = inet_ntop(AF_INET, &ofi_sin_addr(ifa->ifa_addr), name, INET6_ADDRSTRLEN); break; case AF_INET6: ret_ptr = inet_ntop(AF_INET6, &ofi_sin6_addr(ifa->ifa_addr), name, INET6_ADDRSTRLEN); break; default: continue; } if (!ret_ptr) { VERBS_WARN(FI_LOG_FABRIC, "inet_ntop failed: %s(%d)\n", strerror(errno), errno); goto err1; } ret = fi_ibv_create_ep(name, NULL, FI_NUMERICHOST | FI_SOURCE, NULL, &rai, &id); if (ret) continue; ret = fi_ibv_add_rai(verbs_devs, id, rai); if (ret) goto err2; VERBS_DBG(FI_LOG_FABRIC, "Found active interface for verbs device: " "%s with address: %s\n", ibv_get_device_name(id->verbs->device), name); rdma_destroy_ep(id); num_verbs_ifs++; } freeifaddrs(ifaddr); return num_verbs_ifs ? 0 : -FI_ENODATA; err2: rdma_destroy_ep(id); err1: fi_ibv_verbs_devs_free(verbs_devs); freeifaddrs(ifaddr); return ret; }