void fi_log_init(void)
{
struct fi_filter subsys_filter;
int level, i;
char *levelstr = NULL, *provstr = NULL, *subsysstr = NULL;
fi_param_define(NULL, "log_level", FI_PARAM_STRING,
"Specify logging level: warn, trace, info, debug (default: warn)");
fi_param_get_str(NULL, "log_level", &levelstr);
level = fi_convert_log_str(levelstr);
if (level >= 0)
log_mask = ((1 << (level + 1)) - 1);
fi_param_define(NULL, "log_prov", FI_PARAM_STRING,
"Specify specific provider to log (default: all)");
fi_param_get_str(NULL, "log_prov", &provstr);
ofi_create_filter(&prov_log_filter, provstr);
fi_param_define(NULL, "log_subsys", FI_PARAM_STRING,
"Specify specific subsystem to log (default: all)");
fi_param_get_str(NULL, "log_subsys", &subsysstr);
ofi_create_filter(&subsys_filter, subsysstr);
for (i = 0; i < FI_LOG_SUBSYS_MAX; i++) {
if (!ofi_apply_filter(&subsys_filter, log_subsys[i]))
log_mask |= (1 << (i + FI_LOG_SUBSYS_OFFSET));
}
ofi_free_filter(&subsys_filter);
}
static void psmx2_init_env(void)
{
if (getenv("OMPI_COMM_WORLD_RANK") || getenv("PMI_RANK"))
psmx2_env.name_server = 0;
fi_param_get_bool(&psmx2_prov, "name_server", &psmx2_env.name_server);
fi_param_get_bool(&psmx2_prov, "tagged_rma", &psmx2_env.tagged_rma);
fi_param_get_str(&psmx2_prov, "uuid", &psmx2_env.uuid);
fi_param_get_int(&psmx2_prov, "delay", &psmx2_env.delay);
fi_param_get_int(&psmx2_prov, "timeout", &psmx2_env.timeout);
fi_param_get_int(&psmx2_prov, "prog_interval", &psmx2_env.prog_interval);
fi_param_get_str(&psmx2_prov, "prog_affinity", &psmx2_env.prog_affinity);
}
void ofi_hook_init(void)
{
char *param_val = NULL;
fi_param_define(NULL, "hook", FI_PARAM_STRING,
"Intercept calls to underlying provider and apply "
"the specified functionality to them. Hook option: "
"perf (gather performance data)");
fi_param_get_str(NULL, "hook", ¶m_val);
hooks_enabled = 0;
if (!param_val)
return;
if (!strcasecmp(param_val, "noop")) {
FI_INFO(&core_prov, FI_LOG_CORE, "Noop hook requested\n");
hooks_enabled |= (1 << HOOK_NOOP);
}
if (!strcasecmp(param_val, "perf")) {
FI_INFO(&core_prov, FI_LOG_CORE, "Perf hook requested\n");
hooks_enabled |= (1 << HOOK_PERF);
}
}
void fi_ini(void)
{
char *param_val = NULL;
pthread_mutex_lock(&ofi_ini_lock);
if (ofi_init)
goto unlock;
fi_param_init();
fi_log_init();
fi_util_init();
ofi_osd_init();
fi_param_define(NULL, "provider", FI_PARAM_STRING,
"Only use specified provider (default: all available)");
fi_param_define(NULL, "fork_unsafe", FI_PARAM_BOOL,
"Whether use of fork() may be unsafe for some providers"
" (default: no). Setting this to yes could improve"
" performance at the expense of making fork() potentially"
" unsafe");
fi_param_get_str(NULL, "provider", ¶m_val);
fi_create_filter(&prov_filter, param_val);
#ifdef HAVE_LIBDL
int n = 0;
char **dirs;
char *provdir = NULL;
void *dlhandle;
/* If dlopen fails, assume static linking and just return
without error */
dlhandle = dlopen(NULL, RTLD_NOW);
if (dlhandle == NULL) {
goto libdl_done;
}
dlclose(dlhandle);
fi_param_define(NULL, "provider_path", FI_PARAM_STRING,
"Search for providers in specific path (default: "
PROVDLDIR ")");
fi_param_get_str(NULL, "provider_path", &provdir);
if (!provdir)
provdir = PROVDLDIR;
dirs = split_and_alloc(provdir, ":");
if (dirs) {
for (n = 0; dirs[n]; ++n) {
fi_ini_dir(dirs[n]);
}
free_string_array(dirs);
}
libdl_done:
#endif
fi_register_provider(PSM2_INIT, NULL);
fi_register_provider(PSM_INIT, NULL);
fi_register_provider(USNIC_INIT, NULL);
fi_register_provider(MLX_INIT, NULL);
fi_register_provider(VERBS_INIT, NULL);
fi_register_provider(GNI_INIT, NULL);
fi_register_provider(RXM_INIT, NULL);
fi_register_provider(BGQ_INIT, NULL);
/* Initialize the socket(s) provider last. This will result in
it being the least preferred provider. */
fi_register_provider(UDP_INIT, NULL);
fi_register_provider(SOCKETS_INIT, NULL);
/* Before you add ANYTHING here, read the comment above!!! */
/* Seriously, read it! */
ofi_init = 1;
unlock:
pthread_mutex_unlock(&ofi_ini_lock);
}
static int
fi_ibv_rdm_find_sysaddrs(struct fi_ibv_rdm_sysaddr *iface_addr,
struct fi_ibv_rdm_sysaddr *lo_addr)
{
struct ifaddrs *ifaddr, *ifa;
char iface[IFNAMSIZ];
char *iface_tmp = "ib";
size_t iface_len = 2;
int ret;
if (!iface_addr || !lo_addr) {
return -FI_EINVAL;
}
iface_addr->is_found = 0;
lo_addr->is_found = 0;
if (fi_param_get_str(&fi_ibv_prov, "iface", &iface_tmp) == FI_SUCCESS) {
iface_len = strlen(iface_tmp);
if (iface_len > IFNAMSIZ) {
VERBS_INFO(FI_LOG_EP_CTRL,
"Too long iface name: %s, max: %d\n",
iface_tmp, IFNAMSIZ);
return -FI_EINVAL;
}
}
strncpy(iface, iface_tmp, iface_len);
ret = getifaddrs(&ifaddr);
if (ret) {
FI_WARN(&fi_ibv_prov, FI_LOG_FABRIC,
"Unable to get interface addresses\n");
return ret;
}
for (ifa = ifaddr; ifa; ifa = ifa->ifa_next) {
if (!iface_addr->is_found && (ifa->ifa_addr->sa_family == AF_INET) &&
!strncmp(ifa->ifa_name, iface, iface_len)) {
memcpy(&iface_addr->addr, ifa->ifa_addr,
sizeof(iface_addr->addr));
iface_addr->is_found = 1;
FI_INFO(&fi_ibv_prov, FI_LOG_FABRIC,
"iface addr %s:%u\n",
inet_ntoa(iface_addr->addr.sin_addr),
ntohs(iface_addr->addr.sin_port));
}
if (!lo_addr->is_found && (ifa->ifa_addr->sa_family == AF_INET) &&
!strncmp(ifa->ifa_name, "lo", strlen(ifa->ifa_name))) {
memcpy(&lo_addr->addr, ifa->ifa_addr, sizeof(lo_addr->addr));
lo_addr->is_found = 1;
FI_INFO(&fi_ibv_prov, FI_LOG_FABRIC, "lo addr %s:%u\n",
inet_ntoa(lo_addr->addr.sin_addr),
ntohs(lo_addr->addr.sin_port));
}
if (iface_addr->is_found && lo_addr->is_found) {
break;
}
}
freeifaddrs(ifaddr);
return 0;
}
void fi_ini(void)
{
char *param_val = NULL;
pthread_mutex_lock(&common_locks.ini_lock);
if (ofi_init)
goto unlock;
ofi_ordered_provs_init();
fi_param_init();
fi_log_init();
ofi_osd_init();
ofi_pmem_init();
ofi_perf_init();
ofi_hook_init();
fi_param_define(NULL, "provider", FI_PARAM_STRING,
"Only use specified provider (default: all available)");
fi_param_define(NULL, "fork_unsafe", FI_PARAM_BOOL,
"Whether use of fork() may be unsafe for some providers"
" (default: no). Setting this to yes could improve"
" performance at the expense of making fork() potentially"
" unsafe");
fi_param_define(NULL, "universe_size", FI_PARAM_SIZE_T,
"Defines the maximum number of processes that will be"
" used by distribute OFI application. The provider uses"
" this to optimize resource allocations"
" (default: OFI service specific)");
fi_param_get_str(NULL, "provider", ¶m_val);
ofi_create_filter(&prov_filter, param_val);
#ifdef HAVE_LIBDL
int n = 0;
char **dirs;
char *provdir = NULL;
void *dlhandle;
/* If dlopen fails, assume static linking and just return
without error */
dlhandle = dlopen(NULL, RTLD_NOW);
if (dlhandle == NULL) {
goto libdl_done;
}
dlclose(dlhandle);
fi_param_define(NULL, "provider_path", FI_PARAM_STRING,
"Search for providers in specific path (default: "
PROVDLDIR ")");
fi_param_get_str(NULL, "provider_path", &provdir);
if (!provdir)
provdir = PROVDLDIR;
dirs = ofi_split_and_alloc(provdir, ":", NULL);
if (dirs) {
for (n = 0; dirs[n]; ++n) {
ofi_ini_dir(dirs[n]);
}
ofi_free_string_array(dirs);
}
libdl_done:
#endif
ofi_register_provider(PSM2_INIT, NULL);
ofi_register_provider(PSM_INIT, NULL);
ofi_register_provider(USNIC_INIT, NULL);
ofi_register_provider(MLX_INIT, NULL);
ofi_register_provider(GNI_INIT, NULL);
ofi_register_provider(BGQ_INIT, NULL);
ofi_register_provider(NETDIR_INIT, NULL);
ofi_register_provider(SHM_INIT, NULL);
ofi_register_provider(RXM_INIT, NULL);
ofi_register_provider(VERBS_INIT, NULL);
//ofi_register_provider(MRAIL_INIT, NULL);
ofi_register_provider(RSTREAM_INIT, NULL);
{
/* TODO: RXD is not stable for now. Disable it by default */
int enable_rxd = 0;
fi_param_define(NULL, "rxd_enable", FI_PARAM_BOOL,
"Enable RXD provider (default: no)");
fi_param_get_bool(NULL, "rxd_enable", &enable_rxd);
if (enable_rxd)
ofi_register_provider(RXD_INIT, NULL);
}
ofi_register_provider(UDP_INIT, NULL);
ofi_register_provider(SOCKETS_INIT, NULL);
ofi_register_provider(TCP_INIT, NULL);
ofi_init = 1;
unlock:
pthread_mutex_unlock(&common_locks.ini_lock);
}
/* Builds a list of interfaces that correspond to active verbs devices */
static int fi_ibv_getifaddrs(struct dlist_entry *verbs_devs)
{
struct ifaddrs *ifaddr, *ifa;
char name[INET6_ADDRSTRLEN];
struct rdma_addrinfo *rai;
struct rdma_cm_id *id;
const char *ret_ptr;
int ret, num_verbs_ifs = 0;
char *iface = NULL;
size_t iface_len = 0;
int exact_match = 0;
ret = getifaddrs(&ifaddr);
if (ret) {
VERBS_WARN(FI_LOG_FABRIC,
"Unable to get interface addresses\n");
return ret;
}
/* select best iface name based on user's input */
if (fi_param_get_str(&fi_ibv_prov, "iface", &iface) == FI_SUCCESS) {
iface_len = strlen(iface);
if (iface_len > IFNAMSIZ) {
VERBS_INFO(FI_LOG_EP_CTRL,
"Too long iface name: %s, max: %d\n",
iface, IFNAMSIZ);
return -FI_EINVAL;
}
for (ifa = ifaddr; ifa && !exact_match; ifa = ifa->ifa_next)
exact_match = !strcmp(ifa->ifa_name, iface);
}
for (ifa = ifaddr; ifa; ifa = ifa->ifa_next) {
if (!ifa->ifa_addr || !(ifa->ifa_flags & IFF_UP) ||
!strcmp(ifa->ifa_name, "lo"))
continue;
if(iface) {
if(exact_match) {
if(strcmp(ifa->ifa_name, iface))
continue;
} else {
if(strncmp(ifa->ifa_name, iface, iface_len))
continue;
}
}
switch (ifa->ifa_addr->sa_family) {
case AF_INET:
ret_ptr = inet_ntop(AF_INET, &ofi_sin_addr(ifa->ifa_addr),
name, INET6_ADDRSTRLEN);
break;
case AF_INET6:
ret_ptr = inet_ntop(AF_INET6, &ofi_sin6_addr(ifa->ifa_addr),
name, INET6_ADDRSTRLEN);
break;
default:
continue;
}
if (!ret_ptr) {
VERBS_WARN(FI_LOG_FABRIC,
"inet_ntop failed: %s(%d)\n",
strerror(errno), errno);
goto err1;
}
ret = fi_ibv_create_ep(name, NULL, FI_NUMERICHOST | FI_SOURCE,
NULL, &rai, &id);
if (ret)
continue;
ret = fi_ibv_add_rai(verbs_devs, id, rai);
if (ret)
goto err2;
VERBS_DBG(FI_LOG_FABRIC, "Found active interface for verbs device: "
"%s with address: %s\n",
ibv_get_device_name(id->verbs->device), name);
rdma_destroy_ep(id);
num_verbs_ifs++;
}
freeifaddrs(ifaddr);
return num_verbs_ifs ? 0 : -FI_ENODATA;
err2:
rdma_destroy_ep(id);
err1:
fi_ibv_verbs_devs_free(verbs_devs);
freeifaddrs(ifaddr);
return ret;
}
