void fi_log_init(void)
{
struct fi_filter subsys_filter;
int level, i;
char *levelstr = NULL, *provstr = NULL, *subsysstr = NULL;
fi_param_define(NULL, "log_level", FI_PARAM_STRING,
"Specify logging level: warn, trace, info, debug (default: warn)");
fi_param_get_str(NULL, "log_level", &levelstr);
level = fi_convert_log_str(levelstr);
if (level >= 0)
log_mask = ((1 << (level + 1)) - 1);
fi_param_define(NULL, "log_prov", FI_PARAM_STRING,
"Specify specific provider to log (default: all)");
fi_param_get_str(NULL, "log_prov", &provstr);
ofi_create_filter(&prov_log_filter, provstr);
fi_param_define(NULL, "log_subsys", FI_PARAM_STRING,
"Specify specific subsystem to log (default: all)");
fi_param_get_str(NULL, "log_subsys", &subsysstr);
ofi_create_filter(&subsys_filter, subsysstr);
for (i = 0; i < FI_LOG_SUBSYS_MAX; i++) {
if (!ofi_apply_filter(&subsys_filter, log_subsys[i]))
log_mask |= (1 << (i + FI_LOG_SUBSYS_OFFSET));
}
ofi_free_filter(&subsys_filter);
}
/*
* Initialize all available memory monitors
*/
void ofi_monitor_init(void)
{
fastlock_init(&uffd_monitor->lock);
dlist_init(&uffd_monitor->list);
fi_param_define(NULL, "mr_cache_max_size", FI_PARAM_SIZE_T,
"Defines the total number of bytes for all memory"
" regions that may be tracked by the MR cache."
" Setting this will reduce the amount of memory"
" not actively in use that may be registered."
" (default: 0 no limit is enforced)");
fi_param_define(NULL, "mr_cache_max_count", FI_PARAM_SIZE_T,
"Defines the total number of memory regions that"
" may be store in the cache. Setting this will"
" reduce the number of registered regions, regardless"
" of their size, stored in the cache. Setting this"
" to zero will disable MR caching. (default: 1024)");
fi_param_define(NULL, "mr_cache_merge_regions", FI_PARAM_BOOL,
"If set to true, overlapping or adjacent memory"
" regions will be combined into a single, larger"
" region. Merging regions can reduce the cache"
" memory footprint, but can negatively impact"
" performance in some situations. (default: false)");
fi_param_get_size_t(NULL, "mr_cache_max_size", &cache_params.max_size);
fi_param_get_size_t(NULL, "mr_cache_max_count", &cache_params.max_cnt);
fi_param_get_bool(NULL, "mr_cache_merge_regions",
&cache_params.merge_regions);
if (!cache_params.max_size)
cache_params.max_size = SIZE_MAX;
}
static int efa_get_param_size_t(char *param_name, char *param_str,
size_t *param_default)
{
size_t len, ret_len, param;
int ret = FI_SUCCESS;
char *param_help;
len = strlen(param_str) + 50;
param_help = calloc(1, len);
if (!param_help)
return -FI_ENOMEM;
ret_len = snprintf(param_help, len, "%s", param_str);
if (ret_len >= len) {
EFA_WARN(FI_LOG_EP_DATA,
"param_help string size insufficient!\n");
assert(0);
ret = -FI_ETOOSMALL;
goto out;
}
fi_param_define(&efa_prov, param_name, FI_PARAM_INT, param_help);
if (!fi_param_get_size_t(&efa_prov, param_name, ¶m))
*param_default = param;
out:
free(param_help);
return ret;
}
static int fi_ibv_get_param_int(char *param_name, char *param_str,
size_t *param_default)
{
char *param_help;
size_t len, ret;
int param;
len = strlen(param_str) + 50;
param_help = malloc(len);
ret = snprintf(param_help, len, "%s (default: %zu)", param_str,
*param_default);
if (ret >= len) {
VERBS_WARN(FI_LOG_EP_DATA,
"param_help string size insufficient!\n");
free(param_help);
assert(0);
return -FI_ETOOSMALL;
}
fi_param_define(&fi_ibv_prov, param_name, FI_PARAM_INT, param_help);
if (!fi_param_get_int(&fi_ibv_prov, param_name, ¶m))
*param_default = param;
free(param_help);
return 0;
}
void ofi_hook_init(void)
{
char *param_val = NULL;
fi_param_define(NULL, "hook", FI_PARAM_STRING,
"Intercept calls to underlying provider and apply "
"the specified functionality to them. Hook option: "
"perf (gather performance data)");
fi_param_get_str(NULL, "hook", ¶m_val);
hooks_enabled = 0;
if (!param_val)
return;
if (!strcasecmp(param_val, "noop")) {
FI_INFO(&core_prov, FI_LOG_CORE, "Noop hook requested\n");
hooks_enabled |= (1 << HOOK_NOOP);
}
if (!strcasecmp(param_val, "perf")) {
FI_INFO(&core_prov, FI_LOG_CORE, "Perf hook requested\n");
hooks_enabled |= (1 << HOOK_PERF);
}
}
void fi_ini(void)
{
char *param_val = NULL;
pthread_mutex_lock(&ofi_ini_lock);
if (ofi_init)
goto unlock;
fi_param_init();
fi_log_init();
fi_util_init();
ofi_osd_init();
fi_param_define(NULL, "provider", FI_PARAM_STRING,
"Only use specified provider (default: all available)");
fi_param_define(NULL, "fork_unsafe", FI_PARAM_BOOL,
"Whether use of fork() may be unsafe for some providers"
" (default: no). Setting this to yes could improve"
" performance at the expense of making fork() potentially"
" unsafe");
fi_param_get_str(NULL, "provider", ¶m_val);
fi_create_filter(&prov_filter, param_val);
#ifdef HAVE_LIBDL
int n = 0;
char **dirs;
char *provdir = NULL;
void *dlhandle;
/* If dlopen fails, assume static linking and just return
without error */
dlhandle = dlopen(NULL, RTLD_NOW);
if (dlhandle == NULL) {
goto libdl_done;
}
dlclose(dlhandle);
fi_param_define(NULL, "provider_path", FI_PARAM_STRING,
"Search for providers in specific path (default: "
PROVDLDIR ")");
fi_param_get_str(NULL, "provider_path", &provdir);
if (!provdir)
provdir = PROVDLDIR;
dirs = split_and_alloc(provdir, ":");
if (dirs) {
for (n = 0; dirs[n]; ++n) {
fi_ini_dir(dirs[n]);
}
free_string_array(dirs);
}
libdl_done:
#endif
fi_register_provider(PSM2_INIT, NULL);
fi_register_provider(PSM_INIT, NULL);
fi_register_provider(USNIC_INIT, NULL);
fi_register_provider(MLX_INIT, NULL);
fi_register_provider(VERBS_INIT, NULL);
fi_register_provider(GNI_INIT, NULL);
fi_register_provider(RXM_INIT, NULL);
fi_register_provider(BGQ_INIT, NULL);
/* Initialize the socket(s) provider last. This will result in
it being the least preferred provider. */
fi_register_provider(UDP_INIT, NULL);
fi_register_provider(SOCKETS_INIT, NULL);
/* Before you add ANYTHING here, read the comment above!!! */
/* Seriously, read it! */
ofi_init = 1;
unlock:
pthread_mutex_unlock(&ofi_ini_lock);
}
void fi_ini(void)
{
char *param_val = NULL;
pthread_mutex_lock(&common_locks.ini_lock);
if (ofi_init)
goto unlock;
ofi_ordered_provs_init();
fi_param_init();
fi_log_init();
ofi_osd_init();
ofi_pmem_init();
ofi_perf_init();
ofi_hook_init();
fi_param_define(NULL, "provider", FI_PARAM_STRING,
"Only use specified provider (default: all available)");
fi_param_define(NULL, "fork_unsafe", FI_PARAM_BOOL,
"Whether use of fork() may be unsafe for some providers"
" (default: no). Setting this to yes could improve"
" performance at the expense of making fork() potentially"
" unsafe");
fi_param_define(NULL, "universe_size", FI_PARAM_SIZE_T,
"Defines the maximum number of processes that will be"
" used by distribute OFI application. The provider uses"
" this to optimize resource allocations"
" (default: OFI service specific)");
fi_param_get_str(NULL, "provider", ¶m_val);
ofi_create_filter(&prov_filter, param_val);
#ifdef HAVE_LIBDL
int n = 0;
char **dirs;
char *provdir = NULL;
void *dlhandle;
/* If dlopen fails, assume static linking and just return
without error */
dlhandle = dlopen(NULL, RTLD_NOW);
if (dlhandle == NULL) {
goto libdl_done;
}
dlclose(dlhandle);
fi_param_define(NULL, "provider_path", FI_PARAM_STRING,
"Search for providers in specific path (default: "
PROVDLDIR ")");
fi_param_get_str(NULL, "provider_path", &provdir);
if (!provdir)
provdir = PROVDLDIR;
dirs = ofi_split_and_alloc(provdir, ":", NULL);
if (dirs) {
for (n = 0; dirs[n]; ++n) {
ofi_ini_dir(dirs[n]);
}
ofi_free_string_array(dirs);
}
libdl_done:
#endif
ofi_register_provider(PSM2_INIT, NULL);
ofi_register_provider(PSM_INIT, NULL);
ofi_register_provider(USNIC_INIT, NULL);
ofi_register_provider(MLX_INIT, NULL);
ofi_register_provider(GNI_INIT, NULL);
ofi_register_provider(BGQ_INIT, NULL);
ofi_register_provider(NETDIR_INIT, NULL);
ofi_register_provider(SHM_INIT, NULL);
ofi_register_provider(RXM_INIT, NULL);
ofi_register_provider(VERBS_INIT, NULL);
//ofi_register_provider(MRAIL_INIT, NULL);
ofi_register_provider(RSTREAM_INIT, NULL);
{
/* TODO: RXD is not stable for now. Disable it by default */
int enable_rxd = 0;
fi_param_define(NULL, "rxd_enable", FI_PARAM_BOOL,
"Enable RXD provider (default: no)");
fi_param_get_bool(NULL, "rxd_enable", &enable_rxd);
if (enable_rxd)
ofi_register_provider(RXD_INIT, NULL);
}
ofi_register_provider(UDP_INIT, NULL);
ofi_register_provider(SOCKETS_INIT, NULL);
ofi_register_provider(TCP_INIT, NULL);
ofi_init = 1;
unlock:
pthread_mutex_unlock(&common_locks.ini_lock);
}