/*
* Main MCA initialization.
*/
int mca_base_open(void)
{
int param_index;
char *value;
opal_output_stream_t lds;
char hostname[64];
if (!mca_base_opened) {
mca_base_opened = true;
} else {
return OPAL_SUCCESS;
}
/* Register some params */
#if OMPI_WANT_HOME_CONFIG_FILES
asprintf(&value, "%s%c%s"OPAL_PATH_SEP".openmpi"OPAL_PATH_SEP"components", opal_install_dirs.pkglibdir, OPAL_ENV_SEP, opal_home_directory() );
#else
# if defined(__WINDOWS__) && defined(_DEBUG)
asprintf(&value, "%s/debug", opal_install_dirs.pkglibdir);
# else
asprintf(&value, "%s", opal_install_dirs.pkglibdir);
# endif
#endif
mca_base_param_component_path =
mca_base_param_reg_string_name("mca", "component_path",
"Path where to look for Open MPI and ORTE components",
false, false, value, NULL);
free(value);
param_index = mca_base_param_reg_string_name("mca", "verbose",
"Top-level verbosity parameter",
false, false, NULL, NULL);
mca_base_param_reg_int_name("mca", "component_show_load_errors",
"Whether to show errors for components that failed to load or not",
false, false, 1, NULL);
mca_base_param_reg_int_name("mca", "component_disable_dlopen",
"Whether to attempt to disable opening dynamic components or not",
false, false, 0, NULL);
/* What verbosity level do we want? */
mca_base_param_lookup_string(param_index, &value);
memset(&lds, 0, sizeof(lds));
if (NULL != value) {
parse_verbose(value, &lds);
free(value);
} else {
set_defaults(&lds);
}
gethostname(hostname, 64);
asprintf(&lds.lds_prefix, "[%s:%05d] ", hostname, getpid());
opal_output_reopen(0, &lds);
opal_output_verbose(5, 0, "mca: base: opening components");
free(lds.lds_prefix);
/* Open up the component repository */
return mca_base_component_repository_init();
}
/**
* Function for finding and opening either all MCA components, or the one
* that was specifically requested via a MCA parameter.
*/
int mca_btl_base_open(void)
{
int i;
if( ++mca_btl_base_already_opened > 1 ) return OMPI_SUCCESS;
/* Verbose output */
mca_base_param_reg_int_name("btl",
"base_verbose",
"Verbosity level of the BTL framework",
false, false,
0,
&mca_btl_base_verbose);
mca_btl_base_output = opal_output_open(NULL);
opal_output_set_verbosity(mca_btl_base_output, mca_btl_base_verbose);
/* Override the per-BTL "don't run if THREAD_MULTIPLE selected"
embargo? */
mca_base_param_reg_int_name("btl",
"base_thread_multiple_override",
"Enable BTLs that are not normally enabled when MPI_THREAD_MULTIPLE is enabled (THIS IS FOR DEVELOPERS ONLY! SHOULD NOT BE USED BY END USERS!)",
true, false,
0, &i);
mca_btl_base_thread_multiple_override = OPAL_INT_TO_BOOL(i);
/* Open up all available components */
if (OMPI_SUCCESS !=
mca_base_components_open("btl", mca_btl_base_output, mca_btl_base_static_components,
&mca_btl_base_components_opened, true)) {
return OMPI_ERROR;
}
/* Initialize the list so that in mca_btl_base_close(), we can
iterate over it (even if it's empty, as in the case of
ompi_info) */
OBJ_CONSTRUCT(&mca_btl_base_modules_initialized, opal_list_t);
/* register parameters */
mca_base_param_lookup_string(
mca_base_param_register_string("btl","base","include",NULL,NULL), &mca_btl_base_include);
mca_base_param_lookup_string(
mca_base_param_register_string("btl","base","exclude",NULL,NULL), &mca_btl_base_exclude);
mca_base_param_reg_int_name("btl", "base_warn_component_unused",
"This parameter is used to turn on warning messages when certain NICs are not used",
false, false, 1, &mca_btl_base_warn_component_unused);
/* All done */
return OMPI_SUCCESS;
}
/*
* Register some paffinity-wide MCA params
*/
int opal_paffinity_base_register_params(void)
{
int value, id;
static int been_here = 0;
/* We may get called twice; be harmless in that case. */
if (1 == been_here) {
return OPAL_SUCCESS;
}
been_here = 1;
/* Debugging / verbose output */
mca_base_param_reg_int_name("paffinity", "base_verbose",
"Verbosity level of the paffinity framework",
false, false,
0, &value);
if (0 != value) {
opal_paffinity_base_output = opal_output_open(NULL);
} else {
opal_paffinity_base_output = -1;
}
id = mca_base_param_reg_int_name("opal", "paffinity_alone",
"If nonzero, assume that this job is the only (set of) process(es) running on each node and bind processes to processors, starting with processor ID 0",
false, false,
0, NULL);
/* register the historical mpi_paffinity_alone synonym, but don't
* declare it deprecated so we don't scare the users.
*
* Yes, this breaks the abstraction barrier, but as indicated
* on the developer list....live with it. :-)
*/
mca_base_param_reg_syn_name(id, "mpi", "paffinity_alone", false);
mca_base_param_lookup_int(id, &value);
opal_paffinity_alone = OPAL_INT_TO_BOOL(value);
mca_base_param_reg_int_name("paffinity", "base_bound",
"Process affinity was set by an external entity",
true, false,
false, &value);
opal_paffinity_base_bound = OPAL_INT_TO_BOOL(value);
mca_base_param_reg_string_name("paffinity", "base_applied_binding",
"Process affinity was set by an external entity",
true, false,
NULL, &opal_paffinity_base_applied_binding);
return OPAL_SUCCESS;
}
/**
* Initialization of data structures for running under a debugger
* using the MPICH/TotalView parallel debugger interface. Before the
* spawn we need to check if we are being run under a TotalView-like
* debugger; if so then inform applications via an MCA parameter.
*/
void orte_totalview_init_before_spawn(void)
{
if (MPIR_DEBUG_SPAWNED == MPIR_being_debugged) {
int value;
char *s;
if (orte_debug_flag) {
opal_output(0, "Info: Spawned by a debugger");
}
if (mca_base_param_reg_int_name("orte", "mpi_wait_for_totalview",
"Whether the MPI application should wait for a debugger or not",
false, false, (int)false, &value) < 0) {
opal_output(0, "Error: mca_base_param_reg_int_name\n");
}
/* push mca parameter into the environment (not done automatically?) */
s = mca_base_param_environ_variable("orte", "mpi_wait_for_totalview", NULL);
if (ORTE_SUCCESS != opal_setenv(s, "1", true, &environ)) {
opal_output(0, "Error: Can't setenv %s\n", s);
}
free(s);
}
}
/*
* Function for finding and opening either all MCA components, or the one
* that was specifically requested via a MCA parameter.
*/
int opal_carto_base_open(void)
{
int value;
/* Debugging / verbose output */
mca_base_param_reg_int_name("carto", "base_verbose",
"Verbosity level of the carto framework",
false, false,
0, &value);
if (0 != value) {
opal_carto_base_output = opal_output_open(NULL);
} else {
opal_carto_base_output = -1;
}
opal_carto_base_components_opened_valid = false;
/* Open up all available components */
if (OPAL_SUCCESS !=
mca_base_components_open("carto", opal_carto_base_output,
mca_carto_base_static_components,
&opal_carto_base_components_opened,
true)) {
return OPAL_ERROR;
}
opal_carto_base_components_opened_valid = true;
/* All done */
return OPAL_SUCCESS;
}
/*
* CR Init
*/
int ompi_cr_init(void)
{
int val;
/*
* Register some MCA parameters
*/
mca_base_param_reg_int_name("ompi_cr", "verbose",
"Verbose output for the OMPI Checkpoint/Restart functionality",
false, false,
0,
&val);
if(0 != val) {
ompi_cr_output = opal_output_open(NULL);
opal_output_set_verbosity(ompi_cr_output, val);
} else {
ompi_cr_output = opal_cr_output;
}
/* Typically this is not needed. Individual BTLs will set this as needed */
ompi_cr_continue_like_restart = false;
opal_output_verbose(10, ompi_cr_output,
"ompi_cr: init: ompi_cr_init()");
/* Register the OMPI interlevel coordination callback */
opal_cr_reg_coord_callback(ompi_cr_coord, &prev_coord_callback);
return OMPI_SUCCESS;
}
/**
* This function gets called once to check if the program is running in a cuda
* environment.
*/
static void opal_cuda_support_init(void)
{
int id;
CUresult res;
CUcontext cuContext;
if (initialized) {
return;
}
/* Set different levels of verbosity in the cuda related code. */
id = mca_base_param_reg_int_name("opal", "cuda_verbose",
"Set level of opal cuda verbosity",
false, false, 0, &opal_cuda_verbose);
opal_cuda_output = opal_output_open(NULL);
opal_output_set_verbosity(opal_cuda_output, opal_cuda_verbose);
/* Check to see if this process is running in a CUDA context. If so,
* all is good. Currently, just print out a message in verbose mode
* to help with debugging. */
res = cuCtxGetCurrent(&cuContext);
if (CUDA_SUCCESS != res) {
opal_output_verbose(10, opal_cuda_output,
"CUDA: cuCtxGetCurrent failed, CUDA device pointers will not work");
} else {
opal_output_verbose(10, opal_cuda_output,
"CUDA: cuCtxGetCurrent succeeded, CUDA device pointers will work");
}
initialized = true;
}
int opal_crs_base_select(void)
{
int ret, exit_status = OPAL_SUCCESS;
opal_crs_base_component_t *best_component = NULL;
opal_crs_base_module_t *best_module = NULL;
int int_value = 0;
/*
* Note: If we are a tool, then we will manually run the selection routine
* for the checkpointer. The tool will set the MCA parameter
* 'crs_base_do_not_select' before opal_init and then reset it after to
* disable the selection logic.
* This is useful for opal_restart because it reads the metadata file
* that indicates the checkpointer to be used after calling opal_init.
* Therefore it would need to select a specific module, but it doesn't
* know which one until later. It will set the MCA parameter 'crs'
* before calling this function.
*/
mca_base_param_reg_int_name("crs",
"base_do_not_select",
"Do not do the selection of the CRS component",
true, false,
false,
&int_value);
if( OPAL_INT_TO_BOOL(int_value) ) {
opal_output_verbose(10, opal_crs_base_output,
"crs:select: Not selecting at this time!");
return OPAL_SUCCESS;
}
/*
* Select the best component
*/
if( OPAL_SUCCESS != mca_base_select("crs", opal_crs_base_output,
&opal_crs_base_components_available,
(mca_base_module_t **) &best_module,
(mca_base_component_t **) &best_component) ) {
/* This will only happen if no component was selected */
exit_status = OPAL_ERROR;
goto cleanup;
}
/* Save the winner */
opal_crs_base_selected_component = *best_component;
opal_crs = *best_module;
/* Initialize the winner */
if (NULL != best_module) {
if (OPAL_SUCCESS != (ret = opal_crs.crs_init()) ) {
exit_status = ret;
goto cleanup;
}
}
cleanup:
return exit_status;
}
/**
* Function for finding and opening either all MCA components,
* or the one that was specifically requested via a MCA parameter.
*/
int opal_compress_base_open(void)
{
int ret, exit_status = OPAL_SUCCESS;
int value;
char *str_value = NULL;
/* Debugging/Verbose output */
mca_base_param_reg_int_name("compress",
"base_verbose",
"Verbosity level of the COMPRESS framework",
false, false,
0, &value);
if(0 != value) {
opal_compress_base_output = opal_output_open(NULL);
} else {
opal_compress_base_output = -1;
}
opal_output_set_verbosity(opal_compress_base_output, value);
/*
* Which COMPRESS component to open
* - NULL or "" = auto-select
* - "none" = Empty component
* - ow. select that specific component
*/
mca_base_param_reg_string_name("compress", NULL,
"Which COMPRESS component to use (empty = auto-select)",
false, false,
NULL, &str_value);
/* Compression currently only used with C/R */
if( !opal_cr_is_enabled ) {
opal_output_verbose(10, opal_compress_base_output,
"compress:open: FT is not enabled, skipping!");
return OPAL_SUCCESS;
}
/* Open up all available components */
if (OPAL_SUCCESS != (ret = mca_base_components_open("compress",
opal_compress_base_output,
mca_compress_base_static_components,
&opal_compress_base_components_available,
true)) ) {
if( OPAL_ERR_NOT_FOUND == ret &&
NULL != str_value &&
0 == strncmp(str_value, "none", strlen("none")) ) {
exit_status = OPAL_SUCCESS;
} else {
exit_status = OPAL_ERROR;
}
}
if( NULL != str_value ) {
free(str_value);
}
return exit_status;
}
/*
* CR Init
*/
int orte_cr_init(void)
{
int ret, exit_status = ORTE_SUCCESS;
int val;
/*
* OPAL Frameworks
*/
if (OPAL_SUCCESS != (ret = opal_cr_init() ) ) {
exit_status = ret;
goto cleanup;
}
/*
* Register MCA Parameters
*/
mca_base_param_reg_int_name("orte_cr", "verbose",
"Verbose output for the ORTE Checkpoint/Restart functionality",
false, false,
0,
&val);
/*** RHC: This is going to crash-and-burn when the output conversion is
* completed as opal_output will have no idea what opal_cr_output stream means,
* or even worse, will have assigned it to someone else!
*/
if(0 != val) {
orte_cr_output = opal_output_open(NULL);
opal_output_set_verbosity(orte_cr_output, val);
} else {
orte_cr_output = opal_cr_output;
}
opal_output_verbose(10, orte_cr_output,
"orte_cr: init: orte_cr_init()\n");
/* Init ORTE Entry Point Function */
if( ORTE_SUCCESS != (ret = orte_cr_entry_point_init()) ) {
exit_status = ret;
goto cleanup;
}
/* Register the ORTE interlevel coordination callback */
opal_cr_reg_coord_callback(orte_cr_coord, &prev_coord_callback);
/* Typically this is not needed. Individual BTLs will set this as needed */
orte_cr_continue_like_restart = false;
orte_cr_flush_restart_files = true;
cleanup:
return exit_status;
}
int opal_datatype_register_params(void)
{
#if OPAL_ENABLE_DEBUG
mca_base_param_reg_int_name( "mpi", "ddt_unpack_debug",
"Whether to output debugging information in the ddt unpack functions (nonzero = enabled)",
false, false,
opal_unpack_debug, &opal_unpack_debug );
mca_base_param_reg_int_name( "mpi", "ddt_pack_debug",
"Whether to output debugging information in the ddt pack functions (nonzero = enabled)",
false, false,
opal_pack_debug, &opal_pack_debug );
mca_base_param_reg_int_name( "mpi", "ddt_position_debug",
"Non zero lead to output generated by the datatype position functions",
false, false, 0, &opal_position_debug );
mca_base_param_reg_int_name( "mpi", "ddt_copy_debug",
"Whether to output debugging information in the ddt copy functions (nonzero = enabled)",
false, false,
opal_copy_debug, &opal_copy_debug );
#endif /* OPAL_ENABLE_DEBUG */
return OPAL_SUCCESS;
}
/**
* Function for finding and opening either all MCA components, or the one
* that was specifically requested via a MCA parameter.
*/
int orte_debugger_base_open(void)
{
int value;
/* Debugging / verbose output. Always have stream open, with
verbose set by the mca open system... */
orte_debugger_base.output = opal_output_open(NULL);
mca_base_param_reg_int_name("orte",
"output_debugger_proctable",
"Whether or not to output the debugger proctable after launch (default: false)",
true, false, 0, &value);
orte_debugger_base.dump_proctable = OPAL_INT_TO_BOOL(value);
mca_base_param_reg_string_name("orte", "debugger_test_daemon",
"Name of the executable to be used to simulate a debugger colaunch (relative or absolute path)",
false, false, NULL, &orte_debugger_base.test_daemon);
mca_base_param_reg_int_name("orte",
"debugger_test_attach",
"Test debugger colaunch after debugger attachment",
false, false, 0, &value);
orte_debugger_base.test_attach = OPAL_INT_TO_BOOL(value);
/* Open up all available components */
if (ORTE_SUCCESS !=
mca_base_components_open("debugger", orte_debugger_base.output,
mca_debugger_base_static_components,
&orte_debugger_base_components_available,
true)) {
return ORTE_ERROR;
}
/* All done */
return ORTE_SUCCESS;
}
static void register_internal(void)
{
int ival;
warn_nonexistent_if_index =
mca_base_param_reg_int_name("ompi_common_verbs",
"warn_nonexistent_if",
"Warn if non-existent devices and/or ports are specified in device include/exclude MCA parameters "
"(0 = do not warn; any other value = warn)",
false, false,
(int) ompi_common_verbs_warn_nonexistent_if,
&ival);
ompi_common_verbs_warn_nonexistent_if = (bool) ival;
registered = true;
}
int orte_ess_env_get(void)
{
int num_procs;
mca_base_param_reg_int_name("orte", "ess_num_procs",
"Used to discover the number of procs in the job",
true, false, -1, &num_procs);
if (num_procs < 0) {
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
return ORTE_ERR_NOT_FOUND;
}
orte_process_info.num_procs = (orte_std_cntr_t)num_procs;
return ORTE_SUCCESS;
}
int opal_cr_refresh_environ(int prev_pid) {
int val;
char *file_name = NULL;
struct stat file_status;
if( 0 >= prev_pid ) {
prev_pid = getpid();
}
/*
* Make sure the file exists. If it doesn't then this means 2 things:
* 1) We have already executed this function, and
* 2) The file has been deleted on the previous round.
*/
asprintf(&file_name, "%s/%s-%d", opal_tmp_directory(), OPAL_CR_BASE_ENV_NAME, prev_pid);
if(0 != stat(file_name, &file_status) ){
return OPAL_SUCCESS;
}
#if OPAL_ENABLE_CRDEBUG == 1
opal_unsetenv(mca_base_param_env_var("opal_cr_enable_crdebug"), &environ);
#endif
extract_env_vars(prev_pid, file_name);
#if OPAL_ENABLE_CRDEBUG == 1
mca_base_param_reg_int_name("opal_cr", "enable_crdebug",
"Enable checkpoint/restart debugging",
false, false,
0,
&val);
MPIR_debug_with_checkpoint = OPAL_INT_TO_BOOL(val);
opal_output_verbose(10, opal_cr_output,
"opal_cr: init: C/R Debugging Enabled [%s] (refresh)\n",
(MPIR_debug_with_checkpoint ? "True": "False"));
#else
val = 0; /* Silence Compiler warning */
#endif
if( NULL != file_name ){
free(file_name);
file_name = NULL;
}
return OPAL_SUCCESS;
}
/**
* Function for finding and opening either all MCA components, or the one
* that was specifically requested via a MCA parameter.
*/
int mca_bcol_base_open(void)
{
int value, ret;
/*_bcol_base_components_available
* Register some MCA parameters
*/
/* Debugging/Verbose output */
mca_base_param_reg_int_name("bcol",
"base_verbose",
"Verbosity level of BCOL framework",
false, false,
0, &value);
/* get fraemwork id */
mca_bcol_base_output = opal_output_open(NULL);
opal_output_set_verbosity(mca_bcol_base_output, value);
/* Open up all available components */
if (OMPI_SUCCESS !=
mca_base_components_open("bcol", mca_bcol_base_output, mca_bcol_base_static_components,
&mca_bcol_base_components_opened,
true)) {
return OMPI_ERROR;
}
/* figure out which bcol and sbgp components will actually be used */
/* get list of sub-grouping functions to use */
mca_base_param_reg_string_name("bcol","base_string",
"Default set of basic collective components to use ",
false, false, "basesmuma,basesmuma,iboffload,ptpcoll,ugni", &ompi_bcol_bcols_string);
ret = mca_bcol_base_set_components_to_use(&mca_bcol_base_components_opened,
&mca_bcol_base_components_in_use);
/* memory registration compatibilities */
bcol_mpool_compatibility[BCOL_SHARED_MEMORY_UMA][BCOL_SHARED_MEMORY_UMA]=1;
bcol_mpool_compatibility[BCOL_SHARED_MEMORY_UMA][BCOL_SHARED_MEMORY_SOCKET]=1;
bcol_mpool_compatibility[BCOL_SHARED_MEMORY_UMA][BCOL_POINT_TO_POINT]=1;
bcol_mpool_compatibility[BCOL_SHARED_MEMORY_UMA][BCOL_IB_OFFLOAD]=1;
bcol_mpool_compatibility[BCOL_SHARED_MEMORY_SOCKET][BCOL_SHARED_MEMORY_UMA]=1;
bcol_mpool_compatibility[BCOL_POINT_TO_POINT] [BCOL_SHARED_MEMORY_UMA]=1;
bcol_mpool_compatibility[BCOL_IB_OFFLOAD] [BCOL_SHARED_MEMORY_UMA]=1;
return OMPI_SUCCESS;
}
static orte_rmgr_base_module_t *orte_rmgr_urm_init(int* priority)
{
int param, value;
/* if we are NOT an HNP, then we do NOT want to be selected */
if(!orte_process_info.seed) {
return NULL;
}
param = mca_base_param_reg_int_name("orte", "timing",
"Request that critical timing loops be measured",
false, false, 0, &value);
if (value != 0) {
mca_rmgr_urm_component.timing = true;
} else {
mca_rmgr_urm_component.timing = false;
}
/* volunteer to be selected */
*priority = 100;
return &orte_rmgr_urm_module;
}
int orcm_init(orcm_proc_type_t flags)
{
int ret;
char *error;
int i, spin;
if (NULL != getenv("ORCM_MCA_spin")) {
spin = 1;
/* spin until a debugger can attach */
while (0 != spin) {
ret = 0;
while (ret < 10000) {
ret++;
};
}
}
if (!orcm_util_initialized) {
orcm_init_util();
}
/* set the default leader policy */
orcm_default_leader_policy.jobid = ORTE_JOBID_WILDCARD;
orcm_default_leader_policy.vpid = ORTE_VPID_WILDCARD;
/* get the number of max msgs */
mca_base_param_reg_int_name("orcm", "max_buffered_msgs",
"Number of recvd messages to hold in storage from each source",
false, false, ORCM_MAX_MSG_RING_SIZE, &orcm_max_msg_ring_size);
/* independent mode or not */
mca_base_param_reg_int_name("orcm", "sched_kill_dvm",
"Whether or not scheduler kills associated daemons upon termination (default: no)",
false, false, (int)false, &ret);
orcm_sched_kill_dvm = OPAL_INT_TO_BOOL(ret);
/* setup the globals that require initialization */
orcm_triplets = OBJ_NEW(orcm_triplets_array_t);
#ifdef HAVE_QSYSTEM_H
#ifdef Q_SYSTEM_INTFCS_TO_PROBE_FOR_IP_ADDRESS
{
char *eth_ifs[] = Q_SYSTEM_INTFCS_TO_PROBE_FOR_IP_ADDRESS;
char **adds=NULL, *ifs, *envar;
int i, num_ifs;
num_ifs = sizeof(eth_ifs) / sizeof(eth_ifs[0]);
for (i=0; i < num_ifs; i++) {
opal_argv_append_nosize(&adds, eth_ifs[i]);
}
ifs = opal_argv_join(adds, ',');
opal_argv_free(adds);
/* push it into the environ so that the rmcast framework can get it */
asprintf(&envar, "OMPI_MCA_rmcast_base_if_include=%s", ifs);
putenv(envar);
/* cannot release envar as the environ doesn't keep its own copy */
free(ifs);
}
#endif
#endif
/* initialize us */
if (ORTE_SUCCESS != (ret = orte_init(NULL, NULL, flags))) {
error = "orte_init";
goto error;
}
if (!ORCM_PROC_IS_TOOL) {
opal_set_using_threads(true);
}
if (!ORCM_PROC_IS_APP) {
trap_signals();
}
orcm_initialized = true;
return ORCM_SUCCESS;
error:
if (ORCM_ERR_SILENT != ret) {
orte_show_help("help-openrcm-runtime.txt",
"orcm_init:startup:internal-failure",
true, error, ORTE_ERROR_NAME(ret), ret);
}
return ret;
}
void mca_common_cuda_init(void)
{
int id, value;
CUresult res;
CUcontext cuContext;
if (initialized) {
return;
}
/* Set different levels of verbosity in the cuda related code. */
id = mca_base_param_reg_int_name("mpi", "common_cuda_verbose",
"Set level of common cuda verbosity",
false, false, 0, &mca_common_cuda_verbose);
mca_common_cuda_output = opal_output_open(NULL);
opal_output_set_verbosity(mca_common_cuda_output, mca_common_cuda_verbose);
/* Control whether system buffers get CUDA pinned or not. Allows for
* performance analysis. */
id = mca_base_param_reg_int_name("mpi", "common_cuda_register_memory",
"Whether to cuMemHostRegister preallocated BTL buffers",
false, false,
(int) mca_common_cuda_register_memory, &value);
mca_common_cuda_register_memory = OPAL_INT_TO_BOOL(value);
/* Control whether we see warnings when CUDA memory registration fails. This is
* useful when CUDA support is configured in, but we are running a regular MPI
* application without CUDA. */
id = mca_base_param_reg_int_name("mpi", "common_cuda_warning",
"Whether to print warnings when CUDA registration fails",
false, false,
(int) mca_common_cuda_warning, &value);
mca_common_cuda_warning = OPAL_INT_TO_BOOL(value);
/* Check to see if this process is running in a CUDA context. If
* so, all is good. If not, then disable registration of memory. */
res = cuCtxGetCurrent(&cuContext);
if (CUDA_SUCCESS != res) {
if (mca_common_cuda_warning) {
/* Check for the not initialized error since we can make suggestions to
* user for this error. */
if (CUDA_ERROR_NOT_INITIALIZED == res) {
orte_show_help("help-mpi-common-cuda.txt", "cuCtxGetCurrent failed not initialized",
true);
} else {
orte_show_help("help-mpi-common-cuda.txt", "cuCtxGetCurrent failed",
true, res);
}
}
mca_common_cuda_enabled = false;
mca_common_cuda_register_memory = false;
} else if ((CUDA_SUCCESS == res) && (NULL == cuContext)) {
if (mca_common_cuda_warning) {
orte_show_help("help-mpi-common-cuda.txt", "cuCtxGetCurrent returned NULL",
true);
}
mca_common_cuda_enabled = false;
mca_common_cuda_register_memory = false;
} else {
/* All is good. mca_common_cuda_register_memory will retain its original
* value. Normally, that is 1, but the user can override it to disable
* registration of the internal buffers. */
mca_common_cuda_enabled = true;
opal_output_verbose(20, mca_common_cuda_output,
"CUDA: cuCtxGetCurrent succeeded");
}
opal_output_verbose(30, mca_common_cuda_output,
"CUDA: initialized");
initialized = true;
}
int orte_proc_info(void)
{
int tmp;
char *uri, *ptr;
char hostname[ORTE_MAX_HOSTNAME_SIZE];
if (init) {
return ORTE_SUCCESS;
}
init = true;
mca_base_param_reg_string_name("orte", "hnp_uri",
"HNP contact info",
true, false, NULL, &uri);
if (NULL != uri) {
/* the uri value passed to us will have quote marks around it to protect
* the value if passed on the command line. We must remove those
* to have a correct uri string
*/
if ('"' == uri[0]) {
/* if the first char is a quote, then so will the last one be */
uri[strlen(uri)-1] = '\0';
ptr = &uri[1];
} else {
ptr = &uri[0];
}
orte_process_info.my_hnp_uri = strdup(ptr);
free(uri);
}
mca_base_param_reg_string_name("orte", "local_daemon_uri",
"Daemon contact info",
true, false, NULL, &(uri));
if (NULL != uri) {
/* the uri value passed to us may have quote marks around it to protect
* the value if passed on the command line. We must remove those
* to have a correct uri string
*/
if ('"' == uri[0]) {
/* if the first char is a quote, then so will the last one be */
uri[strlen(uri)-1] = '\0';
ptr = &uri[1];
} else {
ptr = &uri[0];
}
orte_process_info.my_daemon_uri = strdup(ptr);
free(uri);
}
mca_base_param_reg_int_name("orte", "app_num",
"Index of the app_context that defines this proc",
true, false, 0, &tmp);
orte_process_info.app_num = tmp;
/* get the process id */
orte_process_info.pid = getpid();
/* get the nodename */
gethostname(hostname, ORTE_MAX_HOSTNAME_SIZE);
orte_process_info.nodename = strdup(hostname);
/* get the number of nodes in the job */
mca_base_param_reg_int_name("orte", "num_nodes",
"Number of nodes in the job",
true, false,
orte_process_info.num_nodes, &tmp);
orte_process_info.num_nodes = tmp;
/* get the number of times this proc has restarted */
mca_base_param_reg_int_name("orte", "num_restarts",
"Number of times this proc has restarted",
true, false, 0, &tmp);
orte_process_info.num_restarts = tmp;
mca_base_param_reg_string_name("orte", "job_name",
"Job name",
true, false, NULL, &orte_process_info.job_name);
mca_base_param_reg_string_name("orte", "job_instance",
"Job instance",
true, false, NULL, &orte_process_info.job_instance);
mca_base_param_reg_string_name("orte", "executable",
"Executable",
true, false, NULL, &orte_process_info.executable);
mca_base_param_reg_int_name("orte", "app_rank",
"Rank of this proc within its app_context",
true, false, 0, &tmp);
orte_process_info.app_rank = tmp;
/* setup the sync buffer */
orte_process_info.sync_buf = OBJ_NEW(opal_buffer_t);
return ORTE_SUCCESS;
}
int orte_util_nidmap_init(opal_buffer_t *buffer)
{
int32_t cnt;
int rc;
opal_byte_object_t *bo;
mca_base_param_reg_int_name("orte", "nidmap_verbose",
"Verbosity of the nidmap subsystem",
true, false, 0, &orte_nidmap_verbose);
if (0 < orte_nidmap_verbose) {
orte_nidmap_output = opal_output_open(NULL);
opal_output_set_verbosity(orte_nidmap_output, orte_nidmap_verbose);
}
/* it is okay if the buffer is empty */
if (NULL == buffer || 0 == buffer->bytes_used) {
return ORTE_SUCCESS;
}
#if OPAL_HAVE_HWLOC
{
hwloc_topology_t topo;
/* extract the topology */
cnt=1;
if (ORTE_SUCCESS != (rc = opal_dss.unpack(buffer, &topo, &cnt, OPAL_HWLOC_TOPO))) {
ORTE_ERROR_LOG(rc);
return rc;
}
if (NULL == opal_hwloc_topology) {
opal_hwloc_topology = topo;
} else {
hwloc_topology_destroy(topo);
}
}
#endif
/* extract the byte object holding the daemonmap */
cnt=1;
if (ORTE_SUCCESS != (rc = opal_dss.unpack(buffer, &bo, &cnt, OPAL_BYTE_OBJECT))) {
ORTE_ERROR_LOG(rc);
return rc;
}
/* unpack the node map */
if (ORTE_SUCCESS != (rc = orte_util_decode_nodemap(bo))) {
ORTE_ERROR_LOG(rc);
return rc;
}
/* the bytes in the object were free'd by the decode */
/* extract the byte object holding the process map */
cnt=1;
if (ORTE_SUCCESS != (rc = opal_dss.unpack(buffer, &bo, &cnt, OPAL_BYTE_OBJECT))) {
ORTE_ERROR_LOG(rc);
return rc;
}
/* unpack the process map */
if (ORTE_SUCCESS != (rc = orte_util_decode_pidmap(bo))) {
ORTE_ERROR_LOG(rc);
return rc;
}
/* the bytes in the object were free'd by the decode */
return ORTE_SUCCESS;
}
int
ompi_comm_start_processes(int count, char **array_of_commands,
char ***array_of_argv,
int *array_of_maxprocs,
MPI_Info *array_of_info,
char *port_name)
{
int rc, i, j, counter;
int have_wdir=0;
bool have_prefix;
int valuelen=OMPI_PATH_MAX, flag=0;
char cwd[OMPI_PATH_MAX];
char host[OMPI_PATH_MAX]; /*** should define OMPI_HOST_MAX ***/
char prefix[OMPI_PATH_MAX];
char *base_prefix;
orte_std_cntr_t num_apps, ai;
orte_jobid_t new_jobid=ORTE_JOBID_INVALID;
orte_app_context_t **apps=NULL;
opal_list_t attributes;
opal_list_item_t *item;
bool timing = false;
struct timeval ompistart, ompistop;
int param, value;
/* parse the info object */
/* check potentially for:
- "host": desired host where to spawn the processes
- "prefix": the path to the root of the directory tree where ompi
executables and libraries can be found
- "arch": desired architecture
- "wdir": directory, where executable can be found
- "path": list of directories where to look for the executable
- "file": filename, where additional information is provided.
- "soft": see page 92 of MPI-2.
*/
/* make sure the progress engine properly trips the event library */
opal_progress_event_increment();
/* check to see if we want timing information */
param = mca_base_param_reg_int_name("ompi", "timing",
"Request that critical timing loops be measured",
false, false, 0, &value);
if (value != 0) {
timing = true;
if (0 != gettimeofday(&ompistart, NULL)) {
opal_output(0, "ompi_comm_start_procs: could not obtain start time");
ompistart.tv_sec = 0;
ompistart.tv_usec = 0;
}
}
/* setup to record the attributes */
OBJ_CONSTRUCT(&attributes, opal_list_t);
/* we want to be able to default the prefix to the one used for this job
* so that the ompi executables and libraries can be found. the user can
* later override this value by providing an MPI_Info value. for now, though,
* let's get the default value off the registry
*/
if (ORTE_SUCCESS != (rc = orte_rmgr.get_app_context(orte_process_info.my_name->jobid, &apps, &num_apps))) {
ORTE_ERROR_LOG(rc);
return rc;
}
/* we'll just use the prefix from the first member of the app_context array.
* this shouldn't matter as they all should be the same. it could be NULL, of
* course (user might not have specified it), so we need to protect against that.
*
* It's possible that no app_contexts are returned (e.g., during a comm_spawn
* from a singleton), so check first
*/
if (NULL != apps && NULL != apps[0]->prefix_dir) {
base_prefix = strdup(apps[0]->prefix_dir);
} else {
base_prefix = NULL;
}
/* cleanup the memory we used */
for (ai = 0; ai < num_apps; ai++) {
OBJ_RELEASE(apps[ai]);
}
if (NULL != apps) free(apps);
/* Convert the list of commands to an array of orte_app_context_t
pointers */
apps = (orte_app_context_t**)malloc(count * sizeof(orte_app_context_t *));
if (NULL == apps) {
ORTE_ERROR_LOG(ORTE_ERR_OUT_OF_RESOURCE);
return ORTE_ERR_OUT_OF_RESOURCE;
}
for (i = 0; i < count; ++i) {
apps[i] = OBJ_NEW(orte_app_context_t);
if (NULL == apps[i]) {
ORTE_ERROR_LOG(ORTE_ERR_OUT_OF_RESOURCE);
/* rollback what was already done */
for (j=0; j < i; j++) OBJ_RELEASE(apps[j]);
opal_progress_event_decrement();
return ORTE_ERR_OUT_OF_RESOURCE;
}
/* copy over the name of the executable */
apps[i]->app = strdup(array_of_commands[i]);
if (NULL == apps[i]->app) {
ORTE_ERROR_LOG(ORTE_ERR_OUT_OF_RESOURCE);
/* rollback what was already done */
for (j=0; j < i; j++) OBJ_RELEASE(apps[j]);
opal_progress_event_decrement();
return ORTE_ERR_OUT_OF_RESOURCE;
}
/* record the number of procs to be generated */
apps[i]->num_procs = array_of_maxprocs[i];
/* copy over the argv array */
counter = 1;
if (MPI_ARGVS_NULL != array_of_argv &&
MPI_ARGV_NULL != array_of_argv[i]) {
/* first need to find out how many entries there are */
j=0;
while (NULL != array_of_argv[i][j]) {
j++;
}
counter += j;
}
/* now copy them over, ensuring to NULL terminate the array */
apps[i]->argv = (char**)malloc((1 + counter) * sizeof(char*));
if (NULL == apps[i]->argv) {
ORTE_ERROR_LOG(ORTE_ERR_OUT_OF_RESOURCE);
/* rollback what was already done */
for (j=0; j < i; j++) {
OBJ_RELEASE(apps[j]);
}
opal_progress_event_decrement();
return ORTE_ERR_OUT_OF_RESOURCE;
}
apps[i]->argv[0] = strdup(array_of_commands[i]);
for (j=1; j < counter; j++) {
apps[i]->argv[j] = strdup(array_of_argv[i][j-1]);
}
apps[i]->argv[counter] = NULL;
/* the environment gets set by the launcher
* all we need to do is add the specific values
* needed for comm_spawn
*/
/* Add environment variable with the contact information for the
child processes.
*/
counter = 1;
apps[i]->env = (char**)malloc((1+counter) * sizeof(char*));
if (NULL == apps[i]->env) {
ORTE_ERROR_LOG(ORTE_ERR_OUT_OF_RESOURCE);
/* rollback what was already done */
for (j=0; j < i; j++) OBJ_RELEASE(apps[j]);
opal_progress_event_decrement();
return ORTE_ERR_OUT_OF_RESOURCE;
}
asprintf(&(apps[i]->env[0]), "OMPI_PARENT_PORT=%s", port_name);
apps[i]->env[1] = NULL;
for (j = 0; NULL != environ[j]; ++j) {
if (0 == strncmp("OMPI_", environ[j], 5)) {
opal_argv_append_nosize(&apps[i]->env, environ[j]);
}
}
/* Check for well-known info keys */
have_wdir = 0;
have_prefix = false;
if ( array_of_info != NULL && array_of_info[i] != MPI_INFO_NULL ) {
/* check for 'wdir' */
ompi_info_get (array_of_info[i], "wdir", valuelen, cwd, &flag);
if ( flag ) {
apps[i]->cwd = cwd;
have_wdir = 1;
}
/* check for 'host' */
ompi_info_get (array_of_info[i], "host", sizeof(host), host, &flag);
if ( flag ) {
apps[i]->num_map = 1;
apps[i]->map_data = (orte_app_context_map_t **) malloc(sizeof(orte_app_context_map_t *));
apps[i]->map_data[0] = OBJ_NEW(orte_app_context_map_t);
apps[i]->map_data[0]->map_type = ORTE_APP_CONTEXT_MAP_HOSTNAME;
apps[i]->map_data[0]->map_data = strdup(host);
}
/* 'path', 'arch', 'file', 'soft' -- to be implemented */
/* check for 'ompi_prefix' (OMPI-specific -- to effect the same
* behavior as --prefix option to orterun)
*/
ompi_info_get (array_of_info[i], "ompi_prefix", sizeof(prefix), prefix, &flag);
if ( flag ) {
apps[i]->prefix_dir = strdup(prefix);
have_prefix = true;
}
}
/* default value: If the user did not tell us where to look for the
executable, we assume the current working directory */
if ( !have_wdir ) {
getcwd(cwd, OMPI_PATH_MAX);
apps[i]->cwd = strdup(cwd);
}
/* if the user told us a new prefix, then we leave it alone. otherwise, if
* a prefix had been provided before, copy that one into the new app_context
* for use by the spawned children
*/
if ( !have_prefix && NULL != base_prefix) {
apps[i]->prefix_dir = strdup(base_prefix);
}
/* leave the map info alone - the launcher will
* decide where to put things
*/
} /* for (i = 0 ; i < count ; ++i) */
/* cleanup */
if (NULL != base_prefix) free(base_prefix);
/* tell the RTE that we want to the children to run inside of our allocation -
* don't go get one just for them
*/
if (ORTE_SUCCESS != (rc = orte_rmgr.add_attribute(&attributes, ORTE_RAS_USE_PARENT_ALLOCATION,
ORTE_JOBID, &(orte_process_info.my_name->jobid),
ORTE_RMGR_ATTR_OVERRIDE))) {
ORTE_ERROR_LOG(rc);
OBJ_DESTRUCT(&attributes);
opal_progress_event_decrement();
return MPI_ERR_SPAWN;
}
/* tell the RTE that we want the children mapped the same way as their parent */
if (ORTE_SUCCESS != (rc = orte_rmgr.add_attribute(&attributes, ORTE_RMAPS_USE_PARENT_PLAN,
ORTE_JOBID, &(orte_process_info.my_name->jobid),
ORTE_RMGR_ATTR_OVERRIDE))) {
ORTE_ERROR_LOG(rc);
OBJ_DESTRUCT(&attributes);
opal_progress_event_decrement();
return MPI_ERR_SPAWN;
}
/* check for timing request - get stop time and report elapsed time if so */
if (timing) {
if (0 != gettimeofday(&ompistop, NULL)) {
opal_output(0, "ompi_comm_start_procs: could not obtain stop time");
} else {
opal_output(0, "ompi_comm_start_procs: time from start to prepare to spawn %ld usec",
(long int)((ompistop.tv_sec - ompistart.tv_sec)*1000000 +
(ompistop.tv_usec - ompistart.tv_usec)));
if (0 != gettimeofday(&ompistart, NULL)) {
opal_output(0, "ompi_comm_start_procs: could not obtain new start time");
ompistart.tv_sec = ompistop.tv_sec;
ompistart.tv_usec = ompistop.tv_usec;
}
}
}
/* spawn procs */
if (ORTE_SUCCESS != (rc = orte_rmgr.spawn_job(apps, count, &new_jobid, 0, NULL, NULL, ORTE_PROC_STATE_NONE, &attributes))) {
ORTE_ERROR_LOG(rc);
opal_progress_event_decrement();
return MPI_ERR_SPAWN;
}
/* check for timing request - get stop time and report elapsed time if so */
if (timing) {
if (0 != gettimeofday(&ompistop, NULL)) {
opal_output(0, "ompi_comm_start_procs: could not obtain stop time");
} else {
opal_output(0, "ompi_comm_start_procs: time to spawn %ld usec",
(long int)((ompistop.tv_sec - ompistart.tv_sec)*1000000 +
(ompistop.tv_usec - ompistart.tv_usec)));
}
}
/* clean up */
opal_progress_event_decrement();
while (NULL != (item = opal_list_remove_first(&attributes))) OBJ_RELEASE(item);
OBJ_DESTRUCT(&attributes);
for ( i=0; i<count; i++) {
OBJ_RELEASE(apps[i]);
}
free (apps);
return OMPI_SUCCESS;
}
int ompi_mpi_init(int argc, char **argv, int requested, int *provided)
{
int ret;
ompi_proc_t** procs;
size_t nprocs;
char *error = NULL;
bool timing = false;
int param, value;
struct timeval ompistart, ompistop;
char *event_val = NULL;
bool orte_setup = false;
orte_grpcomm_collective_t *coll;
char *cmd=NULL, *av=NULL;
/* bitflag of the thread level support provided. To be used
* for the modex in order to work in heterogeneous environments. */
uint8_t threadlevel_bf;
/* Indicate that we have *started* MPI_INIT*. MPI_FINALIZE has
something sorta similar in a static local variable in
ompi_mpi_finalize(). */
ompi_mpi_init_started = true;
/* Setup enough to check get/set MCA params */
if (OPAL_SUCCESS != (ret = opal_init_util(&argc, &argv))) {
error = "ompi_mpi_init: opal_init_util failed";
goto error;
}
if (OPAL_SUCCESS != (ret = opal_arch_set_fortran_logical_size(sizeof(ompi_fortran_logical_t)))) {
error = "ompi_mpi_init: opal_arch_set_fortran_logical_size failed";
goto error;
}
/* _After_ opal_init_util() but _before_ orte_init(), we need to
set an MCA param that tells libevent that it's ok to use any
mechanism in libevent that is available on this platform (e.g.,
epoll and friends). Per opal/event/event.s, we default to
select/poll -- but we know that MPI processes won't be using
pty's with the event engine, so it's ok to relax this
constraint and let any fd-monitoring mechanism be used. */
ret = mca_base_param_reg_string_name("opal", "event_include",
"Internal orted MCA param: tell opal_init() to use a specific mechanism in libevent",
false, false, "all", &event_val);
if (ret >= 0) {
/* We have to explicitly "set" the MCA param value here
because libevent initialization will re-register the MCA
param and therefore override the default. Setting the value
here puts the desired value ("all") in different storage
that is not overwritten if/when the MCA param is
re-registered. This is unless the user has specified a different
value for this MCA parameter. Make sure we check to see if the
default is specified before forcing "all" in case that is not what
the user desires. Note that we do *NOT* set this value as an
environment variable, just so that it won't be inherited by
any spawned processes and potentially cause unintented
side-effects with launching ORTE tools... */
if (0 == strcmp("all", event_val)) {
mca_base_param_set_string(ret, "all");
}
}
if( NULL != event_val ) {
free(event_val);
event_val = NULL;
}
/* check to see if we want timing information */
param = mca_base_param_reg_int_name("ompi", "timing",
"Request that critical timing loops be measured",
false, false, 0, &value);
if (value != 0) {
timing = true;
gettimeofday(&ompistart, NULL);
}
/* if we were not externally started, then we need to setup
* some envars so the MPI_INFO_ENV can get the cmd name
* and argv (but only if the user supplied a non-NULL argv!), and
* the requested thread level
*/
if (NULL == getenv("OMPI_COMMAND") && NULL != argv && NULL != argv[0]) {
asprintf(&cmd, "OMPI_COMMAND=%s", argv[0]);
putenv(cmd);
}
if (NULL == getenv("OMPI_ARGV") && 1 < argc) {
char *tmp;
tmp = opal_argv_join(&argv[1], ' ');
asprintf(&av, "OMPI_ARGV=%s", tmp);
free(tmp);
putenv(av);
}
/* Setup ORTE - note that we are an MPI process */
if (ORTE_SUCCESS != (ret = orte_init(NULL, NULL, ORTE_PROC_MPI))) {
error = "ompi_mpi_init: orte_init failed";
goto error;
}
orte_setup = true;
/* check for timing request - get stop time and report elapsed time if so */
if (timing && 0 == ORTE_PROC_MY_NAME->vpid) {
gettimeofday(&ompistop, NULL);
opal_output(0, "ompi_mpi_init [%ld]: time from start to completion of orte_init %ld usec",
(long)ORTE_PROC_MY_NAME->vpid,
(long int)((ompistop.tv_sec - ompistart.tv_sec)*1000000 +
(ompistop.tv_usec - ompistart.tv_usec)));
gettimeofday(&ompistart, NULL);
}
#if OPAL_HAVE_HWLOC
/* if hwloc is available but didn't get setup for some
* reason, do so now
*/
if (NULL == opal_hwloc_topology) {
if (OPAL_SUCCESS != (ret = opal_hwloc_base_get_topology())) {
error = "Topology init";
goto error;
}
}
#endif
/* Register errhandler callback with orte errmgr */
if (NULL != orte_errmgr.set_fault_callback) {
orte_errmgr.set_fault_callback(ompi_errhandler_runtime_callback);
}
/* Figure out the final MPI thread levels. If we were not
compiled for support for MPI threads, then don't allow
MPI_THREAD_MULTIPLE. Set this stuff up here early in the
process so that other components can make decisions based on
this value. */
/**
* These values are monotonic; MPI_THREAD_SINGLE < MPI_THREAD_FUNNELED
* < MPI_THREAD_SERIALIZED < MPI_THREAD_MULTIPLE.
* If possible, the call will return provided = required. Failing this,
* the call will return the least supported level such that
* provided > required. Finally, if the user requirement cannot be
* satisfied, then the call will return in provided the highest
* supported level.
*/
ompi_mpi_thread_requested = requested;
if (OMPI_ENABLE_THREAD_MULTIPLE == 1) {
ompi_mpi_thread_provided = *provided = requested;
ompi_mpi_main_thread = opal_thread_get_self();
} else {
if (MPI_THREAD_MULTIPLE == requested) {
ompi_mpi_thread_provided = *provided = MPI_THREAD_SERIALIZED;
} else {
ompi_mpi_thread_provided = *provided = requested;
}
ompi_mpi_main_thread = (OPAL_ENABLE_MULTI_THREADS ? opal_thread_get_self() : NULL);
}
ompi_mpi_thread_multiple = (ompi_mpi_thread_provided ==
MPI_THREAD_MULTIPLE);
/* determine the bitflag belonging to the threadlevel_support provided */
memset ( &threadlevel_bf, 0, sizeof(uint8_t));
OMPI_THREADLEVEL_SET_BITFLAG ( ompi_mpi_thread_provided, threadlevel_bf );
/* add this bitflag to the modex */
if ( OMPI_SUCCESS != (ret = ompi_modex_send_string("MPI_THREAD_LEVEL", &threadlevel_bf, sizeof(uint8_t)))) {
error = "ompi_mpi_init: modex send thread level";
goto error;
}
/* Once we've joined the RTE, see if any MCA parameters were
passed to the MPI level */
if (OMPI_SUCCESS != (ret = ompi_mpi_register_params())) {
error = "mca_mpi_register_params() failed";
goto error;
}
/* initialize datatypes. This step should be done early as it will
* create the local convertor and local arch used in the proc
* init.
*/
if (OMPI_SUCCESS != (ret = ompi_datatype_init())) {
error = "ompi_datatype_init() failed";
goto error;
}
/* Initialize OMPI procs */
if (OMPI_SUCCESS != (ret = ompi_proc_init())) {
error = "mca_proc_init() failed";
goto error;
}
/* Initialize the op framework. This has to be done *after*
ddt_init, but befor mca_coll_base_open, since some collective
modules (e.g., the hierarchical coll component) may need ops in
their query function. */
if (OMPI_SUCCESS != (ret = ompi_op_base_open())) {
error = "ompi_op_base_open() failed";
goto error;
}
if (OMPI_SUCCESS !=
(ret = ompi_op_base_find_available(OMPI_ENABLE_PROGRESS_THREADS,
OMPI_ENABLE_THREAD_MULTIPLE))) {
error = "ompi_op_base_find_available() failed";
goto error;
}
if (OMPI_SUCCESS != (ret = ompi_op_init())) {
error = "ompi_op_init() failed";
goto error;
}
/* Open up MPI-related MCA components */
if (OMPI_SUCCESS != (ret = mca_allocator_base_open())) {
error = "mca_allocator_base_open() failed";
goto error;
}
if (OMPI_SUCCESS != (ret = mca_rcache_base_open())) {
error = "mca_rcache_base_open() failed";
goto error;
}
if (OMPI_SUCCESS != (ret = mca_mpool_base_open())) {
error = "mca_mpool_base_open() failed";
goto error;
}
if (OMPI_SUCCESS != (ret = mca_pml_base_open())) {
error = "mca_pml_base_open() failed";
goto error;
}
if (OMPI_SUCCESS != (ret = mca_coll_base_open())) {
error = "mca_coll_base_open() failed";
goto error;
}
if (OMPI_SUCCESS != (ret = ompi_osc_base_open())) {
error = "ompi_osc_base_open() failed";
goto error;
}
#if OPAL_ENABLE_FT_CR == 1
if (OMPI_SUCCESS != (ret = ompi_crcp_base_open())) {
error = "ompi_crcp_base_open() failed";
goto error;
}
#endif
/* In order to reduce the common case for MPI apps (where they
don't use MPI-2 IO or MPI-1 topology functions), the io and
topo frameworks are initialized lazily, at the first use of
relevant functions (e.g., MPI_FILE_*, MPI_CART_*, MPI_GRAPH_*),
so they are not opened here. */
/* Select which MPI components to use */
if (OMPI_SUCCESS !=
(ret = mca_mpool_base_init(OMPI_ENABLE_PROGRESS_THREADS,
OMPI_ENABLE_THREAD_MULTIPLE))) {
error = "mca_mpool_base_init() failed";
goto error;
}
if (OMPI_SUCCESS !=
(ret = mca_pml_base_select(OMPI_ENABLE_PROGRESS_THREADS,
OMPI_ENABLE_THREAD_MULTIPLE))) {
error = "mca_pml_base_select() failed";
goto error;
}
/* check for timing request - get stop time and report elapsed time if so */
if (timing && 0 == ORTE_PROC_MY_NAME->vpid) {
gettimeofday(&ompistop, NULL);
opal_output(0, "ompi_mpi_init[%ld]: time from completion of orte_init to modex %ld usec",
(long)ORTE_PROC_MY_NAME->vpid,
(long int)((ompistop.tv_sec - ompistart.tv_sec)*1000000 +
(ompistop.tv_usec - ompistart.tv_usec)));
gettimeofday(&ompistart, NULL);
}
/* exchange connection info - this function also acts as a barrier
* as it will not return until the exchange is complete
*/
coll = OBJ_NEW(orte_grpcomm_collective_t);
coll->id = orte_process_info.peer_modex;
if (ORTE_SUCCESS != (ret = orte_grpcomm.modex(coll))) {
error = "orte_grpcomm_modex failed";
goto error;
}
/* wait for modex to complete - this may be moved anywhere in mpi_init
* so long as it occurs prior to calling a function that needs
* the modex info!
*/
while (coll->active) {
opal_progress(); /* block in progress pending events */
}
OBJ_RELEASE(coll);
if (timing && 0 == ORTE_PROC_MY_NAME->vpid) {
gettimeofday(&ompistop, NULL);
opal_output(0, "ompi_mpi_init[%ld]: time to execute modex %ld usec",
(long)ORTE_PROC_MY_NAME->vpid,
(long int)((ompistop.tv_sec - ompistart.tv_sec)*1000000 +
(ompistop.tv_usec - ompistart.tv_usec)));
gettimeofday(&ompistart, NULL);
}
/* select buffered send allocator component to be used */
ret=mca_pml_base_bsend_init(OMPI_ENABLE_THREAD_MULTIPLE);
if( OMPI_SUCCESS != ret ) {
error = "mca_pml_base_bsend_init() failed";
goto error;
}
if (OMPI_SUCCESS !=
(ret = mca_coll_base_find_available(OMPI_ENABLE_PROGRESS_THREADS,
OMPI_ENABLE_THREAD_MULTIPLE))) {
error = "mca_coll_base_find_available() failed";
goto error;
}
if (OMPI_SUCCESS !=
(ret = ompi_osc_base_find_available(OMPI_ENABLE_PROGRESS_THREADS,
OMPI_ENABLE_THREAD_MULTIPLE))) {
error = "ompi_osc_base_find_available() failed";
goto error;
}
#if OPAL_ENABLE_FT_CR == 1
if (OMPI_SUCCESS != (ret = ompi_crcp_base_select() ) ) {
error = "ompi_crcp_base_select() failed";
goto error;
}
#endif
/* io and topo components are not selected here -- see comment
above about the io and topo frameworks being loaded lazily */
/* Initialize each MPI handle subsystem */
/* initialize requests */
if (OMPI_SUCCESS != (ret = ompi_request_init())) {
error = "ompi_request_init() failed";
goto error;
}
if (OMPI_SUCCESS != (ret = ompi_message_init())) {
error = "ompi_message_init() failed";
goto error;
}
/* initialize info */
if (OMPI_SUCCESS != (ret = ompi_info_init())) {
error = "ompi_info_init() failed";
goto error;
}
/* initialize error handlers */
if (OMPI_SUCCESS != (ret = ompi_errhandler_init())) {
error = "ompi_errhandler_init() failed";
goto error;
}
/* initialize error codes */
if (OMPI_SUCCESS != (ret = ompi_mpi_errcode_init())) {
error = "ompi_mpi_errcode_init() failed";
goto error;
}
/* initialize internal error codes */
if (OMPI_SUCCESS != (ret = ompi_errcode_intern_init())) {
error = "ompi_errcode_intern_init() failed";
goto error;
}
/* initialize groups */
if (OMPI_SUCCESS != (ret = ompi_group_init())) {
error = "ompi_group_init() failed";
goto error;
}
/* initialize communicators */
if (OMPI_SUCCESS != (ret = ompi_comm_init())) {
error = "ompi_comm_init() failed";
goto error;
}
/* initialize file handles */
if (OMPI_SUCCESS != (ret = ompi_file_init())) {
error = "ompi_file_init() failed";
goto error;
}
/* initialize windows */
if (OMPI_SUCCESS != (ret = ompi_win_init())) {
error = "ompi_win_init() failed";
goto error;
}
/* initialize attribute meta-data structure for comm/win/dtype */
if (OMPI_SUCCESS != (ret = ompi_attr_init())) {
error = "ompi_attr_init() failed";
goto error;
}
/* identify the architectures of remote procs and setup
* their datatype convertors, if required
*/
if (OMPI_SUCCESS != (ret = ompi_proc_complete_init())) {
error = "ompi_proc_complete_init failed";
goto error;
}
/* If thread support was enabled, then setup OPAL to allow for
them. */
if ((OMPI_ENABLE_PROGRESS_THREADS == 1) ||
(*provided != MPI_THREAD_SINGLE)) {
opal_set_using_threads(true);
}
/* start PML/BTL's */
ret = MCA_PML_CALL(enable(true));
if( OMPI_SUCCESS != ret ) {
error = "PML control failed";
goto error;
}
/* add all ompi_proc_t's to PML */
if (NULL == (procs = ompi_proc_world(&nprocs))) {
error = "ompi_proc_world() failed";
goto error;
}
ret = MCA_PML_CALL(add_procs(procs, nprocs));
free(procs);
/* If we got "unreachable", then print a specific error message.
Otherwise, if we got some other failure, fall through to print
a generic message. */
if (OMPI_ERR_UNREACH == ret) {
orte_show_help("help-mpi-runtime",
"mpi_init:startup:pml-add-procs-fail", true);
error = NULL;
goto error;
} else if (OMPI_SUCCESS != ret) {
error = "PML add procs failed";
goto error;
}
MCA_PML_CALL(add_comm(&ompi_mpi_comm_world.comm));
MCA_PML_CALL(add_comm(&ompi_mpi_comm_self.comm));
/*
* Dump all MCA parameters if requested
*/
if (ompi_mpi_show_mca_params) {
ompi_show_all_mca_params(ompi_mpi_comm_world.comm.c_my_rank,
nprocs,
orte_process_info.nodename);
}
/* Do we need to wait for a debugger? */
ompi_wait_for_debugger();
/* check for timing request - get stop time and report elapsed
time if so, then start the clock again */
if (timing && 0 == ORTE_PROC_MY_NAME->vpid) {
gettimeofday(&ompistop, NULL);
opal_output(0, "ompi_mpi_init[%ld]: time from modex to first barrier %ld usec",
(long)ORTE_PROC_MY_NAME->vpid,
(long int)((ompistop.tv_sec - ompistart.tv_sec)*1000000 +
(ompistop.tv_usec - ompistart.tv_usec)));
gettimeofday(&ompistart, NULL);
}
/* wait for everyone to reach this point */
coll = OBJ_NEW(orte_grpcomm_collective_t);
coll->id = orte_process_info.peer_init_barrier;
if (ORTE_SUCCESS != (ret = orte_grpcomm.barrier(coll))) {
error = "orte_grpcomm_barrier failed";
goto error;
}
/* wait for barrier to complete */
while (coll->active) {
opal_progress(); /* block in progress pending events */
}
OBJ_RELEASE(coll);
/* check for timing request - get stop time and report elapsed
time if so, then start the clock again */
if (timing && 0 == ORTE_PROC_MY_NAME->vpid) {
gettimeofday(&ompistop, NULL);
opal_output(0, "ompi_mpi_init[%ld]: time to execute barrier %ld usec",
(long)ORTE_PROC_MY_NAME->vpid,
(long int)((ompistop.tv_sec - ompistart.tv_sec)*1000000 +
(ompistop.tv_usec - ompistart.tv_usec)));
gettimeofday(&ompistart, NULL);
}
#if OMPI_ENABLE_PROGRESS_THREADS == 0
/* Start setting up the event engine for MPI operations. Don't
block in the event library, so that communications don't take
forever between procs in the dynamic code. This will increase
CPU utilization for the remainder of MPI_INIT when we are
blocking on ORTE-level events, but may greatly reduce non-TCP
latency. */
opal_progress_set_event_flag(OPAL_EVLOOP_NONBLOCK);
#endif
/* wire up the mpi interface, if requested. Do this after the
non-block switch for non-TCP performance. Do before the
polling change as anyone with a complex wire-up is going to be
using the oob. */
if (OMPI_SUCCESS != (ret = ompi_init_preconnect_mpi())) {
error = "ompi_mpi_do_preconnect_all() failed";
goto error;
}
/* Setup the publish/subscribe (PUBSUB) framework */
if (OMPI_SUCCESS != (ret = ompi_pubsub_base_open())) {
error = "ompi_pubsub_base_open() failed";
goto error;
}
if (OMPI_SUCCESS != (ret = ompi_pubsub_base_select())) {
error = "ompi_pubsub_base_select() failed";
goto error;
}
/* Setup the dynamic process management (DPM) framework */
if (OMPI_SUCCESS != (ret = ompi_dpm_base_open())) {
error = "ompi_dpm_base_open() failed";
goto error;
}
if (OMPI_SUCCESS != (ret = ompi_dpm_base_select())) {
error = "ompi_dpm_base_select() failed";
goto error;
}
/* Determine the overall threadlevel support of all processes
in MPI_COMM_WORLD. This has to be done before calling
coll_base_comm_select, since some of the collective components
e.g. hierarch, might create subcommunicators. The threadlevel
requested by all processes is required in order to know
which cid allocation algorithm can be used. */
if ( OMPI_SUCCESS !=
( ret = ompi_comm_cid_init ())) {
error = "ompi_mpi_init: ompi_comm_cid_init failed";
goto error;
}
/* Init coll for the comms. This has to be after dpm_base_select,
(since dpm.mark_dyncomm is not set in the communicator creation
function else), but before dpm.dyncom_init, since this function
might require collective for the CID allocation. */
if (OMPI_SUCCESS !=
(ret = mca_coll_base_comm_select(MPI_COMM_WORLD))) {
error = "mca_coll_base_comm_select(MPI_COMM_WORLD) failed";
goto error;
}
if (OMPI_SUCCESS !=
(ret = mca_coll_base_comm_select(MPI_COMM_SELF))) {
error = "mca_coll_base_comm_select(MPI_COMM_SELF) failed";
goto error;
}
/* Check whether we have been spawned or not. We introduce that
at the very end, since we need collectives, datatypes, ptls
etc. up and running here.... */
if (OMPI_SUCCESS != (ret = ompi_dpm.dyn_init())) {
error = "ompi_comm_dyn_init() failed";
goto error;
}
/*
* Startup the Checkpoint/Restart Mech.
* Note: Always do this so tools don't hang when
* in a non-checkpointable build
*/
if (OMPI_SUCCESS != (ret = ompi_cr_init())) {
error = "ompi_cr_init";
goto error;
}
/* Undo OPAL calling opal_progress_event_users_increment() during
opal_init, to get better latency when not using TCP. Do
this *after* dyn_init, as dyn init uses lots of ORTE
communication and we don't want to hinder the performance of
that code. */
opal_progress_event_users_decrement();
/* see if yield_when_idle was specified - if so, use it */
param = mca_base_param_find("mpi", NULL, "yield_when_idle");
mca_base_param_lookup_int(param, &value);
if (value < 0) {
/* if no info is provided, just default to conservative */
opal_progress_set_yield_when_idle(true);
} else {
/* info was provided, so set idle accordingly */
opal_progress_set_yield_when_idle(value == 0 ? false : true);
}
param = mca_base_param_find("mpi", NULL, "event_tick_rate");
mca_base_param_lookup_int(param, &value);
/* negative value means use default - just don't do anything */
if (value >= 0) {
opal_progress_set_event_poll_rate(value);
}
/* At this point, we are fully configured and in MPI mode. Any
communication calls here will work exactly like they would in
the user's code. Setup the connections between procs and warm
them up with simple sends, if requested */
if (OMPI_SUCCESS != (ret = ompi_mpiext_init())) {
error = "ompi_mpiext_init";
goto error;
}
/* Fall through */
error:
if (ret != OMPI_SUCCESS) {
/* Only print a message if one was not already printed */
if (NULL != error) {
const char *err_msg = opal_strerror(ret);
/* If ORTE was not setup yet, don't use orte_show_help */
if (orte_setup) {
orte_show_help("help-mpi-runtime",
"mpi_init:startup:internal-failure", true,
"MPI_INIT", "MPI_INIT", error, err_msg, ret);
} else {
opal_show_help("help-mpi-runtime",
"mpi_init:startup:internal-failure", true,
"MPI_INIT", "MPI_INIT", error, err_msg, ret);
}
}
return ret;
}
/* Initialize the registered datarep list to be empty */
OBJ_CONSTRUCT(&ompi_registered_datareps, opal_list_t);
/* Initialize the arrays used to store the F90 types returned by the
* MPI_Type_create_f90_XXX functions.
*/
OBJ_CONSTRUCT( &ompi_mpi_f90_integer_hashtable, opal_hash_table_t);
opal_hash_table_init(&ompi_mpi_f90_integer_hashtable, 16 /* why not? */);
OBJ_CONSTRUCT( &ompi_mpi_f90_real_hashtable, opal_hash_table_t);
opal_hash_table_init(&ompi_mpi_f90_real_hashtable, FLT_MAX_10_EXP);
OBJ_CONSTRUCT( &ompi_mpi_f90_complex_hashtable, opal_hash_table_t);
opal_hash_table_init(&ompi_mpi_f90_complex_hashtable, FLT_MAX_10_EXP);
/* All done. Wasn't that simple? */
ompi_mpi_initialized = true;
/* check for timing request - get stop time and report elapsed time if so */
if (timing && 0 == ORTE_PROC_MY_NAME->vpid) {
gettimeofday(&ompistop, NULL);
opal_output(0, "ompi_mpi_init[%ld]: time from barrier to complete mpi_init %ld usec",
(long)ORTE_PROC_MY_NAME->vpid,
(long int)((ompistop.tv_sec - ompistart.tv_sec)*1000000 +
(ompistop.tv_usec - ompistart.tv_usec)));
}
return MPI_SUCCESS;
}
int opal_register_params(void)
{
int value;
/*
* This string is going to be used in opal/util/stacktrace.c
*/
{
char *string = NULL;
int j;
int signals[] = {
#ifdef SIGABRT
SIGABRT,
#endif
#ifdef SIGBUS
SIGBUS,
#endif
#ifdef SIGFPE
SIGFPE,
#endif
#ifdef SIGSEGV
SIGSEGV,
#endif
-1
};
for (j = 0 ; signals[j] != -1 ; ++j) {
if (j == 0) {
asprintf(&string, "%d", signals[j]);
} else {
char *tmp;
asprintf(&tmp, "%s,%d", string, signals[j]);
free(string);
string = tmp;
}
}
mca_base_param_reg_string_name("opal", "signal",
"Comma-delimited list of integer signal numbers to Open MPI to attempt to intercept. Upon receipt of the intercepted signal, Open MPI will display a stack trace and abort. Open MPI will *not* replace signals if handlers are already installed by the time MPI_INIT is invoked. Optionally append \":complain\" to any signal number in the comma-delimited list to make Open MPI complain if it detects another signal handler (and therefore does not insert its own).",
false, false, string, NULL);
free(string);
}
#if OMPI_ENABLE_DEBUG
mca_base_param_reg_int_name("opal", "progress_debug",
"Set to non-zero to debug progress engine features",
false, false, 0, NULL);
{
mca_base_param_reg_int_name("opal", "debug_locks",
"Debug mutex usage within Open MPI. On a "
"non-threaded build, this enables integer counters and "
"warning messages when double-locks are detected.",
false, false, 0, &value);
if (value) opal_mutex_check_locks = true;
}
#endif
/*
* Do we want the "warning: your mmap file is on NFS!" message? Per a
* thread on the OMPI devel list
* (http://www.open-mpi.org/community/lists/devel/2011/12/10054.php),
* on some systems, it doesn't seem to matter. But per older threads,
* it definitely does matter on some systems. Perhaps newer kernels
* are smarter about this kind of stuff...? Regardless, we should
* provide the ability to turn off this message for systems where the
* effect doesn't matter.
*
* v1.4.x-specific note: the MCA param name is "shmem_mmap_...",
* where "shmem" is not a framework that exists in the v1.4
* series. This parameter was added right before 1.4.5, and at a
* similar time as 1.5.5 (where the "shmem" framework *does*
* exist). The idea was to have a consistent MCA param name
* starting with v1.4.5. Hence, we put a slightly non-sensiscal
* name here in v1.4.x so that we'd have a correct/good name
* moving forward.
*/
mca_base_param_reg_int_name("shmem",
"mmap_enable_nfs_warning",
"Enable the warning emitted when Open MPI detects that its shared memory backing file is located on a network filesystem (1 = enabled, 0 = disabled).",
false, false,
(int)true, &value);
opal_mmap_on_nfs_warning = OPAL_INT_TO_BOOL(value);
/* Paffinity base also has some parameters */
return opal_paffinity_base_register_params();
}
int ompi_mpi_init(int argc, char **argv, int requested, int *provided)
{
int ret;
ompi_proc_t** procs;
size_t nprocs;
char *error = NULL;
bool timing = false;
int param, value;
struct timeval ompistart, ompistop;
char *event_val = NULL;
opal_paffinity_base_cpu_set_t mask;
bool proc_bound;
#if 0
/* see comment below about sched_yield */
int num_processors;
#endif
bool orte_setup = false;
bool paffinity_enabled = false;
/* Setup enough to check get/set MCA params */
if (ORTE_SUCCESS != (ret = opal_init_util())) {
error = "ompi_mpi_init: opal_init_util failed";
goto error;
}
/* _After_ opal_init_util() but _before_ orte_init(), we need to
set an MCA param that tells libevent that it's ok to use any
mechanism in libevent that is available on this platform (e.g.,
epoll and friends). Per opal/event/event.s, we default to
select/poll -- but we know that MPI processes won't be using
pty's with the event engine, so it's ok to relax this
constraint and let any fd-monitoring mechanism be used. */
ret = mca_base_param_reg_string_name("opal", "event_include",
"Internal orted MCA param: tell opal_init() to use a specific mechanism in libevent",
false, false, "all", &event_val);
if (ret >= 0) {
/* We have to explicitly "set" the MCA param value here
because libevent initialization will re-register the MCA
param and therefore override the default. Setting the value
here puts the desired value ("all") in different storage
that is not overwritten if/when the MCA param is
re-registered. This is unless the user has specified a different
value for this MCA parameter. Make sure we check to see if the
default is specified before forcing "all" in case that is not what
the user desires. Note that we do *NOT* set this value as an
environment variable, just so that it won't be inherited by
any spawned processes and potentially cause unintented
side-effects with launching ORTE tools... */
if (0 == strcmp("all", event_val)) {
mca_base_param_set_string(ret, "all");
}
}
if( NULL != event_val ) {
free(event_val);
event_val = NULL;
}
/* check to see if we want timing information */
param = mca_base_param_reg_int_name("ompi", "timing",
"Request that critical timing loops be measured",
false, false, 0, &value);
if (value != 0) {
timing = true;
gettimeofday(&ompistart, NULL);
}
/* Setup ORTE - note that we are not a tool */
if (ORTE_SUCCESS != (ret = orte_init(ORTE_NON_TOOL))) {
error = "ompi_mpi_init: orte_init failed";
goto error;
}
orte_setup = true;
/* check for timing request - get stop time and report elapsed time if so */
if (timing && 0 == ORTE_PROC_MY_NAME->vpid) {
gettimeofday(&ompistop, NULL);
opal_output(0, "ompi_mpi_init [%ld]: time from start to completion of orte_init %ld usec",
(long)ORTE_PROC_MY_NAME->vpid,
(long int)((ompistop.tv_sec - ompistart.tv_sec)*1000000 +
(ompistop.tv_usec - ompistart.tv_usec)));
gettimeofday(&ompistart, NULL);
}
/* Figure out the final MPI thread levels. If we were not
compiled for support for MPI threads, then don't allow
MPI_THREAD_MULTIPLE. Set this stuff up here early in the
process so that other components can make decisions based on
this value. */
ompi_mpi_thread_requested = requested;
if (OMPI_HAVE_THREAD_SUPPORT == 0) {
ompi_mpi_thread_provided = *provided = MPI_THREAD_SINGLE;
ompi_mpi_main_thread = NULL;
} else if (OMPI_ENABLE_MPI_THREADS == 1) {
ompi_mpi_thread_provided = *provided = requested;
ompi_mpi_main_thread = opal_thread_get_self();
} else {
if (MPI_THREAD_MULTIPLE == requested) {
ompi_mpi_thread_provided = *provided = MPI_THREAD_SERIALIZED;
} else {
ompi_mpi_thread_provided = *provided = requested;
}
ompi_mpi_main_thread = opal_thread_get_self();
}
ompi_mpi_thread_multiple = (ompi_mpi_thread_provided ==
MPI_THREAD_MULTIPLE);
/* Once we've joined the RTE, see if any MCA parameters were
passed to the MPI level */
if (OMPI_SUCCESS != (ret = ompi_mpi_register_params())) {
error = "mca_mpi_register_params() failed";
goto error;
}
/* if it hasn't already been done, setup process affinity.
* First check to see if a slot list was
* specified. If so, use it. If no slot list was specified,
* that's not an error -- just fall through and try the next
* paffinity scheme.
*/
ret = opal_paffinity_base_get(&mask);
if (OPAL_SUCCESS == ret) {
/* paffinity is supported - check for binding */
OPAL_PAFFINITY_PROCESS_IS_BOUND(mask, &proc_bound);
if (proc_bound) {
/* someone external set it - indicate it is set
* so that we know
*/
paffinity_enabled = true;
} else {
/* the system is capable of doing processor affinity, but it
* has not yet been set - see if a slot_list was given
*/
if (NULL != opal_paffinity_base_slot_list) {
/* It's an error if multiple paffinity schemes were specified */
if (opal_paffinity_alone) {
ret = OMPI_ERR_BAD_PARAM;
error = "Multiple processor affinity schemes specified (can only specify one)";
goto error;
}
ret = opal_paffinity_base_slot_list_set((long)ORTE_PROC_MY_NAME->vpid, opal_paffinity_base_slot_list);
if (OPAL_ERR_NOT_FOUND != ret) {
error = "opal_paffinity_base_slot_list_set() returned an error";
goto error;
}
paffinity_enabled = true;
} else if (opal_paffinity_alone) {
/* no slot_list, but they asked for paffinity */
int phys_cpu;
orte_node_rank_t nrank;
if (ORTE_NODE_RANK_INVALID == (nrank = orte_ess.get_node_rank(ORTE_PROC_MY_NAME))) {
error = "Could not get node rank - cannot set processor affinity";
goto error;
}
OPAL_PAFFINITY_CPU_ZERO(mask);
phys_cpu = opal_paffinity_base_get_physical_processor_id(nrank);
if (0 > phys_cpu) {
error = "Could not get physical processor id - cannot set processor affinity";
goto error;
}
OPAL_PAFFINITY_CPU_SET(phys_cpu, mask);
ret = opal_paffinity_base_set(mask);
if (OPAL_SUCCESS != ret) {
error = "Setting processor affinity failed";
goto error;
}
paffinity_enabled = true;
}
}
}
/* If we were able to set processor affinity, try setting up
memory affinity */
if (!opal_maffinity_setup && paffinity_enabled) {
if (OPAL_SUCCESS == opal_maffinity_base_open() &&
OPAL_SUCCESS == opal_maffinity_base_select()) {
opal_maffinity_setup = true;
}
}
/* initialize datatypes. This step should be done early as it will
* create the local convertor and local arch used in the proc
* init.
*/
if (OMPI_SUCCESS != (ret = ompi_ddt_init())) {
error = "ompi_ddt_init() failed";
goto error;
}
/* Initialize OMPI procs */
if (OMPI_SUCCESS != (ret = ompi_proc_init())) {
error = "mca_proc_init() failed";
goto error;
}
/* initialize ops. This has to be done *after* ddt_init, but
befor mca_coll_base_open, since come collective modules
(e.g. the hierarchical) need them in the query function
*/
if (OMPI_SUCCESS != (ret = ompi_op_init())) {
error = "ompi_op_init() failed";
goto error;
}
/* Open up MPI-related MCA components */
if (OMPI_SUCCESS != (ret = mca_allocator_base_open())) {
error = "mca_allocator_base_open() failed";
goto error;
}
if (OMPI_SUCCESS != (ret = mca_rcache_base_open())) {
error = "mca_rcache_base_open() failed";
goto error;
}
if (OMPI_SUCCESS != (ret = mca_mpool_base_open())) {
error = "mca_mpool_base_open() failed";
goto error;
}
if (OMPI_SUCCESS != (ret = mca_pml_base_open())) {
error = "mca_pml_base_open() failed";
goto error;
}
if (OMPI_SUCCESS != (ret = mca_coll_base_open())) {
error = "mca_coll_base_open() failed";
goto error;
}
if (OMPI_SUCCESS != (ret = ompi_osc_base_open())) {
error = "ompi_osc_base_open() failed";
goto error;
}
#if OPAL_ENABLE_FT == 1
if (OMPI_SUCCESS != (ret = ompi_crcp_base_open())) {
error = "ompi_crcp_base_open() failed";
goto error;
}
#endif
/* In order to reduce the common case for MPI apps (where they
don't use MPI-2 IO or MPI-1 topology functions), the io and
topo frameworks are initialized lazily, at the first use of
relevant functions (e.g., MPI_FILE_*, MPI_CART_*, MPI_GRAPH_*),
so they are not opened here. */
/* Select which MPI components to use */
if (OMPI_SUCCESS !=
(ret = mca_mpool_base_init(OMPI_ENABLE_PROGRESS_THREADS,
OMPI_ENABLE_MPI_THREADS))) {
error = "mca_mpool_base_init() failed";
goto error;
}
if (OMPI_SUCCESS !=
(ret = mca_pml_base_select(OMPI_ENABLE_PROGRESS_THREADS,
OMPI_ENABLE_MPI_THREADS))) {
error = "mca_pml_base_select() failed";
goto error;
}
/* select buffered send allocator component to be used */
ret=mca_pml_base_bsend_init(OMPI_ENABLE_MPI_THREADS);
if( OMPI_SUCCESS != ret ) {
error = "mca_pml_base_bsend_init() failed";
goto error;
}
if (OMPI_SUCCESS !=
(ret = mca_coll_base_find_available(OMPI_ENABLE_PROGRESS_THREADS,
OMPI_ENABLE_MPI_THREADS))) {
error = "mca_coll_base_find_available() failed";
goto error;
}
if (OMPI_SUCCESS !=
(ret = ompi_osc_base_find_available(OMPI_ENABLE_PROGRESS_THREADS,
OMPI_ENABLE_MPI_THREADS))) {
error = "ompi_osc_base_find_available() failed";
goto error;
}
#if OPAL_ENABLE_FT == 1
if (OMPI_SUCCESS != (ret = ompi_crcp_base_select() ) ) {
error = "ompi_crcp_base_select() failed";
goto error;
}
#endif
/* io and topo components are not selected here -- see comment
above about the io and topo frameworks being loaded lazily */
/* Initialize each MPI handle subsystem */
/* initialize requests */
if (OMPI_SUCCESS != (ret = ompi_request_init())) {
error = "ompi_request_init() failed";
goto error;
}
/* initialize info */
if (OMPI_SUCCESS != (ret = ompi_info_init())) {
error = "ompi_info_init() failed";
goto error;
}
/* initialize error handlers */
if (OMPI_SUCCESS != (ret = ompi_errhandler_init())) {
error = "ompi_errhandler_init() failed";
goto error;
}
/* initialize error codes */
if (OMPI_SUCCESS != (ret = ompi_mpi_errcode_init())) {
error = "ompi_mpi_errcode_init() failed";
goto error;
}
/* initialize internal error codes */
if (OMPI_SUCCESS != (ret = ompi_errcode_intern_init())) {
error = "ompi_errcode_intern_init() failed";
goto error;
}
/* initialize groups */
if (OMPI_SUCCESS != (ret = ompi_group_init())) {
error = "ompi_group_init() failed";
goto error;
}
/* initialize communicators */
if (OMPI_SUCCESS != (ret = ompi_comm_init())) {
error = "ompi_comm_init() failed";
goto error;
}
/* initialize file handles */
if (OMPI_SUCCESS != (ret = ompi_file_init())) {
error = "ompi_file_init() failed";
goto error;
}
/* initialize windows */
if (OMPI_SUCCESS != (ret = ompi_win_init())) {
error = "ompi_win_init() failed";
goto error;
}
/* initialize attribute meta-data structure for comm/win/dtype */
if (OMPI_SUCCESS != (ret = ompi_attr_init())) {
error = "ompi_attr_init() failed";
goto error;
}
/* check for timing request - get stop time and report elapsed time if so */
if (timing && 0 == ORTE_PROC_MY_NAME->vpid) {
gettimeofday(&ompistop, NULL);
opal_output(0, "ompi_mpi_init[%ld]: time from completion of orte_init to modex %ld usec",
(long)ORTE_PROC_MY_NAME->vpid,
(long int)((ompistop.tv_sec - ompistart.tv_sec)*1000000 +
(ompistop.tv_usec - ompistart.tv_usec)));
gettimeofday(&ompistart, NULL);
}
/* exchange connection info - this function also acts as a barrier
* as it will not return until the exchange is complete
*/
if (OMPI_SUCCESS != (ret = orte_grpcomm.modex(NULL))) {
error = "orte_grpcomm_modex failed";
goto error;
}
if (timing && 0 == ORTE_PROC_MY_NAME->vpid) {
gettimeofday(&ompistop, NULL);
opal_output(0, "ompi_mpi_init[%ld]: time to execute modex %ld usec",
(long)ORTE_PROC_MY_NAME->vpid,
(long int)((ompistop.tv_sec - ompistart.tv_sec)*1000000 +
(ompistop.tv_usec - ompistart.tv_usec)));
gettimeofday(&ompistart, NULL);
}
/* identify the architectures of remote procs and setup
* their datatype convertors, if required
*/
if (OMPI_SUCCESS != (ret = ompi_proc_set_arch())) {
error = "ompi_proc_set_arch failed";
goto error;
}
/* If thread support was enabled, then setup OPAL to allow for
them. */
if ((OMPI_ENABLE_PROGRESS_THREADS == 1) ||
(*provided != MPI_THREAD_SINGLE)) {
opal_set_using_threads(true);
}
/* start PML/BTL's */
ret = MCA_PML_CALL(enable(true));
if( OMPI_SUCCESS != ret ) {
error = "PML control failed";
goto error;
}
/* add all ompi_proc_t's to PML */
if (NULL == (procs = ompi_proc_world(&nprocs))) {
error = "ompi_proc_world() failed";
goto error;
}
ret = MCA_PML_CALL(add_procs(procs, nprocs));
free(procs);
if( OMPI_SUCCESS != ret ) {
error = "PML add procs failed";
goto error;
}
MCA_PML_CALL(add_comm(&ompi_mpi_comm_world.comm));
MCA_PML_CALL(add_comm(&ompi_mpi_comm_self.comm));
/*
* Dump all MCA parameters if requested
*/
if (ompi_mpi_show_mca_params) {
ompi_show_all_mca_params(ompi_mpi_comm_world.comm.c_my_rank,
nprocs,
orte_process_info.nodename);
}
/* Do we need to wait for a debugger? */
ompi_wait_for_debugger();
/* check for timing request - get stop time and report elapsed
time if so, then start the clock again */
if (timing && 0 == ORTE_PROC_MY_NAME->vpid) {
gettimeofday(&ompistop, NULL);
opal_output(0, "ompi_mpi_init[%ld]: time from modex thru complete oob wireup %ld usec",
(long)ORTE_PROC_MY_NAME->vpid,
(long int)((ompistop.tv_sec - ompistart.tv_sec)*1000000 +
(ompistop.tv_usec - ompistart.tv_usec)));
gettimeofday(&ompistart, NULL);
}
/* wait for everyone to reach this point */
if (OMPI_SUCCESS != (ret = orte_grpcomm.barrier())) {
error = "orte_grpcomm_barrier failed";
goto error;
}
/* check for timing request - get stop time and report elapsed
time if so, then start the clock again */
if (timing && 0 == ORTE_PROC_MY_NAME->vpid) {
gettimeofday(&ompistop, NULL);
opal_output(0, "ompi_mpi_init[%ld]: time to execute barrier %ld usec",
(long)ORTE_PROC_MY_NAME->vpid,
(long int)((ompistop.tv_sec - ompistart.tv_sec)*1000000 +
(ompistop.tv_usec - ompistart.tv_usec)));
gettimeofday(&ompistart, NULL);
}
#if OMPI_ENABLE_PROGRESS_THREADS == 0
/* Start setting up the event engine for MPI operations. Don't
block in the event library, so that communications don't take
forever between procs in the dynamic code. This will increase
CPU utilization for the remainder of MPI_INIT when we are
blocking on ORTE-level events, but may greatly reduce non-TCP
latency. */
opal_progress_set_event_flag(OPAL_EVLOOP_NONBLOCK);
#endif
/* wire up the mpi interface, if requested. Do this after the
non-block switch for non-TCP performance. Do before the
polling change as anyone with a complex wire-up is going to be
using the oob. */
if (OMPI_SUCCESS != (ret = ompi_init_preconnect_mpi())) {
error = "ompi_mpi_do_preconnect_all() failed";
goto error;
}
/* Setup the publish/subscribe (PUBSUB) framework */
if (OMPI_SUCCESS != (ret = ompi_pubsub_base_open())) {
error = "ompi_pubsub_base_open() failed";
goto error;
}
if (OMPI_SUCCESS != (ret = ompi_pubsub_base_select())) {
error = "ompi_pubsub_base_select() failed";
goto error;
}
/* Setup the dynamic process management (DPM) framework */
if (OMPI_SUCCESS != (ret = ompi_dpm_base_open())) {
error = "ompi_dpm_base_open() failed";
goto error;
}
if (OMPI_SUCCESS != (ret = ompi_dpm_base_select())) {
error = "ompi_dpm_base_select() failed";
goto error;
}
/* Init coll for the comms. This has to be after dpm_base_select,
(since dpm.mark_dyncomm is not set in the communicator creation
function else), but before dpm.dyncom_init, since this function
might require collective for the CID allocation. */
if (OMPI_SUCCESS !=
(ret = mca_coll_base_comm_select(MPI_COMM_WORLD))) {
error = "mca_coll_base_comm_select(MPI_COMM_WORLD) failed";
goto error;
}
if (OMPI_SUCCESS !=
(ret = mca_coll_base_comm_select(MPI_COMM_SELF))) {
error = "mca_coll_base_comm_select(MPI_COMM_SELF) failed";
goto error;
}
/* Check whether we have been spawned or not. We introduce that
at the very end, since we need collectives, datatypes, ptls
etc. up and running here.... */
if (OMPI_SUCCESS != (ret = ompi_dpm.dyn_init())) {
error = "ompi_comm_dyn_init() failed";
goto error;
}
/*
* Startup the Checkpoint/Restart Mech.
* Note: Always do this so tools don't hang when
* in a non-checkpointable build
*/
if (OMPI_SUCCESS != (ret = ompi_cr_init())) {
error = "ompi_cr_init";
goto error;
}
/* Undo OPAL calling opal_progress_event_users_increment() during
opal_init, to get better latency when not using TCP. Do
this *after* dyn_init, as dyn init uses lots of ORTE
communication and we don't want to hinder the performance of
that code. */
opal_progress_event_users_decrement();
/* see if yield_when_idle was specified - if so, use it */
param = mca_base_param_find("mpi", NULL, "yield_when_idle");
mca_base_param_lookup_int(param, &value);
if (value < 0) {
/* if no info is provided, just default to conservative */
opal_progress_set_yield_when_idle(true);
} else {
/* info was provided, so set idle accordingly */
opal_progress_set_yield_when_idle(value == 0 ? false : true);
}
param = mca_base_param_find("mpi", NULL, "event_tick_rate");
mca_base_param_lookup_int(param, &value);
/* negative value means use default - just don't do anything */
if (value >= 0) {
opal_progress_set_event_poll_rate(value);
}
/* At this point, we are fully configured and in MPI mode. Any
communication calls here will work exactly like they would in
the user's code. Setup the connections between procs and warm
them up with simple sends, if requested */
error:
if (ret != OMPI_SUCCESS) {
const char *err_msg = opal_strerror(ret);
/* If ORTE was not setup yet, don't use orte_show_help */
if (orte_setup) {
orte_show_help("help-mpi-runtime",
"mpi_init:startup:internal-failure", true,
"MPI_INIT", "MPI_INIT", error, err_msg, ret);
} else {
opal_show_help("help-mpi-runtime",
"mpi_init:startup:internal-failure", true,
"MPI_INIT", "MPI_INIT", error, err_msg, ret);
}
return ret;
}
/* Initialize the registered datarep list to be empty */
OBJ_CONSTRUCT(&ompi_registered_datareps, opal_list_t);
/* Initialize the arrays used to store the F90 types returned by the
* MPI_Type_create_f90_XXX functions.
*/
OBJ_CONSTRUCT( &ompi_mpi_f90_integer_hashtable, opal_hash_table_t);
opal_hash_table_init(&ompi_mpi_f90_integer_hashtable, 16 /* why not? */);
OBJ_CONSTRUCT( &ompi_mpi_f90_real_hashtable, opal_hash_table_t);
opal_hash_table_init(&ompi_mpi_f90_real_hashtable, FLT_MAX_10_EXP);
OBJ_CONSTRUCT( &ompi_mpi_f90_complex_hashtable, opal_hash_table_t);
opal_hash_table_init(&ompi_mpi_f90_complex_hashtable, FLT_MAX_10_EXP);
/* All done. Wasn't that simple? */
ompi_mpi_initialized = true;
/* check for timing request - get stop time and report elapsed time if so */
if (timing && 0 == ORTE_PROC_MY_NAME->vpid) {
gettimeofday(&ompistop, NULL);
opal_output(0, "ompi_mpi_init[%ld]: time from barrier p to complete mpi_init %ld usec",
(long)ORTE_PROC_MY_NAME->vpid,
(long int)((ompistop.tv_sec - ompistart.tv_sec)*1000000 +
(ompistop.tv_usec - ompistart.tv_usec)));
}
return MPI_SUCCESS;
}
/**
* Function for finding and opening either all MCA components, or the
* one that was specifically requested via a MCA parameter.
*/
int mca_base_components_open(const char *type_name, int output_id,
const mca_base_component_t **static_components,
opal_list_t *components_available,
bool open_dso_components)
{
int ret, param;
opal_list_item_t *item;
opal_list_t components_found;
char **requested_component_names;
int param_verbose = -1;
int param_type = -1;
int verbose_level;
char *str;
bool include_mode;
#if (OPAL_ENABLE_FT == 1) && (OPAL_ENABLE_FT_CR == 1)
opal_list_item_t *next;
uint32_t open_only_flags = MCA_BASE_METADATA_PARAM_NONE;
const mca_base_component_t *component;
#endif
/* Register MCA parameters */
/* Check to see if it exists first */
if( 0 > (param_type = mca_base_param_find(type_name, NULL, NULL) ) ) {
asprintf(&str, "Default selection set of components for the %s framework (<none>"
" means use all components that can be found)", type_name);
param_type =
mca_base_param_reg_string_name(type_name, NULL, str,
false, false, NULL, NULL);
free(str);
}
param = mca_base_param_find("mca", NULL, "component_show_load_errors");
mca_base_param_lookup_int(param, &ret);
show_errors = OPAL_INT_TO_BOOL(ret);
/* Setup verbosity for this MCA type */
asprintf(&str, "Verbosity level for the %s framework (0 = no verbosity)", type_name);
param_verbose =
mca_base_param_reg_int_name(type_name, "base_verbose",
str, false, false, 0, NULL);
free(str);
mca_base_param_lookup_int(param_verbose, &verbose_level);
if (output_id != 0) {
opal_output_set_verbosity(output_id, verbose_level);
}
opal_output_verbose(10, output_id,
"mca: base: components_open: Looking for %s components",
type_name);
ret = parse_requested(param_type, &include_mode, &requested_component_names);
if( OPAL_SUCCESS != ret ) {
return ret;
}
/* Find and load requested components */
if (OPAL_SUCCESS != (ret =
mca_base_component_find(NULL, type_name, static_components,
requested_component_names, include_mode,
&components_found, open_dso_components)) ) {
return ret;
}
#if (OPAL_ENABLE_FT == 1) && (OPAL_ENABLE_FT_CR == 1)
{
int param_id = -1;
int param_val = 0;
/*
* Extract supported mca parameters for selection contraints
* Supported Options:
* - mca_base_component_distill_checkpoint_ready = Checkpoint Ready
*/
param_id = mca_base_param_reg_int_name("mca", "base_component_distill_checkpoint_ready",
"Distill only those components that are Checkpoint Ready",
false, false,
0, ¶m_val);
if( 0 != param_val ) { /* Select Checkpoint Ready */
open_only_flags |= MCA_BASE_METADATA_PARAM_CHECKPOINT;
}
}
#endif /* (OPAL_ENABLE_FT == 1) && (OPAL_ENABLE_FT_CR == 1) */
/*
* Pre-process the list with parameter constraints
* e.g., If requested to select only CR enabled components
* then only make available those components.
*
* JJH Note: Currently checkpoint/restart is the only user of this
* functionality. If other component constraint options are
* added, then this logic can be used for all contraint
* options.
*/
#if (OPAL_ENABLE_FT == 1) && (OPAL_ENABLE_FT_CR == 1)
if( !(MCA_BASE_METADATA_PARAM_NONE & open_only_flags) ) {
if( MCA_BASE_METADATA_PARAM_CHECKPOINT & open_only_flags) {
opal_output_verbose(10, output_id,
"mca: base: components_open: "
"including only %s components that are checkpoint enabled", type_name);
}
/*
* Check all the components to make sure they adhere to the user
* expressed requirements.
*/
for(item = opal_list_get_first(&components_found);
item != opal_list_get_end(&components_found);
item = next ) {
mca_base_open_only_dummy_component_t *dummy;
mca_base_component_list_item_t *cli = (mca_base_component_list_item_t *) item;
dummy = (mca_base_open_only_dummy_component_t*) cli->cli_component;
component = cli->cli_component;
next = opal_list_get_next(item);
/*
* If the user asked for a checkpoint enabled run
* then only load checkpoint enabled components.
*/
if( MCA_BASE_METADATA_PARAM_CHECKPOINT & open_only_flags) {
if( MCA_BASE_METADATA_PARAM_CHECKPOINT & dummy->data.param_field) {
opal_output_verbose(10, output_id,
"mca: base: components_open: "
"(%s) Component %s is Checkpointable",
type_name,
dummy->version.mca_component_name);
}
else {
opal_output_verbose(10, output_id,
"mca: base: components_open: "
"(%s) Component %s is *NOT* Checkpointable - Disabled",
type_name,
dummy->version.mca_component_name);
opal_list_remove_item(&components_found, item);
/* Make sure to release the component since we are not
* opening it */
mca_base_component_repository_release(component);
}
}
}
}
#endif /* (OPAL_ENABLE_FT == 1) && (OPAL_ENABLE_FT_CR == 1) */
/* Open all remaining components */
ret = open_components(type_name, output_id,
&components_found, components_available);
/* Free resources */
for (item = opal_list_remove_first(&components_found); NULL != item;
item = opal_list_remove_first(&components_found)) {
OBJ_RELEASE(item);
}
OBJ_DESTRUCT(&components_found);
if (NULL != requested_component_names) {
opal_argv_free(requested_component_names);
}
/* All done */
return ret;
}
int opal_register_params(void)
{
int ret;
/*
* This string is going to be used in opal/util/stacktrace.c
*/
{
char *string = NULL;
int j;
int signals[] = {
#ifdef SIGABRT
SIGABRT,
#endif
#ifdef SIGBUS
SIGBUS,
#endif
#ifdef SIGFPE
SIGFPE,
#endif
#ifdef SIGSEGV
SIGSEGV,
#endif
-1
};
for (j = 0 ; signals[j] != -1 ; ++j) {
if (j == 0) {
asprintf(&string, "%d", signals[j]);
} else {
char *tmp;
asprintf(&tmp, "%s,%d", string, signals[j]);
free(string);
string = tmp;
}
}
mca_base_param_reg_string_name("opal", "signal",
"Comma-delimited list of integer signal numbers to Open MPI to attempt to intercept. Upon receipt of the intercepted signal, Open MPI will display a stack trace and abort. Open MPI will *not* replace signals if handlers are already installed by the time MPI_INIT is invoked. Optionally append \":complain\" to any signal number in the comma-delimited list to make Open MPI complain if it detects another signal handler (and therefore does not insert its own).",
false, false, string, NULL);
free(string);
}
{
int j;
mca_base_param_reg_int_name("opal", "profile",
"Set to non-zero to profile component selections",
false, false, (int)false, &j);
opal_profile = OPAL_INT_TO_BOOL(j);
mca_base_param_reg_string_name("opal", "profile_file",
"Name of the file containing the cluster configuration information",
false, false, NULL, &opal_profile_file);
}
#if OPAL_ENABLE_DEBUG
mca_base_param_reg_int_name("opal", "progress_debug",
"Set to non-zero to debug progress engine features",
false, false, 0, NULL);
{
int value;
mca_base_param_reg_int_name("opal", "debug_locks",
"Debug mutex usage within Open MPI. On a "
"non-threaded build, this enables integer counters and "
"warning messages when double-locks are detected.",
false, false, 0, &value);
if (value) opal_mutex_check_locks = true;
}
#endif
/* The ddt engine has a few parameters */
ret = opal_datatype_register_params();
if (OPAL_SUCCESS != ret) {
return ret;
}
/* Paffinity base also has some parameters */
return opal_paffinity_base_register_params();
}
int ompi_mpi_register_params(void)
{
int value;
char *param;
/* Whether we want MPI API function parameter checking or not */
mca_base_param_reg_int_name("mpi", "param_check",
"Whether you want MPI API parameters checked at run-time or not. Possible values are 0 (no checking) and 1 (perform checking at run-time)",
false, false, MPI_PARAM_CHECK, &value);
ompi_mpi_param_check = OPAL_INT_TO_BOOL(value);
if (ompi_mpi_param_check) {
value = 0;
if (MPI_PARAM_CHECK) {
value = 1;
}
if (0 == value) {
orte_show_help("help-mpi-runtime.txt",
"mpi-param-check-enabled-but-compiled-out",
true);
ompi_mpi_param_check = false;
}
}
/*
* opal_progress: decide whether to yield and the event library
* tick rate
*/
/* JMS: Need ORTE data here -- set this to 0 when
exactly/under-subscribed, or 1 when oversubscribed */
mca_base_param_reg_int_name("mpi", "yield_when_idle",
"Yield the processor when waiting for MPI communication (for MPI processes, will default to 1 when oversubscribing nodes)",
false, false, -1, NULL);
mca_base_param_reg_int_name("mpi", "event_tick_rate",
"How often to progress TCP communications (0 = never, otherwise specified in microseconds)",
false, false, -1, NULL);
/* Whether or not to show MPI handle leaks */
mca_base_param_reg_int_name("mpi", "show_handle_leaks",
"Whether MPI_FINALIZE shows all MPI handles that were not freed or not",
false, false,
(int) ompi_debug_show_handle_leaks, &value);
ompi_debug_show_handle_leaks = OPAL_INT_TO_BOOL(value);
/* Whether or not to free MPI handles. Useless without run-time
param checking, so implicitly set that to true if we don't want
to free the handles. */
mca_base_param_reg_int_name("mpi", "no_free_handles",
"Whether to actually free MPI objects when their handles are freed",
false, false,
(int) ompi_debug_no_free_handles, &value);
ompi_debug_no_free_handles = OPAL_INT_TO_BOOL(value);
if (ompi_debug_no_free_handles) {
ompi_mpi_param_check = true;
value = 0;
if (MPI_PARAM_CHECK) {
value = 1;
}
if (0 == value) {
opal_output(0, "WARNING: MCA parameter mpi_no_free_handles set to true, but MPI");
opal_output(0, "WARNING: parameter checking has been compiled out of Open MPI.");
opal_output(0, "WARNING: mpi_no_free_handles is therefore only partially effective!");
}
}
/* Whether or not to show MPI_ALLOC_MEM leaks */
mca_base_param_reg_int_name("mpi", "show_mpi_alloc_mem_leaks",
"If >0, MPI_FINALIZE will show up to this many instances of memory allocated by MPI_ALLOC_MEM that was not freed by MPI_FREE_MEM",
false, false,
ompi_debug_show_mpi_alloc_mem_leaks,
&ompi_debug_show_mpi_alloc_mem_leaks);
/* Whether or not to print all MCA parameters in MPI_INIT */
mca_base_param_reg_string_name("mpi", "show_mca_params",
"Whether to show all MCA parameter values during MPI_INIT or not (good for reproducability of MPI jobs "
"for debug purposes). Accepted values are all, default, file, api, and enviro - or a comma "
"delimited combination of them",
false, false, NULL, ¶m);
if (NULL != param) {
char **args;
int i;
ompi_mpi_show_mca_params = true;
args = opal_argv_split(param, ',');
if (NULL == args) {
opal_output(0, "WARNING: could not parse mpi_show_mca_params request - defaulting to show \"all\"");
show_default_mca_params = true;
show_file_mca_params = true;
show_enviro_mca_params = true;
show_override_mca_params = true;
} else {
for (i=0; NULL != args[i]; i++) {
if (0 == strcasecmp(args[i], "all") || 0 == strcmp(args[i], "1")) {
show_default_mca_params = true;
show_file_mca_params = true;
show_enviro_mca_params = true;
show_override_mca_params = true;
} else if (0 == strcasecmp(args[i], "default")) {
show_default_mca_params = true;
} else if (0 == strcasecmp(args[i], "file")) {
show_file_mca_params = true;
} else if (0 == strcasecmp(args[i], "enviro") ||
0 == strcasecmp(args[i], "env")) {
show_enviro_mca_params = true;
} else if (0 == strcasecmp(args[i], "api")) {
show_override_mca_params = true;
}
}
opal_argv_free(args);
}
}
/* File to use when dumping the parameters */
mca_base_param_reg_string_name("mpi", "show_mca_params_file",
"If mpi_show_mca_params is true, setting this string to a valid filename tells Open MPI to dump all the MCA parameter values into a file suitable for reading via the mca_param_files parameter (good for reproducability of MPI jobs)",
false, false,
"", &ompi_mpi_show_mca_params_file);
/* User-level process pinning controls */
/* Do we want to save hostnames for debugging messages? This can
eat quite a bit of memory... */
mca_base_param_reg_int_name("mpi", "keep_peer_hostnames",
"If nonzero, save the string hostnames of all MPI peer processes (mostly for error / debugging output messages). This can add quite a bit of memory usage to each MPI process.",
false, false, 1, &value);
ompi_mpi_keep_peer_hostnames = OPAL_INT_TO_BOOL(value);
/* MPI_ABORT controls */
mca_base_param_reg_int_name("mpi", "abort_delay",
"If nonzero, print out an identifying message when MPI_ABORT is invoked (hostname, PID of the process that called MPI_ABORT) and delay for that many seconds before exiting (a negative delay value means to never abort). This allows attaching of a debugger before quitting the job.",
false, false,
ompi_mpi_abort_delay,
&ompi_mpi_abort_delay);
mca_base_param_reg_int_name("mpi", "abort_print_stack",
"If nonzero, print out a stack trace when MPI_ABORT is invoked",
false,
/* If we do not have stack trace
capability, make this a read-only
MCA param */
#if OMPI_WANT_PRETTY_PRINT_STACKTRACE && ! defined(__WINDOWS__) && defined(HAVE_BACKTRACE)
false,
#else
true,
#endif
(int) ompi_mpi_abort_print_stack,
&value);
#if OMPI_WANT_PRETTY_PRINT_STACKTRACE && ! defined(__WINDOWS__) && defined(HAVE_BACKTRACE)
/* Only take the value if we have stack trace capability */
ompi_mpi_abort_print_stack = OPAL_INT_TO_BOOL(value);
#else
/* If we do not have stack trace capability, ensure that this is
hard-coded to false */
ompi_mpi_abort_print_stack = false;
#endif
value = mca_base_param_reg_int_name("mpi", "preconnect_mpi",
"Whether to force MPI processes to fully "
"wire-up the MPI connections between MPI "
"processes during "
"MPI_INIT (vs. making connections lazily -- "
"upon the first MPI traffic between each "
"process peer pair)",
false, false, 0, NULL);
mca_base_param_reg_syn_name(value, "mpi", "preconnect_all", true);
/* Leave pinned parameter */
mca_base_param_reg_int_name("mpi", "leave_pinned",
"Whether to use the \"leave pinned\" protocol or not. Enabling this setting can help bandwidth performance when repeatedly sending and receiving large messages with the same buffers over RDMA-based networks (0 = do not use \"leave pinned\" protocol, 1 = use \"leave pinned\" protocol, -1 = allow network to choose at runtime).",
false, false,
ompi_mpi_leave_pinned, &value);
ompi_mpi_leave_pinned = (value >= 1) ? true: false;
mca_base_param_reg_int_name("mpi", "leave_pinned_pipeline",
"Whether to use the \"leave pinned pipeline\" protocol or not.",
false, false,
(int) ompi_mpi_leave_pinned_pipeline, &value);
ompi_mpi_leave_pinned_pipeline = OPAL_INT_TO_BOOL(value);
if (ompi_mpi_leave_pinned && ompi_mpi_leave_pinned_pipeline) {
ompi_mpi_leave_pinned_pipeline = 0;
orte_show_help("help-mpi-runtime.txt",
"mpi-params:leave-pinned-and-pipeline-selected",
true);
}
mca_base_param_reg_int_name("mpi", "warn_on_fork",
"If nonzero, issue a warning if program forks under conditions that could cause system errors",
false, false,
(int) true, &value);
ompi_warn_on_fork = OPAL_INT_TO_BOOL(value);
/* Sparse group storage support */
mca_base_param_reg_int_name("mpi", "have_sparse_group_storage",
"Whether this Open MPI installation supports storing of data in MPI groups in \"sparse\" formats (good for extremely large process count MPI jobs that create many communicators/groups)",
false, true, (int) OMPI_GROUP_SPARSE, NULL);
mca_base_param_reg_int_name("mpi", "use_sparse_group_storage",
"Whether to use \"sparse\" storage formats for MPI groups (only relevant if mpi_have_sparse_group_storage is 1)",
false, false, OMPI_GROUP_SPARSE, &value);
ompi_use_sparse_group_storage = OPAL_INT_TO_BOOL(value);
if (ompi_use_sparse_group_storage) {
value = 0;
if (OMPI_GROUP_SPARSE) {
value = 1;
}
if (0 == value) {
orte_show_help("help-mpi-runtime.txt",
"sparse groups enabled but compiled out",
true);
ompi_use_sparse_group_storage = false;
}
}
/* The ddt engine has a few parameters */
return ompi_ddt_register_params();
}
int orte_register_params(void)
{
int id;
opal_output_stream_t lds;
/* only go thru this once - mpirun calls it twice, which causes
* any error messages to show up twice
*/
if (passed_thru) {
return ORTE_SUCCESS;
}
passed_thru = true;
/* get a clean output channel too - need to do this here because
* we use it below, and orterun and some other tools call this
* function prior to calling orte_init
*/
OBJ_CONSTRUCT(&lds, opal_output_stream_t);
lds.lds_want_stdout = true;
orte_clean_output = opal_output_open(&lds);
OBJ_DESTRUCT(&lds);
orte_help_want_aggregate = true;
(void) mca_base_var_register ("orte", "orte", "base", "help_aggregate",
"If orte_base_help_aggregate is true, duplicate help messages will be aggregated rather than displayed individually. This can be helpful for parallel jobs that experience multiple identical failures; rather than print out the same help/failure message N times, display it once with a count of how many processes sent the same message.",
MCA_BASE_VAR_TYPE_BOOL, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_ALL_EQ,
&orte_help_want_aggregate);
/* LOOK FOR A TMP DIRECTORY BASE */
/* Several options are provided to cover a range of possibilities:
*
* (a) all processes need to use a specified location as the base
* for tmp directories
* (b) daemons on remote nodes need to use a specified location, but
* one different from that used by mpirun
* (c) mpirun needs to use a specified location, but one different
* from that used on remote nodes
*/
orte_tmpdir_base = NULL;
(void) mca_base_var_register ("orte", "orte", NULL, "tmpdir_base",
"Base of the session directory tree to be used by all processes",
MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_ALL_EQ,
&orte_tmpdir_base);
orte_local_tmpdir_base = NULL;
(void) mca_base_var_register ("orte", "orte", NULL, "local_tmpdir_base",
"Base of the session directory tree to be used by orterun/mpirun",
MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_ALL_EQ,
&orte_local_tmpdir_base);
orte_remote_tmpdir_base = NULL;
(void) mca_base_var_register ("orte", "orte", NULL, "remote_tmpdir_base",
"Base of the session directory tree on remote nodes, if required to be different from head node",
MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_ALL_EQ,
&orte_remote_tmpdir_base);
/* if a global tmpdir was specified, then we do not allow specification
* of the local or remote values to avoid confusion
*/
if (NULL != orte_tmpdir_base &&
(NULL != orte_local_tmpdir_base || NULL != orte_remote_tmpdir_base)) {
opal_output(orte_clean_output,
"------------------------------------------------------------------\n"
"The MCA param orte_tmpdir_base was specified, which sets the base\n"
"of the temporary directory tree for all procs. However, values for\n"
"the local and/or remote tmpdir base were also given. This can lead\n"
"to confusion and is therefore not allowed. Please specify either a\n"
"global tmpdir base OR a local/remote tmpdir base value\n"
"------------------------------------------------------------------");
exit(1);
}
if (NULL != orte_tmpdir_base) {
if (NULL != orte_process_info.tmpdir_base) {
free(orte_process_info.tmpdir_base);
}
orte_process_info.tmpdir_base = strdup (orte_tmpdir_base);
} else if (ORTE_PROC_IS_HNP && NULL != orte_local_tmpdir_base) {
/* orterun will pickup the value for its own use */
if (NULL != orte_process_info.tmpdir_base) {
free(orte_process_info.tmpdir_base);
}
orte_process_info.tmpdir_base = strdup (orte_local_tmpdir_base);
} else if (ORTE_PROC_IS_DAEMON && NULL != orte_remote_tmpdir_base) {
/* orterun will pickup the value and forward it along, but must not
* use it in its own work. So only a daemon needs to get it, and the
* daemon will pass it down to its application procs. Note that orterun
* will pass -its- value to any procs local to it
*/
if (NULL != orte_process_info.tmpdir_base) {
free(orte_process_info.tmpdir_base);
}
orte_process_info.tmpdir_base = strdup (orte_remote_tmpdir_base);
}
orte_top_session_dir = NULL;
(void) mca_base_var_register ("orte", "orte", NULL, "top_session_dir",
"Top of the session directory tree for applications",
MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_ALL_EQ,
&orte_top_session_dir);
if (NULL != orte_top_session_dir) {
if (NULL != orte_process_info.top_session_dir) {
free(orte_process_info.top_session_dir);
}
orte_process_info.top_session_dir = strdup(orte_top_session_dir);
}
orte_jobfam_session_dir = NULL;
(void) mca_base_var_register ("orte", "orte", NULL, "jobfam_session_dir",
"The jobfamily session directory for applications",
MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_ALL_EQ,
&orte_jobfam_session_dir);
if (NULL != orte_jobfam_session_dir) {
if (NULL != orte_process_info.jobfam_session_dir) {
free(orte_process_info.jobfam_session_dir);
}
orte_process_info.jobfam_session_dir = strdup(orte_jobfam_session_dir);
}
orte_prohibited_session_dirs = NULL;
(void) mca_base_var_register ("orte", "orte", NULL, "no_session_dirs",
"Prohibited locations for session directories (multiple locations separated by ',', default=NULL)",
MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_ALL,
&orte_prohibited_session_dirs);
orte_create_session_dirs = true;
(void) mca_base_var_register ("orte", "orte", NULL, "create_session_dirs",
"Create session directories",
MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_ALL,
&orte_create_session_dirs);
orte_execute_quiet = false;
(void) mca_base_var_register ("orte", "orte", NULL, "execute_quiet",
"Do not output error and help messages",
MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_ALL,
&orte_execute_quiet);
orte_report_silent_errors = false;
(void) mca_base_var_register ("orte", "orte", NULL, "report_silent_errors",
"Report all errors, including silent ones",
MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_ALL,
&orte_report_silent_errors);
orte_debug_flag = false;
(void) mca_base_var_register ("orte", "orte", NULL, "debug",
"Top-level ORTE debug switch (default: false)",
MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_ALL,
&orte_debug_flag);
orte_debug_verbosity = -1;
(void) mca_base_var_register ("orte", "orte", NULL, "debug_verbose",
"Verbosity level for ORTE debug messages (default: 1)",
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_ALL,
&orte_debug_verbosity);
orte_debug_daemons_file_flag = false;
(void) mca_base_var_register ("orte", "orte", NULL, "debug_daemons_file",
"Whether want stdout/stderr of daemons to go to a file or not",
MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_ALL,
&orte_debug_daemons_file_flag);
/* If --debug-daemons-file was specified, that also implies
--debug-daemons */
if (orte_debug_daemons_file_flag) {
orte_debug_daemons_flag = true;
/* value can't change */
(void) mca_base_var_register ("orte", "orte", NULL, "debug_daemons",
"Whether to debug the ORTE daemons or not",
MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_CONSTANT,
&orte_debug_daemons_flag);
} else {
orte_debug_daemons_flag = false;
(void) mca_base_var_register ("orte", "orte", NULL, "debug_daemons",
"Whether to debug the ORTE daemons or not",
MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_ALL,
&orte_debug_daemons_flag);
}
orte_progress_thread_debug_level = -1;
(void) mca_base_var_register ("orte", "orte", NULL, "progress_thread_debug",
"Debug level for ORTE progress threads",
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_ALL,
&orte_progress_thread_debug_level);
if (0 <= orte_progress_thread_debug_level) {
orte_progress_thread_debug = opal_output_open(NULL);
opal_output_set_verbosity(orte_progress_thread_debug,
orte_progress_thread_debug_level);
}
/* do we want session output left open? */
orte_leave_session_attached = false;
(void) mca_base_var_register ("orte", "orte", NULL, "leave_session_attached",
"Whether applications and/or daemons should leave their sessions "
"attached so that any output can be received - this allows X forwarding "
"without all the attendant debugging output",
MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_ALL,
&orte_leave_session_attached);
/* if any debug level is set, ensure we output debug level dumps */
if (orte_debug_flag || orte_debug_daemons_flag || orte_leave_session_attached) {
orte_devel_level_output = true;
}
/* See comment in orte/tools/orterun/orterun.c about this MCA
param (this param is internal) */
orte_in_parallel_debugger = false;
(void) mca_base_var_register ("orte", "orte", NULL, "in_parallel_debugger",
"Whether the application is being debugged "
"in a parallel debugger (default: false)",
MCA_BASE_VAR_TYPE_BOOL, NULL, 0, MCA_BASE_VAR_FLAG_INTERNAL,
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
&orte_in_parallel_debugger);
orte_debugger_dump_proctable = false;
(void) mca_base_var_register ("orte", "orte", NULL, "output_debugger_proctable",
"Whether or not to output the debugger proctable after launch (default: false)",
MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_ALL,
&orte_debugger_dump_proctable);
orte_debugger_test_daemon = NULL;
(void) mca_base_var_register ("orte", "orte", NULL, "debugger_test_daemon",
"Name of the executable to be used to simulate a debugger colaunch (relative or absolute path)",
MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
&orte_debugger_test_daemon);
orte_debugger_test_attach = false;
(void) mca_base_var_register ("orte", "orte", NULL, "debugger_test_attach",
"Test debugger colaunch after debugger attachment",
MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
&orte_debugger_test_daemon);
orte_debugger_check_rate = 0;
(void) mca_base_var_register ("orte", "orte", NULL, "debugger_check_rate",
"Set rate (in secs) for auto-detect of debugger attachment (0 => do not check)",
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
&orte_debugger_check_rate);
orte_do_not_launch = false;
(void) mca_base_var_register ("orte", "orte", NULL, "do_not_launch",
"Perform all necessary operations to prepare to launch the application, but do not actually launch it",
MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
&orte_do_not_launch);
orted_spin_flag = false;
(void) mca_base_var_register ("orte", "orte", NULL, "daemon_spin",
"Have any orteds spin until we can connect a debugger to them",
MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
&orted_spin_flag);
orted_debug_failure = ORTE_VPID_INVALID;
(void) mca_base_var_register ("orte", "orte", NULL, "daemon_fail",
"Have the specified orted fail after init for debugging purposes",
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
&orted_debug_failure);
orted_debug_failure_delay = 0;
(void) mca_base_var_register ("orte", "orte", NULL, "daemon_fail_delay",
"Have the specified orted fail after specified number of seconds (default: 0 => no delay)",
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
&orted_debug_failure_delay);
orte_startup_timeout = 0;
(void) mca_base_var_register ("orte", "orte", NULL, "startup_timeout",
"Seconds to wait for startup or job launch before declaring failed_to_start (default: 0 => do not check)",
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
&orte_startup_timeout);
/* User-level debugger info string */
orte_base_user_debugger = "totalview @mpirun@ -a @mpirun_args@ : ddt -n @np@ -start @executable@ @executable_argv@ @single_app@ : fxp @mpirun@ -a @mpirun_args@";
(void) mca_base_var_register ("orte", "orte", NULL, "base_user_debugger",
"Sequence of user-level debuggers to search for in orterun",
MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
&orte_base_user_debugger);
#if 0
mca_base_param_reg_int_name("orte", "abort_timeout",
"Max time to wait [in secs] before aborting an ORTE operation (default: 1sec)",
false, false, 1, &value);
orte_max_timeout = 1000000.0 * value; /* convert to usec */
mca_base_param_reg_int_name("orte", "timeout_step",
"Time to wait [in usecs/proc] before aborting an ORTE operation (default: 1000 usec/proc)",
false, false, 1000, &orte_timeout_usec_per_proc);
#endif
/* default hostfile */
orte_default_hostfile = NULL;
(void) mca_base_var_register ("orte", "orte", NULL, "default_hostfile",
"Name of the default hostfile (relative or absolute path, \"none\" to ignore environmental or default MCA param setting)",
MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
&orte_default_hostfile);
if (NULL == orte_default_hostfile) {
/* nothing was given, so define the default */
asprintf(&orte_default_hostfile, "%s/openmpi-default-hostfile", opal_install_dirs.sysconfdir);
/* flag that nothing was given */
orte_default_hostfile_given = false;
} else if (0 == strcmp(orte_default_hostfile, "none")) {
free (orte_default_hostfile);
orte_default_hostfile = NULL;
/* flag that it was given */
orte_default_hostfile_given = true;
} else {
/* flag that it was given */
orte_default_hostfile_given = true;
}
/* default dash-host */
orte_default_dash_host = NULL;
(void) mca_base_var_register ("orte", "orte", NULL, "default_dash_host",
"Default -host setting (specify \"none\" to ignore environmental or default MCA param setting)",
MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
&orte_default_dash_host);
if (NULL != orte_default_dash_host &&
0 == strcmp(orte_default_dash_host, "none")) {
free(orte_default_dash_host);
orte_default_dash_host = NULL;
}
/* regex of nodes in system */
orte_node_regex = NULL;
(void) mca_base_var_register ("orte", "orte", NULL, "node_regex",
"Regular expression defining nodes in the system",
MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
&orte_node_regex);
/* whether or not to keep FQDN hostnames */
orte_keep_fqdn_hostnames = false;
(void) mca_base_var_register ("orte", "orte", NULL, "keep_fqdn_hostnames",
"Whether or not to keep FQDN hostnames [default: no]",
MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
&orte_keep_fqdn_hostnames);
/* whether or not to retain aliases of hostnames */
orte_retain_aliases = false;
(void) mca_base_var_register ("orte", "orte", NULL, "retain_aliases",
"Whether or not to keep aliases for host names [default: no]",
MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
&orte_retain_aliases);
/* which alias to use in MPIR_proctab */
orte_use_hostname_alias = 1;
(void) mca_base_var_register ("orte", "orte", NULL, "hostname_alias_index",
"If hostname aliases are being retained, which one to use for the debugger proc table [default: 1st alias]",
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
&orte_use_hostname_alias);
orte_xml_output = false;
(void) mca_base_var_register ("orte", "orte", NULL, "xml_output",
"Display all output in XML format (default: false)",
MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
&orte_xml_output);
/* whether to tag output */
/* if we requested xml output, be sure to tag the output as well */
orte_tag_output = orte_xml_output;
(void) mca_base_var_register ("orte", "orte", NULL, "tag_output",
"Tag all output with [job,rank] (default: false)",
MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
&orte_tag_output);
if (orte_xml_output) {
orte_tag_output = true;
}
orte_xml_file = NULL;
(void) mca_base_var_register ("orte", "orte", NULL, "xml_file",
"Provide all output in XML format to the specified file",
MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
&orte_xml_file);
if (NULL != orte_xml_file) {
if (ORTE_PROC_IS_HNP && NULL == orte_xml_fp) {
/* only the HNP opens this file! Make sure it only happens once */
orte_xml_fp = fopen(orte_xml_file, "w");
if (NULL == orte_xml_fp) {
opal_output(0, "Could not open specified xml output file: %s", orte_xml_file);
return ORTE_ERROR;
}
}
/* ensure we set the flags to tag output */
orte_xml_output = true;
orte_tag_output = true;
} else {
/* default to stdout */
orte_xml_fp = stdout;
}
/* whether to timestamp output */
orte_timestamp_output = false;
(void) mca_base_var_register ("orte", "orte", NULL, "timestamp_output",
"Timestamp all application process output (default: false)",
MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
&orte_timestamp_output);
/* redirect output into files */
orte_output_filename = NULL;
(void) mca_base_var_register ("orte", "orte", NULL, "output_filename",
"Redirect output from application processes into filename.rank [default: NULL]",
MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
&orte_output_filename);
orte_show_resolved_nodenames = false;
(void) mca_base_var_register ("orte", "orte", NULL, "show_resolved_nodenames",
"Display any node names that are resolved to a different name (default: false)",
MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
&orte_show_resolved_nodenames);
#if 0
/* XXX -- option doesn't appear to do anything */
mca_base_param_reg_int_name("orte", "hetero_apps",
"Indicates that multiple app_contexts are being provided that are a mix of 32/64 bit binaries (default: false)",
false, false, (int) false, &value);
orte_hetero_apps = OPAL_INT_TO_BOOL(value);
#endif
orte_hetero_nodes = false;
(void) mca_base_var_register ("orte", "orte", NULL, "hetero_nodes",
"Nodes in cluster may differ in topology, so send the topology back from each node [Default = false]",
MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
&orte_hetero_nodes);
/* allow specification of the launch agent */
orte_launch_agent = "orted";
(void) mca_base_var_register ("orte", "orte", NULL, "launch_agent",
"Command used to start processes on remote nodes (default: orted)",
MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
&orte_launch_agent);
orte_fork_agent_string = NULL;
(void) mca_base_var_register ("orte", "orte", NULL, "fork_agent",
"Command used to fork processes on remote nodes (default: NULL)",
MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
&orte_fork_agent_string);
if (NULL != orte_fork_agent_string) {
orte_fork_agent = opal_argv_split(orte_fork_agent_string, ' ');
}
/* whether or not to require RM allocation */
orte_allocation_required = false;
(void) mca_base_var_register ("orte", "orte", NULL, "allocation_required",
"Whether or not an allocation by a resource manager is required [default: no]",
MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
&orte_allocation_required);
/* whether or not to map stddiag to stderr */
orte_map_stddiag_to_stderr = false;
(void) mca_base_var_register ("orte", "orte", NULL, "map_stddiag_to_stderr",
"Map output from opal_output to stderr of the local process [default: no]",
MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
&orte_map_stddiag_to_stderr);
/* generate new terminal windows to display output from specified ranks */
orte_xterm = NULL;
(void) mca_base_var_register ("orte", "orte", NULL, "xterm",
"Create a new xterm window and display output from the specified ranks there [default: none]",
MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
&orte_xterm);
if (NULL != orte_xterm) {
/* if an xterm request is given, we have to leave any ssh
* sessions attached so the xterm window manager can get
* back to the controlling terminal
*/
orte_leave_session_attached = true;
/* also want to redirect stddiag output from opal_output
* to stderr from the process so those messages show
* up in the xterm window instead of being forwarded to mpirun
*/
orte_map_stddiag_to_stderr = true;
}
/* whether or not to report launch progress */
orte_report_launch_progress = false;
(void) mca_base_var_register ("orte", "orte", NULL, "report_launch_progress",
"Output a brief periodic report on launch progress [default: no]",
MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
&orte_report_launch_progress);
/* cluster hardware info detected by orte only */
orte_local_cpu_type = NULL;
(void) mca_base_var_register ("orte", "orte", NULL, "cpu_type",
"cpu type detected in node",
MCA_BASE_VAR_TYPE_STRING, NULL, 0, MCA_BASE_VAR_FLAG_INTERNAL,
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
&orte_local_cpu_type);
orte_local_cpu_model = NULL;
(void) mca_base_var_register ("orte", "orte", NULL, "cpu_model",
"cpu model detected in node",
MCA_BASE_VAR_TYPE_STRING, NULL, 0, MCA_BASE_VAR_FLAG_INTERNAL,
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
&orte_local_cpu_model);
/* tool communication controls */
orte_report_events_uri = NULL;
(void) mca_base_var_register ("orte", "orte", NULL, "report_events",
"URI to which events are to be reported (default: NULL)",
MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
&orte_report_events_uri);
if (NULL != orte_report_events_uri) {
orte_report_events = true;
}
/* barrier control */
orte_do_not_barrier = false;
(void) mca_base_var_register ("orte", "orte", NULL, "do_not_barrier",
"Do not barrier in orte_init",
MCA_BASE_VAR_TYPE_BOOL, NULL, 0, MCA_BASE_VAR_FLAG_INTERNAL,
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
&orte_do_not_barrier);
orte_enable_recovery = false;
(void) mca_base_var_register ("orte", "orte", NULL, "enable_recovery",
"Enable recovery from process failure [Default = disabled]",
MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
&orte_enable_recovery);
orte_max_restarts = 0;
(void) mca_base_var_register ("orte", "orte", NULL, "max_restarts",
"Max number of times to restart a failed process",
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
&orte_max_restarts);
if (!orte_enable_recovery && orte_max_restarts != 0) {
if (ORTE_PROC_IS_HNP) {
opal_output(orte_clean_output,
"------------------------------------------------------------------\n"
"The MCA param orte_enable_recovery was not set to true, but\n"
"a value was provided for the number of restarts:\n\n"
"Max restarts: %d\n"
"We are enabling process recovery and continuing execution. To avoid\n"
"this warning in the future, please set the orte_enable_recovery\n"
"param to non-zero.\n"
"------------------------------------------------------------------",
orte_max_restarts);
}
orte_enable_recovery = true;
}
orte_abort_non_zero_exit = true;
(void) mca_base_var_register ("orte", "orte", NULL, "abort_on_non_zero_status",
"Abort the job if any process returns a non-zero exit status - no restart in such cases",
MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
&orte_abort_non_zero_exit);
orte_allowed_exit_without_sync = false;
(void) mca_base_var_register ("orte", "orte", NULL, "allowed_exit_without_sync",
"Process exiting without calling finalize will not trigger job termination",
MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
&orte_allowed_exit_without_sync);
orte_staged_execution = false;
(void) mca_base_var_register ("orte", "orte", NULL, "staged_execution",
"Staged execution is being used",
MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
&orte_staged_execution);
orte_report_child_jobs_separately = false;
(void) mca_base_var_register ("orte", "orte", NULL, "report_child_jobs_separately",
"Return the exit status of the primary job only",
MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
&orte_report_child_jobs_separately);
#if 0
/* XXX -- unused parameter */
mca_base_param_reg_int_name("orte", "child_time_to_exit",
"Max time a spawned child job is allowed to run after the primary job has terminated (seconds)",
false, false,
INT_MAX, &value);
orte_child_time_to_exit.tv_sec = value;
orte_child_time_to_exit.tv_usec = 0;
#endif
orte_stat_history_size = 1;
(void) mca_base_var_register ("orte", "orte", NULL, "stat_history_size",
"Number of stat samples to keep",
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
&orte_stat_history_size);
orte_max_vm_size = -1;
(void) mca_base_var_register ("orte", "orte", NULL, "max_vm_size",
"Maximum size of virtual machine - used to subdivide allocation",
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
&orte_max_vm_size);
if (opal_hwloc_use_hwthreads_as_cpus) {
orte_set_slots = "hwthreads";
} else {
orte_set_slots = "cores";
}
(void) mca_base_var_register ("orte", "orte", NULL, "set_default_slots",
"Set the number of slots on nodes that lack such info to the"
" number of specified objects [a number, \"cores\" (default),"
" \"numas\", \"sockets\", \"hwthreads\" (default if hwthreads_as_cpus is set),"
" or \"none\" to skip this option]",
MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
&orte_set_slots);
/* should we display the allocation after determining it? */
orte_display_allocation = false;
id = mca_base_var_register ("orte", "orte", NULL, "display_alloc",
"Whether to display the allocation after it is determined",
MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
&orte_display_allocation);
/* register a synonym for old name -- should we remove this now? */
mca_base_var_register_synonym (id, "orte", "ras", "base", "display_alloc", MCA_BASE_VAR_SYN_FLAG_DEPRECATED);
/* should we display a detailed (developer-quality) version of the allocation after determining it? */
orte_devel_level_output = false;
id = mca_base_var_register ("orte", "orte", NULL, "display_devel_alloc",
"Whether to display a developer-detail allocation after it is determined",
MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
&orte_devel_level_output);
/* register a synonym for old name -- should we remove this now? */
mca_base_var_register_synonym (id, "orte", "ras", "base", "display_devel_alloc", MCA_BASE_VAR_SYN_FLAG_DEPRECATED);
if (orte_devel_level_output) {
orte_display_allocation = true;
}
/* should we treat any -host directives as "soft" - i.e., desired
* but not required
*/
orte_soft_locations = false;
(void) mca_base_var_register ("orte", "orte", NULL, "soft_locations",
"Treat -host directives as desired, but not required",
MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
&orte_soft_locations);
/* allow specification of the cores to be used by daemons */
orte_daemon_cores = NULL;
(void) mca_base_var_register ("orte", "orte", NULL, "daemon_cores",
"Restrict the ORTE daemons (including mpirun) to operate on the specified cores (comma-separated list of ranges)",
MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_READONLY,
&orte_daemon_cores);
/* cutoff for full modex */
orte_direct_modex_cutoff = UINT32_MAX;
id = mca_base_var_register ("orte", "orte", NULL, "direct_modex_cutoff",
"If the number of processes in the application exceeds the provided value,"
"modex will be done upon demand [default: UINT32_MAX]",
MCA_BASE_VAR_TYPE_UNSIGNED_INT, NULL, 0, 0,
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
&orte_direct_modex_cutoff);
/* register a synonym for old name */
mca_base_var_register_synonym (id, "ompi", "ompi", "hostname", "cutoff", MCA_BASE_VAR_SYN_FLAG_DEPRECATED);
return ORTE_SUCCESS;
}
/**
* Function for finding and opening either all MCA components, or the one
* that was specifically requested via a MCA parameter.
*/
int orte_ns_base_open(void)
{
int param, value, rc;
orte_data_type_t tmp;
opal_output_stream_t kill_prefix;
/* Debugging / verbose output */
/** setup the structure to kill the blasted prefix that opal_output
* now defaults to including so the output can be legible again!
*/
OBJ_CONSTRUCT(&kill_prefix, opal_output_stream_t);
kill_prefix.lds_want_stderr = true;
kill_prefix.lds_prefix = NULL;
param = mca_base_param_reg_int_name("ns", "base_verbose",
"Verbosity level for the ns framework",
false, false, 0, &value);
if (value != 0) {
kill_prefix.lds_verbose_level = value;
}
mca_ns_base_output = opal_output_open(&kill_prefix);
/* register the base system types with the DPS */
tmp = ORTE_NAME;
if (ORTE_SUCCESS != (rc = orte_dss.register_type(orte_ns_base_pack_name,
orte_ns_base_unpack_name,
(orte_dss_copy_fn_t)orte_ns_base_copy_name,
(orte_dss_compare_fn_t)orte_ns_base_compare_name,
(orte_dss_size_fn_t)orte_ns_base_std_size,
(orte_dss_print_fn_t)orte_ns_base_print_name,
(orte_dss_release_fn_t)orte_ns_base_std_release,
ORTE_DSS_UNSTRUCTURED,
"ORTE_NAME", &tmp))) {
ORTE_ERROR_LOG(rc);
return rc;
}
tmp = ORTE_VPID;
if (ORTE_SUCCESS != (rc = orte_dss.register_type(orte_ns_base_pack_vpid,
orte_ns_base_unpack_vpid,
(orte_dss_copy_fn_t)orte_ns_base_copy_vpid,
(orte_dss_compare_fn_t)orte_ns_base_compare_vpid,
(orte_dss_size_fn_t)orte_ns_base_std_size,
(orte_dss_print_fn_t)orte_ns_base_std_print,
(orte_dss_release_fn_t)orte_ns_base_std_release,
ORTE_DSS_UNSTRUCTURED,
"ORTE_VPID", &tmp))) {
ORTE_ERROR_LOG(rc);
return rc;
}
tmp = ORTE_JOBID;
if (ORTE_SUCCESS != (rc = orte_dss.register_type(orte_ns_base_pack_jobid,
orte_ns_base_unpack_jobid,
(orte_dss_copy_fn_t)orte_ns_base_copy_jobid,
(orte_dss_compare_fn_t)orte_ns_base_compare_jobid,
(orte_dss_size_fn_t)orte_ns_base_std_size,
(orte_dss_print_fn_t)orte_ns_base_std_print,
(orte_dss_release_fn_t)orte_ns_base_std_release,
ORTE_DSS_UNSTRUCTURED,
"ORTE_JOBID", &tmp))) {
ORTE_ERROR_LOG(rc);
return rc;
}
tmp = ORTE_CELLID;
if (ORTE_SUCCESS != (rc = orte_dss.register_type(orte_ns_base_pack_cellid,
orte_ns_base_unpack_cellid,
(orte_dss_copy_fn_t)orte_ns_base_copy_cellid,
(orte_dss_compare_fn_t)orte_ns_base_compare_cellid,
(orte_dss_size_fn_t)orte_ns_base_std_size,
(orte_dss_print_fn_t)orte_ns_base_std_print,
(orte_dss_release_fn_t)orte_ns_base_std_release,
ORTE_DSS_UNSTRUCTURED,
"ORTE_CELLID", &tmp))) {
ORTE_ERROR_LOG(rc);
return rc;
}
/* Open up all available components */
if (ORTE_SUCCESS !=
mca_base_components_open("ns", mca_ns_base_output,
mca_ns_base_static_components,
&mca_ns_base_components_available, true)) {
return ORTE_ERROR;
}
/* All done */
return ORTE_SUCCESS;
}
