/* * Local helper function to build an array of all the procs in a * communicator, excluding this process. * * Killing a just the indicated peers must be implemented for * MPI_Abort() to work according to the standard language for * a 'high-quality' implementation. * * It would be nifty if we could differentiate between the * abort scenarios (but we don't, currently): * - MPI_Abort() * - MPI_ERRORS_ARE_FATAL * - Victim of MPI_Abort() */ static void try_kill_peers(ompi_communicator_t *comm, int errcode) { int nprocs; ompi_process_name_t *procs; nprocs = ompi_comm_size(comm); /* ompi_comm_remote_size() returns 0 if not an intercomm, so this is safe */ nprocs += ompi_comm_remote_size(comm); procs = (ompi_process_name_t*) calloc(nprocs, sizeof(ompi_process_name_t)); if (NULL == procs) { /* quick clean orte and get out */ ompi_rte_abort(errno, "Abort: unable to alloc memory to kill procs"); } /* put all the local group procs in the abort list */ int rank, i, count; rank = ompi_comm_rank(comm); for (count = i = 0; i < ompi_comm_size(comm); ++i) { if (rank == i) { /* Don't include this process in the array */ --nprocs; } else { assert(count <= nprocs); procs[count++] = *OMPI_CAST_RTE_NAME(&ompi_group_get_proc_ptr(comm->c_remote_group, i)->super.proc_name); } } /* if requested, kill off remote group procs too */ for (i = 0; i < ompi_comm_remote_size(comm); ++i) { assert(count <= nprocs); procs[count++] = *OMPI_CAST_RTE_NAME(&ompi_group_get_proc_ptr(comm->c_remote_group, i)->super.proc_name); } if (nprocs > 0) { ompi_rte_abort_peers(procs, nprocs, errcode); } /* We could fall through here if ompi_rte_abort_peers() fails, or if (nprocs == 0). Either way, tidy up and let the caller handle it. */ free(procs); }
int ompi_mpi_abort(struct ompi_communicator_t* comm, int errcode, bool kill_remote_of_intercomm) { int count = 0, i, ret; char *msg, *host, hostname[MAXHOSTNAMELEN]; pid_t pid = 0; ompi_process_name_t *abort_procs; int32_t nabort_procs; /* Protection for recursive invocation */ if (have_been_invoked) { return OMPI_SUCCESS; } have_been_invoked = true; /* If MPI is initialized, we know we have a runtime nodename, so use that. Otherwise, call gethostname. */ if (ompi_mpi_initialized) { host = ompi_process_info.nodename; } else { gethostname(hostname, sizeof(hostname)); host = hostname; } pid = getpid(); /* Should we print a stack trace? Not aggregated because they might be different on all processes. */ if (ompi_mpi_abort_print_stack) { char **messages; int len, i; if (OMPI_SUCCESS == opal_backtrace_buffer(&messages, &len)) { for (i = 0; i < len; ++i) { fprintf(stderr, "[%s:%d] [%d] func:%s\n", host, (int) pid, i, messages[i]); fflush(stderr); } free(messages); } else { /* This will print an message if it's unable to print the backtrace, so we don't need an additional "else" clause if opal_backtrace_print() is not supported. */ opal_backtrace_print(stderr, NULL, 1); } } /* Notify the debugger that we're about to abort */ if (errcode < 0 || asprintf(&msg, "[%s:%d] aborting with MPI error %s%s", host, (int) pid, ompi_mpi_errnum_get_string(errcode), ompi_mpi_abort_print_stack ? " (stack trace available on stderr)" : "") < 0) { msg = NULL; } ompi_debugger_notify_abort(msg); if (NULL != msg) { free(msg); } /* Should we wait for a while before aborting? */ if (0 != ompi_mpi_abort_delay) { if (ompi_mpi_abort_delay < 0) { fprintf(stderr ,"[%s:%d] Looping forever (MCA parameter mpi_abort_delay is < 0)\n", host, (int) pid); fflush(stderr); while (1) { sleep(5); } } else { fprintf(stderr, "[%s:%d] Delaying for %d seconds before aborting\n", host, (int) pid, ompi_mpi_abort_delay); do { sleep(1); } while (--ompi_mpi_abort_delay > 0); } } /* If OMPI isn't setup yet/any more, then don't even try killing everyone. OMPI's initialized period covers all runtime initialized period of time, so no need to check that here. Sorry, Charlie... */ if (!ompi_mpi_initialized || ompi_mpi_finalized) { fprintf(stderr, "[%s:%d] Local abort %s completed successfully; not able to aggregate error messages, and not able to guarantee that all other processes were killed!\n", host, (int) pid, ompi_mpi_finalized ? "after MPI_FINALIZE" : "before MPI_INIT"); exit(errcode); } /* abort local procs in the communicator. If the communicator is an intercommunicator AND the abort has explicitly requested that we abort the remote procs, then do that as well. */ nabort_procs = ompi_comm_size(comm); if (kill_remote_of_intercomm) { /* ompi_comm_remote_size() returns 0 if not an intercomm, so this is cool */ nabort_procs += ompi_comm_remote_size(comm); } abort_procs = (ompi_process_name_t*)malloc(sizeof(ompi_process_name_t) * nabort_procs); if (NULL == abort_procs) { /* quick clean orte and get out */ ompi_rte_abort(errcode, "Abort unable to malloc memory to kill procs"); } /* put all the local procs in the abort list */ for (i = 0 ; i < ompi_comm_size(comm) ; ++i) { if (OPAL_EQUAL != ompi_rte_compare_name_fields(OMPI_RTE_CMP_ALL, &comm->c_local_group->grp_proc_pointers[i]->proc_name, OMPI_PROC_MY_NAME)) { assert(count <= nabort_procs); abort_procs[count++] = comm->c_local_group->grp_proc_pointers[i]->proc_name; } else { /* don't terminate me just yet */ nabort_procs--; } } /* if requested, kill off remote procs too */ if (kill_remote_of_intercomm) { for (i = 0 ; i < ompi_comm_remote_size(comm) ; ++i) { if (OPAL_EQUAL != ompi_rte_compare_name_fields(OMPI_RTE_CMP_ALL, &comm->c_remote_group->grp_proc_pointers[i]->proc_name, OMPI_PROC_MY_NAME)) { assert(count <= nabort_procs); abort_procs[count++] = comm->c_remote_group->grp_proc_pointers[i]->proc_name; } else { /* don't terminate me just yet */ nabort_procs--; } } } if (nabort_procs > 0) { /* This must be implemented for MPI_Abort() to work according to the * standard language for a 'high-quality' implementation. * It would be nifty if we could differentiate between the * abort scenarios: * - MPI_Abort() * - MPI_ERRORS_ARE_FATAL * - Victim of MPI_Abort() */ /* * Abort peers in this communicator group. Does not include self. */ if( OMPI_SUCCESS != (ret = ompi_rte_abort_peers(abort_procs, nabort_procs, errcode)) ) { ompi_rte_abort(errcode, "Open MPI failed to abort all of the procs requested (%d).", ret); } } /* now that we've aborted everyone else, gracefully die. */ ompi_rte_abort(errcode, NULL); return OMPI_SUCCESS; }
int ompi_mpi_register_params(void) { int value; /* Whether we want MPI API function parameter checking or not. Disable this by default if parameter checking is compiled out. */ ompi_mpi_param_check = !!(MPI_PARAM_CHECK); (void) mca_base_var_register("ompi", "mpi", NULL, "param_check", "Whether you want MPI API parameters checked at run-time or not. Possible values are 0 (no checking) and 1 (perform checking at run-time)", MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY, &ompi_mpi_param_check); if (ompi_mpi_param_check && !MPI_PARAM_CHECK) { opal_show_help("help-mpi-runtime.txt", "mpi-param-check-enabled-but-compiled-out", true); ompi_mpi_param_check = false; } /* * opal_progress: decide whether to yield and the event library * tick rate */ /* JMS: Need ORTE data here -- set this to 0 when exactly/under-subscribed, or 1 when oversubscribed */ ompi_mpi_yield_when_idle = true; (void) mca_base_var_register("ompi", "mpi", NULL, "yield_when_idle", "Yield the processor when waiting for MPI communication (for MPI processes, will default to 1 when oversubscribing nodes)", MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY, &ompi_mpi_yield_when_idle); ompi_mpi_event_tick_rate = -1; (void) mca_base_var_register("ompi", "mpi", NULL, "event_tick_rate", "How often to progress TCP communications (0 = never, otherwise specified in microseconds)", MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY, &ompi_mpi_event_tick_rate); /* Whether or not to show MPI handle leaks */ ompi_debug_show_handle_leaks = false; (void) mca_base_var_register("ompi", "mpi", NULL, "show_handle_leaks", "Whether MPI_FINALIZE shows all MPI handles that were not freed or not", MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY, &ompi_debug_show_handle_leaks); /* Whether or not to free MPI handles. Useless without run-time param checking, so implicitly set that to true if we don't want to free the handles. */ ompi_debug_no_free_handles = false; (void) mca_base_var_register("ompi", "mpi", NULL, "no_free_handles", "Whether to actually free MPI objects when their handles are freed", MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY, &ompi_debug_no_free_handles); if (ompi_debug_no_free_handles) { ompi_mpi_param_check = true; if (!MPI_PARAM_CHECK) { opal_output(0, "WARNING: MCA parameter mpi_no_free_handles set to true, but MPI"); opal_output(0, "WARNING: parameter checking has been compiled out of Open MPI."); opal_output(0, "WARNING: mpi_no_free_handles is therefore only partially effective!"); } } /* Whether or not to show MPI_ALLOC_MEM leaks */ ompi_debug_show_mpi_alloc_mem_leaks = 0; (void) mca_base_var_register("ompi", "mpi", NULL, "show_mpi_alloc_mem_leaks", "If >0, MPI_FINALIZE will show up to this many instances of memory allocated by MPI_ALLOC_MEM that was not freed by MPI_FREE_MEM", MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY, &ompi_debug_show_mpi_alloc_mem_leaks); /* Whether or not to print all MCA parameters in MPI_INIT */ ompi_mpi_show_mca_params_string = NULL; (void) mca_base_var_register("ompi", "mpi", NULL, "show_mca_params", "Whether to show all MCA parameter values during MPI_INIT or not (good for reproducability of MPI jobs " "for debug purposes). Accepted values are all, default, file, api, and enviro - or a comma " "delimited combination of them", MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY, &ompi_mpi_show_mca_params_string); if (NULL != ompi_mpi_show_mca_params_string) { char **args; int i; ompi_mpi_show_mca_params = true; args = opal_argv_split(ompi_mpi_show_mca_params_string, ','); if (NULL == args) { opal_output(0, "WARNING: could not parse mpi_show_mca_params request - defaulting to show \"all\""); show_default_mca_params = true; show_file_mca_params = true; show_enviro_mca_params = true; show_override_mca_params = true; } else { for (i=0; NULL != args[i]; i++) { if (0 == strcasecmp(args[i], "all") || 0 == strcmp(args[i], "1")) { show_default_mca_params = true; show_file_mca_params = true; show_enviro_mca_params = true; show_override_mca_params = true; } else if (0 == strcasecmp(args[i], "default")) { show_default_mca_params = true; } else if (0 == strcasecmp(args[i], "file")) { show_file_mca_params = true; } else if (0 == strncasecmp(args[i], "env", 3)) { show_enviro_mca_params = true; } else if (0 == strcasecmp(args[i], "api")) { show_override_mca_params = true; } } opal_argv_free(args); } } /* File to use when dumping the parameters */ ompi_mpi_show_mca_params_file = ""; (void) mca_base_var_register("ompi", "mpi", NULL, "show_mca_params_file", "If mpi_show_mca_params is true, setting this string to a valid filename tells Open MPI to dump all the MCA parameter values into a file suitable for reading via the mca_param_files parameter (good for reproducability of MPI jobs)", MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY, &ompi_mpi_show_mca_params_file); /* User-level process pinning controls */ /* MPI_ABORT controls */ ompi_mpi_abort_delay = 0; (void) mca_base_var_register("ompi", "mpi", NULL, "abort_delay", "If nonzero, print out an identifying message when MPI_ABORT is invoked (hostname, PID of the process that called MPI_ABORT) and delay for that many seconds before exiting (a negative delay value means to never abort). This allows attaching of a debugger before quitting the job.", MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY, &ompi_mpi_abort_delay); ompi_mpi_abort_print_stack = false; (void) mca_base_var_register("ompi", "mpi", NULL, "abort_print_stack", "If nonzero, print out a stack trace when MPI_ABORT is invoked", MCA_BASE_VAR_TYPE_BOOL, NULL, 0, /* If we do not have stack trace capability, make this a constant MCA variable */ #if OPAL_WANT_PRETTY_PRINT_STACKTRACE && defined(HAVE_BACKTRACE) 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY, #else MCA_BASE_VAR_FLAG_DEFAULT_ONLY, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_CONSTANT, #endif &ompi_mpi_abort_print_stack); ompi_mpi_preconnect_mpi = false; value = mca_base_var_register("ompi", "mpi", NULL, "preconnect_mpi", "Whether to force MPI processes to fully " "wire-up the MPI connections between MPI " "processes during " "MPI_INIT (vs. making connections lazily -- " "upon the first MPI traffic between each " "process peer pair)", MCA_BASE_VAR_TYPE_BOOL, NULL, 0, MCA_BASE_VAR_FLAG_INTERNAL, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY, &ompi_mpi_preconnect_mpi); mca_base_var_register_synonym(value, "ompi", "mpi", NULL, "preconnect_all", MCA_BASE_VAR_SYN_FLAG_DEPRECATED); /* Leave pinned parameter */ ompi_mpi_leave_pinned = -1; (void) mca_base_var_register("ompi", "mpi", NULL, "leave_pinned", "Whether to use the \"leave pinned\" protocol or not. Enabling this setting can help bandwidth performance when repeatedly sending and receiving large messages with the same buffers over RDMA-based networks (0 = do not use \"leave pinned\" protocol, 1 = use \"leave pinned\" protocol, -1 = allow network to choose at runtime).", MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY, &ompi_mpi_leave_pinned); ompi_mpi_leave_pinned_pipeline = false; (void) mca_base_var_register("ompi", "mpi", NULL, "leave_pinned_pipeline", "Whether to use the \"leave pinned pipeline\" protocol or not.", MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY, &ompi_mpi_leave_pinned_pipeline); if (ompi_mpi_leave_pinned > 0 && ompi_mpi_leave_pinned_pipeline) { ompi_mpi_leave_pinned_pipeline = 0; opal_show_help("help-mpi-runtime.txt", "mpi-params:leave-pinned-and-pipeline-selected", true); } ompi_warn_on_fork = true; (void) mca_base_var_register("ompi", "mpi", NULL, "warn_on_fork", "If nonzero, issue a warning if program forks under conditions that could cause system errors", MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY, &ompi_warn_on_fork); /* Sparse group storage support */ (void) mca_base_var_register("ompi", "mpi", NULL, "have_sparse_group_storage", "Whether this Open MPI installation supports storing of data in MPI groups in \"sparse\" formats (good for extremely large process count MPI jobs that create many communicators/groups)", MCA_BASE_VAR_TYPE_BOOL, NULL, 0, MCA_BASE_VAR_FLAG_DEFAULT_ONLY, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_CONSTANT, &ompi_mpi_have_sparse_group_storage); ompi_use_sparse_group_storage = ompi_mpi_have_sparse_group_storage; (void) mca_base_var_register("ompi", "mpi", NULL, "use_sparse_group_storage", "Whether to use \"sparse\" storage formats for MPI groups (only relevant if mpi_have_sparse_group_storage is 1)", MCA_BASE_VAR_TYPE_BOOL, NULL, 0, ompi_mpi_have_sparse_group_storage ? 0 : MCA_BASE_VAR_FLAG_DEFAULT_ONLY, OPAL_INFO_LVL_9, ompi_mpi_have_sparse_group_storage ? MCA_BASE_VAR_SCOPE_READONLY : MCA_BASE_VAR_SCOPE_CONSTANT, &ompi_use_sparse_group_storage); if (ompi_use_sparse_group_storage && !ompi_mpi_have_sparse_group_storage) { opal_show_help("help-mpi-runtime.txt", "sparse groups enabled but compiled out", true); ompi_use_sparse_group_storage = false; } (void) mca_base_var_register("ompi", "mpi", NULL, "built_with_cuda_support", "Whether CUDA GPU buffer support is built into library or not", MCA_BASE_VAR_TYPE_BOOL, NULL, 0, MCA_BASE_VAR_FLAG_DEFAULT_ONLY, OPAL_INFO_LVL_4, MCA_BASE_VAR_SCOPE_CONSTANT, &ompi_mpi_built_with_cuda_support); /* Current default is to enable CUDA support if it is built into library */ ompi_mpi_cuda_support = ompi_mpi_built_with_cuda_support; (void) mca_base_var_register("ompi", "mpi", NULL, "cuda_support", "Whether CUDA GPU buffer support is enabled or not", MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, OPAL_INFO_LVL_4, MCA_BASE_VAR_SCOPE_LOCAL, &ompi_mpi_cuda_support); if (ompi_mpi_cuda_support && !ompi_mpi_built_with_cuda_support) { opal_show_help("help-mpi-runtime.txt", "no cuda support", true); ompi_rte_abort(1, NULL); } /* cutoff for retrieving hostnames */ ompi_hostname_cutoff = UINT32_MAX; (void) mca_base_var_register ("ompi", "ompi", NULL, "hostname_cutoff", "If the number of processes in the application exceeds the provided value," "hostnames for remote processes will not be retrieved by applications [default: UINT32_MAX]", MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY, &ompi_hostname_cutoff); return OMPI_SUCCESS; }
int ompi_mpi_abort(struct ompi_communicator_t* comm, int errcode) { char *msg, *host, hostname[MAXHOSTNAMELEN]; pid_t pid = 0; /* Protection for recursive invocation */ if (have_been_invoked) { return OMPI_SUCCESS; } have_been_invoked = true; /* If MPI is initialized, we know we have a runtime nodename, so use that. Otherwise, call gethostname. */ if (ompi_rte_initialized) { host = ompi_process_info.nodename; } else { gethostname(hostname, sizeof(hostname)); host = hostname; } pid = getpid(); /* Should we print a stack trace? Not aggregated because they might be different on all processes. */ if (ompi_mpi_abort_print_stack) { char **messages; int len, i; if (OMPI_SUCCESS == opal_backtrace_buffer(&messages, &len)) { for (i = 0; i < len; ++i) { fprintf(stderr, "[%s:%d] [%d] func:%s\n", host, (int) pid, i, messages[i]); fflush(stderr); } free(messages); } else { /* This will print an message if it's unable to print the backtrace, so we don't need an additional "else" clause if opal_backtrace_print() is not supported. */ opal_backtrace_print(stderr, NULL, 1); } } /* Notify the debugger that we're about to abort */ if (errcode < 0 || asprintf(&msg, "[%s:%d] aborting with MPI error %s%s", host, (int) pid, ompi_mpi_errnum_get_string(errcode), ompi_mpi_abort_print_stack ? " (stack trace available on stderr)" : "") < 0) { msg = NULL; } ompi_debugger_notify_abort(msg); if (NULL != msg) { free(msg); } /* Should we wait for a while before aborting? */ if (0 != ompi_mpi_abort_delay) { if (ompi_mpi_abort_delay < 0) { fprintf(stderr ,"[%s:%d] Looping forever (MCA parameter mpi_abort_delay is < 0)\n", host, (int) pid); fflush(stderr); while (1) { sleep(5); } } else { fprintf(stderr, "[%s:%d] Delaying for %d seconds before aborting\n", host, (int) pid, ompi_mpi_abort_delay); do { sleep(1); } while (--ompi_mpi_abort_delay > 0); } } /* If the RTE isn't setup yet/any more, then don't even try killing everyone. Sorry, Charlie... */ if (!ompi_rte_initialized) { fprintf(stderr, "[%s:%d] Local abort %s completed successfully, but am not able to aggregate error messages, and not able to guarantee that all other processes were killed!\n", host, (int) pid, ompi_mpi_finalized ? "after MPI_FINALIZE started" : "before MPI_INIT completed"); _exit(errcode == 0 ? 1 : errcode); } /* If OMPI is initialized and we have a non-NULL communicator, then try to kill just that set of processes */ if (ompi_mpi_initialized && !ompi_mpi_finalized && NULL != comm) { try_kill_peers(comm, errcode); } /* We can fall through to here in a few cases: 1. The attempt to kill just a subset of peers via try_kill_peers() failed (e.g., as of July 2014, ORTE does returns NOT_IMPLENTED from orte_rte_abort_peers()). 2. MPI wasn't initialized, was already finalized, or we got a NULL communicator. In all of these cases, the only sensible thing left to do is to kill the entire job. Wah wah. */ ompi_rte_abort(errcode, NULL); /* Does not return */ }
int oshmem_shmem_abort(int errcode) { char *host, hostname[OPAL_MAXHOSTNAMELEN]; pid_t pid = 0; /* Protection for recursive invocation */ if (have_been_invoked) { return OSHMEM_SUCCESS; } have_been_invoked = true; /* If ORTE is initialized, use its nodename. Otherwise, call gethostname. */ /* If MPI is initialized, we know we have a runtime nodename, so use that. Otherwise, call gethostname. */ if (ompi_rte_initialized) { host = ompi_process_info.nodename; } else { gethostname(hostname, sizeof(hostname)); host = hostname; } pid = getpid(); opal_show_help("help-shmem-api.txt", "shmem-abort", true, OMPI_PROC_MY_NAME->vpid, pid, host, errcode); /* Should we print a stack trace? Not aggregated because they might be different on all processes. */ if (opal_abort_print_stack) { char **messages; int len, i; if (OPAL_SUCCESS == opal_backtrace_buffer(&messages, &len)) { for (i = 0; i < len; ++i) { fprintf(stderr, "[%s:%05d] [%d] func:%s\n", host, (int) pid, i, messages[i]); fflush(stderr); } free(messages); } else { /* This will print an message if it's unable to print the backtrace, so we don't need an additional "else" clause if opal_backtrace_print() is not supported. */ opal_backtrace_print(stderr, NULL, 1); } } /* Wait for a while before aborting */ opal_delay_abort(); if (!oshmem_shmem_initialized) { if (!opal_initialized) { /* TODO help message from SHMEM not from MPI is needed*/ opal_show_help("help-shmem-runtime.txt", "oshmem shmem abort:cannot guarantee all killed", true, host, (int) pid); } else { fprintf(stderr, "[%s:%05d] Local abort completed successfully; not able to aggregate error messages, and not able to guarantee that all other processes were killed!\n", host, (int) pid); } oshmem_shmem_aborted = true; exit(errcode); } /* abort local procs in the communicator. If the communicator is an intercommunicator AND the abort has explicitly requested that we abort the remote procs, then do that as well. */ oshmem_shmem_aborted = true; /* now that we've aborted everyone else, gracefully die. */ ompi_rte_abort(errcode, NULL ); return OSHMEM_SUCCESS; }
int ompi_mpi_register_params(void) { int value; /* Whether we want MPI API function parameter checking or not. Disable this by default if parameter checking is compiled out. */ ompi_mpi_param_check = !!(MPI_PARAM_CHECK); (void) mca_base_var_register("ompi", "mpi", NULL, "param_check", "Whether you want MPI API parameters checked at run-time or not. Possible values are 0 (no checking) and 1 (perform checking at run-time)", MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY, &ompi_mpi_param_check); if (ompi_mpi_param_check && !MPI_PARAM_CHECK) { opal_show_help("help-mpi-runtime.txt", "mpi-param-check-enabled-but-compiled-out", true); ompi_mpi_param_check = false; } /* * opal_progress: decide whether to yield and the event library * tick rate */ /* JMS: Need ORTE data here -- set this to 0 when exactly/under-subscribed, or 1 when oversubscribed */ ompi_mpi_yield_when_idle = true; (void) mca_base_var_register("ompi", "mpi", NULL, "yield_when_idle", "Yield the processor when waiting for MPI communication (for MPI processes, will default to 1 when oversubscribing nodes)", MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY, &ompi_mpi_yield_when_idle); ompi_mpi_event_tick_rate = -1; (void) mca_base_var_register("ompi", "mpi", NULL, "event_tick_rate", "How often to progress TCP communications (0 = never, otherwise specified in microseconds)", MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY, &ompi_mpi_event_tick_rate); /* Whether or not to show MPI handle leaks */ ompi_debug_show_handle_leaks = false; (void) mca_base_var_register("ompi", "mpi", NULL, "show_handle_leaks", "Whether MPI_FINALIZE shows all MPI handles that were not freed or not", MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY, &ompi_debug_show_handle_leaks); /* Whether or not to free MPI handles. Useless without run-time param checking, so implicitly set that to true if we don't want to free the handles. */ ompi_debug_no_free_handles = false; (void) mca_base_var_register("ompi", "mpi", NULL, "no_free_handles", "Whether to actually free MPI objects when their handles are freed", MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY, &ompi_debug_no_free_handles); if (ompi_debug_no_free_handles) { ompi_mpi_param_check = true; if (!MPI_PARAM_CHECK) { opal_output(0, "WARNING: MCA parameter mpi_no_free_handles set to true, but MPI"); opal_output(0, "WARNING: parameter checking has been compiled out of Open MPI."); opal_output(0, "WARNING: mpi_no_free_handles is therefore only partially effective!"); } } /* Whether or not to show MPI_ALLOC_MEM leaks */ ompi_debug_show_mpi_alloc_mem_leaks = 0; (void) mca_base_var_register("ompi", "mpi", NULL, "show_mpi_alloc_mem_leaks", "If >0, MPI_FINALIZE will show up to this many instances of memory allocated by MPI_ALLOC_MEM that was not freed by MPI_FREE_MEM", MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY, &ompi_debug_show_mpi_alloc_mem_leaks); /* Whether or not to print all MCA parameters in MPI_INIT */ ompi_mpi_show_mca_params_string = NULL; (void) mca_base_var_register("ompi", "mpi", NULL, "show_mca_params", "Whether to show all MCA parameter values during MPI_INIT or not (good for reproducability of MPI jobs " "for debug purposes). Accepted values are all, default, file, api, and enviro - or a comma " "delimited combination of them", MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY, &ompi_mpi_show_mca_params_string); if (NULL != ompi_mpi_show_mca_params_string) { char **args; int i; ompi_mpi_show_mca_params = true; args = opal_argv_split(ompi_mpi_show_mca_params_string, ','); if (NULL == args) { opal_output(0, "WARNING: could not parse mpi_show_mca_params request - defaulting to show \"all\""); show_default_mca_params = true; show_file_mca_params = true; show_enviro_mca_params = true; show_override_mca_params = true; } else { for (i=0; NULL != args[i]; i++) { if (0 == strcasecmp(args[i], "all") || 0 == strcmp(args[i], "1")) { show_default_mca_params = true; show_file_mca_params = true; show_enviro_mca_params = true; show_override_mca_params = true; } else if (0 == strcasecmp(args[i], "default")) { show_default_mca_params = true; } else if (0 == strcasecmp(args[i], "file")) { show_file_mca_params = true; } else if (0 == strncasecmp(args[i], "env", 3)) { show_enviro_mca_params = true; } else if (0 == strcasecmp(args[i], "api")) { show_override_mca_params = true; } } opal_argv_free(args); } } /* File to use when dumping the parameters */ (void) mca_base_var_register("ompi", "mpi", NULL, "show_mca_params_file", "If mpi_show_mca_params is true, setting this string to a valid filename tells Open MPI to dump all the MCA parameter values into a file suitable for reading via the mca_param_files parameter (good for reproducability of MPI jobs)", MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY, &ompi_mpi_show_mca_params_file); /* User-level process pinning controls */ ompi_mpi_preconnect_mpi = false; value = mca_base_var_register("ompi", "mpi", NULL, "preconnect_mpi", "Whether to force MPI processes to fully " "wire-up the MPI connections between MPI " "processes during " "MPI_INIT (vs. making connections lazily -- " "upon the first MPI traffic between each " "process peer pair)", MCA_BASE_VAR_TYPE_BOOL, NULL, 0, MCA_BASE_VAR_FLAG_INTERNAL, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY, &ompi_mpi_preconnect_mpi); mca_base_var_register_synonym(value, "ompi", "mpi", NULL, "preconnect_all", MCA_BASE_VAR_SYN_FLAG_DEPRECATED); /* Sparse group storage support */ (void) mca_base_var_register("ompi", "mpi", NULL, "have_sparse_group_storage", "Whether this Open MPI installation supports storing of data in MPI groups in \"sparse\" formats (good for extremely large process count MPI jobs that create many communicators/groups)", MCA_BASE_VAR_TYPE_BOOL, NULL, 0, MCA_BASE_VAR_FLAG_DEFAULT_ONLY, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_CONSTANT, &ompi_mpi_have_sparse_group_storage); ompi_use_sparse_group_storage = ompi_mpi_have_sparse_group_storage; (void) mca_base_var_register("ompi", "mpi", NULL, "use_sparse_group_storage", "Whether to use \"sparse\" storage formats for MPI groups (only relevant if mpi_have_sparse_group_storage is 1)", MCA_BASE_VAR_TYPE_BOOL, NULL, 0, ompi_mpi_have_sparse_group_storage ? 0 : MCA_BASE_VAR_FLAG_DEFAULT_ONLY, OPAL_INFO_LVL_9, ompi_mpi_have_sparse_group_storage ? MCA_BASE_VAR_SCOPE_READONLY : MCA_BASE_VAR_SCOPE_CONSTANT, &ompi_use_sparse_group_storage); if (ompi_use_sparse_group_storage && !ompi_mpi_have_sparse_group_storage) { opal_show_help("help-mpi-runtime.txt", "sparse groups enabled but compiled out", true); ompi_use_sparse_group_storage = false; } value = mca_base_var_find ("opal", "opal", NULL, "cuda_support"); if (0 <= value) { mca_base_var_register_synonym(value, "ompi", "mpi", NULL, "cuda_support", MCA_BASE_VAR_SYN_FLAG_DEPRECATED); } value = mca_base_var_find ("opal", "opal", NULL, "built_with_cuda_support"); if (0 <= value) { mca_base_var_register_synonym(value, "ompi", "mpi", NULL, "built_with_cuda_support", 0); } if (opal_cuda_support && !opal_built_with_cuda_support) { opal_show_help("help-mpi-runtime.txt", "no cuda support", true); ompi_rte_abort(1, NULL); } ompi_add_procs_cutoff = OMPI_ADD_PROCS_CUTOFF_DEFAULT; (void) mca_base_var_register ("ompi", "mpi", NULL, "add_procs_cutoff", "Maximum world size for pre-allocating resources for all " "remote processes. Increasing this limit may improve " "communication performance at the cost of memory usage", MCA_BASE_VAR_TYPE_UNSIGNED_INT, NULL, 0, 0, OPAL_INFO_LVL_3, MCA_BASE_VAR_SCOPE_LOCAL, &ompi_add_procs_cutoff); ompi_mpi_dynamics_enabled = true; (void) mca_base_var_register("ompi", "mpi", NULL, "dynamics_enabled", "Is the MPI dynamic process functionality enabled (e.g., MPI_COMM_SPAWN)? Default is yes, but certain transports and/or environments may disable it.", MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, OPAL_INFO_LVL_4, MCA_BASE_VAR_SCOPE_READONLY, &ompi_mpi_dynamics_enabled); ompi_async_mpi_init = false; (void) mca_base_var_register("ompi", "async", "mpi", "init", "Do not perform a barrier at the end of MPI_Init", MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY, &ompi_async_mpi_init); ompi_async_mpi_finalize = false; (void) mca_base_var_register("ompi", "async", "mpi", "finalize", "Do not perform a barrier at the beginning of MPI_Finalize", MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY, &ompi_async_mpi_finalize); value = mca_base_var_find ("opal", "opal", NULL, "abort_delay"); if (0 <= value) { (void) mca_base_var_register_synonym(value, "ompi", "mpi", NULL, "abort_delay", MCA_BASE_VAR_SYN_FLAG_DEPRECATED); } value = mca_base_var_find ("opal", "opal", NULL, "abort_print_stack"); if (0 <= value) { (void) mca_base_var_register_synonym(value, "ompi", "mpi", NULL, "abort_print_stack", MCA_BASE_VAR_SYN_FLAG_DEPRECATED); } return OMPI_SUCCESS; }