int opal_crs_blcr_restart(opal_crs_base_snapshot_t *base_snapshot, bool spawn_child, pid_t *child_pid)
{
opal_crs_blcr_snapshot_t *snapshot = OBJ_NEW(opal_crs_blcr_snapshot_t);
char **cr_argv = NULL;
char *cr_cmd = NULL;
char *cr_full_cmd = NULL;
int ret;
int exit_status = OPAL_SUCCESS;
int status;
snapshot->super = *base_snapshot;
opal_output_verbose(10, mca_crs_blcr_component.super.output_handle,
"crs:blcr: restart(--, %d)", spawn_child);
/*
* If we need to reconstruct the snapshot,
*/
if(snapshot->super.cold_start) {
if( OPAL_SUCCESS != (ret = blcr_cold_start(snapshot)) ) {
exit_status = OPAL_ERROR;
opal_output(mca_crs_blcr_component.super.output_handle,
"crs:blcr: blcr_restart: Unable to reconstruct the snapshot.");
goto cleanup;
}
}
/*
* Get the restart command
*/
if ( OPAL_SUCCESS != (ret = opal_crs_blcr_restart_cmd(snapshot->context_filename, &cr_cmd)) ) {
exit_status = ret;
goto cleanup;
}
if ( NULL == (cr_argv = opal_argv_split(cr_cmd, ' ')) ) {
exit_status = OPAL_ERROR;
goto cleanup;
}
/* Need to shutdown the event engine before this.
* for some reason the BLCR checkpointer and our event engine don't get
* along very well.
*/
opal_progress_finalize();
(void) mca_base_framework_close(&opal_event_base_framework);
if (!spawn_child) {
cr_full_cmd = opal_argv_join(cr_argv, ' ');
opal_output_verbose(10, mca_crs_blcr_component.super.output_handle,
"crs:blcr: blcr_restart: SELF: exec :(%s, %s):",
blcr_restart_cmd, cr_full_cmd);
status = execvp(blcr_restart_cmd, cr_argv);
if(status < 0) {
opal_output(mca_crs_blcr_component.super.output_handle,
"crs:blcr: blcr_restart: SELF: Child failed to execute :(%d):", status);
}
opal_show_help("help-opal-crs-blcr.txt", "blcr:restart_failed_exec", true,
status,
blcr_restart_cmd,
cr_full_cmd);
exit_status = status;
goto cleanup;
}
/*
* Restart by starting a new process
*/
else {
*child_pid = fork();
if( 0 == *child_pid) {
/* Child Process */
opal_output_verbose(10, mca_crs_blcr_component.super.output_handle,
"crs:blcr: blcr_restart: CHILD: exec :(%s, %s):",
blcr_restart_cmd,
opal_argv_join(cr_argv, ' '));
status = execvp(blcr_restart_cmd, cr_argv);
if(status < 0) {
opal_output(mca_crs_blcr_component.super.output_handle,
"crs:blcr: blcr_restart: CHILD: Child failed to execute :(%d):", status);
}
opal_output(mca_crs_blcr_component.super.output_handle,
"crs:blcr: blcr_restart: CHILD: execvp returned %d", status);
exit_status = status;
goto cleanup;
}
else if(*child_pid > 0) {
/* Parent is done once it is started. */
;
}
else {
opal_output(mca_crs_blcr_component.super.output_handle,
"crs:blcr: blcr_restart: CHILD: fork failed :(%d):", *child_pid);
}
}
cleanup:
if(NULL != cr_cmd)
free(cr_cmd);
if(NULL != cr_argv)
opal_argv_free(cr_argv);
return exit_status;
}
static void launch_daemons(int fd, short args, void *cbdata)
{
orte_app_context_t *app;
orte_node_t *node;
orte_std_cntr_t n;
orte_job_map_t *map;
char *jobid_string = NULL;
char *param;
char **argv = NULL;
int argc;
int rc;
char *tmp;
char** env = NULL;
char *nodelist_flat;
char **nodelist_argv;
char *name_string;
char **custom_strings;
int num_args, i;
char *cur_prefix;
int proc_vpid_index;
bool failed_launch=true;
orte_job_t *daemons;
orte_state_caddy_t *state = (orte_state_caddy_t*)cbdata;
ORTE_ACQUIRE_OBJECT(state);
OPAL_OUTPUT_VERBOSE((1, orte_plm_base_framework.framework_output,
"%s plm:slurm: LAUNCH DAEMONS CALLED",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
/* if we are launching debugger daemons, then just go
* do it - no new daemons will be launched
*/
if (ORTE_FLAG_TEST(state->jdata, ORTE_JOB_FLAG_DEBUGGER_DAEMON)) {
state->jdata->state = ORTE_JOB_STATE_DAEMONS_LAUNCHED;
ORTE_ACTIVATE_JOB_STATE(state->jdata, ORTE_JOB_STATE_DAEMONS_REPORTED);
OBJ_RELEASE(state);
return;
}
/* start by setting up the virtual machine */
daemons = orte_get_job_data_object(ORTE_PROC_MY_NAME->jobid);
if (ORTE_SUCCESS != (rc = orte_plm_base_setup_virtual_machine(state->jdata))) {
ORTE_ERROR_LOG(rc);
goto cleanup;
}
/* if we don't want to launch, then don't attempt to
* launch the daemons - the user really wants to just
* look at the proposed process map
*/
if (orte_do_not_launch) {
/* set the state to indicate the daemons reported - this
* will trigger the daemons_reported event and cause the
* job to move to the following step
*/
state->jdata->state = ORTE_JOB_STATE_DAEMONS_LAUNCHED;
ORTE_ACTIVATE_JOB_STATE(state->jdata, ORTE_JOB_STATE_DAEMONS_REPORTED);
OBJ_RELEASE(state);
return;
}
/* Get the map for this job */
if (NULL == (map = daemons->map)) {
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
rc = ORTE_ERR_NOT_FOUND;
goto cleanup;
}
if (0 == map->num_new_daemons) {
/* set the state to indicate the daemons reported - this
* will trigger the daemons_reported event and cause the
* job to move to the following step
*/
OPAL_OUTPUT_VERBOSE((1, orte_plm_base_framework.framework_output,
"%s plm:slurm: no new daemons to launch",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
state->jdata->state = ORTE_JOB_STATE_DAEMONS_LAUNCHED;
ORTE_ACTIVATE_JOB_STATE(state->jdata, ORTE_JOB_STATE_DAEMONS_REPORTED);
OBJ_RELEASE(state);
return;
}
/* need integer value for command line parameter */
asprintf(&jobid_string, "%lu", (unsigned long) daemons->jobid);
/*
* start building argv array
*/
argv = NULL;
argc = 0;
/*
* SLURM srun OPTIONS
*/
/* add the srun command */
opal_argv_append(&argc, &argv, "srun");
/* start one orted on each node */
opal_argv_append(&argc, &argv, "--ntasks-per-node=1");
if (!orte_enable_recovery) {
/* kill the job if any orteds die */
opal_argv_append(&argc, &argv, "--kill-on-bad-exit");
}
/* ensure the orteds are not bound to a single processor,
* just in case the TaskAffinity option is set by default.
* This will *not* release the orteds from any cpu-set
* constraint, but will ensure it doesn't get
* bound to only one processor
*/
opal_argv_append(&argc, &argv, "--cpu_bind=none");
#if SLURM_CRAY_ENV
/*
* If in a SLURM/Cray env. make sure that Cray PMI is not pulled in,
* neither as a constructor run when orteds start, nor selected
* when pmix components are registered
*/
opal_setenv("PMI_NO_PREINITIALIZE", "1", false, &orte_launch_environ);
opal_setenv("PMI_NO_FORK", "1", false, &orte_launch_environ);
opal_setenv("OMPI_NO_USE_CRAY_PMI", "1", false, &orte_launch_environ);
#endif
/* Append user defined arguments to srun */
if ( NULL != mca_plm_slurm_component.custom_args ) {
custom_strings = opal_argv_split(mca_plm_slurm_component.custom_args, ' ');
num_args = opal_argv_count(custom_strings);
for (i = 0; i < num_args; ++i) {
opal_argv_append(&argc, &argv, custom_strings[i]);
}
opal_argv_free(custom_strings);
}
/* create nodelist */
nodelist_argv = NULL;
for (n=0; n < map->nodes->size; n++ ) {
if (NULL == (node = (orte_node_t*)opal_pointer_array_get_item(map->nodes, n))) {
continue;
}
/* if the daemon already exists on this node, then
* don't include it
*/
if (ORTE_FLAG_TEST(node, ORTE_NODE_FLAG_DAEMON_LAUNCHED)) {
continue;
}
/* otherwise, add it to the list of nodes upon which
* we need to launch a daemon
*/
opal_argv_append_nosize(&nodelist_argv, node->name);
}
if (0 == opal_argv_count(nodelist_argv)) {
orte_show_help("help-plm-slurm.txt", "no-hosts-in-list", true);
rc = ORTE_ERR_FAILED_TO_START;
goto cleanup;
}
nodelist_flat = opal_argv_join(nodelist_argv, ',');
opal_argv_free(nodelist_argv);
/* if we are using all allocated nodes, then srun doesn't
* require any further arguments
*/
if (map->num_new_daemons < orte_num_allocated_nodes) {
asprintf(&tmp, "--nodes=%lu", (unsigned long)map->num_new_daemons);
opal_argv_append(&argc, &argv, tmp);
free(tmp);
asprintf(&tmp, "--nodelist=%s", nodelist_flat);
opal_argv_append(&argc, &argv, tmp);
free(tmp);
}
/* tell srun how many tasks to run */
asprintf(&tmp, "--ntasks=%lu", (unsigned long)map->num_new_daemons);
opal_argv_append(&argc, &argv, tmp);
free(tmp);
OPAL_OUTPUT_VERBOSE((2, orte_plm_base_framework.framework_output,
"%s plm:slurm: launching on nodes %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), nodelist_flat));
free(nodelist_flat);
/*
* ORTED OPTIONS
*/
/* add the daemon command (as specified by user) */
orte_plm_base_setup_orted_cmd(&argc, &argv);
/* Add basic orted command line options, including debug flags */
orte_plm_base_orted_append_basic_args(&argc, &argv,
"slurm", &proc_vpid_index);
/* tell the new daemons the base of the name list so they can compute
* their own name on the other end
*/
rc = orte_util_convert_vpid_to_string(&name_string, map->daemon_vpid_start);
if (ORTE_SUCCESS != rc) {
opal_output(0, "plm_slurm: unable to get daemon vpid as string");
goto cleanup;
}
free(argv[proc_vpid_index]);
argv[proc_vpid_index] = strdup(name_string);
free(name_string);
/* Copy the prefix-directory specified in the
corresponding app_context. If there are multiple,
different prefix's in the app context, complain (i.e., only
allow one --prefix option for the entire slurm run -- we
don't support different --prefix'es for different nodes in
the SLURM plm) */
cur_prefix = NULL;
for (n=0; n < state->jdata->apps->size; n++) {
char * app_prefix_dir;
if (NULL == (app = (orte_app_context_t*)opal_pointer_array_get_item(state->jdata->apps, n))) {
continue;
}
app_prefix_dir = NULL;
orte_get_attribute(&app->attributes, ORTE_APP_PREFIX_DIR, (void**)&app_prefix_dir, OPAL_STRING);
/* Check for already set cur_prefix_dir -- if different,
complain */
if (NULL != app_prefix_dir) {
if (NULL != cur_prefix &&
0 != strcmp (cur_prefix, app_prefix_dir)) {
orte_show_help("help-plm-slurm.txt", "multiple-prefixes",
true, cur_prefix, app_prefix_dir);
goto cleanup;
}
/* If not yet set, copy it; iff set, then it's the
* same anyway
*/
if (NULL == cur_prefix) {
cur_prefix = strdup(app_prefix_dir);
OPAL_OUTPUT_VERBOSE((1, orte_plm_base_framework.framework_output,
"%s plm:slurm: Set prefix:%s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
cur_prefix));
}
free(app_prefix_dir);
}
}
/* protect the args in case someone has a script wrapper around srun */
mca_base_cmd_line_wrap_args(argv);
/* setup environment */
env = opal_argv_copy(orte_launch_environ);
if (0 < opal_output_get_verbosity(orte_plm_base_framework.framework_output)) {
param = opal_argv_join(argv, ' ');
OPAL_OUTPUT_VERBOSE((1, orte_plm_base_framework.framework_output,
"%s plm:slurm: final top-level argv:\n\t%s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
(NULL == param) ? "NULL" : param));
if (NULL != param) free(param);
}
/* exec the daemon(s) */
if (ORTE_SUCCESS != (rc = plm_slurm_start_proc(argc, argv, env, cur_prefix))) {
ORTE_ERROR_LOG(rc);
goto cleanup;
}
/* indicate that the daemons for this job were launched */
state->jdata->state = ORTE_JOB_STATE_DAEMONS_LAUNCHED;
daemons->state = ORTE_JOB_STATE_DAEMONS_LAUNCHED;
/* flag that launch was successful, so far as we currently know */
failed_launch = false;
cleanup:
if (NULL != argv) {
opal_argv_free(argv);
}
if (NULL != env) {
opal_argv_free(env);
}
if(NULL != jobid_string) {
free(jobid_string);
}
/* cleanup the caddy */
OBJ_RELEASE(state);
/* check for failed launch - if so, force terminate */
if (failed_launch) {
ORTE_FORCED_TERMINATE(ORTE_ERROR_DEFAULT_EXIT_CODE);
}
}
int ompi_mpi_init(int argc, char **argv, int requested, int *provided)
{
int ret;
ompi_proc_t** procs;
size_t nprocs;
char *error = NULL;
struct timeval ompistart, ompistop;
bool rte_setup = false;
ompi_rte_collective_t *coll;
char *cmd=NULL, *av=NULL;
/* bitflag of the thread level support provided. To be used
* for the modex in order to work in heterogeneous environments. */
uint8_t threadlevel_bf;
/* Indicate that we have *started* MPI_INIT*. MPI_FINALIZE has
something sorta similar in a static local variable in
ompi_mpi_finalize(). */
ompi_mpi_init_started = true;
/* Setup enough to check get/set MCA params */
if (OPAL_SUCCESS != (ret = opal_init_util(&argc, &argv))) {
error = "ompi_mpi_init: opal_init_util failed";
goto error;
}
/* Register MCA variables */
if (OPAL_SUCCESS != (ret = ompi_register_mca_variables())) {
error = "ompi_mpi_init: ompi_register_mca_variables failed";
goto error;
}
if (OPAL_SUCCESS != (ret = opal_arch_set_fortran_logical_size(sizeof(ompi_fortran_logical_t)))) {
error = "ompi_mpi_init: opal_arch_set_fortran_logical_size failed";
goto error;
}
/* _After_ opal_init_util() but _before_ orte_init(), we need to
set an MCA param that tells libevent that it's ok to use any
mechanism in libevent that is available on this platform (e.g.,
epoll and friends). Per opal/event/event.s, we default to
select/poll -- but we know that MPI processes won't be using
pty's with the event engine, so it's ok to relax this
constraint and let any fd-monitoring mechanism be used. */
ret = mca_base_var_find("opal", "event", "*", "event_include");
if (ret >= 0) {
char *allvalue = "all";
/* We have to explicitly "set" the MCA param value here
because libevent initialization will re-register the MCA
param and therefore override the default. Setting the value
here puts the desired value ("all") in different storage
that is not overwritten if/when the MCA param is
re-registered. This is unless the user has specified a different
value for this MCA parameter. Make sure we check to see if the
default is specified before forcing "all" in case that is not what
the user desires. Note that we do *NOT* set this value as an
environment variable, just so that it won't be inherited by
any spawned processes and potentially cause unintented
side-effects with launching RTE tools... */
mca_base_var_set_value(ret, allvalue, 4, MCA_BASE_VAR_SOURCE_DEFAULT, NULL);
}
if (ompi_enable_timing && 0 == OMPI_PROC_MY_NAME->vpid) {
gettimeofday(&ompistart, NULL);
}
/* if we were not externally started, then we need to setup
* some envars so the MPI_INFO_ENV can get the cmd name
* and argv (but only if the user supplied a non-NULL argv!), and
* the requested thread level
*/
if (NULL == getenv("OMPI_COMMAND") && NULL != argv && NULL != argv[0]) {
asprintf(&cmd, "OMPI_COMMAND=%s", argv[0]);
putenv(cmd);
}
if (NULL == getenv("OMPI_ARGV") && 1 < argc) {
char *tmp;
tmp = opal_argv_join(&argv[1], ' ');
asprintf(&av, "OMPI_ARGV=%s", tmp);
free(tmp);
putenv(av);
}
/* Setup RTE - note that we are an MPI process */
if (OMPI_SUCCESS != (ret = ompi_rte_init(NULL, NULL))) {
error = "ompi_mpi_init: ompi_rte_init failed";
goto error;
}
rte_setup = true;
/* check for timing request - get stop time and report elapsed time if so */
if (ompi_enable_timing && 0 == OMPI_PROC_MY_NAME->vpid) {
gettimeofday(&ompistop, NULL);
opal_output(0, "ompi_mpi_init [%ld]: time from start to completion of rte_init %ld usec",
(long)OMPI_PROC_MY_NAME->vpid,
(long int)((ompistop.tv_sec - ompistart.tv_sec)*1000000 +
(ompistop.tv_usec - ompistart.tv_usec)));
gettimeofday(&ompistart, NULL);
}
#if OPAL_HAVE_HWLOC
/* if hwloc is available but didn't get setup for some
* reason, do so now
*/
if (NULL == opal_hwloc_topology) {
if (OPAL_SUCCESS != (ret = opal_hwloc_base_get_topology())) {
error = "Topology init";
goto error;
}
}
#endif
/* Register the default errhandler callback - RTE will ignore if it
* doesn't support this capability
*/
ompi_rte_register_errhandler(ompi_errhandler_runtime_callback,
OMPI_RTE_ERRHANDLER_LAST);
/* Figure out the final MPI thread levels. If we were not
compiled for support for MPI threads, then don't allow
MPI_THREAD_MULTIPLE. Set this stuff up here early in the
process so that other components can make decisions based on
this value. */
ompi_mpi_thread_level(requested, provided);
/* determine the bitflag belonging to the threadlevel_support provided */
memset ( &threadlevel_bf, 0, sizeof(uint8_t));
OMPI_THREADLEVEL_SET_BITFLAG ( ompi_mpi_thread_provided, threadlevel_bf );
/* add this bitflag to the modex */
if ( OMPI_SUCCESS != (ret = ompi_modex_send_string("MPI_THREAD_LEVEL", &threadlevel_bf, sizeof(uint8_t)))) {
error = "ompi_mpi_init: modex send thread level";
goto error;
}
/* initialize datatypes. This step should be done early as it will
* create the local convertor and local arch used in the proc
* init.
*/
if (OMPI_SUCCESS != (ret = ompi_datatype_init())) {
error = "ompi_datatype_init() failed";
goto error;
}
/* Initialize OMPI procs */
if (OMPI_SUCCESS != (ret = ompi_proc_init())) {
error = "mca_proc_init() failed";
goto error;
}
/* Initialize the op framework. This has to be done *after*
ddt_init, but befor mca_coll_base_open, since some collective
modules (e.g., the hierarchical coll component) may need ops in
their query function. */
if (OMPI_SUCCESS != (ret = mca_base_framework_open(&ompi_op_base_framework, 0))) {
error = "ompi_op_base_open() failed";
goto error;
}
if (OMPI_SUCCESS !=
(ret = ompi_op_base_find_available(OMPI_ENABLE_PROGRESS_THREADS,
OMPI_ENABLE_THREAD_MULTIPLE))) {
error = "ompi_op_base_find_available() failed";
goto error;
}
if (OMPI_SUCCESS != (ret = ompi_op_init())) {
error = "ompi_op_init() failed";
goto error;
}
/* Open up MPI-related MCA components */
if (OMPI_SUCCESS != (ret = mca_base_framework_open(&ompi_allocator_base_framework, 0))) {
error = "mca_allocator_base_open() failed";
goto error;
}
if (OMPI_SUCCESS != (ret = mca_base_framework_open(&ompi_rcache_base_framework, 0))) {
error = "mca_rcache_base_open() failed";
goto error;
}
if (OMPI_SUCCESS != (ret = mca_base_framework_open(&ompi_mpool_base_framework, 0))) {
error = "mca_mpool_base_open() failed";
goto error;
}
if (OMPI_SUCCESS != (ret = mca_base_framework_open(&ompi_pml_base_framework, 0))) {
error = "mca_pml_base_open() failed";
goto error;
}
if (OMPI_SUCCESS != (ret = mca_base_framework_open(&ompi_coll_base_framework, 0))) {
error = "mca_coll_base_open() failed";
goto error;
}
if (OMPI_SUCCESS != (ret = mca_base_framework_open(&ompi_osc_base_framework, 0))) {
error = "ompi_osc_base_open() failed";
goto error;
}
#if OPAL_ENABLE_FT_CR == 1
if (OMPI_SUCCESS != (ret = mca_base_framework_open(&ompi_crcp_base_framework, 0))) {
error = "ompi_crcp_base_open() failed";
goto error;
}
#endif
/* In order to reduce the common case for MPI apps (where they
don't use MPI-2 IO or MPI-1 topology functions), the io and
topo frameworks are initialized lazily, at the first use of
relevant functions (e.g., MPI_FILE_*, MPI_CART_*, MPI_GRAPH_*),
so they are not opened here. */
/* Select which MPI components to use */
if (OMPI_SUCCESS !=
(ret = mca_mpool_base_init(OMPI_ENABLE_PROGRESS_THREADS,
OMPI_ENABLE_THREAD_MULTIPLE))) {
error = "mca_mpool_base_init() failed";
goto error;
}
if (OMPI_SUCCESS !=
(ret = mca_pml_base_select(OMPI_ENABLE_PROGRESS_THREADS,
OMPI_ENABLE_THREAD_MULTIPLE))) {
error = "mca_pml_base_select() failed";
goto error;
}
/* check for timing request - get stop time and report elapsed time if so */
if (ompi_enable_timing && 0 == OMPI_PROC_MY_NAME->vpid) {
gettimeofday(&ompistop, NULL);
opal_output(0, "ompi_mpi_init[%ld]: time from completion of rte_init to modex %ld usec",
(long)OMPI_PROC_MY_NAME->vpid,
(long int)((ompistop.tv_sec - ompistart.tv_sec)*1000000 +
(ompistop.tv_usec - ompistart.tv_usec)));
gettimeofday(&ompistart, NULL);
}
/* exchange connection info - this function also acts as a barrier
* as it will not return until the exchange is complete
*/
coll = OBJ_NEW(ompi_rte_collective_t);
coll->id = ompi_process_info.peer_modex;
if (OMPI_SUCCESS != (ret = ompi_rte_modex(coll))) {
error = "rte_modex failed";
goto error;
}
/* wait for modex to complete - this may be moved anywhere in mpi_init
* so long as it occurs prior to calling a function that needs
* the modex info!
*/
while (coll->active) {
opal_progress(); /* block in progress pending events */
}
OBJ_RELEASE(coll);
if (ompi_enable_timing && 0 == OMPI_PROC_MY_NAME->vpid) {
gettimeofday(&ompistop, NULL);
opal_output(0, "ompi_mpi_init[%ld]: time to execute modex %ld usec",
(long)OMPI_PROC_MY_NAME->vpid,
(long int)((ompistop.tv_sec - ompistart.tv_sec)*1000000 +
(ompistop.tv_usec - ompistart.tv_usec)));
gettimeofday(&ompistart, NULL);
}
/* select buffered send allocator component to be used */
ret=mca_pml_base_bsend_init(OMPI_ENABLE_THREAD_MULTIPLE);
if( OMPI_SUCCESS != ret ) {
error = "mca_pml_base_bsend_init() failed";
goto error;
}
if (OMPI_SUCCESS !=
(ret = mca_coll_base_find_available(OMPI_ENABLE_PROGRESS_THREADS,
OMPI_ENABLE_THREAD_MULTIPLE))) {
error = "mca_coll_base_find_available() failed";
goto error;
}
if (OMPI_SUCCESS !=
(ret = ompi_osc_base_find_available(OMPI_ENABLE_PROGRESS_THREADS,
OMPI_ENABLE_THREAD_MULTIPLE))) {
error = "ompi_osc_base_find_available() failed";
goto error;
}
#if OPAL_ENABLE_FT_CR == 1
if (OMPI_SUCCESS != (ret = ompi_crcp_base_select() ) ) {
error = "ompi_crcp_base_select() failed";
goto error;
}
#endif
/* io and topo components are not selected here -- see comment
above about the io and topo frameworks being loaded lazily */
/* Initialize each MPI handle subsystem */
/* initialize requests */
if (OMPI_SUCCESS != (ret = ompi_request_init())) {
error = "ompi_request_init() failed";
goto error;
}
if (OMPI_SUCCESS != (ret = ompi_message_init())) {
error = "ompi_message_init() failed";
goto error;
}
/* initialize info */
if (OMPI_SUCCESS != (ret = ompi_info_init())) {
error = "ompi_info_init() failed";
goto error;
}
/* initialize error handlers */
if (OMPI_SUCCESS != (ret = ompi_errhandler_init())) {
error = "ompi_errhandler_init() failed";
goto error;
}
/* initialize error codes */
if (OMPI_SUCCESS != (ret = ompi_mpi_errcode_init())) {
error = "ompi_mpi_errcode_init() failed";
goto error;
}
/* initialize internal error codes */
if (OMPI_SUCCESS != (ret = ompi_errcode_intern_init())) {
error = "ompi_errcode_intern_init() failed";
goto error;
}
/* initialize groups */
if (OMPI_SUCCESS != (ret = ompi_group_init())) {
error = "ompi_group_init() failed";
goto error;
}
/* initialize communicators */
if (OMPI_SUCCESS != (ret = ompi_comm_init())) {
error = "ompi_comm_init() failed";
goto error;
}
/* initialize file handles */
if (OMPI_SUCCESS != (ret = ompi_file_init())) {
error = "ompi_file_init() failed";
goto error;
}
/* initialize windows */
if (OMPI_SUCCESS != (ret = ompi_win_init())) {
error = "ompi_win_init() failed";
goto error;
}
/* initialize attribute meta-data structure for comm/win/dtype */
if (OMPI_SUCCESS != (ret = ompi_attr_init())) {
error = "ompi_attr_init() failed";
goto error;
}
/* identify the architectures of remote procs and setup
* their datatype convertors, if required
*/
if (OMPI_SUCCESS != (ret = ompi_proc_complete_init())) {
error = "ompi_proc_complete_init failed";
goto error;
}
/* If thread support was enabled, then setup OPAL to allow for
them. */
if ((OMPI_ENABLE_PROGRESS_THREADS == 1) ||
(*provided != MPI_THREAD_SINGLE)) {
opal_set_using_threads(true);
}
/* start PML/BTL's */
ret = MCA_PML_CALL(enable(true));
if( OMPI_SUCCESS != ret ) {
error = "PML control failed";
goto error;
}
/* add all ompi_proc_t's to PML */
if (NULL == (procs = ompi_proc_world(&nprocs))) {
error = "ompi_proc_world() failed";
goto error;
}
ret = MCA_PML_CALL(add_procs(procs, nprocs));
free(procs);
/* If we got "unreachable", then print a specific error message.
Otherwise, if we got some other failure, fall through to print
a generic message. */
if (OMPI_ERR_UNREACH == ret) {
opal_show_help("help-mpi-runtime",
"mpi_init:startup:pml-add-procs-fail", true);
error = NULL;
goto error;
} else if (OMPI_SUCCESS != ret) {
error = "PML add procs failed";
goto error;
}
MCA_PML_CALL(add_comm(&ompi_mpi_comm_world.comm));
MCA_PML_CALL(add_comm(&ompi_mpi_comm_self.comm));
/*
* Dump all MCA parameters if requested
*/
if (ompi_mpi_show_mca_params) {
ompi_show_all_mca_params(ompi_mpi_comm_world.comm.c_my_rank,
nprocs,
ompi_process_info.nodename);
}
/* Do we need to wait for a debugger? */
ompi_rte_wait_for_debugger();
/* check for timing request - get stop time and report elapsed
time if so, then start the clock again */
if (ompi_enable_timing && 0 == OMPI_PROC_MY_NAME->vpid) {
gettimeofday(&ompistop, NULL);
opal_output(0, "ompi_mpi_init[%ld]: time from modex to first barrier %ld usec",
(long)OMPI_PROC_MY_NAME->vpid,
(long int)((ompistop.tv_sec - ompistart.tv_sec)*1000000 +
(ompistop.tv_usec - ompistart.tv_usec)));
gettimeofday(&ompistart, NULL);
}
/* wait for everyone to reach this point */
coll = OBJ_NEW(ompi_rte_collective_t);
coll->id = ompi_process_info.peer_init_barrier;
if (OMPI_SUCCESS != (ret = ompi_rte_barrier(coll))) {
error = "rte_barrier failed";
goto error;
}
/* wait for barrier to complete */
while (coll->active) {
opal_progress(); /* block in progress pending events */
}
OBJ_RELEASE(coll);
/* check for timing request - get stop time and report elapsed
time if so, then start the clock again */
if (ompi_enable_timing && 0 == OMPI_PROC_MY_NAME->vpid) {
gettimeofday(&ompistop, NULL);
opal_output(0, "ompi_mpi_init[%ld]: time to execute barrier %ld usec",
(long)OMPI_PROC_MY_NAME->vpid,
(long int)((ompistop.tv_sec - ompistart.tv_sec)*1000000 +
(ompistop.tv_usec - ompistart.tv_usec)));
gettimeofday(&ompistart, NULL);
}
#if OMPI_ENABLE_PROGRESS_THREADS == 0
/* Start setting up the event engine for MPI operations. Don't
block in the event library, so that communications don't take
forever between procs in the dynamic code. This will increase
CPU utilization for the remainder of MPI_INIT when we are
blocking on RTE-level events, but may greatly reduce non-TCP
latency. */
opal_progress_set_event_flag(OPAL_EVLOOP_NONBLOCK);
#endif
/* wire up the mpi interface, if requested. Do this after the
non-block switch for non-TCP performance. Do before the
polling change as anyone with a complex wire-up is going to be
using the oob. */
if (OMPI_SUCCESS != (ret = ompi_init_preconnect_mpi())) {
error = "ompi_mpi_do_preconnect_all() failed";
goto error;
}
/* Setup the publish/subscribe (PUBSUB) framework */
if (OMPI_SUCCESS != (ret = mca_base_framework_open(&ompi_pubsub_base_framework, 0))) {
error = "mca_pubsub_base_open() failed";
goto error;
}
if (OMPI_SUCCESS != (ret = ompi_pubsub_base_select())) {
error = "ompi_pubsub_base_select() failed";
goto error;
}
/* Setup the dynamic process management (DPM) framework */
if (OMPI_SUCCESS != (ret = mca_base_framework_open(&ompi_dpm_base_framework, 0))) {
error = "ompi_dpm_base_open() failed";
goto error;
}
if (OMPI_SUCCESS != (ret = ompi_dpm_base_select())) {
error = "ompi_dpm_base_select() failed";
goto error;
}
/* Determine the overall threadlevel support of all processes
in MPI_COMM_WORLD. This has to be done before calling
coll_base_comm_select, since some of the collective components
e.g. hierarch, might create subcommunicators. The threadlevel
requested by all processes is required in order to know
which cid allocation algorithm can be used. */
if ( OMPI_SUCCESS !=
( ret = ompi_comm_cid_init ())) {
error = "ompi_mpi_init: ompi_comm_cid_init failed";
goto error;
}
/* Init coll for the comms. This has to be after dpm_base_select,
(since dpm.mark_dyncomm is not set in the communicator creation
function else), but before dpm.dyncom_init, since this function
might require collective for the CID allocation. */
if (OMPI_SUCCESS !=
(ret = mca_coll_base_comm_select(MPI_COMM_WORLD))) {
error = "mca_coll_base_comm_select(MPI_COMM_WORLD) failed";
goto error;
}
if (OMPI_SUCCESS !=
(ret = mca_coll_base_comm_select(MPI_COMM_SELF))) {
error = "mca_coll_base_comm_select(MPI_COMM_SELF) failed";
goto error;
}
/* Check whether we have been spawned or not. We introduce that
at the very end, since we need collectives, datatypes, ptls
etc. up and running here.... */
if (OMPI_SUCCESS != (ret = ompi_dpm.dyn_init())) {
error = "ompi_comm_dyn_init() failed";
goto error;
}
/*
* Startup the Checkpoint/Restart Mech.
* Note: Always do this so tools don't hang when
* in a non-checkpointable build
*/
if (OMPI_SUCCESS != (ret = ompi_cr_init())) {
error = "ompi_cr_init";
goto error;
}
/* Undo OPAL calling opal_progress_event_users_increment() during
opal_init, to get better latency when not using TCP. Do
this *after* dyn_init, as dyn init uses lots of RTE
communication and we don't want to hinder the performance of
that code. */
opal_progress_event_users_decrement();
/* see if yield_when_idle was specified - if so, use it */
opal_progress_set_yield_when_idle(ompi_mpi_yield_when_idle);
/* negative value means use default - just don't do anything */
if (ompi_mpi_event_tick_rate >= 0) {
opal_progress_set_event_poll_rate(ompi_mpi_event_tick_rate);
}
/* At this point, we are fully configured and in MPI mode. Any
communication calls here will work exactly like they would in
the user's code. Setup the connections between procs and warm
them up with simple sends, if requested */
if (OMPI_SUCCESS != (ret = ompi_mpiext_init())) {
error = "ompi_mpiext_init";
goto error;
}
/* Fall through */
error:
if (ret != OMPI_SUCCESS) {
/* Only print a message if one was not already printed */
if (NULL != error) {
const char *err_msg = opal_strerror(ret);
/* If RTE was not setup yet, don't use opal_show_help */
if (rte_setup) {
opal_show_help("help-mpi-runtime",
"mpi_init:startup:internal-failure", true,
"MPI_INIT", "MPI_INIT", error, err_msg, ret);
} else {
opal_show_help("help-mpi-runtime",
"mpi_init:startup:internal-failure", true,
"MPI_INIT", "MPI_INIT", error, err_msg, ret);
}
}
return ret;
}
/* Initialize the registered datarep list to be empty */
OBJ_CONSTRUCT(&ompi_registered_datareps, opal_list_t);
/* Initialize the arrays used to store the F90 types returned by the
* MPI_Type_create_f90_XXX functions.
*/
OBJ_CONSTRUCT( &ompi_mpi_f90_integer_hashtable, opal_hash_table_t);
opal_hash_table_init(&ompi_mpi_f90_integer_hashtable, 16 /* why not? */);
OBJ_CONSTRUCT( &ompi_mpi_f90_real_hashtable, opal_hash_table_t);
opal_hash_table_init(&ompi_mpi_f90_real_hashtable, FLT_MAX_10_EXP);
OBJ_CONSTRUCT( &ompi_mpi_f90_complex_hashtable, opal_hash_table_t);
opal_hash_table_init(&ompi_mpi_f90_complex_hashtable, FLT_MAX_10_EXP);
/* All done. Wasn't that simple? */
ompi_mpi_initialized = true;
/* check for timing request - get stop time and report elapsed time if so */
if (ompi_enable_timing && 0 == OMPI_PROC_MY_NAME->vpid) {
gettimeofday(&ompistop, NULL);
opal_output(0, "ompi_mpi_init[%ld]: time from barrier to complete mpi_init %ld usec",
(long)OMPI_PROC_MY_NAME->vpid,
(long int)((ompistop.tv_sec - ompistart.tv_sec)*1000000 +
(ompistop.tv_usec - ompistart.tv_usec)));
}
return MPI_SUCCESS;
}
int opal_crs_none_restart(opal_crs_base_snapshot_t *base_snapshot, bool spawn_child, pid_t *child_pid)
{
int exit_status = OPAL_SUCCESS;
char **tmp_argv = NULL;
char **cr_argv = NULL;
int status;
*child_pid = getpid();
if( NULL == base_snapshot->metadata ) {
if (NULL == (base_snapshot->metadata = fopen(base_snapshot->metadata_filename, "a")) ) {
opal_output(0,
"crs:none: checkpoint(): Error: Unable to open the file (%s)",
base_snapshot->metadata_filename);
exit_status = OPAL_ERROR;
goto cleanup;
}
}
opal_crs_base_metadata_read_token(base_snapshot->metadata, CRS_METADATA_CONTEXT, &tmp_argv);
if( NULL == tmp_argv ) {
opal_output(opal_crs_base_framework.framework_output,
"crs:none: none_restart: Error: Failed to read the %s token from the local checkpoint in %s",
CRS_METADATA_CONTEXT, base_snapshot->metadata_filename);
exit_status = OPAL_ERROR;
goto cleanup;
}
if( opal_argv_count(tmp_argv) <= 0 ) {
opal_output_verbose(10, opal_crs_base_framework.framework_output,
"crs:none: none_restart: No command line to exec, so just returning");
exit_status = OPAL_SUCCESS;
goto cleanup;
}
if ( NULL == (cr_argv = opal_argv_split(tmp_argv[0], ' ')) ) {
exit_status = OPAL_ERROR;
goto cleanup;
}
if( !spawn_child ) {
opal_output_verbose(10, opal_crs_base_framework.framework_output,
"crs:none: none_restart: exec :(%s, %s):",
strdup(cr_argv[0]),
opal_argv_join(cr_argv, ' '));
status = execvp(strdup(cr_argv[0]), cr_argv);
if(status < 0) {
opal_output(opal_crs_base_framework.framework_output,
"crs:none: none_restart: Child failed to execute :(%d):", status);
}
opal_output(opal_crs_base_framework.framework_output,
"crs:none: none_restart: execvp returned %d", status);
exit_status = status;
goto cleanup;
} else {
opal_output(opal_crs_base_framework.framework_output,
"crs:none: none_restart: Spawn not implemented");
exit_status = OPAL_ERR_NOT_IMPLEMENTED;
goto cleanup;
}
cleanup:
if (NULL != base_snapshot->metadata) {
fclose(base_snapshot->metadata);
}
base_snapshot->metadata = NULL;
return exit_status;
}
static int status(char *nodes, opal_list_t *images)
{
char *query, *line, *ptr;
FILE *fp;
orcm_pvsn_provision_t *pvn, *pvnptr;
opal_value_t *attr;
int i, rc=ORTE_SUCCESS;
int j;
char **nodelist, **ranges;
OPAL_OUTPUT_VERBOSE((5, orcm_pvsn_base_framework.framework_output,
"%s pvsn:wwulf:status",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
/* if nodes is NULL, then get the info for all nodes. Note
* that this could be a *lot* of info for a large cluster */
if (NULL == nodes) {
(void)asprintf(&query, "%s provision print", cmd);
} else {
/* could be a comma-separated regex, so parse it */
ranges = opal_argv_split(nodes, ',');
nodelist = NULL;
for (i=0; NULL != ranges[i]; i++) {
if (ORTE_SUCCESS != (rc = orte_regex_extract_node_names(ranges[i], &nodelist))) {
ORTE_ERROR_LOG(rc);
opal_argv_free(ranges);
return rc;
}
}
opal_argv_free(ranges);
ptr = opal_argv_join(nodelist, ' ');
opal_argv_free(nodelist);
(void)asprintf(&query, "%s provision print %s", cmd, ptr);
free(ptr);
}
if (NULL == (fp = popen(query, "r"))) {
OPAL_OUTPUT_VERBOSE((5, orcm_pvsn_base_framework.framework_output,
"%s pvsn:wwulf:avail query for provisioning status failed",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
free(query);
return ORCM_ERROR;
}
free(query);
while (NULL != (line = orcm_getline(fp))) {
OPAL_OUTPUT_VERBOSE((5, orcm_pvsn_base_framework.framework_output,
"%s pvsn:wwulf:status got input %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), line));
/* if the line starts with #, it can be ignored */
if ('#' == line[0]) {
free(line);
continue;
}
/* we want the following sections of the output line:
* 0 => node name
* 1 => attribute
* 3 => value
*/
ptr = line;
j=0;
while (NULL != (query = parse_next(ptr, &ptr))) {
switch(j) {
case 0:
/* see if we already have this node */
pvn = NULL;
OPAL_LIST_FOREACH(pvnptr, images, orcm_pvsn_provision_t) {
if (0 == strcmp(pvnptr->nodes, query)) {
pvn = pvnptr;
break;
}
}
if (NULL == pvn) {
pvn = OBJ_NEW(orcm_pvsn_provision_t);
opal_list_append(images, &pvn->super);
pvn->nodes = strdup(query);
/* need to come up with a naming scheme for images */
pvn->image.name = strdup(query);
}
break;
case 1:
attr = OBJ_NEW(opal_value_t);
attr->key = strdup(query);
opal_list_append(&pvn->image.attributes, &attr->super);
break;
case 3:
attr->type = OPAL_STRING;
attr->data.string = strdup(query);
break;
default:
/* just ignore it */
break;
}
j++;
}
free(line);
}
pclose(fp);
return ORCM_SUCCESS;
}
static int s1_init(void)
{
PMI_BOOL initialized;
int spawned;
int rc, ret = OPAL_ERROR;
int i, rank, lrank, nrank;
char *pmix_id, tmp[64];
opal_value_t kv;
char *str;
uint32_t ui32;
opal_process_name_t ldr;
char **localranks=NULL;
if (PMI_SUCCESS != (rc = PMI_Initialized(&initialized))) {
OPAL_PMI_ERROR(rc, "PMI_Initialized");
return OPAL_ERROR;
}
if( PMI_TRUE != initialized && PMI_SUCCESS != (rc = PMI_Init(&spawned)) ) {
OPAL_PMI_ERROR(rc, "PMI_Init");
return OPAL_ERROR;
}
// setup hash table
opal_pmix_base_hash_init();
// Initialize space demands
rc = PMI_KVS_Get_value_length_max(&pmix_vallen_max);
if( PMI_SUCCESS != rc ) {
OPAL_PMI_ERROR(rc, "PMI_KVS_Get_value_length_max");
goto err_exit;
}
pmix_vallen_threshold = pmix_vallen_max * 3;
pmix_vallen_threshold >>= 2;
rc = PMI_KVS_Get_name_length_max(&pmix_kvslen_max);
if (PMI_SUCCESS != rc ) {
OPAL_PMI_ERROR(rc, "PMI_KVS_Get_name_length_max");
goto err_exit;
}
rc = PMI_KVS_Get_key_length_max(&pmix_keylen_max);
if( PMI_SUCCESS != rc ) {
OPAL_PMI_ERROR(rc, "PMI_KVS_Get_key_length_max");
goto err_exit;
}
// Initialize job environment information
pmix_id = (char*)malloc(pmix_vallen_max);
if( pmix_id == NULL ){
ret = OPAL_ERR_OUT_OF_RESOURCE;
goto err_exit;
}
/* Get domain id */
if (PMI_SUCCESS != (rc = PMI_Get_kvs_domain_id(pmix_id, pmix_vallen_max))) {
free(pmix_id);
goto err_exit;
}
/* Slurm PMI provides the job id as an integer followed
* by a '.', followed by essentially a stepid. The first integer
* defines an overall job number. The second integer is the number of
* individual jobs we have run within that allocation.
*/
OBJ_CONSTRUCT(&kv, opal_value_t);
kv.key = strdup(OPAL_PMIX_JOBID);
kv.type = OPAL_STRING;
kv.data.string = pmix_id;
if (OPAL_SUCCESS != (ret = opal_pmix_base_store(&OPAL_PROC_MY_NAME, &kv))) {
OPAL_ERROR_LOG(ret);
OBJ_DESTRUCT(&kv);
goto err_exit;
}
OBJ_DESTRUCT(&kv);
/* get our rank */
ret = PMI_Get_rank(&rank);
if( PMI_SUCCESS != ret ) {
OPAL_PMI_ERROR(ret, "PMI_Get_rank");
goto err_exit;
}
/* save it */
OBJ_CONSTRUCT(&kv, opal_value_t);
kv.key = strdup(OPAL_PMIX_RANK);
kv.type = OPAL_UINT32;
kv.data.uint32 = rank;
if (OPAL_SUCCESS != (ret = opal_pmix_base_store(&OPAL_PROC_MY_NAME, &kv))) {
OPAL_ERROR_LOG(ret);
OBJ_DESTRUCT(&kv);
goto err_exit;
}
OBJ_DESTRUCT(&kv);
/* store our name in the opal_proc_t so that
* debug messages will make sense - an upper
* layer will eventually overwrite it, but that
* won't do any harm */
s1_pname.jobid = strtoul(pmix_id, &str, 10);
s1_pname.jobid = (s1_pname.jobid << 16) & 0xffff0000;
if (NULL != str) {
ui32 = strtoul(str, NULL, 10);
s1_pname.jobid |= (ui32 & 0x0000ffff);
}
free(pmix_id);
ldr.jobid = s1_pname.jobid;
s1_pname.vpid = rank;
opal_proc_set_name(&s1_pname);
opal_output_verbose(2, opal_pmix_base_framework.framework_output,
"%s pmix:s1: assigned tmp name",
OPAL_NAME_PRINT(s1_pname));
pmix_kvs_name = (char*)malloc(pmix_kvslen_max);
if( pmix_kvs_name == NULL ){
ret = OPAL_ERR_OUT_OF_RESOURCE;
goto err_exit;
}
rc = PMI_KVS_Get_my_name(pmix_kvs_name, pmix_kvslen_max);
if( PMI_SUCCESS != rc ) {
OPAL_PMI_ERROR(rc, "PMI_KVS_Get_my_name");
goto err_exit;
}
/* get our local proc info to find our local rank */
if (PMI_SUCCESS != (rc = PMI_Get_clique_size(&nlranks))) {
OPAL_PMI_ERROR(rc, "PMI_Get_clique_size");
return rc;
}
/* save the local size */
OBJ_CONSTRUCT(&kv, opal_value_t);
kv.key = strdup(OPAL_PMIX_LOCAL_SIZE);
kv.type = OPAL_UINT16;
kv.data.uint16 = nlranks;
if (OPAL_SUCCESS != (ret = opal_pmix_base_store(&OPAL_PROC_MY_NAME, &kv))) {
OPAL_ERROR_LOG(ret);
OBJ_DESTRUCT(&kv);
goto err_exit;
}
OBJ_DESTRUCT(&kv);
lrank = 0;
nrank = 0;
ldr.vpid = rank;
if (0 < nlranks) {
/* now get the specific ranks */
lranks = (int*)calloc(nlranks, sizeof(int));
if (NULL == lranks) {
rc = OPAL_ERR_OUT_OF_RESOURCE;
OPAL_ERROR_LOG(rc);
return rc;
}
if (PMI_SUCCESS != (rc = PMI_Get_clique_ranks(lranks, nlranks))) {
OPAL_PMI_ERROR(rc, "PMI_Get_clique_ranks");
free(lranks);
return rc;
}
/* note the local ldr */
ldr.vpid = lranks[0];
/* save this */
memset(tmp, 0, 64);
for (i=0; i < nlranks; i++) {
(void)snprintf(tmp, 64, "%d", lranks[i]);
opal_argv_append_nosize(&localranks, tmp);
if (rank == lranks[i]) {
lrank = i;
nrank = i;
}
}
str = opal_argv_join(localranks, ',');
opal_argv_free(localranks);
OBJ_CONSTRUCT(&kv, opal_value_t);
kv.key = strdup(OPAL_PMIX_LOCAL_PEERS);
kv.type = OPAL_STRING;
kv.data.string = str;
if (OPAL_SUCCESS != (ret = opal_pmix_base_store(&OPAL_PROC_MY_NAME, &kv))) {
OPAL_ERROR_LOG(ret);
OBJ_DESTRUCT(&kv);
goto err_exit;
}
OBJ_DESTRUCT(&kv);
}
/* save the local leader */
OBJ_CONSTRUCT(&kv, opal_value_t);
kv.key = strdup(OPAL_PMIX_LOCALLDR);
kv.type = OPAL_UINT64;
kv.data.uint64 = *(uint64_t*)&ldr;
if (OPAL_SUCCESS != (ret = opal_pmix_base_store(&OPAL_PROC_MY_NAME, &kv))) {
OPAL_ERROR_LOG(ret);
OBJ_DESTRUCT(&kv);
goto err_exit;
}
OBJ_DESTRUCT(&kv);
/* save our local rank */
OBJ_CONSTRUCT(&kv, opal_value_t);
kv.key = strdup(OPAL_PMIX_LOCAL_RANK);
kv.type = OPAL_UINT16;
kv.data.uint16 = lrank;
if (OPAL_SUCCESS != (ret = opal_pmix_base_store(&OPAL_PROC_MY_NAME, &kv))) {
OPAL_ERROR_LOG(ret);
OBJ_DESTRUCT(&kv);
goto err_exit;
}
OBJ_DESTRUCT(&kv);
/* and our node rank */
OBJ_CONSTRUCT(&kv, opal_value_t);
kv.key = strdup(OPAL_PMIX_NODE_RANK);
kv.type = OPAL_UINT16;
kv.data.uint16 = nrank;
if (OPAL_SUCCESS != (ret = opal_pmix_base_store(&OPAL_PROC_MY_NAME, &kv))) {
OPAL_ERROR_LOG(ret);
OBJ_DESTRUCT(&kv);
goto err_exit;
}
OBJ_DESTRUCT(&kv);
/* get universe size */
ret = PMI_Get_universe_size(&i);
if (PMI_SUCCESS != ret) {
OPAL_PMI_ERROR(ret, "PMI_Get_universe_size");
goto err_exit;
}
/* push this into the dstore for subsequent fetches */
OBJ_CONSTRUCT(&kv, opal_value_t);
kv.key = strdup(OPAL_PMIX_UNIV_SIZE);
kv.type = OPAL_UINT32;
kv.data.uint32 = i;
if (OPAL_SUCCESS != (ret = opal_pmix_base_store(&OPAL_PROC_MY_NAME, &kv))) {
OPAL_ERROR_LOG(ret);
OBJ_DESTRUCT(&kv);
goto err_exit;
}
OBJ_DESTRUCT(&kv);
/* get job size */
ret = PMI_Get_size(&i);
if (PMI_SUCCESS != ret) {
OPAL_PMI_ERROR(ret, "PMI_Get_size");
goto err_exit;
}
OBJ_CONSTRUCT(&kv, opal_value_t);
kv.key = strdup(OPAL_PMIX_JOB_SIZE);
kv.type = OPAL_UINT32;
kv.data.uint32 = i;
if (OPAL_SUCCESS != (ret = opal_pmix_base_store(&OPAL_PROC_MY_NAME, &kv))) {
OPAL_ERROR_LOG(ret);
OBJ_DESTRUCT(&kv);
goto err_exit;
}
OBJ_DESTRUCT(&kv);
/* get appnum */
ret = PMI_Get_appnum(&i);
if (PMI_SUCCESS != ret) {
OPAL_PMI_ERROR(ret, "PMI_Get_appnum");
goto err_exit;
}
OBJ_CONSTRUCT(&kv, opal_value_t);
kv.key = strdup(OPAL_PMIX_APPNUM);
kv.type = OPAL_UINT32;
kv.data.uint32 = i;
if (OPAL_SUCCESS != (ret = opal_pmix_base_store(&OPAL_PROC_MY_NAME, &kv))) {
OPAL_ERROR_LOG(ret);
OBJ_DESTRUCT(&kv);
goto err_exit;
}
OBJ_DESTRUCT(&kv);
/* increment the init count */
++pmix_init_count;
return OPAL_SUCCESS;
err_exit:
PMI_Finalize();
return ret;
}
static int parse_args(int argc, char *argv[])
{
int i, ret, len;
opal_cmd_line_t cmd_line;
char **app_env = NULL, **global_env = NULL;
char * tmp_env_var = NULL;
char *argv0 = NULL;
orte_restart_globals_t tmp = { false, /* help */
NULL, /* filename */
NULL, /* appfile */
false, /* verbose */
false, /* forked */
-1, /* seq_number */
NULL, /* hostfile */
-1, /* output*/
false, /* info only */
false, /* app only */
false, /* showme */
NULL}; /* mpirun_opts */
orte_restart_globals = tmp;
#if OPAL_ENABLE_CRDEBUG == 1
orte_restart_globals.enable_crdebug = false;
#endif
#if OPAL_ENABLE_FT_CR == 0
/* Warn and exit if not configured with Checkpoint/Restart */
{
char *str, *args = NULL;
args = opal_cmd_line_get_usage_msg(&cmd_line);
str = opal_show_help_string("help-orte-restart.txt", "usage-no-cr",
true, args);
if (NULL != str) {
printf("%s", str);
free(str);
}
free(args);
return ORTE_ERROR;
}
#endif
/* Parse the command line options */
opal_cmd_line_create(&cmd_line, cmd_line_opts);
mca_base_open();
mca_base_cmd_line_setup(&cmd_line);
ret = opal_cmd_line_parse(&cmd_line, true, false, argc, argv);
if (OPAL_SUCCESS != ret) {
if (OPAL_ERR_SILENT != ret) {
fprintf(stderr, "%s: command line error (%s)\n", argv[0],
opal_strerror(ret));
}
return 1;
}
if (orte_restart_globals.help) {
char *str, *args = NULL;
args = opal_cmd_line_get_usage_msg(&cmd_line);
str = opal_show_help_string("help-orte-restart.txt", "usage", true,
args);
if (NULL != str) {
printf("%s", str);
free(str);
}
free(args);
/* If we show the help message, that should be all we do */
exit(0);
}
/**
* Put all of the MCA arguments in the environment
*/
mca_base_cmd_line_process_args(argv, &app_env, &global_env);
len = opal_argv_count(app_env);
for(i = 0; i < len; ++i) {
putenv(app_env[i]);
}
len = opal_argv_count(global_env);
for(i = 0; i < len; ++i) {
putenv(global_env[i]);
}
(void) mca_base_var_env_name("opal_cr_is_tool", &tmp_env_var);
opal_setenv(tmp_env_var,
"1",
true, &environ);
free(tmp_env_var);
tmp_env_var = NULL;
/**
* Now start parsing our specific arguments
*/
/* get the remaining bits */
argv0 = strdup(argv[0]);
opal_cmd_line_get_tail(&cmd_line, &argc, &argv);
if (0 == argc) {
fprintf(stderr, "%s: Nothing to do\n", argv0);
fprintf(stderr, "Type '%s --help' for usge.\n", argv0);
free(argv0);
return ORTE_ERROR;
}
free(argv0);
orte_restart_globals.snapshot_ref = strdup(argv[0]);
if ( NULL == orte_restart_globals.snapshot_ref ||
0 >= strlen(orte_restart_globals.snapshot_ref) ) {
opal_show_help("help-orte-restart.txt", "invalid_filename", true,
"<none provided>");
return ORTE_ERROR;
}
/* If we have arguments after the command, then assume they
* need to be grouped together.
*/
if(argc > 1) {
orte_restart_globals.snapshot_ref = strdup(opal_argv_join(argv, ' '));
}
return ORTE_SUCCESS;
}
static int store(struct orcm_db_base_module_t *imod,
const char *primary_key,
opal_list_t *kvs)
{
int i, rc;
char *sql, **cmd = NULL, *tmp;
sqlite3_stmt *stmt;
opal_value_t *kv;
mca_db_sqlite_module_t *mod = (mca_db_sqlite_module_t*)imod;
/* setup the insert statement */
for (i=0; i < (int)opal_list_get_size(kvs); i++) {
opal_argv_append_nosize(&cmd, "?");
}
tmp = opal_argv_join(cmd, ',');
asprintf(&sql, "INSERT INTO %s VALUES (%s)", mod->dbfile, tmp);
free(tmp);
opal_argv_free(cmd);
/* use the next worker thread */
ORCM_SQLITE_CMD(prepare_v2(mod->dbhandles[mod->active], sql, strlen(sql)+1, &stmt, NULL), mod->dbhandles[mod->active], &rc);
if (SQLITE_OK != rc) {
return ORCM_ERROR;
}
/* cycle through the provided values and construct
* an insert command for them - note that the values
* MUST be in column-order for the database!
*/
OPAL_LIST_FOREACH(kv, kvs, opal_value_t) {
switch (kv->type) {
case OPAL_STRING:
ORCM_SQLITE_CMD(bind_text(stmt, i, kv->data.string, strlen(kv->data.string), NULL),
mod->dbhandles[mod->active], &rc);
break;
case OPAL_INT32:
ORCM_SQLITE_CMD(bind_int(stmt, i, kv->data.int32), mod->dbhandles[mod->active], &rc);
break;
case OPAL_INT16:
ORCM_SQLITE_CMD(bind_int(stmt, i, kv->data.int16), mod->dbhandles[mod->active], &rc);
break;
case OPAL_PID:
ORCM_SQLITE_CMD(bind_int64(stmt, i, kv->data.pid), mod->dbhandles[mod->active], &rc);
break;
case OPAL_INT64:
ORCM_SQLITE_CMD(bind_int64(stmt, i, kv->data.int64), mod->dbhandles[mod->active], &rc);
break;
case OPAL_FLOAT:
ORCM_SQLITE_CMD(bind_double(stmt, i, kv->data.fval), mod->dbhandles[mod->active], &rc);
break;
case OPAL_TIMEVAL:
asprintf(&tmp, "%d.%06d", (int)kv->data.tv.tv_sec, (int)kv->data.tv.tv_usec);
ORCM_SQLITE_CMD(bind_text(stmt, i, tmp, strlen(tmp), NULL),
mod->dbhandles[mod->active], &rc);
free(tmp);
break;
}
if (SQLITE_OK != rc) {
return ORCM_ERROR;
}
}
ORCM_SQLITE_OP(step(stmt), DONE, mod->dbhandles[mod->active], &rc);
if (SQLITE_OK != rc) {
return ORCM_ERROR;
}
opal_output_verbose(2, orcm_db_base_framework.framework_output,
"INSERTED ROW %d", (int)sqlite3_last_insert_rowid(mod->dbhandles[mod->active]));
/* cycle to the next worker thread */
mod->active++;
if (mod->nthreads < mod->active) {
mod->active = 0;
}
return ORCM_SUCCESS;
}
int orte_pls_rsh_launch(orte_jobid_t jobid)
{
orte_job_map_t *map;
opal_list_item_t *n_item;
orte_mapped_node_t *rmaps_node;
orte_std_cntr_t num_nodes;
orte_vpid_t vpid;
int node_name_index1;
int node_name_index2;
int proc_name_index;
int local_exec_index, local_exec_index_end;
char *jobid_string = NULL;
char *uri, *param;
char **argv = NULL, **tmp;
char *prefix_dir;
int argc;
int rc;
sigset_t sigs;
struct passwd *p;
bool remote_sh = false, remote_csh = false;
bool local_sh = false, local_csh = false;
char *lib_base = NULL, *bin_base = NULL;
orte_pls_daemon_info_t *dmn;
orte_pls_rsh_shell_t shell;
if (mca_pls_rsh_component.timing) {
if (0 != gettimeofday(&joblaunchstart, NULL)) {
opal_output(0, "pls_rsh: could not obtain start time");
joblaunchstart.tv_sec = 0;
joblaunchstart.tv_usec = 0;
}
}
/* setup a list that will contain the info for all the daemons
* so we can store it on the registry when done and use it
* locally to track their state
*/
OBJ_CONSTRUCT(&active_daemons, opal_list_t);
/* Get the map for this job
* We need the entire mapping for a couple of reasons:
* - need the prefix to start with.
* - need to know the nodes we are launching on
* All other mapping responsibilities fall to orted in the fork PLS
*/
rc = orte_rmaps.get_job_map(&map, jobid);
if (ORTE_SUCCESS != rc) {
ORTE_ERROR_LOG(rc);
OBJ_DESTRUCT(&active_daemons);
return rc;
}
/* if the user requested that we re-use daemons,
* launch the procs on any existing, re-usable daemons
*/
if (orte_pls_base.reuse_daemons) {
if (ORTE_SUCCESS != (rc = orte_pls_base_launch_on_existing_daemons(map))) {
ORTE_ERROR_LOG(rc);
OBJ_RELEASE(map);
OBJ_DESTRUCT(&active_daemons);
return rc;
}
}
num_nodes = (orte_std_cntr_t)opal_list_get_size(&map->nodes);
if (0 == num_nodes) {
/* nothing left to do - just return */
OBJ_RELEASE(map);
OBJ_DESTRUCT(&active_daemons);
return ORTE_SUCCESS;
}
if (mca_pls_rsh_component.debug_daemons &&
mca_pls_rsh_component.num_concurrent < num_nodes) {
/* we can't run in this situation, so pretty print the error
* and exit
*/
opal_show_help("help-pls-rsh.txt", "deadlock-params",
true, mca_pls_rsh_component.num_concurrent, num_nodes);
OBJ_RELEASE(map);
OBJ_DESTRUCT(&active_daemons);
return ORTE_ERR_FATAL;
}
/*
* After a discussion between Ralph & Jeff, we concluded that we
* really are handling the prefix dir option incorrectly. It currently
* is associated with an app_context, yet it really refers to the
* location where OpenRTE/Open MPI is installed on a NODE. Fixing
* this right now would involve significant change to orterun as well
* as elsewhere, so we will intentionally leave this incorrect at this
* point. The error, however, is identical to that seen in all prior
* releases of OpenRTE/Open MPI, so our behavior is no worse than before.
*
* A note to fix this, along with ideas on how to do so, has been filed
* on the project's Trac system under "feature enhancement".
*
* For now, default to the prefix_dir provided in the first app_context.
* Since there always MUST be at least one app_context, we are safe in
* doing this.
*/
prefix_dir = map->apps[0]->prefix_dir;
/*
* Allocate a range of vpids for the daemons.
*/
if (num_nodes == 0) {
return ORTE_ERR_BAD_PARAM;
}
rc = orte_ns.reserve_range(0, num_nodes, &vpid);
if (ORTE_SUCCESS != rc) {
goto cleanup;
}
/* setup the orted triggers for passing their launch info */
if (ORTE_SUCCESS != (rc = orte_smr.init_orted_stage_gates(jobid, num_nodes, NULL, NULL))) {
ORTE_ERROR_LOG(rc);
goto cleanup;
}
/* need integer value for command line parameter */
if (ORTE_SUCCESS != (rc = orte_ns.convert_jobid_to_string(&jobid_string, jobid))) {
ORTE_ERROR_LOG(rc);
goto cleanup;
}
/* What is our local shell? */
shell = ORTE_PLS_RSH_SHELL_UNKNOWN;
p = getpwuid(getuid());
if (NULL != p) {
param = p->pw_shell;
shell = find_shell(p->pw_shell);
}
/* If we didn't find it in getpwuid(), try looking at the $SHELL
environment variable (see
https://svn.open-mpi.org/trac/ompi/ticket/1060) */
if (ORTE_PLS_RSH_SHELL_UNKNOWN == shell &&
NULL != (param = getenv("SHELL"))) {
shell = find_shell(param);
}
switch (shell) {
case ORTE_PLS_RSH_SHELL_SH: /* fall through */
case ORTE_PLS_RSH_SHELL_KSH: /* fall through */
case ORTE_PLS_RSH_SHELL_ZSH: /* fall through */
case ORTE_PLS_RSH_SHELL_BASH: local_sh = true; break;
case ORTE_PLS_RSH_SHELL_TCSH: /* fall through */
case ORTE_PLS_RSH_SHELL_CSH: local_csh = true; break;
default:
opal_output(0, "WARNING: local probe returned unhandled shell:%s assuming bash\n",
(NULL != param) ? param : "unknown");
remote_sh = true;
break;
}
if (mca_pls_rsh_component.debug) {
opal_output(0, "pls:rsh: local csh: %d, local sh: %d\n",
local_csh, local_sh);
}
/* What is our remote shell? */
if (mca_pls_rsh_component.assume_same_shell) {
remote_sh = local_sh;
remote_csh = local_csh;
if (mca_pls_rsh_component.debug) {
opal_output(0, "pls:rsh: assuming same remote shell as local shell");
}
} else {
orte_pls_rsh_shell_t shell;
rmaps_node = (orte_mapped_node_t*)opal_list_get_first(&map->nodes);
rc = orte_pls_rsh_probe(rmaps_node, &shell);
if (ORTE_SUCCESS != rc) {
ORTE_ERROR_LOG(rc);
return rc;
}
switch (shell) {
case ORTE_PLS_RSH_SHELL_SH: /* fall through */
case ORTE_PLS_RSH_SHELL_KSH: /* fall through */
case ORTE_PLS_RSH_SHELL_ZSH: /* fall through */
case ORTE_PLS_RSH_SHELL_BASH: remote_sh = true; break;
case ORTE_PLS_RSH_SHELL_TCSH: /* fall through */
case ORTE_PLS_RSH_SHELL_CSH: remote_csh = true; break;
default:
opal_output(0, "WARNING: rsh probe returned unhandled shell; assuming bash\n");
remote_sh = true;
}
}
if (mca_pls_rsh_component.debug) {
opal_output(0, "pls:rsh: remote csh: %d, remote sh: %d\n",
remote_csh, remote_sh);
}
/*
* Build argv array
*/
argv = opal_argv_copy(mca_pls_rsh_component.agent_argv);
argc = mca_pls_rsh_component.agent_argc;
node_name_index1 = argc;
opal_argv_append(&argc, &argv, "<template>");
/* Do we need to source .profile on the remote side? */
if (!(remote_csh || remote_sh)) {
int i;
tmp = opal_argv_split("( test ! -r ./.profile || . ./.profile;", ' ');
if (NULL == tmp) {
return ORTE_ERR_OUT_OF_RESOURCE;
}
for (i = 0; NULL != tmp[i]; ++i) {
opal_argv_append(&argc, &argv, tmp[i]);
}
opal_argv_free(tmp);
}
/* add the daemon command (as specified by user) */
local_exec_index = argc;
opal_argv_append(&argc, &argv, mca_pls_rsh_component.orted);
/* check for debug flags */
orte_pls_base_mca_argv(&argc, &argv);
opal_argv_append(&argc, &argv, "--bootproxy");
opal_argv_append(&argc, &argv, jobid_string);
opal_argv_append(&argc, &argv, "--name");
proc_name_index = argc;
opal_argv_append(&argc, &argv, "<template>");
/* tell the daemon how many procs are in the daemon's job */
opal_argv_append(&argc, &argv, "--num_procs");
asprintf(¶m, "%lu", (unsigned long)(vpid + num_nodes));
opal_argv_append(&argc, &argv, param);
free(param);
/* tell the daemon the starting vpid of the daemon's job */
opal_argv_append(&argc, &argv, "--vpid_start");
opal_argv_append(&argc, &argv, "0");
opal_argv_append(&argc, &argv, "--nodename");
node_name_index2 = argc;
opal_argv_append(&argc, &argv, "<template>");
/* pass along the universe name and location info */
opal_argv_append(&argc, &argv, "--universe");
asprintf(¶m, "%s@%s:%s", orte_universe_info.uid,
orte_universe_info.host, orte_universe_info.name);
opal_argv_append(&argc, &argv, param);
free(param);
/* setup ns contact info */
opal_argv_append(&argc, &argv, "--nsreplica");
if (NULL != orte_process_info.ns_replica_uri) {
uri = strdup(orte_process_info.ns_replica_uri);
} else {
uri = orte_rml.get_uri();
}
asprintf(¶m, "\"%s\"", uri);
opal_argv_append(&argc, &argv, param);
free(uri);
free(param);
/* setup gpr contact info */
opal_argv_append(&argc, &argv, "--gprreplica");
if (NULL != orte_process_info.gpr_replica_uri) {
uri = strdup(orte_process_info.gpr_replica_uri);
} else {
uri = orte_rml.get_uri();
}
asprintf(¶m, "\"%s\"", uri);
opal_argv_append(&argc, &argv, param);
free(uri);
free(param);
local_exec_index_end = argc;
if (!(remote_csh || remote_sh)) {
opal_argv_append(&argc, &argv, ")");
}
if (mca_pls_rsh_component.debug) {
param = opal_argv_join(argv, ' ');
if (NULL != param) {
opal_output(0, "pls:rsh: final template argv:");
opal_output(0, "pls:rsh: %s", param);
free(param);
}
}
/* Figure out the basenames for the libdir and bindir. This
requires some explanation:
- Use opal_install_dirs.libdir and opal_install_dirs.bindir instead of -D'ing some macros
in this directory's Makefile.am because it makes all the
dependencies work out correctly. These are defined in
opal/install_dirs.h.
- After a discussion on the devel-core mailing list, the
developers decided that we should use the local directory
basenames as the basis for the prefix on the remote note.
This does not handle a few notable cases (e.g., f the
libdir/bindir is not simply a subdir under the prefix, if the
libdir/bindir basename is not the same on the remote node as
it is here in the local node, etc.), but we decided that
--prefix was meant to handle "the common case". If you need
something more complex than this, a) edit your shell startup
files to set PATH/LD_LIBRARY_PATH properly on the remove
node, or b) use some new/to-be-defined options that
explicitly allow setting the bindir/libdir on the remote
node. We decided to implement these options (e.g.,
--remote-bindir and --remote-libdir) to orterun when it
actually becomes a problem for someone (vs. a hypothetical
situation).
Hence, for now, we simply take the basename of this install's
libdir and bindir and use it to append this install's prefix
and use that on the remote node.
*/
lib_base = opal_basename(opal_install_dirs.libdir);
bin_base = opal_basename(opal_install_dirs.bindir);
/*
* Iterate through each of the nodes
*/
if (mca_pls_rsh_component.timing) {
/* allocate space to track the start times */
launchstart = (struct timeval*)malloc((num_nodes+vpid) * sizeof(struct timeval));
}
for(n_item = opal_list_get_first(&map->nodes);
n_item != opal_list_get_end(&map->nodes);
n_item = opal_list_get_next(n_item)) {
orte_process_name_t* name;
pid_t pid;
char *exec_path;
char **exec_argv;
rmaps_node = (orte_mapped_node_t*)n_item;
if (mca_pls_rsh_component.timing) {
if (0 != gettimeofday(&launchstart[vpid], NULL)) {
opal_output(0, "pls_rsh: could not obtain start time");
}
}
/* new daemon - setup to record its info */
dmn = OBJ_NEW(orte_pls_daemon_info_t);
dmn->active_job = jobid;
opal_list_append(&active_daemons, &dmn->super);
/* setup node name */
free(argv[node_name_index1]);
if (NULL != rmaps_node->username &&
0 != strlen (rmaps_node->username)) {
asprintf (&argv[node_name_index1], "%s@%s",
rmaps_node->username, rmaps_node->nodename);
} else {
argv[node_name_index1] = strdup(rmaps_node->nodename);
}
free(argv[node_name_index2]);
argv[node_name_index2] = strdup(rmaps_node->nodename);
/* save it in the daemon info */
dmn->nodename = strdup(rmaps_node->nodename);
/* initialize daemons process name */
rc = orte_ns.create_process_name(&name, rmaps_node->cell, 0, vpid);
if (ORTE_SUCCESS != rc) {
ORTE_ERROR_LOG(rc);
goto cleanup;
}
/* save it in the daemon info */
dmn->cell = rmaps_node->cell;
if (ORTE_SUCCESS != (rc = orte_dss.copy((void**)&(dmn->name), name, ORTE_NAME))) {
ORTE_ERROR_LOG(rc);
goto cleanup;
}
/* fork a child to exec the rsh/ssh session */
/* set the process state to "launched" */
if (ORTE_SUCCESS != (rc = orte_smr.set_proc_state(name, ORTE_PROC_STATE_LAUNCHED, 0))) {
ORTE_ERROR_LOG(rc);
goto cleanup;
}
pid = fork();
if (pid < 0) {
rc = ORTE_ERR_OUT_OF_RESOURCE;
goto cleanup;
}
/* child */
if (pid == 0) {
char* name_string;
char** env;
char* var;
long fd, fdmax = sysconf(_SC_OPEN_MAX);
if (mca_pls_rsh_component.debug) {
opal_output(0, "pls:rsh: launching on node %s\n",
rmaps_node->nodename);
}
/* We don't need to sense an oversubscribed condition and set the sched_yield
* for the node as we are only launching the daemons at this time. The daemons
* are now smart enough to set the oversubscribed condition themselves when
* they launch the local procs.
*/
/* Is this a local launch?
*
* Not all node names may be resolvable (if we found
* localhost in the hostfile, for example). So first
* check trivial case of node_name being same as the
* current nodename, which must be local. If that doesn't
* match, check using ifislocal().
*/
if (!mca_pls_rsh_component.force_rsh &&
(0 == strcmp(rmaps_node->nodename, orte_system_info.nodename) ||
opal_ifislocal(rmaps_node->nodename))) {
if (mca_pls_rsh_component.debug) {
opal_output(0, "pls:rsh: %s is a LOCAL node\n",
rmaps_node->nodename);
}
if (mca_pls_rsh_component.timing) {
/* since this is a local launch, the daemon will never reach
* the waitpid callback - so set the start value to
* something nonsensical
*/
launchstart[vpid].tv_sec = 0;
launchstart[vpid].tv_usec = 0;
}
exec_path = opal_path_findv(argv[local_exec_index], 0, environ, NULL);
if (NULL == exec_path && NULL == prefix_dir) {
rc = orte_pls_rsh_fill_exec_path (&exec_path);
if (ORTE_SUCCESS != rc) {
exit(-1); /* the forked process MUST exit */
}
} else {
if (NULL != prefix_dir) {
exec_path = opal_os_path( false, prefix_dir, bin_base, "orted", NULL );
}
/* If we yet did not fill up the execpath, do so now */
if (NULL == exec_path) {
rc = orte_pls_rsh_fill_exec_path (&exec_path);
if (ORTE_SUCCESS != rc) {
exit(-1); /* the forked process MUST exit */
}
}
}
/* If we have a prefix, then modify the PATH and
LD_LIBRARY_PATH environment variables. We're
already in the child process, so it's ok to modify
environ. */
if (NULL != prefix_dir) {
char *oldenv, *newenv;
/* Reset PATH */
newenv = opal_os_path( false, prefix_dir, bin_base, NULL );
oldenv = getenv("PATH");
if (NULL != oldenv) {
char *temp;
asprintf(&temp, "%s:%s", newenv, oldenv );
free( newenv );
newenv = temp;
}
opal_setenv("PATH", newenv, true, &environ);
if (mca_pls_rsh_component.debug) {
opal_output(0, "pls:rsh: reset PATH: %s", newenv);
}
free(newenv);
/* Reset LD_LIBRARY_PATH */
newenv = opal_os_path( false, prefix_dir, lib_base, NULL );
oldenv = getenv("LD_LIBRARY_PATH");
if (NULL != oldenv) {
char* temp;
asprintf(&temp, "%s:%s", newenv, oldenv);
free(newenv);
newenv = temp;
}
opal_setenv("LD_LIBRARY_PATH", newenv, true, &environ);
if (mca_pls_rsh_component.debug) {
opal_output(0, "pls:rsh: reset LD_LIBRARY_PATH: %s",
newenv);
}
free(newenv);
}
/* Since this is a local execution, we need to
potentially whack the final ")" in the argv (if
sh/csh conditionals, from above). Note that we're
modifying the argv[] in the child process, so
there's no need to save this and restore it
afterward -- the parent's argv[] is unmodified. */
if (NULL != argv[local_exec_index_end]) {
free(argv[local_exec_index_end]);
argv[local_exec_index_end] = NULL;
}
/* tell the daemon to setup its own process session/group */
opal_argv_append(&argc, &argv, "--set-sid");
exec_argv = &argv[local_exec_index];
/* Finally, chdir($HOME) because we're making the
assumption that this is what will happen on
remote nodes (via rsh/ssh). This allows a user
to specify a path that is relative to $HOME for
both the cwd and argv[0] and it will work on
all nodes -- including the local nost.
Otherwise, it would work on remote nodes and
not the local node. If the user does not start
in $HOME on the remote nodes... well... let's
hope they start in $HOME. :-) */
var = getenv("HOME");
if (NULL != var) {
if (mca_pls_rsh_component.debug) {
opal_output(0, "pls:rsh: changing to directory %s", var);
}
/* Ignore errors -- what are we going to do?
(and we ignore errors on the remote nodes
in the fork pls, so this is consistent) */
chdir(var);
}
} else {
if (mca_pls_rsh_component.debug) {
opal_output(0, "pls:rsh: %s is a REMOTE node\n",
rmaps_node->nodename);
}
exec_argv = argv;
exec_path = strdup(mca_pls_rsh_component.agent_path);
if (NULL != prefix_dir) {
char *opal_prefix = getenv("OPAL_PREFIX");
if (remote_sh) {
asprintf (&argv[local_exec_index],
"%s%s%s PATH=%s/%s:$PATH ; export PATH ; "
"LD_LIBRARY_PATH=%s/%s:$LD_LIBRARY_PATH ; export LD_LIBRARY_PATH ; "
"%s/%s/%s",
(opal_prefix != NULL ? "OPAL_PREFIX=" : ""),
(opal_prefix != NULL ? opal_prefix : ""),
(opal_prefix != NULL ? " ;" : ""),
prefix_dir, bin_base,
prefix_dir, lib_base,
prefix_dir, bin_base,
mca_pls_rsh_component.orted);
}
if (remote_csh) {
/* [t]csh is a bit more challenging -- we
have to check whether LD_LIBRARY_PATH
is already set before we try to set it.
Must be very careful about obeying
[t]csh's order of evaluation and not
using a variable before it is defined.
See this thread for more details:
http://www.open-mpi.org/community/lists/users/2006/01/0517.php. */
asprintf (&argv[local_exec_index],
"%s%s%s set path = ( %s/%s $path ) ; "
"if ( $?LD_LIBRARY_PATH == 1 ) "
"set OMPI_have_llp ; "
"if ( $?LD_LIBRARY_PATH == 0 ) "
"setenv LD_LIBRARY_PATH %s/%s ; "
"if ( $?OMPI_have_llp == 1 ) "
"setenv LD_LIBRARY_PATH %s/%s:$LD_LIBRARY_PATH ; "
"%s/%s/%s",
(opal_prefix != NULL ? "setenv OPAL_PREFIX " : ""),
(opal_prefix != NULL ? opal_prefix : ""),
(opal_prefix != NULL ? " ;" : ""),
prefix_dir, bin_base,
prefix_dir, lib_base,
prefix_dir, lib_base,
prefix_dir, bin_base,
mca_pls_rsh_component.orted);
}
}
}
/* setup process name */
rc = orte_ns.get_proc_name_string(&name_string, name);
if (ORTE_SUCCESS != rc) {
opal_output(0, "orte_pls_rsh: unable to create process name");
exit(-1);
}
free(argv[proc_name_index]);
argv[proc_name_index] = strdup(name_string);
if (!mca_pls_rsh_component.debug) {
/* setup stdin */
int fd = open("/dev/null", O_RDWR);
dup2(fd, 0);
close(fd);
}
/* close all file descriptors w/ exception of stdin/stdout/stderr */
for(fd=3; fd<fdmax; fd++)
close(fd);
/* Set signal handlers back to the default. Do this close
to the execve() because the event library may (and likely
will) reset them. If we don't do this, the event
library may have left some set that, at least on some
OS's, don't get reset via fork() or exec(). Hence, the
orted could be unkillable (for example). */
set_handler_default(SIGTERM);
set_handler_default(SIGINT);
set_handler_default(SIGHUP);
set_handler_default(SIGPIPE);
set_handler_default(SIGCHLD);
/* Unblock all signals, for many of the same reasons that
we set the default handlers, above. This is noticable
on Linux where the event library blocks SIGTERM, but we
don't want that blocked by the orted (or, more
specifically, we don't want it to be blocked by the
orted and then inherited by the ORTE processes that it
forks, making them unkillable by SIGTERM). */
sigprocmask(0, 0, &sigs);
sigprocmask(SIG_UNBLOCK, &sigs, 0);
/* setup environment */
env = opal_argv_copy(environ);
var = mca_base_param_environ_variable("seed",NULL,NULL);
opal_setenv(var, "0", true, &env);
/* exec the daemon */
if (mca_pls_rsh_component.debug) {
param = opal_argv_join(exec_argv, ' ');
if (NULL != param) {
char* env_array = opal_argv_join( env, ' ' );
opal_output(0, "pls:rsh: executing: (%s) %s [%s]",
exec_path, param, env_array);
free(param); free(env_array);
}
}
execve(exec_path, exec_argv, env);
opal_output(0, "pls:rsh: execv of %s failed with errno=%s(%d)\n",
exec_path, strerror(errno), errno);
exit(-1);
} else { /* father */
OPAL_THREAD_LOCK(&mca_pls_rsh_component.lock);
/* JJH Bug:
* If we are in '--debug-daemons' we keep the ssh connection
* alive for the span of the run. If we use this option
* AND we launch on more than "num_concurrent" machines
* then we will deadlock. No connections are terminated
* until the job is complete, no job is started
* since all the orteds are waiting for all the others
* to come online, and the others ore not launched because
* we are waiting on those that have started to terminate
* their ssh tunnels. :(
*/
if (mca_pls_rsh_component.num_children++ >=
mca_pls_rsh_component.num_concurrent) {
opal_condition_wait(&mca_pls_rsh_component.cond, &mca_pls_rsh_component.lock);
}
OPAL_THREAD_UNLOCK(&mca_pls_rsh_component.lock);
/* setup callback on sigchild - wait until setup above is complete
* as the callback can occur in the call to orte_wait_cb
*/
orte_wait_cb(pid, orte_pls_rsh_wait_daemon, dmn);
/* if required - add delay to avoid problems w/ X11 authentication */
if (mca_pls_rsh_component.debug && mca_pls_rsh_component.delay) {
sleep(mca_pls_rsh_component.delay);
}
vpid++;
}
free(name);
}
/* all done, so store the daemon info on the registry */
if (ORTE_SUCCESS != (rc = orte_pls_base_store_active_daemons(&active_daemons))) {
ORTE_ERROR_LOG(rc);
}
cleanup:
OBJ_RELEASE(map);
if (NULL != lib_base) {
free(lib_base);
}
if (NULL != bin_base) {
free(bin_base);
}
if (NULL != jobid_string) free(jobid_string); /* done with this variable */
if (NULL != argv) opal_argv_free(argv);
return rc;
}
static void launch_daemons(int fd, short args, void *cbdata)
{
orte_job_map_t *map;
char *jobid_string = NULL;
char *param;
char **argv = NULL;
int argc;
int rc;
char *tmp;
char** env = NULL;
char *nodelist_flat;
char **nodelist_argv;
int nodelist_argc;
char *vpid_string;
char **custom_strings;
int num_args, i;
char *cur_prefix;
int proc_vpid_index;
orte_app_context_t *app;
orte_node_t *node;
orte_std_cntr_t nnode;
orte_job_t *daemons;
orte_state_caddy_t *state = (orte_state_caddy_t*)cbdata;
/* if we are launching debugger daemons, then just go
* do it - no new daemons will be launched
*/
if (ORTE_JOB_CONTROL_DEBUGGER_DAEMON & state->jdata->controls) {
state->jdata->state = ORTE_JOB_STATE_DAEMONS_LAUNCHED;
ORTE_ACTIVATE_JOB_STATE(state->jdata, ORTE_JOB_STATE_DAEMONS_REPORTED);
OBJ_RELEASE(state);
return;
}
/* start by setting up the virtual machine */
daemons = orte_get_job_data_object(ORTE_PROC_MY_NAME->jobid);
if (ORTE_SUCCESS != (rc = orte_plm_base_setup_virtual_machine(state->jdata))) {
ORTE_ERROR_LOG(rc);
goto cleanup;
}
/* if we don't want to launch, then don't attempt to
* launch the daemons - the user really wants to just
* look at the proposed process map
*/
if (orte_do_not_launch) {
/* set the state to indicate the daemons reported - this
* will trigger the daemons_reported event and cause the
* job to move to the following step
*/
state->jdata->state = ORTE_JOB_STATE_DAEMONS_LAUNCHED;
ORTE_ACTIVATE_JOB_STATE(state->jdata, ORTE_JOB_STATE_DAEMONS_REPORTED);
OBJ_RELEASE(state);
return;
}
/* Get the map for this job */
if (NULL == (map = daemons->map)) {
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
rc = ORTE_ERR_NOT_FOUND;
goto cleanup;
}
if (0 == map->num_new_daemons) {
/* set the state to indicate the daemons reported - this
* will trigger the daemons_reported event and cause the
* job to move to the following step
*/
OPAL_OUTPUT_VERBOSE((1, orte_plm_base_framework.framework_output,
"%s plm:alps: no new daemons to launch",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
state->jdata->state = ORTE_JOB_STATE_DAEMONS_LAUNCHED;
if (ORTE_JOB_STATE_DAEMONS_REPORTED == daemons->state) {
ORTE_ACTIVATE_JOB_STATE(state->jdata, ORTE_JOB_STATE_DAEMONS_REPORTED);
}
OBJ_RELEASE(state);
return;
}
/* need integer value for command line parameter */
orte_util_convert_jobid_to_string(&jobid_string, daemons->jobid);
/*
* start building argv array
*/
argv = NULL;
argc = 0;
/*
* ALPS aprun OPTIONS
*/
/* add the aprun command */
opal_argv_append(&argc, &argv, mca_plm_alps_component.aprun_cmd);
/* Append user defined arguments to aprun */
if ( NULL != mca_plm_alps_component.custom_args ) {
custom_strings = opal_argv_split(mca_plm_alps_component.custom_args, ' ');
num_args = opal_argv_count(custom_strings);
for (i = 0; i < num_args; ++i) {
opal_argv_append(&argc, &argv, custom_strings[i]);
}
opal_argv_free(custom_strings);
}
/* number of processors needed */
opal_argv_append(&argc, &argv, "-n");
asprintf(&tmp, "%lu", (unsigned long) map->num_new_daemons);
opal_argv_append(&argc, &argv, tmp);
free(tmp);
opal_argv_append(&argc, &argv, "-N");
opal_argv_append(&argc, &argv, "1");
opal_argv_append(&argc, &argv, "-cc");
opal_argv_append(&argc, &argv, "none");
/* create nodelist */
nodelist_argv = NULL;
nodelist_argc = 0;
for (nnode=0; nnode < map->nodes->size; nnode++) {
if (NULL == (node = (orte_node_t*)opal_pointer_array_get_item(map->nodes, nnode))) {
continue;
}
/* if the daemon already exists on this node, then
* don't include it
*/
if (node->daemon_launched) {
continue;
}
/* otherwise, add it to the list of nodes upon which
* we need to launch a daemon
*/
opal_argv_append(&nodelist_argc, &nodelist_argv, node->name);
}
if (0 == opal_argv_count(nodelist_argv)) {
orte_show_help("help-plm-alps.txt", "no-hosts-in-list", true);
rc = ORTE_ERR_FAILED_TO_START;
goto cleanup;
}
nodelist_flat = opal_argv_join(nodelist_argv, ',');
opal_argv_free(nodelist_argv);
/* if we are using all allocated nodes, then alps
* doesn't need a nodelist
*/
if (map->num_new_daemons < orte_num_allocated_nodes) {
opal_argv_append(&argc, &argv, "-L");
opal_argv_append(&argc, &argv, nodelist_flat);
}
/*
* ORTED OPTIONS
*/
/* add the daemon command (as specified by user) */
orte_plm_base_setup_orted_cmd(&argc, &argv);
/* Add basic orted command line options, including debug flags */
orte_plm_base_orted_append_basic_args(&argc, &argv,
NULL,
&proc_vpid_index,
nodelist_flat);
free(nodelist_flat);
/* tell the new daemons the base of the name list so they can compute
* their own name on the other end
*/
rc = orte_util_convert_vpid_to_string(&vpid_string, map->daemon_vpid_start);
if (ORTE_SUCCESS != rc) {
opal_output(0, "plm_alps: unable to create process name");
goto cleanup;
}
free(argv[proc_vpid_index]);
argv[proc_vpid_index] = strdup(vpid_string);
free(vpid_string);
if (mca_plm_alps_component.debug) {
param = opal_argv_join(argv, ' ');
if (NULL != param) {
opal_output(0, "plm:alps: final top-level argv:");
opal_output(0, "plm:alps: %s", param);
free(param);
}
}
/* Copy the prefix-directory specified in the
corresponding app_context. If there are multiple,
different prefix's in the app context, complain (i.e., only
allow one --prefix option for the entire alps run -- we
don't support different --prefix'es for different nodes in
the ALPS plm) */
cur_prefix = NULL;
for (i=0; i < state->jdata->apps->size; i++) {
char *app_prefix_dir;
if (NULL == (app = (orte_app_context_t*)opal_pointer_array_get_item(state->jdata->apps, i))) {
continue;
}
app_prefix_dir = app->prefix_dir;
/* Check for already set cur_prefix_dir -- if different,
complain */
if (NULL != app_prefix_dir) {
if (NULL != cur_prefix &&
0 != strcmp (cur_prefix, app_prefix_dir)) {
orte_show_help("help-plm-alps.txt", "multiple-prefixes",
true, cur_prefix, app_prefix_dir);
goto cleanup;
}
/* If not yet set, copy it; iff set, then it's the
same anyway */
if (NULL == cur_prefix) {
cur_prefix = strdup(app_prefix_dir);
if (mca_plm_alps_component.debug) {
opal_output (0, "plm:alps: Set prefix:%s",
cur_prefix);
}
}
}
}
/* setup environment */
env = opal_argv_copy(orte_launch_environ);
if (0 < opal_output_get_verbosity(orte_plm_base_framework.framework_output)) {
param = opal_argv_join(argv, ' ');
OPAL_OUTPUT_VERBOSE((1, orte_plm_base_framework.framework_output,
"%s plm:alps: final top-level argv:\n\t%s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
(NULL == param) ? "NULL" : param));
if (NULL != param) free(param);
}
/* exec the daemon(s) */
if (ORTE_SUCCESS != (rc = plm_alps_start_proc(argc, argv, env, cur_prefix))) {
ORTE_ERROR_LOG(rc);
goto cleanup;
}
/* indicate that the daemons for this job were launched */
state->jdata->state = ORTE_JOB_STATE_DAEMONS_LAUNCHED;
daemons->state = ORTE_JOB_STATE_DAEMONS_LAUNCHED;
/* flag that launch was successful, so far as we currently know */
failed_launch = false;
cleanup:
if (NULL != argv) {
opal_argv_free(argv);
}
if (NULL != env) {
opal_argv_free(env);
}
if(NULL != jobid_string) {
free(jobid_string);
}
/* cleanup the caddy */
OBJ_RELEASE(state);
/* check for failed launch - if so, force terminate */
if (failed_launch) {
ORTE_FORCED_TERMINATE(ORTE_ERROR_DEFAULT_EXIT_CODE);
}
}
/* stuff proc attributes for sending back to a proc */
int orte_pmix_server_register_nspace(orte_job_t *jdata, bool force)
{
int rc;
orte_proc_t *pptr;
int i, k, n;
opal_list_t *info, *pmap;
opal_value_t *kv;
orte_node_t *node, *mynode;
opal_vpid_t vpid;
char **list, **procs, **micro, *tmp, *regex;
orte_job_t *dmns;
orte_job_map_t *map;
orte_app_context_t *app;
uid_t uid;
gid_t gid;
opal_list_t *cache;
hwloc_obj_t machine;
opal_output_verbose(2, orte_pmix_server_globals.output,
"%s register nspace for %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_JOBID_PRINT(jdata->jobid));
/* setup the info list */
info = OBJ_NEW(opal_list_t);
uid = geteuid();
gid = getegid();
/* pass our nspace/rank */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_SERVER_NSPACE);
kv->data.string = strdup(ORTE_JOBID_PRINT(ORTE_PROC_MY_NAME->jobid));
kv->type = OPAL_STRING;
opal_list_append(info, &kv->super);
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_SERVER_RANK);
kv->data.uint32 = ORTE_PROC_MY_NAME->vpid;
kv->type = OPAL_UINT32;
opal_list_append(info, &kv->super);
/* jobid */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_JOBID);
kv->data.string = strdup(ORTE_JOBID_PRINT(jdata->jobid));
kv->type = OPAL_STRING;
opal_list_append(info, &kv->super);
/* offset */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_NPROC_OFFSET);
kv->data.uint32 = jdata->offset;
kv->type = OPAL_UINT32;
opal_list_append(info, &kv->super);
/* check for cached values to add to the job info */
cache = NULL;
if (orte_get_attribute(&jdata->attributes, ORTE_JOB_INFO_CACHE, (void**)&cache, OPAL_PTR) &&
NULL != cache) {
while (NULL != (kv = (opal_value_t*)opal_list_remove_first(cache))) {
opal_list_append(info, &kv->super);
}
orte_remove_attribute(&jdata->attributes, ORTE_JOB_INFO_CACHE);
OBJ_RELEASE(cache);
}
/* assemble the node and proc map info */
list = NULL;
procs = NULL;
map = jdata->map;
for (i=0; i < map->nodes->size; i++) {
micro = NULL;
if (NULL != (node = (orte_node_t*)opal_pointer_array_get_item(map->nodes, i))) {
opal_argv_append_nosize(&list, node->name);
/* assemble all the ranks for this job that are on this node */
for (k=0; k < node->procs->size; k++) {
if (NULL != (pptr = (orte_proc_t*)opal_pointer_array_get_item(node->procs, k))) {
if (jdata->jobid == pptr->name.jobid) {
opal_argv_append_nosize(µ, ORTE_VPID_PRINT(pptr->name.vpid));
}
}
}
/* assemble the rank/node map */
if (NULL != micro) {
tmp = opal_argv_join(micro, ',');
opal_argv_free(micro);
opal_argv_append_nosize(&procs, tmp);
free(tmp);
}
}
}
/* let the PMIx server generate the nodemap regex */
if (NULL != list) {
tmp = opal_argv_join(list, ',');
opal_argv_free(list);
list = NULL;
if (OPAL_SUCCESS != (rc = opal_pmix.generate_regex(tmp, ®ex))) {
ORTE_ERROR_LOG(rc);
free(tmp);
OPAL_LIST_RELEASE(info);
return rc;
}
free(tmp);
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_NODE_MAP);
kv->type = OPAL_STRING;
kv->data.string = regex;
opal_list_append(info, &kv->super);
}
/* let the PMIx server generate the procmap regex */
if (NULL != procs) {
tmp = opal_argv_join(procs, ';');
opal_argv_free(procs);
procs = NULL;
if (OPAL_SUCCESS != (rc = opal_pmix.generate_ppn(tmp, ®ex))) {
ORTE_ERROR_LOG(rc);
free(tmp);
OPAL_LIST_RELEASE(info);
return rc;
}
free(tmp);
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_PROC_MAP);
kv->type = OPAL_STRING;
kv->data.string = regex;
opal_list_append(info, &kv->super);
}
/* get our local node */
if (NULL == (dmns = orte_get_job_data_object(ORTE_PROC_MY_NAME->jobid))) {
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
OPAL_LIST_RELEASE(info);
return ORTE_ERR_NOT_FOUND;
}
if (NULL == (pptr = (orte_proc_t*)opal_pointer_array_get_item(dmns->procs, ORTE_PROC_MY_NAME->vpid))) {
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
OPAL_LIST_RELEASE(info);
return ORTE_ERR_NOT_FOUND;
}
mynode = pptr->node;
if (NULL == mynode) {
/* cannot happen */
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
OPAL_LIST_RELEASE(info);
return ORTE_ERR_NOT_FOUND;
}
/* pass our node ID */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_NODEID);
kv->type = OPAL_UINT32;
kv->data.uint32 = mynode->index;
opal_list_append(info, &kv->super);
/* pass our node size */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_NODE_SIZE);
kv->type = OPAL_UINT32;
kv->data.uint32 = mynode->num_procs;
opal_list_append(info, &kv->super);
/* pass the number of nodes in the job */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_NUM_NODES);
kv->type = OPAL_UINT32;
kv->data.uint32 = map->num_nodes;
opal_list_append(info, &kv->super);
/* univ size */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_UNIV_SIZE);
kv->type = OPAL_UINT32;
kv->data.uint32 = jdata->total_slots_alloc;
opal_list_append(info, &kv->super);
/* job size */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_JOB_SIZE);
kv->type = OPAL_UINT32;
kv->data.uint32 = jdata->num_procs;
opal_list_append(info, &kv->super);
/* number of apps in this job */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_JOB_NUM_APPS);
kv->type = OPAL_UINT32;
kv->data.uint32 = jdata->num_apps;
opal_list_append(info, &kv->super);
/* local size */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_LOCAL_SIZE);
kv->type = OPAL_UINT32;
kv->data.uint32 = jdata->num_local_procs;
opal_list_append(info, &kv->super);
/* max procs */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_MAX_PROCS);
kv->type = OPAL_UINT32;
kv->data.uint32 = jdata->total_slots_alloc;
opal_list_append(info, &kv->super);
/* topology signature */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_TOPOLOGY_SIGNATURE);
kv->type = OPAL_STRING;
kv->data.string = strdup(orte_topo_signature);
opal_list_append(info, &kv->super);
/* total available physical memory */
machine = hwloc_get_next_obj_by_type (opal_hwloc_topology, HWLOC_OBJ_MACHINE, NULL);
if (NULL != machine) {
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_AVAIL_PHYS_MEMORY);
kv->type = OPAL_UINT64;
#if HWLOC_API_VERSION < 0x20000
kv->data.uint64 = machine->memory.total_memory;
#else
kv->data.uint64 = machine->total_memory;
#endif
opal_list_append(info, &kv->super);
}
/* pass the mapping policy used for this job */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_MAPBY);
kv->type = OPAL_STRING;
kv->data.string = strdup(orte_rmaps_base_print_mapping(jdata->map->mapping));
opal_list_append(info, &kv->super);
/* pass the ranking policy used for this job */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_RANKBY);
kv->type = OPAL_STRING;
kv->data.string = strdup(orte_rmaps_base_print_ranking(jdata->map->ranking));
opal_list_append(info, &kv->super);
/* pass the binding policy used for this job */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_BINDTO);
kv->type = OPAL_STRING;
kv->data.string = strdup(opal_hwloc_base_print_binding(jdata->map->binding));
opal_list_append(info, &kv->super);
/* register any local clients */
vpid = ORTE_VPID_MAX;
micro = NULL;
for (i=0; i < mynode->procs->size; i++) {
if (NULL == (pptr = (orte_proc_t*)opal_pointer_array_get_item(mynode->procs, i))) {
continue;
}
if (pptr->name.jobid == jdata->jobid) {
opal_argv_append_nosize(µ, ORTE_VPID_PRINT(pptr->name.vpid));
if (pptr->name.vpid < vpid) {
vpid = pptr->name.vpid;
}
/* go ahead and register this client */
if (OPAL_SUCCESS != (rc = opal_pmix.server_register_client(&pptr->name, uid, gid,
(void*)pptr, NULL, NULL))) {
ORTE_ERROR_LOG(rc);
}
}
}
if (NULL != micro) {
/* pass the local peers */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_LOCAL_PEERS);
kv->type = OPAL_STRING;
kv->data.string = opal_argv_join(micro, ',');
opal_argv_free(micro);
opal_list_append(info, &kv->super);
}
/* pass the local ldr */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_LOCALLDR);
kv->type = OPAL_VPID;
kv->data.name.vpid = vpid;
opal_list_append(info, &kv->super);
/* for each proc in this job, create an object that
* includes the info describing the proc so the recipient has a complete
* picture. This allows procs to connect to each other without
* any further info exchange, assuming the underlying transports
* support it. We also pass all the proc-specific data here so
* that each proc can lookup info about every other proc in the job */
for (n=0; n < map->nodes->size; n++) {
if (NULL == (node = (orte_node_t*)opal_pointer_array_get_item(map->nodes, n))) {
continue;
}
/* cycle across each proc on this node, passing all data that
* varies by proc */
for (i=0; i < node->procs->size; i++) {
if (NULL == (pptr = (orte_proc_t*)opal_pointer_array_get_item(node->procs, i))) {
continue;
}
/* only consider procs from this job */
if (pptr->name.jobid != jdata->jobid) {
continue;
}
/* setup the proc map object */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_PROC_DATA);
kv->type = OPAL_PTR;
kv->data.ptr = OBJ_NEW(opal_list_t);
opal_list_append(info, &kv->super);
pmap = kv->data.ptr;
/* must start with rank */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_RANK);
kv->type = OPAL_VPID;
kv->data.name.vpid = pptr->name.vpid;
opal_list_append(pmap, &kv->super);
/* location, for local procs */
if (node == mynode) {
tmp = NULL;
if (orte_get_attribute(&pptr->attributes, ORTE_PROC_CPU_BITMAP, (void**)&tmp, OPAL_STRING) &&
NULL != tmp) {
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_LOCALITY_STRING);
kv->type = OPAL_STRING;
kv->data.string = opal_hwloc_base_get_locality_string(opal_hwloc_topology, tmp);
opal_list_append(pmap, &kv->super);
free(tmp);
} else {
/* the proc is not bound */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_LOCALITY_STRING);
kv->type = OPAL_STRING;
kv->data.string = NULL;
opal_list_append(pmap, &kv->super);
}
}
/* global/univ rank */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_GLOBAL_RANK);
kv->type = OPAL_VPID;
kv->data.name.vpid = pptr->name.vpid + jdata->offset;
opal_list_append(pmap, &kv->super);
if (1 < jdata->num_apps) {
/* appnum */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_APPNUM);
kv->type = OPAL_UINT32;
kv->data.uint32 = pptr->app_idx;
opal_list_append(pmap, &kv->super);
/* app ldr */
app = (orte_app_context_t*)opal_pointer_array_get_item(jdata->apps, pptr->app_idx);
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_APPLDR);
kv->type = OPAL_VPID;
kv->data.name.vpid = app->first_rank;
opal_list_append(pmap, &kv->super);
/* app rank */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_APP_RANK);
kv->type = OPAL_VPID;
kv->data.name.vpid = pptr->app_rank;
opal_list_append(pmap, &kv->super);
/* app size */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_APP_SIZE);
kv->type = OPAL_UINT32;
kv->data.uint32 = app->num_procs;
opal_list_append(info, &kv->super);
}
/* local rank */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_LOCAL_RANK);
kv->type = OPAL_UINT16;
kv->data.uint16 = pptr->local_rank;
opal_list_append(pmap, &kv->super);
/* node rank */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_NODE_RANK);
kv->type = OPAL_UINT16;
kv->data.uint32 = pptr->node_rank;
opal_list_append(pmap, &kv->super);
/* node ID */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_NODEID);
kv->type = OPAL_UINT32;
kv->data.uint32 = pptr->node->index;
opal_list_append(pmap, &kv->super);
if (map->num_nodes < orte_hostname_cutoff) {
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_HOSTNAME);
kv->type = OPAL_STRING;
kv->data.string = strdup(pptr->node->name);
opal_list_append(pmap, &kv->super);
}
}
}
/* mark the job as registered */
orte_set_attribute(&jdata->attributes, ORTE_JOB_NSPACE_REGISTERED, ORTE_ATTR_LOCAL, NULL, OPAL_BOOL);
/* pass it down */
/* we are in an event, so no need to callback */
rc = opal_pmix.server_register_nspace(jdata->jobid,
jdata->num_local_procs,
info, NULL, NULL);
OPAL_LIST_RELEASE(info);
/* if the user has connected us to an external server, then we must
* assume there is going to be some cross-mpirun exchange, and so
* we protect against that situation by publishing the job info
* for this job - this allows any subsequent "connect" to retrieve
* the job info */
if (NULL != orte_data_server_uri) {
opal_buffer_t buf;
OBJ_CONSTRUCT(&buf, opal_buffer_t);
if (OPAL_SUCCESS != (rc = opal_dss.pack(&buf, &jdata, 1, ORTE_JOB))) {
ORTE_ERROR_LOG(rc);
OBJ_DESTRUCT(&buf);
return rc;
}
info = OBJ_NEW(opal_list_t);
/* create a key-value with the key being the string jobid
* and the value being the byte object */
kv = OBJ_NEW(opal_value_t);
orte_util_convert_jobid_to_string(&kv->key, jdata->jobid);
kv->type = OPAL_BYTE_OBJECT;
opal_dss.unload(&buf, (void**)&kv->data.bo.bytes, &kv->data.bo.size);
OBJ_DESTRUCT(&buf);
opal_list_append(info, &kv->super);
/* set the range to be session */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_RANGE);
kv->type = OPAL_UINT;
kv->data.uint = OPAL_PMIX_RANGE_SESSION;
opal_list_append(info, &kv->super);
/* set the persistence to be app */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_PERSISTENCE);
kv->type = OPAL_INT;
kv->data.integer = OPAL_PMIX_PERSIST_APP;
opal_list_append(info, &kv->super);
/* add our effective userid to the directives */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_USERID);
kv->type = OPAL_UINT32;
kv->data.uint32 = geteuid();
opal_list_append(info, &kv->super);
/* now publish it */
if (ORTE_SUCCESS != (rc = pmix_server_publish_fn(ORTE_PROC_MY_NAME,
info, mycbfunc, info))) {
ORTE_ERROR_LOG(rc);
}
}
return rc;
}
int orte_regex_create(char *nodelist, char **regexp)
{
char *node;
char prefix[ORTE_MAX_NODE_PREFIX];
int i, j, len, startnum, nodenum, numdigits;
bool found, fullname;
char *suffix, *sfx;
orte_regex_node_t *ndreg;
orte_regex_range_t *range;
opal_list_t nodeids;
opal_list_item_t *item, *itm2;
char **regexargs = NULL, *tmp, *tmp2;
char *cptr;
/* define the default */
*regexp = NULL;
cptr = strchr(nodelist, ',');
if (NULL == cptr) {
/* if there is only one node, don't bother */
*regexp = strdup(nodelist);
return ORTE_SUCCESS;
}
/* setup the list of results */
OBJ_CONSTRUCT(&nodeids, opal_list_t);
/* cycle thru the array of nodenames */
node = nodelist;
while (NULL != (cptr = strchr(node, ',')) || 0 < strlen(node)) {
if (NULL != cptr) {
*cptr = '\0';
}
/* determine this node's prefix by looking for first non-alpha char */
fullname = false;
len = strlen(node);
startnum = -1;
memset(prefix, 0, ORTE_MAX_NODE_PREFIX);
numdigits = 0;
for (i=0, j=0; i < len; i++) {
if (!isalpha(node[i])) {
/* found a non-alpha char */
if (!isdigit(node[i])) {
/* if it is anything but a digit, we just use
* the entire name
*/
fullname = true;
break;
}
/* count the size of the numeric field - but don't
* add the digits to the prefix
*/
numdigits++;
if (startnum < 0) {
/* okay, this defines end of the prefix */
startnum = i;
}
continue;
}
if (startnum < 0) {
prefix[j++] = node[i];
}
}
if (fullname || startnum < 0) {
/* can't compress this name - just add it to the list */
ndreg = OBJ_NEW(orte_regex_node_t);
ndreg->prefix = strdup(node);
opal_list_append(&nodeids, &ndreg->super);
/* move to the next posn */
if (NULL == cptr) {
break;
}
node = cptr + 1;
continue;
}
/* convert the digits and get any suffix */
nodenum = strtol(&node[startnum], &sfx, 10);
if (NULL != sfx) {
suffix = strdup(sfx);
} else {
suffix = NULL;
}
/* is this nodeid already on our list? */
found = false;
for (item = opal_list_get_first(&nodeids);
!found && item != opal_list_get_end(&nodeids);
item = opal_list_get_next(item)) {
ndreg = (orte_regex_node_t*)item;
if (0 < strlen(prefix) && NULL == ndreg->prefix) {
continue;
}
if (0 == strlen(prefix) && NULL != ndreg->prefix) {
continue;
}
if (0 < strlen(prefix) && NULL != ndreg->prefix
&& 0 != strcmp(prefix, ndreg->prefix)) {
continue;
}
if (NULL == suffix && NULL != ndreg->suffix) {
continue;
}
if (NULL != suffix && NULL == ndreg->suffix) {
continue;
}
if (NULL != suffix && NULL != ndreg->suffix &&
0 != strcmp(suffix, ndreg->suffix)) {
continue;
}
if (numdigits != ndreg->num_digits) {
continue;
}
/* found a match - flag it */
found = true;
/* get the last range on this nodeid - we do this
* to preserve order
*/
range = (orte_regex_range_t*)opal_list_get_last(&ndreg->ranges);
if (NULL == range) {
/* first range for this nodeid */
range = OBJ_NEW(orte_regex_range_t);
range->start = nodenum;
range->cnt = 1;
opal_list_append(&ndreg->ranges, &range->super);
break;
}
/* see if the node number is out of sequence */
if (nodenum != (range->start + range->cnt)) {
/* start a new range */
range = OBJ_NEW(orte_regex_range_t);
range->start = nodenum;
range->cnt = 1;
opal_list_append(&ndreg->ranges, &range->super);
break;
}
/* everything matches - just increment the cnt */
range->cnt++;
break;
}
if (!found) {
/* need to add it */
ndreg = OBJ_NEW(orte_regex_node_t);
if (0 < strlen(prefix)) {
ndreg->prefix = strdup(prefix);
}
if (NULL != suffix) {
ndreg->suffix = strdup(suffix);
}
ndreg->num_digits = numdigits;
opal_list_append(&nodeids, &ndreg->super);
/* record the first range for this nodeid - we took
* care of names we can't compress above
*/
range = OBJ_NEW(orte_regex_range_t);
range->start = nodenum;
range->cnt = 1;
opal_list_append(&ndreg->ranges, &range->super);
}
if (NULL != suffix) {
free(suffix);
}
/* move to the next posn */
if (NULL == cptr) {
break;
}
node = cptr + 1;
}
/* begin constructing the regular expression */
while (NULL != (item = opal_list_remove_first(&nodeids))) {
ndreg = (orte_regex_node_t*)item;
/* if no ranges, then just add the name */
if (0 == opal_list_get_size(&ndreg->ranges)) {
if (NULL != ndreg->prefix) {
/* solitary node */
asprintf(&tmp, "%s", ndreg->prefix);
opal_argv_append_nosize(®exargs, tmp);
free(tmp);
}
OBJ_RELEASE(ndreg);
continue;
}
/* start the regex for this nodeid with the prefix */
if (NULL != ndreg->prefix) {
asprintf(&tmp, "%s[%d:", ndreg->prefix, ndreg->num_digits);
} else {
asprintf(&tmp, "[%d:", ndreg->num_digits);
}
/* add the ranges */
while (NULL != (itm2 = opal_list_remove_first(&ndreg->ranges))) {
range = (orte_regex_range_t*)itm2;
if (1 == range->cnt) {
asprintf(&tmp2, "%s%d,", tmp, range->start);
} else {
asprintf(&tmp2, "%s%d-%d,", tmp, range->start, range->start + range->cnt - 1);
}
free(tmp);
tmp = tmp2;
OBJ_RELEASE(range);
}
/* replace the final comma */
tmp[strlen(tmp)-1] = ']';
if (NULL != ndreg->suffix) {
/* add in the suffix, if provided */
asprintf(&tmp2, "%s%s", tmp, ndreg->suffix);
free(tmp);
tmp = tmp2;
}
opal_argv_append_nosize(®exargs, tmp);
free(tmp);
OBJ_RELEASE(ndreg);
}
/* assemble final result */
*regexp = opal_argv_join(regexargs, ',');
/* cleanup */
opal_argv_free(regexargs);
OBJ_DESTRUCT(&nodeids);
return ORTE_SUCCESS;
}
static int list_all_snapshots(void) {
int ret, exit_status = ORTE_SUCCESS;
opal_list_t *all_snapshots = NULL;
opal_list_item_t* item = NULL;
orte_sstore_base_global_snapshot_info_t *global_snapshot = NULL;
int s;
all_snapshots = OBJ_NEW(opal_list_t);
if( ORTE_SUCCESS != (ret = orte_sstore_base_get_all_snapshots(all_snapshots, NULL)) ) {
opal_output(0, "Error: Unable to list the checkpoints in the directory <%s>\n",
orte_sstore_base_global_snapshot_dir);
ORTE_ERROR_LOG(ret);
exit_status = ret;
goto cleanup;
}
/*
* For each reference
*/
for(item = opal_list_get_first(all_snapshots);
item != opal_list_get_end(all_snapshots);
item = opal_list_get_next(item) ) {
global_snapshot = (orte_sstore_base_global_snapshot_info_t*)item;
/*
* Get a list of valid sequence numbers
*/
if( ORTE_SUCCESS != (ret = orte_sstore_base_find_all_seq_nums(global_snapshot,
&(global_snapshot->num_seqs),
&(global_snapshot->all_seqs)))) {
ORTE_ERROR_LOG(ret);
exit_status = ret;
goto cleanup;
}
s = 0; /* Silence a compiler warning */
#if OPAL_ENABLE_CRDEBUG == 1
/* Pretty print the result - C/R Debug version */
if( orte_checkpoint_globals.enable_crdebug ) {
for(s = 0; s < global_snapshot->num_seqs; ++s) {
printf("-s %s %s\n", global_snapshot->all_seqs[s], global_snapshot->reference);
}
}
else
#endif
{
/* Pretty print the result */
printf("Snapshot Ref.: %s\t[",
global_snapshot->reference);
if( 0 >= global_snapshot->num_seqs ) {
printf("No Valid Checkpoints");
} else {
printf("%s",
opal_argv_join(global_snapshot->all_seqs, ','));
}
printf("]\n");
}
}
cleanup:
while (NULL != (item = opal_list_remove_first(all_snapshots))) {
OBJ_RELEASE(item);
}
OBJ_RELEASE(all_snapshots);
return exit_status;
}
void orcm_cfgi_base_dump(char **dumped, char *pfx, void *ptr, int type)
{
orcm_cfgi_app_t *app;
orcm_cfgi_exec_t *exec;
orcm_cfgi_version_t *vers;
orcm_cfgi_run_t *run;
orcm_cfgi_bin_t *bin;
orcm_cfgi_caddy_t *caddy;
orte_job_t *jdt;
int i;
char *output, *tmp, *tmp2, *pfx2;
/* protection */
if (NULL == ptr) {
opal_output(0, "DUMP WITH NULL PTR");
return;
}
switch(type) {
case ORCM_CFGI_APP:
app = (orcm_cfgi_app_t*)ptr;
/* create the string output */
asprintf(&output, "%sApplication: %s\tMax instances: %d\tNum instances: %d",
(NULL == pfx) ? "" : pfx,
(NULL == app->application) ? "NULL" : app->application,
app->max_instances, app->num_instances);
asprintf(&pfx2, "%s ", (NULL == pfx) ? "" : pfx);
for (i=0; i < app->executables.size; i++) {
if (NULL == (exec = (orcm_cfgi_exec_t*)opal_pointer_array_get_item(&app->executables, i))) {
continue;
}
orcm_cfgi_base_dump(&tmp, pfx2, exec, ORCM_CFGI_EXEC);
asprintf(&tmp2, "%s\n%s", output, tmp);
free(tmp);
free(output);
output = tmp2;
}
break;
case ORCM_CFGI_EXEC:
exec = (orcm_cfgi_exec_t*)ptr;
asprintf(&output, "%sExec: %s\tProc limit: %d\tTotal procs: %d",
(NULL == pfx) ? "" : pfx,
(NULL == exec->appname) ? "NULL" : exec->appname,
exec->process_limit, exec->total_procs);
asprintf(&pfx2, "%s ", (NULL == pfx) ? "" : pfx);
for (i=0; i < exec->versions.size; i++) {
if (NULL == (vers = (orcm_cfgi_version_t*)opal_pointer_array_get_item(&exec->versions, i))) {
continue;
}
orcm_cfgi_base_dump(&tmp, pfx2, vers, ORCM_CFGI_VERSION);
asprintf(&tmp2, "%s\n%s", output, tmp);
free(tmp);
free(output);
output = tmp2;
}
free(pfx2);
break;
case ORCM_CFGI_VERSION:
vers = (orcm_cfgi_version_t*)ptr;
if (NULL != vers->argv) {
tmp = opal_argv_join(vers->argv, ' ');
} else {
tmp = NULL;
}
asprintf(&output, "%sExec: %s\tVersion: %s\tArgv: %s\tInstalled: %s",
(NULL == pfx) ? "" : pfx,
(NULL == vers->exec) ? "NULL" : ((NULL == vers->exec->appname) ? "NULL" : vers->exec->appname),
(NULL == vers->version) ? "NULL" : vers->version,
(NULL == tmp) ? "NULL" : tmp,
(NULL == vers->mod_time) ? "NULL" : vers->mod_time);
if (NULL != tmp) {
free(tmp);
}
asprintf(&pfx2, "%s ", (NULL == pfx) ? "" : pfx);
for (i=0; i < vers->binaries.size; i++) {
if (NULL == (bin = (orcm_cfgi_bin_t*)opal_pointer_array_get_item(&vers->binaries, i))) {
continue;
}
orcm_cfgi_base_dump(&tmp, pfx2, bin, ORCM_CFGI_BIN);
asprintf(&tmp2, "%s\n%s", output, tmp);
free(tmp);
free(output);
output = tmp2;
}
free(pfx2);
break;
case ORCM_CFGI_RUN:
run = (orcm_cfgi_run_t*)ptr;
/* create the string output */
asprintf(&output, "%sApp: %s\tInstance: %s",
(NULL == pfx) ? "" : pfx,
(NULL == run->application) ? "NULL" : run->application,
(NULL == run->instance) ? "NULL" : run->instance);
asprintf(&pfx2, "%s ", (NULL == pfx) ? "" : pfx);
for (i=0; i < run->binaries.size; i++) {
if (NULL == (bin = (orcm_cfgi_bin_t*)opal_pointer_array_get_item(&run->binaries, i))) {
continue;
}
orcm_cfgi_base_dump(&tmp, pfx2, bin, ORCM_CFGI_BIN);
asprintf(&tmp2, "%s\n%s", output, tmp);
free(tmp);
free(output);
output = tmp2;
}
free(pfx2);
break;
case ORCM_CFGI_BIN:
bin = (orcm_cfgi_bin_t*)ptr;
asprintf(&output, "%sApp: %s\tVersion: %s\tBinary: %s\tNum procs: %d",
(NULL == pfx) ? "" : pfx,
(NULL == bin->appname) ? "NULL" : bin->appname,
(NULL == bin->version) ? "NULL" : bin->version,
(NULL == bin->binary) ? "NULL" : bin->binary, bin->num_procs);
break;
case ORCM_CFGI_CADDY:
caddy = (orcm_cfgi_caddy_t*)ptr;
asprintf(&output, "%sCaddy Cmd: %d\tRun: %s\tJdata: %s",
(NULL == pfx) ? "" : pfx,
(int)caddy->cmd,
(NULL == caddy->run) ? "NULL" : "NON-NULL",
(NULL == caddy->jdata) ? "NULL" : ORTE_JOBID_PRINT(caddy->jdata->jobid));
asprintf(&pfx2, "%s ", (NULL == pfx) ? "" : pfx);
if (NULL != caddy->run) {
orcm_cfgi_base_dump(&tmp, pfx2, caddy->run, ORCM_CFGI_RUN);
asprintf(&tmp2, "%s\n%s", output, tmp);
free(tmp);
free(output);
output = tmp2;
}
break;
default:
opal_output(0, "%s UNRECOGNIZED TYPE %d",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), type);
}
if (NULL != dumped) {
*dumped = output;
} else {
opal_output(0, "\n%s", output);
free(output);
}
}
int orte_rmaps_base_filter_nodes(orte_app_context_t *app,
opal_list_t *nodes, bool remove)
{
int rc=ORTE_ERR_TAKE_NEXT_OPTION;
/* did the app_context contain a hostfile? */
if (NULL != app->hostfile) {
/* yes - filter the node list through the file, removing
* any nodes not found in the file
*/
if (ORTE_SUCCESS != (rc = orte_util_filter_hostfile_nodes(nodes, app->hostfile, remove))) {
ORTE_ERROR_LOG(rc);
return rc;
}
/** check that anything is here */
if (0 == opal_list_get_size(nodes)) {
orte_show_help("help-orte-rmaps-base.txt", "orte-rmaps-base:no-mapped-node",
true, app->app, "-hostfile", app->hostfile);
return ORTE_ERR_SILENT;
}
}
/* did the app_context contain an add-hostfile? */
if (NULL != app->add_hostfile) {
/* yes - filter the node list through the file, removing
* any nodes not found in the file
*/
if (ORTE_SUCCESS != (rc = orte_util_filter_hostfile_nodes(nodes, app->add_hostfile, remove))) {
ORTE_ERROR_LOG(rc);
return rc;
}
/** check that anything is here */
if (0 == opal_list_get_size(nodes)) {
orte_show_help("help-orte-rmaps-base.txt", "orte-rmaps-base:no-mapped-node",
true, app->app, "-add-hostfile", app->hostfile);
return ORTE_ERR_SILENT;
}
}
/* now filter the list through any -host specification */
if (!orte_soft_locations && NULL != app->dash_host) {
if (ORTE_SUCCESS != (rc = orte_util_filter_dash_host_nodes(nodes, app->dash_host, remove))) {
ORTE_ERROR_LOG(rc);
return rc;
}
/** check that anything is left! */
if (0 == opal_list_get_size(nodes)) {
char *foo;
foo = opal_argv_join(app->dash_host, ',');
orte_show_help("help-orte-rmaps-base.txt", "orte-rmaps-base:no-mapped-node",
true, app->app, "-host", foo);
free(foo);
return ORTE_ERR_SILENT;
}
}
/* now filter the list through any add-host specification */
if (NULL != app->add_host) {
if (ORTE_SUCCESS != (rc = orte_util_filter_dash_host_nodes(nodes, app->add_host, remove))) {
ORTE_ERROR_LOG(rc);
return rc;
}
/** check that anything is left! */
if (0 == opal_list_get_size(nodes)) {
char *foo;
foo = opal_argv_join(app->dash_host, ',');
orte_show_help("help-orte-rmaps-base.txt", "orte-rmaps-base:no-mapped-node",
true, app->app, "-add-host", foo);
free(foo);
return ORTE_ERR_SILENT;
}
}
return rc;
}
static void launch_daemons(int fd, short args, void *cbdata)
{
orte_job_map_t *map;
size_t num_nodes;
char *param;
char **argv = NULL;
int argc;
int rc;
char** env = NULL;
char **nodelist_argv;
char *nodelist;
int nodelist_argc;
char *vpid_string;
int i;
char *cur_prefix;
int proc_vpid_index = 0;
bool failed_launch = true;
orte_app_context_t *app;
orte_node_t *node;
orte_std_cntr_t nnode;
orte_job_t *daemons;
orte_state_caddy_t *state = (orte_state_caddy_t*)cbdata;
orte_job_t *jdata = state->jdata;
/* start by setting up the virtual machine */
daemons = orte_get_job_data_object(ORTE_PROC_MY_NAME->jobid);
if (ORTE_SUCCESS != (rc = orte_plm_base_setup_virtual_machine(jdata))) {
ORTE_ERROR_LOG(rc);
goto cleanup;
}
/* if we don't want to launch, then don't attempt to
* launch the daemons - the user really wants to just
* look at the proposed process map
*/
if (orte_do_not_launch) {
/* set the state to indicate the daemons reported - this
* will trigger the daemons_reported event and cause the
* job to move to the following step
*/
state->jdata->state = ORTE_JOB_STATE_DAEMONS_LAUNCHED;
ORTE_ACTIVATE_JOB_STATE(state->jdata, ORTE_JOB_STATE_DAEMONS_REPORTED);
OBJ_RELEASE(state);
return;
}
OPAL_OUTPUT_VERBOSE((1, orte_plm_base_framework.framework_output,
"%s plm:lsf: launching vm",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
/* Get the map for this job */
if (NULL == (map = daemons->map)) {
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
rc = ORTE_ERR_NOT_FOUND;
goto cleanup;
}
num_nodes = map->num_new_daemons;
if (0 == num_nodes) {
/* set the state to indicate the daemons reported - this
* will trigger the daemons_reported event and cause the
* job to move to the following step
*/
OPAL_OUTPUT_VERBOSE((1, orte_plm_base_framework.framework_output,
"%s plm:lsf: no new daemons to launch",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
state->jdata->state = ORTE_JOB_STATE_DAEMONS_LAUNCHED;
ORTE_ACTIVATE_JOB_STATE(state->jdata, ORTE_JOB_STATE_DAEMONS_REPORTED);
OBJ_RELEASE(state);
return;
}
/* create nodelist */
nodelist_argv = NULL;
nodelist_argc = 0;
for (nnode=0; nnode < map->nodes->size; nnode++) {
if (NULL == (node = (orte_node_t*)opal_pointer_array_get_item(map->nodes, nnode))) {
continue;
}
/* if the daemon already exists on this node, then
* don't include it
*/
if (ORTE_FLAG_TEST(node, ORTE_NODE_FLAG_DAEMON_LAUNCHED)) {
continue;
}
/* otherwise, add it to the list of nodes upon which
* we need to launch a daemon
*/
opal_argv_append(&nodelist_argc, &nodelist_argv, node->name);
}
nodelist = opal_argv_join(nodelist_argv, ',');
/*
* start building argv array
*/
argv = NULL;
argc = 0;
/*
* ORTED OPTIONS
*/
/* add the daemon command (as specified by user) */
orte_plm_base_setup_orted_cmd(&argc, &argv);
/* Add basic orted command line options */
orte_plm_base_orted_append_basic_args(&argc, &argv,
"lsf",
&proc_vpid_index,
nodelist);
free(nodelist);
/* tell the new daemons the base of the name list so they can compute
* their own name on the other end
*/
rc = orte_util_convert_vpid_to_string(&vpid_string, map->daemon_vpid_start);
if (ORTE_SUCCESS != rc) {
opal_output(0, "plm_lsf: unable to get daemon vpid as string");
goto cleanup;
}
free(argv[proc_vpid_index]);
argv[proc_vpid_index] = strdup(vpid_string);
free(vpid_string);
/* protect the args in case someone has a script wrapper */
mca_base_cmd_line_wrap_args(argv);
if (0 < opal_output_get_verbosity(orte_plm_base_framework.framework_output)) {
param = opal_argv_join(argv, ' ');
if (NULL != param) {
opal_output(0, "plm:lsf: final top-level argv:");
opal_output(0, "plm:lsf: %s", param);
free(param);
}
}
/* Copy the prefix-directory specified in the
corresponding app_context. If there are multiple,
different prefix's in the app context, complain (i.e., only
allow one --prefix option for the entire lsf run -- we
don't support different --prefix'es for different nodes in
the LSF plm) */
cur_prefix = NULL;
for (i=0; i < jdata->apps->size; i++) {
char *app_prefix_dir=NULL;
if (NULL == (app = (orte_app_context_t*)opal_pointer_array_get_item(jdata->apps, i))) {
continue;
}
if (orte_get_attribute(&app->attributes, ORTE_APP_PREFIX_DIR, (void**)&app_prefix_dir, OPAL_STRING) &&
NULL != app_prefix_dir) {
/* Check for already set cur_prefix_dir -- if different,
complain */
if (NULL != cur_prefix &&
0 != strcmp (cur_prefix, app_prefix_dir)) {
orte_show_help("help-plm-lsf.txt", "multiple-prefixes",
true, cur_prefix, app_prefix_dir);
rc = ORTE_ERR_FAILED_TO_START;
goto cleanup;
}
/* If not yet set, copy it; iff set, then it's the
same anyway */
if (NULL == cur_prefix) {
cur_prefix = strdup(app_prefix_dir);
OPAL_OUTPUT_VERBOSE((1, orte_plm_base_framework.framework_output,
"%s plm:lsf: Set prefix:%s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), cur_prefix));
}
free(app_prefix_dir);
}
}
/* setup environment */
env = opal_argv_copy(orte_launch_environ);
/* lsb_launch tampers with SIGCHLD.
* After the call to lsb_launch, the signal handler for SIGCHLD is NULL.
* So, we disable the SIGCHLD handler of libevent for the duration of
* the call to lsb_launch
*/
orte_wait_disable();
/* exec the daemon(s). Do NOT wait for lsb_launch to complete as
* it only completes when the processes it starts - in this case,
* the orteds - complete. We need to go ahead and return so
* orterun can do the rest of its stuff. Instead, we'll catch any
* failures and deal with them elsewhere
*/
if (lsb_launch(nodelist_argv, argv, LSF_DJOB_REPLACE_ENV | LSF_DJOB_NOWAIT, env) < 0) {
ORTE_ERROR_LOG(ORTE_ERR_FAILED_TO_START);
opal_output(0, "lsb_launch failed: %d", rc);
rc = ORTE_ERR_FAILED_TO_START;
orte_wait_enable(); /* re-enable our SIGCHLD handler */
goto cleanup;
}
orte_wait_enable(); /* re-enable our SIGCHLD handler */
/* indicate that the daemons for this job were launched */
state->jdata->state = ORTE_JOB_STATE_DAEMONS_LAUNCHED;
daemons->state = ORTE_JOB_STATE_DAEMONS_LAUNCHED;
/* flag that launch was successful, so far as we currently know */
failed_launch = false;
cleanup:
if (NULL != argv) {
opal_argv_free(argv);
}
if (NULL != env) {
opal_argv_free(env);
}
/* cleanup the caddy */
OBJ_RELEASE(state);
/* check for failed launch - if so, force terminate */
if (failed_launch) {
ORTE_FORCED_TERMINATE(ORTE_ERROR_DEFAULT_EXIT_CODE);
}
}
/*
* Notice that the user restart callback is not called here, but always from
* opal_init for the self module.
*/
int opal_crs_self_restart(opal_crs_base_snapshot_t *base_snapshot, bool spawn_child, pid_t *child_pid)
{
opal_crs_self_snapshot_t *snapshot = OBJ_NEW(opal_crs_self_snapshot_t);
char **cr_argv = NULL;
char * cr_cmd = NULL;
int ret;
int exit_status = OPAL_SUCCESS;
int status;
snapshot->super = *base_snapshot;
opal_output_verbose(10, mca_crs_self_component.super.output_handle,
"crs:self: restart(%d)", spawn_child);
/*
* If we need to reconstruct the snapshot
*/
if(snapshot->super.cold_start) {
if( OPAL_SUCCESS != (ret = self_cold_start(snapshot)) ){
exit_status = ret;
opal_output(mca_crs_self_component.super.output_handle,
"crs:blcr: blcr_restart: Unable to reconstruct the snapshot.");
goto cleanup;
}
}
/*
* JJH: Check to make sure the application exists?
*/
/*
* Get the restart command
*/
if ( OPAL_SUCCESS != (ret = opal_crs_self_restart_cmd(snapshot, &cr_cmd)) ) {
exit_status = ret;
goto cleanup;
}
if ( NULL == (cr_argv = opal_argv_split(cr_cmd, ' ')) ) {
exit_status = OPAL_ERROR;
goto cleanup;
}
if (!spawn_child) {
opal_output_verbose(10, mca_crs_self_component.super.output_handle,
"crs:self: self_restart: SELF: exec :(%s, %s):",
strdup(cr_argv[0]),
opal_argv_join(cr_argv, ' '));
status = execvp(strdup(cr_argv[0]), cr_argv);
if(status < 0) {
opal_output(mca_crs_self_component.super.output_handle,
"crs:self: self_restart: SELF: Child failed to execute :(%d):", status);
}
opal_output(mca_crs_self_component.super.output_handle,
"crs:self: self_restart: SELF: execvp returned %d", status);
exit_status = status;
goto cleanup;
}
else {
*child_pid = fork();
if( *child_pid == 0) {
/* Child Process */
opal_output_verbose(10, mca_crs_self_component.super.output_handle,
"crs:self: self_restart: CHILD: exec :(%s, %s):",
strdup(cr_argv[0]),
opal_argv_join(cr_argv, ' '));
status = execvp(strdup(cr_argv[0]), cr_argv);
if(status < 0) {
opal_output(mca_crs_self_component.super.output_handle,
"crs:self: self_restart: CHILD: Child failed to execute :(%d):", status);
}
opal_output(mca_crs_self_component.super.output_handle,
"crs:self: self_restart: CHILD: execvp returned %d", status);
exit_status = status;
goto cleanup;
}
else if(*child_pid > 0) {
/* Parent is done once it is started. */
;
}
else {
opal_output(mca_crs_self_component.super.output_handle,
"crs:self: self_restart: CHILD: fork failed :(%d):", *child_pid);
}
}
cleanup:
if( NULL != cr_cmd)
free(cr_cmd);
if( NULL != cr_argv)
opal_argv_free(cr_argv);
return exit_status;
}
/*
* Register MCA parameters
*/
int ompi_btl_openib_connect_base_register(void)
{
int i, j, save;
char **temp = NULL, *string = NULL, *all_cpc_names = NULL;
char *cpc_include = NULL, *cpc_exclude = NULL;
/* Make an MCA parameter to select which connect module to use */
for (i = 0; NULL != all[i]; ++i) {
/* The CPC name "empty" is reserved for "fake" CPC modules */
if (0 != strcmp(all[i]->cbc_name, "empty")) {
opal_argv_append_nosize(&temp, all[i]->cbc_name);
}
}
all_cpc_names = opal_argv_join(temp, ',');
opal_argv_free(temp);
asprintf(&string,
"Method used to select OpenFabrics connections (valid values: %s)",
all_cpc_names);
mca_base_param_reg_string(&mca_btl_openib_component.super.btl_version,
"cpc_include", string, false, false,
NULL, &cpc_include);
free(string);
asprintf(&string,
"Method used to exclude OpenFabrics connections (valid values: %s)",
all_cpc_names);
mca_base_param_reg_string(&mca_btl_openib_component.super.btl_version,
"cpc_exclude", string, false, false,
NULL, &cpc_exclude);
free(string);
/* Parse the if_[in|ex]clude paramters to come up with a list of
CPCs that are available */
available = (ompi_btl_openib_connect_base_component_t **) calloc(1, sizeof(all));
/* If we have an "include" list, then find all those CPCs and put
them in available[] */
if (NULL != cpc_include) {
mca_btl_openib_component.cpc_explicitly_defined = true;
temp = opal_argv_split(cpc_include, ',');
for (save = j = 0; NULL != temp[j]; ++j) {
for (i = 0; NULL != all[i]; ++i) {
if (0 == strcmp(temp[j], all[i]->cbc_name)) {
opal_output(-1, "include: saving %s", all[i]->cbc_name);
available[save++] = all[i];
++num_available;
break;
}
}
if (NULL == all[i]) {
orte_show_help("help-mpi-btl-openib-cpc-base.txt",
"cpc name not found", true,
"include", orte_process_info.nodename,
"include", cpc_include, temp[j],
all_cpc_names);
opal_argv_free(temp);
free(all_cpc_names);
return OMPI_ERR_NOT_FOUND;
}
}
opal_argv_free(temp);
}
/* Otherwise, if we have an "exclude" list, take all the CPCs that
are not in that list and put them in available[] */
else if (NULL != cpc_exclude) {
mca_btl_openib_component.cpc_explicitly_defined = true;
temp = opal_argv_split(cpc_exclude, ',');
/* First: error check -- ensure that all the names are valid */
for (j = 0; NULL != temp[j]; ++j) {
for (i = 0; NULL != all[i]; ++i) {
if (0 == strcmp(temp[j], all[i]->cbc_name)) {
break;
}
}
if (NULL == all[i]) {
orte_show_help("help-mpi-btl-openib-cpc-base.txt",
"cpc name not found", true,
"exclude", orte_process_info.nodename,
"exclude", cpc_exclude, temp[j],
all_cpc_names);
opal_argv_free(temp);
free(all_cpc_names);
return OMPI_ERR_NOT_FOUND;
}
}
/* Now do the exclude */
for (save = i = 0; NULL != all[i]; ++i) {
for (j = 0; NULL != temp[j]; ++j) {
if (0 == strcmp(temp[j], all[i]->cbc_name)) {
break;
}
}
if (NULL == temp[j]) {
opal_output(-1, "exclude: saving %s", all[i]->cbc_name);
available[save++] = all[i];
++num_available;
}
}
opal_argv_free(temp);
}
/* If there's no include/exclude list, copy all[] into available[] */
else {
opal_output(-1, "no include or exclude: saving all");
memcpy(available, all, sizeof(all));
num_available = (sizeof(all) /
sizeof(ompi_btl_openib_connect_base_module_t *)) - 1;
}
/* Call the register function on all the CPCs so that they may
setup any MCA params specific to the connection type */
for (i = 0; NULL != available[i]; ++i) {
if (NULL != available[i]->cbc_register) {
available[i]->cbc_register();
}
}
return OMPI_SUCCESS;
}
static int cray_init(void)
{
int i, spawned, size, rank, appnum, my_node;
int rc, ret = OPAL_ERROR;
char *pmapping = NULL;
char buf[PMI2_MAX_ATTRVALUE];
int found;
int major, minor, revision;
uint32_t jobfam;
opal_value_t kv;
opal_process_name_t ldr;
char nmtmp[64];
char *str, **localranks = NULL;
++pmix_init_count;
/* if we can't startup PMI, we can't be used */
if ( PMI2_Initialized () ) {
opal_output_verbose(10, opal_pmix_base_framework.framework_output,
"%s pmix:cray: pmi already initialized",
OPAL_NAME_PRINT(pmix_pname));
return OPAL_SUCCESS;
}
size = -1;
rank = -1;
appnum = -1;
if (PMI_SUCCESS != (rc = PMI2_Init(&spawned, &size, &rank, &appnum))) {
opal_show_help("help-pmix-base.txt", "pmix2-init-failed", true, rc);
return OPAL_ERROR;
}
if( size < 0 || rank < 0 ){
opal_show_help("help-pmix-base.txt", "pmix2-init-returned-bad-values", true);
goto err_exit;
}
pmix_size = size;
pmix_rank = rank;
pmix_appnum = appnum;
pmix_vallen_max = PMI2_MAX_VALLEN;
pmix_kvslen_max = PMI2_MAX_VALLEN; // FIX ME: What to put here for versatility?
pmix_keylen_max = PMI2_MAX_KEYLEN;
pmix_vallen_threshold = PMI2_MAX_VALLEN * 3;
pmix_vallen_threshold >>= 2;
/*
* get the version info
*/
if (PMI_SUCCESS != PMI_Get_version_info(&major,&minor,&revision)) {
return OPAL_ERROR;
}
snprintf(cray_pmi_version, sizeof(cray_pmi_version),
"%d.%d.%d", major, minor, revision);
pmix_kvs_name = (char*)malloc(pmix_kvslen_max);
if( pmix_kvs_name == NULL ){
PMI2_Finalize();
ret = OPAL_ERR_OUT_OF_RESOURCE;
goto err_exit;
}
rc = PMI2_Job_GetId(pmix_kvs_name, pmix_kvslen_max);
if( PMI_SUCCESS != rc ) {
OPAL_PMI_ERROR(rc, "PMI2_Job_GetId");
goto err_exit;
}
rc = sscanf(pmix_kvs_name,"kvs_%u",&jobfam);
if (rc != 1) {
opal_output_verbose(10, opal_pmix_base_framework.framework_output,
"%s pmix:cray: pmix_kvs_name %s",
OPAL_NAME_PRINT(pmix_pname), pmix_kvs_name);
rc = OPAL_ERROR;
goto err_exit;
}
pmix_jobid = jobfam << 16;
/* store our name in the opal_proc_t so that
* debug messages will make sense - an upper
* layer will eventually overwrite it, but that
* won't do any harm */
pmix_pname.jobid = pmix_jobid;
pmix_pname.vpid = pmix_rank;
opal_proc_set_name(&pmix_pname);
opal_output_verbose(10, opal_pmix_base_framework.framework_output,
"%s pmix:cray: assigned tmp name %d %d pmix_kvs_name %s",
OPAL_NAME_PRINT(pmix_pname),pmix_pname.jobid,pmix_pname.vpid,pmix_kvs_name);
pmapping = (char*)malloc(PMI2_MAX_VALLEN);
if( pmapping == NULL ){
rc = OPAL_ERR_OUT_OF_RESOURCE;
OPAL_ERROR_LOG(rc);
return rc;
}
rc = PMI2_Info_GetJobAttr("PMI_process_mapping", pmapping, PMI2_MAX_VALLEN, &found);
if( !found || PMI_SUCCESS != rc ) {
OPAL_PMI_ERROR(rc,"PMI2_Info_GetJobAttr");
return OPAL_ERROR;
}
pmix_lranks = pmix_cray_parse_pmap(pmapping, pmix_rank, &my_node, &pmix_nlranks);
if (NULL == pmix_lranks) {
rc = OPAL_ERR_OUT_OF_RESOURCE;
OPAL_ERROR_LOG(rc);
return rc;
}
free(pmapping);
// setup hash table
opal_pmix_base_hash_init();
/* save the job size */
OBJ_CONSTRUCT(&kv, opal_value_t);
kv.key = strdup(OPAL_PMIX_JOB_SIZE);
kv.type = OPAL_UINT32;
kv.data.uint32 = pmix_size;
if (OPAL_SUCCESS != (rc = opal_pmix_base_store(&OPAL_PROC_MY_NAME, &kv))) {
OPAL_ERROR_LOG(rc);
OBJ_DESTRUCT(&kv);
goto err_exit;
}
OBJ_DESTRUCT(&kv);
/* save the appnum */
OBJ_CONSTRUCT(&kv, opal_value_t);
kv.key = strdup(OPAL_PMIX_APPNUM);
kv.type = OPAL_UINT32;
kv.data.uint32 = pmix_appnum;
if (OPAL_SUCCESS != (ret = opal_pmix_base_store(&OPAL_PROC_MY_NAME, &kv))) {
OPAL_ERROR_LOG(ret);
OBJ_DESTRUCT(&kv);
goto err_exit;
}
OBJ_DESTRUCT(&kv);
rc = PMI2_Info_GetJobAttr("universeSize", buf, 16, &found);
if( PMI_SUCCESS != rc ) {
OPAL_PMI_ERROR(rc, "PMI_Get_universe_size");
goto err_exit;
}
pmix_usize = atoi(buf);
OBJ_CONSTRUCT(&kv, opal_value_t);
kv.key = strdup(OPAL_PMIX_UNIV_SIZE);
kv.type = OPAL_UINT32;
kv.data.uint32 = pmix_usize;
if (OPAL_SUCCESS != (rc = opal_pmix_base_store(&OPAL_PROC_MY_NAME, &kv))) {
OPAL_ERROR_LOG(rc);
OBJ_DESTRUCT(&kv);
goto err_exit;
}
OBJ_DESTRUCT(&kv);
OBJ_CONSTRUCT(&kv, opal_value_t);
kv.key = strdup(OPAL_PMIX_JOBID);
kv.type = OPAL_UINT32;
kv.data.uint32 = pmix_jobid;
if (OPAL_SUCCESS != (ret = opal_pmix_base_store(&OPAL_PROC_MY_NAME, &kv))) {
OPAL_ERROR_LOG(ret);
OBJ_DESTRUCT(&kv);
goto err_exit;
}
OBJ_DESTRUCT(&kv);
/* save the local size */
OBJ_CONSTRUCT(&kv, opal_value_t);
kv.key = strdup(OPAL_PMIX_LOCAL_SIZE);
kv.type = OPAL_UINT32;
kv.data.uint32 = pmix_nlranks;
if (OPAL_SUCCESS != (rc = opal_pmix_base_store(&OPAL_PROC_MY_NAME, &kv))) {
OPAL_ERROR_LOG(rc);
OBJ_DESTRUCT(&kv);
goto err_exit;
}
OBJ_DESTRUCT(&kv);
ldr.vpid = pmix_lranks[0];
ldr.jobid = pmix_pname.jobid;
/* find ourselves and build up a string for local peer info */
memset(nmtmp, 0, 64);
for (i=0; i < pmix_nlranks; i++) {
ret = snprintf(nmtmp, 64, "%d", pmix_lranks[i]);
opal_argv_append_nosize(&localranks, nmtmp);
if (pmix_rank == pmix_lranks[i]) {
pmix_lrank = i;
pmix_nrank = i;
}
}
str = opal_argv_join(localranks, ',');
opal_argv_free(localranks);
OBJ_CONSTRUCT(&kv, opal_value_t);
kv.key = strdup(OPAL_PMIX_LOCAL_PEERS);
kv.type = OPAL_STRING;
kv.data.string = str;
if (OPAL_SUCCESS != (ret = opal_pmix_base_store(&OPAL_PROC_MY_NAME, &kv))) {
OPAL_ERROR_LOG(ret);
OBJ_DESTRUCT(&kv);
goto err_exit;
}
OBJ_DESTRUCT(&kv);
/* save the local leader */
OBJ_CONSTRUCT(&kv, opal_value_t);
kv.key = strdup(OPAL_PMIX_LOCALLDR);
kv.type = OPAL_UINT64;
kv.data.uint64 = *(uint64_t*)&ldr;
if (OPAL_SUCCESS != (ret = opal_pmix_base_store(&OPAL_PROC_MY_NAME, &kv))) {
OPAL_ERROR_LOG(ret);
OBJ_DESTRUCT(&kv);
goto err_exit;
}
/* save our local rank */
OBJ_CONSTRUCT(&kv, opal_value_t);
kv.key = strdup(OPAL_PMIX_LOCAL_RANK);
kv.type = OPAL_UINT16;
kv.data.uint16 = pmix_lrank;
if (OPAL_SUCCESS != (ret = opal_pmix_base_store(&OPAL_PROC_MY_NAME, &kv))) {
OPAL_ERROR_LOG(ret);
OBJ_DESTRUCT(&kv);
goto err_exit;
}
/* and our node rank */
OBJ_CONSTRUCT(&kv, opal_value_t);
kv.key = strdup(OPAL_PMIX_NODE_RANK);
kv.type = OPAL_UINT16;
kv.data.uint16 = pmix_nrank;
if (OPAL_SUCCESS != (ret = opal_pmix_base_store(&OPAL_PROC_MY_NAME, &kv))) {
OPAL_ERROR_LOG(ret);
OBJ_DESTRUCT(&kv);
goto err_exit;
}
OBJ_DESTRUCT(&kv);
return OPAL_SUCCESS;
err_exit:
PMI2_Finalize();
return ret;
}
static void recv_data(int fd, short args, void *cbdata)
{
bool found;
int i, rc;
orte_node_t *nd, *nd2;
opal_list_t nds, ndtmp;
opal_list_item_t *item, *itm;
char recv_msg[8192];
int nbytes, idx, sjob;
char **alloc, *nodelist, *tpn;
local_jobtracker_t *ptr, *jtrk;
local_apptracker_t *aptrk;
orte_app_context_t *app;
orte_jobid_t jobid;
orte_job_t *jdata;
char **dash_host = NULL;
opal_output_verbose(2, orte_ras_base_framework.framework_output,
"%s ras:slurm: dynamic allocation - data recvd",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
/* read the data from the socket and put it in the
* nodes field of op
*/
memset(recv_msg, 0, sizeof(recv_msg));
nbytes = read(fd, recv_msg, sizeof(recv_msg) - 1);
opal_output_verbose(2, orte_ras_base_framework.framework_output,
"%s ras:slurm: dynamic allocation msg: %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), recv_msg);
/* check if we got something */
if (0 == nbytes || 0 == strlen(recv_msg) || strstr(recv_msg, "failure") != NULL) {
/* show an error here - basically, a "nothing was available"
* message
*/
orte_show_help("help-ras-slurm.txt", "slurm-dyn-alloc-failed", true,
(0 == strlen(recv_msg)) ? "NO MSG" : recv_msg);
ORTE_ACTIVATE_JOB_STATE(NULL, ORTE_JOB_STATE_ALLOC_FAILED);
return;
}
/* break the message into its component parts, separated by colons */
alloc = opal_argv_split(recv_msg, ':');
/* the first section contains the ORTE jobid for this allocation */
tpn = strchr(alloc[0], '=');
orte_util_convert_string_to_jobid(&jobid, tpn+1);
/* get the corresponding job object */
jdata = orte_get_job_data_object(jobid);
jtrk = NULL;
/* find the associated tracking object */
for (item = opal_list_get_first(&jobs);
item != opal_list_get_end(&jobs);
item = opal_list_get_next(item)) {
ptr = (local_jobtracker_t*)item;
if (ptr->jobid == jobid) {
jtrk = ptr;
break;
}
}
if (NULL == jtrk) {
orte_show_help("help-ras-slurm.txt", "slurm-dyn-alloc-failed", true, "NO JOB TRACKER");
ORTE_ACTIVATE_JOB_STATE(NULL, ORTE_JOB_STATE_ALLOC_FAILED);
opal_argv_free(alloc);
return;
}
/* stop the timeout event */
opal_event_del(&jtrk->timeout_ev);
/* cycle across all the remaining parts - each is the allocation for
* an app in this job
*/
OBJ_CONSTRUCT(&nds, opal_list_t);
OBJ_CONSTRUCT(&ndtmp, opal_list_t);
idx = -1;
sjob = -1;
nodelist = NULL;
tpn = NULL;
for (i=1; NULL != alloc[i]; i++) {
if (ORTE_SUCCESS != parse_alloc_msg(alloc[i], &idx, &sjob, &nodelist, &tpn)) {
orte_show_help("help-ras-slurm.txt", "slurm-dyn-alloc-failed", true, jtrk->cmd);
ORTE_ACTIVATE_JOB_STATE(jdata, ORTE_JOB_STATE_ALLOC_FAILED);
opal_argv_free(alloc);
if (NULL != nodelist) {
free(nodelist);
}
if (NULL != tpn) {
free(tpn);
}
return;
}
if (idx < 0) {
orte_show_help("help-ras-slurm.txt", "slurm-dyn-alloc-failed", true, jtrk->cmd);
ORTE_ACTIVATE_JOB_STATE(jdata, ORTE_JOB_STATE_ALLOC_FAILED);
opal_argv_free(alloc);
free(nodelist);
free(tpn);
return;
}
if (NULL == (app = (orte_app_context_t*)opal_pointer_array_get_item(jdata->apps, idx))) {
orte_show_help("help-ras-slurm.txt", "slurm-dyn-alloc-failed", true, jtrk->cmd);
ORTE_ACTIVATE_JOB_STATE(jdata, ORTE_JOB_STATE_ALLOC_FAILED);
opal_argv_free(alloc);
free(nodelist);
free(tpn);
return;
}
/* release the current dash_host as that contained the *desired* allocation */
orte_remove_attribute(&app->attributes, ORTE_APP_DASH_HOST);
/* track the Slurm jobid */
if (NULL == (aptrk = (local_apptracker_t*)opal_pointer_array_get_item(&jtrk->apps, idx))) {
aptrk = OBJ_NEW(local_apptracker_t);
opal_pointer_array_set_item(&jtrk->apps, idx, aptrk);
}
aptrk->sjob = sjob;
/* since the nodelist/tpn may contain regular expressions, parse them */
if (ORTE_SUCCESS != (rc = orte_ras_slurm_discover(nodelist, tpn, &ndtmp))) {
ORTE_ERROR_LOG(rc);
ORTE_ACTIVATE_JOB_STATE(jdata, ORTE_JOB_STATE_ALLOC_FAILED);
opal_argv_free(alloc);
free(nodelist);
free(tpn);
return;
}
/* transfer the discovered nodes to our node list, and construct
* the new dash_host entry to match what was allocated
*/
while (NULL != (item = opal_list_remove_first(&ndtmp))) {
nd = (orte_node_t*)item;
opal_argv_append_nosize(&dash_host, nd->name);
/* check for duplicates */
found = false;
for (itm = opal_list_get_first(&nds);
itm != opal_list_get_end(&nds);
itm = opal_list_get_next(itm)) {
nd2 = (orte_node_t*)itm;
if (0 == strcmp(nd->name, nd2->name)) {
found = true;
nd2->slots += nd->slots;
OBJ_RELEASE(item);
break;
}
}
if (!found) {
/* append the new node to our list */
opal_list_append(&nds, item);
}
}
/* cleanup */
free(nodelist);
free(tpn);
}
/* cleanup */
opal_argv_free(alloc);
OBJ_DESTRUCT(&ndtmp);
if (NULL != dash_host) {
tpn = opal_argv_join(dash_host, ',');
for (idx=0; idx < jdata->apps->size; idx++) {
if (NULL == (app = (orte_app_context_t*)opal_pointer_array_get_item(jdata->apps, idx))) {
orte_show_help("help-ras-slurm.txt", "slurm-dyn-alloc-failed", true, jtrk->cmd);
ORTE_ACTIVATE_JOB_STATE(jdata, ORTE_JOB_STATE_ALLOC_FAILED);
opal_argv_free(dash_host);
free(tpn);
return;
}
orte_set_attribute(&app->attributes, ORTE_APP_DASH_HOST, ORTE_ATTR_LOCAL, (void*)tpn, OPAL_STRING);
}
opal_argv_free(dash_host);
free(tpn);
}
if (opal_list_is_empty(&nds)) {
/* if we get here, then we were able to contact slurm,
* which means we are in an actively managed cluster.
* However, slurm indicated that nothing is currently
* available that meets our requirements. This is a fatal
* situation - we do NOT have the option of running on
* user-specified hosts as the cluster is managed.
*/
OBJ_DESTRUCT(&nds);
orte_show_help("help-ras-base.txt", "ras-base:no-allocation", true);
ORTE_FORCED_TERMINATE(ORTE_ERROR_DEFAULT_EXIT_CODE);
}
/* store the found nodes */
if (ORTE_SUCCESS != (rc = orte_ras_base_node_insert(&nds, jdata))) {
ORTE_ERROR_LOG(rc);
OBJ_DESTRUCT(&nds);
ORTE_FORCED_TERMINATE(ORTE_ERROR_DEFAULT_EXIT_CODE);
return;
}
OBJ_DESTRUCT(&nds);
/* default to no-oversubscribe-allowed for managed systems */
if (!(ORTE_MAPPING_SUBSCRIBE_GIVEN & ORTE_GET_MAPPING_DIRECTIVE(orte_rmaps_base.mapping))) {
ORTE_SET_MAPPING_DIRECTIVE(orte_rmaps_base.mapping, ORTE_MAPPING_NO_OVERSUBSCRIBE);
}
/* flag that the allocation is managed */
orte_managed_allocation = true;
/* move the job along */
ORTE_ACTIVATE_JOB_STATE(jdata, ORTE_JOB_STATE_ALLOCATION_COMPLETE);
/* all done */
return;
}
opal_argv_append_nosize(®exargs, tmp); free(tmp); /* the stdin target for the job */ asprintf(&tmp, "STDIN=%d", (int)jdata->stdin_target); opal_argv_append_nosize(®exargs, tmp); free(tmp); /* the app_context for the job - can only be one! Just include * the required portions */ app = (orte_app_context_t*)opal_pointer_array_get_item(jdata->apps, 0); asprintf(&tmp, "APP=\"%s:%s\"", app->app, app->cwd); opal_argv_append_nosize(®exargs, tmp); free(tmp); tmp2 = opal_argv_join(app->argv, '#'); asprintf(&tmp, "ARGV=\"%s\"", (NULL == tmp2) ? "NULL" : tmp2); free(tmp2); opal_argv_append_nosize(®exargs, tmp); free(tmp); tmp2 = opal_argv_join(app->env, '#'); asprintf(&tmp, "ENV=\"%s\"", (NULL == tmp2) ? "NULL" : tmp2); free(tmp2); opal_argv_append_nosize(®exargs, tmp); free(tmp); /* next comes the starting daemon vpid */ asprintf(&tmp, "DVPID=%s", ORTE_VPID_PRINT(jdata->map->daemon_vpid_start)); opal_argv_append_nosize(®exargs, tmp); free(tmp);
/* we cannot use the RML to communicate with SLURM as it doesn't
* understand our internal protocol, so we have to do a bare-bones
* exchange based on sockets
*/
static int dyn_allocate(orte_job_t *jdata)
{
char *cmd_str, **cmd=NULL, *tmp, *jstring;
char *node_list;
orte_app_context_t *app;
int i;
struct timeval tv;
local_jobtracker_t *jtrk;
int64_t i64, *i64ptr;
if (NULL == mca_ras_slurm_component.config_file) {
opal_output(0, "Cannot perform dynamic allocation as no Slurm configuration file provided");
return ORTE_ERR_NOT_FOUND;
}
/* track this request */
jtrk = OBJ_NEW(local_jobtracker_t);
jtrk->jobid = jdata->jobid;
opal_list_append(&jobs, &jtrk->super);
/* construct the command - note that the jdata structure contains
* a field for the minimum number of nodes required for the job.
* The node list can be constructed from the union of all the nodes
* contained in the dash_host field of the app_contexts. So you'll
* need to do a little work to build the command. We don't currently
* have a field in the jdata structure for "mandatory" vs "optional"
* allocations, so we'll have to add that someday. Likewise, you may
* want to provide a param to adjust the timeout value
*/
/* construct the cmd string */
opal_argv_append_nosize(&cmd, "allocate");
/* add the jobid */
orte_util_convert_jobid_to_string(&jstring, jdata->jobid);
opal_asprintf(&tmp, "jobid=%s", jstring);
opal_argv_append_nosize(&cmd, tmp);
free(tmp);
free(jstring);
/* if we want the allocation for all apps in one shot,
* then tell slurm
*
* RHC: we don't currently have the ability to handle
* rolling allocations in the rest of the code base
*/
#if 0
if (!mca_ras_slurm_component.rolling_alloc) {
opal_argv_append_nosize(&cmd, "return=all");
}
#else
opal_argv_append_nosize(&cmd, "return=all");
#endif
/* pass the timeout */
opal_asprintf(&tmp, "timeout=%d", mca_ras_slurm_component.timeout);
opal_argv_append_nosize(&cmd, tmp);
free(tmp);
/* for each app, add its allocation request info */
i64ptr = &i64;
for (i=0; i < jdata->apps->size; i++) {
if (NULL == (app = (orte_app_context_t*)opal_pointer_array_get_item(jdata->apps, i))) {
continue;
}
/* add the app id, preceded by a colon separator */
opal_asprintf(&tmp, ": app=%d", (int)app->idx);
opal_argv_append_nosize(&cmd, tmp);
free(tmp);
/* add the number of process "slots" we need */
opal_asprintf(&tmp, "np=%d", app->num_procs);
opal_argv_append_nosize(&cmd, tmp);
free(tmp);
/* if we were given a minimum number of nodes, pass it along */
if (orte_get_attribute(&app->attributes, ORTE_APP_MIN_NODES, (void**)&i64ptr, OPAL_INT64)) {
opal_asprintf(&tmp, "N=%ld", (long int)i64);
opal_argv_append_nosize(&cmd, tmp);
free(tmp);
}
/* add the list of nodes, if one was given, ensuring
* that each node only appears once
*/
node_list = get_node_list(app);
if (NULL != node_list) {
opal_asprintf(&tmp, "node_list=%s", node_list);
opal_argv_append_nosize(&cmd, tmp);
free(node_list);
free(tmp);
}
/* add the mandatory/optional flag */
if (orte_get_attribute(&app->attributes, ORTE_APP_MANDATORY, NULL, OPAL_BOOL)) {
opal_argv_append_nosize(&cmd, "flag=mandatory");
} else {
opal_argv_append_nosize(&cmd, "flag=optional");
}
}
/* assemble it into the final cmd to be sent */
cmd_str = opal_argv_join(cmd, ' ');
opal_argv_free(cmd);
/* start a timer - if the response to our request doesn't appear
* in the defined time, then we will error out as Slurm isn't
* responding to us
*/
opal_event_evtimer_set(orte_event_base, &jtrk->timeout_ev, timeout, jtrk);
tv.tv_sec = mca_ras_slurm_component.timeout * 2;
tv.tv_usec = 0;
opal_event_evtimer_add(&jtrk->timeout_ev, &tv);
opal_output_verbose(2, orte_ras_base_framework.framework_output,
"%s slurm:dynalloc cmd_str = %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
cmd_str);
if (send(socket_fd, cmd_str, strlen(cmd_str)+1, 0) < 0) {
ORTE_ERROR_LOG(ORTE_ERR_COMM_FAILURE);
}
free(cmd_str);
/* we cannot wait here for a response as we
* are already in an event. So return a value
* that indicates we are waiting for an
* allocation so the base functions know
* that they shouldn't progress the job
*/
return ORTE_ERR_ALLOCATION_PENDING;
}
static int setup_fork(orte_job_t *jdata,
orte_app_context_t *app)
{
int i;
char *param;
bool oversubscribed;
orte_node_t *node;
char **envcpy, **nps, **firstranks;
char *npstring, *firstrankstring;
char *num_app_ctx;
bool takeus = false;
orte_app_context_t* tmp_app;
opal_output_verbose(1, orte_schizo_base_framework.framework_output,
"%s schizo:ompi: setup_fork",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
if (NULL != orte_schizo_base.personalities) {
/* see if we are included */
for (i=0; NULL != jdata->personality[i]; i++) {
if (0 == strcmp(jdata->personality[i], "ompi")) {
takeus = true;
break;
}
}
if (!takeus) {
return ORTE_ERR_TAKE_NEXT_OPTION;
}
}
/* see if the mapper thinks we are oversubscribed */
oversubscribed = false;
if (NULL == (node = (orte_node_t*)opal_pointer_array_get_item(orte_node_pool, ORTE_PROC_MY_NAME->vpid))) {
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
return ORTE_ERR_NOT_FOUND;
}
if (ORTE_FLAG_TEST(node, ORTE_NODE_FLAG_OVERSUBSCRIBED)) {
oversubscribed = true;
}
/* setup base environment: copy the current environ and merge
in the app context environ */
if (NULL != app->env) {
/* manually free original context->env to avoid a memory leak */
char **tmp = app->env;
envcpy = opal_environ_merge(orte_launch_environ, app->env);
if (NULL != tmp) {
opal_argv_free(tmp);
}
} else {
envcpy = opal_argv_copy(orte_launch_environ);
}
app->env = envcpy;
/* special case handling for --prefix: this is somewhat icky,
but at least some users do this. :-\ It is possible that
when using --prefix, the user will also "-x PATH" and/or
"-x LD_LIBRARY_PATH", which would therefore clobber the
work that was done in the prior pls to ensure that we have
the prefix at the beginning of the PATH and
LD_LIBRARY_PATH. So examine the context->env and see if we
find PATH or LD_LIBRARY_PATH. If found, that means the
prior work was clobbered, and we need to re-prefix those
variables. */
param = NULL;
orte_get_attribute(&app->attributes, ORTE_APP_PREFIX_DIR, (void**)¶m, OPAL_STRING);
for (i = 0; NULL != param && NULL != app->env && NULL != app->env[i]; ++i) {
char *newenv;
/* Reset PATH */
if (0 == strncmp("PATH=", app->env[i], 5)) {
asprintf(&newenv, "%s/bin:%s", param, app->env[i] + 5);
opal_setenv("PATH", newenv, true, &app->env);
free(newenv);
}
/* Reset LD_LIBRARY_PATH */
else if (0 == strncmp("LD_LIBRARY_PATH=", app->env[i], 16)) {
asprintf(&newenv, "%s/lib:%s", param, app->env[i] + 16);
opal_setenv("LD_LIBRARY_PATH", newenv, true, &app->env);
free(newenv);
}
}
if (NULL != param) {
free(param);
}
/* pass my contact info to the local proc so we can talk */
opal_setenv("OMPI_MCA_orte_local_daemon_uri", orte_process_info.my_daemon_uri, true, &app->env);
/* pass the hnp's contact info to the local proc in case it
* needs it
*/
if (NULL != orte_process_info.my_hnp_uri) {
opal_setenv("OMPI_MCA_orte_hnp_uri", orte_process_info.my_hnp_uri, true, &app->env);
}
/* setup yield schedule - do not override any user-supplied directive! */
if (oversubscribed) {
opal_setenv("OMPI_MCA_mpi_yield_when_idle", "1", false, &app->env);
} else {
opal_setenv("OMPI_MCA_mpi_yield_when_idle", "0", false, &app->env);
}
/* set the app_context number into the environment */
asprintf(¶m, "%ld", (long)app->idx);
opal_setenv("OMPI_MCA_orte_app_num", param, true, &app->env);
free(param);
/* although the total_slots_alloc is the universe size, users
* would appreciate being given a public environmental variable
* that also represents this value - something MPI specific - so
* do that here. Also required by the ompi_attributes code!
*
* AND YES - THIS BREAKS THE ABSTRACTION BARRIER TO SOME EXTENT.
* We know - just live with it
*/
asprintf(¶m, "%ld", (long)jdata->total_slots_alloc);
opal_setenv("OMPI_UNIVERSE_SIZE", param, true, &app->env);
free(param);
/* pass the number of nodes involved in this job */
asprintf(¶m, "%ld", (long)(jdata->map->num_nodes));
opal_setenv("OMPI_MCA_orte_num_nodes", param, true, &app->env);
free(param);
/* pass a param telling the child what type and model of cpu we are on,
* if we know it. If hwloc has the value, use what it knows. Otherwise,
* see if we were explicitly given it and use that value.
*/
hwloc_obj_t obj;
char *htmp;
if (NULL != opal_hwloc_topology) {
obj = hwloc_get_root_obj(opal_hwloc_topology);
if (NULL != (htmp = (char*)hwloc_obj_get_info_by_name(obj, "CPUType")) ||
NULL != (htmp = orte_local_cpu_type)) {
opal_setenv("OMPI_MCA_orte_cpu_type", htmp, true, &app->env);
}
if (NULL != (htmp = (char*)hwloc_obj_get_info_by_name(obj, "CPUModel")) ||
NULL != (htmp = orte_local_cpu_model)) {
opal_setenv("OMPI_MCA_orte_cpu_model", htmp, true, &app->env);
}
} else {
if (NULL != orte_local_cpu_type) {
opal_setenv("OMPI_MCA_orte_cpu_type", orte_local_cpu_type, true, &app->env);
}
if (NULL != orte_local_cpu_model) {
opal_setenv("OMPI_MCA_orte_cpu_model", orte_local_cpu_model, true, &app->env);
}
}
/* get shmem's best component name so we can provide a hint to the shmem
* framework. the idea here is to have someone figure out what component to
* select (via the shmem framework) and then have the rest of the
* components in shmem obey that decision. for more details take a look at
* the shmem framework in opal.
*/
if (NULL != (param = opal_shmem_base_best_runnable_component_name())) {
opal_setenv("OMPI_MCA_shmem_RUNTIME_QUERY_hint", param, true, &app->env);
free(param);
}
/* Set an info MCA param that tells the launched processes that
* any binding policy was applied by us (e.g., so that
* MPI_INIT doesn't try to bind itself)
*/
opal_setenv("OMPI_MCA_orte_bound_at_launch", "1", true, &app->env);
/* tell the ESS to avoid the singleton component - but don't override
* anything that may have been provided elsewhere
*/
opal_setenv("OMPI_MCA_ess", "^singleton", false, &app->env);
/* ensure that the spawned process ignores direct launch components,
* but do not overrride anything we were given */
opal_setenv("OMPI_MCA_pmix", "^s1,s2,cray", false, &app->env);
/* since we want to pass the name as separate components, make sure
* that the "name" environmental variable is cleared!
*/
opal_unsetenv("OMPI_MCA_orte_ess_name", &app->env);
asprintf(¶m, "%ld", (long)jdata->num_procs);
opal_setenv("OMPI_MCA_orte_ess_num_procs", param, true, &app->env);
/* although the num_procs is the comm_world size, users
* would appreciate being given a public environmental variable
* that also represents this value - something MPI specific - so
* do that here.
*
* AND YES - THIS BREAKS THE ABSTRACTION BARRIER TO SOME EXTENT.
* We know - just live with it
*/
opal_setenv("OMPI_COMM_WORLD_SIZE", param, true, &app->env);
free(param);
/* users would appreciate being given a public environmental variable
* that also represents this value - something MPI specific - so
* do that here.
*
* AND YES - THIS BREAKS THE ABSTRACTION BARRIER TO SOME EXTENT.
* We know - just live with it
*/
asprintf(¶m, "%ld", (long)jdata->num_local_procs);
opal_setenv("OMPI_COMM_WORLD_LOCAL_SIZE", param, true, &app->env);
free(param);
/* forcibly set the local tmpdir base and top session dir to match ours */
opal_setenv("OMPI_MCA_orte_tmpdir_base", orte_process_info.tmpdir_base, true, &app->env);
opal_setenv("OMPI_MCA_orte_top_session_dir", orte_process_info.top_session_dir, true, &app->env);
/* MPI-3 requires we provide some further info to the procs,
* so we pass them as envars to avoid introducing further
* ORTE calls in the MPI layer
*/
asprintf(&num_app_ctx, "%lu", (unsigned long)jdata->num_apps);
/* build some common envars we need to pass for MPI-3 compatibility */
nps = NULL;
firstranks = NULL;
for (i=0; i < jdata->apps->size; i++) {
if (NULL == (tmp_app = (orte_app_context_t*)opal_pointer_array_get_item(jdata->apps, i))) {
continue;
}
opal_argv_append_nosize(&nps, ORTE_VPID_PRINT(tmp_app->num_procs));
opal_argv_append_nosize(&firstranks, ORTE_VPID_PRINT(tmp_app->first_rank));
}
npstring = opal_argv_join(nps, ' ');
firstrankstring = opal_argv_join(firstranks, ' ');
opal_argv_free(nps);
opal_argv_free(firstranks);
/* add the MPI-3 envars */
opal_setenv("OMPI_NUM_APP_CTX", num_app_ctx, true, &app->env);
opal_setenv("OMPI_FIRST_RANKS", firstrankstring, true, &app->env);
opal_setenv("OMPI_APP_CTX_NUM_PROCS", npstring, true, &app->env);
free(num_app_ctx);
free(firstrankstring);
free(npstring);
return ORTE_SUCCESS;
}
static int parse_args(int argc, char *argv[])
{
int i, ret, len;
opal_cmd_line_t cmd_line;
char **app_env = NULL, **global_env = NULL;
opal_restart_globals.help = false;
opal_restart_globals.verbose = false;
opal_restart_globals.snapshot_ref = NULL;
opal_restart_globals.snapshot_loc = NULL;
opal_restart_globals.snapshot_metadata = NULL;
opal_restart_globals.snapshot_cache = NULL;
opal_restart_globals.snapshot_compress = NULL;
opal_restart_globals.snapshot_compress_postfix = NULL;
opal_restart_globals.output = 0;
/* Parse the command line options */
opal_cmd_line_create(&cmd_line, cmd_line_opts);
mca_base_open();
mca_base_cmd_line_setup(&cmd_line);
ret = opal_cmd_line_parse(&cmd_line, false, false, argc, argv);
if (OPAL_SUCCESS != ret) {
if (OPAL_ERR_SILENT != ret) {
fprintf(stderr, "%s: command line error (%s)\n", argv[0],
opal_strerror(ret));
}
return 1;
}
if (opal_restart_globals.help ) {
char *str, *args = NULL;
args = opal_cmd_line_get_usage_msg(&cmd_line);
str = opal_show_help_string("help-opal-restart.txt", "usage", true,
args);
if (NULL != str) {
printf("%s", str);
free(str);
}
free(args);
/* If we show the help message, that should be all we do */
exit(0);
}
/**
* Put all of the MCA arguments in the environment
*/
mca_base_cmd_line_process_args(&cmd_line, &app_env, &global_env);
len = opal_argv_count(app_env);
for(i = 0; i < len; ++i) {
putenv(app_env[i]);
}
len = opal_argv_count(global_env);
for(i = 0; i < len; ++i) {
putenv(global_env[i]);
}
/**
* Now start parsing our specific arguments
*/
/* get the remaining bits */
opal_cmd_line_get_tail(&cmd_line, &argc, &argv);
if ( NULL == opal_restart_globals.snapshot_ref ||
0 >= strlen(opal_restart_globals.snapshot_ref) ) {
opal_show_help("help-opal-restart.txt", "invalid_filename", true,
"<none provided>");
return OPAL_ERROR;
}
/* If we have arguments after the command, then assume they
* need to be grouped together.
* Useful in the 'mca crs self' instance.
*/
if(argc > 0) {
opal_restart_globals.snapshot_ref = strdup(opal_argv_join(argv, ' '));
}
return OPAL_SUCCESS;
}
int main(int argc, char *argv[])
{
int rc, i, j;
opal_cmd_line_t cmd_line;
char *param, *value;
orte_job_t *jdata=NULL;
orte_app_context_t *app;
char *uri, *ptr;
/* Setup and parse the command line */
memset(&myglobals, 0, sizeof(myglobals));
/* find our basename (the name of the executable) so that we can
use it in pretty-print error messages */
myglobals.basename = opal_basename(argv[0]);
opal_cmd_line_create(&cmd_line, cmd_line_init);
mca_base_cmd_line_setup(&cmd_line);
if (OPAL_SUCCESS != (rc = opal_cmd_line_parse(&cmd_line, true,
argc, argv)) ) {
if (OPAL_ERR_SILENT != rc) {
fprintf(stderr, "%s: command line error (%s)\n", argv[0],
opal_strerror(rc));
}
return rc;
}
/* print version if requested. Do this before check for help so
that --version --help works as one might expect. */
if (myglobals.version) {
char *str;
str = opal_info_make_version_str("all",
OPAL_MAJOR_VERSION, OPAL_MINOR_VERSION,
OPAL_RELEASE_VERSION,
OPAL_GREEK_VERSION,
OPAL_REPO_REV);
if (NULL != str) {
fprintf(stdout, "%s %s\n\nReport bugs to %s\n",
myglobals.basename, str, PACKAGE_BUGREPORT);
free(str);
}
exit(0);
}
/* check if we are running as root - if we are, then only allow
* us to proceed if the allow-run-as-root flag was given. Otherwise,
* exit with a giant warning flag
*/
if (0 == geteuid() && !myglobals.run_as_root) {
fprintf(stderr, "--------------------------------------------------------------------------\n");
if (myglobals.help) {
fprintf(stderr, "%s cannot provide the help message when run as root\n", myglobals.basename);
} else {
/* show_help is not yet available, so print an error manually */
fprintf(stderr, "%s has detected an attempt to run as root.\n", myglobals.basename);
}
fprintf(stderr, " This is *strongly* discouraged as any mistake (e.g., in defining TMPDIR) or bug can\n");
fprintf(stderr, "result in catastrophic damage to the OS file system, leaving\n");
fprintf(stderr, "your system in an unusable state.\n\n");
fprintf(stderr, "You can override this protection by adding the --allow-run-as-root\n");
fprintf(stderr, "option to your cmd line. However, we reiterate our strong advice\n");
fprintf(stderr, "against doing so - please do so at your own risk.\n");
fprintf(stderr, "--------------------------------------------------------------------------\n");
exit(1);
}
/*
* Since this process can now handle MCA/GMCA parameters, make sure to
* process them.
* NOTE: It is "safe" to call mca_base_cmd_line_process_args() before
* opal_init_util() since mca_base_cmd_line_process_args() does *not*
* depend upon opal_init_util() functionality.
*/
if (OPAL_SUCCESS != mca_base_cmd_line_process_args(&cmd_line, &environ, &environ)) {
exit(1);
}
/* Need to initialize OPAL so that install_dirs are filled in */
if (OPAL_SUCCESS != opal_init(&argc, &argv)) {
exit(1);
}
/* Check for help request */
if (myglobals.help) {
char *str, *args = NULL;
char *project_name = NULL;
if (0 == strcmp(myglobals.basename, "mpirun")) {
project_name = "Open MPI";
} else {
project_name = "OpenRTE";
}
args = opal_cmd_line_get_usage_msg(&cmd_line);
str = opal_show_help_string("help-orterun.txt", "orterun:usage", false,
myglobals.basename, project_name, OPAL_VERSION,
myglobals.basename, args,
PACKAGE_BUGREPORT);
if (NULL != str) {
printf("%s", str);
free(str);
}
free(args);
/* If someone asks for help, that should be all we do */
exit(0);
}
/* flag that I am the HNP */
orte_process_info.proc_type = ORTE_PROC_HNP;
/* Setup MCA params */
orte_register_params();
/* specify the DVM state machine */
opal_setenv("OMPI_MCA_state", "dvm", true, &environ);
/* Intialize our Open RTE environment */
if (ORTE_SUCCESS != (rc = orte_init(&argc, &argv, ORTE_PROC_HNP))) {
/* cannot call ORTE_ERROR_LOG as it could be the errmgr
* never got loaded!
*/
return rc;
}
/* finalize OPAL. As it was opened again from orte_init->opal_init
* we continue to have a reference count on it. So we have to finalize it twice...
*/
opal_finalize();
/* check for request to report uri */
uri = orte_rml.get_contact_info();
if (NULL != myglobals.report_uri) {
FILE *fp;
if (0 == strcmp(myglobals.report_uri, "-")) {
/* if '-', then output to stdout */
printf("VMURI: %s\n", uri);
} else if (0 == strcmp(myglobals.report_uri, "+")) {
/* if '+', output to stderr */
fprintf(stderr, "VMURI: %s\n", uri);
} else if (0 == strncasecmp(myglobals.report_uri, "file:", strlen("file:"))) {
ptr = strchr(myglobals.report_uri, ':');
++ptr;
fp = fopen(ptr, "w");
if (NULL == fp) {
orte_show_help("help-orterun.txt", "orterun:write_file", false,
myglobals.basename, "pid", ptr);
exit(0);
}
fprintf(fp, "%s\n", uri);
fclose(fp);
} else {
fp = fopen(myglobals.report_uri, "w");
if (NULL == fp) {
orte_show_help("help-orterun.txt", "orterun:write_file", false,
myglobals.basename, "pid", myglobals.report_uri);
exit(0);
}
fprintf(fp, "%s\n", uri);
fclose(fp);
}
free(uri);
} else {
printf("VMURI: %s\n", uri);
}
/* get the daemon job object - was created by ess/hnp component */
if (NULL == (jdata = orte_get_job_data_object(ORTE_PROC_MY_NAME->jobid))) {
orte_show_help("help-orterun.txt", "bad-job-object", true,
myglobals.basename);
exit(0);
}
/* also should have created a daemon "app" */
if (NULL == (app = (orte_app_context_t*)opal_pointer_array_get_item(jdata->apps, 0))) {
orte_show_help("help-orterun.txt", "bad-app-object", true,
myglobals.basename);
exit(0);
}
/* Did the user specify a prefix, or want prefix by default? */
if (opal_cmd_line_is_taken(&cmd_line, "prefix") || want_prefix_by_default) {
size_t param_len;
/* if both the prefix was given and we have a prefix
* given above, check to see if they match
*/
if (opal_cmd_line_is_taken(&cmd_line, "prefix") &&
NULL != myglobals.prefix) {
/* if they don't match, then that merits a warning */
param = strdup(opal_cmd_line_get_param(&cmd_line, "prefix", 0, 0));
/* ensure we strip any trailing '/' */
if (0 == strcmp(OPAL_PATH_SEP, &(param[strlen(param)-1]))) {
param[strlen(param)-1] = '\0';
}
value = strdup(myglobals.prefix);
if (0 == strcmp(OPAL_PATH_SEP, &(value[strlen(value)-1]))) {
value[strlen(value)-1] = '\0';
}
if (0 != strcmp(param, value)) {
orte_show_help("help-orterun.txt", "orterun:app-prefix-conflict",
true, myglobals.basename, value, param);
/* let the global-level prefix take precedence since we
* know that one is being used
*/
free(param);
param = strdup(myglobals.prefix);
}
free(value);
} else if (NULL != myglobals.prefix) {
param = myglobals.prefix;
} else if (opal_cmd_line_is_taken(&cmd_line, "prefix")){
/* must be --prefix alone */
param = strdup(opal_cmd_line_get_param(&cmd_line, "prefix", 0, 0));
} else {
/* --enable-orterun-prefix-default was given to orterun */
param = strdup(opal_install_dirs.prefix);
}
if (NULL != param) {
/* "Parse" the param, aka remove superfluous path_sep. */
param_len = strlen(param);
while (0 == strcmp (OPAL_PATH_SEP, &(param[param_len-1]))) {
param[param_len-1] = '\0';
param_len--;
if (0 == param_len) {
orte_show_help("help-orterun.txt", "orterun:empty-prefix",
true, myglobals.basename, myglobals.basename);
return ORTE_ERR_FATAL;
}
}
orte_set_attribute(&app->attributes, ORTE_APP_PREFIX_DIR, ORTE_ATTR_GLOBAL, param, OPAL_STRING);
free(param);
}
}
/* Did the user specify a hostfile. Need to check for both
* hostfile and machine file.
* We can only deal with one hostfile per app context, otherwise give an error.
*/
if (0 < (j = opal_cmd_line_get_ninsts(&cmd_line, "hostfile"))) {
if(1 < j) {
orte_show_help("help-orterun.txt", "orterun:multiple-hostfiles",
true, myglobals.basename, NULL);
return ORTE_ERR_FATAL;
} else {
value = opal_cmd_line_get_param(&cmd_line, "hostfile", 0, 0);
orte_set_attribute(&app->attributes, ORTE_APP_HOSTFILE, ORTE_ATTR_LOCAL, value, OPAL_STRING);
}
}
if (0 < (j = opal_cmd_line_get_ninsts(&cmd_line, "machinefile"))) {
if(1 < j || orte_get_attribute(&app->attributes, ORTE_APP_HOSTFILE, NULL, OPAL_STRING)) {
orte_show_help("help-orterun.txt", "orterun:multiple-hostfiles",
true, myglobals.basename, NULL);
return ORTE_ERR_FATAL;
} else {
value = opal_cmd_line_get_param(&cmd_line, "machinefile", 0, 0);
orte_set_attribute(&app->attributes, ORTE_APP_HOSTFILE, ORTE_ATTR_LOCAL, value, OPAL_STRING);
}
}
/* Did the user specify any hosts? */
if (0 < (j = opal_cmd_line_get_ninsts(&cmd_line, "host"))) {
char **targ=NULL, *tval;
for (i = 0; i < j; ++i) {
value = opal_cmd_line_get_param(&cmd_line, "host", i, 0);
opal_argv_append_nosize(&targ, value);
}
tval = opal_argv_join(targ, ',');
orte_set_attribute(&app->attributes, ORTE_APP_DASH_HOST, ORTE_ATTR_LOCAL, tval, OPAL_STRING);
opal_argv_free(targ);
free(tval);
}
OBJ_DESTRUCT(&cmd_line);
/* setup to listen for commands sent specifically to me, even though I would probably
* be the one sending them! Unfortunately, since I am a participating daemon,
* there are times I need to send a command to "all daemons", and that means *I* have
* to receive it too
*/
orte_rml.recv_buffer_nb(ORTE_NAME_WILDCARD, ORTE_RML_TAG_DAEMON,
ORTE_RML_PERSISTENT, orte_daemon_recv, NULL);
/* override the notify_completed state so we can send a message
* back to anyone who submits a job to us telling them the job
* completed */
if (ORTE_SUCCESS != (rc = orte_state.set_job_state_callback(ORTE_JOB_STATE_NOTIFY_COMPLETED, notify_requestor))) {
ORTE_ERROR_LOG(rc);
ORTE_UPDATE_EXIT_STATUS(rc);
exit(orte_exit_status);
}
/* spawn the DVM - we skip the initial steps as this
* isn't a user-level application */
ORTE_ACTIVATE_JOB_STATE(jdata, ORTE_JOB_STATE_ALLOCATE);
/* loop the event lib until an exit event is detected */
while (orte_event_base_active) {
opal_event_loop(orte_event_base, OPAL_EVLOOP_ONCE);
}
/* cleanup and leave */
orte_finalize();
if (orte_debug_flag) {
fprintf(stderr, "exiting with status %d\n", orte_exit_status);
}
exit(orte_exit_status);
}
int orcm_init(orcm_proc_type_t flags)
{
int ret;
char *error;
int i, spin;
if (NULL != getenv("ORCM_MCA_spin")) {
spin = 1;
/* spin until a debugger can attach */
while (0 != spin) {
ret = 0;
while (ret < 10000) {
ret++;
};
}
}
if (!orcm_util_initialized) {
orcm_init_util();
}
/* set the default leader policy */
orcm_default_leader_policy.jobid = ORTE_JOBID_WILDCARD;
orcm_default_leader_policy.vpid = ORTE_VPID_WILDCARD;
/* get the number of max msgs */
mca_base_param_reg_int_name("orcm", "max_buffered_msgs",
"Number of recvd messages to hold in storage from each source",
false, false, ORCM_MAX_MSG_RING_SIZE, &orcm_max_msg_ring_size);
/* independent mode or not */
mca_base_param_reg_int_name("orcm", "sched_kill_dvm",
"Whether or not scheduler kills associated daemons upon termination (default: no)",
false, false, (int)false, &ret);
orcm_sched_kill_dvm = OPAL_INT_TO_BOOL(ret);
/* setup the globals that require initialization */
orcm_triplets = OBJ_NEW(orcm_triplets_array_t);
#ifdef HAVE_QSYSTEM_H
#ifdef Q_SYSTEM_INTFCS_TO_PROBE_FOR_IP_ADDRESS
{
char *eth_ifs[] = Q_SYSTEM_INTFCS_TO_PROBE_FOR_IP_ADDRESS;
char **adds=NULL, *ifs, *envar;
int i, num_ifs;
num_ifs = sizeof(eth_ifs) / sizeof(eth_ifs[0]);
for (i=0; i < num_ifs; i++) {
opal_argv_append_nosize(&adds, eth_ifs[i]);
}
ifs = opal_argv_join(adds, ',');
opal_argv_free(adds);
/* push it into the environ so that the rmcast framework can get it */
asprintf(&envar, "OMPI_MCA_rmcast_base_if_include=%s", ifs);
putenv(envar);
/* cannot release envar as the environ doesn't keep its own copy */
free(ifs);
}
#endif
#endif
/* initialize us */
if (ORTE_SUCCESS != (ret = orte_init(NULL, NULL, flags))) {
error = "orte_init";
goto error;
}
if (!ORCM_PROC_IS_TOOL) {
opal_set_using_threads(true);
}
if (!ORCM_PROC_IS_APP) {
trap_signals();
}
orcm_initialized = true;
return ORCM_SUCCESS;
error:
if (ORCM_ERR_SILENT != ret) {
orte_show_help("help-openrcm-runtime.txt",
"orcm_init:startup:internal-failure",
true, error, ORTE_ERROR_NAME(ret), ret);
}
return ret;
}
int
main(int argc, char *argv[])
{
int exit_status = 0, ret, flags = 0, i;
int exec_argc = 0, user_argc = 0;
char **exec_argv = NULL, **user_argv = NULL;
char *exec_command, *base_argv0 = NULL;
bool disable_flags = true;
bool real_flag = false;
if (OPAL_SUCCESS != (ret = opal_init_util(&argc, &argv))) {
return ret;
}
/****************************************************
*
* Setup compiler information
*
****************************************************/
base_argv0 = opal_basename(argv[0]);
#if defined(EXEEXT)
if( 0 != strlen(EXEEXT) ) {
char extension[] = EXEEXT;
char* temp = strstr( base_argv0, extension );
char* old_match = temp;
while( NULL != temp ) {
old_match = temp;
temp = strstr( temp + 1, extension );
}
/* Only if there was a match of .exe, erase the last occurence of .exe */
if ( NULL != old_match ) {
*old_match = '\0';
}
}
#endif /* defined(EXEEXT) */
if (OPAL_SUCCESS != (ret = data_init(base_argv0))) {
fprintf(stderr, "Error parsing data file %s: %s\n", base_argv0, opal_strerror(ret));
return ret;
}
for (i = 1 ; i < argc && user_data_idx < 0 ; ++i) {
user_data_idx = find_options_index(argv[i]);
}
/* if we didn't find a match, look for the NULL (base case) options */
if (user_data_idx < 0) {
user_data_idx = default_data_idx;
}
/* if we still didn't find a match, abort */
if (user_data_idx < 0) {
char *flat = opal_argv_join(argv, ' ');
opal_show_help("help-opal-wrapper.txt", "no-options-support", true,
base_argv0, flat, NULL);
free(flat);
exit(1);
}
/* compiler */
load_env_data(options_data[user_data_idx].project_short, options_data[user_data_idx].compiler_env, &options_data[user_data_idx].compiler);
/* preprocessor flags */
load_env_data_argv(options_data[user_data_idx].project_short, "CPPFLAGS", &options_data[user_data_idx].preproc_flags);
/* compiler flags */
load_env_data_argv(options_data[user_data_idx].project_short, options_data[user_data_idx].compiler_flags_env,
&options_data[user_data_idx].comp_flags);
/* linker flags */
load_env_data_argv(options_data[user_data_idx].project_short, "LDFLAGS", &options_data[user_data_idx].link_flags);
/* libs */
load_env_data_argv(options_data[user_data_idx].project_short, "LIBS", &options_data[user_data_idx].libs);
/****************************************************
*
* Sanity Checks
*
****************************************************/
if (NULL != options_data[user_data_idx].req_file) {
/* make sure the language is supported */
if (0 == strcmp(options_data[user_data_idx].req_file, "not supported")) {
opal_show_help("help-opal-wrapper.txt", "no-language-support", true,
options_data[user_data_idx].language, base_argv0, NULL);
exit_status = 1;
goto cleanup;
}
if (options_data[user_data_idx].req_file[0] != '\0') {
char *filename;
struct stat buf;
filename = opal_os_path( false, options_data[user_data_idx].path_libdir, options_data[user_data_idx].req_file, NULL );
if (0 != stat(filename, &buf)) {
opal_show_help("help-opal-wrapper.txt", "file-not-found", true,
base_argv0, options_data[user_data_idx].req_file, options_data[user_data_idx].language, NULL);
}
}
}
/****************************************************
*
* Parse user flags
*
****************************************************/
flags = COMP_WANT_COMMAND|COMP_WANT_PREPROC|
COMP_WANT_COMPILE|COMP_WANT_LINK;
user_argv = opal_argv_copy(argv + 1);
user_argc = opal_argv_count(user_argv);
for (i = 0 ; i < user_argc ; ++i) {
if (0 == strncmp(user_argv[i], "-showme", strlen("-showme")) ||
0 == strncmp(user_argv[i], "--showme", strlen("--showme")) ||
0 == strncmp(user_argv[i], "-show", strlen("-show")) ||
0 == strncmp(user_argv[i], "--show", strlen("--show"))) {
bool done_now = false;
/* check for specific things we want to see. First three
still invoke all the building routines. Last set want
to parse out certain flags, so we don't go through the
normal build routine - skip to cleanup. */
if (0 == strncmp(user_argv[i], "-showme:command", strlen("-showme:command")) ||
0 == strncmp(user_argv[i], "--showme:command", strlen("--showme:command"))) {
flags = COMP_WANT_COMMAND;
/* we know what we want, so don't process any more args */
done_now = true;
} else if (0 == strncmp(user_argv[i], "-showme:compile", strlen("-showme:compile")) ||
0 == strncmp(user_argv[i], "--showme:compile", strlen("--showme:compile"))) {
flags = COMP_WANT_PREPROC|COMP_WANT_COMPILE;
/* we know what we want, so don't process any more args */
done_now = true;
} else if (0 == strncmp(user_argv[i], "-showme:link", strlen("-showme:link")) ||
0 == strncmp(user_argv[i], "--showme:link", strlen("--showme:link"))) {
flags = COMP_WANT_COMPILE|COMP_WANT_LINK;
/* we know what we want, so don't process any more args */
done_now = true;
} else if (0 == strncmp(user_argv[i], "-showme:incdirs", strlen("-showme:incdirs")) ||
0 == strncmp(user_argv[i], "--showme:incdirs", strlen("--showme:incdirs"))) {
print_flags(options_data[user_data_idx].preproc_flags, OPAL_INCLUDE_FLAG);
goto cleanup;
} else if (0 == strncmp(user_argv[i], "-showme:libdirs", strlen("-showme:libdirs")) ||
0 == strncmp(user_argv[i], "--showme:libdirs", strlen("--showme:libdirs"))) {
print_flags(options_data[user_data_idx].link_flags, OPAL_LIBDIR_FLAG);
goto cleanup;
} else if (0 == strncmp(user_argv[i], "-showme:libs", strlen("-showme:libs")) ||
0 == strncmp(user_argv[i], "--showme:libs", strlen("--showme:libs"))) {
print_flags(options_data[user_data_idx].libs, "-l");
goto cleanup;
} else if (0 == strncmp(user_argv[i], "-showme:version", strlen("-showme:version")) ||
0 == strncmp(user_argv[i], "--showme:version", strlen("--showme:version"))) {
char * str;
str = opal_show_help_string("help-opal-wrapper.txt",
"version", false,
argv[0], options_data[user_data_idx].project, options_data[user_data_idx].version, options_data[user_data_idx].language, NULL);
if (NULL != str) {
printf("%s", str);
free(str);
}
goto cleanup;
} else if (0 == strncmp(user_argv[i], "-showme:help", strlen("-showme:help")) ||
0 == strncmp(user_argv[i], "--showme:help", strlen("--showme:help"))) {
char *str;
str = opal_show_help_string("help-opal-wrapper.txt", "usage",
false, argv[0],
options_data[user_data_idx].project,
NULL);
if (NULL != str) {
printf("%s", str);
free(str);
}
exit_status = 0;
goto cleanup;
} else if (0 == strncmp(user_argv[i], "-showme:", strlen("-showme:")) ||
0 == strncmp(user_argv[i], "--showme:", strlen("--showme:"))) {
fprintf(stderr, "%s: unrecognized option: %s\n", argv[0],
user_argv[i]);
fprintf(stderr, "Type '%s --showme:help' for usage.\n",
argv[0]);
exit_status = 1;
goto cleanup;
}
flags |= (COMP_DRY_RUN|COMP_SHOW_ERROR);
/* remove element from user_argv */
opal_argv_delete(&user_argc, &user_argv, i, 1);
--i;
if (done_now) {
disable_flags = false;
break;
}
} else if (0 == strcmp(user_argv[i], "-c")) {
flags &= ~COMP_WANT_LINK;
real_flag = true;
} else if (0 == strcmp(user_argv[i], "-E") ||
0 == strcmp(user_argv[i], "-M")) {
flags &= ~(COMP_WANT_COMPILE | COMP_WANT_LINK);
real_flag = true;
} else if (0 == strcmp(user_argv[i], "-S")) {
flags &= ~COMP_WANT_LINK;
real_flag = true;
} else if (0 == strcmp(user_argv[i], "-lpmpi")) {
flags |= COMP_WANT_PMPI;
/* remove element from user_argv */
opal_argv_delete(&user_argc, &user_argv, i, 1);
--i;
} else if (0 == strcmp(user_argv[i], "-static") ||
0 == strcmp(user_argv[i], "--static") ||
0 == strcmp(user_argv[i], "-Bstatic") ||
0 == strcmp(user_argv[i], "-Wl,-static") ||
0 == strcmp(user_argv[i], "-Wl,--static") ||
0 == strcmp(user_argv[i], "-Wl,-Bstatic")) {
flags |= COMP_WANT_STATIC;
} else if (0 == strcmp(user_argv[i], "-dynamic") ||
0 == strcmp(user_argv[i], "--dynamic") ||
0 == strcmp(user_argv[i], "-Bdynamic") ||
0 == strcmp(user_argv[i], "-Wl,-dynamic") ||
0 == strcmp(user_argv[i], "-Wl,--dynamic") ||
0 == strcmp(user_argv[i], "-Wl,-Bdynamic")) {
flags &= ~COMP_WANT_STATIC;
} else if (0 == strcmp(user_argv[i], "--openmpi:linkall")) {
/* This is an intentionally undocummented wrapper compiler
switch. It should only be used by Open MPI developers
-- not end users. It will cause mpicc to use the
static library list, even if we're compiling
dynamically (i.e., it'll specifically -lopen-rte and
-lopen-pal (and all their dependent libs)). We provide
this flag for test MPI applications that also invoke
ORTE and/or OPAL function calls.
On some systems (e.g., OS X), if the top-level
application calls ORTE/OPAL functions and you don't -l
ORTE and OPAL, then the functions won't be resolved at
link time (i.e., the implicit library dependencies of
libmpi won't be pulled in at link time), and therefore
the link will fail. This flag will cause the wrapper
to explicitly list the ORTE and OPAL libs on the
underlying compiler command line, so the application
will therefore link properly. */
flags |= COMP_WANT_LINKALL;
/* remove element from user_argv */
opal_argv_delete(&user_argc, &user_argv, i, 1);
} else if ('-' != user_argv[i][0]) {
disable_flags = false;
flags |= COMP_SHOW_ERROR;
real_flag = true;
} else {
/* if the option flag is one that we use to determine
which set of compiler data to use, don't count it as a
real option */
if (find_options_index(user_argv[i]) < 0) {
real_flag = true;
}
}
}
/* clear out the want_flags if we got no arguments not starting
with a - (dash) and -showme wasn't given OR -showme was given
and we had at least one more non-showme argument that started
with a - (dash) and no other non-dash arguments. Some examples:
opal_wrapper : clear our flags
opal_wrapper -v : clear our flags
opal_wrapper -E a.c : don't clear our flags
opal_wrapper a.c : don't clear our flags
opal_wrapper -showme : don't clear our flags
opal_wrapper -showme -v : clear our flags
opal_wrapper -showme -E a.c : don't clear our flags
opal_wrapper -showme a.c : don't clear our flags
*/
if (disable_flags && !((flags & COMP_DRY_RUN) && !real_flag)) {
flags &= ~(COMP_WANT_PREPROC|COMP_WANT_COMPILE|COMP_WANT_LINK);
}
/****************************************************
*
* Assemble the command line
*
****************************************************/
/* compiler (may be multiple arguments, so split) */
if (flags & COMP_WANT_COMMAND) {
exec_argv = opal_argv_split(options_data[user_data_idx].compiler, ' ');
exec_argc = opal_argv_count(exec_argv);
} else {
exec_argv = (char **) malloc(sizeof(char*));
exec_argv[0] = NULL;
exec_argc = 0;
}
/* This error would normally not happen unless the user edits the
wrapper data files manually */
if (NULL == exec_argv) {
opal_show_help("help-opal-wrapper.txt", "no-compiler-specified", true);
return 1;
}
if (flags & COMP_WANT_COMPILE) {
opal_argv_insert(&exec_argv, exec_argc,
options_data[user_data_idx].comp_flags_prefix);
exec_argc = opal_argv_count(exec_argv);
}
/* Per https://svn.open-mpi.org/trac/ompi/ticket/2201, add all the
user arguments before anything else. */
opal_argv_insert(&exec_argv, exec_argc, user_argv);
exec_argc = opal_argv_count(exec_argv);
/* preproc flags */
if (flags & COMP_WANT_PREPROC) {
opal_argv_insert(&exec_argv, exec_argc, options_data[user_data_idx].preproc_flags);
exec_argc = opal_argv_count(exec_argv);
}
/* compiler flags */
if (flags & COMP_WANT_COMPILE) {
opal_argv_insert(&exec_argv, exec_argc, options_data[user_data_idx].comp_flags);
exec_argc = opal_argv_count(exec_argv);
}
/* link flags and libs */
if (flags & COMP_WANT_LINK) {
bool have_static_lib;
bool have_dyn_lib;
bool use_static_libs;
char *filename1, *filename2;
struct stat buf;
opal_argv_insert(&exec_argv, exec_argc, options_data[user_data_idx].link_flags);
exec_argc = opal_argv_count(exec_argv);
/* Are we linking statically? If so, decide what libraries to
list. It depends on two factors:
1. Was --static (etc.) specified?
2. Does OMPI have static, dynamic, or both libraries installed?
Here's a matrix showing what we'll do in all 6 cases:
What's installed --static no --static
---------------- ---------- -----------
ompi .so libs -lmpi -lmpi
ompi .a libs all all
ompi both libs all -lmpi
*/
filename1 = opal_os_path( false, options_data[user_data_idx].path_libdir, options_data[user_data_idx].static_lib_file, NULL );
if (0 == stat(filename1, &buf)) {
have_static_lib = true;
} else {
have_static_lib = false;
}
filename2 = opal_os_path( false, options_data[user_data_idx].path_libdir, options_data[user_data_idx].dyn_lib_file, NULL );
if (0 == stat(filename2, &buf)) {
have_dyn_lib = true;
} else {
have_dyn_lib = false;
}
/* Determine which set of libs to use: dynamic or static. Be
pedantic to make the code easy to read. */
if (flags & COMP_WANT_LINKALL) {
/* If --openmpi:linkall was specified, list all the libs
(i.e., the static libs) if they're available, either in
static or dynamic form. */
if (have_static_lib || have_dyn_lib) {
use_static_libs = true;
} else {
fprintf(stderr, "The linkall option has failed as we were unable to find either static or dynamic libs\n"
"Files looked for:\n Static: %s\n Dynamic: %s\n",
filename1, filename2);
free(filename1);
free(filename2);
exit(1);
}
} else if (flags & COMP_WANT_STATIC) {
/* If --static (or something like it) was specified, if we
have the static libs, then use them. Otherwise, use
the dynamic libs. */
if (have_static_lib) {
use_static_libs = true;
} else {
use_static_libs = false;
}
} else {
/* If --static (or something like it) was NOT specified
(or if --dyanic, or something like it, was specified),
if we have the dynamic libs, then use them. Otherwise,
use the static libs. */
if (have_dyn_lib) {
use_static_libs = false;
} else {
use_static_libs = true;
}
}
free(filename1);
free(filename2);
if (use_static_libs) {
opal_argv_insert(&exec_argv, exec_argc, options_data[user_data_idx].libs_static);
} else {
opal_argv_insert(&exec_argv, exec_argc, options_data[user_data_idx].libs);
}
exec_argc = opal_argv_count(exec_argv);
}
/****************************************************
*
* Execute the command
*
****************************************************/
if (flags & COMP_DRY_RUN) {
exec_command = opal_argv_join(exec_argv, ' ');
printf("%s\n", exec_command);
} else {
char *tmp;
#if 0
exec_command = opal_argv_join(exec_argv, ' ');
printf("command: %s\n", exec_command);
#endif
tmp = opal_path_findv(exec_argv[0], 0, environ, NULL);
if (NULL == tmp) {
opal_show_help("help-opal-wrapper.txt", "no-compiler-found", true,
exec_argv[0], NULL);
errno = 0;
exit_status = 1;
} else {
int status;
free(exec_argv[0]);
exec_argv[0] = tmp;
ret = opal_few(exec_argv, &status);
exit_status = WIFEXITED(status) ? WEXITSTATUS(status) :
(WIFSIGNALED(status) ? WTERMSIG(status) :
(WIFSTOPPED(status) ? WSTOPSIG(status) : 255));
if( (OPAL_SUCCESS != ret) || ((0 != exit_status) && (flags & COMP_SHOW_ERROR)) ) {
char* exec_command = opal_argv_join(exec_argv, ' ');
if( OPAL_SUCCESS != ret ) {
opal_show_help("help-opal-wrapper.txt", "spawn-failed", true,
exec_argv[0], strerror(status), exec_command, NULL);
} else {
#if 0
opal_show_help("help-opal-wrapper.txt", "compiler-failed", true,
exec_argv[0], exit_status, exec_command, NULL);
#endif
}
free(exec_command);
}
}
}
/****************************************************
*
* Cleanup
*
****************************************************/
cleanup:
opal_argv_free(exec_argv);
opal_argv_free(user_argv);
if (NULL != base_argv0) free(base_argv0);
if (OPAL_SUCCESS != (ret = data_finalize())) {
return ret;
}
if (OPAL_SUCCESS != (ret = opal_finalize_util())) {
return ret;
}
return exit_status;
}
static int mca_btl_udapl_modify_ia_list(DAT_COUNT *num_info_entries,
DAT_PROVIDER_INFO* datinfo)
{
int i,j,k,found;
DAT_PROVIDER_INFO* tmp_datinfo = NULL;
DAT_COUNT tmp_num_entries = 0;
tmp_datinfo = malloc((*num_info_entries) * sizeof(DAT_PROVIDER_INFO));
if(NULL == tmp_datinfo) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
for (i = 0; i < *num_info_entries; i++) {
j = 0;
found = 0;
/* search for datinfo entry on the if_list list */
while (mca_btl_udapl_component.if_list[j]) {
if (0 == strcmp(datinfo[i].ia_name,
mca_btl_udapl_component.if_list[j])) {
found = 1;
/* remove from if_list */
k = opal_argv_count(mca_btl_udapl_component.if_list);
opal_argv_delete(&k, &(mca_btl_udapl_component.if_list),
j, 1);
break;
}
j++;
}
if (found) {
if (NULL != mca_btl_udapl_component.if_include_list) {
/* explicitly include */
tmp_datinfo[tmp_num_entries] = datinfo[i];
tmp_num_entries++;
}
/* if this is if_exclude case and match found do nothing */
} else {
/* if this is if_include case and match not found do nothing */
if (NULL != mca_btl_udapl_component.if_exclude_list) {
/* not found for exclude case so actually include here */
tmp_datinfo[tmp_num_entries] = datinfo[i];
tmp_num_entries++;
}
}
}
/* set new values */
*num_info_entries = tmp_num_entries;
for (j = 0; j < *num_info_entries; j++) {
datinfo[j] = tmp_datinfo[j];
}
/* if if_list not NULL, either not found or user error */
if (opal_argv_count(mca_btl_udapl_component.if_list)) {
char *str = opal_argv_join(mca_btl_udapl_component.if_list, ',');
BTL_UDAPL_VERBOSE_HELP(VERBOSE_SHOW_HELP,
("help-mpi-btl-udapl.txt", "nonexistent entry",
true, orte_process_info.nodename,
((NULL != mca_btl_udapl_component.if_include) ?
"in" : "ex"), str));
free(str);
}
free(tmp_datinfo);
return OMPI_SUCCESS;
}
int orte_daemon(int argc, char *argv[])
{
int ret = 0;
opal_cmd_line_t *cmd_line = NULL;
int i;
opal_buffer_t *buffer;
char hostname[OPAL_MAXHOSTNAMELEN];
pmix_value_t val;
pmix_proc_t proc;
pmix_status_t prc;
myxfer_t xfer;
pmix_data_buffer_t pbuf;
pmix_byte_object_t pbo;
opal_byte_object_t bo, *boptr;
/* initialize the globals */
memset(&orted_globals, 0, sizeof(orted_globals));
/* initialize the singleton died pipe to an illegal value so we can detect it was set */
orted_globals.singleton_died_pipe = -1;
bucket = OBJ_NEW(opal_buffer_t);
/* setup to check common command line options that just report and die */
cmd_line = OBJ_NEW(opal_cmd_line_t);
if (OPAL_SUCCESS != opal_cmd_line_create(cmd_line, orte_cmd_line_opts)) {
OBJ_RELEASE(cmd_line);
exit(1);
}
mca_base_cmd_line_setup(cmd_line);
if (ORTE_SUCCESS != (ret = opal_cmd_line_parse(cmd_line, false, false,
argc, argv))) {
char *args = NULL;
args = opal_cmd_line_get_usage_msg(cmd_line);
fprintf(stderr, "Usage: %s [OPTION]...\n%s\n", argv[0], args);
free(args);
OBJ_RELEASE(cmd_line);
return ret;
}
/*
* Since this process can now handle MCA/GMCA parameters, make sure to
* process them.
*/
mca_base_cmd_line_process_args(cmd_line, &environ, &environ);
/* Ensure that enough of OPAL is setup for us to be able to run */
/*
* NOTE: (JJH)
* We need to allow 'mca_base_cmd_line_process_args()' to process command
* line arguments *before* calling opal_init_util() since the command
* line could contain MCA parameters that affect the way opal_init_util()
* functions. AMCA parameters are one such option normally received on the
* command line that affect the way opal_init_util() behaves.
* It is "safe" to call mca_base_cmd_line_process_args() before
* opal_init_util() since mca_base_cmd_line_process_args() does *not*
* depend upon opal_init_util() functionality.
*/
if (OPAL_SUCCESS != opal_init_util(&argc, &argv)) {
fprintf(stderr, "OPAL failed to initialize -- orted aborting\n");
exit(1);
}
/* save the environment for launch purposes. This MUST be
* done so that we can pass it to any local procs we
* spawn - otherwise, those local procs won't see any
* non-MCA envars that were set in the enviro when the
* orted was executed - e.g., by .csh
*/
orte_launch_environ = opal_argv_copy(environ);
/* purge any ess/pmix flags set in the environ when we were launched */
opal_unsetenv(OPAL_MCA_PREFIX"ess", &orte_launch_environ);
opal_unsetenv(OPAL_MCA_PREFIX"pmix", &orte_launch_environ);
/* if orte_daemon_debug is set, let someone know we are alive right
* away just in case we have a problem along the way
*/
if (orted_globals.debug) {
gethostname(hostname, sizeof(hostname));
fprintf(stderr, "Daemon was launched on %s - beginning to initialize\n", hostname);
}
/* check for help request */
if (orted_globals.help) {
char *args = NULL;
args = opal_cmd_line_get_usage_msg(cmd_line);
orte_show_help("help-orted.txt", "orted:usage", false,
argv[0], args);
free(args);
return 1;
}
#if defined(HAVE_SETSID)
/* see if we were directed to separate from current session */
if (orted_globals.set_sid) {
setsid();
}
#endif
/* see if they want us to spin until they can connect a debugger to us */
i=0;
while (orted_spin_flag) {
i++;
if (1000 < i) i=0;
}
/* detach from controlling terminal
* otherwise, remain attached so output can get to us
*/
if(!orte_debug_flag &&
!orte_debug_daemons_flag &&
orted_globals.daemonize) {
opal_daemon_init(NULL);
}
/* Set the flag telling OpenRTE that I am NOT a
* singleton, but am "infrastructure" - prevents setting
* up incorrect infrastructure that only a singleton would
* require.
*/
if (orted_globals.hnp) {
if (ORTE_SUCCESS != (ret = orte_init(&argc, &argv, ORTE_PROC_HNP))) {
ORTE_ERROR_LOG(ret);
return ret;
}
} else {
if (ORTE_SUCCESS != (ret = orte_init(&argc, &argv, ORTE_PROC_DAEMON))) {
ORTE_ERROR_LOG(ret);
return ret;
}
}
/* finalize the OPAL utils. As they are opened again from orte_init->opal_init
* we continue to have a reference count on them. So we have to finalize them twice...
*/
opal_finalize_util();
/* bind ourselves if so directed */
if (NULL != orte_daemon_cores) {
char **cores=NULL, tmp[128];
hwloc_obj_t pu;
hwloc_cpuset_t ours, res;
int core;
/* could be a collection of comma-delimited ranges, so
* use our handy utility to parse it
*/
orte_util_parse_range_options(orte_daemon_cores, &cores);
if (NULL != cores) {
ours = hwloc_bitmap_alloc();
hwloc_bitmap_zero(ours);
res = hwloc_bitmap_alloc();
for (i=0; NULL != cores[i]; i++) {
core = strtoul(cores[i], NULL, 10);
if (NULL == (pu = opal_hwloc_base_get_pu(opal_hwloc_topology, core, OPAL_HWLOC_LOGICAL))) {
/* turn off the show help forwarding as we won't
* be able to cycle the event library to send
*/
orte_show_help_finalize();
/* the message will now come out locally */
orte_show_help("help-orted.txt", "orted:cannot-bind",
true, orte_process_info.nodename,
orte_daemon_cores);
ret = ORTE_ERR_NOT_SUPPORTED;
hwloc_bitmap_free(ours);
hwloc_bitmap_free(res);
goto DONE;
}
hwloc_bitmap_or(res, ours, pu->cpuset);
hwloc_bitmap_copy(ours, res);
}
/* if the result is all zeros, then don't bind */
if (!hwloc_bitmap_iszero(ours)) {
(void)hwloc_set_cpubind(opal_hwloc_topology, ours, 0);
if (opal_hwloc_report_bindings) {
opal_hwloc_base_cset2mapstr(tmp, sizeof(tmp), opal_hwloc_topology, ours);
opal_output(0, "Daemon %s is bound to cores %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), tmp);
}
}
/* cleanup */
hwloc_bitmap_free(ours);
hwloc_bitmap_free(res);
opal_argv_free(cores);
}
}
if ((int)ORTE_VPID_INVALID != orted_debug_failure) {
orted_globals.abort=false;
/* some vpid was ordered to fail. The value can be positive
* or negative, depending upon the desired method for failure,
* so need to check both here
*/
if (0 > orted_debug_failure) {
orted_debug_failure = -1*orted_debug_failure;
orted_globals.abort = true;
}
/* are we the specified vpid? */
if ((int)ORTE_PROC_MY_NAME->vpid == orted_debug_failure) {
/* if the user specified we delay, then setup a timer
* and have it kill us
*/
if (0 < orted_debug_failure_delay) {
ORTE_TIMER_EVENT(orted_debug_failure_delay, 0, shutdown_callback, ORTE_SYS_PRI);
} else {
opal_output(0, "%s is executing clean %s", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
orted_globals.abort ? "abort" : "abnormal termination");
/* do -not- call finalize as this will send a message to the HNP
* indicating clean termination! Instead, just forcibly cleanup
* the local session_dir tree and exit
*/
orte_session_dir_cleanup(ORTE_JOBID_WILDCARD);
/* if we were ordered to abort, do so */
if (orted_globals.abort) {
abort();
}
/* otherwise, return with non-zero status */
ret = ORTE_ERROR_DEFAULT_EXIT_CODE;
goto DONE;
}
}
}
/* insert our contact info into our process_info struct so we
* have it for later use and set the local daemon field to our name
*/
orte_oob_base_get_addr(&orte_process_info.my_daemon_uri);
if (NULL == orte_process_info.my_daemon_uri) {
/* no way to communicate */
ret = ORTE_ERROR;
goto DONE;
}
ORTE_PROC_MY_DAEMON->jobid = ORTE_PROC_MY_NAME->jobid;
ORTE_PROC_MY_DAEMON->vpid = ORTE_PROC_MY_NAME->vpid;
PMIX_VALUE_LOAD(&val, orte_process_info.my_daemon_uri, OPAL_STRING);
(void)opal_snprintf_jobid(proc.nspace, PMIX_MAX_NSLEN, ORTE_PROC_MY_NAME->jobid);
proc.rank = ORTE_PROC_MY_NAME->vpid;
if (PMIX_SUCCESS != (prc = PMIx_Store_internal(&proc, PMIX_PROC_URI, &val))) {
PMIX_ERROR_LOG(prc);
PMIX_VALUE_DESTRUCT(&val);
ret = ORTE_ERROR;
goto DONE;
}
PMIX_VALUE_DESTRUCT(&val);
/* if I am also the hnp, then update that contact info field too */
if (ORTE_PROC_IS_HNP) {
orte_process_info.my_hnp_uri = strdup(orte_process_info.my_daemon_uri);
ORTE_PROC_MY_HNP->jobid = ORTE_PROC_MY_NAME->jobid;
ORTE_PROC_MY_HNP->vpid = ORTE_PROC_MY_NAME->vpid;
}
/* setup the primary daemon command receive function */
orte_rml.recv_buffer_nb(ORTE_NAME_WILDCARD, ORTE_RML_TAG_DAEMON,
ORTE_RML_PERSISTENT, orte_daemon_recv, NULL);
/* output a message indicating we are alive, our name, and our pid
* for debugging purposes
*/
if (orte_debug_daemons_flag) {
fprintf(stderr, "Daemon %s checking in as pid %ld on host %s\n",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), (long)orte_process_info.pid,
orte_process_info.nodename);
}
/* We actually do *not* want the orted to voluntarily yield() the
processor more than necessary. The orted already blocks when
it is doing nothing, so it doesn't use any more CPU cycles than
it should; but when it *is* doing something, we do not want it
to be unnecessarily delayed because it voluntarily yielded the
processor in the middle of its work.
For example: when a message arrives at the orted, we want the
OS to wake up the orted in a timely fashion (which most OS's
seem good about doing) and then we want the orted to process
the message as fast as possible. If the orted yields and lets
aggressive MPI applications get the processor back, it may be a
long time before the OS schedules the orted to run again
(particularly if there is no IO event to wake it up). Hence,
routed OOB messages (for example) may be significantly delayed
before being delivered to MPI processes, which can be
problematic in some scenarios (e.g., COMM_SPAWN, BTL's that
require OOB messages for wireup, etc.). */
opal_progress_set_yield_when_idle(false);
/* Change the default behavior of libevent such that we want to
continually block rather than blocking for the default timeout
and then looping around the progress engine again. There
should be nothing in the orted that cannot block in libevent
until "something" happens (i.e., there's no need to keep
cycling through progress because the only things that should
happen will happen in libevent). This is a minor optimization,
but what the heck... :-) */
opal_progress_set_event_flag(OPAL_EVLOOP_ONCE);
/* if requested, report my uri to the indicated pipe */
if (orted_globals.uri_pipe > 0) {
orte_job_t *jdata;
orte_proc_t *p;
orte_node_t *node;
orte_app_context_t *app;
char *tmp, *nptr, *sysinfo;
char **singenv=NULL, *string_key, *env_str;
/* setup the singleton's job */
jdata = OBJ_NEW(orte_job_t);
/* default to ompi for now */
opal_argv_append_nosize(&jdata->personality, "ompi");
orte_plm_base_create_jobid(jdata);
opal_hash_table_set_value_uint32(orte_job_data, jdata->jobid, jdata);
/* must create a map for it (even though it has no
* info in it) so that the job info will be picked
* up in subsequent pidmaps or other daemons won't
* know how to route
*/
jdata->map = OBJ_NEW(orte_job_map_t);
/* setup an app_context for the singleton */
app = OBJ_NEW(orte_app_context_t);
app->app = strdup("singleton");
app->num_procs = 1;
opal_pointer_array_add(jdata->apps, app);
jdata->num_apps = 1;
/* setup a proc object for the singleton - since we
* -must- be the HNP, and therefore we stored our
* node on the global node pool, and since the singleton
* -must- be on the same node as us, indicate that
*/
p = OBJ_NEW(orte_proc_t);
p->name.jobid = jdata->jobid;
p->name.vpid = 0;
p->parent = 0;
ORTE_FLAG_SET(p, ORTE_PROC_FLAG_ALIVE);
p->state = ORTE_PROC_STATE_RUNNING;
p->app_idx = 0;
/* obviously, it is on my node */
node = (orte_node_t*)opal_pointer_array_get_item(orte_node_pool, 0);
p->node = node;
OBJ_RETAIN(node); /* keep accounting straight */
opal_pointer_array_add(jdata->procs, p);
jdata->num_procs = 1;
/* add the node to the job map */
OBJ_RETAIN(node);
opal_pointer_array_add(jdata->map->nodes, node);
jdata->map->num_nodes++;
/* and it obviously is on the node */
OBJ_RETAIN(p);
opal_pointer_array_add(node->procs, p);
node->num_procs++;
/* and obviously it is one of my local procs */
OBJ_RETAIN(p);
opal_pointer_array_add(orte_local_children, p);
jdata->num_local_procs = 1;
/* set the trivial */
p->local_rank = 0;
p->node_rank = 0;
p->app_rank = 0;
p->state = ORTE_PROC_STATE_RUNNING;
p->app_idx = 0;
ORTE_FLAG_SET(p, ORTE_PROC_FLAG_LOCAL);
/* register the singleton's nspace with our PMIx server */
if (ORTE_SUCCESS != (ret = orte_pmix_server_register_nspace(jdata))) {
ORTE_ERROR_LOG(ret);
goto DONE;
}
/* use setup fork to create the envars needed by the singleton */
(void)opal_snprintf_jobid(proc.nspace, PMIX_MAX_NSLEN, p->name.jobid);
proc.rank = p->name.vpid;
if (PMIX_SUCCESS != (prc = PMIx_server_setup_fork(&proc, &singenv))) {
PMIX_ERROR_LOG(prc);
ret = ORTE_ERROR;
goto DONE;
}
/* append the transport key to the envars needed by the singleton */
if (!orte_get_attribute(&jdata->attributes, ORTE_JOB_TRANSPORT_KEY, (void**)&string_key, OPAL_STRING) || NULL == string_key) {
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
goto DONE;
}
opal_asprintf(&env_str, OPAL_MCA_PREFIX"orte_precondition_transports=%s", string_key);
opal_argv_append_nosize(&singenv, env_str);
free(env_str);
nptr = opal_argv_join(singenv, '*');
opal_argv_free(singenv);
/* create a string that contains our uri + sysinfo + PMIx server URI envars */
orte_util_convert_sysinfo_to_string(&sysinfo, orte_local_cpu_type, orte_local_cpu_model);
opal_asprintf(&tmp, "%s[%s]%s", orte_process_info.my_daemon_uri, sysinfo, nptr);
free(sysinfo);
free(nptr);
/* pass that info to the singleton */
if (OPAL_SUCCESS != (ret = opal_fd_write(orted_globals.uri_pipe, strlen(tmp)+1, tmp))) { ; /* need to add 1 to get the NULL */
ORTE_ERROR_LOG(ret);
goto DONE;
}
/* cleanup */
free(tmp);
close(orted_globals.uri_pipe);
/* since a singleton spawned us, we need to harvest
* any MCA params from the local environment so
* we can pass them along to any subsequent daemons
* we may start as the result of a comm_spawn
*/
for (i=0; NULL != environ[i]; i++) {
if (0 == strncmp(environ[i], OPAL_MCA_PREFIX, 9)) {
/* make a copy to manipulate */
tmp = strdup(environ[i]);
/* find the equal sign */
nptr = strchr(tmp, '=');
*nptr = '\0';
nptr++;
/* add the mca param to the orted cmd line */
opal_argv_append_nosize(&orted_cmd_line, "-"OPAL_MCA_CMD_LINE_ID);
opal_argv_append_nosize(&orted_cmd_line, &tmp[9]);
opal_argv_append_nosize(&orted_cmd_line, nptr);
free(tmp);
}
}
}
/* if we were given a pipe to monitor for singleton termination, set that up */
if (orted_globals.singleton_died_pipe > 0) {
/* register shutdown handler */
pipe_handler = (opal_event_t*)malloc(sizeof(opal_event_t));
opal_event_set(orte_event_base, pipe_handler,
orted_globals.singleton_died_pipe,
OPAL_EV_READ,
pipe_closed,
pipe_handler);
opal_event_add(pipe_handler, NULL);
}
/* If I have a parent, then save his contact info so
* any messages we send can flow thru him.
*/
orte_parent_uri = NULL;
(void) mca_base_var_register ("orte", "orte", NULL, "parent_uri",
"URI for the parent if tree launch is enabled.",
MCA_BASE_VAR_TYPE_STRING, NULL, 0,
MCA_BASE_VAR_FLAG_INTERNAL,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_CONSTANT,
&orte_parent_uri);
if (NULL != orte_parent_uri) {
/* set the contact info into our local database */
ret = orte_rml_base_parse_uris(orte_parent_uri, ORTE_PROC_MY_PARENT, NULL);
if (ORTE_SUCCESS != ret) {
ORTE_ERROR_LOG(ret);
goto DONE;
}
PMIX_VALUE_LOAD(&val, orte_parent_uri, OPAL_STRING);
OPAL_PMIX_CONVERT_NAME(&proc, ORTE_PROC_MY_PARENT);
if (PMIX_SUCCESS != (prc = PMIx_Store_internal(&proc, PMIX_PROC_URI, &val))) {
PMIX_ERROR_LOG(prc);
PMIX_VALUE_DESTRUCT(&val);
ret = ORTE_ERROR;
goto DONE;
}
PMIX_VALUE_DESTRUCT(&val);
/* tell the routed module that we have a path
* back to the HNP
*/
if (ORTE_SUCCESS != (ret = orte_routed.update_route(ORTE_PROC_MY_HNP, ORTE_PROC_MY_PARENT))) {
ORTE_ERROR_LOG(ret);
goto DONE;
}
/* and a path to our parent */
if (ORTE_SUCCESS != (ret = orte_routed.update_route(ORTE_PROC_MY_PARENT, ORTE_PROC_MY_PARENT))) {
ORTE_ERROR_LOG(ret);
goto DONE;
}
/* set the lifeline to point to our parent so that we
* can handle the situation if that lifeline goes away
*/
if (ORTE_SUCCESS != (ret = orte_routed.set_lifeline(ORTE_PROC_MY_PARENT))) {
ORTE_ERROR_LOG(ret);
goto DONE;
}
}
/* if we are not the HNP...the only time we will be an HNP
* is if we are launched by a singleton to provide support
* for it
*/
if (!ORTE_PROC_IS_HNP) {
orte_process_name_t target;
/* setup the rollup callback */
orte_rml.recv_buffer_nb(ORTE_NAME_WILDCARD, ORTE_RML_TAG_ORTED_CALLBACK,
ORTE_RML_PERSISTENT, rollup, NULL);
/* define the target jobid */
target.jobid = ORTE_PROC_MY_NAME->jobid;
if (orte_fwd_mpirun_port || orte_static_ports || NULL != orte_parent_uri) {
/* we start by sending to ourselves */
target.vpid = ORTE_PROC_MY_NAME->vpid;
/* since we will be waiting for any children to send us
* their rollup info before sending to our parent, save
* a little time in the launch phase by "warming up" the
* connection to our parent while we wait for our children */
buffer = OBJ_NEW(opal_buffer_t); // zero-byte message
orte_rml.recv_buffer_nb(ORTE_PROC_MY_PARENT, ORTE_RML_TAG_NODE_REGEX_REPORT,
ORTE_RML_PERSISTENT, node_regex_report, &node_regex_waiting);
node_regex_waiting = true;
if (0 > (ret = orte_rml.send_buffer_nb(ORTE_PROC_MY_PARENT, buffer,
ORTE_RML_TAG_WARMUP_CONNECTION,
orte_rml_send_callback, NULL))) {
ORTE_ERROR_LOG(ret);
OBJ_RELEASE(buffer);
goto DONE;
}
} else {
target.vpid = 0;
}
/* send the information to the orted report-back point - this function
* will process the data, but also counts the number of
* orteds that reported back so the launch procedure can continue.
* We need to do this at the last possible second as the HNP
* can turn right around and begin issuing orders to us
*/
buffer = OBJ_NEW(opal_buffer_t);
/* insert our name for rollup purposes */
if (ORTE_SUCCESS != (ret = opal_dss.pack(buffer, ORTE_PROC_MY_NAME, 1, ORTE_NAME))) {
ORTE_ERROR_LOG(ret);
OBJ_RELEASE(buffer);
goto DONE;
}
/* get any connection info we may have pushed */
{
pmix_info_t *info;
size_t ninfo;
pmix_value_t *vptr;
OPAL_PMIX_CONVERT_NAME(&proc, ORTE_PROC_MY_NAME);
boptr = &bo;
bo.bytes = NULL;
bo.size = 0;
if (PMIX_SUCCESS == PMIx_Get(&proc, NULL, NULL, 0, &vptr) && NULL != vptr) {
/* the data is returned as a pmix_data_array_t */
if (PMIX_DATA_ARRAY != vptr->type || NULL == vptr->data.darray ||
PMIX_INFO != vptr->data.darray->type || NULL == vptr->data.darray->array) {
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
OBJ_RELEASE(buffer);
goto DONE;
}
/* use the PMIx data support to pack it */
info = (pmix_info_t*)vptr->data.darray->array;
ninfo = vptr->data.darray->size;
PMIX_DATA_BUFFER_CONSTRUCT(&pbuf);
if (PMIX_SUCCESS != (prc = PMIx_Data_pack(&proc, &pbuf, &ninfo, 1, PMIX_SIZE))) {
PMIX_ERROR_LOG(prc);
ret = ORTE_ERROR;
OBJ_RELEASE(buffer);
goto DONE;
}
if (PMIX_SUCCESS != (prc = PMIx_Data_pack(&proc, &pbuf, info, ninfo, PMIX_INFO))) {
PMIX_ERROR_LOG(prc);
ret = ORTE_ERROR;
OBJ_RELEASE(buffer);
goto DONE;
}
PMIX_DATA_BUFFER_UNLOAD(&pbuf, pbo.bytes, pbo.size);
bo.bytes = (uint8_t*)pbo.bytes;
bo.size = pbo.size;
PMIX_VALUE_RELEASE(vptr);
}
if (ORTE_SUCCESS != (ret = opal_dss.pack(buffer, &boptr, 1, OPAL_BYTE_OBJECT))) {
ORTE_ERROR_LOG(ret);
OBJ_RELEASE(buffer);
goto DONE;
}
}
/* include our node name */
opal_dss.pack(buffer, &orte_process_info.nodename, 1, OPAL_STRING);
/* if requested, include any non-loopback aliases for this node */
if (orte_retain_aliases) {
char **aliases=NULL;
uint8_t naliases, ni;
char hostname[OPAL_MAXHOSTNAMELEN];
/* if we stripped the prefix or removed the fqdn,
* include full hostname as an alias
*/
gethostname(hostname, sizeof(hostname));
if (strlen(orte_process_info.nodename) < strlen(hostname)) {
opal_argv_append_nosize(&aliases, hostname);
}
opal_ifgetaliases(&aliases);
naliases = opal_argv_count(aliases);
if (ORTE_SUCCESS != (ret = opal_dss.pack(buffer, &naliases, 1, OPAL_UINT8))) {
ORTE_ERROR_LOG(ret);
OBJ_RELEASE(buffer);
opal_argv_free(aliases);
goto DONE;
}
for (ni=0; ni < naliases; ni++) {
if (ORTE_SUCCESS != (ret = opal_dss.pack(buffer, &aliases[ni], 1, OPAL_STRING))) {
ORTE_ERROR_LOG(ret);
OBJ_RELEASE(buffer);
opal_argv_free(aliases);
goto DONE;
}
}
opal_argv_free(aliases);
}
/* always send back our topology signature - this is a small string
* and won't hurt anything */
if (ORTE_SUCCESS != (ret = opal_dss.pack(buffer, &orte_topo_signature, 1, OPAL_STRING))) {
ORTE_ERROR_LOG(ret);
}
/* if we are rank=1, then send our topology back - otherwise, mpirun
* will request it if necessary */
if (1 == ORTE_PROC_MY_NAME->vpid) {
opal_buffer_t data;
int8_t flag;
uint8_t *cmpdata;
size_t cmplen;
/* setup an intermediate buffer */
OBJ_CONSTRUCT(&data, opal_buffer_t);
if (ORTE_SUCCESS != (ret = opal_dss.pack(&data, &opal_hwloc_topology, 1, OPAL_HWLOC_TOPO))) {
ORTE_ERROR_LOG(ret);
}
if (opal_compress.compress_block((uint8_t*)data.base_ptr, data.bytes_used,
&cmpdata, &cmplen)) {
/* the data was compressed - mark that we compressed it */
flag = 1;
if (ORTE_SUCCESS != (ret = opal_dss.pack(buffer, &flag, 1, OPAL_INT8))) {
ORTE_ERROR_LOG(ret);
free(cmpdata);
OBJ_DESTRUCT(&data);
}
/* pack the compressed length */
if (ORTE_SUCCESS != (ret = opal_dss.pack(buffer, &cmplen, 1, OPAL_SIZE))) {
ORTE_ERROR_LOG(ret);
free(cmpdata);
OBJ_DESTRUCT(&data);
}
/* pack the uncompressed length */
if (ORTE_SUCCESS != (ret = opal_dss.pack(buffer, &data.bytes_used, 1, OPAL_SIZE))) {
ORTE_ERROR_LOG(ret);
free(cmpdata);
OBJ_DESTRUCT(&data);
}
/* pack the compressed info */
if (ORTE_SUCCESS != (ret = opal_dss.pack(buffer, cmpdata, cmplen, OPAL_UINT8))) {
ORTE_ERROR_LOG(ret);
free(cmpdata);
OBJ_DESTRUCT(&data);
}
OBJ_DESTRUCT(&data);
free(cmpdata);
} else {
/* mark that it was not compressed */
flag = 0;
if (ORTE_SUCCESS != (ret = opal_dss.pack(buffer, &flag, 1, OPAL_INT8))) {
ORTE_ERROR_LOG(ret);
OBJ_DESTRUCT(&data);
free(cmpdata);
}
/* transfer the payload across */
opal_dss.copy_payload(buffer, &data);
OBJ_DESTRUCT(&data);
}
}
/* collect our network inventory */
memset(&xfer, 0, sizeof(myxfer_t));
OPAL_PMIX_CONSTRUCT_LOCK(&xfer.lock);
if (PMIX_SUCCESS != (prc = PMIx_server_collect_inventory(NULL, 0, infocbfunc, &xfer))) {
PMIX_ERROR_LOG(prc);
ret = ORTE_ERR_NOT_SUPPORTED;
goto DONE;
}
OPAL_PMIX_WAIT_THREAD(&xfer.lock);
if (NULL != xfer.info) {
PMIX_DATA_BUFFER_CONSTRUCT(&pbuf);
if (PMIX_SUCCESS != (prc = PMIx_Data_pack(NULL, &pbuf, &xfer.ninfo, 1, PMIX_SIZE))) {
PMIX_ERROR_LOG(prc);
ret = ORTE_ERROR;
OBJ_RELEASE(buffer);
goto DONE;
}
if (PMIX_SUCCESS != (prc = PMIx_Data_pack(NULL, &pbuf, xfer.info, xfer.ninfo, PMIX_INFO))) {
PMIX_ERROR_LOG(prc);
ret = ORTE_ERROR;
OBJ_RELEASE(buffer);
goto DONE;
}
PMIX_DATA_BUFFER_UNLOAD(&pbuf, pbo.bytes, pbo.size);
bo.bytes = (uint8_t*)pbo.bytes;
bo.size = pbo.size;
if (ORTE_SUCCESS != (ret = opal_dss.pack(buffer, &boptr, 1, OPAL_BYTE_OBJECT))) {
ORTE_ERROR_LOG(ret);
OBJ_RELEASE(buffer);
goto DONE;
}
}
/* send it to the designated target */
if (0 > (ret = orte_rml.send_buffer_nb(&target, buffer,
ORTE_RML_TAG_ORTED_CALLBACK,
orte_rml_send_callback, NULL))) {
ORTE_ERROR_LOG(ret);
OBJ_RELEASE(buffer);
goto DONE;
}
}
/* if we are tree-spawning, then we need to capture the MCA params
* from our cmd line so we can pass them along to the daemons we spawn -
* otherwise, only the first layer of daemons will ever see them
*/
if (orted_globals.tree_spawn) {
int j, k;
bool ignore;
char *no_keep[] = {
"orte_hnp_uri",
"orte_ess_jobid",
"orte_ess_vpid",
"orte_ess_num_procs",
"orte_parent_uri",
"mca_base_env_list",
NULL
};
for (i=0; i < argc; i++) {
if (0 == strcmp("-"OPAL_MCA_CMD_LINE_ID, argv[i]) ||
0 == strcmp("--"OPAL_MCA_CMD_LINE_ID, argv[i]) ) {
ignore = false;
/* see if this is something we cannot pass along */
for (k=0; NULL != no_keep[k]; k++) {
if (0 == strcmp(no_keep[k], argv[i+1])) {
ignore = true;
break;
}
}
if (!ignore) {
/* see if this is already present so we at least can
* avoid growing the cmd line with duplicates
*/
if (NULL != orted_cmd_line) {
for (j=0; NULL != orted_cmd_line[j]; j++) {
if (0 == strcmp(argv[i+1], orted_cmd_line[j])) {
/* already here - ignore it */
ignore = true;
break;
}
}
}
if (!ignore) {
opal_argv_append_nosize(&orted_cmd_line, argv[i]);
opal_argv_append_nosize(&orted_cmd_line, argv[i+1]);
opal_argv_append_nosize(&orted_cmd_line, argv[i+2]);
}
}
i += 2;
}
}
}
if (orte_debug_daemons_flag) {
opal_output(0, "%s orted: up and running - waiting for commands!", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
}
ret = ORTE_SUCCESS;
/* loop the event lib until an exit event is detected */
while (orte_event_base_active) {
opal_event_loop(orte_event_base, OPAL_EVLOOP_ONCE);
}
ORTE_ACQUIRE_OBJECT(orte_event_base_active);
/* ensure all local procs are dead */
orte_odls.kill_local_procs(NULL);
DONE:
/* update the exit status, in case it wasn't done */
ORTE_UPDATE_EXIT_STATUS(ret);
/* cleanup and leave */
orte_finalize();
opal_finalize_util();
orte_session_dir_cleanup(ORTE_JOBID_WILDCARD);
/* cleanup the process info */
orte_proc_info_finalize();
if (orte_debug_flag) {
fprintf(stderr, "exiting with status %d\n", orte_exit_status);
}
exit(orte_exit_status);
}
static int s2_init(void)
{
int spawned, size, rank, appnum;
int rc, ret = OPAL_ERROR;
char buf[16];
int found;
int my_node;
uint32_t stepid;
int i;
opal_process_name_t ldr;
opal_value_t kv;
char **localranks;
char *str;
char nmtmp[64];
opal_process_name_t wildcard_rank;
/* if we can't startup PMI, we can't be used */
if ( PMI2_Initialized () ) {
return OPAL_SUCCESS;
}
size = -1;
rank = -1;
appnum = -1;
// setup hash table so we always can finalize it
opal_pmix_base_hash_init();
if (PMI2_SUCCESS != (rc = PMI2_Init(&spawned, &size, &rank, &appnum))) {
opal_show_help("help-pmix-base.txt", "pmix2-init-failed", true, rc);
return OPAL_ERROR;
}
if( size < 0 || rank < 0 ){
opal_show_help("help-pmix-base.txt", "pmix2-init-returned-bad-values", true);
goto err_exit;
}
s2_jsize = size;
s2_rank = rank;
s2_appnum = appnum;
pmix_vallen_max = PMI2_MAX_VALLEN;
pmix_kvslen_max = PMI2_MAX_VALLEN; // FIX ME: What to put here for versatility?
pmix_keylen_max = PMI2_MAX_KEYLEN;
pmix_vallen_threshold = PMI2_MAX_VALLEN * 3;
pmix_vallen_threshold >>= 2;
pmix_kvs_name = (char*)malloc(pmix_kvslen_max);
if( pmix_kvs_name == NULL ){
PMI2_Finalize();
ret = OPAL_ERR_OUT_OF_RESOURCE;
goto err_exit;
}
rc = PMI2_Job_GetId(pmix_kvs_name, pmix_kvslen_max);
if( PMI2_SUCCESS != rc ) {
OPAL_PMI_ERROR(rc, "PMI2_Job_GetId");
free(pmix_kvs_name);
goto err_exit;
}
/* store our name in the opal_proc_t so that
* debug messages will make sense - an upper
* layer will eventually overwrite it, but that
* won't do any harm */
s2_pname.jobid = strtoul(pmix_kvs_name, &str, 10);
s2_pname.jobid = (s2_pname.jobid << 16) & 0xffff0000;
if (NULL != str) {
stepid = strtoul(str, NULL, 10);
s2_pname.jobid |= (stepid & 0x0000ffff);
}
s2_pname.vpid = s2_rank;
opal_proc_set_name(&s2_pname);
opal_output_verbose(2, opal_pmix_base_framework.framework_output,
"%s pmix:s2: assigned tmp name",
OPAL_NAME_PRINT(s2_pname));
/* setup wildcard rank*/
wildcard_rank = OPAL_PROC_MY_NAME;
wildcard_rank.vpid = OPAL_VPID_WILDCARD;
/* Slurm PMI provides the job id as an integer followed
* by a '.', followed by essentially a stepid. The first integer
* defines an overall job number. The second integer is the number of
* individual jobs we have run within that allocation.
*/
OBJ_CONSTRUCT(&kv, opal_value_t);
kv.key = strdup(OPAL_PMIX_JOBID);
kv.type = OPAL_UINT32;
kv.data.uint32 = s2_pname.jobid;
if (OPAL_SUCCESS != (ret = opal_pmix_base_store(&wildcard_rank, &kv))) {
OPAL_ERROR_LOG(ret);
OBJ_DESTRUCT(&kv);
goto err_exit;
}
OBJ_DESTRUCT(&kv); // frees pmix_kvs_name
/* save the job size */
OBJ_CONSTRUCT(&kv, opal_value_t);
kv.key = strdup(OPAL_PMIX_JOB_SIZE);
kv.type = OPAL_UINT32;
kv.data.uint32 = size;
if (OPAL_SUCCESS != (rc = opal_pmix_base_store(&wildcard_rank, &kv))) {
OPAL_ERROR_LOG(rc);
OBJ_DESTRUCT(&kv);
goto err_exit;
}
OBJ_DESTRUCT(&kv);
/* save the appnum */
OBJ_CONSTRUCT(&kv, opal_value_t);
kv.key = strdup(OPAL_PMIX_APPNUM);
kv.type = OPAL_UINT32;
kv.data.uint32 = appnum;
if (OPAL_SUCCESS != (ret = opal_pmix_base_store(&OPAL_PROC_MY_NAME, &kv))) {
OPAL_ERROR_LOG(ret);
OBJ_DESTRUCT(&kv);
goto err_exit;
}
OBJ_DESTRUCT(&kv);
rc = PMI2_Info_GetJobAttr("universeSize", buf, 16, &found);
if( PMI2_SUCCESS != rc ) {
OPAL_PMI_ERROR(rc, "PMI_Get_universe_size");
goto err_exit;
}
/* save it */
OBJ_CONSTRUCT(&kv, opal_value_t);
kv.key = strdup(OPAL_PMIX_UNIV_SIZE);
kv.type = OPAL_UINT32;
kv.data.uint32 = atoi(buf);
if (OPAL_SUCCESS != (rc = opal_pmix_base_store(&wildcard_rank, &kv))) {
OPAL_ERROR_LOG(rc);
OBJ_DESTRUCT(&kv);
goto err_exit;
}
OBJ_DESTRUCT(&kv);
/* push this into the dstore for subsequent fetches */
OBJ_CONSTRUCT(&kv, opal_value_t);
kv.key = strdup(OPAL_PMIX_MAX_PROCS);
kv.type = OPAL_UINT32;
kv.data.uint32 = atoi(buf);
if (OPAL_SUCCESS != (ret = opal_pmix_base_store(&wildcard_rank, &kv))) {
OPAL_ERROR_LOG(ret);
OBJ_DESTRUCT(&kv);
goto err_exit;
}
OBJ_DESTRUCT(&kv);
char *pmapping = (char*)malloc(PMI2_MAX_VALLEN);
if( pmapping == NULL ){
rc = OPAL_ERR_OUT_OF_RESOURCE;
OPAL_ERROR_LOG(rc);
return rc;
}
rc = PMI2_Info_GetJobAttr("PMI_process_mapping", pmapping, PMI2_MAX_VALLEN, &found);
if( !found || PMI2_SUCCESS != rc ) {
OPAL_PMI_ERROR(rc,"PMI2_Info_GetJobAttr");
return OPAL_ERROR;
}
s2_lranks = mca_common_pmi2_parse_pmap(pmapping, s2_pname.vpid, &my_node, &s2_nlranks);
if (NULL == s2_lranks) {
rc = OPAL_ERR_OUT_OF_RESOURCE;
OPAL_ERROR_LOG(rc);
return rc;
}
free(pmapping);
/* save the local size */
OBJ_CONSTRUCT(&kv, opal_value_t);
kv.key = strdup(OPAL_PMIX_LOCAL_SIZE);
kv.type = OPAL_UINT32;
kv.data.uint32 = s2_nlranks;
if (OPAL_SUCCESS != (rc = opal_pmix_base_store(&wildcard_rank, &kv))) {
OPAL_ERROR_LOG(rc);
OBJ_DESTRUCT(&kv);
goto err_exit;
}
OBJ_DESTRUCT(&kv);
s2_lrank = 0;
s2_nrank = 0;
ldr.vpid = rank;
localranks = NULL;
if (0 < s2_nlranks && NULL != s2_lranks) {
/* note the local ldr */
ldr.vpid = s2_lranks[0];
/* find ourselves */
ldr.jobid = s2_pname.jobid;
ldr.vpid = s2_pname.vpid;
memset(nmtmp, 0, 64);
for (i=0; i < s2_nlranks; i++) {
(void)snprintf(nmtmp, 64, "%d", s2_lranks[i]);
opal_argv_append_nosize(&localranks, nmtmp);
if (s2_rank == s2_lranks[i]) {
s2_lrank = i;
s2_nrank = i;
}
}
str = opal_argv_join(localranks, ',');
opal_argv_free(localranks);
OBJ_CONSTRUCT(&kv, opal_value_t);
kv.key = strdup(OPAL_PMIX_LOCAL_PEERS);
kv.type = OPAL_STRING;
kv.data.string = str;
if (OPAL_SUCCESS != (ret = opal_pmix_base_store(&wildcard_rank, &kv))) {
OPAL_ERROR_LOG(ret);
OBJ_DESTRUCT(&kv);
goto err_exit;
}
OBJ_DESTRUCT(&kv);
}
/* save the local leader */
OBJ_CONSTRUCT(&kv, opal_value_t);
kv.key = strdup(OPAL_PMIX_LOCALLDR);
kv.type = OPAL_UINT64;
kv.data.uint64 = *(uint64_t*)&ldr;
if (OPAL_SUCCESS != (ret = opal_pmix_base_store(&OPAL_PROC_MY_NAME, &kv))) {
OPAL_ERROR_LOG(ret);
OBJ_DESTRUCT(&kv);
goto err_exit;
}
OBJ_DESTRUCT(&kv);
/* save our local rank */
OBJ_CONSTRUCT(&kv, opal_value_t);
kv.key = strdup(OPAL_PMIX_LOCAL_RANK);
kv.type = OPAL_UINT16;
kv.data.uint16 = s2_lrank;
if (OPAL_SUCCESS != (ret = opal_pmix_base_store(&OPAL_PROC_MY_NAME, &kv))) {
OPAL_ERROR_LOG(ret);
OBJ_DESTRUCT(&kv);
goto err_exit;
}
OBJ_DESTRUCT(&kv);
/* and our node rank */
OBJ_CONSTRUCT(&kv, opal_value_t);
kv.key = strdup(OPAL_PMIX_NODE_RANK);
kv.type = OPAL_UINT16;
kv.data.uint16 = s2_nrank;
if (OPAL_SUCCESS != (ret = opal_pmix_base_store(&OPAL_PROC_MY_NAME, &kv))) {
OPAL_ERROR_LOG(ret);
OBJ_DESTRUCT(&kv);
goto err_exit;
}
OBJ_DESTRUCT(&kv);
/* increment the init count */
++pmix_init_count;
return OPAL_SUCCESS;
err_exit:
PMI2_Finalize();
return ret;
}
