int opal_compress_bzip_compress_nb(char * fname, char **cname, char **postfix, pid_t *child_pid)
{
char **argv = NULL;
char * base_fname = NULL;
char * dir_fname = NULL;
int status;
bool is_dir;
is_dir = is_directory(fname);
*child_pid = fork();
if( *child_pid == 0 ) { /* Child */
char * cmd;
dir_fname = opal_dirname(fname);
base_fname = opal_basename(fname);
chdir(dir_fname);
if( is_dir ) {
#if 0
opal_compress_base_tar_create(&base_fname);
asprintf(cname, "%s.bz2", base_fname);
asprintf(&cmd, "bzip2 %s", base_fname);
#else
asprintf(cname, "%s.tar.bz2", base_fname);
asprintf(&cmd, "tar -jcf %s %s", *cname, base_fname);
#endif
} else {
asprintf(cname, "%s.bz2", base_fname);
asprintf(&cmd, "bzip2 %s", base_fname);
}
opal_output_verbose(10, mca_compress_bzip_component.super.output_handle,
"compress:bzip: compress_nb(%s -> [%s])",
fname, *cname);
opal_output_verbose(10, mca_compress_bzip_component.super.output_handle,
"compress:bzip: compress_nb() command [%s]",
cmd);
argv = opal_argv_split(cmd, ' ');
status = execvp(argv[0], argv);
opal_output(0, "compress:bzip: compress_nb: Failed to exec child [%s] status = %d\n", cmd, status);
exit(OPAL_ERROR);
}
else if( *child_pid > 0 ) {
if( is_dir ) {
*postfix = strdup(".tar.bz2");
} else {
*postfix = strdup(".bz2");
}
asprintf(cname, "%s%s", fname, *postfix);
}
else {
return OPAL_ERROR;
}
return OPAL_SUCCESS;
}
static int orte_cr_coord_post_restart(void) {
int ret, exit_status = ORTE_SUCCESS;
orte_proc_type_t prev_type = ORTE_PROC_TYPE_NONE;
char * tmp_dir = NULL;
opal_output_verbose(10, orte_cr_output,
"orte_cr: coord_post_restart: orte_cr_coord_post_restart()");
/*
* Add the previous session directory for cleanup
*/
opal_crs_base_cleanup_append(orte_process_info.job_session_dir, true);
tmp_dir = opal_dirname(orte_process_info.job_session_dir);
if( NULL != tmp_dir ) {
opal_crs_base_cleanup_append(tmp_dir, true);
free(tmp_dir);
tmp_dir = NULL;
}
/*
* Refresh System information
*/
prev_type = orte_process_info.proc_type;
if( ORTE_SUCCESS != (ret = orte_proc_info_finalize()) ) {
exit_status = ret;
}
if( NULL != orte_process_info.my_hnp_uri ) {
free(orte_process_info.my_hnp_uri);
orte_process_info.my_hnp_uri = NULL;
}
if( NULL != orte_process_info.my_daemon_uri ) {
free(orte_process_info.my_daemon_uri);
orte_process_info.my_daemon_uri = NULL;
}
if( ORTE_SUCCESS != (ret = orte_proc_info()) ) {
exit_status = ret;
}
orte_process_info.proc_type = prev_type;
orte_process_info.my_name = *ORTE_NAME_INVALID;
/*
* Notify the ESS
*/
if( NULL != orte_ess.ft_event ) {
if( ORTE_SUCCESS != (ret = orte_ess.ft_event(OPAL_CRS_RESTART))) {
exit_status = ret;
goto cleanup;
}
}
cleanup:
return exit_status;
}
static int ras_tm_open(void)
{
mca_base_component_t *c = &mca_ras_tm_component.super.ras_version;
char *pbs_nodefile_env = NULL;
char *default_nodefile_dir = NULL;
bool free_default_nodefile_dir = false;
param_priority =
mca_base_param_reg_int(c,
"priority",
"Priority of the tm ras component",
false, false, 100, NULL);
/* try to detect the default directory */
pbs_nodefile_env = getenv("PBS_NODEFILE");
if ( NULL != pbs_nodefile_env ) {
default_nodefile_dir = opal_dirname(pbs_nodefile_env);
if ( NULL != default_nodefile_dir ) {
free_default_nodefile_dir = true;
} else {
default_nodefile_dir = "/var/torque/aux";
}
} else {
default_nodefile_dir = "/var/torque/aux";
}
mca_base_param_reg_string(c, "nodefile_dir",
"The directory where the PBS nodefile can be found",
false, false, default_nodefile_dir,
&mca_ras_tm_component.nodefile_dir);
if ( free_default_nodefile_dir ) {
free(default_nodefile_dir);
}
return ORTE_SUCCESS;
}
static int setup_app(orte_app_context_t *app)
{
int i;
char *newenv, *pth, *t2;
bool takeus = false;
if (NULL != orte_schizo_base.personalities) {
/* see if we are included */
for (i=0; NULL != orte_schizo_base.personalities[i]; i++) {
if (0 == strcmp(orte_schizo_base.personalities[i], "singularity")) {
takeus = true;
break;
}
}
}
if (!takeus) {
/* even if they didn't specify, check to see if
* this involves a singularity container */
if (0 != strcmp(app->argv[0],"singularity") &&
0 != strcmp(app->argv[0],"sapprun") &&
NULL == strstr(app->argv[0], ".sapp")) {
/* guess not! */
return ORTE_ERR_TAKE_NEXT_OPTION;
}
}
opal_output_verbose(1, orte_schizo_base_framework.framework_output,
"%s schizo:singularity: checking app %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), app->argv[0]);
if (0 < strlen(OPAL_SINGULARITY_PATH)) {
asprintf(&pth, "%s/singularity", OPAL_SINGULARITY_PATH);
} else {
/* since we allow for detecting singularity's presence, it
* is possible that we found it in the PATH, but not in a
* standard location. Check for that here */
pth = opal_path_findv("singularity", X_OK, app->env, NULL);
if (NULL == pth) {
/* cannot execute */
return ORTE_ERR_TAKE_NEXT_OPTION;
}
}
/* find the path and prepend it with the path to Singularity */
for (i = 0; NULL != app->env && NULL != app->env[i]; ++i) {
/* add to PATH */
if (0 == strncmp("PATH=", app->env[i], 5)) {
t2 = opal_dirname(pth);
asprintf(&newenv, "%s:%s", t2, app->env[i] + 5);
opal_setenv("PATH", newenv, true, &app->env);
free(newenv);
free(t2);
break;
}
}
free(pth);
if (0 == strcmp(app->argv[0], "singularity")) {
/* we don't want the backend to setup a cache dir */
orte_set_attribute(&app->attributes, ORTE_APP_NO_CACHEDIR, ORTE_ATTR_GLOBAL, NULL, OPAL_BOOL);
}
/* export an envar to permit shared memory operations */
opal_setenv("SINGULARITY_NO_NAMESPACE_PID", "1", true, &app->env);
return ORTE_SUCCESS;
}
static int rte_finalize(void)
{
char *contact_path;
char *jobfam_dir;
if (signals_set) {
/* Remove the epipe handler */
opal_event_signal_del(&epipe_handler);
/* remove the term handler */
opal_event_del(&term_handler);
/** Remove the USR signal handlers */
opal_event_signal_del(&sigusr1_handler);
opal_event_signal_del(&sigusr2_handler);
if (orte_forward_job_control) {
opal_event_signal_del(&sigtstp_handler);
opal_event_signal_del(&sigcont_handler);
}
signals_set = false;
}
/* close the dfs */
(void) mca_base_framework_close(&orte_dfs_base_framework);
(void) mca_base_framework_close(&orte_filem_base_framework);
/* output any lingering stdout/err data */
fflush(stdout);
fflush(stderr);
(void) mca_base_framework_close(&orte_iof_base_framework);
(void) mca_base_framework_close(&orte_rtc_base_framework);
(void) mca_base_framework_close(&orte_odls_base_framework);
(void) mca_base_framework_close(&orte_rmaps_base_framework);
(void) mca_base_framework_close(&orte_ras_base_framework);
(void) mca_base_framework_close(&orte_grpcomm_base_framework);
(void) mca_base_framework_close(&opal_dstore_base_framework);
(void) mca_base_framework_close(&orte_routed_base_framework);
(void) mca_base_framework_close(&orte_plm_base_framework);
(void) mca_base_framework_close(&orte_errmgr_base_framework);
(void) mca_base_framework_close(&orte_state_base_framework);
/* cleanup the pstat stuff */
(void) mca_base_framework_close(&opal_pstat_base_framework);
/* remove my contact info file, if we have session directories */
if (NULL != orte_process_info.job_session_dir) {
jobfam_dir = opal_dirname(orte_process_info.job_session_dir);
contact_path = opal_os_path(false, jobfam_dir, "contact.txt", NULL);
free(jobfam_dir);
unlink(contact_path);
free(contact_path);
}
/* shutdown the messaging frameworks */
(void) mca_base_framework_close(&orte_rml_base_framework);
(void) mca_base_framework_close(&orte_oob_base_framework);
/* ensure we scrub the session directory tree */
orte_session_dir_cleanup(ORTE_JOBID_WILDCARD);
/* close the xml output file, if open */
if (orte_xml_output) {
fprintf(orte_xml_fp, "</mpirun>\n");
fflush(orte_xml_fp);
if (stdout != orte_xml_fp) {
fclose(orte_xml_fp);
}
}
return ORTE_SUCCESS;
}
static int rte_init(void)
{
int ret;
char *error = NULL;
char *contact_path, *jobfam_dir;
orte_job_t *jdata;
orte_node_t *node;
orte_proc_t *proc;
orte_app_context_t *app;
char **aliases, *aptr;
/* run the prolog */
if (ORTE_SUCCESS != (ret = orte_ess_base_std_prolog())) {
error = "orte_ess_base_std_prolog";
goto error;
}
/* setup callback for SIGPIPE */
setup_sighandler(SIGPIPE, &epipe_handler, epipe_signal_callback);
/** setup callbacks for abort signals - from this point
* forward, we need to abort in a manner that allows us
* to cleanup. However, we cannot directly use libevent
* to trap these signals as otherwise we cannot respond
* to them if we are stuck in an event! So instead use
* the basic POSIX trap functions to handle the signal,
* and then let that signal handler do some magic to
* avoid the hang
*
* NOTE: posix traps don't allow us to do anything major
* in them, so use a pipe tied to a libevent event to
* reach a "safe" place where the termination event can
* be created
*/
pipe(term_pipe);
/* setup an event to attempt normal termination on signal */
opal_event_set(orte_event_base, &term_handler, term_pipe[0], OPAL_EV_READ, clean_abort, NULL);
opal_event_set_priority(&term_handler, ORTE_ERROR_PRI);
opal_event_add(&term_handler, NULL);
/* Set both ends of this pipe to be close-on-exec so that no
children inherit it */
if (opal_fd_set_cloexec(term_pipe[0]) != OPAL_SUCCESS ||
opal_fd_set_cloexec(term_pipe[1]) != OPAL_SUCCESS) {
error = "unable to set the pipe to CLOEXEC";
goto error;
}
/* point the signal trap to a function that will activate that event */
signal(SIGTERM, abort_signal_callback);
signal(SIGINT, abort_signal_callback);
signal(SIGHUP, abort_signal_callback);
/** setup callbacks for signals we should foward */
setup_sighandler(SIGUSR1, &sigusr1_handler, signal_forward_callback);
setup_sighandler(SIGUSR2, &sigusr2_handler, signal_forward_callback);
setup_sighandler(SIGTSTP, &sigtstp_handler, signal_forward_callback);
setup_sighandler(SIGCONT, &sigcont_handler, signal_forward_callback);
signals_set = true;
#if OPAL_HAVE_HWLOC
{
hwloc_obj_t obj;
unsigned i, j;
/* get the local topology */
if (NULL == opal_hwloc_topology) {
if (OPAL_SUCCESS != opal_hwloc_base_get_topology()) {
error = "topology discovery";
goto error;
}
}
/* remove the hostname from the topology. Unfortunately, hwloc
* decided to add the source hostname to the "topology", thus
* rendering it unusable as a pure topological description. So
* we remove that information here.
*/
obj = hwloc_get_root_obj(opal_hwloc_topology);
for (i=0; i < obj->infos_count; i++) {
if (NULL == obj->infos[i].name ||
NULL == obj->infos[i].value) {
continue;
}
if (0 == strncmp(obj->infos[i].name, "HostName", strlen("HostName"))) {
free(obj->infos[i].name);
free(obj->infos[i].value);
/* left justify the array */
for (j=i; j < obj->infos_count-1; j++) {
obj->infos[j] = obj->infos[j+1];
}
obj->infos[obj->infos_count-1].name = NULL;
obj->infos[obj->infos_count-1].value = NULL;
obj->infos_count--;
break;
}
}
if (4 < opal_output_get_verbosity(orte_ess_base_framework.framework_output)) {
opal_output(0, "%s Topology Info:", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
opal_dss.dump(0, opal_hwloc_topology, OPAL_HWLOC_TOPO);
}
}
#endif
/* if we are using xml for output, put an mpirun start tag */
if (orte_xml_output) {
fprintf(orte_xml_fp, "<mpirun>\n");
fflush(orte_xml_fp);
}
/* open and setup the opal_pstat framework so we can provide
* process stats if requested
*/
if (ORTE_SUCCESS != (ret = mca_base_framework_open(&opal_pstat_base_framework, 0))) {
ORTE_ERROR_LOG(ret);
error = "opal_pstat_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = opal_pstat_base_select())) {
ORTE_ERROR_LOG(ret);
error = "opal_pstat_base_select";
goto error;
}
/* open and setup the state machine */
if (ORTE_SUCCESS != (ret = mca_base_framework_open(&orte_state_base_framework, 0))) {
ORTE_ERROR_LOG(ret);
error = "orte_state_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_state_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_state_base_select";
goto error;
}
/* open the errmgr */
if (ORTE_SUCCESS != (ret = mca_base_framework_open(&orte_errmgr_base_framework, 0))) {
ORTE_ERROR_LOG(ret);
error = "orte_errmgr_base_open";
goto error;
}
/* Since we are the HNP, then responsibility for
* defining the name falls to the PLM component for our
* respective environment - hence, we have to open the PLM
* first and select that component.
*/
if (ORTE_SUCCESS != (ret = mca_base_framework_open(&orte_plm_base_framework, 0))) {
ORTE_ERROR_LOG(ret);
error = "orte_plm_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_plm_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_plm_base_select";
goto error;
}
/* if we were spawned by a singleton, our jobid was given to us */
if (NULL != orte_ess_base_jobid) {
if (ORTE_SUCCESS != (ret = orte_util_convert_string_to_jobid(&ORTE_PROC_MY_NAME->jobid, orte_ess_base_jobid))) {
ORTE_ERROR_LOG(ret);
error = "convert_string_to_jobid";
goto error;
}
ORTE_PROC_MY_NAME->vpid = 0;
} else {
if (ORTE_SUCCESS != (ret = orte_plm.set_hnp_name())) {
ORTE_ERROR_LOG(ret);
error = "orte_plm_set_hnp_name";
goto error;
}
}
/* Setup the communication infrastructure */
/*
* OOB Layer
*/
if (ORTE_SUCCESS != (ret = mca_base_framework_open(&orte_oob_base_framework, 0))) {
ORTE_ERROR_LOG(ret);
error = "orte_oob_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_oob_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_oob_base_select";
goto error;
}
/*
* Runtime Messaging Layer
*/
if (ORTE_SUCCESS != (ret = mca_base_framework_open(&orte_rml_base_framework, 0))) {
ORTE_ERROR_LOG(ret);
error = "orte_rml_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_rml_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_rml_base_select";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_errmgr_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_errmgr_base_select";
goto error;
}
/* setup the global job and node arrays */
orte_job_data = OBJ_NEW(opal_pointer_array_t);
if (ORTE_SUCCESS != (ret = opal_pointer_array_init(orte_job_data,
1,
ORTE_GLOBAL_ARRAY_MAX_SIZE,
1))) {
ORTE_ERROR_LOG(ret);
error = "setup job array";
goto error;
}
orte_node_pool = OBJ_NEW(opal_pointer_array_t);
if (ORTE_SUCCESS != (ret = opal_pointer_array_init(orte_node_pool,
ORTE_GLOBAL_ARRAY_BLOCK_SIZE,
ORTE_GLOBAL_ARRAY_MAX_SIZE,
ORTE_GLOBAL_ARRAY_BLOCK_SIZE))) {
ORTE_ERROR_LOG(ret);
error = "setup node array";
goto error;
}
orte_node_topologies = OBJ_NEW(opal_pointer_array_t);
if (ORTE_SUCCESS != (ret = opal_pointer_array_init(orte_node_topologies,
ORTE_GLOBAL_ARRAY_BLOCK_SIZE,
ORTE_GLOBAL_ARRAY_MAX_SIZE,
ORTE_GLOBAL_ARRAY_BLOCK_SIZE))) {
ORTE_ERROR_LOG(ret);
error = "setup node topologies array";
goto error;
}
/* init the nidmap - just so we register that verbosity */
orte_util_nidmap_init(NULL);
/* Setup the job data object for the daemons */
/* create and store the job data object */
jdata = OBJ_NEW(orte_job_t);
jdata->jobid = ORTE_PROC_MY_NAME->jobid;
opal_pointer_array_set_item(orte_job_data, 0, jdata);
/* mark that the daemons have reported as we are the
* only ones in the system right now, and we definitely
* are running!
*/
jdata->state = ORTE_JOB_STATE_DAEMONS_REPORTED;
/* every job requires at least one app */
app = OBJ_NEW(orte_app_context_t);
opal_pointer_array_set_item(jdata->apps, 0, app);
jdata->num_apps++;
/* create and store a node object where we are */
node = OBJ_NEW(orte_node_t);
node->name = strdup(orte_process_info.nodename);
node->index = opal_pointer_array_set_item(orte_node_pool, 0, node);
#if OPAL_HAVE_HWLOC
/* add it to the array of known topologies */
opal_pointer_array_add(orte_node_topologies, opal_hwloc_topology);
#endif
/* create and store a proc object for us */
proc = OBJ_NEW(orte_proc_t);
proc->name.jobid = ORTE_PROC_MY_NAME->jobid;
proc->name.vpid = ORTE_PROC_MY_NAME->vpid;
proc->pid = orte_process_info.pid;
proc->rml_uri = orte_rml.get_contact_info();
proc->state = ORTE_PROC_STATE_RUNNING;
OBJ_RETAIN(node); /* keep accounting straight */
proc->node = node;
opal_pointer_array_set_item(jdata->procs, proc->name.vpid, proc);
/* record that the daemon (i.e., us) is on this node
* NOTE: we do not add the proc object to the node's
* proc array because we are not an application proc.
* Instead, we record it in the daemon field of the
* node object
*/
OBJ_RETAIN(proc); /* keep accounting straight */
node->daemon = proc;
ORTE_FLAG_SET(node, ORTE_NODE_FLAG_DAEMON_LAUNCHED);
node->state = ORTE_NODE_STATE_UP;
/* if we are to retain aliases, get ours */
if (orte_retain_aliases) {
aliases = NULL;
opal_ifgetaliases(&aliases);
/* add our own local name to it */
opal_argv_append_nosize(&aliases, orte_process_info.nodename);
aptr = opal_argv_join(aliases, ',');
opal_argv_free(aliases);
orte_set_attribute(&node->attributes, ORTE_NODE_ALIAS, ORTE_ATTR_LOCAL, aptr, OPAL_STRING);
free(aptr);
}
/* record that the daemon job is running */
jdata->num_procs = 1;
jdata->state = ORTE_JOB_STATE_RUNNING;
/* obviously, we have "reported" */
jdata->num_reported = 1;
/*
* Routed system
*/
if (ORTE_SUCCESS != (ret = mca_base_framework_open(&orte_routed_base_framework, 0))) {
ORTE_ERROR_LOG(ret);
error = "orte_rml_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_routed_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_routed_base_select";
goto error;
}
/* datastore - ensure we don't pickup the pmi component, but
* don't override anything set by user
*/
if (NULL == getenv("OMPI_MCA_dstore")) {
putenv("OMPI_MCA_dstore=^pmi");
}
if (ORTE_SUCCESS != (ret = mca_base_framework_open(&opal_dstore_base_framework, 0))) {
ORTE_ERROR_LOG(ret);
error = "opal_dstore_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = opal_dstore_base_select())) {
ORTE_ERROR_LOG(ret);
error = "opal_dstore_base_select";
goto error;
}
/* create the handles */
if (0 > (opal_dstore_peer = opal_dstore.open("PEER"))) {
error = "opal dstore global";
ret = ORTE_ERR_FATAL;
goto error;
}
if (0 > (opal_dstore_internal = opal_dstore.open("INTERNAL"))) {
error = "opal dstore internal";
ret = ORTE_ERR_FATAL;
goto error;
}
if (0 > (opal_dstore_nonpeer = opal_dstore.open("NONPEER"))) {
error = "opal dstore nonpeer";
ret = ORTE_ERR_FATAL;
goto error;
}
/*
* Group communications
*/
if (ORTE_SUCCESS != (ret = mca_base_framework_open(&orte_grpcomm_base_framework, 0))) {
ORTE_ERROR_LOG(ret);
error = "orte_grpcomm_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_grpcomm_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_grpcomm_base_select";
goto error;
}
/* Now provide a chance for the PLM
* to perform any module-specific init functions. This
* needs to occur AFTER the communications are setup
* as it may involve starting a non-blocking recv
*/
if (ORTE_SUCCESS != (ret = orte_plm.init())) {
ORTE_ERROR_LOG(ret);
error = "orte_plm_init";
goto error;
}
/*
* Setup the remaining resource
* management and errmgr frameworks - application procs
* and daemons do not open these frameworks as they only use
* the hnp proxy support in the PLM framework.
*/
if (ORTE_SUCCESS != (ret = mca_base_framework_open(&orte_ras_base_framework, 0))) {
ORTE_ERROR_LOG(ret);
error = "orte_ras_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_ras_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_ras_base_find_available";
goto error;
}
if (ORTE_SUCCESS != (ret = mca_base_framework_open(&orte_rmaps_base_framework, 0))) {
ORTE_ERROR_LOG(ret);
error = "orte_rmaps_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_rmaps_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_rmaps_base_find_available";
goto error;
}
#if OPAL_HAVE_HWLOC
{
char *coprocessors, **sns;
uint32_t h;
int idx;
/* if a topology file was given, then the rmaps framework open
* will have reset our topology. Ensure we always get the right
* one by setting our node topology afterwards
*/
node->topology = opal_hwloc_topology;
/* init the hash table, if necessary */
if (NULL == orte_coprocessors) {
orte_coprocessors = OBJ_NEW(opal_hash_table_t);
opal_hash_table_init(orte_coprocessors, orte_process_info.num_procs);
}
/* detect and add any coprocessors */
coprocessors = opal_hwloc_base_find_coprocessors(opal_hwloc_topology);
if (NULL != coprocessors) {
/* separate the serial numbers of the coprocessors
* on this host
*/
sns = opal_argv_split(coprocessors, ',');
for (idx=0; NULL != sns[idx]; idx++) {
/* compute the hash */
OPAL_HASH_STR(sns[idx], h);
/* mark that this coprocessor is hosted by this node */
opal_hash_table_set_value_uint32(orte_coprocessors, h, (void*)&(ORTE_PROC_MY_NAME->vpid));
}
opal_argv_free(sns);
free(coprocessors);
orte_coprocessors_detected = true;
}
/* see if I am on a coprocessor */
coprocessors = opal_hwloc_base_check_on_coprocessor();
if (NULL != coprocessors) {
orte_set_attribute(&node->attributes, ORTE_NODE_SERIAL_NUMBER, ORTE_ATTR_LOCAL, coprocessors, OPAL_STRING);
free(coprocessors);
orte_coprocessors_detected = true;
}
}
#endif
/* Open/select the odls */
if (ORTE_SUCCESS != (ret = mca_base_framework_open(&orte_odls_base_framework, 0))) {
ORTE_ERROR_LOG(ret);
error = "orte_odls_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_odls_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_odls_base_select";
goto error;
}
/* Open/select the rtc */
if (ORTE_SUCCESS != (ret = mca_base_framework_open(&orte_rtc_base_framework, 0))) {
ORTE_ERROR_LOG(ret);
error = "orte_rtc_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_rtc_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_rtc_base_select";
goto error;
}
/* enable communication with the rml */
if (ORTE_SUCCESS != (ret = orte_rml.enable_comm())) {
ORTE_ERROR_LOG(ret);
error = "orte_rml.enable_comm";
goto error;
}
/* we are an hnp, so update the contact info field for later use */
orte_process_info.my_hnp_uri = orte_rml.get_contact_info();
proc->rml_uri = strdup(orte_process_info.my_hnp_uri);
/* we are also officially a daemon, so better update that field too */
orte_process_info.my_daemon_uri = strdup(orte_process_info.my_hnp_uri);
/* setup the orte_show_help system to recv remote output */
orte_rml.recv_buffer_nb(ORTE_NAME_WILDCARD, ORTE_RML_TAG_SHOW_HELP,
ORTE_RML_PERSISTENT, orte_show_help_recv, NULL);
/* setup my session directory */
if (orte_create_session_dirs) {
OPAL_OUTPUT_VERBOSE((2, orte_debug_output,
"%s setting up session dir with\n\ttmpdir: %s\n\thost %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
(NULL == orte_process_info.tmpdir_base) ? "UNDEF" : orte_process_info.tmpdir_base,
orte_process_info.nodename));
/* take a pass thru the session directory code to fillin the
* tmpdir names - don't create anything yet
*/
if (ORTE_SUCCESS != (ret = orte_session_dir(false,
orte_process_info.tmpdir_base,
orte_process_info.nodename, NULL,
ORTE_PROC_MY_NAME))) {
ORTE_ERROR_LOG(ret);
error = "orte_session_dir define";
goto error;
}
/* clear the session directory just in case there are
* stale directories laying around
*/
orte_session_dir_cleanup(ORTE_JOBID_WILDCARD);
/* now actually create the directory tree */
if (ORTE_SUCCESS != (ret = orte_session_dir(true,
orte_process_info.tmpdir_base,
orte_process_info.nodename, NULL,
ORTE_PROC_MY_NAME))) {
ORTE_ERROR_LOG(ret);
error = "orte_session_dir";
goto error;
}
/* Once the session directory location has been established, set
the opal_output hnp file location to be in the
proc-specific session directory. */
opal_output_set_output_file_info(orte_process_info.proc_session_dir,
"output-", NULL, NULL);
/* save my contact info in a file for others to find */
jobfam_dir = opal_dirname(orte_process_info.job_session_dir);
contact_path = opal_os_path(false, jobfam_dir, "contact.txt", NULL);
free(jobfam_dir);
OPAL_OUTPUT_VERBOSE((2, orte_debug_output,
"%s writing contact file %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
contact_path));
if (ORTE_SUCCESS != (ret = orte_write_hnp_contact_file(contact_path))) {
OPAL_OUTPUT_VERBOSE((2, orte_debug_output,
"%s writing contact file failed with error %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_ERROR_NAME(ret)));
} else {
OPAL_OUTPUT_VERBOSE((2, orte_debug_output,
"%s wrote contact file",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
}
free(contact_path);
}
/* setup the routed info - the selected routed component
* will know what to do.
*/
if (ORTE_SUCCESS != (ret = orte_routed.init_routes(ORTE_PROC_MY_NAME->jobid, NULL))) {
ORTE_ERROR_LOG(ret);
error = "orte_routed.init_routes";
goto error;
}
/* setup I/O forwarding system - must come after we init routes */
if (ORTE_SUCCESS != (ret = mca_base_framework_open(&orte_iof_base_framework, 0))) {
ORTE_ERROR_LOG(ret);
error = "orte_iof_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_iof_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_iof_base_select";
goto error;
}
/* setup the FileM */
if (ORTE_SUCCESS != (ret = mca_base_framework_open(&orte_filem_base_framework, 0))) {
ORTE_ERROR_LOG(ret);
error = "orte_filem_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_filem_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_filem_base_select";
goto error;
}
#if OPAL_ENABLE_FT_CR == 1
/*
* Setup the SnapC
*/
if (ORTE_SUCCESS != (ret = mca_base_framework_open(&orte_snapc_base_framework, 0))) {
ORTE_ERROR_LOG(ret);
error = "orte_snapc_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = mca_base_framework_open(&orte_sstore_base_framework, 0))) {
ORTE_ERROR_LOG(ret);
error = "orte_sstore_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_snapc_base_select(ORTE_PROC_IS_HNP, ORTE_PROC_IS_APP))) {
ORTE_ERROR_LOG(ret);
error = "orte_snapc_base_select";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_sstore_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_sstore_base_select";
goto error;
}
/* For HNP, ORTE doesn't need the OPAL CR stuff */
opal_cr_set_enabled(false);
#else
opal_cr_set_enabled(false);
#endif
/*
* Initalize the CR setup
* Note: Always do this, even in non-FT builds.
* If we don't some user level tools may hang.
*/
if (ORTE_SUCCESS != (ret = orte_cr_init())) {
ORTE_ERROR_LOG(ret);
error = "orte_cr_init";
goto error;
}
/* setup the dfs framework */
if (ORTE_SUCCESS != (ret = mca_base_framework_open(&orte_dfs_base_framework, 0))) {
ORTE_ERROR_LOG(ret);
error = "orte_dfs_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_dfs_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_dfs_select";
goto error;
}
/* if a tool has launched us and is requesting event reports,
* then set its contact info into the comm system
*/
if (orte_report_events) {
if (ORTE_SUCCESS != (ret = orte_util_comm_connect_tool(orte_report_events_uri))) {
error = "could not connect to tool";
goto error;
}
}
/* We actually do *not* want an HNP to voluntarily yield() the
processor more than necessary. Orterun already blocks when
it is doing nothing, so it doesn't use any more CPU cycles than
it should; but when it *is* doing something, we do not want it
to be unnecessarily delayed because it voluntarily yielded the
processor in the middle of its work.
For example: when a message arrives at orterun, we want the
OS to wake us up in a timely fashion (which most OS's
seem good about doing) and then we want orterun to process
the message as fast as possible. If orterun yields and lets
aggressive MPI applications get the processor back, it may be a
long time before the OS schedules orterun to run again
(particularly if there is no IO event to wake it up). Hence,
routed OOB messages (for example) may be significantly delayed
before being delivered to MPI processes, which can be
problematic in some scenarios (e.g., COMM_SPAWN, BTL's that
require OOB messages for wireup, etc.). */
opal_progress_set_yield_when_idle(false);
return ORTE_SUCCESS;
error:
if (ORTE_ERR_SILENT != ret && !orte_report_silent_errors) {
orte_show_help("help-orte-runtime.txt",
"orte_init:startup:internal-failure",
true, error, ORTE_ERROR_NAME(ret), ret);
}
return ORTE_ERR_SILENT;
}
/**
* Function for finding and opening either all MCA components, or the one
* that was specifically requested via a MCA parameter.
*/
static int orte_iof_base_open(mca_base_open_flag_t flags)
{
int rc, xmlfd;
/* did the user request we print output to files? */
if (NULL != orte_output_filename) {
/* we will setup the files themselves as needed in the iof
* module. For now, let's see if the filename contains a
* path, or just a name
*/
char *path;
path = opal_dirname(orte_output_filename);
if (NULL == path) {
return ORTE_ERR_OUT_OF_RESOURCE;
}
if (0 != strcmp(path, orte_output_filename)) {
/* there is a path in this name - ensure that the directory
* exists, and create it if not
*/
if (ORTE_SUCCESS != (rc = opal_os_dirpath_create(path, S_IRWXU))) {
free(path);
return rc;
}
}
free(path);
}
/* daemons do not need to do this as they do not write out stdout/err */
if (!ORTE_PROC_IS_DAEMON ||
(ORTE_PROC_IS_DAEMON && ORTE_PROC_IS_CM)) {
if (orte_xml_output) {
if (NULL != orte_xml_fp) {
/* user wants all xml-formatted output sent to file */
xmlfd = fileno(orte_xml_fp);
} else {
xmlfd = 1;
}
/* setup the stdout event */
ORTE_IOF_SINK_DEFINE(&orte_iof_base.iof_write_stdout, ORTE_PROC_MY_NAME,
xmlfd, ORTE_IOF_STDOUT, orte_iof_base_write_handler);
/* don't create a stderr event - all output will go to
* the stdout channel
*/
} else {
/* setup the stdout event */
ORTE_IOF_SINK_DEFINE(&orte_iof_base.iof_write_stdout, ORTE_PROC_MY_NAME,
1, ORTE_IOF_STDOUT, orte_iof_base_write_handler);
/* setup the stderr event */
ORTE_IOF_SINK_DEFINE(&orte_iof_base.iof_write_stderr, ORTE_PROC_MY_NAME,
2, ORTE_IOF_STDERR, orte_iof_base_write_handler);
}
/* do NOT set these file descriptors to non-blocking. If we do so,
* we set the file descriptor to non-blocking for everyone that has
* that file descriptor, which includes everyone else in our shell
* pipeline chain. (See
* http://lists.freebsd.org/pipermail/freebsd-hackers/2005-January/009742.html).
* This causes things like "mpirun -np 1 big_app | cat" to lose
* output, because cat's stdout is then ALSO non-blocking and cat
* isn't built to deal with that case (same with almost all other
* unix text utils).
*/
}
/* Open up all available components */
return mca_base_framework_components_open(&orte_iof_base_framework, flags);
}
int opal_compress_gzip_decompress_nb(char * cname, char **fname, pid_t *child_pid)
{
char * cmd = NULL;
char **argv = NULL;
char * dir_cname = NULL;
pid_t loc_pid = 0;
int status;
bool is_tar = false;
if( 0 == strncmp(&(cname[strlen(cname)-7]), ".tar.gz", strlen(".tar.gz")) ) {
is_tar = true;
}
*fname = strdup(cname);
if( is_tar ) {
/* Strip off '.tar.gz' */
(*fname)[strlen(cname)-7] = '\0';
} else {
/* Strip off '.gz' */
(*fname)[strlen(cname)-3] = '\0';
}
opal_output_verbose(10, mca_compress_gzip_component.super.output_handle,
"compress:gzip: decompress_nb(%s -> [%s])",
cname, *fname);
*child_pid = fork();
if( *child_pid == 0 ) { /* Child */
dir_cname = opal_dirname(cname);
chdir(dir_cname);
/* Fork(gunzip) */
loc_pid = fork();
if( loc_pid == 0 ) { /* Child */
asprintf(&cmd, "gunzip %s", cname);
opal_output_verbose(10, mca_compress_gzip_component.super.output_handle,
"compress:gzip: decompress_nb() command [%s]",
cmd);
argv = opal_argv_split(cmd, ' ');
status = execvp(argv[0], argv);
opal_output(0, "compress:gzip: decompress_nb: Failed to exec child [%s] status = %d\n", cmd, status);
exit(OPAL_ERROR);
}
else if( loc_pid > 0 ) { /* Parent */
waitpid(loc_pid, &status, 0);
if( !WIFEXITED(status) ) {
opal_output(0, "compress:gzip: decompress_nb: Failed to bunzip the file [%s] status = %d\n", cname, status);
exit(OPAL_ERROR);
}
}
else {
exit(OPAL_ERROR);
}
/* tar_decompress */
if( is_tar ) {
/* Strip off '.gz' leaving just '.tar' */
cname[strlen(cname)-3] = '\0';
opal_compress_base_tar_extract(&cname);
}
/* Once this child is done, then directly exit */
exit(OPAL_SUCCESS);
}
else if( *child_pid > 0 ) {
;
}
else {
return OPAL_ERROR;
}
if( NULL != cmd ) {
free(cmd);
cmd = NULL;
}
return OPAL_SUCCESS;
}
int
main(int argc, char *argv[])
{
int ret, exit_status = ORTE_SUCCESS;
pid_t child_pid = 0;
orte_sstore_base_global_snapshot_info_t *snapshot = NULL;
char *basedir = NULL;
char *tmp_str = NULL;
/***************
* Initialize
***************/
if (ORTE_SUCCESS != (ret = initialize(argc, argv))) {
exit_status = ret;
goto cleanup;
}
snapshot = OBJ_NEW(orte_sstore_base_global_snapshot_info_t);
if( opal_path_is_absolute(orte_restart_globals.snapshot_ref) ) {
basedir = opal_dirname(orte_restart_globals.snapshot_ref);
tmp_str = opal_basename(orte_restart_globals.snapshot_ref);
free(orte_restart_globals.snapshot_ref);
orte_restart_globals.snapshot_ref = strdup(tmp_str);
free(tmp_str);
tmp_str = NULL;
} else if( NULL != strchr(orte_restart_globals.snapshot_ref, '/') ) {
basedir = opal_dirname(orte_restart_globals.snapshot_ref);
tmp_str = opal_basename(orte_restart_globals.snapshot_ref);
free(orte_restart_globals.snapshot_ref);
orte_restart_globals.snapshot_ref = strdup(tmp_str);
free(tmp_str);
tmp_str = NULL;
} else {
basedir = NULL; /* Use MCA parameter */
}
/*
* Note: If the seq # passed is -1, then the largest seq # is selected,
* ow the seq # requested is selected if available
* 'basedir': Snapshot Base location to look in. If NULL then MCA parameter is used
*/
if( ORTE_SUCCESS != (ret = orte_sstore.request_restart_handle(&(snapshot->ss_handle),
basedir,
orte_restart_globals.snapshot_ref,
orte_restart_globals.seq_number,
snapshot))) {
opal_show_help("help-orte-restart.txt", "invalid_filename", true,
orte_restart_globals.snapshot_ref);
exit_status = ret;
goto cleanup;
}
orte_restart_globals.seq_number = snapshot->seq_num;
if(orte_restart_globals.info_only ) {
if (ORTE_SUCCESS != (ret = snapshot_info(snapshot))) {
exit_status = ret;
goto cleanup;
}
exit_status = ORTE_SUCCESS;
goto cleanup;
}
/******************************
* Create the app file to use with mpirun/orterun
******************************/
if( ORTE_SUCCESS != (ret = create_appfile(snapshot) ) ) {
exit_status = ret;
goto cleanup;
}
if( orte_restart_globals.app_only ) {
printf("Created Appfile:\n\t%s\n", orte_restart_globals.appfile);
exit_status = ORTE_SUCCESS;
goto cleanup;
}
/******************************
* Restart in this process [mpirun/orterun]
******************************/
if( orte_restart_globals.verbose ) {
opal_output_verbose(10, orte_restart_globals.output,
"Restarting from file (%s)",
orte_restart_globals.snapshot_ref);
if( orte_restart_globals.forked ) {
opal_output_verbose(10, orte_restart_globals.output,
"\t Forking off a child");
} else {
opal_output_verbose(10, orte_restart_globals.output,
"\t Exec in self");
}
}
if( ORTE_SUCCESS != (ret = spawn_children(snapshot, &child_pid)) ) {
opal_show_help("help-orte-restart.txt", "restart_cmd_failure", true,
orte_restart_globals.snapshot_ref, ret);
exit_status = ret;
goto cleanup;
}
/***************
* Cleanup
***************/
cleanup:
if( NULL != basedir ) {
free(basedir);
basedir = NULL;
}
if( NULL != tmp_str ) {
free(tmp_str);
tmp_str = NULL;
}
if( NULL != snapshot ) {
OBJ_RELEASE(snapshot);
snapshot = NULL;
}
if (OPAL_SUCCESS != (ret = finalize())) {
return ret;
}
return exit_status;
}
/*
* Construct the session directory and create it if necessary
*/
int orte_session_dir(bool create,
char *prefix, char *hostid,
char *batchid, orte_process_name_t *proc)
{
char *fulldirpath = NULL,
*frontend = NULL,
*sav = NULL;
int rc = ORTE_SUCCESS;
char *local_prefix = NULL;
/* use the specified prefix, if one was given */
if (NULL != prefix) {
local_prefix = strdup(prefix);
}
/*
* Get the session directory full name
*/
if( ORTE_SUCCESS != ( rc = orte_session_dir_get_name(&fulldirpath,
&local_prefix,
&frontend,
hostid,
batchid, proc) ) ) {
if (ORTE_ERR_FATAL == rc) {
/* this indicates we should abort quietly */
rc = ORTE_ERR_SILENT;
goto cleanup;
}
/* otherwise, bark a little first */
ORTE_ERROR_LOG(rc);
goto cleanup;
}
/*
* Now that we have the full path, go ahead and create it if necessary
*/
if( create ) {
if( ORTE_SUCCESS != (rc = orte_create_dir(fulldirpath) ) ) {
ORTE_ERROR_LOG(rc);
goto cleanup;
}
}
/*
* if we are not creating, then just verify that the path is OK
*/
else {
if( ORTE_SUCCESS != (rc = opal_os_dirpath_access(fulldirpath, 0) )) {
/* it is okay for the path not to be found - don't error
* log that case, but do error log others
*/
if (ORTE_ERR_NOT_FOUND != rc) {
ORTE_ERROR_LOG(rc);
}
goto cleanup;
}
}
/*
* If we are creating the directory tree, then force overwrite of the
* global structure fields
*/
if (create) {
if (NULL != orte_process_info.tmpdir_base) {
free(orte_process_info.tmpdir_base);
orte_process_info.tmpdir_base = NULL;
}
if (NULL != orte_process_info.top_session_dir) {
free(orte_process_info.top_session_dir);
orte_process_info.top_session_dir = NULL;
}
}
/*
* Update some of the global structures if they are empty
*/
if (NULL == orte_process_info.tmpdir_base) {
orte_process_info.tmpdir_base = strdup(local_prefix);
}
if (NULL == orte_process_info.top_session_dir &&
NULL != frontend) {
orte_process_info.top_session_dir = strdup(frontend);
}
/*
* Set the process session directory
*/
if (ORTE_VPID_INVALID != proc->vpid) {
if (create) { /* overwrite if creating */
if (NULL != orte_process_info.proc_session_dir) {
free(orte_process_info.proc_session_dir);
orte_process_info.proc_session_dir = NULL;
}
}
if (NULL == orte_process_info.proc_session_dir) {
orte_process_info.proc_session_dir = strdup(fulldirpath);
}
/* Strip off last part of directory structure */
sav = opal_dirname(fulldirpath);
free(fulldirpath);
fulldirpath = sav;
sav = NULL;
}
/*
* Set the job session directory
*/
if (ORTE_JOBID_INVALID != proc->jobid) {
if (create) { /* overwrite if creating */
if (NULL != orte_process_info.job_session_dir) {
free(orte_process_info.job_session_dir);
orte_process_info.job_session_dir = NULL;
}
}
if (NULL == orte_process_info.job_session_dir) {
orte_process_info.job_session_dir = strdup(fulldirpath);
}
}
if (orte_debug_flag) {
opal_output(0, "procdir: %s",
OMPI_PRINTF_FIX_STRING(orte_process_info.proc_session_dir));
opal_output(0, "jobdir: %s",
OMPI_PRINTF_FIX_STRING(orte_process_info.job_session_dir));
opal_output(0, "top: %s",
OMPI_PRINTF_FIX_STRING(orte_process_info.top_session_dir));
opal_output(0, "tmp: %s",
OMPI_PRINTF_FIX_STRING(orte_process_info.tmpdir_base));
}
cleanup:
if (NULL != local_prefix) {
free(local_prefix);
}
if(NULL != fulldirpath) {
free(fulldirpath);
}
if(NULL != frontend) {
free(frontend);
}
return rc;
}
static int initialize(int argc, char *argv[])
{
int ret, exit_status = OPAL_SUCCESS;
char * tmp_env_var = NULL;
/*
* Make sure to init util before parse_args
* to ensure installdirs is setup properly
* before calling mca_base_open();
*/
if( OPAL_SUCCESS != (ret = opal_init_util(&argc, &argv)) ) {
return ret;
}
/*
* Parse Command line arguments
*/
if (OPAL_SUCCESS != (ret = parse_args(argc, argv))) {
exit_status = ret;
goto cleanup;
}
/*
* Setup OPAL Output handle from the verbose argument
*/
if( opal_restart_globals.verbose ) {
opal_restart_globals.output = opal_output_open(NULL);
opal_output_set_verbosity(opal_restart_globals.output, 10);
} else {
opal_restart_globals.output = 0; /* Default=STDOUT */
}
/*
* Turn off the selection of the CRS component,
* we need to do that later
*/
(void) mca_base_var_env_name("crs_base_do_not_select", &tmp_env_var);
opal_setenv(tmp_env_var,
"1", /* turn off the selection */
true, &environ);
free(tmp_env_var);
tmp_env_var = NULL;
/*
* Make sure we select the proper compress component.
*/
if( NULL != opal_restart_globals.snapshot_compress ) {
(void) mca_base_var_env_name("compress", &tmp_env_var);
opal_setenv(tmp_env_var,
opal_restart_globals.snapshot_compress,
true, &environ);
free(tmp_env_var);
tmp_env_var = NULL;
}
/*
* Initialize the OPAL layer
*/
if (OPAL_SUCCESS != (ret = opal_init(&argc, &argv))) {
exit_status = ret;
goto cleanup;
}
/*
* If the checkpoint was compressed, then decompress it before continuing
*/
if( NULL != opal_restart_globals.snapshot_compress ) {
char * zip_dir = NULL;
char * tmp_str = NULL;
/* Make sure to clear the selection for the restart,
* this way the user can swich compression mechanism
* across restart
*/
(void) mca_base_var_env_name("compress", &tmp_env_var);
opal_unsetenv(tmp_env_var, &environ);
free(tmp_env_var);
tmp_env_var = NULL;
opal_asprintf(&zip_dir, "%s/%s%s",
opal_restart_globals.snapshot_loc,
opal_restart_globals.snapshot_ref,
opal_restart_globals.snapshot_compress_postfix);
if (0 > (ret = access(zip_dir, F_OK)) ) {
opal_output(opal_restart_globals.output,
"Error: Unable to access the file [%s]!",
zip_dir);
exit_status = OPAL_ERROR;
goto cleanup;
}
opal_output_verbose(10, opal_restart_globals.output,
"Decompressing (%s)",
zip_dir);
opal_compress.decompress(zip_dir, &tmp_str);
if( NULL != zip_dir ) {
free(zip_dir);
zip_dir = NULL;
}
if( NULL != tmp_str ) {
free(tmp_str);
tmp_str = NULL;
}
}
/*
* If a cache directory has been suggested, see if it exists
*/
if( NULL != opal_restart_globals.snapshot_cache ) {
if(0 == (ret = access(opal_restart_globals.snapshot_cache, F_OK)) ) {
opal_output_verbose(10, opal_restart_globals.output,
"Using the cached snapshot (%s) instead of (%s)",
opal_restart_globals.snapshot_cache,
opal_restart_globals.snapshot_loc);
if( NULL != opal_restart_globals.snapshot_loc ) {
free(opal_restart_globals.snapshot_loc);
opal_restart_globals.snapshot_loc = NULL;
}
opal_restart_globals.snapshot_loc = opal_dirname(opal_restart_globals.snapshot_cache);
} else {
opal_show_help("help-opal-restart.txt", "cache_not_avail", true,
opal_restart_globals.snapshot_cache,
opal_restart_globals.snapshot_loc);
}
}
/*
* Mark this process as a tool
*/
opal_cr_is_tool = true;
cleanup:
return exit_status;
}
static int rte_init(char flags)
{
int ret;
char *error = NULL;
char *contact_path, *jobfam_dir;
orte_job_t *jdata;
orte_node_t *node;
orte_proc_t *proc;
int value;
/* initialize the global list of local children and job data */
OBJ_CONSTRUCT(&orte_local_children, opal_list_t);
OBJ_CONSTRUCT(&orte_local_jobdata, opal_list_t);
/* run the prolog */
if (ORTE_SUCCESS != (ret = orte_ess_base_std_prolog())) {
error = "orte_ess_base_std_prolog";
goto error;
}
/* if we are using xml for output, put a basename start tag */
if (orte_xml_output) {
fprintf(orte_xml_fp, "<%s>\n", orte_cmd_basename);
fflush(orte_xml_fp);
}
/* determine the topology info */
if (0 == orte_default_num_sockets_per_board) {
/* we weren't given a number, so try to determine it */
if (OPAL_SUCCESS != opal_paffinity_base_get_socket_info(&value)) {
/* can't get any info - default to 1 */
value = 1;
}
orte_default_num_sockets_per_board = (uint8_t)value;
}
if (0 == orte_default_num_cores_per_socket) {
/* we weren't given a number, so try to determine it */
if (OPAL_SUCCESS != (ret = opal_paffinity_base_get_core_info(0, &value))) {
/* don't have topo info - can we at least get #processors? */
if (OPAL_SUCCESS != opal_paffinity_base_get_processor_info(&value)) {
/* can't get any info - default to 1 */
value = 1;
}
}
orte_default_num_cores_per_socket = (uint8_t)value;
}
/* Since we are the HNP, then responsibility for
* defining the name falls to the PLM component for our
* respective environment - hence, we have to open the PLM
* first and select that component. Note that ONLY the
* HNP ever uses a PLM component anyway
*/
if (ORTE_SUCCESS != (ret = orte_plm_base_open())) {
ORTE_ERROR_LOG(ret);
error = "orte_plm_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_plm_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_plm_base_select";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_plm.set_hnp_name())) {
ORTE_ERROR_LOG(ret);
error = "orte_plm_set_hnp_name";
goto error;
}
/* Setup the communication infrastructure */
/*
* Runtime Messaging Layer
*/
if (ORTE_SUCCESS != (ret = orte_rml_base_open())) {
ORTE_ERROR_LOG(ret);
error = "orte_rml_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_rml_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_rml_base_select";
goto error;
}
/*
* Routed system
*/
if (ORTE_SUCCESS != (ret = orte_routed_base_open())) {
ORTE_ERROR_LOG(ret);
error = "orte_routed_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_routed_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_routed_base_select";
goto error;
}
/*
* Group communications
*/
if (ORTE_SUCCESS != (ret = orte_grpcomm_base_open())) {
ORTE_ERROR_LOG(ret);
error = "orte_grpcomm_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_grpcomm_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_grpcomm_base_select";
goto error;
}
/* Now provide a chance for the PLM
* to perform any module-specific init functions. This
* needs to occur AFTER the communications are setup
* as it may involve starting a non-blocking recv
*/
if (ORTE_SUCCESS != (ret = orte_plm.init())) {
ORTE_ERROR_LOG(ret);
error = "orte_plm_init";
goto error;
}
/*
* Setup the remaining resource
* management and errmgr frameworks - application procs
* and daemons do not open these frameworks as they only use
* the hnp proxy support in the PLM framework.
*/
if (ORTE_SUCCESS != (ret = orte_ras_base_open())) {
ORTE_ERROR_LOG(ret);
error = "orte_ras_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_ras_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_ras_base_find_available";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_rmaps_base_open())) {
ORTE_ERROR_LOG(ret);
error = "orte_rmaps_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_rmaps_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_rmaps_base_find_available";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_errmgr_base_open())) {
error = "orte_errmgr_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_errmgr_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_errmgr_base_select";
goto error;
}
/* Open/select the odls */
if (ORTE_SUCCESS != (ret = orte_odls_base_open())) {
ORTE_ERROR_LOG(ret);
error = "orte_odls_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_odls_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_odls_base_select";
goto error;
}
/* enable communication with the rml */
if (ORTE_SUCCESS != (ret = orte_rml.enable_comm())) {
ORTE_ERROR_LOG(ret);
error = "orte_rml.enable_comm";
goto error;
}
#if !ORTE_DISABLE_FULL_SUPPORT
/* setup the orte_show_help system to recv remote output */
ret = orte_rml.recv_buffer_nb(ORTE_NAME_WILDCARD, ORTE_RML_TAG_SHOW_HELP,
ORTE_RML_NON_PERSISTENT, orte_show_help_recv, NULL);
if (ret != ORTE_SUCCESS && ret != ORTE_ERR_NOT_IMPLEMENTED) {
ORTE_ERROR_LOG(ret);
error = "setup receive for orte_show_help";
goto error;
}
#endif
/* setup my session directory */
OPAL_OUTPUT_VERBOSE((2, orte_debug_output,
"%s setting up session dir with\n\ttmpdir: %s\n\thost %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
(NULL == orte_process_info.tmpdir_base) ? "UNDEF" : orte_process_info.tmpdir_base,
orte_process_info.nodename));
if (ORTE_SUCCESS != (ret = orte_session_dir(true,
orte_process_info.tmpdir_base,
orte_process_info.nodename, NULL,
ORTE_PROC_MY_NAME))) {
ORTE_ERROR_LOG(ret);
error = "orte_session_dir";
goto error;
}
/* Once the session directory location has been established, set
the opal_output hnp file location to be in the
proc-specific session directory. */
opal_output_set_output_file_info(orte_process_info.proc_session_dir,
"output-", NULL, NULL);
/* save my contact info in a file for others to find */
jobfam_dir = opal_dirname(orte_process_info.job_session_dir);
contact_path = opal_os_path(false, jobfam_dir, "contact.txt", NULL);
free(jobfam_dir);
OPAL_OUTPUT_VERBOSE((2, orte_debug_output,
"%s writing contact file %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
contact_path));
if (ORTE_SUCCESS != (ret = orte_write_hnp_contact_file(contact_path))) {
OPAL_OUTPUT_VERBOSE((2, orte_debug_output,
"%s writing contact file failed with error %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_ERROR_NAME(ret)));
} else {
OPAL_OUTPUT_VERBOSE((2, orte_debug_output,
"%s wrote contact file",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
}
free(contact_path);
/* Setup the job data object for the daemons */
/* create and store the job data object */
jdata = OBJ_NEW(orte_job_t);
jdata->jobid = ORTE_PROC_MY_NAME->jobid;
opal_pointer_array_add(orte_job_data, jdata);
/* create and store a node object where we are */
node = OBJ_NEW(orte_node_t);
node->name = strdup(orte_process_info.nodename);
node->arch = orte_process_info.arch;
node->index = opal_pointer_array_add(orte_node_pool, node);
/* create and store a proc object for us */
proc = OBJ_NEW(orte_proc_t);
proc->name.jobid = ORTE_PROC_MY_NAME->jobid;
proc->name.vpid = ORTE_PROC_MY_NAME->vpid;
proc->pid = orte_process_info.pid;
proc->rml_uri = orte_rml.get_contact_info();
proc->state = ORTE_PROC_STATE_RUNNING;
OBJ_RETAIN(node); /* keep accounting straight */
proc->node = node;
proc->nodename = node->name;
opal_pointer_array_add(jdata->procs, proc);
/* record that the daemon (i.e., us) is on this node
* NOTE: we do not add the proc object to the node's
* proc array because we are not an application proc.
* Instead, we record it in the daemon field of the
* node object
*/
OBJ_RETAIN(proc); /* keep accounting straight */
node->daemon = proc;
node->daemon_launched = true;
node->state = ORTE_NODE_STATE_UP;
/* record that the daemon job is running */
jdata->num_procs = 1;
jdata->state = ORTE_JOB_STATE_RUNNING;
/* setup the routed info - the selected routed component
* will know what to do.
*/
if (ORTE_SUCCESS != (ret = orte_routed.init_routes(ORTE_PROC_MY_NAME->jobid, NULL))) {
ORTE_ERROR_LOG(ret);
error = "orte_routed.init_routes";
goto error;
}
/* setup I/O forwarding system - must come after we init routes */
if (ORTE_SUCCESS != (ret = orte_iof_base_open())) {
ORTE_ERROR_LOG(ret);
error = "orte_iof_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_iof_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_iof_base_select";
goto error;
}
/* setup the FileM */
if (ORTE_SUCCESS != (ret = orte_filem_base_open())) {
ORTE_ERROR_LOG(ret);
error = "orte_filem_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_filem_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_filem_base_select";
goto error;
}
#if OPAL_ENABLE_FT == 1
/*
* Setup the SnapC
*/
if (ORTE_SUCCESS != (ret = orte_snapc_base_open())) {
ORTE_ERROR_LOG(ret);
error = "orte_snapc_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_snapc_base_select(orte_process_info.hnp, !orte_process_info.daemon))) {
ORTE_ERROR_LOG(ret);
error = "orte_snapc_base_select";
goto error;
}
/* For HNP, ORTE doesn't need the OPAL CR stuff */
opal_cr_set_enabled(false);
#else
opal_cr_set_enabled(false);
#endif
/*
* Initalize the CR setup
* Note: Always do this, even in non-FT builds.
* If we don't some user level tools may hang.
*/
if (ORTE_SUCCESS != (ret = orte_cr_init())) {
ORTE_ERROR_LOG(ret);
error = "orte_cr_init";
goto error;
}
/* setup the notifier system */
if (ORTE_SUCCESS != (ret = orte_notifier_base_open())) {
ORTE_ERROR_LOG(ret);
error = "orte_notifer_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_notifier_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_notifer_select";
goto error;
}
return ORTE_SUCCESS;
error:
if (ORTE_ERR_SILENT != ret) {
orte_show_help("help-orte-runtime.txt",
"orte_init:startup:internal-failure",
true, error, ORTE_ERROR_NAME(ret), ret);
}
/* cleanup the global list of local children and job data */
OBJ_DESTRUCT(&orte_local_children);
OBJ_DESTRUCT(&orte_local_jobdata);
return ret;
}
/*
* Construct the session directory and create it if necessary
*/
int orte_session_dir(bool create,
char *prefix, char *usr, char *hostid,
char *batchid, char *univ, char *job, char *proc)
{
char *fulldirpath = NULL,
*frontend = NULL,
*sav = NULL;
int return_code = ORTE_SUCCESS, rtn;
/* This indicates if the prefix was set, and so if it fails then we
* should try with the default prefixes.*/
bool dbl_check_prefix = false;
if( NULL != prefix)
dbl_check_prefix = true;
try_again:
/*
* If the first attempt at the path creation failed, try with a null
* prefix. unless the original prefix was null, then we fail.
*/
if(!dbl_check_prefix && /* an indicator that we are trying a second time */
NULL != prefix) {
free(prefix);
prefix = NULL;
}
/*
* Get the session directory full name
* First try it with the specified prefix.
*/
if( ORTE_SUCCESS != ( rtn = orte_session_dir_get_name(&fulldirpath,
&prefix,
&frontend,
usr, hostid,
batchid, univ, job,
proc) ) ) {
return_code = rtn;
/*
* If the first attempt at the path creation failed, try with a null
* prefix. unless the original prefix was null, then we fail :(
*/
if(dbl_check_prefix) {
dbl_check_prefix = false;
goto try_again;
}
else {
goto cleanup;
}
}
/*
* Now that we have the full path, go ahead and create it if necessary
*/
if( create ) {
if( ORTE_SUCCESS != (rtn = orte_create_dir(fulldirpath) ) ) {
return_code = rtn;
if(dbl_check_prefix) {
dbl_check_prefix = false;
goto try_again;
}
else {
goto cleanup;
}
}
}
/*
* if we are not creating, then just verify that the path is OK
*/
else {
if( ORTE_SUCCESS != (rtn = opal_os_dirpath_access(fulldirpath, 0) )) {
/* It is not valid so we give up and return an error */
return_code = rtn;
if(dbl_check_prefix) {
dbl_check_prefix = false;
goto try_again;
}
else {
goto cleanup;
}
}
}
return_code = ORTE_SUCCESS;
/*
* If we are creating the directory tree, the overwrite the
* global structure fields
*/
if (create) {
if (NULL != orte_process_info.tmpdir_base) {
free(orte_process_info.tmpdir_base);
orte_process_info.tmpdir_base = NULL;
}
if (NULL != orte_process_info.top_session_dir) {
free(orte_process_info.top_session_dir);
orte_process_info.top_session_dir = NULL;
}
}
/*
* Update some of the global structures if they are empty
*/
if (NULL == orte_process_info.tmpdir_base)
orte_process_info.tmpdir_base = strdup(prefix);
if (NULL == orte_process_info.top_session_dir)
orte_process_info.top_session_dir = strdup(frontend);
/*
* Set the process session directory
*/
if (NULL != proc) {
if (create) { /* overwrite if creating */
if (NULL != orte_process_info.proc_session_dir) {
free(orte_process_info.proc_session_dir);
orte_process_info.proc_session_dir = NULL;
}
}
if (NULL == orte_process_info.proc_session_dir) {
orte_process_info.proc_session_dir = strdup(fulldirpath);
}
/* Strip off last part of directory structure */
sav = opal_dirname(fulldirpath);
free(fulldirpath);
fulldirpath = sav;
sav = NULL;
}
/*
* Set the job session directory
*/
if (NULL != job) {
if (create) { /* overwrite if creating */
if (NULL != orte_process_info.job_session_dir) {
free(orte_process_info.job_session_dir);
orte_process_info.job_session_dir = NULL;
}
}
if (NULL == orte_process_info.job_session_dir) {
orte_process_info.job_session_dir = strdup(fulldirpath);
}
/* Strip off last part of directory structure */
sav = opal_dirname(fulldirpath);
free(fulldirpath);
fulldirpath = sav;
sav = NULL;
}
/*
* Set the universe session directory
*/
if (create) { /* overwrite if creating */
if (NULL != orte_process_info.universe_session_dir) {
free(orte_process_info.universe_session_dir);
orte_process_info.universe_session_dir = NULL;
}
}
if (NULL == orte_process_info.universe_session_dir) {
orte_process_info.universe_session_dir = strdup(fulldirpath);
}
if (orte_debug_flag) {
opal_output(0, "procdir: %s",
OMPI_PRINTF_FIX_STRING(orte_process_info.proc_session_dir));
opal_output(0, "jobdir: %s",
OMPI_PRINTF_FIX_STRING(orte_process_info.job_session_dir));
opal_output(0, "unidir: %s",
OMPI_PRINTF_FIX_STRING(orte_process_info.universe_session_dir));
opal_output(0, "top: %s",
OMPI_PRINTF_FIX_STRING(orte_process_info.top_session_dir));
opal_output(0, "tmp: %s",
OMPI_PRINTF_FIX_STRING(orte_process_info.tmpdir_base));
}
cleanup:
if(NULL != fulldirpath)
free(fulldirpath);
if(NULL != frontend)
free(frontend);
if(NULL != sav)
free(sav);
return return_code;
}
static int rte_init(void)
{
int ret;
char *error = NULL;
char *contact_path, *jobfam_dir;
orte_job_t *jdata;
orte_node_t *node;
orte_proc_t *proc;
int value;
/* initialize the global list of local children and job data */
OBJ_CONSTRUCT(&orte_local_children, opal_list_t);
OBJ_CONSTRUCT(&orte_local_jobdata, opal_list_t);
/* run the prolog */
if (ORTE_SUCCESS != (ret = orte_ess_base_std_prolog())) {
error = "orte_ess_base_std_prolog";
goto error;
}
/* determine the topology info */
if (0 == orte_default_num_sockets_per_board) {
/* we weren't given a number, so try to determine it */
if (OPAL_SUCCESS != opal_paffinity_base_get_socket_info(&value)) {
/* can't get any info - default to 1 */
value = 1;
}
orte_default_num_sockets_per_board = (uint8_t)value;
}
if (0 == orte_default_num_cores_per_socket) {
/* we weren't given a number, so try to determine it */
if (OPAL_SUCCESS != (ret = opal_paffinity_base_get_core_info(0, &value))) {
/* don't have topo info - can we at least get #processors? */
if (OPAL_SUCCESS != opal_paffinity_base_get_processor_info(&value)) {
/* can't get any info - default to 1 */
value = 1;
}
}
orte_default_num_cores_per_socket = (uint8_t)value;
}
/* if we are using xml for output, put an mpirun start tag */
if (orte_xml_output) {
fprintf(orte_xml_fp, "<mpirun>\n");
fflush(orte_xml_fp);
}
/* open and setup the opal_pstat framework so we can provide
* process stats if requested
*/
if (ORTE_SUCCESS != (ret = opal_pstat_base_open())) {
ORTE_ERROR_LOG(ret);
error = "opal_pstat_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = opal_pstat_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_pstat_base_select";
goto error;
}
/* open and setup the local resource discovery framework */
if (ORTE_SUCCESS != (ret = opal_sysinfo_base_open())) {
ORTE_ERROR_LOG(ret);
error = "opal_sysinfo_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = opal_sysinfo_base_select())) {
ORTE_ERROR_LOG(ret);
error = "opal_sysinfo_base_select";
goto error;
}
/* Since we are the HNP, then responsibility for
* defining the name falls to the PLM component for our
* respective environment - hence, we have to open the PLM
* first and select that component.
*/
if (ORTE_SUCCESS != (ret = orte_plm_base_open())) {
ORTE_ERROR_LOG(ret);
error = "orte_plm_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_plm_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_plm_base_select";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_plm.set_hnp_name())) {
ORTE_ERROR_LOG(ret);
error = "orte_plm_set_hnp_name";
goto error;
}
/* Setup the communication infrastructure */
/*
* Runtime Messaging Layer
*/
if (ORTE_SUCCESS != (ret = orte_rml_base_open())) {
ORTE_ERROR_LOG(ret);
error = "orte_rml_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_rml_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_rml_base_select";
goto error;
}
/*
* Routed system
*/
if (ORTE_SUCCESS != (ret = orte_routed_base_open())) {
ORTE_ERROR_LOG(ret);
error = "orte_routed_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_routed_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_routed_base_select";
goto error;
}
/*
* Group communications
*/
if (ORTE_SUCCESS != (ret = orte_grpcomm_base_open())) {
ORTE_ERROR_LOG(ret);
error = "orte_grpcomm_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_grpcomm_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_grpcomm_base_select";
goto error;
}
/* multicast */
#if ORTE_ENABLE_MULTICAST
if (ORTE_SUCCESS != (ret = orte_rmcast_base_open())) {
ORTE_ERROR_LOG(ret);
error = "orte_rmcast_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_rmcast_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_rmcast_base_select";
goto error;
}
#endif
/* Now provide a chance for the PLM
* to perform any module-specific init functions. This
* needs to occur AFTER the communications are setup
* as it may involve starting a non-blocking recv
*/
if (ORTE_SUCCESS != (ret = orte_plm.init())) {
ORTE_ERROR_LOG(ret);
error = "orte_plm_init";
goto error;
}
/*
* Setup the remaining resource
* management and errmgr frameworks - application procs
* and daemons do not open these frameworks as they only use
* the hnp proxy support in the PLM framework.
*/
if (ORTE_SUCCESS != (ret = orte_ras_base_open())) {
ORTE_ERROR_LOG(ret);
error = "orte_ras_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_ras_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_ras_base_find_available";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_rmaps_base_open())) {
ORTE_ERROR_LOG(ret);
error = "orte_rmaps_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_rmaps_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_rmaps_base_find_available";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_errmgr_base_open())) {
error = "orte_errmgr_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_errmgr_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_errmgr_base_select";
goto error;
}
/* Open/select the odls */
if (ORTE_SUCCESS != (ret = orte_odls_base_open())) {
ORTE_ERROR_LOG(ret);
error = "orte_odls_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_odls_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_odls_base_select";
goto error;
}
/* enable communication with the rml */
if (ORTE_SUCCESS != (ret = orte_rml.enable_comm())) {
ORTE_ERROR_LOG(ret);
error = "orte_rml.enable_comm";
goto error;
}
/* we are an hnp, so update the contact info field for later use */
orte_process_info.my_hnp_uri = orte_rml.get_contact_info();
/* we are also officially a daemon, so better update that field too */
orte_process_info.my_daemon_uri = orte_rml.get_contact_info();
#if !ORTE_DISABLE_FULL_SUPPORT
/* setup the orte_show_help system to recv remote output */
ret = orte_rml.recv_buffer_nb(ORTE_NAME_WILDCARD, ORTE_RML_TAG_SHOW_HELP,
ORTE_RML_NON_PERSISTENT, orte_show_help_recv, NULL);
if (ret != ORTE_SUCCESS && ret != ORTE_ERR_NOT_IMPLEMENTED) {
ORTE_ERROR_LOG(ret);
error = "setup receive for orte_show_help";
goto error;
}
#endif
/* setup my session directory */
OPAL_OUTPUT_VERBOSE((2, orte_debug_output,
"%s setting up session dir with\n\ttmpdir: %s\n\thost %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
(NULL == orte_process_info.tmpdir_base) ? "UNDEF" : orte_process_info.tmpdir_base,
orte_process_info.nodename));
if (ORTE_SUCCESS != (ret = orte_session_dir(true,
orte_process_info.tmpdir_base,
orte_process_info.nodename, NULL,
ORTE_PROC_MY_NAME))) {
ORTE_ERROR_LOG(ret);
error = "orte_session_dir";
goto error;
}
/* Once the session directory location has been established, set
the opal_output hnp file location to be in the
proc-specific session directory. */
opal_output_set_output_file_info(orte_process_info.proc_session_dir,
"output-", NULL, NULL);
/* save my contact info in a file for others to find */
jobfam_dir = opal_dirname(orte_process_info.job_session_dir);
contact_path = opal_os_path(false, jobfam_dir, "contact.txt", NULL);
free(jobfam_dir);
OPAL_OUTPUT_VERBOSE((2, orte_debug_output,
"%s writing contact file %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
contact_path));
if (ORTE_SUCCESS != (ret = orte_write_hnp_contact_file(contact_path))) {
OPAL_OUTPUT_VERBOSE((2, orte_debug_output,
"%s writing contact file failed with error %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_ERROR_NAME(ret)));
} else {
OPAL_OUTPUT_VERBOSE((2, orte_debug_output,
"%s wrote contact file",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
}
free(contact_path);
/* setup the global job and node arrays */
orte_job_data = OBJ_NEW(opal_pointer_array_t);
if (ORTE_SUCCESS != (ret = opal_pointer_array_init(orte_job_data,
1,
ORTE_GLOBAL_ARRAY_MAX_SIZE,
1))) {
ORTE_ERROR_LOG(ret);
error = "setup job array";
goto error;
}
orte_node_pool = OBJ_NEW(opal_pointer_array_t);
if (ORTE_SUCCESS != (ret = opal_pointer_array_init(orte_node_pool,
ORTE_GLOBAL_ARRAY_BLOCK_SIZE,
ORTE_GLOBAL_ARRAY_MAX_SIZE,
ORTE_GLOBAL_ARRAY_BLOCK_SIZE))) {
ORTE_ERROR_LOG(ret);
error = "setup node array";
goto error;
}
/* Setup the job data object for the daemons */
/* create and store the job data object */
jdata = OBJ_NEW(orte_job_t);
jdata->jobid = ORTE_PROC_MY_NAME->jobid;
opal_pointer_array_set_item(orte_job_data, 0, jdata);
/* create and store a node object where we are */
node = OBJ_NEW(orte_node_t);
node->name = strdup(orte_process_info.nodename);
node->index = opal_pointer_array_add(orte_node_pool, node);
/* create and store a proc object for us */
proc = OBJ_NEW(orte_proc_t);
proc->name.jobid = ORTE_PROC_MY_NAME->jobid;
proc->name.vpid = ORTE_PROC_MY_NAME->vpid;
proc->pid = orte_process_info.pid;
proc->rml_uri = orte_rml.get_contact_info();
proc->state = ORTE_PROC_STATE_RUNNING;
OBJ_RETAIN(node); /* keep accounting straight */
proc->node = node;
proc->nodename = node->name;
opal_pointer_array_add(jdata->procs, proc);
/* record that the daemon (i.e., us) is on this node
* NOTE: we do not add the proc object to the node's
* proc array because we are not an application proc.
* Instead, we record it in the daemon field of the
* node object
*/
OBJ_RETAIN(proc); /* keep accounting straight */
node->daemon = proc;
node->daemon_launched = true;
node->state = ORTE_NODE_STATE_UP;
/* record that the daemon job is running */
jdata->num_procs = 1;
jdata->state = ORTE_JOB_STATE_RUNNING;
/* setup the routed info - the selected routed component
* will know what to do.
*/
if (ORTE_SUCCESS != (ret = orte_routed.init_routes(ORTE_PROC_MY_NAME->jobid, NULL))) {
ORTE_ERROR_LOG(ret);
error = "orte_routed.init_routes";
goto error;
}
/* setup I/O forwarding system - must come after we init routes */
if (ORTE_SUCCESS != (ret = orte_iof_base_open())) {
ORTE_ERROR_LOG(ret);
error = "orte_iof_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_iof_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_iof_base_select";
goto error;
}
/* setup the FileM */
if (ORTE_SUCCESS != (ret = orte_filem_base_open())) {
ORTE_ERROR_LOG(ret);
error = "orte_filem_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_filem_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_filem_base_select";
goto error;
}
#if OPAL_ENABLE_FT_CR == 1
/*
* Setup the SnapC
*/
if (ORTE_SUCCESS != (ret = orte_snapc_base_open())) {
ORTE_ERROR_LOG(ret);
error = "orte_snapc_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_snapc_base_select(ORTE_PROC_IS_HNP, !ORTE_PROC_IS_DAEMON))) {
ORTE_ERROR_LOG(ret);
error = "orte_snapc_base_select";
goto error;
}
/* For HNP, ORTE doesn't need the OPAL CR stuff */
opal_cr_set_enabled(false);
#else
opal_cr_set_enabled(false);
#endif
/*
* Initalize the CR setup
* Note: Always do this, even in non-FT builds.
* If we don't some user level tools may hang.
*/
if (ORTE_SUCCESS != (ret = orte_cr_init())) {
ORTE_ERROR_LOG(ret);
error = "orte_cr_init";
goto error;
}
/* setup the notifier system */
if (ORTE_SUCCESS != (ret = orte_notifier_base_open())) {
ORTE_ERROR_LOG(ret);
error = "orte_notifer_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_notifier_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_notifer_select";
goto error;
}
/* if a tool has launched us and is requesting event reports,
* then set its contact info into the comm system
*/
if (orte_report_events) {
if (ORTE_SUCCESS != (ret = orte_util_comm_connect_tool(orte_report_events_uri))) {
error = "could not connect to tool";
goto error;
}
}
/* We actually do *not* want an HNP to voluntarily yield() the
processor more than necessary. Orterun already blocks when
it is doing nothing, so it doesn't use any more CPU cycles than
it should; but when it *is* doing something, we do not want it
to be unnecessarily delayed because it voluntarily yielded the
processor in the middle of its work.
For example: when a message arrives at orterun, we want the
OS to wake us up in a timely fashion (which most OS's
seem good about doing) and then we want orterun to process
the message as fast as possible. If orterun yields and lets
aggressive MPI applications get the processor back, it may be a
long time before the OS schedules orterun to run again
(particularly if there is no IO event to wake it up). Hence,
routed OOB messages (for example) may be significantly delayed
before being delivered to MPI processes, which can be
problematic in some scenarios (e.g., COMM_SPAWN, BTL's that
require OOB messages for wireup, etc.). */
opal_progress_set_yield_when_idle(false);
return ORTE_SUCCESS;
error:
if (ORTE_ERR_SILENT != ret) {
orte_show_help("help-orte-runtime.txt",
"orte_init:startup:internal-failure",
true, error, ORTE_ERROR_NAME(ret), ret);
}
/* cleanup the global list of local children and job data */
OBJ_DESTRUCT(&orte_local_children);
OBJ_DESTRUCT(&orte_local_jobdata);
return ret;
}
/*
* Construct the session directory and create it if necessary
*/
int orte_session_dir(bool create,
char *prefix, char *hostid,
char *batchid, orte_process_name_t *proc)
{
char *fulldirpath = NULL,
*frontend = NULL,
*sav = NULL;
int rc = ORTE_SUCCESS;
char *local_prefix = NULL;
/* use the specified prefix, if one was given */
if (NULL != prefix) {
local_prefix = strdup(prefix);
}
/*
* Get the session directory full name
*/
if (ORTE_SUCCESS != (rc = orte_session_dir_get_name(&fulldirpath,
&local_prefix,
&frontend,
hostid,
batchid, proc))) {
if (ORTE_ERR_FATAL == rc) {
/* this indicates we should abort quietly */
rc = ORTE_ERR_SILENT;
goto cleanup;
}
/* otherwise, bark a little first */
ORTE_ERROR_LOG(rc);
goto cleanup;
}
/*
* Now that we have the full path, go ahead and create it if necessary
*/
if( create ) {
if( ORTE_SUCCESS != (rc = orte_create_dir(fulldirpath) ) ) {
ORTE_ERROR_LOG(rc);
goto cleanup;
}
}
/* update global structure fields */
if (NULL != orte_process_info.tmpdir_base) {
free(orte_process_info.tmpdir_base);
}
orte_process_info.tmpdir_base = strdup(local_prefix);
if (NULL != orte_process_info.top_session_dir) {
free(orte_process_info.top_session_dir);
orte_process_info.top_session_dir = NULL;
}
if (NULL != frontend) {
orte_process_info.top_session_dir = strdup(frontend);
}
/*
* Set the process session directory
*/
if (ORTE_VPID_INVALID != proc->vpid) {
if (NULL != orte_process_info.proc_session_dir) {
free(orte_process_info.proc_session_dir);
}
orte_process_info.proc_session_dir = strdup(fulldirpath);
/* Strip off last part of directory structure */
sav = opal_dirname(fulldirpath);
free(fulldirpath);
fulldirpath = sav;
sav = NULL;
}
/*
* Set the job session directory
*/
if (ORTE_JOBID_INVALID != proc->jobid) {
if (NULL != orte_process_info.job_session_dir) {
free(orte_process_info.job_session_dir);
}
orte_process_info.job_session_dir = strdup(fulldirpath);
}
if (orte_debug_flag) {
opal_output(0, "procdir: %s",
OMPI_PRINTF_FIX_STRING(orte_process_info.proc_session_dir));
opal_output(0, "jobdir: %s",
OMPI_PRINTF_FIX_STRING(orte_process_info.job_session_dir));
opal_output(0, "top: %s",
OMPI_PRINTF_FIX_STRING(orte_process_info.top_session_dir));
opal_output(0, "tmp: %s",
OMPI_PRINTF_FIX_STRING(orte_process_info.tmpdir_base));
}
cleanup:
if (NULL != local_prefix) {
free(local_prefix);
}
if(NULL != fulldirpath) {
free(fulldirpath);
}
if(NULL != frontend) {
free(frontend);
}
return rc;
}
