static int setup_child(orte_job_t *jdata,
orte_proc_t *child,
orte_app_context_t *app)
{
char *param, *value;
int rc, i;
int32_t nrestarts=0, *nrptr;
bool takeus = false;
/* see if we are included */
for (i=0; NULL != jdata->personality[i]; i++) {
if (0 == strcmp(jdata->personality[i], "ompi")) {
takeus = true;
break;
}
}
if (!takeus) {
return ORTE_ERR_TAKE_NEXT_OPTION;
}
/* setup the jobid */
if (ORTE_SUCCESS != (rc = orte_util_convert_jobid_to_string(&value, child->name.jobid))) {
ORTE_ERROR_LOG(rc);
return rc;
}
opal_setenv("OMPI_MCA_ess_base_jobid", value, true, &app->env);
free(value);
/* setup the vpid */
if (ORTE_SUCCESS != (rc = orte_util_convert_vpid_to_string(&value, child->name.vpid))) {
ORTE_ERROR_LOG(rc);
return rc;
}
opal_setenv("OMPI_MCA_ess_base_vpid", value, true, &app->env);
/* although the vpid IS the process' rank within the job, users
* would appreciate being given a public environmental variable
* that also represents this value - something MPI specific - so
* do that here.
*
* AND YES - THIS BREAKS THE ABSTRACTION BARRIER TO SOME EXTENT.
* We know - just live with it
*/
opal_setenv("OMPI_COMM_WORLD_RANK", value, true, &app->env);
free(value); /* done with this now */
/* users would appreciate being given a public environmental variable
* that also represents the local rank value - something MPI specific - so
* do that here.
*
* AND YES - THIS BREAKS THE ABSTRACTION BARRIER TO SOME EXTENT.
* We know - just live with it
*/
if (ORTE_LOCAL_RANK_INVALID == child->local_rank) {
ORTE_ERROR_LOG(ORTE_ERR_VALUE_OUT_OF_BOUNDS);
rc = ORTE_ERR_VALUE_OUT_OF_BOUNDS;
return rc;
}
asprintf(&value, "%lu", (unsigned long) child->local_rank);
opal_setenv("OMPI_COMM_WORLD_LOCAL_RANK", value, true, &app->env);
free(value);
/* users would appreciate being given a public environmental variable
* that also represents the node rank value - something MPI specific - so
* do that here.
*
* AND YES - THIS BREAKS THE ABSTRACTION BARRIER TO SOME EXTENT.
* We know - just live with it
*/
if (ORTE_NODE_RANK_INVALID == child->node_rank) {
ORTE_ERROR_LOG(ORTE_ERR_VALUE_OUT_OF_BOUNDS);
rc = ORTE_ERR_VALUE_OUT_OF_BOUNDS;
return rc;
}
asprintf(&value, "%lu", (unsigned long) child->node_rank);
opal_setenv("OMPI_COMM_WORLD_NODE_RANK", value, true, &app->env);
/* set an mca param for it too */
opal_setenv("OMPI_MCA_orte_ess_node_rank", value, true, &app->env);
free(value);
/* provide the identifier for the PMIx connection - the
* PMIx connection is made prior to setting the process
* name itself. Although in most cases the ID and the
* process name are the same, it isn't necessarily
* required */
orte_util_convert_process_name_to_string(&value, &child->name);
opal_setenv("PMIX_ID", value, true, &app->env);
free(value);
nrptr = &nrestarts;
if (orte_get_attribute(&child->attributes, ORTE_PROC_NRESTARTS, (void**)&nrptr, OPAL_INT32)) {
/* pass the number of restarts for this proc - will be zero for
* an initial start, but procs would like to know if they are being
* restarted so they can take appropriate action
*/
asprintf(&value, "%d", nrestarts);
opal_setenv("OMPI_MCA_orte_num_restarts", value, true, &app->env);
free(value);
}
/* if the proc should not barrier in orte_init, tell it */
if (orte_get_attribute(&child->attributes, ORTE_PROC_NOBARRIER, NULL, OPAL_BOOL)
|| 0 < nrestarts) {
opal_setenv("OMPI_MCA_orte_do_not_barrier", "1", true, &app->env);
}
/* if we are using staged execution, tell it */
if (orte_staged_execution) {
opal_setenv("OMPI_MCA_orte_staged_execution", "1", true, &app->env);
}
/* if the proc isn't going to forward IO, then we need to flag that
* it has "completed" iof termination as otherwise it will never fire
*/
if (!ORTE_FLAG_TEST(jdata, ORTE_JOB_FLAG_FORWARD_OUTPUT)) {
ORTE_FLAG_SET(child, ORTE_PROC_FLAG_IOF_COMPLETE);
}
/* construct the proc's session dir name */
if (NULL != orte_process_info.tmpdir_base) {
value = strdup(orte_process_info.tmpdir_base);
} else {
value = NULL;
}
param = NULL;
if (ORTE_SUCCESS != (rc = orte_session_dir_get_name(¶m, &value, NULL,
orte_process_info.nodename,
NULL, &child->name))) {
ORTE_ERROR_LOG(rc);
if (NULL != value) {
free(value);
}
return rc;
}
free(value);
/* pass an envar so the proc can find any files it had prepositioned */
opal_setenv("OMPI_FILE_LOCATION", param, true, &app->env);
/* if the user wanted the cwd to be the proc's session dir, then
* switch to that location now
*/
if (orte_get_attribute(&app->attributes, ORTE_APP_SSNDIR_CWD, NULL, OPAL_BOOL)) {
/* create the session dir - may not exist */
if (OPAL_SUCCESS != (rc = opal_os_dirpath_create(param, S_IRWXU))) {
ORTE_ERROR_LOG(rc);
/* doesn't exist with correct permissions, and/or we can't
* create it - either way, we are done
*/
free(param);
return rc;
}
/* change to it */
if (0 != chdir(param)) {
free(param);
return ORTE_ERROR;
}
/* It seems that chdir doesn't
* adjust the $PWD enviro variable when it changes the directory. This
* can cause a user to get a different response when doing getcwd vs
* looking at the enviro variable. To keep this consistent, we explicitly
* ensure that the PWD enviro variable matches the CWD we moved to.
*
* NOTE: if a user's program does a chdir(), then $PWD will once
* again not match getcwd! This is beyond our control - we are only
* ensuring they start out matching.
*/
opal_setenv("PWD", param, true, &app->env);
/* update the initial wdir value too */
opal_setenv("OMPI_MCA_initial_wdir", param, true, &app->env);
}
free(param);
return ORTE_SUCCESS;
}
static void launch_daemons(int fd, short args, void *cbdata)
{
orte_job_map_t *map;
char *jobid_string = NULL;
char *param;
char **argv = NULL;
int argc;
int rc;
char *tmp;
char** env = NULL;
char *nodelist_flat;
char **nodelist_argv;
int nodelist_argc;
char *vpid_string;
char **custom_strings;
int num_args, i;
char *cur_prefix;
int proc_vpid_index;
orte_app_context_t *app;
orte_node_t *node;
orte_std_cntr_t nnode;
orte_job_t *daemons;
orte_state_caddy_t *state = (orte_state_caddy_t*)cbdata;
/* if we are launching debugger daemons, then just go
* do it - no new daemons will be launched
*/
if (ORTE_FLAG_TEST(state->jdata, ORTE_JOB_FLAG_DEBUGGER_DAEMON)) {
state->jdata->state = ORTE_JOB_STATE_DAEMONS_LAUNCHED;
ORTE_ACTIVATE_JOB_STATE(state->jdata, ORTE_JOB_STATE_DAEMONS_REPORTED);
OBJ_RELEASE(state);
return;
}
/* start by setting up the virtual machine */
daemons = orte_get_job_data_object(ORTE_PROC_MY_NAME->jobid);
if (ORTE_SUCCESS != (rc = orte_plm_base_setup_virtual_machine(state->jdata))) {
ORTE_ERROR_LOG(rc);
goto cleanup;
}
/* if we don't want to launch, then don't attempt to
* launch the daemons - the user really wants to just
* look at the proposed process map
*/
if (orte_do_not_launch) {
/* set the state to indicate the daemons reported - this
* will trigger the daemons_reported event and cause the
* job to move to the following step
*/
state->jdata->state = ORTE_JOB_STATE_DAEMONS_LAUNCHED;
ORTE_ACTIVATE_JOB_STATE(state->jdata, ORTE_JOB_STATE_DAEMONS_REPORTED);
OBJ_RELEASE(state);
return;
}
/* Get the map for this job */
if (NULL == (map = daemons->map)) {
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
rc = ORTE_ERR_NOT_FOUND;
goto cleanup;
}
if (0 == map->num_new_daemons) {
/* set the state to indicate the daemons reported - this
* will trigger the daemons_reported event and cause the
* job to move to the following step
*/
OPAL_OUTPUT_VERBOSE((1, orte_plm_base_framework.framework_output,
"%s plm:alps: no new daemons to launch",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
state->jdata->state = ORTE_JOB_STATE_DAEMONS_LAUNCHED;
ORTE_ACTIVATE_JOB_STATE(state->jdata, ORTE_JOB_STATE_DAEMONS_REPORTED);
OBJ_RELEASE(state);
return;
}
/* need integer value for command line parameter */
orte_util_convert_jobid_to_string(&jobid_string, daemons->jobid);
/*
* start building argv array
*/
argv = NULL;
argc = 0;
/*
* ALPS aprun OPTIONS
*/
/* add the aprun command */
opal_argv_append(&argc, &argv, mca_plm_alps_component.aprun_cmd);
/* Append user defined arguments to aprun */
if ( NULL != mca_plm_alps_component.custom_args ) {
custom_strings = opal_argv_split(mca_plm_alps_component.custom_args, ' ');
num_args = opal_argv_count(custom_strings);
for (i = 0; i < num_args; ++i) {
opal_argv_append(&argc, &argv, custom_strings[i]);
}
opal_argv_free(custom_strings);
}
/* number of processors needed */
opal_argv_append(&argc, &argv, "-n");
asprintf(&tmp, "%lu", (unsigned long) map->num_new_daemons);
opal_argv_append(&argc, &argv, tmp);
free(tmp);
opal_argv_append(&argc, &argv, "-N");
opal_argv_append(&argc, &argv, "1");
opal_argv_append(&argc, &argv, "-cc");
opal_argv_append(&argc, &argv, "none");
/* create nodelist */
nodelist_argv = NULL;
nodelist_argc = 0;
for (nnode=0; nnode < map->nodes->size; nnode++) {
if (NULL == (node = (orte_node_t*)opal_pointer_array_get_item(map->nodes, nnode))) {
continue;
}
/* if the daemon already exists on this node, then
* don't include it
*/
if (ORTE_FLAG_TEST(node, ORTE_NODE_FLAG_DAEMON_LAUNCHED)) {
continue;
}
/* otherwise, add it to the list of nodes upon which
* we need to launch a daemon
*/
opal_argv_append(&nodelist_argc, &nodelist_argv, node->name);
}
if (0 == opal_argv_count(nodelist_argv)) {
orte_show_help("help-plm-alps.txt", "no-hosts-in-list", true);
rc = ORTE_ERR_FAILED_TO_START;
goto cleanup;
}
nodelist_flat = opal_argv_join(nodelist_argv, ',');
opal_argv_free(nodelist_argv);
/* if we are using all allocated nodes, then alps
* doesn't need a nodelist, or if running without a batch scheduler
*/
if ((map->num_new_daemons < orte_num_allocated_nodes) || (orte_num_allocated_nodes == 0)) {
opal_argv_append(&argc, &argv, "-L");
opal_argv_append(&argc, &argv, nodelist_flat);
}
/*
* ORTED OPTIONS
*/
/* add the daemon command (as specified by user) */
orte_plm_base_setup_orted_cmd(&argc, &argv);
/* Add basic orted command line options, including debug flags */
orte_plm_base_orted_append_basic_args(&argc, &argv,
NULL,
&proc_vpid_index,
nodelist_flat);
free(nodelist_flat);
/* tell the new daemons the base of the name list so they can compute
* their own name on the other end
*/
rc = orte_util_convert_vpid_to_string(&vpid_string, map->daemon_vpid_start);
if (ORTE_SUCCESS != rc) {
opal_output(0, "plm_alps: unable to create process name");
goto cleanup;
}
free(argv[proc_vpid_index]);
argv[proc_vpid_index] = strdup(vpid_string);
free(vpid_string);
if (mca_plm_alps_component.debug) {
param = opal_argv_join(argv, ' ');
if (NULL != param) {
opal_output(0, "plm:alps: final top-level argv:");
opal_output(0, "plm:alps: %s", param);
free(param);
}
}
/* Copy the prefix-directory specified in the
corresponding app_context. If there are multiple,
different prefix's in the app context, complain (i.e., only
allow one --prefix option for the entire alps run -- we
don't support different --prefix'es for different nodes in
the ALPS plm) */
cur_prefix = NULL;
for (i=0; i < state->jdata->apps->size; i++) {
char *app_prefix_dir = NULL;
if (NULL == (app = (orte_app_context_t*)opal_pointer_array_get_item(state->jdata->apps, i))) {
continue;
}
orte_get_attribute(&app->attributes, ORTE_APP_PREFIX_DIR, (void**)&app_prefix_dir, OPAL_STRING);
/* Check for already set cur_prefix_dir -- if different,
complain */
if (NULL != app_prefix_dir) {
if (NULL != cur_prefix &&
0 != strcmp (cur_prefix, app_prefix_dir)) {
orte_show_help("help-plm-alps.txt", "multiple-prefixes",
true, cur_prefix, app_prefix_dir);
goto cleanup;
}
/* If not yet set, copy it; iff set, then it's the
same anyway */
if (NULL == cur_prefix) {
cur_prefix = strdup(app_prefix_dir);
if (mca_plm_alps_component.debug) {
opal_output (0, "plm:alps: Set prefix:%s",
cur_prefix);
}
}
free(app_prefix_dir);
}
}
/* protect the args in case someone has a script wrapper around aprun */
mca_base_cmd_line_wrap_args(argv);
/* setup environment */
env = opal_argv_copy(orte_launch_environ);
if (0 < opal_output_get_verbosity(orte_plm_base_framework.framework_output)) {
param = opal_argv_join(argv, ' ');
OPAL_OUTPUT_VERBOSE((1, orte_plm_base_framework.framework_output,
"%s plm:alps: final top-level argv:\n\t%s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
(NULL == param) ? "NULL" : param));
if (NULL != param) free(param);
}
/* exec the daemon(s) */
if (ORTE_SUCCESS != (rc = plm_alps_start_proc(argc, argv, env, cur_prefix))) {
ORTE_ERROR_LOG(rc);
goto cleanup;
}
/* indicate that the daemons for this job were launched */
state->jdata->state = ORTE_JOB_STATE_DAEMONS_LAUNCHED;
daemons->state = ORTE_JOB_STATE_DAEMONS_LAUNCHED;
/* flag that launch was successful, so far as we currently know */
failed_launch = false;
cleanup:
if (NULL != argv) {
opal_argv_free(argv);
}
if (NULL != env) {
opal_argv_free(env);
}
if(NULL != jobid_string) {
free(jobid_string);
}
/* cleanup the caddy */
OBJ_RELEASE(state);
/* check for failed launch - if so, force terminate */
if (failed_launch) {
ORTE_FORCED_TERMINATE(ORTE_ERROR_DEFAULT_EXIT_CODE);
}
}
/* we cannot use the RML to communicate with SLURM as it doesn't
* understand our internal protocol, so we have to do a bare-bones
* exchange based on sockets
*/
static int dyn_allocate(orte_job_t *jdata)
{
char *cmd_str, **cmd=NULL, *tmp, *jstring;
char *node_list;
orte_app_context_t *app;
int i;
struct timeval tv;
local_jobtracker_t *jtrk;
int64_t i64, *i64ptr;
if (NULL == mca_ras_slurm_component.config_file) {
opal_output(0, "Cannot perform dynamic allocation as no Slurm configuration file provided");
return ORTE_ERR_NOT_FOUND;
}
/* track this request */
jtrk = OBJ_NEW(local_jobtracker_t);
jtrk->jobid = jdata->jobid;
opal_list_append(&jobs, &jtrk->super);
/* construct the command - note that the jdata structure contains
* a field for the minimum number of nodes required for the job.
* The node list can be constructed from the union of all the nodes
* contained in the dash_host field of the app_contexts. So you'll
* need to do a little work to build the command. We don't currently
* have a field in the jdata structure for "mandatory" vs "optional"
* allocations, so we'll have to add that someday. Likewise, you may
* want to provide a param to adjust the timeout value
*/
/* construct the cmd string */
opal_argv_append_nosize(&cmd, "allocate");
/* add the jobid */
orte_util_convert_jobid_to_string(&jstring, jdata->jobid);
asprintf(&tmp, "jobid=%s", jstring);
opal_argv_append_nosize(&cmd, tmp);
free(tmp);
free(jstring);
/* if we want the allocation for all apps in one shot,
* then tell slurm
*
* RHC: we don't currently have the ability to handle
* rolling allocations in the rest of the code base
*/
#if 0
if (!mca_ras_slurm_component.rolling_alloc) {
opal_argv_append_nosize(&cmd, "return=all");
}
#else
opal_argv_append_nosize(&cmd, "return=all");
#endif
/* pass the timeout */
asprintf(&tmp, "timeout=%d", mca_ras_slurm_component.timeout);
opal_argv_append_nosize(&cmd, tmp);
free(tmp);
/* for each app, add its allocation request info */
i64ptr = &i64;
for (i=0; i < jdata->apps->size; i++) {
if (NULL == (app = (orte_app_context_t*)opal_pointer_array_get_item(jdata->apps, i))) {
continue;
}
/* add the app id, preceded by a colon separator */
asprintf(&tmp, ": app=%d", (int)app->idx);
opal_argv_append_nosize(&cmd, tmp);
free(tmp);
/* add the number of process "slots" we need */
asprintf(&tmp, "np=%d", app->num_procs);
opal_argv_append_nosize(&cmd, tmp);
free(tmp);
/* if we were given a minimum number of nodes, pass it along */
if (orte_get_attribute(&app->attributes, ORTE_APP_MIN_NODES, (void**)&i64ptr, OPAL_INT64)) {
asprintf(&tmp, "N=%ld", (long int)i64);
opal_argv_append_nosize(&cmd, tmp);
free(tmp);
}
/* add the list of nodes, if one was given, ensuring
* that each node only appears once
*/
node_list = get_node_list(app);
if (NULL != node_list) {
asprintf(&tmp, "node_list=%s", node_list);
opal_argv_append_nosize(&cmd, tmp);
free(node_list);
free(tmp);
}
/* add the mandatory/optional flag */
if (orte_get_attribute(&app->attributes, ORTE_APP_MANDATORY, NULL, OPAL_BOOL)) {
opal_argv_append_nosize(&cmd, "flag=mandatory");
} else {
opal_argv_append_nosize(&cmd, "flag=optional");
}
}
/* assemble it into the final cmd to be sent */
cmd_str = opal_argv_join(cmd, ' ');
opal_argv_free(cmd);
/* start a timer - if the response to our request doesn't appear
* in the defined time, then we will error out as Slurm isn't
* responding to us
*/
opal_event_evtimer_set(orte_event_base, &jtrk->timeout_ev, timeout, jtrk);
tv.tv_sec = mca_ras_slurm_component.timeout * 2;
tv.tv_usec = 0;
opal_event_evtimer_add(&jtrk->timeout_ev, &tv);
opal_output_verbose(2, orte_ras_base_framework.framework_output,
"%s slurm:dynalloc cmd_str = %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
cmd_str);
if (send(socket_fd, cmd_str, strlen(cmd_str)+1, 0) < 0) {
ORTE_ERROR_LOG(ORTE_ERR_COMM_FAILURE);
}
free(cmd_str);
/* we cannot wait here for a response as we
* are already in an event. So return a value
* that indicates we are waiting for an
* allocation so the base functions know
* that they shouldn't progress the job
*/
return ORTE_ERR_ALLOCATION_PENDING;
}
int orte_plm_base_fork_hnp(void)
{
int p[2], death_pipe[2];
char *cmd;
char **argv = NULL;
int argc;
char *param, *cptr, *pmix_uri;
sigset_t sigs;
int buffer_length, num_chars_read, chunk;
char *orted_uri;
int rc;
orte_jobid_t jobid;
/* A pipe is used to communicate between the parent and child to
indicate whether the exec ultimately succeeded or failed. The
child sets the pipe to be close-on-exec; the child only ever
writes anything to the pipe if there is an error (e.g.,
executable not found, exec() fails, etc.). The parent does a
blocking read on the pipe; if the pipe closed with no data,
then the exec() succeeded. If the parent reads something from
the pipe, then the child was letting us know that it failed.
*/
if (pipe(p) < 0) {
ORTE_ERROR_LOG(ORTE_ERR_SYS_LIMITS_PIPES);
return ORTE_ERR_SYS_LIMITS_PIPES;
}
/* we also have to give the HNP a pipe it can watch to know when
* we terminated. Since the HNP is going to be a child of us, it
* can't just use waitpid to see when we leave - so it will watch
* the pipe instead
*/
if (pipe(death_pipe) < 0) {
ORTE_ERROR_LOG(ORTE_ERR_SYS_LIMITS_PIPES);
return ORTE_ERR_SYS_LIMITS_PIPES;
}
/* find the orted binary using the install_dirs support - this also
* checks to ensure that we can see this executable and it *is* executable by us
*/
cmd = opal_path_access("orted", opal_install_dirs.bindir, X_OK);
if (NULL == cmd) {
/* guess we couldn't do it - best to abort */
ORTE_ERROR_LOG(ORTE_ERR_FILE_NOT_EXECUTABLE);
close(p[0]);
close(p[1]);
return ORTE_ERR_FILE_NOT_EXECUTABLE;
}
/* okay, setup an appropriate argv */
opal_argv_append(&argc, &argv, "orted");
/* tell the daemon it is to be the HNP */
opal_argv_append(&argc, &argv, "--hnp");
/* tell the daemon to get out of our process group */
opal_argv_append(&argc, &argv, "--set-sid");
/* tell the daemon to report back its uri so we can connect to it */
opal_argv_append(&argc, &argv, "--report-uri");
asprintf(¶m, "%d", p[1]);
opal_argv_append(&argc, &argv, param);
free(param);
/* give the daemon a pipe it can watch to tell when we have died */
opal_argv_append(&argc, &argv, "--singleton-died-pipe");
asprintf(¶m, "%d", death_pipe[0]);
opal_argv_append(&argc, &argv, param);
free(param);
/* add any debug flags */
if (orte_debug_flag) {
opal_argv_append(&argc, &argv, "--debug");
}
if (orte_debug_daemons_flag) {
opal_argv_append(&argc, &argv, "--debug-daemons");
}
if (orte_debug_daemons_file_flag) {
if (!orte_debug_daemons_flag) {
opal_argv_append(&argc, &argv, "--debug-daemons");
}
opal_argv_append(&argc, &argv, "--debug-daemons-file");
}
/* indicate that it must use the novm state machine */
opal_argv_append(&argc, &argv, "-"OPAL_MCA_CMD_LINE_ID);
opal_argv_append(&argc, &argv, "state_novm_select");
opal_argv_append(&argc, &argv, "1");
/* pass it a jobid to match my job family */
opal_argv_append(&argc, &argv, "-"OPAL_MCA_CMD_LINE_ID);
opal_argv_append(&argc, &argv, "ess_base_jobid");
jobid = ORTE_DAEMON_JOBID(ORTE_PROC_MY_NAME->jobid);
if (ORTE_SUCCESS != (rc = orte_util_convert_jobid_to_string(¶m, jobid))) {
ORTE_ERROR_LOG(rc);
return rc;
}
opal_argv_append(&argc, &argv, param);
free(param);
/* Fork off the child */
orte_process_info.hnp_pid = fork();
if(orte_process_info.hnp_pid < 0) {
ORTE_ERROR_LOG(ORTE_ERR_SYS_LIMITS_CHILDREN);
close(p[0]);
close(p[1]);
close(death_pipe[0]);
close(death_pipe[1]);
free(cmd);
opal_argv_free(argv);
return ORTE_ERR_SYS_LIMITS_CHILDREN;
}
if (orte_process_info.hnp_pid == 0) {
close(p[0]);
close(death_pipe[1]);
/* I am the child - exec me */
/* Set signal handlers back to the default. Do this close
to the execve() because the event library may (and likely
will) reset them. If we don't do this, the event
library may have left some set that, at least on some
OS's, don't get reset via fork() or exec(). Hence, the
orted could be unkillable (for example). */
set_handler_default(SIGTERM);
set_handler_default(SIGINT);
set_handler_default(SIGHUP);
set_handler_default(SIGPIPE);
set_handler_default(SIGCHLD);
/* Unblock all signals, for many of the same reasons that
we set the default handlers, above. This is noticable
on Linux where the event library blocks SIGTERM, but we
don't want that blocked by the orted (or, more
specifically, we don't want it to be blocked by the
orted and then inherited by the ORTE processes that it
forks, making them unkillable by SIGTERM). */
sigprocmask(0, 0, &sigs);
sigprocmask(SIG_UNBLOCK, &sigs, 0);
execv(cmd, argv);
/* if I get here, the execv failed! */
orte_show_help("help-ess-base.txt", "ess-base:execv-error",
true, cmd, strerror(errno));
exit(1);
} else {
/* I am the parent - wait to hear something back and
* report results
*/
close(p[1]); /* parent closes the write - orted will write its contact info to it*/
close(death_pipe[0]); /* parent closes the death_pipe's read */
opal_argv_free(argv);
/* setup the buffer to read the HNP's uri */
buffer_length = ORTE_URI_MSG_LGTH;
chunk = ORTE_URI_MSG_LGTH-1;
num_chars_read = 0;
orted_uri = (char*)malloc(buffer_length);
while (chunk == (rc = read(p[0], &orted_uri[num_chars_read], chunk))) {
/* we read an entire buffer - better get more */
num_chars_read += chunk;
buffer_length += ORTE_URI_MSG_LGTH;
orted_uri = realloc((void*)orted_uri, buffer_length);
}
num_chars_read += rc;
if (num_chars_read <= 0) {
/* we didn't get anything back - this is bad */
ORTE_ERROR_LOG(ORTE_ERR_HNP_COULD_NOT_START);
free(orted_uri);
return ORTE_ERR_HNP_COULD_NOT_START;
}
/* parse the sysinfo from the returned info - must
* start from the end of the string as the uri itself
* can contain brackets */
if (NULL == (param = strrchr(orted_uri, '['))) {
ORTE_ERROR_LOG(ORTE_ERR_COMM_FAILURE);
free(orted_uri);
return ORTE_ERR_COMM_FAILURE;
}
*param = '\0'; /* terminate the uri string */
++param; /* point to the start of the sysinfo */
/* find the end of the sysinfo */
if (NULL == (cptr = strchr(param, ']'))) {
ORTE_ERROR_LOG(ORTE_ERR_COMM_FAILURE);
free(orted_uri);
return ORTE_ERR_COMM_FAILURE;
}
*cptr = '\0'; /* terminate the sysinfo string */
++cptr; /* point to the start of the pmix uri */
/* convert the sysinfo string */
if (ORTE_SUCCESS != (rc = orte_util_convert_string_to_sysinfo(&orte_local_cpu_type,
&orte_local_cpu_model, param))) {
ORTE_ERROR_LOG(rc);
free(orted_uri);
return rc;
}
/* save the daemon uri - we will process it later */
orte_process_info.my_daemon_uri = strdup(orted_uri);
/* Set the contact info in the RML - this won't actually establish
* the connection, but just tells the RML how to reach the daemon
* if/when we attempt to send to it
*/
orte_rml.set_contact_info(orte_process_info.my_daemon_uri);
if (ORTE_SUCCESS != (rc = orte_rml_base_parse_uris(orte_process_info.my_daemon_uri,
ORTE_PROC_MY_DAEMON, NULL))) {
ORTE_ERROR_LOG(rc);
return rc;
}
/* likewise, since this is also the HNP, set that uri too */
orte_process_info.my_hnp_uri = strdup(orted_uri);
orte_rml.set_contact_info(orte_process_info.my_hnp_uri);
if (ORTE_SUCCESS != (rc = orte_rml_base_parse_uris(orte_process_info.my_hnp_uri,
ORTE_PROC_MY_HNP, NULL))) {
ORTE_ERROR_LOG(rc);
return rc;
}
/* push the pmix_uri into our environment - need to protect it */
(void)asprintf(&pmix_uri, "PMIX_SERVER_URI=%s", cptr);
putenv(pmix_uri);
/* now re-init the pmix framework so we can connect when required */
if (OPAL_SUCCESS != (rc = opal_pmix.init())) {
ORTE_ERROR_LOG(rc);
return rc;
}
/* now call fence to push our own modex data into the
* newly-launched HNP in case someone else needs it */
if (OPAL_SUCCESS != (rc = opal_pmix.fence(NULL, 0))) {
ORTE_ERROR_LOG(rc);
return rc;
}
/* all done - report success */
free(orted_uri);
return ORTE_SUCCESS;
}
}
/* setup argv for daemon process */
static int setup_daemon_proc_env_and_argv(orte_proc_t* proc, char ***pargv,
int *argc, char ***penv)
{
orte_job_t* daemons;
int rc;
char* param;
/* get daemon job object */
daemons = orte_get_job_data_object(ORTE_PROC_MY_NAME->jobid);
*penv = opal_argv_copy(orte_launch_environ);
/* prepend orted to argv */
opal_argv_append(argc, pargv, "orted");
/* ess */
opal_argv_append(argc, pargv, "-mca");
opal_argv_append(argc, pargv, "ess");
opal_argv_append(argc, pargv, "env");
/* jobid */
opal_argv_append(argc, pargv, "-mca");
opal_argv_append(argc, pargv, "orte_ess_jobid");
if (ORTE_SUCCESS != (rc = orte_util_convert_jobid_to_string(¶m, ORTE_PROC_MY_NAME->jobid))) {
ORTE_ERROR_LOG(rc);
return rc;
}
opal_argv_append(argc, pargv, param);
free(param);
/* vpid */
opal_argv_append(argc, pargv, "-mca");
opal_argv_append(argc, pargv, "orte_ess_vpid");
if (ORTE_SUCCESS != (rc = orte_util_convert_vpid_to_string(¶m, proc->name.vpid))) {
ORTE_ERROR_LOG(rc);
return rc;
}
opal_argv_append(argc, pargv, param);
free(param);
/* num processes */
opal_argv_append(argc, pargv, "-mca");
opal_argv_append(argc, pargv, "orte_ess_num_procs");
asprintf(¶m, "%lu", daemons->num_procs);
opal_argv_append(argc, pargv, param);
free(param);
/* pass the uri of the hnp */
asprintf(¶m, "\\\"%s\\\"", orte_rml.get_contact_info());
opal_argv_append(argc, pargv, "-mca");
opal_argv_append(argc, pargv, "orte_hnp_uri");
opal_argv_append(argc, pargv, param);
free(param);
/* oob */
opal_argv_append(argc, pargv, "-mca");
opal_argv_append(argc, pargv, "oob");
opal_argv_append(argc, pargv, "tcp");
/* odls */
opal_argv_append(argc, pargv, "-mca");
opal_argv_append(argc, pargv, "odls");
opal_argv_append(argc, pargv, "yarn");
/* add stdout, stderr to orted */
opal_argv_append_nosize(pargv, "1><LOG_DIR>/stdout");
opal_argv_append_nosize(pargv, "2><LOG_DIR>/stderr");
/* orted */
opal_argv_append(argc, pargv, "-mca");
opal_argv_append(argc, pargv, "state");
opal_argv_append(argc, pargv, "orted");
/* print launch commandline and env when this env is specified */
if (getenv("HAMSTER_VERBOSE")) {
char* join_argv = opal_argv_join(*pargv, ' ');
OPAL_OUTPUT_VERBOSE((5, orte_plm_globals.output, "%s plm:yarn launch_daemon argv=%s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), join_argv));
if (join_argv) {
free(join_argv);
}
}
return 0;
}
/*
* setup env and argv for specified process
*/
static int setup_proc_env_and_argv(orte_job_t* jdata, orte_app_context_t* app,
orte_proc_t* proc, char ***pargv, char ***penv)
{
char* param;
char* param2;
char* value;
char* vp_id_str;
char* job_id_str;
int rc;
int i, num_nodes;
/* obtain app->argv */
if (!(app->argv)) {
opal_output(0, "%s plm::yarn::setup_proc_env_and_argv: app->argv is null",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
return ORTE_ERROR;
}
*pargv = opal_argv_copy(app->argv);
if (ORTE_SUCCESS != orte_util_convert_jobid_to_string(&job_id_str, jdata->jobid)) {
ORTE_ERROR_LOG(ORTE_ERR_OUT_OF_RESOURCE);
return ORTE_ERR_OUT_OF_RESOURCE;
}
if (ORTE_SUCCESS != orte_util_convert_vpid_to_string(&vp_id_str, proc->name.vpid)) {
ORTE_ERROR_LOG(ORTE_ERR_OUT_OF_RESOURCE);
return ORTE_ERR_OUT_OF_RESOURCE;
}
// add stdout, stderr to app
opal_argv_append_nosize(pargv, "1><LOG_DIR>/stdout");
opal_argv_append_nosize(pargv, "2><LOG_DIR>/stderr");
// add java executor to app
opal_argv_prepend_nosize(pargv, vp_id_str);
opal_argv_prepend_nosize(pargv, job_id_str);
opal_argv_prepend_nosize(pargv, "com.pivotal.hamster.yarnexecutor.YarnExecutor");
opal_argv_prepend_nosize(pargv, "hamster-core.jar");
opal_argv_prepend_nosize(pargv, "-cp");
opal_argv_prepend_nosize(pargv, getenv("HAMSTER_JAVA_OPT")==NULL ? "-Xmx32M -Xms8M" : getenv("HAMSTER_JAVA_OPT"));
opal_argv_prepend_nosize(pargv, "$JAVA_HOME/bin/java");
/* obtain app->env */
*penv = opal_environ_merge(environ, app->env);
if (!proc->node) {
opal_output(0, "%s plm::yarn::setup_proc_env_and_argv: node of proc[%d] is NULL",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), proc->name.vpid);
return ORTE_ERROR;
}
if (!proc->node->daemon) {
opal_output(0, "%s plm::yarn::setup_proc_env_and_argv: daemon of node[%s] is NULL",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), proc->node->name);
return ORTE_ERROR;
}
/* set the app_context number into the environment */ //??
param = mca_base_param_env_var("orte_app_num");
asprintf(¶m2, "%ld", (long)app->idx);
opal_setenv(param, param2, true, penv);
free(param);
free(param2);
// pass the daemon's name
param = mca_base_param_env_var("orte_local_daemon_uri");
opal_setenv(param, proc->node->daemon->rml_uri, true, penv);
free(param);
/* pass my contact info */
param = mca_base_param_env_var("orte_hnp_uri");
opal_setenv(param, orte_process_info.my_hnp_uri, true, penv);
free(param);
/* pass the jobid */
param = mca_base_param_env_var("orte_ess_jobid");
opal_setenv(param, job_id_str, true, penv);
free(param);
free(job_id_str);
/* pass the rank */
param = mca_base_param_env_var("orte_ess_vpid");
opal_setenv(param, vp_id_str, true, penv);
free(param);
opal_setenv("OMPI_COMM_WORLD_RANK", vp_id_str, true, penv);
free(vp_id_str); /* done with this now */
/* pass local rank */
asprintf(&value, "%lu", (unsigned long) proc->local_rank);
opal_setenv("OMPI_COMM_WORLD_LOCAL_RANK", value, true, penv);
free(value);
/* pass node rank */
asprintf(&value, "%lu", (unsigned long) proc->node_rank);
opal_setenv("OMPI_COMM_WORLD_NODE_RANK", value, true, penv);
/* set an mca param for it too */
param = mca_base_param_env_var("orte_ess_node_rank");
opal_setenv(param, value, true, penv);
free(param);
free(value);
/* pass a param telling the child what model of cpu we are on,
* if we know it
*/
if (NULL != orte_local_cpu_type) {
param = mca_base_param_env_var("orte_cpu_type");
/* do not overwrite what the user may have provided */
opal_setenv(param, orte_local_cpu_type, false, penv);
free(param);
}
if (NULL != orte_local_cpu_model) {
param = mca_base_param_env_var("orte_cpu_model");
/* do not overwrite what the user may have provided */
opal_setenv(param, orte_local_cpu_model, false, penv);
free(param);
}
/* pass the number of nodes involved in this job */
param = mca_base_param_env_var("orte_num_nodes");
/* we have to count the number of nodes as the size of orte_node_pool
* is only guaranteed to be equal or larger than that number - i.e.,
* the pointer_array increases the size by a block each time, so some
* of the locations are left empty
*/
num_nodes = 0;
for (i = 0; i < orte_node_pool->size; i++) {
if (NULL != opal_pointer_array_get_item(orte_node_pool, i)) {
num_nodes++;
}
}
asprintf(&value, "%d", num_nodes);
opal_setenv(param, value, true, penv);
free(param);
free(value);
/* setup yield schedule */
param = mca_base_param_env_var("mpi_yield_when_idle");
opal_setenv(param, "0", false, penv);
free(param);
/* set MPI universe envar */
orte_ess_env_put(jdata->num_procs, proc->node->num_procs, penv);
asprintf(&value, "%ld", (long) jdata->num_procs);
opal_setenv("OMPI_UNIVERSE_SIZE", value, true, penv);
free(value);
/* pass collective ids for the std MPI operations */
param = mca_base_param_env_var("orte_peer_modex_id");
asprintf(&value, "%d", jdata->peer_modex);
opal_setenv(param, value, true, penv);
free(param);
free(value);
param = mca_base_param_env_var("orte_peer_init_barrier_id");
asprintf(&value, "%d", jdata->peer_init_barrier);
opal_setenv(param, value, true, penv);
free(param);
free(value);
param = mca_base_param_env_var("orte_peer_fini_barrier_id");
asprintf(&value, "%d", jdata->peer_fini_barrier);
opal_setenv(param, value, true, penv);
free(param);
free(value);
/* finally, we will set/unset some mca param to select modules */
opal_unsetenv("OMPI_MCA_plm", penv);
opal_unsetenv("OMPI_MCA_ras", penv);
opal_unsetenv("OMPI_MCA_ess", penv);
opal_unsetenv("OMPI_MCA_state", penv);
opal_unsetenv("OMPI_MCA_errmgr", penv);
return 0;
}
int orte_ess_base_orted_setup(char **hosts)
{
int ret = ORTE_ERROR;
int fd;
char log_file[PATH_MAX];
char *jobidstring;
char *error = NULL;
orte_job_t *jdata;
orte_proc_t *proc;
orte_app_context_t *app;
orte_node_t *node;
char *param;
plm_in_use = false;
/* setup callback for SIGPIPE */
setup_sighandler(SIGPIPE, &epipe_handler, epipe_signal_callback);
/* Set signal handlers to catch kill signals so we can properly clean up
* after ourselves.
*/
setup_sighandler(SIGTERM, &term_handler, shutdown_signal);
setup_sighandler(SIGINT, &int_handler, shutdown_signal);
/** setup callbacks for signals we should ignore */
setup_sighandler(SIGUSR1, &sigusr1_handler, signal_callback);
setup_sighandler(SIGUSR2, &sigusr2_handler, signal_callback);
signals_set = true;
#if OPAL_HAVE_HWLOC
{
hwloc_obj_t obj;
unsigned i, j;
/* get the local topology */
if (NULL == opal_hwloc_topology) {
if (OPAL_SUCCESS != opal_hwloc_base_get_topology()) {
error = "topology discovery";
goto error;
}
}
/* remove the hostname from the topology. Unfortunately, hwloc
* decided to add the source hostname to the "topology", thus
* rendering it unusable as a pure topological description. So
* we remove that information here.
*/
obj = hwloc_get_root_obj(opal_hwloc_topology);
for (i=0; i < obj->infos_count; i++) {
if (NULL == obj->infos[i].name ||
NULL == obj->infos[i].value) {
continue;
}
if (0 == strncmp(obj->infos[i].name, "HostName", strlen("HostName"))) {
free(obj->infos[i].name);
free(obj->infos[i].value);
/* left justify the array */
for (j=i; j < obj->infos_count-1; j++) {
obj->infos[j] = obj->infos[j+1];
}
obj->infos[obj->infos_count-1].name = NULL;
obj->infos[obj->infos_count-1].value = NULL;
obj->infos_count--;
break;
}
}
if (4 < opal_output_get_verbosity(orte_ess_base_framework.framework_output)) {
opal_output(0, "%s Topology Info:", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
opal_dss.dump(0, opal_hwloc_topology, OPAL_HWLOC_TOPO);
}
}
#endif
/* setup the global nidmap/pidmap object */
orte_nidmap.bytes = NULL;
orte_nidmap.size = 0;
orte_pidmap.bytes = NULL;
orte_pidmap.size = 0;
/* open and setup the opal_pstat framework so we can provide
* process stats if requested
*/
if (ORTE_SUCCESS != (ret = mca_base_framework_open(&opal_pstat_base_framework, 0))) {
ORTE_ERROR_LOG(ret);
error = "opal_pstat_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = opal_pstat_base_select())) {
ORTE_ERROR_LOG(ret);
error = "opal_pstat_base_select";
goto error;
}
/* open and setup the state machine */
if (ORTE_SUCCESS != (ret = mca_base_framework_open(&orte_state_base_framework, 0))) {
ORTE_ERROR_LOG(ret);
error = "orte_state_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_state_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_state_base_select";
goto error;
}
/* open the errmgr */
if (ORTE_SUCCESS != (ret = mca_base_framework_open(&orte_errmgr_base_framework, 0))) {
ORTE_ERROR_LOG(ret);
error = "orte_errmgr_base_open";
goto error;
}
/* some environments allow remote launches - e.g., ssh - so
* open and select something -only- if we are given
* a specific module to use
*/
(void) mca_base_var_env_name("plm", ¶m);
plm_in_use = !!(getenv(param));
free (param);
if (plm_in_use) {
if (ORTE_SUCCESS != (ret = mca_base_framework_open(&orte_plm_base_framework, 0))) {
ORTE_ERROR_LOG(ret);
error = "orte_plm_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_plm_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_plm_base_select";
goto error;
}
}
/* Setup the communication infrastructure */
if (ORTE_SUCCESS != (ret = mca_base_framework_open(&orte_oob_base_framework, 0))) {
ORTE_ERROR_LOG(ret);
error = "orte_oob_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_oob_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_oob_base_select";
goto error;
}
if (ORTE_SUCCESS != (ret = mca_base_framework_open(&orte_rml_base_framework, 0))) {
ORTE_ERROR_LOG(ret);
error = "orte_rml_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_rml_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_rml_base_select";
goto error;
}
/* select the errmgr */
if (ORTE_SUCCESS != (ret = orte_errmgr_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_errmgr_base_select";
goto error;
}
/* Routed system */
if (ORTE_SUCCESS != (ret = mca_base_framework_open(&orte_routed_base_framework, 0))) {
ORTE_ERROR_LOG(ret);
error = "orte_rml_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_routed_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_routed_base_select";
goto error;
}
/* database */
if (ORTE_SUCCESS != (ret = mca_base_framework_open(&opal_db_base_framework, 0))) {
ORTE_ERROR_LOG(ret);
error = "orte_db_base_open";
goto error;
}
/* always restrict daemons to local database components */
if (ORTE_SUCCESS != (ret = opal_db_base_select(true))) {
ORTE_ERROR_LOG(ret);
error = "orte_db_base_select";
goto error;
}
/* set our id */
opal_db.set_id((opal_identifier_t*)ORTE_PROC_MY_NAME);
/*
* Group communications
*/
if (ORTE_SUCCESS != (ret = mca_base_framework_open(&orte_grpcomm_base_framework, 0))) {
ORTE_ERROR_LOG(ret);
error = "orte_grpcomm_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_grpcomm_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_grpcomm_base_select";
goto error;
}
/* Open/select the odls */
if (ORTE_SUCCESS != (ret = mca_base_framework_open(&orte_odls_base_framework, 0))) {
ORTE_ERROR_LOG(ret);
error = "orte_odls_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_odls_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_odls_base_select";
goto error;
}
/* enable communication with the rml */
if (ORTE_SUCCESS != (ret = orte_rml.enable_comm())) {
ORTE_ERROR_LOG(ret);
error = "orte_rml.enable_comm";
goto error;
}
/* initialize the nidmaps */
if (ORTE_SUCCESS != (ret = orte_util_nidmap_init(NULL))) {
ORTE_ERROR_LOG(ret);
error = "orte_util_nidmap_init";
goto error;
}
#if ORTE_ENABLE_STATIC_PORTS
/* if we are using static ports, then we need to setup
* the daemon info so the RML can function properly
* without requiring a wireup stage. This must be done
* after we enable_comm as that function determines our
* own port, which we need in order to construct the nidmap
*/
if (orte_static_ports) {
/* define the routing tree so we know the pattern
* if we are trying to setup common or static ports
*/
orte_routed.update_routing_plan();
/* extract the node info from the environment and
* build a nidmap from it
*/
if (ORTE_SUCCESS != (ret = orte_util_build_daemon_nidmap(hosts))) {
ORTE_ERROR_LOG(ret);
error = "construct daemon map from static ports";
goto error;
}
}
#endif
/* be sure to update the routing tree so the initial "phone home"
* to mpirun goes through the tree if static ports were enabled - still
* need to do it anyway just to initialize things
*/
orte_routed.update_routing_plan();
/* Now provide a chance for the PLM
* to perform any module-specific init functions. This
* needs to occur AFTER the communications are setup
* as it may involve starting a non-blocking recv
* Do this only if a specific PLM was given to us - the
* orted has no need of the proxy PLM at all
*/
if (plm_in_use) {
if (ORTE_SUCCESS != (ret = orte_plm.init())) {
ORTE_ERROR_LOG(ret);
error = "orte_plm_init";
goto error;
}
}
/* setup my session directory */
if (orte_create_session_dirs) {
OPAL_OUTPUT_VERBOSE((2, orte_ess_base_framework.framework_output,
"%s setting up session dir with\n\ttmpdir: %s\n\thost %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
(NULL == orte_process_info.tmpdir_base) ? "UNDEF" : orte_process_info.tmpdir_base,
orte_process_info.nodename));
/* take a pass thru the session directory code to fillin the
* tmpdir names - don't create anything yet
*/
if (ORTE_SUCCESS != (ret = orte_session_dir(false,
orte_process_info.tmpdir_base,
orte_process_info.nodename, NULL,
ORTE_PROC_MY_NAME))) {
ORTE_ERROR_LOG(ret);
error = "orte_session_dir define";
goto error;
}
/* clear the session directory just in case there are
* stale directories laying around
*/
orte_session_dir_cleanup(ORTE_JOBID_WILDCARD);
/* now actually create the directory tree */
if (ORTE_SUCCESS != (ret = orte_session_dir(true,
orte_process_info.tmpdir_base,
orte_process_info.nodename, NULL,
ORTE_PROC_MY_NAME))) {
ORTE_ERROR_LOG(ret);
error = "orte_session_dir";
goto error;
}
/* Once the session directory location has been established, set
the opal_output env file location to be in the
proc-specific session directory. */
opal_output_set_output_file_info(orte_process_info.proc_session_dir,
"output-", NULL, NULL);
/* setup stdout/stderr */
if (orte_debug_daemons_file_flag) {
/* if we are debugging to a file, then send stdout/stderr to
* the orted log file
*/
/* get my jobid */
if (ORTE_SUCCESS != (ret = orte_util_convert_jobid_to_string(&jobidstring,
ORTE_PROC_MY_NAME->jobid))) {
ORTE_ERROR_LOG(ret);
error = "convert_jobid";
goto error;
}
/* define a log file name in the session directory */
snprintf(log_file, PATH_MAX, "output-orted-%s-%s.log",
jobidstring, orte_process_info.nodename);
log_path = opal_os_path(false,
orte_process_info.tmpdir_base,
orte_process_info.top_session_dir,
log_file,
NULL);
fd = open(log_path, O_RDWR|O_CREAT|O_TRUNC, 0640);
if (fd < 0) {
/* couldn't open the file for some reason, so
* just connect everything to /dev/null
*/
fd = open("/dev/null", O_RDWR|O_CREAT|O_TRUNC, 0666);
} else {
dup2(fd, STDOUT_FILENO);
dup2(fd, STDERR_FILENO);
if(fd != STDOUT_FILENO && fd != STDERR_FILENO) {
close(fd);
}
}
}
}
/* setup the global job and node arrays */
orte_job_data = OBJ_NEW(opal_pointer_array_t);
if (ORTE_SUCCESS != (ret = opal_pointer_array_init(orte_job_data,
1,
ORTE_GLOBAL_ARRAY_MAX_SIZE,
1))) {
ORTE_ERROR_LOG(ret);
error = "setup job array";
goto error;
}
orte_node_pool = OBJ_NEW(opal_pointer_array_t);
if (ORTE_SUCCESS != (ret = opal_pointer_array_init(orte_node_pool,
ORTE_GLOBAL_ARRAY_BLOCK_SIZE,
ORTE_GLOBAL_ARRAY_MAX_SIZE,
ORTE_GLOBAL_ARRAY_BLOCK_SIZE))) {
ORTE_ERROR_LOG(ret);
error = "setup node array";
goto error;
}
orte_node_topologies = OBJ_NEW(opal_pointer_array_t);
if (ORTE_SUCCESS != (ret = opal_pointer_array_init(orte_node_topologies,
ORTE_GLOBAL_ARRAY_BLOCK_SIZE,
ORTE_GLOBAL_ARRAY_MAX_SIZE,
ORTE_GLOBAL_ARRAY_BLOCK_SIZE))) {
ORTE_ERROR_LOG(ret);
error = "setup node topologies array";
goto error;
}
/* Setup the job data object for the daemons */
/* create and store the job data object */
jdata = OBJ_NEW(orte_job_t);
jdata->jobid = ORTE_PROC_MY_NAME->jobid;
opal_pointer_array_set_item(orte_job_data, 0, jdata);
/* every job requires at least one app */
app = OBJ_NEW(orte_app_context_t);
opal_pointer_array_set_item(jdata->apps, 0, app);
jdata->num_apps++;
/* create and store a node object where we are */
node = OBJ_NEW(orte_node_t);
node->name = strdup(orte_process_info.nodename);
node->index = opal_pointer_array_set_item(orte_node_pool, ORTE_PROC_MY_NAME->vpid, node);
#if OPAL_HAVE_HWLOC
/* point our topology to the one detected locally */
node->topology = opal_hwloc_topology;
#endif
/* create and store a proc object for us */
proc = OBJ_NEW(orte_proc_t);
proc->name.jobid = ORTE_PROC_MY_NAME->jobid;
proc->name.vpid = ORTE_PROC_MY_NAME->vpid;
proc->pid = orte_process_info.pid;
proc->rml_uri = orte_rml.get_contact_info();
proc->state = ORTE_PROC_STATE_RUNNING;
opal_pointer_array_set_item(jdata->procs, proc->name.vpid, proc);
/* record that the daemon (i.e., us) is on this node
* NOTE: we do not add the proc object to the node's
* proc array because we are not an application proc.
* Instead, we record it in the daemon field of the
* node object
*/
OBJ_RETAIN(proc); /* keep accounting straight */
node->daemon = proc;
node->daemon_launched = true;
node->state = ORTE_NODE_STATE_UP;
/* now point our proc node field to the node */
OBJ_RETAIN(node); /* keep accounting straight */
proc->node = node;
/* record that the daemon job is running */
jdata->num_procs = 1;
jdata->state = ORTE_JOB_STATE_RUNNING;
/* obviously, we have "reported" */
jdata->num_reported = 1;
/* setup the routed info - the selected routed component
* will know what to do.
*/
if (ORTE_SUCCESS != (ret = orte_routed.init_routes(ORTE_PROC_MY_NAME->jobid, NULL))) {
ORTE_ERROR_LOG(ret);
error = "orte_routed.init_routes";
goto error;
}
/* setup I/O forwarding system - must come after we init routes */
if (ORTE_SUCCESS != (ret = mca_base_framework_open(&orte_iof_base_framework, 0))) {
ORTE_ERROR_LOG(ret);
error = "orte_iof_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_iof_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_iof_base_select";
goto error;
}
/* setup the FileM */
if (ORTE_SUCCESS != (ret = mca_base_framework_open(&orte_filem_base_framework, 0))) {
ORTE_ERROR_LOG(ret);
error = "orte_filem_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_filem_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_filem_base_select";
goto error;
}
#if OPAL_ENABLE_FT_CR == 1
/*
* Setup the SnapC
*/
if (ORTE_SUCCESS != (ret = mca_base_framework_open(&orte_snapc_base_framework, 0))) {
ORTE_ERROR_LOG(ret);
error = "orte_snapc_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = mca_base_framework_open(&orte_sstore_base_framework, 0))) {
ORTE_ERROR_LOG(ret);
error = "orte_sstore_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_snapc_base_select(!ORTE_PROC_IS_HNP, ORTE_PROC_IS_DAEMON))) {
ORTE_ERROR_LOG(ret);
error = "orte_snapc_base_select";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_sstore_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_sstore_base_select";
goto error;
}
/* For daemons, ORTE doesn't need the OPAL CR stuff */
opal_cr_set_enabled(false);
#else
opal_cr_set_enabled(false);
#endif
/*
* Initalize the CR setup
* Note: Always do this, even in non-FT builds.
* If we don't some user level tools may hang.
*/
if (ORTE_SUCCESS != (ret = orte_cr_init())) {
ORTE_ERROR_LOG(ret);
error = "orte_cr_init";
goto error;
}
/* setup the DFS framework */
if (ORTE_SUCCESS != (ret = mca_base_framework_open(&orte_dfs_base_framework, 0))) {
ORTE_ERROR_LOG(ret);
error = "orte_dfs_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_dfs_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_dfs_select";
goto error;
}
return ORTE_SUCCESS;
error:
orte_show_help("help-orte-runtime.txt",
"orte_init:startup:internal-failure",
true, error, ORTE_ERROR_NAME(ret), ret);
return ORTE_ERR_SILENT;
}
/* stuff proc attributes for sending back to a proc */
int orte_pmix_server_register_nspace(orte_job_t *jdata, bool force)
{
int rc;
orte_proc_t *pptr;
int i, k, n;
opal_list_t *info, *pmap;
opal_value_t *kv;
orte_node_t *node, *mynode;
opal_vpid_t vpid;
char **list, **procs, **micro, *tmp, *regex;
orte_job_t *dmns;
orte_job_map_t *map;
orte_app_context_t *app;
uid_t uid;
gid_t gid;
opal_list_t *cache;
hwloc_obj_t machine;
opal_output_verbose(2, orte_pmix_server_globals.output,
"%s register nspace for %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_JOBID_PRINT(jdata->jobid));
/* setup the info list */
info = OBJ_NEW(opal_list_t);
uid = geteuid();
gid = getegid();
/* pass our nspace/rank */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_SERVER_NSPACE);
kv->data.string = strdup(ORTE_JOBID_PRINT(ORTE_PROC_MY_NAME->jobid));
kv->type = OPAL_STRING;
opal_list_append(info, &kv->super);
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_SERVER_RANK);
kv->data.uint32 = ORTE_PROC_MY_NAME->vpid;
kv->type = OPAL_UINT32;
opal_list_append(info, &kv->super);
/* jobid */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_JOBID);
kv->data.string = strdup(ORTE_JOBID_PRINT(jdata->jobid));
kv->type = OPAL_STRING;
opal_list_append(info, &kv->super);
/* offset */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_NPROC_OFFSET);
kv->data.uint32 = jdata->offset;
kv->type = OPAL_UINT32;
opal_list_append(info, &kv->super);
/* check for cached values to add to the job info */
cache = NULL;
if (orte_get_attribute(&jdata->attributes, ORTE_JOB_INFO_CACHE, (void**)&cache, OPAL_PTR) &&
NULL != cache) {
while (NULL != (kv = (opal_value_t*)opal_list_remove_first(cache))) {
opal_list_append(info, &kv->super);
}
orte_remove_attribute(&jdata->attributes, ORTE_JOB_INFO_CACHE);
OBJ_RELEASE(cache);
}
/* assemble the node and proc map info */
list = NULL;
procs = NULL;
map = jdata->map;
for (i=0; i < map->nodes->size; i++) {
micro = NULL;
if (NULL != (node = (orte_node_t*)opal_pointer_array_get_item(map->nodes, i))) {
opal_argv_append_nosize(&list, node->name);
/* assemble all the ranks for this job that are on this node */
for (k=0; k < node->procs->size; k++) {
if (NULL != (pptr = (orte_proc_t*)opal_pointer_array_get_item(node->procs, k))) {
if (jdata->jobid == pptr->name.jobid) {
opal_argv_append_nosize(µ, ORTE_VPID_PRINT(pptr->name.vpid));
}
}
}
/* assemble the rank/node map */
if (NULL != micro) {
tmp = opal_argv_join(micro, ',');
opal_argv_free(micro);
opal_argv_append_nosize(&procs, tmp);
free(tmp);
}
}
}
/* let the PMIx server generate the nodemap regex */
if (NULL != list) {
tmp = opal_argv_join(list, ',');
opal_argv_free(list);
list = NULL;
if (OPAL_SUCCESS != (rc = opal_pmix.generate_regex(tmp, ®ex))) {
ORTE_ERROR_LOG(rc);
free(tmp);
OPAL_LIST_RELEASE(info);
return rc;
}
free(tmp);
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_NODE_MAP);
kv->type = OPAL_STRING;
kv->data.string = regex;
opal_list_append(info, &kv->super);
}
/* let the PMIx server generate the procmap regex */
if (NULL != procs) {
tmp = opal_argv_join(procs, ';');
opal_argv_free(procs);
procs = NULL;
if (OPAL_SUCCESS != (rc = opal_pmix.generate_ppn(tmp, ®ex))) {
ORTE_ERROR_LOG(rc);
free(tmp);
OPAL_LIST_RELEASE(info);
return rc;
}
free(tmp);
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_PROC_MAP);
kv->type = OPAL_STRING;
kv->data.string = regex;
opal_list_append(info, &kv->super);
}
/* get our local node */
if (NULL == (dmns = orte_get_job_data_object(ORTE_PROC_MY_NAME->jobid))) {
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
OPAL_LIST_RELEASE(info);
return ORTE_ERR_NOT_FOUND;
}
if (NULL == (pptr = (orte_proc_t*)opal_pointer_array_get_item(dmns->procs, ORTE_PROC_MY_NAME->vpid))) {
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
OPAL_LIST_RELEASE(info);
return ORTE_ERR_NOT_FOUND;
}
mynode = pptr->node;
if (NULL == mynode) {
/* cannot happen */
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
OPAL_LIST_RELEASE(info);
return ORTE_ERR_NOT_FOUND;
}
/* pass our node ID */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_NODEID);
kv->type = OPAL_UINT32;
kv->data.uint32 = mynode->index;
opal_list_append(info, &kv->super);
/* pass our node size */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_NODE_SIZE);
kv->type = OPAL_UINT32;
kv->data.uint32 = mynode->num_procs;
opal_list_append(info, &kv->super);
/* pass the number of nodes in the job */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_NUM_NODES);
kv->type = OPAL_UINT32;
kv->data.uint32 = map->num_nodes;
opal_list_append(info, &kv->super);
/* univ size */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_UNIV_SIZE);
kv->type = OPAL_UINT32;
kv->data.uint32 = jdata->total_slots_alloc;
opal_list_append(info, &kv->super);
/* job size */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_JOB_SIZE);
kv->type = OPAL_UINT32;
kv->data.uint32 = jdata->num_procs;
opal_list_append(info, &kv->super);
/* number of apps in this job */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_JOB_NUM_APPS);
kv->type = OPAL_UINT32;
kv->data.uint32 = jdata->num_apps;
opal_list_append(info, &kv->super);
/* local size */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_LOCAL_SIZE);
kv->type = OPAL_UINT32;
kv->data.uint32 = jdata->num_local_procs;
opal_list_append(info, &kv->super);
/* max procs */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_MAX_PROCS);
kv->type = OPAL_UINT32;
kv->data.uint32 = jdata->total_slots_alloc;
opal_list_append(info, &kv->super);
/* topology signature */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_TOPOLOGY_SIGNATURE);
kv->type = OPAL_STRING;
kv->data.string = strdup(orte_topo_signature);
opal_list_append(info, &kv->super);
/* total available physical memory */
machine = hwloc_get_next_obj_by_type (opal_hwloc_topology, HWLOC_OBJ_MACHINE, NULL);
if (NULL != machine) {
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_AVAIL_PHYS_MEMORY);
kv->type = OPAL_UINT64;
#if HWLOC_API_VERSION < 0x20000
kv->data.uint64 = machine->memory.total_memory;
#else
kv->data.uint64 = machine->total_memory;
#endif
opal_list_append(info, &kv->super);
}
/* pass the mapping policy used for this job */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_MAPBY);
kv->type = OPAL_STRING;
kv->data.string = strdup(orte_rmaps_base_print_mapping(jdata->map->mapping));
opal_list_append(info, &kv->super);
/* pass the ranking policy used for this job */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_RANKBY);
kv->type = OPAL_STRING;
kv->data.string = strdup(orte_rmaps_base_print_ranking(jdata->map->ranking));
opal_list_append(info, &kv->super);
/* pass the binding policy used for this job */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_BINDTO);
kv->type = OPAL_STRING;
kv->data.string = strdup(opal_hwloc_base_print_binding(jdata->map->binding));
opal_list_append(info, &kv->super);
/* register any local clients */
vpid = ORTE_VPID_MAX;
micro = NULL;
for (i=0; i < mynode->procs->size; i++) {
if (NULL == (pptr = (orte_proc_t*)opal_pointer_array_get_item(mynode->procs, i))) {
continue;
}
if (pptr->name.jobid == jdata->jobid) {
opal_argv_append_nosize(µ, ORTE_VPID_PRINT(pptr->name.vpid));
if (pptr->name.vpid < vpid) {
vpid = pptr->name.vpid;
}
/* go ahead and register this client */
if (OPAL_SUCCESS != (rc = opal_pmix.server_register_client(&pptr->name, uid, gid,
(void*)pptr, NULL, NULL))) {
ORTE_ERROR_LOG(rc);
}
}
}
if (NULL != micro) {
/* pass the local peers */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_LOCAL_PEERS);
kv->type = OPAL_STRING;
kv->data.string = opal_argv_join(micro, ',');
opal_argv_free(micro);
opal_list_append(info, &kv->super);
}
/* pass the local ldr */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_LOCALLDR);
kv->type = OPAL_VPID;
kv->data.name.vpid = vpid;
opal_list_append(info, &kv->super);
/* for each proc in this job, create an object that
* includes the info describing the proc so the recipient has a complete
* picture. This allows procs to connect to each other without
* any further info exchange, assuming the underlying transports
* support it. We also pass all the proc-specific data here so
* that each proc can lookup info about every other proc in the job */
for (n=0; n < map->nodes->size; n++) {
if (NULL == (node = (orte_node_t*)opal_pointer_array_get_item(map->nodes, n))) {
continue;
}
/* cycle across each proc on this node, passing all data that
* varies by proc */
for (i=0; i < node->procs->size; i++) {
if (NULL == (pptr = (orte_proc_t*)opal_pointer_array_get_item(node->procs, i))) {
continue;
}
/* only consider procs from this job */
if (pptr->name.jobid != jdata->jobid) {
continue;
}
/* setup the proc map object */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_PROC_DATA);
kv->type = OPAL_PTR;
kv->data.ptr = OBJ_NEW(opal_list_t);
opal_list_append(info, &kv->super);
pmap = kv->data.ptr;
/* must start with rank */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_RANK);
kv->type = OPAL_VPID;
kv->data.name.vpid = pptr->name.vpid;
opal_list_append(pmap, &kv->super);
/* location, for local procs */
if (node == mynode) {
tmp = NULL;
if (orte_get_attribute(&pptr->attributes, ORTE_PROC_CPU_BITMAP, (void**)&tmp, OPAL_STRING) &&
NULL != tmp) {
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_LOCALITY_STRING);
kv->type = OPAL_STRING;
kv->data.string = opal_hwloc_base_get_locality_string(opal_hwloc_topology, tmp);
opal_list_append(pmap, &kv->super);
free(tmp);
} else {
/* the proc is not bound */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_LOCALITY_STRING);
kv->type = OPAL_STRING;
kv->data.string = NULL;
opal_list_append(pmap, &kv->super);
}
}
/* global/univ rank */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_GLOBAL_RANK);
kv->type = OPAL_VPID;
kv->data.name.vpid = pptr->name.vpid + jdata->offset;
opal_list_append(pmap, &kv->super);
if (1 < jdata->num_apps) {
/* appnum */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_APPNUM);
kv->type = OPAL_UINT32;
kv->data.uint32 = pptr->app_idx;
opal_list_append(pmap, &kv->super);
/* app ldr */
app = (orte_app_context_t*)opal_pointer_array_get_item(jdata->apps, pptr->app_idx);
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_APPLDR);
kv->type = OPAL_VPID;
kv->data.name.vpid = app->first_rank;
opal_list_append(pmap, &kv->super);
/* app rank */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_APP_RANK);
kv->type = OPAL_VPID;
kv->data.name.vpid = pptr->app_rank;
opal_list_append(pmap, &kv->super);
/* app size */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_APP_SIZE);
kv->type = OPAL_UINT32;
kv->data.uint32 = app->num_procs;
opal_list_append(info, &kv->super);
}
/* local rank */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_LOCAL_RANK);
kv->type = OPAL_UINT16;
kv->data.uint16 = pptr->local_rank;
opal_list_append(pmap, &kv->super);
/* node rank */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_NODE_RANK);
kv->type = OPAL_UINT16;
kv->data.uint32 = pptr->node_rank;
opal_list_append(pmap, &kv->super);
/* node ID */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_NODEID);
kv->type = OPAL_UINT32;
kv->data.uint32 = pptr->node->index;
opal_list_append(pmap, &kv->super);
if (map->num_nodes < orte_hostname_cutoff) {
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_HOSTNAME);
kv->type = OPAL_STRING;
kv->data.string = strdup(pptr->node->name);
opal_list_append(pmap, &kv->super);
}
}
}
/* mark the job as registered */
orte_set_attribute(&jdata->attributes, ORTE_JOB_NSPACE_REGISTERED, ORTE_ATTR_LOCAL, NULL, OPAL_BOOL);
/* pass it down */
/* we are in an event, so no need to callback */
rc = opal_pmix.server_register_nspace(jdata->jobid,
jdata->num_local_procs,
info, NULL, NULL);
OPAL_LIST_RELEASE(info);
/* if the user has connected us to an external server, then we must
* assume there is going to be some cross-mpirun exchange, and so
* we protect against that situation by publishing the job info
* for this job - this allows any subsequent "connect" to retrieve
* the job info */
if (NULL != orte_data_server_uri) {
opal_buffer_t buf;
OBJ_CONSTRUCT(&buf, opal_buffer_t);
if (OPAL_SUCCESS != (rc = opal_dss.pack(&buf, &jdata, 1, ORTE_JOB))) {
ORTE_ERROR_LOG(rc);
OBJ_DESTRUCT(&buf);
return rc;
}
info = OBJ_NEW(opal_list_t);
/* create a key-value with the key being the string jobid
* and the value being the byte object */
kv = OBJ_NEW(opal_value_t);
orte_util_convert_jobid_to_string(&kv->key, jdata->jobid);
kv->type = OPAL_BYTE_OBJECT;
opal_dss.unload(&buf, (void**)&kv->data.bo.bytes, &kv->data.bo.size);
OBJ_DESTRUCT(&buf);
opal_list_append(info, &kv->super);
/* set the range to be session */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_RANGE);
kv->type = OPAL_UINT;
kv->data.uint = OPAL_PMIX_RANGE_SESSION;
opal_list_append(info, &kv->super);
/* set the persistence to be app */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_PERSISTENCE);
kv->type = OPAL_INT;
kv->data.integer = OPAL_PMIX_PERSIST_APP;
opal_list_append(info, &kv->super);
/* add our effective userid to the directives */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_USERID);
kv->type = OPAL_UINT32;
kv->data.uint32 = geteuid();
opal_list_append(info, &kv->super);
/* now publish it */
if (ORTE_SUCCESS != (rc = pmix_server_publish_fn(ORTE_PROC_MY_NAME,
info, mycbfunc, info))) {
ORTE_ERROR_LOG(rc);
}
}
return rc;
}
int orte_ess_base_tool_setup(void)
{
int ret;
char *error = NULL;
opal_list_t transports;
orte_jobid_t jobid;
orte_vpid_t vpid;
/* setup the PMIx framework - ensure it skips all non-PMIx components,
* but do not override anything we were given */
opal_setenv("OMPI_MCA_pmix", "^s1,s2,cray,isolated", false, &environ);
if (OPAL_SUCCESS != (ret = mca_base_framework_open(&opal_pmix_base_framework, 0))) {
ORTE_ERROR_LOG(ret);
error = "orte_pmix_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = opal_pmix_base_select())) {
ORTE_ERROR_LOG(ret);
error = "opal_pmix_base_select";
goto error;
}
/* set the event base */
opal_pmix_base_set_evbase(orte_event_base);
/* we have to define our name here */
if (NULL != orte_ess_base_jobid &&
NULL != orte_ess_base_vpid) {
opal_output_verbose(2, orte_ess_base_framework.framework_output,
"ess:tool:obtaining name from environment");
if (ORTE_SUCCESS != (ret = orte_util_convert_string_to_jobid(&jobid, orte_ess_base_jobid))) {
return(ret);
}
ORTE_PROC_MY_NAME->jobid = jobid;
if (ORTE_SUCCESS != (ret = orte_util_convert_string_to_vpid(&vpid, orte_ess_base_vpid))) {
return(ret);
}
ORTE_PROC_MY_NAME->vpid = vpid;
} else {
/* If we are a tool with no name, then define it here */
uint16_t jobfam;
uint32_t hash32;
uint32_t bias;
opal_output_verbose(2, orte_ess_base_framework.framework_output,
"ess:tool:computing name");
/* hash the nodename */
OPAL_HASH_STR(orte_process_info.nodename, hash32);
bias = (uint32_t)orte_process_info.pid;
/* fold in the bias */
hash32 = hash32 ^ bias;
/* now compress to 16-bits */
jobfam = (uint16_t)(((0x0000ffff & (0xffff0000 & hash32) >> 16)) ^ (0x0000ffff & hash32));
/* set the name */
ORTE_PROC_MY_NAME->jobid = 0xffff0000 & ((uint32_t)jobfam << 16);
ORTE_PROC_MY_NAME->vpid = 0;
}
/* my name is set, xfer it to the OPAL layer */
orte_process_info.super.proc_name = *(opal_process_name_t*)ORTE_PROC_MY_NAME;
/* initialize - PMIx may set our name here if we attach to
* a PMIx server */
if (NULL != opal_pmix.tool_init) {
opal_list_t info;
opal_value_t *kv;
OBJ_CONSTRUCT(&info, opal_list_t);
/* pass our name so the PMIx layer can use it */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_TOOL_NSPACE);
orte_util_convert_jobid_to_string(&kv->data.string, ORTE_PROC_MY_NAME->jobid);
kv->type = OPAL_STRING;
opal_list_append(&info, &kv->super);
/* ditto for our rank */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_TOOL_RANK);
kv->data.name.vpid = ORTE_PROC_MY_NAME->vpid;
kv->type = OPAL_VPID;
opal_list_append(&info, &kv->super);
/* ORTE tools don't need to connect to a PMIx server as
* they will connect via the OOB */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_TOOL_DO_NOT_CONNECT);
kv->data.flag = true;
kv->type = OPAL_BOOL;
opal_list_append(&info, &kv->super);
if (OPAL_SUCCESS != (ret = opal_pmix.tool_init(&info))) {
ORTE_ERROR_LOG(ret);
error = "opal_pmix.init";
OPAL_LIST_DESTRUCT(&info);
goto error;
}
OPAL_LIST_DESTRUCT(&info);
ORTE_PROC_MY_NAME->jobid = OPAL_PROC_MY_NAME.jobid;
ORTE_PROC_MY_NAME->vpid = OPAL_PROC_MY_NAME.vpid;
}
orte_process_info.super.proc_hostname = strdup(orte_process_info.nodename);
orte_process_info.super.proc_flags = OPAL_PROC_ALL_LOCAL;
orte_process_info.super.proc_arch = opal_local_arch;
opal_proc_local_set(&orte_process_info.super);
/* open and setup the state machine */
if (ORTE_SUCCESS != (ret = mca_base_framework_open(&orte_state_base_framework, 0))) {
ORTE_ERROR_LOG(ret);
error = "orte_state_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_state_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_state_base_select";
goto error;
}
/* open and setup the error manager */
if (ORTE_SUCCESS != (ret = mca_base_framework_open(&orte_errmgr_base_framework, 0))) {
ORTE_ERROR_LOG(ret);
error = "orte_errmgr_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_errmgr_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_errmgr_base_select";
goto error;
}
/* Setup the communication infrastructure */
/* Routed system */
if (ORTE_SUCCESS != (ret = mca_base_framework_open(&orte_routed_base_framework, 0))) {
ORTE_ERROR_LOG(ret);
error = "orte_rml_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_routed_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_routed_base_select";
goto error;
}
if (ORTE_SUCCESS != (ret = mca_base_framework_open(&orte_oob_base_framework, 0))) {
ORTE_ERROR_LOG(ret);
error = "orte_oob_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_oob_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_oob_base_select";
goto error;
}
/* Runtime Messaging Layer */
if (ORTE_SUCCESS != (ret = mca_base_framework_open(&orte_rml_base_framework, 0))) {
ORTE_ERROR_LOG(ret);
error = "orte_rml_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_rml_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_rml_base_select";
goto error;
}
/* get a conduit for our use - we never route IO over fabric */
OBJ_CONSTRUCT(&transports, opal_list_t);
orte_set_attribute(&transports, ORTE_RML_TRANSPORT_TYPE,
ORTE_ATTR_LOCAL, orte_mgmt_transport, OPAL_STRING);
orte_mgmt_conduit = orte_rml.open_conduit(&transports);
OPAL_LIST_DESTRUCT(&transports);
/* since I am a tool, then all I really want to do is communicate.
* So setup communications and be done - finding the HNP
* to which I want to communicate and setting up a route for
* that link is my responsibility
*/
/* we -may- need to know the name of the head
* of our session directory tree, particularly the
* tmp base where any other session directories on
* this node might be located
*/
ret = orte_session_setup_base(ORTE_PROC_MY_NAME);
if (ORTE_SUCCESS != ret ) {
ORTE_ERROR_LOG(ret);
error = "define session dir names";
goto error;
}
/* setup I/O forwarding system - must come after we init routes */
if (NULL != orte_process_info.my_hnp_uri) {
/* only do this if we were given an HNP */
if (ORTE_SUCCESS != (ret = mca_base_framework_open(&orte_iof_base_framework, 0))) {
ORTE_ERROR_LOG(ret);
error = "orte_iof_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_iof_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_iof_base_select";
goto error;
}
/* if we were given an HNP, then also setup the PLM in case this
* tool wants to request that we spawn something for it */
if (ORTE_SUCCESS != (ret = mca_base_framework_open(&orte_plm_base_framework, 0))) {
ORTE_ERROR_LOG(ret);
error = "orte_plm_base_open";
goto error;
}
/* we don't select the plm framework as we only want the
* base proxy functions */
}
#if OPAL_ENABLE_FT_CR == 1
/*
* Setup the SnapC
*/
if (ORTE_SUCCESS != (ret = mca_base_framework_open(&orte_snapc_base_framework, 0))) {
ORTE_ERROR_LOG(ret);
error = "orte_snapc_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = mca_base_framework_open(&orte_sstore_base_framework, 0))) {
ORTE_ERROR_LOG(ret);
error = "orte_sstore_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_snapc_base_select(ORTE_PROC_IS_HNP, ORTE_PROC_IS_APP))) {
ORTE_ERROR_LOG(ret);
error = "orte_snapc_base_select";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_sstore_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_sstore_base_select";
goto error;
}
/* Tools do not need all the OPAL CR stuff */
opal_cr_set_enabled(false);
#endif
return ORTE_SUCCESS;
error:
orte_show_help("help-orte-runtime.txt",
"orte_init:startup:internal-failure",
true, error, ORTE_ERROR_NAME(ret), ret);
return ret;
}
