int orte_plm_poe_component_query(mca_base_module_t **module, int *priority)
{
/* if we are NOT an HNP, then don't select us */
if (!ORTE_PROC_IS_HNP) {
*priority = -1;
*module = NULL;
return ORTE_ERROR;
}
/* if we can't find the required elements, don't select us */
mca_plm_poe_component.path = opal_path_findv(mca_plm_poe_component.argv[0], 0, environ, NULL);
if (NULL == mca_plm_poe_component.path) {
*priority = -1;
*module = NULL;
return ORTE_ERROR;
}
mca_plm_poe_component.env = opal_path_findv(mca_plm_poe_component.env, 0, environ, NULL);
if (NULL == mca_plm_poe_component.env) {
*priority = -1;
*module = NULL;
return ORTE_ERROR;
}
/* guess we are good to go */
*priority = mca_plm_poe_component.priority;
*module = (mca_base_module_t*) &orte_plm_poe_module;
return ORTE_SUCCESS;
}
/**
* Try to figure out the absolute path based on the application name
* (usually argv[0]). If the path is already absolute return a copy, if
* it start with . look into the current directory, if not dig into
* the $PATH.
* In case of error or if executable was not found (as an example if
* the application did a cwd between the start and this call), the
* function will return NULL. Otherwise, an newly allocated string
* will be returned.
*/
char* opal_find_absolute_path( char* app_name )
{
char* abs_app_name;
char cwd[OPAL_PATH_MAX], *pcwd;
if( opal_path_is_absolute(app_name) ) { /* already absolute path */
abs_app_name = app_name;
} else if ( '.' == app_name[0] ||
NULL != strchr(app_name, OPAL_PATH_SEP[0])) {
/* the app is in the current directory or below it */
pcwd = getcwd( cwd, OPAL_PATH_MAX );
if( NULL == pcwd ) {
/* too bad there is no way we can get the app absolute name */
return NULL;
}
abs_app_name = opal_os_path( false, pcwd, app_name, NULL );
} else {
/* Otherwise try to search for the application in the PATH ... */
abs_app_name = opal_path_findv( app_name, X_OK, NULL, NULL );
}
if( NULL != abs_app_name ) {
char* resolved_path = (char*)malloc(OPAL_PATH_MAX);
#if !defined(__WINDOWS__)
realpath( abs_app_name, resolved_path );
#else
#ifdef HAVE_SHLWAPI_H
PathCanonicalize(resolved_path, abs_app_name);
#endif
#endif /* !defined(__WINDOWS__) */
if( abs_app_name != app_name ) free(abs_app_name);
return resolved_path;
}
return NULL;
}
static int init(void)
{
char cwd[OPAL_PATH_MAX];
int rc;
OPAL_OUTPUT_VERBOSE((5, orcm_pvsn_base_framework.framework_output,
"%s pvsn:wwulf:init",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
if (OPAL_SUCCESS != (rc = opal_getcwd(cwd, OPAL_PATH_MAX))) {
return rc;
}
/* check to see if we can execute wwsh */
cmd = opal_path_findv("wwsh", X_OK, environ, cwd);
if (NULL == cmd) {
return ORTE_ERR_EXE_NOT_FOUND;
}
OPAL_OUTPUT_VERBOSE((5, orcm_pvsn_base_framework.framework_output,
"%s pvsn:wwulf:init path to wwsh %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), cmd));
return ORCM_SUCCESS;
}
int orte_util_check_context_app(orte_app_context_t *context, char **env)
{
char *tmp;
/* If the app is a naked filename, we need to do a path search for
it. orterun will send in whatever the user specified (e.g.,
"orterun -np 2 uptime"), so in some cases, we need to search
the path to verify that we can find it. Here's the
possibilities:
1. The user specified an absolute pathname for the executable.
We simply need to verify that it exists and we can run it.
2. The user specified a relative pathname for the executable.
Ditto with #1 -- based on the cwd, we need to verify that it
exists and we can run it.
3. The user specified a naked filename. We need to search the
path, find a match, and verify that we can run it.
Note that in some cases, we won't be doing this work here --
bproc, for example, does not use the fork pls for launching, so
it does this same work over there. */
tmp = opal_basename(context->argv[0]);
if (strlen(tmp) == strlen(context->argv[0])) {
/* If this is a naked executable -- no relative or absolute
pathname -- then search the PATH for it */
free(tmp);
tmp = opal_path_findv(context->argv[0], X_OK, env, context->cwd);
if (NULL == tmp) {
return ORTE_ERR_EXE_NOT_FOUND;
}
if (NULL != context->app) free(context->app);
context->app = tmp;
} else {
if (0 != access(context->app, X_OK)) {
return ORTE_ERR_EXE_NOT_ACCESSIBLE;
}
}
/* All was good */
return ORTE_SUCCESS;
}
static int plm_alps_start_proc(int argc, char **argv, char **env,
char *prefix)
{
int fd;
pid_t alps_pid;
char *exec_argv = opal_path_findv(argv[0], 0, env, NULL);
if (NULL == exec_argv) {
return ORTE_ERR_NOT_FOUND;
}
alps_pid = fork();
if (-1 == alps_pid) {
ORTE_ERROR_LOG(ORTE_ERR_SYS_LIMITS_CHILDREN);
return ORTE_ERR_SYS_LIMITS_CHILDREN;
}
alpsrun = OBJ_NEW(orte_proc_t);
alpsrun->pid = alps_pid;
/* be sure to mark it as alive so we don't instantly fire */
ORTE_FLAG_SET(alpsrun, ORTE_PROC_FLAG_ALIVE);
/* setup the waitpid so we can find out if alps succeeds! */
orte_wait_cb(alpsrun, alps_wait_cb, NULL);
if (0 == alps_pid) { /* child */
char *bin_base = NULL, *lib_base = NULL;
/* Figure out the basenames for the libdir and bindir. There
is a lengthy comment about this in plm_rsh_module.c
explaining all the rationale for how / why we're doing
this. */
lib_base = opal_basename(opal_install_dirs.libdir);
bin_base = opal_basename(opal_install_dirs.bindir);
/* If we have a prefix, then modify the PATH and
LD_LIBRARY_PATH environment variables. */
if (NULL != prefix) {
char *oldenv, *newenv;
/* Reset PATH */
oldenv = getenv("PATH");
if (NULL != oldenv) {
asprintf(&newenv, "%s/%s:%s", prefix, bin_base, oldenv);
} else {
asprintf(&newenv, "%s/%s", prefix, bin_base);
}
opal_setenv("PATH", newenv, true, &env);
if (mca_plm_alps_component.debug) {
opal_output(0, "plm:alps: reset PATH: %s", newenv);
}
free(newenv);
/* Reset LD_LIBRARY_PATH */
oldenv = getenv("LD_LIBRARY_PATH");
if (NULL != oldenv) {
asprintf(&newenv, "%s/%s:%s", prefix, lib_base, oldenv);
} else {
asprintf(&newenv, "%s/%s", prefix, lib_base);
}
opal_setenv("LD_LIBRARY_PATH", newenv, true, &env);
if (mca_plm_alps_component.debug) {
opal_output(0, "plm:alps: reset LD_LIBRARY_PATH: %s",
newenv);
}
free(newenv);
}
fd = open("/dev/null", O_CREAT|O_WRONLY|O_TRUNC, 0666);
if(fd > 0) {
dup2(fd, 0);
}
/* When not in debug mode and --debug-daemons was not passed,
* tie stdout/stderr to dev null so we don't see messages from orted */
if (0 == mca_plm_alps_component.debug && !orte_debug_daemons_flag) {
if (fd >= 0) {
if (fd != 1) {
dup2(fd,1);
}
if (fd != 2) {
dup2(fd,2);
}
}
}
if (fd > 2) {
close(fd);
}
/* get the alps process out of orterun's process group so that
signals sent from the shell (like those resulting from
cntl-c) don't get sent to alps */
setpgid(0, 0);
execve(exec_argv, argv, env);
opal_output(0, "plm:alps:start_proc: exec failed");
/* don't return - need to exit - returning would be bad -
we're not in the calling process anymore */
exit(1);
} else { /* parent */
/* just in case, make sure that the alps process is not in our
process group any more. Stevens says always do this on both
sides of the fork... */
setpgid(alps_pid, alps_pid);
free(exec_argv);
}
return ORTE_SUCCESS;
}
static int plm_slurm_start_proc(int argc, char **argv, char **env,
char *prefix)
{
int fd;
int srun_pid;
char *exec_argv = opal_path_findv(argv[0], 0, env, NULL);
orte_proc_t *dummy;
if (NULL == exec_argv) {
return ORTE_ERR_NOT_FOUND;
}
srun_pid = fork();
if (-1 == srun_pid) {
ORTE_ERROR_LOG(ORTE_ERR_SYS_LIMITS_CHILDREN);
free(exec_argv);
return ORTE_ERR_SYS_LIMITS_CHILDREN;
}
/* setup a dummy proc object to track the srun */
dummy = OBJ_NEW(orte_proc_t);
dummy->pid = srun_pid;
/* be sure to mark it as alive so we don't instantly fire */
ORTE_FLAG_SET(dummy, ORTE_PROC_FLAG_ALIVE);
/* setup the waitpid so we can find out if srun succeeds! */
orte_wait_cb(dummy, srun_wait_cb, NULL);
if (0 == srun_pid) { /* child */
char *bin_base = NULL, *lib_base = NULL;
/* Figure out the basenames for the libdir and bindir. There
is a lengthy comment about this in plm_rsh_module.c
explaining all the rationale for how / why we're doing
this. */
lib_base = opal_basename(opal_install_dirs.libdir);
bin_base = opal_basename(opal_install_dirs.bindir);
/* If we have a prefix, then modify the PATH and
LD_LIBRARY_PATH environment variables. */
if (NULL != prefix) {
char *oldenv, *newenv;
/* Reset PATH */
oldenv = getenv("PATH");
if (NULL != oldenv) {
asprintf(&newenv, "%s/%s:%s", prefix, bin_base, oldenv);
} else {
asprintf(&newenv, "%s/%s", prefix, bin_base);
}
opal_setenv("PATH", newenv, true, &env);
OPAL_OUTPUT_VERBOSE((1, orte_plm_base_framework.framework_output,
"%s plm:slurm: reset PATH: %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
newenv));
free(newenv);
/* Reset LD_LIBRARY_PATH */
oldenv = getenv("LD_LIBRARY_PATH");
if (NULL != oldenv) {
asprintf(&newenv, "%s/%s:%s", prefix, lib_base, oldenv);
} else {
asprintf(&newenv, "%s/%s", prefix, lib_base);
}
opal_setenv("LD_LIBRARY_PATH", newenv, true, &env);
OPAL_OUTPUT_VERBOSE((1, orte_plm_base_framework.framework_output,
"%s plm:slurm: reset LD_LIBRARY_PATH: %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
newenv));
free(newenv);
}
fd = open("/dev/null", O_CREAT|O_RDWR|O_TRUNC, 0666);
if (fd >= 0) {
dup2(fd, 0);
/* When not in debug mode and --debug-daemons was not passed,
* tie stdout/stderr to dev null so we don't see messages from orted
* EXCEPT if the user has requested that we leave sessions attached
*/
if (0 > opal_output_get_verbosity(orte_plm_base_framework.framework_output) &&
!orte_debug_daemons_flag && !orte_leave_session_attached) {
dup2(fd,1);
dup2(fd,2);
}
/* Don't leave the extra fd to /dev/null open */
if (fd > 2) {
close(fd);
}
}
/* get the srun process out of orterun's process group so that
signals sent from the shell (like those resulting from
cntl-c) don't get sent to srun */
setpgid(0, 0);
execve(exec_argv, argv, env);
opal_output(0, "plm:slurm:start_proc: exec failed");
/* don't return - need to exit - returning would be bad -
we're not in the calling process anymore */
exit(1);
} else { /* parent */
/* just in case, make sure that the srun process is not in our
process group any more. Stevens says always do this on both
sides of the fork... */
setpgid(srun_pid, srun_pid);
/* if this is the primary launch - i.e., not a comm_spawn of a
* child job - then save the pid
*/
if (!primary_pid_set) {
primary_srun_pid = srun_pid;
primary_pid_set = true;
}
free(exec_argv);
}
return ORTE_SUCCESS;
}
/*
* Process one line from the orte_base_user_debugger MCA param and
* look for that debugger in the path. If we find it, fill in
* new_argv.
*/
static int process(char *orig_line, char *basename, opal_cmd_line_t *cmd_line,
int argc, char **argv, char ***new_argv, int num_procs)
{
int ret = ORTE_SUCCESS;
int i, j, count;
char *line = NULL, *tmp = NULL, *full_line = strdup(orig_line);
char **orterun_argv = NULL, **executable_argv = NULL, **line_argv = NULL;
char cwd[OPAL_PATH_MAX];
bool used_num_procs = false;
bool single_app = false;
bool fail_needed_executable = false;
line = full_line;
if (NULL == line) {
ret = ORTE_ERR_OUT_OF_RESOURCE;
goto out;
}
/* Trim off whitespace at the beginning and ending of line */
for (i = 0; '\0' != line[i] && isspace(line[i]); ++line) {
continue;
}
for (i = strlen(line) - 2; i > 0 && isspace(line[i]); ++i) {
line[i] = '\0';
}
if (strlen(line) <= 0) {
ret = ORTE_ERROR;
goto out;
}
/* Get the tail of the command line (i.e., the user executable /
argv) */
opal_cmd_line_get_tail(cmd_line, &i, &executable_argv);
/* Make a new copy of the orterun command line args, without the
orterun token itself, and without the --debug, --debugger, and
-tv flags. */
orterun_argv = opal_argv_copy(argv);
count = opal_argv_count(orterun_argv);
opal_argv_delete(&count, &orterun_argv, 0, 1);
for (i = 0; NULL != orterun_argv[i]; ++i) {
count = opal_argv_count(orterun_argv);
if (0 == strcmp(orterun_argv[i], "-debug") ||
0 == strcmp(orterun_argv[i], "--debug")) {
opal_argv_delete(&count, &orterun_argv, i, 1);
} else if (0 == strcmp(orterun_argv[i], "-tv") ||
0 == strcmp(orterun_argv[i], "--tv")) {
opal_argv_delete(&count, &orterun_argv, i, 1);
} else if (0 == strcmp(orterun_argv[i], "--debugger") ||
0 == strcmp(orterun_argv[i], "-debugger")) {
opal_argv_delete(&count, &orterun_argv, i, 2);
}
}
/* Replace @@ tokens - line should never realistically be bigger
than MAX_INT, so just cast to int to remove compiler warning */
*new_argv = NULL;
line_argv = opal_argv_split(line, ' ');
if (NULL == line_argv) {
ret = ORTE_ERR_NOT_FOUND;
goto out;
}
for (i = 0; NULL != line_argv[i]; ++i) {
if (0 == strcmp(line_argv[i], "@mpirun@") ||
0 == strcmp(line_argv[i], "@orterun@")) {
opal_argv_append_nosize(new_argv, argv[0]);
} else if (0 == strcmp(line_argv[i], "@mpirun_args@") ||
0 == strcmp(line_argv[i], "@orterun_args@")) {
for (j = 0; NULL != orterun_argv && NULL != orterun_argv[j]; ++j) {
opal_argv_append_nosize(new_argv, orterun_argv[j]);
}
} else if (0 == strcmp(line_argv[i], "@np@")) {
asprintf(&tmp, "%d", num_procs);
opal_argv_append_nosize(new_argv, tmp);
free(tmp);
} else if (0 == strcmp(line_argv[i], "@single_app@")) {
/* This token is only a flag; it is not replaced with any
alternate text */
single_app = true;
} else if (0 == strcmp(line_argv[i], "@executable@")) {
/* If we found the executable, paste it in. Otherwise,
this is a possible error. */
if (NULL != executable_argv) {
opal_argv_append_nosize(new_argv, executable_argv[0]);
} else {
fail_needed_executable = true;
}
} else if (0 == strcmp(line_argv[i], "@executable_argv@")) {
/* If we found the tail, paste in the argv. Otherwise,
this is a possible error. */
if (NULL != executable_argv) {
for (j = 1; NULL != executable_argv[j]; ++j) {
opal_argv_append_nosize(new_argv, executable_argv[j]);
}
} else {
fail_needed_executable = true;
}
} else {
/* It wasn't a special token, so just copy it over */
opal_argv_append_nosize(new_argv, line_argv[i]);
}
}
/* Can we find argv[0] in the path? */
getcwd(cwd, OPAL_PATH_MAX);
tmp = opal_path_findv((*new_argv)[0], X_OK, environ, cwd);
if (NULL != tmp) {
free(tmp);
/* Ok, we found a good debugger. Check for some error
conditions. */
tmp = opal_argv_join(argv, ' ');
/* We do not support launching a debugger that requires the
-np value if the user did not specify -np on the command
line. */
if (used_num_procs && 0 == num_procs) {
free(tmp);
tmp = opal_argv_join(orterun_argv, ' ');
orte_show_help("help-orterun.txt", "debugger requires -np",
true, (*new_argv)[0], argv[0], tmp,
(*new_argv)[0]);
/* Fall through to free / fail, below */
}
/* Some debuggers do not support launching MPMD */
else if (single_app && NULL != strchr(tmp, ':')) {
orte_show_help("help-orterun.txt",
"debugger only accepts single app", true,
(*new_argv)[0], (*new_argv)[0]);
/* Fall through to free / fail, below */
}
/* Some debuggers do not use orterun/mpirun, and therefore
must have an executable to run (e.g., cannot use mpirun's
app context file feature). */
else if (fail_needed_executable) {
orte_show_help("help-orterun.txt",
"debugger requires executable", true,
(*new_argv)[0], argv[0], (*new_argv)[0], argv[0],
(*new_argv)[0]);
/* Fall through to free / fail, below */
}
/* Otherwise, we succeeded. Return happiness. */
else {
goto out;
}
free(tmp);
}
/* All done -- didn't find it */
opal_argv_free(*new_argv);
*new_argv = NULL;
ret = ORTE_ERR_NOT_FOUND;
out:
if (NULL != orterun_argv) {
opal_argv_free(orterun_argv);
}
if (NULL != executable_argv) {
opal_argv_free(executable_argv);
}
if (NULL != line_argv) {
opal_argv_free(line_argv);
}
if (NULL != tmp) {
free(tmp);
}
if (NULL != full_line) {
free(full_line);
}
return ret;
}
static int setup_app(orte_app_context_t *app)
{
int i;
char *newenv, *pth, *t2;
bool takeus = false;
if (NULL != orte_schizo_base.personalities) {
/* see if we are included */
for (i=0; NULL != orte_schizo_base.personalities[i]; i++) {
if (0 == strcmp(orte_schizo_base.personalities[i], "singularity")) {
takeus = true;
break;
}
}
}
if (!takeus) {
/* even if they didn't specify, check to see if
* this involves a singularity container */
if (0 != strcmp(app->argv[0],"singularity") &&
0 != strcmp(app->argv[0],"sapprun") &&
NULL == strstr(app->argv[0], ".sapp")) {
/* guess not! */
return ORTE_ERR_TAKE_NEXT_OPTION;
}
}
opal_output_verbose(1, orte_schizo_base_framework.framework_output,
"%s schizo:singularity: checking app %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), app->argv[0]);
if (0 < strlen(OPAL_SINGULARITY_PATH)) {
asprintf(&pth, "%s/singularity", OPAL_SINGULARITY_PATH);
} else {
/* since we allow for detecting singularity's presence, it
* is possible that we found it in the PATH, but not in a
* standard location. Check for that here */
pth = opal_path_findv("singularity", X_OK, app->env, NULL);
if (NULL == pth) {
/* cannot execute */
return ORTE_ERR_TAKE_NEXT_OPTION;
}
}
/* find the path and prepend it with the path to Singularity */
for (i = 0; NULL != app->env && NULL != app->env[i]; ++i) {
/* add to PATH */
if (0 == strncmp("PATH=", app->env[i], 5)) {
t2 = opal_dirname(pth);
asprintf(&newenv, "%s:%s", t2, app->env[i] + 5);
opal_setenv("PATH", newenv, true, &app->env);
free(newenv);
free(t2);
break;
}
}
free(pth);
if (0 == strcmp(app->argv[0], "singularity")) {
/* we don't want the backend to setup a cache dir */
orte_set_attribute(&app->attributes, ORTE_APP_NO_CACHEDIR, ORTE_ATTR_GLOBAL, NULL, OPAL_BOOL);
}
/* export an envar to permit shared memory operations */
opal_setenv("SINGULARITY_NO_NAMESPACE_PID", "1", true, &app->env);
return ORTE_SUCCESS;
}
/*
* Process one line from the orte_base_user_debugger MCA param and
* look for that debugger in the path. If we find it, fill in
* new_argv.
*/
static int process(char *orig_line, char *basename, opal_cmd_line_t *cmd_line,
int argc, char **argv, char ***new_argv)
{
int i;
char *line, *full_line = strdup(orig_line);
char *user_argv, *tmp, *tmp2, **tmp_argv, **executable;
char cwd[PATH_MAX];
bool used_num_procs = false;
bool fail_single_app = false;
bool fail_needed_executable = false;
line = full_line;
if (NULL == line) {
return ORTE_ERR_OUT_OF_RESOURCE;
}
/* Trim off whitespace at the beginning and ending of line */
for (i = 0; '\0' != line[i] && isspace(line[i]); ++line) {
continue;
}
for (i = strlen(line) - 2; i > 0 && isspace(line[i]); ++i) {
line[i] = '\0';
}
if (strlen(line) <= 0) {
return ORTE_ERROR;
}
/* Get the tail of the command line (i.e., the user executable /
argv) */
opal_cmd_line_get_tail(cmd_line, &i, &executable);
/* Remove --debug, --debugger, and -tv from the user command line
params */
if (1 == argc) {
user_argv = strdup("");
} else {
tmp_argv = opal_argv_copy(argv);
for (i = 0; NULL != tmp_argv[i]; ++i) {
if (0 == strcmp(tmp_argv[i], "-debug") ||
0 == strcmp(tmp_argv[i], "--debug")) {
free(tmp_argv[i]);
tmp_argv[i] = strdup("");
} else if (0 == strcmp(tmp_argv[i], "-tv") ||
0 == strcmp(tmp_argv[i], "--tv")) {
free(tmp_argv[i]);
tmp_argv[i] = strdup("");
} else if (0 == strcmp(tmp_argv[i], "--debugger") ||
0 == strcmp(tmp_argv[i], "-debugger")) {
free(tmp_argv[i]);
tmp_argv[i] = strdup("");
if (NULL != tmp_argv[i + 1]) {
++i;
free(tmp_argv[i]);
tmp_argv[i] = strdup("");
}
}
}
user_argv = opal_argv_join(tmp_argv + 1, ' ');
opal_argv_free(tmp_argv);
}
/* Replace @@ tokens - line should never realistically be bigger
than MAX_INT, so just cast to int to remove compiler warning */
for (i = 0; i < (int) strlen(line); ++i) {
tmp = NULL;
if (0 == strncmp(line + i, "@mpirun@", 8)) {
line[i] = '\0';
asprintf(&tmp, "%s%s%s", line, argv[0], line + i + 8);
} else if (0 == strncmp(line + i, "@orterun@", 9)) {
line[i] = '\0';
asprintf(&tmp, "%s%s%s", line, argv[0], line + i + 9);
} else if (0 == strncmp(line + i, "@mpirun_args@", 13)) {
line[i] = '\0';
asprintf(&tmp, "%s%s%s", line, user_argv, line + i + 13);
} else if (0 == strncmp(line + i, "@orterun_args@", 14)) {
line[i] = '\0';
asprintf(&tmp, "%s%s%s", line, user_argv, line + i + 14);
} else if (0 == strncmp(line + i, "@np@", 4)) {
line[i] = '\0';
asprintf(&tmp, "%s%d%s", line, orterun_globals.num_procs,
line + i + 4);
used_num_procs = true;
} else if (0 == strncmp(line + i, "@single_app@", 12)) {
line[i] = '\0';
/* This token is only a flag; it is not replaced with any
alternate text */
asprintf(&tmp, "%s%s", line, line + i + 12);
tmp2 = opal_argv_join(executable + 1, ' ');
if (NULL != strchr(tmp2, ':')) {
fail_single_app = true;
}
free(tmp2);
} else if (0 == strncmp(line + i, "@executable@", 12)) {
line[i] = '\0';
/* If we found the executable, paste it in. Otherwise,
this is a possible error. */
if (NULL != executable) {
asprintf(&tmp, "%s%s%s", line, executable[0], line + i + 12);
} else {
fail_needed_executable = true;
}
} else if (0 == strncmp(line + i, "@executable_argv@", 17)) {
line[i] = '\0';
/* If we found the tail, paste in the argv. Otherwise,
this is a possible error. */
if (NULL != executable) {
if (NULL != executable[1]) {
/* Put in the argv */
tmp2 = opal_argv_join(executable + 1, ' ');
asprintf(&tmp, "%s%s%s", line, tmp2, line + i + 17);
free(tmp2);
} else {
/* There is no argv; just paste the front and back
together, removing the @token@ */
asprintf(&tmp, "%s%s", line, line + i + 17);
}
} else {
fail_needed_executable = true;
}
}
if (NULL != tmp) {
free(full_line);
full_line = line = tmp;
--i;
}
}
/* Split up into argv */
*new_argv = opal_argv_split(line, ' ');
free(line);
/* Can we find argv[0] in the path? */
getcwd(cwd, PATH_MAX);
tmp = opal_path_findv((*new_argv)[0], X_OK, environ, cwd);
if (NULL != tmp) {
free(tmp);
/* Ok, we found a good debugger. Check for some error
conditions. */
/* We do not support launching a debugger that requires the
-np value if the user did not specify -np on the command
line. */
if (used_num_procs && 0 == orterun_globals.num_procs) {
opal_show_help("help-orterun.txt", "debugger requires -np",
true, (*new_argv)[0], argv[0], user_argv,
(*new_argv)[0]);
/* Fall through to free / fail, below */
}
/* Some debuggers do not support launching MPMD */
else if (fail_single_app) {
opal_show_help("help-orterun.txt",
"debugger only accepts single app", true,
(*new_argv)[0], (*new_argv)[0]);
/* Fall through to free / fail, below */
}
/* Some debuggers do not use orterun/mpirun, and therefore
must have an executable to run (e.g., cannot use mpirun's
app context file feature). */
else if (fail_needed_executable) {
opal_show_help("help-orterun.txt",
"debugger requires executable", true,
(*new_argv)[0], argv[0], (*new_argv)[0], argv[0],
(*new_argv)[0]);
/* Fall through to free / fail, below */
}
/* Otherwise, we succeeded. Return happiness. */
else {
return ORTE_SUCCESS;
}
}
/* All done -- didn't find it */
opal_argv_free(*new_argv);
*new_argv = NULL;
return ORTE_ERR_NOT_FOUND;
}
int
main(int argc, char *argv[])
{
int exit_status = 0, ret, flags = 0, i;
int exec_argc = 0, user_argc = 0;
char **exec_argv = NULL, **user_argv = NULL;
char *exec_command, *base_argv0 = NULL;
bool disable_flags = true;
bool real_flag = false;
if (OPAL_SUCCESS != (ret = opal_init_util())) {
return ret;
}
/****************************************************
*
* Setup compiler information
*
****************************************************/
base_argv0 = opal_basename(argv[0]);
#if defined(EXEEXT)
if( 0 != strlen(EXEEXT) ) {
char extension[] = EXEEXT;
char* temp = strstr( base_argv0, extension );
char* old_match = temp;
while( NULL != temp ) {
old_match = temp;
temp = strstr( temp + 1, extension );
}
/* Only if there was a match of .exe, erase the last occurence of .exe */
if ( NULL != old_match ) {
*old_match = '\0';
}
}
#endif /* defined(EXEEXT) */
if (OPAL_SUCCESS != (ret = data_init(base_argv0))) {
fprintf(stderr, "Error parsing data file %s: %s\n", base_argv0, opal_strerror(ret));
return ret;
}
for (i = 1 ; i < argc && user_data_idx < 0 ; ++i) {
user_data_idx = find_options_index(argv[i]);
}
/* if we didn't find a match, look for the NULL (base case) options */
if (user_data_idx < 0) {
user_data_idx = default_data_idx;
}
/* if we still didn't find a match, abort */
if (user_data_idx < 0) {
char *flat = opal_argv_join(argv, ' ');
opal_show_help("help-opal-wrapper.txt", "no-options-support", true,
base_argv0, flat, NULL);
free(flat);
exit(1);
}
/* compiler */
load_env_data(options_data[user_data_idx].project_short, options_data[user_data_idx].compiler_env, &options_data[user_data_idx].compiler);
/* preprocessor flags */
load_env_data_argv(options_data[user_data_idx].project_short, "CPPFLAGS", &options_data[user_data_idx].preproc_flags);
/* compiler flags */
load_env_data_argv(options_data[user_data_idx].project_short, options_data[user_data_idx].compiler_flags_env,
&options_data[user_data_idx].comp_flags);
/* linker flags */
load_env_data_argv(options_data[user_data_idx].project_short, "LDFLAGS", &options_data[user_data_idx].link_flags);
/* libs */
load_env_data_argv(options_data[user_data_idx].project_short, "LIBS", &options_data[user_data_idx].libs);
/****************************************************
*
* Sanity Checks
*
****************************************************/
if (NULL != options_data[user_data_idx].req_file) {
/* make sure the language is supported */
if (0 == strcmp(options_data[user_data_idx].req_file, "not supported")) {
opal_show_help("help-opal-wrapper.txt", "no-language-support", true,
options_data[user_data_idx].language, base_argv0, NULL);
exit_status = 1;
goto cleanup;
}
if (options_data[user_data_idx].req_file[0] != '\0') {
char *filename;
struct stat buf;
filename = opal_os_path( false, options_data[user_data_idx].path_libdir, options_data[user_data_idx].req_file, NULL );
if (0 != stat(filename, &buf)) {
opal_show_help("help-opal-wrapper.txt", "file-not-found", true,
base_argv0, options_data[user_data_idx].req_file, options_data[user_data_idx].language, NULL);
}
}
}
/****************************************************
*
* Parse user flags
*
****************************************************/
flags = COMP_WANT_COMMAND|COMP_WANT_PREPROC|
COMP_WANT_COMPILE|COMP_WANT_LINK;
user_argv = opal_argv_copy(argv + 1);
user_argc = opal_argv_count(user_argv);
for (i = 0 ; i < user_argc ; ++i) {
if (0 == strncmp(user_argv[i], "-showme", strlen("-showme")) ||
0 == strncmp(user_argv[i], "--showme", strlen("--showme")) ||
0 == strncmp(user_argv[i], "-show", strlen("-show")) ||
0 == strncmp(user_argv[i], "--show", strlen("--show"))) {
bool done_now = false;
/* check for specific things we want to see. First three
still invoke all the building routines. Last set want
to parse out certain flags, so we don't go through the
normal build routine - skip to cleanup. */
if (0 == strncmp(user_argv[i], "-showme:command", strlen("-showme:command")) ||
0 == strncmp(user_argv[i], "--showme:command", strlen("--showme:command"))) {
flags = COMP_WANT_COMMAND;
/* we know what we want, so don't process any more args */
done_now = true;
} else if (0 == strncmp(user_argv[i], "-showme:compile", strlen("-showme:compile")) ||
0 == strncmp(user_argv[i], "--showme:compile", strlen("--showme:compile"))) {
flags = COMP_WANT_PREPROC|COMP_WANT_COMPILE;
/* we know what we want, so don't process any more args */
done_now = true;
} else if (0 == strncmp(user_argv[i], "-showme:link", strlen("-showme:link")) ||
0 == strncmp(user_argv[i], "--showme:link", strlen("--showme:link"))) {
flags = COMP_WANT_COMPILE|COMP_WANT_LINK;
/* we know what we want, so don't process any more args */
done_now = true;
} else if (0 == strncmp(user_argv[i], "-showme:incdirs", strlen("-showme:incdirs")) ||
0 == strncmp(user_argv[i], "--showme:incdirs", strlen("--showme:incdirs"))) {
print_flags(options_data[user_data_idx].preproc_flags, OPAL_INCLUDE_FLAG);
goto cleanup;
} else if (0 == strncmp(user_argv[i], "-showme:libdirs", strlen("-showme:libdirs")) ||
0 == strncmp(user_argv[i], "--showme:libdirs", strlen("--showme:libdirs"))) {
print_flags(options_data[user_data_idx].link_flags, OPAL_LIBDIR_FLAG);
goto cleanup;
} else if (0 == strncmp(user_argv[i], "-showme:libs", strlen("-showme:libs")) ||
0 == strncmp(user_argv[i], "--showme:libs", strlen("--showme:libs"))) {
print_flags(options_data[user_data_idx].libs, "-l");
goto cleanup;
} else if (0 == strncmp(user_argv[i], "-showme:version", strlen("-showme:version")) ||
0 == strncmp(user_argv[i], "--showme:version", strlen("--showme:version"))) {
opal_show_help("help-opal-wrapper.txt", "version", false,
argv[0], options_data[user_data_idx].project, options_data[user_data_idx].version, options_data[user_data_idx].language, NULL);
goto cleanup;
} else if (0 == strncmp(user_argv[i], "-showme:", strlen("-showme:")) ||
0 == strncmp(user_argv[i], "--showme:", strlen("--showme:"))) {
opal_show_help("help-opal-wrapper.txt", "usage", true,
argv[0], options_data[user_data_idx].project, NULL);
goto cleanup;
}
flags |= (COMP_DRY_RUN|COMP_SHOW_ERROR);
/* remove element from user_argv */
opal_argv_delete(&user_argc, &user_argv, i, 1);
--i;
if (done_now) {
disable_flags = false;
break;
}
} else if (0 == strcmp(user_argv[i], "-c")) {
flags &= ~COMP_WANT_LINK;
real_flag = true;
} else if (0 == strcmp(user_argv[i], "-E") ||
0 == strcmp(user_argv[i], "-M")) {
flags &= ~(COMP_WANT_COMPILE | COMP_WANT_LINK);
real_flag = true;
} else if (0 == strcmp(user_argv[i], "-S")) {
flags &= ~COMP_WANT_LINK;
real_flag = true;
} else if (0 == strcmp(user_argv[i], "-lpmpi")) {
flags |= COMP_WANT_PMPI;
/* remove element from user_argv */
opal_argv_delete(&user_argc, &user_argv, i, 1);
--i;
} else if ('-' != user_argv[i][0]) {
disable_flags = false;
flags |= COMP_SHOW_ERROR;
real_flag = true;
} else {
/* if the option flag is one that we use to determine
which set of compiler data to use, don't count it as a
real option */
if (find_options_index(user_argv[i]) < 0) {
real_flag = true;
}
}
}
/* clear out the want_flags if we got no arguments not starting
with a - (dash) and -showme wasn't given OR -showme was given
and we had at least one more non-showme argument that started
with a - (dash) and no other non-dash arguments. Some examples:
opal_wrapper : clear our flags
opal_wrapper -v : clear our flags
opal_wrapper -E a.c : don't clear our flags
opal_wrapper a.c : don't clear our flags
opal_wrapper -showme : don't clear our flags
opal_wrapper -showme -v : clear our flags
opal_wrapper -showme -E a.c : don't clear our flags
opal_wrapper -showme a.c : don't clear our flags
*/
if (disable_flags && !((flags & COMP_DRY_RUN) && !real_flag)) {
flags &= ~(COMP_WANT_PREPROC|COMP_WANT_COMPILE|COMP_WANT_LINK);
}
#if !OMPI_ENABLE_MPI_PROFILING
/* sanity check */
if (flags & COMP_WANT_PMPI) {
opal_show_help("help-opal-wrapper.txt", "no-profiling-support", true,
argv[0], NULL);
}
#endif
/****************************************************
*
* Assemble the command line
*
****************************************************/
/* compiler (may be multiple arguments, so split) */
if (flags & COMP_WANT_COMMAND) {
exec_argv = opal_argv_split(options_data[user_data_idx].compiler, ' ');
exec_argc = opal_argv_count(exec_argv);
} else {
exec_argv = (char **) malloc(sizeof(char*));
exec_argv[0] = NULL;
exec_argc = 0;
}
/* Per https://svn.open-mpi.org/trac/ompi/ticket/2201, add all the
user arguments before anything else. */
opal_argv_insert(&exec_argv, exec_argc, user_argv);
exec_argc = opal_argv_count(exec_argv);
/* preproc flags */
if (flags & COMP_WANT_PREPROC) {
opal_argv_insert(&exec_argv, exec_argc, options_data[user_data_idx].preproc_flags);
exec_argc = opal_argv_count(exec_argv);
}
/* compiler flags */
if (flags & COMP_WANT_COMPILE) {
opal_argv_insert(&exec_argv, exec_argc, options_data[user_data_idx].comp_flags);
/* Deal with languages like Fortran 90 that have special
places and flags for modules or whatever */
if (options_data[user_data_idx].module_option != NULL) {
char *line;
asprintf(&line, "%s%s", options_data[user_data_idx].module_option, options_data[user_data_idx].path_libdir);
opal_argv_append_nosize(&exec_argv, line);
free(line);
}
exec_argc = opal_argv_count(exec_argv);
}
/* link flags and libs */
if (flags & COMP_WANT_LINK) {
opal_argv_insert(&exec_argv, exec_argc, options_data[user_data_idx].link_flags);
exec_argc = opal_argv_count(exec_argv);
opal_argv_insert(&exec_argv, exec_argc, options_data[user_data_idx].libs);
exec_argc = opal_argv_count(exec_argv);
}
/****************************************************
*
* Execute the command
*
****************************************************/
if (flags & COMP_DRY_RUN) {
exec_command = opal_argv_join(exec_argv, ' ');
printf("%s\n", exec_command);
} else {
char *tmp;
#if 0
exec_command = opal_argv_join(exec_argv, ' ');
printf("command: %s\n", exec_command);
#endif
tmp = opal_path_findv(exec_argv[0], 0, environ, NULL);
if (NULL == tmp) {
opal_show_help("help-opal-wrapper.txt", "no-compiler-found", true,
exec_argv[0], NULL);
errno = 0;
exit_status = 1;
} else {
int status;
free(exec_argv[0]);
exec_argv[0] = tmp;
ret = opal_few(exec_argv, &status);
exit_status = WIFEXITED(status) ? WEXITSTATUS(status) :
(WIFSIGNALED(status) ? WTERMSIG(status) :
(WIFSTOPPED(status) ? WSTOPSIG(status) : 255));
if( (OPAL_SUCCESS != ret) || ((0 != exit_status) && (flags & COMP_SHOW_ERROR)) ) {
char* exec_command = opal_argv_join(exec_argv, ' ');
if( OPAL_SUCCESS != ret ) {
opal_show_help("help-opal-wrapper.txt", "spawn-failed", true,
exec_argv[0], strerror(status), exec_command, NULL);
} else {
#if 0
opal_show_help("help-opal-wrapper.txt", "compiler-failed", true,
exec_argv[0], exit_status, exec_command, NULL);
#endif
}
free(exec_command);
}
}
}
/****************************************************
*
* Cleanup
*
****************************************************/
cleanup:
opal_argv_free(exec_argv);
opal_argv_free(user_argv);
if (NULL != base_argv0) free(base_argv0);
if (OPAL_SUCCESS != (ret = data_finalize())) {
return ret;
}
if (OPAL_SUCCESS != (ret = opal_finalize_util())) {
return ret;
}
return exit_status;
}
int
main(int argc, char *argv[])
{
int exit_status = 0, ret, flags = 0, i;
int exec_argc = 0, user_argc = 0;
char **exec_argv = NULL, **user_argv = NULL;
char *exec_command, *base_argv0 = NULL;
bool disable_flags = true;
bool real_flag = false;
if (OPAL_SUCCESS != (ret = opal_init_util(&argc, &argv))) {
return ret;
}
/****************************************************
*
* Setup compiler information
*
****************************************************/
base_argv0 = opal_basename(argv[0]);
#if defined(EXEEXT)
if( 0 != strlen(EXEEXT) ) {
char extension[] = EXEEXT;
char* temp = strstr( base_argv0, extension );
char* old_match = temp;
while( NULL != temp ) {
old_match = temp;
temp = strstr( temp + 1, extension );
}
/* Only if there was a match of .exe, erase the last occurence of .exe */
if ( NULL != old_match ) {
*old_match = '\0';
}
}
#endif /* defined(EXEEXT) */
if (OPAL_SUCCESS != (ret = data_init(base_argv0))) {
fprintf(stderr, "Error parsing data file %s: %s\n", base_argv0, opal_strerror(ret));
return ret;
}
for (i = 1 ; i < argc && user_data_idx < 0 ; ++i) {
user_data_idx = find_options_index(argv[i]);
}
/* if we didn't find a match, look for the NULL (base case) options */
if (user_data_idx < 0) {
user_data_idx = default_data_idx;
}
/* if we still didn't find a match, abort */
if (user_data_idx < 0) {
char *flat = opal_argv_join(argv, ' ');
opal_show_help("help-opal-wrapper.txt", "no-options-support", true,
base_argv0, flat, NULL);
free(flat);
exit(1);
}
/* compiler */
load_env_data(options_data[user_data_idx].project_short, options_data[user_data_idx].compiler_env, &options_data[user_data_idx].compiler);
/* preprocessor flags */
load_env_data_argv(options_data[user_data_idx].project_short, "CPPFLAGS", &options_data[user_data_idx].preproc_flags);
/* compiler flags */
load_env_data_argv(options_data[user_data_idx].project_short, options_data[user_data_idx].compiler_flags_env,
&options_data[user_data_idx].comp_flags);
/* linker flags */
load_env_data_argv(options_data[user_data_idx].project_short, "LDFLAGS", &options_data[user_data_idx].link_flags);
/* libs */
load_env_data_argv(options_data[user_data_idx].project_short, "LIBS", &options_data[user_data_idx].libs);
/****************************************************
*
* Sanity Checks
*
****************************************************/
if (NULL != options_data[user_data_idx].req_file) {
/* make sure the language is supported */
if (0 == strcmp(options_data[user_data_idx].req_file, "not supported")) {
opal_show_help("help-opal-wrapper.txt", "no-language-support", true,
options_data[user_data_idx].language, base_argv0, NULL);
exit_status = 1;
goto cleanup;
}
if (options_data[user_data_idx].req_file[0] != '\0') {
char *filename;
struct stat buf;
filename = opal_os_path( false, options_data[user_data_idx].path_libdir, options_data[user_data_idx].req_file, NULL );
if (0 != stat(filename, &buf)) {
opal_show_help("help-opal-wrapper.txt", "file-not-found", true,
base_argv0, options_data[user_data_idx].req_file, options_data[user_data_idx].language, NULL);
}
}
}
/****************************************************
*
* Parse user flags
*
****************************************************/
flags = COMP_WANT_COMMAND|COMP_WANT_PREPROC|
COMP_WANT_COMPILE|COMP_WANT_LINK;
user_argv = opal_argv_copy(argv + 1);
user_argc = opal_argv_count(user_argv);
for (i = 0 ; i < user_argc ; ++i) {
if (0 == strncmp(user_argv[i], "-showme", strlen("-showme")) ||
0 == strncmp(user_argv[i], "--showme", strlen("--showme")) ||
0 == strncmp(user_argv[i], "-show", strlen("-show")) ||
0 == strncmp(user_argv[i], "--show", strlen("--show"))) {
bool done_now = false;
/* check for specific things we want to see. First three
still invoke all the building routines. Last set want
to parse out certain flags, so we don't go through the
normal build routine - skip to cleanup. */
if (0 == strncmp(user_argv[i], "-showme:command", strlen("-showme:command")) ||
0 == strncmp(user_argv[i], "--showme:command", strlen("--showme:command"))) {
flags = COMP_WANT_COMMAND;
/* we know what we want, so don't process any more args */
done_now = true;
} else if (0 == strncmp(user_argv[i], "-showme:compile", strlen("-showme:compile")) ||
0 == strncmp(user_argv[i], "--showme:compile", strlen("--showme:compile"))) {
flags = COMP_WANT_PREPROC|COMP_WANT_COMPILE;
/* we know what we want, so don't process any more args */
done_now = true;
} else if (0 == strncmp(user_argv[i], "-showme:link", strlen("-showme:link")) ||
0 == strncmp(user_argv[i], "--showme:link", strlen("--showme:link"))) {
flags = COMP_WANT_COMPILE|COMP_WANT_LINK;
/* we know what we want, so don't process any more args */
done_now = true;
} else if (0 == strncmp(user_argv[i], "-showme:incdirs", strlen("-showme:incdirs")) ||
0 == strncmp(user_argv[i], "--showme:incdirs", strlen("--showme:incdirs"))) {
print_flags(options_data[user_data_idx].preproc_flags, OPAL_INCLUDE_FLAG);
goto cleanup;
} else if (0 == strncmp(user_argv[i], "-showme:libdirs", strlen("-showme:libdirs")) ||
0 == strncmp(user_argv[i], "--showme:libdirs", strlen("--showme:libdirs"))) {
print_flags(options_data[user_data_idx].link_flags, OPAL_LIBDIR_FLAG);
goto cleanup;
} else if (0 == strncmp(user_argv[i], "-showme:libs", strlen("-showme:libs")) ||
0 == strncmp(user_argv[i], "--showme:libs", strlen("--showme:libs"))) {
print_flags(options_data[user_data_idx].libs, "-l");
goto cleanup;
} else if (0 == strncmp(user_argv[i], "-showme:version", strlen("-showme:version")) ||
0 == strncmp(user_argv[i], "--showme:version", strlen("--showme:version"))) {
char * str;
str = opal_show_help_string("help-opal-wrapper.txt",
"version", false,
argv[0], options_data[user_data_idx].project, options_data[user_data_idx].version, options_data[user_data_idx].language, NULL);
if (NULL != str) {
printf("%s", str);
free(str);
}
goto cleanup;
} else if (0 == strncmp(user_argv[i], "-showme:help", strlen("-showme:help")) ||
0 == strncmp(user_argv[i], "--showme:help", strlen("--showme:help"))) {
char *str;
str = opal_show_help_string("help-opal-wrapper.txt", "usage",
false, argv[0],
options_data[user_data_idx].project,
NULL);
if (NULL != str) {
printf("%s", str);
free(str);
}
exit_status = 0;
goto cleanup;
} else if (0 == strncmp(user_argv[i], "-showme:", strlen("-showme:")) ||
0 == strncmp(user_argv[i], "--showme:", strlen("--showme:"))) {
fprintf(stderr, "%s: unrecognized option: %s\n", argv[0],
user_argv[i]);
fprintf(stderr, "Type '%s --showme:help' for usage.\n",
argv[0]);
exit_status = 1;
goto cleanup;
}
flags |= (COMP_DRY_RUN|COMP_SHOW_ERROR);
/* remove element from user_argv */
opal_argv_delete(&user_argc, &user_argv, i, 1);
--i;
if (done_now) {
disable_flags = false;
break;
}
} else if (0 == strcmp(user_argv[i], "-c")) {
flags &= ~COMP_WANT_LINK;
real_flag = true;
} else if (0 == strcmp(user_argv[i], "-E") ||
0 == strcmp(user_argv[i], "-M")) {
flags &= ~(COMP_WANT_COMPILE | COMP_WANT_LINK);
real_flag = true;
} else if (0 == strcmp(user_argv[i], "-S")) {
flags &= ~COMP_WANT_LINK;
real_flag = true;
} else if (0 == strcmp(user_argv[i], "-lpmpi")) {
flags |= COMP_WANT_PMPI;
/* remove element from user_argv */
opal_argv_delete(&user_argc, &user_argv, i, 1);
--i;
} else if (0 == strcmp(user_argv[i], "-static") ||
0 == strcmp(user_argv[i], "--static") ||
0 == strcmp(user_argv[i], "-Bstatic") ||
0 == strcmp(user_argv[i], "-Wl,-static") ||
0 == strcmp(user_argv[i], "-Wl,--static") ||
0 == strcmp(user_argv[i], "-Wl,-Bstatic")) {
flags |= COMP_WANT_STATIC;
} else if (0 == strcmp(user_argv[i], "-dynamic") ||
0 == strcmp(user_argv[i], "--dynamic") ||
0 == strcmp(user_argv[i], "-Bdynamic") ||
0 == strcmp(user_argv[i], "-Wl,-dynamic") ||
0 == strcmp(user_argv[i], "-Wl,--dynamic") ||
0 == strcmp(user_argv[i], "-Wl,-Bdynamic")) {
flags &= ~COMP_WANT_STATIC;
} else if (0 == strcmp(user_argv[i], "--openmpi:linkall")) {
/* This is an intentionally undocummented wrapper compiler
switch. It should only be used by Open MPI developers
-- not end users. It will cause mpicc to use the
static library list, even if we're compiling
dynamically (i.e., it'll specifically -lopen-rte and
-lopen-pal (and all their dependent libs)). We provide
this flag for test MPI applications that also invoke
ORTE and/or OPAL function calls.
On some systems (e.g., OS X), if the top-level
application calls ORTE/OPAL functions and you don't -l
ORTE and OPAL, then the functions won't be resolved at
link time (i.e., the implicit library dependencies of
libmpi won't be pulled in at link time), and therefore
the link will fail. This flag will cause the wrapper
to explicitly list the ORTE and OPAL libs on the
underlying compiler command line, so the application
will therefore link properly. */
flags |= COMP_WANT_LINKALL;
/* remove element from user_argv */
opal_argv_delete(&user_argc, &user_argv, i, 1);
} else if ('-' != user_argv[i][0]) {
disable_flags = false;
flags |= COMP_SHOW_ERROR;
real_flag = true;
} else {
/* if the option flag is one that we use to determine
which set of compiler data to use, don't count it as a
real option */
if (find_options_index(user_argv[i]) < 0) {
real_flag = true;
}
}
}
/* clear out the want_flags if we got no arguments not starting
with a - (dash) and -showme wasn't given OR -showme was given
and we had at least one more non-showme argument that started
with a - (dash) and no other non-dash arguments. Some examples:
opal_wrapper : clear our flags
opal_wrapper -v : clear our flags
opal_wrapper -E a.c : don't clear our flags
opal_wrapper a.c : don't clear our flags
opal_wrapper -showme : don't clear our flags
opal_wrapper -showme -v : clear our flags
opal_wrapper -showme -E a.c : don't clear our flags
opal_wrapper -showme a.c : don't clear our flags
*/
if (disable_flags && !((flags & COMP_DRY_RUN) && !real_flag)) {
flags &= ~(COMP_WANT_PREPROC|COMP_WANT_COMPILE|COMP_WANT_LINK);
}
/****************************************************
*
* Assemble the command line
*
****************************************************/
/* compiler (may be multiple arguments, so split) */
if (flags & COMP_WANT_COMMAND) {
exec_argv = opal_argv_split(options_data[user_data_idx].compiler, ' ');
exec_argc = opal_argv_count(exec_argv);
} else {
exec_argv = (char **) malloc(sizeof(char*));
exec_argv[0] = NULL;
exec_argc = 0;
}
/* This error would normally not happen unless the user edits the
wrapper data files manually */
if (NULL == exec_argv) {
opal_show_help("help-opal-wrapper.txt", "no-compiler-specified", true);
return 1;
}
if (flags & COMP_WANT_COMPILE) {
opal_argv_insert(&exec_argv, exec_argc,
options_data[user_data_idx].comp_flags_prefix);
exec_argc = opal_argv_count(exec_argv);
}
/* Per https://svn.open-mpi.org/trac/ompi/ticket/2201, add all the
user arguments before anything else. */
opal_argv_insert(&exec_argv, exec_argc, user_argv);
exec_argc = opal_argv_count(exec_argv);
/* preproc flags */
if (flags & COMP_WANT_PREPROC) {
opal_argv_insert(&exec_argv, exec_argc, options_data[user_data_idx].preproc_flags);
exec_argc = opal_argv_count(exec_argv);
}
/* compiler flags */
if (flags & COMP_WANT_COMPILE) {
opal_argv_insert(&exec_argv, exec_argc, options_data[user_data_idx].comp_flags);
exec_argc = opal_argv_count(exec_argv);
}
/* link flags and libs */
if (flags & COMP_WANT_LINK) {
bool have_static_lib;
bool have_dyn_lib;
bool use_static_libs;
char *filename1, *filename2;
struct stat buf;
opal_argv_insert(&exec_argv, exec_argc, options_data[user_data_idx].link_flags);
exec_argc = opal_argv_count(exec_argv);
/* Are we linking statically? If so, decide what libraries to
list. It depends on two factors:
1. Was --static (etc.) specified?
2. Does OMPI have static, dynamic, or both libraries installed?
Here's a matrix showing what we'll do in all 6 cases:
What's installed --static no --static
---------------- ---------- -----------
ompi .so libs -lmpi -lmpi
ompi .a libs all all
ompi both libs all -lmpi
*/
filename1 = opal_os_path( false, options_data[user_data_idx].path_libdir, options_data[user_data_idx].static_lib_file, NULL );
if (0 == stat(filename1, &buf)) {
have_static_lib = true;
} else {
have_static_lib = false;
}
filename2 = opal_os_path( false, options_data[user_data_idx].path_libdir, options_data[user_data_idx].dyn_lib_file, NULL );
if (0 == stat(filename2, &buf)) {
have_dyn_lib = true;
} else {
have_dyn_lib = false;
}
/* Determine which set of libs to use: dynamic or static. Be
pedantic to make the code easy to read. */
if (flags & COMP_WANT_LINKALL) {
/* If --openmpi:linkall was specified, list all the libs
(i.e., the static libs) if they're available, either in
static or dynamic form. */
if (have_static_lib || have_dyn_lib) {
use_static_libs = true;
} else {
fprintf(stderr, "The linkall option has failed as we were unable to find either static or dynamic libs\n"
"Files looked for:\n Static: %s\n Dynamic: %s\n",
filename1, filename2);
free(filename1);
free(filename2);
exit(1);
}
} else if (flags & COMP_WANT_STATIC) {
/* If --static (or something like it) was specified, if we
have the static libs, then use them. Otherwise, use
the dynamic libs. */
if (have_static_lib) {
use_static_libs = true;
} else {
use_static_libs = false;
}
} else {
/* If --static (or something like it) was NOT specified
(or if --dyanic, or something like it, was specified),
if we have the dynamic libs, then use them. Otherwise,
use the static libs. */
if (have_dyn_lib) {
use_static_libs = false;
} else {
use_static_libs = true;
}
}
free(filename1);
free(filename2);
if (use_static_libs) {
opal_argv_insert(&exec_argv, exec_argc, options_data[user_data_idx].libs_static);
} else {
opal_argv_insert(&exec_argv, exec_argc, options_data[user_data_idx].libs);
}
exec_argc = opal_argv_count(exec_argv);
}
/****************************************************
*
* Execute the command
*
****************************************************/
if (flags & COMP_DRY_RUN) {
exec_command = opal_argv_join(exec_argv, ' ');
printf("%s\n", exec_command);
} else {
char *tmp;
#if 0
exec_command = opal_argv_join(exec_argv, ' ');
printf("command: %s\n", exec_command);
#endif
tmp = opal_path_findv(exec_argv[0], 0, environ, NULL);
if (NULL == tmp) {
opal_show_help("help-opal-wrapper.txt", "no-compiler-found", true,
exec_argv[0], NULL);
errno = 0;
exit_status = 1;
} else {
int status;
free(exec_argv[0]);
exec_argv[0] = tmp;
ret = opal_few(exec_argv, &status);
exit_status = WIFEXITED(status) ? WEXITSTATUS(status) :
(WIFSIGNALED(status) ? WTERMSIG(status) :
(WIFSTOPPED(status) ? WSTOPSIG(status) : 255));
if( (OPAL_SUCCESS != ret) || ((0 != exit_status) && (flags & COMP_SHOW_ERROR)) ) {
char* exec_command = opal_argv_join(exec_argv, ' ');
if( OPAL_SUCCESS != ret ) {
opal_show_help("help-opal-wrapper.txt", "spawn-failed", true,
exec_argv[0], strerror(status), exec_command, NULL);
} else {
#if 0
opal_show_help("help-opal-wrapper.txt", "compiler-failed", true,
exec_argv[0], exit_status, exec_command, NULL);
#endif
}
free(exec_command);
}
}
}
/****************************************************
*
* Cleanup
*
****************************************************/
cleanup:
opal_argv_free(exec_argv);
opal_argv_free(user_argv);
if (NULL != base_argv0) free(base_argv0);
if (OPAL_SUCCESS != (ret = data_finalize())) {
return ret;
}
if (OPAL_SUCCESS != (ret = opal_finalize_util())) {
return ret;
}
return exit_status;
}
static int parse_file(char *filename,
opal_pointer_array_t *apps)
{
char *cptr, *executable, *version, *argv, *tmp, *binary;
int token, rc=ORCM_SUCCESS;
int i, ival;
orcm_cfgi_app_t *app=NULL, *aptr, *curapp;
orcm_cfgi_exec_t *exec=NULL, *eptr;
orcm_cfgi_version_t *vers=NULL, *vptr;
bool found;
struct stat buf;
orcm_cfgi_file_in = fopen(filename, "r");
if (NULL == orcm_cfgi_file_in) {
opal_output(0, "%s UNABLE TO OPEN FILE %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), filename);
return ORTE_ERR_FILE_OPEN_FAILURE;
}
OPAL_OUTPUT_VERBOSE((2, orcm_cfgi_base.output,
"%s PARSING CONFIGURATION FILE %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), filename));
while (!orcm_cfgi_file_done) {
token = orcm_cfgi_file_lex();
switch (token) {
case ORCM_CFGI_FILE_DONE:
orcm_cfgi_file_done = true;
if (NULL != app) {
if (1 < opal_output_get_verbosity(orcm_cfgi_base.output)) {
opal_output(0, "READ INSTALL CONFIG:");
orcm_cfgi_base_dump(NULL, NULL, app, ORCM_CFGI_APP);
}
/* if the app hasn't been added yet */
if (app->idx < 0) {
/* see if we already have a definition for this app */
found = false;
for (i=0; i < apps->size; i++) {
if (NULL == (aptr = (orcm_cfgi_app_t*)opal_pointer_array_get_item(apps, i))) {
continue;
}
if (0 == strcmp(app->application, aptr->application)) {
/* found app */
found = true;
break;
}
}
if (!found) {
/* this is a new app - check for validity */
if (!orcm_cfgi_app_definition_valid(app)) {
opal_output(0, "%s APP %s NOT VALID",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
(NULL == app->application) ? "NULL" : app->application);
rc = ORTE_ERR_BAD_PARAM;
OBJ_RELEASE(app);
goto depart;
}
/* store it before leaving */
app->idx = opal_pointer_array_add(apps, app);
}
}
}
break;
case ORCM_CFGI_FILE_NEWLINE:
break;
case ORCM_CFGI_FILE_APPLICATION:
if (NULL == (cptr = parse_string())) {
orte_show_help("help-cfgi-file.txt", "no-app-name", true);
rc = ORTE_ERR_BAD_PARAM;
goto depart;
}
if (NULL == app) {
/* start a new app - create record for it */
app = OBJ_NEW(orcm_cfgi_app_t);
app->application = strdup(cptr);
exec = NULL; /* start the exec over */
vers = NULL;
} else {
/* are we already working this app */
if (0 != strcmp(cptr, app->application)) {
/* this starts a different app */
if (1 < opal_output_get_verbosity(orcm_cfgi_base.output)) {
opal_output(0, "READ INSTALL CONFIG:");
orcm_cfgi_base_dump(NULL, NULL, app, ORCM_CFGI_APP);
}
/* check for validity */
if (!orcm_cfgi_app_definition_valid(app)) {
opal_output(0, "%s APP %s NOT VALID",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
(NULL == app->application) ? "NULL" : app->application);
rc = ORTE_ERR_BAD_PARAM;
OBJ_RELEASE(app);
free(cptr);
goto depart;
}
if (app->idx < 0) {
/* hasn't been added yet, so do so
* before we start on the new one
*/
app->idx = opal_pointer_array_add(apps, app);
}
/* see if we already have a definition for this app */
found = false;
for (i=0; i < orcm_cfgi_base.installed_apps.size; i++) {
if (NULL == (aptr = (orcm_cfgi_app_t*)opal_pointer_array_get_item(apps, i))) {
continue;
}
if (0 == strcmp(cptr, aptr->application)) {
/* found app */
app = aptr;
found = true;
break;
}
}
if (!found) {
/* start a new app - create record for it */
app = OBJ_NEW(orcm_cfgi_app_t);
app->application = strdup(cptr);
}
}
exec = NULL; /* start the exec over */
vers = NULL;
}
free(cptr);
break;
case ORCM_CFGI_FILE_MAX_INSTANCES:
/* if we don't have an active app, that's an error */
if (NULL == app) {
/* cannot process this */
rc = ORTE_ERR_BAD_PARAM;
goto depart;
}
if ((app->max_instances = parse_int()) < 0) {
orte_show_help("help-cfgi-file.txt", "bad-max-instances", true,
filename, orcm_cfgi_file_line, token,
orcm_cfgi_file_value.sval);
OBJ_RELEASE(app);
rc = ORTE_ERR_BAD_PARAM;
goto depart;
}
break;
case ORCM_CFGI_FILE_EXECUTABLE:
/* if we don't have an active app, that's an error */
if (NULL == app) {
/* cannot process this */
rc = ORTE_ERR_BAD_PARAM;
goto depart;
}
if (NULL == (executable = parse_string())) {
orte_show_help("help-cfgi-file.txt", "no-exec", true,
filename, orcm_cfgi_file_line, token,
orcm_cfgi_file_value.sval);
OBJ_RELEASE(app);
rc = ORTE_ERR_BAD_PARAM;
goto depart;
}
/* do I already have an entry for this executable in this app? */
exec = NULL;
for (i=0; i < app->executables.size; i++) {
if (NULL == (eptr = (orcm_cfgi_exec_t*)opal_pointer_array_get_item(&app->executables, i))) {
continue;
}
if (0 == strcmp(eptr->appname, executable)) {
exec = eptr;
break;
}
}
if (NULL == exec) {
/* nope - add this executable */
exec = OBJ_NEW(orcm_cfgi_exec_t);
exec->appname = strdup(executable);
exec->idx = opal_pointer_array_add(&app->executables, exec);
}
free(executable);
break;
case ORCM_CFGI_FILE_VERSION:
/* if we don't have an active job, that's an error */
if (NULL == app) {
/* cannot process this */
rc = ORTE_ERR_BAD_PARAM;
goto depart;
}
/* if we don't have an active exec, that's an error */
if (NULL == exec) {
/* cannot process this */
OBJ_RELEASE(app);
rc = ORTE_ERR_BAD_PARAM;
goto depart;
}
if (NULL == (version = parse_string())) {
orte_show_help("help-cfgi-file.txt", "no-version", true,
filename, orcm_cfgi_file_line, token,
orcm_cfgi_file_value.sval);
OBJ_RELEASE(app);
rc = ORTE_ERR_BAD_PARAM;
goto depart;
}
/* do we already have a record of this version? */
vers = NULL;
for (i=0; i < exec->versions.size; i++) {
if (NULL == (vptr = (orcm_cfgi_version_t*)opal_pointer_array_get_item(&exec->versions, i))) {
continue;
}
if (0 == strcmp(vptr->version, version)) {
vers = vptr;
break;
}
}
if (NULL == vers) {
/* nope - add it */
vers = OBJ_NEW(orcm_cfgi_version_t);
vers->exec = exec;
vers->version = strdup(version);
vers->idx = opal_pointer_array_add(&exec->versions, vers);
/* get the modification time stamp for that binary */
asprintf(&binary, "%s_%s", exec->appname, version);
tmp = opal_path_findv(binary, X_OK, environ, NULL);
free(binary);
if (NULL == tmp) {
/* binary wasn't found - just leave it */
free(version);
break;
}
if (0 > stat(tmp, &buf)) {
/* cannot stat file */
OPAL_OUTPUT_VERBOSE((1, orcm_cfgi_base.output,
"%s could not stat %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), tmp));
free(tmp);
free(version);
break;
}
vers->mod_time = strdup(ctime(&buf.st_mtime));
/* strip the ending cr */
vers->mod_time[strlen(vers->mod_time)-1] = '\0';
OPAL_OUTPUT_VERBOSE((1, orcm_cfgi_base.output,
"%s BIN %s MODTIME %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
binary, vers->mod_time));
free(tmp);
}
free(version);
break;
case ORCM_CFGI_FILE_ARGV:
/* if we don't have an active job, that's an error */
if (NULL == app) {
/* cannot process this */
rc = ORTE_ERR_BAD_PARAM;
goto depart;
}
/* if we don't have an active exec, that's an error */
if (NULL == exec) {
/* cannot process this */
OBJ_RELEASE(app);
rc = ORTE_ERR_BAD_PARAM;
goto depart;
}
/* if we don't have an active version, that's an error */
if (NULL == vers) {
/* cannot process this */
OBJ_RELEASE(app);
rc = ORTE_ERR_BAD_PARAM;
goto depart;
}
if (NULL == (argv = parse_string())) {
orte_show_help("help-cfgi-file.txt", "no-argv", true,
filename, orcm_cfgi_file_line, token,
orcm_cfgi_file_value.sval);
OBJ_RELEASE(app);
rc = ORTE_ERR_BAD_PARAM;
goto depart;
}
vers->argv = opal_argv_split(argv, ' ');
free(argv);
break;
case ORCM_CFGI_FILE_PROCESS_LIMIT:
/* if we don't have an active job, that's an error */
if (NULL == app) {
/* cannot process this */
rc = ORTE_ERR_BAD_PARAM;
goto depart;
}
/* if we don't have an active exec, that's an error */
if (NULL == exec) {
/* cannot process this */
OBJ_RELEASE(app);
rc = ORTE_ERR_BAD_PARAM;
goto depart;
}
/* set the process limit */
if ((exec->process_limit = parse_int()) < 0) {
orte_show_help("help-cfgi-file.txt", "bad-max-procs", true,
filename, orcm_cfgi_file_line, token,
orcm_cfgi_file_value.sval);
OBJ_RELEASE(app);
rc = ORTE_ERR_BAD_PARAM;
goto depart;
}
break;
case ORCM_CFGI_FILE_STRING:
case ORCM_CFGI_FILE_QUOTED_STRING:
orte_show_help("help-cfgi-file.txt", "parse_error_string",
true,
filename,
orcm_cfgi_file_line,
token,
orcm_cfgi_file_value.sval);
break;
case ORCM_CFGI_FILE_INT:
orte_show_help("help-cfgi-file.txt", "parse_error_int",
true,
filename,
orcm_cfgi_file_line,
token,
orcm_cfgi_file_value.ival);
break;
default:
orte_show_help("help-cfgi-file.txt", "parse_error",
true,
filename,
orcm_cfgi_file_line,
token);
}
}
depart:
fclose(orcm_cfgi_file_in);
orcm_cfgi_file_done = false;
return rc;
}
int orte_pls_rsh_launch(orte_jobid_t jobid)
{
orte_job_map_t *map;
opal_list_item_t *n_item;
orte_mapped_node_t *rmaps_node;
orte_std_cntr_t num_nodes;
orte_vpid_t vpid;
int node_name_index1;
int node_name_index2;
int proc_name_index;
int local_exec_index, local_exec_index_end;
char *jobid_string = NULL;
char *uri, *param;
char **argv = NULL, **tmp;
char *prefix_dir;
int argc;
int rc;
sigset_t sigs;
struct passwd *p;
bool remote_sh = false, remote_csh = false;
bool local_sh = false, local_csh = false;
char *lib_base = NULL, *bin_base = NULL;
orte_pls_daemon_info_t *dmn;
orte_pls_rsh_shell_t shell;
if (mca_pls_rsh_component.timing) {
if (0 != gettimeofday(&joblaunchstart, NULL)) {
opal_output(0, "pls_rsh: could not obtain start time");
joblaunchstart.tv_sec = 0;
joblaunchstart.tv_usec = 0;
}
}
/* setup a list that will contain the info for all the daemons
* so we can store it on the registry when done and use it
* locally to track their state
*/
OBJ_CONSTRUCT(&active_daemons, opal_list_t);
/* Get the map for this job
* We need the entire mapping for a couple of reasons:
* - need the prefix to start with.
* - need to know the nodes we are launching on
* All other mapping responsibilities fall to orted in the fork PLS
*/
rc = orte_rmaps.get_job_map(&map, jobid);
if (ORTE_SUCCESS != rc) {
ORTE_ERROR_LOG(rc);
OBJ_DESTRUCT(&active_daemons);
return rc;
}
/* if the user requested that we re-use daemons,
* launch the procs on any existing, re-usable daemons
*/
if (orte_pls_base.reuse_daemons) {
if (ORTE_SUCCESS != (rc = orte_pls_base_launch_on_existing_daemons(map))) {
ORTE_ERROR_LOG(rc);
OBJ_RELEASE(map);
OBJ_DESTRUCT(&active_daemons);
return rc;
}
}
num_nodes = (orte_std_cntr_t)opal_list_get_size(&map->nodes);
if (0 == num_nodes) {
/* nothing left to do - just return */
OBJ_RELEASE(map);
OBJ_DESTRUCT(&active_daemons);
return ORTE_SUCCESS;
}
if (mca_pls_rsh_component.debug_daemons &&
mca_pls_rsh_component.num_concurrent < num_nodes) {
/* we can't run in this situation, so pretty print the error
* and exit
*/
opal_show_help("help-pls-rsh.txt", "deadlock-params",
true, mca_pls_rsh_component.num_concurrent, num_nodes);
OBJ_RELEASE(map);
OBJ_DESTRUCT(&active_daemons);
return ORTE_ERR_FATAL;
}
/*
* After a discussion between Ralph & Jeff, we concluded that we
* really are handling the prefix dir option incorrectly. It currently
* is associated with an app_context, yet it really refers to the
* location where OpenRTE/Open MPI is installed on a NODE. Fixing
* this right now would involve significant change to orterun as well
* as elsewhere, so we will intentionally leave this incorrect at this
* point. The error, however, is identical to that seen in all prior
* releases of OpenRTE/Open MPI, so our behavior is no worse than before.
*
* A note to fix this, along with ideas on how to do so, has been filed
* on the project's Trac system under "feature enhancement".
*
* For now, default to the prefix_dir provided in the first app_context.
* Since there always MUST be at least one app_context, we are safe in
* doing this.
*/
prefix_dir = map->apps[0]->prefix_dir;
/*
* Allocate a range of vpids for the daemons.
*/
if (num_nodes == 0) {
return ORTE_ERR_BAD_PARAM;
}
rc = orte_ns.reserve_range(0, num_nodes, &vpid);
if (ORTE_SUCCESS != rc) {
goto cleanup;
}
/* setup the orted triggers for passing their launch info */
if (ORTE_SUCCESS != (rc = orte_smr.init_orted_stage_gates(jobid, num_nodes, NULL, NULL))) {
ORTE_ERROR_LOG(rc);
goto cleanup;
}
/* need integer value for command line parameter */
if (ORTE_SUCCESS != (rc = orte_ns.convert_jobid_to_string(&jobid_string, jobid))) {
ORTE_ERROR_LOG(rc);
goto cleanup;
}
/* What is our local shell? */
shell = ORTE_PLS_RSH_SHELL_UNKNOWN;
p = getpwuid(getuid());
if (NULL != p) {
param = p->pw_shell;
shell = find_shell(p->pw_shell);
}
/* If we didn't find it in getpwuid(), try looking at the $SHELL
environment variable (see
https://svn.open-mpi.org/trac/ompi/ticket/1060) */
if (ORTE_PLS_RSH_SHELL_UNKNOWN == shell &&
NULL != (param = getenv("SHELL"))) {
shell = find_shell(param);
}
switch (shell) {
case ORTE_PLS_RSH_SHELL_SH: /* fall through */
case ORTE_PLS_RSH_SHELL_KSH: /* fall through */
case ORTE_PLS_RSH_SHELL_ZSH: /* fall through */
case ORTE_PLS_RSH_SHELL_BASH: local_sh = true; break;
case ORTE_PLS_RSH_SHELL_TCSH: /* fall through */
case ORTE_PLS_RSH_SHELL_CSH: local_csh = true; break;
default:
opal_output(0, "WARNING: local probe returned unhandled shell:%s assuming bash\n",
(NULL != param) ? param : "unknown");
remote_sh = true;
break;
}
if (mca_pls_rsh_component.debug) {
opal_output(0, "pls:rsh: local csh: %d, local sh: %d\n",
local_csh, local_sh);
}
/* What is our remote shell? */
if (mca_pls_rsh_component.assume_same_shell) {
remote_sh = local_sh;
remote_csh = local_csh;
if (mca_pls_rsh_component.debug) {
opal_output(0, "pls:rsh: assuming same remote shell as local shell");
}
} else {
orte_pls_rsh_shell_t shell;
rmaps_node = (orte_mapped_node_t*)opal_list_get_first(&map->nodes);
rc = orte_pls_rsh_probe(rmaps_node, &shell);
if (ORTE_SUCCESS != rc) {
ORTE_ERROR_LOG(rc);
return rc;
}
switch (shell) {
case ORTE_PLS_RSH_SHELL_SH: /* fall through */
case ORTE_PLS_RSH_SHELL_KSH: /* fall through */
case ORTE_PLS_RSH_SHELL_ZSH: /* fall through */
case ORTE_PLS_RSH_SHELL_BASH: remote_sh = true; break;
case ORTE_PLS_RSH_SHELL_TCSH: /* fall through */
case ORTE_PLS_RSH_SHELL_CSH: remote_csh = true; break;
default:
opal_output(0, "WARNING: rsh probe returned unhandled shell; assuming bash\n");
remote_sh = true;
}
}
if (mca_pls_rsh_component.debug) {
opal_output(0, "pls:rsh: remote csh: %d, remote sh: %d\n",
remote_csh, remote_sh);
}
/*
* Build argv array
*/
argv = opal_argv_copy(mca_pls_rsh_component.agent_argv);
argc = mca_pls_rsh_component.agent_argc;
node_name_index1 = argc;
opal_argv_append(&argc, &argv, "<template>");
/* Do we need to source .profile on the remote side? */
if (!(remote_csh || remote_sh)) {
int i;
tmp = opal_argv_split("( test ! -r ./.profile || . ./.profile;", ' ');
if (NULL == tmp) {
return ORTE_ERR_OUT_OF_RESOURCE;
}
for (i = 0; NULL != tmp[i]; ++i) {
opal_argv_append(&argc, &argv, tmp[i]);
}
opal_argv_free(tmp);
}
/* add the daemon command (as specified by user) */
local_exec_index = argc;
opal_argv_append(&argc, &argv, mca_pls_rsh_component.orted);
/* check for debug flags */
orte_pls_base_mca_argv(&argc, &argv);
opal_argv_append(&argc, &argv, "--bootproxy");
opal_argv_append(&argc, &argv, jobid_string);
opal_argv_append(&argc, &argv, "--name");
proc_name_index = argc;
opal_argv_append(&argc, &argv, "<template>");
/* tell the daemon how many procs are in the daemon's job */
opal_argv_append(&argc, &argv, "--num_procs");
asprintf(¶m, "%lu", (unsigned long)(vpid + num_nodes));
opal_argv_append(&argc, &argv, param);
free(param);
/* tell the daemon the starting vpid of the daemon's job */
opal_argv_append(&argc, &argv, "--vpid_start");
opal_argv_append(&argc, &argv, "0");
opal_argv_append(&argc, &argv, "--nodename");
node_name_index2 = argc;
opal_argv_append(&argc, &argv, "<template>");
/* pass along the universe name and location info */
opal_argv_append(&argc, &argv, "--universe");
asprintf(¶m, "%s@%s:%s", orte_universe_info.uid,
orte_universe_info.host, orte_universe_info.name);
opal_argv_append(&argc, &argv, param);
free(param);
/* setup ns contact info */
opal_argv_append(&argc, &argv, "--nsreplica");
if (NULL != orte_process_info.ns_replica_uri) {
uri = strdup(orte_process_info.ns_replica_uri);
} else {
uri = orte_rml.get_uri();
}
asprintf(¶m, "\"%s\"", uri);
opal_argv_append(&argc, &argv, param);
free(uri);
free(param);
/* setup gpr contact info */
opal_argv_append(&argc, &argv, "--gprreplica");
if (NULL != orte_process_info.gpr_replica_uri) {
uri = strdup(orte_process_info.gpr_replica_uri);
} else {
uri = orte_rml.get_uri();
}
asprintf(¶m, "\"%s\"", uri);
opal_argv_append(&argc, &argv, param);
free(uri);
free(param);
local_exec_index_end = argc;
if (!(remote_csh || remote_sh)) {
opal_argv_append(&argc, &argv, ")");
}
if (mca_pls_rsh_component.debug) {
param = opal_argv_join(argv, ' ');
if (NULL != param) {
opal_output(0, "pls:rsh: final template argv:");
opal_output(0, "pls:rsh: %s", param);
free(param);
}
}
/* Figure out the basenames for the libdir and bindir. This
requires some explanation:
- Use opal_install_dirs.libdir and opal_install_dirs.bindir instead of -D'ing some macros
in this directory's Makefile.am because it makes all the
dependencies work out correctly. These are defined in
opal/install_dirs.h.
- After a discussion on the devel-core mailing list, the
developers decided that we should use the local directory
basenames as the basis for the prefix on the remote note.
This does not handle a few notable cases (e.g., f the
libdir/bindir is not simply a subdir under the prefix, if the
libdir/bindir basename is not the same on the remote node as
it is here in the local node, etc.), but we decided that
--prefix was meant to handle "the common case". If you need
something more complex than this, a) edit your shell startup
files to set PATH/LD_LIBRARY_PATH properly on the remove
node, or b) use some new/to-be-defined options that
explicitly allow setting the bindir/libdir on the remote
node. We decided to implement these options (e.g.,
--remote-bindir and --remote-libdir) to orterun when it
actually becomes a problem for someone (vs. a hypothetical
situation).
Hence, for now, we simply take the basename of this install's
libdir and bindir and use it to append this install's prefix
and use that on the remote node.
*/
lib_base = opal_basename(opal_install_dirs.libdir);
bin_base = opal_basename(opal_install_dirs.bindir);
/*
* Iterate through each of the nodes
*/
if (mca_pls_rsh_component.timing) {
/* allocate space to track the start times */
launchstart = (struct timeval*)malloc((num_nodes+vpid) * sizeof(struct timeval));
}
for(n_item = opal_list_get_first(&map->nodes);
n_item != opal_list_get_end(&map->nodes);
n_item = opal_list_get_next(n_item)) {
orte_process_name_t* name;
pid_t pid;
char *exec_path;
char **exec_argv;
rmaps_node = (orte_mapped_node_t*)n_item;
if (mca_pls_rsh_component.timing) {
if (0 != gettimeofday(&launchstart[vpid], NULL)) {
opal_output(0, "pls_rsh: could not obtain start time");
}
}
/* new daemon - setup to record its info */
dmn = OBJ_NEW(orte_pls_daemon_info_t);
dmn->active_job = jobid;
opal_list_append(&active_daemons, &dmn->super);
/* setup node name */
free(argv[node_name_index1]);
if (NULL != rmaps_node->username &&
0 != strlen (rmaps_node->username)) {
asprintf (&argv[node_name_index1], "%s@%s",
rmaps_node->username, rmaps_node->nodename);
} else {
argv[node_name_index1] = strdup(rmaps_node->nodename);
}
free(argv[node_name_index2]);
argv[node_name_index2] = strdup(rmaps_node->nodename);
/* save it in the daemon info */
dmn->nodename = strdup(rmaps_node->nodename);
/* initialize daemons process name */
rc = orte_ns.create_process_name(&name, rmaps_node->cell, 0, vpid);
if (ORTE_SUCCESS != rc) {
ORTE_ERROR_LOG(rc);
goto cleanup;
}
/* save it in the daemon info */
dmn->cell = rmaps_node->cell;
if (ORTE_SUCCESS != (rc = orte_dss.copy((void**)&(dmn->name), name, ORTE_NAME))) {
ORTE_ERROR_LOG(rc);
goto cleanup;
}
/* fork a child to exec the rsh/ssh session */
/* set the process state to "launched" */
if (ORTE_SUCCESS != (rc = orte_smr.set_proc_state(name, ORTE_PROC_STATE_LAUNCHED, 0))) {
ORTE_ERROR_LOG(rc);
goto cleanup;
}
pid = fork();
if (pid < 0) {
rc = ORTE_ERR_OUT_OF_RESOURCE;
goto cleanup;
}
/* child */
if (pid == 0) {
char* name_string;
char** env;
char* var;
long fd, fdmax = sysconf(_SC_OPEN_MAX);
if (mca_pls_rsh_component.debug) {
opal_output(0, "pls:rsh: launching on node %s\n",
rmaps_node->nodename);
}
/* We don't need to sense an oversubscribed condition and set the sched_yield
* for the node as we are only launching the daemons at this time. The daemons
* are now smart enough to set the oversubscribed condition themselves when
* they launch the local procs.
*/
/* Is this a local launch?
*
* Not all node names may be resolvable (if we found
* localhost in the hostfile, for example). So first
* check trivial case of node_name being same as the
* current nodename, which must be local. If that doesn't
* match, check using ifislocal().
*/
if (!mca_pls_rsh_component.force_rsh &&
(0 == strcmp(rmaps_node->nodename, orte_system_info.nodename) ||
opal_ifislocal(rmaps_node->nodename))) {
if (mca_pls_rsh_component.debug) {
opal_output(0, "pls:rsh: %s is a LOCAL node\n",
rmaps_node->nodename);
}
if (mca_pls_rsh_component.timing) {
/* since this is a local launch, the daemon will never reach
* the waitpid callback - so set the start value to
* something nonsensical
*/
launchstart[vpid].tv_sec = 0;
launchstart[vpid].tv_usec = 0;
}
exec_path = opal_path_findv(argv[local_exec_index], 0, environ, NULL);
if (NULL == exec_path && NULL == prefix_dir) {
rc = orte_pls_rsh_fill_exec_path (&exec_path);
if (ORTE_SUCCESS != rc) {
exit(-1); /* the forked process MUST exit */
}
} else {
if (NULL != prefix_dir) {
exec_path = opal_os_path( false, prefix_dir, bin_base, "orted", NULL );
}
/* If we yet did not fill up the execpath, do so now */
if (NULL == exec_path) {
rc = orte_pls_rsh_fill_exec_path (&exec_path);
if (ORTE_SUCCESS != rc) {
exit(-1); /* the forked process MUST exit */
}
}
}
/* If we have a prefix, then modify the PATH and
LD_LIBRARY_PATH environment variables. We're
already in the child process, so it's ok to modify
environ. */
if (NULL != prefix_dir) {
char *oldenv, *newenv;
/* Reset PATH */
newenv = opal_os_path( false, prefix_dir, bin_base, NULL );
oldenv = getenv("PATH");
if (NULL != oldenv) {
char *temp;
asprintf(&temp, "%s:%s", newenv, oldenv );
free( newenv );
newenv = temp;
}
opal_setenv("PATH", newenv, true, &environ);
if (mca_pls_rsh_component.debug) {
opal_output(0, "pls:rsh: reset PATH: %s", newenv);
}
free(newenv);
/* Reset LD_LIBRARY_PATH */
newenv = opal_os_path( false, prefix_dir, lib_base, NULL );
oldenv = getenv("LD_LIBRARY_PATH");
if (NULL != oldenv) {
char* temp;
asprintf(&temp, "%s:%s", newenv, oldenv);
free(newenv);
newenv = temp;
}
opal_setenv("LD_LIBRARY_PATH", newenv, true, &environ);
if (mca_pls_rsh_component.debug) {
opal_output(0, "pls:rsh: reset LD_LIBRARY_PATH: %s",
newenv);
}
free(newenv);
}
/* Since this is a local execution, we need to
potentially whack the final ")" in the argv (if
sh/csh conditionals, from above). Note that we're
modifying the argv[] in the child process, so
there's no need to save this and restore it
afterward -- the parent's argv[] is unmodified. */
if (NULL != argv[local_exec_index_end]) {
free(argv[local_exec_index_end]);
argv[local_exec_index_end] = NULL;
}
/* tell the daemon to setup its own process session/group */
opal_argv_append(&argc, &argv, "--set-sid");
exec_argv = &argv[local_exec_index];
/* Finally, chdir($HOME) because we're making the
assumption that this is what will happen on
remote nodes (via rsh/ssh). This allows a user
to specify a path that is relative to $HOME for
both the cwd and argv[0] and it will work on
all nodes -- including the local nost.
Otherwise, it would work on remote nodes and
not the local node. If the user does not start
in $HOME on the remote nodes... well... let's
hope they start in $HOME. :-) */
var = getenv("HOME");
if (NULL != var) {
if (mca_pls_rsh_component.debug) {
opal_output(0, "pls:rsh: changing to directory %s", var);
}
/* Ignore errors -- what are we going to do?
(and we ignore errors on the remote nodes
in the fork pls, so this is consistent) */
chdir(var);
}
} else {
if (mca_pls_rsh_component.debug) {
opal_output(0, "pls:rsh: %s is a REMOTE node\n",
rmaps_node->nodename);
}
exec_argv = argv;
exec_path = strdup(mca_pls_rsh_component.agent_path);
if (NULL != prefix_dir) {
char *opal_prefix = getenv("OPAL_PREFIX");
if (remote_sh) {
asprintf (&argv[local_exec_index],
"%s%s%s PATH=%s/%s:$PATH ; export PATH ; "
"LD_LIBRARY_PATH=%s/%s:$LD_LIBRARY_PATH ; export LD_LIBRARY_PATH ; "
"%s/%s/%s",
(opal_prefix != NULL ? "OPAL_PREFIX=" : ""),
(opal_prefix != NULL ? opal_prefix : ""),
(opal_prefix != NULL ? " ;" : ""),
prefix_dir, bin_base,
prefix_dir, lib_base,
prefix_dir, bin_base,
mca_pls_rsh_component.orted);
}
if (remote_csh) {
/* [t]csh is a bit more challenging -- we
have to check whether LD_LIBRARY_PATH
is already set before we try to set it.
Must be very careful about obeying
[t]csh's order of evaluation and not
using a variable before it is defined.
See this thread for more details:
http://www.open-mpi.org/community/lists/users/2006/01/0517.php. */
asprintf (&argv[local_exec_index],
"%s%s%s set path = ( %s/%s $path ) ; "
"if ( $?LD_LIBRARY_PATH == 1 ) "
"set OMPI_have_llp ; "
"if ( $?LD_LIBRARY_PATH == 0 ) "
"setenv LD_LIBRARY_PATH %s/%s ; "
"if ( $?OMPI_have_llp == 1 ) "
"setenv LD_LIBRARY_PATH %s/%s:$LD_LIBRARY_PATH ; "
"%s/%s/%s",
(opal_prefix != NULL ? "setenv OPAL_PREFIX " : ""),
(opal_prefix != NULL ? opal_prefix : ""),
(opal_prefix != NULL ? " ;" : ""),
prefix_dir, bin_base,
prefix_dir, lib_base,
prefix_dir, lib_base,
prefix_dir, bin_base,
mca_pls_rsh_component.orted);
}
}
}
/* setup process name */
rc = orte_ns.get_proc_name_string(&name_string, name);
if (ORTE_SUCCESS != rc) {
opal_output(0, "orte_pls_rsh: unable to create process name");
exit(-1);
}
free(argv[proc_name_index]);
argv[proc_name_index] = strdup(name_string);
if (!mca_pls_rsh_component.debug) {
/* setup stdin */
int fd = open("/dev/null", O_RDWR);
dup2(fd, 0);
close(fd);
}
/* close all file descriptors w/ exception of stdin/stdout/stderr */
for(fd=3; fd<fdmax; fd++)
close(fd);
/* Set signal handlers back to the default. Do this close
to the execve() because the event library may (and likely
will) reset them. If we don't do this, the event
library may have left some set that, at least on some
OS's, don't get reset via fork() or exec(). Hence, the
orted could be unkillable (for example). */
set_handler_default(SIGTERM);
set_handler_default(SIGINT);
set_handler_default(SIGHUP);
set_handler_default(SIGPIPE);
set_handler_default(SIGCHLD);
/* Unblock all signals, for many of the same reasons that
we set the default handlers, above. This is noticable
on Linux where the event library blocks SIGTERM, but we
don't want that blocked by the orted (or, more
specifically, we don't want it to be blocked by the
orted and then inherited by the ORTE processes that it
forks, making them unkillable by SIGTERM). */
sigprocmask(0, 0, &sigs);
sigprocmask(SIG_UNBLOCK, &sigs, 0);
/* setup environment */
env = opal_argv_copy(environ);
var = mca_base_param_environ_variable("seed",NULL,NULL);
opal_setenv(var, "0", true, &env);
/* exec the daemon */
if (mca_pls_rsh_component.debug) {
param = opal_argv_join(exec_argv, ' ');
if (NULL != param) {
char* env_array = opal_argv_join( env, ' ' );
opal_output(0, "pls:rsh: executing: (%s) %s [%s]",
exec_path, param, env_array);
free(param); free(env_array);
}
}
execve(exec_path, exec_argv, env);
opal_output(0, "pls:rsh: execv of %s failed with errno=%s(%d)\n",
exec_path, strerror(errno), errno);
exit(-1);
} else { /* father */
OPAL_THREAD_LOCK(&mca_pls_rsh_component.lock);
/* JJH Bug:
* If we are in '--debug-daemons' we keep the ssh connection
* alive for the span of the run. If we use this option
* AND we launch on more than "num_concurrent" machines
* then we will deadlock. No connections are terminated
* until the job is complete, no job is started
* since all the orteds are waiting for all the others
* to come online, and the others ore not launched because
* we are waiting on those that have started to terminate
* their ssh tunnels. :(
*/
if (mca_pls_rsh_component.num_children++ >=
mca_pls_rsh_component.num_concurrent) {
opal_condition_wait(&mca_pls_rsh_component.cond, &mca_pls_rsh_component.lock);
}
OPAL_THREAD_UNLOCK(&mca_pls_rsh_component.lock);
/* setup callback on sigchild - wait until setup above is complete
* as the callback can occur in the call to orte_wait_cb
*/
orte_wait_cb(pid, orte_pls_rsh_wait_daemon, dmn);
/* if required - add delay to avoid problems w/ X11 authentication */
if (mca_pls_rsh_component.debug && mca_pls_rsh_component.delay) {
sleep(mca_pls_rsh_component.delay);
}
vpid++;
}
free(name);
}
/* all done, so store the daemon info on the registry */
if (ORTE_SUCCESS != (rc = orte_pls_base_store_active_daemons(&active_daemons))) {
ORTE_ERROR_LOG(rc);
}
cleanup:
OBJ_RELEASE(map);
if (NULL != lib_base) {
free(lib_base);
}
if (NULL != bin_base) {
free(bin_base);
}
if (NULL != jobid_string) free(jobid_string); /* done with this variable */
if (NULL != argv) opal_argv_free(argv);
return rc;
}