static bool var_info_insert(var_info v_info)
{
char buf[MAX_STR];
/* This extra renaming step is here because of a slight difference
in hash table semantics-- in dhash (which env uses), a scan will
see hidden bindings, but not so w/ Jeff's hash implementation
*/
snprintf(buf, MAX_STR, "%s::%d", v_info->name, acnt.collection_count++);
if(v_info->visible)
hash_table_insert(collection_hash,(hash_key)rstrdup(banshee_nonptr_region, buf),
(hash_data)v_info->t_type);
switch (v_info->kind)
{
case vk_local :
{
env_add(local_env, v_info->name, v_info);
return TRUE;
}
break;
case vk_static :
{
env_add(file_env, v_info->name, v_info);
return TRUE;
}
break;
case vk_global :
{
assert(!seen_global(v_info->name));
//env_add(global_var_env, v_info->name, v_info);
hash_table_insert(global_var_hash, (hash_key)v_info->name,
(hash_data)v_info);
return FALSE;
}
break;
}
assert(0);
return FALSE;
}
/*
* env_add, but only if it does not already exist
*/
static void
env_add_if_not(const char *name, const char *value)
{
int i, len;
len = strlen(name);
for (i=0; i<envp_pos; i++) {
if (strchr(envp[i], '=') - envp[i] != len)
continue;
if (!strncmp(envp[i], name, len))
return;
}
env_add(name, value);
}
int env_put2(const char *s,const char *t)
{
char *u;
int slen;
if (!env_isinit) if (!env_init()) return 0;
slen = str_len(s);
u = alloc(slen + str_len(t) + 2);
if (!u) return 0;
str_copy(u,s);
u[slen] = '=';
str_copy(u + slen + 1,t);
if (!env_add(u)) { alloc_free(u); return 0; }
return 1;
}
// parse and store the environment setting
void env_store(const char *str, ENV_OP op) {
EUID_ASSERT();
assert(str);
// some basic checking
if (*str == '\0')
goto errexit;
char *ptr = strchr(str, '=');
if (op == SETENV) {
if (!ptr)
goto errexit;
ptr++;
if (*ptr == '\0')
goto errexit;
}
// build list entry
Env *env = malloc(sizeof(Env));
if (!env)
errExit("malloc");
memset(env, 0, sizeof(Env));
env->name = strdup(str);
if (env->name == NULL)
errExit("strdup");
if (op == SETENV) {
char *ptr2 = strchr(env->name, '=');
assert(ptr2);
*ptr2 = '\0';
env->value = ptr2 + 1;
}
env->op = op;
// add entry to the list
env_add(env);
return;
errexit:
fprintf(stderr, "Error: invalid --env setting\n");
exit(1);
}
void define_internal_attribute(const char *name,
void (*handle_ndecl)(nesc_attribute attr,
nesc_declaration ndecl),
void (*handle_decl)(nesc_attribute attr,
data_declaration ddecl),
void (*handle_tag)(nesc_attribute attr,
tag_declaration tdecl),
void (*handle_field)(nesc_attribute attr,
field_declaration fdecl),
void (*handle_type)(nesc_attribute attr,
type *t),
...)
{
va_list args;
field_declaration *next_field;
word attr_word;
type_element attr_tag;
tag_declaration attr_decl;
struct internal_attribute *iattr;
/* Build and declare the attribute */
current.env = global_env;
attr_word = build_word(parse_region, name);
attr_tag = start_struct(dummy_location, kind_attribute_ref, attr_word);
attr_decl = CAST(tag_ref, attr_tag)->tdecl;
attr_decl->fields = new_env(parse_region, NULL);
next_field = &attr_decl->fieldlist;
/* Fields. A fieldname, fieldtype argument list, terminated with a
null fieldname. We build a semi-fake struct for these.
*/
va_start(args, handle_type);
for (;;)
{
const char *field_name = va_arg(args, const char *);
field_declaration field;
if (!field_name)
break;
field = ralloc(parse_region, struct field_declaration);
field->containing_tag = attr_decl;
*next_field = field;
next_field = &field->next;
field->name = field_name;
field->type = va_arg(args, type);
field->bitwidth = field->offset = cval_unknown_number;
env_add(attr_decl->fields, field_name, field);
}
va_end(args);
/* Add to internal attributes table */
iattr = ralloc(permanent, struct internal_attribute);
iattr->name = name;
iattr->handle_ndecl = handle_ndecl;
iattr->handle_decl = handle_decl;
iattr->handle_tag = handle_tag;
iattr->handle_field = handle_field;
iattr->handle_type = handle_type;
env_add(internal_attributes, name, iattr);
}
struct value env_lookup(struct env e, char *name)
{
int i;
struct value err = {.type = VERR, .v = 3};
for(i = e.top - 1; i >= 0; i--)
{
if(strcmp(name, e.tab[i].name) == 0)
return e.tab[i].v;
}
return err;
}
struct value value_int(int v)
{
struct value x = {.type = VINT, .v = v};
return x;
}
struct value value_bool(bool b)
{
struct value x = {.type = VBOOL, .b = b};
return x;
}
struct value value_err(struct value v1, struct value v2, int e)
{
struct value x = {.type = VERR, .v = e};
if(v1.type == VERR)
return v1;
if(v2.type == VERR)
return v2;
return x;
}
struct sexp *atom_i(int v)
{
struct sexp *s = malloc(sizeof(struct sexp));
s->type = ATOM_I;
s->atom_i = v;
return s;
}
struct sexp *atom_b(bool v)
{
struct sexp *s = malloc(sizeof(struct sexp));
s->type = ATOM_B;
s->atom_b = v;
return s;
}
struct sexp *atom_n(char *n)
{
struct sexp *s = malloc(sizeof(struct sexp));
s->type = ATOM_N;
strcpy(s->atom_n, n);
return s;
}
struct sexp *sexp(enum op op, struct sexp *s1, struct sexp *s2, struct sexp *s3)
{
struct sexp *s = malloc(sizeof(struct sexp));
s->type = SEXP;
s->sexp.op = op;
s->sexp.s1 = s1;
s->sexp.s2 = s2;
s->sexp.s3 = s3;
return s;
}
struct value eval(struct env env, struct sexp *s)
{
struct value v1, v2, v3;
switch(s->type)
{
case ATOM_B:
return value_bool(s->atom_b);
case ATOM_I:
return value_int(s->atom_i);
case ATOM_N:
return env_lookup(env, s->atom_n);
default:
switch(s->sexp.op) {
case ADD:
v1 = eval(env, s->sexp.s1);
v2 = eval(env, s->sexp.s2);
if(v1.type == VINT &&
v2.type == VINT)
return value_int(v1.v + v2.v);
else
return value_err(v1, v2, 1);
case SUB:
v1 = eval(env, s->sexp.s1);
v2 = eval(env, s->sexp.s2);
if(v1.type == VINT &&
v2.type == VINT)
return value_int(v1.v - v2.v);
else
return value_err(v1, v2, 1);
case MUL:
v1 = eval(env, s->sexp.s1);
v2 = eval(env, s->sexp.s2);
if(v1.type == VINT &&
v2.type == VINT)
return value_int(v1.v * v2.v);
else
return value_err(v1, v2, 1);
case DIV:
v1 = eval(env, s->sexp.s1);
v2 = eval(env, s->sexp.s2);
if(v1.type == VINT &&
v2.type == VINT)
if(v2.v == 0)
return value_err(v1, v1, 2);
else
return value_int(v1.v / v2.v);
else
return value_err(v1, v2, 1);
case LT:
v1 = eval(env, s->sexp.s1);
v2 = eval(env, s->sexp.s2);
if(v1.type == VINT &&
v2.type == VINT)
return value_bool(v1.v < v2.v);
else
return value_err(v1, v2, 1);
case GT:
v1 = eval(env, s->sexp.s1);
v2 = eval(env, s->sexp.s2);
if(v1.type == VINT &&
v2.type == VINT)
return value_bool(v1.v > v2.v);
else
return value_err(v1, v2, 1);
case EQ:
v1 = eval(env, s->sexp.s1);
v2 = eval(env, s->sexp.s2);
if(v1.type == VINT &&
v2.type == VINT)
return value_bool(v1.v == v2.v);
else
return value_err(v1, v2, 1);
case IF:
v1 = eval(env, s->sexp.s1);
if(v1.type == VBOOL)
if(v1.b)
return eval(env, s->sexp.s2);
else
return eval(env, s->sexp.s3);
else
return value_err(v1, v1, 1);
case LET:
v1 = eval(env, s->sexp.s2);
if (v1.type == VERR) {
return v1;
}
return eval(env_add(env, s->sexp.s1->atom_n, eval(env, s->sexp.s2)), s->sexp.s3);
}
}
}
char line[256];
int pos;
char tok[16];
void run(struct sexp *s)
{
struct value v = eval(env0(), s);
if(v.type == VINT)
printf("%d\n", v.v);
else if(v.type == VBOOL)
printf("%s\n", v.b ? "true" : "false");
else {
if(v.v == 1)
printf("Type Mismatch\n");
else if (v.v == 2)
printf("Division By Zero\n");
else
printf("Unbound Identifier\n");
}
}
void lex()
{
int i;
tok[0] = 0;
while(line[pos] == ' ')
pos++;
switch(line[pos]) {
case '(':
case ')':
case '+':
case '-':
case '*':
case '/':
case '<':
case '>':
case '=':
tok[0] = line[pos];
tok[1] = 0;
pos++;
return;
default:
i = 0;
if(line[pos] >= '0' && line[pos] <= '9') {
while(line[pos] >= '0' && line[pos] <= '9')
tok[i++] = line[pos++];
tok[i] = 0;
return;
}
if(line[pos] >= 'a' && line[pos] <= 'z') {
while(line[pos] >= 'a' && line[pos] <= 'z')
tok[i++] = line[pos++];
tok[i] = 0;
return;
}
}
}
/**
* Run initial preparations for the shell
*
**/
void init(int argc, char *argv[]) {
int i;
char* script;
zshmode = 0;
debug = 0;
xflag = 0;
fileflag = 0;
// working directory
cwd = (char*)malloc(500*sizeof(char));
getcwd(cwd, 500);
// zsh prompt counter
env_com_count = 1;
// username and hostname
env_username = getenv("USER");
gethostname(env_hostname, HOSTSIZE);
// get term width
struct winsize ws;
ioctl(STDIN_FILENO, TIOCGWINSZ, &ws);
env_tty_width = ws.ws_col;
// setup background process handling
proc_init();
// setup env variables
env_init();
// set debug, xflag, and fileflag
if (argc > 1) {
for(i=1; i<argc; i++) {
if(!strcmp(argv[i], "-d") && (i + 1) < argc ) {
debug = !strcmp(argv[i+++1], "1");
dprint("debug mode activated");
}
else if(!strcmp(argv[i], "-x")) {
xflag = 1;
}
else if(!strcmp(argv[i], "-zsh")) {
zshmode = 1;
}
else if(strcmp(argv[i-1], "-d") && !fileflag) {
fileflag = 1;
script = argv[i];
if(i+1 < argc) {
int j;
//printf("argc = %d\n", argc);
for(j = i + 1; j < argc; j++) {
char *tc = malloc(100 * sizeof(char));
//char *dollar = malloc(100 * sizeof(char));
//sprintf(dollar, "%s", "");
sprintf(tc, "%d", j-i);
//strcat(dollar, tc);
//strcat(dollar, "=");
//strcat(dollar, argv[j]);
//printf("dollar = %s\n", dollar);
env_add(tc, argv[j]);
//putenv(dollar);
free(tc);
//free(dollar);
}
}
_run_file(script);
}
}
}
}
static int fdt_spawn_connection(server *srv,
fdt_server *s_server) {
int status;
struct timeval tv = { 0, 100 * 1000 };
struct sockaddr_un fdt_addr_un;
struct sockaddr *fdt_addr;
socklen_t servlen;
if (buffer_is_empty(s_server->unixsocket)) {
log_error_write(srv, __FILE__, __LINE__, "s",
"Socket path missing");
return -1;
}
memset(&fdt_addr, 0, sizeof(fdt_addr));
fdt_addr_un.sun_family = AF_UNIX;
strcpy(fdt_addr_un.sun_path, s_server->unixsocket->ptr);
servlen = s_server->unixsocket->used + sizeof(fdt_addr_un.sun_family);
fdt_addr = (struct sockaddr *) &fdt_addr_un;
/** We first try to connect to the backend to see if it is already present or not. If it is present then we do not need to fork that process*/
if (-1 == (s_server->fdt_sfd = socket(AF_UNIX, SOCK_STREAM, 0))) {
log_error_write(srv, __FILE__, __LINE__, "ss",
"failed:", strerror(errno));
return -1;
}
if (-1 == connect(s_server->fdt_sfd, fdt_addr, servlen)) {
/* server is not up, spawn it */
log_error_write(srv, __FILE__, __LINE__, "sb",
"spawning a new fdt proc, socket:", s_server->unixsocket);
pid_t child;
int val;
if (errno != ENOENT &&
!buffer_is_empty(s_server->unixsocket)) {
unlink(s_server->unixsocket->ptr);
}
close(s_server->fdt_sfd);
/** here we create a socket bind to address and then go in listen state. Reason behind this is that the server process is ideally suppose to start and then listen on this socket but it so happens that there is a race condition that lighttpd does connect before the child could do listen so in that case connect fails 2nd time so the best way to do is do the listen in parent itself and then do fork and then after fork in child process that listen socket will still be open. While in parent process we can close that socket and reopen it again and do connect*/
/* reopen socket */
if (-1 == (s_server->fdt_sfd = socket(AF_UNIX, SOCK_STREAM, 0))) {
log_error_write(srv, __FILE__, __LINE__, "ss",
"socket failed:", strerror(errno));
return -1;
}
val = 1;
if (setsockopt(s_server->fdt_sfd, SOL_SOCKET, SO_REUSEADDR, &val, sizeof(val)) < 0) {
log_error_write(srv, __FILE__, __LINE__, "ss",
"socketsockopt failed:", strerror(errno));
return -1;
}
/* create socket */
if (-1 == bind(s_server->fdt_sfd, fdt_addr, servlen)) {
log_error_write(srv, __FILE__, __LINE__, "sbs",
"bind failed for:",
s_server->service_name,
strerror(errno));
return -1;
}
if (-1 == listen(s_server->fdt_sfd, 1024)) {
log_error_write(srv, __FILE__, __LINE__, "ss",
"listen failed:", strerror(errno));
return -1;
}
switch ((child = fork())) {
case 0: {
size_t i = 0;
char *c;
char_array env;
char_array arg;
/* create environment */
env.ptr = NULL;
env.size = 0;
env.used = 0;
arg.ptr = NULL;
arg.size = 0;
arg.used = 0;
/** build clean enviornment basically says what all environ variables do you want to copy from the environ of the lighttpd and pass it to child process. Currently we dont need this right now but we mihgt need it later on
// build clean environment
if (s_server->bin_env_copy->used) {
for (i = 0; i < s_server->bin_env_copy->used; i++) {
data_string *ds = (data_string *)s_server->bin_env_copy->data[i];
char *ge;
if (NULL != (ge = getenv(ds->value->ptr))) {
env_add(&env, CONST_BUF_LEN(ds->value), ge, strlen(ge));
}
}
} else {
for (i = 0; environ[i]; i++) {
char *eq;
if (NULL != (eq = strchr(environ[i], '='))) {
env_add(&env, environ[i], eq - environ[i], eq+1, strlen(eq+1));
}
}
}*/
for (i = 0; environ[i]; i++) {
char *eq;
if (NULL != (eq = strchr(environ[i], '='))) {
env_add(&env, environ[i], eq - environ[i], eq+1, strlen(eq+1));
}
}
/* create environment */
for (i = 0; i < s_server->bin_env->used; i++) {
data_string *ds = (data_string *)s_server->bin_env->data[i];
env_add(&env, CONST_BUF_LEN(ds->key), CONST_BUF_LEN(ds->value));
}
/** The socket file descirtor handle is passed in environmental variables to the client.*/
char *env_key = calloc(1, sizeof(char)*10);
char *env_value = calloc(1, sizeof(char)*4);
memcpy(env_key, "SOCKET_FD", 9);
sprintf(env_value,"%d", s_server->fdt_sfd);
env_add(&env, env_key, 9, env_value, strlen(env_value));
/* log_error_write(srv, __FILE__, __LINE__, "ss",
"socket fd in child process is", env_value);*/
free(env_key);
free(env_value);
env.ptr[env.used] = NULL;
parse_binpath(&arg, s_server->bin_path);
/* chdir into the base of the bin-path,
* search for the last / */
if (NULL != (c = strrchr(arg.ptr[0], '/'))) {
*c = '\0';
/* change to the physical directory */
if (-1 == chdir(arg.ptr[0])) {
*c = '/';
log_error_write(srv, __FILE__, __LINE__, "sss", "chdir failed:", strerror(errno), arg.ptr[0]);
}
*c = '/';
}
reset_signals();
/* exec the fdt process */
execve(arg.ptr[0], arg.ptr, env.ptr);
log_error_write(srv, __FILE__, __LINE__, "sbs",
"execve failed for:", s_server->bin_path, strerror(errno));
exit(errno);
break;
}
case -1:
/* error */
break;
default:
/* father */
/* wait */
select(0, NULL, NULL, NULL, &tv);
switch (waitpid(child, &status, WNOHANG)) {
case 0:
/* child still running after timeout, good */
break;
case -1:
/* no PID found ? should never happen */
log_error_write(srv, __FILE__, __LINE__, "ss",
"pid not found:", strerror(errno));
return -1;
default:
log_error_write(srv, __FILE__, __LINE__, "sbs",
"the fdt-transfer-backend", s_server->bin_path, "failed to start:");
/* the child should not terminate at all */
if (WIFEXITED(status)) {
log_error_write(srv, __FILE__, __LINE__, "sdb",
"child exited with status",
WEXITSTATUS(status), s_server->bin_path);
} else if (WIFSIGNALED(status)) {
log_error_write(srv, __FILE__, __LINE__, "sd",
"terminated by signal:",
WTERMSIG(status));
if (WTERMSIG(status) == 11) {
log_error_write(srv, __FILE__, __LINE__, "s",
"to be exact: it segfaulted, crashed, died, ... you get the idea." );
}
} else {
log_error_write(srv, __FILE__, __LINE__, "sd",
"child died somehow:",
status);
}
return -1;
}
/* register process */
s_server->pid = child;
close(s_server->fdt_sfd);
/* reopen socket */
if (-1 == (s_server->fdt_sfd = socket(AF_UNIX, SOCK_STREAM, 0))) {
log_error_write(srv, __FILE__, __LINE__, "ss",
"socket failed:", strerror(errno));
return -1;
}
if (-1 == connect(s_server->fdt_sfd, fdt_addr, servlen)) {
log_error_write(srv, __FILE__, __LINE__, "ss",
"connect failed 2nd time This should never happen:", strerror(errno));
return -1;
}
fdevent_register(srv->ev, s_server->fdt_sfd, fdt_handle_fdevent, s_server);
if (-1 == fdevent_fcntl_set(srv->ev, s_server->fdt_sfd)) {
log_error_write(srv, __FILE__, __LINE__, "ss",
"fcntl failed:", strerror(errno));
return HANDLER_ERROR;
}
fdevent_event_set(srv->ev, NULL, s_server->fdt_sfd, FDEVENT_HUP);
break;
}
} else {
s_server->pid = 0;
log_error_write(srv, __FILE__, __LINE__, "sb",
"(debug) socket is already used; won't spawn:",
s_server->service_name);
}
s_server->state = PROC_STATE_RUNNING;
return 0;
}
/*
* Start the tasks, with much stuff in the environment. If concurrent
* master, this could be on behalf of some other mpiexec to which we
* will forward any event/error results.
*/
int
start_tasks(int spawn)
{
int i, ret = 0;
char *nargv[3];
char pwd[PATH_MAX];
char *cp;
int conns[3]; /* expected connections to the stdio process */
int master_port = 0;
const char *user_shell;
growstr_t *g;
int gmpi_port[2];
int pmi_fd;
int task_start, task_end;
const char *mpiexec_redir_helper_path;
char *psm_uuid = NULL;
int tv_port = 0;
/* for looping from 0..numtasks in the case of MPI_Spawn */
task_start = spawns[spawn].task_start;
task_end = spawns[spawn].task_end;
/*
* Get the pwd. Probably can trust libc not to overflow this,
* but who knows.
*/
if (!getcwd(pwd, sizeof(pwd)))
error("%s: no current working directory", __func__);
pwd[sizeof(pwd)-1] = '\0';
/*
* Eventually use the user's preferred shell.
*/
if ((cp = getenv("SHELL")))
user_shell = cp;
else if (pswd->pw_shell)
user_shell = pswd->pw_shell;
else
user_shell = "/bin/sh"; /* assume again */
/*
* Rewrite argv to go through user's shell, just like rsh.
* $SHELL, "-c", "cd <path>; exec <argv0> <argv1>..."
* But to change the working dir and not frighten weak shells like tcsh,
* we must detect that the dir actually exists on the far side before
* committing to the cd. Use /bin/sh for this task, hoping it exists
* everywhere we'll be. Then there's also a bit of quoting nightmare
* to handle too. So we'll end up with:
* rsh node "/bin/sh -c 'if test -d $dir ; then cd $dir ; fi ; $SHELL -c
* \'exec argv0 argv1 ...\''"
* but with argv* (including the executable, argv0) changed to replace
* all occurrences of ' with '\''.
*/
nargv[0] = strsave("/bin/sh"); /* assume this exists everywhere */
nargv[1] = strsave("-c");
/* exec_line constructed for each process */
g = growstr_init();
/*
* Start stdio stream handler process, if anybody gets stdin,
* or !nostdout.
*/
if (cl_args->which_stdin == STDIN_NONE)
conns[0] = 0;
else if (cl_args->which_stdin == STDIN_ONE) {
if (spawn == 0)
conns[0] = 1;
else
conns[0] = 0; /* already connected the single stdin */
} else if (cl_args->which_stdin == STDIN_ALL) {
/* total processes which connect stdin */
conns[0] = 0;
for (i=task_start; i<task_end; i++)
conns[0] += tasks[i].num_copies;
}
if (cl_args->nostdout)
conns[1] = conns[2] = 0;
else
/* even for p4 and shmem, not with multiplicity */
conns[1] = conns[2] = task_end - task_start;
/*
* Initialize listener sockets for gm and ib, since these will be
* used to implement MPI_Abort in the stdio listener later.
*/
if (cl_args->comm == COMM_MPICH_GM) {
prepare_gm_startup_ports(gmpi_port);
} else if (cl_args->comm == COMM_MPICH_IB) {
master_port = prepare_ib_startup_port(&gmpi_fd[0]);
gmpi_fd[1] = -1;
} else if (cl_args->comm == COMM_MPICH_PSM) {
master_port = prepare_psm_startup_port(&gmpi_fd[0]);
gmpi_fd[1] = -1;
} else if (cl_args->comm == COMM_MPICH_RAI) {
master_port = prepare_rai_startup_port();
gmpi_fd[0] = -1;
gmpi_fd[1] = -1;
} else {
gmpi_fd[0] = -1;
gmpi_fd[1] = -1;
}
pmi_fd = -1;
if (cl_args->comm == COMM_MPICH2_PMI) {
/* stdio listener handles all PMI activity, even startup */
if (spawn == 0)
master_port = prepare_pmi_startup_port(&pmi_fd);
else
master_port = stdio_msg_parent_say_more_tasks(
task_end - task_start, conns);
}
/* flush output buffer, else forked child will have the output too */
fflush(stdout);
/* fork the listener (unless we're just spawning more tasks) */
if (spawn == 0)
stdio_fork(conns, gmpi_fd, pmi_fd);
if (pmi_fd >= 0)
close(pmi_fd); /* child has it now */
numtasks_waiting_start = 0;
if (cl_args->comm == COMM_NONE)
/* do not complain if they exit before all other tasks are up */
startup_complete = 1;
else
startup_complete = 0;
/*
* Start signal handling _after_ stdio child is up.
*/
handle_signals(0, 0, killall);
/*
* environment variables common to all tasks
*/
env_init();
/* override user env with these */
if (cl_args->comm == COMM_MPICH_GM) {
env_add_int("GMPI_MAGIC", atoi(jobid));
/* PBS always gives us the "mother superior" node first in the list */
env_add("GMPI_MASTER", nodes[0].name);
env_add_int("GMPI_PORT", gmpi_port[0]); /* 1.2.5..10 */
env_add_int("GMPI_PORT1", gmpi_port[0]); /* 1.2.4..8a */
env_add_int("GMPI_PORT2", gmpi_port[1]);
env_add_int("GMPI_NP", numtasks);
env_add_int("GMPI_BOARD", -1);
/* ditto for new MX version */
env_add_int("MXMPI_MAGIC", atoi(jobid));
env_add("MXMPI_MASTER", nodes[0].name);
env_add_int("MXMPI_PORT", gmpi_port[0]);
env_add_int("MXMPI_NP", numtasks);
env_add_int("MXMPI_BOARD", -1);
/* for MACOSX to override default malloc */
env_add_int("DYLD_FORCE_FLAT_NAMESPACE", 1);
}
if (cl_args->comm == COMM_EMP) {
growstr_t *emphosts = growstr_init();
for (i=0; i<numtasks; i++)
growstr_printf(emphosts, "%s%s", (i > 0 ? " " : ""),
nodes[tasks[i].node].mpname);
env_add("EMPHOSTS", emphosts->s);
growstr_free(emphosts);
}
if (cl_args->comm == COMM_MPICH_IB || cl_args->comm == COMM_MPICH_RAI) {
int len;
char *cq, *cr;
env_add("MPIRUN_HOST", nodes[0].name); /* master address */
env_add_int("MPIRUN_PORT", master_port);
env_add_int("MPIRUN_NPROCS", numtasks);
env_add_int("MPIRUN_ID", atoi(jobid)); /* global job id */
/*
* pmgr_version >= 3 needs this terribly long string in every task.
* Since it may be quite large, we do the allocation by hand and
* skip some growstr overhead.
*/
len = numtasks; /* separating colons and terminal \0 */
for (i=0; i<numtasks; i++)
len += strlen(nodes[tasks[i].node].name);
cq = cp = Malloc(len);
for (i=0; i<numtasks; i++) {
for (cr=nodes[tasks[i].node].name; *cr; cr++)
*cq++ = *cr;
*cq++ = ':';
}
--cq;
*cq = '\0';
env_add("MPIRUN_PROCESSES", cp);
free(cp);
}
if (cl_args->comm == COMM_MPICH2_PMI) {
growstr_t *hp = growstr_init();
growstr_printf(hp, "%s:%d", nodes[0].name, master_port);
env_add("PMI_PORT", hp->s);
growstr_free(hp);
if (spawn > 0)
env_add_int("PMI_SPAWNED", 1);
}
if (cl_args->comm == COMM_PORTALS) {
growstr_t *nidmap = growstr_init();
growstr_t *pidmap = growstr_init();
portals_build_nidpid_maps(spawn, nidmap, pidmap);
env_add("PTL_NIDMAP", nidmap->s);
env_add("PTL_PIDMAP", pidmap->s);
growstr_free(nidmap);
growstr_free(pidmap);
env_add("PTL_IFACE", "eth0"); /* XXX: no way to know */
}
if (cl_args->comm == COMM_MPICH_P4 && numtasks > 1)
master_port = prepare_p4_master_port();
if (cl_args->comm == COMM_MPICH_PSM) {
/* We need to generate a uuid of the form
* 9dea0f22-39a4-462a-80c9-b60b28cdfd38. If /usr/bin/uuidgen exists,
* we should probably just use that.
* 4bytes-2bytes-2bytes-2bytes-6bytes
*/
char uuid_packed[16];
unsigned char *p = (unsigned char *) uuid_packed;
int fd, rret;
fd = open("/dev/urandom", O_RDONLY);
if (fd < 0)
error_errno("%s: open /dev/urandom", __func__);
rret = read_full_ret(fd, uuid_packed, sizeof(uuid_packed));
if (rret < 0)
error_errno("%s: read /dev/urandom", __func__);
if (rret != sizeof(uuid_packed))
error("%s: short read /dev/urandom", __func__);
close(fd);
psm_uuid = Malloc(37); /* 16 * 2 + 4 + 1 */
snprintf(psm_uuid, 37,
"%02x%02x%02x%02x-%02x%02x-%02x%02x-"
"%02x%02x-%02x%02x%02x%02x%02x%02x",
p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7],
p[8], p[9], p[10], p[11], p[12], p[13], p[14], p[15]);
psm_uuid[36] = '\0';
}
/*
* Ports on which to talk to listener process for stdout/stderr
* connection (if !-nostdout).
*/
if (stdio_port(1) >= 0)
env_add_int("MPIEXEC_STDOUT_PORT", stdio_port(1));
if (stdio_port(2) >= 0)
env_add_int("MPIEXEC_STDERR_PORT", stdio_port(2));
/*
* Add our hostname too, for use by the redir-helper. And resolve
* it now via the user's path for use by the spawns.
*/
if (HAVE_PBSPRO_HELPER) {
env_add("MPIEXEC_HOST", nodes[0].name);
mpiexec_redir_helper_path = resolve_exe("mpiexec-redir-helper", 1);
}
/* now the env as given from pbs */
env_add_environ();
/* if pbs did not give us these, put in some defaults */
env_add_if_not("PATH", _PATH_DEFPATH);
env_add_if_not("USER", pswd->pw_name);
/*
* Set up for totalview attach. Returns local port number that will be
* used in command startup to tell processes how to find us. These two
* env vars are necessary in all processes. The first tells them to
* consume the tv_ready message. The second is checked in MPI_Init to
* determine if they should wait for all processes to be attached by
* totalview.
*/
if (cl_args->tview && cl_args->comm == COMM_MPICH2_PMI) {
env_add_int("PMI_TOTALVIEW", 1);
env_add_int("MPIEXEC_DEBUG", 1);
tv_port = tv_startup(task_end - task_start);
}
/*
* Spawn each task, adding its private env vars.
* numspawned set to zero earlier before signal handler setup;
* both it and i walk the iterations in the loop.
*/
for (i=task_start; i<task_end; i++) {
env_push();
if (cl_args->comm == COMM_MPICH_GM) {
/* build proc-specific gmpi_opts in envp */
env_add_int("GMPI_ID", i);
env_add_int("MXMPI_ID", i);
env_add("GMPI_SLAVE", nodes[tasks[i].node].name); /* 1.2.5..10 */
}
if (cl_args->comm == COMM_SHMEM) {
/* earlier in get_hosts we checked that there is only one task */
env_add_int("MPICH_NP", tasks[0].num_copies);
}
if (cl_args->comm == COMM_MPICH_IB || cl_args->comm == COMM_MPICH_RAI)
env_add_int("MPIRUN_RANK", i);
if (cl_args->comm == COMM_MPICH_IB) {
/* hack for topspin adaptation of mvapich 0.9.2 */
env_add("MPIRUN_NODENAME", nodes[tasks[i].node].name);
}
if (cl_args->comm == COMM_MPICH2_PMI) {
/* task id is always 0-based, even for spawn */
env_add_int("PMI_ID", i - task_start);
if (strcmp(nodes[tasks[i].node].mpname,
nodes[tasks[i].node].name) != 0)
env_add("MPICH_INTERFACE_HOSTNAME",
nodes[tasks[i].node].mpname);
}
if (cl_args->comm == COMM_MPICH_PSM) {
/* build one big string with everything in it */
char buf[2048];
snprintf(buf, sizeof(buf) - 1,
"%d %d %s %d %d %d %d %d %d %d %s",
0, /* protocol version */
0x4, /* protocol flags, ASYNC_SHUTDOWN=0x4 */
nodes[0].name, /* spawner host */
master_port, /* spawner port */
atoi(jobid), /* spawner jobid */
numtasks, /* COMM_WORLD size */
i - task_start, /* COMM_WORLD rank for this process */
nodes[tasks[i].node].numcpu, /* num local ranks */
tasks[i].cpu_index[0], /* my local rank */
60, /* timeout... */
psm_uuid);
buf[sizeof(buf) - 1] = '\0';
env_add("MPI_SPAWNER", buf);
}
if (cl_args->comm == COMM_PORTALS)
env_add_int("PTL_MY_RID", i);
if (cl_args->comm == COMM_NONE)
env_add_int("MPIEXEC_RANK", i);
/* either no stdin, or just to proc #0, or to all of them */
if (cl_args->which_stdin == STDIN_ONE && i == 0) {
env_add_int("MPIEXEC_STDIN_PORT", stdio_port(0));
/* do not add _HOST for p4, since we don't want
* the children of the big or remote master to
* connect. This _PORT is just for PBS, not for MPICH. */
}
if (cl_args->which_stdin == STDIN_ALL) {
env_add_int("MPIEXEC_STDIN_PORT", stdio_port(0));
if (cl_args->comm == COMM_MPICH_P4)
/* slave processes need to be told which host, as the stdin
* connection happens not in pbs_mom, but in mpich/p4 library
* code when it spawns each of the other tasks. */
env_add("MPIEXEC_STDIN_HOST", nodes[0].name);
}
env_terminate();
/* build proc-specific command line */
growstr_zero(g);
g->translate_single_quote = 0;
/*
* Totalview is a bit odd, even hackish perhaps. Send the pid
* the just-starting process to ourselves via /dev/tcp, some sort
* of virtual device that makes a TCP connection as told and sends
* the echoed data.
*/
if (cl_args->tview && cl_args->comm == COMM_MPICH2_PMI)
growstr_printf(g, "if hash nc > /dev/null; then printf %%10d $$ | nc %s %d; else printf %%10d $$ > /dev/tcp/%s/%d; fi; "
"if test -d \"%s\"; then cd \"%s\"; fi; exec %s -c ",
nodes[0].name, tv_port,
nodes[0].name, tv_port,
pwd, pwd, user_shell);
else
growstr_printf(g,
"if test -d \"%s\"; then cd \"%s\"; fi; exec %s -c ",
pwd, pwd, user_shell);
growstr_append(g, "'exec ");
g->translate_single_quote = 1;
/*
* PBSPro environments do not know how to redirect standard streams.
* So we fork a helper program that lives in the user's PATH, hopefully
* the same place as mpiexec, that does the redirection then execs the
* actual executable. This will break on OpenPBS or Torque, although
* I guess the redir helper could unset the env vars, but I'd rather
* people just didn't use the redir helper in that case.
*/
if (HAVE_PBSPRO_HELPER)
growstr_printf(g, "%s ", mpiexec_redir_helper_path);
/*
* The executable, or a debugger wrapper around it. In the mpich2
* case we don't need any special args.
*/
if (cl_args->tview && cl_args->comm != COMM_MPICH2_PMI) {
if (i == 0)
growstr_printf(g, "%s %s -a -mpichtv", tvname,
tasks[i].conf->exe);
else
growstr_printf(g, "%s -mpichtv", tasks[i].conf->exe);
} else
growstr_printf(g, "%s", tasks[i].conf->exe);
/* process arguments _before_ p4 arguments to allow xterm/gdb hack */
if (tasks[i].conf->args)
growstr_printf(g, " %s", tasks[i].conf->args);
if (cl_args->comm == COMM_MPICH_P4) {
/*
* Pass the cwd to ch_p4, else it tries to chdir(exedir). Thanks
* to Ben Webb <*****@*****.**> for fixing this.
*/
growstr_printf(g, " -p4wd %s", pwd);
/* The actual flag names are just for debugging; they're not used
* but the order is important. */
growstr_printf(g, " -execer_id mpiexec");
growstr_printf(g, " -master_host %s", nodes[tasks[0].node].mpname);
growstr_printf(g, " -my_hostname %s", nodes[tasks[i].node].mpname);
growstr_printf(g, " -my_nodenum %d", i);
growstr_printf(g, " -my_numprocs %d", tasks[i].num_copies);
growstr_printf(g, " -total_numnodes %d", numtasks);
growstr_printf(g, " -master_port %d", master_port);
if (i == 0 && numtasks > 1) {
int j;
/* list of: <hostname> <procs-on-that-node> */
growstr_printf(g, " -remote_info");
for (j=1; j<numtasks; j++)
growstr_printf(g, " %s %d",
nodes[tasks[j].node].mpname, tasks[j].num_copies);
}
}
g->translate_single_quote = 0;
growstr_printf(g, "'"); /* close quote for 'exec myjob ...' */
nargv[2] = g->s;
/*
* Dump all the info if sufficiently verbose.
*/
debug(2, "%s: command to %d/%d %s: %s", __func__, i, numtasks,
nodes[tasks[i].node].name, nargv[2]);
if (cl_args->verbose > 2) {
int j;
debug(3, "%s: environment to %d/%d %s", __func__, i,
numtasks, nodes[tasks[i].node].name);
for (j=0; (cp = envp[j]); j++)
printf("env %2d %s\n", j, cp);
}
if (concurrent_master) {
tm_event_t evt;
int err;
/* Note, would like to add obit immediately, but that is
* not allowed until the START message is polled.
*/
err = tm_spawn(list_count(nargv), nargv, envp,
nodes[tasks[i].node].ids[tasks[i].cpu_index[0]],
&tasks[i].tid, &evt);
if (err != TM_SUCCESS)
error_tm(err, "%s: tm_spawn task %d", __func__, i);
evt_add(evt, -1, i, EVT_START);
} else {
concurrent_request_spawn(i, list_count(nargv), nargv, envp,
nodes[tasks[i].node].ids[tasks[i].cpu_index[0]]);
}
tasks[i].done = DONE_NOT; /* has now been started */
env_pop();
++numspawned;
++numtasks_waiting_start;
if (cl_args->comm == COMM_MPICH_P4 && i == 0 && numtasks > 1) {
ret = wait_task_start();
if (ret)
break; /* don't bother trying to start the rest */
ret = read_p4_master_port(&master_port);
if (ret)
break;
}
/*
* Pay attention to incoming tasks so they don't time out while
* we're starting up all the others, non blocking.
*/
if (cl_args->comm == COMM_MPICH_IB) {
int one = 1;
for (;;) {
ret = service_ib_startup(one);
one = 0; /* only report the new task that first time */
if (ret < 0) {
ret = 1;
goto out;
}
if (ret == 0) /* nothing accomplished */
break;
}
}
if (cl_args->comm == COMM_MPICH_GM) {
int one = 1;
for (;;) {
ret = service_gm_startup(one);
one = 0; /* only report the new task that first time */
if (ret < 0) {
ret = 1;
goto out;
}
if (ret == 0) /* nothing accomplished */
break;
}
}
if (cl_args->comm == COMM_MPICH_PSM) {
int one = 1;
for (;;) {
ret = service_psm_startup(one);
one = 0; /* only report the new task that first time */
if (ret < 0) {
ret = 1;
goto out;
}
if (ret == 0) /* nothing accomplished */
break;
}
}
if (cl_args->tview && cl_args->comm == COMM_MPICH2_PMI)
tv_accept_one(i);
}
if (cl_args->tview && cl_args->comm == COMM_MPICH2_PMI)
tv_complete();
/* don't need these anymore */
free(nargv[0]);
free(nargv[1]);
growstr_free(g);
if (cl_args->comm == COMM_MPICH_PSM)
free(psm_uuid);
if (ret)
goto out;
/*
* Wait for spawn events and submit obit requests.
*/
while (numtasks_waiting_start) {
ret = wait_task_start();
if (ret)
goto out;
}
debug(1, "All %d task%s (spawn %d) started", task_end - task_start,
task_end - task_start > 1 ? "s": "", spawn);
/*
* Finalize mpi-specific startup protocal, e.g. wait for all tasks to
* checkin, perform barrier, etc.
*/
if (cl_args->comm == COMM_MPICH_GM)
ret = read_gm_startup_ports();
if (cl_args->comm == COMM_MPICH_IB)
ret = read_ib_startup_ports();
if (cl_args->comm == COMM_MPICH_PSM)
ret = read_psm_startup_ports();
if (cl_args->comm == COMM_MPICH_RAI)
ret = read_rai_startup_ports();
if (ret == 0)
startup_complete = 1;
out:
return ret;
}
