HYD_status HYDU_str_alloc_and_join(char **strlist, char **strjoin)
{
int len = 0, i, count;
HYD_status status = HYD_SUCCESS;
HYDU_FUNC_ENTER();
for (i = 0; strlist[i] != NULL; i++) {
len += strlen(strlist[i]);
}
HYDU_MALLOC(*strjoin, char *, len + 1, status);
count = 0;
(*strjoin)[0] = 0;
for (i = 0; strlist[i] != NULL; i++) {
HYDU_snprintf(*strjoin + count, len - count + 1, "%s", strlist[i]);
count += strlen(strlist[i]);
}
fn_exit:
HYDU_FUNC_EXIT();
return status;
fn_fail:
goto fn_exit;
}
char *HYDU_size_t_to_str(size_t x)
{
int len = 1, i;
size_t max = 10;
char *str = NULL;
HYD_status status = HYD_SUCCESS;
HYDU_FUNC_ENTER();
while (x >= max) {
len++;
max *= 10;
}
len++;
HYDU_MALLOC(str, char *, len, status);
HYDU_ERR_POP(status, "unable to allocate memory\n");
for (i = 0; i < len; i++)
str[i] = '0';
HYDU_snprintf(str, len, "%llu", (unsigned long long) x);
fn_exit:
HYDU_FUNC_EXIT();
return str;
fn_fail:
goto fn_exit;
}
static HYD_status pmi_kvsname_fn(char *arg, char ***argv)
{
HYDU_snprintf(HYD_pmcd_pmip.local.kvs->kvsname, PMI_MAXKVSLEN, "%s", **argv);
(*argv)++;
return HYD_SUCCESS;
}
HYD_status HYD_pmcd_pmi_add_kvs(const char *key, char *val, struct HYD_pmcd_pmi_kvs *kvs,
int *ret)
{
struct HYD_pmcd_pmi_kvs_pair *key_pair, *run;
HYD_status status = HYD_SUCCESS;
HYDU_FUNC_ENTER();
HYDU_MALLOC(key_pair, struct HYD_pmcd_pmi_kvs_pair *, sizeof(struct HYD_pmcd_pmi_kvs_pair),
status);
HYDU_snprintf(key_pair->key, PMI_MAXKEYLEN, "%s", key);
HYDU_snprintf(key_pair->val, PMI_MAXVALLEN, "%s", val);
key_pair->next = NULL;
*ret = 0;
if (kvs->key_pair == NULL) {
kvs->key_pair = key_pair;
}
else {
run = kvs->key_pair;
while (run->next) {
if (!strcmp(run->key, key_pair->key)) {
/* duplicate key found */
*ret = -1;
break;
}
run = run->next;
}
run->next = key_pair;
}
fn_exit:
HYDU_FUNC_EXIT();
return status;
fn_fail:
goto fn_exit;
}
static HYD_status pmi_spawner_kvsname_fn(char *arg, char ***argv)
{
HYD_status status = HYD_SUCCESS;
HYDU_MALLOC(HYD_pmcd_pmip.local.spawner_kvsname, char *, PMI_MAXKVSLEN, status);
HYDU_snprintf(HYD_pmcd_pmip.local.spawner_kvsname, PMI_MAXKVSLEN, "%s", **argv);
(*argv)++;
fn_exit:
return status;
fn_fail:
goto fn_exit;
}
HYD_status HYD_pmcd_pmi_allocate_kvs(struct HYD_pmcd_pmi_kvs ** kvs, int pgid)
{
HYD_status status = HYD_SUCCESS;
HYDU_FUNC_ENTER();
HYDU_MALLOC(*kvs, struct HYD_pmcd_pmi_kvs *, sizeof(struct HYD_pmcd_pmi_kvs), status);
HYDU_snprintf((*kvs)->kvs_name, PMI_MAXKVSLEN, "kvs_%d_%d", (int) getpid(), pgid);
(*kvs)->key_pair = NULL;
fn_exit:
HYDU_FUNC_EXIT();
return status;
fn_fail:
goto fn_exit;
}
char *HYDU_int_to_str_pad(int x, int maxlen)
{
int len = 1, max = 10, y;
int actual_len, i;
char *str = NULL;
HYD_status status = HYD_SUCCESS;
HYDU_FUNC_ENTER();
if (x < 0) {
len++;
y = -x;
}
else
y = x;
while (y >= max) {
len++;
max *= 10;
}
if (len > maxlen)
actual_len = len + 1;
else
actual_len = maxlen + 1;
HYDU_MALLOC(str, char *, actual_len, status);
HYDU_ERR_POP(status, "unable to allocate memory\n");
for (i = 0; i < actual_len; i++)
str[i] = '0';
HYDU_snprintf(str + actual_len - len - 1, len + 1, "%d", x);
fn_exit:
HYDU_FUNC_EXIT();
return str;
fn_fail:
goto fn_exit;
}
static HYD_status cmd_response(int fd, int pid, char *cmd)
{
char cmdlen[7];
struct HYD_pmcd_hdr hdr;
int sent, closed;
HYD_status status = HYD_SUCCESS;
HYDU_FUNC_ENTER();
HYD_pmcd_init_header(&hdr);
hdr.cmd = PMI_RESPONSE;
hdr.pid = pid;
hdr.pmi_version = 2;
hdr.buflen = 6 + strlen(cmd);
status = HYDU_sock_write(fd, &hdr, sizeof(hdr), &sent, &closed, HYDU_SOCK_COMM_MSGWAIT);
HYDU_ERR_POP(status, "unable to send PMI_RESPONSE header to proxy\n");
HYDU_ASSERT(!closed, status);
HYDU_snprintf(cmdlen, 7, "%6u", (unsigned) strlen(cmd));
status = HYDU_sock_write(fd, cmdlen, 6, &sent, &closed, HYDU_SOCK_COMM_MSGWAIT);
HYDU_ERR_POP(status, "error writing PMI line\n");
HYDU_ASSERT(!closed, status);
if (HYD_server_info.user_global.debug) {
HYDU_dump(stdout, "PMI response to fd %d pid %d: %s\n", fd, pid, cmd);
}
status = HYDU_sock_write(fd, cmd, strlen(cmd), &sent, &closed, HYDU_SOCK_COMM_MSGWAIT);
HYDU_ERR_POP(status, "error writing PMI line\n");
HYDU_ASSERT(!closed, status);
fn_exit:
HYDU_FUNC_EXIT();
return status;
fn_fail:
goto fn_exit;
}
HYD_status HYDT_bscd_ll_launch_procs(char **args, struct HYD_proxy *proxy_list,
int *control_fd)
{
int idx, i, total_procs, node_count;
int *pid, *fd_list, exec_idx;
char *targs[HYD_NUM_TMP_STRINGS], *node_list_str = NULL;
char *path = NULL, *extra_arg_list = NULL, *extra_arg, quoted_exec_string[HYD_TMP_STRLEN];
struct HYD_proxy *proxy;
struct HYDT_topo_cpuset_t cpuset;
HYD_status status = HYD_SUCCESS;
HYDU_FUNC_ENTER();
/* We use the following priority order for the executable path:
* (1) user-specified; (2) search in path; (3) Hard-coded
* location */
if (HYDT_bsci_info.launcher_exec)
path = HYDU_strdup(HYDT_bsci_info.launcher_exec);
if (!path)
path = HYDU_find_full_path("poe");
if (!path)
path = HYDU_strdup("/usr/bin/poe");
idx = 0;
targs[idx++] = HYDU_strdup(path);
if (!strcmp(HYDT_bsci_info.rmk, "ll")) {
HYDU_ERR_SETANDJUMP(status, HYD_INTERNAL_ERROR,
"ll does not support user-defined host lists\n");
}
/* Check how many nodes are being passed for the launch */
status = HYDTI_bscd_ll_query_node_count(&total_procs);
HYDU_ERR_POP(status, "unable to query for the node count\n");
node_count = 0;
for (proxy = proxy_list; proxy; proxy = proxy->next)
node_count++;
if (total_procs != node_count)
HYDU_ERR_SETANDJUMP(status, HYD_INTERNAL_ERROR,
"processes to be launched have to cover all nodes\n");
MPL_env2str("HYDRA_LAUNCHER_EXTRA_ARGS", (const char **) &extra_arg_list);
if (extra_arg_list) {
extra_arg = strtok(extra_arg_list, " ");
while (extra_arg) {
targs[idx++] = HYDU_strdup(extra_arg);
extra_arg = strtok(NULL, " ");
}
}
/* Fill in the remaining arguments */
exec_idx = idx;
for (i = 0; args[i]; i++)
targs[idx++] = HYDU_strdup(args[i]);
/* Create a quoted version of the exec string, which is only used
* when the executable is not launched directly, but through an
* actual launcher */
HYDU_snprintf(quoted_exec_string, HYD_TMP_STRLEN, "\"%s\"", targs[exec_idx]);
HYDU_FREE(targs[exec_idx]);
targs[exec_idx] = quoted_exec_string;
/* Increase pid list to accommodate the new pid */
HYDU_MALLOC(pid, int *, (HYD_bscu_pid_count + 1) * sizeof(int), status);
for (i = 0; i < HYD_bscu_pid_count; i++)
pid[i] = HYD_bscu_pid_list[i];
HYDU_FREE(HYD_bscu_pid_list);
HYD_bscu_pid_list = pid;
/* Increase fd list to accommodate these new fds */
HYDU_MALLOC(fd_list, int *, (HYD_bscu_fd_count + 3) * sizeof(int), status);
for (i = 0; i < HYD_bscu_fd_count; i++)
fd_list[i] = HYD_bscu_fd_list[i];
HYDU_FREE(HYD_bscu_fd_list);
HYD_bscu_fd_list = fd_list;
/* append proxy ID as -1 */
targs[idx++] = HYDU_int_to_str(-1);
targs[idx++] = NULL;
HYDT_topo_cpuset_zero(&cpuset);
status = HYDU_create_process(targs, NULL, NULL, &fd_stdout, &fd_stderr,
&HYD_bscu_pid_list[HYD_bscu_pid_count++], cpuset);
HYDU_ERR_POP(status, "create process returned error\n");
HYD_bscu_fd_list[HYD_bscu_fd_count++] = fd_stdout;
HYD_bscu_fd_list[HYD_bscu_fd_count++] = fd_stderr;
status = HYDT_dmx_register_fd(1, &fd_stdout, HYD_POLLIN,
(void *) (size_t) STDOUT_FILENO, HYDT_bscu_stdio_cb);
HYDU_ERR_POP(status, "demux returned error registering fd\n");
status = HYDT_dmx_register_fd(1, &fd_stderr, HYD_POLLIN,
(void *) (size_t) STDERR_FILENO, HYDT_bscu_stdio_cb);
HYDU_ERR_POP(status, "demux returned error registering fd\n");
fn_exit:
if (node_list_str)
HYDU_FREE(node_list_str);
HYDU_free_strlist(targs);
if (path)
HYDU_FREE(path);
HYDU_FUNC_EXIT();
return status;
fn_fail:
goto fn_exit;
}
static HYD_status fn_spawn(int fd, int pid, int pgid, char *args[])
{
struct HYD_pg *pg;
struct HYD_pmcd_pmi_pg_scratch *pg_scratch;
struct HYD_proxy *proxy;
struct HYD_pmcd_token *tokens;
struct HYD_exec *exec_list = NULL, *exec;
struct HYD_env *env;
struct HYD_node *node;
char key[PMI_MAXKEYLEN], *val;
int nprocs, preput_num, info_num, ret;
char *execname, *path = NULL;
struct HYD_pmcd_token_segment *segment_list = NULL;
int token_count, i, j, k, new_pgid, total_spawns;
int argcnt, num_segments;
char *control_port, *proxy_args[HYD_NUM_TMP_STRINGS] = { NULL };
char *tmp[HYD_NUM_TMP_STRINGS];
HYD_status status = HYD_SUCCESS;
HYDU_FUNC_ENTER();
for (i = 0; args[i]; i++)
mcmd_args[mcmd_num_args++] = HYDU_strdup(args[i]);
mcmd_args[mcmd_num_args] = NULL;
status = HYD_pmcd_pmi_args_to_tokens(mcmd_args, &tokens, &token_count);
HYDU_ERR_POP(status, "unable to convert args to tokens\n");
/* Here's the order of things we do:
*
* 1. Break the token list into multiple segments, each segment
* corresponding to a command. Each command represents
* information for one executable.
*
* 2. Allocate a process group for the new set of spawned
* processes
*
* 3. Get all the common keys and deal with them
*
* 4. Create an executable list based on the segments.
*
* 5. Create a proxy list using the created executable list and
* spawn it.
*/
/* Break the token list into multiple segments and create an
* executable list based on the segments. */
val = HYD_pmcd_pmi_find_token_keyval(tokens, token_count, "totspawns");
HYDU_ERR_CHKANDJUMP(status, val == NULL, HYD_INTERNAL_ERROR,
"unable to find token: totspawns\n");
total_spawns = atoi(val);
HYDU_MALLOC(segment_list, struct HYD_pmcd_token_segment *,
total_spawns * sizeof(struct HYD_pmcd_token_segment), status);
segment_tokens(tokens, token_count, segment_list, &num_segments);
if (num_segments != total_spawns) {
/* We didn't read the entire PMI string; wait for the rest to
* arrive */
goto fn_exit;
}
else {
/* Got the entire PMI string; free the arguments and reset */
HYDU_free_strlist(mcmd_args);
mcmd_num_args = 0;
}
/* Allocate a new process group */
for (pg = &HYD_server_info.pg_list; pg->next; pg = pg->next);
new_pgid = pg->pgid + 1;
status = HYDU_alloc_pg(&pg->next, new_pgid);
HYDU_ERR_POP(status, "unable to allocate process group\n");
pg = pg->next;
proxy = HYD_pmcd_pmi_find_proxy(fd);
HYDU_ASSERT(proxy, status);
pg->spawner_pg = proxy->pg;
for (j = 0; j < total_spawns; j++) {
/* For each segment, we create an exec structure */
val = HYD_pmcd_pmi_find_token_keyval(&tokens[segment_list[j].start_idx],
segment_list[j].token_count, "nprocs");
HYDU_ERR_CHKANDJUMP(status, val == NULL, HYD_INTERNAL_ERROR,
"unable to find token: nprocs\n");
nprocs = atoi(val);
pg->pg_process_count += nprocs;
val = HYD_pmcd_pmi_find_token_keyval(&tokens[segment_list[j].start_idx],
segment_list[j].token_count, "argcnt");
HYDU_ERR_CHKANDJUMP(status, val == NULL, HYD_INTERNAL_ERROR,
"unable to find token: argcnt\n");
argcnt = atoi(val);
val = HYD_pmcd_pmi_find_token_keyval(&tokens[segment_list[j].start_idx],
segment_list[j].token_count, "info_num");
if (val)
info_num = atoi(val);
else
info_num = 0;
if (exec_list == NULL) {
status = HYDU_alloc_exec(&exec_list);
HYDU_ERR_POP(status, "unable to allocate exec\n");
exec_list->appnum = 0;
exec = exec_list;
}
else {
for (exec = exec_list; exec->next; exec = exec->next);
status = HYDU_alloc_exec(&exec->next);
HYDU_ERR_POP(status, "unable to allocate exec\n");
exec->next->appnum = exec->appnum + 1;
exec = exec->next;
}
/* Info keys */
for (i = 0; i < info_num; i++) {
char *info_key, *info_val;
HYDU_snprintf(key, PMI_MAXKEYLEN, "info_key_%d", i);
val = HYD_pmcd_pmi_find_token_keyval(&tokens[segment_list[j].start_idx],
segment_list[j].token_count, key);
HYDU_ERR_CHKANDJUMP(status, val == NULL, HYD_INTERNAL_ERROR,
"unable to find token: %s\n", key);
info_key = val;
HYDU_snprintf(key, PMI_MAXKEYLEN, "info_val_%d", i);
val = HYD_pmcd_pmi_find_token_keyval(&tokens[segment_list[j].start_idx],
segment_list[j].token_count, key);
HYDU_ERR_CHKANDJUMP(status, val == NULL, HYD_INTERNAL_ERROR,
"unable to find token: %s\n", key);
info_val = val;
if (!strcmp(info_key, "path")) {
path = HYDU_strdup(info_val);
}
else if (!strcmp(info_key, "wdir")) {
exec->wdir = HYDU_strdup(info_val);
}
else if (!strcmp(info_key, "host")) {
status = HYDU_process_mfile_token(info_val, 1, &pg->user_node_list);
HYDU_ERR_POP(status, "error create node list\n");
}
else if (!strcmp(info_key, "hostfile")) {
status =
HYDU_parse_hostfile(info_val, &pg->user_node_list, HYDU_process_mfile_token);
HYDU_ERR_POP(status, "error parsing hostfile\n");
}
else {
/* Unrecognized info key; ignore */
}
}
status = HYDU_correct_wdir(&exec->wdir);
HYDU_ERR_POP(status, "unable to correct wdir\n");
val = HYD_pmcd_pmi_find_token_keyval(&tokens[segment_list[j].start_idx],
segment_list[j].token_count, "execname");
HYDU_ERR_CHKANDJUMP(status, val == NULL, HYD_INTERNAL_ERROR,
"unable to find token: execname\n");
if (path == NULL)
execname = HYDU_strdup(val);
else {
i = 0;
tmp[i++] = HYDU_strdup(path);
tmp[i++] = HYDU_strdup("/");
tmp[i++] = HYDU_strdup(val);
tmp[i++] = NULL;
status = HYDU_str_alloc_and_join(tmp, &execname);
HYDU_ERR_POP(status, "error while joining strings\n");
HYDU_free_strlist(tmp);
}
i = 0;
exec->exec[i++] = execname;
for (k = 0; k < argcnt; k++) {
HYDU_snprintf(key, PMI_MAXKEYLEN, "arg%d", k + 1);
val = HYD_pmcd_pmi_find_token_keyval(&tokens[segment_list[j].start_idx],
segment_list[j].token_count, key);
HYDU_ERR_CHKANDJUMP(status, val == NULL, HYD_INTERNAL_ERROR,
"unable to find token: %s\n", key);
exec->exec[i++] = HYDU_strdup(val);
}
exec->exec[i++] = NULL;
exec->proc_count = nprocs;
/* It is not clear what kind of environment needs to get
* passed to the spawned process. Don't set anything here, and
* let the proxy do whatever it does by default. */
exec->env_prop = NULL;
status = HYDU_env_create(&env, "PMI_SPAWNED", "1");
HYDU_ERR_POP(status, "unable to create PMI_SPAWNED environment\n");
exec->user_env = env;
}
status = HYD_pmcd_pmi_alloc_pg_scratch(pg);
HYDU_ERR_POP(status, "unable to allocate pg scratch space\n");
if (pg->user_node_list) {
pg->pg_core_count = 0;
for (i = 0, node = pg->user_node_list; node; node = node->next, i++) {
pg->pg_core_count += node->core_count;
node->node_id = i;
}
}
else {
pg->pg_core_count = HYD_server_info.pg_list.pg_core_count;
}
pg->pg_process_count = 0;
for (exec = exec_list; exec; exec = exec->next)
pg->pg_process_count += exec->proc_count;
pg_scratch = (struct HYD_pmcd_pmi_pg_scratch *) pg->pg_scratch;
/* Get the common keys and deal with them */
val = HYD_pmcd_pmi_find_token_keyval(tokens, token_count, "preput_num");
if (val)
preput_num = atoi(val);
else
preput_num = 0;
for (i = 0; i < preput_num; i++) {
char *preput_key, *preput_val;
HYDU_snprintf(key, PMI_MAXKEYLEN, "preput_key_%d", i);
val = HYD_pmcd_pmi_find_token_keyval(tokens, token_count, key);
HYDU_ERR_CHKANDJUMP(status, val == NULL, HYD_INTERNAL_ERROR,
"unable to find token: %s\n", key);
preput_key = val;
HYDU_snprintf(key, PMI_MAXKEYLEN, "preput_val_%d", i);
val = HYD_pmcd_pmi_find_token_keyval(tokens, token_count, key);
HYDU_ERR_CHKANDJUMP(status, val == NULL, HYD_INTERNAL_ERROR,
"unable to find token: %s\n", key);
preput_val = val;
status = HYD_pmcd_pmi_add_kvs(preput_key, preput_val, pg_scratch->kvs, &ret);
HYDU_ERR_POP(status, "unable to add keypair to kvs\n");
}
/* Create the proxy list */
if (pg->user_node_list) {
status = HYDU_create_proxy_list(exec_list, pg->user_node_list, pg);
HYDU_ERR_POP(status, "error creating proxy list\n");
}
else {
status = HYDU_create_proxy_list(exec_list, HYD_server_info.node_list, pg);
HYDU_ERR_POP(status, "error creating proxy list\n");
}
HYDU_free_exec_list(exec_list);
status = HYDU_sock_create_and_listen_portstr(HYD_server_info.user_global.iface,
HYD_server_info.local_hostname,
HYD_server_info.port_range, &control_port,
HYD_pmcd_pmiserv_control_listen_cb,
(void *) (size_t) new_pgid);
HYDU_ERR_POP(status, "unable to create PMI port\n");
if (HYD_server_info.user_global.debug)
HYDU_dump(stdout, "Got a control port string of %s\n", control_port);
/* Go to the last PG */
for (pg = &HYD_server_info.pg_list; pg->next; pg = pg->next);
status = HYD_pmcd_pmi_fill_in_proxy_args(proxy_args, control_port, new_pgid);
HYDU_ERR_POP(status, "unable to fill in proxy arguments\n");
HYDU_FREE(control_port);
status = HYD_pmcd_pmi_fill_in_exec_launch_info(pg);
HYDU_ERR_POP(status, "unable to fill in executable arguments\n");
status = HYDT_bsci_launch_procs(proxy_args, pg->proxy_list, NULL);
HYDU_ERR_POP(status, "launcher cannot launch processes\n");
{
char *cmd_str[HYD_NUM_TMP_STRINGS], *cmd;
i = 0;
cmd_str[i++] = HYDU_strdup("cmd=spawn_result rc=0");
cmd_str[i++] = HYDU_strdup("\n");
cmd_str[i++] = NULL;
status = HYDU_str_alloc_and_join(cmd_str, &cmd);
HYDU_ERR_POP(status, "unable to join strings\n");
HYDU_free_strlist(cmd_str);
status = cmd_response(fd, pid, cmd);
HYDU_ERR_POP(status, "error writing PMI line\n");
HYDU_FREE(cmd);
}
fn_exit:
HYD_pmcd_pmi_free_tokens(tokens, token_count);
HYDU_free_strlist(proxy_args);
if (segment_list)
HYDU_FREE(segment_list);
HYDU_FUNC_EXIT();
return status;
fn_fail:
goto fn_exit;
}
static HYD_status fn_spawn(int fd, int pid, int pgid, char *args[])
{
struct HYD_pg *pg;
struct HYD_pmcd_pmi_pg_scratch *pg_scratch;
struct HYD_proxy *proxy;
struct HYD_pmcd_token *tokens;
struct HYD_exec *exec_list = NULL, *exec;
struct HYD_env *env;
struct HYD_node *node;
char *thrid;
char key[PMI_MAXKEYLEN], *val;
int maxprocs, preputcount, infokeycount, ret;
int ncmds;
char *execname, *path = NULL;
struct HYD_pmcd_token_segment *segment_list = NULL;
int token_count, i, j, k, new_pgid;
int argcnt, num_segments;
struct HYD_string_stash proxy_stash;
char *control_port;
struct HYD_string_stash stash;
HYD_status status = HYD_SUCCESS;
HYDU_FUNC_ENTER();
status = HYD_pmcd_pmi_args_to_tokens(args, &tokens, &token_count);
HYDU_ERR_POP(status, "unable to convert args to tokens\n");
thrid = HYD_pmcd_pmi_find_token_keyval(tokens, token_count, "thrid");
/* Here's the order of things we do:
*
* 1. Break the token list into multiple segments, each segment
* corresponding to a command. Each command represents
* information for one executable.
*
* 2. Allocate a process group for the new set of spawned
* processes
*
* 3. Get all the common keys and deal with them
*
* 4. Create an executable list based on the segments.
*
* 5. Create a proxy list using the created executable list and
* spawn it.
*/
val = HYD_pmcd_pmi_find_token_keyval(tokens, token_count, "ncmds");
HYDU_ERR_CHKANDJUMP(status, val == NULL, HYD_INTERNAL_ERROR, "unable to find token: ncmds\n");
ncmds = atoi(val);
HYDU_MALLOC(segment_list, struct HYD_pmcd_token_segment *,
(ncmds + 1) * sizeof(struct HYD_pmcd_token_segment), status);
segment_tokens(tokens, token_count, segment_list, &num_segments);
HYDU_ASSERT((ncmds + 1) == num_segments, status);
/* Allocate a new process group */
for (pg = &HYD_server_info.pg_list; pg->next; pg = pg->next);
new_pgid = pg->pgid + 1;
status = HYDU_alloc_pg(&pg->next, new_pgid);
HYDU_ERR_POP(status, "unable to allocate process group\n");
pg = pg->next;
proxy = HYD_pmcd_pmi_find_proxy(fd);
HYDU_ASSERT(proxy, status);
pg->spawner_pg = proxy->pg;
for (j = 1; j <= ncmds; j++) {
/* For each segment, we create an exec structure */
val = HYD_pmcd_pmi_find_token_keyval(&tokens[segment_list[j].start_idx],
segment_list[j].token_count, "maxprocs");
HYDU_ERR_CHKANDJUMP(status, val == NULL, HYD_INTERNAL_ERROR,
"unable to find token: maxprocs\n");
maxprocs = atoi(val);
pg->pg_process_count += maxprocs;
val = HYD_pmcd_pmi_find_token_keyval(&tokens[segment_list[j].start_idx],
segment_list[j].token_count, "argc");
HYDU_ERR_CHKANDJUMP(status, val == NULL, HYD_INTERNAL_ERROR,
"unable to find token: argc\n");
argcnt = atoi(val);
val = HYD_pmcd_pmi_find_token_keyval(&tokens[segment_list[j].start_idx],
segment_list[j].token_count, "infokeycount");
if (val)
infokeycount = atoi(val);
else
infokeycount = 0;
if (exec_list == NULL) {
status = HYDU_alloc_exec(&exec_list);
HYDU_ERR_POP(status, "unable to allocate exec\n");
exec_list->appnum = 0;
exec = exec_list;
}
else {
for (exec = exec_list; exec->next; exec = exec->next);
status = HYDU_alloc_exec(&exec->next);
HYDU_ERR_POP(status, "unable to allocate exec\n");
exec->next->appnum = exec->appnum + 1;
exec = exec->next;
}
/* Info keys */
for (i = 0; i < infokeycount; i++) {
char *info_key, *info_val;
HYDU_snprintf(key, PMI_MAXKEYLEN, "infokey%d", i);
val = HYD_pmcd_pmi_find_token_keyval(&tokens[segment_list[j].start_idx],
segment_list[j].token_count, key);
HYDU_ERR_CHKANDJUMP(status, val == NULL, HYD_INTERNAL_ERROR,
"unable to find token: %s\n", key);
info_key = val;
HYDU_snprintf(key, PMI_MAXKEYLEN, "infoval%d", i);
val = HYD_pmcd_pmi_find_token_keyval(&tokens[segment_list[j].start_idx],
segment_list[j].token_count, key);
HYDU_ERR_CHKANDJUMP(status, val == NULL, HYD_INTERNAL_ERROR,
"unable to find token: %s\n", key);
info_val = val;
if (!strcmp(info_key, "path")) {
path = HYDU_strdup(info_val);
}
else if (!strcmp(info_key, "wdir")) {
exec->wdir = HYDU_strdup(info_val);
}
else if (!strcmp(info_key, "host") || !strcmp(info_key, "hosts")) {
char *host = strtok(info_val, ",");
while (host) {
status = HYDU_process_mfile_token(host, 1, &pg->user_node_list);
HYDU_ERR_POP(status, "error creating node list\n");
host = strtok(NULL, ",");
}
}
else if (!strcmp(info_key, "hostfile")) {
status = HYDU_parse_hostfile(info_val, &pg->user_node_list,
HYDU_process_mfile_token);
HYDU_ERR_POP(status, "error parsing hostfile\n");
}
else {
/* Unrecognized info key; ignore */
}
}
status = HYDU_correct_wdir(&exec->wdir);
HYDU_ERR_POP(status, "unable to correct wdir\n");
val = HYD_pmcd_pmi_find_token_keyval(&tokens[segment_list[j].start_idx],
segment_list[j].token_count, "subcmd");
HYDU_ERR_CHKANDJUMP(status, val == NULL, HYD_INTERNAL_ERROR,
"unable to find token: subcmd\n");
if (path == NULL)
execname = HYDU_strdup(val);
else {
HYD_STRING_STASH_INIT(stash);
HYD_STRING_STASH(stash, HYDU_strdup(path), status);
HYD_STRING_STASH(stash, HYDU_strdup("/"), status);
HYD_STRING_STASH(stash, HYDU_strdup(val), status);
HYD_STRING_SPIT(stash, execname, status);
}
i = 0;
exec->exec[i++] = execname;
for (k = 0; k < argcnt; k++) {
HYDU_snprintf(key, PMI_MAXKEYLEN, "argv%d", k);
val = HYD_pmcd_pmi_find_token_keyval(&tokens[segment_list[j].start_idx],
segment_list[j].token_count, key);
HYDU_ERR_CHKANDJUMP(status, val == NULL, HYD_INTERNAL_ERROR,
"unable to find token: %s\n", key);
exec->exec[i++] = HYDU_strdup(val);
}
exec->exec[i++] = NULL;
exec->proc_count = maxprocs;
/* It is not clear what kind of environment needs to get
* passed to the spawned process. Don't set anything here, and
* let the proxy do whatever it does by default. */
exec->env_prop = NULL;
status = HYDU_env_create(&env, "PMI_SPAWNED", "1");
HYDU_ERR_POP(status, "unable to create PMI_SPAWNED environment\n");
exec->user_env = env;
}
status = HYD_pmcd_pmi_alloc_pg_scratch(pg);
HYDU_ERR_POP(status, "unable to allocate pg scratch space\n");
pg->pg_process_count = 0;
for (exec = exec_list; exec; exec = exec->next)
pg->pg_process_count += exec->proc_count;
pg_scratch = (struct HYD_pmcd_pmi_pg_scratch *) pg->pg_scratch;
/* Get the common keys and deal with them */
val = HYD_pmcd_pmi_find_token_keyval(tokens, token_count, "preputcount");
HYDU_ERR_CHKANDJUMP(status, val == NULL, HYD_INTERNAL_ERROR,
"unable to find token: preputcount\n");
preputcount = atoi(val);
for (i = 0; i < preputcount; i++) {
char *preput_key, *preput_val;
HYDU_snprintf(key, PMI_MAXKEYLEN, "ppkey%d", i);
val = HYD_pmcd_pmi_find_token_keyval(tokens, token_count, key);
HYDU_ERR_CHKANDJUMP(status, val == NULL, HYD_INTERNAL_ERROR,
"unable to find token: %s\n", key);
preput_key = val;
HYDU_snprintf(key, PMI_MAXKEYLEN, "ppval%d", i);
val = HYD_pmcd_pmi_find_token_keyval(tokens, token_count, key);
HYDU_ERR_CHKANDJUMP(status, val == NULL, HYD_INTERNAL_ERROR,
"unable to find token: %s\n", key);
preput_val = val;
status = HYD_pmcd_pmi_add_kvs(preput_key, preput_val, pg_scratch->kvs, &ret);
HYDU_ERR_POP(status, "unable to add keypair to kvs\n");
}
/* Create the proxy list */
if (pg->user_node_list) {
status = HYDU_create_proxy_list(exec_list, pg->user_node_list, pg);
HYDU_ERR_POP(status, "error creating proxy list\n");
}
else {
status = HYDU_create_proxy_list(exec_list, HYD_server_info.node_list, pg);
HYDU_ERR_POP(status, "error creating proxy list\n");
}
HYDU_free_exec_list(exec_list);
if (pg->user_node_list) {
pg->pg_core_count = 0;
for (i = 0, node = pg->user_node_list; node; node = node->next, i++)
pg->pg_core_count += node->core_count;
}
else {
pg->pg_core_count = 0;
for (proxy = pg->proxy_list; proxy; proxy = proxy->next)
pg->pg_core_count += proxy->node->core_count;
}
status = HYDU_sock_create_and_listen_portstr(HYD_server_info.user_global.iface,
HYD_server_info.localhost,
HYD_server_info.port_range, &control_port,
HYD_pmcd_pmiserv_control_listen_cb,
(void *) (size_t) new_pgid);
HYDU_ERR_POP(status, "unable to create PMI port\n");
if (HYD_server_info.user_global.debug)
HYDU_dump(stdout, "Got a control port string of %s\n", control_port);
/* Go to the last PG */
for (pg = &HYD_server_info.pg_list; pg->next; pg = pg->next);
status = HYD_pmcd_pmi_fill_in_proxy_args(&proxy_stash, control_port, new_pgid);
HYDU_ERR_POP(status, "unable to fill in proxy arguments\n");
HYDU_FREE(control_port);
status = HYD_pmcd_pmi_fill_in_exec_launch_info(pg);
HYDU_ERR_POP(status, "unable to fill in executable arguments\n");
status = HYDT_bsci_launch_procs(proxy_stash.strlist, pg->proxy_list, NULL);
HYDU_ERR_POP(status, "launcher cannot launch processes\n");
{
char *cmd;
HYD_STRING_STASH_INIT(stash);
HYD_STRING_STASH(stash, HYDU_strdup("cmd=spawn-response;"), status);
if (thrid) {
HYD_STRING_STASH(stash, HYDU_strdup("thrid="), status);
HYD_STRING_STASH(stash, HYDU_strdup(thrid), status);
HYD_STRING_STASH(stash, HYDU_strdup(";"), status);
}
HYD_STRING_STASH(stash, HYDU_strdup("rc=0;"), status);
HYD_STRING_STASH(stash, HYDU_strdup("jobid="), status);
HYD_STRING_STASH(stash, HYDU_strdup(pg_scratch->kvs->kvsname), status);
HYD_STRING_STASH(stash, HYDU_strdup(";"), status);
HYD_STRING_STASH(stash, HYDU_strdup("nerrs=0;"), status);
HYD_STRING_SPIT(stash, cmd, status);
status = cmd_response(fd, pid, cmd);
HYDU_ERR_POP(status, "send command failed\n");
HYDU_FREE(cmd);
}
fn_exit:
HYD_pmcd_pmi_free_tokens(tokens, token_count);
HYD_STRING_STASH_FREE(proxy_stash);
if (segment_list)
HYDU_FREE(segment_list);
HYDU_FUNC_EXIT();
return status;
fn_fail:
goto fn_exit;
}
HYD_status HYD_pmcd_pmip_get_params(char **t_argv)
{
char **argv = t_argv;
static char dbg_prefix[2 * MAX_HOSTNAME_LEN];
HYD_status status = HYD_SUCCESS;
HYDU_FUNC_ENTER();
argv++;
do {
/* Get the proxy arguments */
status = HYDU_parse_array(&argv, HYD_pmcd_pmip_match_table);
HYDU_ERR_POP(status, "error parsing input array\n");
/* No more arguments left */
if (!(*argv))
break;
} while (1);
/* Verify the arguments we got */
if (HYD_pmcd_pmip.upstream.server_name == NULL)
HYDU_ERR_SETANDJUMP(status, HYD_INTERNAL_ERROR, "server name not available\n");
if (HYD_pmcd_pmip.upstream.server_port == -1)
HYDU_ERR_SETANDJUMP(status, HYD_INTERNAL_ERROR, "server port not available\n");
if (HYD_pmcd_pmip.user_global.demux == NULL)
HYDU_ERR_SETANDJUMP(status, HYD_INTERNAL_ERROR, "demux engine not available\n");
if (HYD_pmcd_pmip.user_global.debug == -1)
HYD_pmcd_pmip.user_global.debug = 0;
status = HYDT_bsci_init(HYD_pmcd_pmip.user_global.rmk,
HYD_pmcd_pmip.user_global.launcher,
HYD_pmcd_pmip.user_global.launcher_exec,
0 /* disable x */ , HYD_pmcd_pmip.user_global.debug);
HYDU_ERR_POP(status, "proxy unable to initialize bootstrap server\n");
if (HYD_pmcd_pmip.local.id == -1) {
/* We didn't get a proxy ID during launch; query the launcher
* for it. */
status = HYDT_bsci_query_proxy_id(&HYD_pmcd_pmip.local.id);
HYDU_ERR_POP(status, "unable to query launcher for proxy ID\n");
}
if (HYD_pmcd_pmip.local.id == -1)
HYDU_ERR_SETANDJUMP(status, HYD_INTERNAL_ERROR, "proxy ID not available\n");
if (HYD_pmcd_pmip.local.pgid == -1)
HYDU_ERR_SETANDJUMP(status, HYD_INTERNAL_ERROR, "PG ID not available\n");
if (HYD_pmcd_pmip.local.retries == -1)
HYD_pmcd_pmip.local.retries = 0;
HYDU_dbg_finalize();
HYDU_snprintf(dbg_prefix, 2 * MAX_HOSTNAME_LEN, "proxy:%d:%d",
HYD_pmcd_pmip.local.pgid, HYD_pmcd_pmip.local.id);
status = HYDU_dbg_init((const char *) dbg_prefix);
HYDU_ERR_POP(status, "unable to initialization debugging\n");
fn_exit:
HYDU_FUNC_EXIT();
return status;
fn_fail:
goto fn_exit;
}
