HYD_status HYDU_sock_create_and_listen_portstr(char *iface, char *hostname, char *port_range, char **port_str, HYD_status(*callback) (int fd, HYD_event_t events, void *userp), void *userp) { int listenfd; char *sport, *real_port_range, *ip = NULL; uint16_t port; HYD_status status = HYD_SUCCESS; /* Listen on a port in the port range */ port = 0; real_port_range = port_range ? MPL_strdup(port_range) : NULL; status = HYDU_sock_listen(&listenfd, real_port_range, &port); HYDU_ERR_POP(status, "unable to listen on port\n"); /* Register the listening socket with the demux engine */ status = HYDT_dmx_register_fd(1, &listenfd, HYD_POLLIN, userp, callback); HYDU_ERR_POP(status, "unable to register fd\n"); /* Create a port string for MPI processes to use to connect to */ if (iface) { status = HYDU_sock_get_iface_ip(iface, &ip); HYDU_ERR_POP(status, "unable to get network interface IP\n"); } else if (hostname) { ip = MPL_strdup(hostname); } else { char localhost[MAX_HOSTNAME_LEN] = { 0 }; if (gethostname(localhost, MAX_HOSTNAME_LEN) < 0) HYDU_ERR_SETANDJUMP(status, HYD_SOCK_ERROR, "unable to get local hostname\n"); ip = MPL_strdup(localhost); } sport = HYDU_int_to_str(port); HYDU_MALLOC_OR_JUMP(*port_str, char *, strlen(ip) + 1 + strlen(sport) + 1, status); MPL_snprintf(*port_str, strlen(ip) + 1 + strlen(sport) + 1, "%s:%s", ip, sport); MPL_free(sport); fn_exit: if (ip) MPL_free(ip); return status; fn_fail: goto fn_exit; }
HYD_status HYDT_bscd_ll_launch_procs(char **args, struct HYD_proxy *proxy_list, int *control_fd) { int idx, i, total_procs, node_count; int *pid, *fd_list, exec_idx; char *targs[HYD_NUM_TMP_STRINGS], *node_list_str = NULL; char *path = NULL, *extra_arg_list = NULL, *extra_arg, quoted_exec_string[HYD_TMP_STRLEN]; struct HYD_proxy *proxy; struct HYDT_topo_cpuset_t cpuset; HYD_status status = HYD_SUCCESS; HYDU_FUNC_ENTER(); /* We use the following priority order for the executable path: * (1) user-specified; (2) search in path; (3) Hard-coded * location */ if (HYDT_bsci_info.launcher_exec) path = HYDU_strdup(HYDT_bsci_info.launcher_exec); if (!path) path = HYDU_find_full_path("poe"); if (!path) path = HYDU_strdup("/usr/bin/poe"); idx = 0; targs[idx++] = HYDU_strdup(path); if (!strcmp(HYDT_bsci_info.rmk, "ll")) { HYDU_ERR_SETANDJUMP(status, HYD_INTERNAL_ERROR, "ll does not support user-defined host lists\n"); } /* Check how many nodes are being passed for the launch */ status = HYDTI_bscd_ll_query_node_count(&total_procs); HYDU_ERR_POP(status, "unable to query for the node count\n"); node_count = 0; for (proxy = proxy_list; proxy; proxy = proxy->next) node_count++; if (total_procs != node_count) HYDU_ERR_SETANDJUMP(status, HYD_INTERNAL_ERROR, "processes to be launched have to cover all nodes\n"); MPL_env2str("HYDRA_LAUNCHER_EXTRA_ARGS", (const char **) &extra_arg_list); if (extra_arg_list) { extra_arg = strtok(extra_arg_list, " "); while (extra_arg) { targs[idx++] = HYDU_strdup(extra_arg); extra_arg = strtok(NULL, " "); } } /* Fill in the remaining arguments */ exec_idx = idx; for (i = 0; args[i]; i++) targs[idx++] = HYDU_strdup(args[i]); /* Create a quoted version of the exec string, which is only used * when the executable is not launched directly, but through an * actual launcher */ HYDU_snprintf(quoted_exec_string, HYD_TMP_STRLEN, "\"%s\"", targs[exec_idx]); HYDU_FREE(targs[exec_idx]); targs[exec_idx] = quoted_exec_string; /* Increase pid list to accommodate the new pid */ HYDU_MALLOC(pid, int *, (HYD_bscu_pid_count + 1) * sizeof(int), status); for (i = 0; i < HYD_bscu_pid_count; i++) pid[i] = HYD_bscu_pid_list[i]; HYDU_FREE(HYD_bscu_pid_list); HYD_bscu_pid_list = pid; /* Increase fd list to accommodate these new fds */ HYDU_MALLOC(fd_list, int *, (HYD_bscu_fd_count + 3) * sizeof(int), status); for (i = 0; i < HYD_bscu_fd_count; i++) fd_list[i] = HYD_bscu_fd_list[i]; HYDU_FREE(HYD_bscu_fd_list); HYD_bscu_fd_list = fd_list; /* append proxy ID as -1 */ targs[idx++] = HYDU_int_to_str(-1); targs[idx++] = NULL; HYDT_topo_cpuset_zero(&cpuset); status = HYDU_create_process(targs, NULL, NULL, &fd_stdout, &fd_stderr, &HYD_bscu_pid_list[HYD_bscu_pid_count++], cpuset); HYDU_ERR_POP(status, "create process returned error\n"); HYD_bscu_fd_list[HYD_bscu_fd_count++] = fd_stdout; HYD_bscu_fd_list[HYD_bscu_fd_count++] = fd_stderr; status = HYDT_dmx_register_fd(1, &fd_stdout, HYD_POLLIN, (void *) (size_t) STDOUT_FILENO, HYDT_bscu_stdio_cb); HYDU_ERR_POP(status, "demux returned error registering fd\n"); status = HYDT_dmx_register_fd(1, &fd_stderr, HYD_POLLIN, (void *) (size_t) STDERR_FILENO, HYDT_bscu_stdio_cb); HYDU_ERR_POP(status, "demux returned error registering fd\n"); fn_exit: if (node_list_str) HYDU_FREE(node_list_str); HYDU_free_strlist(targs); if (path) HYDU_FREE(path); HYDU_FUNC_EXIT(); return status; fn_fail: goto fn_exit; }
HYD_status HYDT_bscd_slurm_launch_procs(char **args, struct HYD_proxy *proxy_list, int *control_fd) { int num_hosts, idx, i; int *pid, *fd_list; char *targs[HYD_NUM_TMP_STRINGS], *node_list_str = NULL; char *path = NULL, *extra_arg_list = NULL, *extra_arg; struct HYD_proxy *proxy; struct HYDT_topo_cpuset_t cpuset; HYD_status status = HYD_SUCCESS; HYDU_FUNC_ENTER(); /* We use the following priority order for the executable path: * (1) user-specified; (2) search in path; (3) Hard-coded * location */ if (HYDT_bsci_info.launcher_exec) path = HYDU_strdup(HYDT_bsci_info.launcher_exec); if (!path) path = HYDU_find_full_path("srun"); if (!path) path = HYDU_strdup("/usr/bin/srun"); idx = 0; targs[idx++] = HYDU_strdup(path); if (strcmp(HYDT_bsci_info.rmk, "slurm")) { targs[idx++] = HYDU_strdup("--nodelist"); status = proxy_list_to_node_str(proxy_list, &node_list_str); HYDU_ERR_POP(status, "unable to build a node list string\n"); targs[idx++] = HYDU_strdup(node_list_str); } num_hosts = 0; for (proxy = proxy_list; proxy; proxy = proxy->next) num_hosts++; targs[idx++] = HYDU_strdup("-N"); targs[idx++] = HYDU_int_to_str(num_hosts); targs[idx++] = HYDU_strdup("-n"); targs[idx++] = HYDU_int_to_str(num_hosts); MPL_env2str("HYDRA_LAUNCHER_EXTRA_ARGS", (const char **) &extra_arg_list); if (extra_arg_list) { extra_arg = strtok(extra_arg_list, " "); while (extra_arg) { targs[idx++] = HYDU_strdup(extra_arg); extra_arg = strtok(NULL, " "); } } /* Fill in the remaining arguments */ /* We do not need to create a quoted version of the string for * SLURM. It seems to be internally quoting it anyway. */ for (i = 0; args[i]; i++) targs[idx++] = HYDU_strdup(args[i]); /* Increase pid list to accommodate the new pid */ HYDU_MALLOC(pid, int *, (HYD_bscu_pid_count + 1) * sizeof(int), status); for (i = 0; i < HYD_bscu_pid_count; i++) pid[i] = HYD_bscu_pid_list[i]; HYDU_FREE(HYD_bscu_pid_list); HYD_bscu_pid_list = pid; /* Increase fd list to accommodate these new fds */ HYDU_MALLOC(fd_list, int *, (HYD_bscu_fd_count + 3) * sizeof(int), status); for (i = 0; i < HYD_bscu_fd_count; i++) fd_list[i] = HYD_bscu_fd_list[i]; HYDU_FREE(HYD_bscu_fd_list); HYD_bscu_fd_list = fd_list; /* append proxy ID as -1 */ targs[idx++] = HYDU_int_to_str(-1); targs[idx++] = NULL; if (HYDT_bsci_info.debug) { HYDU_dump(stdout, "Launch arguments: "); HYDU_print_strlist(targs); } HYDT_topo_cpuset_zero(&cpuset); status = HYDU_create_process(targs, NULL, NULL, &fd_stdout, &fd_stderr, &HYD_bscu_pid_list[HYD_bscu_pid_count++], cpuset); HYDU_ERR_POP(status, "create process returned error\n"); HYD_bscu_fd_list[HYD_bscu_fd_count++] = fd_stdout; HYD_bscu_fd_list[HYD_bscu_fd_count++] = fd_stderr; status = HYDT_dmx_register_fd(1, &fd_stdout, HYD_POLLIN, (void *) (size_t) STDOUT_FILENO, HYDT_bscu_stdio_cb); HYDU_ERR_POP(status, "demux returned error registering fd\n"); status = HYDT_dmx_register_fd(1, &fd_stderr, HYD_POLLIN, (void *) (size_t) STDERR_FILENO, HYDT_bscu_stdio_cb); HYDU_ERR_POP(status, "demux returned error registering fd\n"); fn_exit: if (node_list_str) HYDU_FREE(node_list_str); HYDU_free_strlist(targs); if (path) HYDU_FREE(path); HYDU_FUNC_EXIT(); return status; fn_fail: goto fn_exit; }
int main(int argc, char **argv) { int i, count, pid, ret_status, sent, closed, ret, done; struct HYD_pmcd_hdr hdr; HYD_status status = HYD_SUCCESS; status = HYDU_dbg_init("proxy:unset"); HYDU_ERR_POP(status, "unable to initialization debugging\n"); status = HYDU_set_signal(SIGPIPE, signal_cb); HYDU_ERR_POP(status, "unable to set SIGPIPE\n"); status = HYDU_set_signal(SIGTSTP, signal_cb); HYDU_ERR_POP(status, "unable to set SIGTSTP\n"); status = HYDU_set_common_signals(signal_cb); HYDU_ERR_POP(status, "unable to set common signals\n"); status = init_params(); HYDU_ERR_POP(status, "Error initializing proxy params\n"); status = HYD_pmcd_pmip_get_params(argv); HYDU_ERR_POP(status, "bad parameters passed to the proxy\n"); status = HYDT_dmx_init(&HYD_pmcd_pmip.user_global.demux); HYDU_ERR_POP(status, "unable to initialize the demux engine\n"); status = HYDT_ftb_init(); HYDU_ERR_POP(status, "unable to initialize FTB\n"); /* See if HYDI_CONTROL_FD is set before trying to connect upstream */ ret = MPL_env2int("HYDI_CONTROL_FD", &HYD_pmcd_pmip.upstream.control); if (ret < 0) { HYDU_ERR_POP(status, "error reading HYDI_CONTROL_FD environment\n"); } else if (ret == 0) { status = HYDU_sock_connect(HYD_pmcd_pmip.upstream.server_name, HYD_pmcd_pmip.upstream.server_port, &HYD_pmcd_pmip.upstream.control, HYD_pmcd_pmip.local.retries, HYD_CONNECT_DELAY); HYDU_ERR_POP(status, "unable to connect to server %s at port %d (check for firewalls!)\n", HYD_pmcd_pmip.upstream.server_name, HYD_pmcd_pmip.upstream.server_port); } status = HYDU_sock_write(HYD_pmcd_pmip.upstream.control, &HYD_pmcd_pmip.local.id, sizeof(HYD_pmcd_pmip.local.id), &sent, &closed, HYDU_SOCK_COMM_MSGWAIT); HYDU_ERR_POP(status, "unable to send the proxy ID to the server\n"); if (closed) goto fn_fail; status = HYDT_dmx_register_fd(1, &HYD_pmcd_pmip.upstream.control, HYD_POLLIN, NULL, HYD_pmcd_pmip_control_cmd_cb); HYDU_ERR_POP(status, "unable to register fd\n"); while (1) { /* Wait for some event to occur */ status = HYDT_dmx_wait_for_event(-1); HYDU_ERR_POP(status, "demux engine error waiting for event\n"); /* Check to see if there's any open read socket left; if there * are, we will just wait for more events. */ count = 0; for (i = 0; i < HYD_pmcd_pmip.local.proxy_process_count; i++) { if (HYD_pmcd_pmip.downstream.out[i] != HYD_FD_CLOSED) count++; if (HYD_pmcd_pmip.downstream.err[i] != HYD_FD_CLOSED) count++; if (count) break; } if (!count) break; } /* Now wait for the processes to finish */ done = 0; while (1) { pid = waitpid(-1, &ret_status, 0); /* Find the pid and mark it as complete. */ if (pid > 0) for (i = 0; i < HYD_pmcd_pmip.local.proxy_process_count; i++) if (HYD_pmcd_pmip.downstream.pid[i] == pid) { if (HYD_pmcd_pmip.downstream.forced_cleanup) { /* If it is a forced cleanup, the exit status * is either already set or we have to ignore * it */ if (HYD_pmcd_pmip.downstream.exit_status[i] == -1) HYD_pmcd_pmip.downstream.exit_status[i] = 0; else HYD_pmcd_pmip.downstream.exit_status[i] = ret_status; } else { HYD_pmcd_pmip.downstream.exit_status[i] = ret_status; } done++; } /* If no more processes are pending, break out */ if (done == HYD_pmcd_pmip.local.proxy_process_count) break; /* Check if there are any messages from the launcher */ status = HYDT_dmx_wait_for_event(0); HYDU_IGNORE_TIMEOUT(status); HYDU_ERR_POP(status, "demux engine error waiting for event\n"); } /* Send the exit status upstream */ HYD_pmcd_init_header(&hdr); hdr.cmd = EXIT_STATUS; status = HYDU_sock_write(HYD_pmcd_pmip.upstream.control, &hdr, sizeof(hdr), &sent, &closed, HYDU_SOCK_COMM_MSGWAIT); HYDU_ERR_POP(status, "unable to send EXIT_STATUS command upstream\n"); HYDU_ASSERT(!closed, status); status = HYDU_sock_write(HYD_pmcd_pmip.upstream.control, HYD_pmcd_pmip.downstream.exit_status, HYD_pmcd_pmip.local.proxy_process_count * sizeof(int), &sent, &closed, HYDU_SOCK_COMM_MSGWAIT); HYDU_ERR_POP(status, "unable to return exit status upstream\n"); HYDU_ASSERT(!closed, status); status = HYDT_dmx_deregister_fd(HYD_pmcd_pmip.upstream.control); HYDU_ERR_POP(status, "unable to deregister fd\n"); close(HYD_pmcd_pmip.upstream.control); status = HYDT_dmx_finalize(); HYDU_ERR_POP(status, "error returned from demux finalize\n"); status = HYDT_ftb_finalize(); HYDU_ERR_POP(status, "unable to finalize FTB\n"); status = HYDT_bsci_finalize(); HYDU_ERR_POP(status, "unable to finalize the bootstrap device\n"); /* cleanup the params structure */ cleanup_params(); fn_exit: HYDU_dbg_finalize(); return status; fn_fail: /* kill all processes */ HYD_pmcd_pmip_send_signal(SIGKILL); goto fn_exit; }