int AMUDP_SPMDCustomSpawn(int nproc, int argc, char **argv, char **extra_env) {
int i;
char *spawn_cmd = NULL;
char *spawn_servers = NULL;
char workerservers[1024];
char workercmd[1024];
char nproc_str[10];
char cwd[1024];
char cmd[1024];
#if PLATFORM_OS_MSWINDOWS || PLATFORM_OS_CYGWIN
int spawn_use_create_process;
#endif
int spawn_route_output;
int pid;
if (!AMUDP_SPMDSpawnRunning) {
AMUDP_Err("Spawn functions should never be run directly - only passed to AMUDP_SPMDStartup()");
return FALSE;
}
pid = getpid();
spawn_cmd = AMUDP_getenv_prefixed_withdefault("CSPAWN_CMD","");
if (!strlen(spawn_cmd)) {
AMUDP_Err("You must set the " AMUDP_ENV_PREFIX_STR"_CSPAWN_CMD environment variable to use the custom spawn function");
return FALSE;
}
spawn_servers = AMUDP_getenv_prefixed_withdefault("CSPAWN_SERVERS","");
if (!strlen(spawn_servers)) spawn_servers = NULL;
spawn_route_output =
strcmp(AMUDP_getenv_prefixed_withdefault("CSPAWN_ROUTE_OUTPUT","0"),"0");
#if PLATFORM_OS_MSWINDOWS || PLATFORM_OS_CYGWIN
spawn_use_create_process =
strcmp(AMUDP_getenv_prefixed_withdefault("CSPAWN_USE_CREATE_PROCESS","0"),"0");
#endif
if (spawn_servers) { /* build server list */
char *p = spawn_servers;
char servername[255];
char serverDelim[2];
strcpy(serverDelim," "); /* default to space */
workerservers[0] = '\0';
for (i = 0; i < nproc; i++) { /* check we have enough servers & copy the right number */
char *end;
while (*p && strchr(SSH_SERVERS_DELIM_CHARS, *p)) p++;
end = p + strcspn(p, SSH_SERVERS_DELIM_CHARS);
if (p == end) {
printf("Not enough machines in environment variable " AMUDP_ENV_PREFIX_STR"_CSPAWN_SERVERS to satisfy request for (%i).\n"
"Only (%i) machines available: %s\n", nproc, i, spawn_servers);
return FALSE;
}
strncpy(servername, p, end-p);
servername[end-p] = '\0';
if (workerservers[0]) strcat(workerservers, serverDelim);
if (*end) serverDelim[0] = *end;
strcat(workerservers, servername);
if (*end) p = end+1;
else p = end;
}
}
sprintf(nproc_str, "%i", nproc);
if (!getcwd(cwd, 1024)) {
printf("Error calling getcwd()\n");
return FALSE;
}
/* build the worker command */
{ char temp[2048]; // TODO: allocate dynamically
temp[0] = '\0';
if (extra_env && extra_env[0]) {
const char *envcmd = AMUDP_getenv_prefixed_withdefault("ENV_CMD", "env");
strcat(temp,envcmd);
strcat(temp," ");
for (i=0; extra_env[i]; ++i) {
strcat(temp,"'");
strcat(temp,extra_env[i]);
strcat(temp,"' ");
}
}
for (i = 0; i < argc; i++) {
AMUDP_assert(argv[i] != NULL);
char *qarg = quote_for_local(argv[i]);
strcat(temp,qarg);
strcat(temp," ");
AMUDP_free(qarg);
}
AMUDP_assert(!argv[i]);
#if PLATFORM_OS_MSWINDOWS || PLATFORM_OS_CYGWIN
strcpy(workercmd, temp);
#else
sprintf(workercmd, "/bin/sh -c \"%s%s\" || ( echo \"spawn failed.\" ; kill %i ) ",
(AMUDP_SilentMode?"":"echo connected to `uname -n`... ; "),
temp, pid
);
#endif
}
strcpy(cmd, spawn_cmd);
{ char tmp[1024];
char *p = cmd;
while ((p = strchr(p, '%'))) {
const char *replacement;
switch (*(p+1)) {
case 'M': case 'm':
if (!spawn_servers) { /* user failed to provide servers and now is asking for them */
AMUDP_Err("You must set the " AMUDP_ENV_PREFIX_STR"_CSPAWN_SERVERS environment "
"variable to use the %%M option in " AMUDP_ENV_PREFIX_STR"_CSPAWN_CMD");
}
replacement = workerservers;
break;
case 'N': case 'n': replacement = nproc_str; break;
case 'C': case 'c': replacement = workercmd; break;
case 'D': case 'd': replacement = cwd; break;
case 'P': case 'p': replacement = argv[0]; break;
case '%': replacement = "%"; break;
default: replacement = "";
}
strcpy(tmp, cmd);
tmp[p-cmd] = '\0';
strcat(tmp, replacement);
if (*(p+1)) strcat(tmp, p+2);
p += strlen(replacement);
strcpy(cmd, tmp);
}
}
AMUDP_SPMDRedirectStdsockets = spawn_route_output;
{
#if !PLATFORM_OS_MSWINDOWS
int forkRet;
forkRet = fork(); /* fork a new process to hold cmd master */
if (forkRet == -1) {
perror("fork");
return FALSE;
}
else if (forkRet != 0) {
return TRUE;
}
else
#endif
{ /* this is the child - exec the new process */
if (!AMUDP_SilentMode)
printf("system(%s)\n", cmd); fflush(stdout);
#if PLATFORM_OS_MSWINDOWS || PLATFORM_OS_CYGWIN
if (spawn_use_create_process) {
STARTUPINFO si;
PROCESS_INFORMATION pi;
DWORD code;
ZeroMemory(&si, sizeof(STARTUPINFO));
si.cb = sizeof(STARTUPINFO);
si.hStdError = GetStdHandle(STD_OUTPUT_HANDLE);
si.hStdOutput = GetStdHandle(STD_ERROR_HANDLE);
si.dwFlags |= STARTF_USESTDHANDLES;
if ( TRUE != CreateProcess(0, strdup(cmd), 0, 0, TRUE, 0, 0, 0, &si, &pi)
|| WAIT_FAILED == WaitForSingleObject(pi.hProcess, INFINITE)
|| 0 == GetExitCodeProcess(pi.hProcess, &code)
|| 0 != code)
AMUDP_FatalErr("Failed while calling CreateProcess() with custom spawn command:\n%s", cmd);
} else
#endif
if (system(cmd) != 0)
AMUDP_FatalErr("Failed while calling system() with custom spawn command:\n%s", cmd);
exit(0);
}
}
return TRUE;
}
/* ------------------------------------------------------------------------------------
* spawn jobs on local machine
* ------------------------------------------------------------------------------------ */
extern int AMUDP_SPMDLocalSpawn(int nproc, int argc, char **argv, char **extra_env) {
/* just a simple fork/exec */
int i;
if (!AMUDP_SPMDSpawnRunning) {
AMUDP_Err("Spawn functions should never be run directly - only passed to AMUDP_SPMDStartup()");
return FALSE;
}
/* Temporarily assume values from extra_env[] (which we modify in-place) */
char **save_env = NULL;
int envc = 0;
if (extra_env && extra_env[0]) {
for (envc=0; extra_env[envc]; ++envc) {/*empty*/}
save_env = (char **)AMUDP_malloc(sizeof(char *)*envc);
for (i=0;i<envc;++i) {
char *var = extra_env[i];
char *delim = strchr(var,'=');
AMUDP_assert(delim);
*delim = '\0';
save_env[i] = getenv(var);
setenv(var,delim+1,1);
}
}
for (i = 0; i < nproc; i++) {
#if PLATFORM_OS_MSWINDOWS && !PLATFORM_OS_CYGWIN
if (_spawnv(_P_NOWAIT, argv[0], argv) == -1)
AMUDP_FatalErr("failed _spawnv()");
#elif PLATFORM_ARCH_CRAYX1
{ char **nargv = (char **)AMUDP_malloc(sizeof(char *)*(argc+2));
nargv[0] = argv[0];
memcpy(nargv+1,argv,argc*sizeof(char *));
nargv[argc+1] = NULL;
if (execsp(nargv, environ, NULL) == -1)
AMUDP_FatalErr("failed execsp()");
}
#else
int forkRet = fork();
if (forkRet == -1) {
perror("fork");
return FALSE;
} else if (forkRet != 0) continue; /* this is the parent, will go back to the top of the loop */
else { /* this is the child - exec the new process */
/* could close some resources here (like AMUDP_SPMDListenSocket) but not strictly necessary */
#if 0
/* put new process in a separate process group */
if (setsid() == -1) perror("setsid");
#endif
/* exec the program, with the given arguments */
execv(argv[0], argv);
/* if execv returns, an error occurred */
perror("execv");
_exit(1); /* use _exit() to prevent corrupting parent's io buffers */
} /* child */
#endif
}
/* Restore saved environment var(s) */
for (i=0;i<envc;++i) {
char *name = extra_env[i];
if (save_env[i]) {
setenv(name,save_env[i],1);
} else {
unsetenv(name);
}
/* Revert our in-place modification of extra_env[]: */
name[strlen(name)] = '=';
}
AMUDP_free(save_env);
return TRUE;
}
int AMUDP_SPMDSshSpawn(int nproc, int argc, char **argv, char **extra_env) {
int i;
const char *ssh_servers;
const char *ssh_cmd;
const char *ssh_options;
const char *ssh_remote_path;
char cwd[1024];
const char *p;
char *cmd1, *cmd2;
size_t cmd1_sz, cmd2_sz;
int pid;
if (!AMUDP_SPMDSpawnRunning) {
AMUDP_Err("Spawn functions should never be run directly - only passed to AMUDP_SPMDStartup()");
return FALSE;
}
pid = getpid();
ssh_servers = AMUDP_getenv_prefixed_withdefault("SSH_SERVERS","");
if (!strlen(ssh_servers)) {
printf("Environment variable SSH_SERVERS is missing.\n");
return FALSE;
}
if (!getcwd(cwd, 1024)) {
printf("Error calling getcwd()\n");
return FALSE;
}
ssh_remote_path = quote_for_local(AMUDP_getenv_prefixed_withdefault("SSH_REMOTE_PATH", cwd));
ssh_cmd = AMUDP_getenv_prefixed_withdefault("SSH_CMD", "ssh");
int isOpenSSH = 0; /* figure out if we're using OpenSSH */
{ char cmdtmp[1024];
sprintf(cmdtmp,"%s -v 2>&1 | grep OpenSSH", ssh_cmd);
FILE *pop = popen(cmdtmp,"r");
while (!feof(pop) && !ferror(pop)) {
int next = fgetc(pop);
if (next != EOF && !isspace(next)) {
isOpenSSH = 1;
break;
}
}
pclose(pop);
}
ssh_options = AMUDP_getenv_prefixed_withdefault("SSH_OPTIONS","");
p = ssh_servers;
for (i = 0; i < nproc; i++) { /* check we have enough servers */
const char *end;
while (*p && strchr(SSH_SERVERS_DELIM_CHARS, *p)) p++;
end = p + strcspn(p, SSH_SERVERS_DELIM_CHARS);
if (p == end) {
printf("Not enough machines in environment variable SSH_SERVERS to satisfy request for (%i).\n"
"Only (%i) machines available: %s\n", nproc, i, ssh_servers);
return FALSE;
}
if (*end) p = end+1;
else p = end;
}
const char *envcmd = AMUDP_getenv_prefixed_withdefault("ENV_CMD", "env");
cmd1_sz = 1; /* terminating nul */
if (extra_env && extra_env[0]) {
cmd1_sz += 1 + strlen(envcmd); // "env "
for (i=0; extra_env[i]; ++i) {
cmd1_sz += strlen(extra_env[i]) + 3; // "'%s' "
}
}
char **quoted_args = (char**)AMUDP_malloc(argc * sizeof(char*));
for (i = 0; i < argc; i++) {
AMUDP_assert(argv[i]);
quoted_args[i] = quote_for_local(argv[i]);
cmd1_sz += strlen(quoted_args[i]) + 1; // +1 for trailing space
}
cmd1 = (char *)AMUDP_malloc(cmd1_sz);
{ char *tmp = cmd1;
if (extra_env && extra_env[0]) {
tmp += sprintf(tmp, "%s ", envcmd);
for (i=0; extra_env[i]; ++i) {
tmp += sprintf(tmp, "'%s' ", extra_env[i]);
}
}
for (i = 0; i < argc; i++) {
AMUDP_assert(quoted_args[i]);
tmp += sprintf(tmp, "%s ", quoted_args[i]);
AMUDP_free(quoted_args[i]);
}
AMUDP_free(quoted_args);
AMUDP_assert(!argv[i]);
*tmp = '\0';
AMUDP_assert(strlen(cmd1) == cmd1_sz - 1);
}
cmd2_sz = cmd1_sz + strlen(ssh_remote_path) + 1024; /* estimated */
cmd2 = (char *)AMUDP_malloc(cmd2_sz);
p = ssh_servers;
for (i = 0; i < nproc; i++) {
char ssh_server[255];
const char *end;
while (*p && strchr(SSH_SERVERS_DELIM_CHARS, *p)) p++;
end = p + strcspn(p, SSH_SERVERS_DELIM_CHARS);
AMUDP_assert(p != end);
strncpy(ssh_server, p, (end-p));
ssh_server[end-p] = '\0';
/* build the ssh command */
snprintf(cmd2, cmd2_sz, "%s %s %s %s %s %s \" %s cd %s ; %s\" "
" || ( echo \"connection to %s failed.\" ; kill %i ) "
"%s",
ssh_cmd,
(isOpenSSH?"-f":""), /* go into background and nullify stdin */
#if SSH_SUPRESSNEWKEYPROMPT
(isOpenSSH?"-o 'StrictHostKeyChecking no'":""),
#else
"",
#endif
#if SSH_PREVENTRSHFALLBACK
(isOpenSSH?"-o 'FallBackToRsh no'":""),
#else
"",
#endif
ssh_options, ssh_server,
(AMUDP_SilentMode?"":"echo connected to \\$HOST... ;"),
ssh_remote_path, cmd1, ssh_server, pid,
#if SSH_PARALLELSPAWN
"&"
#else
""
#endif
);
if (!AMUDP_SilentMode)
printf("system(%s)\n", cmd2); fflush(stdout);
if (system(cmd2) == -1) {
printf("Failed to call system() to spawn");
AMUDP_free(cmd1);
AMUDP_free(cmd2);
return FALSE;
}
if (*end) p = end+1;
else p = end;
}
AMUDP_free(cmd1);
AMUDP_free(cmd2);
return TRUE;
}
int AMUDP_SPMDSshSpawn(int nproc, int argc, char **argv) {
int i;
const char *ssh_servers;
const char *ssh_cmd;
const char *ssh_options;
const char *ssh_remote_path;
char cwd[1024];
const char *p;
char cmd1[1024],cmd2[1024];
int pid;
if (!AMUDP_SPMDSpawnRunning) {
AMUDP_Err("Spawn functions should never be run directly - only passed to AMUDP_SPMDStartup()");
return FALSE;
}
pid = getpid();
ssh_servers = AMUDP_getenv_prefixed_withdefault("SSH_SERVERS","");
if (!strlen(ssh_servers)) {
printf("Environment variable SSH_SERVERS is missing.\n");
return FALSE;
}
if (!getcwd(cwd, 1024)) {
printf("Error calling getcwd()\n");
return FALSE;
}
ssh_remote_path = AMUDP_getenv_prefixed_withdefault("SSH_REMOTE_PATH", cwd);
ssh_cmd = AMUDP_getenv_prefixed_withdefault("SSH_CMD", "ssh");
int isOpenSSH = 0; /* figure out if we're using OpenSSH */
{ char cmdtmp[1024];
sprintf(cmdtmp,"%s -v 2>&1 | grep OpenSSH", ssh_cmd);
FILE *pop = popen(cmdtmp,"r");
while (!feof(pop) && !ferror(pop)) {
int next = fgetc(pop);
if (next != EOF && !isspace(next)) {
isOpenSSH = 1;
break;
}
}
pclose(pop);
}
ssh_options = AMUDP_getenv_prefixed_withdefault("SSH_OPTIONS","");
cmd1[0] = '\0';
for (i = 0; i < argc; i++) {
AMUDP_assert(argv[i] != NULL);
strcat(cmd1,"'");
strcat(cmd1,argv[i]);
strcat(cmd1,"' ");
}
AMUDP_assert(!argv[i]);
p = ssh_servers;
for (i = 0; i < nproc; i++) { /* check we have enough servers */
const char *end;
while (*p && strchr(SSH_SERVERS_DELIM_CHARS, *p)) p++;
end = p + strcspn(p, SSH_SERVERS_DELIM_CHARS);
if (p == end) {
printf("Not enough machines in environment variable SSH_SERVERS to satisfy request for (%i).\n"
"Only (%i) machines available: %s\n", nproc, i, ssh_servers);
return FALSE;
}
if (*end) p = end+1;
else p = end;
}
p = ssh_servers;
for (i = 0; i < nproc; i++) {
char ssh_server[255];
const char *end;
while (*p && strchr(SSH_SERVERS_DELIM_CHARS, *p)) p++;
end = p + strcspn(p, SSH_SERVERS_DELIM_CHARS);
AMUDP_assert(p != end);
strncpy(ssh_server, p, (end-p));
ssh_server[end-p] = '\0';
/* build the ssh command */
sprintf(cmd2, "%s %s %s %s %s %s \" %s cd '%s' ; %s\" "
" || ( echo \"connection to %s failed.\" ; kill %i ) "
"%s",
ssh_cmd,
(isOpenSSH?"-f":""), /* go into background and nullify stdin */
#if SSH_SUPRESSNEWKEYPROMPT
(isOpenSSH?"-o 'StrictHostKeyChecking no'":""),
#else
"",
#endif
#if SSH_PREVENTRSHFALLBACK
(isOpenSSH?"-o 'FallBackToRsh no'":""),
#else
"",
#endif
ssh_options, ssh_server,
(AMUDP_SilentMode?"":"echo connected to \\$HOST... ;"),
ssh_remote_path, cmd1, ssh_server, pid,
#if SSH_PARALLELSPAWN
"&"
#else
""
#endif
);
if (!AMUDP_SilentMode)
printf("system(%s)\n", cmd2); fflush(stdout);
if (system(cmd2) == -1) {
printf("Failed to call system() to spawn");
return FALSE;
}
if (*end) p = end+1;
else p = end;
}
return TRUE;
}