/* return a vnode number matching targethost */
int findtargethost(char *allnodes, char *targethost)
{
int i;
char *ptr;
int vnode = 0;
if ((ptr = strchr(targethost, '/')) != NULL)
{
*ptr = '\0';
ptr++;
vnode = atoi(ptr);
}
for (i = 0; i < numnodes; i++)
{
if (!strcmp(allnodes + (i*PBS_MAXNODENAME), targethost))
{
if (vnode == 0)
return(i);
vnode--;
}
}
if (i == numnodes)
{
fprintf(stderr, "%s: %s not found\n", id, targethost);
tm_finalize();
exit(1);
}
return(-1);
}
HYD_status HYDT_bscd_pbs_launcher_finalize(void)
{
int err;
HYD_status status = HYD_SUCCESS;
HYDU_FUNC_ENTER();
#if defined(HAVE_TM_H)
err = tm_finalize();
HYDU_ERR_CHKANDJUMP(status, err != TM_SUCCESS, HYD_INTERNAL_ERROR,
"error calling tm_finalize\n");
#endif /* HAVE_TM_H */
if (HYDT_bscd_pbs_sys) {
if (HYDT_bscd_pbs_sys->task_id)
HYDU_FREE(HYDT_bscd_pbs_sys->task_id);
if (HYDT_bscd_pbs_sys->spawn_events)
HYDU_FREE(HYDT_bscd_pbs_sys->spawn_events);
HYDU_FREE(HYDT_bscd_pbs_sys);
}
fn_exit:
HYDU_FUNC_EXIT();
return status;
fn_fail:
goto fn_exit;
}
void
mom_reconnect(void)
{
int c, rc;
struct tm_roots rootrot;
for (;;)
{
tm_finalize();
sigprocmask(SIG_UNBLOCK, &allsigs, NULL);
sleep(2);
sigprocmask(SIG_BLOCK, &allsigs, NULL);
/* attempt to reconnect */
rc = tm_init(0, &rootrot);
if (rc == TM_SUCCESS)
{
fprintf(stderr, "%s: reconnected\n",
id);
/* resend obit requests */
for (c = 0; c < numnodes; ++c)
{
if (*(events_obit + c) != TM_NULL_EVENT)
{
rc = obit_submit(c);
if (rc != TM_SUCCESS)
{
break; /* reconnect again */
}
}
else if (verbose)
{
fprintf(stderr, "%s: skipping obit resend for %u\n",
id,
*(tid + c));
}
}
break;
}
}
return;
} /* END mom_reconnect() */
/* return a vnode number matching targethost */
int findtargethost(char *allnodes, char *targethost)
{
int i;
char *ptr;
int vnode = 0;
if ((ptr = strchr(targethost, '/')) != NULL)
{
*ptr = '\0';
ptr++;
vnode = atoi(ptr);
}
for (i = 0; i < numnodes; i++)
{
if (!strcmp(allnodes + (i*PBS_MAXNODENAME), targethost))
{
if (vnode == 0)
return(i);
vnode--;
}
else
{
/* Sometimes the allnodes will return the FQDN of the host
and the PBS_NODEFILE will have the short name of the host.
See if the shortname mataches */
std::string targetname(targethost);
std::string the_host(allnodes + (i*PBS_MAXNODENAME));
std::size_t dot = the_host.find_first_of(".");
if(dot != std::string::npos)
{
the_host[dot] = '\0';
std::string shortname(the_host.c_str());
if (shortname.compare(targetname) == 0)
{
if (vnode == 0)
return(i);
vnode--;
}
}
}
}
if (i == numnodes)
{
fprintf(stderr, "%s: %s not found\n", id, targethost);
tm_finalize();
exit(1);
}
return(-1);
}
/*
* Free stuff
*/
static int plm_tm_finalize(void)
{
int rc;
/* cleanup any pending recvs */
if (ORTE_SUCCESS != (rc = orte_plm_base_comm_stop())) {
ORTE_ERROR_LOG(rc);
}
if (connected) {
tm_finalize();
connected = false;
}
return ORTE_SUCCESS;
}
void Kickoff_PBS(const Node_info *ddinodes,const Cmdline_info *info) {
char ddiinfo[] = "-ddi";
char procid[8];
char portid[8];
char nodeid[8];
char snodes[8];
char sprocs[8];
char **rargs;
char **argv = info->argv;
int i,j,r,iarg,nargs = info->ddiarg + info->nnodes + 8;
int inode,ncpus,np = info->nprocs;
int ntests;
if(info->nnodes == 1) return;
int tm_errno;
tm_task_id *tid;
tm_event_t *spawn;
tm_event_t polled;
struct tm_roots roots;
tm_node_id *nodelist;
/* ---------------------------------- *\
Initialize PBS Task Management API
\* ---------------------------------- */
if(tm_init(0, &roots) != TM_SUCCESS) {
fprintf(stderr, " ddikick.x: tm_init failed\n");
Fatal_error(911);
}
if(tm_nodeinfo(&nodelist, &np) != TM_SUCCESS) {
fprintf(stderr, " ddikick.x: tm_nodeinfo failed.\n");
Fatal_error(911);
}
tid = (tm_task_id *) Malloc(2*np*sizeof(tm_task_id));
spawn = (tm_event_t *) Malloc(2*np*sizeof(tm_event_t));
for(i=0; i<2*np; i++) {
*(tid + i) = TM_NULL_TASK;
*(spawn + i) = TM_NULL_EVENT;
}
/* ----------------------------------------- *\
Initialize arguments to kickoff DDI tasks
\* ----------------------------------------- */
rargs = (char **) Malloc(nargs*sizeof(char*));
sprintf(portid, "%d", info->kickoffport);
sprintf(snodes, "%d", info->nnodes);
sprintf(sprocs, "%d", info->nprocs);
for(i=1,r=0; i<info->ddiarg-1; i++) rargs[r++] = argv[i];
rargs[r++] = ddiinfo;
rargs[r++] = info->kickoffhost; /* kickoff host name */
rargs[r++] = portid; /* kickoff port number */
rargs[r++] = nodeid; /* rank of this node */
rargs[r++] = procid; /* rank of this process */
rargs[r++] = snodes; /* number of nodes */
rargs[r++] = sprocs; /* number of processors */
for(i=0,iarg=info->nodearg; i<info->nnodes; i++,iarg++) {
rargs[r++] = argv[iarg];
}
rargs[r] = NULL;
/* ------------------------ *\
Spawn DDI tasks to nodes
\* ------------------------ */
ncpus=ddinodes[0].cpus+ddinodes[1].cpus;
for(i=ddinodes[0].cpus,inode=1; i<np; i++) {
if(i == ncpus) ncpus += ddinodes[++inode].cpus;
sprintf(nodeid,"%d",inode);
sprintf(procid,"%d",i);
# if DDI_DEBUG
DEBUG_START(DEBUG_MAX)
fprintf(stdout,"DDI Process %i PBS tm_spawn arguments: ",i);
for(iarg=0; iarg<r; iarg++) fprintf(stdout,"%s ",rargs[iarg]);
fprintf(stdout,"\n");
DEBUG_END()
# endif
/* ------------------------- *\
Spawn DDI Compute Process
\* ------------------------- */
if(tm_spawn(r,rargs,NULL,*(nodelist+i),(tid+i),spawn+i) != TM_SUCCESS) {
fprintf(stderr," ddikick.x: tm_spawn failed.\n");
Fatal_error(911);
}
/* ---------------------------------- *\
No data server on single node runs
\* ---------------------------------- */
if(info->nnodes == 1) continue;
# if DDI_DEBUG
DEBUG_START(DEBUG_MAX)
fprintf(stdout,"DDI Process %i PBS tm_spawn arguments: ",j);
for(iarg=0; iarg<r; iarg++) fprintf(stdout,"%s ",rargs[iarg]);
fprintf(stdout,"\n");
DEBUG_END()
# endif
j = i+np;
sprintf(procid,"%d",j);
/* --------------------- *\
Spawn DDI Data Server
\* --------------------- */
if(tm_spawn(r,rargs,NULL,*(nodelist+i),(tid+j),spawn+j) != TM_SUCCESS) {
fprintf(stderr," ddikick.x: tm_spawn failed.\n");
Fatal_error(911);
} }
/* -------------------------------------------------------- *\
Poll PBS to ensure each DDI process started successfully
\* -------------------------------------------------------- */
ntests = np-ddinodes[0].cpus;
if(USING_DATA_SERVERS()) ntests *= 2;
for(i=ntests; i--; ) {
if(tm_poll(TM_NULL_EVENT,&polled,1,&tm_errno) != TM_SUCCESS) {
fprintf(stderr," ddikick.x: tm_poll failed.\n");
Fatal_error(911);
}
for(j=0; j<np; j++) {
if(polled == *(spawn+j)) {
if(tm_errno) {
fprintf(stderr," ddikick.x: error spawning DDI task %i.\n",j);
Fatal_error(911);
} else {
# if DDI_DEBUG
DEBUG_START(DEBUG_MAX)
fprintf(stdout," ddikick.x: DDI task %i started.\n",j);
DEBUG_END()
# endif
} }
if(info->nnodes == 1) continue;
if(polled == *(spawn+j+np)) {
if(tm_errno) {
fprintf(stderr," ddikick.x: error spawning DDI task %i.\n",j+np);
Fatal_error(911);
} else {
# if DDI_DEBUG
DEBUG_START(DEBUG_MAX)
fprintf(stdout," ddikick.x: DDI task %i started.\n",j+np);
DEBUG_END()
# endif
} } } }
/* -------------------------------------- *\
Close the link to the PBS Task Manager
\* -------------------------------------- */
tm_finalize();
/* ---------------- *\
Free used memory
\* ---------------- */
free(tid);
free(spawn);
free(rargs);
}
int main(
int argc,
char *argv[])
{
int c;
int err = 0;
int ncopies = -1;
int onenode = -1;
int rc;
struct tm_roots rootrot;
int nspawned = 0;
tm_node_id *nodelist;
int start;
int stop;
int sync = 0;
int pernode = 0;
char *targethost = NULL;
char *allnodes;
struct sigaction act;
char **ioenv;
extern int optind;
extern char *optarg;
int posixly_correct_set_by_caller = 0;
char *envstr;
id = malloc(60 * sizeof(char));
if (id == NULL)
{
fprintf(stderr, "%s: malloc failed, (%d)\n",
id,
errno);
return(1);
}
sprintf(id, "pbsdsh%s",
((getenv("PBSDEBUG") != NULL) && (getenv("PBS_TASKNUM") != NULL))
? getenv("PBS_TASKNUM")
: "");
#ifdef __GNUC__
/* If it's already set, we won't unset it later */
if (getenv("POSIXLY_CORRECT") != NULL)
posixly_correct_set_by_caller = 1;
envstr = strdup("POSIXLY_CORRECT=1");
putenv(envstr);
#endif
while ((c = getopt(argc, argv, "c:n:h:osuv")) != EOF)
{
switch (c)
{
case 'c':
ncopies = atoi(optarg);
if (ncopies <= 0)
{
err = 1;
}
break;
case 'h':
targethost = strdup(optarg); /* run on this 1 hostname */
break;
case 'n':
onenode = atoi(optarg);
if (onenode < 0)
{
err = 1;
}
break;
case 'o':
grabstdio = 1;
break;
case 's':
sync = 1; /* force synchronous spawns */
break;
case 'u':
pernode = 1; /* run once per node (unique hostnames) */
break;
case 'v':
verbose = 1; /* turn on verbose output */
break;
default:
err = 1;
break;
} /* END switch (c) */
} /* END while ((c = getopt()) != EOF) */
if ((err != 0) || ((onenode >= 0) && (ncopies >= 1)))
{
fprintf(stderr, "Usage: %s [-c copies][-o][-s][-u][-v] program [args]...]\n",
argv[0]);
fprintf(stderr, " %s [-n nodenumber][-o][-s][-u][-v] program [args]...\n",
argv[0]);
fprintf(stderr, " %s [-h hostname][-o][-v] program [args]...\n",
argv[0]);
fprintf(stderr, "Where -c copies = run copy of \"args\" on the first \"copies\" nodes,\n");
fprintf(stderr, " -n nodenumber = run a copy of \"args\" on the \"nodenumber\"-th node,\n");
fprintf(stderr, " -o = capture stdout of processes,\n");
fprintf(stderr, " -s = forces synchronous execution,\n");
fprintf(stderr, " -u = run on unique hostnames,\n");
fprintf(stderr, " -h = run on this specific hostname,\n");
fprintf(stderr, " -v = forces verbose output.\n");
exit(1);
}
#ifdef __GNUC__
if (!posixly_correct_set_by_caller)
{
putenv("POSIXLY_CORRECT");
free(envstr);
}
#endif
if (getenv("PBS_ENVIRONMENT") == NULL)
{
fprintf(stderr, "%s: not executing under PBS\n",
id);
return(1);
}
/*
* Set up interface to the Task Manager
*/
if ((rc = tm_init(0, &rootrot)) != TM_SUCCESS)
{
fprintf(stderr, "%s: tm_init failed, rc = %s (%d)\n",
id,
get_ecname(rc),
rc);
return(1);
}
sigemptyset(&allsigs);
sigaddset(&allsigs, SIGHUP);
sigaddset(&allsigs, SIGINT);
sigaddset(&allsigs, SIGTERM);
act.sa_mask = allsigs;
act.sa_flags = 0;
/* We want to abort system calls and call a function. */
#ifdef SA_INTERRUPT
act.sa_flags |= SA_INTERRUPT;
#endif
act.sa_handler = bailout;
sigaction(SIGHUP, &act, NULL);
sigaction(SIGINT, &act, NULL);
sigaction(SIGTERM, &act, NULL);
#ifdef DEBUG
if (rootrot.tm_parent == TM_NULL_TASK)
{
fprintf(stderr, "%s: I am the mother of all tasks\n",
id);
}
else
{
fprintf(stderr, "%s: I am but a child in the scheme of things\n",
id);
}
#endif /* DEBUG */
if ((rc = tm_nodeinfo(&nodelist, &numnodes)) != TM_SUCCESS)
{
fprintf(stderr, "%s: tm_nodeinfo failed, rc = %s (%d)\n",
id,
get_ecname(rc),
rc);
return(1);
}
/* nifty unique/hostname code */
if (pernode || targethost)
{
allnodes = gethostnames(nodelist);
if (targethost)
{
onenode = findtargethost(allnodes, targethost);
}
else
{
numnodes = uniquehostlist(nodelist, allnodes);
}
free(allnodes);
if (targethost)
free(targethost);
}
/* We already checked the lower bounds in the argument processing,
now we check the upper bounds */
if ((onenode >= numnodes) || (ncopies > numnodes))
{
fprintf(stderr, "%s: only %d nodes available\n",
id,
numnodes);
return(1);
}
/* malloc space for various arrays based on number of nodes/tasks */
tid = (tm_task_id *)calloc(numnodes, sizeof(tm_task_id));
events_spawn = (tm_event_t *)calloc(numnodes, sizeof(tm_event_t));
events_obit = (tm_event_t *)calloc(numnodes, sizeof(tm_event_t));
ev = (int *)calloc(numnodes, sizeof(int));
if ((tid == NULL) ||
(events_spawn == NULL) ||
(events_obit == NULL) ||
(ev == NULL))
{
/* FAILURE - cannot alloc memory */
fprintf(stderr, "%s: memory alloc of task ids failed\n",
id);
return(1);
}
for (c = 0; c < numnodes; c++)
{
*(tid + c) = TM_NULL_TASK;
*(events_spawn + c) = TM_NULL_EVENT;
*(events_obit + c) = TM_NULL_EVENT;
*(ev + c) = 0;
} /* END for (c) */
/* Now spawn the program to where it goes */
if (onenode >= 0)
{
/* Spawning one copy onto logical node "onenode" */
start = onenode;
stop = onenode + 1;
}
else if (ncopies >= 0)
{
/* Spawn a copy of the program to the first "ncopies" nodes */
start = 0;
stop = ncopies;
}
else
{
/* Spawn a copy on all nodes */
start = 0;
stop = numnodes;
}
if ((ioenv = calloc(2, sizeof(char *)))==NULL)
{
/* FAILURE - cannot alloc memory */
fprintf(stderr,"%s: memory alloc of ioenv failed\n",
id);
return(1);
}
if (grabstdio != 0)
{
stdoutfd = build_listener(&stdoutport);
if ((*ioenv = calloc(50,sizeof(char *))) == NULL)
{
/* FAILURE - cannot alloc memory */
fprintf(stderr,"%s: memory alloc of *ioenv failed\n",
id);
return(1);
}
snprintf(*ioenv,49,"TM_STDOUT_PORT=%d",
stdoutport);
FD_ZERO(&permrfsd);
}
sigprocmask(SIG_BLOCK, &allsigs, NULL);
for (c = start; c < stop; ++c)
{
if ((rc = tm_spawn(
argc - optind,
argv + optind,
ioenv,
*(nodelist + c),
tid + c,
events_spawn + c)) != TM_SUCCESS)
{
fprintf(stderr, "%s: spawn failed on node %d err %s\n",
id,
c,
get_ecname(rc));
}
else
{
if (verbose)
fprintf(stderr, "%s: spawned task %d\n",
id,
c);
++nspawned;
if (sync)
wait_for_task(&nspawned); /* one at a time */
}
} /* END for (c) */
if (sync == 0)
wait_for_task(&nspawned); /* wait for all to finish */
/*
* Terminate interface with Task Manager
*/
tm_finalize();
return 0;
} /* END main() */
char *gethostnames(
tm_node_id *nodelist)
{
char *allnodes;
char *rescinfo;
tm_event_t *rescevent;
tm_event_t resultevent;
char *hoststart;
int rc, tm_errno, i, j;
allnodes = calloc(numnodes, PBS_MAXNODENAME + 1 + sizeof(char));
rescinfo = calloc(numnodes, RESCSTRLEN + 1 + sizeof(char));
rescevent = calloc(numnodes, sizeof(tm_event_t));
if (!allnodes || !rescinfo || !rescevent)
{
fprintf(stderr, "%s: malloc failed!\n",
id);
tm_finalize();
exit(1);
}
/* submit resource requests */
for (i = 0; i < numnodes; i++)
{
if (tm_rescinfo(
nodelist[i],
rescinfo + (i*RESCSTRLEN),
RESCSTRLEN - 1,
rescevent + i) != TM_SUCCESS)
{
fprintf(stderr, "%s: error from tm_rescinfo()\n", id);
tm_finalize();
exit(1);
}
}
/* read back resource requests */
for (j = 0, i = 0; i < numnodes; i++)
{
rc = tm_poll(TM_NULL_EVENT, &resultevent, 1, &tm_errno);
if ((rc != TM_SUCCESS) || (tm_errno != TM_SUCCESS))
{
fprintf(stderr, "%s: error from tm_poll() %d\n",
id,
rc);
tm_finalize();
exit(1);
}
for (j = 0; j < numnodes; j++)
{
if (*(rescevent + j) == resultevent)
break;
}
if (j == numnodes)
{
fprintf(stderr, "%s: unknown resource result\n", id);
tm_finalize();
exit(1);
}
if (verbose)
fprintf(stderr, "%s: rescinfo from %d: %s\n", id, j, rescinfo + (j*RESCSTRLEN));
strtok(rescinfo + (j*RESCSTRLEN), " ");
hoststart = strtok(NULL, " ");
if (hoststart == NULL)
{
fprintf(stderr, "%s: can't find a hostname in resource result\n", id);
tm_finalize();
exit(1);
}
strcpy(allnodes + (j*PBS_MAXNODENAME), hoststart);
}
free(rescinfo);
free(rescevent);
return(allnodes);
}
void wait_for_task(
int *nspawned) /* number of tasks spawned */
{
int c;
tm_event_t eventpolled;
int nobits = 0;
int rc;
int tm_errno;
while (*nspawned || nobits)
{
if (grabstdio)
getstdout();
if (verbose)
{
}
if (fire_phasers)
{
tm_event_t event;
for (c = 0; c < numnodes; c++)
{
if (*(tid + c) == TM_NULL_TASK)
continue;
fprintf(stderr, "%s: killing task %u signal %d\n",
id,
*(tid + c),
fire_phasers);
tm_kill(*(tid + c), fire_phasers, &event);
}
tm_finalize();
exit(1);
}
sigprocmask(SIG_UNBLOCK, &allsigs, NULL);
rc = tm_poll(TM_NULL_EVENT, &eventpolled, !grabstdio, &tm_errno);
sigprocmask(SIG_BLOCK, &allsigs, NULL);
if (rc != TM_SUCCESS)
{
fprintf(stderr, "%s: Event poll failed, error %s\n",
id,
get_ecname(rc));
if (rc == TM_ENOTCONNECTED)
{
mom_reconnect();
}
else
{
exit(2);
}
}
if (eventpolled == TM_NULL_EVENT)
continue;
for (c = 0; c < numnodes; ++c)
{
if (eventpolled == *(events_spawn + c))
{
/* spawn event returned - register obit */
if (verbose)
{
fprintf(stderr, "%s: spawn event returned: %d (%d spawns and %d obits outstanding)\n",
id,
c,
*nspawned,
nobits);
}
(*nspawned)--;
if (tm_errno)
{
fprintf(stderr, "%s: error %d on spawn\n",
id,
tm_errno);
continue;
}
rc = obit_submit(c);
if (rc == TM_SUCCESS)
{
if ((*(events_obit + c) != TM_NULL_EVENT) &&
(*(events_obit + c) != TM_ERROR_EVENT))
{
nobits++;
}
}
}
else if (eventpolled == *(events_obit + c))
{
/* obit event, let's check it out */
if (tm_errno == TM_ESYSTEM)
{
if (verbose)
{
fprintf(stderr, "%s: error TM_ESYSTEM on obit (resubmitting)\n",
id);
}
sleep(2); /* Give the world a second to take a breath */
obit_submit(c);
continue; /* Go poll again */
}
if (tm_errno != 0)
{
fprintf(stderr, "%s: error %d on obit for task %d\n",
id,
tm_errno,
c);
}
/* task exited */
if (verbose)
{
fprintf(stderr, "%s: obit event returned: %d (%d spawns and %d obits outstanding)\n",
id,
c,
*nspawned,
nobits);
}
nobits--;
*(tid + c) = TM_NULL_TASK;
*(events_obit + c) = TM_NULL_EVENT;
if ((verbose != 0) || (*(ev + c) != 0))
{
fprintf(stderr, "%s: task %d exit status %d\n",
id,
c,
*(ev + c));
}
}
}
}
return;
} /* END wait_for_task() */
