/* return a vnode number matching targethost */ int findtargethost(char *allnodes, char *targethost) { int i; char *ptr; int vnode = 0; if ((ptr = strchr(targethost, '/')) != NULL) { *ptr = '\0'; ptr++; vnode = atoi(ptr); } for (i = 0; i < numnodes; i++) { if (!strcmp(allnodes + (i*PBS_MAXNODENAME), targethost)) { if (vnode == 0) return(i); vnode--; } } if (i == numnodes) { fprintf(stderr, "%s: %s not found\n", id, targethost); tm_finalize(); exit(1); } return(-1); }
HYD_status HYDT_bscd_pbs_launcher_finalize(void) { int err; HYD_status status = HYD_SUCCESS; HYDU_FUNC_ENTER(); #if defined(HAVE_TM_H) err = tm_finalize(); HYDU_ERR_CHKANDJUMP(status, err != TM_SUCCESS, HYD_INTERNAL_ERROR, "error calling tm_finalize\n"); #endif /* HAVE_TM_H */ if (HYDT_bscd_pbs_sys) { if (HYDT_bscd_pbs_sys->task_id) HYDU_FREE(HYDT_bscd_pbs_sys->task_id); if (HYDT_bscd_pbs_sys->spawn_events) HYDU_FREE(HYDT_bscd_pbs_sys->spawn_events); HYDU_FREE(HYDT_bscd_pbs_sys); } fn_exit: HYDU_FUNC_EXIT(); return status; fn_fail: goto fn_exit; }
void mom_reconnect(void) { int c, rc; struct tm_roots rootrot; for (;;) { tm_finalize(); sigprocmask(SIG_UNBLOCK, &allsigs, NULL); sleep(2); sigprocmask(SIG_BLOCK, &allsigs, NULL); /* attempt to reconnect */ rc = tm_init(0, &rootrot); if (rc == TM_SUCCESS) { fprintf(stderr, "%s: reconnected\n", id); /* resend obit requests */ for (c = 0; c < numnodes; ++c) { if (*(events_obit + c) != TM_NULL_EVENT) { rc = obit_submit(c); if (rc != TM_SUCCESS) { break; /* reconnect again */ } } else if (verbose) { fprintf(stderr, "%s: skipping obit resend for %u\n", id, *(tid + c)); } } break; } } return; } /* END mom_reconnect() */
/* return a vnode number matching targethost */ int findtargethost(char *allnodes, char *targethost) { int i; char *ptr; int vnode = 0; if ((ptr = strchr(targethost, '/')) != NULL) { *ptr = '\0'; ptr++; vnode = atoi(ptr); } for (i = 0; i < numnodes; i++) { if (!strcmp(allnodes + (i*PBS_MAXNODENAME), targethost)) { if (vnode == 0) return(i); vnode--; } else { /* Sometimes the allnodes will return the FQDN of the host and the PBS_NODEFILE will have the short name of the host. See if the shortname mataches */ std::string targetname(targethost); std::string the_host(allnodes + (i*PBS_MAXNODENAME)); std::size_t dot = the_host.find_first_of("."); if(dot != std::string::npos) { the_host[dot] = '\0'; std::string shortname(the_host.c_str()); if (shortname.compare(targetname) == 0) { if (vnode == 0) return(i); vnode--; } } } } if (i == numnodes) { fprintf(stderr, "%s: %s not found\n", id, targethost); tm_finalize(); exit(1); } return(-1); }
/* * Free stuff */ static int plm_tm_finalize(void) { int rc; /* cleanup any pending recvs */ if (ORTE_SUCCESS != (rc = orte_plm_base_comm_stop())) { ORTE_ERROR_LOG(rc); } if (connected) { tm_finalize(); connected = false; } return ORTE_SUCCESS; }
void Kickoff_PBS(const Node_info *ddinodes,const Cmdline_info *info) { char ddiinfo[] = "-ddi"; char procid[8]; char portid[8]; char nodeid[8]; char snodes[8]; char sprocs[8]; char **rargs; char **argv = info->argv; int i,j,r,iarg,nargs = info->ddiarg + info->nnodes + 8; int inode,ncpus,np = info->nprocs; int ntests; if(info->nnodes == 1) return; int tm_errno; tm_task_id *tid; tm_event_t *spawn; tm_event_t polled; struct tm_roots roots; tm_node_id *nodelist; /* ---------------------------------- *\ Initialize PBS Task Management API \* ---------------------------------- */ if(tm_init(0, &roots) != TM_SUCCESS) { fprintf(stderr, " ddikick.x: tm_init failed\n"); Fatal_error(911); } if(tm_nodeinfo(&nodelist, &np) != TM_SUCCESS) { fprintf(stderr, " ddikick.x: tm_nodeinfo failed.\n"); Fatal_error(911); } tid = (tm_task_id *) Malloc(2*np*sizeof(tm_task_id)); spawn = (tm_event_t *) Malloc(2*np*sizeof(tm_event_t)); for(i=0; i<2*np; i++) { *(tid + i) = TM_NULL_TASK; *(spawn + i) = TM_NULL_EVENT; } /* ----------------------------------------- *\ Initialize arguments to kickoff DDI tasks \* ----------------------------------------- */ rargs = (char **) Malloc(nargs*sizeof(char*)); sprintf(portid, "%d", info->kickoffport); sprintf(snodes, "%d", info->nnodes); sprintf(sprocs, "%d", info->nprocs); for(i=1,r=0; i<info->ddiarg-1; i++) rargs[r++] = argv[i]; rargs[r++] = ddiinfo; rargs[r++] = info->kickoffhost; /* kickoff host name */ rargs[r++] = portid; /* kickoff port number */ rargs[r++] = nodeid; /* rank of this node */ rargs[r++] = procid; /* rank of this process */ rargs[r++] = snodes; /* number of nodes */ rargs[r++] = sprocs; /* number of processors */ for(i=0,iarg=info->nodearg; i<info->nnodes; i++,iarg++) { rargs[r++] = argv[iarg]; } rargs[r] = NULL; /* ------------------------ *\ Spawn DDI tasks to nodes \* ------------------------ */ ncpus=ddinodes[0].cpus+ddinodes[1].cpus; for(i=ddinodes[0].cpus,inode=1; i<np; i++) { if(i == ncpus) ncpus += ddinodes[++inode].cpus; sprintf(nodeid,"%d",inode); sprintf(procid,"%d",i); # if DDI_DEBUG DEBUG_START(DEBUG_MAX) fprintf(stdout,"DDI Process %i PBS tm_spawn arguments: ",i); for(iarg=0; iarg<r; iarg++) fprintf(stdout,"%s ",rargs[iarg]); fprintf(stdout,"\n"); DEBUG_END() # endif /* ------------------------- *\ Spawn DDI Compute Process \* ------------------------- */ if(tm_spawn(r,rargs,NULL,*(nodelist+i),(tid+i),spawn+i) != TM_SUCCESS) { fprintf(stderr," ddikick.x: tm_spawn failed.\n"); Fatal_error(911); } /* ---------------------------------- *\ No data server on single node runs \* ---------------------------------- */ if(info->nnodes == 1) continue; # if DDI_DEBUG DEBUG_START(DEBUG_MAX) fprintf(stdout,"DDI Process %i PBS tm_spawn arguments: ",j); for(iarg=0; iarg<r; iarg++) fprintf(stdout,"%s ",rargs[iarg]); fprintf(stdout,"\n"); DEBUG_END() # endif j = i+np; sprintf(procid,"%d",j); /* --------------------- *\ Spawn DDI Data Server \* --------------------- */ if(tm_spawn(r,rargs,NULL,*(nodelist+i),(tid+j),spawn+j) != TM_SUCCESS) { fprintf(stderr," ddikick.x: tm_spawn failed.\n"); Fatal_error(911); } } /* -------------------------------------------------------- *\ Poll PBS to ensure each DDI process started successfully \* -------------------------------------------------------- */ ntests = np-ddinodes[0].cpus; if(USING_DATA_SERVERS()) ntests *= 2; for(i=ntests; i--; ) { if(tm_poll(TM_NULL_EVENT,&polled,1,&tm_errno) != TM_SUCCESS) { fprintf(stderr," ddikick.x: tm_poll failed.\n"); Fatal_error(911); } for(j=0; j<np; j++) { if(polled == *(spawn+j)) { if(tm_errno) { fprintf(stderr," ddikick.x: error spawning DDI task %i.\n",j); Fatal_error(911); } else { # if DDI_DEBUG DEBUG_START(DEBUG_MAX) fprintf(stdout," ddikick.x: DDI task %i started.\n",j); DEBUG_END() # endif } } if(info->nnodes == 1) continue; if(polled == *(spawn+j+np)) { if(tm_errno) { fprintf(stderr," ddikick.x: error spawning DDI task %i.\n",j+np); Fatal_error(911); } else { # if DDI_DEBUG DEBUG_START(DEBUG_MAX) fprintf(stdout," ddikick.x: DDI task %i started.\n",j+np); DEBUG_END() # endif } } } } /* -------------------------------------- *\ Close the link to the PBS Task Manager \* -------------------------------------- */ tm_finalize(); /* ---------------- *\ Free used memory \* ---------------- */ free(tid); free(spawn); free(rargs); }
int main( int argc, char *argv[]) { int c; int err = 0; int ncopies = -1; int onenode = -1; int rc; struct tm_roots rootrot; int nspawned = 0; tm_node_id *nodelist; int start; int stop; int sync = 0; int pernode = 0; char *targethost = NULL; char *allnodes; struct sigaction act; char **ioenv; extern int optind; extern char *optarg; int posixly_correct_set_by_caller = 0; char *envstr; id = malloc(60 * sizeof(char)); if (id == NULL) { fprintf(stderr, "%s: malloc failed, (%d)\n", id, errno); return(1); } sprintf(id, "pbsdsh%s", ((getenv("PBSDEBUG") != NULL) && (getenv("PBS_TASKNUM") != NULL)) ? getenv("PBS_TASKNUM") : ""); #ifdef __GNUC__ /* If it's already set, we won't unset it later */ if (getenv("POSIXLY_CORRECT") != NULL) posixly_correct_set_by_caller = 1; envstr = strdup("POSIXLY_CORRECT=1"); putenv(envstr); #endif while ((c = getopt(argc, argv, "c:n:h:osuv")) != EOF) { switch (c) { case 'c': ncopies = atoi(optarg); if (ncopies <= 0) { err = 1; } break; case 'h': targethost = strdup(optarg); /* run on this 1 hostname */ break; case 'n': onenode = atoi(optarg); if (onenode < 0) { err = 1; } break; case 'o': grabstdio = 1; break; case 's': sync = 1; /* force synchronous spawns */ break; case 'u': pernode = 1; /* run once per node (unique hostnames) */ break; case 'v': verbose = 1; /* turn on verbose output */ break; default: err = 1; break; } /* END switch (c) */ } /* END while ((c = getopt()) != EOF) */ if ((err != 0) || ((onenode >= 0) && (ncopies >= 1))) { fprintf(stderr, "Usage: %s [-c copies][-o][-s][-u][-v] program [args]...]\n", argv[0]); fprintf(stderr, " %s [-n nodenumber][-o][-s][-u][-v] program [args]...\n", argv[0]); fprintf(stderr, " %s [-h hostname][-o][-v] program [args]...\n", argv[0]); fprintf(stderr, "Where -c copies = run copy of \"args\" on the first \"copies\" nodes,\n"); fprintf(stderr, " -n nodenumber = run a copy of \"args\" on the \"nodenumber\"-th node,\n"); fprintf(stderr, " -o = capture stdout of processes,\n"); fprintf(stderr, " -s = forces synchronous execution,\n"); fprintf(stderr, " -u = run on unique hostnames,\n"); fprintf(stderr, " -h = run on this specific hostname,\n"); fprintf(stderr, " -v = forces verbose output.\n"); exit(1); } #ifdef __GNUC__ if (!posixly_correct_set_by_caller) { putenv("POSIXLY_CORRECT"); free(envstr); } #endif if (getenv("PBS_ENVIRONMENT") == NULL) { fprintf(stderr, "%s: not executing under PBS\n", id); return(1); } /* * Set up interface to the Task Manager */ if ((rc = tm_init(0, &rootrot)) != TM_SUCCESS) { fprintf(stderr, "%s: tm_init failed, rc = %s (%d)\n", id, get_ecname(rc), rc); return(1); } sigemptyset(&allsigs); sigaddset(&allsigs, SIGHUP); sigaddset(&allsigs, SIGINT); sigaddset(&allsigs, SIGTERM); act.sa_mask = allsigs; act.sa_flags = 0; /* We want to abort system calls and call a function. */ #ifdef SA_INTERRUPT act.sa_flags |= SA_INTERRUPT; #endif act.sa_handler = bailout; sigaction(SIGHUP, &act, NULL); sigaction(SIGINT, &act, NULL); sigaction(SIGTERM, &act, NULL); #ifdef DEBUG if (rootrot.tm_parent == TM_NULL_TASK) { fprintf(stderr, "%s: I am the mother of all tasks\n", id); } else { fprintf(stderr, "%s: I am but a child in the scheme of things\n", id); } #endif /* DEBUG */ if ((rc = tm_nodeinfo(&nodelist, &numnodes)) != TM_SUCCESS) { fprintf(stderr, "%s: tm_nodeinfo failed, rc = %s (%d)\n", id, get_ecname(rc), rc); return(1); } /* nifty unique/hostname code */ if (pernode || targethost) { allnodes = gethostnames(nodelist); if (targethost) { onenode = findtargethost(allnodes, targethost); } else { numnodes = uniquehostlist(nodelist, allnodes); } free(allnodes); if (targethost) free(targethost); } /* We already checked the lower bounds in the argument processing, now we check the upper bounds */ if ((onenode >= numnodes) || (ncopies > numnodes)) { fprintf(stderr, "%s: only %d nodes available\n", id, numnodes); return(1); } /* malloc space for various arrays based on number of nodes/tasks */ tid = (tm_task_id *)calloc(numnodes, sizeof(tm_task_id)); events_spawn = (tm_event_t *)calloc(numnodes, sizeof(tm_event_t)); events_obit = (tm_event_t *)calloc(numnodes, sizeof(tm_event_t)); ev = (int *)calloc(numnodes, sizeof(int)); if ((tid == NULL) || (events_spawn == NULL) || (events_obit == NULL) || (ev == NULL)) { /* FAILURE - cannot alloc memory */ fprintf(stderr, "%s: memory alloc of task ids failed\n", id); return(1); } for (c = 0; c < numnodes; c++) { *(tid + c) = TM_NULL_TASK; *(events_spawn + c) = TM_NULL_EVENT; *(events_obit + c) = TM_NULL_EVENT; *(ev + c) = 0; } /* END for (c) */ /* Now spawn the program to where it goes */ if (onenode >= 0) { /* Spawning one copy onto logical node "onenode" */ start = onenode; stop = onenode + 1; } else if (ncopies >= 0) { /* Spawn a copy of the program to the first "ncopies" nodes */ start = 0; stop = ncopies; } else { /* Spawn a copy on all nodes */ start = 0; stop = numnodes; } if ((ioenv = calloc(2, sizeof(char *)))==NULL) { /* FAILURE - cannot alloc memory */ fprintf(stderr,"%s: memory alloc of ioenv failed\n", id); return(1); } if (grabstdio != 0) { stdoutfd = build_listener(&stdoutport); if ((*ioenv = calloc(50,sizeof(char *))) == NULL) { /* FAILURE - cannot alloc memory */ fprintf(stderr,"%s: memory alloc of *ioenv failed\n", id); return(1); } snprintf(*ioenv,49,"TM_STDOUT_PORT=%d", stdoutport); FD_ZERO(&permrfsd); } sigprocmask(SIG_BLOCK, &allsigs, NULL); for (c = start; c < stop; ++c) { if ((rc = tm_spawn( argc - optind, argv + optind, ioenv, *(nodelist + c), tid + c, events_spawn + c)) != TM_SUCCESS) { fprintf(stderr, "%s: spawn failed on node %d err %s\n", id, c, get_ecname(rc)); } else { if (verbose) fprintf(stderr, "%s: spawned task %d\n", id, c); ++nspawned; if (sync) wait_for_task(&nspawned); /* one at a time */ } } /* END for (c) */ if (sync == 0) wait_for_task(&nspawned); /* wait for all to finish */ /* * Terminate interface with Task Manager */ tm_finalize(); return 0; } /* END main() */
char *gethostnames( tm_node_id *nodelist) { char *allnodes; char *rescinfo; tm_event_t *rescevent; tm_event_t resultevent; char *hoststart; int rc, tm_errno, i, j; allnodes = calloc(numnodes, PBS_MAXNODENAME + 1 + sizeof(char)); rescinfo = calloc(numnodes, RESCSTRLEN + 1 + sizeof(char)); rescevent = calloc(numnodes, sizeof(tm_event_t)); if (!allnodes || !rescinfo || !rescevent) { fprintf(stderr, "%s: malloc failed!\n", id); tm_finalize(); exit(1); } /* submit resource requests */ for (i = 0; i < numnodes; i++) { if (tm_rescinfo( nodelist[i], rescinfo + (i*RESCSTRLEN), RESCSTRLEN - 1, rescevent + i) != TM_SUCCESS) { fprintf(stderr, "%s: error from tm_rescinfo()\n", id); tm_finalize(); exit(1); } } /* read back resource requests */ for (j = 0, i = 0; i < numnodes; i++) { rc = tm_poll(TM_NULL_EVENT, &resultevent, 1, &tm_errno); if ((rc != TM_SUCCESS) || (tm_errno != TM_SUCCESS)) { fprintf(stderr, "%s: error from tm_poll() %d\n", id, rc); tm_finalize(); exit(1); } for (j = 0; j < numnodes; j++) { if (*(rescevent + j) == resultevent) break; } if (j == numnodes) { fprintf(stderr, "%s: unknown resource result\n", id); tm_finalize(); exit(1); } if (verbose) fprintf(stderr, "%s: rescinfo from %d: %s\n", id, j, rescinfo + (j*RESCSTRLEN)); strtok(rescinfo + (j*RESCSTRLEN), " "); hoststart = strtok(NULL, " "); if (hoststart == NULL) { fprintf(stderr, "%s: can't find a hostname in resource result\n", id); tm_finalize(); exit(1); } strcpy(allnodes + (j*PBS_MAXNODENAME), hoststart); } free(rescinfo); free(rescevent); return(allnodes); }
void wait_for_task( int *nspawned) /* number of tasks spawned */ { int c; tm_event_t eventpolled; int nobits = 0; int rc; int tm_errno; while (*nspawned || nobits) { if (grabstdio) getstdout(); if (verbose) { } if (fire_phasers) { tm_event_t event; for (c = 0; c < numnodes; c++) { if (*(tid + c) == TM_NULL_TASK) continue; fprintf(stderr, "%s: killing task %u signal %d\n", id, *(tid + c), fire_phasers); tm_kill(*(tid + c), fire_phasers, &event); } tm_finalize(); exit(1); } sigprocmask(SIG_UNBLOCK, &allsigs, NULL); rc = tm_poll(TM_NULL_EVENT, &eventpolled, !grabstdio, &tm_errno); sigprocmask(SIG_BLOCK, &allsigs, NULL); if (rc != TM_SUCCESS) { fprintf(stderr, "%s: Event poll failed, error %s\n", id, get_ecname(rc)); if (rc == TM_ENOTCONNECTED) { mom_reconnect(); } else { exit(2); } } if (eventpolled == TM_NULL_EVENT) continue; for (c = 0; c < numnodes; ++c) { if (eventpolled == *(events_spawn + c)) { /* spawn event returned - register obit */ if (verbose) { fprintf(stderr, "%s: spawn event returned: %d (%d spawns and %d obits outstanding)\n", id, c, *nspawned, nobits); } (*nspawned)--; if (tm_errno) { fprintf(stderr, "%s: error %d on spawn\n", id, tm_errno); continue; } rc = obit_submit(c); if (rc == TM_SUCCESS) { if ((*(events_obit + c) != TM_NULL_EVENT) && (*(events_obit + c) != TM_ERROR_EVENT)) { nobits++; } } } else if (eventpolled == *(events_obit + c)) { /* obit event, let's check it out */ if (tm_errno == TM_ESYSTEM) { if (verbose) { fprintf(stderr, "%s: error TM_ESYSTEM on obit (resubmitting)\n", id); } sleep(2); /* Give the world a second to take a breath */ obit_submit(c); continue; /* Go poll again */ } if (tm_errno != 0) { fprintf(stderr, "%s: error %d on obit for task %d\n", id, tm_errno, c); } /* task exited */ if (verbose) { fprintf(stderr, "%s: obit event returned: %d (%d spawns and %d obits outstanding)\n", id, c, *nspawned, nobits); } nobits--; *(tid + c) = TM_NULL_TASK; *(events_obit + c) = TM_NULL_EVENT; if ((verbose != 0) || (*(ev + c) != 0)) { fprintf(stderr, "%s: task %d exit status %d\n", id, c, *(ev + c)); } } } } return; } /* END wait_for_task() */