/* save a job array struct to disk returns zero if no errors*/
int array_save(
job_array *pa)
{
int fds;
char namebuf[MAXPATHLEN];
array_request_node *rn;
int num_tokens = 0;
snprintf(namebuf, sizeof(namebuf), "%s%s%s",
path_arrays, pa->ai_qs.fileprefix, ARRAY_FILE_SUFFIX);
fds = open(namebuf, O_Sync | O_TRUNC | O_WRONLY | O_CREAT, 0600);
if (fds < 0)
{
return -1;
}
if (write_ac_socket(fds, &(pa->ai_qs), sizeof(struct array_info)) == -1)
{
unlink(namebuf);
close(fds);
return -1;
}
/* count number of request tokens left */
for (rn = (array_request_node*)GET_NEXT(pa->request_tokens), num_tokens = 0;
rn != NULL;
rn = (array_request_node*)GET_NEXT(rn->request_tokens_link), num_tokens++);
if (write_ac_socket(fds, &num_tokens, sizeof(num_tokens)) == -1)
{
unlink(namebuf);
close(fds);
return -1;
}
if (num_tokens > 0)
{
for (rn = (array_request_node*)GET_NEXT(pa->request_tokens); rn != NULL;
rn = (array_request_node*)GET_NEXT(rn->request_tokens_link))
{
if (write_ac_socket(fds, rn, sizeof(array_request_node)) == -1)
{
unlink(namebuf);
close(fds);
return -1;
}
}
}
close(fds);
return(PBSE_NONE);
} /* END array_save() */
static int put_4byte(
int sock, /* socket to read from */
unsigned int val) /* 4 byte interger to write */
{
int amt;
union
{
int unl;
char unc[sizeof(unsigned int)];
} un;
un.unl = htonl(val);
amt = write_ac_socket(sock, (char *)(un.unc + sizeof(unsigned int) - 4), 4);
if (amt != 4)
{
/* FAILURE */
return(-1);
}
/* SUCCESS */
return(0);
}
int write_buffer(
char *buf,
int len,
int fds)
{
int written;
while ((written = write_ac_socket(fds,buf,len)) != len)
{
if ((errno == EINTR) &&
(written == -1))
continue;
else if (written > 0)
{
len -= written;
buf += written;
}
else
{
log_err(errno,__func__,"Unable to write to file or socket");
return(-1);
}
}
return(PBSE_NONE);
} /* END write_buffer */
/*
* change_directory_as_needed()
*
* Changes the directory to the job's home directory for user pelogs
*
* @param which - the kind of pelog we're preparing to execute
* @param pjob - the job whose pelog we're preparing to execute
* @post-cond: we have changed to the appropriate directory if possible and necessary
*/
void change_directory_as_needed(
int which,
job *pjob)
{
if ((which == PE_PROLOGUSER) ||
(which == PE_EPILOGUSER) ||
(which == PE_PROLOGUSERJOB) ||
(which == PE_EPILOGUSERJOB))
{
if (chdir(pjob->ji_grpcache->gc_homedir) != 0)
{
/* warn only, no failure */
sprintf(log_buffer,
"PBS: chdir to %s failed: %s (running user %s in current directory)",
pjob->ji_grpcache->gc_homedir,
strerror(errno),
which == PE_PROLOGUSER ? "prologue" : "epilogue");
if (write_ac_socket(2, log_buffer, strlen(log_buffer)) == -1) {}
fsync(2);
}
}
} /* END change_directory_as_needed() */
int DIS_tcp_wflush(
struct tcp_chan *chan) /* I */
{
size_t ct;
int i;
char *pb = NULL;
char *pbs_debug = NULL;
struct tcpdisbuf *tp;
pbs_debug = getenv("PBSDEBUG");
tp = &chan->writebuf;
pb = tp->tdis_thebuf;
ct = tp->tdis_trailp - tp->tdis_thebuf;
while ((i = write_ac_socket(chan->sock, pb, ct)) != (ssize_t)ct)
{
if (i == -1)
{
if (errno == EINTR)
{
continue;
}
/* FAILURE */
if (pbs_debug != NULL)
{
fprintf(stderr,
"TCP write of %d bytes (%.32s) [sock=%d] failed, errno=%d (%s)\n",
(int)ct, pb, chan->sock, errno, strerror(errno));
}
return(-1);
} /* END if (i == -1) */
else
{
ct -= i;
pb += i;
}
} /* END while (i) */
/* SUCCESS */
tp->tdis_eod = tp->tdis_leadp;
tcp_pack_buff(tp);
return(0);
} /* END DIS_tcp_wflush() */
void track_save(
struct work_task *pwt) /* unused */
{
int fd;
const char *myid = "save_track";
time_t time_now = time(NULL);
work_task *wt;
/* set task for next round trip */
if (pwt) /* set up another work task for next time period */
{
free(pwt->wt_mutex);
free(pwt);
wt = set_task(WORK_Timed, (long)time_now + PBS_SAVE_TRACK_TM, track_save, (char *)NULL, FALSE);
if (wt == NULL)
log_err(errno, myid, (char *)"Unable to set task for save");
}
if (server.sv_trackmodifed == 0)
return; /* nothing to do this time */
fd = open(path_track, O_WRONLY, 0);
if (fd < 0)
{
log_err(errno, myid, (char *)"Unable to open tracking file");
return;
}
if (write_ac_socket(fd, (char *)server.sv_track, server.sv_tracksize * sizeof(struct tracking)) !=
(ssize_t)(server.sv_tracksize * sizeof(struct tracking)))
{
log_err(errno, myid, (char *)"failed to write to track file");
}
if (close(fd) < 0)
{
log_err(errno, myid, (char *)"failed to close track file after saving");
return;
}
server.sv_trackmodifed = 0;
return;
}
int write_munge_temp_file(
struct batch_request *preq, /* I */
char *mungeFileName) /* I */
{
int fd;
int cred_size;
int bytes_written;
int rc;
if ((fd = open(mungeFileName, O_CREAT | O_RDWR, S_IRUSR | S_IWUSR)) < 0)
{
req_reject(PBSE_SYSTEM, 0, preq, NULL, "could not create temporary munge file");
return(-1);
}
if ((cred_size = strlen(preq->rq_ind.rq_authen.rq_cred)) == 0)
{
req_reject(PBSE_BADCRED, 0, preq, NULL, "munge credential invalid");
close(fd);
return(-1);
}
bytes_written = write_ac_socket(fd, preq->rq_ind.rq_authen.rq_cred, cred_size);
if ((bytes_written == -1) ||
(bytes_written != cred_size))
{
req_reject(PBSE_SYSTEM, 0, preq, NULL, "could not write credential to temporary munge file");
close(fd);
return(-1);
}
if ((rc = fsync(fd)) < 0)
{
close(fd);
return(rc);
}
close(fd);
return(PBSE_NONE);
} /* END write_munge_temp_file() */
int run_pelog(
int which, /* I (one of PE_*) */
char *specpelog, /* I - script path */
job *pjob, /* I - associated job */
int pe_io_type, /* I - io type */
int deletejob) /* I - called before a job being deleted (purge -p) */
{
struct sigaction act;
struct sigaction oldact;
char *arg[12];
int fds1 = 0;
int fds2 = 0;
int fd_input;
char resc_list[2048];
char resc_used[2048];
struct stat sbuf;
char sid[20];
char exit_stat[11];
int waitst;
int isjoined; /* boolean */
char buf[MAXPATHLEN + 1024];
char pelog[MAXPATHLEN + 1024];
uid_t real_uid;
gid_t *real_gids = NULL;
gid_t real_gid;
int num_gids;
int jobtypespecified = 0;
resource *r;
char *EmptyString = (char *)"";
int LastArg;
int aindex;
int rc;
char *ptr;
int moabenvcnt = 14; /* # of entries in moabenvs */
static char *moabenvs[] = {
(char *)"MOAB_NODELIST",
(char *)"MOAB_JOBID",
(char *)"MOAB_JOBNAME",
(char *)"MOAB_USER",
(char *)"MOAB_GROUP",
(char *)"MOAB_CLASS",
(char *)"MOAB_TASKMAP",
(char *)"MOAB_QOS",
(char *)"MOAB_PARTITION",
(char *)"MOAB_PROCCOUNT",
(char *)"MOAB_NODECOUNT",
(char *)"MOAB_MACHINE",
(char *)"MOAB_JOBARRAYINDEX",
(char *)"MOAB_JOBARRAYRANGE"
};
if ((pjob == NULL) || (specpelog == NULL) || (specpelog[0] == '\0'))
{
return(0);
}
ptr = pjob->ji_wattr[JOB_ATR_jobtype].at_val.at_str;
if (ptr != NULL)
{
jobtypespecified = 1;
snprintf(pelog,sizeof(pelog),"%s.%s",
specpelog,
ptr);
}
else
{
snprintf(pelog, sizeof(pelog), "%s", specpelog);
}
real_uid = getuid();
real_gid = getgid();
if ((num_gids = getgroups(0, real_gids)) < 0)
{
log_err(errno, __func__, (char *)"getgroups failed\n");
return(-1);
}
/* to support root squashing, become the user before performing file checks */
if ((which == PE_PROLOGUSER) ||
(which == PE_EPILOGUSER) ||
(which == PE_PROLOGUSERJOB) ||
(which == PE_EPILOGUSERJOB))
{
real_gids = (gid_t *)calloc(num_gids, sizeof(gid_t));
if (real_gids == NULL)
{
log_err(ENOMEM, __func__, (char *)"Cannot allocate memory! FAILURE\n");
return(-1);
}
if (getgroups(num_gids,real_gids) < 0)
{
log_err(errno, __func__, (char *)"getgroups failed\n");
free(real_gids);
return(-1);
}
/* pjob->ji_grpcache will not be set if using LDAP and LDAP not set */
/* It is possible that ji_grpcache failed to allocate as well.
Make sure ji_grpcache is not NULL */
if (pjob->ji_grpcache != NULL)
{
if (setgroups(
pjob->ji_grpcache->gc_ngroup,
(gid_t *)pjob->ji_grpcache->gc_groups) != 0)
{
snprintf(log_buffer,sizeof(log_buffer),
"setgroups() for UID = %lu failed: %s\n",
(unsigned long)pjob->ji_qs.ji_un.ji_momt.ji_exuid,
strerror(errno));
log_err(errno, __func__, log_buffer);
undo_set_euid_egid(which,real_uid,real_gid,num_gids,real_gids,__func__);
free(real_gids);
return(-1);
}
}
else
{
sprintf(log_buffer, "pjob->ji_grpcache is null. check_pwd likely failed.");
log_err(-1, __func__, log_buffer);
undo_set_euid_egid(which,real_uid,real_gid,num_gids,real_gids,__func__);
free(real_gids);
return(-1);
}
if (setegid(pjob->ji_qs.ji_un.ji_momt.ji_exgid) != 0)
{
snprintf(log_buffer,sizeof(log_buffer),
"setegid(%lu) for UID = %lu failed: %s\n",
(unsigned long)pjob->ji_qs.ji_un.ji_momt.ji_exgid,
(unsigned long)pjob->ji_qs.ji_un.ji_momt.ji_exuid,
strerror(errno));
log_err(errno, __func__, log_buffer);
undo_set_euid_egid(which,real_uid,real_gid,num_gids,real_gids,__func__);
free(real_gids);
return(-1);
}
if (setuid_ext(pjob->ji_qs.ji_un.ji_momt.ji_exuid, TRUE) != 0)
{
snprintf(log_buffer,sizeof(log_buffer),
"seteuid(%lu) failed: %s\n",
(unsigned long)pjob->ji_qs.ji_un.ji_momt.ji_exuid,
strerror(errno));
log_err(errno, __func__, log_buffer);
undo_set_euid_egid(which,real_uid,real_gid,num_gids,real_gids,__func__);
free(real_gids);
return(-1);
}
}
rc = stat(pelog,&sbuf);
if ((rc == -1) && (jobtypespecified == 1))
{
snprintf(pelog, sizeof(pelog), "%s", specpelog);
rc = stat(pelog,&sbuf);
}
if (rc == -1)
{
if (errno == ENOENT || errno == EBADF)
{
/* epilog/prolog script does not exist */
if (LOGLEVEL >= 5)
{
static char tmpBuf[1024];
sprintf(log_buffer, "%s script '%s' for job %s does not exist (cwd: %s,pid: %d)",
PPEType[which],
(pelog[0] != '\0') ? pelog : "NULL",
pjob->ji_qs.ji_jobid,
getcwd(tmpBuf, sizeof(tmpBuf)),
getpid());
log_record(PBSEVENT_SYSTEM, 0, __func__, log_buffer);
}
#ifdef ENABLE_CSA
if ((which == PE_EPILOGUSER) && (!strcmp(pelog, path_epiloguser)))
{
/*
* Add a workload management end record
*/
if (LOGLEVEL >= 8)
{
sprintf(log_buffer, "%s calling add_wkm_end from run_pelog() - no user epilog",
pjob->ji_qs.ji_jobid);
log_err(-1, __func__, log_buffer);
}
add_wkm_end(pjob->ji_wattr[JOB_ATR_pagg_id].at_val.at_ll,
pjob->ji_qs.ji_un.ji_momt.ji_exitstat, pjob->ji_qs.ji_jobid);
}
#endif /* ENABLE_CSA */
undo_set_euid_egid(which,real_uid,real_gid,num_gids,real_gids,__func__);
free(real_gids);
return(0);
}
undo_set_euid_egid(which,real_uid,real_gid,num_gids,real_gids,__func__);
free(real_gids);
return(pelog_err(pjob,pelog,errno,(char *)"cannot stat"));
}
if (LOGLEVEL >= 5)
{
sprintf(log_buffer,"running %s script '%s' for job %s",
PPEType[which],
(pelog[0] != '\0') ? pelog : "NULL",
pjob->ji_qs.ji_jobid);
log_ext(-1, __func__, log_buffer, LOG_DEBUG); /* not actually an error--but informational */
}
/* script must be owned by root, be regular file, read and execute by user *
* and not writeable by group or other */
if (reduceprologchecks == TRUE)
{
if ((!S_ISREG(sbuf.st_mode)) ||
(!(sbuf.st_mode & (S_IXUSR | S_IXGRP | S_IXOTH))))
{
undo_set_euid_egid(which,real_uid,real_gid,num_gids,real_gids,__func__);
free(real_gids);
return(pelog_err(pjob,pelog,-1, (char *)"permission Error"));
}
}
else
{
if (which == PE_PROLOGUSERJOB || which == PE_EPILOGUSERJOB)
{
if ((sbuf.st_uid != pjob->ji_qs.ji_un.ji_momt.ji_exuid) ||
(!S_ISREG(sbuf.st_mode)) ||
((sbuf.st_mode & (S_IRUSR | S_IXUSR)) != (S_IRUSR | S_IXUSR)) ||
(sbuf.st_mode & (S_IWGRP | S_IWOTH)))
{
undo_set_euid_egid(which,real_uid,real_gid,num_gids,real_gids,__func__);
free(real_gids);
return(pelog_err(pjob,pelog,-1, (char *)"permission Error"));
}
}
else if ((sbuf.st_uid != 0) ||
(!S_ISREG(sbuf.st_mode)) ||
((sbuf.st_mode & (S_IRUSR | S_IXUSR)) != (S_IRUSR | S_IXUSR)) ||\
(sbuf.st_mode & (S_IWGRP | S_IWOTH)))
{
undo_set_euid_egid(which,real_uid,real_gid,num_gids,real_gids,__func__);
free(real_gids);
return(pelog_err(pjob,pelog,-1, (char *)"permission Error"));
}
if ((which == PE_PROLOGUSER) || (which == PE_EPILOGUSER))
{
/* script must also be read and execute by other */
if ((sbuf.st_mode & (S_IROTH | S_IXOTH)) != (S_IROTH | S_IXOTH))
{
undo_set_euid_egid(which,real_uid,real_gid,num_gids,real_gids,__func__);
free(real_gids);
return(pelog_err(pjob, pelog, -1, (char *)"permission Error"));
}
}
} /* END !reduceprologchecks */
fd_input = pe_input(pjob->ji_qs.ji_jobid);
if (fd_input < 0)
{
undo_set_euid_egid(which,real_uid,real_gid,num_gids,real_gids,__func__);
free(real_gids);
return(pelog_err(pjob, pelog, -2, (char *)"no pro/epilogue input file"));
}
run_exit = 0;
child = fork();
if (child > 0)
{
int KillSent = FALSE;
/* parent - watch for prolog/epilog to complete */
close(fd_input);
/* switch back to root if necessary */
undo_set_euid_egid(which,real_uid,real_gid,num_gids,real_gids,__func__);
free(real_gids);
act.sa_handler = pelogalm;
sigemptyset(&act.sa_mask);
act.sa_flags = 0;
sigaction(SIGALRM, &act, &oldact);
/* it would be nice if the harvest routine could block for 5 seconds,
and if the prolog is not complete in that time, mark job as prolog
pending, append prolog child, and continue */
/* main loop should attempt to harvest prolog in non-blocking mode.
If unsuccessful after timeout, job should be terminated, and failure
reported. If successful, mom should unset prolog pending, and
continue with job start sequence. Mom should report job as running
while prologpending flag is set. (NOTE: must track per job prolog
start time)
*/
alarm(pe_alarm_time);
while (waitpid(child, &waitst, 0) < 0)
{
if (errno != EINTR)
{
/* exit loop. non-alarm based failure occurred */
run_exit = -3;
MOMPrologFailureCount++;
break;
}
if (run_exit == -4)
{
if (KillSent == FALSE)
{
MOMPrologTimeoutCount++;
/* timeout occurred */
KillSent = TRUE;
/* NOTE: prolog/epilog may be locked in KERNEL space and unkillable */
alarm(5);
}
else
{
/* cannot kill prolog/epilog, give up */
run_exit = -5;
break;
}
}
} /* END while (wait(&waitst) < 0) */
/* epilog/prolog child completed */
#ifdef ENABLE_CSA
if ((which == PE_EPILOGUSER) && (!strcmp(pelog, path_epiloguser)))
{
/*
* Add a workload management end record
*/
if (LOGLEVEL >= 8)
{
sprintf(log_buffer, "%s calling add_wkm_end from run_pelog() - after user epilog",
pjob->ji_qs.ji_jobid);
log_err(-1, __func__, log_buffer);
}
add_wkm_end(pjob->ji_wattr[JOB_ATR_pagg_id].at_val.at_ll,
pjob->ji_qs.ji_un.ji_momt.ji_exitstat, pjob->ji_qs.ji_jobid);
}
#endif /* ENABLE_CSA */
alarm(0);
/* restore the previous handler */
sigaction(SIGALRM, &oldact, 0);
if (run_exit == 0)
{
if (WIFEXITED(waitst))
{
run_exit = WEXITSTATUS(waitst);
}
}
}
else
{
/* child - run script */
log_close(0);
if (lockfds >= 0)
{
close(lockfds);
lockfds = -1;
}
net_close(-1);
if (fd_input != 0)
{
close(0);
if (dup(fd_input) == -1) {}
close(fd_input);
}
if (pe_io_type == PE_IO_TYPE_NULL)
{
/* no output, force to /dev/null */
fds1 = open("/dev/null", O_WRONLY, 0600);
fds2 = open("/dev/null", O_WRONLY, 0600);
}
else if (pe_io_type == PE_IO_TYPE_STD)
{
/* open job standard out/error */
/*
* We need to know if files are joined or not.
* If they are then open the correct file and duplicate it to the other
*/
isjoined = is_joined(pjob);
switch (isjoined)
{
case -1:
fds2 = open_std_file(pjob, StdErr, O_WRONLY | O_APPEND,
pjob->ji_qs.ji_un.ji_momt.ji_exgid);
fds1 = (fds2 < 0)?-1:dup(fds2);
break;
case 1:
fds1 = open_std_file(pjob, StdOut, O_WRONLY | O_APPEND,
pjob->ji_qs.ji_un.ji_momt.ji_exgid);
fds2 = (fds1 < 0)?-1:dup(fds1);
break;
default:
fds1 = open_std_file(pjob, StdOut, O_WRONLY | O_APPEND,
pjob->ji_qs.ji_un.ji_momt.ji_exgid);
fds2 = open_std_file(pjob, StdErr, O_WRONLY | O_APPEND,
pjob->ji_qs.ji_un.ji_momt.ji_exgid);
break;
}
}
if (!deletejob)
if ((fds1 < 0) ||
(fds2 < 0))
{
if (fds1 >= 0)
close(fds1);
if (fds2 >= 0)
close(fds2);
exit(-1);
}
if (pe_io_type != PE_IO_TYPE_ASIS)
{
/* If PE_IO_TYPE_ASIS, leave as is, already open to job */
if (fds1 != 1)
{
close(1);
if (dup(fds1) >= 0)
{
close(fds1);
}
}
if (fds2 != 2)
{
close(2);
if (dup(fds2) >= 0)
{
close(fds2);
}
}
}
if ((which == PE_PROLOGUSER) ||
(which == PE_EPILOGUSER) ||
(which == PE_PROLOGUSERJOB) ||
(which == PE_EPILOGUSERJOB))
{
if (chdir(pjob->ji_grpcache->gc_homedir) != 0)
{
/* warn only, no failure */
sprintf(log_buffer,
"PBS: chdir to %s failed: %s (running user %s in current directory)",
pjob->ji_grpcache->gc_homedir,
strerror(errno),
which == PE_PROLOGUSER ? "prologue" : "epilogue");
if (write_ac_socket(2, log_buffer, strlen(log_buffer)) == -1) {}
fsync(2);
}
}
/* for both prolog and epilog */
if (DEBUGMODE == 1)
{
fprintf(stderr, "PELOGINFO: script:'%s' jobid:'%s' euser:'******' egroup:'%s' jobname:'%s' SSID:'%ld' RESC:'%s'\n",
pelog,
pjob->ji_qs.ji_jobid,
pjob->ji_wattr[JOB_ATR_euser].at_val.at_str,
pjob->ji_wattr[JOB_ATR_egroup].at_val.at_str,
pjob->ji_wattr[JOB_ATR_jobname].at_val.at_str,
pjob->ji_wattr[JOB_ATR_session_id].at_val.at_long,
resc_to_string(pjob, JOB_ATR_resource, resc_list, sizeof(resc_list)));
}
arg[0] = pelog;
arg[1] = pjob->ji_qs.ji_jobid;
arg[2] = pjob->ji_wattr[JOB_ATR_euser].at_val.at_str;
arg[3] = pjob->ji_wattr[JOB_ATR_egroup].at_val.at_str;
arg[4] = pjob->ji_wattr[JOB_ATR_jobname].at_val.at_str;
/* NOTE: inside child */
if ((which == PE_EPILOG) ||
(which == PE_EPILOGUSER) ||
(which == PE_EPILOGUSERJOB))
{
/* for epilog only */
sprintf(sid, "%ld",
pjob->ji_wattr[JOB_ATR_session_id].at_val.at_long);
sprintf(exit_stat,"%d",
pjob->ji_qs.ji_un.ji_momt.ji_exitstat);
arg[5] = sid;
arg[6] = resc_to_string(pjob, JOB_ATR_resource, resc_list, sizeof(resc_list));
arg[7] = resc_to_string(pjob, JOB_ATR_resc_used, resc_used, sizeof(resc_used));
arg[8] = pjob->ji_wattr[JOB_ATR_in_queue].at_val.at_str;
arg[9] = pjob->ji_wattr[JOB_ATR_account].at_val.at_str;
arg[10] = exit_stat;
arg[11] = NULL;
LastArg = 11;
}
else
{
/* prolog */
arg[5] = resc_to_string(pjob, JOB_ATR_resource, resc_list, sizeof(resc_list));
arg[6] = pjob->ji_wattr[JOB_ATR_in_queue].at_val.at_str;
arg[7] = pjob->ji_wattr[JOB_ATR_account].at_val.at_str;
arg[8] = NULL;
LastArg = 8;
}
for (aindex = 0;aindex < LastArg;aindex++)
{
if (arg[aindex] == NULL)
arg[aindex] = EmptyString;
} /* END for (aindex) */
/*
* Pass Resource_List.nodes request in environment
* to allow pro/epi-logue setup/teardown of system
* settings. --pw, 2 Jan 02
* Fixed to use putenv for sysV compatibility.
* --troy, 11 jun 03
*
*/
r = find_resc_entry(
&pjob->ji_wattr[JOB_ATR_resource],
find_resc_def(svr_resc_def, (char *)"nodes", svr_resc_size));
if (r != NULL)
{
/* setenv("PBS_RESOURCE_NODES",r->rs_value.at_val.at_str,1); */
const char *ppn_str = "ppn=";
int num_nodes = 1;
int num_ppn = 1;
/* PBS_RESOURCE_NODES */
put_env_var("PBS_RESOURCE_NODES", r->rs_value.at_val.at_str);
/* PBS_NUM_NODES */
num_nodes = strtol(r->rs_value.at_val.at_str, NULL, 10);
/*
* InitUserEnv() also calculates num_nodes and num_ppn the same way
*/
if (num_nodes != 0)
{
char *tmp;
char *other_reqs;
/* get the ppn */
if ((tmp = strstr(r->rs_value.at_val.at_str,ppn_str)) != NULL)
{
tmp += strlen(ppn_str);
num_ppn = strtol(tmp, NULL, 10);
}
other_reqs = r->rs_value.at_val.at_str;
while ((other_reqs = strchr(other_reqs, '+')) != NULL)
{
other_reqs += 1;
num_nodes += strtol(other_reqs, &other_reqs, 10);
}
}
sprintf(buf, "%d", num_nodes);
put_env_var("PBS_NUM_NODES", buf);
/* PBS_NUM_PPN */
sprintf(buf, "%d", num_ppn);
put_env_var("PBS_NUM_PPN", buf);
/* PBS_NP */
sprintf(buf, "%d", pjob->ji_numvnod);
put_env_var("PBS_NP", buf);
} /* END if (r != NULL) */
r = find_resc_entry(
&pjob->ji_wattr[JOB_ATR_resource],
find_resc_def(svr_resc_def, (char *)"gres", svr_resc_size));
if (r != NULL)
{
/* setenv("PBS_RESOURCE_NODES",r->rs_value.at_val.at_str,1); */
put_env_var("PBS_RESOURCE_GRES", r->rs_value.at_val.at_str);
}
if (TTmpDirName(pjob, buf, sizeof(buf)))
{
put_env_var("TMPDIR", buf);
}
/* Set PBS_SCHED_HINT */
{
char *envname = (char *)"PBS_SCHED_HINT";
char *envval;
if ((envval = get_job_envvar(pjob, envname)) != NULL)
{
put_env_var("PBS_SCHED_HINT", envval);
}
}
/* Set PBS_NODENUM */
sprintf(buf, "%d",
pjob->ji_nodeid);
put_env_var("PBS_NODENUM", buf);
/* Set PBS_MSHOST */
put_env_var("PBS_MSHOST", pjob->ji_vnods[0].vn_host->hn_host);
/* Set PBS_NODEFILE */
if (pjob->ji_flags & MOM_HAS_NODEFILE)
{
sprintf(buf, "%s/%s",
path_aux,
pjob->ji_qs.ji_jobid);
put_env_var("PBS_NODEFILE", buf);
}
/* Set PBS_O_WORKDIR */
{
char *workdir_val;
workdir_val = get_job_envvar(pjob,"PBS_O_WORKDIR");
if (workdir_val != NULL)
{
put_env_var("PBS_O_WORKDIR", workdir_val);
}
}
/* SET BEOWULF_JOB_MAP */
{
struct array_strings *vstrs;
int VarIsSet = 0;
int j;
vstrs = pjob->ji_wattr[JOB_ATR_variables].at_val.at_arst;
for (j = 0;j < vstrs->as_usedptr;++j)
{
if (!strncmp(
vstrs->as_string[j],
"BEOWULF_JOB_MAP=",
strlen("BEOWULF_JOB_MAP=")))
{
VarIsSet = 1;
break;
}
}
if (VarIsSet == 1)
{
char *val = strchr(vstrs->as_string[j], '=');
if (val != NULL)
put_env_var("BEOWULF_JOB_MAP", val+1);
}
}
/* Set some Moab env variables if they exist */
if ((which == PE_PROLOG) || (which == PE_EPILOG))
{
char *tmp_val;
for (aindex=0;aindex<moabenvcnt;aindex++)
{
tmp_val = get_job_envvar(pjob,moabenvs[aindex]);
if (tmp_val != NULL)
{
put_env_var(moabenvs[aindex], tmp_val);
}
}
}
/*
* if we want to run as user then we need to reset real user permissions
* since it seems that some OSs use real not effective user id when execv'ing
*/
if ((which == PE_PROLOGUSER) ||
(which == PE_EPILOGUSER) ||
(which == PE_PROLOGUSERJOB) ||
(which == PE_EPILOGUSERJOB))
{
setuid_ext(pbsuser, TRUE);
setegid(pbsgroup);
if (setgid(pjob->ji_qs.ji_un.ji_momt.ji_exgid) != 0)
{
snprintf(log_buffer,sizeof(log_buffer),
"setgid(%lu) for UID = %lu failed: %s\n",
(unsigned long)pjob->ji_qs.ji_un.ji_momt.ji_exgid,
(unsigned long)pjob->ji_qs.ji_un.ji_momt.ji_exuid,
strerror(errno));
log_err(errno, __func__, log_buffer);
exit(-1);
}
if (setuid_ext(pjob->ji_qs.ji_un.ji_momt.ji_exuid, FALSE) != 0)
{
snprintf(log_buffer,sizeof(log_buffer),
"setuid(%lu) failed: %s\n",
(unsigned long)pjob->ji_qs.ji_un.ji_momt.ji_exuid,
strerror(errno));
log_err(errno, __func__, log_buffer);
exit(-1);
}
}
execv(pelog,arg);
sprintf(log_buffer,"execv of %s failed: %s\n",
pelog,
strerror(errno));
if (write_ac_socket(2, log_buffer, strlen(log_buffer)) == -1)
{
/* cannot write message to stderr */
/* NO-OP */
}
fsync(2);
exit(255);
} /* END else () */
switch (run_exit)
{
case 0:
/* SUCCESS */
/* NO-OP */
break;
case - 3:
pelog_err(pjob, pelog, run_exit, (char *)"child wait interrupted");
break;
case - 4:
pelog_err(pjob, pelog, run_exit, (char *)"prolog/epilog timeout occurred, child cleaned up");
break;
case - 5:
pelog_err(pjob, pelog, run_exit, (char *) "prolog/epilog timeout occurred, cannot kill child");
break;
default:
pelog_err(pjob, pelog, run_exit, (char *)"nonzero p/e exit status");
break;
} /* END switch (run_exit) */
return(run_exit);
} /* END run_pelog() */
void req_mvjobfile(
struct batch_request *preq) /* I */
{
int fds;
enum job_file jft;
int oflag;
job *pj;
struct passwd *pwd;
jft = (enum job_file)preq->rq_ind.rq_jobfile.rq_type;
if (preq->rq_ind.rq_jobfile.rq_sequence == 0)
oflag = O_CREAT | O_WRONLY | O_TRUNC;
else
oflag = O_CREAT | O_WRONLY | O_APPEND;
pj = locate_new_job(preq->rq_conn, NULL);
if (pj == NULL)
pj = mom_find_job(preq->rq_ind.rq_jobfile.rq_jobid);
if (pj == NULL)
{
snprintf(log_buffer, 1024, "cannot find job %s for move of %s file",
preq->rq_ind.rq_jobfile.rq_jobid,
TJobFileType[jft]);
log_err(-1, __func__, log_buffer);
req_reject(PBSE_UNKJOBID, 0, preq, NULL, NULL);
return;
}
if ((pj->ji_grpcache == NULL) &&
(check_pwd(pj) == NULL))
{
req_reject(PBSE_UNKJOBID, 0, preq, NULL, NULL);
return;
}
if ((pwd = getpwnam_ext(pj->ji_wattr[JOB_ATR_euser].at_val.at_str)) == NULL)
{
/* FAILURE */
req_reject(PBSE_MOMREJECT, 0, preq, NULL, "password lookup failed");
return;
}
if ((fds = open_std_file(pj, jft, oflag, pwd->pw_gid)) < 0)
{
int keeping = 1;
char *path = std_file_name(pj, jft, &keeping);
snprintf(log_buffer,sizeof(log_buffer),
"Cannot create file %s",
path);
req_reject(PBSE_SYSTEM, 0, preq, NULL, log_buffer);
return;
}
if (write_ac_socket(
fds,
preq->rq_ind.rq_jobfile.rq_data,
preq->rq_ind.rq_jobfile.rq_size) != preq->rq_ind.rq_jobfile.rq_size)
{
req_reject(PBSE_SYSTEM, 0, preq, NULL, "cannot create file");
}
else
{
if (LOGLEVEL >= 6)
{
sprintf(log_buffer, "successfully moved %s file for job '%s'",
TJobFileType[jft],
preq->rq_ind.rq_jobfile.rq_jobid);
log_record(
PBSEVENT_JOB,
PBS_EVENTCLASS_JOB,
pj->ji_qs.ji_jobid,
log_buffer);
}
reply_ack(preq);
}
close(fds);
return;
} /* END req_mvjobfile() */
void req_jobscript(
struct batch_request *preq) /* ptr to the decoded request*/
{
int fds;
char namebuf[MAXPATHLEN];
job *pj;
int filemode = 0700;
errno = 0;
pj = locate_new_job(preq->rq_conn, preq->rq_ind.rq_jobfile.rq_jobid);
if (pj == NULL)
{
log_err(errno, __func__, (char *)"cannot locate new job");
req_reject(PBSE_IVALREQ, 0, preq, NULL, NULL);
return;
}
/* what is the difference between JOB_SUBSTATE_TRANSIN and TRANSICM? */
if (pj->ji_qs.ji_substate != JOB_SUBSTATE_TRANSIN)
{
if (errno == 0)
{
sprintf(log_buffer, "job %s in unexpected state '%s'",
pj->ji_qs.ji_jobid,
PJobSubState[pj->ji_qs.ji_substate]);
}
else
{
sprintf(log_buffer, "job %s in unexpected state '%s' (errno=%d - %s)",
pj->ji_qs.ji_jobid,
PJobSubState[pj->ji_qs.ji_substate],
errno,
strerror(errno));
}
log_err(errno, __func__, log_buffer);
req_reject(PBSE_IVALREQ, 0, preq, mom_host, log_buffer);
return;
}
/* mom - if job has been checkpointed, discard script,already have it */
if (pj->ji_qs.ji_svrflags & JOB_SVFLG_CHECKPOINT_FILE)
{
/* SUCCESS - do nothing, ignore script */
reply_ack(preq);
return;
}
if (multi_mom)
{
snprintf(namebuf, sizeof(namebuf), "%s%s%d%s",
path_jobs, pj->ji_qs.ji_fileprefix, pbs_rm_port, JOB_SCRIPT_SUFFIX);
}
else
{
snprintf(namebuf, sizeof(namebuf), "%s%s%s",
path_jobs, pj->ji_qs.ji_fileprefix, JOB_SCRIPT_SUFFIX);
}
if (pj->ji_qs.ji_un.ji_newt.ji_scriptsz == 0)
{
/* NOTE: fail is job script already exists */
fds = open(namebuf, O_WRONLY | O_CREAT | O_EXCL | O_Sync, filemode);
}
else
{
fds = open(namebuf, O_WRONLY | O_APPEND | O_Sync, filemode);
}
if (fds < 0)
{
char tmpLine[1024];
snprintf(tmpLine, sizeof(tmpLine), "cannot open '%s' errno=%d - %s",
namebuf, errno, strerror(errno));
/* FAILURE */
/* NOTE: log_err may modify errno */
log_err(errno, __func__, msg_script_open);
req_reject(PBSE_INTERNAL, 0, preq, mom_host, tmpLine);
return;
}
if (write_ac_socket(
fds,
preq->rq_ind.rq_jobfile.rq_data,
(unsigned)preq->rq_ind.rq_jobfile.rq_size) != preq->rq_ind.rq_jobfile.rq_size)
{
/* FAILURE */
log_err(errno, __func__, msg_script_write);
req_reject(PBSE_INTERNAL, 0, preq, mom_host, "cannot write job command file");
close(fds);
return;
}
close(fds);
pj->ji_qs.ji_un.ji_newt.ji_scriptsz += preq->rq_ind.rq_jobfile.rq_size;
/* job has a script file */
pj->ji_qs.ji_svrflags =
(pj->ji_qs.ji_svrflags & ~JOB_SVFLG_CHECKPOINT_FILE) | JOB_SVFLG_SCRIPT;
/* SUCCESS */
reply_ack(preq);
return;
} /* END req_jobscript() */
int save_acl(
pbs_attribute *attr, /* acl pbs_attribute */
attribute_def *pdef, /* pbs_attribute def structure */
const char *subdir, /* sub-directory path */
const char *name) /* parent object name = file name */
{
static const char *this_function_name = "save_acl";
int fds;
char filename1[MAXPATHLEN];
char filename2[MAXPATHLEN];
tlist_head head;
int i;
svrattrl *pentry;
char log_buf[LOCAL_LOG_BUF_SIZE];
if ((attr->at_flags & ATR_VFLAG_MODIFY) == 0)
{
return(0); /* Not modified, don't bother */
}
attr->at_flags &= ~ATR_VFLAG_MODIFY;
snprintf(filename1, sizeof(filename1), "%s%s/%s",
path_priv, subdir, name);
if ((attr->at_flags & ATR_VFLAG_SET) == 0)
{
/* has been unset, delete the file */
unlink(filename1);
return(0);
}
snprintf(filename2, sizeof(filename2), "%s.new",
filename1);
fds = open(filename2, O_WRONLY | O_CREAT | O_TRUNC | O_Sync, 0600);
if (fds < 0)
{
snprintf(log_buf, sizeof(log_buf), "unable to open acl file '%s'", filename2);
log_err(errno, this_function_name, log_buf);
return(-1);
}
CLEAR_HEAD(head);
i = pdef->at_encode(attr, &head, pdef->at_name, (char *)0, ATR_ENCODE_SAVE, ATR_DFLAG_ACCESS);
if (i < 0)
{
log_err(-1, this_function_name, (char *)"unable to encode acl");
close(fds);
unlink(filename2);
return(-1);
}
pentry = (svrattrl *)GET_NEXT(head);
if (pentry != NULL)
{
/* write entry, but without terminating null */
while ((i = write_ac_socket(fds, pentry->al_value, pentry->al_valln - 1)) != pentry->al_valln - 1)
{
if ((i == -1) && (errno == EINTR))
continue;
log_err(errno, this_function_name, (char *)"wrote incorrect amount");
close(fds);
unlink(filename2);
return(-1);
}
free(pentry);
}
close(fds);
unlink(filename1);
if (link(filename2, filename1) < 0)
{
log_err(errno, this_function_name, (char *)"unable to relink file");
return(-1);
}
unlink(filename2);
attr->at_flags &= ~ATR_VFLAG_MODIFY; /* clear modified flag */
return(0);
}
int job_save(
job *pjob, /* pointer to job structure */
int updatetype, /* 0=quick, 1=full, 2=new */
int mom_port) /* if 0 ignore otherwise append to end of job name. this is for multi-mom mode */
{
int fds;
int i;
char namebuf1[MAXPATHLEN];
char namebuf2[MAXPATHLEN];
char save_buf[SAVEJOB_BUF_SIZE];
const char *tmp_ptr = NULL;
size_t buf_remaining = sizeof(save_buf);
int openflags;
int redo;
time_t time_now = time(NULL);
#ifdef PBS_MOM
tmp_ptr = JOB_FILE_SUFFIX;
#else
if (pjob->ji_is_array_template == TRUE)
tmp_ptr = (char *)JOB_FILE_TMP_SUFFIX;
else
tmp_ptr = (char *)JOB_FILE_SUFFIX;
#endif
if (mom_port)
{
snprintf(namebuf1, MAXPATHLEN, "%s%s%d%s",
path_jobs, pjob->ji_qs.ji_fileprefix, mom_port, tmp_ptr);
snprintf(namebuf2, MAXPATHLEN, "%s%s%d%s",
path_jobs, pjob->ji_qs.ji_fileprefix, mom_port, JOB_FILE_COPY);
}
else
{
snprintf(namebuf1, MAXPATHLEN, "%s%s%s",
path_jobs, pjob->ji_qs.ji_fileprefix, tmp_ptr);
snprintf(namebuf2, MAXPATHLEN, "%s%s%s",
path_jobs, pjob->ji_qs.ji_fileprefix, JOB_FILE_COPY);
}
/* if ji_modified is set, ie an pbs_attribute changed, then update mtime */
if (pjob->ji_modified)
{
pjob->ji_wattr[JOB_ATR_mtime].at_val.at_long = time_now;
}
if (updatetype == SAVEJOB_QUICK)
{
openflags = O_WRONLY | O_Sync;
/* NOTE: open, do not create */
fds = open(namebuf1, openflags, 0600);
if (fds < 0)
{
char tmpLine[1024];
snprintf(tmpLine, sizeof(tmpLine), "cannot open file '%s' for job %s in state %s (%s)",
namebuf1,
pjob->ji_qs.ji_jobid,
PJobSubState[MAX(0,pjob->ji_qs.ji_substate)],
(updatetype == 0) ? "quick" : "full");
log_err(errno, "job_save", tmpLine);
/* FAILURE */
return(-1);
}
/* just write the "critical" base structure to the file */
while ((i = write_ac_socket(fds, (char *)&pjob->ji_qs, sizeof(pjob->ji_qs))) != sizeof(pjob->ji_qs))
{
if ((i < 0) && (errno == EINTR))
{
/* retry the write */
if (lseek(fds, (off_t)0, SEEK_SET) < 0)
{
log_err(errno, "job_save", (char *)"lseek");
close(fds);
return(-1);
}
continue;
}
else
{
log_err(errno, "job_save", (char *)"quickwrite");
close(fds);
/* FAILURE */
return(-1);
}
}
close(fds);
}
else /* SAVEJOB_FULL, SAVEJOB_NEW, SAVEJOB_ARY */
{
/*
* write the whole structure to the file.
* For a update, this is done to a new file to protect the
* old against crashs.
* The file is written in four parts:
* (1) the job structure,
* (2) the attribtes in "encoded" form,
* (3) the attributes in the "external" form, and last
* (4) the dependency list.
*/
openflags = O_CREAT | O_WRONLY | O_Sync;
/* NOTE: create file if required */
if (updatetype == SAVEJOB_NEW)
fds = open(namebuf1, openflags, 0600);
else
fds = open(namebuf2, openflags, 0600);
if (fds < 0)
{
log_err(errno, "job_save", (char *)"open for full save");
return(-1);
}
for (i = 0; i < MAX_SAVE_TRIES; i++)
{
redo = 0; /* try to save twice */
#ifndef PBS_MOM
lock_ss();
#endif
if (save_struct((char *)&pjob->ji_qs, sizeof(pjob->ji_qs), fds, save_buf, &buf_remaining, sizeof(save_buf)) != PBSE_NONE)
{
redo++;
}
else if (save_attr(job_attr_def,
pjob->ji_wattr,
JOB_ATR_LAST,
fds,
save_buf,
&buf_remaining,
sizeof(save_buf)) != PBSE_NONE)
{
redo++;
}
#ifdef PBS_MOM
else if (save_tmsock(pjob, fds, save_buf, &buf_remaining, sizeof(save_buf)) != PBSE_NONE)
{
redo++;
}
#endif /* PBS_MOM */
else if (write_buffer(save_buf, sizeof(save_buf) - buf_remaining, fds) != PBSE_NONE)
{
redo++;
}
#ifndef PBS_MOM
unlock_ss();
#endif
if (redo != 0)
{
if (lseek(fds, (off_t)0, SEEK_SET) < 0)
{
log_err(errno, "job_save", (char *)"full lseek");
}
}
else
{
break;
}
} /* END for (i) */
close(fds);
if (i >= MAX_SAVE_TRIES)
{
if (updatetype == SAVEJOB_FULL)
unlink(namebuf2);
return(-1);
}
if (updatetype == SAVEJOB_FULL)
{
unlink(namebuf1);
if (link(namebuf2, namebuf1) == -1)
{
log_event(
PBSEVENT_ERROR | PBSEVENT_SECURITY,
PBS_EVENTCLASS_JOB,
pjob->ji_qs.ji_jobid,
(char *)"Link in job_save failed");
}
else
{
unlink(namebuf2);
}
}
pjob->ji_modified = 0;
} /* END (updatetype == SAVEJOB_QUICK) */
return(PBSE_NONE);
} /* END job_save() */
void prepare_and_run_pelog_as_child(
job *pjob,
int pe_io_type,
int delete_job,
char *specpelog,
char *pelog,
int which,
int parent_read,
int parent_write,
int kid_read,
int kid_write,
int fd_input)
{
char *arg[12];
handle_pipes_as_child(parent_read, parent_write, kid_read, kid_write);
close_handles_as_child();
// setup stdin for the pelog
if (fd_input != 0)
{
close(0);
if (dup(fd_input) == -1) {}
close(fd_input);
}
setup_pelog_outputs(pjob, pe_io_type, delete_job, specpelog);
change_directory_as_needed(which, pjob);
/* for both prolog and epilog */
if (DEBUGMODE == 1)
{
char resc_list[2048];
fprintf(stderr, "PELOGINFO: script:'%s' jobid:'%s' euser:'******' egroup:'%s' jobname:'%s' SSID:'%ld' RESC:'%s'\n",
pelog,
pjob->ji_qs.ji_jobid,
pjob->ji_wattr[JOB_ATR_euser].at_val.at_str,
pjob->ji_wattr[JOB_ATR_egroup].at_val.at_str,
pjob->ji_wattr[JOB_ATR_jobname].at_val.at_str,
pjob->ji_wattr[JOB_ATR_session_id].at_val.at_long,
resc_to_string(pjob, JOB_ATR_resource, resc_list, sizeof(resc_list)));
}
setup_pelog_arguments(pelog, pjob, which, arg);
setup_pelog_environment(pjob, which);
/*
* if we want to run as user then we need to reset real user permissions
* since it seems that some OSs use real not effective user id when execv'ing
*/
if ((which == PE_PROLOGUSER) ||
(which == PE_EPILOGUSER) ||
(which == PE_PROLOGUSERJOB) ||
(which == PE_EPILOGUSERJOB))
{
setuid_ext(pbsuser, TRUE);
setegid(pbsgroup);
if (become_the_user(pjob) != PBSE_NONE)
{
exit(-1);
}
}
execv(pelog,arg);
/* should not be reached, but clean up if error */
sprintf(log_buffer,"execv of %s failed: %s\n",
pelog,
strerror(errno));
if (write_ac_socket(2, log_buffer, strlen(log_buffer)) == -1)
{
/* cannot write message to stderr */
/* NO-OP */
}
fsync(2);
exit(255);
} /* END prepare_and_run_pelog_as_child() */
