static int
_handle_terminate(int fd, slurmd_job_t *job, uid_t uid)
{
int rc = SLURM_SUCCESS;
int errnum = 0;
debug("_handle_terminate for job %u.%u",
job->jobid, job->stepid);
step_terminate_monitor_start(job->jobid, job->stepid);
debug3(" uid = %d", uid);
if (uid != job->uid && !_slurm_authorized_user(uid)) {
debug("terminate req from uid %ld for job %u.%u "
"owned by uid %ld",
(long)uid, job->jobid, job->stepid, (long)job->uid);
rc = -1;
errnum = EPERM;
goto done;
}
/*
* Sanity checks
*/
if (job->cont_id == 0) {
debug ("step %u.%u invalid container [cont_id:%"PRIu64"]",
job->jobid, job->stepid, job->cont_id);
rc = -1;
errnum = ESLURMD_JOB_NOTRUNNING;
goto done;
}
/*
* Signal the container with SIGKILL
*/
pthread_mutex_lock(&suspend_mutex);
if (suspended) {
debug("Terminating suspended job step %u.%u",
job->jobid, job->stepid);
}
if (slurm_container_signal(job->cont_id, SIGKILL) < 0) {
rc = -1;
errnum = errno;
verbose("Error sending SIGKILL signal to %u.%u: %m",
job->jobid, job->stepid);
} else {
verbose("Sent SIGKILL signal to %u.%u",
job->jobid, job->stepid);
}
pthread_mutex_unlock(&suspend_mutex);
done:
/* Send the return code and errnum */
safe_write(fd, &rc, sizeof(int));
safe_write(fd, &errnum, sizeof(int));
return SLURM_SUCCESS;
rwfail:
return SLURM_FAILURE;
}
static int
_handle_resume(int fd, slurmd_job_t *job, uid_t uid)
{
int rc = SLURM_SUCCESS;
int errnum = 0;
debug("_handle_resume for job %u.%u",
job->jobid, job->stepid);
debug3(" uid = %d", uid);
if (!_slurm_authorized_user(uid)) {
debug("job step resume request from uid %ld for job %u.%u ",
(long)uid, job->jobid, job->stepid);
rc = -1;
errnum = EPERM;
goto done;
}
if (job->cont_id == 0) {
debug ("step %u.%u invalid container [cont_id:%"PRIu64"]",
job->jobid, job->stepid, job->cont_id);
rc = -1;
errnum = ESLURMD_JOB_NOTRUNNING;
goto done;
}
jobacct_gather_g_resume_poll();
/*
* Signal the container
*/
pthread_mutex_lock(&suspend_mutex);
if (!suspended) {
rc = -1;
errnum = ESLURMD_STEP_NOTSUSPENDED;
pthread_mutex_unlock(&suspend_mutex);
goto done;
} else {
if (slurm_container_signal(job->cont_id, SIGCONT) < 0) {
verbose("Error resuming %u.%u: %m",
job->jobid, job->stepid);
} else {
verbose("Resumed %u.%u", job->jobid, job->stepid);
}
suspended = false;
}
pthread_mutex_unlock(&suspend_mutex);
done:
/* Send the return code and errno */
safe_write(fd, &rc, sizeof(int));
safe_write(fd, &errnum, sizeof(int));
return SLURM_SUCCESS;
rwfail:
return SLURM_FAILURE;
}
static int
_handle_signal_container(int fd, slurmd_job_t *job, uid_t uid)
{
int rc = SLURM_SUCCESS;
int errnum = 0;
int sig;
static int msg_sent = 0;
debug("_handle_signal_container for job %u.%u",
job->jobid, job->stepid);
safe_read(fd, &sig, sizeof(int));
debug3(" uid = %d", uid);
if (uid != job->uid && !_slurm_authorized_user(uid)) {
debug("kill container req from uid %ld for job %u.%u "
"owned by uid %ld",
(long)uid, job->jobid, job->stepid, (long)job->uid);
rc = -1;
errnum = EPERM;
goto done;
}
/*
* Sanity checks
*/
if (job->cont_id == 0) {
debug ("step %u.%u invalid container [cont_id:%"PRIu64"]",
job->jobid, job->stepid, job->cont_id);
rc = -1;
errnum = ESLURMD_JOB_NOTRUNNING;
goto done;
}
if ((job->nodeid == 0) && (msg_sent == 0) &&
(job->state < SLURMSTEPD_STEP_ENDING)) {
time_t now = time(NULL);
char entity[24], time_str[24];
if (job->stepid == SLURM_BATCH_SCRIPT) {
snprintf(entity, sizeof(entity), "JOB %u", job->jobid);
} else {
snprintf(entity, sizeof(entity), "STEP %u.%u",
job->jobid, job->stepid);
}
slurm_make_time_str(&now, time_str, sizeof(time_str));
/* Not really errors,
* but we want messages displayed by default */
if (sig == SIG_TIME_LIMIT) {
error("*** %s CANCELLED AT %s DUE TO TIME LIMIT ***",
entity, time_str);
msg_sent = 1;
} else if (sig == SIG_PREEMPTED) {
error("*** %s CANCELLED AT %s DUE TO PREEMPTION ***",
entity, time_str);
msg_sent = 1;
} else if (sig == SIG_NODE_FAIL) {
error("*** %s CANCELLED AT %s DUE TO NODE FAILURE ***",
entity, time_str);
msg_sent = 1;
} else if (sig == SIG_FAILURE) {
error("*** %s FAILED (non-zero exit code or other "
"failure mode) ***", entity);
msg_sent = 1;
} else if ((sig == SIGTERM) || (sig == SIGKILL)) {
error("*** %s CANCELLED AT %s ***", entity, time_str);
msg_sent = 1;
}
}
if ((sig == SIG_TIME_LIMIT) || (sig == SIG_NODE_FAIL) ||
(sig == SIG_PREEMPTED) || (sig == SIG_FAILURE))
goto done;
if (sig == SIG_DEBUG_WAKE) {
int i;
for (i = 0; i < job->node_tasks; i++)
pdebug_wake_process(job, job->task[i]->pid);
goto done;
}
if (sig == SIG_ABORT) {
sig = SIGKILL;
job->aborted = true;
}
pthread_mutex_lock(&suspend_mutex);
if (suspended && (sig != SIGKILL)) {
rc = -1;
errnum = ESLURMD_STEP_SUSPENDED;
pthread_mutex_unlock(&suspend_mutex);
goto done;
}
/*
* Signal the container
*/
if (slurm_container_signal(job->cont_id, sig) < 0) {
rc = -1;
errnum = errno;
verbose("Error sending signal %d to %u.%u: %m",
sig, job->jobid, job->stepid);
} else {
verbose("Sent signal %d to %u.%u",
sig, job->jobid, job->stepid);
}
pthread_mutex_unlock(&suspend_mutex);
done:
/* Send the return code and errnum */
safe_write(fd, &rc, sizeof(int));
safe_write(fd, &errnum, sizeof(int));
return SLURM_SUCCESS;
rwfail:
return SLURM_FAILURE;
}
static int
_handle_suspend(int fd, slurmd_job_t *job, uid_t uid)
{
int rc = SLURM_SUCCESS;
int errnum = 0;
debug("_handle_suspend for job %u.%u",
job->jobid, job->stepid);
debug3(" uid = %d", uid);
if (!_slurm_authorized_user(uid)) {
debug("job step suspend request from uid %ld for job %u.%u ",
(long)uid, job->jobid, job->stepid);
rc = -1;
errnum = EPERM;
goto done;
}
if (job->cont_id == 0) {
debug ("step %u.%u invalid container [cont_id:%"PRIu64"]",
job->jobid, job->stepid, job->cont_id);
rc = -1;
errnum = ESLURMD_JOB_NOTRUNNING;
goto done;
}
jobacct_gather_g_suspend_poll();
/*
* Signal the container
*/
pthread_mutex_lock(&suspend_mutex);
if (suspended) {
rc = -1;
errnum = ESLURMD_STEP_SUSPENDED;
pthread_mutex_unlock(&suspend_mutex);
goto done;
} else {
/* SIGTSTP is sent first to let MPI daemons stop their
* tasks, then we send SIGSTOP to stop everything else */
if (slurm_container_signal(job->cont_id, SIGTSTP) < 0) {
verbose("Error suspending %u.%u (SIGTSTP): %m",
job->jobid, job->stepid);
} else
sleep(1);
if (slurm_container_signal(job->cont_id, SIGSTOP) < 0) {
verbose("Error suspending %u.%u (SIGSTOP): %m",
job->jobid, job->stepid);
} else {
verbose("Suspended %u.%u", job->jobid, job->stepid);
}
suspended = true;
}
pthread_mutex_unlock(&suspend_mutex);
done:
/* Send the return code and errno */
safe_write(fd, &rc, sizeof(int));
safe_write(fd, &errnum, sizeof(int));
return SLURM_SUCCESS;
rwfail:
return SLURM_FAILURE;
}
static int
_handle_suspend(int fd, slurmd_job_t *job, uid_t uid)
{
int rc = SLURM_SUCCESS;
int errnum = 0;
debug("_handle_suspend for job %u.%u",
job->jobid, job->stepid);
debug3(" uid = %d", uid);
if (!_slurm_authorized_user(uid)) {
debug("job step suspend request from uid %ld for job %u.%u ",
(long)uid, job->jobid, job->stepid);
rc = -1;
errnum = EPERM;
goto done;
}
if (job->cont_id == 0) {
debug ("step %u.%u invalid container [cont_id:%"PRIu64"]",
job->jobid, job->stepid, job->cont_id);
rc = -1;
errnum = ESLURMD_JOB_NOTRUNNING;
goto done;
}
jobacct_gather_g_suspend_poll();
/*
* Signal the container
*/
pthread_mutex_lock(&suspend_mutex);
if (suspended) {
rc = -1;
errnum = ESLURMD_STEP_SUSPENDED;
pthread_mutex_unlock(&suspend_mutex);
goto done;
} else {
/* SIGTSTP is sent first to let MPI daemons stop their tasks,
* then wait 2 seconds, then send SIGSTOP to the spawned
* process's container to stop everything else.
*
* In some cases, 1 second has proven insufficient. Longer
* delays may help insure that all MPI tasks have been stopped
* (that depends upon the MPI implementaiton used), but will
* also permit longer time periods when more than one job can
* be running on each resource (not good). */
if (slurm_container_signal(job->cont_id, SIGTSTP) < 0) {
verbose("Error suspending %u.%u (SIGTSTP): %m",
job->jobid, job->stepid);
} else
sleep(2);
if (slurm_container_signal(job->cont_id, SIGSTOP) < 0) {
verbose("Error suspending %u.%u (SIGSTOP): %m",
job->jobid, job->stepid);
} else {
verbose("Suspended %u.%u", job->jobid, job->stepid);
}
suspended = true;
}
pthread_mutex_unlock(&suspend_mutex);
done:
/* Send the return code and errno */
safe_write(fd, &rc, sizeof(int));
safe_write(fd, &errnum, sizeof(int));
return SLURM_SUCCESS;
rwfail:
return SLURM_FAILURE;
}
