// Take all of the scheduled job events that were queued up while we
// weren't running and add those jobs to the set of running jobs. This
// also requires switching their state in the kvs (to trigger events
// in the scheduler)
static int handle_queued_events (ctx_t *ctx)
{
job_t *job = NULL;
int *jobid = NULL;
flux_kvsdir_t *kvs_dir;
flux_t *h = ctx->h;
zlist_t *queued_events = ctx->queued_events;
zlist_t *running_jobs = ctx->running_jobs;
double sim_time = ctx->sim_state->sim_time;
while (zlist_size (queued_events) > 0) {
jobid = zlist_pop (queued_events);
if (!(kvs_dir = job_kvsdir (ctx->h, *jobid)))
log_err_exit ("job_kvsdir (id=%d)", *jobid);
job = pull_job_from_kvs (*jobid, kvs_dir);
if (update_job_state (ctx, *jobid, kvs_dir, J_STARTING, sim_time) < 0) {
flux_log (h,
LOG_ERR,
"failed to set job %d's state to starting",
*jobid);
return -1;
}
if (update_job_state (ctx, *jobid, kvs_dir, J_RUNNING, sim_time) < 0) {
flux_log (h,
LOG_ERR,
"failed to set job %d's state to running",
*jobid);
return -1;
}
if (update_job_state (ctx, *jobid, kvs_dir, J_COMPLETING, sim_time) < 0) {
flux_log (h,
LOG_ERR,
"failed to set job %d's state to completing",
*jobid);
return -1;
}
flux_log (h,
LOG_INFO,
"job %d's state to starting then running",
*jobid);
job->start_time = ctx->sim_state->sim_time;
zlist_append (running_jobs, job);
}
return 0;
}
// Update sched timer as necessary (to trigger an event in sched)
// Also change the state of the job in the KVS
static int complete_job (ctx_t *ctx, job_t *job, double completion_time)
{
flux_t *h = ctx->h;
int rc;
flux_log (h, LOG_INFO, "Job %d completed", job->id);
update_job_state (ctx, job->id, job->kvs_dir, J_COMPLETE, completion_time);
set_event_timer (ctx, "sched", ctx->sim_state->sim_time + .00001);
rc = set_job_timestamps (job->kvs_dir,
SIM_TIME_NONE, // starting
SIM_TIME_NONE, // running
SIM_TIME_NONE, // completing
completion_time,
job->io_time);
if (rc < 0)
flux_log_error (h, "%s: set_job_timestamps", __FUNCTION__);
free_job (job);
return rc;
}
void
job_force_termination(srun_job_t *job)
{
static int kill_sent = 0;
static time_t last_msg = 0;
if (kill_sent == 0) {
info("forcing job termination");
/* Sends SIGKILL to tasks directly */
update_job_state(job, SRUN_JOB_FORCETERM);
} else {
time_t now = time(NULL);
if (last_msg != now) {
info("job abort in progress");
last_msg = now;
}
if (kill_sent == 1) {
/* Try sending SIGKILL through slurmctld */
slurm_kill_job_step(job->jobid, job->stepid, SIGKILL);
}
}
kill_sent++;
}
extern int launch_p_step_launch(
srun_job_t *job, slurm_step_io_fds_t *cio_fds, uint32_t *global_rc,
slurm_step_launch_callbacks_t *step_callbacks)
{
slurm_step_launch_params_t launch_params;
slurm_step_launch_callbacks_t callbacks;
int rc = 0;
bool first_launch = 0;
slurm_step_launch_params_t_init(&launch_params);
memcpy(&callbacks, step_callbacks, sizeof(callbacks));
if (!task_state) {
task_state = task_state_create(job->ntasks);
local_srun_job = job;
local_global_rc = global_rc;
first_launch = 1;
} else
task_state_alter(task_state, job->ntasks);
launch_params.gid = opt.gid;
launch_params.alias_list = job->alias_list;
launch_params.argc = opt.argc;
launch_params.argv = opt.argv;
launch_params.multi_prog = opt.multi_prog ? true : false;
launch_params.cwd = opt.cwd;
launch_params.slurmd_debug = opt.slurmd_debug;
launch_params.buffered_stdio = !opt.unbuffered;
launch_params.labelio = opt.labelio ? true : false;
launch_params.remote_output_filename =fname_remote_string(job->ofname);
launch_params.remote_input_filename = fname_remote_string(job->ifname);
launch_params.remote_error_filename = fname_remote_string(job->efname);
launch_params.task_prolog = opt.task_prolog;
launch_params.task_epilog = opt.task_epilog;
launch_params.cpu_bind = opt.cpu_bind;
launch_params.cpu_bind_type = opt.cpu_bind_type;
launch_params.mem_bind = opt.mem_bind;
launch_params.mem_bind_type = opt.mem_bind_type;
launch_params.open_mode = opt.open_mode;
if (opt.acctg_freq >= 0)
launch_params.acctg_freq = opt.acctg_freq;
launch_params.pty = opt.pty;
if (opt.cpus_set)
launch_params.cpus_per_task = opt.cpus_per_task;
else
launch_params.cpus_per_task = 1;
launch_params.cpu_freq = opt.cpu_freq;
launch_params.task_dist = opt.distribution;
launch_params.ckpt_dir = opt.ckpt_dir;
launch_params.restart_dir = opt.restart_dir;
launch_params.preserve_env = opt.preserve_env;
launch_params.spank_job_env = opt.spank_job_env;
launch_params.spank_job_env_size = opt.spank_job_env_size;
launch_params.user_managed_io = opt.user_managed_io;
memcpy(&launch_params.local_fds, cio_fds, sizeof(slurm_step_io_fds_t));
if (MPIR_being_debugged) {
launch_params.parallel_debug = true;
pmi_server_max_threads(1);
} else {
launch_params.parallel_debug = false;
}
/* Normally this isn't used, but if an outside process (other
than srun (poe) is using this logic to launch tasks then we
can use this to signal the step.
*/
callbacks.task_start = _task_start;
/* If poe is using this code with multi-prog it always returns
1 for each task which could be confusing since no real
error happened.
*/
if (!launch_params.multi_prog
|| (!callbacks.step_signal
|| (callbacks.step_signal == launch_g_fwd_signal))) {
callbacks.task_finish = _task_finish;
}
mpir_init(job->ctx_params.task_count);
update_job_state(job, SRUN_JOB_LAUNCHING);
launch_start_time = time(NULL);
if (first_launch) {
if (slurm_step_launch(job->step_ctx, &launch_params,
&callbacks) != SLURM_SUCCESS) {
rc = errno;
*local_global_rc = errno;
error("Application launch failed: %m");
slurm_step_launch_abort(job->step_ctx);
slurm_step_launch_wait_finish(job->step_ctx);
goto cleanup;
}
} else {
if (slurm_step_launch_add(job->step_ctx, &launch_params,
job->nodelist, job->fir_nodeid)
!= SLURM_SUCCESS) {
rc = errno;
*local_global_rc = errno;
error("Application launch add failed: %m");
slurm_step_launch_abort(job->step_ctx);
slurm_step_launch_wait_finish(job->step_ctx);
goto cleanup;
}
}
update_job_state(job, SRUN_JOB_STARTING);
if (slurm_step_launch_wait_start(job->step_ctx) == SLURM_SUCCESS) {
update_job_state(job, SRUN_JOB_RUNNING);
/* Only set up MPIR structures if the step launched
* correctly. */
if (opt.multi_prog)
mpir_set_multi_name(job->ctx_params.task_count,
launch_params.argv[0]);
else
mpir_set_executable_names(launch_params.argv[0]);
MPIR_debug_state = MPIR_DEBUG_SPAWNED;
if (opt.debugger_test)
mpir_dump_proctable();
else
MPIR_Breakpoint(job);
} else {
info("Job step %u.%u aborted before step completely launched.",
job->jobid, job->stepid);
}
cleanup:
return rc;
}
