/* * Wait until a job is ready to execute or enters some failed state * RET 1: job ready to run * 0: job can't run (cancelled, failure state, timeout, etc.) */ extern int scontrol_job_ready(char *job_id_str) { int rc; uint32_t job_id; job_id = atoi(job_id_str); if (job_id <= 0) { fprintf(stderr, "Invalid job_id %s", job_id_str); return SLURM_ERROR; } if (cluster_flags & CLUSTER_FLAG_BG) { resource_allocation_response_msg_t *alloc; rc = slurm_allocation_lookup(job_id, &alloc); if (rc == SLURM_SUCCESS) { rc = _wait_bluegene_block_ready(alloc); slurm_free_resource_allocation_response_msg(alloc); } else { error("slurm_allocation_lookup: %m"); rc = SLURM_ERROR; } } else rc = _wait_nodes_ready(job_id); return rc; }
/* * slurm_signal_job - send the specified signal to all steps of an existing job * IN job_id - the job's id * IN signal - signal number * RET 0 on success, otherwise return -1 and set errno to indicate the error */ extern int slurm_signal_job (uint32_t job_id, uint16_t signal) { int rc = SLURM_SUCCESS; resource_allocation_response_msg_t *alloc_info = NULL; signal_tasks_msg_t rpc; if (slurm_allocation_lookup(job_id, &alloc_info)) { rc = slurm_get_errno(); goto fail1; } /* same remote procedure call for each node */ rpc.job_id = job_id; rpc.signal = (uint32_t)signal; rpc.flags = KILL_STEPS_ONLY; rc = _local_send_recv_rc_msgs(alloc_info->node_list, REQUEST_SIGNAL_TASKS, &rpc); slurm_free_resource_allocation_response_msg(alloc_info); fail1: if (rc) { slurm_seterrno_ret(rc); } else { return SLURM_SUCCESS; } }
/* * slurm_terminate_job_step - terminates a job step by sending a * REQUEST_TERMINATE_TASKS rpc to all slurmd of a job step. * IN job_id - the job's id * IN step_id - the job step's id - use SLURM_BATCH_SCRIPT as the step_id * to terminate a job's batch script * RET 0 on success, otherwise return -1 and set errno to indicate the error */ extern int slurm_terminate_job_step (uint32_t job_id, uint32_t step_id) { resource_allocation_response_msg_t *alloc_info = NULL; job_step_info_response_msg_t *step_info = NULL; int rc = 0; int i; int save_errno = 0; if (slurm_allocation_lookup(job_id, &alloc_info)) { return -1; } /* * The controller won't give us info about the batch script job step, * so we need to handle that seperately. */ if (step_id == SLURM_BATCH_SCRIPT) { rc = _terminate_batch_script_step(alloc_info); slurm_free_resource_allocation_response_msg(alloc_info); errno = rc; return rc ? -1 : 0; } /* * Otherwise, look through the list of job step info and find * the one matching step_id. Terminate that step. */ rc = slurm_get_job_steps((time_t)0, job_id, step_id, &step_info, SHOW_ALL); if (rc != 0) { save_errno = errno; goto fail; } for (i = 0; i < step_info->job_step_count; i++) { if ((step_info->job_steps[i].job_id == job_id) && (step_info->job_steps[i].step_id == step_id)) { rc = _terminate_job_step(&step_info->job_steps[i], alloc_info); save_errno = errno; break; } } slurm_free_job_step_info_response_msg(step_info); fail: slurm_free_resource_allocation_response_msg(alloc_info); errno = save_errno; return rc ? -1 : 0; }
static void _wait_for_allocation_response(uint32_t job_id, const listen_t *listen, uint16_t msg_type, int timeout, void **resp) { int errnum, rc; info("job %u queued and waiting for resources", job_id); *resp = NULL; if ((rc = _wait_for_alloc_rpc(listen, timeout)) == 1) rc = _accept_msg_connection(listen->fd, msg_type, resp); if (rc <= 0) { errnum = errno; /* Maybe the resource allocation response RPC got lost * in the mail; surely it should have arrived by now. * Let's see if the controller thinks that the allocation * has been granted. */ if (msg_type == RESPONSE_RESOURCE_ALLOCATION) { if (slurm_allocation_lookup(job_id, (resource_allocation_response_msg_t **) resp) >= 0) return; } else if (msg_type == RESPONSE_JOB_PACK_ALLOCATION) { if (slurm_pack_job_lookup(job_id, (List *) resp) >= 0) return; } else { error("%s: Invalid msg_type (%u)", __func__, msg_type); } if (slurm_get_errno() == ESLURM_JOB_PENDING) { debug3("Still waiting for allocation"); errno = errnum; return; } else { debug3("Unable to confirm allocation for job %u: %m", job_id); return; } } info("job %u has been allocated resources", job_id); return; }
static void _update_job_size(uint32_t job_id) { resource_allocation_response_msg_t *alloc_info; char *fname_csh = NULL, *fname_sh = NULL; FILE *resize_csh = NULL, *resize_sh = NULL; if (!getenv("SLURM_JOBID")) return; /*No job environment here to update */ if (slurm_allocation_lookup(job_id, &alloc_info) != SLURM_SUCCESS) { slurm_perror("slurm_allocation_lookup"); return; } xstrfmtcat(fname_csh, "slurm_job_%u_resize.csh", job_id); xstrfmtcat(fname_sh, "slurm_job_%u_resize.sh", job_id); (void) unlink(fname_csh); (void) unlink(fname_sh); if (!(resize_csh = fopen(fname_csh, "w"))) { fprintf(stderr, "Could not create file %s: %s\n", fname_csh, strerror(errno)); goto fini; } if (!(resize_sh = fopen(fname_sh, "w"))) { fprintf(stderr, "Could not create file %s: %s\n", fname_sh, strerror(errno)); goto fini; } /* * Make files executable */ if (chmod(fname_csh, 0700) == -1) error("%s: chmod(%s): %m", __func__, fname_csh); if (chmod(fname_sh, 0700) == -1) error("%s: chmod(%s): %m", __func__, fname_sh); if (getenv("SLURM_NODELIST")) { fprintf(resize_sh, "export SLURM_NODELIST=\"%s\"\n", alloc_info->node_list); fprintf(resize_csh, "setenv SLURM_NODELIST \"%s\"\n", alloc_info->node_list); } if (getenv("SLURM_JOB_NODELIST")) { fprintf(resize_sh, "export SLURM_JOB_NODELIST=\"%s\"\n", alloc_info->node_list); fprintf(resize_csh, "setenv SLURM_JOB_NODELIST \"%s\"\n", alloc_info->node_list); } if (getenv("SLURM_NNODES")) { fprintf(resize_sh, "export SLURM_NNODES=%u\n", alloc_info->node_cnt); fprintf(resize_csh, "setenv SLURM_NNODES %u\n", alloc_info->node_cnt); } if (getenv("SLURM_JOB_NUM_NODES")) { fprintf(resize_sh, "export SLURM_JOB_NUM_NODES=%u\n", alloc_info->node_cnt); fprintf(resize_csh, "setenv SLURM_JOB_NUM_NODES %u\n", alloc_info->node_cnt); } if (getenv("SLURM_JOB_CPUS_PER_NODE")) { char *tmp; tmp = uint32_compressed_to_str(alloc_info->num_cpu_groups, alloc_info->cpus_per_node, alloc_info->cpu_count_reps); fprintf(resize_sh, "export SLURM_JOB_CPUS_PER_NODE=\"%s\"\n", tmp); fprintf(resize_csh, "setenv SLURM_JOB_CPUS_PER_NODE \"%s\"\n", tmp); xfree(tmp); } if (getenv("SLURM_TASKS_PER_NODE")) { /* We don't have sufficient information to recreate these */ fprintf(resize_sh, "unset SLURM_NPROCS\n"); fprintf(resize_csh, "unsetenv SLURM_NPROCS\n"); fprintf(resize_sh, "unset SLURM_NTASKS\n"); fprintf(resize_csh, "unsetenv SLURM_NTASKS\n"); fprintf(resize_sh, "unset SLURM_TASKS_PER_NODE\n"); fprintf(resize_csh, "unsetenv SLURM_TASKS_PER_NODE\n"); } printf("To reset Slurm environment variables, execute\n"); printf(" For bash or sh shells: . ./%s\n", fname_sh); printf(" For csh shells: source ./%s\n", fname_csh); fini: slurm_free_resource_allocation_response_msg(alloc_info); xfree(fname_csh); xfree(fname_sh); if (resize_csh) fclose(resize_csh); if (resize_sh) fclose(resize_sh); }
/* returns 1 if job and nodes are ready for job to begin, 0 otherwise */ static int _wait_nodes_ready(resource_allocation_response_msg_t *alloc) { int is_ready = 0, i, rc; double cur_delay = 0; double cur_sleep = 0; int suspend_time, resume_time, max_delay; bool job_killed = false; suspend_time = slurm_get_suspend_timeout(); resume_time = slurm_get_resume_timeout(); if ((suspend_time == 0) || (resume_time == 0)) return 1; /* Power save mode disabled */ max_delay = suspend_time + resume_time; max_delay *= 5; /* Allow for ResumeRate support */ pending_job_id = alloc->job_id; for (i = 0; cur_delay < max_delay; i++) { if (i) { cur_sleep = POLL_SLEEP * i; if (i == 1) { verbose("Waiting for nodes to boot (delay looping %d times @ %f secs x index)", max_delay, POLL_SLEEP); } else { debug("Waited %f sec and still waiting: next sleep for %f sec", cur_delay, cur_sleep); } usleep(1000000 * cur_sleep); cur_delay += cur_sleep; } rc = slurm_job_node_ready(alloc->job_id); if (rc == READY_JOB_FATAL) break; /* fatal error */ if ((rc == READY_JOB_ERROR) || (rc == EAGAIN)) continue; /* retry */ if ((rc & READY_JOB_STATE) == 0) { /* job killed */ job_killed = true; break; } if (rc & READY_NODE_STATE) { /* job and node ready */ is_ready = 1; break; } if (destroy_job) break; } if (is_ready) { resource_allocation_response_msg_t *resp; char *tmp_str; if (i > 0) verbose("Nodes %s are ready for job", alloc->node_list); if (alloc->alias_list && !xstrcmp(alloc->alias_list, "TBD") && (slurm_allocation_lookup(pending_job_id, &resp) == SLURM_SUCCESS)) { tmp_str = alloc->alias_list; alloc->alias_list = resp->alias_list; resp->alias_list = tmp_str; slurm_free_resource_allocation_response_msg(resp); } } else if (!destroy_job) { if (job_killed) { error("Job allocation %u has been revoked", alloc->job_id); destroy_job = true; } else error("Nodes %s are still not ready", alloc->node_list); } else /* allocation_interrupted and slurmctld not responing */ is_ready = 0; pending_job_id = 0; return is_ready; }