/* * load into the storage information about a job, * typically when it begins execution, but possibly earlier */ extern int jobacct_storage_g_job_start(void *db_conn, struct job_record *job_ptr) { if (slurm_acct_storage_init(NULL) < 0) return SLURM_ERROR; if (enforce & ACCOUNTING_ENFORCE_NO_JOBS) return SLURM_SUCCESS; /* A pending job's start_time is it's expected initiation time * (changed in slurm v2.1). Rather than changing a bunch of code * in the accounting_storage plugins and SlurmDBD, just clear * start_time before accounting and restore it later. * If an update for a job that is being requeued[hold] happens, * we don't want to modify the start_time of the old record. * Pending + Completing is equivalent to Requeue. */ if (IS_JOB_PENDING(job_ptr) && !IS_JOB_COMPLETING(job_ptr)) { int rc; time_t orig_start_time = job_ptr->start_time; job_ptr->start_time = (time_t) 0; rc = (*(ops.job_start))(db_conn, job_ptr); job_ptr->start_time = orig_start_time; return rc; } return (*(ops.job_start))(db_conn, job_ptr); }
extern List find_preemptable_jobs(struct job_record *job_ptr) { ListIterator job_iterator; struct job_record *job_p; List preemptee_job_list = NULL; /* Validate the preemptor job */ if (job_ptr == NULL) { error("find_preemptable_jobs: job_ptr is NULL"); return preemptee_job_list; } if (!IS_JOB_PENDING(job_ptr)) { error("find_preemptable_jobs: job %u not pending", job_ptr->job_id); return preemptee_job_list; } if (job_ptr->part_ptr == NULL) { error("find_preemptable_jobs: job %u has NULL partition ptr", job_ptr->job_id); return preemptee_job_list; } if (job_ptr->part_ptr->node_bitmap == NULL) { error("find_preemptable_jobs: partition %s node_bitmap=NULL", job_ptr->part_ptr->name); return preemptee_job_list; } /* Build an array of pointers to preemption candidates */ job_iterator = list_iterator_create(job_list); while ((job_p = (struct job_record *) list_next(job_iterator))) { if (!IS_JOB_RUNNING(job_p) && !IS_JOB_SUSPENDED(job_p)) continue; if ((job_p->part_ptr == NULL) || (job_p->part_ptr->priority_tier >= job_ptr->part_ptr->priority_tier) || (job_p->part_ptr->preempt_mode == PREEMPT_MODE_OFF)) continue; if ((job_p->node_bitmap == NULL) || (bit_overlap(job_p->node_bitmap, job_ptr->part_ptr->node_bitmap) == 0)) continue; if (job_ptr->details && (job_ptr->details->expanding_jobid == job_p->job_id)) continue; /* This job is a preemption candidate */ if (preemptee_job_list == NULL) { preemptee_job_list = list_create(NULL); } list_append(preemptee_job_list, job_p); } list_iterator_destroy(job_iterator); if (preemptee_job_list && youngest_order) list_sort(preemptee_job_list, _sort_by_youngest); else if (preemptee_job_list) list_sort(preemptee_job_list, _sort_by_prio); return preemptee_job_list; }
/* NOTE: if job has already completed, we append "EXITCODE=#" to * the state name */ static char * _get_job_state(struct job_record *job_ptr) { char *state_str; static char return_msg[128]; if (IS_JOB_COMPLETING(job_ptr)) { /* Give configured KillWait+10 for job * to clear out, then then consider job * done. Moab will allocate jobs to * nodes that are already Idle. */ int age = (int) difftime(time(NULL), job_ptr->end_time); if (age < (kill_wait+10)) return "Running"; } if (IS_JOB_RUNNING(job_ptr)) return "Running"; if (IS_JOB_SUSPENDED(job_ptr)) return "Suspended"; if (IS_JOB_PENDING(job_ptr)) return "Idle"; if (IS_JOB_COMPLETE(job_ptr) || IS_JOB_FAILED(job_ptr)) state_str = "Completed"; else /* JOB_CANCELLED, JOB_TIMEOUT, JOB_NODE_FAIL, etc. */ state_str = "Removed"; snprintf(return_msg, sizeof(return_msg), "%s;EXITCODE=%u", state_str, WEXITSTATUS(job_ptr->exit_code)); return return_msg; }
static int _get_node_cnt(job_info_t * job) { int node_cnt = 0; /* For PENDING jobs, return the maximum of the requested nodelist, * requested maximum number of nodes, or requested CPUs rounded * to nearest node. * * For COMPLETING jobs, the job->nodes nodelist has already been * altered to list only the nodes still in the comp state, and * thus we count only those nodes toward the total nodes still * allocated to this job. */ if (IS_JOB_PENDING(job)) { node_cnt = _nodes_in_list(job->req_nodes); node_cnt = MAX(node_cnt, job->num_nodes); if ((node_cnt == 1) && (job->num_cpus > 1) && job->ntasks_per_node && (job->ntasks_per_node != (uint16_t) NO_VAL)) { int num_tasks = job->num_cpus; if (job->cpus_per_task != (uint16_t) NO_VAL) num_tasks /= job->cpus_per_task; node_cnt = (num_tasks + 1) / job->ntasks_per_node; if (node_cnt > num_tasks) node_cnt = num_tasks; else if (!node_cnt) node_cnt = 1; } } else node_cnt = _nodes_in_list(job->nodes); return node_cnt; }
int _print_job_job_id(job_info_t * job, int width, bool right, char* suffix) { if (job == NULL) { /* Print the Header instead */ _print_str("JOBID", width, right, true); } else if ((job->array_task_id != NO_VAL) && !params.array_flag && IS_JOB_PENDING(job) && job->node_inx) { uint32_t i, max_task_id = 0; char id[FORMAT_STRING_SIZE], task_str[FORMAT_STRING_SIZE]; bitstr_t *task_bits; for (i = 1; i <= job->node_inx[0]; i++) max_task_id = MAX(max_task_id, job->node_inx[i]); task_bits = bit_alloc(max_task_id + 1); for (i = 1; i <= job->node_inx[0]; i++) bit_set(task_bits, job->node_inx[i]); bit_fmt(task_str, sizeof(task_str), task_bits); snprintf(id, FORMAT_STRING_SIZE, "%u_[%s]", job->array_job_id, task_str); _print_str(id, width, right, true); bit_free(task_bits); } else if (job->array_task_id != NO_VAL) { char id[FORMAT_STRING_SIZE]; snprintf(id, FORMAT_STRING_SIZE, "%u_%u", job->array_job_id, job->array_task_id); _print_str(id, width, right, true); } else { char id[FORMAT_STRING_SIZE]; snprintf(id, FORMAT_STRING_SIZE, "%u", job->job_id); _print_str(id, width, right, true); } if (suffix) printf("%s", suffix); return SLURM_SUCCESS; }
/* Code taken from job_info.c calculate cummulative run time for a job */ static time_t _get_job_runtime(struct job_record *job_ptr) { time_t end_time, run_time; if (IS_JOB_PENDING(job_ptr)) run_time = 0; else if (IS_JOB_SUSPENDED(job_ptr)) run_time = job_ptr->pre_sus_time; else { if (IS_JOB_RUNNING(job_ptr) || (job_ptr->end_time == 0)) end_time = time(NULL); else end_time = job_ptr->end_time; if (job_ptr->suspend_time) { run_time = (time_t) (difftime(end_time, job_ptr->suspend_time) + job_ptr->pre_sus_time); } else { run_time = (time_t) difftime(end_time, job_ptr->start_time); } } return run_time; }
/* * srun_user_message - Send arbitrary message to an srun job (no job steps) */ extern int srun_user_message(struct job_record *job_ptr, char *msg) { slurm_addr_t * addr; srun_user_msg_t *msg_arg; xassert(job_ptr); if (!IS_JOB_PENDING(job_ptr) && !IS_JOB_RUNNING(job_ptr)) return ESLURM_ALREADY_DONE; if (job_ptr->other_port && job_ptr->resp_host && job_ptr->resp_host[0]) { addr = xmalloc(sizeof(struct sockaddr_in)); slurm_set_addr(addr, job_ptr->other_port, job_ptr->resp_host); msg_arg = xmalloc(sizeof(srun_user_msg_t)); msg_arg->job_id = job_ptr->job_id; msg_arg->msg = xstrdup(msg); _srun_agent_launch(addr, job_ptr->resp_host, SRUN_USER_MSG, msg_arg); return SLURM_SUCCESS; } else if (job_ptr->batch_flag && IS_JOB_RUNNING(job_ptr)) { #ifndef HAVE_FRONT_END struct node_record *node_ptr; #endif job_notify_msg_t *notify_msg_ptr; agent_arg_t *agent_arg_ptr; #ifdef HAVE_FRONT_END if (job_ptr->batch_host == NULL) return ESLURM_DISABLED; /* no allocated nodes */ agent_arg_ptr = (agent_arg_t *) xmalloc(sizeof(agent_arg_t)); agent_arg_ptr->hostlist = hostlist_create(job_ptr->batch_host); #else node_ptr = find_first_node_record(job_ptr->node_bitmap); if (node_ptr == NULL) return ESLURM_DISABLED; /* no allocated nodes */ agent_arg_ptr = (agent_arg_t *) xmalloc(sizeof(agent_arg_t)); agent_arg_ptr->hostlist = hostlist_create(node_ptr->name); #endif if (agent_arg_ptr->hostlist == NULL) fatal("hostlist_create: malloc failure"); notify_msg_ptr = (job_notify_msg_t *) xmalloc(sizeof(job_notify_msg_t)); notify_msg_ptr->job_id = job_ptr->job_id; notify_msg_ptr->message = xstrdup(msg); agent_arg_ptr->node_count = 1; agent_arg_ptr->retry = 0; agent_arg_ptr->msg_type = REQUEST_JOB_NOTIFY; agent_arg_ptr->msg_args = (void *) notify_msg_ptr; /* Launch the RPC via agent */ agent_queue_request(agent_arg_ptr); return SLURM_SUCCESS; } return ESLURM_DISABLED; }
static bool _merge_job_array(List l, job_info_t * job_ptr) { job_info_t *list_job_ptr; ListIterator iter; bool merge = false; if (params.array_flag) return merge; if (job_ptr->array_task_id == NO_VAL) return merge; if (!IS_JOB_PENDING(job_ptr)) return merge; xfree(job_ptr->node_inx); if (!l) return merge; iter = list_iterator_create(l); while ((list_job_ptr = list_next(iter))) { if ((list_job_ptr->array_task_id == NO_VAL) || (job_ptr->array_job_id != list_job_ptr->array_job_id) || (!IS_JOB_PENDING(list_job_ptr))) continue; /* We re-purpose the job's node_inx array to store the * array_task_id values */ if (!list_job_ptr->node_inx) { list_job_ptr->node_inx = xmalloc(sizeof(int32_t) * 0xffff); list_job_ptr->node_inx[0] = 1; /* offset */ list_job_ptr->node_inx[1] = list_job_ptr->array_task_id; } list_job_ptr->node_inx[0]++; list_job_ptr->node_inx[list_job_ptr->node_inx[0]] = job_ptr->array_task_id; merge = true; break; } list_iterator_destroy(iter); return merge; }
/* Apply usage with decay factor. Call standard functions */ static void _ft_decay_apply_new_usage(struct job_record *job, time_t *start) { if (!decay_apply_new_usage(job, start)) return; /* Priority 0 is reserved for held jobs. Also skip priority * calculation for non-pending jobs. */ if ((job->priority == 0) || !IS_JOB_PENDING(job)) return; set_priority_factors(*start, job); last_job_update = time(NULL); }
extern int job_modify(struct job_descriptor *job_desc, struct job_record *job_ptr, uint32_t submit_uid) { uint16_t job_mcdram, job_numa; int mcdram_cnt, numa_cnt; char *tmp_str; if (!job_desc->features) return SLURM_SUCCESS; if (!IS_JOB_PENDING(job_ptr)) return ESLURM_JOB_NOT_PENDING; job_mcdram = knl_mcdram_parse(job_desc->features, "&"); mcdram_cnt = knl_mcdram_bits_cnt(job_mcdram); if (mcdram_cnt > 1) { /* Multiple MCDRAM options */ return ESLURM_INVALID_KNL; } else if (mcdram_cnt == 0) { if (job_desc->features && job_desc->features[0]) xstrcat(job_desc->features, "&"); tmp_str = knl_mcdram_str(default_mcdram); xstrcat(job_desc->features, tmp_str); xfree(tmp_str); } else if ((job_mcdram & avail_mcdram) == 0) { /* Unavailable option */ return ESLURM_INVALID_KNL; } job_numa = knl_numa_parse(job_desc->features, "&"); numa_cnt = knl_numa_bits_cnt(job_numa); if (numa_cnt > 1) { /* Multiple NUMA options */ return ESLURM_INVALID_KNL; } else if (numa_cnt == 0) { if (job_desc->features && job_desc->features[0]) xstrcat(job_desc->features, "&"); tmp_str = knl_numa_str(default_numa); xstrcat(job_desc->features, tmp_str); xfree(tmp_str); } else if ((job_numa & avail_numa) == 0) { /* Unavailable NUMA option */ return ESLURM_INVALID_KNL; } return SLURM_SUCCESS; }
long job_time_used(job_info_t * job_ptr) { time_t end_time; if ((job_ptr->start_time == 0) || IS_JOB_PENDING(job_ptr)) return 0L; if (IS_JOB_SUSPENDED(job_ptr)) return (long) job_ptr->pre_sus_time; if (IS_JOB_RUNNING(job_ptr) || (job_ptr->end_time == 0)) end_time = time(NULL); else end_time = job_ptr->end_time; if (job_ptr->suspend_time) return (long) (difftime(end_time, job_ptr->suspend_time) + job_ptr->pre_sus_time); return (long) (difftime(end_time, job_ptr->start_time)); }
/* * load into the storage information about a job, * typically when it begins execution, but possibly earlier */ extern int jobacct_storage_g_job_start(void *db_conn, struct job_record *job_ptr) { if (slurm_acct_storage_init(NULL) < 0) return SLURM_ERROR; /* A pending job's start_time is it's expected initiation time * (changed in slurm v2.1). Rather than changing a bunch of code * in the accounting_storage plugins and SlurmDBD, just clear * start_time before accounting and restore it later. */ if (IS_JOB_PENDING(job_ptr)) { int rc; time_t orig_start_time = job_ptr->start_time; job_ptr->start_time = (time_t) 0; rc = (*(g_acct_storage_context->ops.job_start))( db_conn, job_ptr); job_ptr->start_time = orig_start_time; return rc; } return (*(g_acct_storage_context->ops.job_start))(db_conn, job_ptr); }
int _print_job_reason_list(job_info_t * job, int width, bool right, char* suffix) { if (job == NULL) { /* Print the Header instead */ char *title = "NODELIST(REASON)"; if (params.cluster_flags & CLUSTER_FLAG_BG) title = "MIDPLANELIST(REASON)"; _print_str(title, width, right, false); } else if (!IS_JOB_COMPLETING(job) && (IS_JOB_PENDING(job) || IS_JOB_TIMEOUT(job) || IS_JOB_FAILED(job))) { char id[FORMAT_STRING_SIZE], *reason; if (job->state_desc) reason = job->state_desc; else reason = job_reason_string(job->state_reason); snprintf(id, FORMAT_STRING_SIZE, "(%s)", reason); _print_str(id, width, right, true); } else { char *nodes = xstrdup(job->nodes); char *ionodes = NULL; select_g_select_jobinfo_get(job->select_jobinfo, SELECT_JOBDATA_IONODES, &ionodes); if (ionodes) { xstrfmtcat(nodes, "[%s]", ionodes); xfree(ionodes); _print_str(nodes, width, right, false); } else _print_nodes(nodes, width, right, false); xfree(nodes); } if (suffix) printf("%s", suffix); return SLURM_SUCCESS; }
static bool _pending_pack_jobs(struct job_record *job_ptr) { struct job_record *pack_leader, *pack_job; ListIterator iter; bool pending_job = false; if (job_ptr->pack_job_id == 0) return false; pack_leader = find_job_record(job_ptr->pack_job_id); if (!pack_leader) { error("Job pack leader %pJ not found", job_ptr); return false; } if (!pack_leader->pack_job_list) { error("Job pack leader %pJ lacks pack_job_list", job_ptr); return false; } iter = list_iterator_create(pack_leader->pack_job_list); while ((pack_job = (struct job_record *) list_next(iter))) { if (pack_leader->pack_job_id != pack_job->pack_job_id) { error("%s: Bad pack_job_list for %pJ", __func__, pack_leader); continue; } if (IS_JOB_PENDING(pack_job)) { pending_job = true; break; } } list_iterator_destroy(iter); return pending_job; }
static int _get_node_cnt(job_info_t * job) { int node_cnt = 0, round; /* For PENDING jobs, return the maximum of the requested nodelist, * requested maximum number of nodes, or requested CPUs rounded * to nearest node. * * For COMPLETING jobs, the job->nodes nodelist has already been * altered to list only the nodes still in the comp state, and * thus we count only those nodes toward the total nodes still * allocated to this job. */ if (IS_JOB_PENDING(job)) { node_cnt = _nodes_in_list(job->req_nodes); node_cnt = MAX(node_cnt, job->num_nodes); round = job->num_cpus + params.max_cpus - 1; round /= params.max_cpus; /* round up */ node_cnt = MAX(node_cnt, round); } else node_cnt = _nodes_in_list(job->nodes); return node_cnt; }
/* * acct_policy_update_pending_job - Make sure the limits imposed on a * job on submission are correct after an update to a qos or * association. If the association/qos limits prevent * the job from ever running (lowered limits since job submission), * then cancel the job. */ extern int acct_policy_update_pending_job(struct job_record *job_ptr) { job_desc_msg_t job_desc; uint16_t limit_set_max_cpus = 0; uint16_t limit_set_max_nodes = 0; uint16_t limit_set_time = 0; bool update_accounting = false; struct job_details *details_ptr; int rc = SLURM_SUCCESS; /* check to see if we are enforcing associations and the job * is pending or if we are even enforcing limits. */ if (!accounting_enforce || !IS_JOB_PENDING(job_ptr) || !(accounting_enforce & ACCOUNTING_ENFORCE_LIMITS)) return SLURM_SUCCESS; details_ptr = job_ptr->details; if (!details_ptr) { error("acct_policy_update_pending_job: no details"); return SLURM_ERROR; } /* set up the job desc to make sure things are the way we * need. */ slurm_init_job_desc_msg(&job_desc); job_desc.min_cpus = details_ptr->min_cpus; /* Only set this value if not set from a limit */ if (job_ptr->limit_set_max_cpus == ADMIN_SET_LIMIT) limit_set_max_cpus = job_ptr->limit_set_max_cpus; else if ((details_ptr->max_cpus != NO_VAL) && !job_ptr->limit_set_max_cpus) job_desc.max_cpus = details_ptr->max_cpus; job_desc.min_nodes = details_ptr->min_nodes; /* Only set this value if not set from a limit */ if (job_ptr->limit_set_max_nodes == ADMIN_SET_LIMIT) limit_set_max_nodes = job_ptr->limit_set_max_nodes; else if ((details_ptr->max_nodes != NO_VAL) && !job_ptr->limit_set_max_nodes) job_desc.max_nodes = details_ptr->max_nodes; else job_desc.max_nodes = 0; /* Only set this value if not set from a limit */ if (job_ptr->limit_set_time == ADMIN_SET_LIMIT) limit_set_time = job_ptr->limit_set_time; else if ((job_ptr->time_limit != NO_VAL) && !job_ptr->limit_set_time) job_desc.time_limit = job_ptr->time_limit; if (!acct_policy_validate(&job_desc, job_ptr->part_ptr, job_ptr->assoc_ptr, job_ptr->qos_ptr, &limit_set_max_cpus, &limit_set_max_nodes, &limit_set_time, 0)) { info("acct_policy_update_pending_job: exceeded " "association/qos's cpu, node or " "time limit for job %d", job_ptr->job_id); _cancel_job(job_ptr); return SLURM_ERROR; } /* If it isn't an admin set limit replace it. */ if (!limit_set_max_cpus && (job_ptr->limit_set_max_cpus == 1)) { details_ptr->max_cpus = NO_VAL; job_ptr->limit_set_max_cpus = 0; update_accounting = true; } else if (limit_set_max_cpus != ADMIN_SET_LIMIT) { if (details_ptr->max_cpus != job_desc.max_cpus) { details_ptr->max_cpus = job_desc.max_cpus; update_accounting = true; } job_ptr->limit_set_max_cpus = limit_set_max_cpus; } if (!limit_set_max_nodes && (job_ptr->limit_set_max_nodes == 1)) { details_ptr->max_nodes = 0; job_ptr->limit_set_max_nodes = 0; update_accounting = true; } else if (limit_set_max_nodes != ADMIN_SET_LIMIT) { if (details_ptr->max_nodes != job_desc.max_nodes) { details_ptr->max_nodes = job_desc.max_nodes; update_accounting = true; } job_ptr->limit_set_max_nodes = limit_set_max_nodes; } if (!limit_set_time && (job_ptr->limit_set_time == 1)) { job_ptr->time_limit = NO_VAL; job_ptr->limit_set_time = 0; update_accounting = true; } else if (limit_set_time != ADMIN_SET_LIMIT) { if (job_ptr->time_limit != job_desc.time_limit) { job_ptr->time_limit = job_desc.time_limit; update_accounting = true; } job_ptr->limit_set_time = limit_set_time; } if (update_accounting) { last_job_update = time(NULL); debug("limits changed for job %u: updating accounting", job_ptr->job_id); if (details_ptr->begin_time) { /* Update job record in accounting to reflect changes */ jobacct_storage_g_job_start(acct_db_conn, job_ptr); } } return rc; }
static char * _will_run_test2(uint32_t jobid, time_t start_time, char *node_list, uint32_t *preemptee, int preemptee_cnt, int *err_code, char **err_msg) { struct job_record *job_ptr = NULL, *pre_ptr; struct part_record *part_ptr; bitstr_t *avail_bitmap = NULL, *resv_bitmap = NULL; bitstr_t *exc_core_bitmap = NULL; time_t start_res; uint32_t min_nodes, max_nodes, req_nodes; List preemptee_candidates = NULL, preempted_jobs = NULL; time_t orig_start_time; char *reply_msg = NULL; int i, rc; bool resv_overlap = false; xassert(node_list); debug2("wiki2: will_run2 job_id=%u start_time=%u node_list=%s", jobid, (uint32_t)start_time, node_list); job_ptr = find_job_record(jobid); if (job_ptr == NULL) { *err_code = -700; *err_msg = "No such job"; error("wiki: Failed to find job %u", jobid); return NULL; } if ((job_ptr->details == NULL) || (!IS_JOB_PENDING(job_ptr))) { *err_code = -700; *err_msg = "WillRun not applicable to non-pending job"; error("wiki: WillRun on non-pending job %u", jobid); return NULL; } part_ptr = job_ptr->part_ptr; if (part_ptr == NULL) { *err_code = -700; *err_msg = "Job lacks a partition"; error("wiki: Job %u lacks a partition", jobid); return NULL; } if (node_name2bitmap(node_list, false, &avail_bitmap) != 0) { *err_code = -700; *err_msg = "Invalid available nodes value"; error("wiki: Attempt to set invalid available node " "list for job %u, %s", jobid, node_list); return NULL; } /* Enforce reservation: access control, time and nodes */ start_res = start_time; rc = job_test_resv(job_ptr, &start_res, true, &resv_bitmap, &exc_core_bitmap, &resv_overlap); if (rc != SLURM_SUCCESS) { *err_code = -730; *err_msg = "Job denied access to reservation"; error("wiki: reservation access denied for job %u", jobid); FREE_NULL_BITMAP(avail_bitmap); FREE_NULL_BITMAP(exc_core_bitmap); return NULL; } bit_and(avail_bitmap, resv_bitmap); FREE_NULL_BITMAP(resv_bitmap); /* Only consider nodes that are not DOWN or DRAINED */ bit_and(avail_bitmap, avail_node_bitmap); /* Consider only nodes in this job's partition */ if (part_ptr->node_bitmap) bit_and(avail_bitmap, part_ptr->node_bitmap); else { *err_code = -730; *err_msg = "Job's partition has no nodes"; error("wiki: no nodes in partition %s for job %u", part_ptr->name, jobid); FREE_NULL_BITMAP(avail_bitmap); FREE_NULL_BITMAP(exc_core_bitmap); return NULL; } if (job_req_node_filter(job_ptr, avail_bitmap) != SLURM_SUCCESS) { /* Job probably has invalid feature list */ *err_code = -730; *err_msg = "Job's required features not available " "on selected nodes"; error("wiki: job %u not runnable on hosts=%s", jobid, node_list); FREE_NULL_BITMAP(avail_bitmap); FREE_NULL_BITMAP(exc_core_bitmap); return NULL; } if (job_ptr->details->exc_node_bitmap) { bit_not(job_ptr->details->exc_node_bitmap); bit_and(avail_bitmap, job_ptr->details->exc_node_bitmap); bit_not(job_ptr->details->exc_node_bitmap); } if ((job_ptr->details->req_node_bitmap) && (!bit_super_set(job_ptr->details->req_node_bitmap, avail_bitmap))) { *err_code = -730; *err_msg = "Job's required nodes not available"; error("wiki: job %u not runnable on hosts=%s", jobid, node_list); FREE_NULL_BITMAP(avail_bitmap); FREE_NULL_BITMAP(exc_core_bitmap); return NULL; } min_nodes = MAX(job_ptr->details->min_nodes, part_ptr->min_nodes); if (job_ptr->details->max_nodes == 0) max_nodes = part_ptr->max_nodes; else max_nodes = MIN(job_ptr->details->max_nodes, part_ptr->max_nodes); max_nodes = MIN(max_nodes, 500000); /* prevent overflows */ if (job_ptr->details->max_nodes) req_nodes = max_nodes; else req_nodes = min_nodes; if (min_nodes > max_nodes) { /* job's min_nodes exceeds partitions max_nodes */ *err_code = -730; *err_msg = "Job's min_nodes > max_nodes"; error("wiki: job %u not runnable on hosts=%s", jobid, node_list); FREE_NULL_BITMAP(avail_bitmap); FREE_NULL_BITMAP(exc_core_bitmap); return NULL; } if (preemptee_cnt) { preemptee_candidates = list_create(NULL); for (i=0; i<preemptee_cnt; i++) { if ((pre_ptr = find_job_record(preemptee[i]))) list_append(preemptee_candidates, pre_ptr); } } orig_start_time = job_ptr->start_time; rc = select_g_job_test(job_ptr, avail_bitmap, min_nodes, max_nodes, req_nodes, SELECT_MODE_WILL_RUN, preemptee_candidates, &preempted_jobs, exc_core_bitmap); FREE_NULL_LIST(preemptee_candidates); if (rc == SLURM_SUCCESS) { char *hostlist, *sep, tmp_str[128]; uint32_t pre_cnt = 0, proc_cnt = 0; #ifdef HAVE_BG select_g_select_jobinfo_get(job_ptr->select_jobinfo, SELECT_JOBDATA_NODE_CNT, &proc_cnt); #else proc_cnt = job_ptr->total_cpus; #endif snprintf(tmp_str, sizeof(tmp_str), "STARTINFO=%u TASKS=%u STARTTIME=%u NODES=", job_ptr->job_id, proc_cnt, (uint32_t) job_ptr->start_time); xstrcat(reply_msg, tmp_str); hostlist = bitmap2node_name(avail_bitmap); xstrcat(reply_msg, hostlist); xfree(hostlist); if (preempted_jobs) { while ((pre_ptr = list_pop(preempted_jobs))) { if (pre_cnt++) sep = ","; else sep = " PREEMPT="; snprintf(tmp_str, sizeof(tmp_str), "%s%u", sep, pre_ptr->job_id); xstrcat(reply_msg, tmp_str); } FREE_NULL_LIST(preempted_jobs); } } else { xstrcat(reply_msg, "Jobs not runable on selected nodes"); error("wiki: jobs not runnable on nodes"); } /* Restore pending job's expected start time */ job_ptr->start_time = orig_start_time; FREE_NULL_BITMAP(avail_bitmap); FREE_NULL_BITMAP(exc_core_bitmap); return reply_msg; }
extern void get_job(void) { int error_code = -1, i, recs; static int printed_jobs = 0; static int count = 0; static job_info_msg_t *job_info_ptr = NULL, *new_job_ptr = NULL; job_info_t *job_ptr = NULL; uint16_t show_flags = 0; bitstr_t *nodes_req = NULL; static uint16_t last_flags = 0; if (params.all_flag) show_flags |= SHOW_ALL; if (job_info_ptr) { if (show_flags != last_flags) job_info_ptr->last_update = 0; error_code = slurm_load_jobs(job_info_ptr->last_update, &new_job_ptr, show_flags); if (error_code == SLURM_SUCCESS) slurm_free_job_info_msg(job_info_ptr); else if (slurm_get_errno() == SLURM_NO_CHANGE_IN_DATA) { error_code = SLURM_SUCCESS; new_job_ptr = job_info_ptr; } } else error_code = slurm_load_jobs((time_t) NULL, &new_job_ptr, show_flags); last_flags = show_flags; if (error_code) { if (quiet_flag != 1) { if (!params.commandline) { mvwprintw(text_win, main_ycord, 1, "slurm_load_jobs: %s", slurm_strerror(slurm_get_errno())); main_ycord++; } else { printf("slurm_load_jobs: %s\n", slurm_strerror(slurm_get_errno())); } } } if (!params.no_header) _print_header_job(); if (new_job_ptr) recs = new_job_ptr->record_count; else recs = 0; if (!params.commandline) if ((text_line_cnt+printed_jobs) > count) text_line_cnt--; printed_jobs = 0; count = 0; if (params.hl) nodes_req = get_requested_node_bitmap(); for (i = 0; i < recs; i++) { job_ptr = &(new_job_ptr->job_array[i]); if (!IS_JOB_PENDING(job_ptr) && !IS_JOB_RUNNING(job_ptr) && !IS_JOB_SUSPENDED(job_ptr) && !IS_JOB_COMPLETING(job_ptr)) continue; /* job has completed */ if (nodes_req) { int overlap = 0; bitstr_t *loc_bitmap = bit_alloc(bit_size(nodes_req)); inx2bitstr(loc_bitmap, job_ptr->node_inx); overlap = bit_overlap(loc_bitmap, nodes_req); FREE_NULL_BITMAP(loc_bitmap); if (!overlap) continue; } if (job_ptr->node_inx[0] != -1) { int j = 0; job_ptr->num_nodes = 0; while (job_ptr->node_inx[j] >= 0) { job_ptr->num_nodes += (job_ptr->node_inx[j + 1] + 1) - job_ptr->node_inx[j]; set_grid_inx(job_ptr->node_inx[j], job_ptr->node_inx[j + 1], count); j += 2; } if (!params.commandline) { if ((count >= text_line_cnt) && (printed_jobs < (getmaxy(text_win) - 4))) { job_ptr->num_cpus = (int)letters[count%62]; wattron(text_win, COLOR_PAIR(colors[count%6])); _print_text_job(job_ptr); wattroff(text_win, COLOR_PAIR(colors[count%6])); printed_jobs++; } } else { job_ptr->num_cpus = (int)letters[count%62]; _print_text_job(job_ptr); } count++; } if (count == 128) count = 0; } for (i = 0; i < recs; i++) { job_ptr = &(new_job_ptr->job_array[i]); if (!IS_JOB_PENDING(job_ptr)) continue; /* job has completed */ if (!params.commandline) { if ((count>=text_line_cnt) && (printed_jobs < (getmaxy(text_win) - 4))) { xfree(job_ptr->nodes); job_ptr->nodes = xstrdup("waiting..."); job_ptr->num_cpus = (int) letters[count%62]; wattron(text_win, COLOR_PAIR(colors[count%6])); _print_text_job(job_ptr); wattroff(text_win, COLOR_PAIR(colors[count%6])); printed_jobs++; } } else { xfree(job_ptr->nodes); job_ptr->nodes = xstrdup("waiting..."); job_ptr->num_cpus = (int) letters[count%62]; _print_text_job(job_ptr); printed_jobs++; } count++; if (count == 128) count = 0; } if (params.commandline && params.iterate) printf("\n"); if (!params.commandline) main_ycord++; job_info_ptr = new_job_ptr; return; }
/* * slurm_sprint_job_info - output information about a specific Slurm * job based upon message as loaded using slurm_load_jobs * IN job_ptr - an individual job information record pointer * IN one_liner - print as a single line if true * RET out - char * containing formatted output (must be freed after call) * NULL is returned on failure. */ extern char * slurm_sprint_job_info ( job_info_t * job_ptr, int one_liner ) { int i, j; char time_str[32], *group_name, *user_name; char tmp1[128], tmp2[128], tmp3[128], tmp4[128], tmp5[128], *tmp6_ptr; char tmp_line[512]; char *ionodes = NULL; uint16_t exit_status = 0, term_sig = 0; job_resources_t *job_resrcs = job_ptr->job_resrcs; char *out = NULL; time_t run_time; uint32_t min_nodes, max_nodes = 0; char *nodelist = "NodeList"; bitstr_t *core_bitmap; char *host; int sock_inx, sock_reps, last; int abs_node_inx, rel_node_inx; int bit_inx, bit_reps; uint32_t *last_mem_alloc_ptr = NULL; uint32_t last_mem_alloc = NO_VAL; char *last_hosts; hostlist_t hl, hl_last; char select_buf[122]; uint32_t cluster_flags = slurmdb_setup_cluster_flags(); if (cluster_flags & CLUSTER_FLAG_BG) { nodelist = "MidplaneList"; select_g_select_jobinfo_get(job_ptr->select_jobinfo, SELECT_JOBDATA_IONODES, &ionodes); } /****** Line 1 ******/ snprintf(tmp_line, sizeof(tmp_line), "JobId=%u ", job_ptr->job_id); out = xstrdup(tmp_line); if (job_ptr->array_job_id) { snprintf(tmp_line, sizeof(tmp_line), "ArrayJobId=%u ArrayTaskId=%u ", job_ptr->array_job_id, job_ptr->array_task_id); xstrcat(out, tmp_line); } snprintf(tmp_line, sizeof(tmp_line), "Name=%s", job_ptr->name); xstrcat(out, tmp_line); if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); /****** Line 2 ******/ user_name = uid_to_string((uid_t) job_ptr->user_id); group_name = gid_to_string((gid_t) job_ptr->group_id); snprintf(tmp_line, sizeof(tmp_line), "UserId=%s(%u) GroupId=%s(%u)", user_name, job_ptr->user_id, group_name, job_ptr->group_id); xfree(user_name); xfree(group_name); xstrcat(out, tmp_line); if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); /****** Line 3 ******/ snprintf(tmp_line, sizeof(tmp_line), "Priority=%u Account=%s QOS=%s", job_ptr->priority, job_ptr->account, job_ptr->qos); xstrcat(out, tmp_line); if (slurm_get_track_wckey()) { snprintf(tmp_line, sizeof(tmp_line), " WCKey=%s", job_ptr->wckey); xstrcat(out, tmp_line); } if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); /****** Line 4 ******/ if (job_ptr->state_desc) { /* Replace white space with underscore for easier parsing */ for (j=0; job_ptr->state_desc[j]; j++) { if (isspace((int)job_ptr->state_desc[j])) job_ptr->state_desc[j] = '_'; } tmp6_ptr = job_ptr->state_desc; } else tmp6_ptr = job_reason_string(job_ptr->state_reason); snprintf(tmp_line, sizeof(tmp_line), "JobState=%s Reason=%s Dependency=%s", job_state_string(job_ptr->job_state), tmp6_ptr, job_ptr->dependency); xstrcat(out, tmp_line); if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); /****** Line 5 ******/ snprintf(tmp_line, sizeof(tmp_line), "Requeue=%u Restarts=%u BatchFlag=%u ", job_ptr->requeue, job_ptr->restart_cnt, job_ptr->batch_flag); xstrcat(out, tmp_line); if (WIFSIGNALED(job_ptr->exit_code)) term_sig = WTERMSIG(job_ptr->exit_code); exit_status = WEXITSTATUS(job_ptr->exit_code); snprintf(tmp_line, sizeof(tmp_line), "ExitCode=%u:%u", exit_status, term_sig); xstrcat(out, tmp_line); if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); /****** Line 5a (optional) ******/ if (!(job_ptr->show_flags & SHOW_DETAIL)) goto line6; if (WIFSIGNALED(job_ptr->derived_ec)) term_sig = WTERMSIG(job_ptr->derived_ec); else term_sig = 0; exit_status = WEXITSTATUS(job_ptr->derived_ec); snprintf(tmp_line, sizeof(tmp_line), "DerivedExitCode=%u:%u", exit_status, term_sig); xstrcat(out, tmp_line); if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); /****** Line 6 ******/ line6: snprintf(tmp_line, sizeof(tmp_line), "RunTime="); xstrcat(out, tmp_line); if (IS_JOB_PENDING(job_ptr)) run_time = 0; else if (IS_JOB_SUSPENDED(job_ptr)) run_time = job_ptr->pre_sus_time; else { time_t end_time; if (IS_JOB_RUNNING(job_ptr) || (job_ptr->end_time == 0)) end_time = time(NULL); else end_time = job_ptr->end_time; if (job_ptr->suspend_time) { run_time = (time_t) (difftime(end_time, job_ptr->suspend_time) + job_ptr->pre_sus_time); } else run_time = (time_t) difftime(end_time, job_ptr->start_time); } secs2time_str(run_time, tmp1, sizeof(tmp1)); sprintf(tmp_line, "%s ", tmp1); xstrcat(out, tmp_line); snprintf(tmp_line, sizeof(tmp_line), "TimeLimit="); xstrcat(out, tmp_line); if (job_ptr->time_limit == NO_VAL) sprintf(tmp_line, "Partition_Limit"); else { mins2time_str(job_ptr->time_limit, tmp_line, sizeof(tmp_line)); } xstrcat(out, tmp_line); snprintf(tmp_line, sizeof(tmp_line), " TimeMin="); xstrcat(out, tmp_line); if (job_ptr->time_min == 0) sprintf(tmp_line, "N/A"); else { mins2time_str(job_ptr->time_min, tmp_line, sizeof(tmp_line)); } xstrcat(out, tmp_line); if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); /****** Line 7 ******/ slurm_make_time_str((time_t *)&job_ptr->submit_time, time_str, sizeof(time_str)); snprintf(tmp_line, sizeof(tmp_line), "SubmitTime=%s ", time_str); xstrcat(out, tmp_line); slurm_make_time_str((time_t *)&job_ptr->eligible_time, time_str, sizeof(time_str)); snprintf(tmp_line, sizeof(tmp_line), "EligibleTime=%s", time_str); xstrcat(out, tmp_line); if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); /****** Line 8 (optional) ******/ if (job_ptr->resize_time) { slurm_make_time_str((time_t *)&job_ptr->resize_time, time_str, sizeof(time_str)); snprintf(tmp_line, sizeof(tmp_line), "ResizeTime=%s", time_str); xstrcat(out, tmp_line); if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); } /****** Line 9 ******/ slurm_make_time_str((time_t *)&job_ptr->start_time, time_str, sizeof(time_str)); snprintf(tmp_line, sizeof(tmp_line), "StartTime=%s ", time_str); xstrcat(out, tmp_line); snprintf(tmp_line, sizeof(tmp_line), "EndTime="); xstrcat(out, tmp_line); if ((job_ptr->time_limit == INFINITE) && (job_ptr->end_time > time(NULL))) sprintf(tmp_line, "Unknown"); else { slurm_make_time_str ((time_t *)&job_ptr->end_time, time_str, sizeof(time_str)); sprintf(tmp_line, "%s", time_str); } xstrcat(out, tmp_line); if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); /****** Line 10 ******/ if (job_ptr->preempt_time == 0) sprintf(tmp_line, "PreemptTime=None "); else { slurm_make_time_str((time_t *)&job_ptr->preempt_time, time_str, sizeof(time_str)); snprintf(tmp_line, sizeof(tmp_line), "PreemptTime=%s ", time_str); } xstrcat(out, tmp_line); if (job_ptr->suspend_time) { slurm_make_time_str ((time_t *)&job_ptr->suspend_time, time_str, sizeof(time_str)); } else { strncpy(time_str, "None", sizeof(time_str)); } snprintf(tmp_line, sizeof(tmp_line), "SuspendTime=%s SecsPreSuspend=%ld", time_str, (long int)job_ptr->pre_sus_time); xstrcat(out, tmp_line); if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); /****** Line 11 ******/ snprintf(tmp_line, sizeof(tmp_line), "Partition=%s AllocNode:Sid=%s:%u", job_ptr->partition, job_ptr->alloc_node, job_ptr->alloc_sid); xstrcat(out, tmp_line); if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); /****** Line 12 ******/ snprintf(tmp_line, sizeof(tmp_line), "Req%s=%s Exc%s=%s", nodelist, job_ptr->req_nodes, nodelist, job_ptr->exc_nodes); xstrcat(out, tmp_line); if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); /****** Line 13 ******/ xstrfmtcat(out, "%s=", nodelist); xstrcat(out, job_ptr->nodes); if (job_ptr->nodes && ionodes) { snprintf(tmp_line, sizeof(tmp_line), "[%s]", ionodes); xstrcat(out, tmp_line); xfree(ionodes); } if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); /****** Line 14 (optional) ******/ if (job_ptr->batch_host) { snprintf(tmp_line, sizeof(tmp_line), "BatchHost=%s", job_ptr->batch_host); xstrcat(out, tmp_line); if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); } /****** Line 15 ******/ if (cluster_flags & CLUSTER_FLAG_BG) { select_g_select_jobinfo_get(job_ptr->select_jobinfo, SELECT_JOBDATA_NODE_CNT, &min_nodes); if ((min_nodes == 0) || (min_nodes == NO_VAL)) { min_nodes = job_ptr->num_nodes; max_nodes = job_ptr->max_nodes; } else if (job_ptr->max_nodes) max_nodes = min_nodes; } else { min_nodes = job_ptr->num_nodes; max_nodes = job_ptr->max_nodes; } _sprint_range(tmp1, sizeof(tmp1), job_ptr->num_cpus, job_ptr->max_cpus); _sprint_range(tmp2, sizeof(tmp2), min_nodes, max_nodes); if (job_ptr->sockets_per_node == (uint16_t) NO_VAL) strcpy(tmp3, "*"); else snprintf(tmp3, sizeof(tmp3), "%u", job_ptr->sockets_per_node); if (job_ptr->cores_per_socket == (uint16_t) NO_VAL) strcpy(tmp4, "*"); else snprintf(tmp4, sizeof(tmp4), "%u", job_ptr->cores_per_socket); if (job_ptr->threads_per_core == (uint16_t) NO_VAL) strcpy(tmp5, "*"); else snprintf(tmp5, sizeof(tmp5), "%u", job_ptr->threads_per_core); snprintf(tmp_line, sizeof(tmp_line), "NumNodes=%s NumCPUs=%s CPUs/Task=%u ReqS:C:T=%s:%s:%s", tmp2, tmp1, job_ptr->cpus_per_task, tmp3, tmp4, tmp5); xstrcat(out, tmp_line); if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); if (!job_resrcs) goto line15; if (cluster_flags & CLUSTER_FLAG_BG) { if ((job_resrcs->cpu_array_cnt > 0) && (job_resrcs->cpu_array_value) && (job_resrcs->cpu_array_reps)) { int length = 0; xstrcat(out, "CPUs="); length += 10; for (i = 0; i < job_resrcs->cpu_array_cnt; i++) { if (length > 70) { /* skip to last CPU group entry */ if (i < job_resrcs->cpu_array_cnt - 1) { continue; } /* add ellipsis before last entry */ xstrcat(out, "...,"); length += 4; } snprintf(tmp_line, sizeof(tmp_line), "%d", job_resrcs->cpus[i]); xstrcat(out, tmp_line); length += strlen(tmp_line); if (job_resrcs->cpu_array_reps[i] > 1) { snprintf(tmp_line, sizeof(tmp_line), "*%d", job_resrcs->cpu_array_reps[i]); xstrcat(out, tmp_line); length += strlen(tmp_line); } if (i < job_resrcs->cpu_array_cnt - 1) { xstrcat(out, ","); length++; } } if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); } } else { if (!job_resrcs->core_bitmap) goto line15; last = bit_fls(job_resrcs->core_bitmap); if (last == -1) goto line15; hl = hostlist_create(job_ptr->nodes); if (!hl) { error("slurm_sprint_job_info: hostlist_create: %s", job_ptr->nodes); return NULL; } hl_last = hostlist_create(NULL); if (!hl_last) { error("slurm_sprint_job_info: hostlist_create: NULL"); hostlist_destroy(hl); return NULL; } bit_inx = 0; i = sock_inx = sock_reps = 0; abs_node_inx = job_ptr->node_inx[i]; /* tmp1[] stores the current cpu(s) allocated */ tmp2[0] = '\0'; /* stores last cpu(s) allocated */ for (rel_node_inx=0; rel_node_inx < job_resrcs->nhosts; rel_node_inx++) { if (sock_reps >= job_resrcs->sock_core_rep_count[sock_inx]) { sock_inx++; sock_reps = 0; } sock_reps++; bit_reps = job_resrcs->sockets_per_node[sock_inx] * job_resrcs->cores_per_socket[sock_inx]; core_bitmap = bit_alloc(bit_reps); for (j=0; j < bit_reps; j++) { if (bit_test(job_resrcs->core_bitmap, bit_inx)) bit_set(core_bitmap, j); bit_inx++; } bit_fmt(tmp1, sizeof(tmp1), core_bitmap); FREE_NULL_BITMAP(core_bitmap); host = hostlist_shift(hl); /* * If the allocation values for this host are not the same as the * last host, print the report of the last group of hosts that had * identical allocation values. */ if (strcmp(tmp1, tmp2) || (last_mem_alloc_ptr != job_resrcs->memory_allocated) || (job_resrcs->memory_allocated && (last_mem_alloc != job_resrcs->memory_allocated[rel_node_inx]))) { if (hostlist_count(hl_last)) { last_hosts = hostlist_ranged_string_xmalloc( hl_last); snprintf(tmp_line, sizeof(tmp_line), " Nodes=%s CPU_IDs=%s Mem=%u", last_hosts, tmp2, last_mem_alloc_ptr ? last_mem_alloc : 0); xfree(last_hosts); xstrcat(out, tmp_line); if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); hostlist_destroy(hl_last); hl_last = hostlist_create(NULL); } strcpy(tmp2, tmp1); last_mem_alloc_ptr = job_resrcs->memory_allocated; if (last_mem_alloc_ptr) last_mem_alloc = job_resrcs-> memory_allocated[rel_node_inx]; else last_mem_alloc = NO_VAL; } hostlist_push_host(hl_last, host); free(host); if (bit_inx > last) break; if (abs_node_inx > job_ptr->node_inx[i+1]) { i += 2; abs_node_inx = job_ptr->node_inx[i]; } else { abs_node_inx++; } } if (hostlist_count(hl_last)) { last_hosts = hostlist_ranged_string_xmalloc(hl_last); snprintf(tmp_line, sizeof(tmp_line), " Nodes=%s CPU_IDs=%s Mem=%u", last_hosts, tmp2, last_mem_alloc_ptr ? last_mem_alloc : 0); xfree(last_hosts); xstrcat(out, tmp_line); if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); } hostlist_destroy(hl); hostlist_destroy(hl_last); } /****** Line 15 ******/ line15: if (job_ptr->pn_min_memory & MEM_PER_CPU) { job_ptr->pn_min_memory &= (~MEM_PER_CPU); tmp6_ptr = "CPU"; } else tmp6_ptr = "Node"; if (cluster_flags & CLUSTER_FLAG_BG) { convert_num_unit((float)job_ptr->pn_min_cpus, tmp1, sizeof(tmp1), UNIT_NONE); snprintf(tmp_line, sizeof(tmp_line), "MinCPUsNode=%s", tmp1); } else { snprintf(tmp_line, sizeof(tmp_line), "MinCPUsNode=%u", job_ptr->pn_min_cpus); } xstrcat(out, tmp_line); convert_num_unit((float)job_ptr->pn_min_memory, tmp1, sizeof(tmp1), UNIT_MEGA); convert_num_unit((float)job_ptr->pn_min_tmp_disk, tmp2, sizeof(tmp2), UNIT_MEGA); snprintf(tmp_line, sizeof(tmp_line), " MinMemory%s=%s MinTmpDiskNode=%s", tmp6_ptr, tmp1, tmp2); xstrcat(out, tmp_line); if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); /****** Line 16 ******/ snprintf(tmp_line, sizeof(tmp_line), "Features=%s Gres=%s Reservation=%s", job_ptr->features, job_ptr->gres, job_ptr->resv_name); xstrcat(out, tmp_line); if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); /****** Line 17 ******/ snprintf(tmp_line, sizeof(tmp_line), "Shared=%s Contiguous=%d Licenses=%s Network=%s", (job_ptr->shared == 0 ? "0" : job_ptr->shared == 1 ? "1" : "OK"), job_ptr->contiguous, job_ptr->licenses, job_ptr->network); xstrcat(out, tmp_line); if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); /****** Line 18 ******/ snprintf(tmp_line, sizeof(tmp_line), "Command=%s", job_ptr->command); xstrcat(out, tmp_line); if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); /****** Line 19 ******/ snprintf(tmp_line, sizeof(tmp_line), "WorkDir=%s", job_ptr->work_dir); xstrcat(out, tmp_line); if (cluster_flags & CLUSTER_FLAG_BG) { /****** Line 20 (optional) ******/ select_g_select_jobinfo_sprint(job_ptr->select_jobinfo, select_buf, sizeof(select_buf), SELECT_PRINT_BG_ID); if (select_buf[0] != '\0') { if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); snprintf(tmp_line, sizeof(tmp_line), "Block_ID=%s", select_buf); xstrcat(out, tmp_line); } /****** Line 21 (optional) ******/ select_g_select_jobinfo_sprint(job_ptr->select_jobinfo, select_buf, sizeof(select_buf), SELECT_PRINT_MIXED_SHORT); if (select_buf[0] != '\0') { if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); xstrcat(out, select_buf); } if (cluster_flags & CLUSTER_FLAG_BGL) { /****** Line 22 (optional) ******/ select_g_select_jobinfo_sprint( job_ptr->select_jobinfo, select_buf, sizeof(select_buf), SELECT_PRINT_BLRTS_IMAGE); if (select_buf[0] != '\0') { if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); snprintf(tmp_line, sizeof(tmp_line), "BlrtsImage=%s", select_buf); xstrcat(out, tmp_line); } } /****** Line 23 (optional) ******/ select_g_select_jobinfo_sprint(job_ptr->select_jobinfo, select_buf, sizeof(select_buf), SELECT_PRINT_LINUX_IMAGE); if (select_buf[0] != '\0') { if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); if (cluster_flags & CLUSTER_FLAG_BGL) snprintf(tmp_line, sizeof(tmp_line), "LinuxImage=%s", select_buf); else snprintf(tmp_line, sizeof(tmp_line), "CnloadImage=%s", select_buf); xstrcat(out, tmp_line); } /****** Line 24 (optional) ******/ select_g_select_jobinfo_sprint(job_ptr->select_jobinfo, select_buf, sizeof(select_buf), SELECT_PRINT_MLOADER_IMAGE); if (select_buf[0] != '\0') { if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); snprintf(tmp_line, sizeof(tmp_line), "MloaderImage=%s", select_buf); xstrcat(out, tmp_line); } /****** Line 25 (optional) ******/ select_g_select_jobinfo_sprint(job_ptr->select_jobinfo, select_buf, sizeof(select_buf), SELECT_PRINT_RAMDISK_IMAGE); if (select_buf[0] != '\0') { if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); if (cluster_flags & CLUSTER_FLAG_BGL) snprintf(tmp_line, sizeof(tmp_line), "RamDiskImage=%s", select_buf); else snprintf(tmp_line, sizeof(tmp_line), "IoloadImage=%s", select_buf); xstrcat(out, tmp_line); } } /****** Line 26 (optional) ******/ if (job_ptr->comment) { if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); snprintf(tmp_line, sizeof(tmp_line), "Comment=%s ", job_ptr->comment); xstrcat(out, tmp_line); } /****** Line 27 (optional) ******/ if (job_ptr->batch_script) { if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); xstrcat(out, "BatchScript=\n"); xstrcat(out, job_ptr->batch_script); } /****** Line 28 (optional) ******/ if (job_ptr->req_switch) { char time_buf[32]; if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); secs2time_str((time_t) job_ptr->wait4switch, time_buf, sizeof(time_buf)); snprintf(tmp_line, sizeof(tmp_line), "Switches=%u@%s\n", job_ptr->req_switch, time_buf); xstrcat(out, tmp_line); } /****** Line 29 (optional) ******/ if (one_liner) xstrcat(out, "\n"); else xstrcat(out, "\n\n"); return out; }
/* filter job records per input specifications, * returns >0 if job should be filter out (not printed) */ static int _filter_job(job_info_t * job) { int filter; ListIterator iterator; uint32_t *user; uint16_t *state_id; char *account, *part, *qos, *name; squeue_job_step_t *job_step_id; if (params.job_list) { filter = 1; iterator = list_iterator_create(params.job_list); while ((job_step_id = list_next(iterator))) { if (((job_step_id->array_id == (uint16_t) NO_VAL) && ((job_step_id->job_id == job->array_job_id) || (job_step_id->job_id == job->job_id))) || ((job_step_id->array_id == job->array_task_id) && (job_step_id->job_id == job->array_job_id))) { filter = 0; break; } } list_iterator_destroy(iterator); if (filter == 1) return 1; } if (params.part_list) { char *token = NULL, *last = NULL, *tmp_name = NULL; filter = 1; if (job->partition) { tmp_name = xstrdup(job->partition); token = strtok_r(tmp_name, ",", &last); } while (token && filter) { iterator = list_iterator_create(params.part_list); while ((part = list_next(iterator))) { if (strcmp(part, token) == 0) { filter = 0; break; } } list_iterator_destroy(iterator); token = strtok_r(NULL, ",", &last); } xfree(tmp_name); if (filter == 1) return 2; } if (params.account_list) { filter = 1; iterator = list_iterator_create(params.account_list); while ((account = list_next(iterator))) { if ((job->account != NULL) && (strcasecmp(account, job->account) == 0)) { filter = 0; break; } } list_iterator_destroy(iterator); if (filter == 1) return 2; } if (params.qos_list) { filter = 1; iterator = list_iterator_create(params.qos_list); while ((qos = list_next(iterator))) { if ((job->qos != NULL) && (strcasecmp(qos, job->qos) == 0)) { filter = 0; break; } } list_iterator_destroy(iterator); if (filter == 1) return 2; } if (params.state_list) { filter = 1; iterator = list_iterator_create(params.state_list); while ((state_id = list_next(iterator))) { if ((*state_id == job->job_state) || ((*state_id == JOB_COMPLETING) && (*state_id & job->job_state)) || ((*state_id == JOB_CONFIGURING) && (*state_id & job->job_state))) { filter = 0; break; } } list_iterator_destroy(iterator); if (filter == 1) return 3; } else { if (!IS_JOB_PENDING(job) && !IS_JOB_RUNNING(job) && !IS_JOB_SUSPENDED(job) && !IS_JOB_COMPLETING(job)) return 4; } if ((params.nodes) && ((job->nodes == NULL) || (!hostset_intersects(params.nodes, job->nodes)))) return 5; if (params.user_list) { filter = 1; iterator = list_iterator_create(params.user_list); while ((user = list_next(iterator))) { if (*user == job->user_id) { filter = 0; break; } } list_iterator_destroy(iterator); if (filter == 1) return 6; } if (params.reservation) { if ((job->resv_name == NULL) || (strcmp(job->resv_name, params.reservation))) { return 7; } } if (params.name_list) { filter = 1; iterator = list_iterator_create(params.name_list); while ((name = list_next(iterator))) { if ((job->name != NULL) && (strcasecmp(name, job->name) == 0)) { filter = 0; break; } } list_iterator_destroy(iterator); if (filter == 1) return 8; } return 0; }
static char * _will_run_test(uint32_t jobid, time_t start_time, char *node_list, int *err_code, char **err_msg) { struct job_record *job_ptr = NULL; struct part_record *part_ptr; bitstr_t *avail_bitmap = NULL, *resv_bitmap = NULL; bitstr_t *exc_core_bitmap = NULL; char *hostlist, *reply_msg = NULL; uint32_t min_nodes, max_nodes, req_nodes; int rc; time_t start_res, orig_start_time; List preemptee_candidates; debug2("wiki2: will_run job_id=%u start_time=%u node_list=%s", jobid, (uint32_t)start_time, node_list); job_ptr = find_job_record(jobid); if (job_ptr == NULL) { *err_code = -700; *err_msg = "No such job"; error("wiki: Failed to find job %u", jobid); return NULL; } if ((job_ptr->details == NULL) || (!IS_JOB_PENDING(job_ptr))) { *err_code = -700; *err_msg = "WillRun not applicable to non-pending job"; error("wiki: WillRun on non-pending job %u", jobid); return NULL; } part_ptr = job_ptr->part_ptr; if (part_ptr == NULL) { *err_code = -700; *err_msg = "Job lacks a partition"; error("wiki: Job %u lacks a partition", jobid); return NULL; } if ((node_list == NULL) || (node_list[0] == '\0')) { /* assume all nodes available to job for testing */ avail_bitmap = bit_copy(avail_node_bitmap); } else if (node_name2bitmap(node_list, false, &avail_bitmap) != 0) { *err_code = -700; *err_msg = "Invalid available nodes value"; error("wiki: Attempt to set invalid available node " "list for job %u, %s", jobid, node_list); return NULL; } /* Enforce reservation: access control, time and nodes */ start_res = start_time; rc = job_test_resv(job_ptr, &start_res, true, &resv_bitmap, &exc_core_bitmap); if (rc != SLURM_SUCCESS) { *err_code = -730; *err_msg = "Job denied access to reservation"; error("wiki: reservation access denied for job %u", jobid); FREE_NULL_BITMAP(avail_bitmap); return NULL; } start_time = MAX(start_time, start_res); bit_and(avail_bitmap, resv_bitmap); FREE_NULL_BITMAP(resv_bitmap); /* Only consider nodes that are not DOWN or DRAINED */ bit_and(avail_bitmap, avail_node_bitmap); /* Consider only nodes in this job's partition */ if (part_ptr->node_bitmap) bit_and(avail_bitmap, part_ptr->node_bitmap); else { *err_code = -730; *err_msg = "Job's partition has no nodes"; error("wiki: no nodes in partition %s for job %u", part_ptr->name, jobid); FREE_NULL_BITMAP(avail_bitmap); return NULL; } if (job_req_node_filter(job_ptr, avail_bitmap) != SLURM_SUCCESS) { /* Job probably has invalid feature list */ *err_code = -730; *err_msg = "Job's required features not available " "on selected nodes"; error("wiki: job %u not runnable on hosts=%s", jobid, node_list); FREE_NULL_BITMAP(avail_bitmap); return NULL; } if (job_ptr->details->exc_node_bitmap) { bit_not(job_ptr->details->exc_node_bitmap); bit_and(avail_bitmap, job_ptr->details->exc_node_bitmap); bit_not(job_ptr->details->exc_node_bitmap); } if ((job_ptr->details->req_node_bitmap) && (!bit_super_set(job_ptr->details->req_node_bitmap, avail_bitmap))) { *err_code = -730; *err_msg = "Job's required nodes not available"; error("wiki: job %u not runnable on hosts=%s", jobid, node_list); FREE_NULL_BITMAP(avail_bitmap); return NULL; } min_nodes = MAX(job_ptr->details->min_nodes, part_ptr->min_nodes); if (job_ptr->details->max_nodes == 0) max_nodes = part_ptr->max_nodes; else max_nodes = MIN(job_ptr->details->max_nodes, part_ptr->max_nodes); max_nodes = MIN(max_nodes, 500000); /* prevent overflows */ if (job_ptr->details->max_nodes) req_nodes = max_nodes; else req_nodes = min_nodes; if (min_nodes > max_nodes) { /* job's min_nodes exceeds partitions max_nodes */ *err_code = -730; *err_msg = "Job's min_nodes > max_nodes"; error("wiki: job %u not runnable on hosts=%s", jobid, node_list); FREE_NULL_BITMAP(avail_bitmap); return NULL; } preemptee_candidates = slurm_find_preemptable_jobs(job_ptr); orig_start_time = job_ptr->start_time; rc = select_g_job_test(job_ptr, avail_bitmap, min_nodes, max_nodes, req_nodes, SELECT_MODE_WILL_RUN, preemptee_candidates, NULL, exc_core_bitmap); if (preemptee_candidates) list_destroy(preemptee_candidates); if (rc == SLURM_SUCCESS) { char tmp_str[128]; *err_code = 0; uint32_t proc_cnt = 0; xstrcat(reply_msg, "STARTINFO="); #ifdef HAVE_BG select_g_select_jobinfo_get(job_ptr->select_jobinfo, SELECT_JOBDATA_NODE_CNT, &proc_cnt); #else proc_cnt = job_ptr->total_cpus; #endif snprintf(tmp_str, sizeof(tmp_str), "%u:%u@%u,", jobid, proc_cnt, (uint32_t) job_ptr->start_time); xstrcat(reply_msg, tmp_str); hostlist = bitmap2node_name(avail_bitmap); xstrcat(reply_msg, hostlist); xfree(hostlist); } else { xstrcat(reply_msg, "Jobs not runable on selected nodes"); error("wiki: jobs not runnable on nodes"); } /* Restore pending job's expected start time */ job_ptr->start_time = orig_start_time; FREE_NULL_BITMAP(avail_bitmap); return reply_msg; }
static int _job_modify(uint32_t jobid, char *bank_ptr, char *depend_ptr, char *new_hostlist, uint32_t new_node_cnt, char *part_name_ptr, uint32_t new_time_limit, char *name_ptr, char *start_ptr, char *feature_ptr, char *env_ptr, char *comment_ptr, char *gres_ptr, char *wckey_ptr) { struct job_record *job_ptr; time_t now = time(NULL); bool update_accounting = false; job_ptr = find_job_record(jobid); if (job_ptr == NULL) { error("wiki: MODIFYJOB has invalid jobid %u", jobid); return ESLURM_INVALID_JOB_ID; } if (IS_JOB_FINISHED(job_ptr) || (job_ptr->details == NULL)) { info("wiki: MODIFYJOB jobid %u is finished", jobid); return ESLURM_DISABLED; } if (comment_ptr) { info("wiki: change job %u comment %s", jobid, comment_ptr); xfree(job_ptr->comment); job_ptr->comment = xstrdup(comment_ptr); last_job_update = now; } if (depend_ptr) { int rc = update_job_dependency(job_ptr, depend_ptr); if (rc == SLURM_SUCCESS) { info("wiki: changed job %u dependency to %s", jobid, depend_ptr); } else { error("wiki: changing job %u dependency to %s", jobid, depend_ptr); return EINVAL; } } if (env_ptr) { bool have_equal = false; char old_sep[1]; int begin = 0, i; if (job_ptr->batch_flag == 0) { error("wiki: attempt to set environment variables " "for non-batch job %u", jobid); return ESLURM_DISABLED; } for (i=0; ; i++) { if (env_ptr[i] == '=') { if (have_equal) { error("wiki: setting job %u invalid " "environment variables: %s", jobid, env_ptr); return EINVAL; } have_equal = true; if (env_ptr[i+1] == '\"') { for (i+=2; ; i++) { if (env_ptr[i] == '\0') { error("wiki: setting job %u " "invalid environment " "variables: %s", jobid, env_ptr); return EINVAL; } if (env_ptr[i] == '\"') { i++; break; } if (env_ptr[i] == '\\') { i++; } } } else if (env_ptr[i+1] == '\'') { for (i+=2; ; i++) { if (env_ptr[i] == '\0') { error("wiki: setting job %u " "invalid environment " "variables: %s", jobid, env_ptr); return EINVAL; } if (env_ptr[i] == '\'') { i++; break; } if (env_ptr[i] == '\\') { i++; } } } } if (isspace(env_ptr[i]) || (env_ptr[i] == ',')) { if (!have_equal) { error("wiki: setting job %u invalid " "environment variables: %s", jobid, env_ptr); return EINVAL; } old_sep[0] = env_ptr[i]; env_ptr[i] = '\0'; xrealloc(job_ptr->details->env_sup, sizeof(char *) * (job_ptr->details->env_cnt+1)); job_ptr->details->env_sup [job_ptr->details->env_cnt++] = xstrdup(&env_ptr[begin]); info("wiki: for job %u add env: %s", jobid, &env_ptr[begin]); env_ptr[i] = old_sep[0]; if (isspace(old_sep[0])) break; begin = i + 1; have_equal = false; } } } if (new_time_limit) { time_t old_time = job_ptr->time_limit; job_ptr->time_limit = new_time_limit; info("wiki: change job %u time_limit to %u", jobid, new_time_limit); /* Update end_time based upon change * to preserve suspend time info */ job_ptr->end_time = job_ptr->end_time + ((job_ptr->time_limit - old_time) * 60); last_job_update = now; } if (bank_ptr && (update_job_account("wiki", job_ptr, bank_ptr) != SLURM_SUCCESS)) { return EINVAL; } if (feature_ptr) { if (IS_JOB_PENDING(job_ptr) && (job_ptr->details)) { info("wiki: change job %u features to %s", jobid, feature_ptr); job_ptr->details->features = xstrdup(feature_ptr); last_job_update = now; } else { error("wiki: MODIFYJOB features of non-pending " "job %u", jobid); return ESLURM_DISABLED; } } if (start_ptr) { char *end_ptr; uint32_t begin_time = strtol(start_ptr, &end_ptr, 10); if (IS_JOB_PENDING(job_ptr) && (job_ptr->details)) { info("wiki: change job %u begin time to %u", jobid, begin_time); job_ptr->details->begin_time = begin_time; last_job_update = now; update_accounting = true; } else { error("wiki: MODIFYJOB begin_time of non-pending " "job %u", jobid); return ESLURM_DISABLED; } } if (name_ptr) { if (IS_JOB_PENDING(job_ptr)) { info("wiki: change job %u name %s", jobid, name_ptr); xfree(job_ptr->name); job_ptr->name = xstrdup(name_ptr); last_job_update = now; update_accounting = true; } else { error("wiki: MODIFYJOB name of non-pending job %u", jobid); return ESLURM_DISABLED; } } if (new_hostlist) { int rc = 0, task_cnt; hostlist_t hl; char *tasklist; if (!IS_JOB_PENDING(job_ptr) || !job_ptr->details) { /* Job is done, nothing to reset */ if (new_hostlist == '\0') goto host_fini; error("wiki: MODIFYJOB hostlist of non-pending " "job %u", jobid); return ESLURM_DISABLED; } xfree(job_ptr->details->req_nodes); FREE_NULL_BITMAP(job_ptr->details->req_node_bitmap); if (new_hostlist == '\0') goto host_fini; tasklist = moab2slurm_task_list(new_hostlist, &task_cnt); if (tasklist == NULL) { rc = 1; goto host_fini; } hl = hostlist_create(tasklist); if (hl == 0) { rc = 1; goto host_fini; } hostlist_uniq(hl); hostlist_sort(hl); job_ptr->details->req_nodes = hostlist_ranged_string_xmalloc(hl); hostlist_destroy(hl); if (job_ptr->details->req_nodes == NULL) { rc = 1; goto host_fini; } if (node_name2bitmap(job_ptr->details->req_nodes, false, &job_ptr->details->req_node_bitmap)) { rc = 1; goto host_fini; } host_fini: if (rc) { info("wiki: change job %u invalid hostlist %s", jobid, new_hostlist); xfree(job_ptr->details->req_nodes); return EINVAL; } else { info("wiki: change job %u hostlist %s", jobid, new_hostlist); update_accounting = true; } } if (part_name_ptr) { struct part_record *part_ptr; if (!IS_JOB_PENDING(job_ptr)) { error("wiki: MODIFYJOB partition of non-pending " "job %u", jobid); return ESLURM_DISABLED; } part_ptr = find_part_record(part_name_ptr); if (part_ptr == NULL) { error("wiki: MODIFYJOB has invalid partition %s", part_name_ptr); return ESLURM_INVALID_PARTITION_NAME; } info("wiki: change job %u partition %s", jobid, part_name_ptr); xfree(job_ptr->partition); job_ptr->partition = xstrdup(part_name_ptr); job_ptr->part_ptr = part_ptr; last_job_update = now; update_accounting = true; } if (new_node_cnt) { job_desc_msg_t job_desc; #ifdef HAVE_BG uint16_t geometry[SYSTEM_DIMENSIONS] = {(uint16_t) NO_VAL}; static uint16_t cpus_per_node = 0; if (!cpus_per_node) { select_g_alter_node_cnt(SELECT_GET_NODE_CPU_CNT, &cpus_per_node); } #endif if(!IS_JOB_PENDING(job_ptr) || !job_ptr->details) { error("wiki: MODIFYJOB node count of non-pending " "job %u", jobid); return ESLURM_DISABLED; } memset(&job_desc, 0, sizeof(job_desc_msg_t)); job_desc.min_nodes = new_node_cnt; job_desc.max_nodes = NO_VAL; job_desc.select_jobinfo = select_g_select_jobinfo_alloc(); select_g_alter_node_cnt(SELECT_SET_NODE_CNT, &job_desc); select_g_select_jobinfo_free(job_desc.select_jobinfo); job_ptr->details->min_nodes = job_desc.min_nodes; if (job_ptr->details->max_nodes && (job_ptr->details->max_nodes < job_desc.min_nodes)) job_ptr->details->max_nodes = job_desc.min_nodes; info("wiki: change job %u min_nodes to %u", jobid, new_node_cnt); #ifdef HAVE_BG job_ptr->details->min_cpus = job_desc.min_cpus; job_ptr->details->max_cpus = job_desc.max_cpus; job_ptr->details->pn_min_cpus = job_desc.pn_min_cpus; new_node_cnt = job_ptr->details->min_cpus; if (cpus_per_node) new_node_cnt /= cpus_per_node; /* This is only set up so accounting is set up correctly */ select_g_select_jobinfo_set(job_ptr->select_jobinfo, SELECT_JOBDATA_NODE_CNT, &new_node_cnt); /* reset geo since changing this makes any geo potentially invalid */ select_g_select_jobinfo_set(job_ptr->select_jobinfo, SELECT_JOBDATA_GEOMETRY, geometry); #endif last_job_update = now; update_accounting = true; } if (gres_ptr) { char *orig_gres; if (!IS_JOB_PENDING(job_ptr)) { error("wiki: MODIFYJOB GRES of non-pending job %u", jobid); return ESLURM_DISABLED; } orig_gres = job_ptr->gres; job_ptr->gres = NULL; if (gres_ptr[0]) job_ptr->gres = xstrdup(gres_ptr); if (gres_plugin_job_state_validate(job_ptr->gres, &job_ptr->gres_list)) { error("wiki: MODIFYJOB Invalid GRES=%s", gres_ptr); xfree(job_ptr->gres); job_ptr->gres = orig_gres; return ESLURM_INVALID_GRES; } xfree(orig_gres); } if (wckey_ptr) { int rc = update_job_wckey("update_job", job_ptr, wckey_ptr); if (rc != SLURM_SUCCESS) { error("wiki: MODIFYJOB Invalid WCKEY=%s", wckey_ptr); return rc; } } if (update_accounting) { if (job_ptr->details && job_ptr->details->begin_time) { /* Update job record in accounting to reflect * the changes */ jobacct_storage_g_job_start(acct_db_conn, job_ptr); } } return SLURM_SUCCESS; }
static int _job_modify(uint32_t jobid, char *bank_ptr, char *depend_ptr, char *new_hostlist, uint32_t new_node_cnt, char *part_name_ptr, uint32_t new_time_limit) { struct job_record *job_ptr; bool update_accounting = false; job_ptr = find_job_record(jobid); if (job_ptr == NULL) { error("wiki: MODIFYJOB has invalid jobid %u", jobid); return ESLURM_INVALID_JOB_ID; } if (IS_JOB_FINISHED(job_ptr)) { error("wiki: MODIFYJOB jobid %u is finished", jobid); return ESLURM_DISABLED; } if (depend_ptr) { int rc = update_job_dependency(job_ptr, depend_ptr); if (rc == SLURM_SUCCESS) { info("wiki: changed job %u dependency to %s", jobid, depend_ptr); } else { error("wiki: changing job %u dependency to %s", jobid, depend_ptr); return EINVAL; } } if (new_time_limit) { time_t old_time = job_ptr->time_limit; job_ptr->time_limit = new_time_limit; info("wiki: change job %u time_limit to %u", jobid, new_time_limit); /* Update end_time based upon change * to preserve suspend time info */ job_ptr->end_time = job_ptr->end_time + ((job_ptr->time_limit - old_time) * 60); last_job_update = time(NULL); } if (bank_ptr) { if (update_job_account("wiki", job_ptr, bank_ptr) != SLURM_SUCCESS) return EINVAL; else update_accounting = true; } if (new_hostlist) { int rc = 0, task_cnt; hostlist_t hl; char *tasklist; if (!IS_JOB_PENDING(job_ptr) || !job_ptr->details) { /* Job is done, nothing to reset */ if (new_hostlist == '\0') goto host_fini; error("wiki: MODIFYJOB tasklist of non-pending " "job %u", jobid); return ESLURM_DISABLED; } xfree(job_ptr->details->req_nodes); FREE_NULL_BITMAP(job_ptr->details->req_node_bitmap); if (new_hostlist == '\0') goto host_fini; tasklist = moab2slurm_task_list(new_hostlist, &task_cnt); if (tasklist == NULL) { rc = 1; goto host_fini; } hl = hostlist_create(tasklist); if (hl == 0) { rc = 1; goto host_fini; } hostlist_uniq(hl); hostlist_sort(hl); job_ptr->details->req_nodes = hostlist_ranged_string_xmalloc(hl); hostlist_destroy(hl); if (job_ptr->details->req_nodes == NULL) { rc = 1; goto host_fini; } if (node_name2bitmap(job_ptr->details->req_nodes, false, &job_ptr->details->req_node_bitmap)) { rc = 1; goto host_fini; } host_fini: if (rc) { info("wiki: change job %u invalid hostlist %s", jobid, new_hostlist); xfree(job_ptr->details->req_nodes); return EINVAL; } else { info("wiki: change job %u hostlist %s", jobid, new_hostlist); update_accounting = true; } } if (part_name_ptr) { struct part_record *part_ptr; if (!IS_JOB_PENDING(job_ptr)) { error("wiki: MODIFYJOB partition of non-pending " "job %u", jobid); return ESLURM_DISABLED; } part_ptr = find_part_record(part_name_ptr); if (part_ptr == NULL) { error("wiki: MODIFYJOB has invalid partition %s", part_name_ptr); return ESLURM_INVALID_PARTITION_NAME; } info("wiki: change job %u partition %s", jobid, part_name_ptr); xfree(job_ptr->partition); job_ptr->partition = xstrdup(part_name_ptr); job_ptr->part_ptr = part_ptr; last_job_update = time(NULL); update_accounting = true; } if (new_node_cnt) { if (IS_JOB_PENDING(job_ptr) && job_ptr->details) { job_ptr->details->min_nodes = new_node_cnt; if (job_ptr->details->max_nodes && (job_ptr->details->max_nodes < new_node_cnt)) job_ptr->details->max_nodes = new_node_cnt; info("wiki: change job %u min_nodes to %u", jobid, new_node_cnt); last_job_update = time(NULL); update_accounting = true; } else { error("wiki: MODIFYJOB node count of non-pending " "job %u", jobid); return ESLURM_DISABLED; } } if (update_accounting) { /* Update job record in accounting to reflect changes */ jobacct_storage_job_start_direct(acct_db_conn, job_ptr); } return SLURM_SUCCESS; }
static int _attempt_backfill(void) { DEF_TIMERS; bool filter_root = false; List job_queue; job_queue_rec_t *job_queue_rec; slurmdb_qos_rec_t *qos_ptr = NULL; int i, j, node_space_recs; struct job_record *job_ptr; struct part_record *part_ptr; uint32_t end_time, end_reserve; uint32_t time_limit, comp_time_limit, orig_time_limit; uint32_t min_nodes, max_nodes, req_nodes; bitstr_t *avail_bitmap = NULL, *resv_bitmap = NULL; bitstr_t *exc_core_bitmap = NULL; time_t now, sched_start, later_start, start_res, resv_end; node_space_map_t *node_space; struct timeval bf_time1, bf_time2; int sched_timeout = 2, yield_sleep = 1; int rc = 0; int job_test_count = 0; uint32_t *uid = NULL, nuser = 0; uint16_t *njobs = NULL; bool already_counted; uint32_t reject_array_job_id = 0; #ifdef HAVE_CRAY /* * Run a Basil Inventory immediately before setting up the schedule * plan, to avoid race conditions caused by ALPS node state change. * Needs to be done with the node-state lock taken. */ START_TIMER; if (select_g_reconfigure()) { debug4("backfill: not scheduling due to ALPS"); return SLURM_SUCCESS; } END_TIMER; if (debug_flags & DEBUG_FLAG_BACKFILL) info("backfill: ALPS inventory completed, %s", TIME_STR); /* The Basil inventory can take a long time to complete. Process * pending RPCs before starting the backfill scheduling logic */ _yield_locks(1); #endif START_TIMER; if (debug_flags & DEBUG_FLAG_BACKFILL) info("backfill: beginning"); sched_start = now = time(NULL); if (slurm_get_root_filter()) filter_root = true; job_queue = build_job_queue(true); if (list_count(job_queue) == 0) { debug("backfill: no jobs to backfill"); list_destroy(job_queue); return 0; } gettimeofday(&bf_time1, NULL); slurmctld_diag_stats.bf_queue_len = list_count(job_queue); slurmctld_diag_stats.bf_queue_len_sum += slurmctld_diag_stats. bf_queue_len; slurmctld_diag_stats.bf_last_depth = 0; slurmctld_diag_stats.bf_last_depth_try = 0; slurmctld_diag_stats.bf_when_last_cycle = now; bf_last_yields = 0; slurmctld_diag_stats.bf_active = 1; node_space = xmalloc(sizeof(node_space_map_t) * (max_backfill_job_cnt + 3)); node_space[0].begin_time = sched_start; node_space[0].end_time = sched_start + backfill_window; node_space[0].avail_bitmap = bit_copy(avail_node_bitmap); node_space[0].next = 0; node_space_recs = 1; if (debug_flags & DEBUG_FLAG_BACKFILL) _dump_node_space_table(node_space); if (max_backfill_job_per_user) { uid = xmalloc(BF_MAX_USERS * sizeof(uint32_t)); njobs = xmalloc(BF_MAX_USERS * sizeof(uint16_t)); } while ((job_queue_rec = (job_queue_rec_t *) list_pop_bottom(job_queue, sort_job_queue2))) { job_ptr = job_queue_rec->job_ptr; orig_time_limit = job_ptr->time_limit; if ((time(NULL) - sched_start) >= sched_timeout) { uint32_t save_time_limit = job_ptr->time_limit; job_ptr->time_limit = orig_time_limit; if (debug_flags & DEBUG_FLAG_BACKFILL) { END_TIMER; info("backfill: completed yielding locks " "after testing %d jobs, %s", job_test_count, TIME_STR); } if (_yield_locks(yield_sleep) && !backfill_continue) { if (debug_flags & DEBUG_FLAG_BACKFILL) { info("backfill: system state changed, " "breaking out after testing %d " "jobs", job_test_count); } rc = 1; break; } job_ptr->time_limit = save_time_limit; /* Reset backfill scheduling timers, resume testing */ sched_start = time(NULL); job_test_count = 0; START_TIMER; } part_ptr = job_queue_rec->part_ptr; job_test_count++; xfree(job_queue_rec); if (!IS_JOB_PENDING(job_ptr)) continue; /* started in other partition */ if (!avail_front_end(job_ptr)) continue; /* No available frontend for this job */ if (job_ptr->array_task_id != (uint16_t) NO_VAL) { if (reject_array_job_id == job_ptr->array_job_id) continue; /* already rejected array element */ /* assume reject whole array for now, clear if OK */ reject_array_job_id = job_ptr->array_job_id; } job_ptr->part_ptr = part_ptr; if (debug_flags & DEBUG_FLAG_BACKFILL) info("backfill test for job %u", job_ptr->job_id); slurmctld_diag_stats.bf_last_depth++; already_counted = false; if (max_backfill_job_per_user) { for (j = 0; j < nuser; j++) { if (job_ptr->user_id == uid[j]) { njobs[j]++; if (debug_flags & DEBUG_FLAG_BACKFILL) debug("backfill: user %u: " "#jobs %u", uid[j], njobs[j]); break; } } if (j == nuser) { /* user not found */ if (nuser < BF_MAX_USERS) { uid[j] = job_ptr->user_id; njobs[j] = 1; nuser++; } else { error("backfill: too many users in " "queue. Consider increasing " "BF_MAX_USERS"); } if (debug_flags & DEBUG_FLAG_BACKFILL) debug2("backfill: found new user %u. " "Total #users now %u", job_ptr->user_id, nuser); } else { if (njobs[j] > max_backfill_job_per_user) { /* skip job */ if (debug_flags & DEBUG_FLAG_BACKFILL) debug("backfill: have already " "checked %u jobs for " "user %u; skipping " "job %u", max_backfill_job_per_user, job_ptr->user_id, job_ptr->job_id); continue; } } } if (((part_ptr->state_up & PARTITION_SCHED) == 0) || (part_ptr->node_bitmap == NULL)) continue; if ((part_ptr->flags & PART_FLAG_ROOT_ONLY) && filter_root) continue; if ((!job_independent(job_ptr, 0)) || (license_job_test(job_ptr, time(NULL)) != SLURM_SUCCESS)) continue; /* Determine minimum and maximum node counts */ min_nodes = MAX(job_ptr->details->min_nodes, part_ptr->min_nodes); if (job_ptr->details->max_nodes == 0) max_nodes = part_ptr->max_nodes; else max_nodes = MIN(job_ptr->details->max_nodes, part_ptr->max_nodes); max_nodes = MIN(max_nodes, 500000); /* prevent overflows */ if (job_ptr->details->max_nodes) req_nodes = max_nodes; else req_nodes = min_nodes; if (min_nodes > max_nodes) { /* job's min_nodes exceeds partition's max_nodes */ continue; } /* Determine job's expected completion time */ if (job_ptr->time_limit == NO_VAL) { if (part_ptr->max_time == INFINITE) time_limit = 365 * 24 * 60; /* one year */ else time_limit = part_ptr->max_time; } else { if (part_ptr->max_time == INFINITE) time_limit = job_ptr->time_limit; else time_limit = MIN(job_ptr->time_limit, part_ptr->max_time); } comp_time_limit = time_limit; qos_ptr = job_ptr->qos_ptr; if (qos_ptr && (qos_ptr->flags & QOS_FLAG_NO_RESERVE) && slurm_get_preempt_mode()) time_limit = job_ptr->time_limit = 1; else if (job_ptr->time_min && (job_ptr->time_min < time_limit)) time_limit = job_ptr->time_limit = job_ptr->time_min; /* Determine impact of any resource reservations */ later_start = now; TRY_LATER: if ((time(NULL) - sched_start) >= sched_timeout) { uint32_t save_time_limit = job_ptr->time_limit; job_ptr->time_limit = orig_time_limit; if (debug_flags & DEBUG_FLAG_BACKFILL) { END_TIMER; info("backfill: completed yielding locks 2" "after testing %d jobs, %s", job_test_count, TIME_STR); } if (_yield_locks(yield_sleep) && !backfill_continue) { if (debug_flags & DEBUG_FLAG_BACKFILL) { info("backfill: system state changed, " "breaking out after testing %d " "jobs", job_test_count); } rc = 1; break; } job_ptr->time_limit = save_time_limit; /* Reset backfill scheduling timers, resume testing */ sched_start = time(NULL); job_test_count = 1; START_TIMER; } FREE_NULL_BITMAP(avail_bitmap); FREE_NULL_BITMAP(exc_core_bitmap); start_res = later_start; later_start = 0; j = job_test_resv(job_ptr, &start_res, true, &avail_bitmap, &exc_core_bitmap); if (j != SLURM_SUCCESS) { job_ptr->time_limit = orig_time_limit; continue; } if (start_res > now) end_time = (time_limit * 60) + start_res; else end_time = (time_limit * 60) + now; resv_end = find_resv_end(start_res); /* Identify usable nodes for this job */ bit_and(avail_bitmap, part_ptr->node_bitmap); bit_and(avail_bitmap, up_node_bitmap); for (j=0; ; ) { if ((node_space[j].end_time > start_res) && node_space[j].next && (later_start == 0)) later_start = node_space[j].end_time; if (node_space[j].end_time <= start_res) ; else if (node_space[j].begin_time <= end_time) { bit_and(avail_bitmap, node_space[j].avail_bitmap); } else break; if ((j = node_space[j].next) == 0) break; } if ((resv_end++) && ((later_start == 0) || (resv_end < later_start))) { later_start = resv_end; } if (job_ptr->details->exc_node_bitmap) { bit_not(job_ptr->details->exc_node_bitmap); bit_and(avail_bitmap, job_ptr->details->exc_node_bitmap); bit_not(job_ptr->details->exc_node_bitmap); } /* Test if insufficient nodes remain OR * required nodes missing OR * nodes lack features */ if ((bit_set_count(avail_bitmap) < min_nodes) || ((job_ptr->details->req_node_bitmap) && (!bit_super_set(job_ptr->details->req_node_bitmap, avail_bitmap))) || (job_req_node_filter(job_ptr, avail_bitmap))) { if (later_start) { job_ptr->start_time = 0; goto TRY_LATER; } /* Job can not start until too far in the future */ job_ptr->time_limit = orig_time_limit; job_ptr->start_time = sched_start + backfill_window; continue; } /* Identify nodes which are definitely off limits */ FREE_NULL_BITMAP(resv_bitmap); resv_bitmap = bit_copy(avail_bitmap); bit_not(resv_bitmap); /* this is the time consuming operation */ debug2("backfill: entering _try_sched for job %u.", job_ptr->job_id); if (!already_counted) { slurmctld_diag_stats.bf_last_depth_try++; already_counted = true; } j = _try_sched(job_ptr, &avail_bitmap, min_nodes, max_nodes, req_nodes, exc_core_bitmap); now = time(NULL); if (j != SLURM_SUCCESS) { job_ptr->time_limit = orig_time_limit; job_ptr->start_time = 0; continue; /* not runable */ } if (start_res > job_ptr->start_time) { job_ptr->start_time = start_res; last_job_update = now; } if (job_ptr->start_time <= now) { uint32_t save_time_limit = job_ptr->time_limit; int rc = _start_job(job_ptr, resv_bitmap); if (qos_ptr && (qos_ptr->flags & QOS_FLAG_NO_RESERVE)) { if (orig_time_limit == NO_VAL) job_ptr->time_limit = comp_time_limit; else job_ptr->time_limit = orig_time_limit; job_ptr->end_time = job_ptr->start_time + (job_ptr->time_limit * 60); } else if ((rc == SLURM_SUCCESS) && job_ptr->time_min) { /* Set time limit as high as possible */ job_ptr->time_limit = comp_time_limit; job_ptr->end_time = job_ptr->start_time + (comp_time_limit * 60); _reset_job_time_limit(job_ptr, now, node_space); time_limit = job_ptr->time_limit; } else { job_ptr->time_limit = orig_time_limit; } if (rc == ESLURM_ACCOUNTING_POLICY) { /* Unknown future start time, just skip job */ job_ptr->start_time = 0; continue; } else if (rc != SLURM_SUCCESS) { /* Planned to start job, but something bad * happended. */ job_ptr->start_time = 0; break; } else { /* Started this job, move to next one */ reject_array_job_id = 0; /* Update the database if job time limit * changed and move to next job */ if (save_time_limit != job_ptr->time_limit) jobacct_storage_g_job_start(acct_db_conn, job_ptr); continue; } } else job_ptr->time_limit = orig_time_limit; if (later_start && (job_ptr->start_time > later_start)) { /* Try later when some nodes currently reserved for * pending jobs are free */ job_ptr->start_time = 0; goto TRY_LATER; } if (job_ptr->start_time > (sched_start + backfill_window)) { /* Starts too far in the future to worry about */ continue; } if (node_space_recs >= max_backfill_job_cnt) { /* Already have too many jobs to deal with */ break; } end_reserve = job_ptr->start_time + (time_limit * 60); if (_test_resv_overlap(node_space, avail_bitmap, job_ptr->start_time, end_reserve)) { /* This job overlaps with an existing reservation for * job to be backfill scheduled, which the sched * plugin does not know about. Try again later. */ later_start = job_ptr->start_time; job_ptr->start_time = 0; goto TRY_LATER; } /* * Add reservation to scheduling table if appropriate */ if (qos_ptr && (qos_ptr->flags & QOS_FLAG_NO_RESERVE)) continue; reject_array_job_id = 0; bit_not(avail_bitmap); _add_reservation(job_ptr->start_time, end_reserve, avail_bitmap, node_space, &node_space_recs); if (debug_flags & DEBUG_FLAG_BACKFILL) _dump_node_space_table(node_space); } xfree(uid); xfree(njobs); FREE_NULL_BITMAP(avail_bitmap); FREE_NULL_BITMAP(exc_core_bitmap); FREE_NULL_BITMAP(resv_bitmap); for (i=0; ; ) { FREE_NULL_BITMAP(node_space[i].avail_bitmap); if ((i = node_space[i].next) == 0) break; } xfree(node_space); list_destroy(job_queue); gettimeofday(&bf_time2, NULL); _do_diag_stats(&bf_time1, &bf_time2, yield_sleep); if (debug_flags & DEBUG_FLAG_BACKFILL) { END_TIMER; info("backfill: completed testing %d jobs, %s", job_test_count, TIME_STR); } return rc; }
/* ensure that all jobs running in SLURM are accounted for. * this procedure assumes that the gs data has already been * locked by the caller! */ static void _scan_slurm_job_list(void) { struct job_record *job_ptr; struct gs_part *p_ptr; int i; ListIterator job_iterator; char *part_name; if (!job_list) { /* no jobs */ if (slurmctld_conf.debug_flags & DEBUG_FLAG_GANG) info("gang: _scan_slurm_job_list: job_list NULL"); return; } if (slurmctld_conf.debug_flags & DEBUG_FLAG_GANG) info("gang: _scan_slurm_job_list: job_list exists..."); job_iterator = list_iterator_create(job_list); while ((job_ptr = (struct job_record *) list_next(job_iterator))) { if (slurmctld_conf.debug_flags & DEBUG_FLAG_GANG) { info("gang: _scan_slurm_job_list: checking job %u", job_ptr->job_id); } if (IS_JOB_PENDING(job_ptr)) continue; if (IS_JOB_SUSPENDED(job_ptr) && (job_ptr->priority == 0)) continue; /* not suspended by us */ if (job_ptr->part_ptr && job_ptr->part_ptr->name) part_name = job_ptr->part_ptr->name; else part_name = job_ptr->partition; if (IS_JOB_SUSPENDED(job_ptr) || IS_JOB_RUNNING(job_ptr)) { /* are we tracking this job already? */ p_ptr = list_find_first(gs_part_list, _find_gs_part, part_name); if (!p_ptr) /* no partition */ continue; i = _find_job_index(p_ptr, job_ptr->job_id); if (i >= 0) /* we're tracking it, so continue */ continue; /* We're not tracking this job. Resume it if it's * suspended, and then add it to the job list. */ if (IS_JOB_SUSPENDED(job_ptr) && job_ptr->priority) { /* The likely scenario here is that the * failed over, and this is a job that gang * had previously suspended. It's not possible * to determine the previous order of jobs * without preserving gang state, which is not * worth the extra infrastructure. Just resume * the job and then add it to the job list. */ _resume_job(job_ptr->job_id); } _add_job_to_part(p_ptr, job_ptr); continue; } /* if the job is not pending, suspended, or running, then * it's completing or completed. Make sure we've released * this job */ p_ptr = list_find_first(gs_part_list, _find_gs_part, part_name); if (!p_ptr) /* no partition */ continue; _remove_job_from_part(job_ptr->job_id, p_ptr, false); } list_iterator_destroy(job_iterator); /* now that all of the old jobs have been flushed out, * update the active row of all partitions */ _update_all_active_rows(); return; }
/* * scontrol_hold - perform some job hold/release operation * IN op - hold/release operation * IN job_str - a job ID or job name * RET 0 if no slurm error, errno otherwise. parsing error prints * error message and returns 0 */ extern int scontrol_hold(char *op, char *job_str) { static uint32_t last_job_id = NO_VAL; static job_info_msg_t *jobs = NULL; job_array_resp_msg_t *resp = NULL; int i, rc = SLURM_SUCCESS, rc2; int j; job_desc_msg_t job_msg; uint32_t job_id = 0; char *job_name = NULL; char *job_id_str = NULL; slurm_job_info_t *job_ptr; if (job_str && !xstrncasecmp(job_str, "JobID=", 6)) job_str += 6; if (job_str && !xstrncasecmp(job_str, "Job=", 4)) job_str += 4; slurm_init_job_desc_msg (&job_msg); if ((xstrncasecmp(op, "holdu", 5) == 0) || (xstrncasecmp(op, "uhold", 5) == 0)) { job_msg.priority = 0; job_msg.alloc_sid = ALLOC_SID_USER_HOLD; } else if (xstrncasecmp(op, "hold", 4) == 0) { job_msg.priority = 0; job_msg.alloc_sid = 0; } else job_msg.priority = INFINITE; if (_is_job_id(job_str)) { while ((job_msg.job_id_str = _next_job_id())) { rc2 = slurm_update_job2(&job_msg, &resp); if (rc2 != SLURM_SUCCESS) { rc2 = slurm_get_errno(); rc = MAX(rc, rc2); exit_code = 1; if (quiet_flag != 1) { fprintf(stderr, "%s for job %s\n", slurm_strerror(rc2), job_msg.job_id_str); } } else if (resp) { for (i = 0; i < resp->job_array_count; i++) { if ((resp->error_code[i]==SLURM_SUCCESS) && (resp->job_array_count == 1)) continue; exit_code = 1; if (quiet_flag == 1) continue; fprintf(stderr, "%s: %s\n", resp->job_array_id[i], slurm_strerror(resp-> error_code[i])); } slurm_free_job_array_resp(resp); resp = NULL; } job_msg.job_id_str = _next_job_id(); } return rc; } else if (job_str) { if (!xstrncasecmp(job_str, "Name=", 5)) { job_str += 5; job_id = 0; job_name = job_str; last_job_id = NO_VAL; } else { exit_code = 1; rc = ESLURM_INVALID_JOB_ID; slurm_seterrno(rc); if (quiet_flag != 1) { fprintf(stderr, "%s for job %s\n", slurm_strerror(rc), job_str); } return rc; } } else { last_job_id = NO_VAL; /* Refresh cache on next call */ return 0; } if (last_job_id != job_id) { if (scontrol_load_job(&jobs, job_id)) { if (quiet_flag == -1) slurm_perror ("slurm_load_job error"); return 1; } last_job_id = job_id; } /* set current user, needed e.g., for AllowGroups checks */ for (i = 0, job_ptr = jobs->job_array; i < jobs->record_count; i++, job_ptr++) { if (xstrcmp(job_name, job_ptr->name)) continue; if (!IS_JOB_PENDING(job_ptr)) { if (job_ptr->array_task_id != NO_VAL) continue; slurm_seterrno(ESLURM_JOB_NOT_PENDING); rc = MAX(rc, ESLURM_JOB_NOT_PENDING); } if (job_ptr->array_task_str) { xstrfmtcat(job_id_str, "%u_%s", job_ptr->array_job_id, job_ptr->array_task_str); } else if (job_ptr->array_task_id != NO_VAL) { xstrfmtcat(job_id_str, "%u_%u", job_ptr->array_job_id, job_ptr->array_task_id); } else { xstrfmtcat(job_id_str, "%u", job_ptr->job_id); } job_msg.job_id_str = job_id_str; rc2 = slurm_update_job2(&job_msg, &resp); if (rc2 != SLURM_SUCCESS) { rc2 = slurm_get_errno(); rc = MAX(rc, rc2); exit_code = 1; if (quiet_flag != 1) { fprintf(stderr, "%s for job %s\n", slurm_strerror(rc2), job_msg.job_id_str); } } else if (resp) { for (j = 0; j < resp->job_array_count; j++) { if ((resp->error_code[j] == SLURM_SUCCESS) && (resp->job_array_count == 1)) continue; exit_code = 1; if (quiet_flag == 1) continue; fprintf(stderr, "%s: %s\n", resp->job_array_id[j], slurm_strerror(resp->error_code[j])); } slurm_free_job_array_resp(resp); resp = NULL; } xfree(job_id_str); } return rc; }
/* * Attempt to start a job * jobid (IN) - job id * task_cnt (IN) - total count of tasks to start * hostlist (IN) - SLURM hostlist expression with no repeated hostnames * tasklist (IN/OUT) - comma separated list of hosts with tasks to be started, * list hostname once per task to start * comment_ptr (IN) - new comment field for the job or NULL for no change * err_code (OUT) - Moab error code * err_msg (OUT) - Moab error message */ static int _start_job(uint32_t jobid, int task_cnt, char *hostlist, char *tasklist, char *comment_ptr, int *err_code, char **err_msg) { int rc = 0, old_task_cnt = 1; struct job_record *job_ptr; /* Write lock on job info, read lock on node info */ slurmctld_lock_t job_write_lock = { NO_LOCK, WRITE_LOCK, READ_LOCK, NO_LOCK }; char *new_node_list = NULL; static char tmp_msg[128]; bitstr_t *new_bitmap = (bitstr_t *) NULL; bitstr_t *save_req_bitmap = (bitstr_t *) NULL; bitoff_t i, bsize; int ll; /* layout info index */ char *node_name, *node_idx, *node_cur, *save_req_nodes = NULL; size_t node_name_len; static uint32_t cr_test = 0, cr_enabled = 0; if (cr_test == 0) { select_g_get_info_from_plugin(SELECT_CR_PLUGIN, NULL, &cr_enabled); cr_test = 1; } lock_slurmctld(job_write_lock); job_ptr = find_job_record(jobid); if (job_ptr == NULL) { *err_code = -700; *err_msg = "No such job"; error("wiki: Failed to find job %u", jobid); rc = -1; goto fini; } if ((job_ptr->details == NULL) || (!IS_JOB_PENDING(job_ptr))) { *err_code = -700; *err_msg = "Job not pending, can't start"; error("wiki: Attempt to start job %u in state %s", jobid, job_state_string(job_ptr->job_state)); rc = -1; goto fini; } if (comment_ptr) { char *reserved = strstr(comment_ptr, "RESERVED:"); if (reserved) { reserved += 9; job_ptr->details->reserved_resources = strtol(reserved, NULL, 10); } xfree(job_ptr->comment); job_ptr->comment = xstrdup(comment_ptr); } if (task_cnt) { new_node_list = xstrdup(hostlist); if (node_name2bitmap(new_node_list, false, &new_bitmap) != 0) { *err_code = -700; *err_msg = "Invalid TASKLIST"; error("wiki: Attempt to set invalid node list for " "job %u, %s", jobid, hostlist); xfree(new_node_list); rc = -1; goto fini; } if (!bit_super_set(new_bitmap, avail_node_bitmap)) { /* Selected node is UP and not responding * or it just went DOWN */ *err_code = -700; *err_msg = "TASKLIST includes non-responsive node"; error("wiki: Attempt to use non-responsive nodes for " "job %u, %s", jobid, hostlist); xfree(new_node_list); FREE_NULL_BITMAP(new_bitmap); rc = -1; goto fini; } /* User excluded node list incompatible with Wiki * Exclude all nodes not explicitly requested */ FREE_NULL_BITMAP(job_ptr->details->exc_node_bitmap); job_ptr->details->exc_node_bitmap = bit_copy(new_bitmap); bit_not(job_ptr->details->exc_node_bitmap); } /* Build layout information from tasklist (assuming that Moab * sends a non-bracketed list of nodes, repeated as many times * as cpus should be used per node); at this point, node names * are comma-separated. This is _not_ a fast algorithm as it * performs many string compares. */ xfree(job_ptr->details->req_node_layout); if (task_cnt && cr_enabled) { uint16_t cpus_per_task = MAX(1, job_ptr->details->cpus_per_task); job_ptr->details->req_node_layout = (uint16_t *) xmalloc(bit_set_count(new_bitmap) * sizeof(uint16_t)); bsize = bit_size(new_bitmap); for (i = 0, ll = -1; i < bsize; i++) { if (!bit_test(new_bitmap, i)) continue; ll++; node_name = node_record_table_ptr[i].name; node_name_len = strlen(node_name); if (node_name_len == 0) continue; node_cur = tasklist; while (*node_cur) { if ((node_idx = strstr(node_cur, node_name))) { if ((node_idx[node_name_len] == ',') || (node_idx[node_name_len] == '\0')) { job_ptr->details-> req_node_layout[ll] += cpus_per_task; } node_cur = strchr(node_idx, ','); if (node_cur) continue; } break; } } } /* save and update job state to start now */ save_req_nodes = job_ptr->details->req_nodes; job_ptr->details->req_nodes = new_node_list; save_req_bitmap = job_ptr->details->req_node_bitmap; job_ptr->details->req_node_bitmap = new_bitmap; old_task_cnt = job_ptr->details->min_cpus; job_ptr->details->min_cpus = MAX(task_cnt, old_task_cnt); job_ptr->priority = 100000000; fini: unlock_slurmctld(job_write_lock); if (rc) return rc; /* No errors so far */ (void) schedule(INFINITE); /* provides own locking */ /* Check to insure the job was actually started */ lock_slurmctld(job_write_lock); if (job_ptr->job_id != jobid) job_ptr = find_job_record(jobid); if (job_ptr && (job_ptr->job_id == jobid) && (!IS_JOB_RUNNING(job_ptr))) { uint16_t wait_reason = 0; char *wait_string; if (IS_JOB_FAILED(job_ptr)) wait_string = "Invalid request, job aborted"; else { wait_reason = job_ptr->state_reason; if (wait_reason == WAIT_HELD) { /* some job is completing, slurmctld did * not even try to schedule this job */ wait_reason = WAIT_RESOURCES; } wait_string = job_reason_string(wait_reason); job_ptr->state_reason = WAIT_HELD; xfree(job_ptr->state_desc); } *err_code = -910 - wait_reason; snprintf(tmp_msg, sizeof(tmp_msg), "Could not start job %u(%s): %s", jobid, new_node_list, wait_string); *err_msg = tmp_msg; error("wiki: %s", tmp_msg); /* restore some of job state */ job_ptr->priority = 0; job_ptr->details->min_cpus = old_task_cnt; rc = -1; } if (job_ptr && (job_ptr->job_id == jobid) && job_ptr->details) { /* Restore required node list in case job requeued */ xfree(job_ptr->details->req_nodes); job_ptr->details->req_nodes = save_req_nodes; FREE_NULL_BITMAP(job_ptr->details->req_node_bitmap); job_ptr->details->req_node_bitmap = save_req_bitmap; FREE_NULL_BITMAP(job_ptr->details->exc_node_bitmap); xfree(job_ptr->details->req_node_layout); } else { error("wiki: start_job(%u) job missing", jobid); xfree(save_req_nodes); FREE_NULL_BITMAP(save_req_bitmap); } unlock_slurmctld(job_write_lock); schedule_node_save(); /* provides own locking */ schedule_job_save(); /* provides own locking */ return rc; }
static int _verify_job_ids(void) { job_info_t *job_ptr; int i, j, rc = 0; if (opt.job_cnt == 0) return rc; opt.job_found = xmalloc(sizeof(bool) * opt.job_cnt); opt.job_pend = xmalloc(sizeof(bool) * opt.job_cnt); job_ptr = job_buffer_ptr->job_array; for (i = 0; i < job_buffer_ptr->record_count; i++, job_ptr++) { /* NOTE: We re-use the job's "assoc_id" value as a flag to * record if the job is referenced in the job list supplied * by the user. */ job_ptr->assoc_id = 0; if (IS_JOB_FINISHED(job_ptr)) job_ptr->job_id = 0; if (job_ptr->job_id == 0) continue; for (j = 0; j < opt.job_cnt; j++) { if (opt.array_id[j] == NO_VAL) { if ((opt.job_id[j] == job_ptr->job_id) || ((opt.job_id[j] == job_ptr->array_job_id) && (opt.step_id[j] == SLURM_BATCH_SCRIPT))) { opt.job_found[j] = true; } } else if (opt.array_id[j] == INFINITE) { if (opt.job_id[j] == job_ptr->array_job_id) { opt.job_found[j] = true; } } else if (opt.job_id[j] != job_ptr->array_job_id) { continue; } else if (_is_task_in_job(job_ptr, opt.array_id[j])) { opt.job_found[j] = true; } if (opt.job_found[j]) { if (IS_JOB_PENDING(job_ptr)) opt.job_pend[j] = true; job_ptr->assoc_id = 1; } } if (job_ptr->assoc_id == 0) job_ptr->job_id = 0; } for (j = 0; j < opt.job_cnt; j++) { char *job_id_str = NULL; if (!opt.job_found[j]) rc = 1; else continue; if (opt.verbose < 0) { ; } else if (opt.array_id[j] == NO_VAL) { xstrfmtcat(job_id_str, "%u", opt.job_id[j]); } else if (opt.array_id[j] == INFINITE) { xstrfmtcat(job_id_str, "%u_*", opt.job_id[j]); } else { xstrfmtcat(job_id_str, "%u_%u", opt.job_id[j], opt.array_id[j]); } if (opt.verbose < 0) { ; } else if (opt.step_id[j] == SLURM_BATCH_SCRIPT) { error("Kill job error on job id %s: %s", job_id_str, slurm_strerror(ESLURM_INVALID_JOB_ID)); } else { error("Kill job error on job step id %s.%u: %s", job_id_str, opt.step_id[j], slurm_strerror(ESLURM_INVALID_JOB_ID)); } xfree(job_id_str); /* Avoid this job in the cancel_job logic */ opt.job_id[j] = 0; } return rc; }
/* * Attempt to allocate resources and begin file staging for pending jobs. */ extern int bb_p_job_try_stage_in(List job_queue) { bb_job_queue_rec_t *job_rec; List job_candidates; ListIterator job_iter; struct job_record *job_ptr; uint64_t bb_size; int rc; if (bb_state.bb_config.debug_flag) info("%s: %s", plugin_type, __func__); if (!bb_state.bb_config.start_stage_in) return SLURM_ERROR; /* Identify candidates to be allocated burst buffers */ job_candidates = list_create(bb_job_queue_del); job_iter = list_iterator_create(job_queue); while ((job_ptr = list_next(job_iter))) { if (!IS_JOB_PENDING(job_ptr) || (job_ptr->start_time == 0) || (job_ptr->burst_buffer == NULL) || (job_ptr->burst_buffer[0] == '\0')) continue; if (job_ptr->array_recs && (job_ptr->array_task_id == NO_VAL)) continue; bb_size = _get_bb_size(job_ptr); if (bb_size == 0) continue; job_rec = xmalloc(sizeof(bb_job_queue_rec_t)); job_rec->job_ptr = job_ptr; job_rec->bb_size = bb_size; list_push(job_candidates, job_rec); } list_iterator_destroy(job_iter); /* Sort in order of expected start time */ list_sort(job_candidates, bb_job_queue_sort); pthread_mutex_lock(&bb_state.bb_mutex); bb_set_use_time(&bb_state); job_iter = list_iterator_create(job_candidates); while ((job_rec = list_next(job_iter))) { job_ptr = job_rec->job_ptr; bb_size = job_rec->bb_size; if (bb_find_alloc_rec(&bb_state, job_ptr)) continue; rc = _test_size_limit(job_ptr, bb_size); if (rc == 1) continue; else if (rc == 2) break; _alloc_job_bb(job_ptr, bb_size); } list_iterator_destroy(job_iter); pthread_mutex_unlock(&bb_state.bb_mutex); FREE_NULL_LIST(job_candidates); return SLURM_SUCCESS; }
static void _cancel_jobid_by_state(uint32_t job_state, int filter_cnt, int *rc) { job_cancel_info_t *cancel_info; job_info_t *job_ptr; pthread_t dummy; int err, i, j; if (opt.job_cnt == 0) return; for (j = 0; j < opt.job_cnt; j++) { if (opt.job_id[j] == 0) continue; if ((job_state == JOB_PENDING) && !opt.job_pend[j]) continue; job_ptr = job_buffer_ptr->job_array; for (i = 0; i < job_buffer_ptr->record_count; i++, job_ptr++) { if (IS_JOB_FINISHED(job_ptr)) job_ptr->job_id = 0; if (job_ptr->job_id == 0) continue; if ((opt.step_id[j] != SLURM_BATCH_SCRIPT) && IS_JOB_PENDING(job_ptr)) { /* User specified #.# for step, but the job ID * may be job array leader with part of job * array running with other tasks pending */ continue; } opt.job_found[j] = false; if (opt.array_id[j] == NO_VAL) { if ((opt.job_id[j] == job_ptr->job_id) || ((opt.job_id[j] == job_ptr->array_job_id) && (opt.step_id[j] == SLURM_BATCH_SCRIPT))) { opt.job_found[j] = true; } } else if (opt.array_id[j] == INFINITE) { if (opt.job_id[j] == job_ptr->array_job_id) { opt.job_found[j] = true; } } else if (opt.job_id[j] != job_ptr->array_job_id) { continue; } else if (_is_task_in_job(job_ptr, opt.array_id[j])) { opt.job_found[j] = true; } if (!opt.job_found[j]) continue; if (opt.interactive && (_confirmation(job_ptr, opt.step_id[j]) == 0)) { job_ptr->job_id = 0; /* Don't check again */ continue; } slurm_mutex_lock(&num_active_threads_lock); num_active_threads++; while (num_active_threads > MAX_THREADS) { slurm_cond_wait(&num_active_threads_cond, &num_active_threads_lock); } slurm_mutex_unlock(&num_active_threads_lock); cancel_info = (job_cancel_info_t *) xmalloc(sizeof(job_cancel_info_t)); cancel_info->rc = rc; cancel_info->sig = opt.signal; cancel_info->num_active_threads = &num_active_threads; cancel_info->num_active_threads_lock = &num_active_threads_lock; cancel_info->num_active_threads_cond = &num_active_threads_cond; if (opt.step_id[j] == SLURM_BATCH_SCRIPT) { cancel_info->job_id_str = _build_jobid_str(job_ptr); err = pthread_create(&dummy, &attr, _cancel_job_id, cancel_info); if (err) /* Run in-line as needed */ _cancel_job_id(cancel_info); job_ptr->job_id = 0; } else { cancel_info->job_id = job_ptr->job_id; cancel_info->step_id = opt.step_id[j]; err = pthread_create(&dummy, &attr, _cancel_step_id, cancel_info); if (err) /* Run in-line as needed */ _cancel_step_id(cancel_info); } if (opt.interactive) { /* Print any error message for first job before * starting confirmation of next job */ slurm_mutex_lock(&num_active_threads_lock); while (num_active_threads > 0) { slurm_cond_wait(&num_active_threads_cond, &num_active_threads_lock); } slurm_mutex_unlock(&num_active_threads_lock); } } } }