/* Compute resource usage for the given job on all available resources
*
* IN: job_ptr - pointer to the job requesting resources
* IN: node_map - bitmap of available nodes
* IN/OUT: core_map - bitmap of available cores
* IN: cr_node_cnt - total number of nodes in the cluster
* IN: cr_type - resource type
* OUT: cpu_cnt - number of cpus that can be used by this job
* IN: test_only - ignore allocated memory check
* RET SLURM_SUCCESS index of selected node or -1 if none
*/
static int _get_res_usage(struct job_record *job_ptr, bitstr_t *node_map,
bitstr_t *core_map, uint32_t cr_node_cnt,
struct node_use_record *node_usage,
uint16_t cr_type, uint16_t **cpu_cnt_ptr,
bool test_only)
{
uint16_t *cpu_cnt, max_cpu_cnt = 0, part_lln_flag = 0;
int i, i_first, i_last;
int node_inx = -1;
if (cr_node_cnt != node_record_count) {
error("select/serial: node count inconsistent with slurmctld");
return SLURM_ERROR;
}
if (!job_ptr) {
error("select/serial: NULL job pointer");
return SLURM_ERROR;
}
if (job_ptr->part_ptr && (job_ptr->part_ptr->flags & PART_FLAG_LLN))
part_lln_flag = 1;
if (job_ptr->details && job_ptr->details->req_node_bitmap)
bit_and(node_map, job_ptr->details->req_node_bitmap);
cpu_cnt = xmalloc(cr_node_cnt * sizeof(uint16_t));
i_first = bit_ffs(node_map);
if (i_first >= 0)
i_last = bit_fls(node_map);
else
i_last = -2;
for (i = i_first; i <= i_last; i++) {
if (!bit_test(node_map, i))
continue;
cpu_cnt[i] = _can_job_run_on_node(job_ptr, core_map, i,
node_usage, cr_type,
test_only);
if (!(cr_type & CR_LLN) && !part_lln_flag && cpu_cnt[i]) {
bit_nclear(node_map, 0, (node_record_count - 1));
bit_set(node_map, i);
node_inx = i;
break; /* select/serial: only need one node */
}
}
if ((cr_type & CR_LLN) || part_lln_flag) {
for (i = i_first; i <= i_last; i++) {
if (cpu_cnt[i] > max_cpu_cnt) {
max_cpu_cnt = cpu_cnt[i];
node_inx = i;
}
}
if (node_inx >= 0) {
bit_nclear(node_map, 0, (node_record_count - 1));
bit_set(node_map, node_inx);
}
}
*cpu_cnt_ptr = cpu_cnt;
return node_inx;
}
/* Rebuild avail_feature_list for given node configuration structure */
extern void build_avail_feature_list(struct config_record *config_ptr)
{
node_feature_t *feature_ptr;
ListIterator feature_iter;
char *tmp_str, *token, *last = NULL;
/* Clear these nodes from the feature_list record,
* then restore as needed */
feature_iter = list_iterator_create(avail_feature_list);
bit_not(config_ptr->node_bitmap);
while ((feature_ptr = (node_feature_t *) list_next(feature_iter))) {
bit_and(feature_ptr->node_bitmap, config_ptr->node_bitmap);
}
list_iterator_destroy(feature_iter);
bit_not(config_ptr->node_bitmap);
if (config_ptr->feature) {
tmp_str = xstrdup(config_ptr->feature);
token = strtok_r(tmp_str, ",", &last);
while (token) {
_add_config_feature(avail_feature_list, token,
config_ptr->node_bitmap);
token = strtok_r(NULL, ",", &last);
}
xfree(tmp_str);
}
}
/* Return 1 if job fits in this row, else return 0 */
static int _job_fits_in_active_row(struct job_record *job_ptr,
struct gs_part *p_ptr)
{
job_resources_t *job_res = job_ptr->job_resrcs;
int count;
bitstr_t *job_map;
uint16_t job_gr_type;
if ((p_ptr->active_resmap == NULL) || (p_ptr->jobs_active == 0))
return 1;
job_gr_type = _get_part_gr_type(job_ptr->part_ptr);
if ((job_gr_type == GS_CPU2) || (job_gr_type == GS_CORE) ||
(job_gr_type == GS_SOCKET)) {
return job_fits_into_cores(job_res, p_ptr->active_resmap,
gs_bits_per_node);
}
/* job_gr_type == GS_NODE || job_gr_type == GS_CPU */
job_map = bit_copy(job_res->node_bitmap);
bit_and(job_map, p_ptr->active_resmap);
/* any set bits indicate contention for the same resource */
count = bit_set_count(job_map);
if (slurmctld_conf.debug_flags & DEBUG_FLAG_GANG)
info("gang: _job_fits_in_active_row: %d bits conflict", count);
FREE_NULL_BITMAP(job_map);
if (count == 0)
return 1;
if (job_gr_type == GS_CPU) {
/* For GS_CPU we check the CPU arrays */
return _can_cpus_fit(job_ptr, p_ptr);
}
return 0;
}
/* Create a reservation for a job in the future */
static void _add_reservation(uint32_t start_time, uint32_t end_reserve,
bitstr_t *res_bitmap,
node_space_map_t *node_space,
int *node_space_recs)
{
bool placed = false;
int i, j;
for (j=0; ; ) {
if (node_space[j].end_time > start_time) {
/* insert start entry record */
i = *node_space_recs;
node_space[i].begin_time = start_time;
node_space[i].end_time = node_space[j].end_time;
node_space[j].end_time = start_time;
node_space[i].avail_bitmap =
bit_copy(node_space[j].avail_bitmap);
node_space[i].next = node_space[j].next;
node_space[j].next = i;
(*node_space_recs)++;
placed = true;
}
if (node_space[j].end_time == start_time) {
/* no need to insert new start entry record */
placed = true;
}
if (placed == true) {
j = node_space[j].next;
if (j && (end_reserve < node_space[j].end_time)) {
/* insert end entry record */
i = *node_space_recs;
node_space[i].begin_time = end_reserve;
node_space[i].end_time = node_space[j].
end_time;
node_space[j].end_time = end_reserve;
node_space[i].avail_bitmap =
bit_copy(node_space[j].avail_bitmap);
node_space[i].next = node_space[j].next;
node_space[j].next = i;
(*node_space_recs)++;
}
break;
}
if ((j = node_space[j].next) == 0)
break;
}
for (j=0; ; ) {
if ((node_space[j].begin_time >= start_time) &&
(node_space[j].end_time <= end_reserve))
bit_and(node_space[j].avail_bitmap, res_bitmap);
if ((node_space[j].begin_time >= end_reserve) ||
((j = node_space[j].next) == 0))
break;
}
}
/* Compute resource usage for the given job on all available resources
*
* IN: job_ptr - pointer to the job requesting resources
* IN: node_map - bitmap of available nodes
* IN/OUT: core_map - bitmap of available cores
* IN: cr_node_cnt - total number of nodes in the cluster
* IN: cr_type - resource type
* OUT: cpu_cnt - number of cpus that can be used by this job
* IN: test_only - ignore allocated memory check
* RET SLURM_SUCCESS index of selected node or -1 if none
*/
static int _get_res_usage(struct job_record *job_ptr, bitstr_t *node_map,
bitstr_t *core_map, uint32_t cr_node_cnt,
struct node_use_record *node_usage,
uint16_t cr_type, uint16_t **cpu_cnt_ptr,
bool test_only)
{
uint16_t *cpu_cnt;
uint32_t n;
int i_first, i_last;
int node_inx = -1;
if (cr_node_cnt != node_record_count) {
error("select/serial: node count inconsistent with slurmctld");
return SLURM_ERROR;
}
if (job_ptr->details && job_ptr->details->req_node_bitmap)
bit_and(node_map, job_ptr->details->req_node_bitmap);
cpu_cnt = xmalloc(cr_node_cnt * sizeof(uint16_t));
i_first = bit_ffs(node_map);
if (i_first >= 0)
i_last = bit_fls(node_map);
else
i_last = -2;
for (n = i_first; n <= i_last; n++) {
if (!bit_test(node_map, n))
continue;
cpu_cnt[n] = _can_job_run_on_node(job_ptr, core_map, n,
node_usage, cr_type,
test_only);
if (cpu_cnt[n]) {
bit_nclear(node_map, 0, (node_record_count - 1));
bit_set(node_map, n);
node_inx = n;
break; /* select/serial: only need one node */
}
}
*cpu_cnt_ptr = cpu_cnt;
return node_inx;
}
/* Given a config_record with it's bitmap already set, update feature_list */
extern void build_config_feature_list(struct config_record *config_ptr)
{
struct features_record *feature_ptr;
ListIterator feature_iter;
int i, j;
char *tmp_str, *token, *last = NULL;
/* Clear these nodes from the feature_list record,
* then restore as needed */
feature_iter = list_iterator_create(feature_list);
if (feature_iter == NULL)
fatal("list_iterator_create malloc failure");
bit_not(config_ptr->node_bitmap);
while ((feature_ptr = (struct features_record *)
list_next(feature_iter))) {
bit_and(feature_ptr->node_bitmap, config_ptr->node_bitmap);
}
list_iterator_destroy(feature_iter);
bit_not(config_ptr->node_bitmap);
if (config_ptr->feature) {
i = strlen(config_ptr->feature) + 1; /* oversized */
tmp_str = xmalloc(i);
/* Remove white space from feature specification */
for (i=0, j=0; config_ptr->feature[i]; i++) {
if (!isspace((int)config_ptr->feature[i]))
tmp_str[j++] = config_ptr->feature[i];
}
if (i != j)
strcpy(config_ptr->feature, tmp_str);
token = strtok_r(tmp_str, ",", &last);
while (token) {
_add_config_feature(token, config_ptr->node_bitmap);
token = strtok_r(NULL, ",", &last);
}
xfree(tmp_str);
}
}
/* Release reserved ports for a job step
* RET SLURM_SUCCESS or an error code */
extern void resv_port_free(struct step_record *step_ptr)
{
int i, j;
if (step_ptr->resv_port_array == NULL)
return;
bit_not(step_ptr->step_node_bitmap);
for (i=0; i<step_ptr->resv_port_cnt; i++) {
if ((step_ptr->resv_port_array[i] < port_resv_min) ||
(step_ptr->resv_port_array[i] > port_resv_max))
continue;
j = step_ptr->resv_port_array[i] - port_resv_min;
bit_and(port_resv_table[j], step_ptr->step_node_bitmap);
}
bit_not(step_ptr->step_node_bitmap);
xfree(step_ptr->resv_port_array);
debug("freed ports %s for step %u.%u",
step_ptr->resv_ports,
step_ptr->job_ptr->job_id, step_ptr->step_id);
}
static char * _will_run_test2(uint32_t jobid, time_t start_time,
char *node_list,
uint32_t *preemptee, int preemptee_cnt,
int *err_code, char **err_msg)
{
struct job_record *job_ptr = NULL, *pre_ptr;
struct part_record *part_ptr;
bitstr_t *avail_bitmap = NULL, *resv_bitmap = NULL;
bitstr_t *exc_core_bitmap = NULL;
time_t start_res;
uint32_t min_nodes, max_nodes, req_nodes;
List preemptee_candidates = NULL, preempted_jobs = NULL;
time_t orig_start_time;
char *reply_msg = NULL;
int i, rc;
bool resv_overlap = false;
xassert(node_list);
debug2("wiki2: will_run2 job_id=%u start_time=%u node_list=%s",
jobid, (uint32_t)start_time, node_list);
job_ptr = find_job_record(jobid);
if (job_ptr == NULL) {
*err_code = -700;
*err_msg = "No such job";
error("wiki: Failed to find job %u", jobid);
return NULL;
}
if ((job_ptr->details == NULL) || (!IS_JOB_PENDING(job_ptr))) {
*err_code = -700;
*err_msg = "WillRun not applicable to non-pending job";
error("wiki: WillRun on non-pending job %u", jobid);
return NULL;
}
part_ptr = job_ptr->part_ptr;
if (part_ptr == NULL) {
*err_code = -700;
*err_msg = "Job lacks a partition";
error("wiki: Job %u lacks a partition", jobid);
return NULL;
}
if (node_name2bitmap(node_list, false, &avail_bitmap) != 0) {
*err_code = -700;
*err_msg = "Invalid available nodes value";
error("wiki: Attempt to set invalid available node "
"list for job %u, %s", jobid, node_list);
return NULL;
}
/* Enforce reservation: access control, time and nodes */
start_res = start_time;
rc = job_test_resv(job_ptr, &start_res, true, &resv_bitmap,
&exc_core_bitmap, &resv_overlap);
if (rc != SLURM_SUCCESS) {
*err_code = -730;
*err_msg = "Job denied access to reservation";
error("wiki: reservation access denied for job %u", jobid);
FREE_NULL_BITMAP(avail_bitmap);
FREE_NULL_BITMAP(exc_core_bitmap);
return NULL;
}
bit_and(avail_bitmap, resv_bitmap);
FREE_NULL_BITMAP(resv_bitmap);
/* Only consider nodes that are not DOWN or DRAINED */
bit_and(avail_bitmap, avail_node_bitmap);
/* Consider only nodes in this job's partition */
if (part_ptr->node_bitmap)
bit_and(avail_bitmap, part_ptr->node_bitmap);
else {
*err_code = -730;
*err_msg = "Job's partition has no nodes";
error("wiki: no nodes in partition %s for job %u",
part_ptr->name, jobid);
FREE_NULL_BITMAP(avail_bitmap);
FREE_NULL_BITMAP(exc_core_bitmap);
return NULL;
}
if (job_req_node_filter(job_ptr, avail_bitmap) != SLURM_SUCCESS) {
/* Job probably has invalid feature list */
*err_code = -730;
*err_msg = "Job's required features not available "
"on selected nodes";
error("wiki: job %u not runnable on hosts=%s",
jobid, node_list);
FREE_NULL_BITMAP(avail_bitmap);
FREE_NULL_BITMAP(exc_core_bitmap);
return NULL;
}
if (job_ptr->details->exc_node_bitmap) {
bit_not(job_ptr->details->exc_node_bitmap);
bit_and(avail_bitmap, job_ptr->details->exc_node_bitmap);
bit_not(job_ptr->details->exc_node_bitmap);
}
if ((job_ptr->details->req_node_bitmap) &&
(!bit_super_set(job_ptr->details->req_node_bitmap,
avail_bitmap))) {
*err_code = -730;
*err_msg = "Job's required nodes not available";
error("wiki: job %u not runnable on hosts=%s",
jobid, node_list);
FREE_NULL_BITMAP(avail_bitmap);
FREE_NULL_BITMAP(exc_core_bitmap);
return NULL;
}
min_nodes = MAX(job_ptr->details->min_nodes, part_ptr->min_nodes);
if (job_ptr->details->max_nodes == 0)
max_nodes = part_ptr->max_nodes;
else
max_nodes = MIN(job_ptr->details->max_nodes,
part_ptr->max_nodes);
max_nodes = MIN(max_nodes, 500000); /* prevent overflows */
if (job_ptr->details->max_nodes)
req_nodes = max_nodes;
else
req_nodes = min_nodes;
if (min_nodes > max_nodes) {
/* job's min_nodes exceeds partitions max_nodes */
*err_code = -730;
*err_msg = "Job's min_nodes > max_nodes";
error("wiki: job %u not runnable on hosts=%s",
jobid, node_list);
FREE_NULL_BITMAP(avail_bitmap);
FREE_NULL_BITMAP(exc_core_bitmap);
return NULL;
}
if (preemptee_cnt) {
preemptee_candidates = list_create(NULL);
for (i=0; i<preemptee_cnt; i++) {
if ((pre_ptr = find_job_record(preemptee[i])))
list_append(preemptee_candidates, pre_ptr);
}
}
orig_start_time = job_ptr->start_time;
rc = select_g_job_test(job_ptr, avail_bitmap, min_nodes, max_nodes,
req_nodes, SELECT_MODE_WILL_RUN,
preemptee_candidates, &preempted_jobs,
exc_core_bitmap);
FREE_NULL_LIST(preemptee_candidates);
if (rc == SLURM_SUCCESS) {
char *hostlist, *sep, tmp_str[128];
uint32_t pre_cnt = 0, proc_cnt = 0;
#ifdef HAVE_BG
select_g_select_jobinfo_get(job_ptr->select_jobinfo,
SELECT_JOBDATA_NODE_CNT, &proc_cnt);
#else
proc_cnt = job_ptr->total_cpus;
#endif
snprintf(tmp_str, sizeof(tmp_str),
"STARTINFO=%u TASKS=%u STARTTIME=%u NODES=",
job_ptr->job_id, proc_cnt,
(uint32_t) job_ptr->start_time);
xstrcat(reply_msg, tmp_str);
hostlist = bitmap2node_name(avail_bitmap);
xstrcat(reply_msg, hostlist);
xfree(hostlist);
if (preempted_jobs) {
while ((pre_ptr = list_pop(preempted_jobs))) {
if (pre_cnt++)
sep = ",";
else
sep = " PREEMPT=";
snprintf(tmp_str, sizeof(tmp_str), "%s%u",
sep, pre_ptr->job_id);
xstrcat(reply_msg, tmp_str);
}
FREE_NULL_LIST(preempted_jobs);
}
} else {
xstrcat(reply_msg, "Jobs not runable on selected nodes");
error("wiki: jobs not runnable on nodes");
}
/* Restore pending job's expected start time */
job_ptr->start_time = orig_start_time;
FREE_NULL_BITMAP(avail_bitmap);
FREE_NULL_BITMAP(exc_core_bitmap);
return reply_msg;
}
/* cr_job_test - does most of the real work for select_p_job_test(), which
* includes contiguous selection, load-leveling and max_share logic
*
* PROCEDURE:
*
* Step 1: compare nodes in "avail" bitmap with current node state data
* to find available nodes that match the job request
*
* Step 2: check resources in "avail" bitmap with allocated resources from
* higher priority partitions (busy resources are UNavailable)
*
* Step 3: select resource usage on remaining resources in "avail" bitmap
* for this job, with the placement influenced by existing
* allocations
*/
extern int cr_job_test(struct job_record *job_ptr, bitstr_t *bitmap, int mode,
uint16_t cr_type, enum node_cr_state job_node_req,
uint32_t cr_node_cnt,
struct part_res_record *cr_part_ptr,
struct node_use_record *node_usage)
{
static int gang_mode = -1;
int error_code = SLURM_SUCCESS;
bitstr_t *orig_map, *avail_cores, *free_cores;
bitstr_t *tmpcore = NULL;
bool test_only;
uint32_t c, i, j, k, n, csize, save_mem = 0;
job_resources_t *job_res;
struct job_details *details_ptr;
struct part_res_record *p_ptr, *jp_ptr;
uint16_t *cpu_count;
if (gang_mode == -1) {
if (slurm_get_preempt_mode() & PREEMPT_MODE_GANG)
gang_mode = 1;
else
gang_mode = 0;
}
details_ptr = job_ptr->details;
free_job_resources(&job_ptr->job_resrcs);
if (mode == SELECT_MODE_TEST_ONLY)
test_only = true;
else /* SELECT_MODE_RUN_NOW || SELECT_MODE_WILL_RUN */
test_only = false;
/* check node_state and update the node bitmap as necessary */
if (!test_only) {
error_code = _verify_node_state(cr_part_ptr, job_ptr,
bitmap, cr_type, node_usage,
job_node_req);
if (error_code != SLURM_SUCCESS)
return error_code;
}
if (select_debug_flags & DEBUG_FLAG_SELECT_TYPE) {
info("select/serial: evaluating job %u on %u nodes",
job_ptr->job_id, bit_set_count(bitmap));
}
orig_map = bit_copy(bitmap);
avail_cores = _make_core_bitmap(bitmap);
/* test to make sure that this job can succeed with all avail_cores
* if 'no' then return FAIL
* if 'yes' then we will seek the optimal placement for this job
* within avail_cores
*/
free_cores = bit_copy(avail_cores);
cpu_count = _select_nodes(job_ptr, bitmap, cr_node_cnt, free_cores,
node_usage, cr_type, test_only);
if (cpu_count == NULL) {
/* job cannot fit */
FREE_NULL_BITMAP(orig_map);
FREE_NULL_BITMAP(free_cores);
FREE_NULL_BITMAP(avail_cores);
if (select_debug_flags & DEBUG_FLAG_SELECT_TYPE) {
info("select/serial: cr_job_test: test 0 fail: "
"insufficient resources");
}
return SLURM_ERROR;
} else if (test_only) {
FREE_NULL_BITMAP(orig_map);
FREE_NULL_BITMAP(free_cores);
FREE_NULL_BITMAP(avail_cores);
xfree(cpu_count);
if (select_debug_flags & DEBUG_FLAG_SELECT_TYPE)
info("select/serial: cr_job_test: test 0 pass: "******"test_only");
return SLURM_SUCCESS;
}
if (cr_type == CR_MEMORY) {
/* CR_MEMORY does not care about existing CPU allocations,
* so we can jump right to job allocation from here */
goto alloc_job;
}
xfree(cpu_count);
if (select_debug_flags & DEBUG_FLAG_SELECT_TYPE) {
info("select/serial: cr_job_test: test 0 pass - "
"job fits on given resources");
}
/* now that we know that this job can run with the given resources,
* let's factor in the existing allocations and seek the optimal set
* of resources for this job. Here is the procedure:
*
* Step 1: Seek idle CPUs across all partitions. If successful then
* place job and exit. If not successful, then continue. Two
* related items to note:
* 1. Jobs that don't share CPUs finish with step 1.
* 2. The remaining steps assume sharing or preemption.
*
* Step 2: Remove resources that are in use by higher-priority
* partitions, and test that job can still succeed. If not
* then exit.
*
* Step 3: Seek idle nodes among the partitions with the same
* priority as the job's partition. If successful then
* goto Step 6. If not then continue:
*
* Step 4: Seek placement within the job's partition. Search
* row-by-row. If no placement if found, then exit. If a row
* is found, then continue:
*
* Step 5: Place job and exit. FIXME! Here is where we need a
* placement algorithm that recognizes existing job
* boundaries and tries to "overlap jobs" as efficiently
* as possible.
*
* Step 6: Place job and exit. FIXME! here is we use a placement
* algorithm similar to Step 5 on jobs from lower-priority
* partitions.
*/
/*** Step 1 ***/
bit_copybits(bitmap, orig_map);
bit_copybits(free_cores, avail_cores);
/* remove all existing allocations from free_cores */
tmpcore = bit_copy(free_cores);
for (p_ptr = cr_part_ptr; p_ptr; p_ptr = p_ptr->next) {
if (!p_ptr->row)
continue;
for (i = 0; i < p_ptr->num_rows; i++) {
if (!p_ptr->row[i].row_bitmap)
continue;
bit_copybits(tmpcore, p_ptr->row[i].row_bitmap);
bit_not(tmpcore); /* set bits now "free" resources */
bit_and(free_cores, tmpcore);
}
}
cpu_count = _select_nodes(job_ptr, bitmap, cr_node_cnt, free_cores,
node_usage, cr_type, test_only);
if (cpu_count) {
/* job fits! We're done. */
if (select_debug_flags & DEBUG_FLAG_SELECT_TYPE) {
info("select/serial: cr_job_test: test 1 pass - "
"idle resources found");
}
goto alloc_job;
}
if ((gang_mode == 0) && (job_node_req == NODE_CR_ONE_ROW)) {
/* This job CANNOT share CPUs regardless of priority,
* so we fail here. Note that Shared=EXCLUSIVE was already
* addressed in _verify_node_state() and job preemption
* removes jobs from simulated resource allocation map
* before this point. */
if (select_debug_flags & DEBUG_FLAG_SELECT_TYPE) {
info("select/serial: cr_job_test: test 1 fail - "
"no idle resources available");
}
goto alloc_job;
}
if (select_debug_flags & DEBUG_FLAG_SELECT_TYPE) {
info("select/serial: cr_job_test: test 1 fail - "
"not enough idle resources");
}
/*** Step 2 ***/
bit_copybits(bitmap, orig_map);
bit_copybits(free_cores, avail_cores);
for (jp_ptr = cr_part_ptr; jp_ptr; jp_ptr = jp_ptr->next) {
if (jp_ptr->part_ptr == job_ptr->part_ptr)
break;
}
if (!jp_ptr) {
fatal("select/serial: could not find partition for job %u",
job_ptr->job_id);
return SLURM_ERROR; /* Fix CLANG false positive */
}
/* remove existing allocations (jobs) from higher-priority partitions
* from avail_cores */
for (p_ptr = cr_part_ptr; p_ptr; p_ptr = p_ptr->next) {
if ((p_ptr->part_ptr->priority <= jp_ptr->part_ptr->priority) &&
(p_ptr->part_ptr->preempt_mode != PREEMPT_MODE_OFF))
continue;
if (!p_ptr->row)
continue;
for (i = 0; i < p_ptr->num_rows; i++) {
if (!p_ptr->row[i].row_bitmap)
continue;
bit_copybits(tmpcore, p_ptr->row[i].row_bitmap);
bit_not(tmpcore); /* set bits now "free" resources */
bit_and(free_cores, tmpcore);
}
}
/* make these changes permanent */
bit_copybits(avail_cores, free_cores);
cpu_count = _select_nodes(job_ptr, bitmap, cr_node_cnt, free_cores,
node_usage, cr_type, test_only);
if (!cpu_count) {
/* job needs resources that are currently in use by
* higher-priority jobs, so fail for now */
if (select_debug_flags & DEBUG_FLAG_SELECT_TYPE) {
info("select/serial: cr_job_test: test 2 fail - "
"resources busy with higher priority jobs");
}
goto alloc_job;
}
xfree(cpu_count);
if (select_debug_flags & DEBUG_FLAG_SELECT_TYPE) {
info("select/serial: cr_job_test: test 2 pass - "
"available resources for this priority");
}
/*** Step 3 ***/
bit_copybits(bitmap, orig_map);
bit_copybits(free_cores, avail_cores);
/* remove existing allocations (jobs) from same-priority partitions
* from avail_cores */
for (p_ptr = cr_part_ptr; p_ptr; p_ptr = p_ptr->next) {
if (p_ptr->part_ptr->priority != jp_ptr->part_ptr->priority)
continue;
if (!p_ptr->row)
continue;
for (i = 0; i < p_ptr->num_rows; i++) {
if (!p_ptr->row[i].row_bitmap)
continue;
bit_copybits(tmpcore, p_ptr->row[i].row_bitmap);
bit_not(tmpcore); /* set bits now "free" resources */
bit_and(free_cores, tmpcore);
}
}
cpu_count = _select_nodes(job_ptr, bitmap, cr_node_cnt, free_cores,
node_usage, cr_type, test_only);
if (cpu_count) {
/* jobs from low-priority partitions are the only thing left
* in our way. for now we'll ignore them, but FIXME: we need
* a good placement algorithm here that optimizes "job overlap"
* between this job (in these idle nodes) and the low-priority
* jobs */
if (select_debug_flags & DEBUG_FLAG_SELECT_TYPE) {
info("select/serial: cr_job_test: test 3 pass - "
"found resources");
}
goto alloc_job;
}
if (select_debug_flags & DEBUG_FLAG_SELECT_TYPE) {
info("select/serial: cr_job_test: test 3 fail - "
"not enough idle resources in same priority");
}
/*** Step 4 ***/
/* try to fit the job into an existing row
*
* tmpcore = worker core_bitmap
* free_cores = core_bitmap to be built
* avail_cores = static core_bitmap of all available cores
*/
if (!jp_ptr || !jp_ptr->row) {
/* there's no existing jobs in this partition, so place
* the job in avail_cores. FIXME: still need a good
* placement algorithm here that optimizes "job overlap"
* between this job (in these idle nodes) and existing
* jobs in the other partitions with <= priority to
* this partition */
bit_copybits(bitmap, orig_map);
bit_copybits(free_cores, avail_cores);
cpu_count = _select_nodes(job_ptr, bitmap, cr_node_cnt,
free_cores, node_usage, cr_type,
test_only);
if (select_debug_flags & DEBUG_FLAG_SELECT_TYPE) {
info("select/serial: cr_job_test: test 4 pass - "
"first row found");
}
goto alloc_job;
}
cr_sort_part_rows(jp_ptr);
c = jp_ptr->num_rows;
if (job_node_req != NODE_CR_AVAILABLE)
c = 1;
for (i = 0; i < c; i++) {
if (!jp_ptr->row[i].row_bitmap)
break;
bit_copybits(bitmap, orig_map);
bit_copybits(free_cores, avail_cores);
bit_copybits(tmpcore, jp_ptr->row[i].row_bitmap);
bit_not(tmpcore);
bit_and(free_cores, tmpcore);
cpu_count = _select_nodes(job_ptr, bitmap, cr_node_cnt,
free_cores, node_usage, cr_type,
test_only);
if (cpu_count) {
if (select_debug_flags & DEBUG_FLAG_SELECT_TYPE) {
info("select/serial: cr_job_test: "
"test 4 pass - row %i", i);
}
break;
}
if (select_debug_flags & DEBUG_FLAG_SELECT_TYPE) {
info("select/serial: cr_job_test: "
"test 4 fail - row %i", i);
}
}
if ((i < c) && !jp_ptr->row[i].row_bitmap) {
/* we've found an empty row, so use it */
bit_copybits(bitmap, orig_map);
bit_copybits(free_cores, avail_cores);
if (select_debug_flags & DEBUG_FLAG_SELECT_TYPE) {
info("select/serial: cr_job_test: "
"test 4 trying empty row %i",i);
}
cpu_count = _select_nodes(job_ptr, bitmap, cr_node_cnt,
free_cores, node_usage, cr_type,
test_only);
}
if (!cpu_count) {
/* job can't fit into any row, so exit */
if (select_debug_flags & DEBUG_FLAG_SELECT_TYPE) {
info("select/serial: cr_job_test: test 4 fail - "
"busy partition");
}
goto alloc_job;
}
/*** CONSTRUCTION ZONE FOR STEPs 5 AND 6 ***
* Note that while the job may have fit into a row, it should
* still be run through a good placement algorithm here that
* optimizes "job overlap" between this job (in these idle nodes)
* and existing jobs in the other partitions with <= priority to
* this partition */
alloc_job:
/* at this point we've found a good set of
* bits to allocate to this job:
* - bitmap is the set of nodes to allocate
* - free_cores is the set of allocated cores
* - cpu_count is the number of cpus per allocated node
*
* Next steps are to cleanup the worker variables,
* create the job_resources struct,
* distribute the job on the bits, and exit
*/
FREE_NULL_BITMAP(orig_map);
FREE_NULL_BITMAP(avail_cores);
FREE_NULL_BITMAP(tmpcore);
if (!cpu_count) {
/* we were sent here to cleanup and exit */
FREE_NULL_BITMAP(free_cores);
if (select_debug_flags & DEBUG_FLAG_SELECT_TYPE) {
info("select/serial: exiting cr_job_test with no "
"allocation");
}
return SLURM_ERROR;
}
/* At this point we have:
* - a bitmap of selected nodes
* - a free_cores bitmap of usable cores on each selected node
* - a per-alloc-node cpu_count array
*/
if ((mode != SELECT_MODE_WILL_RUN) && (job_ptr->part_ptr == NULL))
error_code = EINVAL;
if ((error_code == SLURM_SUCCESS) && (mode == SELECT_MODE_WILL_RUN))
job_ptr->total_cpus = 1;
if ((error_code != SLURM_SUCCESS) || (mode != SELECT_MODE_RUN_NOW)) {
FREE_NULL_BITMAP(free_cores);
xfree(cpu_count);
return error_code;
}
n = bit_ffs(bitmap);
if (n < 0) {
FREE_NULL_BITMAP(free_cores);
xfree(cpu_count);
return error_code;
}
if (select_debug_flags & DEBUG_FLAG_SELECT_TYPE) {
info("select/serial: cr_job_test: distributing job %u",
job_ptr->job_id);
}
/** create the struct_job_res **/
job_res = create_job_resources();
job_res->node_bitmap = bit_copy(bitmap);
job_res->nodes = bitmap2node_name(bitmap);
job_res->nhosts = bit_set_count(bitmap);
job_res->ncpus = job_res->nhosts;
if (job_ptr->details->ntasks_per_node)
job_res->ncpus *= details_ptr->ntasks_per_node;
job_res->ncpus = MAX(job_res->ncpus,
details_ptr->min_cpus);
job_res->ncpus = MAX(job_res->ncpus,
details_ptr->pn_min_cpus);
job_res->node_req = job_node_req;
job_res->cpus = cpu_count;
job_res->cpus_used = xmalloc(job_res->nhosts *
sizeof(uint16_t));
job_res->memory_allocated = xmalloc(job_res->nhosts *
sizeof(uint32_t));
job_res->memory_used = xmalloc(job_res->nhosts *
sizeof(uint32_t));
/* store the hardware data for the selected nodes */
error_code = build_job_resources(job_res, node_record_table_ptr,
select_fast_schedule);
if (error_code != SLURM_SUCCESS) {
free_job_resources(&job_res);
FREE_NULL_BITMAP(free_cores);
return error_code;
}
c = 0;
csize = bit_size(job_res->core_bitmap);
j = cr_get_coremap_offset(n);
k = cr_get_coremap_offset(n + 1);
for (; j < k; j++, c++) {
if (!bit_test(free_cores, j))
continue;
if (c >= csize) {
error("select/serial: cr_job_test "
"core_bitmap index error on node %s",
select_node_record[n].node_ptr->name);
drain_nodes(select_node_record[n].node_ptr->name,
"Bad core count", getuid());
free_job_resources(&job_res);
FREE_NULL_BITMAP(free_cores);
return SLURM_ERROR;
}
bit_set(job_res->core_bitmap, c);
break;
}
if (select_debug_flags & DEBUG_FLAG_SELECT_TYPE) {
info("select/serial: cr_job_test: job %u ncpus %u cbits %u/%d "
"nbits %u", job_ptr->job_id,
job_res->ncpus, bit_set_count(free_cores), 1,
job_res->nhosts);
}
FREE_NULL_BITMAP(free_cores);
/* distribute the tasks and clear any unused cores */
job_ptr->job_resrcs = job_res;
error_code = cr_dist(job_ptr, cr_type);
if (error_code != SLURM_SUCCESS) {
free_job_resources(&job_ptr->job_resrcs);
return error_code;
}
/* translate job_res->cpus array into format with rep count */
job_ptr->total_cpus = build_job_resources_cpu_array(job_res);
if (!(cr_type & CR_MEMORY))
return error_code;
/* load memory allocated array */
save_mem = details_ptr->pn_min_memory;
if (save_mem & MEM_PER_CPU) {
/* memory is per-cpu */
save_mem &= (~MEM_PER_CPU);
job_res->memory_allocated[0] = job_res->cpus[0] * save_mem;
} else {
/* memory is per-node */
job_res->memory_allocated[0] = save_mem;
}
return error_code;
}
void CPU::exec16(const Instruction16 &insn) {
switch(insn.OP) {
case 0x00:
switch(insn.rform.func4) {
// nop!
case 0x00: /* noting */ break;
// mlfh! rDg0, rAg1
case 0x01: g0[insn.rform.rD] = g1[insn.rform.rA]; break;
// mhfl! rDg1, rAg0
case 0x02: g1[insn.rform.rD] = g0[insn.rform.rA]; break;
// mv! rDg0, rAg0
case 0x03: g0[insn.rform.rD] = g0[insn.rform.rA]; break;
// br{cond}! rAg0
case 0x04: if(conditional(insn.rform.rD)) branch(g0[insn.rform.rA] - 2, false); break;
// t{cond}!
case 0x05: T = conditional(insn.rform.rD); break;
default: debugDump();
}
break;
case 0x01: {
uint32_t &rA = g0[insn.rform.rA];
// uint32_t &rD = g0[insn.rform.rD];
switch(insn.rform.func4) {
// mtce{lh}! rA
case 0x00:
switch(insn.rform.rD) {
case 0x00: CEL = rA; break;
case 0x01: CEH = rA; break;
}
break;
// mfce{lh}! rA
case 0x01:
switch(insn.rform.rD) {
case 0x00: rA = CEL; break;
case 0x01: rA = CEH; break;
}
break;
default: debugDump();
}
} break;
case 0x02: {
uint32_t &rA = g0[insn.rform.rA];
uint32_t &rD = g0[insn.rform.rD];
uint32_t &rAh = g0[insn.rhform.rA];
uint32_t &rDh = g[insn.rhform.H][insn.rhform.rD];
switch(insn.rform.func4) {
// add! rDg0, rAg0
case 0x00: rD = add(rD, rA, true); break;
// sub! rDg0, rAg0
case 0x01: rD = sub(rD, rA, true); break;
// neg! rDg0, rAg0
case 0x02: rD = sub(0, rA, true); break;
// cmp! rDg0, rAg0
case 0x03: sub(rD, rA, true); break;
// and! rDg0, rAg0
case 0x04: rD = bit_and(rD, rA, true); break;
// or! rDg0, rAg0
case 0x05: rD = bit_or(rD, rA, true); break;
// not! rDg0, rAg0
case 0x06: rD = bit_xor(rA, ~0, true); break;
// xor! rDg0, rAg0
case 0x07: rD = bit_xor(rD, rA, true); break;
// lw! rDg0, [rAg0]
case 0x08: rD = miu.readU32(rA); break;
// lh! rDg0, [rAg0]
case 0x09: rD = sign_extend(miu.readU16(rA), 16); break;
// pop! rDgh, [rAg0]
case 0x0A: rDh = miu.readU32(rAh); rAh += 4; break;
// lbu! rDg0, [rAg0]
case 0x0B: rD = miu.readU8(rA); break;
// sw! rDg0, [rAg0]
case 0x0C: miu.writeU32(rA, rD); break;
// sh! rDg0, [rAg0]
case 0x0D: miu.writeU16(rA, rD); break;
// push! rDgh, [rAg0]
case 0x0E: miu.writeU32(rAh -= 4, rDh); break;
// sb! rDg0, [rAg0]
case 0x0F: miu.writeU8(rA, rD); break;
}
} break;
case 0x03: {
// j[l]! imm11
if(insn.jform.LK)
link();
pc &= 0xFFFFF000;
pc |= (insn.jform.Disp11 << 1) - 2;
} break;
case 0x04: {
// b{cond}! imm8
if(conditional(insn.bxform.EC))
pc += (sign_extend(insn.bxform.Imm8, 8) << 1) - 2;
} break;
case 0x05:
// ldiu! imm8
g0[insn.iform2.rD] = insn.iform2.Imm8;
break;
case 0x06: {
uint32_t &rD = g0[insn.iform1.rD];
uint32_t imm = 1 << insn.iform1.Imm5;
switch(insn.iform1.func3) {
// srli! rD, imm5
case 0x03: rD = srl(rD, insn.iform1.Imm5, true); break;
// bitclr! rD, imm5
case 0x04: rD = bit_and(rD, ~imm, true); break;
// bitset! rD, imm5
case 0x05: rD = bit_or(rD, imm, true); break;
// bittst! rD, imm5
case 0x06: bit_and(rD, imm, true); break;
default: debugDump();
}
} break;
case 0x07: {
uint32_t &rD = g0[insn.iform1.rD];
uint32_t imm = insn.iform1.Imm5 << 2;
switch(insn.iform1.func3) {
// lwp! rDg0, imm
case 0x00: rD = miu.readU32(r2 + imm); break;
// lbup! rDg0, imm
case 0x01: rD = miu.readU8(r2 + imm); break;
// lhp! rDg0, imm
case 0x03: rD = sign_extend(miu.readU8(r2 + imm), 16); break;
// swp! rDg0, imm
case 0x04: miu.writeU32(r2 + imm, rD); break;
// shp! rDg0, imm
case 0x05: miu.writeU16(r2 + imm, rD); break;
// sbp! rDg0, imm
case 0x07: miu.writeU32(r2 + imm, rD); break;
default: debugDump();
}
} break;
default: debugDump();
}
}
BitMap& BitMap::operator &=(const BitMap& b){
return bit_and(b);
}
static char * _will_run_test(uint32_t jobid, time_t start_time,
char *node_list, int *err_code, char **err_msg)
{
struct job_record *job_ptr = NULL;
struct part_record *part_ptr;
bitstr_t *avail_bitmap = NULL, *resv_bitmap = NULL;
bitstr_t *exc_core_bitmap = NULL;
char *hostlist, *reply_msg = NULL;
uint32_t min_nodes, max_nodes, req_nodes;
int rc;
time_t start_res, orig_start_time;
List preemptee_candidates;
debug2("wiki2: will_run job_id=%u start_time=%u node_list=%s",
jobid, (uint32_t)start_time, node_list);
job_ptr = find_job_record(jobid);
if (job_ptr == NULL) {
*err_code = -700;
*err_msg = "No such job";
error("wiki: Failed to find job %u", jobid);
return NULL;
}
if ((job_ptr->details == NULL) || (!IS_JOB_PENDING(job_ptr))) {
*err_code = -700;
*err_msg = "WillRun not applicable to non-pending job";
error("wiki: WillRun on non-pending job %u", jobid);
return NULL;
}
part_ptr = job_ptr->part_ptr;
if (part_ptr == NULL) {
*err_code = -700;
*err_msg = "Job lacks a partition";
error("wiki: Job %u lacks a partition", jobid);
return NULL;
}
if ((node_list == NULL) || (node_list[0] == '\0')) {
/* assume all nodes available to job for testing */
avail_bitmap = bit_copy(avail_node_bitmap);
} else if (node_name2bitmap(node_list, false, &avail_bitmap) != 0) {
*err_code = -700;
*err_msg = "Invalid available nodes value";
error("wiki: Attempt to set invalid available node "
"list for job %u, %s", jobid, node_list);
return NULL;
}
/* Enforce reservation: access control, time and nodes */
start_res = start_time;
rc = job_test_resv(job_ptr, &start_res, true, &resv_bitmap,
&exc_core_bitmap);
if (rc != SLURM_SUCCESS) {
*err_code = -730;
*err_msg = "Job denied access to reservation";
error("wiki: reservation access denied for job %u", jobid);
FREE_NULL_BITMAP(avail_bitmap);
return NULL;
}
start_time = MAX(start_time, start_res);
bit_and(avail_bitmap, resv_bitmap);
FREE_NULL_BITMAP(resv_bitmap);
/* Only consider nodes that are not DOWN or DRAINED */
bit_and(avail_bitmap, avail_node_bitmap);
/* Consider only nodes in this job's partition */
if (part_ptr->node_bitmap)
bit_and(avail_bitmap, part_ptr->node_bitmap);
else {
*err_code = -730;
*err_msg = "Job's partition has no nodes";
error("wiki: no nodes in partition %s for job %u",
part_ptr->name, jobid);
FREE_NULL_BITMAP(avail_bitmap);
return NULL;
}
if (job_req_node_filter(job_ptr, avail_bitmap) != SLURM_SUCCESS) {
/* Job probably has invalid feature list */
*err_code = -730;
*err_msg = "Job's required features not available "
"on selected nodes";
error("wiki: job %u not runnable on hosts=%s",
jobid, node_list);
FREE_NULL_BITMAP(avail_bitmap);
return NULL;
}
if (job_ptr->details->exc_node_bitmap) {
bit_not(job_ptr->details->exc_node_bitmap);
bit_and(avail_bitmap, job_ptr->details->exc_node_bitmap);
bit_not(job_ptr->details->exc_node_bitmap);
}
if ((job_ptr->details->req_node_bitmap) &&
(!bit_super_set(job_ptr->details->req_node_bitmap,
avail_bitmap))) {
*err_code = -730;
*err_msg = "Job's required nodes not available";
error("wiki: job %u not runnable on hosts=%s",
jobid, node_list);
FREE_NULL_BITMAP(avail_bitmap);
return NULL;
}
min_nodes = MAX(job_ptr->details->min_nodes, part_ptr->min_nodes);
if (job_ptr->details->max_nodes == 0)
max_nodes = part_ptr->max_nodes;
else
max_nodes = MIN(job_ptr->details->max_nodes,
part_ptr->max_nodes);
max_nodes = MIN(max_nodes, 500000); /* prevent overflows */
if (job_ptr->details->max_nodes)
req_nodes = max_nodes;
else
req_nodes = min_nodes;
if (min_nodes > max_nodes) {
/* job's min_nodes exceeds partitions max_nodes */
*err_code = -730;
*err_msg = "Job's min_nodes > max_nodes";
error("wiki: job %u not runnable on hosts=%s",
jobid, node_list);
FREE_NULL_BITMAP(avail_bitmap);
return NULL;
}
preemptee_candidates = slurm_find_preemptable_jobs(job_ptr);
orig_start_time = job_ptr->start_time;
rc = select_g_job_test(job_ptr, avail_bitmap,
min_nodes, max_nodes, req_nodes,
SELECT_MODE_WILL_RUN,
preemptee_candidates, NULL, exc_core_bitmap);
if (preemptee_candidates)
list_destroy(preemptee_candidates);
if (rc == SLURM_SUCCESS) {
char tmp_str[128];
*err_code = 0;
uint32_t proc_cnt = 0;
xstrcat(reply_msg, "STARTINFO=");
#ifdef HAVE_BG
select_g_select_jobinfo_get(job_ptr->select_jobinfo,
SELECT_JOBDATA_NODE_CNT,
&proc_cnt);
#else
proc_cnt = job_ptr->total_cpus;
#endif
snprintf(tmp_str, sizeof(tmp_str), "%u:%u@%u,",
jobid, proc_cnt, (uint32_t) job_ptr->start_time);
xstrcat(reply_msg, tmp_str);
hostlist = bitmap2node_name(avail_bitmap);
xstrcat(reply_msg, hostlist);
xfree(hostlist);
} else {
xstrcat(reply_msg, "Jobs not runable on selected nodes");
error("wiki: jobs not runnable on nodes");
}
/* Restore pending job's expected start time */
job_ptr->start_time = orig_start_time;
FREE_NULL_BITMAP(avail_bitmap);
return reply_msg;
}
/*
* finds the best match for a given job request
*
*
* OUT - block_id of matched block, NULL otherwise
* returns 1 for error (no match)
*
*/
static int _find_best_block_match(List block_list,
int *blocks_added,
struct job_record* job_ptr,
bitstr_t* slurm_block_bitmap,
uint32_t min_nodes, uint32_t max_nodes,
uint32_t req_nodes,
bg_record_t** found_bg_record,
uint16_t query_mode, int avail_cpus)
{
bg_record_t *bg_record = NULL;
uint16_t req_geometry[SYSTEM_DIMENSIONS];
uint16_t target_size = 0;
uint32_t req_procs = job_ptr->details->min_cpus;
select_ba_request_t request;
int i, dim;
int overlap_check = 0;
int allow = 0;
int check_image = 1;
uint32_t max_cpus = job_ptr->details->max_cpus;
char tmp_char[256];
static int total_cpus = 0;
int rc = SLURM_SUCCESS;
int create_try = 0;
List overlapped_list = NULL;
bool is_test = SELECT_IS_TEST(query_mode);
if (!total_cpus) {
int *cluster_dims = select_g_ba_get_dims();
total_cpus = 1;
for (dim=0; dim<SYSTEM_DIMENSIONS; dim++)
total_cpus *= cluster_dims[dim];
total_cpus *= bg_conf->cpus_per_mp;
}
if (req_nodes > max_nodes) {
error("can't run this job max mps is %u asking for %u",
max_nodes, req_nodes);
return SLURM_ERROR;
}
if (!is_test && (req_procs > avail_cpus)) {
if (bg_conf->slurm_debug_flags & DEBUG_FLAG_BG_PICK)
info("asking for %u I only have %d",
req_procs, avail_cpus);
return SLURM_ERROR;
}
if (!block_list) {
error("_find_best_block_match: There is no block_list");
return SLURM_ERROR;
}
memset(&request, 0, sizeof(select_ba_request_t));
get_select_jobinfo(job_ptr->select_jobinfo->data,
SELECT_JOBDATA_CONN_TYPE, &request.conn_type);
get_select_jobinfo(job_ptr->select_jobinfo->data,
SELECT_JOBDATA_GEOMETRY, &req_geometry);
get_select_jobinfo(job_ptr->select_jobinfo->data,
SELECT_JOBDATA_ROTATE, &request.rotate);
if ((rc = _check_images(job_ptr, &request)) == SLURM_ERROR)
goto end_it;
if (req_geometry[0] != 0 && req_geometry[0] != (uint16_t)NO_VAL) {
char tmp_geo[SYSTEM_DIMENSIONS+1];
target_size = 1;
for (i=0; i<SYSTEM_DIMENSIONS; i++) {
target_size *= req_geometry[i];
tmp_geo[i] = alpha_num[req_geometry[i]];
}
tmp_geo[i] = '\0';
if (target_size != min_nodes) {
debug2("min_nodes not set correctly %u "
"should be %u from %s",
min_nodes, target_size,
tmp_geo);
min_nodes = target_size;
}
if (!req_nodes)
req_nodes = min_nodes;
} else {
req_geometry[0] = (uint16_t)NO_VAL;
target_size = min_nodes;
}
*found_bg_record = NULL;
allow = 0;
memcpy(request.geometry, req_geometry, sizeof(req_geometry));
request.deny_pass = (uint16_t)NO_VAL;
request.save_name = NULL;
request.size = target_size;
request.procs = req_procs;
request.elongate = request.rotate;
/* request.start[0] = 1; */
/* request.start[1] = 2; */
/* request.start[2] = 0; */
/* request.start[3] = 2; */
/* request.start_req = 1; */
if (job_ptr->details->req_node_bitmap)
request.avail_mp_bitmap = job_ptr->details->req_node_bitmap;
else
request.avail_mp_bitmap = slurm_block_bitmap;
/* since we only look at procs after this and not nodes we
* need to set a max_cpus if given
*/
if (max_cpus == (uint32_t)NO_VAL)
max_cpus = max_nodes * bg_conf->cpus_per_mp;
while (1) {
/* Here we are creating a list of all the blocks that
* have overlapped jobs so if we don't find one that
* works we will have can look and see the earliest
* the job can start. This doesn't apply to Dynamic mode.
*/
if (is_test && SELECT_IS_CHECK_FULL_SET(query_mode)
&& bg_conf->layout_mode != LAYOUT_DYNAMIC)
overlapped_list = list_create(NULL);
bg_record = _find_matching_block(block_list,
job_ptr,
slurm_block_bitmap,
&request,
max_cpus,
&allow, check_image,
overlap_check,
overlapped_list,
query_mode);
/* this could get altered in _find_matching_block so we
need to reset it */
memcpy(request.geometry, req_geometry, sizeof(req_geometry));
if (!bg_record && overlapped_list
&& list_count(overlapped_list)) {
ListIterator itr =
list_iterator_create(overlapped_list);
bg_record_t *tmp_rec = NULL;
while ((tmp_rec = list_next(itr))) {
if (!bg_record ||
(tmp_rec->job_ptr->end_time <
bg_record->job_ptr->end_time))
bg_record = tmp_rec;
}
list_iterator_destroy(itr);
}
if (overlapped_list)
list_destroy(overlapped_list);
/* set the bitmap and do other allocation activities */
if (bg_record) {
#ifdef HAVE_BG_L_P
if (!is_test) {
if (bridge_block_check_mp_states(
bg_record->bg_block_id, 1)
!= SLURM_SUCCESS) {
/* check_block_mp_states will
set this block in the main
list to an error state, but
we aren't looking
at the main list, so we
need to set this copy of
the block in an
error state as well.
*/
bg_record->job_running =
BLOCK_ERROR_STATE;
bg_record->state |= BG_BLOCK_ERROR_FLAG;
error("_find_best_block_match: Picked "
"block (%s) had some issues with "
"hardware, trying a different "
"one.",
bg_record->bg_block_id);
continue;
}
}
#endif
format_node_name(bg_record, tmp_char, sizeof(tmp_char));
debug("_find_best_block_match %s <%s>",
bg_record->bg_block_id, tmp_char);
bit_and(slurm_block_bitmap, bg_record->mp_bitmap);
rc = SLURM_SUCCESS;
*found_bg_record = bg_record;
goto end_it;
}
/* see if we can just reset the image and reboot the block */
if (allow) {
check_image = 0;
allow = 0;
continue;
}
check_image = 1;
/* all these assume that the *bg_record is NULL */
if (bg_conf->layout_mode == LAYOUT_OVERLAP
&& !is_test && overlap_check < 2) {
overlap_check++;
continue;
}
if (create_try || bg_conf->layout_mode != LAYOUT_DYNAMIC)
goto no_match;
if ((rc = _dynamically_request(block_list, blocks_added,
&request,
job_ptr->details->req_nodes,
query_mode))
== SLURM_SUCCESS) {
create_try = 1;
continue;
}
/* Only look at the full system if we aren't going to
preempt jobs later and look.
*/
if (is_test && SELECT_IS_CHECK_FULL_SET(query_mode)) {
List new_blocks = NULL;
List job_list = list_create(NULL);
ListIterator itr = NULL;
if (bg_conf->slurm_debug_flags & DEBUG_FLAG_BG_PICK)
info("trying with empty machine");
/* Here we need to make sure the blocks in the
job list are those in the block list so go
through and grab them and add them to a
separate list.
*/
itr = list_iterator_create(block_list);
while ((bg_record = list_next(itr))) {
if (bg_record->job_running != NO_JOB_RUNNING)
list_append(job_list, bg_record);
/* Since the error blocks are at the
end we only really need to look at
the first one to make sure it will
work, so don't add more than one to
the job list.
We do need to check for at least
one error block because that lets
us know not to hold up the entire
machine for a job that won't run
until the error is removed which
could be a very long time.
*/
if (bg_record->job_running == BLOCK_ERROR_STATE)
break;
}
list_iterator_destroy(itr);
/* Block list is already in the correct order,
earliest avaliable first,
so the job list will also be. No need to
sort. */
while (1) {
bool track_down_nodes = true;
if ((bg_record = list_pop(job_list))) {
if (bg_record->job_ptr) {
if (bg_conf->slurm_debug_flags
& DEBUG_FLAG_BG_PICK)
info("taking off "
"%d(%s) started "
"at %ld ends "
"at %ld",
bg_record->
job_running,
bg_record->
bg_block_id,
bg_record->
job_ptr->
start_time,
bg_record->
job_ptr->
end_time);
/* Mark the block as
not running a job,
this should
correspond to the
pointer in the
block_list. We
only look at the
job_running var so
don't remove the
job_ptr.
*/
bg_record->job_running =
NO_JOB_RUNNING;
} else if ((bg_record->job_running
== BLOCK_ERROR_STATE)
&& (bg_conf->
slurm_debug_flags
& DEBUG_FLAG_BG_PICK))
info("taking off (%s) "
"which is in an "
"error state",
bg_record->bg_block_id);
} else
/* This means we didn't have
any jobs to take off
anymore so we are making
sure we can look at every
node on the system.
*/
track_down_nodes = false;
if (!(new_blocks = create_dynamic_block(
block_list, &request, job_list,
track_down_nodes))) {
if (errno == ESLURM_INTERCONNECT_FAILURE
|| !list_count(job_list)) {
char *nodes;
if (slurmctld_conf.
slurmctld_debug < 5)
break;
nodes = bitmap2node_name(
slurm_block_bitmap);
debug("job %u not "
"runable on %s",
job_ptr->job_id,
nodes);
xfree(nodes);
break;
}
continue;
}
rc = SLURM_SUCCESS;
/* outside of the job_test_list this
* gets destroyed later, so don't worry
* about it now
*/
(*found_bg_record) = list_pop(new_blocks);
if (!(*found_bg_record)) {
list_destroy(new_blocks);
if (!bg_record) {
/* This should never happen */
error("got an empty list back");
rc = SLURM_ERROR;
break;
}
if (bg_conf->slurm_debug_flags
& DEBUG_FLAG_BG_PICK)
info("Appears we are trying "
"to place this job on "
"the block we just "
"removed %s.",
bg_record->bg_block_id);
/* This means we placed the job on
the block we just popped off.
*/
bit_and(slurm_block_bitmap,
bg_record->mp_bitmap);
*found_bg_record = bg_record;
break;
}
bit_and(slurm_block_bitmap,
(*found_bg_record)->mp_bitmap);
if (bg_record) {
(*found_bg_record)->job_running =
bg_record->job_running;
(*found_bg_record)->job_ptr
= bg_record->job_ptr;
}
list_destroy(new_blocks);
break;
}
list_destroy(job_list);
goto end_it;
} else {
break;
}
}
no_match:
debug("_find_best_block_match none found");
rc = SLURM_ERROR;
end_it:
xfree(request.blrtsimage);
xfree(request.linuximage);
xfree(request.mloaderimage);
xfree(request.ramdiskimage);
return rc;
}
/*
* route_p_split_hostlist - logic to split an input hostlist into
* a set of hostlists to forward to.
*
* IN: hl - hostlist_t - list of every node to send message to
* will be empty on return;
* OUT: sp_hl - hostlist_t** - the array of hostlists that will be malloced
* OUT: count - int* - the count of created hostlists
* RET: SLURM_SUCCESS - int
*
* Note: created hostlist will have to be freed independently using
* hostlist_destroy by the caller.
* Note: the hostlist_t array will have to be xfree.
*/
extern int route_p_split_hostlist(hostlist_t hl,
hostlist_t** sp_hl,
int* count)
{
int i, j, k, hl_ndx, msg_count, sw_count, lst_count;
char *buf;
bitstr_t *nodes_bitmap = NULL; /* nodes in message list */
bitstr_t *fwd_bitmap = NULL; /* nodes in forward list */
msg_count = hostlist_count(hl);
if (switch_record_cnt == 0) {
/* configs have not already been processed */
slurm_conf_init(NULL);
if (init_node_conf()) {
fatal("ROUTE: Failed to init slurm config");
}
if (build_all_nodeline_info(false)) {
fatal("ROUTE: Failed to build node config");
}
rehash_node();
if (slurm_topo_build_config() != SLURM_SUCCESS) {
fatal("ROUTE: Failed to build topology config");
}
}
*sp_hl = (hostlist_t*) xmalloc(switch_record_cnt * sizeof(hostlist_t));
/* create bitmap of nodes to send message too */
if (hostlist2bitmap (hl, false, &nodes_bitmap) != SLURM_SUCCESS) {
buf = hostlist_ranged_string_xmalloc(hl);
fatal("ROUTE: Failed to make bitmap from hostlist=%s.", buf);
}
/* Find lowest level switch containing all the nodes in the list */
j = 0;
for (i = 0; i <= switch_levels; i++) {
for (j=0; j<switch_record_cnt; j++) {
if (switch_record_table[j].level == i) {
if (bit_super_set(nodes_bitmap,
switch_record_table[j].
node_bitmap)) {
/* All nodes in message list are in
* this switch */
break;
}
}
}
if (j < switch_record_cnt) {
/* Got here via break after bit_super_set */
break; // 'j' is our switch
} /* else, no switches at this level reach all nodes */
}
if (i > switch_levels) {
/* This can only happen if trying to schedule multiple physical
* clusters as a single logical cluster under the control of a
* single slurmctld daemon, and sending something like a
* node_registation request to all nodes.
* Revert to default behavior*/
if (debug_flags & DEBUG_FLAG_ROUTE) {
buf = hostlist_ranged_string_xmalloc(hl);
debug("ROUTE: didn't find switch containing nodes=%s",
buf);
xfree(buf);
}
FREE_NULL_BITMAP(nodes_bitmap);
xfree(*sp_hl);
return route_split_hostlist_treewidth(hl, sp_hl, count);
}
if (switch_record_table[j].level == 0) {
/* This is a leaf switch. Construct list based on TreeWidth */
FREE_NULL_BITMAP(nodes_bitmap);
xfree(*sp_hl);
return route_split_hostlist_treewidth(hl, sp_hl, count);
}
/* loop through children, construction a hostlist for each child switch
* with nodes in the message list */
hl_ndx = 0;
lst_count = 0;
for (i=0; i < switch_record_table[j].num_switches; i++) {
k = switch_record_table[j].switch_index[i];
fwd_bitmap = bit_copy(switch_record_table[k].node_bitmap);
bit_and(fwd_bitmap, nodes_bitmap);
sw_count = bit_set_count(fwd_bitmap);
if (sw_count == 0) {
continue; /* no nodes on this switch in message list */
}
(*sp_hl)[hl_ndx] = bitmap2hostlist(fwd_bitmap);
/* Now remove nodes from this switch from message list */
bit_not(fwd_bitmap);
bit_and(nodes_bitmap, fwd_bitmap);
FREE_NULL_BITMAP(fwd_bitmap);
if (debug_flags & DEBUG_FLAG_ROUTE) {
buf = hostlist_ranged_string_xmalloc((*sp_hl)[hl_ndx]);
debug("ROUTE: ... sublist[%d] switch=%s :: %s",
i, switch_record_table[i].name, buf);
xfree(buf);
}
hl_ndx++;
lst_count += sw_count;
if (lst_count == msg_count)
break; /* all nodes in message are in a child list */
}
FREE_NULL_BITMAP(nodes_bitmap);
*count = hl_ndx;
return SLURM_SUCCESS;
}
int
main(int argc, char *argv[])
{
note("Testing static decl");
{
bitstr_t bit_decl(bs, 65);
/*bitstr_t *bsp = bs;*/
bit_set(bs,9);
bit_set(bs,14);
TEST(bit_test(bs,9), "bit 9 set");
TEST(!bit_test(bs,12), "bit 12 not set");
TEST(bit_test(bs,14), "bit 14 set" );
/*bit_free(bsp);*/ /* triggers TEST in bit_free - OK */
}
note("Testing basic vixie functions");
{
bitstr_t *bs = bit_alloc(16), *bs2;
/*bit_set(bs, 42);*/ /* triggers TEST in bit_set - OK */
bit_set(bs,9);
bit_set(bs,14);
TEST(bit_test(bs,9), "bit 9 set");
TEST(!bit_test(bs,12), "bit 12 not set" );
TEST(bit_test(bs,14), "bit 14 set");
bs2 = bit_copy(bs);
bit_fill_gaps(bs2);
TEST(bit_ffs(bs2) == 9, "first bit set = 9 ");
TEST(bit_fls(bs2) == 14, "last bit set = 14");
TEST(bit_set_count(bs2) == 6, "bitstring");
TEST(bit_test(bs2,12), "bitstring");
TEST(bit_super_set(bs,bs2) == 1, "bitstring");
TEST(bit_super_set(bs2,bs) == 0, "bitstring");
bit_clear(bs,14);
TEST(!bit_test(bs,14), "bitstring");
bit_nclear(bs,9,14);
TEST(!bit_test(bs,9), "bitstring");
TEST(!bit_test(bs,12), "bitstring");
TEST(!bit_test(bs,14), "bitstring");
bit_nset(bs,9,14);
TEST(bit_test(bs,9), "bitstring");
TEST(bit_test(bs,12), "bitstring");
TEST(bit_test(bs,14), "bitstring");
TEST(bit_ffs(bs) == 9, "ffs");
TEST(bit_ffc(bs) == 0, "ffc");
bit_nset(bs,0,8);
TEST(bit_ffc(bs) == 15, "ffc");
bit_free(bs);
/*bit_set(bs,9); */ /* triggers TEST in bit_set - OK */
}
note("Testing and/or/not");
{
bitstr_t *bs1 = bit_alloc(128);
bitstr_t *bs2 = bit_alloc(128);
bit_set(bs1, 100);
bit_set(bs1, 104);
bit_set(bs2, 100);
bit_and(bs1, bs2);
TEST(bit_test(bs1, 100), "and");
TEST(!bit_test(bs1, 104), "and");
bit_set(bs2, 110);
bit_set(bs2, 111);
bit_set(bs2, 112);
bit_or(bs1, bs2);
TEST(bit_test(bs1, 100), "or");
TEST(bit_test(bs1, 110), "or");
TEST(bit_test(bs1, 111), "or");
TEST(bit_test(bs1, 112), "or");
bit_not(bs1);
TEST(!bit_test(bs1, 100), "not");
TEST(bit_test(bs1, 12), "not");
bit_free(bs1);
bit_free(bs2);
}
note("testing bit selection");
{
bitstr_t *bs1 = bit_alloc(128), *bs2;
bit_set(bs1, 21);
bit_set(bs1, 100);
bit_fill_gaps(bs1);
bs2 = bit_pick_cnt(bs1,20);
if (bs2) {
TEST(bit_set_count(bs2) == 20, "pick");
TEST(bit_ffs(bs2) == 21, "pick");
TEST(bit_fls(bs2) == 40, "pick");
bit_free(bs2);
}
else
TEST(0, "alloc fail");
bit_free(bs1);
}
note("Testing realloc");
{
bitstr_t *bs = bit_alloc(1);
TEST(bit_ffs(bs) == -1, "bitstring");
bit_set(bs,0);
/*bit_set(bs, 1000);*/ /* triggers TEST in bit_set - OK */
bs = bit_realloc(bs,1048576);
bit_set(bs,1000);
bit_set(bs,1048575);
TEST(bit_test(bs, 0), "bitstring");
TEST(bit_test(bs, 1000), "bitstring");
TEST(bit_test(bs, 1048575), "bitstring");
TEST(bit_set_count(bs) == 3, "bitstring");
bit_clear(bs,0);
bit_clear(bs,1000);
TEST(bit_set_count(bs) == 1, "bitstring");
TEST(bit_ffs(bs) == 1048575, "bitstring");
bit_free(bs);
}
note("Testing bit_fmt");
{
char tmpstr[1024];
bitstr_t *bs = bit_alloc(1024);
TEST(!strcmp(bit_fmt(tmpstr,sizeof(tmpstr),bs), ""), "bitstring");
bit_set(bs,42);
TEST(!strcmp(bit_fmt(tmpstr,sizeof(tmpstr),bs), "42"), "bitstring");
bit_set(bs,102);
TEST(!strcmp(bit_fmt(tmpstr,sizeof(tmpstr),bs), "42,102"), "bitstring");
bit_nset(bs,9,14);
TEST(!strcmp(bit_fmt(tmpstr,sizeof(tmpstr), bs),
"9-14,42,102"), "bitstring");
}
note("Testing bit_nffc/bit_nffs");
{
bitstr_t *bs = bit_alloc(1024);
bit_set(bs, 2);
bit_set(bs, 6);
bit_set(bs, 7);
bit_nset(bs,12,1018);
TEST(bit_nffc(bs, 2) == 0, "bitstring");
TEST(bit_nffc(bs, 3) == 3, "bitstring");
TEST(bit_nffc(bs, 4) == 8, "bitstring");
TEST(bit_nffc(bs, 5) == 1019, "bitstring");
TEST(bit_nffc(bs, 6) == -1, "bitstring");
TEST(bit_nffs(bs, 1) == 2, "bitstring");
TEST(bit_nffs(bs, 2) == 6, "bitstring");
TEST(bit_nffs(bs, 100) == 12, "bitstring");
TEST(bit_nffs(bs, 1023) == -1, "bitstring");
bit_free(bs);
}
note("Testing bit_unfmt");
{
bitstr_t *bs = bit_alloc(1024);
bitstr_t *bs2 = bit_alloc(1024);
char tmpstr[4096];
bit_set(bs,1);
bit_set(bs,3);
bit_set(bs,30);
bit_nset(bs,42,64);
bit_nset(bs,97,1000);
bit_fmt(tmpstr, sizeof(tmpstr), bs);
TEST(bit_unfmt(bs2, tmpstr) != -1, "bitstring");
TEST(bit_equal(bs, bs2), "bitstring");
}
totals();
return failed;
}
static int _attempt_backfill(void)
{
bool filter_root = false;
List job_queue;
job_queue_rec_t *job_queue_rec;
slurmdb_qos_rec_t *qos_ptr = NULL;
int i, j, node_space_recs;
struct job_record *job_ptr;
struct part_record *part_ptr;
uint32_t end_time, end_reserve;
uint32_t time_limit, comp_time_limit, orig_time_limit;
uint32_t min_nodes, max_nodes, req_nodes;
bitstr_t *avail_bitmap = NULL, *resv_bitmap = NULL;
time_t now = time(NULL), sched_start, later_start, start_res;
node_space_map_t *node_space;
static int sched_timeout = 0;
int this_sched_timeout = 0, rc = 0;
sched_start = now;
if (sched_timeout == 0) {
sched_timeout = slurm_get_msg_timeout() / 2;
sched_timeout = MAX(sched_timeout, 1);
sched_timeout = MIN(sched_timeout, 10);
}
this_sched_timeout = sched_timeout;
#ifdef HAVE_CRAY
/*
* Run a Basil Inventory immediately before setting up the schedule
* plan, to avoid race conditions caused by ALPS node state change.
* Needs to be done with the node-state lock taken.
*/
if (select_g_reconfigure()) {
debug4("backfill: not scheduling due to ALPS");
return SLURM_SUCCESS;
}
#endif
if (slurm_get_root_filter())
filter_root = true;
job_queue = build_job_queue(true);
if (list_count(job_queue) <= 1) {
debug("backfill: no jobs to backfill");
list_destroy(job_queue);
return 0;
}
node_space = xmalloc(sizeof(node_space_map_t) *
(max_backfill_job_cnt + 3));
node_space[0].begin_time = sched_start;
node_space[0].end_time = sched_start + backfill_window;
node_space[0].avail_bitmap = bit_copy(avail_node_bitmap);
node_space[0].next = 0;
node_space_recs = 1;
if (debug_flags & DEBUG_FLAG_BACKFILL)
_dump_node_space_table(node_space);
while ((job_queue_rec = (job_queue_rec_t *)
list_pop_bottom(job_queue, sort_job_queue2))) {
job_ptr = job_queue_rec->job_ptr;
part_ptr = job_queue_rec->part_ptr;
xfree(job_queue_rec);
if (!IS_JOB_PENDING(job_ptr))
continue; /* started in other partition */
job_ptr->part_ptr = part_ptr;
if (debug_flags & DEBUG_FLAG_BACKFILL)
info("backfill test for job %u", job_ptr->job_id);
if ((job_ptr->state_reason == WAIT_ASSOC_JOB_LIMIT) ||
(job_ptr->state_reason == WAIT_ASSOC_RESOURCE_LIMIT) ||
(job_ptr->state_reason == WAIT_ASSOC_TIME_LIMIT) ||
(job_ptr->state_reason == WAIT_QOS_JOB_LIMIT) ||
(job_ptr->state_reason == WAIT_QOS_RESOURCE_LIMIT) ||
(job_ptr->state_reason == WAIT_QOS_TIME_LIMIT) ||
!acct_policy_job_runnable(job_ptr)) {
debug2("backfill: job %u is not allowed to run now. "
"Skipping it. State=%s. Reason=%s. Priority=%u",
job_ptr->job_id,
job_state_string(job_ptr->job_state),
job_reason_string(job_ptr->state_reason),
job_ptr->priority);
continue;
}
if (((part_ptr->state_up & PARTITION_SCHED) == 0) ||
(part_ptr->node_bitmap == NULL))
continue;
if ((part_ptr->flags & PART_FLAG_ROOT_ONLY) && filter_root)
continue;
if ((!job_independent(job_ptr, 0)) ||
(license_job_test(job_ptr, time(NULL)) != SLURM_SUCCESS))
continue;
/* Determine minimum and maximum node counts */
min_nodes = MAX(job_ptr->details->min_nodes,
part_ptr->min_nodes);
if (job_ptr->details->max_nodes == 0)
max_nodes = part_ptr->max_nodes;
else
max_nodes = MIN(job_ptr->details->max_nodes,
part_ptr->max_nodes);
max_nodes = MIN(max_nodes, 500000); /* prevent overflows */
if (job_ptr->details->max_nodes)
req_nodes = max_nodes;
else
req_nodes = min_nodes;
if (min_nodes > max_nodes) {
/* job's min_nodes exceeds partition's max_nodes */
continue;
}
/* Determine job's expected completion time */
if (job_ptr->time_limit == NO_VAL) {
if (part_ptr->max_time == INFINITE)
time_limit = 365 * 24 * 60; /* one year */
else
time_limit = part_ptr->max_time;
} else {
if (part_ptr->max_time == INFINITE)
time_limit = job_ptr->time_limit;
else
time_limit = MIN(job_ptr->time_limit,
part_ptr->max_time);
}
comp_time_limit = time_limit;
orig_time_limit = job_ptr->time_limit;
if (qos_ptr && (qos_ptr->flags & QOS_FLAG_NO_RESERVE))
time_limit = job_ptr->time_limit = 1;
else if (job_ptr->time_min && (job_ptr->time_min < time_limit))
time_limit = job_ptr->time_limit = job_ptr->time_min;
/* Determine impact of any resource reservations */
later_start = now;
TRY_LATER:
FREE_NULL_BITMAP(avail_bitmap);
start_res = later_start;
later_start = 0;
j = job_test_resv(job_ptr, &start_res, true, &avail_bitmap);
if (j != SLURM_SUCCESS) {
job_ptr->time_limit = orig_time_limit;
continue;
}
if (start_res > now)
end_time = (time_limit * 60) + start_res;
else
end_time = (time_limit * 60) + now;
/* Identify usable nodes for this job */
bit_and(avail_bitmap, part_ptr->node_bitmap);
bit_and(avail_bitmap, up_node_bitmap);
for (j=0; ; ) {
if ((node_space[j].end_time > start_res) &&
node_space[j].next && (later_start == 0))
later_start = node_space[j].end_time;
if (node_space[j].end_time <= start_res)
;
else if (node_space[j].begin_time <= end_time) {
bit_and(avail_bitmap,
node_space[j].avail_bitmap);
} else
break;
if ((j = node_space[j].next) == 0)
break;
}
if (job_ptr->details->exc_node_bitmap) {
bit_not(job_ptr->details->exc_node_bitmap);
bit_and(avail_bitmap,
job_ptr->details->exc_node_bitmap);
bit_not(job_ptr->details->exc_node_bitmap);
}
/* Test if insufficient nodes remain OR
* required nodes missing OR
* nodes lack features */
if ((bit_set_count(avail_bitmap) < min_nodes) ||
((job_ptr->details->req_node_bitmap) &&
(!bit_super_set(job_ptr->details->req_node_bitmap,
avail_bitmap))) ||
(job_req_node_filter(job_ptr, avail_bitmap))) {
if (later_start) {
job_ptr->start_time = 0;
goto TRY_LATER;
}
job_ptr->time_limit = orig_time_limit;
continue;
}
/* Identify nodes which are definitely off limits */
FREE_NULL_BITMAP(resv_bitmap);
resv_bitmap = bit_copy(avail_bitmap);
bit_not(resv_bitmap);
if ((time(NULL) - sched_start) >= this_sched_timeout) {
debug("backfill: loop taking too long, yielding locks");
if (_yield_locks()) {
debug("backfill: system state changed, "
"breaking out");
rc = 1;
break;
} else {
this_sched_timeout += sched_timeout;
}
}
/* this is the time consuming operation */
debug2("backfill: entering _try_sched for job %u.",
job_ptr->job_id);
j = _try_sched(job_ptr, &avail_bitmap,
min_nodes, max_nodes, req_nodes);
debug2("backfill: finished _try_sched for job %u.",
job_ptr->job_id);
now = time(NULL);
if (j != SLURM_SUCCESS) {
job_ptr->time_limit = orig_time_limit;
job_ptr->start_time = 0;
continue; /* not runable */
}
if (start_res > job_ptr->start_time) {
job_ptr->start_time = start_res;
last_job_update = now;
}
if (job_ptr->start_time <= now) {
int rc = _start_job(job_ptr, resv_bitmap);
if (qos_ptr && (qos_ptr->flags & QOS_FLAG_NO_RESERVE))
job_ptr->time_limit = orig_time_limit;
else if ((rc == SLURM_SUCCESS) && job_ptr->time_min) {
/* Set time limit as high as possible */
job_ptr->time_limit = comp_time_limit;
job_ptr->end_time = job_ptr->start_time +
(comp_time_limit * 60);
_reset_job_time_limit(job_ptr, now,
node_space);
time_limit = job_ptr->time_limit;
} else {
job_ptr->time_limit = orig_time_limit;
}
if (rc == ESLURM_ACCOUNTING_POLICY) {
/* Unknown future start time, just skip job */
job_ptr->start_time = 0;
continue;
} else if (rc != SLURM_SUCCESS) {
/* Planned to start job, but something bad
* happended. */
job_ptr->start_time = 0;
break;
} else {
/* Started this job, move to next one */
continue;
}
} else
job_ptr->time_limit = orig_time_limit;
if (later_start && (job_ptr->start_time > later_start)) {
/* Try later when some nodes currently reserved for
* pending jobs are free */
job_ptr->start_time = 0;
goto TRY_LATER;
}
if (job_ptr->start_time > (sched_start + backfill_window)) {
/* Starts too far in the future to worry about */
continue;
}
if (node_space_recs >= max_backfill_job_cnt) {
/* Already have too many jobs to deal with */
break;
}
end_reserve = job_ptr->start_time + (time_limit * 60);
if (_test_resv_overlap(node_space, avail_bitmap,
job_ptr->start_time, end_reserve)) {
/* This job overlaps with an existing reservation for
* job to be backfill scheduled, which the sched
* plugin does not know about. Try again later. */
later_start = job_ptr->start_time;
job_ptr->start_time = 0;
goto TRY_LATER;
}
/*
* Add reservation to scheduling table if appropriate
*/
qos_ptr = job_ptr->qos_ptr;
if (qos_ptr && (qos_ptr->flags & QOS_FLAG_NO_RESERVE))
continue;
bit_not(avail_bitmap);
_add_reservation(job_ptr->start_time, end_reserve,
avail_bitmap, node_space, &node_space_recs);
if (debug_flags & DEBUG_FLAG_BACKFILL)
_dump_node_space_table(node_space);
}
FREE_NULL_BITMAP(avail_bitmap);
FREE_NULL_BITMAP(resv_bitmap);
for (i=0; ; ) {
FREE_NULL_BITMAP(node_space[i].avail_bitmap);
if ((i = node_space[i].next) == 0)
break;
}
xfree(node_space);
list_destroy(job_queue);
return rc;
}
/* Create a reservation for a job in the future */
static void _add_reservation(uint32_t start_time, uint32_t end_reserve,
bitstr_t *res_bitmap,
node_space_map_t *node_space,
int *node_space_recs)
{
bool placed = false;
int i, j;
/* If we decrease the resolution of our timing information, this can
* decrease the number of records managed and increase performance */
start_time = (start_time / backfill_resolution) * backfill_resolution;
end_reserve = (end_reserve / backfill_resolution) * backfill_resolution;
for (j=0; ; ) {
if (node_space[j].end_time > start_time) {
/* insert start entry record */
i = *node_space_recs;
node_space[i].begin_time = start_time;
node_space[i].end_time = node_space[j].end_time;
node_space[j].end_time = start_time;
node_space[i].avail_bitmap =
bit_copy(node_space[j].avail_bitmap);
node_space[i].next = node_space[j].next;
node_space[j].next = i;
(*node_space_recs)++;
placed = true;
}
if (node_space[j].end_time == start_time) {
/* no need to insert new start entry record */
placed = true;
}
if (placed == true) {
j = node_space[j].next;
if (j && (end_reserve < node_space[j].end_time)) {
/* insert end entry record */
i = *node_space_recs;
node_space[i].begin_time = end_reserve;
node_space[i].end_time = node_space[j].
end_time;
node_space[j].end_time = end_reserve;
node_space[i].avail_bitmap =
bit_copy(node_space[j].avail_bitmap);
node_space[i].next = node_space[j].next;
node_space[j].next = i;
(*node_space_recs)++;
}
break;
}
if ((j = node_space[j].next) == 0)
break;
}
for (j=0; ; ) {
if ((node_space[j].begin_time >= start_time) &&
(node_space[j].end_time <= end_reserve))
bit_and(node_space[j].avail_bitmap, res_bitmap);
if ((node_space[j].begin_time >= end_reserve) ||
((j = node_space[j].next) == 0))
break;
}
}
void CPU::exec32(const Instruction32 &insn) {
switch(insn.OP) {
case 0x00: {
uint32_t &rD = r[insn.spform.rD];
uint32_t &rA = r[insn.spform.rA];
uint32_t &rB = r[insn.spform.rB];
switch(insn.spform.func6) {
// nop
case 0x00: /* nothing */ break;
// br{cond}[l] rA
case 0x04: if(conditional(insn.spform.rB)) branch(rA - 4, insn.spform.CU); break;
// add[.c] rD, rA, rB
case 0x08: rD = add(rA, rB, insn.spform.CU); break;
// addc[.c] rD, rA, rB
case 0x09: rD = addc(rA, rB, insn.spform.CU); break;
// sub[.c] rD, rA, rB
case 0x0A: rD = sub(rA, rB, insn.spform.CU); break;
// subc[.c] rD, rA, rB
case 0x0B: rD = subc(rA, rB, insn.spform.CU); break;
// cmp{tcs}.c rA, rB
case 0x0C: cmp(rA, rB, insn.spform.rD & 0x03, insn.spform.CU); break;
// cmpz{tcs}.c rA, rB
case 0x0D: cmp(rA, 0, insn.spform.rD & 0x03, insn.spform.CU); break;
// neg[.c] rD, rA
case 0x0F: rD = sub(0, rA, insn.spform.CU); break;
// and[.c] rD, rA, rB
case 0x10: rD = bit_and(rA, rB, insn.spform.CU); break;
// or[.c] rD, rA, rB
case 0x11: rD = bit_or(rA, rB, insn.spform.CU); break;
// not[.c] rD, rA, rB
case 0x12: rD = bit_xor(rA, ~0, insn.spform.CU); break;
// xor[.c] rD, rA, rB
case 0x13: rD = bit_or(rA, rB, insn.spform.CU); break;
// bitclr[.c] rD, rA, imm5
case 0x14: rD = bit_and(rA, ~(1 << insn.spform.rB), insn.spform.CU); break;
// bitset[.c] rD, rA, imm5
case 0x15: rD = bit_or(rA, 1 << insn.spform.rB, insn.spform.CU); break;
// bittst.c rA, imm5
case 0x16: bit_and(rA, 1 << insn.spform.rB, insn.spform.CU); break;
// bittgl[.c] rA, imm5
case 0x17: rD = bit_xor(rA, 1 << insn.spform.rB, insn.spform.CU); break;
// sll[.c] rA, imm5
case 0x18: rD = sll(rA, insn.spform.rB, insn.spform.CU); break;
// srl[.c] rA, imm5
case 0x1A: rD = srl(rA, insn.spform.rB, insn.spform.CU); break;
// sra[.c] rA, imm5
case 0x1B: rD = sra(rA, insn.spform.rB, insn.spform.CU); break;
// mul rA, rD
case 0x20: ce_op(rA, rD, std::multiplies<int64_t>()); break;
// mulu rA, rD
case 0x21: ce_op(rA, rD, std::multiplies<uint64_t>()); break;
// div rA, rD
case 0x22: ce_op(rA, rD, std::divides<int64_t>()); break;
// divu rA, rD
case 0x23: ce_op(rA, rD, std::divides<uint64_t>()); break;
// mfce{hl} rD[, rA]
case 0x24:
switch(insn.spform.rB) {
case 0x01: rD = CEL; break;
case 0x02: rD = CEH; break;
case 0x03: rD = CEH; rA = CEL; break;
}
break;
// mtce{hl} rD[, rA]
case 0x25:
switch(insn.spform.rB) {
case 0x01: CEL = rD; break;
case 0x02: CEH = rD; break;
case 0x03: CEH = rD; CEL = rA; break;
}
break;
// mfsr rA, Srn
case 0x28: rA = sr[insn.spform.rB];
// mtsr rA, Srn
case 0x29: sr[insn.spform.rB] = rA;
// t{cond}
case 0x2A: T = conditional(insn.spform.rB); break;
// mv{cond} rD, rA
case 0x2B: if(conditional(insn.spform.rB)) rD = rA; break;
// extsb[.c] rD, rA
case 0x2C: rD = sign_extend(rA, 8); if(insn.spform.CU) basic_flags(rD); break;
// extsh[.c] rD, rA
case 0x2D: rD = sign_extend(rA, 16); if(insn.spform.CU) basic_flags(rD); break;
// extzb[.c] rD, rA
case 0x2E: rD = bit_and(rA, 0x000000FF, insn.spform.CU); break;
// extzh[.c] rD, rA
case 0x2F: rD = bit_and(rA, 0x0000FFFF, insn.spform.CU); break;
// slli[.c] rD, rA, imm5
case 0x38: rD = sll(rA, insn.spform.rB, insn.spform.CU); break;
// srli[.c] rD, rA, imm5
case 0x3A: rD = srl(rA, insn.spform.rB, insn.spform.CU); break;
// srai[.c] rD, rA, imm5
case 0x3B: rD = sra(rA, insn.spform.rB, insn.spform.CU); break;
default: debugDump();
}
} break;
case 0x01: {
uint32_t &rD = r[insn.iform.rD];
switch(insn.iform.func3) {
// addi[.c] rD, imm16
case 0x00: rD = add(rD, sign_extend(insn.iform.Imm16, 16), insn.iform.CU); break;
// cmpi.c rD, imm16
case 0x02: cmp(rD, sign_extend(insn.iform.Imm16, 16), 3, insn.iform.CU); break;
// andi.c rD, imm16
case 0x04: rD = bit_and(rD, insn.iform.Imm16, insn.iform.CU); break;
// ori.c rD, imm16
case 0x05: rD = bit_or(rD, insn.iform.Imm16, insn.iform.CU); break;
// ldi rD, imm16
case 0x06: rD = sign_extend(insn.iform.Imm16, 16); break;
default: debugDump();
}
} break;
case 0x02: {
// j[l] imm24
if(insn.jform.LK)
link();
// Update PC
pc &= 0xFC000000;
pc |= (insn.jform.Disp24 << 1) - 4;
} break;
case 0x03: {
uint32_t &rD = r[insn.rixform.rD];
uint32_t &rA = r[insn.rixform.rA];
// Pre-increment
rA += sign_extend(insn.rixform.Imm12, 12);
switch(insn.rixform.func3) {
// lw rD, [rA, imm12]+
case 0x00: rD = miu.readU32(rA); break;
// lh rD, [rA, imm12]+
case 0x01: rD = sign_extend(miu.readU16(rA), 16); break;
// lhu rD, [rA, imm12]+
case 0x02: rD = miu.readU16(rA); break;
// lb rD, [rA, imm12]+
case 0x03: rD = sign_extend(miu.readU8(rA), 8); break;
// sw rD, [rA, imm12]+
case 0x04: miu.writeU32(rA, rD); break;
// sh rD, [rA, imm12]+
case 0x05: miu.writeU16(rA, rD); break;
// lbu rD, [rA, imm12]+
case 0x06: rD = miu.readU8(rA); break;
// sb rD, [rA, imm12]+
case 0x07: miu.writeU8(rA, rD); break;
default: debugDump();
}
} break;
case 0x04: {
// b{cond}[l]
if(conditional(insn.bcform.BC)) {
if(insn.bcform.LK)
link();
pc += sign_extend(((insn.bcform.Disp18_9 << 9) | insn.bcform.Disp8_0) << 1, 20) - 4;
}
} break;
case 0x05: {
uint32_t &rD = r[insn.iform.rD];
uint32_t imm16 = insn.iform.Imm16 << 16;
switch(insn.iform.func3) {
// addis[.c] rD, imm16
case 0x00: rD = add(rD, imm16, insn.iform.CU); break;
// cmpis.c rD, imm16
case 0x02: cmp(rD, imm16, 3, insn.iform.CU); break;
// andis.c rD, imm16
case 0x04: rD = bit_and(rD, imm16, insn.iform.CU); break;
// oris.c rD, imm16
case 0x05: rD = bit_or(rD, imm16, insn.iform.CU); break;
// ldis rD, imm16
case 0x06: rD = imm16; break;
default: debugDump();
}
} break;
case 0x06: {
uint32_t &rD = r[insn.crform.rD];
uint32_t &crA = cr[insn.crform.crA];
switch(insn.crform.CR_OP) {
// mtcr rD, crA
case 0x00: crA = rD; break;
// mfcr rD, crA
case 0x01: rD = crA; break;
// rte
case 0x84: branch(cr5 - 4, false); /* TODO: missing PSR */ break;
default: debugDump();
}
} break;
case 0x07: {
uint32_t &rD = r[insn.rixform.rD];
uint32_t &rA = r[insn.rixform.rA];
switch(insn.rixform.func3) {
// lw rD, [rA]+, imm12
case 0x00: rD = miu.readU32(rA); break;
// lh rD, [rA]+, imm12
case 0x01: rD = sign_extend(miu.readU16(rA), 16); break;
// lhu rD, [rA]+, imm12
case 0x02: rD = miu.readU16(rA); break;
// lb rD, [rA]+, imm12
case 0x03: rD = sign_extend(miu.readU8(rA), 8); break;
// sw rD, [rA]+, imm12
case 0x04: miu.writeU32(rA, rD); break;
// sh rD, [rA]+, imm12
case 0x05: miu.writeU16(rA, rD); break;
// lbu rD, [rA]+, imm12
case 0x06: rD = miu.readU8(rA); break;
// sb rD, [rA]+, imm12
case 0x07: miu.writeU8(rA, rD); break;
default: debugDump();
}
// Post-increment
rA += sign_extend(insn.rixform.Imm12, 12);
} break;
case 0x08: {
// addri[.c] rD, rA, imm14
uint32_t &rD = r[insn.riform.rD];
uint32_t &rA = r[insn.riform.rA];
uint32_t imm14 = sign_extend(insn.riform.Imm14, 14);
rD = add(rA, imm14, insn.riform.CU);
} break;
case 0x0C: {
// andri[.c] rD, rA, imm14
uint32_t &rD = r[insn.riform.rD];
uint32_t &rA = r[insn.riform.rA];
uint32_t imm14 = insn.riform.Imm14;
rD = bit_and(rA, imm14, insn.riform.CU);
} break;
case 0x0D: {
// orri[.c] rD, rA, imm14
uint32_t &rD = r[insn.riform.rD];
uint32_t &rA = r[insn.riform.rA];
uint32_t imm14 = insn.riform.Imm14;
rD = bit_or(rA, imm14, insn.riform.CU);
} break;
case 0x10: {
// lw rD, [rA, imm15]
uint32_t &rD = r[insn.mform.rD];
uint32_t &rA = r[insn.mform.rA];
uint32_t imm15 = sign_extend(insn.mform.Imm15, 15);
rD = miu.readU32(rA + imm15);
} break;
case 0x11: {
// lh rD, [rA, imm15]
uint32_t &rD = r[insn.mform.rD];
uint32_t &rA = r[insn.mform.rA];
uint32_t imm15 = sign_extend(insn.mform.Imm15, 15);
rD = sign_extend(miu.readU16(rA + imm15), 16);
} break;
case 0x12: {
// lhu rD, [rA, imm15]
uint32_t &rD = r[insn.mform.rD];
uint32_t &rA = r[insn.mform.rA];
uint32_t imm15 = sign_extend(insn.mform.Imm15, 15);
rD = miu.readU16(rA + imm15);
} break;
case 0x13: {
// lb rD, [rA, imm15]
uint32_t &rD = r[insn.mform.rD];
uint32_t &rA = r[insn.mform.rA];
uint32_t imm15 = sign_extend(insn.mform.Imm15, 15);
rD = sign_extend(miu.readU8(rA + imm15), 8);
} break;
case 0x14: {
// sw rD, [rA, imm15]
uint32_t &rD = r[insn.mform.rD];
uint32_t &rA = r[insn.mform.rA];
uint32_t imm15 = sign_extend(insn.mform.Imm15, 15);
miu.writeU32(rA + imm15, rD);
} break;
case 0x15: {
// sh rD, [rA, imm15]
uint32_t &rD = r[insn.mform.rD];
uint32_t &rA = r[insn.mform.rA];
uint32_t imm15 = sign_extend(insn.mform.Imm15, 15);
miu.writeU16(rA + imm15, rD);
} break;
case 0x16: {
// lbu rD, [rA, imm15]
uint32_t &rD = r[insn.mform.rD];
uint32_t &rA = r[insn.mform.rA];
uint32_t imm15 = sign_extend(insn.mform.Imm15, 15);
rD = miu.readU8(rA + imm15);
} break;
case 0x17: {
// sb rD, [rA, imm15]
uint32_t &rD = r[insn.mform.rD];
uint32_t &rA = r[insn.mform.rA];
uint32_t imm15 = sign_extend(insn.mform.Imm15, 15);
miu.writeU8(rA + imm15, rD);
} break;
case 0x18:
// cache op, [rA, imm15]
break;
default: debugDump();
}
}
static int _attempt_backfill(void)
{
DEF_TIMERS;
bool filter_root = false;
List job_queue;
job_queue_rec_t *job_queue_rec;
slurmdb_qos_rec_t *qos_ptr = NULL;
int i, j, node_space_recs;
struct job_record *job_ptr;
struct part_record *part_ptr, **bf_part_ptr = NULL;
uint32_t end_time, end_reserve;
uint32_t time_limit, comp_time_limit, orig_time_limit, part_time_limit;
uint32_t min_nodes, max_nodes, req_nodes;
bitstr_t *avail_bitmap = NULL, *resv_bitmap = NULL;
bitstr_t *exc_core_bitmap = NULL, *non_cg_bitmap = NULL;
time_t now, sched_start, later_start, start_res, resv_end, window_end;
node_space_map_t *node_space;
struct timeval bf_time1, bf_time2;
int rc = 0;
int job_test_count = 0;
uint32_t *uid = NULL, nuser = 0, bf_parts = 0, *bf_part_jobs = NULL;
uint16_t *njobs = NULL;
bool already_counted;
uint32_t reject_array_job_id = 0;
struct part_record *reject_array_part = NULL;
uint32_t job_start_cnt = 0, start_time;
time_t config_update = slurmctld_conf.last_update;
time_t part_update = last_part_update;
struct timeval start_tv;
bf_last_yields = 0;
#ifdef HAVE_ALPS_CRAY
/*
* Run a Basil Inventory immediately before setting up the schedule
* plan, to avoid race conditions caused by ALPS node state change.
* Needs to be done with the node-state lock taken.
*/
START_TIMER;
if (select_g_reconfigure()) {
debug4("backfill: not scheduling due to ALPS");
return SLURM_SUCCESS;
}
END_TIMER;
if (debug_flags & DEBUG_FLAG_BACKFILL)
info("backfill: ALPS inventory completed, %s", TIME_STR);
/* The Basil inventory can take a long time to complete. Process
* pending RPCs before starting the backfill scheduling logic */
_yield_locks(1000000);
#endif
START_TIMER;
if (debug_flags & DEBUG_FLAG_BACKFILL)
info("backfill: beginning");
else
debug("backfill: beginning");
sched_start = now = time(NULL);
gettimeofday(&start_tv, NULL);
if (slurm_get_root_filter())
filter_root = true;
job_queue = build_job_queue(true, true);
if (list_count(job_queue) == 0) {
if (debug_flags & DEBUG_FLAG_BACKFILL)
info("backfill: no jobs to backfill");
else
debug("backfill: no jobs to backfill");
list_destroy(job_queue);
return 0;
}
gettimeofday(&bf_time1, NULL);
non_cg_bitmap = bit_copy(cg_node_bitmap);
bit_not(non_cg_bitmap);
slurmctld_diag_stats.bf_queue_len = list_count(job_queue);
slurmctld_diag_stats.bf_queue_len_sum += slurmctld_diag_stats.
bf_queue_len;
slurmctld_diag_stats.bf_last_depth = 0;
slurmctld_diag_stats.bf_last_depth_try = 0;
slurmctld_diag_stats.bf_when_last_cycle = now;
slurmctld_diag_stats.bf_active = 1;
node_space = xmalloc(sizeof(node_space_map_t) *
(max_backfill_job_cnt * 2 + 1));
node_space[0].begin_time = sched_start;
window_end = sched_start + backfill_window;
node_space[0].end_time = window_end;
node_space[0].avail_bitmap = bit_copy(avail_node_bitmap);
node_space[0].next = 0;
node_space_recs = 1;
if (debug_flags & DEBUG_FLAG_BACKFILL_MAP)
_dump_node_space_table(node_space);
if (max_backfill_job_per_part) {
ListIterator part_iterator;
struct part_record *part_ptr;
bf_parts = list_count(part_list);
bf_part_ptr = xmalloc(sizeof(struct part_record *) * bf_parts);
bf_part_jobs = xmalloc(sizeof(int) * bf_parts);
part_iterator = list_iterator_create(part_list);
i = 0;
while ((part_ptr = (struct part_record *)
list_next(part_iterator))) {
bf_part_ptr[i++] = part_ptr;
}
list_iterator_destroy(part_iterator);
}
if (max_backfill_job_per_user) {
uid = xmalloc(BF_MAX_USERS * sizeof(uint32_t));
njobs = xmalloc(BF_MAX_USERS * sizeof(uint16_t));
}
sort_job_queue(job_queue);
while (1) {
job_queue_rec = (job_queue_rec_t *) list_pop(job_queue);
if (!job_queue_rec) {
if (debug_flags & DEBUG_FLAG_BACKFILL)
info("backfill: reached end of job queue");
break;
}
if (slurmctld_config.shutdown_time)
break;
if (((defer_rpc_cnt > 0) &&
(slurmctld_config.server_thread_count >= defer_rpc_cnt)) ||
(_delta_tv(&start_tv) >= sched_timeout)) {
if (debug_flags & DEBUG_FLAG_BACKFILL) {
END_TIMER;
info("backfill: completed yielding locks "
"after testing %u(%d) jobs, %s",
slurmctld_diag_stats.bf_last_depth,
job_test_count, TIME_STR);
}
if ((_yield_locks(yield_sleep) && !backfill_continue) ||
(slurmctld_conf.last_update != config_update) ||
(last_part_update != part_update)) {
if (debug_flags & DEBUG_FLAG_BACKFILL) {
info("backfill: system state changed, "
"breaking out after testing "
"%u(%d) jobs",
slurmctld_diag_stats.bf_last_depth,
job_test_count);
}
rc = 1;
xfree(job_queue_rec);
break;
}
/* cg_node_bitmap may be changed */
bit_copybits(non_cg_bitmap, cg_node_bitmap);
bit_not(non_cg_bitmap);
/* Reset backfill scheduling timers, resume testing */
sched_start = time(NULL);
gettimeofday(&start_tv, NULL);
job_test_count = 0;
START_TIMER;
}
job_ptr = job_queue_rec->job_ptr;
/* With bf_continue configured, the original job could have
* been cancelled and purged. Validate pointer here. */
if ((job_ptr->magic != JOB_MAGIC) ||
(job_ptr->job_id != job_queue_rec->job_id)) {
xfree(job_queue_rec);
continue;
}
orig_time_limit = job_ptr->time_limit;
part_ptr = job_queue_rec->part_ptr;
job_test_count++;
slurmctld_diag_stats.bf_last_depth++;
already_counted = false;
xfree(job_queue_rec);
if (!IS_JOB_PENDING(job_ptr))
continue; /* started in other partition */
if (!avail_front_end(job_ptr))
continue; /* No available frontend for this job */
if (job_ptr->array_task_id != NO_VAL) {
if ((reject_array_job_id == job_ptr->array_job_id) &&
(reject_array_part == part_ptr))
continue; /* already rejected array element */
/* assume reject whole array for now, clear if OK */
reject_array_job_id = job_ptr->array_job_id;
reject_array_part = part_ptr;
}
job_ptr->part_ptr = part_ptr;
if (debug_flags & DEBUG_FLAG_BACKFILL) {
info("backfill test for JobID=%u Prio=%u Partition=%s",
job_ptr->job_id, job_ptr->priority,
job_ptr->part_ptr->name);
}
if (max_backfill_job_per_part) {
bool skip_job = false;
for (j = 0; j < bf_parts; j++) {
if (bf_part_ptr[j] != job_ptr->part_ptr)
continue;
if (bf_part_jobs[j]++ >=
max_backfill_job_per_part)
skip_job = true;
break;
}
if (skip_job) {
if (debug_flags & DEBUG_FLAG_BACKFILL)
info("backfill: have already "
"checked %u jobs for "
"partition %s; skipping "
"job %u",
max_backfill_job_per_part,
job_ptr->part_ptr->name,
job_ptr->job_id);
continue;
}
}
if (max_backfill_job_per_user) {
for (j = 0; j < nuser; j++) {
if (job_ptr->user_id == uid[j]) {
njobs[j]++;
if (debug_flags & DEBUG_FLAG_BACKFILL)
debug("backfill: user %u: "
"#jobs %u",
uid[j], njobs[j]);
break;
}
}
if (j == nuser) { /* user not found */
static bool bf_max_user_msg = true;
if (nuser < BF_MAX_USERS) {
uid[j] = job_ptr->user_id;
njobs[j] = 1;
nuser++;
} else if (bf_max_user_msg) {
bf_max_user_msg = false;
error("backfill: too many users in "
"queue. Consider increasing "
"BF_MAX_USERS");
}
if (debug_flags & DEBUG_FLAG_BACKFILL)
debug2("backfill: found new user %u. "
"Total #users now %u",
job_ptr->user_id, nuser);
} else {
if (njobs[j] >= max_backfill_job_per_user) {
/* skip job */
if (debug_flags & DEBUG_FLAG_BACKFILL)
info("backfill: have already "
"checked %u jobs for "
"user %u; skipping "
"job %u",
max_backfill_job_per_user,
job_ptr->user_id,
job_ptr->job_id);
continue;
}
}
}
if (((part_ptr->state_up & PARTITION_SCHED) == 0) ||
(part_ptr->node_bitmap == NULL) ||
((part_ptr->flags & PART_FLAG_ROOT_ONLY) && filter_root)) {
if (debug_flags & DEBUG_FLAG_BACKFILL)
info("backfill: partition %s not usable",
job_ptr->part_ptr->name);
continue;
}
if ((!job_independent(job_ptr, 0)) ||
(license_job_test(job_ptr, time(NULL)) != SLURM_SUCCESS)) {
if (debug_flags & DEBUG_FLAG_BACKFILL)
info("backfill: job %u not runable now",
job_ptr->job_id);
continue;
}
/* Determine minimum and maximum node counts */
min_nodes = MAX(job_ptr->details->min_nodes,
part_ptr->min_nodes);
if (job_ptr->details->max_nodes == 0)
max_nodes = part_ptr->max_nodes;
else
max_nodes = MIN(job_ptr->details->max_nodes,
part_ptr->max_nodes);
max_nodes = MIN(max_nodes, 500000); /* prevent overflows */
if (job_ptr->details->max_nodes)
req_nodes = max_nodes;
else
req_nodes = min_nodes;
if (min_nodes > max_nodes) {
if (debug_flags & DEBUG_FLAG_BACKFILL)
info("backfill: job %u node count too high",
job_ptr->job_id);
continue;
}
/* Determine job's expected completion time */
if (part_ptr->max_time == INFINITE)
part_time_limit = 365 * 24 * 60; /* one year */
else
part_time_limit = part_ptr->max_time;
if (job_ptr->time_limit == NO_VAL) {
time_limit = part_time_limit;
} else {
if (part_ptr->max_time == INFINITE)
time_limit = job_ptr->time_limit;
else
time_limit = MIN(job_ptr->time_limit,
part_time_limit);
}
comp_time_limit = time_limit;
qos_ptr = job_ptr->qos_ptr;
if (qos_ptr && (qos_ptr->flags & QOS_FLAG_NO_RESERVE) &&
slurm_get_preempt_mode())
time_limit = job_ptr->time_limit = 1;
else if (job_ptr->time_min && (job_ptr->time_min < time_limit))
time_limit = job_ptr->time_limit = job_ptr->time_min;
/* Determine impact of any resource reservations */
later_start = now;
TRY_LATER:
if (slurmctld_config.shutdown_time)
break;
if (((defer_rpc_cnt > 0) &&
(slurmctld_config.server_thread_count >= defer_rpc_cnt)) ||
(_delta_tv(&start_tv) >= sched_timeout)) {
uint32_t save_job_id = job_ptr->job_id;
uint32_t save_time_limit = job_ptr->time_limit;
job_ptr->time_limit = orig_time_limit;
if (debug_flags & DEBUG_FLAG_BACKFILL) {
END_TIMER;
info("backfill: completed yielding locks "
"after testing %u(%d) jobs, %s",
slurmctld_diag_stats.bf_last_depth,
job_test_count, TIME_STR);
}
if ((_yield_locks(yield_sleep) && !backfill_continue) ||
(slurmctld_conf.last_update != config_update) ||
(last_part_update != part_update)) {
if (debug_flags & DEBUG_FLAG_BACKFILL) {
info("backfill: system state changed, "
"breaking out after testing "
"%u(%d) jobs",
slurmctld_diag_stats.bf_last_depth,
job_test_count);
}
rc = 1;
break;
}
/* cg_node_bitmap may be changed */
bit_copybits(non_cg_bitmap, cg_node_bitmap);
bit_not(non_cg_bitmap);
/* With bf_continue configured, the original job could
* have been scheduled or cancelled and purged.
* Revalidate job the record here. */
if ((job_ptr->magic != JOB_MAGIC) ||
(job_ptr->job_id != save_job_id))
continue;
if (!IS_JOB_PENDING(job_ptr))
continue;
if (!avail_front_end(job_ptr))
continue; /* No available frontend */
job_ptr->time_limit = save_time_limit;
/* Reset backfill scheduling timers, resume testing */
sched_start = time(NULL);
gettimeofday(&start_tv, NULL);
job_test_count = 1;
START_TIMER;
}
FREE_NULL_BITMAP(avail_bitmap);
FREE_NULL_BITMAP(exc_core_bitmap);
start_res = later_start;
later_start = 0;
j = job_test_resv(job_ptr, &start_res, true, &avail_bitmap,
&exc_core_bitmap);
if (j != SLURM_SUCCESS) {
if (debug_flags & DEBUG_FLAG_BACKFILL)
info("backfill: job %u reservation defer",
job_ptr->job_id);
job_ptr->time_limit = orig_time_limit;
continue;
}
if (start_res > now)
end_time = (time_limit * 60) + start_res;
else
end_time = (time_limit * 60) + now;
resv_end = find_resv_end(start_res);
/* Identify usable nodes for this job */
bit_and(avail_bitmap, part_ptr->node_bitmap);
bit_and(avail_bitmap, up_node_bitmap);
bit_and(avail_bitmap, non_cg_bitmap);
for (j=0; ; ) {
if ((node_space[j].end_time > start_res) &&
node_space[j].next && (later_start == 0))
later_start = node_space[j].end_time;
if (node_space[j].end_time <= start_res)
;
else if (node_space[j].begin_time <= end_time) {
bit_and(avail_bitmap,
node_space[j].avail_bitmap);
} else
break;
if ((j = node_space[j].next) == 0)
break;
}
if (resv_end && (++resv_end < window_end) &&
((later_start == 0) || (resv_end < later_start))) {
later_start = resv_end;
}
if (job_ptr->details->exc_node_bitmap) {
bit_not(job_ptr->details->exc_node_bitmap);
bit_and(avail_bitmap,
job_ptr->details->exc_node_bitmap);
bit_not(job_ptr->details->exc_node_bitmap);
}
/* Test if insufficient nodes remain OR
* required nodes missing OR
* nodes lack features OR
* no change since previously tested nodes (only changes
* in other partition nodes) */
if ((bit_set_count(avail_bitmap) < min_nodes) ||
((job_ptr->details->req_node_bitmap) &&
(!bit_super_set(job_ptr->details->req_node_bitmap,
avail_bitmap))) ||
(job_req_node_filter(job_ptr, avail_bitmap))) {
if (later_start) {
job_ptr->start_time = 0;
goto TRY_LATER;
}
/* Job can not start until too far in the future */
job_ptr->time_limit = orig_time_limit;
job_ptr->start_time = sched_start + backfill_window;
continue;
}
/* Identify nodes which are definitely off limits */
FREE_NULL_BITMAP(resv_bitmap);
resv_bitmap = bit_copy(avail_bitmap);
bit_not(resv_bitmap);
/* this is the time consuming operation */
debug2("backfill: entering _try_sched for job %u.",
job_ptr->job_id);
if (!already_counted) {
slurmctld_diag_stats.bf_last_depth_try++;
already_counted = true;
}
if (debug_flags & DEBUG_FLAG_BACKFILL_MAP)
_dump_job_test(job_ptr, avail_bitmap, start_res);
j = _try_sched(job_ptr, &avail_bitmap, min_nodes, max_nodes,
req_nodes, exc_core_bitmap);
now = time(NULL);
if (j != SLURM_SUCCESS) {
job_ptr->time_limit = orig_time_limit;
job_ptr->start_time = 0;
continue; /* not runable */
}
if (start_res > job_ptr->start_time) {
job_ptr->start_time = start_res;
last_job_update = now;
}
if (job_ptr->start_time <= now) { /* Can start now */
uint32_t save_time_limit = job_ptr->time_limit;
uint32_t hard_limit;
bool reset_time = false;
int rc = _start_job(job_ptr, resv_bitmap);
if (qos_ptr && (qos_ptr->flags & QOS_FLAG_NO_RESERVE)) {
if (orig_time_limit == NO_VAL) {
acct_policy_alter_job(
job_ptr, comp_time_limit);
job_ptr->time_limit = comp_time_limit;
} else {
acct_policy_alter_job(
job_ptr, orig_time_limit);
job_ptr->time_limit = orig_time_limit;
}
} else if ((rc == SLURM_SUCCESS) && job_ptr->time_min) {
/* Set time limit as high as possible */
acct_policy_alter_job(job_ptr, comp_time_limit);
job_ptr->time_limit = comp_time_limit;
reset_time = true;
} else if (orig_time_limit == NO_VAL) {
acct_policy_alter_job(job_ptr, comp_time_limit);
job_ptr->time_limit = comp_time_limit;
} else {
acct_policy_alter_job(job_ptr, orig_time_limit);
job_ptr->time_limit = orig_time_limit;
}
if (job_ptr->time_limit == INFINITE)
hard_limit = 365 * 24 * 60; /* one year */
else
hard_limit = job_ptr->time_limit;
job_ptr->end_time = job_ptr->start_time +
(hard_limit * 60);
if (reset_time) {
_reset_job_time_limit(job_ptr, now,
node_space);
time_limit = job_ptr->time_limit;
}
if (rc == ESLURM_ACCOUNTING_POLICY) {
/* Unknown future start time, just skip job */
job_ptr->start_time = 0;
continue;
} else if (rc != SLURM_SUCCESS) {
if (debug_flags & DEBUG_FLAG_BACKFILL) {
info("backfill: planned start of job %u"
" failed: %s", job_ptr->job_id,
slurm_strerror(rc));
}
/* Drop through and reserve these resources.
* Likely due to state changes during sleep.
* Make best-effort based upon original state */
job_ptr->time_limit = orig_time_limit;
later_start = 0;
} else {
/* Started this job, move to next one */
reject_array_job_id = 0;
reject_array_part = NULL;
/* Update the database if job time limit
* changed and move to next job */
if (save_time_limit != job_ptr->time_limit)
jobacct_storage_g_job_start(acct_db_conn,
job_ptr);
job_start_cnt++;
if (max_backfill_jobs_start &&
(job_start_cnt >= max_backfill_jobs_start)){
if (debug_flags & DEBUG_FLAG_BACKFILL) {
info("backfill: bf_max_job_start"
" limit of %d reached",
max_backfill_jobs_start);
}
break;
}
continue;
}
} else {
job_ptr->time_limit = orig_time_limit;
}
start_time = job_ptr->start_time;
end_reserve = job_ptr->start_time + (time_limit * 60);
start_time = (start_time / backfill_resolution) *
backfill_resolution;
end_reserve = (end_reserve / backfill_resolution) *
backfill_resolution;
if (later_start && (start_time > later_start)) {
/* Try later when some nodes currently reserved for
* pending jobs are free */
job_ptr->start_time = 0;
goto TRY_LATER;
}
if (job_ptr->start_time > (sched_start + backfill_window)) {
/* Starts too far in the future to worry about */
if (debug_flags & DEBUG_FLAG_BACKFILL)
_dump_job_sched(job_ptr, end_reserve,
avail_bitmap);
continue;
}
if (node_space_recs >= max_backfill_job_cnt) {
if (debug_flags & DEBUG_FLAG_BACKFILL) {
info("backfill: table size limit of %u reached",
max_backfill_job_cnt);
}
break;
}
if ((job_ptr->start_time > now) &&
_test_resv_overlap(node_space, avail_bitmap,
start_time, end_reserve)) {
/* This job overlaps with an existing reservation for
* job to be backfill scheduled, which the sched
* plugin does not know about. Try again later. */
later_start = job_ptr->start_time;
job_ptr->start_time = 0;
goto TRY_LATER;
}
/*
* Add reservation to scheduling table if appropriate
*/
if (debug_flags & DEBUG_FLAG_BACKFILL)
_dump_job_sched(job_ptr, end_reserve, avail_bitmap);
if (qos_ptr && (qos_ptr->flags & QOS_FLAG_NO_RESERVE))
continue;
reject_array_job_id = 0;
reject_array_part = NULL;
if (debug_flags & DEBUG_FLAG_BACKFILL)
_dump_job_sched(job_ptr, end_reserve, avail_bitmap);
xfree(job_ptr->sched_nodes);
job_ptr->sched_nodes = bitmap2node_name(avail_bitmap);
bit_not(avail_bitmap);
_add_reservation(start_time, end_reserve,
avail_bitmap, node_space, &node_space_recs);
if (debug_flags & DEBUG_FLAG_BACKFILL_MAP)
_dump_node_space_table(node_space);
}
xfree(bf_part_jobs);
xfree(bf_part_ptr);
xfree(uid);
xfree(njobs);
FREE_NULL_BITMAP(avail_bitmap);
FREE_NULL_BITMAP(exc_core_bitmap);
FREE_NULL_BITMAP(resv_bitmap);
FREE_NULL_BITMAP(non_cg_bitmap);
for (i=0; ; ) {
FREE_NULL_BITMAP(node_space[i].avail_bitmap);
if ((i = node_space[i].next) == 0)
break;
}
xfree(node_space);
list_destroy(job_queue);
gettimeofday(&bf_time2, NULL);
_do_diag_stats(&bf_time1, &bf_time2, yield_sleep);
if (debug_flags & DEBUG_FLAG_BACKFILL) {
END_TIMER;
info("backfill: completed testing %u(%d) jobs, %s",
slurmctld_diag_stats.bf_last_depth,
job_test_count, TIME_STR);
}
return rc;
}
/* Create a reservation for a job in the future */
static void _add_reservation(uint32_t start_time, uint32_t end_reserve,
bitstr_t *res_bitmap,
node_space_map_t *node_space,
int *node_space_recs)
{
bool placed = false;
int i, j;
#if 0
info("add job start:%u end:%u", start_time, end_reserve);
for (j = 0; ; ) {
info("node start:%u end:%u",
(uint32_t) node_space[j].begin_time,
(uint32_t) node_space[j].end_time);
if ((j = node_space[j].next) == 0)
break;
}
#endif
start_time = MAX(start_time, node_space[0].begin_time);
for (j = 0; ; ) {
if (node_space[j].end_time > start_time) {
/* insert start entry record */
i = *node_space_recs;
node_space[i].begin_time = start_time;
node_space[i].end_time = node_space[j].end_time;
node_space[j].end_time = start_time;
node_space[i].avail_bitmap =
bit_copy(node_space[j].avail_bitmap);
node_space[i].next = node_space[j].next;
node_space[j].next = i;
(*node_space_recs)++;
placed = true;
}
if (node_space[j].end_time == start_time) {
/* no need to insert new start entry record */
placed = true;
}
if (placed == true) {
while ((j = node_space[j].next)) {
if (end_reserve < node_space[j].end_time) {
/* insert end entry record */
i = *node_space_recs;
node_space[i].begin_time = end_reserve;
node_space[i].end_time = node_space[j].
end_time;
node_space[j].end_time = end_reserve;
node_space[i].avail_bitmap =
bit_copy(node_space[j].
avail_bitmap);
node_space[i].next = node_space[j].next;
node_space[j].next = i;
(*node_space_recs)++;
break;
}
if (end_reserve == node_space[j].end_time) {
break;
}
}
break;
}
if ((j = node_space[j].next) == 0)
break;
}
for (j = 0; ; ) {
if ((node_space[j].begin_time >= start_time) &&
(node_space[j].end_time <= end_reserve))
bit_and(node_space[j].avail_bitmap, res_bitmap);
if ((node_space[j].begin_time >= end_reserve) ||
((j = node_space[j].next) == 0))
break;
}
/* Drop records with identical bitmaps (up to one record).
* This can significantly improve performance of the backfill tests. */
for (i = 0; ; ) {
if ((j = node_space[i].next) == 0)
break;
if (!bit_equal(node_space[i].avail_bitmap,
node_space[j].avail_bitmap)) {
i = j;
continue;
}
node_space[i].end_time = node_space[j].end_time;
node_space[i].next = node_space[j].next;
FREE_NULL_BITMAP(node_space[j].avail_bitmap);
break;
}
}
static int _attempt_backfill(void)
{
DEF_TIMERS;
bool filter_root = false;
List job_queue;
job_queue_rec_t *job_queue_rec;
slurmdb_qos_rec_t *qos_ptr = NULL;
int i, j, node_space_recs;
struct job_record *job_ptr;
struct part_record *part_ptr;
uint32_t end_time, end_reserve;
uint32_t time_limit, comp_time_limit, orig_time_limit;
uint32_t min_nodes, max_nodes, req_nodes;
bitstr_t *avail_bitmap = NULL, *resv_bitmap = NULL;
bitstr_t *exc_core_bitmap = NULL;
time_t now, sched_start, later_start, start_res, resv_end;
node_space_map_t *node_space;
struct timeval bf_time1, bf_time2;
int sched_timeout = 2, yield_sleep = 1;
int rc = 0;
int job_test_count = 0;
uint32_t *uid = NULL, nuser = 0;
uint16_t *njobs = NULL;
bool already_counted;
uint32_t reject_array_job_id = 0;
#ifdef HAVE_CRAY
/*
* Run a Basil Inventory immediately before setting up the schedule
* plan, to avoid race conditions caused by ALPS node state change.
* Needs to be done with the node-state lock taken.
*/
START_TIMER;
if (select_g_reconfigure()) {
debug4("backfill: not scheduling due to ALPS");
return SLURM_SUCCESS;
}
END_TIMER;
if (debug_flags & DEBUG_FLAG_BACKFILL)
info("backfill: ALPS inventory completed, %s", TIME_STR);
/* The Basil inventory can take a long time to complete. Process
* pending RPCs before starting the backfill scheduling logic */
_yield_locks(1);
#endif
START_TIMER;
if (debug_flags & DEBUG_FLAG_BACKFILL)
info("backfill: beginning");
sched_start = now = time(NULL);
if (slurm_get_root_filter())
filter_root = true;
job_queue = build_job_queue(true);
if (list_count(job_queue) == 0) {
debug("backfill: no jobs to backfill");
list_destroy(job_queue);
return 0;
}
gettimeofday(&bf_time1, NULL);
slurmctld_diag_stats.bf_queue_len = list_count(job_queue);
slurmctld_diag_stats.bf_queue_len_sum += slurmctld_diag_stats.
bf_queue_len;
slurmctld_diag_stats.bf_last_depth = 0;
slurmctld_diag_stats.bf_last_depth_try = 0;
slurmctld_diag_stats.bf_when_last_cycle = now;
bf_last_yields = 0;
slurmctld_diag_stats.bf_active = 1;
node_space = xmalloc(sizeof(node_space_map_t) *
(max_backfill_job_cnt + 3));
node_space[0].begin_time = sched_start;
node_space[0].end_time = sched_start + backfill_window;
node_space[0].avail_bitmap = bit_copy(avail_node_bitmap);
node_space[0].next = 0;
node_space_recs = 1;
if (debug_flags & DEBUG_FLAG_BACKFILL)
_dump_node_space_table(node_space);
if (max_backfill_job_per_user) {
uid = xmalloc(BF_MAX_USERS * sizeof(uint32_t));
njobs = xmalloc(BF_MAX_USERS * sizeof(uint16_t));
}
while ((job_queue_rec = (job_queue_rec_t *)
list_pop_bottom(job_queue, sort_job_queue2))) {
job_ptr = job_queue_rec->job_ptr;
orig_time_limit = job_ptr->time_limit;
if ((time(NULL) - sched_start) >= sched_timeout) {
uint32_t save_time_limit = job_ptr->time_limit;
job_ptr->time_limit = orig_time_limit;
if (debug_flags & DEBUG_FLAG_BACKFILL) {
END_TIMER;
info("backfill: completed yielding locks "
"after testing %d jobs, %s",
job_test_count, TIME_STR);
}
if (_yield_locks(yield_sleep) && !backfill_continue) {
if (debug_flags & DEBUG_FLAG_BACKFILL) {
info("backfill: system state changed, "
"breaking out after testing %d "
"jobs", job_test_count);
}
rc = 1;
break;
}
job_ptr->time_limit = save_time_limit;
/* Reset backfill scheduling timers, resume testing */
sched_start = time(NULL);
job_test_count = 0;
START_TIMER;
}
part_ptr = job_queue_rec->part_ptr;
job_test_count++;
xfree(job_queue_rec);
if (!IS_JOB_PENDING(job_ptr))
continue; /* started in other partition */
if (!avail_front_end(job_ptr))
continue; /* No available frontend for this job */
if (job_ptr->array_task_id != (uint16_t) NO_VAL) {
if (reject_array_job_id == job_ptr->array_job_id)
continue; /* already rejected array element */
/* assume reject whole array for now, clear if OK */
reject_array_job_id = job_ptr->array_job_id;
}
job_ptr->part_ptr = part_ptr;
if (debug_flags & DEBUG_FLAG_BACKFILL)
info("backfill test for job %u", job_ptr->job_id);
slurmctld_diag_stats.bf_last_depth++;
already_counted = false;
if (max_backfill_job_per_user) {
for (j = 0; j < nuser; j++) {
if (job_ptr->user_id == uid[j]) {
njobs[j]++;
if (debug_flags & DEBUG_FLAG_BACKFILL)
debug("backfill: user %u: "
"#jobs %u",
uid[j], njobs[j]);
break;
}
}
if (j == nuser) { /* user not found */
if (nuser < BF_MAX_USERS) {
uid[j] = job_ptr->user_id;
njobs[j] = 1;
nuser++;
} else {
error("backfill: too many users in "
"queue. Consider increasing "
"BF_MAX_USERS");
}
if (debug_flags & DEBUG_FLAG_BACKFILL)
debug2("backfill: found new user %u. "
"Total #users now %u",
job_ptr->user_id, nuser);
} else {
if (njobs[j] > max_backfill_job_per_user) {
/* skip job */
if (debug_flags & DEBUG_FLAG_BACKFILL)
debug("backfill: have already "
"checked %u jobs for "
"user %u; skipping "
"job %u",
max_backfill_job_per_user,
job_ptr->user_id,
job_ptr->job_id);
continue;
}
}
}
if (((part_ptr->state_up & PARTITION_SCHED) == 0) ||
(part_ptr->node_bitmap == NULL))
continue;
if ((part_ptr->flags & PART_FLAG_ROOT_ONLY) && filter_root)
continue;
if ((!job_independent(job_ptr, 0)) ||
(license_job_test(job_ptr, time(NULL)) != SLURM_SUCCESS))
continue;
/* Determine minimum and maximum node counts */
min_nodes = MAX(job_ptr->details->min_nodes,
part_ptr->min_nodes);
if (job_ptr->details->max_nodes == 0)
max_nodes = part_ptr->max_nodes;
else
max_nodes = MIN(job_ptr->details->max_nodes,
part_ptr->max_nodes);
max_nodes = MIN(max_nodes, 500000); /* prevent overflows */
if (job_ptr->details->max_nodes)
req_nodes = max_nodes;
else
req_nodes = min_nodes;
if (min_nodes > max_nodes) {
/* job's min_nodes exceeds partition's max_nodes */
continue;
}
/* Determine job's expected completion time */
if (job_ptr->time_limit == NO_VAL) {
if (part_ptr->max_time == INFINITE)
time_limit = 365 * 24 * 60; /* one year */
else
time_limit = part_ptr->max_time;
} else {
if (part_ptr->max_time == INFINITE)
time_limit = job_ptr->time_limit;
else
time_limit = MIN(job_ptr->time_limit,
part_ptr->max_time);
}
comp_time_limit = time_limit;
qos_ptr = job_ptr->qos_ptr;
if (qos_ptr && (qos_ptr->flags & QOS_FLAG_NO_RESERVE) &&
slurm_get_preempt_mode())
time_limit = job_ptr->time_limit = 1;
else if (job_ptr->time_min && (job_ptr->time_min < time_limit))
time_limit = job_ptr->time_limit = job_ptr->time_min;
/* Determine impact of any resource reservations */
later_start = now;
TRY_LATER:
if ((time(NULL) - sched_start) >= sched_timeout) {
uint32_t save_time_limit = job_ptr->time_limit;
job_ptr->time_limit = orig_time_limit;
if (debug_flags & DEBUG_FLAG_BACKFILL) {
END_TIMER;
info("backfill: completed yielding locks 2"
"after testing %d jobs, %s",
job_test_count, TIME_STR);
}
if (_yield_locks(yield_sleep) && !backfill_continue) {
if (debug_flags & DEBUG_FLAG_BACKFILL) {
info("backfill: system state changed, "
"breaking out after testing %d "
"jobs", job_test_count);
}
rc = 1;
break;
}
job_ptr->time_limit = save_time_limit;
/* Reset backfill scheduling timers, resume testing */
sched_start = time(NULL);
job_test_count = 1;
START_TIMER;
}
FREE_NULL_BITMAP(avail_bitmap);
FREE_NULL_BITMAP(exc_core_bitmap);
start_res = later_start;
later_start = 0;
j = job_test_resv(job_ptr, &start_res, true, &avail_bitmap,
&exc_core_bitmap);
if (j != SLURM_SUCCESS) {
job_ptr->time_limit = orig_time_limit;
continue;
}
if (start_res > now)
end_time = (time_limit * 60) + start_res;
else
end_time = (time_limit * 60) + now;
resv_end = find_resv_end(start_res);
/* Identify usable nodes for this job */
bit_and(avail_bitmap, part_ptr->node_bitmap);
bit_and(avail_bitmap, up_node_bitmap);
for (j=0; ; ) {
if ((node_space[j].end_time > start_res) &&
node_space[j].next && (later_start == 0))
later_start = node_space[j].end_time;
if (node_space[j].end_time <= start_res)
;
else if (node_space[j].begin_time <= end_time) {
bit_and(avail_bitmap,
node_space[j].avail_bitmap);
} else
break;
if ((j = node_space[j].next) == 0)
break;
}
if ((resv_end++) &&
((later_start == 0) || (resv_end < later_start))) {
later_start = resv_end;
}
if (job_ptr->details->exc_node_bitmap) {
bit_not(job_ptr->details->exc_node_bitmap);
bit_and(avail_bitmap,
job_ptr->details->exc_node_bitmap);
bit_not(job_ptr->details->exc_node_bitmap);
}
/* Test if insufficient nodes remain OR
* required nodes missing OR
* nodes lack features */
if ((bit_set_count(avail_bitmap) < min_nodes) ||
((job_ptr->details->req_node_bitmap) &&
(!bit_super_set(job_ptr->details->req_node_bitmap,
avail_bitmap))) ||
(job_req_node_filter(job_ptr, avail_bitmap))) {
if (later_start) {
job_ptr->start_time = 0;
goto TRY_LATER;
}
/* Job can not start until too far in the future */
job_ptr->time_limit = orig_time_limit;
job_ptr->start_time = sched_start + backfill_window;
continue;
}
/* Identify nodes which are definitely off limits */
FREE_NULL_BITMAP(resv_bitmap);
resv_bitmap = bit_copy(avail_bitmap);
bit_not(resv_bitmap);
/* this is the time consuming operation */
debug2("backfill: entering _try_sched for job %u.",
job_ptr->job_id);
if (!already_counted) {
slurmctld_diag_stats.bf_last_depth_try++;
already_counted = true;
}
j = _try_sched(job_ptr, &avail_bitmap, min_nodes, max_nodes,
req_nodes, exc_core_bitmap);
now = time(NULL);
if (j != SLURM_SUCCESS) {
job_ptr->time_limit = orig_time_limit;
job_ptr->start_time = 0;
continue; /* not runable */
}
if (start_res > job_ptr->start_time) {
job_ptr->start_time = start_res;
last_job_update = now;
}
if (job_ptr->start_time <= now) {
uint32_t save_time_limit = job_ptr->time_limit;
int rc = _start_job(job_ptr, resv_bitmap);
if (qos_ptr && (qos_ptr->flags & QOS_FLAG_NO_RESERVE)) {
if (orig_time_limit == NO_VAL)
job_ptr->time_limit = comp_time_limit;
else
job_ptr->time_limit = orig_time_limit;
job_ptr->end_time = job_ptr->start_time +
(job_ptr->time_limit * 60);
} else if ((rc == SLURM_SUCCESS) && job_ptr->time_min) {
/* Set time limit as high as possible */
job_ptr->time_limit = comp_time_limit;
job_ptr->end_time = job_ptr->start_time +
(comp_time_limit * 60);
_reset_job_time_limit(job_ptr, now,
node_space);
time_limit = job_ptr->time_limit;
} else {
job_ptr->time_limit = orig_time_limit;
}
if (rc == ESLURM_ACCOUNTING_POLICY) {
/* Unknown future start time, just skip job */
job_ptr->start_time = 0;
continue;
} else if (rc != SLURM_SUCCESS) {
/* Planned to start job, but something bad
* happended. */
job_ptr->start_time = 0;
break;
} else {
/* Started this job, move to next one */
reject_array_job_id = 0;
/* Update the database if job time limit
* changed and move to next job */
if (save_time_limit != job_ptr->time_limit)
jobacct_storage_g_job_start(acct_db_conn,
job_ptr);
continue;
}
} else
job_ptr->time_limit = orig_time_limit;
if (later_start && (job_ptr->start_time > later_start)) {
/* Try later when some nodes currently reserved for
* pending jobs are free */
job_ptr->start_time = 0;
goto TRY_LATER;
}
if (job_ptr->start_time > (sched_start + backfill_window)) {
/* Starts too far in the future to worry about */
continue;
}
if (node_space_recs >= max_backfill_job_cnt) {
/* Already have too many jobs to deal with */
break;
}
end_reserve = job_ptr->start_time + (time_limit * 60);
if (_test_resv_overlap(node_space, avail_bitmap,
job_ptr->start_time, end_reserve)) {
/* This job overlaps with an existing reservation for
* job to be backfill scheduled, which the sched
* plugin does not know about. Try again later. */
later_start = job_ptr->start_time;
job_ptr->start_time = 0;
goto TRY_LATER;
}
/*
* Add reservation to scheduling table if appropriate
*/
if (qos_ptr && (qos_ptr->flags & QOS_FLAG_NO_RESERVE))
continue;
reject_array_job_id = 0;
bit_not(avail_bitmap);
_add_reservation(job_ptr->start_time, end_reserve,
avail_bitmap, node_space, &node_space_recs);
if (debug_flags & DEBUG_FLAG_BACKFILL)
_dump_node_space_table(node_space);
}
xfree(uid);
xfree(njobs);
FREE_NULL_BITMAP(avail_bitmap);
FREE_NULL_BITMAP(exc_core_bitmap);
FREE_NULL_BITMAP(resv_bitmap);
for (i=0; ; ) {
FREE_NULL_BITMAP(node_space[i].avail_bitmap);
if ((i = node_space[i].next) == 0)
break;
}
xfree(node_space);
list_destroy(job_queue);
gettimeofday(&bf_time2, NULL);
_do_diag_stats(&bf_time1, &bf_time2, yield_sleep);
if (debug_flags & DEBUG_FLAG_BACKFILL) {
END_TIMER;
info("backfill: completed testing %d jobs, %s",
job_test_count, TIME_STR);
}
return rc;
}
static void _validate_switches(void)
{
slurm_conf_switches_t *ptr, **ptr_array;
int depth, i, j;
struct switch_record *switch_ptr, *prior_ptr;
hostlist_t hl, invalid_hl = NULL;
char *child, *buf;
bool have_root = false;
bitstr_t *multi_homed_bitmap = NULL; /* nodes on >1 leaf switch */
bitstr_t *switches_bitmap = NULL; /* nodes on any leaf switch */
bitstr_t *tmp_bitmap = NULL;
_free_switch_record_table();
switch_record_cnt = _read_topo_file(&ptr_array);
if (switch_record_cnt == 0) {
error("No switches configured");
s_p_hashtbl_destroy(conf_hashtbl);
return;
}
switch_record_table = xmalloc(sizeof(struct switch_record) *
switch_record_cnt);
multi_homed_bitmap = bit_alloc(node_record_count);
switch_ptr = switch_record_table;
for (i=0; i<switch_record_cnt; i++, switch_ptr++) {
ptr = ptr_array[i];
switch_ptr->name = xstrdup(ptr->switch_name);
/* See if switch name has already been defined. */
prior_ptr = switch_record_table;
for (j=0; j<i; j++, prior_ptr++) {
if (strcmp(switch_ptr->name, prior_ptr->name) == 0) {
fatal("Switch (%s) has already been defined",
prior_ptr->name);
}
}
switch_ptr->link_speed = ptr->link_speed;
if (ptr->nodes) {
switch_ptr->level = 0; /* leaf switch */
switch_ptr->nodes = xstrdup(ptr->nodes);
if (_node_name2bitmap(ptr->nodes,
&switch_ptr->node_bitmap,
&invalid_hl)) {
fatal("Invalid node name (%s) in switch "
"config (%s)",
ptr->nodes, ptr->switch_name);
}
if (switches_bitmap) {
tmp_bitmap = bit_copy(switch_ptr->node_bitmap);
bit_and(tmp_bitmap, switches_bitmap);
bit_or(multi_homed_bitmap, tmp_bitmap);
FREE_NULL_BITMAP(tmp_bitmap);
bit_or(switches_bitmap,
switch_ptr->node_bitmap);
} else {
switches_bitmap = bit_copy(switch_ptr->
node_bitmap);
}
} else if (ptr->switches) {
switch_ptr->level = -1; /* determine later */
switch_ptr->switches = xstrdup(ptr->switches);
} else {
fatal("Switch configuration (%s) lacks children",
ptr->switch_name);
}
}
for (depth=1; ; depth++) {
bool resolved = true;
switch_ptr = switch_record_table;
for (i=0; i<switch_record_cnt; i++, switch_ptr++) {
if (switch_ptr->level != -1)
continue;
hl = hostlist_create(switch_ptr->switches);
if (!hl) {
fatal("Invalid switches: %s",
switch_ptr->switches);
}
while ((child = hostlist_pop(hl))) {
j = _get_switch_inx(child);
if ((j < 0) || (j == i)) {
fatal("Switch configuration %s has "
"invalid child (%s)",
switch_ptr->name, child);
}
if (switch_record_table[j].level == -1) {
/* Children not resolved */
resolved = false;
switch_ptr->level = -1;
FREE_NULL_BITMAP(switch_ptr->
node_bitmap);
free(child);
break;
}
if (switch_ptr->level == -1) {
switch_ptr->level = 1 +
switch_record_table[j].level;
switch_ptr->node_bitmap =
bit_copy(switch_record_table[j].
node_bitmap);
} else {
switch_ptr->level =
MAX(switch_ptr->level,
(switch_record_table[j].
level + 1));
bit_or(switch_ptr->node_bitmap,
switch_record_table[j].
node_bitmap);
}
free(child);
}
hostlist_destroy(hl);
}
if (resolved)
break;
if (depth > 20) /* Prevent infinite loop */
fatal("Switch configuration is not a tree");
}
switch_levels = 0;
switch_ptr = switch_record_table;
for (i=0; i<switch_record_cnt; i++, switch_ptr++) {
switch_levels = MAX(switch_levels, switch_ptr->level);
if (switch_ptr->node_bitmap == NULL)
error("switch %s has no nodes", switch_ptr->name);
}
if (switches_bitmap) {
bit_not(switches_bitmap);
i = bit_set_count(switches_bitmap);
if (i > 0) {
child = bitmap2node_name(switches_bitmap);
error("WARNING: switches lack access to %d nodes: %s",
i, child);
xfree(child);
}
FREE_NULL_BITMAP(switches_bitmap);
} else
fatal("switches contain no nodes");
if (invalid_hl) {
buf = hostlist_ranged_string_xmalloc(invalid_hl);
error("WARNING: Invalid hostnames in switch configuration: %s",
buf);
xfree(buf);
hostlist_destroy(invalid_hl);
}
/* Report nodes on multiple leaf switches,
* possibly due to bad configuration file */
i = bit_set_count(multi_homed_bitmap);
if (i > 0) {
child = bitmap2node_name(multi_homed_bitmap);
error("WARNING: Multiple leaf switches contain nodes: %s",
child);
xfree(child);
}
FREE_NULL_BITMAP(multi_homed_bitmap);
/* Create array of indexes of children of each switch,
* and see if any switch can reach all nodes */
for (i = 0; i < switch_record_cnt; i++) {
if (switch_record_table[i].level != 0) {
_find_child_switches (i);
}
if (node_record_count ==
bit_set_count(switch_record_table[i].node_bitmap)) {
have_root = true;
}
}
if (!have_root) {
info("TOPOLOGY: warning -- no switch can reach all nodes"
" through its descendants."
"Do not use route/topology");
}
s_p_hashtbl_destroy(conf_hashtbl);
_log_switches();
}
