static int _attempt_backfill(void)
{
DEF_TIMERS;
bool filter_root = false;
List job_queue;
job_queue_rec_t *job_queue_rec;
slurmdb_qos_rec_t *qos_ptr = NULL;
int i, j, node_space_recs;
struct job_record *job_ptr;
struct part_record *part_ptr, **bf_part_ptr = NULL;
uint32_t end_time, end_reserve;
uint32_t time_limit, comp_time_limit, orig_time_limit, part_time_limit;
uint32_t min_nodes, max_nodes, req_nodes;
bitstr_t *avail_bitmap = NULL, *resv_bitmap = NULL;
bitstr_t *exc_core_bitmap = NULL, *non_cg_bitmap = NULL;
bitstr_t *previous_bitmap = NULL;
time_t now, sched_start, later_start, start_res, resv_end;
node_space_map_t *node_space;
struct timeval bf_time1, bf_time2;
int rc = 0;
int job_test_count = 0;
uint32_t *uid = NULL, nuser = 0, bf_parts = 0, *bf_part_jobs = NULL;
uint16_t *njobs = NULL;
bool already_counted;
uint32_t reject_array_job_id = 0;
struct part_record *reject_array_part = NULL;
uint32_t job_start_cnt = 0;
time_t config_update = slurmctld_conf.last_update;
time_t part_update = last_part_update;
struct timeval start_tv;
bf_last_yields = 0;
#ifdef HAVE_ALPS_CRAY
/*
* Run a Basil Inventory immediately before setting up the schedule
* plan, to avoid race conditions caused by ALPS node state change.
* Needs to be done with the node-state lock taken.
*/
START_TIMER;
if (select_g_reconfigure()) {
debug4("backfill: not scheduling due to ALPS");
return SLURM_SUCCESS;
}
END_TIMER;
if (debug_flags & DEBUG_FLAG_BACKFILL)
info("backfill: ALPS inventory completed, %s", TIME_STR);
/* The Basil inventory can take a long time to complete. Process
* pending RPCs before starting the backfill scheduling logic */
_yield_locks(1000000);
#endif
START_TIMER;
if (debug_flags & DEBUG_FLAG_BACKFILL)
info("backfill: beginning");
else
debug("backfill: beginning");
sched_start = now = time(NULL);
gettimeofday(&start_tv, NULL);
if (slurm_get_root_filter())
filter_root = true;
job_queue = build_job_queue(true, true);
if (list_count(job_queue) == 0) {
debug("backfill: no jobs to backfill");
list_destroy(job_queue);
return 0;
}
gettimeofday(&bf_time1, NULL);
non_cg_bitmap = bit_copy(cg_node_bitmap);
bit_not(non_cg_bitmap);
slurmctld_diag_stats.bf_queue_len = list_count(job_queue);
slurmctld_diag_stats.bf_queue_len_sum += slurmctld_diag_stats.
bf_queue_len;
slurmctld_diag_stats.bf_last_depth = 0;
slurmctld_diag_stats.bf_last_depth_try = 0;
slurmctld_diag_stats.bf_when_last_cycle = now;
slurmctld_diag_stats.bf_active = 1;
node_space = xmalloc(sizeof(node_space_map_t) *
(max_backfill_job_cnt * 2 + 1));
node_space[0].begin_time = sched_start;
node_space[0].end_time = sched_start + backfill_window;
node_space[0].avail_bitmap = bit_copy(avail_node_bitmap);
node_space[0].next = 0;
node_space_recs = 1;
if (debug_flags & DEBUG_FLAG_BACKFILL)
_dump_node_space_table(node_space);
if (max_backfill_job_per_part) {
ListIterator part_iterator;
struct part_record *part_ptr;
bf_parts = list_count(part_list);
bf_part_ptr = xmalloc(sizeof(struct part_record *) * bf_parts);
bf_part_jobs = xmalloc(sizeof(int) * bf_parts);
part_iterator = list_iterator_create(part_list);
i = 0;
while ((part_ptr = (struct part_record *)
list_next(part_iterator))) {
bf_part_ptr[i++] = part_ptr;
}
list_iterator_destroy(part_iterator);
}
if (max_backfill_job_per_user) {
uid = xmalloc(BF_MAX_USERS * sizeof(uint32_t));
njobs = xmalloc(BF_MAX_USERS * sizeof(uint16_t));
}
sort_job_queue(job_queue);
while ((job_queue_rec = (job_queue_rec_t *) list_pop(job_queue))) {
if (slurmctld_config.shutdown_time)
break;
if (((defer_rpc_cnt > 0) &&
(slurmctld_config.server_thread_count >= defer_rpc_cnt)) ||
(_delta_tv(&start_tv) >= sched_timeout)) {
if (debug_flags & DEBUG_FLAG_BACKFILL) {
END_TIMER;
info("backfill: completed yielding locks "
"after testing %d jobs, %s",
job_test_count, TIME_STR);
}
if ((_yield_locks(yield_sleep) && !backfill_continue) ||
(slurmctld_conf.last_update != config_update) ||
(last_part_update != part_update)) {
if (debug_flags & DEBUG_FLAG_BACKFILL) {
info("backfill: system state changed, "
"breaking out after testing %d "
"jobs", job_test_count);
}
rc = 1;
xfree(job_queue_rec);
break;
}
/* cg_node_bitmap may be changed */
bit_copybits(non_cg_bitmap, cg_node_bitmap);
bit_not(non_cg_bitmap);
/* Reset backfill scheduling timers, resume testing */
sched_start = time(NULL);
gettimeofday(&start_tv, NULL);
job_test_count = 0;
START_TIMER;
}
job_ptr = job_queue_rec->job_ptr;
/* With bf_continue configured, the original job could have
* been cancelled and purged. Validate pointer here. */
if ((job_ptr->magic != JOB_MAGIC) ||
(job_ptr->job_id != job_queue_rec->job_id)) {
xfree(job_queue_rec);
continue;
}
orig_time_limit = job_ptr->time_limit;
part_ptr = job_queue_rec->part_ptr;
job_test_count++;
xfree(job_queue_rec);
if (!IS_JOB_PENDING(job_ptr))
continue; /* started in other partition */
if (!avail_front_end(job_ptr))
continue; /* No available frontend for this job */
if (job_ptr->array_task_id != NO_VAL) {
if ((reject_array_job_id == job_ptr->array_job_id) &&
(reject_array_part == part_ptr))
continue; /* already rejected array element */
/* assume reject whole array for now, clear if OK */
reject_array_job_id = job_ptr->array_job_id;
reject_array_part = part_ptr;
}
job_ptr->part_ptr = part_ptr;
if (debug_flags & DEBUG_FLAG_BACKFILL)
info("backfill test for job %u", job_ptr->job_id);
slurmctld_diag_stats.bf_last_depth++;
already_counted = false;
if (max_backfill_job_per_part) {
bool skip_job = false;
for (j = 0; j < bf_parts; j++) {
if (bf_part_ptr[j] != job_ptr->part_ptr)
continue;
if (bf_part_jobs[j]++ >=
max_backfill_job_per_part)
skip_job = true;
break;
}
if (skip_job) {
if (debug_flags & DEBUG_FLAG_BACKFILL)
debug("backfill: have already "
"checked %u jobs for "
"partition %s; skipping "
"job %u",
max_backfill_job_per_part,
job_ptr->part_ptr->name,
job_ptr->job_id);
continue;
}
}
if (max_backfill_job_per_user) {
for (j = 0; j < nuser; j++) {
if (job_ptr->user_id == uid[j]) {
njobs[j]++;
if (debug_flags & DEBUG_FLAG_BACKFILL)
debug("backfill: user %u: "
"#jobs %u",
uid[j], njobs[j]);
break;
}
}
if (j == nuser) { /* user not found */
static bool bf_max_user_msg = true;
if (nuser < BF_MAX_USERS) {
uid[j] = job_ptr->user_id;
njobs[j] = 1;
nuser++;
} else if (bf_max_user_msg) {
bf_max_user_msg = false;
error("backfill: too many users in "
"queue. Consider increasing "
"BF_MAX_USERS");
}
if (debug_flags & DEBUG_FLAG_BACKFILL)
debug2("backfill: found new user %u. "
"Total #users now %u",
job_ptr->user_id, nuser);
} else {
if (njobs[j] >= max_backfill_job_per_user) {
/* skip job */
if (debug_flags & DEBUG_FLAG_BACKFILL)
debug("backfill: have already "
"checked %u jobs for "
"user %u; skipping "
"job %u",
max_backfill_job_per_user,
job_ptr->user_id,
job_ptr->job_id);
continue;
}
}
}
if (((part_ptr->state_up & PARTITION_SCHED) == 0) ||
(part_ptr->node_bitmap == NULL))
continue;
if ((part_ptr->flags & PART_FLAG_ROOT_ONLY) && filter_root)
continue;
if ((!job_independent(job_ptr, 0)) ||
(license_job_test(job_ptr, time(NULL)) != SLURM_SUCCESS))
continue;
/* Determine minimum and maximum node counts */
min_nodes = MAX(job_ptr->details->min_nodes,
part_ptr->min_nodes);
if (job_ptr->details->max_nodes == 0)
max_nodes = part_ptr->max_nodes;
else
max_nodes = MIN(job_ptr->details->max_nodes,
part_ptr->max_nodes);
max_nodes = MIN(max_nodes, 500000); /* prevent overflows */
if (job_ptr->details->max_nodes)
req_nodes = max_nodes;
else
req_nodes = min_nodes;
if (min_nodes > max_nodes) {
/* job's min_nodes exceeds partition's max_nodes */
continue;
}
/* Determine job's expected completion time */
if (part_ptr->max_time == INFINITE)
part_time_limit = 365 * 24 * 60; /* one year */
else
part_time_limit = part_ptr->max_time;
if (job_ptr->time_limit == NO_VAL) {
time_limit = part_time_limit;
} else {
if (part_ptr->max_time == INFINITE)
time_limit = job_ptr->time_limit;
else
time_limit = MIN(job_ptr->time_limit,
part_time_limit);
}
comp_time_limit = time_limit;
qos_ptr = job_ptr->qos_ptr;
if (qos_ptr && (qos_ptr->flags & QOS_FLAG_NO_RESERVE) &&
slurm_get_preempt_mode())
time_limit = job_ptr->time_limit = 1;
else if (job_ptr->time_min && (job_ptr->time_min < time_limit))
time_limit = job_ptr->time_limit = job_ptr->time_min;
/* Determine impact of any resource reservations */
later_start = now;
FREE_NULL_BITMAP(previous_bitmap);
TRY_LATER:
if (slurmctld_config.shutdown_time)
break;
if (((defer_rpc_cnt > 0) &&
(slurmctld_config.server_thread_count >= defer_rpc_cnt)) ||
(_delta_tv(&start_tv) >= sched_timeout)) {
uint32_t save_job_id = job_ptr->job_id;
uint32_t save_time_limit = job_ptr->time_limit;
job_ptr->time_limit = orig_time_limit;
if (debug_flags & DEBUG_FLAG_BACKFILL) {
END_TIMER;
info("backfill: completed yielding locks "
"after testing %d jobs, %s",
job_test_count, TIME_STR);
}
if ((_yield_locks(yield_sleep) && !backfill_continue) ||
(slurmctld_conf.last_update != config_update) ||
(last_part_update != part_update)) {
if (debug_flags & DEBUG_FLAG_BACKFILL) {
info("backfill: system state changed, "
"breaking out after testing %d "
"jobs", job_test_count);
}
rc = 1;
break;
}
/* cg_node_bitmap may be changed */
bit_copybits(non_cg_bitmap, cg_node_bitmap);
bit_not(non_cg_bitmap);
/* With bf_continue configured, the original job could
* have been scheduled or cancelled and purged.
* Revalidate job the record here. */
if ((job_ptr->magic != JOB_MAGIC) ||
(job_ptr->job_id != save_job_id))
continue;
if (!IS_JOB_PENDING(job_ptr))
continue;
if (!avail_front_end(job_ptr))
continue; /* No available frontend */
job_ptr->time_limit = save_time_limit;
/* Reset backfill scheduling timers, resume testing */
sched_start = time(NULL);
gettimeofday(&start_tv, NULL);
job_test_count = 1;
START_TIMER;
}
FREE_NULL_BITMAP(avail_bitmap);
FREE_NULL_BITMAP(exc_core_bitmap);
start_res = later_start;
later_start = 0;
j = job_test_resv(job_ptr, &start_res, true, &avail_bitmap,
&exc_core_bitmap);
if (j != SLURM_SUCCESS) {
job_ptr->time_limit = orig_time_limit;
continue;
}
if (start_res > now)
end_time = (time_limit * 60) + start_res;
else
end_time = (time_limit * 60) + now;
resv_end = find_resv_end(start_res);
/* Identify usable nodes for this job */
bit_and(avail_bitmap, part_ptr->node_bitmap);
bit_and(avail_bitmap, up_node_bitmap);
bit_and(avail_bitmap, non_cg_bitmap);
for (j=0; ; ) {
if ((node_space[j].end_time > start_res) &&
node_space[j].next && (later_start == 0))
later_start = node_space[j].end_time;
if (node_space[j].end_time <= start_res)
;
else if (node_space[j].begin_time <= end_time) {
bit_and(avail_bitmap,
node_space[j].avail_bitmap);
} else
break;
if ((j = node_space[j].next) == 0)
break;
}
if ((resv_end++) &&
((later_start == 0) || (resv_end < later_start))) {
later_start = resv_end;
}
if (job_ptr->details->exc_node_bitmap) {
bit_not(job_ptr->details->exc_node_bitmap);
bit_and(avail_bitmap,
job_ptr->details->exc_node_bitmap);
bit_not(job_ptr->details->exc_node_bitmap);
}
/* Test if insufficient nodes remain OR
* required nodes missing OR
* nodes lack features OR
* no change since previously tested nodes (only changes
* in other partition nodes) */
if ((bit_set_count(avail_bitmap) < min_nodes) ||
((job_ptr->details->req_node_bitmap) &&
(!bit_super_set(job_ptr->details->req_node_bitmap,
avail_bitmap))) ||
(job_req_node_filter(job_ptr, avail_bitmap)) ||
(previous_bitmap &&
bit_equal(previous_bitmap, avail_bitmap))) {
if (later_start) {
job_ptr->start_time = 0;
goto TRY_LATER;
}
/* Job can not start until too far in the future */
job_ptr->time_limit = orig_time_limit;
job_ptr->start_time = sched_start + backfill_window;
continue;
}
FREE_NULL_BITMAP(previous_bitmap);
previous_bitmap = bit_copy(avail_bitmap);
/* Identify nodes which are definitely off limits */
FREE_NULL_BITMAP(resv_bitmap);
resv_bitmap = bit_copy(avail_bitmap);
bit_not(resv_bitmap);
/* this is the time consuming operation */
debug2("backfill: entering _try_sched for job %u.",
job_ptr->job_id);
if (!already_counted) {
slurmctld_diag_stats.bf_last_depth_try++;
already_counted = true;
}
if (debug_flags & DEBUG_FLAG_BACKFILL)
_dump_job_test(job_ptr, avail_bitmap, start_res);
j = _try_sched(job_ptr, &avail_bitmap, min_nodes, max_nodes,
req_nodes, exc_core_bitmap);
now = time(NULL);
if (j != SLURM_SUCCESS) {
job_ptr->time_limit = orig_time_limit;
job_ptr->start_time = 0;
continue; /* not runable */
}
if (start_res > job_ptr->start_time) {
job_ptr->start_time = start_res;
last_job_update = now;
}
if (job_ptr->start_time <= now) {
uint32_t save_time_limit = job_ptr->time_limit;
int rc = _start_job(job_ptr, resv_bitmap);
if (qos_ptr && (qos_ptr->flags & QOS_FLAG_NO_RESERVE)) {
if (orig_time_limit == NO_VAL) {
acct_policy_alter_job(
job_ptr, comp_time_limit);
job_ptr->time_limit = comp_time_limit;
} else {
acct_policy_alter_job(
job_ptr, orig_time_limit);
job_ptr->time_limit = orig_time_limit;
}
job_ptr->end_time = job_ptr->start_time +
(job_ptr->time_limit * 60);
} else if ((rc == SLURM_SUCCESS) && job_ptr->time_min) {
/* Set time limit as high as possible */
acct_policy_alter_job(job_ptr, comp_time_limit);
job_ptr->time_limit = comp_time_limit;
job_ptr->end_time = job_ptr->start_time +
(comp_time_limit * 60);
_reset_job_time_limit(job_ptr, now,
node_space);
time_limit = job_ptr->time_limit;
} else if (orig_time_limit == NO_VAL) {
acct_policy_alter_job(job_ptr, comp_time_limit);
job_ptr->time_limit = comp_time_limit;
job_ptr->end_time = job_ptr->start_time +
(job_ptr->time_limit * 60);
} else {
acct_policy_alter_job(job_ptr, orig_time_limit);
job_ptr->time_limit = orig_time_limit;
job_ptr->end_time = job_ptr->start_time +
(job_ptr->time_limit * 60);
}
if (rc == ESLURM_ACCOUNTING_POLICY) {
/* Unknown future start time, just skip job */
job_ptr->start_time = 0;
continue;
} else if (rc != SLURM_SUCCESS) {
/* Planned to start job, but something bad
* happened. */
job_ptr->start_time = 0;
break;
} else {
/* Started this job, move to next one */
reject_array_job_id = 0;
reject_array_part = NULL;
/* Update the database if job time limit
* changed and move to next job */
if (save_time_limit != job_ptr->time_limit)
jobacct_storage_g_job_start(acct_db_conn,
job_ptr);
if (max_backfill_jobs_start &&
(++job_start_cnt >= max_backfill_jobs_start))
break;
continue;
}
} else
job_ptr->time_limit = orig_time_limit;
if (later_start && (job_ptr->start_time > later_start)) {
/* Try later when some nodes currently reserved for
* pending jobs are free */
job_ptr->start_time = 0;
goto TRY_LATER;
}
if (job_ptr->start_time > (sched_start + backfill_window)) {
/* Starts too far in the future to worry about */
continue;
}
if (node_space_recs >= max_backfill_job_cnt) {
/* Already have too many jobs to deal with */
break;
}
end_reserve = job_ptr->start_time + (time_limit * 60);
if (_test_resv_overlap(node_space, avail_bitmap,
job_ptr->start_time, end_reserve)) {
/* This job overlaps with an existing reservation for
* job to be backfill scheduled, which the sched
* plugin does not know about. Try again later. */
later_start = job_ptr->start_time;
job_ptr->start_time = 0;
goto TRY_LATER;
}
/*
* Add reservation to scheduling table if appropriate
*/
if (qos_ptr && (qos_ptr->flags & QOS_FLAG_NO_RESERVE))
continue;
reject_array_job_id = 0;
reject_array_part = NULL;
if (debug_flags & DEBUG_FLAG_BACKFILL)
_dump_job_sched(job_ptr, end_reserve, avail_bitmap);
xfree(job_ptr->sched_nodes);
job_ptr->sched_nodes = bitmap2node_name(avail_bitmap);
bit_not(avail_bitmap);
_add_reservation(job_ptr->start_time, end_reserve,
avail_bitmap, node_space, &node_space_recs);
if (debug_flags & DEBUG_FLAG_BACKFILL)
_dump_node_space_table(node_space);
}
xfree(bf_part_jobs);
xfree(bf_part_ptr);
xfree(uid);
xfree(njobs);
FREE_NULL_BITMAP(avail_bitmap);
FREE_NULL_BITMAP(exc_core_bitmap);
FREE_NULL_BITMAP(resv_bitmap);
FREE_NULL_BITMAP(non_cg_bitmap);
FREE_NULL_BITMAP(previous_bitmap);
for (i=0; ; ) {
FREE_NULL_BITMAP(node_space[i].avail_bitmap);
if ((i = node_space[i].next) == 0)
break;
}
xfree(node_space);
list_destroy(job_queue);
gettimeofday(&bf_time2, NULL);
_do_diag_stats(&bf_time1, &bf_time2, yield_sleep);
if (debug_flags & DEBUG_FLAG_BACKFILL) {
END_TIMER;
info("backfill: completed testing %d jobs, %s",
job_test_count, TIME_STR);
}
return rc;
}
/* Add the given job to the "active" structures of
* the given partition and increment the run count */
static void _add_job_to_active(struct job_record *job_ptr,
struct gs_part *p_ptr)
{
job_resources_t *job_res = job_ptr->job_resrcs;
uint16_t job_gr_type;
/* add job to active_resmap */
job_gr_type = _get_part_gr_type(job_ptr->part_ptr);
if ((job_gr_type == GS_CPU2) || (job_gr_type == GS_CORE) ||
(job_gr_type == GS_SOCKET)) {
if (p_ptr->jobs_active == 0 && p_ptr->active_resmap) {
uint32_t size = bit_size(p_ptr->active_resmap);
bit_nclear(p_ptr->active_resmap, 0, size-1);
}
add_job_to_cores(job_res, &(p_ptr->active_resmap),
gs_bits_per_node);
if (job_gr_type == GS_SOCKET)
_fill_sockets(job_res->node_bitmap, p_ptr);
} else { /* GS_NODE or GS_CPU */
if (!p_ptr->active_resmap) {
if (slurmctld_conf.debug_flags & DEBUG_FLAG_GANG) {
info("gang: _add_job_to_active: job %u first",
job_ptr->job_id);
}
p_ptr->active_resmap = bit_copy(job_res->node_bitmap);
} else if (p_ptr->jobs_active == 0) {
if (slurmctld_conf.debug_flags & DEBUG_FLAG_GANG) {
info("gang: _add_job_to_active: job %u copied",
job_ptr->job_id);
}
bit_copybits(p_ptr->active_resmap,
job_res->node_bitmap);
} else {
if (slurmctld_conf.debug_flags & DEBUG_FLAG_GANG) {
info("gang: _add_job_to_active: adding job %u",
job_ptr->job_id);
}
bit_or(p_ptr->active_resmap, job_res->node_bitmap);
}
}
/* add job to the active_cpus array */
if (job_gr_type == GS_CPU) {
uint32_t i, a, sz = bit_size(p_ptr->active_resmap);
if (!p_ptr->active_cpus) {
/* create active_cpus array */
p_ptr->active_cpus = xmalloc(sz * sizeof(uint16_t));
}
if (p_ptr->jobs_active == 0) {
/* overwrite the existing values in active_cpus */
for (a = 0, i = 0; i < sz; i++) {
if (bit_test(job_res->node_bitmap, i)) {
p_ptr->active_cpus[i] =
job_res->cpus[a++];
} else {
p_ptr->active_cpus[i] = 0;
}
}
} else {
/* add job to existing jobs in the active cpus */
for (a = 0, i = 0; i < sz; i++) {
if (bit_test(job_res->node_bitmap, i)) {
uint16_t limit = _get_phys_bit_cnt(i);
p_ptr->active_cpus[i] +=
job_res->cpus[a++];
/* when adding shadows, the resources
* may get overcommitted */
if (p_ptr->active_cpus[i] > limit)
p_ptr->active_cpus[i] = limit;
}
}
}
}
p_ptr->jobs_active += 1;
}
/* cr_job_test - does most of the real work for select_p_job_test(), which
* includes contiguous selection, load-leveling and max_share logic
*
* PROCEDURE:
*
* Step 1: compare nodes in "avail" bitmap with current node state data
* to find available nodes that match the job request
*
* Step 2: check resources in "avail" bitmap with allocated resources from
* higher priority partitions (busy resources are UNavailable)
*
* Step 3: select resource usage on remaining resources in "avail" bitmap
* for this job, with the placement influenced by existing
* allocations
*/
extern int cr_job_test(struct job_record *job_ptr, bitstr_t *bitmap, int mode,
uint16_t cr_type, enum node_cr_state job_node_req,
uint32_t cr_node_cnt,
struct part_res_record *cr_part_ptr,
struct node_use_record *node_usage)
{
static int gang_mode = -1;
int error_code = SLURM_SUCCESS;
bitstr_t *orig_map, *avail_cores, *free_cores;
bitstr_t *tmpcore = NULL;
bool test_only;
uint32_t c, i, j, k, n, csize, save_mem = 0;
job_resources_t *job_res;
struct job_details *details_ptr;
struct part_res_record *p_ptr, *jp_ptr;
uint16_t *cpu_count;
if (gang_mode == -1) {
if (slurm_get_preempt_mode() & PREEMPT_MODE_GANG)
gang_mode = 1;
else
gang_mode = 0;
}
details_ptr = job_ptr->details;
free_job_resources(&job_ptr->job_resrcs);
if (mode == SELECT_MODE_TEST_ONLY)
test_only = true;
else /* SELECT_MODE_RUN_NOW || SELECT_MODE_WILL_RUN */
test_only = false;
/* check node_state and update the node bitmap as necessary */
if (!test_only) {
error_code = _verify_node_state(cr_part_ptr, job_ptr,
bitmap, cr_type, node_usage,
job_node_req);
if (error_code != SLURM_SUCCESS)
return error_code;
}
if (select_debug_flags & DEBUG_FLAG_SELECT_TYPE) {
info("select/serial: evaluating job %u on %u nodes",
job_ptr->job_id, bit_set_count(bitmap));
}
orig_map = bit_copy(bitmap);
avail_cores = _make_core_bitmap(bitmap);
/* test to make sure that this job can succeed with all avail_cores
* if 'no' then return FAIL
* if 'yes' then we will seek the optimal placement for this job
* within avail_cores
*/
free_cores = bit_copy(avail_cores);
cpu_count = _select_nodes(job_ptr, bitmap, cr_node_cnt, free_cores,
node_usage, cr_type, test_only);
if (cpu_count == NULL) {
/* job cannot fit */
FREE_NULL_BITMAP(orig_map);
FREE_NULL_BITMAP(free_cores);
FREE_NULL_BITMAP(avail_cores);
if (select_debug_flags & DEBUG_FLAG_SELECT_TYPE) {
info("select/serial: cr_job_test: test 0 fail: "
"insufficient resources");
}
return SLURM_ERROR;
} else if (test_only) {
FREE_NULL_BITMAP(orig_map);
FREE_NULL_BITMAP(free_cores);
FREE_NULL_BITMAP(avail_cores);
xfree(cpu_count);
if (select_debug_flags & DEBUG_FLAG_SELECT_TYPE)
info("select/serial: cr_job_test: test 0 pass: "******"test_only");
return SLURM_SUCCESS;
}
if (cr_type == CR_MEMORY) {
/* CR_MEMORY does not care about existing CPU allocations,
* so we can jump right to job allocation from here */
goto alloc_job;
}
xfree(cpu_count);
if (select_debug_flags & DEBUG_FLAG_SELECT_TYPE) {
info("select/serial: cr_job_test: test 0 pass - "
"job fits on given resources");
}
/* now that we know that this job can run with the given resources,
* let's factor in the existing allocations and seek the optimal set
* of resources for this job. Here is the procedure:
*
* Step 1: Seek idle CPUs across all partitions. If successful then
* place job and exit. If not successful, then continue. Two
* related items to note:
* 1. Jobs that don't share CPUs finish with step 1.
* 2. The remaining steps assume sharing or preemption.
*
* Step 2: Remove resources that are in use by higher-priority
* partitions, and test that job can still succeed. If not
* then exit.
*
* Step 3: Seek idle nodes among the partitions with the same
* priority as the job's partition. If successful then
* goto Step 6. If not then continue:
*
* Step 4: Seek placement within the job's partition. Search
* row-by-row. If no placement if found, then exit. If a row
* is found, then continue:
*
* Step 5: Place job and exit. FIXME! Here is where we need a
* placement algorithm that recognizes existing job
* boundaries and tries to "overlap jobs" as efficiently
* as possible.
*
* Step 6: Place job and exit. FIXME! here is we use a placement
* algorithm similar to Step 5 on jobs from lower-priority
* partitions.
*/
/*** Step 1 ***/
bit_copybits(bitmap, orig_map);
bit_copybits(free_cores, avail_cores);
/* remove all existing allocations from free_cores */
tmpcore = bit_copy(free_cores);
for (p_ptr = cr_part_ptr; p_ptr; p_ptr = p_ptr->next) {
if (!p_ptr->row)
continue;
for (i = 0; i < p_ptr->num_rows; i++) {
if (!p_ptr->row[i].row_bitmap)
continue;
bit_copybits(tmpcore, p_ptr->row[i].row_bitmap);
bit_not(tmpcore); /* set bits now "free" resources */
bit_and(free_cores, tmpcore);
}
}
cpu_count = _select_nodes(job_ptr, bitmap, cr_node_cnt, free_cores,
node_usage, cr_type, test_only);
if (cpu_count) {
/* job fits! We're done. */
if (select_debug_flags & DEBUG_FLAG_SELECT_TYPE) {
info("select/serial: cr_job_test: test 1 pass - "
"idle resources found");
}
goto alloc_job;
}
if ((gang_mode == 0) && (job_node_req == NODE_CR_ONE_ROW)) {
/* This job CANNOT share CPUs regardless of priority,
* so we fail here. Note that Shared=EXCLUSIVE was already
* addressed in _verify_node_state() and job preemption
* removes jobs from simulated resource allocation map
* before this point. */
if (select_debug_flags & DEBUG_FLAG_SELECT_TYPE) {
info("select/serial: cr_job_test: test 1 fail - "
"no idle resources available");
}
goto alloc_job;
}
if (select_debug_flags & DEBUG_FLAG_SELECT_TYPE) {
info("select/serial: cr_job_test: test 1 fail - "
"not enough idle resources");
}
/*** Step 2 ***/
bit_copybits(bitmap, orig_map);
bit_copybits(free_cores, avail_cores);
for (jp_ptr = cr_part_ptr; jp_ptr; jp_ptr = jp_ptr->next) {
if (jp_ptr->part_ptr == job_ptr->part_ptr)
break;
}
if (!jp_ptr) {
fatal("select/serial: could not find partition for job %u",
job_ptr->job_id);
return SLURM_ERROR; /* Fix CLANG false positive */
}
/* remove existing allocations (jobs) from higher-priority partitions
* from avail_cores */
for (p_ptr = cr_part_ptr; p_ptr; p_ptr = p_ptr->next) {
if ((p_ptr->part_ptr->priority <= jp_ptr->part_ptr->priority) &&
(p_ptr->part_ptr->preempt_mode != PREEMPT_MODE_OFF))
continue;
if (!p_ptr->row)
continue;
for (i = 0; i < p_ptr->num_rows; i++) {
if (!p_ptr->row[i].row_bitmap)
continue;
bit_copybits(tmpcore, p_ptr->row[i].row_bitmap);
bit_not(tmpcore); /* set bits now "free" resources */
bit_and(free_cores, tmpcore);
}
}
/* make these changes permanent */
bit_copybits(avail_cores, free_cores);
cpu_count = _select_nodes(job_ptr, bitmap, cr_node_cnt, free_cores,
node_usage, cr_type, test_only);
if (!cpu_count) {
/* job needs resources that are currently in use by
* higher-priority jobs, so fail for now */
if (select_debug_flags & DEBUG_FLAG_SELECT_TYPE) {
info("select/serial: cr_job_test: test 2 fail - "
"resources busy with higher priority jobs");
}
goto alloc_job;
}
xfree(cpu_count);
if (select_debug_flags & DEBUG_FLAG_SELECT_TYPE) {
info("select/serial: cr_job_test: test 2 pass - "
"available resources for this priority");
}
/*** Step 3 ***/
bit_copybits(bitmap, orig_map);
bit_copybits(free_cores, avail_cores);
/* remove existing allocations (jobs) from same-priority partitions
* from avail_cores */
for (p_ptr = cr_part_ptr; p_ptr; p_ptr = p_ptr->next) {
if (p_ptr->part_ptr->priority != jp_ptr->part_ptr->priority)
continue;
if (!p_ptr->row)
continue;
for (i = 0; i < p_ptr->num_rows; i++) {
if (!p_ptr->row[i].row_bitmap)
continue;
bit_copybits(tmpcore, p_ptr->row[i].row_bitmap);
bit_not(tmpcore); /* set bits now "free" resources */
bit_and(free_cores, tmpcore);
}
}
cpu_count = _select_nodes(job_ptr, bitmap, cr_node_cnt, free_cores,
node_usage, cr_type, test_only);
if (cpu_count) {
/* jobs from low-priority partitions are the only thing left
* in our way. for now we'll ignore them, but FIXME: we need
* a good placement algorithm here that optimizes "job overlap"
* between this job (in these idle nodes) and the low-priority
* jobs */
if (select_debug_flags & DEBUG_FLAG_SELECT_TYPE) {
info("select/serial: cr_job_test: test 3 pass - "
"found resources");
}
goto alloc_job;
}
if (select_debug_flags & DEBUG_FLAG_SELECT_TYPE) {
info("select/serial: cr_job_test: test 3 fail - "
"not enough idle resources in same priority");
}
/*** Step 4 ***/
/* try to fit the job into an existing row
*
* tmpcore = worker core_bitmap
* free_cores = core_bitmap to be built
* avail_cores = static core_bitmap of all available cores
*/
if (!jp_ptr || !jp_ptr->row) {
/* there's no existing jobs in this partition, so place
* the job in avail_cores. FIXME: still need a good
* placement algorithm here that optimizes "job overlap"
* between this job (in these idle nodes) and existing
* jobs in the other partitions with <= priority to
* this partition */
bit_copybits(bitmap, orig_map);
bit_copybits(free_cores, avail_cores);
cpu_count = _select_nodes(job_ptr, bitmap, cr_node_cnt,
free_cores, node_usage, cr_type,
test_only);
if (select_debug_flags & DEBUG_FLAG_SELECT_TYPE) {
info("select/serial: cr_job_test: test 4 pass - "
"first row found");
}
goto alloc_job;
}
cr_sort_part_rows(jp_ptr);
c = jp_ptr->num_rows;
if (job_node_req != NODE_CR_AVAILABLE)
c = 1;
for (i = 0; i < c; i++) {
if (!jp_ptr->row[i].row_bitmap)
break;
bit_copybits(bitmap, orig_map);
bit_copybits(free_cores, avail_cores);
bit_copybits(tmpcore, jp_ptr->row[i].row_bitmap);
bit_not(tmpcore);
bit_and(free_cores, tmpcore);
cpu_count = _select_nodes(job_ptr, bitmap, cr_node_cnt,
free_cores, node_usage, cr_type,
test_only);
if (cpu_count) {
if (select_debug_flags & DEBUG_FLAG_SELECT_TYPE) {
info("select/serial: cr_job_test: "
"test 4 pass - row %i", i);
}
break;
}
if (select_debug_flags & DEBUG_FLAG_SELECT_TYPE) {
info("select/serial: cr_job_test: "
"test 4 fail - row %i", i);
}
}
if ((i < c) && !jp_ptr->row[i].row_bitmap) {
/* we've found an empty row, so use it */
bit_copybits(bitmap, orig_map);
bit_copybits(free_cores, avail_cores);
if (select_debug_flags & DEBUG_FLAG_SELECT_TYPE) {
info("select/serial: cr_job_test: "
"test 4 trying empty row %i",i);
}
cpu_count = _select_nodes(job_ptr, bitmap, cr_node_cnt,
free_cores, node_usage, cr_type,
test_only);
}
if (!cpu_count) {
/* job can't fit into any row, so exit */
if (select_debug_flags & DEBUG_FLAG_SELECT_TYPE) {
info("select/serial: cr_job_test: test 4 fail - "
"busy partition");
}
goto alloc_job;
}
/*** CONSTRUCTION ZONE FOR STEPs 5 AND 6 ***
* Note that while the job may have fit into a row, it should
* still be run through a good placement algorithm here that
* optimizes "job overlap" between this job (in these idle nodes)
* and existing jobs in the other partitions with <= priority to
* this partition */
alloc_job:
/* at this point we've found a good set of
* bits to allocate to this job:
* - bitmap is the set of nodes to allocate
* - free_cores is the set of allocated cores
* - cpu_count is the number of cpus per allocated node
*
* Next steps are to cleanup the worker variables,
* create the job_resources struct,
* distribute the job on the bits, and exit
*/
FREE_NULL_BITMAP(orig_map);
FREE_NULL_BITMAP(avail_cores);
FREE_NULL_BITMAP(tmpcore);
if (!cpu_count) {
/* we were sent here to cleanup and exit */
FREE_NULL_BITMAP(free_cores);
if (select_debug_flags & DEBUG_FLAG_SELECT_TYPE) {
info("select/serial: exiting cr_job_test with no "
"allocation");
}
return SLURM_ERROR;
}
/* At this point we have:
* - a bitmap of selected nodes
* - a free_cores bitmap of usable cores on each selected node
* - a per-alloc-node cpu_count array
*/
if ((mode != SELECT_MODE_WILL_RUN) && (job_ptr->part_ptr == NULL))
error_code = EINVAL;
if ((error_code == SLURM_SUCCESS) && (mode == SELECT_MODE_WILL_RUN))
job_ptr->total_cpus = 1;
if ((error_code != SLURM_SUCCESS) || (mode != SELECT_MODE_RUN_NOW)) {
FREE_NULL_BITMAP(free_cores);
xfree(cpu_count);
return error_code;
}
n = bit_ffs(bitmap);
if (n < 0) {
FREE_NULL_BITMAP(free_cores);
xfree(cpu_count);
return error_code;
}
if (select_debug_flags & DEBUG_FLAG_SELECT_TYPE) {
info("select/serial: cr_job_test: distributing job %u",
job_ptr->job_id);
}
/** create the struct_job_res **/
job_res = create_job_resources();
job_res->node_bitmap = bit_copy(bitmap);
job_res->nodes = bitmap2node_name(bitmap);
job_res->nhosts = bit_set_count(bitmap);
job_res->ncpus = job_res->nhosts;
if (job_ptr->details->ntasks_per_node)
job_res->ncpus *= details_ptr->ntasks_per_node;
job_res->ncpus = MAX(job_res->ncpus,
details_ptr->min_cpus);
job_res->ncpus = MAX(job_res->ncpus,
details_ptr->pn_min_cpus);
job_res->node_req = job_node_req;
job_res->cpus = cpu_count;
job_res->cpus_used = xmalloc(job_res->nhosts *
sizeof(uint16_t));
job_res->memory_allocated = xmalloc(job_res->nhosts *
sizeof(uint32_t));
job_res->memory_used = xmalloc(job_res->nhosts *
sizeof(uint32_t));
/* store the hardware data for the selected nodes */
error_code = build_job_resources(job_res, node_record_table_ptr,
select_fast_schedule);
if (error_code != SLURM_SUCCESS) {
free_job_resources(&job_res);
FREE_NULL_BITMAP(free_cores);
return error_code;
}
c = 0;
csize = bit_size(job_res->core_bitmap);
j = cr_get_coremap_offset(n);
k = cr_get_coremap_offset(n + 1);
for (; j < k; j++, c++) {
if (!bit_test(free_cores, j))
continue;
if (c >= csize) {
error("select/serial: cr_job_test "
"core_bitmap index error on node %s",
select_node_record[n].node_ptr->name);
drain_nodes(select_node_record[n].node_ptr->name,
"Bad core count", getuid());
free_job_resources(&job_res);
FREE_NULL_BITMAP(free_cores);
return SLURM_ERROR;
}
bit_set(job_res->core_bitmap, c);
break;
}
if (select_debug_flags & DEBUG_FLAG_SELECT_TYPE) {
info("select/serial: cr_job_test: job %u ncpus %u cbits %u/%d "
"nbits %u", job_ptr->job_id,
job_res->ncpus, bit_set_count(free_cores), 1,
job_res->nhosts);
}
FREE_NULL_BITMAP(free_cores);
/* distribute the tasks and clear any unused cores */
job_ptr->job_resrcs = job_res;
error_code = cr_dist(job_ptr, cr_type);
if (error_code != SLURM_SUCCESS) {
free_job_resources(&job_ptr->job_resrcs);
return error_code;
}
/* translate job_res->cpus array into format with rep count */
job_ptr->total_cpus = build_job_resources_cpu_array(job_res);
if (!(cr_type & CR_MEMORY))
return error_code;
/* load memory allocated array */
save_mem = details_ptr->pn_min_memory;
if (save_mem & MEM_PER_CPU) {
/* memory is per-cpu */
save_mem &= (~MEM_PER_CPU);
job_res->memory_allocated[0] = job_res->cpus[0] * save_mem;
} else {
/* memory is per-node */
job_res->memory_allocated[0] = save_mem;
}
return error_code;
}
