/* Reset the node_bitmap in a job_resources data structure
* This is needed after a restart/reconfiguration since nodes can
* be added or removed from the system resulting in changing in
* the bitmap size or bit positions */
extern int reset_node_bitmap(job_resources_t *job_resrcs_ptr, uint32_t job_id)
{
int i;
if (!job_resrcs_ptr)
return SLURM_SUCCESS;
if (job_resrcs_ptr->node_bitmap)
FREE_NULL_BITMAP(job_resrcs_ptr->node_bitmap);
if (job_resrcs_ptr->nodes &&
(node_name2bitmap(job_resrcs_ptr->nodes, false,
&job_resrcs_ptr->node_bitmap))) {
error("Invalid nodes (%s) for job_id %u",
job_resrcs_ptr->nodes, job_id);
return SLURM_ERROR;
} else if (job_resrcs_ptr->nodes == NULL) {
job_resrcs_ptr->node_bitmap = bit_alloc(node_record_count);
}
i = bit_set_count(job_resrcs_ptr->node_bitmap);
if (job_resrcs_ptr->nhosts != i) {
error("Invalid change in resource allocation node count for "
"job %u, %u to %d", job_id, job_resrcs_ptr->nhosts, i);
return SLURM_ERROR;
}
return SLURM_SUCCESS;
}
/*
* build_all_nodeline_info - get a array of slurm_conf_node_t structures
* from the slurm.conf reader, build table, and set values
* IN set_bitmap - if true, set node_bitmap in config record (used by slurmd)
* IN tres_cnt - number of TRES configured on system (used on controller side)
* RET 0 if no error, error code otherwise
*/
extern int build_all_nodeline_info (bool set_bitmap, int tres_cnt)
{
slurm_conf_node_t *node, **ptr_array;
struct config_record *config_ptr = NULL;
int count;
int i, rc, max_rc = SLURM_SUCCESS;
count = slurm_conf_nodename_array(&ptr_array);
if (count == 0)
fatal("No NodeName information available!");
for (i = 0; i < count; i++) {
node = ptr_array[i];
config_ptr = create_config_record();
config_ptr->nodes = xstrdup(node->nodenames);
config_ptr->cpus = node->cpus;
config_ptr->boards = node->boards;
config_ptr->sockets = node->sockets;
config_ptr->cores = node->cores;
config_ptr->core_spec_cnt = node->core_spec_cnt;
config_ptr->cpu_spec_list = xstrdup(node->cpu_spec_list);
config_ptr->threads = node->threads;
config_ptr->real_memory = node->real_memory;
config_ptr->mem_spec_limit = node->mem_spec_limit;
config_ptr->tmp_disk = node->tmp_disk;
if (tres_cnt) {
config_ptr->tres_weights_str =
xstrdup(node->tres_weights_str);
config_ptr->tres_weights =
slurm_get_tres_weight_array(
node->tres_weights_str,
tres_cnt);
}
config_ptr->weight = node->weight;
if (node->feature && node->feature[0])
config_ptr->feature = xstrdup(node->feature);
if (node->gres && node->gres[0])
config_ptr->gres = xstrdup(node->gres);
rc = _build_single_nodeline_info(node, config_ptr);
max_rc = MAX(max_rc, rc);
}
if (set_bitmap) {
ListIterator config_iterator;
config_iterator = list_iterator_create(config_list);
while ((config_ptr = (struct config_record *)
list_next(config_iterator))) {
node_name2bitmap(config_ptr->nodes, true,
&config_ptr->node_bitmap);
}
list_iterator_destroy(config_iterator);
}
return max_rc;
}
/*
* Attempt to start a job
* jobid (IN) - job id
* task_cnt (IN) - total count of tasks to start
* hostlist (IN) - SLURM hostlist expression with no repeated hostnames
* tasklist (IN/OUT) - comma separated list of hosts with tasks to be started,
* list hostname once per task to start
* comment_ptr (IN) - new comment field for the job or NULL for no change
* err_code (OUT) - Moab error code
* err_msg (OUT) - Moab error message
*/
static int _start_job(uint32_t jobid, int task_cnt, char *hostlist,
char *tasklist, char *comment_ptr,
int *err_code, char **err_msg)
{
int rc = 0, old_task_cnt = 1;
struct job_record *job_ptr;
/* Write lock on job info, read lock on node info */
slurmctld_lock_t job_write_lock = {
NO_LOCK, WRITE_LOCK, READ_LOCK, NO_LOCK };
char *new_node_list = NULL;
static char tmp_msg[128];
bitstr_t *new_bitmap = (bitstr_t *) NULL;
bitstr_t *save_req_bitmap = (bitstr_t *) NULL;
bitoff_t i, bsize;
int ll; /* layout info index */
char *node_name, *node_idx, *node_cur, *save_req_nodes = NULL;
size_t node_name_len;
static uint32_t cr_test = 0, cr_enabled = 0;
if (cr_test == 0) {
select_g_get_info_from_plugin(SELECT_CR_PLUGIN, NULL,
&cr_enabled);
cr_test = 1;
}
lock_slurmctld(job_write_lock);
job_ptr = find_job_record(jobid);
if (job_ptr == NULL) {
*err_code = -700;
*err_msg = "No such job";
error("wiki: Failed to find job %u", jobid);
rc = -1;
goto fini;
}
if ((job_ptr->details == NULL) || (!IS_JOB_PENDING(job_ptr))) {
*err_code = -700;
*err_msg = "Job not pending, can't start";
error("wiki: Attempt to start job %u in state %s",
jobid, job_state_string(job_ptr->job_state));
rc = -1;
goto fini;
}
if (comment_ptr) {
char *reserved = strstr(comment_ptr, "RESERVED:");
if (reserved) {
reserved += 9;
job_ptr->details->reserved_resources =
strtol(reserved, NULL, 10);
}
xfree(job_ptr->comment);
job_ptr->comment = xstrdup(comment_ptr);
}
if (task_cnt) {
new_node_list = xstrdup(hostlist);
if (node_name2bitmap(new_node_list, false, &new_bitmap) != 0) {
*err_code = -700;
*err_msg = "Invalid TASKLIST";
error("wiki: Attempt to set invalid node list for "
"job %u, %s",
jobid, hostlist);
xfree(new_node_list);
rc = -1;
goto fini;
}
if (!bit_super_set(new_bitmap, avail_node_bitmap)) {
/* Selected node is UP and not responding
* or it just went DOWN */
*err_code = -700;
*err_msg = "TASKLIST includes non-responsive node";
error("wiki: Attempt to use non-responsive nodes for "
"job %u, %s",
jobid, hostlist);
xfree(new_node_list);
FREE_NULL_BITMAP(new_bitmap);
rc = -1;
goto fini;
}
/* User excluded node list incompatible with Wiki
* Exclude all nodes not explicitly requested */
FREE_NULL_BITMAP(job_ptr->details->exc_node_bitmap);
job_ptr->details->exc_node_bitmap = bit_copy(new_bitmap);
bit_not(job_ptr->details->exc_node_bitmap);
}
/* Build layout information from tasklist (assuming that Moab
* sends a non-bracketed list of nodes, repeated as many times
* as cpus should be used per node); at this point, node names
* are comma-separated. This is _not_ a fast algorithm as it
* performs many string compares. */
xfree(job_ptr->details->req_node_layout);
if (task_cnt && cr_enabled) {
uint16_t cpus_per_task = MAX(1, job_ptr->details->cpus_per_task);
job_ptr->details->req_node_layout = (uint16_t *)
xmalloc(bit_set_count(new_bitmap) * sizeof(uint16_t));
bsize = bit_size(new_bitmap);
for (i = 0, ll = -1; i < bsize; i++) {
if (!bit_test(new_bitmap, i))
continue;
ll++;
node_name = node_record_table_ptr[i].name;
node_name_len = strlen(node_name);
if (node_name_len == 0)
continue;
node_cur = tasklist;
while (*node_cur) {
if ((node_idx = strstr(node_cur, node_name))) {
if ((node_idx[node_name_len] == ',') ||
(node_idx[node_name_len] == '\0')) {
job_ptr->details->
req_node_layout[ll] +=
cpus_per_task;
}
node_cur = strchr(node_idx, ',');
if (node_cur)
continue;
}
break;
}
}
}
/* save and update job state to start now */
save_req_nodes = job_ptr->details->req_nodes;
job_ptr->details->req_nodes = new_node_list;
save_req_bitmap = job_ptr->details->req_node_bitmap;
job_ptr->details->req_node_bitmap = new_bitmap;
old_task_cnt = job_ptr->details->min_cpus;
job_ptr->details->min_cpus = MAX(task_cnt, old_task_cnt);
job_ptr->priority = 100000000;
fini: unlock_slurmctld(job_write_lock);
if (rc)
return rc;
/* No errors so far */
(void) schedule(INFINITE); /* provides own locking */
/* Check to insure the job was actually started */
lock_slurmctld(job_write_lock);
if (job_ptr->job_id != jobid)
job_ptr = find_job_record(jobid);
if (job_ptr && (job_ptr->job_id == jobid) &&
(!IS_JOB_RUNNING(job_ptr))) {
uint16_t wait_reason = 0;
char *wait_string;
if (IS_JOB_FAILED(job_ptr))
wait_string = "Invalid request, job aborted";
else {
wait_reason = job_ptr->state_reason;
if (wait_reason == WAIT_HELD) {
/* some job is completing, slurmctld did
* not even try to schedule this job */
wait_reason = WAIT_RESOURCES;
}
wait_string = job_reason_string(wait_reason);
job_ptr->state_reason = WAIT_HELD;
xfree(job_ptr->state_desc);
}
*err_code = -910 - wait_reason;
snprintf(tmp_msg, sizeof(tmp_msg),
"Could not start job %u(%s): %s",
jobid, new_node_list, wait_string);
*err_msg = tmp_msg;
error("wiki: %s", tmp_msg);
/* restore some of job state */
job_ptr->priority = 0;
job_ptr->details->min_cpus = old_task_cnt;
rc = -1;
}
if (job_ptr && (job_ptr->job_id == jobid) && job_ptr->details) {
/* Restore required node list in case job requeued */
xfree(job_ptr->details->req_nodes);
job_ptr->details->req_nodes = save_req_nodes;
FREE_NULL_BITMAP(job_ptr->details->req_node_bitmap);
job_ptr->details->req_node_bitmap = save_req_bitmap;
FREE_NULL_BITMAP(job_ptr->details->exc_node_bitmap);
xfree(job_ptr->details->req_node_layout);
} else {
error("wiki: start_job(%u) job missing", jobid);
xfree(save_req_nodes);
FREE_NULL_BITMAP(save_req_bitmap);
}
unlock_slurmctld(job_write_lock);
schedule_node_save(); /* provides own locking */
schedule_job_save(); /* provides own locking */
return rc;
}
static int _job_modify(uint32_t jobid, char *bank_ptr,
char *depend_ptr, char *new_hostlist,
uint32_t new_node_cnt, char *part_name_ptr,
uint32_t new_time_limit, char *name_ptr,
char *start_ptr, char *feature_ptr, char *env_ptr,
char *comment_ptr, char *gres_ptr, char *wckey_ptr)
{
struct job_record *job_ptr;
time_t now = time(NULL);
bool update_accounting = false;
job_ptr = find_job_record(jobid);
if (job_ptr == NULL) {
error("wiki: MODIFYJOB has invalid jobid %u", jobid);
return ESLURM_INVALID_JOB_ID;
}
if (IS_JOB_FINISHED(job_ptr) || (job_ptr->details == NULL)) {
info("wiki: MODIFYJOB jobid %u is finished", jobid);
return ESLURM_DISABLED;
}
if (comment_ptr) {
info("wiki: change job %u comment %s", jobid, comment_ptr);
xfree(job_ptr->comment);
job_ptr->comment = xstrdup(comment_ptr);
last_job_update = now;
}
if (depend_ptr) {
int rc = update_job_dependency(job_ptr, depend_ptr);
if (rc == SLURM_SUCCESS) {
info("wiki: changed job %u dependency to %s",
jobid, depend_ptr);
} else {
error("wiki: changing job %u dependency to %s",
jobid, depend_ptr);
return EINVAL;
}
}
if (env_ptr) {
bool have_equal = false;
char old_sep[1];
int begin = 0, i;
if (job_ptr->batch_flag == 0) {
error("wiki: attempt to set environment variables "
"for non-batch job %u", jobid);
return ESLURM_DISABLED;
}
for (i=0; ; i++) {
if (env_ptr[i] == '=') {
if (have_equal) {
error("wiki: setting job %u invalid "
"environment variables: %s",
jobid, env_ptr);
return EINVAL;
}
have_equal = true;
if (env_ptr[i+1] == '\"') {
for (i+=2; ; i++) {
if (env_ptr[i] == '\0') {
error("wiki: setting job %u "
"invalid environment "
"variables: %s",
jobid, env_ptr);
return EINVAL;
}
if (env_ptr[i] == '\"') {
i++;
break;
}
if (env_ptr[i] == '\\') {
i++;
}
}
} else if (env_ptr[i+1] == '\'') {
for (i+=2; ; i++) {
if (env_ptr[i] == '\0') {
error("wiki: setting job %u "
"invalid environment "
"variables: %s",
jobid, env_ptr);
return EINVAL;
}
if (env_ptr[i] == '\'') {
i++;
break;
}
if (env_ptr[i] == '\\') {
i++;
}
}
}
}
if (isspace(env_ptr[i]) || (env_ptr[i] == ',')) {
if (!have_equal) {
error("wiki: setting job %u invalid "
"environment variables: %s",
jobid, env_ptr);
return EINVAL;
}
old_sep[0] = env_ptr[i];
env_ptr[i] = '\0';
xrealloc(job_ptr->details->env_sup,
sizeof(char *) *
(job_ptr->details->env_cnt+1));
job_ptr->details->env_sup
[job_ptr->details->env_cnt++] =
xstrdup(&env_ptr[begin]);
info("wiki: for job %u add env: %s",
jobid, &env_ptr[begin]);
env_ptr[i] = old_sep[0];
if (isspace(old_sep[0]))
break;
begin = i + 1;
have_equal = false;
}
}
}
if (new_time_limit) {
time_t old_time = job_ptr->time_limit;
job_ptr->time_limit = new_time_limit;
info("wiki: change job %u time_limit to %u",
jobid, new_time_limit);
/* Update end_time based upon change
* to preserve suspend time info */
job_ptr->end_time = job_ptr->end_time +
((job_ptr->time_limit -
old_time) * 60);
last_job_update = now;
}
if (bank_ptr &&
(update_job_account("wiki", job_ptr, bank_ptr) != SLURM_SUCCESS)) {
return EINVAL;
}
if (feature_ptr) {
if (IS_JOB_PENDING(job_ptr) && (job_ptr->details)) {
info("wiki: change job %u features to %s",
jobid, feature_ptr);
job_ptr->details->features = xstrdup(feature_ptr);
last_job_update = now;
} else {
error("wiki: MODIFYJOB features of non-pending "
"job %u", jobid);
return ESLURM_DISABLED;
}
}
if (start_ptr) {
char *end_ptr;
uint32_t begin_time = strtol(start_ptr, &end_ptr, 10);
if (IS_JOB_PENDING(job_ptr) && (job_ptr->details)) {
info("wiki: change job %u begin time to %u",
jobid, begin_time);
job_ptr->details->begin_time = begin_time;
last_job_update = now;
update_accounting = true;
} else {
error("wiki: MODIFYJOB begin_time of non-pending "
"job %u", jobid);
return ESLURM_DISABLED;
}
}
if (name_ptr) {
if (IS_JOB_PENDING(job_ptr)) {
info("wiki: change job %u name %s", jobid, name_ptr);
xfree(job_ptr->name);
job_ptr->name = xstrdup(name_ptr);
last_job_update = now;
update_accounting = true;
} else {
error("wiki: MODIFYJOB name of non-pending job %u",
jobid);
return ESLURM_DISABLED;
}
}
if (new_hostlist) {
int rc = 0, task_cnt;
hostlist_t hl;
char *tasklist;
if (!IS_JOB_PENDING(job_ptr) || !job_ptr->details) {
/* Job is done, nothing to reset */
if (new_hostlist == '\0')
goto host_fini;
error("wiki: MODIFYJOB hostlist of non-pending "
"job %u", jobid);
return ESLURM_DISABLED;
}
xfree(job_ptr->details->req_nodes);
FREE_NULL_BITMAP(job_ptr->details->req_node_bitmap);
if (new_hostlist == '\0')
goto host_fini;
tasklist = moab2slurm_task_list(new_hostlist, &task_cnt);
if (tasklist == NULL) {
rc = 1;
goto host_fini;
}
hl = hostlist_create(tasklist);
if (hl == 0) {
rc = 1;
goto host_fini;
}
hostlist_uniq(hl);
hostlist_sort(hl);
job_ptr->details->req_nodes =
hostlist_ranged_string_xmalloc(hl);
hostlist_destroy(hl);
if (job_ptr->details->req_nodes == NULL) {
rc = 1;
goto host_fini;
}
if (node_name2bitmap(job_ptr->details->req_nodes, false,
&job_ptr->details->req_node_bitmap)) {
rc = 1;
goto host_fini;
}
host_fini: if (rc) {
info("wiki: change job %u invalid hostlist %s",
jobid, new_hostlist);
xfree(job_ptr->details->req_nodes);
return EINVAL;
} else {
info("wiki: change job %u hostlist %s",
jobid, new_hostlist);
update_accounting = true;
}
}
if (part_name_ptr) {
struct part_record *part_ptr;
if (!IS_JOB_PENDING(job_ptr)) {
error("wiki: MODIFYJOB partition of non-pending "
"job %u", jobid);
return ESLURM_DISABLED;
}
part_ptr = find_part_record(part_name_ptr);
if (part_ptr == NULL) {
error("wiki: MODIFYJOB has invalid partition %s",
part_name_ptr);
return ESLURM_INVALID_PARTITION_NAME;
}
info("wiki: change job %u partition %s",
jobid, part_name_ptr);
xfree(job_ptr->partition);
job_ptr->partition = xstrdup(part_name_ptr);
job_ptr->part_ptr = part_ptr;
last_job_update = now;
update_accounting = true;
}
if (new_node_cnt) {
job_desc_msg_t job_desc;
#ifdef HAVE_BG
uint16_t geometry[SYSTEM_DIMENSIONS] = {(uint16_t) NO_VAL};
static uint16_t cpus_per_node = 0;
if (!cpus_per_node) {
select_g_alter_node_cnt(SELECT_GET_NODE_CPU_CNT,
&cpus_per_node);
}
#endif
if(!IS_JOB_PENDING(job_ptr) || !job_ptr->details) {
error("wiki: MODIFYJOB node count of non-pending "
"job %u", jobid);
return ESLURM_DISABLED;
}
memset(&job_desc, 0, sizeof(job_desc_msg_t));
job_desc.min_nodes = new_node_cnt;
job_desc.max_nodes = NO_VAL;
job_desc.select_jobinfo = select_g_select_jobinfo_alloc();
select_g_alter_node_cnt(SELECT_SET_NODE_CNT, &job_desc);
select_g_select_jobinfo_free(job_desc.select_jobinfo);
job_ptr->details->min_nodes = job_desc.min_nodes;
if (job_ptr->details->max_nodes &&
(job_ptr->details->max_nodes < job_desc.min_nodes))
job_ptr->details->max_nodes = job_desc.min_nodes;
info("wiki: change job %u min_nodes to %u",
jobid, new_node_cnt);
#ifdef HAVE_BG
job_ptr->details->min_cpus = job_desc.min_cpus;
job_ptr->details->max_cpus = job_desc.max_cpus;
job_ptr->details->pn_min_cpus = job_desc.pn_min_cpus;
new_node_cnt = job_ptr->details->min_cpus;
if (cpus_per_node)
new_node_cnt /= cpus_per_node;
/* This is only set up so accounting is set up correctly */
select_g_select_jobinfo_set(job_ptr->select_jobinfo,
SELECT_JOBDATA_NODE_CNT,
&new_node_cnt);
/* reset geo since changing this makes any geo
potentially invalid */
select_g_select_jobinfo_set(job_ptr->select_jobinfo,
SELECT_JOBDATA_GEOMETRY,
geometry);
#endif
last_job_update = now;
update_accounting = true;
}
if (gres_ptr) {
char *orig_gres;
if (!IS_JOB_PENDING(job_ptr)) {
error("wiki: MODIFYJOB GRES of non-pending job %u",
jobid);
return ESLURM_DISABLED;
}
orig_gres = job_ptr->gres;
job_ptr->gres = NULL;
if (gres_ptr[0])
job_ptr->gres = xstrdup(gres_ptr);
if (gres_plugin_job_state_validate(job_ptr->gres,
&job_ptr->gres_list)) {
error("wiki: MODIFYJOB Invalid GRES=%s", gres_ptr);
xfree(job_ptr->gres);
job_ptr->gres = orig_gres;
return ESLURM_INVALID_GRES;
}
xfree(orig_gres);
}
if (wckey_ptr) {
int rc = update_job_wckey("update_job", job_ptr, wckey_ptr);
if (rc != SLURM_SUCCESS) {
error("wiki: MODIFYJOB Invalid WCKEY=%s", wckey_ptr);
return rc;
}
}
if (update_accounting) {
if (job_ptr->details && job_ptr->details->begin_time) {
/* Update job record in accounting to reflect
* the changes */
jobacct_storage_g_job_start(acct_db_conn, job_ptr);
}
}
return SLURM_SUCCESS;
}
/* Initialize power_save module parameters.
* Return 0 on valid configuration to run power saving,
* otherwise log the problem and return -1 */
static int _init_power_config(void)
{
slurm_ctl_conf_t *conf = slurm_conf_lock();
last_config = slurmctld_conf.last_update;
idle_time = conf->suspend_time - 1;
suspend_rate = conf->suspend_rate;
resume_timeout = conf->resume_timeout;
resume_rate = conf->resume_rate;
slurmd_timeout = conf->slurmd_timeout;
suspend_timeout = conf->suspend_timeout;
_clear_power_config();
if (conf->suspend_program)
suspend_prog = xstrdup(conf->suspend_program);
if (conf->resume_program)
resume_prog = xstrdup(conf->resume_program);
if (conf->suspend_exc_nodes)
exc_nodes = xstrdup(conf->suspend_exc_nodes);
if (conf->suspend_exc_parts)
exc_parts = xstrdup(conf->suspend_exc_parts);
slurm_conf_unlock();
if (idle_time < 0) { /* not an error */
debug("power_save module disabled, SuspendTime < 0");
return -1;
}
if (suspend_rate < 0) {
error("power_save module disabled, SuspendRate < 0");
return -1;
}
if (resume_rate < 0) {
error("power_save module disabled, ResumeRate < 0");
return -1;
}
if (suspend_prog == NULL) {
error("power_save module disabled, NULL SuspendProgram");
return -1;
} else if (!_valid_prog(suspend_prog)) {
error("power_save module disabled, invalid SuspendProgram %s",
suspend_prog);
return -1;
}
if (resume_prog == NULL) {
error("power_save module disabled, NULL ResumeProgram");
return -1;
} else if (!_valid_prog(resume_prog)) {
error("power_save module disabled, invalid ResumeProgram %s",
resume_prog);
return -1;
}
if (exc_nodes &&
(node_name2bitmap(exc_nodes, false, &exc_node_bitmap))) {
error("power_save module disabled, "
"invalid SuspendExcNodes %s", exc_nodes);
return -1;
}
if (exc_parts) {
char *tmp = NULL, *one_part = NULL, *part_list = NULL;
struct part_record *part_ptr = NULL;
int rc = 0;
part_list = xstrdup(exc_parts);
one_part = strtok_r(part_list, ",", &tmp);
while (one_part != NULL) {
part_ptr = find_part_record(one_part);
if (!part_ptr) {
error("power_save module disabled, "
"invalid SuspendExcPart %s",
one_part);
rc = -1;
break;
}
if (exc_node_bitmap)
bit_or(exc_node_bitmap, part_ptr->node_bitmap);
else
exc_node_bitmap = bit_copy(part_ptr->
node_bitmap);
one_part = strtok_r(NULL, ",", &tmp);
}
xfree(part_list);
if (rc)
return rc;
}
if (exc_node_bitmap) {
char *tmp = bitmap2node_name(exc_node_bitmap);
debug("power_save module, excluded nodes %s", tmp);
xfree(tmp);
}
return 0;
}
static int _job_modify(uint32_t jobid, char *bank_ptr,
char *depend_ptr, char *new_hostlist,
uint32_t new_node_cnt, char *part_name_ptr,
uint32_t new_time_limit)
{
struct job_record *job_ptr;
bool update_accounting = false;
job_ptr = find_job_record(jobid);
if (job_ptr == NULL) {
error("wiki: MODIFYJOB has invalid jobid %u", jobid);
return ESLURM_INVALID_JOB_ID;
}
if (IS_JOB_FINISHED(job_ptr)) {
error("wiki: MODIFYJOB jobid %u is finished", jobid);
return ESLURM_DISABLED;
}
if (depend_ptr) {
int rc = update_job_dependency(job_ptr, depend_ptr);
if (rc == SLURM_SUCCESS) {
info("wiki: changed job %u dependency to %s",
jobid, depend_ptr);
} else {
error("wiki: changing job %u dependency to %s",
jobid, depend_ptr);
return EINVAL;
}
}
if (new_time_limit) {
time_t old_time = job_ptr->time_limit;
job_ptr->time_limit = new_time_limit;
info("wiki: change job %u time_limit to %u",
jobid, new_time_limit);
/* Update end_time based upon change
* to preserve suspend time info */
job_ptr->end_time = job_ptr->end_time +
((job_ptr->time_limit -
old_time) * 60);
last_job_update = time(NULL);
}
if (bank_ptr) {
if (update_job_account("wiki", job_ptr, bank_ptr)
!= SLURM_SUCCESS)
return EINVAL;
else
update_accounting = true;
}
if (new_hostlist) {
int rc = 0, task_cnt;
hostlist_t hl;
char *tasklist;
if (!IS_JOB_PENDING(job_ptr) || !job_ptr->details) {
/* Job is done, nothing to reset */
if (new_hostlist == '\0')
goto host_fini;
error("wiki: MODIFYJOB tasklist of non-pending "
"job %u", jobid);
return ESLURM_DISABLED;
}
xfree(job_ptr->details->req_nodes);
FREE_NULL_BITMAP(job_ptr->details->req_node_bitmap);
if (new_hostlist == '\0')
goto host_fini;
tasklist = moab2slurm_task_list(new_hostlist, &task_cnt);
if (tasklist == NULL) {
rc = 1;
goto host_fini;
}
hl = hostlist_create(tasklist);
if (hl == 0) {
rc = 1;
goto host_fini;
}
hostlist_uniq(hl);
hostlist_sort(hl);
job_ptr->details->req_nodes =
hostlist_ranged_string_xmalloc(hl);
hostlist_destroy(hl);
if (job_ptr->details->req_nodes == NULL) {
rc = 1;
goto host_fini;
}
if (node_name2bitmap(job_ptr->details->req_nodes, false,
&job_ptr->details->req_node_bitmap)) {
rc = 1;
goto host_fini;
}
host_fini: if (rc) {
info("wiki: change job %u invalid hostlist %s",
jobid, new_hostlist);
xfree(job_ptr->details->req_nodes);
return EINVAL;
} else {
info("wiki: change job %u hostlist %s",
jobid, new_hostlist);
update_accounting = true;
}
}
if (part_name_ptr) {
struct part_record *part_ptr;
if (!IS_JOB_PENDING(job_ptr)) {
error("wiki: MODIFYJOB partition of non-pending "
"job %u", jobid);
return ESLURM_DISABLED;
}
part_ptr = find_part_record(part_name_ptr);
if (part_ptr == NULL) {
error("wiki: MODIFYJOB has invalid partition %s",
part_name_ptr);
return ESLURM_INVALID_PARTITION_NAME;
}
info("wiki: change job %u partition %s",
jobid, part_name_ptr);
xfree(job_ptr->partition);
job_ptr->partition = xstrdup(part_name_ptr);
job_ptr->part_ptr = part_ptr;
last_job_update = time(NULL);
update_accounting = true;
}
if (new_node_cnt) {
if (IS_JOB_PENDING(job_ptr) && job_ptr->details) {
job_ptr->details->min_nodes = new_node_cnt;
if (job_ptr->details->max_nodes
&& (job_ptr->details->max_nodes < new_node_cnt))
job_ptr->details->max_nodes = new_node_cnt;
info("wiki: change job %u min_nodes to %u",
jobid, new_node_cnt);
last_job_update = time(NULL);
update_accounting = true;
} else {
error("wiki: MODIFYJOB node count of non-pending "
"job %u", jobid);
return ESLURM_DISABLED;
}
}
if (update_accounting) {
/* Update job record in accounting to reflect changes */
jobacct_storage_job_start_direct(acct_db_conn, job_ptr);
}
return SLURM_SUCCESS;
}
static char * _will_run_test2(uint32_t jobid, time_t start_time,
char *node_list,
uint32_t *preemptee, int preemptee_cnt,
int *err_code, char **err_msg)
{
struct job_record *job_ptr = NULL, *pre_ptr;
struct part_record *part_ptr;
bitstr_t *avail_bitmap = NULL, *resv_bitmap = NULL;
bitstr_t *exc_core_bitmap = NULL;
time_t start_res;
uint32_t min_nodes, max_nodes, req_nodes;
List preemptee_candidates = NULL, preempted_jobs = NULL;
time_t orig_start_time;
char *reply_msg = NULL;
int i, rc;
bool resv_overlap = false;
xassert(node_list);
debug2("wiki2: will_run2 job_id=%u start_time=%u node_list=%s",
jobid, (uint32_t)start_time, node_list);
job_ptr = find_job_record(jobid);
if (job_ptr == NULL) {
*err_code = -700;
*err_msg = "No such job";
error("wiki: Failed to find job %u", jobid);
return NULL;
}
if ((job_ptr->details == NULL) || (!IS_JOB_PENDING(job_ptr))) {
*err_code = -700;
*err_msg = "WillRun not applicable to non-pending job";
error("wiki: WillRun on non-pending job %u", jobid);
return NULL;
}
part_ptr = job_ptr->part_ptr;
if (part_ptr == NULL) {
*err_code = -700;
*err_msg = "Job lacks a partition";
error("wiki: Job %u lacks a partition", jobid);
return NULL;
}
if (node_name2bitmap(node_list, false, &avail_bitmap) != 0) {
*err_code = -700;
*err_msg = "Invalid available nodes value";
error("wiki: Attempt to set invalid available node "
"list for job %u, %s", jobid, node_list);
return NULL;
}
/* Enforce reservation: access control, time and nodes */
start_res = start_time;
rc = job_test_resv(job_ptr, &start_res, true, &resv_bitmap,
&exc_core_bitmap, &resv_overlap);
if (rc != SLURM_SUCCESS) {
*err_code = -730;
*err_msg = "Job denied access to reservation";
error("wiki: reservation access denied for job %u", jobid);
FREE_NULL_BITMAP(avail_bitmap);
FREE_NULL_BITMAP(exc_core_bitmap);
return NULL;
}
bit_and(avail_bitmap, resv_bitmap);
FREE_NULL_BITMAP(resv_bitmap);
/* Only consider nodes that are not DOWN or DRAINED */
bit_and(avail_bitmap, avail_node_bitmap);
/* Consider only nodes in this job's partition */
if (part_ptr->node_bitmap)
bit_and(avail_bitmap, part_ptr->node_bitmap);
else {
*err_code = -730;
*err_msg = "Job's partition has no nodes";
error("wiki: no nodes in partition %s for job %u",
part_ptr->name, jobid);
FREE_NULL_BITMAP(avail_bitmap);
FREE_NULL_BITMAP(exc_core_bitmap);
return NULL;
}
if (job_req_node_filter(job_ptr, avail_bitmap) != SLURM_SUCCESS) {
/* Job probably has invalid feature list */
*err_code = -730;
*err_msg = "Job's required features not available "
"on selected nodes";
error("wiki: job %u not runnable on hosts=%s",
jobid, node_list);
FREE_NULL_BITMAP(avail_bitmap);
FREE_NULL_BITMAP(exc_core_bitmap);
return NULL;
}
if (job_ptr->details->exc_node_bitmap) {
bit_not(job_ptr->details->exc_node_bitmap);
bit_and(avail_bitmap, job_ptr->details->exc_node_bitmap);
bit_not(job_ptr->details->exc_node_bitmap);
}
if ((job_ptr->details->req_node_bitmap) &&
(!bit_super_set(job_ptr->details->req_node_bitmap,
avail_bitmap))) {
*err_code = -730;
*err_msg = "Job's required nodes not available";
error("wiki: job %u not runnable on hosts=%s",
jobid, node_list);
FREE_NULL_BITMAP(avail_bitmap);
FREE_NULL_BITMAP(exc_core_bitmap);
return NULL;
}
min_nodes = MAX(job_ptr->details->min_nodes, part_ptr->min_nodes);
if (job_ptr->details->max_nodes == 0)
max_nodes = part_ptr->max_nodes;
else
max_nodes = MIN(job_ptr->details->max_nodes,
part_ptr->max_nodes);
max_nodes = MIN(max_nodes, 500000); /* prevent overflows */
if (job_ptr->details->max_nodes)
req_nodes = max_nodes;
else
req_nodes = min_nodes;
if (min_nodes > max_nodes) {
/* job's min_nodes exceeds partitions max_nodes */
*err_code = -730;
*err_msg = "Job's min_nodes > max_nodes";
error("wiki: job %u not runnable on hosts=%s",
jobid, node_list);
FREE_NULL_BITMAP(avail_bitmap);
FREE_NULL_BITMAP(exc_core_bitmap);
return NULL;
}
if (preemptee_cnt) {
preemptee_candidates = list_create(NULL);
for (i=0; i<preemptee_cnt; i++) {
if ((pre_ptr = find_job_record(preemptee[i])))
list_append(preemptee_candidates, pre_ptr);
}
}
orig_start_time = job_ptr->start_time;
rc = select_g_job_test(job_ptr, avail_bitmap, min_nodes, max_nodes,
req_nodes, SELECT_MODE_WILL_RUN,
preemptee_candidates, &preempted_jobs,
exc_core_bitmap);
FREE_NULL_LIST(preemptee_candidates);
if (rc == SLURM_SUCCESS) {
char *hostlist, *sep, tmp_str[128];
uint32_t pre_cnt = 0, proc_cnt = 0;
#ifdef HAVE_BG
select_g_select_jobinfo_get(job_ptr->select_jobinfo,
SELECT_JOBDATA_NODE_CNT, &proc_cnt);
#else
proc_cnt = job_ptr->total_cpus;
#endif
snprintf(tmp_str, sizeof(tmp_str),
"STARTINFO=%u TASKS=%u STARTTIME=%u NODES=",
job_ptr->job_id, proc_cnt,
(uint32_t) job_ptr->start_time);
xstrcat(reply_msg, tmp_str);
hostlist = bitmap2node_name(avail_bitmap);
xstrcat(reply_msg, hostlist);
xfree(hostlist);
if (preempted_jobs) {
while ((pre_ptr = list_pop(preempted_jobs))) {
if (pre_cnt++)
sep = ",";
else
sep = " PREEMPT=";
snprintf(tmp_str, sizeof(tmp_str), "%s%u",
sep, pre_ptr->job_id);
xstrcat(reply_msg, tmp_str);
}
FREE_NULL_LIST(preempted_jobs);
}
} else {
xstrcat(reply_msg, "Jobs not runable on selected nodes");
error("wiki: jobs not runnable on nodes");
}
/* Restore pending job's expected start time */
job_ptr->start_time = orig_start_time;
FREE_NULL_BITMAP(avail_bitmap);
FREE_NULL_BITMAP(exc_core_bitmap);
return reply_msg;
}
static char * _will_run_test(uint32_t jobid, time_t start_time,
char *node_list, int *err_code, char **err_msg)
{
struct job_record *job_ptr = NULL;
struct part_record *part_ptr;
bitstr_t *avail_bitmap = NULL, *resv_bitmap = NULL;
bitstr_t *exc_core_bitmap = NULL;
char *hostlist, *reply_msg = NULL;
uint32_t min_nodes, max_nodes, req_nodes;
int rc;
time_t start_res, orig_start_time;
List preemptee_candidates;
debug2("wiki2: will_run job_id=%u start_time=%u node_list=%s",
jobid, (uint32_t)start_time, node_list);
job_ptr = find_job_record(jobid);
if (job_ptr == NULL) {
*err_code = -700;
*err_msg = "No such job";
error("wiki: Failed to find job %u", jobid);
return NULL;
}
if ((job_ptr->details == NULL) || (!IS_JOB_PENDING(job_ptr))) {
*err_code = -700;
*err_msg = "WillRun not applicable to non-pending job";
error("wiki: WillRun on non-pending job %u", jobid);
return NULL;
}
part_ptr = job_ptr->part_ptr;
if (part_ptr == NULL) {
*err_code = -700;
*err_msg = "Job lacks a partition";
error("wiki: Job %u lacks a partition", jobid);
return NULL;
}
if ((node_list == NULL) || (node_list[0] == '\0')) {
/* assume all nodes available to job for testing */
avail_bitmap = bit_copy(avail_node_bitmap);
} else if (node_name2bitmap(node_list, false, &avail_bitmap) != 0) {
*err_code = -700;
*err_msg = "Invalid available nodes value";
error("wiki: Attempt to set invalid available node "
"list for job %u, %s", jobid, node_list);
return NULL;
}
/* Enforce reservation: access control, time and nodes */
start_res = start_time;
rc = job_test_resv(job_ptr, &start_res, true, &resv_bitmap,
&exc_core_bitmap);
if (rc != SLURM_SUCCESS) {
*err_code = -730;
*err_msg = "Job denied access to reservation";
error("wiki: reservation access denied for job %u", jobid);
FREE_NULL_BITMAP(avail_bitmap);
return NULL;
}
start_time = MAX(start_time, start_res);
bit_and(avail_bitmap, resv_bitmap);
FREE_NULL_BITMAP(resv_bitmap);
/* Only consider nodes that are not DOWN or DRAINED */
bit_and(avail_bitmap, avail_node_bitmap);
/* Consider only nodes in this job's partition */
if (part_ptr->node_bitmap)
bit_and(avail_bitmap, part_ptr->node_bitmap);
else {
*err_code = -730;
*err_msg = "Job's partition has no nodes";
error("wiki: no nodes in partition %s for job %u",
part_ptr->name, jobid);
FREE_NULL_BITMAP(avail_bitmap);
return NULL;
}
if (job_req_node_filter(job_ptr, avail_bitmap) != SLURM_SUCCESS) {
/* Job probably has invalid feature list */
*err_code = -730;
*err_msg = "Job's required features not available "
"on selected nodes";
error("wiki: job %u not runnable on hosts=%s",
jobid, node_list);
FREE_NULL_BITMAP(avail_bitmap);
return NULL;
}
if (job_ptr->details->exc_node_bitmap) {
bit_not(job_ptr->details->exc_node_bitmap);
bit_and(avail_bitmap, job_ptr->details->exc_node_bitmap);
bit_not(job_ptr->details->exc_node_bitmap);
}
if ((job_ptr->details->req_node_bitmap) &&
(!bit_super_set(job_ptr->details->req_node_bitmap,
avail_bitmap))) {
*err_code = -730;
*err_msg = "Job's required nodes not available";
error("wiki: job %u not runnable on hosts=%s",
jobid, node_list);
FREE_NULL_BITMAP(avail_bitmap);
return NULL;
}
min_nodes = MAX(job_ptr->details->min_nodes, part_ptr->min_nodes);
if (job_ptr->details->max_nodes == 0)
max_nodes = part_ptr->max_nodes;
else
max_nodes = MIN(job_ptr->details->max_nodes,
part_ptr->max_nodes);
max_nodes = MIN(max_nodes, 500000); /* prevent overflows */
if (job_ptr->details->max_nodes)
req_nodes = max_nodes;
else
req_nodes = min_nodes;
if (min_nodes > max_nodes) {
/* job's min_nodes exceeds partitions max_nodes */
*err_code = -730;
*err_msg = "Job's min_nodes > max_nodes";
error("wiki: job %u not runnable on hosts=%s",
jobid, node_list);
FREE_NULL_BITMAP(avail_bitmap);
return NULL;
}
preemptee_candidates = slurm_find_preemptable_jobs(job_ptr);
orig_start_time = job_ptr->start_time;
rc = select_g_job_test(job_ptr, avail_bitmap,
min_nodes, max_nodes, req_nodes,
SELECT_MODE_WILL_RUN,
preemptee_candidates, NULL, exc_core_bitmap);
if (preemptee_candidates)
list_destroy(preemptee_candidates);
if (rc == SLURM_SUCCESS) {
char tmp_str[128];
*err_code = 0;
uint32_t proc_cnt = 0;
xstrcat(reply_msg, "STARTINFO=");
#ifdef HAVE_BG
select_g_select_jobinfo_get(job_ptr->select_jobinfo,
SELECT_JOBDATA_NODE_CNT,
&proc_cnt);
#else
proc_cnt = job_ptr->total_cpus;
#endif
snprintf(tmp_str, sizeof(tmp_str), "%u:%u@%u,",
jobid, proc_cnt, (uint32_t) job_ptr->start_time);
xstrcat(reply_msg, tmp_str);
hostlist = bitmap2node_name(avail_bitmap);
xstrcat(reply_msg, hostlist);
xfree(hostlist);
} else {
xstrcat(reply_msg, "Jobs not runable on selected nodes");
error("wiki: jobs not runnable on nodes");
}
/* Restore pending job's expected start time */
job_ptr->start_time = orig_start_time;
FREE_NULL_BITMAP(avail_bitmap);
return reply_msg;
}
extern void process_nodes(bg_record_t *bg_record, bool startup)
{
int j=0;
int diff=0;
int largest_diff=-1;
uint16_t best_start[SYSTEM_DIMENSIONS];
uint16_t start[SYSTEM_DIMENSIONS];
uint16_t end[SYSTEM_DIMENSIONS];
bool start_set=0;
ListIterator itr;
ba_mp_t* ba_mp = NULL;
int dim;
static char tmp_char[SYSTEM_DIMENSIONS+1],
tmp_char2[SYSTEM_DIMENSIONS+1];
static int *cluster_dims = NULL;
if (!cluster_dims) {
/* do some initing that only needs to happen once. */
cluster_dims = select_g_ba_get_dims();
memset(tmp_char, 0, sizeof(tmp_char));
memset(tmp_char2, 0, sizeof(tmp_char2));
}
if (!bg_record->ba_mp_list || !list_count(bg_record->ba_mp_list)) {
char *nodes = bg_record->mp_str;
if (!bg_record->ba_mp_list)
bg_record->ba_mp_list = list_create(destroy_ba_mp);
memset(&best_start, 0, sizeof(best_start));
//bg_record->mp_count = 0;
if ((bg_record->conn_type[0] >= SELECT_SMALL) && (!startup))
error("process_nodes: "
"We shouldn't be here there could be some "
"badness if we use this logic %s",
bg_record->mp_str);
while (nodes[j] != '\0') {
int mid = j + SYSTEM_DIMENSIONS + 1;
int fin = mid + SYSTEM_DIMENSIONS + 1;
if (((nodes[j] == '[') || (nodes[j] == ',')) &&
((nodes[mid] == 'x') || (nodes[mid] == '-')) &&
((nodes[fin] == ']') || (nodes[fin] == ','))) {
j++; /* Skip leading '[' or ',' */
for (dim = 0; dim < SYSTEM_DIMENSIONS;
dim++, j++)
start[dim] = select_char2coord(
nodes[j]);
j++; /* Skip middle 'x' or '-' */
for (dim = 0; dim < SYSTEM_DIMENSIONS;
dim++, j++)
end[dim] = select_char2coord(nodes[j]);
diff = end[0]-start[0];
_addto_mp_list(bg_record, start, end);
} else if ((nodes[j] >= '0'&& nodes[j] <= '9')
|| (nodes[j] >= 'A' && nodes[j] <= 'Z')) {
for (dim = 0; dim < SYSTEM_DIMENSIONS;
dim++, j++)
start[dim] = select_char2coord(
nodes[j]);
diff = 0;
_addto_mp_list(bg_record, start, start);
} else {
j++;
continue;
}
if (diff > largest_diff) {
largest_diff = diff;
memcpy(best_start, start, sizeof(best_start));
if (bg_conf->slurm_debug_level
>= LOG_LEVEL_DEBUG3) {
for (dim = 0;
dim < SYSTEM_DIMENSIONS;
dim++)
tmp_char[dim] = alpha_num[
best_start[dim]];
debug3("process_nodes: start is now %s",
tmp_char);
}
}
if (bg_record->mp_str[j] != ',')
break;
}
if (largest_diff == -1)
fatal("No hostnames given here");
memcpy(bg_record->start, best_start, sizeof(bg_record->start));
start_set = 1;
if (bg_conf->slurm_debug_level >= LOG_LEVEL_DEBUG3) {
for (dim = 0; dim < SYSTEM_DIMENSIONS; dim++) {
tmp_char[dim] = alpha_num[best_start[dim]];
tmp_char2[dim] =
alpha_num[bg_record->start[dim]];
}
debug3("process_nodes: start is %s %s",
tmp_char, tmp_char2);
}
}
memset(bg_record->geo, 0, sizeof(bg_record->geo));
for (dim = 0; dim < SYSTEM_DIMENSIONS; dim++) {
end[dim] = (int16_t)-1;
if (!start_set)
bg_record->start[dim] = HOSTLIST_BASE;
}
list_sort(bg_record->ba_mp_list, (ListCmpF) _ba_mp_cmpf_inc);
bg_record->mp_count = 0;
itr = list_iterator_create(bg_record->ba_mp_list);
while ((ba_mp = list_next(itr))) {
if (!ba_mp->used)
continue;
bg_record->mp_count++;
debug3("process_nodes: %s is included in this block",
ba_mp->coord_str);
for (dim = 0; dim < SYSTEM_DIMENSIONS; dim++) {
if (ba_mp->coord[dim] > (int16_t)end[dim]) {
bg_record->geo[dim]++;
end[dim] = ba_mp->coord[dim];
}
if (!start_set && (ba_mp->coord[dim] <
(int16_t)bg_record->start[dim]))
bg_record->start[dim] = ba_mp->coord[dim];
}
}
list_iterator_destroy(itr);
if (bg_conf->slurm_debug_level >= LOG_LEVEL_DEBUG3) {
for (dim = 0; dim < SYSTEM_DIMENSIONS; dim++) {
tmp_char[dim] = alpha_num[bg_record->geo[dim]];
tmp_char2[dim] = alpha_num[bg_record->start[dim]];
}
debug3("process_nodes: geo = %s mp count is %d start is %s",
tmp_char, bg_record->mp_count, tmp_char2);
}
/* This check is for sub midplane systems to figure out what
the largest block can be.
*/
for (dim = 0; dim < SYSTEM_DIMENSIONS; dim++) {
if (cluster_dims[dim] > 1)
break;
}
if (dim < SYSTEM_DIMENSIONS) {
/* means we have more than 1 base partition */
for (dim = 0; dim < SYSTEM_DIMENSIONS; dim++) {
if (bg_record->geo[dim] != cluster_dims[dim])
break;
}
if (dim == SYSTEM_DIMENSIONS)
bg_record->full_block = 1;
} else if (bg_record->cnode_cnt == bg_conf->mp_cnode_cnt)
bg_record->full_block = 1;
FREE_NULL_BITMAP(bg_record->mp_bitmap);
if (node_name2bitmap(bg_record->mp_str,
false,
&bg_record->mp_bitmap)) {
fatal("process_nodes: Unable to convert nodes %s to bitmap",
bg_record->mp_str);
}
return;
}
/* Perform any power change work to nodes */
static void _do_power_work(time_t now)
{
static time_t last_log = 0, last_work_scan = 0;
int i, wake_cnt = 0, sleep_cnt = 0, susp_total = 0;
time_t delta_t;
uint32_t susp_state;
bitstr_t *wake_node_bitmap = NULL, *sleep_node_bitmap = NULL;
struct node_record *node_ptr;
bool run_suspend = false;
if (last_work_scan == 0) {
if (exc_nodes &&
(node_name2bitmap(exc_nodes, false, &exc_node_bitmap))) {
error("Invalid SuspendExcNodes %s ignored", exc_nodes);
}
if (exc_parts) {
char *tmp = NULL, *one_part = NULL, *part_list = NULL;
struct part_record *part_ptr = NULL;
part_list = xstrdup(exc_parts);
one_part = strtok_r(part_list, ",", &tmp);
while (one_part != NULL) {
part_ptr = find_part_record(one_part);
if (!part_ptr) {
error("Invalid SuspendExcPart %s ignored",
one_part);
} else if (exc_node_bitmap) {
bit_or(exc_node_bitmap,
part_ptr->node_bitmap);
} else {
exc_node_bitmap =
bit_copy(part_ptr->node_bitmap);
}
one_part = strtok_r(NULL, ",", &tmp);
}
xfree(part_list);
}
if (exc_node_bitmap) {
char *tmp = bitmap2node_name(exc_node_bitmap);
info("power_save module, excluded nodes %s", tmp);
xfree(tmp);
}
}
/* Set limit on counts of nodes to have state changed */
delta_t = now - last_work_scan;
if (delta_t >= 60) {
suspend_cnt_f = 0.0;
resume_cnt_f = 0.0;
} else {
float rate = (60 - delta_t) / 60.0;
suspend_cnt_f *= rate;
resume_cnt_f *= rate;
}
suspend_cnt = (suspend_cnt_f + 0.5);
resume_cnt = (resume_cnt_f + 0.5);
if (now > (last_suspend + suspend_timeout)) {
/* ready to start another round of node suspends */
run_suspend = true;
if (last_suspend) {
bit_nclear(suspend_node_bitmap, 0,
(node_record_count - 1));
bit_nclear(resume_node_bitmap, 0,
(node_record_count - 1));
last_suspend = (time_t) 0;
}
}
last_work_scan = now;
/* Build bitmaps identifying each node which should change state */
for (i = 0, node_ptr = node_record_table_ptr;
i < node_record_count; i++, node_ptr++) {
susp_state = IS_NODE_POWER_SAVE(node_ptr);
if (susp_state)
susp_total++;
/* Resume nodes as appropriate */
if (susp_state &&
((resume_rate == 0) || (resume_cnt < resume_rate)) &&
(bit_test(suspend_node_bitmap, i) == 0) &&
(IS_NODE_ALLOCATED(node_ptr) ||
(node_ptr->last_idle > (now - idle_time)))) {
if (wake_node_bitmap == NULL) {
wake_node_bitmap =
bit_alloc(node_record_count);
}
wake_cnt++;
resume_cnt++;
resume_cnt_f++;
node_ptr->node_state &= (~NODE_STATE_POWER_SAVE);
node_ptr->node_state |= NODE_STATE_POWER_UP;
node_ptr->node_state |= NODE_STATE_NO_RESPOND;
bit_clear(power_node_bitmap, i);
bit_clear(avail_node_bitmap, i);
node_ptr->last_response = now + resume_timeout;
bit_set(wake_node_bitmap, i);
bit_set(resume_node_bitmap, i);
}
/* Suspend nodes as appropriate */
if (run_suspend &&
(susp_state == 0) &&
((suspend_rate == 0) || (suspend_cnt < suspend_rate)) &&
(IS_NODE_IDLE(node_ptr) || IS_NODE_DOWN(node_ptr)) &&
(node_ptr->sus_job_cnt == 0) &&
(!IS_NODE_COMPLETING(node_ptr)) &&
(!IS_NODE_POWER_UP(node_ptr)) &&
(node_ptr->last_idle != 0) &&
(node_ptr->last_idle < (now - idle_time)) &&
((exc_node_bitmap == NULL) ||
(bit_test(exc_node_bitmap, i) == 0))) {
if (sleep_node_bitmap == NULL) {
sleep_node_bitmap =
bit_alloc(node_record_count);
}
sleep_cnt++;
suspend_cnt++;
suspend_cnt_f++;
node_ptr->node_state |= NODE_STATE_POWER_SAVE;
node_ptr->node_state &= (~NODE_STATE_NO_RESPOND);
if (!IS_NODE_DOWN(node_ptr) &&
!IS_NODE_DRAIN(node_ptr) &&
!IS_NODE_FAIL(node_ptr))
bit_set(avail_node_bitmap, i);
bit_set(power_node_bitmap, i);
bit_set(sleep_node_bitmap, i);
bit_set(suspend_node_bitmap, i);
last_suspend = now;
}
}
if (((now - last_log) > 600) && (susp_total > 0)) {
info("Power save mode: %d nodes", susp_total);
last_log = now;
}
if (sleep_node_bitmap) {
char *nodes;
nodes = bitmap2node_name(sleep_node_bitmap);
if (nodes)
_do_suspend(nodes);
else
error("power_save: bitmap2nodename");
xfree(nodes);
FREE_NULL_BITMAP(sleep_node_bitmap);
/* last_node_update could be changed already by another thread!
last_node_update = now; */
}
if (wake_node_bitmap) {
char *nodes;
nodes = bitmap2node_name(wake_node_bitmap);
if (nodes)
_do_resume(nodes);
else
error("power_save: bitmap2nodename");
xfree(nodes);
FREE_NULL_BITMAP(wake_node_bitmap);
/* last_node_update could be changed already by another thread!
last_node_update = now; */
}
}