/* If slurmctld crashes, the node state that it recovers could differ * from the actual hardware state (e.g. ResumeProgram failed to complete). * To address that, when a node that should be powered up for a running * job is not responding, they try running ResumeProgram again. */ static void _re_wake(void) { struct node_record *node_ptr; bitstr_t *wake_node_bitmap = NULL; int i; node_ptr = node_record_table_ptr; for (i=0; i<node_record_count; i++, node_ptr++) { if (IS_NODE_ALLOCATED(node_ptr) && IS_NODE_NO_RESPOND(node_ptr) && !IS_NODE_POWER_SAVE(node_ptr) && (bit_test(suspend_node_bitmap, i) == 0) && (bit_test(resume_node_bitmap, i) == 0)) { if (wake_node_bitmap == NULL) { wake_node_bitmap = bit_alloc(node_record_count); } bit_set(wake_node_bitmap, i); } } if (wake_node_bitmap) { char *nodes; nodes = bitmap2node_name(wake_node_bitmap); if (nodes) { pid_t pid = _run_prog(resume_prog, nodes, NULL); info("power_save: pid %d rewaking nodes %s", (int) pid, nodes); } else error("power_save: bitmap2nodename"); xfree(nodes); FREE_NULL_BITMAP(wake_node_bitmap); } }
static void _do_suspend(char *host) { pid_t pid = _run_prog(suspend_prog, host, NULL); #if _DEBUG info("power_save: pid %d suspending nodes %s", (int) pid, host); #else verbose("power_save: pid %d suspending nodes %s", (int) pid, host); #endif }
static void _do_resume(char *host) { pid_t pid = _run_prog(resume_prog, host, NULL); #if _DEBUG info("power_save: pid %d waking nodes %s", (int) pid, host); #else verbose("power_save: pid %d waking nodes %s", (int) pid, host); #endif }
static void _do_suspend(char *host) { #if _DEBUG info("power_save: suspending nodes %s", host); #else verbose("power_save: suspending nodes %s", host); #endif _run_prog(suspend_prog, host, NULL); }
static void _do_resume(char *host) { #if _DEBUG info("power_save: waking nodes %s", host); #else verbose("power_save: waking nodes %s", host); #endif _run_prog(resume_prog, host, NULL); }
/* power_job_reboot - Reboot compute nodes for a job from the head node */ extern int power_job_reboot(struct job_record *job_ptr) { int rc = SLURM_SUCCESS; int i, i_first, i_last; struct node_record *node_ptr; bitstr_t *wake_node_bitmap = NULL; time_t now = time(NULL); char *nodes, *features = NULL; wake_node_bitmap = bit_alloc(node_record_count); i_first = bit_ffs(job_ptr->node_bitmap); i_last = bit_fls(job_ptr->node_bitmap); for (i = i_first; i <= i_last; i++) { if (!bit_test(job_ptr->node_bitmap, i)) continue; node_ptr = node_record_table_ptr + i; resume_cnt++; resume_cnt_f++; node_ptr->node_state &= (~NODE_STATE_POWER_SAVE); node_ptr->node_state |= NODE_STATE_POWER_UP; node_ptr->node_state |= NODE_STATE_NO_RESPOND; bit_clear(power_node_bitmap, i); bit_clear(avail_node_bitmap, i); node_ptr->last_response = now + resume_timeout; bit_set(wake_node_bitmap, i); bit_set(resume_node_bitmap, i); } nodes = bitmap2node_name(wake_node_bitmap); if (nodes) { #if _DEBUG info("power_save: reboot nodes %s", nodes); #else verbose("power_save: reboot nodes %s", nodes); #endif if (job_ptr->details && job_ptr->details->features) features = xlate_features(job_ptr->details->features); _run_prog(resume_prog, nodes, features); xfree(features); } else { error("power_save: bitmap2nodename"); rc = SLURM_ERROR; } xfree(nodes); FREE_NULL_BITMAP(wake_node_bitmap); last_node_update = now; return rc; }
/* power_job_reboot - Reboot compute nodes for a job from the head node */ extern int power_job_reboot(struct job_record *job_ptr) { int rc = SLURM_SUCCESS; int i, i_first, i_last; struct node_record *node_ptr; bitstr_t *boot_node_bitmap = NULL; time_t now = time(NULL); char *nodes, *features = NULL; pid_t pid; boot_node_bitmap = node_features_reboot(job_ptr); if (boot_node_bitmap == NULL) return SLURM_SUCCESS; i_first = bit_ffs(boot_node_bitmap); if (i_first >= 0) i_last = bit_fls(boot_node_bitmap); else i_last = i_first - 1; for (i = i_first; i <= i_last; i++) { if (!bit_test(boot_node_bitmap, i)) continue; node_ptr = node_record_table_ptr + i; resume_cnt++; resume_cnt_f++; node_ptr->node_state &= (~NODE_STATE_POWER_SAVE); node_ptr->node_state |= NODE_STATE_POWER_UP; node_ptr->node_state |= NODE_STATE_NO_RESPOND; bit_clear(power_node_bitmap, i); bit_clear(avail_node_bitmap, i); node_ptr->last_response = now + resume_timeout; bit_set(resume_node_bitmap, i); } nodes = bitmap2node_name(boot_node_bitmap); if (nodes) { job_ptr->job_state |= JOB_CONFIGURING; job_ptr->wait_all_nodes = 1; if (job_ptr->details && job_ptr->details->features && node_features_g_user_update(job_ptr->user_id)) { features = node_features_g_job_xlate( job_ptr->details->features); } pid = _run_prog(resume_prog, nodes, features); #if _DEBUG info("power_save: pid %d reboot nodes %s features %s", (int) pid, nodes, features); #else verbose("power_save: pid %d reboot nodes %s features %s", (int) pid, nodes, features); #endif xfree(features); } else { error("power_save: bitmap2nodename"); rc = SLURM_ERROR; } xfree(nodes); FREE_NULL_BITMAP(boot_node_bitmap); last_node_update = now; return rc; }