/* block_state_mutex should be locked before calling this function */
extern int bg_reset_block(bg_record_t *bg_record)
{
int rc = SLURM_SUCCESS;
if (bg_record) {
if (bg_record->job_running > NO_JOB_RUNNING) {
bg_record->job_running = NO_JOB_RUNNING;
bg_record->job_ptr = NULL;
}
/* remove user from list */
if (bg_record->target_name) {
if (strcmp(bg_record->target_name,
bg_conf->slurm_user_name)) {
xfree(bg_record->target_name);
bg_record->target_name =
xstrdup(bg_conf->slurm_user_name);
}
update_block_user(bg_record, 1);
} else {
bg_record->target_name =
xstrdup(bg_conf->slurm_user_name);
}
/* Don't reset these (boot_(state/count)), they will be
reset when state changes, and needs to outlast a job
allocation.
*/
/* bg_record->boot_state = 0; */
/* bg_record->boot_count = 0; */
last_bg_update = time(NULL);
/* Only remove from the job_running list if
job_running == NO_JOB_RUNNING, since blocks in
error state could also be in this list and we don't
want to remove them.
*/
if (bg_record->job_running == NO_JOB_RUNNING)
if (remove_from_bg_list(bg_lists->job_running,
bg_record)
== SLURM_SUCCESS) {
num_unused_cpus += bg_record->cpu_cnt;
}
} else {
error("No block given to reset");
rc = SLURM_ERROR;
}
return rc;
}
extern int set_block_user(bg_record_t *bg_record)
{
int rc = 0;
if (bg_conf->slurm_debug_flags & DEBUG_FLAG_SELECT_TYPE)
info("resetting the boot state flag and "
"counter for block %s.",
bg_record->bg_block_id);
bg_record->boot_state = BG_BLOCK_FREE;
bg_record->boot_count = 0;
if ((rc = update_block_user(bg_record, 1)) == 1) {
last_bg_update = time(NULL);
rc = SLURM_SUCCESS;
} else if (rc == -1) {
error("Unable to add user name to block %s. "
"Cancelling job.",
bg_record->bg_block_id);
rc = SLURM_ERROR;
}
xfree(bg_record->target_name);
bg_record->target_name = xstrdup(bg_conf->slurm_user_name);
return rc;
}
extern int bg_status_update_block_state(bg_record_t *bg_record,
uint16_t state,
List kill_job_list)
{
bool skipped_dealloc = false;
kill_job_struct_t *freeit = NULL;
int updated = 0;
uint16_t real_state = bg_record->state & (~BG_BLOCK_ERROR_FLAG);
if (real_state == state)
return 0;
debug("state of Block %s was %s and now is %s",
bg_record->bg_block_id,
bg_block_state_string(bg_record->state),
bg_block_state_string(state));
/*
check to make sure block went
through freeing correctly
*/
if ((real_state != BG_BLOCK_TERM
&& !(bg_record->state & BG_BLOCK_ERROR_FLAG))
&& state == BG_BLOCK_FREE)
skipped_dealloc = 1;
else if ((real_state == BG_BLOCK_INITED)
&& (state == BG_BLOCK_BOOTING)) {
/* This means the user did a reboot through
mpirun but we missed the state
change */
debug("Block %s skipped rebooting, "
"but it really is. "
"Setting target_name back to %s",
bg_record->bg_block_id,
bg_record->user_name);
xfree(bg_record->target_name);
bg_record->target_name = xstrdup(bg_record->user_name);
} else if ((real_state == BG_BLOCK_TERM)
&& (state == BG_BLOCK_BOOTING))
/* This is a funky state IBM says
isn't a bug, but all their
documentation says this doesn't
happen, but IBM says oh yeah, you
weren't really suppose to notice
that. So we will just skip this
state and act like this didn't happen. */
goto nochange_state;
real_state = state;
if (bg_record->state & BG_BLOCK_ERROR_FLAG)
state |= BG_BLOCK_ERROR_FLAG;
bg_record->state = state;
if (real_state == BG_BLOCK_TERM || skipped_dealloc)
_block_is_deallocating(bg_record, kill_job_list);
else if (real_state == BG_BLOCK_BOOTING) {
debug("Setting bootflag for %s", bg_record->bg_block_id);
bg_record->boot_state = 1;
} else if (real_state == BG_BLOCK_FREE) {
if (remove_from_bg_list(bg_lists->job_running, bg_record)
== SLURM_SUCCESS)
num_unused_cpus += bg_record->cpu_cnt;
remove_from_bg_list(bg_lists->booted,
bg_record);
} else if (real_state & BG_BLOCK_ERROR_FLAG) {
if (bg_record->boot_state)
error("Block %s in an error state while booting.",
bg_record->bg_block_id);
else
error("Block %s in an error state.",
bg_record->bg_block_id);
remove_from_bg_list(bg_lists->booted, bg_record);
trigger_block_error();
} else if (real_state == BG_BLOCK_INITED) {
if (!block_ptr_exist_in_list(bg_lists->booted, bg_record))
list_push(bg_lists->booted, bg_record);
}
updated = 1;
nochange_state:
/* check the boot state */
debug3("boot state for block %s is %d",
bg_record->bg_block_id, bg_record->boot_state);
if (bg_record->boot_state) {
if (bg_record->state & BG_BLOCK_ERROR_FLAG) {
/* If we get an error on boot that
* means it is a transparent L3 error
* and should be trying to fix
* itself. If this is the case we
* just hang out waiting for the state
* to go to free where we will try to
* boot again below.
*/
return updated;
}
switch (real_state) {
case BG_BLOCK_BOOTING:
debug3("checking to make sure user %s "
"is the user.",
bg_record->target_name);
if (update_block_user(bg_record, 0) == 1)
last_bg_update = time(NULL);
if (bg_record->job_ptr) {
bg_record->job_ptr->job_state |=
JOB_CONFIGURING;
last_job_update = time(NULL);
}
break;
case BG_BLOCK_FREE:
if (bg_record->boot_count < RETRY_BOOT_COUNT) {
bridge_block_boot(bg_record);
if (bg_record->magic == BLOCK_MAGIC) {
debug("boot count for block %s is %d",
bg_record->bg_block_id,
bg_record->boot_count);
bg_record->boot_count++;
}
} else {
char *reason = (char *)
"status_check: Boot fails ";
error("Couldn't boot Block %s for user %s",
bg_record->bg_block_id,
bg_record->target_name);
slurm_mutex_unlock(&block_state_mutex);
requeue_and_error(bg_record, reason);
slurm_mutex_lock(&block_state_mutex);
bg_record->boot_state = 0;
bg_record->boot_count = 0;
if (remove_from_bg_list(
bg_lists->job_running, bg_record)
== SLURM_SUCCESS)
num_unused_cpus += bg_record->cpu_cnt;
remove_from_bg_list(bg_lists->booted,
bg_record);
}
break;
case BG_BLOCK_INITED:
debug("block %s is ready.",
bg_record->bg_block_id);
if (bg_record->job_ptr) {
bg_record->job_ptr->job_state &=
(~JOB_CONFIGURING);
last_job_update = time(NULL);
}
/* boot flags are reset here */
if (kill_job_list &&
set_block_user(bg_record) == SLURM_ERROR) {
freeit = (kill_job_struct_t *)
xmalloc(sizeof(kill_job_struct_t));
freeit->jobid = bg_record->job_running;
list_push(kill_job_list, freeit);
}
break;
case BG_BLOCK_TERM:
debug2("Block %s is in a deallocating state "
"during a boot. Doing nothing until "
"free state.",
bg_record->bg_block_id);
break;
case BG_BLOCK_REBOOTING:
debug2("Block %s is rebooting.",
bg_record->bg_block_id);
break;
default:
debug("Hey the state of block "
"%s is %d(%s) doing nothing.",
bg_record->bg_block_id,
real_state,
bg_block_state_string(bg_record->state));
break;
}
}
return updated;
}
