extern int bg_status_update_block_state(bg_record_t *bg_record,
uint16_t state,
List kill_job_list)
{
bool skipped_dealloc = false;
kill_job_struct_t *freeit = NULL;
int updated = 0;
uint16_t real_state = bg_record->state & (~BG_BLOCK_ERROR_FLAG);
if (real_state == state)
return 0;
debug("state of Block %s was %s and now is %s",
bg_record->bg_block_id,
bg_block_state_string(bg_record->state),
bg_block_state_string(state));
/*
check to make sure block went
through freeing correctly
*/
if ((real_state != BG_BLOCK_TERM
&& !(bg_record->state & BG_BLOCK_ERROR_FLAG))
&& state == BG_BLOCK_FREE)
skipped_dealloc = 1;
else if ((real_state == BG_BLOCK_INITED)
&& (state == BG_BLOCK_BOOTING)) {
/* This means the user did a reboot through
mpirun but we missed the state
change */
debug("Block %s skipped rebooting, "
"but it really is. "
"Setting target_name back to %s",
bg_record->bg_block_id,
bg_record->user_name);
xfree(bg_record->target_name);
bg_record->target_name = xstrdup(bg_record->user_name);
} else if ((real_state == BG_BLOCK_TERM)
&& (state == BG_BLOCK_BOOTING))
/* This is a funky state IBM says
isn't a bug, but all their
documentation says this doesn't
happen, but IBM says oh yeah, you
weren't really suppose to notice
that. So we will just skip this
state and act like this didn't happen. */
goto nochange_state;
real_state = state;
if (bg_record->state & BG_BLOCK_ERROR_FLAG)
state |= BG_BLOCK_ERROR_FLAG;
bg_record->state = state;
if (real_state == BG_BLOCK_TERM || skipped_dealloc)
_block_is_deallocating(bg_record, kill_job_list);
else if (real_state == BG_BLOCK_BOOTING) {
debug("Setting bootflag for %s", bg_record->bg_block_id);
bg_record->boot_state = 1;
} else if (real_state == BG_BLOCK_FREE) {
if (remove_from_bg_list(bg_lists->job_running, bg_record)
== SLURM_SUCCESS)
num_unused_cpus += bg_record->cpu_cnt;
remove_from_bg_list(bg_lists->booted,
bg_record);
} else if (real_state & BG_BLOCK_ERROR_FLAG) {
if (bg_record->boot_state)
error("Block %s in an error state while booting.",
bg_record->bg_block_id);
else
error("Block %s in an error state.",
bg_record->bg_block_id);
remove_from_bg_list(bg_lists->booted, bg_record);
trigger_block_error();
} else if (real_state == BG_BLOCK_INITED) {
if (!block_ptr_exist_in_list(bg_lists->booted, bg_record))
list_push(bg_lists->booted, bg_record);
}
updated = 1;
nochange_state:
/* check the boot state */
debug3("boot state for block %s is %d",
bg_record->bg_block_id, bg_record->boot_state);
if (bg_record->boot_state) {
if (bg_record->state & BG_BLOCK_ERROR_FLAG) {
/* If we get an error on boot that
* means it is a transparent L3 error
* and should be trying to fix
* itself. If this is the case we
* just hang out waiting for the state
* to go to free where we will try to
* boot again below.
*/
return updated;
}
switch (real_state) {
case BG_BLOCK_BOOTING:
debug3("checking to make sure user %s "
"is the user.",
bg_record->target_name);
if (update_block_user(bg_record, 0) == 1)
last_bg_update = time(NULL);
if (bg_record->job_ptr) {
bg_record->job_ptr->job_state |=
JOB_CONFIGURING;
last_job_update = time(NULL);
}
break;
case BG_BLOCK_FREE:
if (bg_record->boot_count < RETRY_BOOT_COUNT) {
bridge_block_boot(bg_record);
if (bg_record->magic == BLOCK_MAGIC) {
debug("boot count for block %s is %d",
bg_record->bg_block_id,
bg_record->boot_count);
bg_record->boot_count++;
}
} else {
char *reason = (char *)
"status_check: Boot fails ";
error("Couldn't boot Block %s for user %s",
bg_record->bg_block_id,
bg_record->target_name);
slurm_mutex_unlock(&block_state_mutex);
requeue_and_error(bg_record, reason);
slurm_mutex_lock(&block_state_mutex);
bg_record->boot_state = 0;
bg_record->boot_count = 0;
if (remove_from_bg_list(
bg_lists->job_running, bg_record)
== SLURM_SUCCESS)
num_unused_cpus += bg_record->cpu_cnt;
remove_from_bg_list(bg_lists->booted,
bg_record);
}
break;
case BG_BLOCK_INITED:
debug("block %s is ready.",
bg_record->bg_block_id);
if (bg_record->job_ptr) {
bg_record->job_ptr->job_state &=
(~JOB_CONFIGURING);
last_job_update = time(NULL);
}
/* boot flags are reset here */
if (kill_job_list &&
set_block_user(bg_record) == SLURM_ERROR) {
freeit = (kill_job_struct_t *)
xmalloc(sizeof(kill_job_struct_t));
freeit->jobid = bg_record->job_running;
list_push(kill_job_list, freeit);
}
break;
case BG_BLOCK_TERM:
debug2("Block %s is in a deallocating state "
"during a boot. Doing nothing until "
"free state.",
bg_record->bg_block_id);
break;
case BG_BLOCK_REBOOTING:
debug2("Block %s is rebooting.",
bg_record->bg_block_id);
break;
default:
debug("Hey the state of block "
"%s is %d(%s) doing nothing.",
bg_record->bg_block_id,
real_state,
bg_block_state_string(bg_record->state));
break;
}
}
return updated;
}
extern int bg_status_update_block_state(bg_record_t *bg_record,
uint16_t state,
List kill_job_list)
{
bool skipped_dealloc = false;
kill_job_struct_t *freeit = NULL;
int updated = 0;
uint16_t real_state = bg_record->state & (~BG_BLOCK_ERROR_FLAG);
if (real_state == state)
return 0;
debug("state of Block %s was %s and now is %s",
bg_record->bg_block_id,
bg_block_state_string(bg_record->state),
bg_block_state_string(state));
/*
check to make sure block went
through freeing correctly
*/
if ((real_state != BG_BLOCK_TERM
&& !(bg_record->state & BG_BLOCK_ERROR_FLAG))
&& state == BG_BLOCK_FREE)
skipped_dealloc = 1;
else if ((real_state == BG_BLOCK_INITED)
&& (state == BG_BLOCK_BOOTING)) {
/* This means the user did a reboot through
mpirun but we missed the state
change */
debug("Block %s skipped rebooting, "
"but it really is.",
bg_record->bg_block_id);
} else if ((real_state == BG_BLOCK_TERM)
&& (state == BG_BLOCK_BOOTING))
/* This is a funky state IBM says
isn't a bug, but all their
documentation says this doesn't
happen, but IBM says oh yeah, you
weren't really suppose to notice
that. So we will just skip this
state and act like this didn't happen. */
goto nochange_state;
real_state = state;
if (bg_record->state & BG_BLOCK_ERROR_FLAG)
state |= BG_BLOCK_ERROR_FLAG;
bg_record->state = state;
if (real_state == BG_BLOCK_TERM || skipped_dealloc)
_block_is_deallocating(bg_record, kill_job_list);
else if (real_state == BG_BLOCK_BOOTING) {
debug("Setting bootflag for %s", bg_record->bg_block_id);
bg_record->boot_state = 1;
} else if (real_state == BG_BLOCK_FREE) {
/* Make sure block is cleaned up. If there are
* running jobs on the block this happens when they
* are cleaned off. */
if (bg_record->job_running == NO_JOB_RUNNING
&& (!bg_record->job_list
|| !list_count(bg_record->job_list)))
bg_reset_block(bg_record, NULL);
remove_from_bg_list(bg_lists->booted, bg_record);
} else if (real_state & BG_BLOCK_ERROR_FLAG) {
if (bg_record->boot_state)
error("Block %s in an error state while booting.",
bg_record->bg_block_id);
else
error("Block %s in an error state.",
bg_record->bg_block_id);
remove_from_bg_list(bg_lists->booted, bg_record);
trigger_block_error();
} else if (real_state == BG_BLOCK_INITED) {
if (!block_ptr_exist_in_list(bg_lists->booted, bg_record))
list_push(bg_lists->booted, bg_record);
}
updated = 1;
last_bg_update = time(NULL);
nochange_state:
/* check the boot state */
debug3("boot state for block %s is %d",
bg_record->bg_block_id, bg_record->boot_state);
if (bg_record->boot_state) {
if (bg_record->state & BG_BLOCK_ERROR_FLAG) {
/* If we get an error on boot that
* means it is a transparent L3 error
* and should be trying to fix
* itself. If this is the case we
* just hang out waiting for the state
* to go to free where we will try to
* boot again below.
*/
return updated;
}
switch (real_state) {
case BG_BLOCK_BOOTING:
if (bg_record->job_ptr
&& !IS_JOB_CONFIGURING(bg_record->job_ptr)) {
debug3("Setting job %u on block %s "
"to configuring",
bg_record->job_ptr->job_id,
bg_record->bg_block_id);
bg_record->job_ptr->job_state |=
JOB_CONFIGURING;
last_job_update = time(NULL);
} else if (bg_record->job_list
&& list_count(bg_record->job_list)) {
struct job_record *job_ptr;
ListIterator job_itr =
list_iterator_create(
bg_record->job_list);
while ((job_ptr = list_next(job_itr))) {
if (job_ptr->magic != JOB_MAGIC) {
error("bg_status_update_"
"block_state: 1 "
"bad magic found when "
"looking at block %s",
bg_record->bg_block_id);
list_delete_item(job_itr);
continue;
}
job_ptr->job_state |= JOB_CONFIGURING;
}
list_iterator_destroy(job_itr);
last_job_update = time(NULL);
}
break;
case BG_BLOCK_FREE:
if (bg_record->boot_count < RETRY_BOOT_COUNT) {
bridge_block_boot(bg_record);
if (bg_record->magic == BLOCK_MAGIC) {
debug("boot count for block %s is %d",
bg_record->bg_block_id,
bg_record->boot_count);
bg_record->boot_count++;
}
} else {
char *reason = (char *)
"status_check: Boot fails ";
error("Couldn't boot Block %s",
bg_record->bg_block_id);
/* We can't push on the kill_job_list
here since we have to put this
block in an error and that means
the killing has to take place
before the erroring of the block.
*/
slurm_mutex_unlock(&block_state_mutex);
unlock_slurmctld(job_read_lock);
requeue_and_error(bg_record, reason);
lock_slurmctld(job_read_lock);
slurm_mutex_lock(&block_state_mutex);
bg_record->boot_state = 0;
bg_record->boot_count = 0;
remove_from_bg_list(bg_lists->booted,
bg_record);
}
break;
case BG_BLOCK_INITED:
debug("block %s is ready.",
bg_record->bg_block_id);
if (bg_record->job_ptr
&& IS_JOB_CONFIGURING(bg_record->job_ptr)) {
bg_record->job_ptr->job_state &=
(~JOB_CONFIGURING);
last_job_update = time(NULL);
} else if (bg_record->job_list
&& list_count(bg_record->job_list)) {
struct job_record *job_ptr;
ListIterator job_itr =
list_iterator_create(
bg_record->job_list);
while ((job_ptr = list_next(job_itr))) {
if (job_ptr->magic != JOB_MAGIC) {
error("bg_status_update_"
"block_state: 2 "
"bad magic found when "
"looking at block %s",
bg_record->bg_block_id);
list_delete_item(job_itr);
continue;
}
job_ptr->job_state &=
(~JOB_CONFIGURING);
}
list_iterator_destroy(job_itr);
last_job_update = time(NULL);
}
bg_record->boot_state = 0;
bg_record->boot_count = 0;
if (kill_job_list &&
bridge_block_sync_users(bg_record)
== SLURM_ERROR) {
freeit = (kill_job_struct_t *)
xmalloc(sizeof(kill_job_struct_t));
freeit->jobid = bg_record->job_running;
list_push(kill_job_list, freeit);
}
break;
case BG_BLOCK_TERM:
debug2("Block %s is in a deallocating state "
"during a boot. Doing nothing until "
"free state.",
bg_record->bg_block_id);
break;
case BG_BLOCK_REBOOTING:
debug2("Block %s is rebooting.",
bg_record->bg_block_id);
break;
default:
debug("Hey the state of block "
"%s is %d(%s) doing nothing.",
bg_record->bg_block_id,
real_state,
bg_block_state_string(bg_record->state));
break;
}
}
return updated;
}
