extern int up_nodecard(char *mp_name, bitstr_t *ionode_bitmap)
{
ListIterator itr = NULL;
bg_record_t *bg_record = NULL;
struct node_record *node_ptr = NULL;
int mp_bit = 0;
int ret = 0;
xassert(mp_name);
xassert(ionode_bitmap);
node_ptr = find_node_record(mp_name);
if (!node_ptr) {
error ("down_sub_node_blocks: invalid node specified %s",
mp_name);
return EINVAL;
}
mp_bit = (node_ptr - node_record_table_ptr);
slurm_mutex_lock(&block_state_mutex);
itr = list_iterator_create(bg_lists->main);
while ((bg_record = list_next(itr))) {
if (bg_record->job_running != BLOCK_ERROR_STATE)
continue;
if (!bit_test(bg_record->mp_bitmap, mp_bit))
continue;
if (!bit_overlap(bg_record->ionode_bitmap, ionode_bitmap)) {
continue;
}
resume_block(bg_record);
}
list_iterator_destroy(itr);
slurm_mutex_unlock(&block_state_mutex);
/* FIX ME: This needs to call the opposite of
slurm_drain_nodes which does not yet exist.
*/
if ((ret = node_already_down(mp_name))) {
/* means it was drained */
if (ret == 2) {
/* debug("node %s put back into service after " */
/* "being in an error state", */
/* mp_name); */
}
}
return SLURM_SUCCESS;
}
/* Find the specified BlueGene node ID and drain it from SLURM */
static void _configure_node_down(rm_bp_id_t bp_id, my_bluegene_t *my_bg)
{
int bp_num, i, rc;
rm_bp_id_t bpid;
rm_BP_t *my_bp;
rm_location_t bp_loc;
rm_BP_state_t bp_state;
char bg_down_node[128];
if ((rc = bridge_get_data(my_bg, RM_BPNum, &bp_num)) != SLURM_SUCCESS) {
error("bridge_get_data(RM_BPNum): %s", bg_err_str(rc));
bp_num = 0;
}
for (i=0; i<bp_num; i++) {
if (i) {
if ((rc = bridge_get_data(my_bg, RM_NextBP, &my_bp))
!= SLURM_SUCCESS) {
error("bridge_get_data(RM_NextBP): %s",
bg_err_str(rc));
continue;
}
} else {
if ((rc = bridge_get_data(my_bg, RM_FirstBP, &my_bp))
!=
SLURM_SUCCESS) {
error("bridge_get_data(RM_FirstBP): %s",
bg_err_str(rc));
continue;
}
}
if ((rc = bridge_get_data(my_bp, RM_BPID, &bpid))
!= SLURM_SUCCESS) {
error("bridge_get_data(RM_BPID): %s",
bg_err_str(rc));
continue;
}
if (!bpid) {
error("No BPID was returned from database");
continue;
}
if (strcmp(bp_id, bpid) != 0) { /* different midplane */
free(bpid);
continue;
}
free(bpid);
if ((rc = bridge_get_data(my_bp, RM_BPState, &bp_state))
!= SLURM_SUCCESS) {
error("bridge_get_data(RM_BPState): %s",
bg_err_str(rc));
continue;
}
if (bp_state != RM_BP_UP) /* already down */
continue;
if ((rc = bridge_get_data(my_bp, RM_BPLoc, &bp_loc))
!= SLURM_SUCCESS) {
error("bridge_get_data(RM_BPLoc): %s",
bg_err_str(rc));
continue;
}
/* make sure we have this midplane in the system */
if (bp_loc.X >= DIM_SIZE[X]
|| bp_loc.Y >= DIM_SIZE[Y]
|| bp_loc.Z >= DIM_SIZE[Z]) {
debug4("node %s%c%c%c isn't configured",
bg_conf->slurm_node_prefix,
alpha_num[bp_loc.X], alpha_num[bp_loc.Y],
alpha_num[bp_loc.Z]);
continue;
}
snprintf(bg_down_node, sizeof(bg_down_node), "%s%c%c%c",
bg_conf->slurm_node_prefix,
alpha_num[bp_loc.X], alpha_num[bp_loc.Y],
alpha_num[bp_loc.Z]);
if (node_already_down(bg_down_node))
break;
error("switch for node %s is bad", bg_down_node);
slurm_drain_nodes(bg_down_node,
"select_bluegene: MMCS switch not UP",
slurm_get_slurm_user_id());
break;
}
}
/*
* This could potentially lock the node lock in the slurmctld with
* slurm_drain_node, or slurm_fail_job so if slurmctld_locked is called we
* will call the functions without locking the locks again.
*/
extern int down_nodecard(char *mp_name, bitoff_t io_start,
bool slurmctld_locked)
{
List requests = NULL;
List delete_list = NULL;
ListIterator itr = NULL;
bg_record_t *bg_record = NULL, *found_record = NULL, tmp_record;
bg_record_t *smallest_bg_record = NULL;
struct node_record *node_ptr = NULL;
int mp_bit = 0;
static int io_cnt = NO_VAL;
static int create_size = NO_VAL;
static select_ba_request_t blockreq;
int rc = SLURM_SUCCESS;
char *reason = "select_bluegene: nodecard down";
xassert(mp_name);
if (io_cnt == NO_VAL) {
io_cnt = 1;
/* Translate 1 nodecard count to ionode count */
if ((io_cnt *= bg_conf->io_ratio))
io_cnt--;
/* make sure we create something that is able to be
created */
if (bg_conf->smallest_block < bg_conf->nodecard_cnode_cnt)
create_size = bg_conf->nodecard_cnode_cnt;
else
create_size = bg_conf->smallest_block;
}
node_ptr = find_node_record(mp_name);
if (!node_ptr) {
error ("down_sub_node_blocks: invalid node specified '%s'",
mp_name);
return EINVAL;
}
/* this is here for sanity check to make sure we don't core on
these bits when we set them below. */
if (io_start >= bg_conf->ionodes_per_mp
|| (io_start+io_cnt) >= bg_conf->ionodes_per_mp) {
debug("io %d-%d not configured on this "
"system, only %d ionodes per midplane",
io_start, io_start+io_cnt, bg_conf->ionodes_per_mp);
return EINVAL;
}
mp_bit = (node_ptr - node_record_table_ptr);
memset(&blockreq, 0, sizeof(select_ba_request_t));
blockreq.conn_type[0] = SELECT_SMALL;
blockreq.save_name = mp_name;
debug3("here setting node %d of %d and ionodes %d-%d of %d",
mp_bit, node_record_count, io_start,
io_start+io_cnt, bg_conf->ionodes_per_mp);
memset(&tmp_record, 0, sizeof(bg_record_t));
tmp_record.mp_count = 1;
tmp_record.cnode_cnt = bg_conf->nodecard_cnode_cnt;
tmp_record.mp_bitmap = bit_alloc(node_record_count);
bit_set(tmp_record.mp_bitmap, mp_bit);
tmp_record.ionode_bitmap = bit_alloc(bg_conf->ionodes_per_mp);
bit_nset(tmp_record.ionode_bitmap, io_start, io_start+io_cnt);
slurm_mutex_lock(&block_state_mutex);
itr = list_iterator_create(bg_lists->main);
while ((bg_record = list_next(itr))) {
if (!bit_test(bg_record->mp_bitmap, mp_bit))
continue;
if (!blocks_overlap(bg_record, &tmp_record))
continue;
if (bg_record->job_running > NO_JOB_RUNNING) {
if (slurmctld_locked)
job_fail(bg_record->job_running);
else
slurm_fail_job(bg_record->job_running);
}
/* If Running Dynamic mode and the block is
smaller than the create size just continue on.
*/
if ((bg_conf->layout_mode == LAYOUT_DYNAMIC)
&& (bg_record->cnode_cnt < create_size)) {
if (!delete_list)
delete_list = list_create(NULL);
list_append(delete_list, bg_record);
continue;
}
/* keep track of the smallest size that is at least
the size of create_size. */
if (!smallest_bg_record ||
(smallest_bg_record->cnode_cnt > bg_record->cnode_cnt))
smallest_bg_record = bg_record;
}
list_iterator_destroy(itr);
slurm_mutex_unlock(&block_state_mutex);
if (bg_conf->layout_mode != LAYOUT_DYNAMIC) {
debug3("running non-dynamic mode");
/* This should never happen, but just in case... */
if (delete_list)
list_destroy(delete_list);
/* If we found a block that is smaller or equal to a
midplane we will just mark it in an error state as
opposed to draining the node.
*/
if (smallest_bg_record
&& (smallest_bg_record->cnode_cnt < bg_conf->mp_cnode_cnt)){
if (smallest_bg_record->state & BG_BLOCK_ERROR_FLAG) {
rc = SLURM_NO_CHANGE_IN_DATA;
goto cleanup;
}
rc = put_block_in_error_state(
smallest_bg_record, reason);
goto cleanup;
}
debug("No block under 1 midplane available for this nodecard. "
"Draining the whole node.");
if (!node_already_down(mp_name)) {
if (slurmctld_locked)
drain_nodes(mp_name, reason,
slurm_get_slurm_user_id());
else
slurm_drain_nodes(mp_name, reason,
slurm_get_slurm_user_id());
}
rc = SLURM_SUCCESS;
goto cleanup;
}
/* below is only for Dynamic mode */
if (delete_list) {
int cnt_set = 0;
bitstr_t *iobitmap = bit_alloc(bg_conf->ionodes_per_mp);
/* don't lock here since it is handled inside
the put_block_in_error_state
*/
itr = list_iterator_create(delete_list);
while ((bg_record = list_next(itr))) {
debug2("combining smaller than nodecard "
"dynamic block %s",
bg_record->bg_block_id);
while (bg_record->job_running > NO_JOB_RUNNING)
sleep(1);
bit_or(iobitmap, bg_record->ionode_bitmap);
cnt_set++;
}
list_iterator_destroy(itr);
list_destroy(delete_list);
if (!cnt_set) {
FREE_NULL_BITMAP(iobitmap);
rc = SLURM_ERROR;
goto cleanup;
}
/* set the start to be the same as the start of the
ionode_bitmap. If no ionodes set (not a small
block) set io_start = 0. */
if ((io_start = bit_ffs(iobitmap)) == -1) {
io_start = 0;
if (create_size > bg_conf->nodecard_cnode_cnt)
blockreq.small128 = 4;
else
blockreq.small32 = 16;
} else if (create_size <= bg_conf->nodecard_cnode_cnt)
blockreq.small32 = 1;
else
/* this should never happen */
blockreq.small128 = 1;
FREE_NULL_BITMAP(iobitmap);
} else if (smallest_bg_record) {
debug2("smallest dynamic block is %s",
smallest_bg_record->bg_block_id);
if (smallest_bg_record->state & BG_BLOCK_ERROR_FLAG) {
rc = SLURM_NO_CHANGE_IN_DATA;
goto cleanup;
}
while (smallest_bg_record->job_running > NO_JOB_RUNNING)
sleep(1);
if (smallest_bg_record->cnode_cnt == create_size) {
rc = put_block_in_error_state(
smallest_bg_record, reason);
goto cleanup;
}
if (create_size > smallest_bg_record->cnode_cnt) {
/* we should never get here. This means we
* have a create_size that is bigger than a
* block that is already made.
*/
rc = put_block_in_error_state(
smallest_bg_record, reason);
goto cleanup;
}
debug3("node count is %d", smallest_bg_record->cnode_cnt);
switch(smallest_bg_record->cnode_cnt) {
#ifndef HAVE_BGL
case 64:
blockreq.small32 = 2;
break;
case 256:
blockreq.small32 = 8;
break;
#endif
case 128:
blockreq.small32 = 4;
break;
case 512:
default:
blockreq.small32 = 16;
break;
}
if (create_size != bg_conf->nodecard_cnode_cnt) {
blockreq.small128 = blockreq.small32 / 4;
blockreq.small32 = 0;
io_start = 0;
} else if ((io_start =
bit_ffs(smallest_bg_record->ionode_bitmap)) == -1)
/* set the start to be the same as the start of the
ionode_bitmap. If no ionodes set (not a small
block) set io_start = 0. */
io_start = 0;
} else {
switch(create_size) {
#ifndef HAVE_BGL
case 64:
blockreq.small64 = 8;
break;
case 256:
blockreq.small256 = 2;
#endif
case 32:
blockreq.small32 = 16;
break;
case 128:
blockreq.small128 = 4;
break;
case 512:
if (!node_already_down(mp_name)) {
char *reason = "select_bluegene: nodecard down";
if (slurmctld_locked)
drain_nodes(mp_name, reason,
slurm_get_slurm_user_id());
else
slurm_drain_nodes(
mp_name, reason,
slurm_get_slurm_user_id());
}
rc = SLURM_SUCCESS;
goto cleanup;
break;
default:
error("Unknown create size of %d", create_size);
break;
}
/* since we don't have a block in this midplane
we need to start at the beginning. */
io_start = 0;
/* we also need a bg_block to pretend to be the
smallest block that takes up the entire midplane. */
}
/* Here we need to add blocks that take up nodecards on this
midplane. Since Slurm only keeps track of midplanes
natively this is the only want to handle this case.
*/
requests = list_create(destroy_bg_record);
add_bg_record(requests, NULL, &blockreq, 1, io_start);
slurm_mutex_lock(&block_state_mutex);
delete_list = list_create(NULL);
while ((bg_record = list_pop(requests))) {
itr = list_iterator_create(bg_lists->main);
while ((found_record = list_next(itr))) {
if (!blocks_overlap(bg_record, found_record))
continue;
list_push(delete_list, found_record);
list_remove(itr);
}
list_iterator_destroy(itr);
/* we need to add this record since it doesn't exist */
if (bridge_block_create(bg_record) == SLURM_ERROR) {
destroy_bg_record(bg_record);
error("down_sub_node_blocks: "
"unable to configure block in api");
continue;
}
debug("adding block %s to fill in small blocks "
"around bad nodecards",
bg_record->bg_block_id);
print_bg_record(bg_record);
list_append(bg_lists->main, bg_record);
if (bit_overlap(bg_record->ionode_bitmap,
tmp_record.ionode_bitmap)) {
/* here we know the error block doesn't exist
so just set the state here */
slurm_mutex_unlock(&block_state_mutex);
rc = put_block_in_error_state(bg_record, reason);
slurm_mutex_lock(&block_state_mutex);
}
}
list_destroy(requests);
if (delete_list) {
slurm_mutex_unlock(&block_state_mutex);
free_block_list(NO_VAL, delete_list, 0, 0);
list_destroy(delete_list);
}
slurm_mutex_lock(&block_state_mutex);
sort_bg_record_inc_size(bg_lists->main);
slurm_mutex_unlock(&block_state_mutex);
last_bg_update = time(NULL);
cleanup:
FREE_NULL_BITMAP(tmp_record.mp_bitmap);
FREE_NULL_BITMAP(tmp_record.ionode_bitmap);
return rc;
}
