extern int *slurmdb_setup_cluster_dim_size(void)
{
if (working_cluster_rec)
return working_cluster_rec->dim_size;
return select_g_ba_get_dims();
}
static void _addto_mp_list(bg_record_t *bg_record,
uint16_t *start, uint16_t *end)
{
uint16_t coords[SYSTEM_DIMENSIONS];
int dim;
static int *cluster_dims = NULL;
static char start_char[SYSTEM_DIMENSIONS+1],
end_char[SYSTEM_DIMENSIONS+1],
dim_char[SYSTEM_DIMENSIONS+1];
if (!cluster_dims) {
/* do some setup that only needs to happen once. */
cluster_dims = select_g_ba_get_dims();
memset(start_char, 0, sizeof(start_char));
memset(end_char, 0, sizeof(end_char));
memset(dim_char, 0, sizeof(dim_char));
for (dim = 0; dim<SYSTEM_DIMENSIONS; dim++)
dim_char[dim] = alpha_num[cluster_dims[dim]];
}
for (dim = 0; dim<SYSTEM_DIMENSIONS; dim++) {
if ((int16_t)start[dim] < 0) {
for (dim = 0; dim<SYSTEM_DIMENSIONS; dim++)
start_char[dim] = alpha_num[start[dim]];
fatal("bluegene.conf starting coordinate "
"is invalid: %s",
start_char);
}
if (end[dim] >= cluster_dims[dim]) {
for (dim = 0; dim<SYSTEM_DIMENSIONS; dim++) {
start_char[dim] = alpha_num[start[dim]];
end_char[dim] = alpha_num[end[dim]];
dim_char[dim] = alpha_num[cluster_dims[dim]];
}
fatal("bluegene.conf matrix size exceeds space "
"defined in "
"slurm.conf %sx%s => %s",
start_char, end_char, dim_char);
}
}
if (bg_conf->slurm_debug_level >= LOG_LEVEL_DEBUG3) {
for (dim = 0; dim<SYSTEM_DIMENSIONS; dim++) {
start_char[dim] = alpha_num[start[dim]];
end_char[dim] = alpha_num[end[dim]];
}
debug3("adding mps: %sx%s", start_char, end_char);
debug3("slurm.conf: %s", dim_char);
}
_append_ba_mps(bg_record->ba_mp_list, 0, start, end, coords);
}
/*
* Read and process the bluegene.conf configuration file so to interpret what
* blocks are static/dynamic, torus/mesh, etc.
*/
extern int read_bg_conf(void)
{
int i;
int count = 0;
s_p_hashtbl_t *tbl = NULL;
char *layout = NULL;
select_ba_request_t **blockreq_array = NULL;
image_t **image_array = NULL;
image_t *image = NULL;
static time_t last_config_update = (time_t) 0;
struct stat config_stat;
ListIterator itr = NULL;
char* bg_conf_file = NULL;
static int *dims = NULL;
if (!dims)
dims = select_g_ba_get_dims();
if (bg_conf->slurm_debug_flags & DEBUG_FLAG_SELECT_TYPE)
info("Reading the bluegene.conf file");
/* check if config file has changed */
bg_conf_file = _get_bg_conf();
if (stat(bg_conf_file, &config_stat) < 0)
fatal("can't stat bluegene.conf file %s: %m", bg_conf_file);
if (last_config_update) {
_reopen_bridge_log();
if (last_config_update == config_stat.st_mtime) {
if (bg_conf->slurm_debug_flags & DEBUG_FLAG_SELECT_TYPE)
info("%s unchanged", bg_conf_file);
} else {
info("Restart slurmctld for %s changes "
"to take effect",
bg_conf_file);
}
last_config_update = config_stat.st_mtime;
xfree(bg_conf_file);
return SLURM_SUCCESS;
}
last_config_update = config_stat.st_mtime;
/* initialization */
/* bg_conf defined in bg_node_alloc.h */
tbl = s_p_hashtbl_create(bg_conf_file_options);
if (s_p_parse_file(tbl, NULL, bg_conf_file, false) == SLURM_ERROR)
fatal("something wrong with opening/reading bluegene "
"conf file");
xfree(bg_conf_file);
#ifdef HAVE_BGL
if (s_p_get_array((void ***)&image_array,
&count, "AltBlrtsImage", tbl)) {
for (i = 0; i < count; i++) {
list_append(bg_conf->blrts_list, image_array[i]);
image_array[i] = NULL;
}
}
if (!s_p_get_string(&bg_conf->default_blrtsimage, "BlrtsImage", tbl)) {
if (!list_count(bg_conf->blrts_list))
fatal("BlrtsImage not configured "
"in bluegene.conf");
itr = list_iterator_create(bg_conf->blrts_list);
image = list_next(itr);
image->def = true;
list_iterator_destroy(itr);
bg_conf->default_blrtsimage = xstrdup(image->name);
info("Warning: using %s as the default BlrtsImage. "
"If this isn't correct please set BlrtsImage",
bg_conf->default_blrtsimage);
} else {
if (bg_conf->slurm_debug_flags & DEBUG_FLAG_SELECT_TYPE)
info("default BlrtsImage %s",
bg_conf->default_blrtsimage);
image = xmalloc(sizeof(image_t));
image->name = xstrdup(bg_conf->default_blrtsimage);
image->def = true;
image->groups = NULL;
/* we want it to be first */
list_push(bg_conf->blrts_list, image);
}
if (s_p_get_array((void ***)&image_array,
&count, "AltLinuxImage", tbl)) {
for (i = 0; i < count; i++) {
list_append(bg_conf->linux_list, image_array[i]);
image_array[i] = NULL;
}
}
if (!s_p_get_string(&bg_conf->default_linuximage, "LinuxImage", tbl)) {
if (!list_count(bg_conf->linux_list))
fatal("LinuxImage not configured "
"in bluegene.conf");
itr = list_iterator_create(bg_conf->linux_list);
image = list_next(itr);
image->def = true;
list_iterator_destroy(itr);
bg_conf->default_linuximage = xstrdup(image->name);
info("Warning: using %s as the default LinuxImage. "
"If this isn't correct please set LinuxImage",
bg_conf->default_linuximage);
} else {
if (bg_conf->slurm_debug_flags & DEBUG_FLAG_SELECT_TYPE)
info("default LinuxImage %s",
bg_conf->default_linuximage);
image = xmalloc(sizeof(image_t));
image->name = xstrdup(bg_conf->default_linuximage);
image->def = true;
image->groups = NULL;
/* we want it to be first */
list_push(bg_conf->linux_list, image);
}
if (s_p_get_array((void ***)&image_array,
&count, "AltRamDiskImage", tbl)) {
for (i = 0; i < count; i++) {
list_append(bg_conf->ramdisk_list, image_array[i]);
image_array[i] = NULL;
}
}
if (!s_p_get_string(&bg_conf->default_ramdiskimage,
"RamDiskImage", tbl)) {
if (!list_count(bg_conf->ramdisk_list))
fatal("RamDiskImage not configured "
"in bluegene.conf");
itr = list_iterator_create(bg_conf->ramdisk_list);
image = list_next(itr);
image->def = true;
list_iterator_destroy(itr);
bg_conf->default_ramdiskimage = xstrdup(image->name);
info("Warning: using %s as the default RamDiskImage. "
"If this isn't correct please set RamDiskImage",
bg_conf->default_ramdiskimage);
} else {
if (bg_conf->slurm_debug_flags & DEBUG_FLAG_SELECT_TYPE)
info("default RamDiskImage %s",
bg_conf->default_ramdiskimage);
image = xmalloc(sizeof(image_t));
image->name = xstrdup(bg_conf->default_ramdiskimage);
image->def = true;
image->groups = NULL;
/* we want it to be first */
list_push(bg_conf->ramdisk_list, image);
}
#elif defined HAVE_BGP
if (s_p_get_array((void ***)&image_array,
&count, "AltCnloadImage", tbl)) {
for (i = 0; i < count; i++) {
list_append(bg_conf->linux_list, image_array[i]);
image_array[i] = NULL;
}
}
if (!s_p_get_string(&bg_conf->default_linuximage, "CnloadImage", tbl)) {
if (!list_count(bg_conf->linux_list))
fatal("CnloadImage not configured "
"in bluegene.conf");
itr = list_iterator_create(bg_conf->linux_list);
image = list_next(itr);
image->def = true;
list_iterator_destroy(itr);
bg_conf->default_linuximage = xstrdup(image->name);
info("Warning: using %s as the default CnloadImage. "
"If this isn't correct please set CnloadImage",
bg_conf->default_linuximage);
} else {
if (bg_conf->slurm_debug_flags & DEBUG_FLAG_SELECT_TYPE)
info("default CnloadImage %s",
bg_conf->default_linuximage);
image = xmalloc(sizeof(image_t));
image->name = xstrdup(bg_conf->default_linuximage);
image->def = true;
image->groups = NULL;
/* we want it to be first */
list_push(bg_conf->linux_list, image);
}
if (s_p_get_array((void ***)&image_array,
&count, "AltIoloadImage", tbl)) {
for (i = 0; i < count; i++) {
list_append(bg_conf->ramdisk_list, image_array[i]);
image_array[i] = NULL;
}
}
if (!s_p_get_string(&bg_conf->default_ramdiskimage,
"IoloadImage", tbl)) {
if (!list_count(bg_conf->ramdisk_list))
fatal("IoloadImage not configured "
"in bluegene.conf");
itr = list_iterator_create(bg_conf->ramdisk_list);
image = list_next(itr);
image->def = true;
list_iterator_destroy(itr);
bg_conf->default_ramdiskimage = xstrdup(image->name);
info("Warning: using %s as the default IoloadImage. "
"If this isn't correct please set IoloadImage",
bg_conf->default_ramdiskimage);
} else {
if (bg_conf->slurm_debug_flags & DEBUG_FLAG_SELECT_TYPE)
info("default IoloadImage %s",
bg_conf->default_ramdiskimage);
image = xmalloc(sizeof(image_t));
image->name = xstrdup(bg_conf->default_ramdiskimage);
image->def = true;
image->groups = NULL;
/* we want it to be first */
list_push(bg_conf->ramdisk_list, image);
}
#endif
if (s_p_get_array((void ***)&image_array,
&count, "AltMloaderImage", tbl)) {
for (i = 0; i < count; i++) {
list_append(bg_conf->mloader_list, image_array[i]);
image_array[i] = NULL;
}
}
if (!s_p_get_string(&bg_conf->default_mloaderimage,
"MloaderImage", tbl)) {
if (!list_count(bg_conf->mloader_list))
fatal("MloaderImage not configured "
"in bluegene.conf");
itr = list_iterator_create(bg_conf->mloader_list);
image = list_next(itr);
image->def = true;
list_iterator_destroy(itr);
bg_conf->default_mloaderimage = xstrdup(image->name);
info("Warning: using %s as the default MloaderImage. "
"If this isn't correct please set MloaderImage",
bg_conf->default_mloaderimage);
} else {
if (bg_conf->slurm_debug_flags & DEBUG_FLAG_SELECT_TYPE)
info("default MloaderImage %s",
bg_conf->default_mloaderimage);
image = xmalloc(sizeof(image_t));
image->name = xstrdup(bg_conf->default_mloaderimage);
image->def = true;
image->groups = NULL;
/* we want it to be first */
list_push(bg_conf->mloader_list, image);
}
if (!s_p_get_uint16(
&bg_conf->mp_cnode_cnt, "BasePartitionNodeCnt", tbl)) {
error("BasePartitionNodeCnt not configured in bluegene.conf "
"defaulting to 512 as BasePartitionNodeCnt");
bg_conf->mp_cnode_cnt = 512;
bg_conf->quarter_cnode_cnt = 128;
} else {
if (bg_conf->mp_cnode_cnt <= 0)
fatal("You should have more than 0 nodes "
"per base partition");
bg_conf->quarter_cnode_cnt = bg_conf->mp_cnode_cnt/4;
}
/* bg_conf->cpus_per_mp should had already been set from the
* node_init */
if (bg_conf->cpus_per_mp < bg_conf->mp_cnode_cnt) {
fatal("For some reason we have only %u cpus per mp, but "
"have %u cnodes per mp. You need at least the same "
"number of cpus as you have cnodes per mp. "
"Check the NodeName Procs= "
"definition in the slurm.conf.",
bg_conf->cpus_per_mp, bg_conf->mp_cnode_cnt);
}
bg_conf->cpu_ratio = bg_conf->cpus_per_mp/bg_conf->mp_cnode_cnt;
if (!bg_conf->cpu_ratio)
fatal("We appear to have less than 1 cpu on a cnode. "
"You specified %u for BasePartitionNodeCnt "
"in the blugene.conf and %u cpus "
"for each node in the slurm.conf",
bg_conf->mp_cnode_cnt, bg_conf->cpus_per_mp);
num_unused_cpus = 1;
for (i = 0; i<SYSTEM_DIMENSIONS; i++)
num_unused_cpus *= dims[i];
num_unused_cpus *= bg_conf->cpus_per_mp;
if (!s_p_get_uint16(
&bg_conf->nodecard_cnode_cnt, "NodeCardNodeCnt", tbl)) {
error("NodeCardNodeCnt not configured in bluegene.conf "
"defaulting to 32 as NodeCardNodeCnt");
bg_conf->nodecard_cnode_cnt = 32;
}
if (bg_conf->nodecard_cnode_cnt<=0)
fatal("You should have more than 0 nodes per nodecard");
bg_conf->mp_nodecard_cnt =
bg_conf->mp_cnode_cnt / bg_conf->nodecard_cnode_cnt;
if (!s_p_get_uint16(&bg_conf->ionodes_per_mp, "Numpsets", tbl))
fatal("Warning: Numpsets not configured in bluegene.conf");
if (!bg_conf->ionodes_per_mp) {
if (!s_p_get_uint16(&bg_conf->ionodes_per_mp,
"IONodesPerMP", tbl))
fatal("Warning: IONodesPerMP not configured "
"in bluegene.conf");
}
#ifdef HAVE_BGQ
/* You can only have 16 ionodes per midplane */
if (bg_conf->ionodes_per_mp > bg_conf->mp_nodecard_cnt)
bg_conf->ionodes_per_mp = bg_conf->mp_nodecard_cnt;
#endif
if (bg_conf->ionodes_per_mp) {
bitstr_t *tmp_bitmap = NULL;
int small_size = 1;
/* THIS IS A HACK TO MAKE A 1 NODECARD SYSTEM WORK */
if (bg_conf->mp_cnode_cnt == bg_conf->nodecard_cnode_cnt) {
bg_conf->quarter_ionode_cnt = 2;
bg_conf->nodecard_ionode_cnt = 2;
} else {
bg_conf->quarter_ionode_cnt = bg_conf->ionodes_per_mp/4;
bg_conf->nodecard_ionode_cnt =
bg_conf->quarter_ionode_cnt/4;
}
/* How many nodecards per ionode */
bg_conf->nc_ratio =
((double)bg_conf->mp_cnode_cnt
/ (double)bg_conf->nodecard_cnode_cnt)
/ (double)bg_conf->ionodes_per_mp;
/* How many ionodes per nodecard */
bg_conf->io_ratio =
(double)bg_conf->ionodes_per_mp /
((double)bg_conf->mp_cnode_cnt
/ (double)bg_conf->nodecard_cnode_cnt);
//info("got %f %f", bg_conf->nc_ratio, bg_conf->io_ratio);
/* figure out the smallest block we can have on the
system */
#ifdef HAVE_BGL
if (bg_conf->io_ratio >= 1)
bg_conf->smallest_block=32;
else
bg_conf->smallest_block=128;
#else
if (bg_conf->io_ratio >= 2)
bg_conf->smallest_block=16;
else if (bg_conf->io_ratio == 1)
bg_conf->smallest_block=32;
else if (bg_conf->io_ratio == .5)
bg_conf->smallest_block=64;
else if (bg_conf->io_ratio == .25)
bg_conf->smallest_block=128;
else if (bg_conf->io_ratio == .125)
bg_conf->smallest_block=256;
else {
error("unknown ioratio %f. Can't figure out "
"smallest block size, setting it to midplane",
bg_conf->io_ratio);
bg_conf->smallest_block = 512;
}
#endif
if (bg_conf->slurm_debug_flags & DEBUG_FLAG_SELECT_TYPE)
info("Smallest block possible on this system is %u",
bg_conf->smallest_block);
/* below we are creating all the possible bitmaps for
* each size of small block
*/
if ((int)bg_conf->nodecard_ionode_cnt < 1) {
bg_conf->nodecard_ionode_cnt = 0;
} else {
bg_lists->valid_small32 = list_create(_destroy_bitmap);
if ((small_size = bg_conf->nodecard_ionode_cnt))
small_size--;
i = 0;
while (i<bg_conf->ionodes_per_mp) {
tmp_bitmap = bit_alloc(bg_conf->ionodes_per_mp);
bit_nset(tmp_bitmap, i, i+small_size);
i += small_size+1;
list_append(bg_lists->valid_small32,
tmp_bitmap);
}
}
/* If we only have 1 nodecard just jump to the end
since this will never need to happen below.
Pretty much a hack to avoid seg fault;). */
if (bg_conf->mp_cnode_cnt == bg_conf->nodecard_cnode_cnt)
goto no_calc;
bg_lists->valid_small128 = list_create(_destroy_bitmap);
if ((small_size = bg_conf->quarter_ionode_cnt))
small_size--;
i = 0;
while (i<bg_conf->ionodes_per_mp) {
tmp_bitmap = bit_alloc(bg_conf->ionodes_per_mp);
bit_nset(tmp_bitmap, i, i+small_size);
i += small_size+1;
list_append(bg_lists->valid_small128, tmp_bitmap);
}
#ifndef HAVE_BGL
bg_lists->valid_small64 = list_create(_destroy_bitmap);
if ((small_size = bg_conf->nodecard_ionode_cnt * 2))
small_size--;
i = 0;
while (i<bg_conf->ionodes_per_mp) {
tmp_bitmap = bit_alloc(bg_conf->ionodes_per_mp);
bit_nset(tmp_bitmap, i, i+small_size);
i += small_size+1;
list_append(bg_lists->valid_small64, tmp_bitmap);
}
bg_lists->valid_small256 = list_create(_destroy_bitmap);
if ((small_size = bg_conf->quarter_ionode_cnt * 2))
small_size--;
i = 0;
while (i<bg_conf->ionodes_per_mp) {
tmp_bitmap = bit_alloc(bg_conf->ionodes_per_mp);
bit_nset(tmp_bitmap, i, i+small_size);
i += small_size+1;
list_append(bg_lists->valid_small256, tmp_bitmap);
}
#endif
} else {
fatal("your ionodes_per_mp is 0");
}
no_calc:
if (!s_p_get_uint16(&bg_conf->bridge_api_verb, "BridgeAPIVerbose", tbl))
info("Warning: BridgeAPIVerbose not configured "
"in bluegene.conf");
if (!s_p_get_string(&bg_conf->bridge_api_file,
"BridgeAPILogFile", tbl))
info("BridgeAPILogFile not configured in bluegene.conf");
else
_reopen_bridge_log();
if (s_p_get_string(&layout, "DenyPassthrough", tbl)) {
if (strstr(layout, "A"))
ba_deny_pass |= PASS_DENY_A;
if (strstr(layout, "X"))
ba_deny_pass |= PASS_DENY_X;
if (strstr(layout, "Y"))
ba_deny_pass |= PASS_DENY_Y;
if (strstr(layout, "Z"))
ba_deny_pass |= PASS_DENY_Z;
if (!strcasecmp(layout, "ALL"))
ba_deny_pass |= PASS_DENY_ALL;
bg_conf->deny_pass = ba_deny_pass;
xfree(layout);
}
if (!s_p_get_string(&layout, "LayoutMode", tbl)) {
info("Warning: LayoutMode was not specified in bluegene.conf "
"defaulting to STATIC partitioning");
bg_conf->layout_mode = LAYOUT_STATIC;
} else {
if (!strcasecmp(layout,"STATIC"))
bg_conf->layout_mode = LAYOUT_STATIC;
else if (!strcasecmp(layout,"OVERLAP"))
bg_conf->layout_mode = LAYOUT_OVERLAP;
else if (!strcasecmp(layout,"DYNAMIC"))
bg_conf->layout_mode = LAYOUT_DYNAMIC;
else {
fatal("I don't understand this LayoutMode = %s",
layout);
}
xfree(layout);
}
/* add blocks defined in file */
if (bg_conf->layout_mode != LAYOUT_DYNAMIC) {
if (!s_p_get_array((void ***)&blockreq_array,
&count, "BPs", tbl)) {
info("WARNING: no blocks defined in bluegene.conf, "
"only making full system block");
/* create_full_system_block(NULL); */
}
for (i = 0; i < count; i++) {
add_bg_record(bg_lists->main, NULL,
blockreq_array[i], 0, 0);
}
}
s_p_hashtbl_destroy(tbl);
return SLURM_SUCCESS;
}
/*
* finds the best match for a given job request
*
*
* OUT - block_id of matched block, NULL otherwise
* returns 1 for error (no match)
*
*/
static int _find_best_block_match(List block_list,
int *blocks_added,
struct job_record* job_ptr,
bitstr_t* slurm_block_bitmap,
uint32_t min_nodes, uint32_t max_nodes,
uint32_t req_nodes,
bg_record_t** found_bg_record,
uint16_t query_mode, int avail_cpus)
{
bg_record_t *bg_record = NULL;
uint16_t req_geometry[SYSTEM_DIMENSIONS];
uint16_t target_size = 0;
uint32_t req_procs = job_ptr->details->min_cpus;
select_ba_request_t request;
int i, dim;
int overlap_check = 0;
int allow = 0;
int check_image = 1;
uint32_t max_cpus = job_ptr->details->max_cpus;
char tmp_char[256];
static int total_cpus = 0;
int rc = SLURM_SUCCESS;
int create_try = 0;
List overlapped_list = NULL;
bool is_test = SELECT_IS_TEST(query_mode);
if (!total_cpus) {
int *cluster_dims = select_g_ba_get_dims();
total_cpus = 1;
for (dim=0; dim<SYSTEM_DIMENSIONS; dim++)
total_cpus *= cluster_dims[dim];
total_cpus *= bg_conf->cpus_per_mp;
}
if (req_nodes > max_nodes) {
error("can't run this job max mps is %u asking for %u",
max_nodes, req_nodes);
return SLURM_ERROR;
}
if (!is_test && (req_procs > avail_cpus)) {
if (bg_conf->slurm_debug_flags & DEBUG_FLAG_BG_PICK)
info("asking for %u I only have %d",
req_procs, avail_cpus);
return SLURM_ERROR;
}
if (!block_list) {
error("_find_best_block_match: There is no block_list");
return SLURM_ERROR;
}
memset(&request, 0, sizeof(select_ba_request_t));
get_select_jobinfo(job_ptr->select_jobinfo->data,
SELECT_JOBDATA_CONN_TYPE, &request.conn_type);
get_select_jobinfo(job_ptr->select_jobinfo->data,
SELECT_JOBDATA_GEOMETRY, &req_geometry);
get_select_jobinfo(job_ptr->select_jobinfo->data,
SELECT_JOBDATA_ROTATE, &request.rotate);
if ((rc = _check_images(job_ptr, &request)) == SLURM_ERROR)
goto end_it;
if (req_geometry[0] != 0 && req_geometry[0] != (uint16_t)NO_VAL) {
char tmp_geo[SYSTEM_DIMENSIONS+1];
target_size = 1;
for (i=0; i<SYSTEM_DIMENSIONS; i++) {
target_size *= req_geometry[i];
tmp_geo[i] = alpha_num[req_geometry[i]];
}
tmp_geo[i] = '\0';
if (target_size != min_nodes) {
debug2("min_nodes not set correctly %u "
"should be %u from %s",
min_nodes, target_size,
tmp_geo);
min_nodes = target_size;
}
if (!req_nodes)
req_nodes = min_nodes;
} else {
req_geometry[0] = (uint16_t)NO_VAL;
target_size = min_nodes;
}
*found_bg_record = NULL;
allow = 0;
memcpy(request.geometry, req_geometry, sizeof(req_geometry));
request.deny_pass = (uint16_t)NO_VAL;
request.save_name = NULL;
request.size = target_size;
request.procs = req_procs;
request.elongate = request.rotate;
/* request.start[0] = 1; */
/* request.start[1] = 2; */
/* request.start[2] = 0; */
/* request.start[3] = 2; */
/* request.start_req = 1; */
if (job_ptr->details->req_node_bitmap)
request.avail_mp_bitmap = job_ptr->details->req_node_bitmap;
else
request.avail_mp_bitmap = slurm_block_bitmap;
/* since we only look at procs after this and not nodes we
* need to set a max_cpus if given
*/
if (max_cpus == (uint32_t)NO_VAL)
max_cpus = max_nodes * bg_conf->cpus_per_mp;
while (1) {
/* Here we are creating a list of all the blocks that
* have overlapped jobs so if we don't find one that
* works we will have can look and see the earliest
* the job can start. This doesn't apply to Dynamic mode.
*/
if (is_test && SELECT_IS_CHECK_FULL_SET(query_mode)
&& bg_conf->layout_mode != LAYOUT_DYNAMIC)
overlapped_list = list_create(NULL);
bg_record = _find_matching_block(block_list,
job_ptr,
slurm_block_bitmap,
&request,
max_cpus,
&allow, check_image,
overlap_check,
overlapped_list,
query_mode);
/* this could get altered in _find_matching_block so we
need to reset it */
memcpy(request.geometry, req_geometry, sizeof(req_geometry));
if (!bg_record && overlapped_list
&& list_count(overlapped_list)) {
ListIterator itr =
list_iterator_create(overlapped_list);
bg_record_t *tmp_rec = NULL;
while ((tmp_rec = list_next(itr))) {
if (!bg_record ||
(tmp_rec->job_ptr->end_time <
bg_record->job_ptr->end_time))
bg_record = tmp_rec;
}
list_iterator_destroy(itr);
}
if (overlapped_list)
list_destroy(overlapped_list);
/* set the bitmap and do other allocation activities */
if (bg_record) {
#ifdef HAVE_BG_L_P
if (!is_test) {
if (bridge_block_check_mp_states(
bg_record->bg_block_id, 1)
!= SLURM_SUCCESS) {
/* check_block_mp_states will
set this block in the main
list to an error state, but
we aren't looking
at the main list, so we
need to set this copy of
the block in an
error state as well.
*/
bg_record->job_running =
BLOCK_ERROR_STATE;
bg_record->state |= BG_BLOCK_ERROR_FLAG;
error("_find_best_block_match: Picked "
"block (%s) had some issues with "
"hardware, trying a different "
"one.",
bg_record->bg_block_id);
continue;
}
}
#endif
format_node_name(bg_record, tmp_char, sizeof(tmp_char));
debug("_find_best_block_match %s <%s>",
bg_record->bg_block_id, tmp_char);
bit_and(slurm_block_bitmap, bg_record->mp_bitmap);
rc = SLURM_SUCCESS;
*found_bg_record = bg_record;
goto end_it;
}
/* see if we can just reset the image and reboot the block */
if (allow) {
check_image = 0;
allow = 0;
continue;
}
check_image = 1;
/* all these assume that the *bg_record is NULL */
if (bg_conf->layout_mode == LAYOUT_OVERLAP
&& !is_test && overlap_check < 2) {
overlap_check++;
continue;
}
if (create_try || bg_conf->layout_mode != LAYOUT_DYNAMIC)
goto no_match;
if ((rc = _dynamically_request(block_list, blocks_added,
&request,
job_ptr->details->req_nodes,
query_mode))
== SLURM_SUCCESS) {
create_try = 1;
continue;
}
/* Only look at the full system if we aren't going to
preempt jobs later and look.
*/
if (is_test && SELECT_IS_CHECK_FULL_SET(query_mode)) {
List new_blocks = NULL;
List job_list = list_create(NULL);
ListIterator itr = NULL;
if (bg_conf->slurm_debug_flags & DEBUG_FLAG_BG_PICK)
info("trying with empty machine");
/* Here we need to make sure the blocks in the
job list are those in the block list so go
through and grab them and add them to a
separate list.
*/
itr = list_iterator_create(block_list);
while ((bg_record = list_next(itr))) {
if (bg_record->job_running != NO_JOB_RUNNING)
list_append(job_list, bg_record);
/* Since the error blocks are at the
end we only really need to look at
the first one to make sure it will
work, so don't add more than one to
the job list.
We do need to check for at least
one error block because that lets
us know not to hold up the entire
machine for a job that won't run
until the error is removed which
could be a very long time.
*/
if (bg_record->job_running == BLOCK_ERROR_STATE)
break;
}
list_iterator_destroy(itr);
/* Block list is already in the correct order,
earliest avaliable first,
so the job list will also be. No need to
sort. */
while (1) {
bool track_down_nodes = true;
if ((bg_record = list_pop(job_list))) {
if (bg_record->job_ptr) {
if (bg_conf->slurm_debug_flags
& DEBUG_FLAG_BG_PICK)
info("taking off "
"%d(%s) started "
"at %ld ends "
"at %ld",
bg_record->
job_running,
bg_record->
bg_block_id,
bg_record->
job_ptr->
start_time,
bg_record->
job_ptr->
end_time);
/* Mark the block as
not running a job,
this should
correspond to the
pointer in the
block_list. We
only look at the
job_running var so
don't remove the
job_ptr.
*/
bg_record->job_running =
NO_JOB_RUNNING;
} else if ((bg_record->job_running
== BLOCK_ERROR_STATE)
&& (bg_conf->
slurm_debug_flags
& DEBUG_FLAG_BG_PICK))
info("taking off (%s) "
"which is in an "
"error state",
bg_record->bg_block_id);
} else
/* This means we didn't have
any jobs to take off
anymore so we are making
sure we can look at every
node on the system.
*/
track_down_nodes = false;
if (!(new_blocks = create_dynamic_block(
block_list, &request, job_list,
track_down_nodes))) {
if (errno == ESLURM_INTERCONNECT_FAILURE
|| !list_count(job_list)) {
char *nodes;
if (slurmctld_conf.
slurmctld_debug < 5)
break;
nodes = bitmap2node_name(
slurm_block_bitmap);
debug("job %u not "
"runable on %s",
job_ptr->job_id,
nodes);
xfree(nodes);
break;
}
continue;
}
rc = SLURM_SUCCESS;
/* outside of the job_test_list this
* gets destroyed later, so don't worry
* about it now
*/
(*found_bg_record) = list_pop(new_blocks);
if (!(*found_bg_record)) {
list_destroy(new_blocks);
if (!bg_record) {
/* This should never happen */
error("got an empty list back");
rc = SLURM_ERROR;
break;
}
if (bg_conf->slurm_debug_flags
& DEBUG_FLAG_BG_PICK)
info("Appears we are trying "
"to place this job on "
"the block we just "
"removed %s.",
bg_record->bg_block_id);
/* This means we placed the job on
the block we just popped off.
*/
bit_and(slurm_block_bitmap,
bg_record->mp_bitmap);
*found_bg_record = bg_record;
break;
}
bit_and(slurm_block_bitmap,
(*found_bg_record)->mp_bitmap);
if (bg_record) {
(*found_bg_record)->job_running =
bg_record->job_running;
(*found_bg_record)->job_ptr
= bg_record->job_ptr;
}
list_destroy(new_blocks);
break;
}
list_destroy(job_list);
goto end_it;
} else {
break;
}
}
no_match:
debug("_find_best_block_match none found");
rc = SLURM_ERROR;
end_it:
xfree(request.blrtsimage);
xfree(request.linuximage);
xfree(request.mloaderimage);
xfree(request.ramdiskimage);
return rc;
}
/*
* Read and process the bluegene.conf configuration file so to interpret what
* blocks are static/dynamic, torus/mesh, etc.
*/
extern int read_bg_conf(void)
{
int i;
bool tmp_bool = 0;
int count = 0;
s_p_hashtbl_t *tbl = NULL;
char *tmp_char = NULL;
select_ba_request_t **blockreq_array = NULL;
image_t **image_array = NULL;
image_t *image = NULL;
static time_t last_config_update = (time_t) 0;
struct stat config_stat;
ListIterator itr = NULL;
char* bg_conf_file = NULL;
static int *dims = NULL;
if (!dims)
dims = select_g_ba_get_dims();
if (bg_conf->slurm_debug_flags & DEBUG_FLAG_SELECT_TYPE)
info("Reading the bluegene.conf file");
/* check if config file has changed */
bg_conf_file = get_extra_conf_path("bluegene.conf");
if (stat(bg_conf_file, &config_stat) < 0)
fatal("can't stat bluegene.conf file %s: %m", bg_conf_file);
if (last_config_update) {
_reopen_bridge_log();
if (last_config_update == config_stat.st_mtime) {
if (bg_conf->slurm_debug_flags & DEBUG_FLAG_SELECT_TYPE)
info("%s unchanged", bg_conf_file);
} else {
info("Restart slurmctld for %s changes "
"to take effect",
bg_conf_file);
}
last_config_update = config_stat.st_mtime;
xfree(bg_conf_file);
return SLURM_SUCCESS;
}
last_config_update = config_stat.st_mtime;
/* initialization */
/* bg_conf defined in bg_node_alloc.h */
if (!(tbl = config_make_tbl(bg_conf_file)))
fatal("something wrong with opening/reading bluegene "
"conf file");
xfree(bg_conf_file);
#ifdef HAVE_BGL
if (s_p_get_array((void ***)&image_array,
&count, "AltBlrtsImage", tbl)) {
for (i = 0; i < count; i++) {
list_append(bg_conf->blrts_list, image_array[i]);
image_array[i] = NULL;
}
}
if (!s_p_get_string(&bg_conf->default_blrtsimage, "BlrtsImage", tbl)) {
if (!list_count(bg_conf->blrts_list))
fatal("BlrtsImage not configured "
"in bluegene.conf");
itr = list_iterator_create(bg_conf->blrts_list);
image = list_next(itr);
image->def = true;
list_iterator_destroy(itr);
bg_conf->default_blrtsimage = xstrdup(image->name);
info("Warning: using %s as the default BlrtsImage. "
"If this isn't correct please set BlrtsImage",
bg_conf->default_blrtsimage);
} else {
if (bg_conf->slurm_debug_flags & DEBUG_FLAG_SELECT_TYPE)
info("default BlrtsImage %s",
bg_conf->default_blrtsimage);
image = xmalloc(sizeof(image_t));
image->name = xstrdup(bg_conf->default_blrtsimage);
image->def = true;
image->groups = NULL;
/* we want it to be first */
list_push(bg_conf->blrts_list, image);
}
if (s_p_get_array((void ***)&image_array,
&count, "AltLinuxImage", tbl)) {
for (i = 0; i < count; i++) {
list_append(bg_conf->linux_list, image_array[i]);
image_array[i] = NULL;
}
}
if (!s_p_get_string(&bg_conf->default_linuximage, "LinuxImage", tbl)) {
if (!list_count(bg_conf->linux_list))
fatal("LinuxImage not configured "
"in bluegene.conf");
itr = list_iterator_create(bg_conf->linux_list);
image = list_next(itr);
image->def = true;
list_iterator_destroy(itr);
bg_conf->default_linuximage = xstrdup(image->name);
info("Warning: using %s as the default LinuxImage. "
"If this isn't correct please set LinuxImage",
bg_conf->default_linuximage);
} else {
if (bg_conf->slurm_debug_flags & DEBUG_FLAG_SELECT_TYPE)
info("default LinuxImage %s",
bg_conf->default_linuximage);
image = xmalloc(sizeof(image_t));
image->name = xstrdup(bg_conf->default_linuximage);
image->def = true;
image->groups = NULL;
/* we want it to be first */
list_push(bg_conf->linux_list, image);
}
if (s_p_get_array((void ***)&image_array,
&count, "AltRamDiskImage", tbl)) {
for (i = 0; i < count; i++) {
list_append(bg_conf->ramdisk_list, image_array[i]);
image_array[i] = NULL;
}
}
if (!s_p_get_string(&bg_conf->default_ramdiskimage,
"RamDiskImage", tbl)) {
if (!list_count(bg_conf->ramdisk_list))
fatal("RamDiskImage not configured "
"in bluegene.conf");
itr = list_iterator_create(bg_conf->ramdisk_list);
image = list_next(itr);
image->def = true;
list_iterator_destroy(itr);
bg_conf->default_ramdiskimage = xstrdup(image->name);
info("Warning: using %s as the default RamDiskImage. "
"If this isn't correct please set RamDiskImage",
bg_conf->default_ramdiskimage);
} else {
if (bg_conf->slurm_debug_flags & DEBUG_FLAG_SELECT_TYPE)
info("default RamDiskImage %s",
bg_conf->default_ramdiskimage);
image = xmalloc(sizeof(image_t));
image->name = xstrdup(bg_conf->default_ramdiskimage);
image->def = true;
image->groups = NULL;
/* we want it to be first */
list_push(bg_conf->ramdisk_list, image);
}
#elif defined HAVE_BGP
if (s_p_get_array((void ***)&image_array,
&count, "AltCnloadImage", tbl)) {
for (i = 0; i < count; i++) {
list_append(bg_conf->linux_list, image_array[i]);
image_array[i] = NULL;
}
}
if (!s_p_get_string(&bg_conf->default_linuximage, "CnloadImage", tbl)) {
if (!list_count(bg_conf->linux_list))
fatal("CnloadImage not configured "
"in bluegene.conf");
itr = list_iterator_create(bg_conf->linux_list);
image = list_next(itr);
image->def = true;
list_iterator_destroy(itr);
bg_conf->default_linuximage = xstrdup(image->name);
info("Warning: using %s as the default CnloadImage. "
"If this isn't correct please set CnloadImage",
bg_conf->default_linuximage);
} else {
if (bg_conf->slurm_debug_flags & DEBUG_FLAG_SELECT_TYPE)
info("default CnloadImage %s",
bg_conf->default_linuximage);
image = xmalloc(sizeof(image_t));
image->name = xstrdup(bg_conf->default_linuximage);
image->def = true;
image->groups = NULL;
/* we want it to be first */
list_push(bg_conf->linux_list, image);
}
if (s_p_get_array((void ***)&image_array,
&count, "AltIoloadImage", tbl)) {
for (i = 0; i < count; i++) {
list_append(bg_conf->ramdisk_list, image_array[i]);
image_array[i] = NULL;
}
}
if (!s_p_get_string(&bg_conf->default_ramdiskimage,
"IoloadImage", tbl)) {
if (!list_count(bg_conf->ramdisk_list))
fatal("IoloadImage not configured "
"in bluegene.conf");
itr = list_iterator_create(bg_conf->ramdisk_list);
image = list_next(itr);
image->def = true;
list_iterator_destroy(itr);
bg_conf->default_ramdiskimage = xstrdup(image->name);
info("Warning: using %s as the default IoloadImage. "
"If this isn't correct please set IoloadImage",
bg_conf->default_ramdiskimage);
} else {
if (bg_conf->slurm_debug_flags & DEBUG_FLAG_SELECT_TYPE)
info("default IoloadImage %s",
bg_conf->default_ramdiskimage);
image = xmalloc(sizeof(image_t));
image->name = xstrdup(bg_conf->default_ramdiskimage);
image->def = true;
image->groups = NULL;
/* we want it to be first */
list_push(bg_conf->ramdisk_list, image);
}
#endif
if (s_p_get_array((void ***)&image_array,
&count, "AltMloaderImage", tbl)) {
for (i = 0; i < count; i++) {
list_append(bg_conf->mloader_list, image_array[i]);
image_array[i] = NULL;
}
}
if (!s_p_get_string(&bg_conf->default_mloaderimage,
"MloaderImage", tbl)) {
if (!list_count(bg_conf->mloader_list))
fatal("MloaderImage not configured "
"in bluegene.conf");
itr = list_iterator_create(bg_conf->mloader_list);
image = list_next(itr);
image->def = true;
list_iterator_destroy(itr);
bg_conf->default_mloaderimage = xstrdup(image->name);
info("Warning: using %s as the default MloaderImage. "
"If this isn't correct please set MloaderImage",
bg_conf->default_mloaderimage);
} else {
if (bg_conf->slurm_debug_flags & DEBUG_FLAG_SELECT_TYPE)
info("default MloaderImage %s",
bg_conf->default_mloaderimage);
image = xmalloc(sizeof(image_t));
image->name = xstrdup(bg_conf->default_mloaderimage);
image->def = true;
image->groups = NULL;
/* we want it to be first */
list_push(bg_conf->mloader_list, image);
}
if (!s_p_get_uint16(&bg_conf->mp_cnode_cnt, "MidplaneNodeCnt", tbl)) {
if (!s_p_get_uint16(&bg_conf->mp_cnode_cnt,
"BasePartitionNodeCnt", tbl)) {
error("MidplaneNodeCnt not configured in bluegene.conf "
"defaulting to 512 as MidplaneNodeCnt");
bg_conf->mp_cnode_cnt = 512;
}
}
if (bg_conf->mp_cnode_cnt <= 0)
fatal("You should have more than 0 nodes "
"per midplane");
bg_conf->actual_cnodes_per_mp = bg_conf->mp_cnode_cnt;
bg_conf->quarter_cnode_cnt = bg_conf->mp_cnode_cnt/4;
/* bg_conf->cpus_per_mp should had already been set from the
* node_init */
if (bg_conf->cpus_per_mp < bg_conf->mp_cnode_cnt) {
fatal("For some reason we have only %u cpus per mp, but "
"have %u cnodes per mp. You need at least the same "
"number of cpus as you have cnodes per mp. "
"Check the NodeName CPUs= "
"definition in the slurm.conf.",
bg_conf->cpus_per_mp, bg_conf->mp_cnode_cnt);
}
bg_conf->cpu_ratio = bg_conf->cpus_per_mp/bg_conf->mp_cnode_cnt;
if (!bg_conf->cpu_ratio)
fatal("We appear to have less than 1 cpu on a cnode. "
"You specified %u for MidplaneNodeCnt "
"in the blugene.conf and %u cpus "
"for each node in the slurm.conf",
bg_conf->mp_cnode_cnt, bg_conf->cpus_per_mp);
num_unused_cpus = 1;
for (i = 0; i<SYSTEM_DIMENSIONS; i++)
num_unused_cpus *= dims[i];
num_unused_cpus *= bg_conf->cpus_per_mp;
num_possible_unused_cpus = num_unused_cpus;
if (!s_p_get_uint16(&bg_conf->nodecard_cnode_cnt,
"NodeBoardNodeCnt", tbl)) {
if (!s_p_get_uint16(&bg_conf->nodecard_cnode_cnt,
"NodeCardNodeCnt", tbl)) {
error("NodeCardNodeCnt not configured in bluegene.conf "
"defaulting to 32 as NodeCardNodeCnt");
bg_conf->nodecard_cnode_cnt = 32;
}
}
if (bg_conf->nodecard_cnode_cnt <= 0)
fatal("You should have more than 0 nodes per nodecard");
bg_conf->mp_nodecard_cnt =
bg_conf->mp_cnode_cnt / bg_conf->nodecard_cnode_cnt;
if (!s_p_get_uint16(&bg_conf->ionodes_per_mp, "IONodesPerMP", tbl))
if (!s_p_get_uint16(&bg_conf->ionodes_per_mp, "Numpsets", tbl))
fatal("Warning: IONodesPerMP not configured "
"in bluegene.conf");
s_p_get_uint16(&bg_conf->max_block_err, "MaxBlockInError", tbl);
tmp_bool = 0;
s_p_get_boolean(&tmp_bool, "SubMidplaneSystem", tbl);
bg_conf->sub_mp_sys = tmp_bool;
#ifdef HAVE_BGQ
tmp_bool = 0;
s_p_get_boolean(&tmp_bool, "AllowSubBlockAllocations", tbl);
bg_conf->sub_blocks = tmp_bool;
/* You can only have 16 ionodes per midplane */
if (bg_conf->ionodes_per_mp > bg_conf->mp_nodecard_cnt)
bg_conf->ionodes_per_mp = bg_conf->mp_nodecard_cnt;
#endif
for (i=0; i<SYSTEM_DIMENSIONS; i++)
bg_conf->default_conn_type[i] = (uint16_t)NO_VAL;
s_p_get_string(&tmp_char, "DefaultConnType", tbl);
if (tmp_char) {
verify_conn_type(tmp_char, bg_conf->default_conn_type);
if ((bg_conf->default_conn_type[0] != SELECT_MESH)
&& (bg_conf->default_conn_type[0] != SELECT_TORUS))
fatal("Can't have a DefaultConnType of %s "
"(only Mesh or Torus values are valid).",
tmp_char);
xfree(tmp_char);
} else
bg_conf->default_conn_type[0] = SELECT_TORUS;
#ifndef HAVE_BG_L_P
int first_conn_type = bg_conf->default_conn_type[0];
for (i=1; i<SYSTEM_DIMENSIONS; i++) {
if (bg_conf->default_conn_type[i] == (uint16_t)NO_VAL)
bg_conf->default_conn_type[i] = first_conn_type;
else if (bg_conf->default_conn_type[i] >= SELECT_SMALL)
fatal("Can't have a DefaultConnType of %s "
"(only Mesh or Torus values are valid).",
tmp_char);
}
#endif
if (bg_conf->ionodes_per_mp) {
bitstr_t *tmp_bitmap = NULL;
int small_size = 1;
/* THIS IS A HACK TO MAKE A 1 NODECARD SYSTEM WORK,
* Sometime on a Q system the nodecard isn't in the 0
* spot so only do this if you know it is in that
* spot. Otherwise say the whole midplane is there
* and just make blocks over the whole thing. They
* you can error out the blocks that aren't usable. */
if (bg_conf->sub_mp_sys
&& bg_conf->mp_cnode_cnt == bg_conf->nodecard_cnode_cnt) {
#ifdef HAVE_BGQ
bg_conf->quarter_ionode_cnt = 1;
bg_conf->nodecard_ionode_cnt = 1;
#else
bg_conf->quarter_ionode_cnt = 2;
bg_conf->nodecard_ionode_cnt = 2;
#endif
} else {
bg_conf->quarter_ionode_cnt = bg_conf->ionodes_per_mp/4;
bg_conf->nodecard_ionode_cnt =
bg_conf->quarter_ionode_cnt/4;
}
/* How many nodecards per ionode */
bg_conf->nc_ratio =
((double)bg_conf->mp_cnode_cnt
/ (double)bg_conf->nodecard_cnode_cnt)
/ (double)bg_conf->ionodes_per_mp;
/* How many ionodes per nodecard */
bg_conf->io_ratio =
(double)bg_conf->ionodes_per_mp /
((double)bg_conf->mp_cnode_cnt
/ (double)bg_conf->nodecard_cnode_cnt);
/* How many cnodes per ionode */
bg_conf->ionode_cnode_cnt =
bg_conf->nodecard_cnode_cnt * bg_conf->nc_ratio;
//info("got %f %f", bg_conf->nc_ratio, bg_conf->io_ratio);
/* figure out the smallest block we can have on the
system */
#ifdef HAVE_BGL
if (bg_conf->io_ratio >= 1)
bg_conf->smallest_block=32;
else
bg_conf->smallest_block=128;
#else
if (bg_conf->io_ratio >= 2)
bg_conf->smallest_block=16;
else if (bg_conf->io_ratio == 1)
bg_conf->smallest_block=32;
else if (bg_conf->io_ratio == .5)
bg_conf->smallest_block=64;
else if (bg_conf->io_ratio == .25)
bg_conf->smallest_block=128;
else if (bg_conf->io_ratio == .125)
bg_conf->smallest_block=256;
else {
error("unknown ioratio %f. Can't figure out "
"smallest block size, setting it to midplane",
bg_conf->io_ratio);
bg_conf->smallest_block = 512;
}
#endif
if (bg_conf->slurm_debug_flags & DEBUG_FLAG_SELECT_TYPE)
info("Smallest block possible on this system is %u",
bg_conf->smallest_block);
/* below we are creating all the possible bitmaps for
* each size of small block
*/
if ((int)bg_conf->nodecard_ionode_cnt < 1) {
bg_conf->nodecard_ionode_cnt = 0;
} else {
bg_lists->valid_small32 = list_create(_destroy_bitmap);
/* This is suppose to be = and not ==, we only
want to decrement when small_size equals
something.
*/
if ((small_size = bg_conf->nodecard_ionode_cnt))
small_size--;
i = 0;
while (i<bg_conf->ionodes_per_mp) {
tmp_bitmap = bit_alloc(bg_conf->ionodes_per_mp);
bit_nset(tmp_bitmap, i, i+small_size);
i += small_size+1;
list_append(bg_lists->valid_small32,
tmp_bitmap);
}
}
/* If we only have 1 nodecard just jump to the end
since this will never need to happen below.
Pretty much a hack to avoid seg fault;). */
if (bg_conf->mp_cnode_cnt == bg_conf->nodecard_cnode_cnt)
goto no_calc;
bg_lists->valid_small128 = list_create(_destroy_bitmap);
if ((small_size = bg_conf->quarter_ionode_cnt))
small_size--;
i = 0;
while (i<bg_conf->ionodes_per_mp) {
tmp_bitmap = bit_alloc(bg_conf->ionodes_per_mp);
bit_nset(tmp_bitmap, i, i+small_size);
i += small_size+1;
list_append(bg_lists->valid_small128, tmp_bitmap);
}
#ifndef HAVE_BGL
bg_lists->valid_small64 = list_create(_destroy_bitmap);
if ((small_size = bg_conf->nodecard_ionode_cnt * 2))
small_size--;
i = 0;
while (i<bg_conf->ionodes_per_mp) {
tmp_bitmap = bit_alloc(bg_conf->ionodes_per_mp);
bit_nset(tmp_bitmap, i, i+small_size);
i += small_size+1;
list_append(bg_lists->valid_small64, tmp_bitmap);
}
bg_lists->valid_small256 = list_create(_destroy_bitmap);
if ((small_size = bg_conf->quarter_ionode_cnt * 2))
small_size--;
i = 0;
while (i<bg_conf->ionodes_per_mp) {
tmp_bitmap = bit_alloc(bg_conf->ionodes_per_mp);
bit_nset(tmp_bitmap, i, i+small_size);
i += small_size+1;
list_append(bg_lists->valid_small256, tmp_bitmap);
}
#endif
} else {
fatal("your ionodes_per_mp is 0");
}
no_calc:
if (!s_p_get_uint16(&bg_conf->bridge_api_verb, "BridgeAPIVerbose", tbl))
info("Warning: BridgeAPIVerbose not configured "
"in bluegene.conf");
if (!s_p_get_string(&bg_conf->bridge_api_file,
"BridgeAPILogFile", tbl))
info("BridgeAPILogFile not configured in bluegene.conf");
else
_reopen_bridge_log();
if (s_p_get_string(&tmp_char, "DenyPassthrough", tbl)) {
if (strstr(tmp_char, "A"))
ba_deny_pass |= PASS_DENY_A;
if (strstr(tmp_char, "X"))
ba_deny_pass |= PASS_DENY_X;
if (strstr(tmp_char, "Y"))
ba_deny_pass |= PASS_DENY_Y;
if (strstr(tmp_char, "Z"))
ba_deny_pass |= PASS_DENY_Z;
if (!xstrcasecmp(tmp_char, "ALL"))
ba_deny_pass |= PASS_DENY_ALL;
bg_conf->deny_pass = ba_deny_pass;
xfree(tmp_char);
}
if (!s_p_get_string(&tmp_char, "LayoutMode", tbl)) {
info("Warning: LayoutMode was not specified in bluegene.conf "
"defaulting to STATIC partitioning");
bg_conf->layout_mode = LAYOUT_STATIC;
} else {
if (!xstrcasecmp(tmp_char,"STATIC"))
bg_conf->layout_mode = LAYOUT_STATIC;
else if (!xstrcasecmp(tmp_char,"OVERLAP"))
bg_conf->layout_mode = LAYOUT_OVERLAP;
else if (!xstrcasecmp(tmp_char,"DYNAMIC"))
bg_conf->layout_mode = LAYOUT_DYNAMIC;
else {
fatal("I don't understand this LayoutMode = %s",
tmp_char);
}
xfree(tmp_char);
}
/* add blocks defined in file */
if (bg_conf->layout_mode != LAYOUT_DYNAMIC) {
if (!s_p_get_array((void ***)&blockreq_array,
&count, "MPs", tbl)) {
if (!s_p_get_array((void ***)&blockreq_array,
&count, "BPs", tbl)) {
info("WARNING: no blocks defined in "
"bluegene.conf, "
"only making full system block");
/* create_full_system_block(NULL); */
if (bg_conf->sub_mp_sys ||
(bg_conf->mp_cnode_cnt ==
bg_conf->nodecard_cnode_cnt))
fatal("On a sub-midplane system you "
"need to define the blocks you "
"want on your system.");
}
}
for (i = 0; i < count; i++) {
add_bg_record(bg_lists->main, NULL,
blockreq_array[i], 0, 0);
}
} else if (bg_conf->sub_mp_sys ||
(bg_conf->mp_cnode_cnt == bg_conf->nodecard_cnode_cnt))
/* we can't do dynamic here on a sub-midplane system */
fatal("On a sub-midplane system we can only do OVERLAP or "
"STATIC LayoutMode. Please update your bluegene.conf.");
#ifdef HAVE_BGQ
if ((bg_recover != NOT_FROM_CONTROLLER) && assoc_mgr_qos_list
&& s_p_get_string(&tmp_char, "RebootQOSList", tbl)) {
bool valid;
char *token, *last = NULL;
slurmdb_qos_rec_t *qos = NULL;
assoc_mgr_lock_t locks = { NO_LOCK, NO_LOCK, NO_LOCK,
READ_LOCK, NO_LOCK,
NO_LOCK, NO_LOCK };
/* Lock here to avoid g_qos_count changing under us */
assoc_mgr_lock(&locks);
bg_conf->reboot_qos_bitmap = bit_alloc(g_qos_count);
itr = list_iterator_create(assoc_mgr_qos_list);
token = strtok_r(tmp_char, ",", &last);
while (token) {
valid = false;
while((qos = list_next(itr))) {
if (!xstrcasecmp(token, qos->name)) {
bit_set(bg_conf->reboot_qos_bitmap,
qos->id);
valid = true;
break;
}
}
if (!valid)
error("Invalid RebootQOSList value: %s", token);
list_iterator_reset(itr);
token = strtok_r(NULL, ",", &last);
}
list_iterator_destroy(itr);
xfree(tmp_char);
assoc_mgr_unlock(&locks);
}
#endif
s_p_hashtbl_destroy(tbl);
return SLURM_SUCCESS;
}
extern int create_full_system_block(List bg_found_block_list)
{
int rc = SLURM_SUCCESS;
ListIterator itr;
bg_record_t *bg_record = NULL;
char *name = NULL;
List records = NULL;
uint16_t geo[SYSTEM_DIMENSIONS];
int i;
select_ba_request_t blockreq;
List results = NULL;
struct part_record *part_ptr = NULL;
bitstr_t *bitmap = bit_alloc(node_record_count);
static int *dims = NULL;
bool larger = 0;
char start_char[SYSTEM_DIMENSIONS+1];
char geo_char[SYSTEM_DIMENSIONS+1];
if (!dims) {
dims = select_g_ba_get_dims();
memset(start_char, 0, sizeof(start_char));
memset(geo_char, 0, sizeof(geo_char));
}
/* Locks are already in place to protect part_list here */
itr = list_iterator_create(part_list);
while ((part_ptr = list_next(itr))) {
/* we only want to use mps that are in
* partitions
*/
if (!part_ptr->node_bitmap) {
debug4("Partition %s doesn't have any nodes in it.",
part_ptr->name);
continue;
}
bit_or(bitmap, part_ptr->node_bitmap);
}
list_iterator_destroy(itr);
bit_not(bitmap);
if (bit_ffs(bitmap) != -1) {
error("We don't have the entire system covered by partitions, "
"can't create full system block");
FREE_NULL_BITMAP(bitmap);
return SLURM_ERROR;
}
FREE_NULL_BITMAP(bitmap);
/* Here we are adding a block that in for the entire machine
just in case it isn't in the bluegene.conf file.
*/
slurm_mutex_lock(&block_state_mutex);
for (i=0; i<SYSTEM_DIMENSIONS; i++) {
geo[i] = dims[i] - 1;
if (geo[i] > 0)
larger = 1;
geo_char[i] = alpha_num[geo[i]];
start_char[i] = alpha_num[0];
}
i = (10+strlen(bg_conf->slurm_node_prefix));
name = xmalloc(i);
if (!larger)
snprintf(name, i, "%s%s",
bg_conf->slurm_node_prefix, start_char);
else
snprintf(name, i, "%s[%sx%s]",
bg_conf->slurm_node_prefix, start_char, geo_char);
if (bg_found_block_list) {
itr = list_iterator_create(bg_found_block_list);
while ((bg_record = (bg_record_t *) list_next(itr)) != NULL) {
if (!strcmp(name, bg_record->mp_str)) {
xfree(name);
list_iterator_destroy(itr);
/* don't create total already there */
goto no_total;
}
}
list_iterator_destroy(itr);
} else {
error("create_full_system_block: no bg_found_block_list 2");
}
if (bg_lists->main) {
itr = list_iterator_create(bg_lists->main);
while ((bg_record = (bg_record_t *) list_next(itr))
!= NULL) {
if (!strcmp(name, bg_record->mp_str)) {
xfree(name);
list_iterator_destroy(itr);
/* don't create total already there */
goto no_total;
}
}
list_iterator_destroy(itr);
} else {
xfree(name);
error("create_overlapped_blocks: no bg_lists->main 3");
rc = SLURM_ERROR;
goto no_total;
}
records = list_create(destroy_bg_record);
memset(&blockreq, 0, sizeof(select_ba_request_t));
blockreq.save_name = name;
for (i=0; i<SYSTEM_DIMENSIONS; i++)
blockreq.conn_type[i] = SELECT_TORUS;
add_bg_record(records, NULL, &blockreq, 0 , 0);
xfree(name);
bg_record = (bg_record_t *) list_pop(records);
if (!bg_record) {
error("Nothing was returned from full system create");
rc = SLURM_ERROR;
goto no_total;
}
reset_ba_system(false);
for(i=0; i<SYSTEM_DIMENSIONS; i++) {
geo_char[i] = alpha_num[bg_record->geo[i]];
start_char[i] = alpha_num[bg_record->start[i]];
}
debug2("adding %s %s %s", bg_record->mp_str, start_char, geo_char);
if (bg_record->ba_mp_list)
list_flush(bg_record->ba_mp_list);
else
bg_record->ba_mp_list = list_create(destroy_ba_mp);
#ifdef HAVE_BGQ
results = list_create(destroy_ba_mp);
#else
results = list_create(NULL);
#endif
name = set_bg_block(results,
bg_record->start,
bg_record->geo,
bg_record->conn_type);
if (!name) {
error("I was unable to make the full system block.");
list_destroy(results);
list_iterator_destroy(itr);
slurm_mutex_unlock(&block_state_mutex);
return SLURM_ERROR;
}
xfree(name);
if (bg_record->ba_mp_list)
list_destroy(bg_record->ba_mp_list);
#ifdef HAVE_BGQ
bg_record->ba_mp_list = results;
results = NULL;
#else
bg_record->ba_mp_list = list_create(destroy_ba_mp);
copy_node_path(results, &bg_record->ba_mp_list);
list_destroy(results);
#endif
if ((rc = bridge_block_create(bg_record)) == SLURM_ERROR) {
error("create_full_system_block: "
"unable to configure block in api");
destroy_bg_record(bg_record);
goto no_total;
}
print_bg_record(bg_record);
list_append(bg_lists->main, bg_record);
no_total:
if (records)
list_destroy(records);
slurm_mutex_unlock(&block_state_mutex);
return rc;
}
extern void process_nodes(bg_record_t *bg_record, bool startup)
{
int j=0;
int diff=0;
int largest_diff=-1;
uint16_t best_start[SYSTEM_DIMENSIONS];
uint16_t start[SYSTEM_DIMENSIONS];
uint16_t end[SYSTEM_DIMENSIONS];
bool start_set=0;
ListIterator itr;
ba_mp_t* ba_mp = NULL;
int dim;
static char tmp_char[SYSTEM_DIMENSIONS+1],
tmp_char2[SYSTEM_DIMENSIONS+1];
static int *cluster_dims = NULL;
if (!cluster_dims) {
/* do some initing that only needs to happen once. */
cluster_dims = select_g_ba_get_dims();
memset(tmp_char, 0, sizeof(tmp_char));
memset(tmp_char2, 0, sizeof(tmp_char2));
}
if (!bg_record->ba_mp_list || !list_count(bg_record->ba_mp_list)) {
char *nodes = bg_record->mp_str;
if (!bg_record->ba_mp_list)
bg_record->ba_mp_list = list_create(destroy_ba_mp);
memset(&best_start, 0, sizeof(best_start));
//bg_record->mp_count = 0;
if ((bg_record->conn_type[0] >= SELECT_SMALL) && (!startup))
error("process_nodes: "
"We shouldn't be here there could be some "
"badness if we use this logic %s",
bg_record->mp_str);
while (nodes[j] != '\0') {
int mid = j + SYSTEM_DIMENSIONS + 1;
int fin = mid + SYSTEM_DIMENSIONS + 1;
if (((nodes[j] == '[') || (nodes[j] == ',')) &&
((nodes[mid] == 'x') || (nodes[mid] == '-')) &&
((nodes[fin] == ']') || (nodes[fin] == ','))) {
j++; /* Skip leading '[' or ',' */
for (dim = 0; dim < SYSTEM_DIMENSIONS;
dim++, j++)
start[dim] = select_char2coord(
nodes[j]);
j++; /* Skip middle 'x' or '-' */
for (dim = 0; dim < SYSTEM_DIMENSIONS;
dim++, j++)
end[dim] = select_char2coord(nodes[j]);
diff = end[0]-start[0];
_addto_mp_list(bg_record, start, end);
} else if ((nodes[j] >= '0'&& nodes[j] <= '9')
|| (nodes[j] >= 'A' && nodes[j] <= 'Z')) {
for (dim = 0; dim < SYSTEM_DIMENSIONS;
dim++, j++)
start[dim] = select_char2coord(
nodes[j]);
diff = 0;
_addto_mp_list(bg_record, start, start);
} else {
j++;
continue;
}
if (diff > largest_diff) {
largest_diff = diff;
memcpy(best_start, start, sizeof(best_start));
if (bg_conf->slurm_debug_level
>= LOG_LEVEL_DEBUG3) {
for (dim = 0;
dim < SYSTEM_DIMENSIONS;
dim++)
tmp_char[dim] = alpha_num[
best_start[dim]];
debug3("process_nodes: start is now %s",
tmp_char);
}
}
if (bg_record->mp_str[j] != ',')
break;
}
if (largest_diff == -1)
fatal("No hostnames given here");
memcpy(bg_record->start, best_start, sizeof(bg_record->start));
start_set = 1;
if (bg_conf->slurm_debug_level >= LOG_LEVEL_DEBUG3) {
for (dim = 0; dim < SYSTEM_DIMENSIONS; dim++) {
tmp_char[dim] = alpha_num[best_start[dim]];
tmp_char2[dim] =
alpha_num[bg_record->start[dim]];
}
debug3("process_nodes: start is %s %s",
tmp_char, tmp_char2);
}
}
memset(bg_record->geo, 0, sizeof(bg_record->geo));
for (dim = 0; dim < SYSTEM_DIMENSIONS; dim++) {
end[dim] = (int16_t)-1;
if (!start_set)
bg_record->start[dim] = HOSTLIST_BASE;
}
list_sort(bg_record->ba_mp_list, (ListCmpF) _ba_mp_cmpf_inc);
bg_record->mp_count = 0;
itr = list_iterator_create(bg_record->ba_mp_list);
while ((ba_mp = list_next(itr))) {
if (!ba_mp->used)
continue;
bg_record->mp_count++;
debug3("process_nodes: %s is included in this block",
ba_mp->coord_str);
for (dim = 0; dim < SYSTEM_DIMENSIONS; dim++) {
if (ba_mp->coord[dim] > (int16_t)end[dim]) {
bg_record->geo[dim]++;
end[dim] = ba_mp->coord[dim];
}
if (!start_set && (ba_mp->coord[dim] <
(int16_t)bg_record->start[dim]))
bg_record->start[dim] = ba_mp->coord[dim];
}
}
list_iterator_destroy(itr);
if (bg_conf->slurm_debug_level >= LOG_LEVEL_DEBUG3) {
for (dim = 0; dim < SYSTEM_DIMENSIONS; dim++) {
tmp_char[dim] = alpha_num[bg_record->geo[dim]];
tmp_char2[dim] = alpha_num[bg_record->start[dim]];
}
debug3("process_nodes: geo = %s mp count is %d start is %s",
tmp_char, bg_record->mp_count, tmp_char2);
}
/* This check is for sub midplane systems to figure out what
the largest block can be.
*/
for (dim = 0; dim < SYSTEM_DIMENSIONS; dim++) {
if (cluster_dims[dim] > 1)
break;
}
if (dim < SYSTEM_DIMENSIONS) {
/* means we have more than 1 base partition */
for (dim = 0; dim < SYSTEM_DIMENSIONS; dim++) {
if (bg_record->geo[dim] != cluster_dims[dim])
break;
}
if (dim == SYSTEM_DIMENSIONS)
bg_record->full_block = 1;
} else if (bg_record->cnode_cnt == bg_conf->mp_cnode_cnt)
bg_record->full_block = 1;
FREE_NULL_BITMAP(bg_record->mp_bitmap);
if (node_name2bitmap(bg_record->mp_str,
false,
&bg_record->mp_bitmap)) {
fatal("process_nodes: Unable to convert nodes %s to bitmap",
bg_record->mp_str);
}
return;
}
