/**
* basil_geometry - Check node attributes, resolve (X,Y,Z) coordinates.
*
* Checks both SDB database and ALPS inventory for consistency. The inventory
* part is identical to basil_inventory(), with the difference of being called
* before valid bitmaps exist, from select_g_node_init().
* Its dependencies are:
* - it needs reset_job_bitmaps() in order to rebuild node_bitmap fields,
* - it relies on _sync_nodes_to_jobs() to
* o kill active jobs on nodes now marked DOWN,
* o reset node state to ALLOCATED if it has been marked IDLE here (which is
* an error case, since there is no longer an ALPS reservation for the job,
* this is caught by the subsequent basil_inventory()).
*/
extern int basil_geometry(struct node_record *node_ptr_array, int node_cnt)
{
struct node_record *node_ptr, *end = node_ptr_array + node_cnt;
enum basil_version version = get_basil_version();
struct basil_inventory *inv;
/* General mySQL */
MYSQL *handle;
MYSQL_STMT *stmt = NULL;
/* Input parameters */
unsigned int node_id;
/*
* Use a left outer join here since the attributes table may not be
* populated for a given nodeid (e.g. when the node has been disabled
* on the SMW via 'xtcli disable').
* The processor table has more authoritative information, if a nodeid
* is not listed there, it does not exist.
*/
const char query[] = "SELECT x_coord, y_coord, z_coord,"
" cab_position, cab_row, cage, slot, cpu,"
" LOG2(coremask+1), availmem, "
" processor_type "
"FROM processor LEFT JOIN attributes "
"ON processor_id = nodeid "
"WHERE processor_id = ? ";
const int PARAM_COUNT = 1; /* node id */
MYSQL_BIND params[PARAM_COUNT];
int x_coord, y_coord, z_coord;
int cab, row, cage, slot, cpu;
unsigned int node_cpus, node_mem;
char proc_type[BASIL_STRING_SHORT];
MYSQL_BIND bind_cols[COLUMN_COUNT];
my_bool is_null[COLUMN_COUNT];
my_bool is_error[COLUMN_COUNT];
int is_gemini, i;
time_t now = time(NULL);
memset(params, 0, sizeof(params));
params[0].buffer_type = MYSQL_TYPE_LONG;
params[0].is_unsigned = true;
params[0].is_null = (my_bool *)0;
params[0].buffer = (char *)&node_id;
memset(bind_cols, 0, sizeof(bind_cols));
for (i = 0; i < COLUMN_COUNT; i ++) {
bind_cols[i].is_null = &is_null[i];
bind_cols[i].error = &is_error[i];
if (i == COL_TYPE) {
bind_cols[i].buffer_type = MYSQL_TYPE_STRING;
bind_cols[i].buffer_length = sizeof(proc_type);
bind_cols[i].buffer = proc_type;
} else {
bind_cols[i].buffer_type = MYSQL_TYPE_LONG;
bind_cols[i].is_unsigned = (i >= COL_CORES);
}
}
bind_cols[COL_X].buffer = (char *)&x_coord;
bind_cols[COL_Y].buffer = (char *)&y_coord;
bind_cols[COL_Z].buffer = (char *)&z_coord;
bind_cols[COL_CAB].buffer = (char *)&cab;
bind_cols[COL_ROW].buffer = (char *)&row;
bind_cols[COL_CAGE].buffer = (char *)&cage;
bind_cols[COL_SLOT].buffer = (char *)&slot;
bind_cols[COL_CPU].buffer = (char *)&cpu;
bind_cols[COL_CORES].buffer = (char *)&node_cpus;
bind_cols[COL_MEMORY].buffer = (char *)&node_mem;
inv = get_full_inventory(version);
if (inv == NULL)
fatal("failed to get initial BASIL inventory");
info("BASIL %s initial INVENTORY: %d/%d batch nodes available",
bv_names_long[version], inv->batch_avail, inv->batch_total);
handle = cray_connect_sdb();
if (handle == NULL)
fatal("can not connect to XTAdmin database on the SDB");
is_gemini = cray_is_gemini_system(handle);
if (is_gemini < 0)
fatal("can not determine Cray XT/XE system type");
stmt = prepare_stmt(handle, query, params, PARAM_COUNT,
bind_cols, COLUMN_COUNT);
if (stmt == NULL)
fatal("can not prepare statement to resolve Cray coordinates");
for (node_ptr = node_record_table_ptr; node_ptr < end; node_ptr++) {
struct basil_node *node;
char *reason = NULL;
if ((node_ptr->name == NULL) ||
(sscanf(node_ptr->name, "nid%05u", &node_id) != 1)) {
error("can not read basil_node_id from %s",
node_ptr->name);
continue;
}
if (exec_stmt(stmt, query, bind_cols, COLUMN_COUNT) < 0)
fatal("can not resolve %s coordinates", node_ptr->name);
if (fetch_stmt(stmt) == 0) {
#if _DEBUG
info("proc_type:%s cpus:%u memory:%u",
proc_type, node_cpus, node_mem);
info("row:%u cage:%u slot:%u cpu:%u xyz:%u:%u:%u",
row, cage, slot, cpu, x_coord, y_coord, z_coord);
#endif
if (strcmp(proc_type, "compute") != 0) {
/*
* Switching a compute node to be a service node
* can not happen at runtime: requires a reboot.
*/
fatal("Node '%s' is a %s node. "
"Only compute nodes can appear in slurm.conf.",
node_ptr->name, proc_type);
} else if (is_null[COL_CORES] || is_null[COL_MEMORY]) {
/*
* This can happen if a node has been disabled
* on the SMW (using 'xtcli disable <nid>'). The
* node will still be listed in the 'processor'
* table, but have no 'attributes' entry (NULL
* values for CPUs/memory). Also, the node will
* be invisible to ALPS, which is why we need to
* set it down here already.
*/
node_cpus = node_mem = 0;
reason = "node data unknown - disabled on SMW?";
} else if (is_null[COL_X] || is_null[COL_Y]
|| is_null[COL_Z]) {
/*
* Similar case to the one above, observed when
* a blade has been removed. Node will not
* likely show up in ALPS.
*/
x_coord = y_coord = z_coord = 0;
reason = "unknown coordinates - hardware failure?";
} else if (node_cpus < node_ptr->config_ptr->cpus) {
/*
* FIXME: Might reconsider this policy.
*
* FastSchedule is ignored here, it requires the
* slurm.conf to be consistent with hardware.
*
* Assumption is that CPU/Memory do not change
* at runtime (Cray has no hot-swappable parts).
*
* Hence checking it in basil_inventory() would
* mean a lot of runtime overhead.
*/
fatal("slurm.conf: node %s has only Procs=%d",
node_ptr->name, node_cpus);
} else if (node_mem < node_ptr->config_ptr->real_memory) {
fatal("slurm.conf: node %s has RealMemory=%d",
node_ptr->name, node_mem);
}
} else if (is_gemini) {
fatal("Non-existing Gemini node '%s' in slurm.conf",
node_ptr->name);
} else {
fatal("Non-existing SeaStar node '%s' in slurm.conf",
node_ptr->name);
}
if (!is_gemini) {
/*
* SeaStar: each node has unique coordinates
*/
if (node_ptr->arch == NULL)
node_ptr->arch = xstrdup("XT");
} else {
/*
* Gemini: each 2 nodes share the same network
* interface (i.e., nodes 0/1 and 2/3 each have
* the same coordinates).
*/
if (node_ptr->arch == NULL)
node_ptr->arch = xstrdup("XE");
}
xfree(node_ptr->node_hostname);
xfree(node_ptr->comm_name);
/*
* Convention: since we are using SLURM in frontend-mode,
* we use Node{Addr,HostName} as follows.
*
* NodeAddr: <X><Y><Z> coordinates in base-36 encoding
*
* NodeHostName: c#-#c#s#n# using the NID convention
* <cabinet>-<row><chassis><slot><node>
* - each cabinet can accommodate 3 chassis (c1..c3)
* - each chassis has 8 slots (s0..s7)
* - each slot contains 2 or 4 nodes (n0..n3)
* o either 2 service nodes (n0/n3)
* o or 4 compute nodes (n0..n3)
* o or 2 gemini chips (g0/g1 serving n0..n3)
*
* Example: c0-0c1s0n1
* - c0- = cabinet 0
* - 0 = row 0
* - c1 = chassis 1
* - s0 = slot 0
* - n1 = node 1
*/
node_ptr->node_hostname = xstrdup_printf("c%u-%uc%us%un%u", cab,
row, cage, slot, cpu);
node_ptr->comm_name = xstrdup_printf("%c%c%c",
_enc_coord(x_coord),
_enc_coord(y_coord),
_enc_coord(z_coord));
dim_size[0] = MAX(dim_size[0], (x_coord - 1));
dim_size[1] = MAX(dim_size[1], (y_coord - 1));
dim_size[2] = MAX(dim_size[2], (z_coord - 1));
#if _DEBUG
info("%s %s %s cpus=%u, mem=%u reason=%s", node_ptr->name,
node_ptr->node_hostname, node_ptr->comm_name,
node_cpus, node_mem, reason);
#endif
/*
* Check the current state reported by ALPS inventory, unless it
* is already evident that the node has some other problem.
*/
if (reason == NULL) {
for (node = inv->f->node_head; node; node = node->next)
if (node->node_id == node_id)
break;
if (node == NULL) {
reason = "not visible to ALPS - check hardware";
} else if (node->state == BNS_DOWN) {
reason = "ALPS marked it DOWN";
} else if (node->state == BNS_UNAVAIL) {
reason = "node is UNAVAILABLE";
} else if (node->state == BNS_ROUTE) {
reason = "node does ROUTING";
} else if (node->state == BNS_SUSPECT) {
reason = "entered SUSPECT mode";
} else if (node->state == BNS_ADMINDOWN) {
reason = "node is ADMINDOWN";
} else if (node->state != BNS_UP) {
reason = "state not UP";
} else if (node->role != BNR_BATCH) {
reason = "mode not BATCH";
} else if (node->arch != BNA_XT) {
reason = "arch not XT/XE";
}
}
/* Base state entirely derives from ALPS
* NOTE: The node bitmaps are not defined when this code is
* initially executed. */
node_ptr->node_state &= NODE_STATE_FLAGS;
if (reason) {
if (node_ptr->down_time == 0)
node_ptr->down_time = now;
if (IS_NODE_DOWN(node_ptr)) {
/* node still down */
debug("Initial DOWN node %s - %s",
node_ptr->name, node_ptr->reason);
} else if (slurmctld_conf.slurmd_timeout &&
((now - node_ptr->down_time) <
slurmctld_conf.slurmd_timeout)) {
node_ptr->node_state |= NODE_STATE_NO_RESPOND;
} else {
info("Initial DOWN node %s - %s",
node_ptr->name, reason);
node_ptr->reason = xstrdup(reason);
/* Node state flags preserved above */
node_ptr->node_state |= NODE_STATE_DOWN;
clusteracct_storage_g_node_down(acct_db_conn,
node_ptr,
now, NULL,
slurm_get_slurm_user_id());
}
} else {
bool node_up_flag = IS_NODE_DOWN(node_ptr) &&
!IS_NODE_DRAIN(node_ptr) &&
!IS_NODE_FAIL(node_ptr);
node_ptr->down_time = 0;
if (node_is_allocated(node))
node_ptr->node_state |= NODE_STATE_ALLOCATED;
else
node_ptr->node_state |= NODE_STATE_IDLE;
node_ptr->node_state &= (~NODE_STATE_NO_RESPOND);
xfree(node_ptr->reason);
if (node_up_flag) {
info("ALPS returned node %s to service",
node_ptr->name);
clusteracct_storage_g_node_up(acct_db_conn,
node_ptr, now);
}
}
free_stmt_result(stmt);
}
if (stmt_close(stmt))
error("error closing statement: %s", mysql_stmt_error(stmt));
cray_close_sdb(handle);
free_inv(inv);
return SLURM_SUCCESS;
}
/* block_state_mutex should be locked before calling */
static int _check_all_blocks_error(int node_inx, time_t event_time,
char *reason)
{
bg_record_t *bg_record = NULL;
ListIterator itr = NULL;
struct node_record send_node, *node_ptr;
struct config_record config_rec;
int total_cpus = 0;
int rc = SLURM_SUCCESS;
xassert(node_inx <= node_record_count);
node_ptr = &node_record_table_ptr[node_inx];
/* only do this if the node isn't in the DRAINED state.
DRAINING is ok */
if (IS_NODE_DRAINED(node_ptr))
return rc;
memset(&send_node, 0, sizeof(struct node_record));
memset(&config_rec, 0, sizeof(struct config_record));
send_node.name = xstrdup(node_ptr->name);
send_node.config_ptr = &config_rec;
/* here we need to check if there are any other blocks on this
midplane and adjust things correctly */
itr = list_iterator_create(bg_lists->main);
while ((bg_record = list_next(itr))) {
/* only look at other nodes in error state */
if (!(bg_record->state & BG_BLOCK_ERROR_FLAG))
continue;
if (!bit_test(bg_record->mp_bitmap, node_inx))
continue;
if (bg_record->cpu_cnt >= bg_conf->cpus_per_mp) {
total_cpus = bg_conf->cpus_per_mp;
break;
} else
total_cpus += bg_record->cpu_cnt;
}
list_iterator_destroy(itr);
send_node.cpus = total_cpus;
config_rec.cpus = total_cpus;
if (send_node.cpus) {
if (!reason)
reason = "update block: setting partial node down.";
if (!node_ptr->reason)
node_ptr->reason = xstrdup(reason);
node_ptr->reason_time = event_time;
node_ptr->reason_uid = slurm_get_slurm_user_id();
send_node.node_state = NODE_STATE_ERROR;
rc = clusteracct_storage_g_node_down(acct_db_conn,
&send_node, event_time,
reason,
node_ptr->reason_uid);
} else {
if (node_ptr->reason)
xfree(node_ptr->reason);
node_ptr->reason_time = 0;
node_ptr->reason_uid = NO_VAL;
send_node.node_state = NODE_STATE_IDLE;
rc = clusteracct_storage_g_node_up(acct_db_conn,
&send_node, event_time);
}
xfree(send_node.name);
return rc;
}
