static int _hostset_from_ranges(const pmix_proc_t *procs, size_t nprocs,
hostlist_t *hl_out)
{
int i;
hostlist_t hl = hostlist_create("");
pmixp_namespace_t *nsptr = NULL;
for (i = 0; i < nprocs; i++) {
char *node = NULL;
hostlist_t tmp;
nsptr = pmixp_nspaces_find(procs[i].nspace);
if (NULL == nsptr) {
goto err_exit;
}
if (procs[i].rank == PMIX_RANK_WILDCARD) {
tmp = hostlist_copy(nsptr->hl);
} else {
tmp = pmixp_nspace_rankhosts(nsptr, &procs[i].rank, 1);
}
while (NULL != (node = hostlist_pop(tmp))) {
hostlist_push(hl, node);
free(node);
}
hostlist_destroy(tmp);
}
hostlist_uniq(hl);
*hl_out = hl;
return SLURM_SUCCESS;
err_exit:
hostlist_destroy(hl);
return SLURM_ERROR;
}
int main(int argc, char *argv[])
{
log_options_t log_opts = LOG_OPTS_INITIALIZER;
hostlist_t hl = NULL;
char *node_name;
pthread_attr_t attr_work;
pthread_t thread_work = 0;
xstrfmtcat(prog_name, "%s[%u]", argv[0], (uint32_t) getpid());
_read_config();
log_opts.stderr_level = LOG_LEVEL_QUIET;
log_opts.syslog_level = LOG_LEVEL_QUIET;
if (slurm_get_debug_flags() && DEBUG_FLAG_NODE_FEATURES)
log_opts.logfile_level += 3;
(void) log_init(argv[0], log_opts, LOG_DAEMON, log_file);
if ((hl = hostlist_create(argv[1])) == NULL) {
error("%s: Invalid hostlist (%s)", prog_name, argv[1]);
exit(2);
}
while ((node_name = hostlist_pop(hl))) {
slurm_mutex_lock(&thread_cnt_mutex);
while (1) {
if (thread_cnt <= MAX_THREADS) {
thread_cnt++;
break;
} else { /* wait for state change and retry */
pthread_cond_wait(&thread_cnt_cond,
&thread_cnt_mutex);
}
}
slurm_mutex_unlock(&thread_cnt_mutex);
slurm_attr_init(&attr_work);
(void) pthread_attr_setdetachstate
(&attr_work, PTHREAD_CREATE_DETACHED);
if (pthread_create(&thread_work, &attr_work, _node_update,
(void *) node_name)) {
_node_update((void *) node_name);
}
slurm_attr_destroy(&attr_work);
}
/* Wait for work threads to complete */
slurm_mutex_lock(&thread_cnt_mutex);
while (1) {
if (thread_cnt == 0)
break;
else /* wait for state change and retry */
pthread_cond_wait(&thread_cnt_cond, &thread_cnt_mutex);
}
slurm_mutex_unlock(&thread_cnt_mutex);
hostlist_destroy(hl);
exit(0);
}
int main(int argc, char **argv)
{
ListIterator itr = NULL;
uint32_t req_cpufreq = NO_VAL;
uint32_t stepid = NO_VAL;
slurmdb_selected_step_t *selected_step = NULL;
#ifdef HAVE_ALPS_CRAY
error("The sstat command is not supported on Cray systems");
return 1;
#endif
#ifdef HAVE_BG
error("The sstat command is not supported on IBM BlueGene systems");
return 1;
#endif
slurm_conf_init(NULL);
print_fields_list = list_create(NULL);
print_fields_itr = list_iterator_create(print_fields_list);
parse_command_line(argc, argv);
if (!params.opt_job_list || !list_count(params.opt_job_list)) {
error("You didn't give me any jobs to stat.");
return 1;
}
print_fields_header(print_fields_list);
itr = list_iterator_create(params.opt_job_list);
while ((selected_step = list_next(itr))) {
char *nodelist = NULL;
bool free_nodelist = false;
if (selected_step->stepid == INFINITE) {
/* get the batch step info */
job_info_msg_t *job_ptr = NULL;
hostlist_t hl;
if (slurm_load_job(
&job_ptr, selected_step->jobid, SHOW_ALL)) {
error("couldn't get info for job %u",
selected_step->jobid);
continue;
}
stepid = NO_VAL;
hl = hostlist_create(job_ptr->job_array[0].nodes);
nodelist = hostlist_pop(hl);
free_nodelist = true;
hostlist_destroy(hl);
slurm_free_job_info_msg(job_ptr);
} else if (selected_step->stepid != NO_VAL) {
stepid = selected_step->stepid;
} else if (params.opt_all_steps) {
job_step_info_response_msg_t *step_ptr = NULL;
int i = 0;
if (slurm_get_job_steps(
0, selected_step->jobid, NO_VAL,
&step_ptr, SHOW_ALL)) {
error("couldn't get steps for job %u",
selected_step->jobid);
continue;
}
for (i = 0; i < step_ptr->job_step_count; i++) {
_do_stat(selected_step->jobid,
step_ptr->job_steps[i].step_id,
step_ptr->job_steps[i].nodes,
step_ptr->job_steps[i].cpu_freq);
}
slurm_free_job_step_info_response_msg(step_ptr);
continue;
} else {
/* get the first running step to query against. */
job_step_info_response_msg_t *step_ptr = NULL;
if (slurm_get_job_steps(
0, selected_step->jobid, NO_VAL,
&step_ptr, SHOW_ALL)) {
error("couldn't get steps for job %u",
selected_step->jobid);
continue;
}
if (!step_ptr->job_step_count) {
error("no steps running for job %u",
selected_step->jobid);
continue;
}
stepid = step_ptr->job_steps[0].step_id;
nodelist = step_ptr->job_steps[0].nodes;
req_cpufreq = step_ptr->job_steps[0].cpu_freq;
}
_do_stat(selected_step->jobid, stepid, nodelist, req_cpufreq);
if (free_nodelist && nodelist)
free(nodelist);
}
list_iterator_destroy(itr);
xfree(params.opt_field_list);
if (params.opt_job_list)
list_destroy(params.opt_job_list);
if (print_fields_itr)
list_iterator_destroy(print_fields_itr);
if (print_fields_list)
list_destroy(print_fields_list);
return 0;
}
int main(int argc, char *argv[])
{
log_options_t log_opts = LOG_OPTS_INITIALIZER;
char *features, *save_ptr = NULL, *tok;
update_node_msg_t node_msg;
int rc = SLURM_SUCCESS;
hostlist_t hl = NULL;
char *node_name;
pthread_attr_t attr_work;
pthread_t thread_work = 0;
prog_name = argv[0];
_read_config();
log_opts.stderr_level = LOG_LEVEL_QUIET;
log_opts.syslog_level = LOG_LEVEL_QUIET;
if (slurm_get_debug_flags() && DEBUG_FLAG_NODE_FEATURES)
log_opts.logfile_level += 3;
(void) log_init(argv[0], log_opts, LOG_DAEMON, log_file);
/* Parse the MCDRAM and NUMA boot options */
if (argc == 3) {
features = xstrdup(argv[2]);
tok = strtok_r(features, ",", &save_ptr);
while (tok) {
printf("%s\n", tok);
if (!strcasecmp(tok, "a2a") ||
!strcasecmp(tok, "hemi") ||
!strcasecmp(tok, "quad") ||
!strcasecmp(tok, "snc2") ||
!strcasecmp(tok, "snc4")) {
xfree(mcdram_mode);
mcdram_mode = xstrdup(tok);
} else if (!strcasecmp(tok, "cache") ||
!strcasecmp(tok, "equal") ||
!strcasecmp(tok, "flat")) {
xfree(numa_mode);
numa_mode = xstrdup(tok);
}
tok = strtok_r(NULL, ",", &save_ptr);
}
xfree(features);
}
/* Spawn threads to change MCDRAM and NUMA states and start node
* reboot process */
if ((hl = hostlist_create(argv[1])) == NULL) {
error("%s: Invalid hostlist (%s)", prog_name, argv[1]);
exit(2);
}
node_bitmap = bit_alloc(100000);
while ((node_name = hostlist_pop(hl))) {
slurm_mutex_lock(&thread_cnt_mutex);
while (1) {
if (thread_cnt <= MAX_THREADS) {
thread_cnt++;
break;
} else { /* wait for state change and retry */
pthread_cond_wait(&thread_cnt_cond,
&thread_cnt_mutex);
}
}
slurm_mutex_unlock(&thread_cnt_mutex);
slurm_attr_init(&attr_work);
(void) pthread_attr_setdetachstate
(&attr_work, PTHREAD_CREATE_DETACHED);
if (pthread_create(&thread_work, &attr_work, _node_update,
(void *) node_name)) {
_node_update((void *) node_name);
}
slurm_attr_destroy(&attr_work);
}
/* Wait for work threads to complete */
slurm_mutex_lock(&thread_cnt_mutex);
while (1) {
if (thread_cnt == 0)
break;
else /* wait for state change and retry */
pthread_cond_wait(&thread_cnt_cond, &thread_cnt_mutex);
}
slurm_mutex_unlock(&thread_cnt_mutex);
hostlist_destroy(hl);
xfree(mcdram_mode);
xfree(numa_mode);
/* Wait for all nodes to change state to "on" */
_wait_all_nodes_on();
if ((argc == 3) && !syscfg_path) {
slurm_init_update_node_msg(&node_msg);
node_msg.node_names = argv[1];
node_msg.features_act = argv[2];
rc = slurm_update_node(&node_msg);
}
if (rc == SLURM_SUCCESS) {
exit(0);
} else {
error("%s: slurm_update_node(\'%s\', \'%s\'): %s\n",
prog_name, argv[1], argv[2],
slurm_strerror(slurm_get_errno()));
exit(1);
}
}
/*
* Parse an MPMD file and determine count and layout of each task for use
* with Cray systems. Builds the mpmd_set structure in the job record.
*
* IN/OUT job - job step details, builds mpmd_set structure
* IN gtid - Array of global task IDs, indexed by node_id and task
*/
extern void multi_prog_parse(stepd_step_rec_t *job, uint32_t **gtid)
{
int i, j, line_num = 0, rank_id, total_ranks = 0;
char *line = NULL, *local_data = NULL;
char *end_ptr = NULL, *save_ptr = NULL, *tmp_str = NULL;
char *rank_spec = NULL, *cmd_spec = NULL, *args_spec = NULL;
char *p = NULL;
char **tmp_args, **tmp_cmd, *one_rank;
uint32_t *ranks_node_id = NULL; /* Node ID for each rank */
uint32_t *node_id2nid = NULL; /* Map Slurm node ID to Cray NID name */
bool last_line_break = false, line_break = false;
char *last_rank_spec = NULL;
int args_len, line_len;
hostlist_t hl;
tmp_args = xmalloc(sizeof(char *) * job->ntasks);
tmp_cmd = xmalloc(sizeof(char *) * job->ntasks);
node_id2nid = xmalloc(sizeof(uint32_t) * job->nnodes);
ranks_node_id = xmalloc(sizeof(uint32_t) * job->ntasks);
local_data = xstrdup(job->argv[1]);
while (1) {
if (line_num)
line = strtok_r(NULL, "\n", &save_ptr);
else
line = strtok_r(local_data, "\n", &save_ptr);
if (!line)
break;
line_num++;
line_len = strlen(line);
if ((line_len > 0) && (line[line_len - 1] == '\\'))
line_break = true;
else
line_break = false;
if (last_line_break && last_rank_spec) {
xstrfmtcat(tmp_str, "[%s]", last_rank_spec);
hl = hostlist_create(tmp_str);
xfree(tmp_str);
if (!hl)
goto fail;
while ((one_rank = hostlist_pop(hl))) {
rank_id = strtol(one_rank, &end_ptr, 10);
if ((end_ptr[0] != '\0') || (rank_id < 0) ||
(rank_id >= job->ntasks)) {
free(one_rank);
hostlist_destroy(hl);
goto fail;
}
free(one_rank);
args_len = strlen(tmp_args[rank_id]);
if (!tmp_args[rank_id] ||
tmp_args[rank_id][args_len - 1] != '\\') {
hostlist_destroy(hl);
goto fail;
}
tmp_args[rank_id][args_len -1] = '\0';
xstrcat(tmp_args[rank_id], line);
}
hostlist_destroy(hl);
last_line_break = line_break;
continue;
}
last_line_break = line_break;
p = line;
while ((*p != '\0') && isspace(*p)) /* remove leading spaces */
p++;
if (*p == '#') /* only whole-line comments handled */
continue;
if (*p == '\0') /* blank line ignored */
continue;
rank_spec = p; /* Rank specification for this line */
while ((*p != '\0') && !isspace(*p))
p++;
if (*p == '\0')
goto fail;
*p++ = '\0';
while ((*p != '\0') && isspace(*p)) /* remove leading spaces */
p++;
if (*p == '\0') /* blank line ignored */
continue;
cmd_spec = p; /* command only */
while ((*p != '\0') && !isspace(*p))
p++;
if (isspace(*p))
*p++ = '\0';
while ((*p != '\0') && isspace(*p)) /* remove leading spaces */
p++;
if (*p == '\0')
args_spec = NULL; /* no arguments */
else
args_spec = p; /* arguments string */
xstrfmtcat(tmp_str, "[%s]", rank_spec);
hl = hostlist_create(tmp_str);
xfree(tmp_str);
if (!hl)
goto fail;
while ((one_rank = hostlist_pop(hl))) {
rank_id = strtol(one_rank, &end_ptr, 10);
if ((end_ptr[0] != '\0') || (rank_id < 0) ||
(rank_id >= job->ntasks)) {
free(one_rank);
hostlist_destroy(hl);
goto fail;
}
free(one_rank);
if (tmp_args[rank_id]) /* duplicate record for rank */
xfree(tmp_args[rank_id]);
if (tmp_cmd[rank_id]) /* duplicate record for rank */
xfree(tmp_cmd[rank_id]);
else
total_ranks++;
tmp_args[rank_id] = xstrdup(args_spec);
tmp_cmd[rank_id] = xstrdup(cmd_spec);
}
hostlist_destroy(hl);
if (line_break)
last_rank_spec = rank_spec;
}
if (total_ranks != job->ntasks)
goto fail;
if (job->msg->complete_nodelist &&
((hl = hostlist_create(job->msg->complete_nodelist)))) {
i = 0;
while ((one_rank = hostlist_shift(hl))) {
if (i >= job->nnodes) {
error("MPMD more nodes in nodelist than count "
"(cnt:%u nodelist:%s)", job->nnodes,
job->msg->complete_nodelist);
}
for (j = 0; one_rank[j] && !isdigit(one_rank[j]); j++)
;
node_id2nid[i++] = strtol(one_rank + j, &end_ptr, 10);
free(one_rank);
}
hostlist_destroy(hl);
}
for (i = 0; i < job->nnodes; i++) {
if (!job->task_cnts) {
error("MPMD job->task_cnts is NULL");
break;
}
if (!job->task_cnts[i]) {
error("MPMD job->task_cnts[%d] is NULL", i);
break;
}
if (!gtid) {
error("MPMD gtid is NULL");
break;
}
if (!gtid[i]) {
error("MPMD gtid[%d] is NULL", i);
break;
}
for (j = 0; j < job->task_cnts[i]; j++) {
if (gtid[i][j] >= job->ntasks) {
error("MPMD gtid[%d][%d] is invalid (%u >= %u)",
i, j, gtid[i][j], job->ntasks);
break;
}
ranks_node_id[gtid[i][j]] = i;
}
}
job->mpmd_set = xmalloc(sizeof(mpmd_set_t));
job->mpmd_set->apid = SLURM_ID_HASH(job->jobid, job->stepid);
job->mpmd_set->args = xmalloc(sizeof(char *) * job->ntasks);
job->mpmd_set->command = xmalloc(sizeof(char *) * job->ntasks);
job->mpmd_set->first_pe = xmalloc(sizeof(int) * job->ntasks);
job->mpmd_set->start_pe = xmalloc(sizeof(int) * job->ntasks);
job->mpmd_set->total_pe = xmalloc(sizeof(int) * job->ntasks);
job->mpmd_set->placement = xmalloc(sizeof(int) * job->ntasks);
for (i = 0, j = 0; i < job->ntasks; i++) {
job->mpmd_set->placement[i] = node_id2nid[ranks_node_id[i]];
if (i == 0) {
job->mpmd_set->num_cmds++;
if (ranks_node_id[i] == job->nodeid)
job->mpmd_set->first_pe[j] = i;
else
job->mpmd_set->first_pe[j] = -1;
job->mpmd_set->args[j] = xstrdup(tmp_args[i]);
job->mpmd_set->command[j] = xstrdup(tmp_cmd[i]);
job->mpmd_set->start_pe[j] = i;
job->mpmd_set->total_pe[j]++;
} else if (!xstrcmp(tmp_cmd[i-1], tmp_cmd[i]) &&
!xstrcmp(tmp_args[i-1], tmp_args[i]) &&
!xstrchr(tmp_args[i-1], '%')) {
if ((ranks_node_id[i] == job->nodeid) &&
(job->mpmd_set->first_pe[j] == -1))
job->mpmd_set->first_pe[j] = i;
job->mpmd_set->total_pe[j]++;
} else {
j++;
if (ranks_node_id[i] == job->nodeid)
job->mpmd_set->first_pe[j] = i;
else
job->mpmd_set->first_pe[j] = -1;
job->mpmd_set->num_cmds++;
job->mpmd_set->args[j] = xstrdup(tmp_args[i]);
job->mpmd_set->command[j] = xstrdup(tmp_cmd[i]);
job->mpmd_set->start_pe[j] = i;
job->mpmd_set->total_pe[j]++;
}
}
#if _DEBUG
info("MPMD Apid:%"PRIu64"", job->mpmd_set->apid);
info("MPMD NumPEs:%u", job->ntasks); /* Total rank count */
info("MPMD NumPEsHere:%u", job->node_tasks); /* Node's rank count */
info("MPMD NumCmds:%d", job->mpmd_set->num_cmds);
for (i = 0; i < job->mpmd_set->num_cmds; i++) {
info("MPMD Cmd:%s Args:%s FirstPE:%d StartPE:%d TotalPEs:%d ",
job->mpmd_set->command[i], job->mpmd_set->args[i],
job->mpmd_set->first_pe[i], job->mpmd_set->start_pe[i],
job->mpmd_set->total_pe[i]);
}
for (i = 0; i < job->ntasks; i++) {
info("MPMD Placement[%d]:nid%5.5d",
i, job->mpmd_set->placement[i]);
}
#endif
fini: for (i = 0; i < job->ntasks; i++) {
xfree(tmp_args[i]);
xfree(tmp_cmd[i]);
}
xfree(tmp_args);
xfree(tmp_cmd);
xfree(local_data);
xfree(node_id2nid);
xfree(ranks_node_id);
return;
fail: error("Invalid MPMD configuration line %d", line_num);
goto fini;
}
static void _validate_switches(void)
{
slurm_conf_switches_t *ptr, **ptr_array;
int depth, i, j;
struct switch_record *switch_ptr, *prior_ptr;
hostlist_t hl, invalid_hl = NULL;
char *child, *buf;
bool have_root = false;
bitstr_t *multi_homed_bitmap = NULL; /* nodes on >1 leaf switch */
bitstr_t *switches_bitmap = NULL; /* nodes on any leaf switch */
bitstr_t *tmp_bitmap = NULL;
_free_switch_record_table();
switch_record_cnt = _read_topo_file(&ptr_array);
if (switch_record_cnt == 0) {
error("No switches configured");
s_p_hashtbl_destroy(conf_hashtbl);
return;
}
switch_record_table = xmalloc(sizeof(struct switch_record) *
switch_record_cnt);
multi_homed_bitmap = bit_alloc(node_record_count);
switch_ptr = switch_record_table;
for (i=0; i<switch_record_cnt; i++, switch_ptr++) {
ptr = ptr_array[i];
switch_ptr->name = xstrdup(ptr->switch_name);
/* See if switch name has already been defined. */
prior_ptr = switch_record_table;
for (j=0; j<i; j++, prior_ptr++) {
if (strcmp(switch_ptr->name, prior_ptr->name) == 0) {
fatal("Switch (%s) has already been defined",
prior_ptr->name);
}
}
switch_ptr->link_speed = ptr->link_speed;
if (ptr->nodes) {
switch_ptr->level = 0; /* leaf switch */
switch_ptr->nodes = xstrdup(ptr->nodes);
if (_node_name2bitmap(ptr->nodes,
&switch_ptr->node_bitmap,
&invalid_hl)) {
fatal("Invalid node name (%s) in switch "
"config (%s)",
ptr->nodes, ptr->switch_name);
}
if (switches_bitmap) {
tmp_bitmap = bit_copy(switch_ptr->node_bitmap);
bit_and(tmp_bitmap, switches_bitmap);
bit_or(multi_homed_bitmap, tmp_bitmap);
FREE_NULL_BITMAP(tmp_bitmap);
bit_or(switches_bitmap,
switch_ptr->node_bitmap);
} else {
switches_bitmap = bit_copy(switch_ptr->
node_bitmap);
}
} else if (ptr->switches) {
switch_ptr->level = -1; /* determine later */
switch_ptr->switches = xstrdup(ptr->switches);
} else {
fatal("Switch configuration (%s) lacks children",
ptr->switch_name);
}
}
for (depth=1; ; depth++) {
bool resolved = true;
switch_ptr = switch_record_table;
for (i=0; i<switch_record_cnt; i++, switch_ptr++) {
if (switch_ptr->level != -1)
continue;
hl = hostlist_create(switch_ptr->switches);
if (!hl) {
fatal("Invalid switches: %s",
switch_ptr->switches);
}
while ((child = hostlist_pop(hl))) {
j = _get_switch_inx(child);
if ((j < 0) || (j == i)) {
fatal("Switch configuration %s has "
"invalid child (%s)",
switch_ptr->name, child);
}
if (switch_record_table[j].level == -1) {
/* Children not resolved */
resolved = false;
switch_ptr->level = -1;
FREE_NULL_BITMAP(switch_ptr->
node_bitmap);
free(child);
break;
}
if (switch_ptr->level == -1) {
switch_ptr->level = 1 +
switch_record_table[j].level;
switch_ptr->node_bitmap =
bit_copy(switch_record_table[j].
node_bitmap);
} else {
switch_ptr->level =
MAX(switch_ptr->level,
(switch_record_table[j].
level + 1));
bit_or(switch_ptr->node_bitmap,
switch_record_table[j].
node_bitmap);
}
free(child);
}
hostlist_destroy(hl);
}
if (resolved)
break;
if (depth > 20) /* Prevent infinite loop */
fatal("Switch configuration is not a tree");
}
switch_levels = 0;
switch_ptr = switch_record_table;
for (i=0; i<switch_record_cnt; i++, switch_ptr++) {
switch_levels = MAX(switch_levels, switch_ptr->level);
if (switch_ptr->node_bitmap == NULL)
error("switch %s has no nodes", switch_ptr->name);
}
if (switches_bitmap) {
bit_not(switches_bitmap);
i = bit_set_count(switches_bitmap);
if (i > 0) {
child = bitmap2node_name(switches_bitmap);
error("WARNING: switches lack access to %d nodes: %s",
i, child);
xfree(child);
}
FREE_NULL_BITMAP(switches_bitmap);
} else
fatal("switches contain no nodes");
if (invalid_hl) {
buf = hostlist_ranged_string_xmalloc(invalid_hl);
error("WARNING: Invalid hostnames in switch configuration: %s",
buf);
xfree(buf);
hostlist_destroy(invalid_hl);
}
/* Report nodes on multiple leaf switches,
* possibly due to bad configuration file */
i = bit_set_count(multi_homed_bitmap);
if (i > 0) {
child = bitmap2node_name(multi_homed_bitmap);
error("WARNING: Multiple leaf switches contain nodes: %s",
child);
xfree(child);
}
FREE_NULL_BITMAP(multi_homed_bitmap);
/* Create array of indexes of children of each switch,
* and see if any switch can reach all nodes */
for (i = 0; i < switch_record_cnt; i++) {
if (switch_record_table[i].level != 0) {
_find_child_switches (i);
}
if (node_record_count ==
bit_set_count(switch_record_table[i].node_bitmap)) {
have_root = true;
}
}
if (!have_root) {
info("TOPOLOGY: warning -- no switch can reach all nodes"
" through its descendants."
"Do not use route/topology");
}
s_p_hashtbl_destroy(conf_hashtbl);
_log_switches();
}
