static void _atfork_child() { slurm_mutex_unlock(&log_lock); }
/* Perform job initiation work */
static void _start_agent(bg_action_t *bg_action_ptr)
{
int rc, set_user_rc = SLURM_SUCCESS;
bg_record_t *bg_record = NULL;
bg_record_t *found_record = NULL;
ListIterator itr;
List delete_list = NULL;
int requeue_job = 0;
uint32_t req_job_id = bg_action_ptr->job_ptr->job_id;
bool block_inited = 0;
bool delete_it = 0;
slurm_mutex_lock(&block_state_mutex);
bg_record = find_bg_record_in_list(bg_lists->main,
bg_action_ptr->bg_block_id);
if (!bg_record) {
bg_record->modifying = 0;
slurm_mutex_unlock(&block_state_mutex);
error("block %s not found in bg_lists->main",
bg_action_ptr->bg_block_id);
bg_requeue_job(req_job_id, 1, 0, JOB_BOOT_FAIL, false);
return;
}
if ((bg_record->job_running <= NO_JOB_RUNNING)
&& !find_job_in_bg_record(bg_record, req_job_id)) {
bg_record->modifying = 0;
// bg_reset_block(bg_record); should already happened
slurm_mutex_unlock(&block_state_mutex);
debug("job %u finished during the queueing job "
"(everything is ok)",
req_job_id);
return;
}
if ((bg_record->state == BG_BLOCK_TERM) || bg_record->free_cnt) {
/* It doesn't appear state of a small block
(conn_type) is held on a BGP system so
if we to reset it so, just set the reboot flag and
handle it later in that code. */
bg_action_ptr->reboot = 1;
}
delete_list = list_create(NULL);
itr = list_iterator_create(bg_lists->main);
while ((found_record = list_next(itr))) {
if (bg_record == found_record)
continue;
if (!blocks_overlap(bg_record, found_record)) {
debug2("block %s isn't part of %s",
found_record->bg_block_id,
bg_record->bg_block_id);
continue;
}
if (found_record->job_ptr
|| (found_record->job_list
&& list_count(found_record->job_list))) {
struct job_record *job_ptr = found_record->job_ptr;
if (!found_record->job_ptr)
job_ptr = find_job_in_bg_record(
found_record, NO_VAL);
error("Trying to start job %u on block %s, "
"but there is a job %u running on an overlapping "
"block %s it will not end until %ld. "
"This should never happen.",
req_job_id,
bg_record->bg_block_id,
job_ptr->job_id,
found_record->bg_block_id,
job_ptr->end_time);
requeue_job = 1;
break;
}
debug2("need to make sure %s is free, it's part of %s",
found_record->bg_block_id,
bg_record->bg_block_id);
list_push(delete_list, found_record);
}
list_iterator_destroy(itr);
if (requeue_job) {
FREE_NULL_LIST(delete_list);
bg_reset_block(bg_record, bg_action_ptr->job_ptr);
bg_record->modifying = 0;
slurm_mutex_unlock(&block_state_mutex);
bg_requeue_job(req_job_id, 0, 0, JOB_BOOT_FAIL, false);
return;
}
slurm_mutex_unlock(&block_state_mutex);
if (bg_conf->layout_mode == LAYOUT_DYNAMIC)
delete_it = 1;
free_block_list(req_job_id, delete_list, delete_it, 1);
FREE_NULL_LIST(delete_list);
while (1) {
slurm_mutex_lock(&block_state_mutex);
/* Failure will unlock block_state_mutex so no need to
unlock before return. No need to reset modifying
here if the block doesn't exist.
*/
if (!_make_sure_block_still_exists(bg_action_ptr, bg_record)) {
error("Problem with deallocating blocks to run job %u "
"on block %s", req_job_id,
bg_action_ptr->bg_block_id);
return;
}
/* If another thread is freeing this block we need to
wait until it is done or we will get into a state
where this job will be killed.
*/
if (!bg_record->free_cnt)
break;
debug("Waiting for block %s to free for job %u. "
"%d thread(s) trying to free it",
bg_record->bg_block_id, req_job_id,
bg_record->free_cnt);
slurm_mutex_unlock(&block_state_mutex);
sleep(1);
}
/* This was set in the start_job function to close the above
window where a job could be mistakenly requeued if another
thread is trying to free this block as we are trying to run
on it, which is fine since we will reboot it later.
*/
bg_record->modifying = 0;
if ((bg_record->job_running <= NO_JOB_RUNNING)
&& !find_job_in_bg_record(bg_record, req_job_id)) {
// bg_reset_block(bg_record); should already happened
slurm_mutex_unlock(&block_state_mutex);
debug("job %u already finished before boot",
req_job_id);
return;
}
if (bg_record->job_list
&& (bg_action_ptr->job_ptr->total_cpus != bg_record->cpu_cnt)
&& (list_count(bg_record->job_list) != 1)) {
/* We don't allow modification of a block or reboot of
a block if we are running multiple jobs on the
block.
*/
debug2("no reboot");
goto no_reboot;
}
rc = 0;
#ifdef HAVE_BGL
if (bg_action_ptr->blrtsimage
&& xstrcasecmp(bg_action_ptr->blrtsimage, bg_record->blrtsimage)) {
debug3("changing BlrtsImage from %s to %s",
bg_record->blrtsimage, bg_action_ptr->blrtsimage);
xfree(bg_record->blrtsimage);
bg_record->blrtsimage = xstrdup(bg_action_ptr->blrtsimage);
rc = 1;
}
#elif defined HAVE_BGP
if ((bg_action_ptr->conn_type[0] >= SELECT_SMALL)
&& (bg_action_ptr->conn_type[0] != bg_record->conn_type[0])) {
if (bg_conf->slurm_debug_level >= LOG_LEVEL_DEBUG3) {
char *req_conn_type =
conn_type_string_full(bg_action_ptr->conn_type);
char *conn_type =
conn_type_string_full(bg_record->conn_type);
debug3("changing small block mode from %s to %s",
conn_type, req_conn_type);
xfree(req_conn_type);
xfree(conn_type);
}
rc = 1;
# ifndef HAVE_BG_FILES
/* since we don't check state on an emulated system we
* have to change it here
*/
bg_record->conn_type[0] = bg_action_ptr->conn_type[0];
# endif
}
#endif
#ifdef HAVE_BG_L_P
if (bg_action_ptr->linuximage
&& xstrcasecmp(bg_action_ptr->linuximage, bg_record->linuximage)) {
# ifdef HAVE_BGL
debug3("changing LinuxImage from %s to %s",
bg_record->linuximage, bg_action_ptr->linuximage);
# else
debug3("changing CnloadImage from %s to %s",
bg_record->linuximage, bg_action_ptr->linuximage);
# endif
xfree(bg_record->linuximage);
bg_record->linuximage = xstrdup(bg_action_ptr->linuximage);
rc = 1;
}
if (bg_action_ptr->ramdiskimage
&& xstrcasecmp(bg_action_ptr->ramdiskimage,
bg_record->ramdiskimage)) {
# ifdef HAVE_BGL
debug3("changing RamDiskImage from %s to %s",
bg_record->ramdiskimage, bg_action_ptr->ramdiskimage);
# else
debug3("changing IoloadImage from %s to %s",
bg_record->ramdiskimage, bg_action_ptr->ramdiskimage);
# endif
xfree(bg_record->ramdiskimage);
bg_record->ramdiskimage = xstrdup(bg_action_ptr->ramdiskimage);
rc = 1;
}
#endif
if (bg_action_ptr->mloaderimage
&& xstrcasecmp(bg_action_ptr->mloaderimage,
bg_record->mloaderimage)) {
debug3("changing MloaderImage from %s to %s",
bg_record->mloaderimage, bg_action_ptr->mloaderimage);
xfree(bg_record->mloaderimage);
bg_record->mloaderimage = xstrdup(bg_action_ptr->mloaderimage);
rc = 1;
}
if (rc || bg_action_ptr->reboot) {
bg_record->modifying = 1;
/* Increment free_cnt to make sure we don't loose this
* block since bg_free_block will unlock block_state_mutex.
*/
bg_record->free_cnt++;
bg_free_block(bg_record, 1, 1);
bg_record->free_cnt--;
#if defined HAVE_BG_FILES && defined HAVE_BG_L_P
#ifdef HAVE_BGL
if ((rc = bridge_block_modify(bg_record->bg_block_id,
RM_MODIFY_BlrtsImg,
bg_record->blrtsimage))
!= SLURM_SUCCESS)
error("bridge_block_modify(RM_MODIFY_BlrtsImg): %s",
bg_err_str(rc));
if ((rc = bridge_block_modify(bg_record->bg_block_id,
RM_MODIFY_LinuxImg,
bg_record->linuximage))
!= SLURM_SUCCESS)
error("bridge_block_modify(RM_MODIFY_LinuxImg): %s",
bg_err_str(rc));
if ((rc = bridge_block_modify(bg_record->bg_block_id,
RM_MODIFY_RamdiskImg,
bg_record->ramdiskimage))
!= SLURM_SUCCESS)
error("bridge_block_modify(RM_MODIFY_RamdiskImg): %s",
bg_err_str(rc));
#elif defined HAVE_BGP
if ((rc = bridge_block_modify(bg_record->bg_block_id,
RM_MODIFY_CnloadImg,
bg_record->linuximage))
!= SLURM_SUCCESS)
error("bridge_block_modify(RM_MODIFY_CnloadImg): %s",
bg_err_str(rc));
if ((rc = bridge_block_modify(bg_record->bg_block_id,
RM_MODIFY_IoloadImg,
bg_record->ramdiskimage))
!= SLURM_SUCCESS)
error("bridge_block_modify(RM_MODIFY_IoloadImg): %s",
bg_err_str(rc));
if (bg_action_ptr->conn_type[0] > SELECT_SMALL) {
char *conn_type = NULL;
switch(bg_action_ptr->conn_type[0]) {
case SELECT_HTC_S:
conn_type = "s";
break;
case SELECT_HTC_D:
conn_type = "d";
break;
case SELECT_HTC_V:
conn_type = "v";
break;
case SELECT_HTC_L:
conn_type = "l";
break;
default:
break;
}
/* the option has to be set before the pool can be
set */
if ((rc = bridge_block_modify(
bg_record->bg_block_id,
RM_MODIFY_Options,
conn_type)) != SLURM_SUCCESS)
error("bridge_set_data(RM_MODIFY_Options): %s",
bg_err_str(rc));
}
#endif
if ((rc = bridge_block_modify(bg_record->bg_block_id,
RM_MODIFY_MloaderImg,
bg_record->mloaderimage))
!= SLURM_SUCCESS)
error("bridge_block_modify(RM_MODIFY_MloaderImg): %s",
bg_err_str(rc));
#endif
bg_record->modifying = 0;
}
no_reboot:
if (bg_record->state == BG_BLOCK_FREE) {
if ((rc = bridge_block_boot(bg_record)) != SLURM_SUCCESS) {
char reason[200];
bg_record->boot_state = 0;
bg_record->boot_count = 0;
if (rc == BG_ERROR_INVALID_STATE)
snprintf(reason, sizeof(reason),
"Block %s is in an incompatible "
"state. This usually means "
"hardware is allocated "
"by another block (maybe outside "
"of SLURM).",
bg_record->bg_block_id);
else
snprintf(reason, sizeof(reason),
"Couldn't boot block %s: %s",
bg_record->bg_block_id,
bg_err_str(rc));
slurm_mutex_unlock(&block_state_mutex);
requeue_and_error(bg_record, reason);
return;
}
} else if (bg_record->state == BG_BLOCK_BOOTING) {
#ifdef HAVE_BG_FILES
bg_record->boot_state = 1;
#else
if (!block_ptr_exist_in_list(bg_lists->booted, bg_record))
list_push(bg_lists->booted, bg_record);
bg_record->state = BG_BLOCK_INITED;
last_bg_update = time(NULL);
#endif
}
if ((bg_record->job_running <= NO_JOB_RUNNING)
&& !find_job_in_bg_record(bg_record, req_job_id)) {
slurm_mutex_unlock(&block_state_mutex);
debug("job %u finished during the start of the boot "
"(everything is ok)",
req_job_id);
return;
}
/* Don't reset boot_count, it will be reset when state
changes, and needs to outlast a job allocation.
*/
/* bg_record->boot_count = 0; */
if (bg_record->state == BG_BLOCK_INITED) {
debug("block %s is already ready.", bg_record->bg_block_id);
/* Just in case reset the boot flags */
bg_record->boot_state = 0;
bg_record->boot_count = 0;
set_user_rc = bridge_block_sync_users(bg_record);
block_inited = 1;
}
slurm_mutex_unlock(&block_state_mutex);
/* This lock needs to happen after the block_state_mutex to
avoid deadlock.
*/
if (block_inited && bg_action_ptr->job_ptr) {
slurmctld_lock_t job_write_lock = {
NO_LOCK, WRITE_LOCK, NO_LOCK, NO_LOCK, NO_LOCK };
lock_slurmctld(job_write_lock);
bg_action_ptr->job_ptr->job_state &= (~JOB_CONFIGURING);
last_job_update = time(NULL);
unlock_slurmctld(job_write_lock);
}
if (set_user_rc == SLURM_ERROR) {
sleep(2);
/* wait for the slurmd to begin
the batch script, slurm_fail_job()
is a no-op if issued prior
to the script initiation do clean up just
in case the fail job isn't ran */
(void) slurm_fail_job(req_job_id, JOB_BOOT_FAIL);
}
}
/*
* Synchronize BG block state to that of currently active jobs.
* This can recover from slurmctld crashes when block usership
* changes were queued
*/
extern int sync_jobs(List job_list)
{
ListIterator itr;
struct job_record *job_ptr = NULL;
List block_list = NULL, kill_list = NULL;
static bool run_already = false;
bg_record_t *bg_record = NULL;
/* Execute only on initial startup. We don't support bgblock
* creation on demand today, so there is no need to re-sync data. */
if (run_already)
return SLURM_SUCCESS;
run_already = true;
if (!job_list) {
error("sync_jobs: no job_list");
return SLURM_ERROR;
}
slurm_mutex_lock(&block_state_mutex);
/* Insure that all running jobs own the specified block */
itr = list_iterator_create(job_list);
while ((job_ptr = list_next(itr))) {
bg_action_t *bg_action_ptr = NULL;
if (!IS_JOB_RUNNING(job_ptr) && !IS_JOB_COMPLETING(job_ptr))
continue;
bg_action_ptr = xmalloc(sizeof(bg_action_t));
if (IS_JOB_COMPLETING(job_ptr))
bg_action_ptr->op = TERM_OP;
else
bg_action_ptr->op = START_OP;
bg_action_ptr->job_ptr = job_ptr;
get_select_jobinfo(job_ptr->select_jobinfo->data,
SELECT_JOBDATA_BLOCK_ID,
&(bg_action_ptr->bg_block_id));
#ifdef HAVE_BG_L_P
# ifdef HAVE_BGL
get_select_jobinfo(job_ptr->select_jobinfo->data,
SELECT_JOBDATA_BLRTS_IMAGE,
&(bg_action_ptr->blrtsimage));
# else
get_select_jobinfo(job_ptr->select_jobinfo->data,
SELECT_JOBDATA_CONN_TYPE,
&(bg_action_ptr->conn_type));
# endif
get_select_jobinfo(job_ptr->select_jobinfo->data,
SELECT_JOBDATA_LINUX_IMAGE,
&(bg_action_ptr->linuximage));
get_select_jobinfo(job_ptr->select_jobinfo->data,
SELECT_JOBDATA_RAMDISK_IMAGE,
&(bg_action_ptr->ramdiskimage));
#endif
get_select_jobinfo(job_ptr->select_jobinfo->data,
SELECT_JOBDATA_MLOADER_IMAGE,
&(bg_action_ptr->mloaderimage));
if (bg_action_ptr->bg_block_id == NULL) {
error("Running job %u has bgblock==NULL",
job_ptr->job_id);
} else if (job_ptr->nodes == NULL) {
error("Running job %u has nodes==NULL",
job_ptr->job_id);
} else if (!(bg_record = find_bg_record_in_list(
bg_lists->main,
bg_action_ptr->bg_block_id))) {
error("Kill job %u belongs to defunct "
"bgblock %s",
job_ptr->job_id,
bg_action_ptr->bg_block_id);
}
if (!bg_record) {
/* Can't fail it just now, we have locks in
place. */
bg_status_add_job_kill_list(job_ptr, &kill_list);
_destroy_bg_action(bg_action_ptr);
continue;
}
/* _sync_agent will destroy the bg_action_ptr */
_sync_agent(bg_action_ptr, bg_record);
}
list_iterator_destroy(itr);
block_list = list_create(destroy_bg_record);
itr = list_iterator_create(bg_lists->main);
while ((bg_record = list_next(itr))) {
bg_record_t *rm_record;
if (bg_record->job_ptr
|| (bg_record->job_list
&& list_count(bg_record->job_list)))
continue;
rm_record = xmalloc(sizeof(bg_record_t));
rm_record->magic = BLOCK_MAGIC;
rm_record->bg_block_id = xstrdup(bg_record->bg_block_id);
rm_record->mp_str = xstrdup(bg_record->mp_str);
list_append(block_list, rm_record);
}
list_iterator_destroy(itr);
slurm_mutex_unlock(&block_state_mutex);
if (kill_list) {
/* slurmctld is already locked up, so handle this right after
* the unlock of block_state_mutex.
*/
bg_status_process_kill_job_list(kill_list, JOB_BOOT_FAIL, 1);
FREE_NULL_LIST(kill_list);
}
/* Insure that all other blocks are free of users */
if (block_list) {
itr = list_iterator_create(block_list);
while ((bg_record = list_next(itr))) {
info("Queue clearing of users of BG block %s",
bg_record->bg_block_id);
term_jobs_on_block(bg_record->bg_block_id);
}
list_iterator_destroy(itr);
FREE_NULL_LIST(block_list);
} else {
/* this should never happen,
* vestigial logic */
error("sync_jobs: no block_list");
return SLURM_ERROR;
}
return SLURM_SUCCESS;
}
/*
* init_power_save - Initialize the power save module. Started as a
* pthread. Terminates automatically at slurmctld shutdown time.
* Input and output are unused.
*/
static void *_init_power_save(void *arg)
{
/* Locks: Read nodes */
slurmctld_lock_t node_read_lock = {
NO_LOCK, READ_LOCK, NO_LOCK, NO_LOCK };
/* Locks: Write nodes */
slurmctld_lock_t node_write_lock = {
NO_LOCK, WRITE_LOCK, NO_LOCK, NO_LOCK };
time_t now, boot_time = 0, last_power_scan = 0;
if (_init_power_config())
goto fini;
suspend_node_bitmap = bit_alloc(node_record_count);
resume_node_bitmap = bit_alloc(node_record_count);
while (slurmctld_config.shutdown_time == 0) {
sleep(1);
if (_reap_procs() < 2) {
debug("power_save programs getting backlogged");
continue;
}
if ((last_config != slurmctld_conf.last_update) &&
(_init_power_config())) {
info("power_save mode has been disabled due to "
"configuration changes");
goto fini;
}
now = time(NULL);
if (boot_time == 0)
boot_time = now;
/* Only run every 60 seconds or after a node state change,
* whichever happens first */
if ((last_node_update >= last_power_scan) ||
(now >= (last_power_scan + 60))) {
lock_slurmctld(node_write_lock);
_do_power_work(now);
unlock_slurmctld(node_write_lock);
last_power_scan = now;
}
if (slurmd_timeout &&
(now > (boot_time + (slurmd_timeout / 2)))) {
lock_slurmctld(node_read_lock);
_re_wake();
unlock_slurmctld(node_read_lock);
/* prevent additional executions */
boot_time += (365 * 24 * 60 * 60);
slurmd_timeout = 0;
}
}
fini: _clear_power_config();
FREE_NULL_BITMAP(suspend_node_bitmap);
FREE_NULL_BITMAP(resume_node_bitmap);
_shutdown_power();
slurm_mutex_lock(&power_mutex);
power_save_enabled = false;
slurm_mutex_unlock(&power_mutex);
pthread_exit(NULL);
return NULL;
}
extern int as_mysql_user_no_assocs_or_no_uid(
mysql_conn_t *mysql_conn,
slurmdb_assoc_cond_t *assoc_cond,
List ret_list)
{
int rc = SLURM_SUCCESS;
char *query = NULL;
MYSQL_RES *result = NULL;
MYSQL_ROW row;
List use_cluster_list = as_mysql_cluster_list;
ListIterator itr = NULL;
char *cluster_name = NULL;
xassert(ret_list);
query = xstrdup_printf("select name from %s where deleted=0",
user_table);
if (assoc_cond &&
assoc_cond->user_list && list_count(assoc_cond->user_list)) {
int set = 0;
char *object = NULL;
xstrcat(query, " && (");
itr = list_iterator_create(assoc_cond->user_list);
while ((object = list_next(itr))) {
if (set)
xstrcat(query, " || ");
xstrfmtcat(query, "name='%s'", object);
set = 1;
}
list_iterator_destroy(itr);
xstrcat(query, ")");
}
if (!(result = mysql_db_query_ret(
mysql_conn, query, 0))) {
xfree(query);
return SLURM_ERROR;
}
xfree(query);
if (assoc_cond &&
assoc_cond->cluster_list && list_count(assoc_cond->cluster_list))
use_cluster_list = assoc_cond->cluster_list;
else
slurm_mutex_lock(&as_mysql_cluster_list_lock);
itr = list_iterator_create(use_cluster_list);
while ((row = mysql_fetch_row(result))) {
MYSQL_RES *result2 = NULL;
int cnt = 0;
slurmdb_assoc_rec_t *assoc = NULL;
uid_t pw_uid;
if (uid_from_string (row[0], &pw_uid) < 0) {
assoc = xmalloc(sizeof(slurmdb_assoc_rec_t));
list_append(ret_list, assoc);
assoc->id = SLURMDB_PROBLEM_USER_NO_UID;
assoc->user = xstrdup(row[0]);
continue;
}
/* See if we have at least 1 association in the system */
while ((cluster_name = list_next(itr))) {
if (query)
xstrcat(query, " union ");
xstrfmtcat(query,
"select distinct id_assoc from \"%s_%s\" "
"where deleted=0 && "
"user='******'",
cluster_name, assoc_table, row[0]);
}
list_iterator_reset(itr);
if (query)
xstrcat(query, " limit 1");
if (!(result2 = mysql_db_query_ret(
mysql_conn, query, 0))) {
xfree(query);
rc = SLURM_ERROR;
break;
}
xfree(query);
cnt = mysql_num_rows(result2);
mysql_free_result(result2);
if (cnt)
continue;
assoc = xmalloc(sizeof(slurmdb_assoc_rec_t));
list_append(ret_list, assoc);
assoc->id = SLURMDB_PROBLEM_USER_NO_ASSOC;
assoc->user = xstrdup(row[0]);
}
mysql_free_result(result);
list_iterator_destroy(itr);
if (use_cluster_list == as_mysql_cluster_list)
slurm_mutex_unlock(&as_mysql_cluster_list_lock);
return rc;
}
/*
* _get_process_data() - Build a table of all current processes
*
* IN: pid.
*
* OUT: none
*
* THREADSAFE! Only one thread ever gets here.
*
* Assumption:
* Any file with a name of the form "/proc/[0-9]+/stat"
* is a Linux-style stat entry. We disregard the data if they look
* wrong.
*/
static void _get_process_data(void)
{
struct procsinfo proc;
pid_t *pids = NULL;
int npids = 0;
int i;
uint32_t total_job_mem = 0, total_job_vsize = 0;
int pid = 0;
static int processing = 0;
prec_t *prec = NULL;
struct jobacctinfo *jobacct = NULL;
List prec_list = NULL;
ListIterator itr;
ListIterator itr2;
if (!pgid_plugin && (cont_id == (uint64_t)NO_VAL)) {
debug("cont_id hasn't been set yet not running poll");
return;
}
if(processing) {
debug("already running, returning");
return;
}
processing = 1;
prec_list = list_create(_destroy_prec);
if(!pgid_plugin) {
/* get only the processes in the proctrack container */
slurm_container_get_pids(cont_id, &pids, &npids);
if (!npids) {
debug4("no pids in this container %"PRIu64"", cont_id);
goto finished;
}
for (i = 0; i < npids; i++) {
pid = pids[i];
if(!getprocs(&proc, sizeof(proc), 0, 0, &pid, 1))
continue; /* Assume the process went away */
prec = xmalloc(sizeof(prec_t));
list_append(prec_list, prec);
prec->pid = proc.pi_pid;
prec->ppid = proc.pi_ppid;
prec->usec = proc.pi_ru.ru_utime.tv_sec +
proc.pi_ru.ru_utime.tv_usec * 1e-6;
prec->ssec = proc.pi_ru.ru_stime.tv_sec +
proc.pi_ru.ru_stime.tv_usec * 1e-6;
prec->pages = proc.pi_majflt;
prec->rss = (proc.pi_trss + proc.pi_drss) * pagesize;
//prec->rss *= 1024;
prec->vsize = (proc.pi_tsize / 1024);
prec->vsize += (proc.pi_dvm * pagesize);
//prec->vsize *= 1024;
/* debug("vsize = %f = (%d/1024)+(%d*%d)", */
/* prec->vsize, proc.pi_tsize, proc.pi_dvm, pagesize); */
}
} else {
while(getprocs(&proc, sizeof(proc), 0, 0, &pid, 1) == 1) {
prec = xmalloc(sizeof(prec_t));
list_append(prec_list, prec);
prec->pid = proc.pi_pid;
prec->ppid = proc.pi_ppid;
prec->usec = proc.pi_ru.ru_utime.tv_sec +
proc.pi_ru.ru_utime.tv_usec * 1e-6;
prec->ssec = proc.pi_ru.ru_stime.tv_sec +
proc.pi_ru.ru_stime.tv_usec * 1e-6;
prec->pages = proc.pi_majflt;
prec->rss = (proc.pi_trss + proc.pi_drss) * pagesize;
//prec->rss *= 1024;
prec->vsize = (proc.pi_tsize / 1024);
prec->vsize += (proc.pi_dvm * pagesize);
//prec->vsize *= 1024;
/* debug("vsize = %f = (%d/1024)+(%d*%d)", */
/* prec->vsize, proc.pi_tsize, proc.pi_dvm, pagesize); */
}
}
if(!list_count(prec_list))
goto finished;
slurm_mutex_lock(&jobacct_lock);
if(!task_list || !list_count(task_list)) {
slurm_mutex_unlock(&jobacct_lock);
goto finished;
}
itr = list_iterator_create(task_list);
while((jobacct = list_next(itr))) {
itr2 = list_iterator_create(prec_list);
while((prec = list_next(itr2))) {
//debug2("pid %d ? %d", prec->ppid, jobacct->pid);
if (prec->pid == jobacct->pid) {
/* find all my descendents */
_get_offspring_data(prec_list, prec,
prec->pid);
/* tally their usage */
jobacct->max_rss = jobacct->tot_rss =
MAX(jobacct->max_rss, (int)prec->rss);
total_job_mem += jobacct->max_rss;
jobacct->max_vsize = jobacct->tot_vsize =
MAX(jobacct->max_vsize,
(int)prec->vsize);
total_job_vsize += prec->vsize;
jobacct->max_pages = jobacct->tot_pages =
MAX(jobacct->max_pages, prec->pages);
jobacct->min_cpu = jobacct->tot_cpu =
MAX(jobacct->min_cpu,
(prec->usec + prec->ssec));
debug2("%d size now %d %d time %d",
jobacct->pid, jobacct->max_rss,
jobacct->max_vsize, jobacct->tot_cpu);
break;
}
}
list_iterator_destroy(itr2);
}
list_iterator_destroy(itr);
slurm_mutex_unlock(&jobacct_lock);
if (jobacct_mem_limit) {
debug("Step %u.%u memory used:%u limit:%u KB",
jobacct_job_id, jobacct_step_id,
total_job_mem, jobacct_mem_limit);
}
if (jobacct_job_id && jobacct_mem_limit &&
(total_job_mem > jobacct_mem_limit)) {
if (jobacct_step_id == NO_VAL) {
error("Job %u exceeded %u KB memory limit, being "
"killed", jobacct_job_id, jobacct_mem_limit);
} else {
error("Step %u.%u exceeded %u KB memory limit, being "
"killed", jobacct_job_id, jobacct_step_id,
jobacct_mem_limit);
}
_acct_kill_step();
} else if (jobacct_job_id && jobacct_vmem_limit &&
(total_job_vsize > jobacct_vmem_limit)) {
if (jobacct_step_id == NO_VAL) {
error("Job %u exceeded %u KB virtual memory limit, "
"being killed", jobacct_job_id,
jobacct_vmem_limit);
} else {
error("Step %u.%u exceeded %u KB virtual memory "
"limit, being killed", jobacct_job_id,
jobacct_step_id, jobacct_vmem_limit);
}
_acct_kill_step();
}
finished:
list_destroy(prec_list);
processing = 0;
return;
}
static int _check_for_booted_overlapping_blocks(
List block_list, ListIterator bg_record_itr,
bg_record_t *bg_record, int overlap_check, List overlapped_list,
uint16_t query_mode)
{
bg_record_t *found_record = NULL;
ListIterator itr = NULL;
int rc = 0;
int overlap = 0;
bool is_test = SELECT_IS_TEST(query_mode);
/* this test only is for actually picking a block not testing */
if (is_test && bg_conf->layout_mode == LAYOUT_DYNAMIC)
return rc;
/* Make sure no other blocks are under this block
are booted and running jobs
*/
itr = list_iterator_create(block_list);
while ((found_record = (bg_record_t*)list_next(itr)) != NULL) {
if ((!found_record->bg_block_id)
|| (bg_record == found_record)) {
if (bg_conf->slurm_debug_flags & DEBUG_FLAG_BG_PICK)
info("Don't need to look at myself %s %s",
bg_record->bg_block_id,
found_record->bg_block_id);
continue;
}
slurm_mutex_lock(&block_state_mutex);
overlap = blocks_overlap(bg_record, found_record);
slurm_mutex_unlock(&block_state_mutex);
if (overlap) {
overlap = 0;
/* make the available time on this block
* (bg_record) the max of this found_record's job
* or the one already set if in overlapped_block_list
* since we aren't setting job_running we
* don't have to remove them since the
* block_list should always be destroyed afterwards.
*/
if (is_test && overlapped_list
&& found_record->job_ptr
&& bg_record->job_running == NO_JOB_RUNNING) {
ListIterator itr = list_iterator_create(
overlapped_list);
bg_record_t *tmp_rec = NULL;
if (bg_conf->slurm_debug_flags
& DEBUG_FLAG_BG_PICK)
info("found overlapping block %s "
"overlapped %s with job %u",
found_record->bg_block_id,
bg_record->bg_block_id,
found_record->job_ptr->job_id);
while ((tmp_rec = list_next(itr))) {
if (tmp_rec == bg_record)
break;
}
list_iterator_destroy(itr);
if (tmp_rec && tmp_rec->job_ptr->end_time
< found_record->job_ptr->end_time)
tmp_rec->job_ptr =
found_record->job_ptr;
else if (!tmp_rec) {
bg_record->job_ptr =
found_record->job_ptr;
list_append(overlapped_list,
bg_record);
}
}
/* We already know this block doesn't work
* right now so we will if there is another
* overlapping block that ends later
*/
if (rc)
continue;
/* This test is here to check if the block we
* chose is not booted or if there is a block
* overlapping that we could avoid freeing if
* we choose something else
*/
if (bg_conf->layout_mode == LAYOUT_OVERLAP
&& ((overlap_check == 0 && bg_record->state
!= BG_BLOCK_INITED)
|| (overlap_check == 1 && found_record->state
!= BG_BLOCK_FREE))) {
if (!is_test) {
rc = 1;
break;
}
}
if (((bg_conf->layout_mode == LAYOUT_DYNAMIC)
|| ((!SELECT_IS_CHECK_FULL_SET(query_mode)
|| SELECT_IS_MODE_RUN_NOW(query_mode))
&& (bg_conf->layout_mode != LAYOUT_DYNAMIC)))
&& ((found_record->job_running != NO_JOB_RUNNING)
|| (found_record->state
& BG_BLOCK_ERROR_FLAG))) {
if ((found_record->job_running
== BLOCK_ERROR_STATE)
|| (found_record->state
& BG_BLOCK_ERROR_FLAG))
error("can't use %s, "
"overlapping block %s "
"is in an error state.",
bg_record->bg_block_id,
found_record->bg_block_id);
else if (bg_conf->slurm_debug_flags
& DEBUG_FLAG_BG_PICK)
info("can't use %s, there is "
"a job (%d) running on "
"an overlapping "
"block %s",
bg_record->bg_block_id,
found_record->job_running,
found_record->bg_block_id);
if (bg_conf->layout_mode == LAYOUT_DYNAMIC) {
List tmp_list = list_create(NULL);
/* this will remove and
* destroy the memory for
* bg_record
*/
list_remove(bg_record_itr);
slurm_mutex_lock(&block_state_mutex);
if (bg_record->original) {
if (bg_conf->slurm_debug_flags
& DEBUG_FLAG_BG_PICK)
info("This was a "
"copy %s",
bg_record->
bg_block_id);
found_record =
bg_record->original;
} else {
if (bg_conf->slurm_debug_flags
& DEBUG_FLAG_BG_PICK)
info("looking for "
"original");
found_record =
find_org_in_bg_list(
bg_lists->main,
bg_record);
}
if (bg_conf->slurm_debug_flags
& DEBUG_FLAG_BG_PICK)
info("Removing unusable block "
"%s from the system.",
bg_record->bg_block_id);
if (!found_record) {
if (bg_conf->slurm_debug_flags
& DEBUG_FLAG_BG_PICK)
info("This record %s "
"wasn't found in "
"the "
"bg_lists->main, "
"no big deal, it "
"probably wasn't "
"added",
bg_record->
bg_block_id);
found_record = bg_record;
} else
destroy_bg_record(bg_record);
list_push(tmp_list, found_record);
slurm_mutex_unlock(&block_state_mutex);
/* We need to make sure if a
job is running here to not
call the regular method since
we are inside the job write
lock already.
*/
if (found_record->job_ptr
&& !IS_JOB_FINISHED(
found_record->job_ptr)) {
info("Somehow block %s "
"is being freed, but "
"appears to already have "
"a job %u(%u) running "
"on it.",
found_record->bg_block_id,
found_record->
job_ptr->job_id,
found_record->job_running);
if (job_requeue(0,
found_record->
job_ptr->job_id,
-1,
(uint16_t)
NO_VAL,
false)) {
error("Couldn't "
"requeue job %u, "
"failing it: %s",
found_record->
job_ptr->job_id,
slurm_strerror(
rc));
job_fail(found_record->
job_ptr->
job_id);
}
}
free_block_list(NO_VAL, tmp_list, 0, 0);
list_destroy(tmp_list);
}
rc = 1;
if (!is_test)
break;
}
}
}
list_iterator_destroy(itr);
return rc;
}
int proctrack_p_get_pids (uint64_t cont_id, pid_t **pids, int *npids)
{
int rc = SLURM_ERROR;
int i = 0;
int t = 0;
pid_t *p;
*npids = 0;
slurm_mutex_lock (&lua_lock);
lua_getglobal (L, "proctrack_g_get_pids");
if (lua_isnil (L, -1))
goto out;
lua_pushnumber (L, cont_id);
if (lua_pcall (L, 1, 1, 0) != 0) {
error ("%s: %s: %s",
"proctrack/lua",
__func__,
lua_tostring (L, -1));
goto out;
}
/*
* list of PIDs should be returned in a table from the lua
* script. If a table wasn't returned then generate an error:
*/
if (!lua_istable(L, -1)) {
error ("%s: %s: function should return a table",
"proctrack/lua",
__func__);
goto out;
}
/*
* Save absolute position of table in lua stack
*/
t = lua_gettop (L);
/*
* Get table size and create array for slurm
*/
*npids = lua_objlen (L, t);
p = (pid_t *) xmalloc (*npids * sizeof (pid_t));
/*
* Traverse table/array at position t on the stack:
*/
lua_pushnil (L);
while (lua_next (L, t)) {
p [i++] = lua_tonumber (L, -1);
/*
* pop value off stack, leave key for lua_next()
*/
lua_pop (L, 1);
}
lua_pop (L, 1);
*pids = p;
rc = SLURM_SUCCESS;
out:
slurm_mutex_unlock (&lua_lock);
return rc;
}
/*
* Initialize the core specialization plugin.
*
* RET - slurm error code
*/
extern int core_spec_g_init(void)
{
int retval = SLURM_SUCCESS;
char *plugin_type = "core_spec";
char *core_spec_plugin_type = NULL;
char *last = NULL, *core_spec_plugin_list, *core_spec = NULL;
if (init_run && (g_core_spec_context_num >= 0))
return retval;
slurm_mutex_lock(&g_core_spec_context_lock);
if (g_core_spec_context_num >= 0)
goto done;
core_spec_plugin_type = slurm_get_core_spec_plugin();
g_core_spec_context_num = 0; /* mark it before anything else */
if ((core_spec_plugin_type == NULL) ||
(core_spec_plugin_type[0] == '\0'))
goto done;
core_spec_plugin_list = core_spec_plugin_type;
while ((core_spec =
strtok_r(core_spec_plugin_list, ",", &last))) {
xrealloc(ops,
sizeof(core_spec_ops_t) *
(g_core_spec_context_num + 1));
xrealloc(g_core_spec_context, (sizeof(plugin_context_t *)
* (g_core_spec_context_num + 1)));
if (xstrncmp(core_spec, "core_spec/", 10) == 0)
core_spec += 10; /* backward compatibility */
core_spec = xstrdup_printf("core_spec/%s",
core_spec);
g_core_spec_context[g_core_spec_context_num] =
plugin_context_create(
plugin_type, core_spec,
(void **)&ops[g_core_spec_context_num],
syms, sizeof(syms));
if (!g_core_spec_context[g_core_spec_context_num]) {
error("cannot create %s context for %s",
plugin_type, core_spec);
xfree(core_spec);
retval = SLURM_ERROR;
break;
}
xfree(core_spec);
g_core_spec_context_num++;
core_spec_plugin_list = NULL; /* for next iteration */
}
init_run = true;
done:
slurm_mutex_unlock(&g_core_spec_context_lock);
xfree(core_spec_plugin_type);
if (retval != SLURM_SUCCESS)
core_spec_g_fini();
return retval;
}
static void *
_cancel_step_id (void *ci)
{
int error_code = SLURM_SUCCESS, i;
job_cancel_info_t *cancel_info = (job_cancel_info_t *)ci;
uint32_t job_id = cancel_info->job_id;
uint32_t step_id = cancel_info->step_id;
bool sig_set = true;
DEF_TIMERS;
if (cancel_info->sig == (uint16_t) NO_VAL) {
cancel_info->sig = SIGKILL;
sig_set = false;
}
if (!cancel_info->job_id_str) {
if (cancel_info->array_job_id &&
(cancel_info->array_task_id == INFINITE)) {
xstrfmtcat(cancel_info->job_id_str, "%u_*",
cancel_info->array_job_id);
} else if (cancel_info->array_job_id) {
xstrfmtcat(cancel_info->job_id_str, "%u_%u",
cancel_info->array_job_id,
cancel_info->array_task_id);
} else {
xstrfmtcat(cancel_info->job_id_str, "%u",
cancel_info->job_id);
}
}
for (i = 0; i < MAX_CANCEL_RETRY; i++) {
if (cancel_info->sig == SIGKILL) {
verbose("Terminating step %s.%u",
cancel_info->job_id_str, step_id);
} else {
verbose("Signal %u to step %s.%u",
cancel_info->sig,
cancel_info->job_id_str, step_id);
}
_add_delay();
START_TIMER;
if ((!sig_set) || opt.ctld)
error_code = slurm_kill_job_step(job_id, step_id,
cancel_info->sig);
else if (cancel_info->sig == SIGKILL)
error_code = slurm_terminate_job_step(job_id, step_id);
else
error_code = slurm_signal_job_step(job_id, step_id,
cancel_info->sig);
END_TIMER;
slurm_mutex_lock(&max_delay_lock);
max_resp_time = MAX(max_resp_time, DELTA_TIMER);
slurm_mutex_unlock(&max_delay_lock);
if ((error_code == 0) ||
((errno != ESLURM_TRANSITION_STATE_NO_UPDATE) &&
(errno != ESLURM_JOB_PENDING)))
break;
verbose("Job is in transistional state, retrying");
sleep(5 + i);
}
if (error_code) {
error_code = slurm_get_errno();
if ((opt.verbose > 0) || (error_code != ESLURM_ALREADY_DONE))
error("Kill job error on job step id %s: %s",
cancel_info->job_id_str,
slurm_strerror(slurm_get_errno()));
if ((error_code == ESLURM_ALREADY_DONE) &&
(cancel_info->sig == SIGKILL)) {
error_code = 0; /* Ignore error if job done */
}
}
/* Purposely free the struct passed in here, so the caller doesn't have
* to keep track of it, but don't destroy the mutex and condition
* variables contained. */
slurm_mutex_lock(cancel_info->num_active_threads_lock);
*(cancel_info->rc) = MAX(*(cancel_info->rc), error_code);
(*(cancel_info->num_active_threads))--;
slurm_cond_signal(cancel_info->num_active_threads_cond);
slurm_mutex_unlock(cancel_info->num_active_threads_lock);
xfree(cancel_info->job_id_str);
xfree(cancel_info);
return NULL;
}
/*
* _thread_update_node_energy calls _read_ipmi_values and updates all values
* for node consumption
*/
static int _update_node_filesystem(void)
{
static acct_filesystem_data_t fls;
static acct_filesystem_data_t current;
static acct_filesystem_data_t previous;
static bool first = true;
int cc;
slurm_mutex_lock(&lustre_lock);
cc = _read_lustre_counters();
if (cc != SLURM_SUCCESS) {
error("%s: Cannot read lustre counters", __func__);
slurm_mutex_unlock(&lustre_lock);
return SLURM_FAILURE;
}
if (first) {
/* First time initialize the counters and return.
*/
previous.reads = lustre_se.all_lustre_nb_reads;
previous.writes = lustre_se.all_lustre_nb_writes;
previous.read_size
= (double)lustre_se.all_lustre_read_bytes/1048576.0;
previous.write_size
= (double)lustre_se.all_lustre_write_bytes/1048576.0;
first = false;
memset(&lustre_se, 0, sizeof(lustre_sens_t));
slurm_mutex_unlock(&lustre_lock);
return SLURM_SUCCESS;
}
/* Compute the current values read from all lustre-xxxx
* directories
*/
current.reads = lustre_se.all_lustre_nb_reads;
current.writes = lustre_se.all_lustre_nb_writes;
current.read_size = (double)lustre_se.all_lustre_read_bytes/1048576.0;
current.write_size = (double)lustre_se.all_lustre_write_bytes/1048576.0;
/* Now compute the difference between the two snapshots
* and send it to hdf5 log.
*/
fls.reads = fls.reads + (current.reads - previous.reads);
fls.writes = fls.writes + (current.writes - previous.writes);
fls.read_size = fls.read_size
+ (current.read_size - previous.read_size);
fls.write_size = fls.write_size
+ (current.write_size - previous.write_size);
acct_gather_profile_g_add_sample_data(ACCT_GATHER_PROFILE_LUSTRE, &fls);
/* Save current as previous and clean up the working
* data structure.
*/
memcpy(&previous, ¤t, sizeof(acct_filesystem_data_t));
memset(&lustre_se, 0, sizeof(lustre_sens_t));
info("%s: num reads %"PRIu64" nums write %"PRIu64" "
"read %f MB wrote %f MB",
__func__, fls.reads, fls.writes, fls.read_size, fls.write_size);
slurm_mutex_unlock(&lustre_lock);
return SLURM_SUCCESS;
}
static void *
_cancel_job_id (void *ci)
{
int error_code = SLURM_SUCCESS, i;
job_cancel_info_t *cancel_info = (job_cancel_info_t *)ci;
bool sig_set = true;
uint16_t flags = 0;
char *job_type = "";
DEF_TIMERS;
if (cancel_info->sig == (uint16_t) NO_VAL) {
cancel_info->sig = SIGKILL;
sig_set = false;
}
if (opt.batch) {
flags |= KILL_JOB_BATCH;
job_type = "batch ";
}
if (opt.full) {
flags |= KILL_FULL_JOB;
job_type = "full ";
}
if (cancel_info->array_flag)
flags |= KILL_JOB_ARRAY;
if (!cancel_info->job_id_str) {
if (cancel_info->array_job_id &&
(cancel_info->array_task_id == INFINITE)) {
xstrfmtcat(cancel_info->job_id_str, "%u_*",
cancel_info->array_job_id);
} else if (cancel_info->array_job_id) {
xstrfmtcat(cancel_info->job_id_str, "%u_%u",
cancel_info->array_job_id,
cancel_info->array_task_id);
} else {
xstrfmtcat(cancel_info->job_id_str, "%u",
cancel_info->job_id);
}
}
if (!sig_set) {
verbose("Terminating %sjob %s", job_type,
cancel_info->job_id_str);
} else {
verbose("Signal %u to %sjob %s", cancel_info->sig, job_type,
cancel_info->job_id_str);
}
for (i = 0; i < MAX_CANCEL_RETRY; i++) {
_add_delay();
START_TIMER;
error_code = slurm_kill_job2(cancel_info->job_id_str,
cancel_info->sig, flags);
END_TIMER;
slurm_mutex_lock(&max_delay_lock);
max_resp_time = MAX(max_resp_time, DELTA_TIMER);
slurm_mutex_unlock(&max_delay_lock);
if ((error_code == 0) ||
(errno != ESLURM_TRANSITION_STATE_NO_UPDATE))
break;
verbose("Job is in transistional state, retrying");
sleep(5 + i);
}
if (error_code) {
error_code = slurm_get_errno();
if ((opt.verbose > 0) ||
((error_code != ESLURM_ALREADY_DONE) &&
(error_code != ESLURM_INVALID_JOB_ID))) {
error("Kill job error on job id %s: %s",
cancel_info->job_id_str,
slurm_strerror(slurm_get_errno()));
}
if (((error_code == ESLURM_ALREADY_DONE) ||
(error_code == ESLURM_INVALID_JOB_ID)) &&
(cancel_info->sig == SIGKILL)) {
error_code = 0; /* Ignore error if job done */
}
}
/* Purposely free the struct passed in here, so the caller doesn't have
* to keep track of it, but don't destroy the mutex and condition
* variables contained. */
slurm_mutex_lock(cancel_info->num_active_threads_lock);
*(cancel_info->rc) = MAX(*(cancel_info->rc), error_code);
(*(cancel_info->num_active_threads))--;
slurm_cond_signal(cancel_info->num_active_threads_cond);
slurm_mutex_unlock(cancel_info->num_active_threads_lock);
xfree(cancel_info->job_id_str);
xfree(cancel_info);
return NULL;
}
static void
_cancel_jobs_by_state(uint32_t job_state, int filter_cnt, int *rc)
{
int i, err;
job_cancel_info_t *cancel_info;
job_info_t *job_ptr = job_buffer_ptr->job_array;
pthread_t dummy;
/* Spawn a thread to cancel each job or job step marked for
* cancellation */
if (opt.job_cnt) {
_cancel_jobid_by_state(job_state, filter_cnt, rc);
return;
}
for (i = 0; i < job_buffer_ptr->record_count; i++, job_ptr++) {
if (IS_JOB_FINISHED(job_ptr))
job_ptr->job_id = 0;
if (job_ptr->job_id == 0)
continue;
if ((job_state < JOB_END) &&
(job_ptr->job_state != job_state))
continue;
if (opt.interactive &&
(_confirmation(job_ptr, SLURM_BATCH_SCRIPT) == 0)) {
job_ptr->job_id = 0;
continue;
}
cancel_info = (job_cancel_info_t *)
xmalloc(sizeof(job_cancel_info_t));
cancel_info->job_id_str = _build_jobid_str(job_ptr);
cancel_info->rc = rc;
cancel_info->sig = opt.signal;
cancel_info->num_active_threads = &num_active_threads;
cancel_info->num_active_threads_lock =
&num_active_threads_lock;
cancel_info->num_active_threads_cond =
&num_active_threads_cond;
slurm_mutex_lock(&num_active_threads_lock);
num_active_threads++;
while (num_active_threads > MAX_THREADS) {
slurm_cond_wait(&num_active_threads_cond,
&num_active_threads_lock);
}
slurm_mutex_unlock(&num_active_threads_lock);
err = pthread_create(&dummy, &attr, _cancel_job_id,cancel_info);
if (err) /* Run in-line if thread create fails */
_cancel_job_id(cancel_info);
job_ptr->job_id = 0;
if (opt.interactive) {
/* Print any error message for first job before
* starting confirmation of next job */
slurm_mutex_lock(&num_active_threads_lock);
while (num_active_threads > 0) {
slurm_cond_wait(&num_active_threads_cond,
&num_active_threads_lock);
}
slurm_mutex_unlock(&num_active_threads_lock);
}
}
}
static void _cancel_jobid_by_state(uint32_t job_state, int filter_cnt, int *rc)
{
job_cancel_info_t *cancel_info;
job_info_t *job_ptr;
pthread_t dummy;
int err, i, j;
if (opt.job_cnt == 0)
return;
for (j = 0; j < opt.job_cnt; j++) {
if (opt.job_id[j] == 0)
continue;
if ((job_state == JOB_PENDING) && !opt.job_pend[j])
continue;
job_ptr = job_buffer_ptr->job_array;
for (i = 0; i < job_buffer_ptr->record_count; i++, job_ptr++) {
if (IS_JOB_FINISHED(job_ptr))
job_ptr->job_id = 0;
if (job_ptr->job_id == 0)
continue;
if ((opt.step_id[j] != SLURM_BATCH_SCRIPT) &&
IS_JOB_PENDING(job_ptr)) {
/* User specified #.# for step, but the job ID
* may be job array leader with part of job
* array running with other tasks pending */
continue;
}
opt.job_found[j] = false;
if (opt.array_id[j] == NO_VAL) {
if ((opt.job_id[j] == job_ptr->job_id) ||
((opt.job_id[j] == job_ptr->array_job_id) &&
(opt.step_id[j] == SLURM_BATCH_SCRIPT))) {
opt.job_found[j] = true;
}
} else if (opt.array_id[j] == INFINITE) {
if (opt.job_id[j] == job_ptr->array_job_id) {
opt.job_found[j] = true;
}
} else if (opt.job_id[j] != job_ptr->array_job_id) {
continue;
} else if (_is_task_in_job(job_ptr, opt.array_id[j])) {
opt.job_found[j] = true;
}
if (!opt.job_found[j])
continue;
if (opt.interactive &&
(_confirmation(job_ptr, opt.step_id[j]) == 0)) {
job_ptr->job_id = 0; /* Don't check again */
continue;
}
slurm_mutex_lock(&num_active_threads_lock);
num_active_threads++;
while (num_active_threads > MAX_THREADS) {
slurm_cond_wait(&num_active_threads_cond,
&num_active_threads_lock);
}
slurm_mutex_unlock(&num_active_threads_lock);
cancel_info = (job_cancel_info_t *)
xmalloc(sizeof(job_cancel_info_t));
cancel_info->rc = rc;
cancel_info->sig = opt.signal;
cancel_info->num_active_threads = &num_active_threads;
cancel_info->num_active_threads_lock =
&num_active_threads_lock;
cancel_info->num_active_threads_cond =
&num_active_threads_cond;
if (opt.step_id[j] == SLURM_BATCH_SCRIPT) {
cancel_info->job_id_str =
_build_jobid_str(job_ptr);
err = pthread_create(&dummy, &attr,
_cancel_job_id,
cancel_info);
if (err) /* Run in-line as needed */
_cancel_job_id(cancel_info);
job_ptr->job_id = 0;
} else {
cancel_info->job_id = job_ptr->job_id;
cancel_info->step_id = opt.step_id[j];
err = pthread_create(&dummy, &attr,
_cancel_step_id,
cancel_info);
if (err) /* Run in-line as needed */
_cancel_step_id(cancel_info);
}
if (opt.interactive) {
/* Print any error message for first job before
* starting confirmation of next job */
slurm_mutex_lock(&num_active_threads_lock);
while (num_active_threads > 0) {
slurm_cond_wait(&num_active_threads_cond,
&num_active_threads_lock);
}
slurm_mutex_unlock(&num_active_threads_lock);
}
}
}
}
extern void slurmdbd_conf_unlock(void)
{
slurm_mutex_unlock(&conf_mutex);
}
static void *_track_freeing_blocks(void *args)
{
bg_free_block_list_t *bg_free_list = (bg_free_block_list_t *)args;
List track_list = bg_free_list->track_list;
bool destroy = bg_free_list->destroy;
uint32_t job_id = bg_free_list->job_id;
int retry_cnt = 0;
int free_cnt = 0, track_cnt = list_count(track_list);
ListIterator itr = list_iterator_create(track_list);
bg_record_t *bg_record;
bool restore = true;
debug("_track_freeing_blocks: Going to free %d for job %u",
track_cnt, job_id);
while (retry_cnt < MAX_FREE_RETRIES) {
free_cnt = 0;
slurm_mutex_lock(&block_state_mutex);
/* just to make sure state is updated */
bridge_status_update_block_list_state(track_list);
list_iterator_reset(itr);
/* just in case this changes from the update function */
track_cnt = list_count(track_list);
while ((bg_record = list_next(itr))) {
if (bg_record->magic != BLOCK_MAGIC) {
/* update_block_list_state should
remove this already from the list
so we shouldn't ever have this.
*/
error("_track_freeing_blocks: block was "
"already destroyed %p", bg_record);
xassert(0);
free_cnt++;
continue;
}
#ifndef HAVE_BG_FILES
/* Fake a free since we are n deallocating
state before this.
*/
if (!(bg_record->state & BG_BLOCK_ERROR_FLAG)
&& (retry_cnt >= 3))
bg_record->state = BG_BLOCK_FREE;
#endif
if ((bg_record->state == BG_BLOCK_FREE)
|| (bg_record->state & BG_BLOCK_ERROR_FLAG))
free_cnt++;
else if (bg_record->state != BG_BLOCK_TERM)
bg_free_block(bg_record, 0, 1);
}
slurm_mutex_unlock(&block_state_mutex);
if (free_cnt == track_cnt)
break;
debug("_track_freeing_blocks: freed %d of %d for job %u",
free_cnt, track_cnt, job_id);
sleep(FREE_SLEEP_INTERVAL);
retry_cnt++;
}
debug("_track_freeing_blocks: Freed them all for job %u", job_id);
if (destroy)
restore = false;
/* If there is a block in error state we need to keep all
* these blocks around. */
slurm_mutex_lock(&block_state_mutex);
list_iterator_reset(itr);
while ((bg_record = list_next(itr))) {
/* block no longer exists */
if (bg_record->magic != BLOCK_MAGIC)
continue;
if (bg_record->state != BG_BLOCK_FREE) {
restore = true;
break;
}
}
list_iterator_reset(itr);
while ((bg_record = list_next(itr)))
_post_block_free(bg_record, restore);
slurm_mutex_unlock(&block_state_mutex);
last_bg_update = time(NULL);
list_iterator_destroy(itr);
FREE_NULL_LIST(track_list);
xfree(bg_free_list);
return NULL;
}
extern List as_mysql_get_resvs(mysql_conn_t *mysql_conn, uid_t uid,
slurmdb_reservation_cond_t *resv_cond)
{
//DEF_TIMERS;
char *query = NULL;
char *extra = NULL;
char *tmp = NULL;
List resv_list = NULL;
int i=0, is_admin=1;
MYSQL_RES *result = NULL;
MYSQL_ROW row;
uint16_t private_data = 0;
slurmdb_job_cond_t job_cond;
void *curr_cluster = NULL;
List local_cluster_list = NULL;
List use_cluster_list = as_mysql_cluster_list;
ListIterator itr = NULL;
char *cluster_name = NULL;
/* needed if we don't have an resv_cond */
uint16_t with_usage = 0;
/* if this changes you will need to edit the corresponding enum */
char *resv_req_inx[] = {
"id_resv",
"assoclist",
"cpus",
"flags",
"nodelist",
"node_inx",
"resv_name",
"time_start",
"time_end",
};
enum {
RESV_REQ_ID,
RESV_REQ_ASSOCS,
RESV_REQ_CPUS,
RESV_REQ_FLAGS,
RESV_REQ_NODES,
RESV_REQ_NODE_INX,
RESV_REQ_NAME,
RESV_REQ_START,
RESV_REQ_END,
RESV_REQ_COUNT
};
if (!resv_cond) {
xstrcat(extra, " where deleted=0");
goto empty;
}
if (check_connection(mysql_conn) != SLURM_SUCCESS)
return NULL;
private_data = slurm_get_private_data();
if (private_data & PRIVATE_DATA_RESERVATIONS) {
if (!(is_admin = is_user_min_admin_level(
mysql_conn, uid, SLURMDB_ADMIN_OPERATOR))) {
error("Only admins can look at reservations");
errno = ESLURM_ACCESS_DENIED;
return NULL;
}
}
memset(&job_cond, 0, sizeof(slurmdb_job_cond_t));
if (resv_cond->nodes) {
job_cond.usage_start = resv_cond->time_start;
job_cond.usage_end = resv_cond->time_end;
job_cond.used_nodes = resv_cond->nodes;
job_cond.cluster_list = resv_cond->cluster_list;
local_cluster_list = setup_cluster_list_with_inx(
mysql_conn, &job_cond, (void **)&curr_cluster);
} else if (with_usage) {
job_cond.usage_start = resv_cond->time_start;
job_cond.usage_end = resv_cond->time_end;
}
(void) _setup_resv_cond_limits(resv_cond, &extra);
with_usage = resv_cond->with_usage;
if (resv_cond->cluster_list && list_count(resv_cond->cluster_list))
use_cluster_list = resv_cond->cluster_list;
empty:
xfree(tmp);
xstrfmtcat(tmp, "t1.%s", resv_req_inx[i]);
for(i=1; i<RESV_REQ_COUNT; i++) {
xstrfmtcat(tmp, ", t1.%s", resv_req_inx[i]);
}
if (use_cluster_list == as_mysql_cluster_list)
slurm_mutex_lock(&as_mysql_cluster_list_lock);
itr = list_iterator_create(use_cluster_list);
while ((cluster_name = list_next(itr))) {
if (query)
xstrcat(query, " union ");
//START_TIMER;
xstrfmtcat(query, "select distinct %s,'%s' as cluster "
"from \"%s_%s\" as t1%s",
tmp, cluster_name, cluster_name, resv_table, extra);
}
list_iterator_destroy(itr);
if (use_cluster_list == as_mysql_cluster_list)
slurm_mutex_unlock(&as_mysql_cluster_list_lock);
if (query)
xstrcat(query, " order by cluster, resv_name;");
xfree(tmp);
xfree(extra);
debug3("%d(%s:%d) query\n%s",
mysql_conn->conn, THIS_FILE, __LINE__, query);
if (!(result = mysql_db_query_ret(mysql_conn, query, 0))) {
xfree(query);
if (local_cluster_list)
list_destroy(local_cluster_list);
return NULL;
}
xfree(query);
resv_list = list_create(slurmdb_destroy_reservation_rec);
while ((row = mysql_fetch_row(result))) {
slurmdb_reservation_rec_t *resv =
xmalloc(sizeof(slurmdb_reservation_rec_t));
int start = slurm_atoul(row[RESV_REQ_START]);
list_append(resv_list, resv);
if (!good_nodes_from_inx(local_cluster_list, &curr_cluster,
row[RESV_REQ_NODE_INX], start))
continue;
resv->id = slurm_atoul(row[RESV_REQ_ID]);
if (with_usage) {
if (!job_cond.resvid_list)
job_cond.resvid_list = list_create(NULL);
list_append(job_cond.resvid_list, row[RESV_REQ_ID]);
}
resv->name = xstrdup(row[RESV_REQ_NAME]);
resv->cluster = xstrdup(row[RESV_REQ_COUNT]);
resv->cpus = slurm_atoul(row[RESV_REQ_CPUS]);
resv->assocs = xstrdup(row[RESV_REQ_ASSOCS]);
resv->nodes = xstrdup(row[RESV_REQ_NODES]);
resv->time_start = start;
resv->time_end = slurm_atoul(row[RESV_REQ_END]);
resv->flags = slurm_atoul(row[RESV_REQ_FLAGS]);
}
if (local_cluster_list)
list_destroy(local_cluster_list);
if (with_usage && resv_list && list_count(resv_list)) {
List job_list = as_mysql_jobacct_process_get_jobs(
mysql_conn, uid, &job_cond);
ListIterator itr = NULL, itr2 = NULL;
slurmdb_job_rec_t *job = NULL;
slurmdb_reservation_rec_t *resv = NULL;
if (!job_list || !list_count(job_list))
goto no_jobs;
itr = list_iterator_create(job_list);
itr2 = list_iterator_create(resv_list);
while ((job = list_next(itr))) {
int start = job->start;
int end = job->end;
int set = 0;
while ((resv = list_next(itr2))) {
int elapsed = 0;
/* since a reservation could have
changed while a job was running we
have to make sure we get the time
in the correct record.
*/
if (resv->id != job->resvid)
continue;
set = 1;
if (start < resv->time_start)
start = resv->time_start;
if (!end || end > resv->time_end)
end = resv->time_end;
if ((elapsed = (end - start)) < 1)
continue;
if (job->alloc_cpus)
resv->alloc_secs +=
elapsed * job->alloc_cpus;
}
list_iterator_reset(itr2);
if (!set) {
error("we got a job %u with no reservation "
"associatied with it?", job->jobid);
}
}
list_iterator_destroy(itr2);
list_iterator_destroy(itr);
no_jobs:
if (job_list)
list_destroy(job_list);
}
if (job_cond.resvid_list) {
list_destroy(job_cond.resvid_list);
job_cond.resvid_list = NULL;
}
/* free result after we use the list with resv id's in it. */
mysql_free_result(result);
//END_TIMER2("get_resvs");
return resv_list;
}
extern int bg_free_block(bg_record_t *bg_record, bool wait, bool locked)
{
int rc = SLURM_SUCCESS;
int count = 0;
if (!bg_record) {
error("bg_free_block: there was no bg_record");
return SLURM_ERROR;
}
if (!locked)
slurm_mutex_lock(&block_state_mutex);
while (count < MAX_FREE_RETRIES) {
/* block was removed */
if (bg_record->magic != BLOCK_MAGIC) {
error("block was removed while freeing it here");
xassert(0);
if (!locked)
slurm_mutex_unlock(&block_state_mutex);
return SLURM_SUCCESS;
}
/* Reset these here so we don't try to reboot it
when the state goes to free.
*/
bg_record->boot_state = 0;
bg_record->boot_count = 0;
/* Here we don't need to check if the block is still
* in exsistance since this function can't be called on
* the same block twice. It may
* had already been removed at this point also.
*/
#ifdef HAVE_BG_FILES
if (bg_record->state != BG_BLOCK_FREE
&& bg_record->state != BG_BLOCK_TERM) {
if (bg_conf->slurm_debug_flags & DEBUG_FLAG_SELECT_TYPE)
info("bridge_destroy %s",
bg_record->bg_block_id);
rc = bridge_block_free(bg_record);
if (rc != SLURM_SUCCESS) {
if (rc == BG_ERROR_BLOCK_NOT_FOUND) {
debug("block %s is not found",
bg_record->bg_block_id);
bg_record->state = BG_BLOCK_FREE;
break;
} else if (rc == BG_ERROR_FREE) {
if (bg_conf->slurm_debug_flags
& DEBUG_FLAG_SELECT_TYPE)
info("bridge_block_free"
"(%s): %s State = %s",
bg_record->bg_block_id,
bg_err_str(rc),
bg_block_state_string(
bg_record->state));
} else if (rc == BG_ERROR_INVALID_STATE) {
#ifndef HAVE_BGL
/* If the state is error and
we get an incompatible
state back here, it means
we set it ourselves so
break out.
*/
if (bg_record->state
& BG_BLOCK_ERROR_FLAG)
break;
#endif
if (bg_conf->slurm_debug_flags
& DEBUG_FLAG_SELECT_TYPE)
info("bridge_block_free"
"(%s): %s State = %s",
bg_record->bg_block_id,
bg_err_str(rc),
bg_block_state_string(
bg_record->state));
#ifdef HAVE_BGQ
if (bg_record->state != BG_BLOCK_FREE
&& bg_record->state
!= BG_BLOCK_TERM)
bg_record->state = BG_BLOCK_TERM;
#endif
} else {
error("bridge_block_free"
"(%s): %s State = %s",
bg_record->bg_block_id,
bg_err_str(rc),
bg_block_state_string(
bg_record->state));
}
}
}
#else
/* Fake a free since we are n deallocating
state before this.
*/
if (bg_record->state & BG_BLOCK_ERROR_FLAG) {
/* This will set the state to ERROR(Free)
* just in case the state was ERROR(SOMETHING ELSE) */
bg_record->state = BG_BLOCK_ERROR_FLAG;
break;
} else if (!wait || (count >= 3))
bg_record->state = BG_BLOCK_FREE;
else if (bg_record->state != BG_BLOCK_FREE)
bg_record->state = BG_BLOCK_TERM;
#endif
if (!wait || (bg_record->state == BG_BLOCK_FREE)
#ifndef HAVE_BGL
|| (bg_record->state & BG_BLOCK_ERROR_FLAG)
#endif
) {
break;
}
/* If we were locked outside of this we need to unlock
to not cause deadlock on this mutex until we are
done.
*/
slurm_mutex_unlock(&block_state_mutex);
sleep(FREE_SLEEP_INTERVAL);
count++;
slurm_mutex_lock(&block_state_mutex);
}
rc = SLURM_SUCCESS;
if ((bg_record->state == BG_BLOCK_FREE)
|| (bg_record->state & BG_BLOCK_ERROR_FLAG)) {
if (bg_record->err_ratio
&& (bg_record->state == BG_BLOCK_FREE)) {
/* Sometime the realtime server can report
software error on cnodes even though the
block is free. If this is the case we need
to manually clear them.
*/
ba_mp_t *found_ba_mp;
ListIterator itr =
list_iterator_create(bg_record->ba_mp_list);
debug("Block %s is free, but has %u cnodes in error. "
"This can happen if a large block goes into "
"error and then is freed and the state of "
"the block changes before the "
"database informs all the cnodes are back to "
"normal. This is no big deal.",
bg_record->bg_block_id, bg_record->cnode_err_cnt);
while ((found_ba_mp = list_next(itr))) {
if (!found_ba_mp->used)
continue;
if (!found_ba_mp->cnode_err_bitmap)
found_ba_mp->cnode_err_bitmap =
bit_alloc(
bg_conf->mp_cnode_cnt);
bit_nclear(found_ba_mp->cnode_err_bitmap, 0,
bit_size(found_ba_mp->
cnode_err_bitmap)-1);
}
list_iterator_destroy(itr);
bg_record->cnode_err_cnt = 0;
bg_record->err_ratio = 0;
}
remove_from_bg_list(bg_lists->booted, bg_record);
} else if (count >= MAX_FREE_RETRIES) {
/* Something isn't right, go mark this one in an error
state. */
update_block_msg_t block_msg;
if (bg_conf->slurm_debug_flags & DEBUG_FLAG_SELECT_TYPE)
info("bg_free_block: block %s is not in state "
"free (%s), putting it in error state.",
bg_record->bg_block_id,
bg_block_state_string(bg_record->state));
slurm_init_update_block_msg(&block_msg);
block_msg.bg_block_id = bg_record->bg_block_id;
block_msg.state = BG_BLOCK_ERROR_FLAG;
block_msg.reason = "Block would not deallocate";
slurm_mutex_unlock(&block_state_mutex);
select_g_update_block(&block_msg);
slurm_mutex_lock(&block_state_mutex);
rc = SLURM_ERROR;
}
if (!locked)
slurm_mutex_unlock(&block_state_mutex);
return rc;
}
/*
* Try to find resources for a given job request
* IN job_ptr - pointer to job record in slurmctld
* IN/OUT bitmap - nodes available for assignment to job, clear those not to
* be used
* IN min_nodes, max_nodes - minimum and maximum number of nodes to allocate
* to this job (considers slurm block limits)
* IN mode - SELECT_MODE_RUN_NOW: try to schedule job now
* SELECT_MODE_TEST_ONLY: test if job can ever run
* SELECT_MODE_WILL_RUN: determine when and where job can run
* IN preemptee_candidates - List of pointers to jobs which can be preempted.
* IN/OUT preemptee_job_list - Pointer to list of job pointers. These are the
* jobs to be preempted to initiate the pending job. Not set
* if mode=SELECT_MODE_TEST_ONLY or input pointer is NULL.
* RET - SLURM_SUCCESS if job runnable now, error code otherwise
*/
extern int submit_job(struct job_record *job_ptr, bitstr_t *slurm_block_bitmap,
uint32_t min_nodes, uint32_t max_nodes,
uint32_t req_nodes, uint16_t mode,
List preemptee_candidates,
List *preemptee_job_list)
{
int rc = SLURM_SUCCESS;
bg_record_t* bg_record = NULL;
char buf[256];
uint16_t conn_type[SYSTEM_DIMENSIONS];
List block_list = NULL;
int blocks_added = 0;
time_t starttime = time(NULL);
uint16_t local_mode = mode;
int avail_cpus = num_unused_cpus;
int dim = 0;
for (dim=0; dim<SYSTEM_DIMENSIONS; dim++)
conn_type[dim] = (uint16_t)NO_VAL;
if (preemptee_candidates && preemptee_job_list
&& list_count(preemptee_candidates))
local_mode |= SELECT_MODE_PREEMPT_FLAG;
else
local_mode |= SELECT_MODE_CHECK_FULL;
if (bg_conf->layout_mode == LAYOUT_DYNAMIC)
slurm_mutex_lock(&create_dynamic_mutex);
slurm_mutex_lock(&block_state_mutex);
block_list = copy_bg_list(bg_lists->main);
slurm_mutex_unlock(&block_state_mutex);
get_select_jobinfo(job_ptr->select_jobinfo->data,
SELECT_JOBDATA_CONN_TYPE, &conn_type);
if (conn_type[0] == SELECT_NAV) {
if (bg_conf->mp_cnode_cnt == bg_conf->nodecard_cnode_cnt)
conn_type[0] = SELECT_SMALL;
else if (min_nodes > 1) {
for (dim=0; dim<SYSTEM_DIMENSIONS; dim++)
conn_type[dim] = SELECT_TORUS;
} else if (job_ptr->details->min_cpus < bg_conf->cpus_per_mp)
conn_type[0] = SELECT_SMALL;
else {
for (dim=1; dim<SYSTEM_DIMENSIONS; dim++)
conn_type[dim] = SELECT_NAV;
}
set_select_jobinfo(job_ptr->select_jobinfo->data,
SELECT_JOBDATA_CONN_TYPE,
&conn_type);
}
if (slurm_block_bitmap && !bit_set_count(slurm_block_bitmap)) {
error("no nodes given to place job %u.", job_ptr->job_id);
if (bg_conf->layout_mode == LAYOUT_DYNAMIC)
slurm_mutex_unlock(&create_dynamic_mutex);
return SLURM_ERROR;
}
sprint_select_jobinfo(job_ptr->select_jobinfo->data,
buf, sizeof(buf),
SELECT_PRINT_MIXED);
debug("bluegene:submit_job: %u mode=%d %s nodes=%u-%u-%u",
job_ptr->job_id, local_mode, buf,
min_nodes, req_nodes, max_nodes);
#ifdef HAVE_BG_L_P
# ifdef HAVE_BGL
sprint_select_jobinfo(job_ptr->select_jobinfo->data,
buf, sizeof(buf),
SELECT_PRINT_BLRTS_IMAGE);
debug3("BlrtsImage=%s", buf);
# endif
sprint_select_jobinfo(job_ptr->select_jobinfo->data,
buf, sizeof(buf),
SELECT_PRINT_LINUX_IMAGE);
# ifdef HAVE_BGL
debug3("LinuxImage=%s", buf);
# else
debug3("ComputNodeImage=%s", buf);
# endif
sprint_select_jobinfo(job_ptr->select_jobinfo->data,
buf, sizeof(buf),
SELECT_PRINT_RAMDISK_IMAGE);
# ifdef HAVE_BGL
debug3("RamDiskImage=%s", buf);
# else
debug3("RamDiskIoLoadImage=%s", buf);
# endif
#endif
sprint_select_jobinfo(job_ptr->select_jobinfo->data,
buf, sizeof(buf),
SELECT_PRINT_MLOADER_IMAGE);
debug3("MloaderImage=%s", buf);
/* First look at the empty space, and then remove the
preemptable jobs and try again. */
list_sort(block_list, (ListCmpF)bg_record_sort_aval_inc);
rc = _find_best_block_match(block_list, &blocks_added,
job_ptr, slurm_block_bitmap, min_nodes,
max_nodes, req_nodes,
&bg_record, local_mode, avail_cpus);
if (rc == SLURM_SUCCESS && SELECT_IS_PREEMPT_SET(local_mode)) {
ListIterator itr;
ListIterator job_itr;
bg_record_t *found_record;
struct job_record *preempt_job_ptr;
if (bg_conf->slurm_debug_flags & DEBUG_FLAG_BG_PICK)
info("doing preemption");
local_mode |= SELECT_MODE_CHECK_FULL;
job_itr = list_iterator_create(preemptee_candidates);
itr = list_iterator_create(block_list);
while ((preempt_job_ptr = list_next(job_itr))) {
while ((found_record = list_next(itr))) {
if (found_record->job_ptr == preempt_job_ptr) {
/* info("removing job %u running on %s", */
/* preempt_job_ptr->job_id, */
/* found_record->bg_block_id); */
found_record->job_ptr = NULL;
found_record->job_running =
NO_JOB_RUNNING;
avail_cpus += found_record->cpu_cnt;
break;
}
}
if (!found_record) {
list_iterator_reset(itr);
error("Job %u wasn't found running anywhere, "
"can't preempt",
preempt_job_ptr->job_id);
continue;
} else if (job_ptr->details->min_cpus > avail_cpus)
continue;
list_sort(block_list,
(ListCmpF)bg_record_sort_aval_inc);
if ((rc = _find_best_block_match(
block_list, &blocks_added,
job_ptr, slurm_block_bitmap,
min_nodes, max_nodes, req_nodes,
&bg_record, local_mode, avail_cpus))
== SLURM_SUCCESS)
break;
list_iterator_reset(itr);
}
list_iterator_destroy(itr);
list_iterator_destroy(job_itr);
}
if (rc == SLURM_SUCCESS) {
if (!bg_record)
fatal("we got a success, but no block back");
/* Here we see if there is a job running since
* some jobs take awhile to finish we need to
* make sure the time of the end is in the
* future. If it isn't (meaning it is in the
* past or current time) we add 5 seconds to
* it so we don't use the block immediately.
*/
if (bg_record->job_ptr
&& bg_record->job_ptr->end_time) {
if (bg_record->job_ptr->end_time <= starttime)
starttime += 5;
else
starttime = bg_record->job_ptr->end_time;
} else if (bg_record->job_running == BLOCK_ERROR_STATE)
starttime = INFINITE;
/* make sure the job is eligible to run */
if (job_ptr->details->begin_time > starttime)
starttime = job_ptr->details->begin_time;
job_ptr->start_time = starttime;
set_select_jobinfo(job_ptr->select_jobinfo->data,
SELECT_JOBDATA_NODES,
bg_record->mp_str);
set_select_jobinfo(job_ptr->select_jobinfo->data,
SELECT_JOBDATA_IONODES,
bg_record->ionode_str);
if (!bg_record->bg_block_id) {
debug("%d can start unassigned job %u "
"at %ld on %s",
local_mode, job_ptr->job_id,
starttime, bg_record->mp_str);
set_select_jobinfo(job_ptr->select_jobinfo->data,
SELECT_JOBDATA_BLOCK_PTR,
NULL);
set_select_jobinfo(job_ptr->select_jobinfo->data,
SELECT_JOBDATA_NODE_CNT,
&bg_record->cnode_cnt);
} else {
if ((bg_record->ionode_str)
&& (job_ptr->part_ptr->max_share <= 1))
error("Small block used in "
"non-shared partition");
debug("%d(%d) can start job %u "
"at %ld on %s(%s) %d",
local_mode, mode, job_ptr->job_id,
starttime, bg_record->bg_block_id,
bg_record->mp_str,
SELECT_IS_MODE_RUN_NOW(local_mode));
if (SELECT_IS_MODE_RUN_NOW(local_mode)) {
/* Set this up to be the
correct pointer since we
probably are working off a
copy.
*/
if (bg_record->original)
bg_record = bg_record->original;
set_select_jobinfo(
job_ptr->select_jobinfo->data,
SELECT_JOBDATA_BLOCK_PTR,
bg_record);
if (job_ptr) {
bg_record->job_running =
job_ptr->job_id;
bg_record->job_ptr = job_ptr;
job_ptr->job_state |= JOB_CONFIGURING;
last_bg_update = time(NULL);
}
} else {
set_select_jobinfo(
job_ptr->select_jobinfo->data,
SELECT_JOBDATA_BLOCK_PTR,
NULL);
/* Just to make sure we don't
end up using this on
another job, or we have to
wait until preemption is
done.
*/
bg_record->job_ptr = NULL;
bg_record->job_running = NO_JOB_RUNNING;
}
set_select_jobinfo(job_ptr->select_jobinfo->data,
SELECT_JOBDATA_NODE_CNT,
&bg_record->cnode_cnt);
}
if (SELECT_IS_MODE_RUN_NOW(local_mode))
_build_select_struct(job_ptr,
slurm_block_bitmap,
bg_record->cnode_cnt);
/* set up the preempted job list */
if (SELECT_IS_PREEMPT_SET(local_mode)) {
if (*preemptee_job_list)
list_destroy(*preemptee_job_list);
*preemptee_job_list = _get_preemptables(
local_mode, bg_record,
preemptee_candidates);
}
if (!bg_record->bg_block_id) {
/* This is a fake record so we need to
* destroy it after we get the info from
* it. If it was just testing then
* we added this record to the
* block_list. If this is the case
* it will be handled if se sync the
* lists. But we don't want to do
* that so we will set blocks_added to
* 0 so it doesn't happen. */
if (!blocks_added) {
destroy_bg_record(bg_record);
bg_record = NULL;
}
blocks_added = 0;
}
last_job_update = time(NULL);
}
if (bg_conf->layout_mode == LAYOUT_DYNAMIC) {
slurm_mutex_lock(&block_state_mutex);
if (blocks_added)
_sync_block_lists(block_list, bg_lists->main);
slurm_mutex_unlock(&block_state_mutex);
slurm_mutex_unlock(&create_dynamic_mutex);
}
list_destroy(block_list);
return rc;
}
/* block_state_mutex should be unlocked before calling this */
extern void free_block_list(uint32_t job_id, List track_list,
bool destroy, bool wait)
{
bg_record_t *bg_record = NULL;
int retries;
ListIterator itr = NULL;
bg_free_block_list_t *bg_free_list;
pthread_attr_t attr_agent;
pthread_t thread_agent;
List kill_job_list = NULL;
kill_job_struct_t *freeit;
if (!track_list || !list_count(track_list))
return;
bg_free_list = xmalloc(sizeof(bg_free_block_list_t));
bg_free_list->track_list = list_create(NULL);
bg_free_list->destroy = destroy;
bg_free_list->job_id = job_id;
slurm_mutex_lock(&block_state_mutex);
list_transfer(bg_free_list->track_list, track_list);
itr = list_iterator_create(bg_free_list->track_list);
while ((bg_record = list_next(itr))) {
if (bg_record->magic != BLOCK_MAGIC) {
error("block was already destroyed %p", bg_record);
continue;
}
bg_record->free_cnt++;
/* just so we don't over write a different thread that
wants this block destroyed */
if (destroy && !bg_record->destroy)
bg_record->destroy = destroy;
if (destroy && (bg_record->state & BG_BLOCK_ERROR_FLAG))
resume_block(bg_record);
/* This means we are wanting this block free so we can
run this job on it, so it is ok to have the job
remain here. Only checking for jobs should go
below this.
*/
if (bg_record->modifying) {
debug("free_block_list: Just FYI, we are "
"freeing a block (%s) that "
"has at least one pending job.",
bg_record->bg_block_id);
continue;
}
if (bg_record->job_ptr
&& !IS_JOB_FINISHED(bg_record->job_ptr)) {
info("We are freeing a block (%s) that "
"has job %u(%u).",
bg_record->bg_block_id,
bg_record->job_ptr->job_id,
bg_record->job_running);
if (!kill_job_list)
kill_job_list =
bg_status_create_kill_job_list();
freeit = xmalloc(sizeof(kill_job_struct_t));
freeit->jobid = bg_record->job_ptr->job_id;
list_push(kill_job_list, freeit);
} else if (bg_record->job_list
&& list_count(bg_record->job_list)) {
struct job_record *job_ptr;
ListIterator itr;
if (!kill_job_list)
kill_job_list =
bg_status_create_kill_job_list();
info("We are freeing a block (%s) that has at "
"least 1 job.",
bg_record->bg_block_id);
itr = list_iterator_create(bg_record->job_list);
while ((job_ptr = list_next(itr))) {
if ((job_ptr->magic != JOB_MAGIC)
|| IS_JOB_FINISHED(job_ptr))
continue;
freeit = xmalloc(sizeof(kill_job_struct_t));
freeit->jobid = job_ptr->job_id;
list_push(kill_job_list, freeit);
}
list_iterator_destroy(itr);
}
}
list_iterator_destroy(itr);
slurm_mutex_unlock(&block_state_mutex);
if (kill_job_list) {
bg_status_process_kill_job_list(kill_job_list, JOB_FAILED, 0);
FREE_NULL_LIST(kill_job_list);
}
if (wait) {
/* Track_freeing_blocks waits until the list is done
and frees the memory of bg_free_list.
*/
_track_freeing_blocks(bg_free_list);
return;
}
/* _track_freeing_blocks handles cleanup */
slurm_attr_init(&attr_agent);
if (pthread_attr_setdetachstate(&attr_agent, PTHREAD_CREATE_DETACHED))
error("pthread_attr_setdetachstate error %m");
retries = 0;
while (pthread_create(&thread_agent, &attr_agent,
_track_freeing_blocks,
bg_free_list)) {
error("pthread_create error %m");
if (++retries > MAX_PTHREAD_RETRIES)
fatal("Can't create pthread");
/* sleep and retry */
usleep(1000);
}
slurm_attr_destroy(&attr_agent);
return;
}
static int _dynamically_request(List block_list, int *blocks_added,
select_ba_request_t *request,
char *user_req_nodes,
uint16_t query_mode)
{
List list_of_lists = NULL;
List temp_list = NULL;
List new_blocks = NULL;
List job_list = NULL, booted_list = NULL;
ListIterator itr = NULL;
int rc = SLURM_ERROR;
int create_try = 0;
if (bg_conf->slurm_debug_flags & DEBUG_FLAG_BG_PICK)
info("going to create %d", request->size);
list_of_lists = list_create(NULL);
/* If preempt is set and we are checking full system it means
we altered the block list so only look at it.
*/
if (SELECT_IS_PREEMPT_SET(query_mode)
&& SELECT_IS_CHECK_FULL_SET(query_mode)) {
list_append(list_of_lists, block_list);
} else if (user_req_nodes) {
slurm_mutex_lock(&block_state_mutex);
job_list = copy_bg_list(bg_lists->job_running);
list_append(list_of_lists, job_list);
slurm_mutex_unlock(&block_state_mutex);
} else {
slurm_mutex_lock(&block_state_mutex);
list_append(list_of_lists, block_list);
if (list_count(block_list) != list_count(bg_lists->booted)) {
booted_list = copy_bg_list(bg_lists->booted);
list_append(list_of_lists, booted_list);
if (list_count(bg_lists->booted)
!= list_count(bg_lists->job_running)) {
job_list = copy_bg_list(bg_lists->job_running);
list_append(list_of_lists, job_list);
}
} else if (list_count(block_list)
!= list_count(bg_lists->job_running)) {
job_list = copy_bg_list(bg_lists->job_running);
list_append(list_of_lists, job_list);
}
slurm_mutex_unlock(&block_state_mutex);
}
itr = list_iterator_create(list_of_lists);
while ((temp_list = (List)list_next(itr))) {
create_try++;
/* 1- try empty space
2- we see if we can create one in the
unused mps
3- see if we can create one in the non
job running mps
*/
if (bg_conf->slurm_debug_flags & DEBUG_FLAG_BG_PICK)
info("trying with %d", create_try);
if ((new_blocks = create_dynamic_block(
block_list, request, temp_list, true))) {
bg_record_t *bg_record = NULL;
while ((bg_record = list_pop(new_blocks))) {
if (block_exist_in_list(block_list, bg_record))
destroy_bg_record(bg_record);
else if (SELECT_IS_TEST(query_mode)
|| SELECT_IS_PREEMPT_ON_FULL_TEST(
query_mode)) {
/* Here we don't really want
to create the block if we
are testing.
The second test here is to
make sure If we are able to
run here but we just
preempted we should wait a
bit to make sure the
preempted blocks have time
to clear out.
*/
list_append(block_list, bg_record);
(*blocks_added) = 1;
} else {
if (bridge_block_create(bg_record)
== SLURM_ERROR) {
destroy_bg_record(bg_record);
error("_dynamically_request: "
"unable to configure "
"block");
rc = SLURM_ERROR;
break;
}
list_append(block_list, bg_record);
print_bg_record(bg_record);
(*blocks_added) = 1;
}
}
list_destroy(new_blocks);
if (!*blocks_added) {
rc = SLURM_ERROR;
continue;
}
list_sort(block_list,
(ListCmpF)bg_record_sort_aval_inc);
rc = SLURM_SUCCESS;
break;
} else if (errno == ESLURM_INTERCONNECT_FAILURE) {
rc = SLURM_ERROR;
break;
}
}
list_iterator_destroy(itr);
if (list_of_lists)
list_destroy(list_of_lists);
if (job_list)
list_destroy(job_list);
if (booted_list)
list_destroy(booted_list);
return rc;
}
/* block_state_mutex should be locked before calling this */
static int _post_block_free(bg_record_t *bg_record, bool restore)
{
int rc = SLURM_SUCCESS;
if (bg_record->magic != BLOCK_MAGIC) {
error("block already destroyed %p", bg_record);
xassert(0);
return SLURM_ERROR;
}
bg_record->free_cnt--;
if (bg_record->free_cnt == -1) {
info("we got a negative 1 here for %s",
bg_record->bg_block_id);
xassert(0);
return SLURM_SUCCESS;
} else if (bg_record->modifying) {
info("others are modifing this block %s, don't clear it up",
bg_record->bg_block_id);
return SLURM_SUCCESS;
} else if (bg_record->free_cnt) {
if (bg_conf->slurm_debug_flags & DEBUG_FLAG_SELECT_TYPE)
info("%d others are trying to destroy this block %s",
bg_record->free_cnt, bg_record->bg_block_id);
return SLURM_SUCCESS;
}
/* Even if the block is already in error state we need to do this to
avoid any overlapping blocks that may have been created due
to bad hardware.
*/
if ((bg_record->state & (~BG_BLOCK_ERROR_FLAG)) != BG_BLOCK_FREE) {
/* Something isn't right, go mark this one in an error
state. */
update_block_msg_t block_msg;
if (bg_conf->slurm_debug_flags & DEBUG_FLAG_SELECT_TYPE)
info("_post_block_free: block %s is not in state "
"free (%s), putting it in error state.",
bg_record->bg_block_id,
bg_block_state_string(bg_record->state));
slurm_init_update_block_msg(&block_msg);
block_msg.bg_block_id = bg_record->bg_block_id;
block_msg.state = BG_BLOCK_ERROR_FLAG;
block_msg.reason = "Block would not deallocate";
slurm_mutex_unlock(&block_state_mutex);
select_g_update_block(&block_msg);
slurm_mutex_lock(&block_state_mutex);
if (block_ptr_exist_in_list(bg_lists->main, bg_record))
bg_record->destroy = 0;
return SLURM_SUCCESS;
}
/* The reason restore is used on the entire list is if this
* was for a bunch of small blocks. If we record is marked to
* be destroyed and it is bigger than 1 midplane destroy it
* even if restore is true.
*/
if (restore && bg_record->destroy && (bg_record->mp_count > 1))
restore = false;
/* If we are here we are done with the destroy so just reset it. */
bg_record->destroy = 0;
/* A bit of a sanity check to make sure blocks are being
removed out of all the lists.
*/
remove_from_bg_list(bg_lists->booted, bg_record);
if (remove_from_bg_list(bg_lists->job_running, bg_record)
== SLURM_SUCCESS) {
debug2("_post_block_free: we are freeing block %s and "
"it was in the job_running list. This can happen if a "
"block is removed while waiting for mmcs to finish "
"removing the job from the block.",
bg_record->bg_block_id);
num_unused_cpus += bg_record->cpu_cnt;
}
/* If we don't have any mp_counts force block removal */
if (restore && bg_record->mp_count)
return SLURM_SUCCESS;
if (remove_from_bg_list(bg_lists->main, bg_record) != SLURM_SUCCESS) {
/* This should only happen if called from
* bg_job_place.c where the block was never added to
* the list. */
debug("_post_block_free: It appears this block %s isn't "
"in the main list anymore.",
bg_record->bg_block_id);
}
if (bg_conf->slurm_debug_flags & DEBUG_FLAG_SELECT_TYPE)
info("_post_block_free: removing %s from database",
bg_record->bg_block_id);
rc = bridge_block_remove(bg_record);
if (rc != SLURM_SUCCESS) {
if (rc == BG_ERROR_BLOCK_NOT_FOUND) {
debug("_post_block_free: block %s is not found",
bg_record->bg_block_id);
} else {
error("_post_block_free: "
"bridge_block_remove(%s): %s",
bg_record->bg_block_id,
bg_err_str(rc));
}
} else if (bg_conf->slurm_debug_flags & DEBUG_FLAG_SELECT_TYPE)
info("_post_block_free: done %s(%p)",
bg_record->bg_block_id, bg_record);
destroy_bg_record(bg_record);
if (bg_conf->slurm_debug_flags & DEBUG_FLAG_SELECT_TYPE)
info("_post_block_free: destroyed");
return SLURM_SUCCESS;
}
extern int as_mysql_acct_no_users(mysql_conn_t *mysql_conn,
slurmdb_assoc_cond_t *assoc_cond,
List ret_list)
{
int rc = SLURM_SUCCESS;
char *query = NULL, *tmp = NULL;
char *extra = NULL;
int i = 0;
MYSQL_RES *result = NULL;
MYSQL_ROW row;
List use_cluster_list = as_mysql_cluster_list;
ListIterator itr = NULL;
char *cluster_name;
xassert(ret_list);
_setup_assoc_cond_limits(assoc_cond, &extra, 0);
/* if this changes you will need to edit the corresponding enum */
char *assoc_req_inx[] = {
"id_assoc",
"user",
"acct",
"`partition`",
"parent_acct",
};
enum {
ASSOC_REQ_ID,
ASSOC_REQ_USER,
ASSOC_REQ_ACCT,
ASSOC_REQ_PART,
ASSOC_REQ_PARENT,
ASSOC_REQ_COUNT
};
xfree(tmp);
xstrfmtcat(tmp, "%s", assoc_req_inx[i]);
for(i=1; i<ASSOC_REQ_COUNT; i++) {
xstrfmtcat(tmp, ", %s", assoc_req_inx[i]);
}
if (assoc_cond &&
assoc_cond->cluster_list && list_count(assoc_cond->cluster_list))
use_cluster_list = assoc_cond->cluster_list;
else
slurm_mutex_lock(&as_mysql_cluster_list_lock);
itr = list_iterator_create(use_cluster_list);
while ((cluster_name = list_next(itr))) {
/* only get the account associations */
if (query)
xstrcat(query, " union ");
xstrfmtcat(query, "select distinct %s, '%s' as cluster "
"from \"%s_%s\" %s && user='' && lft=(rgt-1) ",
tmp, cluster_name, cluster_name,
assoc_table, extra);
}
list_iterator_destroy(itr);
if (use_cluster_list == as_mysql_cluster_list)
slurm_mutex_unlock(&as_mysql_cluster_list_lock);
if (query)
xstrcat(query, " order by cluster, acct;");
xfree(tmp);
xfree(extra);
if (debug_flags & DEBUG_FLAG_DB_QUERY)
DB_DEBUG(mysql_conn->conn, "query\n%s", query);
if (!(result = mysql_db_query_ret(
mysql_conn, query, 0))) {
xfree(query);
return SLURM_ERROR;
}
xfree(query);
while ((row = mysql_fetch_row(result))) {
slurmdb_assoc_rec_t *assoc =
xmalloc(sizeof(slurmdb_assoc_rec_t));
list_append(ret_list, assoc);
assoc->id = SLURMDB_PROBLEM_ACCT_NO_USERS;
if (row[ASSOC_REQ_USER][0])
assoc->user = xstrdup(row[ASSOC_REQ_USER]);
assoc->acct = xstrdup(row[ASSOC_REQ_ACCT]);
assoc->cluster = xstrdup(row[ASSOC_REQ_COUNT]);
if (row[ASSOC_REQ_PARENT][0])
assoc->parent_acct = xstrdup(row[ASSOC_REQ_PARENT]);
if (row[ASSOC_REQ_PART][0])
assoc->partition = xstrdup(row[ASSOC_REQ_PART]);
}
mysql_free_result(result);
return rc;
}
extern int archive_write_file(Buf buffer, char *cluster_name,
time_t period_start, time_t period_end,
char *arch_dir, char *arch_type,
uint32_t archive_period)
{
int fd = 0;
int rc = SLURM_SUCCESS;
char *old_file = NULL, *new_file = NULL, *reg_file = NULL;
static int high_buffer_size = (1024 * 1024);
static pthread_mutex_t local_file_lock = PTHREAD_MUTEX_INITIALIZER;
xassert(buffer);
slurm_mutex_lock(&local_file_lock);
/* write the buffer to file */
reg_file = _make_archive_name(period_start, period_end,
cluster_name, arch_dir,
arch_type, archive_period);
debug("Storing %s archive for %s at %s",
arch_type, cluster_name, reg_file);
old_file = xstrdup_printf("%s.old", reg_file);
new_file = xstrdup_printf("%s.new", reg_file);
fd = creat(new_file, 0600);
if (fd < 0) {
error("Can't save archive, create file %s error %m", new_file);
rc = SLURM_ERROR;
} else {
int pos = 0, nwrite = get_buf_offset(buffer), amount;
char *data = (char *)get_buf_data(buffer);
high_buffer_size = MAX(nwrite, high_buffer_size);
while (nwrite > 0) {
amount = write(fd, &data[pos], nwrite);
if ((amount < 0) && (errno != EINTR)) {
error("Error writing file %s, %m", new_file);
rc = SLURM_ERROR;
break;
}
nwrite -= amount;
pos += amount;
}
fsync(fd);
close(fd);
}
if (rc)
(void) unlink(new_file);
else { /* file shuffle */
(void) unlink(old_file);
if (link(reg_file, old_file))
debug4("Link(%s, %s): %m", reg_file, old_file);
(void) unlink(reg_file);
if (link(new_file, reg_file))
debug4("Link(%s, %s): %m", new_file, reg_file);
(void) unlink(new_file);
}
xfree(old_file);
xfree(reg_file);
xfree(new_file);
slurm_mutex_unlock(&local_file_lock);
return rc;
}
/*
* Read the slurm configuration file (slurm.conf) and substitute some
* values into the slurmd configuration in preference of the defaults.
*/
static void
_read_config(void)
{
char *path_pubkey = NULL;
slurm_ctl_conf_t *cf = NULL;
uint16_t tmp16 = 0;
#ifndef HAVE_FRONT_END
bool cr_flag = false, gang_flag = false;
#endif
cf = slurm_conf_lock();
slurm_mutex_lock(&conf->config_mutex);
if (conf->conffile == NULL)
conf->conffile = xstrdup(cf->slurm_conf);
conf->slurm_user_id = cf->slurm_user_id;
conf->cr_type = cf->select_type_param;
path_pubkey = xstrdup(cf->job_credential_public_certificate);
if (!conf->logfile)
conf->logfile = xstrdup(cf->slurmd_logfile);
#ifndef HAVE_FRONT_END
if (!strcmp(cf->select_type, "select/cons_res"))
cr_flag = true;
if (cf->preempt_mode & PREEMPT_MODE_GANG)
gang_flag = true;
#endif
slurm_conf_unlock();
/* node_name may already be set from a command line parameter */
if (conf->node_name == NULL)
conf->node_name = slurm_conf_get_nodename(conf->hostname);
/* if we didn't match the form of the hostname already
* stored in conf->hostname, check to see if we match any
* valid aliases */
if (conf->node_name == NULL)
conf->node_name = slurm_conf_get_aliased_nodename();
if (conf->node_name == NULL)
conf->node_name = slurm_conf_get_nodename("localhost");
if (conf->node_name == NULL)
fatal("Unable to determine this slurmd's NodeName");
_massage_pathname(&conf->logfile);
/* set node_addr if relevant */
if ((conf->node_addr == NULL) &&
(conf->node_addr = slurm_conf_get_nodeaddr(conf->hostname)) &&
(strcmp(conf->node_addr, conf->hostname) == 0)) {
xfree(conf->node_addr); /* Sets to NULL */
}
conf->port = slurm_conf_get_port(conf->node_name);
slurm_conf_get_cpus_bsct(conf->node_name,
&conf->conf_cpus, &conf->conf_boards,
&conf->conf_sockets, &conf->conf_cores,
&conf->conf_threads);
/* store hardware properties in slurmd_config */
xfree(conf->block_map);
xfree(conf->block_map_inv);
_update_logging();
_update_nice();
get_cpuinfo(&conf->actual_cpus,
&conf->actual_boards,
&conf->actual_sockets,
&conf->actual_cores,
&conf->actual_threads,
&conf->block_map_size,
&conf->block_map, &conf->block_map_inv);
#ifdef HAVE_FRONT_END
/*
* When running with multiple frontends, the slurmd S:C:T values are not
* relevant, hence ignored by both _register_front_ends (sets all to 1)
* and validate_nodes_via_front_end (uses slurm.conf values).
* Report actual hardware configuration, irrespective of FastSchedule.
*/
conf->cpus = conf->actual_cpus;
conf->boards = conf->actual_boards;
conf->sockets = conf->actual_sockets;
conf->cores = conf->actual_cores;
conf->threads = conf->actual_threads;
#else
/* If the actual resources on a node differ than what is in
* the configuration file and we are using
* cons_res or gang scheduling we have to use what is in the
* configuration file because the slurmctld creates bitmaps
* for scheduling before these nodes check in.
*/
if (((cf->fast_schedule == 0) && !cr_flag && !gang_flag) ||
((cf->fast_schedule == 1) &&
(conf->actual_cpus < conf->conf_cpus))) {
conf->cpus = conf->actual_cpus;
conf->boards = conf->actual_boards;
conf->sockets = conf->actual_sockets;
conf->cores = conf->actual_cores;
conf->threads = conf->actual_threads;
} else {
conf->cpus = conf->conf_cpus;
conf->boards = conf->conf_boards;
conf->sockets = conf->conf_sockets;
conf->cores = conf->conf_cores;
conf->threads = conf->conf_threads;
}
if ((conf->cpus != conf->actual_cpus) ||
(conf->sockets != conf->actual_sockets) ||
(conf->cores != conf->actual_cores) ||
(conf->threads != conf->actual_threads)) {
if (cf->fast_schedule) {
info("Node configuration differs from hardware: "
"CPUs=%u:%u(hw) Boards=%u:%u(hw) "
"SocketsPerBoard=%u:%u(hw) CoresPerSocket=%u:%u(hw) "
"ThreadsPerCore=%u:%u(hw)",
conf->cpus, conf->actual_cpus,
conf->boards, conf->actual_boards,
conf->sockets, conf->actual_sockets,
conf->cores, conf->actual_cores,
conf->threads, conf->actual_threads);
} else if ((cf->fast_schedule == 0) && (cr_flag || gang_flag)) {
error("You are using cons_res or gang scheduling with "
"Fastschedule=0 and node configuration differs "
"from hardware. The node configuration used "
"will be what is in the slurm.conf because of "
"the bitmaps the slurmctld must create before "
"the slurmd registers.\n"
" CPUs=%u:%u(hw) Boards=%u:%u(hw) "
"SocketsPerBoard=%u:%u(hw) CoresPerSocket=%u:%u(hw) "
"ThreadsPerCore=%u:%u(hw)",
conf->cpus, conf->actual_cpus,
conf->boards, conf->actual_boards,
conf->sockets, conf->actual_sockets,
conf->cores, conf->actual_cores,
conf->threads, conf->actual_threads);
}
}
#endif
get_memory(&conf->real_memory_size);
get_up_time(&conf->up_time);
cf = slurm_conf_lock();
get_tmp_disk(&conf->tmp_disk_space, cf->tmp_fs);
_free_and_set(&conf->epilog, xstrdup(cf->epilog));
_free_and_set(&conf->prolog, xstrdup(cf->prolog));
_free_and_set(&conf->tmpfs, xstrdup(cf->tmp_fs));
_free_and_set(&conf->health_check_program,
xstrdup(cf->health_check_program));
_free_and_set(&conf->spooldir, xstrdup(cf->slurmd_spooldir));
_massage_pathname(&conf->spooldir);
_free_and_set(&conf->pidfile, xstrdup(cf->slurmd_pidfile));
_massage_pathname(&conf->pidfile);
_free_and_set(&conf->select_type, xstrdup(cf->select_type));
_free_and_set(&conf->task_prolog, xstrdup(cf->task_prolog));
_free_and_set(&conf->task_epilog, xstrdup(cf->task_epilog));
_free_and_set(&conf->pubkey, path_pubkey);
conf->debug_flags = cf->debug_flags;
conf->propagate_prio = cf->propagate_prio_process;
_free_and_set(&conf->job_acct_gather_freq,
xstrdup(cf->job_acct_gather_freq));
conf->acct_freq_task = (uint16_t)NO_VAL;
tmp16 = acct_gather_parse_freq(PROFILE_TASK,
conf->job_acct_gather_freq);
if (tmp16 != -1)
conf->acct_freq_task = tmp16;
_free_and_set(&conf->acct_gather_energy_type,
xstrdup(cf->acct_gather_energy_type));
_free_and_set(&conf->acct_gather_filesystem_type,
xstrdup(cf->acct_gather_filesystem_type));
_free_and_set(&conf->acct_gather_infiniband_type,
xstrdup(cf->acct_gather_infiniband_type));
_free_and_set(&conf->acct_gather_profile_type,
xstrdup(cf->acct_gather_profile_type));
_free_and_set(&conf->job_acct_gather_type,
xstrdup(cf->job_acct_gather_type));
if ( (conf->node_name == NULL) ||
(conf->node_name[0] == '\0') )
fatal("Node name lookup failure");
if (cf->control_addr == NULL)
fatal("Unable to establish controller machine");
if (cf->slurmctld_port == 0)
fatal("Unable to establish controller port");
conf->slurmd_timeout = cf->slurmd_timeout;
conf->use_pam = cf->use_pam;
conf->task_plugin_param = cf->task_plugin_param;
slurm_mutex_unlock(&conf->config_mutex);
slurm_conf_unlock();
}
/*
* Perform any setup required to initiate a job
* job_ptr IN - pointer to the job being initiated
* RET - SLURM_SUCCESS or an error code
*
* NOTE: This happens in parallel with srun and slurmd spawning
* the job. A prolog script is expected to defer initiation of
* the job script until the BG block is available for use.
*/
extern int start_job(struct job_record *job_ptr)
{
int rc = SLURM_SUCCESS;
bg_record_t *bg_record = NULL;
bg_action_t *bg_action_ptr = NULL;
bg_action_ptr = xmalloc(sizeof(bg_action_t));
bg_action_ptr->op = START_OP;
bg_action_ptr->job_ptr = job_ptr;
get_select_jobinfo(job_ptr->select_jobinfo->data,
SELECT_JOBDATA_BLOCK_ID,
&(bg_action_ptr->bg_block_id));
get_select_jobinfo(job_ptr->select_jobinfo->data,
SELECT_JOBDATA_REBOOT,
&(bg_action_ptr->reboot));
get_select_jobinfo(job_ptr->select_jobinfo->data,
SELECT_JOBDATA_CONN_TYPE,
&(bg_action_ptr->conn_type));
get_select_jobinfo(job_ptr->select_jobinfo->data,
SELECT_JOBDATA_MLOADER_IMAGE,
&(bg_action_ptr->mloaderimage));
#ifdef HAVE_BG_L_P
# ifdef HAVE_BGL
get_select_jobinfo(job_ptr->select_jobinfo->data,
SELECT_JOBDATA_BLRTS_IMAGE,
&(bg_action_ptr->blrtsimage));
if (!bg_action_ptr->blrtsimage) {
bg_action_ptr->blrtsimage =
xstrdup(bg_conf->default_blrtsimage);
set_select_jobinfo(job_ptr->select_jobinfo->data,
SELECT_JOBDATA_BLRTS_IMAGE,
bg_action_ptr->blrtsimage);
}
# elif defined HAVE_BGP
get_select_jobinfo(job_ptr->select_jobinfo->data,
SELECT_JOBDATA_CONN_TYPE,
&(bg_action_ptr->conn_type));
# endif
get_select_jobinfo(job_ptr->select_jobinfo->data,
SELECT_JOBDATA_LINUX_IMAGE,
&(bg_action_ptr->linuximage));
if (!bg_action_ptr->linuximage) {
bg_action_ptr->linuximage =
xstrdup(bg_conf->default_linuximage);
set_select_jobinfo(job_ptr->select_jobinfo->data,
SELECT_JOBDATA_LINUX_IMAGE,
bg_action_ptr->linuximage);
}
get_select_jobinfo(job_ptr->select_jobinfo->data,
SELECT_JOBDATA_RAMDISK_IMAGE,
&(bg_action_ptr->ramdiskimage));
if (!bg_action_ptr->ramdiskimage) {
bg_action_ptr->ramdiskimage =
xstrdup(bg_conf->default_ramdiskimage);
set_select_jobinfo(job_ptr->select_jobinfo->data,
SELECT_JOBDATA_RAMDISK_IMAGE,
bg_action_ptr->ramdiskimage);
}
#endif
if (!bg_action_ptr->mloaderimage) {
bg_action_ptr->mloaderimage =
xstrdup(bg_conf->default_mloaderimage);
set_select_jobinfo(job_ptr->select_jobinfo->data,
SELECT_JOBDATA_MLOADER_IMAGE,
bg_action_ptr->mloaderimage);
}
slurm_mutex_lock(&block_state_mutex);
bg_record = find_bg_record_in_list(bg_lists->main,
bg_action_ptr->bg_block_id);
if (!bg_record) {
slurm_mutex_unlock(&block_state_mutex);
error("bg_record %s doesn't exist, requested for job (%d)",
bg_action_ptr->bg_block_id, job_ptr->job_id);
_destroy_bg_action(bg_action_ptr);
return SLURM_ERROR;
}
last_bg_update = time(NULL);
if (bg_record->job_list) {
if (!find_job_in_bg_record(bg_record, job_ptr->job_id))
list_append(bg_record->job_list, job_ptr);
} else {
bg_record->job_running = bg_action_ptr->job_ptr->job_id;
bg_record->job_ptr = bg_action_ptr->job_ptr;
}
num_unused_cpus -= job_ptr->total_cpus;
if (!block_ptr_exist_in_list(bg_lists->job_running, bg_record))
list_push(bg_lists->job_running, bg_record);
if (!block_ptr_exist_in_list(bg_lists->booted, bg_record))
list_push(bg_lists->booted, bg_record);
/* Just incase something happens to free this block before we
start the job we will make it so this job doesn't get blown
away.
*/
bg_record->modifying = 1;
slurm_mutex_unlock(&block_state_mutex);
info("Queue start of job %u in BG block %s",
job_ptr->job_id,
bg_action_ptr->bg_block_id);
_block_op(bg_action_ptr);
return rc;
}
/*
* Perform any setup required to initiate a job
* job_ptr IN - pointer to the job being initiated
* RET - SLURM_SUCCESS or an error code
*
* NOTE: This happens in parallel with srun and slurmd spawning
* the job. A prolog script is expected to defer initiation of
* the job script until the BG block is available for use.
*/
extern int start_job(struct job_record *job_ptr)
{
int rc = SLURM_SUCCESS;
bg_record_t *bg_record = NULL;
bg_action_t *bg_action_ptr = NULL;
select_jobinfo_t *jobinfo = job_ptr->select_jobinfo->data;
slurm_mutex_lock(&block_state_mutex);
bg_record = jobinfo->bg_record;
if (!bg_record || !block_ptr_exist_in_list(bg_lists->main, bg_record)) {
slurm_mutex_unlock(&block_state_mutex);
error("bg_record %s doesn't exist, requested for job (%d)",
jobinfo->bg_block_id, job_ptr->job_id);
return SLURM_ERROR;
}
if ((jobinfo->conn_type[0] != SELECT_NAV)
&& (jobinfo->conn_type[0] < SELECT_SMALL)) {
int dim;
for (dim=0; dim<SYSTEM_DIMENSIONS; dim++)
jobinfo->conn_type[dim] = bg_record->conn_type[dim];
}
/* If it isn't 0 then it was setup previous (sub-block)
*/
if (jobinfo->geometry[SYSTEM_DIMENSIONS] == 0)
memcpy(jobinfo->geometry, bg_record->geo,
sizeof(bg_record->geo));
if (bg_record->job_list) {
/* Mark the ba_mp cnodes as used now. */
ba_mp_t *ba_mp = list_peek(bg_record->ba_mp_list);
xassert(ba_mp);
xassert(ba_mp->cnode_bitmap);
bit_or(ba_mp->cnode_bitmap, jobinfo->units_avail);
if (!find_job_in_bg_record(bg_record, job_ptr->job_id))
list_append(bg_record->job_list, job_ptr);
} else {
bg_record->job_running = job_ptr->job_id;
bg_record->job_ptr = job_ptr;
}
job_ptr->job_state |= JOB_CONFIGURING;
bg_action_ptr = xmalloc(sizeof(bg_action_t));
bg_action_ptr->op = START_OP;
bg_action_ptr->job_ptr = job_ptr;
/* FIXME: The below get_select_jobinfo calls could be avoided
* by just using the jobinfo as we do above.
*/
get_select_jobinfo(job_ptr->select_jobinfo->data,
SELECT_JOBDATA_BLOCK_ID,
&(bg_action_ptr->bg_block_id));
get_select_jobinfo(job_ptr->select_jobinfo->data,
SELECT_JOBDATA_REBOOT,
&(bg_action_ptr->reboot));
get_select_jobinfo(job_ptr->select_jobinfo->data,
SELECT_JOBDATA_CONN_TYPE,
&(bg_action_ptr->conn_type));
get_select_jobinfo(job_ptr->select_jobinfo->data,
SELECT_JOBDATA_MLOADER_IMAGE,
&(bg_action_ptr->mloaderimage));
#ifdef HAVE_BG_L_P
# ifdef HAVE_BGL
get_select_jobinfo(job_ptr->select_jobinfo->data,
SELECT_JOBDATA_BLRTS_IMAGE,
&(bg_action_ptr->blrtsimage));
if (!bg_action_ptr->blrtsimage) {
bg_action_ptr->blrtsimage =
xstrdup(bg_conf->default_blrtsimage);
set_select_jobinfo(job_ptr->select_jobinfo->data,
SELECT_JOBDATA_BLRTS_IMAGE,
bg_action_ptr->blrtsimage);
}
# elif defined HAVE_BGP
get_select_jobinfo(job_ptr->select_jobinfo->data,
SELECT_JOBDATA_CONN_TYPE,
&(bg_action_ptr->conn_type));
# endif
get_select_jobinfo(job_ptr->select_jobinfo->data,
SELECT_JOBDATA_LINUX_IMAGE,
&(bg_action_ptr->linuximage));
if (!bg_action_ptr->linuximage) {
bg_action_ptr->linuximage =
xstrdup(bg_conf->default_linuximage);
set_select_jobinfo(job_ptr->select_jobinfo->data,
SELECT_JOBDATA_LINUX_IMAGE,
bg_action_ptr->linuximage);
}
get_select_jobinfo(job_ptr->select_jobinfo->data,
SELECT_JOBDATA_RAMDISK_IMAGE,
&(bg_action_ptr->ramdiskimage));
if (!bg_action_ptr->ramdiskimage) {
bg_action_ptr->ramdiskimage =
xstrdup(bg_conf->default_ramdiskimage);
set_select_jobinfo(job_ptr->select_jobinfo->data,
SELECT_JOBDATA_RAMDISK_IMAGE,
bg_action_ptr->ramdiskimage);
}
#endif
if (!bg_action_ptr->mloaderimage) {
bg_action_ptr->mloaderimage =
xstrdup(bg_conf->default_mloaderimage);
set_select_jobinfo(job_ptr->select_jobinfo->data,
SELECT_JOBDATA_MLOADER_IMAGE,
bg_action_ptr->mloaderimage);
}
num_unused_cpus -= job_ptr->total_cpus;
if (!block_ptr_exist_in_list(bg_lists->job_running, bg_record))
list_push(bg_lists->job_running, bg_record);
if (!block_ptr_exist_in_list(bg_lists->booted, bg_record))
list_push(bg_lists->booted, bg_record);
/* Just in case something happens to free this block before we
start the job we will make it so this job doesn't get blown
away.
*/
bg_record->modifying = 1;
last_bg_update = time(NULL);
slurm_mutex_unlock(&block_state_mutex);
info("Queue start of job %u in BG block %s",
job_ptr->job_id,
bg_action_ptr->bg_block_id);
_block_op(bg_action_ptr);
return rc;
}
/*
* read_slurmdbd_conf - load the SlurmDBD configuration from the slurmdbd.conf
* file. Store result into global variable slurmdbd_conf.
* This function can be called more than once.
* RET SLURM_SUCCESS if no error, otherwise an error code
*/
extern int read_slurmdbd_conf(void)
{
s_p_options_t options[] = {
{"ArchiveDir", S_P_STRING},
{"ArchiveEvents", S_P_BOOLEAN},
{"ArchiveJobs", S_P_BOOLEAN},
{"ArchiveResvs", S_P_BOOLEAN},
{"ArchiveScript", S_P_STRING},
{"ArchiveSteps", S_P_BOOLEAN},
{"ArchiveSuspend", S_P_BOOLEAN},
{"AuthInfo", S_P_STRING},
{"AuthType", S_P_STRING},
{"CommitDelay", S_P_UINT16},
{"DbdAddr", S_P_STRING},
{"DbdBackupHost", S_P_STRING},
{"DbdHost", S_P_STRING},
{"DbdPort", S_P_UINT16},
{"DebugFlags", S_P_STRING},
{"DebugLevel", S_P_STRING},
{"DefaultQOS", S_P_STRING},
{"JobPurge", S_P_UINT32},
{"LogFile", S_P_STRING},
{"LogTimeFormat", S_P_STRING},
{"MessageTimeout", S_P_UINT16},
{"PidFile", S_P_STRING},
{"PluginDir", S_P_STRING},
{"PrivateData", S_P_STRING},
{"PurgeEventAfter", S_P_STRING},
{"PurgeJobAfter", S_P_STRING},
{"PurgeResvAfter", S_P_STRING},
{"PurgeStepAfter", S_P_STRING},
{"PurgeSuspendAfter", S_P_STRING},
{"PurgeEventMonths", S_P_UINT32},
{"PurgeJobMonths", S_P_UINT32},
{"PurgeStepMonths", S_P_UINT32},
{"PurgeSuspendMonths", S_P_UINT32},
{"SlurmUser", S_P_STRING},
{"StepPurge", S_P_UINT32},
{"StorageBackupHost", S_P_STRING},
{"StorageHost", S_P_STRING},
{"StorageLoc", S_P_STRING},
{"StoragePass", S_P_STRING},
{"StoragePort", S_P_UINT16},
{"StorageType", S_P_STRING},
{"StorageUser", S_P_STRING},
{"TrackWCKey", S_P_BOOLEAN},
{"TrackSlurmctldDown", S_P_BOOLEAN},
{NULL} };
s_p_hashtbl_t *tbl = NULL;
char *conf_path = NULL;
char *temp_str = NULL;
struct stat buf;
/* Set initial values */
slurm_mutex_lock(&conf_mutex);
if (slurmdbd_conf == NULL) {
slurmdbd_conf = xmalloc(sizeof(slurm_dbd_conf_t));
boot_time = time(NULL);
}
slurmdbd_conf->debug_level = LOG_LEVEL_INFO;
_clear_slurmdbd_conf();
/* Get the slurmdbd.conf path and validate the file */
conf_path = get_extra_conf_path("slurmdbd.conf");
if ((conf_path == NULL) || (stat(conf_path, &buf) == -1)) {
info("No slurmdbd.conf file (%s)", conf_path);
} else {
bool a_events = 0, a_jobs = 0, a_resv = 0,
a_steps = 0, a_suspend = 0;
debug("Reading slurmdbd.conf file %s", conf_path);
tbl = s_p_hashtbl_create(options);
if (s_p_parse_file(tbl, NULL, conf_path, false)
== SLURM_ERROR) {
fatal("Could not open/read/parse slurmdbd.conf file %s",
conf_path);
}
if (!s_p_get_string(&slurmdbd_conf->archive_dir, "ArchiveDir",
tbl))
slurmdbd_conf->archive_dir =
xstrdup(DEFAULT_SLURMDBD_ARCHIVE_DIR);
s_p_get_boolean(&a_events, "ArchiveEvents", tbl);
s_p_get_boolean(&a_jobs, "ArchiveJobs", tbl);
s_p_get_boolean(&a_resv, "ArchiveResvs", tbl);
s_p_get_string(&slurmdbd_conf->archive_script, "ArchiveScript",
tbl);
s_p_get_boolean(&a_steps, "ArchiveSteps", tbl);
s_p_get_boolean(&a_suspend, "ArchiveSuspend", tbl);
s_p_get_string(&slurmdbd_conf->auth_info, "AuthInfo", tbl);
s_p_get_string(&slurmdbd_conf->auth_type, "AuthType", tbl);
s_p_get_uint16(&slurmdbd_conf->commit_delay,
"CommitDelay", tbl);
s_p_get_string(&slurmdbd_conf->dbd_backup,
"DbdBackupHost", tbl);
s_p_get_string(&slurmdbd_conf->dbd_host, "DbdHost", tbl);
s_p_get_string(&slurmdbd_conf->dbd_addr, "DbdAddr", tbl);
s_p_get_uint16(&slurmdbd_conf->dbd_port, "DbdPort", tbl);
if (s_p_get_string(&temp_str, "DebugFlags", tbl)) {
if (debug_str2flags(temp_str,
&slurmdbd_conf->debug_flags)
!= SLURM_SUCCESS)
fatal("DebugFlags invalid: %s", temp_str);
xfree(temp_str);
} else /* Default: no DebugFlags */
slurmdbd_conf->debug_flags = 0;
if (s_p_get_string(&temp_str, "DebugLevel", tbl)) {
slurmdbd_conf->debug_level = log_string2num(temp_str);
if (slurmdbd_conf->debug_level == (uint16_t) NO_VAL)
fatal("Invalid DebugLevel %s", temp_str);
xfree(temp_str);
}
s_p_get_string(&slurmdbd_conf->default_qos, "DefaultQOS", tbl);
if (s_p_get_uint32(&slurmdbd_conf->purge_job,
"JobPurge", tbl)) {
if (!slurmdbd_conf->purge_job)
slurmdbd_conf->purge_job = NO_VAL;
else
slurmdbd_conf->purge_job |=
SLURMDB_PURGE_MONTHS;
}
s_p_get_string(&slurmdbd_conf->log_file, "LogFile", tbl);
if (s_p_get_string(&temp_str, "LogTimeFormat", tbl)) {
if (slurm_strcasestr(temp_str, "iso8601_ms"))
slurmdbd_conf->log_fmt = LOG_FMT_ISO8601_MS;
else if (slurm_strcasestr(temp_str, "iso8601"))
slurmdbd_conf->log_fmt = LOG_FMT_ISO8601;
else if (slurm_strcasestr(temp_str, "rfc5424_ms"))
slurmdbd_conf->log_fmt = LOG_FMT_RFC5424_MS;
else if (slurm_strcasestr(temp_str, "rfc5424"))
slurmdbd_conf->log_fmt = LOG_FMT_RFC5424;
else if (slurm_strcasestr(temp_str, "clock"))
slurmdbd_conf->log_fmt = LOG_FMT_CLOCK;
else if (slurm_strcasestr(temp_str, "short"))
slurmdbd_conf->log_fmt = LOG_FMT_SHORT;
else if (slurm_strcasestr(temp_str, "thread_id"))
slurmdbd_conf->log_fmt = LOG_FMT_THREAD_ID;
xfree(temp_str);
} else
slurmdbd_conf->log_fmt = LOG_FMT_ISO8601_MS;
if (!s_p_get_uint16(&slurmdbd_conf->msg_timeout,
"MessageTimeout", tbl))
slurmdbd_conf->msg_timeout = DEFAULT_MSG_TIMEOUT;
else if (slurmdbd_conf->msg_timeout > 100) {
info("WARNING: MessageTimeout is too high for "
"effective fault-tolerance");
}
s_p_get_string(&slurmdbd_conf->pid_file, "PidFile", tbl);
s_p_get_string(&slurmdbd_conf->plugindir, "PluginDir", tbl);
slurmdbd_conf->private_data = 0; /* default visible to all */
if (s_p_get_string(&temp_str, "PrivateData", tbl)) {
if (slurm_strcasestr(temp_str, "account"))
slurmdbd_conf->private_data
|= PRIVATE_DATA_ACCOUNTS;
if (slurm_strcasestr(temp_str, "job"))
slurmdbd_conf->private_data
|= PRIVATE_DATA_JOBS;
if (slurm_strcasestr(temp_str, "node"))
slurmdbd_conf->private_data
|= PRIVATE_DATA_NODES;
if (slurm_strcasestr(temp_str, "partition"))
slurmdbd_conf->private_data
|= PRIVATE_DATA_PARTITIONS;
if (slurm_strcasestr(temp_str, "reservation"))
slurmdbd_conf->private_data
|= PRIVATE_DATA_RESERVATIONS;
if (slurm_strcasestr(temp_str, "usage"))
slurmdbd_conf->private_data
|= PRIVATE_DATA_USAGE;
if (slurm_strcasestr(temp_str, "user"))
slurmdbd_conf->private_data
|= PRIVATE_DATA_USERS;
if (slurm_strcasestr(temp_str, "all"))
slurmdbd_conf->private_data = 0xffff;
xfree(temp_str);
}
if (s_p_get_string(&temp_str, "PurgeEventAfter", tbl)) {
/* slurmdb_parse_purge will set SLURMDB_PURGE_FLAGS */
if ((slurmdbd_conf->purge_event =
slurmdb_parse_purge(temp_str)) == NO_VAL) {
fatal("Bad value \"%s\" for PurgeEventAfter",
temp_str);
}
xfree(temp_str);
}
if (s_p_get_string(&temp_str, "PurgeJobAfter", tbl)) {
/* slurmdb_parse_purge will set SLURMDB_PURGE_FLAGS */
if ((slurmdbd_conf->purge_job =
slurmdb_parse_purge(temp_str)) == NO_VAL) {
fatal("Bad value \"%s\" for PurgeJobAfter",
temp_str);
}
xfree(temp_str);
}
if (s_p_get_string(&temp_str, "PurgeResvAfter", tbl)) {
/* slurmdb_parse_purge will set SLURMDB_PURGE_FLAGS */
if ((slurmdbd_conf->purge_resv =
slurmdb_parse_purge(temp_str)) == NO_VAL) {
fatal("Bad value \"%s\" for PurgeResvAfter",
temp_str);
}
xfree(temp_str);
}
if (s_p_get_string(&temp_str, "PurgeStepAfter", tbl)) {
/* slurmdb_parse_purge will set SLURMDB_PURGE_FLAGS */
if ((slurmdbd_conf->purge_step =
slurmdb_parse_purge(temp_str)) == NO_VAL) {
fatal("Bad value \"%s\" for PurgeStepAfter",
temp_str);
}
xfree(temp_str);
}
if (s_p_get_string(&temp_str, "PurgeSuspendAfter", tbl)) {
/* slurmdb_parse_purge will set SLURMDB_PURGE_FLAGS */
if ((slurmdbd_conf->purge_suspend =
slurmdb_parse_purge(temp_str)) == NO_VAL) {
fatal("Bad value \"%s\" for PurgeSuspendAfter",
temp_str);
}
xfree(temp_str);
}
if (s_p_get_uint32(&slurmdbd_conf->purge_event,
"PurgeEventMonths", tbl)) {
if (!slurmdbd_conf->purge_event)
slurmdbd_conf->purge_event = NO_VAL;
else
slurmdbd_conf->purge_event |=
SLURMDB_PURGE_MONTHS;
}
if (s_p_get_uint32(&slurmdbd_conf->purge_job,
"PurgeJobMonths", tbl)) {
if (!slurmdbd_conf->purge_job)
slurmdbd_conf->purge_job = NO_VAL;
else
slurmdbd_conf->purge_job |=
SLURMDB_PURGE_MONTHS;
}
if (s_p_get_uint32(&slurmdbd_conf->purge_step,
"PurgeStepMonths", tbl)) {
if (!slurmdbd_conf->purge_step)
slurmdbd_conf->purge_step = NO_VAL;
else
slurmdbd_conf->purge_step |=
SLURMDB_PURGE_MONTHS;
}
if (s_p_get_uint32(&slurmdbd_conf->purge_suspend,
"PurgeSuspendMonths", tbl)) {
if (!slurmdbd_conf->purge_suspend)
slurmdbd_conf->purge_suspend = NO_VAL;
else
slurmdbd_conf->purge_suspend
|= SLURMDB_PURGE_MONTHS;
}
s_p_get_string(&slurmdbd_conf->slurm_user_name,
"SlurmUser", tbl);
if (s_p_get_uint32(&slurmdbd_conf->purge_step,
"StepPurge", tbl)) {
if (!slurmdbd_conf->purge_step)
slurmdbd_conf->purge_step = NO_VAL;
else
slurmdbd_conf->purge_step |=
SLURMDB_PURGE_MONTHS;
}
s_p_get_string(&slurmdbd_conf->storage_backup_host,
"StorageBackupHost", tbl);
s_p_get_string(&slurmdbd_conf->storage_host,
"StorageHost", tbl);
s_p_get_string(&slurmdbd_conf->storage_loc,
"StorageLoc", tbl);
s_p_get_string(&slurmdbd_conf->storage_pass,
"StoragePass", tbl);
s_p_get_uint16(&slurmdbd_conf->storage_port,
"StoragePort", tbl);
s_p_get_string(&slurmdbd_conf->storage_type,
"StorageType", tbl);
s_p_get_string(&slurmdbd_conf->storage_user,
"StorageUser", tbl);
if (!s_p_get_boolean((bool *)&slurmdbd_conf->track_wckey,
"TrackWCKey", tbl))
slurmdbd_conf->track_wckey = false;
if (!s_p_get_boolean((bool *)&slurmdbd_conf->track_ctld,
"TrackSlurmctldDown", tbl))
slurmdbd_conf->track_ctld = false;
if (a_events && slurmdbd_conf->purge_event)
slurmdbd_conf->purge_event |= SLURMDB_PURGE_ARCHIVE;
if (a_jobs && slurmdbd_conf->purge_job)
slurmdbd_conf->purge_job |= SLURMDB_PURGE_ARCHIVE;
if (a_resv && slurmdbd_conf->purge_resv)
slurmdbd_conf->purge_resv |= SLURMDB_PURGE_ARCHIVE;
if (a_steps && slurmdbd_conf->purge_step)
slurmdbd_conf->purge_step |= SLURMDB_PURGE_ARCHIVE;
if (a_suspend && slurmdbd_conf->purge_suspend)
slurmdbd_conf->purge_suspend |= SLURMDB_PURGE_ARCHIVE;
s_p_hashtbl_destroy(tbl);
}
xfree(conf_path);
if (slurmdbd_conf->auth_type == NULL)
slurmdbd_conf->auth_type = xstrdup(DEFAULT_SLURMDBD_AUTHTYPE);
if (slurmdbd_conf->dbd_host == NULL) {
error("slurmdbd.conf lacks DbdHost parameter, "
"using 'localhost'");
slurmdbd_conf->dbd_host = xstrdup("localhost");
}
if (slurmdbd_conf->dbd_addr == NULL)
slurmdbd_conf->dbd_addr = xstrdup(slurmdbd_conf->dbd_host);
if (slurmdbd_conf->pid_file == NULL)
slurmdbd_conf->pid_file = xstrdup(DEFAULT_SLURMDBD_PIDFILE);
if (slurmdbd_conf->dbd_port == 0)
slurmdbd_conf->dbd_port = SLURMDBD_PORT;
if (slurmdbd_conf->plugindir == NULL)
slurmdbd_conf->plugindir = xstrdup(default_plugin_path);
if (slurmdbd_conf->slurm_user_name) {
uid_t pw_uid;
if (uid_from_string (slurmdbd_conf->slurm_user_name,
&pw_uid) < 0)
fatal("Invalid user for SlurmUser %s, ignored",
slurmdbd_conf->slurm_user_name);
else
slurmdbd_conf->slurm_user_id = pw_uid;
} else {
slurmdbd_conf->slurm_user_name = xstrdup("root");
slurmdbd_conf->slurm_user_id = 0;
}
if (slurmdbd_conf->storage_type == NULL)
fatal("StorageType must be specified");
if (!xstrcmp(slurmdbd_conf->storage_type,
"accounting_storage/slurmdbd")) {
fatal("StorageType=%s is invalid in slurmdbd.conf",
slurmdbd_conf->storage_type);
}
if (!slurmdbd_conf->storage_host)
slurmdbd_conf->storage_host = xstrdup(DEFAULT_STORAGE_HOST);
if (!slurmdbd_conf->storage_user)
slurmdbd_conf->storage_user = xstrdup(getlogin());
if (!xstrcmp(slurmdbd_conf->storage_type,
"accounting_storage/mysql")) {
if (!slurmdbd_conf->storage_port)
slurmdbd_conf->storage_port = DEFAULT_MYSQL_PORT;
if (!slurmdbd_conf->storage_loc)
slurmdbd_conf->storage_loc =
xstrdup(DEFAULT_ACCOUNTING_DB);
} else {
if (!slurmdbd_conf->storage_port)
slurmdbd_conf->storage_port = DEFAULT_STORAGE_PORT;
if (!slurmdbd_conf->storage_loc)
slurmdbd_conf->storage_loc =
xstrdup(DEFAULT_STORAGE_LOC);
}
if (slurmdbd_conf->archive_dir) {
if (stat(slurmdbd_conf->archive_dir, &buf) < 0)
fatal("Failed to stat the archive directory %s: %m",
slurmdbd_conf->archive_dir);
if (!(buf.st_mode & S_IFDIR))
fatal("archive directory %s isn't a directory",
slurmdbd_conf->archive_dir);
if (access(slurmdbd_conf->archive_dir, W_OK) < 0)
fatal("archive directory %s is not writable",
slurmdbd_conf->archive_dir);
}
if (slurmdbd_conf->archive_script) {
if (stat(slurmdbd_conf->archive_script, &buf) < 0)
fatal("Failed to stat the archive script %s: %m",
slurmdbd_conf->archive_dir);
if (!(buf.st_mode & S_IFREG))
fatal("archive script %s isn't a regular file",
slurmdbd_conf->archive_script);
if (access(slurmdbd_conf->archive_script, X_OK) < 0)
fatal("archive script %s is not executable",
slurmdbd_conf->archive_script);
}
if (!slurmdbd_conf->purge_event)
slurmdbd_conf->purge_event = NO_VAL;
if (!slurmdbd_conf->purge_job)
slurmdbd_conf->purge_job = NO_VAL;
if (!slurmdbd_conf->purge_resv)
slurmdbd_conf->purge_resv = NO_VAL;
if (!slurmdbd_conf->purge_step)
slurmdbd_conf->purge_step = NO_VAL;
if (!slurmdbd_conf->purge_suspend)
slurmdbd_conf->purge_suspend = NO_VAL;
slurm_mutex_unlock(&conf_mutex);
return SLURM_SUCCESS;
}
extern int as_mysql_add_clusters(mysql_conn_t *mysql_conn, uint32_t uid,
List cluster_list)
{
ListIterator itr = NULL;
int rc = SLURM_SUCCESS;
slurmdb_cluster_rec_t *object = NULL;
char *cols = NULL, *vals = NULL, *extra = NULL,
*query = NULL, *tmp_extra = NULL;
time_t now = time(NULL);
char *user_name = NULL;
int affect_rows = 0;
int added = 0;
List assoc_list = NULL;
slurmdb_association_rec_t *assoc = NULL;
if (check_connection(mysql_conn) != SLURM_SUCCESS)
return ESLURM_DB_CONNECTION;
assoc_list = list_create(slurmdb_destroy_association_rec);
user_name = uid_to_string((uid_t) uid);
/* Since adding tables make it so you can't roll back, if
there is an error there is no way to easily remove entries
in the database, so we will create the tables first and
then after that works out then add them to the mix.
*/
itr = list_iterator_create(cluster_list);
while ((object = list_next(itr))) {
if (!object->name || !object->name[0]) {
error("We need a cluster name to add.");
rc = SLURM_ERROR;
list_remove(itr);
continue;
}
if ((rc = create_cluster_tables(mysql_conn,
object->name))
!= SLURM_SUCCESS) {
xfree(extra);
xfree(cols);
xfree(vals);
added = 0;
if (mysql_errno(mysql_conn->db_conn)
== ER_WRONG_TABLE_NAME)
rc = ESLURM_BAD_NAME;
goto end_it;
}
}
/* Now that all the tables were created successfully lets go
ahead and add it to the system.
*/
list_iterator_reset(itr);
while ((object = list_next(itr))) {
xstrcat(cols, "creation_time, mod_time, acct");
xstrfmtcat(vals, "%ld, %ld, 'root'", now, now);
xstrfmtcat(extra, ", mod_time=%ld", now);
if (object->root_assoc)
setup_association_limits(object->root_assoc, &cols,
&vals, &extra,
QOS_LEVEL_SET, 1);
xstrfmtcat(query,
"insert into %s (creation_time, mod_time, "
"name, classification) "
"values (%ld, %ld, '%s', %u) "
"on duplicate key update deleted=0, mod_time=%ld, "
"control_host='', control_port=0, "
"classification=%u, flags=0",
cluster_table,
now, now, object->name, object->classification,
now, object->classification);
if (debug_flags & DEBUG_FLAG_DB_ASSOC)
DB_DEBUG(mysql_conn->conn, "query\n%s", query);
rc = mysql_db_query(mysql_conn, query);
xfree(query);
if (rc != SLURM_SUCCESS) {
error("Couldn't add cluster %s", object->name);
xfree(extra);
xfree(cols);
xfree(vals);
added=0;
break;
}
affect_rows = last_affected_rows(mysql_conn);
if (!affect_rows) {
debug2("nothing changed %d", affect_rows);
xfree(extra);
xfree(cols);
xfree(vals);
continue;
}
xstrfmtcat(query,
"insert into \"%s_%s\" (%s, lft, rgt) "
"values (%s, 1, 2) "
"on duplicate key update deleted=0, "
"id_assoc=LAST_INSERT_ID(id_assoc)%s;",
object->name, assoc_table, cols,
vals,
extra);
xfree(cols);
xfree(vals);
if (debug_flags & DEBUG_FLAG_DB_ASSOC)
DB_DEBUG(mysql_conn->conn, "query\n%s", query);
rc = mysql_db_query(mysql_conn, query);
xfree(query);
if (rc != SLURM_SUCCESS) {
error("Couldn't add cluster root assoc");
xfree(extra);
added=0;
break;
}
/* we always have a ', ' as the first 2 chars */
tmp_extra = slurm_add_slash_to_quotes(extra+2);
xstrfmtcat(query,
"insert into %s "
"(timestamp, action, name, actor, info) "
"values (%ld, %u, '%s', '%s', '%s');",
txn_table, now, DBD_ADD_CLUSTERS,
object->name, user_name, tmp_extra);
xfree(tmp_extra);
xfree(extra);
debug4("%d(%s:%d) query\n%s",
mysql_conn->conn, THIS_FILE, __LINE__, query);
rc = mysql_db_query(mysql_conn, query);
xfree(query);
if (rc != SLURM_SUCCESS) {
error("Couldn't add txn");
} else {
ListIterator check_itr;
char *tmp_name;
added++;
/* add it to the list and sort */
slurm_mutex_lock(&as_mysql_cluster_list_lock);
check_itr = list_iterator_create(as_mysql_cluster_list);
while ((tmp_name = list_next(check_itr))) {
if (!strcmp(tmp_name, object->name))
break;
}
list_iterator_destroy(check_itr);
if (!tmp_name) {
list_append(as_mysql_cluster_list,
xstrdup(object->name));
list_sort(as_mysql_cluster_list,
(ListCmpF)slurm_sort_char_list_asc);
} else
error("Cluster %s(%s) appears to already be in "
"our cache list, not adding.", tmp_name,
object->name);
slurm_mutex_unlock(&as_mysql_cluster_list_lock);
}
/* Add user root by default to run from the root
* association. This gets popped off so we need to
* read it every time here.
*/
assoc = xmalloc(sizeof(slurmdb_association_rec_t));
slurmdb_init_association_rec(assoc, 0);
list_append(assoc_list, assoc);
assoc->cluster = xstrdup(object->name);
assoc->user = xstrdup("root");
assoc->acct = xstrdup("root");
assoc->is_def = 1;
if (as_mysql_add_assocs(mysql_conn, uid, assoc_list)
== SLURM_ERROR) {
error("Problem adding root user association");
rc = SLURM_ERROR;
}
}
end_it:
list_iterator_destroy(itr);
xfree(user_name);
list_destroy(assoc_list);
if (!added)
reset_mysql_conn(mysql_conn);
return rc;
}
static void _atfork_parent() { slurm_mutex_unlock(&log_lock); }