static void _spawn_signal_thread(uint64_t cont_id, int signal)
{
agent_arg_t *agent_arg_ptr;
agent_arg_ptr = xmalloc(sizeof(agent_arg_t));
agent_arg_ptr->cont_id = cont_id;
agent_arg_ptr->signal = signal;
slurm_thread_create_detached(NULL, _sig_agent, agent_arg_ptr);
}
void heartbeat_start(void)
{
if (!slurmctld_conf.control_addr[1]) {
debug("No BackupController, not launching heartbeat.");
return;
}
slurm_mutex_lock(&heartbeat_mutex);
slurm_thread_create_detached(NULL, _heartbeat_thread, NULL);
heart_beating = true;
slurm_mutex_unlock(&heartbeat_mutex);
}
static void _start_msg_tree_internal(hostlist_t hl, hostlist_t* sp_hl,
fwd_tree_t *fwd_tree_in,
int hl_count)
{
int j;
fwd_tree_t *fwd_tree;
xassert((hl || sp_hl) && !(hl && sp_hl));
xassert(fwd_tree_in);
xassert(fwd_tree_in->p_thr_count);
xassert(fwd_tree_in->tree_mutex);
xassert(fwd_tree_in->notify);
xassert(fwd_tree_in->ret_list);
if (hl)
xassert(hl_count == hostlist_count(hl));
if (fwd_tree_in->timeout <= 0)
/* convert secs to msec */
fwd_tree_in->timeout = slurm_get_msg_timeout() * 1000;
for (j = 0; j < hl_count; j++) {
fwd_tree = xmalloc(sizeof(fwd_tree_t));
memcpy(fwd_tree, fwd_tree_in, sizeof(fwd_tree_t));
if (sp_hl) {
fwd_tree->tree_hl = sp_hl[j];
sp_hl[j] = NULL;
} else if (hl) {
char *name = hostlist_shift(hl);
fwd_tree->tree_hl = hostlist_create(name);
free(name);
}
/*
* Lock and increase thread counter, we need that to protect
* the start_msg_tree waiting loop that was originally designed
* around a "while ((count < host_count))" loop. In case where a
* fwd thread was not able to get all the return codes from
* children, the waiting loop was deadlocked.
*/
slurm_mutex_lock(fwd_tree->tree_mutex);
(*fwd_tree->p_thr_count)++;
slurm_mutex_unlock(fwd_tree->tree_mutex);
slurm_thread_create_detached(NULL, _fwd_tree_thread, fwd_tree);
}
}
static void _forward_msg_internal(hostlist_t hl, hostlist_t* sp_hl,
forward_struct_t *fwd_struct,
header_t *header, int timeout,
int hl_count)
{
int j;
forward_msg_t *fwd_msg = NULL;
char *buf = NULL, *tmp_char = NULL;
if (timeout <= 0)
/* convert secs to msec */
timeout = slurm_get_msg_timeout() * 1000;
for (j = 0; j < hl_count; j++) {
fwd_msg = xmalloc(sizeof(forward_msg_t));
fwd_msg->fwd_struct = fwd_struct;
fwd_msg->timeout = timeout;
memcpy(&fwd_msg->header.orig_addr,
&header->orig_addr,
sizeof(slurm_addr_t));
fwd_msg->header.version = header->version;
fwd_msg->header.flags = header->flags;
fwd_msg->header.msg_type = header->msg_type;
fwd_msg->header.body_length = header->body_length;
fwd_msg->header.ret_list = NULL;
fwd_msg->header.ret_cnt = 0;
if (sp_hl) {
buf = hostlist_ranged_string_xmalloc(sp_hl[j]);
hostlist_destroy(sp_hl[j]);
} else {
tmp_char = hostlist_shift(hl);
buf = xstrdup(tmp_char);
free(tmp_char);
}
forward_init(&fwd_msg->header.forward, NULL);
fwd_msg->header.forward.nodelist = buf;
slurm_thread_create_detached(NULL, _forward_thread, fwd_msg);
}
}
static void _signal_while_allocating(int signo)
{
int *local_signal;
/*
* There are places where _signal_while_allocating() can't be
* put into a thread, but if this isn't on a separate thread
* and we try to print something using the log functions and
* it just so happens to be in a poll or something we can get
* deadlock. So after the signal happens we are able to spawn
* a thread here and avoid the deadlock.
*
* SO, DON'T PRINT ANYTHING IN THIS FUNCTION.
*/
local_signal = xmalloc(sizeof(int));
*local_signal = signo;
slurm_thread_create_detached(NULL, _safe_signal_while_allocating,
local_signal);
}
/*
* The remainder of this file implements the standard Slurm checkpoint API.
*/
extern int slurm_ckpt_op (uint32_t job_id, uint32_t step_id,
struct step_record *step_ptr, uint16_t op,
uint16_t data, char *image_dir, time_t * event_time,
uint32_t *error_code, char **error_msg )
{
int rc = SLURM_SUCCESS;
struct check_job_info *check_ptr;
uint16_t done_sig = 0;
struct job_record *job_ptr;
struct node_record *node_ptr;
char *nodelist;
struct ckpt_req *req_ptr;
/* job/step checked already */
job_ptr = find_job_record(job_id);
if (!job_ptr)
return ESLURM_INVALID_JOB_ID;
if (step_id == SLURM_BATCH_SCRIPT) {
check_ptr = (struct check_job_info *)job_ptr->check_job;
node_ptr = find_first_node_record(job_ptr->node_bitmap);
nodelist = node_ptr->name;
} else {
step_ptr = find_step_record(job_ptr, step_id);
if (!step_ptr)
return ESLURM_INVALID_JOB_ID;
check_ptr = (struct check_job_info *)step_ptr->check_job;
nodelist = step_ptr->step_layout->node_list;
}
xassert(check_ptr);
switch (op) {
case CHECK_ABLE:
if (check_ptr->disabled)
rc = ESLURM_DISABLED;
else {
*event_time = check_ptr->time_stamp;
rc = SLURM_SUCCESS;
}
break;
case CHECK_DISABLE:
check_ptr->disabled++;
break;
case CHECK_ENABLE:
check_ptr->disabled--;
break;
case CHECK_REQUEUE:
if (step_id != SLURM_BATCH_SCRIPT) {
rc = ESLURM_NOT_SUPPORTED;
break;
}
/* no break */
case CHECK_VACATE:
done_sig = SIGTERM;
/* no break */
case CHECK_CREATE:
if (check_ptr->disabled) {
rc = ESLURM_DISABLED;
break;
}
if (check_ptr->time_stamp != 0) {
rc = EALREADY;
break;
}
check_ptr->time_stamp = time(NULL);
check_ptr->error_code = 0;
xfree(check_ptr->error_msg);
req_ptr = xmalloc(sizeof(struct ckpt_req));
req_ptr->gid = job_ptr->group_id;
req_ptr->uid = job_ptr->user_id;
req_ptr->job_id = job_id;
req_ptr->step_id = step_id;
req_ptr->begin_time = check_ptr->time_stamp;
req_ptr->wait = data;
req_ptr->image_dir = xstrdup(image_dir);
req_ptr->nodelist = xstrdup(nodelist);
req_ptr->sig_done = done_sig;
req_ptr->op = op;
slurm_thread_create_detached(NULL, _ckpt_agent_thr, req_ptr);
break;
case CHECK_RESTART:
if (step_id != SLURM_BATCH_SCRIPT) {
rc = ESLURM_NOT_SUPPORTED;
break;
}
/* create a batch job from saved desc */
rc = ESLURM_NOT_SUPPORTED;
/* TODO: save job script */
break;
case CHECK_ERROR:
xassert(error_code);
xassert(error_msg);
*error_code = check_ptr->error_code;
xfree(*error_msg);
*error_msg = xstrdup(check_ptr->error_msg);
break;
default:
error("Invalid checkpoint operation: %d", op);
rc = EINVAL;
}
return rc;
}
static void _launch_app(srun_job_t *job, List srun_job_list, bool got_alloc)
{
ListIterator opt_iter, job_iter;
opt_t *opt_local = NULL;
_launch_app_data_t *opts;
int total_ntasks = 0, total_nnodes = 0, step_cnt = 0, node_offset = 0;
pthread_mutex_t step_mutex = PTHREAD_MUTEX_INITIALIZER;
pthread_cond_t step_cond = PTHREAD_COND_INITIALIZER;
srun_job_t *first_job = NULL;
char *launch_type, *pack_node_list = NULL;
bool need_mpir = false;
uint16_t *tmp_task_cnt = NULL, *pack_task_cnts = NULL;
uint32_t **tmp_tids = NULL, **pack_tids = NULL;
launch_type = slurm_get_launch_type();
if (launch_type && strstr(launch_type, "slurm"))
need_mpir = true;
xfree(launch_type);
if (srun_job_list) {
int pack_step_cnt = list_count(srun_job_list);
first_job = (srun_job_t *) list_peek(srun_job_list);
if (!opt_list) {
if (first_job)
fini_srun(first_job, got_alloc, &global_rc, 0);
fatal("%s: have srun_job_list, but no opt_list",
__func__);
}
job_iter = list_iterator_create(srun_job_list);
while ((job = (srun_job_t *) list_next(job_iter))) {
char *node_list = NULL;
int i, node_inx;
total_ntasks += job->ntasks;
total_nnodes += job->nhosts;
xrealloc(pack_task_cnts, sizeof(uint16_t)*total_nnodes);
(void) slurm_step_ctx_get(job->step_ctx,
SLURM_STEP_CTX_TASKS,
&tmp_task_cnt);
if (!tmp_task_cnt) {
fatal("%s: job %u has NULL task array",
__func__, job->jobid);
break; /* To eliminate CLANG error */
}
memcpy(pack_task_cnts + node_offset, tmp_task_cnt,
sizeof(uint16_t) * job->nhosts);
xrealloc(pack_tids, sizeof(uint32_t *) * total_nnodes);
(void) slurm_step_ctx_get(job->step_ctx,
SLURM_STEP_CTX_TIDS,
&tmp_tids);
if (!tmp_tids) {
fatal("%s: job %u has NULL task ID array",
__func__, job->jobid);
break; /* To eliminate CLANG error */
}
for (node_inx = 0; node_inx < job->nhosts; node_inx++) {
uint32_t *node_tids;
node_tids = xmalloc(sizeof(uint32_t) *
tmp_task_cnt[node_inx]);
for (i = 0; i < tmp_task_cnt[node_inx]; i++) {
node_tids[i] = tmp_tids[node_inx][i] +
job->pack_task_offset;
}
pack_tids[node_offset + node_inx] =
node_tids;
}
(void) slurm_step_ctx_get(job->step_ctx,
SLURM_STEP_CTX_NODE_LIST,
&node_list);
if (!node_list) {
fatal("%s: job %u has NULL hostname",
__func__, job->jobid);
}
if (pack_node_list)
xstrfmtcat(pack_node_list, ",%s", node_list);
else
pack_node_list = xstrdup(node_list);
xfree(node_list);
node_offset += job->nhosts;
}
list_iterator_reset(job_iter);
_reorder_pack_recs(&pack_node_list, &pack_task_cnts,
&pack_tids, total_nnodes);
if (need_mpir)
mpir_init(total_ntasks);
opt_iter = list_iterator_create(opt_list);
while ((opt_local = (opt_t *) list_next(opt_iter))) {
job = (srun_job_t *) list_next(job_iter);
if (!job) {
slurm_mutex_lock(&step_mutex);
while (step_cnt > 0)
slurm_cond_wait(&step_cond,&step_mutex);
slurm_mutex_unlock(&step_mutex);
if (first_job) {
fini_srun(first_job, got_alloc,
&global_rc, 0);
}
fatal("%s: job allocation count does not match request count (%d != %d)",
__func__, list_count(srun_job_list),
list_count(opt_list));
break; /* To eliminate CLANG error */
}
slurm_mutex_lock(&step_mutex);
step_cnt++;
slurm_mutex_unlock(&step_mutex);
job->pack_node_list = xstrdup(pack_node_list);
if ((pack_step_cnt > 1) && pack_task_cnts) {
xassert(node_offset == job->pack_nnodes);
job->pack_task_cnts = xmalloc(sizeof(uint16_t) *
job->pack_nnodes);
memcpy(job->pack_task_cnts, pack_task_cnts,
sizeof(uint16_t) * job->pack_nnodes);
job->pack_tids = xmalloc(sizeof(uint32_t *) *
job->pack_nnodes);
memcpy(job->pack_tids, pack_tids,
sizeof(uint32_t *) * job->pack_nnodes);
}
opts = xmalloc(sizeof(_launch_app_data_t));
opts->got_alloc = got_alloc;
opts->job = job;
opts->opt_local = opt_local;
opts->step_cond = &step_cond;
opts->step_cnt = &step_cnt;
opts->step_mutex = &step_mutex;
opt_local->pack_step_cnt = pack_step_cnt;
slurm_thread_create_detached(NULL, _launch_one_app,
opts);
}
xfree(pack_node_list);
xfree(pack_task_cnts);
list_iterator_destroy(job_iter);
list_iterator_destroy(opt_iter);
slurm_mutex_lock(&step_mutex);
while (step_cnt > 0)
slurm_cond_wait(&step_cond, &step_mutex);
slurm_mutex_unlock(&step_mutex);
if (first_job)
fini_srun(first_job, got_alloc, &global_rc, 0);
} else {
if (need_mpir)
mpir_init(job->ntasks);
opts = xmalloc(sizeof(_launch_app_data_t));
opts->got_alloc = got_alloc;
opts->job = job;
opts->opt_local = &opt;
opt.pack_step_cnt = 1;
_launch_one_app(opts);
fini_srun(job, got_alloc, &global_rc, 0);
}
}
