/*
* slurm_terminate_job_step - terminates a job step by sending a
* REQUEST_TERMINATE_TASKS rpc to all slurmd of a job step.
* IN job_id - the job's id
* IN step_id - the job step's id - use SLURM_BATCH_SCRIPT as the step_id
* to terminate a job's batch script
* RET 0 on success, otherwise return -1 and set errno to indicate the error
*/
extern int
slurm_terminate_job_step (uint32_t job_id, uint32_t step_id)
{
resource_allocation_response_msg_t *alloc_info = NULL;
job_step_info_response_msg_t *step_info = NULL;
int rc = 0;
int i;
int save_errno = 0;
if (slurm_allocation_lookup_lite(job_id, &alloc_info)) {
return -1;
}
/*
* The controller won't give us info about the batch script job step,
* so we need to handle that seperately.
*/
if (step_id == SLURM_BATCH_SCRIPT) {
rc = _terminate_batch_script_step(alloc_info);
slurm_free_resource_allocation_response_msg(alloc_info);
errno = rc;
return rc ? -1 : 0;
}
/*
* Otherwise, look through the list of job step info and find
* the one matching step_id. Terminate that step.
*/
rc = slurm_get_job_steps((time_t)0, job_id, step_id,
&step_info, SHOW_ALL);
if (rc != 0) {
save_errno = errno;
goto fail;
}
for (i = 0; i < step_info->job_step_count; i++) {
if ((step_info->job_steps[i].job_id == job_id) &&
(step_info->job_steps[i].step_id == step_id)) {
rc = _terminate_job_step(&step_info->job_steps[i],
alloc_info);
save_errno = errno;
break;
}
}
slurm_free_job_step_info_response_msg(step_info);
fail:
slurm_free_resource_allocation_response_msg(alloc_info);
errno = save_errno;
return rc ? -1 : 0;
}
/*
* Wait until a job is ready to execute or enters some failed state
* RET 1: job ready to run
* 0: job can't run (cancelled, failure state, timeout, etc.)
*/
extern int scontrol_job_ready(char *job_id_str)
{
int rc;
uint32_t job_id;
job_id = atoi(job_id_str);
if (job_id <= 0) {
fprintf(stderr, "Invalid job_id %s", job_id_str);
return SLURM_ERROR;
}
if (cluster_flags & CLUSTER_FLAG_BG) {
resource_allocation_response_msg_t *alloc;
rc = slurm_allocation_lookup_lite(job_id, &alloc);
if (rc == SLURM_SUCCESS) {
rc = _wait_bluegene_block_ready(alloc);
slurm_free_resource_allocation_response_msg(alloc);
} else {
error("slurm_allocation_lookup_lite: %m");
rc = SLURM_ERROR;
}
} else
rc = _wait_nodes_ready(job_id);
return rc;
}
/*
* slurm_signal_job - send the specified signal to all steps of an existing job
* IN job_id - the job's id
* IN signal - signal number
* RET 0 on success, otherwise return -1 and set errno to indicate the error
*/
extern int
slurm_signal_job (uint32_t job_id, uint16_t signal)
{
int rc = SLURM_SUCCESS;
resource_allocation_response_msg_t *alloc_info = NULL;
signal_job_msg_t rpc;
if (slurm_allocation_lookup_lite(job_id, &alloc_info)) {
rc = slurm_get_errno();
goto fail1;
}
/* same remote procedure call for each node */
rpc.job_id = job_id;
rpc.signal = (uint32_t)signal;
rc = _local_send_recv_rc_msgs(alloc_info->node_list,
REQUEST_SIGNAL_JOB, &rpc);
slurm_free_resource_allocation_response_msg(alloc_info);
fail1:
if (rc) {
slurm_seterrno_ret(rc);
} else {
return SLURM_SUCCESS;
}
}
static void _purge_agent_args(agent_arg_t *agent_arg_ptr)
{
if (agent_arg_ptr == NULL)
return;
hostlist_destroy(agent_arg_ptr->hostlist);
xfree(agent_arg_ptr->addr);
if (agent_arg_ptr->msg_args) {
if (agent_arg_ptr->msg_type == REQUEST_BATCH_JOB_LAUNCH)
_slurmctld_free_batch_job_launch_msg(
agent_arg_ptr->msg_args);
else if (agent_arg_ptr->msg_type ==
RESPONSE_RESOURCE_ALLOCATION)
slurm_free_resource_allocation_response_msg(
agent_arg_ptr->msg_args);
else if ((agent_arg_ptr->msg_type == REQUEST_ABORT_JOB) ||
(agent_arg_ptr->msg_type == REQUEST_TERMINATE_JOB) ||
(agent_arg_ptr->msg_type == REQUEST_KILL_TIMELIMIT))
slurm_free_kill_job_msg(agent_arg_ptr->msg_args);
else if (agent_arg_ptr->msg_type == SRUN_USER_MSG)
slurm_free_srun_user_msg(agent_arg_ptr->msg_args);
else if (agent_arg_ptr->msg_type == SRUN_EXEC)
slurm_free_srun_exec_msg(agent_arg_ptr->msg_args);
else if (agent_arg_ptr->msg_type == SRUN_NODE_FAIL)
slurm_free_srun_node_fail_msg(agent_arg_ptr->msg_args);
else if (agent_arg_ptr->msg_type == SRUN_STEP_MISSING)
slurm_free_srun_step_missing_msg(
agent_arg_ptr->msg_args);
else if (agent_arg_ptr->msg_type == REQUEST_JOB_NOTIFY)
slurm_free_job_notify_msg(agent_arg_ptr->msg_args);
else
xfree(agent_arg_ptr->msg_args);
}
xfree(agent_arg_ptr);
}
static void _free_srun_alloc(void *x)
{
resource_allocation_response_msg_t *alloc_msg;
alloc_msg = (resource_allocation_response_msg_t *) x;
/* NULL working_cluster_rec because it's pointing to global memory */
alloc_msg->working_cluster_rec = NULL;
slurm_free_resource_allocation_response_msg(alloc_msg);
}
static void _update_job_size(uint32_t job_id)
{
resource_allocation_response_msg_t *alloc_info;
char *fname_csh = NULL, *fname_sh = NULL;
FILE *resize_csh = NULL, *resize_sh = NULL;
if (!getenv("SLURM_JOBID"))
return; /*No job environment here to update */
if (slurm_allocation_lookup_lite(job_id, &alloc_info) !=
SLURM_SUCCESS) {
slurm_perror("slurm_allocation_lookup_lite");
return;
}
xstrfmtcat(fname_csh, "slurm_job_%u_resize.csh", job_id);
xstrfmtcat(fname_sh, "slurm_job_%u_resize.sh", job_id);
(void) unlink(fname_csh);
(void) unlink(fname_sh);
if (!(resize_csh = fopen(fname_csh, "w"))) {
fprintf(stderr, "Could not create file %s: %s\n", fname_csh,
strerror(errno));
goto fini;
}
if (!(resize_sh = fopen(fname_sh, "w"))) {
fprintf(stderr, "Could not create file %s: %s\n", fname_sh,
strerror(errno));
goto fini;
}
chmod(fname_csh, 0700); /* Make file executable */
chmod(fname_sh, 0700);
if (getenv("SLURM_NODELIST")) {
fprintf(resize_sh, "export SLURM_NODELIST=\"%s\"\n",
alloc_info->node_list);
fprintf(resize_csh, "setenv SLURM_NODELIST \"%s\"\n",
alloc_info->node_list);
}
if (getenv("SLURM_JOB_NODELIST")) {
fprintf(resize_sh, "export SLURM_JOB_NODELIST=\"%s\"\n",
alloc_info->node_list);
fprintf(resize_csh, "setenv SLURM_JOB_NODELIST \"%s\"\n",
alloc_info->node_list);
}
if (getenv("SLURM_NNODES")) {
fprintf(resize_sh, "export SLURM_NNODES=%u\n",
alloc_info->node_cnt);
fprintf(resize_csh, "setenv SLURM_NNODES %u\n",
alloc_info->node_cnt);
}
if (getenv("SLURM_JOB_NUM_NODES")) {
fprintf(resize_sh, "export SLURM_JOB_NUM_NODES=%u\n",
alloc_info->node_cnt);
fprintf(resize_csh, "setenv SLURM_JOB_NUM_NODES %u\n",
alloc_info->node_cnt);
}
if (getenv("SLURM_JOB_CPUS_PER_NODE")) {
char *tmp;
tmp = uint32_compressed_to_str(alloc_info->num_cpu_groups,
alloc_info->cpus_per_node,
alloc_info->cpu_count_reps);
fprintf(resize_sh, "export SLURM_JOB_CPUS_PER_NODE=\"%s\"\n",
tmp);
fprintf(resize_csh, "setenv SLURM_JOB_CPUS_PER_NODE \"%s\"\n",
tmp);
xfree(tmp);
}
if (getenv("SLURM_TASKS_PER_NODE")) {
/* We don't have sufficient information to recreate this */
fprintf(resize_sh, "unset SLURM_TASKS_PER_NODE\n");
fprintf(resize_csh, "unsetenv SLURM_TASKS_PER_NODE\n");
}
printf("To reset SLURM environment variables, execute\n");
printf(" For bash or sh shells: . ./%s\n", fname_sh);
printf(" For csh shells: source ./%s\n", fname_csh);
fini: slurm_free_resource_allocation_response_msg(alloc_info);
xfree(fname_csh);
xfree(fname_sh);
if (resize_csh)
fclose(resize_csh);
if (resize_sh)
fclose(resize_sh);
}
/*
* slurm_allocate_resources_blocking
* allocate resources for a job request. This call will block until
* the allocation is granted, or the specified timeout limit is reached.
* IN req - description of resource allocation request
* IN timeout - amount of time, in seconds, to wait for a response before
* giving up.
* A timeout of zero will wait indefinitely.
* IN pending_callback - If the allocation cannot be granted immediately,
* the controller will put the job in the PENDING state. If
* pending callback is not NULL, it will be called with the job_id
* of the pending job as the sole parameter.
*
* RET allocation structure on success, NULL on error set errno to
* indicate the error (errno will be ETIMEDOUT if the timeout is reached
* with no allocation granted)
* NOTE: free the response using slurm_free_resource_allocation_response_msg()
*/
resource_allocation_response_msg_t *
slurm_allocate_resources_blocking (const job_desc_msg_t *user_req,
time_t timeout,
void(*pending_callback)(uint32_t job_id))
{
int rc;
slurm_msg_t req_msg;
slurm_msg_t resp_msg;
resource_allocation_response_msg_t *resp = NULL;
char *hostname = NULL;
uint32_t job_id;
job_desc_msg_t *req;
listen_t *listen = NULL;
int errnum = SLURM_SUCCESS;
slurm_msg_t_init(&req_msg);
slurm_msg_t_init(&resp_msg);
/* make a copy of the user's job description struct so that we
* can make changes before contacting the controller */
req = (job_desc_msg_t *)xmalloc(sizeof(job_desc_msg_t));
if (req == NULL)
return NULL;
memcpy(req, user_req, sizeof(job_desc_msg_t));
/*
* set Node and session id for this request
*/
if (req->alloc_sid == NO_VAL)
req->alloc_sid = getsid(0);
if (user_req->alloc_node != NULL) {
req->alloc_node = xstrdup(user_req->alloc_node);
} else if ((hostname = xshort_hostname()) != NULL) {
req->alloc_node = hostname;
} else {
error("Could not get local hostname,"
" forcing immediate allocation mode.");
req->immediate = 1;
}
if (!req->immediate) {
listen = _create_allocation_response_socket(hostname);
if (listen == NULL) {
xfree(req);
return NULL;
}
req->alloc_resp_port = listen->port;
}
req_msg.msg_type = REQUEST_RESOURCE_ALLOCATION;
req_msg.data = req;
rc = slurm_send_recv_controller_msg(&req_msg, &resp_msg);
if (rc == SLURM_SOCKET_ERROR) {
int errnum = errno;
destroy_forward(&req_msg.forward);
destroy_forward(&resp_msg.forward);
if (!req->immediate)
_destroy_allocation_response_socket(listen);
xfree(req);
errno = errnum;
return NULL;
}
switch (resp_msg.msg_type) {
case RESPONSE_SLURM_RC:
if (_handle_rc_msg(&resp_msg) < 0) {
/* will reach this when the allocation fails */
errnum = errno;
} else {
/* shouldn't get here */
errnum = -1;
}
break;
case RESPONSE_RESOURCE_ALLOCATION:
/* Yay, the controller has acknowledged our request! But did
we really get an allocation yet? */
resp = (resource_allocation_response_msg_t *) resp_msg.data;
if (resp->node_cnt > 0) {
/* yes, allocation has been granted */
errno = SLURM_PROTOCOL_SUCCESS;
} else if (!req->immediate) {
if (resp->error_code != SLURM_SUCCESS)
info("%s", slurm_strerror(resp->error_code));
/* no, we need to wait for a response */
job_id = resp->job_id;
slurm_free_resource_allocation_response_msg(resp);
if (pending_callback != NULL)
pending_callback(job_id);
resp = _wait_for_allocation_response(job_id, listen,
timeout);
/* If NULL, we didn't get the allocation in
the time desired, so just free the job id */
if ((resp == NULL) && (errno != ESLURM_ALREADY_DONE)) {
errnum = errno;
slurm_complete_job(job_id, -1);
}
}
break;
default:
errnum = SLURM_UNEXPECTED_MSG_ERROR;
resp = NULL;
}
destroy_forward(&req_msg.forward);
destroy_forward(&resp_msg.forward);
if (!req->immediate)
_destroy_allocation_response_socket(listen);
xfree(req);
errno = errnum;
return resp;
}
resource_allocation_response_msg_t *
allocate_nodes(void)
{
resource_allocation_response_msg_t *resp = NULL;
job_desc_msg_t *j = job_desc_msg_create_from_opts();
slurm_allocation_callbacks_t callbacks;
int i;
if (!j)
return NULL;
/* Do not re-use existing job id when submitting new job
* from within a running job */
if ((j->job_id != NO_VAL) && !opt.jobid_set) {
info("WARNING: Creating SLURM job allocation from within "
"another allocation");
info("WARNING: You are attempting to initiate a second job");
if (!opt.jobid_set) /* Let slurmctld set jobid */
j->job_id = NO_VAL;
}
callbacks.ping = _ping_handler;
callbacks.timeout = _timeout_handler;
callbacks.job_complete = _job_complete_handler;
callbacks.user_msg = _user_msg_handler;
callbacks.node_fail = _node_fail_handler;
/* create message thread to handle pings and such from slurmctld */
msg_thr = slurm_allocation_msg_thr_create(&j->other_port, &callbacks);
/* NOTE: Do not process signals in separate pthread. The signal will
* cause slurm_allocate_resources_blocking() to exit immediately. */
xsignal_unblock(sig_array);
for (i = 0; sig_array[i]; i++)
xsignal(sig_array[i], _signal_while_allocating);
while (!resp) {
resp = slurm_allocate_resources_blocking(j, opt.immediate,
_set_pending_job_id);
if (destroy_job) {
/* cancelled by signal */
break;
} else if (!resp && !_retry()) {
break;
}
}
if (resp && !destroy_job) {
/*
* Allocation granted!
*/
pending_job_id = resp->job_id;
#ifdef HAVE_BG
if (!_wait_bluegene_block_ready(resp)) {
if(!destroy_job)
error("Something is wrong with the "
"boot of the block.");
goto relinquish;
}
#else
if (!_wait_nodes_ready(resp)) {
if(!destroy_job)
error("Something is wrong with the "
"boot of the nodes.");
goto relinquish;
}
#endif
} else if (destroy_job) {
goto relinquish;
}
xsignal_block(sig_array);
job_desc_msg_destroy(j);
return resp;
relinquish:
slurm_free_resource_allocation_response_msg(resp);
if (!destroy_job)
slurm_complete_job(resp->job_id, 1);
exit(error_exit);
return NULL;
}
resource_allocation_response_msg_t *
allocate_nodes(bool handle_signals)
{
resource_allocation_response_msg_t *resp = NULL;
job_desc_msg_t *j = job_desc_msg_create_from_opts();
slurm_allocation_callbacks_t callbacks;
int i;
if (!j)
return NULL;
/* Do not re-use existing job id when submitting new job
* from within a running job */
if ((j->job_id != NO_VAL) && !opt.jobid_set) {
info("WARNING: Creating SLURM job allocation from within "
"another allocation");
info("WARNING: You are attempting to initiate a second job");
if (!opt.jobid_set) /* Let slurmctld set jobid */
j->job_id = NO_VAL;
}
callbacks.ping = _ping_handler;
callbacks.timeout = _timeout_handler;
callbacks.job_complete = _job_complete_handler;
callbacks.job_suspend = NULL;
callbacks.user_msg = _user_msg_handler;
callbacks.node_fail = _node_fail_handler;
/* create message thread to handle pings and such from slurmctld */
msg_thr = slurm_allocation_msg_thr_create(&j->other_port, &callbacks);
/* NOTE: Do not process signals in separate pthread. The signal will
* cause slurm_allocate_resources_blocking() to exit immediately. */
if (handle_signals) {
xsignal_unblock(sig_array);
for (i = 0; sig_array[i]; i++)
xsignal(sig_array[i], _signal_while_allocating);
}
while (!resp) {
resp = slurm_allocate_resources_blocking(j, opt.immediate,
_set_pending_job_id);
if (destroy_job) {
/* cancelled by signal */
break;
} else if (!resp && !_retry()) {
break;
}
}
if (resp && !destroy_job) {
/*
* Allocation granted!
*/
pending_job_id = resp->job_id;
/*
* These values could be changed while the job was
* pending so overwrite the request with what was
* allocated so we don't have issues when we use them
* in the step creation.
*/
if (opt.pn_min_memory != NO_VAL)
opt.pn_min_memory = (resp->pn_min_memory &
(~MEM_PER_CPU));
else if (opt.mem_per_cpu != NO_VAL)
opt.mem_per_cpu = (resp->pn_min_memory &
(~MEM_PER_CPU));
/*
* FIXME: timelimit should probably also be updated
* here since it could also change.
*/
#ifdef HAVE_BG
uint32_t node_cnt = 0;
select_g_select_jobinfo_get(resp->select_jobinfo,
SELECT_JOBDATA_NODE_CNT,
&node_cnt);
if ((node_cnt == 0) || (node_cnt == NO_VAL)) {
opt.min_nodes = node_cnt;
opt.max_nodes = node_cnt;
} /* else we just use the original request */
if (!_wait_bluegene_block_ready(resp)) {
if (!destroy_job)
error("Something is wrong with the "
"boot of the block.");
goto relinquish;
}
#else
opt.min_nodes = resp->node_cnt;
opt.max_nodes = resp->node_cnt;
if (!_wait_nodes_ready(resp)) {
if (!destroy_job)
error("Something is wrong with the "
"boot of the nodes.");
goto relinquish;
}
#endif
} else if (destroy_job) {
goto relinquish;
}
if (handle_signals)
xsignal_block(sig_array);
job_desc_msg_destroy(j);
return resp;
relinquish:
if (resp) {
if (!destroy_job)
slurm_complete_job(resp->job_id, 1);
slurm_free_resource_allocation_response_msg(resp);
}
exit(error_exit);
return NULL;
}
/* returns 1 if job and nodes are ready for job to begin, 0 otherwise */
static int _wait_nodes_ready(resource_allocation_response_msg_t *alloc)
{
int is_ready = 0, i, rc;
int cur_delay = 0;
int suspend_time, resume_time, max_delay;
suspend_time = slurm_get_suspend_timeout();
resume_time = slurm_get_resume_timeout();
if ((suspend_time == 0) || (resume_time == 0))
return 1; /* Power save mode disabled */
max_delay = suspend_time + resume_time;
max_delay *= 5; /* Allow for ResumeRate support */
pending_job_id = alloc->job_id;
for (i = 0; (cur_delay < max_delay); i++) {
if (i) {
if (i == 1)
verbose("Waiting for nodes to boot");
else
debug("still waiting");
sleep(POLL_SLEEP);
cur_delay += POLL_SLEEP;
}
rc = slurm_job_node_ready(alloc->job_id);
if (rc == READY_JOB_FATAL)
break; /* fatal error */
if ((rc == READY_JOB_ERROR) || (rc == EAGAIN))
continue; /* retry */
if ((rc & READY_JOB_STATE) == 0) /* job killed */
break;
if (rc & READY_NODE_STATE) { /* job and node ready */
is_ready = 1;
break;
}
if (destroy_job)
break;
}
if (is_ready) {
resource_allocation_response_msg_t *resp;
char *tmp_str;
if (i > 0)
verbose("Nodes %s are ready for job", alloc->node_list);
if (alloc->alias_list && !strcmp(alloc->alias_list, "TBD") &&
(slurm_allocation_lookup_lite(pending_job_id, &resp)
== SLURM_SUCCESS)) {
tmp_str = alloc->alias_list;
alloc->alias_list = resp->alias_list;
resp->alias_list = tmp_str;
slurm_free_resource_allocation_response_msg(resp);
}
} else if (!destroy_job)
error("Nodes %s are still not ready", alloc->node_list);
else /* allocation_interrupted and slurmctld not responing */
is_ready = 0;
pending_job_id = 0;
return is_ready;
}
int main (int argc, char *argv[])
{
int i, min_nodes = 1, max_nodes = 1, nodes, tasks = 0, rc = 0;
job_desc_msg_t job_req;
resource_allocation_response_msg_t *job_resp;
slurm_step_ctx_params_t step_params[1];
slurm_step_ctx_t *ctx = NULL;
slurm_step_launch_params_t launch[1];
char *task_argv[3];
int *fd_array = NULL;
int num_fd;
if (argc > 1) {
i = atoi(argv[1]);
if (i > 0)
min_nodes = i;
}
if (argc > 2) {
i = atoi(argv[2]);
if (i > 0)
max_nodes = i;
}
if (max_nodes < min_nodes)
max_nodes = min_nodes;
/* Create a job allocation */
slurm_init_job_desc_msg( &job_req );
job_req.min_nodes = min_nodes;
job_req.max_nodes = max_nodes;
job_req.user_id = getuid();
job_req.group_id = getgid();
job_req.time_limit = 1;
if (slurm_allocate_resources(&job_req, &job_resp)) {
slurm_perror ("slurm_allocate_resources");
printf("INFO: min_nodes=%u max_nodes=%u user_id=%u group_id=%u",
job_req.min_nodes, job_req.max_nodes,
job_req.user_id, job_req.group_id);
exit(0);
}
printf("job_id %u\n", job_resp->job_id);
fflush(stdout);
/* Wait for allocation request to be satisfied */
if ((job_resp->node_list == NULL) ||
(strlen(job_resp->node_list) == 0)) {
printf("Waiting for resource allocation\n");
fflush(stdout);
while ((job_resp->node_list == NULL) ||
(strlen(job_resp->node_list) == 0)) {
sleep(5);
if (slurm_allocation_lookup_lite(job_resp->job_id,
&job_resp) &&
(slurm_get_errno() != ESLURM_JOB_PENDING)) {
slurm_perror("slurm_confirm_allocation");
exit(0);
}
}
}
nodes = job_resp->node_cnt;
if (argc > 3)
tasks = atoi(argv[3]);
if (tasks < 1)
tasks = nodes * TASKS_PER_NODE;
if (tasks < nodes) {
fprintf(stderr, "Invalid task count argument\n");
exit(1);
}
printf("Starting %d tasks on %d nodes\n", tasks, nodes);
fflush(stdout);
/*
* Create a job step context.
*/
slurm_step_ctx_params_t_init(step_params);
step_params->job_id = job_resp->job_id;
step_params->min_nodes = nodes;
step_params->task_count = tasks;
ctx = slurm_step_ctx_create(step_params);
if ((ctx == NULL) &&
(slurm_get_errno() == ESLURM_PROLOG_RUNNING)) {
printf("SlurmctldProlog is still running, "
"sleep and try again\n");
sleep(10);
ctx = slurm_step_ctx_create(step_params);
}
if (ctx == NULL) {
slurm_perror("slurm_step_ctx_create");
rc = 1;
goto done;
}
/*
* Hack to run one task per node, regardless of what we set up
* when we created the job step context.
*/
if (slurm_step_ctx_daemon_per_node_hack(ctx) != SLURM_SUCCESS) {
slurm_perror("slurm_step_ctx_daemon_per_node_hack");
rc = 1;
goto done;
}
/*
* Launch the tasks using "user managed" IO.
* "user managed" IO means a TCP stream for each task, directly
* connected to the stdin, stdout, and stderr the task.
*/
slurm_step_launch_params_t_init(launch);
task_argv[0] = "./test7.3.io";
launch->argv = task_argv;
launch->argc = 1;
launch->user_managed_io = true; /* This is the key to using
"user managed" IO */
if (slurm_step_launch(ctx, launch, NULL) != SLURM_SUCCESS) {
slurm_perror("slurm_step_launch");
rc = 1;
goto done;
}
if (slurm_step_launch_wait_start(ctx) != SLURM_SUCCESS) {
slurm_perror("slurm_step_launch_wait_start");
rc =1;
goto done;
}
slurm_step_ctx_get(ctx, SLURM_STEP_CTX_USER_MANAGED_SOCKETS,
&num_fd, &fd_array);
/* Interact with launched tasks as desired */
_do_task_work(fd_array, tasks);
for (i = 0; i < tasks; i++) {
close(fd_array[i]);
}
slurm_step_launch_wait_finish(ctx);
/* Terminate the job killing all tasks */
done: slurm_kill_job(job_resp->job_id, SIGKILL, 0);
/* clean up storage */
slurm_free_resource_allocation_response_msg(job_resp);
if (ctx)
slurm_step_ctx_destroy(ctx);
exit(0);
}
/*
* slurm_allocate_resources_blocking
* allocate resources for a job request. This call will block until
* the allocation is granted, or the specified timeout limit is reached.
* IN req - description of resource allocation request
* IN timeout - amount of time, in seconds, to wait for a response before
* giving up.
* A timeout of zero will wait indefinitely.
* IN pending_callback - If the allocation cannot be granted immediately,
* the controller will put the job in the PENDING state. If
* pending callback is not NULL, it will be called with the job_id
* of the pending job as the sole parameter.
*
* RET allocation structure on success, NULL on error set errno to
* indicate the error (errno will be ETIMEDOUT if the timeout is reached
* with no allocation granted)
* NOTE: free the response using slurm_free_resource_allocation_response_msg()
*/
resource_allocation_response_msg_t *
slurm_allocate_resources_blocking (const job_desc_msg_t *user_req,
time_t timeout,
void(*pending_callback)(uint32_t job_id))
{
int rc;
slurm_msg_t req_msg;
slurm_msg_t resp_msg;
resource_allocation_response_msg_t *resp = NULL;
uint32_t job_id;
job_desc_msg_t *req;
listen_t *listen = NULL;
int errnum = SLURM_SUCCESS;
bool already_done = false;
slurm_msg_t_init(&req_msg);
slurm_msg_t_init(&resp_msg);
/* make a copy of the user's job description struct so that we
* can make changes before contacting the controller */
req = (job_desc_msg_t *)xmalloc(sizeof(job_desc_msg_t));
if (req == NULL)
return NULL;
memcpy(req, user_req, sizeof(job_desc_msg_t));
/*
* set Node and session id for this request
*/
if (req->alloc_sid == NO_VAL)
req->alloc_sid = getsid(0);
if (!req->immediate) {
listen = _create_allocation_response_socket();
if (listen == NULL) {
xfree(req);
return NULL;
}
req->alloc_resp_port = listen->port;
}
req_msg.msg_type = REQUEST_RESOURCE_ALLOCATION;
req_msg.data = req;
rc = slurm_send_recv_controller_msg(&req_msg, &resp_msg,
working_cluster_rec);
if (rc == SLURM_ERROR) {
int errnum = errno;
destroy_forward(&req_msg.forward);
destroy_forward(&resp_msg.forward);
if (!req->immediate)
_destroy_allocation_response_socket(listen);
xfree(req);
errno = errnum;
return NULL;
}
switch (resp_msg.msg_type) {
case RESPONSE_SLURM_RC:
if (_handle_rc_msg(&resp_msg) < 0) {
/* will reach this when the allocation fails */
errnum = errno;
} else {
/* shouldn't get here */
errnum = SLURM_ERROR;
}
break;
case RESPONSE_RESOURCE_ALLOCATION:
/* Yay, the controller has acknowledged our request!
* Test if we have an allocation yet? */
resp = (resource_allocation_response_msg_t *) resp_msg.data;
if (resp->node_cnt > 0) {
/* yes, allocation has been granted */
errno = SLURM_SUCCESS;
} else if (!req->immediate) {
if (resp->error_code != SLURM_SUCCESS)
info("%s", slurm_strerror(resp->error_code));
/* no, we need to wait for a response */
/* print out any user messages before we wait. */
print_multi_line_string(resp->job_submit_user_msg,
-1, LOG_LEVEL_INFO);
job_id = resp->job_id;
slurm_free_resource_allocation_response_msg(resp);
if (pending_callback != NULL)
pending_callback(job_id);
_wait_for_allocation_response(job_id, listen,
RESPONSE_RESOURCE_ALLOCATION,
timeout, (void **) &resp);
/* If NULL, we didn't get the allocation in
the time desired, so just free the job id */
if ((resp == NULL) && (errno != ESLURM_ALREADY_DONE)) {
errnum = errno;
slurm_complete_job(job_id, -1);
}
if ((resp == NULL) && (errno == ESLURM_ALREADY_DONE))
already_done = true;
}
break;
default:
errnum = SLURM_UNEXPECTED_MSG_ERROR;
resp = NULL;
}
destroy_forward(&req_msg.forward);
destroy_forward(&resp_msg.forward);
if (!req->immediate)
_destroy_allocation_response_socket(listen);
xfree(req);
if (!resp && already_done && (errnum == SLURM_SUCCESS))
errnum = ESLURM_ALREADY_DONE;
errno = errnum;
return resp;
}
int main(int argc, char *argv[])
{
log_options_t logopt = LOG_OPTS_STDERR_ONLY;
job_desc_msg_t desc;
resource_allocation_response_msg_t *alloc;
time_t before, after;
allocation_msg_thread_t *msg_thr;
char **env = NULL;
int status = 0;
int retries = 0;
pid_t pid = getpid();
pid_t tpgid = 0;
pid_t rc_pid = 0;
int i, rc = 0;
static char *msg = "Slurm job queue full, sleeping and retrying.";
slurm_allocation_callbacks_t callbacks;
log_init(xbasename(argv[0]), logopt, 0, NULL);
_set_exit_code();
if (spank_init_allocator() < 0) {
error("Failed to initialize plugin stack");
exit(error_exit);
}
/* Be sure to call spank_fini when salloc exits
*/
if (atexit((void (*) (void)) spank_fini) < 0)
error("Failed to register atexit handler for plugins: %m");
if (initialize_and_process_args(argc, argv) < 0) {
error("salloc parameter parsing");
exit(error_exit);
}
/* reinit log with new verbosity (if changed by command line) */
if (opt.verbose || opt.quiet) {
logopt.stderr_level += opt.verbose;
logopt.stderr_level -= opt.quiet;
logopt.prefix_level = 1;
log_alter(logopt, 0, NULL);
}
if (spank_init_post_opt() < 0) {
error("Plugin stack post-option processing failed");
exit(error_exit);
}
_set_spank_env();
_set_submit_dir_env();
if (opt.cwd && chdir(opt.cwd)) {
error("chdir(%s): %m", opt.cwd);
exit(error_exit);
}
if (opt.get_user_env_time >= 0) {
char *user = uid_to_string(opt.uid);
if (strcmp(user, "nobody") == 0) {
error("Invalid user id %u: %m", (uint32_t)opt.uid);
exit(error_exit);
}
env = env_array_user_default(user,
opt.get_user_env_time,
opt.get_user_env_mode);
xfree(user);
if (env == NULL)
exit(error_exit); /* error already logged */
_set_rlimits(env);
}
/*
* Job control for interactive salloc sessions: only if ...
*
* a) input is from a terminal (stdin has valid termios attributes),
* b) controlling terminal exists (non-negative tpgid),
* c) salloc is not run in allocation-only (--no-shell) mode,
* d) salloc runs in its own process group (true in interactive
* shells that support job control),
* e) salloc has been configured at compile-time to support background
* execution and is not currently in the background process group.
*/
if (tcgetattr(STDIN_FILENO, &saved_tty_attributes) < 0) {
/*
* Test existence of controlling terminal (tpgid > 0)
* after first making sure stdin is not redirected.
*/
} else if ((tpgid = tcgetpgrp(STDIN_FILENO)) < 0) {
if (!opt.no_shell) {
error("no controlling terminal: please set --no-shell");
exit(error_exit);
}
} else if ((!opt.no_shell) && (pid == getpgrp())) {
if (tpgid == pid)
is_interactive = true;
#ifdef SALLOC_RUN_FOREGROUND
while (tcgetpgrp(STDIN_FILENO) != pid) {
if (!is_interactive) {
error("Waiting for program to be placed in "
"the foreground");
is_interactive = true;
}
killpg(pid, SIGTTIN);
}
#endif
}
/*
* Reset saved tty attributes at exit, in case a child
* process died before properly resetting terminal.
*/
if (is_interactive)
atexit (_reset_input_mode);
/*
* Request a job allocation
*/
slurm_init_job_desc_msg(&desc);
if (_fill_job_desc_from_opts(&desc) == -1) {
exit(error_exit);
}
if (opt.gid != (gid_t) -1) {
if (setgid(opt.gid) < 0) {
error("setgid: %m");
exit(error_exit);
}
}
callbacks.ping = _ping_handler;
callbacks.timeout = _timeout_handler;
callbacks.job_complete = _job_complete_handler;
callbacks.user_msg = _user_msg_handler;
callbacks.node_fail = _node_fail_handler;
/* create message thread to handle pings and such from slurmctld */
msg_thr = slurm_allocation_msg_thr_create(&desc.other_port,
&callbacks);
/* NOTE: Do not process signals in separate pthread. The signal will
* cause slurm_allocate_resources_blocking() to exit immediately. */
for (i = 0; sig_array[i]; i++)
xsignal(sig_array[i], _signal_while_allocating);
before = time(NULL);
while ((alloc = slurm_allocate_resources_blocking(&desc, opt.immediate,
_pending_callback)) == NULL) {
if ((errno != ESLURM_ERROR_ON_DESC_TO_RECORD_COPY) ||
(retries >= MAX_RETRIES))
break;
if (retries == 0)
error("%s", msg);
else
debug("%s", msg);
sleep (++retries);
}
/* become the user after the allocation has been requested. */
if (opt.uid != (uid_t) -1) {
if (setuid(opt.uid) < 0) {
error("setuid: %m");
exit(error_exit);
}
}
if (alloc == NULL) {
if (allocation_interrupted) {
/* cancelled by signal */
info("Job aborted due to signal");
} else if (errno == EINTR) {
error("Interrupted by signal."
" Allocation request rescinded.");
} else if (opt.immediate &&
((errno == ETIMEDOUT) ||
(errno == ESLURM_NOT_TOP_PRIORITY) ||
(errno == ESLURM_NODES_BUSY))) {
error("Unable to allocate resources: %m");
error_exit = immediate_exit;
} else {
error("Failed to allocate resources: %m");
}
slurm_allocation_msg_thr_destroy(msg_thr);
exit(error_exit);
} else if (!allocation_interrupted) {
/*
* Allocation granted!
*/
info("Granted job allocation %u", alloc->job_id);
pending_job_id = alloc->job_id;
#ifdef HAVE_BG
if (!_wait_bluegene_block_ready(alloc)) {
if(!allocation_interrupted)
error("Something is wrong with the "
"boot of the block.");
goto relinquish;
}
#else
if (!_wait_nodes_ready(alloc)) {
if(!allocation_interrupted)
error("Something is wrong with the "
"boot of the nodes.");
goto relinquish;
}
#endif
}
after = time(NULL);
if (opt.bell == BELL_ALWAYS
|| (opt.bell == BELL_AFTER_DELAY
&& ((after - before) > DEFAULT_BELL_DELAY))) {
_ring_terminal_bell();
}
if (opt.no_shell)
exit(0);
if (allocation_interrupted) {
/* salloc process received a signal after
* slurm_allocate_resources_blocking returned with the
* allocation, but before the new signal handlers were
* registered.
*/
goto relinquish;
}
/*
* Run the user's command.
*/
if (env_array_for_job(&env, alloc, &desc) != SLURM_SUCCESS)
goto relinquish;
/* Add default task count for srun, if not already set */
if (opt.ntasks_set) {
env_array_append_fmt(&env, "SLURM_NTASKS", "%d", opt.ntasks);
/* keep around for old scripts */
env_array_append_fmt(&env, "SLURM_NPROCS", "%d", opt.ntasks);
}
if (opt.cpus_per_task > 1) {
env_array_append_fmt(&env, "SLURM_CPUS_PER_TASK", "%d",
opt.cpus_per_task);
}
if (opt.overcommit) {
env_array_append_fmt(&env, "SLURM_OVERCOMMIT", "%d",
opt.overcommit);
}
if (opt.acctg_freq >= 0) {
env_array_append_fmt(&env, "SLURM_ACCTG_FREQ", "%d",
opt.acctg_freq);
}
if (opt.network)
env_array_append_fmt(&env, "SLURM_NETWORK", "%s", opt.network);
env_array_set_environment(env);
env_array_free(env);
pthread_mutex_lock(&allocation_state_lock);
if (allocation_state == REVOKED) {
error("Allocation was revoked for job %u before command could "
"be run", alloc->job_id);
pthread_mutex_unlock(&allocation_state_lock);
if (slurm_complete_job(alloc->job_id, status) != 0) {
error("Unable to clean up allocation for job %u: %m",
alloc->job_id);
}
return 1;
}
allocation_state = GRANTED;
pthread_mutex_unlock(&allocation_state_lock);
/* Ensure that salloc has initial terminal foreground control. */
if (is_interactive) {
/*
* Ignore remaining job-control signals (other than those in
* sig_array, which at this state act like SIG_IGN).
*/
xsignal(SIGTSTP, SIG_IGN);
xsignal(SIGTTIN, SIG_IGN);
xsignal(SIGTTOU, SIG_IGN);
pid = getpid();
setpgid(pid, pid);
tcsetpgrp(STDIN_FILENO, pid);
}
command_pid = _fork_command(command_argv);
/*
* Wait for command to exit, OR for waitpid to be interrupted by a
* signal. Either way, we are going to release the allocation next.
*/
if (command_pid > 0) {
setpgid(command_pid, command_pid);
if (is_interactive)
tcsetpgrp(STDIN_FILENO, command_pid);
/* NOTE: Do not process signals in separate pthread.
* The signal will cause waitpid() to exit immediately. */
xsignal(SIGHUP, _exit_on_signal);
/* Use WUNTRACED to treat stopped children like terminated ones */
do {
rc_pid = waitpid(command_pid, &status, WUNTRACED);
} while ((rc_pid == -1) && (!exit_flag));
if ((rc_pid == -1) && (errno != EINTR))
error("waitpid for %s failed: %m", command_argv[0]);
}
if (is_interactive)
tcsetpgrp(STDIN_FILENO, pid);
/*
* Relinquish the job allocation (if not already revoked).
*/
relinquish:
pthread_mutex_lock(&allocation_state_lock);
if (allocation_state != REVOKED) {
pthread_mutex_unlock(&allocation_state_lock);
info("Relinquishing job allocation %d", alloc->job_id);
if ((slurm_complete_job(alloc->job_id, status) != 0) &&
(slurm_get_errno() != ESLURM_ALREADY_DONE))
error("Unable to clean up job allocation %d: %m",
alloc->job_id);
pthread_mutex_lock(&allocation_state_lock);
allocation_state = REVOKED;
}
pthread_mutex_unlock(&allocation_state_lock);
slurm_free_resource_allocation_response_msg(alloc);
slurm_allocation_msg_thr_destroy(msg_thr);
/*
* Figure out what return code we should use. If the user's command
* exited normally, return the user's return code.
*/
rc = 1;
if (rc_pid != -1) {
if (WIFEXITED(status)) {
rc = WEXITSTATUS(status);
} else if (WIFSIGNALED(status)) {
verbose("Command \"%s\" was terminated by signal %d",
command_argv[0], WTERMSIG(status));
/* if we get these signals we return a normal
* exit since this was most likely sent from the
* user */
switch(WTERMSIG(status)) {
case SIGHUP:
case SIGINT:
case SIGQUIT:
case SIGKILL:
rc = 0;
break;
default:
break;
}
}
}
return rc;
}
/*
* Allocate nodes from the slurm controller -- retrying the attempt
* if the controller appears to be down, and optionally waiting for
* resources if none are currently available (see opt.immediate)
*
* Returns a pointer to a resource_allocation_response_msg which must
* be freed with slurm_free_resource_allocation_response_msg()
*/
extern resource_allocation_response_msg_t *
allocate_nodes(bool handle_signals, slurm_opt_t *opt_local)
{
srun_opt_t *srun_opt = opt_local->srun_opt;
resource_allocation_response_msg_t *resp = NULL;
job_desc_msg_t *j;
slurm_allocation_callbacks_t callbacks;
int i;
xassert(srun_opt);
if (srun_opt->relative_set && srun_opt->relative)
fatal("--relative option invalid for job allocation request");
if ((j = _job_desc_msg_create_from_opts(&opt)) == NULL)
return NULL;
if (opt_local->clusters &&
(slurmdb_get_first_avail_cluster(j, opt_local->clusters,
&working_cluster_rec)
!= SLURM_SUCCESS)) {
print_db_notok(opt_local->clusters, 0);
return NULL;
}
j->origin_cluster = xstrdup(slurmctld_conf.cluster_name);
/* Do not re-use existing job id when submitting new job
* from within a running job */
if ((j->job_id != NO_VAL) && !opt_local->jobid_set) {
info("WARNING: Creating SLURM job allocation from within "
"another allocation");
info("WARNING: You are attempting to initiate a second job");
if (!opt_local->jobid_set) /* Let slurmctld set jobid */
j->job_id = NO_VAL;
}
callbacks.ping = _ping_handler;
callbacks.timeout = _timeout_handler;
callbacks.job_complete = _job_complete_handler;
callbacks.job_suspend = NULL;
callbacks.user_msg = _user_msg_handler;
callbacks.node_fail = _node_fail_handler;
/* create message thread to handle pings and such from slurmctld */
msg_thr = slurm_allocation_msg_thr_create(&j->other_port, &callbacks);
/* NOTE: Do not process signals in separate pthread. The signal will
* cause slurm_allocate_resources_blocking() to exit immediately. */
if (handle_signals) {
xsignal_unblock(sig_array);
for (i = 0; sig_array[i]; i++)
xsignal(sig_array[i], _signal_while_allocating);
}
while (!resp) {
resp = slurm_allocate_resources_blocking(j,
opt_local->immediate,
_set_pending_job_id);
if (destroy_job) {
/* cancelled by signal */
break;
} else if (!resp && !_retry()) {
break;
}
}
if (resp)
print_multi_line_string(resp->job_submit_user_msg, -1);
if (resp && !destroy_job) {
/*
* Allocation granted!
*/
pending_job_id = resp->job_id;
/*
* These values could be changed while the job was
* pending so overwrite the request with what was
* allocated so we don't have issues when we use them
* in the step creation.
*/
opt_local->pn_min_memory = NO_VAL64;
opt_local->mem_per_cpu = NO_VAL64;
if (resp->pn_min_memory != NO_VAL64) {
if (resp->pn_min_memory & MEM_PER_CPU) {
opt_local->mem_per_cpu = (resp->pn_min_memory &
(~MEM_PER_CPU));
} else {
opt_local->pn_min_memory = resp->pn_min_memory;
}
}
#ifdef HAVE_BG
uint32_t node_cnt = 0;
select_g_select_jobinfo_get(resp->select_jobinfo,
SELECT_JOBDATA_NODE_CNT,
&node_cnt);
if ((node_cnt == 0) || (node_cnt == NO_VAL)) {
opt_local->min_nodes = node_cnt;
opt_local->max_nodes = node_cnt;
} /* else we just use the original request */
if (!_wait_bluegene_block_ready(resp)) {
if (!destroy_job)
error("Something is wrong with the "
"boot of the block.");
goto relinquish;
}
#else
opt_local->min_nodes = resp->node_cnt;
opt_local->max_nodes = resp->node_cnt;
if (resp->working_cluster_rec)
slurm_setup_remote_working_cluster(resp);
if (!_wait_nodes_ready(resp)) {
if (!destroy_job)
error("Something is wrong with the boot of the nodes.");
goto relinquish;
}
#endif
} else if (destroy_job) {
goto relinquish;
}
if (handle_signals)
xsignal_block(sig_array);
job_desc_msg_destroy(j);
return resp;
relinquish:
if (resp) {
if (!destroy_job)
slurm_complete_job(resp->job_id, 1);
slurm_free_resource_allocation_response_msg(resp);
}
exit(error_exit);
return NULL;
}
/* returns 1 if job and nodes are ready for job to begin, 0 otherwise */
static int _wait_nodes_ready(resource_allocation_response_msg_t *alloc)
{
int is_ready = 0, i, rc;
double cur_delay = 0;
double cur_sleep = 0;
int suspend_time, resume_time, max_delay;
bool job_killed = false;
suspend_time = slurm_get_suspend_timeout();
resume_time = slurm_get_resume_timeout();
if ((suspend_time == 0) || (resume_time == 0))
return 1; /* Power save mode disabled */
max_delay = suspend_time + resume_time;
max_delay *= 5; /* Allow for ResumeRate support */
pending_job_id = alloc->job_id;
for (i = 0; cur_delay < max_delay; i++) {
if (i) {
cur_sleep = POLL_SLEEP * i;
if (i == 1) {
verbose("Waiting for nodes to boot (delay looping %d times @ %f secs x index)",
max_delay, POLL_SLEEP);
} else {
debug("Waited %f sec and still waiting: next sleep for %f sec",
cur_delay, cur_sleep);
}
usleep(1000000 * cur_sleep);
cur_delay += cur_sleep;
}
rc = slurm_job_node_ready(alloc->job_id);
if (rc == READY_JOB_FATAL)
break; /* fatal error */
if ((rc == READY_JOB_ERROR) || (rc == EAGAIN))
continue; /* retry */
if ((rc & READY_JOB_STATE) == 0) { /* job killed */
job_killed = true;
break;
}
if (rc & READY_NODE_STATE) { /* job and node ready */
is_ready = 1;
break;
}
if (destroy_job)
break;
}
if (is_ready) {
resource_allocation_response_msg_t *resp;
char *tmp_str;
if (i > 0)
verbose("Nodes %s are ready for job", alloc->node_list);
if (alloc->alias_list && !xstrcmp(alloc->alias_list, "TBD") &&
(slurm_allocation_lookup(pending_job_id, &resp)
== SLURM_SUCCESS)) {
tmp_str = alloc->alias_list;
alloc->alias_list = resp->alias_list;
resp->alias_list = tmp_str;
slurm_free_resource_allocation_response_msg(resp);
}
} else if (!destroy_job) {
if (job_killed) {
error("Job allocation %u has been revoked",
alloc->job_id);
destroy_job = true;
} else
error("Nodes %s are still not ready", alloc->node_list);
} else /* allocation_interrupted and slurmctld not responing */
is_ready = 0;
pending_job_id = 0;
return is_ready;
}
