/* * slurm_terminate_job_step - terminates a job step by sending a * REQUEST_TERMINATE_TASKS rpc to all slurmd of a job step. * IN job_id - the job's id * IN step_id - the job step's id - use SLURM_BATCH_SCRIPT as the step_id * to terminate a job's batch script * RET 0 on success, otherwise return -1 and set errno to indicate the error */ extern int slurm_terminate_job_step (uint32_t job_id, uint32_t step_id) { resource_allocation_response_msg_t *alloc_info = NULL; job_step_info_response_msg_t *step_info = NULL; int rc = 0; int i; int save_errno = 0; if (slurm_allocation_lookup_lite(job_id, &alloc_info)) { return -1; } /* * The controller won't give us info about the batch script job step, * so we need to handle that seperately. */ if (step_id == SLURM_BATCH_SCRIPT) { rc = _terminate_batch_script_step(alloc_info); slurm_free_resource_allocation_response_msg(alloc_info); errno = rc; return rc ? -1 : 0; } /* * Otherwise, look through the list of job step info and find * the one matching step_id. Terminate that step. */ rc = slurm_get_job_steps((time_t)0, job_id, step_id, &step_info, SHOW_ALL); if (rc != 0) { save_errno = errno; goto fail; } for (i = 0; i < step_info->job_step_count; i++) { if ((step_info->job_steps[i].job_id == job_id) && (step_info->job_steps[i].step_id == step_id)) { rc = _terminate_job_step(&step_info->job_steps[i], alloc_info); save_errno = errno; break; } } slurm_free_job_step_info_response_msg(step_info); fail: slurm_free_resource_allocation_response_msg(alloc_info); errno = save_errno; return rc ? -1 : 0; }
/* * Wait until a job is ready to execute or enters some failed state * RET 1: job ready to run * 0: job can't run (cancelled, failure state, timeout, etc.) */ extern int scontrol_job_ready(char *job_id_str) { int rc; uint32_t job_id; job_id = atoi(job_id_str); if (job_id <= 0) { fprintf(stderr, "Invalid job_id %s", job_id_str); return SLURM_ERROR; } if (cluster_flags & CLUSTER_FLAG_BG) { resource_allocation_response_msg_t *alloc; rc = slurm_allocation_lookup_lite(job_id, &alloc); if (rc == SLURM_SUCCESS) { rc = _wait_bluegene_block_ready(alloc); slurm_free_resource_allocation_response_msg(alloc); } else { error("slurm_allocation_lookup_lite: %m"); rc = SLURM_ERROR; } } else rc = _wait_nodes_ready(job_id); return rc; }
/* * slurm_signal_job - send the specified signal to all steps of an existing job * IN job_id - the job's id * IN signal - signal number * RET 0 on success, otherwise return -1 and set errno to indicate the error */ extern int slurm_signal_job (uint32_t job_id, uint16_t signal) { int rc = SLURM_SUCCESS; resource_allocation_response_msg_t *alloc_info = NULL; signal_job_msg_t rpc; if (slurm_allocation_lookup_lite(job_id, &alloc_info)) { rc = slurm_get_errno(); goto fail1; } /* same remote procedure call for each node */ rpc.job_id = job_id; rpc.signal = (uint32_t)signal; rc = _local_send_recv_rc_msgs(alloc_info->node_list, REQUEST_SIGNAL_JOB, &rpc); slurm_free_resource_allocation_response_msg(alloc_info); fail1: if (rc) { slurm_seterrno_ret(rc); } else { return SLURM_SUCCESS; } }
static void _purge_agent_args(agent_arg_t *agent_arg_ptr) { if (agent_arg_ptr == NULL) return; hostlist_destroy(agent_arg_ptr->hostlist); xfree(agent_arg_ptr->addr); if (agent_arg_ptr->msg_args) { if (agent_arg_ptr->msg_type == REQUEST_BATCH_JOB_LAUNCH) _slurmctld_free_batch_job_launch_msg( agent_arg_ptr->msg_args); else if (agent_arg_ptr->msg_type == RESPONSE_RESOURCE_ALLOCATION) slurm_free_resource_allocation_response_msg( agent_arg_ptr->msg_args); else if ((agent_arg_ptr->msg_type == REQUEST_ABORT_JOB) || (agent_arg_ptr->msg_type == REQUEST_TERMINATE_JOB) || (agent_arg_ptr->msg_type == REQUEST_KILL_TIMELIMIT)) slurm_free_kill_job_msg(agent_arg_ptr->msg_args); else if (agent_arg_ptr->msg_type == SRUN_USER_MSG) slurm_free_srun_user_msg(agent_arg_ptr->msg_args); else if (agent_arg_ptr->msg_type == SRUN_EXEC) slurm_free_srun_exec_msg(agent_arg_ptr->msg_args); else if (agent_arg_ptr->msg_type == SRUN_NODE_FAIL) slurm_free_srun_node_fail_msg(agent_arg_ptr->msg_args); else if (agent_arg_ptr->msg_type == SRUN_STEP_MISSING) slurm_free_srun_step_missing_msg( agent_arg_ptr->msg_args); else if (agent_arg_ptr->msg_type == REQUEST_JOB_NOTIFY) slurm_free_job_notify_msg(agent_arg_ptr->msg_args); else xfree(agent_arg_ptr->msg_args); } xfree(agent_arg_ptr); }
static void _free_srun_alloc(void *x) { resource_allocation_response_msg_t *alloc_msg; alloc_msg = (resource_allocation_response_msg_t *) x; /* NULL working_cluster_rec because it's pointing to global memory */ alloc_msg->working_cluster_rec = NULL; slurm_free_resource_allocation_response_msg(alloc_msg); }
static void _update_job_size(uint32_t job_id) { resource_allocation_response_msg_t *alloc_info; char *fname_csh = NULL, *fname_sh = NULL; FILE *resize_csh = NULL, *resize_sh = NULL; if (!getenv("SLURM_JOBID")) return; /*No job environment here to update */ if (slurm_allocation_lookup_lite(job_id, &alloc_info) != SLURM_SUCCESS) { slurm_perror("slurm_allocation_lookup_lite"); return; } xstrfmtcat(fname_csh, "slurm_job_%u_resize.csh", job_id); xstrfmtcat(fname_sh, "slurm_job_%u_resize.sh", job_id); (void) unlink(fname_csh); (void) unlink(fname_sh); if (!(resize_csh = fopen(fname_csh, "w"))) { fprintf(stderr, "Could not create file %s: %s\n", fname_csh, strerror(errno)); goto fini; } if (!(resize_sh = fopen(fname_sh, "w"))) { fprintf(stderr, "Could not create file %s: %s\n", fname_sh, strerror(errno)); goto fini; } chmod(fname_csh, 0700); /* Make file executable */ chmod(fname_sh, 0700); if (getenv("SLURM_NODELIST")) { fprintf(resize_sh, "export SLURM_NODELIST=\"%s\"\n", alloc_info->node_list); fprintf(resize_csh, "setenv SLURM_NODELIST \"%s\"\n", alloc_info->node_list); } if (getenv("SLURM_JOB_NODELIST")) { fprintf(resize_sh, "export SLURM_JOB_NODELIST=\"%s\"\n", alloc_info->node_list); fprintf(resize_csh, "setenv SLURM_JOB_NODELIST \"%s\"\n", alloc_info->node_list); } if (getenv("SLURM_NNODES")) { fprintf(resize_sh, "export SLURM_NNODES=%u\n", alloc_info->node_cnt); fprintf(resize_csh, "setenv SLURM_NNODES %u\n", alloc_info->node_cnt); } if (getenv("SLURM_JOB_NUM_NODES")) { fprintf(resize_sh, "export SLURM_JOB_NUM_NODES=%u\n", alloc_info->node_cnt); fprintf(resize_csh, "setenv SLURM_JOB_NUM_NODES %u\n", alloc_info->node_cnt); } if (getenv("SLURM_JOB_CPUS_PER_NODE")) { char *tmp; tmp = uint32_compressed_to_str(alloc_info->num_cpu_groups, alloc_info->cpus_per_node, alloc_info->cpu_count_reps); fprintf(resize_sh, "export SLURM_JOB_CPUS_PER_NODE=\"%s\"\n", tmp); fprintf(resize_csh, "setenv SLURM_JOB_CPUS_PER_NODE \"%s\"\n", tmp); xfree(tmp); } if (getenv("SLURM_TASKS_PER_NODE")) { /* We don't have sufficient information to recreate this */ fprintf(resize_sh, "unset SLURM_TASKS_PER_NODE\n"); fprintf(resize_csh, "unsetenv SLURM_TASKS_PER_NODE\n"); } printf("To reset SLURM environment variables, execute\n"); printf(" For bash or sh shells: . ./%s\n", fname_sh); printf(" For csh shells: source ./%s\n", fname_csh); fini: slurm_free_resource_allocation_response_msg(alloc_info); xfree(fname_csh); xfree(fname_sh); if (resize_csh) fclose(resize_csh); if (resize_sh) fclose(resize_sh); }
/* * slurm_allocate_resources_blocking * allocate resources for a job request. This call will block until * the allocation is granted, or the specified timeout limit is reached. * IN req - description of resource allocation request * IN timeout - amount of time, in seconds, to wait for a response before * giving up. * A timeout of zero will wait indefinitely. * IN pending_callback - If the allocation cannot be granted immediately, * the controller will put the job in the PENDING state. If * pending callback is not NULL, it will be called with the job_id * of the pending job as the sole parameter. * * RET allocation structure on success, NULL on error set errno to * indicate the error (errno will be ETIMEDOUT if the timeout is reached * with no allocation granted) * NOTE: free the response using slurm_free_resource_allocation_response_msg() */ resource_allocation_response_msg_t * slurm_allocate_resources_blocking (const job_desc_msg_t *user_req, time_t timeout, void(*pending_callback)(uint32_t job_id)) { int rc; slurm_msg_t req_msg; slurm_msg_t resp_msg; resource_allocation_response_msg_t *resp = NULL; char *hostname = NULL; uint32_t job_id; job_desc_msg_t *req; listen_t *listen = NULL; int errnum = SLURM_SUCCESS; slurm_msg_t_init(&req_msg); slurm_msg_t_init(&resp_msg); /* make a copy of the user's job description struct so that we * can make changes before contacting the controller */ req = (job_desc_msg_t *)xmalloc(sizeof(job_desc_msg_t)); if (req == NULL) return NULL; memcpy(req, user_req, sizeof(job_desc_msg_t)); /* * set Node and session id for this request */ if (req->alloc_sid == NO_VAL) req->alloc_sid = getsid(0); if (user_req->alloc_node != NULL) { req->alloc_node = xstrdup(user_req->alloc_node); } else if ((hostname = xshort_hostname()) != NULL) { req->alloc_node = hostname; } else { error("Could not get local hostname," " forcing immediate allocation mode."); req->immediate = 1; } if (!req->immediate) { listen = _create_allocation_response_socket(hostname); if (listen == NULL) { xfree(req); return NULL; } req->alloc_resp_port = listen->port; } req_msg.msg_type = REQUEST_RESOURCE_ALLOCATION; req_msg.data = req; rc = slurm_send_recv_controller_msg(&req_msg, &resp_msg); if (rc == SLURM_SOCKET_ERROR) { int errnum = errno; destroy_forward(&req_msg.forward); destroy_forward(&resp_msg.forward); if (!req->immediate) _destroy_allocation_response_socket(listen); xfree(req); errno = errnum; return NULL; } switch (resp_msg.msg_type) { case RESPONSE_SLURM_RC: if (_handle_rc_msg(&resp_msg) < 0) { /* will reach this when the allocation fails */ errnum = errno; } else { /* shouldn't get here */ errnum = -1; } break; case RESPONSE_RESOURCE_ALLOCATION: /* Yay, the controller has acknowledged our request! But did we really get an allocation yet? */ resp = (resource_allocation_response_msg_t *) resp_msg.data; if (resp->node_cnt > 0) { /* yes, allocation has been granted */ errno = SLURM_PROTOCOL_SUCCESS; } else if (!req->immediate) { if (resp->error_code != SLURM_SUCCESS) info("%s", slurm_strerror(resp->error_code)); /* no, we need to wait for a response */ job_id = resp->job_id; slurm_free_resource_allocation_response_msg(resp); if (pending_callback != NULL) pending_callback(job_id); resp = _wait_for_allocation_response(job_id, listen, timeout); /* If NULL, we didn't get the allocation in the time desired, so just free the job id */ if ((resp == NULL) && (errno != ESLURM_ALREADY_DONE)) { errnum = errno; slurm_complete_job(job_id, -1); } } break; default: errnum = SLURM_UNEXPECTED_MSG_ERROR; resp = NULL; } destroy_forward(&req_msg.forward); destroy_forward(&resp_msg.forward); if (!req->immediate) _destroy_allocation_response_socket(listen); xfree(req); errno = errnum; return resp; }
resource_allocation_response_msg_t * allocate_nodes(void) { resource_allocation_response_msg_t *resp = NULL; job_desc_msg_t *j = job_desc_msg_create_from_opts(); slurm_allocation_callbacks_t callbacks; int i; if (!j) return NULL; /* Do not re-use existing job id when submitting new job * from within a running job */ if ((j->job_id != NO_VAL) && !opt.jobid_set) { info("WARNING: Creating SLURM job allocation from within " "another allocation"); info("WARNING: You are attempting to initiate a second job"); if (!opt.jobid_set) /* Let slurmctld set jobid */ j->job_id = NO_VAL; } callbacks.ping = _ping_handler; callbacks.timeout = _timeout_handler; callbacks.job_complete = _job_complete_handler; callbacks.user_msg = _user_msg_handler; callbacks.node_fail = _node_fail_handler; /* create message thread to handle pings and such from slurmctld */ msg_thr = slurm_allocation_msg_thr_create(&j->other_port, &callbacks); /* NOTE: Do not process signals in separate pthread. The signal will * cause slurm_allocate_resources_blocking() to exit immediately. */ xsignal_unblock(sig_array); for (i = 0; sig_array[i]; i++) xsignal(sig_array[i], _signal_while_allocating); while (!resp) { resp = slurm_allocate_resources_blocking(j, opt.immediate, _set_pending_job_id); if (destroy_job) { /* cancelled by signal */ break; } else if (!resp && !_retry()) { break; } } if (resp && !destroy_job) { /* * Allocation granted! */ pending_job_id = resp->job_id; #ifdef HAVE_BG if (!_wait_bluegene_block_ready(resp)) { if(!destroy_job) error("Something is wrong with the " "boot of the block."); goto relinquish; } #else if (!_wait_nodes_ready(resp)) { if(!destroy_job) error("Something is wrong with the " "boot of the nodes."); goto relinquish; } #endif } else if (destroy_job) { goto relinquish; } xsignal_block(sig_array); job_desc_msg_destroy(j); return resp; relinquish: slurm_free_resource_allocation_response_msg(resp); if (!destroy_job) slurm_complete_job(resp->job_id, 1); exit(error_exit); return NULL; }
resource_allocation_response_msg_t * allocate_nodes(bool handle_signals) { resource_allocation_response_msg_t *resp = NULL; job_desc_msg_t *j = job_desc_msg_create_from_opts(); slurm_allocation_callbacks_t callbacks; int i; if (!j) return NULL; /* Do not re-use existing job id when submitting new job * from within a running job */ if ((j->job_id != NO_VAL) && !opt.jobid_set) { info("WARNING: Creating SLURM job allocation from within " "another allocation"); info("WARNING: You are attempting to initiate a second job"); if (!opt.jobid_set) /* Let slurmctld set jobid */ j->job_id = NO_VAL; } callbacks.ping = _ping_handler; callbacks.timeout = _timeout_handler; callbacks.job_complete = _job_complete_handler; callbacks.job_suspend = NULL; callbacks.user_msg = _user_msg_handler; callbacks.node_fail = _node_fail_handler; /* create message thread to handle pings and such from slurmctld */ msg_thr = slurm_allocation_msg_thr_create(&j->other_port, &callbacks); /* NOTE: Do not process signals in separate pthread. The signal will * cause slurm_allocate_resources_blocking() to exit immediately. */ if (handle_signals) { xsignal_unblock(sig_array); for (i = 0; sig_array[i]; i++) xsignal(sig_array[i], _signal_while_allocating); } while (!resp) { resp = slurm_allocate_resources_blocking(j, opt.immediate, _set_pending_job_id); if (destroy_job) { /* cancelled by signal */ break; } else if (!resp && !_retry()) { break; } } if (resp && !destroy_job) { /* * Allocation granted! */ pending_job_id = resp->job_id; /* * These values could be changed while the job was * pending so overwrite the request with what was * allocated so we don't have issues when we use them * in the step creation. */ if (opt.pn_min_memory != NO_VAL) opt.pn_min_memory = (resp->pn_min_memory & (~MEM_PER_CPU)); else if (opt.mem_per_cpu != NO_VAL) opt.mem_per_cpu = (resp->pn_min_memory & (~MEM_PER_CPU)); /* * FIXME: timelimit should probably also be updated * here since it could also change. */ #ifdef HAVE_BG uint32_t node_cnt = 0; select_g_select_jobinfo_get(resp->select_jobinfo, SELECT_JOBDATA_NODE_CNT, &node_cnt); if ((node_cnt == 0) || (node_cnt == NO_VAL)) { opt.min_nodes = node_cnt; opt.max_nodes = node_cnt; } /* else we just use the original request */ if (!_wait_bluegene_block_ready(resp)) { if (!destroy_job) error("Something is wrong with the " "boot of the block."); goto relinquish; } #else opt.min_nodes = resp->node_cnt; opt.max_nodes = resp->node_cnt; if (!_wait_nodes_ready(resp)) { if (!destroy_job) error("Something is wrong with the " "boot of the nodes."); goto relinquish; } #endif } else if (destroy_job) { goto relinquish; } if (handle_signals) xsignal_block(sig_array); job_desc_msg_destroy(j); return resp; relinquish: if (resp) { if (!destroy_job) slurm_complete_job(resp->job_id, 1); slurm_free_resource_allocation_response_msg(resp); } exit(error_exit); return NULL; }
/* returns 1 if job and nodes are ready for job to begin, 0 otherwise */ static int _wait_nodes_ready(resource_allocation_response_msg_t *alloc) { int is_ready = 0, i, rc; int cur_delay = 0; int suspend_time, resume_time, max_delay; suspend_time = slurm_get_suspend_timeout(); resume_time = slurm_get_resume_timeout(); if ((suspend_time == 0) || (resume_time == 0)) return 1; /* Power save mode disabled */ max_delay = suspend_time + resume_time; max_delay *= 5; /* Allow for ResumeRate support */ pending_job_id = alloc->job_id; for (i = 0; (cur_delay < max_delay); i++) { if (i) { if (i == 1) verbose("Waiting for nodes to boot"); else debug("still waiting"); sleep(POLL_SLEEP); cur_delay += POLL_SLEEP; } rc = slurm_job_node_ready(alloc->job_id); if (rc == READY_JOB_FATAL) break; /* fatal error */ if ((rc == READY_JOB_ERROR) || (rc == EAGAIN)) continue; /* retry */ if ((rc & READY_JOB_STATE) == 0) /* job killed */ break; if (rc & READY_NODE_STATE) { /* job and node ready */ is_ready = 1; break; } if (destroy_job) break; } if (is_ready) { resource_allocation_response_msg_t *resp; char *tmp_str; if (i > 0) verbose("Nodes %s are ready for job", alloc->node_list); if (alloc->alias_list && !strcmp(alloc->alias_list, "TBD") && (slurm_allocation_lookup_lite(pending_job_id, &resp) == SLURM_SUCCESS)) { tmp_str = alloc->alias_list; alloc->alias_list = resp->alias_list; resp->alias_list = tmp_str; slurm_free_resource_allocation_response_msg(resp); } } else if (!destroy_job) error("Nodes %s are still not ready", alloc->node_list); else /* allocation_interrupted and slurmctld not responing */ is_ready = 0; pending_job_id = 0; return is_ready; }
int main (int argc, char *argv[]) { int i, min_nodes = 1, max_nodes = 1, nodes, tasks = 0, rc = 0; job_desc_msg_t job_req; resource_allocation_response_msg_t *job_resp; slurm_step_ctx_params_t step_params[1]; slurm_step_ctx_t *ctx = NULL; slurm_step_launch_params_t launch[1]; char *task_argv[3]; int *fd_array = NULL; int num_fd; if (argc > 1) { i = atoi(argv[1]); if (i > 0) min_nodes = i; } if (argc > 2) { i = atoi(argv[2]); if (i > 0) max_nodes = i; } if (max_nodes < min_nodes) max_nodes = min_nodes; /* Create a job allocation */ slurm_init_job_desc_msg( &job_req ); job_req.min_nodes = min_nodes; job_req.max_nodes = max_nodes; job_req.user_id = getuid(); job_req.group_id = getgid(); job_req.time_limit = 1; if (slurm_allocate_resources(&job_req, &job_resp)) { slurm_perror ("slurm_allocate_resources"); printf("INFO: min_nodes=%u max_nodes=%u user_id=%u group_id=%u", job_req.min_nodes, job_req.max_nodes, job_req.user_id, job_req.group_id); exit(0); } printf("job_id %u\n", job_resp->job_id); fflush(stdout); /* Wait for allocation request to be satisfied */ if ((job_resp->node_list == NULL) || (strlen(job_resp->node_list) == 0)) { printf("Waiting for resource allocation\n"); fflush(stdout); while ((job_resp->node_list == NULL) || (strlen(job_resp->node_list) == 0)) { sleep(5); if (slurm_allocation_lookup_lite(job_resp->job_id, &job_resp) && (slurm_get_errno() != ESLURM_JOB_PENDING)) { slurm_perror("slurm_confirm_allocation"); exit(0); } } } nodes = job_resp->node_cnt; if (argc > 3) tasks = atoi(argv[3]); if (tasks < 1) tasks = nodes * TASKS_PER_NODE; if (tasks < nodes) { fprintf(stderr, "Invalid task count argument\n"); exit(1); } printf("Starting %d tasks on %d nodes\n", tasks, nodes); fflush(stdout); /* * Create a job step context. */ slurm_step_ctx_params_t_init(step_params); step_params->job_id = job_resp->job_id; step_params->min_nodes = nodes; step_params->task_count = tasks; ctx = slurm_step_ctx_create(step_params); if ((ctx == NULL) && (slurm_get_errno() == ESLURM_PROLOG_RUNNING)) { printf("SlurmctldProlog is still running, " "sleep and try again\n"); sleep(10); ctx = slurm_step_ctx_create(step_params); } if (ctx == NULL) { slurm_perror("slurm_step_ctx_create"); rc = 1; goto done; } /* * Hack to run one task per node, regardless of what we set up * when we created the job step context. */ if (slurm_step_ctx_daemon_per_node_hack(ctx) != SLURM_SUCCESS) { slurm_perror("slurm_step_ctx_daemon_per_node_hack"); rc = 1; goto done; } /* * Launch the tasks using "user managed" IO. * "user managed" IO means a TCP stream for each task, directly * connected to the stdin, stdout, and stderr the task. */ slurm_step_launch_params_t_init(launch); task_argv[0] = "./test7.3.io"; launch->argv = task_argv; launch->argc = 1; launch->user_managed_io = true; /* This is the key to using "user managed" IO */ if (slurm_step_launch(ctx, launch, NULL) != SLURM_SUCCESS) { slurm_perror("slurm_step_launch"); rc = 1; goto done; } if (slurm_step_launch_wait_start(ctx) != SLURM_SUCCESS) { slurm_perror("slurm_step_launch_wait_start"); rc =1; goto done; } slurm_step_ctx_get(ctx, SLURM_STEP_CTX_USER_MANAGED_SOCKETS, &num_fd, &fd_array); /* Interact with launched tasks as desired */ _do_task_work(fd_array, tasks); for (i = 0; i < tasks; i++) { close(fd_array[i]); } slurm_step_launch_wait_finish(ctx); /* Terminate the job killing all tasks */ done: slurm_kill_job(job_resp->job_id, SIGKILL, 0); /* clean up storage */ slurm_free_resource_allocation_response_msg(job_resp); if (ctx) slurm_step_ctx_destroy(ctx); exit(0); }
/* * slurm_allocate_resources_blocking * allocate resources for a job request. This call will block until * the allocation is granted, or the specified timeout limit is reached. * IN req - description of resource allocation request * IN timeout - amount of time, in seconds, to wait for a response before * giving up. * A timeout of zero will wait indefinitely. * IN pending_callback - If the allocation cannot be granted immediately, * the controller will put the job in the PENDING state. If * pending callback is not NULL, it will be called with the job_id * of the pending job as the sole parameter. * * RET allocation structure on success, NULL on error set errno to * indicate the error (errno will be ETIMEDOUT if the timeout is reached * with no allocation granted) * NOTE: free the response using slurm_free_resource_allocation_response_msg() */ resource_allocation_response_msg_t * slurm_allocate_resources_blocking (const job_desc_msg_t *user_req, time_t timeout, void(*pending_callback)(uint32_t job_id)) { int rc; slurm_msg_t req_msg; slurm_msg_t resp_msg; resource_allocation_response_msg_t *resp = NULL; uint32_t job_id; job_desc_msg_t *req; listen_t *listen = NULL; int errnum = SLURM_SUCCESS; bool already_done = false; slurm_msg_t_init(&req_msg); slurm_msg_t_init(&resp_msg); /* make a copy of the user's job description struct so that we * can make changes before contacting the controller */ req = (job_desc_msg_t *)xmalloc(sizeof(job_desc_msg_t)); if (req == NULL) return NULL; memcpy(req, user_req, sizeof(job_desc_msg_t)); /* * set Node and session id for this request */ if (req->alloc_sid == NO_VAL) req->alloc_sid = getsid(0); if (!req->immediate) { listen = _create_allocation_response_socket(); if (listen == NULL) { xfree(req); return NULL; } req->alloc_resp_port = listen->port; } req_msg.msg_type = REQUEST_RESOURCE_ALLOCATION; req_msg.data = req; rc = slurm_send_recv_controller_msg(&req_msg, &resp_msg, working_cluster_rec); if (rc == SLURM_ERROR) { int errnum = errno; destroy_forward(&req_msg.forward); destroy_forward(&resp_msg.forward); if (!req->immediate) _destroy_allocation_response_socket(listen); xfree(req); errno = errnum; return NULL; } switch (resp_msg.msg_type) { case RESPONSE_SLURM_RC: if (_handle_rc_msg(&resp_msg) < 0) { /* will reach this when the allocation fails */ errnum = errno; } else { /* shouldn't get here */ errnum = SLURM_ERROR; } break; case RESPONSE_RESOURCE_ALLOCATION: /* Yay, the controller has acknowledged our request! * Test if we have an allocation yet? */ resp = (resource_allocation_response_msg_t *) resp_msg.data; if (resp->node_cnt > 0) { /* yes, allocation has been granted */ errno = SLURM_SUCCESS; } else if (!req->immediate) { if (resp->error_code != SLURM_SUCCESS) info("%s", slurm_strerror(resp->error_code)); /* no, we need to wait for a response */ /* print out any user messages before we wait. */ print_multi_line_string(resp->job_submit_user_msg, -1, LOG_LEVEL_INFO); job_id = resp->job_id; slurm_free_resource_allocation_response_msg(resp); if (pending_callback != NULL) pending_callback(job_id); _wait_for_allocation_response(job_id, listen, RESPONSE_RESOURCE_ALLOCATION, timeout, (void **) &resp); /* If NULL, we didn't get the allocation in the time desired, so just free the job id */ if ((resp == NULL) && (errno != ESLURM_ALREADY_DONE)) { errnum = errno; slurm_complete_job(job_id, -1); } if ((resp == NULL) && (errno == ESLURM_ALREADY_DONE)) already_done = true; } break; default: errnum = SLURM_UNEXPECTED_MSG_ERROR; resp = NULL; } destroy_forward(&req_msg.forward); destroy_forward(&resp_msg.forward); if (!req->immediate) _destroy_allocation_response_socket(listen); xfree(req); if (!resp && already_done && (errnum == SLURM_SUCCESS)) errnum = ESLURM_ALREADY_DONE; errno = errnum; return resp; }
int main(int argc, char *argv[]) { log_options_t logopt = LOG_OPTS_STDERR_ONLY; job_desc_msg_t desc; resource_allocation_response_msg_t *alloc; time_t before, after; allocation_msg_thread_t *msg_thr; char **env = NULL; int status = 0; int retries = 0; pid_t pid = getpid(); pid_t tpgid = 0; pid_t rc_pid = 0; int i, rc = 0; static char *msg = "Slurm job queue full, sleeping and retrying."; slurm_allocation_callbacks_t callbacks; log_init(xbasename(argv[0]), logopt, 0, NULL); _set_exit_code(); if (spank_init_allocator() < 0) { error("Failed to initialize plugin stack"); exit(error_exit); } /* Be sure to call spank_fini when salloc exits */ if (atexit((void (*) (void)) spank_fini) < 0) error("Failed to register atexit handler for plugins: %m"); if (initialize_and_process_args(argc, argv) < 0) { error("salloc parameter parsing"); exit(error_exit); } /* reinit log with new verbosity (if changed by command line) */ if (opt.verbose || opt.quiet) { logopt.stderr_level += opt.verbose; logopt.stderr_level -= opt.quiet; logopt.prefix_level = 1; log_alter(logopt, 0, NULL); } if (spank_init_post_opt() < 0) { error("Plugin stack post-option processing failed"); exit(error_exit); } _set_spank_env(); _set_submit_dir_env(); if (opt.cwd && chdir(opt.cwd)) { error("chdir(%s): %m", opt.cwd); exit(error_exit); } if (opt.get_user_env_time >= 0) { char *user = uid_to_string(opt.uid); if (strcmp(user, "nobody") == 0) { error("Invalid user id %u: %m", (uint32_t)opt.uid); exit(error_exit); } env = env_array_user_default(user, opt.get_user_env_time, opt.get_user_env_mode); xfree(user); if (env == NULL) exit(error_exit); /* error already logged */ _set_rlimits(env); } /* * Job control for interactive salloc sessions: only if ... * * a) input is from a terminal (stdin has valid termios attributes), * b) controlling terminal exists (non-negative tpgid), * c) salloc is not run in allocation-only (--no-shell) mode, * d) salloc runs in its own process group (true in interactive * shells that support job control), * e) salloc has been configured at compile-time to support background * execution and is not currently in the background process group. */ if (tcgetattr(STDIN_FILENO, &saved_tty_attributes) < 0) { /* * Test existence of controlling terminal (tpgid > 0) * after first making sure stdin is not redirected. */ } else if ((tpgid = tcgetpgrp(STDIN_FILENO)) < 0) { if (!opt.no_shell) { error("no controlling terminal: please set --no-shell"); exit(error_exit); } } else if ((!opt.no_shell) && (pid == getpgrp())) { if (tpgid == pid) is_interactive = true; #ifdef SALLOC_RUN_FOREGROUND while (tcgetpgrp(STDIN_FILENO) != pid) { if (!is_interactive) { error("Waiting for program to be placed in " "the foreground"); is_interactive = true; } killpg(pid, SIGTTIN); } #endif } /* * Reset saved tty attributes at exit, in case a child * process died before properly resetting terminal. */ if (is_interactive) atexit (_reset_input_mode); /* * Request a job allocation */ slurm_init_job_desc_msg(&desc); if (_fill_job_desc_from_opts(&desc) == -1) { exit(error_exit); } if (opt.gid != (gid_t) -1) { if (setgid(opt.gid) < 0) { error("setgid: %m"); exit(error_exit); } } callbacks.ping = _ping_handler; callbacks.timeout = _timeout_handler; callbacks.job_complete = _job_complete_handler; callbacks.user_msg = _user_msg_handler; callbacks.node_fail = _node_fail_handler; /* create message thread to handle pings and such from slurmctld */ msg_thr = slurm_allocation_msg_thr_create(&desc.other_port, &callbacks); /* NOTE: Do not process signals in separate pthread. The signal will * cause slurm_allocate_resources_blocking() to exit immediately. */ for (i = 0; sig_array[i]; i++) xsignal(sig_array[i], _signal_while_allocating); before = time(NULL); while ((alloc = slurm_allocate_resources_blocking(&desc, opt.immediate, _pending_callback)) == NULL) { if ((errno != ESLURM_ERROR_ON_DESC_TO_RECORD_COPY) || (retries >= MAX_RETRIES)) break; if (retries == 0) error("%s", msg); else debug("%s", msg); sleep (++retries); } /* become the user after the allocation has been requested. */ if (opt.uid != (uid_t) -1) { if (setuid(opt.uid) < 0) { error("setuid: %m"); exit(error_exit); } } if (alloc == NULL) { if (allocation_interrupted) { /* cancelled by signal */ info("Job aborted due to signal"); } else if (errno == EINTR) { error("Interrupted by signal." " Allocation request rescinded."); } else if (opt.immediate && ((errno == ETIMEDOUT) || (errno == ESLURM_NOT_TOP_PRIORITY) || (errno == ESLURM_NODES_BUSY))) { error("Unable to allocate resources: %m"); error_exit = immediate_exit; } else { error("Failed to allocate resources: %m"); } slurm_allocation_msg_thr_destroy(msg_thr); exit(error_exit); } else if (!allocation_interrupted) { /* * Allocation granted! */ info("Granted job allocation %u", alloc->job_id); pending_job_id = alloc->job_id; #ifdef HAVE_BG if (!_wait_bluegene_block_ready(alloc)) { if(!allocation_interrupted) error("Something is wrong with the " "boot of the block."); goto relinquish; } #else if (!_wait_nodes_ready(alloc)) { if(!allocation_interrupted) error("Something is wrong with the " "boot of the nodes."); goto relinquish; } #endif } after = time(NULL); if (opt.bell == BELL_ALWAYS || (opt.bell == BELL_AFTER_DELAY && ((after - before) > DEFAULT_BELL_DELAY))) { _ring_terminal_bell(); } if (opt.no_shell) exit(0); if (allocation_interrupted) { /* salloc process received a signal after * slurm_allocate_resources_blocking returned with the * allocation, but before the new signal handlers were * registered. */ goto relinquish; } /* * Run the user's command. */ if (env_array_for_job(&env, alloc, &desc) != SLURM_SUCCESS) goto relinquish; /* Add default task count for srun, if not already set */ if (opt.ntasks_set) { env_array_append_fmt(&env, "SLURM_NTASKS", "%d", opt.ntasks); /* keep around for old scripts */ env_array_append_fmt(&env, "SLURM_NPROCS", "%d", opt.ntasks); } if (opt.cpus_per_task > 1) { env_array_append_fmt(&env, "SLURM_CPUS_PER_TASK", "%d", opt.cpus_per_task); } if (opt.overcommit) { env_array_append_fmt(&env, "SLURM_OVERCOMMIT", "%d", opt.overcommit); } if (opt.acctg_freq >= 0) { env_array_append_fmt(&env, "SLURM_ACCTG_FREQ", "%d", opt.acctg_freq); } if (opt.network) env_array_append_fmt(&env, "SLURM_NETWORK", "%s", opt.network); env_array_set_environment(env); env_array_free(env); pthread_mutex_lock(&allocation_state_lock); if (allocation_state == REVOKED) { error("Allocation was revoked for job %u before command could " "be run", alloc->job_id); pthread_mutex_unlock(&allocation_state_lock); if (slurm_complete_job(alloc->job_id, status) != 0) { error("Unable to clean up allocation for job %u: %m", alloc->job_id); } return 1; } allocation_state = GRANTED; pthread_mutex_unlock(&allocation_state_lock); /* Ensure that salloc has initial terminal foreground control. */ if (is_interactive) { /* * Ignore remaining job-control signals (other than those in * sig_array, which at this state act like SIG_IGN). */ xsignal(SIGTSTP, SIG_IGN); xsignal(SIGTTIN, SIG_IGN); xsignal(SIGTTOU, SIG_IGN); pid = getpid(); setpgid(pid, pid); tcsetpgrp(STDIN_FILENO, pid); } command_pid = _fork_command(command_argv); /* * Wait for command to exit, OR for waitpid to be interrupted by a * signal. Either way, we are going to release the allocation next. */ if (command_pid > 0) { setpgid(command_pid, command_pid); if (is_interactive) tcsetpgrp(STDIN_FILENO, command_pid); /* NOTE: Do not process signals in separate pthread. * The signal will cause waitpid() to exit immediately. */ xsignal(SIGHUP, _exit_on_signal); /* Use WUNTRACED to treat stopped children like terminated ones */ do { rc_pid = waitpid(command_pid, &status, WUNTRACED); } while ((rc_pid == -1) && (!exit_flag)); if ((rc_pid == -1) && (errno != EINTR)) error("waitpid for %s failed: %m", command_argv[0]); } if (is_interactive) tcsetpgrp(STDIN_FILENO, pid); /* * Relinquish the job allocation (if not already revoked). */ relinquish: pthread_mutex_lock(&allocation_state_lock); if (allocation_state != REVOKED) { pthread_mutex_unlock(&allocation_state_lock); info("Relinquishing job allocation %d", alloc->job_id); if ((slurm_complete_job(alloc->job_id, status) != 0) && (slurm_get_errno() != ESLURM_ALREADY_DONE)) error("Unable to clean up job allocation %d: %m", alloc->job_id); pthread_mutex_lock(&allocation_state_lock); allocation_state = REVOKED; } pthread_mutex_unlock(&allocation_state_lock); slurm_free_resource_allocation_response_msg(alloc); slurm_allocation_msg_thr_destroy(msg_thr); /* * Figure out what return code we should use. If the user's command * exited normally, return the user's return code. */ rc = 1; if (rc_pid != -1) { if (WIFEXITED(status)) { rc = WEXITSTATUS(status); } else if (WIFSIGNALED(status)) { verbose("Command \"%s\" was terminated by signal %d", command_argv[0], WTERMSIG(status)); /* if we get these signals we return a normal * exit since this was most likely sent from the * user */ switch(WTERMSIG(status)) { case SIGHUP: case SIGINT: case SIGQUIT: case SIGKILL: rc = 0; break; default: break; } } } return rc; }
/* * Allocate nodes from the slurm controller -- retrying the attempt * if the controller appears to be down, and optionally waiting for * resources if none are currently available (see opt.immediate) * * Returns a pointer to a resource_allocation_response_msg which must * be freed with slurm_free_resource_allocation_response_msg() */ extern resource_allocation_response_msg_t * allocate_nodes(bool handle_signals, slurm_opt_t *opt_local) { srun_opt_t *srun_opt = opt_local->srun_opt; resource_allocation_response_msg_t *resp = NULL; job_desc_msg_t *j; slurm_allocation_callbacks_t callbacks; int i; xassert(srun_opt); if (srun_opt->relative_set && srun_opt->relative) fatal("--relative option invalid for job allocation request"); if ((j = _job_desc_msg_create_from_opts(&opt)) == NULL) return NULL; if (opt_local->clusters && (slurmdb_get_first_avail_cluster(j, opt_local->clusters, &working_cluster_rec) != SLURM_SUCCESS)) { print_db_notok(opt_local->clusters, 0); return NULL; } j->origin_cluster = xstrdup(slurmctld_conf.cluster_name); /* Do not re-use existing job id when submitting new job * from within a running job */ if ((j->job_id != NO_VAL) && !opt_local->jobid_set) { info("WARNING: Creating SLURM job allocation from within " "another allocation"); info("WARNING: You are attempting to initiate a second job"); if (!opt_local->jobid_set) /* Let slurmctld set jobid */ j->job_id = NO_VAL; } callbacks.ping = _ping_handler; callbacks.timeout = _timeout_handler; callbacks.job_complete = _job_complete_handler; callbacks.job_suspend = NULL; callbacks.user_msg = _user_msg_handler; callbacks.node_fail = _node_fail_handler; /* create message thread to handle pings and such from slurmctld */ msg_thr = slurm_allocation_msg_thr_create(&j->other_port, &callbacks); /* NOTE: Do not process signals in separate pthread. The signal will * cause slurm_allocate_resources_blocking() to exit immediately. */ if (handle_signals) { xsignal_unblock(sig_array); for (i = 0; sig_array[i]; i++) xsignal(sig_array[i], _signal_while_allocating); } while (!resp) { resp = slurm_allocate_resources_blocking(j, opt_local->immediate, _set_pending_job_id); if (destroy_job) { /* cancelled by signal */ break; } else if (!resp && !_retry()) { break; } } if (resp) print_multi_line_string(resp->job_submit_user_msg, -1); if (resp && !destroy_job) { /* * Allocation granted! */ pending_job_id = resp->job_id; /* * These values could be changed while the job was * pending so overwrite the request with what was * allocated so we don't have issues when we use them * in the step creation. */ opt_local->pn_min_memory = NO_VAL64; opt_local->mem_per_cpu = NO_VAL64; if (resp->pn_min_memory != NO_VAL64) { if (resp->pn_min_memory & MEM_PER_CPU) { opt_local->mem_per_cpu = (resp->pn_min_memory & (~MEM_PER_CPU)); } else { opt_local->pn_min_memory = resp->pn_min_memory; } } #ifdef HAVE_BG uint32_t node_cnt = 0; select_g_select_jobinfo_get(resp->select_jobinfo, SELECT_JOBDATA_NODE_CNT, &node_cnt); if ((node_cnt == 0) || (node_cnt == NO_VAL)) { opt_local->min_nodes = node_cnt; opt_local->max_nodes = node_cnt; } /* else we just use the original request */ if (!_wait_bluegene_block_ready(resp)) { if (!destroy_job) error("Something is wrong with the " "boot of the block."); goto relinquish; } #else opt_local->min_nodes = resp->node_cnt; opt_local->max_nodes = resp->node_cnt; if (resp->working_cluster_rec) slurm_setup_remote_working_cluster(resp); if (!_wait_nodes_ready(resp)) { if (!destroy_job) error("Something is wrong with the boot of the nodes."); goto relinquish; } #endif } else if (destroy_job) { goto relinquish; } if (handle_signals) xsignal_block(sig_array); job_desc_msg_destroy(j); return resp; relinquish: if (resp) { if (!destroy_job) slurm_complete_job(resp->job_id, 1); slurm_free_resource_allocation_response_msg(resp); } exit(error_exit); return NULL; }
/* returns 1 if job and nodes are ready for job to begin, 0 otherwise */ static int _wait_nodes_ready(resource_allocation_response_msg_t *alloc) { int is_ready = 0, i, rc; double cur_delay = 0; double cur_sleep = 0; int suspend_time, resume_time, max_delay; bool job_killed = false; suspend_time = slurm_get_suspend_timeout(); resume_time = slurm_get_resume_timeout(); if ((suspend_time == 0) || (resume_time == 0)) return 1; /* Power save mode disabled */ max_delay = suspend_time + resume_time; max_delay *= 5; /* Allow for ResumeRate support */ pending_job_id = alloc->job_id; for (i = 0; cur_delay < max_delay; i++) { if (i) { cur_sleep = POLL_SLEEP * i; if (i == 1) { verbose("Waiting for nodes to boot (delay looping %d times @ %f secs x index)", max_delay, POLL_SLEEP); } else { debug("Waited %f sec and still waiting: next sleep for %f sec", cur_delay, cur_sleep); } usleep(1000000 * cur_sleep); cur_delay += cur_sleep; } rc = slurm_job_node_ready(alloc->job_id); if (rc == READY_JOB_FATAL) break; /* fatal error */ if ((rc == READY_JOB_ERROR) || (rc == EAGAIN)) continue; /* retry */ if ((rc & READY_JOB_STATE) == 0) { /* job killed */ job_killed = true; break; } if (rc & READY_NODE_STATE) { /* job and node ready */ is_ready = 1; break; } if (destroy_job) break; } if (is_ready) { resource_allocation_response_msg_t *resp; char *tmp_str; if (i > 0) verbose("Nodes %s are ready for job", alloc->node_list); if (alloc->alias_list && !xstrcmp(alloc->alias_list, "TBD") && (slurm_allocation_lookup(pending_job_id, &resp) == SLURM_SUCCESS)) { tmp_str = alloc->alias_list; alloc->alias_list = resp->alias_list; resp->alias_list = tmp_str; slurm_free_resource_allocation_response_msg(resp); } } else if (!destroy_job) { if (job_killed) { error("Job allocation %u has been revoked", alloc->job_id); destroy_job = true; } else error("Nodes %s are still not ready", alloc->node_list); } else /* allocation_interrupted and slurmctld not responing */ is_ready = 0; pending_job_id = 0; return is_ready; }