static int _setup_particulars(uint32_t cluster_flags, char ***dest, dynamic_plugin_data_t *select_jobinfo) { int rc = SLURM_SUCCESS; if (cluster_flags & CLUSTER_FLAG_BG) { char *bg_part_id = NULL; select_g_select_jobinfo_get(select_jobinfo, SELECT_JOBDATA_BLOCK_ID, &bg_part_id); if (bg_part_id) { /* check to see if this is a HTC block or not. */ if (cluster_flags & CLUSTER_FLAG_BGP) { uint16_t conn_type[HIGHEST_DIMENSIONS]; select_g_select_jobinfo_get( select_jobinfo, SELECT_JOBDATA_CONN_TYPE, &conn_type); if (conn_type[0] > SELECT_SMALL) { /* SUBMIT_POOL over rides HTC_SUBMIT_POOL */ setenvf(dest, "SUBMIT_POOL", "%s", bg_part_id); } } setenvf(dest, "MPIRUN_PARTITION", "%s", bg_part_id); setenvf(dest, "MPIRUN_NOFREE", "%d", 1); setenvf(dest, "MPIRUN_NOALLOCATE", "%d", 1); xfree(bg_part_id); } else rc = SLURM_FAILURE; if (rc == SLURM_FAILURE) { error("Can't set MPIRUN_PARTITION " "environment variable"); } } else if (cluster_flags & CLUSTER_FLAG_CRAYXT) { uint32_t resv_id = 0; select_g_select_jobinfo_get(select_jobinfo, SELECT_JOBDATA_RESV_ID, &resv_id); if (resv_id) { setenvf(dest, "BASIL_RESERVATION_ID", "%u", resv_id); } else { error("Can't set BASIL_RESERVATION_ID " "environment variable"); rc = SLURM_FAILURE; } } else if (cluster_flags & CLUSTER_FLAG_AIX) { setenvf(dest, "LOADLBATCH", "%s", "yes"); } return rc; }
int _print_job_nodes(job_info_t * job, int width, bool right, char* suffix) { if (job == NULL) { /* Print the Header instead */ char *title = "NODELIST"; if (params.cluster_flags & CLUSTER_FLAG_BG) title = "MIDPLANELIST"; _print_str(title, width, right, false); } else { char *nodes = xstrdup(job->nodes); char *ionodes = NULL; if (nodes) { select_g_select_jobinfo_get(job->select_jobinfo, SELECT_JOBDATA_IONODES, &ionodes); } if (ionodes) { xstrfmtcat(nodes, "[%s]", ionodes); xfree(ionodes); _print_str(nodes, width, right, false); } else _print_nodes(nodes, width, right, false); xfree(nodes); } if (suffix) printf("%s", suffix); return SLURM_SUCCESS; }
int _print_job_num_nodes(job_info_t * job, int width, bool right_justify, char* suffix) { uint32_t node_cnt = 0; char tmp_char[8]; if (job == NULL) /* Print the Header instead */ _print_str("NODES", width, right_justify, true); else { if (params.cluster_flags & CLUSTER_FLAG_BG) select_g_select_jobinfo_get(job->select_jobinfo, SELECT_JOBDATA_NODE_CNT, &node_cnt); if ((node_cnt == 0) || (node_cnt == NO_VAL)) node_cnt = _get_node_cnt(job); if (params.cluster_flags & CLUSTER_FLAG_BG) convert_num_unit((float)node_cnt, tmp_char, sizeof(tmp_char), UNIT_NONE); else snprintf(tmp_char, sizeof(tmp_char), "%d", node_cnt); _print_str(tmp_char, width, right_justify, true); } if (suffix) printf("%s", suffix); return SLURM_SUCCESS; }
/* returns 1 if job and nodes are ready for job to begin, 0 otherwise */ static int _wait_bluegene_block_ready(resource_allocation_response_msg_t *alloc) { int is_ready = 0, i, rc; char *block_id = NULL; double cur_delay = 0; double cur_sleep = 0; int max_delay = BG_FREE_PREVIOUS_BLOCK + BG_MIN_BLOCK_BOOT + (BG_INCR_BLOCK_BOOT * alloc->node_cnt); select_g_select_jobinfo_get(alloc->select_jobinfo, SELECT_JOBDATA_BLOCK_ID, &block_id); for (i = 0; cur_delay < max_delay; i++) { cur_sleep = POLL_SLEEP * i; if (i == 1) { debug("Waiting for block %s to become ready for job", block_id); } if (i) { usleep(1000000 * cur_sleep); rc = _blocks_dealloc(); if ((rc == 0) || (rc == -1)) cur_delay += cur_sleep; debug2("still waiting"); } rc = slurm_job_node_ready(alloc->job_id); if (rc == READY_JOB_FATAL) break; /* fatal error */ if ((rc == READY_JOB_ERROR) || (rc == EAGAIN)) continue; /* retry */ if ((rc & READY_JOB_STATE) == 0) /* job killed */ break; if (rc & READY_NODE_STATE) { /* job and node ready */ is_ready = 1; break; } if (destroy_job) break; } if (is_ready) debug("Block %s is ready for job", block_id); else if (!destroy_job) error("Block %s still not ready", block_id); else /* destroy_job set and slurmctld not responing */ is_ready = 0; xfree(block_id); return is_ready; }
static int _wait_bluegene_block_ready(resource_allocation_response_msg_t *alloc) { int is_ready = SLURM_ERROR, i, rc = 0; char *block_id = NULL; int cur_delay = 0; int max_delay = BG_FREE_PREVIOUS_BLOCK + BG_MIN_BLOCK_BOOT + (BG_INCR_BLOCK_BOOT * alloc->node_cnt); select_g_select_jobinfo_get(alloc->select_jobinfo, SELECT_JOBDATA_BLOCK_ID, &block_id); for (i=0; (cur_delay < max_delay); i++) { if (i) { if (i == 1) { info("Waiting for block %s to become ready for " "job", block_id); } else debug("still waiting"); sleep(POLL_SLEEP); rc = _blocks_dealloc(); if ((rc == 0) || (rc == -1)) cur_delay += POLL_SLEEP; } rc = slurm_job_node_ready(alloc->job_id); if (rc == READY_JOB_FATAL) break; /* fatal error */ if ((rc == READY_JOB_ERROR) || (rc == EAGAIN)) continue; /* retry */ if ((rc & READY_JOB_STATE) == 0) /* job killed */ break; if (rc & READY_NODE_STATE) { /* job and node ready */ is_ready = SLURM_SUCCESS; break; } } if (is_ready == SLURM_SUCCESS) info("Block %s is ready for job %u", block_id, alloc->job_id); else if ((rc & READY_JOB_STATE) == 0) info("Job %u no longer running", alloc->job_id); else info("Problem running job %u", alloc->job_id); xfree(block_id); return is_ready; }
int _print_job_reason_list(job_info_t * job, int width, bool right, char* suffix) { if (job == NULL) { /* Print the Header instead */ char *title = "NODELIST(REASON)"; if (params.cluster_flags & CLUSTER_FLAG_BG) title = "MIDPLANELIST(REASON)"; _print_str(title, width, right, false); } else if (!IS_JOB_COMPLETING(job) && (IS_JOB_PENDING(job) || IS_JOB_TIMEOUT(job) || IS_JOB_FAILED(job))) { char id[FORMAT_STRING_SIZE], *reason; if (job->state_desc) reason = job->state_desc; else reason = job_reason_string(job->state_reason); snprintf(id, FORMAT_STRING_SIZE, "(%s)", reason); _print_str(id, width, right, true); } else { char *nodes = xstrdup(job->nodes); char *ionodes = NULL; select_g_select_jobinfo_get(job->select_jobinfo, SELECT_JOBDATA_IONODES, &ionodes); if (ionodes) { xstrfmtcat(nodes, "[%s]", ionodes); xfree(ionodes); _print_str(nodes, width, right, false); } else _print_nodes(nodes, width, right, false); xfree(nodes); } if (suffix) printf("%s", suffix); return SLURM_SUCCESS; }
static int _print_record(struct job_record *job_ptr, time_t time, char *data) { static int rc=SLURM_SUCCESS; char *block_id = NULL; if (!job_ptr->details) { error("job_acct: job=%u doesn't exist", job_ptr->job_id); return SLURM_ERROR; } debug2("_print_record, job=%u, \"%s\"", job_ptr->job_id, data); #ifdef HAVE_BG select_g_select_jobinfo_get(job_ptr->select_jobinfo, SELECT_JOBDATA_BLOCK_ID, &block_id); #endif if (!block_id) block_id = xstrdup("-"); slurm_mutex_lock( &logfile_lock ); if (fprintf(LOGFILE, "%u %s %d %d %u %u %s - %s\n", job_ptr->job_id, job_ptr->partition, (int)job_ptr->details->submit_time, (int)time, job_ptr->user_id, job_ptr->group_id, block_id, data) < 0) rc=SLURM_ERROR; #ifdef HAVE_FDATASYNC fdatasync(LOGFILE_FD); #endif slurm_mutex_unlock( &logfile_lock ); xfree(block_id); return rc; }
resource_allocation_response_msg_t * allocate_nodes(bool handle_signals) { resource_allocation_response_msg_t *resp = NULL; job_desc_msg_t *j = job_desc_msg_create_from_opts(); slurm_allocation_callbacks_t callbacks; int i; if (!j) return NULL; /* Do not re-use existing job id when submitting new job * from within a running job */ if ((j->job_id != NO_VAL) && !opt.jobid_set) { info("WARNING: Creating SLURM job allocation from within " "another allocation"); info("WARNING: You are attempting to initiate a second job"); if (!opt.jobid_set) /* Let slurmctld set jobid */ j->job_id = NO_VAL; } callbacks.ping = _ping_handler; callbacks.timeout = _timeout_handler; callbacks.job_complete = _job_complete_handler; callbacks.job_suspend = NULL; callbacks.user_msg = _user_msg_handler; callbacks.node_fail = _node_fail_handler; /* create message thread to handle pings and such from slurmctld */ msg_thr = slurm_allocation_msg_thr_create(&j->other_port, &callbacks); /* NOTE: Do not process signals in separate pthread. The signal will * cause slurm_allocate_resources_blocking() to exit immediately. */ if (handle_signals) { xsignal_unblock(sig_array); for (i = 0; sig_array[i]; i++) xsignal(sig_array[i], _signal_while_allocating); } while (!resp) { resp = slurm_allocate_resources_blocking(j, opt.immediate, _set_pending_job_id); if (destroy_job) { /* cancelled by signal */ break; } else if (!resp && !_retry()) { break; } } if (resp && !destroy_job) { /* * Allocation granted! */ pending_job_id = resp->job_id; /* * These values could be changed while the job was * pending so overwrite the request with what was * allocated so we don't have issues when we use them * in the step creation. */ if (opt.pn_min_memory != NO_VAL) opt.pn_min_memory = (resp->pn_min_memory & (~MEM_PER_CPU)); else if (opt.mem_per_cpu != NO_VAL) opt.mem_per_cpu = (resp->pn_min_memory & (~MEM_PER_CPU)); /* * FIXME: timelimit should probably also be updated * here since it could also change. */ #ifdef HAVE_BG uint32_t node_cnt = 0; select_g_select_jobinfo_get(resp->select_jobinfo, SELECT_JOBDATA_NODE_CNT, &node_cnt); if ((node_cnt == 0) || (node_cnt == NO_VAL)) { opt.min_nodes = node_cnt; opt.max_nodes = node_cnt; } /* else we just use the original request */ if (!_wait_bluegene_block_ready(resp)) { if (!destroy_job) error("Something is wrong with the " "boot of the block."); goto relinquish; } #else opt.min_nodes = resp->node_cnt; opt.max_nodes = resp->node_cnt; if (!_wait_nodes_ready(resp)) { if (!destroy_job) error("Something is wrong with the " "boot of the nodes."); goto relinquish; } #endif } else if (destroy_job) { goto relinquish; } if (handle_signals) xsignal_block(sig_array); job_desc_msg_destroy(j); return resp; relinquish: if (resp) { if (!destroy_job) slurm_complete_job(resp->job_id, 1); slurm_free_resource_allocation_response_msg(resp); } exit(error_exit); return NULL; }
extern int slurm_get_select_jobinfo(dynamic_plugin_data_t *jobinfo, enum select_jobdata_type data_type, void *data) { return select_g_select_jobinfo_get(jobinfo, data_type, data); }
static char * _will_run_test2(uint32_t jobid, time_t start_time, char *node_list, uint32_t *preemptee, int preemptee_cnt, int *err_code, char **err_msg) { struct job_record *job_ptr = NULL, *pre_ptr; struct part_record *part_ptr; bitstr_t *avail_bitmap = NULL, *resv_bitmap = NULL; bitstr_t *exc_core_bitmap = NULL; time_t start_res; uint32_t min_nodes, max_nodes, req_nodes; List preemptee_candidates = NULL, preempted_jobs = NULL; time_t orig_start_time; char *reply_msg = NULL; int i, rc; bool resv_overlap = false; xassert(node_list); debug2("wiki2: will_run2 job_id=%u start_time=%u node_list=%s", jobid, (uint32_t)start_time, node_list); job_ptr = find_job_record(jobid); if (job_ptr == NULL) { *err_code = -700; *err_msg = "No such job"; error("wiki: Failed to find job %u", jobid); return NULL; } if ((job_ptr->details == NULL) || (!IS_JOB_PENDING(job_ptr))) { *err_code = -700; *err_msg = "WillRun not applicable to non-pending job"; error("wiki: WillRun on non-pending job %u", jobid); return NULL; } part_ptr = job_ptr->part_ptr; if (part_ptr == NULL) { *err_code = -700; *err_msg = "Job lacks a partition"; error("wiki: Job %u lacks a partition", jobid); return NULL; } if (node_name2bitmap(node_list, false, &avail_bitmap) != 0) { *err_code = -700; *err_msg = "Invalid available nodes value"; error("wiki: Attempt to set invalid available node " "list for job %u, %s", jobid, node_list); return NULL; } /* Enforce reservation: access control, time and nodes */ start_res = start_time; rc = job_test_resv(job_ptr, &start_res, true, &resv_bitmap, &exc_core_bitmap, &resv_overlap); if (rc != SLURM_SUCCESS) { *err_code = -730; *err_msg = "Job denied access to reservation"; error("wiki: reservation access denied for job %u", jobid); FREE_NULL_BITMAP(avail_bitmap); FREE_NULL_BITMAP(exc_core_bitmap); return NULL; } bit_and(avail_bitmap, resv_bitmap); FREE_NULL_BITMAP(resv_bitmap); /* Only consider nodes that are not DOWN or DRAINED */ bit_and(avail_bitmap, avail_node_bitmap); /* Consider only nodes in this job's partition */ if (part_ptr->node_bitmap) bit_and(avail_bitmap, part_ptr->node_bitmap); else { *err_code = -730; *err_msg = "Job's partition has no nodes"; error("wiki: no nodes in partition %s for job %u", part_ptr->name, jobid); FREE_NULL_BITMAP(avail_bitmap); FREE_NULL_BITMAP(exc_core_bitmap); return NULL; } if (job_req_node_filter(job_ptr, avail_bitmap) != SLURM_SUCCESS) { /* Job probably has invalid feature list */ *err_code = -730; *err_msg = "Job's required features not available " "on selected nodes"; error("wiki: job %u not runnable on hosts=%s", jobid, node_list); FREE_NULL_BITMAP(avail_bitmap); FREE_NULL_BITMAP(exc_core_bitmap); return NULL; } if (job_ptr->details->exc_node_bitmap) { bit_not(job_ptr->details->exc_node_bitmap); bit_and(avail_bitmap, job_ptr->details->exc_node_bitmap); bit_not(job_ptr->details->exc_node_bitmap); } if ((job_ptr->details->req_node_bitmap) && (!bit_super_set(job_ptr->details->req_node_bitmap, avail_bitmap))) { *err_code = -730; *err_msg = "Job's required nodes not available"; error("wiki: job %u not runnable on hosts=%s", jobid, node_list); FREE_NULL_BITMAP(avail_bitmap); FREE_NULL_BITMAP(exc_core_bitmap); return NULL; } min_nodes = MAX(job_ptr->details->min_nodes, part_ptr->min_nodes); if (job_ptr->details->max_nodes == 0) max_nodes = part_ptr->max_nodes; else max_nodes = MIN(job_ptr->details->max_nodes, part_ptr->max_nodes); max_nodes = MIN(max_nodes, 500000); /* prevent overflows */ if (job_ptr->details->max_nodes) req_nodes = max_nodes; else req_nodes = min_nodes; if (min_nodes > max_nodes) { /* job's min_nodes exceeds partitions max_nodes */ *err_code = -730; *err_msg = "Job's min_nodes > max_nodes"; error("wiki: job %u not runnable on hosts=%s", jobid, node_list); FREE_NULL_BITMAP(avail_bitmap); FREE_NULL_BITMAP(exc_core_bitmap); return NULL; } if (preemptee_cnt) { preemptee_candidates = list_create(NULL); for (i=0; i<preemptee_cnt; i++) { if ((pre_ptr = find_job_record(preemptee[i]))) list_append(preemptee_candidates, pre_ptr); } } orig_start_time = job_ptr->start_time; rc = select_g_job_test(job_ptr, avail_bitmap, min_nodes, max_nodes, req_nodes, SELECT_MODE_WILL_RUN, preemptee_candidates, &preempted_jobs, exc_core_bitmap); FREE_NULL_LIST(preemptee_candidates); if (rc == SLURM_SUCCESS) { char *hostlist, *sep, tmp_str[128]; uint32_t pre_cnt = 0, proc_cnt = 0; #ifdef HAVE_BG select_g_select_jobinfo_get(job_ptr->select_jobinfo, SELECT_JOBDATA_NODE_CNT, &proc_cnt); #else proc_cnt = job_ptr->total_cpus; #endif snprintf(tmp_str, sizeof(tmp_str), "STARTINFO=%u TASKS=%u STARTTIME=%u NODES=", job_ptr->job_id, proc_cnt, (uint32_t) job_ptr->start_time); xstrcat(reply_msg, tmp_str); hostlist = bitmap2node_name(avail_bitmap); xstrcat(reply_msg, hostlist); xfree(hostlist); if (preempted_jobs) { while ((pre_ptr = list_pop(preempted_jobs))) { if (pre_cnt++) sep = ","; else sep = " PREEMPT="; snprintf(tmp_str, sizeof(tmp_str), "%s%u", sep, pre_ptr->job_id); xstrcat(reply_msg, tmp_str); } FREE_NULL_LIST(preempted_jobs); } } else { xstrcat(reply_msg, "Jobs not runable on selected nodes"); error("wiki: jobs not runnable on nodes"); } /* Restore pending job's expected start time */ job_ptr->start_time = orig_start_time; FREE_NULL_BITMAP(avail_bitmap); FREE_NULL_BITMAP(exc_core_bitmap); return reply_msg; }
extern stepd_step_rec_t * batch_stepd_step_rec_create(batch_job_launch_msg_t *msg) { stepd_step_rec_t *job; srun_info_t *srun = NULL; char *in_name; xassert(msg != NULL); debug3("entering batch_stepd_step_rec_create"); if (!_valid_uid_gid((uid_t)msg->uid, &(msg->gid), &(msg->user_name))) return NULL; if (_check_acct_freq_task(msg->job_mem, msg->acctg_freq)) return NULL; job = xmalloc(sizeof(stepd_step_rec_t)); job->state = SLURMSTEPD_STEP_STARTING; if (msg->cpus_per_node) job->cpus = msg->cpus_per_node[0]; job->node_tasks = 1; job->ntasks = msg->ntasks; job->jobid = msg->job_id; job->stepid = msg->step_id; job->array_job_id = msg->array_job_id; job->array_task_id = msg->array_task_id; job->batch = true; /* This needs to happen before acct_gather_profile_startpoll and only really looks at the profile in the job. */ acct_gather_profile_g_node_step_start(job); /* needed for the jobacct_gather plugin to start */ acct_gather_profile_startpoll(msg->acctg_freq, conf->job_acct_gather_freq); job->multi_prog = 0; job->open_mode = msg->open_mode; job->overcommit = (bool) msg->overcommit; job->node_name = xstrdup(conf->node_name); job->uid = (uid_t) msg->uid; job->user_name = xstrdup(msg->user_name); job->gid = (gid_t) msg->gid; job->cwd = xstrdup(msg->work_dir); job->ckpt_dir = xstrdup(msg->ckpt_dir); job->restart_dir = xstrdup(msg->restart_dir); job->env = _array_copy(msg->envc, msg->environment); job->eio = eio_handle_create(); job->sruns = list_create((ListDelF) _srun_info_destructor); job->envtp = xmalloc(sizeof(env_t)); job->envtp->jobid = -1; job->envtp->stepid = -1; job->envtp->procid = -1; job->envtp->localid = -1; job->envtp->nodeid = -1; job->envtp->distribution = 0; job->cpu_bind_type = msg->cpu_bind_type; job->cpu_bind = xstrdup(msg->cpu_bind); job->envtp->mem_bind_type = 0; job->envtp->mem_bind = NULL; job->envtp->ckpt_dir = NULL; job->envtp->restart_cnt = msg->restart_cnt; if (msg->cpus_per_node) job->cpus = msg->cpus_per_node[0]; format_core_allocs(msg->cred, conf->node_name, conf->cpus, &job->job_alloc_cores, &job->step_alloc_cores, &job->job_mem, &job->step_mem); if (job->step_mem) jobacct_gather_set_mem_limit(job->jobid, NO_VAL, job->step_mem); else if (job->job_mem) jobacct_gather_set_mem_limit(job->jobid, NO_VAL, job->job_mem); get_cred_gres(msg->cred, conf->node_name, &job->job_gres_list, &job->step_gres_list); srun = srun_info_create(NULL, NULL, NULL); list_append(job->sruns, (void *) srun); if (msg->argc) { job->argc = msg->argc; job->argv = _array_copy(job->argc, msg->argv); } else { job->argc = 1; /* job script has not yet been written out to disk -- * argv will be filled in later by _make_batch_script() */ job->argv = (char **) xmalloc(2 * sizeof(char *)); } job->task = xmalloc(sizeof(stepd_step_task_info_t *)); if (msg->std_err == NULL) msg->std_err = xstrdup(msg->std_out); if (msg->std_in == NULL) in_name = xstrdup("/dev/null"); else in_name = fname_create(job, msg->std_in, 0); job->task[0] = task_info_create(0, 0, in_name, _batchfilename(job, msg->std_out), _batchfilename(job, msg->std_err)); job->task[0]->argc = job->argc; job->task[0]->argv = job->argv; #ifdef HAVE_ALPS_CRAY select_g_select_jobinfo_get(msg->select_jobinfo, SELECT_JOBDATA_RESV_ID, &job->resv_id); #endif return job; }
/* * slurm_sprint_job_step_info - output information about a specific Slurm * job step based upon message as loaded using slurm_get_job_steps * IN job_ptr - an individual job step information record pointer * IN one_liner - print as a single line if true * RET out - char * containing formatted output (must be freed after call) * NULL is returned on failure. */ char * slurm_sprint_job_step_info ( job_step_info_t * job_step_ptr, int one_liner ) { char tmp_node_cnt[40]; char time_str[32]; char limit_str[32]; char tmp_line[128]; char *out = NULL; uint32_t cluster_flags = slurmdb_setup_cluster_flags(); /****** Line 1 ******/ slurm_make_time_str ((time_t *)&job_step_ptr->start_time, time_str, sizeof(time_str)); if (job_step_ptr->time_limit == INFINITE) sprintf(limit_str, "UNLIMITED"); else secs2time_str ((time_t)job_step_ptr->time_limit * 60, limit_str, sizeof(limit_str)); if (job_step_ptr->array_job_id) { if (job_step_ptr->step_id == INFINITE) { /* Pending */ snprintf(tmp_line, sizeof(tmp_line), "StepId=%u_%u.TBD ", job_step_ptr->array_job_id, job_step_ptr->array_task_id); } else { snprintf(tmp_line, sizeof(tmp_line), "StepId=%u_%u.%u ", job_step_ptr->array_job_id, job_step_ptr->array_task_id, job_step_ptr->step_id); } out = xstrdup(tmp_line); } else { if (job_step_ptr->step_id == INFINITE) { /* Pending */ snprintf(tmp_line, sizeof(tmp_line), "StepId=%u.TBD ", job_step_ptr->job_id); } else { snprintf(tmp_line, sizeof(tmp_line), "StepId=%u.%u ", job_step_ptr->job_id, job_step_ptr->step_id); } out = xstrdup(tmp_line); } snprintf(tmp_line, sizeof(tmp_line), "UserId=%u StartTime=%s TimeLimit=%s", job_step_ptr->user_id, time_str, limit_str); xstrcat(out, tmp_line); if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); /****** Line 2 ******/ snprintf(tmp_line, sizeof(tmp_line), "State=%s ", job_state_string(job_step_ptr->state)); xstrcat(out, tmp_line); if (cluster_flags & CLUSTER_FLAG_BG) { char *io_nodes = NULL; select_g_select_jobinfo_get(job_step_ptr->select_jobinfo, SELECT_JOBDATA_IONODES, &io_nodes); if (io_nodes) { snprintf(tmp_line, sizeof(tmp_line), "Partition=%s MidplaneList=%s[%s] Gres=%s", job_step_ptr->partition, job_step_ptr->nodes, io_nodes, job_step_ptr->gres); xfree(io_nodes); } else snprintf(tmp_line, sizeof(tmp_line), "Partition=%s MidplaneList=%s Gres=%s", job_step_ptr->partition, job_step_ptr->nodes, job_step_ptr->gres); } else { snprintf(tmp_line, sizeof(tmp_line), "Partition=%s NodeList=%s Gres=%s", job_step_ptr->partition, job_step_ptr->nodes, job_step_ptr->gres); } xstrcat(out, tmp_line); if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); /****** Line 3 ******/ if (cluster_flags & CLUSTER_FLAG_BGQ) { uint32_t nodes = 0; select_g_select_jobinfo_get(job_step_ptr->select_jobinfo, SELECT_JOBDATA_NODE_CNT, &nodes); convert_num_unit((float)nodes, tmp_node_cnt, sizeof(tmp_node_cnt), UNIT_NONE, CONVERT_NUM_UNIT_EXACT); } else { convert_num_unit((float)_nodes_in_list(job_step_ptr->nodes), tmp_node_cnt, sizeof(tmp_node_cnt), UNIT_NONE, CONVERT_NUM_UNIT_EXACT); } snprintf(tmp_line, sizeof(tmp_line), "Nodes=%s CPUs=%u Tasks=%u Name=%s Network=%s", tmp_node_cnt, job_step_ptr->num_cpus, job_step_ptr->num_tasks, job_step_ptr->name, job_step_ptr->network); xstrcat(out, tmp_line); if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); /****** Line 4 ******/ snprintf(tmp_line, sizeof(tmp_line), "TRES=%s", job_step_ptr->tres_alloc_str); xstrcat(out, tmp_line); if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); /****** Line 5 ******/ snprintf(tmp_line, sizeof(tmp_line), "ResvPorts=%s Checkpoint=%u CheckpointDir=%s", job_step_ptr->resv_ports, job_step_ptr->ckpt_interval, job_step_ptr->ckpt_dir); xstrcat(out, tmp_line); if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); /****** Line 6 ******/ if (cpu_freq_debug(NULL, NULL, tmp_line, sizeof(tmp_line), job_step_ptr->cpu_freq_gov, job_step_ptr->cpu_freq_min, job_step_ptr->cpu_freq_max, NO_VAL) != 0) { xstrcat(out, tmp_line); } else { xstrcat(out, "CPUFreqReq=Default"); } xstrfmtcat(out, " Dist=%s", slurm_step_layout_type_name(job_step_ptr->task_dist)); xstrcat(out, "\n\n"); return out; }
extern int as_mysql_job_start(mysql_conn_t *mysql_conn, struct job_record *job_ptr) { int rc=SLURM_SUCCESS; char *nodes = NULL, *jname = NULL, *node_inx = NULL; int track_steps = 0; char *block_id = NULL, *partition = NULL, *gres_req = NULL, *gres_alloc = NULL; char *query = NULL; int reinit = 0; time_t begin_time, check_time, start_time, submit_time; uint32_t wckeyid = 0; int job_state, node_cnt = 0; uint32_t job_db_inx = job_ptr->db_index; if ((!job_ptr->details || !job_ptr->details->submit_time) && !job_ptr->resize_time) { error("as_mysql_job_start: " "Not inputing this job, it has no submit time."); return SLURM_ERROR; } if (check_connection(mysql_conn) != SLURM_SUCCESS) return ESLURM_DB_CONNECTION; debug2("as_mysql_slurmdb_job_start() called"); job_state = job_ptr->job_state; if (job_ptr->resize_time) { begin_time = job_ptr->resize_time; submit_time = job_ptr->resize_time; start_time = job_ptr->resize_time; } else { begin_time = job_ptr->details->begin_time; submit_time = job_ptr->details->submit_time; start_time = job_ptr->start_time; } /* Since we need a new db_inx make sure the old db_inx * removed. This is most likely the only time we are going to * be notified of the change also so make the state without * the resize. */ if (IS_JOB_RESIZING(job_ptr)) { /* If we have a db_index lets end the previous record. */ if (!job_ptr->db_index) { error("We don't have a db_index for job %u, " "this should only happen when resizing " "jobs and the database interface was down.", job_ptr->job_id); job_ptr->db_index = _get_db_index(mysql_conn, job_ptr->details-> submit_time, job_ptr->job_id, job_ptr->assoc_id); } if (job_ptr->db_index) as_mysql_job_complete(mysql_conn, job_ptr); job_state &= (~JOB_RESIZING); job_ptr->db_index = 0; } job_state &= JOB_STATE_BASE; /* See what we are hearing about here if no start time. If * this job latest time is before the last roll up we will * need to reset it to look at this job. */ if (start_time) check_time = start_time; else if (begin_time) check_time = begin_time; else check_time = submit_time; slurm_mutex_lock(&rollup_lock); if (check_time < global_last_rollup) { MYSQL_RES *result = NULL; MYSQL_ROW row; /* check to see if we are hearing about this time for the * first time. */ query = xstrdup_printf("select job_db_inx " "from \"%s_%s\" where id_job=%u and " "time_submit=%ld and time_eligible=%ld " "and time_start=%ld;", mysql_conn->cluster_name, job_table, job_ptr->job_id, submit_time, begin_time, start_time); debug3("%d(%s:%d) query\n%s", mysql_conn->conn, THIS_FILE, __LINE__, query); if (!(result = mysql_db_query_ret(mysql_conn, query, 0))) { xfree(query); slurm_mutex_unlock(&rollup_lock); return SLURM_ERROR; } xfree(query); if ((row = mysql_fetch_row(result))) { mysql_free_result(result); debug4("revieved an update for a " "job (%u) already known about", job_ptr->job_id); slurm_mutex_unlock(&rollup_lock); goto no_rollup_change; } mysql_free_result(result); if (job_ptr->start_time) debug("Need to reroll usage from %sJob %u " "from %s started then and we are just " "now hearing about it.", slurm_ctime(&check_time), job_ptr->job_id, mysql_conn->cluster_name); else if (begin_time) debug("Need to reroll usage from %sJob %u " "from %s became eligible then and we are just " "now hearing about it.", slurm_ctime(&check_time), job_ptr->job_id, mysql_conn->cluster_name); else debug("Need to reroll usage from %sJob %u " "from %s was submitted then and we are just " "now hearing about it.", slurm_ctime(&check_time), job_ptr->job_id, mysql_conn->cluster_name); global_last_rollup = check_time; slurm_mutex_unlock(&rollup_lock); /* If the times here are later than the daily_rollup or monthly rollup it isn't a big deal since they are always shrunk down to the beginning of each time period. */ query = xstrdup_printf("update \"%s_%s\" set " "hourly_rollup=%ld, " "daily_rollup=%ld, monthly_rollup=%ld", mysql_conn->cluster_name, last_ran_table, check_time, check_time, check_time); debug3("%d(%s:%d) query\n%s", mysql_conn->conn, THIS_FILE, __LINE__, query); rc = mysql_db_query(mysql_conn, query); xfree(query); } else slurm_mutex_unlock(&rollup_lock); no_rollup_change: if (job_ptr->name && job_ptr->name[0]) jname = slurm_add_slash_to_quotes(job_ptr->name); else { jname = xstrdup("allocation"); track_steps = 1; } if (job_ptr->nodes && job_ptr->nodes[0]) nodes = job_ptr->nodes; else nodes = "None assigned"; if (job_ptr->batch_flag) track_steps = 1; if (slurmdbd_conf) { block_id = xstrdup(job_ptr->comment); node_cnt = job_ptr->total_nodes; node_inx = job_ptr->network; } else { char temp_bit[BUF_SIZE]; if (job_ptr->node_bitmap) { node_inx = bit_fmt(temp_bit, sizeof(temp_bit), job_ptr->node_bitmap); } #ifdef HAVE_BG select_g_select_jobinfo_get(job_ptr->select_jobinfo, SELECT_JOBDATA_BLOCK_ID, &block_id); select_g_select_jobinfo_get(job_ptr->select_jobinfo, SELECT_JOBDATA_NODE_CNT, &node_cnt); #else node_cnt = job_ptr->total_nodes; #endif } /* If there is a start_time get the wckeyid. If the job is * cancelled before the job starts we also want to grab it. */ if (job_ptr->assoc_id && (job_ptr->start_time || IS_JOB_CANCELLED(job_ptr))) wckeyid = _get_wckeyid(mysql_conn, &job_ptr->wckey, job_ptr->user_id, mysql_conn->cluster_name, job_ptr->assoc_id); if (job_ptr->partition) partition = slurm_add_slash_to_quotes(job_ptr->partition); if (job_ptr->gres_req) gres_req = slurm_add_slash_to_quotes(job_ptr->gres_req); if (job_ptr->gres_alloc) gres_alloc = slurm_add_slash_to_quotes(job_ptr->gres_alloc); if (!job_ptr->db_index) { if (!begin_time) begin_time = submit_time; query = xstrdup_printf( "insert into \"%s_%s\" " "(id_job, id_array_job, id_array_task, " "id_assoc, id_qos, id_wckey, id_user, " "id_group, nodelist, id_resv, timelimit, " "time_eligible, time_submit, time_start, " "job_name, track_steps, state, priority, cpus_req, " "cpus_alloc, nodes_alloc, mem_req", mysql_conn->cluster_name, job_table); if (job_ptr->account) xstrcat(query, ", account"); if (partition) xstrcat(query, ", `partition`"); if (block_id) xstrcat(query, ", id_block"); if (job_ptr->wckey) xstrcat(query, ", wckey"); if (node_inx) xstrcat(query, ", node_inx"); if (gres_req) xstrcat(query, ", gres_req"); if (gres_alloc) xstrcat(query, ", gres_alloc"); xstrfmtcat(query, ") values (%u, %u, %u, %u, %u, %u, %u, %u, " "'%s', %u, %u, %ld, %ld, %ld, " "'%s', %u, %u, %u, %u, %u, %u, %u", job_ptr->job_id, job_ptr->array_job_id, job_ptr->array_task_id, job_ptr->assoc_id, job_ptr->qos_id, wckeyid, job_ptr->user_id, job_ptr->group_id, nodes, job_ptr->resv_id, job_ptr->time_limit, begin_time, submit_time, start_time, jname, track_steps, job_state, job_ptr->priority, job_ptr->details->min_cpus, job_ptr->total_cpus, node_cnt, job_ptr->details->pn_min_memory); if (job_ptr->account) xstrfmtcat(query, ", '%s'", job_ptr->account); if (partition) xstrfmtcat(query, ", '%s'", partition); if (block_id) xstrfmtcat(query, ", '%s'", block_id); if (job_ptr->wckey) xstrfmtcat(query, ", '%s'", job_ptr->wckey); if (node_inx) xstrfmtcat(query, ", '%s'", node_inx); if (gres_req) xstrfmtcat(query, ", '%s'", gres_req); if (gres_alloc) xstrfmtcat(query, ", '%s'", gres_alloc); xstrfmtcat(query, ") on duplicate key update " "job_db_inx=LAST_INSERT_ID(job_db_inx), " "id_wckey=%u, id_user=%u, id_group=%u, " "nodelist='%s', id_resv=%u, timelimit=%u, " "time_submit=%ld, time_start=%ld, " "job_name='%s', track_steps=%u, id_qos=%u, " "state=greatest(state, %u), priority=%u, " "cpus_req=%u, cpus_alloc=%u, nodes_alloc=%u, " "mem_req=%u, id_array_job=%u, id_array_task=%u", wckeyid, job_ptr->user_id, job_ptr->group_id, nodes, job_ptr->resv_id, job_ptr->time_limit, submit_time, start_time, jname, track_steps, job_ptr->qos_id, job_state, job_ptr->priority, job_ptr->details->min_cpus, job_ptr->total_cpus, node_cnt, job_ptr->details->pn_min_memory, job_ptr->array_job_id, job_ptr->array_task_id); if (job_ptr->account) xstrfmtcat(query, ", account='%s'", job_ptr->account); if (partition) xstrfmtcat(query, ", `partition`='%s'", partition); if (block_id) xstrfmtcat(query, ", id_block='%s'", block_id); if (job_ptr->wckey) xstrfmtcat(query, ", wckey='%s'", job_ptr->wckey); if (node_inx) xstrfmtcat(query, ", node_inx='%s'", node_inx); if (gres_req) xstrfmtcat(query, ", gres_req='%s'", gres_req); if (gres_alloc) xstrfmtcat(query, ", gres_alloc='%s'", gres_alloc); debug3("%d(%s:%d) query\n%s", mysql_conn->conn, THIS_FILE, __LINE__, query); try_again: if (!(job_ptr->db_index = mysql_db_insert_ret_id( mysql_conn, query))) { if (!reinit) { error("It looks like the storage has gone " "away trying to reconnect"); mysql_db_close_db_connection( mysql_conn); /* reconnect */ check_connection(mysql_conn); reinit = 1; goto try_again; } else rc = SLURM_ERROR; } } else { query = xstrdup_printf("update \"%s_%s\" set nodelist='%s', ", mysql_conn->cluster_name, job_table, nodes); if (job_ptr->account) xstrfmtcat(query, "account='%s', ", job_ptr->account); if (partition) xstrfmtcat(query, "`partition`='%s', ", partition); if (block_id) xstrfmtcat(query, "id_block='%s', ", block_id); if (job_ptr->wckey) xstrfmtcat(query, "wckey='%s', ", job_ptr->wckey); if (node_inx) xstrfmtcat(query, "node_inx='%s', ", node_inx); if (gres_req) xstrfmtcat(query, "gres_req='%s', ", gres_req); if (gres_alloc) xstrfmtcat(query, "gres_alloc='%s', ", gres_alloc); xstrfmtcat(query, "time_start=%ld, job_name='%s', state=%u, " "cpus_alloc=%u, nodes_alloc=%u, id_qos=%u, " "id_assoc=%u, id_wckey=%u, id_resv=%u, " "timelimit=%u, mem_req=%u, " "id_array_job=%u, id_array_task=%u, " "time_eligible=%ld where job_db_inx=%d", start_time, jname, job_state, job_ptr->total_cpus, node_cnt, job_ptr->qos_id, job_ptr->assoc_id, wckeyid, job_ptr->resv_id, job_ptr->time_limit, job_ptr->details->pn_min_memory, job_ptr->array_job_id, job_ptr->array_task_id, begin_time, job_ptr->db_index); debug3("%d(%s:%d) query\n%s", mysql_conn->conn, THIS_FILE, __LINE__, query); rc = mysql_db_query(mysql_conn, query); } xfree(block_id); xfree(partition); xfree(gres_req); xfree(gres_alloc); xfree(jname); xfree(query); /* now we will reset all the steps */ if (IS_JOB_RESIZING(job_ptr)) { /* FIXME : Verify this is still needed */ if (IS_JOB_SUSPENDED(job_ptr)) as_mysql_suspend(mysql_conn, job_db_inx, job_ptr); } return rc; }
/* * Allocate nodes from the slurm controller -- retrying the attempt * if the controller appears to be down, and optionally waiting for * resources if none are currently available (see opt.immediate) * * Returns a pointer to a resource_allocation_response_msg which must * be freed with slurm_free_resource_allocation_response_msg() */ extern resource_allocation_response_msg_t * allocate_nodes(bool handle_signals, slurm_opt_t *opt_local) { srun_opt_t *srun_opt = opt_local->srun_opt; resource_allocation_response_msg_t *resp = NULL; job_desc_msg_t *j; slurm_allocation_callbacks_t callbacks; int i; xassert(srun_opt); if (srun_opt->relative_set && srun_opt->relative) fatal("--relative option invalid for job allocation request"); if ((j = _job_desc_msg_create_from_opts(&opt)) == NULL) return NULL; if (opt_local->clusters && (slurmdb_get_first_avail_cluster(j, opt_local->clusters, &working_cluster_rec) != SLURM_SUCCESS)) { print_db_notok(opt_local->clusters, 0); return NULL; } j->origin_cluster = xstrdup(slurmctld_conf.cluster_name); /* Do not re-use existing job id when submitting new job * from within a running job */ if ((j->job_id != NO_VAL) && !opt_local->jobid_set) { info("WARNING: Creating SLURM job allocation from within " "another allocation"); info("WARNING: You are attempting to initiate a second job"); if (!opt_local->jobid_set) /* Let slurmctld set jobid */ j->job_id = NO_VAL; } callbacks.ping = _ping_handler; callbacks.timeout = _timeout_handler; callbacks.job_complete = _job_complete_handler; callbacks.job_suspend = NULL; callbacks.user_msg = _user_msg_handler; callbacks.node_fail = _node_fail_handler; /* create message thread to handle pings and such from slurmctld */ msg_thr = slurm_allocation_msg_thr_create(&j->other_port, &callbacks); /* NOTE: Do not process signals in separate pthread. The signal will * cause slurm_allocate_resources_blocking() to exit immediately. */ if (handle_signals) { xsignal_unblock(sig_array); for (i = 0; sig_array[i]; i++) xsignal(sig_array[i], _signal_while_allocating); } while (!resp) { resp = slurm_allocate_resources_blocking(j, opt_local->immediate, _set_pending_job_id); if (destroy_job) { /* cancelled by signal */ break; } else if (!resp && !_retry()) { break; } } if (resp) print_multi_line_string(resp->job_submit_user_msg, -1); if (resp && !destroy_job) { /* * Allocation granted! */ pending_job_id = resp->job_id; /* * These values could be changed while the job was * pending so overwrite the request with what was * allocated so we don't have issues when we use them * in the step creation. */ opt_local->pn_min_memory = NO_VAL64; opt_local->mem_per_cpu = NO_VAL64; if (resp->pn_min_memory != NO_VAL64) { if (resp->pn_min_memory & MEM_PER_CPU) { opt_local->mem_per_cpu = (resp->pn_min_memory & (~MEM_PER_CPU)); } else { opt_local->pn_min_memory = resp->pn_min_memory; } } #ifdef HAVE_BG uint32_t node_cnt = 0; select_g_select_jobinfo_get(resp->select_jobinfo, SELECT_JOBDATA_NODE_CNT, &node_cnt); if ((node_cnt == 0) || (node_cnt == NO_VAL)) { opt_local->min_nodes = node_cnt; opt_local->max_nodes = node_cnt; } /* else we just use the original request */ if (!_wait_bluegene_block_ready(resp)) { if (!destroy_job) error("Something is wrong with the " "boot of the block."); goto relinquish; } #else opt_local->min_nodes = resp->node_cnt; opt_local->max_nodes = resp->node_cnt; if (resp->working_cluster_rec) slurm_setup_remote_working_cluster(resp); if (!_wait_nodes_ready(resp)) { if (!destroy_job) error("Something is wrong with the boot of the nodes."); goto relinquish; } #endif } else if (destroy_job) { goto relinquish; } if (handle_signals) xsignal_block(sig_array); job_desc_msg_destroy(j); return resp; relinquish: if (resp) { if (!destroy_job) slurm_complete_job(resp->job_id, 1); slurm_free_resource_allocation_response_msg(resp); } exit(error_exit); return NULL; }
static int _print_text_job(job_info_t * job_ptr) { time_t time_diff; int printed = 0; int tempxcord; int prefixlen = 0; int i = 0; int width = 0; char time_buf[20]; char tmp_cnt[8]; uint32_t node_cnt = 0; char *ionodes = NULL, *uname; if (params.cluster_flags & CLUSTER_FLAG_BG) { select_g_select_jobinfo_get(job_ptr->select_jobinfo, SELECT_JOBDATA_IONODES, &ionodes); select_g_select_jobinfo_get(job_ptr->select_jobinfo, SELECT_JOBDATA_NODE_CNT, &node_cnt); if (!xstrcasecmp(job_ptr->nodes,"waiting...")) xfree(ionodes); } else node_cnt = job_ptr->num_nodes; if ((node_cnt == 0) || (node_cnt == NO_VAL)) node_cnt = job_ptr->num_nodes; if (params.cluster_flags & CLUSTER_FLAG_BG) convert_num_unit((float)node_cnt, tmp_cnt, sizeof(tmp_cnt), UNIT_NONE, NO_VAL, CONVERT_NUM_UNIT_EXACT); else snprintf(tmp_cnt, sizeof(tmp_cnt), "%d", node_cnt); if (!params.commandline) { mvwprintw(text_win, main_ycord, main_xcord, "%c", job_ptr->num_cpus); main_xcord += 3; if (job_ptr->array_task_str) { mvwprintw(text_win, main_ycord, main_xcord, "%u_[%s]", job_ptr->array_job_id, job_ptr->array_task_str); } else if (job_ptr->array_task_id != NO_VAL) { mvwprintw(text_win, main_ycord, main_xcord, "%u_%u (%u)", job_ptr->array_job_id, job_ptr->array_task_id, job_ptr->job_id); } else if (job_ptr->pack_job_id) { mvwprintw(text_win, main_ycord, main_xcord, "%u+%u ", job_ptr->pack_job_id, job_ptr->pack_job_offset); } else { mvwprintw(text_win, main_ycord, main_xcord, "%u", job_ptr->job_id); } main_xcord += 19; mvwprintw(text_win, main_ycord, main_xcord, "%.10s", job_ptr->partition); main_xcord += 10; if (params.cluster_flags & CLUSTER_FLAG_BG) { mvwprintw(text_win, main_ycord, main_xcord, "%.16s", select_g_select_jobinfo_sprint( job_ptr->select_jobinfo, time_buf, sizeof(time_buf), SELECT_PRINT_BG_ID)); main_xcord += 18; } if (params.cluster_flags & CLUSTER_FLAG_CRAY_A) { mvwprintw(text_win, main_ycord, main_xcord, "%.16s", select_g_select_jobinfo_sprint( job_ptr->select_jobinfo, time_buf, sizeof(time_buf), SELECT_PRINT_DATA)); main_xcord += 18; } uname = uid_to_string_cached((uid_t) job_ptr->user_id); mvwprintw(text_win, main_ycord, main_xcord, "%.8s", uname); main_xcord += 9; mvwprintw(text_win, main_ycord, main_xcord, "%.9s", job_ptr->name); main_xcord += 10; mvwprintw(text_win, main_ycord, main_xcord, "%.2s", job_state_string_compact(job_ptr->job_state)); main_xcord += 2; if (!xstrcasecmp(job_ptr->nodes,"waiting...")) { sprintf(time_buf,"00:00:00"); } else { time_diff = (time_t) _job_time_used(job_ptr); secs2time_str(time_diff, time_buf, sizeof(time_buf)); } width = strlen(time_buf); mvwprintw(text_win, main_ycord, main_xcord + (10 - width), "%s", time_buf); main_xcord += 11; mvwprintw(text_win, main_ycord, main_xcord, "%5s", tmp_cnt); main_xcord += 6; tempxcord = main_xcord; i=0; while (job_ptr->nodes[i] != '\0') { if ((printed = mvwaddch(text_win, main_ycord, main_xcord, job_ptr->nodes[i])) < 0) { xfree(ionodes); return printed; } main_xcord++; width = getmaxx(text_win) - 1 - main_xcord; if (job_ptr->nodes[i] == '[') prefixlen = i + 1; else if (job_ptr->nodes[i] == ',' && (width - 9) <= 0) { main_ycord++; main_xcord = tempxcord + prefixlen; } i++; } if (ionodes) { mvwprintw(text_win, main_ycord, main_xcord, "[%s]", ionodes); main_xcord += strlen(ionodes)+2; xfree(ionodes); } main_xcord = 1; main_ycord++; } else { if (job_ptr->pack_job_id) { printf("%8u+%u ", job_ptr->pack_job_id, job_ptr->pack_job_offset); } else printf("%8u ", job_ptr->job_id); printf("%9.9s ", job_ptr->partition); if (params.cluster_flags & CLUSTER_FLAG_BG) printf("%16.16s ", select_g_select_jobinfo_sprint( job_ptr->select_jobinfo, time_buf, sizeof(time_buf), SELECT_PRINT_BG_ID)); if (params.cluster_flags & CLUSTER_FLAG_CRAY_A) printf("%16.16s ", select_g_select_jobinfo_sprint( job_ptr->select_jobinfo, time_buf, sizeof(time_buf), SELECT_PRINT_DATA)); uname = uid_to_string_cached((uid_t) job_ptr->user_id); printf("%8.8s ", uname); printf("%6.6s ", job_ptr->name); printf("%2.2s ", job_state_string_compact(job_ptr->job_state)); if (!xstrcasecmp(job_ptr->nodes,"waiting...")) { sprintf(time_buf,"00:00:00"); } else { time_diff = (time_t) _job_time_used(job_ptr); secs2time_str(time_diff, time_buf, sizeof(time_buf)); } printf("%10.10s ", time_buf); printf("%5s ", tmp_cnt); printf("%s", job_ptr->nodes); if (ionodes) { printf("[%s]", ionodes); xfree(ionodes); } printf("\n"); } return printed; }
/* * slurm_sprint_job_info - output information about a specific Slurm * job based upon message as loaded using slurm_load_jobs * IN job_ptr - an individual job information record pointer * IN one_liner - print as a single line if true * RET out - char * containing formatted output (must be freed after call) * NULL is returned on failure. */ extern char * slurm_sprint_job_info ( job_info_t * job_ptr, int one_liner ) { int i, j, k; char time_str[32], *group_name, *user_name; char *gres_last = "", tmp1[128], tmp2[128]; char *tmp6_ptr; char tmp_line[1024 * 128]; char tmp_path[MAXPATHLEN]; char *ionodes = NULL; uint16_t exit_status = 0, term_sig = 0; job_resources_t *job_resrcs = job_ptr->job_resrcs; char *out = NULL; time_t run_time; uint32_t min_nodes, max_nodes = 0; char *nodelist = "NodeList"; bitstr_t *cpu_bitmap; char *host; int sock_inx, sock_reps, last; int abs_node_inx, rel_node_inx; int64_t nice; int bit_inx, bit_reps; uint64_t *last_mem_alloc_ptr = NULL; uint64_t last_mem_alloc = NO_VAL64; char *last_hosts; hostlist_t hl, hl_last; char select_buf[122]; uint32_t cluster_flags = slurmdb_setup_cluster_flags(); uint32_t threads; char *line_end = (one_liner) ? " " : "\n "; if (cluster_flags & CLUSTER_FLAG_BG) { nodelist = "MidplaneList"; select_g_select_jobinfo_get(job_ptr->select_jobinfo, SELECT_JOBDATA_IONODES, &ionodes); } /****** Line 1 ******/ xstrfmtcat(out, "JobId=%u ", job_ptr->job_id); if (job_ptr->array_job_id) { if (job_ptr->array_task_str) { xstrfmtcat(out, "ArrayJobId=%u ArrayTaskId=%s ", job_ptr->array_job_id, job_ptr->array_task_str); } else { xstrfmtcat(out, "ArrayJobId=%u ArrayTaskId=%u ", job_ptr->array_job_id, job_ptr->array_task_id); } } xstrfmtcat(out, "JobName=%s", job_ptr->name); xstrcat(out, line_end); /****** Line 2 ******/ user_name = uid_to_string((uid_t) job_ptr->user_id); group_name = gid_to_string((gid_t) job_ptr->group_id); xstrfmtcat(out, "UserId=%s(%u) GroupId=%s(%u) MCS_label=%s", user_name, job_ptr->user_id, group_name, job_ptr->group_id, (job_ptr->mcs_label==NULL) ? "N/A" : job_ptr->mcs_label); xfree(user_name); xfree(group_name); xstrcat(out, line_end); /****** Line 3 ******/ nice = ((int64_t)job_ptr->nice) - NICE_OFFSET; xstrfmtcat(out, "Priority=%u Nice=%"PRIi64" Account=%s QOS=%s", job_ptr->priority, nice, job_ptr->account, job_ptr->qos); if (slurm_get_track_wckey()) xstrfmtcat(out, " WCKey=%s", job_ptr->wckey); xstrcat(out, line_end); /****** Line 4 ******/ xstrfmtcat(out, "JobState=%s ", job_state_string(job_ptr->job_state)); if (job_ptr->state_desc) { /* Replace white space with underscore for easier parsing */ for (j=0; job_ptr->state_desc[j]; j++) { if (isspace((int)job_ptr->state_desc[j])) job_ptr->state_desc[j] = '_'; } xstrfmtcat(out, "Reason=%s ", job_ptr->state_desc); } else xstrfmtcat(out, "Reason=%s ", job_reason_string(job_ptr->state_reason)); xstrfmtcat(out, "Dependency=%s", job_ptr->dependency); xstrcat(out, line_end); /****** Line 5 ******/ xstrfmtcat(out, "Requeue=%u Restarts=%u BatchFlag=%u Reboot=%u ", job_ptr->requeue, job_ptr->restart_cnt, job_ptr->batch_flag, job_ptr->reboot); if (WIFSIGNALED(job_ptr->exit_code)) term_sig = WTERMSIG(job_ptr->exit_code); exit_status = WEXITSTATUS(job_ptr->exit_code); xstrfmtcat(out, "ExitCode=%u:%u", exit_status, term_sig); xstrcat(out, line_end); /****** Line 5a (optional) ******/ if (job_ptr->show_flags & SHOW_DETAIL) { if (WIFSIGNALED(job_ptr->derived_ec)) term_sig = WTERMSIG(job_ptr->derived_ec); else term_sig = 0; exit_status = WEXITSTATUS(job_ptr->derived_ec); xstrfmtcat(out, "DerivedExitCode=%u:%u", exit_status, term_sig); xstrcat(out, line_end); } /****** Line 6 ******/ if (IS_JOB_PENDING(job_ptr)) run_time = 0; else if (IS_JOB_SUSPENDED(job_ptr)) run_time = job_ptr->pre_sus_time; else { time_t end_time; if (IS_JOB_RUNNING(job_ptr) || (job_ptr->end_time == 0)) end_time = time(NULL); else end_time = job_ptr->end_time; if (job_ptr->suspend_time) { run_time = (time_t) (difftime(end_time, job_ptr->suspend_time) + job_ptr->pre_sus_time); } else run_time = (time_t) difftime(end_time, job_ptr->start_time); } secs2time_str(run_time, time_str, sizeof(time_str)); xstrfmtcat(out, "RunTime=%s ", time_str); if (job_ptr->time_limit == NO_VAL) xstrcat(out, "TimeLimit=Partition_Limit "); else { mins2time_str(job_ptr->time_limit, time_str, sizeof(time_str)); xstrfmtcat(out, "TimeLimit=%s ", time_str); } if (job_ptr->time_min == 0) xstrcat(out, "TimeMin=N/A"); else { mins2time_str(job_ptr->time_min, time_str, sizeof(time_str)); xstrfmtcat(out, "TimeMin=%s", time_str); } xstrcat(out, line_end); /****** Line 7 ******/ slurm_make_time_str(&job_ptr->submit_time, time_str, sizeof(time_str)); xstrfmtcat(out, "SubmitTime=%s ", time_str); slurm_make_time_str(&job_ptr->eligible_time, time_str, sizeof(time_str)); xstrfmtcat(out, "EligibleTime=%s", time_str); xstrcat(out, line_end); /****** Line 8 (optional) ******/ if (job_ptr->resize_time) { slurm_make_time_str(&job_ptr->resize_time, time_str, sizeof(time_str)); xstrfmtcat(out, "ResizeTime=%s", time_str); xstrcat(out, line_end); } /****** Line 9 ******/ slurm_make_time_str(&job_ptr->start_time, time_str, sizeof(time_str)); xstrfmtcat(out, "StartTime=%s ", time_str); if ((job_ptr->time_limit == INFINITE) && (job_ptr->end_time > time(NULL))) xstrcat(out, "EndTime=Unknown "); else { slurm_make_time_str(&job_ptr->end_time, time_str, sizeof(time_str)); xstrfmtcat(out, "EndTime=%s ", time_str); } if (job_ptr->deadline) { slurm_make_time_str(&job_ptr->deadline, time_str, sizeof(time_str)); xstrfmtcat(out, "Deadline=%s", time_str); } else { xstrcat(out, "Deadline=N/A"); } xstrcat(out, line_end); /****** Line 10 ******/ if (job_ptr->preempt_time == 0) xstrcat(out, "PreemptTime=None "); else { slurm_make_time_str(&job_ptr->preempt_time, time_str, sizeof(time_str)); xstrfmtcat(out, "PreemptTime=%s ", time_str); } if (job_ptr->suspend_time) { slurm_make_time_str(&job_ptr->suspend_time, time_str, sizeof(time_str)); xstrfmtcat(out, "SuspendTime=%s ", time_str); } else xstrcat(out, "SuspendTime=None "); xstrfmtcat(out, "SecsPreSuspend=%ld", (long int)job_ptr->pre_sus_time); xstrcat(out, line_end); /****** Line 11 ******/ xstrfmtcat(out, "Partition=%s AllocNode:Sid=%s:%u", job_ptr->partition, job_ptr->alloc_node, job_ptr->alloc_sid); xstrcat(out, line_end); /****** Line 12 ******/ xstrfmtcat(out, "Req%s=%s Exc%s=%s", nodelist, job_ptr->req_nodes, nodelist, job_ptr->exc_nodes); xstrcat(out, line_end); /****** Line 13 ******/ xstrfmtcat(out, "%s=%s", nodelist, job_ptr->nodes); if (job_ptr->nodes && ionodes) { xstrfmtcat(out, "[%s]", ionodes); xfree(ionodes); } if (job_ptr->sched_nodes) xstrfmtcat(out, " Sched%s=%s", nodelist, job_ptr->sched_nodes); xstrcat(out, line_end); /****** Line 14 (optional) ******/ if (job_ptr->batch_host) { xstrfmtcat(out, "BatchHost=%s", job_ptr->batch_host); xstrcat(out, line_end); } /****** Line 14a (optional) ******/ if (job_ptr->fed_siblings) { xstrfmtcat(out, "FedOrigin=%s FedSiblings=%s", job_ptr->fed_origin_str, job_ptr->fed_siblings_str); xstrcat(out, line_end); } /****** Line 15 ******/ if (cluster_flags & CLUSTER_FLAG_BG) { select_g_select_jobinfo_get(job_ptr->select_jobinfo, SELECT_JOBDATA_NODE_CNT, &min_nodes); if ((min_nodes == 0) || (min_nodes == NO_VAL)) { min_nodes = job_ptr->num_nodes; max_nodes = job_ptr->max_nodes; } else if (job_ptr->max_nodes) max_nodes = min_nodes; } else if (IS_JOB_PENDING(job_ptr)) { min_nodes = job_ptr->num_nodes; max_nodes = job_ptr->max_nodes; if (max_nodes && (max_nodes < min_nodes)) min_nodes = max_nodes; } else { min_nodes = job_ptr->num_nodes; max_nodes = 0; } _sprint_range(tmp_line, sizeof(tmp_line), min_nodes, max_nodes); xstrfmtcat(out, "NumNodes=%s ", tmp_line); _sprint_range(tmp_line, sizeof(tmp_line), job_ptr->num_cpus, job_ptr->max_cpus); xstrfmtcat(out, "NumCPUs=%s ", tmp_line); xstrfmtcat(out, "NumTasks=%u ", job_ptr->num_tasks); xstrfmtcat(out, "CPUs/Task=%u ", job_ptr->cpus_per_task); if (job_ptr->boards_per_node == (uint16_t) NO_VAL) xstrcat(out, "ReqB:S:C:T=*:"); else xstrfmtcat(out, "ReqB:S:C:T=%u:", job_ptr->boards_per_node); if (job_ptr->sockets_per_board == (uint16_t) NO_VAL) xstrcat(out, "*:"); else xstrfmtcat(out, "%u:", job_ptr->sockets_per_board); if (job_ptr->cores_per_socket == (uint16_t) NO_VAL) xstrcat(out, "*:"); else xstrfmtcat(out, "%u:", job_ptr->cores_per_socket); if (job_ptr->threads_per_core == (uint16_t) NO_VAL) xstrcat(out, "*"); else xstrfmtcat(out, "%u", job_ptr->threads_per_core); xstrcat(out, line_end); /****** Line 16 ******/ /* Tres should already of been converted at this point from simple */ xstrfmtcat(out, "TRES=%s", job_ptr->tres_alloc_str ? job_ptr->tres_alloc_str : job_ptr->tres_req_str); xstrcat(out, line_end); /****** Line 17 ******/ if (job_ptr->sockets_per_node == (uint16_t) NO_VAL) xstrcat(out, "Socks/Node=* "); else xstrfmtcat(out, "Socks/Node=%u ", job_ptr->sockets_per_node); if (job_ptr->ntasks_per_node == (uint16_t) NO_VAL) xstrcat(out, "NtasksPerN:B:S:C=*:"); else xstrfmtcat(out, "NtasksPerN:B:S:C=%u:", job_ptr->ntasks_per_node); if (job_ptr->ntasks_per_board == (uint16_t) NO_VAL) xstrcat(out, "*:"); else xstrfmtcat(out, "%u:", job_ptr->ntasks_per_board); if ((job_ptr->ntasks_per_socket == (uint16_t) NO_VAL) || (job_ptr->ntasks_per_socket == (uint16_t) INFINITE)) xstrcat(out, "*:"); else xstrfmtcat(out, "%u:", job_ptr->ntasks_per_socket); if ((job_ptr->ntasks_per_core == (uint16_t) NO_VAL) || (job_ptr->ntasks_per_core == (uint16_t) INFINITE)) xstrcat(out, "* "); else xstrfmtcat(out, "%u ", job_ptr->ntasks_per_core); if (job_ptr->core_spec == (uint16_t) NO_VAL) xstrcat(out, "CoreSpec=*"); else if (job_ptr->core_spec & CORE_SPEC_THREAD) xstrfmtcat(out, "ThreadSpec=%d", (job_ptr->core_spec & (~CORE_SPEC_THREAD))); else xstrfmtcat(out, "CoreSpec=%u", job_ptr->core_spec); xstrcat(out, line_end); if (job_resrcs && cluster_flags & CLUSTER_FLAG_BG) { if ((job_resrcs->cpu_array_cnt > 0) && (job_resrcs->cpu_array_value) && (job_resrcs->cpu_array_reps)) { int length = 0; xstrcat(out, "CPUs="); for (i = 0; i < job_resrcs->cpu_array_cnt; i++) { /* only print 60 characters worth of this record */ if (length > 60) { /* skip to last CPU group entry */ if (i < job_resrcs->cpu_array_cnt - 1) { continue; } /* add ellipsis before last entry */ xstrcat(out, "...,"); } length += xstrfmtcat(out, "%d", job_resrcs->cpus[i]); if (job_resrcs->cpu_array_reps[i] > 1) { length += xstrfmtcat(out, "*%d", job_resrcs->cpu_array_reps[i]); } if (i < job_resrcs->cpu_array_cnt - 1) { xstrcat(out, ","); length++; } } xstrcat(out, line_end); } } else if (job_resrcs && job_resrcs->core_bitmap && ((last = bit_fls(job_resrcs->core_bitmap)) != -1)) { hl = hostlist_create(job_resrcs->nodes); if (!hl) { error("slurm_sprint_job_info: hostlist_create: %s", job_resrcs->nodes); return NULL; } hl_last = hostlist_create(NULL); if (!hl_last) { error("slurm_sprint_job_info: hostlist_create: NULL"); hostlist_destroy(hl); return NULL; } bit_inx = 0; i = sock_inx = sock_reps = 0; abs_node_inx = job_ptr->node_inx[i]; gres_last = ""; /* tmp1[] stores the current cpu(s) allocated */ tmp2[0] = '\0'; /* stores last cpu(s) allocated */ for (rel_node_inx=0; rel_node_inx < job_resrcs->nhosts; rel_node_inx++) { if (sock_reps >= job_resrcs->sock_core_rep_count[sock_inx]) { sock_inx++; sock_reps = 0; } sock_reps++; bit_reps = job_resrcs->sockets_per_node[sock_inx] * job_resrcs->cores_per_socket[sock_inx]; host = hostlist_shift(hl); threads = _threads_per_core(host); cpu_bitmap = bit_alloc(bit_reps * threads); for (j = 0; j < bit_reps; j++) { if (bit_test(job_resrcs->core_bitmap, bit_inx)){ for (k = 0; k < threads; k++) bit_set(cpu_bitmap, (j * threads) + k); } bit_inx++; } bit_fmt(tmp1, sizeof(tmp1), cpu_bitmap); FREE_NULL_BITMAP(cpu_bitmap); /* * If the allocation values for this host are not the * same as the last host, print the report of the last * group of hosts that had identical allocation values. */ if (xstrcmp(tmp1, tmp2) || ((rel_node_inx < job_ptr->gres_detail_cnt) && xstrcmp(job_ptr->gres_detail_str[rel_node_inx], gres_last)) || (last_mem_alloc_ptr != job_resrcs->memory_allocated) || (job_resrcs->memory_allocated && (last_mem_alloc != job_resrcs->memory_allocated[rel_node_inx]))) { if (hostlist_count(hl_last)) { last_hosts = hostlist_ranged_string_xmalloc( hl_last); xstrfmtcat(out, " Nodes=%s CPU_IDs=%s " "Mem=%"PRIu64" GRES_IDX=%s", last_hosts, tmp2, last_mem_alloc_ptr ? last_mem_alloc : 0, gres_last); xfree(last_hosts); xstrcat(out, line_end); hostlist_destroy(hl_last); hl_last = hostlist_create(NULL); } strcpy(tmp2, tmp1); if (rel_node_inx < job_ptr->gres_detail_cnt) { gres_last = job_ptr-> gres_detail_str[rel_node_inx]; } else { gres_last = ""; } last_mem_alloc_ptr = job_resrcs->memory_allocated; if (last_mem_alloc_ptr) last_mem_alloc = job_resrcs-> memory_allocated[rel_node_inx]; else last_mem_alloc = NO_VAL64; } hostlist_push_host(hl_last, host); free(host); if (bit_inx > last) break; if (abs_node_inx > job_ptr->node_inx[i+1]) { i += 2; abs_node_inx = job_ptr->node_inx[i]; } else { abs_node_inx++; } } if (hostlist_count(hl_last)) { last_hosts = hostlist_ranged_string_xmalloc(hl_last); xstrfmtcat(out, " Nodes=%s CPU_IDs=%s Mem=%"PRIu64" GRES_IDX=%s", last_hosts, tmp2, last_mem_alloc_ptr ? last_mem_alloc : 0, gres_last); xfree(last_hosts); xstrcat(out, line_end); } hostlist_destroy(hl); hostlist_destroy(hl_last); } /****** Line 18 ******/ if (job_ptr->pn_min_memory & MEM_PER_CPU) { job_ptr->pn_min_memory &= (~MEM_PER_CPU); tmp6_ptr = "CPU"; } else tmp6_ptr = "Node"; if (cluster_flags & CLUSTER_FLAG_BG) { convert_num_unit((float)job_ptr->pn_min_cpus, tmp1, sizeof(tmp1), UNIT_NONE, NO_VAL, CONVERT_NUM_UNIT_EXACT); xstrfmtcat(out, "MinCPUsNode=%s ", tmp1); } else { xstrfmtcat(out, "MinCPUsNode=%u ", job_ptr->pn_min_cpus); } convert_num_unit((float)job_ptr->pn_min_memory, tmp1, sizeof(tmp1), UNIT_MEGA, NO_VAL, CONVERT_NUM_UNIT_EXACT); convert_num_unit((float)job_ptr->pn_min_tmp_disk, tmp2, sizeof(tmp2), UNIT_MEGA, NO_VAL, CONVERT_NUM_UNIT_EXACT); xstrfmtcat(out, "MinMemory%s=%s MinTmpDiskNode=%s", tmp6_ptr, tmp1, tmp2); xstrcat(out, line_end); /****** Line ******/ secs2time_str((time_t)job_ptr->delay_boot, tmp1, sizeof(tmp1)); xstrfmtcat(out, "Features=%s DelayBoot=%s", job_ptr->features, tmp1); xstrcat(out, line_end); /****** Line ******/ xstrfmtcat(out, "Gres=%s Reservation=%s", job_ptr->gres, job_ptr->resv_name); xstrcat(out, line_end); /****** Line 20 ******/ xstrfmtcat(out, "OverSubscribe=%s Contiguous=%d Licenses=%s Network=%s", job_share_string(job_ptr->shared), job_ptr->contiguous, job_ptr->licenses, job_ptr->network); xstrcat(out, line_end); /****** Line 21 ******/ xstrfmtcat(out, "Command=%s", job_ptr->command); xstrcat(out, line_end); /****** Line 22 ******/ xstrfmtcat(out, "WorkDir=%s", job_ptr->work_dir); if (cluster_flags & CLUSTER_FLAG_BG) { /****** Line 23 (optional) ******/ select_g_select_jobinfo_sprint(job_ptr->select_jobinfo, select_buf, sizeof(select_buf), SELECT_PRINT_BG_ID); if (select_buf[0] != '\0') { xstrcat(out, line_end); xstrfmtcat(out, "Block_ID=%s", select_buf); } /****** Line 24 (optional) ******/ select_g_select_jobinfo_sprint(job_ptr->select_jobinfo, select_buf, sizeof(select_buf), SELECT_PRINT_MIXED_SHORT); if (select_buf[0] != '\0') { xstrcat(out, line_end); xstrcat(out, select_buf); } /****** Line 26 (optional) ******/ select_g_select_jobinfo_sprint(job_ptr->select_jobinfo, select_buf, sizeof(select_buf), SELECT_PRINT_LINUX_IMAGE); if (select_buf[0] != '\0') { xstrcat(out, line_end); xstrfmtcat(out, "CnloadImage=%s", select_buf); } /****** Line 27 (optional) ******/ select_g_select_jobinfo_sprint(job_ptr->select_jobinfo, select_buf, sizeof(select_buf), SELECT_PRINT_MLOADER_IMAGE); if (select_buf[0] != '\0') { xstrcat(out, line_end); xstrfmtcat(out, "MloaderImage=%s", select_buf); } /****** Line 28 (optional) ******/ select_g_select_jobinfo_sprint(job_ptr->select_jobinfo, select_buf, sizeof(select_buf), SELECT_PRINT_RAMDISK_IMAGE); if (select_buf[0] != '\0') { xstrcat(out, line_end); xstrfmtcat(out, "IoloadImage=%s", select_buf); } } /****** Line (optional) ******/ if (job_ptr->admin_comment) { xstrcat(out, line_end); xstrfmtcat(out, "AdminComment=%s ", job_ptr->admin_comment); } /****** Line (optional) ******/ if (job_ptr->comment) { xstrcat(out, line_end); xstrfmtcat(out, "Comment=%s ", job_ptr->comment); } /****** Line 30 (optional) ******/ if (job_ptr->batch_flag) { xstrcat(out, line_end); slurm_get_job_stderr(tmp_path, sizeof(tmp_path), job_ptr); xstrfmtcat(out, "StdErr=%s", tmp_path); } /****** Line 31 (optional) ******/ if (job_ptr->batch_flag) { xstrcat(out, line_end); slurm_get_job_stdin(tmp_path, sizeof(tmp_path), job_ptr); xstrfmtcat(out, "StdIn=%s", tmp_path); } /****** Line 32 (optional) ******/ if (job_ptr->batch_flag) { xstrcat(out, line_end); slurm_get_job_stdout(tmp_path, sizeof(tmp_path), job_ptr); xstrfmtcat(out, "StdOut=%s", tmp_path); } /****** Line 33 (optional) ******/ if (job_ptr->batch_script) { xstrcat(out, line_end); xstrcat(out, "BatchScript=\n"); xstrcat(out, job_ptr->batch_script); } /****** Line 34 (optional) ******/ if (job_ptr->req_switch) { char time_buf[32]; xstrcat(out, line_end); secs2time_str((time_t) job_ptr->wait4switch, time_buf, sizeof(time_buf)); xstrfmtcat(out, "Switches=%u@%s\n", job_ptr->req_switch, time_buf); } /****** Line 35 (optional) ******/ if (job_ptr->burst_buffer) { xstrcat(out, line_end); xstrfmtcat(out, "BurstBuffer=%s", job_ptr->burst_buffer); } /****** Line (optional) ******/ if (job_ptr->burst_buffer_state) { xstrcat(out, line_end); xstrfmtcat(out, "BurstBufferState=%s", job_ptr->burst_buffer_state); } /****** Line 36 (optional) ******/ if (cpu_freq_debug(NULL, NULL, tmp1, sizeof(tmp1), job_ptr->cpu_freq_gov, job_ptr->cpu_freq_min, job_ptr->cpu_freq_max, NO_VAL) != 0) { xstrcat(out, line_end); xstrcat(out, tmp1); } /****** Line 37 ******/ xstrcat(out, line_end); xstrfmtcat(out, "Power=%s", power_flags_str(job_ptr->power_flags)); /****** Line 38 (optional) ******/ if (job_ptr->bitflags) { xstrcat(out, line_end); if (job_ptr->bitflags & GRES_ENFORCE_BIND) xstrcat(out, "GresEnforceBind=Yes"); if (job_ptr->bitflags & KILL_INV_DEP) xstrcat(out, "KillOInInvalidDependent=Yes"); if (job_ptr->bitflags & NO_KILL_INV_DEP) xstrcat(out, "KillOInInvalidDependent=No"); if (job_ptr->bitflags & SPREAD_JOB) xstrcat(out, "SpreadJob=Yes"); } /****** END OF JOB RECORD ******/ if (one_liner) xstrcat(out, "\n"); else xstrcat(out, "\n\n"); return out; }
/* * load into the storage the start of a job step */ extern int jobacct_storage_p_step_start(void *db_conn, struct step_record *step_ptr) { char buf[BUFFER_SIZE]; int cpus = 0, rc; char node_list[BUFFER_SIZE]; #ifdef HAVE_BG char *ionodes = NULL; #endif float float_tmp = 0; char *account, *step_name; if (!storage_init) { debug("jobacct init was not called or it failed"); return SLURM_ERROR; } #ifdef HAVE_BG if (step_ptr->job_ptr->details) cpus = step_ptr->job_ptr->details->min_cpus; else cpus = step_ptr->job_ptr->cpu_cnt; select_g_select_jobinfo_get(step_ptr->job_ptr->select_jobinfo, SELECT_JOBDATA_IONODES, &ionodes); if (ionodes) { snprintf(node_list, BUFFER_SIZE, "%s[%s]", step_ptr->job_ptr->nodes, ionodes); xfree(ionodes); } else snprintf(node_list, BUFFER_SIZE, "%s", step_ptr->job_ptr->nodes); #else if (!step_ptr->step_layout || !step_ptr->step_layout->task_cnt) { cpus = step_ptr->job_ptr->total_cpus; snprintf(node_list, BUFFER_SIZE, "%s", step_ptr->job_ptr->nodes); } else { cpus = step_ptr->step_layout->task_cnt; snprintf(node_list, BUFFER_SIZE, "%s", step_ptr->step_layout->node_list); } #endif account = _safe_dup(step_ptr->job_ptr->account); step_name = _safe_dup(step_ptr->name); step_ptr->job_ptr->requid = -1; /* force to -1 for stats to know this * hasn't been set yet */ snprintf(buf, BUFFER_SIZE, _jobstep_format, JOB_STEP, step_ptr->step_id, /* stepid */ JOB_RUNNING, /* completion status */ 0, /* completion code */ cpus, /* number of tasks */ cpus, /* number of cpus */ 0, /* elapsed seconds */ 0, /* total cputime seconds */ 0, /* total cputime seconds */ 0, /* user seconds */ 0, /* user microseconds */ 0, /* system seconds */ 0, /* system microsecs */ 0, /* max rss */ 0, /* max ixrss */ 0, /* max idrss */ 0, /* max isrss */ 0, /* max minflt */ 0, /* max majflt */ 0, /* max nswap */ 0, /* total inblock */ 0, /* total outblock */ 0, /* total msgsnd */ 0, /* total msgrcv */ 0, /* total nsignals */ 0, /* total nvcsw */ 0, /* total nivcsw */ 0, /* max vsize */ 0, /* max vsize task */ float_tmp, /* ave vsize */ 0, /* max rss */ 0, /* max rss task */ float_tmp, /* ave rss */ 0, /* max pages */ 0, /* max pages task */ float_tmp, /* ave pages */ 0, /* min cpu */ 0, /* min cpu task */ float_tmp, /* ave cpu */ step_name, /* step exe name */ node_list, /* name of nodes step running on */ 0, /* max vsize node */ 0, /* max rss node */ 0, /* max pages node */ 0, /* min cpu node */ account, step_ptr->job_ptr->requid); /* requester user id */ rc = _print_record(step_ptr->job_ptr, step_ptr->start_time, buf); xfree(account); xfree(step_name); return rc; }
/* * slurm_sprint_job_step_info - output information about a specific Slurm * job step based upon message as loaded using slurm_get_job_steps * IN job_ptr - an individual job step information record pointer * IN one_liner - print as a single line if true * RET out - char * containing formatted output (must be freed after call) * NULL is returned on failure. */ char * slurm_sprint_job_step_info ( job_step_info_t * job_step_ptr, int one_liner ) { char time_str[32]; char limit_str[32]; char tmp_line[128]; char *out = NULL; uint32_t cluster_flags = slurmdb_setup_cluster_flags(); /****** Line 1 ******/ slurm_make_time_str ((time_t *)&job_step_ptr->start_time, time_str, sizeof(time_str)); if (job_step_ptr->time_limit == INFINITE) sprintf(limit_str, "UNLIMITED"); else secs2time_str ((time_t)job_step_ptr->time_limit * 60, limit_str, sizeof(limit_str)); snprintf(tmp_line, sizeof(tmp_line), "StepId=%u.%u UserId=%u StartTime=%s TimeLimit=%s", job_step_ptr->job_id, job_step_ptr->step_id, job_step_ptr->user_id, time_str, limit_str); out = xstrdup(tmp_line); if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); /****** Line 2 ******/ if (cluster_flags & CLUSTER_FLAG_BG) { char *io_nodes; select_g_select_jobinfo_get(job_step_ptr->select_jobinfo, SELECT_JOBDATA_IONODES, &io_nodes); snprintf(tmp_line, sizeof(tmp_line), "Partition=%s BP_List=%s[%s] Gres=%s", job_step_ptr->partition, job_step_ptr->nodes, io_nodes, job_step_ptr->gres); xfree(io_nodes); } else { snprintf(tmp_line, sizeof(tmp_line), "Partition=%s Nodes=%s Gres=%s", job_step_ptr->partition, job_step_ptr->nodes, job_step_ptr->gres); } xstrcat(out, tmp_line); if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); /****** Line 3 ******/ snprintf(tmp_line, sizeof(tmp_line), "Tasks=%u Name=%s Network=%s", job_step_ptr->num_tasks, job_step_ptr->name, job_step_ptr->network); xstrcat(out, tmp_line); if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); /****** Line 4 ******/ snprintf(tmp_line, sizeof(tmp_line), "ResvPorts=%s Checkpoint=%u CheckpointDir=%s\n\n", job_step_ptr->resv_ports, job_step_ptr->ckpt_interval, job_step_ptr->ckpt_dir); xstrcat(out, tmp_line); return out; }
static char * _will_run_test(uint32_t jobid, time_t start_time, char *node_list, int *err_code, char **err_msg) { struct job_record *job_ptr = NULL; struct part_record *part_ptr; bitstr_t *avail_bitmap = NULL, *resv_bitmap = NULL; bitstr_t *exc_core_bitmap = NULL; char *hostlist, *reply_msg = NULL; uint32_t min_nodes, max_nodes, req_nodes; int rc; time_t start_res, orig_start_time; List preemptee_candidates; debug2("wiki2: will_run job_id=%u start_time=%u node_list=%s", jobid, (uint32_t)start_time, node_list); job_ptr = find_job_record(jobid); if (job_ptr == NULL) { *err_code = -700; *err_msg = "No such job"; error("wiki: Failed to find job %u", jobid); return NULL; } if ((job_ptr->details == NULL) || (!IS_JOB_PENDING(job_ptr))) { *err_code = -700; *err_msg = "WillRun not applicable to non-pending job"; error("wiki: WillRun on non-pending job %u", jobid); return NULL; } part_ptr = job_ptr->part_ptr; if (part_ptr == NULL) { *err_code = -700; *err_msg = "Job lacks a partition"; error("wiki: Job %u lacks a partition", jobid); return NULL; } if ((node_list == NULL) || (node_list[0] == '\0')) { /* assume all nodes available to job for testing */ avail_bitmap = bit_copy(avail_node_bitmap); } else if (node_name2bitmap(node_list, false, &avail_bitmap) != 0) { *err_code = -700; *err_msg = "Invalid available nodes value"; error("wiki: Attempt to set invalid available node " "list for job %u, %s", jobid, node_list); return NULL; } /* Enforce reservation: access control, time and nodes */ start_res = start_time; rc = job_test_resv(job_ptr, &start_res, true, &resv_bitmap, &exc_core_bitmap); if (rc != SLURM_SUCCESS) { *err_code = -730; *err_msg = "Job denied access to reservation"; error("wiki: reservation access denied for job %u", jobid); FREE_NULL_BITMAP(avail_bitmap); return NULL; } start_time = MAX(start_time, start_res); bit_and(avail_bitmap, resv_bitmap); FREE_NULL_BITMAP(resv_bitmap); /* Only consider nodes that are not DOWN or DRAINED */ bit_and(avail_bitmap, avail_node_bitmap); /* Consider only nodes in this job's partition */ if (part_ptr->node_bitmap) bit_and(avail_bitmap, part_ptr->node_bitmap); else { *err_code = -730; *err_msg = "Job's partition has no nodes"; error("wiki: no nodes in partition %s for job %u", part_ptr->name, jobid); FREE_NULL_BITMAP(avail_bitmap); return NULL; } if (job_req_node_filter(job_ptr, avail_bitmap) != SLURM_SUCCESS) { /* Job probably has invalid feature list */ *err_code = -730; *err_msg = "Job's required features not available " "on selected nodes"; error("wiki: job %u not runnable on hosts=%s", jobid, node_list); FREE_NULL_BITMAP(avail_bitmap); return NULL; } if (job_ptr->details->exc_node_bitmap) { bit_not(job_ptr->details->exc_node_bitmap); bit_and(avail_bitmap, job_ptr->details->exc_node_bitmap); bit_not(job_ptr->details->exc_node_bitmap); } if ((job_ptr->details->req_node_bitmap) && (!bit_super_set(job_ptr->details->req_node_bitmap, avail_bitmap))) { *err_code = -730; *err_msg = "Job's required nodes not available"; error("wiki: job %u not runnable on hosts=%s", jobid, node_list); FREE_NULL_BITMAP(avail_bitmap); return NULL; } min_nodes = MAX(job_ptr->details->min_nodes, part_ptr->min_nodes); if (job_ptr->details->max_nodes == 0) max_nodes = part_ptr->max_nodes; else max_nodes = MIN(job_ptr->details->max_nodes, part_ptr->max_nodes); max_nodes = MIN(max_nodes, 500000); /* prevent overflows */ if (job_ptr->details->max_nodes) req_nodes = max_nodes; else req_nodes = min_nodes; if (min_nodes > max_nodes) { /* job's min_nodes exceeds partitions max_nodes */ *err_code = -730; *err_msg = "Job's min_nodes > max_nodes"; error("wiki: job %u not runnable on hosts=%s", jobid, node_list); FREE_NULL_BITMAP(avail_bitmap); return NULL; } preemptee_candidates = slurm_find_preemptable_jobs(job_ptr); orig_start_time = job_ptr->start_time; rc = select_g_job_test(job_ptr, avail_bitmap, min_nodes, max_nodes, req_nodes, SELECT_MODE_WILL_RUN, preemptee_candidates, NULL, exc_core_bitmap); if (preemptee_candidates) list_destroy(preemptee_candidates); if (rc == SLURM_SUCCESS) { char tmp_str[128]; *err_code = 0; uint32_t proc_cnt = 0; xstrcat(reply_msg, "STARTINFO="); #ifdef HAVE_BG select_g_select_jobinfo_get(job_ptr->select_jobinfo, SELECT_JOBDATA_NODE_CNT, &proc_cnt); #else proc_cnt = job_ptr->total_cpus; #endif snprintf(tmp_str, sizeof(tmp_str), "%u:%u@%u,", jobid, proc_cnt, (uint32_t) job_ptr->start_time); xstrcat(reply_msg, tmp_str); hostlist = bitmap2node_name(avail_bitmap); xstrcat(reply_msg, hostlist); xfree(hostlist); } else { xstrcat(reply_msg, "Jobs not runable on selected nodes"); error("wiki: jobs not runnable on nodes"); } /* Restore pending job's expected start time */ job_ptr->start_time = orig_start_time; FREE_NULL_BITMAP(avail_bitmap); return reply_msg; }
/* * slurm_sprint_job_info - output information about a specific Slurm * job based upon message as loaded using slurm_load_jobs * IN job_ptr - an individual job information record pointer * IN one_liner - print as a single line if true * RET out - char * containing formatted output (must be freed after call) * NULL is returned on failure. */ extern char * slurm_sprint_job_info ( job_info_t * job_ptr, int one_liner ) { int i, j; char time_str[32], *group_name, *user_name; char tmp1[128], tmp2[128], tmp3[128], tmp4[128], tmp5[128], *tmp6_ptr; char tmp_line[512]; char *ionodes = NULL; uint16_t exit_status = 0, term_sig = 0; job_resources_t *job_resrcs = job_ptr->job_resrcs; char *out = NULL; time_t run_time; uint32_t min_nodes, max_nodes = 0; char *nodelist = "NodeList"; bitstr_t *core_bitmap; char *host; int sock_inx, sock_reps, last; int abs_node_inx, rel_node_inx; int bit_inx, bit_reps; uint32_t *last_mem_alloc_ptr = NULL; uint32_t last_mem_alloc = NO_VAL; char *last_hosts; hostlist_t hl, hl_last; char select_buf[122]; uint32_t cluster_flags = slurmdb_setup_cluster_flags(); if (cluster_flags & CLUSTER_FLAG_BG) { nodelist = "MidplaneList"; select_g_select_jobinfo_get(job_ptr->select_jobinfo, SELECT_JOBDATA_IONODES, &ionodes); } /****** Line 1 ******/ snprintf(tmp_line, sizeof(tmp_line), "JobId=%u ", job_ptr->job_id); out = xstrdup(tmp_line); if (job_ptr->array_job_id) { snprintf(tmp_line, sizeof(tmp_line), "ArrayJobId=%u ArrayTaskId=%u ", job_ptr->array_job_id, job_ptr->array_task_id); xstrcat(out, tmp_line); } snprintf(tmp_line, sizeof(tmp_line), "Name=%s", job_ptr->name); xstrcat(out, tmp_line); if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); /****** Line 2 ******/ user_name = uid_to_string((uid_t) job_ptr->user_id); group_name = gid_to_string((gid_t) job_ptr->group_id); snprintf(tmp_line, sizeof(tmp_line), "UserId=%s(%u) GroupId=%s(%u)", user_name, job_ptr->user_id, group_name, job_ptr->group_id); xfree(user_name); xfree(group_name); xstrcat(out, tmp_line); if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); /****** Line 3 ******/ snprintf(tmp_line, sizeof(tmp_line), "Priority=%u Account=%s QOS=%s", job_ptr->priority, job_ptr->account, job_ptr->qos); xstrcat(out, tmp_line); if (slurm_get_track_wckey()) { snprintf(tmp_line, sizeof(tmp_line), " WCKey=%s", job_ptr->wckey); xstrcat(out, tmp_line); } if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); /****** Line 4 ******/ if (job_ptr->state_desc) { /* Replace white space with underscore for easier parsing */ for (j=0; job_ptr->state_desc[j]; j++) { if (isspace((int)job_ptr->state_desc[j])) job_ptr->state_desc[j] = '_'; } tmp6_ptr = job_ptr->state_desc; } else tmp6_ptr = job_reason_string(job_ptr->state_reason); snprintf(tmp_line, sizeof(tmp_line), "JobState=%s Reason=%s Dependency=%s", job_state_string(job_ptr->job_state), tmp6_ptr, job_ptr->dependency); xstrcat(out, tmp_line); if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); /****** Line 5 ******/ snprintf(tmp_line, sizeof(tmp_line), "Requeue=%u Restarts=%u BatchFlag=%u ", job_ptr->requeue, job_ptr->restart_cnt, job_ptr->batch_flag); xstrcat(out, tmp_line); if (WIFSIGNALED(job_ptr->exit_code)) term_sig = WTERMSIG(job_ptr->exit_code); exit_status = WEXITSTATUS(job_ptr->exit_code); snprintf(tmp_line, sizeof(tmp_line), "ExitCode=%u:%u", exit_status, term_sig); xstrcat(out, tmp_line); if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); /****** Line 5a (optional) ******/ if (!(job_ptr->show_flags & SHOW_DETAIL)) goto line6; if (WIFSIGNALED(job_ptr->derived_ec)) term_sig = WTERMSIG(job_ptr->derived_ec); else term_sig = 0; exit_status = WEXITSTATUS(job_ptr->derived_ec); snprintf(tmp_line, sizeof(tmp_line), "DerivedExitCode=%u:%u", exit_status, term_sig); xstrcat(out, tmp_line); if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); /****** Line 6 ******/ line6: snprintf(tmp_line, sizeof(tmp_line), "RunTime="); xstrcat(out, tmp_line); if (IS_JOB_PENDING(job_ptr)) run_time = 0; else if (IS_JOB_SUSPENDED(job_ptr)) run_time = job_ptr->pre_sus_time; else { time_t end_time; if (IS_JOB_RUNNING(job_ptr) || (job_ptr->end_time == 0)) end_time = time(NULL); else end_time = job_ptr->end_time; if (job_ptr->suspend_time) { run_time = (time_t) (difftime(end_time, job_ptr->suspend_time) + job_ptr->pre_sus_time); } else run_time = (time_t) difftime(end_time, job_ptr->start_time); } secs2time_str(run_time, tmp1, sizeof(tmp1)); sprintf(tmp_line, "%s ", tmp1); xstrcat(out, tmp_line); snprintf(tmp_line, sizeof(tmp_line), "TimeLimit="); xstrcat(out, tmp_line); if (job_ptr->time_limit == NO_VAL) sprintf(tmp_line, "Partition_Limit"); else { mins2time_str(job_ptr->time_limit, tmp_line, sizeof(tmp_line)); } xstrcat(out, tmp_line); snprintf(tmp_line, sizeof(tmp_line), " TimeMin="); xstrcat(out, tmp_line); if (job_ptr->time_min == 0) sprintf(tmp_line, "N/A"); else { mins2time_str(job_ptr->time_min, tmp_line, sizeof(tmp_line)); } xstrcat(out, tmp_line); if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); /****** Line 7 ******/ slurm_make_time_str((time_t *)&job_ptr->submit_time, time_str, sizeof(time_str)); snprintf(tmp_line, sizeof(tmp_line), "SubmitTime=%s ", time_str); xstrcat(out, tmp_line); slurm_make_time_str((time_t *)&job_ptr->eligible_time, time_str, sizeof(time_str)); snprintf(tmp_line, sizeof(tmp_line), "EligibleTime=%s", time_str); xstrcat(out, tmp_line); if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); /****** Line 8 (optional) ******/ if (job_ptr->resize_time) { slurm_make_time_str((time_t *)&job_ptr->resize_time, time_str, sizeof(time_str)); snprintf(tmp_line, sizeof(tmp_line), "ResizeTime=%s", time_str); xstrcat(out, tmp_line); if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); } /****** Line 9 ******/ slurm_make_time_str((time_t *)&job_ptr->start_time, time_str, sizeof(time_str)); snprintf(tmp_line, sizeof(tmp_line), "StartTime=%s ", time_str); xstrcat(out, tmp_line); snprintf(tmp_line, sizeof(tmp_line), "EndTime="); xstrcat(out, tmp_line); if ((job_ptr->time_limit == INFINITE) && (job_ptr->end_time > time(NULL))) sprintf(tmp_line, "Unknown"); else { slurm_make_time_str ((time_t *)&job_ptr->end_time, time_str, sizeof(time_str)); sprintf(tmp_line, "%s", time_str); } xstrcat(out, tmp_line); if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); /****** Line 10 ******/ if (job_ptr->preempt_time == 0) sprintf(tmp_line, "PreemptTime=None "); else { slurm_make_time_str((time_t *)&job_ptr->preempt_time, time_str, sizeof(time_str)); snprintf(tmp_line, sizeof(tmp_line), "PreemptTime=%s ", time_str); } xstrcat(out, tmp_line); if (job_ptr->suspend_time) { slurm_make_time_str ((time_t *)&job_ptr->suspend_time, time_str, sizeof(time_str)); } else { strncpy(time_str, "None", sizeof(time_str)); } snprintf(tmp_line, sizeof(tmp_line), "SuspendTime=%s SecsPreSuspend=%ld", time_str, (long int)job_ptr->pre_sus_time); xstrcat(out, tmp_line); if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); /****** Line 11 ******/ snprintf(tmp_line, sizeof(tmp_line), "Partition=%s AllocNode:Sid=%s:%u", job_ptr->partition, job_ptr->alloc_node, job_ptr->alloc_sid); xstrcat(out, tmp_line); if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); /****** Line 12 ******/ snprintf(tmp_line, sizeof(tmp_line), "Req%s=%s Exc%s=%s", nodelist, job_ptr->req_nodes, nodelist, job_ptr->exc_nodes); xstrcat(out, tmp_line); if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); /****** Line 13 ******/ xstrfmtcat(out, "%s=", nodelist); xstrcat(out, job_ptr->nodes); if (job_ptr->nodes && ionodes) { snprintf(tmp_line, sizeof(tmp_line), "[%s]", ionodes); xstrcat(out, tmp_line); xfree(ionodes); } if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); /****** Line 14 (optional) ******/ if (job_ptr->batch_host) { snprintf(tmp_line, sizeof(tmp_line), "BatchHost=%s", job_ptr->batch_host); xstrcat(out, tmp_line); if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); } /****** Line 15 ******/ if (cluster_flags & CLUSTER_FLAG_BG) { select_g_select_jobinfo_get(job_ptr->select_jobinfo, SELECT_JOBDATA_NODE_CNT, &min_nodes); if ((min_nodes == 0) || (min_nodes == NO_VAL)) { min_nodes = job_ptr->num_nodes; max_nodes = job_ptr->max_nodes; } else if (job_ptr->max_nodes) max_nodes = min_nodes; } else { min_nodes = job_ptr->num_nodes; max_nodes = job_ptr->max_nodes; } _sprint_range(tmp1, sizeof(tmp1), job_ptr->num_cpus, job_ptr->max_cpus); _sprint_range(tmp2, sizeof(tmp2), min_nodes, max_nodes); if (job_ptr->sockets_per_node == (uint16_t) NO_VAL) strcpy(tmp3, "*"); else snprintf(tmp3, sizeof(tmp3), "%u", job_ptr->sockets_per_node); if (job_ptr->cores_per_socket == (uint16_t) NO_VAL) strcpy(tmp4, "*"); else snprintf(tmp4, sizeof(tmp4), "%u", job_ptr->cores_per_socket); if (job_ptr->threads_per_core == (uint16_t) NO_VAL) strcpy(tmp5, "*"); else snprintf(tmp5, sizeof(tmp5), "%u", job_ptr->threads_per_core); snprintf(tmp_line, sizeof(tmp_line), "NumNodes=%s NumCPUs=%s CPUs/Task=%u ReqS:C:T=%s:%s:%s", tmp2, tmp1, job_ptr->cpus_per_task, tmp3, tmp4, tmp5); xstrcat(out, tmp_line); if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); if (!job_resrcs) goto line15; if (cluster_flags & CLUSTER_FLAG_BG) { if ((job_resrcs->cpu_array_cnt > 0) && (job_resrcs->cpu_array_value) && (job_resrcs->cpu_array_reps)) { int length = 0; xstrcat(out, "CPUs="); length += 10; for (i = 0; i < job_resrcs->cpu_array_cnt; i++) { if (length > 70) { /* skip to last CPU group entry */ if (i < job_resrcs->cpu_array_cnt - 1) { continue; } /* add ellipsis before last entry */ xstrcat(out, "...,"); length += 4; } snprintf(tmp_line, sizeof(tmp_line), "%d", job_resrcs->cpus[i]); xstrcat(out, tmp_line); length += strlen(tmp_line); if (job_resrcs->cpu_array_reps[i] > 1) { snprintf(tmp_line, sizeof(tmp_line), "*%d", job_resrcs->cpu_array_reps[i]); xstrcat(out, tmp_line); length += strlen(tmp_line); } if (i < job_resrcs->cpu_array_cnt - 1) { xstrcat(out, ","); length++; } } if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); } } else { if (!job_resrcs->core_bitmap) goto line15; last = bit_fls(job_resrcs->core_bitmap); if (last == -1) goto line15; hl = hostlist_create(job_ptr->nodes); if (!hl) { error("slurm_sprint_job_info: hostlist_create: %s", job_ptr->nodes); return NULL; } hl_last = hostlist_create(NULL); if (!hl_last) { error("slurm_sprint_job_info: hostlist_create: NULL"); hostlist_destroy(hl); return NULL; } bit_inx = 0; i = sock_inx = sock_reps = 0; abs_node_inx = job_ptr->node_inx[i]; /* tmp1[] stores the current cpu(s) allocated */ tmp2[0] = '\0'; /* stores last cpu(s) allocated */ for (rel_node_inx=0; rel_node_inx < job_resrcs->nhosts; rel_node_inx++) { if (sock_reps >= job_resrcs->sock_core_rep_count[sock_inx]) { sock_inx++; sock_reps = 0; } sock_reps++; bit_reps = job_resrcs->sockets_per_node[sock_inx] * job_resrcs->cores_per_socket[sock_inx]; core_bitmap = bit_alloc(bit_reps); for (j=0; j < bit_reps; j++) { if (bit_test(job_resrcs->core_bitmap, bit_inx)) bit_set(core_bitmap, j); bit_inx++; } bit_fmt(tmp1, sizeof(tmp1), core_bitmap); FREE_NULL_BITMAP(core_bitmap); host = hostlist_shift(hl); /* * If the allocation values for this host are not the same as the * last host, print the report of the last group of hosts that had * identical allocation values. */ if (strcmp(tmp1, tmp2) || (last_mem_alloc_ptr != job_resrcs->memory_allocated) || (job_resrcs->memory_allocated && (last_mem_alloc != job_resrcs->memory_allocated[rel_node_inx]))) { if (hostlist_count(hl_last)) { last_hosts = hostlist_ranged_string_xmalloc( hl_last); snprintf(tmp_line, sizeof(tmp_line), " Nodes=%s CPU_IDs=%s Mem=%u", last_hosts, tmp2, last_mem_alloc_ptr ? last_mem_alloc : 0); xfree(last_hosts); xstrcat(out, tmp_line); if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); hostlist_destroy(hl_last); hl_last = hostlist_create(NULL); } strcpy(tmp2, tmp1); last_mem_alloc_ptr = job_resrcs->memory_allocated; if (last_mem_alloc_ptr) last_mem_alloc = job_resrcs-> memory_allocated[rel_node_inx]; else last_mem_alloc = NO_VAL; } hostlist_push_host(hl_last, host); free(host); if (bit_inx > last) break; if (abs_node_inx > job_ptr->node_inx[i+1]) { i += 2; abs_node_inx = job_ptr->node_inx[i]; } else { abs_node_inx++; } } if (hostlist_count(hl_last)) { last_hosts = hostlist_ranged_string_xmalloc(hl_last); snprintf(tmp_line, sizeof(tmp_line), " Nodes=%s CPU_IDs=%s Mem=%u", last_hosts, tmp2, last_mem_alloc_ptr ? last_mem_alloc : 0); xfree(last_hosts); xstrcat(out, tmp_line); if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); } hostlist_destroy(hl); hostlist_destroy(hl_last); } /****** Line 15 ******/ line15: if (job_ptr->pn_min_memory & MEM_PER_CPU) { job_ptr->pn_min_memory &= (~MEM_PER_CPU); tmp6_ptr = "CPU"; } else tmp6_ptr = "Node"; if (cluster_flags & CLUSTER_FLAG_BG) { convert_num_unit((float)job_ptr->pn_min_cpus, tmp1, sizeof(tmp1), UNIT_NONE); snprintf(tmp_line, sizeof(tmp_line), "MinCPUsNode=%s", tmp1); } else { snprintf(tmp_line, sizeof(tmp_line), "MinCPUsNode=%u", job_ptr->pn_min_cpus); } xstrcat(out, tmp_line); convert_num_unit((float)job_ptr->pn_min_memory, tmp1, sizeof(tmp1), UNIT_MEGA); convert_num_unit((float)job_ptr->pn_min_tmp_disk, tmp2, sizeof(tmp2), UNIT_MEGA); snprintf(tmp_line, sizeof(tmp_line), " MinMemory%s=%s MinTmpDiskNode=%s", tmp6_ptr, tmp1, tmp2); xstrcat(out, tmp_line); if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); /****** Line 16 ******/ snprintf(tmp_line, sizeof(tmp_line), "Features=%s Gres=%s Reservation=%s", job_ptr->features, job_ptr->gres, job_ptr->resv_name); xstrcat(out, tmp_line); if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); /****** Line 17 ******/ snprintf(tmp_line, sizeof(tmp_line), "Shared=%s Contiguous=%d Licenses=%s Network=%s", (job_ptr->shared == 0 ? "0" : job_ptr->shared == 1 ? "1" : "OK"), job_ptr->contiguous, job_ptr->licenses, job_ptr->network); xstrcat(out, tmp_line); if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); /****** Line 18 ******/ snprintf(tmp_line, sizeof(tmp_line), "Command=%s", job_ptr->command); xstrcat(out, tmp_line); if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); /****** Line 19 ******/ snprintf(tmp_line, sizeof(tmp_line), "WorkDir=%s", job_ptr->work_dir); xstrcat(out, tmp_line); if (cluster_flags & CLUSTER_FLAG_BG) { /****** Line 20 (optional) ******/ select_g_select_jobinfo_sprint(job_ptr->select_jobinfo, select_buf, sizeof(select_buf), SELECT_PRINT_BG_ID); if (select_buf[0] != '\0') { if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); snprintf(tmp_line, sizeof(tmp_line), "Block_ID=%s", select_buf); xstrcat(out, tmp_line); } /****** Line 21 (optional) ******/ select_g_select_jobinfo_sprint(job_ptr->select_jobinfo, select_buf, sizeof(select_buf), SELECT_PRINT_MIXED_SHORT); if (select_buf[0] != '\0') { if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); xstrcat(out, select_buf); } if (cluster_flags & CLUSTER_FLAG_BGL) { /****** Line 22 (optional) ******/ select_g_select_jobinfo_sprint( job_ptr->select_jobinfo, select_buf, sizeof(select_buf), SELECT_PRINT_BLRTS_IMAGE); if (select_buf[0] != '\0') { if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); snprintf(tmp_line, sizeof(tmp_line), "BlrtsImage=%s", select_buf); xstrcat(out, tmp_line); } } /****** Line 23 (optional) ******/ select_g_select_jobinfo_sprint(job_ptr->select_jobinfo, select_buf, sizeof(select_buf), SELECT_PRINT_LINUX_IMAGE); if (select_buf[0] != '\0') { if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); if (cluster_flags & CLUSTER_FLAG_BGL) snprintf(tmp_line, sizeof(tmp_line), "LinuxImage=%s", select_buf); else snprintf(tmp_line, sizeof(tmp_line), "CnloadImage=%s", select_buf); xstrcat(out, tmp_line); } /****** Line 24 (optional) ******/ select_g_select_jobinfo_sprint(job_ptr->select_jobinfo, select_buf, sizeof(select_buf), SELECT_PRINT_MLOADER_IMAGE); if (select_buf[0] != '\0') { if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); snprintf(tmp_line, sizeof(tmp_line), "MloaderImage=%s", select_buf); xstrcat(out, tmp_line); } /****** Line 25 (optional) ******/ select_g_select_jobinfo_sprint(job_ptr->select_jobinfo, select_buf, sizeof(select_buf), SELECT_PRINT_RAMDISK_IMAGE); if (select_buf[0] != '\0') { if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); if (cluster_flags & CLUSTER_FLAG_BGL) snprintf(tmp_line, sizeof(tmp_line), "RamDiskImage=%s", select_buf); else snprintf(tmp_line, sizeof(tmp_line), "IoloadImage=%s", select_buf); xstrcat(out, tmp_line); } } /****** Line 26 (optional) ******/ if (job_ptr->comment) { if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); snprintf(tmp_line, sizeof(tmp_line), "Comment=%s ", job_ptr->comment); xstrcat(out, tmp_line); } /****** Line 27 (optional) ******/ if (job_ptr->batch_script) { if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); xstrcat(out, "BatchScript=\n"); xstrcat(out, job_ptr->batch_script); } /****** Line 28 (optional) ******/ if (job_ptr->req_switch) { char time_buf[32]; if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); secs2time_str((time_t) job_ptr->wait4switch, time_buf, sizeof(time_buf)); snprintf(tmp_line, sizeof(tmp_line), "Switches=%u@%s\n", job_ptr->req_switch, time_buf); xstrcat(out, tmp_line); } /****** Line 29 (optional) ******/ if (one_liner) xstrcat(out, "\n"); else xstrcat(out, "\n\n"); return out; }
extern int as_mysql_step_start(mysql_conn_t *mysql_conn, struct step_record *step_ptr) { int cpus = 0, tasks = 0, nodes = 0, task_dist = 0; int rc=SLURM_SUCCESS; char node_list[BUFFER_SIZE]; char *node_inx = NULL, *step_name = NULL; time_t start_time, submit_time; char *query = NULL; if (!step_ptr->job_ptr->db_index && ((!step_ptr->job_ptr->details || !step_ptr->job_ptr->details->submit_time) && !step_ptr->job_ptr->resize_time)) { error("as_mysql_step_start: " "Not inputing this job, it has no submit time."); return SLURM_ERROR; } if (step_ptr->job_ptr->resize_time) { submit_time = start_time = step_ptr->job_ptr->resize_time; if (step_ptr->start_time > submit_time) start_time = step_ptr->start_time; } else { start_time = step_ptr->start_time; submit_time = step_ptr->job_ptr->details->submit_time; } if (check_connection(mysql_conn) != SLURM_SUCCESS) return ESLURM_DB_CONNECTION; if (slurmdbd_conf) { cpus = step_ptr->cpu_count; if (step_ptr->job_ptr->details) tasks = step_ptr->job_ptr->details->num_tasks; else tasks = cpus; snprintf(node_list, BUFFER_SIZE, "%s", step_ptr->job_ptr->nodes); nodes = step_ptr->step_layout->node_cnt; task_dist = step_ptr->step_layout->task_dist; node_inx = step_ptr->network; } else if (step_ptr->step_id == SLURM_BATCH_SCRIPT) { char temp_bit[BUF_SIZE]; if (step_ptr->step_node_bitmap) { node_inx = bit_fmt(temp_bit, sizeof(temp_bit), step_ptr->step_node_bitmap); } /* We overload gres with the node name of where the script was running. */ snprintf(node_list, BUFFER_SIZE, "%s", step_ptr->gres); nodes = cpus = tasks = 1; } else { char *ionodes = NULL, *temp_nodes = NULL; char temp_bit[BUF_SIZE]; if (step_ptr->step_node_bitmap) { node_inx = bit_fmt(temp_bit, sizeof(temp_bit), step_ptr->step_node_bitmap); } #ifdef HAVE_BG_L_P /* Only L and P use this code */ if (step_ptr->job_ptr->details) tasks = cpus = step_ptr->job_ptr->details->min_cpus; else tasks = cpus = step_ptr->job_ptr->cpu_cnt; select_g_select_jobinfo_get(step_ptr->job_ptr->select_jobinfo, SELECT_JOBDATA_NODE_CNT, &nodes); temp_nodes = step_ptr->job_ptr->nodes; #else if (!step_ptr->step_layout || !step_ptr->step_layout->task_cnt) { tasks = cpus = step_ptr->job_ptr->total_cpus; nodes = step_ptr->job_ptr->total_nodes; temp_nodes = step_ptr->job_ptr->nodes; } else { cpus = step_ptr->cpu_count; tasks = step_ptr->step_layout->task_cnt; #ifdef HAVE_BGQ select_g_select_jobinfo_get(step_ptr->select_jobinfo, SELECT_JOBDATA_NODE_CNT, &nodes); #else nodes = step_ptr->step_layout->node_cnt; #endif task_dist = step_ptr->step_layout->task_dist; temp_nodes = step_ptr->step_layout->node_list; } #endif select_g_select_jobinfo_get(step_ptr->select_jobinfo, SELECT_JOBDATA_IONODES, &ionodes); if (ionodes) { snprintf(node_list, BUFFER_SIZE, "%s[%s]", temp_nodes, ionodes); xfree(ionodes); } else snprintf(node_list, BUFFER_SIZE, "%s", temp_nodes); } if (!step_ptr->job_ptr->db_index) { if (!(step_ptr->job_ptr->db_index = _get_db_index(mysql_conn, submit_time, step_ptr->job_ptr->job_id, step_ptr->job_ptr->assoc_id))) { /* If we get an error with this just fall * through to avoid an infinite loop */ if (as_mysql_job_start(mysql_conn, step_ptr->job_ptr) == SLURM_ERROR) { error("couldn't add job %u at step start", step_ptr->job_ptr->job_id); return SLURM_SUCCESS; } } } step_name = slurm_add_slash_to_quotes(step_ptr->name); /* we want to print a -1 for the requid so leave it a %d */ /* The stepid could be -2 so use %d not %u */ query = xstrdup_printf( "insert into \"%s_%s\" (job_db_inx, id_step, time_start, " "step_name, state, " "cpus_alloc, nodes_alloc, task_cnt, nodelist, " "node_inx, task_dist, req_cpufreq) " "values (%d, %d, %d, '%s', %d, %d, %d, %d, " "'%s', '%s', %d, %u) " "on duplicate key update cpus_alloc=%d, nodes_alloc=%d, " "task_cnt=%d, time_end=0, state=%d, " "nodelist='%s', node_inx='%s', task_dist=%d, req_cpufreq=%u", mysql_conn->cluster_name, step_table, step_ptr->job_ptr->db_index, step_ptr->step_id, (int)start_time, step_name, JOB_RUNNING, cpus, nodes, tasks, node_list, node_inx, task_dist, step_ptr->cpu_freq, cpus, nodes, tasks, JOB_RUNNING, node_list, node_inx, task_dist, step_ptr->cpu_freq); debug3("%d(%s:%d) query\n%s", mysql_conn->conn, THIS_FILE, __LINE__, query); rc = mysql_db_query(mysql_conn, query); xfree(query); xfree(step_name); return rc; }
/* * slurm_sprint_job_step_info - output information about a specific Slurm * job step based upon message as loaded using slurm_get_job_steps * IN job_ptr - an individual job step information record pointer * IN one_liner - print as a single line if true * RET out - char * containing formatted output (must be freed after call) * NULL is returned on failure. */ char * slurm_sprint_job_step_info ( job_step_info_t * job_step_ptr, int one_liner ) { char tmp_node_cnt[40]; char time_str[32]; char limit_str[32]; char tmp_line[128]; char *out = NULL; uint32_t cluster_flags = slurmdb_setup_cluster_flags(); /****** Line 1 ******/ slurm_make_time_str ((time_t *)&job_step_ptr->start_time, time_str, sizeof(time_str)); if (job_step_ptr->time_limit == INFINITE) sprintf(limit_str, "UNLIMITED"); else secs2time_str ((time_t)job_step_ptr->time_limit * 60, limit_str, sizeof(limit_str)); snprintf(tmp_line, sizeof(tmp_line), "StepId=%u.%u UserId=%u StartTime=%s TimeLimit=%s", job_step_ptr->job_id, job_step_ptr->step_id, job_step_ptr->user_id, time_str, limit_str); out = xstrdup(tmp_line); if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); /****** Line 2 ******/ snprintf(tmp_line, sizeof(tmp_line), "State=%s ", job_state_string(job_step_ptr->state)); xstrcat(out, tmp_line); if (cluster_flags & CLUSTER_FLAG_BG) { char *io_nodes; select_g_select_jobinfo_get(job_step_ptr->select_jobinfo, SELECT_JOBDATA_IONODES, &io_nodes); if (io_nodes) { snprintf(tmp_line, sizeof(tmp_line), "Partition=%s MidplaneList=%s[%s] Gres=%s", job_step_ptr->partition, job_step_ptr->nodes, io_nodes, job_step_ptr->gres); xfree(io_nodes); } else snprintf(tmp_line, sizeof(tmp_line), "Partition=%s MidplaneList=%s Gres=%s", job_step_ptr->partition, job_step_ptr->nodes, job_step_ptr->gres); } else { snprintf(tmp_line, sizeof(tmp_line), "Partition=%s NodeList=%s Gres=%s", job_step_ptr->partition, job_step_ptr->nodes, job_step_ptr->gres); } xstrcat(out, tmp_line); if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); /****** Line 3 ******/ if (cluster_flags & CLUSTER_FLAG_BGQ) { uint32_t nodes = 0; select_g_select_jobinfo_get(job_step_ptr->select_jobinfo, SELECT_JOBDATA_NODE_CNT, &nodes); convert_num_unit((float)nodes, tmp_node_cnt, sizeof(tmp_node_cnt), UNIT_NONE); } else { convert_num_unit((float)_nodes_in_list(job_step_ptr->nodes), tmp_node_cnt, sizeof(tmp_node_cnt), UNIT_NONE); } snprintf(tmp_line, sizeof(tmp_line), "Nodes=%s Tasks=%u Name=%s Network=%s", tmp_node_cnt, job_step_ptr->num_tasks, job_step_ptr->name, job_step_ptr->network); xstrcat(out, tmp_line); if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); /****** Line 4 ******/ snprintf(tmp_line, sizeof(tmp_line), "ResvPorts=%s Checkpoint=%u CheckpointDir=%s", job_step_ptr->resv_ports, job_step_ptr->ckpt_interval, job_step_ptr->ckpt_dir); xstrcat(out, tmp_line); if (one_liner) xstrcat(out, " "); else xstrcat(out, "\n "); /****** Line 5 ******/ if (job_step_ptr->cpu_freq == NO_VAL) { snprintf(tmp_line, sizeof(tmp_line), "CPUFreqReq=Default\n\n"); } else if (job_step_ptr->cpu_freq & CPU_FREQ_RANGE_FLAG) { switch (job_step_ptr->cpu_freq) { case CPU_FREQ_LOW : snprintf(tmp_line, sizeof(tmp_line), "CPUFreqReq=Low\n\n"); break; case CPU_FREQ_MEDIUM : snprintf(tmp_line, sizeof(tmp_line), "CPUFreqReq=Medium\n\n"); break; case CPU_FREQ_HIGH : snprintf(tmp_line, sizeof(tmp_line), "CPUFreqReq=High\n\n"); break; default : snprintf(tmp_line, sizeof(tmp_line), "CPUFreqReq=Unknown\n\n"); } } else { snprintf(tmp_line, sizeof(tmp_line), "CPUFreqReq=%u\n\n", job_step_ptr->cpu_freq); } xstrcat(out, tmp_line); return out; }
/* * Allocate nodes for heterogeneous/pack job from the slurm controller -- * retrying the attempt if the controller appears to be down, and optionally * waiting for resources if none are currently available (see opt.immediate) * * Returns a pointer to a resource_allocation_response_msg which must * be freed with slurm_free_resource_allocation_response_msg() */ List allocate_pack_nodes(bool handle_signals) { resource_allocation_response_msg_t *resp = NULL; bool jobid_log = true; job_desc_msg_t *j, *first_job = NULL; slurm_allocation_callbacks_t callbacks; ListIterator opt_iter, resp_iter; slurm_opt_t *opt_local, *first_opt = NULL; List job_req_list = NULL, job_resp_list = NULL; uint32_t my_job_id = 0; int i, k; job_req_list = list_create(NULL); opt_iter = list_iterator_create(opt_list); while ((opt_local = list_next(opt_iter))) { srun_opt_t *srun_opt = opt_local->srun_opt; xassert(srun_opt); if (!first_opt) first_opt = opt_local; if (srun_opt->relative_set && srun_opt->relative) fatal("--relative option invalid for job allocation request"); if ((j = _job_desc_msg_create_from_opts(opt_local)) == NULL) return NULL; if (!first_job) first_job = j; j->origin_cluster = xstrdup(slurmctld_conf.cluster_name); /* Do not re-use existing job id when submitting new job * from within a running job */ if ((j->job_id != NO_VAL) && !opt_local->jobid_set) { if (jobid_log) { jobid_log = false; /* log once */ info("WARNING: Creating SLURM job allocation from within " "another allocation"); info("WARNING: You are attempting to initiate a second job"); } if (!opt_local->jobid_set) /* Let slurmctld set jobid */ j->job_id = NO_VAL; } list_append(job_req_list, j); } list_iterator_destroy(opt_iter); if (!first_job) { error("%s: No job requests found", __func__); return NULL; } if (first_opt && first_opt->clusters && (slurmdb_get_first_pack_cluster(job_req_list, first_opt->clusters, &working_cluster_rec) != SLURM_SUCCESS)) { print_db_notok(first_opt->clusters, 0); return NULL; } callbacks.ping = _ping_handler; callbacks.timeout = _timeout_handler; callbacks.job_complete = _job_complete_handler; callbacks.job_suspend = NULL; callbacks.user_msg = _user_msg_handler; callbacks.node_fail = _node_fail_handler; /* create message thread to handle pings and such from slurmctld */ msg_thr = slurm_allocation_msg_thr_create(&first_job->other_port, &callbacks); /* NOTE: Do not process signals in separate pthread. The signal will * cause slurm_allocate_resources_blocking() to exit immediately. */ if (handle_signals) { xsignal_unblock(sig_array); for (i = 0; sig_array[i]; i++) xsignal(sig_array[i], _signal_while_allocating); } while (first_opt && !job_resp_list) { job_resp_list = slurm_allocate_pack_job_blocking(job_req_list, first_opt->immediate, _set_pending_job_id); if (destroy_job) { /* cancelled by signal */ break; } else if (!job_resp_list && !_retry()) { break; } } if (job_resp_list && !destroy_job) { /* * Allocation granted! */ opt_iter = list_iterator_create(opt_list); resp_iter = list_iterator_create(job_resp_list); while ((opt_local = list_next(opt_iter))) { resp = (resource_allocation_response_msg_t *) list_next(resp_iter); if (!resp) break; if (pending_job_id == 0) pending_job_id = resp->job_id; if (my_job_id == 0) { my_job_id = resp->job_id; i = list_count(opt_list); k = list_count(job_resp_list); if (i != k) { error("%s: request count != response count (%d != %d)", __func__, i, k); goto relinquish; } } /* * These values could be changed while the job was * pending so overwrite the request with what was * allocated so we don't have issues when we use them * in the step creation. * * NOTE: pn_min_memory here is an int64, not uint64. * These operations may have some bizarre side effects */ if (opt_local->pn_min_memory != NO_VAL64) opt_local->pn_min_memory = (resp->pn_min_memory & (~MEM_PER_CPU)); else if (opt_local->mem_per_cpu != NO_VAL64) opt_local->mem_per_cpu = (resp->pn_min_memory & (~MEM_PER_CPU)); #ifdef HAVE_BG uint32_t node_cnt = 0; select_g_select_jobinfo_get(resp->select_jobinfo, SELECT_JOBDATA_NODE_CNT, &node_cnt); if ((node_cnt == 0) || (node_cnt == NO_VAL)) { opt_local->min_nodes = node_cnt; opt_local->max_nodes = node_cnt; } /* else we just use the original request */ if (!_wait_bluegene_block_ready(resp)) { if (!destroy_job) error("Something is wrong with the " "boot of the block."); goto relinquish; } #else opt_local->min_nodes = resp->node_cnt; opt_local->max_nodes = resp->node_cnt; if (resp->working_cluster_rec) slurm_setup_remote_working_cluster(resp); if (!_wait_nodes_ready(resp)) { if (!destroy_job) error("Something is wrong with the " "boot of the nodes."); goto relinquish; } #endif } list_iterator_destroy(resp_iter); list_iterator_destroy(opt_iter); } else if (destroy_job) { goto relinquish; } if (handle_signals) xsignal_block(sig_array); return job_resp_list; relinquish: if (job_resp_list) { if (!destroy_job && my_job_id) slurm_complete_job(my_job_id, 1); list_destroy(job_resp_list); } exit(error_exit); return NULL; }
/* * Set in "dest" the environment variables relevant to a SLURM job * allocation, overwriting any environment variables of the same name. * If the address pointed to by "dest" is NULL, memory will automatically be * xmalloc'ed. The array is terminated by a NULL pointer, and thus is * suitable for use by execle() and other env_array_* functions. * * Sets the variables: * SLURM_JOB_ID * SLURM_JOB_NUM_NODES * SLURM_JOB_NODELIST * SLURM_JOB_CPUS_PER_NODE * LOADLBATCH (AIX only) * SLURM_BG_NUM_NODES, MPIRUN_PARTITION, MPIRUN_NOFREE, and * MPIRUN_NOALLOCATE (BG only) * * Sets OBSOLETE variables (needed for MPI, do not remove): * SLURM_JOBID * SLURM_NNODES * SLURM_NODELIST * SLURM_TASKS_PER_NODE */ int env_array_for_job(char ***dest, const resource_allocation_response_msg_t *alloc, const job_desc_msg_t *desc) { char *tmp = NULL; char *dist = NULL, *lllp_dist = NULL; slurm_step_layout_t *step_layout = NULL; uint32_t num_tasks = desc->num_tasks; int rc = SLURM_SUCCESS; uint32_t node_cnt = alloc->node_cnt; uint32_t cluster_flags = slurmdb_setup_cluster_flags(); _setup_particulars(cluster_flags, dest, alloc->select_jobinfo); if (cluster_flags & CLUSTER_FLAG_BG) { select_g_select_jobinfo_get(alloc->select_jobinfo, SELECT_JOBDATA_NODE_CNT, &node_cnt); if (!node_cnt) node_cnt = alloc->node_cnt; env_array_overwrite_fmt(dest, "SLURM_BG_NUM_NODES", "%u", node_cnt); } env_array_overwrite_fmt(dest, "SLURM_JOB_ID", "%u", alloc->job_id); env_array_overwrite_fmt(dest, "SLURM_JOB_NUM_NODES", "%u", node_cnt); env_array_overwrite_fmt(dest, "SLURM_JOB_NODELIST", "%s", alloc->node_list); _set_distribution(desc->task_dist, &dist, &lllp_dist); if(dist) env_array_overwrite_fmt(dest, "SLURM_DISTRIBUTION", "%s", dist); if(desc->task_dist == SLURM_DIST_PLANE) env_array_overwrite_fmt(dest, "SLURM_DIST_PLANESIZE", "%u", desc->plane_size); if(lllp_dist) env_array_overwrite_fmt(dest, "SLURM_DIST_LLLP", "%s", lllp_dist); tmp = uint32_compressed_to_str(alloc->num_cpu_groups, alloc->cpus_per_node, alloc->cpu_count_reps); env_array_overwrite_fmt(dest, "SLURM_JOB_CPUS_PER_NODE", "%s", tmp); xfree(tmp); /* OBSOLETE, but needed by MPI, do not remove */ env_array_overwrite_fmt(dest, "SLURM_JOBID", "%u", alloc->job_id); env_array_overwrite_fmt(dest, "SLURM_NNODES", "%u", node_cnt); env_array_overwrite_fmt(dest, "SLURM_NODELIST", "%s", alloc->node_list); if(num_tasks == NO_VAL) { /* If we know how many tasks we are going to do then we set SLURM_TASKS_PER_NODE */ int i=0; /* If no tasks were given we can figure it out here * by totalling up the cpus and then dividing by the * number of cpus per task */ num_tasks = 0; for (i = 0; i < alloc->num_cpu_groups; i++) { num_tasks += alloc->cpu_count_reps[i] * alloc->cpus_per_node[i]; } if((int)desc->cpus_per_task > 1 && desc->cpus_per_task != (uint16_t)NO_VAL) num_tasks /= desc->cpus_per_task; //num_tasks = desc->min_cpus; } if(desc->task_dist == SLURM_DIST_ARBITRARY) { tmp = desc->req_nodes; env_array_overwrite_fmt(dest, "SLURM_ARBITRARY_NODELIST", "%s", tmp); } else tmp = alloc->node_list; if(!(step_layout = slurm_step_layout_create(tmp, alloc->cpus_per_node, alloc->cpu_count_reps, node_cnt, num_tasks, desc->cpus_per_task, desc->task_dist, desc->plane_size))) return SLURM_ERROR; tmp = _uint16_array_to_str(step_layout->node_cnt, step_layout->tasks); slurm_step_layout_destroy(step_layout); env_array_overwrite_fmt(dest, "SLURM_TASKS_PER_NODE", "%s", tmp); xfree(tmp); return rc; }
/* * js_pg_job_start - load into the storage the start of a job * * IN pg_conn: database connection * IN cluster_name: cluster of the job * IN job_ptr: job just started * RET: error code */ extern int js_pg_job_start(pgsql_conn_t *pg_conn, struct job_record *job_ptr) { int rc=SLURM_SUCCESS, track_steps = 0, reinit = 0; char *jname = NULL, *nodes = NULL, *node_inx = NULL; char *block_id = NULL, *rec = NULL, *query = NULL; time_t begin_time, check_time, start_time, submit_time; int job_state, node_cnt = 0; uint32_t wckeyid = 0; if ((!job_ptr->details || !job_ptr->details->submit_time) && !job_ptr->resize_time) { error("as/pg: job_start: Not inputing this job, " "it has no submit time."); return SLURM_ERROR; } if (check_db_connection(pg_conn) != SLURM_SUCCESS) return ESLURM_DB_CONNECTION; if (! cluster_in_db(pg_conn, pg_conn->cluster_name) ) { error("cluster %s not in db", pg_conn->cluster_name); return SLURM_ERROR; } debug3("as/pg: job_start() called"); job_state = job_ptr->job_state; /* Since we need a new db_inx make sure the old db_inx * removed. This is most likely the only time we are going to * be notified of the change also so make the state without * the resize. */ if (IS_JOB_RESIZING(job_ptr)) { /* If we have a db_index lets end the previous record. */ if (job_ptr->db_index) js_pg_job_complete(pg_conn, job_ptr); else error("We don't have a db_index for job %u, " "this should never happen.", job_ptr->job_id); job_state &= (~JOB_RESIZING); job_ptr->db_index = 0; } job_state &= JOB_STATE_BASE; if (job_ptr->resize_time) { begin_time = job_ptr->resize_time; submit_time = job_ptr->resize_time; start_time = job_ptr->resize_time; } else { begin_time = job_ptr->details->begin_time; submit_time = job_ptr->details->submit_time; start_time = job_ptr->start_time; } /* See what we are hearing about here if no start time. If * this job latest time is before the last roll up we will * need to reset it to look at this job. */ if (start_time) check_time = start_time; else if (begin_time) check_time = begin_time; else check_time = submit_time; slurm_mutex_lock(&usage_rollup_lock); if (check_time < global_last_rollup) { PGresult *result = NULL; /* check to see if we are hearing about this time for the * first time. */ query = xstrdup_printf( "SELECT job_db_inx FROM %s.%s WHERE id_job=%u AND " "time_submit=%ld AND time_eligible=%ld AND time_start=%ld", pg_conn->cluster_name, job_table, job_ptr->job_id, submit_time, begin_time, start_time); result = DEF_QUERY_RET; if (!result) { slurm_mutex_unlock(&usage_rollup_lock); return SLURM_ERROR; } if (PQntuples(result) != 0) { PQclear(result); debug4("revieved an update for a " "job (%u) already known about", job_ptr->job_id); slurm_mutex_unlock(&usage_rollup_lock); goto no_rollup_change; } PQclear(result); if (job_ptr->start_time) debug("Need to reroll usage from %s Job %u " "from %s started then and we are just " "now hearing about it.", ctime(&check_time), job_ptr->job_id, pg_conn->cluster_name); else if (begin_time) debug("Need to reroll usage from %s Job %u " "from %s became eligible then and we are just " "now hearing about it.", ctime(&check_time), job_ptr->job_id, pg_conn->cluster_name); else debug("Need to reroll usage from %s Job %u " "from %s was submitted then and we are just " "now hearing about it.", ctime(&check_time), job_ptr->job_id, pg_conn->cluster_name); global_last_rollup = check_time; slurm_mutex_unlock(&usage_rollup_lock); query = xstrdup_printf("UPDATE %s.%s SET hourly_rollup=%ld, " "daily_rollup=%ld, monthly_rollup=%ld", pg_conn->cluster_name, last_ran_table, check_time, check_time, check_time); rc = DEF_QUERY_RET_RC; } else slurm_mutex_unlock(&usage_rollup_lock); no_rollup_change: if (job_ptr->name && job_ptr->name[0]) jname = xstrdup(job_ptr->name); else { jname = xstrdup("allocation"); track_steps = 1; } if (job_ptr->nodes && job_ptr->nodes[0]) nodes = job_ptr->nodes; else nodes = "None assigned"; if (job_ptr->batch_flag) track_steps = 1; if (slurmdbd_conf) { block_id = xstrdup(job_ptr->comment); node_cnt = job_ptr->total_nodes; node_inx = job_ptr->network; } else { char temp_bit[BUF_SIZE]; if (job_ptr->node_bitmap) { node_inx = bit_fmt(temp_bit, sizeof(temp_bit), job_ptr->node_bitmap); } #ifdef HAVE_BG select_g_select_jobinfo_get(job_ptr->select_jobinfo, SELECT_JOBDATA_BLOCK_ID, &block_id); select_g_select_jobinfo_get(job_ptr->select_jobinfo, SELECT_JOBDATA_NODE_CNT, &node_cnt); #else node_cnt = job_ptr->total_nodes; #endif } /* If there is a start_time get the wckeyid. If the job is * cancelled before the job starts we also want to grab it. */ if (job_ptr->assoc_id && (job_ptr->start_time || IS_JOB_CANCELLED(job_ptr))) wckeyid = get_wckeyid(pg_conn, &job_ptr->wckey, job_ptr->user_id, pg_conn->cluster_name, job_ptr->assoc_id); if (!job_ptr->db_index) { if (!begin_time) begin_time = submit_time; rec = xstrdup_printf( "(0, 0, '%s', '%s', %d, %d, 0, '%s', " "%d, '%s', %d, %d, %d, %d, %d, %d, 0, " "%d, %ld, %ld, %ld, 0, 0, " "%d, '%s', '%s', %d, %d, '%s', %d)", /* job_db_inx=0, not used */ /* deleted=0 */ job_ptr->account ?: "", /* account */ job_ptr->partition ?: "", /* partition */ (int)job_ptr->details->min_cpus, /* cpus_req */ (int)job_ptr->total_cpus, /* cpus_alloc */ /* exit_code=0 */ jname, /* job_name */ (int)job_ptr->assoc_id, /* id_assoc */ block_id ?: "", /* id_block */ (int)job_ptr->job_id, /* id_job */ (int)job_ptr->qos_id, /* id_qos */ (int)job_ptr->resv_id, /* id_resv */ (int)wckeyid, /* id_wckey */ (int)job_ptr->user_id, /* uid */ (int)job_ptr->group_id, /* gid */ /* kill_requid=0 */ (int)job_ptr->time_limit, /* timelimit */ submit_time, /* time_submit */ begin_time, /* time_eligible */ start_time, /* time_start */ /* time_end=0 */ /* time_suspended=0 */ (int)node_cnt, /* nodes_alloc */ nodes ?: "", /* nodelist */ node_inx ?: "", /* node_inx */ (int)job_ptr->priority, /* priority */ (int)job_state, /* state */ job_ptr->wckey ?: "", /* wckey */ (int)track_steps); query = xstrdup_printf("SELECT %s.add_job_start(%s);", pg_conn->cluster_name, rec); xfree(rec); try_again: DEBUG_QUERY; job_ptr->db_index = pgsql_query_ret_id(pg_conn->db_conn, query); if (!job_ptr->db_index) { if (!reinit) { error("It looks like the storage has gone " "away trying to reconnect"); check_db_connection(pg_conn); reinit = 1; goto try_again; } else rc = SLURM_ERROR; } xfree(query); } else {
/* * load into the storage the end of a job step */ extern int jobacct_storage_p_step_complete(void *db_conn, struct step_record *step_ptr) { char buf[BUFFER_SIZE]; time_t now; int elapsed; int comp_status; int cpus = 0, rc; char node_list[BUFFER_SIZE]; struct jobacctinfo *jobacct = (struct jobacctinfo *)step_ptr->jobacct; struct jobacctinfo dummy_jobacct; #ifdef HAVE_BG char *ionodes = NULL; #endif float ave_vsize = 0, ave_rss = 0, ave_pages = 0; float ave_cpu = 0; uint32_t ave_cpu2 = 0; char *account, *step_name; uint32_t exit_code; if (!storage_init) { debug("jobacct init was not called or it failed"); return SLURM_ERROR; } now = time(NULL); if (jobacct == NULL) { /* JobAcctGather=slurmdb_gather/none, no data to process */ memset(&dummy_jobacct, 0, sizeof(dummy_jobacct)); jobacct = &dummy_jobacct; } if ((elapsed=now-step_ptr->start_time)<0) elapsed=0; /* For *very* short jobs, if clock is wrong */ exit_code = step_ptr->exit_code; if (exit_code == NO_VAL) { comp_status = JOB_CANCELLED; exit_code = 0; } else if (exit_code) comp_status = JOB_FAILED; else comp_status = JOB_COMPLETE; #ifdef HAVE_BG if (step_ptr->job_ptr->details) cpus = step_ptr->job_ptr->details->min_cpus; else cpus = step_ptr->job_ptr->cpu_cnt; select_g_select_jobinfo_get(step_ptr->job_ptr->select_jobinfo, SELECT_JOBDATA_IONODES, &ionodes); if (ionodes) { snprintf(node_list, BUFFER_SIZE, "%s[%s]", step_ptr->job_ptr->nodes, ionodes); xfree(ionodes); } else snprintf(node_list, BUFFER_SIZE, "%s", step_ptr->job_ptr->nodes); #else if (!step_ptr->step_layout || !step_ptr->step_layout->task_cnt) { cpus = step_ptr->job_ptr->total_cpus; snprintf(node_list, BUFFER_SIZE, "%s", step_ptr->job_ptr->nodes); } else { cpus = step_ptr->step_layout->task_cnt; snprintf(node_list, BUFFER_SIZE, "%s", step_ptr->step_layout->node_list); } #endif /* figure out the ave of the totals sent */ if (cpus > 0) { ave_vsize = jobacct->tot_vsize; ave_vsize /= cpus; ave_rss = jobacct->tot_rss; ave_rss /= cpus; ave_pages = jobacct->tot_pages; ave_pages /= cpus; ave_cpu = jobacct->tot_cpu; ave_cpu /= cpus; } if (jobacct->min_cpu != (uint32_t)NO_VAL) { ave_cpu2 = jobacct->min_cpu; } account = _safe_dup(step_ptr->job_ptr->account); step_name = _safe_dup(step_ptr->name); snprintf(buf, BUFFER_SIZE, _jobstep_format, JOB_STEP, step_ptr->step_id, /* stepid */ comp_status, /* completion status */ exit_code, /* completion code */ cpus, /* number of tasks */ cpus, /* number of cpus */ elapsed, /* elapsed seconds */ /* total cputime seconds */ jobacct->user_cpu_sec + jobacct->sys_cpu_sec, /* total cputime seconds */ jobacct->user_cpu_usec + jobacct->sys_cpu_usec, jobacct->user_cpu_sec, /* user seconds */ jobacct->user_cpu_usec,/* user microseconds */ jobacct->sys_cpu_sec, /* system seconds */ jobacct->sys_cpu_usec,/* system microsecs */ 0, /* max rss */ 0, /* max ixrss */ 0, /* max idrss */ 0, /* max isrss */ 0, /* max minflt */ 0, /* max majflt */ 0, /* max nswap */ 0, /* total inblock */ 0, /* total outblock */ 0, /* total msgsnd */ 0, /* total msgrcv */ 0, /* total nsignals */ 0, /* total nvcsw */ 0, /* total nivcsw */ jobacct->max_vsize, /* max vsize */ jobacct->max_vsize_id.taskid, /* max vsize node */ ave_vsize, /* ave vsize */ jobacct->max_rss, /* max vsize */ jobacct->max_rss_id.taskid, /* max rss node */ ave_rss, /* ave rss */ jobacct->max_pages, /* max pages */ jobacct->max_pages_id.taskid, /* max pages node */ ave_pages, /* ave pages */ ave_cpu2, /* min cpu */ jobacct->min_cpu_id.taskid, /* min cpu node */ ave_cpu, /* ave cpu */ step_name, /* step exe name */ node_list, /* name of nodes step running on */ jobacct->max_vsize_id.nodeid, /* max vsize task */ jobacct->max_rss_id.nodeid, /* max rss task */ jobacct->max_pages_id.nodeid, /* max pages task */ jobacct->min_cpu_id.nodeid, /* min cpu task */ account, step_ptr->job_ptr->requid); /* requester user id */ rc = _print_record(step_ptr->job_ptr, now, buf); xfree(account); xfree(step_name); return rc; }
slurmd_job_t * job_batch_job_create(batch_job_launch_msg_t *msg) { struct passwd *pwd; slurmd_job_t *job; srun_info_t *srun = NULL; char *in_name; xassert(msg != NULL); debug3("entering batch_job_create"); if ((pwd = _pwd_create((uid_t)msg->uid)) == NULL) { error("uid %ld not found on system", (long) msg->uid); slurm_seterrno (ESLURMD_UID_NOT_FOUND); return NULL; } if (!_valid_gid(pwd, &(msg->gid))) { slurm_seterrno (ESLURMD_GID_NOT_FOUND); _pwd_destroy(pwd); return NULL; } if(msg->job_mem && (msg->acctg_freq != (uint16_t) NO_VAL) && (msg->acctg_freq > conf->job_acct_gather_freq)) { error("Can't set frequency to %u, it is higher than %u. " "We need it to be at least at this level to " "monitor memory usage.", msg->acctg_freq, conf->job_acct_gather_freq); slurm_seterrno (ESLURMD_INVALID_ACCT_FREQ); _pwd_destroy(pwd); return NULL; } job = xmalloc(sizeof(slurmd_job_t)); job->state = SLURMSTEPD_STEP_STARTING; job->pwd = pwd; if (msg->cpus_per_node) job->cpus = msg->cpus_per_node[0]; job->node_tasks = 1; job->ntasks = msg->ntasks; job->jobid = msg->job_id; job->stepid = msg->step_id; job->batch = true; if (msg->acctg_freq != (uint16_t) NO_VAL) jobacct_gather_change_poll(msg->acctg_freq); job->multi_prog = 0; job->open_mode = msg->open_mode; job->overcommit = (bool) msg->overcommit; job->node_name = xstrdup(conf->node_name); job->uid = (uid_t) msg->uid; job->gid = (gid_t) msg->gid; job->cwd = xstrdup(msg->work_dir); job->ckpt_dir = xstrdup(msg->ckpt_dir); job->restart_dir = xstrdup(msg->restart_dir); job->env = _array_copy(msg->envc, msg->environment); job->eio = eio_handle_create(); job->sruns = list_create((ListDelF) _srun_info_destructor); job->envtp = xmalloc(sizeof(env_t)); job->envtp->jobid = -1; job->envtp->stepid = -1; job->envtp->procid = -1; job->envtp->localid = -1; job->envtp->nodeid = -1; job->envtp->distribution = 0; job->cpu_bind_type = msg->cpu_bind_type; job->cpu_bind = xstrdup(msg->cpu_bind); job->envtp->mem_bind_type = 0; job->envtp->mem_bind = NULL; job->envtp->ckpt_dir = NULL; job->envtp->restart_cnt = msg->restart_cnt; if (msg->cpus_per_node) job->cpus = msg->cpus_per_node[0]; format_core_allocs(msg->cred, conf->node_name, &job->job_alloc_cores, &job->step_alloc_cores, &job->job_mem, &job->step_mem); if (job->step_mem) jobacct_gather_set_mem_limit(job->jobid, NO_VAL, job->step_mem); else if (job->job_mem) jobacct_gather_set_mem_limit(job->jobid, NO_VAL, job->job_mem); get_cred_gres(msg->cred, conf->node_name, &job->job_gres_list, &job->step_gres_list); srun = srun_info_create(NULL, NULL, NULL); list_append(job->sruns, (void *) srun); if (msg->argc) { job->argc = msg->argc; job->argv = _array_copy(job->argc, msg->argv); } else { job->argc = 1; /* job script has not yet been written out to disk -- * argv will be filled in later by _make_batch_script() */ job->argv = (char **) xmalloc(2 * sizeof(char *)); } job->task = xmalloc(sizeof(slurmd_task_info_t *)); if (msg->std_err == NULL) msg->std_err = xstrdup(msg->std_out); if (msg->std_in == NULL) in_name = xstrdup("/dev/null"); else in_name = fname_create(job, msg->std_in, 0); job->task[0] = task_info_create(0, 0, in_name, _batchfilename(job, msg->std_out), _batchfilename(job, msg->std_err)); job->task[0]->argc = job->argc; job->task[0]->argv = job->argv; #ifdef HAVE_CRAY select_g_select_jobinfo_get(msg->select_jobinfo, SELECT_JOBDATA_RESV_ID, &job->resv_id); #endif return job; }