static void _state_read_buf(Buf buffer)
{
uint16_t protocol_version = (uint16_t) NO_VAL;
uint32_t min_port, max_port;
int i;
/* Validate state version */
safe_unpack16(&protocol_version, buffer);
debug3("Version in switch_cray header is %u", protocol_version);
if (protocol_version < SLURM_MIN_PROTOCOL_VERSION) {
error("******************************************************");
error("Can't recover switch/cray state, incompatible version");
error("******************************************************");
return;
}
pthread_mutex_lock(&port_mutex);
if (protocol_version >= SLURM_14_11_PROTOCOL_VERSION) {
safe_unpack32(&min_port, buffer);
safe_unpack32(&max_port, buffer);
safe_unpack32(&last_alloc_port, buffer);
/* make sure we are NULL here */
FREE_NULL_BITMAP(port_resv);
unpack_bit_str_hex(&port_resv, buffer);
} else if (protocol_version >= SLURM_MIN_PROTOCOL_VERSION) {
uint8_t port_set = 0;
safe_unpack32(&min_port, buffer);
safe_unpack32(&max_port, buffer);
safe_unpack32(&last_alloc_port, buffer);
/* make sure we are NULL here */
FREE_NULL_BITMAP(port_resv);
port_resv = bit_alloc(PORT_CNT);
for (i = 0; i < PORT_CNT; i++) {
safe_unpack8(&port_set, buffer);
if (port_set)
bit_set(port_resv, i);
}
}
if (!port_resv || (bit_size(port_resv) != PORT_CNT)) {
error("_state_read_buf: Reserve Port size was %d not %d, "
"reallocating",
port_resv ? bit_size(port_resv) : -1, PORT_CNT);
port_resv = bit_realloc(port_resv, PORT_CNT);
}
pthread_mutex_unlock(&port_mutex);
if ((min_port != MIN_PORT) || (max_port != MAX_PORT)) {
error("******************************************************");
error("Can not recover switch/cray state");
error("Changed MIN_PORT (%u != %u) and/or MAX_PORT (%u != %u)",
min_port, MIN_PORT, max_port, MAX_PORT);
error("******************************************************");
return;
}
return;
unpack_error:
CRAY_ERR("unpack error");
return;
}
static int
_handle_completion(int fd, stepd_step_rec_t *job, uid_t uid)
{
int rc = SLURM_SUCCESS;
int errnum = 0;
int first;
int last;
jobacctinfo_t *jobacct = NULL;
int step_rc;
char* buf;
int len;
Buf buffer;
int version; /* For future use */
bool lock_set = false;
debug("_handle_completion for job %u.%u",
job->jobid, job->stepid);
debug3(" uid = %d", uid);
if (!_slurm_authorized_user(uid)) {
debug("step completion message from uid %ld for job %u.%u ",
(long)uid, job->jobid, job->stepid);
rc = -1;
errnum = EPERM;
/* Send the return code and errno */
safe_write(fd, &rc, sizeof(int));
safe_write(fd, &errnum, sizeof(int));
return SLURM_SUCCESS;
}
safe_read(fd, &version, sizeof(int));
safe_read(fd, &first, sizeof(int));
safe_read(fd, &last, sizeof(int));
safe_read(fd, &step_rc, sizeof(int));
/*
* We must not use getinfo over a pipe with slurmd here
* Indeed, slurmstepd does a large use of setinfo over a pipe
* with slurmd and doing the reverse can result in a deadlock
* scenario with slurmd :
* slurmd(lockforread,write)/slurmstepd(write,lockforread)
* Do pack/unpack instead to be sure of independances of
* slurmd and slurmstepd
*/
safe_read(fd, &len, sizeof(int));
buf = xmalloc(len);
safe_read(fd, buf, len);
buffer = create_buf(buf, len);
jobacctinfo_unpack(&jobacct, SLURM_PROTOCOL_VERSION,
PROTOCOL_TYPE_SLURM, buffer, 1);
free_buf(buffer);
/*
* Record the completed nodes
*/
pthread_mutex_lock(&step_complete.lock);
lock_set = true;
if (! step_complete.wait_children) {
rc = -1;
errnum = ETIMEDOUT; /* not used anyway */
goto timeout;
}
/* SlurmUser or root can craft a launch without a valid credential
* ("srun --no-alloc ...") and no tree information can be built
* without the hostlist from the credential. */
if (step_complete.rank >= 0) {
#if 0
char bits_string[128];
debug2("Setting range %d (bit %d) through %d(bit %d)",
first, first-(step_complete.rank+1),
last, last-(step_complete.rank+1));
bit_fmt(bits_string, sizeof(bits_string), step_complete.bits);
debug2(" before bits: %s", bits_string);
#endif
bit_nset(step_complete.bits,
first - (step_complete.rank+1),
last - (step_complete.rank+1));
#if 0
bit_fmt(bits_string, sizeof(bits_string), step_complete.bits);
debug2(" after bits: %s", bits_string);
#endif
}
step_complete.step_rc = MAX(step_complete.step_rc, step_rc);
/************* acct stuff ********************/
jobacctinfo_aggregate(step_complete.jobacct, jobacct);
timeout:
jobacctinfo_destroy(jobacct);
/*********************************************/
/* Send the return code and errno, we do this within the locked
* region to ensure that the stepd doesn't exit before we can
* perform this send. */
safe_write(fd, &rc, sizeof(int));
safe_write(fd, &errnum, sizeof(int));
pthread_cond_signal(&step_complete.cond);
pthread_mutex_unlock(&step_complete.lock);
return SLURM_SUCCESS;
rwfail: if (lock_set) {
pthread_cond_signal(&step_complete.cond);
pthread_mutex_unlock(&step_complete.lock);
}
return SLURM_FAILURE;
}
static void *
_handle_accept(void *arg)
{
/*struct request_params *param = (struct request_params *)arg;*/
int fd = ((struct request_params *)arg)->fd;
stepd_step_rec_t *job = ((struct request_params *)arg)->job;
int req;
int len;
Buf buffer;
void *auth_cred;
int rc;
uid_t uid;
gid_t gid;
debug3("Entering _handle_accept (new thread)");
xfree(arg);
safe_read(fd, &req, sizeof(int));
if (req != REQUEST_CONNECT) {
error("First message must be REQUEST_CONNECT");
goto fail;
}
safe_read(fd, &len, sizeof(int));
buffer = init_buf(len);
safe_read(fd, get_buf_data(buffer), len);
/* Unpack and verify the auth credential */
auth_cred = g_slurm_auth_unpack(buffer);
if (auth_cred == NULL) {
error("Unpacking authentication credential: %s",
g_slurm_auth_errstr(g_slurm_auth_errno(NULL)));
free_buf(buffer);
goto fail;
}
rc = g_slurm_auth_verify(auth_cred, NULL, 2, NULL);
if (rc != SLURM_SUCCESS) {
error("Verifying authentication credential: %s",
g_slurm_auth_errstr(g_slurm_auth_errno(auth_cred)));
(void) g_slurm_auth_destroy(auth_cred);
free_buf(buffer);
goto fail;
}
/* Get the uid & gid from the credential, then destroy it. */
uid = g_slurm_auth_get_uid(auth_cred, NULL);
gid = g_slurm_auth_get_gid(auth_cred, NULL);
debug3(" Identity: uid=%d, gid=%d", uid, gid);
g_slurm_auth_destroy(auth_cred);
free_buf(buffer);
rc = SLURM_SUCCESS;
safe_write(fd, &rc, sizeof(int));
while (1) {
rc = _handle_request(fd, job, uid, gid);
if (rc != SLURM_SUCCESS)
break;
}
if (close(fd) == -1)
error("Closing accepted fd: %m");
pthread_mutex_lock(&message_lock);
message_connections--;
pthread_cond_signal(&message_cond);
pthread_mutex_unlock(&message_lock);
debug3("Leaving _handle_accept");
return NULL;
fail:
rc = SLURM_FAILURE;
safe_write(fd, &rc, sizeof(int));
rwfail:
if (close(fd) == -1)
error("Closing accepted fd after error: %m");
debug("Leaving _handle_accept on an error");
return NULL;
}
/*
* load_all_front_end_state - Load the front_end node state from file, recover
* on slurmctld restart. Execute this after loading the configuration
* file data. Data goes into common storage.
* IN state_only - if true, overwrite only front_end node state and reason
* Use this to overwrite the "UNKNOWN state typically used in slurm.conf
* RET 0 or error code
* NOTE: READ lock_slurmctld config before entry
*/
extern int load_all_front_end_state(bool state_only)
{
#ifdef HAVE_FRONT_END
char *node_name = NULL, *reason = NULL, *data = NULL, *state_file;
int data_allocated, data_read = 0, error_code = 0, node_cnt = 0;
uint32_t node_state;
uint32_t data_size = 0, name_len;
uint32_t reason_uid = NO_VAL;
time_t reason_time = 0;
front_end_record_t *front_end_ptr;
int state_fd;
time_t time_stamp;
Buf buffer;
char *ver_str = NULL;
uint16_t protocol_version = (uint16_t) NO_VAL;
/* read the file */
lock_state_files ();
state_fd = _open_front_end_state_file(&state_file);
if (state_fd < 0) {
info ("No node state file (%s) to recover", state_file);
error_code = ENOENT;
} else {
data_allocated = BUF_SIZE;
data = xmalloc(data_allocated);
while (1) {
data_read = read(state_fd, &data[data_size], BUF_SIZE);
if (data_read < 0) {
if (errno == EINTR)
continue;
else {
error ("Read error on %s: %m",
state_file);
break;
}
} else if (data_read == 0) /* eof */
break;
data_size += data_read;
data_allocated += data_read;
xrealloc(data, data_allocated);
}
close (state_fd);
}
xfree (state_file);
unlock_state_files ();
buffer = create_buf (data, data_size);
safe_unpackstr_xmalloc( &ver_str, &name_len, buffer);
debug3("Version string in front_end_state header is %s", ver_str);
if (ver_str) {
if (!strcmp(ver_str, FRONT_END_STATE_VERSION)) {
safe_unpack16(&protocol_version, buffer);
} else
protocol_version = SLURM_2_6_PROTOCOL_VERSION;
}
if (protocol_version == (uint16_t) NO_VAL) {
error("*****************************************************");
error("Can not recover front_end state, version incompatible");
error("*****************************************************");
xfree(ver_str);
free_buf(buffer);
return EFAULT;
}
xfree(ver_str);
safe_unpack_time(&time_stamp, buffer);
while (remaining_buf (buffer) > 0) {
uint32_t base_state = (uint32_t)NO_VAL;
uint16_t tmp_state;
uint16_t obj_protocol_version = (uint16_t)NO_VAL;;
if (protocol_version >= SLURM_14_11_PROTOCOL_VERSION) {
safe_unpackstr_xmalloc (&node_name, &name_len, buffer);
safe_unpack32 (&node_state, buffer);
safe_unpackstr_xmalloc (&reason, &name_len, buffer);
safe_unpack_time (&reason_time, buffer);
safe_unpack32 (&reason_uid, buffer);
safe_unpack16 (&obj_protocol_version, buffer);
base_state = node_state & NODE_STATE_BASE;
} else if (protocol_version >= SLURM_14_03_PROTOCOL_VERSION) {
safe_unpackstr_xmalloc (&node_name, &name_len, buffer);
safe_unpack16 (&tmp_state, buffer);
safe_unpackstr_xmalloc (&reason, &name_len, buffer);
safe_unpack_time (&reason_time, buffer);
safe_unpack32 (&reason_uid, buffer);
safe_unpack16 (&obj_protocol_version, buffer);
node_state = tmp_state;
base_state = node_state & NODE_STATE_BASE;
} else if (protocol_version >= SLURM_2_6_PROTOCOL_VERSION) {
safe_unpackstr_xmalloc (&node_name, &name_len, buffer);
safe_unpack16 (&tmp_state, buffer);
safe_unpackstr_xmalloc (&reason, &name_len, buffer);
safe_unpack_time (&reason_time, buffer);
safe_unpack32 (&reason_uid, buffer);
node_state = tmp_state;
base_state = node_state & NODE_STATE_BASE;
} else
goto unpack_error;
/* validity test as possible */
/* find record and perform update */
front_end_ptr = find_front_end_record(node_name);
if (front_end_ptr == NULL) {
error("Front_end node %s has vanished from "
"configuration", node_name);
} else if (state_only) {
uint32_t orig_flags;
orig_flags = front_end_ptr->node_state &
NODE_STATE_FLAGS;
if (IS_NODE_UNKNOWN(front_end_ptr)) {
if (base_state == NODE_STATE_DOWN) {
orig_flags &= (~NODE_STATE_COMPLETING);
front_end_ptr->node_state =
NODE_STATE_DOWN | orig_flags;
}
if (node_state & NODE_STATE_DRAIN) {
front_end_ptr->node_state |=
NODE_STATE_DRAIN;
}
if (node_state & NODE_STATE_FAIL) {
front_end_ptr->node_state |=
NODE_STATE_FAIL;
}
}
if (front_end_ptr->reason == NULL) {
front_end_ptr->reason = reason;
reason = NULL; /* Nothing to free */
front_end_ptr->reason_time = reason_time;
front_end_ptr->reason_uid = reason_uid;
}
} else {
front_end_ptr->node_state = node_state;
xfree(front_end_ptr->reason);
front_end_ptr->reason = reason;
reason = NULL; /* Nothing to free */
front_end_ptr->reason_time = reason_time;
front_end_ptr->reason_uid = reason_uid;
front_end_ptr->last_response = (time_t) 0;
}
if (front_end_ptr) {
node_cnt++;
if (obj_protocol_version != (uint16_t)NO_VAL)
front_end_ptr->protocol_version =
obj_protocol_version;
else
front_end_ptr->protocol_version =
protocol_version;
}
xfree(node_name);
xfree(reason);
}
fini: info("Recovered state of %d front_end nodes", node_cnt);
free_buf (buffer);
return error_code;
unpack_error:
error("Incomplete front_end node data checkpoint file");
error_code = EFAULT;
xfree (node_name);
xfree(reason);
goto fini;
#else
return 0;
#endif
}
int
hostkeys_foreach(const char *path, hostkeys_foreach_fn *callback, void *ctx,
const char *host, u_int options)
{
FILE *f;
char line[8192], oline[8192];
u_long linenum = 0;
char *cp, *cp2;
u_int kbits;
int s, r = 0;
struct hostkey_foreach_line lineinfo;
memset(&lineinfo, 0, sizeof(lineinfo));
if (host == NULL && (options & HKF_WANT_MATCH_HOST) != 0)
return SSH_ERR_INVALID_ARGUMENT;
if ((f = fopen(path, "r")) == NULL)
return SSH_ERR_SYSTEM_ERROR;
debug3("%s: reading file \"%s\"", __func__, path);
while (read_keyfile_line(f, path, line, sizeof(line), &linenum) == 0) {
line[strcspn(line, "\n")] = '\0';
strlcpy(oline, line, sizeof(oline));
sshkey_free(lineinfo.key);
memset(&lineinfo, 0, sizeof(lineinfo));
lineinfo.path = path;
lineinfo.linenum = linenum;
lineinfo.line = oline;
lineinfo.status = HKF_STATUS_OK;
/* Skip any leading whitespace, comments and empty lines. */
for (cp = line; *cp == ' ' || *cp == '\t'; cp++)
;
if (!*cp || *cp == '#' || *cp == '\n') {
if ((options & HKF_WANT_MATCH_HOST) == 0) {
lineinfo.status = HKF_STATUS_COMMENT;
if ((r = callback(&lineinfo, ctx)) != 0)
break;
}
continue;
}
if ((lineinfo.marker = check_markers(&cp)) == MRK_ERROR) {
verbose("%s: invalid marker at %s:%lu",
__func__, path, linenum);
if ((options & HKF_WANT_MATCH_HOST) == 0)
goto bad;
continue;
}
/* Find the end of the host name portion. */
for (cp2 = cp; *cp2 && *cp2 != ' ' && *cp2 != '\t'; cp2++)
;
lineinfo.hosts = cp;
*cp2++ = '\0';
/* Check if the host name matches. */
if (host != NULL) {
s = match_maybe_hashed(host, lineinfo.hosts,
&lineinfo.was_hashed);
if (s == 1)
lineinfo.status = HKF_STATUS_HOST_MATCHED;
else if ((options & HKF_WANT_MATCH_HOST) != 0)
continue;
else if (s == -1) {
debug2("%s: %s:%ld: bad host hash \"%.32s\"",
__func__, path, linenum, lineinfo.hosts);
goto bad;
}
}
/* Got a match. Skip host name and any following whitespace */
for (; *cp2 == ' ' || *cp2 == '\t'; cp2++)
;
if (*cp2 == '\0' || *cp2 == '#') {
debug2("%s:%ld: truncated before key", path, linenum);
goto bad;
}
lineinfo.rawkey = cp = cp2;
if ((options & HKF_WANT_PARSE_KEY) != 0) {
/*
* Extract the key from the line. This will skip
* any leading whitespace. Ignore badly formatted
* lines.
*/
if ((lineinfo.key = sshkey_new(KEY_UNSPEC)) == NULL) {
error("%s: sshkey_new failed", __func__);
return SSH_ERR_ALLOC_FAIL;
}
if (!hostfile_read_key(&cp, &kbits, lineinfo.key)) {
#ifdef WITH_SSH1
sshkey_free(lineinfo.key);
lineinfo.key = sshkey_new(KEY_RSA1);
if (lineinfo.key == NULL) {
error("%s: sshkey_new fail", __func__);
return SSH_ERR_ALLOC_FAIL;
}
if (!hostfile_read_key(&cp, &kbits,
lineinfo.key))
goto bad;
#else
goto bad;
#endif
}
if (!hostfile_check_key(kbits, lineinfo.key, host,
path, linenum)) {
bad:
lineinfo.status = HKF_STATUS_INVALID;
if ((r = callback(&lineinfo, ctx)) != 0)
break;
continue;
}
}
if ((r = callback(&lineinfo, ctx)) != 0)
break;
}
sshkey_free(lineinfo.key);
fclose(f);
return r;
}
void
mm_get_keystate(struct monitor *pmonitor)
{
Buffer m;
u_char *blob, *p;
u_int bloblen, plen;
u_int32_t seqnr, packets;
u_int64_t blocks;
debug3("%s: Waiting for new keys", __func__);
buffer_init(&m);
mm_request_receive_expect(pmonitor->m_sendfd, MONITOR_REQ_KEYEXPORT, &m);
if (!compat20) {
child_state.ssh1protoflags = buffer_get_int(&m);
child_state.ssh1cipher = buffer_get_int(&m);
child_state.ssh1key = buffer_get_string(&m,
&child_state.ssh1keylen);
child_state.ivout = buffer_get_string(&m,
&child_state.ivoutlen);
child_state.ivin = buffer_get_string(&m, &child_state.ivinlen);
goto skip;
} else {
/* Get the Kex for rekeying */
*pmonitor->m_pkex = mm_get_kex(&m);
}
blob = buffer_get_string(&m, &bloblen);
current_keys[MODE_OUT] = mm_newkeys_from_blob(blob, bloblen);
xfree(blob);
debug3("%s: Waiting for second key", __func__);
blob = buffer_get_string(&m, &bloblen);
current_keys[MODE_IN] = mm_newkeys_from_blob(blob, bloblen);
xfree(blob);
/* Now get sequence numbers for the packets */
seqnr = buffer_get_int(&m);
blocks = buffer_get_int64(&m);
packets = buffer_get_int(&m);
packet_set_state(MODE_OUT, seqnr, blocks, packets);
seqnr = buffer_get_int(&m);
blocks = buffer_get_int64(&m);
packets = buffer_get_int(&m);
packet_set_state(MODE_IN, seqnr, blocks, packets);
skip:
/* Get the key context */
child_state.keyout = buffer_get_string(&m, &child_state.keyoutlen);
child_state.keyin = buffer_get_string(&m, &child_state.keyinlen);
debug3("%s: Getting compression state", __func__);
/* Get compression state */
p = buffer_get_string(&m, &plen);
if (plen != sizeof(child_state.outgoing))
fatal("%s: bad request size", __func__);
memcpy(&child_state.outgoing, p, sizeof(child_state.outgoing));
xfree(p);
p = buffer_get_string(&m, &plen);
if (plen != sizeof(child_state.incoming))
fatal("%s: bad request size", __func__);
memcpy(&child_state.incoming, p, sizeof(child_state.incoming));
xfree(p);
/* Network I/O buffers */
debug3("%s: Getting Network I/O buffers", __func__);
child_state.input = buffer_get_string(&m, &child_state.ilen);
child_state.output = buffer_get_string(&m, &child_state.olen);
buffer_free(&m);
}
int
mm_answer_pwnamallow(int sock, Buffer *m)
{
char *username;
struct passwd *pwent;
int allowed = 0;
debug3("%s", __func__);
if (authctxt->attempt++ != 0)
fatal("%s: multiple attempts for getpwnam", __func__);
username = buffer_get_string(m, NULL);
pwent = getpwnamallow(username);
authctxt->user = xstrdup(username);
setproctitle("%s [priv]", pwent ? username : "******");
xfree(username);
buffer_clear(m);
if (pwent == NULL) {
buffer_put_char(m, 0);
authctxt->pw = fakepw();
goto out;
}
allowed = 1;
authctxt->pw = pwent;
authctxt->valid = 1;
buffer_put_char(m, 1);
buffer_put_string(m, pwent, sizeof(struct passwd));
buffer_put_cstring(m, pwent->pw_name);
buffer_put_cstring(m, "*");
buffer_put_cstring(m, pwent->pw_gecos);
#ifdef HAVE_PW_CLASS_IN_PASSWD
buffer_put_cstring(m, pwent->pw_class);
#endif
buffer_put_cstring(m, pwent->pw_dir);
buffer_put_cstring(m, pwent->pw_shell);
buffer_put_string(m, &options, sizeof(options));
if (options.banner != NULL)
buffer_put_cstring(m, options.banner);
out:
debug3("%s: sending MONITOR_ANS_PWNAM: %d", __func__, allowed);
mm_request_send(sock, MONITOR_ANS_PWNAM, m);
/* For SSHv1 allow authentication now */
if (!compat20)
monitor_permit_authentications(1);
else {
/* Allow service/style information on the auth context */
monitor_permit(mon_dispatch, MONITOR_REQ_AUTHSERV, 1);
monitor_permit(mon_dispatch, MONITOR_REQ_AUTH2_READ_BANNER, 1);
}
#ifdef USE_PAM
if (options.use_pam)
monitor_permit(mon_dispatch, MONITOR_REQ_PAM_START, 1);
#endif
return (0);
}
void
userauth_finish(struct ssh *ssh, int authenticated, const char *method,
const char *submethod)
{
struct authctxt *authctxt = ssh->authctxt;
char *methods;
int r, partial = 0;
if (!authctxt->valid && authenticated)
fatal("INTERNAL ERROR: authenticated invalid user %s",
authctxt->user);
if (authenticated && authctxt->postponed)
fatal("INTERNAL ERROR: authenticated and postponed");
/* Special handling for root */
if (authenticated && authctxt->pw->pw_uid == 0 &&
!auth_root_allowed(method))
authenticated = 0;
if (authenticated && options.num_auth_methods != 0) {
if (!auth2_update_methods_lists(authctxt, method, submethod)) {
authenticated = 0;
partial = 1;
}
}
/* Log before sending the reply */
auth_log(authctxt, authenticated, partial, method, submethod);
if (authctxt->postponed)
return;
if (authenticated == 1) {
/* turn off userauth */
ssh_dispatch_set(ssh, SSH2_MSG_USERAUTH_REQUEST,
&dispatch_protocol_ignore);
if ((r = sshpkt_start(ssh, SSH2_MSG_USERAUTH_SUCCESS)) != 0 ||
(r = sshpkt_send(ssh)) != 0)
fatal("%s: %s", __func__, ssh_err(r));
ssh_packet_write_wait(ssh);
/* now we can break out */
authctxt->success = 1;
} else {
/* Allow initial try of "none" auth without failure penalty */
if (!authctxt->server_caused_failure &&
(authctxt->attempt > 1 || strcmp(method, "none") != 0))
authctxt->failures++;
if (authctxt->failures >= options.max_authtries)
ssh_packet_disconnect(ssh, AUTH_FAIL_MSG,
authctxt->user);
methods = authmethods_get(authctxt);
debug3("%s: failure partial=%d next methods=\"%s\"", __func__,
partial, methods);
if ((r = sshpkt_start(ssh, SSH2_MSG_USERAUTH_FAILURE)) != 0 ||
(r = sshpkt_put_cstring(ssh, methods)) != 0 ||
(r = sshpkt_put_u8(ssh, partial)) != 0 ||
(r = sshpkt_send(ssh)) != 0)
fatal("%s: %s", __func__, ssh_err(r));
ssh_packet_write_wait(ssh);
free(methods);
}
}
/* create a slurmd job structure from a launch tasks message */
extern stepd_step_rec_t *
stepd_step_rec_create(launch_tasks_request_msg_t *msg, uint16_t protocol_version)
{
stepd_step_rec_t *job = NULL;
srun_info_t *srun = NULL;
slurm_addr_t resp_addr;
slurm_addr_t io_addr;
int i, nodeid = NO_VAL;
xassert(msg != NULL);
xassert(msg->complete_nodelist != NULL);
debug3("entering stepd_step_rec_create");
if (!_valid_uid_gid((uid_t)msg->uid, &(msg->gid), &(msg->user_name)))
return NULL;
if (acct_gather_check_acct_freq_task(msg->job_mem_lim, msg->acctg_freq))
return NULL;
job = xmalloc(sizeof(stepd_step_rec_t));
job->msg = msg;
#ifndef HAVE_FRONT_END
nodeid = nodelist_find(msg->complete_nodelist, conf->node_name);
job->node_name = xstrdup(conf->node_name);
#else
nodeid = 0;
job->node_name = xstrdup(msg->complete_nodelist);
#endif
if (nodeid < 0) {
error("couldn't find node %s in %s",
job->node_name, msg->complete_nodelist);
stepd_step_rec_destroy(job);
return NULL;
}
job->state = SLURMSTEPD_STEP_STARTING;
job->node_tasks = msg->tasks_to_launch[nodeid];
i = sizeof(uint16_t) * msg->nnodes;
job->task_cnts = xmalloc(i);
memcpy(job->task_cnts, msg->tasks_to_launch, i);
job->ntasks = msg->ntasks;
job->jobid = msg->job_id;
job->stepid = msg->job_step_id;
job->uid = (uid_t) msg->uid;
job->user_name = xstrdup(msg->user_name);
job->gid = (gid_t) msg->gid;
job->cwd = xstrdup(msg->cwd);
job->task_dist = msg->task_dist;
job->cpu_bind_type = msg->cpu_bind_type;
job->cpu_bind = xstrdup(msg->cpu_bind);
job->mem_bind_type = msg->mem_bind_type;
job->mem_bind = xstrdup(msg->mem_bind);
job->cpu_freq_min = msg->cpu_freq_min;
job->cpu_freq_max = msg->cpu_freq_max;
job->cpu_freq_gov = msg->cpu_freq_gov;
job->ckpt_dir = xstrdup(msg->ckpt_dir);
job->restart_dir = xstrdup(msg->restart_dir);
job->cpus_per_task = msg->cpus_per_task;
job->env = _array_copy(msg->envc, msg->env);
job->array_job_id = msg->job_id;
job->array_task_id = NO_VAL;
for (i = 0; i < msg->envc; i++) {
/* 1234567890123456789 */
if (!strncmp(msg->env[i], "SLURM_ARRAY_JOB_ID=", 19))
job->array_job_id = atoi(msg->env[i] + 19);
/* 12345678901234567890 */
if (!strncmp(msg->env[i], "SLURM_ARRAY_TASK_ID=", 20))
job->array_task_id = atoi(msg->env[i] + 20);
}
job->eio = eio_handle_create(0);
job->sruns = list_create((ListDelF) _srun_info_destructor);
job->clients = list_create(NULL); /* FIXME! Needs destructor */
job->stdout_eio_objs = list_create(NULL); /* FIXME! Needs destructor */
job->stderr_eio_objs = list_create(NULL); /* FIXME! Needs destructor */
job->free_incoming = list_create(NULL); /* FIXME! Needs destructor */
job->incoming_count = 0;
job->free_outgoing = list_create(NULL); /* FIXME! Needs destructor */
job->outgoing_count = 0;
job->outgoing_cache = list_create(NULL); /* FIXME! Needs destructor */
job->envtp = xmalloc(sizeof(env_t));
job->envtp->jobid = -1;
job->envtp->stepid = -1;
job->envtp->procid = -1;
job->envtp->localid = -1;
job->envtp->nodeid = -1;
job->envtp->distribution = 0;
job->envtp->cpu_bind_type = 0;
job->envtp->cpu_bind = NULL;
job->envtp->mem_bind_type = 0;
job->envtp->mem_bind = NULL;
job->envtp->ckpt_dir = NULL;
if (!msg->resp_port)
msg->num_resp_port = 0;
if (msg->num_resp_port) {
job->envtp->comm_port =
msg->resp_port[nodeid % msg->num_resp_port];
memcpy(&resp_addr, &msg->orig_addr, sizeof(slurm_addr_t));
slurm_set_addr(&resp_addr,
msg->resp_port[nodeid % msg->num_resp_port],
NULL);
}
job->user_managed_io = msg->user_managed_io;
if (!msg->io_port)
msg->user_managed_io = 1;
if (!msg->user_managed_io) {
memcpy(&io_addr, &msg->orig_addr, sizeof(slurm_addr_t));
slurm_set_addr(&io_addr,
msg->io_port[nodeid % msg->num_io_port],
NULL);
}
srun = srun_info_create(msg->cred, &resp_addr, &io_addr,
protocol_version);
job->buffered_stdio = msg->buffered_stdio;
job->labelio = msg->labelio;
job->profile = msg->profile;
job->task_prolog = xstrdup(msg->task_prolog);
job->task_epilog = xstrdup(msg->task_epilog);
job->argc = msg->argc;
job->argv = _array_copy(job->argc, msg->argv);
job->nnodes = msg->nnodes;
job->nodeid = nodeid;
job->debug = msg->slurmd_debug;
job->cpus = msg->node_cpus;
job->job_core_spec = msg->job_core_spec;
/* This needs to happen before acct_gather_profile_startpoll
and only really looks at the profile in the job.
*/
acct_gather_profile_g_node_step_start(job);
acct_gather_profile_startpoll(msg->acctg_freq,
conf->job_acct_gather_freq);
job->multi_prog = msg->multi_prog;
job->timelimit = (time_t) -1;
job->task_flags = msg->task_flags;
job->switch_job = msg->switch_job;
job->pty = msg->pty;
job->open_mode = msg->open_mode;
job->options = msg->options;
format_core_allocs(msg->cred, conf->node_name, conf->cpus,
&job->job_alloc_cores, &job->step_alloc_cores,
&job->job_mem, &job->step_mem);
/* If users have configured MemLimitEnforce=no
* in their slurm.conf keep going.
*/
if (job->step_mem
&& conf->mem_limit_enforce) {
jobacct_gather_set_mem_limit(job->jobid, job->stepid,
job->step_mem);
} else if (job->job_mem
&& conf->mem_limit_enforce) {
jobacct_gather_set_mem_limit(job->jobid, job->stepid,
job->job_mem);
}
#ifdef HAVE_ALPS_CRAY
/* This is only used for Cray emulation mode where slurmd is used to
* launch job steps. On a real Cray system, ALPS is used to launch
* the tasks instead of SLURM. SLURM's task launch RPC does NOT
* contain the reservation ID, so just use some non-zero value here
* for testing purposes. */
job->resv_id = 1;
select_g_select_jobinfo_set(msg->select_jobinfo, SELECT_JOBDATA_RESV_ID,
&job->resv_id);
#endif
get_cred_gres(msg->cred, conf->node_name,
&job->job_gres_list, &job->step_gres_list);
list_append(job->sruns, (void *) srun);
_job_init_task_info(job, msg->global_task_ids,
msg->ifname, msg->ofname, msg->efname);
return job;
}
/* To fake a nodecard down do this on the service node.
db2 "update bg{l|p}nodecard set status = 'E' where location =
'Rxx-Mx-Nx' and status='A'"
Reverse the A, and E to bring it back up.
*/
static int _test_nodecard_state(rm_nodecard_t *ncard, int nc_id,
char *node_name, bool slurmctld_locked)
{
int rc = SLURM_SUCCESS;
rm_nodecard_id_t nc_name = NULL;
rm_nodecard_state_t state;
int io_start = 0;
if ((rc = bridge_get_data(ncard,
RM_NodeCardState,
&state)) != SLURM_SUCCESS) {
error("bridge_get_data(RM_NodeCardState): %s",
bg_err_str(rc));
return SLURM_ERROR;
}
if (state == RM_NODECARD_UP)
return SLURM_SUCCESS;
if ((rc = bridge_get_data(ncard,
RM_NodeCardID,
&nc_name)) != SLURM_SUCCESS) {
error("bridge_get_data(RM_NodeCardID): %s", bg_err_str(rc));
return SLURM_ERROR;
}
if (!nc_name) {
error("We didn't get an RM_NodeCardID but rc was SLURM_SUCCESS?");
return SLURM_ERROR;
}
#ifdef HAVE_BGL
if ((rc = bridge_get_data(ncard,
RM_NodeCardQuarter,
&io_start)) != SLURM_SUCCESS) {
error("bridge_get_data(CardQuarter): %s", bg_err_str(rc));
rc = SLURM_ERROR;
goto clean_up;
}
io_start *= bg_conf->quarter_ionode_cnt;
io_start += bg_conf->nodecard_ionode_cnt * (nc_id%4);
#else
/* From the first nodecard id we can figure
out where to start from with the alloc of ionodes.
*/
io_start = atoi((char*)nc_name+1);
io_start *= bg_conf->io_ratio;
#endif
/* On small systems with less than a midplane the
database may see the nodecards there but in missing
state. To avoid getting a bunch of warnings here just
skip over the ones missing.
*/
if (io_start >= bg_conf->ionodes_per_mp) {
rc = SLURM_SUCCESS;
if (state == RM_NODECARD_MISSING) {
debug3("Nodecard %s is missing",
nc_name);
} else {
error("We don't have the system configured "
"for this nodecard %s, we only have "
"%d ionodes and this starts at %d",
nc_name, bg_conf->ionodes_per_mp, io_start);
}
goto clean_up;
}
/* if (!ionode_bitmap) */
/* ionode_bitmap = bit_alloc(bg_conf->ionodes_per_mp); */
/* info("setting %s start %d of %d", */
/* nc_name, io_start, bg_conf->ionodes_per_mp); */
/* bit_nset(ionode_bitmap, io_start, io_start+io_cnt); */
/* we have to handle each nodecard separately to make
sure we don't create holes in the system */
if (down_nodecard(node_name, io_start, slurmctld_locked)
== SLURM_SUCCESS) {
debug("nodecard %s on %s is in an error state",
nc_name, node_name);
} else
debug2("nodecard %s on %s is in an error state, "
"but error was returned when trying to make it so",
nc_name, node_name);
/* Here we want to keep track of any nodecard that
isn't up and return error if it is in the system. */
rc = SLURM_ERROR;
clean_up:
free(nc_name);
return rc;
}
static char *
get_remote_hostname(int sock, int use_dns)
{
struct sockaddr_storage from;
int i;
socklen_t fromlen;
struct addrinfo hints, *ai, *aitop;
char name[NI_MAXHOST], ntop[NI_MAXHOST], ntop2[NI_MAXHOST];
/* Get IP address of client. */
fromlen = sizeof(from);
memset(&from, 0, sizeof(from));
if (getpeername(sock, (struct sockaddr *)&from, &fromlen) < 0) {
debug("getpeername failed: %.100s", strerror(errno));
cleanup_exit(255);
}
if (from.ss_family == AF_INET)
check_ip_options(sock, ntop);
ipv64_normalise_mapped(&from, &fromlen);
if (from.ss_family == AF_INET6)
fromlen = sizeof(struct sockaddr_in6);
if (getnameinfo((struct sockaddr *)&from, fromlen, ntop, sizeof(ntop),
NULL, 0, NI_NUMERICHOST) != 0)
fatal("get_remote_hostname: getnameinfo NI_NUMERICHOST failed");
if (!use_dns)
return xstrdup(ntop);
debug3("Trying to reverse map address %.100s.", ntop);
/* Map the IP address to a host name. */
if (getnameinfo((struct sockaddr *)&from, fromlen, name, sizeof(name),
NULL, 0, NI_NAMEREQD) != 0) {
/* Host name not found. Use ip address. */
return xstrdup(ntop);
}
/*
* if reverse lookup result looks like a numeric hostname,
* someone is trying to trick us by PTR record like following:
* 1.1.1.10.in-addr.arpa. IN PTR 2.3.4.5
*/
memset(&hints, 0, sizeof(hints));
hints.ai_socktype = SOCK_DGRAM; /*dummy*/
hints.ai_flags = AI_NUMERICHOST;
if (getaddrinfo(name, NULL, &hints, &ai) == 0) {
logit("Nasty PTR record \"%s\" is set up for %s, ignoring",
name, ntop);
freeaddrinfo(ai);
return xstrdup(ntop);
}
/*
* Convert it to all lowercase (which is expected by the rest
* of this software).
*/
for (i = 0; name[i]; i++)
if (isupper(name[i]))
name[i] = (char)tolower(name[i]);
/*
* Map it back to an IP address and check that the given
* address actually is an address of this host. This is
* necessary because anyone with access to a name server can
* define arbitrary names for an IP address. Mapping from
* name to IP address can be trusted better (but can still be
* fooled if the intruder has access to the name server of
* the domain).
*/
memset(&hints, 0, sizeof(hints));
hints.ai_family = from.ss_family;
hints.ai_socktype = SOCK_STREAM;
if (getaddrinfo(name, NULL, &hints, &aitop) != 0) {
logit("reverse mapping checking getaddrinfo for %.700s "
"[%s] failed - POSSIBLE BREAK-IN ATTEMPT!", name, ntop);
return xstrdup(ntop);
}
/* Look for the address from the list of addresses. */
for (ai = aitop; ai; ai = ai->ai_next) {
if (getnameinfo(ai->ai_addr, ai->ai_addrlen, ntop2,
sizeof(ntop2), NULL, 0, NI_NUMERICHOST) == 0 &&
(strcmp(ntop, ntop2) == 0))
break;
}
freeaddrinfo(aitop);
/* If we reached the end of the list, the address was not there. */
if (!ai) {
/* Address not found for the host name. */
logit("Address %.100s maps to %.600s, but this does not "
"map back to the address - POSSIBLE BREAK-IN ATTEMPT!",
ntop, name);
return xstrdup(ntop);
}
return xstrdup(name);
}
int
mm_receive_fd(int sock)
{
struct msghdr msg;
union {
struct cmsghdr hdr;
char buf[1024];
} cmsgbuf;
struct cmsghdr *cmsg;
struct iovec vec;
ssize_t n;
char ch;
int fd;
struct pollfd pfd;
if (sizeof(cmsgbuf.buf) < CMSG_SPACE(sizeof(int))) {
error("%s: %zu < %zu, recompile", __func__,
sizeof(cmsgbuf.buf), CMSG_SPACE(sizeof(int)));
return -1;
}
memset(&msg, 0, sizeof(msg));
memset(&cmsgbuf, 0, sizeof(cmsgbuf));
vec.iov_base = &ch;
vec.iov_len = 1;
msg.msg_iov = &vec;
msg.msg_iovlen = 1;
msg.msg_control = &cmsgbuf.buf;
msg.msg_controllen = CMSG_SPACE(sizeof(int));
pfd.fd = sock;
pfd.events = POLLIN;
while ((n = recvmsg(sock, &msg, 0)) == -1 &&
(errno == EAGAIN || errno == EINTR)) {
debug3("%s: recvmsg: %s", __func__, strerror(errno));
(void)poll(&pfd, 1, -1);
}
if (n == -1) {
error("%s: recvmsg: %s", __func__, strerror(errno));
return -1;
}
if (n != 1) {
error("%s: recvmsg: expected received 1 got %zd", __func__, n);
return -1;
}
cmsg = CMSG_FIRSTHDR(&msg);
if (cmsg == NULL) {
error("%s: no message header", __func__);
return -1;
}
if (cmsg->cmsg_type != SCM_RIGHTS) {
error("%s: expected type %d got %d", __func__,
SCM_RIGHTS, cmsg->cmsg_type);
return -1;
}
fd = (*(int *)CMSG_DATA(cmsg));
return fd;
}
/*
* The plane distribution results in a block cyclic of block size
* "plane_size".
* To effectively deal with heterogeneous nodes, we fake a cyclic
* distribution to figure out how many tasks go on each node and
* then make the assignments of task numbers to nodes using the
* user-specified plane size.
* For example:
* plane_size = 2, #tasks = 6, #nodes = 3
*
* Node#: Node0 Node1 Node2
* ----- ----- -----
* #of allocated CPUs: 4 1 1
*
* task distribution: 0 1 2 3
* 4 5
*/
static int _task_layout_plane(slurm_step_layout_t *step_layout,
uint16_t *cpus)
{
int i, j, k, taskid = 0;
bool over_subscribe = false;
uint32_t cur_task[step_layout->node_cnt];
debug3("_task_layout_plane plane_size %u node_cnt %u task_cnt %u",
step_layout->plane_size,
step_layout->node_cnt, step_layout->task_cnt);
if (step_layout->plane_size <= 0)
return SLURM_ERROR;
if (step_layout->tasks == NULL)
return SLURM_ERROR;
/* figure out how many tasks go to each node */
for (j=0; taskid<step_layout->task_cnt; j++) { /* cycle counter */
bool space_remaining = false;
for (i=0; ((i<step_layout->node_cnt)
&& (taskid<step_layout->task_cnt)); i++) {
if ((j<cpus[i]) || over_subscribe) {
taskid++;
step_layout->tasks[i]++;
if ((j+1) < cpus[i])
space_remaining = true;
}
}
if (!space_remaining)
over_subscribe = true;
}
/* now distribute the tasks */
taskid = 0;
for (i=0; i < step_layout->node_cnt; i++) {
step_layout->tids[i] = xmalloc(sizeof(uint32_t)
* step_layout->tasks[i]);
cur_task[i] = 0;
}
for (j=0; taskid<step_layout->task_cnt; j++) { /* cycle counter */
for (i=0; ((i<step_layout->node_cnt)
&& (taskid<step_layout->task_cnt)); i++) {
/* assign a block of 'plane_size' tasks to this node */
for (k=0; ((k<step_layout->plane_size)
&& (cur_task[i] < step_layout->tasks[i])
&& (taskid < step_layout->task_cnt)); k++) {
step_layout->tids[i][cur_task[i]] = taskid;
taskid++;
cur_task[i]++;
}
}
}
if (taskid != step_layout->task_cnt) {
error("_task_layout_plane: Mismatch in task count (%d != %d) ",
taskid, step_layout->task_cnt);
return SLURM_ERROR;
}
#if (0)
/* debugging only */
for (i=0; i < step_layout->node_cnt; i++) {
info("tasks[%d]: %u", i, step_layout->tasks[i]);
}
for (i=0; i < step_layout->node_cnt; i++) {
info ("Host %d _plane_ # of tasks %u", i, step_layout->tasks[i]);
for (j=0; j<step_layout->tasks[i]; j++) {
info ("Host %d _plane_ localid %d taskid %u",
i, j, step_layout->tids[i][j]);
}
}
#endif
return SLURM_SUCCESS;
}
/* use specific set run tasks on each host listed in hostfile
* XXX: Need to handle over-subscribe.
*/
static int _task_layout_hostfile(slurm_step_layout_t *step_layout,
const char *arbitrary_nodes)
{
int i=0, j, taskid = 0, task_cnt=0;
hostlist_iterator_t itr = NULL, itr_task = NULL;
char *host = NULL;
char *host_task = NULL;
hostlist_t job_alloc_hosts = NULL;
hostlist_t step_alloc_hosts = NULL;
debug2("job list is %s", step_layout->node_list);
job_alloc_hosts = hostlist_create(step_layout->node_list);
itr = hostlist_iterator_create(job_alloc_hosts);
if (!arbitrary_nodes) {
error("no hostlist given for arbitrary dist");
return SLURM_ERROR;
}
debug2("list is %s", arbitrary_nodes);
step_alloc_hosts = hostlist_create(arbitrary_nodes);
if (hostlist_count(step_alloc_hosts) != step_layout->task_cnt) {
error("Asked for %u tasks have %d in the nodelist. "
"Check your nodelist, or set the -n option to be %d",
step_layout->task_cnt,
hostlist_count(step_alloc_hosts),
hostlist_count(step_alloc_hosts));
return SLURM_ERROR;
}
itr_task = hostlist_iterator_create(step_alloc_hosts);
while((host = hostlist_next(itr))) {
step_layout->tasks[i] = 0;
while((host_task = hostlist_next(itr_task))) {
if (!strcmp(host, host_task)) {
step_layout->tasks[i]++;
task_cnt++;
}
free(host_task);
if (task_cnt >= step_layout->task_cnt)
break;
}
debug3("%s got %u tasks", host, step_layout->tasks[i]);
if (step_layout->tasks[i] == 0)
goto reset_hosts;
step_layout->tids[i] = xmalloc(sizeof(uint32_t)
* step_layout->tasks[i]);
taskid = 0;
j = 0;
hostlist_iterator_reset(itr_task);
while((host_task = hostlist_next(itr_task))) {
if (!strcmp(host, host_task)) {
step_layout->tids[i][j] = taskid;
j++;
}
taskid++;
free(host_task);
if (j >= step_layout->tasks[i])
break;
}
i++;
reset_hosts:
hostlist_iterator_reset(itr_task);
free(host);
if (i > step_layout->task_cnt)
break;
}
hostlist_iterator_destroy(itr);
hostlist_iterator_destroy(itr_task);
hostlist_destroy(job_alloc_hosts);
hostlist_destroy(step_alloc_hosts);
if (task_cnt != step_layout->task_cnt) {
error("Asked for %u tasks but placed %d. Check your nodelist",
step_layout->task_cnt, task_cnt);
return SLURM_ERROR;
}
return SLURM_SUCCESS;
}
static int
valid_request(struct passwd *pw, char *host, Key **ret, u_char *data,
u_int datalen)
{
Buffer b;
Key *key = NULL;
u_char *pkblob;
u_int blen, len;
char *pkalg, *p;
int pktype, fail;
fail = 0;
buffer_init(&b);
buffer_append(&b, data, datalen);
/* session id, currently limited to SHA1 (20 bytes) */
p = buffer_get_string(&b, &len);
if (len != 20)
fail++;
xfree(p);
if (buffer_get_char(&b) != SSH2_MSG_USERAUTH_REQUEST)
fail++;
/* server user */
buffer_skip_string(&b);
/* service */
p = buffer_get_string(&b, NULL);
if (strcmp("ssh-connection", p) != 0)
fail++;
xfree(p);
/* method */
p = buffer_get_string(&b, NULL);
if (strcmp("hostbased", p) != 0)
fail++;
xfree(p);
/* pubkey */
pkalg = buffer_get_string(&b, NULL);
pkblob = buffer_get_string(&b, &blen);
pktype = key_type_from_name(pkalg);
if (pktype == KEY_UNSPEC)
fail++;
else if ((key = key_from_blob(pkblob, blen)) == NULL)
fail++;
else if (key->type != pktype)
fail++;
xfree(pkalg);
xfree(pkblob);
/* client host name, handle trailing dot */
p = buffer_get_string(&b, &len);
debug2("valid_request: check expect chost %s got %s", host, p);
if (strlen(host) != len - 1)
fail++;
else if (p[len - 1] != '.')
fail++;
else if (strncasecmp(host, p, len - 1) != 0)
fail++;
xfree(p);
/* local user */
p = buffer_get_string(&b, NULL);
if (strcmp(pw->pw_name, p) != 0)
fail++;
xfree(p);
/* end of message */
if (buffer_len(&b) != 0)
fail++;
buffer_free(&b);
debug3("valid_request: fail %d", fail);
if (fail && key != NULL)
key_free(key);
else
*ret = key;
return (fail ? -1 : 0);
}
extern stepd_step_rec_t *
batch_stepd_step_rec_create(batch_job_launch_msg_t *msg)
{
stepd_step_rec_t *job;
srun_info_t *srun = NULL;
char *in_name;
xassert(msg != NULL);
debug3("entering batch_stepd_step_rec_create");
if (!_valid_uid_gid((uid_t)msg->uid, &(msg->gid), &(msg->user_name)))
return NULL;
if (acct_gather_check_acct_freq_task(msg->job_mem, msg->acctg_freq))
return NULL;
job = xmalloc(sizeof(stepd_step_rec_t));
job->state = SLURMSTEPD_STEP_STARTING;
if (msg->cpus_per_node)
job->cpus = msg->cpus_per_node[0];
job->node_tasks = 1;
job->ntasks = msg->ntasks;
job->jobid = msg->job_id;
job->stepid = msg->step_id;
job->array_job_id = msg->array_job_id;
job->array_task_id = msg->array_task_id;
job->job_core_spec = msg->job_core_spec;
job->batch = true;
job->node_name = xstrdup(conf->node_name);
/* This needs to happen before acct_gather_profile_startpoll
and only really looks at the profile in the job.
*/
acct_gather_profile_g_node_step_start(job);
/* needed for the jobacct_gather plugin to start */
acct_gather_profile_startpoll(msg->acctg_freq,
conf->job_acct_gather_freq);
job->multi_prog = 0;
job->open_mode = msg->open_mode;
job->overcommit = (bool) msg->overcommit;
job->uid = (uid_t) msg->uid;
job->user_name = xstrdup(msg->user_name);
job->gid = (gid_t) msg->gid;
job->cwd = xstrdup(msg->work_dir);
job->ckpt_dir = xstrdup(msg->ckpt_dir);
job->restart_dir = xstrdup(msg->restart_dir);
job->env = _array_copy(msg->envc, msg->environment);
job->eio = eio_handle_create(0);
job->sruns = list_create((ListDelF) _srun_info_destructor);
job->envtp = xmalloc(sizeof(env_t));
job->envtp->jobid = -1;
job->envtp->stepid = -1;
job->envtp->procid = -1;
job->envtp->localid = -1;
job->envtp->nodeid = -1;
job->envtp->distribution = 0;
job->cpu_bind_type = msg->cpu_bind_type;
job->cpu_bind = xstrdup(msg->cpu_bind);
job->envtp->mem_bind_type = 0;
job->envtp->mem_bind = NULL;
job->envtp->ckpt_dir = NULL;
job->envtp->restart_cnt = msg->restart_cnt;
if (msg->cpus_per_node)
job->cpus = msg->cpus_per_node[0];
format_core_allocs(msg->cred, conf->node_name, conf->cpus,
&job->job_alloc_cores, &job->step_alloc_cores,
&job->job_mem, &job->step_mem);
if (job->step_mem
&& conf->mem_limit_enforce)
jobacct_gather_set_mem_limit(job->jobid, NO_VAL, job->step_mem);
else if (job->job_mem
&& conf->mem_limit_enforce)
jobacct_gather_set_mem_limit(job->jobid, NO_VAL, job->job_mem);
get_cred_gres(msg->cred, conf->node_name,
&job->job_gres_list, &job->step_gres_list);
srun = srun_info_create(NULL, NULL, NULL, (uint16_t)NO_VAL);
list_append(job->sruns, (void *) srun);
if (msg->argc) {
job->argc = msg->argc;
job->argv = _array_copy(job->argc, msg->argv);
} else {
job->argc = 1;
/* job script has not yet been written out to disk --
* argv will be filled in later by _make_batch_script()
*/
job->argv = (char **) xmalloc(2 * sizeof(char *));
}
job->task = xmalloc(sizeof(stepd_step_task_info_t *));
if (msg->std_err == NULL)
msg->std_err = xstrdup(msg->std_out);
if (msg->std_in == NULL)
in_name = xstrdup("/dev/null");
else
in_name = fname_create(job, msg->std_in, 0);
job->task[0] = task_info_create(0, 0,
in_name,
_batchfilename(job, msg->std_out),
_batchfilename(job, msg->std_err));
job->task[0]->argc = job->argc;
job->task[0]->argv = job->argv;
#ifdef HAVE_ALPS_CRAY
select_g_select_jobinfo_get(msg->select_jobinfo, SELECT_JOBDATA_RESV_ID,
&job->resv_id);
#endif
return job;
}
int
mm_answer_pty(int sock, Buffer *m)
{
extern struct monitor *pmonitor;
Session *s;
int res, fd0;
debug3("%s entering", __func__);
buffer_clear(m);
s = session_new();
if (s == NULL)
goto error;
s->authctxt = authctxt;
s->pw = authctxt->pw;
s->pid = pmonitor->m_pid;
res = pty_allocate(&s->ptyfd, &s->ttyfd, s->tty, sizeof(s->tty));
if (res == 0)
goto error;
pty_setowner(authctxt->pw, s->tty);
buffer_put_int(m, 1);
buffer_put_cstring(m, s->tty);
/* We need to trick ttyslot */
if (dup2(s->ttyfd, 0) == -1)
fatal("%s: dup2", __func__);
mm_record_login(s, authctxt->pw);
/* Now we can close the file descriptor again */
close(0);
/* send messages generated by record_login */
buffer_put_string(m, buffer_ptr(&loginmsg), buffer_len(&loginmsg));
buffer_clear(&loginmsg);
mm_request_send(sock, MONITOR_ANS_PTY, m);
mm_send_fd(sock, s->ptyfd);
mm_send_fd(sock, s->ttyfd);
/* make sure nothing uses fd 0 */
if ((fd0 = open(_PATH_DEVNULL, O_RDONLY)) < 0)
fatal("%s: open(/dev/null): %s", __func__, strerror(errno));
if (fd0 != 0)
error("%s: fd0 %d != 0", __func__, fd0);
/* slave is not needed */
close(s->ttyfd);
s->ttyfd = s->ptyfd;
/* no need to dup() because nobody closes ptyfd */
s->ptymaster = s->ptyfd;
debug3("%s: tty %s ptyfd %d", __func__, s->tty, s->ttyfd);
return (0);
error:
if (s != NULL)
mm_session_close(s);
buffer_put_int(m, 0);
mm_request_send(sock, MONITOR_ANS_PTY, m);
return (0);
}
/*
* Encodes terminal modes for the terminal referenced by fd
* or tiop in a portable manner, and appends the modes to a packet
* being constructed.
*/
void
tty_make_modes(int fd, struct termios *tiop)
{
struct termios tio;
int baud;
Buffer buf;
int tty_op_ospeed, tty_op_ispeed;
void (*put_arg)(Buffer *, u_int);
buffer_init(&buf);
if (compat20) {
tty_op_ospeed = TTY_OP_OSPEED_PROTO2;
tty_op_ispeed = TTY_OP_ISPEED_PROTO2;
put_arg = buffer_put_int;
} else {
tty_op_ospeed = TTY_OP_OSPEED_PROTO1;
tty_op_ispeed = TTY_OP_ISPEED_PROTO1;
put_arg = (void (*)(Buffer *, u_int)) buffer_put_char;
}
if (tiop == NULL) {
if (tcgetattr(fd, &tio) == -1) {
/*logit("tcgetattr: %.100s", strerror(errno));*/
goto end;
}
} else
tio = *tiop;
/* Store input and output baud rates. */
baud = speed_to_baud(cfgetospeed(&tio));
debug3("tty_make_modes: ospeed %d", baud);
buffer_put_char(&buf, tty_op_ospeed);
buffer_put_int(&buf, baud);
baud = speed_to_baud(cfgetispeed(&tio));
debug3("tty_make_modes: ispeed %d", baud);
buffer_put_char(&buf, tty_op_ispeed);
buffer_put_int(&buf, baud);
/* Store values of mode flags. */
#define TTYCHAR(NAME, OP) \
debug3("tty_make_modes: %d %d", OP, tio.c_cc[NAME]); \
buffer_put_char(&buf, OP); \
put_arg(&buf, special_char_encode(tio.c_cc[NAME]));
#define TTYMODE(NAME, FIELD, OP) \
debug3("tty_make_modes: %d %d", OP, ((tio.FIELD & NAME) != 0)); \
buffer_put_char(&buf, OP); \
put_arg(&buf, ((tio.FIELD & NAME) != 0));
#include "ttymodes.h"
#undef TTYCHAR
#undef TTYMODE
end:
/* Mark end of mode data. */
buffer_put_char(&buf, TTY_OP_END);
if (compat20)
packet_put_string(buffer_ptr(&buf), buffer_len(&buf));
else
packet_put_raw(buffer_ptr(&buf), buffer_len(&buf));
buffer_free(&buf);
}
void
monitor_child_preauth(Authctxt *_authctxt, struct monitor *pmonitor)
{
struct mon_table *ent;
int authenticated = 0;
debug3("preauth child monitor started");
authctxt = _authctxt;
memset(authctxt, 0, sizeof(*authctxt));
authctxt->loginmsg = &loginmsg;
if (compat20) {
mon_dispatch = mon_dispatch_proto20;
/* Permit requests for moduli and signatures */
monitor_permit(mon_dispatch, MONITOR_REQ_MODULI, 1);
monitor_permit(mon_dispatch, MONITOR_REQ_SIGN, 1);
} else {
mon_dispatch = mon_dispatch_proto15;
monitor_permit(mon_dispatch, MONITOR_REQ_SESSKEY, 1);
}
/* The first few requests do not require asynchronous access */
while (!authenticated) {
auth_method = "unknown";
authenticated = (monitor_read(pmonitor, mon_dispatch, &ent) == 1);
if (authenticated) {
if (!(ent->flags & MON_AUTHDECIDE))
fatal("%s: unexpected authentication from %d",
__func__, ent->type);
if (authctxt->pw->pw_uid == 0 &&
!auth_root_allowed(auth_method))
authenticated = 0;
#ifdef USE_PAM
/* PAM needs to perform account checks after auth */
if (options.use_pam && authenticated) {
Buffer m;
buffer_init(&m);
mm_request_receive_expect(pmonitor->m_sendfd,
MONITOR_REQ_PAM_ACCOUNT, &m);
authenticated = mm_answer_pam_account(pmonitor->m_sendfd, &m);
buffer_free(&m);
}
#endif
}
if (ent->flags & (MON_AUTHDECIDE|MON_ALOG)) {
auth_log(authctxt, authenticated, auth_method,
compat20 ? " ssh2" : "");
if (!authenticated)
authctxt->failures++;
}
}
if (!authctxt->valid)
fatal("%s: authenticated invalid user", __func__);
if (strcmp(auth_method, "unknown") == 0)
fatal("%s: authentication method name unknown", __func__);
debug("%s: %s has been authenticated by privileged process",
__func__, authctxt->user);
mm_get_keystate(pmonitor);
}
/*
* Decodes terminal modes for the terminal referenced by fd in a portable
* manner from a packet being read.
*/
void
tty_parse_modes(int fd, int *n_bytes_ptr)
{
struct termios tio;
int opcode, baud;
int n_bytes = 0;
int failure = 0;
u_int (*get_arg)(void);
int arg, arg_size;
if (compat20) {
*n_bytes_ptr = packet_get_int();
debug3("tty_parse_modes: SSH2 n_bytes %d", *n_bytes_ptr);
if (*n_bytes_ptr == 0)
return;
get_arg = packet_get_int;
arg_size = 4;
} else {
get_arg = packet_get_char;
arg_size = 1;
}
/*
* Get old attributes for the terminal. We will modify these
* flags. I am hoping that if there are any machine-specific
* modes, they will initially have reasonable values.
*/
if (tcgetattr(fd, &tio) == -1) {
/*logit("tcgetattr: %.100s", strerror(errno));*/
failure = -1;
}
for (;;) {
n_bytes += 1;
opcode = packet_get_char();
switch (opcode) {
case TTY_OP_END:
goto set;
/* XXX: future conflict possible */
case TTY_OP_ISPEED_PROTO1:
case TTY_OP_ISPEED_PROTO2:
n_bytes += 4;
baud = packet_get_int();
debug3("tty_parse_modes: ispeed %d", baud);
if (failure != -1 &&
cfsetispeed(&tio, baud_to_speed(baud)) == -1)
error("cfsetispeed failed for %d", baud);
break;
/* XXX: future conflict possible */
case TTY_OP_OSPEED_PROTO1:
case TTY_OP_OSPEED_PROTO2:
n_bytes += 4;
baud = packet_get_int();
debug3("tty_parse_modes: ospeed %d", baud);
if (failure != -1 &&
cfsetospeed(&tio, baud_to_speed(baud)) == -1)
error("cfsetospeed failed for %d", baud);
break;
#define TTYCHAR(NAME, OP) \
case OP: \
n_bytes += arg_size; \
tio.c_cc[NAME] = special_char_decode(get_arg()); \
debug3("tty_parse_modes: %d %d", OP, tio.c_cc[NAME]); \
break;
#define TTYMODE(NAME, FIELD, OP) \
case OP: \
n_bytes += arg_size; \
if ((arg = get_arg())) \
tio.FIELD |= NAME; \
else \
tio.FIELD &= ~NAME; \
debug3("tty_parse_modes: %d %d", OP, arg); \
break;
#include "ttymodes.h"
#undef TTYCHAR
#undef TTYMODE
default:
debug("Ignoring unsupported tty mode opcode %d (0x%x)",
opcode, opcode);
if (!compat20) {
/*
* SSH1:
* Opcodes 1 to 127 are defined to have
* a one-byte argument.
* Opcodes 128 to 159 are defined to have
* an integer argument.
*/
if (opcode > 0 && opcode < 128) {
n_bytes += 1;
(void) packet_get_char();
break;
} else if (opcode >= 128 && opcode < 160) {
n_bytes += 4;
(void) packet_get_int();
break;
} else {
/*
* It is a truly undefined opcode (160 to 255).
* We have no idea about its arguments. So we
* must stop parsing. Note that some data
* may be left in the packet; hopefully there
* is nothing more coming after the mode data.
*/
logit("parse_tty_modes: unknown opcode %d",
opcode);
goto set;
}
} else {
/*
* SSH2:
* Opcodes 1 to 159 are defined to have
* a uint32 argument.
* Opcodes 160 to 255 are undefined and
* cause parsing to stop.
*/
if (opcode > 0 && opcode < 160) {
n_bytes += 4;
(void) packet_get_int();
break;
} else {
logit("parse_tty_modes: unknown opcode %d",
opcode);
goto set;
}
}
}
}
set:
if (*n_bytes_ptr != n_bytes) {
*n_bytes_ptr = n_bytes;
logit("parse_tty_modes: n_bytes_ptr != n_bytes: %d %d",
*n_bytes_ptr, n_bytes);
return; /* Don't process bytes passed */
}
if (failure == -1)
return; /* Packet parsed ok but tcgetattr() failed */
/* Set the new modes for the terminal. */
if (tcsetattr(fd, TCSANOW, &tio) == -1)
logit("Setting tty modes failed: %.100s", strerror(errno));
}
int
mm_answer_keyallowed(int sock, Buffer *m)
{
Key *key;
char *cuser, *chost;
u_char *blob;
u_int bloblen;
enum mm_keytype type = 0;
int allowed = 0;
debug3("%s entering", __func__);
type = buffer_get_int(m);
cuser = buffer_get_string(m, NULL);
chost = buffer_get_string(m, NULL);
blob = buffer_get_string(m, &bloblen);
key = key_from_blob(blob, bloblen);
if ((compat20 && type == MM_RSAHOSTKEY) ||
(!compat20 && type != MM_RSAHOSTKEY))
fatal("%s: key type and protocol mismatch", __func__);
debug3("%s: key_from_blob: %p", __func__, key);
if (key != NULL && authctxt->valid) {
switch (type) {
case MM_USERKEY:
allowed = options.pubkey_authentication &&
user_key_allowed(authctxt->pw, key);
auth_method = "publickey";
break;
case MM_HOSTKEY:
allowed = options.hostbased_authentication &&
hostbased_key_allowed(authctxt->pw,
cuser, chost, key);
auth_method = "hostbased";
break;
case MM_RSAHOSTKEY:
key->type = KEY_RSA1; /* XXX */
allowed = options.rhosts_rsa_authentication &&
auth_rhosts_rsa_key_allowed(authctxt->pw,
cuser, chost, key);
auth_method = "rsa";
break;
default:
fatal("%s: unknown key type %d", __func__, type);
break;
}
}
if (key != NULL)
key_free(key);
/* clear temporarily storage (used by verify) */
monitor_reset_key_state();
if (allowed) {
/* Save temporarily for comparison in verify */
key_blob = blob;
key_bloblen = bloblen;
key_blobtype = type;
hostbased_cuser = cuser;
hostbased_chost = chost;
} else {
/* Log failed attempt */
auth_log(authctxt, 0, auth_method, compat20 ? " ssh2" : "");
xfree(blob);
xfree(cuser);
xfree(chost);
}
debug3("%s: key %p is %s",
__func__, key, allowed ? "allowed" : "disallowed");
buffer_clear(m);
buffer_put_int(m, allowed);
buffer_put_int(m, forced_command != NULL);
mm_append_debug(m);
mm_request_send(sock, MONITOR_ANS_KEYALLOWED, m);
if (type == MM_RSAHOSTKEY)
monitor_permit(mon_dispatch, MONITOR_REQ_RSACHALLENGE, allowed);
return (0);
}
void
userauth_finish(Authctxt *authctxt, int authenticated, const char *method,
const char *submethod)
{
char *methods;
int partial = 0;
if (!authctxt->valid && authenticated)
fatal("INTERNAL ERROR: authenticated invalid user %s",
authctxt->user);
if (authenticated && authctxt->postponed)
fatal("INTERNAL ERROR: authenticated and postponed");
/* Special handling for root */
if (authenticated && authctxt->pw->pw_uid == 0 &&
!auth_root_allowed(method)) {
authenticated = 0;
#ifdef SSH_AUDIT_EVENTS
PRIVSEP(audit_event(SSH_LOGIN_ROOT_DENIED));
#endif
}
if (authenticated && options.num_auth_methods != 0) {
if (!auth2_update_methods_lists(authctxt, method, submethod)) {
authenticated = 0;
partial = 1;
}
}
/* Log before sending the reply */
auth_log(authctxt, authenticated, partial, method, submethod);
if (authctxt->postponed)
return;
#ifdef USE_PAM
if (options.use_pam && authenticated) {
if (!PRIVSEP(do_pam_account())) {
/* if PAM returned a message, send it to the user */
if (buffer_len(&loginmsg) > 0) {
buffer_append(&loginmsg, "\0", 1);
userauth_send_banner(buffer_ptr(&loginmsg));
packet_write_wait();
}
fatal("Access denied for user %s by PAM account "
"configuration", authctxt->user);
}
}
#endif
#ifdef _UNICOS
if (authenticated && cray_access_denied(authctxt->user)) {
authenticated = 0;
fatal("Access denied for user %s.", authctxt->user);
}
#endif /* _UNICOS */
if (authenticated == 1) {
/* turn off userauth */
dispatch_set(SSH2_MSG_USERAUTH_REQUEST, &dispatch_protocol_ignore);
packet_start(SSH2_MSG_USERAUTH_SUCCESS);
packet_send();
packet_write_wait();
/* now we can break out */
authctxt->success = 1;
} else {
/* Allow initial try of "none" auth without failure penalty */
if (!authctxt->server_caused_failure &&
(authctxt->attempt > 1 || strcmp(method, "none") != 0))
authctxt->failures++;
if (authctxt->failures >= options.max_authtries) {
#ifdef SSH_AUDIT_EVENTS
PRIVSEP(audit_event(SSH_LOGIN_EXCEED_MAXTRIES));
#endif
auth_maxtries_exceeded(authctxt);
}
methods = authmethods_get(authctxt);
debug3("%s: failure partial=%d next methods=\"%s\"", __func__,
partial, methods);
packet_start(SSH2_MSG_USERAUTH_FAILURE);
packet_put_cstring(methods);
packet_put_char(partial);
packet_send();
packet_write_wait();
free(methods);
}
}
/*
* Checks whether key is allowed in output of command.
* returns 1 if the key is allowed or 0 otherwise.
*/
static int
user_key_command_allowed2(struct passwd *user_pw, Key *key)
{
FILE *f;
int ok, found_key = 0;
struct passwd *pw;
struct stat st;
int status, devnull, p[2], i;
pid_t pid;
char *username, errmsg[512];
if (options.authorized_keys_command == NULL ||
options.authorized_keys_command[0] != '/')
return 0;
if (options.authorized_keys_command_user == NULL) {
error("No user for AuthorizedKeysCommand specified, skipping");
return 0;
}
username = percent_expand(options.authorized_keys_command_user,
"u", user_pw->pw_name, (char *)NULL);
pw = getpwnam(username);
if (pw == NULL) {
error("AuthorizedKeysCommandUser \"%s\" not found: %s",
username, strerror(errno));
free(username);
return 0;
}
free(username);
temporarily_use_uid(pw);
if (stat(options.authorized_keys_command, &st) < 0) {
error("Could not stat AuthorizedKeysCommand \"%s\": %s",
options.authorized_keys_command, strerror(errno));
goto out;
}
if (auth_secure_path(options.authorized_keys_command, &st, NULL, 0,
errmsg, sizeof(errmsg)) != 0) {
error("Unsafe AuthorizedKeysCommand: %s", errmsg);
goto out;
}
if (pipe(p) != 0) {
error("%s: pipe: %s", __func__, strerror(errno));
goto out;
}
debug3("Running AuthorizedKeysCommand: \"%s %s\" as \"%s\"",
options.authorized_keys_command, user_pw->pw_name, pw->pw_name);
/*
* Don't want to call this in the child, where it can fatal() and
* run cleanup_exit() code.
*/
restore_uid();
switch ((pid = fork())) {
case -1: /* error */
error("%s: fork: %s", __func__, strerror(errno));
close(p[0]);
close(p[1]);
return 0;
case 0: /* child */
for (i = 0; i < NSIG; i++)
signal(i, SIG_DFL);
if ((devnull = open(_PATH_DEVNULL, O_RDWR)) == -1) {
error("%s: open %s: %s", __func__, _PATH_DEVNULL,
strerror(errno));
_exit(1);
}
/* Keep stderr around a while longer to catch errors */
if (dup2(devnull, STDIN_FILENO) == -1 ||
dup2(p[1], STDOUT_FILENO) == -1) {
error("%s: dup2: %s", __func__, strerror(errno));
_exit(1);
}
closefrom(STDERR_FILENO + 1);
/* Don't use permanently_set_uid() here to avoid fatal() */
if (setresgid(pw->pw_gid, pw->pw_gid, pw->pw_gid) != 0) {
error("setresgid %u: %s", (u_int)pw->pw_gid,
strerror(errno));
_exit(1);
}
if (setresuid(pw->pw_uid, pw->pw_uid, pw->pw_uid) != 0) {
error("setresuid %u: %s", (u_int)pw->pw_uid,
strerror(errno));
_exit(1);
}
/* stdin is pointed to /dev/null at this point */
if (dup2(STDIN_FILENO, STDERR_FILENO) == -1) {
error("%s: dup2: %s", __func__, strerror(errno));
_exit(1);
}
execl(options.authorized_keys_command,
options.authorized_keys_command, user_pw->pw_name, NULL);
error("AuthorizedKeysCommand %s exec failed: %s",
options.authorized_keys_command, strerror(errno));
_exit(127);
default: /* parent */
break;
}
temporarily_use_uid(pw);
close(p[1]);
if ((f = fdopen(p[0], "r")) == NULL) {
error("%s: fdopen: %s", __func__, strerror(errno));
close(p[0]);
/* Don't leave zombie child */
kill(pid, SIGTERM);
while (waitpid(pid, NULL, 0) == -1 && errno == EINTR)
;
goto out;
}
ok = check_authkeys_file(f, options.authorized_keys_command, key, pw);
fclose(f);
while (waitpid(pid, &status, 0) == -1) {
if (errno != EINTR) {
error("%s: waitpid: %s", __func__, strerror(errno));
goto out;
}
}
if (WIFSIGNALED(status)) {
error("AuthorizedKeysCommand %s exited on signal %d",
options.authorized_keys_command, WTERMSIG(status));
goto out;
} else if (WEXITSTATUS(status) != 0) {
error("AuthorizedKeysCommand %s returned status %d",
options.authorized_keys_command, WEXITSTATUS(status));
goto out;
}
found_key = ok;
out:
restore_uid();
return found_key;
}
/* create a slurmd job structure from a launch tasks message */
slurmd_job_t *
job_create(launch_tasks_request_msg_t *msg)
{
struct passwd *pwd = NULL;
slurmd_job_t *job = NULL;
srun_info_t *srun = NULL;
slurm_addr_t resp_addr;
slurm_addr_t io_addr;
int nodeid = NO_VAL;
xassert(msg != NULL);
xassert(msg->complete_nodelist != NULL);
debug3("entering job_create");
if ((pwd = _pwd_create((uid_t)msg->uid)) == NULL) {
error("uid %ld not found on system", (long) msg->uid);
slurm_seterrno (ESLURMD_UID_NOT_FOUND);
return NULL;
}
if (!_valid_gid(pwd, &(msg->gid))) {
slurm_seterrno (ESLURMD_GID_NOT_FOUND);
_pwd_destroy(pwd);
return NULL;
}
if (msg->job_mem_lim && (msg->acctg_freq != (uint16_t) NO_VAL)
&& (msg->acctg_freq > conf->job_acct_gather_freq)) {
error("Can't set frequency to %u, it is higher than %u. "
"We need it to be at least at this level to "
"monitor memory usage.",
msg->acctg_freq, conf->job_acct_gather_freq);
slurm_seterrno (ESLURMD_INVALID_ACCT_FREQ);
_pwd_destroy(pwd);
return NULL;
}
job = xmalloc(sizeof(slurmd_job_t));
#ifndef HAVE_FRONT_END
nodeid = nodelist_find(msg->complete_nodelist, conf->node_name);
job->node_name = xstrdup(conf->node_name);
#else
nodeid = 0;
job->node_name = xstrdup(msg->complete_nodelist);
#endif
if(nodeid < 0) {
error("couldn't find node %s in %s",
job->node_name, msg->complete_nodelist);
job_destroy(job);
return NULL;
}
job->state = SLURMSTEPD_STEP_STARTING;
job->pwd = pwd;
job->node_tasks = msg->tasks_to_launch[nodeid];
job->ntasks = msg->ntasks;
job->jobid = msg->job_id;
job->stepid = msg->job_step_id;
job->uid = (uid_t) msg->uid;
job->gid = (gid_t) msg->gid;
job->cwd = xstrdup(msg->cwd);
job->task_dist = msg->task_dist;
job->cpu_bind_type = msg->cpu_bind_type;
job->cpu_bind = xstrdup(msg->cpu_bind);
job->mem_bind_type = msg->mem_bind_type;
job->mem_bind = xstrdup(msg->mem_bind);
job->ckpt_dir = xstrdup(msg->ckpt_dir);
job->restart_dir = xstrdup(msg->restart_dir);
job->cpus_per_task = msg->cpus_per_task;
job->env = _array_copy(msg->envc, msg->env);
job->eio = eio_handle_create();
job->sruns = list_create((ListDelF) _srun_info_destructor);
job->clients = list_create(NULL); /* FIXME! Needs destructor */
job->stdout_eio_objs = list_create(NULL); /* FIXME! Needs destructor */
job->stderr_eio_objs = list_create(NULL); /* FIXME! Needs destructor */
job->free_incoming = list_create(NULL); /* FIXME! Needs destructor */
job->incoming_count = 0;
job->free_outgoing = list_create(NULL); /* FIXME! Needs destructor */
job->outgoing_count = 0;
job->outgoing_cache = list_create(NULL); /* FIXME! Needs destructor */
job->envtp = xmalloc(sizeof(env_t));
job->envtp->jobid = -1;
job->envtp->stepid = -1;
job->envtp->procid = -1;
job->envtp->localid = -1;
job->envtp->nodeid = -1;
job->envtp->distribution = 0;
job->envtp->cpu_bind_type = 0;
job->envtp->cpu_bind = NULL;
job->envtp->mem_bind_type = 0;
job->envtp->mem_bind = NULL;
job->envtp->ckpt_dir = NULL;
job->envtp->comm_port = msg->resp_port[nodeid % msg->num_resp_port];
memcpy(&resp_addr, &msg->orig_addr, sizeof(slurm_addr_t));
slurm_set_addr(&resp_addr,
msg->resp_port[nodeid % msg->num_resp_port],
NULL);
job->user_managed_io = msg->user_managed_io;
if (!msg->user_managed_io) {
memcpy(&io_addr, &msg->orig_addr, sizeof(slurm_addr_t));
slurm_set_addr(&io_addr,
msg->io_port[nodeid % msg->num_io_port],
NULL);
}
srun = srun_info_create(msg->cred, &resp_addr, &io_addr);
job->buffered_stdio = msg->buffered_stdio;
job->labelio = msg->labelio;
job->task_prolog = xstrdup(msg->task_prolog);
job->task_epilog = xstrdup(msg->task_epilog);
job->argc = msg->argc;
job->argv = _array_copy(job->argc, msg->argv);
job->nnodes = msg->nnodes;
job->nodeid = nodeid;
job->debug = msg->slurmd_debug;
job->cpus = msg->cpus_allocated[nodeid];
if (msg->acctg_freq != (uint16_t) NO_VAL)
jobacct_gather_g_change_poll(msg->acctg_freq);
job->multi_prog = msg->multi_prog;
job->timelimit = (time_t) -1;
job->task_flags = msg->task_flags;
job->switch_job = msg->switch_job;
job->pty = msg->pty;
job->open_mode = msg->open_mode;
job->options = msg->options;
format_core_allocs(msg->cred, conf->node_name,
&job->job_alloc_cores, &job->step_alloc_cores,
&job->job_mem, &job->step_mem);
if (job->step_mem) {
jobacct_common_set_mem_limit(job->jobid, job->stepid,
job->step_mem);
} else if (job->job_mem) {
jobacct_common_set_mem_limit(job->jobid, job->stepid,
job->job_mem);
}
#ifdef HAVE_CRAY
/* This is only used for Cray emulation mode where slurmd is used to
* launch job steps. On a real Cray system, ALPS is used to launch
* the tasks instead of SLURM. SLURM's task launch RPC does NOT
* contain the reservation ID, so just use some non-zero value here
* for testing purposes. */
job->resv_id = 1;
select_g_select_jobinfo_set(msg->select_jobinfo, SELECT_JOBDATA_RESV_ID,
&job->resv_id);
#endif
get_cred_gres(msg->cred, conf->node_name,
&job->job_gres_list, &job->step_gres_list);
list_append(job->sruns, (void *) srun);
_job_init_task_info(job, msg->global_task_ids[nodeid],
msg->ifname, msg->ofname, msg->efname);
return job;
}
static Key *
key_load_private_rsa1(int fd, const char *filename, const char *passphrase,
char **commentp)
{
int i, check1, check2, cipher_type;
off_t len;
Buffer buffer, decrypted;
u_char *cp;
CipherContext ciphercontext;
Cipher *cipher;
Key *prv = NULL;
struct stat st;
if (fstat(fd, &st) < 0) {
error("fstat for key file %.200s failed: %.100s",
filename, strerror(errno));
close(fd);
return NULL;
}
len = st.st_size;
buffer_init(&buffer);
cp = buffer_append_space(&buffer, len);
if (read(fd, cp, (size_t) len) != (size_t) len) {
debug("Read from key file %.200s failed: %.100s", filename,
strerror(errno));
buffer_free(&buffer);
close(fd);
return NULL;
}
/* Check that it is at least big enough to contain the ID string. */
if (len < sizeof(authfile_id_string)) {
debug3("Not a RSA1 key file %.200s.", filename);
buffer_free(&buffer);
close(fd);
return NULL;
}
/*
* Make sure it begins with the id string. Consume the id string
* from the buffer.
*/
for (i = 0; i < sizeof(authfile_id_string); i++)
if (buffer_get_char(&buffer) != authfile_id_string[i]) {
debug3("Not a RSA1 key file %.200s.", filename);
buffer_free(&buffer);
close(fd);
return NULL;
}
/* Read cipher type. */
cipher_type = buffer_get_char(&buffer);
(void) buffer_get_int(&buffer); /* Reserved data. */
/* Read the public key from the buffer. */
(void) buffer_get_int(&buffer);
prv = key_new_private(KEY_RSA1);
buffer_get_bignum(&buffer, prv->rsa->n);
buffer_get_bignum(&buffer, prv->rsa->e);
if (commentp)
*commentp = buffer_get_string(&buffer, NULL);
else
xfree(buffer_get_string(&buffer, NULL));
/* Check that it is a supported cipher. */
cipher = cipher_by_number(cipher_type);
if (cipher == NULL) {
debug("Unsupported cipher %d used in key file %.200s.",
cipher_type, filename);
buffer_free(&buffer);
goto fail;
}
/* Initialize space for decrypted data. */
buffer_init(&decrypted);
cp = buffer_append_space(&decrypted, buffer_len(&buffer));
/* Rest of the buffer is encrypted. Decrypt it using the passphrase. */
cipher_set_key_string(&ciphercontext, cipher, passphrase,
CIPHER_DECRYPT);
cipher_crypt(&ciphercontext, cp,
buffer_ptr(&buffer), buffer_len(&buffer));
cipher_cleanup(&ciphercontext);
memset(&ciphercontext, 0, sizeof(ciphercontext));
buffer_free(&buffer);
check1 = buffer_get_char(&decrypted);
check2 = buffer_get_char(&decrypted);
if (check1 != buffer_get_char(&decrypted) ||
check2 != buffer_get_char(&decrypted)) {
if (strcmp(passphrase, "") != 0)
debug("Bad passphrase supplied for key file %.200s.",
filename);
/* Bad passphrase. */
buffer_free(&decrypted);
goto fail;
}
/* Read the rest of the private key. */
buffer_get_bignum(&decrypted, prv->rsa->d);
buffer_get_bignum(&decrypted, prv->rsa->iqmp); /* u */
/* in SSL and SSH v1 p and q are exchanged */
buffer_get_bignum(&decrypted, prv->rsa->q); /* p */
buffer_get_bignum(&decrypted, prv->rsa->p); /* q */
/* calculate p-1 and q-1 */
rsa_generate_additional_parameters(prv->rsa);
buffer_free(&decrypted);
/* enable blinding */
if (RSA_blinding_on(prv->rsa, NULL) != 1) {
error("key_load_private_rsa1: RSA_blinding_on failed");
goto fail;
}
close(fd);
return prv;
fail:
if (commentp)
xfree(*commentp);
close(fd);
key_free(prv);
return NULL;
}
slurmd_job_t *
job_batch_job_create(batch_job_launch_msg_t *msg)
{
struct passwd *pwd;
slurmd_job_t *job;
srun_info_t *srun = NULL;
char *in_name;
xassert(msg != NULL);
debug3("entering batch_job_create");
if ((pwd = _pwd_create((uid_t)msg->uid)) == NULL) {
error("uid %ld not found on system", (long) msg->uid);
slurm_seterrno (ESLURMD_UID_NOT_FOUND);
return NULL;
}
if (!_valid_gid(pwd, &(msg->gid))) {
slurm_seterrno (ESLURMD_GID_NOT_FOUND);
_pwd_destroy(pwd);
return NULL;
}
if(msg->job_mem && (msg->acctg_freq != (uint16_t) NO_VAL)
&& (msg->acctg_freq > conf->job_acct_gather_freq)) {
error("Can't set frequency to %u, it is higher than %u. "
"We need it to be at least at this level to "
"monitor memory usage.",
msg->acctg_freq, conf->job_acct_gather_freq);
slurm_seterrno (ESLURMD_INVALID_ACCT_FREQ);
_pwd_destroy(pwd);
return NULL;
}
job = xmalloc(sizeof(slurmd_job_t));
job->state = SLURMSTEPD_STEP_STARTING;
job->pwd = pwd;
job->cpus = msg->cpus_per_node[0];
job->node_tasks = 1;
job->ntasks = msg->ntasks;
job->jobid = msg->job_id;
job->stepid = msg->step_id;
job->batch = true;
if (msg->acctg_freq != (uint16_t) NO_VAL)
jobacct_gather_g_change_poll(msg->acctg_freq);
job->multi_prog = 0;
job->open_mode = msg->open_mode;
job->overcommit = (bool) msg->overcommit;
job->node_name = xstrdup(conf->node_name);
job->uid = (uid_t) msg->uid;
job->gid = (gid_t) msg->gid;
job->cwd = xstrdup(msg->work_dir);
job->ckpt_dir = xstrdup(msg->ckpt_dir);
job->restart_dir = xstrdup(msg->restart_dir);
job->env = _array_copy(msg->envc, msg->environment);
job->eio = eio_handle_create();
job->sruns = list_create((ListDelF) _srun_info_destructor);
job->envtp = xmalloc(sizeof(env_t));
job->envtp->jobid = -1;
job->envtp->stepid = -1;
job->envtp->procid = -1;
job->envtp->localid = -1;
job->envtp->nodeid = -1;
job->envtp->distribution = 0;
job->cpu_bind_type = msg->cpu_bind_type;
job->cpu_bind = xstrdup(msg->cpu_bind);
job->envtp->mem_bind_type = 0;
job->envtp->mem_bind = NULL;
job->envtp->ckpt_dir = NULL;
job->envtp->restart_cnt = msg->restart_cnt;
job->cpus_per_task = msg->cpus_per_node[0];
format_core_allocs(msg->cred, conf->node_name,
&job->job_alloc_cores, &job->step_alloc_cores,
&job->job_mem, &job->step_mem);
if (job->step_mem) {
jobacct_common_set_mem_limit(job->jobid, NO_VAL,
job->step_mem);
} else if (job->job_mem)
jobacct_common_set_mem_limit(job->jobid, NO_VAL, job->job_mem);
get_cred_gres(msg->cred, conf->node_name,
&job->job_gres_list, &job->step_gres_list);
srun = srun_info_create(NULL, NULL, NULL);
list_append(job->sruns, (void *) srun);
if (msg->argc) {
job->argc = msg->argc;
job->argv = _array_copy(job->argc, msg->argv);
} else {
job->argc = 1;
/* job script has not yet been written out to disk --
* argv will be filled in later by _make_batch_script()
*/
job->argv = (char **) xmalloc(2 * sizeof(char *));
}
job->task = xmalloc(sizeof(slurmd_task_info_t *));
if (msg->std_err == NULL)
msg->std_err = xstrdup(msg->std_out);
if (msg->std_in == NULL)
in_name = xstrdup("/dev/null");
else
in_name = fname_create(job, msg->std_in, 0);
job->task[0] = task_info_create(0, 0,
in_name,
_batchfilename(job, msg->std_out),
_batchfilename(job, msg->std_err));
job->task[0]->argc = job->argc;
job->task[0]->argv = job->argv;
#ifdef HAVE_CRAY
select_g_select_jobinfo_get(msg->select_jobinfo, SELECT_JOBDATA_RESV_ID,
&job->resv_id);
#endif
return job;
}
static int
_msg_socket_accept(eio_obj_t *obj, List objs)
{
stepd_step_rec_t *job = (stepd_step_rec_t *)obj->arg;
int fd;
struct sockaddr_un addr;
int len = sizeof(addr);
struct request_params *param = NULL;
pthread_attr_t attr;
pthread_t id;
int retries = 0;
debug3("Called _msg_socket_accept");
while ((fd = accept(obj->fd, (struct sockaddr *)&addr,
(socklen_t *)&len)) < 0) {
if (errno == EINTR)
continue;
if (errno == EAGAIN
|| errno == ECONNABORTED
|| errno == EWOULDBLOCK) {
return SLURM_SUCCESS;
}
error("Error on msg accept socket: %m");
obj->shutdown = true;
return SLURM_SUCCESS;
}
pthread_mutex_lock(&message_lock);
message_connections++;
pthread_mutex_unlock(&message_lock);
fd_set_close_on_exec(fd);
fd_set_blocking(fd);
slurm_attr_init(&attr);
if (pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED) != 0) {
error("Unable to set detachstate on attr: %m");
slurm_attr_destroy(&attr);
close(fd);
return SLURM_ERROR;
}
param = xmalloc(sizeof(struct request_params));
param->fd = fd;
param->job = job;
while (pthread_create(&id, &attr, &_handle_accept, (void *)param)) {
error("stepd_api message engine pthread_create: %m");
if (++retries > MAX_RETRIES) {
error("running handle_accept without "
"starting a thread stepd will be "
"unresponsive until done");
_handle_accept((void *)param);
info("stepd should be responsive now");
break;
}
usleep(10); /* sleep and again */
}
slurm_attr_destroy(&attr);
param = NULL;
debug3("Leaving _msg_socket_accept");
return SLURM_SUCCESS;
}
/* sync up core bitmap with new CPU count using a best-fit approach
* on the available resources on each node
*
* "Best-fit" means:
* 1st priority: Use smallest number of boards with sufficient
* available CPUs
* 2nd priority: Use smallest number of sockets with sufficient
* available CPUs
* 3rd priority: Use board combination with the smallest number
* of available CPUs
* 4th priority: Use higher-numbered boards/sockets/cores first
*
* The CPU array contains the distribution of CPUs, which can include
* virtual CPUs (hyperthreads)
*/
static void _block_sync_core_bitmap(struct job_record *job_ptr,
const uint16_t cr_type)
{
uint32_t c, s, i, j, n, b, z, size, csize, core_cnt;
uint16_t cpus, num_bits, vpus = 1;
uint16_t cpus_per_task = job_ptr->details->cpus_per_task;
job_resources_t *job_res = job_ptr->job_resrcs;
bool alloc_cores = false, alloc_sockets = false;
uint16_t ntasks_per_core = 0xffff;
int tmp_cpt = 0;
int count, cpu_min, b_min, elig, s_min, comb_idx, sock_idx;
int elig_idx, comb_brd_idx, sock_list_idx, comb_min, board_num;
int* boards_core_cnt;
int* sort_brds_core_cnt;
int* board_combs;
int* socket_list;
int* elig_brd_combs;
int* elig_core_cnt;
bool* sockets_used;
uint16_t boards_nb;
uint16_t nboards_nb;
uint16_t sockets_nb;
uint16_t ncores_nb;
uint16_t nsockets_nb;
uint16_t sock_per_brd;
uint16_t sock_per_comb;
uint16_t req_cores,best_fit_cores = 0;
uint32_t best_fit_location = 0;
uint64_t ncomb_brd;
bool sufficient, best_fit_sufficient;
if (!job_res)
return;
if (!job_res->core_bitmap) {
error("%s: core_bitmap for job %u is NULL",
__func__, job_ptr->job_id);
return;
}
if (bit_ffs(job_res->core_bitmap) == -1) {
error("%s: core_bitmap for job %u has no bits set",
__func__, job_ptr->job_id);
return;
}
if (cr_type & CR_SOCKET)
alloc_sockets = true;
else if (cr_type & CR_CORE)
alloc_cores = true;
if (job_ptr->details && job_ptr->details->mc_ptr) {
multi_core_data_t *mc_ptr = job_ptr->details->mc_ptr;
if ((mc_ptr->ntasks_per_core != (uint16_t) INFINITE) &&
(mc_ptr->ntasks_per_core)) {
ntasks_per_core = mc_ptr->ntasks_per_core;
}
}
size = bit_size(job_res->node_bitmap);
csize = bit_size(job_res->core_bitmap);
sockets_nb = select_node_record[0].sockets;
sockets_core_cnt = xmalloc(sockets_nb * sizeof(int));
sockets_used = xmalloc(sockets_nb * sizeof(bool));
boards_nb = select_node_record[0].boards;
boards_core_cnt = xmalloc(boards_nb * sizeof(int));
sort_brds_core_cnt = xmalloc(boards_nb * sizeof(int));
for (c = 0, i = 0, n = 0; n < size; n++) {
if (bit_test(job_res->node_bitmap, n) == 0)
continue;
core_cnt = 0;
ncores_nb = select_node_record[n].cores;
nsockets_nb = select_node_record[n].sockets;
nboards_nb = select_node_record[n].boards;
num_bits = nsockets_nb * ncores_nb;
if ((c + num_bits) > csize)
fatal("cons_res: _block_sync_core_bitmap index error");
cpus = job_res->cpus[i];
vpus = cr_cpus_per_core(job_ptr->details, n);
/* compute still required cores on the node */
req_cores = cpus / vpus;
if ( cpus % vpus )
req_cores++;
/* figure out core cnt if task requires more than one core and
* tasks_per_core is 1 */
if ((ntasks_per_core == 1) &&
(cpus_per_task > vpus)) {
/* how many cores a task will consume */
int cores_per_task = (cpus_per_task + vpus - 1) / vpus;
int tasks = cpus / cpus_per_task;
req_cores = tasks * cores_per_task;
}
if (nboards_nb > MAX_BOARDS) {
debug3("cons_res: node[%u]: exceeds max boards; "
"doing best-fit across sockets only", n);
nboards_nb = 1;
}
if ( nsockets_nb > sockets_nb) {
sockets_nb = nsockets_nb;
xrealloc(sockets_core_cnt, sockets_nb * sizeof(int));
xrealloc(sockets_used,sockets_nb * sizeof(bool));
}
if ( nboards_nb > boards_nb) {
boards_nb = nboards_nb;
xrealloc(boards_core_cnt, boards_nb * sizeof(int));
xrealloc(sort_brds_core_cnt, boards_nb * sizeof(int));
}
/* Count available cores on each socket and board */
if (nsockets_nb >= nboards_nb) {
sock_per_brd = nsockets_nb / nboards_nb;
} else {
error("Node socket count lower than board count "
"(%u < %u), job %u node %s",
nsockets_nb, nboards_nb, job_ptr->job_id,
node_record_table_ptr[n].name);
sock_per_brd = 1;
}
for (b = 0; b < nboards_nb; b++) {
boards_core_cnt[b] = 0;
sort_brds_core_cnt[b] = 0;
}
for (s = 0; s < nsockets_nb; s++) {
sockets_core_cnt[s]=0;
sockets_used[s]=false;
b = s/sock_per_brd;
for ( j = c + (s * ncores_nb) ;
j < c + ((s+1) * ncores_nb) ;
j++ ) {
if ( bit_test(job_res->core_bitmap,j) ) {
sockets_core_cnt[s]++;
boards_core_cnt[b]++;
sort_brds_core_cnt[b]++;
}
}
}
/* Sort boards in descending order of available core count */
qsort(sort_brds_core_cnt, nboards_nb, sizeof (int),
_cmp_int_descend);
/* Determine minimum number of boards required for the
* allocation (b_min) */
count = 0;
for (b = 0; b < nboards_nb; b++) {
count+=sort_brds_core_cnt[b];
if (count >= req_cores)
break;
}
b_min = b+1;
sock_per_comb = b_min * sock_per_brd;
/* Allocate space for list of board combinations */
ncomb_brd = comb_counts[nboards_nb-1][b_min-1];
board_combs = xmalloc(ncomb_brd * b_min * sizeof(int));
/* Generate all combinations of b_min boards on the node */
_gen_combs(board_combs, nboards_nb, b_min);
/* Determine which combinations have enough available cores
* for the allocation (eligible board combinations)
*/
elig_brd_combs = xmalloc(ncomb_brd * sizeof(int));
elig_core_cnt = xmalloc(ncomb_brd * sizeof(int));
elig = 0;
for (comb_idx = 0; comb_idx < ncomb_brd; comb_idx++) {
count = 0;
for (comb_brd_idx = 0; comb_brd_idx < b_min;
comb_brd_idx++) {
board_num = board_combs[(comb_idx * b_min)
+ comb_brd_idx];
count += boards_core_cnt[board_num];
}
if (count >= req_cores) {
elig_brd_combs[elig] = comb_idx;
elig_core_cnt[elig] = count;
elig++;
}
}
/* Allocate space for list of sockets for each eligible board
* combination */
socket_list = xmalloc(elig * sock_per_comb * sizeof(int));
/* Generate sorted list of sockets for each eligible board
* combination, and find combination with minimum number
* of sockets and minimum number of cpus required for the
* allocation
*/
s_min = sock_per_comb;
comb_min = 0;
cpu_min = sock_per_comb * ncores_nb;
for (elig_idx = 0; elig_idx < elig; elig_idx++) {
comb_idx = elig_brd_combs[elig_idx];
for (comb_brd_idx = 0; comb_brd_idx < b_min;
comb_brd_idx++) {
board_num = board_combs[(comb_idx * b_min)
+ comb_brd_idx];
sock_list_idx = (elig_idx * sock_per_comb) +
(comb_brd_idx * sock_per_brd);
for (sock_idx = 0; sock_idx < sock_per_brd;
sock_idx++) {
socket_list[sock_list_idx + sock_idx]
= (board_num * sock_per_brd)
+ sock_idx;
}
}
/* Sort this socket list in descending order of
* available core count */
qsort(&socket_list[elig_idx*sock_per_comb],
sock_per_comb, sizeof (int), _cmp_sock);
/* Determine minimum number of sockets required for
* the allocation from this socket list */
count = 0;
for (b = 0; b < sock_per_comb; b++) {
sock_idx =
socket_list[(int)((elig_idx*sock_per_comb)+b)];
count+=sockets_core_cnt[sock_idx];
if (count >= req_cores)
break;
}
b++;
/* Use board combination with minimum number
* of required sockets and minimum number of CPUs
*/
if ((b < s_min) ||
(b == s_min && elig_core_cnt[elig_idx]
<= cpu_min)) {
s_min = b;
comb_min = elig_idx;
cpu_min = elig_core_cnt[elig_idx];
}
}
if (select_debug_flags & DEBUG_FLAG_SELECT_TYPE) {
info("cons_res: best_fit: node[%u]: "
"required cpus: %u, min req boards: %u,",
n, cpus, b_min);
info("cons_res: best_fit: node[%u]: "
"min req sockets: %u, min avail cores: %u",
n, s_min, cpu_min);
}
/* Re-sort socket list for best-fit board combination in
* ascending order of socket number */
qsort(&socket_list[comb_min * sock_per_comb], sock_per_comb,
sizeof (int), _cmp_int_ascend);
xfree(board_combs);
xfree(elig_brd_combs);
xfree(elig_core_cnt);
/* select cores from the sockets of the best-fit board
* combination using a best-fit approach */
tmp_cpt = cpus_per_task;
while ( cpus > 0 ) {
best_fit_cores = 0;
best_fit_sufficient = false;
/* search for the socket with best fit */
for ( z = 0; z < sock_per_comb; z++ ) {
s = socket_list[(comb_min*sock_per_comb)+z];
sufficient = sockets_core_cnt[s] >= req_cores;
if ( (best_fit_cores == 0) ||
(sufficient && !best_fit_sufficient ) ||
(sufficient && (sockets_core_cnt[s] <
best_fit_cores)) ||
(!sufficient && (sockets_core_cnt[s] >
best_fit_cores)) ) {
best_fit_cores = sockets_core_cnt[s];
best_fit_location = s;
best_fit_sufficient = sufficient;
}
}
/* check that we have found a usable socket */
if ( best_fit_cores == 0 )
break;
j = best_fit_location;
if (sock_per_brd)
j /= sock_per_brd;
if (select_debug_flags & DEBUG_FLAG_SELECT_TYPE) {
info("cons_res: best_fit: using node[%u]: "
"board[%u]: socket[%u]: %u cores "
"available", n, j, best_fit_location,
sockets_core_cnt[best_fit_location]);
}
sockets_used[best_fit_location] = true;
for ( j = (c + (best_fit_location * ncores_nb));
j < (c + ((best_fit_location + 1) * ncores_nb));
j++ ) {
/*
* if no more cpus to select
* release remaining cores unless
* we are allocating whole sockets
*/
if (cpus == 0) {
if (alloc_sockets) {
bit_set(job_res->core_bitmap,j);
core_cnt++;
} else {
bit_clear(job_res->core_bitmap,j);
}
continue;
}
/*
* remove cores from socket count and
* cpus count using hyperthreading requirement
*/
if ( bit_test(job_res->core_bitmap, j) ) {
sockets_core_cnt[best_fit_location]--;
core_cnt++;
if (cpus < vpus)
cpus = 0;
else if ((ntasks_per_core == 1) &&
(cpus_per_task > vpus)) {
int used = MIN(tmp_cpt, vpus);
cpus -= used;
if (tmp_cpt <= used)
tmp_cpt = cpus_per_task;
else
tmp_cpt -= used;
} else {
cpus -= vpus;
}
} else if (alloc_sockets) {
/* If the core is not used, add it
* anyway if allocating whole sockets */
bit_set(job_res->core_bitmap, j);
core_cnt++;
}
}
/* loop again if more cpus required */
if ( cpus > 0 )
continue;
/* release remaining cores of the unused sockets */
for (s = 0; s < nsockets_nb; s++) {
if ( sockets_used[s] )
continue;
bit_nclear(job_res->core_bitmap,
c+(s*ncores_nb),
c+((s+1)*ncores_nb)-1);
}
}
xfree(socket_list);
if (cpus > 0) {
/* cpu count should NEVER be greater than the number
* of set bits in the core bitmap for a given node */
fatal("cons_res: cpus computation error");
}
/* adjust cpus count of the current node */
if ((alloc_cores || alloc_sockets) &&
(select_node_record[n].vpus >= 1)) {
job_res->cpus[i] = core_cnt *
select_node_record[n].vpus;
}
i++;
/* move c to the next node in core_bitmap */
c += num_bits;
}
xfree(boards_core_cnt);
xfree(sort_brds_core_cnt);
xfree(sockets_core_cnt);
xfree(sockets_used);
}
int
_handle_request(int fd, stepd_step_rec_t *job, uid_t uid, gid_t gid)
{
int rc = 0;
int req;
debug3("Entering _handle_request");
if ((rc = read(fd, &req, sizeof(int))) != sizeof(int)) {
if (rc == 0) { /* EOF, normal */
return -1;
} else {
debug3("Leaving _handle_request on read error: %m");
return SLURM_FAILURE;
}
}
debug3("Got request %d", req);
rc = SLURM_SUCCESS;
switch (req) {
case REQUEST_SIGNAL_TASK_LOCAL:
debug("Handling REQUEST_SIGNAL_TASK_LOCAL");
rc = _handle_signal_task_local(fd, job, uid);
break;
case REQUEST_SIGNAL_TASK_GLOBAL:
debug("Handling REQUEST_SIGNAL_TASK_GLOBAL (not implemented)");
break;
case REQUEST_SIGNAL_PROCESS_GROUP: /* Defunct */
case REQUEST_SIGNAL_CONTAINER:
debug("Handling REQUEST_SIGNAL_CONTAINER");
rc = _handle_signal_container(fd, job, uid);
break;
case REQUEST_CHECKPOINT_TASKS:
debug("Handling REQUEST_CHECKPOINT_TASKS");
rc = _handle_checkpoint_tasks(fd, job, uid);
break;
case REQUEST_STATE:
debug("Handling REQUEST_STATE");
rc = _handle_state(fd, job);
break;
case REQUEST_INFO:
debug("Handling REQUEST_INFO");
rc = _handle_info(fd, job);
break;
case REQUEST_ATTACH:
debug("Handling REQUEST_ATTACH");
rc = _handle_attach(fd, job, uid);
break;
case REQUEST_PID_IN_CONTAINER:
debug("Handling REQUEST_PID_IN_CONTAINER");
rc = _handle_pid_in_container(fd, job);
break;
case REQUEST_DAEMON_PID:
debug("Handling REQUEST_DAEMON_PID");
rc = _handle_daemon_pid(fd, job);
break;
case REQUEST_STEP_SUSPEND:
debug("Handling REQUEST_STEP_SUSPEND");
rc = _handle_suspend(fd, job, uid);
break;
case REQUEST_STEP_RESUME:
debug("Handling REQUEST_STEP_RESUME");
rc = _handle_resume(fd, job, uid);
break;
case REQUEST_STEP_TERMINATE:
debug("Handling REQUEST_STEP_TERMINATE");
rc = _handle_terminate(fd, job, uid);
break;
case REQUEST_STEP_COMPLETION_V2:
debug("Handling REQUEST_STEP_COMPLETION_V2");
rc = _handle_completion(fd, job, uid);
break;
case REQUEST_STEP_TASK_INFO:
debug("Handling REQUEST_STEP_TASK_INFO");
rc = _handle_task_info(fd, job);
break;
case REQUEST_STEP_STAT:
debug("Handling REQUEST_STEP_STAT");
rc = _handle_stat_jobacct(fd, job, uid);
break;
case REQUEST_STEP_LIST_PIDS:
debug("Handling REQUEST_STEP_LIST_PIDS");
rc = _handle_list_pids(fd, job);
break;
case REQUEST_STEP_RECONFIGURE:
debug("Handling REQUEST_STEP_RECONFIGURE");
rc = _handle_reconfig(fd, job, uid);
break;
case REQUEST_JOB_NOTIFY:
debug("Handling REQUEST_JOB_NOTIFY");
rc = _handle_notify_job(fd, job, uid);
break;
default:
error("Unrecognized request: %d", req);
rc = SLURM_FAILURE;
break;
}
debug3("Leaving _handle_request: %s",
rc ? "SLURM_FAILURE" : "SLURM_SUCCESS");
return rc;
}
extern int as_mysql_hourly_rollup(mysql_conn_t *mysql_conn,
char *cluster_name,
time_t start, time_t end,
uint16_t archive_data)
{
int rc = SLURM_SUCCESS;
int add_sec = 3600;
int i=0;
time_t now = time(NULL);
time_t curr_start = start;
time_t curr_end = curr_start + add_sec;
char *query = NULL;
MYSQL_RES *result = NULL;
MYSQL_ROW row;
ListIterator a_itr = NULL;
ListIterator c_itr = NULL;
ListIterator w_itr = NULL;
ListIterator r_itr = NULL;
List assoc_usage_list = list_create(_destroy_local_id_usage);
List cluster_down_list = list_create(_destroy_local_cluster_usage);
List wckey_usage_list = list_create(_destroy_local_id_usage);
List resv_usage_list = list_create(_destroy_local_resv_usage);
uint16_t track_wckey = slurm_get_track_wckey();
/* char start_char[20], end_char[20]; */
char *job_req_inx[] = {
"job_db_inx",
"id_job",
"id_assoc",
"id_wckey",
"time_eligible",
"time_start",
"time_end",
"time_suspended",
"cpus_alloc",
"cpus_req",
"id_resv"
};
char *job_str = NULL;
enum {
JOB_REQ_DB_INX,
JOB_REQ_JOBID,
JOB_REQ_ASSOCID,
JOB_REQ_WCKEYID,
JOB_REQ_ELG,
JOB_REQ_START,
JOB_REQ_END,
JOB_REQ_SUSPENDED,
JOB_REQ_ACPU,
JOB_REQ_RCPU,
JOB_REQ_RESVID,
JOB_REQ_COUNT
};
char *suspend_req_inx[] = {
"time_start",
"time_end"
};
char *suspend_str = NULL;
enum {
SUSPEND_REQ_START,
SUSPEND_REQ_END,
SUSPEND_REQ_COUNT
};
char *resv_req_inx[] = {
"id_resv",
"assoclist",
"cpus",
"flags",
"time_start",
"time_end"
};
char *resv_str = NULL;
enum {
RESV_REQ_ID,
RESV_REQ_ASSOCS,
RESV_REQ_CPU,
RESV_REQ_FLAGS,
RESV_REQ_START,
RESV_REQ_END,
RESV_REQ_COUNT
};
i=0;
xstrfmtcat(job_str, "%s", job_req_inx[i]);
for(i=1; i<JOB_REQ_COUNT; i++) {
xstrfmtcat(job_str, ", %s", job_req_inx[i]);
}
i=0;
xstrfmtcat(suspend_str, "%s", suspend_req_inx[i]);
for(i=1; i<SUSPEND_REQ_COUNT; i++) {
xstrfmtcat(suspend_str, ", %s", suspend_req_inx[i]);
}
i=0;
xstrfmtcat(resv_str, "%s", resv_req_inx[i]);
for(i=1; i<RESV_REQ_COUNT; i++) {
xstrfmtcat(resv_str, ", %s", resv_req_inx[i]);
}
/* info("begin start %s", ctime(&curr_start)); */
/* info("begin end %s", ctime(&curr_end)); */
a_itr = list_iterator_create(assoc_usage_list);
c_itr = list_iterator_create(cluster_down_list);
w_itr = list_iterator_create(wckey_usage_list);
r_itr = list_iterator_create(resv_usage_list);
while (curr_start < end) {
int last_id = -1;
int last_wckeyid = -1;
int seconds = 0;
int tot_time = 0;
local_cluster_usage_t *loc_c_usage = NULL;
local_cluster_usage_t *c_usage = NULL;
local_resv_usage_t *r_usage = NULL;
local_id_usage_t *a_usage = NULL;
local_id_usage_t *w_usage = NULL;
debug3("%s curr hour is now %ld-%ld",
cluster_name, curr_start, curr_end);
/* info("start %s", ctime(&curr_start)); */
/* info("end %s", ctime(&curr_end)); */
c_usage = _setup_cluster_usage(mysql_conn, cluster_name,
curr_start, curr_end,
cluster_down_list);
// now get the reservations during this time
query = xstrdup_printf("select %s from \"%s_%s\" where "
"(time_start < %ld && time_end >= %ld) "
"order by time_start",
resv_str, cluster_name, resv_table,
curr_end, curr_start);
debug3("%d(%s:%d) query\n%s",
mysql_conn->conn, THIS_FILE, __LINE__, query);
if (!(result = mysql_db_query_ret(
mysql_conn, query, 0))) {
xfree(query);
_destroy_local_cluster_usage(c_usage);
return SLURM_ERROR;
}
xfree(query);
/* If a reservation overlaps another reservation we
total up everything here as if they didn't but when
calculating the total time for a cluster we will
remove the extra time received. This may result in
unexpected results with association based reports
since the association is given the total amount of
time of each reservation, thus equaling more time
than is available. Job/Cluster/Reservation reports
should be fine though since we really don't over
allocate resources. The issue with us not being
able to handle overlapping reservations here is
unless the reservation completely overlaps the
other reservation we have no idea how many cpus
should be removed since this could be a
heterogeneous system. This same problem exists
when a reservation is created with the ignore_jobs
option which will allow jobs to continue to run in the
reservation that aren't suppose to.
*/
while ((row = mysql_fetch_row(result))) {
time_t row_start = slurm_atoul(row[RESV_REQ_START]);
time_t row_end = slurm_atoul(row[RESV_REQ_END]);
uint32_t row_cpu = slurm_atoul(row[RESV_REQ_CPU]);
uint32_t row_flags = slurm_atoul(row[RESV_REQ_FLAGS]);
if (row_start < curr_start)
row_start = curr_start;
if (!row_end || row_end > curr_end)
row_end = curr_end;
/* Don't worry about it if the time is less
* than 1 second.
*/
if ((row_end - row_start) < 1)
continue;
r_usage = xmalloc(sizeof(local_resv_usage_t));
r_usage->id = slurm_atoul(row[RESV_REQ_ID]);
r_usage->local_assocs = list_create(slurm_destroy_char);
slurm_addto_char_list(r_usage->local_assocs,
row[RESV_REQ_ASSOCS]);
r_usage->total_time = (row_end - row_start) * row_cpu;
r_usage->start = row_start;
r_usage->end = row_end;
list_append(resv_usage_list, r_usage);
/* Since this reservation was added to the
cluster and only certain people could run
there we will use this as allocated time on
the system. If the reservation was a
maintenance then we add the time to planned
down time.
*/
/* only record time for the clusters that have
registered. This continue should rarely if
ever happen.
*/
if (!c_usage)
continue;
else if (row_flags & RESERVE_FLAG_MAINT)
c_usage->pd_cpu += r_usage->total_time;
else
c_usage->a_cpu += r_usage->total_time;
/* slurm_make_time_str(&r_usage->start, start_char, */
/* sizeof(start_char)); */
/* slurm_make_time_str(&r_usage->end, end_char, */
/* sizeof(end_char)); */
/* info("adding this much %lld to cluster %s " */
/* "%d %d %s - %s", */
/* r_usage->total_time, c_usage->name, */
/* (row_flags & RESERVE_FLAG_MAINT), */
/* r_usage->id, start_char, end_char); */
}
mysql_free_result(result);
/* now get the jobs during this time only */
query = xstrdup_printf("select %s from \"%s_%s\" where "
"(time_eligible < %ld && "
"(time_end >= %ld || time_end = 0)) "
"order by id_assoc, time_eligible",
job_str, cluster_name, job_table,
curr_end, curr_start);
debug3("%d(%s:%d) query\n%s",
mysql_conn->conn, THIS_FILE, __LINE__, query);
if (!(result = mysql_db_query_ret(
mysql_conn, query, 0))) {
xfree(query);
_destroy_local_cluster_usage(c_usage);
return SLURM_ERROR;
}
xfree(query);
while ((row = mysql_fetch_row(result))) {
uint32_t job_id = slurm_atoul(row[JOB_REQ_JOBID]);
uint32_t assoc_id = slurm_atoul(row[JOB_REQ_ASSOCID]);
uint32_t wckey_id = slurm_atoul(row[JOB_REQ_WCKEYID]);
uint32_t resv_id = slurm_atoul(row[JOB_REQ_RESVID]);
time_t row_eligible = slurm_atoul(row[JOB_REQ_ELG]);
time_t row_start = slurm_atoul(row[JOB_REQ_START]);
time_t row_end = slurm_atoul(row[JOB_REQ_END]);
uint32_t row_acpu = slurm_atoul(row[JOB_REQ_ACPU]);
uint32_t row_rcpu = slurm_atoul(row[JOB_REQ_RCPU]);
seconds = 0;
if (row_start && (row_start < curr_start))
row_start = curr_start;
if (!row_start && row_end)
row_start = row_end;
if (!row_end || row_end > curr_end)
row_end = curr_end;
if (!row_start || ((row_end - row_start) < 1))
goto calc_cluster;
seconds = (row_end - row_start);
if (slurm_atoul(row[JOB_REQ_SUSPENDED])) {
MYSQL_RES *result2 = NULL;
MYSQL_ROW row2;
/* get the suspended time for this job */
query = xstrdup_printf(
"select %s from \"%s_%s\" where "
"(time_start < %ld && (time_end >= %ld "
"|| time_end = 0)) && job_db_inx=%s "
"order by time_start",
suspend_str, cluster_name,
suspend_table,
curr_end, curr_start,
row[JOB_REQ_DB_INX]);
debug4("%d(%s:%d) query\n%s",
mysql_conn->conn, THIS_FILE,
__LINE__, query);
if (!(result2 = mysql_db_query_ret(
mysql_conn,
query, 0))) {
xfree(query);
_destroy_local_cluster_usage(c_usage);
return SLURM_ERROR;
}
xfree(query);
while ((row2 = mysql_fetch_row(result2))) {
time_t local_start = slurm_atoul(
row2[SUSPEND_REQ_START]);
time_t local_end = slurm_atoul(
row2[SUSPEND_REQ_END]);
if (!local_start)
continue;
if (row_start > local_start)
local_start = row_start;
if (row_end < local_end)
local_end = row_end;
tot_time = (local_end - local_start);
if (tot_time < 1)
continue;
seconds -= tot_time;
}
mysql_free_result(result2);
}
if (seconds < 1) {
debug4("This job (%u) was suspended "
"the entire hour", job_id);
continue;
}
if (last_id != assoc_id) {
a_usage = xmalloc(sizeof(local_id_usage_t));
a_usage->id = assoc_id;
list_append(assoc_usage_list, a_usage);
last_id = assoc_id;
}
a_usage->a_cpu += seconds * row_acpu;
if (!track_wckey)
goto calc_cluster;
/* do the wckey calculation */
if (last_wckeyid != wckey_id) {
list_iterator_reset(w_itr);
while ((w_usage = list_next(w_itr)))
if (w_usage->id == wckey_id)
break;
if (!w_usage) {
w_usage = xmalloc(
sizeof(local_id_usage_t));
w_usage->id = wckey_id;
list_append(wckey_usage_list,
w_usage);
}
last_wckeyid = wckey_id;
}
w_usage->a_cpu += seconds * row_acpu;
/* do the cluster allocated calculation */
calc_cluster:
/* Now figure out there was a disconnected
slurmctld durning this job.
*/
list_iterator_reset(c_itr);
while ((loc_c_usage = list_next(c_itr))) {
int temp_end = row_end;
int temp_start = row_start;
if (loc_c_usage->start > temp_start)
temp_start = loc_c_usage->start;
if (loc_c_usage->end < temp_end)
temp_end = loc_c_usage->end;
seconds = (temp_end - temp_start);
if (seconds > 0) {
/* info(" Job %u was running for " */
/* "%"PRIu64" seconds while " */
/* "cluster %s's slurmctld " */
/* "wasn't responding", */
/* job_id, */
/* (uint64_t) */
/* (seconds * row_acpu), */
/* cluster_name); */
loc_c_usage->total_time -=
seconds * row_acpu;
}
}
/* first figure out the reservation */
if (resv_id) {
if (seconds <= 0)
continue;
/* Since we have already added the
entire reservation as used time on
the cluster we only need to
calculate the used time for the
reservation and then divy up the
unused time over the associations
able to run in the reservation.
Since the job was to run, or ran a
reservation we don't care about
eligible time since that could
totally skew the clusters reserved time
since the job may be able to run
outside of the reservation. */
list_iterator_reset(r_itr);
while ((r_usage = list_next(r_itr))) {
/* since the reservation could
have changed in some way,
thus making a new
reservation record in the
database, we have to make
sure all the reservations
are checked to see if such
a thing has happened */
if (r_usage->id == resv_id) {
int temp_end = row_end;
int temp_start = row_start;
if (r_usage->start > temp_start)
temp_start =
r_usage->start;
if (r_usage->end < temp_end)
temp_end = r_usage->end;
if ((temp_end - temp_start)
> 0) {
r_usage->a_cpu +=
(temp_end
- temp_start)
* row_acpu;
}
}
}
continue;
}
/* only record time for the clusters that have
registered. This continue should rarely if
ever happen.
*/
if (!c_usage)
continue;
if (row_start && (seconds > 0)) {
/* info("%d assoc %d adds " */
/* "(%d)(%d-%d) * %d = %d " */
/* "to %d", */
/* job_id, */
/* a_usage->id, */
/* seconds, */
/* row_end, row_start, */
/* row_acpu, */
/* seconds * row_acpu, */
/* row_acpu); */
c_usage->a_cpu += seconds * row_acpu;
}
/* now reserved time */
if (!row_start || (row_start >= c_usage->start)) {
int temp_end = row_start;
int temp_start = row_eligible;
if (c_usage->start > temp_start)
temp_start = c_usage->start;
if (c_usage->end < temp_end)
temp_end = c_usage->end;
seconds = (temp_end - temp_start);
if (seconds > 0) {
/* info("%d assoc %d reserved " */
/* "(%d)(%d-%d) * %d = %d " */
/* "to %d", */
/* job_id, */
/* assoc_id, */
/* seconds, */
/* temp_end, temp_start, */
/* row_rcpu, */
/* seconds * row_rcpu, */
/* row_rcpu); */
c_usage->r_cpu += seconds * row_rcpu;
}
}
}
mysql_free_result(result);
/* now figure out how much more to add to the
associations that could had run in the reservation
*/
list_iterator_reset(r_itr);
while ((r_usage = list_next(r_itr))) {
int64_t idle = r_usage->total_time - r_usage->a_cpu;
char *assoc = NULL;
ListIterator tmp_itr = NULL;
if (idle <= 0)
continue;
/* now divide that time by the number of
associations in the reservation and add
them to each association */
seconds = idle / list_count(r_usage->local_assocs);
/* info("resv %d got %d for seconds for %d assocs", */
/* r_usage->id, seconds, */
/* list_count(r_usage->local_assocs)); */
tmp_itr = list_iterator_create(r_usage->local_assocs);
while ((assoc = list_next(tmp_itr))) {
uint32_t associd = slurm_atoul(assoc);
if (last_id != associd) {
list_iterator_reset(a_itr);
while ((a_usage = list_next(a_itr))) {
if (a_usage->id == associd) {
last_id = a_usage->id;
break;
}
}
}
if (!a_usage) {
a_usage = xmalloc(
sizeof(local_id_usage_t));
a_usage->id = associd;
list_append(assoc_usage_list, a_usage);
last_id = associd;
}
a_usage->a_cpu += seconds;
}
list_iterator_destroy(tmp_itr);
}
/* now apply the down time from the slurmctld disconnects */
list_iterator_reset(c_itr);
while ((loc_c_usage = list_next(c_itr)))
c_usage->d_cpu += loc_c_usage->total_time;
if ((rc = _process_cluster_usage(
mysql_conn, cluster_name, curr_start,
curr_end, now, c_usage)) != SLURM_SUCCESS) {
_destroy_local_cluster_usage(c_usage);
goto end_it;
}
list_iterator_reset(a_itr);
while ((a_usage = list_next(a_itr))) {
/* info("association (%d) %d alloc %d", */
/* a_usage->id, last_id, */
/* a_usage->a_cpu); */
if (query) {
xstrfmtcat(query,
", (%ld, %ld, %d, %ld, %"PRIu64")",
now, now,
a_usage->id, curr_start,
a_usage->a_cpu);
} else {
xstrfmtcat(query,
"insert into \"%s_%s\" "
"(creation_time, "
"mod_time, id_assoc, time_start, "
"alloc_cpu_secs) values "
"(%ld, %ld, %d, %ld, %"PRIu64")",
cluster_name, assoc_hour_table,
now, now,
a_usage->id, curr_start,
a_usage->a_cpu);
}
}
if (query) {
xstrfmtcat(query,
" on duplicate key update "
"mod_time=%ld, "
"alloc_cpu_secs=VALUES(alloc_cpu_secs);",
now);
debug3("%d(%s:%d) query\n%s",
mysql_conn->conn, THIS_FILE, __LINE__, query);
rc = mysql_db_query(mysql_conn, query);
xfree(query);
if (rc != SLURM_SUCCESS) {
error("Couldn't add assoc hour rollup");
_destroy_local_cluster_usage(c_usage);
goto end_it;
}
}
if (!track_wckey)
goto end_loop;
list_iterator_reset(w_itr);
while ((w_usage = list_next(w_itr))) {
/* info("association (%d) %d alloc %d", */
/* w_usage->id, last_id, */
/* w_usage->a_cpu); */
if (query) {
xstrfmtcat(query,
", (%ld, %ld, %d, %ld, %"PRIu64")",
now, now,
w_usage->id, curr_start,
w_usage->a_cpu);
} else {
xstrfmtcat(query,
"insert into \"%s_%s\" "
"(creation_time, "
"mod_time, id_wckey, time_start, "
"alloc_cpu_secs) values "
"(%ld, %ld, %d, %ld, %"PRIu64")",
cluster_name, wckey_hour_table,
now, now,
w_usage->id, curr_start,
w_usage->a_cpu);
}
}
if (query) {
xstrfmtcat(query,
" on duplicate key update "
"mod_time=%ld, "
"alloc_cpu_secs=VALUES(alloc_cpu_secs);",
now);
debug3("%d(%s:%d) query\n%s",
mysql_conn->conn, THIS_FILE, __LINE__, query);
rc = mysql_db_query(mysql_conn, query);
xfree(query);
if (rc != SLURM_SUCCESS) {
error("Couldn't add wckey hour rollup");
_destroy_local_cluster_usage(c_usage);
goto end_it;
}
}
end_loop:
_destroy_local_cluster_usage(c_usage);
list_flush(assoc_usage_list);
list_flush(cluster_down_list);
list_flush(wckey_usage_list);
list_flush(resv_usage_list);
curr_start = curr_end;
curr_end = curr_start + add_sec;
}
end_it:
xfree(suspend_str);
xfree(job_str);
xfree(resv_str);
list_iterator_destroy(a_itr);
list_iterator_destroy(c_itr);
list_iterator_destroy(w_itr);
list_iterator_destroy(r_itr);
list_destroy(assoc_usage_list);
list_destroy(cluster_down_list);
list_destroy(wckey_usage_list);
list_destroy(resv_usage_list);
/* info("stop start %s", ctime(&curr_start)); */
/* info("stop end %s", ctime(&curr_end)); */
/* go check to see if we archive and purge */
if (rc == SLURM_SUCCESS)
rc = _process_purge(mysql_conn, cluster_name, archive_data,
SLURMDB_PURGE_HOURS);
return rc;
}
