/*
* _thread_ipmi_run is the thread calling ipmi and launching _thread_ipmi_write
*/
static void *_thread_ipmi_run(void *no_data)
{
// need input (attr)
struct timeval tvnow;
struct timespec abs;
flag_energy_accounting_shutdown = false;
if (debug_flags & DEBUG_FLAG_ENERGY)
info("ipmi-thread: launched");
(void) pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL);
(void) pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, NULL);
slurm_mutex_lock(&ipmi_mutex);
if (_thread_init() != SLURM_SUCCESS) {
if (debug_flags & DEBUG_FLAG_ENERGY)
info("ipmi-thread: aborted");
slurm_mutex_unlock(&ipmi_mutex);
slurm_cond_signal(&launch_cond);
return NULL;
}
(void) pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL);
slurm_mutex_unlock(&ipmi_mutex);
flag_thread_started = true;
slurm_cond_signal(&launch_cond);
/* setup timer */
gettimeofday(&tvnow, NULL);
abs.tv_sec = tvnow.tv_sec;
abs.tv_nsec = tvnow.tv_usec * 1000;
//loop until slurm stop
while (!flag_energy_accounting_shutdown) {
slurm_mutex_lock(&ipmi_mutex);
_thread_update_node_energy();
/* Sleep until the next time. */
abs.tv_sec += slurm_ipmi_conf.freq;
slurm_cond_timedwait(&ipmi_cond, &ipmi_mutex, &abs);
slurm_mutex_unlock(&ipmi_mutex);
}
if (debug_flags & DEBUG_FLAG_ENERGY)
info("ipmi-thread: ended");
return NULL;
}
static void _my_sleep(int secs)
{
struct timespec ts = {0, 0};
struct timeval now;
gettimeofday(&now, NULL);
ts.tv_sec = now.tv_sec + secs;
ts.tv_nsec = now.tv_usec * 1000;
slurm_mutex_lock(&term_lock);
if (!stop_builtin)
slurm_cond_timedwait(&term_cond, &term_lock, &ts);
slurm_mutex_unlock(&term_lock);
}
static void _slice_sleep(void)
{
struct timespec ts = {0, 0};
struct timeval now;
gettimeofday(&now, NULL);
ts.tv_sec = now.tv_sec + timeslicer_seconds;
ts.tv_nsec = now.tv_usec * 1000;
slurm_mutex_lock(&term_lock);
if (!thread_shutdown)
slurm_cond_timedwait(&term_cond, &term_lock, &ts);
slurm_mutex_unlock(&term_lock);
}
/* Sleep function, also handles termination signal */
extern void bb_sleep(bb_state_t *state_ptr, int add_secs)
{
struct timespec ts = {0, 0};
struct timeval tv = {0, 0};
if (gettimeofday(&tv, NULL)) { /* Some error */
sleep(1);
return;
}
ts.tv_sec = tv.tv_sec + add_secs;
ts.tv_nsec = tv.tv_usec * 1000;
slurm_mutex_lock(&state_ptr->term_mutex);
if (!state_ptr->term_flag) {
slurm_cond_timedwait(&state_ptr->term_cond,
&state_ptr->term_mutex, &ts);
}
slurm_mutex_unlock(&state_ptr->term_mutex);
}
static void *_thread_launcher(void *no_data)
{
//what arg would countain? frequency, socket?
struct timeval tvnow;
struct timespec abs;
slurm_thread_create(&thread_ipmi_id_run, _thread_ipmi_run, NULL);
/* setup timer */
gettimeofday(&tvnow, NULL);
abs.tv_sec = tvnow.tv_sec + slurm_ipmi_conf.timeout;
abs.tv_nsec = tvnow.tv_usec * 1000;
slurm_mutex_lock(&launch_mutex);
slurm_cond_timedwait(&launch_cond, &launch_mutex, &abs);
slurm_mutex_unlock(&launch_mutex);
if (!flag_thread_started) {
error("%s threads failed to start in a timely manner",
plugin_name);
flag_energy_accounting_shutdown = true;
/*
* It is a known thing we can hang up on IPMI calls cancel if
* we must.
*/
pthread_cancel(thread_ipmi_id_run);
/*
* Unlock just to make sure since we could have canceled the
* thread while in the lock.
*/
slurm_mutex_unlock(&ipmi_mutex);
}
return NULL;
}
/*
* _msg_aggregation_sender()
*
* Start and terminate message collection windows.
* Send collected msgs to next collector node or final destination
* at window expiration.
*/
static void * _msg_aggregation_sender(void *arg)
{
struct timeval now;
struct timespec timeout;
slurm_msg_t msg;
composite_msg_t cmp;
msg_collection.running = 1;
slurm_mutex_lock(&msg_collection.mutex);
while (msg_collection.running) {
/* Wait for a new msg to be collected */
slurm_cond_wait(&msg_collection.cond, &msg_collection.mutex);
if (!msg_collection.running &&
!list_count(msg_collection.msg_list))
break;
/* A msg has been collected; start new window */
gettimeofday(&now, NULL);
timeout.tv_sec = now.tv_sec + (msg_collection.window / 1000);
timeout.tv_nsec = (now.tv_usec * 1000) +
(1000000 * (msg_collection.window % 1000));
timeout.tv_sec += timeout.tv_nsec / 1000000000;
timeout.tv_nsec %= 1000000000;
slurm_cond_timedwait(&msg_collection.cond,
&msg_collection.mutex, &timeout);
if (!msg_collection.running &&
!list_count(msg_collection.msg_list))
break;
msg_collection.max_msgs = true;
/* Msg collection window has expired and message collection
* is suspended; now build and send composite msg */
memset(&msg, 0, sizeof(slurm_msg_t));
memset(&cmp, 0, sizeof(composite_msg_t));
memcpy(&cmp.sender, &msg_collection.node_addr,
sizeof(slurm_addr_t));
cmp.msg_list = msg_collection.msg_list;
msg_collection.msg_list =
list_create(slurm_free_comp_msg_list);
msg_collection.max_msgs = false;
slurm_msg_t_init(&msg);
msg.msg_type = MESSAGE_COMPOSITE;
msg.protocol_version = SLURM_PROTOCOL_VERSION;
msg.data = &cmp;
if (_send_to_next_collector(&msg) != SLURM_SUCCESS) {
error("_msg_aggregation_engine: Unable to send "
"composite msg: %m");
}
FREE_NULL_LIST(cmp.msg_list);
/* Resume message collection */
slurm_cond_broadcast(&msg_collection.cond);
}
slurm_mutex_unlock(&msg_collection.mutex);
return NULL;
}
static void *_agent(void *x)
{
int cnt, rc;
Buf buffer;
struct timespec abs_time;
static time_t fail_time = 0;
int sigarray[] = {SIGUSR1, 0};
slurmdbd_msg_t list_req;
dbd_list_msg_t list_msg;
list_req.msg_type = DBD_SEND_MULT_MSG;
list_req.data = &list_msg;
memset(&list_msg, 0, sizeof(dbd_list_msg_t));
/* DEF_TIMERS; */
/* Prepare to catch SIGUSR1 to interrupt pending
* I/O and terminate in a timely fashion. */
xsignal(SIGUSR1, _sig_handler);
xsignal_unblock(sigarray);
while (*slurmdbd_conn->shutdown == 0) {
/* START_TIMER; */
slurm_mutex_lock(&slurmdbd_lock);
if (halt_agent)
slurm_cond_wait(&slurmdbd_cond, &slurmdbd_lock);
if ((slurmdbd_conn->fd < 0) &&
(difftime(time(NULL), fail_time) >= 10)) {
/* The connection to Slurm DBD is not open */
_open_slurmdbd_conn(1);
if (slurmdbd_conn->fd < 0)
fail_time = time(NULL);
}
slurm_mutex_lock(&agent_lock);
if (agent_list && slurmdbd_conn->fd)
cnt = list_count(agent_list);
else
cnt = 0;
if ((cnt == 0) || (slurmdbd_conn->fd < 0) ||
(fail_time && (difftime(time(NULL), fail_time) < 10))) {
slurm_mutex_unlock(&slurmdbd_lock);
abs_time.tv_sec = time(NULL) + 10;
abs_time.tv_nsec = 0;
slurm_cond_timedwait(&agent_cond, &agent_lock,
&abs_time);
slurm_mutex_unlock(&agent_lock);
continue;
} else if ((cnt > 0) && ((cnt % 100) == 0))
info("slurmdbd: agent queue size %u", cnt);
/* Leave item on the queue until processing complete */
if (agent_list) {
int handle_agent_count = 1000;
if (cnt > handle_agent_count) {
int agent_count = 0;
ListIterator agent_itr =
list_iterator_create(agent_list);
list_msg.my_list = list_create(NULL);
while ((buffer = list_next(agent_itr))) {
list_enqueue(list_msg.my_list, buffer);
agent_count++;
if (agent_count > handle_agent_count)
break;
}
list_iterator_destroy(agent_itr);
buffer = pack_slurmdbd_msg(
&list_req, SLURM_PROTOCOL_VERSION);
} else if (cnt > 1) {
list_msg.my_list = agent_list;
buffer = pack_slurmdbd_msg(
&list_req, SLURM_PROTOCOL_VERSION);
} else
buffer = (Buf) list_peek(agent_list);
} else
buffer = NULL;
slurm_mutex_unlock(&agent_lock);
if (buffer == NULL) {
slurm_mutex_unlock(&slurmdbd_lock);
slurm_mutex_lock(&assoc_cache_mutex);
if (slurmdbd_conn->fd >= 0 && running_cache)
slurm_cond_signal(&assoc_cache_cond);
slurm_mutex_unlock(&assoc_cache_mutex);
continue;
}
/* NOTE: agent_lock is clear here, so we can add more
* requests to the queue while waiting for this RPC to
* complete. */
rc = slurm_persist_send_msg(slurmdbd_conn, buffer);
if (rc != SLURM_SUCCESS) {
if (*slurmdbd_conn->shutdown) {
slurm_mutex_unlock(&slurmdbd_lock);
break;
}
error("slurmdbd: Failure sending message: %d: %m", rc);
} else if (list_msg.my_list) {
rc = _handle_mult_rc_ret();
} else {
rc = _get_return_code();
if (rc == EAGAIN) {
if (*slurmdbd_conn->shutdown) {
slurm_mutex_unlock(&slurmdbd_lock);
break;
}
error("slurmdbd: Failure with "
"message need to resend: %d: %m", rc);
}
}
slurm_mutex_unlock(&slurmdbd_lock);
slurm_mutex_lock(&assoc_cache_mutex);
if (slurmdbd_conn->fd >= 0 && running_cache)
slurm_cond_signal(&assoc_cache_cond);
slurm_mutex_unlock(&assoc_cache_mutex);
slurm_mutex_lock(&agent_lock);
if (agent_list && (rc == SLURM_SUCCESS)) {
/*
* If we sent a mult_msg we just need to free buffer,
* we don't need to requeue, just mark list_msg.my_list
* as NULL as that is the sign we sent a mult_msg.
*/
if (list_msg.my_list) {
if (list_msg.my_list != agent_list)
FREE_NULL_LIST(list_msg.my_list);
list_msg.my_list = NULL;
} else
buffer = (Buf) list_dequeue(agent_list);
free_buf(buffer);
fail_time = 0;
} else {
/* We need to free a mult_msg even on failure */
if (list_msg.my_list) {
if (list_msg.my_list != agent_list)
FREE_NULL_LIST(list_msg.my_list);
list_msg.my_list = NULL;
free_buf(buffer);
}
fail_time = time(NULL);
}
slurm_mutex_unlock(&agent_lock);
/* END_TIMER; */
/* info("at the end with %s", TIME_STR); */
}
slurm_mutex_lock(&agent_lock);
_save_dbd_state();
FREE_NULL_LIST(agent_list);
slurm_mutex_unlock(&agent_lock);
return NULL;
}
static void *_heartbeat_thread(void *no_data)
{
/*
* The frequency needs to be faster than slurmctld_timeout,
* or the backup controller may try to assume control.
* One-fourth is very conservative, one-half should be sufficient.
* Have it happen at least every 30 seconds if the timeout is quite
* large.
*/
int beat = MIN(slurmctld_conf.slurmctld_timeout / 4, 30);
time_t now;
uint64_t nl;
struct timespec ts = {0, 0};
char *reg_file, *new_file;
int fd;
debug("Heartbeat thread started, beating every %d seconds.", beat);
slurm_mutex_lock(&heartbeat_mutex);
while (heart_beating) {
now = time(NULL);
ts.tv_sec = now + beat;
debug3("Heartbeat at %ld", now);
/*
* Rebuild file path each beat just in case someone changes
* StateSaveLocation and runs reconfigure.
*/
reg_file = xstrdup_printf("%s/heartbeat",
slurmctld_conf.state_save_location);
new_file = xstrdup_printf("%s.new", reg_file);
nl = HTON_uint64((uint64_t) now);
fd = open(new_file, O_CREAT|O_WRONLY|O_TRUNC|O_CLOEXEC, 0600);
if (fd < 0) {
error("%s: heartbeat file creation failed to %s.",
__func__, new_file);
goto delay;
}
if (write(fd, &nl, sizeof(uint64_t)) != sizeof(uint64_t)) {
error("%s: heartbeat write failed to %s.",
__func__, new_file);
close(fd);
(void) unlink(new_file);
goto delay;
}
if (write(fd, &backup_inx, sizeof(int)) != sizeof(int)) {
error("%s: heartbeat write failed to %s.",
__func__, new_file);
close(fd);
(void) unlink(new_file);
goto delay;
}
if (fsync_and_close(fd, "heartbeat")) {
(void) unlink(new_file);
goto delay;
}
/* shuffle files around */
(void) unlink(reg_file);
if (link(new_file, reg_file))
debug("%s: unable to create link for %s -> %s, %m",
__func__, new_file, reg_file);
(void) unlink(new_file);
delay:
xfree(reg_file);
xfree(new_file);
slurm_cond_timedwait(&heartbeat_cond, &heartbeat_mutex, &ts);
}
slurm_mutex_unlock(&heartbeat_mutex);
return NULL;
}
