/* * _thread_ipmi_run is the thread calling ipmi and launching _thread_ipmi_write */ static void *_thread_ipmi_run(void *no_data) { // need input (attr) struct timeval tvnow; struct timespec abs; flag_energy_accounting_shutdown = false; if (debug_flags & DEBUG_FLAG_ENERGY) info("ipmi-thread: launched"); (void) pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL); (void) pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, NULL); slurm_mutex_lock(&ipmi_mutex); if (_thread_init() != SLURM_SUCCESS) { if (debug_flags & DEBUG_FLAG_ENERGY) info("ipmi-thread: aborted"); slurm_mutex_unlock(&ipmi_mutex); slurm_cond_signal(&launch_cond); return NULL; } (void) pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL); slurm_mutex_unlock(&ipmi_mutex); flag_thread_started = true; slurm_cond_signal(&launch_cond); /* setup timer */ gettimeofday(&tvnow, NULL); abs.tv_sec = tvnow.tv_sec; abs.tv_nsec = tvnow.tv_usec * 1000; //loop until slurm stop while (!flag_energy_accounting_shutdown) { slurm_mutex_lock(&ipmi_mutex); _thread_update_node_energy(); /* Sleep until the next time. */ abs.tv_sec += slurm_ipmi_conf.freq; slurm_cond_timedwait(&ipmi_cond, &ipmi_mutex, &abs); slurm_mutex_unlock(&ipmi_mutex); } if (debug_flags & DEBUG_FLAG_ENERGY) info("ipmi-thread: ended"); return NULL; }
static void _my_sleep(int secs) { struct timespec ts = {0, 0}; struct timeval now; gettimeofday(&now, NULL); ts.tv_sec = now.tv_sec + secs; ts.tv_nsec = now.tv_usec * 1000; slurm_mutex_lock(&term_lock); if (!stop_builtin) slurm_cond_timedwait(&term_cond, &term_lock, &ts); slurm_mutex_unlock(&term_lock); }
static void _slice_sleep(void) { struct timespec ts = {0, 0}; struct timeval now; gettimeofday(&now, NULL); ts.tv_sec = now.tv_sec + timeslicer_seconds; ts.tv_nsec = now.tv_usec * 1000; slurm_mutex_lock(&term_lock); if (!thread_shutdown) slurm_cond_timedwait(&term_cond, &term_lock, &ts); slurm_mutex_unlock(&term_lock); }
/* Sleep function, also handles termination signal */ extern void bb_sleep(bb_state_t *state_ptr, int add_secs) { struct timespec ts = {0, 0}; struct timeval tv = {0, 0}; if (gettimeofday(&tv, NULL)) { /* Some error */ sleep(1); return; } ts.tv_sec = tv.tv_sec + add_secs; ts.tv_nsec = tv.tv_usec * 1000; slurm_mutex_lock(&state_ptr->term_mutex); if (!state_ptr->term_flag) { slurm_cond_timedwait(&state_ptr->term_cond, &state_ptr->term_mutex, &ts); } slurm_mutex_unlock(&state_ptr->term_mutex); }
static void *_thread_launcher(void *no_data) { //what arg would countain? frequency, socket? struct timeval tvnow; struct timespec abs; slurm_thread_create(&thread_ipmi_id_run, _thread_ipmi_run, NULL); /* setup timer */ gettimeofday(&tvnow, NULL); abs.tv_sec = tvnow.tv_sec + slurm_ipmi_conf.timeout; abs.tv_nsec = tvnow.tv_usec * 1000; slurm_mutex_lock(&launch_mutex); slurm_cond_timedwait(&launch_cond, &launch_mutex, &abs); slurm_mutex_unlock(&launch_mutex); if (!flag_thread_started) { error("%s threads failed to start in a timely manner", plugin_name); flag_energy_accounting_shutdown = true; /* * It is a known thing we can hang up on IPMI calls cancel if * we must. */ pthread_cancel(thread_ipmi_id_run); /* * Unlock just to make sure since we could have canceled the * thread while in the lock. */ slurm_mutex_unlock(&ipmi_mutex); } return NULL; }
/* * _msg_aggregation_sender() * * Start and terminate message collection windows. * Send collected msgs to next collector node or final destination * at window expiration. */ static void * _msg_aggregation_sender(void *arg) { struct timeval now; struct timespec timeout; slurm_msg_t msg; composite_msg_t cmp; msg_collection.running = 1; slurm_mutex_lock(&msg_collection.mutex); while (msg_collection.running) { /* Wait for a new msg to be collected */ slurm_cond_wait(&msg_collection.cond, &msg_collection.mutex); if (!msg_collection.running && !list_count(msg_collection.msg_list)) break; /* A msg has been collected; start new window */ gettimeofday(&now, NULL); timeout.tv_sec = now.tv_sec + (msg_collection.window / 1000); timeout.tv_nsec = (now.tv_usec * 1000) + (1000000 * (msg_collection.window % 1000)); timeout.tv_sec += timeout.tv_nsec / 1000000000; timeout.tv_nsec %= 1000000000; slurm_cond_timedwait(&msg_collection.cond, &msg_collection.mutex, &timeout); if (!msg_collection.running && !list_count(msg_collection.msg_list)) break; msg_collection.max_msgs = true; /* Msg collection window has expired and message collection * is suspended; now build and send composite msg */ memset(&msg, 0, sizeof(slurm_msg_t)); memset(&cmp, 0, sizeof(composite_msg_t)); memcpy(&cmp.sender, &msg_collection.node_addr, sizeof(slurm_addr_t)); cmp.msg_list = msg_collection.msg_list; msg_collection.msg_list = list_create(slurm_free_comp_msg_list); msg_collection.max_msgs = false; slurm_msg_t_init(&msg); msg.msg_type = MESSAGE_COMPOSITE; msg.protocol_version = SLURM_PROTOCOL_VERSION; msg.data = &cmp; if (_send_to_next_collector(&msg) != SLURM_SUCCESS) { error("_msg_aggregation_engine: Unable to send " "composite msg: %m"); } FREE_NULL_LIST(cmp.msg_list); /* Resume message collection */ slurm_cond_broadcast(&msg_collection.cond); } slurm_mutex_unlock(&msg_collection.mutex); return NULL; }
static void *_agent(void *x) { int cnt, rc; Buf buffer; struct timespec abs_time; static time_t fail_time = 0; int sigarray[] = {SIGUSR1, 0}; slurmdbd_msg_t list_req; dbd_list_msg_t list_msg; list_req.msg_type = DBD_SEND_MULT_MSG; list_req.data = &list_msg; memset(&list_msg, 0, sizeof(dbd_list_msg_t)); /* DEF_TIMERS; */ /* Prepare to catch SIGUSR1 to interrupt pending * I/O and terminate in a timely fashion. */ xsignal(SIGUSR1, _sig_handler); xsignal_unblock(sigarray); while (*slurmdbd_conn->shutdown == 0) { /* START_TIMER; */ slurm_mutex_lock(&slurmdbd_lock); if (halt_agent) slurm_cond_wait(&slurmdbd_cond, &slurmdbd_lock); if ((slurmdbd_conn->fd < 0) && (difftime(time(NULL), fail_time) >= 10)) { /* The connection to Slurm DBD is not open */ _open_slurmdbd_conn(1); if (slurmdbd_conn->fd < 0) fail_time = time(NULL); } slurm_mutex_lock(&agent_lock); if (agent_list && slurmdbd_conn->fd) cnt = list_count(agent_list); else cnt = 0; if ((cnt == 0) || (slurmdbd_conn->fd < 0) || (fail_time && (difftime(time(NULL), fail_time) < 10))) { slurm_mutex_unlock(&slurmdbd_lock); abs_time.tv_sec = time(NULL) + 10; abs_time.tv_nsec = 0; slurm_cond_timedwait(&agent_cond, &agent_lock, &abs_time); slurm_mutex_unlock(&agent_lock); continue; } else if ((cnt > 0) && ((cnt % 100) == 0)) info("slurmdbd: agent queue size %u", cnt); /* Leave item on the queue until processing complete */ if (agent_list) { int handle_agent_count = 1000; if (cnt > handle_agent_count) { int agent_count = 0; ListIterator agent_itr = list_iterator_create(agent_list); list_msg.my_list = list_create(NULL); while ((buffer = list_next(agent_itr))) { list_enqueue(list_msg.my_list, buffer); agent_count++; if (agent_count > handle_agent_count) break; } list_iterator_destroy(agent_itr); buffer = pack_slurmdbd_msg( &list_req, SLURM_PROTOCOL_VERSION); } else if (cnt > 1) { list_msg.my_list = agent_list; buffer = pack_slurmdbd_msg( &list_req, SLURM_PROTOCOL_VERSION); } else buffer = (Buf) list_peek(agent_list); } else buffer = NULL; slurm_mutex_unlock(&agent_lock); if (buffer == NULL) { slurm_mutex_unlock(&slurmdbd_lock); slurm_mutex_lock(&assoc_cache_mutex); if (slurmdbd_conn->fd >= 0 && running_cache) slurm_cond_signal(&assoc_cache_cond); slurm_mutex_unlock(&assoc_cache_mutex); continue; } /* NOTE: agent_lock is clear here, so we can add more * requests to the queue while waiting for this RPC to * complete. */ rc = slurm_persist_send_msg(slurmdbd_conn, buffer); if (rc != SLURM_SUCCESS) { if (*slurmdbd_conn->shutdown) { slurm_mutex_unlock(&slurmdbd_lock); break; } error("slurmdbd: Failure sending message: %d: %m", rc); } else if (list_msg.my_list) { rc = _handle_mult_rc_ret(); } else { rc = _get_return_code(); if (rc == EAGAIN) { if (*slurmdbd_conn->shutdown) { slurm_mutex_unlock(&slurmdbd_lock); break; } error("slurmdbd: Failure with " "message need to resend: %d: %m", rc); } } slurm_mutex_unlock(&slurmdbd_lock); slurm_mutex_lock(&assoc_cache_mutex); if (slurmdbd_conn->fd >= 0 && running_cache) slurm_cond_signal(&assoc_cache_cond); slurm_mutex_unlock(&assoc_cache_mutex); slurm_mutex_lock(&agent_lock); if (agent_list && (rc == SLURM_SUCCESS)) { /* * If we sent a mult_msg we just need to free buffer, * we don't need to requeue, just mark list_msg.my_list * as NULL as that is the sign we sent a mult_msg. */ if (list_msg.my_list) { if (list_msg.my_list != agent_list) FREE_NULL_LIST(list_msg.my_list); list_msg.my_list = NULL; } else buffer = (Buf) list_dequeue(agent_list); free_buf(buffer); fail_time = 0; } else { /* We need to free a mult_msg even on failure */ if (list_msg.my_list) { if (list_msg.my_list != agent_list) FREE_NULL_LIST(list_msg.my_list); list_msg.my_list = NULL; free_buf(buffer); } fail_time = time(NULL); } slurm_mutex_unlock(&agent_lock); /* END_TIMER; */ /* info("at the end with %s", TIME_STR); */ } slurm_mutex_lock(&agent_lock); _save_dbd_state(); FREE_NULL_LIST(agent_list); slurm_mutex_unlock(&agent_lock); return NULL; }
static void *_heartbeat_thread(void *no_data) { /* * The frequency needs to be faster than slurmctld_timeout, * or the backup controller may try to assume control. * One-fourth is very conservative, one-half should be sufficient. * Have it happen at least every 30 seconds if the timeout is quite * large. */ int beat = MIN(slurmctld_conf.slurmctld_timeout / 4, 30); time_t now; uint64_t nl; struct timespec ts = {0, 0}; char *reg_file, *new_file; int fd; debug("Heartbeat thread started, beating every %d seconds.", beat); slurm_mutex_lock(&heartbeat_mutex); while (heart_beating) { now = time(NULL); ts.tv_sec = now + beat; debug3("Heartbeat at %ld", now); /* * Rebuild file path each beat just in case someone changes * StateSaveLocation and runs reconfigure. */ reg_file = xstrdup_printf("%s/heartbeat", slurmctld_conf.state_save_location); new_file = xstrdup_printf("%s.new", reg_file); nl = HTON_uint64((uint64_t) now); fd = open(new_file, O_CREAT|O_WRONLY|O_TRUNC|O_CLOEXEC, 0600); if (fd < 0) { error("%s: heartbeat file creation failed to %s.", __func__, new_file); goto delay; } if (write(fd, &nl, sizeof(uint64_t)) != sizeof(uint64_t)) { error("%s: heartbeat write failed to %s.", __func__, new_file); close(fd); (void) unlink(new_file); goto delay; } if (write(fd, &backup_inx, sizeof(int)) != sizeof(int)) { error("%s: heartbeat write failed to %s.", __func__, new_file); close(fd); (void) unlink(new_file); goto delay; } if (fsync_and_close(fd, "heartbeat")) { (void) unlink(new_file); goto delay; } /* shuffle files around */ (void) unlink(reg_file); if (link(new_file, reg_file)) debug("%s: unable to create link for %s -> %s, %m", __func__, new_file, reg_file); (void) unlink(new_file); delay: xfree(reg_file); xfree(new_file); slurm_cond_timedwait(&heartbeat_cond, &heartbeat_mutex, &ts); } slurm_mutex_unlock(&heartbeat_mutex); return NULL; }