static void _handle_stats(List prec_list, char *proc_stat_file,
char *proc_io_file, jag_callbacks_t *callbacks)
{
static int no_share_data = -1;
FILE *stat_fp = NULL;
FILE *io_fp = NULL;
int fd, fd2;
jag_prec_t *prec = NULL;
if (no_share_data == -1) {
char *acct_params = slurm_get_jobacct_gather_params();
if (acct_params && strstr(acct_params, "NoShare"))
no_share_data = 1;
else
no_share_data = 0;
xfree(acct_params);
}
if (!(stat_fp = fopen(proc_stat_file, "r")))
return; /* Assume the process went away */
/*
* Close the file on exec() of user tasks.
*
* NOTE: If we fork() slurmstepd after the
* fopen() above and before the fcntl() below,
* then the user task may have this extra file
* open, which can cause problems for
* checkpoint/restart, but this should be a very rare
* problem in practice.
*/
fd = fileno(stat_fp);
fcntl(fd, F_SETFD, FD_CLOEXEC);
prec = xmalloc(sizeof(jag_prec_t));
if (_get_process_data_line(fd, prec)) {
if (no_share_data)
_remove_share_data(proc_stat_file, prec);
list_append(prec_list, prec);
if ((io_fp = fopen(proc_io_file, "r"))) {
fd2 = fileno(io_fp);
fcntl(fd2, F_SETFD, FD_CLOEXEC);
_get_process_io_data_line(fd2, prec);
fclose(io_fp);
}
if (callbacks->prec_extra)
(*(callbacks->prec_extra))(prec, my_pagesize);
} else
xfree(prec);
fclose(stat_fp);
}
static void _handle_stats(List prec_list, char *proc_stat_file, char *proc_io_file,
char *proc_smaps_file, jag_callbacks_t *callbacks)
{
static int no_share_data = -1;
static int use_pss = -1;
FILE *stat_fp = NULL;
FILE *io_fp = NULL;
int fd, fd2;
jag_prec_t *prec = NULL;
if (no_share_data == -1) {
char *acct_params = slurm_get_jobacct_gather_params();
if (acct_params && xstrcasestr(acct_params, "NoShare"))
no_share_data = 1;
else
no_share_data = 0;
if (acct_params && xstrcasestr(acct_params, "UsePss"))
use_pss = 1;
else
use_pss = 0;
xfree(acct_params);
}
if (!(stat_fp = fopen(proc_stat_file, "r")))
return; /* Assume the process went away */
/*
* Close the file on exec() of user tasks.
*
* NOTE: If we fork() slurmstepd after the
* fopen() above and before the fcntl() below,
* then the user task may have this extra file
* open, which can cause problems for
* checkpoint/restart, but this should be a very rare
* problem in practice.
*/
fd = fileno(stat_fp);
if (fcntl(fd, F_SETFD, FD_CLOEXEC) == -1)
error("%s: fcntl(%s): %m", __func__, proc_stat_file);
prec = try_xmalloc(sizeof(jag_prec_t));
if (prec == NULL) { /* Avoid killing slurmstepd on malloc failure */
fclose(stat_fp);
return;
}
if (!_get_process_data_line(fd, prec)) {
xfree(prec);
fclose(stat_fp);
return;
}
fclose(stat_fp);
/* Remove shared data from rss */
if (no_share_data)
_remove_share_data(proc_stat_file, prec);
/* Use PSS instead if RSS */
if (use_pss) {
if (_get_pss(proc_smaps_file, prec) == -1) {
xfree(prec);
return;
}
}
list_append(prec_list, prec);
if ((io_fp = fopen(proc_io_file, "r"))) {
fd2 = fileno(io_fp);
if (fcntl(fd2, F_SETFD, FD_CLOEXEC) == -1)
error("%s: fcntl: %m", __func__);
_get_process_io_data_line(fd2, prec);
fclose(io_fp);
}
if (callbacks->prec_extra)
(*(callbacks->prec_extra))(prec);
}
extern void jag_common_poll_data(
List task_list, bool pgid_plugin, uint64_t cont_id,
jag_callbacks_t *callbacks, bool profile)
{
/* Update the data */
List prec_list = NULL;
uint64_t total_job_mem = 0, total_job_vsize = 0;
ListIterator itr;
jag_prec_t *prec = NULL;
struct jobacctinfo *jobacct = NULL;
static int processing = 0;
char sbuf[72];
int energy_counted = 0;
time_t ct;
static int no_over_memory_kill = -1;
xassert(callbacks);
if (!pgid_plugin && (cont_id == (uint64_t)NO_VAL)) {
debug("cont_id hasn't been set yet not running poll");
return;
}
if (processing) {
debug("already running, returning");
return;
}
processing = 1;
if (no_over_memory_kill == -1) {
char *acct_params = slurm_get_jobacct_gather_params();
if (acct_params && strstr(acct_params, "NoOverMemoryKill"))
no_over_memory_kill = 1;
else
no_over_memory_kill = 0;
xfree(acct_params);
}
if (!callbacks->get_precs)
callbacks->get_precs = _get_precs;
ct = time(NULL);
prec_list = (*(callbacks->get_precs))(task_list, pgid_plugin, cont_id,
callbacks);
if (!list_count(prec_list) || !task_list || !list_count(task_list))
goto finished; /* We have no business being here! */
itr = list_iterator_create(task_list);
while ((jobacct = list_next(itr))) {
uint32_t cpu_calc;
uint32_t last_total_cputime;
if (!(prec = list_find_first(prec_list, _find_prec, jobacct)))
continue;
#if _DEBUG
info("pid:%u ppid:%u rss:%d KB",
prec->pid, prec->ppid, prec->rss);
#endif
/* find all my descendents */
if (callbacks->get_offspring_data)
(*(callbacks->get_offspring_data))
(prec_list, prec, prec->pid);
last_total_cputime = jobacct->tot_cpu;
cpu_calc = (prec->ssec + prec->usec)/hertz;
/* tally their usage */
jobacct->max_rss =
MAX(jobacct->max_rss, prec->rss);
jobacct->tot_rss = prec->rss;
total_job_mem += prec->rss;
jobacct->max_vsize =
MAX(jobacct->max_vsize, prec->vsize);
jobacct->tot_vsize = prec->vsize;
total_job_vsize += prec->vsize;
jobacct->max_pages =
MAX(jobacct->max_pages, prec->pages);
jobacct->tot_pages = prec->pages;
jobacct->max_disk_read = MAX(
jobacct->max_disk_read,
prec->disk_read);
jobacct->tot_disk_read = prec->disk_read;
jobacct->max_disk_write = MAX(
jobacct->max_disk_write,
prec->disk_write);
jobacct->tot_disk_write = prec->disk_write;
jobacct->min_cpu =
MAX(jobacct->min_cpu, cpu_calc);
/* Update the cpu times
*/
jobacct->tot_cpu = cpu_calc;
jobacct->user_cpu_sec = prec->usec/hertz;
jobacct->sys_cpu_sec = prec->ssec/hertz;
debug2("%s: %d mem size %"PRIu64" %"PRIu64" "
"time %u(%u+%u)", __func__,
jobacct->pid, jobacct->max_rss,
jobacct->max_vsize, jobacct->tot_cpu,
jobacct->user_cpu_sec,
jobacct->sys_cpu_sec);
/* compute frequency */
jobacct->this_sampled_cputime =
cpu_calc - last_total_cputime;
_get_sys_interface_freq_line(
prec->last_cpu,
"cpuinfo_cur_freq", sbuf);
jobacct->act_cpufreq =
_update_weighted_freq(jobacct, sbuf);
debug("%s: Task average frequency = %u "
"pid %d mem size %"PRIu64" %"PRIu64" "
"time %u(%u+%u)", __func__,
jobacct->act_cpufreq,
jobacct->pid, jobacct->max_rss,
jobacct->max_vsize, jobacct->tot_cpu,
jobacct->user_cpu_sec,
jobacct->sys_cpu_sec);
/* get energy consumption
* only once is enough since we
* report per node energy consumption */
debug2("energycounted = %d", energy_counted);
if (energy_counted == 0) {
acct_gather_energy_g_get_data(
energy_profile,
&jobacct->energy);
debug2("getjoules_task energy = %"PRIu64,
jobacct->energy.consumed_energy);
energy_counted = 1;
}
if (profile &&
acct_gather_profile_g_is_active(ACCT_GATHER_PROFILE_TASK)) {
jobacct->cur_time = ct;
_record_profile(jobacct);
jobacct->last_tot_disk_read = jobacct->tot_disk_read;
jobacct->last_tot_disk_write = jobacct->tot_disk_write;
jobacct->last_total_cputime = jobacct->tot_cpu;
jobacct->last_time = jobacct->cur_time;
}
}
list_iterator_destroy(itr);
if (!no_over_memory_kill)
jobacct_gather_handle_mem_limit(total_job_mem, total_job_vsize);
finished:
FREE_NULL_LIST(prec_list);
processing = 0;
}
static void _handle_stats(List prec_list, char *proc_stat_file,
char *proc_io_file, char *proc_smaps_file,
jag_callbacks_t *callbacks,
int tres_count)
{
static int no_share_data = -1;
static int use_pss = -1;
FILE *stat_fp = NULL;
FILE *io_fp = NULL;
int fd, fd2, i;
jag_prec_t *prec = NULL;
if (no_share_data == -1) {
char *acct_params = slurm_get_jobacct_gather_params();
if (acct_params && xstrcasestr(acct_params, "NoShare"))
no_share_data = 1;
else
no_share_data = 0;
if (acct_params && xstrcasestr(acct_params, "UsePss"))
use_pss = 1;
else
use_pss = 0;
xfree(acct_params);
}
if (!(stat_fp = fopen(proc_stat_file, "r")))
return; /* Assume the process went away */
/*
* Close the file on exec() of user tasks.
*
* NOTE: If we fork() slurmstepd after the
* fopen() above and before the fcntl() below,
* then the user task may have this extra file
* open, which can cause problems for
* checkpoint/restart, but this should be a very rare
* problem in practice.
*/
fd = fileno(stat_fp);
if (fcntl(fd, F_SETFD, FD_CLOEXEC) == -1)
error("%s: fcntl(%s): %m", __func__, proc_stat_file);
prec = xmalloc(sizeof(jag_prec_t));
if (!tres_count) {
assoc_mgr_lock_t locks = {
NO_LOCK, NO_LOCK, NO_LOCK, NO_LOCK,
READ_LOCK, NO_LOCK, NO_LOCK };
assoc_mgr_lock(&locks);
tres_count = g_tres_count;
assoc_mgr_unlock(&locks);
}
prec->tres_count = tres_count;
prec->tres_data = xmalloc(prec->tres_count *
sizeof(acct_gather_data_t));
/* Initialize read/writes */
for (i = 0; i < prec->tres_count; i++) {
prec->tres_data[i].num_reads = INFINITE64;
prec->tres_data[i].num_writes = INFINITE64;
prec->tres_data[i].size_read = INFINITE64;
prec->tres_data[i].size_write = INFINITE64;
}
if (!_get_process_data_line(fd, prec)) {
xfree(prec->tres_data);
xfree(prec);
fclose(stat_fp);
return;
}
fclose(stat_fp);
if (acct_gather_filesystem_g_get_data(prec->tres_data) < 0) {
debug2("problem retrieving filesystem data");
}
if (acct_gather_interconnect_g_get_data(prec->tres_data) < 0) {
debug2("problem retrieving interconnect data");
}
/* Remove shared data from rss */
if (no_share_data)
_remove_share_data(proc_stat_file, prec);
/* Use PSS instead if RSS */
if (use_pss) {
if (_get_pss(proc_smaps_file, prec) == -1) {
xfree(prec->tres_data);
xfree(prec);
return;
}
}
list_append(prec_list, prec);
if ((io_fp = fopen(proc_io_file, "r"))) {
fd2 = fileno(io_fp);
if (fcntl(fd2, F_SETFD, FD_CLOEXEC) == -1)
error("%s: fcntl: %m", __func__);
_get_process_io_data_line(fd2, prec);
fclose(io_fp);
}
}
