static void _handle_stats(List prec_list, char *proc_stat_file,
char *proc_io_file, jag_callbacks_t *callbacks)
{
static int no_share_data = -1;
FILE *stat_fp = NULL;
FILE *io_fp = NULL;
int fd, fd2;
jag_prec_t *prec = NULL;
if (no_share_data == -1) {
char *acct_params = slurm_get_jobacct_gather_params();
if (acct_params && strstr(acct_params, "NoShare"))
no_share_data = 1;
else
no_share_data = 0;
xfree(acct_params);
}
if (!(stat_fp = fopen(proc_stat_file, "r")))
return; /* Assume the process went away */
/*
* Close the file on exec() of user tasks.
*
* NOTE: If we fork() slurmstepd after the
* fopen() above and before the fcntl() below,
* then the user task may have this extra file
* open, which can cause problems for
* checkpoint/restart, but this should be a very rare
* problem in practice.
*/
fd = fileno(stat_fp);
fcntl(fd, F_SETFD, FD_CLOEXEC);
prec = xmalloc(sizeof(jag_prec_t));
if (_get_process_data_line(fd, prec)) {
if (no_share_data)
_remove_share_data(proc_stat_file, prec);
list_append(prec_list, prec);
if ((io_fp = fopen(proc_io_file, "r"))) {
fd2 = fileno(io_fp);
fcntl(fd2, F_SETFD, FD_CLOEXEC);
_get_process_io_data_line(fd2, prec);
fclose(io_fp);
}
if (callbacks->prec_extra)
(*(callbacks->prec_extra))(prec, my_pagesize);
} else
xfree(prec);
fclose(stat_fp);
}
static void _handle_stats(List prec_list, char *proc_stat_file,
char *proc_io_file, char *proc_smaps_file,
jag_callbacks_t *callbacks,
int tres_count)
{
static int no_share_data = -1;
static int use_pss = -1;
FILE *stat_fp = NULL;
FILE *io_fp = NULL;
int fd, fd2, i;
jag_prec_t *prec = NULL;
if (no_share_data == -1) {
char *acct_params = slurm_get_jobacct_gather_params();
if (acct_params && xstrcasestr(acct_params, "NoShare"))
no_share_data = 1;
else
no_share_data = 0;
if (acct_params && xstrcasestr(acct_params, "UsePss"))
use_pss = 1;
else
use_pss = 0;
xfree(acct_params);
}
if (!(stat_fp = fopen(proc_stat_file, "r")))
return; /* Assume the process went away */
/*
* Close the file on exec() of user tasks.
*
* NOTE: If we fork() slurmstepd after the
* fopen() above and before the fcntl() below,
* then the user task may have this extra file
* open, which can cause problems for
* checkpoint/restart, but this should be a very rare
* problem in practice.
*/
fd = fileno(stat_fp);
if (fcntl(fd, F_SETFD, FD_CLOEXEC) == -1)
error("%s: fcntl(%s): %m", __func__, proc_stat_file);
prec = xmalloc(sizeof(jag_prec_t));
if (!tres_count) {
assoc_mgr_lock_t locks = {
NO_LOCK, NO_LOCK, NO_LOCK, NO_LOCK,
READ_LOCK, NO_LOCK, NO_LOCK };
assoc_mgr_lock(&locks);
tres_count = g_tres_count;
assoc_mgr_unlock(&locks);
}
prec->tres_count = tres_count;
prec->tres_data = xmalloc(prec->tres_count *
sizeof(acct_gather_data_t));
/* Initialize read/writes */
for (i = 0; i < prec->tres_count; i++) {
prec->tres_data[i].num_reads = INFINITE64;
prec->tres_data[i].num_writes = INFINITE64;
prec->tres_data[i].size_read = INFINITE64;
prec->tres_data[i].size_write = INFINITE64;
}
if (!_get_process_data_line(fd, prec)) {
xfree(prec->tres_data);
xfree(prec);
fclose(stat_fp);
return;
}
fclose(stat_fp);
if (acct_gather_filesystem_g_get_data(prec->tres_data) < 0) {
debug2("problem retrieving filesystem data");
}
if (acct_gather_interconnect_g_get_data(prec->tres_data) < 0) {
debug2("problem retrieving interconnect data");
}
/* Remove shared data from rss */
if (no_share_data)
_remove_share_data(proc_stat_file, prec);
/* Use PSS instead if RSS */
if (use_pss) {
if (_get_pss(proc_smaps_file, prec) == -1) {
xfree(prec->tres_data);
xfree(prec);
return;
}
}
list_append(prec_list, prec);
if ((io_fp = fopen(proc_io_file, "r"))) {
fd2 = fileno(io_fp);
if (fcntl(fd2, F_SETFD, FD_CLOEXEC) == -1)
error("%s: fcntl: %m", __func__);
_get_process_io_data_line(fd2, prec);
fclose(io_fp);
}
}
static void _handle_stats(List prec_list, char *proc_stat_file, char *proc_io_file,
char *proc_smaps_file, jag_callbacks_t *callbacks)
{
static int no_share_data = -1;
static int use_pss = -1;
FILE *stat_fp = NULL;
FILE *io_fp = NULL;
int fd, fd2;
jag_prec_t *prec = NULL;
if (no_share_data == -1) {
char *acct_params = slurm_get_jobacct_gather_params();
if (acct_params && xstrcasestr(acct_params, "NoShare"))
no_share_data = 1;
else
no_share_data = 0;
if (acct_params && xstrcasestr(acct_params, "UsePss"))
use_pss = 1;
else
use_pss = 0;
xfree(acct_params);
}
if (!(stat_fp = fopen(proc_stat_file, "r")))
return; /* Assume the process went away */
/*
* Close the file on exec() of user tasks.
*
* NOTE: If we fork() slurmstepd after the
* fopen() above and before the fcntl() below,
* then the user task may have this extra file
* open, which can cause problems for
* checkpoint/restart, but this should be a very rare
* problem in practice.
*/
fd = fileno(stat_fp);
if (fcntl(fd, F_SETFD, FD_CLOEXEC) == -1)
error("%s: fcntl(%s): %m", __func__, proc_stat_file);
prec = try_xmalloc(sizeof(jag_prec_t));
if (prec == NULL) { /* Avoid killing slurmstepd on malloc failure */
fclose(stat_fp);
return;
}
if (!_get_process_data_line(fd, prec)) {
xfree(prec);
fclose(stat_fp);
return;
}
fclose(stat_fp);
/* Remove shared data from rss */
if (no_share_data)
_remove_share_data(proc_stat_file, prec);
/* Use PSS instead if RSS */
if (use_pss) {
if (_get_pss(proc_smaps_file, prec) == -1) {
xfree(prec);
return;
}
}
list_append(prec_list, prec);
if ((io_fp = fopen(proc_io_file, "r"))) {
fd2 = fileno(io_fp);
if (fcntl(fd2, F_SETFD, FD_CLOEXEC) == -1)
error("%s: fcntl: %m", __func__);
_get_process_io_data_line(fd2, prec);
fclose(io_fp);
}
if (callbacks->prec_extra)
(*(callbacks->prec_extra))(prec);
}
/*
* _get_process_data() - Build a table of all current processes
*
* IN: pid.
*
* OUT: none
*
* THREADSAFE! Only one thread ever gets here.
*
* Assumption:
* Any file with a name of the form "/proc/[0-9]+/stat"
* is a Linux-style stat entry. We disregard the data if they look
* wrong.
*/
static void _get_process_data(void)
{
static int slash_proc_open = 0;
struct dirent *slash_proc_entry;
char *iptr = NULL, *optr = NULL;
FILE *stat_fp = NULL;
char proc_stat_file[256]; /* Allow ~20x extra length */
List prec_list = NULL;
pid_t *pids = NULL;
int npids = 0;
uint32_t total_job_mem = 0, total_job_vsize = 0;
int i, fd;
ListIterator itr;
ListIterator itr2;
prec_t *prec = NULL;
struct jobacctinfo *jobacct = NULL;
static int processing = 0;
long hertz;
if (!pgid_plugin && (cont_id == (uint64_t)NO_VAL)) {
debug("cont_id hasn't been set yet not running poll");
return;
}
if(processing) {
debug("already running, returning");
return;
}
processing = 1;
prec_list = list_create(_destroy_prec);
hertz = sysconf(_SC_CLK_TCK);
if (hertz < 1) {
error ("_get_process_data: unable to get clock rate");
hertz = 100; /* default on many systems */
}
if(!pgid_plugin) {
/* get only the processes in the proctrack container */
slurm_container_get_pids(cont_id, &pids, &npids);
if(!npids) {
debug4("no pids in this container %"PRIu64"", cont_id);
goto finished;
}
for (i = 0; i < npids; i++) {
snprintf(proc_stat_file, 256,
"/proc/%d/stat", pids[i]);
if ((stat_fp = fopen(proc_stat_file, "r"))==NULL)
continue; /* Assume the process went away */
/*
* Close the file on exec() of user tasks.
*
* NOTE: If we fork() slurmstepd after the
* fopen() above and before the fcntl() below,
* then the user task may have this extra file
* open, which can cause problems for
* checkpoint/restart, but this should be a very rare
* problem in practice.
*/
fd = fileno(stat_fp);
fcntl(fd, F_SETFD, FD_CLOEXEC);
prec = xmalloc(sizeof(prec_t));
if (_get_process_data_line(fd, prec))
list_append(prec_list, prec);
else
xfree(prec);
fclose(stat_fp);
}
} else {
slurm_mutex_lock(&reading_mutex);
if (slash_proc_open) {
rewinddir(slash_proc);
} else {
slash_proc=opendir("/proc");
if (slash_proc == NULL) {
perror("opening /proc");
slurm_mutex_unlock(&reading_mutex);
goto finished;
}
slash_proc_open=1;
}
strcpy(proc_stat_file, "/proc/");
while ((slash_proc_entry = readdir(slash_proc))) {
/* Save a few cyles by simulating
strcat(statFileName, slash_proc_entry->d_name);
strcat(statFileName, "/stat");
while checking for a numeric filename (which really
should be a pid).
*/
optr = proc_stat_file + sizeof("/proc");
iptr = slash_proc_entry->d_name;
i = 0;
do {
if((*iptr < '0')
|| ((*optr++ = *iptr++) > '9')) {
i = -1;
break;
}
} while (*iptr);
if(i == -1)
continue;
iptr = (char*)"/stat";
do {
*optr++ = *iptr++;
} while (*iptr);
*optr = 0;
if ((stat_fp = fopen(proc_stat_file,"r"))==NULL)
continue; /* Assume the process went away */
/*
* Close the file on exec() of user tasks.
*
* NOTE: If we fork() slurmstepd after the
* fopen() above and before the fcntl() below,
* then the user task may have this extra file
* open, which can cause problems for
* checkpoint/restart, but this should be a very rare
* problem in practice.
*/
fd = fileno(stat_fp);
fcntl(fd, F_SETFD, FD_CLOEXEC);
prec = xmalloc(sizeof(prec_t));
if (_get_process_data_line(fd, prec))
list_append(prec_list, prec);
else
xfree(prec);
fclose(stat_fp);
}
slurm_mutex_unlock(&reading_mutex);
}
if (!list_count(prec_list)) {
goto finished; /* We have no business being here! */
}
slurm_mutex_lock(&jobacct_lock);
if(!task_list || !list_count(task_list)) {
slurm_mutex_unlock(&jobacct_lock);
goto finished;
}
itr = list_iterator_create(task_list);
while((jobacct = list_next(itr))) {
itr2 = list_iterator_create(prec_list);
while((prec = list_next(itr2))) {
if (prec->pid == jobacct->pid) {
#if _DEBUG
info("pid:%u ppid:%u rss:%d KB",
prec->pid, prec->ppid, prec->rss);
#endif
/* find all my descendents */
_get_offspring_data(prec_list,
prec, prec->pid);
/* tally their usage */
jobacct->max_rss = jobacct->tot_rss =
MAX(jobacct->max_rss, prec->rss);
total_job_mem += prec->rss;
jobacct->max_vsize = jobacct->tot_vsize =
MAX(jobacct->max_vsize, prec->vsize);
total_job_vsize += prec->vsize;
jobacct->max_pages = jobacct->tot_pages =
MAX(jobacct->max_pages, prec->pages);
jobacct->min_cpu = jobacct->tot_cpu =
MAX(jobacct->min_cpu,
(prec->ssec / hertz +
prec->usec / hertz));
debug2("%d mem size %u %u time %u(%u+%u)",
jobacct->pid, jobacct->max_rss,
jobacct->max_vsize, jobacct->tot_cpu,
prec->usec, prec->ssec);
break;
}
}
list_iterator_destroy(itr2);
}
list_iterator_destroy(itr);
slurm_mutex_unlock(&jobacct_lock);
if (jobacct_mem_limit) {
if (jobacct_step_id == NO_VAL) {
debug("Job %u memory used:%u limit:%u KB",
jobacct_job_id, total_job_mem, jobacct_mem_limit);
} else {
debug("Step %u.%u memory used:%u limit:%u KB",
jobacct_job_id, jobacct_step_id,
total_job_mem, jobacct_mem_limit);
}
}
if (jobacct_job_id && jobacct_mem_limit &&
(total_job_mem > jobacct_mem_limit)) {
if (jobacct_step_id == NO_VAL) {
error("Job %u exceeded %u KB memory limit, being "
"killed", jobacct_job_id, jobacct_mem_limit);
} else {
error("Step %u.%u exceeded %u KB memory limit, being "
"killed", jobacct_job_id, jobacct_step_id,
jobacct_mem_limit);
}
_acct_kill_step();
} else if (jobacct_job_id && jobacct_vmem_limit &&
(total_job_vsize > jobacct_vmem_limit)) {
if (jobacct_step_id == NO_VAL) {
error("Job %u exceeded %u KB virtual memory limit, "
"being killed", jobacct_job_id,
jobacct_vmem_limit);
} else {
error("Step %u.%u exceeded %u KB virtual memory "
"limit, being killed", jobacct_job_id,
jobacct_step_id, jobacct_vmem_limit);
}
_acct_kill_step();
}
finished:
list_destroy(prec_list);
processing = 0;
return;
}