static void _record_profile(struct jobacctinfo *jobacct)
{
enum {
FIELD_CPUFREQ,
FIELD_CPUTIME,
FIELD_CPUUTIL,
FIELD_RSS,
FIELD_VMSIZE,
FIELD_PAGES,
FIELD_READ,
FIELD_WRITE,
FIELD_CNT
};
acct_gather_profile_dataset_t dataset[] = {
{ "CPUFrequency", PROFILE_FIELD_UINT64 },
{ "CPUTime", PROFILE_FIELD_DOUBLE },
{ "CPUUtilization", PROFILE_FIELD_DOUBLE },
{ "RSS", PROFILE_FIELD_UINT64 },
{ "VMSize", PROFILE_FIELD_UINT64 },
{ "Pages", PROFILE_FIELD_UINT64 },
{ "ReadMB", PROFILE_FIELD_DOUBLE },
{ "WriteMB", PROFILE_FIELD_DOUBLE },
{ NULL, PROFILE_FIELD_NOT_SET }
};
static int profile_gid = -1;
double et;
union {
double d;
uint64_t u64;
} data[FIELD_CNT];
if (profile_gid == -1)
profile_gid = acct_gather_profile_g_create_group("Tasks");
/* Create the dataset first */
if (jobacct->dataset_id < 0) {
char ds_name[32];
snprintf(ds_name, sizeof(ds_name), "%u", jobacct->id.taskid);
jobacct->dataset_id = acct_gather_profile_g_create_dataset(
ds_name, profile_gid, dataset);
if (jobacct->dataset_id == SLURM_ERROR) {
error("JobAcct: Failed to create the dataset for "
"task %d",
jobacct->pid);
return;
}
}
if (jobacct->dataset_id < 0)
return;
data[FIELD_CPUFREQ].u64 = jobacct->act_cpufreq;
data[FIELD_RSS].u64 = jobacct->tot_rss;
data[FIELD_VMSIZE].u64 = jobacct->tot_vsize;
data[FIELD_PAGES].u64 = jobacct->tot_pages;
/* delta from last snapshot */
if (!jobacct->last_time) {
data[FIELD_CPUTIME].d = 0.0;
data[FIELD_CPUUTIL].d = 0.0;
data[FIELD_READ].d = 0.0;
data[FIELD_WRITE].d = 0.0;
} else {
data[FIELD_CPUTIME].d =
jobacct->tot_cpu - jobacct->last_total_cputime;
et = (jobacct->cur_time - jobacct->last_time);
if (!et)
data[FIELD_CPUUTIL].d = 0.0;
else
data[FIELD_CPUUTIL].d =
(100.0 * (double)data[FIELD_CPUTIME].d) /
((double) et);
data[FIELD_READ].d = jobacct->tot_disk_read -
jobacct->last_tot_disk_read;
data[FIELD_WRITE].d = jobacct->tot_disk_write -
jobacct->last_tot_disk_write;
}
if (debug_flags & DEBUG_FLAG_PROFILE) {
char str[256];
info("PROFILE-Task: %s", acct_gather_profile_dataset_str(
dataset, data, str, sizeof(str)));
}
acct_gather_profile_g_add_sample_data(jobacct->dataset_id,
(void *)data, jobacct->cur_time);
}
/*
* _thread_update_node_energy calls _read_ipmi_values and updates all values
* for node consumption
*/
static int _update_node_filesystem(void)
{
static acct_gather_data_t previous;
static int dataset_id = -1;
static bool first = true;
acct_gather_data_t current;
enum {
FIELD_READ,
FIELD_READMB,
FIELD_WRITE,
FIELD_WRITEMB,
FIELD_CNT
};
acct_gather_profile_dataset_t dataset[] = {
{ "Reads", PROFILE_FIELD_UINT64 },
{ "ReadMB", PROFILE_FIELD_DOUBLE },
{ "Writes", PROFILE_FIELD_UINT64 },
{ "WriteMB", PROFILE_FIELD_DOUBLE },
{ NULL, PROFILE_FIELD_NOT_SET }
};
union {
double d;
uint64_t u64;
} data[FIELD_CNT];
slurm_mutex_lock(&lustre_lock);
if (_read_lustre_counters() != SLURM_SUCCESS) {
error("%s: Cannot read lustre counters", __func__);
slurm_mutex_unlock(&lustre_lock);
return SLURM_ERROR;
}
if (first) {
dataset_id = acct_gather_profile_g_create_dataset(
"Filesystem", NO_PARENT, dataset);
if (dataset_id == SLURM_ERROR) {
error("FileSystem: Failed to create the dataset "
"for Lustre");
slurm_mutex_unlock(&lustre_lock);
return SLURM_ERROR;
}
previous.num_reads = lustre_se.all_lustre_nb_reads;
previous.num_writes = lustre_se.all_lustre_nb_writes;
previous.size_read = lustre_se.all_lustre_read_bytes;
previous.size_write = lustre_se.all_lustre_write_bytes;
first = false;
}
if (dataset_id < 0) {
slurm_mutex_unlock(&lustre_lock);
return SLURM_ERROR;
}
/* Compute the current values read from all lustre-xxxx directories */
current.num_reads = lustre_se.all_lustre_nb_reads;
current.num_writes = lustre_se.all_lustre_nb_writes;
current.size_read = lustre_se.all_lustre_read_bytes;
current.size_write = lustre_se.all_lustre_write_bytes;
/* record sample */
data[FIELD_READ].u64 = current.num_reads - previous.num_reads;
data[FIELD_READMB].d =
(double)(current.size_read - previous.size_read) / (1 << 20);
data[FIELD_WRITE].u64 = current.num_writes - previous.num_writes;
data[FIELD_WRITEMB].d =
(double)(current.size_write - previous.size_write) / (1 << 20);
if (debug_flags & DEBUG_FLAG_PROFILE) {
char str[256];
info("PROFILE-Lustre: %s", acct_gather_profile_dataset_str(
dataset, data, str, sizeof(str)));
}
acct_gather_profile_g_add_sample_data(dataset_id, (void *)data,
lustre_se.update_time);
/* Save current as previous and clean up the working
* data structure.
*/
memcpy(&previous, ¤t, sizeof(acct_gather_data_t));
memset(&lustre_se, 0, sizeof(lustre_sens_t));
slurm_mutex_unlock(&lustre_lock);
return SLURM_SUCCESS;
}
extern void jag_common_poll_data(
List task_list, bool pgid_plugin, uint64_t cont_id,
jag_callbacks_t *callbacks)
{
/* Update the data */
List prec_list = NULL;
uint32_t total_job_mem = 0, total_job_vsize = 0;
ListIterator itr;
ListIterator itr2;
jag_prec_t *prec = NULL;
struct jobacctinfo *jobacct = NULL;
static int processing = 0;
char sbuf[72];
int energy_counted = 0;
static int first = 1;
xassert(callbacks);
if (!pgid_plugin && (cont_id == (uint64_t)NO_VAL)) {
debug("cont_id hasn't been set yet not running poll");
return;
}
if (processing) {
debug("already running, returning");
return;
}
processing = 1;
if (!callbacks->get_precs)
callbacks->get_precs = _get_precs;
prec_list = (*(callbacks->get_precs))(task_list, pgid_plugin, cont_id,
callbacks);
if (!list_count(prec_list) || !task_list || !list_count(task_list))
goto finished; /* We have no business being here! */
itr = list_iterator_create(task_list);
while ((jobacct = list_next(itr))) {
itr2 = list_iterator_create(prec_list);
while ((prec = list_next(itr2))) {
if (prec->pid == jobacct->pid) {
uint32_t cpu_calc =
(prec->ssec + prec->usec)/hertz;
#if _DEBUG
info("pid:%u ppid:%u rss:%d KB",
prec->pid, prec->ppid, prec->rss);
#endif
/* find all my descendents */
if (callbacks->get_offspring_data)
(*(callbacks->get_offspring_data))
(prec_list, prec, prec->pid);
/* tally their usage */
jobacct->max_rss =
MAX(jobacct->max_rss, prec->rss);
jobacct->tot_rss = prec->rss;
total_job_mem += prec->rss;
jobacct->max_vsize =
MAX(jobacct->max_vsize, prec->vsize);
jobacct->tot_vsize = prec->vsize;
total_job_vsize += prec->vsize;
jobacct->max_pages =
MAX(jobacct->max_pages, prec->pages);
jobacct->tot_pages = prec->pages;
jobacct->max_disk_read = MAX(
jobacct->max_disk_read,
prec->disk_read);
jobacct->tot_disk_read = prec->disk_read;
jobacct->max_disk_write = MAX(
jobacct->max_disk_write,
prec->disk_write);
jobacct->tot_disk_write = prec->disk_write;
jobacct->min_cpu =
MAX(jobacct->min_cpu, cpu_calc);
jobacct->last_total_cputime = jobacct->tot_cpu;
jobacct->tot_cpu = cpu_calc;
debug2("%d mem size %u %u time %u(%u+%u)",
jobacct->pid, jobacct->max_rss,
jobacct->max_vsize, jobacct->tot_cpu,
prec->usec, prec->ssec);
/* compute frequency */
jobacct->this_sampled_cputime =
cpu_calc - jobacct->last_total_cputime;
_get_sys_interface_freq_line(
prec->last_cpu,
"cpuinfo_cur_freq", sbuf);
jobacct->act_cpufreq =
_update_weighted_freq(jobacct, sbuf);
debug2("Task average frequency = %u "
"pid %d mem size %u %u time %u(%u+%u)",
jobacct->act_cpufreq,
jobacct->pid, jobacct->max_rss,
jobacct->max_vsize, jobacct->tot_cpu,
prec->usec, prec->ssec);
/* get energy consumption
* only once is enough since we
* report per node energy consumption */
debug2("energycounted = %d", energy_counted);
if (energy_counted == 0) {
acct_gather_energy_g_get_data(
energy_profile,
&jobacct->energy);
debug2("getjoules_task energy = %u",
jobacct->energy.consumed_energy);
energy_counted = 1;
}
/* We only profile on after the first poll. */
if (!first)
acct_gather_profile_g_add_sample_data(
ACCT_GATHER_PROFILE_TASK,
jobacct);
break;
}
}
list_iterator_destroy(itr2);
}
list_iterator_destroy(itr);
jobacct_gather_handle_mem_limit(total_job_mem, total_job_vsize);
finished:
list_destroy(prec_list);
processing = 0;
first = 0;
}
