//! Collect values in counters
static void intel_snb_cbo_collect(struct stats_type *type)
{
int i;
for (i = 0; i < nr_cpus; i++) {
char cpu[80];
char core_id_path[80];
char socket[80];
char socket_id_path[80];
char cpu_box[80];
int socket_id = -1;
int core_id = -1;
int box;
/* Only collect uncore counters on core 0 of a socket. */
snprintf(core_id_path, sizeof(core_id_path),
"/sys/devices/system/cpu/cpu%d/topology/core_id", i);
if (pscanf(core_id_path, "%d", &core_id) != 1) {
ERROR("cannot read core id file `%s': %m\n", core_id_path);
continue;
}
if (core_id != 0)
continue;
/* Get socket number. */
snprintf(socket_id_path, sizeof(socket_id_path),
"/sys/devices/system/cpu/cpu%d/topology/physical_package_id", i);
if (pscanf(socket_id_path, "%d", &socket_id) != 1) {
ERROR("cannot read socket id file `%s': %m\n", socket_id_path);
continue;
}
snprintf(cpu, sizeof(cpu), "%d", i);
snprintf(socket, sizeof(socket), "%d", socket_id);
if (cpu_is_sandybridge(cpu))
{
for (box = 0; box < 8; box++)
{
snprintf(cpu_box, sizeof(cpu_box), "%d/%d", socket_id, box);
intel_snb_cbo_collect_box(type, cpu, cpu_box, box);
}
}
}
}
const char *
cpu_perc(void)
{
static long double a[7];
long double b[7], sum;
memcpy(b, a, sizeof(b));
/* cpu user nice system idle iowait irq softirq */
if (pscanf("/proc/stat", "%*s %Lf %Lf %Lf %Lf %Lf %Lf %Lf",
&a[0], &a[1], &a[2], &a[3], &a[4], &a[5], &a[6])
!= 7) {
return NULL;
}
if (b[0] == 0) {
return NULL;
}
sum = (b[0] + b[1] + b[2] + b[3] + b[4] + b[5] + b[6]) -
(a[0] + a[1] + a[2] + a[3] + a[4] + a[5] + a[6]);
if (sum == 0) {
return NULL;
}
return bprintf("%d", (int)(100 *
((b[0] + b[1] + b[2] + b[5] + b[6]) -
(a[0] + a[1] + a[2] + a[5] + a[6])) / sum));
}
/* main: test pscanf */
int main(void)
{
int i;
float f;
char c, str[100];
pscanf("%c%d %f %s", &c, &i, &f, str);
printf("c: %c, i: %d, f: %f, str: %s\n", c, i, f, str);
return 0;
}
//! Collect values of counters
static void intel_snb_pcu_collect(struct stats_type *type)
{
// CPUs 0 and 8 have core_id 0 on Stampede at least
int i;
for (i = 0; i < nr_cpus; i++) {
char cpu[80];
char core_id_path[80];
int core_id = -1;
char socket[80];
char socket_id_path[80];
int socket_id = -1;
char pcu[80];
/* Only collect uncore counters on core 0 of a socket. */
snprintf(core_id_path, sizeof(core_id_path),
"/sys/devices/system/cpu/cpu%d/topology/core_id", i);
if (pscanf(core_id_path, "%d", &core_id) != 1) {
ERROR("cannot read core id file `%s': %m\n", core_id_path); /* errno */
continue;
}
if (core_id != 0)
continue;
/* Get socket number. */
snprintf(socket_id_path, sizeof(socket_id_path),
"/sys/devices/system/cpu/cpu%d/topology/physical_package_id", i);
if (pscanf(socket_id_path, "%d", &socket_id) != 1) {
ERROR("cannot read socket id file `%s': %m\n", socket_id_path);
continue;
}
snprintf(cpu, sizeof(cpu), "%d", i);
snprintf(socket, sizeof(socket), "%d", socket_id);
if (cpu_is_sandybridge(cpu))
{
snprintf(pcu, sizeof(pcu), "%d", socket_id);
intel_snb_pcu_collect_socket(type, cpu, pcu);
}
}
}
const char *
ram_total(void)
{
uintmax_t total;
if (pscanf("/proc/meminfo", "MemTotal: %ju kB\n", &total)
!= 1) {
return NULL;
}
return fmt_human(total * 1024, 1024);
}
const char *
cpu_freq(void)
{
uintmax_t freq;
/* in kHz */
if (pscanf("/sys/devices/system/cpu/cpu0/cpufreq/"
"scaling_cur_freq", "%ju", &freq) != 1) {
return NULL;
}
return fmt_human(freq * 1000, 1000);
}
static int
find_ib_device_events(ib_device_t *dev, int extended)
{
int nevents = 0;
DIR *cnt_dir = NULL;
char counters_path[128];
snprintf(counters_path, sizeof(counters_path), "%s/%s/ports/%d/counters%s",
ib_dir_path, dev->dev_name, dev->dev_port, (extended?"_ext":""));
cnt_dir = opendir(counters_path);
if (cnt_dir == NULL) {
SUBDBG("cannot open counters directory `%s'\n", counters_path);
goto out;
}
struct dirent *ev_ent;
/* iterate over all the events */
while ((ev_ent = readdir(cnt_dir)) != NULL) {
char *ev_name = ev_ent->d_name;
long long value = -1;
char event_path[160];
char counter_name[80];
if (ev_name[0] == '.')
continue;
/* Check that we can read an integer from the counter file */
snprintf(event_path, sizeof(event_path), "%s/%s", counters_path, ev_name);
if (pscanf(event_path, "%lld", &value) != 1) {
SUBDBG("cannot read value for event '%s'\n", ev_name);
continue;
}
/* Create new counter */
snprintf(counter_name, sizeof(counter_name), "%s_%d%s:%s",
dev->dev_name, dev->dev_port, (extended?"_ext":""), ev_name);
if (add_ib_counter(counter_name, ev_name, extended, dev))
{
SUBDBG("Added new counter `%s'\n", counter_name);
nevents += 1;
}
}
out:
if (cnt_dir != NULL)
closedir(cnt_dir);
return (nevents);
}
const char *
ram_free(void)
{
uintmax_t free;
if (pscanf("/proc/meminfo",
"MemTotal: %ju kB\n"
"MemFree: %ju kB\n"
"MemAvailable: %ju kB\n",
&free, &free, &free) != 3) {
return NULL;
}
return fmt_human(free * 1024, 1024);
}
int stats_wr_hdr(struct stats_buffer *sf)
{
struct utsname uts_buf;
unsigned long long uptime = 0;
uname(&uts_buf);
pscanf("/proc/uptime", "%llu", &uptime);
sf->sf_data = "";
sf_printf(sf, "%c%s %s\n", SF_PROPERTY_CHAR, STATS_PROGRAM, STATS_VERSION);
sf_printf(sf, "%chostname %s\n", SF_PROPERTY_CHAR, uts_buf.nodename);
sf_printf(sf, "%cuname %s %s %s %s\n", SF_PROPERTY_CHAR, uts_buf.sysname,
uts_buf.machine, uts_buf.release, uts_buf.version);
sf_printf(sf, "%cuptime %llu\n", SF_PROPERTY_CHAR, uptime);
size_t i = 0;
struct stats_type *type;
while ((type = stats_type_for_each(&i)) != NULL) {
if (!type->st_enabled)
continue;
TRACE("type %s, schema_len %zu\n", type->st_name, type->st_schema.sc_len);
/* Write schema. */
sf_printf(sf, "%c%s", SF_SCHEMA_CHAR, type->st_name);
/* MOVEME */
size_t j;
for (j = 0; j < type->st_schema.sc_len; j++) {
struct schema_entry *se = type->st_schema.sc_ent[j];
sf_printf(sf, " %s", se->se_key);
if (se->se_type == SE_CONTROL)
sf_printf(sf, ",C");
if (se->se_type == SE_EVENT)
sf_printf(sf, ",E");
if (se->se_unit != NULL)
sf_printf(sf, ",U=%s", se->se_unit);
if (se->se_width != 0)
sf_printf(sf, ",W=%u", se->se_width);
}
sf_printf(sf, "\n");
}
send(sf);
return 0;
}
const char *
ram_used(void)
{
uintmax_t total, free, buffers, cached;
if (pscanf("/proc/meminfo",
"MemTotal: %ju kB\n"
"MemFree: %ju kB\n"
"MemAvailable: %ju kB\n"
"Buffers: %ju kB\n"
"Cached: %ju kB\n",
&total, &free, &buffers, &buffers, &cached) != 5) {
return NULL;
}
return fmt_human((total - free - buffers - cached) * 1024,
1024);
}
//! Configure and start counters
static int intel_snb_cbo_begin(struct stats_type *type)
{
int nr = 0;
uint64_t cbo_events[8][4] = {
{ RxR_OCCUPANCY, LLC_LOOKUP, COUNTER0_OCCUPANCY, CLOCK_TICKS, },
{ RxR_OCCUPANCY, LLC_LOOKUP, COUNTER0_OCCUPANCY, CLOCK_TICKS, },
{ RxR_OCCUPANCY, LLC_LOOKUP, COUNTER0_OCCUPANCY, CLOCK_TICKS, },
{ RxR_OCCUPANCY, LLC_LOOKUP, COUNTER0_OCCUPANCY, CLOCK_TICKS, },
{ RxR_OCCUPANCY, LLC_LOOKUP, COUNTER0_OCCUPANCY, CLOCK_TICKS, },
{ RxR_OCCUPANCY, LLC_LOOKUP, COUNTER0_OCCUPANCY, CLOCK_TICKS, },
{ RxR_OCCUPANCY, LLC_LOOKUP, COUNTER0_OCCUPANCY, CLOCK_TICKS, },
{ RxR_OCCUPANCY, LLC_LOOKUP, COUNTER0_OCCUPANCY, CLOCK_TICKS, },
};
int i;
for (i = 0; i < nr_cpus; i++) {
char cpu[80];
char core_id_path[80];
int core_id = -1;
int box;
/* Only program uncore counters on core 0 of a socket. */
snprintf(core_id_path, sizeof(core_id_path), "/sys/devices/system/cpu/cpu%d/topology/core_id", i);
if (pscanf(core_id_path, "%d", &core_id) != 1) {
ERROR("cannot read core id file `%s': %m\n", core_id_path); /* errno */
continue;
}
if (core_id != 0)
continue;
snprintf(cpu, sizeof(cpu), "%d", i);
if (cpu_is_sandybridge(cpu))
{
for (box = 0; box < 8; box++)
if (intel_snb_cbo_begin_box(cpu, box, cbo_events[box], 4) == 0)
nr++;
}
}
return nr > 0 ? 0 : -1;
}
static long long
read_ib_counter_value(int index)
{
char ev_file[128];
long long value = 0ll;
infiniband_native_event_entry_t *iter = &infiniband_native_events[index];
snprintf(ev_file, sizeof(ev_file), "%s/%s/ports/%d/counters%s/%s",
ib_dir_path, iter->device->dev_name,
iter->device->dev_port, (iter->extended?"_ext":""),
iter->file_name);
if (pscanf(ev_file, "%lld", &value) != 1) {
PAPIERROR("cannot read value for counter '%s'\n", iter->name);
} else
{
SUBDBG("Counter '%s': %lld\n", iter->name, value);
}
return (value);
}
const char *
ram_perc(void)
{
uintmax_t total, free, buffers, cached;
if (pscanf("/proc/meminfo",
"MemTotal: %ju kB\n"
"MemFree: %ju kB\n"
"MemAvailable: %ju kB\n"
"Buffers: %ju kB\n"
"Cached: %ju kB\n",
&total, &free, &buffers, &buffers, &cached) != 5) {
return NULL;
}
if (total == 0) {
return NULL;
}
return bprintf("%d", 100 * ((total - free) - (buffers + cached))
/ total);
}
//! Configure and start counters
static int intel_snb_pcu_begin(struct stats_type *type)
{
int nr = 0;
uint64_t pcu_events[4] = {FREQ_MAX_TEMP_CYCLES,
FREQ_MAX_POWER_CYCLES,
FREQ_MIN_IO_CYCLES,
FREQ_MIN_SNOOP_CYCLES
};
int i;
for (i = 0; i < nr_cpus; i++) {
char cpu[80];
char core_id_path[80];
int core_id = -1;
/* Only program uncore counters on core 0 of a socket. */
snprintf(core_id_path, sizeof(core_id_path), "/sys/devices/system/cpu/cpu%d/topology/core_id", i);
if (pscanf(core_id_path, "%d", &core_id) != 1) {
ERROR("cannot read core id file `%s': %m\n", core_id_path); /* errno */
continue;
}
if (core_id != 0)
continue;
snprintf(cpu, sizeof(cpu), "%d", i);
if (cpu_is_sandybridge(cpu))
{
if (intel_snb_pcu_begin_socket(cpu, pcu_events,4) == 0)
nr++; /* HARD */
}
}
return nr > 0 ? 0 : -1;
}
static void ps_collect_loadavg(struct stats *stats)
{
const char *path = "/proc/loadavg";
unsigned long long load[3][2];
unsigned long long nr_running = 0, nr_threads = 0;
memset(load, 0, sizeof(load));
/* Ignore last_pid (sixth field). */
if (pscanf(path, "%llu.%llu %llu.%llu %llu.%llu %llu/%llu",
&load[0][0], &load[0][1],
&load[1][0], &load[1][1],
&load[2][0], &load[2][1],
&nr_running, &nr_threads) != 8) {
/* XXX */
return;
}
stats_set(stats, "load_1", load[0][0] * 100 + load[0][1]);
stats_set(stats, "load_5", load[1][0] * 100 + load[1][1]);
stats_set(stats, "load_15", load[2][0] * 100 + load[2][1]);
stats_set(stats, "nr_running", nr_running);
stats_set(stats, "nr_threads", nr_threads);
}
static int
find_ib_devices()
{
DIR *ib_dir = NULL;
int result = PAPI_OK;
num_events = 0;
ib_dir = opendir(ib_dir_path);
if (ib_dir == NULL) {
SUBDBG("cannot open `%s'\n", ib_dir_path);
strncpy(_infiniband_vector.cmp_info.disabled_reason,
"Infiniband sysfs interface not found", PAPI_MAX_STR_LEN);
result = PAPI_ENOSUPP;
goto out;
}
struct dirent *hca_ent;
while ((hca_ent = readdir(ib_dir)) != NULL) {
char *hca = hca_ent->d_name;
char ports_path[80];
DIR *ports_dir = NULL;
if (hca[0] == '.')
goto next_hca;
snprintf(ports_path, sizeof(ports_path), "%s/%s/ports", ib_dir_path, hca);
ports_dir = opendir(ports_path);
if (ports_dir == NULL) {
SUBDBG("cannot open `%s'\n", ports_path);
goto next_hca;
}
struct dirent *port_ent;
while ((port_ent = readdir(ports_dir)) != NULL) {
int port = atoi(port_ent->d_name);
if (port <= 0)
continue;
/* Check that port is active. .../HCA/ports/PORT/state should read "4: ACTIVE." */
int state = -1;
char state_path[80];
snprintf(state_path, sizeof(state_path), "%s/%s/ports/%d/state", ib_dir_path, hca, port);
if (pscanf(state_path, "%d", &state) != 1) {
SUBDBG("cannot read state of IB HCA `%s' port %d\n", hca, port);
continue;
}
if (state != 4) {
SUBDBG("skipping inactive IB HCA `%s', port %d, state %d\n", hca, port, state);
continue;
}
/* Create dev name (HCA/PORT) and get stats for dev. */
SUBDBG("Found IB device `%s', port %d\n", hca, port);
ib_device_t *dev = add_ib_device(hca, port);
if (!dev)
continue;
// do we want to check for short counters only if no extended counters found?
num_events += find_ib_device_events(dev, 1); // check if we have extended (64bit) counters
num_events += find_ib_device_events(dev, 0); // check also for short counters
}
next_hca:
if (ports_dir != NULL)
closedir(ports_dir);
}
if (root_device == 0) // no active devices found
{
strncpy(_infiniband_vector.cmp_info.disabled_reason,
"No active Infiniband ports found", PAPI_MAX_STR_LEN);
result = PAPI_ENOIMPL;
} else if (num_events == 0)
{
strncpy(_infiniband_vector.cmp_info.disabled_reason,
"No supported Infiniband events found", PAPI_MAX_STR_LEN);
result = PAPI_ENOIMPL;
} else
{
// Events are stored in a linked list, in reverse order than how I found them
// Revert them again, so that they are in finding order, not that it matters.
int i = num_events - 1;
// now allocate memory to store the counters into the native table
infiniband_native_events = (infiniband_native_event_entry_t*)
papi_calloc(sizeof(infiniband_native_event_entry_t), num_events);
ib_counter_t *iter = root_counter;
while (iter != 0)
{
infiniband_native_events[i].name = iter->ev_name;
infiniband_native_events[i].file_name = iter->ev_file_name;
infiniband_native_events[i].device = iter->ev_device;
infiniband_native_events[i].extended = iter->extended;
infiniband_native_events[i].resources.selector = i + 1;
infiniband_native_events[i].description =
make_ib_event_description(iter->ev_file_name, iter->extended);
ib_counter_t *tmp = iter;
iter = iter->next;
papi_free(tmp);
-- i;
}
root_counter = 0;
}
out:
if (ib_dir != NULL)
closedir(ib_dir);
return (result);
}
static void collect_ib_sw(struct stats_type *type)
{
const char *ib_dir_path = "/sys/class/infiniband";
DIR *ib_dir = NULL;
ib_dir = opendir(ib_dir_path);
if (ib_dir == NULL) {
ERROR("cannot open `%s': %m\n", ib_dir_path);
goto out;
}
struct dirent *hca_ent;
while ((hca_ent = readdir(ib_dir)) != NULL) {
char *hca = hca_ent->d_name;
char ports_path[80];
DIR *ports_dir = NULL;
if (hca[0] == '.')
goto next_hca;
snprintf(ports_path, sizeof(ports_path), "%s/%s/ports", ib_dir_path, hca);
ports_dir = opendir(ports_path);
if (ports_dir == NULL) {
ERROR("cannot open `%s': %m\n", ports_path);
goto next_hca;
}
struct dirent *port_ent;
while ((port_ent = readdir(ports_dir)) != NULL) {
int port = atoi(port_ent->d_name);
if (port <= 0)
continue;
/* Check that port is active. .../HCA/ports/PORT/state should read "4: ACTIVE." */
int state = -1;
char state_path[80];
snprintf(state_path, sizeof(state_path), "/sys/class/infiniband/%s/ports/%d/state", hca, port);
if (pscanf(state_path, "%d", &state) != 1) {
ERROR("cannot read state of IB HCA `%s' port %d: %m\n", hca, port);
continue;
}
if (state != 4) {
TRACE("skipping inactive IB HCA `%s', port %d, state %d\n", hca, port, state);
continue;
}
/* Create dev name (HCA/PORT) and get stats for dev. */
char dev[80];
snprintf(dev, sizeof(dev), "%s/%d", hca, port);
TRACE("IB HCA `%s', port %d, dev `%s'\n", hca, port, dev);
struct stats *stats = get_current_stats(type, dev);
if (stats == NULL)
continue;
collect_hca_port(stats, hca, port);
}
next_hca:
if (ports_dir != NULL)
closedir(ports_dir);
}
out:
if (ib_dir != NULL)
closedir(ib_dir);
}
int main(int argc, char *argv[])
{
int lock_fd = -1;
int lock_timeout = 30;
const char *current_path = STATS_DIR_PATH"/current";
const char *mark = NULL;
int rc = 0;
struct option opts[] = {
{ "help", 0, 0, 'h' },
{ "mark", 0, 0, 'm' },
{ NULL, 0, 0, 0 },
};
int c;
while ((c = getopt_long(argc, argv, "hm:", opts, 0)) != -1) {
switch (c) {
case 'h':
usage();
exit(0);
case 'm':
mark = optarg;
break;
case '?':
fprintf(stderr, "Try `%s --help' for more information.\n", program_invocation_short_name);
exit(1);
}
}
umask(022);
if (!(optind < argc))
FATAL("must specify a command\n");
const char *cmd_str = argv[optind];
char **arg_list = argv + optind + 1;
size_t arg_count = argc - optind - 1;
enum {
cmd_begin,
cmd_collect,
cmd_end,
cmd_rotate,
} cmd;
if (strcmp(cmd_str, "begin") == 0)
cmd = cmd_begin;
else if (strcmp(cmd_str, "collect") == 0)
cmd = cmd_collect;
else if (strcmp(cmd_str, "end") == 0)
cmd = cmd_end;
else if (strcmp(cmd_str, "rotate") == 0)
cmd = cmd_rotate;
else
FATAL("invalid command `%s'\n", cmd_str);
lock_fd = open_lock_timeout(STATS_LOCK_PATH, lock_timeout);
if (lock_fd < 0)
FATAL("cannot acquire lock\n");
if (cmd == cmd_rotate) {
if (unlink(current_path) < 0 && errno != ENOENT) {
ERROR("cannot unlink `%s': %m\n", current_path);
rc = 1;
}
goto out;
}
current_time = time(NULL);
pscanf(JOBID_FILE_PATH, "%79s", current_jobid);
nr_cpus = sysconf(_SC_NPROCESSORS_ONLN);
if (mkdir(STATS_DIR_PATH, 0777) < 0) {
if (errno != EEXIST)
FATAL("cannot create directory `%s': %m\n", STATS_DIR_PATH);
}
struct stats_file sf;
if (stats_file_open(&sf, current_path) < 0) {
rc = 1;
goto out;
}
int enable_all = 0;
int select_all = cmd != cmd_collect || arg_count == 0;
if (sf.sf_empty) {
char *link_path = strf("%s/%ld", STATS_DIR_PATH, current_time);
if (link_path == NULL)
ERROR("cannot create path: %m\n");
else if (link(current_path, link_path) < 0)
ERROR("cannot link `%s' to `%s': %m\n", current_path, link_path);
free(link_path);
enable_all = 1;
select_all = 1;
}
size_t i;
struct stats_type *type;
if (cmd == cmd_collect) {
/* If arg_count is zero then we select all below. */
for (i = 0; i < arg_count; i++) {
type = stats_type_get(arg_list[i]);
if (type == NULL) {
ERROR("unknown type `%s'\n", arg_list[i]);
continue;
}
type->st_selected = 1;
}
}
i = 0;
while ((type = stats_type_for_each(&i)) != NULL) {
if (enable_all)
type->st_enabled = 1;
if (!type->st_enabled)
continue;
if (stats_type_init(type) < 0) {
type->st_enabled = 0;
continue;
}
if (select_all)
type->st_selected = 1;
if (cmd == cmd_begin && type->st_begin != NULL)
(*type->st_begin)(type);
if (type->st_enabled && type->st_selected)
(*type->st_collect)(type);
}
if (mark != NULL)
stats_file_mark(&sf, "%s", mark);
else if (cmd == cmd_begin || cmd == cmd_end)
/* On begin set mark to "begin JOBID", and similar for end. */
stats_file_mark(&sf, "%s %s", cmd_str, arg_count > 0 ? arg_list[0] : "-");
if (stats_file_close(&sf) < 0)
rc = 1;
/* Cleanup. */
i = 0;
while ((type = stats_type_for_each(&i)) != NULL)
stats_type_destroy(type);
out:
return rc;
}
