/* Sysprof -- Sampling, systemwide CPU profiler

* Copyright 2004, Red Hat, Inc.

* Copyright 2004, 2005, Soeren Sandmann

*

* This program is free software; you can redistribute it and/or modify

* it under the terms of the GNU General Public License as published by

* the Free Software Foundation; either version 2 of the License, or

* (at your option) any later version.

*

* This program is distributed in the hope that it will be useful,

* but WITHOUT ANY WARRANTY; without even the implied warranty of

* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

* GNU General Public License for more details.

*

* You should have received a copy of the GNU General Public License

* along with this program; if not, write to the Free Software

* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.

*/

#include <stdint.h>

#include <glib.h>

#include <unistd.h>

#include <sys/syscall.h>

#include <sys/types.h>

#include <string.h>

#include <sys/mman.h>

#include <stdlib.h>

#include <stdio.h>

#include <sys/ioctl.h>

#include <errno.h>

#include "stackstash.h"

#include "collector.h"

#include "watch.h"

#include "elfparser.h"

#include "tracker.h"

#include <linux/perf_event.h>

#include "util.h"

#define d_print(...)

#define N_PAGES 32 /* Number of pages in the ringbuffer */

#define N_WAKEUP_EVENTS 149

typedef struct counter_t counter_t;

typedef struct sample_event_t sample_event_t;

typedef struct mmap_event_t mmap_event_t;

typedef struct comm_event_t comm_event_t;

typedef struct exit_event_t exit_event_t;

typedef struct fork_event_t fork_event_t;

typedef union counter_event_t counter_event_t;

static void process_event (Collector *collector,

counter_t *counter,

counter_event_t *event);

struct counter_t

};

struct sample_event_t

};

struct comm_event_t

};

struct mmap_event_t

};

struct fork_event_t

};

struct exit_event_t

};

union counter_event_t

{

struct perf_event_header header;

mmap_event_t mmap;

comm_event_t comm;

sample_event_t sample;

fork_event_t fork;

exit_event_t exit;

};

struct Collector

};

static int

get_n_cpus (void)

{

return sysconf (_SC_NPROCESSORS_ONLN);

}

static int

sysprof_perf_counter_open (struct perf_event_attr *attr,

unsigned long flags)

{

#ifndef __NR_perf_counter_open

#if defined(__i386__)

#define __NR_perf_counter_open 336

#elif defined(__x86_64__)

#define __NR_perf_counter_open 298

#elif defined(__arm__)

#define __NR_perf_counter_open 364

#elif defined(__bfin__)

#define __NR_perf_counter_open 369

#elif defined(__frv__)

#define __NR_perf_counter_open 336

#elif defined(__m68k__)

#define __NR_perf_counter_open 332

#elif defined(__MICROBLAZE__)

#define __NR_perf_counter_open 366

#elif defined(__mips__) && defined(_ABIO32)

#define __NR_perf_counter_open 4333

#elif defined(__mips__) && defined(_ABIN32)

#define __NR_perf_counter_open 6296

#elif defined(__mips__) && defined(_ABI64)

#define __NR_perf_counter_open 5292

#elif defined(__mn10300__)

#define __NR_perf_counter_open 337

#elif defined(__hppa__)

#define __NR_perf_counter_open 318

#elif defined(__powerpc__) || defined(__powerpc64__)

#define __NR_perf_counter_open 319

#elif defined(__s390__)

#define __NR_perf_counter_open 331

#elif defined(__sh__) && (!defined(__SH5__) || __SH5__ == 32)

#define __NR_perf_counter_open 336

#elif defined(__sh__) && defined(__SH5__) && __SH5__ == 64

#define __NR_perf_counter_open 364

#elif defined(__sparc__) || defined(__sparc64__)

#define __NR_perf_counter_open 327

#endif

#endif

attr->size = sizeof(*attr);

return syscall (__NR_perf_counter_open, attr, pid, cpu, group_fd, flags);

}

static double

timeval_to_ms (const GTimeVal *timeval)

{

return (timeval->tv_sec * G_USEC_PER_SEC + timeval->tv_usec) / 1000.0;

}

static double

time_diff (const GTimeVal *first,

const GTimeVal *second)

{

double first_ms = timeval_to_ms (first);

double second_ms = timeval_to_ms (second);

return first_ms - second_ms;

}

#define RESET_DEAD_PERIOD 250

static gboolean

in_dead_period (Collector *collector)

{

GTimeVal now;

double diff;

g_get_current_time (&now);

diff = time_diff (&now, &collector->latest_reset);

if (diff >= 0.0 && diff < RESET_DEAD_PERIOD)

return TRUE;

return FALSE;

}

static int

get_page_size (void)

{

static int page_size;

static gboolean has_page_size = FALSE;

if (!has_page_size)

{

page_size = getpagesize();

has_page_size = TRUE;

}

return page_size;

}

static void

on_read (gpointer data)

{

counter_t *counter = data;

int mask = (N_PAGES * get_page_size() - 1);

int n_bytes = mask + 1;

gboolean skip_samples;

Collector *collector;

uint64_t head, tail;

collector = counter->collector;

tail = counter->tail;

head = counter->mmap_page->data_head;

rmb();

if (head < tail)

{

g_warning ("sysprof fails at ring buffers (head "FMT64", tail "FMT64"\n", head, tail);

tail = head;

}

#if 0

/* Verify that the double mapping works */

x = g_random_int() & mask;

g_assert (*(counter->data + x) == *(counter->data + x + n_bytes));

#endif

skip_samples = in_dead_period (collector);

#if 0

g_print ("n bytes %d\n", head - tail);

#endif

while (head - tail >= sizeof (struct perf_event_header))

{

struct perf_event_header *header;

guint8 buffer[4096];

guint8 *free_me;

free_me = NULL;

/* Note that:

*

* - perf events are a multiple of 64 bits

* - the perf event header is 64 bits

* - the data area is a multiple of 64 bits

*

* which means there will always be space for one header, which means we

* can safely dereference the size field.

*/

header = (struct perf_event_header *)(counter->data + (tail & mask));

if (header->size > head - tail)

{

/* The kernel did not generate a complete event.

* I don't think that can happen, but we may as well

* be paranoid.

*/

break;

}

if (counter->data + (tail & mask) + header->size > counter->data + n_bytes)

{

int n_before, n_after;

guint8 *b;

if (header->size > sizeof (buffer))

free_me = b = g_malloc (header->size);

else

b = buffer;

n_after = (tail & mask) + header->size - n_bytes;

n_before = header->size - n_after;

memcpy (b, counter->data + (tail & mask), n_before);

memcpy (b + n_before, counter->data, n_after);

header = (struct perf_event_header *)b;

}

if (!skip_samples || header->type != PERF_RECORD_SAMPLE)

{

if (header->type == PERF_RECORD_SAMPLE)

collector->n_samples++;

process_event (collector, counter, (counter_event_t *)header);

}

if (free_me)

g_free (free_me);

tail += header->size;

}

counter->tail = tail;

counter->mmap_page->data_tail = tail;

if (collector->callback)

{

if (collector->n_samples - collector->prev_samples >= N_WAKEUP_EVENTS)

{

gboolean first_sample = collector->prev_samples == 0;

collector->callback (first_sample, collector->data);

collector->prev_samples = collector->n_samples;

}

}

}

static void *

fail (GError **err, const char *what)

{

g_set_error (err, COLLECTOR_ERROR, COLLECTOR_ERROR_FAILED,

"%s: %s", what, g_strerror (errno));

return NULL;

}

static void *

map_buffer (counter_t *counter, GError **err)

{

int n_bytes = N_PAGES * get_page_size();

void *address;

address = mmap (NULL, n_bytes + get_page_size(), PROT_READ | PROT_WRITE, MAP_SHARED, counter->fd, 0);

if (address == MAP_FAILED)

return fail (err, "mmap");

return address;

}

static gboolean

counter_set_output (counter_t *counter, int output)

{

return ioctl (counter->fd, PERF_EVENT_IOC_SET_OUTPUT, output) == 0;

}

static void

counter_enable (counter_t *counter)

{

ioctl (counter->fd, PERF_EVENT_IOC_ENABLE);

}

static void

counter_disable (counter_t *counter)

{

d_print ("disable\n");

ioctl (counter->fd, PERF_EVENT_IOC_DISABLE);

}

static counter_t *

counter_new (Collector *collector,

GError **err)

{

struct perf_event_attr attr;

counter_t *counter;

int fd;

counter = g_new (counter_t, 1);

memset (&attr, 0, sizeof (attr));

attr.type = PERF_TYPE_HARDWARE;

attr.config = PERF_COUNT_HW_CPU_CYCLES;

attr.sample_period = 1200000 ; /* In number of clock cycles -

* FIXME: consider using frequency instead

*/

attr.sample_type = PERF_SAMPLE_IP | PERF_SAMPLE_TID | PERF_SAMPLE_CALLCHAIN;

attr.wakeup_events = N_WAKEUP_EVENTS;

attr.disabled = TRUE;

attr.mmap = 1;

attr.comm = 1;

attr.task = 1;

attr.exclude_idle = 1;

if (!collector->use_hw_counters || (fd = sysprof_perf_counter_open (&attr, pid, cpu, -1, 0)) < 0)

{

attr.type = PERF_TYPE_SOFTWARE;

attr.config = PERF_COUNT_SW_CPU_CLOCK;

attr.sample_period = 1000000;

fd = sysprof_perf_counter_open (&attr, pid, cpu, -1, 0);

}

if (fd < 0)

return fail (err, "Could not open performance counter");

counter->collector = collector;

counter->fd = fd;

counter->cpu = cpu;

if (output && counter_set_output (counter, output->fd))

{

counter->mmap_page = NULL;

counter->data = NULL;

counter->tail = 0;

}

else

{

counter->mmap_page = map_buffer (counter, err);

if (!counter->mmap_page || counter->mmap_page == MAP_FAILED)

return NULL;

counter->data = (uint8_t *)counter->mmap_page + get_page_size ();

counter->tail = 0;

fd_add_watch (fd, counter);

fd_set_read_callback (fd, on_read);

}

return counter;

}

static void

counter_free (counter_t *counter)

{

d_print ("munmap\n");

munmap (counter->mmap_page, (N_PAGES + 1) * get_page_size());

fd_remove_watch (counter->fd);

close (counter->fd);

g_free (counter);

}

/*

* Collector

*/

static void

enable_counters (Collector *collector)

{

GList *list;

d_print ("enable\n");

for (list = collector->counters; list != NULL; list = list->next)

{

counter_t *counter = list->data;

counter_enable (counter);

}

}

static void

disable_counters (Collector *collector)

{

GList *list;

d_print ("disable\n");

for (list = collector->counters; list != NULL; list = list->next)

{

counter_t *counter = list->data;

counter_disable (counter);

}

}

void

collector_reset (Collector *collector)

{

/* Disable the counters so that we won't track

* the activity of tracker_free()/tracker_new()

*

* They will still record fork/mmap/etc. so

* we can keep an accurate log of process creation

*/

if (collector->counters)

{

d_print ("disable counters\n");

disable_counters (collector);

}

if (collector->tracker)

{

tracker_free (collector->tracker);

collector->tracker = tracker_new ();

}

collector->n_samples = 0;

collector->prev_samples = 0;

g_get_current_time (&collector->latest_reset);

if (collector->counters)

{

d_print ("enable counters\n");

enable_counters (collector);

}

}

/* callback is called whenever a new sample arrives */

Collector *

collector_new (gboolean use_hw_counters,

CollectorFunc callback,

gpointer data)

{

Collector *collector = g_new0 (Collector, 1);

collector->callback = callback;

collector->data = data;

collector->tracker = NULL;

collector->use_hw_counters = use_hw_counters;

collector_reset (collector);

return collector;

}

static void

process_mmap (Collector *collector, mmap_event_t *mmap)

{

tracker_add_map (collector->tracker,

mmap->pid,

mmap->addr,

mmap->addr + mmap->len,

mmap->pgoff,

0, /* inode */

mmap->filename);

}

static void

process_comm (Collector *collector, comm_event_t *comm)

{

d_print ("pid, tid: %d %d", comm->pid, comm->tid);

tracker_add_process (collector->tracker,

comm->pid,

comm->comm);

}

static void

process_fork (Collector *collector, fork_event_t *fork)

{

d_print ("ppid: %d pid: %d ptid: %d tid %d\n",

fork->ppid, fork->pid, fork->ptid, fork->tid);

tracker_add_fork (collector->tracker, fork->ppid, fork->pid);

}

static void

process_exit (Collector *collector, exit_event_t *exit)

{

d_print ("for %d %d", exit->pid, exit->tid);

tracker_add_exit (collector->tracker, exit->pid);

}

static void

process_sample (Collector *collector,

sample_event_t *sample)

{

uint64_t *ips;

int n_ips;

d_print ("pid, tid: %d %d", sample->pid, sample->tid);

if (sample->n_ips == 0)

{

uint64_t trace[3];

if (sample->header.misc & PERF_RECORD_MISC_KERNEL)

{

trace[0] = PERF_CONTEXT_KERNEL;

trace[1] = sample->ip;

trace[2] = PERF_CONTEXT_USER;

ips = trace;

n_ips = 3;

}

else

{

trace[0] = PERF_CONTEXT_USER;

trace[1] = sample->ip;

ips = trace;

n_ips = 2;

}

}

else

{

ips = sample->ips;

n_ips = sample->n_ips;

}

tracker_add_sample (collector->tracker,

sample->pid, ips, n_ips);

}

static void

process_event (Collector *collector,

counter_t *counter,

counter_event_t *event)

{

char *name;

switch (event->header.type)

{

case PERF_RECORD_MMAP: name = "mmap"; break;

case PERF_RECORD_LOST: name = "lost"; break;

case PERF_RECORD_COMM: name = "comm"; break;

case PERF_RECORD_EXIT: name = "exit"; break;

case PERF_RECORD_THROTTLE: name = "throttle"; break;

case PERF_RECORD_UNTHROTTLE: name = "unthrottle"; break;

case PERF_RECORD_FORK: name = "fork"; break;

case PERF_RECORD_READ: name = "read"; break;

case PERF_RECORD_SAMPLE: name = "samp"; break;

default: name = "unknown"; break;

}

d_print ("cpu %d :: %s :: ", counter->cpu, name);

switch (event->header.type)

{

case PERF_RECORD_MMAP:

process_mmap (collector, &event->mmap);

break;

case PERF_RECORD_LOST:

g_print ("lost event\n");

break;

case PERF_RECORD_COMM:

process_comm (collector, &event->comm);

break;

case PERF_RECORD_EXIT:

process_exit (collector, &event->exit);

break;

case PERF_RECORD_THROTTLE:

g_print ("throttle\n");

break;

case PERF_RECORD_UNTHROTTLE:

g_print ("unthrottle\n");

break;

case PERF_RECORD_FORK:

process_fork (collector, &event->fork);

break;

case PERF_RECORD_READ:

break;

case PERF_RECORD_SAMPLE:

process_sample (collector, &event->sample);

break;

default:

g_warning ("unknown event: %d (%d)\n",

event->header.type, event->header.size);

break;

}

d_print ("\n");

}

gboolean

collector_start (Collector *collector,

pid_t pid,

GError **err)

{

int n_cpus = get_n_cpus ();

int i;

counter_t *output;

if (!collector->tracker)

collector->tracker = tracker_new ();

output = NULL;

for (i = 0; i < n_cpus; ++i)

{

counter_t *counter = counter_new (collector, pid, i, output, err);

if (!counter)

{

GList *list;

for (list = collector->counters; list != NULL; list = list->next)

counter_free (list->data);

collector->tracker = NULL;

return FALSE;

}

collector->counters = g_list_append (collector->counters, counter);

if (!output)

output = counter;

}

enable_counters (collector);

return TRUE;

}

void

collector_stop (Collector *collector)

{

GList *list;

if (!collector->counters)

return;

/* Read any remaining data */

for (list = collector->counters; list != NULL; list = list->next)

{

counter_t *counter = list->data;

if (counter->data)

on_read (counter);

counter_free (counter);

}

g_list_free (collector->counters);

collector->counters = NULL;

}

int

collector_get_n_samples (Collector *collector)

{

return collector->n_samples;

}

Profile *

collector_create_profile (Collector *collector)

{

/* The collector must be stopped when you create a profile */

g_assert (!collector->counters);

return tracker_create_profile (collector->tracker);

}

GQuark

collector_error_quark (void)

{

static GQuark q = 0;

if (q == 0)

q = g_quark_from_static_string ("collector-error-quark");

return q;

}