static int annotate_release(struct inode *inode, struct file *file)
{
int cpu = 0;
/* synchronize between cores */
spin_lock(&annotate_lock);
if (per_cpu(gator_buffer, cpu)[ANNOTATE_BUF] && buffer_check_space(cpu, ANNOTATE_BUF, MAXSIZE_PACK64 + 3 * MAXSIZE_PACK32)) {
uint32_t pid = current->pid;
gator_buffer_write_packed_int(cpu, ANNOTATE_BUF, get_physical_cpu());
gator_buffer_write_packed_int(cpu, ANNOTATE_BUF, pid);
/* time */
gator_buffer_write_packed_int64(cpu, ANNOTATE_BUF, 0);
/* size */
gator_buffer_write_packed_int(cpu, ANNOTATE_BUF, 0);
}
/* Check and commit; commit is set to occur once buffer is 3/4 full */
buffer_check(cpu, ANNOTATE_BUF, gator_get_time());
spin_unlock(&annotate_lock);
return 0;
}
GATOR_DEFINE_PROBE(cpu_migrate_begin, TP_PROTO(u64 timestamp, u32 cpu_hwid))
{
const int cpu = get_physical_cpu();
gator_timer_offline((void *)1);
gator_timer_offline_dispatch(cpu, true);
}
static int gator_events_irq_read(int **buffer)
{
int len, value;
int cpu = get_physical_cpu();
len = 0;
if (hardirq_enabled) {
value = atomic_read(&per_cpu(irqCnt, cpu)[HARDIRQ]);
atomic_sub(value, &per_cpu(irqCnt, cpu)[HARDIRQ]);
per_cpu(irqGet, cpu)[len++] = hardirq_key;
per_cpu(irqGet, cpu)[len++] = value;
}
if (softirq_enabled) {
value = atomic_read(&per_cpu(irqCnt, cpu)[SOFTIRQ]);
atomic_sub(value, &per_cpu(irqCnt, cpu)[SOFTIRQ]);
per_cpu(irqGet, cpu)[len++] = softirq_key;
per_cpu(irqGet, cpu)[len++] = value;
}
if (buffer)
*buffer = per_cpu(irqGet, cpu);
return len;
}
static void marshal_backtrace_footer(u64 time)
{
int cpu = get_physical_cpu();
gator_buffer_write_packed_int(cpu, BACKTRACE_BUF, MESSAGE_END_BACKTRACE);
/* Check and commit; commit is set to occur once buffer is 3/4 full */
buffer_check(cpu, BACKTRACE_BUF, time);
}
static void marshal_backtrace(unsigned long address, int cookie, int in_kernel)
{
int cpu = get_physical_cpu();
if (cookie == 0 && !in_kernel)
cookie = UNRESOLVED_COOKIE;
gator_buffer_write_packed_int(cpu, BACKTRACE_BUF, cookie);
gator_buffer_write_packed_int64(cpu, BACKTRACE_BUF, address);
}
static void marshal_cookie(int cookie, const char *text)
{
int cpu = get_physical_cpu();
/* buffer_check_space already called by marshal_cookie_header */
gator_buffer_write_packed_int(cpu, NAME_BUF, MESSAGE_COOKIE);
gator_buffer_write_packed_int(cpu, NAME_BUF, cookie);
gator_buffer_write_string(cpu, NAME_BUF, text);
buffer_check(cpu, NAME_BUF, gator_get_time());
}
GATOR_DEFINE_PROBE(cpu_migrate_finish, TP_PROTO(u64 timestamp, u32 cpu_hwid))
{
int cpu;
gator_update_cpu_mapping(cpu_hwid);
// get_physical_cpu must be called after gator_update_cpu_mapping
cpu = get_physical_cpu();
gator_timer_online_dispatch(cpu, true);
gator_timer_online((void *)1);
}
GATOR_DEFINE_PROBE(sched_switch, TP_PROTO(struct task_struct *prev, struct task_struct *next, unsigned int prev_ip))
#endif
{
unsigned long flags;
// disable interrupts to synchronize with gator_events_sched_read()
// spinlocks not needed since percpu buffers are used
local_irq_save(flags);
per_cpu(schedCnt, get_physical_cpu())[SCHED_SWITCH]++;
local_irq_restore(flags);
}
static bool marshal_event_header(u64 time)
{
unsigned long flags, cpu = get_physical_cpu();
bool retval = false;
local_irq_save(flags);
if (buffer_check_space(cpu, BLOCK_COUNTER_BUF, MAXSIZE_PACK32 + MAXSIZE_PACK64)) {
gator_buffer_write_packed_int(cpu, BLOCK_COUNTER_BUF, 0); /* key of zero indicates a timestamp */
gator_buffer_write_packed_int64(cpu, BLOCK_COUNTER_BUF, time);
retval = true;
}
local_irq_restore(flags);
return retval;
}
static void marshal_core_name(const int core, const int cpuid, const char *name)
{
int cpu = get_physical_cpu();
unsigned long flags;
local_irq_save(flags);
if (buffer_check_space(cpu, SUMMARY_BUF, MAXSIZE_PACK32 + MAXSIZE_CORE_NAME)) {
gator_buffer_write_packed_int(cpu, SUMMARY_BUF, MESSAGE_CORE_NAME);
gator_buffer_write_packed_int(cpu, SUMMARY_BUF, core);
gator_buffer_write_packed_int(cpu, SUMMARY_BUF, cpuid);
gator_buffer_write_string(cpu, SUMMARY_BUF, name);
}
/* Commit core names now so that they can show up in live */
local_irq_restore(flags);
gator_commit_buffer(cpu, SUMMARY_BUF, gator_get_time());
}
static void marshal_thread_name(int pid, char *name)
{
unsigned long flags, cpu;
u64 time;
local_irq_save(flags);
cpu = get_physical_cpu();
time = gator_get_time();
if (buffer_check_space(cpu, NAME_BUF, TASK_COMM_LEN + 3 * MAXSIZE_PACK32 + MAXSIZE_PACK64)) {
gator_buffer_write_packed_int(cpu, NAME_BUF, MESSAGE_THREAD_NAME);
gator_buffer_write_packed_int64(cpu, NAME_BUF, time);
gator_buffer_write_packed_int(cpu, NAME_BUF, pid);
gator_buffer_write_string(cpu, NAME_BUF, name);
}
local_irq_restore(flags);
buffer_check(cpu, NAME_BUF, time);
}
static void marshal_idle(int core, int state)
{
unsigned long flags, cpu;
u64 time;
local_irq_save(flags);
cpu = get_physical_cpu();
time = gator_get_time();
if (buffer_check_space(cpu, IDLE_BUF, MAXSIZE_PACK64 + 2 * MAXSIZE_PACK32)) {
gator_buffer_write_packed_int(cpu, IDLE_BUF, state);
gator_buffer_write_packed_int64(cpu, IDLE_BUF, time);
gator_buffer_write_packed_int(cpu, IDLE_BUF, core);
}
local_irq_restore(flags);
/* Check and commit; commit is set to occur once buffer is 3/4 full */
buffer_check(cpu, IDLE_BUF, time);
}
static void __maybe_unused marshal_event_single64(int core, int key, long long value)
{
unsigned long flags, cpu;
u64 time;
local_irq_save(flags);
cpu = get_physical_cpu();
time = gator_get_time();
if (buffer_check_space(cpu, COUNTER_BUF, 2 * MAXSIZE_PACK64 + 2 * MAXSIZE_PACK32)) {
gator_buffer_write_packed_int64(cpu, COUNTER_BUF, time);
gator_buffer_write_packed_int(cpu, COUNTER_BUF, core);
gator_buffer_write_packed_int(cpu, COUNTER_BUF, key);
gator_buffer_write_packed_int64(cpu, COUNTER_BUF, value);
}
local_irq_restore(flags);
/* Check and commit; commit is set to occur once buffer is 3/4 full */
buffer_check(cpu, COUNTER_BUF, time);
}
static bool marshal_backtrace_header(int exec_cookie, int tgid, int pid, u64 time)
{
int cpu = get_physical_cpu();
if (!buffer_check_space(cpu, BACKTRACE_BUF, MAXSIZE_PACK64 + 5 * MAXSIZE_PACK32 + gator_backtrace_depth * 2 * MAXSIZE_PACK32)) {
/* Check and commit; commit is set to occur once buffer is 3/4 full */
buffer_check(cpu, BACKTRACE_BUF, time);
return false;
}
gator_buffer_write_packed_int64(cpu, BACKTRACE_BUF, time);
gator_buffer_write_packed_int(cpu, BACKTRACE_BUF, exec_cookie);
gator_buffer_write_packed_int(cpu, BACKTRACE_BUF, tgid);
gator_buffer_write_packed_int(cpu, BACKTRACE_BUF, pid);
return true;
}
static void marshal_link(int cookie, int tgid, int pid)
{
unsigned long cpu = get_physical_cpu(), flags;
u64 time;
local_irq_save(flags);
time = gator_get_time();
if (buffer_check_space(cpu, ACTIVITY_BUF, MAXSIZE_PACK64 + 5 * MAXSIZE_PACK32)) {
gator_buffer_write_packed_int(cpu, ACTIVITY_BUF, MESSAGE_LINK);
gator_buffer_write_packed_int64(cpu, ACTIVITY_BUF, time);
gator_buffer_write_packed_int(cpu, ACTIVITY_BUF, cookie);
gator_buffer_write_packed_int(cpu, ACTIVITY_BUF, tgid);
gator_buffer_write_packed_int(cpu, ACTIVITY_BUF, pid);
}
local_irq_restore(flags);
/* Check and commit; commit is set to occur once buffer is 3/4 full */
buffer_check(cpu, ACTIVITY_BUF, time);
}
static void marshal_sched_trace_exit(int tgid, int pid)
{
unsigned long cpu = get_physical_cpu(), flags;
u64 time;
if (!per_cpu(gator_buffer, cpu)[SCHED_TRACE_BUF])
return;
local_irq_save(flags);
time = gator_get_time();
if (buffer_check_space(cpu, SCHED_TRACE_BUF, MAXSIZE_PACK64 + 2 * MAXSIZE_PACK32)) {
gator_buffer_write_packed_int(cpu, SCHED_TRACE_BUF, MESSAGE_SCHED_EXIT);
gator_buffer_write_packed_int64(cpu, SCHED_TRACE_BUF, time);
gator_buffer_write_packed_int(cpu, SCHED_TRACE_BUF, pid);
}
local_irq_restore(flags);
/* Check and commit; commit is set to occur once buffer is 3/4 full */
buffer_check(cpu, SCHED_TRACE_BUF, time);
}
static int report_trace(struct stackframe *frame, void *d)
{
unsigned int *depth = d, cookie = NO_COOKIE;
unsigned long addr = frame->pc;
if (*depth) {
#if defined(MODULE)
unsigned int cpu = get_physical_cpu();
struct module *mod = __module_address(addr);
if (mod) {
cookie = get_cookie(cpu, current, mod->name, false);
addr = addr - (unsigned long)mod->module_core;
}
#endif
marshal_backtrace(addr & ~1, cookie);
(*depth)--;
}
return *depth == 0;
}
static int gator_events_sched_read(int **buffer)
{
unsigned long flags;
int len, value;
int cpu = get_physical_cpu();
len = 0;
if (sched_switch_enabled) {
local_irq_save(flags);
value = per_cpu(schedCnt, cpu)[SCHED_SWITCH];
per_cpu(schedCnt, cpu)[SCHED_SWITCH] = 0;
local_irq_restore(flags);
per_cpu(schedGet, cpu)[len++] = sched_switch_key;
per_cpu(schedGet, cpu)[len++] = value;
}
if (buffer)
*buffer = per_cpu(schedGet, cpu);
return len;
}
static int gator_events_irq_online(int **buffer, bool migrate)
{
int len = 0, cpu = get_physical_cpu();
// synchronization with the irq_exit functions is not necessary as the values are being reset
if (hardirq_enabled) {
atomic_set(&per_cpu(irqCnt, cpu)[HARDIRQ], 0);
per_cpu(irqGet, cpu)[len++] = hardirq_key;
per_cpu(irqGet, cpu)[len++] = 0;
}
if (softirq_enabled) {
atomic_set(&per_cpu(irqCnt, cpu)[SOFTIRQ], 0);
per_cpu(irqGet, cpu)[len++] = softirq_key;
per_cpu(irqGet, cpu)[len++] = 0;
}
if (buffer)
*buffer = per_cpu(irqGet, cpu);
return len;
}
static void marshal_event64(int len, long long *buffer64)
{
unsigned long i, flags, cpu = get_physical_cpu();
if (len <= 0)
return;
/* length must be even since all data is a (key, value) pair */
if (len & 0x1) {
pr_err("gator: invalid counter data detected and discarded\n");
return;
}
/* events must be written in key,value pairs */
local_irq_save(flags);
for (i = 0; i < len; i += 2) {
if (!buffer_check_space(cpu, BLOCK_COUNTER_BUF, 2 * MAXSIZE_PACK64))
break;
gator_buffer_write_packed_int64(cpu, BLOCK_COUNTER_BUF, buffer64[i]);
gator_buffer_write_packed_int64(cpu, BLOCK_COUNTER_BUF, buffer64[i + 1]);
}
local_irq_restore(flags);
}
static void marshal_activity_switch(int core, int key, int activity, int pid, int state)
{
unsigned long cpu = get_physical_cpu(), flags;
u64 time;
if (!per_cpu(gator_buffer, cpu)[ACTIVITY_BUF])
return;
local_irq_save(flags);
time = gator_get_time();
if (buffer_check_space(cpu, ACTIVITY_BUF, MAXSIZE_PACK64 + 5 * MAXSIZE_PACK32)) {
gator_buffer_write_packed_int(cpu, ACTIVITY_BUF, MESSAGE_SWITCH);
gator_buffer_write_packed_int64(cpu, ACTIVITY_BUF, time);
gator_buffer_write_packed_int(cpu, ACTIVITY_BUF, core);
gator_buffer_write_packed_int(cpu, ACTIVITY_BUF, key);
gator_buffer_write_packed_int(cpu, ACTIVITY_BUF, activity);
gator_buffer_write_packed_int(cpu, ACTIVITY_BUF, pid);
gator_buffer_write_packed_int(cpu, ACTIVITY_BUF, state);
}
local_irq_restore(flags);
/* Check and commit; commit is set to occur once buffer is 3/4 full */
buffer_check(cpu, ACTIVITY_BUF, time);
}
static ssize_t annotate_write(struct file *file, char const __user *buf, size_t count_orig, loff_t *offset)
{
int pid, cpu, header_size, available, contiguous, length1, length2, size, count = count_orig & 0x7fffffff;
bool interrupt_context;
if (*offset)
return -EINVAL;
interrupt_context = in_interrupt();
/* Annotations are not supported in interrupt context, but may work
* if you comment out the the next four lines of code. By doing so,
* annotations in interrupt context can result in deadlocks and lost
* data.
*/
if (interrupt_context) {
pr_warning("gator: Annotations are not supported in interrupt context. Edit gator_annotate.c in the gator driver to enable annotations in interrupt context.\n");
return -EINVAL;
}
retry:
/* synchronize between cores and with collect_annotations */
spin_lock(&annotate_lock);
if (!collect_annotations) {
/* Not collecting annotations, tell the caller everything was written */
size = count_orig;
goto annotate_write_out;
}
/* Annotation only uses a single per-cpu buffer as the data must be in order to the engine */
cpu = 0;
if (current == NULL)
pid = 0;
else
pid = current->pid;
/* determine total size of the payload */
header_size = MAXSIZE_PACK32 * 3 + MAXSIZE_PACK64;
available = buffer_bytes_available(cpu, ANNOTATE_BUF) - header_size;
size = count < available ? count : available;
if (size <= 0) {
/* Buffer is full, wait until space is available */
spin_unlock(&annotate_lock);
/* Drop the annotation as blocking is not allowed in interrupt context */
if (interrupt_context)
return -EINVAL;
wait_event_interruptible(gator_annotate_wait, buffer_bytes_available(cpu, ANNOTATE_BUF) > header_size || !collect_annotations);
/* Check to see if a signal is pending */
if (signal_pending(current))
return -EINTR;
goto retry;
}
/* synchronize shared variables annotateBuf and annotatePos */
if (per_cpu(gator_buffer, cpu)[ANNOTATE_BUF]) {
u64 time = gator_get_time();
gator_buffer_write_packed_int(cpu, ANNOTATE_BUF, get_physical_cpu());
gator_buffer_write_packed_int(cpu, ANNOTATE_BUF, pid);
gator_buffer_write_packed_int64(cpu, ANNOTATE_BUF, time);
gator_buffer_write_packed_int(cpu, ANNOTATE_BUF, size);
/* determine the sizes to capture, length1 + length2 will equal size */
contiguous = contiguous_space_available(cpu, ANNOTATE_BUF);
if (size < contiguous) {
length1 = size;
length2 = 0;
} else {
length1 = contiguous;
length2 = size - contiguous;
}
if (annotate_copy(file, buf, length1) != 0) {
size = -EINVAL;
goto annotate_write_out;
}
if (length2 > 0 && annotate_copy(file, &buf[length1], length2) != 0) {
size = -EINVAL;
goto annotate_write_out;
}
/* Check and commit; commit is set to occur once buffer is 3/4 full */
buffer_check(cpu, ANNOTATE_BUF, time);
}
annotate_write_out:
spin_unlock(&annotate_lock);
/* return the number of bytes written */
return size;
}
static int gator_events_meminfo_read_proc(long long **buffer, struct task_struct *task)
{
struct mm_struct *mm;
u64 share = 0;
int i;
long long value;
int len = 0;
int cpu = get_physical_cpu();
long long *buf = per_cpu(proc_buffer, cpu);
if (!proc_global_enabled)
return 0;
/* Collect the memory stats of the process instead of the thread */
if (task->group_leader != NULL)
task = task->group_leader;
/* get_task_mm/mmput is not needed in this context because the task and it's mm are required as part of the sched_switch */
mm = task->mm;
if (mm == NULL)
return 0;
/* Derived from task_statm in fs/proc/task_mmu.c */
if (meminfo_enabled[MEMINFO_MEMUSED] || proc_enabled[PROC_SHARE]) {
share = get_mm_counter(mm,
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 32) && LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 34)
file_rss
#else
MM_FILEPAGES
#endif
);
}
/* key of 1 indicates a pid */
buf[len++] = 1;
buf[len++] = task->pid;
for (i = 0; i < PROC_COUNT; ++i) {
if (proc_enabled[i]) {
switch (i) {
case PROC_SIZE:
value = mm->total_vm;
break;
case PROC_SHARE:
value = share;
break;
case PROC_TEXT:
value = (PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK)) >> PAGE_SHIFT;
break;
case PROC_DATA:
value = mm->total_vm - mm->shared_vm;
break;
}
buf[len++] = proc_keys[i];
buf[len++] = value * PAGE_SIZE;
}
}
if (meminfo_enabled[MEMINFO_MEMUSED]) {
value = share + get_mm_counter(mm,
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 32) && LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 34)
anon_rss
#else
MM_ANONPAGES
#endif
);
/* Send resident for this pid */
buf[len++] = meminfo_keys[MEMINFO_MEMUSED];
buf[len++] = value * PAGE_SIZE;
}
/* Clear pid */
buf[len++] = 1;
buf[len++] = 0;
if (buffer)
*buffer = buf;
return len;
}
GATOR_DEFINE_PROBE(softirq_exit, TP_PROTO(unsigned int vec_nr))
#endif
{
atomic_inc(&per_cpu(irqCnt, get_physical_cpu())[SOFTIRQ]);
}
GATOR_DEFINE_PROBE(irq_handler_exit,
TP_PROTO(int irq, struct irqaction *action, int ret))
{
atomic_inc(&per_cpu(irqCnt, get_physical_cpu())[HARDIRQ]);
}
static bool marshal_cookie_header(const char *text)
{
int cpu = get_physical_cpu();
return buffer_check_space(cpu, NAME_BUF, strlen(text) + 3 * MAXSIZE_PACK32);
}
