static int annotate_release(struct inode *inode, struct file *file)
{
int cpu = 0;
/* synchronize between cores */
spin_lock(&annotate_lock);
if (per_cpu(gator_buffer, cpu)[ANNOTATE_BUF] && buffer_check_space(cpu, ANNOTATE_BUF, MAXSIZE_PACK64 + 3 * MAXSIZE_PACK32)) {
uint32_t pid = current->pid;
gator_buffer_write_packed_int(cpu, ANNOTATE_BUF, get_physical_cpu());
gator_buffer_write_packed_int(cpu, ANNOTATE_BUF, pid);
/* time */
gator_buffer_write_packed_int64(cpu, ANNOTATE_BUF, 0);
/* size */
gator_buffer_write_packed_int(cpu, ANNOTATE_BUF, 0);
}
/* Check and commit; commit is set to occur once buffer is 3/4 full */
buffer_check(cpu, ANNOTATE_BUF, gator_get_time());
spin_unlock(&annotate_lock);
return 0;
}
static void ebs_overflow_handler(struct perf_event *event, struct perf_sample_data *data, struct pt_regs *regs)
#endif
{
unsigned int value, delta, cpu = smp_processor_id(), buftype = EVENT_BUF;
if (event != per_cpu(pevent, cpu))
return;
if (buffer_check_space(cpu, buftype, 5 * MAXSIZE_PACK32 + MAXSIZE_PACK64)) {
value = local64_read(&event->count);
delta = value - per_cpu(prev_value, cpu);
per_cpu(prev_value, cpu) = value;
// Counters header
gator_buffer_write_packed_int(cpu, buftype, MESSAGE_COUNTERS); // type
gator_buffer_write_packed_int64(cpu, buftype, gator_get_time()); // time
// Output counter
gator_buffer_write_packed_int(cpu, buftype, 2); // length
gator_buffer_write_packed_int(cpu, buftype, per_cpu(key, cpu)); // key
gator_buffer_write_packed_int(cpu, buftype, delta); // delta
// End Counters, length of zero
gator_buffer_write_packed_int(cpu, buftype, 0);
}
// Output backtrace
if (buffer_check_space(cpu, buftype, gator_backtrace_depth * 2 * MAXSIZE_PACK32))
gator_add_sample(cpu, buftype, regs);
// Check and commit; commit is set to occur once buffer is 3/4 full
buffer_check(cpu, buftype);
}
static void marshal_cookie(int cookie, const char *text)
{
int cpu = get_physical_cpu();
/* buffer_check_space already called by marshal_cookie_header */
gator_buffer_write_packed_int(cpu, NAME_BUF, MESSAGE_COOKIE);
gator_buffer_write_packed_int(cpu, NAME_BUF, cookie);
gator_buffer_write_string(cpu, NAME_BUF, text);
buffer_check(cpu, NAME_BUF, gator_get_time());
}
static irqreturn_t armv7_pmnc_interrupt(int irq, void *arg)
{
unsigned int cnt, cpu = smp_processor_id(), buftype = EVENT_BUF;
struct pt_regs * const regs = get_irq_regs();
u32 flags;
// Stop irq generation
armv7_pmnc_write(armv7_pmnc_read() & ~PMNC_E);
// Get and reset overflow status flags
flags = armv7_pmnc_reset_interrupt();
// Counters header
gator_buffer_write_packed_int(cpu, buftype, MESSAGE_COUNTERS); // type
gator_buffer_write_packed_int64(cpu, buftype, gator_get_time()); // time
// Cycle counter
if (flags & (1 << 31)) {
int value = armv7_ccnt_read(pmnc_count[CCNT]); // overrun
gator_buffer_write_packed_int(cpu, buftype, 2); // length
gator_buffer_write_packed_int(cpu, buftype, pmnc_key[CCNT]); // key
gator_buffer_write_packed_int(cpu, buftype, value); // value
}
// PMNC counters
for (cnt = CNT0; cnt < CNTMAX; cnt++) {
if (flags & (1 << (cnt - CNT0))) {
int value = armv7_cntn_read(cnt, pmnc_count[cnt]); // overrun
gator_buffer_write_packed_int(cpu, buftype, 2); // length
gator_buffer_write_packed_int(cpu, buftype, pmnc_key[cnt]); // key
gator_buffer_write_packed_int(cpu, buftype, value); // value
}
}
// End Counters, length of zero
gator_buffer_write_packed_int(cpu, buftype, 0);
// Output backtrace
gator_add_sample(cpu, buftype, regs);
// Check and commit; commit is set to occur once buffer is 3/4 full
event_buffer_check(cpu);
// Allow irq generation
armv7_pmnc_write(armv7_pmnc_read() | PMNC_E);
return IRQ_HANDLED;
}
static void marshal_core_name(const int core, const int cpuid, const char *name)
{
int cpu = get_physical_cpu();
unsigned long flags;
local_irq_save(flags);
if (buffer_check_space(cpu, SUMMARY_BUF, MAXSIZE_PACK32 + MAXSIZE_CORE_NAME)) {
gator_buffer_write_packed_int(cpu, SUMMARY_BUF, MESSAGE_CORE_NAME);
gator_buffer_write_packed_int(cpu, SUMMARY_BUF, core);
gator_buffer_write_packed_int(cpu, SUMMARY_BUF, cpuid);
gator_buffer_write_string(cpu, SUMMARY_BUF, name);
}
/* Commit core names now so that they can show up in live */
local_irq_restore(flags);
gator_commit_buffer(cpu, SUMMARY_BUF, gator_get_time());
}
static void marshal_thread_name(int pid, char *name)
{
unsigned long flags, cpu;
u64 time;
local_irq_save(flags);
cpu = get_physical_cpu();
time = gator_get_time();
if (buffer_check_space(cpu, NAME_BUF, TASK_COMM_LEN + 3 * MAXSIZE_PACK32 + MAXSIZE_PACK64)) {
gator_buffer_write_packed_int(cpu, NAME_BUF, MESSAGE_THREAD_NAME);
gator_buffer_write_packed_int64(cpu, NAME_BUF, time);
gator_buffer_write_packed_int(cpu, NAME_BUF, pid);
gator_buffer_write_string(cpu, NAME_BUF, name);
}
local_irq_restore(flags);
buffer_check(cpu, NAME_BUF, time);
}
static void marshal_idle(int core, int state)
{
unsigned long flags, cpu;
u64 time;
local_irq_save(flags);
cpu = get_physical_cpu();
time = gator_get_time();
if (buffer_check_space(cpu, IDLE_BUF, MAXSIZE_PACK64 + 2 * MAXSIZE_PACK32)) {
gator_buffer_write_packed_int(cpu, IDLE_BUF, state);
gator_buffer_write_packed_int64(cpu, IDLE_BUF, time);
gator_buffer_write_packed_int(cpu, IDLE_BUF, core);
}
local_irq_restore(flags);
/* Check and commit; commit is set to occur once buffer is 3/4 full */
buffer_check(cpu, IDLE_BUF, time);
}
static void marshal_summary(long long timestamp, long long uptime, long long monotonic_delta, const char *uname)
{
unsigned long flags;
int cpu = 0;
char buf[32];
local_irq_save(flags);
gator_buffer_write_packed_int(cpu, SUMMARY_BUF, MESSAGE_SUMMARY);
gator_buffer_write_string(cpu, SUMMARY_BUF, NEWLINE_CANARY);
gator_buffer_write_packed_int64(cpu, SUMMARY_BUF, timestamp);
gator_buffer_write_packed_int64(cpu, SUMMARY_BUF, uptime);
gator_buffer_write_packed_int64(cpu, SUMMARY_BUF, monotonic_delta);
gator_buffer_write_string(cpu, SUMMARY_BUF, "uname");
gator_buffer_write_string(cpu, SUMMARY_BUF, uname);
gator_buffer_write_string(cpu, SUMMARY_BUF, "PAGESIZE");
snprintf(buf, sizeof(buf), "%lu", PAGE_SIZE);
gator_buffer_write_string(cpu, SUMMARY_BUF, buf);
#if GATOR_IKS_SUPPORT
gator_buffer_write_string(cpu, SUMMARY_BUF, "iks");
gator_buffer_write_string(cpu, SUMMARY_BUF, "");
#endif
#ifdef CONFIG_PREEMPT_RTB
gator_buffer_write_string(cpu, SUMMARY_BUF, "preempt_rtb");
gator_buffer_write_string(cpu, SUMMARY_BUF, "");
#endif
#ifdef CONFIG_PREEMPT_RT_FULL
gator_buffer_write_string(cpu, SUMMARY_BUF, "preempt_rt_full");
gator_buffer_write_string(cpu, SUMMARY_BUF, "");
#endif
/* Let Streamline know which GPU is used so that it can label the GPU Activity appropriately. This is a temporary fix, to be improved in a future release. */
#ifdef MALI_SUPPORT
gator_buffer_write_string(cpu, SUMMARY_BUF, "mali_type");
#if (MALI_SUPPORT == MALI_4xx)
gator_buffer_write_string(cpu, SUMMARY_BUF, "4xx");
#elif (MALI_SUPPORT == MALI_MIDGARD)
gator_buffer_write_string(cpu, SUMMARY_BUF, "6xx");
#else
gator_buffer_write_string(cpu, SUMMARY_BUF, "unknown");
#endif
#endif
gator_buffer_write_string(cpu, SUMMARY_BUF, "");
/* Commit the buffer now so it can be one of the first frames read by Streamline */
local_irq_restore(flags);
gator_commit_buffer(cpu, SUMMARY_BUF, gator_get_time());
}
static void __maybe_unused marshal_event_single64(int core, int key, long long value)
{
unsigned long flags, cpu;
u64 time;
local_irq_save(flags);
cpu = get_physical_cpu();
time = gator_get_time();
if (buffer_check_space(cpu, COUNTER_BUF, 2 * MAXSIZE_PACK64 + 2 * MAXSIZE_PACK32)) {
gator_buffer_write_packed_int64(cpu, COUNTER_BUF, time);
gator_buffer_write_packed_int(cpu, COUNTER_BUF, core);
gator_buffer_write_packed_int(cpu, COUNTER_BUF, key);
gator_buffer_write_packed_int64(cpu, COUNTER_BUF, value);
}
local_irq_restore(flags);
/* Check and commit; commit is set to occur once buffer is 3/4 full */
buffer_check(cpu, COUNTER_BUF, time);
}
static void marshal_link(int cookie, int tgid, int pid)
{
unsigned long cpu = get_physical_cpu(), flags;
u64 time;
local_irq_save(flags);
time = gator_get_time();
if (buffer_check_space(cpu, ACTIVITY_BUF, MAXSIZE_PACK64 + 5 * MAXSIZE_PACK32)) {
gator_buffer_write_packed_int(cpu, ACTIVITY_BUF, MESSAGE_LINK);
gator_buffer_write_packed_int64(cpu, ACTIVITY_BUF, time);
gator_buffer_write_packed_int(cpu, ACTIVITY_BUF, cookie);
gator_buffer_write_packed_int(cpu, ACTIVITY_BUF, tgid);
gator_buffer_write_packed_int(cpu, ACTIVITY_BUF, pid);
}
local_irq_restore(flags);
/* Check and commit; commit is set to occur once buffer is 3/4 full */
buffer_check(cpu, ACTIVITY_BUF, time);
}
static void marshal_sched_trace_exit(int tgid, int pid)
{
unsigned long cpu = get_physical_cpu(), flags;
u64 time;
if (!per_cpu(gator_buffer, cpu)[SCHED_TRACE_BUF])
return;
local_irq_save(flags);
time = gator_get_time();
if (buffer_check_space(cpu, SCHED_TRACE_BUF, MAXSIZE_PACK64 + 2 * MAXSIZE_PACK32)) {
gator_buffer_write_packed_int(cpu, SCHED_TRACE_BUF, MESSAGE_SCHED_EXIT);
gator_buffer_write_packed_int64(cpu, SCHED_TRACE_BUF, time);
gator_buffer_write_packed_int(cpu, SCHED_TRACE_BUF, pid);
}
local_irq_restore(flags);
/* Check and commit; commit is set to occur once buffer is 3/4 full */
buffer_check(cpu, SCHED_TRACE_BUF, time);
}
static void gator_event_sampling_online(void)
{
int cpu = smp_processor_id(), buftype = EVENT_BUF;
// read the counter and toss the invalid data, return zero instead
struct perf_event * ev = per_cpu(pevent, cpu);
if (ev != NULL && ev->state == PERF_EVENT_STATE_ACTIVE) {
ev->pmu->read(ev);
per_cpu(prev_value, cpu) = local64_read(&ev->count);
// Counters header
gator_buffer_write_packed_int(cpu, buftype, MESSAGE_COUNTERS); // type
gator_buffer_write_packed_int64(cpu, buftype, gator_get_time()); // time
// Output counter
gator_buffer_write_packed_int(cpu, buftype, 2); // length
gator_buffer_write_packed_int(cpu, buftype, per_cpu(key, cpu)); // key
gator_buffer_write_packed_int(cpu, buftype, 0); // delta - zero for initialization
// End Counters, length of zero
gator_buffer_write_packed_int(cpu, buftype, 0);
}
}
static void marshal_activity_switch(int core, int key, int activity, int pid, int state)
{
unsigned long cpu = get_physical_cpu(), flags;
u64 time;
if (!per_cpu(gator_buffer, cpu)[ACTIVITY_BUF])
return;
local_irq_save(flags);
time = gator_get_time();
if (buffer_check_space(cpu, ACTIVITY_BUF, MAXSIZE_PACK64 + 5 * MAXSIZE_PACK32)) {
gator_buffer_write_packed_int(cpu, ACTIVITY_BUF, MESSAGE_SWITCH);
gator_buffer_write_packed_int64(cpu, ACTIVITY_BUF, time);
gator_buffer_write_packed_int(cpu, ACTIVITY_BUF, core);
gator_buffer_write_packed_int(cpu, ACTIVITY_BUF, key);
gator_buffer_write_packed_int(cpu, ACTIVITY_BUF, activity);
gator_buffer_write_packed_int(cpu, ACTIVITY_BUF, pid);
gator_buffer_write_packed_int(cpu, ACTIVITY_BUF, state);
}
local_irq_restore(flags);
/* Check and commit; commit is set to occur once buffer is 3/4 full */
buffer_check(cpu, ACTIVITY_BUF, time);
}
static ssize_t annotate_write(struct file *file, char const __user *buf, size_t count_orig, loff_t *offset)
{
int pid, cpu, header_size, available, contiguous, length1, length2, size, count = count_orig & 0x7fffffff;
bool interrupt_context;
if (*offset)
return -EINVAL;
interrupt_context = in_interrupt();
/* Annotations are not supported in interrupt context, but may work
* if you comment out the the next four lines of code. By doing so,
* annotations in interrupt context can result in deadlocks and lost
* data.
*/
if (interrupt_context) {
pr_warning("gator: Annotations are not supported in interrupt context. Edit gator_annotate.c in the gator driver to enable annotations in interrupt context.\n");
return -EINVAL;
}
retry:
/* synchronize between cores and with collect_annotations */
spin_lock(&annotate_lock);
if (!collect_annotations) {
/* Not collecting annotations, tell the caller everything was written */
size = count_orig;
goto annotate_write_out;
}
/* Annotation only uses a single per-cpu buffer as the data must be in order to the engine */
cpu = 0;
if (current == NULL)
pid = 0;
else
pid = current->pid;
/* determine total size of the payload */
header_size = MAXSIZE_PACK32 * 3 + MAXSIZE_PACK64;
available = buffer_bytes_available(cpu, ANNOTATE_BUF) - header_size;
size = count < available ? count : available;
if (size <= 0) {
/* Buffer is full, wait until space is available */
spin_unlock(&annotate_lock);
/* Drop the annotation as blocking is not allowed in interrupt context */
if (interrupt_context)
return -EINVAL;
wait_event_interruptible(gator_annotate_wait, buffer_bytes_available(cpu, ANNOTATE_BUF) > header_size || !collect_annotations);
/* Check to see if a signal is pending */
if (signal_pending(current))
return -EINTR;
goto retry;
}
/* synchronize shared variables annotateBuf and annotatePos */
if (per_cpu(gator_buffer, cpu)[ANNOTATE_BUF]) {
u64 time = gator_get_time();
gator_buffer_write_packed_int(cpu, ANNOTATE_BUF, get_physical_cpu());
gator_buffer_write_packed_int(cpu, ANNOTATE_BUF, pid);
gator_buffer_write_packed_int64(cpu, ANNOTATE_BUF, time);
gator_buffer_write_packed_int(cpu, ANNOTATE_BUF, size);
/* determine the sizes to capture, length1 + length2 will equal size */
contiguous = contiguous_space_available(cpu, ANNOTATE_BUF);
if (size < contiguous) {
length1 = size;
length2 = 0;
} else {
length1 = contiguous;
length2 = size - contiguous;
}
if (annotate_copy(file, buf, length1) != 0) {
size = -EINVAL;
goto annotate_write_out;
}
if (length2 > 0 && annotate_copy(file, &buf[length1], length2) != 0) {
size = -EINVAL;
goto annotate_write_out;
}
/* Check and commit; commit is set to occur once buffer is 3/4 full */
buffer_check(cpu, ANNOTATE_BUF, time);
}
annotate_write_out:
spin_unlock(&annotate_lock);
/* return the number of bytes written */
return size;
}