static void ebs_overflow_handler(struct perf_event *event, struct perf_sample_data *data, struct pt_regs *regs)
#endif
{
unsigned int value, delta, cpu = smp_processor_id(), buftype = EVENT_BUF;
if (event != per_cpu(pevent, cpu))
return;
if (buffer_check_space(cpu, buftype, 5 * MAXSIZE_PACK32 + MAXSIZE_PACK64)) {
value = local64_read(&event->count);
delta = value - per_cpu(prev_value, cpu);
per_cpu(prev_value, cpu) = value;
// Counters header
gator_buffer_write_packed_int(cpu, buftype, MESSAGE_COUNTERS); // type
gator_buffer_write_packed_int64(cpu, buftype, gator_get_time()); // time
// Output counter
gator_buffer_write_packed_int(cpu, buftype, 2); // length
gator_buffer_write_packed_int(cpu, buftype, per_cpu(key, cpu)); // key
gator_buffer_write_packed_int(cpu, buftype, delta); // delta
// End Counters, length of zero
gator_buffer_write_packed_int(cpu, buftype, 0);
}
// Output backtrace
if (buffer_check_space(cpu, buftype, gator_backtrace_depth * 2 * MAXSIZE_PACK32))
gator_add_sample(cpu, buftype, regs);
// Check and commit; commit is set to occur once buffer is 3/4 full
buffer_check(cpu, buftype);
}
static int annotate_release(struct inode *inode, struct file *file)
{
int cpu = 0;
/* synchronize between cores */
spin_lock(&annotate_lock);
if (per_cpu(gator_buffer, cpu)[ANNOTATE_BUF] && buffer_check_space(cpu, ANNOTATE_BUF, MAXSIZE_PACK64 + 3 * MAXSIZE_PACK32)) {
uint32_t pid = current->pid;
gator_buffer_write_packed_int(cpu, ANNOTATE_BUF, get_physical_cpu());
gator_buffer_write_packed_int(cpu, ANNOTATE_BUF, pid);
/* time */
gator_buffer_write_packed_int64(cpu, ANNOTATE_BUF, 0);
/* size */
gator_buffer_write_packed_int(cpu, ANNOTATE_BUF, 0);
}
/* Check and commit; commit is set to occur once buffer is 3/4 full */
buffer_check(cpu, ANNOTATE_BUF, gator_get_time());
spin_unlock(&annotate_lock);
return 0;
}
static bool marshal_event_header(u64 time)
{
unsigned long flags, cpu = get_physical_cpu();
bool retval = false;
local_irq_save(flags);
if (buffer_check_space(cpu, BLOCK_COUNTER_BUF, MAXSIZE_PACK32 + MAXSIZE_PACK64)) {
gator_buffer_write_packed_int(cpu, BLOCK_COUNTER_BUF, 0); /* key of zero indicates a timestamp */
gator_buffer_write_packed_int64(cpu, BLOCK_COUNTER_BUF, time);
retval = true;
}
local_irq_restore(flags);
return retval;
}
static void marshal_core_name(const int core, const int cpuid, const char *name)
{
int cpu = get_physical_cpu();
unsigned long flags;
local_irq_save(flags);
if (buffer_check_space(cpu, SUMMARY_BUF, MAXSIZE_PACK32 + MAXSIZE_CORE_NAME)) {
gator_buffer_write_packed_int(cpu, SUMMARY_BUF, MESSAGE_CORE_NAME);
gator_buffer_write_packed_int(cpu, SUMMARY_BUF, core);
gator_buffer_write_packed_int(cpu, SUMMARY_BUF, cpuid);
gator_buffer_write_string(cpu, SUMMARY_BUF, name);
}
/* Commit core names now so that they can show up in live */
local_irq_restore(flags);
gator_commit_buffer(cpu, SUMMARY_BUF, gator_get_time());
}
static void marshal_thread_name(int pid, char *name)
{
unsigned long flags, cpu;
u64 time;
local_irq_save(flags);
cpu = get_physical_cpu();
time = gator_get_time();
if (buffer_check_space(cpu, NAME_BUF, TASK_COMM_LEN + 3 * MAXSIZE_PACK32 + MAXSIZE_PACK64)) {
gator_buffer_write_packed_int(cpu, NAME_BUF, MESSAGE_THREAD_NAME);
gator_buffer_write_packed_int64(cpu, NAME_BUF, time);
gator_buffer_write_packed_int(cpu, NAME_BUF, pid);
gator_buffer_write_string(cpu, NAME_BUF, name);
}
local_irq_restore(flags);
buffer_check(cpu, NAME_BUF, time);
}
static void marshal_idle(int core, int state)
{
unsigned long flags, cpu;
u64 time;
local_irq_save(flags);
cpu = get_physical_cpu();
time = gator_get_time();
if (buffer_check_space(cpu, IDLE_BUF, MAXSIZE_PACK64 + 2 * MAXSIZE_PACK32)) {
gator_buffer_write_packed_int(cpu, IDLE_BUF, state);
gator_buffer_write_packed_int64(cpu, IDLE_BUF, time);
gator_buffer_write_packed_int(cpu, IDLE_BUF, core);
}
local_irq_restore(flags);
/* Check and commit; commit is set to occur once buffer is 3/4 full */
buffer_check(cpu, IDLE_BUF, time);
}
static void __maybe_unused marshal_event_single64(int core, int key, long long value)
{
unsigned long flags, cpu;
u64 time;
local_irq_save(flags);
cpu = get_physical_cpu();
time = gator_get_time();
if (buffer_check_space(cpu, COUNTER_BUF, 2 * MAXSIZE_PACK64 + 2 * MAXSIZE_PACK32)) {
gator_buffer_write_packed_int64(cpu, COUNTER_BUF, time);
gator_buffer_write_packed_int(cpu, COUNTER_BUF, core);
gator_buffer_write_packed_int(cpu, COUNTER_BUF, key);
gator_buffer_write_packed_int64(cpu, COUNTER_BUF, value);
}
local_irq_restore(flags);
/* Check and commit; commit is set to occur once buffer is 3/4 full */
buffer_check(cpu, COUNTER_BUF, time);
}
static bool marshal_backtrace_header(int exec_cookie, int tgid, int pid, u64 time)
{
int cpu = get_physical_cpu();
if (!buffer_check_space(cpu, BACKTRACE_BUF, MAXSIZE_PACK64 + 5 * MAXSIZE_PACK32 + gator_backtrace_depth * 2 * MAXSIZE_PACK32)) {
/* Check and commit; commit is set to occur once buffer is 3/4 full */
buffer_check(cpu, BACKTRACE_BUF, time);
return false;
}
gator_buffer_write_packed_int64(cpu, BACKTRACE_BUF, time);
gator_buffer_write_packed_int(cpu, BACKTRACE_BUF, exec_cookie);
gator_buffer_write_packed_int(cpu, BACKTRACE_BUF, tgid);
gator_buffer_write_packed_int(cpu, BACKTRACE_BUF, pid);
return true;
}
static void marshal_link(int cookie, int tgid, int pid)
{
unsigned long cpu = get_physical_cpu(), flags;
u64 time;
local_irq_save(flags);
time = gator_get_time();
if (buffer_check_space(cpu, ACTIVITY_BUF, MAXSIZE_PACK64 + 5 * MAXSIZE_PACK32)) {
gator_buffer_write_packed_int(cpu, ACTIVITY_BUF, MESSAGE_LINK);
gator_buffer_write_packed_int64(cpu, ACTIVITY_BUF, time);
gator_buffer_write_packed_int(cpu, ACTIVITY_BUF, cookie);
gator_buffer_write_packed_int(cpu, ACTIVITY_BUF, tgid);
gator_buffer_write_packed_int(cpu, ACTIVITY_BUF, pid);
}
local_irq_restore(flags);
/* Check and commit; commit is set to occur once buffer is 3/4 full */
buffer_check(cpu, ACTIVITY_BUF, time);
}
static void marshal_sched_trace_exit(int tgid, int pid)
{
unsigned long cpu = get_physical_cpu(), flags;
u64 time;
if (!per_cpu(gator_buffer, cpu)[SCHED_TRACE_BUF])
return;
local_irq_save(flags);
time = gator_get_time();
if (buffer_check_space(cpu, SCHED_TRACE_BUF, MAXSIZE_PACK64 + 2 * MAXSIZE_PACK32)) {
gator_buffer_write_packed_int(cpu, SCHED_TRACE_BUF, MESSAGE_SCHED_EXIT);
gator_buffer_write_packed_int64(cpu, SCHED_TRACE_BUF, time);
gator_buffer_write_packed_int(cpu, SCHED_TRACE_BUF, pid);
}
local_irq_restore(flags);
/* Check and commit; commit is set to occur once buffer is 3/4 full */
buffer_check(cpu, SCHED_TRACE_BUF, time);
}
static void marshal_activity_switch(int core, int key, int activity, int pid, int state)
{
unsigned long cpu = get_physical_cpu(), flags;
u64 time;
if (!per_cpu(gator_buffer, cpu)[ACTIVITY_BUF])
return;
local_irq_save(flags);
time = gator_get_time();
if (buffer_check_space(cpu, ACTIVITY_BUF, MAXSIZE_PACK64 + 5 * MAXSIZE_PACK32)) {
gator_buffer_write_packed_int(cpu, ACTIVITY_BUF, MESSAGE_SWITCH);
gator_buffer_write_packed_int64(cpu, ACTIVITY_BUF, time);
gator_buffer_write_packed_int(cpu, ACTIVITY_BUF, core);
gator_buffer_write_packed_int(cpu, ACTIVITY_BUF, key);
gator_buffer_write_packed_int(cpu, ACTIVITY_BUF, activity);
gator_buffer_write_packed_int(cpu, ACTIVITY_BUF, pid);
gator_buffer_write_packed_int(cpu, ACTIVITY_BUF, state);
}
local_irq_restore(flags);
/* Check and commit; commit is set to occur once buffer is 3/4 full */
buffer_check(cpu, ACTIVITY_BUF, time);
}
static void marshal_event64(int len, long long *buffer64)
{
unsigned long i, flags, cpu = get_physical_cpu();
if (len <= 0)
return;
/* length must be even since all data is a (key, value) pair */
if (len & 0x1) {
pr_err("gator: invalid counter data detected and discarded\n");
return;
}
/* events must be written in key,value pairs */
local_irq_save(flags);
for (i = 0; i < len; i += 2) {
if (!buffer_check_space(cpu, BLOCK_COUNTER_BUF, 2 * MAXSIZE_PACK64))
break;
gator_buffer_write_packed_int64(cpu, BLOCK_COUNTER_BUF, buffer64[i]);
gator_buffer_write_packed_int64(cpu, BLOCK_COUNTER_BUF, buffer64[i + 1]);
}
local_irq_restore(flags);
}
static bool marshal_cookie_header(const char *text)
{
int cpu = get_physical_cpu();
return buffer_check_space(cpu, NAME_BUF, strlen(text) + 3 * MAXSIZE_PACK32);
}
