static unsigned long __user *
user_backtrace(unsigned long __user *tail,
struct quadd_callchain *callchain_data,
struct vm_area_struct *stack_vma)
{
unsigned long value, value_lr = 0, value_fp = 0;
unsigned long __user *fp_prev = NULL;
if (check_vma_address((unsigned long)tail, stack_vma))
return NULL;
if (__copy_from_user_inatomic(&value, tail, sizeof(unsigned long)))
return NULL;
if (!check_vma_address(value, stack_vma)) {
/* clang's frame */
value_fp = value;
if (check_vma_address((unsigned long)(tail + 1), stack_vma))
return NULL;
if (__copy_from_user_inatomic(&value_lr, tail + 1,
sizeof(unsigned long)))
return NULL;
} else {
/* gcc's frame */
if (__copy_from_user_inatomic(&value_fp, tail - 1,
sizeof(unsigned long)))
return NULL;
if (check_vma_address(value_fp, stack_vma))
return NULL;
value_lr = value;
}
fp_prev = (unsigned long __user *)value_fp;
if (value_lr < QUADD_USER_SPACE_MIN_ADDR)
return NULL;
quadd_callchain_store(callchain_data, value_lr);
if (fp_prev <= tail)
return NULL;
return fp_prev;
}
static void
read_all_sources(struct pt_regs *regs, struct task_struct *task)
{
u32 state, extra_data = 0;
int i, vec_idx = 0, bt_size = 0;
int nr_events = 0, nr_positive_events = 0;
struct pt_regs *user_regs;
struct quadd_iovec vec[5];
struct hrt_event_value events[QUADD_MAX_COUNTERS];
u32 events_extra[QUADD_MAX_COUNTERS];
struct quadd_record_data record_data;
struct quadd_sample_data *s = &record_data.sample;
struct quadd_ctx *ctx = hrt.quadd_ctx;
struct quadd_cpu_context *cpu_ctx = this_cpu_ptr(hrt.cpu_ctx);
struct quadd_callchain *cc = &cpu_ctx->cc;
if (!regs)
return;
if (atomic_read(&cpu_ctx->nr_active) == 0)
return;
if (!task)
task = current;
rcu_read_lock();
if (!task_nsproxy(task)) {
rcu_read_unlock();
return;
}
rcu_read_unlock();
if (ctx->pmu && ctx->pmu_info.active)
nr_events += read_source(ctx->pmu, regs,
events, QUADD_MAX_COUNTERS);
if (ctx->pl310 && ctx->pl310_info.active)
nr_events += read_source(ctx->pl310, regs,
events + nr_events,
QUADD_MAX_COUNTERS - nr_events);
if (!nr_events)
return;
if (user_mode(regs))
user_regs = regs;
else
user_regs = current_pt_regs();
if (get_sample_data(s, regs, task))
return;
vec[vec_idx].base = &extra_data;
vec[vec_idx].len = sizeof(extra_data);
vec_idx++;
s->reserved = 0;
if (ctx->param.backtrace) {
cc->unw_method = hrt.unw_method;
bt_size = quadd_get_user_callchain(user_regs, cc, ctx, task);
if (!bt_size && !user_mode(regs)) {
unsigned long pc = instruction_pointer(user_regs);
cc->nr = 0;
#ifdef CONFIG_ARM64
cc->cs_64 = compat_user_mode(user_regs) ? 0 : 1;
#else
cc->cs_64 = 0;
#endif
bt_size += quadd_callchain_store(cc, pc,
QUADD_UNW_TYPE_KCTX);
}
if (bt_size > 0) {
int ip_size = cc->cs_64 ? sizeof(u64) : sizeof(u32);
int nr_types = DIV_ROUND_UP(bt_size, 8);
vec[vec_idx].base = cc->cs_64 ?
(void *)cc->ip_64 : (void *)cc->ip_32;
vec[vec_idx].len = bt_size * ip_size;
vec_idx++;
vec[vec_idx].base = cc->types;
vec[vec_idx].len = nr_types * sizeof(cc->types[0]);
vec_idx++;
if (cc->cs_64)
extra_data |= QUADD_SED_IP64;
}
extra_data |= cc->unw_method << QUADD_SED_UNW_METHOD_SHIFT;
s->reserved |= cc->unw_rc << QUADD_SAMPLE_URC_SHIFT;
}
s->callchain_nr = bt_size;
record_data.record_type = QUADD_RECORD_TYPE_SAMPLE;
s->events_flags = 0;
for (i = 0; i < nr_events; i++) {
u32 value = events[i].value;
if (value > 0) {
s->events_flags |= 1 << i;
events_extra[nr_positive_events++] = value;
}
}
if (nr_positive_events == 0)
return;
vec[vec_idx].base = events_extra;
vec[vec_idx].len = nr_positive_events * sizeof(events_extra[0]);
vec_idx++;
state = task->state;
if (state) {
s->state = 1;
vec[vec_idx].base = &state;
vec[vec_idx].len = sizeof(state);
vec_idx++;
} else {
s->state = 0;
}
quadd_put_sample(&record_data, vec, vec_idx);
}
unsigned int
quadd_get_user_callchain(struct pt_regs *regs,
struct quadd_callchain *callchain_data)
{
unsigned long fp, sp, pc, reg;
struct vm_area_struct *vma, *vma_pc;
unsigned long __user *tail = NULL;
struct mm_struct *mm = current->mm;
callchain_data->nr = 0;
if (!regs || !user_mode(regs) || !mm)
return 0;
if (thumb_mode(regs))
return 0;
fp = regs->ARM_fp;
sp = regs->ARM_sp;
pc = regs->ARM_pc;
if (fp == 0 || fp < sp || fp & 0x3)
return 0;
vma = find_vma(mm, sp);
if (check_vma_address(fp, vma))
return 0;
if (__copy_from_user_inatomic(®, (unsigned long __user *)fp,
sizeof(unsigned long)))
return 0;
if (reg > fp &&
!check_vma_address(reg, vma)) {
unsigned long value;
int read_lr = 0;
if (!check_vma_address(fp + sizeof(unsigned long), vma)) {
if (__copy_from_user_inatomic(
&value,
(unsigned long __user *)fp + 1,
sizeof(unsigned long)))
return 0;
vma_pc = find_vma(mm, pc);
read_lr = 1;
}
if (!read_lr || check_vma_address(value, vma_pc)) {
/* gcc: fp --> short frame tail (fp) */
if (regs->ARM_lr < QUADD_USER_SPACE_MIN_ADDR)
return 0;
quadd_callchain_store(callchain_data, regs->ARM_lr);
tail = (unsigned long __user *)reg;
}
}
if (!tail)
tail = (unsigned long __user *)fp;
while (tail && !((unsigned long)tail & 0x3))
tail = user_backtrace(tail, callchain_data, vma);
return callchain_data->nr;
}
