int check_hw_breakpoint(struct pt_regs *regs)
{
if (regs->debugcause & BIT(DEBUGCAUSE_IBREAK_BIT)) {
int i;
struct perf_event **bp = this_cpu_ptr(bp_on_reg);
for (i = 0; i < XCHAL_NUM_IBREAK; ++i) {
if (bp[i] && !bp[i]->attr.disabled &&
regs->pc == bp[i]->attr.bp_addr)
perf_bp_event(bp[i], regs);
}
return 0;
} else if (regs->debugcause & BIT(DEBUGCAUSE_DBREAK_BIT)) {
struct perf_event **bp = this_cpu_ptr(wp_on_reg);
int dbnum = (regs->debugcause & DEBUGCAUSE_DBNUM_MASK) >>
DEBUGCAUSE_DBNUM_SHIFT;
if (dbnum < XCHAL_NUM_DBREAK && bp[dbnum]) {
if (user_mode(regs)) {
perf_bp_event(bp[dbnum], regs);
} else {
set_thread_flag(TIF_DB_DISABLED);
xtensa_wsr(0, SREG_DBREAKC + dbnum);
}
} else {
WARN_ONCE(1,
"Wrong/unconfigured DBNUM reported in DEBUGCAUSE: %d\n",
dbnum);
}
return 0;
}
return -ENOENT;
}
int __kprobes hw_breakpoint_handler(struct die_args *args)
{
int rc = NOTIFY_STOP;
struct perf_event *bp;
struct pt_regs *regs = args->regs;
int stepped = 1;
struct arch_hw_breakpoint *info;
unsigned int instr;
unsigned long dar = regs->dar;
set_dabr(0);
rcu_read_lock();
bp = __get_cpu_var(bp_per_reg);
if (!bp)
goto out;
info = counter_arch_bp(bp);
if (bp->overflow_handler == ptrace_triggered) {
perf_bp_event(bp, regs);
rc = NOTIFY_DONE;
goto out;
}
info->extraneous_interrupt = !((bp->attr.bp_addr <= dar) &&
(dar - bp->attr.bp_addr < bp->attr.bp_len));
if (user_mode(regs)) {
bp->ctx->task->thread.last_hit_ubp = bp;
regs->msr |= MSR_SE;
goto out;
}
stepped = 0;
instr = 0;
if (!__get_user_inatomic(instr, (unsigned int *) regs->nip))
stepped = emulate_step(regs, instr);
if (!stepped) {
WARN(1, "Unable to handle hardware breakpoint. Breakpoint at "
"0x%lx will be disabled.", info->address);
perf_event_disable(bp);
goto out;
}
if (!info->extraneous_interrupt)
perf_bp_event(bp, regs);
set_dabr(info->address | info->type | DABR_TRANSLATION);
out:
rcu_read_unlock();
return rc;
}
/*
* Handle single-step exceptions following a DABR hit.
*/
int __kprobes single_step_dabr_instruction(struct die_args *args)
{
struct pt_regs *regs = args->regs;
struct perf_event *bp = NULL;
struct arch_hw_breakpoint *info;
bp = current->thread.last_hit_ubp;
/*
* Check if we are single-stepping as a result of a
* previous HW Breakpoint exception
*/
if (!bp)
return NOTIFY_DONE;
info = counter_arch_bp(bp);
/*
* We shall invoke the user-defined callback function in the single
* stepping handler to confirm to 'trigger-after-execute' semantics
*/
if (!info->extraneous_interrupt)
perf_bp_event(bp, regs);
set_dabr(info->address | info->type | DABR_TRANSLATION, info->dabrx);
current->thread.last_hit_ubp = NULL;
/*
* If the process was being single-stepped by ptrace, let the
* other single-step actions occur (e.g. generate SIGTRAP).
*/
if (test_thread_flag(TIF_SINGLESTEP))
return NOTIFY_DONE;
return NOTIFY_STOP;
}
int __kprobes single_step_dabr_instruction(struct die_args *args)
{
struct pt_regs *regs = args->regs;
struct perf_event *bp = NULL;
struct arch_hw_breakpoint *bp_info;
bp = current->thread.last_hit_ubp;
if (!bp)
return NOTIFY_DONE;
bp_info = counter_arch_bp(bp);
if (!bp_info->extraneous_interrupt)
perf_bp_event(bp, regs);
set_dabr(bp_info->address | bp_info->type | DABR_TRANSLATION);
current->thread.last_hit_ubp = NULL;
if (test_thread_flag(TIF_SINGLESTEP))
return NOTIFY_DONE;
return NOTIFY_STOP;
}
static int __kprobes hw_breakpoint_handler(struct die_args *args)
{
int cpu, i, rc = NOTIFY_STOP;
struct perf_event *bp;
unsigned int cmf, resume_mask;
cmf = sh_ubc->triggered_mask();
if (unlikely(!cmf))
return NOTIFY_DONE;
resume_mask = sh_ubc->active_mask();
sh_ubc->disable_all();
cpu = get_cpu();
for (i = 0; i < sh_ubc->num_events; i++) {
unsigned long event_mask = (1 << i);
if (likely(!(cmf & event_mask)))
continue;
rcu_read_lock();
bp = per_cpu(bp_per_reg[i], cpu);
if (bp)
rc = NOTIFY_DONE;
sh_ubc->clear_triggered_mask(event_mask);
if (!bp) {
rcu_read_unlock();
break;
}
if (bp->overflow_handler == ptrace_triggered)
resume_mask &= ~(1 << i);
perf_bp_event(bp, args->regs);
if (!arch_check_bp_in_kernelspace(bp)) {
siginfo_t info;
info.si_signo = args->signr;
info.si_errno = notifier_to_errno(rc);
info.si_code = TRAP_HWBKPT;
force_sig_info(args->signr, &info, current);
}
rcu_read_unlock();
}
if (cmf == 0)
rc = NOTIFY_DONE;
sh_ubc->enable_all(resume_mask);
put_cpu();
return rc;
}
static int watchpoint_handler(unsigned long addr, unsigned int esr,
struct pt_regs *regs)
{
int i, step = 0, *kernel_step, access;
u32 ctrl_reg;
u64 val, alignment_mask;
struct perf_event *wp, **slots;
struct debug_info *debug_info;
struct arch_hw_breakpoint *info;
struct arch_hw_breakpoint_ctrl ctrl;
slots = this_cpu_ptr(wp_on_reg);
debug_info = ¤t->thread.debug;
for (i = 0; i < core_num_wrps; ++i) {
rcu_read_lock();
wp = slots[i];
if (wp == NULL)
goto unlock;
info = counter_arch_bp(wp);
/* AArch32 watchpoints are either 4 or 8 bytes aligned. */
if (is_compat_task()) {
if (info->ctrl.len == ARM_BREAKPOINT_LEN_8)
alignment_mask = 0x7;
else
alignment_mask = 0x3;
} else {
alignment_mask = 0x7;
}
/* Check if the watchpoint value matches. */
val = read_wb_reg(AARCH64_DBG_REG_WVR, i);
if (val != (addr & ~alignment_mask))
goto unlock;
/* Possible match, check the byte address select to confirm. */
ctrl_reg = read_wb_reg(AARCH64_DBG_REG_WCR, i);
decode_ctrl_reg(ctrl_reg, &ctrl);
if (!((1 << (addr & alignment_mask)) & ctrl.len))
goto unlock;
/*
* Check that the access type matches.
* 0 => load, otherwise => store
*/
access = (esr & AARCH64_ESR_ACCESS_MASK) ? HW_BREAKPOINT_W :
HW_BREAKPOINT_R;
if (!(access & hw_breakpoint_type(wp)))
goto unlock;
info->trigger = addr;
perf_bp_event(wp, regs);
/* Do we need to handle the stepping? */
if (!wp->overflow_handler)
step = 1;
unlock:
rcu_read_unlock();
}
if (!step)
return 0;
/*
* We always disable EL0 watchpoints because the kernel can
* cause these to fire via an unprivileged access.
*/
toggle_bp_registers(AARCH64_DBG_REG_WCR, DBG_ACTIVE_EL0, 0);
if (user_mode(regs)) {
debug_info->wps_disabled = 1;
/* If we're already stepping a breakpoint, just return. */
if (debug_info->bps_disabled)
return 0;
if (test_thread_flag(TIF_SINGLESTEP))
debug_info->suspended_step = 1;
else
user_enable_single_step(current);
} else {
toggle_bp_registers(AARCH64_DBG_REG_WCR, DBG_ACTIVE_EL1, 0);
kernel_step = this_cpu_ptr(&stepping_kernel_bp);
if (*kernel_step != ARM_KERNEL_STEP_NONE)
return 0;
if (kernel_active_single_step()) {
*kernel_step = ARM_KERNEL_STEP_SUSPEND;
} else {
*kernel_step = ARM_KERNEL_STEP_ACTIVE;
kernel_enable_single_step(regs);
}
}
return 0;
}
/*
* Debug exception handlers.
*/
static int breakpoint_handler(unsigned long unused, unsigned int esr,
struct pt_regs *regs)
{
int i, step = 0, *kernel_step;
u32 ctrl_reg;
u64 addr, val;
struct perf_event *bp, **slots;
struct debug_info *debug_info;
struct arch_hw_breakpoint_ctrl ctrl;
slots = this_cpu_ptr(bp_on_reg);
addr = instruction_pointer(regs);
debug_info = ¤t->thread.debug;
for (i = 0; i < core_num_brps; ++i) {
rcu_read_lock();
bp = slots[i];
if (bp == NULL)
goto unlock;
/* Check if the breakpoint value matches. */
val = read_wb_reg(AARCH64_DBG_REG_BVR, i);
if (val != (addr & ~0x3))
goto unlock;
/* Possible match, check the byte address select to confirm. */
ctrl_reg = read_wb_reg(AARCH64_DBG_REG_BCR, i);
decode_ctrl_reg(ctrl_reg, &ctrl);
if (!((1 << (addr & 0x3)) & ctrl.len))
goto unlock;
counter_arch_bp(bp)->trigger = addr;
perf_bp_event(bp, regs);
/* Do we need to handle the stepping? */
if (!bp->overflow_handler)
step = 1;
unlock:
rcu_read_unlock();
}
if (!step)
return 0;
if (user_mode(regs)) {
debug_info->bps_disabled = 1;
toggle_bp_registers(AARCH64_DBG_REG_BCR, DBG_ACTIVE_EL0, 0);
/* If we're already stepping a watchpoint, just return. */
if (debug_info->wps_disabled)
return 0;
if (test_thread_flag(TIF_SINGLESTEP))
debug_info->suspended_step = 1;
else
user_enable_single_step(current);
} else {
toggle_bp_registers(AARCH64_DBG_REG_BCR, DBG_ACTIVE_EL1, 0);
kernel_step = this_cpu_ptr(&stepping_kernel_bp);
if (*kernel_step != ARM_KERNEL_STEP_NONE)
return 0;
if (kernel_active_single_step()) {
*kernel_step = ARM_KERNEL_STEP_SUSPEND;
} else {
*kernel_step = ARM_KERNEL_STEP_ACTIVE;
kernel_enable_single_step(regs);
}
}
return 0;
}
static int __kprobes hw_breakpoint_handler(struct die_args *args)
{
int cpu, i, rc = NOTIFY_STOP;
struct perf_event *bp;
unsigned int cmf, resume_mask;
/*
* Do an early return if none of the channels triggered.
*/
cmf = sh_ubc->triggered_mask();
if (unlikely(!cmf))
return NOTIFY_DONE;
/*
* By default, resume all of the active channels.
*/
resume_mask = sh_ubc->active_mask();
/*
* Disable breakpoints during exception handling.
*/
sh_ubc->disable_all();
cpu = get_cpu();
for (i = 0; i < sh_ubc->num_events; i++) {
unsigned long event_mask = (1 << i);
if (likely(!(cmf & event_mask)))
continue;
/*
* The counter may be concurrently released but that can only
* occur from a call_rcu() path. We can then safely fetch
* the breakpoint, use its callback, touch its counter
* while we are in an rcu_read_lock() path.
*/
rcu_read_lock();
bp = per_cpu(bp_per_reg[i], cpu);
if (bp)
rc = NOTIFY_DONE;
/*
* Reset the condition match flag to denote completion of
* exception handling.
*/
sh_ubc->clear_triggered_mask(event_mask);
/*
* bp can be NULL due to concurrent perf counter
* removing.
*/
if (!bp) {
rcu_read_unlock();
break;
}
/*
* Don't restore the channel if the breakpoint is from
* ptrace, as it always operates in one-shot mode.
*/
if (bp->overflow_handler == ptrace_triggered)
resume_mask &= ~(1 << i);
perf_bp_event(bp, args->regs);
/* Deliver the signal to userspace */
if (arch_check_va_in_userspace(bp->attr.bp_addr,
bp->attr.bp_len)) {
siginfo_t info;
info.si_signo = args->signr;
info.si_errno = notifier_to_errno(rc);
info.si_code = TRAP_HWBKPT;
force_sig_info(args->signr, &info, current);
}
rcu_read_unlock();
}
if (cmf == 0)
rc = NOTIFY_DONE;
sh_ubc->enable_all(resume_mask);
put_cpu();
return rc;
}
/*
* Handle debug exception notifications.
*
* Return value is either NOTIFY_STOP or NOTIFY_DONE as explained below.
*
* NOTIFY_DONE returned if one of the following conditions is true.
* i) When the causative address is from user-space and the exception
* is a valid one, i.e. not triggered as a result of lazy debug register
* switching
* ii) When there are more bits than trap<n> set in DR6 register (such
* as BD, BS or BT) indicating that more than one debug condition is
* met and requires some more action in do_debug().
*
* NOTIFY_STOP returned for all other cases
*
*/
static int __kprobes hw_breakpoint_handler(struct die_args *args)
{
int i, cpu, rc = NOTIFY_STOP;
struct perf_event *bp;
unsigned long dr7, dr6;
unsigned long *dr6_p;
/* The DR6 value is pointed by args->err */
dr6_p = (unsigned long *)ERR_PTR(args->err);
dr6 = *dr6_p;
/* If it's a single step, TRAP bits are random */
if (dr6 & DR_STEP)
return NOTIFY_DONE;
/* Do an early return if no trap bits are set in DR6 */
if ((dr6 & DR_TRAP_BITS) == 0)
return NOTIFY_DONE;
get_debugreg(dr7, 7);
/* Disable breakpoints during exception handling */
set_debugreg(0UL, 7);
/*
* Assert that local interrupts are disabled
* Reset the DRn bits in the virtualized register value.
* The ptrace trigger routine will add in whatever is needed.
*/
current->thread.debugreg6 &= ~DR_TRAP_BITS;
cpu = get_cpu();
/* Handle all the breakpoints that were triggered */
for (i = 0; i < HBP_NUM; ++i) {
if (likely(!(dr6 & (DR_TRAP0 << i))))
continue;
/*
* The counter may be concurrently released but that can only
* occur from a call_rcu() path. We can then safely fetch
* the breakpoint, use its callback, touch its counter
* while we are in an rcu_read_lock() path.
*/
rcu_read_lock();
bp = per_cpu(bp_per_reg[i], cpu);
/*
* Reset the 'i'th TRAP bit in dr6 to denote completion of
* exception handling
*/
(*dr6_p) &= ~(DR_TRAP0 << i);
/*
* bp can be NULL due to lazy debug register switching
* or due to concurrent perf counter removing.
*/
if (!bp) {
rcu_read_unlock();
break;
}
perf_bp_event(bp, args->regs);
/*
* Set up resume flag to avoid breakpoint recursion when
* returning back to origin.
*/
if (bp->hw.info.type == X86_BREAKPOINT_EXECUTE)
args->regs->flags |= X86_EFLAGS_RF;
rcu_read_unlock();
}
/*
* Further processing in do_debug() is needed for a) user-space
* breakpoints (to generate signals) and b) when the system has
* taken exception due to multiple causes
*/
if ((current->thread.debugreg6 & DR_TRAP_BITS) ||
(dr6 & (~DR_TRAP_BITS)))
rc = NOTIFY_DONE;
set_debugreg(dr7, 7);
put_cpu();
return rc;
}
/*
* Handle debug exception notifications.
*
* Return value is either NOTIFY_STOP or NOTIFY_DONE as explained below.
*
* NOTIFY_DONE returned if one of the following conditions is true.
* i) When the causative address is from user-space and the exception
* is a valid one, i.e. not triggered as a result of lazy debug register
* switching
* ii) When there are more bits than trap<n> set in DR6 register (such
* as BD, BS or BT) indicating that more than one debug condition is
* met and requires some more action in do_debug().
*
* NOTIFY_STOP returned for all other cases
*
*/
static int __kprobes hw_breakpoint_handler(struct die_args *args)
{
int i, cpu, rc = NOTIFY_STOP;
struct perf_event *bp;
unsigned long dr7, dr6;
unsigned long *dr6_p;
/* The DR6 value is pointed by args->err */
dr6_p = (unsigned long *)ERR_PTR(args->err);
dr6 = *dr6_p;
/* If it's a single step, TRAP bits are random */
if (dr6 & DR_STEP)
return NOTIFY_DONE;
/* Do an early return if no trap bits are set in DR6 */
if ((dr6 & DR_TRAP_BITS) == 0)
return NOTIFY_DONE;
get_debugreg(dr7, 7);
/* Disable breakpoints during exception handling */
set_debugreg(0UL, 7);
/*
* Assert that local interrupts are disabled
* Reset the DRn bits in the virtualized register value.
* The ptrace trigger routine will add in whatever is needed.
*/
current->thread.debugreg6 &= ~DR_TRAP_BITS;
cpu = get_cpu();
/* Handle all the breakpoints that were triggered */
for (i = 0; i < HBP_NUM; ++i) {
if (likely(!(dr6 & (DR_TRAP0 << i))))
continue;
/*
* The counter may be concurrently released but that can only
* occur from a call_rcu() path. We can then safely fetch
* the breakpoint, use its callback, touch its counter
* while we are in an rcu_read_lock() path.
*/
rcu_read_lock();
bp = per_cpu(bp_per_reg[i], cpu);
if (bp)
rc = NOTIFY_DONE;
/*
* Reset the 'i'th TRAP bit in dr6 to denote completion of
* exception handling
*/
(*dr6_p) &= ~(DR_TRAP0 << i);
/*
* bp can be NULL due to lazy debug register switching
* or due to concurrent perf counter removing.
*/
if (!bp) {
rcu_read_unlock();
break;
}
perf_bp_event(bp, args->regs);
rcu_read_unlock();
}
if (dr6 & (~DR_TRAP_BITS))
rc = NOTIFY_DONE;
set_debugreg(dr7, 7);
put_cpu();
return rc;
}
static int __kprobes hw_breakpoint_handler(struct die_args *args)
{
int i, cpu, rc = NOTIFY_STOP;
struct perf_event *bp;
unsigned long dr7, dr6;
unsigned long *dr6_p;
/* */
dr6_p = (unsigned long *)ERR_PTR(args->err);
dr6 = *dr6_p;
/* */
if (dr6 & DR_STEP)
return NOTIFY_DONE;
/* */
if ((dr6 & DR_TRAP_BITS) == 0)
return NOTIFY_DONE;
get_debugreg(dr7, 7);
/* */
set_debugreg(0UL, 7);
/*
*/
current->thread.debugreg6 &= ~DR_TRAP_BITS;
cpu = get_cpu();
/* */
for (i = 0; i < HBP_NUM; ++i) {
if (likely(!(dr6 & (DR_TRAP0 << i))))
continue;
/*
*/
rcu_read_lock();
bp = per_cpu(bp_per_reg[i], cpu);
/*
*/
(*dr6_p) &= ~(DR_TRAP0 << i);
/*
*/
if (!bp) {
rcu_read_unlock();
break;
}
perf_bp_event(bp, args->regs);
/*
*/
if (bp->hw.info.type == X86_BREAKPOINT_EXECUTE)
args->regs->flags |= X86_EFLAGS_RF;
rcu_read_unlock();
}
/*
*/
if ((current->thread.debugreg6 & DR_TRAP_BITS) ||
(dr6 & (~DR_TRAP_BITS)))
rc = NOTIFY_DONE;
set_debugreg(dr7, 7);
put_cpu();
return rc;
}
/*
* Handle debug exception notifications.
*/
int __kprobes hw_breakpoint_handler(struct die_args *args)
{
int rc = NOTIFY_STOP;
struct perf_event *bp;
struct pt_regs *regs = args->regs;
int stepped = 1;
struct arch_hw_breakpoint *info;
unsigned int instr;
unsigned long dar = regs->dar;
/* Disable breakpoints during exception handling */
set_dabr(0, 0);
/*
* The counter may be concurrently released but that can only
* occur from a call_rcu() path. We can then safely fetch
* the breakpoint, use its callback, touch its counter
* while we are in an rcu_read_lock() path.
*/
rcu_read_lock();
bp = __get_cpu_var(bp_per_reg);
if (!bp)
goto out;
info = counter_arch_bp(bp);
/*
* Return early after invoking user-callback function without restoring
* DABR if the breakpoint is from ptrace which always operates in
* one-shot mode. The ptrace-ed process will receive the SIGTRAP signal
* generated in do_dabr().
*/
if (bp->overflow_handler == ptrace_triggered) {
perf_bp_event(bp, regs);
rc = NOTIFY_DONE;
goto out;
}
/*
* Verify if dar lies within the address range occupied by the symbol
* being watched to filter extraneous exceptions. If it doesn't,
* we still need to single-step the instruction, but we don't
* generate an event.
*/
info->extraneous_interrupt = !((bp->attr.bp_addr <= dar) &&
(dar - bp->attr.bp_addr < bp->attr.bp_len));
/* Do not emulate user-space instructions, instead single-step them */
if (user_mode(regs)) {
current->thread.last_hit_ubp = bp;
regs->msr |= MSR_SE;
goto out;
}
stepped = 0;
instr = 0;
if (!__get_user_inatomic(instr, (unsigned int *) regs->nip))
stepped = emulate_step(regs, instr);
/*
* emulate_step() could not execute it. We've failed in reliably
* handling the hw-breakpoint. Unregister it and throw a warning
* message to let the user know about it.
*/
if (!stepped) {
WARN(1, "Unable to handle hardware breakpoint. Breakpoint at "
"0x%lx will be disabled.", info->address);
perf_event_disable(bp);
goto out;
}
/*
* As a policy, the callback is invoked in a 'trigger-after-execute'
* fashion
*/
if (!info->extraneous_interrupt)
perf_bp_event(bp, regs);
set_dabr(info->address | info->type | DABR_TRANSLATION, info->dabrx);
out:
rcu_read_unlock();
return rc;
}