int __kprobes hw_breakpoint_handler(struct die_args *args)
{
int rc = NOTIFY_STOP;
struct perf_event *bp;
struct pt_regs *regs = args->regs;
int stepped = 1;
struct arch_hw_breakpoint *info;
unsigned int instr;
unsigned long dar = regs->dar;
set_dabr(0);
rcu_read_lock();
bp = __get_cpu_var(bp_per_reg);
if (!bp)
goto out;
info = counter_arch_bp(bp);
if (bp->overflow_handler == ptrace_triggered) {
perf_bp_event(bp, regs);
rc = NOTIFY_DONE;
goto out;
}
info->extraneous_interrupt = !((bp->attr.bp_addr <= dar) &&
(dar - bp->attr.bp_addr < bp->attr.bp_len));
if (user_mode(regs)) {
bp->ctx->task->thread.last_hit_ubp = bp;
regs->msr |= MSR_SE;
goto out;
}
stepped = 0;
instr = 0;
if (!__get_user_inatomic(instr, (unsigned int *) regs->nip))
stepped = emulate_step(regs, instr);
if (!stepped) {
WARN(1, "Unable to handle hardware breakpoint. Breakpoint at "
"0x%lx will be disabled.", info->address);
perf_event_disable(bp);
goto out;
}
if (!info->extraneous_interrupt)
perf_bp_event(bp, regs);
set_dabr(info->address | info->type | DABR_TRANSLATION);
out:
rcu_read_unlock();
return rc;
}
/*
* Handle single-step exceptions following a DABR hit.
*/
int __kprobes single_step_dabr_instruction(struct die_args *args)
{
struct pt_regs *regs = args->regs;
struct perf_event *bp = NULL;
struct arch_hw_breakpoint *info;
bp = current->thread.last_hit_ubp;
/*
* Check if we are single-stepping as a result of a
* previous HW Breakpoint exception
*/
if (!bp)
return NOTIFY_DONE;
info = counter_arch_bp(bp);
/*
* We shall invoke the user-defined callback function in the single
* stepping handler to confirm to 'trigger-after-execute' semantics
*/
if (!info->extraneous_interrupt)
perf_bp_event(bp, regs);
set_dabr(info->address | info->type | DABR_TRANSLATION, info->dabrx);
current->thread.last_hit_ubp = NULL;
/*
* If the process was being single-stepped by ptrace, let the
* other single-step actions occur (e.g. generate SIGTRAP).
*/
if (test_thread_flag(TIF_SINGLESTEP))
return NOTIFY_DONE;
return NOTIFY_STOP;
}
void do_dabr(struct pt_regs *regs, unsigned long address,
unsigned long error_code)
{
siginfo_t info;
if (notify_die(DIE_DABR_MATCH, "dabr_match", regs, error_code,
11, SIGSEGV) == NOTIFY_STOP)
return;
if (debugger_dabr_match(regs))
return;
/* Clear the DAC and struct entries. One shot trigger */
#if defined(CONFIG_BOOKE)
mtspr(SPRN_DBCR0, mfspr(SPRN_DBCR0) & ~(DBSR_DAC1R | DBSR_DAC1W
| DBCR0_IDM));
#endif
/* Clear the DABR */
set_dabr(0);
/* Deliver the signal to userspace */
info.si_signo = SIGTRAP;
info.si_errno = 0;
info.si_code = TRAP_HWBKPT;
info.si_addr = (void __user *)address;
force_sig_info(SIGTRAP, &info, current);
}
static void set_debug_reg_defaults(struct thread_struct *thread)
{
if (thread->dabr) {
thread->dabr = 0;
set_dabr(0);
}
}
static int do_signal(struct pt_regs *regs)
{
sigset_t *oldset;
siginfo_t info;
int signr;
struct k_sigaction ka;
int ret;
int is32 = is_32bit_task();
if (current_thread_info()->local_flags & _TLF_RESTORE_SIGMASK)
oldset = ¤t->saved_sigmask;
else
oldset = ¤t->blocked;
signr = get_signal_to_deliver(&info, &ka, regs, NULL);
check_syscall_restart(regs, &ka, signr > 0);
if (signr <= 0) {
struct thread_info *ti = current_thread_info();
if (ti->local_flags & _TLF_RESTORE_SIGMASK) {
ti->local_flags &= ~_TLF_RESTORE_SIGMASK;
sigprocmask(SIG_SETMASK, ¤t->saved_sigmask, NULL);
}
regs->trap = 0;
return 0;
}
#ifndef CONFIG_PPC_ADV_DEBUG_REGS
if (current->thread.dabr)
set_dabr(current->thread.dabr);
#endif
thread_change_pc(current, regs);
if (is32) {
if (ka.sa.sa_flags & SA_SIGINFO)
ret = handle_rt_signal32(signr, &ka, &info, oldset,
regs);
else
ret = handle_signal32(signr, &ka, &info, oldset,
regs);
} else {
ret = handle_rt_signal64(signr, &ka, &info, oldset, regs);
}
regs->trap = 0;
if (ret) {
block_sigmask(&ka, signr);
current_thread_info()->local_flags &= ~_TLF_RESTORE_SIGMASK;
tracehook_signal_handler(signr, &info, &ka, regs,
test_thread_flag(TIF_SINGLESTEP));
}
return ret;
}
/*
* Restores the breakpoint on the debug registers.
* Invoke this function if it is known that the execution context is
* about to change to cause loss of MSR_SE settings.
*/
void thread_change_pc(struct task_struct *tsk, struct pt_regs *regs)
{
struct arch_hw_breakpoint *info;
if (likely(!tsk->thread.last_hit_ubp))
return;
info = counter_arch_bp(tsk->thread.last_hit_ubp);
regs->msr &= ~MSR_SE;
set_dabr(info->address | info->type | DABR_TRANSLATION, info->dabrx);
tsk->thread.last_hit_ubp = NULL;
}
int arch_install_hw_breakpoint(struct perf_event *bp)
{
struct arch_hw_breakpoint *info = counter_arch_bp(bp);
struct perf_event **slot = &__get_cpu_var(bp_per_reg);
*slot = bp;
if (current->thread.last_hit_ubp != bp)
set_dabr(info->address | info->type | DABR_TRANSLATION);
return 0;
}
/*
* Uninstall the breakpoint contained in the given counter.
*
* First we search the debug address register it uses and then we disable
* it.
*
* Atomic: we hold the counter->ctx->lock and we only handle variables
* and registers local to this cpu.
*/
void arch_uninstall_hw_breakpoint(struct perf_event *bp)
{
struct perf_event **slot = &__get_cpu_var(bp_per_reg);
if (*slot != bp) {
WARN_ONCE(1, "Can't find the breakpoint");
return;
}
*slot = NULL;
set_dabr(0, 0);
}
/*
* Note that 'init' is a special process: it doesn't get signals it doesn't
* want to handle. Thus you cannot kill init even with a SIGKILL even by
* mistake.
*/
int do_signal(sigset_t *oldset, struct pt_regs *regs)
{
siginfo_t info;
int signr;
struct k_sigaction ka;
/*
* If the current thread is 32 bit - invoke the
* 32 bit signal handling code
*/
if (test_thread_flag(TIF_32BIT))
return do_signal32(oldset, regs);
if (!oldset)
oldset = ¤t->blocked;
signr = get_signal_to_deliver(&info, &ka, regs, NULL);
if (signr > 0) {
/* Whee! Actually deliver the signal. */
if (TRAP(regs) == 0x0C00)
syscall_restart(regs, &ka);
/*
* Reenable the DABR before delivering the signal to
* user space. The DABR will have been cleared if it
* triggered inside the kernel.
*/
if (current->thread.dabr)
set_dabr(current->thread.dabr);
return handle_signal(signr, &ka, &info, oldset, regs);
}
if (TRAP(regs) == 0x0C00) { /* System Call! */
if ((int)regs->result == -ERESTARTNOHAND ||
(int)regs->result == -ERESTARTSYS ||
(int)regs->result == -ERESTARTNOINTR) {
regs->gpr[3] = regs->orig_gpr3;
regs->nip -= 4; /* Back up & retry system call */
regs->result = 0;
} else if ((int)regs->result == -ERESTART_RESTARTBLOCK) {
regs->gpr[0] = __NR_restart_syscall;
regs->nip -= 4;
regs->result = 0;
}
}
return 0;
}
/*
* Install a perf counter breakpoint.
*
* We seek a free debug address register and use it for this
* breakpoint.
*
* Atomic: we hold the counter->ctx->lock and we only handle variables
* and registers local to this cpu.
*/
int arch_install_hw_breakpoint(struct perf_event *bp)
{
struct arch_hw_breakpoint *info = counter_arch_bp(bp);
struct perf_event **slot = &__get_cpu_var(bp_per_reg);
*slot = bp;
/*
* Do not install DABR values if the instruction must be single-stepped.
* If so, DABR will be populated in single_step_dabr_instruction().
*/
if (current->thread.last_hit_ubp != bp)
set_dabr(info->address | info->type | DABR_TRANSLATION, info->dabrx);
return 0;
}
/* Deliver the signal to userspace */
info.si_signo = SIGTRAP;
info.si_errno = breakpt; /* breakpoint or watchpoint id */
info.si_code = signal_code;
info.si_addr = (void __user *)address;
force_sig_info(SIGTRAP, &info, current);
}
#else /* !CONFIG_PPC_ADV_DEBUG_REGS */
void do_dabr(struct pt_regs *regs, unsigned long address,
unsigned long error_code)
{
siginfo_t info;
if (notify_die(DIE_DABR_MATCH, "dabr_match", regs, error_code,
11, SIGSEGV) == NOTIFY_STOP)
return;
if (debugger_dabr_match(regs))
return;
/* Clear the DABR */
set_dabr(0);
/* Deliver the signal to userspace */
info.si_signo = SIGTRAP;
info.si_errno = 0;
info.si_code = TRAP_HWBKPT;
info.si_addr = (void __user *)address;
force_sig_info(SIGTRAP, &info, current);
}
int __kprobes single_step_dabr_instruction(struct die_args *args)
{
struct pt_regs *regs = args->regs;
struct perf_event *bp = NULL;
struct arch_hw_breakpoint *bp_info;
bp = current->thread.last_hit_ubp;
if (!bp)
return NOTIFY_DONE;
bp_info = counter_arch_bp(bp);
if (!bp_info->extraneous_interrupt)
perf_bp_event(bp, regs);
set_dabr(bp_info->address | bp_info->type | DABR_TRANSLATION);
current->thread.last_hit_ubp = NULL;
if (test_thread_flag(TIF_SINGLESTEP))
return NOTIFY_DONE;
return NOTIFY_STOP;
}
int do_signal(sigset_t *oldset, struct pt_regs *regs)
{
siginfo_t info;
int signr;
struct k_sigaction ka;
int ret;
int is32 = is_32bit_task();
if (test_thread_flag(TIF_RESTORE_SIGMASK))
oldset = ¤t->saved_sigmask;
else if (!oldset)
oldset = ¤t->blocked;
signr = get_signal_to_deliver(&info, &ka, regs, NULL);
/* Is there any syscall restart business here ? */
check_syscall_restart(regs, &ka, signr > 0);
if (signr <= 0) {
/* No signal to deliver -- put the saved sigmask back */
if (test_thread_flag(TIF_RESTORE_SIGMASK)) {
clear_thread_flag(TIF_RESTORE_SIGMASK);
sigprocmask(SIG_SETMASK, ¤t->saved_sigmask, NULL);
}
return 0; /* no signals delivered */
}
/*
* Reenable the DABR before delivering the signal to
* user space. The DABR will have been cleared if it
* triggered inside the kernel.
*/
if (current->thread.dabr)
set_dabr(current->thread.dabr);
if (is32) {
if (ka.sa.sa_flags & SA_SIGINFO)
ret = handle_rt_signal32(signr, &ka, &info, oldset,
regs);
else
ret = handle_signal32(signr, &ka, &info, oldset,
regs);
} else {
ret = handle_rt_signal64(signr, &ka, &info, oldset, regs);
}
if (ret) {
spin_lock_irq(¤t->sighand->siglock);
sigorsets(¤t->blocked, ¤t->blocked,
&ka.sa.sa_mask);
if (!(ka.sa.sa_flags & SA_NODEFER))
sigaddset(¤t->blocked, signr);
recalc_sigpending();
spin_unlock_irq(¤t->sighand->siglock);
/*
* A signal was successfully delivered; the saved sigmask is in
* its frame, and we can clear the TIF_RESTORE_SIGMASK flag.
*/
if (test_thread_flag(TIF_RESTORE_SIGMASK))
clear_thread_flag(TIF_RESTORE_SIGMASK);
}
return ret;
}
static int do_signal_pending(sigset_t *oldset, struct pt_regs *regs)
{
siginfo_t info;
int signr;
struct k_sigaction ka;
int ret;
int is32 = is_32bit_task();
#ifdef CONFIG_PREEMPT_RT
/*
* Fully-preemptible kernel does not need interrupts disabled:
*/
local_irq_enable();
preempt_check_resched();
#endif
if (current_thread_info()->local_flags & _TLF_RESTORE_SIGMASK)
oldset = ¤t->saved_sigmask;
else if (!oldset)
oldset = ¤t->blocked;
signr = get_signal_to_deliver(&info, &ka, regs, NULL);
/* Is there any syscall restart business here ? */
check_syscall_restart(regs, &ka, signr > 0);
if (signr <= 0) {
struct thread_info *ti = current_thread_info();
/* No signal to deliver -- put the saved sigmask back */
if (ti->local_flags & _TLF_RESTORE_SIGMASK) {
ti->local_flags &= ~_TLF_RESTORE_SIGMASK;
sigprocmask(SIG_SETMASK, ¤t->saved_sigmask, NULL);
}
return 0; /* no signals delivered */
}
/*
* Reenable the DABR before delivering the signal to
* user space. The DABR will have been cleared if it
* triggered inside the kernel.
*/
if (current->thread.dabr) {
set_dabr(current->thread.dabr);
#if defined(CONFIG_BOOKE)
mtspr(SPRN_DBCR0, current->thread.dbcr0);
#endif
}
if (is32) {
if (ka.sa.sa_flags & SA_SIGINFO)
ret = handle_rt_signal32(signr, &ka, &info, oldset,
regs);
else
ret = handle_signal32(signr, &ka, &info, oldset,
regs);
} else {
ret = handle_rt_signal64(signr, &ka, &info, oldset, regs);
}
if (ret) {
spin_lock_irq(¤t->sighand->siglock);
sigorsets(¤t->blocked, ¤t->blocked,
&ka.sa.sa_mask);
if (!(ka.sa.sa_flags & SA_NODEFER))
sigaddset(¤t->blocked, signr);
recalc_sigpending();
spin_unlock_irq(¤t->sighand->siglock);
/*
* A signal was successfully delivered; the saved sigmask is in
* its frame, and we can clear the TLF_RESTORE_SIGMASK flag.
*/
current_thread_info()->local_flags &= ~_TLF_RESTORE_SIGMASK;
/*
* Let tracing know that we've done the handler setup.
*/
tracehook_signal_handler(signr, &info, &ka, regs,
test_thread_flag(TIF_SINGLESTEP));
}
return ret;
}
/*
* Handle debug exception notifications.
*/
int __kprobes hw_breakpoint_handler(struct die_args *args)
{
int rc = NOTIFY_STOP;
struct perf_event *bp;
struct pt_regs *regs = args->regs;
int stepped = 1;
struct arch_hw_breakpoint *info;
unsigned int instr;
unsigned long dar = regs->dar;
/* Disable breakpoints during exception handling */
set_dabr(0, 0);
/*
* The counter may be concurrently released but that can only
* occur from a call_rcu() path. We can then safely fetch
* the breakpoint, use its callback, touch its counter
* while we are in an rcu_read_lock() path.
*/
rcu_read_lock();
bp = __get_cpu_var(bp_per_reg);
if (!bp)
goto out;
info = counter_arch_bp(bp);
/*
* Return early after invoking user-callback function without restoring
* DABR if the breakpoint is from ptrace which always operates in
* one-shot mode. The ptrace-ed process will receive the SIGTRAP signal
* generated in do_dabr().
*/
if (bp->overflow_handler == ptrace_triggered) {
perf_bp_event(bp, regs);
rc = NOTIFY_DONE;
goto out;
}
/*
* Verify if dar lies within the address range occupied by the symbol
* being watched to filter extraneous exceptions. If it doesn't,
* we still need to single-step the instruction, but we don't
* generate an event.
*/
info->extraneous_interrupt = !((bp->attr.bp_addr <= dar) &&
(dar - bp->attr.bp_addr < bp->attr.bp_len));
/* Do not emulate user-space instructions, instead single-step them */
if (user_mode(regs)) {
current->thread.last_hit_ubp = bp;
regs->msr |= MSR_SE;
goto out;
}
stepped = 0;
instr = 0;
if (!__get_user_inatomic(instr, (unsigned int *) regs->nip))
stepped = emulate_step(regs, instr);
/*
* emulate_step() could not execute it. We've failed in reliably
* handling the hw-breakpoint. Unregister it and throw a warning
* message to let the user know about it.
*/
if (!stepped) {
WARN(1, "Unable to handle hardware breakpoint. Breakpoint at "
"0x%lx will be disabled.", info->address);
perf_event_disable(bp);
goto out;
}
/*
* As a policy, the callback is invoked in a 'trigger-after-execute'
* fashion
*/
if (!info->extraneous_interrupt)
perf_bp_event(bp, regs);
set_dabr(info->address | info->type | DABR_TRANSLATION, info->dabrx);
out:
rcu_read_unlock();
return rc;
}
static int do_signal(struct pt_regs *regs)
{
sigset_t *oldset;
siginfo_t info;
int signr;
struct k_sigaction ka;
int ret;
int is32 = is_32bit_task();
if (current_thread_info()->local_flags & _TLF_RESTORE_SIGMASK)
oldset = ¤t->saved_sigmask;
else
oldset = ¤t->blocked;
signr = get_signal_to_deliver(&info, &ka, regs, NULL);
/* Is there any syscall restart business here ? */
check_syscall_restart(regs, &ka, signr > 0);
if (signr <= 0) {
struct thread_info *ti = current_thread_info();
/* No signal to deliver -- put the saved sigmask back */
if (ti->local_flags & _TLF_RESTORE_SIGMASK) {
ti->local_flags &= ~_TLF_RESTORE_SIGMASK;
sigprocmask(SIG_SETMASK, ¤t->saved_sigmask, NULL);
}
regs->trap = 0;
return 0; /* no signals delivered */
}
#ifndef CONFIG_PPC_ADV_DEBUG_REGS
/*
* Reenable the DABR before delivering the signal to
* user space. The DABR will have been cleared if it
* triggered inside the kernel.
*/
if (current->thread.dabr)
set_dabr(current->thread.dabr);
#endif
/* Re-enable the breakpoints for the signal stack */
thread_change_pc(current, regs);
if (is32) {
if (ka.sa.sa_flags & SA_SIGINFO)
ret = handle_rt_signal32(signr, &ka, &info, oldset,
regs);
else
ret = handle_signal32(signr, &ka, &info, oldset,
regs);
} else {
ret = handle_rt_signal64(signr, &ka, &info, oldset, regs);
}
regs->trap = 0;
if (ret) {
block_sigmask(&ka, signr);
/*
* A signal was successfully delivered; the saved sigmask is in
* its frame, and we can clear the TLF_RESTORE_SIGMASK flag.
*/
current_thread_info()->local_flags &= ~_TLF_RESTORE_SIGMASK;
/*
* Let tracing know that we've done the handler setup.
*/
tracehook_signal_handler(signr, &info, &ka, regs,
test_thread_flag(TIF_SINGLESTEP));
}
return ret;
}
/*
* Note that 'init' is a special process: it doesn't get signals it doesn't
* want to handle. Thus you cannot kill init even with a SIGKILL even by
* mistake.
*/
int do_signal(sigset_t *oldset, struct pt_regs *regs)
{
siginfo_t info;
int signr;
struct k_sigaction ka;
/*
* If the current thread is 32 bit - invoke the
* 32 bit signal handling code
*/
if (test_thread_flag(TIF_32BIT))
return do_signal32(oldset, regs);
if (test_thread_flag(TIF_RESTORE_SIGMASK))
oldset = ¤t->saved_sigmask;
else if (!oldset)
oldset = ¤t->blocked;
signr = get_signal_to_deliver(&info, &ka, regs, NULL);
if (signr > 0) {
int ret;
/* Whee! Actually deliver the signal. */
if (TRAP(regs) == 0x0C00)
syscall_restart(regs, &ka);
/*
* Reenable the DABR before delivering the signal to
* user space. The DABR will have been cleared if it
* triggered inside the kernel.
*/
if (current->thread.dabr)
set_dabr(current->thread.dabr);
ret = handle_signal(signr, &ka, &info, oldset, regs);
/* If a signal was successfully delivered, the saved sigmask is in
its frame, and we can clear the TIF_RESTORE_SIGMASK flag */
if (ret && test_thread_flag(TIF_RESTORE_SIGMASK))
clear_thread_flag(TIF_RESTORE_SIGMASK);
return ret;
}
if (TRAP(regs) == 0x0C00) { /* System Call! */
if ((int)regs->result == -ERESTARTNOHAND ||
(int)regs->result == -ERESTARTSYS ||
(int)regs->result == -ERESTARTNOINTR) {
regs->gpr[3] = regs->orig_gpr3;
regs->nip -= 4; /* Back up & retry system call */
regs->result = 0;
} else if ((int)regs->result == -ERESTART_RESTARTBLOCK) {
regs->gpr[0] = __NR_restart_syscall;
regs->nip -= 4;
regs->result = 0;
}
}
/* No signal to deliver -- put the saved sigmask back */
if (test_thread_flag(TIF_RESTORE_SIGMASK)) {
clear_thread_flag(TIF_RESTORE_SIGMASK);
sigprocmask(SIG_SETMASK, ¤t->saved_sigmask, NULL);
}
return 0;
}