static int __kprobes kprobe_handler(struct pt_regs *regs) { struct kprobe *p; void *addr = (void *)regs->pc; int ret = 0; pr_debug("kprobe_handler: kprobe_running=%p\n", kprobe_running()); /* * We don't want to be preempted for the entire * duration of kprobe processing */ preempt_disable(); /* Check that we're not recursing */ if (kprobe_running()) { p = get_kprobe(addr); if (p) { if (kprobe_status == KPROBE_HIT_SS) { printk("FIXME: kprobe hit while single-stepping!\n"); goto no_kprobe; } printk("FIXME: kprobe hit while handling another kprobe\n"); goto no_kprobe; } else { p = kprobe_running(); if (p->break_handler && p->break_handler(p, regs)) goto ss_probe; } /* If it's not ours, can't be delete race, (we hold lock). */ goto no_kprobe; } p = get_kprobe(addr); if (!p) goto no_kprobe; kprobe_status = KPROBE_HIT_ACTIVE; set_current_kprobe(p); if (p->pre_handler && p->pre_handler(p, regs)) /* handler has already set things up, so skip ss setup */ return 1; ss_probe: prepare_singlestep(p, regs); kprobe_status = KPROBE_HIT_SS; return 1; no_kprobe: preempt_enable_no_resched(); return ret; }
static int __kprobes kprobe_handler(struct pt_regs *regs) { struct kprobe *p; void *addr = (void *)regs->pc; int ret = 0; pr_debug("kprobe_handler: kprobe_running=%p\n", kprobe_running()); preempt_disable(); if (kprobe_running()) { p = get_kprobe(addr); if (p) { if (kprobe_status == KPROBE_HIT_SS) { printk("FIXME: kprobe hit while single-stepping!\n"); goto no_kprobe; } printk("FIXME: kprobe hit while handling another kprobe\n"); goto no_kprobe; } else { p = kprobe_running(); if (p->break_handler && p->break_handler(p, regs)) goto ss_probe; } goto no_kprobe; } p = get_kprobe(addr); if (!p) goto no_kprobe; kprobe_status = KPROBE_HIT_ACTIVE; set_current_kprobe(p); if (p->pre_handler && p->pre_handler(p, regs)) return 1; ss_probe: prepare_singlestep(p, regs); kprobe_status = KPROBE_HIT_SS; return 1; no_kprobe: preempt_enable_no_resched(); return ret; }
/* * Wrapper routine to for handling exceptions. */ int kprobe_exceptions_notify(struct notifier_block *self, unsigned long val, void *data) { struct die_args *args = (struct die_args *)data; int ret = NOTIFY_DONE; /* * Interrupts are not disabled here. We need to disable * preemption, because kprobe_running() uses smp_processor_id(). */ preempt_disable(); switch (val) { case DIE_IABR_MATCH: case DIE_DABR_MATCH: case DIE_BPT: if (kprobe_handler(args->regs)) ret = NOTIFY_STOP; break; case DIE_SSTEP: if (post_kprobe_handler(args->regs)) ret = NOTIFY_STOP; break; case DIE_GPF: case DIE_PAGE_FAULT: if (kprobe_running() && kprobe_fault_handler(args->regs, args->trapnr)) ret = NOTIFY_STOP; break; default: break; } preempt_enable(); return ret; }
/* Ftrace callback handler for kprobes */ void __kprobes kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip, struct ftrace_ops *ops, struct pt_regs *regs) { struct kprobe *p; struct kprobe_ctlblk *kcb; unsigned long flags; /* Disable irq for emulating a breakpoint and avoiding preempt */ local_irq_save(flags); p = get_kprobe((kprobe_opcode_t *)ip); if (unlikely(!p) || kprobe_disabled(p)) goto end; kcb = get_kprobe_ctlblk(); if (kprobe_running()) { kprobes_inc_nmissed_count(p); } else { /* Kprobe handler expects regs->ip = ip + 1 as breakpoint hit */ regs->ip = ip + sizeof(kprobe_opcode_t); __this_cpu_write(current_kprobe, p); kcb->kprobe_status = KPROBE_HIT_ACTIVE; if (!p->pre_handler || !p->pre_handler(p, regs)) __skip_singlestep(p, regs, kcb); /* * If pre_handler returns !0, it sets regs->ip and * resets current kprobe. */ } end: local_irq_restore(flags); }
static inline void save_previous_kprobe(struct kprobe_ctlblk *kcb) { kcb->prev_kprobe.kp = kprobe_running(); kcb->prev_kprobe.status = kcb->kprobe_status; kcb->prev_kprobe.old_eflags = kcb->kprobe_old_eflags; kcb->prev_kprobe.saved_eflags = kcb->kprobe_saved_eflags; }
static void __kprobes save_previous_kprobe(struct kprobe_ctlblk *kcb) { kcb->prev_kprobe.kp = kprobe_running(); kcb->prev_kprobe.status = kcb->kprobe_status; kcb->prev_kprobe.orig_tnpc = kcb->kprobe_orig_tnpc; kcb->prev_kprobe.orig_tstate_pil = kcb->kprobe_orig_tstate_pil; }
/* * Wrapper routine to for handling exceptions. */ int __kprobes kprobe_exceptions_notify(struct notifier_block *self, unsigned long val, void *data) { struct die_args *args = (struct die_args *)data; int ret = NOTIFY_DONE; pr_debug("kprobe_exceptions_notify: val=%lu, data=%p\n", val, data); switch (val) { case DIE_BREAKPOINT: if (kprobe_handler(args->regs)) ret = NOTIFY_STOP; break; case DIE_SSTEP: if (post_kprobe_handler(args->regs)) ret = NOTIFY_STOP; break; case DIE_FAULT: if (kprobe_running() && kprobe_fault_handler(args->regs, args->trapnr)) ret = NOTIFY_STOP; break; default: break; } return ret; }
static inline int notify_page_fault(struct pt_regs *regs, int trap) { int ret = 0; if (!user_mode(regs)) { if (kprobe_running() && kprobe_fault_handler(regs, trap)) ret = 1; } return ret; }
static inline int notify_page_fault(struct pt_regs *regs) { int ret = 0; /* kprobe_running() needs smp_processor_id() */ if (kprobes_built_in() && !user_mode(regs)) { preempt_disable(); if (kprobe_running() && kprobe_fault_handler(regs, 14)) ret = 1; preempt_enable(); } return ret; }
int __kprobes kprobe_fault_handler(struct pt_regs *regs, unsigned int fsr) { struct kprobe *cur = kprobe_running(); struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); switch (kcb->kprobe_status) { case KPROBE_HIT_SS: case KPROBE_REENTER: /* * We are here because the instruction being single * stepped caused a page fault. We reset the current * kprobe and the PC to point back to the probe address * and allow the page fault handler to continue as a * normal page fault. */ regs->ARM_pc = (long)cur->addr; if (kcb->kprobe_status == KPROBE_REENTER) { restore_previous_kprobe(kcb); } else { reset_current_kprobe(); } break; case KPROBE_HIT_ACTIVE: case KPROBE_HIT_SSDONE: /* * We increment the nmissed count for accounting, * we can also use npre/npostfault count for accounting * these specific fault cases. */ kprobes_inc_nmissed_count(cur); /* * We come here because instructions in the pre/post * handler caused the page_fault, this could happen * if handler tries to access user space by * copy_from_user(), get_user() etc. Let the * user-specified handler try to fix it. */ if (cur->fault_handler && cur->fault_handler(cur, regs, fsr)) return 1; break; default: break; } return 0; }
static int __kprobes kprobe_fault_handler(struct pt_regs *regs, int trapnr) { struct kprobe *cur = kprobe_running(); pr_debug("kprobe_fault_handler: trapnr=%d\n", trapnr); if (cur->fault_handler && cur->fault_handler(cur, regs, trapnr)) return 1; if (kprobe_status & KPROBE_HIT_SS) { resume_execution(cur, regs); preempt_enable_no_resched(); } return 0; }
static int __kprobes post_kprobe_handler(struct pt_regs *regs) { struct kprobe *cur = kprobe_running(); pr_debug("post_kprobe_handler, cur=%p\n", cur); if (!cur) return 0; if (cur->post_handler) { kprobe_status = KPROBE_HIT_SSDONE; cur->post_handler(cur, regs, 0); } resume_execution(cur, regs); reset_current_kprobe(); preempt_enable_no_resched(); return 1; }
/* Ftrace callback handler for kprobes */ void kprobe_ftrace_handler(unsigned long nip, unsigned long parent_nip, struct ftrace_ops *ops, struct pt_regs *regs) { struct kprobe *p; struct kprobe_ctlblk *kcb; preempt_disable(); p = get_kprobe((kprobe_opcode_t *)nip); if (unlikely(!p) || kprobe_disabled(p)) goto end; kcb = get_kprobe_ctlblk(); if (kprobe_running()) { kprobes_inc_nmissed_count(p); } else { unsigned long orig_nip = regs->nip; /* * On powerpc, NIP is *before* this instruction for the * pre handler */ regs->nip -= MCOUNT_INSN_SIZE; __this_cpu_write(current_kprobe, p); kcb->kprobe_status = KPROBE_HIT_ACTIVE; if (!p->pre_handler || !p->pre_handler(p, regs)) __skip_singlestep(p, regs, kcb, orig_nip); else { /* * If pre_handler returns !0, it sets regs->nip and * resets current kprobe. In this case, we should not * re-enable preemption. */ return; } } end: preempt_enable_no_resched(); }
static inline int post_kprobe_handler(struct pt_regs *regs) { if (!kprobe_running()) return 0; if (current_kprobe->post_handler) current_kprobe->post_handler(current_kprobe, regs, 0); resume_execution(current_kprobe, regs); regs->msr |= kprobe_saved_msr; unlock_kprobes(); preempt_enable_no_resched(); /* * if somebody else is singlestepping across a probe point, msr * will have SE set, in which case, continue the remaining processing * of do_debug, as if this is not a probe hit. */ if (regs->msr & MSR_SE) return 0; return 1; }
static int __kprobes kprobe_handler(struct pt_regs *regs) { struct kprobe *p; void *addr = (void *) regs->tpc; int ret = 0; struct kprobe_ctlblk *kcb; /* * We don't want to be preempted for the entire * duration of kprobe processing */ preempt_disable(); kcb = get_kprobe_ctlblk(); if (kprobe_running()) { p = get_kprobe(addr); if (p) { if (kcb->kprobe_status == KPROBE_HIT_SS) { regs->tstate = ((regs->tstate & ~TSTATE_PIL) | kcb->kprobe_orig_tstate_pil); goto no_kprobe; } /* We have reentered the kprobe_handler(), since * another probe was hit while within the handler. * We here save the original kprobes variables and * just single step on the instruction of the new probe * without calling any user handlers. */ save_previous_kprobe(kcb); set_current_kprobe(p, regs, kcb); kprobes_inc_nmissed_count(p); kcb->kprobe_status = KPROBE_REENTER; prepare_singlestep(p, regs, kcb); return 1; } else { if (*(u32 *)addr != BREAKPOINT_INSTRUCTION) { /* The breakpoint instruction was removed by * another cpu right after we hit, no further * handling of this interrupt is appropriate */ ret = 1; goto no_kprobe; } p = __get_cpu_var(current_kprobe); if (p->break_handler && p->break_handler(p, regs)) goto ss_probe; } goto no_kprobe; } p = get_kprobe(addr); if (!p) { if (*(u32 *)addr != BREAKPOINT_INSTRUCTION) { /* * The breakpoint instruction was removed right * after we hit it. Another cpu has removed * either a probepoint or a debugger breakpoint * at this address. In either case, no further * handling of this interrupt is appropriate. */ ret = 1; } /* Not one of ours: let kernel handle it */ goto no_kprobe; } set_current_kprobe(p, regs, kcb); kcb->kprobe_status = KPROBE_HIT_ACTIVE; if (p->pre_handler && p->pre_handler(p, regs)) return 1; ss_probe: prepare_singlestep(p, regs, kcb); kcb->kprobe_status = KPROBE_HIT_SS; return 1; no_kprobe: preempt_enable_no_resched(); return ret; }
/* * Called with IRQs disabled. IRQs must remain disabled from that point * all the way until processing this kprobe is complete. The current * kprobes implementation cannot process more than one nested level of * kprobe, and that level is reserved for user kprobe handlers, so we can't * risk encountering a new kprobe in an interrupt handler. */ void __kprobes kprobe_handler(struct pt_regs *regs) { struct kprobe *p, *cur; struct kprobe_ctlblk *kcb; kcb = get_kprobe_ctlblk(); cur = kprobe_running(); #ifdef CONFIG_THUMB2_KERNEL /* * First look for a probe which was registered using an address with * bit 0 set, this is the usual situation for pointers to Thumb code. * If not found, fallback to looking for one with bit 0 clear. */ p = get_kprobe((kprobe_opcode_t *)(regs->ARM_pc | 1)); if (!p) p = get_kprobe((kprobe_opcode_t *)regs->ARM_pc); #else /* ! CONFIG_THUMB2_KERNEL */ p = get_kprobe((kprobe_opcode_t *)regs->ARM_pc); #endif if (p) { if (cur) { /* Kprobe is pending, so we're recursing. */ switch (kcb->kprobe_status) { case KPROBE_HIT_ACTIVE: case KPROBE_HIT_SSDONE: /* A pre- or post-handler probe got us here. */ kprobes_inc_nmissed_count(p); save_previous_kprobe(kcb); set_current_kprobe(p); kcb->kprobe_status = KPROBE_REENTER; singlestep(p, regs, kcb); restore_previous_kprobe(kcb); break; default: /* impossible cases */ BUG(); } } else if (p->ainsn.insn_check_cc(regs->ARM_cpsr)) { /* Probe hit and conditional execution check ok. */ set_current_kprobe(p); kcb->kprobe_status = KPROBE_HIT_ACTIVE; /* * If we have no pre-handler or it returned 0, we * continue with normal processing. If we have a * pre-handler and it returned non-zero, it prepped * for calling the break_handler below on re-entry, * so get out doing nothing more here. */ if (!p->pre_handler || !p->pre_handler(p, regs)) { kcb->kprobe_status = KPROBE_HIT_SS; singlestep(p, regs, kcb); if (p->post_handler) { kcb->kprobe_status = KPROBE_HIT_SSDONE; p->post_handler(p, regs, 0); } reset_current_kprobe(); } } else { /* * Probe hit but conditional execution check failed, * so just skip the instruction and continue as if * nothing had happened. */ singlestep_skip(p, regs); } } else if (cur) { /* We probably hit a jprobe. Call its break handler. */ if (cur->break_handler && cur->break_handler(cur, regs)) { kcb->kprobe_status = KPROBE_HIT_SS; singlestep(cur, regs, kcb); if (cur->post_handler) { kcb->kprobe_status = KPROBE_HIT_SSDONE; cur->post_handler(cur, regs, 0); } } reset_current_kprobe(); } else { /* * The probe was removed and a race is in progress. * There is nothing we can do about it. Let's restart * the instruction. By the time we can restart, the * real instruction will be there. */ } }
static void __kprobes save_previous_kprobe(struct kprobe_ctlblk *kcb) { kcb->prev_kprobe.kp = kprobe_running(); kcb->prev_kprobe.status = kcb->kprobe_status; }
static inline int kprobe_handler(struct pt_regs *regs) { struct kprobe *p; int ret = 0; unsigned int *addr = (unsigned int *)regs->nip; /* Check we're not actually recursing */ if (kprobe_running()) { /* We *are* holding lock here, so this is safe. Disarm the probe we just hit, and ignore it. */ p = get_kprobe(addr); if (p) { if (kprobe_status == KPROBE_HIT_SS) { regs->msr &= ~MSR_SE; regs->msr |= kprobe_saved_msr; unlock_kprobes(); goto no_kprobe; } disarm_kprobe(p, regs); ret = 1; } else { p = current_kprobe; if (p->break_handler && p->break_handler(p, regs)) { goto ss_probe; } } /* If it's not ours, can't be delete race, (we hold lock). */ goto no_kprobe; } lock_kprobes(); p = get_kprobe(addr); if (!p) { unlock_kprobes(); if (*addr != BREAKPOINT_INSTRUCTION) { /* * PowerPC has multiple variants of the "trap" * instruction. If the current instruction is a * trap variant, it could belong to someone else */ kprobe_opcode_t cur_insn = *addr; if (IS_TW(cur_insn) || IS_TD(cur_insn) || IS_TWI(cur_insn) || IS_TDI(cur_insn)) goto no_kprobe; /* * The breakpoint instruction was removed right * after we hit it. Another cpu has removed * either a probepoint or a debugger breakpoint * at this address. In either case, no further * handling of this interrupt is appropriate. */ ret = 1; } /* Not one of ours: let kernel handle it */ goto no_kprobe; } kprobe_status = KPROBE_HIT_ACTIVE; current_kprobe = p; kprobe_saved_msr = regs->msr; if (p->pre_handler && p->pre_handler(p, regs)) /* handler has already set things up, so skip ss setup */ return 1; ss_probe: prepare_singlestep(p, regs); kprobe_status = KPROBE_HIT_SS; /* * This preempt_disable() matches the preempt_enable_no_resched() * in post_kprobe_handler(). */ preempt_disable(); return 1; no_kprobe: return ret; }
/* * Called with IRQs disabled. IRQs must remain disabled from that point * all the way until processing this kprobe is complete. The current * kprobes implementation cannot process more than one nested level of * kprobe, and that level is reserved for user kprobe handlers, so we can't * risk encountering a new kprobe in an interrupt handler. */ void __kprobes kprobe_handler(struct pt_regs *regs) { struct kprobe *p, *cur; struct kprobe_ctlblk *kcb; kprobe_opcode_t *addr = (kprobe_opcode_t *)regs->ARM_pc; kcb = get_kprobe_ctlblk(); cur = kprobe_running(); p = get_kprobe(addr); if (p) { if (cur) { /* Kprobe is pending, so we're recursing. */ switch (kcb->kprobe_status) { case KPROBE_HIT_ACTIVE: case KPROBE_HIT_SSDONE: /* A pre- or post-handler probe got us here. */ kprobes_inc_nmissed_count(p); save_previous_kprobe(kcb); set_current_kprobe(p); kcb->kprobe_status = KPROBE_REENTER; singlestep(p, regs, kcb); restore_previous_kprobe(kcb); break; default: /* impossible cases */ BUG(); } } else { set_current_kprobe(p); kcb->kprobe_status = KPROBE_HIT_ACTIVE; /* * If we have no pre-handler or it returned 0, we * continue with normal processing. If we have a * pre-handler and it returned non-zero, it prepped * for calling the break_handler below on re-entry, * so get out doing nothing more here. */ if (!p->pre_handler || !p->pre_handler(p, regs)) { kcb->kprobe_status = KPROBE_HIT_SS; singlestep(p, regs, kcb); if (p->post_handler) { kcb->kprobe_status = KPROBE_HIT_SSDONE; p->post_handler(p, regs, 0); } reset_current_kprobe(); } } } else if (cur) { /* We probably hit a jprobe. Call its break handler. */ if (cur->break_handler && cur->break_handler(cur, regs)) { kcb->kprobe_status = KPROBE_HIT_SS; singlestep(cur, regs, kcb); if (cur->post_handler) { kcb->kprobe_status = KPROBE_HIT_SSDONE; cur->post_handler(cur, regs, 0); } } reset_current_kprobe(); } else { /* * The probe was removed and a race is in progress. * There is nothing we can do about it. Let's restart * the instruction. By the time we can restart, the * real instruction will be there. */ } }
static int __kprobes kprobe_handler(struct pt_regs *regs) { struct kprobe *p; void *addr = (void *) regs->tpc; int ret = 0; preempt_disable(); if (kprobe_running()) { /* We *are* holding lock here, so this is safe. * Disarm the probe we just hit, and ignore it. */ p = get_kprobe(addr); if (p) { if (kprobe_status == KPROBE_HIT_SS) { regs->tstate = ((regs->tstate & ~TSTATE_PIL) | current_kprobe_orig_tstate_pil); unlock_kprobes(); goto no_kprobe; } /* We have reentered the kprobe_handler(), since * another probe was hit while within the handler. * We here save the original kprobes variables and * just single step on the instruction of the new probe * without calling any user handlers. */ save_previous_kprobe(); set_current_kprobe(p, regs); p->nmissed++; kprobe_status = KPROBE_REENTER; prepare_singlestep(p, regs); return 1; } else { p = current_kprobe; if (p->break_handler && p->break_handler(p, regs)) goto ss_probe; } /* If it's not ours, can't be delete race, (we hold lock). */ goto no_kprobe; } lock_kprobes(); p = get_kprobe(addr); if (!p) { unlock_kprobes(); if (*(u32 *)addr != BREAKPOINT_INSTRUCTION) { /* * The breakpoint instruction was removed right * after we hit it. Another cpu has removed * either a probepoint or a debugger breakpoint * at this address. In either case, no further * handling of this interrupt is appropriate. */ ret = 1; } /* Not one of ours: let kernel handle it */ goto no_kprobe; } set_current_kprobe(p, regs); kprobe_status = KPROBE_HIT_ACTIVE; if (p->pre_handler && p->pre_handler(p, regs)) return 1; ss_probe: prepare_singlestep(p, regs); kprobe_status = KPROBE_HIT_SS; return 1; no_kprobe: preempt_enable_no_resched(); return ret; }