void do_ip27_dbe(struct pt_regs *regs) { unsigned long fixup, errst0, errst1; int cpu = LOCAL_HUB_L(PI_CPU_NUM); fixup = search_dbe_table(regs->cp0_epc); if (fixup) { long new_epc; new_epc = fixup_exception(dpf_reg, fixup, regs->cp0_epc); regs->cp0_epc = new_epc; return; } printk("Slice %c got dbe at 0x%lx\n", 'A' + cpu, regs->cp0_epc); printk("Hub information:\n"); printk("ERR_INT_PEND = 0x%06lx\n", LOCAL_HUB_L(PI_ERR_INT_PEND)); errst0 = LOCAL_HUB_L(cpu ? PI_ERR_STATUS0_B : PI_ERR_STATUS0_A); errst1 = LOCAL_HUB_L(cpu ? PI_ERR_STATUS1_B : PI_ERR_STATUS1_A); dump_hub_information(errst0, errst1); show_regs(regs); dump_tlb_all(); while(1); force_sig(SIGBUS, current); }
static void __do_kernel_fault(struct mm_struct *mm, unsigned long addr, unsigned int esr, struct pt_regs *regs) { #if defined(CONFIG_HTC_DEBUG_RTB) static int enable_logk_die = 1; #endif if (fixup_exception(regs)) return; #if defined(CONFIG_HTC_DEBUG_RTB) if (enable_logk_die) { uncached_logk(LOGK_DIE, (void *)regs->pc); uncached_logk(LOGK_DIE, (void *)regs->regs[30]); uncached_logk(LOGK_DIE, (void *)addr); msm_rtb_disable(); enable_logk_die = 0; } #endif bust_spinlocks(1); pr_alert("Unable to handle kernel %s at virtual address %08lx\n", (addr < PAGE_SIZE) ? "NULL pointer dereference" : "paging request", addr); show_pte(mm, addr); die("Oops", regs, esr); bust_spinlocks(0); do_exit(SIGKILL); }
void do_page_fault(struct pt_regs *regs, unsigned int vector) { /* Kernel space exception fixup check */ if (fixup_exception(regs)) return; unsigned long cr2 = read_cr2(); #ifdef CONFIG_KGDB kgdb_breakpoint(); #endif struct siginfo s; memset(&s, 0, sizeof(struct siginfo)); s.si_signo = SIGSEGV; s.si_errno = 0; s.si_addr = (void *)cr2; s.si_code = SEGV_ACCERR; // SEGV_MAPERR; int i = sigsend(current->aspace->id, ANY_ID, SIGSEGV, &s); printk(">> PAGE FAULT! Sent signal %d. CR2 is %p, cpu %d\n", i, (void *)cr2, this_cpu); show_registers(regs); /* If the fault occurred in kernel space, or the init_task, go down */ if ( (regs->rip >= PAGE_OFFSET) || (current->aspace->id == INIT_ASPACE_ID)) { local_irq_disable(); halt(); } }
/* * The kernel tried to access some page that wasn't present. */ static void __do_kernel_fault(struct mm_struct *mm, unsigned long addr, unsigned int esr, struct pt_regs *regs) { #if defined(CONFIG_HTC_DEBUG_RTB) static int enable_logk_die = 1; #endif /* * Are we prepared to handle this kernel fault? */ if (fixup_exception(regs)) return; #if defined(CONFIG_HTC_DEBUG_RTB) if (enable_logk_die) { uncached_logk(LOGK_DIE, (void *)regs->pc); uncached_logk(LOGK_DIE, (void *)regs->regs[30]); uncached_logk(LOGK_DIE, (void *)addr); /* Disable RTB here to avoid weird recursive spinlock/printk behaviors */ msm_rtb_disable(); enable_logk_die = 0; } #endif /* * No handler, we'll have to terminate things with extreme prejudice. */ bust_spinlocks(1); pr_alert("Unable to handle kernel %s at virtual address %08lx\n", (addr < PAGE_SIZE) ? "NULL pointer dereference" : "paging request", addr); show_pte(mm, addr); die("Oops", regs, esr); bust_spinlocks(0); do_exit(SIGKILL); }
int send_fault_sig(struct pt_regs *regs) { int signo, si_code; void __user *addr; signo = current->thread.signo; si_code = current->thread.code; addr = (void __user *)current->thread.faddr; pr_debug("send_fault_sig: %p,%d,%d\n", addr, signo, si_code); if (user_mode(regs)) { force_sig_fault(signo, si_code, addr, current); } else { if (fixup_exception(regs)) return -1; //if (signo == SIGBUS) // force_sig_fault(si_signo, si_code, addr, current); /* * Oops. The kernel tried to access some bad page. We'll have to * terminate things with extreme prejudice. */ if ((unsigned long)addr < PAGE_SIZE) pr_alert("Unable to handle kernel NULL pointer dereference"); else pr_alert("Unable to handle kernel access"); pr_cont(" at virtual address %p\n", addr); die_if_kernel("Oops", regs, 0 /*error_code*/); do_exit(SIGKILL); } return 1; }
void do_page_fault(struct pt_regs *regs, unsigned int vector) { /* Kernel space exception fixup check */ if (fixup_exception(regs)) return; unsigned long cr2 = read_cr2(); #ifdef CONFIG_KGDB kgdb_breakpoint(); #endif struct siginfo s; memset(&s, 0, sizeof(struct siginfo)); s.si_signo = SIGSEGV; s.si_errno = 0; s.si_addr = (void *)cr2; s.si_code = SEGV_ACCERR; // SEGV_MAPERR; int i = sigsend(current->aspace->id, ANY_ID, SIGSEGV, &s); printk(">> PAGE FAULT! Sent signal %d: CR2 is %p\n", i, s.si_addr); show_registers(regs); while (1) {} }
dotraplinkage void __kprobes do_general_protection(struct pt_regs *regs, long error_code) { struct task_struct *tsk; conditional_sti(regs); #ifdef CONFIG_X86_32 if (lazy_iobitmap_copy()) { /* restart the faulting instruction */ return; } if (regs->flags & X86_VM_MASK) goto gp_in_vm86; #endif tsk = current; if (!user_mode(regs)) goto gp_in_kernel; tsk->thread.error_code = error_code; tsk->thread.trap_no = 13; if (show_unhandled_signals && unhandled_signal(tsk, SIGSEGV) && printk_ratelimit()) { printk(KERN_INFO "%s[%d] general protection ip:%lx sp:%lx error:%lx", tsk->comm, task_pid_nr(tsk), regs->ip, regs->sp, error_code); print_vma_addr(" in ", regs->ip); printk("\n"); } force_sig(SIGSEGV, tsk); return; #ifdef CONFIG_X86_32 gp_in_vm86: local_irq_enable(); handle_vm86_fault((struct kernel_vm86_regs *) regs, error_code); return; #endif gp_in_kernel: if (fixup_exception(regs)) return; tsk->thread.error_code = error_code; tsk->thread.trap_no = 13; if (notify_die(DIE_GPF, "general protection fault", regs, error_code, 13, SIGSEGV) == NOTIFY_STOP) return; die("general protection fault", regs, error_code); }
static int kernel_math_error(struct pt_regs *regs, const char *str, int trapnr) { if (fixup_exception(regs)) return 1; notify_die(DIE_GPF, str, regs, 0, trapnr, SIGFPE); /* Illegal floating point operation in the kernel */ current->thread.trap_no = trapnr; die(str, regs, 0); return 0; }
void do_dbe(struct pt_regs *regs) { unsigned long fixup; fixup = search_dbe_table(regs->cp0_epc); if (fixup) { long new_epc; new_epc = fixup_exception(dpf_reg, fixup, regs->cp0_epc); regs->cp0_epc = new_epc; return; } printk("Got dbe at 0x%lx\n", regs->cp0_epc); show_regs(regs); dump_tlb_all(); while(1); force_sig(SIGBUS, current); }
static __inline__ void do_trap(int trapnr, int signr, const char * str, struct pt_regs * regs, long error_code, siginfo_t *info) { if (user_mode(regs)) { /* trap_signal */ struct task_struct *tsk = current; tsk->thread.error_code = error_code; tsk->thread.trap_no = trapnr; if (info) force_sig_info(signr, info, tsk); else force_sig(signr, tsk); return; } else { /* kernel_trap */ if (!fixup_exception(regs)) die(str, regs, error_code); return; } }
/* * Set up exception handlers for tracing and breakpoints */ void handle_exception(struct pt_regs *regs) { int trap = (regs->cp0_cause & 0x7c) >> 2; if (fixup_exception(regs)) { return; } if (atomic_read(&debugger_active)) kgdb_nmihook(smp_processor_id(), regs); if (atomic_read(&kgdb_setting_breakpoint)) if ((trap == 9) && (regs->cp0_epc == (unsigned long)breakinst)) regs->cp0_epc += 4; kgdb_handle_exception(0, compute_signal(trap), 0, regs); /* In SMP mode, __flush_cache_all does IPI */ __flush_cache_all(); }
/* * The kernel tried to access some page that wasn't present. */ static void __do_kernel_fault(struct mm_struct *mm, unsigned long addr, unsigned int esr, struct pt_regs *regs) { /* * Are we prepared to handle this kernel fault? */ if (fixup_exception(regs)) return; /* * No handler, we'll have to terminate things with extreme prejudice. */ bust_spinlocks(1); pr_alert("Unable to handle kernel %s at virtual address %08lx\n", (addr < PAGE_SIZE) ? "NULL pointer dereference" : "paging request", addr); show_pte(mm, addr); die("Oops", regs, esr); bust_spinlocks(0); do_exit(SIGKILL); }
/* Alignment exception handler */ asmlinkage void handle_unaligned_c(struct pt_regs *fp, int cause) { unsigned long addr = RDCTL(CTL_BADADDR); cause >>= 2; fp->ea -= 4; if (fixup_exception(fp)) return; if (!user_mode(fp)) { pr_alert("Unaligned access from kernel mode, this might be a hardware\n"); pr_alert("problem, dump registers and restart the instruction\n"); pr_alert(" BADADDR 0x%08lx\n", addr); pr_alert(" cause %d\n", cause); pr_alert(" op-code 0x%08lx\n", *(unsigned long *)(fp->ea)); show_regs(fp); return; } _exception(SIGBUS, fp, BUS_ADRALN, addr); }
static inline void emulate_load_store_insn(struct pt_regs *regs, unsigned long addr, unsigned long pc) { union mips_instruction insn; unsigned long value, fixup; regs->regs[0] = 0; /* * This load never faults. */ __get_user(insn.word, (unsigned int *)pc); switch (insn.i_format.opcode) { /* * These are instructions that a compiler doesn't generate. We * can assume therefore that the code is MIPS-aware and * really buggy. Emulating these instructions would break the * semantics anyway. */ case ll_op: case lld_op: case sc_op: case scd_op: /* * For these instructions the only way to create an address * error is an attempted access to kernel/supervisor address * space. */ case ldl_op: case ldr_op: case lwl_op: case lwr_op: case sdl_op: case sdr_op: case swl_op: case swr_op: case lb_op: case lbu_op: case sb_op: goto sigbus; /* * The remaining opcodes are the ones that are really of interest. */ case lh_op: check_axs(pc, addr, 2); __asm__( ".set\tnoat\n" #ifdef __BIG_ENDIAN "1:\tlb\t%0,0(%1)\n" "2:\tlbu\t$1,1(%1)\n\t" #endif #ifdef __LITTLE_ENDIAN "1:\tlb\t%0,1(%1)\n" "2:\tlbu\t$1,0(%1)\n\t" #endif "sll\t%0,0x8\n\t" "or\t%0,$1\n\t" ".set\tat\n\t" ".section\t__ex_table,\"a\"\n\t" STR(PTR)"\t1b,%2\n\t" STR(PTR)"\t2b,%2\n\t" ".previous" :"=&r" (value) :"r" (addr), "i" (&&fault) :"$1"); regs->regs[insn.i_format.rt] = value; return; case lw_op: check_axs(pc, addr, 4); __asm__( #ifdef __BIG_ENDIAN "1:\tlwl\t%0,(%1)\n" "2:\tlwr\t%0,3(%1)\n\t" #endif #ifdef __LITTLE_ENDIAN "1:\tlwl\t%0,3(%1)\n" "2:\tlwr\t%0,(%1)\n\t" #endif ".section\t__ex_table,\"a\"\n\t" STR(PTR)"\t1b,%2\n\t" STR(PTR)"\t2b,%2\n\t" ".previous" :"=&r" (value) :"r" (addr), "i" (&&fault)); regs->regs[insn.i_format.rt] = value; return; case lhu_op: check_axs(pc, addr, 2); __asm__( ".set\tnoat\n" #ifdef __BIG_ENDIAN "1:\tlbu\t%0,0(%1)\n" "2:\tlbu\t$1,1(%1)\n\t" #endif #ifdef __LITTLE_ENDIAN "1:\tlbu\t%0,1(%1)\n" "2:\tlbu\t$1,0(%1)\n\t" #endif "sll\t%0,0x8\n\t" "or\t%0,$1\n\t" ".set\tat\n\t" ".section\t__ex_table,\"a\"\n\t" STR(PTR)"\t1b,%2\n\t" STR(PTR)"\t2b,%2\n\t" ".previous" :"=&r" (value) :"r" (addr), "i" (&&fault) :"$1"); regs->regs[insn.i_format.rt] = value; return; case lwu_op: check_axs(pc, addr, 4); __asm__( #ifdef __BIG_ENDIAN "1:\tlwl\t%0,(%1)\n" "2:\tlwr\t%0,3(%1)\n\t" #endif #ifdef __LITTLE_ENDIAN "1:\tlwl\t%0,3(%1)\n" "2:\tlwr\t%0,(%1)\n\t" #endif ".section\t__ex_table,\"a\"\n\t" STR(PTR)"\t1b,%2\n\t" STR(PTR)"\t2b,%2\n\t" ".previous" :"=&r" (value) :"r" (addr), "i" (&&fault)); value &= 0xffffffff; regs->regs[insn.i_format.rt] = value; return; case ld_op: check_axs(pc, addr, 8); __asm__( ".set\tmips3\n" #ifdef __BIG_ENDIAN "1:\tldl\t%0,(%1)\n" "2:\tldr\t%0,7(%1)\n\t" #endif #ifdef __LITTLE_ENDIAN "1:\tldl\t%0,7(%1)\n" "2:\tldr\t%0,(%1)\n\t" #endif ".set\tmips0\n\t" ".section\t__ex_table,\"a\"\n\t" STR(PTR)"\t1b,%2\n\t" STR(PTR)"\t2b,%2\n\t" ".previous" :"=&r" (value) :"r" (addr), "i" (&&fault)); regs->regs[insn.i_format.rt] = value; return; case sh_op: check_axs(pc, addr, 2); value = regs->regs[insn.i_format.rt]; __asm__( #ifdef __BIG_ENDIAN ".set\tnoat\n" "1:\tsb\t%0,1(%1)\n\t" "srl\t$1,%0,0x8\n" "2:\tsb\t$1,0(%1)\n\t" ".set\tat\n\t" #endif #ifdef __LITTLE_ENDIAN ".set\tnoat\n" "1:\tsb\t%0,0(%1)\n\t" "srl\t$1,%0,0x8\n" "2:\tsb\t$1,1(%1)\n\t" ".set\tat\n\t" #endif ".section\t__ex_table,\"a\"\n\t" STR(PTR)"\t1b,%2\n\t" STR(PTR)"\t2b,%2\n\t" ".previous" : /* no outputs */ :"r" (value), "r" (addr), "i" (&&fault) :"$1"); return; case sw_op: check_axs(pc, addr, 4); value = regs->regs[insn.i_format.rt]; __asm__( #ifdef __BIG_ENDIAN "1:\tswl\t%0,(%1)\n" "2:\tswr\t%0,3(%1)\n\t" #endif #ifdef __LITTLE_ENDIAN "1:\tswl\t%0,3(%1)\n" "2:\tswr\t%0,(%1)\n\t" #endif ".section\t__ex_table,\"a\"\n\t" STR(PTR)"\t1b,%2\n\t" STR(PTR)"\t2b,%2\n\t" ".previous" : /* no outputs */ :"r" (value), "r" (addr), "i" (&&fault)); return; case sd_op: check_axs(pc, addr, 8); value = regs->regs[insn.i_format.rt]; __asm__( ".set\tmips3\n" #ifdef __BIG_ENDIAN "1:\tsdl\t%0,(%1)\n" "2:\tsdr\t%0,7(%1)\n\t" #endif #ifdef __LITTLE_ENDIAN "1:\tsdl\t%0,7(%1)\n" "2:\tsdr\t%0,(%1)\n\t" #endif ".set\tmips0\n\t" ".section\t__ex_table,\"a\"\n\t" STR(PTR)"\t1b,%2\n\t" STR(PTR)"\t2b,%2\n\t" ".previous" : /* no outputs */ :"r" (value), "r" (addr), "i" (&&fault)); return; case lwc1_op: case ldc1_op: case swc1_op: case sdc1_op: /* * I herewith declare: this does not happen. So send SIGBUS. */ goto sigbus; case lwc2_op: case ldc2_op: case swc2_op: case sdc2_op: /* * These are the coprocessor 2 load/stores. The current * implementations don't use cp2 and cp2 should always be * disabled in c0_status. So send SIGILL. * (No longer true: The Sony Praystation uses cp2 for * 3D matrix operations. Dunno if that thingy has a MMU ...) */ default: /* * Pheeee... We encountered an yet unknown instruction or * cache coherence problem. Die sucker, die ... */ goto sigill; } return; fault: /* Did we have an exception handler installed? */ fixup = search_exception_table(regs->cp0_epc); if (fixup) { long new_epc; new_epc = fixup_exception(dpf_reg, fixup, regs->cp0_epc); printk(KERN_DEBUG "%s: Forwarding exception at [<%lx>] (%lx)\n", current->comm, regs->cp0_epc, new_epc); regs->cp0_epc = new_epc; return; } lock_kernel(); send_sig(SIGSEGV, current, 1); unlock_kernel(); return; sigbus: lock_kernel(); send_sig(SIGBUS, current, 1); unlock_kernel(); return; sigill: lock_kernel(); send_sig(SIGILL, current, 1); unlock_kernel(); return; }
static void __kprobes do_trap(int trapnr, int signr, const char *str, struct pt_regs *regs, long error_code, siginfo_t *info) { struct task_struct *tsk = current; #ifdef CONFIG_X86_32 if (v8086_mode(regs)) { /* * traps 0, 1, 3, 4, and 5 should be forwarded to vm86. * On nmi (interrupt 2), do_trap should not be called. */ if (trapnr < 6) goto vm86_trap; goto trap_signal; } #endif if (!user_mode_novm(regs)) goto kernel_trap; #ifdef CONFIG_X86_32 trap_signal: #endif /* * We want error_code and trap_no set for userspace faults and * kernelspace faults which result in die(), but not * kernelspace faults which are fixed up. die() gives the * process no chance to handle the signal and notice the * kernel fault information, so that won't result in polluting * the information about previously queued, but not yet * delivered, faults. See also do_general_protection below. */ tsk->thread.error_code = error_code; tsk->thread.trap_no = trapnr; #ifdef CONFIG_X86_64 if (show_unhandled_signals && unhandled_signal(tsk, signr) && printk_ratelimit()) { printk(KERN_INFO "%s[%d] trap %s ip:%lx sp:%lx error:%lx", tsk->comm, task_pid_nr(tsk), str, regs->ip, regs->sp, error_code); print_vma_addr(" in ", regs->ip); printk("\n"); } #endif if (info) force_sig_info(signr, info, tsk); else force_sig(signr, tsk); return; kernel_trap: if (!fixup_exception(regs)) { tsk->thread.error_code = error_code; tsk->thread.trap_no = trapnr; #if defined(CONFIG_X86_32) && defined(CONFIG_PAX_KERNEXEC) if (trapnr == 12 && ((regs->cs & 0xFFFF) == __KERNEL_CS || (regs->cs & 0xFFFF) == __KERNEXEC_KERNEL_CS)) str = "PAX: suspicious stack segment fault"; #endif die(str, regs, error_code); } #ifdef CONFIG_PAX_REFCOUNT if (trapnr == 4) pax_report_refcount_overflow(regs); #endif return; #ifdef CONFIG_X86_32 vm86_trap: if (handle_vm86_trap((struct kernel_vm86_regs *) regs, error_code, trapnr)) goto trap_signal; return; #endif }
/* * This routine handles page faults. It determines the address and the * problem, and then passes it off to one of the appropriate routines. */ asmlinkage void do_page_fault(struct pt_regs *regs) { struct task_struct *tsk; struct vm_area_struct *vma; struct mm_struct *mm; unsigned long addr, cause; unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; int fault, code = SEGV_MAPERR; cause = regs->scause; addr = regs->sbadaddr; tsk = current; mm = tsk->mm; /* * Fault-in kernel-space virtual memory on-demand. * The 'reference' page table is init_mm.pgd. * * NOTE! We MUST NOT take any locks for this case. We may * be in an interrupt or a critical region, and should * only copy the information from the master page table, * nothing more. */ if (unlikely((addr >= VMALLOC_START) && (addr <= VMALLOC_END))) goto vmalloc_fault; /* Enable interrupts if they were enabled in the parent context. */ if (likely(regs->sstatus & SR_PIE)) local_irq_enable(); /* * If we're in an interrupt, have no user context, or are running * in an atomic region, then we must not take the fault. */ if (unlikely(in_atomic() || !mm)) goto no_context; if (user_mode(regs)) flags |= FAULT_FLAG_USER; perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, addr); retry: down_read(&mm->mmap_sem); vma = find_vma(mm, addr); if (unlikely(!vma)) goto bad_area; if (likely(vma->vm_start <= addr)) goto good_area; if (unlikely(!(vma->vm_flags & VM_GROWSDOWN))) goto bad_area; if (unlikely(expand_stack(vma, addr))) goto bad_area; /* * Ok, we have a good vm_area for this memory access, so * we can handle it. */ good_area: code = SEGV_ACCERR; switch (cause) { case EXC_INST_ACCESS: if (!(vma->vm_flags & VM_EXEC)) goto bad_area; break; case EXC_LOAD_ACCESS: if (!(vma->vm_flags & VM_READ)) goto bad_area; break; case EXC_STORE_ACCESS: if (!(vma->vm_flags & VM_WRITE)) goto bad_area; flags |= FAULT_FLAG_WRITE; break; default: panic("%s: unhandled cause %lu", __func__, cause); } /* * If for any reason at all we could not handle the fault, * make sure we exit gracefully rather than endlessly redo * the fault. */ fault = handle_mm_fault(mm, vma, addr, flags); /* * If we need to retry but a fatal signal is pending, handle the * signal first. We do not need to release the mmap_sem because it * would already be released in __lock_page_or_retry in mm/filemap.c. */ if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(tsk)) return; if (unlikely(fault & VM_FAULT_ERROR)) { if (fault & VM_FAULT_OOM) goto out_of_memory; else if (fault & VM_FAULT_SIGBUS) goto do_sigbus; BUG(); } /* * Major/minor page fault accounting is only done on the * initial attempt. If we go through a retry, it is extremely * likely that the page will be found in page cache at that point. */ if (flags & FAULT_FLAG_ALLOW_RETRY) { if (fault & VM_FAULT_MAJOR) { tsk->maj_flt++; perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, regs, addr); } else { tsk->min_flt++; perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, regs, addr); } if (fault & VM_FAULT_RETRY) { /* * Clear FAULT_FLAG_ALLOW_RETRY to avoid any risk * of starvation. */ flags &= ~(FAULT_FLAG_ALLOW_RETRY); flags |= FAULT_FLAG_TRIED; /* * No need to up_read(&mm->mmap_sem) as we would * have already released it in __lock_page_or_retry * in mm/filemap.c. */ goto retry; } } up_read(&mm->mmap_sem); return; /* * Something tried to access memory that isn't in our memory map. * Fix it, but check if it's kernel or user first. */ bad_area: up_read(&mm->mmap_sem); /* User mode accesses just cause a SIGSEGV */ if (user_mode(regs)) { do_trap(regs, SIGSEGV, code, addr, tsk); return; } no_context: /* Are we prepared to handle this kernel fault? */ if (fixup_exception(regs)) { return; } /* * Oops. The kernel tried to access some bad page. We'll have to * terminate things with extreme prejudice. */ bust_spinlocks(1); pr_alert("Unable to handle kernel %s at virtual address " REG_FMT "\n", (addr < PAGE_SIZE) ? "NULL pointer dereference" : "paging request", addr); die(regs, "Oops"); do_exit(SIGKILL); /* * We ran out of memory, call the OOM killer, and return the userspace * (which will retry the fault, or kill us if we got oom-killed). */ out_of_memory: up_read(&mm->mmap_sem); if (!user_mode(regs)) goto no_context; pagefault_out_of_memory(); return; do_sigbus: up_read(&mm->mmap_sem); /* Kernel mode? Handle exceptions or die */ if (!user_mode(regs)) goto no_context; do_trap(regs, SIGBUS, BUS_ADRERR, addr, tsk); return; vmalloc_fault: { pgd_t *pgd, *pgd_k; pud_t *pud, *pud_k; pmd_t *pmd, *pmd_k; pte_t *pte_k; int index; if (user_mode(regs)) goto bad_area; /* * Synchronize this task's top level page-table * with the 'reference' page table. * * Do _not_ use "tsk->active_mm->pgd" here. * We might be inside an interrupt in the middle * of a task switch. */ index = pgd_index(addr); pgd = (pgd_t *)pfn_to_virt(csr_read(sptbr)) + index; pgd_k = init_mm.pgd + index; if (!pgd_present(*pgd_k)) goto no_context; set_pgd(pgd, *pgd_k); pud = pud_offset(pgd, addr); pud_k = pud_offset(pgd_k, addr); if (!pud_present(*pud_k)) goto no_context; /* Since the vmalloc area is global, it is unnecessary to copy individual PTEs */ pmd = pmd_offset(pud, addr); pmd_k = pmd_offset(pud_k, addr); if (!pmd_present(*pmd_k)) goto no_context; set_pmd(pmd, *pmd_k); /* Make sure the actual PTE exists as well to * catch kernel vmalloc-area accesses to non-mapped * addresses. If we don't do this, this will just * silently loop forever. */ pte_k = pte_offset_kernel(pmd_k, addr); if (!pte_present(*pte_k)) goto no_context; return; } }
static void __kprobes do_trap(int trapnr, int signr, char *str, struct pt_regs *regs, long error_code, siginfo_t *info) { struct task_struct *tsk = current; #ifdef CONFIG_X86_32 if (regs->flags & X86_VM_MASK) { /* * traps 0, 1, 3, 4, and 5 should be forwarded to vm86. * On nmi (interrupt 2), do_trap should not be called. */ if (trapnr < 6) goto vm86_trap; goto trap_signal; } #endif if (!user_mode(regs)) goto kernel_trap; #ifdef CONFIG_X86_32 trap_signal: #endif /* * We want error_code and trap_no set for userspace faults and * kernelspace faults which result in die(), but not * kernelspace faults which are fixed up. die() gives the * process no chance to handle the signal and notice the * kernel fault information, so that won't result in polluting * the information about previously queued, but not yet * delivered, faults. See also do_general_protection below. */ tsk->thread.error_code = error_code; tsk->thread.trap_no = trapnr; #ifdef CONFIG_X86_64 if (show_unhandled_signals && unhandled_signal(tsk, signr) && printk_ratelimit()) { printk(KERN_INFO "%s[%d] trap %s ip:%lx sp:%lx error:%lx", tsk->comm, tsk->pid, str, regs->ip, regs->sp, error_code); print_vma_addr(" in ", regs->ip); printk("\n"); } #endif if (info) force_sig_info(signr, info, tsk); else force_sig(signr, tsk); return; kernel_trap: if (!fixup_exception(regs)) { tsk->thread.error_code = error_code; tsk->thread.trap_no = trapnr; die(str, regs, error_code); } return; #ifdef CONFIG_X86_32 vm86_trap: if (handle_vm86_trap((struct kernel_vm86_regs *) regs, error_code, trapnr)) goto trap_signal; return; #endif }
/* * This routine handles page faults. It determines the address, * and the problem, and then passes it off to one of the appropriate * routines. */ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long cause, unsigned long address) { struct vm_area_struct *vma = NULL; struct task_struct *tsk = current; struct mm_struct *mm = tsk->mm; int code = SEGV_MAPERR; int fault; unsigned int flags = 0; cause >>= 2; /* Restart the instruction */ regs->ea -= 4; /* * We fault-in kernel-space virtual memory on-demand. The * 'reference' page table is init_mm.pgd. * * NOTE! We MUST NOT take any locks for this case. We may * be in an interrupt or a critical region, and should * only copy the information from the master page table, * nothing more. */ if (unlikely(address >= VMALLOC_START && address <= VMALLOC_END)) { if (user_mode(regs)) goto bad_area_nosemaphore; else goto vmalloc_fault; } if (unlikely(address >= TASK_SIZE)) goto bad_area_nosemaphore; /* * If we're in an interrupt or have no user * context, we must not take the fault.. */ if (in_atomic() || !mm) goto bad_area_nosemaphore; down_read(&mm->mmap_sem); vma = find_vma(mm, address); if (!vma) goto bad_area; if (vma->vm_start <= address) goto good_area; if (!(vma->vm_flags & VM_GROWSDOWN)) goto bad_area; if (expand_stack(vma, address)) goto bad_area; /* * Ok, we have a good vm_area for this memory access, so * we can handle it.. */ good_area: code = SEGV_ACCERR; switch (cause) { case EXC_SUPERV_INSN_ACCESS: goto bad_area; case EXC_SUPERV_DATA_ACCESS: goto bad_area; case EXC_X_PROTECTION_FAULT: if (!(vma->vm_flags & VM_EXEC)) goto bad_area; break; case EXC_R_PROTECTION_FAULT: if (!(vma->vm_flags & VM_READ)) goto bad_area; break; case EXC_W_PROTECTION_FAULT: flags = FAULT_FLAG_WRITE; if (!(vma->vm_flags & VM_WRITE)) goto bad_area; break; } survive: /* * If for any reason at all we couldn't handle the fault, * make sure we exit gracefully rather than endlessly redo * the fault. */ fault = handle_mm_fault(mm, vma, address, flags); if (unlikely(fault & VM_FAULT_ERROR)) { if (fault & VM_FAULT_OOM) goto out_of_memory; else if (fault & VM_FAULT_SIGBUS) goto do_sigbus; BUG(); } if (fault & VM_FAULT_MAJOR) tsk->maj_flt++; else tsk->min_flt++; up_read(&mm->mmap_sem); return; /* * Something tried to access memory that isn't in our memory map.. * Fix it, but check if it's kernel or user first.. */ bad_area: up_read(&mm->mmap_sem); bad_area_nosemaphore: /* User mode accesses just cause a SIGSEGV */ if (user_mode(regs)) { _exception(SIGSEGV, regs, code, address); return; } no_context: /* Are we prepared to handle this kernel fault? */ if (fixup_exception(regs)) return; /* * Oops. The kernel tried to access some bad page. We'll have to * terminate things with extreme prejudice. */ bust_spinlocks(1); pr_alert("Unable to handle kernel %s at virtual address %08lx", address < PAGE_SIZE ? "NULL pointer dereference" : "paging request", address); pr_alert("ea = %08lx, ra = %08lx, cause = %ld\n", regs->ea, regs->ra, cause); panic("Oops"); return; /* * We ran out of memory, or some other thing happened to us that made * us unable to handle the page fault gracefully. */ out_of_memory: up_read(&mm->mmap_sem); if (is_global_init(tsk)) { yield(); down_read(&mm->mmap_sem); goto survive; } pr_info("VM: killing process %s\n", tsk->comm); if (user_mode(regs)) do_group_exit(SIGKILL); goto no_context; do_sigbus: up_read(&mm->mmap_sem); /* Kernel mode? Handle exceptions or die */ if (!user_mode(regs)) goto no_context; _exception(SIGBUS, regs, BUS_ADRERR, address); return; vmalloc_fault: { /* * Synchronize this task's top level page-table * with the 'reference' page table. * * Do _not_ use "tsk" here. We might be inside * an interrupt in the middle of a task switch.. */ int offset = pgd_index(address); pgd_t *pgd, *pgd_k; pud_t *pud, *pud_k; pmd_t *pmd, *pmd_k; pte_t *pte_k; #if 1 /* FIXME: Is this entirely correct ? */ pgd = pgd_current + offset; #else pgd = ¤t->mm->pgd[offset]; #endif pgd_k = init_mm.pgd + offset; if (!pgd_present(*pgd_k)) goto no_context; set_pgd(pgd, *pgd_k); pud = pud_offset(pgd, address); pud_k = pud_offset(pgd_k, address); if (!pud_present(*pud_k)) goto no_context; pmd = pmd_offset(pud, address); pmd_k = pmd_offset(pud_k, address); if (!pmd_present(*pmd_k)) goto no_context; set_pmd(pmd, *pmd_k); pte_k = pte_offset_kernel(pmd_k, address); if (!pte_present(*pte_k)) goto no_context; flush_tlb_one(address); return; } }
/* * This routine handles page faults. It determines the address, * and the problem, and then passes it off to one of the appropriate * routines. */ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long cause, unsigned long address) { struct vm_area_struct *vma = NULL; struct task_struct *tsk = current; struct mm_struct *mm = tsk->mm; int code = SEGV_MAPERR; int fault; unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; cause >>= 2; /* Restart the instruction */ regs->ea -= 4; /* * We fault-in kernel-space virtual memory on-demand. The * 'reference' page table is init_mm.pgd. * * NOTE! We MUST NOT take any locks for this case. We may * be in an interrupt or a critical region, and should * only copy the information from the master page table, * nothing more. */ if (unlikely(address >= VMALLOC_START && address <= VMALLOC_END)) { if (user_mode(regs)) goto bad_area_nosemaphore; else goto vmalloc_fault; } if (unlikely(address >= TASK_SIZE)) goto bad_area_nosemaphore; /* * If we're in an interrupt or have no user * context, we must not take the fault.. */ if (faulthandler_disabled() || !mm) goto bad_area_nosemaphore; if (user_mode(regs)) flags |= FAULT_FLAG_USER; if (!down_read_trylock(&mm->mmap_sem)) { if (!user_mode(regs) && !search_exception_tables(regs->ea)) goto bad_area_nosemaphore; retry: down_read(&mm->mmap_sem); } vma = find_vma(mm, address); if (!vma) goto bad_area; if (vma->vm_start <= address) goto good_area; if (!(vma->vm_flags & VM_GROWSDOWN)) goto bad_area; if (expand_stack(vma, address)) goto bad_area; /* * Ok, we have a good vm_area for this memory access, so * we can handle it.. */ good_area: code = SEGV_ACCERR; switch (cause) { case EXC_SUPERV_INSN_ACCESS: goto bad_area; case EXC_SUPERV_DATA_ACCESS: goto bad_area; case EXC_X_PROTECTION_FAULT: if (!(vma->vm_flags & VM_EXEC)) goto bad_area; break; case EXC_R_PROTECTION_FAULT: if (!(vma->vm_flags & VM_READ)) goto bad_area; break; case EXC_W_PROTECTION_FAULT: if (!(vma->vm_flags & VM_WRITE)) goto bad_area; flags = FAULT_FLAG_WRITE; break; } /* * If for any reason at all we couldn't handle the fault, * make sure we exit gracefully rather than endlessly redo * the fault. */ fault = handle_mm_fault(mm, vma, address, flags); if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) return; if (unlikely(fault & VM_FAULT_ERROR)) { if (fault & VM_FAULT_OOM) goto out_of_memory; else if (fault & VM_FAULT_SIGSEGV) goto bad_area; else if (fault & VM_FAULT_SIGBUS) goto do_sigbus; BUG(); } /* * Major/minor page fault accounting is only done on the * initial attempt. If we go through a retry, it is extremely * likely that the page will be found in page cache at that point. */ if (flags & FAULT_FLAG_ALLOW_RETRY) { if (fault & VM_FAULT_MAJOR) current->maj_flt++; else current->min_flt++; if (fault & VM_FAULT_RETRY) { /* Clear FAULT_FLAG_ALLOW_RETRY to avoid any risk * of starvation. */ flags &= ~FAULT_FLAG_ALLOW_RETRY; flags |= FAULT_FLAG_TRIED; /* * No need to up_read(&mm->mmap_sem) as we would * have already released it in __lock_page_or_retry * in mm/filemap.c. */ goto retry; } } up_read(&mm->mmap_sem); return; /* * Something tried to access memory that isn't in our memory map.. * Fix it, but check if it's kernel or user first.. */ bad_area: up_read(&mm->mmap_sem); bad_area_nosemaphore: /* User mode accesses just cause a SIGSEGV */ if (user_mode(regs)) { if (unhandled_signal(current, SIGSEGV) && printk_ratelimit()) { pr_info("%s: unhandled page fault (%d) at 0x%08lx, " "cause %ld\n", current->comm, SIGSEGV, address, cause); show_regs(regs); } _exception(SIGSEGV, regs, code, address); return; } no_context: /* Are we prepared to handle this kernel fault? */ if (fixup_exception(regs)) return; /* * Oops. The kernel tried to access some bad page. We'll have to * terminate things with extreme prejudice. */ bust_spinlocks(1); pr_alert("Unable to handle kernel %s at virtual address %08lx", address < PAGE_SIZE ? "NULL pointer dereference" : "paging request", address); pr_alert("ea = %08lx, ra = %08lx, cause = %ld\n", regs->ea, regs->ra, cause); panic("Oops"); return; /* * We ran out of memory, or some other thing happened to us that made * us unable to handle the page fault gracefully. */ out_of_memory: up_read(&mm->mmap_sem); if (!user_mode(regs)) goto no_context; pagefault_out_of_memory(); return; do_sigbus: up_read(&mm->mmap_sem); /* Kernel mode? Handle exceptions or die */ if (!user_mode(regs)) goto no_context; _exception(SIGBUS, regs, BUS_ADRERR, address); return; vmalloc_fault: { /* * Synchronize this task's top level page-table * with the 'reference' page table. * * Do _not_ use "tsk" here. We might be inside * an interrupt in the middle of a task switch.. */ int offset = pgd_index(address); pgd_t *pgd, *pgd_k; pud_t *pud, *pud_k; pmd_t *pmd, *pmd_k; pte_t *pte_k; pgd = pgd_current + offset; pgd_k = init_mm.pgd + offset; if (!pgd_present(*pgd_k)) goto no_context; set_pgd(pgd, *pgd_k); pud = pud_offset(pgd, address); pud_k = pud_offset(pgd_k, address); if (!pud_present(*pud_k)) goto no_context; pmd = pmd_offset(pud, address); pmd_k = pmd_offset(pud_k, address); if (!pmd_present(*pmd_k)) goto no_context; set_pmd(pmd, *pmd_k); pte_k = pte_offset_kernel(pmd_k, address); if (!pte_present(*pte_k)) goto no_context; flush_tlb_one(address); return; } }
dotraplinkage void __kprobes do_general_protection(struct pt_regs *regs, long error_code) { struct task_struct *tsk; conditional_sti(regs); #ifdef CONFIG_X86_32 if (v8086_mode(regs)) goto gp_in_vm86; #endif tsk = current; if (!user_mode_novm(regs)) goto gp_in_kernel; #if defined(CONFIG_X86_32) && defined(CONFIG_PAX_PAGEEXEC) if (!nx_enabled && tsk->mm && (tsk->mm->pax_flags & MF_PAX_PAGEEXEC)) { struct mm_struct *mm = tsk->mm; unsigned long limit; down_write(&mm->mmap_sem); limit = mm->context.user_cs_limit; if (limit < TASK_SIZE) { track_exec_limit(mm, limit, TASK_SIZE, VM_EXEC); up_write(&mm->mmap_sem); return; } up_write(&mm->mmap_sem); } #endif tsk->thread.error_code = error_code; tsk->thread.trap_no = 13; if (show_unhandled_signals && unhandled_signal(tsk, SIGSEGV) && printk_ratelimit()) { printk(KERN_INFO "%s[%d] general protection ip:%lx sp:%lx error:%lx", tsk->comm, task_pid_nr(tsk), regs->ip, regs->sp, error_code); print_vma_addr(" in ", regs->ip); printk("\n"); } force_sig(SIGSEGV, tsk); return; #ifdef CONFIG_X86_32 gp_in_vm86: local_irq_enable(); handle_vm86_fault((struct kernel_vm86_regs *) regs, error_code); return; #endif gp_in_kernel: if (fixup_exception(regs)) return; tsk->thread.error_code = error_code; tsk->thread.trap_no = 13; if (notify_die(DIE_GPF, "general protection fault", regs, error_code, 13, SIGSEGV) == NOTIFY_STOP) return; #if defined(CONFIG_X86_32) && defined(CONFIG_PAX_KERNEXEC) if ((regs->cs & 0xFFFF) == __KERNEL_CS) die("PAX: suspicious general protection fault", regs, error_code); else #endif die("general protection fault", regs, error_code); }
void do_page_fault(struct pt_regs *regs, int write, unsigned long address, unsigned long cause_code) { struct vm_area_struct *vma = NULL; struct task_struct *tsk = current; struct mm_struct *mm = tsk->mm; siginfo_t info; int fault, ret; unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; /* * We fault-in kernel-space virtual memory on-demand. The * 'reference' page table is init_mm.pgd. * * NOTE! We MUST NOT take any locks for this case. We may * be in an interrupt or a critical region, and should * only copy the information from the master page table, * nothing more. */ if (address >= VMALLOC_START && address <= VMALLOC_END) { ret = handle_vmalloc_fault(address); if (unlikely(ret)) goto bad_area_nosemaphore; else return; } info.si_code = SEGV_MAPERR; /* * If we're in an interrupt or have no user * context, we must not take the fault.. */ if (in_atomic() || !mm) goto no_context; if (user_mode(regs)) flags |= FAULT_FLAG_USER; retry: down_read(&mm->mmap_sem); vma = find_vma(mm, address); if (!vma) goto bad_area; if (vma->vm_start <= address) goto good_area; if (!(vma->vm_flags & VM_GROWSDOWN)) goto bad_area; if (expand_stack(vma, address)) goto bad_area; /* * Ok, we have a good vm_area for this memory access, so * we can handle it.. */ good_area: info.si_code = SEGV_ACCERR; /* Handle protection violation, execute on heap or stack */ if (cause_code == ((ECR_V_PROTV << 16) | ECR_C_PROTV_INST_FETCH)) goto bad_area; if (write) { if (!(vma->vm_flags & VM_WRITE)) goto bad_area; flags |= FAULT_FLAG_WRITE; } else { if (!(vma->vm_flags & (VM_READ | VM_EXEC))) goto bad_area; } survive: /* * If for any reason at all we couldn't handle the fault, * make sure we exit gracefully rather than endlessly redo * the fault. */ fault = handle_mm_fault(mm, vma, address, flags); /* If Pagefault was interrupted by SIGKILL, exit page fault "early" */ if (unlikely(fatal_signal_pending(current))) { if ((fault & VM_FAULT_ERROR) && !(fault & VM_FAULT_RETRY)) up_read(&mm->mmap_sem); if (user_mode(regs)) return; } if (likely(!(fault & VM_FAULT_ERROR))) { if (flags & FAULT_FLAG_ALLOW_RETRY) { /* To avoid updating stats twice for retry case */ if (fault & VM_FAULT_MAJOR) tsk->maj_flt++; else tsk->min_flt++; if (fault & VM_FAULT_RETRY) { flags &= ~FAULT_FLAG_ALLOW_RETRY; flags |= FAULT_FLAG_TRIED; goto retry; } } /* Fault Handled Gracefully */ up_read(&mm->mmap_sem); return; } /* TBD: switch to pagefault_out_of_memory() */ if (fault & VM_FAULT_OOM) goto out_of_memory; else if (fault & VM_FAULT_SIGBUS) goto do_sigbus; /* no man's land */ BUG(); /* * Something tried to access memory that isn't in our memory map.. * Fix it, but check if it's kernel or user first.. */ bad_area: up_read(&mm->mmap_sem); bad_area_nosemaphore: /* User mode accesses just cause a SIGSEGV */ if (user_mode(regs)) { tsk->thread.fault_address = address; tsk->thread.cause_code = cause_code; info.si_signo = SIGSEGV; info.si_errno = 0; /* info.si_code has been set above */ info.si_addr = (void __user *)address; force_sig_info(SIGSEGV, &info, tsk); return; } no_context: /* Are we prepared to handle this kernel fault? * * (The kernel has valid exception-points in the source * when it acesses user-memory. When it fails in one * of those points, we find it in a table and do a jump * to some fixup code that loads an appropriate error * code) */ if (fixup_exception(regs)) return; die("Oops", regs, address, cause_code); out_of_memory: if (is_global_init(tsk)) { yield(); goto survive; } up_read(&mm->mmap_sem); if (user_mode(regs)) do_group_exit(SIGKILL); /* This will never return */ goto no_context; do_sigbus: up_read(&mm->mmap_sem); if (!user_mode(regs)) goto no_context; tsk->thread.fault_address = address; tsk->thread.cause_code = cause_code; info.si_signo = SIGBUS; info.si_errno = 0; info.si_code = BUS_ADRERR; info.si_addr = (void __user *)address; force_sig_info(SIGBUS, &info, tsk); }
/* * This routine is responsible for faulting in user pages. * It passes the work off to one of the appropriate routines. * It returns true if the fault was successfully handled. */ static int handle_page_fault(struct pt_regs *regs, int fault_num, int is_page_fault, unsigned long address, int write) { struct task_struct *tsk; struct mm_struct *mm; struct vm_area_struct *vma; unsigned long stack_offset; int fault; int si_code; int is_kernel_mode; pgd_t *pgd; /* on TILE, protection faults are always writes */ if (!is_page_fault) write = 1; flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; is_kernel_mode = (EX1_PL(regs->ex1) != USER_PL); tsk = validate_current(); /* * Check to see if we might be overwriting the stack, and bail * out if so. The page fault code is a relatively likely * place to get trapped in an infinite regress, and once we * overwrite the whole stack, it becomes very hard to recover. */ stack_offset = stack_pointer & (THREAD_SIZE-1); if (stack_offset < THREAD_SIZE / 8) { pr_alert("Potential stack overrun: sp %#lx\n", stack_pointer); show_regs(regs); pr_alert("Killing current process %d/%s\n", tsk->pid, tsk->comm); do_group_exit(SIGKILL); } /* * Early on, we need to check for migrating PTE entries; * see homecache.c. If we find a migrating PTE, we wait until * the backing page claims to be done migrating, then we proceed. * For kernel PTEs, we rewrite the PTE and return and retry. * Otherwise, we treat the fault like a normal "no PTE" fault, * rather than trying to patch up the existing PTE. */ pgd = get_current_pgd(); if (handle_migrating_pte(pgd, fault_num, address, regs->pc, is_kernel_mode, write)) return 1; si_code = SEGV_MAPERR; /* * We fault-in kernel-space virtual memory on-demand. The * 'reference' page table is init_mm.pgd. * * NOTE! We MUST NOT take any locks for this case. We may * be in an interrupt or a critical region, and should * only copy the information from the master page table, * nothing more. * * This verifies that the fault happens in kernel space * and that the fault was not a protection fault. */ if (unlikely(address >= TASK_SIZE && !is_arch_mappable_range(address, 0))) { if (is_kernel_mode && is_page_fault && vmalloc_fault(pgd, address) >= 0) return 1; /* * Don't take the mm semaphore here. If we fixup a prefetch * fault we could otherwise deadlock. */ mm = NULL; /* happy compiler */ vma = NULL; goto bad_area_nosemaphore; } /* * If we're trying to touch user-space addresses, we must * be either at PL0, or else with interrupts enabled in the * kernel, so either way we can re-enable interrupts here * unless we are doing atomic access to user space with * interrupts disabled. */ if (!(regs->flags & PT_FLAGS_DISABLE_IRQ)) local_irq_enable(); mm = tsk->mm; /* * If we're in an interrupt, have no user context or are running in an * atomic region then we must not take the fault. */ if (in_atomic() || !mm) { vma = NULL; /* happy compiler */ goto bad_area_nosemaphore; } if (!is_kernel_mode) flags |= FAULT_FLAG_USER; /* * When running in the kernel we expect faults to occur only to * addresses in user space. All other faults represent errors in the * kernel and should generate an OOPS. Unfortunately, in the case of an * erroneous fault occurring in a code path which already holds mmap_sem * we will deadlock attempting to validate the fault against the * address space. Luckily the kernel only validly references user * space from well defined areas of code, which are listed in the * exceptions table. * * As the vast majority of faults will be valid we will only perform * the source reference check when there is a possibility of a deadlock. * Attempt to lock the address space, if we cannot we then validate the * source. If this is invalid we can skip the address space check, * thus avoiding the deadlock. */ if (!down_read_trylock(&mm->mmap_sem)) { if (is_kernel_mode && !search_exception_tables(regs->pc)) { vma = NULL; /* happy compiler */ goto bad_area_nosemaphore; } down_read(&mm->mmap_sem); } vma = find_vma(mm, address); if (!vma) goto bad_area; if (vma->vm_start <= address) goto good_area; if (!(vma->vm_flags & VM_GROWSDOWN)) goto bad_area; if (regs->sp < PAGE_OFFSET) { /* * accessing the stack below sp is always a bug. */ if (address < regs->sp) goto bad_area; } if (expand_stack(vma, address)) goto bad_area; /* * Ok, we have a good vm_area for this memory access, so * we can handle it.. */ good_area: si_code = SEGV_ACCERR; if (fault_num == INT_ITLB_MISS) { if (!(vma->vm_flags & VM_EXEC)) goto bad_area; } else if (write) { #ifdef TEST_VERIFY_AREA if (!is_page_fault && regs->cs == KERNEL_CS) pr_err("WP fault at "REGFMT"\n", regs->eip); #endif if (!(vma->vm_flags & VM_WRITE)) goto bad_area; flags |= FAULT_FLAG_WRITE; } else { if (!is_page_fault || !(vma->vm_flags & VM_READ)) goto bad_area; } /* * If for any reason at all we couldn't handle the fault, * make sure we exit gracefully rather than endlessly redo * the fault. */ fault = handle_mm_fault(mm, vma, address, write); if (unlikely(fault & VM_FAULT_ERROR)) { if (fault & VM_FAULT_OOM) goto out_of_memory; else if (fault & VM_FAULT_SIGBUS) goto do_sigbus; BUG(); } if (fault & VM_FAULT_MAJOR) tsk->maj_flt++; else tsk->min_flt++; #if CHIP_HAS_TILE_DMA() || CHIP_HAS_SN_PROC() /* * If this was an asynchronous fault, * restart the appropriate engine. */ switch (fault_num) { #if CHIP_HAS_TILE_DMA() case INT_DMATLB_MISS: case INT_DMATLB_MISS_DWNCL: case INT_DMATLB_ACCESS: case INT_DMATLB_ACCESS_DWNCL: __insn_mtspr(SPR_DMA_CTR, SPR_DMA_CTR__REQUEST_MASK); break; #endif #if CHIP_HAS_SN_PROC() case INT_SNITLB_MISS: case INT_SNITLB_MISS_DWNCL: __insn_mtspr(SPR_SNCTL, __insn_mfspr(SPR_SNCTL) & ~SPR_SNCTL__FRZPROC_MASK); break; #endif } #endif up_read(&mm->mmap_sem); return 1; /* * Something tried to access memory that isn't in our memory map.. * Fix it, but check if it's kernel or user first.. */ bad_area: up_read(&mm->mmap_sem); bad_area_nosemaphore: /* User mode accesses just cause a SIGSEGV */ if (!is_kernel_mode) { /* * It's possible to have interrupts off here. */ local_irq_enable(); force_sig_info_fault("segfault", SIGSEGV, si_code, address, fault_num, tsk, regs); return 0; } no_context: /* Are we prepared to handle this kernel fault? */ if (fixup_exception(regs)) return 0; /* * Oops. The kernel tried to access some bad page. We'll have to * terminate things with extreme prejudice. */ bust_spinlocks(1); /* FIXME: no lookup_address() yet */ #ifdef SUPPORT_LOOKUP_ADDRESS if (fault_num == INT_ITLB_MISS) { pte_t *pte = lookup_address(address); if (pte && pte_present(*pte) && !pte_exec_kernel(*pte)) pr_crit("kernel tried to execute" " non-executable page - exploit attempt?" " (uid: %d)\n", current->uid); } #endif if (address < PAGE_SIZE) pr_alert("Unable to handle kernel NULL pointer dereference\n"); else pr_alert("Unable to handle kernel paging request\n"); pr_alert(" at virtual address "REGFMT", pc "REGFMT"\n", address, regs->pc); show_regs(regs); if (unlikely(tsk->pid < 2)) { panic("Kernel page fault running %s!", is_idle_task(tsk) ? "the idle task" : "init"); } /* * More FIXME: we should probably copy the i386 here and * implement a generic die() routine. Not today. */ #ifdef SUPPORT_DIE die("Oops", regs); #endif bust_spinlocks(1); do_group_exit(SIGKILL); /* * We ran out of memory, or some other thing happened to us that made * us unable to handle the page fault gracefully. */ out_of_memory: up_read(&mm->mmap_sem); if (is_kernel_mode) goto no_context; pagefault_out_of_memory(); return 0; do_sigbus: up_read(&mm->mmap_sem); /* Kernel mode? Handle exceptions or die */ if (is_kernel_mode) goto no_context; force_sig_info_fault("bus error", SIGBUS, BUS_ADRERR, address, fault_num, tsk, regs); return 0; }
static void emulate_load_store_MIPS16e(struct pt_regs *regs, void __user * addr) { unsigned long value; unsigned int res; int reg; unsigned long orig31; u16 __user *pc16; unsigned long origpc; union mips16e_instruction mips16inst, oldinst; origpc = regs->cp0_epc; orig31 = regs->regs[31]; pc16 = (unsigned short __user *)msk_isa16_mode(origpc); /* * This load never faults. */ __get_user(mips16inst.full, pc16); oldinst = mips16inst; /* skip EXTEND instruction */ if (mips16inst.ri.opcode == MIPS16e_extend_op) { pc16++; __get_user(mips16inst.full, pc16); } else if (delay_slot(regs)) { /* skip jump instructions */ /* JAL/JALX are 32 bits but have OPCODE in first short int */ if (mips16inst.ri.opcode == MIPS16e_jal_op) pc16++; pc16++; if (get_user(mips16inst.full, pc16)) goto sigbus; } switch (mips16inst.ri.opcode) { case MIPS16e_i64_op: /* I64 or RI64 instruction */ switch (mips16inst.i64.func) { /* I64/RI64 func field check */ case MIPS16e_ldpc_func: case MIPS16e_ldsp_func: reg = reg16to32[mips16inst.ri64.ry]; goto loadDW; case MIPS16e_sdsp_func: reg = reg16to32[mips16inst.ri64.ry]; goto writeDW; case MIPS16e_sdrasp_func: reg = 29; /* GPRSP */ goto writeDW; } goto sigbus; case MIPS16e_swsp_op: case MIPS16e_lwpc_op: case MIPS16e_lwsp_op: reg = reg16to32[mips16inst.ri.rx]; break; case MIPS16e_i8_op: if (mips16inst.i8.func != MIPS16e_swrasp_func) goto sigbus; reg = 29; /* GPRSP */ break; default: reg = reg16to32[mips16inst.rri.ry]; break; } switch (mips16inst.ri.opcode) { case MIPS16e_lb_op: case MIPS16e_lbu_op: case MIPS16e_sb_op: goto sigbus; case MIPS16e_lh_op: if (!access_ok(VERIFY_READ, addr, 2)) goto sigbus; LoadHW(addr, value, res); if (res) goto fault; MIPS16e_compute_return_epc(regs, &oldinst); regs->regs[reg] = value; break; case MIPS16e_lhu_op: if (!access_ok(VERIFY_READ, addr, 2)) goto sigbus; LoadHWU(addr, value, res); if (res) goto fault; MIPS16e_compute_return_epc(regs, &oldinst); regs->regs[reg] = value; break; case MIPS16e_lw_op: case MIPS16e_lwpc_op: case MIPS16e_lwsp_op: if (!access_ok(VERIFY_READ, addr, 4)) goto sigbus; LoadW(addr, value, res); if (res) goto fault; MIPS16e_compute_return_epc(regs, &oldinst); regs->regs[reg] = value; break; case MIPS16e_lwu_op: #ifdef CONFIG_64BIT /* * A 32-bit kernel might be running on a 64-bit processor. But * if we're on a 32-bit processor and an i-cache incoherency * or race makes us see a 64-bit instruction here the sdl/sdr * would blow up, so for now we don't handle unaligned 64-bit * instructions on 32-bit kernels. */ if (!access_ok(VERIFY_READ, addr, 4)) goto sigbus; LoadWU(addr, value, res); if (res) goto fault; MIPS16e_compute_return_epc(regs, &oldinst); regs->regs[reg] = value; break; #endif /* CONFIG_64BIT */ /* Cannot handle 64-bit instructions in 32-bit kernel */ goto sigill; case MIPS16e_ld_op: loadDW: #ifdef CONFIG_64BIT /* * A 32-bit kernel might be running on a 64-bit processor. But * if we're on a 32-bit processor and an i-cache incoherency * or race makes us see a 64-bit instruction here the sdl/sdr * would blow up, so for now we don't handle unaligned 64-bit * instructions on 32-bit kernels. */ if (!access_ok(VERIFY_READ, addr, 8)) goto sigbus; LoadDW(addr, value, res); if (res) goto fault; MIPS16e_compute_return_epc(regs, &oldinst); regs->regs[reg] = value; break; #endif /* CONFIG_64BIT */ /* Cannot handle 64-bit instructions in 32-bit kernel */ goto sigill; case MIPS16e_sh_op: if (!access_ok(VERIFY_WRITE, addr, 2)) goto sigbus; MIPS16e_compute_return_epc(regs, &oldinst); value = regs->regs[reg]; StoreHW(addr, value, res); if (res) goto fault; break; case MIPS16e_sw_op: case MIPS16e_swsp_op: case MIPS16e_i8_op: /* actually - MIPS16e_swrasp_func */ if (!access_ok(VERIFY_WRITE, addr, 4)) goto sigbus; MIPS16e_compute_return_epc(regs, &oldinst); value = regs->regs[reg]; StoreW(addr, value, res); if (res) goto fault; break; case MIPS16e_sd_op: writeDW: #ifdef CONFIG_64BIT /* * A 32-bit kernel might be running on a 64-bit processor. But * if we're on a 32-bit processor and an i-cache incoherency * or race makes us see a 64-bit instruction here the sdl/sdr * would blow up, so for now we don't handle unaligned 64-bit * instructions on 32-bit kernels. */ if (!access_ok(VERIFY_WRITE, addr, 8)) goto sigbus; MIPS16e_compute_return_epc(regs, &oldinst); value = regs->regs[reg]; StoreDW(addr, value, res); if (res) goto fault; break; #endif /* CONFIG_64BIT */ /* Cannot handle 64-bit instructions in 32-bit kernel */ goto sigill; default: /* * Pheeee... We encountered an yet unknown instruction or * cache coherence problem. Die sucker, die ... */ goto sigill; } #ifdef CONFIG_DEBUG_FS unaligned_instructions++; #endif return; fault: /* roll back jump/branch */ regs->cp0_epc = origpc; regs->regs[31] = orig31; /* Did we have an exception handler installed? */ if (fixup_exception(regs)) return; die_if_kernel("Unhandled kernel unaligned access", regs); force_sig(SIGSEGV, current); return; sigbus: die_if_kernel("Unhandled kernel unaligned access", regs); force_sig(SIGBUS, current); return; sigill: die_if_kernel ("Unhandled kernel unaligned access or invalid instruction", regs); force_sig(SIGILL, current); }
static void emulate_load_store_insn(struct pt_regs *regs, void __user *addr, unsigned int __user *pc) { union mips_instruction insn; unsigned long value; unsigned int res; unsigned long origpc; unsigned long orig31; void __user *fault_addr = NULL; origpc = (unsigned long)pc; orig31 = regs->regs[31]; perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, 0); /* * This load never faults. */ __get_user(insn.word, pc); switch (insn.i_format.opcode) { /* * These are instructions that a compiler doesn't generate. We * can assume therefore that the code is MIPS-aware and * really buggy. Emulating these instructions would break the * semantics anyway. */ case ll_op: case lld_op: case sc_op: case scd_op: /* * For these instructions the only way to create an address * error is an attempted access to kernel/supervisor address * space. */ case ldl_op: case ldr_op: case lwl_op: case lwr_op: case sdl_op: case sdr_op: case swl_op: case swr_op: case lb_op: case lbu_op: case sb_op: goto sigbus; case spec3_op: if (insn.r_format.func != lx_op) goto sigill; switch (insn.r_format.re) { case lwx_op: if (!access_ok(VERIFY_READ, addr, 4)) goto sigbus; LoadW(addr, value, res); if (res) goto fault; compute_return_epc(regs); regs->regs[insn.r_format.rd] = value; break; case lhx_op: if (!access_ok(VERIFY_READ, addr, 2)) goto sigbus; LoadHW(addr, value, res); if (res) goto fault; compute_return_epc(regs); regs->regs[insn.r_format.rd] = value; break; #ifdef CONFIG_64BIT case ldx_op: if (!access_ok(VERIFY_READ, addr, 8)) goto sigbus; LoadDW(addr, value, res); if (res) goto fault; compute_return_epc(regs); regs->regs[insn.r_format.rd] = value; break; case lwux_op: if (!access_ok(VERIFY_READ, addr, 4)) goto sigbus; LoadWU(addr, value, res); if (res) goto fault; compute_return_epc(regs); regs->regs[insn.r_format.rd] = value; break; #endif /* CONFIG_64BIT */ case lhux_op: if (!access_ok(VERIFY_READ, addr, 2)) goto sigbus; LoadHWU(addr, value, res); if (res) goto fault; compute_return_epc(regs); regs->regs[insn.r_format.rd] = value; break; case lbux_op: case lbx_op: goto sigbus; default: goto sigill; } break; /* * The remaining opcodes are the ones that are really of * interest. */ case lh_op: if (!access_ok(VERIFY_READ, addr, 2)) goto sigbus; LoadHW(addr, value, res); if (res) goto fault; compute_return_epc(regs); regs->regs[insn.i_format.rt] = value; break; case lw_op: if (!access_ok(VERIFY_READ, addr, 4)) goto sigbus; LoadW(addr, value, res); if (res) goto fault; compute_return_epc(regs); regs->regs[insn.i_format.rt] = value; break; case lhu_op: if (!access_ok(VERIFY_READ, addr, 2)) goto sigbus; LoadHWU(addr, value, res); if (res) goto fault; compute_return_epc(regs); regs->regs[insn.i_format.rt] = value; break; case lwu_op: #ifdef CONFIG_64BIT /* * A 32-bit kernel might be running on a 64-bit processor. But * if we're on a 32-bit processor and an i-cache incoherency * or race makes us see a 64-bit instruction here the sdl/sdr * would blow up, so for now we don't handle unaligned 64-bit * instructions on 32-bit kernels. */ if (!access_ok(VERIFY_READ, addr, 4)) goto sigbus; LoadWU(addr, value, res); if (res) goto fault; compute_return_epc(regs); regs->regs[insn.i_format.rt] = value; break; #endif /* CONFIG_64BIT */ /* Cannot handle 64-bit instructions in 32-bit kernel */ goto sigill; case ld_op: #ifdef CONFIG_64BIT /* * A 32-bit kernel might be running on a 64-bit processor. But * if we're on a 32-bit processor and an i-cache incoherency * or race makes us see a 64-bit instruction here the sdl/sdr * would blow up, so for now we don't handle unaligned 64-bit * instructions on 32-bit kernels. */ if (!access_ok(VERIFY_READ, addr, 8)) goto sigbus; LoadDW(addr, value, res); if (res) goto fault; compute_return_epc(regs); regs->regs[insn.i_format.rt] = value; break; #endif /* CONFIG_64BIT */ /* Cannot handle 64-bit instructions in 32-bit kernel */ goto sigill; case sh_op: if (!access_ok(VERIFY_WRITE, addr, 2)) goto sigbus; compute_return_epc(regs); value = regs->regs[insn.i_format.rt]; StoreHW(addr, value, res); if (res) goto fault; break; case sw_op: if (!access_ok(VERIFY_WRITE, addr, 4)) goto sigbus; compute_return_epc(regs); value = regs->regs[insn.i_format.rt]; StoreW(addr, value, res); if (res) goto fault; break; case sd_op: #ifdef CONFIG_64BIT /* * A 32-bit kernel might be running on a 64-bit processor. But * if we're on a 32-bit processor and an i-cache incoherency * or race makes us see a 64-bit instruction here the sdl/sdr * would blow up, so for now we don't handle unaligned 64-bit * instructions on 32-bit kernels. */ if (!access_ok(VERIFY_WRITE, addr, 8)) goto sigbus; compute_return_epc(regs); value = regs->regs[insn.i_format.rt]; StoreDW(addr, value, res); if (res) goto fault; break; #endif /* CONFIG_64BIT */ /* Cannot handle 64-bit instructions in 32-bit kernel */ goto sigill; case lwc1_op: case ldc1_op: case swc1_op: case sdc1_op: die_if_kernel("Unaligned FP access in kernel code", regs); BUG_ON(!used_math()); BUG_ON(!is_fpu_owner()); lose_fpu(1); /* Save FPU state for the emulator. */ res = fpu_emulator_cop1Handler(regs, ¤t->thread.fpu, 1, &fault_addr); own_fpu(1); /* Restore FPU state. */ /* Signal if something went wrong. */ process_fpemu_return(res, fault_addr); if (res == 0) break; return; /* * COP2 is available to implementor for application specific use. * It's up to applications to register a notifier chain and do * whatever they have to do, including possible sending of signals. */ case lwc2_op: cu2_notifier_call_chain(CU2_LWC2_OP, regs); break; case ldc2_op: cu2_notifier_call_chain(CU2_LDC2_OP, regs); break; case swc2_op: cu2_notifier_call_chain(CU2_SWC2_OP, regs); break; case sdc2_op: cu2_notifier_call_chain(CU2_SDC2_OP, regs); break; default: /* * Pheeee... We encountered an yet unknown instruction or * cache coherence problem. Die sucker, die ... */ goto sigill; } #ifdef CONFIG_DEBUG_FS unaligned_instructions++; #endif return; fault: /* roll back jump/branch */ regs->cp0_epc = origpc; regs->regs[31] = orig31; /* Did we have an exception handler installed? */ if (fixup_exception(regs)) return; die_if_kernel("Unhandled kernel unaligned access", regs); force_sig(SIGSEGV, current); return; sigbus: die_if_kernel("Unhandled kernel unaligned access", regs); force_sig(SIGBUS, current); return; sigill: die_if_kernel ("Unhandled kernel unaligned access or invalid instruction", regs); force_sig(SIGILL, current); }
static void emulate_load_store_microMIPS(struct pt_regs *regs, void __user *addr) { unsigned long value; unsigned int res; int i; unsigned int reg = 0, rvar; unsigned long orig31; u16 __user *pc16; u16 halfword; unsigned int word; unsigned long origpc, contpc; union mips_instruction insn; struct mm_decoded_insn mminsn; void __user *fault_addr = NULL; origpc = regs->cp0_epc; orig31 = regs->regs[31]; mminsn.micro_mips_mode = 1; /* * This load never faults. */ pc16 = (unsigned short __user *)msk_isa16_mode(regs->cp0_epc); __get_user(halfword, pc16); pc16++; contpc = regs->cp0_epc + 2; word = ((unsigned int)halfword << 16); mminsn.pc_inc = 2; if (!mm_insn_16bit(halfword)) { __get_user(halfword, pc16); pc16++; contpc = regs->cp0_epc + 4; mminsn.pc_inc = 4; word |= halfword; } mminsn.insn = word; if (get_user(halfword, pc16)) goto fault; mminsn.next_pc_inc = 2; word = ((unsigned int)halfword << 16); if (!mm_insn_16bit(halfword)) { pc16++; if (get_user(halfword, pc16)) goto fault; mminsn.next_pc_inc = 4; word |= halfword; } mminsn.next_insn = word; insn = (union mips_instruction)(mminsn.insn); if (mm_isBranchInstr(regs, mminsn, &contpc)) insn = (union mips_instruction)(mminsn.next_insn); /* Parse instruction to find what to do */ switch (insn.mm_i_format.opcode) { case mm_pool32a_op: switch (insn.mm_x_format.func) { case mm_lwxs_op: reg = insn.mm_x_format.rd; goto loadW; } goto sigbus; case mm_pool32b_op: switch (insn.mm_m_format.func) { case mm_lwp_func: reg = insn.mm_m_format.rd; if (reg == 31) goto sigbus; if (!access_ok(VERIFY_READ, addr, 8)) goto sigbus; LoadW(addr, value, res); if (res) goto fault; regs->regs[reg] = value; addr += 4; LoadW(addr, value, res); if (res) goto fault; regs->regs[reg + 1] = value; goto success; case mm_swp_func: reg = insn.mm_m_format.rd; if (reg == 31) goto sigbus; if (!access_ok(VERIFY_WRITE, addr, 8)) goto sigbus; value = regs->regs[reg]; StoreW(addr, value, res); if (res) goto fault; addr += 4; value = regs->regs[reg + 1]; StoreW(addr, value, res); if (res) goto fault; goto success; case mm_ldp_func: #ifdef CONFIG_64BIT reg = insn.mm_m_format.rd; if (reg == 31) goto sigbus; if (!access_ok(VERIFY_READ, addr, 16)) goto sigbus; LoadDW(addr, value, res); if (res) goto fault; regs->regs[reg] = value; addr += 8; LoadDW(addr, value, res); if (res) goto fault; regs->regs[reg + 1] = value; goto success; #endif /* CONFIG_64BIT */ goto sigill; case mm_sdp_func: #ifdef CONFIG_64BIT reg = insn.mm_m_format.rd; if (reg == 31) goto sigbus; if (!access_ok(VERIFY_WRITE, addr, 16)) goto sigbus; value = regs->regs[reg]; StoreDW(addr, value, res); if (res) goto fault; addr += 8; value = regs->regs[reg + 1]; StoreDW(addr, value, res); if (res) goto fault; goto success; #endif /* CONFIG_64BIT */ goto sigill; case mm_lwm32_func: reg = insn.mm_m_format.rd; rvar = reg & 0xf; if ((rvar > 9) || !reg) goto sigill; if (reg & 0x10) { if (!access_ok (VERIFY_READ, addr, 4 * (rvar + 1))) goto sigbus; } else { if (!access_ok(VERIFY_READ, addr, 4 * rvar)) goto sigbus; } if (rvar == 9) rvar = 8; for (i = 16; rvar; rvar--, i++) { LoadW(addr, value, res); if (res) goto fault; addr += 4; regs->regs[i] = value; } if ((reg & 0xf) == 9) { LoadW(addr, value, res); if (res) goto fault; addr += 4; regs->regs[30] = value; } if (reg & 0x10) { LoadW(addr, value, res); if (res) goto fault; regs->regs[31] = value; } goto success; case mm_swm32_func: reg = insn.mm_m_format.rd; rvar = reg & 0xf; if ((rvar > 9) || !reg) goto sigill; if (reg & 0x10) { if (!access_ok (VERIFY_WRITE, addr, 4 * (rvar + 1))) goto sigbus; } else { if (!access_ok(VERIFY_WRITE, addr, 4 * rvar)) goto sigbus; } if (rvar == 9) rvar = 8; for (i = 16; rvar; rvar--, i++) { value = regs->regs[i]; StoreW(addr, value, res); if (res) goto fault; addr += 4; } if ((reg & 0xf) == 9) { value = regs->regs[30]; StoreW(addr, value, res); if (res) goto fault; addr += 4; } if (reg & 0x10) { value = regs->regs[31]; StoreW(addr, value, res); if (res) goto fault; } goto success; case mm_ldm_func: #ifdef CONFIG_64BIT reg = insn.mm_m_format.rd; rvar = reg & 0xf; if ((rvar > 9) || !reg) goto sigill; if (reg & 0x10) { if (!access_ok (VERIFY_READ, addr, 8 * (rvar + 1))) goto sigbus; } else { if (!access_ok(VERIFY_READ, addr, 8 * rvar)) goto sigbus; } if (rvar == 9) rvar = 8; for (i = 16; rvar; rvar--, i++) { LoadDW(addr, value, res); if (res) goto fault; addr += 4; regs->regs[i] = value; } if ((reg & 0xf) == 9) { LoadDW(addr, value, res); if (res) goto fault; addr += 8; regs->regs[30] = value; } if (reg & 0x10) { LoadDW(addr, value, res); if (res) goto fault; regs->regs[31] = value; } goto success; #endif /* CONFIG_64BIT */ goto sigill; case mm_sdm_func: #ifdef CONFIG_64BIT reg = insn.mm_m_format.rd; rvar = reg & 0xf; if ((rvar > 9) || !reg) goto sigill; if (reg & 0x10) { if (!access_ok (VERIFY_WRITE, addr, 8 * (rvar + 1))) goto sigbus; } else { if (!access_ok(VERIFY_WRITE, addr, 8 * rvar)) goto sigbus; } if (rvar == 9) rvar = 8; for (i = 16; rvar; rvar--, i++) { value = regs->regs[i]; StoreDW(addr, value, res); if (res) goto fault; addr += 8; } if ((reg & 0xf) == 9) { value = regs->regs[30]; StoreDW(addr, value, res); if (res) goto fault; addr += 8; } if (reg & 0x10) { value = regs->regs[31]; StoreDW(addr, value, res); if (res) goto fault; } goto success; #endif /* CONFIG_64BIT */ goto sigill; /* LWC2, SWC2, LDC2, SDC2 are not serviced */ } goto sigbus; case mm_pool32c_op: switch (insn.mm_m_format.func) { case mm_lwu_func: reg = insn.mm_m_format.rd; goto loadWU; } /* LL,SC,LLD,SCD are not serviced */ goto sigbus; case mm_pool32f_op: switch (insn.mm_x_format.func) { case mm_lwxc1_func: case mm_swxc1_func: case mm_ldxc1_func: case mm_sdxc1_func: goto fpu_emul; } goto sigbus; case mm_ldc132_op: case mm_sdc132_op: case mm_lwc132_op: case mm_swc132_op: fpu_emul: /* roll back jump/branch */ regs->cp0_epc = origpc; regs->regs[31] = orig31; die_if_kernel("Unaligned FP access in kernel code", regs); BUG_ON(!used_math()); BUG_ON(!is_fpu_owner()); lose_fpu(1); /* save the FPU state for the emulator */ res = fpu_emulator_cop1Handler(regs, ¤t->thread.fpu, 1, &fault_addr); own_fpu(1); /* restore FPU state */ /* If something went wrong, signal */ process_fpemu_return(res, fault_addr); if (res == 0) goto success; return; case mm_lh32_op: reg = insn.mm_i_format.rt; goto loadHW; case mm_lhu32_op: reg = insn.mm_i_format.rt; goto loadHWU; case mm_lw32_op: reg = insn.mm_i_format.rt; goto loadW; case mm_sh32_op: reg = insn.mm_i_format.rt; goto storeHW; case mm_sw32_op: reg = insn.mm_i_format.rt; goto storeW; case mm_ld32_op: reg = insn.mm_i_format.rt; goto loadDW; case mm_sd32_op: reg = insn.mm_i_format.rt; goto storeDW; case mm_pool16c_op: switch (insn.mm16_m_format.func) { case mm_lwm16_op: reg = insn.mm16_m_format.rlist; rvar = reg + 1; if (!access_ok(VERIFY_READ, addr, 4 * rvar)) goto sigbus; for (i = 16; rvar; rvar--, i++) { LoadW(addr, value, res); if (res) goto fault; addr += 4; regs->regs[i] = value; } LoadW(addr, value, res); if (res) goto fault; regs->regs[31] = value; goto success; case mm_swm16_op: reg = insn.mm16_m_format.rlist; rvar = reg + 1; if (!access_ok(VERIFY_WRITE, addr, 4 * rvar)) goto sigbus; for (i = 16; rvar; rvar--, i++) { value = regs->regs[i]; StoreW(addr, value, res); if (res) goto fault; addr += 4; } value = regs->regs[31]; StoreW(addr, value, res); if (res) goto fault; goto success; } goto sigbus; case mm_lhu16_op: reg = reg16to32[insn.mm16_rb_format.rt]; goto loadHWU; case mm_lw16_op: reg = reg16to32[insn.mm16_rb_format.rt]; goto loadW; case mm_sh16_op: reg = reg16to32st[insn.mm16_rb_format.rt]; goto storeHW; case mm_sw16_op: reg = reg16to32st[insn.mm16_rb_format.rt]; goto storeW; case mm_lwsp16_op: reg = insn.mm16_r5_format.rt; goto loadW; case mm_swsp16_op: reg = insn.mm16_r5_format.rt; goto storeW; case mm_lwgp16_op: reg = reg16to32[insn.mm16_r3_format.rt]; goto loadW; default: goto sigill; } loadHW: if (!access_ok(VERIFY_READ, addr, 2)) goto sigbus; LoadHW(addr, value, res); if (res) goto fault; regs->regs[reg] = value; goto success; loadHWU: if (!access_ok(VERIFY_READ, addr, 2)) goto sigbus; LoadHWU(addr, value, res); if (res) goto fault; regs->regs[reg] = value; goto success; loadW: if (!access_ok(VERIFY_READ, addr, 4)) goto sigbus; LoadW(addr, value, res); if (res) goto fault; regs->regs[reg] = value; goto success; loadWU: #ifdef CONFIG_64BIT /* * A 32-bit kernel might be running on a 64-bit processor. But * if we're on a 32-bit processor and an i-cache incoherency * or race makes us see a 64-bit instruction here the sdl/sdr * would blow up, so for now we don't handle unaligned 64-bit * instructions on 32-bit kernels. */ if (!access_ok(VERIFY_READ, addr, 4)) goto sigbus; LoadWU(addr, value, res); if (res) goto fault; regs->regs[reg] = value; goto success; #endif /* CONFIG_64BIT */ /* Cannot handle 64-bit instructions in 32-bit kernel */ goto sigill; loadDW: #ifdef CONFIG_64BIT /* * A 32-bit kernel might be running on a 64-bit processor. But * if we're on a 32-bit processor and an i-cache incoherency * or race makes us see a 64-bit instruction here the sdl/sdr * would blow up, so for now we don't handle unaligned 64-bit * instructions on 32-bit kernels. */ if (!access_ok(VERIFY_READ, addr, 8)) goto sigbus; LoadDW(addr, value, res); if (res) goto fault; regs->regs[reg] = value; goto success; #endif /* CONFIG_64BIT */ /* Cannot handle 64-bit instructions in 32-bit kernel */ goto sigill; storeHW: if (!access_ok(VERIFY_WRITE, addr, 2)) goto sigbus; value = regs->regs[reg]; StoreHW(addr, value, res); if (res) goto fault; goto success; storeW: if (!access_ok(VERIFY_WRITE, addr, 4)) goto sigbus; value = regs->regs[reg]; StoreW(addr, value, res); if (res) goto fault; goto success; storeDW: #ifdef CONFIG_64BIT /* * A 32-bit kernel might be running on a 64-bit processor. But * if we're on a 32-bit processor and an i-cache incoherency * or race makes us see a 64-bit instruction here the sdl/sdr * would blow up, so for now we don't handle unaligned 64-bit * instructions on 32-bit kernels. */ if (!access_ok(VERIFY_WRITE, addr, 8)) goto sigbus; value = regs->regs[reg]; StoreDW(addr, value, res); if (res) goto fault; goto success; #endif /* CONFIG_64BIT */ /* Cannot handle 64-bit instructions in 32-bit kernel */ goto sigill; success: regs->cp0_epc = contpc; /* advance or branch */ #ifdef CONFIG_DEBUG_FS unaligned_instructions++; #endif return; fault: /* roll back jump/branch */ regs->cp0_epc = origpc; regs->regs[31] = orig31; /* Did we have an exception handler installed? */ if (fixup_exception(regs)) return; die_if_kernel("Unhandled kernel unaligned access", regs); force_sig(SIGSEGV, current); return; sigbus: die_if_kernel("Unhandled kernel unaligned access", regs); force_sig(SIGBUS, current); return; sigill: die_if_kernel ("Unhandled kernel unaligned access or invalid instruction", regs); force_sig(SIGILL, current); }
/* * This routine handles page faults. It determines the address, * and the problem, and then passes it off to one of the appropriate * routines. */ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long write, unsigned long address) { struct vm_area_struct * vma = NULL; struct task_struct *tsk = current; struct mm_struct *mm = tsk->mm; const int field = sizeof(unsigned long) * 2; siginfo_t info; #if 0 printk("Cpu%d[%s:%d:%0*lx:%ld:%0*lx]\n", smp_processor_id(), current->comm, current->pid, field, address, write, field, regs->cp0_epc); #endif info.si_code = SEGV_MAPERR; /* * We fault-in kernel-space virtual memory on-demand. The * 'reference' page table is init_mm.pgd. * * NOTE! We MUST NOT take any locks for this case. We may * be in an interrupt or a critical region, and should * only copy the information from the master page table, * nothing more. */ if (unlikely(address >= VMALLOC_START && address <= VMALLOC_END)) goto vmalloc_fault; /* * If we're in an interrupt or have no user * context, we must not take the fault.. */ if (in_atomic() || !mm) goto bad_area_nosemaphore; down_read(&mm->mmap_sem); vma = find_vma(mm, address); if (!vma) goto bad_area; if (vma->vm_start <= address) goto good_area; if (!(vma->vm_flags & VM_GROWSDOWN)) goto bad_area; if (expand_stack(vma, address)) goto bad_area; /* * Ok, we have a good vm_area for this memory access, so * we can handle it.. */ good_area: info.si_code = SEGV_ACCERR; if (write) { if (!(vma->vm_flags & VM_WRITE)) goto bad_area; } else { if (!(vma->vm_flags & (VM_READ | VM_WRITE | VM_EXEC))) goto bad_area; } survive: /* * If for any reason at all we couldn't handle the fault, * make sure we exit gracefully rather than endlessly redo * the fault. */ switch (handle_mm_fault(mm, vma, address, write)) { case VM_FAULT_MINOR: tsk->min_flt++; break; case VM_FAULT_MAJOR: tsk->maj_flt++; break; case VM_FAULT_SIGBUS: goto do_sigbus; case VM_FAULT_OOM: goto out_of_memory; default: BUG(); } up_read(&mm->mmap_sem); return; /* * Something tried to access memory that isn't in our memory map.. * Fix it, but check if it's kernel or user first.. */ bad_area: up_read(&mm->mmap_sem); bad_area_nosemaphore: /* User mode accesses just cause a SIGSEGV */ if (user_mode(regs)) { tsk->thread.cp0_badvaddr = address; tsk->thread.error_code = write; #if 0 printk("do_page_fault() #2: sending SIGSEGV to %s for " "invalid %s\n%0*lx (epc == %0*lx, ra == %0*lx)\n", tsk->comm, write ? "write access to" : "read access from", field, address, field, (unsigned long) regs->cp0_epc, field, (unsigned long) regs->regs[31]); #endif info.si_signo = SIGSEGV; info.si_errno = 0; /* info.si_code has been set above */ info.si_addr = (void __user *) address; force_sig_info(SIGSEGV, &info, tsk); return; } no_context: /* Are we prepared to handle this kernel fault? */ if (fixup_exception(regs)) { current->thread.cp0_baduaddr = address; return; } /* * Oops. The kernel tried to access some bad page. We'll have to * terminate things with extreme prejudice. */ bust_spinlocks(1); printk(KERN_ALERT "CPU %d Unable to handle kernel paging request at " "virtual address %0*lx, epc == %0*lx, ra == %0*lx\n", smp_processor_id(), field, address, field, regs->cp0_epc, field, regs->regs[31]); die("Oops", regs); /* * We ran out of memory, or some other thing happened to us that made * us unable to handle the page fault gracefully. */ out_of_memory: up_read(&mm->mmap_sem); if (is_init(tsk)) { yield(); down_read(&mm->mmap_sem); goto survive; } printk("VM: killing process %s\n", tsk->comm); if (user_mode(regs)) do_exit(SIGKILL); goto no_context; do_sigbus: up_read(&mm->mmap_sem); /* Kernel mode? Handle exceptions or die */ if (!user_mode(regs)) goto no_context; else /* * Send a sigbus, regardless of whether we were in kernel * or user mode. */ #if 0 printk("do_page_fault() #3: sending SIGBUS to %s for " "invalid %s\n%0*lx (epc == %0*lx, ra == %0*lx)\n", tsk->comm, write ? "write access to" : "read access from", field, address, field, (unsigned long) regs->cp0_epc, field, (unsigned long) regs->regs[31]); #endif tsk->thread.cp0_badvaddr = address; info.si_signo = SIGBUS; info.si_errno = 0; info.si_code = BUS_ADRERR; info.si_addr = (void __user *) address; force_sig_info(SIGBUS, &info, tsk); return; vmalloc_fault: { /* * Synchronize this task's top level page-table * with the 'reference' page table. * * Do _not_ use "tsk" here. We might be inside * an interrupt in the middle of a task switch.. */ int offset = __pgd_offset(address); pgd_t *pgd, *pgd_k; pud_t *pud, *pud_k; pmd_t *pmd, *pmd_k; pte_t *pte_k; pgd = (pgd_t *) pgd_current[smp_processor_id()] + offset; pgd_k = init_mm.pgd + offset; if (!pgd_present(*pgd_k)) goto no_context; set_pgd(pgd, *pgd_k); pud = pud_offset(pgd, address); pud_k = pud_offset(pgd_k, address); if (!pud_present(*pud_k)) goto no_context; pmd = pmd_offset(pud, address); pmd_k = pmd_offset(pud_k, address); if (!pmd_present(*pmd_k)) goto no_context; set_pmd(pmd, *pmd_k); pte_k = pte_offset_kernel(pmd_k, address); if (!pte_present(*pte_k)) goto no_context; return; } }
/* * This routine handles page faults. It determines the address, * and the problem, and then passes it off to one of the appropriate * routines. */ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long write, unsigned long address) { struct vm_area_struct * vma; struct task_struct *tsk = current; struct mm_struct *mm = tsk->mm; unsigned long fixup; siginfo_t info; /* * We fault-in kernel-space virtual memory on-demand. The * 'reference' page table is init_mm.pgd. * * NOTE! We MUST NOT take any locks for this case. We may * be in an interrupt or a critical region, and should * only copy the information from the master page table, * nothing more. */ if (address >= VMALLOC_START) goto vmalloc_fault; info.si_code = SEGV_MAPERR; /* * If we're in an interrupt or have no user * context, we must not take the fault.. */ if (in_interrupt() || !mm) goto no_context; #if 0 printk("[%s:%d:%08lx:%ld:%08lx]\n", current->comm, current->pid, address, write, regs->cp0_epc); #endif down_read(&mm->mmap_sem); vma = find_vma(mm, address); if (!vma) goto bad_area; if (vma->vm_start <= address) goto good_area; if (expand_stack(vma, address)) goto bad_area; /* * Ok, we have a good vm_area for this memory access, so * we can handle it.. */ good_area: info.si_code = SEGV_ACCERR; if (write) { if (!(vma->vm_flags & VM_WRITE)) goto bad_area; } else { if (!(vma->vm_flags & (VM_READ | VM_EXEC))) goto bad_area; } survive: /* * If for any reason at all we couldn't handle the fault, * make sure we exit gracefully rather than endlessly redo * the fault. */ switch (handle_mm_fault(mm, vma, address, write)) { case 1: tsk->min_flt++; break; case 2: tsk->maj_flt++; break; case 0: goto do_sigbus; default: goto out_of_memory; } up_read(&mm->mmap_sem); return; /* * Something tried to access memory that isn't in our memory map.. * Fix it, but check if it's kernel or user first.. */ bad_area: up_read(&mm->mmap_sem); bad_area_nosemaphore: /* User mode accesses just cause a SIGSEGV */ if (user_mode(regs)) { tsk->thread.cp0_badvaddr = address; tsk->thread.error_code = write; #if 0 printk("do_page_fault() #2: sending SIGSEGV to %s for illegal %s\n" "%08lx (epc == %08lx, ra == %08lx)\n", tsk->comm, write ? "write access to" : "read access from", address, (unsigned long) regs->cp0_epc, (unsigned long) regs->regs[31]); #endif info.si_signo = SIGSEGV; info.si_errno = 0; /* info.si_code has been set above */ info.si_addr = (void *) address; force_sig_info(SIGSEGV, &info, tsk); return; } no_context: /* Are we prepared to handle this kernel fault? */ fixup = search_exception_table(exception_epc(regs)); if (fixup) { long new_epc; tsk->thread.cp0_baduaddr = address; new_epc = fixup_exception(dpf_reg, fixup, regs->cp0_epc); if (development_version) printk(KERN_DEBUG "%s: Exception at [<%lx>] (%lx)\n", tsk->comm, regs->cp0_epc, new_epc); regs->cp0_epc = new_epc; return; } /* * Oops. The kernel tried to access some bad page. We'll have to * terminate things with extreme prejudice. */ printk(KERN_ALERT "Unable to handle kernel paging request at virtual " "address %08lx, epc == %08lx, ra == %08lx\n", address, regs->cp0_epc, regs->regs[31]); die("Oops", regs); /* Game over. */ /* * We ran out of memory, or some other thing happened to us that made * us unable to handle the page fault gracefully. */ out_of_memory: if (tsk->pid == 1) { yield(); goto survive; } up_read(&mm->mmap_sem); printk(KERN_NOTICE "VM: killing process %s\n", tsk->comm); if (user_mode(regs)) do_exit(SIGKILL); goto no_context; do_sigbus: up_read(&mm->mmap_sem); /* * Send a sigbus, regardless of whether we were in kernel * or user mode. */ tsk->thread.cp0_badvaddr = address; info.si_signo = SIGBUS; info.si_errno = 0; info.si_code = BUS_ADRERR; info.si_addr = (void *) address; force_sig_info(SIGBUS, &info, tsk); /* Kernel mode? Handle exceptions or die */ if (!user_mode(regs)) goto no_context; return; vmalloc_fault: { /* * Synchronize this task's top level page-table * with the 'reference' page table. */ int offset = pgd_index(address); pgd_t *pgd, *pgd_k; pmd_t *pmd, *pmd_k; pgd = tsk->active_mm->pgd + offset; pgd_k = init_mm.pgd + offset; if (!pgd_present(*pgd)) { if (!pgd_present(*pgd_k)) goto bad_area_nosemaphore; set_pgd(pgd, *pgd_k); return; } pmd = pmd_offset(pgd, address); pmd_k = pmd_offset(pgd_k, address); if (pmd_present(*pmd) || !pmd_present(*pmd_k)) goto bad_area_nosemaphore; set_pmd(pmd, *pmd_k); } }
/* * This routine handles page faults. It determines the address, * and the problem, and then passes it off to one of the appropriate * routines. */ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long write, unsigned long mmu_meh) { struct vm_area_struct *vma = NULL; struct task_struct *tsk = current; struct mm_struct *mm = tsk->mm; int si_code; int fault; unsigned long address = mmu_meh & PAGE_MASK; si_code = SEGV_MAPERR; #ifndef CONFIG_CPU_HAS_TLBI /* * We fault-in kernel-space virtual memory on-demand. The * 'reference' page table is init_mm.pgd. * * NOTE! We MUST NOT take any locks for this case. We may * be in an interrupt or a critical region, and should * only copy the information from the master page table, * nothing more. */ if (unlikely(address >= VMALLOC_START) && unlikely(address <= VMALLOC_END)) { /* * Synchronize this task's top level page-table * with the 'reference' page table. * * Do _not_ use "tsk" here. We might be inside * an interrupt in the middle of a task switch.. */ int offset = __pgd_offset(address); pgd_t *pgd, *pgd_k; pud_t *pud, *pud_k; pmd_t *pmd, *pmd_k; pte_t *pte_k; unsigned long pgd_base; pgd_base = tlb_get_pgd(); pgd = (pgd_t *)pgd_base + offset; pgd_k = init_mm.pgd + offset; if (!pgd_present(*pgd_k)) goto no_context; set_pgd(pgd, *pgd_k); pud = (pud_t *)pgd; pud_k = (pud_t *)pgd_k; if (!pud_present(*pud_k)) goto no_context; pmd = pmd_offset(pud, address); pmd_k = pmd_offset(pud_k, address); if (!pmd_present(*pmd_k)) goto no_context; set_pmd(pmd, *pmd_k); pte_k = pte_offset_kernel(pmd_k, address); if (!pte_present(*pte_k)) goto no_context; return; } #endif /* * If we're in an interrupt or have no user * context, we must not take the fault.. */ if (in_atomic() || !mm) goto bad_area_nosemaphore; down_read(&mm->mmap_sem); vma = find_vma(mm, address); if (!vma) goto bad_area; if (vma->vm_start <= address) goto good_area; if (!(vma->vm_flags & VM_GROWSDOWN)) goto bad_area; if (expand_stack(vma, address)) goto bad_area; /* * Ok, we have a good vm_area for this memory access, so * we can handle it.. */ good_area: si_code = SEGV_ACCERR; if (write) { if (!(vma->vm_flags & VM_WRITE)) goto bad_area; } else { if (!(vma->vm_flags & (VM_READ | VM_WRITE | VM_EXEC))) goto bad_area; } /* * If for any reason at all we couldn't handle the fault, * make sure we exit gracefully rather than endlessly redo * the fault. */ fault = handle_mm_fault(vma, address, write ? FAULT_FLAG_WRITE : 0); if (unlikely(fault & VM_FAULT_ERROR)) { if (fault & VM_FAULT_OOM) goto out_of_memory; else if (fault & VM_FAULT_SIGBUS) goto do_sigbus; else if (fault & VM_FAULT_SIGSEGV) goto bad_area; BUG(); } if (fault & VM_FAULT_MAJOR) tsk->maj_flt++; else tsk->min_flt++; up_read(&mm->mmap_sem); return; /* * Something tried to access memory that isn't in our memory map.. * Fix it, but check if it's kernel or user first.. */ bad_area: up_read(&mm->mmap_sem); bad_area_nosemaphore: /* User mode accesses just cause a SIGSEGV */ if (user_mode(regs)) { force_sig_fault(SIGSEGV, si_code, (void __user *)address, current); return; } no_context: /* Are we prepared to handle this kernel fault? */ if (fixup_exception(regs)) return; /* * Oops. The kernel tried to access some bad page. We'll have to * terminate things with extreme prejudice. */ bust_spinlocks(1); pr_alert("Unable to handle kernel paging request at virtual " "address 0x%08lx, pc: 0x%08lx\n", address, regs->pc); die_if_kernel("Oops", regs, write); out_of_memory: /* * We ran out of memory, call the OOM killer, and return the userspace * (which will retry the fault, or kill us if we got oom-killed). */ pagefault_out_of_memory(); return; do_sigbus: up_read(&mm->mmap_sem); /* Kernel mode? Handle exceptions or die */ if (!user_mode(regs)) goto no_context; force_sig_fault(SIGBUS, BUS_ADRERR, (void __user *)address, current); }
dotraplinkage void __kprobes do_general_protection(struct pt_regs *regs, long error_code) { struct task_struct *tsk; conditional_sti(regs); #ifdef CONFIG_X86_32 if (regs->flags & X86_VM_MASK) goto gp_in_vm86; #endif tsk = current; if (!user_mode(regs)) goto gp_in_kernel; #ifdef CONFIG_X86_32 { int cpu; int ok; cpu = get_cpu(); ok = check_lazy_exec_limit(cpu, regs, error_code); put_cpu(); if (ok) return; if (print_fatal_signals) { printk(KERN_ERR "#GPF(%ld[seg:%lx]) at %08lx, CPU#%d.\n", error_code, error_code/8, regs->ip, smp_processor_id()); printk(KERN_ERR "exec_limit: %08lx, user_cs: %08x/%08x.\n", current->mm->context.exec_limit, current->mm->context.user_cs.a, current->mm->context.user_cs.b); } } #endif /*CONFIG_X86_32*/ tsk->thread.error_code = error_code; tsk->thread.trap_no = 13; if (show_unhandled_signals && unhandled_signal(tsk, SIGSEGV) && printk_ratelimit()) { printk(KERN_INFO "%s[%d] general protection ip:%lx sp:%lx error:%lx", tsk->comm, task_pid_nr(tsk), regs->ip, regs->sp, error_code); print_vma_addr(" in ", regs->ip); printk("\n"); } force_sig(SIGSEGV, tsk); return; #ifdef CONFIG_X86_32 gp_in_vm86: local_irq_enable(); handle_vm86_fault((struct kernel_vm86_regs *) regs, error_code); return; #endif gp_in_kernel: if (fixup_exception(regs)) return; tsk->thread.error_code = error_code; tsk->thread.trap_no = 13; if (notify_die(DIE_GPF, "general protection fault", regs, error_code, 13, SIGSEGV) == NOTIFY_STOP) return; die("general protection fault", regs, error_code); }