void do_page_fault(struct pt_regs *regs, int fault_num, unsigned long address, unsigned long write) { enum ctx_state prev_state = exception_enter(); __do_page_fault(regs, fault_num, address, write); exception_exit(prev_state); }
dotraplinkage void do_page_fault(struct pt_regs *regs, long error_code) { unsigned long address = read_cr2(); __do_page_fault(regs, error_code, address); printk("BUG: unable to handle kernel paging request at %#lx\n", address); printk("CPU: %d PID: %u Comm: %s\n", smp_processor_id(), current->pid, current->comm); show_regs(regs); panic("panic at #PF"); }
static int __kprobes do_page_fault(unsigned long addr, unsigned int esr, struct pt_regs *regs) { struct task_struct *tsk; struct mm_struct *mm; int fault, sig, code; unsigned long vm_flags = VM_READ | VM_WRITE | VM_EXEC; unsigned int mm_flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; tsk = current; mm = tsk->mm; /* Enable interrupts if they were enabled in the parent context. */ if (interrupts_enabled(regs)) local_irq_enable(); /* * If we're in an interrupt or have no user context, we must not take * the fault. */ if (in_atomic() || !mm) goto no_context; if (user_mode(regs)) mm_flags |= FAULT_FLAG_USER; if (esr & ESR_LNX_EXEC) { vm_flags = VM_EXEC; } else if ((esr & ESR_EL1_WRITE) && !(esr & ESR_EL1_CM)) { vm_flags = VM_WRITE; mm_flags |= FAULT_FLAG_WRITE; } /* * As per x86, we may deadlock here. However, since the kernel only * validly references user space from well defined areas of the code, * we can bug out early if this is from code which shouldn't. */ if (!down_read_trylock(&mm->mmap_sem)) { if (!user_mode(regs) && !search_exception_tables(regs->pc)) goto no_context; retry: down_read(&mm->mmap_sem); } else { /* * The above down_read_trylock() might have succeeded in which * case, we'll have missed the might_sleep() from down_read(). */ might_sleep(); #ifdef CONFIG_DEBUG_VM if (!user_mode(regs) && !search_exception_tables(regs->pc)) goto no_context; #endif } fault = __do_page_fault(mm, addr, mm_flags, vm_flags, tsk); /* * If we need to retry but a fatal signal is pending, handle the * signal first. We do not need to release the mmap_sem because it * would already be released in __lock_page_or_retry in mm/filemap.c. */ if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) return 0; /* * Major/minor page fault accounting is only done on the initial * attempt. If we go through a retry, it is extremely likely that the * page will be found in page cache at that point. */ perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, addr); if (mm_flags & FAULT_FLAG_ALLOW_RETRY) { if (fault & VM_FAULT_MAJOR) { tsk->maj_flt++; perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, regs, addr); } else { tsk->min_flt++; perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, regs, addr); } if (fault & VM_FAULT_RETRY) { /* * Clear FAULT_FLAG_ALLOW_RETRY to avoid any risk of * starvation. */ mm_flags &= ~FAULT_FLAG_ALLOW_RETRY; mm_flags |= FAULT_FLAG_TRIED; goto retry; } } up_read(&mm->mmap_sem); /* * Handle the "normal" case first - VM_FAULT_MAJOR / VM_FAULT_MINOR */ if (likely(!(fault & (VM_FAULT_ERROR | VM_FAULT_BADMAP | VM_FAULT_BADACCESS)))) return 0; /* * If we are in kernel mode at this point, we have no context to * handle this fault with. */ if (!user_mode(regs)) goto no_context; if (fault & VM_FAULT_OOM) { /* * We ran out of memory, call the OOM killer, and return to * userspace (which will retry the fault, or kill us if we got * oom-killed). */ pagefault_out_of_memory(); return 0; } if (fault & VM_FAULT_SIGBUS) { /* * We had some memory, but were unable to successfully fix up * this page fault. */ sig = SIGBUS; code = BUS_ADRERR; } else { /* * Something tried to access memory that isn't in our memory * map. */ sig = SIGSEGV; code = fault == VM_FAULT_BADACCESS ? SEGV_ACCERR : SEGV_MAPERR; } __do_user_fault(tsk, addr, esr, sig, code, regs); return 0; no_context: __do_kernel_fault(mm, addr, esr, regs); return 0; }
static int do_page_fault(unsigned long addr, int mode, struct pt_regs *regs) { struct task_struct *tsk; struct mm_struct *mm; unsigned long fixup; int fault; tsk = current; mm = tsk->mm; /* * We fault-in kernel-space virtual memory on-demand. The * 'reference' page table is init_mm.pgd. * * NOTE! We MUST NOT take any locks for this case. We may * be in an interrupt or a critical region, and should * only copy the information from the master page table, * nothing more. */ if (addr >= TASK_SIZE) goto vmalloc_fault; /* * If we're in an interrupt or have no user * context, we must not take the fault.. */ if (in_interrupt() || !mm) goto no_context; down(&mm->mmap_sem); fault = __do_page_fault(mm, addr, mode, tsk); up(&mm->mmap_sem); ret: /* * Handle the "normal" case first */ if (fault > 0) return 0; /* * We had some memory, but were unable to * successfully fix up this page fault. */ if (fault == 0) goto do_sigbus; /* * If we are in kernel mode at this point, we * have no context to handle this fault with. */ if (!user_mode(regs)) goto no_context; if (fault == -3) { /* * We ran out of memory, or some other thing happened to * us that made us unable to handle the page fault gracefully. */ printk("VM: killing process %s\n", tsk->comm); do_exit(SIGKILL); } else { /* * Something tried to access memory that isn't in our memory map.. * User mode accesses just cause a SIGSEGV */ struct siginfo si; #ifdef CONFIG_DEBUG_USER printk(KERN_DEBUG "%s: unhandled page fault at pc=0x%08lx, " "lr=0x%08lx (bad address=0x%08lx, code %d)\n", tsk->comm, regs->ARM_pc, regs->ARM_lr, addr, mode); #endif tsk->thread.address = addr; tsk->thread.error_code = mode; tsk->thread.trap_no = 14; si.si_signo = SIGSEGV; si.si_errno = 0; si.si_code = fault == -1 ? SEGV_ACCERR : SEGV_MAPERR; si.si_addr = (void *)addr; force_sig_info(SIGSEGV, &si, tsk); } return 0; /* * We ran out of memory, or some other thing happened to us that made * us unable to handle the page fault gracefully. */ do_sigbus: /* * Send a sigbus, regardless of whether we were in kernel * or user mode. */ tsk->thread.address = addr; tsk->thread.error_code = mode; tsk->thread.trap_no = 14; force_sig(SIGBUS, tsk); /* Kernel mode? Handle exceptions or die */ if (user_mode(regs)) return 0; no_context: /* Are we prepared to handle this kernel fault? */ if ((fixup = search_exception_table(instruction_pointer(regs))) != 0) { #ifdef DEBUG printk(KERN_DEBUG "%s: Exception at [<%lx>] addr=%lx (fixup: %lx)\n", tsk->comm, regs->ARM_pc, addr, fixup); #endif regs->ARM_pc = fixup; return 0; } /* * Oops. The kernel tried to access some bad page. We'll have to * terminate things with extreme prejudice. */ printk(KERN_ALERT "Unable to handle kernel %s at virtual address %08lx\n", (addr < PAGE_SIZE) ? "NULL pointer dereference" : "paging request", addr); show_pte(mm, addr); die("Oops", regs, mode); do_exit(SIGKILL); return 0; vmalloc_fault: fault = __do_vmalloc_fault(addr, mm); goto ret; }
int do_page_fault(unsigned long addr, int error_code, struct pt_regs *regs) { struct task_struct *tsk; struct mm_struct *mm; int fault; #if defined(CONFIG_KGDB) if (kgdb_fault_expected) kgdb_handle_bus_error(); #endif tsk = current; mm = tsk->mm; /* * If we're in an interrupt or have no user * context, we must not take the fault.. */ if (in_interrupt() || !mm) goto no_context; TRACE_TRAP_ENTRY(14, instruction_pointer(regs)); down_read(&mm->mmap_sem); fault = __do_page_fault(mm, addr, error_code, tsk); up_read(&mm->mmap_sem); TRACE_EVENT(TRACE_EV_TRAP_EXIT, NULL); /* * Handle the "normal" case first */ if (fault > 0) return 0; /* * We had some memory, but were unable to * successfully fix up this page fault. */ if (fault == 0) goto do_sigbus; /* * If we are in kernel mode at this point, we * have no context to handle this fault with. */ if (!user_mode(regs)) goto no_context; if (fault == -3) { /* * We ran out of memory, or some other thing happened to * us that made us unable to handle the page fault gracefully. */ printk("VM: killing process %s\n", tsk->comm); do_exit(SIGKILL); } else __do_user_fault(tsk, addr, error_code, fault == -1 ? SEGV_ACCERR : SEGV_MAPERR, regs); return 0; /* * We ran out of memory, or some other thing happened to us that made * us unable to handle the page fault gracefully. */ do_sigbus: /* * Send a sigbus, regardless of whether we were in kernel * or user mode. */ tsk->thread.address = addr; tsk->thread.error_code = error_code; tsk->thread.trap_no = 14; force_sig(SIGBUS, tsk); #ifdef CONFIG_DEBUG_USER printk(KERN_DEBUG "%s: sigbus at 0x%08lx, pc=0x%08lx\n", current->comm, addr, instruction_pointer(regs)); #endif /* Kernel mode? Handle exceptions or die */ if (user_mode(regs)) return 0; no_context: __do_kernel_fault(mm, addr, error_code, regs); return 0; }
static int __kprobes do_page_fault(unsigned long addr, unsigned int esr, struct pt_regs *regs) { struct task_struct *tsk; struct mm_struct *mm; int fault, sig, code; unsigned long vm_flags = VM_READ | VM_WRITE | VM_EXEC; unsigned int mm_flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; tsk = current; mm = tsk->mm; if (interrupts_enabled(regs)) local_irq_enable(); if (in_atomic() || !mm) goto no_context; if (user_mode(regs)) mm_flags |= FAULT_FLAG_USER; if (esr & ESR_LNX_EXEC) { vm_flags = VM_EXEC; } else if (esr & ESR_EL1_WRITE) { vm_flags = VM_WRITE; mm_flags |= FAULT_FLAG_WRITE; } if (!down_read_trylock(&mm->mmap_sem)) { if (!user_mode(regs) && !search_exception_tables(regs->pc)) goto no_context; retry: down_read(&mm->mmap_sem); } else { might_sleep(); #ifdef CONFIG_DEBUG_VM if (!user_mode(regs) && !search_exception_tables(regs->pc)) goto no_context; #endif } fault = __do_page_fault(mm, addr, mm_flags, vm_flags, tsk); if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) return 0; perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, addr); if (mm_flags & FAULT_FLAG_ALLOW_RETRY) { if (fault & VM_FAULT_MAJOR) { tsk->maj_flt++; perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, regs, addr); } else { tsk->min_flt++; perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, regs, addr); } if (fault & VM_FAULT_RETRY) { mm_flags &= ~FAULT_FLAG_ALLOW_RETRY; mm_flags |= FAULT_FLAG_TRIED; goto retry; } } up_read(&mm->mmap_sem); if (likely(!(fault & (VM_FAULT_ERROR | VM_FAULT_BADMAP | VM_FAULT_BADACCESS)))) return 0; if (!user_mode(regs)) goto no_context; if (fault & VM_FAULT_OOM) { pagefault_out_of_memory(); return 0; } if (fault & VM_FAULT_SIGBUS) { sig = SIGBUS; code = BUS_ADRERR; } else { sig = SIGSEGV; code = fault == VM_FAULT_BADACCESS ? SEGV_ACCERR : SEGV_MAPERR; } __do_user_fault(tsk, addr, esr, sig, code, regs); return 0; no_context: __do_kernel_fault(mm, addr, esr, regs); return 0; }