void do_page_fault(struct pt_regs *regs, int fault_num,
unsigned long address, unsigned long write)
{
enum ctx_state prev_state = exception_enter();
__do_page_fault(regs, fault_num, address, write);
exception_exit(prev_state);
}
dotraplinkage void do_page_fault(struct pt_regs *regs, long error_code)
{
unsigned long address = read_cr2();
__do_page_fault(regs, error_code, address);
printk("BUG: unable to handle kernel paging request at %#lx\n", address);
printk("CPU: %d PID: %u Comm: %s\n", smp_processor_id(), current->pid, current->comm);
show_regs(regs);
panic("panic at #PF");
}
static int __kprobes do_page_fault(unsigned long addr, unsigned int esr,
struct pt_regs *regs)
{
struct task_struct *tsk;
struct mm_struct *mm;
int fault, sig, code;
unsigned long vm_flags = VM_READ | VM_WRITE | VM_EXEC;
unsigned int mm_flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
tsk = current;
mm = tsk->mm;
/* Enable interrupts if they were enabled in the parent context. */
if (interrupts_enabled(regs))
local_irq_enable();
/*
* If we're in an interrupt or have no user context, we must not take
* the fault.
*/
if (in_atomic() || !mm)
goto no_context;
if (user_mode(regs))
mm_flags |= FAULT_FLAG_USER;
if (esr & ESR_LNX_EXEC) {
vm_flags = VM_EXEC;
} else if ((esr & ESR_EL1_WRITE) && !(esr & ESR_EL1_CM)) {
vm_flags = VM_WRITE;
mm_flags |= FAULT_FLAG_WRITE;
}
/*
* As per x86, we may deadlock here. However, since the kernel only
* validly references user space from well defined areas of the code,
* we can bug out early if this is from code which shouldn't.
*/
if (!down_read_trylock(&mm->mmap_sem)) {
if (!user_mode(regs) && !search_exception_tables(regs->pc))
goto no_context;
retry:
down_read(&mm->mmap_sem);
} else {
/*
* The above down_read_trylock() might have succeeded in which
* case, we'll have missed the might_sleep() from down_read().
*/
might_sleep();
#ifdef CONFIG_DEBUG_VM
if (!user_mode(regs) && !search_exception_tables(regs->pc))
goto no_context;
#endif
}
fault = __do_page_fault(mm, addr, mm_flags, vm_flags, tsk);
/*
* If we need to retry but a fatal signal is pending, handle the
* signal first. We do not need to release the mmap_sem because it
* would already be released in __lock_page_or_retry in mm/filemap.c.
*/
if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current))
return 0;
/*
* Major/minor page fault accounting is only done on the initial
* attempt. If we go through a retry, it is extremely likely that the
* page will be found in page cache at that point.
*/
perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, addr);
if (mm_flags & FAULT_FLAG_ALLOW_RETRY) {
if (fault & VM_FAULT_MAJOR) {
tsk->maj_flt++;
perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, regs,
addr);
} else {
tsk->min_flt++;
perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, regs,
addr);
}
if (fault & VM_FAULT_RETRY) {
/*
* Clear FAULT_FLAG_ALLOW_RETRY to avoid any risk of
* starvation.
*/
mm_flags &= ~FAULT_FLAG_ALLOW_RETRY;
mm_flags |= FAULT_FLAG_TRIED;
goto retry;
}
}
up_read(&mm->mmap_sem);
/*
* Handle the "normal" case first - VM_FAULT_MAJOR / VM_FAULT_MINOR
*/
if (likely(!(fault & (VM_FAULT_ERROR | VM_FAULT_BADMAP |
VM_FAULT_BADACCESS))))
return 0;
/*
* If we are in kernel mode at this point, we have no context to
* handle this fault with.
*/
if (!user_mode(regs))
goto no_context;
if (fault & VM_FAULT_OOM) {
/*
* We ran out of memory, call the OOM killer, and return to
* userspace (which will retry the fault, or kill us if we got
* oom-killed).
*/
pagefault_out_of_memory();
return 0;
}
if (fault & VM_FAULT_SIGBUS) {
/*
* We had some memory, but were unable to successfully fix up
* this page fault.
*/
sig = SIGBUS;
code = BUS_ADRERR;
} else {
/*
* Something tried to access memory that isn't in our memory
* map.
*/
sig = SIGSEGV;
code = fault == VM_FAULT_BADACCESS ?
SEGV_ACCERR : SEGV_MAPERR;
}
__do_user_fault(tsk, addr, esr, sig, code, regs);
return 0;
no_context:
__do_kernel_fault(mm, addr, esr, regs);
return 0;
}
static int do_page_fault(unsigned long addr, int mode, struct pt_regs *regs)
{
struct task_struct *tsk;
struct mm_struct *mm;
unsigned long fixup;
int fault;
tsk = current;
mm = tsk->mm;
/*
* We fault-in kernel-space virtual memory on-demand. The
* 'reference' page table is init_mm.pgd.
*
* NOTE! We MUST NOT take any locks for this case. We may
* be in an interrupt or a critical region, and should
* only copy the information from the master page table,
* nothing more.
*/
if (addr >= TASK_SIZE)
goto vmalloc_fault;
/*
* If we're in an interrupt or have no user
* context, we must not take the fault..
*/
if (in_interrupt() || !mm)
goto no_context;
down(&mm->mmap_sem);
fault = __do_page_fault(mm, addr, mode, tsk);
up(&mm->mmap_sem);
ret:
/*
* Handle the "normal" case first
*/
if (fault > 0)
return 0;
/*
* We had some memory, but were unable to
* successfully fix up this page fault.
*/
if (fault == 0)
goto do_sigbus;
/*
* If we are in kernel mode at this point, we
* have no context to handle this fault with.
*/
if (!user_mode(regs))
goto no_context;
if (fault == -3) {
/*
* We ran out of memory, or some other thing happened to
* us that made us unable to handle the page fault gracefully.
*/
printk("VM: killing process %s\n", tsk->comm);
do_exit(SIGKILL);
} else {
/*
* Something tried to access memory that isn't in our memory map..
* User mode accesses just cause a SIGSEGV
*/
struct siginfo si;
#ifdef CONFIG_DEBUG_USER
printk(KERN_DEBUG "%s: unhandled page fault at pc=0x%08lx, "
"lr=0x%08lx (bad address=0x%08lx, code %d)\n",
tsk->comm, regs->ARM_pc, regs->ARM_lr, addr, mode);
#endif
tsk->thread.address = addr;
tsk->thread.error_code = mode;
tsk->thread.trap_no = 14;
si.si_signo = SIGSEGV;
si.si_errno = 0;
si.si_code = fault == -1 ? SEGV_ACCERR : SEGV_MAPERR;
si.si_addr = (void *)addr;
force_sig_info(SIGSEGV, &si, tsk);
}
return 0;
/*
* We ran out of memory, or some other thing happened to us that made
* us unable to handle the page fault gracefully.
*/
do_sigbus:
/*
* Send a sigbus, regardless of whether we were in kernel
* or user mode.
*/
tsk->thread.address = addr;
tsk->thread.error_code = mode;
tsk->thread.trap_no = 14;
force_sig(SIGBUS, tsk);
/* Kernel mode? Handle exceptions or die */
if (user_mode(regs))
return 0;
no_context:
/* Are we prepared to handle this kernel fault? */
if ((fixup = search_exception_table(instruction_pointer(regs))) != 0) {
#ifdef DEBUG
printk(KERN_DEBUG "%s: Exception at [<%lx>] addr=%lx (fixup: %lx)\n",
tsk->comm, regs->ARM_pc, addr, fixup);
#endif
regs->ARM_pc = fixup;
return 0;
}
/*
* Oops. The kernel tried to access some bad page. We'll have to
* terminate things with extreme prejudice.
*/
printk(KERN_ALERT "Unable to handle kernel %s at virtual address %08lx\n",
(addr < PAGE_SIZE) ? "NULL pointer dereference" : "paging request", addr);
show_pte(mm, addr);
die("Oops", regs, mode);
do_exit(SIGKILL);
return 0;
vmalloc_fault:
fault = __do_vmalloc_fault(addr, mm);
goto ret;
}
int do_page_fault(unsigned long addr, int error_code, struct pt_regs *regs)
{
struct task_struct *tsk;
struct mm_struct *mm;
int fault;
#if defined(CONFIG_KGDB)
if (kgdb_fault_expected)
kgdb_handle_bus_error();
#endif
tsk = current;
mm = tsk->mm;
/*
* If we're in an interrupt or have no user
* context, we must not take the fault..
*/
if (in_interrupt() || !mm)
goto no_context;
TRACE_TRAP_ENTRY(14, instruction_pointer(regs));
down_read(&mm->mmap_sem);
fault = __do_page_fault(mm, addr, error_code, tsk);
up_read(&mm->mmap_sem);
TRACE_EVENT(TRACE_EV_TRAP_EXIT, NULL);
/*
* Handle the "normal" case first
*/
if (fault > 0)
return 0;
/*
* We had some memory, but were unable to
* successfully fix up this page fault.
*/
if (fault == 0)
goto do_sigbus;
/*
* If we are in kernel mode at this point, we
* have no context to handle this fault with.
*/
if (!user_mode(regs))
goto no_context;
if (fault == -3) {
/*
* We ran out of memory, or some other thing happened to
* us that made us unable to handle the page fault gracefully.
*/
printk("VM: killing process %s\n", tsk->comm);
do_exit(SIGKILL);
} else
__do_user_fault(tsk, addr, error_code, fault == -1 ?
SEGV_ACCERR : SEGV_MAPERR, regs);
return 0;
/*
* We ran out of memory, or some other thing happened to us that made
* us unable to handle the page fault gracefully.
*/
do_sigbus:
/*
* Send a sigbus, regardless of whether we were in kernel
* or user mode.
*/
tsk->thread.address = addr;
tsk->thread.error_code = error_code;
tsk->thread.trap_no = 14;
force_sig(SIGBUS, tsk);
#ifdef CONFIG_DEBUG_USER
printk(KERN_DEBUG "%s: sigbus at 0x%08lx, pc=0x%08lx\n",
current->comm, addr, instruction_pointer(regs));
#endif
/* Kernel mode? Handle exceptions or die */
if (user_mode(regs))
return 0;
no_context:
__do_kernel_fault(mm, addr, error_code, regs);
return 0;
}
static int __kprobes do_page_fault(unsigned long addr, unsigned int esr,
struct pt_regs *regs)
{
struct task_struct *tsk;
struct mm_struct *mm;
int fault, sig, code;
unsigned long vm_flags = VM_READ | VM_WRITE | VM_EXEC;
unsigned int mm_flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
tsk = current;
mm = tsk->mm;
if (interrupts_enabled(regs))
local_irq_enable();
if (in_atomic() || !mm)
goto no_context;
if (user_mode(regs))
mm_flags |= FAULT_FLAG_USER;
if (esr & ESR_LNX_EXEC) {
vm_flags = VM_EXEC;
} else if (esr & ESR_EL1_WRITE) {
vm_flags = VM_WRITE;
mm_flags |= FAULT_FLAG_WRITE;
}
if (!down_read_trylock(&mm->mmap_sem)) {
if (!user_mode(regs) && !search_exception_tables(regs->pc))
goto no_context;
retry:
down_read(&mm->mmap_sem);
} else {
might_sleep();
#ifdef CONFIG_DEBUG_VM
if (!user_mode(regs) && !search_exception_tables(regs->pc))
goto no_context;
#endif
}
fault = __do_page_fault(mm, addr, mm_flags, vm_flags, tsk);
if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current))
return 0;
perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, addr);
if (mm_flags & FAULT_FLAG_ALLOW_RETRY) {
if (fault & VM_FAULT_MAJOR) {
tsk->maj_flt++;
perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, regs,
addr);
} else {
tsk->min_flt++;
perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, regs,
addr);
}
if (fault & VM_FAULT_RETRY) {
mm_flags &= ~FAULT_FLAG_ALLOW_RETRY;
mm_flags |= FAULT_FLAG_TRIED;
goto retry;
}
}
up_read(&mm->mmap_sem);
if (likely(!(fault & (VM_FAULT_ERROR | VM_FAULT_BADMAP |
VM_FAULT_BADACCESS))))
return 0;
if (!user_mode(regs))
goto no_context;
if (fault & VM_FAULT_OOM) {
pagefault_out_of_memory();
return 0;
}
if (fault & VM_FAULT_SIGBUS) {
sig = SIGBUS;
code = BUS_ADRERR;
} else {
sig = SIGSEGV;
code = fault == VM_FAULT_BADACCESS ?
SEGV_ACCERR : SEGV_MAPERR;
}
__do_user_fault(tsk, addr, esr, sig, code, regs);
return 0;
no_context:
__do_kernel_fault(mm, addr, esr, regs);
return 0;
}
