void do_ip27_dbe(struct pt_regs *regs)
{
unsigned long fixup, errst0, errst1;
int cpu = LOCAL_HUB_L(PI_CPU_NUM);
fixup = search_dbe_table(regs->cp0_epc);
if (fixup) {
long new_epc;
new_epc = fixup_exception(dpf_reg, fixup, regs->cp0_epc);
regs->cp0_epc = new_epc;
return;
}
printk("Slice %c got dbe at 0x%lx\n", 'A' + cpu, regs->cp0_epc);
printk("Hub information:\n");
printk("ERR_INT_PEND = 0x%06lx\n", LOCAL_HUB_L(PI_ERR_INT_PEND));
errst0 = LOCAL_HUB_L(cpu ? PI_ERR_STATUS0_B : PI_ERR_STATUS0_A);
errst1 = LOCAL_HUB_L(cpu ? PI_ERR_STATUS1_B : PI_ERR_STATUS1_A);
dump_hub_information(errst0, errst1);
show_regs(regs);
dump_tlb_all();
while(1);
force_sig(SIGBUS, current);
}
static void __do_kernel_fault(struct mm_struct *mm, unsigned long addr,
unsigned int esr, struct pt_regs *regs)
{
#if defined(CONFIG_HTC_DEBUG_RTB)
static int enable_logk_die = 1;
#endif
if (fixup_exception(regs))
return;
#if defined(CONFIG_HTC_DEBUG_RTB)
if (enable_logk_die) {
uncached_logk(LOGK_DIE, (void *)regs->pc);
uncached_logk(LOGK_DIE, (void *)regs->regs[30]);
uncached_logk(LOGK_DIE, (void *)addr);
msm_rtb_disable();
enable_logk_die = 0;
}
#endif
bust_spinlocks(1);
pr_alert("Unable to handle kernel %s at virtual address %08lx\n",
(addr < PAGE_SIZE) ? "NULL pointer dereference" :
"paging request", addr);
show_pte(mm, addr);
die("Oops", regs, esr);
bust_spinlocks(0);
do_exit(SIGKILL);
}
void
do_page_fault(struct pt_regs *regs, unsigned int vector)
{
/* Kernel space exception fixup check */
if (fixup_exception(regs))
return;
unsigned long cr2 = read_cr2();
#ifdef CONFIG_KGDB
kgdb_breakpoint();
#endif
struct siginfo s;
memset(&s, 0, sizeof(struct siginfo));
s.si_signo = SIGSEGV;
s.si_errno = 0;
s.si_addr = (void *)cr2;
s.si_code = SEGV_ACCERR; // SEGV_MAPERR;
int i = sigsend(current->aspace->id, ANY_ID, SIGSEGV, &s);
printk(">> PAGE FAULT! Sent signal %d. CR2 is %p, cpu %d\n", i, (void *)cr2, this_cpu);
show_registers(regs);
/* If the fault occurred in kernel space, or the init_task, go down */
if ( (regs->rip >= PAGE_OFFSET) ||
(current->aspace->id == INIT_ASPACE_ID))
{
local_irq_disable();
halt();
}
}
/*
* The kernel tried to access some page that wasn't present.
*/
static void __do_kernel_fault(struct mm_struct *mm, unsigned long addr,
unsigned int esr, struct pt_regs *regs)
{
#if defined(CONFIG_HTC_DEBUG_RTB)
static int enable_logk_die = 1;
#endif
/*
* Are we prepared to handle this kernel fault?
*/
if (fixup_exception(regs))
return;
#if defined(CONFIG_HTC_DEBUG_RTB)
if (enable_logk_die) {
uncached_logk(LOGK_DIE, (void *)regs->pc);
uncached_logk(LOGK_DIE, (void *)regs->regs[30]);
uncached_logk(LOGK_DIE, (void *)addr);
/* Disable RTB here to avoid weird recursive spinlock/printk behaviors */
msm_rtb_disable();
enable_logk_die = 0;
}
#endif
/*
* No handler, we'll have to terminate things with extreme prejudice.
*/
bust_spinlocks(1);
pr_alert("Unable to handle kernel %s at virtual address %08lx\n",
(addr < PAGE_SIZE) ? "NULL pointer dereference" :
"paging request", addr);
show_pte(mm, addr);
die("Oops", regs, esr);
bust_spinlocks(0);
do_exit(SIGKILL);
}
int send_fault_sig(struct pt_regs *regs)
{
int signo, si_code;
void __user *addr;
signo = current->thread.signo;
si_code = current->thread.code;
addr = (void __user *)current->thread.faddr;
pr_debug("send_fault_sig: %p,%d,%d\n", addr, signo, si_code);
if (user_mode(regs)) {
force_sig_fault(signo, si_code, addr, current);
} else {
if (fixup_exception(regs))
return -1;
//if (signo == SIGBUS)
// force_sig_fault(si_signo, si_code, addr, current);
/*
* Oops. The kernel tried to access some bad page. We'll have to
* terminate things with extreme prejudice.
*/
if ((unsigned long)addr < PAGE_SIZE)
pr_alert("Unable to handle kernel NULL pointer dereference");
else
pr_alert("Unable to handle kernel access");
pr_cont(" at virtual address %p\n", addr);
die_if_kernel("Oops", regs, 0 /*error_code*/);
do_exit(SIGKILL);
}
return 1;
}
void
do_page_fault(struct pt_regs *regs, unsigned int vector)
{
/* Kernel space exception fixup check */
if (fixup_exception(regs))
return;
unsigned long cr2 = read_cr2();
#ifdef CONFIG_KGDB
kgdb_breakpoint();
#endif
struct siginfo s;
memset(&s, 0, sizeof(struct siginfo));
s.si_signo = SIGSEGV;
s.si_errno = 0;
s.si_addr = (void *)cr2;
s.si_code = SEGV_ACCERR; // SEGV_MAPERR;
int i = sigsend(current->aspace->id, ANY_ID, SIGSEGV, &s);
printk(">> PAGE FAULT! Sent signal %d: CR2 is %p\n", i, s.si_addr);
show_registers(regs);
while (1) {}
}
dotraplinkage void __kprobes
do_general_protection(struct pt_regs *regs, long error_code)
{
struct task_struct *tsk;
conditional_sti(regs);
#ifdef CONFIG_X86_32
if (lazy_iobitmap_copy()) {
/* restart the faulting instruction */
return;
}
if (regs->flags & X86_VM_MASK)
goto gp_in_vm86;
#endif
tsk = current;
if (!user_mode(regs))
goto gp_in_kernel;
tsk->thread.error_code = error_code;
tsk->thread.trap_no = 13;
if (show_unhandled_signals && unhandled_signal(tsk, SIGSEGV) &&
printk_ratelimit()) {
printk(KERN_INFO
"%s[%d] general protection ip:%lx sp:%lx error:%lx",
tsk->comm, task_pid_nr(tsk),
regs->ip, regs->sp, error_code);
print_vma_addr(" in ", regs->ip);
printk("\n");
}
force_sig(SIGSEGV, tsk);
return;
#ifdef CONFIG_X86_32
gp_in_vm86:
local_irq_enable();
handle_vm86_fault((struct kernel_vm86_regs *) regs, error_code);
return;
#endif
gp_in_kernel:
if (fixup_exception(regs))
return;
tsk->thread.error_code = error_code;
tsk->thread.trap_no = 13;
if (notify_die(DIE_GPF, "general protection fault", regs,
error_code, 13, SIGSEGV) == NOTIFY_STOP)
return;
die("general protection fault", regs, error_code);
}
static int kernel_math_error(struct pt_regs *regs, const char *str, int trapnr)
{
if (fixup_exception(regs))
return 1;
notify_die(DIE_GPF, str, regs, 0, trapnr, SIGFPE);
/* Illegal floating point operation in the kernel */
current->thread.trap_no = trapnr;
die(str, regs, 0);
return 0;
}
void do_dbe(struct pt_regs *regs)
{
unsigned long fixup;
fixup = search_dbe_table(regs->cp0_epc);
if (fixup) {
long new_epc;
new_epc = fixup_exception(dpf_reg, fixup, regs->cp0_epc);
regs->cp0_epc = new_epc;
return;
}
printk("Got dbe at 0x%lx\n", regs->cp0_epc);
show_regs(regs);
dump_tlb_all();
while(1);
force_sig(SIGBUS, current);
}
static __inline__ void do_trap(int trapnr, int signr, const char * str,
struct pt_regs * regs, long error_code, siginfo_t *info)
{
if (user_mode(regs)) {
/* trap_signal */
struct task_struct *tsk = current;
tsk->thread.error_code = error_code;
tsk->thread.trap_no = trapnr;
if (info)
force_sig_info(signr, info, tsk);
else
force_sig(signr, tsk);
return;
} else {
/* kernel_trap */
if (!fixup_exception(regs))
die(str, regs, error_code);
return;
}
}
/*
* Set up exception handlers for tracing and breakpoints
*/
void handle_exception(struct pt_regs *regs)
{
int trap = (regs->cp0_cause & 0x7c) >> 2;
if (fixup_exception(regs)) {
return;
}
if (atomic_read(&debugger_active))
kgdb_nmihook(smp_processor_id(), regs);
if (atomic_read(&kgdb_setting_breakpoint))
if ((trap == 9) && (regs->cp0_epc == (unsigned long)breakinst))
regs->cp0_epc += 4;
kgdb_handle_exception(0, compute_signal(trap), 0, regs);
/* In SMP mode, __flush_cache_all does IPI */
__flush_cache_all();
}
/*
* The kernel tried to access some page that wasn't present.
*/
static void __do_kernel_fault(struct mm_struct *mm, unsigned long addr,
unsigned int esr, struct pt_regs *regs)
{
/*
* Are we prepared to handle this kernel fault?
*/
if (fixup_exception(regs))
return;
/*
* No handler, we'll have to terminate things with extreme prejudice.
*/
bust_spinlocks(1);
pr_alert("Unable to handle kernel %s at virtual address %08lx\n",
(addr < PAGE_SIZE) ? "NULL pointer dereference" :
"paging request", addr);
show_pte(mm, addr);
die("Oops", regs, esr);
bust_spinlocks(0);
do_exit(SIGKILL);
}
/* Alignment exception handler */
asmlinkage void handle_unaligned_c(struct pt_regs *fp, int cause)
{
unsigned long addr = RDCTL(CTL_BADADDR);
cause >>= 2;
fp->ea -= 4;
if (fixup_exception(fp))
return;
if (!user_mode(fp)) {
pr_alert("Unaligned access from kernel mode, this might be a hardware\n");
pr_alert("problem, dump registers and restart the instruction\n");
pr_alert(" BADADDR 0x%08lx\n", addr);
pr_alert(" cause %d\n", cause);
pr_alert(" op-code 0x%08lx\n", *(unsigned long *)(fp->ea));
show_regs(fp);
return;
}
_exception(SIGBUS, fp, BUS_ADRALN, addr);
}
static inline void
emulate_load_store_insn(struct pt_regs *regs,
unsigned long addr,
unsigned long pc)
{
union mips_instruction insn;
unsigned long value, fixup;
regs->regs[0] = 0;
/*
* This load never faults.
*/
__get_user(insn.word, (unsigned int *)pc);
switch (insn.i_format.opcode) {
/*
* These are instructions that a compiler doesn't generate. We
* can assume therefore that the code is MIPS-aware and
* really buggy. Emulating these instructions would break the
* semantics anyway.
*/
case ll_op:
case lld_op:
case sc_op:
case scd_op:
/*
* For these instructions the only way to create an address
* error is an attempted access to kernel/supervisor address
* space.
*/
case ldl_op:
case ldr_op:
case lwl_op:
case lwr_op:
case sdl_op:
case sdr_op:
case swl_op:
case swr_op:
case lb_op:
case lbu_op:
case sb_op:
goto sigbus;
/*
* The remaining opcodes are the ones that are really of interest.
*/
case lh_op:
check_axs(pc, addr, 2);
__asm__(
".set\tnoat\n"
#ifdef __BIG_ENDIAN
"1:\tlb\t%0,0(%1)\n"
"2:\tlbu\t$1,1(%1)\n\t"
#endif
#ifdef __LITTLE_ENDIAN
"1:\tlb\t%0,1(%1)\n"
"2:\tlbu\t$1,0(%1)\n\t"
#endif
"sll\t%0,0x8\n\t"
"or\t%0,$1\n\t"
".set\tat\n\t"
".section\t__ex_table,\"a\"\n\t"
STR(PTR)"\t1b,%2\n\t"
STR(PTR)"\t2b,%2\n\t"
".previous"
:"=&r" (value)
:"r" (addr), "i" (&&fault)
:"$1");
regs->regs[insn.i_format.rt] = value;
return;
case lw_op:
check_axs(pc, addr, 4);
__asm__(
#ifdef __BIG_ENDIAN
"1:\tlwl\t%0,(%1)\n"
"2:\tlwr\t%0,3(%1)\n\t"
#endif
#ifdef __LITTLE_ENDIAN
"1:\tlwl\t%0,3(%1)\n"
"2:\tlwr\t%0,(%1)\n\t"
#endif
".section\t__ex_table,\"a\"\n\t"
STR(PTR)"\t1b,%2\n\t"
STR(PTR)"\t2b,%2\n\t"
".previous"
:"=&r" (value)
:"r" (addr), "i" (&&fault));
regs->regs[insn.i_format.rt] = value;
return;
case lhu_op:
check_axs(pc, addr, 2);
__asm__(
".set\tnoat\n"
#ifdef __BIG_ENDIAN
"1:\tlbu\t%0,0(%1)\n"
"2:\tlbu\t$1,1(%1)\n\t"
#endif
#ifdef __LITTLE_ENDIAN
"1:\tlbu\t%0,1(%1)\n"
"2:\tlbu\t$1,0(%1)\n\t"
#endif
"sll\t%0,0x8\n\t"
"or\t%0,$1\n\t"
".set\tat\n\t"
".section\t__ex_table,\"a\"\n\t"
STR(PTR)"\t1b,%2\n\t"
STR(PTR)"\t2b,%2\n\t"
".previous"
:"=&r" (value)
:"r" (addr), "i" (&&fault)
:"$1");
regs->regs[insn.i_format.rt] = value;
return;
case lwu_op:
check_axs(pc, addr, 4);
__asm__(
#ifdef __BIG_ENDIAN
"1:\tlwl\t%0,(%1)\n"
"2:\tlwr\t%0,3(%1)\n\t"
#endif
#ifdef __LITTLE_ENDIAN
"1:\tlwl\t%0,3(%1)\n"
"2:\tlwr\t%0,(%1)\n\t"
#endif
".section\t__ex_table,\"a\"\n\t"
STR(PTR)"\t1b,%2\n\t"
STR(PTR)"\t2b,%2\n\t"
".previous"
:"=&r" (value)
:"r" (addr), "i" (&&fault));
value &= 0xffffffff;
regs->regs[insn.i_format.rt] = value;
return;
case ld_op:
check_axs(pc, addr, 8);
__asm__(
".set\tmips3\n"
#ifdef __BIG_ENDIAN
"1:\tldl\t%0,(%1)\n"
"2:\tldr\t%0,7(%1)\n\t"
#endif
#ifdef __LITTLE_ENDIAN
"1:\tldl\t%0,7(%1)\n"
"2:\tldr\t%0,(%1)\n\t"
#endif
".set\tmips0\n\t"
".section\t__ex_table,\"a\"\n\t"
STR(PTR)"\t1b,%2\n\t"
STR(PTR)"\t2b,%2\n\t"
".previous"
:"=&r" (value)
:"r" (addr), "i" (&&fault));
regs->regs[insn.i_format.rt] = value;
return;
case sh_op:
check_axs(pc, addr, 2);
value = regs->regs[insn.i_format.rt];
__asm__(
#ifdef __BIG_ENDIAN
".set\tnoat\n"
"1:\tsb\t%0,1(%1)\n\t"
"srl\t$1,%0,0x8\n"
"2:\tsb\t$1,0(%1)\n\t"
".set\tat\n\t"
#endif
#ifdef __LITTLE_ENDIAN
".set\tnoat\n"
"1:\tsb\t%0,0(%1)\n\t"
"srl\t$1,%0,0x8\n"
"2:\tsb\t$1,1(%1)\n\t"
".set\tat\n\t"
#endif
".section\t__ex_table,\"a\"\n\t"
STR(PTR)"\t1b,%2\n\t"
STR(PTR)"\t2b,%2\n\t"
".previous"
: /* no outputs */
:"r" (value), "r" (addr), "i" (&&fault)
:"$1");
return;
case sw_op:
check_axs(pc, addr, 4);
value = regs->regs[insn.i_format.rt];
__asm__(
#ifdef __BIG_ENDIAN
"1:\tswl\t%0,(%1)\n"
"2:\tswr\t%0,3(%1)\n\t"
#endif
#ifdef __LITTLE_ENDIAN
"1:\tswl\t%0,3(%1)\n"
"2:\tswr\t%0,(%1)\n\t"
#endif
".section\t__ex_table,\"a\"\n\t"
STR(PTR)"\t1b,%2\n\t"
STR(PTR)"\t2b,%2\n\t"
".previous"
: /* no outputs */
:"r" (value), "r" (addr), "i" (&&fault));
return;
case sd_op:
check_axs(pc, addr, 8);
value = regs->regs[insn.i_format.rt];
__asm__(
".set\tmips3\n"
#ifdef __BIG_ENDIAN
"1:\tsdl\t%0,(%1)\n"
"2:\tsdr\t%0,7(%1)\n\t"
#endif
#ifdef __LITTLE_ENDIAN
"1:\tsdl\t%0,7(%1)\n"
"2:\tsdr\t%0,(%1)\n\t"
#endif
".set\tmips0\n\t"
".section\t__ex_table,\"a\"\n\t"
STR(PTR)"\t1b,%2\n\t"
STR(PTR)"\t2b,%2\n\t"
".previous"
: /* no outputs */
:"r" (value), "r" (addr), "i" (&&fault));
return;
case lwc1_op:
case ldc1_op:
case swc1_op:
case sdc1_op:
/*
* I herewith declare: this does not happen. So send SIGBUS.
*/
goto sigbus;
case lwc2_op:
case ldc2_op:
case swc2_op:
case sdc2_op:
/*
* These are the coprocessor 2 load/stores. The current
* implementations don't use cp2 and cp2 should always be
* disabled in c0_status. So send SIGILL.
* (No longer true: The Sony Praystation uses cp2 for
* 3D matrix operations. Dunno if that thingy has a MMU ...)
*/
default:
/*
* Pheeee... We encountered an yet unknown instruction or
* cache coherence problem. Die sucker, die ...
*/
goto sigill;
}
return;
fault:
/* Did we have an exception handler installed? */
fixup = search_exception_table(regs->cp0_epc);
if (fixup) {
long new_epc;
new_epc = fixup_exception(dpf_reg, fixup, regs->cp0_epc);
printk(KERN_DEBUG "%s: Forwarding exception at [<%lx>] (%lx)\n",
current->comm, regs->cp0_epc, new_epc);
regs->cp0_epc = new_epc;
return;
}
lock_kernel();
send_sig(SIGSEGV, current, 1);
unlock_kernel();
return;
sigbus:
lock_kernel();
send_sig(SIGBUS, current, 1);
unlock_kernel();
return;
sigill:
lock_kernel();
send_sig(SIGILL, current, 1);
unlock_kernel();
return;
}
static void __kprobes
do_trap(int trapnr, int signr, const char *str, struct pt_regs *regs,
long error_code, siginfo_t *info)
{
struct task_struct *tsk = current;
#ifdef CONFIG_X86_32
if (v8086_mode(regs)) {
/*
* traps 0, 1, 3, 4, and 5 should be forwarded to vm86.
* On nmi (interrupt 2), do_trap should not be called.
*/
if (trapnr < 6)
goto vm86_trap;
goto trap_signal;
}
#endif
if (!user_mode_novm(regs))
goto kernel_trap;
#ifdef CONFIG_X86_32
trap_signal:
#endif
/*
* We want error_code and trap_no set for userspace faults and
* kernelspace faults which result in die(), but not
* kernelspace faults which are fixed up. die() gives the
* process no chance to handle the signal and notice the
* kernel fault information, so that won't result in polluting
* the information about previously queued, but not yet
* delivered, faults. See also do_general_protection below.
*/
tsk->thread.error_code = error_code;
tsk->thread.trap_no = trapnr;
#ifdef CONFIG_X86_64
if (show_unhandled_signals && unhandled_signal(tsk, signr) &&
printk_ratelimit()) {
printk(KERN_INFO
"%s[%d] trap %s ip:%lx sp:%lx error:%lx",
tsk->comm, task_pid_nr(tsk), str,
regs->ip, regs->sp, error_code);
print_vma_addr(" in ", regs->ip);
printk("\n");
}
#endif
if (info)
force_sig_info(signr, info, tsk);
else
force_sig(signr, tsk);
return;
kernel_trap:
if (!fixup_exception(regs)) {
tsk->thread.error_code = error_code;
tsk->thread.trap_no = trapnr;
#if defined(CONFIG_X86_32) && defined(CONFIG_PAX_KERNEXEC)
if (trapnr == 12 && ((regs->cs & 0xFFFF) == __KERNEL_CS || (regs->cs & 0xFFFF) == __KERNEXEC_KERNEL_CS))
str = "PAX: suspicious stack segment fault";
#endif
die(str, regs, error_code);
}
#ifdef CONFIG_PAX_REFCOUNT
if (trapnr == 4)
pax_report_refcount_overflow(regs);
#endif
return;
#ifdef CONFIG_X86_32
vm86_trap:
if (handle_vm86_trap((struct kernel_vm86_regs *) regs,
error_code, trapnr))
goto trap_signal;
return;
#endif
}
/*
* This routine handles page faults. It determines the address and the
* problem, and then passes it off to one of the appropriate routines.
*/
asmlinkage void do_page_fault(struct pt_regs *regs)
{
struct task_struct *tsk;
struct vm_area_struct *vma;
struct mm_struct *mm;
unsigned long addr, cause;
unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
int fault, code = SEGV_MAPERR;
cause = regs->scause;
addr = regs->sbadaddr;
tsk = current;
mm = tsk->mm;
/*
* Fault-in kernel-space virtual memory on-demand.
* The 'reference' page table is init_mm.pgd.
*
* NOTE! We MUST NOT take any locks for this case. We may
* be in an interrupt or a critical region, and should
* only copy the information from the master page table,
* nothing more.
*/
if (unlikely((addr >= VMALLOC_START) && (addr <= VMALLOC_END)))
goto vmalloc_fault;
/* Enable interrupts if they were enabled in the parent context. */
if (likely(regs->sstatus & SR_PIE))
local_irq_enable();
/*
* If we're in an interrupt, have no user context, or are running
* in an atomic region, then we must not take the fault.
*/
if (unlikely(in_atomic() || !mm))
goto no_context;
if (user_mode(regs))
flags |= FAULT_FLAG_USER;
perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, addr);
retry:
down_read(&mm->mmap_sem);
vma = find_vma(mm, addr);
if (unlikely(!vma))
goto bad_area;
if (likely(vma->vm_start <= addr))
goto good_area;
if (unlikely(!(vma->vm_flags & VM_GROWSDOWN)))
goto bad_area;
if (unlikely(expand_stack(vma, addr)))
goto bad_area;
/*
* Ok, we have a good vm_area for this memory access, so
* we can handle it.
*/
good_area:
code = SEGV_ACCERR;
switch (cause) {
case EXC_INST_ACCESS:
if (!(vma->vm_flags & VM_EXEC))
goto bad_area;
break;
case EXC_LOAD_ACCESS:
if (!(vma->vm_flags & VM_READ))
goto bad_area;
break;
case EXC_STORE_ACCESS:
if (!(vma->vm_flags & VM_WRITE))
goto bad_area;
flags |= FAULT_FLAG_WRITE;
break;
default:
panic("%s: unhandled cause %lu", __func__, cause);
}
/*
* If for any reason at all we could not handle the fault,
* make sure we exit gracefully rather than endlessly redo
* the fault.
*/
fault = handle_mm_fault(mm, vma, addr, flags);
/*
* If we need to retry but a fatal signal is pending, handle the
* signal first. We do not need to release the mmap_sem because it
* would already be released in __lock_page_or_retry in mm/filemap.c.
*/
if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(tsk))
return;
if (unlikely(fault & VM_FAULT_ERROR)) {
if (fault & VM_FAULT_OOM)
goto out_of_memory;
else if (fault & VM_FAULT_SIGBUS)
goto do_sigbus;
BUG();
}
/*
* Major/minor page fault accounting is only done on the
* initial attempt. If we go through a retry, it is extremely
* likely that the page will be found in page cache at that point.
*/
if (flags & FAULT_FLAG_ALLOW_RETRY) {
if (fault & VM_FAULT_MAJOR) {
tsk->maj_flt++;
perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, regs, addr);
} else {
tsk->min_flt++;
perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, regs, addr);
}
if (fault & VM_FAULT_RETRY) {
/*
* Clear FAULT_FLAG_ALLOW_RETRY to avoid any risk
* of starvation.
*/
flags &= ~(FAULT_FLAG_ALLOW_RETRY);
flags |= FAULT_FLAG_TRIED;
/*
* No need to up_read(&mm->mmap_sem) as we would
* have already released it in __lock_page_or_retry
* in mm/filemap.c.
*/
goto retry;
}
}
up_read(&mm->mmap_sem);
return;
/*
* Something tried to access memory that isn't in our memory map.
* Fix it, but check if it's kernel or user first.
*/
bad_area:
up_read(&mm->mmap_sem);
/* User mode accesses just cause a SIGSEGV */
if (user_mode(regs)) {
do_trap(regs, SIGSEGV, code, addr, tsk);
return;
}
no_context:
/* Are we prepared to handle this kernel fault? */
if (fixup_exception(regs)) {
return;
}
/*
* Oops. The kernel tried to access some bad page. We'll have to
* terminate things with extreme prejudice.
*/
bust_spinlocks(1);
pr_alert("Unable to handle kernel %s at virtual address " REG_FMT "\n",
(addr < PAGE_SIZE) ? "NULL pointer dereference" :
"paging request", addr);
die(regs, "Oops");
do_exit(SIGKILL);
/*
* We ran out of memory, call the OOM killer, and return the userspace
* (which will retry the fault, or kill us if we got oom-killed).
*/
out_of_memory:
up_read(&mm->mmap_sem);
if (!user_mode(regs))
goto no_context;
pagefault_out_of_memory();
return;
do_sigbus:
up_read(&mm->mmap_sem);
/* Kernel mode? Handle exceptions or die */
if (!user_mode(regs))
goto no_context;
do_trap(regs, SIGBUS, BUS_ADRERR, addr, tsk);
return;
vmalloc_fault:
{
pgd_t *pgd, *pgd_k;
pud_t *pud, *pud_k;
pmd_t *pmd, *pmd_k;
pte_t *pte_k;
int index;
if (user_mode(regs))
goto bad_area;
/*
* Synchronize this task's top level page-table
* with the 'reference' page table.
*
* Do _not_ use "tsk->active_mm->pgd" here.
* We might be inside an interrupt in the middle
* of a task switch.
*/
index = pgd_index(addr);
pgd = (pgd_t *)pfn_to_virt(csr_read(sptbr)) + index;
pgd_k = init_mm.pgd + index;
if (!pgd_present(*pgd_k))
goto no_context;
set_pgd(pgd, *pgd_k);
pud = pud_offset(pgd, addr);
pud_k = pud_offset(pgd_k, addr);
if (!pud_present(*pud_k))
goto no_context;
/* Since the vmalloc area is global, it is unnecessary
to copy individual PTEs */
pmd = pmd_offset(pud, addr);
pmd_k = pmd_offset(pud_k, addr);
if (!pmd_present(*pmd_k))
goto no_context;
set_pmd(pmd, *pmd_k);
/* Make sure the actual PTE exists as well to
* catch kernel vmalloc-area accesses to non-mapped
* addresses. If we don't do this, this will just
* silently loop forever.
*/
pte_k = pte_offset_kernel(pmd_k, addr);
if (!pte_present(*pte_k))
goto no_context;
return;
}
}
static void __kprobes
do_trap(int trapnr, int signr, char *str, struct pt_regs *regs,
long error_code, siginfo_t *info)
{
struct task_struct *tsk = current;
#ifdef CONFIG_X86_32
if (regs->flags & X86_VM_MASK) {
/*
* traps 0, 1, 3, 4, and 5 should be forwarded to vm86.
* On nmi (interrupt 2), do_trap should not be called.
*/
if (trapnr < 6)
goto vm86_trap;
goto trap_signal;
}
#endif
if (!user_mode(regs))
goto kernel_trap;
#ifdef CONFIG_X86_32
trap_signal:
#endif
/*
* We want error_code and trap_no set for userspace faults and
* kernelspace faults which result in die(), but not
* kernelspace faults which are fixed up. die() gives the
* process no chance to handle the signal and notice the
* kernel fault information, so that won't result in polluting
* the information about previously queued, but not yet
* delivered, faults. See also do_general_protection below.
*/
tsk->thread.error_code = error_code;
tsk->thread.trap_no = trapnr;
#ifdef CONFIG_X86_64
if (show_unhandled_signals && unhandled_signal(tsk, signr) &&
printk_ratelimit()) {
printk(KERN_INFO
"%s[%d] trap %s ip:%lx sp:%lx error:%lx",
tsk->comm, tsk->pid, str,
regs->ip, regs->sp, error_code);
print_vma_addr(" in ", regs->ip);
printk("\n");
}
#endif
if (info)
force_sig_info(signr, info, tsk);
else
force_sig(signr, tsk);
return;
kernel_trap:
if (!fixup_exception(regs)) {
tsk->thread.error_code = error_code;
tsk->thread.trap_no = trapnr;
die(str, regs, error_code);
}
return;
#ifdef CONFIG_X86_32
vm86_trap:
if (handle_vm86_trap((struct kernel_vm86_regs *) regs,
error_code, trapnr))
goto trap_signal;
return;
#endif
}
/*
* This routine handles page faults. It determines the address,
* and the problem, and then passes it off to one of the appropriate
* routines.
*/
asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long cause,
unsigned long address)
{
struct vm_area_struct *vma = NULL;
struct task_struct *tsk = current;
struct mm_struct *mm = tsk->mm;
int code = SEGV_MAPERR;
int fault;
unsigned int flags = 0;
cause >>= 2;
/* Restart the instruction */
regs->ea -= 4;
/*
* We fault-in kernel-space virtual memory on-demand. The
* 'reference' page table is init_mm.pgd.
*
* NOTE! We MUST NOT take any locks for this case. We may
* be in an interrupt or a critical region, and should
* only copy the information from the master page table,
* nothing more.
*/
if (unlikely(address >= VMALLOC_START && address <= VMALLOC_END)) {
if (user_mode(regs))
goto bad_area_nosemaphore;
else
goto vmalloc_fault;
}
if (unlikely(address >= TASK_SIZE))
goto bad_area_nosemaphore;
/*
* If we're in an interrupt or have no user
* context, we must not take the fault..
*/
if (in_atomic() || !mm)
goto bad_area_nosemaphore;
down_read(&mm->mmap_sem);
vma = find_vma(mm, address);
if (!vma)
goto bad_area;
if (vma->vm_start <= address)
goto good_area;
if (!(vma->vm_flags & VM_GROWSDOWN))
goto bad_area;
if (expand_stack(vma, address))
goto bad_area;
/*
* Ok, we have a good vm_area for this memory access, so
* we can handle it..
*/
good_area:
code = SEGV_ACCERR;
switch (cause) {
case EXC_SUPERV_INSN_ACCESS:
goto bad_area;
case EXC_SUPERV_DATA_ACCESS:
goto bad_area;
case EXC_X_PROTECTION_FAULT:
if (!(vma->vm_flags & VM_EXEC))
goto bad_area;
break;
case EXC_R_PROTECTION_FAULT:
if (!(vma->vm_flags & VM_READ))
goto bad_area;
break;
case EXC_W_PROTECTION_FAULT:
flags = FAULT_FLAG_WRITE;
if (!(vma->vm_flags & VM_WRITE))
goto bad_area;
break;
}
survive:
/*
* If for any reason at all we couldn't handle the fault,
* make sure we exit gracefully rather than endlessly redo
* the fault.
*/
fault = handle_mm_fault(mm, vma, address, flags);
if (unlikely(fault & VM_FAULT_ERROR)) {
if (fault & VM_FAULT_OOM)
goto out_of_memory;
else if (fault & VM_FAULT_SIGBUS)
goto do_sigbus;
BUG();
}
if (fault & VM_FAULT_MAJOR)
tsk->maj_flt++;
else
tsk->min_flt++;
up_read(&mm->mmap_sem);
return;
/*
* Something tried to access memory that isn't in our memory map..
* Fix it, but check if it's kernel or user first..
*/
bad_area:
up_read(&mm->mmap_sem);
bad_area_nosemaphore:
/* User mode accesses just cause a SIGSEGV */
if (user_mode(regs)) {
_exception(SIGSEGV, regs, code, address);
return;
}
no_context:
/* Are we prepared to handle this kernel fault? */
if (fixup_exception(regs))
return;
/*
* Oops. The kernel tried to access some bad page. We'll have to
* terminate things with extreme prejudice.
*/
bust_spinlocks(1);
pr_alert("Unable to handle kernel %s at virtual address %08lx",
address < PAGE_SIZE ? "NULL pointer dereference" :
"paging request", address);
pr_alert("ea = %08lx, ra = %08lx, cause = %ld\n", regs->ea, regs->ra,
cause);
panic("Oops");
return;
/*
* We ran out of memory, or some other thing happened to us that made
* us unable to handle the page fault gracefully.
*/
out_of_memory:
up_read(&mm->mmap_sem);
if (is_global_init(tsk)) {
yield();
down_read(&mm->mmap_sem);
goto survive;
}
pr_info("VM: killing process %s\n", tsk->comm);
if (user_mode(regs))
do_group_exit(SIGKILL);
goto no_context;
do_sigbus:
up_read(&mm->mmap_sem);
/* Kernel mode? Handle exceptions or die */
if (!user_mode(regs))
goto no_context;
_exception(SIGBUS, regs, BUS_ADRERR, address);
return;
vmalloc_fault:
{
/*
* Synchronize this task's top level page-table
* with the 'reference' page table.
*
* Do _not_ use "tsk" here. We might be inside
* an interrupt in the middle of a task switch..
*/
int offset = pgd_index(address);
pgd_t *pgd, *pgd_k;
pud_t *pud, *pud_k;
pmd_t *pmd, *pmd_k;
pte_t *pte_k;
#if 1
/* FIXME: Is this entirely correct ? */
pgd = pgd_current + offset;
#else
pgd = ¤t->mm->pgd[offset];
#endif
pgd_k = init_mm.pgd + offset;
if (!pgd_present(*pgd_k))
goto no_context;
set_pgd(pgd, *pgd_k);
pud = pud_offset(pgd, address);
pud_k = pud_offset(pgd_k, address);
if (!pud_present(*pud_k))
goto no_context;
pmd = pmd_offset(pud, address);
pmd_k = pmd_offset(pud_k, address);
if (!pmd_present(*pmd_k))
goto no_context;
set_pmd(pmd, *pmd_k);
pte_k = pte_offset_kernel(pmd_k, address);
if (!pte_present(*pte_k))
goto no_context;
flush_tlb_one(address);
return;
}
}
/*
* This routine handles page faults. It determines the address,
* and the problem, and then passes it off to one of the appropriate
* routines.
*/
asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long cause,
unsigned long address)
{
struct vm_area_struct *vma = NULL;
struct task_struct *tsk = current;
struct mm_struct *mm = tsk->mm;
int code = SEGV_MAPERR;
int fault;
unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
cause >>= 2;
/* Restart the instruction */
regs->ea -= 4;
/*
* We fault-in kernel-space virtual memory on-demand. The
* 'reference' page table is init_mm.pgd.
*
* NOTE! We MUST NOT take any locks for this case. We may
* be in an interrupt or a critical region, and should
* only copy the information from the master page table,
* nothing more.
*/
if (unlikely(address >= VMALLOC_START && address <= VMALLOC_END)) {
if (user_mode(regs))
goto bad_area_nosemaphore;
else
goto vmalloc_fault;
}
if (unlikely(address >= TASK_SIZE))
goto bad_area_nosemaphore;
/*
* If we're in an interrupt or have no user
* context, we must not take the fault..
*/
if (faulthandler_disabled() || !mm)
goto bad_area_nosemaphore;
if (user_mode(regs))
flags |= FAULT_FLAG_USER;
if (!down_read_trylock(&mm->mmap_sem)) {
if (!user_mode(regs) && !search_exception_tables(regs->ea))
goto bad_area_nosemaphore;
retry:
down_read(&mm->mmap_sem);
}
vma = find_vma(mm, address);
if (!vma)
goto bad_area;
if (vma->vm_start <= address)
goto good_area;
if (!(vma->vm_flags & VM_GROWSDOWN))
goto bad_area;
if (expand_stack(vma, address))
goto bad_area;
/*
* Ok, we have a good vm_area for this memory access, so
* we can handle it..
*/
good_area:
code = SEGV_ACCERR;
switch (cause) {
case EXC_SUPERV_INSN_ACCESS:
goto bad_area;
case EXC_SUPERV_DATA_ACCESS:
goto bad_area;
case EXC_X_PROTECTION_FAULT:
if (!(vma->vm_flags & VM_EXEC))
goto bad_area;
break;
case EXC_R_PROTECTION_FAULT:
if (!(vma->vm_flags & VM_READ))
goto bad_area;
break;
case EXC_W_PROTECTION_FAULT:
if (!(vma->vm_flags & VM_WRITE))
goto bad_area;
flags = FAULT_FLAG_WRITE;
break;
}
/*
* If for any reason at all we couldn't handle the fault,
* make sure we exit gracefully rather than endlessly redo
* the fault.
*/
fault = handle_mm_fault(mm, vma, address, flags);
if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current))
return;
if (unlikely(fault & VM_FAULT_ERROR)) {
if (fault & VM_FAULT_OOM)
goto out_of_memory;
else if (fault & VM_FAULT_SIGSEGV)
goto bad_area;
else if (fault & VM_FAULT_SIGBUS)
goto do_sigbus;
BUG();
}
/*
* Major/minor page fault accounting is only done on the
* initial attempt. If we go through a retry, it is extremely
* likely that the page will be found in page cache at that point.
*/
if (flags & FAULT_FLAG_ALLOW_RETRY) {
if (fault & VM_FAULT_MAJOR)
current->maj_flt++;
else
current->min_flt++;
if (fault & VM_FAULT_RETRY) {
/* Clear FAULT_FLAG_ALLOW_RETRY to avoid any risk
* of starvation. */
flags &= ~FAULT_FLAG_ALLOW_RETRY;
flags |= FAULT_FLAG_TRIED;
/*
* No need to up_read(&mm->mmap_sem) as we would
* have already released it in __lock_page_or_retry
* in mm/filemap.c.
*/
goto retry;
}
}
up_read(&mm->mmap_sem);
return;
/*
* Something tried to access memory that isn't in our memory map..
* Fix it, but check if it's kernel or user first..
*/
bad_area:
up_read(&mm->mmap_sem);
bad_area_nosemaphore:
/* User mode accesses just cause a SIGSEGV */
if (user_mode(regs)) {
if (unhandled_signal(current, SIGSEGV) && printk_ratelimit()) {
pr_info("%s: unhandled page fault (%d) at 0x%08lx, "
"cause %ld\n", current->comm, SIGSEGV, address, cause);
show_regs(regs);
}
_exception(SIGSEGV, regs, code, address);
return;
}
no_context:
/* Are we prepared to handle this kernel fault? */
if (fixup_exception(regs))
return;
/*
* Oops. The kernel tried to access some bad page. We'll have to
* terminate things with extreme prejudice.
*/
bust_spinlocks(1);
pr_alert("Unable to handle kernel %s at virtual address %08lx",
address < PAGE_SIZE ? "NULL pointer dereference" :
"paging request", address);
pr_alert("ea = %08lx, ra = %08lx, cause = %ld\n", regs->ea, regs->ra,
cause);
panic("Oops");
return;
/*
* We ran out of memory, or some other thing happened to us that made
* us unable to handle the page fault gracefully.
*/
out_of_memory:
up_read(&mm->mmap_sem);
if (!user_mode(regs))
goto no_context;
pagefault_out_of_memory();
return;
do_sigbus:
up_read(&mm->mmap_sem);
/* Kernel mode? Handle exceptions or die */
if (!user_mode(regs))
goto no_context;
_exception(SIGBUS, regs, BUS_ADRERR, address);
return;
vmalloc_fault:
{
/*
* Synchronize this task's top level page-table
* with the 'reference' page table.
*
* Do _not_ use "tsk" here. We might be inside
* an interrupt in the middle of a task switch..
*/
int offset = pgd_index(address);
pgd_t *pgd, *pgd_k;
pud_t *pud, *pud_k;
pmd_t *pmd, *pmd_k;
pte_t *pte_k;
pgd = pgd_current + offset;
pgd_k = init_mm.pgd + offset;
if (!pgd_present(*pgd_k))
goto no_context;
set_pgd(pgd, *pgd_k);
pud = pud_offset(pgd, address);
pud_k = pud_offset(pgd_k, address);
if (!pud_present(*pud_k))
goto no_context;
pmd = pmd_offset(pud, address);
pmd_k = pmd_offset(pud_k, address);
if (!pmd_present(*pmd_k))
goto no_context;
set_pmd(pmd, *pmd_k);
pte_k = pte_offset_kernel(pmd_k, address);
if (!pte_present(*pte_k))
goto no_context;
flush_tlb_one(address);
return;
}
}
dotraplinkage void __kprobes
do_general_protection(struct pt_regs *regs, long error_code)
{
struct task_struct *tsk;
conditional_sti(regs);
#ifdef CONFIG_X86_32
if (v8086_mode(regs))
goto gp_in_vm86;
#endif
tsk = current;
if (!user_mode_novm(regs))
goto gp_in_kernel;
#if defined(CONFIG_X86_32) && defined(CONFIG_PAX_PAGEEXEC)
if (!nx_enabled && tsk->mm && (tsk->mm->pax_flags & MF_PAX_PAGEEXEC)) {
struct mm_struct *mm = tsk->mm;
unsigned long limit;
down_write(&mm->mmap_sem);
limit = mm->context.user_cs_limit;
if (limit < TASK_SIZE) {
track_exec_limit(mm, limit, TASK_SIZE, VM_EXEC);
up_write(&mm->mmap_sem);
return;
}
up_write(&mm->mmap_sem);
}
#endif
tsk->thread.error_code = error_code;
tsk->thread.trap_no = 13;
if (show_unhandled_signals && unhandled_signal(tsk, SIGSEGV) &&
printk_ratelimit()) {
printk(KERN_INFO
"%s[%d] general protection ip:%lx sp:%lx error:%lx",
tsk->comm, task_pid_nr(tsk),
regs->ip, regs->sp, error_code);
print_vma_addr(" in ", regs->ip);
printk("\n");
}
force_sig(SIGSEGV, tsk);
return;
#ifdef CONFIG_X86_32
gp_in_vm86:
local_irq_enable();
handle_vm86_fault((struct kernel_vm86_regs *) regs, error_code);
return;
#endif
gp_in_kernel:
if (fixup_exception(regs))
return;
tsk->thread.error_code = error_code;
tsk->thread.trap_no = 13;
if (notify_die(DIE_GPF, "general protection fault", regs,
error_code, 13, SIGSEGV) == NOTIFY_STOP)
return;
#if defined(CONFIG_X86_32) && defined(CONFIG_PAX_KERNEXEC)
if ((regs->cs & 0xFFFF) == __KERNEL_CS)
die("PAX: suspicious general protection fault", regs, error_code);
else
#endif
die("general protection fault", regs, error_code);
}
void do_page_fault(struct pt_regs *regs, int write, unsigned long address,
unsigned long cause_code)
{
struct vm_area_struct *vma = NULL;
struct task_struct *tsk = current;
struct mm_struct *mm = tsk->mm;
siginfo_t info;
int fault, ret;
unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
/*
* We fault-in kernel-space virtual memory on-demand. The
* 'reference' page table is init_mm.pgd.
*
* NOTE! We MUST NOT take any locks for this case. We may
* be in an interrupt or a critical region, and should
* only copy the information from the master page table,
* nothing more.
*/
if (address >= VMALLOC_START && address <= VMALLOC_END) {
ret = handle_vmalloc_fault(address);
if (unlikely(ret))
goto bad_area_nosemaphore;
else
return;
}
info.si_code = SEGV_MAPERR;
/*
* If we're in an interrupt or have no user
* context, we must not take the fault..
*/
if (in_atomic() || !mm)
goto no_context;
if (user_mode(regs))
flags |= FAULT_FLAG_USER;
retry:
down_read(&mm->mmap_sem);
vma = find_vma(mm, address);
if (!vma)
goto bad_area;
if (vma->vm_start <= address)
goto good_area;
if (!(vma->vm_flags & VM_GROWSDOWN))
goto bad_area;
if (expand_stack(vma, address))
goto bad_area;
/*
* Ok, we have a good vm_area for this memory access, so
* we can handle it..
*/
good_area:
info.si_code = SEGV_ACCERR;
/* Handle protection violation, execute on heap or stack */
if (cause_code == ((ECR_V_PROTV << 16) | ECR_C_PROTV_INST_FETCH))
goto bad_area;
if (write) {
if (!(vma->vm_flags & VM_WRITE))
goto bad_area;
flags |= FAULT_FLAG_WRITE;
} else {
if (!(vma->vm_flags & (VM_READ | VM_EXEC)))
goto bad_area;
}
survive:
/*
* If for any reason at all we couldn't handle the fault,
* make sure we exit gracefully rather than endlessly redo
* the fault.
*/
fault = handle_mm_fault(mm, vma, address, flags);
/* If Pagefault was interrupted by SIGKILL, exit page fault "early" */
if (unlikely(fatal_signal_pending(current))) {
if ((fault & VM_FAULT_ERROR) && !(fault & VM_FAULT_RETRY))
up_read(&mm->mmap_sem);
if (user_mode(regs))
return;
}
if (likely(!(fault & VM_FAULT_ERROR))) {
if (flags & FAULT_FLAG_ALLOW_RETRY) {
/* To avoid updating stats twice for retry case */
if (fault & VM_FAULT_MAJOR)
tsk->maj_flt++;
else
tsk->min_flt++;
if (fault & VM_FAULT_RETRY) {
flags &= ~FAULT_FLAG_ALLOW_RETRY;
flags |= FAULT_FLAG_TRIED;
goto retry;
}
}
/* Fault Handled Gracefully */
up_read(&mm->mmap_sem);
return;
}
/* TBD: switch to pagefault_out_of_memory() */
if (fault & VM_FAULT_OOM)
goto out_of_memory;
else if (fault & VM_FAULT_SIGBUS)
goto do_sigbus;
/* no man's land */
BUG();
/*
* Something tried to access memory that isn't in our memory map..
* Fix it, but check if it's kernel or user first..
*/
bad_area:
up_read(&mm->mmap_sem);
bad_area_nosemaphore:
/* User mode accesses just cause a SIGSEGV */
if (user_mode(regs)) {
tsk->thread.fault_address = address;
tsk->thread.cause_code = cause_code;
info.si_signo = SIGSEGV;
info.si_errno = 0;
/* info.si_code has been set above */
info.si_addr = (void __user *)address;
force_sig_info(SIGSEGV, &info, tsk);
return;
}
no_context:
/* Are we prepared to handle this kernel fault?
*
* (The kernel has valid exception-points in the source
* when it acesses user-memory. When it fails in one
* of those points, we find it in a table and do a jump
* to some fixup code that loads an appropriate error
* code)
*/
if (fixup_exception(regs))
return;
die("Oops", regs, address, cause_code);
out_of_memory:
if (is_global_init(tsk)) {
yield();
goto survive;
}
up_read(&mm->mmap_sem);
if (user_mode(regs))
do_group_exit(SIGKILL); /* This will never return */
goto no_context;
do_sigbus:
up_read(&mm->mmap_sem);
if (!user_mode(regs))
goto no_context;
tsk->thread.fault_address = address;
tsk->thread.cause_code = cause_code;
info.si_signo = SIGBUS;
info.si_errno = 0;
info.si_code = BUS_ADRERR;
info.si_addr = (void __user *)address;
force_sig_info(SIGBUS, &info, tsk);
}
/*
* This routine is responsible for faulting in user pages.
* It passes the work off to one of the appropriate routines.
* It returns true if the fault was successfully handled.
*/
static int handle_page_fault(struct pt_regs *regs,
int fault_num,
int is_page_fault,
unsigned long address,
int write)
{
struct task_struct *tsk;
struct mm_struct *mm;
struct vm_area_struct *vma;
unsigned long stack_offset;
int fault;
int si_code;
int is_kernel_mode;
pgd_t *pgd;
/* on TILE, protection faults are always writes */
if (!is_page_fault)
write = 1;
flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
is_kernel_mode = (EX1_PL(regs->ex1) != USER_PL);
tsk = validate_current();
/*
* Check to see if we might be overwriting the stack, and bail
* out if so. The page fault code is a relatively likely
* place to get trapped in an infinite regress, and once we
* overwrite the whole stack, it becomes very hard to recover.
*/
stack_offset = stack_pointer & (THREAD_SIZE-1);
if (stack_offset < THREAD_SIZE / 8) {
pr_alert("Potential stack overrun: sp %#lx\n",
stack_pointer);
show_regs(regs);
pr_alert("Killing current process %d/%s\n",
tsk->pid, tsk->comm);
do_group_exit(SIGKILL);
}
/*
* Early on, we need to check for migrating PTE entries;
* see homecache.c. If we find a migrating PTE, we wait until
* the backing page claims to be done migrating, then we proceed.
* For kernel PTEs, we rewrite the PTE and return and retry.
* Otherwise, we treat the fault like a normal "no PTE" fault,
* rather than trying to patch up the existing PTE.
*/
pgd = get_current_pgd();
if (handle_migrating_pte(pgd, fault_num, address, regs->pc,
is_kernel_mode, write))
return 1;
si_code = SEGV_MAPERR;
/*
* We fault-in kernel-space virtual memory on-demand. The
* 'reference' page table is init_mm.pgd.
*
* NOTE! We MUST NOT take any locks for this case. We may
* be in an interrupt or a critical region, and should
* only copy the information from the master page table,
* nothing more.
*
* This verifies that the fault happens in kernel space
* and that the fault was not a protection fault.
*/
if (unlikely(address >= TASK_SIZE &&
!is_arch_mappable_range(address, 0))) {
if (is_kernel_mode && is_page_fault &&
vmalloc_fault(pgd, address) >= 0)
return 1;
/*
* Don't take the mm semaphore here. If we fixup a prefetch
* fault we could otherwise deadlock.
*/
mm = NULL; /* happy compiler */
vma = NULL;
goto bad_area_nosemaphore;
}
/*
* If we're trying to touch user-space addresses, we must
* be either at PL0, or else with interrupts enabled in the
* kernel, so either way we can re-enable interrupts here
* unless we are doing atomic access to user space with
* interrupts disabled.
*/
if (!(regs->flags & PT_FLAGS_DISABLE_IRQ))
local_irq_enable();
mm = tsk->mm;
/*
* If we're in an interrupt, have no user context or are running in an
* atomic region then we must not take the fault.
*/
if (in_atomic() || !mm) {
vma = NULL; /* happy compiler */
goto bad_area_nosemaphore;
}
if (!is_kernel_mode)
flags |= FAULT_FLAG_USER;
/*
* When running in the kernel we expect faults to occur only to
* addresses in user space. All other faults represent errors in the
* kernel and should generate an OOPS. Unfortunately, in the case of an
* erroneous fault occurring in a code path which already holds mmap_sem
* we will deadlock attempting to validate the fault against the
* address space. Luckily the kernel only validly references user
* space from well defined areas of code, which are listed in the
* exceptions table.
*
* As the vast majority of faults will be valid we will only perform
* the source reference check when there is a possibility of a deadlock.
* Attempt to lock the address space, if we cannot we then validate the
* source. If this is invalid we can skip the address space check,
* thus avoiding the deadlock.
*/
if (!down_read_trylock(&mm->mmap_sem)) {
if (is_kernel_mode &&
!search_exception_tables(regs->pc)) {
vma = NULL; /* happy compiler */
goto bad_area_nosemaphore;
}
down_read(&mm->mmap_sem);
}
vma = find_vma(mm, address);
if (!vma)
goto bad_area;
if (vma->vm_start <= address)
goto good_area;
if (!(vma->vm_flags & VM_GROWSDOWN))
goto bad_area;
if (regs->sp < PAGE_OFFSET) {
/*
* accessing the stack below sp is always a bug.
*/
if (address < regs->sp)
goto bad_area;
}
if (expand_stack(vma, address))
goto bad_area;
/*
* Ok, we have a good vm_area for this memory access, so
* we can handle it..
*/
good_area:
si_code = SEGV_ACCERR;
if (fault_num == INT_ITLB_MISS) {
if (!(vma->vm_flags & VM_EXEC))
goto bad_area;
} else if (write) {
#ifdef TEST_VERIFY_AREA
if (!is_page_fault && regs->cs == KERNEL_CS)
pr_err("WP fault at "REGFMT"\n", regs->eip);
#endif
if (!(vma->vm_flags & VM_WRITE))
goto bad_area;
flags |= FAULT_FLAG_WRITE;
} else {
if (!is_page_fault || !(vma->vm_flags & VM_READ))
goto bad_area;
}
/*
* If for any reason at all we couldn't handle the fault,
* make sure we exit gracefully rather than endlessly redo
* the fault.
*/
fault = handle_mm_fault(mm, vma, address, write);
if (unlikely(fault & VM_FAULT_ERROR)) {
if (fault & VM_FAULT_OOM)
goto out_of_memory;
else if (fault & VM_FAULT_SIGBUS)
goto do_sigbus;
BUG();
}
if (fault & VM_FAULT_MAJOR)
tsk->maj_flt++;
else
tsk->min_flt++;
#if CHIP_HAS_TILE_DMA() || CHIP_HAS_SN_PROC()
/*
* If this was an asynchronous fault,
* restart the appropriate engine.
*/
switch (fault_num) {
#if CHIP_HAS_TILE_DMA()
case INT_DMATLB_MISS:
case INT_DMATLB_MISS_DWNCL:
case INT_DMATLB_ACCESS:
case INT_DMATLB_ACCESS_DWNCL:
__insn_mtspr(SPR_DMA_CTR, SPR_DMA_CTR__REQUEST_MASK);
break;
#endif
#if CHIP_HAS_SN_PROC()
case INT_SNITLB_MISS:
case INT_SNITLB_MISS_DWNCL:
__insn_mtspr(SPR_SNCTL,
__insn_mfspr(SPR_SNCTL) &
~SPR_SNCTL__FRZPROC_MASK);
break;
#endif
}
#endif
up_read(&mm->mmap_sem);
return 1;
/*
* Something tried to access memory that isn't in our memory map..
* Fix it, but check if it's kernel or user first..
*/
bad_area:
up_read(&mm->mmap_sem);
bad_area_nosemaphore:
/* User mode accesses just cause a SIGSEGV */
if (!is_kernel_mode) {
/*
* It's possible to have interrupts off here.
*/
local_irq_enable();
force_sig_info_fault("segfault", SIGSEGV, si_code, address,
fault_num, tsk, regs);
return 0;
}
no_context:
/* Are we prepared to handle this kernel fault? */
if (fixup_exception(regs))
return 0;
/*
* Oops. The kernel tried to access some bad page. We'll have to
* terminate things with extreme prejudice.
*/
bust_spinlocks(1);
/* FIXME: no lookup_address() yet */
#ifdef SUPPORT_LOOKUP_ADDRESS
if (fault_num == INT_ITLB_MISS) {
pte_t *pte = lookup_address(address);
if (pte && pte_present(*pte) && !pte_exec_kernel(*pte))
pr_crit("kernel tried to execute"
" non-executable page - exploit attempt?"
" (uid: %d)\n", current->uid);
}
#endif
if (address < PAGE_SIZE)
pr_alert("Unable to handle kernel NULL pointer dereference\n");
else
pr_alert("Unable to handle kernel paging request\n");
pr_alert(" at virtual address "REGFMT", pc "REGFMT"\n",
address, regs->pc);
show_regs(regs);
if (unlikely(tsk->pid < 2)) {
panic("Kernel page fault running %s!",
is_idle_task(tsk) ? "the idle task" : "init");
}
/*
* More FIXME: we should probably copy the i386 here and
* implement a generic die() routine. Not today.
*/
#ifdef SUPPORT_DIE
die("Oops", regs);
#endif
bust_spinlocks(1);
do_group_exit(SIGKILL);
/*
* We ran out of memory, or some other thing happened to us that made
* us unable to handle the page fault gracefully.
*/
out_of_memory:
up_read(&mm->mmap_sem);
if (is_kernel_mode)
goto no_context;
pagefault_out_of_memory();
return 0;
do_sigbus:
up_read(&mm->mmap_sem);
/* Kernel mode? Handle exceptions or die */
if (is_kernel_mode)
goto no_context;
force_sig_info_fault("bus error", SIGBUS, BUS_ADRERR, address,
fault_num, tsk, regs);
return 0;
}
static void emulate_load_store_MIPS16e(struct pt_regs *regs, void __user * addr)
{
unsigned long value;
unsigned int res;
int reg;
unsigned long orig31;
u16 __user *pc16;
unsigned long origpc;
union mips16e_instruction mips16inst, oldinst;
origpc = regs->cp0_epc;
orig31 = regs->regs[31];
pc16 = (unsigned short __user *)msk_isa16_mode(origpc);
/*
* This load never faults.
*/
__get_user(mips16inst.full, pc16);
oldinst = mips16inst;
/* skip EXTEND instruction */
if (mips16inst.ri.opcode == MIPS16e_extend_op) {
pc16++;
__get_user(mips16inst.full, pc16);
} else if (delay_slot(regs)) {
/* skip jump instructions */
/* JAL/JALX are 32 bits but have OPCODE in first short int */
if (mips16inst.ri.opcode == MIPS16e_jal_op)
pc16++;
pc16++;
if (get_user(mips16inst.full, pc16))
goto sigbus;
}
switch (mips16inst.ri.opcode) {
case MIPS16e_i64_op: /* I64 or RI64 instruction */
switch (mips16inst.i64.func) { /* I64/RI64 func field check */
case MIPS16e_ldpc_func:
case MIPS16e_ldsp_func:
reg = reg16to32[mips16inst.ri64.ry];
goto loadDW;
case MIPS16e_sdsp_func:
reg = reg16to32[mips16inst.ri64.ry];
goto writeDW;
case MIPS16e_sdrasp_func:
reg = 29; /* GPRSP */
goto writeDW;
}
goto sigbus;
case MIPS16e_swsp_op:
case MIPS16e_lwpc_op:
case MIPS16e_lwsp_op:
reg = reg16to32[mips16inst.ri.rx];
break;
case MIPS16e_i8_op:
if (mips16inst.i8.func != MIPS16e_swrasp_func)
goto sigbus;
reg = 29; /* GPRSP */
break;
default:
reg = reg16to32[mips16inst.rri.ry];
break;
}
switch (mips16inst.ri.opcode) {
case MIPS16e_lb_op:
case MIPS16e_lbu_op:
case MIPS16e_sb_op:
goto sigbus;
case MIPS16e_lh_op:
if (!access_ok(VERIFY_READ, addr, 2))
goto sigbus;
LoadHW(addr, value, res);
if (res)
goto fault;
MIPS16e_compute_return_epc(regs, &oldinst);
regs->regs[reg] = value;
break;
case MIPS16e_lhu_op:
if (!access_ok(VERIFY_READ, addr, 2))
goto sigbus;
LoadHWU(addr, value, res);
if (res)
goto fault;
MIPS16e_compute_return_epc(regs, &oldinst);
regs->regs[reg] = value;
break;
case MIPS16e_lw_op:
case MIPS16e_lwpc_op:
case MIPS16e_lwsp_op:
if (!access_ok(VERIFY_READ, addr, 4))
goto sigbus;
LoadW(addr, value, res);
if (res)
goto fault;
MIPS16e_compute_return_epc(regs, &oldinst);
regs->regs[reg] = value;
break;
case MIPS16e_lwu_op:
#ifdef CONFIG_64BIT
/*
* A 32-bit kernel might be running on a 64-bit processor. But
* if we're on a 32-bit processor and an i-cache incoherency
* or race makes us see a 64-bit instruction here the sdl/sdr
* would blow up, so for now we don't handle unaligned 64-bit
* instructions on 32-bit kernels.
*/
if (!access_ok(VERIFY_READ, addr, 4))
goto sigbus;
LoadWU(addr, value, res);
if (res)
goto fault;
MIPS16e_compute_return_epc(regs, &oldinst);
regs->regs[reg] = value;
break;
#endif /* CONFIG_64BIT */
/* Cannot handle 64-bit instructions in 32-bit kernel */
goto sigill;
case MIPS16e_ld_op:
loadDW:
#ifdef CONFIG_64BIT
/*
* A 32-bit kernel might be running on a 64-bit processor. But
* if we're on a 32-bit processor and an i-cache incoherency
* or race makes us see a 64-bit instruction here the sdl/sdr
* would blow up, so for now we don't handle unaligned 64-bit
* instructions on 32-bit kernels.
*/
if (!access_ok(VERIFY_READ, addr, 8))
goto sigbus;
LoadDW(addr, value, res);
if (res)
goto fault;
MIPS16e_compute_return_epc(regs, &oldinst);
regs->regs[reg] = value;
break;
#endif /* CONFIG_64BIT */
/* Cannot handle 64-bit instructions in 32-bit kernel */
goto sigill;
case MIPS16e_sh_op:
if (!access_ok(VERIFY_WRITE, addr, 2))
goto sigbus;
MIPS16e_compute_return_epc(regs, &oldinst);
value = regs->regs[reg];
StoreHW(addr, value, res);
if (res)
goto fault;
break;
case MIPS16e_sw_op:
case MIPS16e_swsp_op:
case MIPS16e_i8_op: /* actually - MIPS16e_swrasp_func */
if (!access_ok(VERIFY_WRITE, addr, 4))
goto sigbus;
MIPS16e_compute_return_epc(regs, &oldinst);
value = regs->regs[reg];
StoreW(addr, value, res);
if (res)
goto fault;
break;
case MIPS16e_sd_op:
writeDW:
#ifdef CONFIG_64BIT
/*
* A 32-bit kernel might be running on a 64-bit processor. But
* if we're on a 32-bit processor and an i-cache incoherency
* or race makes us see a 64-bit instruction here the sdl/sdr
* would blow up, so for now we don't handle unaligned 64-bit
* instructions on 32-bit kernels.
*/
if (!access_ok(VERIFY_WRITE, addr, 8))
goto sigbus;
MIPS16e_compute_return_epc(regs, &oldinst);
value = regs->regs[reg];
StoreDW(addr, value, res);
if (res)
goto fault;
break;
#endif /* CONFIG_64BIT */
/* Cannot handle 64-bit instructions in 32-bit kernel */
goto sigill;
default:
/*
* Pheeee... We encountered an yet unknown instruction or
* cache coherence problem. Die sucker, die ...
*/
goto sigill;
}
#ifdef CONFIG_DEBUG_FS
unaligned_instructions++;
#endif
return;
fault:
/* roll back jump/branch */
regs->cp0_epc = origpc;
regs->regs[31] = orig31;
/* Did we have an exception handler installed? */
if (fixup_exception(regs))
return;
die_if_kernel("Unhandled kernel unaligned access", regs);
force_sig(SIGSEGV, current);
return;
sigbus:
die_if_kernel("Unhandled kernel unaligned access", regs);
force_sig(SIGBUS, current);
return;
sigill:
die_if_kernel
("Unhandled kernel unaligned access or invalid instruction", regs);
force_sig(SIGILL, current);
}
static void emulate_load_store_insn(struct pt_regs *regs,
void __user *addr, unsigned int __user *pc)
{
union mips_instruction insn;
unsigned long value;
unsigned int res;
unsigned long origpc;
unsigned long orig31;
void __user *fault_addr = NULL;
origpc = (unsigned long)pc;
orig31 = regs->regs[31];
perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, 0);
/*
* This load never faults.
*/
__get_user(insn.word, pc);
switch (insn.i_format.opcode) {
/*
* These are instructions that a compiler doesn't generate. We
* can assume therefore that the code is MIPS-aware and
* really buggy. Emulating these instructions would break the
* semantics anyway.
*/
case ll_op:
case lld_op:
case sc_op:
case scd_op:
/*
* For these instructions the only way to create an address
* error is an attempted access to kernel/supervisor address
* space.
*/
case ldl_op:
case ldr_op:
case lwl_op:
case lwr_op:
case sdl_op:
case sdr_op:
case swl_op:
case swr_op:
case lb_op:
case lbu_op:
case sb_op:
goto sigbus;
case spec3_op:
if (insn.r_format.func != lx_op)
goto sigill;
switch (insn.r_format.re) {
case lwx_op:
if (!access_ok(VERIFY_READ, addr, 4))
goto sigbus;
LoadW(addr, value, res);
if (res)
goto fault;
compute_return_epc(regs);
regs->regs[insn.r_format.rd] = value;
break;
case lhx_op:
if (!access_ok(VERIFY_READ, addr, 2))
goto sigbus;
LoadHW(addr, value, res);
if (res)
goto fault;
compute_return_epc(regs);
regs->regs[insn.r_format.rd] = value;
break;
#ifdef CONFIG_64BIT
case ldx_op:
if (!access_ok(VERIFY_READ, addr, 8))
goto sigbus;
LoadDW(addr, value, res);
if (res)
goto fault;
compute_return_epc(regs);
regs->regs[insn.r_format.rd] = value;
break;
case lwux_op:
if (!access_ok(VERIFY_READ, addr, 4))
goto sigbus;
LoadWU(addr, value, res);
if (res)
goto fault;
compute_return_epc(regs);
regs->regs[insn.r_format.rd] = value;
break;
#endif /* CONFIG_64BIT */
case lhux_op:
if (!access_ok(VERIFY_READ, addr, 2))
goto sigbus;
LoadHWU(addr, value, res);
if (res)
goto fault;
compute_return_epc(regs);
regs->regs[insn.r_format.rd] = value;
break;
case lbux_op:
case lbx_op:
goto sigbus;
default:
goto sigill;
}
break;
/*
* The remaining opcodes are the ones that are really of
* interest.
*/
case lh_op:
if (!access_ok(VERIFY_READ, addr, 2))
goto sigbus;
LoadHW(addr, value, res);
if (res)
goto fault;
compute_return_epc(regs);
regs->regs[insn.i_format.rt] = value;
break;
case lw_op:
if (!access_ok(VERIFY_READ, addr, 4))
goto sigbus;
LoadW(addr, value, res);
if (res)
goto fault;
compute_return_epc(regs);
regs->regs[insn.i_format.rt] = value;
break;
case lhu_op:
if (!access_ok(VERIFY_READ, addr, 2))
goto sigbus;
LoadHWU(addr, value, res);
if (res)
goto fault;
compute_return_epc(regs);
regs->regs[insn.i_format.rt] = value;
break;
case lwu_op:
#ifdef CONFIG_64BIT
/*
* A 32-bit kernel might be running on a 64-bit processor. But
* if we're on a 32-bit processor and an i-cache incoherency
* or race makes us see a 64-bit instruction here the sdl/sdr
* would blow up, so for now we don't handle unaligned 64-bit
* instructions on 32-bit kernels.
*/
if (!access_ok(VERIFY_READ, addr, 4))
goto sigbus;
LoadWU(addr, value, res);
if (res)
goto fault;
compute_return_epc(regs);
regs->regs[insn.i_format.rt] = value;
break;
#endif /* CONFIG_64BIT */
/* Cannot handle 64-bit instructions in 32-bit kernel */
goto sigill;
case ld_op:
#ifdef CONFIG_64BIT
/*
* A 32-bit kernel might be running on a 64-bit processor. But
* if we're on a 32-bit processor and an i-cache incoherency
* or race makes us see a 64-bit instruction here the sdl/sdr
* would blow up, so for now we don't handle unaligned 64-bit
* instructions on 32-bit kernels.
*/
if (!access_ok(VERIFY_READ, addr, 8))
goto sigbus;
LoadDW(addr, value, res);
if (res)
goto fault;
compute_return_epc(regs);
regs->regs[insn.i_format.rt] = value;
break;
#endif /* CONFIG_64BIT */
/* Cannot handle 64-bit instructions in 32-bit kernel */
goto sigill;
case sh_op:
if (!access_ok(VERIFY_WRITE, addr, 2))
goto sigbus;
compute_return_epc(regs);
value = regs->regs[insn.i_format.rt];
StoreHW(addr, value, res);
if (res)
goto fault;
break;
case sw_op:
if (!access_ok(VERIFY_WRITE, addr, 4))
goto sigbus;
compute_return_epc(regs);
value = regs->regs[insn.i_format.rt];
StoreW(addr, value, res);
if (res)
goto fault;
break;
case sd_op:
#ifdef CONFIG_64BIT
/*
* A 32-bit kernel might be running on a 64-bit processor. But
* if we're on a 32-bit processor and an i-cache incoherency
* or race makes us see a 64-bit instruction here the sdl/sdr
* would blow up, so for now we don't handle unaligned 64-bit
* instructions on 32-bit kernels.
*/
if (!access_ok(VERIFY_WRITE, addr, 8))
goto sigbus;
compute_return_epc(regs);
value = regs->regs[insn.i_format.rt];
StoreDW(addr, value, res);
if (res)
goto fault;
break;
#endif /* CONFIG_64BIT */
/* Cannot handle 64-bit instructions in 32-bit kernel */
goto sigill;
case lwc1_op:
case ldc1_op:
case swc1_op:
case sdc1_op:
die_if_kernel("Unaligned FP access in kernel code", regs);
BUG_ON(!used_math());
BUG_ON(!is_fpu_owner());
lose_fpu(1); /* Save FPU state for the emulator. */
res = fpu_emulator_cop1Handler(regs, ¤t->thread.fpu, 1,
&fault_addr);
own_fpu(1); /* Restore FPU state. */
/* Signal if something went wrong. */
process_fpemu_return(res, fault_addr);
if (res == 0)
break;
return;
/*
* COP2 is available to implementor for application specific use.
* It's up to applications to register a notifier chain and do
* whatever they have to do, including possible sending of signals.
*/
case lwc2_op:
cu2_notifier_call_chain(CU2_LWC2_OP, regs);
break;
case ldc2_op:
cu2_notifier_call_chain(CU2_LDC2_OP, regs);
break;
case swc2_op:
cu2_notifier_call_chain(CU2_SWC2_OP, regs);
break;
case sdc2_op:
cu2_notifier_call_chain(CU2_SDC2_OP, regs);
break;
default:
/*
* Pheeee... We encountered an yet unknown instruction or
* cache coherence problem. Die sucker, die ...
*/
goto sigill;
}
#ifdef CONFIG_DEBUG_FS
unaligned_instructions++;
#endif
return;
fault:
/* roll back jump/branch */
regs->cp0_epc = origpc;
regs->regs[31] = orig31;
/* Did we have an exception handler installed? */
if (fixup_exception(regs))
return;
die_if_kernel("Unhandled kernel unaligned access", regs);
force_sig(SIGSEGV, current);
return;
sigbus:
die_if_kernel("Unhandled kernel unaligned access", regs);
force_sig(SIGBUS, current);
return;
sigill:
die_if_kernel
("Unhandled kernel unaligned access or invalid instruction", regs);
force_sig(SIGILL, current);
}
static void emulate_load_store_microMIPS(struct pt_regs *regs,
void __user *addr)
{
unsigned long value;
unsigned int res;
int i;
unsigned int reg = 0, rvar;
unsigned long orig31;
u16 __user *pc16;
u16 halfword;
unsigned int word;
unsigned long origpc, contpc;
union mips_instruction insn;
struct mm_decoded_insn mminsn;
void __user *fault_addr = NULL;
origpc = regs->cp0_epc;
orig31 = regs->regs[31];
mminsn.micro_mips_mode = 1;
/*
* This load never faults.
*/
pc16 = (unsigned short __user *)msk_isa16_mode(regs->cp0_epc);
__get_user(halfword, pc16);
pc16++;
contpc = regs->cp0_epc + 2;
word = ((unsigned int)halfword << 16);
mminsn.pc_inc = 2;
if (!mm_insn_16bit(halfword)) {
__get_user(halfword, pc16);
pc16++;
contpc = regs->cp0_epc + 4;
mminsn.pc_inc = 4;
word |= halfword;
}
mminsn.insn = word;
if (get_user(halfword, pc16))
goto fault;
mminsn.next_pc_inc = 2;
word = ((unsigned int)halfword << 16);
if (!mm_insn_16bit(halfword)) {
pc16++;
if (get_user(halfword, pc16))
goto fault;
mminsn.next_pc_inc = 4;
word |= halfword;
}
mminsn.next_insn = word;
insn = (union mips_instruction)(mminsn.insn);
if (mm_isBranchInstr(regs, mminsn, &contpc))
insn = (union mips_instruction)(mminsn.next_insn);
/* Parse instruction to find what to do */
switch (insn.mm_i_format.opcode) {
case mm_pool32a_op:
switch (insn.mm_x_format.func) {
case mm_lwxs_op:
reg = insn.mm_x_format.rd;
goto loadW;
}
goto sigbus;
case mm_pool32b_op:
switch (insn.mm_m_format.func) {
case mm_lwp_func:
reg = insn.mm_m_format.rd;
if (reg == 31)
goto sigbus;
if (!access_ok(VERIFY_READ, addr, 8))
goto sigbus;
LoadW(addr, value, res);
if (res)
goto fault;
regs->regs[reg] = value;
addr += 4;
LoadW(addr, value, res);
if (res)
goto fault;
regs->regs[reg + 1] = value;
goto success;
case mm_swp_func:
reg = insn.mm_m_format.rd;
if (reg == 31)
goto sigbus;
if (!access_ok(VERIFY_WRITE, addr, 8))
goto sigbus;
value = regs->regs[reg];
StoreW(addr, value, res);
if (res)
goto fault;
addr += 4;
value = regs->regs[reg + 1];
StoreW(addr, value, res);
if (res)
goto fault;
goto success;
case mm_ldp_func:
#ifdef CONFIG_64BIT
reg = insn.mm_m_format.rd;
if (reg == 31)
goto sigbus;
if (!access_ok(VERIFY_READ, addr, 16))
goto sigbus;
LoadDW(addr, value, res);
if (res)
goto fault;
regs->regs[reg] = value;
addr += 8;
LoadDW(addr, value, res);
if (res)
goto fault;
regs->regs[reg + 1] = value;
goto success;
#endif /* CONFIG_64BIT */
goto sigill;
case mm_sdp_func:
#ifdef CONFIG_64BIT
reg = insn.mm_m_format.rd;
if (reg == 31)
goto sigbus;
if (!access_ok(VERIFY_WRITE, addr, 16))
goto sigbus;
value = regs->regs[reg];
StoreDW(addr, value, res);
if (res)
goto fault;
addr += 8;
value = regs->regs[reg + 1];
StoreDW(addr, value, res);
if (res)
goto fault;
goto success;
#endif /* CONFIG_64BIT */
goto sigill;
case mm_lwm32_func:
reg = insn.mm_m_format.rd;
rvar = reg & 0xf;
if ((rvar > 9) || !reg)
goto sigill;
if (reg & 0x10) {
if (!access_ok
(VERIFY_READ, addr, 4 * (rvar + 1)))
goto sigbus;
} else {
if (!access_ok(VERIFY_READ, addr, 4 * rvar))
goto sigbus;
}
if (rvar == 9)
rvar = 8;
for (i = 16; rvar; rvar--, i++) {
LoadW(addr, value, res);
if (res)
goto fault;
addr += 4;
regs->regs[i] = value;
}
if ((reg & 0xf) == 9) {
LoadW(addr, value, res);
if (res)
goto fault;
addr += 4;
regs->regs[30] = value;
}
if (reg & 0x10) {
LoadW(addr, value, res);
if (res)
goto fault;
regs->regs[31] = value;
}
goto success;
case mm_swm32_func:
reg = insn.mm_m_format.rd;
rvar = reg & 0xf;
if ((rvar > 9) || !reg)
goto sigill;
if (reg & 0x10) {
if (!access_ok
(VERIFY_WRITE, addr, 4 * (rvar + 1)))
goto sigbus;
} else {
if (!access_ok(VERIFY_WRITE, addr, 4 * rvar))
goto sigbus;
}
if (rvar == 9)
rvar = 8;
for (i = 16; rvar; rvar--, i++) {
value = regs->regs[i];
StoreW(addr, value, res);
if (res)
goto fault;
addr += 4;
}
if ((reg & 0xf) == 9) {
value = regs->regs[30];
StoreW(addr, value, res);
if (res)
goto fault;
addr += 4;
}
if (reg & 0x10) {
value = regs->regs[31];
StoreW(addr, value, res);
if (res)
goto fault;
}
goto success;
case mm_ldm_func:
#ifdef CONFIG_64BIT
reg = insn.mm_m_format.rd;
rvar = reg & 0xf;
if ((rvar > 9) || !reg)
goto sigill;
if (reg & 0x10) {
if (!access_ok
(VERIFY_READ, addr, 8 * (rvar + 1)))
goto sigbus;
} else {
if (!access_ok(VERIFY_READ, addr, 8 * rvar))
goto sigbus;
}
if (rvar == 9)
rvar = 8;
for (i = 16; rvar; rvar--, i++) {
LoadDW(addr, value, res);
if (res)
goto fault;
addr += 4;
regs->regs[i] = value;
}
if ((reg & 0xf) == 9) {
LoadDW(addr, value, res);
if (res)
goto fault;
addr += 8;
regs->regs[30] = value;
}
if (reg & 0x10) {
LoadDW(addr, value, res);
if (res)
goto fault;
regs->regs[31] = value;
}
goto success;
#endif /* CONFIG_64BIT */
goto sigill;
case mm_sdm_func:
#ifdef CONFIG_64BIT
reg = insn.mm_m_format.rd;
rvar = reg & 0xf;
if ((rvar > 9) || !reg)
goto sigill;
if (reg & 0x10) {
if (!access_ok
(VERIFY_WRITE, addr, 8 * (rvar + 1)))
goto sigbus;
} else {
if (!access_ok(VERIFY_WRITE, addr, 8 * rvar))
goto sigbus;
}
if (rvar == 9)
rvar = 8;
for (i = 16; rvar; rvar--, i++) {
value = regs->regs[i];
StoreDW(addr, value, res);
if (res)
goto fault;
addr += 8;
}
if ((reg & 0xf) == 9) {
value = regs->regs[30];
StoreDW(addr, value, res);
if (res)
goto fault;
addr += 8;
}
if (reg & 0x10) {
value = regs->regs[31];
StoreDW(addr, value, res);
if (res)
goto fault;
}
goto success;
#endif /* CONFIG_64BIT */
goto sigill;
/* LWC2, SWC2, LDC2, SDC2 are not serviced */
}
goto sigbus;
case mm_pool32c_op:
switch (insn.mm_m_format.func) {
case mm_lwu_func:
reg = insn.mm_m_format.rd;
goto loadWU;
}
/* LL,SC,LLD,SCD are not serviced */
goto sigbus;
case mm_pool32f_op:
switch (insn.mm_x_format.func) {
case mm_lwxc1_func:
case mm_swxc1_func:
case mm_ldxc1_func:
case mm_sdxc1_func:
goto fpu_emul;
}
goto sigbus;
case mm_ldc132_op:
case mm_sdc132_op:
case mm_lwc132_op:
case mm_swc132_op:
fpu_emul:
/* roll back jump/branch */
regs->cp0_epc = origpc;
regs->regs[31] = orig31;
die_if_kernel("Unaligned FP access in kernel code", regs);
BUG_ON(!used_math());
BUG_ON(!is_fpu_owner());
lose_fpu(1); /* save the FPU state for the emulator */
res = fpu_emulator_cop1Handler(regs, ¤t->thread.fpu, 1,
&fault_addr);
own_fpu(1); /* restore FPU state */
/* If something went wrong, signal */
process_fpemu_return(res, fault_addr);
if (res == 0)
goto success;
return;
case mm_lh32_op:
reg = insn.mm_i_format.rt;
goto loadHW;
case mm_lhu32_op:
reg = insn.mm_i_format.rt;
goto loadHWU;
case mm_lw32_op:
reg = insn.mm_i_format.rt;
goto loadW;
case mm_sh32_op:
reg = insn.mm_i_format.rt;
goto storeHW;
case mm_sw32_op:
reg = insn.mm_i_format.rt;
goto storeW;
case mm_ld32_op:
reg = insn.mm_i_format.rt;
goto loadDW;
case mm_sd32_op:
reg = insn.mm_i_format.rt;
goto storeDW;
case mm_pool16c_op:
switch (insn.mm16_m_format.func) {
case mm_lwm16_op:
reg = insn.mm16_m_format.rlist;
rvar = reg + 1;
if (!access_ok(VERIFY_READ, addr, 4 * rvar))
goto sigbus;
for (i = 16; rvar; rvar--, i++) {
LoadW(addr, value, res);
if (res)
goto fault;
addr += 4;
regs->regs[i] = value;
}
LoadW(addr, value, res);
if (res)
goto fault;
regs->regs[31] = value;
goto success;
case mm_swm16_op:
reg = insn.mm16_m_format.rlist;
rvar = reg + 1;
if (!access_ok(VERIFY_WRITE, addr, 4 * rvar))
goto sigbus;
for (i = 16; rvar; rvar--, i++) {
value = regs->regs[i];
StoreW(addr, value, res);
if (res)
goto fault;
addr += 4;
}
value = regs->regs[31];
StoreW(addr, value, res);
if (res)
goto fault;
goto success;
}
goto sigbus;
case mm_lhu16_op:
reg = reg16to32[insn.mm16_rb_format.rt];
goto loadHWU;
case mm_lw16_op:
reg = reg16to32[insn.mm16_rb_format.rt];
goto loadW;
case mm_sh16_op:
reg = reg16to32st[insn.mm16_rb_format.rt];
goto storeHW;
case mm_sw16_op:
reg = reg16to32st[insn.mm16_rb_format.rt];
goto storeW;
case mm_lwsp16_op:
reg = insn.mm16_r5_format.rt;
goto loadW;
case mm_swsp16_op:
reg = insn.mm16_r5_format.rt;
goto storeW;
case mm_lwgp16_op:
reg = reg16to32[insn.mm16_r3_format.rt];
goto loadW;
default:
goto sigill;
}
loadHW:
if (!access_ok(VERIFY_READ, addr, 2))
goto sigbus;
LoadHW(addr, value, res);
if (res)
goto fault;
regs->regs[reg] = value;
goto success;
loadHWU:
if (!access_ok(VERIFY_READ, addr, 2))
goto sigbus;
LoadHWU(addr, value, res);
if (res)
goto fault;
regs->regs[reg] = value;
goto success;
loadW:
if (!access_ok(VERIFY_READ, addr, 4))
goto sigbus;
LoadW(addr, value, res);
if (res)
goto fault;
regs->regs[reg] = value;
goto success;
loadWU:
#ifdef CONFIG_64BIT
/*
* A 32-bit kernel might be running on a 64-bit processor. But
* if we're on a 32-bit processor and an i-cache incoherency
* or race makes us see a 64-bit instruction here the sdl/sdr
* would blow up, so for now we don't handle unaligned 64-bit
* instructions on 32-bit kernels.
*/
if (!access_ok(VERIFY_READ, addr, 4))
goto sigbus;
LoadWU(addr, value, res);
if (res)
goto fault;
regs->regs[reg] = value;
goto success;
#endif /* CONFIG_64BIT */
/* Cannot handle 64-bit instructions in 32-bit kernel */
goto sigill;
loadDW:
#ifdef CONFIG_64BIT
/*
* A 32-bit kernel might be running on a 64-bit processor. But
* if we're on a 32-bit processor and an i-cache incoherency
* or race makes us see a 64-bit instruction here the sdl/sdr
* would blow up, so for now we don't handle unaligned 64-bit
* instructions on 32-bit kernels.
*/
if (!access_ok(VERIFY_READ, addr, 8))
goto sigbus;
LoadDW(addr, value, res);
if (res)
goto fault;
regs->regs[reg] = value;
goto success;
#endif /* CONFIG_64BIT */
/* Cannot handle 64-bit instructions in 32-bit kernel */
goto sigill;
storeHW:
if (!access_ok(VERIFY_WRITE, addr, 2))
goto sigbus;
value = regs->regs[reg];
StoreHW(addr, value, res);
if (res)
goto fault;
goto success;
storeW:
if (!access_ok(VERIFY_WRITE, addr, 4))
goto sigbus;
value = regs->regs[reg];
StoreW(addr, value, res);
if (res)
goto fault;
goto success;
storeDW:
#ifdef CONFIG_64BIT
/*
* A 32-bit kernel might be running on a 64-bit processor. But
* if we're on a 32-bit processor and an i-cache incoherency
* or race makes us see a 64-bit instruction here the sdl/sdr
* would blow up, so for now we don't handle unaligned 64-bit
* instructions on 32-bit kernels.
*/
if (!access_ok(VERIFY_WRITE, addr, 8))
goto sigbus;
value = regs->regs[reg];
StoreDW(addr, value, res);
if (res)
goto fault;
goto success;
#endif /* CONFIG_64BIT */
/* Cannot handle 64-bit instructions in 32-bit kernel */
goto sigill;
success:
regs->cp0_epc = contpc; /* advance or branch */
#ifdef CONFIG_DEBUG_FS
unaligned_instructions++;
#endif
return;
fault:
/* roll back jump/branch */
regs->cp0_epc = origpc;
regs->regs[31] = orig31;
/* Did we have an exception handler installed? */
if (fixup_exception(regs))
return;
die_if_kernel("Unhandled kernel unaligned access", regs);
force_sig(SIGSEGV, current);
return;
sigbus:
die_if_kernel("Unhandled kernel unaligned access", regs);
force_sig(SIGBUS, current);
return;
sigill:
die_if_kernel
("Unhandled kernel unaligned access or invalid instruction", regs);
force_sig(SIGILL, current);
}
/*
* This routine handles page faults. It determines the address,
* and the problem, and then passes it off to one of the appropriate
* routines.
*/
asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long write,
unsigned long address)
{
struct vm_area_struct * vma = NULL;
struct task_struct *tsk = current;
struct mm_struct *mm = tsk->mm;
const int field = sizeof(unsigned long) * 2;
siginfo_t info;
#if 0
printk("Cpu%d[%s:%d:%0*lx:%ld:%0*lx]\n", smp_processor_id(),
current->comm, current->pid, field, address, write,
field, regs->cp0_epc);
#endif
info.si_code = SEGV_MAPERR;
/*
* We fault-in kernel-space virtual memory on-demand. The
* 'reference' page table is init_mm.pgd.
*
* NOTE! We MUST NOT take any locks for this case. We may
* be in an interrupt or a critical region, and should
* only copy the information from the master page table,
* nothing more.
*/
if (unlikely(address >= VMALLOC_START && address <= VMALLOC_END))
goto vmalloc_fault;
/*
* If we're in an interrupt or have no user
* context, we must not take the fault..
*/
if (in_atomic() || !mm)
goto bad_area_nosemaphore;
down_read(&mm->mmap_sem);
vma = find_vma(mm, address);
if (!vma)
goto bad_area;
if (vma->vm_start <= address)
goto good_area;
if (!(vma->vm_flags & VM_GROWSDOWN))
goto bad_area;
if (expand_stack(vma, address))
goto bad_area;
/*
* Ok, we have a good vm_area for this memory access, so
* we can handle it..
*/
good_area:
info.si_code = SEGV_ACCERR;
if (write) {
if (!(vma->vm_flags & VM_WRITE))
goto bad_area;
} else {
if (!(vma->vm_flags & (VM_READ | VM_WRITE | VM_EXEC)))
goto bad_area;
}
survive:
/*
* If for any reason at all we couldn't handle the fault,
* make sure we exit gracefully rather than endlessly redo
* the fault.
*/
switch (handle_mm_fault(mm, vma, address, write)) {
case VM_FAULT_MINOR:
tsk->min_flt++;
break;
case VM_FAULT_MAJOR:
tsk->maj_flt++;
break;
case VM_FAULT_SIGBUS:
goto do_sigbus;
case VM_FAULT_OOM:
goto out_of_memory;
default:
BUG();
}
up_read(&mm->mmap_sem);
return;
/*
* Something tried to access memory that isn't in our memory map..
* Fix it, but check if it's kernel or user first..
*/
bad_area:
up_read(&mm->mmap_sem);
bad_area_nosemaphore:
/* User mode accesses just cause a SIGSEGV */
if (user_mode(regs)) {
tsk->thread.cp0_badvaddr = address;
tsk->thread.error_code = write;
#if 0
printk("do_page_fault() #2: sending SIGSEGV to %s for "
"invalid %s\n%0*lx (epc == %0*lx, ra == %0*lx)\n",
tsk->comm,
write ? "write access to" : "read access from",
field, address,
field, (unsigned long) regs->cp0_epc,
field, (unsigned long) regs->regs[31]);
#endif
info.si_signo = SIGSEGV;
info.si_errno = 0;
/* info.si_code has been set above */
info.si_addr = (void __user *) address;
force_sig_info(SIGSEGV, &info, tsk);
return;
}
no_context:
/* Are we prepared to handle this kernel fault? */
if (fixup_exception(regs)) {
current->thread.cp0_baduaddr = address;
return;
}
/*
* Oops. The kernel tried to access some bad page. We'll have to
* terminate things with extreme prejudice.
*/
bust_spinlocks(1);
printk(KERN_ALERT "CPU %d Unable to handle kernel paging request at "
"virtual address %0*lx, epc == %0*lx, ra == %0*lx\n",
smp_processor_id(), field, address, field, regs->cp0_epc,
field, regs->regs[31]);
die("Oops", regs);
/*
* We ran out of memory, or some other thing happened to us that made
* us unable to handle the page fault gracefully.
*/
out_of_memory:
up_read(&mm->mmap_sem);
if (is_init(tsk)) {
yield();
down_read(&mm->mmap_sem);
goto survive;
}
printk("VM: killing process %s\n", tsk->comm);
if (user_mode(regs))
do_exit(SIGKILL);
goto no_context;
do_sigbus:
up_read(&mm->mmap_sem);
/* Kernel mode? Handle exceptions or die */
if (!user_mode(regs))
goto no_context;
else
/*
* Send a sigbus, regardless of whether we were in kernel
* or user mode.
*/
#if 0
printk("do_page_fault() #3: sending SIGBUS to %s for "
"invalid %s\n%0*lx (epc == %0*lx, ra == %0*lx)\n",
tsk->comm,
write ? "write access to" : "read access from",
field, address,
field, (unsigned long) regs->cp0_epc,
field, (unsigned long) regs->regs[31]);
#endif
tsk->thread.cp0_badvaddr = address;
info.si_signo = SIGBUS;
info.si_errno = 0;
info.si_code = BUS_ADRERR;
info.si_addr = (void __user *) address;
force_sig_info(SIGBUS, &info, tsk);
return;
vmalloc_fault:
{
/*
* Synchronize this task's top level page-table
* with the 'reference' page table.
*
* Do _not_ use "tsk" here. We might be inside
* an interrupt in the middle of a task switch..
*/
int offset = __pgd_offset(address);
pgd_t *pgd, *pgd_k;
pud_t *pud, *pud_k;
pmd_t *pmd, *pmd_k;
pte_t *pte_k;
pgd = (pgd_t *) pgd_current[smp_processor_id()] + offset;
pgd_k = init_mm.pgd + offset;
if (!pgd_present(*pgd_k))
goto no_context;
set_pgd(pgd, *pgd_k);
pud = pud_offset(pgd, address);
pud_k = pud_offset(pgd_k, address);
if (!pud_present(*pud_k))
goto no_context;
pmd = pmd_offset(pud, address);
pmd_k = pmd_offset(pud_k, address);
if (!pmd_present(*pmd_k))
goto no_context;
set_pmd(pmd, *pmd_k);
pte_k = pte_offset_kernel(pmd_k, address);
if (!pte_present(*pte_k))
goto no_context;
return;
}
}
/*
* This routine handles page faults. It determines the address,
* and the problem, and then passes it off to one of the appropriate
* routines.
*/
asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long write,
unsigned long address)
{
struct vm_area_struct * vma;
struct task_struct *tsk = current;
struct mm_struct *mm = tsk->mm;
unsigned long fixup;
siginfo_t info;
/*
* We fault-in kernel-space virtual memory on-demand. The
* 'reference' page table is init_mm.pgd.
*
* NOTE! We MUST NOT take any locks for this case. We may
* be in an interrupt or a critical region, and should
* only copy the information from the master page table,
* nothing more.
*/
if (address >= VMALLOC_START)
goto vmalloc_fault;
info.si_code = SEGV_MAPERR;
/*
* If we're in an interrupt or have no user
* context, we must not take the fault..
*/
if (in_interrupt() || !mm)
goto no_context;
#if 0
printk("[%s:%d:%08lx:%ld:%08lx]\n", current->comm, current->pid,
address, write, regs->cp0_epc);
#endif
down_read(&mm->mmap_sem);
vma = find_vma(mm, address);
if (!vma)
goto bad_area;
if (vma->vm_start <= address)
goto good_area;
if (expand_stack(vma, address))
goto bad_area;
/*
* Ok, we have a good vm_area for this memory access, so
* we can handle it..
*/
good_area:
info.si_code = SEGV_ACCERR;
if (write) {
if (!(vma->vm_flags & VM_WRITE))
goto bad_area;
} else {
if (!(vma->vm_flags & (VM_READ | VM_EXEC)))
goto bad_area;
}
survive:
/*
* If for any reason at all we couldn't handle the fault,
* make sure we exit gracefully rather than endlessly redo
* the fault.
*/
switch (handle_mm_fault(mm, vma, address, write)) {
case 1:
tsk->min_flt++;
break;
case 2:
tsk->maj_flt++;
break;
case 0:
goto do_sigbus;
default:
goto out_of_memory;
}
up_read(&mm->mmap_sem);
return;
/*
* Something tried to access memory that isn't in our memory map..
* Fix it, but check if it's kernel or user first..
*/
bad_area:
up_read(&mm->mmap_sem);
bad_area_nosemaphore:
/* User mode accesses just cause a SIGSEGV */
if (user_mode(regs)) {
tsk->thread.cp0_badvaddr = address;
tsk->thread.error_code = write;
#if 0
printk("do_page_fault() #2: sending SIGSEGV to %s for illegal %s\n"
"%08lx (epc == %08lx, ra == %08lx)\n",
tsk->comm,
write ? "write access to" : "read access from",
address,
(unsigned long) regs->cp0_epc,
(unsigned long) regs->regs[31]);
#endif
info.si_signo = SIGSEGV;
info.si_errno = 0;
/* info.si_code has been set above */
info.si_addr = (void *) address;
force_sig_info(SIGSEGV, &info, tsk);
return;
}
no_context:
/* Are we prepared to handle this kernel fault? */
fixup = search_exception_table(exception_epc(regs));
if (fixup) {
long new_epc;
tsk->thread.cp0_baduaddr = address;
new_epc = fixup_exception(dpf_reg, fixup, regs->cp0_epc);
if (development_version)
printk(KERN_DEBUG "%s: Exception at [<%lx>] (%lx)\n",
tsk->comm, regs->cp0_epc, new_epc);
regs->cp0_epc = new_epc;
return;
}
/*
* Oops. The kernel tried to access some bad page. We'll have to
* terminate things with extreme prejudice.
*/
printk(KERN_ALERT "Unable to handle kernel paging request at virtual "
"address %08lx, epc == %08lx, ra == %08lx\n",
address, regs->cp0_epc, regs->regs[31]);
die("Oops", regs);
/* Game over. */
/*
* We ran out of memory, or some other thing happened to us that made
* us unable to handle the page fault gracefully.
*/
out_of_memory:
if (tsk->pid == 1) {
yield();
goto survive;
}
up_read(&mm->mmap_sem);
printk(KERN_NOTICE "VM: killing process %s\n", tsk->comm);
if (user_mode(regs))
do_exit(SIGKILL);
goto no_context;
do_sigbus:
up_read(&mm->mmap_sem);
/*
* Send a sigbus, regardless of whether we were in kernel
* or user mode.
*/
tsk->thread.cp0_badvaddr = address;
info.si_signo = SIGBUS;
info.si_errno = 0;
info.si_code = BUS_ADRERR;
info.si_addr = (void *) address;
force_sig_info(SIGBUS, &info, tsk);
/* Kernel mode? Handle exceptions or die */
if (!user_mode(regs))
goto no_context;
return;
vmalloc_fault:
{
/*
* Synchronize this task's top level page-table
* with the 'reference' page table.
*/
int offset = pgd_index(address);
pgd_t *pgd, *pgd_k;
pmd_t *pmd, *pmd_k;
pgd = tsk->active_mm->pgd + offset;
pgd_k = init_mm.pgd + offset;
if (!pgd_present(*pgd)) {
if (!pgd_present(*pgd_k))
goto bad_area_nosemaphore;
set_pgd(pgd, *pgd_k);
return;
}
pmd = pmd_offset(pgd, address);
pmd_k = pmd_offset(pgd_k, address);
if (pmd_present(*pmd) || !pmd_present(*pmd_k))
goto bad_area_nosemaphore;
set_pmd(pmd, *pmd_k);
}
}
/*
* This routine handles page faults. It determines the address,
* and the problem, and then passes it off to one of the appropriate
* routines.
*/
asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long write,
unsigned long mmu_meh)
{
struct vm_area_struct *vma = NULL;
struct task_struct *tsk = current;
struct mm_struct *mm = tsk->mm;
int si_code;
int fault;
unsigned long address = mmu_meh & PAGE_MASK;
si_code = SEGV_MAPERR;
#ifndef CONFIG_CPU_HAS_TLBI
/*
* We fault-in kernel-space virtual memory on-demand. The
* 'reference' page table is init_mm.pgd.
*
* NOTE! We MUST NOT take any locks for this case. We may
* be in an interrupt or a critical region, and should
* only copy the information from the master page table,
* nothing more.
*/
if (unlikely(address >= VMALLOC_START) &&
unlikely(address <= VMALLOC_END)) {
/*
* Synchronize this task's top level page-table
* with the 'reference' page table.
*
* Do _not_ use "tsk" here. We might be inside
* an interrupt in the middle of a task switch..
*/
int offset = __pgd_offset(address);
pgd_t *pgd, *pgd_k;
pud_t *pud, *pud_k;
pmd_t *pmd, *pmd_k;
pte_t *pte_k;
unsigned long pgd_base;
pgd_base = tlb_get_pgd();
pgd = (pgd_t *)pgd_base + offset;
pgd_k = init_mm.pgd + offset;
if (!pgd_present(*pgd_k))
goto no_context;
set_pgd(pgd, *pgd_k);
pud = (pud_t *)pgd;
pud_k = (pud_t *)pgd_k;
if (!pud_present(*pud_k))
goto no_context;
pmd = pmd_offset(pud, address);
pmd_k = pmd_offset(pud_k, address);
if (!pmd_present(*pmd_k))
goto no_context;
set_pmd(pmd, *pmd_k);
pte_k = pte_offset_kernel(pmd_k, address);
if (!pte_present(*pte_k))
goto no_context;
return;
}
#endif
/*
* If we're in an interrupt or have no user
* context, we must not take the fault..
*/
if (in_atomic() || !mm)
goto bad_area_nosemaphore;
down_read(&mm->mmap_sem);
vma = find_vma(mm, address);
if (!vma)
goto bad_area;
if (vma->vm_start <= address)
goto good_area;
if (!(vma->vm_flags & VM_GROWSDOWN))
goto bad_area;
if (expand_stack(vma, address))
goto bad_area;
/*
* Ok, we have a good vm_area for this memory access, so
* we can handle it..
*/
good_area:
si_code = SEGV_ACCERR;
if (write) {
if (!(vma->vm_flags & VM_WRITE))
goto bad_area;
} else {
if (!(vma->vm_flags & (VM_READ | VM_WRITE | VM_EXEC)))
goto bad_area;
}
/*
* If for any reason at all we couldn't handle the fault,
* make sure we exit gracefully rather than endlessly redo
* the fault.
*/
fault = handle_mm_fault(vma, address, write ? FAULT_FLAG_WRITE : 0);
if (unlikely(fault & VM_FAULT_ERROR)) {
if (fault & VM_FAULT_OOM)
goto out_of_memory;
else if (fault & VM_FAULT_SIGBUS)
goto do_sigbus;
else if (fault & VM_FAULT_SIGSEGV)
goto bad_area;
BUG();
}
if (fault & VM_FAULT_MAJOR)
tsk->maj_flt++;
else
tsk->min_flt++;
up_read(&mm->mmap_sem);
return;
/*
* Something tried to access memory that isn't in our memory map..
* Fix it, but check if it's kernel or user first..
*/
bad_area:
up_read(&mm->mmap_sem);
bad_area_nosemaphore:
/* User mode accesses just cause a SIGSEGV */
if (user_mode(regs)) {
force_sig_fault(SIGSEGV, si_code, (void __user *)address, current);
return;
}
no_context:
/* Are we prepared to handle this kernel fault? */
if (fixup_exception(regs))
return;
/*
* Oops. The kernel tried to access some bad page. We'll have to
* terminate things with extreme prejudice.
*/
bust_spinlocks(1);
pr_alert("Unable to handle kernel paging request at virtual "
"address 0x%08lx, pc: 0x%08lx\n", address, regs->pc);
die_if_kernel("Oops", regs, write);
out_of_memory:
/*
* We ran out of memory, call the OOM killer, and return the userspace
* (which will retry the fault, or kill us if we got oom-killed).
*/
pagefault_out_of_memory();
return;
do_sigbus:
up_read(&mm->mmap_sem);
/* Kernel mode? Handle exceptions or die */
if (!user_mode(regs))
goto no_context;
force_sig_fault(SIGBUS, BUS_ADRERR, (void __user *)address, current);
}
dotraplinkage void __kprobes
do_general_protection(struct pt_regs *regs, long error_code)
{
struct task_struct *tsk;
conditional_sti(regs);
#ifdef CONFIG_X86_32
if (regs->flags & X86_VM_MASK)
goto gp_in_vm86;
#endif
tsk = current;
if (!user_mode(regs))
goto gp_in_kernel;
#ifdef CONFIG_X86_32
{
int cpu;
int ok;
cpu = get_cpu();
ok = check_lazy_exec_limit(cpu, regs, error_code);
put_cpu();
if (ok)
return;
if (print_fatal_signals) {
printk(KERN_ERR "#GPF(%ld[seg:%lx]) at %08lx, CPU#%d.\n",
error_code, error_code/8, regs->ip, smp_processor_id());
printk(KERN_ERR "exec_limit: %08lx, user_cs: %08x/%08x.\n",
current->mm->context.exec_limit,
current->mm->context.user_cs.a,
current->mm->context.user_cs.b);
}
}
#endif /*CONFIG_X86_32*/
tsk->thread.error_code = error_code;
tsk->thread.trap_no = 13;
if (show_unhandled_signals && unhandled_signal(tsk, SIGSEGV) &&
printk_ratelimit()) {
printk(KERN_INFO
"%s[%d] general protection ip:%lx sp:%lx error:%lx",
tsk->comm, task_pid_nr(tsk),
regs->ip, regs->sp, error_code);
print_vma_addr(" in ", regs->ip);
printk("\n");
}
force_sig(SIGSEGV, tsk);
return;
#ifdef CONFIG_X86_32
gp_in_vm86:
local_irq_enable();
handle_vm86_fault((struct kernel_vm86_regs *) regs, error_code);
return;
#endif
gp_in_kernel:
if (fixup_exception(regs))
return;
tsk->thread.error_code = error_code;
tsk->thread.trap_no = 13;
if (notify_die(DIE_GPF, "general protection fault", regs,
error_code, 13, SIGSEGV) == NOTIFY_STOP)
return;
die("general protection fault", regs, error_code);
}
