void
abort_handler(struct trapframe *tf, int type)
{
struct vm_map *map;
struct pcb *pcb;
struct thread *td;
u_int user, far, fsr;
vm_prot_t ftype;
void *onfault;
vm_offset_t va;
int error = 0;
struct ksig ksig;
struct proc *p;
if (type == 1)
return (prefetch_abort_handler(tf));
/* Grab FAR/FSR before enabling interrupts */
far = cpu_faultaddress();
fsr = cpu_faultstatus();
#if 0
printf("data abort: fault address=%p (from pc=%p lr=%p)\n",
(void*)far, (void*)tf->tf_pc, (void*)tf->tf_svc_lr);
#endif
/* Update vmmeter statistics */
#if 0
vmexp.traps++;
#endif
td = curthread;
p = td->td_proc;
PCPU_INC(cnt.v_trap);
/* Data abort came from user mode? */
user = TRAP_USERMODE(tf);
if (user) {
td->td_pticks = 0;
td->td_frame = tf;
if (td->td_cowgen != td->td_proc->p_cowgen)
thread_cow_update(td);
}
/* Grab the current pcb */
pcb = td->td_pcb;
/* Re-enable interrupts if they were enabled previously */
if (td->td_md.md_spinlock_count == 0) {
if (__predict_true(tf->tf_spsr & PSR_I) == 0)
enable_interrupts(PSR_I);
if (__predict_true(tf->tf_spsr & PSR_F) == 0)
enable_interrupts(PSR_F);
}
/* Invoke the appropriate handler, if necessary */
if (__predict_false(data_aborts[fsr & FAULT_TYPE_MASK].func != NULL)) {
if ((data_aborts[fsr & FAULT_TYPE_MASK].func)(tf, fsr, far,
td, &ksig)) {
goto do_trapsignal;
}
goto out;
}
/*
* At this point, we're dealing with one of the following data aborts:
*
* FAULT_TRANS_S - Translation -- Section
* FAULT_TRANS_P - Translation -- Page
* FAULT_DOMAIN_S - Domain -- Section
* FAULT_DOMAIN_P - Domain -- Page
* FAULT_PERM_S - Permission -- Section
* FAULT_PERM_P - Permission -- Page
*
* These are the main virtual memory-related faults signalled by
* the MMU.
*/
/*
* Make sure the Program Counter is sane. We could fall foul of
* someone executing Thumb code, in which case the PC might not
* be word-aligned. This would cause a kernel alignment fault
* further down if we have to decode the current instruction.
* XXX: It would be nice to be able to support Thumb at some point.
*/
if (__predict_false((tf->tf_pc & 3) != 0)) {
if (user) {
/*
* Give the user an illegal instruction signal.
*/
/* Deliver a SIGILL to the process */
ksig.signb = SIGILL;
ksig.code = 0;
goto do_trapsignal;
}
/*
* The kernel never executes Thumb code.
*/
printf("\ndata_abort_fault: Misaligned Kernel-mode "
"Program Counter\n");
dab_fatal(tf, fsr, far, td, &ksig);
}
va = trunc_page((vm_offset_t)far);
/*
* It is only a kernel address space fault iff:
* 1. user == 0 and
* 2. pcb_onfault not set or
* 3. pcb_onfault set and not LDRT/LDRBT/STRT/STRBT instruction.
*/
if (user == 0 && (va >= VM_MIN_KERNEL_ADDRESS ||
(va < VM_MIN_ADDRESS && vector_page == ARM_VECTORS_LOW)) &&
__predict_true((pcb->pcb_onfault == NULL ||
(ReadWord(tf->tf_pc) & 0x05200000) != 0x04200000))) {
map = kernel_map;
/* Was the fault due to the FPE/IPKDB ? */
if (__predict_false((tf->tf_spsr & PSR_MODE)==PSR_UND32_MODE)) {
/*
* Force exit via userret()
* This is necessary as the FPE is an extension to
* userland that actually runs in a priveledged mode
* but uses USR mode permissions for its accesses.
*/
user = 1;
ksig.signb = SIGSEGV;
ksig.code = 0;
goto do_trapsignal;
}
} else {
map = &td->td_proc->p_vmspace->vm_map;
}
/*
* We need to know whether the page should be mapped as R or R/W.
* On armv4, the fault status register does not indicate whether
* the access was a read or write. We know that a permission fault
* can only be the result of a write to a read-only location, so we
* can deal with those quickly. Otherwise we need to disassemble
* the faulting instruction to determine if it was a write.
*/
if (IS_PERMISSION_FAULT(fsr))
ftype = VM_PROT_WRITE;
else {
u_int insn = ReadWord(tf->tf_pc);
if (((insn & 0x0c100000) == 0x04000000) || /* STR/STRB */
((insn & 0x0e1000b0) == 0x000000b0) || /* STRH/STRD */
((insn & 0x0a100000) == 0x08000000)) { /* STM/CDT */
ftype = VM_PROT_WRITE;
} else {
if ((insn & 0x0fb00ff0) == 0x01000090) /* SWP */
ftype = VM_PROT_READ | VM_PROT_WRITE;
else
ftype = VM_PROT_READ;
}
}
/*
* See if the fault is as a result of ref/mod emulation,
* or domain mismatch.
*/
#ifdef DEBUG
last_fault_code = fsr;
#endif
if (td->td_critnest != 0 || WITNESS_CHECK(WARN_SLEEPOK | WARN_GIANTOK,
NULL, "Kernel page fault") != 0)
goto fatal_pagefault;
if (pmap_fault_fixup(vmspace_pmap(td->td_proc->p_vmspace), va, ftype,
user)) {
goto out;
}
onfault = pcb->pcb_onfault;
pcb->pcb_onfault = NULL;
error = vm_fault(map, va, ftype, VM_FAULT_NORMAL);
pcb->pcb_onfault = onfault;
if (__predict_true(error == 0))
goto out;
fatal_pagefault:
if (user == 0) {
if (pcb->pcb_onfault) {
tf->tf_r0 = error;
tf->tf_pc = (register_t)(intptr_t) pcb->pcb_onfault;
return;
}
printf("\nvm_fault(%p, %x, %x, 0) -> %x\n", map, va, ftype,
error);
dab_fatal(tf, fsr, far, td, &ksig);
}
if (error == ENOMEM) {
printf("VM: pid %d (%s), uid %d killed: "
"out of swap\n", td->td_proc->p_pid, td->td_name,
(td->td_proc->p_ucred) ?
td->td_proc->p_ucred->cr_uid : -1);
ksig.signb = SIGKILL;
} else {
ksig.signb = SIGSEGV;
}
ksig.code = 0;
do_trapsignal:
call_trapsignal(td, ksig.signb, ksig.code);
out:
/* If returning to user mode, make sure to invoke userret() */
if (user)
userret(td, tf);
}
/*
* Abort handler.
*
* FAR, FSR, and everything what can be lost after enabling
* interrupts must be grabbed before the interrupts will be
* enabled. Note that when interrupts will be enabled, we
* could even migrate to another CPU ...
*
* TODO: move quick cases to ASM
*/
void
abort_handler(struct trapframe *tf, int prefetch)
{
struct thread *td;
vm_offset_t far, va;
int idx, rv;
uint32_t fsr;
struct ksig ksig;
struct proc *p;
struct pcb *pcb;
struct vm_map *map;
struct vmspace *vm;
vm_prot_t ftype;
bool usermode;
#ifdef INVARIANTS
void *onfault;
#endif
td = curthread;
fsr = (prefetch) ? cp15_ifsr_get(): cp15_dfsr_get();
#if __ARM_ARCH >= 7
far = (prefetch) ? cp15_ifar_get() : cp15_dfar_get();
#else
far = (prefetch) ? TRAPF_PC(tf) : cp15_dfar_get();
#endif
idx = FSR_TO_FAULT(fsr);
usermode = TRAPF_USERMODE(tf); /* Abort came from user mode? */
if (usermode)
td->td_frame = tf;
CTR6(KTR_TRAP, "%s: fsr %#x (idx %u) far %#x prefetch %u usermode %d",
__func__, fsr, idx, far, prefetch, usermode);
/*
* Firstly, handle aborts that are not directly related to mapping.
*/
if (__predict_false(idx == FAULT_EA_IMPREC)) {
abort_imprecise(tf, fsr, prefetch, usermode);
return;
}
if (__predict_false(idx == FAULT_DEBUG)) {
abort_debug(tf, fsr, prefetch, usermode, far);
return;
}
/*
* ARM has a set of unprivileged load and store instructions
* (LDRT/LDRBT/STRT/STRBT ...) which are supposed to be used in other
* than user mode and OS should recognize their aborts and behave
* appropriately. However, there is no way how to do that reasonably
* in general unless we restrict the handling somehow.
*
* For now, these instructions are used only in copyin()/copyout()
* like functions where usermode buffers are checked in advance that
* they are not from KVA space. Thus, no action is needed here.
*/
#ifdef ARM_NEW_PMAP
rv = pmap_fault(PCPU_GET(curpmap), far, fsr, idx, usermode);
if (rv == 0) {
return;
} else if (rv == EFAULT) {
call_trapsignal(td, SIGSEGV, SEGV_MAPERR, far);
userret(td, tf);
return;
}
#endif
/*
* Now, when we handled imprecise and debug aborts, the rest of
* aborts should be really related to mapping.
*/
PCPU_INC(cnt.v_trap);
#ifdef KDB
if (kdb_active) {
kdb_reenter();
goto out;
}
#endif
if (__predict_false((td->td_pflags & TDP_NOFAULTING) != 0)) {
/*
* Due to both processor errata and lazy TLB invalidation when
* access restrictions are removed from virtual pages, memory
* accesses that are allowed by the physical mapping layer may
* nonetheless cause one spurious page fault per virtual page.
* When the thread is executing a "no faulting" section that
* is bracketed by vm_fault_{disable,enable}_pagefaults(),
* every page fault is treated as a spurious page fault,
* unless it accesses the same virtual address as the most
* recent page fault within the same "no faulting" section.
*/
if (td->td_md.md_spurflt_addr != far ||
(td->td_pflags & TDP_RESETSPUR) != 0) {
td->td_md.md_spurflt_addr = far;
td->td_pflags &= ~TDP_RESETSPUR;
tlb_flush_local(far & ~PAGE_MASK);
return;
}
} else {
/*
* If we get a page fault while in a critical section, then
* it is most likely a fatal kernel page fault. The kernel
* is already going to panic trying to get a sleep lock to
* do the VM lookup, so just consider it a fatal trap so the
* kernel can print out a useful trap message and even get
* to the debugger.
*
* If we get a page fault while holding a non-sleepable
* lock, then it is most likely a fatal kernel page fault.
* If WITNESS is enabled, then it's going to whine about
* bogus LORs with various VM locks, so just skip to the
* fatal trap handling directly.
*/
if (td->td_critnest != 0 ||
WITNESS_CHECK(WARN_SLEEPOK | WARN_GIANTOK, NULL,
"Kernel page fault") != 0) {
abort_fatal(tf, idx, fsr, far, prefetch, td, &ksig);
return;
}
}
/* Re-enable interrupts if they were enabled previously. */
if (td->td_md.md_spinlock_count == 0) {
if (__predict_true(tf->tf_spsr & PSR_I) == 0)
enable_interrupts(PSR_I);
if (__predict_true(tf->tf_spsr & PSR_F) == 0)
enable_interrupts(PSR_F);
}
p = td->td_proc;
if (usermode) {
td->td_pticks = 0;
if (td->td_cowgen != p->p_cowgen)
thread_cow_update(td);
}
/* Invoke the appropriate handler, if necessary. */
if (__predict_false(aborts[idx].func != NULL)) {
if ((aborts[idx].func)(tf, idx, fsr, far, prefetch, td, &ksig))
goto do_trapsignal;
goto out;
}
/*
* Don't pass faulting cache operation to vm_fault(). We don't want
* to handle all vm stuff at this moment.
*/
pcb = td->td_pcb;
if (__predict_false(pcb->pcb_onfault == cachebailout)) {
tf->tf_r0 = far; /* return failing address */
tf->tf_pc = (register_t)pcb->pcb_onfault;
return;
}
/* Handle remaining I-cache aborts. */
if (idx == FAULT_ICACHE) {
if (abort_icache(tf, idx, fsr, far, prefetch, td, &ksig))
goto do_trapsignal;
goto out;
}
/*
* At this point, we're dealing with one of the following aborts:
*
* FAULT_TRAN_xx - Translation
* FAULT_PERM_xx - Permission
*
* These are the main virtual memory-related faults signalled by
* the MMU.
*/
/* fusubailout is used by [fs]uswintr to avoid page faulting. */
pcb = td->td_pcb;
if (__predict_false(pcb->pcb_onfault == fusubailout)) {
tf->tf_r0 = EFAULT;
tf->tf_pc = (register_t)pcb->pcb_onfault;
return;
}
va = trunc_page(far);
if (va >= KERNBASE) {
/*
* Don't allow user-mode faults in kernel address space.
*/
if (usermode)
goto nogo;
map = kernel_map;
} else {
/*
* This is a fault on non-kernel virtual memory. If curproc
* is NULL or curproc->p_vmspace is NULL the fault is fatal.
*/
vm = (p != NULL) ? p->p_vmspace : NULL;
if (vm == NULL)
goto nogo;
map = &vm->vm_map;
if (!usermode && (td->td_intr_nesting_level != 0 ||
pcb->pcb_onfault == NULL)) {
abort_fatal(tf, idx, fsr, far, prefetch, td, &ksig);
return;
}
}
ftype = (fsr & FSR_WNR) ? VM_PROT_WRITE : VM_PROT_READ;
if (prefetch)
ftype |= VM_PROT_EXECUTE;
#ifdef DEBUG
last_fault_code = fsr;
#endif
#ifndef ARM_NEW_PMAP
if (pmap_fault_fixup(vmspace_pmap(td->td_proc->p_vmspace), va, ftype,
usermode)) {
goto out;
}
#endif
#ifdef INVARIANTS
onfault = pcb->pcb_onfault;
pcb->pcb_onfault = NULL;
#endif
/* Fault in the page. */
rv = vm_fault(map, va, ftype, VM_FAULT_NORMAL);
#ifdef INVARIANTS
pcb->pcb_onfault = onfault;
#endif
if (__predict_true(rv == KERN_SUCCESS))
goto out;
nogo:
if (!usermode) {
if (td->td_intr_nesting_level == 0 &&
pcb->pcb_onfault != NULL) {
tf->tf_r0 = rv;
tf->tf_pc = (int)pcb->pcb_onfault;
return;
}
CTR2(KTR_TRAP, "%s: vm_fault() failed with %d", __func__, rv);
abort_fatal(tf, idx, fsr, far, prefetch, td, &ksig);
return;
}
ksig.sig = SIGSEGV;
ksig.code = (rv == KERN_PROTECTION_FAILURE) ? SEGV_ACCERR : SEGV_MAPERR;
ksig.addr = far;
do_trapsignal:
call_trapsignal(td, ksig.sig, ksig.code, ksig.addr);
out:
if (usermode)
userret(td, tf);
}
void
trap(struct trapframe *frame)
{
struct thread *td = curthread;
struct proc *p = td->td_proc;
int i = 0, ucode = 0, code;
u_int type;
register_t addr = 0;
vm_offset_t eva;
ksiginfo_t ksi;
#ifdef POWERFAIL_NMI
static int lastalert = 0;
#endif
PCPU_INC(cnt.v_trap);
type = frame->tf_trapno;
#ifdef SMP
/* Handler for NMI IPIs used for stopping CPUs. */
if (type == T_NMI) {
if (ipi_nmi_handler() == 0)
goto out;
}
#endif /* SMP */
#ifdef KDB
if (kdb_active) {
kdb_reenter();
goto out;
}
#endif
if (type == T_RESERVED) {
trap_fatal(frame, 0);
goto out;
}
#ifdef HWPMC_HOOKS
/*
* CPU PMCs interrupt using an NMI so we check for that first.
* If the HWPMC module is active, 'pmc_hook' will point to
* the function to be called. A return value of '1' from the
* hook means that the NMI was handled by it and that we can
* return immediately.
*/
if (type == T_NMI && pmc_intr &&
(*pmc_intr)(PCPU_GET(cpuid), frame))
goto out;
#endif
if (type == T_MCHK) {
if (!mca_intr())
trap_fatal(frame, 0);
goto out;
}
#ifdef KDTRACE_HOOKS
/*
* A trap can occur while DTrace executes a probe. Before
* executing the probe, DTrace blocks re-scheduling and sets
* a flag in it's per-cpu flags to indicate that it doesn't
* want to fault. On returning from the probe, the no-fault
* flag is cleared and finally re-scheduling is enabled.
*
* If the DTrace kernel module has registered a trap handler,
* call it and if it returns non-zero, assume that it has
* handled the trap and modified the trap frame so that this
* function can return normally.
*/
if ((type == T_PROTFLT || type == T_PAGEFLT) &&
dtrace_trap_func != NULL)
if ((*dtrace_trap_func)(frame, type))
goto out;
if (type == T_DTRACE_PROBE || type == T_DTRACE_RET ||
type == T_BPTFLT) {
struct reg regs;
fill_frame_regs(frame, ®s);
if (type == T_DTRACE_PROBE &&
dtrace_fasttrap_probe_ptr != NULL &&
dtrace_fasttrap_probe_ptr(®s) == 0)
goto out;
if (type == T_BPTFLT &&
dtrace_pid_probe_ptr != NULL &&
dtrace_pid_probe_ptr(®s) == 0)
goto out;
if (type == T_DTRACE_RET &&
dtrace_return_probe_ptr != NULL &&
dtrace_return_probe_ptr(®s) == 0)
goto out;
}
#endif
if ((frame->tf_eflags & PSL_I) == 0) {
/*
* Buggy application or kernel code has disabled
* interrupts and then trapped. Enabling interrupts
* now is wrong, but it is better than running with
* interrupts disabled until they are accidentally
* enabled later.
*/
if (ISPL(frame->tf_cs) == SEL_UPL || (frame->tf_eflags & PSL_VM))
uprintf(
"pid %ld (%s): trap %d with interrupts disabled\n",
(long)curproc->p_pid, curthread->td_name, type);
else if (type != T_BPTFLT && type != T_TRCTRAP &&
frame->tf_eip != (int)cpu_switch_load_gs) {
/*
* XXX not quite right, since this may be for a
* multiple fault in user mode.
*/
printf("kernel trap %d with interrupts disabled\n",
type);
/*
* Page faults need interrupts disabled until later,
* and we shouldn't enable interrupts while holding
* a spin lock or if servicing an NMI.
*/
if (type != T_NMI && type != T_PAGEFLT &&
td->td_md.md_spinlock_count == 0)
enable_intr();
}
}
eva = 0;
code = frame->tf_err;
if (type == T_PAGEFLT) {
/*
* For some Cyrix CPUs, %cr2 is clobbered by
* interrupts. This problem is worked around by using
* an interrupt gate for the pagefault handler. We
* are finally ready to read %cr2 and then must
* reenable interrupts.
*
* If we get a page fault while in a critical section, then
* it is most likely a fatal kernel page fault. The kernel
* is already going to panic trying to get a sleep lock to
* do the VM lookup, so just consider it a fatal trap so the
* kernel can print out a useful trap message and even get
* to the debugger.
*
* If we get a page fault while holding a non-sleepable
* lock, then it is most likely a fatal kernel page fault.
* If WITNESS is enabled, then it's going to whine about
* bogus LORs with various VM locks, so just skip to the
* fatal trap handling directly.
*/
eva = rcr2();
if (td->td_critnest != 0 ||
WITNESS_CHECK(WARN_SLEEPOK | WARN_GIANTOK, NULL,
"Kernel page fault") != 0)
trap_fatal(frame, eva);
else
enable_intr();
}
if ((ISPL(frame->tf_cs) == SEL_UPL) ||
((frame->tf_eflags & PSL_VM) &&
!(PCPU_GET(curpcb)->pcb_flags & PCB_VM86CALL))) {
/* user trap */
td->td_pticks = 0;
td->td_frame = frame;
addr = frame->tf_eip;
if (td->td_ucred != p->p_ucred)
cred_update_thread(td);
switch (type) {
case T_PRIVINFLT: /* privileged instruction fault */
i = SIGILL;
ucode = ILL_PRVOPC;
break;
case T_BPTFLT: /* bpt instruction fault */
case T_TRCTRAP: /* trace trap */
enable_intr();
frame->tf_eflags &= ~PSL_T;
i = SIGTRAP;
ucode = (type == T_TRCTRAP ? TRAP_TRACE : TRAP_BRKPT);
break;
case T_ARITHTRAP: /* arithmetic trap */
#ifdef DEV_NPX
ucode = npxtrap();
if (ucode == -1)
goto userout;
#else
ucode = 0;
#endif
i = SIGFPE;
break;
/*
* The following two traps can happen in
* vm86 mode, and, if so, we want to handle
* them specially.
*/
case T_PROTFLT: /* general protection fault */
case T_STKFLT: /* stack fault */
if (frame->tf_eflags & PSL_VM) {
i = vm86_emulate((struct vm86frame *)frame);
if (i == 0)
goto user;
break;
}
i = SIGBUS;
ucode = (type == T_PROTFLT) ? BUS_OBJERR : BUS_ADRERR;
break;
case T_SEGNPFLT: /* segment not present fault */
i = SIGBUS;
ucode = BUS_ADRERR;
break;
case T_TSSFLT: /* invalid TSS fault */
i = SIGBUS;
ucode = BUS_OBJERR;
break;
case T_DOUBLEFLT: /* double fault */
default:
i = SIGBUS;
ucode = BUS_OBJERR;
break;
case T_PAGEFLT: /* page fault */
i = trap_pfault(frame, TRUE, eva);
#if defined(I586_CPU) && !defined(NO_F00F_HACK)
if (i == -2) {
/*
* The f00f hack workaround has triggered, so
* treat the fault as an illegal instruction
* (T_PRIVINFLT) instead of a page fault.
*/
type = frame->tf_trapno = T_PRIVINFLT;
/* Proceed as in that case. */
ucode = ILL_PRVOPC;
i = SIGILL;
break;
}
#endif
if (i == -1)
goto userout;
if (i == 0)
goto user;
if (i == SIGSEGV)
ucode = SEGV_MAPERR;
else {
if (prot_fault_translation == 0) {
/*
* Autodetect.
* This check also covers the images
* without the ABI-tag ELF note.
*/
if (SV_CURPROC_ABI() == SV_ABI_FREEBSD
&& p->p_osrel >= P_OSREL_SIGSEGV) {
i = SIGSEGV;
ucode = SEGV_ACCERR;
} else {
i = SIGBUS;
ucode = BUS_PAGE_FAULT;
}
} else if (prot_fault_translation == 1) {
/*
* Always compat mode.
*/
i = SIGBUS;
ucode = BUS_PAGE_FAULT;
} else {
/*
* Always SIGSEGV mode.
*/
i = SIGSEGV;
ucode = SEGV_ACCERR;
}
}
addr = eva;
break;
case T_DIVIDE: /* integer divide fault */
ucode = FPE_INTDIV;
i = SIGFPE;
break;
#ifdef DEV_ISA
case T_NMI:
#ifdef POWERFAIL_NMI
#ifndef TIMER_FREQ
# define TIMER_FREQ 1193182
#endif
if (time_second - lastalert > 10) {
log(LOG_WARNING, "NMI: power fail\n");
sysbeep(880, hz);
lastalert = time_second;
}
goto userout;
#else /* !POWERFAIL_NMI */
/* machine/parity/power fail/"kitchen sink" faults */
if (isa_nmi(code) == 0) {
#ifdef KDB
/*
* NMI can be hooked up to a pushbutton
* for debugging.
*/
if (kdb_on_nmi) {
printf ("NMI ... going to debugger\n");
kdb_trap(type, 0, frame);
}
#endif /* KDB */
goto userout;
} else if (panic_on_nmi)
panic("NMI indicates hardware failure");
break;
#endif /* POWERFAIL_NMI */
#endif /* DEV_ISA */
case T_OFLOW: /* integer overflow fault */
ucode = FPE_INTOVF;
i = SIGFPE;
break;
case T_BOUND: /* bounds check fault */
ucode = FPE_FLTSUB;
i = SIGFPE;
break;
case T_DNA:
#ifdef DEV_NPX
KASSERT(PCB_USER_FPU(td->td_pcb),
("kernel FPU ctx has leaked"));
/* transparent fault (due to context switch "late") */
if (npxdna())
goto userout;
#endif
uprintf("pid %d killed due to lack of floating point\n",
p->p_pid);
i = SIGKILL;
ucode = 0;
break;
case T_FPOPFLT: /* FPU operand fetch fault */
ucode = ILL_COPROC;
i = SIGILL;
break;
case T_XMMFLT: /* SIMD floating-point exception */
ucode = 0; /* XXX */
i = SIGFPE;
break;
}
} else {
/* kernel trap */
KASSERT(cold || td->td_ucred != NULL,
("kernel trap doesn't have ucred"));
switch (type) {
case T_PAGEFLT: /* page fault */
(void) trap_pfault(frame, FALSE, eva);
goto out;
case T_DNA:
#ifdef DEV_NPX
KASSERT(!PCB_USER_FPU(td->td_pcb),
("Unregistered use of FPU in kernel"));
if (npxdna())
goto out;
#endif
break;
case T_ARITHTRAP: /* arithmetic trap */
case T_XMMFLT: /* SIMD floating-point exception */
case T_FPOPFLT: /* FPU operand fetch fault */
/*
* XXXKIB for now disable any FPU traps in kernel
* handler registration seems to be overkill
*/
trap_fatal(frame, 0);
goto out;
/*
* The following two traps can happen in
* vm86 mode, and, if so, we want to handle
* them specially.
*/
case T_PROTFLT: /* general protection fault */
case T_STKFLT: /* stack fault */
if (frame->tf_eflags & PSL_VM) {
i = vm86_emulate((struct vm86frame *)frame);
if (i != 0)
/*
* returns to original process
*/
vm86_trap((struct vm86frame *)frame);
goto out;
}
if (type == T_STKFLT)
break;
/* FALL THROUGH */
case T_SEGNPFLT: /* segment not present fault */
if (PCPU_GET(curpcb)->pcb_flags & PCB_VM86CALL)
break;
/*
* Invalid %fs's and %gs's can be created using
* procfs or PT_SETREGS or by invalidating the
* underlying LDT entry. This causes a fault
* in kernel mode when the kernel attempts to
* switch contexts. Lose the bad context
* (XXX) so that we can continue, and generate
* a signal.
*/
if (frame->tf_eip == (int)cpu_switch_load_gs) {
PCPU_GET(curpcb)->pcb_gs = 0;
#if 0
PROC_LOCK(p);
kern_psignal(p, SIGBUS);
PROC_UNLOCK(p);
#endif
goto out;
}
if (td->td_intr_nesting_level != 0)
break;
/*
* Invalid segment selectors and out of bounds
* %eip's and %esp's can be set up in user mode.
* This causes a fault in kernel mode when the
* kernel tries to return to user mode. We want
* to get this fault so that we can fix the
* problem here and not have to check all the
* selectors and pointers when the user changes
* them.
*/
if (frame->tf_eip == (int)doreti_iret) {
frame->tf_eip = (int)doreti_iret_fault;
goto out;
}
if (frame->tf_eip == (int)doreti_popl_ds) {
frame->tf_eip = (int)doreti_popl_ds_fault;
goto out;
}
if (frame->tf_eip == (int)doreti_popl_es) {
frame->tf_eip = (int)doreti_popl_es_fault;
goto out;
}
if (frame->tf_eip == (int)doreti_popl_fs) {
frame->tf_eip = (int)doreti_popl_fs_fault;
goto out;
}
if (PCPU_GET(curpcb)->pcb_onfault != NULL) {
frame->tf_eip =
(int)PCPU_GET(curpcb)->pcb_onfault;
goto out;
}
break;
case T_TSSFLT:
/*
* PSL_NT can be set in user mode and isn't cleared
* automatically when the kernel is entered. This
* causes a TSS fault when the kernel attempts to
* `iret' because the TSS link is uninitialized. We
* want to get this fault so that we can fix the
* problem here and not every time the kernel is
* entered.
*/
if (frame->tf_eflags & PSL_NT) {
frame->tf_eflags &= ~PSL_NT;
goto out;
}
break;
case T_TRCTRAP: /* trace trap */
if (frame->tf_eip == (int)IDTVEC(lcall_syscall)) {
/*
* We've just entered system mode via the
* syscall lcall. Continue single stepping
* silently until the syscall handler has
* saved the flags.
*/
goto out;
}
if (frame->tf_eip == (int)IDTVEC(lcall_syscall) + 1) {
/*
* The syscall handler has now saved the
* flags. Stop single stepping it.
*/
frame->tf_eflags &= ~PSL_T;
goto out;
}
/*
* Ignore debug register trace traps due to
* accesses in the user's address space, which
* can happen under several conditions such as
* if a user sets a watchpoint on a buffer and
* then passes that buffer to a system call.
* We still want to get TRCTRAPS for addresses
* in kernel space because that is useful when
* debugging the kernel.
*/
if (user_dbreg_trap() &&
!(PCPU_GET(curpcb)->pcb_flags & PCB_VM86CALL)) {
/*
* Reset breakpoint bits because the
* processor doesn't
*/
load_dr6(rdr6() & 0xfffffff0);
goto out;
}
/*
* FALLTHROUGH (TRCTRAP kernel mode, kernel address)
*/
case T_BPTFLT:
/*
* If KDB is enabled, let it handle the debugger trap.
* Otherwise, debugger traps "can't happen".
*/
#ifdef KDB
if (kdb_trap(type, 0, frame))
goto out;
#endif
break;
#ifdef DEV_ISA
case T_NMI:
#ifdef POWERFAIL_NMI
if (time_second - lastalert > 10) {
log(LOG_WARNING, "NMI: power fail\n");
sysbeep(880, hz);
lastalert = time_second;
}
goto out;
#else /* !POWERFAIL_NMI */
/* machine/parity/power fail/"kitchen sink" faults */
if (isa_nmi(code) == 0) {
#ifdef KDB
/*
* NMI can be hooked up to a pushbutton
* for debugging.
*/
if (kdb_on_nmi) {
printf ("NMI ... going to debugger\n");
kdb_trap(type, 0, frame);
}
#endif /* KDB */
goto out;
} else if (panic_on_nmi == 0)
goto out;
/* FALLTHROUGH */
#endif /* POWERFAIL_NMI */
#endif /* DEV_ISA */
}
trap_fatal(frame, eva);
goto out;
}
/* Translate fault for emulators (e.g. Linux) */
if (*p->p_sysent->sv_transtrap)
i = (*p->p_sysent->sv_transtrap)(i, type);
ksiginfo_init_trap(&ksi);
ksi.ksi_signo = i;
ksi.ksi_code = ucode;
ksi.ksi_addr = (void *)addr;
ksi.ksi_trapno = type;
trapsignal(td, &ksi);
#ifdef DEBUG
if (type <= MAX_TRAP_MSG) {
uprintf("fatal process exception: %s",
trap_msg[type]);
if ((type == T_PAGEFLT) || (type == T_PROTFLT))
uprintf(", fault VA = 0x%lx", (u_long)eva);
uprintf("\n");
}
#endif
user:
userret(td, frame);
mtx_assert(&Giant, MA_NOTOWNED);
KASSERT(PCB_USER_FPU(td->td_pcb),
("Return from trap with kernel FPU ctx leaked"));
userout:
out:
return;
}
void
trap(struct trapframe *frame)
{
struct thread *td = curthread;
struct proc *p = td->td_proc;
int i = 0, ucode = 0, code;
u_int type;
register_t addr = 0;
ksiginfo_t ksi;
PCPU_INC(cnt.v_trap);
type = frame->tf_trapno;
#ifdef SMP
#ifdef STOP_NMI
/* Handler for NMI IPIs used for stopping CPUs. */
if (type == T_NMI) {
if (ipi_nmi_handler() == 0)
goto out;
}
#endif /* STOP_NMI */
#endif /* SMP */
#ifdef KDB
if (kdb_active) {
kdb_reenter();
goto out;
}
#endif
#ifdef HWPMC_HOOKS
/*
* CPU PMCs interrupt using an NMI. If the PMC module is
* active, pass the 'rip' value to the PMC module's interrupt
* handler. A return value of '1' from the handler means that
* the NMI was handled by it and we can return immediately.
*/
if (type == T_NMI && pmc_intr &&
(*pmc_intr)(PCPU_GET(cpuid), frame))
goto out;
#endif
if (type == T_MCHK) {
if (!mca_intr())
trap_fatal(frame, 0);
goto out;
}
#ifdef KDTRACE_HOOKS
/*
* A trap can occur while DTrace executes a probe. Before
* executing the probe, DTrace blocks re-scheduling and sets
* a flag in it's per-cpu flags to indicate that it doesn't
* want to fault. On returning from the the probe, the no-fault
* flag is cleared and finally re-scheduling is enabled.
*
* If the DTrace kernel module has registered a trap handler,
* call it and if it returns non-zero, assume that it has
* handled the trap and modified the trap frame so that this
* function can return normally.
*/
if (dtrace_trap_func != NULL)
if ((*dtrace_trap_func)(frame, type))
goto out;
#endif
if ((frame->tf_rflags & PSL_I) == 0) {
/*
* Buggy application or kernel code has disabled
* interrupts and then trapped. Enabling interrupts
* now is wrong, but it is better than running with
* interrupts disabled until they are accidentally
* enabled later.
*/
if (ISPL(frame->tf_cs) == SEL_UPL)
printf(
"pid %ld (%s): trap %d with interrupts disabled\n",
(long)curproc->p_pid, curproc->p_comm, type);
else if (type != T_NMI && type != T_BPTFLT &&
type != T_TRCTRAP) {
/*
* XXX not quite right, since this may be for a
* multiple fault in user mode.
*/
printf("kernel trap %d with interrupts disabled\n",
type);
/*
* We shouldn't enable interrupts while holding a
* spin lock or servicing an NMI.
*/
if (type != T_NMI && td->td_md.md_spinlock_count == 0)
enable_intr();
}
}
code = frame->tf_err;
if (type == T_PAGEFLT) {
/*
* If we get a page fault while in a critical section, then
* it is most likely a fatal kernel page fault. The kernel
* is already going to panic trying to get a sleep lock to
* do the VM lookup, so just consider it a fatal trap so the
* kernel can print out a useful trap message and even get
* to the debugger.
*
* If we get a page fault while holding a non-sleepable
* lock, then it is most likely a fatal kernel page fault.
* If WITNESS is enabled, then it's going to whine about
* bogus LORs with various VM locks, so just skip to the
* fatal trap handling directly.
*/
if (td->td_critnest != 0 ||
WITNESS_CHECK(WARN_SLEEPOK | WARN_GIANTOK, NULL,
"Kernel page fault") != 0)
trap_fatal(frame, frame->tf_addr);
}
if (ISPL(frame->tf_cs) == SEL_UPL) {
/* user trap */
td->td_pticks = 0;
td->td_frame = frame;
addr = frame->tf_rip;
if (td->td_ucred != p->p_ucred)
cred_update_thread(td);
switch (type) {
case T_PRIVINFLT: /* privileged instruction fault */
i = SIGILL;
ucode = ILL_PRVOPC;
break;
case T_BPTFLT: /* bpt instruction fault */
case T_TRCTRAP: /* trace trap */
enable_intr();
frame->tf_rflags &= ~PSL_T;
i = SIGTRAP;
ucode = (type == T_TRCTRAP ? TRAP_TRACE : TRAP_BRKPT);
break;
case T_ARITHTRAP: /* arithmetic trap */
ucode = fputrap();
if (ucode == -1)
goto userout;
i = SIGFPE;
break;
case T_PROTFLT: /* general protection fault */
i = SIGBUS;
ucode = BUS_OBJERR;
break;
case T_STKFLT: /* stack fault */
case T_SEGNPFLT: /* segment not present fault */
i = SIGBUS;
ucode = BUS_ADRERR;
break;
case T_TSSFLT: /* invalid TSS fault */
i = SIGBUS;
ucode = BUS_OBJERR;
break;
case T_DOUBLEFLT: /* double fault */
default:
i = SIGBUS;
ucode = BUS_OBJERR;
break;
case T_PAGEFLT: /* page fault */
addr = frame->tf_addr;
#ifdef KSE
if (td->td_pflags & TDP_SA)
thread_user_enter(td);
#endif
i = trap_pfault(frame, TRUE);
if (i == -1)
goto userout;
if (i == 0)
goto user;
if (i == SIGSEGV)
ucode = SEGV_MAPERR;
else {
if (prot_fault_translation == 0) {
/*
* Autodetect.
* This check also covers the images
* without the ABI-tag ELF note.
*/
if ((curproc->p_sysent ==
&elf64_freebsd_sysvec
#ifdef COMPAT_IA32
|| curproc->p_sysent ==
&ia32_freebsd_sysvec
#endif
) && p->p_osrel >= 700004) {
i = SIGSEGV;
ucode = SEGV_ACCERR;
} else {
i = SIGBUS;
ucode = BUS_PAGE_FAULT;
}
} else if (prot_fault_translation == 1) {
/*
* Always compat mode.
*/
i = SIGBUS;
ucode = BUS_PAGE_FAULT;
} else {
/*
* Always SIGSEGV mode.
*/
i = SIGSEGV;
ucode = SEGV_ACCERR;
}
}
break;
case T_DIVIDE: /* integer divide fault */
ucode = FPE_INTDIV;
i = SIGFPE;
break;
#ifdef DEV_ISA
case T_NMI:
/* machine/parity/power fail/"kitchen sink" faults */
/* XXX Giant */
if (isa_nmi(code) == 0) {
#ifdef KDB
/*
* NMI can be hooked up to a pushbutton
* for debugging.
*/
if (kdb_on_nmi) {
printf ("NMI ... going to debugger\n");
kdb_trap(type, 0, frame);
}
#endif /* KDB */
goto userout;
} else if (panic_on_nmi)
panic("NMI indicates hardware failure");
break;
#endif /* DEV_ISA */
case T_OFLOW: /* integer overflow fault */
ucode = FPE_INTOVF;
i = SIGFPE;
break;
case T_BOUND: /* bounds check fault */
ucode = FPE_FLTSUB;
i = SIGFPE;
break;
case T_DNA:
/* transparent fault (due to context switch "late") */
fpudna();
goto userout;
case T_FPOPFLT: /* FPU operand fetch fault */
ucode = ILL_COPROC;
i = SIGILL;
break;
case T_XMMFLT: /* SIMD floating-point exception */
ucode = 0; /* XXX */
i = SIGFPE;
break;
}
} else {
/* kernel trap */
KASSERT(cold || td->td_ucred != NULL,
("kernel trap doesn't have ucred"));
switch (type) {
case T_PAGEFLT: /* page fault */
(void) trap_pfault(frame, FALSE);
goto out;
case T_DNA:
/*
* The kernel is apparently using fpu for copying.
* XXX this should be fatal unless the kernel has
* registered such use.
*/
fpudna();
printf("fpudna in kernel mode!\n");
goto out;
case T_STKFLT: /* stack fault */
break;
case T_PROTFLT: /* general protection fault */
case T_SEGNPFLT: /* segment not present fault */
if (td->td_intr_nesting_level != 0)
break;
/*
* Invalid segment selectors and out of bounds
* %rip's and %rsp's can be set up in user mode.
* This causes a fault in kernel mode when the
* kernel tries to return to user mode. We want
* to get this fault so that we can fix the
* problem here and not have to check all the
* selectors and pointers when the user changes
* them.
*/
if (frame->tf_rip == (long)doreti_iret) {
frame->tf_rip = (long)doreti_iret_fault;
goto out;
}
if (PCPU_GET(curpcb)->pcb_onfault != NULL) {
frame->tf_rip =
(long)PCPU_GET(curpcb)->pcb_onfault;
goto out;
}
break;
case T_TSSFLT:
/*
* PSL_NT can be set in user mode and isn't cleared
* automatically when the kernel is entered. This
* causes a TSS fault when the kernel attempts to
* `iret' because the TSS link is uninitialized. We
* want to get this fault so that we can fix the
* problem here and not every time the kernel is
* entered.
*/
if (frame->tf_rflags & PSL_NT) {
frame->tf_rflags &= ~PSL_NT;
goto out;
}
break;
case T_TRCTRAP: /* trace trap */
/*
* Ignore debug register trace traps due to
* accesses in the user's address space, which
* can happen under several conditions such as
* if a user sets a watchpoint on a buffer and
* then passes that buffer to a system call.
* We still want to get TRCTRAPS for addresses
* in kernel space because that is useful when
* debugging the kernel.
*/
if (user_dbreg_trap()) {
/*
* Reset breakpoint bits because the
* processor doesn't
*/
/* XXX check upper bits here */
load_dr6(rdr6() & 0xfffffff0);
goto out;
}
/*
* FALLTHROUGH (TRCTRAP kernel mode, kernel address)
*/
case T_BPTFLT:
/*
* If KDB is enabled, let it handle the debugger trap.
* Otherwise, debugger traps "can't happen".
*/
#ifdef KDB
if (kdb_trap(type, 0, frame))
goto out;
#endif
break;
#ifdef DEV_ISA
case T_NMI:
/* XXX Giant */
/* machine/parity/power fail/"kitchen sink" faults */
if (isa_nmi(code) == 0) {
#ifdef KDB
/*
* NMI can be hooked up to a pushbutton
* for debugging.
*/
if (kdb_on_nmi) {
printf ("NMI ... going to debugger\n");
kdb_trap(type, 0, frame);
}
#endif /* KDB */
goto out;
} else if (panic_on_nmi == 0)
goto out;
/* FALLTHROUGH */
#endif /* DEV_ISA */
}
trap_fatal(frame, 0);
goto out;
}
/* Translate fault for emulators (e.g. Linux) */
if (*p->p_sysent->sv_transtrap)
i = (*p->p_sysent->sv_transtrap)(i, type);
ksiginfo_init_trap(&ksi);
ksi.ksi_signo = i;
ksi.ksi_code = ucode;
ksi.ksi_trapno = type;
ksi.ksi_addr = (void *)addr;
trapsignal(td, &ksi);
#ifdef DEBUG
if (type <= MAX_TRAP_MSG) {
uprintf("fatal process exception: %s",
trap_msg[type]);
if ((type == T_PAGEFLT) || (type == T_PROTFLT))
uprintf(", fault VA = 0x%lx", frame->tf_addr);
uprintf("\n");
}
#endif
user:
userret(td, frame);
mtx_assert(&Giant, MA_NOTOWNED);
userout:
out:
return;
}
