void
trap(struct trapframe *frame)
{
#ifdef KDTRACE_HOOKS
struct reg regs;
#endif
struct thread *td = curthread;
struct proc *p = td->td_proc;
int i = 0, ucode = 0, code;
u_int type;
register_t addr = 0;
ksiginfo_t ksi;
PCPU_INC(cnt.v_trap);
type = frame->tf_trapno;
#ifdef SMP
/* Handler for NMI IPIs used for stopping CPUs. */
if (type == T_NMI) {
if (ipi_nmi_handler() == 0)
goto out;
}
#endif /* SMP */
#ifdef KDB
if (kdb_active) {
kdb_reenter();
goto out;
}
#endif
if (type == T_RESERVED) {
trap_fatal(frame, 0);
goto out;
}
#ifdef HWPMC_HOOKS
/*
* CPU PMCs interrupt using an NMI. If the PMC module is
* active, pass the 'rip' value to the PMC module's interrupt
* handler. A return value of '1' from the handler means that
* the NMI was handled by it and we can return immediately.
*/
if (type == T_NMI && pmc_intr &&
(*pmc_intr)(PCPU_GET(cpuid), frame))
goto out;
#endif
if (type == T_MCHK) {
mca_intr();
goto out;
}
#ifdef KDTRACE_HOOKS
/*
* A trap can occur while DTrace executes a probe. Before
* executing the probe, DTrace blocks re-scheduling and sets
* a flag in its per-cpu flags to indicate that it doesn't
* want to fault. On returning from the probe, the no-fault
* flag is cleared and finally re-scheduling is enabled.
*/
if (dtrace_trap_func != NULL && (*dtrace_trap_func)(frame, type))
goto out;
#endif
if ((frame->tf_rflags & PSL_I) == 0) {
/*
* Buggy application or kernel code has disabled
* interrupts and then trapped. Enabling interrupts
* now is wrong, but it is better than running with
* interrupts disabled until they are accidentally
* enabled later.
*/
if (ISPL(frame->tf_cs) == SEL_UPL)
uprintf(
"pid %ld (%s): trap %d with interrupts disabled\n",
(long)curproc->p_pid, curthread->td_name, type);
else if (type != T_NMI && type != T_BPTFLT &&
type != T_TRCTRAP) {
/*
* XXX not quite right, since this may be for a
* multiple fault in user mode.
*/
printf("kernel trap %d with interrupts disabled\n",
type);
/*
* We shouldn't enable interrupts while holding a
* spin lock.
*/
if (td->td_md.md_spinlock_count == 0)
enable_intr();
}
}
code = frame->tf_err;
if (ISPL(frame->tf_cs) == SEL_UPL) {
/* user trap */
td->td_pticks = 0;
td->td_frame = frame;
addr = frame->tf_rip;
if (td->td_ucred != p->p_ucred)
cred_update_thread(td);
switch (type) {
case T_PRIVINFLT: /* privileged instruction fault */
i = SIGILL;
ucode = ILL_PRVOPC;
break;
case T_BPTFLT: /* bpt instruction fault */
case T_TRCTRAP: /* trace trap */
enable_intr();
#ifdef KDTRACE_HOOKS
if (type == T_BPTFLT) {
fill_frame_regs(frame, ®s);
if (dtrace_pid_probe_ptr != NULL &&
dtrace_pid_probe_ptr(®s) == 0)
goto out;
}
#endif
frame->tf_rflags &= ~PSL_T;
i = SIGTRAP;
ucode = (type == T_TRCTRAP ? TRAP_TRACE : TRAP_BRKPT);
break;
case T_ARITHTRAP: /* arithmetic trap */
ucode = fputrap_x87();
if (ucode == -1)
goto userout;
i = SIGFPE;
break;
case T_PROTFLT: /* general protection fault */
i = SIGBUS;
ucode = BUS_OBJERR;
break;
case T_STKFLT: /* stack fault */
case T_SEGNPFLT: /* segment not present fault */
i = SIGBUS;
ucode = BUS_ADRERR;
break;
case T_TSSFLT: /* invalid TSS fault */
i = SIGBUS;
ucode = BUS_OBJERR;
break;
case T_DOUBLEFLT: /* double fault */
default:
i = SIGBUS;
ucode = BUS_OBJERR;
break;
case T_PAGEFLT: /* page fault */
addr = frame->tf_addr;
i = trap_pfault(frame, TRUE);
if (i == -1)
goto userout;
if (i == 0)
goto user;
if (i == SIGSEGV)
ucode = SEGV_MAPERR;
else {
if (prot_fault_translation == 0) {
/*
* Autodetect.
* This check also covers the images
* without the ABI-tag ELF note.
*/
if (SV_CURPROC_ABI() == SV_ABI_FREEBSD
&& p->p_osrel >= P_OSREL_SIGSEGV) {
i = SIGSEGV;
ucode = SEGV_ACCERR;
} else {
i = SIGBUS;
ucode = BUS_PAGE_FAULT;
}
} else if (prot_fault_translation == 1) {
/*
* Always compat mode.
*/
i = SIGBUS;
ucode = BUS_PAGE_FAULT;
} else {
/*
* Always SIGSEGV mode.
*/
i = SIGSEGV;
ucode = SEGV_ACCERR;
}
}
break;
case T_DIVIDE: /* integer divide fault */
ucode = FPE_INTDIV;
i = SIGFPE;
break;
#ifdef DEV_ISA
case T_NMI:
/* machine/parity/power fail/"kitchen sink" faults */
if (isa_nmi(code) == 0) {
#ifdef KDB
/*
* NMI can be hooked up to a pushbutton
* for debugging.
*/
if (kdb_on_nmi) {
printf ("NMI ... going to debugger\n");
kdb_trap(type, 0, frame);
}
#endif /* KDB */
goto userout;
} else if (panic_on_nmi)
panic("NMI indicates hardware failure");
break;
#endif /* DEV_ISA */
case T_OFLOW: /* integer overflow fault */
ucode = FPE_INTOVF;
i = SIGFPE;
break;
case T_BOUND: /* bounds check fault */
ucode = FPE_FLTSUB;
i = SIGFPE;
break;
case T_DNA:
/* transparent fault (due to context switch "late") */
KASSERT(PCB_USER_FPU(td->td_pcb),
("kernel FPU ctx has leaked"));
fpudna();
goto userout;
case T_FPOPFLT: /* FPU operand fetch fault */
ucode = ILL_COPROC;
i = SIGILL;
break;
case T_XMMFLT: /* SIMD floating-point exception */
ucode = fputrap_sse();
if (ucode == -1)
goto userout;
i = SIGFPE;
break;
#ifdef KDTRACE_HOOKS
case T_DTRACE_RET:
enable_intr();
fill_frame_regs(frame, ®s);
if (dtrace_return_probe_ptr != NULL &&
dtrace_return_probe_ptr(®s) == 0)
goto out;
break;
#endif
}
} else {
/* kernel trap */
KASSERT(cold || td->td_ucred != NULL,
("kernel trap doesn't have ucred"));
switch (type) {
case T_PAGEFLT: /* page fault */
(void) trap_pfault(frame, FALSE);
goto out;
case T_DNA:
KASSERT(!PCB_USER_FPU(td->td_pcb),
("Unregistered use of FPU in kernel"));
fpudna();
goto out;
case T_ARITHTRAP: /* arithmetic trap */
case T_XMMFLT: /* SIMD floating-point exception */
case T_FPOPFLT: /* FPU operand fetch fault */
/*
* XXXKIB for now disable any FPU traps in kernel
* handler registration seems to be overkill
*/
trap_fatal(frame, 0);
goto out;
case T_STKFLT: /* stack fault */
break;
case T_PROTFLT: /* general protection fault */
case T_SEGNPFLT: /* segment not present fault */
if (td->td_intr_nesting_level != 0)
break;
/*
* Invalid segment selectors and out of bounds
* %rip's and %rsp's can be set up in user mode.
* This causes a fault in kernel mode when the
* kernel tries to return to user mode. We want
* to get this fault so that we can fix the
* problem here and not have to check all the
* selectors and pointers when the user changes
* them.
*/
if (frame->tf_rip == (long)doreti_iret) {
frame->tf_rip = (long)doreti_iret_fault;
goto out;
}
if (frame->tf_rip == (long)ld_ds) {
frame->tf_rip = (long)ds_load_fault;
goto out;
}
if (frame->tf_rip == (long)ld_es) {
frame->tf_rip = (long)es_load_fault;
goto out;
}
if (frame->tf_rip == (long)ld_fs) {
frame->tf_rip = (long)fs_load_fault;
goto out;
}
if (frame->tf_rip == (long)ld_gs) {
frame->tf_rip = (long)gs_load_fault;
goto out;
}
if (frame->tf_rip == (long)ld_gsbase) {
frame->tf_rip = (long)gsbase_load_fault;
goto out;
}
if (frame->tf_rip == (long)ld_fsbase) {
frame->tf_rip = (long)fsbase_load_fault;
goto out;
}
if (curpcb->pcb_onfault != NULL) {
frame->tf_rip = (long)curpcb->pcb_onfault;
goto out;
}
break;
case T_TSSFLT:
/*
* PSL_NT can be set in user mode and isn't cleared
* automatically when the kernel is entered. This
* causes a TSS fault when the kernel attempts to
* `iret' because the TSS link is uninitialized. We
* want to get this fault so that we can fix the
* problem here and not every time the kernel is
* entered.
*/
if (frame->tf_rflags & PSL_NT) {
frame->tf_rflags &= ~PSL_NT;
goto out;
}
break;
case T_TRCTRAP: /* trace trap */
/*
* Ignore debug register trace traps due to
* accesses in the user's address space, which
* can happen under several conditions such as
* if a user sets a watchpoint on a buffer and
* then passes that buffer to a system call.
* We still want to get TRCTRAPS for addresses
* in kernel space because that is useful when
* debugging the kernel.
*/
if (user_dbreg_trap()) {
/*
* Reset breakpoint bits because the
* processor doesn't
*/
/* XXX check upper bits here */
load_dr6(rdr6() & 0xfffffff0);
goto out;
}
/*
* FALLTHROUGH (TRCTRAP kernel mode, kernel address)
*/
case T_BPTFLT:
/*
* If KDB is enabled, let it handle the debugger trap.
* Otherwise, debugger traps "can't happen".
*/
#ifdef KDB
if (kdb_trap(type, 0, frame))
goto out;
#endif
break;
#ifdef DEV_ISA
case T_NMI:
/* machine/parity/power fail/"kitchen sink" faults */
if (isa_nmi(code) == 0) {
#ifdef KDB
/*
* NMI can be hooked up to a pushbutton
* for debugging.
*/
if (kdb_on_nmi) {
printf ("NMI ... going to debugger\n");
kdb_trap(type, 0, frame);
}
#endif /* KDB */
goto out;
} else if (panic_on_nmi == 0)
goto out;
/* FALLTHROUGH */
#endif /* DEV_ISA */
}
trap_fatal(frame, 0);
goto out;
}
/* Translate fault for emulators (e.g. Linux) */
if (*p->p_sysent->sv_transtrap)
i = (*p->p_sysent->sv_transtrap)(i, type);
ksiginfo_init_trap(&ksi);
ksi.ksi_signo = i;
ksi.ksi_code = ucode;
ksi.ksi_trapno = type;
ksi.ksi_addr = (void *)addr;
if (uprintf_signal) {
uprintf("pid %d comm %s: signal %d err %lx code %d type %d "
"addr 0x%lx rsp 0x%lx rip 0x%lx "
"<%02x %02x %02x %02x %02x %02x %02x %02x>\n",
p->p_pid, p->p_comm, i, frame->tf_err, ucode, type, addr,
frame->tf_rsp, frame->tf_rip,
fubyte((void *)(frame->tf_rip + 0)),
fubyte((void *)(frame->tf_rip + 1)),
fubyte((void *)(frame->tf_rip + 2)),
fubyte((void *)(frame->tf_rip + 3)),
fubyte((void *)(frame->tf_rip + 4)),
fubyte((void *)(frame->tf_rip + 5)),
fubyte((void *)(frame->tf_rip + 6)),
fubyte((void *)(frame->tf_rip + 7)));
}
KASSERT((read_rflags() & PSL_I) != 0, ("interrupts disabled"));
trapsignal(td, &ksi);
user:
userret(td, frame);
KASSERT(PCB_USER_FPU(td->td_pcb),
("Return from trap with kernel FPU ctx leaked"));
userout:
out:
return;
}
/*
* Process an asynchronous software trap.
* This is relatively easy.
* This function will return with preemption disabled.
*/
void
ast(struct trapframe *framep)
{
struct thread *td;
struct proc *p;
int flags;
int sig;
td = curthread;
p = td->td_proc;
CTR3(KTR_SYSC, "ast: thread %p (pid %d, %s)", td, p->p_pid,
p->p_comm);
KASSERT(TRAPF_USERMODE(framep), ("ast in kernel mode"));
WITNESS_WARN(WARN_PANIC, NULL, "Returning to user mode");
mtx_assert(&Giant, MA_NOTOWNED);
THREAD_LOCK_ASSERT(td, MA_NOTOWNED);
td->td_frame = framep;
td->td_pticks = 0;
/*
* This updates the td_flag's for the checks below in one
* "atomic" operation with turning off the astpending flag.
* If another AST is triggered while we are handling the
* AST's saved in flags, the astpending flag will be set and
* ast() will be called again.
*/
thread_lock(td);
flags = td->td_flags;
td->td_flags &= ~(TDF_ASTPENDING | TDF_NEEDSIGCHK | TDF_NEEDSUSPCHK |
TDF_NEEDRESCHED | TDF_ALRMPEND | TDF_PROFPEND | TDF_MACPEND);
thread_unlock(td);
PCPU_INC(cnt.v_trap);
if (td->td_ucred != p->p_ucred)
cred_update_thread(td);
if (td->td_pflags & TDP_OWEUPC && p->p_flag & P_PROFIL) {
addupc_task(td, td->td_profil_addr, td->td_profil_ticks);
td->td_profil_ticks = 0;
td->td_pflags &= ~TDP_OWEUPC;
}
#ifdef HWPMC_HOOKS
/* Handle Software PMC callchain capture. */
if (PMC_IS_PENDING_CALLCHAIN(td))
PMC_CALL_HOOK_UNLOCKED(td, PMC_FN_USER_CALLCHAIN_SOFT, (void *) framep);
#endif
if (flags & TDF_ALRMPEND) {
PROC_LOCK(p);
kern_psignal(p, SIGVTALRM);
PROC_UNLOCK(p);
}
if (flags & TDF_PROFPEND) {
PROC_LOCK(p);
kern_psignal(p, SIGPROF);
PROC_UNLOCK(p);
}
#ifdef MAC
if (flags & TDF_MACPEND)
mac_thread_userret(td);
#endif
if (flags & TDF_NEEDRESCHED) {
#ifdef KTRACE
if (KTRPOINT(td, KTR_CSW))
ktrcsw(1, 1, __func__);
#endif
thread_lock(td);
sched_prio(td, td->td_user_pri);
mi_switch(SW_INVOL | SWT_NEEDRESCHED, NULL);
thread_unlock(td);
#ifdef KTRACE
if (KTRPOINT(td, KTR_CSW))
ktrcsw(0, 1, __func__);
#endif
}
/*
* Check for signals. Unlocked reads of p_pendingcnt or
* p_siglist might cause process-directed signal to be handled
* later.
*/
if (flags & TDF_NEEDSIGCHK || p->p_pendingcnt > 0 ||
!SIGISEMPTY(p->p_siglist)) {
PROC_LOCK(p);
mtx_lock(&p->p_sigacts->ps_mtx);
while ((sig = cursig(td)) != 0)
postsig(sig);
mtx_unlock(&p->p_sigacts->ps_mtx);
PROC_UNLOCK(p);
}
/*
* We need to check to see if we have to exit or wait due to a
* single threading requirement or some other STOP condition.
*/
if (flags & TDF_NEEDSUSPCHK) {
PROC_LOCK(p);
thread_suspend_check(0);
PROC_UNLOCK(p);
}
if (td->td_pflags & TDP_OLDMASK) {
td->td_pflags &= ~TDP_OLDMASK;
kern_sigprocmask(td, SIG_SETMASK, &td->td_oldsigmask, NULL, 0);
}
userret(td, framep);
}
void
trap(struct trapframe *frame)
{
struct thread *td = curthread;
struct proc *p = td->td_proc;
int i = 0, ucode = 0, code;
u_int type;
register_t addr = 0;
vm_offset_t eva;
ksiginfo_t ksi;
#ifdef POWERFAIL_NMI
static int lastalert = 0;
#endif
PCPU_INC(cnt.v_trap);
type = frame->tf_trapno;
#ifdef SMP
/* Handler for NMI IPIs used for stopping CPUs. */
if (type == T_NMI) {
if (ipi_nmi_handler() == 0)
goto out;
}
#endif /* SMP */
#ifdef KDB
if (kdb_active) {
kdb_reenter();
goto out;
}
#endif
if (type == T_RESERVED) {
trap_fatal(frame, 0);
goto out;
}
#ifdef HWPMC_HOOKS
/*
* CPU PMCs interrupt using an NMI so we check for that first.
* If the HWPMC module is active, 'pmc_hook' will point to
* the function to be called. A return value of '1' from the
* hook means that the NMI was handled by it and that we can
* return immediately.
*/
if (type == T_NMI && pmc_intr &&
(*pmc_intr)(PCPU_GET(cpuid), frame))
goto out;
#endif
if (type == T_MCHK) {
if (!mca_intr())
trap_fatal(frame, 0);
goto out;
}
#ifdef KDTRACE_HOOKS
/*
* A trap can occur while DTrace executes a probe. Before
* executing the probe, DTrace blocks re-scheduling and sets
* a flag in it's per-cpu flags to indicate that it doesn't
* want to fault. On returning from the probe, the no-fault
* flag is cleared and finally re-scheduling is enabled.
*
* If the DTrace kernel module has registered a trap handler,
* call it and if it returns non-zero, assume that it has
* handled the trap and modified the trap frame so that this
* function can return normally.
*/
if ((type == T_PROTFLT || type == T_PAGEFLT) &&
dtrace_trap_func != NULL)
if ((*dtrace_trap_func)(frame, type))
goto out;
if (type == T_DTRACE_PROBE || type == T_DTRACE_RET ||
type == T_BPTFLT) {
struct reg regs;
fill_frame_regs(frame, ®s);
if (type == T_DTRACE_PROBE &&
dtrace_fasttrap_probe_ptr != NULL &&
dtrace_fasttrap_probe_ptr(®s) == 0)
goto out;
if (type == T_BPTFLT &&
dtrace_pid_probe_ptr != NULL &&
dtrace_pid_probe_ptr(®s) == 0)
goto out;
if (type == T_DTRACE_RET &&
dtrace_return_probe_ptr != NULL &&
dtrace_return_probe_ptr(®s) == 0)
goto out;
}
#endif
if ((frame->tf_eflags & PSL_I) == 0) {
/*
* Buggy application or kernel code has disabled
* interrupts and then trapped. Enabling interrupts
* now is wrong, but it is better than running with
* interrupts disabled until they are accidentally
* enabled later.
*/
if (ISPL(frame->tf_cs) == SEL_UPL || (frame->tf_eflags & PSL_VM))
uprintf(
"pid %ld (%s): trap %d with interrupts disabled\n",
(long)curproc->p_pid, curthread->td_name, type);
else if (type != T_BPTFLT && type != T_TRCTRAP &&
frame->tf_eip != (int)cpu_switch_load_gs) {
/*
* XXX not quite right, since this may be for a
* multiple fault in user mode.
*/
printf("kernel trap %d with interrupts disabled\n",
type);
/*
* Page faults need interrupts disabled until later,
* and we shouldn't enable interrupts while holding
* a spin lock or if servicing an NMI.
*/
if (type != T_NMI && type != T_PAGEFLT &&
td->td_md.md_spinlock_count == 0)
enable_intr();
}
}
eva = 0;
code = frame->tf_err;
if (type == T_PAGEFLT) {
/*
* For some Cyrix CPUs, %cr2 is clobbered by
* interrupts. This problem is worked around by using
* an interrupt gate for the pagefault handler. We
* are finally ready to read %cr2 and then must
* reenable interrupts.
*
* If we get a page fault while in a critical section, then
* it is most likely a fatal kernel page fault. The kernel
* is already going to panic trying to get a sleep lock to
* do the VM lookup, so just consider it a fatal trap so the
* kernel can print out a useful trap message and even get
* to the debugger.
*
* If we get a page fault while holding a non-sleepable
* lock, then it is most likely a fatal kernel page fault.
* If WITNESS is enabled, then it's going to whine about
* bogus LORs with various VM locks, so just skip to the
* fatal trap handling directly.
*/
eva = rcr2();
if (td->td_critnest != 0 ||
WITNESS_CHECK(WARN_SLEEPOK | WARN_GIANTOK, NULL,
"Kernel page fault") != 0)
trap_fatal(frame, eva);
else
enable_intr();
}
if ((ISPL(frame->tf_cs) == SEL_UPL) ||
((frame->tf_eflags & PSL_VM) &&
!(PCPU_GET(curpcb)->pcb_flags & PCB_VM86CALL))) {
/* user trap */
td->td_pticks = 0;
td->td_frame = frame;
addr = frame->tf_eip;
if (td->td_ucred != p->p_ucred)
cred_update_thread(td);
switch (type) {
case T_PRIVINFLT: /* privileged instruction fault */
i = SIGILL;
ucode = ILL_PRVOPC;
break;
case T_BPTFLT: /* bpt instruction fault */
case T_TRCTRAP: /* trace trap */
enable_intr();
frame->tf_eflags &= ~PSL_T;
i = SIGTRAP;
ucode = (type == T_TRCTRAP ? TRAP_TRACE : TRAP_BRKPT);
break;
case T_ARITHTRAP: /* arithmetic trap */
#ifdef DEV_NPX
ucode = npxtrap();
if (ucode == -1)
goto userout;
#else
ucode = 0;
#endif
i = SIGFPE;
break;
/*
* The following two traps can happen in
* vm86 mode, and, if so, we want to handle
* them specially.
*/
case T_PROTFLT: /* general protection fault */
case T_STKFLT: /* stack fault */
if (frame->tf_eflags & PSL_VM) {
i = vm86_emulate((struct vm86frame *)frame);
if (i == 0)
goto user;
break;
}
i = SIGBUS;
ucode = (type == T_PROTFLT) ? BUS_OBJERR : BUS_ADRERR;
break;
case T_SEGNPFLT: /* segment not present fault */
i = SIGBUS;
ucode = BUS_ADRERR;
break;
case T_TSSFLT: /* invalid TSS fault */
i = SIGBUS;
ucode = BUS_OBJERR;
break;
case T_DOUBLEFLT: /* double fault */
default:
i = SIGBUS;
ucode = BUS_OBJERR;
break;
case T_PAGEFLT: /* page fault */
i = trap_pfault(frame, TRUE, eva);
#if defined(I586_CPU) && !defined(NO_F00F_HACK)
if (i == -2) {
/*
* The f00f hack workaround has triggered, so
* treat the fault as an illegal instruction
* (T_PRIVINFLT) instead of a page fault.
*/
type = frame->tf_trapno = T_PRIVINFLT;
/* Proceed as in that case. */
ucode = ILL_PRVOPC;
i = SIGILL;
break;
}
#endif
if (i == -1)
goto userout;
if (i == 0)
goto user;
if (i == SIGSEGV)
ucode = SEGV_MAPERR;
else {
if (prot_fault_translation == 0) {
/*
* Autodetect.
* This check also covers the images
* without the ABI-tag ELF note.
*/
if (SV_CURPROC_ABI() == SV_ABI_FREEBSD
&& p->p_osrel >= P_OSREL_SIGSEGV) {
i = SIGSEGV;
ucode = SEGV_ACCERR;
} else {
i = SIGBUS;
ucode = BUS_PAGE_FAULT;
}
} else if (prot_fault_translation == 1) {
/*
* Always compat mode.
*/
i = SIGBUS;
ucode = BUS_PAGE_FAULT;
} else {
/*
* Always SIGSEGV mode.
*/
i = SIGSEGV;
ucode = SEGV_ACCERR;
}
}
addr = eva;
break;
case T_DIVIDE: /* integer divide fault */
ucode = FPE_INTDIV;
i = SIGFPE;
break;
#ifdef DEV_ISA
case T_NMI:
#ifdef POWERFAIL_NMI
#ifndef TIMER_FREQ
# define TIMER_FREQ 1193182
#endif
if (time_second - lastalert > 10) {
log(LOG_WARNING, "NMI: power fail\n");
sysbeep(880, hz);
lastalert = time_second;
}
goto userout;
#else /* !POWERFAIL_NMI */
/* machine/parity/power fail/"kitchen sink" faults */
if (isa_nmi(code) == 0) {
#ifdef KDB
/*
* NMI can be hooked up to a pushbutton
* for debugging.
*/
if (kdb_on_nmi) {
printf ("NMI ... going to debugger\n");
kdb_trap(type, 0, frame);
}
#endif /* KDB */
goto userout;
} else if (panic_on_nmi)
panic("NMI indicates hardware failure");
break;
#endif /* POWERFAIL_NMI */
#endif /* DEV_ISA */
case T_OFLOW: /* integer overflow fault */
ucode = FPE_INTOVF;
i = SIGFPE;
break;
case T_BOUND: /* bounds check fault */
ucode = FPE_FLTSUB;
i = SIGFPE;
break;
case T_DNA:
#ifdef DEV_NPX
KASSERT(PCB_USER_FPU(td->td_pcb),
("kernel FPU ctx has leaked"));
/* transparent fault (due to context switch "late") */
if (npxdna())
goto userout;
#endif
uprintf("pid %d killed due to lack of floating point\n",
p->p_pid);
i = SIGKILL;
ucode = 0;
break;
case T_FPOPFLT: /* FPU operand fetch fault */
ucode = ILL_COPROC;
i = SIGILL;
break;
case T_XMMFLT: /* SIMD floating-point exception */
ucode = 0; /* XXX */
i = SIGFPE;
break;
}
} else {
/* kernel trap */
KASSERT(cold || td->td_ucred != NULL,
("kernel trap doesn't have ucred"));
switch (type) {
case T_PAGEFLT: /* page fault */
(void) trap_pfault(frame, FALSE, eva);
goto out;
case T_DNA:
#ifdef DEV_NPX
KASSERT(!PCB_USER_FPU(td->td_pcb),
("Unregistered use of FPU in kernel"));
if (npxdna())
goto out;
#endif
break;
case T_ARITHTRAP: /* arithmetic trap */
case T_XMMFLT: /* SIMD floating-point exception */
case T_FPOPFLT: /* FPU operand fetch fault */
/*
* XXXKIB for now disable any FPU traps in kernel
* handler registration seems to be overkill
*/
trap_fatal(frame, 0);
goto out;
/*
* The following two traps can happen in
* vm86 mode, and, if so, we want to handle
* them specially.
*/
case T_PROTFLT: /* general protection fault */
case T_STKFLT: /* stack fault */
if (frame->tf_eflags & PSL_VM) {
i = vm86_emulate((struct vm86frame *)frame);
if (i != 0)
/*
* returns to original process
*/
vm86_trap((struct vm86frame *)frame);
goto out;
}
if (type == T_STKFLT)
break;
/* FALL THROUGH */
case T_SEGNPFLT: /* segment not present fault */
if (PCPU_GET(curpcb)->pcb_flags & PCB_VM86CALL)
break;
/*
* Invalid %fs's and %gs's can be created using
* procfs or PT_SETREGS or by invalidating the
* underlying LDT entry. This causes a fault
* in kernel mode when the kernel attempts to
* switch contexts. Lose the bad context
* (XXX) so that we can continue, and generate
* a signal.
*/
if (frame->tf_eip == (int)cpu_switch_load_gs) {
PCPU_GET(curpcb)->pcb_gs = 0;
#if 0
PROC_LOCK(p);
kern_psignal(p, SIGBUS);
PROC_UNLOCK(p);
#endif
goto out;
}
if (td->td_intr_nesting_level != 0)
break;
/*
* Invalid segment selectors and out of bounds
* %eip's and %esp's can be set up in user mode.
* This causes a fault in kernel mode when the
* kernel tries to return to user mode. We want
* to get this fault so that we can fix the
* problem here and not have to check all the
* selectors and pointers when the user changes
* them.
*/
if (frame->tf_eip == (int)doreti_iret) {
frame->tf_eip = (int)doreti_iret_fault;
goto out;
}
if (frame->tf_eip == (int)doreti_popl_ds) {
frame->tf_eip = (int)doreti_popl_ds_fault;
goto out;
}
if (frame->tf_eip == (int)doreti_popl_es) {
frame->tf_eip = (int)doreti_popl_es_fault;
goto out;
}
if (frame->tf_eip == (int)doreti_popl_fs) {
frame->tf_eip = (int)doreti_popl_fs_fault;
goto out;
}
if (PCPU_GET(curpcb)->pcb_onfault != NULL) {
frame->tf_eip =
(int)PCPU_GET(curpcb)->pcb_onfault;
goto out;
}
break;
case T_TSSFLT:
/*
* PSL_NT can be set in user mode and isn't cleared
* automatically when the kernel is entered. This
* causes a TSS fault when the kernel attempts to
* `iret' because the TSS link is uninitialized. We
* want to get this fault so that we can fix the
* problem here and not every time the kernel is
* entered.
*/
if (frame->tf_eflags & PSL_NT) {
frame->tf_eflags &= ~PSL_NT;
goto out;
}
break;
case T_TRCTRAP: /* trace trap */
if (frame->tf_eip == (int)IDTVEC(lcall_syscall)) {
/*
* We've just entered system mode via the
* syscall lcall. Continue single stepping
* silently until the syscall handler has
* saved the flags.
*/
goto out;
}
if (frame->tf_eip == (int)IDTVEC(lcall_syscall) + 1) {
/*
* The syscall handler has now saved the
* flags. Stop single stepping it.
*/
frame->tf_eflags &= ~PSL_T;
goto out;
}
/*
* Ignore debug register trace traps due to
* accesses in the user's address space, which
* can happen under several conditions such as
* if a user sets a watchpoint on a buffer and
* then passes that buffer to a system call.
* We still want to get TRCTRAPS for addresses
* in kernel space because that is useful when
* debugging the kernel.
*/
if (user_dbreg_trap() &&
!(PCPU_GET(curpcb)->pcb_flags & PCB_VM86CALL)) {
/*
* Reset breakpoint bits because the
* processor doesn't
*/
load_dr6(rdr6() & 0xfffffff0);
goto out;
}
/*
* FALLTHROUGH (TRCTRAP kernel mode, kernel address)
*/
case T_BPTFLT:
/*
* If KDB is enabled, let it handle the debugger trap.
* Otherwise, debugger traps "can't happen".
*/
#ifdef KDB
if (kdb_trap(type, 0, frame))
goto out;
#endif
break;
#ifdef DEV_ISA
case T_NMI:
#ifdef POWERFAIL_NMI
if (time_second - lastalert > 10) {
log(LOG_WARNING, "NMI: power fail\n");
sysbeep(880, hz);
lastalert = time_second;
}
goto out;
#else /* !POWERFAIL_NMI */
/* machine/parity/power fail/"kitchen sink" faults */
if (isa_nmi(code) == 0) {
#ifdef KDB
/*
* NMI can be hooked up to a pushbutton
* for debugging.
*/
if (kdb_on_nmi) {
printf ("NMI ... going to debugger\n");
kdb_trap(type, 0, frame);
}
#endif /* KDB */
goto out;
} else if (panic_on_nmi == 0)
goto out;
/* FALLTHROUGH */
#endif /* POWERFAIL_NMI */
#endif /* DEV_ISA */
}
trap_fatal(frame, eva);
goto out;
}
/* Translate fault for emulators (e.g. Linux) */
if (*p->p_sysent->sv_transtrap)
i = (*p->p_sysent->sv_transtrap)(i, type);
ksiginfo_init_trap(&ksi);
ksi.ksi_signo = i;
ksi.ksi_code = ucode;
ksi.ksi_addr = (void *)addr;
ksi.ksi_trapno = type;
trapsignal(td, &ksi);
#ifdef DEBUG
if (type <= MAX_TRAP_MSG) {
uprintf("fatal process exception: %s",
trap_msg[type]);
if ((type == T_PAGEFLT) || (type == T_PROTFLT))
uprintf(", fault VA = 0x%lx", (u_long)eva);
uprintf("\n");
}
#endif
user:
userret(td, frame);
mtx_assert(&Giant, MA_NOTOWNED);
KASSERT(PCB_USER_FPU(td->td_pcb),
("Return from trap with kernel FPU ctx leaked"));
userout:
out:
return;
}
/*
* void prefetch_abort_handler(trapframe_t *tf)
*
* Abort handler called when instruction execution occurs at
* a non existent or restricted (access permissions) memory page.
* If the address is invalid and we were in SVC mode then panic as
* the kernel should never prefetch abort.
* If the address is invalid and the page is mapped then the user process
* does no have read permission so send it a signal.
* Otherwise fault the page in and try again.
*/
void
prefetch_abort_handler(trapframe_t *tf)
{
struct thread *td;
struct proc * p;
struct vm_map *map;
vm_offset_t fault_pc, va;
int error = 0;
struct ksig ksig;
#if 0
/* Update vmmeter statistics */
uvmexp.traps++;
#endif
#if 0
printf("prefetch abort handler: %p %p\n", (void*)tf->tf_pc,
(void*)tf->tf_usr_lr);
#endif
td = curthread;
p = td->td_proc;
PCPU_INC(cnt.v_trap);
if (TRAP_USERMODE(tf)) {
td->td_frame = tf;
if (td->td_ucred != td->td_proc->p_ucred)
cred_update_thread(td);
}
fault_pc = tf->tf_pc;
if (td->td_md.md_spinlock_count == 0) {
if (__predict_true(tf->tf_spsr & I32_bit) == 0)
enable_interrupts(I32_bit);
if (__predict_true(tf->tf_spsr & F32_bit) == 0)
enable_interrupts(F32_bit);
}
/* See if the cpu state needs to be fixed up */
switch (prefetch_abort_fixup(tf, &ksig)) {
case ABORT_FIXUP_RETURN:
return;
case ABORT_FIXUP_FAILED:
/* Deliver a SIGILL to the process */
ksig.signb = SIGILL;
ksig.code = 0;
td->td_frame = tf;
goto do_trapsignal;
default:
break;
}
/* Prefetch aborts cannot happen in kernel mode */
if (__predict_false(!TRAP_USERMODE(tf)))
dab_fatal(tf, 0, tf->tf_pc, NULL, &ksig);
td->td_pticks = 0;
/* Ok validate the address, can only execute in USER space */
if (__predict_false(fault_pc >= VM_MAXUSER_ADDRESS ||
(fault_pc < VM_MIN_ADDRESS && vector_page == ARM_VECTORS_LOW))) {
ksig.signb = SIGSEGV;
ksig.code = 0;
goto do_trapsignal;
}
map = &td->td_proc->p_vmspace->vm_map;
va = trunc_page(fault_pc);
/*
* See if the pmap can handle this fault on its own...
*/
#ifdef DEBUG
last_fault_code = -1;
#endif
if (pmap_fault_fixup(map->pmap, va, VM_PROT_READ, 1))
goto out;
if (map != kernel_map) {
PROC_LOCK(p);
p->p_lock++;
PROC_UNLOCK(p);
}
error = vm_fault(map, va, VM_PROT_READ | VM_PROT_EXECUTE,
VM_FAULT_NORMAL);
if (map != kernel_map) {
PROC_LOCK(p);
p->p_lock--;
PROC_UNLOCK(p);
}
if (__predict_true(error == 0))
goto out;
if (error == ENOMEM) {
printf("VM: pid %d (%s), uid %d killed: "
"out of swap\n", td->td_proc->p_pid, td->td_name,
(td->td_proc->p_ucred) ?
td->td_proc->p_ucred->cr_uid : -1);
ksig.signb = SIGKILL;
} else {
ksig.signb = SIGSEGV;
}
ksig.code = 0;
do_trapsignal:
call_trapsignal(td, ksig.signb, ksig.code);
out:
userret(td, tf);
}
void
trap(struct trapframe *frame)
{
struct thread *td;
struct proc *p;
int sig, type, user;
ksiginfo_t ksi;
#ifdef KDB
if (kdb_active) {
kdb_reenter();
return;
}
#endif
PCPU_INC(cnt.v_trap);
td = curthread;
p = td->td_proc;
type = frame->exc;
sig = 0;
user = (frame->srr1 & PSL_PR) ? 1 : 0;
CTR3(KTR_TRAP, "trap: %s type=%s (%s)", p->p_comm,
trapname(type), user ? "user" : "kernel");
if (user) {
td->td_frame = frame;
if (td->td_ucred != p->p_ucred)
cred_update_thread(td);
/* User Mode Traps */
switch (type) {
case EXC_DSI:
case EXC_ISI:
sig = trap_pfault(frame, 1);
break;
case EXC_SC:
syscall(frame);
break;
case EXC_ALI:
if (fix_unaligned(td, frame) != 0)
sig = SIGBUS;
else
frame->srr0 += 4;
break;
case EXC_DEBUG: /* Single stepping */
mtspr(SPR_DBSR, mfspr(SPR_DBSR));
frame->srr1 &= ~PSL_DE;
frame->cpu.booke.dbcr0 &= ~(DBCR0_IDM || DBCR0_IC);
sig = SIGTRAP;
break;
case EXC_PGM: /* Program exception */
#ifdef FPU_EMU
sig = fpu_emulate(frame,
(struct fpreg *)&td->td_pcb->pcb_fpu);
#else
/* XXX SIGILL for non-trap instructions. */
sig = SIGTRAP;
#endif
break;
default:
trap_fatal(frame);
}
} else {
/* Kernel Mode Traps */
KASSERT(cold || td->td_ucred != NULL,
("kernel trap doesn't have ucred"));
switch (type) {
case EXC_DEBUG:
mtspr(SPR_DBSR, mfspr(SPR_DBSR));
kdb_trap(frame->exc, 0, frame);
return;
case EXC_DSI:
if (trap_pfault(frame, 0) == 0)
return;
break;
case EXC_MCHK:
if (handle_onfault(frame))
return;
break;
#ifdef KDB
case EXC_PGM:
if (frame->cpu.booke.esr & ESR_PTR)
kdb_trap(EXC_PGM, 0, frame);
return;
#endif
default:
break;
}
trap_fatal(frame);
}
if (sig != 0) {
if (p->p_sysent->sv_transtrap != NULL)
sig = (p->p_sysent->sv_transtrap)(sig, type);
ksiginfo_init_trap(&ksi);
ksi.ksi_signo = sig;
ksi.ksi_code = type; /* XXX, not POSIX */
/* ksi.ksi_addr = ? */
ksi.ksi_trapno = type;
trapsignal(td, &ksi);
}
userret(td, frame);
}
void
data_abort_handler(trapframe_t *tf)
{
struct vm_map *map;
struct pcb *pcb;
struct thread *td;
u_int user, far, fsr;
vm_prot_t ftype;
void *onfault;
vm_offset_t va;
int error = 0;
struct ksig ksig;
struct proc *p;
/* Grab FAR/FSR before enabling interrupts */
far = cpu_faultaddress();
fsr = cpu_faultstatus();
#if 0
printf("data abort: fault address=%p (from pc=%p lr=%p)\n",
(void*)far, (void*)tf->tf_pc, (void*)tf->tf_svc_lr);
#endif
/* Update vmmeter statistics */
#if 0
vmexp.traps++;
#endif
td = curthread;
p = td->td_proc;
PCPU_INC(cnt.v_trap);
/* Data abort came from user mode? */
user = TRAP_USERMODE(tf);
if (user) {
td->td_pticks = 0;
td->td_frame = tf;
if (td->td_ucred != td->td_proc->p_ucred)
cred_update_thread(td);
}
/* Grab the current pcb */
pcb = td->td_pcb;
/* Re-enable interrupts if they were enabled previously */
if (td->td_md.md_spinlock_count == 0) {
if (__predict_true(tf->tf_spsr & I32_bit) == 0)
enable_interrupts(I32_bit);
if (__predict_true(tf->tf_spsr & F32_bit) == 0)
enable_interrupts(F32_bit);
}
/* Invoke the appropriate handler, if necessary */
if (__predict_false(data_aborts[fsr & FAULT_TYPE_MASK].func != NULL)) {
if ((data_aborts[fsr & FAULT_TYPE_MASK].func)(tf, fsr, far,
td, &ksig)) {
goto do_trapsignal;
}
goto out;
}
/*
* At this point, we're dealing with one of the following data aborts:
*
* FAULT_TRANS_S - Translation -- Section
* FAULT_TRANS_P - Translation -- Page
* FAULT_DOMAIN_S - Domain -- Section
* FAULT_DOMAIN_P - Domain -- Page
* FAULT_PERM_S - Permission -- Section
* FAULT_PERM_P - Permission -- Page
*
* These are the main virtual memory-related faults signalled by
* the MMU.
*/
/* fusubailout is used by [fs]uswintr to avoid page faulting */
if (__predict_false(pcb->pcb_onfault == fusubailout)) {
tf->tf_r0 = EFAULT;
tf->tf_pc = (register_t)(intptr_t) pcb->pcb_onfault;
return;
}
/*
* Make sure the Program Counter is sane. We could fall foul of
* someone executing Thumb code, in which case the PC might not
* be word-aligned. This would cause a kernel alignment fault
* further down if we have to decode the current instruction.
* XXX: It would be nice to be able to support Thumb at some point.
*/
if (__predict_false((tf->tf_pc & 3) != 0)) {
if (user) {
/*
* Give the user an illegal instruction signal.
*/
/* Deliver a SIGILL to the process */
ksig.signb = SIGILL;
ksig.code = 0;
goto do_trapsignal;
}
/*
* The kernel never executes Thumb code.
*/
printf("\ndata_abort_fault: Misaligned Kernel-mode "
"Program Counter\n");
dab_fatal(tf, fsr, far, td, &ksig);
}
/* See if the cpu state needs to be fixed up */
switch (data_abort_fixup(tf, fsr, far, td, &ksig)) {
case ABORT_FIXUP_RETURN:
return;
case ABORT_FIXUP_FAILED:
/* Deliver a SIGILL to the process */
ksig.signb = SIGILL;
ksig.code = 0;
goto do_trapsignal;
default:
break;
}
va = trunc_page((vm_offset_t)far);
/*
* It is only a kernel address space fault iff:
* 1. user == 0 and
* 2. pcb_onfault not set or
* 3. pcb_onfault set and not LDRT/LDRBT/STRT/STRBT instruction.
*/
if (user == 0 && (va >= VM_MIN_KERNEL_ADDRESS ||
(va < VM_MIN_ADDRESS && vector_page == ARM_VECTORS_LOW)) &&
__predict_true((pcb->pcb_onfault == NULL ||
(ReadWord(tf->tf_pc) & 0x05200000) != 0x04200000))) {
map = kernel_map;
/* Was the fault due to the FPE/IPKDB ? */
if (__predict_false((tf->tf_spsr & PSR_MODE)==PSR_UND32_MODE)) {
/*
* Force exit via userret()
* This is necessary as the FPE is an extension to
* userland that actually runs in a priveledged mode
* but uses USR mode permissions for its accesses.
*/
user = 1;
ksig.signb = SIGSEGV;
ksig.code = 0;
goto do_trapsignal;
}
} else {
map = &td->td_proc->p_vmspace->vm_map;
}
/*
* We need to know whether the page should be mapped
* as R or R/W. The MMU does not give us the info as
* to whether the fault was caused by a read or a write.
*
* However, we know that a permission fault can only be
* the result of a write to a read-only location, so
* we can deal with those quickly.
*
* Otherwise we need to disassemble the instruction
* responsible to determine if it was a write.
*/
if (IS_PERMISSION_FAULT(fsr))
ftype = VM_PROT_WRITE;
else {
u_int insn = ReadWord(tf->tf_pc);
if (((insn & 0x0c100000) == 0x04000000) || /* STR/STRB */
((insn & 0x0e1000b0) == 0x000000b0) || /* STRH/STRD */
((insn & 0x0a100000) == 0x08000000)) { /* STM/CDT */
ftype = VM_PROT_WRITE;
} else {
if ((insn & 0x0fb00ff0) == 0x01000090) /* SWP */
ftype = VM_PROT_READ | VM_PROT_WRITE;
else
ftype = VM_PROT_READ;
}
}
/*
* See if the fault is as a result of ref/mod emulation,
* or domain mismatch.
*/
#ifdef DEBUG
last_fault_code = fsr;
#endif
if (pmap_fault_fixup(vmspace_pmap(td->td_proc->p_vmspace), va, ftype,
user)) {
goto out;
}
onfault = pcb->pcb_onfault;
pcb->pcb_onfault = NULL;
if (map != kernel_map) {
PROC_LOCK(p);
p->p_lock++;
PROC_UNLOCK(p);
}
error = vm_fault(map, va, ftype, VM_FAULT_NORMAL);
pcb->pcb_onfault = onfault;
if (map != kernel_map) {
PROC_LOCK(p);
p->p_lock--;
PROC_UNLOCK(p);
}
if (__predict_true(error == 0))
goto out;
if (user == 0) {
if (pcb->pcb_onfault) {
tf->tf_r0 = error;
tf->tf_pc = (register_t)(intptr_t) pcb->pcb_onfault;
return;
}
printf("\nvm_fault(%p, %x, %x, 0) -> %x\n", map, va, ftype,
error);
dab_fatal(tf, fsr, far, td, &ksig);
}
if (error == ENOMEM) {
printf("VM: pid %d (%s), uid %d killed: "
"out of swap\n", td->td_proc->p_pid, td->td_name,
(td->td_proc->p_ucred) ?
td->td_proc->p_ucred->cr_uid : -1);
ksig.signb = SIGKILL;
} else {
ksig.signb = SIGSEGV;
}
ksig.code = 0;
do_trapsignal:
call_trapsignal(td, ksig.signb, ksig.code);
out:
/* If returning to user mode, make sure to invoke userret() */
if (user)
userret(td, tf);
}
static inline int
syscallenter(struct thread *td, struct syscall_args *sa)
{
struct proc *p;
int error, traced;
PCPU_INC(cnt.v_syscall);
p = td->td_proc;
td->td_pticks = 0;
if (td->td_ucred != p->p_ucred)
cred_update_thread(td);
if (p->p_flag & P_TRACED) {
traced = 1;
PROC_LOCK(p);
td->td_dbgflags &= ~TDB_USERWR;
td->td_dbgflags |= TDB_SCE;
PROC_UNLOCK(p);
} else
traced = 0;
error = (p->p_sysent->sv_fetch_syscall_args)(td, sa);
#ifdef KTRACE
if (KTRPOINT(td, KTR_SYSCALL))
ktrsyscall(sa->code, sa->narg, sa->args);
#endif
CTR6(KTR_SYSC,
"syscall: td=%p pid %d %s (%#lx, %#lx, %#lx)",
td, td->td_proc->p_pid, syscallname(p, sa->code),
sa->args[0], sa->args[1], sa->args[2]);
if (error == 0) {
STOPEVENT(p, S_SCE, sa->narg);
if (p->p_flag & P_TRACED && p->p_stops & S_PT_SCE) {
PROC_LOCK(p);
ptracestop((td), SIGTRAP);
PROC_UNLOCK(p);
}
if (td->td_dbgflags & TDB_USERWR) {
/*
* Reread syscall number and arguments if
* debugger modified registers or memory.
*/
error = (p->p_sysent->sv_fetch_syscall_args)(td, sa);
#ifdef KTRACE
if (KTRPOINT(td, KTR_SYSCALL))
ktrsyscall(sa->code, sa->narg, sa->args);
#endif
if (error != 0)
goto retval;
}
#ifdef CAPABILITY_MODE
/*
* In capability mode, we only allow access to system calls
* flagged with SYF_CAPENABLED.
*/
if (IN_CAPABILITY_MODE(td) &&
!(sa->callp->sy_flags & SYF_CAPENABLED)) {
error = ECAPMODE;
goto retval;
}
#endif
error = syscall_thread_enter(td, sa->callp);
if (error != 0)
goto retval;
#ifdef KDTRACE_HOOKS
/*
* If the systrace module has registered it's probe
* callback and if there is a probe active for the
* syscall 'entry', process the probe.
*/
if (systrace_probe_func != NULL && sa->callp->sy_entry != 0)
(*systrace_probe_func)(sa->callp->sy_entry, sa->code,
sa->callp, sa->args, 0);
#endif
AUDIT_SYSCALL_ENTER(sa->code, td);
error = (sa->callp->sy_call)(td, sa->args);
AUDIT_SYSCALL_EXIT(error, td);
/* Save the latest error return value. */
td->td_errno = error;
#ifdef KDTRACE_HOOKS
/*
* If the systrace module has registered it's probe
* callback and if there is a probe active for the
* syscall 'return', process the probe.
*/
if (systrace_probe_func != NULL && sa->callp->sy_return != 0)
(*systrace_probe_func)(sa->callp->sy_return, sa->code,
sa->callp, NULL, (error) ? -1 : td->td_retval[0]);
#endif
syscall_thread_exit(td, sa->callp);
CTR4(KTR_SYSC, "syscall: p=%p error=%d return %#lx %#lx",
p, error, td->td_retval[0], td->td_retval[1]);
}
retval:
if (traced) {
PROC_LOCK(p);
td->td_dbgflags &= ~TDB_SCE;
PROC_UNLOCK(p);
}
(p->p_sysent->sv_set_syscall_retval)(td, error);
return (error);
}
void
trap(struct trapframe *frame)
{
struct thread *td = curthread;
struct proc *p = td->td_proc;
int i = 0, ucode = 0, code;
u_int type;
register_t addr = 0;
ksiginfo_t ksi;
PCPU_INC(cnt.v_trap);
type = frame->tf_trapno;
#ifdef SMP
#ifdef STOP_NMI
/* Handler for NMI IPIs used for stopping CPUs. */
if (type == T_NMI) {
if (ipi_nmi_handler() == 0)
goto out;
}
#endif /* STOP_NMI */
#endif /* SMP */
#ifdef KDB
if (kdb_active) {
kdb_reenter();
goto out;
}
#endif
#ifdef HWPMC_HOOKS
/*
* CPU PMCs interrupt using an NMI. If the PMC module is
* active, pass the 'rip' value to the PMC module's interrupt
* handler. A return value of '1' from the handler means that
* the NMI was handled by it and we can return immediately.
*/
if (type == T_NMI && pmc_intr &&
(*pmc_intr)(PCPU_GET(cpuid), frame))
goto out;
#endif
if (type == T_MCHK) {
if (!mca_intr())
trap_fatal(frame, 0);
goto out;
}
#ifdef KDTRACE_HOOKS
/*
* A trap can occur while DTrace executes a probe. Before
* executing the probe, DTrace blocks re-scheduling and sets
* a flag in it's per-cpu flags to indicate that it doesn't
* want to fault. On returning from the the probe, the no-fault
* flag is cleared and finally re-scheduling is enabled.
*
* If the DTrace kernel module has registered a trap handler,
* call it and if it returns non-zero, assume that it has
* handled the trap and modified the trap frame so that this
* function can return normally.
*/
if (dtrace_trap_func != NULL)
if ((*dtrace_trap_func)(frame, type))
goto out;
#endif
if ((frame->tf_rflags & PSL_I) == 0) {
/*
* Buggy application or kernel code has disabled
* interrupts and then trapped. Enabling interrupts
* now is wrong, but it is better than running with
* interrupts disabled until they are accidentally
* enabled later.
*/
if (ISPL(frame->tf_cs) == SEL_UPL)
printf(
"pid %ld (%s): trap %d with interrupts disabled\n",
(long)curproc->p_pid, curproc->p_comm, type);
else if (type != T_NMI && type != T_BPTFLT &&
type != T_TRCTRAP) {
/*
* XXX not quite right, since this may be for a
* multiple fault in user mode.
*/
printf("kernel trap %d with interrupts disabled\n",
type);
/*
* We shouldn't enable interrupts while holding a
* spin lock or servicing an NMI.
*/
if (type != T_NMI && td->td_md.md_spinlock_count == 0)
enable_intr();
}
}
code = frame->tf_err;
if (type == T_PAGEFLT) {
/*
* If we get a page fault while in a critical section, then
* it is most likely a fatal kernel page fault. The kernel
* is already going to panic trying to get a sleep lock to
* do the VM lookup, so just consider it a fatal trap so the
* kernel can print out a useful trap message and even get
* to the debugger.
*
* If we get a page fault while holding a non-sleepable
* lock, then it is most likely a fatal kernel page fault.
* If WITNESS is enabled, then it's going to whine about
* bogus LORs with various VM locks, so just skip to the
* fatal trap handling directly.
*/
if (td->td_critnest != 0 ||
WITNESS_CHECK(WARN_SLEEPOK | WARN_GIANTOK, NULL,
"Kernel page fault") != 0)
trap_fatal(frame, frame->tf_addr);
}
if (ISPL(frame->tf_cs) == SEL_UPL) {
/* user trap */
td->td_pticks = 0;
td->td_frame = frame;
addr = frame->tf_rip;
if (td->td_ucred != p->p_ucred)
cred_update_thread(td);
switch (type) {
case T_PRIVINFLT: /* privileged instruction fault */
i = SIGILL;
ucode = ILL_PRVOPC;
break;
case T_BPTFLT: /* bpt instruction fault */
case T_TRCTRAP: /* trace trap */
enable_intr();
frame->tf_rflags &= ~PSL_T;
i = SIGTRAP;
ucode = (type == T_TRCTRAP ? TRAP_TRACE : TRAP_BRKPT);
break;
case T_ARITHTRAP: /* arithmetic trap */
ucode = fputrap();
if (ucode == -1)
goto userout;
i = SIGFPE;
break;
case T_PROTFLT: /* general protection fault */
i = SIGBUS;
ucode = BUS_OBJERR;
break;
case T_STKFLT: /* stack fault */
case T_SEGNPFLT: /* segment not present fault */
i = SIGBUS;
ucode = BUS_ADRERR;
break;
case T_TSSFLT: /* invalid TSS fault */
i = SIGBUS;
ucode = BUS_OBJERR;
break;
case T_DOUBLEFLT: /* double fault */
default:
i = SIGBUS;
ucode = BUS_OBJERR;
break;
case T_PAGEFLT: /* page fault */
addr = frame->tf_addr;
#ifdef KSE
if (td->td_pflags & TDP_SA)
thread_user_enter(td);
#endif
i = trap_pfault(frame, TRUE);
if (i == -1)
goto userout;
if (i == 0)
goto user;
if (i == SIGSEGV)
ucode = SEGV_MAPERR;
else {
if (prot_fault_translation == 0) {
/*
* Autodetect.
* This check also covers the images
* without the ABI-tag ELF note.
*/
if ((curproc->p_sysent ==
&elf64_freebsd_sysvec
#ifdef COMPAT_IA32
|| curproc->p_sysent ==
&ia32_freebsd_sysvec
#endif
) && p->p_osrel >= 700004) {
i = SIGSEGV;
ucode = SEGV_ACCERR;
} else {
i = SIGBUS;
ucode = BUS_PAGE_FAULT;
}
} else if (prot_fault_translation == 1) {
/*
* Always compat mode.
*/
i = SIGBUS;
ucode = BUS_PAGE_FAULT;
} else {
/*
* Always SIGSEGV mode.
*/
i = SIGSEGV;
ucode = SEGV_ACCERR;
}
}
break;
case T_DIVIDE: /* integer divide fault */
ucode = FPE_INTDIV;
i = SIGFPE;
break;
#ifdef DEV_ISA
case T_NMI:
/* machine/parity/power fail/"kitchen sink" faults */
/* XXX Giant */
if (isa_nmi(code) == 0) {
#ifdef KDB
/*
* NMI can be hooked up to a pushbutton
* for debugging.
*/
if (kdb_on_nmi) {
printf ("NMI ... going to debugger\n");
kdb_trap(type, 0, frame);
}
#endif /* KDB */
goto userout;
} else if (panic_on_nmi)
panic("NMI indicates hardware failure");
break;
#endif /* DEV_ISA */
case T_OFLOW: /* integer overflow fault */
ucode = FPE_INTOVF;
i = SIGFPE;
break;
case T_BOUND: /* bounds check fault */
ucode = FPE_FLTSUB;
i = SIGFPE;
break;
case T_DNA:
/* transparent fault (due to context switch "late") */
fpudna();
goto userout;
case T_FPOPFLT: /* FPU operand fetch fault */
ucode = ILL_COPROC;
i = SIGILL;
break;
case T_XMMFLT: /* SIMD floating-point exception */
ucode = 0; /* XXX */
i = SIGFPE;
break;
}
} else {
/* kernel trap */
KASSERT(cold || td->td_ucred != NULL,
("kernel trap doesn't have ucred"));
switch (type) {
case T_PAGEFLT: /* page fault */
(void) trap_pfault(frame, FALSE);
goto out;
case T_DNA:
/*
* The kernel is apparently using fpu for copying.
* XXX this should be fatal unless the kernel has
* registered such use.
*/
fpudna();
printf("fpudna in kernel mode!\n");
goto out;
case T_STKFLT: /* stack fault */
break;
case T_PROTFLT: /* general protection fault */
case T_SEGNPFLT: /* segment not present fault */
if (td->td_intr_nesting_level != 0)
break;
/*
* Invalid segment selectors and out of bounds
* %rip's and %rsp's can be set up in user mode.
* This causes a fault in kernel mode when the
* kernel tries to return to user mode. We want
* to get this fault so that we can fix the
* problem here and not have to check all the
* selectors and pointers when the user changes
* them.
*/
if (frame->tf_rip == (long)doreti_iret) {
frame->tf_rip = (long)doreti_iret_fault;
goto out;
}
if (PCPU_GET(curpcb)->pcb_onfault != NULL) {
frame->tf_rip =
(long)PCPU_GET(curpcb)->pcb_onfault;
goto out;
}
break;
case T_TSSFLT:
/*
* PSL_NT can be set in user mode and isn't cleared
* automatically when the kernel is entered. This
* causes a TSS fault when the kernel attempts to
* `iret' because the TSS link is uninitialized. We
* want to get this fault so that we can fix the
* problem here and not every time the kernel is
* entered.
*/
if (frame->tf_rflags & PSL_NT) {
frame->tf_rflags &= ~PSL_NT;
goto out;
}
break;
case T_TRCTRAP: /* trace trap */
/*
* Ignore debug register trace traps due to
* accesses in the user's address space, which
* can happen under several conditions such as
* if a user sets a watchpoint on a buffer and
* then passes that buffer to a system call.
* We still want to get TRCTRAPS for addresses
* in kernel space because that is useful when
* debugging the kernel.
*/
if (user_dbreg_trap()) {
/*
* Reset breakpoint bits because the
* processor doesn't
*/
/* XXX check upper bits here */
load_dr6(rdr6() & 0xfffffff0);
goto out;
}
/*
* FALLTHROUGH (TRCTRAP kernel mode, kernel address)
*/
case T_BPTFLT:
/*
* If KDB is enabled, let it handle the debugger trap.
* Otherwise, debugger traps "can't happen".
*/
#ifdef KDB
if (kdb_trap(type, 0, frame))
goto out;
#endif
break;
#ifdef DEV_ISA
case T_NMI:
/* XXX Giant */
/* machine/parity/power fail/"kitchen sink" faults */
if (isa_nmi(code) == 0) {
#ifdef KDB
/*
* NMI can be hooked up to a pushbutton
* for debugging.
*/
if (kdb_on_nmi) {
printf ("NMI ... going to debugger\n");
kdb_trap(type, 0, frame);
}
#endif /* KDB */
goto out;
} else if (panic_on_nmi == 0)
goto out;
/* FALLTHROUGH */
#endif /* DEV_ISA */
}
trap_fatal(frame, 0);
goto out;
}
/* Translate fault for emulators (e.g. Linux) */
if (*p->p_sysent->sv_transtrap)
i = (*p->p_sysent->sv_transtrap)(i, type);
ksiginfo_init_trap(&ksi);
ksi.ksi_signo = i;
ksi.ksi_code = ucode;
ksi.ksi_trapno = type;
ksi.ksi_addr = (void *)addr;
trapsignal(td, &ksi);
#ifdef DEBUG
if (type <= MAX_TRAP_MSG) {
uprintf("fatal process exception: %s",
trap_msg[type]);
if ((type == T_PAGEFLT) || (type == T_PROTFLT))
uprintf(", fault VA = 0x%lx", frame->tf_addr);
uprintf("\n");
}
#endif
user:
userret(td, frame);
mtx_assert(&Giant, MA_NOTOWNED);
userout:
out:
return;
}
void
ia32_syscall(struct trapframe *frame)
{
caddr_t params;
int i;
struct sysent *callp;
struct thread *td = curthread;
struct proc *p = td->td_proc;
register_t orig_tf_rflags;
int error;
int narg;
u_int32_t args[8];
u_int64_t args64[8];
u_int code;
ksiginfo_t ksi;
PCPU_INC(cnt.v_syscall);
td->td_pticks = 0;
td->td_frame = frame;
if (td->td_ucred != p->p_ucred)
cred_update_thread(td);
params = (caddr_t)frame->tf_rsp + sizeof(u_int32_t);
code = frame->tf_rax;
orig_tf_rflags = frame->tf_rflags;
if (p->p_sysent->sv_prepsyscall) {
/*
* The prep code is MP aware.
*/
(*p->p_sysent->sv_prepsyscall)(frame, args, &code, ¶ms);
} else {
/*
* Need to check if this is a 32 bit or 64 bit syscall.
* fuword is MP aware.
*/
if (code == SYS_syscall) {
/*
* Code is first argument, followed by actual args.
*/
code = fuword32(params);
params += sizeof(int);
} else if (code == SYS___syscall) {
/*
* Like syscall, but code is a quad, so as to maintain
* quad alignment for the rest of the arguments.
* We use a 32-bit fetch in case params is not
* aligned.
*/
code = fuword32(params);
params += sizeof(quad_t);
}
}
if (p->p_sysent->sv_mask)
code &= p->p_sysent->sv_mask;
if (code >= p->p_sysent->sv_size)
callp = &p->p_sysent->sv_table[0];
else
callp = &p->p_sysent->sv_table[code];
narg = callp->sy_narg;
/*
* copyin and the ktrsyscall()/ktrsysret() code is MP-aware
*/
if (params != NULL && narg != 0)
error = copyin(params, (caddr_t)args,
(u_int)(narg * sizeof(int)));
else
error = 0;
for (i = 0; i < narg; i++)
args64[i] = args[i];
#ifdef KTRACE
if (KTRPOINT(td, KTR_SYSCALL))
ktrsyscall(code, narg, args64);
#endif
CTR4(KTR_SYSC, "syscall enter thread %p pid %d proc %s code %d", td,
td->td_proc->p_pid, td->td_proc->p_comm, code);
if (error == 0) {
td->td_retval[0] = 0;
td->td_retval[1] = frame->tf_rdx;
STOPEVENT(p, S_SCE, narg);
PTRACESTOP_SC(p, td, S_PT_SCE);
AUDIT_SYSCALL_ENTER(code, td);
error = (*callp->sy_call)(td, args64);
AUDIT_SYSCALL_EXIT(error, td);
}
switch (error) {
case 0:
frame->tf_rax = td->td_retval[0];
frame->tf_rdx = td->td_retval[1];
frame->tf_rflags &= ~PSL_C;
break;
case ERESTART:
/*
* Reconstruct pc, assuming lcall $X,y is 7 bytes,
* int 0x80 is 2 bytes. We saved this in tf_err.
*/
frame->tf_rip -= frame->tf_err;
break;
case EJUSTRETURN:
break;
default:
if (p->p_sysent->sv_errsize) {
if (error >= p->p_sysent->sv_errsize)
error = -1; /* XXX */
else
error = p->p_sysent->sv_errtbl[error];
}
frame->tf_rax = error;
frame->tf_rflags |= PSL_C;
break;
}
/*
* Traced syscall.
*/
if (orig_tf_rflags & PSL_T) {
frame->tf_rflags &= ~PSL_T;
ksiginfo_init_trap(&ksi);
ksi.ksi_signo = SIGTRAP;
ksi.ksi_code = TRAP_TRACE;
ksi.ksi_addr = (void *)frame->tf_rip;
trapsignal(td, &ksi);
}
/*
* Check for misbehavior.
*/
WITNESS_WARN(WARN_PANIC, NULL, "System call %s returning",
(code >= 0 && code < SYS_MAXSYSCALL) ? freebsd32_syscallnames[code] : "???");
KASSERT(td->td_critnest == 0,
("System call %s returning in a critical section",
(code >= 0 && code < SYS_MAXSYSCALL) ? freebsd32_syscallnames[code] : "???"));
KASSERT(td->td_locks == 0,
("System call %s returning with %d locks held",
(code >= 0 && code < SYS_MAXSYSCALL) ? freebsd32_syscallnames[code] : "???",
td->td_locks));
/*
* Handle reschedule and other end-of-syscall issues
*/
userret(td, frame);
CTR4(KTR_SYSC, "syscall exit thread %p pid %d proc %s code %d", td,
td->td_proc->p_pid, td->td_proc->p_comm, code);
#ifdef KTRACE
if (KTRPOINT(td, KTR_SYSRET))
ktrsysret(code, error, td->td_retval[0]);
#endif
/*
* This works because errno is findable through the
* register set. If we ever support an emulation where this
* is not the case, this code will need to be revisited.
*/
STOPEVENT(p, S_SCX, code);
PTRACESTOP_SC(p, td, S_PT_SCX);
}
void
trap(struct trapframe *frame)
{
struct thread *td;
struct proc *p;
#ifdef KDTRACE_HOOKS
uint32_t inst;
#endif
int sig, type, user;
u_int ucode;
ksiginfo_t ksi;
PCPU_INC(cnt.v_trap);
td = curthread;
p = td->td_proc;
type = ucode = frame->exc;
sig = 0;
user = frame->srr1 & PSL_PR;
CTR3(KTR_TRAP, "trap: %s type=%s (%s)", td->td_name,
trapname(type), user ? "user" : "kernel");
#ifdef KDTRACE_HOOKS
/*
* A trap can occur while DTrace executes a probe. Before
* executing the probe, DTrace blocks re-scheduling and sets
* a flag in it's per-cpu flags to indicate that it doesn't
* want to fault. On returning from the probe, the no-fault
* flag is cleared and finally re-scheduling is enabled.
*
* If the DTrace kernel module has registered a trap handler,
* call it and if it returns non-zero, assume that it has
* handled the trap and modified the trap frame so that this
* function can return normally.
*/
/*
* XXXDTRACE: add pid probe handler here (if ever)
*/
if (dtrace_trap_func != NULL && (*dtrace_trap_func)(frame, type))
return;
#endif
if (user) {
td->td_pticks = 0;
td->td_frame = frame;
if (td->td_ucred != p->p_ucred)
cred_update_thread(td);
/* User Mode Traps */
switch (type) {
case EXC_RUNMODETRC:
case EXC_TRC:
frame->srr1 &= ~PSL_SE;
sig = SIGTRAP;
ucode = TRAP_TRACE;
break;
#ifdef __powerpc64__
case EXC_ISE:
case EXC_DSE:
if (handle_user_slb_spill(&p->p_vmspace->vm_pmap,
(type == EXC_ISE) ? frame->srr0 :
frame->cpu.aim.dar) != 0) {
sig = SIGSEGV;
ucode = SEGV_MAPERR;
}
break;
#endif
case EXC_DSI:
case EXC_ISI:
sig = trap_pfault(frame, 1);
if (sig == SIGSEGV)
ucode = SEGV_MAPERR;
break;
case EXC_SC:
syscall(frame);
break;
case EXC_FPU:
KASSERT((td->td_pcb->pcb_flags & PCB_FPU) != PCB_FPU,
("FPU already enabled for thread"));
enable_fpu(td);
break;
case EXC_VEC:
KASSERT((td->td_pcb->pcb_flags & PCB_VEC) != PCB_VEC,
("Altivec already enabled for thread"));
enable_vec(td);
break;
case EXC_VECAST_G4:
case EXC_VECAST_G5:
/*
* We get a VPU assist exception for IEEE mode
* vector operations on denormalized floats.
* Emulating this is a giant pain, so for now,
* just switch off IEEE mode and treat them as
* zero.
*/
save_vec(td);
td->td_pcb->pcb_vec.vscr |= ALTIVEC_VSCR_NJ;
enable_vec(td);
break;
case EXC_ALI:
if (fix_unaligned(td, frame) != 0) {
sig = SIGBUS;
ucode = BUS_ADRALN;
}
else
frame->srr0 += 4;
break;
case EXC_PGM:
/* Identify the trap reason */
if (frame->srr1 & EXC_PGM_TRAP) {
#ifdef KDTRACE_HOOKS
inst = fuword32((const void *)frame->srr0);
if (inst == 0x0FFFDDDD && dtrace_pid_probe_ptr != NULL) {
struct reg regs;
fill_regs(td, ®s);
(*dtrace_pid_probe_ptr)(®s);
break;
}
#endif
sig = SIGTRAP;
ucode = TRAP_BRKPT;
} else {
sig = ppc_instr_emulate(frame, td->td_pcb);
if (sig == SIGILL) {
if (frame->srr1 & EXC_PGM_PRIV)
ucode = ILL_PRVOPC;
else if (frame->srr1 & EXC_PGM_ILLEGAL)
ucode = ILL_ILLOPC;
} else if (sig == SIGFPE)
ucode = FPE_FLTINV; /* Punt for now, invalid operation. */
}
break;
case EXC_MCHK:
/*
* Note that this may not be recoverable for the user
* process, depending on the type of machine check,
* but it at least prevents the kernel from dying.
*/
sig = SIGBUS;
ucode = BUS_OBJERR;
break;
default:
trap_fatal(frame);
}
} else {
/* Kernel Mode Traps */
KASSERT(cold || td->td_ucred != NULL,
("kernel trap doesn't have ucred"));
switch (type) {
#ifdef KDTRACE_HOOKS
case EXC_PGM:
if (frame->srr1 & EXC_PGM_TRAP) {
if (*(uint32_t *)frame->srr0 == 0x7c810808) {
if (dtrace_invop_jump_addr != NULL) {
dtrace_invop_jump_addr(frame);
return;
}
}
}
break;
#endif
#ifdef __powerpc64__
case EXC_DSE:
if ((frame->cpu.aim.dar & SEGMENT_MASK) == USER_ADDR) {
__asm __volatile ("slbmte %0, %1" ::
"r"(td->td_pcb->pcb_cpu.aim.usr_vsid),
"r"(USER_SLB_SLBE));
return;
}
break;
#endif
case EXC_DSI:
if (trap_pfault(frame, 0) == 0)
return;
break;
case EXC_MCHK:
if (handle_onfault(frame))
return;
break;
default:
break;
}
trap_fatal(frame);
}
