/*ARGSUSED*/ void trap(struct trapframe *frame) { struct proc *p = curproc; int type = (int)frame->tf_trapno; struct pcb *pcb; extern char doreti_iret[], resume_iret[]; caddr_t onfault; int error; uint64_t cr2; union sigval sv; uvmexp.traps++; pcb = (p != NULL && p->p_addr != NULL) ? &p->p_addr->u_pcb : NULL; #ifdef DEBUG if (trapdebug) { printf("trap %d code %lx rip %lx cs %lx rflags %lx cr2 %lx " "cpl %x\n", type, frame->tf_err, frame->tf_rip, frame->tf_cs, frame->tf_rflags, rcr2(), curcpu()->ci_ilevel); printf("curproc %p\n", curproc); if (curproc) printf("pid %d\n", p->p_pid); } #endif if (!KERNELMODE(frame->tf_cs, frame->tf_rflags)) { type |= T_USER; p->p_md.md_regs = frame; } switch (type) { default: we_re_toast: #ifdef KGDB if (kgdb_trap(type, frame)) return; else { /* * If this is a breakpoint, don't panic * if we're not connected. */ if (type == T_BPTFLT) { printf("kgdb: ignored %s\n", trap_type[type]); return; } } #endif #ifdef DDB if (kdb_trap(type, 0, frame)) return; #endif if (frame->tf_trapno < trap_types) printf("fatal %s", trap_type[frame->tf_trapno]); else printf("unknown trap %ld", (u_long)frame->tf_trapno); printf(" in %s mode\n", (type & T_USER) ? "user" : "supervisor"); printf("trap type %d code %lx rip %lx cs %lx rflags %lx cr2 " " %lx cpl %x rsp %lx\n", type, frame->tf_err, (u_long)frame->tf_rip, frame->tf_cs, frame->tf_rflags, rcr2(), curcpu()->ci_ilevel, frame->tf_rsp); panic("trap type %d, code=%lx, pc=%lx", type, frame->tf_err, frame->tf_rip); /*NOTREACHED*/ case T_PROTFLT: case T_SEGNPFLT: case T_ALIGNFLT: case T_TSSFLT: if (p == NULL) goto we_re_toast; /* Check for copyin/copyout fault. */ if (pcb->pcb_onfault != 0) { error = EFAULT; copyfault: frame->tf_rip = (u_int64_t)pcb->pcb_onfault; frame->tf_rax = error; return; } /* * Check for failure during return to user mode. * We do this by looking at the address of the * instruction that faulted. */ if (frame->tf_rip == (u_int64_t)doreti_iret) { frame->tf_rip = (u_int64_t)resume_iret; return; } goto we_re_toast; case T_PROTFLT|T_USER: /* protection fault */ case T_TSSFLT|T_USER: case T_SEGNPFLT|T_USER: case T_STKFLT|T_USER: case T_NMI|T_USER: #ifdef TRAP_SIGDEBUG printf("pid %d (%s): BUS at rip %lx addr %lx\n", p->p_pid, p->p_comm, frame->tf_rip, rcr2()); frame_dump(frame); #endif sv.sival_ptr = (void *)frame->tf_rip; KERNEL_LOCK(); trapsignal(p, SIGBUS, type & ~T_USER, BUS_OBJERR, sv); KERNEL_UNLOCK(); goto out; case T_ALIGNFLT|T_USER: sv.sival_ptr = (void *)frame->tf_rip; KERNEL_LOCK(); trapsignal(p, SIGBUS, type & ~T_USER, BUS_ADRALN, sv); KERNEL_UNLOCK(); goto out; case T_PRIVINFLT|T_USER: /* privileged instruction fault */ sv.sival_ptr = (void *)frame->tf_rip; KERNEL_LOCK(); trapsignal(p, SIGILL, type & ~T_USER, ILL_PRVOPC, sv); KERNEL_UNLOCK(); goto out; case T_FPOPFLT|T_USER: /* coprocessor operand fault */ #ifdef TRAP_SIGDEBUG printf("pid %d (%s): ILL at rip %lx addr %lx\n", p->p_pid, p->p_comm, frame->tf_rip, rcr2()); frame_dump(frame); #endif sv.sival_ptr = (void *)frame->tf_rip; KERNEL_LOCK(); trapsignal(p, SIGILL, type & ~T_USER, ILL_COPROC, sv); KERNEL_UNLOCK(); goto out; case T_ASTFLT|T_USER: /* Allow process switch */ uvmexp.softs++; if (p->p_flag & P_OWEUPC) { KERNEL_LOCK(); ADDUPROF(p); KERNEL_UNLOCK(); } /* Allow a forced task switch. */ if (curcpu()->ci_want_resched) preempt(NULL); goto out; case T_BOUND|T_USER: sv.sival_ptr = (void *)frame->tf_rip; KERNEL_LOCK(); trapsignal(p, SIGFPE, type &~ T_USER, FPE_FLTSUB, sv); KERNEL_UNLOCK(); goto out; case T_OFLOW|T_USER: sv.sival_ptr = (void *)frame->tf_rip; KERNEL_LOCK(); trapsignal(p, SIGFPE, type &~ T_USER, FPE_INTOVF, sv); KERNEL_UNLOCK(); goto out; case T_DIVIDE|T_USER: sv.sival_ptr = (void *)frame->tf_rip; KERNEL_LOCK(); trapsignal(p, SIGFPE, type &~ T_USER, FPE_INTDIV, sv); KERNEL_UNLOCK(); goto out; case T_ARITHTRAP|T_USER: case T_XMM|T_USER: fputrap(frame); goto out; case T_PAGEFLT: /* allow page faults in kernel mode */ if (p == NULL) goto we_re_toast; cr2 = rcr2(); KERNEL_LOCK(); goto faultcommon; case T_PAGEFLT|T_USER: { /* page fault */ vaddr_t va, fa; struct vmspace *vm; struct vm_map *map; vm_prot_t ftype; extern struct vm_map *kernel_map; cr2 = rcr2(); KERNEL_LOCK(); faultcommon: vm = p->p_vmspace; if (vm == NULL) goto we_re_toast; fa = cr2; va = trunc_page((vaddr_t)cr2); /* * It is only a kernel address space fault iff: * 1. (type & T_USER) == 0 and * 2. pcb_onfault not set or * 3. pcb_onfault set but supervisor space fault * The last can occur during an exec() copyin where the * argument space is lazy-allocated. */ if (type == T_PAGEFLT && va >= VM_MIN_KERNEL_ADDRESS) map = kernel_map; else map = &vm->vm_map; if (frame->tf_err & PGEX_W) ftype = VM_PROT_WRITE; else if (frame->tf_err & PGEX_I) ftype = VM_PROT_EXECUTE; else ftype = VM_PROT_READ; #ifdef DIAGNOSTIC if (map == kernel_map && va == 0) { printf("trap: bad kernel access at %lx\n", va); goto we_re_toast; } #endif /* Fault the original page in. */ onfault = pcb->pcb_onfault; pcb->pcb_onfault = NULL; error = uvm_fault(map, va, frame->tf_err & PGEX_P? VM_FAULT_PROTECT : VM_FAULT_INVALID, ftype); pcb->pcb_onfault = onfault; if (error == 0) { if (map != kernel_map) uvm_grow(p, va); if (type == T_PAGEFLT) { KERNEL_UNLOCK(); return; } KERNEL_UNLOCK(); goto out; } if (error == EACCES) { error = EFAULT; } if (type == T_PAGEFLT) { if (pcb->pcb_onfault != 0) { KERNEL_UNLOCK(); goto copyfault; } printf("uvm_fault(%p, 0x%lx, 0, %d) -> %x\n", map, va, ftype, error); goto we_re_toast; } if (error == ENOMEM) { printf("UVM: pid %d (%s), uid %d killed: out of swap\n", p->p_pid, p->p_comm, p->p_cred && p->p_ucred ? (int)p->p_ucred->cr_uid : -1); sv.sival_ptr = (void *)fa; trapsignal(p, SIGKILL, T_PAGEFLT, SEGV_MAPERR, sv); } else { #ifdef TRAP_SIGDEBUG printf("pid %d (%s): SEGV at rip %lx addr %lx\n", p->p_pid, p->p_comm, frame->tf_rip, va); frame_dump(frame); #endif sv.sival_ptr = (void *)fa; trapsignal(p, SIGSEGV, T_PAGEFLT, SEGV_MAPERR, sv); } KERNEL_UNLOCK(); break; } case T_TRCTRAP: goto we_re_toast; case T_BPTFLT|T_USER: /* bpt instruction fault */ case T_TRCTRAP|T_USER: /* trace trap */ #ifdef MATH_EMULATE trace: #endif KERNEL_LOCK(); trapsignal(p, SIGTRAP, type &~ T_USER, TRAP_BRKPT, sv); KERNEL_UNLOCK(); break; #if NISA > 0 case T_NMI: #if defined(KGDB) || defined(DDB) /* NMI can be hooked up to a pushbutton for debugging */ printf ("NMI ... going to debugger\n"); #ifdef KGDB if (kgdb_trap(type, frame)) return; #endif #ifdef DDB if (kdb_trap(type, 0, frame)) return; #endif #endif /* KGDB || DDB */ /* machine/parity/power fail/"kitchen sink" faults */ if (x86_nmi() != 0) goto we_re_toast; else return; #endif /* NISA > 0 */ } if ((type & T_USER) == 0) return; out: userret(p); }
void trap(struct trapframe *frame) { struct thread *td = curthread; struct proc *p = td->td_proc; int i = 0, ucode = 0, code; u_int type; register_t addr = 0; ksiginfo_t ksi; PCPU_INC(cnt.v_trap); type = frame->tf_trapno; #ifdef SMP #ifdef STOP_NMI /* Handler for NMI IPIs used for stopping CPUs. */ if (type == T_NMI) { if (ipi_nmi_handler() == 0) goto out; } #endif /* STOP_NMI */ #endif /* SMP */ #ifdef KDB if (kdb_active) { kdb_reenter(); goto out; } #endif #ifdef HWPMC_HOOKS /* * CPU PMCs interrupt using an NMI. If the PMC module is * active, pass the 'rip' value to the PMC module's interrupt * handler. A return value of '1' from the handler means that * the NMI was handled by it and we can return immediately. */ if (type == T_NMI && pmc_intr && (*pmc_intr)(PCPU_GET(cpuid), frame)) goto out; #endif if (type == T_MCHK) { if (!mca_intr()) trap_fatal(frame, 0); goto out; } #ifdef KDTRACE_HOOKS /* * A trap can occur while DTrace executes a probe. Before * executing the probe, DTrace blocks re-scheduling and sets * a flag in it's per-cpu flags to indicate that it doesn't * want to fault. On returning from the the probe, the no-fault * flag is cleared and finally re-scheduling is enabled. * * If the DTrace kernel module has registered a trap handler, * call it and if it returns non-zero, assume that it has * handled the trap and modified the trap frame so that this * function can return normally. */ if (dtrace_trap_func != NULL) if ((*dtrace_trap_func)(frame, type)) goto out; #endif if ((frame->tf_rflags & PSL_I) == 0) { /* * Buggy application or kernel code has disabled * interrupts and then trapped. Enabling interrupts * now is wrong, but it is better than running with * interrupts disabled until they are accidentally * enabled later. */ if (ISPL(frame->tf_cs) == SEL_UPL) printf( "pid %ld (%s): trap %d with interrupts disabled\n", (long)curproc->p_pid, curproc->p_comm, type); else if (type != T_NMI && type != T_BPTFLT && type != T_TRCTRAP) { /* * XXX not quite right, since this may be for a * multiple fault in user mode. */ printf("kernel trap %d with interrupts disabled\n", type); /* * We shouldn't enable interrupts while holding a * spin lock or servicing an NMI. */ if (type != T_NMI && td->td_md.md_spinlock_count == 0) enable_intr(); } } code = frame->tf_err; if (type == T_PAGEFLT) { /* * If we get a page fault while in a critical section, then * it is most likely a fatal kernel page fault. The kernel * is already going to panic trying to get a sleep lock to * do the VM lookup, so just consider it a fatal trap so the * kernel can print out a useful trap message and even get * to the debugger. * * If we get a page fault while holding a non-sleepable * lock, then it is most likely a fatal kernel page fault. * If WITNESS is enabled, then it's going to whine about * bogus LORs with various VM locks, so just skip to the * fatal trap handling directly. */ if (td->td_critnest != 0 || WITNESS_CHECK(WARN_SLEEPOK | WARN_GIANTOK, NULL, "Kernel page fault") != 0) trap_fatal(frame, frame->tf_addr); } if (ISPL(frame->tf_cs) == SEL_UPL) { /* user trap */ td->td_pticks = 0; td->td_frame = frame; addr = frame->tf_rip; if (td->td_ucred != p->p_ucred) cred_update_thread(td); switch (type) { case T_PRIVINFLT: /* privileged instruction fault */ i = SIGILL; ucode = ILL_PRVOPC; break; case T_BPTFLT: /* bpt instruction fault */ case T_TRCTRAP: /* trace trap */ enable_intr(); frame->tf_rflags &= ~PSL_T; i = SIGTRAP; ucode = (type == T_TRCTRAP ? TRAP_TRACE : TRAP_BRKPT); break; case T_ARITHTRAP: /* arithmetic trap */ ucode = fputrap(); if (ucode == -1) goto userout; i = SIGFPE; break; case T_PROTFLT: /* general protection fault */ i = SIGBUS; ucode = BUS_OBJERR; break; case T_STKFLT: /* stack fault */ case T_SEGNPFLT: /* segment not present fault */ i = SIGBUS; ucode = BUS_ADRERR; break; case T_TSSFLT: /* invalid TSS fault */ i = SIGBUS; ucode = BUS_OBJERR; break; case T_DOUBLEFLT: /* double fault */ default: i = SIGBUS; ucode = BUS_OBJERR; break; case T_PAGEFLT: /* page fault */ addr = frame->tf_addr; #ifdef KSE if (td->td_pflags & TDP_SA) thread_user_enter(td); #endif i = trap_pfault(frame, TRUE); if (i == -1) goto userout; if (i == 0) goto user; if (i == SIGSEGV) ucode = SEGV_MAPERR; else { if (prot_fault_translation == 0) { /* * Autodetect. * This check also covers the images * without the ABI-tag ELF note. */ if ((curproc->p_sysent == &elf64_freebsd_sysvec #ifdef COMPAT_IA32 || curproc->p_sysent == &ia32_freebsd_sysvec #endif ) && p->p_osrel >= 700004) { i = SIGSEGV; ucode = SEGV_ACCERR; } else { i = SIGBUS; ucode = BUS_PAGE_FAULT; } } else if (prot_fault_translation == 1) { /* * Always compat mode. */ i = SIGBUS; ucode = BUS_PAGE_FAULT; } else { /* * Always SIGSEGV mode. */ i = SIGSEGV; ucode = SEGV_ACCERR; } } break; case T_DIVIDE: /* integer divide fault */ ucode = FPE_INTDIV; i = SIGFPE; break; #ifdef DEV_ISA case T_NMI: /* machine/parity/power fail/"kitchen sink" faults */ /* XXX Giant */ if (isa_nmi(code) == 0) { #ifdef KDB /* * NMI can be hooked up to a pushbutton * for debugging. */ if (kdb_on_nmi) { printf ("NMI ... going to debugger\n"); kdb_trap(type, 0, frame); } #endif /* KDB */ goto userout; } else if (panic_on_nmi) panic("NMI indicates hardware failure"); break; #endif /* DEV_ISA */ case T_OFLOW: /* integer overflow fault */ ucode = FPE_INTOVF; i = SIGFPE; break; case T_BOUND: /* bounds check fault */ ucode = FPE_FLTSUB; i = SIGFPE; break; case T_DNA: /* transparent fault (due to context switch "late") */ fpudna(); goto userout; case T_FPOPFLT: /* FPU operand fetch fault */ ucode = ILL_COPROC; i = SIGILL; break; case T_XMMFLT: /* SIMD floating-point exception */ ucode = 0; /* XXX */ i = SIGFPE; break; } } else { /* kernel trap */ KASSERT(cold || td->td_ucred != NULL, ("kernel trap doesn't have ucred")); switch (type) { case T_PAGEFLT: /* page fault */ (void) trap_pfault(frame, FALSE); goto out; case T_DNA: /* * The kernel is apparently using fpu for copying. * XXX this should be fatal unless the kernel has * registered such use. */ fpudna(); printf("fpudna in kernel mode!\n"); goto out; case T_STKFLT: /* stack fault */ break; case T_PROTFLT: /* general protection fault */ case T_SEGNPFLT: /* segment not present fault */ if (td->td_intr_nesting_level != 0) break; /* * Invalid segment selectors and out of bounds * %rip's and %rsp's can be set up in user mode. * This causes a fault in kernel mode when the * kernel tries to return to user mode. We want * to get this fault so that we can fix the * problem here and not have to check all the * selectors and pointers when the user changes * them. */ if (frame->tf_rip == (long)doreti_iret) { frame->tf_rip = (long)doreti_iret_fault; goto out; } if (PCPU_GET(curpcb)->pcb_onfault != NULL) { frame->tf_rip = (long)PCPU_GET(curpcb)->pcb_onfault; goto out; } break; case T_TSSFLT: /* * PSL_NT can be set in user mode and isn't cleared * automatically when the kernel is entered. This * causes a TSS fault when the kernel attempts to * `iret' because the TSS link is uninitialized. We * want to get this fault so that we can fix the * problem here and not every time the kernel is * entered. */ if (frame->tf_rflags & PSL_NT) { frame->tf_rflags &= ~PSL_NT; goto out; } break; case T_TRCTRAP: /* trace trap */ /* * Ignore debug register trace traps due to * accesses in the user's address space, which * can happen under several conditions such as * if a user sets a watchpoint on a buffer and * then passes that buffer to a system call. * We still want to get TRCTRAPS for addresses * in kernel space because that is useful when * debugging the kernel. */ if (user_dbreg_trap()) { /* * Reset breakpoint bits because the * processor doesn't */ /* XXX check upper bits here */ load_dr6(rdr6() & 0xfffffff0); goto out; } /* * FALLTHROUGH (TRCTRAP kernel mode, kernel address) */ case T_BPTFLT: /* * If KDB is enabled, let it handle the debugger trap. * Otherwise, debugger traps "can't happen". */ #ifdef KDB if (kdb_trap(type, 0, frame)) goto out; #endif break; #ifdef DEV_ISA case T_NMI: /* XXX Giant */ /* machine/parity/power fail/"kitchen sink" faults */ if (isa_nmi(code) == 0) { #ifdef KDB /* * NMI can be hooked up to a pushbutton * for debugging. */ if (kdb_on_nmi) { printf ("NMI ... going to debugger\n"); kdb_trap(type, 0, frame); } #endif /* KDB */ goto out; } else if (panic_on_nmi == 0) goto out; /* FALLTHROUGH */ #endif /* DEV_ISA */ } trap_fatal(frame, 0); goto out; } /* Translate fault for emulators (e.g. Linux) */ if (*p->p_sysent->sv_transtrap) i = (*p->p_sysent->sv_transtrap)(i, type); ksiginfo_init_trap(&ksi); ksi.ksi_signo = i; ksi.ksi_code = ucode; ksi.ksi_trapno = type; ksi.ksi_addr = (void *)addr; trapsignal(td, &ksi); #ifdef DEBUG if (type <= MAX_TRAP_MSG) { uprintf("fatal process exception: %s", trap_msg[type]); if ((type == T_PAGEFLT) || (type == T_PROTFLT)) uprintf(", fault VA = 0x%lx", frame->tf_addr); uprintf("\n"); } #endif user: userret(td, frame); mtx_assert(&Giant, MA_NOTOWNED); userout: out: return; }