/* * Simplified back end of syscall(), used when returning from fork() * directly into user mode. * * This code will return back into the fork trampoline code which then * runs doreti. * * NOTE: The mplock is not held at any point. */ void generic_lwp_return(struct lwp *lp, struct trapframe *frame) { struct proc *p = lp->lwp_proc; /* * Newly forked processes are given a kernel priority. We have to * adjust the priority to a normal user priority and fake entry * into the kernel (call userenter()) to install a passive release * function just in case userret() decides to stop the process. This * can occur when ^Z races a fork. If we do not install the passive * release function the current process designation will not be * released when the thread goes to sleep. */ lwkt_setpri_self(TDPRI_USER_NORM); userenter(lp->lwp_thread, p); userret(lp, frame, 0); #ifdef KTRACE if (KTRPOINT(lp->lwp_thread, KTR_SYSRET)) ktrsysret(lp, SYS_fork, 0, 0); #endif lp->lwp_flags |= LWP_PASSIVE_ACQ; userexit(lp); lp->lwp_flags &= ~LWP_PASSIVE_ACQ; }
/* * Simplified back end of syscall(), used when returning from fork() * directly into user mode. * * This code will return back into the fork trampoline code which then * runs doreti. */ void generic_lwp_return(struct lwp *lp, struct trapframe *frame) { struct proc *p = lp->lwp_proc; /* * Check for exit-race. If one lwp exits the process concurrent with * another lwp creating a new thread, the two operations may cross * each other resulting in the newly-created lwp not receiving a * KILL signal. */ if (p->p_flags & P_WEXIT) { lwpsignal(p, lp, SIGKILL); } /* * Newly forked processes are given a kernel priority. We have to * adjust the priority to a normal user priority and fake entry * into the kernel (call userenter()) to install a passive release * function just in case userret() decides to stop the process. This * can occur when ^Z races a fork. If we do not install the passive * release function the current process designation will not be * released when the thread goes to sleep. */ lwkt_setpri_self(TDPRI_USER_NORM); userenter(lp->lwp_thread, p); userret(lp, frame, 0); #ifdef KTRACE if (KTRPOINT(lp->lwp_thread, KTR_SYSRET)) ktrsysret(lp, SYS_fork, 0, 0); #endif lp->lwp_flags |= LWP_PASSIVE_ACQ; userexit(lp); lp->lwp_flags &= ~LWP_PASSIVE_ACQ; }
/* * abort_imprecise() handles the following abort: * * FAULT_EA_IMPREC - Imprecise External Abort * * The imprecise means that we don't know where the abort happened, * thus FAR is undefined. The abort should not never fire, but hot * plugging or accidental hardware failure can be the cause of it. * If the abort happens, it can even be on different (thread) context. * Without any additional support, the abort is fatal, as we do not * know what really happened. * * QQQ: Some additional functionality, like pcb_onfault but global, * can be implemented. Imprecise handlers could be registered * which tell us if the abort is caused by something they know * about. They should return one of three codes like: * FAULT_IS_MINE, * FAULT_CAN_BE_MINE, * FAULT_IS_NOT_MINE. * The handlers should be called until some of them returns * FAULT_IS_MINE value or all was called. If all handlers return * FAULT_IS_NOT_MINE value, then the abort is fatal. */ static __inline void abort_imprecise(struct trapframe *tf, u_int fsr, u_int prefetch, bool usermode) { /* * XXX - We can got imprecise abort as result of access * to not-present PCI/PCIe configuration space. */ #if 0 goto out; #endif abort_fatal(tf, FAULT_EA_IMPREC, fsr, 0, prefetch, curthread, NULL); /* * Returning from this function means that we ignore * the abort for good reason. Note that imprecise abort * could fire any time even in user mode. */ #if 0 out: if (usermode) userret(curthread, tf); #endif }
void swi_handler(trapframe_t *frame) { struct thread *td = curthread; td->td_frame = frame; td->td_pticks = 0; /* * Make sure the program counter is correctly aligned so we * don't take an alignment fault trying to read the opcode. */ if (__predict_false(((frame->tf_pc - INSN_SIZE) & 3) != 0)) { call_trapsignal(td, SIGILL, 0); userret(td, frame); return; } /* * Enable interrupts if they were enabled before the exception. * Since all syscalls *should* come from user mode it will always * be safe to enable them, but check anyway. */ if (td->td_md.md_spinlock_count == 0) { if (__predict_true(frame->tf_spsr & I32_bit) == 0) enable_interrupts(I32_bit); if (__predict_true(frame->tf_spsr & F32_bit) == 0) enable_interrupts(F32_bit); } syscall(td, frame); }
void ast(struct trapframe *tf) { struct proc *p = curproc;; #ifdef acorn26 /* Enable interrupts if they were enabled before the trap. */ if ((tf->tf_r15 & R15_IRQ_DISABLE) == 0) int_on(); #else /* Interrupts were restored by exception_exit. */ #endif uvmexp.traps++; uvmexp.softs++; #ifdef DEBUG if (p == NULL) panic("ast: no curproc!"); if (&p->p_addr->u_pcb == 0) panic("ast: no pcb!"); #endif if (p->p_flag & P_OWEUPC) { p->p_flag &= ~P_OWEUPC; ADDUPROF(p); } /* Allow a forced task switch. */ if (want_resched) preempt(0); userret(p, tf->tf_pc, p->p_sticks); /* XXX */ }
/* * Simplified back end of syscall(), used when returning from fork() * directly into user mode. Giant is not held on entry, and must not * be held on return. This function is passed in to fork_exit() as the * first parameter and is called when returning to a new userland process. */ void fork_return(struct thread *td, struct trapframe *frame) { struct proc *p, *dbg; p = td->td_proc; if (td->td_dbgflags & TDB_STOPATFORK) { sx_xlock(&proctree_lock); PROC_LOCK(p); if ((p->p_pptr->p_flag & (P_TRACED | P_FOLLOWFORK)) == (P_TRACED | P_FOLLOWFORK)) { /* * If debugger still wants auto-attach for the * parent's children, do it now. */ dbg = p->p_pptr->p_pptr; p->p_flag |= P_TRACED; p->p_oppid = p->p_pptr->p_pid; CTR2(KTR_PTRACE, "fork_return: attaching to new child pid %d: oppid %d", p->p_pid, p->p_oppid); proc_reparent(p, dbg); sx_xunlock(&proctree_lock); td->td_dbgflags |= TDB_CHILD | TDB_SCX; ptracestop(td, SIGSTOP); td->td_dbgflags &= ~(TDB_CHILD | TDB_SCX); } else { /* * ... otherwise clear the request. */ sx_xunlock(&proctree_lock); td->td_dbgflags &= ~TDB_STOPATFORK; cv_broadcast(&p->p_dbgwait); } PROC_UNLOCK(p); } else if (p->p_flag & P_TRACED || td->td_dbgflags & TDB_BORN) { /* * This is the start of a new thread in a traced * process. Report a system call exit event. */ PROC_LOCK(p); td->td_dbgflags |= TDB_SCX; _STOPEVENT(p, S_SCX, td->td_dbg_sc_code); if ((p->p_stops & S_PT_SCX) != 0 || (td->td_dbgflags & TDB_BORN) != 0) ptracestop(td, SIGTRAP); td->td_dbgflags &= ~(TDB_SCX | TDB_BORN); PROC_UNLOCK(p); } userret(td, frame); #ifdef KTRACE if (KTRPOINT(td, KTR_SYSRET)) ktrsysret(SYS_fork, 0, 0); #endif }
void trap_handle_userexit(struct trapframe *frame, int sticks) { struct lwp *lp = curthread->td_lwp; if (lp) { userret(lp, frame, sticks); userexit(lp); } }
/* * Handle an AST for the current process. */ void ast() { struct cpu_info *ci = curcpu(); struct proc *p = ci->ci_curproc; p->p_md.md_astpending = 0; atomic_add_int(&uvmexp.softs, 1); mi_ast(p, ci->ci_want_resched); userret(p); }
/* * startlwp: start of a new LWP. */ void startlwp(void *arg) { ucontext_t *uc = arg; lwp_t *l = curlwp; int error __diagused; error = cpu_setmcontext(l, &uc->uc_mcontext, uc->uc_flags); KASSERT(error == 0); kmem_free(uc, sizeof(ucontext_t)); userret(l); }
void data_abort_handler(struct trapframe *tf) { vaddr_t pc, va; vsize_t asize; struct proc *p; struct lwp *l; vm_prot_t atype; bool usrmode, twopages; struct vm_map *map; /* * Data aborts in kernel mode are possible (copyout etc), so * we hope the compiler (or programmer) has ensured that * R14_svc gets saved. * * We may need to fix up an STM or LDM instruction. This * involves seeing if the base was being written back, and if * so resetting it (by counting the number of registers being * transferred) before retrying (ARM 2 ds pp 10 & 33). */ /* Enable interrupts if they were enabled before the trap. */ if ((tf->tf_r15 & R15_IRQ_DISABLE) == 0) int_on(); uvmexp.traps++; l = curlwp; if (l == NULL) l = &lwp0; p = l->l_proc; if ((tf->tf_r15 & R15_MODE) == R15_MODE_USR) { l->l_addr->u_pcb.pcb_tf = tf; LWP_CACHE_CREDS(l, p); } pc = tf->tf_r15 & R15_PC; data_abort_fixup(tf); va = data_abort_address(tf, &asize); atype = data_abort_atype(tf); usrmode = data_abort_usrmode(tf); twopages = (trunc_page(va) != round_page(va + asize) - PAGE_SIZE); if (!usrmode && va >= VM_MIN_KERNEL_ADDRESS) map = kernel_map; else map = &p->p_vmspace->vm_map; do_fault(tf, l, map, va, atype); if (twopages) do_fault(tf, l, map, va + asize - 4, atype); if ((tf->tf_r15 & R15_MODE) == R15_MODE_USR) userret(l); }
/* * Start a new LWP */ void startlwp32(void *arg) { ucontext32_t * const uc = arg; int error __diagused; error = cpu_setmcontext32(curlwp, &uc->uc_mcontext, uc->uc_flags); KASSERT(error == 0); // Even though this is a ucontext32_t, the space allocated was for a // full ucontext_t kmem_free(uc, sizeof(ucontext_t)); userret(curlwp); }
/* * Start a new LWP */ void startlwp(void *arg) { ucontext_t * const uc = arg; lwp_t * const l = curlwp; int error __diagused; error = cpu_setmcontext(l, &uc->uc_mcontext, uc->uc_flags); KASSERT(error == 0); kmem_free(uc, sizeof(ucontext_t)); /* XXX - profiling spoiled here */ userret(l, l->l_md.md_utf, l->l_proc->p_sticks); }
void startlwp32(void *arg) { ucontext32_t *uc = arg; lwp_t *l = curlwp; int error __diagused; error = cpu_setmcontext32(l, &uc->uc_mcontext, uc->uc_flags); KASSERT(error == 0); /* Note: we are freeing ucontext_t, not ucontext32_t. */ kmem_free(arg, sizeof(ucontext_t)); userret(l, 0, 0); }
void prefetch_abort_handler(struct trapframe *tf) { vaddr_t pc; struct proc *p; struct lwp *l; /* Enable interrupts if they were enabled before the trap. */ if ((tf->tf_r15 & R15_IRQ_DISABLE) == 0) int_on(); /* * XXX Not done yet: * Check if the page being requested is already present. If * so, call the undefined instruction handler instead (ARM3 ds * p15). */ uvmexp.traps++; l = curlwp; if (l == NULL) l = &lwp0; p = l->l_proc; if ((tf->tf_r15 & R15_MODE) == R15_MODE_USR) { l->l_addr->u_pcb.pcb_tf = tf; LWP_CACHE_CREDS(l, p); } if ((tf->tf_r15 & R15_MODE) != R15_MODE_USR) { #ifdef DDB db_printf("Prefetch abort in kernel mode\n"); kdb_trap(T_FAULT, tf); #else #ifdef DEBUG printf("Prefetch abort:\n"); printregs(tf); #endif panic("prefetch abort in kernel mode"); #endif } /* User-mode prefetch abort */ pc = tf->tf_r15 & R15_PC; do_fault(tf, l, &p->p_vmspace->vm_map, pc, VM_PROT_EXECUTE); userret(l); }
void child_return(void *arg) { struct lwp *l = arg; struct proc *p = l->l_proc; userret(l, l->l_md.md_regs->tf_iioq_head, 0); #ifdef KTRACE if (KTRPOINT(p, KTR_SYSRET)) ktrsysret(p, SYS_fork, 0, 0); #endif #ifdef DEBUG frame_sanity_check(l->l_md.md_regs, l); #endif /* DEBUG */ }
void child_return(void *arg) { lwp_t *l = arg; struct trapframe *frame = l->l_addr->u_pcb.pcb_tf; frame->tf_r0 = 0; #ifdef __PROG32 frame->tf_spsr &= ~PSR_C_bit; /* carry bit */ #else frame->tf_r15 &= ~R15_FLAG_C; /* carry bit */ #endif userret(l); ktrsysret(SYS_fork, 0, 0); }
/* * startlwp: * * Start a new LWP. */ void startlwp(void *arg) { int err; ucontext_t *uc = arg; struct lwp *l = curlwp; err = cpu_setmcontext(l, &uc->uc_mcontext, uc->uc_flags); #ifdef DIAGNOSTIC if (err) printf("Error %d from cpu_setmcontext.", err); #endif pool_put(&lwp_uc_pool, uc); userret(l); }
/* * Start a new LWP */ void startlwp(void *arg) { int err; ucontext_t *uc = arg; struct lwp *l = curlwp; err = cpu_setmcontext(l, &uc->uc_mcontext, uc->uc_flags); #if DIAGNOSTIC if (err) { printf("Error %d from cpu_setmcontext.", err); } #endif pool_put(&lwp_uc_pool, uc); userret(l, l->l_md.md_regs->tf_iioq_head, 0); }
void child_return(void *arg) { lwp_t *l = arg; register_t rval[2]; struct pcb *pcb = lwp_getpcb(l); ucontext_t *ucp = &pcb->pcb_userret_ucp; /* return value zero */ rval[0] = 0; rval[1] = 0; md_syscall_set_returnargs(l, ucp, 0, rval); aprint_debug("child return! lwp %p\n", l); userret(l); ktrsysret(SYS_fork, 0, 0); }
/* * If syscallemu specific data is present for the process, verify that the * caller is allowed to execute system calls. If not, deliver a SIGILL to * the process. When syscallemu specific data is not present, simply defer * to the original syscall handler. */ static void x86_syscall_emu(struct trapframe *frame) { void (*md_syscall)(struct trapframe *) = NULL; struct syscallemu_data *sce; register_t rip_call; struct proc *p; ksiginfo_t ksi; lwp_t *l; l = curlwp; p = l->l_proc; rip_call = X86_TF_RIP(frame) - frame->tf_err; /* Determine if we need to emulate the system call */ sce = syscallemu_getsce(p); if (sce) { if ((rip_call >= sce->sce_user_start && rip_call < sce->sce_user_end) || (rip_call + frame->tf_err >= sce->sce_user_start && rip_call + frame->tf_err < sce->sce_user_end)) { md_syscall = NULL; } else { md_syscall = sce->sce_md_syscall; } } else { md_syscall = p->p_md.md_syscall; } if (md_syscall == NULL) { /* If emulating, deliver SIGILL to process */ X86_TF_RIP(frame) = rip_call; KSI_INIT_TRAP(&ksi); ksi.ksi_signo = SIGILL; ksi.ksi_code = ILL_ILLTRP; ksi.ksi_addr = (void *)X86_TF_RIP(frame); ksi.ksi_trap = 0; trapsignal(l, &ksi); userret(l); } else { /* Not emulating, so treat as a normal syscall */ KASSERT(md_syscall != NULL); md_syscall(frame); } }
/* * abort_debug() handles the following abort: * * FAULT_DEBUG - Debug Event * */ static __inline void abort_debug(struct trapframe *tf, u_int fsr, u_int prefetch, u_int usermode, u_int far) { if (usermode) { struct thread *td; td = curthread; call_trapsignal(td, SIGTRAP, TRAP_BRKPT, far); userret(td, tf); } else { #ifdef KDB kdb_trap(T_BREAKPOINT, 0, tf); #else printf("No debugger in kernel.\n"); #endif } }
/* * Handle asynchronous software traps. */ void ast(struct trapframe *frame) { struct cpu_info *ci = curcpu(); struct proc *p = ci->ci_curproc; uvmexp.softs++; p->p_md.md_astpending = 0; if (p->p_flag & P_OWEUPC) { KERNEL_LOCK(); ADDUPROF(p); KERNEL_UNLOCK(); } if (ci->ci_want_resched) preempt(NULL); userret(p); }
/* * Process the tail end of a fork() for the child. */ void child_return(void *arg) { struct proc *p = arg; /* * Return values in the frame set by cpu_fork(). */ KERNEL_PROC_UNLOCK(p); userret(p); #ifdef KTRACE if (KTRPOINT(p, KTR_SYSRET)) { KERNEL_PROC_LOCK(p); ktrsysret(p, SYS_fork, 0, 0); KERNEL_PROC_UNLOCK(p); } #endif }
/* * Handle an AST for the current process. */ void ast() { struct cpu_info *ci = curcpu(); struct proc *p = ci->ci_curproc; uvmexp.softs++; if (p->p_md.md_astpending == 0) panic("unexpected ast p %p astpending %p\n", p, &p->p_md.md_astpending); p->p_md.md_astpending = 0; if (p->p_flag & P_OWEUPC) { ADDUPROF(p); } if (ci->ci_want_resched) preempt(NULL); userret(p); }
void prefetch_abort_handler(struct trapframe *tf) { struct lwp * const l = curlwp; struct proc * const p = l->l_proc; /* Enable interrupts if they were enabled before the trap. */ if ((tf->tf_r15 & R15_IRQ_DISABLE) == 0) int_on(); /* * XXX Not done yet: * Check if the page being requested is already present. If * so, call the undefined instruction handler instead (ARM3 ds * p15). */ curcpu()->ci_data.cpu_ntrap++; if (TRAP_USERMODE(tf)) { lwp_settrapframe(l, tf); LWP_CACHE_CREDS(l, p); } else { #ifdef DDB db_printf("Prefetch abort in kernel mode\n"); kdb_trap(T_FAULT, tf); #else #ifdef DEBUG printf("Prefetch abort:\n"); printregs(tf); #endif panic("prefetch abort in kernel mode"); #endif } /* User-mode prefetch abort */ vaddr_t pc = tf->tf_r15 & R15_PC; do_fault(tf, l, &p->p_vmspace->vm_map, pc, VM_PROT_EXECUTE); userret(l); }
void ast(struct trapframe *tf) { struct proc *p = curproc; /* Interrupts were restored by exception_exit. */ uvmexp.traps++; #ifdef DEBUG if (p == NULL) panic("ast: no curproc!"); if (&p->p_addr->u_pcb == 0) panic("ast: no pcb!"); #endif uvmexp.softs++; mi_ast(p, want_resched); userret(p); }
void ast(struct trapframe *tf) { struct lwp * const l = curlwp; #ifdef acorn26 /* Enable interrupts if they were enabled before the trap. */ if ((tf->tf_r15 & R15_IRQ_DISABLE) == 0) int_on(); #else /* Interrupts were restored by exception_exit. */ #endif #ifdef __PROG32 KASSERT(VALID_R15_PSR(tf->tf_pc, tf->tf_spsr)); #endif #ifdef __HAVE_PREEMPTION kpreempt_disable(); #endif struct cpu_info * const ci = curcpu(); ci->ci_data.cpu_ntrap++; KDASSERT(ci->ci_cpl == IPL_NONE); const int want_resched = ci->ci_want_resched; #ifdef __HAVE_PREEMPTION kpreempt_enable(); #endif if (l->l_pflag & LP_OWEUPC) { l->l_pflag &= ~LP_OWEUPC; ADDUPROF(l); } /* Allow a forced task switch. */ if (want_resched) preempt(); userret(l); }
void prefetch_abort_handler(struct trapframe *tf) { vaddr_t pc; struct proc *p; /* Enable interrupts if they were enabled before the trap. */ if ((tf->tf_r15 & R15_IRQ_DISABLE) == 0) int_on(); /* * XXX Not done yet: * Check if the page being requested is already present. If * so, call the undefined instruction handler instead (ARM3 ds * p15). */ uvmexp.traps++; p = curproc; if (p == NULL) p = &proc0; if ((tf->tf_r15 & R15_MODE) == R15_MODE_USR) p->p_addr->u_pcb.pcb_tf = tf; if ((tf->tf_r15 & R15_MODE) != R15_MODE_USR) { #ifdef DEBUG printf("Prefetch abort:\n"); printregs(tf); #endif panic("prefetch abort in kernel mode"); } /* User-mode prefetch abort */ pc = tf->tf_r15 & R15_PC; do_fault(tf, p, &p->p_vmspace->vm_map, pc, VM_PROT_EXECUTE); userret(p); }
static void syscall(struct thread *td, trapframe_t *frame) { struct syscall_args sa; int error; #ifndef __ARM_EABI__ sa.insn = *(uint32_t *)(frame->tf_pc - INSN_SIZE); switch (sa.insn & SWI_OS_MASK) { case 0: /* XXX: we need our own one. */ break; default: call_trapsignal(td, SIGILL, 0); userret(td, frame); return; } #endif sa.nap = 4; error = syscallenter(td, &sa); KASSERT(error != 0 || td->td_ar == NULL, ("returning from syscall with td_ar set!")); syscallret(td, error, &sa); }
/* * syscall(frame): * System call request from POSIX system call gate interface to kernel. * Like trap(), argument is call by reference. */ void linux_syscall(struct trapframe *frame) { register const struct sysent *callp; struct lwp *l; int error; register_t code, args[6], rval[2]; l = curlwp; LWP_CACHE_CREDS(l, l->l_proc); code = frame->tf_eax & (LINUX_SYS_NSYSENT - 1); callp = linux_sysent; callp += code; /* * Linux passes the args in ebx, ecx, edx, esi, edi, ebp, in * increasing order. */ args[0] = frame->tf_ebx; args[1] = frame->tf_ecx; args[2] = frame->tf_edx; args[3] = frame->tf_esi; args[4] = frame->tf_edi; args[5] = frame->tf_ebp; rval[0] = 0; rval[1] = 0; if (__predict_false(l->l_proc->p_trace_enabled)) { error = trace_enter(code, args, callp->sy_narg); if (__predict_true(error == 0)) { error = sy_call(callp, l, args, rval); code = frame->tf_eax & (LINUX_SYS_NSYSENT - 1); trace_exit(code, rval, error); } } else error = sy_call(callp, l, args, rval); if (__predict_true(error == 0)) { frame->tf_eax = rval[0]; /* * XXX The linux libc code I (dsl) looked at doesn't use the * carry bit. * Values above 0xfffff000 are assumed to be errno values and * not result codes! */ frame->tf_eflags &= ~PSL_C; /* carry bit */ } else { switch (error) { case ERESTART: /* * The offset to adjust the PC by depends on whether * we entered the kernel through the trap or call gate. * We save the instruction size in tf_err on entry. */ frame->tf_eip -= frame->tf_err; break; case EJUSTRETURN: /* nothing to do */ break; default: error = native_to_linux_errno[error]; frame->tf_eax = error; frame->tf_eflags |= PSL_C; /* carry bit */ break; } } userret(l); }