/* * uvm_emap_consume: update the current CPU and LWP to the given generation * of the emap. In a case of LWP migration to a different CPU after block * or preempt, uvm_emap_switch() will synchronise. * * => may be called from both interrupt and thread context. */ void uvm_emap_consume(u_int gen) { struct cpu_info *ci; struct uvm_cpu *ucpu; lwp_t *l = curlwp; u_int curgen; if (gen == UVM_EMAP_INACTIVE) { return; } /* * No need to synchronise if generation number of current CPU is * newer than the number of this LWP. * * This test assumes two's complement arithmetic and allows * ~2B missed updates before it will produce bad results. */ KPREEMPT_DISABLE(l); ci = l->l_cpu; ucpu = ci->ci_data.cpu_uvm; if (__predict_true((signed int)(ucpu->emap_gen - gen) >= 0)) { l->l_emap_gen = ucpu->emap_gen; KPREEMPT_ENABLE(l); return; } /* * Record the current generation _before_ issuing the TLB flush. * No need for a memory barrier before, as reading a stale value * for uvm_emap_gen is not a problem. * * pmap_emap_sync() must implicitly perform a full memory barrier, * which prevents us from fetching a value from after the TLB flush * has occurred (which would be bad). * * We can race with an interrupt on the current CPU updating the * counter to a newer value. This could cause us to set a stale * value into ucpu->emap_gen, overwriting a newer update from the * interrupt. However, it does not matter since: * (1) Interrupts always run to completion or block. * (2) Interrupts will only ever install a newer value and, * (3) We will roll the value forward later. */ curgen = uvm_emap_gen_return(); pmap_emap_sync(true); ucpu->emap_gen = curgen; l->l_emap_gen = curgen; KASSERT((signed int)(curgen - gen) >= 0); KPREEMPT_ENABLE(l); }
void fputrap(struct trapframe *frame) { register struct lwp *l = curcpu()->ci_fpcurlwp; struct pcb *pcb = lwp_getpcb(l); struct savefpu *sfp = &pcb->pcb_savefpu; uint32_t mxcsr, statbits; uint16_t cw; ksiginfo_t ksi; KPREEMPT_DISABLE(l); x86_enable_intr(); /* * At this point, fpcurlwp should be curlwp. If it wasn't, the TS bit * should be set, and we should have gotten a DNA exception. */ KASSERT(l == curlwp); fxsave(sfp); pcb->pcb_savefpu_i387.fp_ex_tw = sfp->fp_fxsave.fx_ftw; pcb->pcb_savefpu_i387.fp_ex_sw = sfp->fp_fxsave.fx_fsw; if (frame->tf_trapno == T_XMM) { mxcsr = sfp->fp_fxsave.fx_mxcsr; statbits = mxcsr; mxcsr &= ~0x3f; x86_ldmxcsr(&mxcsr); } else { fninit(); fwait(); cw = sfp->fp_fxsave.fx_fcw; fldcw(&cw); fwait(); statbits = sfp->fp_fxsave.fx_fsw; } KPREEMPT_ENABLE(l); KSI_INIT_TRAP(&ksi); ksi.ksi_signo = SIGFPE; ksi.ksi_addr = (void *)frame->tf_rip; ksi.ksi_code = x86fpflags_to_ksiginfo(statbits); ksi.ksi_trap = statbits; (*l->l_proc->p_emul->e_trapsignal)(l, &ksi); }
/* * mutex_vector_enter: * * Support routine for mutex_enter() that must handle all cases. In * the LOCKDEBUG case, mutex_enter() is always aliased here, even if * fast-path stubs are available. If a mutex_spin_enter() stub is * not available, then it is also aliased directly here. */ void mutex_vector_enter(kmutex_t *mtx) { uintptr_t owner, curthread; turnstile_t *ts; #ifdef MULTIPROCESSOR u_int count; #endif LOCKSTAT_COUNTER(spincnt); LOCKSTAT_COUNTER(slpcnt); LOCKSTAT_TIMER(spintime); LOCKSTAT_TIMER(slptime); LOCKSTAT_FLAG(lsflag); /* * Handle spin mutexes. */ if (MUTEX_SPIN_P(mtx)) { #if defined(LOCKDEBUG) && defined(MULTIPROCESSOR) u_int spins = 0; #endif MUTEX_SPIN_SPLRAISE(mtx); MUTEX_WANTLOCK(mtx); #ifdef FULL if (MUTEX_SPINBIT_LOCK_TRY(mtx)) { MUTEX_LOCKED(mtx); return; } #if !defined(MULTIPROCESSOR) MUTEX_ABORT(mtx, "locking against myself"); #else /* !MULTIPROCESSOR */ LOCKSTAT_ENTER(lsflag); LOCKSTAT_START_TIMER(lsflag, spintime); count = SPINLOCK_BACKOFF_MIN; /* * Spin testing the lock word and do exponential backoff * to reduce cache line ping-ponging between CPUs. */ do { if (panicstr != NULL) break; while (MUTEX_SPINBIT_LOCKED_P(mtx)) { SPINLOCK_BACKOFF(count); #ifdef LOCKDEBUG if (SPINLOCK_SPINOUT(spins)) MUTEX_ABORT(mtx, "spinout"); #endif /* LOCKDEBUG */ } } while (!MUTEX_SPINBIT_LOCK_TRY(mtx)); if (count != SPINLOCK_BACKOFF_MIN) { LOCKSTAT_STOP_TIMER(lsflag, spintime); LOCKSTAT_EVENT(lsflag, mtx, LB_SPIN_MUTEX | LB_SPIN, 1, spintime); } LOCKSTAT_EXIT(lsflag); #endif /* !MULTIPROCESSOR */ #endif /* FULL */ MUTEX_LOCKED(mtx); return; } curthread = (uintptr_t)curlwp; MUTEX_DASSERT(mtx, MUTEX_ADAPTIVE_P(mtx)); MUTEX_ASSERT(mtx, curthread != 0); MUTEX_WANTLOCK(mtx); if (panicstr == NULL) { LOCKDEBUG_BARRIER(&kernel_lock, 1); } LOCKSTAT_ENTER(lsflag); /* * Adaptive mutex; spin trying to acquire the mutex. If we * determine that the owner is not running on a processor, * then we stop spinning, and sleep instead. */ KPREEMPT_DISABLE(curlwp); for (owner = mtx->mtx_owner;;) { if (!MUTEX_OWNED(owner)) { /* * Mutex owner clear could mean two things: * * * The mutex has been released. * * The owner field hasn't been set yet. * * Try to acquire it again. If that fails, * we'll just loop again. */ if (MUTEX_ACQUIRE(mtx, curthread)) break; owner = mtx->mtx_owner; continue; } if (__predict_false(panicstr != NULL)) { KPREEMPT_ENABLE(curlwp); return; } if (__predict_false(MUTEX_OWNER(owner) == curthread)) { MUTEX_ABORT(mtx, "locking against myself"); } #ifdef MULTIPROCESSOR /* * Check to see if the owner is running on a processor. * If so, then we should just spin, as the owner will * likely release the lock very soon. */ if (mutex_oncpu(owner)) { LOCKSTAT_START_TIMER(lsflag, spintime); count = SPINLOCK_BACKOFF_MIN; do { KPREEMPT_ENABLE(curlwp); SPINLOCK_BACKOFF(count); KPREEMPT_DISABLE(curlwp); owner = mtx->mtx_owner; } while (mutex_oncpu(owner)); LOCKSTAT_STOP_TIMER(lsflag, spintime); LOCKSTAT_COUNT(spincnt, 1); if (!MUTEX_OWNED(owner)) continue; } #endif ts = turnstile_lookup(mtx); /* * Once we have the turnstile chain interlock, mark the * mutex has having waiters. If that fails, spin again: * chances are that the mutex has been released. */ if (!MUTEX_SET_WAITERS(mtx, owner)) { turnstile_exit(mtx); owner = mtx->mtx_owner; continue; } #ifdef MULTIPROCESSOR /* * mutex_exit() is permitted to release the mutex without * any interlocking instructions, and the following can * occur as a result: * * CPU 1: MUTEX_SET_WAITERS() CPU2: mutex_exit() * ---------------------------- ---------------------------- * .. acquire cache line * .. test for waiters * acquire cache line <- lose cache line * lock cache line .. * verify mutex is held .. * set waiters .. * unlock cache line .. * lose cache line -> acquire cache line * .. clear lock word, waiters * return success * * There is another race that can occur: a third CPU could * acquire the mutex as soon as it is released. Since * adaptive mutexes are primarily spin mutexes, this is not * something that we need to worry about too much. What we * do need to ensure is that the waiters bit gets set. * * To allow the unlocked release, we need to make some * assumptions here: * * o Release is the only non-atomic/unlocked operation * that can be performed on the mutex. (It must still * be atomic on the local CPU, e.g. in case interrupted * or preempted). * * o At any given time, MUTEX_SET_WAITERS() can only ever * be in progress on one CPU in the system - guaranteed * by the turnstile chain lock. * * o No other operations other than MUTEX_SET_WAITERS() * and release can modify a mutex with a non-zero * owner field. * * o The result of a successful MUTEX_SET_WAITERS() call * is an unbuffered write that is immediately visible * to all other processors in the system. * * o If the holding LWP switches away, it posts a store * fence before changing curlwp, ensuring that any * overwrite of the mutex waiters flag by mutex_exit() * completes before the modification of curlwp becomes * visible to this CPU. * * o mi_switch() posts a store fence before setting curlwp * and before resuming execution of an LWP. * * o _kernel_lock() posts a store fence before setting * curcpu()->ci_biglock_wanted, and after clearing it. * This ensures that any overwrite of the mutex waiters * flag by mutex_exit() completes before the modification * of ci_biglock_wanted becomes visible. * * We now post a read memory barrier (after setting the * waiters field) and check the lock holder's status again. * Some of the possible outcomes (not an exhaustive list): * * 1. The on-CPU check returns true: the holding LWP is * running again. The lock may be released soon and * we should spin. Importantly, we can't trust the * value of the waiters flag. * * 2. The on-CPU check returns false: the holding LWP is * not running. We now have the opportunity to check * if mutex_exit() has blatted the modifications made * by MUTEX_SET_WAITERS(). * * 3. The on-CPU check returns false: the holding LWP may * or may not be running. It has context switched at * some point during our check. Again, we have the * chance to see if the waiters bit is still set or * has been overwritten. * * 4. The on-CPU check returns false: the holding LWP is * running on a CPU, but wants the big lock. It's OK * to check the waiters field in this case. * * 5. The has-waiters check fails: the mutex has been * released, the waiters flag cleared and another LWP * now owns the mutex. * * 6. The has-waiters check fails: the mutex has been * released. * * If the waiters bit is not set it's unsafe to go asleep, * as we might never be awoken. */ if ((membar_consumer(), mutex_oncpu(owner)) || (membar_consumer(), !MUTEX_HAS_WAITERS(mtx))) { turnstile_exit(mtx); owner = mtx->mtx_owner; continue; } #endif /* MULTIPROCESSOR */ LOCKSTAT_START_TIMER(lsflag, slptime); turnstile_block(ts, TS_WRITER_Q, mtx, &mutex_syncobj); LOCKSTAT_STOP_TIMER(lsflag, slptime); LOCKSTAT_COUNT(slpcnt, 1); owner = mtx->mtx_owner; } KPREEMPT_ENABLE(curlwp); LOCKSTAT_EVENT(lsflag, mtx, LB_ADAPTIVE_MUTEX | LB_SLEEP1, slpcnt, slptime); LOCKSTAT_EVENT(lsflag, mtx, LB_ADAPTIVE_MUTEX | LB_SPIN, spincnt, spintime); LOCKSTAT_EXIT(lsflag); MUTEX_DASSERT(mtx, MUTEX_OWNER(mtx->mtx_owner) == curthread); MUTEX_LOCKED(mtx); }
/* * rw_vector_enter: * * Acquire a rwlock. */ void rw_vector_enter(krwlock_t *rw, const krw_t op) { uintptr_t owner, incr, need_wait, set_wait, curthread, next; turnstile_t *ts; int queue; lwp_t *l; LOCKSTAT_TIMER(slptime); LOCKSTAT_TIMER(slpcnt); LOCKSTAT_TIMER(spintime); LOCKSTAT_COUNTER(spincnt); LOCKSTAT_FLAG(lsflag); l = curlwp; curthread = (uintptr_t)l; RW_ASSERT(rw, !cpu_intr_p()); RW_ASSERT(rw, curthread != 0); RW_WANTLOCK(rw, op); if (panicstr == NULL) { LOCKDEBUG_BARRIER(&kernel_lock, 1); } /* * We play a slight trick here. If we're a reader, we want * increment the read count. If we're a writer, we want to * set the owner field and whe WRITE_LOCKED bit. * * In the latter case, we expect those bits to be zero, * therefore we can use an add operation to set them, which * means an add operation for both cases. */ if (__predict_true(op == RW_READER)) { incr = RW_READ_INCR; set_wait = RW_HAS_WAITERS; need_wait = RW_WRITE_LOCKED | RW_WRITE_WANTED; queue = TS_READER_Q; } else { RW_DASSERT(rw, op == RW_WRITER); incr = curthread | RW_WRITE_LOCKED; set_wait = RW_HAS_WAITERS | RW_WRITE_WANTED; need_wait = RW_WRITE_LOCKED | RW_THREAD; queue = TS_WRITER_Q; } LOCKSTAT_ENTER(lsflag); KPREEMPT_DISABLE(curlwp); for (owner = rw->rw_owner; ;) { /* * Read the lock owner field. If the need-to-wait * indicator is clear, then try to acquire the lock. */ if ((owner & need_wait) == 0) { next = rw_cas(rw, owner, (owner + incr) & ~RW_WRITE_WANTED); if (__predict_true(next == owner)) { /* Got it! */ membar_enter(); break; } /* * Didn't get it -- spin around again (we'll * probably sleep on the next iteration). */ owner = next; continue; } if (__predict_false(panicstr != NULL)) { kpreempt_enable(); return; } if (__predict_false(RW_OWNER(rw) == curthread)) { rw_abort(rw, __func__, "locking against myself"); } /* * If the lock owner is running on another CPU, and * there are no existing waiters, then spin. */ if (rw_oncpu(owner)) { LOCKSTAT_START_TIMER(lsflag, spintime); u_int count = SPINLOCK_BACKOFF_MIN; do { KPREEMPT_ENABLE(curlwp); SPINLOCK_BACKOFF(count); KPREEMPT_DISABLE(curlwp); owner = rw->rw_owner; } while (rw_oncpu(owner)); LOCKSTAT_STOP_TIMER(lsflag, spintime); LOCKSTAT_COUNT(spincnt, 1); if ((owner & need_wait) == 0) continue; } /* * Grab the turnstile chain lock. Once we have that, we * can adjust the waiter bits and sleep queue. */ ts = turnstile_lookup(rw); /* * Mark the rwlock as having waiters. If the set fails, * then we may not need to sleep and should spin again. * Reload rw_owner because turnstile_lookup() may have * spun on the turnstile chain lock. */ owner = rw->rw_owner; if ((owner & need_wait) == 0 || rw_oncpu(owner)) { turnstile_exit(rw); continue; } next = rw_cas(rw, owner, owner | set_wait); if (__predict_false(next != owner)) { turnstile_exit(rw); owner = next; continue; } LOCKSTAT_START_TIMER(lsflag, slptime); turnstile_block(ts, queue, rw, &rw_syncobj); LOCKSTAT_STOP_TIMER(lsflag, slptime); LOCKSTAT_COUNT(slpcnt, 1); /* * No need for a memory barrier because of context switch. * If not handed the lock, then spin again. */ if (op == RW_READER || (rw->rw_owner & RW_THREAD) == curthread) break; owner = rw->rw_owner; } KPREEMPT_ENABLE(curlwp); LOCKSTAT_EVENT(lsflag, rw, LB_RWLOCK | (op == RW_WRITER ? LB_SLEEP1 : LB_SLEEP2), slpcnt, slptime); LOCKSTAT_EVENT(lsflag, rw, LB_RWLOCK | LB_SPIN, spincnt, spintime); LOCKSTAT_EXIT(lsflag); RW_DASSERT(rw, (op != RW_READER && RW_OWNER(rw) == curthread) || (op == RW_READER && RW_COUNT(rw) != 0)); RW_LOCKED(rw, op); }
/* * Implement device not available (DNA) exception * * If we were the last lwp to use the FPU, we can simply return. * Otherwise, we save the previous state, if necessary, and restore * our last saved state. */ void fpudna(struct cpu_info *ci) { uint16_t cw; uint32_t mxcsr; struct lwp *l, *fl; struct pcb *pcb; int s; if (ci->ci_fpsaving) { /* Recursive trap. */ x86_enable_intr(); return; } /* Lock out IPIs and disable preemption. */ s = splhigh(); x86_enable_intr(); /* Save state on current CPU. */ l = ci->ci_curlwp; pcb = lwp_getpcb(l); fl = ci->ci_fpcurlwp; if (fl != NULL) { /* * It seems we can get here on Xen even if we didn't * switch lwp. In this case do nothing */ if (fl == l) { KASSERT(pcb->pcb_fpcpu == ci); clts(); splx(s); return; } KASSERT(fl != l); fpusave_cpu(true); KASSERT(ci->ci_fpcurlwp == NULL); } /* Save our state if on a remote CPU. */ if (pcb->pcb_fpcpu != NULL) { /* Explicitly disable preemption before dropping spl. */ KPREEMPT_DISABLE(l); splx(s); fpusave_lwp(l, true); KASSERT(pcb->pcb_fpcpu == NULL); s = splhigh(); KPREEMPT_ENABLE(l); } /* * Restore state on this CPU, or initialize. Ensure that * the entire update is atomic with respect to FPU-sync IPIs. */ clts(); ci->ci_fpcurlwp = l; pcb->pcb_fpcpu = ci; if ((l->l_md.md_flags & MDL_USEDFPU) == 0) { fninit(); cw = pcb->pcb_savefpu.fp_fxsave.fx_fcw; fldcw(&cw); mxcsr = pcb->pcb_savefpu.fp_fxsave.fx_mxcsr; x86_ldmxcsr(&mxcsr); l->l_md.md_flags |= MDL_USEDFPU; } else { /* * AMD FPU's do not restore FIP, FDP, and FOP on fxrstor, * leaking other process's execution history. Clear them * manually. */ static const double zero = 0.0; int status; /* * Clear the ES bit in the x87 status word if it is currently * set, in order to avoid causing a fault in the upcoming load. */ fnstsw(&status); if (status & 0x80) fnclex(); /* * Load the dummy variable into the x87 stack. This mangles * the x87 stack, but we don't care since we're about to call * fxrstor() anyway. */ fldummy(&zero); fxrstor(&pcb->pcb_savefpu); } KASSERT(ci == curcpu()); splx(s); }
void percpu_putref(percpu_t *pc) { KPREEMPT_ENABLE(curlwp); }
void kpreempt_enable(void) { KPREEMPT_ENABLE(curlwp); }