/*
* uvm_emap_consume: update the current CPU and LWP to the given generation
* of the emap. In a case of LWP migration to a different CPU after block
* or preempt, uvm_emap_switch() will synchronise.
*
* => may be called from both interrupt and thread context.
*/
void
uvm_emap_consume(u_int gen)
{
struct cpu_info *ci;
struct uvm_cpu *ucpu;
lwp_t *l = curlwp;
u_int curgen;
if (gen == UVM_EMAP_INACTIVE) {
return;
}
/*
* No need to synchronise if generation number of current CPU is
* newer than the number of this LWP.
*
* This test assumes two's complement arithmetic and allows
* ~2B missed updates before it will produce bad results.
*/
KPREEMPT_DISABLE(l);
ci = l->l_cpu;
ucpu = ci->ci_data.cpu_uvm;
if (__predict_true((signed int)(ucpu->emap_gen - gen) >= 0)) {
l->l_emap_gen = ucpu->emap_gen;
KPREEMPT_ENABLE(l);
return;
}
/*
* Record the current generation _before_ issuing the TLB flush.
* No need for a memory barrier before, as reading a stale value
* for uvm_emap_gen is not a problem.
*
* pmap_emap_sync() must implicitly perform a full memory barrier,
* which prevents us from fetching a value from after the TLB flush
* has occurred (which would be bad).
*
* We can race with an interrupt on the current CPU updating the
* counter to a newer value. This could cause us to set a stale
* value into ucpu->emap_gen, overwriting a newer update from the
* interrupt. However, it does not matter since:
* (1) Interrupts always run to completion or block.
* (2) Interrupts will only ever install a newer value and,
* (3) We will roll the value forward later.
*/
curgen = uvm_emap_gen_return();
pmap_emap_sync(true);
ucpu->emap_gen = curgen;
l->l_emap_gen = curgen;
KASSERT((signed int)(curgen - gen) >= 0);
KPREEMPT_ENABLE(l);
}
void
fputrap(struct trapframe *frame)
{
register struct lwp *l = curcpu()->ci_fpcurlwp;
struct pcb *pcb = lwp_getpcb(l);
struct savefpu *sfp = &pcb->pcb_savefpu;
uint32_t mxcsr, statbits;
uint16_t cw;
ksiginfo_t ksi;
KPREEMPT_DISABLE(l);
x86_enable_intr();
/*
* At this point, fpcurlwp should be curlwp. If it wasn't, the TS bit
* should be set, and we should have gotten a DNA exception.
*/
KASSERT(l == curlwp);
fxsave(sfp);
pcb->pcb_savefpu_i387.fp_ex_tw = sfp->fp_fxsave.fx_ftw;
pcb->pcb_savefpu_i387.fp_ex_sw = sfp->fp_fxsave.fx_fsw;
if (frame->tf_trapno == T_XMM) {
mxcsr = sfp->fp_fxsave.fx_mxcsr;
statbits = mxcsr;
mxcsr &= ~0x3f;
x86_ldmxcsr(&mxcsr);
} else {
fninit();
fwait();
cw = sfp->fp_fxsave.fx_fcw;
fldcw(&cw);
fwait();
statbits = sfp->fp_fxsave.fx_fsw;
}
KPREEMPT_ENABLE(l);
KSI_INIT_TRAP(&ksi);
ksi.ksi_signo = SIGFPE;
ksi.ksi_addr = (void *)frame->tf_rip;
ksi.ksi_code = x86fpflags_to_ksiginfo(statbits);
ksi.ksi_trap = statbits;
(*l->l_proc->p_emul->e_trapsignal)(l, &ksi);
}
/*
* mutex_vector_enter:
*
* Support routine for mutex_enter() that must handle all cases. In
* the LOCKDEBUG case, mutex_enter() is always aliased here, even if
* fast-path stubs are available. If a mutex_spin_enter() stub is
* not available, then it is also aliased directly here.
*/
void
mutex_vector_enter(kmutex_t *mtx)
{
uintptr_t owner, curthread;
turnstile_t *ts;
#ifdef MULTIPROCESSOR
u_int count;
#endif
LOCKSTAT_COUNTER(spincnt);
LOCKSTAT_COUNTER(slpcnt);
LOCKSTAT_TIMER(spintime);
LOCKSTAT_TIMER(slptime);
LOCKSTAT_FLAG(lsflag);
/*
* Handle spin mutexes.
*/
if (MUTEX_SPIN_P(mtx)) {
#if defined(LOCKDEBUG) && defined(MULTIPROCESSOR)
u_int spins = 0;
#endif
MUTEX_SPIN_SPLRAISE(mtx);
MUTEX_WANTLOCK(mtx);
#ifdef FULL
if (MUTEX_SPINBIT_LOCK_TRY(mtx)) {
MUTEX_LOCKED(mtx);
return;
}
#if !defined(MULTIPROCESSOR)
MUTEX_ABORT(mtx, "locking against myself");
#else /* !MULTIPROCESSOR */
LOCKSTAT_ENTER(lsflag);
LOCKSTAT_START_TIMER(lsflag, spintime);
count = SPINLOCK_BACKOFF_MIN;
/*
* Spin testing the lock word and do exponential backoff
* to reduce cache line ping-ponging between CPUs.
*/
do {
if (panicstr != NULL)
break;
while (MUTEX_SPINBIT_LOCKED_P(mtx)) {
SPINLOCK_BACKOFF(count);
#ifdef LOCKDEBUG
if (SPINLOCK_SPINOUT(spins))
MUTEX_ABORT(mtx, "spinout");
#endif /* LOCKDEBUG */
}
} while (!MUTEX_SPINBIT_LOCK_TRY(mtx));
if (count != SPINLOCK_BACKOFF_MIN) {
LOCKSTAT_STOP_TIMER(lsflag, spintime);
LOCKSTAT_EVENT(lsflag, mtx,
LB_SPIN_MUTEX | LB_SPIN, 1, spintime);
}
LOCKSTAT_EXIT(lsflag);
#endif /* !MULTIPROCESSOR */
#endif /* FULL */
MUTEX_LOCKED(mtx);
return;
}
curthread = (uintptr_t)curlwp;
MUTEX_DASSERT(mtx, MUTEX_ADAPTIVE_P(mtx));
MUTEX_ASSERT(mtx, curthread != 0);
MUTEX_WANTLOCK(mtx);
if (panicstr == NULL) {
LOCKDEBUG_BARRIER(&kernel_lock, 1);
}
LOCKSTAT_ENTER(lsflag);
/*
* Adaptive mutex; spin trying to acquire the mutex. If we
* determine that the owner is not running on a processor,
* then we stop spinning, and sleep instead.
*/
KPREEMPT_DISABLE(curlwp);
for (owner = mtx->mtx_owner;;) {
if (!MUTEX_OWNED(owner)) {
/*
* Mutex owner clear could mean two things:
*
* * The mutex has been released.
* * The owner field hasn't been set yet.
*
* Try to acquire it again. If that fails,
* we'll just loop again.
*/
if (MUTEX_ACQUIRE(mtx, curthread))
break;
owner = mtx->mtx_owner;
continue;
}
if (__predict_false(panicstr != NULL)) {
KPREEMPT_ENABLE(curlwp);
return;
}
if (__predict_false(MUTEX_OWNER(owner) == curthread)) {
MUTEX_ABORT(mtx, "locking against myself");
}
#ifdef MULTIPROCESSOR
/*
* Check to see if the owner is running on a processor.
* If so, then we should just spin, as the owner will
* likely release the lock very soon.
*/
if (mutex_oncpu(owner)) {
LOCKSTAT_START_TIMER(lsflag, spintime);
count = SPINLOCK_BACKOFF_MIN;
do {
KPREEMPT_ENABLE(curlwp);
SPINLOCK_BACKOFF(count);
KPREEMPT_DISABLE(curlwp);
owner = mtx->mtx_owner;
} while (mutex_oncpu(owner));
LOCKSTAT_STOP_TIMER(lsflag, spintime);
LOCKSTAT_COUNT(spincnt, 1);
if (!MUTEX_OWNED(owner))
continue;
}
#endif
ts = turnstile_lookup(mtx);
/*
* Once we have the turnstile chain interlock, mark the
* mutex has having waiters. If that fails, spin again:
* chances are that the mutex has been released.
*/
if (!MUTEX_SET_WAITERS(mtx, owner)) {
turnstile_exit(mtx);
owner = mtx->mtx_owner;
continue;
}
#ifdef MULTIPROCESSOR
/*
* mutex_exit() is permitted to release the mutex without
* any interlocking instructions, and the following can
* occur as a result:
*
* CPU 1: MUTEX_SET_WAITERS() CPU2: mutex_exit()
* ---------------------------- ----------------------------
* .. acquire cache line
* .. test for waiters
* acquire cache line <- lose cache line
* lock cache line ..
* verify mutex is held ..
* set waiters ..
* unlock cache line ..
* lose cache line -> acquire cache line
* .. clear lock word, waiters
* return success
*
* There is another race that can occur: a third CPU could
* acquire the mutex as soon as it is released. Since
* adaptive mutexes are primarily spin mutexes, this is not
* something that we need to worry about too much. What we
* do need to ensure is that the waiters bit gets set.
*
* To allow the unlocked release, we need to make some
* assumptions here:
*
* o Release is the only non-atomic/unlocked operation
* that can be performed on the mutex. (It must still
* be atomic on the local CPU, e.g. in case interrupted
* or preempted).
*
* o At any given time, MUTEX_SET_WAITERS() can only ever
* be in progress on one CPU in the system - guaranteed
* by the turnstile chain lock.
*
* o No other operations other than MUTEX_SET_WAITERS()
* and release can modify a mutex with a non-zero
* owner field.
*
* o The result of a successful MUTEX_SET_WAITERS() call
* is an unbuffered write that is immediately visible
* to all other processors in the system.
*
* o If the holding LWP switches away, it posts a store
* fence before changing curlwp, ensuring that any
* overwrite of the mutex waiters flag by mutex_exit()
* completes before the modification of curlwp becomes
* visible to this CPU.
*
* o mi_switch() posts a store fence before setting curlwp
* and before resuming execution of an LWP.
*
* o _kernel_lock() posts a store fence before setting
* curcpu()->ci_biglock_wanted, and after clearing it.
* This ensures that any overwrite of the mutex waiters
* flag by mutex_exit() completes before the modification
* of ci_biglock_wanted becomes visible.
*
* We now post a read memory barrier (after setting the
* waiters field) and check the lock holder's status again.
* Some of the possible outcomes (not an exhaustive list):
*
* 1. The on-CPU check returns true: the holding LWP is
* running again. The lock may be released soon and
* we should spin. Importantly, we can't trust the
* value of the waiters flag.
*
* 2. The on-CPU check returns false: the holding LWP is
* not running. We now have the opportunity to check
* if mutex_exit() has blatted the modifications made
* by MUTEX_SET_WAITERS().
*
* 3. The on-CPU check returns false: the holding LWP may
* or may not be running. It has context switched at
* some point during our check. Again, we have the
* chance to see if the waiters bit is still set or
* has been overwritten.
*
* 4. The on-CPU check returns false: the holding LWP is
* running on a CPU, but wants the big lock. It's OK
* to check the waiters field in this case.
*
* 5. The has-waiters check fails: the mutex has been
* released, the waiters flag cleared and another LWP
* now owns the mutex.
*
* 6. The has-waiters check fails: the mutex has been
* released.
*
* If the waiters bit is not set it's unsafe to go asleep,
* as we might never be awoken.
*/
if ((membar_consumer(), mutex_oncpu(owner)) ||
(membar_consumer(), !MUTEX_HAS_WAITERS(mtx))) {
turnstile_exit(mtx);
owner = mtx->mtx_owner;
continue;
}
#endif /* MULTIPROCESSOR */
LOCKSTAT_START_TIMER(lsflag, slptime);
turnstile_block(ts, TS_WRITER_Q, mtx, &mutex_syncobj);
LOCKSTAT_STOP_TIMER(lsflag, slptime);
LOCKSTAT_COUNT(slpcnt, 1);
owner = mtx->mtx_owner;
}
KPREEMPT_ENABLE(curlwp);
LOCKSTAT_EVENT(lsflag, mtx, LB_ADAPTIVE_MUTEX | LB_SLEEP1,
slpcnt, slptime);
LOCKSTAT_EVENT(lsflag, mtx, LB_ADAPTIVE_MUTEX | LB_SPIN,
spincnt, spintime);
LOCKSTAT_EXIT(lsflag);
MUTEX_DASSERT(mtx, MUTEX_OWNER(mtx->mtx_owner) == curthread);
MUTEX_LOCKED(mtx);
}
/*
* rw_vector_enter:
*
* Acquire a rwlock.
*/
void
rw_vector_enter(krwlock_t *rw, const krw_t op)
{
uintptr_t owner, incr, need_wait, set_wait, curthread, next;
turnstile_t *ts;
int queue;
lwp_t *l;
LOCKSTAT_TIMER(slptime);
LOCKSTAT_TIMER(slpcnt);
LOCKSTAT_TIMER(spintime);
LOCKSTAT_COUNTER(spincnt);
LOCKSTAT_FLAG(lsflag);
l = curlwp;
curthread = (uintptr_t)l;
RW_ASSERT(rw, !cpu_intr_p());
RW_ASSERT(rw, curthread != 0);
RW_WANTLOCK(rw, op);
if (panicstr == NULL) {
LOCKDEBUG_BARRIER(&kernel_lock, 1);
}
/*
* We play a slight trick here. If we're a reader, we want
* increment the read count. If we're a writer, we want to
* set the owner field and whe WRITE_LOCKED bit.
*
* In the latter case, we expect those bits to be zero,
* therefore we can use an add operation to set them, which
* means an add operation for both cases.
*/
if (__predict_true(op == RW_READER)) {
incr = RW_READ_INCR;
set_wait = RW_HAS_WAITERS;
need_wait = RW_WRITE_LOCKED | RW_WRITE_WANTED;
queue = TS_READER_Q;
} else {
RW_DASSERT(rw, op == RW_WRITER);
incr = curthread | RW_WRITE_LOCKED;
set_wait = RW_HAS_WAITERS | RW_WRITE_WANTED;
need_wait = RW_WRITE_LOCKED | RW_THREAD;
queue = TS_WRITER_Q;
}
LOCKSTAT_ENTER(lsflag);
KPREEMPT_DISABLE(curlwp);
for (owner = rw->rw_owner; ;) {
/*
* Read the lock owner field. If the need-to-wait
* indicator is clear, then try to acquire the lock.
*/
if ((owner & need_wait) == 0) {
next = rw_cas(rw, owner, (owner + incr) &
~RW_WRITE_WANTED);
if (__predict_true(next == owner)) {
/* Got it! */
membar_enter();
break;
}
/*
* Didn't get it -- spin around again (we'll
* probably sleep on the next iteration).
*/
owner = next;
continue;
}
if (__predict_false(panicstr != NULL)) {
kpreempt_enable();
return;
}
if (__predict_false(RW_OWNER(rw) == curthread)) {
rw_abort(rw, __func__, "locking against myself");
}
/*
* If the lock owner is running on another CPU, and
* there are no existing waiters, then spin.
*/
if (rw_oncpu(owner)) {
LOCKSTAT_START_TIMER(lsflag, spintime);
u_int count = SPINLOCK_BACKOFF_MIN;
do {
KPREEMPT_ENABLE(curlwp);
SPINLOCK_BACKOFF(count);
KPREEMPT_DISABLE(curlwp);
owner = rw->rw_owner;
} while (rw_oncpu(owner));
LOCKSTAT_STOP_TIMER(lsflag, spintime);
LOCKSTAT_COUNT(spincnt, 1);
if ((owner & need_wait) == 0)
continue;
}
/*
* Grab the turnstile chain lock. Once we have that, we
* can adjust the waiter bits and sleep queue.
*/
ts = turnstile_lookup(rw);
/*
* Mark the rwlock as having waiters. If the set fails,
* then we may not need to sleep and should spin again.
* Reload rw_owner because turnstile_lookup() may have
* spun on the turnstile chain lock.
*/
owner = rw->rw_owner;
if ((owner & need_wait) == 0 || rw_oncpu(owner)) {
turnstile_exit(rw);
continue;
}
next = rw_cas(rw, owner, owner | set_wait);
if (__predict_false(next != owner)) {
turnstile_exit(rw);
owner = next;
continue;
}
LOCKSTAT_START_TIMER(lsflag, slptime);
turnstile_block(ts, queue, rw, &rw_syncobj);
LOCKSTAT_STOP_TIMER(lsflag, slptime);
LOCKSTAT_COUNT(slpcnt, 1);
/*
* No need for a memory barrier because of context switch.
* If not handed the lock, then spin again.
*/
if (op == RW_READER || (rw->rw_owner & RW_THREAD) == curthread)
break;
owner = rw->rw_owner;
}
KPREEMPT_ENABLE(curlwp);
LOCKSTAT_EVENT(lsflag, rw, LB_RWLOCK |
(op == RW_WRITER ? LB_SLEEP1 : LB_SLEEP2), slpcnt, slptime);
LOCKSTAT_EVENT(lsflag, rw, LB_RWLOCK | LB_SPIN, spincnt, spintime);
LOCKSTAT_EXIT(lsflag);
RW_DASSERT(rw, (op != RW_READER && RW_OWNER(rw) == curthread) ||
(op == RW_READER && RW_COUNT(rw) != 0));
RW_LOCKED(rw, op);
}
/*
* Implement device not available (DNA) exception
*
* If we were the last lwp to use the FPU, we can simply return.
* Otherwise, we save the previous state, if necessary, and restore
* our last saved state.
*/
void
fpudna(struct cpu_info *ci)
{
uint16_t cw;
uint32_t mxcsr;
struct lwp *l, *fl;
struct pcb *pcb;
int s;
if (ci->ci_fpsaving) {
/* Recursive trap. */
x86_enable_intr();
return;
}
/* Lock out IPIs and disable preemption. */
s = splhigh();
x86_enable_intr();
/* Save state on current CPU. */
l = ci->ci_curlwp;
pcb = lwp_getpcb(l);
fl = ci->ci_fpcurlwp;
if (fl != NULL) {
/*
* It seems we can get here on Xen even if we didn't
* switch lwp. In this case do nothing
*/
if (fl == l) {
KASSERT(pcb->pcb_fpcpu == ci);
clts();
splx(s);
return;
}
KASSERT(fl != l);
fpusave_cpu(true);
KASSERT(ci->ci_fpcurlwp == NULL);
}
/* Save our state if on a remote CPU. */
if (pcb->pcb_fpcpu != NULL) {
/* Explicitly disable preemption before dropping spl. */
KPREEMPT_DISABLE(l);
splx(s);
fpusave_lwp(l, true);
KASSERT(pcb->pcb_fpcpu == NULL);
s = splhigh();
KPREEMPT_ENABLE(l);
}
/*
* Restore state on this CPU, or initialize. Ensure that
* the entire update is atomic with respect to FPU-sync IPIs.
*/
clts();
ci->ci_fpcurlwp = l;
pcb->pcb_fpcpu = ci;
if ((l->l_md.md_flags & MDL_USEDFPU) == 0) {
fninit();
cw = pcb->pcb_savefpu.fp_fxsave.fx_fcw;
fldcw(&cw);
mxcsr = pcb->pcb_savefpu.fp_fxsave.fx_mxcsr;
x86_ldmxcsr(&mxcsr);
l->l_md.md_flags |= MDL_USEDFPU;
} else {
/*
* AMD FPU's do not restore FIP, FDP, and FOP on fxrstor,
* leaking other process's execution history. Clear them
* manually.
*/
static const double zero = 0.0;
int status;
/*
* Clear the ES bit in the x87 status word if it is currently
* set, in order to avoid causing a fault in the upcoming load.
*/
fnstsw(&status);
if (status & 0x80)
fnclex();
/*
* Load the dummy variable into the x87 stack. This mangles
* the x87 stack, but we don't care since we're about to call
* fxrstor() anyway.
*/
fldummy(&zero);
fxrstor(&pcb->pcb_savefpu);
}
KASSERT(ci == curcpu());
splx(s);
}
void
percpu_putref(percpu_t *pc)
{
KPREEMPT_ENABLE(curlwp);
}
void
kpreempt_enable(void)
{
KPREEMPT_ENABLE(curlwp);
}
