/*
* mutex_spin_retry:
*
* Support routine for mutex_spin_enter(). Assumes that the caller
* has already raised the SPL, and adjusted counters.
*/
void
mutex_spin_retry(kmutex_t *mtx)
{
#ifdef MULTIPROCESSOR
u_int count;
LOCKSTAT_TIMER(spintime);
LOCKSTAT_FLAG(lsflag);
#ifdef LOCKDEBUG
u_int spins = 0;
#endif /* LOCKDEBUG */
MUTEX_WANTLOCK(mtx);
LOCKSTAT_ENTER(lsflag);
LOCKSTAT_START_TIMER(lsflag, spintime);
count = SPINLOCK_BACKOFF_MIN;
/*
* Spin testing the lock word and do exponential backoff
* to reduce cache line ping-ponging between CPUs.
*/
do {
if (panicstr != NULL)
break;
while (MUTEX_SPINBIT_LOCKED_P(mtx)) {
SPINLOCK_BACKOFF(count);
#ifdef LOCKDEBUG
if (SPINLOCK_SPINOUT(spins))
MUTEX_ABORT(mtx, "spinout");
#endif /* LOCKDEBUG */
}
} while (!MUTEX_SPINBIT_LOCK_TRY(mtx));
LOCKSTAT_STOP_TIMER(lsflag, spintime);
LOCKSTAT_EVENT(lsflag, mtx, LB_SPIN_MUTEX | LB_SPIN, 1, spintime);
LOCKSTAT_EXIT(lsflag);
MUTEX_LOCKED(mtx);
#else /* MULTIPROCESSOR */
MUTEX_ABORT(mtx, "locking against myself");
#endif /* MULTIPROCESSOR */
}
/*
* mutex_vector_enter:
*
* Support routine for mutex_enter() that must handle all cases. In
* the LOCKDEBUG case, mutex_enter() is always aliased here, even if
* fast-path stubs are available. If a mutex_spin_enter() stub is
* not available, then it is also aliased directly here.
*/
void
mutex_vector_enter(kmutex_t *mtx)
{
uintptr_t owner, curthread;
turnstile_t *ts;
#ifdef MULTIPROCESSOR
u_int count;
#endif
LOCKSTAT_COUNTER(spincnt);
LOCKSTAT_COUNTER(slpcnt);
LOCKSTAT_TIMER(spintime);
LOCKSTAT_TIMER(slptime);
LOCKSTAT_FLAG(lsflag);
/*
* Handle spin mutexes.
*/
if (MUTEX_SPIN_P(mtx)) {
#if defined(LOCKDEBUG) && defined(MULTIPROCESSOR)
u_int spins = 0;
#endif
MUTEX_SPIN_SPLRAISE(mtx);
MUTEX_WANTLOCK(mtx);
#ifdef FULL
if (MUTEX_SPINBIT_LOCK_TRY(mtx)) {
MUTEX_LOCKED(mtx);
return;
}
#if !defined(MULTIPROCESSOR)
MUTEX_ABORT(mtx, "locking against myself");
#else /* !MULTIPROCESSOR */
LOCKSTAT_ENTER(lsflag);
LOCKSTAT_START_TIMER(lsflag, spintime);
count = SPINLOCK_BACKOFF_MIN;
/*
* Spin testing the lock word and do exponential backoff
* to reduce cache line ping-ponging between CPUs.
*/
do {
if (panicstr != NULL)
break;
while (MUTEX_SPINBIT_LOCKED_P(mtx)) {
SPINLOCK_BACKOFF(count);
#ifdef LOCKDEBUG
if (SPINLOCK_SPINOUT(spins))
MUTEX_ABORT(mtx, "spinout");
#endif /* LOCKDEBUG */
}
} while (!MUTEX_SPINBIT_LOCK_TRY(mtx));
if (count != SPINLOCK_BACKOFF_MIN) {
LOCKSTAT_STOP_TIMER(lsflag, spintime);
LOCKSTAT_EVENT(lsflag, mtx,
LB_SPIN_MUTEX | LB_SPIN, 1, spintime);
}
LOCKSTAT_EXIT(lsflag);
#endif /* !MULTIPROCESSOR */
#endif /* FULL */
MUTEX_LOCKED(mtx);
return;
}
curthread = (uintptr_t)curlwp;
MUTEX_DASSERT(mtx, MUTEX_ADAPTIVE_P(mtx));
MUTEX_ASSERT(mtx, curthread != 0);
MUTEX_WANTLOCK(mtx);
if (panicstr == NULL) {
LOCKDEBUG_BARRIER(&kernel_lock, 1);
}
LOCKSTAT_ENTER(lsflag);
/*
* Adaptive mutex; spin trying to acquire the mutex. If we
* determine that the owner is not running on a processor,
* then we stop spinning, and sleep instead.
*/
KPREEMPT_DISABLE(curlwp);
for (owner = mtx->mtx_owner;;) {
if (!MUTEX_OWNED(owner)) {
/*
* Mutex owner clear could mean two things:
*
* * The mutex has been released.
* * The owner field hasn't been set yet.
*
* Try to acquire it again. If that fails,
* we'll just loop again.
*/
if (MUTEX_ACQUIRE(mtx, curthread))
break;
owner = mtx->mtx_owner;
continue;
}
if (__predict_false(panicstr != NULL)) {
KPREEMPT_ENABLE(curlwp);
return;
}
if (__predict_false(MUTEX_OWNER(owner) == curthread)) {
MUTEX_ABORT(mtx, "locking against myself");
}
#ifdef MULTIPROCESSOR
/*
* Check to see if the owner is running on a processor.
* If so, then we should just spin, as the owner will
* likely release the lock very soon.
*/
if (mutex_oncpu(owner)) {
LOCKSTAT_START_TIMER(lsflag, spintime);
count = SPINLOCK_BACKOFF_MIN;
do {
KPREEMPT_ENABLE(curlwp);
SPINLOCK_BACKOFF(count);
KPREEMPT_DISABLE(curlwp);
owner = mtx->mtx_owner;
} while (mutex_oncpu(owner));
LOCKSTAT_STOP_TIMER(lsflag, spintime);
LOCKSTAT_COUNT(spincnt, 1);
if (!MUTEX_OWNED(owner))
continue;
}
#endif
ts = turnstile_lookup(mtx);
/*
* Once we have the turnstile chain interlock, mark the
* mutex has having waiters. If that fails, spin again:
* chances are that the mutex has been released.
*/
if (!MUTEX_SET_WAITERS(mtx, owner)) {
turnstile_exit(mtx);
owner = mtx->mtx_owner;
continue;
}
#ifdef MULTIPROCESSOR
/*
* mutex_exit() is permitted to release the mutex without
* any interlocking instructions, and the following can
* occur as a result:
*
* CPU 1: MUTEX_SET_WAITERS() CPU2: mutex_exit()
* ---------------------------- ----------------------------
* .. acquire cache line
* .. test for waiters
* acquire cache line <- lose cache line
* lock cache line ..
* verify mutex is held ..
* set waiters ..
* unlock cache line ..
* lose cache line -> acquire cache line
* .. clear lock word, waiters
* return success
*
* There is another race that can occur: a third CPU could
* acquire the mutex as soon as it is released. Since
* adaptive mutexes are primarily spin mutexes, this is not
* something that we need to worry about too much. What we
* do need to ensure is that the waiters bit gets set.
*
* To allow the unlocked release, we need to make some
* assumptions here:
*
* o Release is the only non-atomic/unlocked operation
* that can be performed on the mutex. (It must still
* be atomic on the local CPU, e.g. in case interrupted
* or preempted).
*
* o At any given time, MUTEX_SET_WAITERS() can only ever
* be in progress on one CPU in the system - guaranteed
* by the turnstile chain lock.
*
* o No other operations other than MUTEX_SET_WAITERS()
* and release can modify a mutex with a non-zero
* owner field.
*
* o The result of a successful MUTEX_SET_WAITERS() call
* is an unbuffered write that is immediately visible
* to all other processors in the system.
*
* o If the holding LWP switches away, it posts a store
* fence before changing curlwp, ensuring that any
* overwrite of the mutex waiters flag by mutex_exit()
* completes before the modification of curlwp becomes
* visible to this CPU.
*
* o mi_switch() posts a store fence before setting curlwp
* and before resuming execution of an LWP.
*
* o _kernel_lock() posts a store fence before setting
* curcpu()->ci_biglock_wanted, and after clearing it.
* This ensures that any overwrite of the mutex waiters
* flag by mutex_exit() completes before the modification
* of ci_biglock_wanted becomes visible.
*
* We now post a read memory barrier (after setting the
* waiters field) and check the lock holder's status again.
* Some of the possible outcomes (not an exhaustive list):
*
* 1. The on-CPU check returns true: the holding LWP is
* running again. The lock may be released soon and
* we should spin. Importantly, we can't trust the
* value of the waiters flag.
*
* 2. The on-CPU check returns false: the holding LWP is
* not running. We now have the opportunity to check
* if mutex_exit() has blatted the modifications made
* by MUTEX_SET_WAITERS().
*
* 3. The on-CPU check returns false: the holding LWP may
* or may not be running. It has context switched at
* some point during our check. Again, we have the
* chance to see if the waiters bit is still set or
* has been overwritten.
*
* 4. The on-CPU check returns false: the holding LWP is
* running on a CPU, but wants the big lock. It's OK
* to check the waiters field in this case.
*
* 5. The has-waiters check fails: the mutex has been
* released, the waiters flag cleared and another LWP
* now owns the mutex.
*
* 6. The has-waiters check fails: the mutex has been
* released.
*
* If the waiters bit is not set it's unsafe to go asleep,
* as we might never be awoken.
*/
if ((membar_consumer(), mutex_oncpu(owner)) ||
(membar_consumer(), !MUTEX_HAS_WAITERS(mtx))) {
turnstile_exit(mtx);
owner = mtx->mtx_owner;
continue;
}
#endif /* MULTIPROCESSOR */
LOCKSTAT_START_TIMER(lsflag, slptime);
turnstile_block(ts, TS_WRITER_Q, mtx, &mutex_syncobj);
LOCKSTAT_STOP_TIMER(lsflag, slptime);
LOCKSTAT_COUNT(slpcnt, 1);
owner = mtx->mtx_owner;
}
KPREEMPT_ENABLE(curlwp);
LOCKSTAT_EVENT(lsflag, mtx, LB_ADAPTIVE_MUTEX | LB_SLEEP1,
slpcnt, slptime);
LOCKSTAT_EVENT(lsflag, mtx, LB_ADAPTIVE_MUTEX | LB_SPIN,
spincnt, spintime);
LOCKSTAT_EXIT(lsflag);
MUTEX_DASSERT(mtx, MUTEX_OWNER(mtx->mtx_owner) == curthread);
MUTEX_LOCKED(mtx);
}
/*
* rw_vector_enter:
*
* Acquire a rwlock.
*/
void
rw_vector_enter(krwlock_t *rw, const krw_t op)
{
uintptr_t owner, incr, need_wait, set_wait, curthread, next;
turnstile_t *ts;
int queue;
lwp_t *l;
LOCKSTAT_TIMER(slptime);
LOCKSTAT_TIMER(slpcnt);
LOCKSTAT_TIMER(spintime);
LOCKSTAT_COUNTER(spincnt);
LOCKSTAT_FLAG(lsflag);
l = curlwp;
curthread = (uintptr_t)l;
RW_ASSERT(rw, !cpu_intr_p());
RW_ASSERT(rw, curthread != 0);
RW_WANTLOCK(rw, op);
if (panicstr == NULL) {
LOCKDEBUG_BARRIER(&kernel_lock, 1);
}
/*
* We play a slight trick here. If we're a reader, we want
* increment the read count. If we're a writer, we want to
* set the owner field and whe WRITE_LOCKED bit.
*
* In the latter case, we expect those bits to be zero,
* therefore we can use an add operation to set them, which
* means an add operation for both cases.
*/
if (__predict_true(op == RW_READER)) {
incr = RW_READ_INCR;
set_wait = RW_HAS_WAITERS;
need_wait = RW_WRITE_LOCKED | RW_WRITE_WANTED;
queue = TS_READER_Q;
} else {
RW_DASSERT(rw, op == RW_WRITER);
incr = curthread | RW_WRITE_LOCKED;
set_wait = RW_HAS_WAITERS | RW_WRITE_WANTED;
need_wait = RW_WRITE_LOCKED | RW_THREAD;
queue = TS_WRITER_Q;
}
LOCKSTAT_ENTER(lsflag);
KPREEMPT_DISABLE(curlwp);
for (owner = rw->rw_owner; ;) {
/*
* Read the lock owner field. If the need-to-wait
* indicator is clear, then try to acquire the lock.
*/
if ((owner & need_wait) == 0) {
next = rw_cas(rw, owner, (owner + incr) &
~RW_WRITE_WANTED);
if (__predict_true(next == owner)) {
/* Got it! */
membar_enter();
break;
}
/*
* Didn't get it -- spin around again (we'll
* probably sleep on the next iteration).
*/
owner = next;
continue;
}
if (__predict_false(panicstr != NULL)) {
kpreempt_enable();
return;
}
if (__predict_false(RW_OWNER(rw) == curthread)) {
rw_abort(rw, __func__, "locking against myself");
}
/*
* If the lock owner is running on another CPU, and
* there are no existing waiters, then spin.
*/
if (rw_oncpu(owner)) {
LOCKSTAT_START_TIMER(lsflag, spintime);
u_int count = SPINLOCK_BACKOFF_MIN;
do {
KPREEMPT_ENABLE(curlwp);
SPINLOCK_BACKOFF(count);
KPREEMPT_DISABLE(curlwp);
owner = rw->rw_owner;
} while (rw_oncpu(owner));
LOCKSTAT_STOP_TIMER(lsflag, spintime);
LOCKSTAT_COUNT(spincnt, 1);
if ((owner & need_wait) == 0)
continue;
}
/*
* Grab the turnstile chain lock. Once we have that, we
* can adjust the waiter bits and sleep queue.
*/
ts = turnstile_lookup(rw);
/*
* Mark the rwlock as having waiters. If the set fails,
* then we may not need to sleep and should spin again.
* Reload rw_owner because turnstile_lookup() may have
* spun on the turnstile chain lock.
*/
owner = rw->rw_owner;
if ((owner & need_wait) == 0 || rw_oncpu(owner)) {
turnstile_exit(rw);
continue;
}
next = rw_cas(rw, owner, owner | set_wait);
if (__predict_false(next != owner)) {
turnstile_exit(rw);
owner = next;
continue;
}
LOCKSTAT_START_TIMER(lsflag, slptime);
turnstile_block(ts, queue, rw, &rw_syncobj);
LOCKSTAT_STOP_TIMER(lsflag, slptime);
LOCKSTAT_COUNT(slpcnt, 1);
/*
* No need for a memory barrier because of context switch.
* If not handed the lock, then spin again.
*/
if (op == RW_READER || (rw->rw_owner & RW_THREAD) == curthread)
break;
owner = rw->rw_owner;
}
KPREEMPT_ENABLE(curlwp);
LOCKSTAT_EVENT(lsflag, rw, LB_RWLOCK |
(op == RW_WRITER ? LB_SLEEP1 : LB_SLEEP2), slpcnt, slptime);
LOCKSTAT_EVENT(lsflag, rw, LB_RWLOCK | LB_SPIN, spincnt, spintime);
LOCKSTAT_EXIT(lsflag);
RW_DASSERT(rw, (op != RW_READER && RW_OWNER(rw) == curthread) ||
(op == RW_READER && RW_COUNT(rw) != 0));
RW_LOCKED(rw, op);
}