/*
* When we apply priority inheritance, we must grab the owner's thread lock
* while already holding the waiter's thread lock. If both thread locks are
* turnstile locks, this can lead to deadlock: while we hold L1 and try to
* grab L2, some unrelated thread may be applying priority inheritance to
* some other blocking chain, holding L2 and trying to grab L1. The most
* obvious solution -- do a lock_try() for the owner lock -- isn't quite
* sufficient because it can cause livelock: each thread may hold one lock,
* try to grab the other, fail, bail out, and try again, looping forever.
* To prevent livelock we must define a winner, i.e. define an arbitrary
* lock ordering on the turnstile locks. For simplicity we declare that
* virtual address order defines lock order, i.e. if L1 < L2, then the
* correct lock ordering is L1, L2. Thus the thread that holds L1 and
* wants L2 should spin until L2 is available, but the thread that holds
* L2 and can't get L1 on the first try must drop L2 and return failure.
* Moreover, the losing thread must not reacquire L2 until the winning
* thread has had a chance to grab it; to ensure this, the losing thread
* must grab L1 after dropping L2, thus spinning until the winner is done.
* Complicating matters further, note that the owner's thread lock pointer
* can change (i.e. be pointed at a different lock) while we're trying to
* grab it. If that happens, we must unwind our state and try again.
*
* On success, returns 1 with both locks held.
* On failure, returns 0 with neither lock held.
*/
static int
turnstile_interlock(lock_t *wlp, lock_t *volatile *olpp)
{
ASSERT(LOCK_HELD(wlp));
for (;;) {
volatile lock_t *olp = *olpp;
/*
* If the locks are identical, there's nothing to do.
*/
if (olp == wlp)
return (1);
if (lock_try((lock_t *)olp)) {
/*
* If 'olp' is still the right lock, return success.
* Otherwise, drop 'olp' and try the dance again.
*/
if (olp == *olpp)
return (1);
lock_clear((lock_t *)olp);
} else {
hrtime_t spin_time = 0;
/*
* If we're grabbing the locks out of order, we lose.
* Drop the waiter's lock, and then grab and release
* the owner's lock to ensure that we won't retry
* until the winner is done (as described above).
*/
if (olp >= (lock_t *)turnstile_table && olp < wlp) {
lock_clear(wlp);
lock_set((lock_t *)olp);
lock_clear((lock_t *)olp);
return (0);
}
/*
* We're grabbing the locks in the right order,
* so spin until the owner's lock either becomes
* available or spontaneously changes.
*/
spin_time =
LOCKSTAT_START_TIME(LS_TURNSTILE_INTERLOCK_SPIN);
while (olp == *olpp && LOCK_HELD(olp)) {
if (panicstr)
return (1);
SMT_PAUSE();
}
LOCKSTAT_RECORD_TIME(LS_TURNSTILE_INTERLOCK_SPIN,
olp, spin_time);
}
}
}
void
lock_set_spl_spin(lock_t *lp, int new_pil, ushort_t *old_pil_addr, int old_pil)
{
int spin_count = 1;
int backoff; /* current backoff */
int backctr; /* ctr for backoff */
if (panicstr)
return;
if (ncpus == 1)
panic("lock_set_spl: %p lock held and only one CPU", lp);
ASSERT(new_pil > LOCK_LEVEL);
if (&plat_lock_delay) {
backoff = 0;
} else {
backoff = BACKOFF_BASE;
}
do {
splx(old_pil);
while (LOCK_HELD(lp)) {
if (panicstr) {
*old_pil_addr = (ushort_t)splr(new_pil);
return;
}
spin_count++;
/*
* Add an exponential backoff delay before trying again
* to touch the mutex data structure.
* spin_count test and call to nulldev are to prevent
* compiler optimizer from eliminating the delay loop.
*/
if (&plat_lock_delay) {
plat_lock_delay(&backoff);
} else {
for (backctr = backoff; backctr; backctr--) {
if (!spin_count) (void) nulldev();
}
backoff = backoff << 1; /* double it */
if (backoff > BACKOFF_CAP) {
backoff = BACKOFF_CAP;
}
SMT_PAUSE();
}
}
old_pil = splr(new_pil);
} while (!lock_spin_try(lp));
*old_pil_addr = (ushort_t)old_pil;
if (spin_count) {
LOCKSTAT_RECORD(LS_LOCK_SET_SPL_SPIN, lp, spin_count);
}
LOCKSTAT_RECORD(LS_LOCK_SET_SPL_ACQUIRE, lp, spin_count);
}
int
mutex_owned(kmutex_t *mp)
{
mutex_impl_t *lp = (mutex_impl_t *)mp;
if (panicstr)
return (1);
if (MUTEX_TYPE_ADAPTIVE(lp))
return (MUTEX_OWNER(lp) == curthread);
return (LOCK_HELD(&lp->m_spin.m_spinlock));
}
/*
* Simple C support for the cases where spin locks miss on the first try.
*/
void
lock_set_spin(lock_t *lp)
{
int spin_count = 1;
int backoff; /* current backoff */
int backctr; /* ctr for backoff */
if (panicstr)
return;
if (ncpus == 1)
panic("lock_set: %p lock held and only one CPU", lp);
if (&plat_lock_delay) {
backoff = 0;
} else {
backoff = BACKOFF_BASE;
}
while (LOCK_HELD(lp) || !lock_spin_try(lp)) {
if (panicstr)
return;
spin_count++;
/*
* Add an exponential backoff delay before trying again
* to touch the mutex data structure.
* the spin_count test and call to nulldev are to prevent
* the compiler optimizer from eliminating the delay loop.
*/
if (&plat_lock_delay) {
plat_lock_delay(&backoff);
} else {
/* delay */
for (backctr = backoff; backctr; backctr--) {
if (!spin_count) (void) nulldev();
}
backoff = backoff << 1; /* double it */
if (backoff > BACKOFF_CAP) {
backoff = BACKOFF_CAP;
}
SMT_PAUSE();
}
}
if (spin_count) {
LOCKSTAT_RECORD(LS_LOCK_SET_SPIN, lp, spin_count);
}
LOCKSTAT_RECORD0(LS_LOCK_SET_ACQUIRE, lp);
}