/* * mutex_wakeup: * * Support routine for mutex_exit() that wakes up all waiters. * We assume that the mutex has been released, but it need not * be. */ void mutex_wakeup(kmutex_t *mtx) { turnstile_t *ts; ts = turnstile_lookup(mtx); if (ts == NULL) { turnstile_exit(mtx); return; } MUTEX_CLEAR_WAITERS(mtx); turnstile_wakeup(ts, TS_WRITER_Q, TS_WAITERS(ts, TS_WRITER_Q), NULL); }
/* * mutex_vector_exit() is called from mutex_exit() if the lock is not * adaptive, has waiters, or is not owned by the current thread (panic). */ void mutex_vector_exit(mutex_impl_t *lp) { turnstile_t *ts; if (MUTEX_TYPE_SPIN(lp)) { lock_clear_splx(&lp->m_spin.m_spinlock, lp->m_spin.m_oldspl); return; } if (MUTEX_OWNER(lp) != curthread) { mutex_panic("mutex_exit: not owner", lp); return; } ts = turnstile_lookup(lp); MUTEX_CLEAR_LOCK_AND_WAITERS(lp); if (ts == NULL) turnstile_exit(lp); else turnstile_wakeup(ts, TS_WRITER_Q, ts->ts_waiters, NULL); LOCKSTAT_RECORD0(LS_MUTEX_EXIT_RELEASE, lp); }
/* * mutex_vector_exit: * * Support routine for mutex_exit() that handles all cases. */ void mutex_vector_exit(kmutex_t *mtx) { turnstile_t *ts; uintptr_t curthread; if (MUTEX_SPIN_P(mtx)) { #ifdef FULL if (__predict_false(!MUTEX_SPINBIT_LOCKED_P(mtx))) { if (panicstr != NULL) return; MUTEX_ABORT(mtx, "exiting unheld spin mutex"); } MUTEX_UNLOCKED(mtx); MUTEX_SPINBIT_LOCK_UNLOCK(mtx); #endif MUTEX_SPIN_SPLRESTORE(mtx); return; } if (__predict_false((uintptr_t)panicstr | cold)) { MUTEX_UNLOCKED(mtx); MUTEX_RELEASE(mtx); return; } curthread = (uintptr_t)curlwp; MUTEX_DASSERT(mtx, curthread != 0); MUTEX_ASSERT(mtx, MUTEX_OWNER(mtx->mtx_owner) == curthread); MUTEX_UNLOCKED(mtx); #if !defined(LOCKDEBUG) __USE(curthread); #endif #ifdef LOCKDEBUG /* * Avoid having to take the turnstile chain lock every time * around. Raise the priority level to splhigh() in order * to disable preemption and so make the following atomic. */ { int s = splhigh(); if (!MUTEX_HAS_WAITERS(mtx)) { MUTEX_RELEASE(mtx); splx(s); return; } splx(s); } #endif /* * Get this lock's turnstile. This gets the interlock on * the sleep queue. Once we have that, we can clear the * lock. If there was no turnstile for the lock, there * were no waiters remaining. */ ts = turnstile_lookup(mtx); if (ts == NULL) { MUTEX_RELEASE(mtx); turnstile_exit(mtx); } else { MUTEX_RELEASE(mtx); turnstile_wakeup(ts, TS_WRITER_Q, TS_WAITERS(ts, TS_WRITER_Q), NULL); } }