/*
* mutex_wakeup:
*
* Support routine for mutex_exit() that wakes up all waiters.
* We assume that the mutex has been released, but it need not
* be.
*/
void
mutex_wakeup(kmutex_t *mtx)
{
turnstile_t *ts;
ts = turnstile_lookup(mtx);
if (ts == NULL) {
turnstile_exit(mtx);
return;
}
MUTEX_CLEAR_WAITERS(mtx);
turnstile_wakeup(ts, TS_WRITER_Q, TS_WAITERS(ts, TS_WRITER_Q), NULL);
}
/*
* mutex_vector_exit() is called from mutex_exit() if the lock is not
* adaptive, has waiters, or is not owned by the current thread (panic).
*/
void
mutex_vector_exit(mutex_impl_t *lp)
{
turnstile_t *ts;
if (MUTEX_TYPE_SPIN(lp)) {
lock_clear_splx(&lp->m_spin.m_spinlock, lp->m_spin.m_oldspl);
return;
}
if (MUTEX_OWNER(lp) != curthread) {
mutex_panic("mutex_exit: not owner", lp);
return;
}
ts = turnstile_lookup(lp);
MUTEX_CLEAR_LOCK_AND_WAITERS(lp);
if (ts == NULL)
turnstile_exit(lp);
else
turnstile_wakeup(ts, TS_WRITER_Q, ts->ts_waiters, NULL);
LOCKSTAT_RECORD0(LS_MUTEX_EXIT_RELEASE, lp);
}
/*
* mutex_vector_exit:
*
* Support routine for mutex_exit() that handles all cases.
*/
void
mutex_vector_exit(kmutex_t *mtx)
{
turnstile_t *ts;
uintptr_t curthread;
if (MUTEX_SPIN_P(mtx)) {
#ifdef FULL
if (__predict_false(!MUTEX_SPINBIT_LOCKED_P(mtx))) {
if (panicstr != NULL)
return;
MUTEX_ABORT(mtx, "exiting unheld spin mutex");
}
MUTEX_UNLOCKED(mtx);
MUTEX_SPINBIT_LOCK_UNLOCK(mtx);
#endif
MUTEX_SPIN_SPLRESTORE(mtx);
return;
}
if (__predict_false((uintptr_t)panicstr | cold)) {
MUTEX_UNLOCKED(mtx);
MUTEX_RELEASE(mtx);
return;
}
curthread = (uintptr_t)curlwp;
MUTEX_DASSERT(mtx, curthread != 0);
MUTEX_ASSERT(mtx, MUTEX_OWNER(mtx->mtx_owner) == curthread);
MUTEX_UNLOCKED(mtx);
#if !defined(LOCKDEBUG)
__USE(curthread);
#endif
#ifdef LOCKDEBUG
/*
* Avoid having to take the turnstile chain lock every time
* around. Raise the priority level to splhigh() in order
* to disable preemption and so make the following atomic.
*/
{
int s = splhigh();
if (!MUTEX_HAS_WAITERS(mtx)) {
MUTEX_RELEASE(mtx);
splx(s);
return;
}
splx(s);
}
#endif
/*
* Get this lock's turnstile. This gets the interlock on
* the sleep queue. Once we have that, we can clear the
* lock. If there was no turnstile for the lock, there
* were no waiters remaining.
*/
ts = turnstile_lookup(mtx);
if (ts == NULL) {
MUTEX_RELEASE(mtx);
turnstile_exit(mtx);
} else {
MUTEX_RELEASE(mtx);
turnstile_wakeup(ts, TS_WRITER_Q,
TS_WAITERS(ts, TS_WRITER_Q), NULL);
}
}
