void _sx_xunlock(struct sx *sx, const char *file, int line) { _sx_assert(sx, SX_XLOCKED, file, line); mtx_lock(sx->sx_lock); MPASS(sx->sx_cnt == -1); WITNESS_UNLOCK(&sx->sx_object, LOP_EXCLUSIVE, file, line); /* Release. */ sx->sx_cnt++; sx->sx_xholder = NULL; /* * Wake up waiters if there are any. Give precedence to slock waiters. */ if (sx->sx_shrd_wcnt > 0) cv_broadcast(&sx->sx_shrd_cv); else if (sx->sx_excl_wcnt > 0) cv_signal(&sx->sx_excl_cv); LOCK_LOG_LOCK("XUNLOCK", &sx->sx_object, 0, 0, file, line); mtx_unlock(sx->sx_lock); }
/* * Try to do a non-blocking upgrade from a shared lock to an exclusive lock. * This will only succeed if this thread holds a single shared lock. * Return 1 if if the upgrade succeed, 0 otherwise. */ int sx_try_upgrade_(struct sx *sx, const char *file, int line) { uintptr_t x; int success; if (SCHEDULER_STOPPED()) return (1); KASSERT(sx->sx_lock != SX_LOCK_DESTROYED, ("sx_try_upgrade() of destroyed sx @ %s:%d", file, line)); _sx_assert(sx, SA_SLOCKED, file, line); /* * Try to switch from one shared lock to an exclusive lock. We need * to maintain the SX_LOCK_EXCLUSIVE_WAITERS flag if set so that * we will wake up the exclusive waiters when we drop the lock. */ x = sx->sx_lock & SX_LOCK_EXCLUSIVE_WAITERS; success = atomic_cmpset_ptr(&sx->sx_lock, SX_SHARERS_LOCK(1) | x, (uintptr_t)curthread | x); LOCK_LOG_TRY("XUPGRADE", &sx->lock_object, 0, success, file, line); if (success) { WITNESS_UPGRADE(&sx->lock_object, LOP_EXCLUSIVE | LOP_TRYLOCK, file, line); LOCKSTAT_RECORD0(sx__upgrade, sx); } return (success); }
/* * Downgrade an unrecursed exclusive lock into a single shared lock. */ void _sx_downgrade(struct sx *sx, const char *file, int line) { uintptr_t x; int wakeup_swapper; KASSERT(sx->sx_lock != SX_LOCK_DESTROYED, ("sx_downgrade() of destroyed sx @ %s:%d", file, line)); _sx_assert(sx, SA_XLOCKED | SA_NOTRECURSED, file, line); #ifndef INVARIANTS if (sx_recursed(sx)) panic("downgrade of a recursed lock"); #endif WITNESS_DOWNGRADE(&sx->lock_object, 0, file, line); /* * Try to switch from an exclusive lock with no shared waiters * to one sharer with no shared waiters. If there are * exclusive waiters, we don't need to lock the sleep queue so * long as we preserve the flag. We do one quick try and if * that fails we grab the sleepq lock to keep the flags from * changing and do it the slow way. * * We have to lock the sleep queue if there are shared waiters * so we can wake them up. */ x = sx->sx_lock; if (!(x & SX_LOCK_SHARED_WAITERS) && atomic_cmpset_rel_ptr(&sx->sx_lock, x, SX_SHARERS_LOCK(1) | (x & SX_LOCK_EXCLUSIVE_WAITERS))) { LOCK_LOG_LOCK("XDOWNGRADE", &sx->lock_object, 0, 0, file, line); return; } /* * Lock the sleep queue so we can read the waiters bits * without any races and wakeup any shared waiters. */ sleepq_lock(&sx->lock_object); /* * Preserve SX_LOCK_EXCLUSIVE_WAITERS while downgraded to a single * shared lock. If there are any shared waiters, wake them up. */ wakeup_swapper = 0; x = sx->sx_lock; atomic_store_rel_ptr(&sx->sx_lock, SX_SHARERS_LOCK(1) | (x & SX_LOCK_EXCLUSIVE_WAITERS)); if (x & SX_LOCK_SHARED_WAITERS) wakeup_swapper = sleepq_broadcast(&sx->lock_object, SLEEPQ_SX, 0, SQ_SHARED_QUEUE); sleepq_release(&sx->lock_object); LOCK_LOG_LOCK("XDOWNGRADE", &sx->lock_object, 0, 0, file, line); LOCKSTAT_RECORD0(LS_SX_DOWNGRADE_DOWNGRADE, sx); if (wakeup_swapper) kick_proc0(); }
void _sx_sunlock(struct sx *sx, const char *file, int line) { _sx_assert(sx, SX_SLOCKED, file, line); mtx_lock(sx->sx_lock); WITNESS_UNLOCK(&sx->sx_object, 0, file, line); /* Release. */ sx->sx_cnt--; /* * If we just released the last shared lock, wake any waiters up, giving * exclusive lockers precedence. In order to make sure that exclusive * lockers won't be blocked forever, don't wake shared lock waiters if * there are exclusive lock waiters. */ if (sx->sx_excl_wcnt > 0) { if (sx->sx_cnt == 0) cv_signal(&sx->sx_excl_cv); } else if (sx->sx_shrd_wcnt > 0) cv_broadcast(&sx->sx_shrd_cv); LOCK_LOG_LOCK("SUNLOCK", &sx->sx_object, 0, 0, file, line); mtx_unlock(sx->sx_lock); }
void _sx_sunlock(struct sx *sx, const char *file, int line) { if (SCHEDULER_STOPPED()) return; KASSERT(sx->sx_lock != SX_LOCK_DESTROYED, ("sx_sunlock() of destroyed sx @ %s:%d", file, line)); _sx_assert(sx, SA_SLOCKED, file, line); WITNESS_UNLOCK(&sx->lock_object, 0, file, line); LOCK_LOG_LOCK("SUNLOCK", &sx->lock_object, 0, 0, file, line); __sx_sunlock(sx, file, line); curthread->td_locks--; }
void _sx_sunlock(struct sx *sx, const char *file, int line) { MPASS(curthread != NULL); KASSERT(sx->sx_lock != SX_LOCK_DESTROYED, ("sx_sunlock() of destroyed sx @ %s:%d", file, line)); _sx_assert(sx, SA_SLOCKED, file, line); curthread->td_locks--; WITNESS_UNLOCK(&sx->lock_object, 0, file, line); LOCK_LOG_LOCK("SUNLOCK", &sx->lock_object, 0, 0, file, line); __sx_sunlock(sx, file, line); LOCKSTAT_PROFILE_RELEASE_LOCK(LS_SX_SUNLOCK_RELEASE, sx); }
void _sx_xunlock(struct sx *sx, const char *file, int line) { MPASS(curthread != NULL); KASSERT(sx->sx_lock != SX_LOCK_DESTROYED, ("sx_xunlock() of destroyed sx @ %s:%d", file, line)); _sx_assert(sx, SA_XLOCKED, file, line); curthread->td_locks--; WITNESS_UNLOCK(&sx->lock_object, LOP_EXCLUSIVE, file, line); LOCK_LOG_LOCK("XUNLOCK", &sx->lock_object, 0, sx->sx_recurse, file, line); if (!sx_recursed(sx)) LOCKSTAT_PROFILE_RELEASE_LOCK(LS_SX_XUNLOCK_RELEASE, sx); __sx_xunlock(sx, curthread, file, line); }
void _sx_downgrade(struct sx *sx, const char *file, int line) { _sx_assert(sx, SX_XLOCKED, file, line); mtx_lock(sx->sx_lock); MPASS(sx->sx_cnt == -1); WITNESS_DOWNGRADE(&sx->sx_object, 0, file, line); sx->sx_cnt = 1; sx->sx_xholder = NULL; if (sx->sx_shrd_wcnt > 0) cv_broadcast(&sx->sx_shrd_cv); LOCK_LOG_LOCK("XDOWNGRADE", &sx->sx_object, 0, 0, file, line); mtx_unlock(sx->sx_lock); }
int _sx_try_upgrade(struct sx *sx, const char *file, int line) { _sx_assert(sx, SX_SLOCKED, file, line); mtx_lock(sx->sx_lock); if (sx->sx_cnt == 1) { sx->sx_cnt = -1; sx->sx_xholder = curthread; LOCK_LOG_TRY("XUPGRADE", &sx->sx_object, 0, 1, file, line); WITNESS_UPGRADE(&sx->sx_object, LOP_EXCLUSIVE | LOP_TRYLOCK, file, line); mtx_unlock(sx->sx_lock); return (1); } else { LOCK_LOG_TRY("XUPGRADE", &sx->sx_object, 0, 0, file, line); mtx_unlock(sx->sx_lock); return (0); } }