static unsigned int NOINLINE
hw_lock_lock_contended(hw_lock_t lock, uintptr_t data, uint64_t timeout, boolean_t do_panic)
{
uint64_t end = 0;
uintptr_t holder = lock->lock_data;
int i;
if (timeout == 0)
timeout = LOCK_PANIC_TIMEOUT;
#if CONFIG_DTRACE
uint64_t begin;
boolean_t dtrace_enabled = lockstat_probemap[LS_LCK_SPIN_LOCK_SPIN] != 0;
if (__improbable(dtrace_enabled))
begin = mach_absolute_time();
#endif
for ( ; ; ) {
for (i = 0; i < LOCK_SNOOP_SPINS; i++) {
cpu_pause();
#if (!__ARM_ENABLE_WFE_) || (LOCK_PRETEST)
holder = ordered_load_hw(lock);
if (holder != 0)
continue;
#endif
if (atomic_compare_exchange(&lock->lock_data, 0, data,
memory_order_acquire_smp, TRUE)) {
#if CONFIG_DTRACE
if (__improbable(dtrace_enabled)) {
uint64_t spintime = mach_absolute_time() - begin;
if (spintime > dtrace_spin_threshold)
LOCKSTAT_RECORD2(LS_LCK_SPIN_LOCK_SPIN, lock, spintime, dtrace_spin_threshold);
}
#endif
return 1;
}
}
if (end == 0) {
end = ml_get_timebase() + timeout;
}
else if (ml_get_timebase() >= end)
break;
}
if (do_panic) {
// Capture the actual time spent blocked, which may be higher than the timeout
// if a misbehaving interrupt stole this thread's CPU time.
panic("Spinlock timeout after %llu ticks, %p = %lx",
(ml_get_timebase() - end + timeout), lock, holder);
}
return 0;
}
/*
* Routine: lck_rw_lock_exclusive
*/
void
lck_rw_lock_exclusive(
lck_rw_t *lck)
{
int i;
wait_result_t res;
#if MACH_LDEBUG
int decrementer;
#endif /* MACH_LDEBUG */
boolean_t istate;
#if CONFIG_DTRACE
uint64_t wait_interval = 0;
int slept = 0;
int readers_at_sleep;
#endif
istate = lck_interlock_lock(lck);
#if CONFIG_DTRACE
readers_at_sleep = lck->lck_rw_shared_count;
#endif
#if MACH_LDEBUG
decrementer = DECREMENTER_TIMEOUT;
#endif /* MACH_LDEBUG */
/*
* Try to acquire the lck_rw_want_write bit.
*/
while (lck->lck_rw_want_write) {
KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_EXCLUSIVE_CODE) | DBG_FUNC_START, (int)lck, 0, 0, 0, 0);
/*
* Either sleeping or spinning is happening, start
* a timing of our delay interval now.
*/
#if CONFIG_DTRACE
if ((lockstat_probemap[LS_LCK_RW_LOCK_EXCL_SPIN] || lockstat_probemap[LS_LCK_RW_LOCK_EXCL_BLOCK]) && wait_interval == 0) {
wait_interval = mach_absolute_time();
} else {
wait_interval = -1;
}
#endif
i = lock_wait_time[lck->lck_rw_can_sleep ? 1 : 0];
if (i != 0) {
lck_interlock_unlock(lck, istate);
#if MACH_LDEBUG
if (!--decrementer)
Debugger("timeout - lck_rw_want_write");
#endif /* MACH_LDEBUG */
while (--i != 0 && lck->lck_rw_want_write)
lck_rw_lock_pause(istate);
istate = lck_interlock_lock(lck);
}
if (lck->lck_rw_can_sleep && lck->lck_rw_want_write) {
lck->lck_w_waiting = TRUE;
res = assert_wait(RW_LOCK_WRITER_EVENT(lck), THREAD_UNINT);
if (res == THREAD_WAITING) {
lck_interlock_unlock(lck, istate);
res = thread_block(THREAD_CONTINUE_NULL);
#if CONFIG_DTRACE
slept = 1;
#endif
istate = lck_interlock_lock(lck);
}
}
KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_EXCLUSIVE_CODE) | DBG_FUNC_END, (int)lck, res, 0, 0, 0);
}
lck->lck_rw_want_write = TRUE;
/* Wait for readers (and upgrades) to finish */
#if MACH_LDEBUG
decrementer = DECREMENTER_TIMEOUT;
#endif /* MACH_LDEBUG */
while ((lck->lck_rw_shared_count != 0) || lck->lck_rw_want_upgrade) {
i = lock_wait_time[lck->lck_rw_can_sleep ? 1 : 0];
KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_EXCLUSIVE1_CODE) | DBG_FUNC_START,
(int)lck, lck->lck_rw_shared_count, lck->lck_rw_want_upgrade, i, 0);
#if CONFIG_DTRACE
/*
* Either sleeping or spinning is happening, start
* a timing of our delay interval now. If we set it
* to -1 we don't have accurate data so we cannot later
* decide to record a dtrace spin or sleep event.
*/
if ((lockstat_probemap[LS_LCK_RW_LOCK_EXCL_SPIN] || lockstat_probemap[LS_LCK_RW_LOCK_EXCL_BLOCK]) && wait_interval == 0) {
wait_interval = mach_absolute_time();
} else {
wait_interval = (unsigned) -1;
}
#endif
if (i != 0) {
lck_interlock_unlock(lck, istate);
#if MACH_LDEBUG
if (!--decrementer)
Debugger("timeout - wait for readers");
#endif /* MACH_LDEBUG */
while (--i != 0 && (lck->lck_rw_shared_count != 0 ||
lck->lck_rw_want_upgrade))
lck_rw_lock_pause(istate);
istate = lck_interlock_lock(lck);
}
if (lck->lck_rw_can_sleep && (lck->lck_rw_shared_count != 0 || lck->lck_rw_want_upgrade)) {
lck->lck_w_waiting = TRUE;
res = assert_wait(RW_LOCK_WRITER_EVENT(lck), THREAD_UNINT);
if (res == THREAD_WAITING) {
lck_interlock_unlock(lck, istate);
res = thread_block(THREAD_CONTINUE_NULL);
#if CONFIG_DTRACE
slept = 1;
#endif
istate = lck_interlock_lock(lck);
}
}
KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_EXCLUSIVE1_CODE) | DBG_FUNC_END,
(int)lck, lck->lck_rw_shared_count, lck->lck_rw_want_upgrade, res, 0);
}
lck_interlock_unlock(lck, istate);
#if CONFIG_DTRACE
/*
* Decide what latencies we suffered that are Dtrace events.
* If we have set wait_interval, then we either spun or slept.
* At least we get out from under the interlock before we record
* which is the best we can do here to minimize the impact
* of the tracing.
* If we have set wait_interval to -1, then dtrace was not enabled when we
* started sleeping/spinning so we don't record this event.
*/
if (wait_interval != 0 && wait_interval != (unsigned) -1) {
if (slept == 0) {
LOCKSTAT_RECORD2(LS_LCK_RW_LOCK_EXCL_SPIN, lck,
mach_absolute_time() - wait_interval, 1);
} else {
/*
* For the blocking case, we also record if when we blocked
* it was held for read or write, and how many readers.
* Notice that above we recorded this before we dropped
* the interlock so the count is accurate.
*/
LOCKSTAT_RECORD4(LS_LCK_RW_LOCK_EXCL_BLOCK, lck,
mach_absolute_time() - wait_interval, 1,
(readers_at_sleep == 0 ? 1 : 0), readers_at_sleep);
}
}
LOCKSTAT_RECORD(LS_LCK_RW_LOCK_EXCL_ACQUIRE, lck, 1);
#endif
}
boolean_t
lck_rw_lock_shared_to_exclusive(
lck_rw_t *lck)
{
int i;
boolean_t do_wakeup = FALSE;
wait_result_t res;
#if MACH_LDEBUG
int decrementer;
#endif /* MACH_LDEBUG */
boolean_t istate;
#if CONFIG_DTRACE
uint64_t wait_interval = 0;
int slept = 0;
int readers_at_sleep = 0;
#endif
istate = lck_interlock_lock(lck);
lck->lck_rw_shared_count--;
if (lck->lck_rw_want_upgrade) {
KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_SH_TO_EX_CODE) | DBG_FUNC_START,
(int)lck, lck->lck_rw_shared_count, lck->lck_rw_want_upgrade, 0, 0);
/*
* Someone else has requested upgrade.
* Since we've released a read lock, wake
* him up.
*/
if (lck->lck_w_waiting && (lck->lck_rw_shared_count == 0)) {
lck->lck_w_waiting = FALSE;
do_wakeup = TRUE;
}
lck_interlock_unlock(lck, istate);
if (do_wakeup)
thread_wakeup(RW_LOCK_WRITER_EVENT(lck));
KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_SH_TO_EX_CODE) | DBG_FUNC_END,
(int)lck, lck->lck_rw_shared_count, lck->lck_rw_want_upgrade, 0, 0);
return (FALSE);
}
lck->lck_rw_want_upgrade = TRUE;
#if MACH_LDEBUG
decrementer = DECREMENTER_TIMEOUT;
#endif /* MACH_LDEBUG */
while (lck->lck_rw_shared_count != 0) {
#if CONFIG_DTRACE
if (lockstat_probemap[LS_LCK_RW_LOCK_SHARED_TO_EXCL_SPIN] && wait_interval == 0) {
wait_interval = mach_absolute_time();
readers_at_sleep = lck->lck_rw_shared_count;
} else {
wait_interval = -1;
}
#endif
i = lock_wait_time[lck->lck_rw_can_sleep ? 1 : 0];
KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_SH_TO_EX1_CODE) | DBG_FUNC_START,
(int)lck, lck->lck_rw_shared_count, i, 0, 0);
if (i != 0) {
lck_interlock_unlock(lck, istate);
#if MACH_LDEBUG
if (!--decrementer)
Debugger("timeout - lck_rw_shared_count");
#endif /* MACH_LDEBUG */
while (--i != 0 && lck->lck_rw_shared_count != 0)
lck_rw_lock_pause(istate);
istate = lck_interlock_lock(lck);
}
if (lck->lck_rw_can_sleep && lck->lck_rw_shared_count != 0) {
lck->lck_w_waiting = TRUE;
res = assert_wait(RW_LOCK_WRITER_EVENT(lck), THREAD_UNINT);
if (res == THREAD_WAITING) {
lck_interlock_unlock(lck, istate);
res = thread_block(THREAD_CONTINUE_NULL);
#if CONFIG_DTRACE
slept = 1;
#endif
istate = lck_interlock_lock(lck);
}
}
KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_SH_TO_EX1_CODE) | DBG_FUNC_END,
(int)lck, lck->lck_rw_shared_count, 0, 0, 0);
}
lck_interlock_unlock(lck, istate);
#if CONFIG_DTRACE
/*
* We infer whether we took the sleep/spin path above by checking readers_at_sleep.
*/
if (wait_interval != 0 && wait_interval != (unsigned) -1 && readers_at_sleep) {
if (slept == 0) {
LOCKSTAT_RECORD2(LS_LCK_RW_LOCK_SHARED_TO_EXCL_SPIN, lck, mach_absolute_time() - wait_interval, 0);
} else {
LOCKSTAT_RECORD4(LS_LCK_RW_LOCK_SHARED_TO_EXCL_BLOCK, lck,
mach_absolute_time() - wait_interval, 1,
(readers_at_sleep == 0 ? 1 : 0), readers_at_sleep);
}
}
LOCKSTAT_RECORD(LS_LCK_RW_LOCK_SHARED_TO_EXCL_UPGRADE, lck, 1);
#endif
return (TRUE);
}
/*
* Routine: lck_rw_lock_shared_gen
*/
void
lck_rw_lock_shared_gen(
lck_rw_t *lck)
{
int i;
wait_result_t res;
#if MACH_LDEBUG
int decrementer;
#endif /* MACH_LDEBUG */
boolean_t istate;
#if CONFIG_DTRACE
uint64_t wait_interval = 0;
int slept = 0;
int readers_at_sleep;
#endif
istate = lck_interlock_lock(lck);
#if CONFIG_DTRACE
readers_at_sleep = lck->lck_rw_shared_count;
#endif
#if MACH_LDEBUG
decrementer = DECREMENTER_TIMEOUT;
#endif /* MACH_LDEBUG */
while ((lck->lck_rw_want_write || lck->lck_rw_want_upgrade) &&
((lck->lck_rw_shared_count == 0) || lck->lck_rw_priv_excl)) {
i = lock_wait_time[lck->lck_rw_can_sleep ? 1 : 0];
KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_SHARED_CODE) | DBG_FUNC_START,
(int)lck, lck->lck_rw_want_write, lck->lck_rw_want_upgrade, i, 0);
#if CONFIG_DTRACE
if ((lockstat_probemap[LS_LCK_RW_LOCK_SHARED_SPIN] || lockstat_probemap[LS_LCK_RW_LOCK_SHARED_BLOCK]) && wait_interval == 0) {
wait_interval = mach_absolute_time();
} else {
wait_interval = -1;
}
#endif
if (i != 0) {
lck_interlock_unlock(lck, istate);
#if MACH_LDEBUG
if (!--decrementer)
Debugger("timeout - wait no writers");
#endif /* MACH_LDEBUG */
while (--i != 0 &&
(lck->lck_rw_want_write || lck->lck_rw_want_upgrade) &&
((lck->lck_rw_shared_count == 0) || lck->lck_rw_priv_excl))
lck_rw_lock_pause(istate);
istate = lck_interlock_lock(lck);
}
if (lck->lck_rw_can_sleep &&
(lck->lck_rw_want_write || lck->lck_rw_want_upgrade) &&
((lck->lck_rw_shared_count == 0) || lck->lck_rw_priv_excl)) {
lck->lck_r_waiting = TRUE;
res = assert_wait(RW_LOCK_READER_EVENT(lck), THREAD_UNINT);
if (res == THREAD_WAITING) {
lck_interlock_unlock(lck, istate);
res = thread_block(THREAD_CONTINUE_NULL);
#if CONFIG_DTRACE
slept = 1;
#endif
istate = lck_interlock_lock(lck);
}
}
KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_RW_LCK_SHARED_CODE) | DBG_FUNC_END,
(int)lck, lck->lck_rw_want_write, lck->lck_rw_want_upgrade, res, 0);
}
lck->lck_rw_shared_count++;
lck_interlock_unlock(lck, istate);
#if CONFIG_DTRACE
if (wait_interval != 0 && wait_interval != (unsigned) -1) {
if (slept == 0) {
LOCKSTAT_RECORD2(LS_LCK_RW_LOCK_SHARED_SPIN, lck, mach_absolute_time() - wait_interval, 0);
} else {
LOCKSTAT_RECORD4(LS_LCK_RW_LOCK_SHARED_BLOCK, lck,
mach_absolute_time() - wait_interval, 0,
(readers_at_sleep == 0 ? 1 : 0), readers_at_sleep);
}
}
LOCKSTAT_RECORD(LS_LCK_RW_LOCK_SHARED_ACQUIRE, lck, 0);
#endif
}
