/*
* Routine: lck_rw_sleep
*/
wait_result_t
lck_rw_sleep(
lck_rw_t *lck,
lck_sleep_action_t lck_sleep_action,
event_t event,
wait_interrupt_t interruptible)
{
wait_result_t res;
lck_rw_type_t lck_rw_type;
thread_t thread = current_thread();
if ((lck_sleep_action & ~LCK_SLEEP_MASK) != 0)
panic("Invalid lock sleep action %x\n", lck_sleep_action);
if (lck_sleep_action & LCK_SLEEP_PROMOTED_PRI) {
/*
* Although we are dropping the RW lock, the intent in most cases
* is that this thread remains as an observer, since it may hold
* some secondary resource, but must yield to avoid deadlock. In
* this situation, make sure that the thread is boosted to the
* RW lock ceiling while blocked, so that it can re-acquire the
* RW lock at that priority.
*/
thread->rwlock_count++;
}
res = assert_wait(event, interruptible);
if (res == THREAD_WAITING) {
lck_rw_type = lck_rw_done(lck);
res = thread_block(THREAD_CONTINUE_NULL);
if (!(lck_sleep_action & LCK_SLEEP_UNLOCK)) {
if (!(lck_sleep_action & (LCK_SLEEP_SHARED|LCK_SLEEP_EXCLUSIVE)))
lck_rw_lock(lck, lck_rw_type);
else if (lck_sleep_action & LCK_SLEEP_EXCLUSIVE)
lck_rw_lock_exclusive(lck);
else
lck_rw_lock_shared(lck);
}
}
else
if (lck_sleep_action & LCK_SLEEP_UNLOCK)
(void)lck_rw_done(lck);
if (lck_sleep_action & LCK_SLEEP_PROMOTED_PRI) {
if ((thread->rwlock_count-- == 1 /* field now 0 */) && (thread->sched_flags & TH_SFLAG_RW_PROMOTED)) {
/* sched_flags checked without lock, but will be rechecked while clearing */
/* Only if the caller wanted the lck_rw_t returned unlocked should we drop to 0 */
assert(lck_sleep_action & LCK_SLEEP_UNLOCK);
lck_rw_clear_promotion(thread);
}
}
return res;
}
/*
* Routine: lck_mtx_sleep
*/
wait_result_t
lck_mtx_sleep(
lck_mtx_t *lck,
lck_sleep_action_t lck_sleep_action,
event_t event,
wait_interrupt_t interruptible)
{
wait_result_t res;
thread_t thread = current_thread();
KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_MTX_SLEEP_CODE) | DBG_FUNC_START,
VM_KERNEL_UNSLIDE_OR_PERM(lck), (int)lck_sleep_action, VM_KERNEL_UNSLIDE_OR_PERM(event), (int)interruptible, 0);
if ((lck_sleep_action & ~LCK_SLEEP_MASK) != 0)
panic("Invalid lock sleep action %x\n", lck_sleep_action);
if (lck_sleep_action & LCK_SLEEP_PROMOTED_PRI) {
/*
* We overload the RW lock promotion to give us a priority ceiling
* during the time that this thread is asleep, so that when it
* is re-awakened (and not yet contending on the mutex), it is
* runnable at a reasonably high priority.
*/
thread->rwlock_count++;
}
res = assert_wait(event, interruptible);
if (res == THREAD_WAITING) {
lck_mtx_unlock(lck);
res = thread_block(THREAD_CONTINUE_NULL);
if (!(lck_sleep_action & LCK_SLEEP_UNLOCK)) {
if ((lck_sleep_action & LCK_SLEEP_SPIN))
lck_mtx_lock_spin(lck);
else if ((lck_sleep_action & LCK_SLEEP_SPIN_ALWAYS))
lck_mtx_lock_spin_always(lck);
else
lck_mtx_lock(lck);
}
}
else
if (lck_sleep_action & LCK_SLEEP_UNLOCK)
lck_mtx_unlock(lck);
if (lck_sleep_action & LCK_SLEEP_PROMOTED_PRI) {
if ((thread->rwlock_count-- == 1 /* field now 0 */) && (thread->sched_flags & TH_SFLAG_RW_PROMOTED)) {
/* sched_flags checked without lock, but will be rechecked while clearing */
lck_rw_clear_promotion(thread);
}
}
KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_MTX_SLEEP_CODE) | DBG_FUNC_END, (int)res, 0, 0, 0, 0);
return res;
}
/*
* Routine: lck_mtx_sleep_deadline
*/
wait_result_t
lck_mtx_sleep_deadline(
lck_mtx_t *lck,
lck_sleep_action_t lck_sleep_action,
event_t event,
wait_interrupt_t interruptible,
uint64_t deadline)
{
wait_result_t res;
thread_t thread = current_thread();
KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_MTX_SLEEP_DEADLINE_CODE) | DBG_FUNC_START,
VM_KERNEL_UNSLIDE_OR_PERM(lck), (int)lck_sleep_action, VM_KERNEL_UNSLIDE_OR_PERM(event), (int)interruptible, 0);
if ((lck_sleep_action & ~LCK_SLEEP_MASK) != 0)
panic("Invalid lock sleep action %x\n", lck_sleep_action);
if (lck_sleep_action & LCK_SLEEP_PROMOTED_PRI) {
/*
* See lck_mtx_sleep().
*/
thread->rwlock_count++;
}
res = assert_wait_deadline(event, interruptible, deadline);
if (res == THREAD_WAITING) {
lck_mtx_unlock(lck);
res = thread_block(THREAD_CONTINUE_NULL);
if (!(lck_sleep_action & LCK_SLEEP_UNLOCK)) {
if ((lck_sleep_action & LCK_SLEEP_SPIN))
lck_mtx_lock_spin(lck);
else
lck_mtx_lock(lck);
}
}
else
if (lck_sleep_action & LCK_SLEEP_UNLOCK)
lck_mtx_unlock(lck);
if (lck_sleep_action & LCK_SLEEP_PROMOTED_PRI) {
if ((thread->rwlock_count-- == 1 /* field now 0 */) && (thread->sched_flags & TH_SFLAG_RW_PROMOTED)) {
/* sched_flags checked without lock, but will be rechecked while clearing */
lck_rw_clear_promotion(thread);
}
}
KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_MTX_SLEEP_DEADLINE_CODE) | DBG_FUNC_END, (int)res, 0, 0, 0, 0);
return res;
}
/*
* Routine: lck_rw_sleep_deadline
*/
wait_result_t
lck_rw_sleep_deadline(
lck_rw_t *lck,
lck_sleep_action_t lck_sleep_action,
event_t event,
wait_interrupt_t interruptible,
uint64_t deadline)
{
wait_result_t res;
lck_rw_type_t lck_rw_type;
thread_t thread = current_thread();
if ((lck_sleep_action & ~LCK_SLEEP_MASK) != 0)
panic("Invalid lock sleep action %x\n", lck_sleep_action);
if (lck_sleep_action & LCK_SLEEP_PROMOTED_PRI) {
thread->rwlock_count++;
}
res = assert_wait_deadline(event, interruptible, deadline);
if (res == THREAD_WAITING) {
lck_rw_type = lck_rw_done(lck);
res = thread_block(THREAD_CONTINUE_NULL);
if (!(lck_sleep_action & LCK_SLEEP_UNLOCK)) {
if (!(lck_sleep_action & (LCK_SLEEP_SHARED|LCK_SLEEP_EXCLUSIVE)))
lck_rw_lock(lck, lck_rw_type);
else if (lck_sleep_action & LCK_SLEEP_EXCLUSIVE)
lck_rw_lock_exclusive(lck);
else
lck_rw_lock_shared(lck);
}
}
else
if (lck_sleep_action & LCK_SLEEP_UNLOCK)
(void)lck_rw_done(lck);
if (lck_sleep_action & LCK_SLEEP_PROMOTED_PRI) {
if ((thread->rwlock_count-- == 1 /* field now 0 */) && (thread->sched_flags & TH_SFLAG_RW_PROMOTED)) {
/* sched_flags checked without lock, but will be rechecked while clearing */
/* Only if the caller wanted the lck_rw_t returned unlocked should we drop to 0 */
assert(lck_sleep_action & LCK_SLEEP_UNLOCK);
lck_rw_clear_promotion(thread);
}
}
return res;
}
