/* * Routine: lck_rw_sleep */ wait_result_t lck_rw_sleep( lck_rw_t *lck, lck_sleep_action_t lck_sleep_action, event_t event, wait_interrupt_t interruptible) { wait_result_t res; lck_rw_type_t lck_rw_type; thread_t thread = current_thread(); if ((lck_sleep_action & ~LCK_SLEEP_MASK) != 0) panic("Invalid lock sleep action %x\n", lck_sleep_action); if (lck_sleep_action & LCK_SLEEP_PROMOTED_PRI) { /* * Although we are dropping the RW lock, the intent in most cases * is that this thread remains as an observer, since it may hold * some secondary resource, but must yield to avoid deadlock. In * this situation, make sure that the thread is boosted to the * RW lock ceiling while blocked, so that it can re-acquire the * RW lock at that priority. */ thread->rwlock_count++; } res = assert_wait(event, interruptible); if (res == THREAD_WAITING) { lck_rw_type = lck_rw_done(lck); res = thread_block(THREAD_CONTINUE_NULL); if (!(lck_sleep_action & LCK_SLEEP_UNLOCK)) { if (!(lck_sleep_action & (LCK_SLEEP_SHARED|LCK_SLEEP_EXCLUSIVE))) lck_rw_lock(lck, lck_rw_type); else if (lck_sleep_action & LCK_SLEEP_EXCLUSIVE) lck_rw_lock_exclusive(lck); else lck_rw_lock_shared(lck); } } else if (lck_sleep_action & LCK_SLEEP_UNLOCK) (void)lck_rw_done(lck); if (lck_sleep_action & LCK_SLEEP_PROMOTED_PRI) { if ((thread->rwlock_count-- == 1 /* field now 0 */) && (thread->sched_flags & TH_SFLAG_RW_PROMOTED)) { /* sched_flags checked without lock, but will be rechecked while clearing */ /* Only if the caller wanted the lck_rw_t returned unlocked should we drop to 0 */ assert(lck_sleep_action & LCK_SLEEP_UNLOCK); lck_rw_clear_promotion(thread); } } return res; }
/* * Routine: lck_mtx_sleep */ wait_result_t lck_mtx_sleep( lck_mtx_t *lck, lck_sleep_action_t lck_sleep_action, event_t event, wait_interrupt_t interruptible) { wait_result_t res; thread_t thread = current_thread(); KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_MTX_SLEEP_CODE) | DBG_FUNC_START, VM_KERNEL_UNSLIDE_OR_PERM(lck), (int)lck_sleep_action, VM_KERNEL_UNSLIDE_OR_PERM(event), (int)interruptible, 0); if ((lck_sleep_action & ~LCK_SLEEP_MASK) != 0) panic("Invalid lock sleep action %x\n", lck_sleep_action); if (lck_sleep_action & LCK_SLEEP_PROMOTED_PRI) { /* * We overload the RW lock promotion to give us a priority ceiling * during the time that this thread is asleep, so that when it * is re-awakened (and not yet contending on the mutex), it is * runnable at a reasonably high priority. */ thread->rwlock_count++; } res = assert_wait(event, interruptible); if (res == THREAD_WAITING) { lck_mtx_unlock(lck); res = thread_block(THREAD_CONTINUE_NULL); if (!(lck_sleep_action & LCK_SLEEP_UNLOCK)) { if ((lck_sleep_action & LCK_SLEEP_SPIN)) lck_mtx_lock_spin(lck); else if ((lck_sleep_action & LCK_SLEEP_SPIN_ALWAYS)) lck_mtx_lock_spin_always(lck); else lck_mtx_lock(lck); } } else if (lck_sleep_action & LCK_SLEEP_UNLOCK) lck_mtx_unlock(lck); if (lck_sleep_action & LCK_SLEEP_PROMOTED_PRI) { if ((thread->rwlock_count-- == 1 /* field now 0 */) && (thread->sched_flags & TH_SFLAG_RW_PROMOTED)) { /* sched_flags checked without lock, but will be rechecked while clearing */ lck_rw_clear_promotion(thread); } } KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_MTX_SLEEP_CODE) | DBG_FUNC_END, (int)res, 0, 0, 0, 0); return res; }
/* * Routine: lck_mtx_sleep_deadline */ wait_result_t lck_mtx_sleep_deadline( lck_mtx_t *lck, lck_sleep_action_t lck_sleep_action, event_t event, wait_interrupt_t interruptible, uint64_t deadline) { wait_result_t res; thread_t thread = current_thread(); KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_MTX_SLEEP_DEADLINE_CODE) | DBG_FUNC_START, VM_KERNEL_UNSLIDE_OR_PERM(lck), (int)lck_sleep_action, VM_KERNEL_UNSLIDE_OR_PERM(event), (int)interruptible, 0); if ((lck_sleep_action & ~LCK_SLEEP_MASK) != 0) panic("Invalid lock sleep action %x\n", lck_sleep_action); if (lck_sleep_action & LCK_SLEEP_PROMOTED_PRI) { /* * See lck_mtx_sleep(). */ thread->rwlock_count++; } res = assert_wait_deadline(event, interruptible, deadline); if (res == THREAD_WAITING) { lck_mtx_unlock(lck); res = thread_block(THREAD_CONTINUE_NULL); if (!(lck_sleep_action & LCK_SLEEP_UNLOCK)) { if ((lck_sleep_action & LCK_SLEEP_SPIN)) lck_mtx_lock_spin(lck); else lck_mtx_lock(lck); } } else if (lck_sleep_action & LCK_SLEEP_UNLOCK) lck_mtx_unlock(lck); if (lck_sleep_action & LCK_SLEEP_PROMOTED_PRI) { if ((thread->rwlock_count-- == 1 /* field now 0 */) && (thread->sched_flags & TH_SFLAG_RW_PROMOTED)) { /* sched_flags checked without lock, but will be rechecked while clearing */ lck_rw_clear_promotion(thread); } } KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_LOCKS, LCK_MTX_SLEEP_DEADLINE_CODE) | DBG_FUNC_END, (int)res, 0, 0, 0, 0); return res; }
/* * Routine: lck_rw_sleep_deadline */ wait_result_t lck_rw_sleep_deadline( lck_rw_t *lck, lck_sleep_action_t lck_sleep_action, event_t event, wait_interrupt_t interruptible, uint64_t deadline) { wait_result_t res; lck_rw_type_t lck_rw_type; thread_t thread = current_thread(); if ((lck_sleep_action & ~LCK_SLEEP_MASK) != 0) panic("Invalid lock sleep action %x\n", lck_sleep_action); if (lck_sleep_action & LCK_SLEEP_PROMOTED_PRI) { thread->rwlock_count++; } res = assert_wait_deadline(event, interruptible, deadline); if (res == THREAD_WAITING) { lck_rw_type = lck_rw_done(lck); res = thread_block(THREAD_CONTINUE_NULL); if (!(lck_sleep_action & LCK_SLEEP_UNLOCK)) { if (!(lck_sleep_action & (LCK_SLEEP_SHARED|LCK_SLEEP_EXCLUSIVE))) lck_rw_lock(lck, lck_rw_type); else if (lck_sleep_action & LCK_SLEEP_EXCLUSIVE) lck_rw_lock_exclusive(lck); else lck_rw_lock_shared(lck); } } else if (lck_sleep_action & LCK_SLEEP_UNLOCK) (void)lck_rw_done(lck); if (lck_sleep_action & LCK_SLEEP_PROMOTED_PRI) { if ((thread->rwlock_count-- == 1 /* field now 0 */) && (thread->sched_flags & TH_SFLAG_RW_PROMOTED)) { /* sched_flags checked without lock, but will be rechecked while clearing */ /* Only if the caller wanted the lck_rw_t returned unlocked should we drop to 0 */ assert(lck_sleep_action & LCK_SLEEP_UNLOCK); lck_rw_clear_promotion(thread); } } return res; }