/* * Routine: wait_queue_assert_wait64 * Purpose: * Insert the current thread into the supplied wait queue * waiting for a particular event to be posted to that queue. * Conditions: * nothing of interest locked. */ wait_result_t wait_queue_assert_wait64( wait_queue_t wq, event64_t event, wait_interrupt_t interruptible) { spl_t s; wait_result_t ret; thread_t cur_thread = current_thread(); /* If it is an invalid wait queue, you cant wait on it */ if (!wait_queue_is_valid(wq)) { thread_t thread = current_thread(); return (thread->wait_result = THREAD_RESTART); } s = splsched(); wait_queue_lock(wq); thread_lock(cur_thread); ret = wait_queue_assert_wait64_locked(wq, event, interruptible, cur_thread); thread_unlock(cur_thread); wait_queue_unlock(wq); splx(s); return(ret); }
/* * Routine: wait_queue_assert_wait * Purpose: * Insert the current thread into the supplied wait queue * waiting for a particular event to be posted to that queue. * * Conditions: * nothing of interest locked. */ wait_result_t wait_queue_assert_wait( wait_queue_t wq, event_t event, wait_interrupt_t interruptible, uint64_t deadline) { spl_t s; wait_result_t ret; thread_t thread = current_thread(); /* If it is an invalid wait queue, you can't wait on it */ if (!wait_queue_is_valid(wq)) return (thread->wait_result = THREAD_RESTART); s = splsched(); wait_queue_lock(wq); thread_lock(thread); ret = wait_queue_assert_wait64_locked(wq, CAST_DOWN(event64_t,event), interruptible, deadline, thread); thread_unlock(thread); wait_queue_unlock(wq); splx(s); return(ret); }
/* * Routine: wait_queue_wakeup64_thread * Purpose: * Wakeup the particular thread that was specified if and only * it was in this wait queue (or one of it's set's queues) * and waiting on the specified event. * * This is much safer than just removing the thread from * whatever wait queue it happens to be on. For instance, it * may have already been awoken from the wait you intended to * interrupt and waited on something else (like another * semaphore). * Conditions: * nothing of interest locked * we need to assume spl needs to be raised * Returns: * KERN_SUCCESS - the thread was found waiting and awakened * KERN_NOT_WAITING - the thread was not waiting here */ kern_return_t wait_queue_wakeup64_thread( wait_queue_t wq, event64_t event, thread_t thread, wait_result_t result) { kern_return_t res; spl_t s; if (!wait_queue_is_valid(wq)) { return KERN_INVALID_ARGUMENT; } s = splsched(); wait_queue_lock(wq); res = _wait_queue_select64_thread(wq, event, thread); wait_queue_unlock(wq); if (res == KERN_SUCCESS) { res = thread_go(thread, result); assert(res == KERN_SUCCESS); thread_unlock(thread); splx(s); return res; } splx(s); return KERN_NOT_WAITING; }
/* * Routine: _wait_queue_select64_one * Purpose: * Select the best thread off a wait queue that meet the * supplied criteria. * Conditions: * at splsched * wait queue locked * possibly recursive * Returns: * a locked thread - if one found * Note: * This is where the sync policy of the wait queue comes * into effect. For now, we just assume FIFO. */ static thread_t _wait_queue_select64_one( wait_queue_t wq, event64_t event) { wait_queue_element_t wq_element; wait_queue_element_t wqe_next; thread_t t = THREAD_NULL; queue_t q; assert(wq->wq_fifo); q = &wq->wq_queue; wq_element = (wait_queue_element_t) queue_first(q); while (!queue_end(q, (queue_entry_t)wq_element)) { WAIT_QUEUE_ELEMENT_CHECK(wq, wq_element); wqe_next = (wait_queue_element_t) queue_next((queue_t) wq_element); /* * We may have to recurse if this is a compound wait queue. */ if (wq_element->wqe_type == WAIT_QUEUE_LINK) { wait_queue_link_t wql = (wait_queue_link_t)wq_element; wait_queue_t set_queue; /* * We have to check the set wait queue. */ set_queue = (wait_queue_t)wql->wql_setqueue; wait_queue_lock(set_queue); if (! wait_queue_empty(set_queue)) { t = _wait_queue_select64_one(set_queue, event); } wait_queue_unlock(set_queue); if (t != THREAD_NULL) return t; } else { /* * Otherwise, its a thread. If it is waiting on * the event we are posting to this queue, pull * it off the queue and stick it in out wake_queue. */ thread_t t = (thread_t)wq_element; if (t->wait_event == event) { thread_lock(t); remqueue(q, (queue_entry_t) t); t->wait_queue = WAIT_QUEUE_NULL; t->wait_event = NO_EVENT64; t->at_safe_point = FALSE; return t; /* still locked */ } } wq_element = wqe_next; } return THREAD_NULL; }
/* * Routine: wait_queue_wakeup_all * Purpose: * Wakeup some number of threads that are in the specified * wait queue and waiting on the specified event. * Conditions: * Nothing locked * Returns: * KERN_SUCCESS - Threads were woken up * KERN_NOT_WAITING - No threads were waiting <wq,event> pair */ kern_return_t wait_queue_wakeup_all( wait_queue_t wq, event_t event, wait_result_t result) { kern_return_t ret; spl_t s; if (!wait_queue_is_valid(wq)) { return KERN_INVALID_ARGUMENT; } s = splsched(); wait_queue_lock(wq); // if(!wq->wq_interlock.lock_data) { /* (BRINGUP */ // panic("wait_queue_wakeup_all: we did not get the lock on %p\n", wq); /* (BRINGUP) */ // } ret = wait_queue_wakeup64_all_locked( wq, CAST_DOWN(event64_t,event), result, TRUE); /* lock released */ splx(s); return ret; }
/* * Routine: wait_queue_wakeup_one * Purpose: * Wakeup the most appropriate thread that is in the specified * wait queue for the specified event. * Conditions: * Nothing locked * Returns: * KERN_SUCCESS - Thread was woken up * KERN_NOT_WAITING - No thread was waiting <wq,event> pair */ kern_return_t wait_queue_wakeup_one( wait_queue_t wq, event_t event, wait_result_t result) { thread_t thread; spl_t s; if (!wait_queue_is_valid(wq)) { return KERN_INVALID_ARGUMENT; } s = splsched(); wait_queue_lock(wq); thread = _wait_queue_select64_one(wq, (event64_t)((uint32_t)event)); wait_queue_unlock(wq); if (thread) { kern_return_t res; res = thread_go_locked(thread, result); assert(res == KERN_SUCCESS); thread_unlock(thread); splx(s); return res; } splx(s); return KERN_NOT_WAITING; }
/* * Routine: wait_queue_link_internal * Purpose: * Insert a set wait queue into a wait queue. This * requires us to link the two together using a wait_queue_link * structure that was provided. * Conditions: * The wait queue being inserted must be inited as a set queue * The wait_queue_link structure must already be properly typed */ static kern_return_t wait_queue_link_internal( wait_queue_t wq, wait_queue_set_t wq_set, wait_queue_link_t wql) { wait_queue_element_t wq_element; queue_t q; spl_t s; if (!wait_queue_is_valid(wq) || !wait_queue_is_set(wq_set)) return KERN_INVALID_ARGUMENT; /* * There are probably fewer threads and sets associated with * the wait queue than there are wait queues associated with * the set. So let's validate it that way. */ s = splsched(); wait_queue_lock(wq); q = &wq->wq_queue; wq_element = (wait_queue_element_t) queue_first(q); while (!queue_end(q, (queue_entry_t)wq_element)) { WAIT_QUEUE_ELEMENT_CHECK(wq, wq_element); if ((wq_element->wqe_type == WAIT_QUEUE_LINK || wq_element->wqe_type == WAIT_QUEUE_LINK_NOALLOC) && ((wait_queue_link_t)wq_element)->wql_setqueue == wq_set) { wait_queue_unlock(wq); splx(s); return KERN_ALREADY_IN_SET; } wq_element = (wait_queue_element_t) queue_next((queue_t) wq_element); } /* * Not already a member, so we can add it. */ wqs_lock(wq_set); WAIT_QUEUE_SET_CHECK(wq_set); assert(wql->wql_type == WAIT_QUEUE_LINK || wql->wql_type == WAIT_QUEUE_LINK_NOALLOC); wql->wql_queue = wq; wql_clear_prepost(wql); queue_enter(&wq->wq_queue, wql, wait_queue_link_t, wql_links); wql->wql_setqueue = wq_set; queue_enter(&wq_set->wqs_setlinks, wql, wait_queue_link_t, wql_setlinks); wqs_unlock(wq_set); wait_queue_unlock(wq); splx(s); return KERN_SUCCESS; }
kern_return_t wait_queue_unlink_all( wait_queue_t wq) { wait_queue_element_t wq_element; wait_queue_element_t wq_next_element; wait_queue_set_t wq_set; wait_queue_link_t wql; queue_head_t links_queue_head; queue_t links = &links_queue_head; queue_t q; spl_t s; if (!wait_queue_is_valid(wq)) { return KERN_INVALID_ARGUMENT; } queue_init(links); s = splsched(); wait_queue_lock(wq); q = &wq->wq_queue; wq_element = (wait_queue_element_t) queue_first(q); while (!queue_end(q, (queue_entry_t)wq_element)) { boolean_t alloced; WAIT_QUEUE_ELEMENT_CHECK(wq, wq_element); wq_next_element = (wait_queue_element_t) queue_next((queue_t) wq_element); alloced = (wq_element->wqe_type == WAIT_QUEUE_LINK); if (alloced || wq_element->wqe_type == WAIT_QUEUE_LINK_NOALLOC) { wql = (wait_queue_link_t)wq_element; wq_set = wql->wql_setqueue; wqs_lock(wq_set); wait_queue_unlink_locked(wq, wq_set, wql); wqs_unlock(wq_set); if (alloced) enqueue(links, &wql->wql_links); } wq_element = wq_next_element; } wait_queue_unlock(wq); splx(s); while(!queue_empty(links)) { wql = (wait_queue_link_t) dequeue(links); zfree(_wait_queue_link_zone, wql); } return(KERN_SUCCESS); }
/* * Routine: wait_queue_select64_thread * Purpose: * Look for a thread and remove it from the queues, if * (and only if) the thread is waiting on the supplied * <wait_queue, event> pair. * Conditions: * at splsched * wait queue locked * possibly recursive * Returns: * KERN_NOT_WAITING: Thread is not waiting here. * KERN_SUCCESS: It was, and is now removed (returned locked) */ static kern_return_t _wait_queue_select64_thread( wait_queue_t wq, event64_t event, thread_t thread) { wait_queue_element_t wq_element; wait_queue_element_t wqe_next; kern_return_t res = KERN_NOT_WAITING; queue_t q = &wq->wq_queue; thread_lock(thread); if ((thread->wait_queue == wq) && (thread->wait_event == event)) { remqueue(q, (queue_entry_t) thread); thread->at_safe_point = FALSE; thread->wait_event = NO_EVENT64; thread->wait_queue = WAIT_QUEUE_NULL; /* thread still locked */ return KERN_SUCCESS; } thread_unlock(thread); /* * The wait_queue associated with the thread may be one of this * wait queue's sets. Go see. If so, removing it from * there is like removing it from here. */ wq_element = (wait_queue_element_t) queue_first(q); while (!queue_end(q, (queue_entry_t)wq_element)) { WAIT_QUEUE_ELEMENT_CHECK(wq, wq_element); wqe_next = (wait_queue_element_t) queue_next((queue_t) wq_element); if (wq_element->wqe_type == WAIT_QUEUE_LINK) { wait_queue_link_t wql = (wait_queue_link_t)wq_element; wait_queue_t set_queue; set_queue = (wait_queue_t)wql->wql_setqueue; wait_queue_lock(set_queue); if (! wait_queue_empty(set_queue)) { res = _wait_queue_select64_thread(set_queue, event, thread); } wait_queue_unlock(set_queue); if (res == KERN_SUCCESS) return KERN_SUCCESS; } wq_element = wqe_next; } return res; }
kern_return_t wait_queue_unlink_all( wait_queue_t wq) { wait_queue_element_t wq_element; wait_queue_element_t wq_next_element; wait_queue_set_t wq_set; wait_queue_link_t wql; queue_head_t links_queue_head; queue_t links = &links_queue_head; queue_t q; spl_t s; if (!wait_queue_is_queue(wq)) { return KERN_INVALID_ARGUMENT; } queue_init(links); s = splsched(); wait_queue_lock(wq); q = &wq->wq_queue; wq_element = (wait_queue_element_t) queue_first(q); while (!queue_end(q, (queue_entry_t)wq_element)) { WAIT_QUEUE_ELEMENT_CHECK(wq, wq_element); wq_next_element = (wait_queue_element_t) queue_next((queue_t) wq_element); if (wq_element->wqe_type == WAIT_QUEUE_LINK) { wql = (wait_queue_link_t)wq_element; wq_set = wql->wql_setqueue; wqs_lock(wq_set); wait_queue_unlink_locked(wq, wq_set, wql); wqs_unlock(wq_set); enqueue(links, &wql->wql_links); } wq_element = wq_next_element; } wait_queue_unlock(wq); splx(s); while(!queue_empty(links)) { wql = (wait_queue_link_t) dequeue(links); kfree((vm_offset_t) wql, sizeof(struct wait_queue_link)); } return(KERN_SUCCESS); }
/* * Routine: wait_queue_unlink * Purpose: * Remove the linkage between a wait queue and a set, * freeing the linkage structure. * Conditions: * The wait queue being must be a member set queue */ kern_return_t wait_queue_unlink( wait_queue_t wq, wait_queue_set_t wq_set) { wait_queue_element_t wq_element; wait_queue_link_t wql; queue_t q; spl_t s; if (!wait_queue_is_valid(wq) || !wait_queue_is_set(wq_set)) { return KERN_INVALID_ARGUMENT; } s = splsched(); wait_queue_lock(wq); q = &wq->wq_queue; wq_element = (wait_queue_element_t) queue_first(q); while (!queue_end(q, (queue_entry_t)wq_element)) { WAIT_QUEUE_ELEMENT_CHECK(wq, wq_element); if (wq_element->wqe_type == WAIT_QUEUE_LINK || wq_element->wqe_type == WAIT_QUEUE_LINK_NOALLOC) { wql = (wait_queue_link_t)wq_element; if (wql->wql_setqueue == wq_set) { boolean_t alloced; alloced = (wql->wql_type == WAIT_QUEUE_LINK); wqs_lock(wq_set); wait_queue_unlink_locked(wq, wq_set, wql); wqs_unlock(wq_set); wait_queue_unlock(wq); splx(s); if (alloced) zfree(_wait_queue_link_zone, wql); return KERN_SUCCESS; } } wq_element = (wait_queue_element_t) queue_next((queue_t) wq_element); } wait_queue_unlock(wq); splx(s); return KERN_NOT_IN_SET; }
/* * Routine: wait_queue_member * Purpose: * Indicate if this set queue is a member of the queue * Conditions: * The set queue is just that, a set queue */ boolean_t wait_queue_member( wait_queue_t wq, wait_queue_set_t wq_set) { boolean_t ret; spl_t s; if (!wait_queue_is_set(wq_set)) return FALSE; s = splsched(); wait_queue_lock(wq); ret = wait_queue_member_locked(wq, wq_set); wait_queue_unlock(wq); splx(s); return ret; }
/* * Routine: wait_queue_wakeup64_all * Purpose: * Wakeup some number of threads that are in the specified * wait queue and waiting on the specified event. * Conditions: * Nothing locked * Returns: * KERN_SUCCESS - Threads were woken up * KERN_NOT_WAITING - No threads were waiting <wq,event> pair */ kern_return_t wait_queue_wakeup64_all( wait_queue_t wq, event64_t event, wait_result_t result) { kern_return_t ret; spl_t s; if (!wait_queue_is_valid(wq)) { return KERN_INVALID_ARGUMENT; } s = splsched(); wait_queue_lock(wq); ret = wait_queue_wakeup64_all_locked(wq, event, result, TRUE); /* lock released */ splx(s); return ret; }
/* * Routine: wait_queue_wakeup_one * Purpose: * Wakeup the most appropriate thread that is in the specified * wait queue for the specified event. * Conditions: * Nothing locked * Returns: * KERN_SUCCESS - Thread was woken up * KERN_NOT_WAITING - No thread was waiting <wq,event> pair */ kern_return_t wait_queue_wakeup_one( wait_queue_t wq, event_t event, wait_result_t result, int priority) { thread_t thread; spl_t s; if (!wait_queue_is_valid(wq)) { return KERN_INVALID_ARGUMENT; } s = splsched(); wait_queue_lock(wq); thread = _wait_queue_select64_one(wq, CAST_DOWN(event64_t,event)); wait_queue_unlock(wq); if (thread) { kern_return_t res; if (thread->sched_pri < priority) { if (priority <= MAXPRI) { set_sched_pri(thread, priority); thread->was_promoted_on_wakeup = 1; thread->sched_flags |= TH_SFLAG_PROMOTED; } } res = thread_go(thread, result); assert(res == KERN_SUCCESS); thread_unlock(thread); splx(s); return res; } splx(s); return KERN_NOT_WAITING; }
/* * Routine: wait_queue_unlink * Purpose: * Remove the linkage between a wait queue and a set, * freeing the linkage structure. * Conditions: * The wait queue being must be a member set queue */ kern_return_t wait_queue_unlink( wait_queue_t wq, wait_queue_set_t wq_set) { wait_queue_element_t wq_element; wait_queue_link_t wql; queue_t q; spl_t s; if (!wait_queue_is_queue(wq) || !wait_queue_is_set(wq_set)) { return KERN_INVALID_ARGUMENT; } s = splsched(); wait_queue_lock(wq); q = &wq->wq_queue; wq_element = (wait_queue_element_t) queue_first(q); while (!queue_end(q, (queue_entry_t)wq_element)) { WAIT_QUEUE_ELEMENT_CHECK(wq, wq_element); if (wq_element->wqe_type == WAIT_QUEUE_LINK) { wql = (wait_queue_link_t)wq_element; if (wql->wql_setqueue == wq_set) { wqs_lock(wq_set); wait_queue_unlink_locked(wq, wq_set, wql); wqs_unlock(wq_set); wait_queue_unlock(wq); splx(s); kfree((vm_offset_t)wql, sizeof(struct wait_queue_link)); return KERN_SUCCESS; } } wq_element = (wait_queue_element_t) queue_next((queue_t) wq_element); } wait_queue_unlock(wq); splx(s); return KERN_NOT_IN_SET; }
/* * Routine: wait_queue_unlink_one * Purpose: * Find and unlink one set wait queue * Conditions: * Nothing of interest locked. */ void wait_queue_unlink_one( wait_queue_t wq, wait_queue_set_t *wq_setp) { wait_queue_element_t wq_element; queue_t q; spl_t s; s = splsched(); wait_queue_lock(wq); q = &wq->wq_queue; wq_element = (wait_queue_element_t) queue_first(q); while (!queue_end(q, (queue_entry_t)wq_element)) { if (wq_element->wqe_type == WAIT_QUEUE_LINK) { wait_queue_link_t wql = (wait_queue_link_t)wq_element; wait_queue_set_t wq_set = wql->wql_setqueue; wqs_lock(wq_set); wait_queue_unlink_locked(wq, wq_set, wql); wqs_unlock(wq_set); wait_queue_unlock(wq); splx(s); kfree((vm_offset_t)wql,sizeof(struct wait_queue_link)); *wq_setp = wq_set; return; } wq_element = (wait_queue_element_t) queue_next((queue_t) wq_element); } wait_queue_unlock(wq); splx(s); *wq_setp = WAIT_QUEUE_SET_NULL; }
/* * Routine: wait_queue_peek64_locked * Purpose: * Select the best thread from a wait queue that meet the * supplied criteria, but leave it on the queue it was * found on. The thread, and the actual wait_queue the * thread was found on are identified. * Conditions: * at splsched * wait queue locked * possibly recursive * Returns: * a locked thread - if one found * a locked waitq - the one the thread was found on * Note: * Both the waitq the thread was actually found on, and * the supplied wait queue, are locked after this. */ __private_extern__ void wait_queue_peek64_locked( wait_queue_t wq, event64_t event, thread_t *tp, wait_queue_t *wqp) { wait_queue_element_t wq_element; wait_queue_element_t wqe_next; thread_t t; queue_t q; assert(wq->wq_fifo); *tp = THREAD_NULL; q = &wq->wq_queue; wq_element = (wait_queue_element_t) queue_first(q); while (!queue_end(q, (queue_entry_t)wq_element)) { WAIT_QUEUE_ELEMENT_CHECK(wq, wq_element); wqe_next = (wait_queue_element_t) queue_next((queue_t) wq_element); /* * We may have to recurse if this is a compound wait queue. */ if (wq_element->wqe_type == WAIT_QUEUE_LINK) { wait_queue_link_t wql = (wait_queue_link_t)wq_element; wait_queue_t set_queue; /* * We have to check the set wait queue. */ set_queue = (wait_queue_t)wql->wql_setqueue; wait_queue_lock(set_queue); if (! wait_queue_empty(set_queue)) { wait_queue_peek64_locked(set_queue, event, tp, wqp); } if (*tp != THREAD_NULL) { if (*wqp != set_queue) wait_queue_unlock(set_queue); return; /* thread and its waitq locked */ } wait_queue_unlock(set_queue); } else { /* * Otherwise, its a thread. If it is waiting on * the event we are posting to this queue, return * it locked, but leave it on the queue. */ thread_t t = (thread_t)wq_element; if (t->wait_event == event) { thread_lock(t); *tp = t; *wqp = wq; return; } } wq_element = wqe_next; } }
/* * ROUTINE: ulock_release_internal [internal] * * Releases the ulock. * If any threads are blocked waiting for the ulock, one is woken-up. * */ kern_return_t ulock_release_internal (ulock_t ulock, thread_t thread) { lock_set_t lock_set; if ((lock_set = ulock->lock_set) == LOCK_SET_NULL) return KERN_INVALID_ARGUMENT; lock_set_lock(lock_set); if (!lock_set->active) { lock_set_unlock(lock_set); return KERN_LOCK_SET_DESTROYED; } ulock_lock(ulock); lock_set_unlock(lock_set); if (ulock->holder != thread) { ulock_unlock(ulock); return KERN_INVALID_RIGHT; } /* * If we have a hint that threads might be waiting, * try to transfer the lock ownership to a waiting thread * and wake it up. */ if (ulock->blocked) { wait_queue_t wq = &ulock->wait_queue; thread_t wqthread; spl_t s; s = splsched(); wait_queue_lock(wq); wqthread = wait_queue_wakeup64_identity_locked(wq, LOCK_SET_EVENT, THREAD_AWAKENED, TRUE); /* wait_queue now unlocked, thread locked */ if (wqthread != THREAD_NULL) { thread_unlock(wqthread); splx(s); /* * Transfer ulock ownership * from the current thread to the acquisition thread. */ ulock_ownership_clear(ulock); ulock_ownership_set(ulock, wqthread); ulock_unlock(ulock); return KERN_SUCCESS; } else { ulock->blocked = FALSE; splx(s); } } /* * Disown ulock */ ulock_ownership_clear(ulock); ulock_unlock(ulock); return KERN_SUCCESS; }
/* * Routine: _wait_queue_select64_all * Purpose: * Select all threads off a wait queue that meet the * supplied criteria. * Conditions: * at splsched * wait queue locked * wake_queue initialized and ready for insertion * possibly recursive * Returns: * a queue of locked threads */ static void _wait_queue_select64_all( wait_queue_t wq, event64_t event, queue_t wake_queue) { wait_queue_element_t wq_element; wait_queue_element_t wqe_next; queue_t q; q = &wq->wq_queue; wq_element = (wait_queue_element_t) queue_first(q); while (!queue_end(q, (queue_entry_t)wq_element)) { WAIT_QUEUE_ELEMENT_CHECK(wq, wq_element); wqe_next = (wait_queue_element_t) queue_next((queue_t) wq_element); /* * We may have to recurse if this is a compound wait queue. */ if (wq_element->wqe_type == WAIT_QUEUE_LINK) { wait_queue_link_t wql = (wait_queue_link_t)wq_element; wait_queue_t set_queue; /* * We have to check the set wait queue. */ set_queue = (wait_queue_t)wql->wql_setqueue; wait_queue_lock(set_queue); if (set_queue->wq_isprepost) { wait_queue_set_t wqs = (wait_queue_set_t)set_queue; /* * Preposting is only for sets and wait queue * is the first element of set */ wqs->wqs_refcount++; } if (! wait_queue_empty(set_queue)) _wait_queue_select64_all(set_queue, event, wake_queue); wait_queue_unlock(set_queue); } else { /* * Otherwise, its a thread. If it is waiting on * the event we are posting to this queue, pull * it off the queue and stick it in out wake_queue. */ thread_t t = (thread_t)wq_element; if (t->wait_event == event) { thread_lock(t); remqueue(q, (queue_entry_t) t); enqueue (wake_queue, (queue_entry_t) t); t->wait_queue = WAIT_QUEUE_NULL; t->wait_event = NO_EVENT64; t->at_safe_point = FALSE; /* returned locked */ } } wq_element = wqe_next; } }
/* * ROUTINE: ulock_release_internal [internal] * * Releases the ulock. * If any threads are blocked waiting for the ulock, one is woken-up. * */ kern_return_t ulock_release_internal (ulock_t ulock, thread_t thread) { lock_set_t lock_set; if ((lock_set = ulock->lock_set) == LOCK_SET_NULL) return KERN_INVALID_ARGUMENT; lock_set_lock(lock_set); if (!lock_set->active) { lock_set_unlock(lock_set); return KERN_LOCK_SET_DESTROYED; } ulock_lock(ulock); lock_set_unlock(lock_set); if (ulock->holder != thread) { ulock_unlock(ulock); return KERN_INVALID_RIGHT; } /* * If we have a hint that threads might be waiting, * try to transfer the lock ownership to a waiting thread * and wake it up. */ if (ulock->blocked) { wait_queue_t wq = &ulock->wait_queue; thread_t wqthread; spl_t s; s = splsched(); wait_queue_lock(wq); wqthread = wait_queue_wakeup64_identity_locked(wq, LOCK_SET_EVENT, THREAD_AWAKENED, TRUE); /* wait_queue now unlocked, thread locked */ if (wqthread != THREAD_NULL) { /* * JMM - These ownership transfer macros have a * locking/race problem. To keep the thread from * changing states on us (nullifying the ownership * assignment) we need to keep the thread locked * during the assignment. But we can't because the * macros take an activation lock, which is a mutex. * Since this code was already broken before I got * here, I will leave it for now. */ thread_unlock(wqthread); splx(s); /* * Transfer ulock ownership * from the current thread to the acquisition thread. */ ulock_ownership_clear(ulock); ulock_ownership_set(ulock, wqthread); ulock_unlock(ulock); return KERN_SUCCESS; } else { ulock->blocked = FALSE; splx(s); } } /* * Disown ulock */ ulock_ownership_clear(ulock); ulock_unlock(ulock); return KERN_SUCCESS; }
kern_return_t lock_handoff (lock_set_t lock_set, int lock_id) { ulock_t ulock; int wait_result; if (lock_set == LOCK_SET_NULL) return KERN_INVALID_ARGUMENT; if (lock_id < 0 || lock_id >= lock_set->n_ulocks) return KERN_INVALID_ARGUMENT; retry: lock_set_lock(lock_set); if (!lock_set->active) { lock_set_unlock(lock_set); return KERN_LOCK_SET_DESTROYED; } ulock = (ulock_t) &lock_set->ulock_list[lock_id]; ulock_lock(ulock); lock_set_unlock(lock_set); if (ulock->holder != current_thread()) { ulock_unlock(ulock); return KERN_INVALID_RIGHT; } /* * If the accepting thread (the receiver) is already waiting * to accept the lock from the handoff thread (the sender), * then perform the hand-off now. */ if (ulock->accept_wait) { wait_queue_t wq = &ulock->wait_queue; thread_t thread; spl_t s; /* * See who the lucky devil is, if he is still there waiting. */ s = splsched(); wait_queue_lock(wq); thread = wait_queue_wakeup64_identity_locked( wq, LOCK_SET_HANDOFF, THREAD_AWAKENED, TRUE); /* wait queue unlocked, thread locked */ /* * Transfer lock ownership */ if (thread != THREAD_NULL) { /* * The thread we are transferring to will try * to take the lock on the ulock, and therefore * will wait for us complete the handoff even * through we set the thread running. */ thread_unlock(thread); splx(s); ulock_ownership_clear(ulock); ulock_ownership_set(ulock, thread); ulock->accept_wait = FALSE; ulock_unlock(ulock); return KERN_SUCCESS; } else { /* * OOPS. The accepting thread must have been aborted. * and is racing back to clear the flag that says is * waiting for an accept. He will clear it when we * release the lock, so just fall thru and wait for * the next accept thread (that's the way it is * specified). */ splx(s); } } /* * Indicate that there is a hand-off thread waiting, and then wait * for an accepting thread. */ ulock->ho_wait = TRUE; wait_result = wait_queue_assert_wait64(&ulock->wait_queue, LOCK_SET_HANDOFF, THREAD_ABORTSAFE, 0); ulock_unlock(ulock); if (wait_result == THREAD_WAITING) wait_result = thread_block(THREAD_CONTINUE_NULL); /* * If the thread was woken-up via some action other than * lock_handoff_accept or lock_set_destroy (i.e. thread_terminate), * then we need to clear the ulock's handoff state. */ switch (wait_result) { case THREAD_AWAKENED: /* * we take the ulock lock to syncronize with the * thread that is accepting ownership. */ ulock_lock(ulock); assert(ulock->holder != current_thread()); ulock_unlock(ulock); return KERN_SUCCESS; case THREAD_INTERRUPTED: ulock_lock(ulock); assert(ulock->holder == current_thread()); ulock->ho_wait = FALSE; ulock_unlock(ulock); return KERN_ABORTED; case THREAD_RESTART: goto retry; } panic("lock_handoff"); return KERN_FAILURE; }