/* * Routine: wait_queue_unlink_locked * Purpose: * Undo the linkage between a wait queue and a set. */ static void wait_queue_unlink_locked( wait_queue_t wq, wait_queue_set_t wq_set, wait_queue_link_t wql) { assert(wait_queue_held(wq)); assert(wait_queue_held(&wq_set->wqs_wait_queue)); wql->wql_queue = WAIT_QUEUE_NULL; queue_remove(&wq->wq_queue, wql, wait_queue_link_t, wql_links); wql->wql_setqueue = WAIT_QUEUE_SET_NULL; queue_remove(&wq_set->wqs_setlinks, wql, wait_queue_link_t, wql_setlinks); if (wql_is_preposted(wql)) { queue_t ppq = &wq_set->wqs_preposts; queue_remove(ppq, wql, wait_queue_link_t, wql_preposts); } wql->wql_type = WAIT_QUEUE_UNLINKED; WAIT_QUEUE_CHECK(wq); WAIT_QUEUE_SET_CHECK(wq_set); }
kern_return_t wait_queue_sub_clearrefs( wait_queue_set_t wq_set) { wait_queue_link_t wql; queue_t q; spl_t s; if (!wait_queue_is_set(wq_set)) return KERN_INVALID_ARGUMENT; s = splsched(); wqs_lock(wq_set); q = &wq_set->wqs_preposts; while (!queue_empty(q)) { queue_remove_first(q, wql, wait_queue_link_t, wql_preposts); assert(!wql_is_preposted(wql)); } wqs_unlock(wq_set); splx(s); return KERN_SUCCESS; }
/* * Routine: _wait_queue_select64_all * Purpose: * Select all threads off a wait queue that meet the * supplied criteria. * Conditions: * at splsched * wait queue locked * wake_queue initialized and ready for insertion * possibly recursive * Returns: * a queue of locked threads */ static void _wait_queue_select64_all( wait_queue_t wq, event64_t event, queue_t wake_queue) { wait_queue_element_t wq_element; wait_queue_element_t wqe_next; queue_t q; q = &wq->wq_queue; wq_element = (wait_queue_element_t) queue_first(q); while (!queue_end(q, (queue_entry_t)wq_element)) { WAIT_QUEUE_ELEMENT_CHECK(wq, wq_element); wqe_next = (wait_queue_element_t) queue_next((queue_t) wq_element); /* * We may have to recurse if this is a compound wait queue. */ if (wq_element->wqe_type == WAIT_QUEUE_LINK || wq_element->wqe_type == WAIT_QUEUE_LINK_NOALLOC) { wait_queue_link_t wql = (wait_queue_link_t)wq_element; wait_queue_set_t set_queue = wql->wql_setqueue; /* * We have to check the set wait queue. If it is marked * as pre-post, and it is the "generic event" then mark * it pre-posted now (if not already). */ wqs_lock(set_queue); if (event == NO_EVENT64 && set_queue->wqs_prepost && !wql_is_preposted(wql)) { queue_t ppq = &set_queue->wqs_preposts; queue_enter(ppq, wql, wait_queue_link_t, wql_preposts); } if (! wait_queue_empty(&set_queue->wqs_wait_queue)) _wait_queue_select64_all(&set_queue->wqs_wait_queue, event, wake_queue); wqs_unlock(set_queue); } else { /* * Otherwise, its a thread. If it is waiting on * the event we are posting to this queue, pull * it off the queue and stick it in out wake_queue. */ thread_t t = (thread_t)wq_element; if (t->wait_event == event) { thread_lock(t); remqueue((queue_entry_t) t); enqueue (wake_queue, (queue_entry_t) t); t->wait_queue = WAIT_QUEUE_NULL; t->wait_event = NO_EVENT64; t->at_safe_point = FALSE; /* returned locked */ } } wq_element = wqe_next; } }
/* * Routine: _wait_queue_select64_one * Purpose: * Select the best thread off a wait queue that meet the * supplied criteria. * Conditions: * at splsched * wait queue locked * possibly recursive * Returns: * a locked thread - if one found * Note: * This is where the sync policy of the wait queue comes * into effect. For now, we just assume FIFO/LIFO. */ static thread_t _wait_queue_select64_one( wait_queue_t wq, event64_t event) { wait_queue_element_t wq_element; wait_queue_element_t wqe_next; thread_t t = THREAD_NULL; queue_t q; q = &wq->wq_queue; wq_element = (wait_queue_element_t) queue_first(q); while (!queue_end(q, (queue_entry_t)wq_element)) { WAIT_QUEUE_ELEMENT_CHECK(wq, wq_element); wqe_next = (wait_queue_element_t) queue_next((queue_t) wq_element); /* * We may have to recurse if this is a compound wait queue. */ if (wq_element->wqe_type == WAIT_QUEUE_LINK || wq_element->wqe_type == WAIT_QUEUE_LINK_NOALLOC) { wait_queue_link_t wql = (wait_queue_link_t)wq_element; wait_queue_set_t set_queue = wql->wql_setqueue; /* * We have to check the set wait queue. If the set * supports pre-posting, it isn't already preposted, * and we didn't find a thread in the set, then mark it. * * If we later find a thread, there may be a spurious * pre-post here on this set. The wait side has to check * for that either pre- or post-wait. */ wqs_lock(set_queue); if (! wait_queue_empty(&set_queue->wqs_wait_queue)) { t = _wait_queue_select64_one(&set_queue->wqs_wait_queue, event); } if (t != THREAD_NULL) { wqs_unlock(set_queue); return t; } if (event == NO_EVENT64 && set_queue->wqs_prepost && !wql_is_preposted(wql)) { queue_t ppq = &set_queue->wqs_preposts; queue_enter(ppq, wql, wait_queue_link_t, wql_preposts); } wqs_unlock(set_queue); } else { /* * Otherwise, its a thread. If it is waiting on * the event we are posting to this queue, pull * it off the queue and stick it in out wake_queue. */ t = (thread_t)wq_element; if (t->wait_event == event) { thread_lock(t); remqueue((queue_entry_t) t); t->wait_queue = WAIT_QUEUE_NULL; t->wait_event = NO_EVENT64; t->at_safe_point = FALSE; return t; /* still locked */ } t = THREAD_NULL; } wq_element = wqe_next; } return THREAD_NULL; }
wait_result_t ipc_mqueue_receive_on_thread( ipc_mqueue_t mqueue, mach_msg_option_t option, mach_msg_size_t max_size, mach_msg_timeout_t rcv_timeout, int interruptible, thread_t thread) { ipc_kmsg_queue_t kmsgs; wait_result_t wresult; uint64_t deadline; spl_t s; s = splsched(); imq_lock(mqueue); if (imq_is_set(mqueue)) { queue_t q; q = &mqueue->imq_preposts; /* * If we are waiting on a portset mqueue, we need to see if * any of the member ports have work for us. Ports that * have (or recently had) messages will be linked in the * prepost queue for the portset. By holding the portset's * mqueue lock during the search, we tie up any attempts by * mqueue_deliver or portset membership changes that may * cross our path. */ search_set: while(!queue_empty(q)) { wait_queue_link_t wql; ipc_mqueue_t port_mq; queue_remove_first(q, wql, wait_queue_link_t, wql_preposts); assert(!wql_is_preposted(wql)); /* * This is a lock order violation, so we have to do it * "softly," putting the link back on the prepost list * if it fails (at the tail is fine since the order of * handling messages from different sources in a set is * not guaranteed and we'd like to skip to the next source * if one is available). */ port_mq = (ipc_mqueue_t)wql->wql_queue; if (!imq_lock_try(port_mq)) { queue_enter(q, wql, wait_queue_link_t, wql_preposts); imq_unlock(mqueue); splx(s); mutex_pause(0); s = splsched(); imq_lock(mqueue); goto search_set; /* start again at beginning - SMP */ } /* * If there are no messages on this queue, just skip it * (we already removed the link from the set's prepost queue). */ kmsgs = &port_mq->imq_messages; if (ipc_kmsg_queue_first(kmsgs) == IKM_NULL) { imq_unlock(port_mq); continue; } /* * There are messages, so reinsert the link back * at the tail of the preposted queue (for fairness) * while we still have the portset mqueue locked. */ queue_enter(q, wql, wait_queue_link_t, wql_preposts); imq_unlock(mqueue); /* * Continue on to handling the message with just * the port mqueue locked. */ ipc_mqueue_select_on_thread(port_mq, option, max_size, thread); imq_unlock(port_mq); splx(s); return THREAD_NOT_WAITING; } } else { /* * Receive on a single port. Just try to get the messages. */ kmsgs = &mqueue->imq_messages; if (ipc_kmsg_queue_first(kmsgs) != IKM_NULL) { ipc_mqueue_select_on_thread(mqueue, option, max_size, thread); imq_unlock(mqueue); splx(s); return THREAD_NOT_WAITING; } } /* * Looks like we'll have to block. The mqueue we will * block on (whether the set's or the local port's) is * still locked. */ if (option & MACH_RCV_TIMEOUT) { if (rcv_timeout == 0) { imq_unlock(mqueue); splx(s); thread->ith_state = MACH_RCV_TIMED_OUT; return THREAD_NOT_WAITING; } } thread_lock(thread); thread->ith_state = MACH_RCV_IN_PROGRESS; thread->ith_option = option; thread->ith_msize = max_size; if (option & MACH_RCV_TIMEOUT) clock_interval_to_deadline(rcv_timeout, 1000*NSEC_PER_USEC, &deadline); else deadline = 0; wresult = wait_queue_assert_wait64_locked(&mqueue->imq_wait_queue, IPC_MQUEUE_RECEIVE, interruptible, TIMEOUT_URGENCY_USER_NORMAL, deadline, 0, thread); /* preposts should be detected above, not here */ if (wresult == THREAD_AWAKENED) panic("ipc_mqueue_receive_on_thread: sleep walking"); thread_unlock(thread); imq_unlock(mqueue); splx(s); return wresult; }