void host_notify_port_destroy( ipc_port_t port) { host_notify_t entry; mutex_lock(&host_notify_lock); ip_lock(port); if (ip_kotype(port) == IKOT_HOST_NOTIFY) { entry = (host_notify_t)port->ip_kobject; assert(entry != NULL); ipc_kobject_set_atomically(port, IKO_NULL, IKOT_NONE); ip_unlock(port); assert(entry->port == port); remqueue(NULL, (queue_entry_t)entry); mutex_unlock(&host_notify_lock); zfree(host_notify_zone, (vm_offset_t)entry); ipc_port_release_sonce(port); return; } ip_unlock(port); mutex_unlock(&host_notify_lock); }
/* * Routine: _wait_queue_select64_one * Purpose: * Select the best thread off a wait queue that meet the * supplied criteria. * Conditions: * at splsched * wait queue locked * possibly recursive * Returns: * a locked thread - if one found * Note: * This is where the sync policy of the wait queue comes * into effect. For now, we just assume FIFO. */ static thread_t _wait_queue_select64_one( wait_queue_t wq, event64_t event) { wait_queue_element_t wq_element; wait_queue_element_t wqe_next; thread_t t = THREAD_NULL; queue_t q; assert(wq->wq_fifo); q = &wq->wq_queue; wq_element = (wait_queue_element_t) queue_first(q); while (!queue_end(q, (queue_entry_t)wq_element)) { WAIT_QUEUE_ELEMENT_CHECK(wq, wq_element); wqe_next = (wait_queue_element_t) queue_next((queue_t) wq_element); /* * We may have to recurse if this is a compound wait queue. */ if (wq_element->wqe_type == WAIT_QUEUE_LINK) { wait_queue_link_t wql = (wait_queue_link_t)wq_element; wait_queue_t set_queue; /* * We have to check the set wait queue. */ set_queue = (wait_queue_t)wql->wql_setqueue; wait_queue_lock(set_queue); if (! wait_queue_empty(set_queue)) { t = _wait_queue_select64_one(set_queue, event); } wait_queue_unlock(set_queue); if (t != THREAD_NULL) return t; } else { /* * Otherwise, its a thread. If it is waiting on * the event we are posting to this queue, pull * it off the queue and stick it in out wake_queue. */ thread_t t = (thread_t)wq_element; if (t->wait_event == event) { thread_lock(t); remqueue(q, (queue_entry_t) t); t->wait_queue = WAIT_QUEUE_NULL; t->wait_event = NO_EVENT64; t->at_safe_point = FALSE; return t; /* still locked */ } } wq_element = wqe_next; } return THREAD_NULL; }
/* * Routine: semaphore_destroy * * Destroys a semaphore. This call will only succeed if the * specified task is the SAME task name specified at the semaphore's * creation. * * All threads currently blocked on the semaphore are awoken. These * threads will return with the KERN_TERMINATED error. */ kern_return_t semaphore_destroy( task_t task, semaphore_t semaphore) { int old_count; spl_t spl_level; if (task == TASK_NULL || semaphore == SEMAPHORE_NULL) return KERN_INVALID_ARGUMENT; /* * Disown semaphore */ task_lock(task); if (semaphore->owner != task) { task_unlock(task); return KERN_INVALID_ARGUMENT; } remqueue(&task->semaphore_list, (queue_entry_t) semaphore); semaphore->owner = TASK_NULL; task->semaphores_owned--; task_unlock(task); spl_level = splsched(); semaphore_lock(semaphore); /* * Deactivate semaphore */ assert(semaphore->active); semaphore->active = FALSE; /* * Wakeup blocked threads */ old_count = semaphore->count; semaphore->count = 0; if (old_count < 0) { wait_queue_wakeup64_all_locked(&semaphore->wait_queue, SEMAPHORE_EVENT, THREAD_RESTART, TRUE); /* unlock? */ } else { semaphore_unlock(semaphore); } splx(spl_level); /* * Deallocate * * Drop the semaphore reference, which in turn deallocates the * semaphore structure if the reference count goes to zero. */ ipc_port_dealloc_kernel(semaphore->port); semaphore_dereference(semaphore); return KERN_SUCCESS; }
/* * sched_traditional_processor_queue_shutdown: * * Shutdown a processor run queue by * re-dispatching non-bound threads. * * Associated pset must be locked, and is * returned unlocked. */ static void sched_traditional_processor_queue_shutdown(processor_t processor) { processor_set_t pset = processor->processor_set; run_queue_t rq = runq_for_processor(processor); queue_t queue = rq->queues + rq->highq; int pri = rq->highq; int count = rq->count; thread_t next, thread; queue_head_t tqueue; queue_init(&tqueue); while (count > 0) { thread = (thread_t)(uintptr_t)queue_first(queue); while (!queue_end(queue, (queue_entry_t)thread)) { next = (thread_t)(uintptr_t)queue_next((queue_entry_t)thread); if (thread->bound_processor == PROCESSOR_NULL) { remqueue((queue_entry_t)thread); thread->runq = PROCESSOR_NULL; SCHED_STATS_RUNQ_CHANGE(&rq->runq_stats, rq->count); runq_consider_decr_bound_count(processor, thread); rq->count--; if (SCHED(priority_is_urgent)(pri)) { rq->urgency--; assert(rq->urgency >= 0); } if (queue_empty(queue)) { bitmap_clear(rq->bitmap, pri); rq->highq = bitmap_first(rq->bitmap, NRQS); } enqueue_tail(&tqueue, (queue_entry_t)thread); } count--; thread = next; } queue--; pri--; } pset_unlock(pset); while ((thread = (thread_t)(uintptr_t)dequeue_head(&tqueue)) != THREAD_NULL) { thread_lock(thread); thread_setrun(thread, SCHED_TAILQ); thread_unlock(thread); } }
/* * Routine: wait_queue_select64_thread * Purpose: * Look for a thread and remove it from the queues, if * (and only if) the thread is waiting on the supplied * <wait_queue, event> pair. * Conditions: * at splsched * wait queue locked * possibly recursive * Returns: * KERN_NOT_WAITING: Thread is not waiting here. * KERN_SUCCESS: It was, and is now removed (returned locked) */ static kern_return_t _wait_queue_select64_thread( wait_queue_t wq, event64_t event, thread_t thread) { wait_queue_element_t wq_element; wait_queue_element_t wqe_next; kern_return_t res = KERN_NOT_WAITING; queue_t q = &wq->wq_queue; thread_lock(thread); if ((thread->wait_queue == wq) && (thread->wait_event == event)) { remqueue((queue_entry_t) thread); thread->at_safe_point = FALSE; thread->wait_event = NO_EVENT64; thread->wait_queue = WAIT_QUEUE_NULL; /* thread still locked */ return KERN_SUCCESS; } thread_unlock(thread); /* * The wait_queue associated with the thread may be one of this * wait queue's sets. Go see. If so, removing it from * there is like removing it from here. */ wq_element = (wait_queue_element_t) queue_first(q); while (!queue_end(q, (queue_entry_t)wq_element)) { WAIT_QUEUE_ELEMENT_CHECK(wq, wq_element); wqe_next = (wait_queue_element_t) queue_next((queue_t) wq_element); if (wq_element->wqe_type == WAIT_QUEUE_LINK || wq_element->wqe_type == WAIT_QUEUE_LINK_NOALLOC) { wait_queue_link_t wql = (wait_queue_link_t)wq_element; wait_queue_set_t set_queue = wql->wql_setqueue; wqs_lock(set_queue); if (! wait_queue_empty(&set_queue->wqs_wait_queue)) { res = _wait_queue_select64_thread(&set_queue->wqs_wait_queue, event, thread); } wqs_unlock(set_queue); if (res == KERN_SUCCESS) return KERN_SUCCESS; } wq_element = wqe_next; } return res; }
/* * Routine: wait_queue_pull_thread_locked * Purpose: * Pull a thread off its wait queue and (possibly) unlock * the waitq. * Conditions: * at splsched * wait queue locked * thread locked * Returns: * with the thread still locked. */ void wait_queue_pull_thread_locked( wait_queue_t waitq, thread_t thread, boolean_t unlock) { assert(thread->wait_queue == waitq); remqueue((queue_entry_t)thread ); thread->wait_queue = WAIT_QUEUE_NULL; thread->wait_event = NO_EVENT64; thread->at_safe_point = FALSE; if (unlock) wait_queue_unlock(waitq); }
/* * Port-death routine to clean up reply messages. */ boolean_t tty_queue_clean( queue_t q, ipc_port_t port, boolean_t (*routine)(io_req_t) ) { register io_req_t ior; ior = (io_req_t)queue_first(q); while (!queue_end(q, (queue_entry_t)ior)) { if (ior->io_reply_port == port) { remqueue(q, (queue_entry_t)ior); ior->io_done = routine; iodone(ior); return TRUE; } ior = ior->io_next; } return FALSE; }
/* * sched_traditional_choose_thread_from_runq: * * Locate a thread to execute from the processor run queue * and return it. Only choose a thread with greater or equal * priority. * * Associated pset must be locked. Returns THREAD_NULL * on failure. */ static thread_t sched_traditional_choose_thread_from_runq( processor_t processor, run_queue_t rq, int priority) { queue_t queue = rq->queues + rq->highq; int pri = rq->highq; int count = rq->count; thread_t thread; while (count > 0 && pri >= priority) { thread = (thread_t)(uintptr_t)queue_first(queue); while (!queue_end(queue, (queue_entry_t)thread)) { if (thread->bound_processor == PROCESSOR_NULL || thread->bound_processor == processor) { remqueue((queue_entry_t)thread); thread->runq = PROCESSOR_NULL; SCHED_STATS_RUNQ_CHANGE(&rq->runq_stats, rq->count); rq->count--; if (SCHED(priority_is_urgent)(pri)) { rq->urgency--; assert(rq->urgency >= 0); } if (queue_empty(queue)) { bitmap_clear(rq->bitmap, pri); rq->highq = bitmap_first(rq->bitmap, NRQS); } return (thread); } count--; thread = (thread_t)(uintptr_t)queue_next((queue_entry_t)thread); } queue--; pri--; } return (THREAD_NULL); }
/* * Locate and steal a thread, beginning * at the pset. * * The pset must be locked, and is returned * unlocked. * * Returns the stolen thread, or THREAD_NULL on * failure. */ static thread_t sched_traditional_steal_thread(processor_set_t pset) { processor_set_t nset, cset = pset; processor_t processor; thread_t thread; do { processor = (processor_t)(uintptr_t)queue_first(&cset->active_queue); while (!queue_end(&cset->active_queue, (queue_entry_t)processor)) { if (runq_for_processor(processor)->count > 0) { thread = sched_traditional_steal_processor_thread(processor); if (thread != THREAD_NULL) { remqueue((queue_entry_t)processor); enqueue_tail(&cset->active_queue, (queue_entry_t)processor); pset_unlock(cset); return (thread); } } processor = (processor_t)(uintptr_t)queue_next((queue_entry_t)processor); } nset = next_pset(cset); if (nset != pset) { pset_unlock(cset); cset = nset; pset_lock(cset); } } while (nset != pset); pset_unlock(cset); return (THREAD_NULL); }
/* * Routine: semaphore_destroy_internal * * Disassociate a semaphore from its owning task, mark it inactive, * and set any waiting threads running with THREAD_RESTART. * * Conditions: * task is locked * semaphore is locked * semaphore is owned by the specified task * Returns: * with semaphore unlocked */ static void semaphore_destroy_internal( task_t task, semaphore_t semaphore) { int old_count; /* unlink semaphore from owning task */ assert(semaphore->owner == task); remqueue((queue_entry_t) semaphore); semaphore->owner = TASK_NULL; task->semaphores_owned--; /* * Deactivate semaphore */ assert(semaphore->active); semaphore->active = FALSE; /* * Wakeup blocked threads */ old_count = semaphore->count; semaphore->count = 0; if (old_count < 0) { waitq_wakeup64_all_locked(&semaphore->waitq, SEMAPHORE_EVENT, THREAD_RESTART, NULL, WAITQ_ALL_PRIORITIES, WAITQ_UNLOCK); /* waitq/semaphore is unlocked */ } else { semaphore_unlock(semaphore); } }
/* * Routine: _wait_queue_select64_one * Purpose: * Select the best thread off a wait queue that meet the * supplied criteria. * Conditions: * at splsched * wait queue locked * possibly recursive * Returns: * a locked thread - if one found * Note: * This is where the sync policy of the wait queue comes * into effect. For now, we just assume FIFO/LIFO. */ static thread_t _wait_queue_select64_one( wait_queue_t wq, event64_t event) { wait_queue_element_t wq_element; wait_queue_element_t wqe_next; thread_t t = THREAD_NULL; queue_t q; q = &wq->wq_queue; wq_element = (wait_queue_element_t) queue_first(q); while (!queue_end(q, (queue_entry_t)wq_element)) { WAIT_QUEUE_ELEMENT_CHECK(wq, wq_element); wqe_next = (wait_queue_element_t) queue_next((queue_t) wq_element); /* * We may have to recurse if this is a compound wait queue. */ if (wq_element->wqe_type == WAIT_QUEUE_LINK || wq_element->wqe_type == WAIT_QUEUE_LINK_NOALLOC) { wait_queue_link_t wql = (wait_queue_link_t)wq_element; wait_queue_set_t set_queue = wql->wql_setqueue; /* * We have to check the set wait queue. If the set * supports pre-posting, it isn't already preposted, * and we didn't find a thread in the set, then mark it. * * If we later find a thread, there may be a spurious * pre-post here on this set. The wait side has to check * for that either pre- or post-wait. */ wqs_lock(set_queue); if (! wait_queue_empty(&set_queue->wqs_wait_queue)) { t = _wait_queue_select64_one(&set_queue->wqs_wait_queue, event); } if (t != THREAD_NULL) { wqs_unlock(set_queue); return t; } if (event == NO_EVENT64 && set_queue->wqs_prepost && !wql_is_preposted(wql)) { queue_t ppq = &set_queue->wqs_preposts; queue_enter(ppq, wql, wait_queue_link_t, wql_preposts); } wqs_unlock(set_queue); } else { /* * Otherwise, its a thread. If it is waiting on * the event we are posting to this queue, pull * it off the queue and stick it in out wake_queue. */ t = (thread_t)wq_element; if (t->wait_event == event) { thread_lock(t); remqueue((queue_entry_t) t); t->wait_queue = WAIT_QUEUE_NULL; t->wait_event = NO_EVENT64; t->at_safe_point = FALSE; return t; /* still locked */ } t = THREAD_NULL; } wq_element = wqe_next; } return THREAD_NULL; }
/* * Routine: _wait_queue_select64_all * Purpose: * Select all threads off a wait queue that meet the * supplied criteria. * Conditions: * at splsched * wait queue locked * wake_queue initialized and ready for insertion * possibly recursive * Returns: * a queue of locked threads */ static void _wait_queue_select64_all( wait_queue_t wq, event64_t event, queue_t wake_queue) { wait_queue_element_t wq_element; wait_queue_element_t wqe_next; queue_t q; q = &wq->wq_queue; wq_element = (wait_queue_element_t) queue_first(q); while (!queue_end(q, (queue_entry_t)wq_element)) { WAIT_QUEUE_ELEMENT_CHECK(wq, wq_element); wqe_next = (wait_queue_element_t) queue_next((queue_t) wq_element); /* * We may have to recurse if this is a compound wait queue. */ if (wq_element->wqe_type == WAIT_QUEUE_LINK || wq_element->wqe_type == WAIT_QUEUE_LINK_NOALLOC) { wait_queue_link_t wql = (wait_queue_link_t)wq_element; wait_queue_set_t set_queue = wql->wql_setqueue; /* * We have to check the set wait queue. If it is marked * as pre-post, and it is the "generic event" then mark * it pre-posted now (if not already). */ wqs_lock(set_queue); if (event == NO_EVENT64 && set_queue->wqs_prepost && !wql_is_preposted(wql)) { queue_t ppq = &set_queue->wqs_preposts; queue_enter(ppq, wql, wait_queue_link_t, wql_preposts); } if (! wait_queue_empty(&set_queue->wqs_wait_queue)) _wait_queue_select64_all(&set_queue->wqs_wait_queue, event, wake_queue); wqs_unlock(set_queue); } else { /* * Otherwise, its a thread. If it is waiting on * the event we are posting to this queue, pull * it off the queue and stick it in out wake_queue. */ thread_t t = (thread_t)wq_element; if (t->wait_event == event) { thread_lock(t); remqueue((queue_entry_t) t); enqueue (wake_queue, (queue_entry_t) t); t->wait_queue = WAIT_QUEUE_NULL; t->wait_event = NO_EVENT64; t->at_safe_point = FALSE; /* returned locked */ } } wq_element = wqe_next; } }
/* * Routine: _wait_queue_select64_all * Purpose: * Select all threads off a wait queue that meet the * supplied criteria. * Conditions: * at splsched * wait queue locked * wake_queue initialized and ready for insertion * possibly recursive * Returns: * a queue of locked threads */ static void _wait_queue_select64_all( wait_queue_t wq, event64_t event, queue_t wake_queue) { wait_queue_element_t wq_element; wait_queue_element_t wqe_next; queue_t q; q = &wq->wq_queue; wq_element = (wait_queue_element_t) queue_first(q); while (!queue_end(q, (queue_entry_t)wq_element)) { WAIT_QUEUE_ELEMENT_CHECK(wq, wq_element); wqe_next = (wait_queue_element_t) queue_next((queue_t) wq_element); /* * We may have to recurse if this is a compound wait queue. */ if (wq_element->wqe_type == WAIT_QUEUE_LINK) { wait_queue_link_t wql = (wait_queue_link_t)wq_element; wait_queue_t set_queue; /* * We have to check the set wait queue. */ set_queue = (wait_queue_t)wql->wql_setqueue; wait_queue_lock(set_queue); if (set_queue->wq_isprepost) { wait_queue_set_t wqs = (wait_queue_set_t)set_queue; /* * Preposting is only for sets and wait queue * is the first element of set */ wqs->wqs_refcount++; } if (! wait_queue_empty(set_queue)) _wait_queue_select64_all(set_queue, event, wake_queue); wait_queue_unlock(set_queue); } else { /* * Otherwise, its a thread. If it is waiting on * the event we are posting to this queue, pull * it off the queue and stick it in out wake_queue. */ thread_t t = (thread_t)wq_element; if (t->wait_event == event) { thread_lock(t); remqueue(q, (queue_entry_t) t); enqueue (wake_queue, (queue_entry_t) t); t->wait_queue = WAIT_QUEUE_NULL; t->wait_event = NO_EVENT64; t->at_safe_point = FALSE; /* returned locked */ } } wq_element = wqe_next; } }
kern_return_t processor_shutdown( processor_t processor) { processor_set_t pset; spl_t s; s = splsched(); pset = processor->processor_set; pset_lock(pset); if (processor->state == PROCESSOR_OFF_LINE) { /* * Success if already shutdown. */ pset_unlock(pset); splx(s); return (KERN_SUCCESS); } if (processor->state == PROCESSOR_START) { /* * Failure if currently being started. */ pset_unlock(pset); splx(s); return (KERN_FAILURE); } /* * If the processor is dispatching, let it finish. */ while (processor->state == PROCESSOR_DISPATCHING) { pset_unlock(pset); splx(s); delay(1); s = splsched(); pset_lock(pset); } /* * Success if already being shutdown. */ if (processor->state == PROCESSOR_SHUTDOWN) { pset_unlock(pset); splx(s); return (KERN_SUCCESS); } if (processor->state == PROCESSOR_IDLE) remqueue((queue_entry_t)processor); else if (processor->state == PROCESSOR_RUNNING) remqueue((queue_entry_t)processor); processor->state = PROCESSOR_SHUTDOWN; pset_unlock(pset); processor_doshutdown(processor); splx(s); cpu_exit_wait(processor->cpu_id); return (KERN_SUCCESS); }