void
host_notify_port_destroy(
ipc_port_t port)
{
host_notify_t entry;
mutex_lock(&host_notify_lock);
ip_lock(port);
if (ip_kotype(port) == IKOT_HOST_NOTIFY) {
entry = (host_notify_t)port->ip_kobject;
assert(entry != NULL);
ipc_kobject_set_atomically(port, IKO_NULL, IKOT_NONE);
ip_unlock(port);
assert(entry->port == port);
remqueue(NULL, (queue_entry_t)entry);
mutex_unlock(&host_notify_lock);
zfree(host_notify_zone, (vm_offset_t)entry);
ipc_port_release_sonce(port);
return;
}
ip_unlock(port);
mutex_unlock(&host_notify_lock);
}
/*
* Routine: _wait_queue_select64_one
* Purpose:
* Select the best thread off a wait queue that meet the
* supplied criteria.
* Conditions:
* at splsched
* wait queue locked
* possibly recursive
* Returns:
* a locked thread - if one found
* Note:
* This is where the sync policy of the wait queue comes
* into effect. For now, we just assume FIFO.
*/
static thread_t
_wait_queue_select64_one(
wait_queue_t wq,
event64_t event)
{
wait_queue_element_t wq_element;
wait_queue_element_t wqe_next;
thread_t t = THREAD_NULL;
queue_t q;
assert(wq->wq_fifo);
q = &wq->wq_queue;
wq_element = (wait_queue_element_t) queue_first(q);
while (!queue_end(q, (queue_entry_t)wq_element)) {
WAIT_QUEUE_ELEMENT_CHECK(wq, wq_element);
wqe_next = (wait_queue_element_t)
queue_next((queue_t) wq_element);
/*
* We may have to recurse if this is a compound wait queue.
*/
if (wq_element->wqe_type == WAIT_QUEUE_LINK) {
wait_queue_link_t wql = (wait_queue_link_t)wq_element;
wait_queue_t set_queue;
/*
* We have to check the set wait queue.
*/
set_queue = (wait_queue_t)wql->wql_setqueue;
wait_queue_lock(set_queue);
if (! wait_queue_empty(set_queue)) {
t = _wait_queue_select64_one(set_queue, event);
}
wait_queue_unlock(set_queue);
if (t != THREAD_NULL)
return t;
} else {
/*
* Otherwise, its a thread. If it is waiting on
* the event we are posting to this queue, pull
* it off the queue and stick it in out wake_queue.
*/
thread_t t = (thread_t)wq_element;
if (t->wait_event == event) {
thread_lock(t);
remqueue(q, (queue_entry_t) t);
t->wait_queue = WAIT_QUEUE_NULL;
t->wait_event = NO_EVENT64;
t->at_safe_point = FALSE;
return t; /* still locked */
}
}
wq_element = wqe_next;
}
return THREAD_NULL;
}
/*
* Routine: semaphore_destroy
*
* Destroys a semaphore. This call will only succeed if the
* specified task is the SAME task name specified at the semaphore's
* creation.
*
* All threads currently blocked on the semaphore are awoken. These
* threads will return with the KERN_TERMINATED error.
*/
kern_return_t
semaphore_destroy(
task_t task,
semaphore_t semaphore)
{
int old_count;
spl_t spl_level;
if (task == TASK_NULL || semaphore == SEMAPHORE_NULL)
return KERN_INVALID_ARGUMENT;
/*
* Disown semaphore
*/
task_lock(task);
if (semaphore->owner != task) {
task_unlock(task);
return KERN_INVALID_ARGUMENT;
}
remqueue(&task->semaphore_list, (queue_entry_t) semaphore);
semaphore->owner = TASK_NULL;
task->semaphores_owned--;
task_unlock(task);
spl_level = splsched();
semaphore_lock(semaphore);
/*
* Deactivate semaphore
*/
assert(semaphore->active);
semaphore->active = FALSE;
/*
* Wakeup blocked threads
*/
old_count = semaphore->count;
semaphore->count = 0;
if (old_count < 0) {
wait_queue_wakeup64_all_locked(&semaphore->wait_queue,
SEMAPHORE_EVENT,
THREAD_RESTART,
TRUE); /* unlock? */
} else {
semaphore_unlock(semaphore);
}
splx(spl_level);
/*
* Deallocate
*
* Drop the semaphore reference, which in turn deallocates the
* semaphore structure if the reference count goes to zero.
*/
ipc_port_dealloc_kernel(semaphore->port);
semaphore_dereference(semaphore);
return KERN_SUCCESS;
}
/*
* sched_traditional_processor_queue_shutdown:
*
* Shutdown a processor run queue by
* re-dispatching non-bound threads.
*
* Associated pset must be locked, and is
* returned unlocked.
*/
static void
sched_traditional_processor_queue_shutdown(processor_t processor)
{
processor_set_t pset = processor->processor_set;
run_queue_t rq = runq_for_processor(processor);
queue_t queue = rq->queues + rq->highq;
int pri = rq->highq;
int count = rq->count;
thread_t next, thread;
queue_head_t tqueue;
queue_init(&tqueue);
while (count > 0) {
thread = (thread_t)(uintptr_t)queue_first(queue);
while (!queue_end(queue, (queue_entry_t)thread)) {
next = (thread_t)(uintptr_t)queue_next((queue_entry_t)thread);
if (thread->bound_processor == PROCESSOR_NULL) {
remqueue((queue_entry_t)thread);
thread->runq = PROCESSOR_NULL;
SCHED_STATS_RUNQ_CHANGE(&rq->runq_stats, rq->count);
runq_consider_decr_bound_count(processor, thread);
rq->count--;
if (SCHED(priority_is_urgent)(pri)) {
rq->urgency--; assert(rq->urgency >= 0);
}
if (queue_empty(queue)) {
bitmap_clear(rq->bitmap, pri);
rq->highq = bitmap_first(rq->bitmap, NRQS);
}
enqueue_tail(&tqueue, (queue_entry_t)thread);
}
count--;
thread = next;
}
queue--; pri--;
}
pset_unlock(pset);
while ((thread = (thread_t)(uintptr_t)dequeue_head(&tqueue)) != THREAD_NULL) {
thread_lock(thread);
thread_setrun(thread, SCHED_TAILQ);
thread_unlock(thread);
}
}
/*
* Routine: wait_queue_select64_thread
* Purpose:
* Look for a thread and remove it from the queues, if
* (and only if) the thread is waiting on the supplied
* <wait_queue, event> pair.
* Conditions:
* at splsched
* wait queue locked
* possibly recursive
* Returns:
* KERN_NOT_WAITING: Thread is not waiting here.
* KERN_SUCCESS: It was, and is now removed (returned locked)
*/
static kern_return_t
_wait_queue_select64_thread(
wait_queue_t wq,
event64_t event,
thread_t thread)
{
wait_queue_element_t wq_element;
wait_queue_element_t wqe_next;
kern_return_t res = KERN_NOT_WAITING;
queue_t q = &wq->wq_queue;
thread_lock(thread);
if ((thread->wait_queue == wq) && (thread->wait_event == event)) {
remqueue((queue_entry_t) thread);
thread->at_safe_point = FALSE;
thread->wait_event = NO_EVENT64;
thread->wait_queue = WAIT_QUEUE_NULL;
/* thread still locked */
return KERN_SUCCESS;
}
thread_unlock(thread);
/*
* The wait_queue associated with the thread may be one of this
* wait queue's sets. Go see. If so, removing it from
* there is like removing it from here.
*/
wq_element = (wait_queue_element_t) queue_first(q);
while (!queue_end(q, (queue_entry_t)wq_element)) {
WAIT_QUEUE_ELEMENT_CHECK(wq, wq_element);
wqe_next = (wait_queue_element_t)
queue_next((queue_t) wq_element);
if (wq_element->wqe_type == WAIT_QUEUE_LINK ||
wq_element->wqe_type == WAIT_QUEUE_LINK_NOALLOC) {
wait_queue_link_t wql = (wait_queue_link_t)wq_element;
wait_queue_set_t set_queue = wql->wql_setqueue;
wqs_lock(set_queue);
if (! wait_queue_empty(&set_queue->wqs_wait_queue)) {
res = _wait_queue_select64_thread(&set_queue->wqs_wait_queue,
event,
thread);
}
wqs_unlock(set_queue);
if (res == KERN_SUCCESS)
return KERN_SUCCESS;
}
wq_element = wqe_next;
}
return res;
}
/*
* Routine: wait_queue_pull_thread_locked
* Purpose:
* Pull a thread off its wait queue and (possibly) unlock
* the waitq.
* Conditions:
* at splsched
* wait queue locked
* thread locked
* Returns:
* with the thread still locked.
*/
void
wait_queue_pull_thread_locked(
wait_queue_t waitq,
thread_t thread,
boolean_t unlock)
{
assert(thread->wait_queue == waitq);
remqueue((queue_entry_t)thread );
thread->wait_queue = WAIT_QUEUE_NULL;
thread->wait_event = NO_EVENT64;
thread->at_safe_point = FALSE;
if (unlock)
wait_queue_unlock(waitq);
}
/*
* Port-death routine to clean up reply messages.
*/
boolean_t
tty_queue_clean(
queue_t q,
ipc_port_t port,
boolean_t (*routine)(io_req_t) )
{
register io_req_t ior;
ior = (io_req_t)queue_first(q);
while (!queue_end(q, (queue_entry_t)ior)) {
if (ior->io_reply_port == port) {
remqueue(q, (queue_entry_t)ior);
ior->io_done = routine;
iodone(ior);
return TRUE;
}
ior = ior->io_next;
}
return FALSE;
}
/*
* sched_traditional_choose_thread_from_runq:
*
* Locate a thread to execute from the processor run queue
* and return it. Only choose a thread with greater or equal
* priority.
*
* Associated pset must be locked. Returns THREAD_NULL
* on failure.
*/
static thread_t
sched_traditional_choose_thread_from_runq(
processor_t processor,
run_queue_t rq,
int priority)
{
queue_t queue = rq->queues + rq->highq;
int pri = rq->highq;
int count = rq->count;
thread_t thread;
while (count > 0 && pri >= priority) {
thread = (thread_t)(uintptr_t)queue_first(queue);
while (!queue_end(queue, (queue_entry_t)thread)) {
if (thread->bound_processor == PROCESSOR_NULL ||
thread->bound_processor == processor) {
remqueue((queue_entry_t)thread);
thread->runq = PROCESSOR_NULL;
SCHED_STATS_RUNQ_CHANGE(&rq->runq_stats, rq->count);
rq->count--;
if (SCHED(priority_is_urgent)(pri)) {
rq->urgency--; assert(rq->urgency >= 0);
}
if (queue_empty(queue)) {
bitmap_clear(rq->bitmap, pri);
rq->highq = bitmap_first(rq->bitmap, NRQS);
}
return (thread);
}
count--;
thread = (thread_t)(uintptr_t)queue_next((queue_entry_t)thread);
}
queue--; pri--;
}
return (THREAD_NULL);
}
/*
* Locate and steal a thread, beginning
* at the pset.
*
* The pset must be locked, and is returned
* unlocked.
*
* Returns the stolen thread, or THREAD_NULL on
* failure.
*/
static thread_t
sched_traditional_steal_thread(processor_set_t pset)
{
processor_set_t nset, cset = pset;
processor_t processor;
thread_t thread;
do {
processor = (processor_t)(uintptr_t)queue_first(&cset->active_queue);
while (!queue_end(&cset->active_queue, (queue_entry_t)processor)) {
if (runq_for_processor(processor)->count > 0) {
thread = sched_traditional_steal_processor_thread(processor);
if (thread != THREAD_NULL) {
remqueue((queue_entry_t)processor);
enqueue_tail(&cset->active_queue, (queue_entry_t)processor);
pset_unlock(cset);
return (thread);
}
}
processor = (processor_t)(uintptr_t)queue_next((queue_entry_t)processor);
}
nset = next_pset(cset);
if (nset != pset) {
pset_unlock(cset);
cset = nset;
pset_lock(cset);
}
} while (nset != pset);
pset_unlock(cset);
return (THREAD_NULL);
}
/*
* Routine: semaphore_destroy_internal
*
* Disassociate a semaphore from its owning task, mark it inactive,
* and set any waiting threads running with THREAD_RESTART.
*
* Conditions:
* task is locked
* semaphore is locked
* semaphore is owned by the specified task
* Returns:
* with semaphore unlocked
*/
static void
semaphore_destroy_internal(
task_t task,
semaphore_t semaphore)
{
int old_count;
/* unlink semaphore from owning task */
assert(semaphore->owner == task);
remqueue((queue_entry_t) semaphore);
semaphore->owner = TASK_NULL;
task->semaphores_owned--;
/*
* Deactivate semaphore
*/
assert(semaphore->active);
semaphore->active = FALSE;
/*
* Wakeup blocked threads
*/
old_count = semaphore->count;
semaphore->count = 0;
if (old_count < 0) {
waitq_wakeup64_all_locked(&semaphore->waitq,
SEMAPHORE_EVENT,
THREAD_RESTART, NULL,
WAITQ_ALL_PRIORITIES,
WAITQ_UNLOCK);
/* waitq/semaphore is unlocked */
} else {
semaphore_unlock(semaphore);
}
}
/*
* Routine: _wait_queue_select64_one
* Purpose:
* Select the best thread off a wait queue that meet the
* supplied criteria.
* Conditions:
* at splsched
* wait queue locked
* possibly recursive
* Returns:
* a locked thread - if one found
* Note:
* This is where the sync policy of the wait queue comes
* into effect. For now, we just assume FIFO/LIFO.
*/
static thread_t
_wait_queue_select64_one(
wait_queue_t wq,
event64_t event)
{
wait_queue_element_t wq_element;
wait_queue_element_t wqe_next;
thread_t t = THREAD_NULL;
queue_t q;
q = &wq->wq_queue;
wq_element = (wait_queue_element_t) queue_first(q);
while (!queue_end(q, (queue_entry_t)wq_element)) {
WAIT_QUEUE_ELEMENT_CHECK(wq, wq_element);
wqe_next = (wait_queue_element_t)
queue_next((queue_t) wq_element);
/*
* We may have to recurse if this is a compound wait queue.
*/
if (wq_element->wqe_type == WAIT_QUEUE_LINK ||
wq_element->wqe_type == WAIT_QUEUE_LINK_NOALLOC) {
wait_queue_link_t wql = (wait_queue_link_t)wq_element;
wait_queue_set_t set_queue = wql->wql_setqueue;
/*
* We have to check the set wait queue. If the set
* supports pre-posting, it isn't already preposted,
* and we didn't find a thread in the set, then mark it.
*
* If we later find a thread, there may be a spurious
* pre-post here on this set. The wait side has to check
* for that either pre- or post-wait.
*/
wqs_lock(set_queue);
if (! wait_queue_empty(&set_queue->wqs_wait_queue)) {
t = _wait_queue_select64_one(&set_queue->wqs_wait_queue, event);
}
if (t != THREAD_NULL) {
wqs_unlock(set_queue);
return t;
}
if (event == NO_EVENT64 && set_queue->wqs_prepost && !wql_is_preposted(wql)) {
queue_t ppq = &set_queue->wqs_preposts;
queue_enter(ppq, wql, wait_queue_link_t, wql_preposts);
}
wqs_unlock(set_queue);
} else {
/*
* Otherwise, its a thread. If it is waiting on
* the event we are posting to this queue, pull
* it off the queue and stick it in out wake_queue.
*/
t = (thread_t)wq_element;
if (t->wait_event == event) {
thread_lock(t);
remqueue((queue_entry_t) t);
t->wait_queue = WAIT_QUEUE_NULL;
t->wait_event = NO_EVENT64;
t->at_safe_point = FALSE;
return t; /* still locked */
}
t = THREAD_NULL;
}
wq_element = wqe_next;
}
return THREAD_NULL;
}
/*
* Routine: _wait_queue_select64_all
* Purpose:
* Select all threads off a wait queue that meet the
* supplied criteria.
* Conditions:
* at splsched
* wait queue locked
* wake_queue initialized and ready for insertion
* possibly recursive
* Returns:
* a queue of locked threads
*/
static void
_wait_queue_select64_all(
wait_queue_t wq,
event64_t event,
queue_t wake_queue)
{
wait_queue_element_t wq_element;
wait_queue_element_t wqe_next;
queue_t q;
q = &wq->wq_queue;
wq_element = (wait_queue_element_t) queue_first(q);
while (!queue_end(q, (queue_entry_t)wq_element)) {
WAIT_QUEUE_ELEMENT_CHECK(wq, wq_element);
wqe_next = (wait_queue_element_t)
queue_next((queue_t) wq_element);
/*
* We may have to recurse if this is a compound wait queue.
*/
if (wq_element->wqe_type == WAIT_QUEUE_LINK ||
wq_element->wqe_type == WAIT_QUEUE_LINK_NOALLOC) {
wait_queue_link_t wql = (wait_queue_link_t)wq_element;
wait_queue_set_t set_queue = wql->wql_setqueue;
/*
* We have to check the set wait queue. If it is marked
* as pre-post, and it is the "generic event" then mark
* it pre-posted now (if not already).
*/
wqs_lock(set_queue);
if (event == NO_EVENT64 && set_queue->wqs_prepost && !wql_is_preposted(wql)) {
queue_t ppq = &set_queue->wqs_preposts;
queue_enter(ppq, wql, wait_queue_link_t, wql_preposts);
}
if (! wait_queue_empty(&set_queue->wqs_wait_queue))
_wait_queue_select64_all(&set_queue->wqs_wait_queue, event, wake_queue);
wqs_unlock(set_queue);
} else {
/*
* Otherwise, its a thread. If it is waiting on
* the event we are posting to this queue, pull
* it off the queue and stick it in out wake_queue.
*/
thread_t t = (thread_t)wq_element;
if (t->wait_event == event) {
thread_lock(t);
remqueue((queue_entry_t) t);
enqueue (wake_queue, (queue_entry_t) t);
t->wait_queue = WAIT_QUEUE_NULL;
t->wait_event = NO_EVENT64;
t->at_safe_point = FALSE;
/* returned locked */
}
}
wq_element = wqe_next;
}
}
/*
* Routine: _wait_queue_select64_all
* Purpose:
* Select all threads off a wait queue that meet the
* supplied criteria.
* Conditions:
* at splsched
* wait queue locked
* wake_queue initialized and ready for insertion
* possibly recursive
* Returns:
* a queue of locked threads
*/
static void
_wait_queue_select64_all(
wait_queue_t wq,
event64_t event,
queue_t wake_queue)
{
wait_queue_element_t wq_element;
wait_queue_element_t wqe_next;
queue_t q;
q = &wq->wq_queue;
wq_element = (wait_queue_element_t) queue_first(q);
while (!queue_end(q, (queue_entry_t)wq_element)) {
WAIT_QUEUE_ELEMENT_CHECK(wq, wq_element);
wqe_next = (wait_queue_element_t)
queue_next((queue_t) wq_element);
/*
* We may have to recurse if this is a compound wait queue.
*/
if (wq_element->wqe_type == WAIT_QUEUE_LINK) {
wait_queue_link_t wql = (wait_queue_link_t)wq_element;
wait_queue_t set_queue;
/*
* We have to check the set wait queue.
*/
set_queue = (wait_queue_t)wql->wql_setqueue;
wait_queue_lock(set_queue);
if (set_queue->wq_isprepost) {
wait_queue_set_t wqs = (wait_queue_set_t)set_queue;
/*
* Preposting is only for sets and wait queue
* is the first element of set
*/
wqs->wqs_refcount++;
}
if (! wait_queue_empty(set_queue))
_wait_queue_select64_all(set_queue, event, wake_queue);
wait_queue_unlock(set_queue);
} else {
/*
* Otherwise, its a thread. If it is waiting on
* the event we are posting to this queue, pull
* it off the queue and stick it in out wake_queue.
*/
thread_t t = (thread_t)wq_element;
if (t->wait_event == event) {
thread_lock(t);
remqueue(q, (queue_entry_t) t);
enqueue (wake_queue, (queue_entry_t) t);
t->wait_queue = WAIT_QUEUE_NULL;
t->wait_event = NO_EVENT64;
t->at_safe_point = FALSE;
/* returned locked */
}
}
wq_element = wqe_next;
}
}
kern_return_t
processor_shutdown(
processor_t processor)
{
processor_set_t pset;
spl_t s;
s = splsched();
pset = processor->processor_set;
pset_lock(pset);
if (processor->state == PROCESSOR_OFF_LINE) {
/*
* Success if already shutdown.
*/
pset_unlock(pset);
splx(s);
return (KERN_SUCCESS);
}
if (processor->state == PROCESSOR_START) {
/*
* Failure if currently being started.
*/
pset_unlock(pset);
splx(s);
return (KERN_FAILURE);
}
/*
* If the processor is dispatching, let it finish.
*/
while (processor->state == PROCESSOR_DISPATCHING) {
pset_unlock(pset);
splx(s);
delay(1);
s = splsched();
pset_lock(pset);
}
/*
* Success if already being shutdown.
*/
if (processor->state == PROCESSOR_SHUTDOWN) {
pset_unlock(pset);
splx(s);
return (KERN_SUCCESS);
}
if (processor->state == PROCESSOR_IDLE)
remqueue((queue_entry_t)processor);
else
if (processor->state == PROCESSOR_RUNNING)
remqueue((queue_entry_t)processor);
processor->state = PROCESSOR_SHUTDOWN;
pset_unlock(pset);
processor_doshutdown(processor);
splx(s);
cpu_exit_wait(processor->cpu_id);
return (KERN_SUCCESS);
}
