/*
* swapin_thread: [exported]
*
* This procedure executes as a kernel thread. Threads that need to
* be swapped in are swapped in by this thread.
*/
void __attribute__((noreturn)) swapin_thread_continue(void)
{
for (;;) {
thread_t thread;
spl_t s;
s = splsched();
swapper_lock();
while ((thread = (thread_t) dequeue_head(&swapin_queue))
!= THREAD_NULL) {
kern_return_t kr;
swapper_unlock();
(void) splx(s);
kr = thread_doswapin(thread); /* may block */
s = splsched();
swapper_lock();
if (kr != KERN_SUCCESS) {
enqueue_head(&swapin_queue,
(queue_entry_t) thread);
break;
}
}
assert_wait((event_t) &swapin_queue, FALSE);
swapper_unlock();
(void) splx(s);
counter(c_swapin_thread_block++);
thread_block(swapin_thread_continue);
}
}
static uintptr_t
iopa_alloc(vm_size_t bytes, uint32_t balign)
{
static const uint64_t align_masks[] = {
0xFFFFFFFFFFFFFFFF,
0xAAAAAAAAAAAAAAAA,
0x8888888888888888,
0x8080808080808080,
0x8000800080008000,
0x8000000080000000,
0x8000000000000000,
};
io_pagealloc_t * pa;
uintptr_t addr = 0;
uint32_t count;
uint64_t align;
if (!bytes) bytes = 1;
count = (bytes + kIOPageAllocChunkBytes - 1) / kIOPageAllocChunkBytes;
align = align_masks[log2up((balign + kIOPageAllocChunkBytes - 1) / kIOPageAllocChunkBytes)];
IOSimpleLockLock(gIOPageAllocLock);
pa = (typeof(pa)) queue_first(&gIOPageAllocList);
while (!queue_end(&gIOPageAllocList, &pa->link))
{
addr = iopa_allocinpage(pa, count, align);
if (addr)
{
gIOPageAllocBytes += bytes;
break;
}
pa = (typeof(pa)) queue_next(&pa->link);
}
IOSimpleLockUnlock(gIOPageAllocLock);
if (!addr)
{
pa = iopa_allocpage();
if (pa)
{
addr = iopa_allocinpage(pa, count, align);
IOSimpleLockLock(gIOPageAllocLock);
if (pa->avail) enqueue_head(&gIOPageAllocList, &pa->link);
gIOPageAllocCount++;
if (addr) gIOPageAllocBytes += bytes;
IOSimpleLockUnlock(gIOPageAllocLock);
}
}
if (addr)
{
assert((addr & ((1 << log2up(balign)) - 1)) == 0);
IOStatisticsAlloc(kIOStatisticsMallocAligned, bytes);
#if IOALLOCDEBUG
debug_iomalloc_size += bytes;
#endif
}
return (addr);
}
/*
* _internal_call_release:
*
* Release an internal callout entry which
* is no longer pending (or delayed).
*
* Called with thread_call_lock held.
*/
static __inline__ void
_internal_call_release(
thread_call_t call)
{
if ( call >= internal_call_storage &&
call < &internal_call_storage[INTERNAL_CALL_COUNT] )
enqueue_head(&thread_call_internal_queue, qe(call));
}
/*
* _internal_call_release:
*
* Release an internal callout entry which
* is no longer pending (or delayed).
*
* Called with thread_call_lock held.
*/
static __inline__ void
_internal_call_release(
thread_call_t call)
{
if ( call >= internal_call_storage &&
call < &internal_call_storage[internal_call_count] )
enqueue_head(&thread_call_internal_queue, qe(call));
}
/*
* Routine: semaphore_create
*
* Creates a semaphore.
* The port representing the semaphore is returned as a parameter.
*/
kern_return_t
semaphore_create(
task_t task,
semaphore_t *new_semaphore,
int policy,
int value)
{
semaphore_t s = SEMAPHORE_NULL;
if (task == TASK_NULL || value < 0 || policy > SYNC_POLICY_MAX) {
*new_semaphore = SEMAPHORE_NULL;
return KERN_INVALID_ARGUMENT;
}
s = (semaphore_t) zalloc (semaphore_zone);
if (s == SEMAPHORE_NULL) {
*new_semaphore = SEMAPHORE_NULL;
return KERN_RESOURCE_SHORTAGE;
}
wait_queue_init(&s->wait_queue, policy); /* also inits lock */
s->count = value;
s->ref_count = 1;
/*
* Create and initialize the semaphore port
*/
s->port = ipc_port_alloc_kernel();
if (s->port == IP_NULL) {
/* This will deallocate the semaphore */
semaphore_dereference(s);
*new_semaphore = SEMAPHORE_NULL;
return KERN_RESOURCE_SHORTAGE;
}
ipc_kobject_set (s->port, (ipc_kobject_t) s, IKOT_SEMAPHORE);
/*
* Associate the new semaphore with the task by adding
* the new semaphore to the task's semaphore list.
*
* Associate the task with the new semaphore by having the
* semaphores task pointer point to the owning task's structure.
*/
task_lock(task);
enqueue_head(&task->semaphore_list, (queue_entry_t) s);
task->semaphores_owned++;
s->owner = task;
s->active = TRUE;
task_unlock(task);
*new_semaphore = s;
return KERN_SUCCESS;
}
/*
* Routine: semaphore_create
*
* Creates a semaphore.
* The port representing the semaphore is returned as a parameter.
*/
kern_return_t
semaphore_create(
task_t task,
semaphore_t *new_semaphore,
int policy,
int value)
{
semaphore_t s = SEMAPHORE_NULL;
kern_return_t kret;
*new_semaphore = SEMAPHORE_NULL;
if (task == TASK_NULL || value < 0 || policy > SYNC_POLICY_MAX)
return KERN_INVALID_ARGUMENT;
s = (semaphore_t) zalloc (semaphore_zone);
if (s == SEMAPHORE_NULL)
return KERN_RESOURCE_SHORTAGE;
kret = waitq_init(&s->waitq, policy | SYNC_POLICY_DISABLE_IRQ); /* also inits lock */
if (kret != KERN_SUCCESS) {
zfree(semaphore_zone, s);
return kret;
}
/*
* Initialize the semaphore values.
*/
s->port = IP_NULL;
s->ref_count = 1;
s->count = value;
s->active = TRUE;
s->owner = task;
/*
* Associate the new semaphore with the task by adding
* the new semaphore to the task's semaphore list.
*/
task_lock(task);
enqueue_head(&task->semaphore_list, (queue_entry_t) s);
task->semaphores_owned++;
task_unlock(task);
*new_semaphore = s;
return KERN_SUCCESS;
}
/*
* The scheduler is too messy for my old little brain
*/
void
simpler_thread_setrun(
thread_t th,
boolean_t may_preempt)
{
register struct run_queue *rq;
register int whichq;
/*
* XXX should replace queue with a boolean in this case.
*/
if (default_pset.idle_count > 0) {
processor_t processor;
processor = (processor_t) queue_first(&default_pset.idle_queue);
queue_remove(&default_pset.idle_queue, processor,
processor_t, processor_queue);
default_pset.idle_count--;
processor->next_thread = th;
processor->state = PROCESSOR_DISPATCHING;
return;
}
rq = &(master_processor->runq);
ast_on(cpu_number(), AST_BLOCK);
whichq = (th)->sched_pri;
simple_lock(&(rq)->lock); /* lock the run queue */
enqueue_head(&(rq)->runq[whichq], (queue_entry_t) (th));
if (whichq < (rq)->low || (rq)->count == 0)
(rq)->low = whichq; /* minimize */
(rq)->count++;
#ifdef MIGRATING_THREADS
(th)->shuttle.runq = (rq);
#else
(th)->runq = (rq);
#endif
simple_unlock(&(rq)->lock);
/*
* Turn off first_quantum to allow context switch.
*/
current_processor()->first_quantum = FALSE;
}
/*
* Routine: wait_queue_assert_wait64_locked
* Purpose:
* Insert the current thread into the supplied wait queue
* waiting for a particular event to be posted to that queue.
*
* Conditions:
* The wait queue is assumed locked.
* The waiting thread is assumed locked.
*
*/
__private_extern__ wait_result_t
wait_queue_assert_wait64_locked(
wait_queue_t wq,
event64_t event,
wait_interrupt_t interruptible,
thread_t thread)
{
wait_result_t wait_result;
if (!wait_queue_assert_possible(thread))
panic("wait_queue_assert_wait64_locked");
if (wq->wq_type == _WAIT_QUEUE_SET_inited) {
wait_queue_set_t wqs = (wait_queue_set_t)wq;
if (wqs->wqs_isprepost && wqs->wqs_refcount > 0)
return(THREAD_AWAKENED);
}
/*
* This is the extent to which we currently take scheduling attributes
* into account. If the thread is vm priviledged, we stick it at
* the front of the queue. Later, these queues will honor the policy
* value set at wait_queue_init time.
*/
wait_result = thread_mark_wait_locked(thread, interruptible);
if (wait_result == THREAD_WAITING) {
if (thread->vm_privilege)
enqueue_head(&wq->wq_queue, (queue_entry_t) thread);
else
enqueue_tail(&wq->wq_queue, (queue_entry_t) thread);
thread->wait_event = event;
thread->wait_queue = wq;
}
return(wait_result);
}
/*
* Routine: wait_queue_assert_wait64_locked
* Purpose:
* Insert the current thread into the supplied wait queue
* waiting for a particular event to be posted to that queue.
*
* Conditions:
* The wait queue is assumed locked.
* The waiting thread is assumed locked.
*
*/
__private_extern__ wait_result_t
wait_queue_assert_wait64_locked(
wait_queue_t wq,
event64_t event,
wait_interrupt_t interruptible,
uint64_t deadline,
thread_t thread)
{
wait_result_t wait_result;
boolean_t realtime;
if (!wait_queue_assert_possible(thread))
panic("wait_queue_assert_wait64_locked");
if (wq->wq_type == _WAIT_QUEUE_SET_inited) {
wait_queue_set_t wqs = (wait_queue_set_t)wq;
if (event == NO_EVENT64 && wqs_is_preposted(wqs))
return(THREAD_AWAKENED);
}
/*
* Realtime threads get priority for wait queue placements.
* This allows wait_queue_wakeup_one to prefer a waiting
* realtime thread, similar in principle to performing
* a wait_queue_wakeup_all and allowing scheduler prioritization
* to run the realtime thread, but without causing the
* lock contention of that scenario.
*/
realtime = (thread->sched_pri >= BASEPRI_REALTIME);
/*
* This is the extent to which we currently take scheduling attributes
* into account. If the thread is vm priviledged, we stick it at
* the front of the queue. Later, these queues will honor the policy
* value set at wait_queue_init time.
*/
wait_result = thread_mark_wait_locked(thread, interruptible);
if (wait_result == THREAD_WAITING) {
if (!wq->wq_fifo
|| (thread->options & TH_OPT_VMPRIV)
|| realtime)
enqueue_head(&wq->wq_queue, (queue_entry_t) thread);
else
enqueue_tail(&wq->wq_queue, (queue_entry_t) thread);
thread->wait_event = event;
thread->wait_queue = wq;
if (deadline != 0) {
uint32_t flags;
flags = realtime ? TIMER_CALL_CRITICAL : 0;
if (!timer_call_enter(&thread->wait_timer, deadline, flags))
thread->wait_timer_active++;
thread->wait_timer_is_set = TRUE;
}
}
return(wait_result);
}
