/*
* thread_doswapin:
*
* Swapin the specified thread, if it should be runnable, then put
* it on a run queue. No locks should be held on entry, as it is
* likely that this routine will sleep (waiting for stack allocation).
*/
kern_return_t thread_doswapin(thread_t thread)
{
kern_return_t kr;
spl_t s;
/*
* Allocate the kernel stack.
*/
kr = stack_alloc(thread, thread_continue);
if (kr != KERN_SUCCESS)
return kr;
/*
* Place on run queue.
*/
s = splsched();
thread_lock(thread);
thread->state &= ~(TH_SWAPPED | TH_SW_COMING_IN);
if (thread->state & TH_RUN)
thread_setrun(thread, TRUE);
thread_unlock(thread);
(void) splx(s);
return KERN_SUCCESS;
}
/*
* thread_stack_daemon:
*
* Perform stack allocation as required due to
* invoke failures.
*/
static void
thread_stack_daemon(void)
{
thread_t thread;
simple_lock(&thread_stack_lock);
while ((thread = (thread_t)dequeue_head(&thread_stack_queue)) != THREAD_NULL) {
simple_unlock(&thread_stack_lock);
stack_alloc(thread);
(void)splsched();
thread_lock(thread);
thread_setrun(thread, SCHED_PREEMPT | SCHED_TAILQ);
thread_unlock(thread);
(void)spllo();
simple_lock(&thread_stack_lock);
}
assert_wait((event_t)&thread_stack_queue, THREAD_UNINT);
simple_unlock(&thread_stack_lock);
thread_block((thread_continue_t)thread_stack_daemon);
/*NOTREACHED*/
}
/*
* sched_traditional_processor_queue_shutdown:
*
* Shutdown a processor run queue by
* re-dispatching non-bound threads.
*
* Associated pset must be locked, and is
* returned unlocked.
*/
static void
sched_traditional_processor_queue_shutdown(processor_t processor)
{
processor_set_t pset = processor->processor_set;
run_queue_t rq = runq_for_processor(processor);
queue_t queue = rq->queues + rq->highq;
int pri = rq->highq;
int count = rq->count;
thread_t next, thread;
queue_head_t tqueue;
queue_init(&tqueue);
while (count > 0) {
thread = (thread_t)(uintptr_t)queue_first(queue);
while (!queue_end(queue, (queue_entry_t)thread)) {
next = (thread_t)(uintptr_t)queue_next((queue_entry_t)thread);
if (thread->bound_processor == PROCESSOR_NULL) {
remqueue((queue_entry_t)thread);
thread->runq = PROCESSOR_NULL;
SCHED_STATS_RUNQ_CHANGE(&rq->runq_stats, rq->count);
runq_consider_decr_bound_count(processor, thread);
rq->count--;
if (SCHED(priority_is_urgent)(pri)) {
rq->urgency--; assert(rq->urgency >= 0);
}
if (queue_empty(queue)) {
bitmap_clear(rq->bitmap, pri);
rq->highq = bitmap_first(rq->bitmap, NRQS);
}
enqueue_tail(&tqueue, (queue_entry_t)thread);
}
count--;
thread = next;
}
queue--; pri--;
}
pset_unlock(pset);
while ((thread = (thread_t)(uintptr_t)dequeue_head(&tqueue)) != THREAD_NULL) {
thread_lock(thread);
thread_setrun(thread, SCHED_TAILQ);
thread_unlock(thread);
}
}
void
thread_go(
thread_t thread)
{
int state;
spl_t s;
s = splsched();
thread_lock(thread);
reset_timeout_check(&thread->timer);
state = thread->state;
switch (state & TH_SCHED_STATE) {
case TH_WAIT | TH_SUSP | TH_UNINT:
case TH_WAIT | TH_UNINT:
case TH_WAIT:
/*
* Sleeping and not suspendable - put
* on run queue.
*/
thread->state = (state &~ TH_WAIT) | TH_RUN;
thread->wait_result = THREAD_AWAKENED;
thread_setrun(thread, TRUE);
break;
case TH_WAIT | TH_SUSP:
case TH_RUN | TH_WAIT:
case TH_RUN | TH_WAIT | TH_SUSP:
case TH_RUN | TH_WAIT | TH_UNINT:
case TH_RUN | TH_WAIT | TH_SUSP | TH_UNINT:
/*
* Either already running, or suspended.
*/
thread->state = state & ~TH_WAIT;
thread->wait_result = THREAD_AWAKENED;
break;
default:
/*
* Not waiting.
*/
break;
}
thread_unlock(thread);
splx(s);
}
/*
* Called exclusively from interrupt context
*/
void
evc_signal(evc_t ev)
{
register volatile thread_t thread;
register int state;
spl_t s;
if (ev->sanity != ev)
return;
s = splsched();
simple_lock(&ev->lock);
ev->count++;
if (thread = ev->waiting_thread, thread != THREAD_NULL)
{
ev->waiting_thread = 0;
#if (NCPUS > 1)
retry:
while((thread->state & TH_RUN) || thread->lock.lock_data)
;
#endif
thread_lock(thread);
/* make thread runnable on this processor */
/* taken from clear_wait */
switch ((state = thread->state) & TH_SCHED_STATE)
{
case TH_WAIT | TH_SUSP | TH_UNINT:
case TH_WAIT | TH_UNINT:
case TH_WAIT:
/*
* Sleeping and not suspendable - put
* on run queue.
*/
thread->state = (state &~ TH_WAIT) | TH_RUN;
thread_unlock(thread);
#if NCPUS > 1
thread_setrun(thread, TRUE);
#else
simpler_thread_setrun(thread, TRUE);
#endif
break;
case TH_RUN | TH_WAIT:
#if (NCPUS > 1)
/*
* Legal on MP: between assert_wait()
* and thread_block(), in evc_wait() above.
*
* Mmm. Maybe don't need now that the while(..) check is
* done before the thread lock is grabbed.....
*/
thread_unlock(thread);
goto retry;
#else
/*FALLTHROUGH*/
#endif
case TH_WAIT | TH_SUSP:
case TH_RUN | TH_WAIT | TH_SUSP:
case TH_RUN | TH_WAIT | TH_UNINT:
case TH_RUN | TH_WAIT | TH_SUSP | TH_UNINT:
/*
* Either already running, or suspended.
* Just clear the wait.
*/
thread->state = state &~ TH_WAIT;
thread_unlock(thread);
break;
default:
/*
* Not waiting.
*/
panic("evc_signal.3");
thread_unlock(thread);
break;
}
}
simple_unlock(&ev->lock);
splx(s);
}
/*
* Now running in a thread. Create the rest of the kernel threads
* and the bootstrap task.
*/
void
start_kernel_threads(void)
{
register int i;
/*
* Create the idle threads and the other
* service threads.
*/
for (i = 0; i < NCPUS; i++) {
if (machine_slot[i].is_cpu) {
thread_t th;
spl_t s;
processor_t processor = cpu_to_processor(i);
(void) thread_create_at(kernel_task, &th, idle_thread);
s=splsched();
thread_lock(th);
thread_bind_locked(th, processor);
processor->idle_thread = th;
/*(void) thread_resume(th->top_act);*/
th->state |= TH_RUN;
thread_setrun( th, TRUE, TAIL_Q);
thread_unlock( th );
splx(s);
}
}
(void) kernel_thread(kernel_task, reaper_thread, (char *) 0);
#if THREAD_SWAPPER
(void) kernel_thread(kernel_task, swapin_thread, (char *) 0);
(void) kernel_thread(kernel_task, swapout_thread, (char *) 0);
#endif /* THREAD_SWAPPER */
#if TASK_SWAPPER
if (task_swap_on) {
(void) kernel_thread(kernel_task, task_swapper, (char *) 0);
(void) kernel_thread(kernel_task, task_swap_swapout_thread,
(char *) 0);
}
#endif /* TASK_SWAPPER */
(void) kernel_thread(kernel_task, sched_thread, (char *) 0);
(void) kernel_thread(kernel_task, timeout_thread, (char *) 0);
#if NORMA_VM
(void) kernel_thread(kernel_task, vm_object_thread, (char *) 0);
#endif /* NORMA_VM */
/*
* Create the clock service.
*/
clock_service_create();
/*
* Create the device service.
*/
device_service_create();
/*
* Initialize distributed services, starting
* with distributed IPC and progressing to any
* services layered on top of that.
*
* This stub exists even in non-NORMA systems.
*/
norma_bootstrap();
/*
* Initialize any testing services blocking the main kernel
* thread so that the in-kernel tests run without interference
* from other boot time activities. We will resume this thread
* in kernel_test_thread().
*/
#if KERNEL_TEST
/*
* Initialize the lock that will be used to guard
* variables that will be used in the test synchronization
* scheme.
*/
simple_lock_init(&kernel_test_lock, ETAP_MISC_KERNEL_TEST);
#if PARAGON860
{
char *s;
unsigned int firstnode;
/*
* Only start up loopback tests on boot node.
*/
if ((s = (char *) getbootenv("BOOT_FIRST_NODE")) == 0)
panic("startup");
firstnode = atoi(s);
(void) kernel_thread(kernel_task, kernel_test_thread,
(char * )(dipc_node_self() ==
(node_name) firstnode));
}
#else /* PARAGON860 */
(void) kernel_thread(kernel_task, kernel_test_thread, (char *) 0);
#endif /* PARAGON860 */
{
/*
* The synchronization scheme uses a simple lock, two
* booleans and the wakeup event. The wakeup event will
* be posted by kernel_test_thread().
*/
spl_t s;
s = splsched();
simple_lock(&kernel_test_lock);
while(!kernel_test_thread_sync_done){
assert_wait((event_t) &start_kernel_threads, FALSE);
start_kernel_threads_blocked = TRUE;
simple_unlock(&kernel_test_lock);
splx(s);
thread_block((void (*)(void)) 0);
s = splsched();
simple_lock(&kernel_test_lock);
start_kernel_threads_blocked = FALSE;
}
kernel_test_thread_sync_done = FALSE; /* Reset for next use */
simple_unlock(&kernel_test_lock);
splx(s);
}
#endif /* KERNEL_TEST */
/*
* Start the user bootstrap.
*/
bootstrap_create();
#if XPR_DEBUG
xprinit(); /* XXX */
#endif /* XPR_DEBUG */
#if NCPUS > 1
/*
* Create the shutdown thread.
*/
(void) kernel_thread(kernel_task, action_thread, (char *) 0);
/*
* Allow other CPUs to run.
*
* (this must be last, to allow bootstrap_create to fiddle with
* its child thread before some cpu tries to run it)
*/
start_other_cpus();
#endif /* NCPUS > 1 */
/*
* Become the pageout daemon.
*/
(void) spllo();
vm_pageout();
/*NOTREACHED*/
}
/*
* thread_set_mode_and_absolute_pri:
*
* Set scheduling policy & absolute priority for thread, for deprecated
* thread_set_policy and thread_policy interfaces.
*
* Note that there is no implemented difference between POLICY_RR and POLICY_FIFO.
* Both result in FIXED mode scheduling.
*
* Called with thread mutex locked.
*/
kern_return_t
thread_set_mode_and_absolute_pri(
thread_t thread,
integer_t policy,
integer_t priority)
{
spl_t s;
sched_mode_t mode;
kern_return_t kr = KERN_SUCCESS;
if (thread_is_static_param(thread))
return (KERN_POLICY_STATIC);
if (thread->policy_reset)
return (KERN_SUCCESS);
/* Setting legacy policies on threads kills the current QoS */
if (thread->requested_policy.thrp_qos != THREAD_QOS_UNSPECIFIED) {
thread_mtx_unlock(thread);
kr = thread_remove_qos_policy(thread);
thread_mtx_lock(thread);
if (!thread->active) {
return (KERN_TERMINATED);
}
}
switch (policy) {
case POLICY_TIMESHARE:
mode = TH_MODE_TIMESHARE;
break;
case POLICY_RR:
case POLICY_FIFO:
mode = TH_MODE_FIXED;
break;
default:
panic("unexpected sched policy: %d", policy);
break;
}
s = splsched();
thread_lock(thread);
/* This path isn't allowed to change a thread out of realtime. */
if ((thread->sched_mode != TH_MODE_REALTIME) &&
(thread->saved_mode != TH_MODE_REALTIME)) {
/*
* Reverse engineer and apply the correct importance value
* from the requested absolute priority value.
*/
if (priority >= thread->max_priority)
priority = thread->max_priority - thread->task_priority;
else if (priority >= MINPRI_KERNEL)
priority -= MINPRI_KERNEL;
else if (priority >= MINPRI_RESERVED)
priority -= MINPRI_RESERVED;
else
priority -= BASEPRI_DEFAULT;
priority += thread->task_priority;
if (priority > thread->max_priority)
priority = thread->max_priority;
else if (priority < MINPRI)
priority = MINPRI;
thread->importance = priority - thread->task_priority;
boolean_t removed = thread_run_queue_remove(thread);
thread_set_user_sched_mode(thread, mode);
thread_recompute_priority(thread);
if (removed)
thread_setrun(thread, SCHED_TAILQ);
}
thread_unlock(thread);
splx(s);
sfi_reevaluate(thread);
return (kr);
}
kern_return_t
thread_policy_set_internal(
thread_t thread,
thread_policy_flavor_t flavor,
thread_policy_t policy_info,
mach_msg_type_number_t count)
{
kern_return_t result = KERN_SUCCESS;
spl_t s;
thread_mtx_lock(thread);
if (!thread->active) {
thread_mtx_unlock(thread);
return (KERN_TERMINATED);
}
switch (flavor) {
case THREAD_EXTENDED_POLICY:
{
boolean_t timeshare = TRUE;
if (count >= THREAD_EXTENDED_POLICY_COUNT) {
thread_extended_policy_t info;
info = (thread_extended_policy_t)policy_info;
timeshare = info->timeshare;
}
sched_mode_t mode = (timeshare == TRUE) ? TH_MODE_TIMESHARE : TH_MODE_FIXED;
s = splsched();
thread_lock(thread);
boolean_t removed = thread_run_queue_remove(thread);
thread_set_user_sched_mode(thread, mode);
thread_recompute_priority(thread);
if (removed)
thread_setrun(thread, SCHED_TAILQ);
thread_unlock(thread);
splx(s);
sfi_reevaluate(thread);
break;
}
case THREAD_TIME_CONSTRAINT_POLICY:
{
thread_time_constraint_policy_t info;
if (count < THREAD_TIME_CONSTRAINT_POLICY_COUNT) {
result = KERN_INVALID_ARGUMENT;
break;
}
info = (thread_time_constraint_policy_t)policy_info;
if ( info->constraint < info->computation ||
info->computation > max_rt_quantum ||
info->computation < min_rt_quantum ) {
result = KERN_INVALID_ARGUMENT;
break;
}
s = splsched();
thread_lock(thread);
boolean_t removed = thread_run_queue_remove(thread);
thread->realtime.period = info->period;
thread->realtime.computation = info->computation;
thread->realtime.constraint = info->constraint;
thread->realtime.preemptible = info->preemptible;
thread_set_user_sched_mode(thread, TH_MODE_REALTIME);
thread_recompute_priority(thread);
if (removed)
thread_setrun(thread, SCHED_TAILQ);
thread_unlock(thread);
splx(s);
sfi_reevaluate(thread);
break;
}
case THREAD_PRECEDENCE_POLICY:
{
thread_precedence_policy_t info;
if (count < THREAD_PRECEDENCE_POLICY_COUNT) {
result = KERN_INVALID_ARGUMENT;
break;
}
info = (thread_precedence_policy_t)policy_info;
s = splsched();
thread_lock(thread);
thread->importance = info->importance;
thread_recompute_priority(thread);
thread_unlock(thread);
splx(s);
break;
}
case THREAD_AFFINITY_POLICY:
{
thread_affinity_policy_t info;
if (!thread_affinity_is_supported()) {
result = KERN_NOT_SUPPORTED;
break;
}
if (count < THREAD_AFFINITY_POLICY_COUNT) {
result = KERN_INVALID_ARGUMENT;
break;
}
info = (thread_affinity_policy_t) policy_info;
/*
* Unlock the thread mutex here and
* return directly after calling thread_affinity_set().
* This is necessary for correct lock ordering because
* thread_affinity_set() takes the task lock.
*/
thread_mtx_unlock(thread);
return thread_affinity_set(thread, info->affinity_tag);
}
case THREAD_THROUGHPUT_QOS_POLICY:
{
thread_throughput_qos_policy_t info = (thread_throughput_qos_policy_t) policy_info;
int tqos;
if (count < THREAD_LATENCY_QOS_POLICY_COUNT) {
result = KERN_INVALID_ARGUMENT;
break;
}
if ((result = qos_throughput_policy_validate(info->thread_throughput_qos_tier)) !=
KERN_SUCCESS) {
break;
}
tqos = qos_extract(info->thread_throughput_qos_tier);
thread->effective_policy.t_through_qos = tqos;
}
break;
case THREAD_LATENCY_QOS_POLICY:
{
thread_latency_qos_policy_t info = (thread_latency_qos_policy_t) policy_info;
int lqos;
if (count < THREAD_THROUGHPUT_QOS_POLICY_COUNT) {
result = KERN_INVALID_ARGUMENT;
break;
}
if ((result = qos_latency_policy_validate(info->thread_latency_qos_tier)) !=
KERN_SUCCESS) {
break;
}
lqos = qos_extract(info->thread_latency_qos_tier);
/* The expected use cases (opt-in) of per-thread latency QoS would seem to
* preclude any requirement at present to re-evaluate timers on a thread level
* latency QoS change.
*/
thread->effective_policy.t_latency_qos = lqos;
}
break;
case THREAD_QOS_POLICY:
case THREAD_QOS_POLICY_OVERRIDE:
{
thread_qos_policy_t info = (thread_qos_policy_t)policy_info;
if (count < THREAD_QOS_POLICY_COUNT) {
result = KERN_INVALID_ARGUMENT;
break;
}
if (info->qos_tier < 0 || info->qos_tier >= THREAD_QOS_LAST) {
result = KERN_INVALID_ARGUMENT;
break;
}
if (info->tier_importance > 0 || info->tier_importance < THREAD_QOS_MIN_TIER_IMPORTANCE) {
result = KERN_INVALID_ARGUMENT;
break;
}
if (info->qos_tier == THREAD_QOS_UNSPECIFIED && info->tier_importance != 0) {
result = KERN_INVALID_ARGUMENT;
break;
}
/*
* Going into task policy requires the task mutex,
* because of the way synchronization against the IO policy
* subsystem works.
*
* We need to move thread policy to the thread mutex instead.
* <rdar://problem/15831652> separate thread policy from task policy
*/
if (flavor == THREAD_QOS_POLICY_OVERRIDE) {
int strongest_override = info->qos_tier;
if (info->qos_tier != THREAD_QOS_UNSPECIFIED &&
thread->requested_policy.thrp_qos_override != THREAD_QOS_UNSPECIFIED)
strongest_override = MAX(thread->requested_policy.thrp_qos_override, info->qos_tier);
thread_mtx_unlock(thread);
/* There is a race here. To be closed in <rdar://problem/15831652> separate thread policy from task policy */
proc_set_task_policy(thread->task, thread, TASK_POLICY_ATTRIBUTE, TASK_POLICY_QOS_OVERRIDE, strongest_override);
return (result);
}
thread_mtx_unlock(thread);
proc_set_task_policy2(thread->task, thread, TASK_POLICY_ATTRIBUTE, TASK_POLICY_QOS_AND_RELPRIO, info->qos_tier, -info->tier_importance);
thread_mtx_lock(thread);
if (!thread->active) {
thread_mtx_unlock(thread);
return (KERN_TERMINATED);
}
break;
}
default:
result = KERN_INVALID_ARGUMENT;
break;
}
thread_mtx_unlock(thread);
return (result);
}
