kern_return_t
thread_get_special_port(
thread_act_t thr_act,
int which,
ipc_port_t *portp)
{
ipc_port_t *whichp;
ipc_port_t port;
thread_t thread;
if (!thr_act)
return KERN_INVALID_ARGUMENT;
thread = act_lock_thread(thr_act);
switch (which) {
case THREAD_KERNEL_PORT:
whichp = &thr_act->ith_sself;
break;
default:
act_unlock_thread(thr_act);
return KERN_INVALID_ARGUMENT;
}
if (!thr_act->active) {
act_unlock_thread(thr_act);
return KERN_FAILURE;
}
port = ipc_port_copy_send(*whichp);
act_unlock_thread(thr_act);
*portp = port;
return KERN_SUCCESS;
}
kern_return_t
thread_abort_safely(
thread_act_t act)
{
thread_t thread;
kern_return_t ret;
spl_t s;
if ( act == THR_ACT_NULL )
return (KERN_INVALID_ARGUMENT);
thread = act_lock_thread(act);
if (!act->active) {
act_unlock_thread(act);
return (KERN_TERMINATED);
}
s = splsched();
thread_lock(thread);
if (!thread->at_safe_point ||
clear_wait_internal(thread, THREAD_INTERRUPTED) != KERN_SUCCESS) {
if (!(thread->state & TH_ABORT)) {
thread->state |= (TH_ABORT|TH_ABORT_SAFELY);
install_special_handler_locked(act);
}
}
thread_unlock(thread);
splx(s);
act_unlock_thread(act);
return (KERN_SUCCESS);
}
kern_return_t
thread_info(
thread_act_t thr_act,
thread_flavor_t flavor,
thread_info_t thread_info_out,
mach_msg_type_number_t *thread_info_count)
{
register thread_t thread;
kern_return_t result;
if (thr_act == THR_ACT_NULL)
return (KERN_INVALID_ARGUMENT);
thread = act_lock_thread(thr_act);
if (!thr_act->active) {
act_unlock_thread(thr_act);
return (KERN_TERMINATED);
}
result = thread_info_shuttle(thr_act, flavor,
thread_info_out, thread_info_count);
act_unlock_thread(thr_act);
return (result);
}
/*
* Change thread's machine-dependent state. Called with nothing
* locked. Returns same way.
*/
kern_return_t
thread_set_state(
register thread_act_t act,
int flavor,
thread_state_t state,
mach_msg_type_number_t state_count)
{
kern_return_t result = KERN_SUCCESS;
thread_t thread;
if (act == THR_ACT_NULL || act == current_act())
return (KERN_INVALID_ARGUMENT);
thread = act_lock_thread(act);
if (!act->active) {
act_unlock_thread(act);
return (KERN_TERMINATED);
}
thread_hold(act);
for (;;) {
thread_t thread1;
if ( thread == THREAD_NULL ||
thread->top_act != act )
break;
act_unlock_thread(act);
if (!thread_stop(thread)) {
result = KERN_ABORTED;
(void)act_lock_thread(act);
thread = THREAD_NULL;
break;
}
thread1 = act_lock_thread(act);
if (thread1 == thread)
break;
thread_unstop(thread);
thread = thread1;
}
if (result == KERN_SUCCESS)
result = machine_thread_set_state(act, flavor, state, state_count);
if ( thread != THREAD_NULL &&
thread->top_act == act )
thread_unstop(thread);
thread_release(act);
act_unlock_thread(act);
return (result);
}
kern_return_t
thread_dup(
register thread_act_t target)
{
kern_return_t result = KERN_SUCCESS;
thread_act_t self = current_act();
thread_t thread;
if (target == THR_ACT_NULL || target == self)
return (KERN_INVALID_ARGUMENT);
thread = act_lock_thread(target);
if (!target->active) {
act_unlock_thread(target);
return (KERN_TERMINATED);
}
thread_hold(target);
for (;;) {
thread_t thread1;
if ( thread == THREAD_NULL ||
thread->top_act != target )
break;
act_unlock_thread(target);
if (!thread_stop(thread)) {
result = KERN_ABORTED;
(void)act_lock_thread(target);
thread = THREAD_NULL;
break;
}
thread1 = act_lock_thread(target);
if (thread1 == thread)
break;
thread_unstop(thread);
thread = thread1;
}
if (result == KERN_SUCCESS)
result = machine_thread_dup(self, target);
if ( thread != THREAD_NULL &&
thread->top_act == target )
thread_unstop(thread);
thread_release(target);
act_unlock_thread(target);
return (result);
}
kern_return_t check_actforsig(task_t task, thread_act_t thact, int setast)
{
thread_act_t inc;
thread_act_t ninc;
thread_act_t thr_act;
thread_t th;
int found=0;
task_lock(task);
if (!task->active) {
task_unlock(task);
return(KERN_FAILURE);
}
thr_act = THR_ACT_NULL;
for (inc = (thread_act_t)queue_first(&task->threads);
!queue_end(&task->threads, (queue_entry_t)inc);
inc = ninc) {
if (inc != thact) {
ninc = (thread_act_t)queue_next(&inc->task_threads);
continue;
}
th = act_lock_thread(inc);
if ((inc->active) &&
((th->state & (TH_ABORT|TH_ABORT_SAFELY)) != TH_ABORT)) {
found = 1;
thr_act = inc;
break;
}
act_unlock_thread(inc);
/* ninc = (thread_act_t)queue_next(&inc->thr_acts); */
break;
}
out:
if (found) {
if (setast)
act_set_astbsd(thr_act);
act_unlock_thread(thr_act);
}
task_unlock(task);
if (found)
return(KERN_SUCCESS);
else
return(KERN_FAILURE);
}
/*
* thread_getstatus:
*
* Get the status of the specified thread.
*/
kern_return_t
thread_getstatus(
register thread_act_t act,
int flavor,
thread_state_t tstate,
mach_msg_type_number_t *count)
{
kern_return_t result = KERN_SUCCESS;
thread_t thread;
thread = act_lock_thread(act);
if ( act != current_act() &&
(act->suspend_count == 0 ||
thread == THREAD_NULL ||
(thread->state & TH_RUN) ||
thread->top_act != act) )
result = KERN_FAILURE;
if (result == KERN_SUCCESS)
result = machine_thread_get_state(act, flavor, tstate, count);
act_unlock_thread(act);
return (result);
}
kern_return_t get_signalact(task_t task,thread_act_t * thact, int setast)
{
thread_act_t inc;
thread_act_t ninc;
thread_act_t thr_act;
thread_t th;
task_lock(task);
if (!task->active) {
task_unlock(task);
return(KERN_FAILURE);
}
thr_act = THR_ACT_NULL;
for (inc = (thread_act_t)queue_first(&task->threads);
!queue_end(&task->threads, (queue_entry_t)inc);
inc = ninc) {
th = act_lock_thread(inc);
if ((inc->active) &&
((th->state & (TH_ABORT|TH_ABORT_SAFELY)) != TH_ABORT)) {
thr_act = inc;
break;
}
act_unlock_thread(inc);
ninc = (thread_act_t)queue_next(&inc->task_threads);
}
out:
if (thact)
*thact = thr_act;
if (thr_act) {
if (setast)
act_set_astbsd(thr_act);
act_unlock_thread(thr_act);
}
task_unlock(task);
if (thr_act)
return(KERN_SUCCESS);
else
return(KERN_FAILURE);
}
/*
* Internal routine to terminate a thread.
* Sometimes called with task already locked.
*/
kern_return_t
thread_terminate_internal(
register thread_act_t act)
{
kern_return_t result;
thread_t thread;
thread = act_lock_thread(act);
if (!act->active) {
act_unlock_thread(act);
return (KERN_TERMINATED);
}
act_disable(act);
result = act_abort(act, FALSE);
/*
* Make sure this thread enters the kernel
* Must unlock the act, but leave the shuttle
* captured in this act.
*/
if (thread != current_thread()) {
act_unlock(act);
if (thread_stop(thread))
thread_unstop(thread);
else
result = KERN_ABORTED;
act_lock(act);
}
clear_wait(thread, act->started? THREAD_INTERRUPTED: THREAD_AWAKENED);
act_unlock_thread(act);
return (result);
}
/*
* thread_stop_freeze
* Block the thread in the kernel and freeze the processor set.
* return value:
* TRUE - the thread has blocked interruptibly, is stopped, and
* the processor set assignment is frozen
* FALSE - the thread is no longer in the processor set, so it
* isn't stopped, and the processor set assignment
* is released.
*/
int
thread_stop_freeze( thread_t thread, processor_set_t pset )
{
thread_act_t thr_act;
spl_t s;
/*
* hold it, and wait for it to stop.
*/
thr_act = thread_lock_act(thread);
thread_hold(thr_act);
act_unlock_thread(thr_act);
thread_stop(thread);
s = splsched();
wake_lock(thread);
while( thread->state & (TH_RUN|TH_UNINT) ) {
thread->wake_active = TRUE;
assert_wait((event_t)&thread->wake_active, FALSE);
wake_unlock(thread);
splx(s);
thread_block( (void (*)(void)) 0 );
(void) splsched();
wake_lock(thread);
}
/*
* Now, the thread has blocked uninterruptibly; freeze the
* assignment and make sure it's still part of the processor set.
*/
wake_unlock(thread);
thread_freeze(thread);
thread_lock(thread);
/*
* if the processor set has changed, release the freeze and
* then unstop it.
*/
if( thread->processor_set != pset ) {
thread_unlock(thread);
splx(s);
thread_unfreeze(thread);
thread_unstop(thread);
return FALSE;
}
thread_unlock(thread);
splx(s);
return TRUE;
}
kern_return_t
thread_abort(
register thread_act_t act)
{
kern_return_t result;
thread_t thread;
if (act == THR_ACT_NULL)
return (KERN_INVALID_ARGUMENT);
thread = act_lock_thread(act);
if (!act->active) {
act_unlock_thread(act);
return (KERN_TERMINATED);
}
result = act_abort(act, FALSE);
clear_wait(thread, THREAD_INTERRUPTED);
act_unlock_thread(act);
return (result);
}
kern_return_t
thread_set_special_port(
thread_act_t thr_act,
int which,
ipc_port_t port)
{
ipc_port_t *whichp;
ipc_port_t old;
thread_t thread;
if (thr_act == 0)
return KERN_INVALID_ARGUMENT;
thread = act_lock_thread(thr_act);
switch (which) {
case THREAD_KERNEL_PORT:
whichp = &thr_act->ith_self;
break;
default:
act_unlock_thread(thr_act);
return KERN_INVALID_ARGUMENT;
}
if (!thr_act->active) {
act_unlock_thread(thr_act);
return KERN_FAILURE;
}
old = *whichp;
*whichp = port;
act_unlock_thread(thr_act);
if (IP_VALID(old))
ipc_port_release_send(old);
return KERN_SUCCESS;
}
kern_return_t
thread_suspend(
register thread_act_t act)
{
thread_t thread;
if (act == THR_ACT_NULL || act->task == kernel_task)
return (KERN_INVALID_ARGUMENT);
thread = act_lock_thread(act);
if (!act->active) {
act_unlock_thread(act);
return (KERN_TERMINATED);
}
if ( act->user_stop_count++ == 0 &&
act->suspend_count++ == 0 ) {
install_special_handler(act);
if ( thread != current_thread() &&
thread != THREAD_NULL &&
thread->top_act == act ) {
assert(act->started);
thread_wakeup_one(&act->suspend_count);
act_unlock_thread(act);
thread_wait(thread);
}
else
act_unlock_thread(act);
}
else
act_unlock_thread(act);
return (KERN_SUCCESS);
}
/*
* thread_depress_abort:
*
* Prematurely abort priority depression if there is one.
*/
kern_return_t
thread_depress_abort(
register thread_act_t thr_act)
{
register thread_t thread;
kern_return_t result;
if (thr_act == THR_ACT_NULL)
return (KERN_INVALID_ARGUMENT);
thread = act_lock_thread(thr_act);
/* if activation is terminating, this operation is not meaningful */
if (!thr_act->active) {
act_unlock_thread(thr_act);
return (KERN_TERMINATED);
}
result = _mk_sp_thread_depress_abort(thread, FALSE);
act_unlock_thread(thr_act);
return (result);
}
kern_return_t
thread_resume(
register thread_act_t act)
{
kern_return_t result = KERN_SUCCESS;
thread_t thread;
if (act == THR_ACT_NULL || act->task == kernel_task)
return (KERN_INVALID_ARGUMENT);
thread = act_lock_thread(act);
if (act->active) {
if (act->user_stop_count > 0) {
if ( --act->user_stop_count == 0 &&
--act->suspend_count == 0 &&
thread != THREAD_NULL &&
thread->top_act == act ) {
if (!act->started) {
clear_wait(thread, THREAD_AWAKENED);
act->started = TRUE;
}
else
thread_wakeup_one(&act->suspend_count);
}
}
else
result = KERN_FAILURE;
}
else
result = KERN_TERMINATED;
act_unlock_thread(act);
return (result);
}
kern_return_t
thread_policy_set(
thread_act_t act,
thread_policy_flavor_t flavor,
thread_policy_t policy_info,
mach_msg_type_number_t count)
{
kern_return_t result = KERN_SUCCESS;
thread_t thread;
spl_t s;
if (act == THR_ACT_NULL)
return (KERN_INVALID_ARGUMENT);
thread = act_lock_thread(act);
if (!act->active) {
act_unlock_thread(act);
return (KERN_TERMINATED);
}
assert(thread != THREAD_NULL);
switch (flavor) {
case THREAD_EXTENDED_POLICY:
{
boolean_t timeshare = TRUE;
if (count >= THREAD_EXTENDED_POLICY_COUNT) {
thread_extended_policy_t info;
info = (thread_extended_policy_t)policy_info;
timeshare = info->timeshare;
}
s = splsched();
thread_lock(thread);
if (!(thread->sched_mode & TH_MODE_FAILSAFE)) {
integer_t oldmode = (thread->sched_mode & TH_MODE_TIMESHARE);
thread->sched_mode &= ~TH_MODE_REALTIME;
if (timeshare && !oldmode) {
thread->sched_mode |= TH_MODE_TIMESHARE;
if (thread->state & TH_RUN)
pset_share_incr(thread->processor_set);
}
else if (!timeshare && oldmode) {
thread->sched_mode &= ~TH_MODE_TIMESHARE;
if (thread->state & TH_RUN)
pset_share_decr(thread->processor_set);
}
thread_recompute_priority(thread);
}
else {
thread->safe_mode &= ~TH_MODE_REALTIME;
if (timeshare)
thread->safe_mode |= TH_MODE_TIMESHARE;
else
thread->safe_mode &= ~TH_MODE_TIMESHARE;
}
thread_unlock(thread);
splx(s);
break;
}
case THREAD_TIME_CONSTRAINT_POLICY:
{
thread_time_constraint_policy_t info;
if (count < THREAD_TIME_CONSTRAINT_POLICY_COUNT) {
result = KERN_INVALID_ARGUMENT;
break;
}
info = (thread_time_constraint_policy_t)policy_info;
if ( info->constraint < info->computation ||
info->computation > max_rt_quantum ||
info->computation < min_rt_quantum ) {
result = KERN_INVALID_ARGUMENT;
break;
}
s = splsched();
thread_lock(thread);
thread->realtime.period = info->period;
thread->realtime.computation = info->computation;
thread->realtime.constraint = info->constraint;
thread->realtime.preemptible = info->preemptible;
if (!(thread->sched_mode & TH_MODE_FAILSAFE)) {
if (thread->sched_mode & TH_MODE_TIMESHARE) {
thread->sched_mode &= ~TH_MODE_TIMESHARE;
if (thread->state & TH_RUN)
pset_share_decr(thread->processor_set);
}
thread->sched_mode |= TH_MODE_REALTIME;
thread_recompute_priority(thread);
}
else {
thread->safe_mode &= ~TH_MODE_TIMESHARE;
thread->safe_mode |= TH_MODE_REALTIME;
}
thread_unlock(thread);
splx(s);
break;
}
case THREAD_PRECEDENCE_POLICY:
{
thread_precedence_policy_t info;
if (count < THREAD_PRECEDENCE_POLICY_COUNT) {
result = KERN_INVALID_ARGUMENT;
break;
}
info = (thread_precedence_policy_t)policy_info;
s = splsched();
thread_lock(thread);
thread->importance = info->importance;
thread_recompute_priority(thread);
thread_unlock(thread);
splx(s);
break;
}
default:
result = KERN_INVALID_ARGUMENT;
break;
}
act_unlock_thread(act);
return (result);
}
kern_return_t
thread_policy_get(
thread_act_t act,
thread_policy_flavor_t flavor,
thread_policy_t policy_info,
mach_msg_type_number_t *count,
boolean_t *get_default)
{
kern_return_t result = KERN_SUCCESS;
thread_t thread;
spl_t s;
if (act == THR_ACT_NULL)
return (KERN_INVALID_ARGUMENT);
thread = act_lock_thread(act);
if (!act->active) {
act_unlock_thread(act);
return (KERN_TERMINATED);
}
assert(thread != THREAD_NULL);
switch (flavor) {
case THREAD_EXTENDED_POLICY:
{
boolean_t timeshare = TRUE;
if (!(*get_default)) {
s = splsched();
thread_lock(thread);
if ( !(thread->sched_mode & TH_MODE_REALTIME) &&
!(thread->safe_mode & TH_MODE_REALTIME) ) {
if (!(thread->sched_mode & TH_MODE_FAILSAFE))
timeshare = (thread->sched_mode & TH_MODE_TIMESHARE) != 0;
else
timeshare = (thread->safe_mode & TH_MODE_TIMESHARE) != 0;
}
else
*get_default = TRUE;
thread_unlock(thread);
splx(s);
}
if (*count >= THREAD_EXTENDED_POLICY_COUNT) {
thread_extended_policy_t info;
info = (thread_extended_policy_t)policy_info;
info->timeshare = timeshare;
}
break;
}
case THREAD_TIME_CONSTRAINT_POLICY:
{
thread_time_constraint_policy_t info;
if (*count < THREAD_TIME_CONSTRAINT_POLICY_COUNT) {
result = KERN_INVALID_ARGUMENT;
break;
}
info = (thread_time_constraint_policy_t)policy_info;
if (!(*get_default)) {
s = splsched();
thread_lock(thread);
if ( (thread->sched_mode & TH_MODE_REALTIME) ||
(thread->safe_mode & TH_MODE_REALTIME) ) {
info->period = thread->realtime.period;
info->computation = thread->realtime.computation;
info->constraint = thread->realtime.constraint;
info->preemptible = thread->realtime.preemptible;
}
else
*get_default = TRUE;
thread_unlock(thread);
splx(s);
}
if (*get_default) {
info->period = 0;
info->computation = std_quantum / 2;
info->constraint = std_quantum;
info->preemptible = TRUE;
}
break;
}
case THREAD_PRECEDENCE_POLICY:
{
thread_precedence_policy_t info;
if (*count < THREAD_PRECEDENCE_POLICY_COUNT) {
result = KERN_INVALID_ARGUMENT;
break;
}
info = (thread_precedence_policy_t)policy_info;
if (!(*get_default)) {
s = splsched();
thread_lock(thread);
info->importance = thread->importance;
thread_unlock(thread);
splx(s);
}
else
info->importance = 0;
break;
}
default:
result = KERN_INVALID_ARGUMENT;
break;
}
act_unlock_thread(act);
return (result);
}
/*
* task_swap_swapout_thread: [exported]
*
* Executes as a separate kernel thread.
* Its job is to swap out threads that have been halted by AST_SWAPOUT.
*/
void
task_swap_swapout_thread(void)
{
thread_act_t thr_act;
thread_t thread, nthread;
task_t task;
int s;
thread_swappable(current_act(), FALSE);
stack_privilege(current_thread());
spllo();
while (TRUE) {
task_swapper_lock();
while (! queue_empty(&swapout_thread_q)) {
queue_remove_first(&swapout_thread_q, thr_act,
thread_act_t, swap_queue);
/*
* If we're racing with task_swapin, we need
* to make it safe for it to do remque on the
* thread, so make its links point to itself.
* Allowing this ugliness is cheaper than
* making task_swapin search the entire queue.
*/
act_lock(thr_act);
queue_init((queue_t) &thr_act->swap_queue);
act_unlock(thr_act);
task_swapper_unlock();
/*
* Wait for thread's RUN bit to be deasserted.
*/
thread = act_lock_thread(thr_act);
if (thread == THREAD_NULL)
act_unlock_thread(thr_act);
else {
boolean_t r;
thread_reference(thread);
thread_hold(thr_act);
act_unlock_thread(thr_act);
r = thread_stop_wait(thread);
nthread = act_lock_thread(thr_act);
thread_release(thr_act);
thread_deallocate(thread);
act_unlock_thread(thr_act);
if (!r || nthread != thread) {
task_swapper_lock();
continue;
}
}
task = thr_act->task;
task_lock(task);
/*
* we can race with swapin, which would set the
* state to TASK_SW_IN.
*/
if ((task->swap_state != TASK_SW_OUT) &&
(task->swap_state != TASK_SW_GOING_OUT)) {
task_unlock(task);
task_swapper_lock();
TASK_STATS_INCR(task_sw_race_in_won);
if (thread != THREAD_NULL)
thread_unstop(thread);
continue;
}
nthread = act_lock_thread(thr_act);
if (nthread != thread || thr_act->active == FALSE) {
act_unlock_thread(thr_act);
task_unlock(task);
task_swapper_lock();
TASK_STATS_INCR(task_sw_act_inactive);
if (thread != THREAD_NULL)
thread_unstop(thread);
continue;
}
s = splsched();
if (thread != THREAD_NULL)
thread_lock(thread);
/*
* Thread cannot have been swapped out yet because
* TH_SW_TASK_SWAPPING was set in AST. If task_swapin
* beat us here, we either wouldn't have found it on
* the queue, or the task->swap_state would have
* changed. The synchronization is on the
* task's swap_state and the task_lock.
* The thread can't be swapped in any other way
* because its task has been swapped.
*/
assert(thr_act->swap_state & TH_SW_TASK_SWAPPING);
assert(thread == THREAD_NULL ||
!(thread->state & (TH_SWAPPED_OUT|TH_RUN)));
assert((thr_act->swap_state & TH_SW_STATE) == TH_SW_IN);
/* assert(thread->state & TH_HALTED); */
/* this also clears TH_SW_TASK_SWAPPING flag */
thr_act->swap_state = TH_SW_GOING_OUT;
if (thread != THREAD_NULL) {
if (thread->top_act == thr_act) {
thread->state |= TH_SWAPPED_OUT;
/*
* Once we unlock the task, things can happen
* to the thread, so make sure it's consistent
* for thread_swapout.
*/
}
thread->ref_count++;
thread_unlock(thread);
thread_unstop(thread);
}
splx(s);
act_locked_act_reference(thr_act);
act_unlock_thread(thr_act);
task_unlock(task);
thread_swapout(thr_act); /* do the work */
if (thread != THREAD_NULL)
thread_deallocate(thread);
act_deallocate(thr_act);
task_swapper_lock();
}
assert_wait((event_t)&swapout_thread_q, FALSE);
task_swapper_unlock();
thread_block((void (*)(void)) 0);
}
}
kern_return_t
task_swapin(task_t task, boolean_t make_unswappable)
{
register queue_head_t *list;
register thread_act_t thr_act, next;
thread_t thread;
int s;
boolean_t swappable = TRUE;
task_lock(task);
switch (task->swap_state) {
case TASK_SW_OUT:
{
vm_map_t map = task->map;
/*
* Task has made it all the way out, which means
* that vm_map_res_deallocate has been done; set
* state to TASK_SW_COMING_IN, then bring map
* back in. We could actually be racing with
* the thread_swapout_enqueue, which does the
* vm_map_res_deallocate, but that race is covered.
*/
task->swap_state = TASK_SW_COMING_IN;
assert(task->swap_ast_waiting == 0);
assert(map->res_count >= 0);
task_unlock(task);
mutex_lock(&map->s_lock);
vm_map_res_reference(map);
mutex_unlock(&map->s_lock);
task_lock(task);
assert(task->swap_state == TASK_SW_COMING_IN);
}
break;
case TASK_SW_GOING_OUT:
/*
* Task isn't all the way out yet. There is
* still at least one thread not swapped, and
* vm_map_res_deallocate has not been done.
*/
task->swap_state = TASK_SW_COMING_IN;
assert(task->swap_ast_waiting > 0 ||
(task->swap_ast_waiting == 0 &&
task->thr_act_count == 0));
assert(task->map->res_count > 0);
TASK_STATS_INCR(task_sw_race_going_out);
break;
case TASK_SW_IN:
assert(task->map->res_count > 0);
#if TASK_SW_DEBUG
task_swapper_lock();
if (task_swap_debug && on_swapped_list(task)) {
printf("task 0x%X on list, state is SW_IN\n",
task);
Debugger("");
}
task_swapper_unlock();
#endif /* TASK_SW_DEBUG */
TASK_STATS_INCR(task_sw_race_in);
if (make_unswappable) {
task->swap_state = TASK_SW_UNSWAPPABLE;
task_unlock(task);
task_swapout_ineligible(task);
} else
task_unlock(task);
return(KERN_SUCCESS);
case TASK_SW_COMING_IN:
/*
* Raced with another task_swapin and lost;
* wait for other one to complete first
*/
assert(task->map->res_count >= 0);
/*
* set MAKE_UNSWAPPABLE so that whoever is swapping
* the task in will make it unswappable, and return
*/
if (make_unswappable)
task->swap_flags |= TASK_SW_MAKE_UNSWAPPABLE;
task->swap_flags |= TASK_SW_WANT_IN;
assert_wait((event_t)&task->swap_state, FALSE);
task_unlock(task);
thread_block((void (*)(void)) 0);
TASK_STATS_INCR(task_sw_race_coming_in);
return(KERN_SUCCESS);
case TASK_SW_UNSWAPPABLE:
/*
* This can happen, since task_terminate
* unconditionally calls task_swapin.
*/
task_unlock(task);
return(KERN_SUCCESS);
default:
panic("task_swapin bad state");
break;
}
if (make_unswappable)
task->swap_flags |= TASK_SW_MAKE_UNSWAPPABLE;
assert(task->swap_state == TASK_SW_COMING_IN);
task_swapper_lock();
#if TASK_SW_DEBUG
if (task_swap_debug && !on_swapped_list(task)) {
printf("task 0x%X not on list\n", task);
Debugger("");
}
#endif /* TASK_SW_DEBUG */
queue_remove(&swapped_tasks, task, task_t, swapped_tasks);
tasks_swapped_out--;
task_swapins++;
task_swapper_unlock();
/*
* Iterate through all threads for this task and
* release them, as required. They may not have been swapped
* out yet. The task remains locked throughout.
*/
list = &task->thr_acts;
thr_act = (thread_act_t) queue_first(list);
while (!queue_end(list, (queue_entry_t) thr_act)) {
boolean_t need_to_release;
next = (thread_act_t) queue_next(&thr_act->thr_acts);
/*
* Keep task_swapper_lock across thread handling
* to synchronize with task_swap_swapout_thread
*/
task_swapper_lock();
thread = act_lock_thread(thr_act);
s = splsched();
if (thr_act->ast & AST_SWAPOUT) {
/* thread hasn't gotten the AST yet, just clear it */
thread_ast_clear(thr_act, AST_SWAPOUT);
need_to_release = FALSE;
TASK_STATS_INCR(task_sw_before_ast);
splx(s);
act_unlock_thread(thr_act);
} else {
/*
* If AST_SWAPOUT was cleared, then thread_hold,
* or equivalent was done.
*/
need_to_release = TRUE;
/*
* Thread has hit AST, but it may not have
* been dequeued yet, so we need to check.
* NOTE: the thread may have been dequeued, but
* has not yet been swapped (the task_swapper_lock
* has been dropped, but the thread is not yet
* locked), and the TH_SW_TASK_SWAPPING flag may
* not have been cleared. In this case, we will do
* an extra remque, which the task_swap_swapout_thread
* has made safe, and clear the flag, which is also
* checked by the t_s_s_t before doing the swapout.
*/
if (thread)
thread_lock(thread);
if (thr_act->swap_state & TH_SW_TASK_SWAPPING) {
/*
* hasn't yet been dequeued for swapout,
* so clear flags and dequeue it first.
*/
thr_act->swap_state &= ~TH_SW_TASK_SWAPPING;
assert(thr_act->thread == THREAD_NULL ||
!(thr_act->thread->state &
TH_SWAPPED_OUT));
queue_remove(&swapout_thread_q, thr_act,
thread_act_t, swap_queue);
TASK_STATS_INCR(task_sw_before_swap);
} else {
TASK_STATS_INCR(task_sw_after_swap);
/*
* It's possible that the thread was
* made unswappable before hitting the
* AST, in which case it's still running.
*/
if (thr_act->swap_state == TH_SW_UNSWAPPABLE) {
need_to_release = FALSE;
TASK_STATS_INCR(task_sw_unswappable);
}
}
if (thread)
thread_unlock(thread);
splx(s);
act_unlock_thread(thr_act);
}
task_swapper_unlock();
/*
* thread_release will swap in the thread if it's been
* swapped out.
*/
if (need_to_release) {
act_lock_thread(thr_act);
thread_release(thr_act);
act_unlock_thread(thr_act);
}
thr_act = next;
}
if (task->swap_flags & TASK_SW_MAKE_UNSWAPPABLE) {
task->swap_flags &= ~TASK_SW_MAKE_UNSWAPPABLE;
task->swap_state = TASK_SW_UNSWAPPABLE;
swappable = FALSE;
} else {
task->swap_state = TASK_SW_IN;
}
task_swaprss_in += pmap_resident_count(task->map->pmap);
task_swap_total_time += sched_tick - task->swap_stamp;
/* note when task came back in */
task->swap_stamp = sched_tick;
if (task->swap_flags & TASK_SW_WANT_IN) {
task->swap_flags &= ~TASK_SW_WANT_IN;
thread_wakeup((event_t)&task->swap_state);
}
assert((task->swap_flags & TASK_SW_ELIGIBLE) == 0);
task_unlock(task);
#if TASK_SW_DEBUG
task_swapper_lock();
if (task_swap_debug && on_swapped_list(task)) {
printf("task 0x%X on list at end of swap in\n", task);
Debugger("");
}
task_swapper_unlock();
#endif /* TASK_SW_DEBUG */
/*
* Make the task eligible to be swapped again
*/
if (swappable)
task_swapout_eligible(task);
return(KERN_SUCCESS);
}
task_t
pick_outtask(void)
{
register task_t task;
register task_t target_task = TASK_NULL;
unsigned long task_rss;
unsigned long target_rss = 0;
boolean_t wired;
boolean_t active;
int nactive = 0;
task_swapout_lock();
if (queue_empty(&eligible_tasks)) {
/* not likely to happen */
task_swapout_unlock();
return(TASK_NULL);
}
task = (task_t)queue_first(&eligible_tasks);
while (!queue_end(&eligible_tasks, (queue_entry_t)task)) {
int s;
register thread_act_t thr_act;
thread_t th;
task_lock(task);
#if MACH_RT
/*
* Don't swap real-time tasks.
* XXX Should we enforce that or can we let really critical
* tasks use task_swappable() to make sure they never end up
* n the eligible list ?
*/
if (task->policy & POLICYCLASS_FIXEDPRI) {
goto tryagain;
}
#endif /* MACH_RT */
if (!task->active) {
TASK_STATS_INCR(inactive_task_count);
goto tryagain;
}
if (task->res_act_count == 0) {
TASK_STATS_INCR(empty_task_count);
goto tryagain;
}
assert(!queue_empty(&task->thr_acts));
thr_act = (thread_act_t)queue_first(&task->thr_acts);
active = FALSE;
th = act_lock_thread(thr_act);
s = splsched();
if (th != THREAD_NULL)
thread_lock(th);
if ((th == THREAD_NULL) ||
(th->state == TH_RUN) ||
(th->state & TH_WAIT)) {
/*
* thread is "active": either runnable
* or sleeping. Count it and examine
* it further below.
*/
nactive++;
active = TRUE;
}
if (th != THREAD_NULL)
thread_unlock(th);
splx(s);
act_unlock_thread(thr_act);
if (active &&
(task->swap_state == TASK_SW_IN) &&
((sched_tick - task->swap_stamp) > min_res_time)) {
long rescount = pmap_resident_count(task->map->pmap);
/*
* thread must be "active", task must be swapped
* in and resident for at least min_res_time
*/
#if 0
/* DEBUG Test round-robin strategy. Picking biggest task could cause extreme
* unfairness to such large interactive programs as xterm. Instead, pick the
* first task that has any pages resident:
*/
if (rescount > 1) {
task->ref_count++;
target_task = task;
task_unlock(task);
task_swapout_unlock();
return(target_task);
}
#else
if (rescount > target_rss) {
/*
* task is not swapped, and it has the
* largest rss seen so far.
*/
task->ref_count++;
target_rss = rescount;
assert(target_task != task);
if (target_task != TASK_NULL)
task_deallocate(target_task);
target_task = task;
}
#endif
}
tryagain:
task_unlock(task);
task = (task_t)queue_next(&task->swapped_tasks);
}
task_swapout_unlock();
/* only swap out if there are at least min_active_tasks */
if (nactive < min_active_tasks) {
if (target_task != TASK_NULL) {
task_deallocate(target_task);
target_task = TASK_NULL;
}
}
return(target_task);
}
/*
* task_swapout:
* A reference to the task must be held.
*
* Start swapping out a task by sending an AST_SWAPOUT to each thread.
* When the threads reach a clean point, they queue themselves up on the
* swapout_thread_q to be swapped out by the task_swap_swapout_thread.
* The task can be swapped in at any point in this process.
*
* A task will not be fully swapped out (i.e. its map residence count
* at zero) until all currently-swapped threads run and reach
* a clean point, at which time they will be swapped again,
* decrementing the swap_ast_waiting count on the task.
*
* Locking: no locks held upon entry and exit.
* Task_lock is held throughout this function.
*/
kern_return_t
task_swapout(task_t task)
{
thread_act_t thr_act;
thread_t thread;
queue_head_t *list;
int s;
task_swapout_lock();
task_lock(task);
/*
* NOTE: look into turning these into assertions if they
* are invariants.
*/
if ((task->swap_state != TASK_SW_IN) || (!task->active)) {
task_unlock(task);
task_swapout_unlock();
return(KERN_FAILURE);
}
if (task->swap_flags & TASK_SW_ELIGIBLE) {
queue_remove(&eligible_tasks, task, task_t, swapped_tasks);
task->swap_flags &= ~TASK_SW_ELIGIBLE;
}
task_swapout_unlock();
/* set state to avoid races with task_swappable(FALSE) */
task->swap_state = TASK_SW_GOING_OUT;
task->swap_rss = pmap_resident_count(task->map->pmap);
task_swaprss_out += task->swap_rss;
task->swap_ast_waiting = task->thr_act_count;
/*
* halt all threads in this task:
* We don't need the thread list lock for traversal.
*/
list = &task->thr_acts;
thr_act = (thread_act_t) queue_first(list);
while (!queue_end(list, (queue_entry_t) thr_act)) {
boolean_t swappable;
thread_act_t ract;
thread = act_lock_thread(thr_act);
s = splsched();
if (!thread)
swappable = (thr_act->swap_state != TH_SW_UNSWAPPABLE);
else {
thread_lock(thread);
swappable = TRUE;
for (ract = thread->top_act; ract; ract = ract->lower)
if (ract->swap_state == TH_SW_UNSWAPPABLE) {
swappable = FALSE;
break;
}
}
if (swappable)
thread_ast_set(thr_act, AST_SWAPOUT);
if (thread)
thread_unlock(thread);
splx(s);
assert((thr_act->ast & AST_TERMINATE) == 0);
act_unlock_thread(thr_act);
thr_act = (thread_act_t) queue_next(&thr_act->thr_acts);
}
task->swap_stamp = sched_tick;
task->swap_nswap++;
assert((task->swap_flags&TASK_SW_WANT_IN) == 0);
/* put task on the queue of swapped out tasks */
task_swapper_lock();
#if TASK_SW_DEBUG
if (task_swap_debug && on_swapped_list(task)) {
printf("task 0x%X already on list\n", task);
Debugger("");
}
#endif /* TASK_SW_DEBUG */
queue_enter(&swapped_tasks, task, task_t, swapped_tasks);
tasks_swapped_out++;
task_swapouts++;
task_swapper_unlock();
task_unlock(task);
return(KERN_SUCCESS);
}
void
processor_doaction(
processor_t processor)
{
thread_t this_thread;
spl_t s;
register processor_set_t pset;
#if MACH_HOST
register processor_set_t new_pset;
register thread_t thread;
register thread_t prev_thread = THREAD_NULL;
thread_act_t thr_act;
boolean_t have_pset_ref = FALSE;
#endif /* MACH_HOST */
/*
* Get onto the processor to shutdown
*/
this_thread = current_thread();
thread_bind(this_thread, processor);
thread_block((void (*)(void)) 0);
pset = processor->processor_set;
#if MACH_HOST
/*
* If this is the last processor in the processor_set,
* stop all the threads first.
*/
pset_lock(pset);
if (pset->processor_count == 1) {
thread = (thread_t) queue_first(&pset->threads);
prev_thread = THREAD_NULL;
pset->ref_count++;
have_pset_ref = TRUE;
pset->empty = TRUE;
/*
* loop through freezing the processor set assignment
* and reference counting the threads;
*/
while (!queue_end(&pset->threads, (queue_entry_t) thread)) {
thread_reference(thread);
pset_unlock(pset);
/*
* Freeze the thread on the processor set.
* If it's moved, just release the reference.
* Get the next thread in the processor set list
* from the last one which was frozen.
*/
if( thread_stop_freeze(thread, pset) )
prev_thread = thread;
else
thread_deallocate(thread);
pset_lock(pset);
if( prev_thread != THREAD_NULL )
thread = (thread_t)queue_next(&prev_thread->pset_threads);
else
thread = (thread_t) queue_first(&pset->threads);
}
/*
* Remove the processor from the set so that when the threads
* are unstopped below the ones blocked in the kernel don't
* start running again.
*/
s = splsched();
processor_lock(processor);
pset_remove_processor(pset, processor);
/*
* Prevent race with another processor being added to the set
* See code after Restart_pset:
* while(new_pset->empty && new_pset->processor_count > 0)
*
* ... it tests for the condition where a new processor is
* added to the set while the last one is still being removed.
*/
pset->processor_count++; /* block new processors being added */
assert( pset->processor_count == 1 );
/*
* Release the thread assignment locks, unstop the threads and
* release the thread references which were taken above.
*/
thread = (thread_t) queue_first(&pset->threads);
while( !queue_empty( &pset->threads) && (thread != THREAD_NULL) ) {
prev_thread = thread;
if( queue_end(&pset->threads, (queue_entry_t) thread) )
thread = THREAD_NULL;
else
thread = (thread_t) queue_next(&prev_thread->pset_threads);
pset_unlock(pset);
thread_unfreeze(prev_thread);
thread_unstop(prev_thread);
thread_deallocate(prev_thread);
pset_lock(pset);
}
/*
* allow a processor to be added to the empty pset
*/
pset->processor_count--;
}
else {
/* not last processor in set */
#endif /* MACH_HOST */
/*
* At this point, it is ok to rm the processor from the pset.
*/
s = splsched();
processor_lock(processor);
pset_remove_processor(pset, processor);
#if MACH_HOST
}
pset_unlock(pset);
/*
* Copy the next pset pointer into a local variable and clear
* it because we are taking over its reference.
*/
new_pset = processor->processor_set_next;
processor->processor_set_next = PROCESSOR_SET_NULL;
if (processor->state == PROCESSOR_ASSIGN) {
Restart_pset:
/*
* Nasty problem: we want to lock the target pset, but
* we have to enable interrupts to do that which requires
* dropping the processor lock. While the processor
* is unlocked, it could be reassigned or shutdown.
*/
processor_unlock(processor);
splx(s);
/*
* Lock target pset and handle remove last / assign first race.
* Only happens if there is more than one action thread.
*/
pset_lock(new_pset);
while (new_pset->empty && new_pset->processor_count > 0) {
pset_unlock(new_pset);
while (*(volatile boolean_t *)&new_pset->empty &&
*(volatile int *)&new_pset->processor_count > 0)
/* spin */;
pset_lock(new_pset);
}
/*
* Finally relock the processor and see if something changed.
* The only possibilities are assignment to a different pset
* and shutdown.
*/
s = splsched();
processor_lock(processor);
if (processor->state == PROCESSOR_SHUTDOWN) {
pset_unlock(new_pset);
goto shutdown; /* will release pset reference */
}
if (processor->processor_set_next != PROCESSOR_SET_NULL) {
/*
* Processor was reassigned. Drop the reference
* we have on the wrong new_pset, and get the
* right one. Involves lots of lock juggling.
*/
processor_unlock(processor);
splx(s);
pset_unlock(new_pset);
pset_deallocate(new_pset);
s = splsched();
processor_lock(processor);
new_pset = processor->processor_set_next;
processor->processor_set_next = PROCESSOR_SET_NULL;
goto Restart_pset;
}
/*
* If the pset has been deactivated since the operation
* was requested, redirect to the default pset.
*/
if (!(new_pset->active)) {
pset_unlock(new_pset);
pset_deallocate(new_pset);
new_pset = &default_pset;
pset_lock(new_pset);
new_pset->ref_count++;
}
/*
* Do assignment, then wakeup anyone waiting for it.
* Finally context switch to have it take effect.
*/
pset_add_processor(new_pset, processor);
if (new_pset->empty) {
/*
* Set all the threads loose
*/
thread = (thread_t) queue_first(&new_pset->threads);
while (!queue_end(&new_pset->threads,(queue_entry_t)thread)) {
thr_act = thread_lock_act(thread);
thread_release(thread->top_act);
act_unlock_thread(thr_act);
thread = (thread_t) queue_next(&thread->pset_threads);
}
new_pset->empty = FALSE;
}
processor->processor_set_next = PROCESSOR_SET_NULL;
processor->state = PROCESSOR_RUNNING;
thread_wakeup((event_t)processor);
processor_unlock(processor);
splx(s);
pset_unlock(new_pset);
/*
* Clean up dangling references, and release our binding.
*/
pset_deallocate(new_pset);
if (have_pset_ref)
pset_deallocate(pset);
if (prev_thread != THREAD_NULL)
thread_deallocate(prev_thread);
thread_bind(this_thread, PROCESSOR_NULL);
thread_block((void (*)(void)) 0);
return;
}
shutdown:
#endif /* MACH_HOST */
/*
* Do shutdown, make sure we live when processor dies.
*/
if (processor->state != PROCESSOR_SHUTDOWN) {
printf("state: %d\n", processor->state);
panic("action_thread -- bad processor state");
}
processor_unlock(processor);
/*
* Clean up dangling references, and release our binding.
*/
#if MACH_HOST
if (new_pset != PROCESSOR_SET_NULL)
pset_deallocate(new_pset);
if (have_pset_ref)
pset_deallocate(pset);
if (prev_thread != THREAD_NULL)
thread_deallocate(prev_thread);
#endif /* MACH_HOST */
thread_bind(this_thread, PROCESSOR_NULL);
switch_to_shutdown_context(this_thread,
processor_doshutdown,
processor);
splx(s);
}
/*
* Process an AST_SWAPOUT.
*/
void
swapout_ast()
{
spl_t s;
thread_act_t act;
thread_t thread;
act = current_act();
/*
* Task is being swapped out. First mark it as suspended
* and halted, then call thread_swapout_enqueue to put
* the thread on the queue for task_swap_swapout_threads
* to swap out the thread.
*/
/*
* Don't swap unswappable threads
*/
thread = act_lock_thread(act);
s = splsched();
if (thread)
thread_lock(thread);
if ((act->ast & AST_SWAPOUT) == 0) {
/*
* Race with task_swapin. Abort swapout.
*/
task_swap_ast_aborted++; /* not locked XXX */
if (thread)
thread_unlock(thread);
splx(s);
act_unlock_thread(act);
} else if (act->swap_state == TH_SW_IN) {
/*
* Mark swap_state as TH_SW_TASK_SWAPPING to avoid
* race with thread swapper, which will only
* swap thread if swap_state is TH_SW_IN.
* This way, the thread can only be swapped by
* the task swapping mechanism.
*/
act->swap_state |= TH_SW_TASK_SWAPPING;
/* assert(act->suspend_count == 0); XXX ? */
if (thread)
thread_unlock(thread);
if (act->suspend_count++ == 0) /* inline thread_hold */
install_special_handler(act);
/* self->state |= TH_HALTED; */
thread_ast_clear(act, AST_SWAPOUT);
/*
* Initialize the swap_queue fields to allow an extra
* queue_remove() in task_swapin if we lose the race
* (task_swapin can be called before we complete
* thread_swapout_enqueue).
*/
queue_init((queue_t) &act->swap_queue);
splx(s);
act_unlock_thread(act);
/* this must be called at normal interrupt level */
thread_swapout_enqueue(act);
} else {
/* thread isn't swappable; continue running */
assert(act->swap_state == TH_SW_UNSWAPPABLE);
if (thread)
thread_unlock(thread);
thread_ast_clear(act, AST_SWAPOUT);
splx(s);
act_unlock_thread(act);
}
}
