static void
act_set_ast(
thread_t thread,
ast_t ast)
{
spl_t s = splsched();
if (thread == current_thread()) {
thread_ast_set(thread, ast);
ast_propagate(thread->ast);
}
else {
processor_t processor;
thread_lock(thread);
thread_ast_set(thread, ast);
processor = thread->last_processor;
if ( processor != PROCESSOR_NULL &&
processor->state == PROCESSOR_RUNNING &&
processor->active_thread == thread )
cause_ast_check(processor);
thread_unlock(thread);
}
splx(s);
}
/*
* install_special_handler_locked:
*
* Do the work of installing the special_handler.
*
* Called with the thread mutex and scheduling lock held.
*/
void
install_special_handler_locked(
thread_t thread)
{
ReturnHandler **rh;
/* The work handler must always be the last ReturnHandler on the list,
because it can do tricky things like detach the thr_act. */
for (rh = &thread->handlers; *rh; rh = &(*rh)->next)
continue;
if (rh != &thread->special_handler.next)
*rh = &thread->special_handler;
/*
* Temporarily undepress, so target has
* a chance to do locking required to
* block itself in special_handler().
*/
if (thread->sched_flags & TH_SFLAG_DEPRESSED_MASK)
SCHED(compute_priority)(thread, TRUE);
thread_ast_set(thread, AST_APC);
if (thread == current_thread())
ast_propagate(thread->ast);
else {
processor_t processor = thread->last_processor;
if ( processor != PROCESSOR_NULL &&
processor->state == PROCESSOR_RUNNING &&
processor->active_thread == thread )
cause_ast_check(processor);
}
}
/*
* install_special_handler_locked:
*
* Do the work of installing the special_handler.
*
* Called with the thread mutex and scheduling lock held.
*/
void
install_special_handler_locked(
thread_t thread)
{
/*
* Temporarily undepress, so target has
* a chance to do locking required to
* block itself in special_handler().
*/
if (thread->sched_flags & TH_SFLAG_DEPRESSED_MASK)
thread_recompute_sched_pri(thread, TRUE);
thread_ast_set(thread, AST_APC);
if (thread == current_thread())
ast_propagate(thread->ast);
else {
processor_t processor = thread->last_processor;
if ( processor != PROCESSOR_NULL &&
processor->state == PROCESSOR_RUNNING &&
processor->active_thread == thread )
cause_ast_check(processor);
}
}
/*
* thread_set_apc_ast_locked:
*
* Do the work of registering for the AST_APC callback.
*
* Called with the thread mutex and scheduling lock held.
*/
static void
thread_set_apc_ast_locked(thread_t thread)
{
/*
* Temporarily undepress, so target has
* a chance to do locking required to
* block itself in thread_suspended.
*
* Leaves the depress flag set so we can reinstate when it's blocked.
*/
if (thread->sched_flags & TH_SFLAG_DEPRESSED_MASK)
thread_recompute_sched_pri(thread, TRUE);
thread_ast_set(thread, AST_APC);
if (thread == current_thread()) {
ast_propagate(thread);
} else {
processor_t processor = thread->last_processor;
if (processor != PROCESSOR_NULL &&
processor->state == PROCESSOR_RUNNING &&
processor->active_thread == thread) {
cause_ast_check(processor);
}
}
}
/*
* Mark the current thread for an interrupt-based
* telemetry record, to be sampled at the next AST boundary.
*/
void telemetry_mark_curthread(boolean_t interrupted_userspace)
{
uint32_t ast_bits = 0;
thread_t thread = current_thread();
/*
* If telemetry isn't active for this thread, return and try
* again next time.
*/
if (telemetry_is_active(thread) == FALSE) {
return;
}
ast_bits |= (interrupted_userspace ? AST_TELEMETRY_USER : AST_TELEMETRY_KERNEL);
telemetry_needs_record = FALSE;
thread_ast_set(thread, ast_bits);
ast_propagate(thread);
}
void
act_set_astkevent(thread_t thread, uint16_t bits)
{
spl_t s = splsched();
/*
* Do not send an IPI if the thread is running on
* another processor, wait for the next quantum
* expiration to load the AST.
*/
atomic_fetch_or(&thread->kevent_ast_bits, bits);
thread_ast_set(thread, AST_KEVENT);
if (thread == current_thread()) {
ast_propagate(thread);
}
splx(s);
}
/*
* task_swapout:
* A reference to the task must be held.
*
* Start swapping out a task by sending an AST_SWAPOUT to each thread.
* When the threads reach a clean point, they queue themselves up on the
* swapout_thread_q to be swapped out by the task_swap_swapout_thread.
* The task can be swapped in at any point in this process.
*
* A task will not be fully swapped out (i.e. its map residence count
* at zero) until all currently-swapped threads run and reach
* a clean point, at which time they will be swapped again,
* decrementing the swap_ast_waiting count on the task.
*
* Locking: no locks held upon entry and exit.
* Task_lock is held throughout this function.
*/
kern_return_t
task_swapout(task_t task)
{
thread_act_t thr_act;
thread_t thread;
queue_head_t *list;
int s;
task_swapout_lock();
task_lock(task);
/*
* NOTE: look into turning these into assertions if they
* are invariants.
*/
if ((task->swap_state != TASK_SW_IN) || (!task->active)) {
task_unlock(task);
task_swapout_unlock();
return(KERN_FAILURE);
}
if (task->swap_flags & TASK_SW_ELIGIBLE) {
queue_remove(&eligible_tasks, task, task_t, swapped_tasks);
task->swap_flags &= ~TASK_SW_ELIGIBLE;
}
task_swapout_unlock();
/* set state to avoid races with task_swappable(FALSE) */
task->swap_state = TASK_SW_GOING_OUT;
task->swap_rss = pmap_resident_count(task->map->pmap);
task_swaprss_out += task->swap_rss;
task->swap_ast_waiting = task->thr_act_count;
/*
* halt all threads in this task:
* We don't need the thread list lock for traversal.
*/
list = &task->thr_acts;
thr_act = (thread_act_t) queue_first(list);
while (!queue_end(list, (queue_entry_t) thr_act)) {
boolean_t swappable;
thread_act_t ract;
thread = act_lock_thread(thr_act);
s = splsched();
if (!thread)
swappable = (thr_act->swap_state != TH_SW_UNSWAPPABLE);
else {
thread_lock(thread);
swappable = TRUE;
for (ract = thread->top_act; ract; ract = ract->lower)
if (ract->swap_state == TH_SW_UNSWAPPABLE) {
swappable = FALSE;
break;
}
}
if (swappable)
thread_ast_set(thr_act, AST_SWAPOUT);
if (thread)
thread_unlock(thread);
splx(s);
assert((thr_act->ast & AST_TERMINATE) == 0);
act_unlock_thread(thr_act);
thr_act = (thread_act_t) queue_next(&thr_act->thr_acts);
}
task->swap_stamp = sched_tick;
task->swap_nswap++;
assert((task->swap_flags&TASK_SW_WANT_IN) == 0);
/* put task on the queue of swapped out tasks */
task_swapper_lock();
#if TASK_SW_DEBUG
if (task_swap_debug && on_swapped_list(task)) {
printf("task 0x%X already on list\n", task);
Debugger("");
}
#endif /* TASK_SW_DEBUG */
queue_enter(&swapped_tasks, task, task_t, swapped_tasks);
tasks_swapped_out++;
task_swapouts++;
task_swapper_unlock();
task_unlock(task);
return(KERN_SUCCESS);
}
