/*
* Re-set current processor's per-thread AST flags to those set on thread
* Called at splsched
*/
void
ast_context(thread_t thread)
{
ast_t *pending_ast = ast_pending();
*pending_ast = ((*pending_ast & ~AST_PER_THREAD) | thread->ast);
}
/*
* Set AST flags on current processor
* Called at splsched
*/
void
ast_on(ast_t reasons)
{
ast_t *pending_ast = ast_pending();
*pending_ast |= reasons;
}
/*
* Clear AST flags on current processor
* Called at splsched
*/
void
ast_off(ast_t reasons)
{
ast_t *pending_ast = ast_pending();
*pending_ast &= ~reasons;
}
/*
* Read the requested subset of the AST flags set on the processor
* Return the bits that were set, don't modify the processor
* Called at splsched
*/
ast_t
ast_peek(ast_t reasons)
{
ast_t *pending_ast = ast_pending();
reasons &= *pending_ast;
return reasons;
}
/*
* Consume the requested subset of the AST flags set on the processor
* Return the bits that were set
* Called at splsched
*/
ast_t
ast_consume(ast_t reasons)
{
ast_t *pending_ast = ast_pending();
reasons &= *pending_ast;
*pending_ast &= ~reasons;
return reasons;
}
static kern_return_t
chudxnu_private_chud_ast_callback(
int trapno,
void *regs,
int unused1,
int unused2)
{
#pragma unused (trapno)
#pragma unused (regs)
#pragma unused (unused1)
#pragma unused (unused2)
boolean_t oldlevel = ml_set_interrupts_enabled(FALSE);
ast_t *myast = ast_pending();
kern_return_t retval = KERN_FAILURE;
chudxnu_perfmon_ast_callback_func_t fn = perfmon_ast_callback_fn;
if (*myast & AST_CHUD_URGENT) {
*myast &= ~(AST_CHUD_URGENT | AST_CHUD);
if ((*myast & AST_PREEMPTION) != AST_PREEMPTION)
*myast &= ~(AST_URGENT);
retval = KERN_SUCCESS;
} else if (*myast & AST_CHUD) {
*myast &= ~(AST_CHUD);
retval = KERN_SUCCESS;
}
if (fn) {
x86_thread_state_t state;
mach_msg_type_number_t count;
count = x86_THREAD_STATE_COUNT;
if (chudxnu_thread_get_state(
current_thread(),
x86_THREAD_STATE,
(thread_state_t) &state, &count,
TRUE) == KERN_SUCCESS) {
KERNEL_DEBUG_CONSTANT(
MACHDBG_CODE(DBG_MACH_CHUD,
CHUD_AST_CALLBACK) | DBG_FUNC_NONE,
(uint32_t) fn, 0, 0, 0, 0);
(fn)(
x86_THREAD_STATE,
(thread_state_t) &state,
count);
}
}
ml_set_interrupts_enabled(oldlevel);
return retval;
}
__private_extern__ kern_return_t
chudxnu_perfmon_ast_send_urgent(boolean_t urgent)
{
boolean_t oldlevel = ml_set_interrupts_enabled(FALSE);
ast_t *myast = ast_pending();
if(urgent) {
*myast |= (AST_CHUD_URGENT | AST_URGENT);
} else {
*myast |= (AST_CHUD);
}
KERNEL_DEBUG_CONSTANT(
MACHDBG_CODE(DBG_MACH_CHUD, CHUD_AST_SEND) | DBG_FUNC_NONE,
urgent, 0, 0, 0, 0);
ml_set_interrupts_enabled(oldlevel);
return KERN_SUCCESS;
}
/*
* Called at splsched.
*/
void
ast_taken(
ast_t reasons,
boolean_t enable
)
{
boolean_t preempt_trap = (reasons == AST_PREEMPTION);
ast_t *myast = ast_pending();
thread_t thread = current_thread();
perfASTCallback perf_hook = perfASTHook;
/*
* CHUD hook - all threads including idle processor threads
*/
if (perf_hook) {
if (*myast & AST_CHUD_ALL) {
(*perf_hook)(reasons, myast);
if (*myast == AST_NONE)
return;
}
}
else
*myast &= ~AST_CHUD_ALL;
reasons &= *myast;
*myast &= ~reasons;
/*
* Handle ASTs for all threads
* except idle processor threads.
*/
if (!(thread->state & TH_IDLE)) {
/*
* Check for urgent preemption.
*/
if ( (reasons & AST_URGENT) &&
waitq_wait_possible(thread) ) {
if (reasons & AST_PREEMPT) {
counter(c_ast_taken_block++);
thread_block_reason(THREAD_CONTINUE_NULL, NULL,
reasons & AST_PREEMPTION);
}
reasons &= ~AST_PREEMPTION;
}
/*
* The kernel preempt traps
* skip all other ASTs.
*/
if (!preempt_trap) {
ml_set_interrupts_enabled(enable);
#ifdef MACH_BSD
/*
* Handle BSD hook.
*/
if (reasons & AST_BSD) {
thread_ast_clear(thread, AST_BSD);
bsd_ast(thread);
}
#endif
#if CONFIG_MACF
/*
* Handle MACF hook.
*/
if (reasons & AST_MACF) {
thread_ast_clear(thread, AST_MACF);
mac_thread_userret(thread);
}
#endif
/*
* Thread APC hook.
*/
if (reasons & AST_APC) {
thread_ast_clear(thread, AST_APC);
special_handler(thread);
}
if (reasons & AST_GUARD) {
thread_ast_clear(thread, AST_GUARD);
guard_ast(thread);
}
if (reasons & AST_LEDGER) {
thread_ast_clear(thread, AST_LEDGER);
ledger_ast(thread);
}
/*
* Kernel Profiling Hook
*/
if (reasons & AST_KPERF) {
thread_ast_clear(thread, AST_KPERF);
chudxnu_thread_ast(thread);
}
#if CONFIG_TELEMETRY
if (reasons & AST_TELEMETRY_ALL) {
boolean_t interrupted_userspace = FALSE;
boolean_t is_windowed = FALSE;
assert((reasons & AST_TELEMETRY_ALL) != AST_TELEMETRY_ALL); /* only one is valid at a time */
interrupted_userspace = (reasons & AST_TELEMETRY_USER) ? TRUE : FALSE;
is_windowed = ((reasons & AST_TELEMETRY_WINDOWED) ? TRUE : FALSE);
thread_ast_clear(thread, AST_TELEMETRY_ALL);
telemetry_ast(thread, interrupted_userspace, is_windowed);
}
#endif
ml_set_interrupts_enabled(FALSE);
#if CONFIG_SCHED_SFI
if (reasons & AST_SFI) {
sfi_ast(thread);
}
#endif
/*
* Check for preemption. Conditions may have changed from when the AST_PREEMPT was originally set.
*/
thread_lock(thread);
if (reasons & AST_PREEMPT)
reasons = csw_check(current_processor(), reasons & AST_QUANTUM);
thread_unlock(thread);
assert(waitq_wait_possible(thread));
if (reasons & AST_PREEMPT) {
counter(c_ast_taken_block++);
thread_block_reason((thread_continue_t)thread_exception_return, NULL, reasons & AST_PREEMPTION);
}
}
}
ml_set_interrupts_enabled(enable);
}
