/* * Called at splsched. */ void ast_check( processor_t processor) { thread_t thread = processor->active_thread; if (processor->state == PROCESSOR_RUNNING || processor->state == PROCESSOR_SHUTDOWN) { ast_t preempt; /* * Propagate thread ast to processor. */ pal_ast_check(thread); ast_propagate(thread->ast); /* * Context switch check. */ thread_lock(thread); processor->current_pri = thread->sched_pri; processor->current_thmode = thread->sched_mode; processor->current_sfi_class = thread->sfi_class = sfi_thread_classify(thread); if ((preempt = csw_check(processor, AST_NONE)) != AST_NONE) ast_on(preempt); thread_unlock(thread); } }
/* * Called at splsched. */ void ast_check( processor_t processor) { register thread_t self = processor->active_thread; processor->current_pri = self->sched_pri; if (processor->state == PROCESSOR_RUNNING) { register ast_t preempt; processor_running: /* * Propagate thread ast to processor. */ ast_propagate(self->top_act->ast); /* * Context switch check. */ if ((preempt = csw_check(self, processor)) != AST_NONE) ast_on(preempt); } else if ( processor->state == PROCESSOR_DISPATCHING || processor->state == PROCESSOR_IDLE ) { return; } else if (processor->state == PROCESSOR_SHUTDOWN) goto processor_running; }
/* * install_special_handler_locked: * * Do the work of installing the special_handler. * * Called with the thread mutex and scheduling lock held. */ void install_special_handler_locked( thread_t thread) { ReturnHandler **rh; /* The work handler must always be the last ReturnHandler on the list, because it can do tricky things like detach the thr_act. */ for (rh = &thread->handlers; *rh; rh = &(*rh)->next) continue; if (rh != &thread->special_handler.next) *rh = &thread->special_handler; /* * Temporarily undepress, so target has * a chance to do locking required to * block itself in special_handler(). */ if (thread->sched_flags & TH_SFLAG_DEPRESSED_MASK) SCHED(compute_priority)(thread, TRUE); thread_ast_set(thread, AST_APC); if (thread == current_thread()) ast_propagate(thread->ast); else { processor_t processor = thread->last_processor; if ( processor != PROCESSOR_NULL && processor->state == PROCESSOR_RUNNING && processor->active_thread == thread ) cause_ast_check(processor); } }
static void act_set_ast( thread_t thread, ast_t ast) { spl_t s = splsched(); if (thread == current_thread()) { thread_ast_set(thread, ast); ast_propagate(thread->ast); } else { processor_t processor; thread_lock(thread); thread_ast_set(thread, ast); processor = thread->last_processor; if ( processor != PROCESSOR_NULL && processor->state == PROCESSOR_RUNNING && processor->active_thread == thread ) cause_ast_check(processor); thread_unlock(thread); } splx(s); }
/* * install_special_handler_locked: * * Do the work of installing the special_handler. * * Called with the thread mutex and scheduling lock held. */ void install_special_handler_locked( thread_t thread) { /* * Temporarily undepress, so target has * a chance to do locking required to * block itself in special_handler(). */ if (thread->sched_flags & TH_SFLAG_DEPRESSED_MASK) thread_recompute_sched_pri(thread, TRUE); thread_ast_set(thread, AST_APC); if (thread == current_thread()) ast_propagate(thread->ast); else { processor_t processor = thread->last_processor; if ( processor != PROCESSOR_NULL && processor->state == PROCESSOR_RUNNING && processor->active_thread == thread ) cause_ast_check(processor); } }
/* * thread_set_apc_ast_locked: * * Do the work of registering for the AST_APC callback. * * Called with the thread mutex and scheduling lock held. */ static void thread_set_apc_ast_locked(thread_t thread) { /* * Temporarily undepress, so target has * a chance to do locking required to * block itself in thread_suspended. * * Leaves the depress flag set so we can reinstate when it's blocked. */ if (thread->sched_flags & TH_SFLAG_DEPRESSED_MASK) thread_recompute_sched_pri(thread, TRUE); thread_ast_set(thread, AST_APC); if (thread == current_thread()) { ast_propagate(thread); } else { processor_t processor = thread->last_processor; if (processor != PROCESSOR_NULL && processor->state == PROCESSOR_RUNNING && processor->active_thread == thread) { cause_ast_check(processor); } } }
/* * Handle preemption IPI or IPI in response to setting an AST flag * Triggered by cause_ast_check * Called at splsched */ void ast_check(processor_t processor) { if (processor->state != PROCESSOR_RUNNING && processor->state != PROCESSOR_SHUTDOWN) return; thread_t thread = processor->active_thread; assert(thread == current_thread()); thread_lock(thread); /* * Propagate thread ast to processor. * (handles IPI in response to setting AST flag) */ ast_propagate(thread); boolean_t needs_callout = false; processor->current_pri = thread->sched_pri; processor->current_sfi_class = thread->sfi_class = sfi_thread_classify(thread); processor->current_recommended_pset_type = recommended_pset_type(thread); perfcontrol_class_t thread_class = thread_get_perfcontrol_class(thread); if (thread_class != processor->current_perfctl_class) { /* We updated the perfctl class of this thread from another core. * Since we dont do CLPC callouts from another core, do a callout * here to let CLPC know that the currently running thread has a new * class. */ needs_callout = true; } processor->current_perfctl_class = thread_class; ast_t preempt; if ((preempt = csw_check(processor, AST_NONE)) != AST_NONE) ast_on(preempt); thread_unlock(thread); if (needs_callout) { machine_switch_perfcontrol_state_update(PERFCONTROL_ATTR_UPDATE, mach_approximate_time(), 0, thread); } }
/* * Mark the current thread for an interrupt-based * telemetry record, to be sampled at the next AST boundary. */ void telemetry_mark_curthread(boolean_t interrupted_userspace) { uint32_t ast_bits = 0; thread_t thread = current_thread(); /* * If telemetry isn't active for this thread, return and try * again next time. */ if (telemetry_is_active(thread) == FALSE) { return; } ast_bits |= (interrupted_userspace ? AST_TELEMETRY_USER : AST_TELEMETRY_KERNEL); telemetry_needs_record = FALSE; thread_ast_set(thread, ast_bits); ast_propagate(thread); }
void act_set_astkevent(thread_t thread, uint16_t bits) { spl_t s = splsched(); /* * Do not send an IPI if the thread is running on * another processor, wait for the next quantum * expiration to load the AST. */ atomic_fetch_or(&thread->kevent_ast_bits, bits); thread_ast_set(thread, AST_KEVENT); if (thread == current_thread()) { ast_propagate(thread); } splx(s); }
/* * An AST flag was set while returning to user mode * Called with interrupts disabled, returns with interrupts enabled * May call continuation instead of returning */ void ast_taken_user(void) { assert(ml_get_interrupts_enabled() == FALSE); thread_t thread = current_thread(); /* We are about to return to userspace, there must not be a pending wait */ assert(waitq_wait_possible(thread)); assert((thread->state & TH_IDLE) == 0); /* TODO: Add more 'return to userspace' assertions here */ /* * If this thread was urgently preempted in userspace, * take the preemption before processing the ASTs. * The trap handler will call us again if we have more ASTs, so it's * safe to block in a continuation here. */ if (ast_peek(AST_URGENT) == AST_URGENT) { ast_t urgent_reason = ast_consume(AST_PREEMPTION); assert(urgent_reason & AST_PREEMPT); /* TODO: Should we csw_check again to notice if conditions have changed? */ thread_block_reason(thread_preempted, NULL, urgent_reason); /* NOTREACHED */ } /* * AST_KEVENT does not send an IPI when setting the ast for a thread running in parallel * on a different processor. Only the ast bit on the thread will be set. * * Force a propagate for concurrent updates without an IPI. */ ast_propagate(thread); /* * Consume all non-preemption processor ASTs matching reasons * because we're handling them here. * * If one of the AST handlers blocks in a continuation, * we'll reinstate the unserviced thread-level AST flags * from the thread to the processor on context switch. * If one of the AST handlers sets another AST, * the trap handler will call ast_taken_user again. * * We expect the AST handlers not to thread_exception_return * without an ast_propagate or context switch to reinstate * the per-processor ASTs. * * TODO: Why are AST_DTRACE and AST_KPERF not per-thread ASTs? */ ast_t reasons = ast_consume(AST_PER_THREAD | AST_KPERF | AST_DTRACE); ml_set_interrupts_enabled(TRUE); #if CONFIG_DTRACE if (reasons & AST_DTRACE) { dtrace_ast(); } #endif #ifdef MACH_BSD if (reasons & AST_BSD) { thread_ast_clear(thread, AST_BSD); bsd_ast(thread); } #endif #if CONFIG_MACF if (reasons & AST_MACF) { thread_ast_clear(thread, AST_MACF); mac_thread_userret(thread); } #endif if (reasons & AST_APC) { thread_ast_clear(thread, AST_APC); thread_apc_ast(thread); } if (reasons & AST_GUARD) { thread_ast_clear(thread, AST_GUARD); guard_ast(thread); } if (reasons & AST_LEDGER) { thread_ast_clear(thread, AST_LEDGER); ledger_ast(thread); } if (reasons & AST_KPERF) { thread_ast_clear(thread, AST_KPERF); kperf_kpc_thread_ast(thread); } if (reasons & AST_KEVENT) { thread_ast_clear(thread, AST_KEVENT); uint16_t bits = atomic_exchange(&thread->kevent_ast_bits, 0); if (bits) kevent_ast(thread, bits); } #if CONFIG_TELEMETRY if (reasons & AST_TELEMETRY_ALL) { ast_t telemetry_reasons = reasons & AST_TELEMETRY_ALL; thread_ast_clear(thread, AST_TELEMETRY_ALL); telemetry_ast(thread, telemetry_reasons); } #endif spl_t s = splsched(); #if CONFIG_SCHED_SFI /* * SFI is currently a per-processor AST, not a per-thread AST * TODO: SFI should be a per-thread AST */ if (ast_consume(AST_SFI) == AST_SFI) { sfi_ast(thread); } #endif /* We are about to return to userspace, there must not be a pending wait */ assert(waitq_wait_possible(thread)); /* * We've handled all per-thread ASTs, time to handle non-urgent preemption. * * We delay reading the preemption bits until now in case the thread * blocks while handling per-thread ASTs. * * If one of the AST handlers had managed to set a new AST bit, * thread_exception_return will call ast_taken again. */ ast_t preemption_reasons = ast_consume(AST_PREEMPTION); if (preemption_reasons & AST_PREEMPT) { /* Conditions may have changed from when the AST_PREEMPT was originally set, so re-check. */ thread_lock(thread); preemption_reasons = csw_check(current_processor(), (preemption_reasons & AST_QUANTUM)); thread_unlock(thread); #if CONFIG_SCHED_SFI /* csw_check might tell us that SFI is needed */ if (preemption_reasons & AST_SFI) { sfi_ast(thread); } #endif if (preemption_reasons & AST_PREEMPT) { counter(c_ast_taken_block++); /* switching to a continuation implicitly re-enables interrupts */ thread_block_reason(thread_preempted, NULL, preemption_reasons); /* NOTREACHED */ } } splx(s); }
void ast_check(void) { register int mycpu; register processor_t myprocessor; register thread_t thread = current_thread(); spl_t s = splsched(); mp_disable_preemption(); mycpu = cpu_number(); /* * Check processor state for ast conditions. */ myprocessor = cpu_to_processor(mycpu); switch(myprocessor->state) { case PROCESSOR_OFF_LINE: case PROCESSOR_IDLE: case PROCESSOR_DISPATCHING: /* * No ast. */ break; #if NCPUS > 1 case PROCESSOR_ASSIGN: case PROCESSOR_SHUTDOWN: /* * Need ast to force action thread onto processor. * * XXX Should check if action thread is already there. */ ast_on(mycpu, AST_BLOCK); break; #endif /* NCPUS > 1 */ case PROCESSOR_RUNNING: case PROCESSOR_VIDLE: /* * Propagate thread ast to processor. If we already * need an ast, don't look for more reasons. */ ast_propagate(current_act(), mycpu); if (ast_needed(mycpu)) break; /* * Context switch check. */ if (csw_needed(thread, myprocessor)) { ast_on(mycpu, (myprocessor->first_quantum ? AST_BLOCK : AST_QUANTUM)); } break; default: panic("ast_check: Bad processor state"); } mp_enable_preemption(); splx(s); }
void ast_check(void) { int mycpu = cpu_number(); processor_t myprocessor; thread_t thread = current_thread(); run_queue_t rq; spl_t s = splsched(); /* * Check processor state for ast conditions. */ myprocessor = cpu_to_processor(mycpu); switch(myprocessor->state) { case PROCESSOR_OFF_LINE: case PROCESSOR_IDLE: case PROCESSOR_DISPATCHING: /* * No ast. */ break; #if NCPUS > 1 case PROCESSOR_ASSIGN: case PROCESSOR_SHUTDOWN: /* * Need ast to force action thread onto processor. * * XXX Should check if action thread is already there. */ ast_on(mycpu, AST_BLOCK); break; #endif /* NCPUS > 1 */ case PROCESSOR_RUNNING: /* * Propagate thread ast to processor. If we already * need an ast, don't look for more reasons. */ ast_propagate(thread, mycpu); if (ast_needed(mycpu)) break; /* * Context switch check. The csw_needed macro isn't * used here because the rq->low hint may be wrong, * and fixing it here avoids an extra ast. * First check the easy cases. */ if (thread->state & TH_SUSP || myprocessor->runq.count > 0) { ast_on(mycpu, AST_BLOCK); break; } /* * Update lazy evaluated runq->low if only timesharing. */ #if MACH_FIXPRI if (myprocessor->processor_set->policies & POLICY_FIXEDPRI) { if (csw_needed(thread,myprocessor)) { ast_on(mycpu, AST_BLOCK); break; } else { /* * For fixed priority threads, set first_quantum * so entire new quantum is used. */ if (thread->policy == POLICY_FIXEDPRI) myprocessor->first_quantum = TRUE; } } else { #endif /* MACH_FIXPRI */ rq = &(myprocessor->processor_set->runq); if (!(myprocessor->first_quantum) && (rq->count > 0)) { queue_t q; /* * This is not the first quantum, and there may * be something in the processor_set runq. * Check whether low hint is accurate. */ q = rq->runq + *(volatile int *)&rq->low; if (queue_empty(q)) { int i; /* * Need to recheck and possibly update hint. */ simple_lock(&rq->lock); q = rq->runq + rq->low; if (rq->count > 0) { for (i = rq->low; i < NRQS; i++) { if(!(queue_empty(q))) break; q++; } rq->low = i; } simple_unlock(&rq->lock); } if (rq->low <= thread->sched_pri) { ast_on(mycpu, AST_BLOCK); break; } } #if MACH_FIXPRI } #endif /* MACH_FIXPRI */ break; default: panic("ast_check: Bad processor state (cpu %d processor %08x) state: %d", mycpu, myprocessor, myprocessor->state); } (void) splx(s); }