void timer_call_shutdown( processor_t processor) { timer_call_t call; queue_t queue, myqueue; assert(processor != current_processor()); queue = &PROCESSOR_DATA(processor, timer_call_queue); myqueue = &PROCESSOR_DATA(current_processor(), timer_call_queue); simple_lock(&timer_call_lock); call = TC(queue_first(queue)); while (!queue_end(queue, qe(call))) { _delayed_call_dequeue(call); _delayed_call_enqueue(myqueue, call); call = TC(queue_first(queue)); } call = TC(queue_first(myqueue)); if (!queue_end(myqueue, qe(call))) _set_delayed_call_timer(call); simple_unlock(&timer_call_lock); }
boolean_t swtch_pri( __unused struct swtch_pri_args *args) { register processor_t myprocessor; boolean_t result; disable_preemption(); myprocessor = current_processor(); if (SCHED(processor_queue_empty)(myprocessor) && rt_runq.count == 0) { mp_enable_preemption(); return (FALSE); } enable_preemption(); counter(c_swtch_pri_block++); thread_depress_abstime(thread_depress_time); thread_block_reason((thread_continue_t)swtch_pri_continue, NULL, AST_YIELD); thread_depress_abort_internal(current_thread()); disable_preemption(); myprocessor = current_processor(); result = !SCHED(processor_queue_empty)(myprocessor) || rt_runq.count > 0; enable_preemption(); return (result); }
boolean_t swtch_pri(int pri) { thread_t thread = current_thread(); processor_t myprocessor; #if NCPUS > 1 myprocessor = current_processor(); if (myprocessor->runq.count == 0 && myprocessor->processor_set->runq.count == 0) return(FALSE); #endif /* NCPUS > 1 */ /* * XXX need to think about depression duration. * XXX currently using min quantum. */ thread_depress_priority(thread, min_quantum); counter(c_swtch_pri_block++); thread_block(swtch_pri_continue); if (thread->depress_priority >= 0) (void) thread_depress_abort(thread); myprocessor = current_processor(); return(myprocessor->runq.count > 0 || myprocessor->processor_set->runq.count > 0); }
int kpc_get_shadow_counters(boolean_t all_cpus, uint32_t classes, int *curcpu, uint64_t *buf) { int curcpu_id = current_processor()->cpu_id; uint32_t cfg_count = kpc_configurable_count(), offset = 0; uint64_t pmc_mask = 0ULL; boolean_t enabled; assert(buf); enabled = ml_set_interrupts_enabled(FALSE); curcpu_id = current_processor()->cpu_id; if (curcpu) *curcpu = curcpu_id; for (int cpu = 0; cpu < machine_info.logical_cpu_max; ++cpu) { /* filter if the caller did not request all cpus */ if (!all_cpus && (cpu != curcpu_id)) continue; if (classes & KPC_CLASS_FIXED_MASK) { uint32_t count = kpc_get_counter_count(KPC_CLASS_FIXED_MASK); memcpy(&buf[offset], &FIXED_SHADOW_CPU(cpu, 0), count * sizeof(uint64_t)); offset += count; } if (classes & KPC_CLASS_CONFIGURABLE_MASK) { pmc_mask = kpc_get_configurable_pmc_mask(KPC_CLASS_CONFIGURABLE_MASK); for (uint32_t cfg_ctr = 0; cfg_ctr < cfg_count; ++cfg_ctr) if ((1ULL << cfg_ctr) & pmc_mask) buf[offset++] = CONFIGURABLE_SHADOW_CPU(cpu, cfg_ctr); } if (classes & KPC_CLASS_POWER_MASK) { pmc_mask = kpc_get_configurable_pmc_mask(KPC_CLASS_POWER_MASK); for (uint32_t cfg_ctr = 0; cfg_ctr < cfg_count; ++cfg_ctr) if ((1ULL << cfg_ctr) & pmc_mask) buf[offset++] = CONFIGURABLE_SHADOW_CPU(cpu, cfg_ctr); } } ml_set_interrupts_enabled(enabled); return offset; }
/* * stack_alloc_try: * * Non-blocking attempt to allocate a * stack for a thread. * * Returns TRUE on success. * * Called at splsched. */ boolean_t stack_alloc_try( thread_t thread) { struct stack_cache *cache; vm_offset_t stack; cache = &PROCESSOR_DATA(current_processor(), stack_cache); stack = cache->free; if (stack != 0) { cache->free = stack_next(stack); cache->count--; } else { if (stack_free_list != 0) { stack_lock(); stack = stack_free_list; if (stack != 0) { stack_free_list = stack_next(stack); stack_free_count--; stack_free_delta--; } stack_unlock(); } } if (stack != 0 || (stack = thread->reserved_stack) != 0) { machine_stack_attach(thread, stack); return (TRUE); } return (FALSE); }
void stack_free_stack( vm_offset_t stack) { struct stack_cache *cache; spl_t s; s = splsched(); cache = &PROCESSOR_DATA(current_processor(), stack_cache); if (cache->count < STACK_CACHE_SIZE) { stack_next(stack) = cache->free; cache->free = stack; cache->count++; } else { stack_lock(); stack_next(stack) = stack_free_list; stack_free_list = stack; if (++stack_free_count > stack_free_hiwat) stack_free_hiwat = stack_free_count; stack_free_delta++; stack_unlock(); } splx(s); }
/* may be called from an IPI */ int kpc_get_curcpu_counters(uint32_t classes, int *curcpu, uint64_t *buf) { int enabled=0, offset=0; uint64_t pmc_mask = 0ULL; assert(buf); enabled = ml_set_interrupts_enabled(FALSE); /* grab counters and CPU number as close as possible */ if (curcpu) *curcpu = current_processor()->cpu_id; if (classes & KPC_CLASS_FIXED_MASK) { kpc_get_fixed_counters(&buf[offset]); offset += kpc_get_counter_count(KPC_CLASS_FIXED_MASK); } if (classes & KPC_CLASS_CONFIGURABLE_MASK) { pmc_mask = kpc_get_configurable_pmc_mask(KPC_CLASS_CONFIGURABLE_MASK); kpc_get_configurable_counters(&buf[offset], pmc_mask); offset += kpc_popcount(pmc_mask); } if (classes & KPC_CLASS_POWER_MASK) { pmc_mask = kpc_get_configurable_pmc_mask(KPC_CLASS_POWER_MASK); kpc_get_configurable_counters(&buf[offset], pmc_mask); offset += kpc_popcount(pmc_mask); } ml_set_interrupts_enabled(enabled); return offset; }
boolean_t timer_call_cancel( timer_call_t call) { boolean_t result = TRUE; spl_t s; s = splclock(); simple_lock(&timer_call_lock); if (call->state == DELAYED) { queue_t queue = &PROCESSOR_DATA(current_processor(), timer_call_queue); if (queue_first(queue) == qe(call)) { _delayed_call_dequeue(call); if (!queue_empty(queue)) _set_delayed_call_timer((timer_call_t)queue_first(queue)); } else _delayed_call_dequeue(call); } else result = FALSE; simple_unlock(&timer_call_lock); splx(s); return (result); }
boolean_t timer_call_enter1( timer_call_t call, timer_call_param_t param1, uint64_t deadline) { boolean_t result = TRUE; queue_t queue; spl_t s; s = splclock(); simple_lock(&timer_call_lock); if (call->state == DELAYED) _delayed_call_dequeue(call); else result = FALSE; call->param1 = param1; call->deadline = deadline; queue = &PROCESSOR_DATA(current_processor(), timer_call_queue); _delayed_call_enqueue(queue, call); if (queue_first(queue) == qe(call)) _set_delayed_call_timer(call); simple_unlock(&timer_call_lock); splx(s); return (result); }
/** * etimer_intr * * Timer interrupt routine, called from the realtime clock interrupt * routine. */ void etimer_intr(int inuser, uint64_t iaddr) { uint64_t abstime; rtclock_timer_t *mytimer; cpu_data_t *pp; int32_t latency; pp = current_cpu_datap(); SCHED_STATS_TIMER_POP(current_processor()); abstime = mach_absolute_time(); /* Get the time now */ /* * has a pending clock timer expired? */ mytimer = &pp->rt_timer; /* Point to the event timer */ if (mytimer->deadline <= abstime) { mytimer->has_expired = TRUE; /* Remember that we popped */ mytimer->deadline = timer_queue_expire(&mytimer->queue, abstime); mytimer->has_expired = FALSE; } pp->rtcPop = EndOfAllTime; /* any real deadline will be earlier */ /* * schedule our next deadline */ etimer_resync_deadlines(); }
/* * Routine: cpu_doshutdown * Function: */ void cpu_doshutdown( void) { enable_preemption(); processor_offline(current_processor()); }
boolean_t swtch(void) { register processor_t myprocessor; #if NCPUS > 1 myprocessor = current_processor(); if (myprocessor->runq.count == 0 && myprocessor->processor_set->runq.count == 0) return(FALSE); #endif /* NCPUS > 1 */ counter(c_swtch_block++); thread_block(swtch_continue); myprocessor = current_processor(); return(myprocessor->runq.count > 0 || myprocessor->processor_set->runq.count > 0); }
void ast_taken(void) { thread_t self = current_thread(); ast_t reasons; /* * Interrupts are still disabled. * We must clear need_ast and then enable interrupts. */ reasons = need_ast[cpu_number()]; need_ast[cpu_number()] = AST_ZILCH; (void) spl0(); /* * These actions must not block. */ if (reasons & AST_NETWORK) net_ast(); /* * Make darn sure that we don't call thread_halt_self * or thread_block from the idle thread. */ if (self != current_processor()->idle_thread) { #ifndef MIGRATING_THREADS while (thread_should_halt(self)) thread_halt_self(); #endif /* * One of the previous actions might well have * woken a high-priority thread, so we use * csw_needed in addition to AST_BLOCK. */ if ((reasons & AST_BLOCK) || csw_needed(self, current_processor())) { counter(c_ast_taken_block++); thread_block(thread_exception_return); } } }
void swtch_continue(void) { register processor_t myprocessor; myprocessor = current_processor(); thread_syscall_return(myprocessor->runq.count > 0 || myprocessor->processor_set->runq.count > 0); /*NOTREACHED*/ }
/* * load_context: * * Start the first thread on a processor. */ static void load_context( thread_t thread) { processor_t processor = current_processor(); #define load_context_kprintf(x...) /* kprintf("load_context: " x) */ load_context_kprintf("calling machine_set_current_thread\n"); machine_set_current_thread(thread); load_context_kprintf("calling processor_up\n"); processor_up(processor); PMAP_ACTIVATE_KERNEL(processor->cpu_id); /* * Acquire a stack if none attached. The panic * should never occur since the thread is expected * to have reserved stack. */ load_context_kprintf("thread %p, stack %lx, stackptr %lx\n", thread, thread->kernel_stack, thread->machine.kstackptr); if (!thread->kernel_stack) { load_context_kprintf("calling stack_alloc_try\n"); if (!stack_alloc_try(thread)) panic("load_context"); } /* * The idle processor threads are not counted as * running for load calculations. */ if (!(thread->state & TH_IDLE)) sched_run_incr(); processor->active_thread = thread; processor->current_pri = thread->sched_pri; processor->current_thmode = thread->sched_mode; processor->deadline = UINT64_MAX; thread->last_processor = processor; processor->last_dispatch = mach_absolute_time(); timer_start(&thread->system_timer, processor->last_dispatch); PROCESSOR_DATA(processor, thread_timer) = PROCESSOR_DATA(processor, kernel_timer) = &thread->system_timer; timer_start(&PROCESSOR_DATA(processor, system_state), processor->last_dispatch); PROCESSOR_DATA(processor, current_state) = &PROCESSOR_DATA(processor, system_state); PMAP_ACTIVATE_USER(thread, processor->cpu_id); load_context_kprintf("calling machine_load_context\n"); machine_load_context(thread); /*NOTREACHED*/ }
void swtch_pri_continue(void) { register thread_t thread = current_thread(); register processor_t myprocessor; if (thread->depress_priority >= 0) (void) thread_depress_abort(thread); myprocessor = current_processor(); thread_syscall_return(myprocessor->runq.count > 0 || myprocessor->processor_set->runq.count > 0); /*NOTREACHED*/ }
int64_t dtrace_calc_thread_recent_vtime(thread_t thread) { if (thread != THREAD_NULL) { processor_t processor = current_processor(); uint64_t abstime = mach_absolute_time(); timer_t timer; timer = PROCESSOR_DATA(processor, thread_timer); return timer_grab(&(thread->system_timer)) + timer_grab(&(thread->user_timer)) + (abstime - timer->tstamp); /* XXX need interrupts off to prevent missed time? */ } else return 0; }
static void swtch_continue(void) { register processor_t myprocessor; boolean_t result; disable_preemption(); myprocessor = current_processor(); result = !SCHED(processor_queue_empty)(myprocessor) || rt_runq.count > 0; enable_preemption(); thread_syscall_return(result); /*NOTREACHED*/ }
/* * Adjust the Universal (Posix) time gradually. */ kern_return_t host_adjust_time( host_t host, time_value_t newadj, time_value_t *oldadj) /* OUT */ { time_value_t oadj; integer_t ndelta; spl_t s; if (host == HOST_NULL) return (KERN_INVALID_HOST); ndelta = (newadj.seconds * 1000000) + newadj.microseconds; #if NCPUS > 1 thread_bind(current_thread(), master_processor); mp_disable_preemption(); if (current_processor() != master_processor) { mp_enable_preemption(); thread_block((void (*)(void)) 0); } else { mp_enable_preemption(); } #endif /* NCPUS > 1 */ s = splclock(); oadj.seconds = timedelta / 1000000; oadj.microseconds = timedelta % 1000000; if (timedelta == 0) { if (ndelta > bigadj) tickdelta = 10 * tickadj; else tickdelta = tickadj; } if (ndelta % tickdelta) ndelta = ndelta / tickdelta * tickdelta; timedelta = ndelta; splx(s); #if NCPUS > 1 thread_bind(current_thread(), PROCESSOR_NULL); #endif /* NCPUS > 1 */ *oldadj = oadj; return (KERN_SUCCESS); }
/* * processor_start_thread: * * First thread to execute on a started processor. * * Called at splsched. */ void processor_start_thread(void *machine_param) { processor_t processor = current_processor(); thread_t self = current_thread(); slave_machine_init(machine_param); /* * If running the idle processor thread, * reenter the idle loop, else terminate. */ if (self == processor->idle_thread) thread_block((thread_continue_t)idle_thread); thread_terminate(self); /*NOTREACHED*/ }
/* * The scheduler is too messy for my old little brain */ void simpler_thread_setrun( thread_t th, boolean_t may_preempt) { register struct run_queue *rq; register int whichq; /* * XXX should replace queue with a boolean in this case. */ if (default_pset.idle_count > 0) { processor_t processor; processor = (processor_t) queue_first(&default_pset.idle_queue); queue_remove(&default_pset.idle_queue, processor, processor_t, processor_queue); default_pset.idle_count--; processor->next_thread = th; processor->state = PROCESSOR_DISPATCHING; return; } rq = &(master_processor->runq); ast_on(cpu_number(), AST_BLOCK); whichq = (th)->sched_pri; simple_lock(&(rq)->lock); /* lock the run queue */ enqueue_head(&(rq)->runq[whichq], (queue_entry_t) (th)); if (whichq < (rq)->low || (rq)->count == 0) (rq)->low = whichq; /* minimize */ (rq)->count++; #ifdef MIGRATING_THREADS (th)->shuttle.runq = (rq); #else (th)->runq = (rq); #endif simple_unlock(&(rq)->lock); /* * Turn off first_quantum to allow context switch. */ current_processor()->first_quantum = FALSE; }
/* * Complete the shutdown and place the processor offline. * * Called at splsched in the shutdown context. * This performs a minimal thread_invoke() to the idle thread, * so it needs to be kept in sync with what thread_invoke() does. * * The onlining half of this is done in load_context(). */ void processor_offline( processor_t processor) { assert(processor == current_processor()); assert(processor->active_thread == current_thread()); thread_t old_thread = processor->active_thread; thread_t new_thread = processor->idle_thread; processor->active_thread = new_thread; processor->current_pri = IDLEPRI; processor->current_thmode = TH_MODE_NONE; processor->starting_pri = IDLEPRI; processor->current_sfi_class = SFI_CLASS_KERNEL; processor->deadline = UINT64_MAX; new_thread->last_processor = processor; uint64_t ctime = mach_absolute_time(); processor->last_dispatch = ctime; old_thread->last_run_time = ctime; /* Update processor->thread_timer and ->kernel_timer to point to the new thread */ thread_timer_event(ctime, &new_thread->system_timer); PROCESSOR_DATA(processor, kernel_timer) = &new_thread->system_timer; timer_stop(PROCESSOR_DATA(processor, current_state), ctime); KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, MACHDBG_CODE(DBG_MACH_SCHED, MACH_SCHED) | DBG_FUNC_NONE, old_thread->reason, (uintptr_t)thread_tid(new_thread), old_thread->sched_pri, new_thread->sched_pri, 0); machine_set_current_thread(new_thread); thread_dispatch(old_thread, new_thread); PMAP_DEACTIVATE_KERNEL(processor->cpu_id); cpu_sleep(); panic("zombie processor"); /*NOTREACHED*/ }
static int pmThreadGetUrgency(uint64_t *rt_period, uint64_t *rt_deadline) { int urgency; uint64_t arg1, arg2; urgency = thread_get_urgency(current_processor()->next_thread, &arg1, &arg2); if (urgency == THREAD_URGENCY_REAL_TIME) { if (rt_period != NULL) *rt_period = arg1; if (rt_deadline != NULL) *rt_deadline = arg2; } KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_SCHED, MACH_SCHED_GET_URGENCY), urgency, arg1, arg2, 0, 0); return(urgency); }
/* * Set the Universal (Posix) time. Privileged call. */ kern_return_t host_set_time( host_t host, time_value_t new_time) { spl_t s; if (host == HOST_NULL) return(KERN_INVALID_HOST); #if NCPUS > 1 thread_bind(current_thread(), master_processor); mp_disable_preemption(); if (current_processor() != master_processor) { mp_enable_preemption(); thread_block((void (*)(void)) 0); } else { mp_enable_preemption(); } #endif /* NCPUS > 1 */ s = splhigh(); time = new_time; update_mapped_time(&time); #if PTIME_MACH_RT rtc_gettime_interrupts_disabled((tvalspec_t *)&last_utime_tick); #endif /* PTIME_MACH_RT */ #if 0 (void)bbc_settime((time_value_t *)&time); #endif splx(s); #if NCPUS > 1 thread_bind(current_thread(), PROCESSOR_NULL); #endif /* NCPUS > 1 */ return (KERN_SUCCESS); }
/* * slave_main: * * Load the first thread to start a processor. */ void slave_main(void *machine_param) { processor_t processor = current_processor(); thread_t thread; /* * Use the idle processor thread if there * is no dedicated start up thread. */ if (processor->next_thread == THREAD_NULL) { thread = processor->idle_thread; thread->continuation = (thread_continue_t)processor_start_thread; thread->parameter = machine_param; } else { thread = processor->next_thread; processor->next_thread = THREAD_NULL; } load_context(thread); /*NOTREACHED*/ }
void thread_poll_yield( thread_t self) { spl_t s; assert(self == current_thread()); s = splsched(); if (self->sched_mode == TH_MODE_FIXED) { uint64_t total_computation, abstime; abstime = mach_absolute_time(); total_computation = abstime - self->computation_epoch; total_computation += self->computation_metered; if (total_computation >= max_poll_computation) { processor_t myprocessor = current_processor(); ast_t preempt; thread_lock(self); if (!(self->sched_flags & TH_SFLAG_DEPRESSED_MASK)) { self->sched_pri = DEPRESSPRI; myprocessor->current_pri = self->sched_pri; } self->computation_epoch = abstime; self->computation_metered = 0; self->sched_flags |= TH_SFLAG_POLLDEPRESS; abstime += (total_computation >> sched_poll_yield_shift); if (!timer_call_enter(&self->depress_timer, abstime, TIMER_CALL_USER_CRITICAL)) self->depress_timer_active++; if ((preempt = csw_check(myprocessor, AST_NONE)) != AST_NONE) ast_on(preempt); thread_unlock(self); }
/* * Called at splsched. */ void ast_taken( ast_t reasons, boolean_t enable ) { boolean_t preempt_trap = (reasons == AST_PREEMPTION); ast_t *myast = ast_pending(); thread_t thread = current_thread(); perfASTCallback perf_hook = perfASTHook; /* * CHUD hook - all threads including idle processor threads */ if (perf_hook) { if (*myast & AST_CHUD_ALL) { (*perf_hook)(reasons, myast); if (*myast == AST_NONE) return; } } else *myast &= ~AST_CHUD_ALL; reasons &= *myast; *myast &= ~reasons; /* * Handle ASTs for all threads * except idle processor threads. */ if (!(thread->state & TH_IDLE)) { /* * Check for urgent preemption. */ if ( (reasons & AST_URGENT) && waitq_wait_possible(thread) ) { if (reasons & AST_PREEMPT) { counter(c_ast_taken_block++); thread_block_reason(THREAD_CONTINUE_NULL, NULL, reasons & AST_PREEMPTION); } reasons &= ~AST_PREEMPTION; } /* * The kernel preempt traps * skip all other ASTs. */ if (!preempt_trap) { ml_set_interrupts_enabled(enable); #ifdef MACH_BSD /* * Handle BSD hook. */ if (reasons & AST_BSD) { thread_ast_clear(thread, AST_BSD); bsd_ast(thread); } #endif #if CONFIG_MACF /* * Handle MACF hook. */ if (reasons & AST_MACF) { thread_ast_clear(thread, AST_MACF); mac_thread_userret(thread); } #endif /* * Thread APC hook. */ if (reasons & AST_APC) { thread_ast_clear(thread, AST_APC); special_handler(thread); } if (reasons & AST_GUARD) { thread_ast_clear(thread, AST_GUARD); guard_ast(thread); } if (reasons & AST_LEDGER) { thread_ast_clear(thread, AST_LEDGER); ledger_ast(thread); } /* * Kernel Profiling Hook */ if (reasons & AST_KPERF) { thread_ast_clear(thread, AST_KPERF); chudxnu_thread_ast(thread); } #if CONFIG_TELEMETRY if (reasons & AST_TELEMETRY_ALL) { boolean_t interrupted_userspace = FALSE; boolean_t is_windowed = FALSE; assert((reasons & AST_TELEMETRY_ALL) != AST_TELEMETRY_ALL); /* only one is valid at a time */ interrupted_userspace = (reasons & AST_TELEMETRY_USER) ? TRUE : FALSE; is_windowed = ((reasons & AST_TELEMETRY_WINDOWED) ? TRUE : FALSE); thread_ast_clear(thread, AST_TELEMETRY_ALL); telemetry_ast(thread, interrupted_userspace, is_windowed); } #endif ml_set_interrupts_enabled(FALSE); #if CONFIG_SCHED_SFI if (reasons & AST_SFI) { sfi_ast(thread); } #endif /* * Check for preemption. Conditions may have changed from when the AST_PREEMPT was originally set. */ thread_lock(thread); if (reasons & AST_PREEMPT) reasons = csw_check(current_processor(), reasons & AST_QUANTUM); thread_unlock(thread); assert(waitq_wait_possible(thread)); if (reasons & AST_PREEMPT) { counter(c_ast_taken_block++); thread_block_reason((thread_continue_t)thread_exception_return, NULL, reasons & AST_PREEMPTION); } } } ml_set_interrupts_enabled(enable); }
/* * Now running in a thread. Kick off other services, * invoke user bootstrap, enter pageout loop. */ static void kernel_bootstrap_thread(void) { processor_t processor = current_processor(); #define kernel_bootstrap_thread_kprintf(x...) /* kprintf("kernel_bootstrap_thread: " x) */ kernel_bootstrap_thread_kprintf("calling idle_thread_create\n"); /* * Create the idle processor thread. */ idle_thread_create(processor); /* * N.B. Do not stick anything else * before this point. * * Start up the scheduler services. */ kernel_bootstrap_thread_kprintf("calling sched_startup\n"); sched_startup(); /* * Thread lifecycle maintenance (teardown, stack allocation) */ kernel_bootstrap_thread_kprintf("calling thread_daemon_init\n"); thread_daemon_init(); /* * Thread callout service. */ kernel_bootstrap_thread_kprintf("calling thread_call_initialize\n"); thread_call_initialize(); /* * Remain on current processor as * additional processors come online. */ kernel_bootstrap_thread_kprintf("calling thread_bind\n"); thread_bind(processor); /* * Kick off memory mapping adjustments. */ kernel_bootstrap_thread_kprintf("calling mapping_adjust\n"); mapping_adjust(); /* * Create the clock service. */ kernel_bootstrap_thread_kprintf("calling clock_service_create\n"); clock_service_create(); /* * Create the device service. */ device_service_create(); kth_started = 1; #if (defined(__i386__) || defined(__x86_64__)) && NCOPY_WINDOWS > 0 /* * Create and initialize the physical copy window for processor 0 * This is required before starting kicking off IOKit. */ cpu_physwindow_init(0); #endif vm_kernel_reserved_entry_init(); #if MACH_KDP kernel_bootstrap_kprintf("calling kdp_init\n"); kdp_init(); #endif #if CONFIG_COUNTERS pmc_bootstrap(); #endif #if (defined(__i386__) || defined(__x86_64__)) if (turn_on_log_leaks && !new_nkdbufs) new_nkdbufs = 200000; start_kern_tracing(new_nkdbufs); if (turn_on_log_leaks) log_leaks = 1; #endif #ifdef IOKIT PE_init_iokit(); #endif (void) spllo(); /* Allow interruptions */ #if (defined(__i386__) || defined(__x86_64__)) && NCOPY_WINDOWS > 0 /* * Create and initialize the copy window for processor 0 * This also allocates window space for all other processors. * However, this is dependent on the number of processors - so this call * must be after IOKit has been started because IOKit performs processor * discovery. */ cpu_userwindow_init(0); #endif #if (!defined(__i386__) && !defined(__x86_64__)) if (turn_on_log_leaks && !new_nkdbufs) new_nkdbufs = 200000; start_kern_tracing(new_nkdbufs); if (turn_on_log_leaks) log_leaks = 1; #endif /* * Initialize the shared region module. */ vm_shared_region_init(); vm_commpage_init(); vm_commpage_text_init(); #if CONFIG_MACF mac_policy_initmach(); #endif /* * Initialize the global used for permuting kernel * addresses that may be exported to userland as tokens * using VM_KERNEL_ADDRPERM(). Force the random number * to be odd to avoid mapping a non-zero * word-aligned address to zero via addition. */ vm_kernel_addrperm = (vm_offset_t)early_random() | 1; /* * Start the user bootstrap. */ #ifdef MACH_BSD bsd_init(); #endif /* * Get rid of segments used to bootstrap kext loading. This removes * the KLD, PRELINK symtab, LINKEDIT, and symtab segments/load commands. */ #if 0 OSKextRemoveKextBootstrap(); #endif serial_keyboard_init(); /* Start serial keyboard if wanted */ vm_page_init_local_q(); thread_bind(PROCESSOR_NULL); /* * Become the pageout daemon. */ vm_pageout(); /*NOTREACHED*/ }
uint64_t timer_queue_expire_with_options( mpqueue_head_t *queue, uint64_t deadline, boolean_t rescan) { timer_call_t call = NULL; uint32_t tc_iterations = 0; DBG("timer_queue_expire(%p,)\n", queue); uint64_t cur_deadline = deadline; timer_queue_lock_spin(queue); while (!queue_empty(&queue->head)) { /* Upon processing one or more timer calls, refresh the * deadline to account for time elapsed in the callout */ if (++tc_iterations > 1) cur_deadline = mach_absolute_time(); if (call == NULL) call = TIMER_CALL(queue_first(&queue->head)); if (call->soft_deadline <= cur_deadline) { timer_call_func_t func; timer_call_param_t param0, param1; TCOAL_DEBUG(0xDDDD0000, queue->earliest_soft_deadline, call->soft_deadline, 0, 0, 0); TIMER_KDEBUG_TRACE(KDEBUG_TRACE, DECR_TIMER_EXPIRE | DBG_FUNC_NONE, call, call->soft_deadline, CE(call)->deadline, CE(call)->entry_time, 0); /* Bit 0 of the "soft" deadline indicates that * this particular timer call is rate-limited * and hence shouldn't be processed before its * hard deadline. */ if ((call->soft_deadline & 0x1) && (CE(call)->deadline > cur_deadline)) { if (rescan == FALSE) break; } if (!simple_lock_try(&call->lock)) { /* case (2b) lock inversion, dequeue and skip */ timer_queue_expire_lock_skips++; timer_call_entry_dequeue_async(call); call = NULL; continue; } timer_call_entry_dequeue(call); func = CE(call)->func; param0 = CE(call)->param0; param1 = CE(call)->param1; simple_unlock(&call->lock); timer_queue_unlock(queue); TIMER_KDEBUG_TRACE(KDEBUG_TRACE, DECR_TIMER_CALLOUT | DBG_FUNC_START, call, VM_KERNEL_UNSLIDE(func), param0, param1, 0); #if CONFIG_DTRACE DTRACE_TMR7(callout__start, timer_call_func_t, func, timer_call_param_t, param0, unsigned, call->flags, 0, (call->ttd >> 32), (unsigned) (call->ttd & 0xFFFFFFFF), call); #endif /* Maintain time-to-deadline in per-processor data * structure for thread wakeup deadline statistics. */ uint64_t *ttdp = &(PROCESSOR_DATA(current_processor(), timer_call_ttd)); *ttdp = call->ttd; (*func)(param0, param1); *ttdp = 0; #if CONFIG_DTRACE DTRACE_TMR4(callout__end, timer_call_func_t, func, param0, param1, call); #endif TIMER_KDEBUG_TRACE(KDEBUG_TRACE, DECR_TIMER_CALLOUT | DBG_FUNC_END, call, VM_KERNEL_UNSLIDE(func), param0, param1, 0); call = NULL; timer_queue_lock_spin(queue); } else { if (__probable(rescan == FALSE)) {
/* * Event timer interrupt. * * XXX a drawback of this implementation is that events serviced earlier must not set deadlines * that occur before the entire chain completes. * * XXX a better implementation would use a set of generic callouts and iterate over them */ void timer_intr(int user_mode, uint64_t rip) { uint64_t abstime; rtclock_timer_t *mytimer; cpu_data_t *pp; int64_t latency; uint64_t pmdeadline; boolean_t timer_processed = FALSE; pp = current_cpu_datap(); SCHED_STATS_TIMER_POP(current_processor()); abstime = mach_absolute_time(); /* Get the time now */ /* has a pending clock timer expired? */ mytimer = &pp->rtclock_timer; /* Point to the event timer */ if ((timer_processed = ((mytimer->deadline <= abstime) || (abstime >= (mytimer->queue.earliest_soft_deadline))))) { /* * Log interrupt service latency (-ve value expected by tool) * a non-PM event is expected next. * The requested deadline may be earlier than when it was set * - use MAX to avoid reporting bogus latencies. */ latency = (int64_t) (abstime - MAX(mytimer->deadline, mytimer->when_set)); /* Log zero timer latencies when opportunistically processing * coalesced timers. */ if (latency < 0) { TCOAL_DEBUG(0xEEEE0000, abstime, mytimer->queue.earliest_soft_deadline, abstime - mytimer->queue.earliest_soft_deadline, 0, 0); latency = 0; } KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, DECR_TRAP_LATENCY | DBG_FUNC_NONE, -latency, ((user_mode != 0) ? rip : VM_KERNEL_UNSLIDE(rip)), user_mode, 0, 0); mytimer->has_expired = TRUE; /* Remember that we popped */ mytimer->deadline = timer_queue_expire(&mytimer->queue, abstime); mytimer->has_expired = FALSE; /* Get the time again since we ran a bit */ abstime = mach_absolute_time(); mytimer->when_set = abstime; } /* is it time for power management state change? */ if ((pmdeadline = pmCPUGetDeadline(pp)) && (pmdeadline <= abstime)) { KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, DECR_PM_DEADLINE | DBG_FUNC_START, 0, 0, 0, 0, 0); pmCPUDeadline(pp); KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, DECR_PM_DEADLINE | DBG_FUNC_END, 0, 0, 0, 0, 0); timer_processed = TRUE; } /* schedule our next deadline */ x86_lcpu()->rtcDeadline = EndOfAllTime; timer_resync_deadlines(); if (__improbable(timer_processed == FALSE)) spurious_timers++; }