cpu_idle(void) { cpu_data_t *cpu_data_ptr = getCpuDatap(); uint64_t new_idle_timeout_ticks = 0x0ULL, lastPop; if ((!idle_enable) || (cpu_data_ptr->cpu_signal & SIGPdisabled)) Idle_load_context(); if (!SetIdlePop()) Idle_load_context(); lastPop = cpu_data_ptr->rtcPop; pmap_switch_user_ttb(kernel_pmap); cpu_data_ptr->cpu_active_thread = current_thread(); if (cpu_data_ptr->cpu_user_debug) arm_debug_set(NULL); cpu_data_ptr->cpu_user_debug = NULL; if (cpu_data_ptr->cpu_idle_notify) ((processor_idle_t) cpu_data_ptr->cpu_idle_notify) (cpu_data_ptr->cpu_id, TRUE, &new_idle_timeout_ticks); if (cpu_data_ptr->idle_timer_notify != 0) { if (new_idle_timeout_ticks == 0x0ULL) { /* turn off the idle timer */ cpu_data_ptr->idle_timer_deadline = 0x0ULL; } else { /* set the new idle timeout */ clock_absolutetime_interval_to_deadline(new_idle_timeout_ticks, &cpu_data_ptr->idle_timer_deadline); } timer_resync_deadlines(); if (cpu_data_ptr->rtcPop != lastPop) SetIdlePop(); } #if KPC kpc_idle(); #endif platform_cache_idle_enter(); cpu_idle_wfi((boolean_t) wfi_fast); platform_cache_idle_exit(); ClearIdlePop(TRUE); cpu_idle_exit(FALSE); }
static void acpi_cpu_mwait_ipi_check_wakeup(void *arg) { volatile uint32_t *mcpu_mwait = (volatile uint32_t *)arg; ASSERT(arg != NULL); if (*mcpu_mwait != MWAIT_WAKEUP_IPI) { /* * CPU has been awakened, notify CPU idle notification system. */ cpu_idle_exit(CPU_IDLE_CB_FLAG_IDLE); } else { /* * Toggle interrupt flag to detect pending interrupts. * If interrupt happened, do_interrupt() will notify CPU idle * notification framework so no need to call cpu_idle_exit() * here. */ sti(); SMT_PAUSE(); cli(); } }
/* * enter deep c-state handler */ static void acpi_cpu_cstate(cpu_acpi_cstate_t *cstate) { volatile uint32_t *mcpu_mwait = CPU->cpu_m.mcpu_mwait; cpu_t *cpup = CPU; processorid_t cpu_sid = cpup->cpu_seqid; cpupart_t *cp = cpup->cpu_part; hrtime_t lapic_expire; uint8_t type = cstate->cs_addrspace_id; uint32_t cs_type = cstate->cs_type; int hset_update = 1; boolean_t using_timer; cpu_idle_check_wakeup_t check_func = &acpi_cpu_check_wakeup; /* * Set our mcpu_mwait here, so we can tell if anyone tries to * wake us between now and when we call mwait. No other cpu will * attempt to set our mcpu_mwait until we add ourself to the haltset. */ if (mcpu_mwait) { if (type == ACPI_ADR_SPACE_SYSTEM_IO) { *mcpu_mwait = MWAIT_WAKEUP_IPI; check_func = &acpi_cpu_mwait_ipi_check_wakeup; } else { *mcpu_mwait = MWAIT_HALTED; check_func = &acpi_cpu_mwait_check_wakeup; } } /* * If this CPU is online, and there are multiple CPUs * in the system, then we should note our halting * by adding ourselves to the partition's halted CPU * bitmap. This allows other CPUs to find/awaken us when * work becomes available. */ if (cpup->cpu_flags & CPU_OFFLINE || ncpus == 1) hset_update = 0; /* * Add ourselves to the partition's halted CPUs bitmask * and set our HALTED flag, if necessary. * * When a thread becomes runnable, it is placed on the queue * and then the halted cpuset is checked to determine who * (if anyone) should be awakened. We therefore need to first * add ourselves to the halted cpuset, and and then check if there * is any work available. * * Note that memory barriers after updating the HALTED flag * are not necessary since an atomic operation (updating the bitmap) * immediately follows. On x86 the atomic operation acts as a * memory barrier for the update of cpu_disp_flags. */ if (hset_update) { cpup->cpu_disp_flags |= CPU_DISP_HALTED; bitset_atomic_add(&cp->cp_haltset, cpu_sid); } /* * Check to make sure there's really nothing to do. * Work destined for this CPU may become available after * this check. We'll be notified through the clearing of our * bit in the halted CPU bitmask, and a write to our mcpu_mwait. * * disp_anywork() checks disp_nrunnable, so we do not have to later. */ if (disp_anywork()) { if (hset_update) { cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; bitset_atomic_del(&cp->cp_haltset, cpu_sid); } return; } /* * We're on our way to being halted. * * The local APIC timer can stop in ACPI C2 and deeper c-states. * Try to program the HPET hardware to substitute for this CPU's * LAPIC timer. * cstate_use_timer() could disable the LAPIC Timer. Make sure * to start the LAPIC Timer again before leaving this function. * * Disable interrupts here so we will awaken immediately after halting * if someone tries to poke us between now and the time we actually * halt. */ cli(); using_timer = cstate_use_timer(&lapic_expire, CSTATE_USING_HPET); /* * We check for the presence of our bit after disabling interrupts. * If it's cleared, we'll return. If the bit is cleared after * we check then the cstate_wakeup() will pop us out of the halted * state. * * This means that the ordering of the cstate_wakeup() and the clearing * of the bit by cpu_wakeup is important. * cpu_wakeup() must clear our mc_haltset bit, and then call * cstate_wakeup(). * acpi_cpu_cstate() must disable interrupts, then check for the bit. */ if (hset_update && bitset_in_set(&cp->cp_haltset, cpu_sid) == 0) { (void) cstate_use_timer(&lapic_expire, CSTATE_USING_LAT); sti(); cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; return; } /* * The check for anything locally runnable is here for performance * and isn't needed for correctness. disp_nrunnable ought to be * in our cache still, so it's inexpensive to check, and if there * is anything runnable we won't have to wait for the poke. */ if (cpup->cpu_disp->disp_nrunnable != 0) { (void) cstate_use_timer(&lapic_expire, CSTATE_USING_LAT); sti(); if (hset_update) { cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; bitset_atomic_del(&cp->cp_haltset, cpu_sid); } return; } if (using_timer == B_FALSE) { (void) cstate_use_timer(&lapic_expire, CSTATE_USING_LAT); sti(); /* * We are currently unable to program the HPET to act as this * CPU's proxy LAPIC timer. This CPU cannot enter C2 or deeper * because no timer is set to wake it up while its LAPIC timer * stalls in deep C-States. * Enter C1 instead. * * cstate_wake_cpu() will wake this CPU with an IPI which * works with MWAIT. */ i86_monitor(mcpu_mwait, 0, 0); if ((*mcpu_mwait & ~MWAIT_WAKEUP_IPI) == MWAIT_HALTED) { if (cpu_idle_enter(IDLE_STATE_C1, 0, check_func, (void *)mcpu_mwait) == 0) { if ((*mcpu_mwait & ~MWAIT_WAKEUP_IPI) == MWAIT_HALTED) { i86_mwait(0, 0); } cpu_idle_exit(CPU_IDLE_CB_FLAG_IDLE); } } /* * We're no longer halted */ if (hset_update) { cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; bitset_atomic_del(&cp->cp_haltset, cpu_sid); } return; } if (type == ACPI_ADR_SPACE_FIXED_HARDWARE) { /* * We're on our way to being halted. * To avoid a lost wakeup, arm the monitor before checking * if another cpu wrote to mcpu_mwait to wake us up. */ i86_monitor(mcpu_mwait, 0, 0); if (*mcpu_mwait == MWAIT_HALTED) { if (cpu_idle_enter((uint_t)cs_type, 0, check_func, (void *)mcpu_mwait) == 0) { if (*mcpu_mwait == MWAIT_HALTED) { i86_mwait(cstate->cs_address, 1); } cpu_idle_exit(CPU_IDLE_CB_FLAG_IDLE); } } } else if (type == ACPI_ADR_SPACE_SYSTEM_IO) { uint32_t value; ACPI_TABLE_FADT *gbl_FADT; if (*mcpu_mwait == MWAIT_WAKEUP_IPI) { if (cpu_idle_enter((uint_t)cs_type, 0, check_func, (void *)mcpu_mwait) == 0) { if (*mcpu_mwait == MWAIT_WAKEUP_IPI) { (void) cpu_acpi_read_port( cstate->cs_address, &value, 8); acpica_get_global_FADT(&gbl_FADT); (void) cpu_acpi_read_port( gbl_FADT->XPmTimerBlock.Address, &value, 32); } cpu_idle_exit(CPU_IDLE_CB_FLAG_IDLE); } } } /* * The LAPIC timer may have stopped in deep c-state. * Reprogram this CPU's LAPIC here before enabling interrupts. */ (void) cstate_use_timer(&lapic_expire, CSTATE_USING_LAT); sti(); /* * We're no longer halted */ if (hset_update) { cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; bitset_atomic_del(&cp->cp_haltset, cpu_sid); } }
/* * Interrupt service routine, called with interrupts disabled. */ void apix_do_interrupt(struct regs *rp, trap_trace_rec_t *ttp) { struct cpu *cpu = CPU; int vector = rp->r_trapno, newipl, oldipl = cpu->cpu_pri, ret; apix_vector_t *vecp = NULL; #ifdef TRAPTRACE ttp->ttr_marker = TT_INTERRUPT; ttp->ttr_cpuid = cpu->cpu_id; ttp->ttr_ipl = 0xff; ttp->ttr_pri = (uchar_t)oldipl; ttp->ttr_spl = cpu->cpu_base_spl; ttp->ttr_vector = 0xff; #endif /* TRAPTRACE */ cpu_idle_exit(CPU_IDLE_CB_FLAG_INTR); ++*(uint16_t *)&cpu->cpu_m.mcpu_istamp; /* * If it's a softint go do it now. */ if (rp->r_trapno == T_SOFTINT) { /* * It might be the case that when an interrupt is triggered, * the spl is raised to high by splhigh(). Later when do_splx() * is called to restore the spl, both hardware and software * interrupt pending flags are check and an SOFTINT is faked * accordingly. */ (void) apix_do_pending_hilevel(cpu, rp); (void) apix_do_pending_hardint(cpu, rp); (void) apix_do_softint(rp); ASSERT(!interrupts_enabled()); #ifdef TRAPTRACE ttp->ttr_vector = T_SOFTINT; #endif return; } /* * Send EOI to local APIC */ newipl = (*setlvl)(oldipl, (int *)&rp->r_trapno); #ifdef TRAPTRACE ttp->ttr_ipl = (uchar_t)newipl; #endif /* TRAPTRACE */ /* * Bail if it is a spurious interrupt */ if (newipl == -1) return; vector = rp->r_trapno; vecp = xv_vector(cpu->cpu_id, vector); #ifdef TRAPTRACE ttp->ttr_vector = (short)vector; #endif /* TRAPTRACE */ /* * Direct dispatch for IPI, MSI, MSI-X */ if (vecp && vecp->v_type != APIX_TYPE_FIXED && newipl > MAX(oldipl, cpu->cpu_base_spl)) { caddr_t newsp; if (newipl > LOCK_LEVEL) { if (apix_hilevel_intr_prolog(cpu, newipl, oldipl, rp) == 0) { newsp = cpu->cpu_intr_stack; switch_sp_and_call(newsp, apix_dispatch_hilevel, vector, 0); } else { apix_dispatch_hilevel(vector, 0); } (void) apix_hilevel_intr_epilog(cpu, oldipl); } else { newsp = apix_intr_thread_prolog(cpu, newipl, (caddr_t)rp); switch_sp_and_call(newsp, apix_dispatch_lowlevel, vector, oldipl); } } else { /* Add to per-pil pending queue */ apix_add_pending_hardint(vector); if (newipl <= MAX(oldipl, cpu->cpu_base_spl) || !apixs[cpu->cpu_id]->x_intr_pending) return; } if (apix_do_pending_hilevel(cpu, rp) < 0) return; do { ret = apix_do_pending_hardint(cpu, rp); /* * Deliver any pending soft interrupts. */ (void) apix_do_softint(rp); } while (!ret && LOWLEVEL_PENDING(cpu)); }
/*ARGSUSED*/ void do_interrupt(struct regs *rp, trap_trace_rec_t *ttp) { struct cpu *cpu = CPU; int newipl, oldipl = cpu->cpu_pri; uint_t vector; caddr_t newsp; #ifdef TRAPTRACE ttp->ttr_marker = TT_INTERRUPT; ttp->ttr_ipl = 0xff; ttp->ttr_pri = oldipl; ttp->ttr_spl = cpu->cpu_base_spl; ttp->ttr_vector = 0xff; #endif /* TRAPTRACE */ cpu_idle_exit(CPU_IDLE_CB_FLAG_INTR); ++*(uint16_t *)&cpu->cpu_m.mcpu_istamp; /* * If it's a softint go do it now. */ if (rp->r_trapno == T_SOFTINT) { dosoftint(rp); ASSERT(!interrupts_enabled()); return; } /* * Raise the interrupt priority. */ newipl = (*setlvl)(oldipl, (int *)&rp->r_trapno); #ifdef TRAPTRACE ttp->ttr_ipl = newipl; #endif /* TRAPTRACE */ /* * Bail if it is a spurious interrupt */ if (newipl == -1) return; cpu->cpu_pri = newipl; vector = rp->r_trapno; #ifdef TRAPTRACE ttp->ttr_vector = vector; #endif /* TRAPTRACE */ if (newipl > LOCK_LEVEL) { /* * High priority interrupts run on this cpu's interrupt stack. */ if (hilevel_intr_prolog(cpu, newipl, oldipl, rp) == 0) { newsp = cpu->cpu_intr_stack; switch_sp_and_call(newsp, dispatch_hilevel, vector, 0); } else { /* already on the interrupt stack */ dispatch_hilevel(vector, 0); } (void) hilevel_intr_epilog(cpu, newipl, oldipl, vector); } else { /* * Run this interrupt in a separate thread. */ newsp = intr_thread_prolog(cpu, (caddr_t)rp, newipl); switch_sp_and_call(newsp, dispatch_hardint, vector, oldipl); } #if !defined(__xpv) /* * Deliver any pending soft interrupts. */ if (cpu->cpu_softinfo.st_pending) dosoftint(rp); #endif /* !__xpv */ }