/*
* Callers should disable interrupts before the call and enable
* interrupts after return.
*/
static void __cpuidle acpi_safe_halt(void)
{
if (!tif_need_resched()) {
safe_halt();
local_irq_disable();
}
}
/*
* we use this if we don't have any better idle routine
*/
static void default_idle(void)
{
local_irq_disable();
if (!need_resched())
safe_halt();
else
local_irq_enable();
}
static void acpi_safe_halt(void)
{
current_thread_info()->status &= ~TS_POLLING;
smp_mb();
if (!need_resched()) {
safe_halt();
local_irq_disable();
}
current_thread_info()->status |= TS_POLLING;
}
/* XXX XEN doesn't use default_idle(), poll_idle(). Use xen_idle() instead. */
void xen_idle(void)
{
local_irq_disable();
if (need_resched())
local_irq_enable();
else {
safe_halt();
}
}
static void xen_idle(void)
{
local_irq_disable();
if (need_resched())
local_irq_enable();
else {
current_thread_info()->status &= ~TS_POLLING;
smp_mb__after_clear_bit();
safe_halt();
current_thread_info()->status |= TS_POLLING;
}
}
/*
* Callers should disable interrupts before the call and enable
* interrupts after return.
*/
static void acpi_safe_halt(void)
{
current_thread_info()->status &= ~TS_POLLING;
/*
* TS_POLLING-cleared state must be visible before we
* test NEED_RESCHED:
*/
smp_mb();
if (!need_resched()) {
safe_halt();
local_irq_disable();
}
current_thread_info()->status |= TS_POLLING;
}
/**
* acpi_idle_play_dead - enters an ACPI state for long-term idle (i.e. off-lining)
* @dev: the target CPU
* @index: the index of suggested state
*/
static int acpi_idle_play_dead(struct cpuidle_device *dev, int index)
{
struct acpi_processor_cx *cx = per_cpu(acpi_cstate[index], dev->cpu);
ACPI_FLUSH_CPU_CACHE();
while (1) {
if (cx->entry_method == ACPI_CSTATE_HALT)
safe_halt();
else if (cx->entry_method == ACPI_CSTATE_SYSTEMIO) {
inb(cx->address);
/* See comment in acpi_idle_do_entry() */
inl(acpi_gbl_FADT.xpm_timer_block.address);
} else
return -ENODEV;
}
/* Never reached */
return 0;
}
/*
* We use this if we don't have any better
* idle routine..
*/
void default_idle(void)
{
if (!hlt_counter && boot_cpu_data.hlt_works_ok) {
current_thread_info()->status &= ~TS_POLLING;
/*
* TS_POLLING-cleared state must be visible before we
* test NEED_RESCHED:
*/
smp_mb();
if (!need_resched())
safe_halt(); /* enables interrupts racelessly */
else
local_irq_enable();
current_thread_info()->status |= TS_POLLING;
} else {
local_irq_enable();
/* loop is done by the caller */
cpu_relax();
}
}
static void acpi_idle_do_entry(struct acpi_processor_cx *cx)
{
switch ( cx->entry_method )
{
case ACPI_CSTATE_EM_FFH:
/* Call into architectural FFH based C-state */
acpi_processor_ffh_cstate_enter(cx);
return;
case ACPI_CSTATE_EM_SYSIO:
/* IO port based C-state */
inb(cx->address);
/* Dummy wait op - must do something useless after P_LVL2 read
because chipsets cannot guarantee that STPCLK# signal
gets asserted in time to freeze execution properly. */
inl(pmtmr_ioport);
return;
case ACPI_CSTATE_EM_HALT:
safe_halt();
local_irq_disable();
return;
}
}
static int acpi_idle_play_dead(struct cpuidle_device *dev, int index)
{
struct cpuidle_state_usage *state_usage = &dev->states_usage[index];
struct acpi_processor_cx *cx = cpuidle_get_statedata(state_usage);
ACPI_FLUSH_CPU_CACHE();
while (1) {
if (cx->entry_method == ACPI_CSTATE_HALT)
safe_halt();
else if (cx->entry_method == ACPI_CSTATE_SYSTEMIO) {
inb(cx->address);
inl(acpi_gbl_FADT.xpm_timer_block.address);
} else
return -ENODEV;
}
return 0;
}
/*
* We use this if we don't have any better idle routine..
*/
void default_idle(void)
{
trace_cpu_idle_rcuidle(1, smp_processor_id());
safe_halt();
trace_cpu_idle_rcuidle(PWR_EVENT_EXIT, smp_processor_id());
}
static void acpi_processor_idle(void)
{
struct acpi_processor_power *power = processor_powers[smp_processor_id()];
struct acpi_processor_cx *cx = NULL;
int next_state;
uint64_t t1, t2 = 0;
u32 exp = 0, pred = 0;
u32 irq_traced[4] = { 0 };
if ( max_cstate > 0 && power && !sched_has_urgent_vcpu() &&
(next_state = cpuidle_current_governor->select(power)) > 0 )
{
cx = &power->states[next_state];
if ( power->flags.bm_check && acpi_idle_bm_check()
&& cx->type == ACPI_STATE_C3 )
cx = power->safe_state;
if ( cx->idx > max_cstate )
cx = &power->states[max_cstate];
menu_get_trace_data(&exp, &pred);
}
if ( !cx )
{
if ( pm_idle_save )
pm_idle_save();
else
safe_halt();
return;
}
cpufreq_dbs_timer_suspend();
sched_tick_suspend();
/* sched_tick_suspend() can raise TIMER_SOFTIRQ. Process it now. */
process_pending_softirqs();
/*
* Interrupts must be disabled during bus mastering calculations and
* for C2/C3 transitions.
*/
local_irq_disable();
if ( !cpu_is_haltable(smp_processor_id()) )
{
local_irq_enable();
sched_tick_resume();
cpufreq_dbs_timer_resume();
return;
}
if ( (cx->type == ACPI_STATE_C3) && errata_c6_eoi_workaround() )
cx = power->safe_state;
power->last_state = cx;
/*
* Sleep:
* ------
* Invoke the current Cx state to put the processor to sleep.
*/
switch ( cx->type )
{
case ACPI_STATE_C1:
case ACPI_STATE_C2:
if ( cx->type == ACPI_STATE_C1 || local_apic_timer_c2_ok )
{
/* Get start time (ticks) */
t1 = get_tick();
/* Trace cpu idle entry */
TRACE_4D(TRC_PM_IDLE_ENTRY, cx->idx, t1, exp, pred);
/* Invoke C2 */
acpi_idle_do_entry(cx);
/* Get end time (ticks) */
t2 = get_tick();
trace_exit_reason(irq_traced);
/* Trace cpu idle exit */
TRACE_6D(TRC_PM_IDLE_EXIT, cx->idx, t2,
irq_traced[0], irq_traced[1], irq_traced[2], irq_traced[3]);
/* Update statistics */
acpi_update_idle_stats(power, cx, ticks_elapsed(t1, t2));
/* Re-enable interrupts */
local_irq_enable();
break;
}
case ACPI_STATE_C3:
/*
* Before invoking C3, be aware that TSC/APIC timer may be
* stopped by H/W. Without carefully handling of TSC/APIC stop issues,
* deep C state can't work correctly.
*/
/* preparing APIC stop */
lapic_timer_off();
/* Get start time (ticks) */
t1 = get_tick();
/* Trace cpu idle entry */
TRACE_4D(TRC_PM_IDLE_ENTRY, cx->idx, t1, exp, pred);
/*
* disable bus master
* bm_check implies we need ARB_DIS
* !bm_check implies we need cache flush
* bm_control implies whether we can do ARB_DIS
*
* That leaves a case where bm_check is set and bm_control is
* not set. In that case we cannot do much, we enter C3
* without doing anything.
*/
if ( cx->type != ACPI_STATE_C3 )
/* nothing to be done here */;
else if ( power->flags.bm_check && power->flags.bm_control )
{
spin_lock(&c3_cpu_status.lock);
if ( ++c3_cpu_status.count == num_online_cpus() )
{
/*
* All CPUs are trying to go to C3
* Disable bus master arbitration
*/
acpi_set_register(ACPI_BITREG_ARB_DISABLE, 1);
}
spin_unlock(&c3_cpu_status.lock);
}
else if ( !power->flags.bm_check )
{
/* SMP with no shared cache... Invalidate cache */
ACPI_FLUSH_CPU_CACHE();
}
/* Invoke C3 */
acpi_idle_do_entry(cx);
if ( (cx->type == ACPI_STATE_C3) &&
power->flags.bm_check && power->flags.bm_control )
{
/* Enable bus master arbitration */
spin_lock(&c3_cpu_status.lock);
acpi_set_register(ACPI_BITREG_ARB_DISABLE, 0);
c3_cpu_status.count--;
spin_unlock(&c3_cpu_status.lock);
}
/* Get end time (ticks) */
t2 = get_tick();
/* recovering TSC */
cstate_restore_tsc();
trace_exit_reason(irq_traced);
/* Trace cpu idle exit */
TRACE_6D(TRC_PM_IDLE_EXIT, cx->idx, t2,
irq_traced[0], irq_traced[1], irq_traced[2], irq_traced[3]);
/* Update statistics */
acpi_update_idle_stats(power, cx, ticks_elapsed(t1, t2));
/* Re-enable interrupts */
local_irq_enable();
/* recovering APIC */
lapic_timer_on();
break;
default:
/* Now in C0 */
power->last_state = &power->states[0];
local_irq_enable();
sched_tick_resume();
cpufreq_dbs_timer_resume();
return;
}
/* Now in C0 */
power->last_state = &power->states[0];
sched_tick_resume();
cpufreq_dbs_timer_resume();
if ( cpuidle_current_governor->reflect )
cpuidle_current_governor->reflect(power);
}