static int default_idle(void)
{
long oldval;
unsigned int cpu = smp_processor_id();
while (1) {
oldval = test_and_clear_thread_flag(TIF_NEED_RESCHED);
if (!oldval) {
set_thread_flag(TIF_POLLING_NRFLAG);
while (!need_resched() && !cpu_is_offline(cpu)) {
barrier();
/*
* Go into low thread priority and possibly
* low power mode.
*/
HMT_low();
HMT_very_low();
}
HMT_medium();
clear_thread_flag(TIF_POLLING_NRFLAG);
} else {
set_need_resched();
}
schedule();
if (cpu_is_offline(cpu) && system_state == SYSTEM_RUNNING)
cpu_die();
}
return 0;
}
/*
* The body of the idle task.
*/
void cpu_idle(void)
{
if (ppc_md.idle_loop)
ppc_md.idle_loop(); /* doesn't return */
set_thread_flag(TIF_POLLING_NRFLAG);
while (1) {
tick_nohz_idle_enter();
rcu_idle_enter();
while (!need_resched() && !cpu_should_die()) {
ppc64_runlatch_off();
if (ppc_md.power_save) {
clear_thread_flag(TIF_POLLING_NRFLAG);
/*
* smp_mb is so clearing of TIF_POLLING_NRFLAG
* is ordered w.r.t. need_resched() test.
*/
smp_mb();
local_irq_disable();
/* Don't trace irqs off for idle */
stop_critical_timings();
/* check again after disabling irqs */
if (!need_resched() && !cpu_should_die())
ppc_md.power_save();
start_critical_timings();
/* Some power_save functions return with
* interrupts enabled, some don't.
*/
if (irqs_disabled())
local_irq_enable();
set_thread_flag(TIF_POLLING_NRFLAG);
} else {
/*
* Go into low thread priority and possibly
* low power mode.
*/
HMT_low();
HMT_very_low();
}
}
HMT_medium();
ppc64_runlatch_on();
rcu_idle_exit();
tick_nohz_idle_exit();
if (cpu_should_die()) {
sched_preempt_enable_no_resched();
cpu_die();
}
schedule_preempt_disabled();
}
}
/*
* The body of the idle task.
*/
void cpu_idle(void)
{
if (ppc_md.idle_loop)
ppc_md.idle_loop(); /* doesn't return */
set_thread_flag(TIF_POLLING_NRFLAG);
while (1) {
hrtimer_stop_sched_tick();
while (!need_resched() && !need_resched_delayed() &&
!cpu_should_die()) {
ppc64_runlatch_off();
/*
*
* We have irqs disabled here, so stop latency tracing
* at this point and restart it after we return:
*/
stop_critical_timing();
if (ppc_md.power_save) {
clear_thread_flag(TIF_POLLING_NRFLAG);
/*
* smp_mb is so clearing of TIF_POLLING_NRFLAG
* is ordered w.r.t. need_resched() test.
*/
smp_mb();
local_irq_disable();
/* check again after disabling irqs */
if (!need_resched() && !cpu_should_die())
ppc_md.power_save();
local_irq_enable();
set_thread_flag(TIF_POLLING_NRFLAG);
} else {
/*
* Go into low thread priority and possibly
* low power mode.
*/
HMT_low();
HMT_very_low();
}
touch_critical_timing();
}
HMT_medium();
ppc64_runlatch_on();
if (cpu_should_die())
cpu_die();
hrtimer_restart_sched_tick();
__preempt_enable_no_resched();
schedule();
preempt_disable();
}
}
static int pseries_dedicated_idle(void)
{
long oldval;
struct paca_struct *lpaca = get_paca();
unsigned int cpu = smp_processor_id();
unsigned long start_snooze;
unsigned long *smt_snooze_delay = &__get_cpu_var(smt_snooze_delay);
while (1) {
/*
* Indicate to the HV that we are idle. Now would be
* a good time to find other work to dispatch.
*/
lpaca->lppaca.idle = 1;
oldval = test_and_clear_thread_flag(TIF_NEED_RESCHED);
if (!oldval) {
set_thread_flag(TIF_POLLING_NRFLAG);
start_snooze = __get_tb() +
*smt_snooze_delay * tb_ticks_per_usec;
while (!need_resched() && !cpu_is_offline(cpu)) {
ppc64_runlatch_off();
/*
* Go into low thread priority and possibly
* low power mode.
*/
HMT_low();
HMT_very_low();
if (*smt_snooze_delay != 0 &&
__get_tb() > start_snooze) {
HMT_medium();
dedicated_idle_sleep(cpu);
}
}
HMT_medium();
clear_thread_flag(TIF_POLLING_NRFLAG);
} else {
set_need_resched();
}
lpaca->lppaca.idle = 0;
ppc64_runlatch_on();
schedule();
if (cpu_is_offline(cpu) && system_state == SYSTEM_RUNNING)
cpu_die();
}
}
void cpu_idle(void)
{
if (ppc_md.idle_loop)
ppc_md.idle_loop();
set_thread_flag(TIF_POLLING_NRFLAG);
while (1) {
tick_nohz_idle_enter();
rcu_idle_enter();
while (!need_resched() && !cpu_should_die()) {
ppc64_runlatch_off();
if (ppc_md.power_save) {
clear_thread_flag(TIF_POLLING_NRFLAG);
smp_mb();
local_irq_disable();
stop_critical_timings();
if (!need_resched() && !cpu_should_die())
ppc_md.power_save();
start_critical_timings();
if (irqs_disabled())
local_irq_enable();
set_thread_flag(TIF_POLLING_NRFLAG);
} else {
HMT_low();
HMT_very_low();
}
}
HMT_medium();
ppc64_runlatch_on();
rcu_idle_exit();
tick_nohz_idle_exit();
if (cpu_should_die()) {
sched_preempt_enable_no_resched();
cpu_die();
}
schedule_preempt_disabled();
}
}
/*
* The body of the idle task.
*/
void cpu_idle(void)
{
if (ppc_md.idle_loop)
ppc_md.idle_loop(); /* doesn't return */
set_thread_flag(TIF_POLLING_NRFLAG);
while (1) {
tick_nohz_stop_sched_tick();
while (!need_resched() && !cpu_should_die()) {
ppc64_runlatch_off();
if (ppc_md.power_save) {
clear_thread_flag(TIF_POLLING_NRFLAG);
/*
* smp_mb is so clearing of TIF_POLLING_NRFLAG
* is ordered w.r.t. need_resched() test.
*/
smp_mb();
local_irq_disable();
/* check again after disabling irqs */
if (!need_resched() && !cpu_should_die())
ppc_md.power_save();
local_irq_enable();
set_thread_flag(TIF_POLLING_NRFLAG);
} else {
/*
* Go into low thread priority and possibly
* low power mode.
*/
HMT_low();
HMT_very_low();
}
}
HMT_medium();
ppc64_runlatch_on();
tick_nohz_restart_sched_tick();
if (cpu_should_die())
cpu_die();
preempt_enable_no_resched();
schedule();
preempt_disable();
}
}
static int pseries_shared_idle(void)
{
struct paca_struct *lpaca = get_paca();
unsigned int cpu = smp_processor_id();
while (1) {
/*
* Indicate to the HV that we are idle. Now would be
* a good time to find other work to dispatch.
*/
lpaca->lppaca.idle = 1;
while (!need_resched() && !cpu_is_offline(cpu)) {
local_irq_disable();
ppc64_runlatch_off();
/*
* Yield the processor to the hypervisor. We return if
* an external interrupt occurs (which are driven prior
* to returning here) or if a prod occurs from another
* processor. When returning here, external interrupts
* are enabled.
*
* Check need_resched() again with interrupts disabled
* to avoid a race.
*/
if (!need_resched())
cede_processor();
else
local_irq_enable();
HMT_medium();
}
lpaca->lppaca.idle = 0;
ppc64_runlatch_on();
schedule();
if (cpu_is_offline(cpu) && system_state == SYSTEM_RUNNING)
cpu_die();
}
return 0;
}
/*
* The idle thread, has rather strange semantics for calling pm_idle,
* but this is what x86 does and we need to do the same, so that
* things like cpuidle get called in the same way. The only difference
* is that we always respect 'hlt_counter' to prevent low power idle.
*/
void cpu_idle(void)
{
local_fiq_enable();
/* endless idle loop with no priority at all */
while (1) {
idle_notifier_call_chain(IDLE_START);
tick_nohz_idle_enter();
rcu_idle_enter();
while (!need_resched()) {
/*
* We need to disable interrupts here
* to ensure we don't miss a wakeup call.
*/
local_irq_disable();
#ifdef CONFIG_PL310_ERRATA_769419
wmb();
#endif
if (hlt_counter) {
local_irq_enable();
cpu_relax();
} else if (!need_resched()) {
stop_critical_timings();
if (cpuidle_idle_call())
pm_idle();
start_critical_timings();
/*
* pm_idle functions must always
* return with IRQs enabled.
*/
WARN_ON(irqs_disabled());
} else
local_irq_enable();
}
rcu_idle_exit();
tick_nohz_idle_exit();
idle_notifier_call_chain(IDLE_END);
schedule_preempt_disabled();
#ifdef CONFIG_HOTPLUG_CPU
if (cpu_is_offline(smp_processor_id()))
cpu_die();
#endif
}
}
/*
* The body of the idle task.
*/
void cpu_idle(void)
{
int cpu = smp_processor_id();
for (;;) {
while (!need_resched()) {
if (ppc_md.idle != NULL)
ppc_md.idle();
else
default_idle();
}
if (cpu_is_offline(cpu) && system_state == SYSTEM_RUNNING)
cpu_die();
preempt_enable_no_resched();
schedule();
preempt_disable();
}
}
/*
* The idle thread. We try to conserve power, while trying to keep
* overall latency low. The architecture specific idle is passed
* a value to indicate the level of "idleness" of the system.
*/
void cpu_idle(void)
{
/* endless idle loop with no priority at all */
while (1) {
void (*idle)(void) = pm_idle;
#ifdef CONFIG_HOTPLUG_CPU
if (cpu_is_offline(smp_processor_id()))
cpu_die();
#endif
if (!idle)
idle = default_idle;
tick_nohz_stop_sched_tick(1);
while (!need_resched())
idle();
tick_nohz_restart_sched_tick();
preempt_enable_no_resched();
schedule();
preempt_disable();
}
}
void cpu_idle(void)
{
while (1) {
void (*idle)(void) = pm_idle;
#ifdef CONFIG_HOTPLUG_CPU
if (cpu_is_offline(smp_processor_id()))
cpu_die();
#endif
if (!idle)
idle = default_idle;
tick_nohz_idle_enter();
rcu_idle_enter();
while (!need_resched())
idle();
rcu_idle_exit();
tick_nohz_idle_exit();
schedule_preempt_disabled();
}
}
/*
* The idle thread. There's no useful work to be done, so just try to conserve
* power and have a low exit latency (ie sit in a loop waiting for somebody to
* say that they'd like to reschedule)
*/
ATTRIB_NORET void cpu_idle(void)
{
/* endless idle loop with no priority at all */
while (1) {
while (!need_resched()) {
#ifdef CONFIG_SMTC_IDLE_HOOK_DEBUG
extern void smtc_idle_loop_hook(void);
smtc_idle_loop_hook();
#endif
#ifdef CONFIG_HOTPLUG_CPU
if (cpu_is_offline(smp_processor_id()))
cpu_die();
#endif
if (cpu_wait)
(*cpu_wait)();
}
preempt_enable_no_resched();
schedule();
preempt_disable();
}
}
void arch_cpu_idle_dead(void)
{
cpu_die();
}
int dedicated_idle(void)
{
long oldval;
struct paca_struct *lpaca = get_paca(), *ppaca;
unsigned long start_snooze;
unsigned long *smt_snooze_delay = &__get_cpu_var(smt_snooze_delay);
unsigned int cpu = smp_processor_id();
ppaca = &paca[cpu ^ 1];
while (1) {
/*
* Indicate to the HV that we are idle. Now would be
* a good time to find other work to dispatch.
*/
lpaca->lppaca.xIdle = 1;
oldval = test_and_clear_thread_flag(TIF_NEED_RESCHED);
if (!oldval) {
set_thread_flag(TIF_POLLING_NRFLAG);
start_snooze = __get_tb() +
*smt_snooze_delay * tb_ticks_per_usec;
while (!need_resched() && !cpu_is_offline(cpu)) {
/*
* Go into low thread priority and possibly
* low power mode.
*/
HMT_low();
HMT_very_low();
if (*smt_snooze_delay == 0 ||
__get_tb() < start_snooze)
continue;
HMT_medium();
if (!(ppaca->lppaca.xIdle)) {
local_irq_disable();
/*
* We are about to sleep the thread
* and so wont be polling any
* more.
*/
clear_thread_flag(TIF_POLLING_NRFLAG);
/*
* SMT dynamic mode. Cede will result
* in this thread going dormant, if the
* partner thread is still doing work.
* Thread wakes up if partner goes idle,
* an interrupt is presented, or a prod
* occurs. Returning from the cede
* enables external interrupts.
*/
if (!need_resched())
cede_processor();
else
local_irq_enable();
} else {
/*
* Give the HV an opportunity at the
* processor, since we are not doing
* any work.
*/
poll_pending();
}
}
clear_thread_flag(TIF_POLLING_NRFLAG);
} else {
set_need_resched();
}
HMT_medium();
lpaca->lppaca.xIdle = 0;
schedule();
if (cpu_is_offline(cpu) && system_state == SYSTEM_RUNNING)
cpu_die();
}
return 0;
}