static int tegra114_idle_power_down(struct cpuidle_device *dev,
struct cpuidle_driver *drv,
int index)
{
local_fiq_disable();
tegra_set_cpu_in_lp2();
cpu_pm_enter();
clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_ENTER, &dev->cpu);
call_firmware_op(prepare_idle);
/* Do suspend by ourselves if the firmware does not implement it */
if (call_firmware_op(do_idle) == -ENOSYS)
cpu_suspend(0, tegra30_sleep_cpu_secondary_finish);
clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_EXIT, &dev->cpu);
cpu_pm_exit();
tegra_clear_cpu_in_lp2();
local_fiq_enable();
return index;
}
/*
* bl_enter_powerdown - Programs CPU to enter the specified state
* @dev: cpuidle device
* @drv: The target state to be programmed
* @idx: state index
*
* Called from the CPUidle framework to program the device to the
* specified target state selected by the governor.
*/
static int bl_enter_powerdown(struct cpuidle_device *dev,
struct cpuidle_driver *drv, int idx)
{
struct timespec ts_preidle, ts_postidle, ts_idle;
int ret;
/* Used to keep track of the total time in idle */
getnstimeofday(&ts_preidle);
BUG_ON(!irqs_disabled());
cpu_pm_enter();
clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_ENTER, &dev->cpu);
ret = cpu_suspend((unsigned long) dev, bl_powerdown_finisher);
if (ret)
BUG();
mcpm_cpu_powered_up();
clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_EXIT, &dev->cpu);
cpu_pm_exit();
getnstimeofday(&ts_postidle);
local_irq_enable();
ts_idle = timespec_sub(ts_postidle, ts_preidle);
dev->last_residency = ts_idle.tv_nsec / NSEC_PER_USEC +
ts_idle.tv_sec * USEC_PER_SEC;
return idx;
}
void __devinit vmi_time_bsp_init(void)
{
/*
* On APIC systems, we want local timers to fire on each cpu. We do
* this by programming LVTT to deliver timer events to the IRQ handler
* for IRQ-0, since we can't re-use the APIC local timer handler
* without interfering with that code.
*/
clockevents_notify(CLOCK_EVT_NOTIFY_SUSPEND, NULL);
local_irq_disable();
#ifdef CONFIG_SMP
/*
* XXX handle_percpu_irq only defined for SMP; we need to switch over
* to using it, since this is a local interrupt, which each CPU must
* handle individually without locking out or dropping simultaneous
* local timers on other CPUs. We also don't want to trigger the
* quirk workaround code for interrupts which gets invoked from
* handle_percpu_irq via eoi, so we use our own IRQ chip.
*/
set_irq_chip_and_handler_name(0, &vmi_chip, handle_percpu_irq, "lvtt");
#else
set_irq_chip_and_handler_name(0, &vmi_chip, handle_edge_irq, "lvtt");
#endif
vmi_wiring = VMI_ALARM_WIRED_LVTT;
apic_write(APIC_LVTT, vmi_get_timer_vector());
local_irq_enable();
clockevents_notify(CLOCK_EVT_NOTIFY_RESUME, NULL);
}
static void tegra2_idle_lp2_cpu_1(struct cpuidle_device *dev,
struct cpuidle_state *state, s64 request)
{
#ifdef CONFIG_SMP
struct tegra_twd_context twd_context;
if (request < tegra_lp2_exit_latency) {
tegra2_cpu_clear_resettable();
tegra2_lp3_fall_back(dev);
return;
}
/* Save time this CPU must be awakened by. */
tegra_cpu1_wake_by_time = ktime_to_us(ktime_get()) + request;
smp_wmb();
clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_ENTER, &dev->cpu);
tegra_twd_suspend(&twd_context);
tegra2_sleep_wfi(PLAT_PHYS_OFFSET - PAGE_OFFSET);
tegra2_cpu_clear_resettable();
tegra_cpu1_wake_by_time = LLONG_MAX;
tegra_twd_resume(&twd_context);
clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_EXIT, &dev->cpu);
#endif
}
static int __cpuinit hrtimer_cpu_notify(struct notifier_block *self,
unsigned long action, void *hcpu)
{
int scpu = (long)hcpu;
switch (action) {
case CPU_UP_PREPARE:
case CPU_UP_PREPARE_FROZEN:
init_hrtimers_cpu(scpu);
break;
#ifdef CONFIG_HOTPLUG_CPU
case CPU_DYING:
case CPU_DYING_FROZEN:
clockevents_notify(CLOCK_EVT_NOTIFY_CPU_DYING, &scpu);
break;
case CPU_DEAD:
case CPU_DEAD_FROZEN:
{
clockevents_notify(CLOCK_EVT_NOTIFY_CPU_DEAD, &scpu);
migrate_hrtimers(scpu);
break;
}
#endif
default:
break;
}
return NOTIFY_OK;
}
static int sunxi_cpu_power_down_c2state(struct cpuidle_device *dev, \
struct cpuidle_driver *drv, \
int index)
{
unsigned int mpidr = read_cpuid_mpidr();
unsigned int cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1);
unsigned int cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0);
cpu_pm_enter();
//cpu_cluster_pm_enter();
clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_ENTER, &dev->cpu);
smp_wmb();
cpu_suspend(CPUIDLE_FLAG_C2_STATE, sunxi_powerdown_c2_finisher);
/*
* Since this is called with IRQs enabled, and no arch_spin_lock_irq
* variant exists, we need to disable IRQs manually here.
*/
local_irq_disable();
arch_spin_lock(&sun8i_mcpm_lock);
sun8i_cpu_use_count[cluster][cpu]++;
sun8i_cluster_use_count[cluster]++;
arch_spin_unlock(&sun8i_mcpm_lock);
local_irq_enable();
clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_EXIT, &dev->cpu);
//cpu_cluster_pm_exit();
cpu_pm_exit();
return index;
}
void arch_idle_multi_core(void)
{
u32 reg;
int cpu = smp_processor_id();
#ifdef CONFIG_LOCAL_TIMERS
if (!tick_broadcast_oneshot_active()
|| !tick_oneshot_mode_active())
return;
clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_ENTER, &cpu);
#endif
/* iMX6Q and iMX6DL */
if ((cpu_is_mx6q() && chip_rev >= IMX_CHIP_REVISION_1_2) ||
(cpu_is_mx6dl() && chip_rev >= IMX_CHIP_REVISION_1_1)) {
/*
* This code should only be executed on MX6QTO1.2 or later
* and MX6DL TO1.1 or later.
* These chips have the HW fix for the WAIT mode issue.
* Ensure that the CGPR bit 17 is set to enable the fix.
*/
reg = __raw_readl(MXC_CCM_CGPR);
reg |= MXC_CCM_CGPR_WAIT_MODE_FIX;
__raw_writel(reg, MXC_CCM_CGPR);
ca9_do_idle();
} else
arch_idle_with_workaround(cpu);
#ifdef CONFIG_LOCAL_TIMERS
clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_EXIT, &cpu);
#endif
}
/**
* cpuidle_idle_call - the main idle loop
*
* NOTE: no locks or semaphores should be used here
* return non-zero on failure
*/
int cpuidle_idle_call(void)
{
struct cpuidle_device *dev = __this_cpu_read(cpuidle_devices);
struct cpuidle_driver *drv;
int next_state, entered_state;
if (off)
return -ENODEV;
if (!initialized)
return -ENODEV;
/* check if the device is ready */
if (!dev || !dev->enabled)
return -EBUSY;
drv = cpuidle_get_cpu_driver(dev);
/* ask the governor for the next state */
next_state = cpuidle_curr_governor->select(drv, dev);
if (next_state < 0)
return -EBUSY;
if (need_resched()) {
dev->last_residency = 0;
/* give the governor an opportunity to reflect on the outcome */
if (cpuidle_curr_governor->reflect)
cpuidle_curr_governor->reflect(dev, next_state);
local_irq_enable();
return 0;
}
trace_cpu_idle_rcuidle(next_state, dev->cpu);
if (drv->states[next_state].flags & CPUIDLE_FLAG_TIMER_STOP)
clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_ENTER,
&dev->cpu);
if (cpuidle_state_is_coupled(dev, drv, next_state))
entered_state = cpuidle_enter_state_coupled(dev, drv,
next_state);
else
entered_state = cpuidle_enter_state(dev, drv, next_state);
if (drv->states[next_state].flags & CPUIDLE_FLAG_TIMER_STOP)
clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_EXIT,
&dev->cpu);
trace_cpu_idle_rcuidle(PWR_EVENT_EXIT, dev->cpu);
/* give the governor an opportunity to reflect on the outcome */
if (cpuidle_curr_governor->reflect)
cpuidle_curr_governor->reflect(dev, entered_state);
return 0;
}
static int sunxi_cpu_core_power_down(struct cpuidle_device *dev,
struct cpuidle_driver *drv, int index)
{
cpu_pm_enter();
clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_ENTER, &dev->cpu);
smp_wmb();
cpu_suspend(0, sunxi_powerdown_finisher);
clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_EXIT, &dev->cpu);
cpu_pm_exit();
return index;
}
static bool tegra20_idle_enter_lp2_cpu_1(struct cpuidle_device *dev,
struct cpuidle_driver *drv,
int index)
{
clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_ENTER, &dev->cpu);
cpu_suspend(0, tegra20_sleep_cpu_secondary_finish);
tegra20_cpu_clear_resettable();
clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_EXIT, &dev->cpu);
return true;
}
static bool tegra30_cpu_core_power_down(struct cpuidle_device *dev,
struct cpuidle_driver *drv,
int index)
{
clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_ENTER, &dev->cpu);
smp_wmb();
cpu_suspend(0, tegra30_sleep_cpu_secondary_finish);
clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_EXIT, &dev->cpu);
return true;
}
void arch_idle(void)
{
int cpu = smp_processor_id();
if (enable_wait_mode) {
#ifdef CONFIG_LOCAL_TIMERS
if (!tick_broadcast_oneshot_active()
|| !tick_oneshot_mode_active())
return;
clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_ENTER, &cpu);
#endif
if (enet_is_active)
/* Don't allow the chip to enter WAIT mode if enet is active
* and the GPIO workaround for ENET interrupts is not used,
* since all ENET interrupts donot wake up the SOC.
*/
mxc_cpu_lp_set(WAIT_CLOCKED);
else
mxc_cpu_lp_set(WAIT_UNCLOCKED_POWER_OFF);
if (mem_clk_on_in_wait) {
u32 reg;
/*
* MX6SL, MX6Q (TO1.2 or later) and
* MX6DL (TO1.1 or later) have a bit in
* CCM_CGPR that when cleared keeps the
* clocks to memories ON when ARM is in WFI.
* This mode can be used when IPG clock is
* very low (12MHz) and the ARM:IPG ratio
* perhaps cannot be maintained.
*/
reg = __raw_readl(MXC_CCM_CGPR);
reg &= ~MXC_CCM_CGPR_MEM_IPG_STOP_MASK;
__raw_writel(reg, MXC_CCM_CGPR);
ca9_do_idle();
} else if (num_possible_cpus() == 1)
/* iMX6SL or iMX6DLS */
arch_idle_single_core();
else
arch_idle_multi_core(cpu);
#ifdef CONFIG_LOCAL_TIMERS
clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_EXIT, &cpu);
#endif
} else {
mxc_cpu_lp_set(WAIT_CLOCKED);
ca9_do_idle();
}
}
static void acpi_propagate_timer_broadcast(struct acpi_processor *pr)
{
unsigned long reason;
reason = pr->power.timer_broadcast_on_state < INT_MAX ?
CLOCK_EVT_NOTIFY_BROADCAST_ON : CLOCK_EVT_NOTIFY_BROADCAST_OFF;
clockevents_notify(reason, &pr->id);
}
static void xen_vcpu_notify_restore(void *data)
{
unsigned long reason = (unsigned long)data;
/* Boot processor notified via generic timekeeping_resume() */
if ( smp_processor_id() == 0)
return;
clockevents_notify(reason, NULL);
}
static bool tegra30_cpu_cluster_power_down(struct cpuidle_device *dev,
struct cpuidle_driver *drv,
int index)
{
/* All CPUs entering LP2 is not working.
* Don't let CPU0 enter LP2 when any secondary CPU is online.
*/
if (num_online_cpus() > 1 || !tegra_cpu_rail_off_ready()) {
cpu_do_idle();
return false;
}
clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_ENTER, &dev->cpu);
tegra_idle_lp2_last();
clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_EXIT, &dev->cpu);
return true;
}
/* Actual code that puts the SoC in different idle states */
static int xilinx_enter_idle(struct cpuidle_device *dev,
struct cpuidle_driver *drv, int index)
{
struct timeval before, after;
int idle_time;
local_irq_disable();
do_gettimeofday(&before);
if (index == 0)
/* Wait for interrupt state */
cpu_do_idle();
else if (index == 1) {
unsigned int cpu_id = smp_processor_id();
clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_ENTER, &cpu_id);
/* Devices must be stopped here */
cpu_pm_enter();
/* Add code for DDR self refresh start */
cpu_do_idle();
/*cpu_suspend(foo, bar);*/
/* Add code for DDR self refresh stop */
cpu_pm_exit();
clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_EXIT, &cpu_id);
}
do_gettimeofday(&after);
local_irq_enable();
idle_time = (after.tv_sec - before.tv_sec) * USEC_PER_SEC +
(after.tv_usec - before.tv_usec);
dev->last_residency = idle_time;
return index;
}
static bool tegra20_cpu_cluster_power_down(struct cpuidle_device *dev,
struct cpuidle_driver *drv,
int index)
{
while (tegra20_cpu_is_resettable_soon())
cpu_relax();
if (tegra20_reset_cpu_1() || !tegra_cpu_rail_off_ready())
return false;
clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_ENTER, &dev->cpu);
tegra_idle_lp2_last();
clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_EXIT, &dev->cpu);
if (cpu_online(1))
tegra20_wake_cpu1_from_reset();
return true;
}
/*
* AMD C1E enabled CPUs have a real nasty problem: Some BIOSes set the
* C1E flag only in the secondary CPU, so when we detect the wreckage
* we already have enabled the boot CPU local apic timer. Check, if
* disable_apic_timer is set and the DUMMY flag is cleared. If yes,
* set the DUMMY flag again and force the broadcast mode in the
* clockevents layer.
*/
void __cpuinit check_boot_apic_timer_broadcast(void)
{
if (!disable_apic_timer ||
(lapic_clockevent.features & CLOCK_EVT_FEAT_DUMMY))
return;
printk(KERN_INFO "AMD C1E detected late. Force timer broadcast.\n");
lapic_clockevent.features |= CLOCK_EVT_FEAT_DUMMY;
local_irq_enable();
clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_FORCE, &boot_cpu_id);
local_irq_disable();
}
static int tegra114_idle_power_down(struct cpuidle_device *dev,
struct cpuidle_driver *drv,
int index)
{
local_fiq_disable();
tegra_set_cpu_in_lp2();
cpu_pm_enter();
clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_ENTER, &dev->cpu);
cpu_suspend(0, tegra30_sleep_cpu_secondary_finish);
clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_EXIT, &dev->cpu);
cpu_pm_exit();
tegra_clear_cpu_in_lp2();
local_fiq_enable();
return index;
}
/* Power(C) State timer broadcast control */
static void lapic_timer_state_broadcast(struct acpi_processor *pr,
struct acpi_processor_cx *cx,
int broadcast)
{
int state = cx - pr->power.states;
if (state >= pr->power.timer_broadcast_on_state) {
unsigned long reason;
reason = broadcast ? CLOCK_EVT_NOTIFY_BROADCAST_ENTER :
CLOCK_EVT_NOTIFY_BROADCAST_EXIT;
clockevents_notify(reason, &pr->id);
}
}
static void power_saving_mwait_init(void)
{
unsigned int eax, ebx, ecx, edx;
unsigned int highest_cstate = 0;
unsigned int highest_subcstate = 0;
int i;
if (!boot_cpu_has(X86_FEATURE_MWAIT))
return;
if (boot_cpu_data.cpuid_level < CPUID_MWAIT_LEAF)
return;
cpuid(CPUID_MWAIT_LEAF, &eax, &ebx, &ecx, &edx);
if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED) ||
!(ecx & CPUID5_ECX_INTERRUPT_BREAK))
return;
edx >>= MWAIT_SUBSTATE_SIZE;
for (i = 0; i < 7 && edx; i++, edx >>= MWAIT_SUBSTATE_SIZE) {
if (edx & MWAIT_SUBSTATE_MASK) {
highest_cstate = i;
highest_subcstate = edx & MWAIT_SUBSTATE_MASK;
}
}
power_saving_mwait_eax = (highest_cstate << MWAIT_SUBSTATE_SIZE) |
(highest_subcstate - 1);
for_each_online_cpu(i)
clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_ON, &i);
#if defined(CONFIG_GENERIC_TIME) && defined(CONFIG_X86)
switch (boot_cpu_data.x86_vendor) {
case X86_VENDOR_AMD:
case X86_VENDOR_INTEL:
/*
* AMD Fam10h TSC will tick in all
* C/P/S0/S1 states when this bit is set.
*/
if (boot_cpu_has(X86_FEATURE_NONSTOP_TSC))
return;
/*FALL THROUGH*/
default:
/* TSC could halt in idle, so notify users */
mark_tsc_unstable("TSC halts in idle");
}
#endif
}
static bool tegra30_cpu_cluster_power_down(struct cpuidle_device *dev,
struct cpuidle_driver *drv,
int index)
{
struct cpuidle_state *state = &drv->states[index];
u32 cpu_on_time = state->exit_latency;
u32 cpu_off_time = state->target_residency - state->exit_latency;
/* All CPUs entering LP2 is not working.
* Don't let CPU0 enter LP2 when any secondary CPU is online.
*/
if (num_online_cpus() > 1 || !tegra_cpu_rail_off_ready()) {
cpu_do_idle();
return false;
}
clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_ENTER, &dev->cpu);
tegra_idle_lp2_last(cpu_on_time, cpu_off_time);
clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_EXIT, &dev->cpu);
return true;
}
void sc8825_idle(void)
{
int this_cpu = smp_processor_id();
int val;
if (!need_resched()) {
hw_local_irq_disable();
if (!arch_local_irq_pending()) {
val = os_ctx->idle(os_ctx);
if (0 == val) {
#ifdef CONFIG_CACHE_L2X0
/*l2cache power control, standby mode enable*/
/*L2X0_POWER_CTRL
__raw_writel(1, SPRD_L2_BASE+0xF80);
l2x0_suspend();
*/
#endif
#if defined(CONFIG_LOCAL_TIMERS)
clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_ENTER, &this_cpu);
#endif
cpu_do_idle();
#if defined(CONFIG_LOCAL_TIMERS)
clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_EXIT, &this_cpu);
#endif
#ifdef CONFIG_CACHE_L2X0
/*
l2x0_resume(1);
*/
#endif
}
}
hw_local_irq_enable();
}
local_irq_enable();
return;
}
/* avoid HT sibilings if possible */
if (cpumask_empty(tmp))
cpumask_andnot(tmp, cpu_online_mask, pad_busy_cpus);
if (cpumask_empty(tmp)) {
mutex_unlock(&isolated_cpus_lock);
return;
}
for_each_cpu(cpu, tmp) {
if (cpu_weight[cpu] < min_weight) {
min_weight = cpu_weight[cpu];
preferred_cpu = cpu;
}
}
if (tsk_in_cpu[tsk_index] != -1)
cpumask_clear_cpu(tsk_in_cpu[tsk_index], pad_busy_cpus);
tsk_in_cpu[tsk_index] = preferred_cpu;
cpumask_set_cpu(preferred_cpu, pad_busy_cpus);
cpu_weight[preferred_cpu]++;
mutex_unlock(&isolated_cpus_lock);
set_cpus_allowed_ptr(current, cpumask_of(preferred_cpu));
}
static void exit_round_robin(unsigned int tsk_index)
{
struct cpumask *pad_busy_cpus = to_cpumask(pad_busy_cpus_bits);
cpumask_clear_cpu(tsk_in_cpu[tsk_index], pad_busy_cpus);
tsk_in_cpu[tsk_index] = -1;
}
static unsigned int idle_pct = 5; /* percentage */
static unsigned int round_robin_time = 10; /* second */
static int power_saving_thread(void *data)
{
struct sched_param param = {.sched_priority = 1};
int do_sleep;
unsigned int tsk_index = (unsigned long)data;
u64 last_jiffies = 0;
sched_setscheduler(current, SCHED_RR, ¶m);
while (!kthread_should_stop()) {
int cpu;
u64 expire_time;
try_to_freeze();
/* round robin to cpus */
if (last_jiffies + round_robin_time * HZ < jiffies) {
last_jiffies = jiffies;
round_robin_cpu(tsk_index);
}
do_sleep = 0;
current_thread_info()->status &= ~TS_POLLING;
/*
* TS_POLLING-cleared state must be visible before we test
* NEED_RESCHED:
*/
smp_mb();
expire_time = jiffies + HZ * (100 - idle_pct) / 100;
while (!need_resched()) {
local_irq_disable();
cpu = smp_processor_id();
clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_ENTER,
&cpu);
stop_critical_timings();
__monitor((void *)¤t_thread_info()->flags, 0, 0);
smp_mb();
if (!need_resched())
__mwait(power_saving_mwait_eax, 1);
start_critical_timings();
clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_EXIT,
&cpu);
local_irq_enable();
if (jiffies > expire_time) {
do_sleep = 1;
break;
}
}
current_thread_info()->status |= TS_POLLING;
/*
* current sched_rt has threshold for rt task running time.
* When a rt task uses 95% CPU time, the rt thread will be
* scheduled out for 5% CPU time to not starve other tasks. But
* the mechanism only works when all CPUs have RT task running,
* as if one CPU hasn't RT task, RT task from other CPUs will
* borrow CPU time from this CPU and cause RT task use > 95%
* CPU time. To make 'avoid starvation' work, takes a nap here.
*/
if (do_sleep)
schedule_timeout_killable(HZ * idle_pct / 100);
}
exit_round_robin(tsk_index);
return 0;
}
static struct task_struct *ps_tsks[NR_CPUS];
static unsigned int ps_tsk_num;
static int create_power_saving_task(void)
{
int rc = -ENOMEM;
ps_tsks[ps_tsk_num] = kthread_run(power_saving_thread,
(void *)(unsigned long)ps_tsk_num,
"power_saving/%d", ps_tsk_num);
rc = IS_ERR(ps_tsks[ps_tsk_num]) ? PTR_ERR(ps_tsks[ps_tsk_num]) : 0;
if (!rc)
ps_tsk_num++;
else
ps_tsks[ps_tsk_num] = NULL;
return rc;
}
/* avoid HT sibilings if possible */
if (cpumask_empty(tmp))
cpumask_andnot(tmp, cpu_online_mask, pad_busy_cpus);
if (cpumask_empty(tmp)) {
mutex_unlock(&round_robin_lock);
return;
}
for_each_cpu(cpu, tmp) {
if (cpu_weight[cpu] < min_weight) {
min_weight = cpu_weight[cpu];
preferred_cpu = cpu;
}
}
if (tsk_in_cpu[tsk_index] != -1)
cpumask_clear_cpu(tsk_in_cpu[tsk_index], pad_busy_cpus);
tsk_in_cpu[tsk_index] = preferred_cpu;
cpumask_set_cpu(preferred_cpu, pad_busy_cpus);
cpu_weight[preferred_cpu]++;
mutex_unlock(&round_robin_lock);
set_cpus_allowed_ptr(current, cpumask_of(preferred_cpu));
}
static void exit_round_robin(unsigned int tsk_index)
{
struct cpumask *pad_busy_cpus = to_cpumask(pad_busy_cpus_bits);
cpumask_clear_cpu(tsk_in_cpu[tsk_index], pad_busy_cpus);
tsk_in_cpu[tsk_index] = -1;
}
static unsigned int idle_pct = 5; /* percentage */
static unsigned int round_robin_time = 1; /* second */
static int power_saving_thread(void *data)
{
struct sched_param param = {.sched_priority = 1};
int do_sleep;
unsigned int tsk_index = (unsigned long)data;
u64 last_jiffies = 0;
sched_setscheduler(current, SCHED_RR, ¶m);
set_freezable();
while (!kthread_should_stop()) {
int cpu;
u64 expire_time;
try_to_freeze();
/* round robin to cpus */
if (last_jiffies + round_robin_time * HZ < jiffies) {
last_jiffies = jiffies;
round_robin_cpu(tsk_index);
}
do_sleep = 0;
expire_time = jiffies + HZ * (100 - idle_pct) / 100;
while (!need_resched()) {
if (tsc_detected_unstable && !tsc_marked_unstable) {
/* TSC could halt in idle, so notify users */
mark_tsc_unstable("TSC halts in idle");
tsc_marked_unstable = 1;
}
if (lapic_detected_unstable && !lapic_marked_unstable) {
int i;
/* LAPIC could halt in idle, so notify users */
for_each_online_cpu(i)
clockevents_notify(
CLOCK_EVT_NOTIFY_BROADCAST_ON,
&i);
lapic_marked_unstable = 1;
}
local_irq_disable();
cpu = smp_processor_id();
if (lapic_marked_unstable)
clockevents_notify(
CLOCK_EVT_NOTIFY_BROADCAST_ENTER, &cpu);
stop_critical_timings();
__monitor((void *)¤t_thread_info()->flags, 0, 0);
smp_mb();
if (!need_resched())
__mwait(power_saving_mwait_eax, 1);
start_critical_timings();
if (lapic_marked_unstable)
clockevents_notify(
CLOCK_EVT_NOTIFY_BROADCAST_EXIT, &cpu);
local_irq_enable();
if (jiffies > expire_time) {
do_sleep = 1;
break;
}
}
/*
* current sched_rt has threshold for rt task running time.
* When a rt task uses 95% CPU time, the rt thread will be
* scheduled out for 5% CPU time to not starve other tasks. But
* the mechanism only works when all CPUs have RT task running,
* as if one CPU hasn't RT task, RT task from other CPUs will
* borrow CPU time from this CPU and cause RT task use > 95%
* CPU time. To make 'avoid starvation' work, takes a nap here.
*/
if (do_sleep)
schedule_timeout_killable(HZ * idle_pct / 100);
}
exit_round_robin(tsk_index);
return 0;
}
static struct task_struct *ps_tsks[NR_CPUS];
static unsigned int ps_tsk_num;
static int create_power_saving_task(void)
{
int rc = -ENOMEM;
ps_tsks[ps_tsk_num] = kthread_run(power_saving_thread,
(void *)(unsigned long)ps_tsk_num,
"acpi_pad/%d", ps_tsk_num);
rc = PTR_RET(ps_tsks[ps_tsk_num]);
if (!rc)
ps_tsk_num++;
else
ps_tsks[ps_tsk_num] = NULL;
return rc;
}
static void tegra_cpuidle_setup_bctimer(void *arg)
{
int cpu = smp_processor_id();
clockevents_notify((long)(arg), &cpu);
}
/**
* cpuidle_idle_call - the main idle function
*
* NOTE: no locks or semaphores should be used here
*/
static void cpuidle_idle_call(void)
{
struct cpuidle_device *dev = __this_cpu_read(cpuidle_devices);
struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev);
int next_state, entered_state;
bool broadcast;
/*
* Check if the idle task must be rescheduled. If it is the
* case, exit the function after re-enabling the local irq.
*/
if (need_resched()) {
local_irq_enable();
return;
}
/*
* During the idle period, stop measuring the disabled irqs
* critical sections latencies
*/
stop_critical_timings();
/*
* Tell the RCU framework we are entering an idle section,
* so no more rcu read side critical sections and one more
* step to the grace period
*/
rcu_idle_enter();
/*
* Ask the cpuidle framework to choose a convenient idle state.
* Fall back to the default arch idle method on errors.
*/
next_state = cpuidle_select(drv, dev);
if (next_state < 0) {
use_default:
/*
* We can't use the cpuidle framework, let's use the default
* idle routine.
*/
if (current_clr_polling_and_test())
local_irq_enable();
else
arch_cpu_idle();
goto exit_idle;
}
/*
* The idle task must be scheduled, it is pointless to
* go to idle, just update no idle residency and get
* out of this function
*/
if (current_clr_polling_and_test()) {
dev->last_residency = 0;
entered_state = next_state;
local_irq_enable();
goto exit_idle;
}
broadcast = !!(drv->states[next_state].flags & CPUIDLE_FLAG_TIMER_STOP);
/*
* Tell the time framework to switch to a broadcast timer
* because our local timer will be shutdown. If a local timer
* is used from another cpu as a broadcast timer, this call may
* fail if it is not available
*/
if (broadcast &&
clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_ENTER, &dev->cpu))
goto use_default;
trace_cpu_idle_rcuidle(next_state, dev->cpu);
/*
* Enter the idle state previously returned by the governor decision.
* This function will block until an interrupt occurs and will take
* care of re-enabling the local interrupts
*/
entered_state = cpuidle_enter(drv, dev, next_state);
trace_cpu_idle_rcuidle(PWR_EVENT_EXIT, dev->cpu);
if (broadcast)
clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_EXIT, &dev->cpu);
/*
* Give the governor an opportunity to reflect on the outcome
*/
cpuidle_reflect(dev, entered_state);
exit_idle:
__current_set_polling();
/*
* It is up to the idle functions to reenable local interrupts
*/
if (WARN_ON_ONCE(irqs_disabled()))
local_irq_enable();
rcu_idle_exit();
start_critical_timings();
}
/**
* create_image - Create a hibernation image.
* @platform_mode: Whether or not to use the platform driver.
*
* Execute device drivers' "late" and "noirq" freeze callbacks, create a
* hibernation image and run the drivers' "noirq" and "early" thaw callbacks.
*
* Control reappears in this routine after the subsequent restore.
*/
static int create_image(int platform_mode)
{
int error;
error = dpm_suspend_end(PMSG_FREEZE);
if (error) {
printk(KERN_ERR "PM: Some devices failed to power down, "
"aborting hibernation\n");
return error;
}
error = platform_pre_snapshot(platform_mode);
if (error || hibernation_test(TEST_PLATFORM))
goto Platform_finish;
error = disable_nonboot_cpus();
if (error || hibernation_test(TEST_CPUS))
goto Enable_cpus;
clockevents_notify(CLOCK_EVT_NOTIFY_SUSPEND, NULL);
local_irq_disable();
error = syscore_suspend();
if (error) {
printk(KERN_ERR "PM: Some system devices failed to power down, "
"aborting hibernation\n");
goto Enable_irqs;
}
if (hibernation_test(TEST_CORE) || pm_wakeup_pending())
goto Power_up;
in_suspend = 1;
save_processor_state();
trace_suspend_resume(TPS("machine_suspend"), PM_EVENT_HIBERNATE, true);
error = swsusp_arch_suspend();
trace_suspend_resume(TPS("machine_suspend"), PM_EVENT_HIBERNATE, false);
if (error)
printk(KERN_ERR "PM: Error %d creating hibernation image\n",
error);
/* Restore control flow magically appears here */
restore_processor_state();
if (!in_suspend)
events_check_enabled = false;
platform_leave(platform_mode);
Power_up:
syscore_resume();
Enable_irqs:
clockevents_notify(CLOCK_EVT_NOTIFY_RESUME, NULL);
local_irq_enable();
Enable_cpus:
enable_nonboot_cpus();
Platform_finish:
platform_finish(platform_mode);
dpm_resume_start(in_suspend ?
(error ? PMSG_RECOVER : PMSG_THAW) : PMSG_RESTORE);
return error;
}
/**
* resume_target_kernel - Restore system state from a hibernation image.
* @platform_mode: Whether or not to use the platform driver.
*
* Execute device drivers' "noirq" and "late" freeze callbacks, restore the
* contents of highmem that have not been restored yet from the image and run
* the low-level code that will restore the remaining contents of memory and
* switch to the just restored target kernel.
*/
static int resume_target_kernel(bool platform_mode)
{
int error;
error = dpm_suspend_end(PMSG_QUIESCE);
if (error) {
printk(KERN_ERR "PM: Some devices failed to power down, "
"aborting resume\n");
return error;
}
error = platform_pre_restore(platform_mode);
if (error)
goto Cleanup;
error = disable_nonboot_cpus();
if (error)
goto Enable_cpus;
clockevents_notify(CLOCK_EVT_NOTIFY_SUSPEND, NULL);
local_irq_disable();
error = syscore_suspend();
if (error)
goto Enable_irqs;
save_processor_state();
error = restore_highmem();
if (!error) {
error = swsusp_arch_resume();
/*
* The code below is only ever reached in case of a failure.
* Otherwise, execution continues at the place where
* swsusp_arch_suspend() was called.
*/
BUG_ON(!error);
/*
* This call to restore_highmem() reverts the changes made by
* the previous one.
*/
restore_highmem();
}
/*
* The only reason why swsusp_arch_resume() can fail is memory being
* very tight, so we have to free it as soon as we can to avoid
* subsequent failures.
*/
swsusp_free();
restore_processor_state();
touch_softlockup_watchdog();
syscore_resume();
Enable_irqs:
clockevents_notify(CLOCK_EVT_NOTIFY_RESUME, NULL);
local_irq_enable();
Enable_cpus:
enable_nonboot_cpus();
Cleanup:
platform_restore_cleanup(platform_mode);
dpm_resume_start(PMSG_RECOVER);
return error;
}
/**
* hibernation_platform_enter - Power off the system using the platform driver.
*/
int hibernation_platform_enter(void)
{
int error;
if (!hibernation_ops)
return -ENOSYS;
/*
* We have cancelled the power transition by running
* hibernation_ops->finish() before saving the image, so we should let
* the firmware know that we're going to enter the sleep state after all
*/
error = hibernation_ops->begin();
if (error)
goto Close;
entering_platform_hibernation = true;
suspend_console();
error = dpm_suspend_start(PMSG_HIBERNATE);
if (error) {
if (hibernation_ops->recover)
hibernation_ops->recover();
goto Resume_devices;
}
error = dpm_suspend_end(PMSG_HIBERNATE);
if (error)
goto Resume_devices;
error = hibernation_ops->prepare();
if (error)
goto Platform_finish;
error = disable_nonboot_cpus();
if (error)
goto Platform_finish;
clockevents_notify(CLOCK_EVT_NOTIFY_SUSPEND, NULL);
local_irq_disable();
syscore_suspend();
if (pm_wakeup_pending()) {
error = -EAGAIN;
goto Power_up;
}
hibernation_ops->enter();
/* We should never get here */
while (1);
Power_up:
syscore_resume();
clockevents_notify(CLOCK_EVT_NOTIFY_RESUME, NULL);
local_irq_enable();
enable_nonboot_cpus();
Platform_finish:
hibernation_ops->finish();
dpm_resume_start(PMSG_RESTORE);
Resume_devices:
entering_platform_hibernation = false;
dpm_resume_end(PMSG_RESTORE);
resume_console();
Close:
hibernation_ops->end();
return error;
}