/**
* tick_nohz_idle_exit - restart the idle tick from the idle task
*
* Restart the idle tick when the CPU is woken up from idle
* This also exit the RCU extended quiescent state. The CPU
* can use RCU again after this function is called.
*/
void tick_nohz_idle_exit(void)
{
int cpu = smp_processor_id();
struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
#ifndef CONFIG_VIRT_CPU_ACCOUNTING
unsigned long ticks;
#endif
ktime_t now;
local_irq_disable();
WARN_ON_ONCE(!ts->inidle);
ts->inidle = 0;
if (ts->idle_active || ts->tick_stopped)
now = ktime_get();
if (ts->idle_active)
tick_nohz_stop_idle(cpu, now);
if (!ts->tick_stopped) {
local_irq_enable();
return;
}
/* Update jiffies first */
select_nohz_load_balancer(0);
tick_do_update_jiffies64(now);
#ifdef CONFIG_MTK_SCHED_CMP
update_cpu_load_nohz();
#endif
#ifndef CONFIG_VIRT_CPU_ACCOUNTING
/*
* We stopped the tick in idle. Update process times would miss the
* time we slept as update_process_times does only a 1 tick
* accounting. Enforce that this is accounted to idle !
*/
ticks = jiffies - ts->idle_jiffies;
/*
* We might be one off. Do not randomly account a huge number of ticks!
*/
if (ticks && ticks < LONG_MAX)
account_idle_ticks(ticks);
#endif
calc_load_exit_idle();
touch_softlockup_watchdog();
/*
* Cancel the scheduled timer and restore the tick
*/
ts->tick_stopped = 0;
ts->idle_exittime = now;
tick_nohz_restart(ts, now);
local_irq_enable();
}
static void tick_nohz_kick_tick(int cpu, ktime_t now)
{
#if 0
struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
ktime_t delta;
delta = ktime_sub(hrtimer_get_expires(&ts->sched_timer), now);
if (delta.tv64 <= tick_period.tv64)
return;
tick_nohz_restart(ts, now);
#endif
}
static void tick_nohz_restart_sched_tick(struct tick_sched *ts, ktime_t now)
{
/* Update jiffies first */
tick_do_update_jiffies64(now);
update_cpu_load_nohz();
calc_load_exit_idle();
touch_softlockup_watchdog();
/*
* Cancel the scheduled timer and restore the tick
*/
ts->tick_stopped = 0;
ts->idle_exittime = now;
tick_nohz_restart(ts, now);
}
void tick_nohz_idle_exit(void)
{
int cpu = smp_processor_id();
struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
#ifndef CONFIG_VIRT_CPU_ACCOUNTING
unsigned long ticks;
#endif
ktime_t now;
local_irq_disable();
WARN_ON_ONCE(!ts->inidle);
ts->inidle = 0;
/* Cancel the timer because CPU already waken up from the C-states*/
menu_hrtimer_cancel();
if (ts->idle_active || ts->tick_stopped)
now = ktime_get();
if (ts->idle_active)
tick_nohz_stop_idle(cpu, now);
if (!ts->tick_stopped) {
local_irq_enable();
return;
}
select_nohz_load_balancer(0);
tick_do_update_jiffies64(now);
#ifndef CONFIG_VIRT_CPU_ACCOUNTING
ticks = jiffies - ts->idle_jiffies;
if (ticks && ticks < LONG_MAX)
account_idle_ticks(ticks);
#endif
calc_load_exit_idle();
touch_softlockup_watchdog();
ts->tick_stopped = 0;
ts->idle_exittime = now;
tick_nohz_restart(ts, now);
local_irq_enable();
}
/*
* When NOHZ is enabled and the tick is stopped, we need to kick the
* tick timer from irq_enter() so that the jiffies update is kept
* alive during long running softirqs. That's ugly as hell, but
* correctness is key even if we need to fix the offending softirq in
* the first place.
*
* Note, this is different to tick_nohz_restart. We just kick the
* timer and do not touch the other magic bits which need to be done
* when idle is left.
*/
static void tick_nohz_kick_tick(int cpu, ktime_t now)
{
#if 0
/* Switch back to 2.6.27 behaviour */
struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
ktime_t delta;
/*
* Do not touch the tick device, when the next expiry is either
* already reached or less/equal than the tick period.
*/
delta = ktime_sub(hrtimer_get_expires(&ts->sched_timer), now);
if (delta.tv64 <= tick_period.tv64)
return;
tick_nohz_restart(ts, now);
#endif
}
void tick_nohz_restart_sched_tick(void)
#endif
{
int cpu = smp_processor_id();
struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
#ifndef CONFIG_VIRT_CPU_ACCOUNTING
unsigned long ticks;
#endif
ktime_t now;
local_irq_disable();
if (ts->idle_active || (ts->inidle && ts->tick_stopped))
now = ktime_get();
if (ts->idle_active)
tick_nohz_stop_idle(cpu, now);
if (!ts->inidle || !ts->tick_stopped) {
ts->inidle = 0;
#ifdef CONFIG_DATAPLANE
if (!user_idle)
#endif
local_irq_enable();
return;
}
ts->inidle = 0;
rcu_exit_nohz();
/* Update jiffies first */
select_nohz_load_balancer(0);
tick_do_update_jiffies64(now);
cpumask_clear_cpu(cpu, nohz_cpu_mask);
#ifndef CONFIG_VIRT_CPU_ACCOUNTING
/*
* We stopped the tick in idle. Update process times would miss the
* time we slept as update_process_times does only a 1 tick
* accounting. Enforce that this is accounted to idle !
*/
ticks = jiffies - ts->idle_jiffies;
/*
* We might be one off. Do not randomly account a huge number of ticks!
*/
if (ticks && ticks < LONG_MAX)
#ifdef CONFIG_DATAPLANE
{
if (user_idle) {
cputime_t cpu_time = jiffies_to_cputime(ticks);
account_user_time(current, cpu_time,
cputime_to_scaled(cpu_time));
} else {
account_idle_ticks(ticks);
}
}
#else
account_idle_ticks(ticks);
#endif
#endif
touch_softlockup_watchdog();
/*
* Cancel the scheduled timer and restore the tick
*/
ts->tick_stopped = 0;
ts->idle_exittime = now;
tick_nohz_restart(ts, now);
#ifdef CONFIG_DATAPLANE
if (!user_idle)
#endif
local_irq_enable();
}
