/** * tick_nohz_idle_exit - restart the idle tick from the idle task * * Restart the idle tick when the CPU is woken up from idle * This also exit the RCU extended quiescent state. The CPU * can use RCU again after this function is called. */ void tick_nohz_idle_exit(void) { int cpu = smp_processor_id(); struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu); #ifndef CONFIG_VIRT_CPU_ACCOUNTING unsigned long ticks; #endif ktime_t now; local_irq_disable(); WARN_ON_ONCE(!ts->inidle); ts->inidle = 0; if (ts->idle_active || ts->tick_stopped) now = ktime_get(); if (ts->idle_active) tick_nohz_stop_idle(cpu, now); if (!ts->tick_stopped) { local_irq_enable(); return; } /* Update jiffies first */ select_nohz_load_balancer(0); tick_do_update_jiffies64(now); #ifdef CONFIG_MTK_SCHED_CMP update_cpu_load_nohz(); #endif #ifndef CONFIG_VIRT_CPU_ACCOUNTING /* * We stopped the tick in idle. Update process times would miss the * time we slept as update_process_times does only a 1 tick * accounting. Enforce that this is accounted to idle ! */ ticks = jiffies - ts->idle_jiffies; /* * We might be one off. Do not randomly account a huge number of ticks! */ if (ticks && ticks < LONG_MAX) account_idle_ticks(ticks); #endif calc_load_exit_idle(); touch_softlockup_watchdog(); /* * Cancel the scheduled timer and restore the tick */ ts->tick_stopped = 0; ts->idle_exittime = now; tick_nohz_restart(ts, now); local_irq_enable(); }
static void tick_nohz_kick_tick(int cpu, ktime_t now) { #if 0 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu); ktime_t delta; delta = ktime_sub(hrtimer_get_expires(&ts->sched_timer), now); if (delta.tv64 <= tick_period.tv64) return; tick_nohz_restart(ts, now); #endif }
static void tick_nohz_restart_sched_tick(struct tick_sched *ts, ktime_t now) { /* Update jiffies first */ tick_do_update_jiffies64(now); update_cpu_load_nohz(); calc_load_exit_idle(); touch_softlockup_watchdog(); /* * Cancel the scheduled timer and restore the tick */ ts->tick_stopped = 0; ts->idle_exittime = now; tick_nohz_restart(ts, now); }
void tick_nohz_idle_exit(void) { int cpu = smp_processor_id(); struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu); #ifndef CONFIG_VIRT_CPU_ACCOUNTING unsigned long ticks; #endif ktime_t now; local_irq_disable(); WARN_ON_ONCE(!ts->inidle); ts->inidle = 0; /* Cancel the timer because CPU already waken up from the C-states*/ menu_hrtimer_cancel(); if (ts->idle_active || ts->tick_stopped) now = ktime_get(); if (ts->idle_active) tick_nohz_stop_idle(cpu, now); if (!ts->tick_stopped) { local_irq_enable(); return; } select_nohz_load_balancer(0); tick_do_update_jiffies64(now); #ifndef CONFIG_VIRT_CPU_ACCOUNTING ticks = jiffies - ts->idle_jiffies; if (ticks && ticks < LONG_MAX) account_idle_ticks(ticks); #endif calc_load_exit_idle(); touch_softlockup_watchdog(); ts->tick_stopped = 0; ts->idle_exittime = now; tick_nohz_restart(ts, now); local_irq_enable(); }
/* * When NOHZ is enabled and the tick is stopped, we need to kick the * tick timer from irq_enter() so that the jiffies update is kept * alive during long running softirqs. That's ugly as hell, but * correctness is key even if we need to fix the offending softirq in * the first place. * * Note, this is different to tick_nohz_restart. We just kick the * timer and do not touch the other magic bits which need to be done * when idle is left. */ static void tick_nohz_kick_tick(int cpu, ktime_t now) { #if 0 /* Switch back to 2.6.27 behaviour */ struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu); ktime_t delta; /* * Do not touch the tick device, when the next expiry is either * already reached or less/equal than the tick period. */ delta = ktime_sub(hrtimer_get_expires(&ts->sched_timer), now); if (delta.tv64 <= tick_period.tv64) return; tick_nohz_restart(ts, now); #endif }
void tick_nohz_restart_sched_tick(void) #endif { int cpu = smp_processor_id(); struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu); #ifndef CONFIG_VIRT_CPU_ACCOUNTING unsigned long ticks; #endif ktime_t now; local_irq_disable(); if (ts->idle_active || (ts->inidle && ts->tick_stopped)) now = ktime_get(); if (ts->idle_active) tick_nohz_stop_idle(cpu, now); if (!ts->inidle || !ts->tick_stopped) { ts->inidle = 0; #ifdef CONFIG_DATAPLANE if (!user_idle) #endif local_irq_enable(); return; } ts->inidle = 0; rcu_exit_nohz(); /* Update jiffies first */ select_nohz_load_balancer(0); tick_do_update_jiffies64(now); cpumask_clear_cpu(cpu, nohz_cpu_mask); #ifndef CONFIG_VIRT_CPU_ACCOUNTING /* * We stopped the tick in idle. Update process times would miss the * time we slept as update_process_times does only a 1 tick * accounting. Enforce that this is accounted to idle ! */ ticks = jiffies - ts->idle_jiffies; /* * We might be one off. Do not randomly account a huge number of ticks! */ if (ticks && ticks < LONG_MAX) #ifdef CONFIG_DATAPLANE { if (user_idle) { cputime_t cpu_time = jiffies_to_cputime(ticks); account_user_time(current, cpu_time, cputime_to_scaled(cpu_time)); } else { account_idle_ticks(ticks); } } #else account_idle_ticks(ticks); #endif #endif touch_softlockup_watchdog(); /* * Cancel the scheduled timer and restore the tick */ ts->tick_stopped = 0; ts->idle_exittime = now; tick_nohz_restart(ts, now); #ifdef CONFIG_DATAPLANE if (!user_idle) #endif local_irq_enable(); }