static void iseries_dedicated_idle(void) { set_thread_flag(TIF_POLLING_NRFLAG); while (1) { tick_nohz_idle_enter(); rcu_idle_enter(); if (!need_resched()) { while (!need_resched()) { ppc64_runlatch_off(); HMT_low(); if (hvlpevent_is_pending()) { HMT_medium(); ppc64_runlatch_on(); process_iSeries_events(); } } HMT_medium(); } ppc64_runlatch_on(); rcu_idle_exit(); tick_nohz_idle_exit(); schedule_preempt_disabled(); } }
static void iseries_shared_idle(void) { while (1) { tick_nohz_idle_enter(); rcu_idle_enter(); while (!need_resched() && !hvlpevent_is_pending()) { local_irq_disable(); ppc64_runlatch_off(); /* Recheck with irqs off */ if (!need_resched() && !hvlpevent_is_pending()) yield_shared_processor(); HMT_medium(); local_irq_enable(); } ppc64_runlatch_on(); rcu_idle_exit(); tick_nohz_idle_exit(); if (hvlpevent_is_pending()) process_iSeries_events(); schedule_preempt_disabled(); } }
void cpu_idle(void) { set_thread_flag(TIF_POLLING_NRFLAG); /* endless idle loop with no priority at all */ while (1) { tick_nohz_idle_enter(); rcu_idle_enter(); while (!need_resched()) { check_pgt_cache(); rmb(); clear_thread_flag(TIF_POLLING_NRFLAG); local_irq_disable(); /* Don't trace irqs off for idle */ stop_critical_timings(); if (!need_resched() && powersave != NULL) powersave(); start_critical_timings(); local_irq_enable(); set_thread_flag(TIF_POLLING_NRFLAG); } rcu_idle_exit(); tick_nohz_idle_exit(); preempt_enable_no_resched(); schedule(); preempt_disable(); } }
/* * Generic idle loop implementation * * Called with polling cleared. */ static void do_idle(void) { /* * If the arch has a polling bit, we maintain an invariant: * * Our polling bit is clear if we're not scheduled (i.e. if rq->curr != * rq->idle). This means that, if rq->idle has the polling bit set, * then setting need_resched is guaranteed to cause the CPU to * reschedule. */ __current_set_polling(); tick_nohz_idle_enter(); while (!need_resched()) { check_pgt_cache(); rmb(); if (cpu_is_offline(smp_processor_id())) { cpuhp_report_idle_dead(); arch_cpu_idle_dead(); } local_irq_disable(); arch_cpu_idle_enter(); /* * In poll mode we reenable interrupts and spin. Also if we * detected in the wakeup from idle path that the tick * broadcast device expired for us, we don't want to go deep * idle as we know that the IPI is going to arrive right away. */ if (cpu_idle_force_poll || tick_check_broadcast_expired()) cpu_idle_poll(); else cpuidle_idle_call(); arch_cpu_idle_exit(); } /* * Since we fell out of the loop above, we know TIF_NEED_RESCHED must * be set, propagate it into PREEMPT_NEED_RESCHED. * * This is required because for polling idle loops we will not have had * an IPI to fold the state for us. */ preempt_set_need_resched(); tick_nohz_idle_exit(); __current_clr_polling(); /* * We promise to call sched_ttwu_pending() and reschedule if * need_resched() is set while polling is set. That means that clearing * polling needs to be visible before doing these things. */ smp_mb__after_atomic(); sched_ttwu_pending(); schedule_preempt_disabled(); }
/* * The body of the idle task. */ void cpu_idle(void) { if (ppc_md.idle_loop) ppc_md.idle_loop(); /* doesn't return */ set_thread_flag(TIF_POLLING_NRFLAG); while (1) { tick_nohz_idle_enter(); rcu_idle_enter(); while (!need_resched() && !cpu_should_die()) { ppc64_runlatch_off(); if (ppc_md.power_save) { clear_thread_flag(TIF_POLLING_NRFLAG); /* * smp_mb is so clearing of TIF_POLLING_NRFLAG * is ordered w.r.t. need_resched() test. */ smp_mb(); local_irq_disable(); /* Don't trace irqs off for idle */ stop_critical_timings(); /* check again after disabling irqs */ if (!need_resched() && !cpu_should_die()) ppc_md.power_save(); start_critical_timings(); /* Some power_save functions return with * interrupts enabled, some don't. */ if (irqs_disabled()) local_irq_enable(); set_thread_flag(TIF_POLLING_NRFLAG); } else { /* * Go into low thread priority and possibly * low power mode. */ HMT_low(); HMT_very_low(); } } HMT_medium(); ppc64_runlatch_on(); rcu_idle_exit(); tick_nohz_idle_exit(); if (cpu_should_die()) { sched_preempt_enable_no_resched(); cpu_die(); } schedule_preempt_disabled(); } }
/* * Generic idle loop implementation */ static void cpu_idle_loop(void) { while (1) { tick_nohz_idle_enter(); while (!need_resched()) { check_pgt_cache(); rmb(); if (cpu_is_offline(smp_processor_id())) arch_cpu_idle_dead(); local_irq_disable(); arch_cpu_idle_enter(); /* * In poll mode we reenable interrupts and spin. * * Also if we detected in the wakeup from idle * path that the tick broadcast device expired * for us, we don't want to go deep idle as we * know that the IPI is going to arrive right * away */ if (cpu_idle_force_poll || tick_check_broadcast_expired()) { cpu_idle_poll(); } else { if (!current_clr_polling_and_test()) { stop_critical_timings(); rcu_idle_enter(); arch_cpu_idle(); WARN_ON_ONCE(irqs_disabled()); rcu_idle_exit(); start_critical_timings(); } else { local_irq_enable(); } __current_set_polling(); } arch_cpu_idle_exit(); } /* * Since we fell out of the loop above, we know * TIF_NEED_RESCHED must be set, propagate it into * PREEMPT_NEED_RESCHED. * * This is required because for polling idle loops we will * not have had an IPI to fold the state for us. */ preempt_set_need_resched(); tick_nohz_idle_exit(); schedule_preempt_disabled(); } }
/* * Generic idle loop implementation * * Called with polling cleared. */ static void cpu_idle_loop(void) { while (1) { /* * If the arch has a polling bit, we maintain an invariant: * * Our polling bit is clear if we're not scheduled (i.e. if * rq->curr != rq->idle). This means that, if rq->idle has * the polling bit set, then setting need_resched is * guaranteed to cause the cpu to reschedule. */ __current_set_polling(); tick_nohz_idle_enter(); while (!need_resched()) { check_pgt_cache(); rmb(); local_irq_disable(); arch_cpu_idle_enter(); /* * In poll mode we reenable interrupts and spin. * * Also if we detected in the wakeup from idle * path that the tick broadcast device expired * for us, we don't want to go deep idle as we * know that the IPI is going to arrive right * away */ if (cpu_idle_force_poll || tick_check_broadcast_expired() || __get_cpu_var(idle_force_poll)) cpu_idle_poll(); else cpuidle_idle_call(); arch_cpu_idle_exit(); } tick_nohz_idle_exit(); __current_clr_polling(); /* * We promise to reschedule if need_resched is set while * polling is set. That means that clearing polling * needs to be visible before rescheduling. */ smp_mb__after_atomic(); schedule_preempt_disabled(); if (cpu_is_offline(smp_processor_id())) arch_cpu_idle_dead(); } }
static void __cpuinit xen_play_dead(void) /* used only with HOTPLUG_CPU */ { play_dead_common(); HYPERVISOR_vcpu_op(VCPUOP_down, smp_processor_id(), NULL); cpu_bringup(); /* * commit 4b0c0f294 (tick: Cleanup NOHZ per cpu data on cpu down) * clears certain data that the cpu_idle loop (which called us * and that we return from) expects. The only way to get that * data back is to call: */ tick_nohz_idle_enter(); }
void cpu_idle(void) { if (ppc_md.idle_loop) ppc_md.idle_loop(); set_thread_flag(TIF_POLLING_NRFLAG); while (1) { tick_nohz_idle_enter(); rcu_idle_enter(); while (!need_resched() && !cpu_should_die()) { ppc64_runlatch_off(); if (ppc_md.power_save) { clear_thread_flag(TIF_POLLING_NRFLAG); smp_mb(); local_irq_disable(); stop_critical_timings(); if (!need_resched() && !cpu_should_die()) ppc_md.power_save(); start_critical_timings(); if (irqs_disabled()) local_irq_enable(); set_thread_flag(TIF_POLLING_NRFLAG); } else { HMT_low(); HMT_very_low(); } } HMT_medium(); ppc64_runlatch_on(); rcu_idle_exit(); tick_nohz_idle_exit(); if (cpu_should_die()) { sched_preempt_enable_no_resched(); cpu_die(); } schedule_preempt_disabled(); } }
static void xen_pv_play_dead(void) /* used only with HOTPLUG_CPU */ { play_dead_common(); HYPERVISOR_vcpu_op(VCPUOP_down, xen_vcpu_nr(smp_processor_id()), NULL); cpu_bringup(); /* * commit 4b0c0f294 (tick: Cleanup NOHZ per cpu data on cpu down) * clears certain data that the cpu_idle loop (which called us * and that we return from) expects. The only way to get that * data back is to call: */ tick_nohz_idle_enter(); tick_nohz_idle_stop_tick_protected(); cpuhp_online_idle(CPUHP_AP_ONLINE_IDLE); }
void cpu_idle(void) { while (1) { tick_nohz_idle_enter(); local_irq_disable(); while (!need_resched()) { idle_sleep(); /* interrupts wake us up, but aren't serviced */ local_irq_enable(); /* service interrupt */ local_irq_disable(); } local_irq_enable(); tick_nohz_idle_exit(); schedule(); } }
/* * Generic idle loop implementation */ static void cpu_idle_loop(void) { while (1) { tick_nohz_idle_enter(); while (!need_resched()) { check_pgt_cache(); rmb(); local_irq_disable(); arch_cpu_idle_enter(); /* * In poll mode we reenable interrupts and spin. * * Also if we detected in the wakeup from idle * path that the tick broadcast device expired * for us, we don't want to go deep idle as we * know that the IPI is going to arrive right * away */ if (cpu_idle_force_poll || tick_check_broadcast_expired() || __get_cpu_var(idle_force_poll)) { cpu_idle_poll(); } else { if (!current_clr_polling_and_test()) { stop_critical_timings(); rcu_idle_enter(); arch_cpu_idle(); WARN_ON_ONCE(irqs_disabled()); rcu_idle_exit(); start_critical_timings(); } else { local_irq_enable(); } __current_set_polling(); } arch_cpu_idle_exit(); } tick_nohz_idle_exit(); schedule_preempt_disabled(); if (cpu_is_offline(smp_processor_id())) arch_cpu_idle_dead(); } }
/* * The idle thread, has rather strange semantics for calling pm_idle, * but this is what x86 does and we need to do the same, so that * things like cpuidle get called in the same way. The only difference * is that we always respect 'hlt_counter' to prevent low power idle. */ void cpu_idle(void) { local_fiq_enable(); /* endless idle loop with no priority at all */ while (1) { idle_notifier_call_chain(IDLE_START); tick_nohz_idle_enter(); rcu_idle_enter(); while (!need_resched()) { /* * We need to disable interrupts here * to ensure we don't miss a wakeup call. */ local_irq_disable(); #ifdef CONFIG_PL310_ERRATA_769419 wmb(); #endif if (hlt_counter) { local_irq_enable(); cpu_relax(); } else if (!need_resched()) { stop_critical_timings(); if (cpuidle_idle_call()) pm_idle(); start_critical_timings(); /* * pm_idle functions must always * return with IRQs enabled. */ WARN_ON(irqs_disabled()); } else local_irq_enable(); } rcu_idle_exit(); tick_nohz_idle_exit(); idle_notifier_call_chain(IDLE_END); schedule_preempt_disabled(); #ifdef CONFIG_HOTPLUG_CPU if (cpu_is_offline(smp_processor_id())) cpu_die(); #endif } }
/* * The idle thread. There's no useful work to be * done, so just try to conserve power and have a * low exit latency (ie sit in a loop waiting for * somebody to say that they'd like to reschedule) */ void cpu_idle(void) { int cpu = smp_processor_id(); current_thread_info()->status |= TS_POLLING; if (no_idle_nap) { while (1) { while (!need_resched()) cpu_relax(); schedule(); } } /* endless idle loop with no priority at all */ while (1) { tick_nohz_idle_enter(); rcu_idle_enter(); while (!need_resched()) { if (cpu_is_offline(cpu)) BUG(); /* no HOTPLUG_CPU */ local_irq_disable(); __get_cpu_var(irq_stat).idle_timestamp = jiffies; current_thread_info()->status &= ~TS_POLLING; /* * TS_POLLING-cleared state must be visible before we * test NEED_RESCHED: */ smp_mb(); if (!need_resched()) _cpu_idle(); else local_irq_enable(); current_thread_info()->status |= TS_POLLING; } rcu_idle_exit(); tick_nohz_idle_exit(); schedule_preempt_disabled(); } }
/* * The idle thread. There's no useful work to be * done, so just try to conserve power and have a * low exit latency (ie sit in a loop waiting for * somebody to say that they'd like to reschedule) */ void cpu_idle(void) { int cpu = smp_processor_id(); /* * If we're the non-boot CPU, nothing set the stack canary up * for us. CPU0 already has it initialized but no harm in * doing it again. This is a good place for updating it, as * we wont ever return from this function (so the invalid * canaries already on the stack wont ever trigger). */ boot_init_stack_canary(); current_thread_info()->status |= TS_POLLING; /* endless idle loop with no priority at all */ while (1) { tick_nohz_idle_enter(); rcu_idle_enter(); while (!need_resched()) { check_pgt_cache(); rmb(); if (cpu_is_offline(cpu)) play_dead(); local_touch_nmi(); local_irq_disable(); /* Don't trace irqs off for idle */ stop_critical_timings(); if (cpuidle_idle_call()) pm_idle(); start_critical_timings(); } rcu_idle_exit(); tick_nohz_idle_exit(); preempt_enable_no_resched(); schedule(); preempt_disable(); } }
/* * The idle thread. We try to conserve power, while trying to keep * overall latency low. The architecture specific idle is passed * a value to indicate the level of "idleness" of the system. */ void cpu_idle(void) { /* endless idle loop with no priority at all */ while (1) { void (*idle)(void) = pm_idle; #ifdef CONFIG_HOTPLUG_CPU if (cpu_is_offline(smp_processor_id())) cpu_die(); #endif if (!idle) idle = default_idle; tick_nohz_idle_enter(); rcu_idle_enter(); while (!need_resched()) idle(); rcu_idle_exit(); tick_nohz_idle_exit(); schedule_preempt_disabled(); } }
void cpu_idle(void) { set_thread_flag(TIF_POLLING_NRFLAG); /* endless idle loop with no priority at all */ while (1) { void (*idle)(void) = pm_idle; if (!idle) idle = default_idle; tick_nohz_idle_enter(); rcu_idle_enter(); while (!need_resched()) idle(); rcu_idle_exit(); tick_nohz_idle_exit(); schedule_preempt_disabled(); check_pgt_cache(); } }
/* * The idle thread. There's no useful work to be done, so just try to conserve * power and have a low exit latency (ie sit in a loop waiting for somebody to * say that they'd like to reschedule) */ void __noreturn cpu_idle(void) { int cpu; /* CPU is going idle. */ cpu = smp_processor_id(); /* endless idle loop with no priority at all */ while (1) { tick_nohz_idle_enter(); rcu_idle_enter(); while (!need_resched() && cpu_online(cpu)) { #ifdef CONFIG_MIPS_MT_SMTC extern void smtc_idle_loop_hook(void); smtc_idle_loop_hook(); #endif if (cpu_wait) { /* Don't trace irqs off for idle */ stop_critical_timings(); (*cpu_wait)(); start_critical_timings(); } } #ifdef CONFIG_HOTPLUG_CPU if (!cpu_online(cpu) && !cpu_isset(cpu, cpu_callin_map) && (system_state == SYSTEM_RUNNING || system_state == SYSTEM_BOOTING)) play_dead(); #endif rcu_idle_exit(); tick_nohz_idle_exit(); schedule_preempt_disabled(); } }
/* * The idle thread. There's no useful work to be done, so just try to conserve * power and have a low exit latency (ie sit in a loop waiting for somebody to * say that they'd like to reschedule) */ void cpu_idle(void) { unsigned int cpu = smp_processor_id(); set_thread_flag(TIF_POLLING_NRFLAG); /* endless idle loop with no priority at all */ while (1) { tick_nohz_idle_enter(); rcu_idle_enter(); while (!need_resched()) { check_pgt_cache(); rmb(); if (cpu_is_offline(cpu)) play_dead(); local_irq_disable(); /* Don't trace irqs off for idle */ stop_critical_timings(); if (cpuidle_idle_call()) sh_idle(); /* * Sanity check to ensure that sh_idle() returns * with IRQs enabled */ WARN_ON(irqs_disabled()); start_critical_timings(); } rcu_idle_exit(); tick_nohz_idle_exit(); schedule_preempt_disabled(); } }
static int clamp_thread(void *arg) { int cpunr = (unsigned long)arg; DEFINE_TIMER(wakeup_timer, noop_timer, 0, 0); static const struct sched_param param = { .sched_priority = MAX_USER_RT_PRIO/2, }; unsigned int count = 0; unsigned int target_ratio; set_bit(cpunr, cpu_clamping_mask); set_freezable(); init_timer_on_stack(&wakeup_timer); sched_setscheduler(current, SCHED_FIFO, ¶m); while (true == clamping && !kthread_should_stop() && cpu_online(cpunr)) { int sleeptime; unsigned long target_jiffies; unsigned int guard; unsigned int compensation = 0; int interval; /* jiffies to sleep for each attempt */ unsigned int duration_jiffies = msecs_to_jiffies(duration); unsigned int window_size_now; try_to_freeze(); /* * make sure user selected ratio does not take effect until * the next round. adjust target_ratio if user has changed * target such that we can converge quickly. */ target_ratio = set_target_ratio; guard = 1 + target_ratio/20; window_size_now = window_size; count++; /* * systems may have different ability to enter package level * c-states, thus we need to compensate the injected idle ratio * to achieve the actual target reported by the HW. */ compensation = get_compensation(target_ratio); interval = duration_jiffies*100/(target_ratio+compensation); /* align idle time */ target_jiffies = roundup(jiffies, interval); sleeptime = target_jiffies - jiffies; if (sleeptime <= 0) sleeptime = 1; schedule_timeout_interruptible(sleeptime); /* * only elected controlling cpu can collect stats and update * control parameters. */ if (cpunr == control_cpu && !(count%window_size_now)) { should_skip = powerclamp_adjust_controls(target_ratio, guard, window_size_now); smp_mb(); } if (should_skip) continue; target_jiffies = jiffies + duration_jiffies; mod_timer(&wakeup_timer, target_jiffies); if (unlikely(local_softirq_pending())) continue; /* * stop tick sched during idle time, interrupts are still * allowed. thus jiffies are updated properly. */ preempt_disable(); tick_nohz_idle_enter(); /* mwait until target jiffies is reached */ while (time_before(jiffies, target_jiffies)) { unsigned long ecx = 1; unsigned long eax = target_mwait; /* * REVISIT: may call enter_idle() to notify drivers who * can save power during cpu idle. same for exit_idle() */ local_touch_nmi(); stop_critical_timings(); mwait_idle_with_hints(eax, ecx); start_critical_timings(); atomic_inc(&idle_wakeup_counter); } tick_nohz_idle_exit(); preempt_enable(); } del_timer_sync(&wakeup_timer); clear_bit(cpunr, cpu_clamping_mask); return 0; } /* * 1 HZ polling while clamping is active, useful for userspace * to monitor actual idle ratio. */ static void poll_pkg_cstate(struct work_struct *dummy); static DECLARE_DELAYED_WORK(poll_pkg_cstate_work, poll_pkg_cstate); static void poll_pkg_cstate(struct work_struct *dummy) { static u64 msr_last; static u64 tsc_last; static unsigned long jiffies_last; u64 msr_now; unsigned long jiffies_now; u64 tsc_now; u64 val64; msr_now = pkg_state_counter(); rdtscll(tsc_now); jiffies_now = jiffies; /* calculate pkg cstate vs tsc ratio */ if (!msr_last || !tsc_last) pkg_cstate_ratio_cur = 1; else { if (tsc_now - tsc_last) { val64 = 100 * (msr_now - msr_last); do_div(val64, (tsc_now - tsc_last)); pkg_cstate_ratio_cur = val64; } } /* update record */ msr_last = msr_now; jiffies_last = jiffies_now; tsc_last = tsc_now; if (true == clamping) schedule_delayed_work(&poll_pkg_cstate_work, HZ); } static int start_power_clamp(void) { unsigned long cpu; struct task_struct *thread; /* check if pkg cstate counter is completely 0, abort in this case */ if (!has_pkg_state_counter()) { pr_err("pkg cstate counter not functional, abort\n"); return -EINVAL; } set_target_ratio = clamp(set_target_ratio, 0U, MAX_TARGET_RATIO - 1); /* prevent cpu hotplug */ get_online_cpus(); /* prefer BSP */ control_cpu = 0; if (!cpu_online(control_cpu)) control_cpu = smp_processor_id(); clamping = true; schedule_delayed_work(&poll_pkg_cstate_work, 0); /* start one thread per online cpu */ for_each_online_cpu(cpu) { struct task_struct **p = per_cpu_ptr(powerclamp_thread, cpu); thread = kthread_create_on_node(clamp_thread, (void *) cpu, cpu_to_node(cpu), "kidle_inject/%ld", cpu); /* bind to cpu here */ if (likely(!IS_ERR(thread))) { kthread_bind(thread, cpu); wake_up_process(thread); *p = thread; } } put_online_cpus(); return 0; }
/* * Generic idle loop implementation * * Called with polling cleared. */ static void cpu_idle_loop(void) { while (1) { /* * If the arch has a polling bit, we maintain an invariant: * * Our polling bit is clear if we're not scheduled (i.e. if * rq->curr != rq->idle). This means that, if rq->idle has * the polling bit set, then setting need_resched is * guaranteed to cause the cpu to reschedule. */ __current_set_polling(); tick_nohz_idle_enter(); while (!need_resched()) { check_pgt_cache(); rmb(); if (cpu_is_offline(smp_processor_id())) arch_cpu_idle_dead(); local_irq_disable(); arch_cpu_idle_enter(); /* * In poll mode we reenable interrupts and spin. * * Also if we detected in the wakeup from idle * path that the tick broadcast device expired * for us, we don't want to go deep idle as we * know that the IPI is going to arrive right * away */ if (cpu_idle_force_poll || tick_check_broadcast_expired()) cpu_idle_poll(); else cpuidle_idle_call(); arch_cpu_idle_exit(); /* * We need to test and propagate the TIF_NEED_RESCHED * bit here because we might not have send the * reschedule IPI to idle tasks. */ if (tif_need_resched()) set_preempt_need_resched(); } tick_nohz_idle_exit(); __current_clr_polling(); /* * We promise to call sched_ttwu_pending and reschedule * if need_resched is set while polling is set. That * means that clearing polling needs to be visible * before doing these things. */ smp_mb(); sched_ttwu_pending(); schedule_preempt_disabled(); } }
/* * Generic idle loop implementation * * Called with polling cleared. */ static void cpu_idle_loop(void) { while (1) { /* * If the arch has a polling bit, we maintain an invariant: * * Our polling bit is clear if we're not scheduled (i.e. if * rq->curr != rq->idle). This means that, if rq->idle has * the polling bit set, then setting need_resched is * guaranteed to cause the cpu to reschedule. */ __current_set_polling(); quiet_vmstat(); tick_nohz_idle_enter(); while (!need_resched()) { check_pgt_cache(); rmb(); if (cpu_is_offline(smp_processor_id())) { rcu_cpu_notify(NULL, CPU_DYING_IDLE, (void *)(long)smp_processor_id()); smp_mb(); /* all activity before dead. */ this_cpu_write(cpu_dead_idle, true); arch_cpu_idle_dead(); } local_irq_disable(); arch_cpu_idle_enter(); /* * In poll mode we reenable interrupts and spin. * * Also if we detected in the wakeup from idle * path that the tick broadcast device expired * for us, we don't want to go deep idle as we * know that the IPI is going to arrive right * away */ if (cpu_idle_force_poll || tick_check_broadcast_expired()) cpu_idle_poll(); else cpuidle_idle_call(); arch_cpu_idle_exit(); } /* * Since we fell out of the loop above, we know * TIF_NEED_RESCHED must be set, propagate it into * PREEMPT_NEED_RESCHED. * * This is required because for polling idle loops we will * not have had an IPI to fold the state for us. */ preempt_set_need_resched(); tick_nohz_idle_exit(); __current_clr_polling(); /* * We promise to call sched_ttwu_pending and reschedule * if need_resched is set while polling is set. That * means that clearing polling needs to be visible * before doing these things. */ smp_mb__after_atomic(); sched_ttwu_pending(); schedule_preempt_disabled(); } }