/* * When High resolution timers are active, try to reprogram. Note, that in case * the state has HRTIMER_STATE_CALLBACK set, no reprogramming and no expiry * check happens. The timer gets enqueued into the rbtree. The reprogramming * and expiry check is done in the hrtimer_interrupt or in the softirq. */ static inline int hrtimer_enqueue_reprogram(struct hrtimer *timer, struct hrtimer_clock_base *base) { if (base->cpu_base->hres_active && hrtimer_reprogram(timer, base)) { /* Timer is expired, act upon the callback mode */ switch(timer->cb_mode) { case HRTIMER_CB_IRQSAFE_PERCPU: case HRTIMER_CB_IRQSAFE_UNLOCKED: /* * This is solely for the sched tick emulation with * dynamic tick support to ensure that we do not * restart the tick right on the edge and end up with * the tick timer in the softirq ! The calling site * takes care of this. Also used for hrtimer sleeper ! */ debug_hrtimer_deactivate(timer); return 1; case HRTIMER_CB_SOFTIRQ: /* * Move everything else into the softirq pending list ! */ list_add_tail(&timer->cb_entry, &base->cpu_base->cb_pending); timer->state = HRTIMER_STATE_PENDING; return 1; default: BUG(); } } return 0; }
/* * When High resolution timers are active, try to reprogram. Note, that in case * the state has HRTIMER_STATE_CALLBACK set, no reprogramming and no expiry * check happens. The timer gets enqueued into the rbtree. The reprogramming * and expiry check is done in the hrtimer_interrupt or in the softirq. */ static inline int hrtimer_enqueue_reprogram(struct hrtimer *timer, struct hrtimer_clock_base *base) { if (base->cpu_base->hres_active && hrtimer_reprogram(timer, base)) { spin_unlock(&base->cpu_base->lock); raise_softirq_irqoff(HRTIMER_SOFTIRQ); spin_lock(&base->cpu_base->lock); return 1; } return 0; }
static void run_hrtimer_pending(struct hrtimer_cpu_base *cpu_base) { spin_lock_irq(&cpu_base->lock); while (!list_empty(&cpu_base->cb_pending)) { enum hrtimer_restart (*fn)(struct hrtimer *); struct hrtimer *timer; int restart; timer = list_entry(cpu_base->cb_pending.next, struct hrtimer, cb_entry); debug_hrtimer_deactivate(timer); timer_stats_account_hrtimer(timer); fn = timer->function; __remove_hrtimer(timer, timer->base, HRTIMER_STATE_CALLBACK, 0); spin_unlock_irq(&cpu_base->lock); restart = fn(timer); spin_lock_irq(&cpu_base->lock); timer->state &= ~HRTIMER_STATE_CALLBACK; if (restart == HRTIMER_RESTART) { BUG_ON(hrtimer_active(timer)); /* * Enqueue the timer, allow reprogramming of the event * device */ enqueue_hrtimer(timer, timer->base, 1); } else if (hrtimer_active(timer)) { /* * If the timer was rearmed on another CPU, reprogram * the event device. */ struct hrtimer_clock_base *base = timer->base; if (base->first == &timer->node && hrtimer_reprogram(timer, base)) { /* * Timer is expired. Thus move it from tree to * pending list again. */ __remove_hrtimer(timer, base, HRTIMER_STATE_PENDING, 0); list_add_tail(&timer->cb_entry, &base->cpu_base->cb_pending); } } } spin_unlock_irq(&cpu_base->lock); }
/** * hrtimer_start_range_ns - (re)start an hrtimer on the current CPU * @timer: the timer to be added * @tim: expiry time * @delta_ns: "slack" range for the timer * @mode: expiry mode: absolute (HRTIMER_MODE_ABS) or * relative (HRTIMER_MODE_REL) */ void hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, unsigned long delta_ns, const enum hrtimer_mode mode) { struct hrtimer_clock_base *base, *new_base; unsigned long flags; int leftmost; base = lock_hrtimer_base(timer, &flags); /* Remove an active timer from the queue: */ remove_hrtimer(timer, base, true); if (mode & HRTIMER_MODE_REL) { tim = ktime_add_safe(tim, base->get_time()); /* * CONFIG_TIME_LOW_RES is a temporary way for architectures * to signal that they simply return xtime in * do_gettimeoffset(). In this case we want to round up by * resolution when starting a relative timer, to avoid short * timeouts. This will go away with the GTOD framework. */ #ifdef CONFIG_TIME_LOW_RES tim = ktime_add_safe(tim, ktime_set(0, hrtimer_resolution)); #endif } hrtimer_set_expires_range_ns(timer, tim, delta_ns); /* Switch the timer base, if necessary: */ new_base = switch_hrtimer_base(timer, base, mode & HRTIMER_MODE_PINNED); timer_stats_hrtimer_set_start_info(timer); leftmost = enqueue_hrtimer(timer, new_base); if (!leftmost) goto unlock; if (!hrtimer_is_hres_active(timer)) { /* * Kick to reschedule the next tick to handle the new timer * on dynticks target. */ if (new_base->cpu_base->nohz_active) wake_up_nohz_cpu(new_base->cpu_base->cpu); } else { hrtimer_reprogram(timer, new_base); } unlock: unlock_hrtimer_base(timer, &flags); }
static void run_hrtimer_softirq(struct softirq_action *h) { struct hrtimer_cpu_base *cpu_base = &__get_cpu_var(hrtimer_bases); spin_lock_irq(&cpu_base->lock); while (!list_empty(&cpu_base->cb_pending)) { enum hrtimer_restart (*fn)(struct hrtimer *); struct hrtimer *timer; int restart; timer = list_entry(cpu_base->cb_pending.next, struct hrtimer, cb_entry); timer_stats_account_hrtimer(timer); fn = timer->function; __remove_hrtimer(timer, timer->base, HRTIMER_STATE_CALLBACK, 0); spin_unlock_irq(&cpu_base->lock); restart = fn(timer); spin_lock_irq(&cpu_base->lock); timer->state &= ~HRTIMER_STATE_CALLBACK; if (restart == HRTIMER_RESTART) { BUG_ON(hrtimer_active(timer)); /* * Enqueue the timer, allow reprogramming of the event * device */ enqueue_hrtimer(timer, timer->base, 1); } else if (hrtimer_active(timer)) { /* * If the timer was rearmed on another CPU, reprogram * the event device. */ if (timer->base->first == &timer->node) hrtimer_reprogram(timer, timer->base); } } spin_unlock_irq(&cpu_base->lock); }
/** * hrtimer_start_range_ns - (re)start an hrtimer on the current CPU * @timer: the timer to be added * @tim: expiry time * @delta_ns: "slack" range for the timer * @mode: expiry mode: absolute (HRTIMER_MODE_ABS) or * relative (HRTIMER_MODE_REL) */ void hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, u64 delta_ns, const enum hrtimer_mode mode) { struct hrtimer_clock_base *base, *new_base; unsigned long flags; int leftmost; base = lock_hrtimer_base(timer, &flags); /* Remove an active timer from the queue: */ remove_hrtimer(timer, base, true); if (mode & HRTIMER_MODE_REL) tim = ktime_add_safe(tim, base->get_time()); tim = hrtimer_update_lowres(timer, tim, mode); hrtimer_set_expires_range_ns(timer, tim, delta_ns); /* Switch the timer base, if necessary: */ new_base = switch_hrtimer_base(timer, base, mode & HRTIMER_MODE_PINNED); timer_stats_hrtimer_set_start_info(timer); leftmost = enqueue_hrtimer(timer, new_base); if (!leftmost) goto unlock; if (!hrtimer_is_hres_active(timer)) { /* * Kick to reschedule the next tick to handle the new timer * on dynticks target. */ if (new_base->cpu_base->nohz_active) wake_up_nohz_cpu(new_base->cpu_base->cpu); } else { hrtimer_reprogram(timer, new_base); } unlock: unlock_hrtimer_base(timer, &flags); }
/* * When High resolution timers are active, try to reprogram. Note, that in case * the state has HRTIMER_STATE_CALLBACK set, no reprogramming and no expiry * check happens. The timer gets enqueued into the rbtree. The reprogramming * and expiry check is done in the hrtimer_interrupt or in the softirq. */ static inline int hrtimer_enqueue_reprogram(struct hrtimer *timer, struct hrtimer_clock_base *base) { if (base->cpu_base->hres_active && hrtimer_reprogram(timer, base)) { /* Timer is expired, act upon the callback mode */ switch(timer->cb_mode) { case HRTIMER_CB_IRQSAFE_NO_RESTART: debug_hrtimer_deactivate(timer); /* * We can call the callback from here. No restart * happens, so no danger of recursion */ BUG_ON(timer->function(timer) != HRTIMER_NORESTART); return 1; case HRTIMER_CB_IRQSAFE_NO_SOFTIRQ: /* * This is solely for the sched tick emulation with * dynamic tick support to ensure that we do not * restart the tick right on the edge and end up with * the tick timer in the softirq ! The calling site * takes care of this. */ debug_hrtimer_deactivate(timer); return 1; case HRTIMER_CB_IRQSAFE: case HRTIMER_CB_SOFTIRQ: /* * Move everything else into the softirq pending list ! */ list_add_tail(&timer->cb_entry, &base->cpu_base->cb_pending); timer->state = HRTIMER_STATE_PENDING; return 1; default: BUG(); } } return 0; }
/* * When High resolution timers are active, try to reprogram. Note, that in case * the state has HRTIMER_STATE_CALLBACK set, no reprogramming and no expiry * check happens. The timer gets enqueued into the rbtree. The reprogramming * and expiry check is done in the hrtimer_interrupt or in the softirq. */ static inline int hrtimer_enqueue_reprogram(struct hrtimer *timer, struct hrtimer_clock_base *base) { return base->cpu_base->hres_active && hrtimer_reprogram(timer, base); }
int __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, unsigned long delta_ns, const enum hrtimer_mode mode, int wakeup) { struct hrtimer_clock_base *base, *new_base; unsigned long flags; int ret, leftmost; base = lock_hrtimer_base(timer, &flags); /* Remove an active timer from the queue: */ ret = remove_hrtimer(timer, base); if (mode & HRTIMER_MODE_REL) { tim = ktime_add_safe(tim, base->get_time()); /* * CONFIG_TIME_LOW_RES is a temporary way for architectures * to signal that they simply return xtime in * do_gettimeoffset(). In this case we want to round up by * resolution when starting a relative timer, to avoid short * timeouts. This will go away with the GTOD framework. */ #ifdef CONFIG_TIME_LOW_RES tim = ktime_add_safe(tim, base->resolution); #endif } hrtimer_set_expires_range_ns(timer, tim, delta_ns); /* Switch the timer base, if necessary: */ new_base = switch_hrtimer_base(timer, base, mode & HRTIMER_MODE_PINNED); timer_stats_hrtimer_set_start_info(timer); leftmost = enqueue_hrtimer(timer, new_base); if (!leftmost) { unlock_hrtimer_base(timer, &flags); return ret; } if (!hrtimer_is_hres_active(timer)) { /* * Kick to reschedule the next tick to handle the new timer * on dynticks target. */ wake_up_nohz_cpu(new_base->cpu_base->cpu); } else if (new_base->cpu_base == this_cpu_ptr(&hrtimer_bases) && hrtimer_reprogram(timer, new_base)) { /* * Only allow reprogramming if the new base is on this CPU. * (it might still be on another CPU if the timer was pending) * * XXX send_remote_softirq() ? */ if (wakeup) { /* * We need to drop cpu_base->lock to avoid a * lock ordering issue vs. rq->lock. */ raw_spin_unlock(&new_base->cpu_base->lock); raise_softirq_irqoff(HRTIMER_SOFTIRQ); local_irq_restore(flags); return ret; } else { __raise_softirq_irqoff(HRTIMER_SOFTIRQ); } } unlock_hrtimer_base(timer, &flags); return ret; }
static void run_hrtimer_pending(struct hrtimer_cpu_base *cpu_base) { spin_lock_irq(&cpu_base->lock); while (!list_empty(&cpu_base->cb_pending)) { enum hrtimer_restart (*fn)(struct hrtimer *); struct hrtimer *timer; int restart; int emulate_hardirq_ctx = 0; timer = list_entry(cpu_base->cb_pending.next, struct hrtimer, cb_entry); debug_hrtimer_deactivate(timer); timer_stats_account_hrtimer(timer); fn = timer->function; /* * A timer might have been added to the cb_pending list * when it was migrated during a cpu-offline operation. * Emulate hardirq context for such timers. */ if (timer->cb_mode == HRTIMER_CB_IRQSAFE_PERCPU || timer->cb_mode == HRTIMER_CB_IRQSAFE_UNLOCKED) emulate_hardirq_ctx = 1; __remove_hrtimer(timer, timer->base, HRTIMER_STATE_CALLBACK, 0); spin_unlock_irq(&cpu_base->lock); if (unlikely(emulate_hardirq_ctx)) { local_irq_disable(); restart = fn(timer); local_irq_enable(); } else restart = fn(timer); spin_lock_irq(&cpu_base->lock); timer->state &= ~HRTIMER_STATE_CALLBACK; if (restart == HRTIMER_RESTART) { BUG_ON(hrtimer_active(timer)); /* * Enqueue the timer, allow reprogramming of the event * device */ enqueue_hrtimer(timer, timer->base, 1); } else if (hrtimer_active(timer)) { /* * If the timer was rearmed on another CPU, reprogram * the event device. */ struct hrtimer_clock_base *base = timer->base; if (base->first == &timer->node && hrtimer_reprogram(timer, base)) { /* * Timer is expired. Thus move it from tree to * pending list again. */ __remove_hrtimer(timer, base, HRTIMER_STATE_PENDING, 0); list_add_tail(&timer->cb_entry, &base->cpu_base->cb_pending); } } } spin_unlock_irq(&cpu_base->lock); }