void __init setup_sched_clock(u32 (*read)(void), int bits, unsigned long rate) { unsigned long r, w; u64 res, wrap; char r_unit; if (cd.rate > rate) return; BUG_ON(bits > 32); WARN_ON(!irqs_disabled()); read_sched_clock = read; sched_clock_mask = (1 << bits) - 1; cd.rate = rate; /* calculate the mult/shift to convert counter ticks to ns. */ clocks_calc_mult_shift(&cd.mult, &cd.shift, rate, NSEC_PER_SEC, 0); r = rate; if (r >= 4000000) { r /= 1000000; r_unit = 'M'; } else if (r >= 1000) { r /= 1000; r_unit = 'k'; } else r_unit = ' '; /* calculate how many ns until we wrap */ wrap = cyc_to_ns((1ULL << bits) - 1, cd.mult, cd.shift); do_div(wrap, NSEC_PER_MSEC); w = wrap; /* calculate the ns resolution of this counter */ res = cyc_to_ns(1ULL, cd.mult, cd.shift); pr_info("sched_clock: %u bits at %lu%cHz, resolution %lluns, wraps every %lums\n", bits, r, r_unit, res, w); /* * Start the timer to keep sched_clock() properly updated and * sets the initial epoch. */ sched_clock_timer.data = msecs_to_jiffies(w - (w / 10)); update_sched_clock(); /* * Ensure that sched_clock() starts off at 0ns */ cd.epoch_ns = 0; /* Enable IRQ time accounting if we have a fast enough sched_clock */ if (irqtime > 0 || (irqtime == -1 && rate >= 1000000)) enable_sched_clock_irqtime(); pr_debug("Registered %pF as sched_clock source\n", read); }
static unsigned long long notrace cyc_to_sched_clock(u32 cyc, u32 mask) { u64 epoch_ns; u32 epoch_cyc; if (cd.suspended) return cd.epoch_ns; /* * Load the epoch_cyc and epoch_ns atomically. We do this by * ensuring that we always write epoch_cyc, epoch_ns and * epoch_cyc_copy in strict order, and read them in strict order. * If epoch_cyc and epoch_cyc_copy are not equal, then we're in * the middle of an update, and we should repeat the load. */ do { epoch_cyc = cd.epoch_cyc; smp_rmb(); epoch_ns = cd.epoch_ns; smp_rmb(); } while (epoch_cyc != cd.epoch_cyc_copy); cyc = read_sched_clock(); cyc = (cyc - epoch_cyc) & sched_clock_mask; return epoch_ns + cyc_to_ns(cyc, cd.mult, cd.shift); }
void __init register_current_timer_delay(const struct delay_timer *timer) { u32 new_mult, new_shift; u64 res; clocks_calc_mult_shift(&new_mult, &new_shift, timer->freq, NSEC_PER_SEC, 3600); res = cyc_to_ns(1ULL, new_mult, new_shift); if (res > 1000) { pr_err("Ignoring delay timer %ps, which has insufficient resolution of %lluns\n", timer, res); return; } if (!delay_calibrated && (!delay_res || (res < delay_res))) { pr_info("Switching to timer-based delay loop, resolution %lluns\n", res); delay_timer = timer; lpj_fine = timer->freq / HZ; delay_res = res; /* cpufreq may scale loops_per_jiffy, so keep a private copy */ arm_delay_ops.ticks_per_jiffy = lpj_fine; arm_delay_ops.delay = __timer_delay; arm_delay_ops.const_udelay = __timer_const_udelay; arm_delay_ops.udelay = __timer_udelay; } else { pr_info("Ignoring duplicate/late registration of read_current_timer delay\n"); } }
void __init init_sched_clock(struct clock_data *cd, void (*update)(void), unsigned int clock_bits, unsigned long rate) { unsigned long r, w; u64 res, wrap; char r_unit; sched_clock_update_fn = update; /* calculate the mult/shift to convert counter ticks to ns. */ clocks_calc_mult_shift(&cd->mult, &cd->shift, rate, NSEC_PER_SEC, 60); r = rate; if (r >= 4000000) { r /= 1000000; r_unit = 'M'; } else { r /= 1000; r_unit = 'k'; } /* calculate how many ns until we wrap */ wrap = cyc_to_ns((1ULL << clock_bits) - 1, cd->mult, cd->shift); do_div(wrap, NSEC_PER_MSEC); w = wrap; /* calculate the ns resolution of this counter */ res = cyc_to_ns(1ULL, cd->mult, cd->shift); pr_info("sched_clock: %u bits at %lu%cHz, resolution %lluns, wraps every %lums\n", clock_bits, r, r_unit, res, w); /* * Start the timer to keep sched_clock() properly updated and * sets the initial epoch. */ sched_clock_timer.data = msecs_to_jiffies(w - (w / 10)); update(); /* * Ensure that sched_clock() starts off at 0ns */ cd->epoch_ns = 0; }
void __init setup_sched_clock(u32 (*read)(void), int bits, unsigned long rate) { unsigned long r, w; u64 res, wrap; char r_unit; BUG_ON(bits > 32); WARN_ON(!irqs_disabled()); WARN_ON(read_sched_clock != jiffy_sched_clock_read); read_sched_clock = read; sched_clock_mask = (1 << bits) - 1; clocks_calc_mult_shift(&cd.mult, &cd.shift, rate, NSEC_PER_SEC, 0); r = rate; if (r >= 4000000) { r /= 1000000; r_unit = 'M'; } else if (r >= 1000) { r /= 1000; r_unit = 'k'; } else r_unit = ' '; wrap = cyc_to_ns((1ULL << bits) - 1, cd.mult, cd.shift); do_div(wrap, NSEC_PER_MSEC); w = wrap; res = cyc_to_ns(1ULL, cd.mult, cd.shift); pr_info("sched_clock: %u bits at %lu%cHz, resolution %lluns, wraps every %lums\n", bits, r, r_unit, res, w); sched_clock_timer.data = msecs_to_jiffies(w - (w / 10)); update_sched_clock(); cd.epoch_ns = 0; pr_debug("Registered %pF as sched_clock source\n", read); }
static unsigned long long cyc_to_sched_clock(u32 cyc, u32 mask) { u64 epoch_ns; u32 epoch_cyc; do { epoch_cyc = cd.epoch_cyc; smp_rmb(); epoch_ns = cd.epoch_ns; smp_rmb(); } while (epoch_cyc != cd.epoch_cyc_copy); return epoch_ns + cyc_to_ns((cyc - epoch_cyc) & mask, cd.mult, cd.shift); }
static void notrace update_sched_clock(void) { unsigned long flags; u32 cyc; u64 ns; cyc = read_sched_clock(); ns = cd.epoch_ns + cyc_to_ns((cyc - cd.epoch_cyc) & sched_clock_mask, cd.mult, cd.shift); raw_local_irq_save(flags); cd.epoch_cyc = cyc; smp_wmb(); cd.epoch_ns = ns; smp_wmb(); cd.epoch_cyc_copy = cyc; raw_local_irq_restore(flags); }
/* * Atomically update the sched_clock epoch. */ static void notrace update_sched_clock(void) { unsigned long flags; u32 cyc; u64 ns; cyc = read_sched_clock(); ns = cd.epoch_ns + cyc_to_ns((cyc - cd.epoch_cyc) & sched_clock_mask, cd.mult, cd.shift); /* * Write epoch_cyc and epoch_ns in a way that the update is * detectable in cyc_to_fixed_sched_clock(). */ raw_local_irq_save(flags); cd.epoch_cyc_copy = cyc; smp_wmb(); cd.epoch_ns = ns; smp_wmb(); cd.epoch_cyc = cyc; raw_local_irq_restore(flags); }