static void arm_generic_timer_init_conversion_factors(uint32_t cntfrq)
{
fp_32_64_div_32_32(&cntpct_per_ms, cntfrq, 1000);
fp_32_64_div_32_32(&ms_per_cntpct, 1000, cntfrq);
fp_32_64_div_32_32(&us_per_cntpct, 1000 * 1000, cntfrq);
LTRACEF("cntpct_per_ms: %08x.%08x%08x\n", cntpct_per_ms.l0, cntpct_per_ms.l32, cntpct_per_ms.l64);
LTRACEF("ms_per_cntpct: %08x.%08x%08x\n", ms_per_cntpct.l0, ms_per_cntpct.l32, ms_per_cntpct.l64);
LTRACEF("us_per_cntpct: %08x.%08x%08x\n", us_per_cntpct.l0, us_per_cntpct.l32, us_per_cntpct.l64);
}
void arm_cortex_a9_timer_init(addr_t _scu_control_base, uint32_t freq)
{
scu_control_base = _scu_control_base;
arm_cortex_a9_timer_init_percpu(0);
/* save the timer frequency for later calculations */
timer_freq = freq;
/* precompute the conversion factor for global time to real time */
fp_32_64_div_32_32(&timer_freq_msec_conversion, timer_freq, 1000);
fp_32_64_div_32_32(&timer_freq_usec_conversion_inverse, 1000000, timer_freq);
fp_32_64_div_32_32(&timer_freq_msec_conversion_inverse, 1000, timer_freq);
}
static void calibrate_tsc(bool has_pvclock) {
ASSERT(arch_ints_disabled());
const uint64_t tsc_freq = has_pvclock ? pvclock_get_tsc_freq() : x86_lookup_tsc_freq();
if (tsc_freq != 0) {
tsc_ticks_per_ms = tsc_freq / 1000;
printf("TSC frequency: %" PRIu64 " ticks/ms\n", tsc_ticks_per_ms);
} else {
printf("Could not find TSC frequency: Calibrating TSC with %s\n",
clock_name[calibration_clock]);
uint32_t duration_ms[2] = {2, 4};
uint64_t best_time[2] = {
calibrate_tsc_count(static_cast<uint16_t>(duration_ms[0])),
calibrate_tsc_count(static_cast<uint16_t>(duration_ms[1]))};
while (best_time[0] >= best_time[1] && 2 * duration_ms[1] < MAX_TIMER_INTERVAL) {
duration_ms[0] = duration_ms[1];
duration_ms[1] *= 2;
best_time[0] = best_time[1];
best_time[1] = calibrate_tsc_count(static_cast<uint16_t>(duration_ms[1]));
}
ASSERT(best_time[0] < best_time[1]);
tsc_ticks_per_ms = (best_time[1] - best_time[0]) / (duration_ms[1] - duration_ms[0]);
printf("TSC calibrated: %" PRIu64 " ticks/ms\n", tsc_ticks_per_ms);
}
ASSERT(tsc_ticks_per_ms <= UINT32_MAX);
fp_32_64_div_32_32(&ns_per_tsc, 1000 * 1000, static_cast<uint32_t>(tsc_ticks_per_ms));
fp_32_64_div_32_32(&tsc_per_ns, static_cast<uint32_t>(tsc_ticks_per_ms), 1000 * 1000);
// Add 1ns to conservatively deal with rounding
ns_per_tsc_rounded_up = u32_mul_u64_fp32_64(1, ns_per_tsc) + 1;
LTRACEF("ns_per_tsc: %08x.%08x%08x\n", ns_per_tsc.l0, ns_per_tsc.l32, ns_per_tsc.l64);
}
void arm_cortex_a9_timer_init(addr_t _scu_control_base, uint32_t freq)
{
scu_control_base = _scu_control_base;
/* disable timer */
TIMREG(TIMER_CONTROL) = 0;
/* kill the watchdog */
TIMREG(WDOG_CONTROL) = 0;
/* ack any irqs that may be pending */
TIMREG(TIMER_ISR) = 1;
/* save the timer frequency for later calculations */
timer_freq = freq;
/* precompute the conversion factor for global time to real time */
fp_32_64_div_32_32(&timer_freq_msec_conversion, timer_freq, 1000);
fp_32_64_div_32_32(&timer_freq_usec_conversion_inverse, 1000000, timer_freq);
fp_32_64_div_32_32(&timer_freq_msec_conversion_inverse, 1000, timer_freq);
register_int_handler(CPU_PRIV_TIMER_INT, &platform_tick, NULL);
unmask_interrupt(CPU_PRIV_TIMER_INT);
}
static void set_pit_frequency(uint32_t frequency) {
uint32_t count, remainder;
/* figure out the correct pit_divisor for the desired frequency */
if (frequency <= 18) {
count = 0xffff;
} else if (frequency >= INTERNAL_FREQ) {
count = 1;
} else {
count = INTERNAL_FREQ_3X / frequency;
remainder = INTERNAL_FREQ_3X % frequency;
if (remainder >= INTERNAL_FREQ_3X / 2) {
count += 1;
}
count /= 3;
remainder = count % 3;
if (remainder >= 1) {
count += 1;
}
}
pit_divisor = count & 0xffff;
/*
* funky math that i don't feel like explaining. essentially 32.32 fixed
* point representation of the configured timer delta.
*/
fp_32_64_div_32_32(&us_per_pit, 1000 * 1000 * 3 * count, INTERNAL_FREQ_3X);
// Add 1us to the PIT tick rate to deal with rounding
ns_per_pit_rounded_up = (u32_mul_u64_fp32_64(1, us_per_pit) + 1) * 1000;
//dprintf(DEBUG, "set_pit_frequency: pit_divisor=%04x\n", pit_divisor);
/*
* setup the Programmable Interval Timer
* timer 0, mode 2, binary counter, LSB followed by MSB
*/
outp(I8253_CONTROL_REG, 0x34);
outp(I8253_DATA_REG, static_cast<uint8_t>(pit_divisor)); // LSB
outp(I8253_DATA_REG, static_cast<uint8_t>(pit_divisor >> 8)); // MSB
}
static void pc_init_timer(uint level) {
const struct x86_model_info* cpu_model = x86_get_model();
constant_tsc = false;
if (x86_vendor == X86_VENDOR_INTEL) {
/* This condition taken from Intel 3B 17.15 (Time-Stamp Counter). This
* is the negation of the non-Constant TSC section, since the Constant
* TSC section is incomplete (the behavior is architectural going
* forward, and modern CPUs are not on the list). */
constant_tsc = !((cpu_model->family == 0x6 && cpu_model->model == 0x9) ||
(cpu_model->family == 0x6 && cpu_model->model == 0xd) ||
(cpu_model->family == 0xf && cpu_model->model < 0x3));
}
invariant_tsc = x86_feature_test(X86_FEATURE_INVAR_TSC);
bool has_pvclock = pvclock_is_present();
if (has_pvclock) {
zx_status_t status = pvclock_init();
if (status == ZX_OK) {
invariant_tsc = pvclock_is_stable();
} else {
has_pvclock = false;
}
}
bool has_hpet = hpet_is_present();
if (has_hpet) {
calibration_clock = CLOCK_HPET;
const uint64_t hpet_ms_rate = hpet_ticks_per_ms();
ASSERT(hpet_ms_rate <= UINT32_MAX);
printf("HPET frequency: %" PRIu64 " ticks/ms\n", hpet_ms_rate);
fp_32_64_div_32_32(&ns_per_hpet, 1000 * 1000, static_cast<uint32_t>(hpet_ms_rate));
// Add 1ns to conservatively deal with rounding
ns_per_hpet_rounded_up = u32_mul_u64_fp32_64(1, ns_per_hpet) + 1;
} else {
calibration_clock = CLOCK_PIT;
}
const char* force_wallclock = cmdline_get("kernel.wallclock");
bool use_invariant_tsc = invariant_tsc && (!force_wallclock || !strcmp(force_wallclock, "tsc"));
use_tsc_deadline = use_invariant_tsc &&
x86_feature_test(X86_FEATURE_TSC_DEADLINE);
if (!use_tsc_deadline) {
calibrate_apic_timer();
}
if (use_invariant_tsc) {
calibrate_tsc(has_pvclock);
// Program PIT in the software strobe configuration, but do not load
// the count. This will pause the PIT.
outp(I8253_CONTROL_REG, 0x38);
wall_clock = CLOCK_TSC;
} else {
if (constant_tsc || invariant_tsc) {
// Calibrate the TSC even though it's not as good as we want, so we
// can still let folks still use it for cheap timing.
calibrate_tsc(has_pvclock);
}
if (has_hpet && (!force_wallclock || !strcmp(force_wallclock, "hpet"))) {
wall_clock = CLOCK_HPET;
hpet_set_value(0);
hpet_enable();
} else {
if (force_wallclock && strcmp(force_wallclock, "pit")) {
panic("Could not satisfy kernel.wallclock choice\n");
}
wall_clock = CLOCK_PIT;
set_pit_frequency(1000); // ~1ms granularity
uint32_t irq = apic_io_isa_to_global(ISA_IRQ_PIT);
zx_status_t status = register_int_handler(irq, &pit_timer_tick, NULL);
DEBUG_ASSERT(status == ZX_OK);
unmask_interrupt(irq);
}
}
printf("timer features: constant_tsc %d invariant_tsc %d tsc_deadline %d\n",
constant_tsc, invariant_tsc, use_tsc_deadline);
printf("Using %s as wallclock\n", clock_name[wall_clock]);
}
static void calibrate_apic_timer(void) {
ASSERT(arch_ints_disabled());
const uint64_t apic_freq = x86_lookup_core_crystal_freq();
if (apic_freq != 0) {
ASSERT(apic_freq / 1000 <= UINT32_MAX);
apic_ticks_per_ms = static_cast<uint32_t>(apic_freq / 1000);
apic_divisor = 1;
fp_32_64_div_32_32(&apic_ticks_per_ns, apic_ticks_per_ms, 1000 * 1000);
printf("APIC frequency: %" PRIu32 " ticks/ms\n", apic_ticks_per_ms);
return;
}
printf("Could not find APIC frequency: Calibrating APIC with %s\n",
clock_name[calibration_clock]);
apic_divisor = 1;
outer:
while (apic_divisor != 0) {
uint32_t best_time[2] = {UINT32_MAX, UINT32_MAX};
const uint16_t duration_ms[2] = {2, 4};
for (int trial = 0; trial < 2; ++trial) {
for (int tries = 0; tries < 3; ++tries) {
switch (calibration_clock) {
case CLOCK_HPET:
hpet_calibration_cycle_preamble();
break;
case CLOCK_PIT:
pit_calibration_cycle_preamble(duration_ms[trial]);
break;
default:
PANIC_UNIMPLEMENTED;
}
// Setup APIC timer to count down with interrupt masked
zx_status_t status = apic_timer_set_oneshot(
UINT32_MAX,
apic_divisor,
true);
ASSERT(status == ZX_OK);
switch (calibration_clock) {
case CLOCK_HPET:
hpet_calibration_cycle(duration_ms[trial]);
break;
case CLOCK_PIT:
pit_calibration_cycle(duration_ms[trial]);
break;
default:
PANIC_UNIMPLEMENTED;
}
uint32_t apic_ticks = UINT32_MAX - apic_timer_current_count();
if (apic_ticks < best_time[trial]) {
best_time[trial] = apic_ticks;
}
LTRACEF("Calibration trial %d found %u ticks/ms\n",
tries, apic_ticks);
switch (calibration_clock) {
case CLOCK_HPET:
hpet_calibration_cycle_cleanup();
break;
case CLOCK_PIT:
pit_calibration_cycle_cleanup();
break;
default:
PANIC_UNIMPLEMENTED;
}
}
// If the APIC ran out of time every time, try again with a higher
// divisor
if (best_time[trial] == UINT32_MAX) {
apic_divisor = static_cast<uint8_t>(apic_divisor * 2);
goto outer;
}
}
apic_ticks_per_ms = (best_time[1] - best_time[0]) / (duration_ms[1] - duration_ms[0]);
fp_32_64_div_32_32(&apic_ticks_per_ns, apic_ticks_per_ms, 1000 * 1000);
break;
}
ASSERT(apic_divisor != 0);
printf("APIC timer calibrated: %" PRIu32 " ticks/ms, divisor %d\n",
apic_ticks_per_ms, apic_divisor);
}
