/*
* DFE: Measures the TSC frequency in Hz (64-bit) using the ACPI PM timer
*/
static uint64_t measure_tsc_frequency(void)
{
uint64_t tscStart;
uint64_t tscEnd;
uint64_t tscDelta = 0xffffffffffffffffULL;
unsigned long pollCount;
uint64_t retval = 0;
int i;
/* Time how many TSC ticks elapse in 30 msec using the 8254 PIT
* counter 2. We run this loop 3 times to make sure the cache
* is hot and we take the minimum delta from all of the runs.
* That is to say that we're biased towards measuring the minimum
* number of TSC ticks that occur while waiting for the timer to
* expire. That theoretically helps avoid inconsistencies when
* running under a VM if the TSC is not virtualized and the host
* steals time. The TSC is normally virtualized for VMware.
*/
for(i = 0; i < 10; ++i)
{
enable_PIT2();
set_PIT2_mode0(CALIBRATE_LATCH);
tscStart = rdtsc64();
pollCount = poll_PIT2_gate();
tscEnd = rdtsc64();
/* The poll loop must have run at least a few times for accuracy */
if (pollCount <= 1)
continue;
/* The TSC must increment at LEAST once every millisecond.
* We should have waited exactly 30 msec so the TSC delta should
* be >= 30. Anything less and the processor is way too slow.
*/
if ((tscEnd - tscStart) <= CALIBRATE_TIME_MSEC)
continue;
// tscDelta = MIN(tscDelta, (tscEnd - tscStart))
if ( (tscEnd - tscStart) < tscDelta )
tscDelta = tscEnd - tscStart;
}
/* tscDelta is now the least number of TSC ticks the processor made in
* a timespan of 0.03 s (e.g. 30 milliseconds)
* Linux thus divides by 30 which gives the answer in kiloHertz because
* 1 / ms = kHz. But we're xnu and most of the rest of the code uses
* Hz so we need to convert our milliseconds to seconds. Since we're
* dividing by the milliseconds, we simply multiply by 1000.
*/
/* Unlike linux, we're not limited to 32-bit, but we do need to take care
* that we're going to multiply by 1000 first so we do need at least some
* arithmetic headroom. For now, 32-bit should be enough.
* Also unlike Linux, our compiler can do 64-bit integer arithmetic.
*/
if (tscDelta > (1ULL<<32))
retval = 0;
else
{
retval = tscDelta * 1000 / 30;
}
disable_PIT2();
return retval;
}
void
cpu_exit_wait(
int cpu)
{
cpu_data_t *cdp = cpu_datap(cpu);
boolean_t intrs_enabled;
uint64_t tsc_timeout;
/*
* Wait until the CPU indicates that it has stopped.
* Disable interrupts while the topo lock is held -- arguably
* this should always be done but in this instance it can lead to
* a timeout if long-running interrupt were to occur here.
*/
intrs_enabled = ml_set_interrupts_enabled(FALSE);
simple_lock(&x86_topo_lock);
/* Set a generous timeout of several seconds (in TSC ticks) */
tsc_timeout = rdtsc64() + (10ULL * 1000 * 1000 * 1000);
while ((cdp->lcpu.state != LCPU_HALT)
&& (cdp->lcpu.state != LCPU_OFF)
&& !cdp->lcpu.stopped) {
simple_unlock(&x86_topo_lock);
ml_set_interrupts_enabled(intrs_enabled);
cpu_pause();
if (rdtsc64() > tsc_timeout)
panic("cpu_exit_wait(%d) timeout", cpu);
ml_set_interrupts_enabled(FALSE);
simple_lock(&x86_topo_lock);
}
simple_unlock(&x86_topo_lock);
ml_set_interrupts_enabled(intrs_enabled);
}
/*
* rtc_nanotime_init:
*
* Intialize the nanotime info from the base time.
*/
static inline void
_rtc_nanotime_init(pal_rtc_nanotime_t *rntp, uint64_t base)
{
uint64_t tsc = rdtsc64();
_pal_rtc_nanotime_store(tsc, base, rntp->scale, rntp->shift, rntp);
}
static uint64_t
rtc_lapic_set_tsc_deadline_timer(uint64_t deadline, uint64_t now)
{
uint64_t delta;
uint64_t delta_tsc;
uint64_t tsc = rdtsc64();
uint64_t set = 0;
if (deadline > 0) {
/*
* Convert to TSC
*/
delta = deadline_to_decrementer(deadline, now);
set = now + delta;
delta_tsc = tmrCvt(delta, tscFCvtn2t);
lapic_set_tsc_deadline_timer(tsc + delta_tsc);
} else {
lapic_set_tsc_deadline_timer(0);
}
KERNEL_DEBUG_CONSTANT(
DECR_SET_TSC_DEADLINE | DBG_FUNC_NONE,
now, deadline,
tsc, lapic_get_tsc_deadline_timer(),
0);
return set;
}
uint64_t calc_rdtsc_overhead() {
uint32_t TRIALS = 1000000;
uint64_t* times = (uint64_t*) calloc(TRIALS,sizeof(uint64_t));
uint64_t start=0, end=0;
for(uint32_t i = 0; i < TRIALS; i++){
start = rdtsc64();
end = rdtsc64();
times[i] = (end -start)>0?(end - start):0;
}
gsl_sort_ulong((unsigned long*)times,1,TRIALS);
uint64_t median = gsl_stats_ulong_median_from_sorted_data ((unsigned long*)times,1,TRIALS);
#ifdef DEBUG
double mean = gsl_stats_ulong_mean(times, 1, TRIALS);
double sd = gsl_stats_ulong_sd(times, 1, TRIALS);
uint64_t max = times[TRIALS-1];
uint64_t min = times[0];
#endif
free(times);
#ifdef DEBUG
printf("| Median: %lu | Mean: %6.3f | Std Deviation: %6.3f | Min: %lu | Max: %lu |\n",median,mean,sd,min,max);
#endif
return median;
}
static void update_clock(void)
{
uint64_t delta = rdtsc64() - private_clock.ticks;
int secs;
private_clock.ticks += delta;
secs = (int) (delta / TICKS_PER_SEC);
private_clock.secs += secs;
delta -= (secs * TICKS_PER_SEC);
private_clock.usecs += (int) (delta / TICKS_PER_USEC);
if (private_clock.usecs > 1000000) {
private_clock.usecs -= 1000000;
private_clock.secs++;
}
}
/*
* rtc_clock_napped:
*
* Invoked from power management when we exit from a low C-State (>= C4)
* and the TSC has stopped counting. The nanotime data is updated according
* to the provided value which represents the new value for nanotime.
*/
void
rtc_clock_napped(uint64_t base, uint64_t tsc_base)
{
pal_rtc_nanotime_t *rntp = &pal_rtc_nanotime_info;
uint64_t oldnsecs;
uint64_t newnsecs;
uint64_t tsc;
assert(!ml_get_interrupts_enabled());
tsc = rdtsc64();
oldnsecs = rntp->ns_base + _rtc_tsc_to_nanoseconds(tsc - rntp->tsc_base, rntp);
newnsecs = base + _rtc_tsc_to_nanoseconds(tsc - tsc_base, rntp);
/*
* Only update the base values if time using the new base values
* is later than the time using the old base values.
*/
if (oldnsecs < newnsecs) {
_pal_rtc_nanotime_store(tsc_base, base, rntp->scale, rntp->shift, rntp);
rtc_nanotime_set_commpage(rntp);
}
}
static void gettimeofday_init(void)
{
int days, delta;
struct tm tm;
rtc_read_clock(&tm);
private_clock.ticks = rdtsc64();
/* Calculate the number of days in the year so far */
days = day_of_year(tm.tm_mon, tm.tm_mday, tm.tm_year + 1900);
delta = tm.tm_year - 70;
days += (delta * 365);
/* Figure leap years */
if (delta > 2)
days += (delta - 2) / 4;
private_clock.secs = (days * 86400) + (tm.tm_hour * 3600) +
(tm.tm_min * 60) + tm.tm_sec;
}
/*
* timeRDTSC()
* This routine sets up PIT counter 2 to count down 1/20 of a second.
* It pauses until the value is latched in the counter
* and then reads the time stamp counter to return to the caller.
*/
uint64_t timeRDTSC(void)
{
int attempts = 0;
uint64_t latchTime;
uint64_t saveTime,intermediate;
unsigned int timerValue, lastValue;
//boolean_t int_enabled;
/*
* Table of correction factors to account for
* - timer counter quantization errors, and
* - undercounts 0..5
*/
#define SAMPLE_CLKS_EXACT (((double) CLKNUM) / 20.0)
#define SAMPLE_CLKS_INT ((int) CLKNUM / 20)
#define SAMPLE_NSECS (2000000000LL)
#define SAMPLE_MULTIPLIER (((double)SAMPLE_NSECS)*SAMPLE_CLKS_EXACT)
#define ROUND64(x) ((uint64_t)((x) + 0.5))
uint64_t scale[6] = {
ROUND64(SAMPLE_MULTIPLIER/(double)(SAMPLE_CLKS_INT-0)),
ROUND64(SAMPLE_MULTIPLIER/(double)(SAMPLE_CLKS_INT-1)),
ROUND64(SAMPLE_MULTIPLIER/(double)(SAMPLE_CLKS_INT-2)),
ROUND64(SAMPLE_MULTIPLIER/(double)(SAMPLE_CLKS_INT-3)),
ROUND64(SAMPLE_MULTIPLIER/(double)(SAMPLE_CLKS_INT-4)),
ROUND64(SAMPLE_MULTIPLIER/(double)(SAMPLE_CLKS_INT-5))
};
//int_enabled = ml_set_interrupts_enabled(FALSE);
restart:
if (attempts >= 9) // increase to up to 9 attempts.
{
// This will flash-reboot. TODO: Use tscPanic instead.
printf("Timestamp counter calibation failed with %d attempts\n", attempts);
}
attempts++;
enable_PIT2(); // turn on PIT2
set_PIT2(0); // reset timer 2 to be zero
latchTime = rdtsc64(); // get the time stamp to time
latchTime = get_PIT2(&timerValue) - latchTime; // time how long this takes
set_PIT2(SAMPLE_CLKS_INT); // set up the timer for (almost) 1/20th a second
saveTime = rdtsc64(); // now time how long a 20th a second is...
get_PIT2(&lastValue);
get_PIT2(&lastValue); // read twice, first value may be unreliable
do {
intermediate = get_PIT2(&timerValue);
if (timerValue > lastValue)
{
// Timer wrapped
set_PIT2(0);
disable_PIT2();
goto restart;
}
lastValue = timerValue;
} while (timerValue > 5);
printf("timerValue %d\n",timerValue);
printf("intermediate 0x%016llx\n",intermediate);
printf("saveTime 0x%016llx\n",saveTime);
intermediate -= saveTime; // raw count for about 1/20 second
intermediate *= scale[timerValue]; // rescale measured time spent
intermediate /= SAMPLE_NSECS; // so its exactly 1/20 a second
intermediate += latchTime; // add on our save fudge
set_PIT2(0); // reset timer 2 to be zero
disable_PIT2(); // turn off PIT 2
//ml_set_interrupts_enabled(int_enabled);
return intermediate;
}
