/*
* Calculate the absolute or boot-relative time from the
* machine-specific fast timecounter and the published timehands
* structure read from the shared page.
*
* The lockless reading scheme is similar to the one used to read the
* in-kernel timehands, see sys/kern/kern_tc.c:binuptime(). This code
* is based on the kernel implementation.
*/
static int
binuptime(struct bintime *bt, struct vdso_timekeep *tk, int abs)
{
struct vdso_timehands *th;
uint32_t curr, gen;
u_int delta;
int error;
do {
if (!tk->tk_enabled)
return (ENOSYS);
curr = atomic_load_acq_32(&tk->tk_current);
th = &tk->tk_th[curr];
gen = atomic_load_acq_32(&th->th_gen);
*bt = th->th_offset;
error = tc_delta(th, &delta);
if (error == EAGAIN)
continue;
if (error != 0)
return (error);
bintime_addx(bt, th->th_scale * delta);
if (abs)
bintime_add(bt, &th->th_boottime);
/*
* Ensure that the load of th_offset is completed
* before the load of th_gen.
*/
atomic_thread_fence_acq();
} while (curr != tk->tk_current || gen == 0 || gen != th->th_gen);
return (0);
}
static int
binuptime(struct bintime *bt, struct vdso_timekeep *tk, int abs)
{
struct vdso_timehands *th;
uint32_t curr, gen;
do {
if (!tk->tk_enabled)
return (ENOSYS);
/*
* XXXKIB. The load of tk->tk_current should use
* atomic_load_acq_32 to provide load barrier. But
* since tk points to r/o mapped page, x86
* implementation of atomic_load_acq faults.
*/
curr = tk->tk_current;
rmb();
th = &tk->tk_th[curr];
if (th->th_algo != VDSO_TH_ALGO_1)
return (ENOSYS);
gen = th->th_gen;
*bt = th->th_offset;
bintime_addx(bt, th->th_scale * tc_delta(th));
if (abs)
bintime_add(bt, &th->th_boottime);
/*
* Barrier for load of both tk->tk_current and th->th_gen.
*/
rmb();
} while (curr != tk->tk_current || gen == 0 || gen != th->th_gen);
return (0);
}
/*
* Calculate the absolute or boot-relative time from the
* machine-specific fast timecounter and the published timehands
* structure read from the shared page.
*
* The lockless reading scheme is similar to the one used to read the
* in-kernel timehands, see sys/kern/kern_tc.c:binuptime(). This code
* is based on the kernel implementation.
*/
static int
binuptime(struct bintime *bt, struct vdso_timekeep *tk, int abs)
{
struct vdso_timehands *th;
uint32_t curr, gen;
do {
if (!tk->tk_enabled)
return (ENOSYS);
curr = atomic_load_acq_32(&tk->tk_current);
th = &tk->tk_th[curr];
if (th->th_algo != VDSO_TH_ALGO_1)
return (ENOSYS);
gen = atomic_load_acq_32(&th->th_gen);
*bt = th->th_offset;
bintime_addx(bt, th->th_scale * tc_delta(th));
if (abs)
bintime_add(bt, &th->th_boottime);
/*
* Ensure that the load of th_offset is completed
* before the load of th_gen.
*/
atomic_thread_fence_acq();
} while (curr != tk->tk_current || gen == 0 || gen != th->th_gen);
return (0);
}
/*
* Feed-forward clock absolute time. This should be the preferred way to read
* the feed-forward clock for "wall-clock" type time. The flags allow to compose
* various flavours of absolute time (e.g. with or without leap seconds taken
* into account). If valid pointers are provided, the ffcounter value and an
* upper bound on clock error associated with the bintime are provided.
* NOTE: use ffclock_convert_abs() to differ the conversion of a ffcounter value
* read earlier.
*/
void
ffclock_abstime(ffcounter *ffcount, struct bintime *bt,
struct bintime *error_bound, uint32_t flags)
{
struct ffclock_estimate cest;
ffcounter ffc;
ffcounter update_ffcount;
ffcounter ffdelta_error;
/* Get counter and corresponding time. */
if ((flags & FFCLOCK_FAST) == FFCLOCK_FAST)
ffclock_last_tick(&ffc, bt, flags);
else {
ffclock_read_counter(&ffc);
ffclock_convert_abs(ffc, bt, flags);
}
/* Current ffclock estimate, use update_ffcount as generation number. */
do {
update_ffcount = ffclock_estimate.update_ffcount;
bcopy(&ffclock_estimate, &cest, sizeof(struct ffclock_estimate));
} while (update_ffcount != ffclock_estimate.update_ffcount);
/*
* Leap second adjustment. Total as seen by synchronisation algorithm
* since it started. cest.leapsec_next is the ffcounter prediction of
* when the next leapsecond occurs.
*/
if ((flags & FFCLOCK_LEAPSEC) == FFCLOCK_LEAPSEC) {
bt->sec -= cest.leapsec_total;
if (ffc > cest.leapsec_next)
bt->sec -= cest.leapsec;
}
/* Boot time adjustment, for uptime/monotonic clocks. */
if ((flags & FFCLOCK_UPTIME) == FFCLOCK_UPTIME) {
bintime_sub(bt, &ffclock_boottime);
}
/* Compute error bound if a valid pointer has been passed. */
if (error_bound) {
ffdelta_error = ffc - cest.update_ffcount;
ffclock_convert_diff(ffdelta_error, error_bound);
/* 18446744073709 = int(2^64/1e12), err_bound_rate in [ps/s] */
bintime_mul(error_bound, cest.errb_rate *
(uint64_t)18446744073709LL);
/* 18446744073 = int(2^64 / 1e9), since err_abs in [ns] */
bintime_addx(error_bound, cest.errb_abs *
(uint64_t)18446744073LL);
}
if (ffcount)
*ffcount = ffc;
}
void
binuptime(struct bintime *bt)
{
struct timehands *th;
u_int gen;
do {
th = timehands;
gen = th->th_generation;
*bt = th->th_offset;
bintime_addx(bt, th->th_scale * tc_delta(th));
} while (gen == 0 || gen != th->th_generation);
}
/*
* Initialize the next struct timehands in the ring and make
* it the active timehands. Along the way we might switch to a different
* timecounter and/or do seconds processing in NTP. Slightly magic.
*/
void
tc_windup(void)
{
struct bintime bt;
struct timehands *th, *tho;
u_int64_t scale;
u_int delta, ncount, ogen;
int i;
#ifdef leapsecs
time_t t;
#endif
/*
* Make the next timehands a copy of the current one, but do not
* overwrite the generation or next pointer. While we update
* the contents, the generation must be zero.
*/
tho = timehands;
th = tho->th_next;
ogen = th->th_generation;
th->th_generation = 0;
bcopy(tho, th, offsetof(struct timehands, th_generation));
/*
* Capture a timecounter delta on the current timecounter and if
* changing timecounters, a counter value from the new timecounter.
* Update the offset fields accordingly.
*/
delta = tc_delta(th);
if (th->th_counter != timecounter)
ncount = timecounter->tc_get_timecount(timecounter);
else
ncount = 0;
th->th_offset_count += delta;
th->th_offset_count &= th->th_counter->tc_counter_mask;
bintime_addx(&th->th_offset, th->th_scale * delta);
#ifdef notyet
/*
* Hardware latching timecounters may not generate interrupts on
* PPS events, so instead we poll them. There is a finite risk that
* the hardware might capture a count which is later than the one we
* got above, and therefore possibly in the next NTP second which might
* have a different rate than the current NTP second. It doesn't
* matter in practice.
*/
if (tho->th_counter->tc_poll_pps)
tho->th_counter->tc_poll_pps(tho->th_counter);
#endif
/*
* Deal with NTP second processing. The for loop normally
* iterates at most once, but in extreme situations it might
* keep NTP sane if timeouts are not run for several seconds.
* At boot, the time step can be large when the TOD hardware
* has been read, so on really large steps, we call
* ntp_update_second only twice. We need to call it twice in
* case we missed a leap second.
*/
bt = th->th_offset;
bintime_add(&bt, &boottimebin);
i = bt.sec - tho->th_microtime.tv_sec;
if (i > LARGE_STEP)
i = 2;
for (; i > 0; i--)
ntp_update_second(&th->th_adjustment, &bt.sec);
/* Update the UTC timestamps used by the get*() functions. */
/* XXX shouldn't do this here. Should force non-`get' versions. */
bintime2timeval(&bt, &th->th_microtime);
bintime2timespec(&bt, &th->th_nanotime);
/* Now is a good time to change timecounters. */
if (th->th_counter != timecounter) {
th->th_counter = timecounter;
th->th_offset_count = ncount;
}
/*-
* Recalculate the scaling factor. We want the number of 1/2^64
* fractions of a second per period of the hardware counter, taking
* into account the th_adjustment factor which the NTP PLL/adjtime(2)
* processing provides us with.
*
* The th_adjustment is nanoseconds per second with 32 bit binary
* fraction and we want 64 bit binary fraction of second:
*
* x = a * 2^32 / 10^9 = a * 4.294967296
*
* The range of th_adjustment is +/- 5000PPM so inside a 64bit int
* we can only multiply by about 850 without overflowing, but that
* leaves suitably precise fractions for multiply before divide.
*
* Divide before multiply with a fraction of 2199/512 results in a
* systematic undercompensation of 10PPM of th_adjustment. On a
* 5000PPM adjustment this is a 0.05PPM error. This is acceptable.
*
* We happily sacrifice the lowest of the 64 bits of our result
* to the goddess of code clarity.
*
*/
scale = (u_int64_t)1 << 63;
scale += (th->th_adjustment / 1024) * 2199;
scale /= th->th_counter->tc_frequency;
th->th_scale = scale * 2;
/*
* Now that the struct timehands is again consistent, set the new
* generation number, making sure to not make it zero.
*/
if (++ogen == 0)
ogen = 1;
th->th_generation = ogen;
/* Go live with the new struct timehands. */
time_second = th->th_microtime.tv_sec;
time_uptime = th->th_offset.sec;
timehands = th;
}
void
pps_event(struct pps_state *pps, int event)
{
struct bintime bt;
struct timespec ts, *tsp, *osp;
u_int tcount, *pcount;
int foff, fhard;
pps_seq_t *pseq;
KASSERT(pps != NULL, ("NULL pps pointer in pps_event"));
/* If the timecounter was wound up underneath us, bail out. */
if (pps->capgen == 0 || pps->capgen != pps->capth->th_generation)
return;
/* Things would be easier with arrays. */
if (event == PPS_CAPTUREASSERT) {
tsp = &pps->ppsinfo.assert_timestamp;
osp = &pps->ppsparam.assert_offset;
foff = pps->ppsparam.mode & PPS_OFFSETASSERT;
fhard = pps->kcmode & PPS_CAPTUREASSERT;
pcount = &pps->ppscount[0];
pseq = &pps->ppsinfo.assert_sequence;
} else {
tsp = &pps->ppsinfo.clear_timestamp;
osp = &pps->ppsparam.clear_offset;
foff = pps->ppsparam.mode & PPS_OFFSETCLEAR;
fhard = pps->kcmode & PPS_CAPTURECLEAR;
pcount = &pps->ppscount[1];
pseq = &pps->ppsinfo.clear_sequence;
}
/*
* If the timecounter changed, we cannot compare the count values, so
* we have to drop the rest of the PPS-stuff until the next event.
*/
if (pps->ppstc != pps->capth->th_counter) {
pps->ppstc = pps->capth->th_counter;
*pcount = pps->capcount;
pps->ppscount[2] = pps->capcount;
return;
}
/* Convert the count to a timespec. */
tcount = pps->capcount - pps->capth->th_offset_count;
tcount &= pps->capth->th_counter->tc_counter_mask;
bt = pps->capth->th_offset;
bintime_addx(&bt, pps->capth->th_scale * tcount);
bintime_add(&bt, &boottimebin);
bintime2timespec(&bt, &ts);
/* If the timecounter was wound up underneath us, bail out. */
if (pps->capgen != pps->capth->th_generation)
return;
*pcount = pps->capcount;
(*pseq)++;
*tsp = ts;
if (foff) {
timespecadd(tsp, osp);
if (tsp->tv_nsec < 0) {
tsp->tv_nsec += 1000000000;
tsp->tv_sec -= 1;
}
}
#ifdef PPS_SYNC
if (fhard) {
uint64_t scale;
/*
* Feed the NTP PLL/FLL.
* The FLL wants to know how many (hardware) nanoseconds
* elapsed since the previous event.
*/
tcount = pps->capcount - pps->ppscount[2];
pps->ppscount[2] = pps->capcount;
tcount &= pps->capth->th_counter->tc_counter_mask;
scale = (uint64_t)1 << 63;
scale /= pps->capth->th_counter->tc_frequency;
scale *= 2;
bt.sec = 0;
bt.frac = 0;
bintime_addx(&bt, scale * tcount);
bintime2timespec(&bt, &ts);
hardpps(tsp, ts.tv_nsec + 1000000000 * ts.tv_sec);
}
#endif
}
