void
bintime(struct bintime *bt)
{
binuptime(bt);
bintime_add(bt, &boottimebin);
}
/*
* Step our concept of UTC. This is done by modifying our estimate of
* when we booted.
* XXX: not locked.
*/
void
tc_setclock(struct timespec *ts)
{
struct timespec tbef, taft;
struct bintime bt, bt2;
cpu_tick_calibrate(1);
nanotime(&tbef);
timespec2bintime(ts, &bt);
binuptime(&bt2);
bintime_sub(&bt, &bt2);
bintime_add(&bt2, &boottimebin);
boottimebin = bt;
bintime2timeval(&bt, &boottime);
/* XXX fiddle all the little crinkly bits around the fiords... */
tc_windup();
nanotime(&taft);
if (timestepwarnings) {
log(LOG_INFO,
"Time stepped from %jd.%09ld to %jd.%09ld (%jd.%09ld)\n",
(intmax_t)tbef.tv_sec, tbef.tv_nsec,
(intmax_t)taft.tv_sec, taft.tv_nsec,
(intmax_t)ts->tv_sec, ts->tv_nsec);
}
cpu_tick_calibrate(1);
}
void
microuptime(struct timeval *tvp)
{
struct bintime bt;
binuptime(&bt);
bintime2timeval(&bt, tvp);
}
void
nanouptime(struct timespec *tsp)
{
struct bintime bt;
binuptime(&bt);
bintime2timespec(&bt, tsp);
}
static __inline hrtime_t
cyc_gethrtime(void)
{
struct bintime bt;
binuptime(&bt);
return ((hrtime_t)bt.sec * NANOSEC +
(((uint64_t)NANOSEC * (uint32_t)(bt.frac >> 32)) >> 32));
}
uint64_t
lockstat_nsecs(void)
{
struct bintime bt;
uint64_t ns;
binuptime(&bt);
ns = bt.sec * (uint64_t)1000000000;
ns += ((uint64_t)1000000000 * (uint32_t)(bt.frac >> 32)) >> 32;
return (ns);
}
uint64_t
_lockstat_nsecs(struct lock_object *lo)
{
struct bintime bt;
uint64_t ns;
if ((lo->lo_flags & LO_NOPROFILE) != 0)
return (0);
binuptime(&bt);
ns = bt.sec * (uint64_t)1000000000;
ns += ((uint64_t)1000000000 * (uint32_t)(bt.frac >> 32)) >> 32;
return (ns);
}
/*
* Return the current time in 100 nanosecond units
*/
UINT64
AcpiOsGetTimer(void)
{
struct bintime bt;
UINT64 t;
/* XXX During early boot there is no (decent) timer available yet. */
KASSERT(cold == 0, ("acpi: timer op not yet supported during boot"));
binuptime(&bt);
t = (uint64_t)bt.sec * 10000000;
t += ((uint64_t)10000000 * (uint32_t)(bt.frac >> 32)) >> 32;
return (t);
}
int
__vdso_gettimeofday(struct timeval *tv, struct timezone *tz)
{
struct bintime bt;
int error;
if (tz != NULL)
return (ENOSYS);
if (tk == NULL) {
error = __vdso_gettimekeep(&tk);
if (error != 0 || tk == NULL)
return (ENOSYS);
}
if (tk->tk_ver != VDSO_TK_VER_CURR)
return (ENOSYS);
error = binuptime(&bt, tk, 1);
if (error != 0)
return (error);
bintime2timeval(&bt, tv);
return (0);
}
int
__vdso_clock_gettime(clockid_t clock_id, struct timespec *ts)
{
struct bintime bt;
int abs, error;
if (tk == NULL) {
error = _elf_aux_info(AT_TIMEKEEP, &tk, sizeof(tk));
if (error != 0 || tk == NULL)
return (ENOSYS);
}
if (tk->tk_ver != VDSO_TK_VER_CURR)
return (ENOSYS);
switch (clock_id) {
case CLOCK_REALTIME:
case CLOCK_REALTIME_PRECISE:
case CLOCK_REALTIME_FAST:
case CLOCK_SECOND:
abs = 1;
break;
case CLOCK_MONOTONIC:
case CLOCK_MONOTONIC_PRECISE:
case CLOCK_MONOTONIC_FAST:
case CLOCK_UPTIME:
case CLOCK_UPTIME_PRECISE:
case CLOCK_UPTIME_FAST:
abs = 0;
break;
default:
return (ENOSYS);
}
error = binuptime(&bt, tk, abs);
if (error != 0)
return (error);
bintime2timespec(&bt, ts);
if (clock_id == CLOCK_SECOND)
ts->tv_nsec = 0;
return (0);
}
/*
* Step our concept of UTC. This is done by modifying our estimate of
* when we booted.
* XXX: not locked.
*/
void
tc_setclock(struct timespec *ts)
{
struct timespec ts2;
struct bintime bt, bt2;
binuptime(&bt2);
timespec2bintime(ts, &bt);
bintime_sub(&bt, &bt2);
bintime_add(&bt2, &boottimebin);
boottimebin = bt;
bintime2timeval(&bt, &boottime);
/* XXX fiddle all the little crinkly bits around the fiords... */
tc_windup();
if (timestepwarnings) {
bintime2timespec(&bt2, &ts2);
log(LOG_INFO, "Time stepped from %ld.%09ld to %ld.%09ld\n",
(long)ts2.tv_sec, ts2.tv_nsec,
(long)ts->tv_sec, ts->tv_nsec);
}
}
/*
* Software (low priority) clock interrupt.
* Run periodic events from timeout queue.
*/
void
softclock(void *dummy)
{
struct callout *c;
struct callout_tailq *bucket;
int curticks;
int steps; /* #steps since we last allowed interrupts */
int depth;
int mpcalls;
int mtxcalls;
int gcalls;
#ifdef DIAGNOSTIC
struct bintime bt1, bt2;
struct timespec ts2;
static uint64_t maxdt = 36893488147419102LL; /* 2 msec */
static timeout_t *lastfunc;
#endif
#ifndef MAX_SOFTCLOCK_STEPS
#define MAX_SOFTCLOCK_STEPS 100 /* Maximum allowed value of steps. */
#endif /* MAX_SOFTCLOCK_STEPS */
mpcalls = 0;
mtxcalls = 0;
gcalls = 0;
depth = 0;
steps = 0;
mtx_lock_spin(&callout_lock);
while (softticks != ticks) {
softticks++;
/*
* softticks may be modified by hard clock, so cache
* it while we work on a given bucket.
*/
curticks = softticks;
bucket = &callwheel[curticks & callwheelmask];
c = TAILQ_FIRST(bucket);
while (c) {
depth++;
if (c->c_time != curticks) {
c = TAILQ_NEXT(c, c_links.tqe);
++steps;
if (steps >= MAX_SOFTCLOCK_STEPS) {
nextsoftcheck = c;
/* Give interrupts a chance. */
mtx_unlock_spin(&callout_lock);
; /* nothing */
mtx_lock_spin(&callout_lock);
c = nextsoftcheck;
steps = 0;
}
} else {
void (*c_func)(void *);
void *c_arg;
struct mtx *c_mtx;
int c_flags;
nextsoftcheck = TAILQ_NEXT(c, c_links.tqe);
TAILQ_REMOVE(bucket, c, c_links.tqe);
c_func = c->c_func;
c_arg = c->c_arg;
c_mtx = c->c_mtx;
c_flags = c->c_flags;
if (c->c_flags & CALLOUT_LOCAL_ALLOC) {
c->c_func = NULL;
c->c_flags = CALLOUT_LOCAL_ALLOC;
SLIST_INSERT_HEAD(&callfree, c,
c_links.sle);
curr_callout = NULL;
} else {
c->c_flags =
(c->c_flags & ~CALLOUT_PENDING);
curr_callout = c;
}
curr_cancelled = 0;
mtx_unlock_spin(&callout_lock);
if (c_mtx != NULL) {
mtx_lock(c_mtx);
/*
* The callout may have been cancelled
* while we switched locks.
*/
if (curr_cancelled) {
mtx_unlock(c_mtx);
goto skip;
}
/* The callout cannot be stopped now. */
curr_cancelled = 1;
if (c_mtx == &Giant) {
gcalls++;
CTR3(KTR_CALLOUT,
"callout %p func %p arg %p",
c, c_func, c_arg);
} else {
mtxcalls++;
CTR3(KTR_CALLOUT, "callout mtx"
" %p func %p arg %p",
c, c_func, c_arg);
}
} else {
mpcalls++;
CTR3(KTR_CALLOUT,
"callout mpsafe %p func %p arg %p",
c, c_func, c_arg);
}
#ifdef DIAGNOSTIC
binuptime(&bt1);
#endif
#ifdef MAXHE_TODO
THREAD_NO_SLEEPING();
c_func(c_arg);
THREAD_SLEEPING_OK();
#else
c_func(c_arg);
#endif // MAXHE_TODO
#ifdef DIAGNOSTIC
binuptime(&bt2);
bintime_sub(&bt2, &bt1);
if (bt2.frac > maxdt) {
if (lastfunc != c_func ||
bt2.frac > maxdt * 2) {
bintime2timespec(&bt2, &ts2);
printf(
"Expensive timeout(9) function: %p(%p) %jd.%09ld s\n",
c_func, c_arg,
(intmax_t)ts2.tv_sec,
ts2.tv_nsec);
}
maxdt = bt2.frac;
lastfunc = c_func;
}
#endif
if ((c_flags & CALLOUT_RETURNUNLOCKED) == 0)
mtx_unlock(c_mtx);
skip:
mtx_lock_spin(&callout_lock);
curr_callout = NULL;
if (callout_wait) {
/*
* There is someone waiting
* for the callout to complete.
*/
callout_wait = 0;
mtx_unlock_spin(&callout_lock);
wakeup(&callout_wait);
mtx_lock_spin(&callout_lock);
}
steps = 0;
c = nextsoftcheck;
}
}
}
#ifdef MAXHE_TODO
avg_depth += (depth * 1000 - avg_depth) >> 8;
avg_mpcalls += (mpcalls * 1000 - avg_mpcalls) >> 8;
avg_mtxcalls += (mtxcalls * 1000 - avg_mtxcalls) >> 8;
avg_gcalls += (gcalls * 1000 - avg_gcalls) >> 8;
#endif // MAXHE_TODO
nextsoftcheck = NULL;
mtx_unlock_spin(&callout_lock);
}
void
smp_init_secondary(u_int32_t cpuid)
{
if (cpuid >= MAXCPU)
panic ("cpu id exceeds MAXCPU\n");
/* tlb init */
R4K_SetWIRED(0);
R4K_TLBFlush(num_tlbentries);
R4K_SetWIRED(VMWIRED_ENTRIES);
MachSetPID(0);
Mips_SyncCache();
mips_cp0_status_write(0);
while (!aps_ready)
;
mips_sync(); mips_sync();
/* Initialize curthread. */
KASSERT(PCPU_GET(idlethread) != NULL, ("no idle thread"));
PCPU_SET(curthread, PCPU_GET(idlethread));
mtx_lock_spin(&ap_boot_mtx);
smp_cpus++;
CTR1(KTR_SMP, "SMP: AP CPU #%d Launched", PCPU_GET(cpuid));
/* Build our map of 'other' CPUs. */
PCPU_SET(other_cpus, all_cpus & ~PCPU_GET(cpumask));
printf("SMP: AP CPU #%d Launched!\n", PCPU_GET(cpuid));
if (smp_cpus == mp_ncpus) {
smp_started = 1;
smp_active = 1;
}
mtx_unlock_spin(&ap_boot_mtx);
while (smp_started == 0)
; /* nothing */
/* Enable Interrupt */
mips_cp0_status_write(SR_INT_ENAB);
/* ok, now grab sched_lock and enter the scheduler */
mtx_lock_spin(&sched_lock);
/*
* Correct spinlock nesting. The idle thread context that we are
* borrowing was created so that it would start out with a single
* spin lock (sched_lock) held in fork_trampoline(). Since we've
* explicitly acquired locks in this function, the nesting count
* is now 2 rather than 1. Since we are nested, calling
* spinlock_exit() will simply adjust the counts without allowing
* spin lock using code to interrupt us.
*/
spinlock_exit();
KASSERT(curthread->td_md.md_spinlock_count == 1, ("invalid count"));
binuptime(PCPU_PTR(switchtime));
PCPU_SET(switchticks, ticks);
/* kick off the clock on this cpu */
mips_start_timer();
cpu_throw(NULL, choosethread()); /* doesn't return */
panic("scheduler returned us to %s", __func__);
}
static void
softclock_call_cc(struct callout *c, struct callout_cpu *cc, int *mpcalls,
int *lockcalls, int *gcalls)
{
void (*c_func)(void *);
void *c_arg;
int c_flags;
#ifdef DIAGNOSTIC
struct bintime bt1, bt2;
struct timespec ts2;
static uint64_t maxdt = 36893488147419102LL; /* 2 msec */
static timeout_t *lastfunc;
#endif
KASSERT((c->c_flags & (CALLOUT_PENDING | CALLOUT_ACTIVE)) ==
(CALLOUT_PENDING | CALLOUT_ACTIVE),
("softclock_call_cc: pend|act %p %x", c, c->c_flags));
c_func = c->c_func;
c_arg = c->c_arg;
c_flags = c->c_flags;
if (c->c_flags & CALLOUT_LOCAL_ALLOC)
c->c_flags = CALLOUT_LOCAL_ALLOC;
else
c->c_flags &= ~CALLOUT_PENDING;
cc->cc_curr = c;
cc->cc_cancel = 0;
CC_UNLOCK(cc);
{
(*mpcalls)++;
CTR3(KTR_CALLOUT, "callout mpsafe %p func %p arg %p",
c, c_func, c_arg);
}
#ifdef DIAGNOSTIC
binuptime(&bt1);
#endif
c_func(c_arg);
#ifdef DIAGNOSTIC
binuptime(&bt2);
bintime_sub(&bt2, &bt1);
if (bt2.frac > maxdt) {
if (lastfunc != c_func || bt2.frac > maxdt * 2) {
bintime2timespec(&bt2, &ts2);
printf(
"Expensive timeout(9) function: %p(%p) %jd.%09ld s\n",
c_func, c_arg, (intmax_t)ts2.tv_sec, ts2.tv_nsec);
}
maxdt = bt2.frac;
lastfunc = c_func;
}
#endif
CTR1(KTR_CALLOUT, "callout %p finished", c);
CC_LOCK(cc);
KASSERT(cc->cc_curr == c, ("mishandled cc_curr"));
cc->cc_curr = NULL;
/*
* If the current callout is locally allocated (from
* timeout(9)) then put it on the freelist.
*
* Note: we need to check the cached copy of c_flags because
* if it was not local, then it's not safe to deref the
* callout pointer.
*/
KASSERT((c_flags & CALLOUT_LOCAL_ALLOC) == 0 ||
c->c_flags == CALLOUT_LOCAL_ALLOC,
("corrupted callout"));
if (c_flags & CALLOUT_LOCAL_ALLOC)
callout_cc_del(c, cc);
}
/*
* Software (low priority) clock interrupt.
* Run periodic events from timeout queue.
*/
void
softclock(void *dummy)
{
struct callout *c;
struct callout_tailq *bucket;
int curticks;
int steps; /* #steps since we last allowed interrupts */
int depth;
int mpcalls;
int gcalls;
int wakeup_cookie;
#ifdef DIAGNOSTIC
struct bintime bt1, bt2;
struct timespec ts2;
static uint64_t maxdt = 36893488147419102LL; /* 2 msec */
static timeout_t *lastfunc;
#endif
#ifndef MAX_SOFTCLOCK_STEPS
#define MAX_SOFTCLOCK_STEPS 100 /* Maximum allowed value of steps. */
#endif /* MAX_SOFTCLOCK_STEPS */
mpcalls = 0;
gcalls = 0;
depth = 0;
steps = 0;
mtx_lock_spin(&callout_lock);
while (softticks != ticks) {
softticks++;
/*
* softticks may be modified by hard clock, so cache
* it while we work on a given bucket.
*/
curticks = softticks;
bucket = &callwheel[curticks & callwheelmask];
c = TAILQ_FIRST(bucket);
while (c) {
depth++;
if (c->c_time != curticks) {
c = TAILQ_NEXT(c, c_links.tqe);
++steps;
if (steps >= MAX_SOFTCLOCK_STEPS) {
nextsoftcheck = c;
/* Give interrupts a chance. */
mtx_unlock_spin(&callout_lock);
; /* nothing */
mtx_lock_spin(&callout_lock);
c = nextsoftcheck;
steps = 0;
}
} else {
void (*c_func)(void *);
void *c_arg;
int c_flags;
nextsoftcheck = TAILQ_NEXT(c, c_links.tqe);
TAILQ_REMOVE(bucket, c, c_links.tqe);
c_func = c->c_func;
c_arg = c->c_arg;
c_flags = c->c_flags;
c->c_func = NULL;
if (c->c_flags & CALLOUT_LOCAL_ALLOC) {
c->c_flags = CALLOUT_LOCAL_ALLOC;
SLIST_INSERT_HEAD(&callfree, c,
c_links.sle);
} else {
c->c_flags =
(c->c_flags & ~CALLOUT_PENDING);
}
curr_callout = c;
mtx_unlock_spin(&callout_lock);
if (!(c_flags & CALLOUT_MPSAFE)) {
mtx_lock(&Giant);
gcalls++;
CTR1(KTR_CALLOUT, "callout %p", c_func);
} else {
mpcalls++;
CTR1(KTR_CALLOUT, "callout mpsafe %p",
c_func);
}
#ifdef DIAGNOSTIC
binuptime(&bt1);
mtx_lock(&dont_sleep_in_callout);
#endif
c_func(c_arg);
#ifdef DIAGNOSTIC
mtx_unlock(&dont_sleep_in_callout);
binuptime(&bt2);
bintime_sub(&bt2, &bt1);
if (bt2.frac > maxdt) {
if (lastfunc != c_func ||
bt2.frac > maxdt * 2) {
bintime2timespec(&bt2, &ts2);
printf(
"Expensive timeout(9) function: %p(%p) %jd.%09ld s\n",
c_func, c_arg,
(intmax_t)ts2.tv_sec,
ts2.tv_nsec);
}
maxdt = bt2.frac;
lastfunc = c_func;
}
#endif
if (!(c_flags & CALLOUT_MPSAFE))
mtx_unlock(&Giant);
mtx_lock_spin(&callout_lock);
curr_callout = NULL;
if (wakeup_needed) {
/*
* There might be someone waiting
* for the callout to complete.
*/
wakeup_cookie = wakeup_ctr;
mtx_unlock_spin(&callout_lock);
mtx_lock(&callout_wait_lock);
cv_broadcast(&callout_wait);
wakeup_done_ctr = wakeup_cookie;
mtx_unlock(&callout_wait_lock);
mtx_lock_spin(&callout_lock);
wakeup_needed = 0;
}
steps = 0;
c = nextsoftcheck;
}
}
}
avg_depth += (depth * 1000 - avg_depth) >> 8;
avg_mpcalls += (mpcalls * 1000 - avg_mpcalls) >> 8;
avg_gcalls += (gcalls * 1000 - avg_gcalls) >> 8;
nextsoftcheck = NULL;
mtx_unlock_spin(&callout_lock);
}
/*
* Software (low priority) clock interrupt.
* Run periodic events from timeout queue.
*/
void
softclock(void *arg)
{
struct callout_cpu *cc;
struct callout *c;
struct callout_tailq *bucket;
int curticks;
int steps; /* #steps since we last allowed interrupts */
int depth;
int mpcalls;
int lockcalls;
int gcalls;
#ifdef DIAGNOSTIC
struct bintime bt1, bt2;
struct timespec ts2;
static uint64_t maxdt = 36893488147419102LL; /* 2 msec */
static timeout_t *lastfunc;
#endif
#ifndef MAX_SOFTCLOCK_STEPS
#define MAX_SOFTCLOCK_STEPS 100 /* Maximum allowed value of steps. */
#endif /* MAX_SOFTCLOCK_STEPS */
mpcalls = 0;
lockcalls = 0;
gcalls = 0;
depth = 0;
steps = 0;
cc = (struct callout_cpu *)arg;
CC_LOCK(cc);
while (cc->cc_softticks != ticks) {
/*
* cc_softticks may be modified by hard clock, so cache
* it while we work on a given bucket.
*/
curticks = cc->cc_softticks;
cc->cc_softticks++;
bucket = &cc->cc_callwheel[curticks & callwheelmask];
c = TAILQ_FIRST(bucket);
while (c) {
depth++;
if (c->c_time != curticks) {
c = TAILQ_NEXT(c, c_links.tqe);
++steps;
if (steps >= MAX_SOFTCLOCK_STEPS) {
cc->cc_next = c;
/* Give interrupts a chance. */
CC_UNLOCK(cc);
; /* nothing */
CC_LOCK(cc);
c = cc->cc_next;
steps = 0;
}
} else {
void (*c_func)(void *);
void *c_arg;
struct lock_class *class;
struct lock_object *c_lock;
int c_flags, sharedlock;
cc->cc_next = TAILQ_NEXT(c, c_links.tqe);
TAILQ_REMOVE(bucket, c, c_links.tqe);
class = (c->c_lock != NULL) ?
LOCK_CLASS(c->c_lock) : NULL;
sharedlock = (c->c_flags & CALLOUT_SHAREDLOCK) ?
0 : 1;
c_lock = c->c_lock;
c_func = c->c_func;
c_arg = c->c_arg;
c_flags = c->c_flags;
if (c->c_flags & CALLOUT_LOCAL_ALLOC) {
c->c_flags = CALLOUT_LOCAL_ALLOC;
} else {
c->c_flags =
(c->c_flags & ~CALLOUT_PENDING);
}
cc->cc_curr = c;
cc->cc_cancel = 0;
CC_UNLOCK(cc);
if (c_lock != NULL) {
class->lc_lock(c_lock, sharedlock);
/*
* The callout may have been cancelled
* while we switched locks.
*/
if (cc->cc_cancel) {
class->lc_unlock(c_lock);
goto skip;
}
/* The callout cannot be stopped now. */
cc->cc_cancel = 1;
if (c_lock == &Giant.lock_object) {
gcalls++;
CTR3(KTR_CALLOUT,
"callout %p func %p arg %p",
c, c_func, c_arg);
} else {
lockcalls++;
CTR3(KTR_CALLOUT, "callout lock"
" %p func %p arg %p",
c, c_func, c_arg);
}
} else {
mpcalls++;
CTR3(KTR_CALLOUT,
"callout mpsafe %p func %p arg %p",
c, c_func, c_arg);
}
#ifdef DIAGNOSTIC
binuptime(&bt1);
#endif
THREAD_NO_SLEEPING();
SDT_PROBE(callout_execute, kernel, ,
callout_start, c, 0, 0, 0, 0);
c_func(c_arg);
SDT_PROBE(callout_execute, kernel, ,
callout_end, c, 0, 0, 0, 0);
THREAD_SLEEPING_OK();
#ifdef DIAGNOSTIC
binuptime(&bt2);
bintime_sub(&bt2, &bt1);
if (bt2.frac > maxdt) {
if (lastfunc != c_func ||
bt2.frac > maxdt * 2) {
bintime2timespec(&bt2, &ts2);
printf(
"Expensive timeout(9) function: %p(%p) %jd.%09ld s\n",
c_func, c_arg,
(intmax_t)ts2.tv_sec,
ts2.tv_nsec);
}
maxdt = bt2.frac;
lastfunc = c_func;
}
#endif
CTR1(KTR_CALLOUT, "callout %p finished", c);
if ((c_flags & CALLOUT_RETURNUNLOCKED) == 0)
class->lc_unlock(c_lock);
skip:
CC_LOCK(cc);
/*
* If the current callout is locally
* allocated (from timeout(9))
* then put it on the freelist.
*
* Note: we need to check the cached
* copy of c_flags because if it was not
* local, then it's not safe to deref the
* callout pointer.
*/
if (c_flags & CALLOUT_LOCAL_ALLOC) {
KASSERT(c->c_flags ==
CALLOUT_LOCAL_ALLOC,
("corrupted callout"));
c->c_func = NULL;
SLIST_INSERT_HEAD(&cc->cc_callfree, c,
c_links.sle);
}
cc->cc_curr = NULL;
if (cc->cc_waiting) {
/*
* There is someone waiting
* for the callout to complete.
*/
cc->cc_waiting = 0;
CC_UNLOCK(cc);
wakeup(&cc->cc_waiting);
CC_LOCK(cc);
}
steps = 0;
c = cc->cc_next;
}
}
}
