static int tsc_read_sample(struct tsc_data *ts, struct sample* sample)
{
int x, y, z1, z2, t, p = 0;
u32 val;
val = tsc_read(ts, chval[0]);
if (val & DATA_VALID)
x = val & 0xffff;
else
return -EINVAL;
y = tsc_read(ts, chval[1]) & 0xffff;
z1 = tsc_read(ts, chval[2]) & 0xffff;
z2 = tsc_read(ts, chval[3]) & 0xffff;
if (z1) {
t = ((600 * x) * (z2 - z1));
p = t / (u32) (z1 << 12);
if (p < 0)
p = 0;
}
sample->x = x;
sample->y = y;
sample->p = p;
return 0;
}
/*
* Set up the cpu, thread and interrupt thread structures for
* executing an interrupt thread. The new stack pointer of the
* interrupt thread (which *must* be switched to) is returned.
*/
caddr_t
intr_thread_prolog(struct cpu *cpu, caddr_t stackptr, uint_t pil)
{
struct machcpu *mcpu = &cpu->cpu_m;
kthread_t *t, *volatile it;
ASSERT(pil > 0);
ASSERT((cpu->cpu_intr_actv & (1 << pil)) == 0);
cpu->cpu_intr_actv |= (1 << pil);
/*
* Get set to run an interrupt thread.
* There should always be an interrupt thread, since we
* allocate one for each level on each CPU.
*
* t_intr_start could be zero due to cpu_intr_swtch_enter.
*/
t = cpu->cpu_thread;
if ((t->t_flag & T_INTR_THREAD) && t->t_intr_start != 0) {
hrtime_t intrtime = tsc_read() - t->t_intr_start;
mcpu->intrstat[t->t_pil][0] += intrtime;
cpu->cpu_intracct[cpu->cpu_mstate] += intrtime;
t->t_intr_start = 0;
}
ASSERT(SA((uintptr_t)stackptr) == (uintptr_t)stackptr);
t->t_sp = (uintptr_t)stackptr; /* mark stack in curthread for resume */
/*
* unlink the interrupt thread off the cpu
*
* Note that the code in kcpc_overflow_intr -relies- on the
* ordering of events here - in particular that t->t_lwp of
* the interrupt thread is set to the pinned thread *before*
* curthread is changed.
*/
it = cpu->cpu_intr_thread;
cpu->cpu_intr_thread = it->t_link;
it->t_intr = t;
it->t_lwp = t->t_lwp;
/*
* (threads on the interrupt thread free list could have state
* preset to TS_ONPROC, but it helps in debugging if
* they're TS_FREE.)
*/
it->t_state = TS_ONPROC;
cpu->cpu_thread = it; /* new curthread on this cpu */
it->t_pil = (uchar_t)pil;
it->t_pri = intr_pri + (pri_t)pil;
it->t_intr_start = tsc_read();
return (it->t_stk);
}
void
dosoftint_epilog(struct cpu *cpu, uint_t oldpil)
{
struct machcpu *mcpu = &cpu->cpu_m;
kthread_t *t, *it;
uint_t pil, basespl;
hrtime_t intrtime;
it = cpu->cpu_thread;
pil = it->t_pil;
cpu->cpu_stats.sys.intr[pil - 1]++;
ASSERT(cpu->cpu_intr_actv & (1 << pil));
cpu->cpu_intr_actv &= ~(1 << pil);
intrtime = tsc_read() - it->t_intr_start;
mcpu->intrstat[pil][0] += intrtime;
cpu->cpu_intracct[cpu->cpu_mstate] += intrtime;
/*
* If there is still an interrupted thread underneath this one
* then the interrupt was never blocked and the return is
* fairly simple. Otherwise it isn't.
*/
if ((t = it->t_intr) == NULL) {
/*
* Put thread back on the interrupt thread list.
* This was an interrupt thread, so set CPU's base SPL.
*/
set_base_spl();
it->t_state = TS_FREE;
it->t_link = cpu->cpu_intr_thread;
cpu->cpu_intr_thread = it;
(void) splhigh();
swtch();
/*NOTREACHED*/
}
it->t_link = cpu->cpu_intr_thread;
cpu->cpu_intr_thread = it;
it->t_state = TS_FREE;
cpu->cpu_thread = t;
if (t->t_flag & T_INTR_THREAD)
t->t_intr_start = tsc_read();
basespl = cpu->cpu_base_spl;
pil = MAX(oldpil, basespl);
mcpu->mcpu_pri = pil;
(*setspl)(pil);
}
/*
* intr_get_time() is a resource for interrupt handlers to determine how
* much time has been spent handling the current interrupt. Such a function
* is needed because higher level interrupts can arrive during the
* processing of an interrupt. intr_get_time() only returns time spent in the
* current interrupt handler.
*
* The caller must be calling from an interrupt handler running at a pil
* below or at lock level. Timings are not provided for high-level
* interrupts.
*
* The first time intr_get_time() is called while handling an interrupt,
* it returns the time since the interrupt handler was invoked. Subsequent
* calls will return the time since the prior call to intr_get_time(). Time
* is returned as ticks. Use scalehrtimef() to convert ticks to nsec.
*
* Theory Of Intrstat[][]:
*
* uint64_t intrstat[pil][0..1] is an array indexed by pil level, with two
* uint64_ts per pil.
*
* intrstat[pil][0] is a cumulative count of the number of ticks spent
* handling all interrupts at the specified pil on this CPU. It is
* exported via kstats to the user.
*
* intrstat[pil][1] is always a count of ticks less than or equal to the
* value in [0]. The difference between [1] and [0] is the value returned
* by a call to intr_get_time(). At the start of interrupt processing,
* [0] and [1] will be equal (or nearly so). As the interrupt consumes
* time, [0] will increase, but [1] will remain the same. A call to
* intr_get_time() will return the difference, then update [1] to be the
* same as [0]. Future calls will return the time since the last call.
* Finally, when the interrupt completes, [1] is updated to the same as [0].
*
* Implementation:
*
* intr_get_time() works much like a higher level interrupt arriving. It
* "checkpoints" the timing information by incrementing intrstat[pil][0]
* to include elapsed running time, and by setting t_intr_start to rdtsc.
* It then sets the return value to intrstat[pil][0] - intrstat[pil][1],
* and updates intrstat[pil][1] to be the same as the new value of
* intrstat[pil][0].
*
* In the normal handling of interrupts, after an interrupt handler returns
* and the code in intr_thread() updates intrstat[pil][0], it then sets
* intrstat[pil][1] to the new value of intrstat[pil][0]. When [0] == [1],
* the timings are reset, i.e. intr_get_time() will return [0] - [1] which
* is 0.
*
* Whenever interrupts arrive on a CPU which is handling a lower pil
* interrupt, they update the lower pil's [0] to show time spent in the
* handler that they've interrupted. This results in a growing discrepancy
* between [0] and [1], which is returned the next time intr_get_time() is
* called. Time spent in the higher-pil interrupt will not be returned in
* the next intr_get_time() call from the original interrupt, because
* the higher-pil interrupt's time is accumulated in intrstat[higherpil][].
*/
uint64_t
intr_get_time(void)
{
struct cpu *cpu;
struct machcpu *mcpu;
kthread_t *t;
uint64_t time, delta, ret;
uint_t pil;
cli();
cpu = CPU;
mcpu = &cpu->cpu_m;
t = cpu->cpu_thread;
pil = t->t_pil;
ASSERT((cpu->cpu_intr_actv & CPU_INTR_ACTV_HIGH_LEVEL_MASK) == 0);
ASSERT(t->t_flag & T_INTR_THREAD);
ASSERT(pil != 0);
ASSERT(t->t_intr_start != 0);
time = tsc_read();
delta = time - t->t_intr_start;
t->t_intr_start = time;
time = mcpu->intrstat[pil][0] + delta;
ret = time - mcpu->intrstat[pil][1];
mcpu->intrstat[pil][0] = time;
mcpu->intrstat[pil][1] = time;
cpu->cpu_intracct[cpu->cpu_mstate] += delta;
sti();
return (ret);
}
/*
* Let timestamp.c know that we are suspending. It needs to take
* snapshots of the current time, and do any pre-suspend work.
*/
void
tsc_suspend(void)
{
/*
* What we need to do here, is to get the time we suspended, so that we
* know how much we should add to the resume.
* This routine is called by each CPU, so we need to handle reentry.
*/
if (tsc_gethrtime_enable) {
/*
* We put the tsc_read() inside the lock as it
* as no locking constraints, and it puts the
* aquired value closer to the time stamp (in
* case we delay getting the lock).
*/
mutex_enter(&tod_lock);
tsc_saved_tsc = tsc_read();
tsc_saved_ts = TODOP_GET(tod_ops);
mutex_exit(&tod_lock);
/* We only want to do this once. */
if (tsc_needs_resume == 0) {
if (tsc_delta_onsuspend) {
tsc_adjust_delta(tsc_saved_tsc);
} else {
tsc_adjust_delta(nsec_scale);
}
tsc_suspend_count++;
}
}
invalidate_cache();
tsc_needs_resume = 1;
}
/*
* An interrupt thread is ending a time slice, so compute the interval it
* ran for and update the statistic for its PIL.
*/
void
cpu_intr_swtch_enter(kthread_id_t t)
{
uint64_t interval;
uint64_t start;
cpu_t *cpu;
ASSERT((t->t_flag & T_INTR_THREAD) != 0);
ASSERT(t->t_pil > 0 && t->t_pil <= LOCK_LEVEL);
/*
* We could be here with a zero timestamp. This could happen if:
* an interrupt thread which no longer has a pinned thread underneath
* it (i.e. it blocked at some point in its past) has finished running
* its handler. intr_thread() updated the interrupt statistic for its
* PIL and zeroed its timestamp. Since there was no pinned thread to
* return to, swtch() gets called and we end up here.
*
* Note that we use atomic ops below (cas64 and atomic_add_64), which
* we don't use in the functions above, because we're not called
* with interrupts blocked, but the epilog/prolog functions are.
*/
if (t->t_intr_start) {
do {
start = t->t_intr_start;
interval = tsc_read() - start;
} while (cas64(&t->t_intr_start, start, 0) != start);
cpu = CPU;
cpu->cpu_m.intrstat[t->t_pil][0] += interval;
atomic_add_64((uint64_t *)&cpu->cpu_intracct[cpu->cpu_mstate],
interval);
} else
ASSERT(t->t_intr == NULL);
}
/*
* An interrupt thread is returning from swtch(). Place a starting timestamp
* in its thread structure.
*/
void
cpu_intr_swtch_exit(kthread_id_t t)
{
uint64_t ts;
ASSERT((t->t_flag & T_INTR_THREAD) != 0);
ASSERT(t->t_pil > 0 && t->t_pil <= LOCK_LEVEL);
do {
ts = t->t_intr_start;
} while (cas64(&t->t_intr_start, ts, tsc_read()) != ts);
}
/*
* Called once per second on a CPU from the cyclic subsystem's
* CY_HIGH_LEVEL interrupt. (No longer just cpu0-only)
*/
void
tsc_tick(void)
{
hrtime_t now, delta;
ushort_t spl;
/*
* Before we set the new variables, we set the shadow values. This
* allows for lock free operation in dtrace_gethrtime().
*/
lock_set_spl((lock_t *)&shadow_hres_lock + HRES_LOCK_OFFSET,
ipltospl(CBE_HIGH_PIL), &spl);
shadow_tsc_hrtime_base = tsc_hrtime_base;
shadow_tsc_last = tsc_last;
shadow_nsec_scale = nsec_scale;
shadow_hres_lock++;
splx(spl);
CLOCK_LOCK(&spl);
now = tsc_read();
if (gethrtimef == tsc_gethrtime_delta)
now += tsc_sync_tick_delta[CPU->cpu_id];
if (now < tsc_last) {
/*
* The TSC has just jumped into the past. We assume that
* this is due to a suspend/resume cycle, and we're going
* to use the _current_ value of TSC as the delta. This
* will keep tsc_hrtime_base correct. We're also going to
* assume that rate of tsc does not change after a suspend
* resume (i.e nsec_scale remains the same).
*/
delta = now;
tsc_last_jumped += tsc_last;
tsc_jumped = 1;
} else {
/*
* Determine the number of TSC ticks since the last clock
* tick, and add that to the hrtime base.
*/
delta = now - tsc_last;
}
TSC_CONVERT_AND_ADD(delta, tsc_hrtime_base, nsec_scale);
tsc_last = now;
CLOCK_UNLOCK(spl);
}
void
tsc_hrtimeinit(uint64_t cpu_freq_hz)
{
extern int gethrtime_hires;
longlong_t tsc;
ulong_t flags;
/*
* cpu_freq_hz is the measured cpu frequency in hertz
*/
/*
* We can't accommodate CPUs slower than 31.25 MHz.
*/
ASSERT(cpu_freq_hz > NANOSEC / (1 << NSEC_SHIFT));
nsec_scale =
(uint_t)(((uint64_t)NANOSEC << (32 - NSEC_SHIFT)) / cpu_freq_hz);
nsec_unscale =
(uint_t)(((uint64_t)cpu_freq_hz << (32 - NSEC_SHIFT)) / NANOSEC);
flags = clear_int_flag();
tsc = tsc_read();
(void) tsc_gethrtime();
tsc_max_delta = tsc_read() - tsc;
restore_int_flag(flags);
gethrtimef = tsc_gethrtime;
gethrtimeunscaledf = tsc_gethrtimeunscaled;
scalehrtimef = tsc_scalehrtime;
unscalehrtimef = tsc_unscalehrtime;
hrtime_tick = tsc_tick;
gethrtime_hires = 1;
/*
* Allocate memory for the structure used in the tsc sync logic.
* This structure should be aligned on a multiple of cache line size.
*/
tscp = kmem_zalloc(PAGESIZE, KM_SLEEP);
}
hrtime_t
tsc_gethrtimeunscaled(void)
{
uint32_t old_hres_lock;
hrtime_t tsc;
do {
old_hres_lock = hres_lock;
/* See tsc_tick(). */
tsc = tsc_read() + tsc_last_jumped;
} while ((old_hres_lock & ~1) != hres_lock);
return (tsc);
}
/*
* Called by a CPU which has just been onlined. It is expected that the CPU
* performing the online operation will call tsc_sync_master().
*
* TSC sync is disabled in the context of virtualization. See comments
* above tsc_sync_master.
*/
void
tsc_sync_slave(void)
{
ulong_t flags;
hrtime_t s1;
tsc_sync_t *tsc = tscp;
int cnt;
int hwtype;
hwtype = get_hwenv();
if (!tsc_master_slave_sync_needed || hwtype == HW_XEN_HVM ||
hwtype == HW_VMWARE)
return;
flags = clear_int_flag();
for (cnt = 0; cnt < SYNC_ITERATIONS; cnt++) {
/* Re-fill the cache line */
s1 = tsc->master_tsc;
membar_enter();
tsc_sync_go = TSC_SYNC_GO;
do {
/*
* Do not put an SMT_PAUSE here. For instance,
* if the master and slave are really the same
* hyper-threaded CPU, then you want the master
* to yield to the slave as quickly as possible here,
* but not the other way.
*/
s1 = tsc_read();
} while (tsc->master_tsc == 0);
tsc->slave_tsc = s1;
membar_enter();
tsc_sync_go = TSC_SYNC_DONE;
while (tsc_sync_go != TSC_SYNC_STOP)
SMT_PAUSE();
}
restore_int_flag(flags);
}
hrtime_t
tsc_gethrtime_delta(void)
{
uint32_t old_hres_lock;
hrtime_t tsc, hrt;
ulong_t flags;
do {
old_hres_lock = hres_lock;
/*
* We need to disable interrupts here to assure that we
* don't migrate between the call to tsc_read() and
* adding the CPU's TSC tick delta. Note that disabling
* and reenabling preemption is forbidden here because
* we may be in the middle of a fast trap. In the amd64
* kernel we cannot tolerate preemption during a fast
* trap. See _update_sregs().
*/
flags = clear_int_flag();
tsc = tsc_read() + tsc_sync_tick_delta[CPU->cpu_id];
restore_int_flag(flags);
/* See comments in tsc_gethrtime() above */
if (tsc >= tsc_last) {
tsc -= tsc_last;
} else if (tsc >= tsc_last - 2 * tsc_max_delta) {
tsc = 0;
}
hrt = tsc_hrtime_base;
TSC_CONVERT_AND_ADD(tsc, hrt, nsec_scale);
} while ((old_hres_lock & ~1) != hres_lock);
return (hrt);
}
hrtime_t
tsc_gethrtime(void)
{
uint32_t old_hres_lock;
hrtime_t tsc, hrt;
do {
old_hres_lock = hres_lock;
if ((tsc = tsc_read()) >= tsc_last) {
/*
* It would seem to be obvious that this is true
* (that is, the past is less than the present),
* but it isn't true in the presence of suspend/resume
* cycles. If we manage to call gethrtime()
* after a resume, but before the first call to
* tsc_tick(), we will see the jump. In this case,
* we will simply use the value in TSC as the delta.
*/
tsc -= tsc_last;
} else if (tsc >= tsc_last - 2*tsc_max_delta) {
/*
* There is a chance that tsc_tick() has just run on
* another CPU, and we have drifted just enough so that
* we appear behind tsc_last. In this case, force the
* delta to be zero.
*/
tsc = 0;
}
hrt = tsc_hrtime_base;
TSC_CONVERT_AND_ADD(tsc, hrt, nsec_scale);
} while ((old_hres_lock & ~1) != hres_lock);
return (hrt);
}
static int tsc_start(struct input_dev *dev)
{
struct tsc_data *ts = input_get_drvdata(dev);
unsigned long timeout = jiffies + msecs_to_jiffies(IDLE_TIMEOUT);
u32 val;
clk_enable(ts->clk);
/* Go to idle mode, before any initialization */
while (time_after(timeout, jiffies)) {
if (tsc_read(ts, tscm) & IDLE)
break;
}
if (time_before(timeout, jiffies)) {
dev_warn(ts->dev, "timeout waiting for idle\n");
clk_disable(ts->clk);
return -EIO;
}
/* Configure TSC Control register*/
val = (PONBG | PON | PVSTC(4) | ONE_SHOT | ZMEASURE_EN);
tsc_write(ts, tscm, val);
/* Bring TSC out of reset: Clear AFE reset bit */
val &= ~(AFERST);
tsc_write(ts, tscm, val);
/* Configure all pins for hardware control*/
tsc_write(ts, bwcm, 0);
/* Finally enable the TSC */
tsc_set_bits(ts, tscm, TSC_EN);
return 0;
}
/*
* Get an interrupt thread and swith to it. It's called from do_interrupt().
* The IF flag is cleared and thus all maskable interrupts are blocked at
* the time of calling.
*/
static caddr_t
apix_intr_thread_prolog(struct cpu *cpu, uint_t pil, caddr_t stackptr)
{
apix_impl_t *apixp = apixs[cpu->cpu_id];
struct machcpu *mcpu = &cpu->cpu_m;
hrtime_t now = tsc_read();
kthread_t *t, *volatile it;
ASSERT(pil > mcpu->mcpu_pri && pil > cpu->cpu_base_spl);
apixp->x_intr_pending &= ~(1 << pil);
ASSERT((cpu->cpu_intr_actv & (1 << pil)) == 0);
cpu->cpu_intr_actv |= (1 << pil);
mcpu->mcpu_pri = pil;
/*
* Get set to run interrupt thread.
* There should always be an interrupt thread since we
* allocate one for each level on the CPU.
*/
/* t_intr_start could be zero due to cpu_intr_swtch_enter. */
t = cpu->cpu_thread;
if ((t->t_flag & T_INTR_THREAD) && t->t_intr_start != 0) {
hrtime_t intrtime = now - t->t_intr_start;
mcpu->intrstat[pil][0] += intrtime;
cpu->cpu_intracct[cpu->cpu_mstate] += intrtime;
t->t_intr_start = 0;
}
/*
* Push interrupted thread onto list from new thread.
* Set the new thread as the current one.
* Set interrupted thread's T_SP because if it is the idle thread,
* resume() may use that stack between threads.
*/
ASSERT(SA((uintptr_t)stackptr) == (uintptr_t)stackptr);
t->t_sp = (uintptr_t)stackptr; /* mark stack in curthread for resume */
/*
* Note that the code in kcpc_overflow_intr -relies- on the
* ordering of events here - in particular that t->t_lwp of
* the interrupt thread is set to the pinned thread *before*
* curthread is changed.
*/
it = cpu->cpu_intr_thread;
cpu->cpu_intr_thread = it->t_link;
it->t_intr = t;
it->t_lwp = t->t_lwp;
/*
* (threads on the interrupt thread free list could have state
* preset to TS_ONPROC, but it helps in debugging if
* they're TS_FREE.)
*/
it->t_state = TS_ONPROC;
cpu->cpu_thread = it;
/*
* Initialize thread priority level from intr_pri
*/
it->t_pil = (uchar_t)pil;
it->t_pri = (pri_t)pil + intr_pri;
it->t_intr_start = now;
return (it->t_stk);
}
static int
apix_hilevel_intr_epilog(struct cpu *cpu, uint_t oldpil)
{
struct machcpu *mcpu = &cpu->cpu_m;
uint_t mask, pil;
hrtime_t intrtime;
hrtime_t now = tsc_read();
pil = mcpu->mcpu_pri;
cpu->cpu_stats.sys.intr[pil - 1]++;
ASSERT(cpu->cpu_intr_actv & (1 << pil));
if (pil == 15) {
/*
* To support reentrant level 15 interrupts, we maintain a
* recursion count in the top half of cpu_intr_actv. Only
* when this count hits zero do we clear the PIL 15 bit from
* the lower half of cpu_intr_actv.
*/
uint16_t *refcntp = (uint16_t *)&cpu->cpu_intr_actv + 1;
ASSERT(*refcntp > 0);
if (--(*refcntp) == 0)
cpu->cpu_intr_actv &= ~(1 << pil);
} else {
cpu->cpu_intr_actv &= ~(1 << pil);
}
ASSERT(mcpu->pil_high_start[pil - (LOCK_LEVEL + 1)] != 0);
intrtime = now - mcpu->pil_high_start[pil - (LOCK_LEVEL + 1)];
mcpu->intrstat[pil][0] += intrtime;
cpu->cpu_intracct[cpu->cpu_mstate] += intrtime;
/*
* Check for lower-pil nested high-level interrupt beneath
* current one. If so, place a starting timestamp in its
* pil_high_start entry.
*/
mask = cpu->cpu_intr_actv & CPU_INTR_ACTV_HIGH_LEVEL_MASK;
if (mask != 0) {
int nestpil;
/*
* find PIL of nested interrupt
*/
nestpil = bsrw_insn((uint16_t)mask);
ASSERT(nestpil < pil);
mcpu->pil_high_start[nestpil - (LOCK_LEVEL + 1)] = now;
/*
* (Another high-level interrupt is active below this one,
* so there is no need to check for an interrupt
* thread. That will be done by the lowest priority
* high-level interrupt active.)
*/
} else {
/*
* Check to see if there is a low-level interrupt active.
* If so, place a starting timestamp in the thread
* structure.
*/
kthread_t *t = cpu->cpu_thread;
if (t->t_flag & T_INTR_THREAD)
t->t_intr_start = now;
}
mcpu->mcpu_pri = oldpil;
if (pil < CBE_HIGH_PIL)
(void) (*setlvlx)(oldpil, 0);
return (mask);
}
static int
apix_hilevel_intr_prolog(struct cpu *cpu, uint_t pil, uint_t oldpil,
struct regs *rp)
{
struct machcpu *mcpu = &cpu->cpu_m;
hrtime_t intrtime;
hrtime_t now = tsc_read();
apix_impl_t *apixp = apixs[cpu->cpu_id];
uint_t mask;
ASSERT(pil > mcpu->mcpu_pri && pil > cpu->cpu_base_spl);
if (pil == CBE_HIGH_PIL) { /* 14 */
cpu->cpu_profile_pil = oldpil;
if (USERMODE(rp->r_cs)) {
cpu->cpu_profile_pc = 0;
cpu->cpu_profile_upc = rp->r_pc;
cpu->cpu_cpcprofile_pc = 0;
cpu->cpu_cpcprofile_upc = rp->r_pc;
} else {
cpu->cpu_profile_pc = rp->r_pc;
cpu->cpu_profile_upc = 0;
cpu->cpu_cpcprofile_pc = rp->r_pc;
cpu->cpu_cpcprofile_upc = 0;
}
}
mcpu->mcpu_pri = pil;
mask = cpu->cpu_intr_actv & CPU_INTR_ACTV_HIGH_LEVEL_MASK;
if (mask != 0) {
int nestpil;
/*
* We have interrupted another high-level interrupt.
* Load starting timestamp, compute interval, update
* cumulative counter.
*/
nestpil = bsrw_insn((uint16_t)mask);
intrtime = now -
mcpu->pil_high_start[nestpil - (LOCK_LEVEL + 1)];
mcpu->intrstat[nestpil][0] += intrtime;
cpu->cpu_intracct[cpu->cpu_mstate] += intrtime;
} else {
kthread_t *t = cpu->cpu_thread;
/*
* See if we are interrupting a low-level interrupt thread.
* If so, account for its time slice only if its time stamp
* is non-zero.
*/
if ((t->t_flag & T_INTR_THREAD) != 0 && t->t_intr_start != 0) {
intrtime = now - t->t_intr_start;
mcpu->intrstat[t->t_pil][0] += intrtime;
cpu->cpu_intracct[cpu->cpu_mstate] += intrtime;
t->t_intr_start = 0;
}
}
/* store starting timestamp in CPu structure for this IPL */
mcpu->pil_high_start[pil - (LOCK_LEVEL + 1)] = now;
if (pil == 15) {
/*
* To support reentrant level 15 interrupts, we maintain a
* recursion count in the top half of cpu_intr_actv. Only
* when this count hits zero do we clear the PIL 15 bit from
* the lower half of cpu_intr_actv.
*/
uint16_t *refcntp = (uint16_t *)&cpu->cpu_intr_actv + 1;
(*refcntp)++;
}
cpu->cpu_intr_actv |= (1 << pil);
/* clear pending ipl level bit */
apixp->x_intr_pending &= ~(1 << pil);
return (mask);
}
static caddr_t
apix_do_softint_prolog(struct cpu *cpu, uint_t pil, uint_t oldpil,
caddr_t stackptr)
{
kthread_t *t, *volatile it;
struct machcpu *mcpu = &cpu->cpu_m;
hrtime_t now;
UNREFERENCED_1PARAMETER(oldpil);
ASSERT(pil > mcpu->mcpu_pri && pil > cpu->cpu_base_spl);
atomic_and_32((uint32_t *)&mcpu->mcpu_softinfo.st_pending, ~(1 << pil));
mcpu->mcpu_pri = pil;
now = tsc_read();
/*
* Get set to run interrupt thread.
* There should always be an interrupt thread since we
* allocate one for each level on the CPU.
*/
it = cpu->cpu_intr_thread;
ASSERT(it != NULL);
cpu->cpu_intr_thread = it->t_link;
/* t_intr_start could be zero due to cpu_intr_swtch_enter. */
t = cpu->cpu_thread;
if ((t->t_flag & T_INTR_THREAD) && t->t_intr_start != 0) {
hrtime_t intrtime = now - t->t_intr_start;
mcpu->intrstat[pil][0] += intrtime;
cpu->cpu_intracct[cpu->cpu_mstate] += intrtime;
t->t_intr_start = 0;
}
/*
* Note that the code in kcpc_overflow_intr -relies- on the
* ordering of events here - in particular that t->t_lwp of
* the interrupt thread is set to the pinned thread *before*
* curthread is changed.
*/
it->t_lwp = t->t_lwp;
it->t_state = TS_ONPROC;
/*
* Push interrupted thread onto list from new thread.
* Set the new thread as the current one.
* Set interrupted thread's T_SP because if it is the idle thread,
* resume() may use that stack between threads.
*/
ASSERT(SA((uintptr_t)stackptr) == (uintptr_t)stackptr);
t->t_sp = (uintptr_t)stackptr;
it->t_intr = t;
cpu->cpu_thread = it;
/*
* Set bit for this pil in CPU's interrupt active bitmask.
*/
ASSERT((cpu->cpu_intr_actv & (1 << pil)) == 0);
cpu->cpu_intr_actv |= (1 << pil);
/*
* Initialize thread priority level from intr_pri
*/
it->t_pil = (uchar_t)pil;
it->t_pri = (pri_t)pil + intr_pri;
it->t_intr_start = now;
return (it->t_stk);
}
/*
* Restore all timestamp state based on the snapshots taken at
* suspend time.
*/
void
tsc_resume(void)
{
/*
* We only need to (and want to) do this once. So let the first
* caller handle this (we are locked by the cpu lock), as it
* is preferential that we get the earliest sync.
*/
if (tsc_needs_resume) {
/*
* If using the TSC, adjust the delta based on how long
* we were sleeping (or away). We also adjust for
* migration and a grown TSC.
*/
if (tsc_saved_tsc != 0) {
timestruc_t ts;
hrtime_t now, sleep_tsc = 0;
int sleep_sec;
extern void tsc_tick(void);
extern uint64_t cpu_freq_hz;
/* tsc_read() MUST be before TODOP_GET() */
mutex_enter(&tod_lock);
now = tsc_read();
ts = TODOP_GET(tod_ops);
mutex_exit(&tod_lock);
/* Compute seconds of sleep time */
sleep_sec = ts.tv_sec - tsc_saved_ts.tv_sec;
/*
* If the saved sec is less that or equal to
* the current ts, then there is likely a
* problem with the clock. Assume at least
* one second has passed, so that time goes forward.
*/
if (sleep_sec <= 0) {
sleep_sec = 1;
}
/* How many TSC's should have occured while sleeping */
if (tsc_adjust_seconds)
sleep_tsc = sleep_sec * cpu_freq_hz;
/*
* We also want to subtract from the "sleep_tsc"
* the current value of tsc_read(), so that our
* adjustment accounts for the amount of time we
* have been resumed _or_ an adjustment based on
* the fact that we didn't actually power off the
* CPU (migration is another issue, but _should_
* also comply with this calculation). If the CPU
* never powered off, then:
* 'now == sleep_tsc + saved_tsc'
* and the delta will effectively be "0".
*/
sleep_tsc -= now;
if (tsc_delta_onsuspend) {
tsc_adjust_delta(sleep_tsc);
} else {
tsc_adjust_delta(tsc_saved_tsc + sleep_tsc);
}
tsc_saved_tsc = 0;
tsc_tick();
}
tsc_needs_resume = 0;
}
}
static int __devinit tsc_probe(struct platform_device *pdev)
{
struct device *dev = &pdev->dev;
struct tsc_data *ts;
int error = 0;
u32 rev = 0;
ts = kzalloc(sizeof(struct tsc_data), GFP_KERNEL);
if (!ts) {
dev_err(dev, "cannot allocate device info\n");
return -ENOMEM;
}
ts->dev = dev;
spin_lock_init(&ts->lock);
setup_timer(&ts->timer, tsc_poll, (unsigned long)ts);
platform_set_drvdata(pdev, ts);
ts->tsc_irq = platform_get_irq(pdev, 0);
if (ts->tsc_irq < 0) {
dev_err(dev, "cannot determine device interrupt\n");
error = -ENODEV;
goto error_res;
}
ts->res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
if (!ts->res) {
dev_err(dev, "cannot determine register area\n");
error = -ENODEV;
goto error_res;
}
if (!request_mem_region(ts->res->start, resource_size(ts->res),
pdev->name)) {
dev_err(dev, "cannot claim register memory\n");
ts->res = NULL;
error = -EINVAL;
goto error_res;
}
ts->regs = ioremap(ts->res->start, resource_size(ts->res));
if (!ts->regs) {
dev_err(dev, "cannot map register memory\n");
error = -ENOMEM;
goto error_map;
}
ts->clk = clk_get(dev, NULL);
if (IS_ERR(ts->clk)) {
dev_err(dev, "cannot claim device clock\n");
error = PTR_ERR(ts->clk);
goto error_clk;
}
error = request_threaded_irq(ts->tsc_irq, NULL, tsc_irq, 0,
dev_name(dev), ts);
if (error < 0) {
dev_err(ts->dev, "Could not allocate ts irq\n");
goto error_irq;
}
ts->input_dev = input_allocate_device();
if (!ts->input_dev) {
dev_err(dev, "cannot allocate input device\n");
error = -ENOMEM;
goto error_input;
}
input_set_drvdata(ts->input_dev, ts);
ts->input_dev->name = pdev->name;
ts->input_dev->id.bustype = BUS_HOST;
ts->input_dev->dev.parent = &pdev->dev;
ts->input_dev->open = tsc_start;
ts->input_dev->close = tsc_stop;
clk_enable(ts->clk);
rev = tsc_read(ts, rev);
ts->input_dev->id.product = ((rev >> 8) & 0x07);
ts->input_dev->id.version = ((rev >> 16) & 0xfff);
clk_disable(ts->clk);
__set_bit(EV_KEY, ts->input_dev->evbit);
__set_bit(EV_ABS, ts->input_dev->evbit);
__set_bit(BTN_TOUCH, ts->input_dev->keybit);
input_set_abs_params(ts->input_dev, ABS_X, 0, 0xffff, 5, 0);
input_set_abs_params(ts->input_dev, ABS_Y, 0, 0xffff, 5, 0);
input_set_abs_params(ts->input_dev, ABS_PRESSURE, 0, 4095, 128, 0);
error = input_register_device(ts->input_dev);
if (error < 0) {
dev_err(dev, "failed input device registration\n");
goto error_reg;
}
return 0;
error_reg:
input_free_device(ts->input_dev);
error_input:
free_irq(ts->tsc_irq, ts);
error_irq:
clk_put(ts->clk);
error_clk:
iounmap(ts->regs);
error_map:
release_mem_region(ts->res->start, resource_size(ts->res));
error_res:
platform_set_drvdata(pdev, NULL);
kfree(ts);
return error;
}
/*
* Do all the work necessary to set up the cpu and thread structures
* to dispatch a high-level interrupt.
*
* Returns 0 if we're -not- already on the high-level interrupt stack,
* (and *must* switch to it), non-zero if we are already on that stack.
*
* Called with interrupts masked.
* The 'pil' is already set to the appropriate level for rp->r_trapno.
*/
static int
hilevel_intr_prolog(struct cpu *cpu, uint_t pil, uint_t oldpil, struct regs *rp)
{
struct machcpu *mcpu = &cpu->cpu_m;
uint_t mask;
hrtime_t intrtime;
hrtime_t now = tsc_read();
ASSERT(pil > LOCK_LEVEL);
if (pil == CBE_HIGH_PIL) {
cpu->cpu_profile_pil = oldpil;
if (USERMODE(rp->r_cs)) {
cpu->cpu_profile_pc = 0;
cpu->cpu_profile_upc = rp->r_pc;
cpu->cpu_cpcprofile_pc = 0;
cpu->cpu_cpcprofile_upc = rp->r_pc;
} else {
cpu->cpu_profile_pc = rp->r_pc;
cpu->cpu_profile_upc = 0;
cpu->cpu_cpcprofile_pc = rp->r_pc;
cpu->cpu_cpcprofile_upc = 0;
}
}
mask = cpu->cpu_intr_actv & CPU_INTR_ACTV_HIGH_LEVEL_MASK;
if (mask != 0) {
int nestpil;
/*
* We have interrupted another high-level interrupt.
* Load starting timestamp, compute interval, update
* cumulative counter.
*/
nestpil = bsrw_insn((uint16_t)mask);
ASSERT(nestpil < pil);
intrtime = now -
mcpu->pil_high_start[nestpil - (LOCK_LEVEL + 1)];
mcpu->intrstat[nestpil][0] += intrtime;
cpu->cpu_intracct[cpu->cpu_mstate] += intrtime;
/*
* Another high-level interrupt is active below this one, so
* there is no need to check for an interrupt thread. That
* will be done by the lowest priority high-level interrupt
* active.
*/
} else {
kthread_t *t = cpu->cpu_thread;
/*
* See if we are interrupting a low-level interrupt thread.
* If so, account for its time slice only if its time stamp
* is non-zero.
*/
if ((t->t_flag & T_INTR_THREAD) != 0 && t->t_intr_start != 0) {
intrtime = now - t->t_intr_start;
mcpu->intrstat[t->t_pil][0] += intrtime;
cpu->cpu_intracct[cpu->cpu_mstate] += intrtime;
t->t_intr_start = 0;
}
}
/*
* Store starting timestamp in CPU structure for this PIL.
*/
mcpu->pil_high_start[pil - (LOCK_LEVEL + 1)] = now;
ASSERT((cpu->cpu_intr_actv & (1 << pil)) == 0);
if (pil == 15) {
/*
* To support reentrant level 15 interrupts, we maintain a
* recursion count in the top half of cpu_intr_actv. Only
* when this count hits zero do we clear the PIL 15 bit from
* the lower half of cpu_intr_actv.
*/
uint16_t *refcntp = (uint16_t *)&cpu->cpu_intr_actv + 1;
(*refcntp)++;
}
mask = cpu->cpu_intr_actv;
cpu->cpu_intr_actv |= (1 << pil);
return (mask & CPU_INTR_ACTV_HIGH_LEVEL_MASK);
}
timestamp_t timestamp_from_now_us(timestamp_t us) {
return (timestamp_t)(tsc_read() + timestamp_us_to_ticks((timestamp_t)us));
}
/*
* Called with interrupts disabled
*/
void
intr_thread_epilog(struct cpu *cpu, uint_t vec, uint_t oldpil)
{
struct machcpu *mcpu = &cpu->cpu_m;
kthread_t *t;
kthread_t *it = cpu->cpu_thread; /* curthread */
uint_t pil, basespl;
hrtime_t intrtime;
pil = it->t_pil;
cpu->cpu_stats.sys.intr[pil - 1]++;
ASSERT(it->t_intr_start != 0);
intrtime = tsc_read() - it->t_intr_start;
mcpu->intrstat[pil][0] += intrtime;
cpu->cpu_intracct[cpu->cpu_mstate] += intrtime;
ASSERT(cpu->cpu_intr_actv & (1 << pil));
cpu->cpu_intr_actv &= ~(1 << pil);
/*
* If there is still an interrupted thread underneath this one
* then the interrupt was never blocked and the return is
* fairly simple. Otherwise it isn't.
*/
if ((t = it->t_intr) == NULL) {
/*
* The interrupted thread is no longer pinned underneath
* the interrupt thread. This means the interrupt must
* have blocked, and the interrupted thread has been
* unpinned, and has probably been running around the
* system for a while.
*
* Since there is no longer a thread under this one, put
* this interrupt thread back on the CPU's free list and
* resume the idle thread which will dispatch the next
* thread to run.
*/
#ifdef DEBUG
intr_thread_cnt++;
#endif
cpu->cpu_stats.sys.intrblk++;
/*
* Set CPU's base SPL based on active interrupts bitmask
*/
set_base_spl();
basespl = cpu->cpu_base_spl;
mcpu->mcpu_pri = basespl;
(*setlvlx)(basespl, vec);
(void) splhigh();
it->t_state = TS_FREE;
/*
* Return interrupt thread to pool
*/
it->t_link = cpu->cpu_intr_thread;
cpu->cpu_intr_thread = it;
swtch();
/*NOTREACHED*/
}
/*
* Return interrupt thread to the pool
*/
it->t_link = cpu->cpu_intr_thread;
cpu->cpu_intr_thread = it;
it->t_state = TS_FREE;
basespl = cpu->cpu_base_spl;
pil = MAX(oldpil, basespl);
mcpu->mcpu_pri = pil;
(*setlvlx)(pil, vec);
t->t_intr_start = tsc_read();
cpu->cpu_thread = t;
}
/*
* This is similar to the above, but it cannot actually spin on hres_lock.
* As a result, it caches all of the variables it needs; if the variables
* don't change, it's done.
*/
hrtime_t
dtrace_gethrtime(void)
{
uint32_t old_hres_lock;
hrtime_t tsc, hrt;
ulong_t flags;
do {
old_hres_lock = hres_lock;
/*
* Interrupts are disabled to ensure that the thread isn't
* migrated between the tsc_read() and adding the CPU's
* TSC tick delta.
*/
flags = clear_int_flag();
tsc = tsc_read();
if (gethrtimef == tsc_gethrtime_delta)
tsc += tsc_sync_tick_delta[CPU->cpu_id];
restore_int_flag(flags);
/*
* See the comments in tsc_gethrtime(), above.
*/
if (tsc >= tsc_last)
tsc -= tsc_last;
else if (tsc >= tsc_last - 2*tsc_max_delta)
tsc = 0;
hrt = tsc_hrtime_base;
TSC_CONVERT_AND_ADD(tsc, hrt, nsec_scale);
if ((old_hres_lock & ~1) == hres_lock)
break;
/*
* If we're here, the clock lock is locked -- or it has been
* unlocked and locked since we looked. This may be due to
* tsc_tick() running on another CPU -- or it may be because
* some code path has ended up in dtrace_probe() with
* CLOCK_LOCK held. We'll try to determine that we're in
* the former case by taking another lap if the lock has
* changed since when we first looked at it.
*/
if (old_hres_lock != hres_lock)
continue;
/*
* So the lock was and is locked. We'll use the old data
* instead.
*/
old_hres_lock = shadow_hres_lock;
/*
* Again, disable interrupts to ensure that the thread
* isn't migrated between the tsc_read() and adding
* the CPU's TSC tick delta.
*/
flags = clear_int_flag();
tsc = tsc_read();
if (gethrtimef == tsc_gethrtime_delta)
tsc += tsc_sync_tick_delta[CPU->cpu_id];
restore_int_flag(flags);
/*
* See the comments in tsc_gethrtime(), above.
*/
if (tsc >= shadow_tsc_last)
tsc -= shadow_tsc_last;
else if (tsc >= shadow_tsc_last - 2 * tsc_max_delta)
tsc = 0;
hrt = shadow_tsc_hrtime_base;
TSC_CONVERT_AND_ADD(tsc, hrt, shadow_nsec_scale);
} while ((old_hres_lock & ~1) != shadow_hres_lock);
return (hrt);
}
static void
apix_intr_thread_epilog(struct cpu *cpu, uint_t oldpil)
{
struct machcpu *mcpu = &cpu->cpu_m;
kthread_t *t, *it = cpu->cpu_thread;
uint_t pil, basespl;
hrtime_t intrtime;
hrtime_t now = tsc_read();
pil = it->t_pil;
cpu->cpu_stats.sys.intr[pil - 1]++;
ASSERT(cpu->cpu_intr_actv & (1 << pil));
cpu->cpu_intr_actv &= ~(1 << pil);
ASSERT(it->t_intr_start != 0);
intrtime = now - it->t_intr_start;
mcpu->intrstat[pil][0] += intrtime;
cpu->cpu_intracct[cpu->cpu_mstate] += intrtime;
/*
* If there is still an interrupted thread underneath this one
* then the interrupt was never blocked and the return is
* fairly simple. Otherwise it isn't.
*/
if ((t = it->t_intr) == NULL) {
/*
* The interrupted thread is no longer pinned underneath
* the interrupt thread. This means the interrupt must
* have blocked, and the interrupted thread has been
* unpinned, and has probably been running around the
* system for a while.
*
* Since there is no longer a thread under this one, put
* this interrupt thread back on the CPU's free list and
* resume the idle thread which will dispatch the next
* thread to run.
*/
cpu->cpu_stats.sys.intrblk++;
/*
* Put thread back on the interrupt thread list.
* This was an interrupt thread, so set CPU's base SPL.
*/
set_base_spl();
basespl = cpu->cpu_base_spl;
mcpu->mcpu_pri = basespl;
(*setlvlx)(basespl, 0);
it->t_state = TS_FREE;
/*
* Return interrupt thread to pool
*/
it->t_link = cpu->cpu_intr_thread;
cpu->cpu_intr_thread = it;
(void) splhigh();
sti();
swtch();
/*NOTREACHED*/
panic("dosoftint_epilog: swtch returned");
}
/*
* Return interrupt thread to the pool
*/
it->t_link = cpu->cpu_intr_thread;
cpu->cpu_intr_thread = it;
it->t_state = TS_FREE;
cpu->cpu_thread = t;
if (t->t_flag & T_INTR_THREAD)
t->t_intr_start = now;
basespl = cpu->cpu_base_spl;
mcpu->mcpu_pri = MAX(oldpil, basespl);
(*setlvlx)(mcpu->mcpu_pri, 0);
}
timestamp_t timestamp_ticks_since(timestamp_t timestamp) {
return tsc_read() - (timestamp_t)(timestamp);
}
uint8_t timestamp_expired(timestamp_t timeout) {
return timestamp_after(tsc_read(), timeout);
}
/*
* Called by the master in the TSC sync operation (usually the boot CPU).
* If the slave is discovered to have a skew, gethrtimef will be changed to
* point to tsc_gethrtime_delta(). Calculating skews is precise only when
* the master and slave TSCs are read simultaneously; however, there is no
* algorithm that can read both CPUs in perfect simultaneity. The proposed
* algorithm is an approximate method based on the behaviour of cache
* management. The slave CPU continuously reads TSC and then reads a global
* variable which the master CPU updates. The moment the master's update reaches
* the slave's visibility (being forced by an mfence operation) we use the TSC
* reading taken on the slave. A corresponding TSC read will be taken on the
* master as soon as possible after finishing the mfence operation. But the
* delay between causing the slave to notice the invalid cache line and the
* competion of mfence is not repeatable. This error is heuristically assumed
* to be 1/4th of the total write time as being measured by the two TSC reads
* on the master sandwiching the mfence. Furthermore, due to the nature of
* bus arbitration, contention on memory bus, etc., the time taken for the write
* to reflect globally can vary a lot. So instead of taking a single reading,
* a set of readings are taken and the one with least write time is chosen
* to calculate the final skew.
*
* TSC sync is disabled in the context of virtualization because the CPUs
* assigned to the guest are virtual CPUs which means the real CPUs on which
* guest runs keep changing during life time of guest OS. So we would end up
* calculating TSC skews for a set of CPUs during boot whereas the guest
* might migrate to a different set of physical CPUs at a later point of
* time.
*/
void
tsc_sync_master(processorid_t slave)
{
ulong_t flags, source, min_write_time = ~0UL;
hrtime_t write_time, x, mtsc_after, tdelta;
tsc_sync_t *tsc = tscp;
int cnt;
int hwtype;
hwtype = get_hwenv();
if (!tsc_master_slave_sync_needed || hwtype == HW_XEN_HVM ||
hwtype == HW_VMWARE)
return;
flags = clear_int_flag();
source = CPU->cpu_id;
for (cnt = 0; cnt < SYNC_ITERATIONS; cnt++) {
while (tsc_sync_go != TSC_SYNC_GO)
SMT_PAUSE();
tsc->master_tsc = tsc_read();
membar_enter();
mtsc_after = tsc_read();
while (tsc_sync_go != TSC_SYNC_DONE)
SMT_PAUSE();
write_time = mtsc_after - tsc->master_tsc;
if (write_time <= min_write_time) {
min_write_time = write_time;
/*
* Apply heuristic adjustment only if the calculated
* delta is > 1/4th of the write time.
*/
x = tsc->slave_tsc - mtsc_after;
if (x < 0)
x = -x;
if (x > (min_write_time/4))
/*
* Subtract 1/4th of the measured write time
* from the master's TSC value, as an estimate
* of how late the mfence completion came
* after the slave noticed the cache line
* change.
*/
tdelta = tsc->slave_tsc -
(mtsc_after - (min_write_time/4));
else
tdelta = tsc->slave_tsc - mtsc_after;
tsc_sync_tick_delta[slave] =
tsc_sync_tick_delta[source] - tdelta;
}
tsc->master_tsc = tsc->slave_tsc = write_time = 0;
membar_enter();
tsc_sync_go = TSC_SYNC_STOP;
}
if (tdelta < 0)
tdelta = -tdelta;
if (tdelta > largest_tsc_delta)
largest_tsc_delta = tdelta;
if (min_write_time < shortest_write_time)
shortest_write_time = min_write_time;
/*
* Enable delta variants of tsc functions if the largest of all chosen
* deltas is > smallest of the write time.
*/
if (largest_tsc_delta > shortest_write_time) {
gethrtimef = tsc_gethrtime_delta;
gethrtimeunscaledf = tsc_gethrtimeunscaled_delta;
}
restore_int_flag(flags);
}
static caddr_t
dosoftint_prolog(
struct cpu *cpu,
caddr_t stackptr,
uint32_t st_pending,
uint_t oldpil)
{
kthread_t *t, *volatile it;
struct machcpu *mcpu = &cpu->cpu_m;
uint_t pil;
hrtime_t now;
top:
ASSERT(st_pending == mcpu->mcpu_softinfo.st_pending);
pil = bsrw_insn((uint16_t)st_pending);
if (pil <= oldpil || pil <= cpu->cpu_base_spl)
return (0);
/*
* XX64 Sigh.
*
* This is a transliteration of the i386 assembler code for
* soft interrupts. One question is "why does this need
* to be atomic?" One possible race is -other- processors
* posting soft interrupts to us in set_pending() i.e. the
* CPU might get preempted just after the address computation,
* but just before the atomic transaction, so another CPU would
* actually set the original CPU's st_pending bit. However,
* it looks like it would be simpler to disable preemption there.
* Are there other races for which preemption control doesn't work?
*
* The i386 assembler version -also- checks to see if the bit
* being cleared was actually set; if it wasn't, it rechecks
* for more. This seems a bit strange, as the only code that
* ever clears the bit is -this- code running with interrupts
* disabled on -this- CPU. This code would probably be cheaper:
*
* atomic_and_32((uint32_t *)&mcpu->mcpu_softinfo.st_pending,
* ~(1 << pil));
*
* and t->t_preempt--/++ around set_pending() even cheaper,
* but at this point, correctness is critical, so we slavishly
* emulate the i386 port.
*/
if (atomic_btr32((uint32_t *)
&mcpu->mcpu_softinfo.st_pending, pil) == 0) {
st_pending = mcpu->mcpu_softinfo.st_pending;
goto top;
}
mcpu->mcpu_pri = pil;
(*setspl)(pil);
now = tsc_read();
/*
* Get set to run interrupt thread.
* There should always be an interrupt thread since we
* allocate one for each level on the CPU.
*/
it = cpu->cpu_intr_thread;
cpu->cpu_intr_thread = it->t_link;
/* t_intr_start could be zero due to cpu_intr_swtch_enter. */
t = cpu->cpu_thread;
if ((t->t_flag & T_INTR_THREAD) && t->t_intr_start != 0) {
hrtime_t intrtime = now - t->t_intr_start;
mcpu->intrstat[pil][0] += intrtime;
cpu->cpu_intracct[cpu->cpu_mstate] += intrtime;
t->t_intr_start = 0;
}
/*
* Note that the code in kcpc_overflow_intr -relies- on the
* ordering of events here - in particular that t->t_lwp of
* the interrupt thread is set to the pinned thread *before*
* curthread is changed.
*/
it->t_lwp = t->t_lwp;
it->t_state = TS_ONPROC;
/*
* Push interrupted thread onto list from new thread.
* Set the new thread as the current one.
* Set interrupted thread's T_SP because if it is the idle thread,
* resume() may use that stack between threads.
*/
ASSERT(SA((uintptr_t)stackptr) == (uintptr_t)stackptr);
t->t_sp = (uintptr_t)stackptr;
it->t_intr = t;
cpu->cpu_thread = it;
/*
* Set bit for this pil in CPU's interrupt active bitmask.
*/
ASSERT((cpu->cpu_intr_actv & (1 << pil)) == 0);
cpu->cpu_intr_actv |= (1 << pil);
/*
* Initialize thread priority level from intr_pri
*/
it->t_pil = (uchar_t)pil;
it->t_pri = (pri_t)pil + intr_pri;
it->t_intr_start = now;
return (it->t_stk);
}
