/*
* Return the number of cycles by which our itc differs from the itc on the master
* (time-keeper) CPU. A positive number indicates our itc is ahead of the master,
* negative that it is behind.
*/
static inline long
get_delta (long *rt, long *master)
{
unsigned long best_t0 = 0, best_t1 = ~0UL, best_tm = 0;
unsigned long tcenter, t0, t1, tm;
long i;
for (i = 0; i < NUM_ITERS; ++i) {
t0 = ia64_get_itc();
go[MASTER] = 1;
while (!(tm = go[SLAVE]))
cpu_relax();
go[SLAVE] = 0;
t1 = ia64_get_itc();
if (t1 - t0 < best_t1 - best_t0)
best_t0 = t0, best_t1 = t1, best_tm = tm;
}
*rt = best_t1 - best_t0;
*master = best_tm - best_t0;
/* average best_t0 and best_t1 without overflow: */
tcenter = (best_t0/2 + best_t1/2);
if (best_t0 % 2 + best_t1 % 2 == 2)
++tcenter;
return tcenter - best_tm;
}
static irqreturn_t
timer_interrupt (int irq, void *dev_id, struct pt_regs *regs)
{
unsigned long new_itm;
if (unlikely(cpu_is_offline(smp_processor_id()))) {
return IRQ_HANDLED;
}
platform_timer_interrupt(irq, dev_id, regs);
new_itm = local_cpu_data->itm_next;
if (!time_after(ia64_get_itc(), new_itm))
printk(KERN_ERR "Oops: timer tick before it's due (itc=%lx,itm=%lx)\n",
ia64_get_itc(), new_itm);
profile_tick(CPU_PROFILING, regs);
while (1) {
update_process_times(user_mode(regs));
new_itm += local_cpu_data->itm_delta;
if (smp_processor_id() == TIME_KEEPER_ID) {
/*
* Here we are in the timer irq handler. We have irqs locally
* disabled, but we don't know if the timer_bh is running on
* another CPU. We need to avoid to SMP race by acquiring the
* xtime_lock.
*/
write_seqlock(&xtime_lock);
do_timer(regs);
local_cpu_data->itm_next = new_itm;
write_sequnlock(&xtime_lock);
} else
local_cpu_data->itm_next = new_itm;
if (time_after(new_itm, ia64_get_itc()))
break;
}
do {
/*
* If we're too close to the next clock tick for
* comfort, we increase the safety margin by
* intentionally dropping the next tick(s). We do NOT
* update itm.next because that would force us to call
* do_timer() which in turn would let our clock run
* too fast (with the potentially devastating effect
* of losing monotony of time).
*/
while (!time_after(new_itm, ia64_get_itc() + local_cpu_data->itm_delta/2))
new_itm += local_cpu_data->itm_delta;
ia64_set_itm(new_itm);
/* double check, in case we got hit by a (slow) PMI: */
} while (time_after_eq(ia64_get_itc(), new_itm));
return IRQ_HANDLED;
}
static void
timer_interrupt(int irq, void *dev_id, struct pt_regs *regs)
{
unsigned long new_itm;
new_itm = local_cpu_data->itm_next;
if (!time_after(ia64_get_itc(), new_itm))
printk(KERN_ERR "Oops: timer tick before it's due (itc=%lx,itm=%lx)\n",
ia64_get_itc(), new_itm);
while (1) {
/*
* Do kernel PC profiling here. We multiply the instruction number by
* four so that we can use a prof_shift of 2 to get instruction-level
* instead of just bundle-level accuracy.
*/
if (!user_mode(regs))
do_profile(regs->cr_iip + 4*ia64_psr(regs)->ri);
#ifdef CONFIG_SMP
smp_do_timer(regs);
#endif
new_itm += local_cpu_data->itm_delta;
if (smp_processor_id() == 0) {
/*
* Here we are in the timer irq handler. We have irqs locally
* disabled, but we don't know if the timer_bh is running on
* another CPU. We need to avoid to SMP race by acquiring the
* xtime_lock.
*/
write_lock(&xtime_lock);
do_timer(regs);
local_cpu_data->itm_next = new_itm;
write_unlock(&xtime_lock);
} else
local_cpu_data->itm_next = new_itm;
if (time_after(new_itm, ia64_get_itc()))
break;
}
do {
/*
* If we're too close to the next clock tick for comfort, we increase the
* saftey margin by intentionally dropping the next tick(s). We do NOT update
* itm.next because that would force us to call do_timer() which in turn would
* let our clock run too fast (with the potentially devastating effect of
* losing monotony of time).
*/
while (!time_after(new_itm, ia64_get_itc() + local_cpu_data->itm_delta/2))
new_itm += local_cpu_data->itm_delta;
ia64_set_itm(new_itm);
/* double check, in case we got hit by a (slow) PMI: */
} while (time_after_eq(ia64_get_itc(), new_itm));
}
/*
* Generic udelay assumes that if preemption is allowed and the thread
* migrates to another CPU, that the ITC values are synchronized across
* all CPUs.
*/
static void
ia64_itc_udelay (unsigned long usecs)
{
unsigned long start = ia64_get_itc();
unsigned long end = start + usecs*local_cpu_data->cyc_per_usec;
while (time_before(ia64_get_itc(), end))
cpu_relax();
}
static irqreturn_t
timer_interrupt (int irq, void *dev_id)
{
unsigned long new_itm;
if (cpu_is_offline(smp_processor_id())) {
return IRQ_HANDLED;
}
platform_timer_interrupt(irq, dev_id);
new_itm = local_cpu_data->itm_next;
if (!time_after(ia64_get_itc(), new_itm))
printk(KERN_ERR "Oops: timer tick before it's due (itc=%lx,itm=%lx)\n",
ia64_get_itc(), new_itm);
profile_tick(CPU_PROFILING);
while (1) {
update_process_times(user_mode(get_irq_regs()));
new_itm += local_cpu_data->itm_delta;
if (smp_processor_id() == time_keeper_id)
xtime_update(1);
local_cpu_data->itm_next = new_itm;
if (time_after(new_itm, ia64_get_itc()))
break;
/*
* Allow IPIs to interrupt the timer loop.
*/
local_irq_enable();
local_irq_disable();
}
do {
/*
* If we're too close to the next clock tick for
* comfort, we increase the safety margin by
* intentionally dropping the next tick(s). We do NOT
* update itm.next because that would force us to call
* xtime_update() which in turn would let our clock run
* too fast (with the potentially devastating effect
* of losing monotony of time).
*/
while (!time_after(new_itm, ia64_get_itc() + local_cpu_data->itm_delta/2))
new_itm += local_cpu_data->itm_delta;
ia64_set_itm(new_itm);
/* double check, in case we got hit by a (slow) PMI: */
} while (time_after_eq(ia64_get_itc(), new_itm));
return IRQ_HANDLED;
}
static void
sn2_ipi_flush_all_tlb(struct mm_struct *mm)
{
unsigned long itc;
itc = ia64_get_itc();
smp_flush_tlb_cpumask(*mm_cpumask(mm));
itc = ia64_get_itc() - itc;
__this_cpu_add(ptcstats.shub_ipi_flushes_itc_clocks, itc);
__this_cpu_inc(ptcstats.shub_ipi_flushes);
}
static void
sn2_ipi_flush_all_tlb(struct mm_struct *mm)
{
unsigned long itc;
itc = ia64_get_itc();
smp_flush_tlb_cpumask(*mm_cpumask(mm));
itc = ia64_get_itc() - itc;
__get_cpu_var(ptcstats).shub_ipi_flushes_itc_clocks += itc;
__get_cpu_var(ptcstats).shub_ipi_flushes++;
}
/*
* Transfer the bte_test_buffer from our node to the specified
* destination and print out timing results.
*/
static void
brt_time_xfer(int dest_node, int iterations, int xfer_lines)
{
int iteration;
char *src, *dst;
u64 xfer_len, src_phys, dst_phys;
u64 itc_before, itc_after, mem_intvl, bte_intvl;
xfer_len = xfer_lines * L1_CACHE_BYTES;
src = nodepda->bte_if[0].bte_test_buf;
src_phys = __pa(src);
dst = NODEPDA(dest_node)->bte_if[1].bte_test_buf;
dst_phys = __pa(dst);
mem_intvl = 0;
for (iteration = 0; iteration < iterations; iteration++) {
if (tm_memcpy) {
itc_before = ia64_get_itc();
memcpy(dst, src, xfer_len);
itc_after = ia64_get_itc();
mem_intvl = itc_after - itc_before;
}
itc_before = ia64_get_itc();
bte_copy(src_phys, dst_phys, xfer_len, BTE_NOTIFY, NULL);
itc_after = ia64_get_itc();
bte_intvl = itc_after - itc_before;
if (tm_memcpy) {
printk("%3d,%3d,%3d,%5d,%4ld,%7ld,%3ld,"
"%7ld,%7ld,%7ld\n",
smp_processor_id(), NASID_GET(src),
NASID_GET(dst), xfer_lines,
NSEC(bte_setup_time),
NSEC(bte_transfer_time),
NSEC(bte_tear_down_time),
NSEC(bte_execute_time), NSEC(bte_intvl),
NSEC(mem_intvl));
} else {
printk("%3d,%3d,%3d,%5d,%4ld,%7ld,%3ld,"
"%7ld,%7ld\n",
smp_processor_id(), NASID_GET(src),
NASID_GET(dst), xfer_lines,
NSEC(bte_setup_time),
NSEC(bte_transfer_time),
NSEC(bte_tear_down_time),
NSEC(bte_execute_time), NSEC(bte_intvl));
}
}
}
/*
* Synchronize ar.itc of the current (slave) CPU with the ar.itc of the MASTER CPU
* (normally the time-keeper CPU). We use a closed loop to eliminate the possibility of
* unaccounted-for errors (such as getting a machine check in the middle of a calibration
* step). The basic idea is for the slave to ask the master what itc value it has and to
* read its own itc before and after the master responds. Each iteration gives us three
* timestamps:
*
* slave master
*
* t0 ---\
* ---\
* --->
* tm
* /---
* /---
* t1 <---
*
*
* The goal is to adjust the slave's ar.itc such that tm falls exactly half-way between t0
* and t1. If we achieve this, the clocks are synchronized provided the interconnect
* between the slave and the master is symmetric. Even if the interconnect were
* asymmetric, we would still know that the synchronization error is smaller than the
* roundtrip latency (t0 - t1).
*
* When the interconnect is quiet and symmetric, this lets us synchronize the itc to
* within one or two cycles. However, we can only *guarantee* that the synchronization is
* accurate to within a round-trip time, which is typically in the range of several
* hundred cycles (e.g., ~500 cycles). In practice, this means that the itc's are usually
* almost perfectly synchronized, but we shouldn't assume that the accuracy is much better
* than half a micro second or so.
*/
void
ia64_sync_itc (unsigned int master)
{
long i, delta, adj, adjust_latency = 0, done = 0;
unsigned long flags, rt, master_time_stamp, bound;
#if DEBUG_ITC_SYNC
struct {
long rt; /* roundtrip time */
long master; /* master's timestamp */
long diff; /* difference between midpoint and master's timestamp */
long lat; /* estimate of itc adjustment latency */
} t[NUM_ROUNDS];
#endif
go[MASTER] = 1;
if (smp_call_function_single(master, sync_master, NULL, 1, 0) < 0) {
printk("sync_itc: failed to get attention of CPU %u!\n", master);
return;
}
while (go[MASTER]); /* wait for master to be ready */
spin_lock_irqsave(&itc_sync_lock, flags);
{
for (i = 0; i < NUM_ROUNDS; ++i) {
delta = get_delta(&rt, &master_time_stamp);
if (delta == 0) {
done = 1; /* let's lock on to this... */
bound = rt;
}
if (!done) {
if (i > 0) {
adjust_latency += -delta;
adj = -delta + adjust_latency/4;
} else
adj = -delta;
ia64_set_itc(ia64_get_itc() + adj);
}
#if DEBUG_ITC_SYNC
t[i].rt = rt;
t[i].master = master_time_stamp;
t[i].diff = delta;
t[i].lat = adjust_latency/4;
#endif
}
}
spin_unlock_irqrestore(&itc_sync_lock, flags);
#if DEBUG_ITC_SYNC
for (i = 0; i < NUM_ROUNDS; ++i)
printk("rt=%5ld master=%5ld diff=%5ld adjlat=%5ld\n",
t[i].rt, t[i].master, t[i].diff, t[i].lat);
#endif
printk("CPU %d: synchronized ITC with CPU %u (last diff %ld cycles, maxerr %lu cycles)\n",
smp_processor_id(), master, delta, rt);
}
static u_int
ia64_ih_clock(struct thread *td, u_int xiv, struct trapframe *tf)
{
struct eventtimer *et;
uint64_t itc, load;
uint32_t mode;
PCPU_INC(md.stats.pcs_nclks);
intrcnt[INTRCNT_CLOCK]++;
itc = ia64_get_itc();
PCPU_SET(md.clock, itc);
mode = PCPU_GET(md.clock_mode);
if (mode == CLOCK_ET_PERIODIC) {
load = PCPU_GET(md.clock_load);
ia64_set_itm(itc + load);
} else
ia64_set_itv((1 << 16) | xiv);
ia64_set_eoi(0);
ia64_srlz_d();
et = &ia64_clock_et;
if (et->et_active)
et->et_event_cb(et, et->et_arg);
return (1);
}
static int xen_do_steal_accounting(unsigned long *new_itm)
{
unsigned long delta_itm;
delta_itm = consider_steal_time(*new_itm);
*new_itm += delta_itm;
if (time_after(*new_itm, ia64_get_itc()) && delta_itm)
return 1;
return 0;
}
void
pcpu_initclock(void)
{
PCPU_SET(clockadj, 0);
PCPU_SET(clock, ia64_get_itc());
ia64_set_itm(PCPU_GET(clock) + ia64_clock_reload);
ia64_set_itv(CLOCK_VECTOR); /* highest priority class */
ia64_srlz_d();
}
/*
* Account time for a transition between system, hard irq or soft irq state.
* Note that this function is called with interrupts enabled.
*/
static __u64 vtime_delta(struct task_struct *tsk)
{
struct thread_info *ti = task_thread_info(tsk);
__u64 now, delta_stime;
WARN_ON_ONCE(!irqs_disabled());
now = ia64_get_itc();
delta_stime = now - ti->ac_stamp;
ti->ac_stamp = now;
return delta_stime;
}
/*
* Account time for a transition between system, hard irq or soft irq state.
* Note that this function is called with interrupts enabled.
*/
static cputime_t vtime_delta(struct task_struct *tsk)
{
struct thread_info *ti = task_thread_info(tsk);
cputime_t delta_stime;
__u64 now;
WARN_ON_ONCE(!irqs_disabled());
now = ia64_get_itc();
delta_stime = cycle_to_cputime(ti->ac_stime + (now - ti->ac_stamp));
ti->ac_stime = 0;
ti->ac_stamp = now;
return delta_stime;
}
void
sync_master (void *arg)
{
unsigned long flags, i;
go[MASTER] = 0;
local_irq_save(flags);
{
for (i = 0; i < NUM_ROUNDS*NUM_ITERS; ++i) {
while (!go[MASTER]);
go[MASTER] = 0;
go[SLAVE] = ia64_get_itc();
}
}
local_irq_restore(flags);
}
/*
* Return the number of micro-seconds that elapsed since the last update to jiffy. The
* xtime_lock must be at least read-locked when calling this routine.
*/
static inline unsigned long
gettimeoffset (void)
{
unsigned long elapsed_cycles, lost = jiffies - wall_jiffies;
unsigned long now, last_tick;
# define time_keeper_id 0 /* smp_processor_id() of time-keeper */
last_tick = (cpu_data(time_keeper_id)->itm_next
- (lost + 1)*cpu_data(time_keeper_id)->itm_delta);
now = ia64_get_itc();
if ((long) (now - last_tick) < 0) {
printk(KERN_ERR "CPU %d: now < last_tick (now=0x%lx,last_tick=0x%lx)!\n",
smp_processor_id(), now, last_tick);
return last_time_offset;
}
elapsed_cycles = now - last_tick;
return (elapsed_cycles*local_cpu_data->usec_per_cyc) >> IA64_USEC_PER_CYC_SHIFT;
}
/*
* Encapsulate access to the itm structure for SMP.
*/
void
ia64_cpu_local_tick (void)
{
int cpu = smp_processor_id();
unsigned long shift = 0, delta;
/* arrange for the cycle counter to generate a timer interrupt: */
ia64_set_itv(IA64_TIMER_VECTOR);
delta = local_cpu_data->itm_delta;
/*
* Stagger the timer tick for each CPU so they don't occur all at (almost) the
* same time:
*/
if (cpu) {
unsigned long hi = 1UL << ia64_fls(cpu);
shift = (2*(cpu - hi) + 1) * delta/hi/2;
}
local_cpu_data->itm_next = ia64_get_itc() + delta + shift;
ia64_set_itm(local_cpu_data->itm_next);
}
/*
* Account time for a transition between system, hard irq or soft irq state.
* Note that this function is called with interrupts enabled.
*/
void account_system_vtime(struct task_struct *tsk)
{
struct thread_info *ti = task_thread_info(tsk);
unsigned long flags;
cputime_t delta_stime;
__u64 now;
local_irq_save(flags);
now = ia64_get_itc();
delta_stime = cycle_to_cputime(ti->ac_stime + (now - ti->ac_stamp));
if (irq_count() || idle_task(smp_processor_id()) != tsk)
account_system_time(tsk, 0, delta_stime, delta_stime);
else
account_idle_time(delta_stime);
ti->ac_stime = 0;
ti->ac_stamp = now;
local_irq_restore(flags);
}
/*
* Called from the context switch with interrupts disabled, to charge all
* accumulated times to the current process, and to prepare accounting on
* the next process.
*/
void ia64_account_on_switch(struct task_struct *prev, struct task_struct *next)
{
struct thread_info *pi = task_thread_info(prev);
struct thread_info *ni = task_thread_info(next);
cputime_t delta_stime, delta_utime;
__u64 now;
now = ia64_get_itc();
delta_stime = cycle_to_cputime(pi->ac_stime + (now - pi->ac_stamp));
if (idle_task(smp_processor_id()) != prev)
account_system_time(prev, 0, delta_stime, delta_stime);
else
account_idle_time(delta_stime);
if (pi->ac_utime) {
delta_utime = cycle_to_cputime(pi->ac_utime);
account_user_time(prev, delta_utime, delta_utime);
}
pi->ac_stamp = ni->ac_stamp = now;
ni->ac_stime = ni->ac_utime = 0;
}
static void __init
check_sal_cache_flush (void)
{
unsigned long flags;
int cpu;
u64 vector;
cpu = get_cpu();
local_irq_save(flags);
/*
* Schedule a timer interrupt, wait until it's reported, and see if
* SAL_CACHE_FLUSH drops it.
*/
ia64_set_itv(IA64_TIMER_VECTOR);
ia64_set_itm(ia64_get_itc() + 1000);
while (!ia64_get_irr(IA64_TIMER_VECTOR))
cpu_relax();
ia64_sal_cache_flush(3);
if (ia64_get_irr(IA64_TIMER_VECTOR)) {
vector = ia64_get_ivr();
ia64_eoi();
WARN_ON(vector != IA64_TIMER_VECTOR);
} else {
sal_cache_flush_drops_interrupts = 1;
printk(KERN_ERR "SAL: SAL_CACHE_FLUSH drops interrupts; "
"PAL_CACHE_FLUSH will be used instead\n");
ia64_eoi();
}
local_irq_restore(flags);
put_cpu();
}
static void rthal_adjust_before_relay(unsigned irq, void *cookie)
{
rthal_itm_next[rthal_processor_id()] = ia64_get_itc();
rthal_propagate_irq(irq);
}
static u_int
ia64_get_timecount(struct timecounter* tc)
{
return ia64_get_itc();
}
/*
* Synchronize ar.itc of the current (slave) CPU with the ar.itc of the MASTER CPU
* (normally the time-keeper CPU). We use a closed loop to eliminate the possibility of
* unaccounted-for errors (such as getting a machine check in the middle of a calibration
* step). The basic idea is for the slave to ask the master what itc value it has and to
* read its own itc before and after the master responds. Each iteration gives us three
* timestamps:
*
* slave master
*
* t0 ---\
* ---\
* --->
* tm
* /---
* /---
* t1 <---
*
*
* The goal is to adjust the slave's ar.itc such that tm falls exactly half-way between t0
* and t1. If we achieve this, the clocks are synchronized provided the interconnect
* between the slave and the master is symmetric. Even if the interconnect were
* asymmetric, we would still know that the synchronization error is smaller than the
* roundtrip latency (t0 - t1).
*
* When the interconnect is quiet and symmetric, this lets us synchronize the itc to
* within one or two cycles. However, we can only *guarantee* that the synchronization is
* accurate to within a round-trip time, which is typically in the range of several
* hundred cycles (e.g., ~500 cycles). In practice, this means that the itc's are usually
* almost perfectly synchronized, but we shouldn't assume that the accuracy is much better
* than half a micro second or so.
*/
void
ia64_sync_itc (unsigned int master)
{
long i, delta, adj, adjust_latency = 0, done = 0;
unsigned long flags, rt, master_time_stamp, bound;
#if DEBUG_ITC_SYNC
struct {
long rt; /* roundtrip time */
long master; /* master's timestamp */
long diff; /* difference between midpoint and master's timestamp */
long lat; /* estimate of itc adjustment latency */
} t[NUM_ROUNDS];
#endif
/*
* Make sure local timer ticks are disabled while we sync. If
* they were enabled, we'd have to worry about nasty issues
* like setting the ITC ahead of (or a long time before) the
* next scheduled tick.
*/
BUG_ON((ia64_get_itv() & (1 << 16)) == 0);
go[MASTER] = 1;
if (smp_call_function_single(master, sync_master, NULL, 1, 0) < 0) {
printk(KERN_ERR "sync_itc: failed to get attention of CPU %u!\n", master);
return;
}
while (go[MASTER])
cpu_relax(); /* wait for master to be ready */
spin_lock_irqsave(&itc_sync_lock, flags);
{
for (i = 0; i < NUM_ROUNDS; ++i) {
delta = get_delta(&rt, &master_time_stamp);
if (delta == 0) {
done = 1; /* let's lock on to this... */
bound = rt;
}
if (!done) {
if (i > 0) {
adjust_latency += -delta;
adj = -delta + adjust_latency/4;
} else
adj = -delta;
ia64_set_itc(ia64_get_itc() + adj);
}
#if DEBUG_ITC_SYNC
t[i].rt = rt;
t[i].master = master_time_stamp;
t[i].diff = delta;
t[i].lat = adjust_latency/4;
#endif
}
}
spin_unlock_irqrestore(&itc_sync_lock, flags);
#if DEBUG_ITC_SYNC
for (i = 0; i < NUM_ROUNDS; ++i)
printk("rt=%5ld master=%5ld diff=%5ld adjlat=%5ld\n",
t[i].rt, t[i].master, t[i].diff, t[i].lat);
#endif
printk(KERN_INFO "CPU %d: synchronized ITC with CPU %u (last diff %ld cycles, "
"maxerr %lu cycles)\n", smp_processor_id(), master, delta, rt);
}
static unsigned
ia64_get_timecount(struct timecounter* tc)
{
return ia64_get_itc();
}
static void rthal_set_itv(void)
{
rthal_itm_next[rthal_processor_id()] = ia64_get_itc();
ia64_set_itv(irq_to_vector(rthal_tick_irq));
}
static unsigned long
consider_steal_time(unsigned long new_itm)
{
unsigned long stolen, blocked;
unsigned long delta_itm = 0, stolentick = 0;
int cpu = smp_processor_id();
struct vcpu_runstate_info runstate;
struct task_struct *p = current;
get_runstate_snapshot(&runstate);
/*
* Check for vcpu migration effect
* In this case, itc value is reversed.
* This causes huge stolen value.
* This function just checks and reject this effect.
*/
if (!time_after_eq(runstate.time[RUNSTATE_blocked],
per_cpu(xen_blocked_time, cpu)))
blocked = 0;
if (!time_after_eq(runstate.time[RUNSTATE_runnable] +
runstate.time[RUNSTATE_offline],
per_cpu(xen_stolen_time, cpu)))
stolen = 0;
if (!time_after(delta_itm + new_itm, ia64_get_itc()))
stolentick = ia64_get_itc() - new_itm;
do_div(stolentick, NS_PER_TICK);
stolentick++;
do_div(stolen, NS_PER_TICK);
if (stolen > stolentick)
stolen = stolentick;
stolentick -= stolen;
do_div(blocked, NS_PER_TICK);
if (blocked > stolentick)
blocked = stolentick;
if (stolen > 0 || blocked > 0) {
account_steal_ticks(stolen);
account_idle_ticks(blocked);
run_local_timers();
rcu_check_callbacks(cpu, user_mode(get_irq_regs()));
scheduler_tick();
run_posix_cpu_timers(p);
delta_itm += local_cpu_data->itm_delta * (stolen + blocked);
if (cpu == time_keeper_id)
xtime_update(stolen + blocked);
local_cpu_data->itm_next = delta_itm + new_itm;
per_cpu(xen_stolen_time, cpu) += NS_PER_TICK * stolen;
per_cpu(xen_blocked_time, cpu) += NS_PER_TICK * blocked;
}
return delta_itm;
}
void
interrupt(u_int64_t vector, struct trapframe *framep)
{
struct thread *td;
volatile struct ia64_interrupt_block *ib = IA64_INTERRUPT_BLOCK;
td = curthread;
atomic_add_int(&td->td_intr_nesting_level, 1);
/*
* Handle ExtINT interrupts by generating an INTA cycle to
* read the vector.
*/
if (vector == 0) {
vector = ib->ib_inta;
printf("ExtINT interrupt: vector=%ld\n", vector);
}
if (vector == 255) {/* clock interrupt */
/* CTR0(KTR_INTR, "clock interrupt"); */
cnt.v_intr++;
#ifdef EVCNT_COUNTERS
clock_intr_evcnt.ev_count++;
#else
intrcnt[INTRCNT_CLOCK]++;
#endif
critical_enter();
#ifdef SMP
clks[PCPU_GET(cpuid)]++;
/* Only the BSP runs the real clock */
if (PCPU_GET(cpuid) == 0) {
#endif
handleclock(framep);
/* divide hz (1024) by 8 to get stathz (128) */
if ((++schedclk2 & 0x7) == 0)
statclock((struct clockframe *)framep);
#ifdef SMP
} else {
ia64_set_itm(ia64_get_itc() + itm_reload);
mtx_lock_spin(&sched_lock);
hardclock_process(curthread, TRAPF_USERMODE(framep));
if ((schedclk2 & 0x7) == 0)
statclock_process(curkse, TRAPF_PC(framep),
TRAPF_USERMODE(framep));
mtx_unlock_spin(&sched_lock);
}
#endif
critical_exit();
#ifdef SMP
} else if (vector == ipi_vector[IPI_AST]) {
asts[PCPU_GET(cpuid)]++;
CTR1(KTR_SMP, "IPI_AST, cpuid=%d", PCPU_GET(cpuid));
} else if (vector == ipi_vector[IPI_RENDEZVOUS]) {
rdvs[PCPU_GET(cpuid)]++;
CTR1(KTR_SMP, "IPI_RENDEZVOUS, cpuid=%d", PCPU_GET(cpuid));
smp_rendezvous_action();
} else if (vector == ipi_vector[IPI_STOP]) {
u_int32_t mybit = PCPU_GET(cpumask);
CTR1(KTR_SMP, "IPI_STOP, cpuid=%d", PCPU_GET(cpuid));
savectx(PCPU_GET(pcb));
stopped_cpus |= mybit;
while ((started_cpus & mybit) == 0)
/* spin */;
started_cpus &= ~mybit;
stopped_cpus &= ~mybit;
if (PCPU_GET(cpuid) == 0 && cpustop_restartfunc != NULL) {
void (*f)(void) = cpustop_restartfunc;
cpustop_restartfunc = NULL;
(*f)();
}
} else if (vector == ipi_vector[IPI_TEST]) {
CTR1(KTR_SMP, "IPI_TEST, cpuid=%d", PCPU_GET(cpuid));
mp_ipi_test++;
#endif
} else {
ints[PCPU_GET(cpuid)]++;
ia64_dispatch_intr(framep, vector);
}
atomic_subtract_int(&td->td_intr_nesting_level, 1);
}
/*
* One of these threads is started per cpu. Each thread is responsible
* for loading that cpu's bte interface and then writing to the
* test buffer. The transfers are set in a round-robin fashion.
* The end result is that each test buffer is being written into
* by the previous node and both cpu's at the same time as the
* local bte is transferring it to the next node.
*/
static int
brt_notify_thrd(void *__bind_cpu)
{
int bind_cpu = (long int)__bind_cpu;
int cpu = cpu_logical_map(bind_cpu);
nodepda_t *nxt_node;
long tmout_itc_intvls;
long tmout;
long passes;
long good_xfer_cnt;
u64 src_phys, dst_phys;
int i;
volatile char *src_buf;
u64 *notify;
atomic_inc(&brt_thread_cnt);
daemonize();
set_user_nice(current, 19);
sigfillset(¤t->blocked);
/* Migrate to the right CPU */
set_cpus_allowed(current, 1UL << cpu);
/* Calculate the uSec timeout itc offset. */
tmout_itc_intvls = local_cpu_data->cyc_per_usec * hang_usec;
if (local_cnodeid() == (numnodes - 1)) {
nxt_node = NODEPDA(0);
} else {
nxt_node = NODEPDA(local_cnodeid() + 1);
}
src_buf = nodepda->bte_if[0].bte_test_buf;
src_phys = __pa(src_buf);
dst_phys = __pa(nxt_node->bte_if[0].bte_test_buf);
notify = kmalloc(L1_CACHE_BYTES, GFP_KERNEL);
ASSERT(!((u64) notify & L1_CACHE_MASK));
printk("BTE Hang %d xfer 0x%lx -> 0x%lx, Notify=0x%lx\n",
smp_processor_id(), src_phys, dst_phys, (u64) notify);
passes = 0;
good_xfer_cnt = 0;
/* Loop until signalled to exit. */
while (!brt_exit_flag) {
/*
* A hang will prevent further transfers.
* NOTE: Sometimes, it appears like a hang occurred and
* then transfers begin again. This just means that
* there is NUMA congestion and the hang_usec param
* should be increased.
*/
if (!(*notify & IBLS_BUSY)) {
if ((bte_copy(src_phys,
dst_phys,
4UL * L1_CACHE_BYTES,
BTE_NOTIFY,
(void *)notify)) != BTE_SUCCESS) {
printk("<0>Cpu %d Could not "
"allocate a bte.\n",
smp_processor_id());
continue;
}
tmout = ia64_get_itc() + tmout_itc_intvls;
while ((*notify & IBLS_BUSY) &&
(ia64_get_itc() < tmout)) {
/* Push data out with the processor. */
for (i = 0; i < (4 * L1_CACHE_BYTES);
i += L1_CACHE_BYTES) {
src_buf[i] = (passes % 128);
}
};
if (*notify & IBLS_BUSY) {
printk("<0>Cpu %d BTE appears to have "
"hung.\n", smp_processor_id());
} else {
good_xfer_cnt++;
}
}
/* Every x passes, take a little break. */
if (!(++passes % 40)) {
passes = 0;
schedule_timeout(0.01 * HZ);
}
}
kfree(notify);
printk("Cpu %d had %ld good passes\n",
smp_processor_id(), good_xfer_cnt);
atomic_dec(&brt_thread_cnt);
return (0);
}
