/*
* Synchronize ar.itc of the current (slave) CPU with the ar.itc of the MASTER CPU
* (normally the time-keeper CPU). We use a closed loop to eliminate the possibility of
* unaccounted-for errors (such as getting a machine check in the middle of a calibration
* step). The basic idea is for the slave to ask the master what itc value it has and to
* read its own itc before and after the master responds. Each iteration gives us three
* timestamps:
*
* slave master
*
* t0 ---\
* ---\
* --->
* tm
* /---
* /---
* t1 <---
*
*
* The goal is to adjust the slave's ar.itc such that tm falls exactly half-way between t0
* and t1. If we achieve this, the clocks are synchronized provided the interconnect
* between the slave and the master is symmetric. Even if the interconnect were
* asymmetric, we would still know that the synchronization error is smaller than the
* roundtrip latency (t0 - t1).
*
* When the interconnect is quiet and symmetric, this lets us synchronize the itc to
* within one or two cycles. However, we can only *guarantee* that the synchronization is
* accurate to within a round-trip time, which is typically in the range of several
* hundred cycles (e.g., ~500 cycles). In practice, this means that the itc's are usually
* almost perfectly synchronized, but we shouldn't assume that the accuracy is much better
* than half a micro second or so.
*/
void
ia64_sync_itc (unsigned int master)
{
long i, delta, adj, adjust_latency = 0, done = 0;
unsigned long flags, rt, master_time_stamp, bound;
#if DEBUG_ITC_SYNC
struct {
long rt; /* roundtrip time */
long master; /* master's timestamp */
long diff; /* difference between midpoint and master's timestamp */
long lat; /* estimate of itc adjustment latency */
} t[NUM_ROUNDS];
#endif
go[MASTER] = 1;
if (smp_call_function_single(master, sync_master, NULL, 1, 0) < 0) {
printk("sync_itc: failed to get attention of CPU %u!\n", master);
return;
}
while (go[MASTER]); /* wait for master to be ready */
spin_lock_irqsave(&itc_sync_lock, flags);
{
for (i = 0; i < NUM_ROUNDS; ++i) {
delta = get_delta(&rt, &master_time_stamp);
if (delta == 0) {
done = 1; /* let's lock on to this... */
bound = rt;
}
if (!done) {
if (i > 0) {
adjust_latency += -delta;
adj = -delta + adjust_latency/4;
} else
adj = -delta;
ia64_set_itc(ia64_get_itc() + adj);
}
#if DEBUG_ITC_SYNC
t[i].rt = rt;
t[i].master = master_time_stamp;
t[i].diff = delta;
t[i].lat = adjust_latency/4;
#endif
}
}
spin_unlock_irqrestore(&itc_sync_lock, flags);
#if DEBUG_ITC_SYNC
for (i = 0; i < NUM_ROUNDS; ++i)
printk("rt=%5ld master=%5ld diff=%5ld adjlat=%5ld\n",
t[i].rt, t[i].master, t[i].diff, t[i].lat);
#endif
printk("CPU %d: synchronized ITC with CPU %u (last diff %ld cycles, maxerr %lu cycles)\n",
smp_processor_id(), master, delta, rt);
}
/*
* Synchronize ar.itc of the current (slave) CPU with the ar.itc of the MASTER CPU
* (normally the time-keeper CPU). We use a closed loop to eliminate the possibility of
* unaccounted-for errors (such as getting a machine check in the middle of a calibration
* step). The basic idea is for the slave to ask the master what itc value it has and to
* read its own itc before and after the master responds. Each iteration gives us three
* timestamps:
*
* slave master
*
* t0 ---\
* ---\
* --->
* tm
* /---
* /---
* t1 <---
*
*
* The goal is to adjust the slave's ar.itc such that tm falls exactly half-way between t0
* and t1. If we achieve this, the clocks are synchronized provided the interconnect
* between the slave and the master is symmetric. Even if the interconnect were
* asymmetric, we would still know that the synchronization error is smaller than the
* roundtrip latency (t0 - t1).
*
* When the interconnect is quiet and symmetric, this lets us synchronize the itc to
* within one or two cycles. However, we can only *guarantee* that the synchronization is
* accurate to within a round-trip time, which is typically in the range of several
* hundred cycles (e.g., ~500 cycles). In practice, this means that the itc's are usually
* almost perfectly synchronized, but we shouldn't assume that the accuracy is much better
* than half a micro second or so.
*/
void
ia64_sync_itc (unsigned int master)
{
long i, delta, adj, adjust_latency = 0, done = 0;
unsigned long flags, rt, master_time_stamp, bound;
#if DEBUG_ITC_SYNC
struct {
long rt; /* roundtrip time */
long master; /* master's timestamp */
long diff; /* difference between midpoint and master's timestamp */
long lat; /* estimate of itc adjustment latency */
} t[NUM_ROUNDS];
#endif
/*
* Make sure local timer ticks are disabled while we sync. If
* they were enabled, we'd have to worry about nasty issues
* like setting the ITC ahead of (or a long time before) the
* next scheduled tick.
*/
BUG_ON((ia64_get_itv() & (1 << 16)) == 0);
go[MASTER] = 1;
if (smp_call_function_single(master, sync_master, NULL, 1, 0) < 0) {
printk(KERN_ERR "sync_itc: failed to get attention of CPU %u!\n", master);
return;
}
while (go[MASTER])
cpu_relax(); /* wait for master to be ready */
spin_lock_irqsave(&itc_sync_lock, flags);
{
for (i = 0; i < NUM_ROUNDS; ++i) {
delta = get_delta(&rt, &master_time_stamp);
if (delta == 0) {
done = 1; /* let's lock on to this... */
bound = rt;
}
if (!done) {
if (i > 0) {
adjust_latency += -delta;
adj = -delta + adjust_latency/4;
} else
adj = -delta;
ia64_set_itc(ia64_get_itc() + adj);
}
#if DEBUG_ITC_SYNC
t[i].rt = rt;
t[i].master = master_time_stamp;
t[i].diff = delta;
t[i].lat = adjust_latency/4;
#endif
}
}
spin_unlock_irqrestore(&itc_sync_lock, flags);
#if DEBUG_ITC_SYNC
for (i = 0; i < NUM_ROUNDS; ++i)
printk("rt=%5ld master=%5ld diff=%5ld adjlat=%5ld\n",
t[i].rt, t[i].master, t[i].diff, t[i].lat);
#endif
printk(KERN_INFO "CPU %d: synchronized ITC with CPU %u (last diff %ld cycles, "
"maxerr %lu cycles)\n", smp_processor_id(), master, delta, rt);
}
/*
* cpu_init() initializes state that is per-CPU. This function acts
* as a 'CPU state barrier', nothing should get across.
*/
void
cpu_init (void)
{
extern void __devinit ia64_mmu_init (void *);
unsigned long num_phys_stacked;
pal_vm_info_2_u_t vmi;
unsigned int max_ctx;
struct cpuinfo_ia64 *cpu_info;
void *cpu_data;
cpu_data = per_cpu_init();
get_max_cacheline_size();
/*
* We can't pass "local_cpu_data" to identify_cpu() because we haven't called
* ia64_mmu_init() yet. And we can't call ia64_mmu_init() first because it
* depends on the data returned by identify_cpu(). We break the dependency by
* accessing cpu_data() through the canonical per-CPU address.
*/
cpu_info = cpu_data + ((char *) &__ia64_per_cpu_var(cpu_info) - __per_cpu_start);
identify_cpu(cpu_info);
#ifdef CONFIG_MCKINLEY
{
# define FEATURE_SET 16
struct ia64_pal_retval iprv;
if (cpu_info->family == 0x1f) {
PAL_CALL_PHYS(iprv, PAL_PROC_GET_FEATURES, 0, FEATURE_SET, 0);
if ((iprv.status == 0) && (iprv.v0 & 0x80) && (iprv.v2 & 0x80))
PAL_CALL_PHYS(iprv, PAL_PROC_SET_FEATURES,
(iprv.v1 | 0x80), FEATURE_SET, 0);
}
}
#endif
/* Clear the stack memory reserved for pt_regs: */
memset(ia64_task_regs(current), 0, sizeof(struct pt_regs));
ia64_set_kr(IA64_KR_FPU_OWNER, 0);
/*
* Initialize default control register to defer all speculative faults. The
* kernel MUST NOT depend on a particular setting of these bits (in other words,
* the kernel must have recovery code for all speculative accesses). Turn on
* dcr.lc as per recommendation by the architecture team. Most IA-32 apps
* shouldn't be affected by this (moral: keep your ia32 locks aligned and you'll
* be fine).
*/
ia64_setreg(_IA64_REG_CR_DCR, ( IA64_DCR_DP | IA64_DCR_DK | IA64_DCR_DX | IA64_DCR_DR
| IA64_DCR_DA | IA64_DCR_DD | IA64_DCR_LC));
atomic_inc(&init_mm.mm_count);
current->active_mm = &init_mm;
if (current->mm)
BUG();
ia64_mmu_init(ia64_imva(cpu_data));
#ifdef CONFIG_IA32_SUPPORT
ia32_cpu_init();
#endif
/* Clear ITC to eliminiate sched_clock() overflows in human time. */
ia64_set_itc(0);
/* disable all local interrupt sources: */
ia64_set_itv(1 << 16);
ia64_set_lrr0(1 << 16);
ia64_set_lrr1(1 << 16);
ia64_setreg(_IA64_REG_CR_PMV, 1 << 16);
ia64_setreg(_IA64_REG_CR_CMCV, 1 << 16);
/* clear TPR & XTP to enable all interrupt classes: */
ia64_setreg(_IA64_REG_CR_TPR, 0);
#ifdef CONFIG_SMP
normal_xtp();
#endif
/* set ia64_ctx.max_rid to the maximum RID that is supported by all CPUs: */
if (ia64_pal_vm_summary(NULL, &vmi) == 0)
max_ctx = (1U << (vmi.pal_vm_info_2_s.rid_size - 3)) - 1;
else {
printk(KERN_WARNING "cpu_init: PAL VM summary failed, assuming 18 RID bits\n");
max_ctx = (1U << 15) - 1; /* use architected minimum */
}
while (max_ctx < ia64_ctx.max_ctx) {
unsigned int old = ia64_ctx.max_ctx;
if (cmpxchg(&ia64_ctx.max_ctx, old, max_ctx) == old)
break;
}
if (ia64_pal_rse_info(&num_phys_stacked, NULL) != 0) {
printk(KERN_WARNING "cpu_init: PAL RSE info failed; assuming 96 physical "
"stacked regs\n");
num_phys_stacked = 96;
}
/* size of physical stacked register partition plus 8 bytes: */
__get_cpu_var(ia64_phys_stacked_size_p8) = num_phys_stacked*8 + 8;
platform_cpu_init();
}
