/* * Synchronize ar.itc of the current (slave) CPU with the ar.itc of the MASTER CPU * (normally the time-keeper CPU). We use a closed loop to eliminate the possibility of * unaccounted-for errors (such as getting a machine check in the middle of a calibration * step). The basic idea is for the slave to ask the master what itc value it has and to * read its own itc before and after the master responds. Each iteration gives us three * timestamps: * * slave master * * t0 ---\ * ---\ * ---> * tm * /--- * /--- * t1 <--- * * * The goal is to adjust the slave's ar.itc such that tm falls exactly half-way between t0 * and t1. If we achieve this, the clocks are synchronized provided the interconnect * between the slave and the master is symmetric. Even if the interconnect were * asymmetric, we would still know that the synchronization error is smaller than the * roundtrip latency (t0 - t1). * * When the interconnect is quiet and symmetric, this lets us synchronize the itc to * within one or two cycles. However, we can only *guarantee* that the synchronization is * accurate to within a round-trip time, which is typically in the range of several * hundred cycles (e.g., ~500 cycles). In practice, this means that the itc's are usually * almost perfectly synchronized, but we shouldn't assume that the accuracy is much better * than half a micro second or so. */ void ia64_sync_itc (unsigned int master) { long i, delta, adj, adjust_latency = 0, done = 0; unsigned long flags, rt, master_time_stamp, bound; #if DEBUG_ITC_SYNC struct { long rt; /* roundtrip time */ long master; /* master's timestamp */ long diff; /* difference between midpoint and master's timestamp */ long lat; /* estimate of itc adjustment latency */ } t[NUM_ROUNDS]; #endif go[MASTER] = 1; if (smp_call_function_single(master, sync_master, NULL, 1, 0) < 0) { printk("sync_itc: failed to get attention of CPU %u!\n", master); return; } while (go[MASTER]); /* wait for master to be ready */ spin_lock_irqsave(&itc_sync_lock, flags); { for (i = 0; i < NUM_ROUNDS; ++i) { delta = get_delta(&rt, &master_time_stamp); if (delta == 0) { done = 1; /* let's lock on to this... */ bound = rt; } if (!done) { if (i > 0) { adjust_latency += -delta; adj = -delta + adjust_latency/4; } else adj = -delta; ia64_set_itc(ia64_get_itc() + adj); } #if DEBUG_ITC_SYNC t[i].rt = rt; t[i].master = master_time_stamp; t[i].diff = delta; t[i].lat = adjust_latency/4; #endif } } spin_unlock_irqrestore(&itc_sync_lock, flags); #if DEBUG_ITC_SYNC for (i = 0; i < NUM_ROUNDS; ++i) printk("rt=%5ld master=%5ld diff=%5ld adjlat=%5ld\n", t[i].rt, t[i].master, t[i].diff, t[i].lat); #endif printk("CPU %d: synchronized ITC with CPU %u (last diff %ld cycles, maxerr %lu cycles)\n", smp_processor_id(), master, delta, rt); }
/* * Synchronize ar.itc of the current (slave) CPU with the ar.itc of the MASTER CPU * (normally the time-keeper CPU). We use a closed loop to eliminate the possibility of * unaccounted-for errors (such as getting a machine check in the middle of a calibration * step). The basic idea is for the slave to ask the master what itc value it has and to * read its own itc before and after the master responds. Each iteration gives us three * timestamps: * * slave master * * t0 ---\ * ---\ * ---> * tm * /--- * /--- * t1 <--- * * * The goal is to adjust the slave's ar.itc such that tm falls exactly half-way between t0 * and t1. If we achieve this, the clocks are synchronized provided the interconnect * between the slave and the master is symmetric. Even if the interconnect were * asymmetric, we would still know that the synchronization error is smaller than the * roundtrip latency (t0 - t1). * * When the interconnect is quiet and symmetric, this lets us synchronize the itc to * within one or two cycles. However, we can only *guarantee* that the synchronization is * accurate to within a round-trip time, which is typically in the range of several * hundred cycles (e.g., ~500 cycles). In practice, this means that the itc's are usually * almost perfectly synchronized, but we shouldn't assume that the accuracy is much better * than half a micro second or so. */ void ia64_sync_itc (unsigned int master) { long i, delta, adj, adjust_latency = 0, done = 0; unsigned long flags, rt, master_time_stamp, bound; #if DEBUG_ITC_SYNC struct { long rt; /* roundtrip time */ long master; /* master's timestamp */ long diff; /* difference between midpoint and master's timestamp */ long lat; /* estimate of itc adjustment latency */ } t[NUM_ROUNDS]; #endif /* * Make sure local timer ticks are disabled while we sync. If * they were enabled, we'd have to worry about nasty issues * like setting the ITC ahead of (or a long time before) the * next scheduled tick. */ BUG_ON((ia64_get_itv() & (1 << 16)) == 0); go[MASTER] = 1; if (smp_call_function_single(master, sync_master, NULL, 1, 0) < 0) { printk(KERN_ERR "sync_itc: failed to get attention of CPU %u!\n", master); return; } while (go[MASTER]) cpu_relax(); /* wait for master to be ready */ spin_lock_irqsave(&itc_sync_lock, flags); { for (i = 0; i < NUM_ROUNDS; ++i) { delta = get_delta(&rt, &master_time_stamp); if (delta == 0) { done = 1; /* let's lock on to this... */ bound = rt; } if (!done) { if (i > 0) { adjust_latency += -delta; adj = -delta + adjust_latency/4; } else adj = -delta; ia64_set_itc(ia64_get_itc() + adj); } #if DEBUG_ITC_SYNC t[i].rt = rt; t[i].master = master_time_stamp; t[i].diff = delta; t[i].lat = adjust_latency/4; #endif } } spin_unlock_irqrestore(&itc_sync_lock, flags); #if DEBUG_ITC_SYNC for (i = 0; i < NUM_ROUNDS; ++i) printk("rt=%5ld master=%5ld diff=%5ld adjlat=%5ld\n", t[i].rt, t[i].master, t[i].diff, t[i].lat); #endif printk(KERN_INFO "CPU %d: synchronized ITC with CPU %u (last diff %ld cycles, " "maxerr %lu cycles)\n", smp_processor_id(), master, delta, rt); }
/* * cpu_init() initializes state that is per-CPU. This function acts * as a 'CPU state barrier', nothing should get across. */ void cpu_init (void) { extern void __devinit ia64_mmu_init (void *); unsigned long num_phys_stacked; pal_vm_info_2_u_t vmi; unsigned int max_ctx; struct cpuinfo_ia64 *cpu_info; void *cpu_data; cpu_data = per_cpu_init(); get_max_cacheline_size(); /* * We can't pass "local_cpu_data" to identify_cpu() because we haven't called * ia64_mmu_init() yet. And we can't call ia64_mmu_init() first because it * depends on the data returned by identify_cpu(). We break the dependency by * accessing cpu_data() through the canonical per-CPU address. */ cpu_info = cpu_data + ((char *) &__ia64_per_cpu_var(cpu_info) - __per_cpu_start); identify_cpu(cpu_info); #ifdef CONFIG_MCKINLEY { # define FEATURE_SET 16 struct ia64_pal_retval iprv; if (cpu_info->family == 0x1f) { PAL_CALL_PHYS(iprv, PAL_PROC_GET_FEATURES, 0, FEATURE_SET, 0); if ((iprv.status == 0) && (iprv.v0 & 0x80) && (iprv.v2 & 0x80)) PAL_CALL_PHYS(iprv, PAL_PROC_SET_FEATURES, (iprv.v1 | 0x80), FEATURE_SET, 0); } } #endif /* Clear the stack memory reserved for pt_regs: */ memset(ia64_task_regs(current), 0, sizeof(struct pt_regs)); ia64_set_kr(IA64_KR_FPU_OWNER, 0); /* * Initialize default control register to defer all speculative faults. The * kernel MUST NOT depend on a particular setting of these bits (in other words, * the kernel must have recovery code for all speculative accesses). Turn on * dcr.lc as per recommendation by the architecture team. Most IA-32 apps * shouldn't be affected by this (moral: keep your ia32 locks aligned and you'll * be fine). */ ia64_setreg(_IA64_REG_CR_DCR, ( IA64_DCR_DP | IA64_DCR_DK | IA64_DCR_DX | IA64_DCR_DR | IA64_DCR_DA | IA64_DCR_DD | IA64_DCR_LC)); atomic_inc(&init_mm.mm_count); current->active_mm = &init_mm; if (current->mm) BUG(); ia64_mmu_init(ia64_imva(cpu_data)); #ifdef CONFIG_IA32_SUPPORT ia32_cpu_init(); #endif /* Clear ITC to eliminiate sched_clock() overflows in human time. */ ia64_set_itc(0); /* disable all local interrupt sources: */ ia64_set_itv(1 << 16); ia64_set_lrr0(1 << 16); ia64_set_lrr1(1 << 16); ia64_setreg(_IA64_REG_CR_PMV, 1 << 16); ia64_setreg(_IA64_REG_CR_CMCV, 1 << 16); /* clear TPR & XTP to enable all interrupt classes: */ ia64_setreg(_IA64_REG_CR_TPR, 0); #ifdef CONFIG_SMP normal_xtp(); #endif /* set ia64_ctx.max_rid to the maximum RID that is supported by all CPUs: */ if (ia64_pal_vm_summary(NULL, &vmi) == 0) max_ctx = (1U << (vmi.pal_vm_info_2_s.rid_size - 3)) - 1; else { printk(KERN_WARNING "cpu_init: PAL VM summary failed, assuming 18 RID bits\n"); max_ctx = (1U << 15) - 1; /* use architected minimum */ } while (max_ctx < ia64_ctx.max_ctx) { unsigned int old = ia64_ctx.max_ctx; if (cmpxchg(&ia64_ctx.max_ctx, old, max_ctx) == old) break; } if (ia64_pal_rse_info(&num_phys_stacked, NULL) != 0) { printk(KERN_WARNING "cpu_init: PAL RSE info failed; assuming 96 physical " "stacked regs\n"); num_phys_stacked = 96; } /* size of physical stacked register partition plus 8 bytes: */ __get_cpu_var(ia64_phys_stacked_size_p8) = num_phys_stacked*8 + 8; platform_cpu_init(); }