/** * set_mtrr - update mtrrs on all processors * @reg: mtrr in question * @base: mtrr base * @size: mtrr size * @type: mtrr type * * This is kinda tricky, but fortunately, Intel spelled it out for us cleanly: * * 1. Send IPI to do the following: * 2. Disable Interrupts * 3. Wait for all procs to do so * 4. Enter no-fill cache mode * 5. Flush caches * 6. Clear PGE bit * 7. Flush all TLBs * 8. Disable all range registers * 9. Update the MTRRs * 10. Enable all range registers * 11. Flush all TLBs and caches again * 12. Enter normal cache mode and reenable caching * 13. Set PGE * 14. Wait for buddies to catch up * 15. Enable interrupts. * * What does that mean for us? Well, first we set data.count to the number * of CPUs. As each CPU disables interrupts, it'll decrement it once. We wait * until it hits 0 and proceed. We set the data.gate flag and reset data.count. * Meanwhile, they are waiting for that flag to be set. Once it's set, each * CPU goes through the transition of updating MTRRs. The CPU vendors may each do it * differently, so we call mtrr_if->set() callback and let them take care of it. * When they're done, they again decrement data->count and wait for data.gate to * be reset. * When we finish, we wait for data.count to hit 0 and toggle the data.gate flag. * Everyone then enables interrupts and we all continue on. * * Note that the mechanism is the same for UP systems, too; all the SMP stuff * becomes nops. */ static void set_mtrr(unsigned int reg, unsigned long base, unsigned long size, mtrr_type type) { struct set_mtrr_data data; unsigned long flags; data.smp_reg = reg; data.smp_base = base; data.smp_size = size; data.smp_type = type; atomic_set(&data.count, num_booting_cpus() - 1); /* make sure data.count is visible before unleashing other CPUs */ smp_wmb(); atomic_set(&data.gate,0); /* Start the ball rolling on other CPUs */ if (smp_call_function(ipi_handler, &data, 0) != 0) panic("mtrr: timed out waiting for other CPUs\n"); local_irq_save(flags); while(atomic_read(&data.count)) cpu_relax(); /* ok, reset count and toggle gate */ atomic_set(&data.count, num_booting_cpus() - 1); smp_wmb(); atomic_set(&data.gate,1); /* do our MTRR business */ /* HACK! * We use this same function to initialize the mtrrs on boot. * The state of the boot cpu's mtrrs has been saved, and we want * to replicate across all the APs. * If we're doing that @reg is set to something special... */ if (reg != ~0U) mtrr_if->set(reg,base,size,type); /* wait for the others */ while(atomic_read(&data.count)) cpu_relax(); atomic_set(&data.count, num_booting_cpus() - 1); smp_wmb(); atomic_set(&data.gate,0); /* * Wait here for everyone to have seen the gate change * So we're the last ones to touch 'data' */ while(atomic_read(&data.count)) cpu_relax(); local_irq_restore(flags); }
static void __init synchronize_tsc_ap (void) { int i; /* * Not every cpu is online at the time * this gets called, so we first wait for the BP to * finish SMP initialization: */ while (!atomic_read(&tsc_start_flag)) mb(); for (i = 0; i < NR_LOOPS; i++) { atomic_inc(&tsc_count_start); while (atomic_read(&tsc_count_start) != num_booting_cpus()) mb(); rdtscll(tsc_values[smp_processor_id()]); if (i == NR_LOOPS-1) write_tsc(0, 0); atomic_inc(&tsc_count_stop); while (atomic_read(&tsc_count_stop) != num_booting_cpus()) mb(); } }
/** * set_mtrr - update mtrrs on all processors * @reg: mtrr in question * @base: mtrr base * @size: mtrr size * @type: mtrr type * * This is kinda tricky, but fortunately, Intel spelled it out for us cleanly: * * 1. Queue work to do the following on all processors: * 2. Disable Interrupts * 3. Wait for all procs to do so * 4. Enter no-fill cache mode * 5. Flush caches * 6. Clear PGE bit * 7. Flush all TLBs * 8. Disable all range registers * 9. Update the MTRRs * 10. Enable all range registers * 11. Flush all TLBs and caches again * 12. Enter normal cache mode and reenable caching * 13. Set PGE * 14. Wait for buddies to catch up * 15. Enable interrupts. * * What does that mean for us? Well, first we set data.count to the number * of CPUs. As each CPU announces that it started the rendezvous handler by * decrementing the count, We reset data.count and set the data.gate flag * allowing all the cpu's to proceed with the work. As each cpu disables * interrupts, it'll decrement data.count once. We wait until it hits 0 and * proceed. We clear the data.gate flag and reset data.count. Meanwhile, they * are waiting for that flag to be cleared. Once it's cleared, each * CPU goes through the transition of updating MTRRs. * The CPU vendors may each do it differently, * so we call mtrr_if->set() callback and let them take care of it. * When they're done, they again decrement data->count and wait for data.gate * to be set. * When we finish, we wait for data.count to hit 0 and toggle the data.gate flag * Everyone then enables interrupts and we all continue on. * * Note that the mechanism is the same for UP systems, too; all the SMP stuff * becomes nops. */ static void set_mtrr(unsigned int reg, unsigned long base, unsigned long size, mtrr_type type) { struct set_mtrr_data data; unsigned long flags; int cpu; #ifdef CONFIG_SMP /* * If this cpu is not yet active, we are in the cpu online path. There * can be no stop_machine() in parallel, as stop machine ensures this * by using get_online_cpus(). We can skip taking the stop_cpus_mutex, * as we don't need it and also we can't afford to block while waiting * for the mutex. * * If this cpu is active, we need to prevent stop_machine() happening * in parallel by taking the stop cpus mutex. * * Also, this is called in the context of cpu online path or in the * context where cpu hotplug is prevented. So checking the active status * of the raw_smp_processor_id() is safe. */ if (cpu_active(raw_smp_processor_id())) mutex_lock(&stop_cpus_mutex); #endif preempt_disable(); data.smp_reg = reg; data.smp_base = base; data.smp_size = size; data.smp_type = type; atomic_set(&data.count, num_booting_cpus() - 1); /* Make sure data.count is visible before unleashing other CPUs */ smp_wmb(); atomic_set(&data.gate, 0); /* Start the ball rolling on other CPUs */ for_each_online_cpu(cpu) { struct cpu_stop_work *work = &per_cpu(mtrr_work, cpu); if (cpu == smp_processor_id()) continue; stop_one_cpu_nowait(cpu, mtrr_work_handler, &data, work); } while (atomic_read(&data.count)) cpu_relax(); /* Ok, reset count and toggle gate */ atomic_set(&data.count, num_booting_cpus() - 1); smp_wmb(); atomic_set(&data.gate, 1); local_irq_save(flags); while (atomic_read(&data.count)) cpu_relax(); /* Ok, reset count and toggle gate */ atomic_set(&data.count, num_booting_cpus() - 1); smp_wmb(); atomic_set(&data.gate, 0); /* Do our MTRR business */ /* * HACK! * We use this same function to initialize the mtrrs on boot. * The state of the boot cpu's mtrrs has been saved, and we want * to replicate across all the APs. * If we're doing that @reg is set to something special... */ if (reg != ~0U) mtrr_if->set(reg, base, size, type); else if (!mtrr_aps_delayed_init) mtrr_if->set_all(); /* Wait for the others */ while (atomic_read(&data.count)) cpu_relax(); atomic_set(&data.count, num_booting_cpus() - 1); smp_wmb(); atomic_set(&data.gate, 1); /* * Wait here for everyone to have seen the gate change * So we're the last ones to touch 'data' */ while (atomic_read(&data.count)) cpu_relax(); local_irq_restore(flags); preempt_enable(); #ifdef CONFIG_SMP if (cpu_active(raw_smp_processor_id())) mutex_unlock(&stop_cpus_mutex); #endif }
/** * set_mtrr - update mtrrs on all processors * @reg: mtrr in question * @base: mtrr base * @size: mtrr size * @type: mtrr type * * This is kinda tricky, but fortunately, Intel spelled it out for us cleanly: * * 1. Queue work to do the following on all processors: * 2. Disable Interrupts * 3. Wait for all procs to do so * 4. Enter no-fill cache mode * 5. Flush caches * 6. Clear PGE bit * 7. Flush all TLBs * 8. Disable all range registers * 9. Update the MTRRs * 10. Enable all range registers * 11. Flush all TLBs and caches again * 12. Enter normal cache mode and reenable caching * 13. Set PGE * 14. Wait for buddies to catch up * 15. Enable interrupts. * * What does that mean for us? Well, first we set data.count to the number * of CPUs. As each CPU announces that it started the rendezvous handler by * decrementing the count, We reset data.count and set the data.gate flag * allowing all the cpu's to proceed with the work. As each cpu disables * interrupts, it'll decrement data.count once. We wait until it hits 0 and * proceed. We clear the data.gate flag and reset data.count. Meanwhile, they * are waiting for that flag to be cleared. Once it's cleared, each * CPU goes through the transition of updating MTRRs. * The CPU vendors may each do it differently, * so we call mtrr_if->set() callback and let them take care of it. * When they're done, they again decrement data->count and wait for data.gate * to be set. * When we finish, we wait for data.count to hit 0 and toggle the data.gate flag * Everyone then enables interrupts and we all continue on. * * Note that the mechanism is the same for UP systems, too; all the SMP stuff * becomes nops. */ static void set_mtrr(unsigned int reg, unsigned long base, unsigned long size, mtrr_type type) { struct set_mtrr_data data; unsigned long flags; int cpu; preempt_disable(); data.smp_reg = reg; data.smp_base = base; data.smp_size = size; data.smp_type = type; atomic_set(&data.count, num_booting_cpus() - 1); /* Make sure data.count is visible before unleashing other CPUs */ smp_wmb(); atomic_set(&data.gate, 0); /* Start the ball rolling on other CPUs */ for_each_online_cpu(cpu) { struct cpu_stop_work *work = &per_cpu(mtrr_work, cpu); if (cpu == smp_processor_id()) continue; stop_one_cpu_nowait(cpu, mtrr_work_handler, &data, work); } while (atomic_read(&data.count)) cpu_relax(); /* Ok, reset count and toggle gate */ atomic_set(&data.count, num_booting_cpus() - 1); smp_wmb(); atomic_set(&data.gate, 1); local_irq_save(flags); while (atomic_read(&data.count)) cpu_relax(); /* Ok, reset count and toggle gate */ atomic_set(&data.count, num_booting_cpus() - 1); smp_wmb(); atomic_set(&data.gate, 0); /* Do our MTRR business */ /* * HACK! * * We use this same function to initialize the mtrrs during boot, * resume, runtime cpu online and on an explicit request to set a * specific MTRR. * * During boot or suspend, the state of the boot cpu's mtrrs has been * saved, and we want to replicate that across all the cpus that come * online (either at the end of boot or resume or during a runtime cpu * online). If we're doing that, @reg is set to something special and on * this cpu we still do mtrr_if->set_all(). During boot/resume, this * is unnecessary if at this point we are still on the cpu that started * the boot/resume sequence. But there is no guarantee that we are still * on the same cpu. So we do mtrr_if->set_all() on this cpu aswell to be * sure that we are in sync with everyone else. */ if (reg != ~0U) mtrr_if->set(reg, base, size, type); else mtrr_if->set_all(); /* Wait for the others */ while (atomic_read(&data.count)) cpu_relax(); atomic_set(&data.count, num_booting_cpus() - 1); smp_wmb(); atomic_set(&data.gate, 1); /* * Wait here for everyone to have seen the gate change * So we're the last ones to touch 'data' */ while (atomic_read(&data.count)) cpu_relax(); local_irq_restore(flags); preempt_enable(); }
/** * set_mtrr - update mtrrs on all processors * @reg: mtrr in question * @base: mtrr base * @size: mtrr size * @type: mtrr type * * This is kinda tricky, but fortunately, Intel spelled it out for us cleanly: * * 1. Queue work to do the following on all processors: * 2. Disable Interrupts * 3. Wait for all procs to do so * 4. Enter no-fill cache mode * 5. Flush caches * 6. Clear PGE bit * 7. Flush all TLBs * 8. Disable all range registers * 9. Update the MTRRs * 10. Enable all range registers * 11. Flush all TLBs and caches again * 12. Enter normal cache mode and reenable caching * 13. Set PGE * 14. Wait for buddies to catch up * 15. Enable interrupts. * * What does that mean for us? Well, first we set data.count to the number * of CPUs. As each CPU announces that it started the rendezvous handler by * decrementing the count, We reset data.count and set the data.gate flag * allowing all the cpu's to proceed with the work. As each cpu disables * interrupts, it'll decrement data.count once. We wait until it hits 0 and * proceed. We clear the data.gate flag and reset data.count. Meanwhile, they * are waiting for that flag to be cleared. Once it's cleared, each * CPU goes through the transition of updating MTRRs. * The CPU vendors may each do it differently, * so we call mtrr_if->set() callback and let them take care of it. * When they're done, they again decrement data->count and wait for data.gate * to be set. * When we finish, we wait for data.count to hit 0 and toggle the data.gate flag * Everyone then enables interrupts and we all continue on. * * Note that the mechanism is the same for UP systems, too; all the SMP stuff * becomes nops. */ static void set_mtrr(unsigned int reg, unsigned long base, unsigned long size, mtrr_type type) { struct set_mtrr_data data; unsigned long flags; int cpu; preempt_disable(); data.smp_reg = reg; data.smp_base = base; data.smp_size = size; data.smp_type = type; atomic_set(&data.count, num_booting_cpus() - 1); /* Make sure data.count is visible before unleashing other CPUs */ smp_wmb(); atomic_set(&data.gate, 0); /* Start the ball rolling on other CPUs */ for_each_online_cpu(cpu) { struct cpu_stop_work *work = &per_cpu(mtrr_work, cpu); if (cpu == smp_processor_id()) continue; stop_one_cpu_nowait(cpu, mtrr_work_handler, &data, work); } while (atomic_read(&data.count)) cpu_relax(); /* Ok, reset count and toggle gate */ atomic_set(&data.count, num_booting_cpus() - 1); smp_wmb(); atomic_set(&data.gate, 1); local_irq_save(flags); while (atomic_read(&data.count)) cpu_relax(); /* Ok, reset count and toggle gate */ atomic_set(&data.count, num_booting_cpus() - 1); smp_wmb(); atomic_set(&data.gate, 0); /* Do our MTRR business */ /* * HACK! * We use this same function to initialize the mtrrs on boot. * The state of the boot cpu's mtrrs has been saved, and we want * to replicate across all the APs. * If we're doing that @reg is set to something special... */ if (reg != ~0U) mtrr_if->set(reg, base, size, type); else if (!mtrr_aps_delayed_init) mtrr_if->set_all(); /* Wait for the others */ while (atomic_read(&data.count)) cpu_relax(); atomic_set(&data.count, num_booting_cpus() - 1); smp_wmb(); atomic_set(&data.gate, 1); /* * Wait here for everyone to have seen the gate change * So we're the last ones to touch 'data' */ while (atomic_read(&data.count)) cpu_relax(); local_irq_restore(flags); preempt_enable(); }
static void __init synchronize_tsc_bp (void) { int i; unsigned long long t0; unsigned long long sum, avg; long long delta; unsigned long one_usec; int buggy = 0; printk(KERN_INFO "checking TSC synchronization across %u CPUs: ", num_booting_cpus()); /* convert from kcyc/sec to cyc/usec */ one_usec = cpu_khz / 1000; atomic_set(&tsc_start_flag, 1); wmb(); /* * We loop a few times to get a primed instruction cache, * then the last pass is more or less synchronized and * the BP and APs set their cycle counters to zero all at * once. This reduces the chance of having random offsets * between the processors, and guarantees that the maximum * delay between the cycle counters is never bigger than * the latency of information-passing (cachelines) between * two CPUs. */ for (i = 0; i < NR_LOOPS; i++) { /* * all APs synchronize but they loop on '== num_cpus' */ while (atomic_read(&tsc_count_start) != num_booting_cpus()-1) mb(); atomic_set(&tsc_count_stop, 0); wmb(); /* * this lets the APs save their current TSC: */ atomic_inc(&tsc_count_start); rdtscll(tsc_values[smp_processor_id()]); /* * We clear the TSC in the last loop: */ if (i == NR_LOOPS-1) write_tsc(0, 0); /* * Wait for all APs to leave the synchronization point: */ while (atomic_read(&tsc_count_stop) != num_booting_cpus()-1) mb(); atomic_set(&tsc_count_start, 0); wmb(); atomic_inc(&tsc_count_stop); } sum = 0; for (i = 0; i < NR_CPUS; i++) { if (cpu_isset(i, cpu_callout_map)) { t0 = tsc_values[i]; sum += t0; } } avg = sum; do_div(avg, num_booting_cpus()); sum = 0; for (i = 0; i < NR_CPUS; i++) { if (!cpu_isset(i, cpu_callout_map)) continue; delta = tsc_values[i] - avg; if (delta < 0) delta = -delta; /* * We report bigger than 2 microseconds clock differences. */ if (delta > 2*one_usec) { long realdelta; if (!buggy) { buggy = 1; printk("\n"); } realdelta = delta; do_div(realdelta, one_usec); if (tsc_values[i] < avg) realdelta = -realdelta; printk(KERN_INFO "CPU#%d had %ld usecs TSC skew, fixed it up.\n", i, realdelta); } sum += delta; } if (!buggy) printk("passed.\n"); }