void
enable_K6_wt_alloc(void)
{
quad_t size;
u_int64_t whcr;
u_long eflags;
eflags = read_eflags();
cpu_disable_intr();
wbinvd();
#ifdef CPU_DISABLE_CACHE
/*
* Certain K6-2 box becomes unstable when write allocation is
* enabled.
*/
/*
* The AMD-K6 processer provides the 64-bit Test Register 12(TR12),
* but only the Cache Inhibit(CI) (bit 3 of TR12) is suppported.
* All other bits in TR12 have no effect on the processer's operation.
* The I/O Trap Restart function (bit 9 of TR12) is always enabled
* on the AMD-K6.
*/
wrmsr(0x0000000e, (u_int64_t)0x0008);
#endif
/* Don't assume that memory size is aligned with 4M. */
if (Maxmem > 0)
size = ((Maxmem >> 8) + 3) >> 2;
else
/*
* IBM Blue Lightning
*/
static void
init_bluelightning(void)
{
u_long eflags;
eflags = read_eflags();
cpu_disable_intr();
load_cr0(rcr0() | CR0_CD | CR0_NW);
invd();
#ifdef CPU_BLUELIGHTNING_FPU_OP_CACHE
wrmsr(0x1000, 0x9c92LL); /* FP operand can be cacheable on Cyrix FPU */
#else
wrmsr(0x1000, 0x1c92LL); /* Intel FPU */
#endif
/* Enables 13MB and 0-640KB cache. */
wrmsr(0x1001, (0xd0LL << 32) | 0x3ff);
#ifdef CPU_BLUELIGHTNING_3X
wrmsr(0x1002, 0x04000000LL); /* Enables triple-clock mode. */
#else
wrmsr(0x1002, 0x03000000LL); /* Enables double-clock mode. */
#endif
/* Enable caching in CR0. */
load_cr0(rcr0() & ~(CR0_CD | CR0_NW)); /* CD = 0 and NW = 0 */
invd();
write_eflags(eflags);
}
/*
* Initialize BBL_CR_CTL3 (Control register 3: used to configure the
* L2 cache).
*/
void
init_mendocino(void)
{
#ifdef CPU_PPRO2CELERON
u_long eflags;
u_int64_t bbl_cr_ctl3;
eflags = read_eflags();
cpu_disable_intr();
load_cr0(rcr0() | CR0_CD | CR0_NW);
wbinvd();
bbl_cr_ctl3 = rdmsr(0x11e);
/* If the L2 cache is configured, do nothing. */
if (!(bbl_cr_ctl3 & 1)) {
bbl_cr_ctl3 = 0x134052bLL;
/* Set L2 Cache Latency (Default: 5). */
#ifdef CPU_CELERON_L2_LATENCY
#if CPU_L2_LATENCY > 15
#error invalid CPU_L2_LATENCY.
#endif
bbl_cr_ctl3 |= CPU_L2_LATENCY << 1;
#else
bbl_cr_ctl3 |= 5 << 1;
#endif
wrmsr(0x11e, bbl_cr_ctl3);
}
load_cr0(rcr0() & ~(CR0_CD | CR0_NW));
write_eflags(eflags);
#endif /* CPU_PPRO2CELERON */
}
/*
* Cyrix 6x86MX (code-named M2)
*
* XXX - What should I do here? Please let me know.
*/
static void
init_6x86MX(void)
{
u_long eflags;
u_char ccr3, ccr4;
eflags = read_eflags();
cpu_disable_intr();
load_cr0(rcr0() | CR0_CD | CR0_NW);
wbinvd();
/* Initialize CCR0. */
write_cyrix_reg(CCR0, read_cyrix_reg(CCR0) | CCR0_NC1);
/* Initialize CCR1. */
#ifdef CPU_CYRIX_NO_LOCK
write_cyrix_reg(CCR1, read_cyrix_reg(CCR1) | CCR1_NO_LOCK);
#else
write_cyrix_reg(CCR1, read_cyrix_reg(CCR1) & ~CCR1_NO_LOCK);
#endif
/* Initialize CCR2. */
#ifdef CPU_SUSP_HLT
write_cyrix_reg(CCR2, read_cyrix_reg(CCR2) | CCR2_SUSP_HLT);
#else
write_cyrix_reg(CCR2, read_cyrix_reg(CCR2) & ~CCR2_SUSP_HLT);
#endif
ccr3 = read_cyrix_reg(CCR3);
write_cyrix_reg(CCR3, CCR3_MAPEN0);
/* Initialize CCR4. */
ccr4 = read_cyrix_reg(CCR4);
ccr4 &= ~CCR4_IOMASK;
#ifdef CPU_IORT
write_cyrix_reg(CCR4, ccr4 | (CPU_IORT & CCR4_IOMASK));
#else
write_cyrix_reg(CCR4, ccr4 | 7);
#endif
/* Initialize CCR5. */
#ifdef CPU_WT_ALLOC
write_cyrix_reg(CCR5, read_cyrix_reg(CCR5) | CCR5_WT_ALLOC);
#endif
/* Restore CCR3. */
write_cyrix_reg(CCR3, ccr3);
/* Unlock NW bit in CR0. */
write_cyrix_reg(CCR2, read_cyrix_reg(CCR2) & ~CCR2_LOCK_NW);
load_cr0(rcr0() & ~(CR0_CD | CR0_NW)); /* CD = 0 and NW = 0 */
/* Lock NW bit in CR0. */
write_cyrix_reg(CCR2, read_cyrix_reg(CCR2) | CCR2_LOCK_NW);
write_eflags(eflags);
}
static int
acpi_timer_test(void)
{
uint32_t last, this;
int min, max, max2, n, delta;
register_t s;
min = INT32_MAX;
max = max2 = 0;
/* Test the timer with interrupts disabled to get accurate results. */
#if defined(__i386__)
s = read_eflags();
#elif defined(__x86_64__)
s = read_rflags();
#else
#error "no read_eflags"
#endif
cpu_disable_intr();
AcpiGetTimer(&last);
for (n = 0; n < 2000; n++) {
AcpiGetTimer(&this);
delta = acpi_TimerDelta(this, last);
if (delta > max) {
max2 = max;
max = delta;
} else if (delta > max2) {
max2 = delta;
}
if (delta < min)
min = delta;
last = this;
}
#if defined(__i386__)
write_eflags(s);
#elif defined(__x86_64__)
write_rflags(s);
#else
#error "no read_eflags"
#endif
delta = max2 - min;
if ((max - min > 8 || delta > 3) && vmm_guest == VMM_GUEST_NONE)
n = 0;
else if (min < 0 || max == 0 || max2 == 0)
n = 0;
else
n = 1;
if (bootverbose) {
kprintf("ACPI timer looks %s min = %d, max = %d, width = %d\n",
n ? "GOOD" : "BAD ",
min, max, max - min);
}
return (n);
}
/*
* There are i486 based upgrade products for i386 machines.
* In this case, BIOS doesn't enables CPU cache.
*/
void
init_i486_on_386(void)
{
u_long eflags;
eflags = read_eflags();
cpu_disable_intr();
load_cr0(rcr0() & ~(CR0_CD | CR0_NW)); /* CD = 0, NW = 0 */
write_eflags(eflags);
}
static void
ioapic_abi_intr_teardown(int intr)
{
const struct ioapic_irqmap *map;
int vector, select;
uint32_t value;
register_t ef;
KASSERT(intr >= 0 && intr < IOAPIC_HWI_VECTORS,
("ioapic teardown, invalid irq %d\n", intr));
map = &ioapic_irqmaps[mycpuid][intr];
KASSERT(IOAPIC_IMT_ISHWI(map),
("ioapic teardown, not hwi irq %d, type %d, cpu%d",
intr, map->im_type, mycpuid));
if (map->im_type != IOAPIC_IMT_LEGACY)
return;
KASSERT(ioapic_irqs[intr].io_addr != NULL,
("ioapic teardown, no GSI information, irq %d\n", intr));
ef = read_rflags();
cpu_disable_intr();
/*
* Teardown an interrupt vector. The vector should already be
* installed in the cpu's IDT, but make sure.
*/
IOAPIC_INTRDIS(intr);
vector = IDT_OFFSET + intr;
/*
* In order to avoid losing an EOI for a level interrupt, which
* is vector based, make sure that the IO APIC is programmed for
* edge-triggering first, then reprogrammed with the new vector.
* This should clear the IRR bit.
*/
imen_lock();
select = ioapic_irqs[intr].io_idx;
value = ioapic_read(ioapic_irqs[intr].io_addr, select);
ioapic_write(ioapic_irqs[intr].io_addr, select,
(value & ~APIC_TRIGMOD_MASK));
ioapic_write(ioapic_irqs[intr].io_addr, select,
(value & ~IOART_INTVEC) | vector);
imen_unlock();
write_rflags(ef);
}
static int
acpi_timer_test(void)
{
uint32_t last, this;
int min, max, n, delta;
register_t s;
min = 10000000;
max = 0;
/* Test the timer with interrupts disabled to get accurate results. */
#if defined(__i386__)
s = read_eflags();
#elif defined(__x86_64__)
s = read_rflags();
#else
#error "no read_eflags"
#endif
cpu_disable_intr();
last = acpi_timer_read();
for (n = 0; n < 2000; n++) {
this = acpi_timer_read();
delta = acpi_TimerDelta(this, last);
if (delta > max)
max = delta;
else if (delta < min)
min = delta;
last = this;
}
#if defined(__i386__)
write_eflags(s);
#elif defined(__x86_64__)
write_rflags(s);
#else
#error "no read_eflags"
#endif
if (max - min > 2)
n = 0;
else if (min < 0 || max == 0)
n = 0;
else
n = 1;
if (bootverbose) {
kprintf("ACPI timer looks %s min = %d, max = %d, width = %d\n",
n ? "GOOD" : "BAD ",
min, max, max - min);
}
return (n);
}
static void
ioapic_abi_intr_setup(int intr, int flags)
{
const struct ioapic_irqmap *map;
int vector, select;
uint32_t value;
register_t ef;
KASSERT(intr >= 0 && intr < IOAPIC_HWI_VECTORS,
("ioapic setup, invalid irq %d", intr));
map = &ioapic_irqmaps[mycpuid][intr];
KASSERT(IOAPIC_IMT_ISHWI(map),
("ioapic setup, not hwi irq %d, type %d, cpu%d",
intr, map->im_type, mycpuid));
if (map->im_type != IOAPIC_IMT_LEGACY)
return;
KASSERT(ioapic_irqs[intr].io_addr != NULL,
("ioapic setup, no GSI information, irq %d", intr));
ef = read_rflags();
cpu_disable_intr();
vector = IDT_OFFSET + intr;
/*
* Now reprogram the vector in the IO APIC. In order to avoid
* losing an EOI for a level interrupt, which is vector based,
* make sure that the IO APIC is programmed for edge-triggering
* first, then reprogrammed with the new vector. This should
* clear the IRR bit.
*/
imen_lock();
select = ioapic_irqs[intr].io_idx;
value = ioapic_read(ioapic_irqs[intr].io_addr, select);
value |= IOART_INTMSET;
ioapic_write(ioapic_irqs[intr].io_addr, select,
(value & ~APIC_TRIGMOD_MASK));
ioapic_write(ioapic_irqs[intr].io_addr, select,
(value & ~IOART_INTVEC) | vector);
imen_unlock();
IOAPIC_INTREN(intr);
write_rflags(ef);
}
/*
* Cyrix 486S/DX series
*/
static void
init_cy486dx(void)
{
u_long eflags;
u_char ccr2;
eflags = read_eflags();
cpu_disable_intr();
invd();
ccr2 = read_cyrix_reg(CCR2);
#ifdef CPU_SUSP_HLT
ccr2 |= CCR2_SUSP_HLT;
#endif
write_cyrix_reg(CCR2, ccr2);
write_eflags(eflags);
}
void
icu_definit(void)
{
register_t ef;
KKASSERT(MachIntrABI.type == MACHINTR_ICU);
ef = read_rflags();
cpu_disable_intr();
/* Leave interrupts masked */
outb(IO_ICU1 + ICU_IMR_OFFSET, 0xff);
outb(IO_ICU2 + ICU_IMR_OFFSET, 0xff);
MachIntrABI.setdefault();
icu_init();
write_rflags(ef);
}
/*
* Cyrix 486SLC/DLC/SR/DR series
*/
static void
init_486dlc(void)
{
u_long eflags;
u_char ccr0;
eflags = read_eflags();
cpu_disable_intr();
invd();
ccr0 = read_cyrix_reg(CCR0);
#ifndef CYRIX_CACHE_WORKS
ccr0 |= CCR0_NC1 | CCR0_BARB;
write_cyrix_reg(CCR0, ccr0);
invd();
#else
ccr0 &= ~CCR0_NC0;
#ifndef CYRIX_CACHE_REALLY_WORKS
ccr0 |= CCR0_NC1 | CCR0_BARB;
#else
ccr0 |= CCR0_NC1;
#endif
#ifdef CPU_DIRECT_MAPPED_CACHE
ccr0 |= CCR0_CO; /* Direct mapped mode. */
#endif
write_cyrix_reg(CCR0, ccr0);
/* Clear non-cacheable region. */
write_cyrix_reg(NCR1+2, NCR_SIZE_0K);
write_cyrix_reg(NCR2+2, NCR_SIZE_0K);
write_cyrix_reg(NCR3+2, NCR_SIZE_0K);
write_cyrix_reg(NCR4+2, NCR_SIZE_0K);
write_cyrix_reg(0, 0); /* dummy write */
/* Enable caching in CR0. */
load_cr0(rcr0() & ~(CR0_CD | CR0_NW)); /* CD = 0 and NW = 0 */
invd();
#endif /* !CYRIX_CACHE_WORKS */
write_eflags(eflags);
}
/*
* Enable write allocate feature of AMD processors.
* Following two functions require the Maxmem variable being set.
*/
void
enable_K5_wt_alloc(void)
{
u_int64_t msr;
/*
* Write allocate is supported only on models 1, 2, and 3, with
* a stepping of 4 or greater.
*/
if (((cpu_id & 0xf0) > 0) && ((cpu_id & 0x0f) > 3)) {
cpu_disable_intr();
msr = rdmsr(0x83); /* HWCR */
wrmsr(0x83, msr & !(0x10));
/*
* We have to tell the chip where the top of memory is,
* since video cards could have frame bufferes there,
* memory-mapped I/O could be there, etc.
*/
if(Maxmem > 0)
msr = Maxmem / 16;
else
msr = 0;
msr |= AMD_WT_ALLOC_TME | AMD_WT_ALLOC_FRE;
/*
* There is no way to know wheter 15-16M hole exists or not.
* Therefore, we disable write allocate for this range.
*/
wrmsr(0x86, 0x0ff00f0);
msr |= AMD_WT_ALLOC_PRE;
wrmsr(0x85, msr);
msr=rdmsr(0x83);
wrmsr(0x83, msr|0x10); /* enable write allocate */
cpu_enable_intr();
}
}
void
icu_reinit_noioapic(void)
{
register_t ef;
KKASSERT(MachIntrABI.type == MACHINTR_ICU);
KKASSERT(ioapic_enable == 0);
crit_enter();
ef = read_rflags();
cpu_disable_intr();
/* Leave interrupts masked */
outb(IO_ICU1 + ICU_IMR_OFFSET, 0xff);
outb(IO_ICU2 + ICU_IMR_OFFSET, 0xff);
icu_init();
MachIntrABI.stabilize();
write_rflags(ef);
MachIntrABI.cleanup();
crit_exit();
}
/*
* Invalidate the specified va across all cpus associated with the pmap.
* If va == (vm_offset_t)-1, we invltlb() instead of invlpg(). The operation
* will be done fully synchronously with storing npte into *ptep and returning
* opte.
*
* If ptep is NULL the operation will execute semi-synchronously.
* ptep must be NULL if npgs > 1
*/
pt_entry_t
pmap_inval_smp(pmap_t pmap, vm_offset_t va, int npgs,
pt_entry_t *ptep, pt_entry_t npte)
{
globaldata_t gd = mycpu;
pmap_inval_info_t *info;
pt_entry_t opte = 0;
int cpu = gd->gd_cpuid;
cpumask_t tmpmask;
unsigned long rflags;
/*
* Initialize invalidation for pmap and enter critical section.
*/
if (pmap == NULL)
pmap = &kernel_pmap;
pmap_inval_init(pmap);
/*
* Shortcut single-cpu case if possible.
*/
if (CPUMASK_CMPMASKEQ(pmap->pm_active, gd->gd_cpumask)) {
/*
* Convert to invltlb if there are too many pages to
* invlpg on.
*/
if (npgs > MAX_INVAL_PAGES) {
npgs = 0;
va = (vm_offset_t)-1;
}
/*
* Invalidate the specified pages, handle invltlb if requested.
*/
while (npgs) {
--npgs;
if (ptep) {
opte = atomic_swap_long(ptep, npte);
++ptep;
}
if (va == (vm_offset_t)-1)
break;
cpu_invlpg((void *)va);
va += PAGE_SIZE;
}
if (va == (vm_offset_t)-1)
cpu_invltlb();
pmap_inval_done(pmap);
return opte;
}
/*
* We need a critical section to prevent getting preempted while
* we setup our command. A preemption might execute its own
* pmap_inval*() command and create confusion below.
*
* tsc_target is our watchdog timeout that will attempt to recover
* from a lost IPI. Set to 1/16 second for now.
*/
info = &invinfo[cpu];
info->tsc_target = rdtsc() + (tsc_frequency * LOOPRECOVER_TIMEOUT1);
/*
* We must wait for other cpus which may still be finishing up a
* prior operation that we requested.
*
* We do not have to disable interrupts here. An Xinvltlb can occur
* at any time (even within a critical section), but it will not
* act on our command until we set our done bits.
*/
while (CPUMASK_TESTNZERO(info->done)) {
#ifdef LOOPRECOVER
if (loopwdog(info)) {
info->failed = 1;
loopdebug("A", info);
/* XXX recover from possible bug */
CPUMASK_ASSZERO(info->done);
}
#endif
cpu_pause();
}
KKASSERT(info->mode == INVDONE);
/*
* Must set our cpu in the invalidation scan mask before
* any possibility of [partial] execution (remember, XINVLTLB
* can interrupt a critical section).
*/
ATOMIC_CPUMASK_ORBIT(smp_invmask, cpu);
info->va = va;
info->npgs = npgs;
info->ptep = ptep;
info->npte = npte;
info->opte = 0;
#ifdef LOOPRECOVER
info->failed = 0;
#endif
info->mode = INVSTORE;
tmpmask = pmap->pm_active; /* volatile (bits may be cleared) */
cpu_ccfence();
CPUMASK_ANDMASK(tmpmask, smp_active_mask);
/*
* If ptep is NULL the operation can be semi-synchronous, which means
* we can improve performance by flagging and removing idle cpus
* (see the idleinvlclr function in mp_machdep.c).
*
* Typically kernel page table operation is semi-synchronous.
*/
if (ptep == NULL)
smp_smurf_idleinvlclr(&tmpmask);
CPUMASK_ORBIT(tmpmask, cpu);
info->mask = tmpmask;
/*
* Command may start executing the moment 'done' is initialized,
* disable current cpu interrupt to prevent 'done' field from
* changing (other cpus can't clear done bits until the originating
* cpu clears its mask bit, but other cpus CAN start clearing their
* mask bits).
*/
#ifdef LOOPRECOVER
info->sigmask = tmpmask;
CHECKSIGMASK(info);
#endif
cpu_sfence();
rflags = read_rflags();
cpu_disable_intr();
ATOMIC_CPUMASK_COPY(info->done, tmpmask);
/* execution can begin here due to races */
/*
* Pass our copy of the done bits (so they don't change out from
* under us) to generate the Xinvltlb interrupt on the targets.
*/
smp_invlpg(&tmpmask);
opte = info->opte;
KKASSERT(info->mode == INVDONE);
/*
* Target cpus will be in their loop exiting concurrently with our
* cleanup. They will not lose the bitmask they obtained before so
* we can safely clear this bit.
*/
ATOMIC_CPUMASK_NANDBIT(smp_invmask, cpu);
write_rflags(rflags);
pmap_inval_done(pmap);
return opte;
}
/*
* Cyrix 6x86
*
* XXX - What should I do here? Please let me know.
*/
static void
init_6x86(void)
{
u_long eflags;
u_char ccr3, ccr4;
eflags = read_eflags();
cpu_disable_intr();
load_cr0(rcr0() | CR0_CD | CR0_NW);
wbinvd();
/* Initialize CCR0. */
write_cyrix_reg(CCR0, read_cyrix_reg(CCR0) | CCR0_NC1);
/* Initialize CCR1. */
#ifdef CPU_CYRIX_NO_LOCK
write_cyrix_reg(CCR1, read_cyrix_reg(CCR1) | CCR1_NO_LOCK);
#else
write_cyrix_reg(CCR1, read_cyrix_reg(CCR1) & ~CCR1_NO_LOCK);
#endif
/* Initialize CCR2. */
#ifdef CPU_SUSP_HLT
write_cyrix_reg(CCR2, read_cyrix_reg(CCR2) | CCR2_SUSP_HLT);
#else
write_cyrix_reg(CCR2, read_cyrix_reg(CCR2) & ~CCR2_SUSP_HLT);
#endif
ccr3 = read_cyrix_reg(CCR3);
write_cyrix_reg(CCR3, CCR3_MAPEN0);
/* Initialize CCR4. */
ccr4 = read_cyrix_reg(CCR4);
ccr4 |= CCR4_DTE;
ccr4 &= ~CCR4_IOMASK;
#ifdef CPU_IORT
write_cyrix_reg(CCR4, ccr4 | (CPU_IORT & CCR4_IOMASK));
#else
write_cyrix_reg(CCR4, ccr4 | 7);
#endif
/* Initialize CCR5. */
#ifdef CPU_WT_ALLOC
write_cyrix_reg(CCR5, read_cyrix_reg(CCR5) | CCR5_WT_ALLOC);
#endif
/* Restore CCR3. */
write_cyrix_reg(CCR3, ccr3);
/* Unlock NW bit in CR0. */
write_cyrix_reg(CCR2, read_cyrix_reg(CCR2) & ~CCR2_LOCK_NW);
/*
* Earlier revision of the 6x86 CPU could crash the system if
* L1 cache is in write-back mode.
*/
if ((cyrix_did & 0xff00) > 0x1600)
load_cr0(rcr0() & ~(CR0_CD | CR0_NW)); /* CD = 0 and NW = 0 */
else {
/* Revision 2.6 and lower. */
#ifdef CYRIX_CACHE_REALLY_WORKS
load_cr0(rcr0() & ~(CR0_CD | CR0_NW)); /* CD = 0 and NW = 0 */
#else
load_cr0((rcr0() & ~CR0_CD) | CR0_NW); /* CD = 0 and NW = 1 */
#endif
}
/* Lock NW bit in CR0. */
write_cyrix_reg(CCR2, read_cyrix_reg(CCR2) | CCR2_LOCK_NW);
write_eflags(eflags);
}
/*
* Called with a critical section held and interrupts enabled.
*/
int
pmap_inval_intr(cpumask_t *cpumaskp, int toolong)
{
globaldata_t gd = mycpu;
pmap_inval_info_t *info;
int loopme = 0;
int cpu;
cpumask_t cpumask;
/*
* Check all cpus for invalidations we may need to service.
*/
cpu_ccfence();
cpu = gd->gd_cpuid;
cpumask = *cpumaskp;
while (CPUMASK_TESTNZERO(cpumask)) {
int n = BSFCPUMASK(cpumask);
#ifdef LOOPRECOVER
KKASSERT(n >= 0 && n < MAXCPU);
#endif
CPUMASK_NANDBIT(cpumask, n);
info = &invinfo[n];
/*
* Due to interrupts/races we can catch a new operation
* in an older interrupt. A fence is needed once we detect
* the (not) done bit.
*/
if (!CPUMASK_TESTBIT(info->done, cpu))
continue;
cpu_lfence();
#ifdef LOOPRECOVER
if (toolong) {
kprintf("pminvl %d->%d %08jx %08jx mode=%d\n",
cpu, n, info->done.ary[0], info->mask.ary[0],
info->mode);
}
#endif
/*
* info->mask and info->done always contain the originating
* cpu until the originator is done. Targets may still be
* present in info->done after the originator is done (they
* will be finishing up their loops).
*
* Clear info->mask bits on other cpus to indicate that they
* have quiesced (entered the loop). Once the other mask bits
* are clear we can execute the operation on the original,
* then clear the mask and done bits on the originator. The
* targets will then finish up their side and clear their
* done bits.
*
* The command is considered 100% done when all done bits have
* been cleared.
*/
if (n != cpu) {
/*
* Command state machine for 'other' cpus.
*/
if (CPUMASK_TESTBIT(info->mask, cpu)) {
/*
* Other cpu indicate to originator that they
* are quiesced.
*/
ATOMIC_CPUMASK_NANDBIT(info->mask, cpu);
loopme = 1;
} else if (info->ptep &&
CPUMASK_TESTBIT(info->mask, n)) {
/*
* Other cpu must wait for the originator (n)
* to complete its command if ptep is not NULL.
*/
loopme = 1;
} else {
/*
* Other cpu detects that the originator has
* completed its command, or there was no
* command.
*
* Now that the page table entry has changed,
* we can follow up with our own invalidation.
*/
vm_offset_t va = info->va;
int npgs;
if (va == (vm_offset_t)-1 ||
info->npgs > MAX_INVAL_PAGES) {
cpu_invltlb();
} else {
for (npgs = info->npgs; npgs; --npgs) {
cpu_invlpg((void *)va);
va += PAGE_SIZE;
}
}
ATOMIC_CPUMASK_NANDBIT(info->done, cpu);
/* info invalid now */
/* loopme left alone */
}
} else if (CPUMASK_TESTBIT(info->mask, cpu)) {
/*
* Originator is waiting for other cpus
*/
if (CPUMASK_CMPMASKNEQ(info->mask, gd->gd_cpumask)) {
/*
* Originator waits for other cpus to enter
* their loop (aka quiesce).
*
* If this bugs out the IPI may have been lost,
* try to reissue by resetting our own
* reentrancy bit and clearing the smurf mask
* for the cpus that did not respond, then
* reissuing the IPI.
*/
loopme = 1;
#ifdef LOOPRECOVER
if (loopwdog(info)) {
info->failed = 1;
loopdebug("C", info);
/* XXX recover from possible bug */
mdcpu->gd_xinvaltlb = 0;
ATOMIC_CPUMASK_NANDMASK(smp_smurf_mask,
info->mask);
cpu_disable_intr();
smp_invlpg(&smp_active_mask);
/*
* Force outer-loop retest of Xinvltlb
* requests (see mp_machdep.c).
*/
mdcpu->gd_xinvaltlb = 2;
cpu_enable_intr();
}
#endif
} else {
/*
* Originator executes operation and clears
* mask to allow other cpus to finish.
*/
KKASSERT(info->mode != INVDONE);
if (info->mode == INVSTORE) {
if (info->ptep)
info->opte = atomic_swap_long(info->ptep, info->npte);
CHECKSIGMASK(info);
ATOMIC_CPUMASK_NANDBIT(info->mask, cpu);
CHECKSIGMASK(info);
} else {
if (atomic_cmpset_long(info->ptep,
info->opte, info->npte)) {
info->success = 1;
} else {
info->success = 0;
}
CHECKSIGMASK(info);
ATOMIC_CPUMASK_NANDBIT(info->mask, cpu);
CHECKSIGMASK(info);
}
loopme = 1;
}
} else {
/*
* Originator does not have to wait for the other
* cpus to finish. It clears its done bit. A new
* command will not be initiated by the originator
* until the other cpus have cleared their done bits
* (asynchronously).
*/
vm_offset_t va = info->va;
int npgs;
if (va == (vm_offset_t)-1 ||
info->npgs > MAX_INVAL_PAGES) {
cpu_invltlb();
} else {
for (npgs = info->npgs; npgs; --npgs) {
cpu_invlpg((void *)va);
va += PAGE_SIZE;
}
}
/* leave loopme alone */
/* other cpus may still be finishing up */
/* can't race originator since that's us */
info->mode = INVDONE;
ATOMIC_CPUMASK_NANDBIT(info->done, cpu);
}
}
return loopme;
}
/*
* API function - invalidate the pte at (va) and replace *ptep with npte
* atomically only if *ptep equals opte, across the pmap's active cpus.
*
* Returns 1 on success, 0 on failure (caller typically retries).
*/
int
pmap_inval_smp_cmpset(pmap_t pmap, vm_offset_t va, pt_entry_t *ptep,
pt_entry_t opte, pt_entry_t npte)
{
globaldata_t gd = mycpu;
pmap_inval_info_t *info;
int success;
int cpu = gd->gd_cpuid;
cpumask_t tmpmask;
unsigned long rflags;
/*
* Initialize invalidation for pmap and enter critical section.
*/
if (pmap == NULL)
pmap = &kernel_pmap;
pmap_inval_init(pmap);
/*
* Shortcut single-cpu case if possible.
*/
if (CPUMASK_CMPMASKEQ(pmap->pm_active, gd->gd_cpumask)) {
if (atomic_cmpset_long(ptep, opte, npte)) {
if (va == (vm_offset_t)-1)
cpu_invltlb();
else
cpu_invlpg((void *)va);
pmap_inval_done(pmap);
return 1;
} else {
pmap_inval_done(pmap);
return 0;
}
}
/*
* We need a critical section to prevent getting preempted while
* we setup our command. A preemption might execute its own
* pmap_inval*() command and create confusion below.
*/
info = &invinfo[cpu];
info->tsc_target = rdtsc() + (tsc_frequency * LOOPRECOVER_TIMEOUT1);
/*
* We must wait for other cpus which may still be finishing
* up a prior operation.
*/
while (CPUMASK_TESTNZERO(info->done)) {
#ifdef LOOPRECOVER
if (loopwdog(info)) {
info->failed = 1;
loopdebug("B", info);
/* XXX recover from possible bug */
CPUMASK_ASSZERO(info->done);
}
#endif
cpu_pause();
}
KKASSERT(info->mode == INVDONE);
/*
* Must set our cpu in the invalidation scan mask before
* any possibility of [partial] execution (remember, XINVLTLB
* can interrupt a critical section).
*/
ATOMIC_CPUMASK_ORBIT(smp_invmask, cpu);
info->va = va;
info->npgs = 1; /* unused */
info->ptep = ptep;
info->npte = npte;
info->opte = opte;
#ifdef LOOPRECOVER
info->failed = 0;
#endif
info->mode = INVCMPSET;
info->success = 0;
tmpmask = pmap->pm_active; /* volatile */
cpu_ccfence();
CPUMASK_ANDMASK(tmpmask, smp_active_mask);
CPUMASK_ORBIT(tmpmask, cpu);
info->mask = tmpmask;
/*
* Command may start executing the moment 'done' is initialized,
* disable current cpu interrupt to prevent 'done' field from
* changing (other cpus can't clear done bits until the originating
* cpu clears its mask bit).
*/
#ifdef LOOPRECOVER
info->sigmask = tmpmask;
CHECKSIGMASK(info);
#endif
cpu_sfence();
rflags = read_rflags();
cpu_disable_intr();
ATOMIC_CPUMASK_COPY(info->done, tmpmask);
/*
* Pass our copy of the done bits (so they don't change out from
* under us) to generate the Xinvltlb interrupt on the targets.
*/
smp_invlpg(&tmpmask);
success = info->success;
KKASSERT(info->mode == INVDONE);
ATOMIC_CPUMASK_NANDBIT(smp_invmask, cpu);
write_rflags(rflags);
pmap_inval_done(pmap);
return success;
}
/*
* Cyrix 5x86
*/
static void
init_5x86(void)
{
u_long eflags;
u_char ccr2, ccr3, ccr4, pcr0;
eflags = read_eflags();
cpu_disable_intr();
load_cr0(rcr0() | CR0_CD | CR0_NW);
wbinvd();
read_cyrix_reg(CCR3); /* dummy */
/* Initialize CCR2. */
ccr2 = read_cyrix_reg(CCR2);
ccr2 |= CCR2_WB;
#ifdef CPU_SUSP_HLT
ccr2 |= CCR2_SUSP_HLT;
#else
ccr2 &= ~CCR2_SUSP_HLT;
#endif
ccr2 |= CCR2_WT1;
write_cyrix_reg(CCR2, ccr2);
/* Initialize CCR4. */
ccr3 = read_cyrix_reg(CCR3);
write_cyrix_reg(CCR3, CCR3_MAPEN0);
ccr4 = read_cyrix_reg(CCR4);
ccr4 |= CCR4_DTE;
ccr4 |= CCR4_MEM;
#ifdef CPU_FASTER_5X86_FPU
ccr4 |= CCR4_FASTFPE;
#else
ccr4 &= ~CCR4_FASTFPE;
#endif
ccr4 &= ~CCR4_IOMASK;
/********************************************************************
* WARNING: The "BIOS Writers Guide" mentions that I/O recovery time
* should be 0 for errata fix.
********************************************************************/
#ifdef CPU_IORT
ccr4 |= CPU_IORT & CCR4_IOMASK;
#endif
write_cyrix_reg(CCR4, ccr4);
/* Initialize PCR0. */
/****************************************************************
* WARNING: RSTK_EN and LOOP_EN could make your system unstable.
* BTB_EN might make your system unstable.
****************************************************************/
pcr0 = read_cyrix_reg(PCR0);
#ifdef CPU_RSTK_EN
pcr0 |= PCR0_RSTK;
#else
pcr0 &= ~PCR0_RSTK;
#endif
#ifdef CPU_BTB_EN
pcr0 |= PCR0_BTB;
#else
pcr0 &= ~PCR0_BTB;
#endif
#ifdef CPU_LOOP_EN
pcr0 |= PCR0_LOOP;
#else
pcr0 &= ~PCR0_LOOP;
#endif
/****************************************************************
* WARNING: if you use a memory mapped I/O device, don't use
* DISABLE_5X86_LSSER option, which may reorder memory mapped
* I/O access.
* IF YOUR MOTHERBOARD HAS PCI BUS, DON'T DISABLE LSSER.
****************************************************************/
#ifdef CPU_DISABLE_5X86_LSSER
pcr0 &= ~PCR0_LSSER;
#else
pcr0 |= PCR0_LSSER;
#endif
write_cyrix_reg(PCR0, pcr0);
/* Restore CCR3. */
write_cyrix_reg(CCR3, ccr3);
read_cyrix_reg(0x80); /* dummy */
/* Unlock NW bit in CR0. */
write_cyrix_reg(CCR2, read_cyrix_reg(CCR2) & ~CCR2_LOCK_NW);
load_cr0((rcr0() & ~CR0_CD) | CR0_NW); /* CD = 0, NW = 1 */
/* Lock NW bit in CR0. */
write_cyrix_reg(CCR2, read_cyrix_reg(CCR2) | CCR2_LOCK_NW);
write_eflags(eflags);
}
