void ioapic_init() {
// TODO: multiple ioapics, etc.
if(!pmem_reserve(IOAPIC_BASE, IOAPIC_VSZ))
fatal("failed to reserve physical I/O APIC memory!\n");
if(!vmem_map(spc_current(), IOAPIC_BASE, (void*)IOAPIC_VIRTUAL,
PG_WRITABLE | PG_NONCACHABLE | PG_WRITETHROUGH | PG_GLOBAL))
fatal("failed to map I/O APIC memory!\n");
_the_ioapic = (ioapic_t*)IOAPIC_VIRTUAL;
uint32_t idreg = ioapic_read(_the_ioapic, IOAPIC_REG_ID);
uint32_t vreg = ioapic_read(_the_ioapic, IOAPIC_REG_VER);
/* initially clear all redirection entries. resets all
* entries to edge-triggered, active high, disabled, and
* not routed to any cpu */
for(uint32_t i = 0; i <= IOAPIC_MAX_REDIR(vreg); ++i) {
ioapic_write(_the_ioapic, IOAPIC_REG_TABLE + (2 * i), IOAPIC_INT_MASKED | (IRQ_NUM(i)));
ioapic_write(_the_ioapic, IOAPIC_REG_TABLE + ((2 * i) + 1), 0);
}
info("i/o apic %d version: 0x%x, max irq redirections: %d\n",
IOAPIC_ID(idreg), IOAPIC_VER(vreg), IOAPIC_MAX_REDIR(vreg));
}
void
apic_set_redir(struct ioapic_softc *sc, int pin, int idt_vec,
struct cpu_info *ci)
{
u_int32_t redlo;
u_int32_t redhi = 0;
int delmode;
struct ioapic_pin *pp;
struct mp_intr_map *map;
pp = &sc->sc_pins[pin];
map = pp->ip_map;
redlo = (map == NULL) ? IOAPIC_REDLO_MASK : map->redir;
delmode = (redlo & IOAPIC_REDLO_DEL_MASK) >> IOAPIC_REDLO_DEL_SHIFT;
/* XXX magic numbers */
if ((delmode != 0) && (delmode != 1))
;
else if (pp->ip_type == IST_NONE) {
redlo |= IOAPIC_REDLO_MASK;
} else {
redlo |= (idt_vec & 0xff);
redlo &= ~IOAPIC_REDLO_DEL_MASK;
redlo |= (IOAPIC_REDLO_DEL_FIXED << IOAPIC_REDLO_DEL_SHIFT);
redlo &= ~IOAPIC_REDLO_DSTMOD;
/*
* Destination: BSP CPU
*
* XXX will want to distribute interrupts across cpu's
* eventually. most likely, we'll want to vector each
* interrupt to a specific CPU and load-balance across
* cpu's. but there's no point in doing that until after
* most interrupts run without the kernel lock.
*/
redhi |= (ci->ci_apicid << IOAPIC_REDHI_DEST_SHIFT);
/* XXX derive this bit from BIOS info */
if (pp->ip_type == IST_LEVEL)
redlo |= IOAPIC_REDLO_LEVEL;
else
redlo &= ~IOAPIC_REDLO_LEVEL;
if (map != NULL && ((map->flags & 3) == MPS_INTPO_DEF)) {
if (pp->ip_type == IST_LEVEL)
redlo |= IOAPIC_REDLO_ACTLO;
else
redlo &= ~IOAPIC_REDLO_ACTLO;
}
}
/* Do atomic write */
ioapic_write(sc, IOAPIC_REDLO(pin), IOAPIC_REDLO_MASK);
ioapic_write(sc, IOAPIC_REDHI(pin), redhi);
ioapic_write(sc, IOAPIC_REDLO(pin), redlo);
if (mp_verbose)
ioapic_print_redir(sc, "int", pin);
}
static void
ioapic_abi_intr_teardown(int intr)
{
const struct ioapic_irqmap *map;
int vector, select;
uint32_t value;
register_t ef;
KASSERT(intr >= 0 && intr < IOAPIC_HWI_VECTORS,
("ioapic teardown, invalid irq %d\n", intr));
map = &ioapic_irqmaps[mycpuid][intr];
KASSERT(IOAPIC_IMT_ISHWI(map),
("ioapic teardown, not hwi irq %d, type %d, cpu%d",
intr, map->im_type, mycpuid));
if (map->im_type != IOAPIC_IMT_LEGACY)
return;
KASSERT(ioapic_irqs[intr].io_addr != NULL,
("ioapic teardown, no GSI information, irq %d\n", intr));
ef = read_rflags();
cpu_disable_intr();
/*
* Teardown an interrupt vector. The vector should already be
* installed in the cpu's IDT, but make sure.
*/
IOAPIC_INTRDIS(intr);
vector = IDT_OFFSET + intr;
/*
* In order to avoid losing an EOI for a level interrupt, which
* is vector based, make sure that the IO APIC is programmed for
* edge-triggering first, then reprogrammed with the new vector.
* This should clear the IRR bit.
*/
imen_lock();
select = ioapic_irqs[intr].io_idx;
value = ioapic_read(ioapic_irqs[intr].io_addr, select);
ioapic_write(ioapic_irqs[intr].io_addr, select,
(value & ~APIC_TRIGMOD_MASK));
ioapic_write(ioapic_irqs[intr].io_addr, select,
(value & ~IOART_INTVEC) | vector);
imen_unlock();
write_rflags(ef);
}
static void
ioapic_abi_intr_setup(int intr, int flags)
{
const struct ioapic_irqmap *map;
int vector, select;
uint32_t value;
register_t ef;
KASSERT(intr >= 0 && intr < IOAPIC_HWI_VECTORS,
("ioapic setup, invalid irq %d", intr));
map = &ioapic_irqmaps[mycpuid][intr];
KASSERT(IOAPIC_IMT_ISHWI(map),
("ioapic setup, not hwi irq %d, type %d, cpu%d",
intr, map->im_type, mycpuid));
if (map->im_type != IOAPIC_IMT_LEGACY)
return;
KASSERT(ioapic_irqs[intr].io_addr != NULL,
("ioapic setup, no GSI information, irq %d", intr));
ef = read_rflags();
cpu_disable_intr();
vector = IDT_OFFSET + intr;
/*
* Now reprogram the vector in the IO APIC. In order to avoid
* losing an EOI for a level interrupt, which is vector based,
* make sure that the IO APIC is programmed for edge-triggering
* first, then reprogrammed with the new vector. This should
* clear the IRR bit.
*/
imen_lock();
select = ioapic_irqs[intr].io_idx;
value = ioapic_read(ioapic_irqs[intr].io_addr, select);
value |= IOART_INTMSET;
ioapic_write(ioapic_irqs[intr].io_addr, select,
(value & ~APIC_TRIGMOD_MASK));
ioapic_write(ioapic_irqs[intr].io_addr, select,
(value & ~IOART_INTVEC) | vector);
imen_unlock();
IOAPIC_INTREN(intr);
write_rflags(ef);
}
static void apic_setup(void)
{
/* Set the IOAPIC ID to the static value used in the MP/ACPI tables. */
ioapic_write(0x00, IOAPIC_ID);
/* NMIs are delivered direct to the BSP. */
lapic_write(APIC_SPIV, APIC_SPIV_APIC_ENABLED | 0xFF);
lapic_write(APIC_LVT0, (APIC_MODE_EXTINT << 8) | APIC_LVT_MASKED);
lapic_write(APIC_LVT1, APIC_MODE_NMI << 8);
/* 8259A ExtInts are delivered through IOAPIC pin 0 (Virtual Wire Mode). */
ioapic_write(0x10, APIC_DM_EXTINT);
ioapic_write(0x11, SET_APIC_ID(LAPIC_ID(0)));
}
static void ioapic_unmask_irq(int irq, unsigned flags)
{
struct ioapic_info *ioa = &global_ioapics[irq / 24];
int vec = irq % 24;
/* transfer the act-low/level-trig pattern into the APIC setting word
* directly.
*/
assert(IHF_ACT_LOW == (1 << 0));
assert(IHF_LEVEL_TRIG == (1 << 2));
int trig_pol = (flags & (IHF_ACT_LOW | IHF_LEVEL_TRIG)) << 13;
assert(CHECK_FLAG(trig_pol, 1 << 13)
== CHECK_FLAG(flags, IHF_ACT_LOW));
assert(CHECK_FLAG(trig_pol, 1 << 15)
== CHECK_FLAG(flags, IHF_LEVEL_TRIG));
/* bit 16 = interrupt mask bit.
* 15 = trigger mode (0 = edge, 1 = level)
* 13 = pin polarity (0 = active high, 1 = active low)
* 7..0 = interrupt vector (0x10..0xff)
*
* other fields select fixed (regular) delivery to a physical LAPIC (CPU
* 0, since mung is still not a SMP kernel).
*/
uint32_t ctl = ioapic_read(ioa, IOREDTBL(vec) + 0);
ctl &= ~((1 << 16) | (1 << 13) | (1 << 15) | 0xff);
ctl |= trig_pol | (0x20 + irq);
ioapic_write(ioa, IOREDTBL(vec) + 0, ctl);
}
void ioapic_add(unsigned num, paddr_t base, unsigned irqbase)
{
unsigned i;
ioapics[num].base = kvmap(base, IOAPIC_SIZE);
ioapics[num].irq = irqbase;
ioapics[num].pins = 1 + ((ioapic_read(num, IO_VER) >> 16) & 0xff);
/* Mask all interrupts */
for (i = 0; i < ioapics[num].pins; i++) {
ioapic_write(num, IO_RED_LO(i), 0x00010000);
ioapic_write(num, IO_RED_HI(i), 0x00000000);
}
printf("IOAPIC ID: %02d PA: %08llx VA: %p IRQ:%02d PINS: %02d\n",
num, base, ioapics[num].base, irqbase, ioapics[num].pins);
}
void ioapic_enable_irq(int irq)
{
struct apic_irq_to_pin pin_record = irq_map[irq];
int pin = pin_record.pin;
int chip = pin_record.chip;
if (pin != -1 && chip != -1) {
int addr = APIC_IO_REDTBL + pin * 2;
struct io_redirection_register reg;
reg.low = ioapic_read(chip, addr);
reg.masked = 0;
ioapic_write(chip, addr, reg.low);
// kprintf("IRQ enabled, vector: %x, IRQ: %d\n", reg.vector, irq);
}
/*
int i;
for (i = 0; i < 32; i++) {
int addr = APIC_IO_REDTBL + i * 2;
struct io_redirection_register reg;
reg.low = ioapic_read(chip, addr);
kprintf("IRQ query, vector: %x, IRQ: %d\n", reg.vector, irq);
}
panic("here");*/
}
void ioapic_mask(bool_t mask, uint32_t ioapic, uint32_t pin)
{
int index = ioapic * IOAPIC_IRQ_LINES + pin;
if (ioapic >= num_ioapics || pin >= IOAPIC_IRQ_LINES) {
/* silently ignore requests to non existent parts of the interrupt space */
return;
}
if (mask) {
ioredtbl_state[index] |= IOREDTBL_LOW_INTERRUPT_MASK;
} else {
ioredtbl_state[index] &= ~IOREDTBL_LOW_INTERRUPT_MASK;
/* it should not be possible to be unmasking an interrupt, without
* it having been mapped to a vector, assert that this is the case */
assert((ioredtbl_state[index] & 0xff) != 0);
}
ioapic_write(ioapic, IOAPIC_REGSEL, IOREDTBL_LOW(pin));
ioapic_write(ioapic, IOAPIC_WINDOW, ioredtbl_state[index]);
}
void gsi_disable(unsigned gsi)
{
uint32_t lo;
assert(gsi < gsis_no);
lo = ioapic_read(gsis[gsi].ioapic, IO_RED_LO(gsis[gsi].pin));
lo |= 0x10000L; /* MASK */
ioapic_write(gsis[gsi].ioapic, IO_RED_LO(gsis[gsi].pin), lo);;
}
static void ioapic_mask_irq(int irq)
{
assert(!x86_irq_is_enabled());
struct ioapic_info *ioa = &global_ioapics[irq / 24];
int vec = irq % 24;
uint32_t ctl = ioapic_read(ioa, IOREDTBL(vec) + 0);
ctl |= 1 << 16;
ioapic_write(ioa, IOREDTBL(vec) + 0, ctl);
}
void gsi_register(unsigned gsi, unsigned vect)
{
uint32_t lo;
assert(gsi < gsis_no);
assert(vect < 256);
lo = ioapic_read(gsis[gsi].ioapic, IO_RED_LO(gsis[gsi].pin));
ioapic_write(gsis[gsi].ioapic, IO_RED_LO(gsis[gsi].pin),
lo | vect);
}
static void single_ioapic_init(word_t ioapic, cpu_id_t delivery_cpu)
{
uint32_t i;
/* Mask all the IRQs. In doing so we happen to set
* the vector to 0, which we can assert against in
* mask_interrupt to ensure a vector is assigned
* before we unmask */
for (i = 0; i < IOAPIC_IRQ_LINES; i++) {
/* Send to desired cpu */
ioapic_write(ioapic, IOAPIC_REGSEL, IOREDTBL_HIGH(i));
ioapic_write(ioapic, IOAPIC_WINDOW, (ioapic_read(ioapic,
IOAPIC_WINDOW) & MASK(IOREDTBL_HIGH_RESERVED_BITS)) | (delivery_cpu << IOREDTBL_HIGH_RESERVED_BITS));
/* mask and set 0 vector */
ioredtbl_state[i] = IOREDTBL_LOW_INTERRUPT_MASK;
ioapic_write(ioapic, IOAPIC_REGSEL, IOREDTBL_LOW(i));
/* The upper 16 bits are reserved, so we make sure to preserve them */
ioredtbl_state[i] |= ioapic_read(ioapic, IOAPIC_WINDOW) & ~MASK(16);
ioapic_write(ioapic, IOAPIC_WINDOW, ioredtbl_state[i]);
}
}
void ioapic_init_redirection_table(int chip, int pin, int vector, int trigger, int polarity)
{
int addr = APIC_IO_REDTBL + pin * 2;
// kprintf("Init redirection table, pin: %d, addr: %x, vector: %d\n", pin, addr, vector);
//
struct io_redirection_register reg;
reg.low = ioapic_read(chip, addr);
reg.high = ioapic_read(chip, addr + 1);
reg.destmod = APIC_DESTMOD_PHYS;
reg.delmod = APIC_DELMOD_FIXED;
reg.trigger_mode = trigger;
reg.intpol = polarity;
reg.vector = vector;
reg.masked = 1;
reg.dest = 0; //0xff;
ioapic_write(chip, addr, reg.low);
ioapic_write(chip, addr + 1, reg.high);
}
void gsi_setup_done(void)
{
unsigned i;
uint32_t hi, lo;
lo = 0; /* No vector, fixed destination */
lo |= 1L << 16; /* Masked */
hi = lapic_getcurrent() << 24; /* Current Processor */
for (i = 0; i < gsis_no; i++) {
/* Now that we have the proper GSI to IRQ mapping, resolve the
* IOAPIC/PIN of the GSI. */
irqresolve(i);
/* Setup Masked IOAPIC entry with no vector information */
switch (gsis[i].mode) {
default:
printf("Warning: GSI table corrupted. "
"Setting GSI %d to EDGE\n", i);
case EDGE:
break;
case LVLHI:
lo |= (1L << 15);
break;
case LVLLO:
lo |= ((1L << 15) | (1L << 13));
break;
}
ioapic_write(gsis[i].ioapic, IO_RED_LO(gsis[i].pin), lo);
ioapic_write(gsis[i].ioapic, IO_RED_HI(gsis[i].pin), hi);
}
/* 1:1 map GSI <-> Kernel IRQ */
for (i = 0; i < gsis_no; i++)
gsi_register(i, VECT_IRQ0 + i);
}
void ioapic_map_pin_to_vector(word_t ioapic, word_t pin, word_t level,
word_t polarity, word_t vector)
{
uint32_t ioredtbl_high = 0;
uint32_t index = 0;
index = ioapic * IOAPIC_IRQ_LINES + pin;
ioapic_write(ioapic, IOAPIC_REGSEL, IOREDTBL_HIGH(pin));
ioredtbl_high = ioapic_read(ioapic, IOAPIC_WINDOW) & MASK(IOREDTBL_HIGH_RESERVED_BITS);
/* delivery mode: physical mode only, using APIC ID */
ioredtbl_high |= (ioapic_target_cpu << IOREDTBL_HIGH_RESERVED_BITS);
ioapic_write(ioapic, IOAPIC_WINDOW, ioredtbl_high);
/* we do not need to add IRQ_INT_OFFSET to the vector here */
ioredtbl_state[index] = IOREDTBL_LOW_INTERRUPT_MASK |
(level << IOREDTBL_LOW_TRIGGER_MODE_SHIFT) |
(polarity << IOREDTBL_LOW_POLARITY_SHIFT) |
vector;
ioapic_write(ioapic, IOAPIC_REGSEL, IOREDTBL_LOW(pin));
/* the upper 16 bits are reserved */
ioredtbl_state[index] |= ioapic_read(ioapic, IOAPIC_WINDOW) & ~MASK(16);
ioapic_write(ioapic, IOAPIC_WINDOW, ioredtbl_state[index]);
}
void ioapic_disable_irq(int irq)
{
struct apic_irq_to_pin pin_record = irq_map[irq];
int pin = pin_record.pin;
int chip = pin_record.chip;
if (pin != -1 && chip != -1) {
volatile struct io_redirection_register reg;
reg.low = ioapic_read(chip, APIC_IO_REDTBL + pin * 2);
reg.masked = 1;
ioapic_write(chip, APIC_IO_REDTBL + pin * 2, reg.low);
}
}
static void apic_setup(void)
{
/*
* This would the The Right Thing To Do (tm), if only qemu negotiated
* with Xen where the IO-APIC actually sits (which is currently hard
* coded in Xen and can't be controlled externally). Uncomment this code
* once that changed.
ioapic_base_address |= (pci_readb(PCI_ISA_DEVFN, 0x80) & 0x3f) << 10;
*/
ioapic_version = ioapic_read(0x01) & 0xff;
/* Set the IOAPIC ID to the static value used in the MP/ACPI tables. */
ioapic_write(0x00, IOAPIC_ID);
/* NMIs are delivered direct to the BSP. */
lapic_write(APIC_SPIV, APIC_SPIV_APIC_ENABLED | 0xFF);
lapic_write(APIC_LVT0, (APIC_MODE_EXTINT << 8) | APIC_LVT_MASKED);
lapic_write(APIC_LVT1, APIC_MODE_NMI << 8);
/* 8259A ExtInts are delivered through IOAPIC pin 0 (Virtual Wire Mode). */
ioapic_write(0x10, APIC_DM_EXTINT);
ioapic_write(0x11, SET_APIC_ID(LAPIC_ID(0)));
}
int
ioapic_activate(struct device *self, int act)
{
struct ioapic_softc *sc = (struct ioapic_softc *)self;
switch (act) {
case DVACT_RESUME:
/* On resume, reset the APIC id, like we do on boot */
ioapic_write(sc, IOAPIC_ID,
(ioapic_read(sc, IOAPIC_ID) & ~IOAPIC_ID_MASK) |
(sc->sc_apicid << IOAPIC_ID_SHIFT));
}
return (0);
}
void ioapic_change_io_redirection_table(int chip, int pin, int dest, int vec, u32 flags)
{
u32 dlvr;
if (flags & APIC_LOPRI) {
dlvr = APIC_DELMOD_LOWPRI;
} else {
dlvr = APIC_DELMOD_FIXED;
}
struct io_redirection_register reg;
reg.low = ioapic_read(chip, APIC_IO_REDTBL + pin * 2);
reg.high = ioapic_read(chip, APIC_IO_REDTBL + pin * 2 + 1);
reg.dest = dest;
reg.destmod = APIC_DESTMOD_PHYS;
reg.trigger_mode = APIC_TRIGMOD_EDGE;
reg.intpol = APIC_POLARITY_HIGH;
reg.delmod = dlvr;
reg.vector = vec;
ioapic_write(chip, APIC_IO_REDTBL + pin * 2, reg.low);
ioapic_write(chip, APIC_IO_REDTBL + pin * 2 + 1, reg.high);
}
/* Reprogram the APIC ID, and check that it actually got set. */
void
ioapic_set_id(struct ioapic_softc *sc) {
u_int8_t apic_id;
ioapic_write(sc, IOAPIC_ID,
(ioapic_read(sc, IOAPIC_ID) & ~IOAPIC_ID_MASK) |
(sc->sc_apicid << IOAPIC_ID_SHIFT));
apic_id = (ioapic_read(sc, IOAPIC_ID) & IOAPIC_ID_MASK) >>
IOAPIC_ID_SHIFT;
if (apic_id != sc->sc_apicid)
printf(", can't remap to apid %d\n", sc->sc_apicid);
else
printf(", remapped to apid %d\n", sc->sc_apicid);
}
int do_ioapic(struct guest_thread *vm_thread, uint64_t gpa, int destreg,
uint64_t *regp, int store)
{
// TODO: compute an index for the ioapic array.
int ix = 0;
uint32_t offset = gpa & 0xfffff;
/* basic sanity tests. */
DPRINTF("%s: %p 0x%x %p %s\n", __func__, (void *)gpa, destreg, regp, store ? "write" : "read");
if ((offset != 0) && (offset != 0x10)) {
DPRINTF("Bad register offset: 0x%x and has to be 0x0 or 0x10\n", offset);
return -1;
}
if (store) {
ioapic_write(ix, offset, *regp);
} else {
*regp = ioapic_read(ix, offset);
}
}
void IOAPIC::SetupIDsFromMPC() {
union IO_APIC_reg_00 reg_00;
if(system->cpuid->vendor_code != VENDOR_INTEL || APIC_XAPIC(system->smp->apic_version[system->smp->boot_cpu_physical_apicid])) {
printk("xAPIC detected, skipping setting IO-APIC IDs\n");
return;
}
for (int apic = 0; apic < system->smp->nr_ioapics; apic++) {
reg_00.raw = ioapic_read(apic, 0);
int old_id = system->smp->mp_ioapics[apic].mpc_apicid;
if (old_id >= parent->GetPhysicalBroadcast()) {
printk("BIOS bug, IO-APIC#%d ID is %d in the MPC table, fixing up to %d\n",
apic, old_id, reg_00.bits.ID);
system->smp->mp_ioapics[apic].mpc_apicid = reg_00.bits.ID;
}
if (old_id != system->smp->mp_ioapics[apic].mpc_apicid)
for (int i = 0; i < system->smp->mp_irq_entries; i++)
if (system->smp->mp_irqs[i].mpc_dstapic == old_id)
system->smp->mp_irqs[i].mpc_dstapic = system->smp->mp_ioapics[apic].mpc_apicid;
printk( "Changing IO-APIC physical APIC ID to %d ...", system->smp->mp_ioapics[apic].mpc_apicid);
reg_00.bits.ID = system->smp->mp_ioapics[apic].mpc_apicid;
ioapic_write(apic, 0, reg_00.raw);
reg_00.raw = ioapic_read(apic, 0);
if (reg_00.bits.ID != system->smp->mp_ioapics[apic].mpc_apicid)
printk("could not set ID!\n");
else
printk(" ok.\n");
}
}
void IOAPIC::MaskEntry(int apic, int pin) {
union entry_union eu;
eu.entry.mask = 1;
ioapic_write(apic, 0x10 + 2 * pin, eu.w1);
ioapic_write(apic, 0x11 + 2 * pin, eu.w2);
}
void IOAPIC::WriteEntry(int apic, int pin, struct IO_APIC_route_entry e) {
union entry_union eu;
eu.entry = e;
ioapic_write(apic, 0x11 + 2*pin, eu.w2);
ioapic_write(apic, 0x10 + 2*pin, eu.w1);
}
void ioapic_enable(uint8_t num, uint32_t cpuid) {
info("enable irq %d (int %d) on cpu %d\n", num, IRQ_NUM(num), cpuid);
ioapic_write(_the_ioapic, IOAPIC_REG_TABLE + (2 * num), (IRQ_NUM(num)));
ioapic_write(_the_ioapic, IOAPIC_REG_TABLE + ((2 * num) + 1), cpuid << 24);
}
void ioapic_disable(uint8_t num, uint32_t cpuid) {
info("disable irq %d (int %d) on cpu %d\n", num, IRQ_NUM(num), cpuid);
ioapic_write(_the_ioapic, IOAPIC_REG_TABLE + (2 * num), IOAPIC_INT_MASKED | (IRQ_NUM(num)));
ioapic_write(_the_ioapic, IOAPIC_REG_TABLE + ((2 * num) + 1), 0);
}
