/* support for the IA-PC IOAPIC architecture, i.e. LAPICs and I/O APICs */

#include <stdio.h>

#include <stdlib.h>

#include <stdint.h>

#include <stdbool.h>

#include <string.h>

#include <assert.h>

#include <ccan/compiler/compiler.h>

#include <ccan/darray/darray.h>

#include <ukernel/ioport.h>

#include <ukernel/interrupt.h>

#include <ukernel/x86.h>

#include <ukernel/acpi.h>

#include <ukernel/mm.h>

#include <ukernel/cpu.h>

#include <ukernel/util.h>

#include <ukernel/misc.h>

/* fields in IA32_APIC_BASE */

#define AB_BOOTSTRAP_CPU (1 << 8)

#define AB_GLOBAL_ENABLE (1 << 11)

/* base address is bits 31..12, i.e. 32-bit ~PAGE_MASK */

/* LAPIC MMIO registers (at 16-byte offsets, of which the low 32 bits can be

* read (perhaps also written) with the 32-bit MMIO functions.)

*/

#define APIC_LAPIC_ID 0x20 /* local APIC ID (RO [partially RW]) */

#define APIC_LAPIC_VER 0x30 /* LAPIC version (RO) */

#define APIC_TPR 0x80 /* task priority (RW) */

#define APIC_APR 0x90 /* arbitrarion priority (RO, >p4) */

#define APIC_PPR 0xa0 /* CPU priority (RO) */

#define APIC_EOI 0xb0 /* EOI register (WO) */

#define APIC_RRD 0xc0 /* remote read register (RO, >p4) */

#define APIC_SIVR 0xf0 /* spurious-interrupt vector (RW) */

#define APIC_ERRSTATUS 0x280 /* error status register (RO) */

/* long registers. valid indexes are 0..7, low to high dword; except for

* APIC_ICR(), which only has 64 bits.

*/

#define APIC_ISR(ix) (0x100 + (ix) * 0x10) /* in-service register (RO) */

#define APIC_TMR(ix) (0x180 + (ix) * 0x10) /* trigger mode reg (RO) */

#define APIC_IRR(ix) (0x200 + (ix) * 0x10) /* interrupt request reg (RO) */

#define APIC_ICR(ix) (0x300 + (ix) * 0x10) /* 0..1: interrupt cmd reg (RW) */

#define APIC_LVT_TIMER 0x320 /* local timer vector */

#define APIC_LVT_THERM 0x330 /* same, but thermal */

#define APIC_LVT_PERF 0x340 /* same, performance monitoring */

#define APIC_LVT_LINT0 0x350 /* same, LINT0 */

#define APIC_LVT_LINT1 0x360 /* same, LINT1 */

#define APIC_LVT_ERROR 0x370 /* same, error vector */

#define APIC_TIMER_INIT 0x380 /* LAPIC timer initial count register */

#define APIC_TIMER_CUR 0x390 /* LAPIC timer current count register (RO) */

#define APIC_TIMER_DCR 0x3e0 /* LAPIC timer divide cfg register */

/* etc... (add as needed) */

/* IOAPIC registers (via ioapic_{read,write}()) */

#define IOAPICID 0x00

#define IOAPICVER 0x01

#define IOAPICARB 0x02

#define IOREDTBL(n) (0x10 + 2 * (n))

/* (this is intended as per-CPU. as there is no malloc in apic_init(), it'll

* be allocated on a "max CPUs allowed by this kernel" basis in the BSS

* section.)

*/

struct lapic_info

};

struct ioapic_info

};

bool apic_enabled = false, apic_disable_opt = false;

struct lapic_info cpu_apics[1]; /* still an UP kernel. */

struct ioapic_info *global_ioapics = NULL;

int num_global_ioapics = 0;

#if 0

/* TODO: export this somewhere */

void apic_send_self_ipi(int vector)

{

assert(vector > 0xf && vector <= 0xff);

struct lapic_info *apic = &cpu_apics[0];

/* self-interrupt destination shorthand, vector as given */

mm_outl(apic->base_addr + APIC_ICR(0), vector | (1 << 18));

}

#endif

/* there are at most 256 32-bit registers in a pre-2013 IOAPIC. */

static inline void ioapic_write(

struct ioapic_info *ioa,

uint8_t address,

uint32_t value)

{

assert(!x86_irq_is_enabled());

mm_outb(ioa->base_addr + 0, address); /* IOREGSEL */

mm_outl(ioa->base_addr + 0x10, value); /* IOWIN */

}

static inline uint32_t ioapic_read(struct ioapic_info *ioa, uint8_t address)

{

assert(!x86_irq_is_enabled());

mm_outb(ioa->base_addr + 0, address);

return mm_inl(ioa->base_addr + 0x10);

}

/* EOI is actually sent to the local APIC, not the I/O APIC as the function's

* name would indicate.

*/

static void ioapic_send_eoi(int irq)

{

assert(!x86_irq_is_enabled());

struct lapic_info *apic = &cpu_apics[0];

/* the specs say that 0 is the right thing to write here. */

mm_outl(apic->base_addr + APIC_EOI, 0);

}

static void ioapic_mask_irq(int irq)

{

assert(!x86_irq_is_enabled());

struct ioapic_info *ioa = &global_ioapics[irq / 24];

int vec = irq % 24;

uint32_t ctl = ioapic_read(ioa, IOREDTBL(vec) + 0);

ctl |= 1 << 16;

ioapic_write(ioa, IOREDTBL(vec) + 0, ctl);

}

static void ioapic_unmask_irq(int irq, unsigned flags)

{

struct ioapic_info *ioa = &global_ioapics[irq / 24];

int vec = irq % 24;

/* transfer the act-low/level-trig pattern into the APIC setting word

* directly.

*/

assert(IHF_ACT_LOW == (1 << 0));

assert(IHF_LEVEL_TRIG == (1 << 2));

int trig_pol = (flags & (IHF_ACT_LOW | IHF_LEVEL_TRIG)) << 13;

assert(CHECK_FLAG(trig_pol, 1 << 13)

== CHECK_FLAG(flags, IHF_ACT_LOW));

assert(CHECK_FLAG(trig_pol, 1 << 15)

== CHECK_FLAG(flags, IHF_LEVEL_TRIG));

/* bit 16 = interrupt mask bit.

* 15 = trigger mode (0 = edge, 1 = level)

* 13 = pin polarity (0 = active high, 1 = active low)

* 7..0 = interrupt vector (0x10..0xff)

*

* other fields select fixed (regular) delivery to a physical LAPIC (CPU

* 0, since mung is still not a SMP kernel).

*/

uint32_t ctl = ioapic_read(ioa, IOREDTBL(vec) + 0);

ctl &= ~((1 << 16) | (1 << 13) | (1 << 15) | 0xff);

ctl |= trig_pol | (0x20 + irq);

ioapic_write(ioa, IOREDTBL(vec) + 0, ctl);

}

void isr_apic_bottom(struct x86_exregs *regs)

{

struct lapic_info *apic = &cpu_apics[0];

#if 0

printf("%s: frame at %p, exregs=%p, frame_len=%u (size=%u)\n",

__func__, &apic, regs, (unsigned)x86_frame_len(regs),

(unsigned)sizeof(*regs));

#endif

/* figure out which interrupt this is from the ISRs. */

int vecnum;

bool vec_found = false;

for(int i=0; i <= apic->max_vector; i += 32) {

uint32_t isr_limb = mm_inl(apic->base_addr + APIC_ISR(i >> 5));

if(i == 0) isr_limb &= ~0xffffu;

if(isr_limb != 0) {

vecnum = i + ffsl(isr_limb) - 1;

vec_found = true;

break;

}

}

if(!vec_found) {

apic->num_spurious++;

ioapic_send_eoi(0); /* just in case. */

return;

}

#if 0

if(vecnum == 0x21) {

printf("i'm a keyboard, toot toot\n");

#define KBD_STATUS_REG 0x64

#define KBD_DATA_REG 0x60

#define KBD_STAT_OBF 0x01

for(;;) {

uint8_t st = inb(KBD_STATUS_REG);

if((st & KBD_STAT_OBF) == 0) break;

inb(KBD_DATA_REG); /* and throw it away */

}

} else {

printf("APIC interrupt vector %d\n", vecnum);

}

#endif

regs->reason = vecnum;

isr_irq_bottom(regs);

return;

#if 0

/* TODO: move this into an x86_print_exregs() function */

printf("exregs: reason=%#lx, es=%#lx, ds=%#lx, cs=%#lx, ss=%#lx\n"

"\tedi=%#lx, esi=%#lx, ebp=%#lx, __esp=%#lx\n"

"\tebx=%#lx, edx=%#lx, ecx=%#lx, eax=%#lx\n"

"\terror=%#lx, eip=%#lx, eflags=%#lx, esp=%#lx\n",

regs->reason, regs->es, regs->ds, regs->cs, regs->ss,

regs->edi, regs->esi, regs->ebp, regs->__esp,

regs->ebx, regs->edx, regs->ecx, regs->eax,

regs->error, regs->eip, regs->eflags, regs->esp);

#endif

}

int ioapic_route_legacy_irq(int irqnum, int ioa_vector)

{

uint32_t global_int = irqnum;

int polarity = 0; /* active high */

int level_sens = 0; /* edge sensitive */

if(irqnum < 16) {

/* adjust per MADT source override (ISA bus) */

for(int offs = 0, len = 0,

max_offs = acpi_madt->h.length - sizeof(acpi_madt->h) - 1;

offs <= max_offs;

offs += len)

{

assert(offs >= 0);

const uint8_t *b = &acpi_madt->data[offs];

const int devtype = b[0];

len = b[1];

if(len == 0 || offs + len - 1 > max_offs) break;

if(devtype != 2) continue;

uint8_t bus = b[2], source = b[3];

if(bus == 0 && source == irqnum) {

memcpy(&global_int, &b[4], 4);

uint16_t flags;

memcpy(&flags, &b[8], 2);

switch(flags & 0x3) {

case 0: /* bus default, for ISA it's active-low */

case 3: /* explicit active-low */

polarity = 1;

break;

case 1: /* active high */

polarity = 0;

break;

default:

panic("unknown polarity attribute in MADT!");

}

switch((flags >> 2) & 0x3) {

case 0: case 1:

/* default = edge sensitive */

level_sens = 0;

break;

case 3:

/* level */

level_sens = 1;

break;

default:

panic("unknown sensitivity attribute in MADT!");

}

printf("IOAPIC: override IRQ%d -> int# %u (act-%s, %s)\n",

irqnum, global_int, polarity == 1 ? "low" : "high",

level_sens ? "edge" : "level");

break;

}

}

}

/* find the relevant IOAPIC. */

struct ioapic_info *ioa = NULL;

int redir;

for(int i=0, base=0; i < num_global_ioapics; i++) {

struct ioapic_info *cand = &global_ioapics[i];

if(base <= global_int && base + cand->max_redir >= global_int) {

ioa = cand;

redir = global_int - base;

break;

}

base += cand->max_redir + 1;

}

if(ioa == NULL) {

printf("I/O APIC for int# %u not found!\n", global_int);

panic("error return goes here");

}

ioapic_write(ioa, IOREDTBL(redir) + 1, 0); /* dest APIC 0 (UP mode) */

/* not masked, physical dest mode, fixed delivery, vector=@vector */

ioapic_write(ioa, IOREDTBL(redir) + 0,

(level_sens << 15) | (polarity << 13) | ioa_vector);

return 0;

}

/* per-CPU LAPIC init. */

static COLD void lapic_init(void)

{

struct lapic_info *apic = &cpu_apics[0];

uint64_t base_msr = x86_rdmsr(IA32_APIC_BASE);

printf("APIC_BASE=%#08x:%08x",

(unsigned)(base_msr >> 32), (unsigned)base_msr);

*apic = (struct lapic_info){

/* FIXME: handle 36 bits, or as indicated by CPUID leaf 80000008h */

.phys_base = (uint32_t)base_msr & ~0xfffu,

.bootstrap = CHECK_FLAG(base_msr, AB_BOOTSTRAP_CPU),

.enabled = CHECK_FLAG(base_msr, AB_GLOBAL_ENABLE),

};

printf(": addr=%#08x, bootstrap=%s, enabled=%s\n",

apic->base_addr, btos(apic->bootstrap), btos(apic->enabled));

/* flip the enable bit */

base_msr |= AB_GLOBAL_ENABLE;

x86_wrmsr(IA32_APIC_BASE, base_msr);

/* map the APIC in wherever. its MMIO space is, conveniently enough,

* exactly 4k big.

*/

uintptr_t resv = reserve_heap_page();

/* FIXME: set a flag that makes the APIC range uncacheable. */

put_supervisor_page(resv, apic->phys_base >> PAGE_BITS);

apic->base_addr = resv;

apic->max_vector = 0xff; /* TODO: use for something */

uint32_t lapic_id = mm_inl(apic->base_addr + APIC_LAPIC_ID),

lapic_ver = mm_inl(apic->base_addr + APIC_LAPIC_VER);

printf("LAPIC_id=%u, LAPIC_ver=%#08x\n", lapic_id, lapic_ver);

uint32_t tpr = mm_inl(apic->base_addr + APIC_TPR);

printf(" TPR=%#08x", tpr);

uint32_t sivr = mm_inl(apic->base_addr + APIC_SIVR);

printf(" SIVR=%#08x\n", sivr);

#if 0

printf(" LVT: timer=%#08x, therm=%#08x, perf=%#08x, l0=%#08x, l1=%#08x\n",

mm_inl(apic->base_addr + APIC_LVT_TIMER),

mm_inl(apic->base_addr + APIC_LVT_THERM),

mm_inl(apic->base_addr + APIC_LVT_PERF),

mm_inl(apic->base_addr + APIC_LVT_LINT0),

mm_inl(apic->base_addr + APIC_LVT_LINT1));

printf(" error=%#08x\n", mm_inl(apic->base_addr + APIC_LVT_ERROR));

#endif

/* enable local APIC. */

mm_orl(apic->base_addr + APIC_SIVR, 1 << 8);

}

COLD int apic_probe(void)

{

apic_enabled = false;

if(!CHECK_FLAG(get_features()->edx, X86_FEATURE_D_APIC)) {

printf("APIC not detected in CPUID flags.\n");

return -1;

}

if(apic_disable_opt) {

printf("APIC found, but instructed to disable it\n");

return -1;

}

lapic_init(); /* per CPU */

apic_enabled = true;

return 0;

}

COLD int ioapic_init(struct pic_ops *ops)

{

if(acpi_madt == NULL) {

/* TODO: crawl over MPS tables instead */

printf("Can't find IOAPIC without ACPI MADT!\n");

return -1;

}

/* scan the MADT for I/O APIC entries (type 1) */

darray(struct ioapic_info) ioas = darray_new();

for(int offs = 0, len = 0,

max_offs = acpi_madt->h.length - sizeof(acpi_madt->h) - 1;

offs <= max_offs;

offs += len)

{

assert(offs >= 0);

const uint8_t *b = &acpi_madt->data[offs];

const int devtype = b[0];

len = b[1];

if(len == 0 || offs + len - 1 > max_offs) break;

if(devtype != 1) continue;

struct ioapic_info info;

info.base_addr = 0;

info.apic_id = b[2];

memcpy(&info.phys_base, &b[4], 4);

memcpy(&info.gsi_base, &b[8], 4);

darray_push(ioas, info);

}

if(darray_empty(ioas)) {

printf("No I/O APIC found in MADT!\n");

darray_free(ioas);

return -1;

}

num_global_ioapics = ioas.size;

global_ioapics = realloc(ioas.item,

sizeof(struct ioapic_info) * ioas.size);

if(global_ioapics == NULL) {

/* wasting some RAM, in ur punani */

global_ioapics = ioas.item;

}

printf("... seeing %d global I/O APICs.\n", num_global_ioapics);

/* TODO: could also sort them so that this invariant is true, but

* whatever.

*/

for(int i=0; i < num_global_ioapics; i++) {

if(global_ioapics[i].apic_id != i) {

printf("Weird I/O APIC order: offset=%d, id=%d!\n",

i, global_ioapics[i].apic_id);

panic("foo!");

}

}

/* map them in & dump some info. */

for(int i=0; i < num_global_ioapics; i++) {

uintptr_t resv = reserve_heap_page();

struct ioapic_info *ioa = &global_ioapics[i];

/* TODO: disable caching for MMIO maps! */

put_supervisor_page(resv, ioa->phys_base >> 12);

ioa->base_addr = resv;

printf("I/O APIC %d:\n", (int)ioa->apic_id);

uint32_t ver = ioapic_read(ioa, IOAPICVER);

printf(" IOAPICID=%#08x, IOAPICVER=%#08x\n",

ioapic_read(ioa, IOAPICID), ver);

ioa->max_redir = (ver >> 16) & 0xff;

}

/* mask all vectors and set them to a 1:1 mapping, with exceptions made by

* ioapic_route_legacy_irq() later.

*

* FIXME: this is likely not good enough. Bochs maps the interval timer to

* IRQ#2 (which is the unused cascade line for XT-PIC), which is fine for

* the microkernel, but other things are a bit iffy.

*/

for(int i=0; i < num_global_ioapics; i++) {

struct ioapic_info *ioa = &global_ioapics[i];

for(int line=0; line < 24; line++) {

uint32_t destctl = ioapic_read(ioa, IOREDTBL(line) + 1),

intctl = ioapic_read(ioa, IOREDTBL(line) + 0);

destctl = 0; /* dest = physical 0 (UP mode) */

/* physical destination mode, fixed delivery mode, also reset the

* vector field

*/

intctl &= ~((1 << 11) | 0x700 | 0xff);

intctl |= 0x10000; /* mask */

intctl |= i * 24 + line;

ioapic_write(ioa, IOREDTBL(line) + 1, destctl);

ioapic_write(ioa, IOREDTBL(line) + 0, intctl);

}

}

*ops = (struct pic_ops){

.send_eoi = &ioapic_send_eoi,

.mask_irq = &ioapic_mask_irq,

.unmask_irq = &ioapic_unmask_irq,

};

/* preallocate all vectors, permit defer, and direct to userspace. */

int num_ints = num_global_ioapics * 24 - 1;

set_irq_handler(num_ints, NULL, 0);

return num_ints;

}