int
msi_release(int *irqs, int count)
{
struct msi_intsrc *msi, *first;
int i;
mtx_lock(&msi_lock);
first = (struct msi_intsrc *)intr_lookup_source(irqs[0]);
if (first == NULL) {
mtx_unlock(&msi_lock);
return (ENOENT);
}
/* Make sure this isn't an MSI-X message. */
if (first->msi_msix) {
mtx_unlock(&msi_lock);
return (EINVAL);
}
/* Make sure this message is allocated to a group. */
if (first->msi_first == NULL) {
mtx_unlock(&msi_lock);
return (ENXIO);
}
/*
* Make sure this is the start of a group and that we are releasing
* the entire group.
*/
if (first->msi_first != first || first->msi_count != count) {
mtx_unlock(&msi_lock);
return (EINVAL);
}
KASSERT(first->msi_dev != NULL, ("unowned group"));
/* Clear all the extra messages in the group. */
for (i = 1; i < count; i++) {
msi = (struct msi_intsrc *)intr_lookup_source(irqs[i]);
KASSERT(msi->msi_first == first, ("message not in group"));
KASSERT(msi->msi_dev == first->msi_dev, ("owner mismatch"));
msi->msi_first = NULL;
msi->msi_dev = NULL;
apic_free_vector(msi->msi_cpu, msi->msi_vector, msi->msi_irq);
msi->msi_vector = 0;
}
/* Clear out the first message. */
first->msi_first = NULL;
first->msi_dev = NULL;
apic_free_vector(first->msi_cpu, first->msi_vector, first->msi_irq);
first->msi_vector = 0;
first->msi_count = 0;
first->msi_maxcount = 0;
free(first->msi_irqs, M_MSI);
first->msi_irqs = NULL;
mtx_unlock(&msi_lock);
return (0);
}
static void
atpic_register_sources(struct pic *pic)
{
struct atpic *ap = (struct atpic *)pic;
struct atpic_intsrc *ai;
int i;
/*
* If any of the ISA IRQs have an interrupt source already, then
* assume that the I/O APICs are being used and don't register any
* of our interrupt sources. This makes sure we don't accidentally
* use mixed mode. The "accidental" use could otherwise occur on
* machines that route the ACPI SCI interrupt to a different ISA
* IRQ (at least one machine routes it to IRQ 13) thus disabling
* that APIC ISA routing and allowing the ATPIC source for that IRQ
* to leak through. We used to depend on this feature for routing
* IRQ0 via mixed mode, but now we don't use mixed mode at all.
*
* To avoid the slave not register sources after the master
* registers its sources, register all IRQs when this function is
* called on the master.
*/
if (ap != &atpics[MASTER])
return;
for (i = 0; i < NUM_ISA_IRQS; i++)
if (intr_lookup_source(i) != NULL)
return;
/* Loop through all interrupt sources and add them. */
for (i = 0, ai = atintrs; i < NUM_ISA_IRQS; i++, ai++) {
if (i == ICU_SLAVEID)
continue;
intr_register_source(&ai->at_intsrc);
}
}
int
msix_release(int irq)
{
struct msi_intsrc *msi;
mtx_lock(&msi_lock);
msi = (struct msi_intsrc *)intr_lookup_source(irq);
if (msi == NULL) {
mtx_unlock(&msi_lock);
return (ENOENT);
}
/* Make sure this is an MSI-X message. */
if (!msi->msi_msix) {
mtx_unlock(&msi_lock);
return (EINVAL);
}
KASSERT(msi->msi_dev != NULL, ("unowned message"));
/* Clear out the message. */
msi->msi_first = NULL;
msi->msi_dev = NULL;
apic_free_vector(msi->msi_cpu, msi->msi_vector, msi->msi_irq);
msi->msi_vector = 0;
msi->msi_msix = 0;
msi->msi_count = 0;
msi->msi_maxcount = 0;
mtx_unlock(&msi_lock);
return (0);
}
static int
nexus_xen_config_intr(device_t dev, int irq, enum intr_trigger trig,
enum intr_polarity pol)
{
int ret;
/*
* ISA and PCI intline IRQs are not preregistered on Xen, so
* intercept calls to configure those and register them on the fly.
*/
if ((irq < first_msi_irq) && (intr_lookup_source(irq) == NULL)) {
ret = xen_register_pirq(irq, trig, pol);
if (ret != 0)
return (ret);
nexus_add_irq(irq);
}
return (intr_config_intr(irq, trig, pol));
}
void
evtchn_do_upcall(struct trapframe *frame)
{
unsigned long l1, l2;
unsigned int l1i, l2i, port;
int irq, cpu;
shared_info_t *s;
vcpu_info_t *vcpu_info;
cpu = PCPU_GET(cpuid);
s = HYPERVISOR_shared_info;
vcpu_info = &s->vcpu_info[cpu];
vcpu_info->evtchn_upcall_pending = 0;
/* NB. No need for a barrier here -- XCHG is a barrier on x86. */
l1 = xen_xchg(&vcpu_info->evtchn_pending_sel, 0);
while (l1 != 0) {
l1i = __ffs(l1);
l1 &= ~(1 << l1i);
while ((l2 = active_evtchns(cpu, s, l1i)) != 0) {
l2i = __ffs(l2);
port = (l1i * LONG_BIT) + l2i;
if ((irq = evtchn_to_irq[port]) != -1) {
struct intsrc *isrc = intr_lookup_source(irq);
/*
* ack
*/
mask_evtchn(port);
clear_evtchn(port);
intr_execute_handlers(isrc, frame);
} else {
evtchn_device_upcall(port);
}
}
}
}
int
msi_map(int irq, uint64_t *addr, uint32_t *data)
{
struct msi_intsrc *msi;
mtx_lock(&msi_lock);
msi = (struct msi_intsrc *)intr_lookup_source(irq);
if (msi == NULL) {
mtx_unlock(&msi_lock);
return (ENOENT);
}
/* Make sure this message is allocated to a device. */
if (msi->msi_dev == NULL) {
mtx_unlock(&msi_lock);
return (ENXIO);
}
/*
* If this message isn't an MSI-X message, make sure it's part
* of a group, and switch to the first message in the
* group.
*/
if (!msi->msi_msix) {
if (msi->msi_first == NULL) {
mtx_unlock(&msi_lock);
return (ENXIO);
}
msi = msi->msi_first;
}
*addr = INTEL_ADDR(msi);
*data = INTEL_DATA(msi);
mtx_unlock(&msi_lock);
return (0);
}
/**
* @brief Main vmbus driver initialization routine.
*
* Here, we
* - initialize the vmbus driver context
* - setup various driver entry points
* - invoke the vmbus hv main init routine
* - get the irq resource
* - invoke the vmbus to add the vmbus root device
* - setup the vmbus root device
* - retrieve the channel offers
*/
static int
vmbus_bus_init(void)
{
struct ioapic_intsrc {
struct intsrc io_intsrc;
u_int io_irq;
u_int io_intpin:8;
u_int io_vector:8;
u_int io_cpu:8;
u_int io_activehi:1;
u_int io_edgetrigger:1;
u_int io_masked:1;
int io_bus:4;
uint32_t io_lowreg;
};
int ret;
unsigned int vector = 0;
struct intsrc *isrc;
struct ioapic_intsrc *intpin;
if (vmbus_inited)
return (0);
vmbus_inited = 1;
ret = hv_vmbus_init();
if (ret) {
if(bootverbose)
printf("Error VMBUS: Hypervisor Initialization Failed!\n");
return (ret);
}
ret = swi_add(&hv_msg_intr_event, "hv_msg", vmbus_msg_swintr,
NULL, SWI_CLOCK, 0, &msg_swintr);
if (ret)
goto cleanup;
/*
* Message SW interrupt handler checks a per-CPU page and
* thus the thread needs to be bound to CPU-0 - which is where
* all interrupts are processed.
*/
ret = intr_event_bind(hv_msg_intr_event, 0);
if (ret)
goto cleanup1;
ret = swi_add(&hv_event_intr_event, "hv_event", hv_vmbus_on_events,
NULL, SWI_CLOCK, 0, &event_swintr);
if (ret)
goto cleanup1;
intr_res = bus_alloc_resource(vmbus_devp,
SYS_RES_IRQ, &vmbus_rid, vmbus_irq, vmbus_irq, 1, RF_ACTIVE);
if (intr_res == NULL) {
ret = ENOMEM; /* XXXKYS: Need a better errno */
goto cleanup2;
}
/*
* Setup interrupt filter handler
*/
ret = bus_setup_intr(vmbus_devp, intr_res,
INTR_TYPE_NET | INTR_MPSAFE, hv_vmbus_isr, NULL,
NULL, &vmbus_cookiep);
if (ret != 0)
goto cleanup3;
ret = bus_bind_intr(vmbus_devp, intr_res, 0);
if (ret != 0)
goto cleanup4;
isrc = intr_lookup_source(vmbus_irq);
if ((isrc == NULL) || (isrc->is_event == NULL)) {
ret = EINVAL;
goto cleanup4;
}
/* vector = isrc->is_event->ie_vector; */
intpin = (struct ioapic_intsrc *)isrc;
vector = intpin->io_vector;
if(bootverbose)
printf("VMBUS: irq 0x%x vector 0x%x\n", vmbus_irq, vector);
/**
* Notify the hypervisor of our irq.
*/
smp_rendezvous(NULL, hv_vmbus_synic_init, NULL, &vector);
/**
* Connect to VMBus in the root partition
*/
ret = hv_vmbus_connect();
if (ret)
goto cleanup4;
hv_vmbus_request_channel_offers();
return (ret);
cleanup4:
/*
* remove swi, bus and intr resource
*/
bus_teardown_intr(vmbus_devp, intr_res, vmbus_cookiep);
cleanup3:
bus_release_resource(vmbus_devp, SYS_RES_IRQ, vmbus_rid, intr_res);
cleanup2:
swi_remove(event_swintr);
cleanup1:
swi_remove(msg_swintr);
cleanup:
hv_vmbus_cleanup();
return (ret);
}
void
nexus_init_resources(void)
{
int irq;
/*
* XXX working notes:
*
* - IRQ resource creation should be moved to the PIC/APIC driver.
* - DRQ resource creation should be moved to the DMAC driver.
* - The above should be sorted to probe earlier than any child busses.
*
* - Leave I/O and memory creation here, as child probes may need them.
* (especially eg. ACPI)
*/
/*
* IRQ's are on the mainboard on old systems, but on the ISA part
* of PCI->ISA bridges. There would be multiple sets of IRQs on
* multi-ISA-bus systems. PCI interrupts are routed to the ISA
* component, so in a way, PCI can be a partial child of an ISA bus(!).
* APIC interrupts are global though.
*/
irq_rman.rm_start = 0;
irq_rman.rm_type = RMAN_ARRAY;
irq_rman.rm_descr = "Interrupt request lines";
irq_rman.rm_end = NUM_IO_INTS - 1;
if (rman_init(&irq_rman))
panic("nexus_init_resources irq_rman");
/*
* We search for regions of existing IRQs and add those to the IRQ
* resource manager.
*/
for (irq = 0; irq < NUM_IO_INTS; irq++)
if (intr_lookup_source(irq) != NULL)
if (rman_manage_region(&irq_rman, irq, irq) != 0)
panic("nexus_init_resources irq_rman add");
/*
* ISA DMA on PCI systems is implemented in the ISA part of each
* PCI->ISA bridge and the channels can be duplicated if there are
* multiple bridges. (eg: laptops with docking stations)
*/
drq_rman.rm_start = 0;
#ifdef PC98
drq_rman.rm_end = 3;
#else
drq_rman.rm_end = 7;
#endif
drq_rman.rm_type = RMAN_ARRAY;
drq_rman.rm_descr = "DMA request lines";
/* XXX drq 0 not available on some machines */
if (rman_init(&drq_rman)
|| rman_manage_region(&drq_rman,
drq_rman.rm_start, drq_rman.rm_end))
panic("nexus_init_resources drq_rman");
/*
* However, IO ports and Memory truely are global at this level,
* as are APIC interrupts (however many IO APICS there turn out
* to be on large systems..)
*/
port_rman.rm_start = 0;
port_rman.rm_end = 0xffff;
port_rman.rm_type = RMAN_ARRAY;
port_rman.rm_descr = "I/O ports";
if (rman_init(&port_rman)
|| rman_manage_region(&port_rman, 0, 0xffff))
panic("nexus_init_resources port_rman");
mem_rman.rm_start = 0;
mem_rman.rm_end = ~0ul;
mem_rman.rm_type = RMAN_ARRAY;
mem_rman.rm_descr = "I/O memory addresses";
if (rman_init(&mem_rman)
|| rman_manage_region(&mem_rman, 0, ~0))
panic("nexus_init_resources mem_rman");
}
int
msix_alloc(device_t dev, int *irq)
{
struct msi_intsrc *msi;
u_int cpu;
int i, vector;
if (!msi_enabled)
return (ENXIO);
again:
mtx_lock(&msi_lock);
/* Find a free IRQ. */
for (i = FIRST_MSI_INT; i < FIRST_MSI_INT + NUM_MSI_INTS; i++) {
msi = (struct msi_intsrc *)intr_lookup_source(i);
/* End of allocated sources, so break. */
if (msi == NULL)
break;
/* Stop at the first free source. */
if (msi->msi_dev == NULL)
break;
}
/* Do we need to create a new source? */
if (msi == NULL) {
/* If we would exceed the max, give up. */
if (i + 1 > FIRST_MSI_INT + NUM_MSI_INTS) {
mtx_unlock(&msi_lock);
return (ENXIO);
}
mtx_unlock(&msi_lock);
/* Create a new source. */
msi_create_source();
goto again;
}
/* Allocate an IDT vector. */
cpu = intr_next_cpu();
vector = apic_alloc_vector(cpu, i);
if (vector == 0) {
mtx_unlock(&msi_lock);
return (ENOSPC);
}
if (bootverbose)
printf("msi: routing MSI-X IRQ %d to local APIC %u vector %u\n",
msi->msi_irq, cpu, vector);
/* Setup source. */
msi->msi_cpu = cpu;
msi->msi_dev = dev;
msi->msi_first = msi;
msi->msi_vector = vector;
msi->msi_msix = 1;
msi->msi_count = 1;
msi->msi_maxcount = 1;
msi->msi_irqs = NULL;
KASSERT(msi->msi_intsrc.is_handlers == 0, ("dead MSI-X has handlers"));
mtx_unlock(&msi_lock);
*irq = i;
return (0);
}
/*
* Try to allocate 'count' interrupt sources with contiguous IDT values.
*/
int
msi_alloc(device_t dev, int count, int maxcount, int *irqs)
{
struct msi_intsrc *msi, *fsrc;
u_int cpu;
int cnt, i, *mirqs, vector;
if (!msi_enabled)
return (ENXIO);
if (count > 1)
mirqs = malloc(count * sizeof(*mirqs), M_MSI, M_WAITOK);
else
mirqs = NULL;
again:
mtx_lock(&msi_lock);
/* Try to find 'count' free IRQs. */
cnt = 0;
for (i = FIRST_MSI_INT; i < FIRST_MSI_INT + NUM_MSI_INTS; i++) {
msi = (struct msi_intsrc *)intr_lookup_source(i);
/* End of allocated sources, so break. */
if (msi == NULL)
break;
/* If this is a free one, save its IRQ in the array. */
if (msi->msi_dev == NULL) {
irqs[cnt] = i;
cnt++;
if (cnt == count)
break;
}
}
/* Do we need to create some new sources? */
if (cnt < count) {
/* If we would exceed the max, give up. */
if (i + (count - cnt) > FIRST_MSI_INT + NUM_MSI_INTS) {
mtx_unlock(&msi_lock);
free(mirqs, M_MSI);
return (ENXIO);
}
mtx_unlock(&msi_lock);
/* We need count - cnt more sources. */
while (cnt < count) {
msi_create_source();
cnt++;
}
goto again;
}
/* Ok, we now have the IRQs allocated. */
KASSERT(cnt == count, ("count mismatch"));
/* Allocate 'count' IDT vectors. */
cpu = intr_next_cpu();
vector = apic_alloc_vectors(cpu, irqs, count, maxcount);
if (vector == 0) {
mtx_unlock(&msi_lock);
free(mirqs, M_MSI);
return (ENOSPC);
}
/* Assign IDT vectors and make these messages owned by 'dev'. */
fsrc = (struct msi_intsrc *)intr_lookup_source(irqs[0]);
for (i = 0; i < count; i++) {
msi = (struct msi_intsrc *)intr_lookup_source(irqs[i]);
msi->msi_cpu = cpu;
msi->msi_dev = dev;
msi->msi_vector = vector + i;
if (bootverbose)
printf(
"msi: routing MSI IRQ %d to local APIC %u vector %u\n",
msi->msi_irq, msi->msi_cpu, msi->msi_vector);
msi->msi_first = fsrc;
KASSERT(msi->msi_intsrc.is_handlers == 0,
("dead MSI has handlers"));
}
fsrc->msi_count = count;
fsrc->msi_maxcount = maxcount;
if (count > 1)
bcopy(irqs, mirqs, count * sizeof(*mirqs));
fsrc->msi_irqs = mirqs;
mtx_unlock(&msi_lock);
return (0);
}
static int
msi_assign_cpu(struct intsrc *isrc, u_int apic_id)
{
struct msi_intsrc *sib, *msi = (struct msi_intsrc *)isrc;
int old_vector;
u_int old_id;
int i, vector;
/*
* Only allow CPUs to be assigned to the first message for an
* MSI group.
*/
if (msi->msi_first != msi)
return (EINVAL);
/* Store information to free existing irq. */
old_vector = msi->msi_vector;
old_id = msi->msi_cpu;
if (old_id == apic_id)
return (0);
/* Allocate IDT vectors on this cpu. */
if (msi->msi_count > 1) {
KASSERT(msi->msi_msix == 0, ("MSI-X message group"));
vector = apic_alloc_vectors(apic_id, msi->msi_irqs,
msi->msi_count, msi->msi_maxcount);
} else
vector = apic_alloc_vector(apic_id, msi->msi_irq);
if (vector == 0)
return (ENOSPC);
msi->msi_cpu = apic_id;
msi->msi_vector = vector;
if (msi->msi_intsrc.is_handlers > 0)
apic_enable_vector(msi->msi_cpu, msi->msi_vector);
if (bootverbose)
printf("msi: Assigning %s IRQ %d to local APIC %u vector %u\n",
msi->msi_msix ? "MSI-X" : "MSI", msi->msi_irq,
msi->msi_cpu, msi->msi_vector);
for (i = 1; i < msi->msi_count; i++) {
sib = (struct msi_intsrc *)intr_lookup_source(msi->msi_irqs[i]);
sib->msi_cpu = apic_id;
sib->msi_vector = vector + i;
if (sib->msi_intsrc.is_handlers > 0)
apic_enable_vector(sib->msi_cpu, sib->msi_vector);
if (bootverbose)
printf(
"msi: Assigning MSI IRQ %d to local APIC %u vector %u\n",
sib->msi_irq, sib->msi_cpu, sib->msi_vector);
}
BUS_REMAP_INTR(device_get_parent(msi->msi_dev), msi->msi_dev,
msi->msi_irq);
/*
* Free the old vector after the new one is established. This is done
* to prevent races where we could miss an interrupt.
*/
if (msi->msi_intsrc.is_handlers > 0)
apic_disable_vector(old_id, old_vector);
apic_free_vector(old_id, old_vector, msi->msi_irq);
for (i = 1; i < msi->msi_count; i++) {
sib = (struct msi_intsrc *)intr_lookup_source(msi->msi_irqs[i]);
if (sib->msi_intsrc.is_handlers > 0)
apic_disable_vector(old_id, old_vector + i);
apic_free_vector(old_id, old_vector + i, msi->msi_irqs[i]);
}
return (0);
}
int
msix_alloc(device_t dev, int *irq)
{
struct msi_intsrc *msi;
u_int cpu;
int i, vector;
#ifdef ACPI_DMAR
u_int cookie;
int error;
#endif
if (!msi_enabled) {
printf("%s:%d MSIX disabled - thus MSIX\n", __FILE__, __LINE__);
return (ENXIO);
}
again:
mtx_lock(&msi_lock);
/* Find a free IRQ. */
for (i = FIRST_MSI_INT; i < FIRST_MSI_INT + NUM_MSI_INTS; i++) {
msi = (struct msi_intsrc *)intr_lookup_source(i);
/* End of allocated sources, so break. */
if (msi == NULL)
break;
/* Stop at the first free source. */
if (msi->msi_dev == NULL)
break;
}
/* Do we need to create a new source? */
if (msi == NULL) {
/* If we would exceed the max, give up. */
if (i + 1 > FIRST_MSI_INT + NUM_MSI_INTS) {
mtx_unlock(&msi_lock);
printf("%s:%d MSIX allocation failure\n", __FILE__, __LINE__);
return (ENXIO);
}
mtx_unlock(&msi_lock);
/* Create a new source. */
msi_create_source();
goto again;
}
/* Allocate an IDT vector. */
cpu = intr_next_cpu();
vector = apic_alloc_vector(cpu, i);
if (vector == 0) {
mtx_unlock(&msi_lock);
return (ENOSPC);
}
msi->msi_dev = dev;
#ifdef ACPI_DMAR
mtx_unlock(&msi_lock);
error = iommu_alloc_msi_intr(dev, &cookie, 1);
mtx_lock(&msi_lock);
if (error == EOPNOTSUPP)
error = 0;
if (error != 0) {
msi->msi_dev = NULL;
apic_free_vector(cpu, vector, i);
return (error);
}
msi->msi_remap_cookie = cookie;
#endif
if (bootverbose)
printf("msi: routing MSI-X IRQ %d to local APIC %u vector %u\n",
msi->msi_irq, cpu, vector);
/* Setup source. */
msi->msi_cpu = cpu;
msi->msi_first = msi;
msi->msi_vector = vector;
msi->msi_msix = 1;
msi->msi_count = 1;
msi->msi_maxcount = 1;
msi->msi_irqs = NULL;
KASSERT(msi->msi_intsrc.is_handlers == 0, ("dead MSI-X has handlers"));
mtx_unlock(&msi_lock);
*irq = i;
return (0);
}
int
msi_map(int irq, uint64_t *addr, uint32_t *data)
{
struct msi_intsrc *msi;
int error;
#ifdef ACPI_DMAR
struct msi_intsrc *msi1;
int i, k;
#endif
mtx_lock(&msi_lock);
msi = (struct msi_intsrc *)intr_lookup_source(irq);
if (msi == NULL) {
mtx_unlock(&msi_lock);
return (ENOENT);
}
/* Make sure this message is allocated to a device. */
if (msi->msi_dev == NULL) {
mtx_unlock(&msi_lock);
return (ENXIO);
}
/*
* If this message isn't an MSI-X message, make sure it's part
* of a group, and switch to the first message in the
* group.
*/
if (!msi->msi_msix) {
if (msi->msi_first == NULL) {
mtx_unlock(&msi_lock);
return (ENXIO);
}
msi = msi->msi_first;
}
#ifdef ACPI_DMAR
if (!msi->msi_msix) {
for (k = msi->msi_count - 1, i = FIRST_MSI_INT; k > 0 &&
i < FIRST_MSI_INT + NUM_MSI_INTS; i++) {
if (i == msi->msi_irq)
continue;
msi1 = (struct msi_intsrc *)intr_lookup_source(i);
if (!msi1->msi_msix && msi1->msi_first == msi) {
mtx_unlock(&msi_lock);
iommu_map_msi_intr(msi1->msi_dev,
msi1->msi_cpu, msi1->msi_vector,
msi1->msi_remap_cookie, NULL, NULL);
k--;
mtx_lock(&msi_lock);
}
}
}
mtx_unlock(&msi_lock);
error = iommu_map_msi_intr(msi->msi_dev, msi->msi_cpu,
msi->msi_vector, msi->msi_remap_cookie, addr, data);
#else
mtx_unlock(&msi_lock);
error = EOPNOTSUPP;
#endif
if (error == EOPNOTSUPP) {
*addr = INTEL_ADDR(msi);
*data = INTEL_DATA(msi);
error = 0;
}
return (error);
}