int
msi_release(int *irqs, int count)
{
struct msi_intsrc *msi, *first;
int i;
mtx_lock(&msi_lock);
first = (struct msi_intsrc *)intr_lookup_source(irqs[0]);
if (first == NULL) {
mtx_unlock(&msi_lock);
return (ENOENT);
}
/* Make sure this isn't an MSI-X message. */
if (first->msi_msix) {
mtx_unlock(&msi_lock);
return (EINVAL);
}
/* Make sure this message is allocated to a group. */
if (first->msi_first == NULL) {
mtx_unlock(&msi_lock);
return (ENXIO);
}
/*
* Make sure this is the start of a group and that we are releasing
* the entire group.
*/
if (first->msi_first != first || first->msi_count != count) {
mtx_unlock(&msi_lock);
return (EINVAL);
}
KASSERT(first->msi_dev != NULL, ("unowned group"));
/* Clear all the extra messages in the group. */
for (i = 1; i < count; i++) {
msi = (struct msi_intsrc *)intr_lookup_source(irqs[i]);
KASSERT(msi->msi_first == first, ("message not in group"));
KASSERT(msi->msi_dev == first->msi_dev, ("owner mismatch"));
msi->msi_first = NULL;
msi->msi_dev = NULL;
apic_free_vector(msi->msi_cpu, msi->msi_vector, msi->msi_irq);
msi->msi_vector = 0;
}
/* Clear out the first message. */
first->msi_first = NULL;
first->msi_dev = NULL;
apic_free_vector(first->msi_cpu, first->msi_vector, first->msi_irq);
first->msi_vector = 0;
first->msi_count = 0;
first->msi_maxcount = 0;
free(first->msi_irqs, M_MSI);
first->msi_irqs = NULL;
mtx_unlock(&msi_lock);
return (0);
}
int
msix_release(int irq)
{
struct msi_intsrc *msi;
mtx_lock(&msi_lock);
msi = (struct msi_intsrc *)intr_lookup_source(irq);
if (msi == NULL) {
mtx_unlock(&msi_lock);
return (ENOENT);
}
/* Make sure this is an MSI-X message. */
if (!msi->msi_msix) {
mtx_unlock(&msi_lock);
return (EINVAL);
}
KASSERT(msi->msi_dev != NULL, ("unowned message"));
/* Clear out the message. */
msi->msi_first = NULL;
msi->msi_dev = NULL;
apic_free_vector(msi->msi_cpu, msi->msi_vector, msi->msi_irq);
msi->msi_vector = 0;
msi->msi_msix = 0;
msi->msi_count = 0;
msi->msi_maxcount = 0;
mtx_unlock(&msi_lock);
return (0);
}
int
msix_alloc(device_t dev, int *irq)
{
struct msi_intsrc *msi;
u_int cpu;
int i, vector;
#ifdef ACPI_DMAR
u_int cookie;
int error;
#endif
if (!msi_enabled) {
printf("%s:%d MSIX disabled - thus MSIX\n", __FILE__, __LINE__);
return (ENXIO);
}
again:
mtx_lock(&msi_lock);
/* Find a free IRQ. */
for (i = FIRST_MSI_INT; i < FIRST_MSI_INT + NUM_MSI_INTS; i++) {
msi = (struct msi_intsrc *)intr_lookup_source(i);
/* End of allocated sources, so break. */
if (msi == NULL)
break;
/* Stop at the first free source. */
if (msi->msi_dev == NULL)
break;
}
/* Do we need to create a new source? */
if (msi == NULL) {
/* If we would exceed the max, give up. */
if (i + 1 > FIRST_MSI_INT + NUM_MSI_INTS) {
mtx_unlock(&msi_lock);
printf("%s:%d MSIX allocation failure\n", __FILE__, __LINE__);
return (ENXIO);
}
mtx_unlock(&msi_lock);
/* Create a new source. */
msi_create_source();
goto again;
}
/* Allocate an IDT vector. */
cpu = intr_next_cpu();
vector = apic_alloc_vector(cpu, i);
if (vector == 0) {
mtx_unlock(&msi_lock);
return (ENOSPC);
}
msi->msi_dev = dev;
#ifdef ACPI_DMAR
mtx_unlock(&msi_lock);
error = iommu_alloc_msi_intr(dev, &cookie, 1);
mtx_lock(&msi_lock);
if (error == EOPNOTSUPP)
error = 0;
if (error != 0) {
msi->msi_dev = NULL;
apic_free_vector(cpu, vector, i);
return (error);
}
msi->msi_remap_cookie = cookie;
#endif
if (bootverbose)
printf("msi: routing MSI-X IRQ %d to local APIC %u vector %u\n",
msi->msi_irq, cpu, vector);
/* Setup source. */
msi->msi_cpu = cpu;
msi->msi_first = msi;
msi->msi_vector = vector;
msi->msi_msix = 1;
msi->msi_count = 1;
msi->msi_maxcount = 1;
msi->msi_irqs = NULL;
KASSERT(msi->msi_intsrc.is_handlers == 0, ("dead MSI-X has handlers"));
mtx_unlock(&msi_lock);
*irq = i;
return (0);
}
static int
msi_assign_cpu(struct intsrc *isrc, u_int apic_id)
{
struct msi_intsrc *sib, *msi = (struct msi_intsrc *)isrc;
int old_vector;
u_int old_id;
int i, vector;
/*
* Only allow CPUs to be assigned to the first message for an
* MSI group.
*/
if (msi->msi_first != msi)
return (EINVAL);
/* Store information to free existing irq. */
old_vector = msi->msi_vector;
old_id = msi->msi_cpu;
if (old_id == apic_id)
return (0);
/* Allocate IDT vectors on this cpu. */
if (msi->msi_count > 1) {
KASSERT(msi->msi_msix == 0, ("MSI-X message group"));
vector = apic_alloc_vectors(apic_id, msi->msi_irqs,
msi->msi_count, msi->msi_maxcount);
} else
vector = apic_alloc_vector(apic_id, msi->msi_irq);
if (vector == 0)
return (ENOSPC);
msi->msi_cpu = apic_id;
msi->msi_vector = vector;
if (msi->msi_intsrc.is_handlers > 0)
apic_enable_vector(msi->msi_cpu, msi->msi_vector);
if (bootverbose)
printf("msi: Assigning %s IRQ %d to local APIC %u vector %u\n",
msi->msi_msix ? "MSI-X" : "MSI", msi->msi_irq,
msi->msi_cpu, msi->msi_vector);
for (i = 1; i < msi->msi_count; i++) {
sib = (struct msi_intsrc *)intr_lookup_source(msi->msi_irqs[i]);
sib->msi_cpu = apic_id;
sib->msi_vector = vector + i;
if (sib->msi_intsrc.is_handlers > 0)
apic_enable_vector(sib->msi_cpu, sib->msi_vector);
if (bootverbose)
printf(
"msi: Assigning MSI IRQ %d to local APIC %u vector %u\n",
sib->msi_irq, sib->msi_cpu, sib->msi_vector);
}
BUS_REMAP_INTR(device_get_parent(msi->msi_dev), msi->msi_dev,
msi->msi_irq);
/*
* Free the old vector after the new one is established. This is done
* to prevent races where we could miss an interrupt.
*/
if (msi->msi_intsrc.is_handlers > 0)
apic_disable_vector(old_id, old_vector);
apic_free_vector(old_id, old_vector, msi->msi_irq);
for (i = 1; i < msi->msi_count; i++) {
sib = (struct msi_intsrc *)intr_lookup_source(msi->msi_irqs[i]);
if (sib->msi_intsrc.is_handlers > 0)
apic_disable_vector(old_id, old_vector + i);
apic_free_vector(old_id, old_vector + i, msi->msi_irqs[i]);
}
return (0);
}
/*
* Try to allocate 'count' interrupt sources with contiguous IDT values.
*/
int
msi_alloc(device_t dev, int count, int maxcount, int *irqs)
{
struct msi_intsrc *msi, *fsrc;
u_int cpu;
int cnt, i, *mirqs, vector;
#ifdef ACPI_DMAR
u_int cookies[count];
int error;
#endif
if (!msi_enabled)
return (ENXIO);
if (count > 1)
mirqs = malloc(count * sizeof(*mirqs), M_MSI, M_WAITOK);
else
mirqs = NULL;
again:
mtx_lock(&msi_lock);
/* Try to find 'count' free IRQs. */
cnt = 0;
for (i = FIRST_MSI_INT; i < FIRST_MSI_INT + NUM_MSI_INTS; i++) {
msi = (struct msi_intsrc *)intr_lookup_source(i);
/* End of allocated sources, so break. */
if (msi == NULL)
break;
/* If this is a free one, save its IRQ in the array. */
if (msi->msi_dev == NULL) {
irqs[cnt] = i;
cnt++;
if (cnt == count)
break;
}
}
/* Do we need to create some new sources? */
if (cnt < count) {
/* If we would exceed the max, give up. */
if (i + (count - cnt) > FIRST_MSI_INT + NUM_MSI_INTS) {
mtx_unlock(&msi_lock);
free(mirqs, M_MSI);
return (ENXIO);
}
mtx_unlock(&msi_lock);
/* We need count - cnt more sources. */
while (cnt < count) {
msi_create_source();
cnt++;
}
goto again;
}
/* Ok, we now have the IRQs allocated. */
KASSERT(cnt == count, ("count mismatch"));
/* Allocate 'count' IDT vectors. */
cpu = intr_next_cpu();
vector = apic_alloc_vectors(cpu, irqs, count, maxcount);
if (vector == 0) {
mtx_unlock(&msi_lock);
free(mirqs, M_MSI);
return (ENOSPC);
}
#ifdef ACPI_DMAR
mtx_unlock(&msi_lock);
error = iommu_alloc_msi_intr(dev, cookies, count);
mtx_lock(&msi_lock);
if (error == EOPNOTSUPP)
error = 0;
if (error != 0) {
for (i = 0; i < count; i++)
apic_free_vector(cpu, vector + i, irqs[i]);
free(mirqs, M_MSI);
return (error);
}
for (i = 0; i < count; i++) {
msi = (struct msi_intsrc *)intr_lookup_source(irqs[i]);
msi->msi_remap_cookie = cookies[i];
}
#endif
/* Assign IDT vectors and make these messages owned by 'dev'. */
fsrc = (struct msi_intsrc *)intr_lookup_source(irqs[0]);
for (i = 0; i < count; i++) {
msi = (struct msi_intsrc *)intr_lookup_source(irqs[i]);
msi->msi_cpu = cpu;
msi->msi_dev = dev;
msi->msi_vector = vector + i;
if (bootverbose)
printf(
"msi: routing MSI IRQ %d to local APIC %u vector %u\n",
msi->msi_irq, msi->msi_cpu, msi->msi_vector);
msi->msi_first = fsrc;
KASSERT(msi->msi_intsrc.is_handlers == 0,
("dead MSI has handlers"));
}
fsrc->msi_count = count;
fsrc->msi_maxcount = maxcount;
if (count > 1)
bcopy(irqs, mirqs, count * sizeof(*mirqs));
fsrc->msi_irqs = mirqs;
mtx_unlock(&msi_lock);
return (0);
}