static void __init intel_cleanup_irq_remapping(void)
{
struct dmar_drhd_unit *drhd;
struct intel_iommu *iommu;
for_each_iommu(iommu, drhd) {
if (ecap_ir_support(iommu->ecap)) {
iommu_disable_irq_remapping(iommu);
intel_teardown_irq_remapping(iommu);
}
}
if (x2apic_supported())
pr_warn("Failed to enable irq remapping. You are vulnerable to irq-injection attacks.\n");
}
static void disable_irq_remapping(void)
{
struct dmar_drhd_unit *drhd;
struct intel_iommu *iommu = NULL;
/*
* Disable Interrupt-remapping for all the DRHD's now.
*/
for_each_iommu(iommu, drhd) {
if (!ecap_ir_support(iommu->ecap))
continue;
iommu_disable_irq_remapping(iommu);
}
}
static int iommu_load_old_irte(struct intel_iommu *iommu)
{
struct irte *old_ir_table;
phys_addr_t irt_phys;
unsigned int i;
size_t size;
u64 irta;
if (!is_kdump_kernel()) {
pr_warn("IRQ remapping was enabled on %s but we are not in kdump mode\n",
iommu->name);
clear_ir_pre_enabled(iommu);
iommu_disable_irq_remapping(iommu);
return -EINVAL;
}
/* Check whether the old ir-table has the same size as ours */
irta = dmar_readq(iommu->reg + DMAR_IRTA_REG);
if ((irta & INTR_REMAP_TABLE_REG_SIZE_MASK)
!= INTR_REMAP_TABLE_REG_SIZE)
return -EINVAL;
irt_phys = irta & VTD_PAGE_MASK;
size = INTR_REMAP_TABLE_ENTRIES*sizeof(struct irte);
/* Map the old IR table */
old_ir_table = memremap(irt_phys, size, MEMREMAP_WB);
if (!old_ir_table)
return -ENOMEM;
/* Copy data over */
memcpy(iommu->ir_table->base, old_ir_table, size);
__iommu_flush_cache(iommu, iommu->ir_table->base, size);
/*
* Now check the table for used entries and mark those as
* allocated in the bitmap
*/
for (i = 0; i < INTR_REMAP_TABLE_ENTRIES; i++) {
if (iommu->ir_table->base[i].present)
bitmap_set(iommu->ir_table->bitmap, i, 1);
}
memunmap(old_ir_table);
return 0;
}
static int __init intel_enable_irq_remapping(void)
{
struct dmar_drhd_unit *drhd;
int setup = 0;
int eim = 0;
if (parse_ioapics_under_ir() != 1) {
printk(KERN_INFO "Not enable interrupt remapping\n");
return -1;
}
if (x2apic_supported()) {
eim = !dmar_x2apic_optout();
WARN(!eim, KERN_WARNING
"Your BIOS is broken and requested that x2apic be disabled\n"
"This will leave your machine vulnerable to irq-injection attacks\n"
"Use 'intremap=no_x2apic_optout' to override BIOS request\n");
}
for_each_drhd_unit(drhd) {
struct intel_iommu *iommu = drhd->iommu;
/*
* If the queued invalidation is already initialized,
* shouldn't disable it.
*/
if (iommu->qi)
continue;
/*
* Clear previous faults.
*/
dmar_fault(-1, iommu);
/*
* Disable intr remapping and queued invalidation, if already
* enabled prior to OS handover.
*/
iommu_disable_irq_remapping(iommu);
dmar_disable_qi(iommu);
}
/*
* check for the Interrupt-remapping support
*/
for_each_drhd_unit(drhd) {
struct intel_iommu *iommu = drhd->iommu;
if (!ecap_ir_support(iommu->ecap))
continue;
if (eim && !ecap_eim_support(iommu->ecap)) {
printk(KERN_INFO "DRHD %Lx: EIM not supported by DRHD, "
" ecap %Lx\n", drhd->reg_base_addr, iommu->ecap);
return -1;
}
}
/*
* Enable queued invalidation for all the DRHD's.
*/
for_each_drhd_unit(drhd) {
int ret;
struct intel_iommu *iommu = drhd->iommu;
ret = dmar_enable_qi(iommu);
if (ret) {
printk(KERN_ERR "DRHD %Lx: failed to enable queued, "
" invalidation, ecap %Lx, ret %d\n",
drhd->reg_base_addr, iommu->ecap, ret);
return -1;
}
}
/*
* Setup Interrupt-remapping for all the DRHD's now.
*/
for_each_drhd_unit(drhd) {
struct intel_iommu *iommu = drhd->iommu;
if (!ecap_ir_support(iommu->ecap))
continue;
if (intel_setup_irq_remapping(iommu, eim))
goto error;
setup = 1;
}
if (!setup)
goto error;
irq_remapping_enabled = 1;
pr_info("Enabled IRQ remapping in %s mode\n", eim ? "x2apic" : "xapic");
return eim ? IRQ_REMAP_X2APIC_MODE : IRQ_REMAP_XAPIC_MODE;
error:
/*
* handle error condition gracefully here!
*/
return -1;
}
static int __init intel_enable_irq_remapping(void)
{
struct dmar_drhd_unit *drhd;
struct intel_iommu *iommu;
bool x2apic_present;
int setup = 0;
int eim = 0;
x2apic_present = x2apic_supported();
if (parse_ioapics_under_ir() != 1) {
printk(KERN_INFO "Not enable interrupt remapping\n");
goto error;
}
if (x2apic_present) {
eim = !dmar_x2apic_optout();
if (!eim)
printk(KERN_WARNING
"Your BIOS is broken and requested that x2apic be disabled.\n"
"This will slightly decrease performance.\n"
"Use 'intremap=no_x2apic_optout' to override BIOS request.\n");
}
for_each_iommu(iommu, drhd) {
/*
* If the queued invalidation is already initialized,
* shouldn't disable it.
*/
if (iommu->qi)
continue;
/*
* Clear previous faults.
*/
dmar_fault(-1, iommu);
/*
* Disable intr remapping and queued invalidation, if already
* enabled prior to OS handover.
*/
iommu_disable_irq_remapping(iommu);
dmar_disable_qi(iommu);
}
/*
* check for the Interrupt-remapping support
*/
for_each_iommu(iommu, drhd) {
if (!ecap_ir_support(iommu->ecap))
continue;
if (eim && !ecap_eim_support(iommu->ecap)) {
printk(KERN_INFO "DRHD %Lx: EIM not supported by DRHD, "
" ecap %Lx\n", drhd->reg_base_addr, iommu->ecap);
goto error;
}
}
/*
* Enable queued invalidation for all the DRHD's.
*/
for_each_iommu(iommu, drhd) {
int ret = dmar_enable_qi(iommu);
if (ret) {
printk(KERN_ERR "DRHD %Lx: failed to enable queued, "
" invalidation, ecap %Lx, ret %d\n",
drhd->reg_base_addr, iommu->ecap, ret);
goto error;
}
}
static int intel_setup_irq_remapping(struct intel_iommu *iommu)
{
struct ir_table *ir_table;
struct fwnode_handle *fn;
unsigned long *bitmap;
struct page *pages;
if (iommu->ir_table)
return 0;
ir_table = kzalloc(sizeof(struct ir_table), GFP_KERNEL);
if (!ir_table)
return -ENOMEM;
pages = alloc_pages_node(iommu->node, GFP_KERNEL | __GFP_ZERO,
INTR_REMAP_PAGE_ORDER);
if (!pages) {
pr_err("IR%d: failed to allocate pages of order %d\n",
iommu->seq_id, INTR_REMAP_PAGE_ORDER);
goto out_free_table;
}
bitmap = kcalloc(BITS_TO_LONGS(INTR_REMAP_TABLE_ENTRIES),
sizeof(long), GFP_ATOMIC);
if (bitmap == NULL) {
pr_err("IR%d: failed to allocate bitmap\n", iommu->seq_id);
goto out_free_pages;
}
fn = irq_domain_alloc_named_id_fwnode("INTEL-IR", iommu->seq_id);
if (!fn)
goto out_free_bitmap;
iommu->ir_domain =
irq_domain_create_hierarchy(arch_get_ir_parent_domain(),
0, INTR_REMAP_TABLE_ENTRIES,
fn, &intel_ir_domain_ops,
iommu);
irq_domain_free_fwnode(fn);
if (!iommu->ir_domain) {
pr_err("IR%d: failed to allocate irqdomain\n", iommu->seq_id);
goto out_free_bitmap;
}
iommu->ir_msi_domain =
arch_create_remap_msi_irq_domain(iommu->ir_domain,
"INTEL-IR-MSI",
iommu->seq_id);
ir_table->base = page_address(pages);
ir_table->bitmap = bitmap;
iommu->ir_table = ir_table;
/*
* If the queued invalidation is already initialized,
* shouldn't disable it.
*/
if (!iommu->qi) {
/*
* Clear previous faults.
*/
dmar_fault(-1, iommu);
dmar_disable_qi(iommu);
if (dmar_enable_qi(iommu)) {
pr_err("Failed to enable queued invalidation\n");
goto out_free_bitmap;
}
}
init_ir_status(iommu);
if (ir_pre_enabled(iommu)) {
if (!is_kdump_kernel()) {
pr_warn("IRQ remapping was enabled on %s but we are not in kdump mode\n",
iommu->name);
clear_ir_pre_enabled(iommu);
iommu_disable_irq_remapping(iommu);
} else if (iommu_load_old_irte(iommu))
pr_err("Failed to copy IR table for %s from previous kernel\n",
iommu->name);
else
pr_info("Copied IR table for %s from previous kernel\n",
iommu->name);
}
iommu_set_irq_remapping(iommu, eim_mode);
return 0;
out_free_bitmap:
kfree(bitmap);
out_free_pages:
__free_pages(pages, INTR_REMAP_PAGE_ORDER);
out_free_table:
kfree(ir_table);
iommu->ir_table = NULL;
return -ENOMEM;
}
static int __init intel_enable_irq_remapping(void)
{
struct dmar_drhd_unit *drhd;
bool x2apic_present;
int setup = 0;
int eim = 0;
x2apic_present = x2apic_supported();
if (parse_ioapics_under_ir() != 1) {
printk(KERN_INFO "Not enable interrupt remapping\n");
goto error;
}
if (x2apic_present) {
eim = !dmar_x2apic_optout();
if (!eim)
printk(KERN_WARNING
"Your BIOS is broken and requested that x2apic be disabled.\n"
"This will slightly decrease performance.\n"
"Use 'intremap=no_x2apic_optout' to override BIOS request.\n");
}
for_each_drhd_unit(drhd) {
struct intel_iommu *iommu = drhd->iommu;
/*
* If the queued invalidation is already initialized,
* shouldn't disable it.
*/
if (iommu->qi)
continue;
/*
* Clear previous faults.
*/
dmar_fault(-1, iommu);
/*
* Disable intr remapping and queued invalidation, if already
* enabled prior to OS handover.
*/
iommu_disable_irq_remapping(iommu);
dmar_disable_qi(iommu);
}
/*
* check for the Interrupt-remapping support
*/
for_each_drhd_unit(drhd) {
struct intel_iommu *iommu = drhd->iommu;
if (!ecap_ir_support(iommu->ecap))
continue;
if (eim && !ecap_eim_support(iommu->ecap)) {
printk(KERN_INFO "DRHD %Lx: EIM not supported by DRHD, "
" ecap %Lx\n", drhd->reg_base_addr, iommu->ecap);
goto error;
}
}
/*
* Enable queued invalidation for all the DRHD's.
*/
for_each_drhd_unit(drhd) {
int ret;
struct intel_iommu *iommu = drhd->iommu;
ret = dmar_enable_qi(iommu);
if (ret) {
printk(KERN_ERR "DRHD %Lx: failed to enable queued, "
" invalidation, ecap %Lx, ret %d\n",
drhd->reg_base_addr, iommu->ecap, ret);
goto error;
}
}
/*
* Setup Interrupt-remapping for all the DRHD's now.
*/
for_each_drhd_unit(drhd) {
struct intel_iommu *iommu = drhd->iommu;
if (!ecap_ir_support(iommu->ecap))
continue;
if (intel_setup_irq_remapping(iommu, eim))
goto error;
setup = 1;
}
if (!setup)
goto error;
irq_remapping_enabled = 1;
/*
* VT-d has a different layout for IO-APIC entries when
* interrupt remapping is enabled. So it needs a special routine
* to print IO-APIC entries for debugging purposes too.
*/
x86_io_apic_ops.print_entries = intel_ir_io_apic_print_entries;
pr_info("Enabled IRQ remapping in %s mode\n", eim ? "x2apic" : "xapic");
return eim ? IRQ_REMAP_X2APIC_MODE : IRQ_REMAP_XAPIC_MODE;
error:
/*
* handle error condition gracefully here!
*/
if (x2apic_present)
pr_warn("Failed to enable irq remapping. You are vulnerable to irq-injection attacks.\n");
return -1;
}