static void __init intel_cleanup_irq_remapping(void)
{
struct dmar_drhd_unit *drhd;
struct intel_iommu *iommu;
for_each_iommu(iommu, drhd) {
if (ecap_ir_support(iommu->ecap)) {
iommu_disable_irq_remapping(iommu);
intel_teardown_irq_remapping(iommu);
}
}
if (x2apic_supported())
pr_warn("Failed to enable irq remapping. You are vulnerable to irq-injection attacks.\n");
}
int __init enable_IR(void)
{
#ifdef CONFIG_INTR_REMAP
if (!intr_remapping_supported()) {
pr_debug("intr-remapping not supported\n");
return 0;
}
if (!x2apic_preenabled && skip_ioapic_setup) {
pr_info("Skipped enabling intr-remap because of skipping "
"io-apic setup\n");
return 0;
}
if (enable_intr_remapping(x2apic_supported()))
return 0;
pr_info("Enabled Interrupt-remapping\n");
return 1;
#endif
return 0;
}
static int __init hyperv_prepare_irq_remapping(void)
{
struct fwnode_handle *fn;
int i;
if (!hypervisor_is_type(X86_HYPER_MS_HYPERV) ||
!x2apic_supported())
return -ENODEV;
fn = irq_domain_alloc_named_id_fwnode("HYPERV-IR", 0);
if (!fn)
return -ENOMEM;
ioapic_ir_domain =
irq_domain_create_hierarchy(arch_get_ir_parent_domain(),
0, IOAPIC_REMAPPING_ENTRY, fn,
&hyperv_ir_domain_ops, NULL);
irq_domain_free_fwnode(fn);
/*
* Hyper-V doesn't provide irq remapping function for
* IO-APIC and so IO-APIC only accepts 8-bit APIC ID.
* Cpu's APIC ID is read from ACPI MADT table and APIC IDs
* in the MADT table on Hyper-v are sorted monotonic increasingly.
* APIC ID reflects cpu topology. There maybe some APIC ID
* gaps when cpu number in a socket is not power of two. Prepare
* max cpu affinity for IOAPIC irqs. Scan cpu 0-255 and set cpu
* into ioapic_max_cpumask if its APIC ID is less than 256.
*/
for (i = min_t(unsigned int, num_possible_cpus() - 1, 255); i >= 0; i--)
if (cpu_physical_id(i) < 256)
cpumask_set_cpu(i, &ioapic_max_cpumask);
return 0;
}
static int __init intel_enable_irq_remapping(void)
{
struct dmar_drhd_unit *drhd;
int setup = 0;
int eim = 0;
if (parse_ioapics_under_ir() != 1) {
printk(KERN_INFO "Not enable interrupt remapping\n");
return -1;
}
if (x2apic_supported()) {
eim = !dmar_x2apic_optout();
WARN(!eim, KERN_WARNING
"Your BIOS is broken and requested that x2apic be disabled\n"
"This will leave your machine vulnerable to irq-injection attacks\n"
"Use 'intremap=no_x2apic_optout' to override BIOS request\n");
}
for_each_drhd_unit(drhd) {
struct intel_iommu *iommu = drhd->iommu;
/*
* If the queued invalidation is already initialized,
* shouldn't disable it.
*/
if (iommu->qi)
continue;
/*
* Clear previous faults.
*/
dmar_fault(-1, iommu);
/*
* Disable intr remapping and queued invalidation, if already
* enabled prior to OS handover.
*/
iommu_disable_irq_remapping(iommu);
dmar_disable_qi(iommu);
}
/*
* check for the Interrupt-remapping support
*/
for_each_drhd_unit(drhd) {
struct intel_iommu *iommu = drhd->iommu;
if (!ecap_ir_support(iommu->ecap))
continue;
if (eim && !ecap_eim_support(iommu->ecap)) {
printk(KERN_INFO "DRHD %Lx: EIM not supported by DRHD, "
" ecap %Lx\n", drhd->reg_base_addr, iommu->ecap);
return -1;
}
}
/*
* Enable queued invalidation for all the DRHD's.
*/
for_each_drhd_unit(drhd) {
int ret;
struct intel_iommu *iommu = drhd->iommu;
ret = dmar_enable_qi(iommu);
if (ret) {
printk(KERN_ERR "DRHD %Lx: failed to enable queued, "
" invalidation, ecap %Lx, ret %d\n",
drhd->reg_base_addr, iommu->ecap, ret);
return -1;
}
}
/*
* Setup Interrupt-remapping for all the DRHD's now.
*/
for_each_drhd_unit(drhd) {
struct intel_iommu *iommu = drhd->iommu;
if (!ecap_ir_support(iommu->ecap))
continue;
if (intel_setup_irq_remapping(iommu, eim))
goto error;
setup = 1;
}
if (!setup)
goto error;
irq_remapping_enabled = 1;
pr_info("Enabled IRQ remapping in %s mode\n", eim ? "x2apic" : "xapic");
return eim ? IRQ_REMAP_X2APIC_MODE : IRQ_REMAP_XAPIC_MODE;
error:
/*
* handle error condition gracefully here!
*/
return -1;
}
static int __init intel_enable_irq_remapping(void)
{
struct dmar_drhd_unit *drhd;
struct intel_iommu *iommu;
bool x2apic_present;
int setup = 0;
int eim = 0;
x2apic_present = x2apic_supported();
if (parse_ioapics_under_ir() != 1) {
printk(KERN_INFO "Not enable interrupt remapping\n");
goto error;
}
if (x2apic_present) {
eim = !dmar_x2apic_optout();
if (!eim)
printk(KERN_WARNING
"Your BIOS is broken and requested that x2apic be disabled.\n"
"This will slightly decrease performance.\n"
"Use 'intremap=no_x2apic_optout' to override BIOS request.\n");
}
for_each_iommu(iommu, drhd) {
/*
* If the queued invalidation is already initialized,
* shouldn't disable it.
*/
if (iommu->qi)
continue;
/*
* Clear previous faults.
*/
dmar_fault(-1, iommu);
/*
* Disable intr remapping and queued invalidation, if already
* enabled prior to OS handover.
*/
iommu_disable_irq_remapping(iommu);
dmar_disable_qi(iommu);
}
/*
* check for the Interrupt-remapping support
*/
for_each_iommu(iommu, drhd) {
if (!ecap_ir_support(iommu->ecap))
continue;
if (eim && !ecap_eim_support(iommu->ecap)) {
printk(KERN_INFO "DRHD %Lx: EIM not supported by DRHD, "
" ecap %Lx\n", drhd->reg_base_addr, iommu->ecap);
goto error;
}
}
/*
* Enable queued invalidation for all the DRHD's.
*/
for_each_iommu(iommu, drhd) {
int ret = dmar_enable_qi(iommu);
if (ret) {
printk(KERN_ERR "DRHD %Lx: failed to enable queued, "
" invalidation, ecap %Lx, ret %d\n",
drhd->reg_base_addr, iommu->ecap, ret);
goto error;
}
}
static int __init intel_prepare_irq_remapping(void)
{
struct dmar_drhd_unit *drhd;
struct intel_iommu *iommu;
int eim = 0;
if (irq_remap_broken) {
pr_warn("This system BIOS has enabled interrupt remapping\n"
"on a chipset that contains an erratum making that\n"
"feature unstable. To maintain system stability\n"
"interrupt remapping is being disabled. Please\n"
"contact your BIOS vendor for an update\n");
add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK);
return -ENODEV;
}
if (dmar_table_init() < 0)
return -ENODEV;
if (!dmar_ir_support())
return -ENODEV;
if (parse_ioapics_under_ir()) {
pr_info("Not enabling interrupt remapping\n");
goto error;
}
/* First make sure all IOMMUs support IRQ remapping */
for_each_iommu(iommu, drhd)
if (!ecap_ir_support(iommu->ecap))
goto error;
/* Detect remapping mode: lapic or x2apic */
if (x2apic_supported()) {
eim = !dmar_x2apic_optout();
if (!eim) {
pr_info("x2apic is disabled because BIOS sets x2apic opt out bit.");
pr_info("Use 'intremap=no_x2apic_optout' to override the BIOS setting.\n");
}
}
for_each_iommu(iommu, drhd) {
if (eim && !ecap_eim_support(iommu->ecap)) {
pr_info("%s does not support EIM\n", iommu->name);
eim = 0;
}
}
eim_mode = eim;
if (eim)
pr_info("Queued invalidation will be enabled to support x2apic and Intr-remapping.\n");
/* Do the initializations early */
for_each_iommu(iommu, drhd) {
if (intel_setup_irq_remapping(iommu)) {
pr_err("Failed to setup irq remapping for %s\n",
iommu->name);
goto error;
}
}
return 0;
error:
intel_cleanup_irq_remapping();
return -ENODEV;
}
static int __init intel_enable_irq_remapping(void)
{
struct dmar_drhd_unit *drhd;
bool x2apic_present;
int setup = 0;
int eim = 0;
x2apic_present = x2apic_supported();
if (parse_ioapics_under_ir() != 1) {
printk(KERN_INFO "Not enable interrupt remapping\n");
goto error;
}
if (x2apic_present) {
eim = !dmar_x2apic_optout();
if (!eim)
printk(KERN_WARNING
"Your BIOS is broken and requested that x2apic be disabled.\n"
"This will slightly decrease performance.\n"
"Use 'intremap=no_x2apic_optout' to override BIOS request.\n");
}
for_each_drhd_unit(drhd) {
struct intel_iommu *iommu = drhd->iommu;
/*
* If the queued invalidation is already initialized,
* shouldn't disable it.
*/
if (iommu->qi)
continue;
/*
* Clear previous faults.
*/
dmar_fault(-1, iommu);
/*
* Disable intr remapping and queued invalidation, if already
* enabled prior to OS handover.
*/
iommu_disable_irq_remapping(iommu);
dmar_disable_qi(iommu);
}
/*
* check for the Interrupt-remapping support
*/
for_each_drhd_unit(drhd) {
struct intel_iommu *iommu = drhd->iommu;
if (!ecap_ir_support(iommu->ecap))
continue;
if (eim && !ecap_eim_support(iommu->ecap)) {
printk(KERN_INFO "DRHD %Lx: EIM not supported by DRHD, "
" ecap %Lx\n", drhd->reg_base_addr, iommu->ecap);
goto error;
}
}
/*
* Enable queued invalidation for all the DRHD's.
*/
for_each_drhd_unit(drhd) {
int ret;
struct intel_iommu *iommu = drhd->iommu;
ret = dmar_enable_qi(iommu);
if (ret) {
printk(KERN_ERR "DRHD %Lx: failed to enable queued, "
" invalidation, ecap %Lx, ret %d\n",
drhd->reg_base_addr, iommu->ecap, ret);
goto error;
}
}
/*
* Setup Interrupt-remapping for all the DRHD's now.
*/
for_each_drhd_unit(drhd) {
struct intel_iommu *iommu = drhd->iommu;
if (!ecap_ir_support(iommu->ecap))
continue;
if (intel_setup_irq_remapping(iommu, eim))
goto error;
setup = 1;
}
if (!setup)
goto error;
irq_remapping_enabled = 1;
/*
* VT-d has a different layout for IO-APIC entries when
* interrupt remapping is enabled. So it needs a special routine
* to print IO-APIC entries for debugging purposes too.
*/
x86_io_apic_ops.print_entries = intel_ir_io_apic_print_entries;
pr_info("Enabled IRQ remapping in %s mode\n", eim ? "x2apic" : "xapic");
return eim ? IRQ_REMAP_X2APIC_MODE : IRQ_REMAP_XAPIC_MODE;
error:
/*
* handle error condition gracefully here!
*/
if (x2apic_present)
pr_warn("Failed to enable irq remapping. You are vulnerable to irq-injection attacks.\n");
return -1;
}
void __init enable_IR_x2apic(void)
{
unsigned long flags;
struct IO_APIC_route_entry **ioapic_entries = NULL;
int ret, x2apic_enabled = 0;
int dmar_table_init_ret = 0;
#ifdef CONFIG_INTR_REMAP
dmar_table_init_ret = dmar_table_init();
if (dmar_table_init_ret)
pr_debug("dmar_table_init() failed with %d:\n",
dmar_table_init_ret);
#endif
ioapic_entries = alloc_ioapic_entries();
if (!ioapic_entries) {
pr_err("Allocate ioapic_entries failed\n");
goto out;
}
ret = save_IO_APIC_setup(ioapic_entries);
if (ret) {
pr_info("Saving IO-APIC state failed: %d\n", ret);
goto out;
}
local_irq_save(flags);
mask_8259A();
mask_IO_APIC_setup(ioapic_entries);
if (dmar_table_init_ret)
ret = 0;
else
ret = enable_IR();
if (!ret) {
if (max_physical_apicid > 255 || !kvm_para_available())
goto nox2apic;
x2apic_force_phys();
}
x2apic_enabled = 1;
if (x2apic_supported() && !x2apic_mode) {
x2apic_mode = 1;
enable_x2apic();
pr_info("Enabled x2apic\n");
}
nox2apic:
if (!ret)
restore_IO_APIC_setup(ioapic_entries);
unmask_8259A();
local_irq_restore(flags);
out:
if (ioapic_entries)
free_ioapic_entries(ioapic_entries);
if (x2apic_enabled)
return;
if (x2apic_preenabled)
panic("x2apic: enabled by BIOS but kernel init failed.");
else if (cpu_has_x2apic)
pr_info("Not enabling x2apic, Intr-remapping init failed.\n");
}
int __init enable_intr_remapping(void)
{
struct dmar_drhd_unit *drhd;
int setup = 0;
int eim = 0;
if (parse_ioapics_under_ir() != 1) {
printk(KERN_INFO "Not enable interrupt remapping\n");
return -1;
}
if (x2apic_supported()) {
eim = !dmar_x2apic_optout();
WARN(!eim, KERN_WARNING
"Your BIOS is broken and requested that x2apic be disabled\n"
"This will leave your machine vulnerable to irq-injection attacks\n"
"Use 'intremap=no_x2apic_optout' to override BIOS request\n");
}
for_each_drhd_unit(drhd) {
struct intel_iommu *iommu = drhd->iommu;
if (iommu->qi)
continue;
dmar_fault(-1, iommu);
iommu_disable_intr_remapping(iommu);
dmar_disable_qi(iommu);
}
for_each_drhd_unit(drhd) {
struct intel_iommu *iommu = drhd->iommu;
if (!ecap_ir_support(iommu->ecap))
continue;
if (eim && !ecap_eim_support(iommu->ecap)) {
printk(KERN_INFO "DRHD %Lx: EIM not supported by DRHD, "
" ecap %Lx\n", drhd->reg_base_addr, iommu->ecap);
return -1;
}
}
for_each_drhd_unit(drhd) {
int ret;
struct intel_iommu *iommu = drhd->iommu;
ret = dmar_enable_qi(iommu);
if (ret) {
printk(KERN_ERR "DRHD %Lx: failed to enable queued, "
" invalidation, ecap %Lx, ret %d\n",
drhd->reg_base_addr, iommu->ecap, ret);
return -1;
}
}
for_each_drhd_unit(drhd) {
struct intel_iommu *iommu = drhd->iommu;
if (!ecap_ir_support(iommu->ecap))
continue;
if (setup_intr_remapping(iommu, eim))
goto error;
setup = 1;
}
if (!setup)
goto error;
intr_remapping_enabled = 1;
pr_info("Enabled IRQ remapping in %s mode\n", eim ? "x2apic" : "xapic");
return eim ? IRQ_REMAP_X2APIC_MODE : IRQ_REMAP_XAPIC_MODE;
error:
return -1;
}
