static void iommu_disable_intr_remapping(struct intel_iommu *iommu)
{
unsigned long flags;
u32 sts;
if (!ecap_ir_support(iommu->ecap))
return;
/*
* global invalidation of interrupt entry cache before disabling
* interrupt-remapping.
*/
qi_global_iec(iommu);
spin_lock_irqsave(&iommu->register_lock, flags);
sts = dmar_readq(iommu->reg + DMAR_GSTS_REG);
if (!(sts & DMA_GSTS_IRES))
goto end;
iommu->gcmd &= ~DMA_GCMD_IRE;
writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
readl, !(sts & DMA_GSTS_IRES), sts);
end:
spin_unlock_irqrestore(&iommu->register_lock, flags);
}
int reenable_intr_remapping(int eim)
{
struct dmar_drhd_unit *drhd;
int setup = 0;
struct intel_iommu *iommu = NULL;
for_each_iommu(iommu, drhd)
if (iommu->qi)
dmar_reenable_qi(iommu);
/*
* Setup Interrupt-remapping for all the DRHD's now.
*/
for_each_iommu(iommu, drhd) {
if (!ecap_ir_support(iommu->ecap))
continue;
/* Set up interrupt remapping for iommu.*/
iommu_set_intr_remapping(iommu, eim);
setup = 1;
}
if (!setup)
goto error;
return 0;
error:
/*
* handle error condition gracefully here!
*/
return -1;
}
/*
* Finds the assocaition between IOAPIC's and its Interrupt-remapping
* hardware unit.
*/
int __init parse_ioapics_under_ir(void)
{
struct dmar_drhd_unit *drhd;
int ir_supported = 0;
int ioapic_idx;
for_each_drhd_unit(drhd) {
struct intel_iommu *iommu = drhd->iommu;
if (ecap_ir_support(iommu->ecap)) {
if (ir_parse_ioapic_hpet_scope(drhd->hdr, iommu))
return -1;
ir_supported = 1;
}
}
if (!ir_supported)
return 0;
for (ioapic_idx = 0; ioapic_idx < nr_ioapics; ioapic_idx++) {
int ioapic_id = mpc_ioapic_id(ioapic_idx);
if (!map_ioapic_to_ir(ioapic_id)) {
pr_err(FW_BUG "ioapic %d has no mapping iommu, "
"interrupt remapping will be disabled\n",
ioapic_id);
return -1;
}
}
return 1;
}
static int __init intel_irq_remapping_supported(void)
{
struct dmar_drhd_unit *drhd;
if (disable_irq_remap)
return 0;
if (irq_remap_broken) {
printk(KERN_WARNING
"This system BIOS has enabled interrupt remapping\n"
"on a chipset that contains an erratum making that\n"
"feature unstable. To maintain system stability\n"
"interrupt remapping is being disabled. Please\n"
"contact your BIOS vendor for an update\n");
add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK);
disable_irq_remap = 1;
return 0;
}
if (!dmar_ir_support())
return 0;
for_each_drhd_unit(drhd) {
struct intel_iommu *iommu = drhd->iommu;
if (!ecap_ir_support(iommu->ecap))
return 0;
}
return 1;
}
int reenable_intr_remapping(int eim)
{
struct dmar_drhd_unit *drhd;
int setup = 0;
struct intel_iommu *iommu = NULL;
for_each_iommu(iommu, drhd)
if (iommu->qi)
dmar_reenable_qi(iommu);
for_each_iommu(iommu, drhd) {
if (!ecap_ir_support(iommu->ecap))
continue;
iommu_set_intr_remapping(iommu, eim);
setup = 1;
}
if (!setup)
goto error;
return 0;
error:
return -1;
}
/*
* Setup Interrupt-remapping for all the DRHD's now.
*/
for_each_iommu(iommu, drhd) {
if (!ecap_ir_support(iommu->ecap))
continue;
if (intel_setup_irq_remapping(iommu, eim))
goto error;
setup = 1;
}
void disable_intr_remapping(void)
{
struct dmar_drhd_unit *drhd;
struct intel_iommu *iommu = NULL;
for_each_iommu(iommu, drhd) {
if (!ecap_ir_support(iommu->ecap))
continue;
iommu_disable_intr_remapping(iommu);
}
}
static void __init intel_cleanup_irq_remapping(void)
{
struct dmar_drhd_unit *drhd;
struct intel_iommu *iommu;
for_each_iommu(iommu, drhd) {
if (ecap_ir_support(iommu->ecap)) {
iommu_disable_irq_remapping(iommu);
intel_teardown_irq_remapping(iommu);
}
}
if (x2apic_supported())
pr_warn("Failed to enable irq remapping. You are vulnerable to irq-injection attacks.\n");
}
void disable_intr_remapping(void)
{
struct dmar_drhd_unit *drhd;
struct intel_iommu *iommu = NULL;
/*
* Disable Interrupt-remapping for all the DRHD's now.
*/
for_each_iommu(iommu, drhd) {
if (!ecap_ir_support(iommu->ecap))
continue;
iommu_disable_intr_remapping(iommu);
}
}
static int __init intel_prepare_irq_remapping(void)
{
struct dmar_drhd_unit *drhd;
struct intel_iommu *iommu;
if (irq_remap_broken) {
printk(KERN_WARNING
"This system BIOS has enabled interrupt remapping\n"
"on a chipset that contains an erratum making that\n"
"feature unstable. To maintain system stability\n"
"interrupt remapping is being disabled. Please\n"
"contact your BIOS vendor for an update\n");
add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK);
return -ENODEV;
}
if (dmar_table_init() < 0)
return -ENODEV;
if (!dmar_ir_support())
return -ENODEV;
if (parse_ioapics_under_ir() != 1) {
printk(KERN_INFO "Not enabling interrupt remapping\n");
goto error;
}
/* First make sure all IOMMUs support IRQ remapping */
for_each_iommu(iommu, drhd)
if (!ecap_ir_support(iommu->ecap))
goto error;
/* Do the allocations early */
for_each_iommu(iommu, drhd)
if (intel_setup_irq_remapping(iommu))
goto error;
return 0;
error:
intel_cleanup_irq_remapping();
return -ENODEV;
}
/*
* Finds the assocaition between IOAPIC's and its Interrupt-remapping
* hardware unit.
*/
static int __init parse_ioapics_under_ir(void)
{
struct dmar_drhd_unit *drhd;
struct intel_iommu *iommu;
bool ir_supported = false;
int ioapic_idx;
for_each_iommu(iommu, drhd) {
int ret;
if (!ecap_ir_support(iommu->ecap))
continue;
ret = ir_parse_ioapic_hpet_scope(drhd->hdr, iommu);
if (ret)
return ret;
ir_supported = true;
}
int __init intr_remapping_supported(void)
{
struct dmar_drhd_unit *drhd;
if (disable_intremap)
return 0;
if (!dmar_ir_support())
return 0;
for_each_drhd_unit(drhd) {
struct intel_iommu *iommu = drhd->iommu;
if (!ecap_ir_support(iommu->ecap))
return 0;
}
return 1;
}
/*
* Disable Interrupt Remapping.
*/
static void disable_intr_remapping(struct intel_iommu *iommu)
{
unsigned long flags;
u32 sts;
if (!ecap_ir_support(iommu->ecap))
return;
spin_lock_irqsave(&iommu->register_lock, flags);
sts = dmar_readq(iommu->reg + DMAR_GSTS_REG);
if (!(sts & DMA_GSTS_IRES))
goto end;
iommu->gcmd &= ~DMA_GCMD_IRE;
writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
readl, !(sts & DMA_GSTS_IRES), sts);
end:
spin_unlock_irqrestore(&iommu->register_lock, flags);
}
/*
* Finds the assocaition between IOAPIC's and its Interrupt-remapping
* hardware unit.
*/
int __init parse_ioapics_under_ir(void)
{
struct dmar_drhd_unit *drhd;
int ir_supported = 0;
for_each_drhd_unit(drhd) {
struct intel_iommu *iommu = drhd->iommu;
if (ecap_ir_support(iommu->ecap)) {
if (ir_parse_ioapic_hpet_scope(drhd->hdr, iommu))
return -1;
ir_supported = 1;
}
}
if (ir_supported && ir_ioapic_num != nr_ioapics) {
printk(KERN_WARNING
"Not all IO-APIC's listed under remapping hardware\n");
return -1;
}
return ir_supported;
}
/*
* Finds the assocaition between IOAPIC's and its Interrupt-remapping
* hardware unit.
*/
int __init parse_ioapics_under_ir(void)
{
struct dmar_drhd_unit *drhd;
int ir_supported = 0;
for_each_drhd_unit(drhd) {
struct intel_iommu *iommu = drhd->iommu;
if (ecap_ir_support(iommu->ecap)) {
if (ir_parse_ioapic_hpet_scope(drhd->hdr, iommu))
return -1;
ir_supported = 1;
}
}
if (ir_supported && ir_ioapic_num != nr_ioapics) {
// printk(KERN_WARNING
;
return -1;
}
return ir_supported;
}
int __init enable_intr_remapping(int eim)
{
struct dmar_drhd_unit *drhd;
int setup = 0;
if (parse_ioapics_under_ir() != 1) {
printk(KERN_INFO "Not enable interrupt remapping\n");
return -1;
}
for_each_drhd_unit(drhd) {
struct intel_iommu *iommu = drhd->iommu;
/*
* If the queued invalidation is already initialized,
* shouldn't disable it.
*/
if (iommu->qi)
continue;
/*
* Clear previous faults.
*/
dmar_fault(-1, iommu);
/*
* Disable intr remapping and queued invalidation, if already
* enabled prior to OS handover.
*/
iommu_disable_intr_remapping(iommu);
dmar_disable_qi(iommu);
}
/*
* check for the Interrupt-remapping support
*/
for_each_drhd_unit(drhd) {
struct intel_iommu *iommu = drhd->iommu;
if (!ecap_ir_support(iommu->ecap))
continue;
if (eim && !ecap_eim_support(iommu->ecap)) {
printk(KERN_INFO "DRHD %Lx: EIM not supported by DRHD, "
" ecap %Lx\n", drhd->reg_base_addr, iommu->ecap);
return -1;
}
}
/*
* Enable queued invalidation for all the DRHD's.
*/
for_each_drhd_unit(drhd) {
int ret;
struct intel_iommu *iommu = drhd->iommu;
ret = dmar_enable_qi(iommu);
if (ret) {
printk(KERN_ERR "DRHD %Lx: failed to enable queued, "
" invalidation, ecap %Lx, ret %d\n",
drhd->reg_base_addr, iommu->ecap, ret);
return -1;
}
}
/*
* Setup Interrupt-remapping for all the DRHD's now.
*/
for_each_drhd_unit(drhd) {
struct intel_iommu *iommu = drhd->iommu;
if (!ecap_ir_support(iommu->ecap))
continue;
if (setup_intr_remapping(iommu, eim))
goto error;
setup = 1;
}
if (!setup)
goto error;
intr_remapping_enabled = 1;
return 0;
error:
/*
* handle error condition gracefully here!
*/
return -1;
}
static int __init intel_enable_irq_remapping(void)
{
struct dmar_drhd_unit *drhd;
int setup = 0;
int eim = 0;
if (parse_ioapics_under_ir() != 1) {
printk(KERN_INFO "Not enable interrupt remapping\n");
return -1;
}
if (x2apic_supported()) {
eim = !dmar_x2apic_optout();
WARN(!eim, KERN_WARNING
"Your BIOS is broken and requested that x2apic be disabled\n"
"This will leave your machine vulnerable to irq-injection attacks\n"
"Use 'intremap=no_x2apic_optout' to override BIOS request\n");
}
for_each_drhd_unit(drhd) {
struct intel_iommu *iommu = drhd->iommu;
/*
* If the queued invalidation is already initialized,
* shouldn't disable it.
*/
if (iommu->qi)
continue;
/*
* Clear previous faults.
*/
dmar_fault(-1, iommu);
/*
* Disable intr remapping and queued invalidation, if already
* enabled prior to OS handover.
*/
iommu_disable_irq_remapping(iommu);
dmar_disable_qi(iommu);
}
/*
* check for the Interrupt-remapping support
*/
for_each_drhd_unit(drhd) {
struct intel_iommu *iommu = drhd->iommu;
if (!ecap_ir_support(iommu->ecap))
continue;
if (eim && !ecap_eim_support(iommu->ecap)) {
printk(KERN_INFO "DRHD %Lx: EIM not supported by DRHD, "
" ecap %Lx\n", drhd->reg_base_addr, iommu->ecap);
return -1;
}
}
/*
* Enable queued invalidation for all the DRHD's.
*/
for_each_drhd_unit(drhd) {
int ret;
struct intel_iommu *iommu = drhd->iommu;
ret = dmar_enable_qi(iommu);
if (ret) {
printk(KERN_ERR "DRHD %Lx: failed to enable queued, "
" invalidation, ecap %Lx, ret %d\n",
drhd->reg_base_addr, iommu->ecap, ret);
return -1;
}
}
/*
* Setup Interrupt-remapping for all the DRHD's now.
*/
for_each_drhd_unit(drhd) {
struct intel_iommu *iommu = drhd->iommu;
if (!ecap_ir_support(iommu->ecap))
continue;
if (intel_setup_irq_remapping(iommu, eim))
goto error;
setup = 1;
}
if (!setup)
goto error;
irq_remapping_enabled = 1;
pr_info("Enabled IRQ remapping in %s mode\n", eim ? "x2apic" : "xapic");
return eim ? IRQ_REMAP_X2APIC_MODE : IRQ_REMAP_XAPIC_MODE;
error:
/*
* handle error condition gracefully here!
*/
return -1;
}
int __init enable_intr_remapping(int eim)
{
struct dmar_drhd_unit *drhd;
int setup = 0;
/*
* check for the Interrupt-remapping support
*/
for_each_drhd_unit(drhd) {
struct intel_iommu *iommu = drhd->iommu;
if (!ecap_ir_support(iommu->ecap))
continue;
if (eim && !ecap_eim_support(iommu->ecap)) {
printk(KERN_INFO "DRHD %Lx: EIM not supported by DRHD, "
" ecap %Lx\n", drhd->reg_base_addr, iommu->ecap);
return -1;
}
}
/*
* Enable queued invalidation for all the DRHD's.
*/
for_each_drhd_unit(drhd) {
int ret;
struct intel_iommu *iommu = drhd->iommu;
ret = dmar_enable_qi(iommu);
if (ret) {
printk(KERN_ERR "DRHD %Lx: failed to enable queued, "
" invalidation, ecap %Lx, ret %d\n",
drhd->reg_base_addr, iommu->ecap, ret);
return -1;
}
}
/*
* Setup Interrupt-remapping for all the DRHD's now.
*/
for_each_drhd_unit(drhd) {
struct intel_iommu *iommu = drhd->iommu;
if (!ecap_ir_support(iommu->ecap))
continue;
if (setup_intr_remapping(iommu, eim))
goto error;
setup = 1;
}
if (!setup)
goto error;
intr_remapping_enabled = 1;
return 0;
error:
/*
* handle error condition gracefully here!
*/
return -1;
}
static int __init intel_enable_irq_remapping(void)
{
struct dmar_drhd_unit *drhd;
struct intel_iommu *iommu;
bool x2apic_present;
int setup = 0;
int eim = 0;
x2apic_present = x2apic_supported();
if (parse_ioapics_under_ir() != 1) {
printk(KERN_INFO "Not enable interrupt remapping\n");
goto error;
}
if (x2apic_present) {
eim = !dmar_x2apic_optout();
if (!eim)
printk(KERN_WARNING
"Your BIOS is broken and requested that x2apic be disabled.\n"
"This will slightly decrease performance.\n"
"Use 'intremap=no_x2apic_optout' to override BIOS request.\n");
}
for_each_iommu(iommu, drhd) {
/*
* If the queued invalidation is already initialized,
* shouldn't disable it.
*/
if (iommu->qi)
continue;
/*
* Clear previous faults.
*/
dmar_fault(-1, iommu);
/*
* Disable intr remapping and queued invalidation, if already
* enabled prior to OS handover.
*/
iommu_disable_irq_remapping(iommu);
dmar_disable_qi(iommu);
}
/*
* check for the Interrupt-remapping support
*/
for_each_iommu(iommu, drhd) {
if (!ecap_ir_support(iommu->ecap))
continue;
if (eim && !ecap_eim_support(iommu->ecap)) {
printk(KERN_INFO "DRHD %Lx: EIM not supported by DRHD, "
" ecap %Lx\n", drhd->reg_base_addr, iommu->ecap);
goto error;
}
}
/*
* Enable queued invalidation for all the DRHD's.
*/
for_each_iommu(iommu, drhd) {
int ret = dmar_enable_qi(iommu);
if (ret) {
printk(KERN_ERR "DRHD %Lx: failed to enable queued, "
" invalidation, ecap %Lx, ret %d\n",
drhd->reg_base_addr, iommu->ecap, ret);
goto error;
}
}
static int __init intel_prepare_irq_remapping(void)
{
struct dmar_drhd_unit *drhd;
struct intel_iommu *iommu;
int eim = 0;
if (irq_remap_broken) {
pr_warn("This system BIOS has enabled interrupt remapping\n"
"on a chipset that contains an erratum making that\n"
"feature unstable. To maintain system stability\n"
"interrupt remapping is being disabled. Please\n"
"contact your BIOS vendor for an update\n");
add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK);
return -ENODEV;
}
if (dmar_table_init() < 0)
return -ENODEV;
if (!dmar_ir_support())
return -ENODEV;
if (parse_ioapics_under_ir()) {
pr_info("Not enabling interrupt remapping\n");
goto error;
}
/* First make sure all IOMMUs support IRQ remapping */
for_each_iommu(iommu, drhd)
if (!ecap_ir_support(iommu->ecap))
goto error;
/* Detect remapping mode: lapic or x2apic */
if (x2apic_supported()) {
eim = !dmar_x2apic_optout();
if (!eim) {
pr_info("x2apic is disabled because BIOS sets x2apic opt out bit.");
pr_info("Use 'intremap=no_x2apic_optout' to override the BIOS setting.\n");
}
}
for_each_iommu(iommu, drhd) {
if (eim && !ecap_eim_support(iommu->ecap)) {
pr_info("%s does not support EIM\n", iommu->name);
eim = 0;
}
}
eim_mode = eim;
if (eim)
pr_info("Queued invalidation will be enabled to support x2apic and Intr-remapping.\n");
/* Do the initializations early */
for_each_iommu(iommu, drhd) {
if (intel_setup_irq_remapping(iommu)) {
pr_err("Failed to setup irq remapping for %s\n",
iommu->name);
goto error;
}
}
return 0;
error:
intel_cleanup_irq_remapping();
return -ENODEV;
}
static int __init intel_enable_irq_remapping(void)
{
struct dmar_drhd_unit *drhd;
bool x2apic_present;
int setup = 0;
int eim = 0;
x2apic_present = x2apic_supported();
if (parse_ioapics_under_ir() != 1) {
printk(KERN_INFO "Not enable interrupt remapping\n");
goto error;
}
if (x2apic_present) {
eim = !dmar_x2apic_optout();
if (!eim)
printk(KERN_WARNING
"Your BIOS is broken and requested that x2apic be disabled.\n"
"This will slightly decrease performance.\n"
"Use 'intremap=no_x2apic_optout' to override BIOS request.\n");
}
for_each_drhd_unit(drhd) {
struct intel_iommu *iommu = drhd->iommu;
/*
* If the queued invalidation is already initialized,
* shouldn't disable it.
*/
if (iommu->qi)
continue;
/*
* Clear previous faults.
*/
dmar_fault(-1, iommu);
/*
* Disable intr remapping and queued invalidation, if already
* enabled prior to OS handover.
*/
iommu_disable_irq_remapping(iommu);
dmar_disable_qi(iommu);
}
/*
* check for the Interrupt-remapping support
*/
for_each_drhd_unit(drhd) {
struct intel_iommu *iommu = drhd->iommu;
if (!ecap_ir_support(iommu->ecap))
continue;
if (eim && !ecap_eim_support(iommu->ecap)) {
printk(KERN_INFO "DRHD %Lx: EIM not supported by DRHD, "
" ecap %Lx\n", drhd->reg_base_addr, iommu->ecap);
goto error;
}
}
/*
* Enable queued invalidation for all the DRHD's.
*/
for_each_drhd_unit(drhd) {
int ret;
struct intel_iommu *iommu = drhd->iommu;
ret = dmar_enable_qi(iommu);
if (ret) {
printk(KERN_ERR "DRHD %Lx: failed to enable queued, "
" invalidation, ecap %Lx, ret %d\n",
drhd->reg_base_addr, iommu->ecap, ret);
goto error;
}
}
/*
* Setup Interrupt-remapping for all the DRHD's now.
*/
for_each_drhd_unit(drhd) {
struct intel_iommu *iommu = drhd->iommu;
if (!ecap_ir_support(iommu->ecap))
continue;
if (intel_setup_irq_remapping(iommu, eim))
goto error;
setup = 1;
}
if (!setup)
goto error;
irq_remapping_enabled = 1;
/*
* VT-d has a different layout for IO-APIC entries when
* interrupt remapping is enabled. So it needs a special routine
* to print IO-APIC entries for debugging purposes too.
*/
x86_io_apic_ops.print_entries = intel_ir_io_apic_print_entries;
pr_info("Enabled IRQ remapping in %s mode\n", eim ? "x2apic" : "xapic");
return eim ? IRQ_REMAP_X2APIC_MODE : IRQ_REMAP_XAPIC_MODE;
error:
/*
* handle error condition gracefully here!
*/
if (x2apic_present)
pr_warn("Failed to enable irq remapping. You are vulnerable to irq-injection attacks.\n");
return -1;
}
int __init enable_intr_remapping(void)
{
struct dmar_drhd_unit *drhd;
int setup = 0;
int eim = 0;
if (parse_ioapics_under_ir() != 1) {
printk(KERN_INFO "Not enable interrupt remapping\n");
return -1;
}
if (x2apic_supported()) {
eim = !dmar_x2apic_optout();
WARN(!eim, KERN_WARNING
"Your BIOS is broken and requested that x2apic be disabled\n"
"This will leave your machine vulnerable to irq-injection attacks\n"
"Use 'intremap=no_x2apic_optout' to override BIOS request\n");
}
for_each_drhd_unit(drhd) {
struct intel_iommu *iommu = drhd->iommu;
if (iommu->qi)
continue;
dmar_fault(-1, iommu);
iommu_disable_intr_remapping(iommu);
dmar_disable_qi(iommu);
}
for_each_drhd_unit(drhd) {
struct intel_iommu *iommu = drhd->iommu;
if (!ecap_ir_support(iommu->ecap))
continue;
if (eim && !ecap_eim_support(iommu->ecap)) {
printk(KERN_INFO "DRHD %Lx: EIM not supported by DRHD, "
" ecap %Lx\n", drhd->reg_base_addr, iommu->ecap);
return -1;
}
}
for_each_drhd_unit(drhd) {
int ret;
struct intel_iommu *iommu = drhd->iommu;
ret = dmar_enable_qi(iommu);
if (ret) {
printk(KERN_ERR "DRHD %Lx: failed to enable queued, "
" invalidation, ecap %Lx, ret %d\n",
drhd->reg_base_addr, iommu->ecap, ret);
return -1;
}
}
for_each_drhd_unit(drhd) {
struct intel_iommu *iommu = drhd->iommu;
if (!ecap_ir_support(iommu->ecap))
continue;
if (setup_intr_remapping(iommu, eim))
goto error;
setup = 1;
}
if (!setup)
goto error;
intr_remapping_enabled = 1;
pr_info("Enabled IRQ remapping in %s mode\n", eim ? "x2apic" : "xapic");
return eim ? IRQ_REMAP_X2APIC_MODE : IRQ_REMAP_XAPIC_MODE;
error:
return -1;
}
