static int
dmar_remap_intr(device_t dev, device_t child, u_int irq)
{
struct dmar_unit *unit;
struct dmar_msi_data *dmd;
uint64_t msi_addr;
uint32_t msi_data;
int i, error;
unit = device_get_softc(dev);
for (i = 0; i < DMAR_INTR_TOTAL; i++) {
dmd = &unit->intrs[i];
if (irq == dmd->irq) {
error = PCIB_MAP_MSI(device_get_parent(
device_get_parent(dev)),
dev, irq, &msi_addr, &msi_data);
if (error != 0)
return (error);
DMAR_LOCK(unit);
(dmd->disable_intr)(unit);
dmar_write4(unit, dmd->msi_data_reg, msi_data);
dmar_write4(unit, dmd->msi_addr_reg, msi_addr);
dmar_write4(unit, dmd->msi_uaddr_reg, msi_addr >> 32);
(dmd->enable_intr)(unit);
DMAR_UNLOCK(unit);
return (0);
}
}
static void
dmar_qi_advance_tail(struct dmar_unit *unit)
{
DMAR_ASSERT_LOCKED(unit);
dmar_write4(unit, DMAR_IQT_REG, unit->inv_queue_tail);
}
static void
dmar_qi_task(void *arg, int pending __unused)
{
struct dmar_unit *unit;
struct dmar_map_entry *entry;
uint32_t ics;
unit = arg;
DMAR_LOCK(unit);
for (;;) {
entry = TAILQ_FIRST(&unit->tlb_flush_entries);
if (entry == NULL)
break;
if (!dmar_qi_seq_processed(unit, &entry->gseq))
break;
TAILQ_REMOVE(&unit->tlb_flush_entries, entry, dmamap_link);
DMAR_UNLOCK(unit);
dmar_domain_free_entry(entry, (entry->flags &
DMAR_MAP_ENTRY_QI_NF) == 0);
DMAR_LOCK(unit);
}
ics = dmar_read4(unit, DMAR_ICS_REG);
if ((ics & DMAR_ICS_IWC) != 0) {
ics = DMAR_ICS_IWC;
dmar_write4(unit, DMAR_ICS_REG, ics);
}
if (unit->inv_seq_waiters > 0)
wakeup(&unit->inv_seq_waiters);
DMAR_UNLOCK(unit);
}
void
dmar_disable_qi_intr(struct dmar_unit *unit)
{
uint32_t iectl;
DMAR_ASSERT_LOCKED(unit);
KASSERT(DMAR_HAS_QI(unit), ("dmar%d: QI is not supported", unit->unit));
iectl = dmar_read4(unit, DMAR_IECTL_REG);
dmar_write4(unit, DMAR_IECTL_REG, iectl | DMAR_IECTL_IM);
}
int
dmar_init_qi(struct dmar_unit *unit)
{
uint64_t iqa;
uint32_t ics;
int qi_sz;
if (!DMAR_HAS_QI(unit) || (unit->hw_cap & DMAR_CAP_CM) != 0)
return (0);
unit->qi_enabled = 1;
TUNABLE_INT_FETCH("hw.dmar.qi", &unit->qi_enabled);
if (!unit->qi_enabled)
return (0);
TAILQ_INIT(&unit->tlb_flush_entries);
TASK_INIT(&unit->qi_task, 0, dmar_qi_task, unit);
unit->qi_taskqueue = taskqueue_create_fast("dmarqf", M_WAITOK,
taskqueue_thread_enqueue, &unit->qi_taskqueue);
taskqueue_start_threads(&unit->qi_taskqueue, 1, PI_AV,
"dmar%d qi taskq", unit->unit);
unit->inv_waitd_gen = 0;
unit->inv_waitd_seq = 1;
qi_sz = DMAR_IQA_QS_DEF;
TUNABLE_INT_FETCH("hw.dmar.qi_size", &qi_sz);
if (qi_sz > DMAR_IQA_QS_MAX)
qi_sz = DMAR_IQA_QS_MAX;
unit->inv_queue_size = (1ULL << qi_sz) * PAGE_SIZE;
/* Reserve one descriptor to prevent wraparound. */
unit->inv_queue_avail = unit->inv_queue_size - DMAR_IQ_DESCR_SZ;
/* The invalidation queue reads by DMARs are always coherent. */
unit->inv_queue = kmem_alloc_contig(kernel_arena, unit->inv_queue_size,
M_WAITOK | M_ZERO, 0, dmar_high, PAGE_SIZE, 0, VM_MEMATTR_DEFAULT);
unit->inv_waitd_seq_hw_phys = pmap_kextract(
(vm_offset_t)&unit->inv_waitd_seq_hw);
DMAR_LOCK(unit);
dmar_write8(unit, DMAR_IQT_REG, 0);
iqa = pmap_kextract(unit->inv_queue);
iqa |= qi_sz;
dmar_write8(unit, DMAR_IQA_REG, iqa);
dmar_enable_qi(unit);
ics = dmar_read4(unit, DMAR_ICS_REG);
if ((ics & DMAR_ICS_IWC) != 0) {
ics = DMAR_ICS_IWC;
dmar_write4(unit, DMAR_ICS_REG, ics);
}
dmar_enable_qi_intr(unit);
DMAR_UNLOCK(unit);
return (0);
}
static int
dmar_disable_qi(struct dmar_unit *unit)
{
int error;
DMAR_ASSERT_LOCKED(unit);
unit->hw_gcmd &= ~DMAR_GCMD_QIE;
dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd);
DMAR_WAIT_UNTIL(((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_QIES)
== 0));
return (error);
}
static int
dmar_disable_qi(struct dmar_unit *unit)
{
DMAR_ASSERT_LOCKED(unit);
unit->hw_gcmd &= ~DMAR_GCMD_QIE;
dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd);
/* XXXKIB should have a timeout */
while ((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_QIES) != 0)
cpu_spinwait();
return (0);
}
static int
dmar_alloc_irq(device_t dev, struct dmar_unit *unit, int idx)
{
device_t pcib;
struct dmar_msi_data *dmd;
uint64_t msi_addr;
uint32_t msi_data;
int error;
dmd = &unit->intrs[idx];
pcib = device_get_parent(device_get_parent(dev)); /* Really not pcib */
error = PCIB_ALLOC_MSIX(pcib, dev, &dmd->irq);
if (error != 0) {
device_printf(dev, "cannot allocate %s interrupt, %d\n",
dmd->name, error);
goto err1;
}
error = bus_set_resource(dev, SYS_RES_IRQ, dmd->irq_rid,
dmd->irq, 1);
if (error != 0) {
device_printf(dev, "cannot set %s interrupt resource, %d\n",
dmd->name, error);
goto err2;
}
dmd->irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ,
&dmd->irq_rid, RF_ACTIVE);
if (dmd->irq_res == NULL) {
device_printf(dev,
"cannot allocate resource for %s interrupt\n", dmd->name);
error = ENXIO;
goto err3;
}
error = bus_setup_intr(dev, dmd->irq_res, INTR_TYPE_MISC,
dmd->handler, NULL, unit, &dmd->intr_handle);
if (error != 0) {
device_printf(dev, "cannot setup %s interrupt, %d\n",
dmd->name, error);
goto err4;
}
bus_describe_intr(dev, dmd->irq_res, dmd->intr_handle, "%s", dmd->name);
error = PCIB_MAP_MSI(pcib, dev, dmd->irq, &msi_addr, &msi_data);
if (error != 0) {
device_printf(dev, "cannot map %s interrupt, %d\n",
dmd->name, error);
goto err5;
}
dmar_write4(unit, dmd->msi_data_reg, msi_data);
dmar_write4(unit, dmd->msi_addr_reg, msi_addr);
/* Only for xAPIC mode */
dmar_write4(unit, dmd->msi_uaddr_reg, msi_addr >> 32);
return (0);
err5:
bus_teardown_intr(dev, dmd->irq_res, dmd->intr_handle);
err4:
bus_release_resource(dev, SYS_RES_IRQ, dmd->irq_rid, dmd->irq_res);
err3:
bus_delete_resource(dev, SYS_RES_IRQ, dmd->irq_rid);
err2:
PCIB_RELEASE_MSIX(pcib, dev, dmd->irq);
dmd->irq = -1;
err1:
return (error);
}
