static void
dmar_qi_advance_tail(struct dmar_unit *unit)
{
DMAR_ASSERT_LOCKED(unit);
dmar_write4(unit, DMAR_IQT_REG, unit->inv_queue_tail);
}
static void
dmar_qi_emit_wait_seq(struct dmar_unit *unit, struct dmar_qi_genseq *pseq,
bool emit_wait)
{
struct dmar_qi_genseq gsec;
uint32_t seq;
KASSERT(pseq != NULL, ("wait descriptor with no place for seq"));
DMAR_ASSERT_LOCKED(unit);
if (unit->inv_waitd_seq == 0xffffffff) {
gsec.gen = unit->inv_waitd_gen;
gsec.seq = unit->inv_waitd_seq;
dmar_qi_ensure(unit, 1);
dmar_qi_emit_wait_descr(unit, gsec.seq, false, true, false);
dmar_qi_advance_tail(unit);
while (!dmar_qi_seq_processed(unit, &gsec))
cpu_spinwait();
unit->inv_waitd_gen++;
unit->inv_waitd_seq = 1;
}
seq = unit->inv_waitd_seq++;
pseq->gen = unit->inv_waitd_gen;
pseq->seq = seq;
if (emit_wait) {
dmar_qi_ensure(unit, 1);
dmar_qi_emit_wait_descr(unit, seq, true, true, false);
}
}
static void
dmar_qi_ensure(struct dmar_unit *unit, int descr_count)
{
uint32_t head;
int bytes;
DMAR_ASSERT_LOCKED(unit);
bytes = descr_count << DMAR_IQ_DESCR_SZ_SHIFT;
for (;;) {
if (bytes <= unit->inv_queue_avail)
break;
/* refill */
head = dmar_read4(unit, DMAR_IQH_REG);
head &= DMAR_IQH_MASK;
unit->inv_queue_avail = head - unit->inv_queue_tail -
DMAR_IQ_DESCR_SZ;
if (head <= unit->inv_queue_tail)
unit->inv_queue_avail += unit->inv_queue_size;
if (bytes <= unit->inv_queue_avail)
break;
/*
* No space in the queue, do busy wait. Hardware must
* make a progress. But first advance the tail to
* inform the descriptor streamer about entries we
* might have already filled, otherwise they could
* clog the whole queue..
*/
dmar_qi_advance_tail(unit);
unit->inv_queue_full++;
cpu_spinwait();
}
unit->inv_queue_avail -= bytes;
}
void
dmar_qi_invalidate_locked(struct dmar_domain *domain, dmar_gaddr_t base,
dmar_gaddr_t size, struct dmar_qi_genseq *pseq)
{
struct dmar_unit *unit;
dmar_gaddr_t isize;
int am;
unit = domain->dmar;
DMAR_ASSERT_LOCKED(unit);
for (; size > 0; base += isize, size -= isize) {
am = calc_am(unit, base, size, &isize);
dmar_qi_ensure(unit, 1);
dmar_qi_emit(unit, DMAR_IQ_DESCR_IOTLB_INV |
DMAR_IQ_DESCR_IOTLB_PAGE | DMAR_IQ_DESCR_IOTLB_DW |
DMAR_IQ_DESCR_IOTLB_DR |
DMAR_IQ_DESCR_IOTLB_DID(domain->domain),
base | am);
}
if (pseq != NULL) {
dmar_qi_ensure(unit, 1);
dmar_qi_emit_wait_seq(unit, pseq);
}
dmar_qi_advance_tail(unit);
}
void
dmar_disable_qi_intr(struct dmar_unit *unit)
{
uint32_t iectl;
DMAR_ASSERT_LOCKED(unit);
KASSERT(DMAR_HAS_QI(unit), ("dmar%d: QI is not supported", unit->unit));
iectl = dmar_read4(unit, DMAR_IECTL_REG);
dmar_write4(unit, DMAR_IECTL_REG, iectl | DMAR_IECTL_IM);
}
static int
dmar_disable_qi(struct dmar_unit *unit)
{
int error;
DMAR_ASSERT_LOCKED(unit);
unit->hw_gcmd &= ~DMAR_GCMD_QIE;
dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd);
DMAR_WAIT_UNTIL(((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_QIES)
== 0));
return (error);
}
void
dmar_qi_invalidate_iec_glob(struct dmar_unit *unit)
{
struct dmar_qi_genseq gseq;
DMAR_ASSERT_LOCKED(unit);
dmar_qi_ensure(unit, 2);
dmar_qi_emit(unit, DMAR_IQ_DESCR_IEC_INV, 0);
dmar_qi_emit_wait_seq(unit, &gseq, true);
dmar_qi_advance_tail(unit);
dmar_qi_wait_for_seq(unit, &gseq, false);
}
void
dmar_qi_invalidate_ctx_glob_locked(struct dmar_unit *unit)
{
struct dmar_qi_genseq gseq;
DMAR_ASSERT_LOCKED(unit);
dmar_qi_ensure(unit, 2);
dmar_qi_emit(unit, DMAR_IQ_DESCR_CTX_INV | DMAR_IQ_DESCR_CTX_GLOB, 0);
dmar_qi_emit_wait_seq(unit, &gseq, true);
dmar_qi_advance_tail(unit);
dmar_qi_wait_for_seq(unit, &gseq, false);
}
static int
dmar_disable_qi(struct dmar_unit *unit)
{
DMAR_ASSERT_LOCKED(unit);
unit->hw_gcmd &= ~DMAR_GCMD_QIE;
dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd);
/* XXXKIB should have a timeout */
while ((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_QIES) != 0)
cpu_spinwait();
return (0);
}
static void
dmar_qi_emit_wait_descr(struct dmar_unit *unit, uint32_t seq, bool intr,
bool memw, bool fence)
{
DMAR_ASSERT_LOCKED(unit);
dmar_qi_emit(unit, DMAR_IQ_DESCR_WAIT_ID |
(intr ? DMAR_IQ_DESCR_WAIT_IF : 0) |
(memw ? DMAR_IQ_DESCR_WAIT_SW : 0) |
(fence ? DMAR_IQ_DESCR_WAIT_FN : 0) |
(memw ? DMAR_IQ_DESCR_WAIT_SD(seq) : 0),
memw ? unit->inv_waitd_seq_hw_phys : 0);
}
void
dmar_qi_invalidate_iotlb_glob_locked(struct dmar_unit *unit)
{
struct dmar_qi_genseq gseq;
DMAR_ASSERT_LOCKED(unit);
dmar_qi_ensure(unit, 2);
dmar_qi_emit(unit, DMAR_IQ_DESCR_IOTLB_INV | DMAR_IQ_DESCR_IOTLB_GLOB |
DMAR_IQ_DESCR_IOTLB_DW | DMAR_IQ_DESCR_IOTLB_DR, 0);
dmar_qi_emit_wait_seq(unit, &gseq);
dmar_qi_advance_tail(unit);
dmar_qi_wait_for_seq(unit, &gseq, false);
}
static void
dmar_qi_wait_for_seq(struct dmar_unit *unit, const struct dmar_qi_genseq *gseq)
{
DMAR_ASSERT_LOCKED(unit);
unit->inv_seq_waiters++;
while (!dmar_qi_seq_processed(unit, gseq)) {
if (cold) {
cpu_spinwait();
} else {
msleep(&unit->inv_seq_waiters, &unit->lock, 0,
"dmarse", hz);
}
}
unit->inv_seq_waiters--;
}
static void
dmar_qi_emit(struct dmar_unit *unit, uint64_t data1, uint64_t data2)
{
DMAR_ASSERT_LOCKED(unit);
*(volatile uint64_t *)(unit->inv_queue + unit->inv_queue_tail) = data1;
unit->inv_queue_tail += DMAR_IQ_DESCR_SZ / 2;
KASSERT(unit->inv_queue_tail <= unit->inv_queue_size,
("tail overflow 0x%x 0x%jx", unit->inv_queue_tail,
(uintmax_t)unit->inv_queue_size));
unit->inv_queue_tail &= unit->inv_queue_size - 1;
*(volatile uint64_t *)(unit->inv_queue + unit->inv_queue_tail) = data2;
unit->inv_queue_tail += DMAR_IQ_DESCR_SZ / 2;
KASSERT(unit->inv_queue_tail <= unit->inv_queue_size,
("tail overflow 0x%x 0x%jx", unit->inv_queue_tail,
(uintmax_t)unit->inv_queue_size));
unit->inv_queue_tail &= unit->inv_queue_size - 1;
}
void
dmar_qi_invalidate_iec(struct dmar_unit *unit, u_int start, u_int cnt)
{
struct dmar_qi_genseq gseq;
u_int c, l;
DMAR_ASSERT_LOCKED(unit);
KASSERT(start < unit->irte_cnt && start < start + cnt &&
start + cnt <= unit->irte_cnt,
("inv iec overflow %d %d %d", unit->irte_cnt, start, cnt));
for (; cnt > 0; cnt -= c, start += c) {
l = ffs(start | cnt) - 1;
c = 1 << l;
dmar_qi_ensure(unit, 1);
dmar_qi_emit(unit, DMAR_IQ_DESCR_IEC_INV |
DMAR_IQ_DESCR_IEC_IDX | DMAR_IQ_DESCR_IEC_IIDX(start) |
DMAR_IQ_DESCR_IEC_IM(l), 0);
}
dmar_qi_ensure(unit, 1);
dmar_qi_emit_wait_seq(unit, &gseq, true);
dmar_qi_advance_tail(unit);
/*
* The caller of the function, in particular,
* dmar_ir_program_irte(), may be called from the context
* where the sleeping is forbidden (in fact, the
* intr_table_lock mutex may be held, locked from
* intr_shuffle_irqs()). Wait for the invalidation completion
* using the busy wait.
*
* The impact on the interrupt input setup code is small, the
* expected overhead is comparable with the chipset register
* read. It is more harmful for the parallel DMA operations,
* since we own the dmar unit lock until whole invalidation
* queue is processed, which includes requests possibly issued
* before our request.
*/
dmar_qi_wait_for_seq(unit, &gseq, true);
}