static void __arm_lpae_free_pgtable(struct arm_lpae_io_pgtable *data, int lvl, arm_lpae_iopte *ptep) { arm_lpae_iopte *start, *end; unsigned long table_size; if (lvl == ARM_LPAE_START_LVL(data)) table_size = data->pgd_size; else table_size = ARM_LPAE_GRANULE(data); start = ptep; /* Only leaf entries at the last level */ if (lvl == ARM_LPAE_MAX_LEVELS - 1) end = ptep; else end = (void *)ptep + table_size; while (ptep != end) { arm_lpae_iopte pte = *ptep++; if (!pte || iopte_leaf(pte, lvl)) continue; __arm_lpae_free_pgtable(data, lvl + 1, iopte_deref(pte, data)); } __arm_lpae_free_pages(start, table_size, &data->iop.cfg); }
static phys_addr_t arm_lpae_iova_to_phys(struct io_pgtable_ops *ops, unsigned long iova) { struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops); arm_lpae_iopte pte, *ptep = data->pgd; int lvl = ARM_LPAE_START_LVL(data); do { /* Valid IOPTE pointer? */ if (!ptep) return 0; /* Grab the IOPTE we're interested in */ pte = *(ptep + ARM_LPAE_LVL_IDX(iova, lvl, data)); /* Valid entry? */ if (!pte) return 0; /* Leaf entry? */ if (iopte_leaf(pte,lvl)) goto found_translation; /* Take it to the next level */ ptep = iopte_deref(pte, data); } while (++lvl < ARM_LPAE_MAX_LEVELS); /* Ran out of page tables to walk */ return 0; found_translation: iova &= (ARM_LPAE_GRANULE(data) - 1); return ((phys_addr_t)iopte_to_pfn(pte,data) << data->pg_shift) | iova; }
/* IOPTE accessors */ static arm_lpae_iopte *iopte_deref(arm_lpae_iopte pte, struct arm_lpae_io_pgtable *d) { int rc; virtual_addr_t va = 0x0; physical_addr_t pa = ((pte) & ((1ULL << ARM_LPAE_MAX_ADDR_BITS) - 1) & ~(ARM_LPAE_GRANULE(d) - 1ULL)); rc = vmm_host_pa2va(pa, &va); BUG_ON(rc); return (arm_lpae_iopte *)va; }
static int __arm_lpae_unmap(struct arm_lpae_io_pgtable *data, unsigned long iova, size_t size, int lvl, arm_lpae_iopte *ptep) { arm_lpae_iopte pte; struct io_pgtable *iop = &data->iop; size_t blk_size = ARM_LPAE_BLOCK_SIZE(lvl, data); /* Something went horribly wrong and we ran out of page table */ if (WARN_ON(lvl == ARM_LPAE_MAX_LEVELS)) return 0; ptep += ARM_LPAE_LVL_IDX(iova, lvl, data); pte = *ptep; if (WARN_ON(!pte)) return 0; /* If the size matches this level, we're in the right place */ if (size == blk_size) { __arm_lpae_set_pte(ptep, 0, &iop->cfg); if (!iopte_leaf(pte, lvl)) { /* Also flush any partial walks */ io_pgtable_tlb_add_flush(iop, iova, size, ARM_LPAE_GRANULE(data), false); io_pgtable_tlb_sync(iop); ptep = iopte_deref(pte, data); __arm_lpae_free_pgtable(data, lvl + 1, ptep); } else { io_pgtable_tlb_add_flush(iop, iova, size, size, true); } return size; } else if (iopte_leaf(pte, lvl)) { /* * Insert a table at the next level to map the old region, * minus the part we want to unmap */ return arm_lpae_split_blk_unmap(data, iova, size, iopte_prot(pte), lvl, ptep, blk_size); } /* Keep on walkin' */ ptep = iopte_deref(pte, data); return __arm_lpae_unmap(data, iova, size, lvl + 1, ptep); }
static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, physical_addr_t iova, physical_addr_t paddr, size_t size, arm_lpae_iopte prot, int lvl, arm_lpae_iopte *ptep) { int rc; physical_addr_t pa; arm_lpae_iopte *cptep, pte; size_t block_size = ARM_LPAE_BLOCK_SIZE(lvl, data); struct io_pgtable_cfg *cfg = &data->iop.cfg; /* Find our entry at the current level */ ptep += ARM_LPAE_LVL_IDX(iova, lvl, data); /* If we can install a leaf entry at this level, then do so */ if (size == block_size && (size & cfg->pgsize_bitmap)) return arm_lpae_init_pte(data, iova, paddr, prot, lvl, ptep); /* We can't allocate tables at the final level */ if (WARN_ON(lvl >= ARM_LPAE_MAX_LEVELS - 1)) return VMM_EINVALID; /* Grab a pointer to the next level */ pte = *ptep; if (!pte) { cptep = __arm_lpae_alloc_pages(ARM_LPAE_GRANULE(data), cfg); if (!cptep) return VMM_ENOMEM; rc = vmm_host_va2pa((virtual_addr_t)cptep, &pa); if (rc) return rc; pte = pa | ARM_LPAE_PTE_TYPE_TABLE; if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_NS) pte |= ARM_LPAE_PTE_NSTABLE; __arm_lpae_set_pte(ptep, pte, cfg); } else { cptep = iopte_deref(pte, data); } /* Rinse, repeat */ return __arm_lpae_map(data, iova, paddr, size, prot, lvl + 1, cptep); }
static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova, phys_addr_t paddr, size_t size, arm_lpae_iopte prot, int lvl, arm_lpae_iopte *ptep) { arm_lpae_iopte *cptep, pte; size_t block_size = ARM_LPAE_BLOCK_SIZE(lvl, data); struct io_pgtable_cfg *cfg = &data->iop.cfg; /* Find our entry at the current level */ ptep += ARM_LPAE_LVL_IDX(iova, lvl, data); /* If we can install a leaf entry at this level, then do so */ if (size == block_size && (size & cfg->pgsize_bitmap)) return arm_lpae_init_pte(data, iova, paddr, prot, lvl, ptep); /* We can't allocate tables at the final level */ if (WARN_ON(lvl >= ARM_LPAE_MAX_LEVELS - 1)) return -EINVAL; /* Grab a pointer to the next level */ pte = *ptep; if (!pte) { cptep = __arm_lpae_alloc_pages(ARM_LPAE_GRANULE(data), GFP_ATOMIC, cfg); if (!cptep) return -ENOMEM; pte = __pa(cptep) | ARM_LPAE_PTE_TYPE_TABLE; if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_NS) pte |= ARM_LPAE_PTE_NSTABLE; __arm_lpae_set_pte(ptep, pte, cfg); } else if (!iopte_leaf(pte, lvl)) { cptep = iopte_deref(pte, data); } else { /* We require an unmap first */ WARN_ON(!selftest_running); return -EEXIST; } /* Rinse, repeat */ return __arm_lpae_map(data, iova, paddr, size, prot, lvl + 1, cptep); }
static struct io_pgtable * arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie) { u64 reg; struct arm_lpae_io_pgtable *data; if (cfg->quirks & ~IO_PGTABLE_QUIRK_ARM_NS) return NULL; data = arm_lpae_alloc_pgtable(cfg); if (!data) return NULL; /* TCR */ reg = (ARM_LPAE_TCR_SH_IS << ARM_LPAE_TCR_SH0_SHIFT) | (ARM_LPAE_TCR_RGN_WBWA << ARM_LPAE_TCR_IRGN0_SHIFT) | (ARM_LPAE_TCR_RGN_WBWA << ARM_LPAE_TCR_ORGN0_SHIFT); switch (ARM_LPAE_GRANULE(data)) { case SZ_4K: reg |= ARM_LPAE_TCR_TG0_4K; break; case SZ_16K: reg |= ARM_LPAE_TCR_TG0_16K; break; case SZ_64K: reg |= ARM_LPAE_TCR_TG0_64K; break; } switch (cfg->oas) { case 32: reg |= (ARM_LPAE_TCR_PS_32_BIT << ARM_LPAE_TCR_IPS_SHIFT); break; case 36: reg |= (ARM_LPAE_TCR_PS_36_BIT << ARM_LPAE_TCR_IPS_SHIFT); break; case 40: reg |= (ARM_LPAE_TCR_PS_40_BIT << ARM_LPAE_TCR_IPS_SHIFT); break; case 42: reg |= (ARM_LPAE_TCR_PS_42_BIT << ARM_LPAE_TCR_IPS_SHIFT); break; case 44: reg |= (ARM_LPAE_TCR_PS_44_BIT << ARM_LPAE_TCR_IPS_SHIFT); break; case 48: reg |= (ARM_LPAE_TCR_PS_48_BIT << ARM_LPAE_TCR_IPS_SHIFT); break; default: goto out_free_data; } reg |= (64ULL - cfg->ias) << ARM_LPAE_TCR_T0SZ_SHIFT; /* Disable speculative walks through TTBR1 */ reg |= ARM_LPAE_TCR_EPD1; cfg->arm_lpae_s1_cfg.tcr = reg; /* MAIRs */ reg = (ARM_LPAE_MAIR_ATTR_NC << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_NC)) | (ARM_LPAE_MAIR_ATTR_WBRWA << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_CACHE)) | (ARM_LPAE_MAIR_ATTR_DEVICE << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_DEV)); cfg->arm_lpae_s1_cfg.mair[0] = reg; cfg->arm_lpae_s1_cfg.mair[1] = 0; /* Looking good; allocate a pgd */ data->pgd = __arm_lpae_alloc_pages(data->pgd_size, GFP_KERNEL, cfg); if (!data->pgd) goto out_free_data; /* Ensure the empty pgd is visible before any actual TTBR write */ wmb(); /* TTBRs */ cfg->arm_lpae_s1_cfg.ttbr[0] = virt_to_phys(data->pgd); cfg->arm_lpae_s1_cfg.ttbr[1] = 0; return &data->iop; out_free_data: kfree(data); return NULL; }