void __init page_table_range_init(unsigned long start, unsigned long end, pgd_t *pgd_base) { pgd_t *pgd; pud_t *pud; pmd_t *pmd; int pgd_idx; unsigned long vaddr; vaddr = start & PMD_MASK; end = (end + PMD_SIZE - 1) & PMD_MASK; pgd_idx = pgd_index(vaddr); pgd = pgd_base + pgd_idx; for ( ; (pgd_idx < PTRS_PER_PGD) && (vaddr != end); pgd++, pgd_idx++) { BUG_ON(pgd_none(*pgd)); pud = pud_offset(pgd, 0); BUG_ON(pud_none(*pud)); pmd = pmd_offset(pud, 0); if (!pmd_present(*pmd)) { pte_t *pte_table; pte_table = (pte_t *)alloc_bootmem_low_pages(PAGE_SIZE); memset(pte_table, 0, PAGE_SIZE); pmd_populate_kernel(&init_mm, pmd, pte_table); } vaddr += PMD_SIZE; } }
/** * kvm_mips_walk_pgd() - Walk page table with optional allocation. * @pgd: Page directory pointer. * @addr: Address to index page table using. * @cache: MMU page cache to allocate new page tables from, or NULL. * * Walk the page tables pointed to by @pgd to find the PTE corresponding to the * address @addr. If page tables don't exist for @addr, they will be created * from the MMU cache if @cache is not NULL. * * Returns: Pointer to pte_t corresponding to @addr. * NULL if a page table doesn't exist for @addr and !@cache. * NULL if a page table allocation failed. */ static pte_t *kvm_mips_walk_pgd(pgd_t *pgd, struct kvm_mmu_memory_cache *cache, unsigned long addr) { pud_t *pud; pmd_t *pmd; pgd += pgd_index(addr); if (pgd_none(*pgd)) { /* Not used on MIPS yet */ BUG(); return NULL; } pud = pud_offset(pgd, addr); if (pud_none(*pud)) { pmd_t *new_pmd; if (!cache) return NULL; new_pmd = mmu_memory_cache_alloc(cache); pmd_init((unsigned long)new_pmd, (unsigned long)invalid_pte_table); pud_populate(NULL, pud, new_pmd); } pmd = pmd_offset(pud, addr); if (pmd_none(*pmd)) { pte_t *new_pte; if (!cache) return NULL; new_pte = mmu_memory_cache_alloc(cache); clear_page(new_pte); pmd_populate_kernel(NULL, pmd, new_pte); } return pte_offset(pmd, addr); }
static int create_hyp_pmd_mappings(pud_t *pud, unsigned long start, unsigned long end, unsigned long pfn, pgprot_t prot) { pmd_t *pmd; pte_t *pte; unsigned long addr, next; addr = start; do { pmd = pmd_offset(pud, addr); BUG_ON(pmd_sect(*pmd)); if (pmd_none(*pmd)) { pte = pte_alloc_one_kernel(NULL, addr); if (!pte) { kvm_err("Cannot allocate Hyp pte\n"); return -ENOMEM; } pmd_populate_kernel(NULL, pmd, pte); get_page(virt_to_page(pmd)); kvm_flush_dcache_to_poc(pmd, sizeof(*pmd)); } next = pmd_addr_end(addr, end); create_hyp_pte_mappings(pmd, addr, next, pfn, prot); pfn += (next - addr) >> PAGE_SHIFT; } while (addr = next, addr != end); return 0; }
static void __init kasan_populate_pmd(pmd_t *pmd, unsigned long addr, unsigned long end, int nid) { pte_t *pte; if (pmd_none(*pmd)) { void *p; if (boot_cpu_has(X86_FEATURE_PSE) && ((end - addr) == PMD_SIZE) && IS_ALIGNED(addr, PMD_SIZE)) { p = early_alloc(PMD_SIZE, nid, false); if (p && pmd_set_huge(pmd, __pa(p), PAGE_KERNEL)) return; else if (p) memblock_free(__pa(p), PMD_SIZE); } p = early_alloc(PAGE_SIZE, nid, true); pmd_populate_kernel(&init_mm, pmd, p); } pte = pte_offset_kernel(pmd, addr); do { pte_t entry; void *p; if (!pte_none(*pte)) continue; p = early_alloc(PAGE_SIZE, nid, true); entry = pfn_pte(PFN_DOWN(__pa(p)), PAGE_KERNEL); set_pte_at(&init_mm, addr, pte, entry); } while (pte++, addr += PAGE_SIZE, addr != end); }
static pte_t __init *early_pte_alloc_kernel(pmd_t *pmdp, unsigned long va) { if (pmd_none(*pmdp)) { pte_t *ptep = early_alloc_pgtable(PTE_FRAG_SIZE); pmd_populate_kernel(&init_mm, pmdp, ptep); } return pte_offset_kernel(pmdp, va); }
static void __init zero_pmd_populate(pud_t *pud, unsigned long addr, unsigned long end) { pmd_t *pmd = pmd_offset(pud, addr); unsigned long next; do { next = pmd_addr_end(addr, end); if (IS_ALIGNED(addr, PMD_SIZE) && end - addr >= PMD_SIZE) { pmd_populate_kernel(&init_mm, pmd, kasan_zero_pte); continue; } if (pmd_none(*pmd)) { pmd_populate_kernel(&init_mm, pmd, early_alloc(PAGE_SIZE, NUMA_NO_NODE)); } zero_pte_populate(pmd, addr, next); } while (pmd++, addr = next, addr != end); }
static int stage2_set_pte(struct kvm *kvm, struct kvm_mmu_memory_cache *cache, phys_addr_t addr, const pte_t *new_pte, unsigned long flags) { pmd_t *pmd; pte_t *pte, old_pte; bool iomap = flags & KVM_S2PTE_FLAG_IS_IOMAP; bool logging_active = flags & KVM_S2_FLAG_LOGGING_ACTIVE; VM_BUG_ON(logging_active && !cache); /* Create stage-2 page table mapping - Levels 0 and 1 */ pmd = stage2_get_pmd(kvm, cache, addr); if (!pmd) { /* * Ignore calls from kvm_set_spte_hva for unallocated * address ranges. */ return 0; } /* * While dirty page logging - dissolve huge PMD, then continue on to * allocate page. */ if (logging_active) stage2_dissolve_pmd(kvm, addr, pmd); /* Create stage-2 page mappings - Level 2 */ if (pmd_none(*pmd)) { if (!cache) return 0; /* ignore calls from kvm_set_spte_hva */ pte = mmu_memory_cache_alloc(cache); kvm_clean_pte(pte); pmd_populate_kernel(NULL, pmd, pte); get_page(virt_to_page(pmd)); } pte = pte_offset_kernel(pmd, addr); if (iomap && pte_present(*pte)) return -EFAULT; /* Create 2nd stage page table mapping - Level 3 */ old_pte = *pte; kvm_set_pte(pte, *new_pte); if (pte_present(old_pte)) kvm_tlb_flush_vmid_ipa(kvm, addr); else get_page(virt_to_page(pte)); return 0; }
static pte_t * __init one_page_table_init(pmd_t *pmd) { if (pmd_none(*pmd)) { pte_t *pte; pte = alloc_bootmem_pages(PAGE_SIZE); pmd_populate_kernel(&init_mm, pmd, pte); BUG_ON(pte != pte_offset_kernel(pmd, 0)); } return pte_offset_kernel(pmd, 0); }
static noinline pte_t * __init alloc_kmap_pgtable(unsigned long kvaddr) { pgd_t *pgd_k; pud_t *pud_k; pmd_t *pmd_k; pte_t *pte_k; pgd_k = pgd_offset_k(kvaddr); pud_k = pud_offset(pgd_k, kvaddr); pmd_k = pmd_offset(pud_k, kvaddr); pte_k = (pte_t *)alloc_bootmem_low_pages(PAGE_SIZE); pmd_populate_kernel(&init_mm, pmd_k, pte_k); return pte_k; }
static void __init kasan_early_pte_populate(pmd_t *pmd, unsigned long addr, unsigned long end) { pte_t *pte; unsigned long next; if (pmd_none(*pmd)) pmd_populate_kernel(&init_mm, pmd, kasan_zero_pte); pte = pte_offset_kimg(pmd, addr); do { next = addr + PAGE_SIZE; set_pte(pte, pfn_pte(virt_to_pfn(kasan_zero_page), PAGE_KERNEL)); } while (pte++, addr = next, addr != end && pte_none(*pte)); }
static int stage2_set_pte(struct kvm *kvm, struct kvm_mmu_memory_cache *cache, phys_addr_t addr, const pte_t *new_pte, bool iomap) { pmd_t *pmd; pte_t *pte, old_pte; /* Create stage-2 page table mapping - Levels 0 and 1 */ pmd = stage2_get_pmd(kvm, cache, addr); if (!pmd) { /* * Ignore calls from kvm_set_spte_hva for unallocated * address ranges. */ return 0; } /* Create stage-2 page mappings - Level 2 */ if (pmd_none(*pmd)) { if (!cache) return 0; /* ignore calls from kvm_set_spte_hva */ pte = mmu_memory_cache_alloc(cache); kvm_clean_pte(pte); pmd_populate_kernel(NULL, pmd, pte); get_page(virt_to_page(pmd)); } pte = pte_offset_kernel(pmd, addr); if (iomap && pte_present(*pte)) return -EFAULT; /* Create 2nd stage page table mapping - Level 3 */ old_pte = *pte; if (pte_present(old_pte)) { /* Skip page table update if there is no change */ if (pte_val(old_pte) == pte_val(*new_pte)) return 0; kvm_set_pte(pte, __pte(0)); kvm_tlb_flush_vmid_ipa(kvm, addr); } else { get_page(virt_to_page(pte)); } kvm_set_pte(pte, *new_pte); return 0; }
static int stage2_set_pte(struct kvm *kvm, struct kvm_mmu_memory_cache *cache, phys_addr_t addr, const pte_t *new_pte, bool iomap) { pgd_t *pgd; pud_t *pud; pmd_t *pmd; pte_t *pte, old_pte; /* Create 2nd stage page table mapping - Level 1 */ pgd = kvm->arch.pgd + pgd_index(addr); pud = pud_offset(pgd, addr); if (pud_none(*pud)) { if (!cache) return 0; /* ignore calls from kvm_set_spte_hva */ pmd = mmu_memory_cache_alloc(cache); pud_populate(NULL, pud, pmd); get_page(virt_to_page(pud)); } pmd = pmd_offset(pud, addr); /* Create 2nd stage page table mapping - Level 2 */ if (pmd_none(*pmd)) { if (!cache) return 0; /* ignore calls from kvm_set_spte_hva */ pte = mmu_memory_cache_alloc(cache); kvm_clean_pte(pte); pmd_populate_kernel(NULL, pmd, pte); get_page(virt_to_page(pmd)); } pte = pte_offset_kernel(pmd, addr); if (iomap && pte_present(*pte)) return -EFAULT; /* Create 2nd stage page table mapping - Level 3 */ old_pte = *pte; kvm_set_pte(pte, *new_pte); if (pte_present(old_pte)) kvm_tlb_flush_vmid_ipa(kvm, addr); else get_page(virt_to_page(pte)); return 0; }
static int early_map_kernel_page(unsigned long ea, unsigned long pa, pgprot_t flags, unsigned int map_page_size, int nid, unsigned long region_start, unsigned long region_end) { unsigned long pfn = pa >> PAGE_SHIFT; pgd_t *pgdp; pud_t *pudp; pmd_t *pmdp; pte_t *ptep; pgdp = pgd_offset_k(ea); if (pgd_none(*pgdp)) { pudp = early_alloc_pgtable(PUD_TABLE_SIZE, nid, region_start, region_end); pgd_populate(&init_mm, pgdp, pudp); } pudp = pud_offset(pgdp, ea); if (map_page_size == PUD_SIZE) { ptep = (pte_t *)pudp; goto set_the_pte; } if (pud_none(*pudp)) { pmdp = early_alloc_pgtable(PMD_TABLE_SIZE, nid, region_start, region_end); pud_populate(&init_mm, pudp, pmdp); } pmdp = pmd_offset(pudp, ea); if (map_page_size == PMD_SIZE) { ptep = pmdp_ptep(pmdp); goto set_the_pte; } if (!pmd_present(*pmdp)) { ptep = early_alloc_pgtable(PAGE_SIZE, nid, region_start, region_end); pmd_populate_kernel(&init_mm, pmdp, ptep); } ptep = pte_offset_kernel(pmdp, ea); set_the_pte: set_pte_at(&init_mm, ea, ptep, pfn_pte(pfn, flags)); smp_wmb(); return 0; }
/* * map_kernel_page currently only called by __ioremap * map_kernel_page adds an entry to the ioremap page table * and adds an entry to the HPT, possibly bolting it */ int __ref map_kernel_page(unsigned long ea, unsigned long pa, pgprot_t prot) { pgd_t *pgdp; pud_t *pudp; pmd_t *pmdp; pte_t *ptep; BUILD_BUG_ON(TASK_SIZE_USER64 > PGTABLE_RANGE); if (slab_is_available()) { pgdp = pgd_offset_k(ea); pudp = pud_alloc(&init_mm, pgdp, ea); if (!pudp) return -ENOMEM; pmdp = pmd_alloc(&init_mm, pudp, ea); if (!pmdp) return -ENOMEM; ptep = pte_alloc_kernel(pmdp, ea); if (!ptep) return -ENOMEM; } else { pgdp = pgd_offset_k(ea); #ifndef __PAGETABLE_PUD_FOLDED if (pgd_none(*pgdp)) { pudp = early_alloc_pgtable(PUD_TABLE_SIZE); pgd_populate(&init_mm, pgdp, pudp); } #endif /* !__PAGETABLE_PUD_FOLDED */ pudp = pud_offset(pgdp, ea); if (pud_none(*pudp)) { pmdp = early_alloc_pgtable(PMD_TABLE_SIZE); pud_populate(&init_mm, pudp, pmdp); } pmdp = pmd_offset(pudp, ea); if (!pmd_present(*pmdp)) { ptep = early_alloc_pgtable(PAGE_SIZE); pmd_populate_kernel(&init_mm, pmdp, ptep); } ptep = pte_offset_kernel(pmdp, ea); } set_pte_at(&init_mm, ea, ptep, pfn_pte(pa >> PAGE_SHIFT, prot)); smp_wmb(); return 0; }
static int copy_pte(pmd_t *dst_pmdp, pmd_t *src_pmdp, unsigned long start, unsigned long end) { pte_t *src_ptep; pte_t *dst_ptep; unsigned long addr = start; dst_ptep = (pte_t *)get_safe_page(GFP_ATOMIC); if (!dst_ptep) return -ENOMEM; pmd_populate_kernel(&init_mm, dst_pmdp, dst_ptep); dst_ptep = pte_offset_kernel(dst_pmdp, start); src_ptep = pte_offset_kernel(src_pmdp, start); do { _copy_pte(dst_ptep, src_ptep, addr); } while (dst_ptep++, src_ptep++, addr += PAGE_SIZE, addr != end); return 0; }
/** * kasan_populate_zero_shadow - populate shadow memory region with * kasan_zero_page * @shadow_start - start of the memory range to populate * @shadow_end - end of the memory range to populate */ void __init kasan_populate_zero_shadow(const void *shadow_start, const void *shadow_end) { unsigned long addr = (unsigned long)shadow_start; unsigned long end = (unsigned long)shadow_end; pgd_t *pgd = pgd_offset_k(addr); unsigned long next; do { next = pgd_addr_end(addr, end); if (IS_ALIGNED(addr, PGDIR_SIZE) && end - addr >= PGDIR_SIZE) { pud_t *pud; pmd_t *pmd; /* * kasan_zero_pud should be populated with pmds * at this moment. * [pud,pmd]_populate*() below needed only for * 3,2 - level page tables where we don't have * puds,pmds, so pgd_populate(), pud_populate() * is noops. */ pgd_populate(&init_mm, pgd, kasan_zero_pud); pud = pud_offset(pgd, addr); pud_populate(&init_mm, pud, kasan_zero_pmd); pmd = pmd_offset(pud, addr); pmd_populate_kernel(&init_mm, pmd, kasan_zero_pte); continue; } if (pgd_none(*pgd)) { pgd_populate(&init_mm, pgd, early_alloc(PAGE_SIZE, NUMA_NO_NODE)); } zero_pud_populate(pgd, addr, next); } while (pgd++, addr = next, addr != end); }
/* * Copies length bytes, starting at src_start into an new page, * perform cache maintentance, then maps it at the specified address low * address as executable. * * This is used by hibernate to copy the code it needs to execute when * overwriting the kernel text. This function generates a new set of page * tables, which it loads into ttbr0. * * Length is provided as we probably only want 4K of data, even on a 64K * page system. */ static int create_safe_exec_page(void *src_start, size_t length, unsigned long dst_addr, phys_addr_t *phys_dst_addr, void *(*allocator)(gfp_t mask), gfp_t mask) { int rc = 0; pgd_t *pgdp; pud_t *pudp; pmd_t *pmdp; pte_t *ptep; unsigned long dst = (unsigned long)allocator(mask); if (!dst) { rc = -ENOMEM; goto out; } memcpy((void *)dst, src_start, length); flush_icache_range(dst, dst + length); pgdp = pgd_offset_raw(allocator(mask), dst_addr); if (pgd_none(READ_ONCE(*pgdp))) { pudp = allocator(mask); if (!pudp) { rc = -ENOMEM; goto out; } pgd_populate(&init_mm, pgdp, pudp); } pudp = pud_offset(pgdp, dst_addr); if (pud_none(READ_ONCE(*pudp))) { pmdp = allocator(mask); if (!pmdp) { rc = -ENOMEM; goto out; } pud_populate(&init_mm, pudp, pmdp); } pmdp = pmd_offset(pudp, dst_addr); if (pmd_none(READ_ONCE(*pmdp))) { ptep = allocator(mask); if (!ptep) { rc = -ENOMEM; goto out; } pmd_populate_kernel(&init_mm, pmdp, ptep); } ptep = pte_offset_kernel(pmdp, dst_addr); set_pte(ptep, pfn_pte(virt_to_pfn(dst), PAGE_KERNEL_EXEC)); /* * Load our new page tables. A strict BBM approach requires that we * ensure that TLBs are free of any entries that may overlap with the * global mappings we are about to install. * * For a real hibernate/resume cycle TTBR0 currently points to a zero * page, but TLBs may contain stale ASID-tagged entries (e.g. for EFI * runtime services), while for a userspace-driven test_resume cycle it * points to userspace page tables (and we must point it at a zero page * ourselves). Elsewhere we only (un)install the idmap with preemption * disabled, so T0SZ should be as required regardless. */ cpu_set_reserved_ttbr0(); local_flush_tlb_all(); write_sysreg(phys_to_ttbr(virt_to_phys(pgdp)), ttbr0_el1); isb(); *phys_dst_addr = virt_to_phys((void *)dst); out: return rc; }