static int walk_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end, struct mm_walk *walk) { p4d_t *p4d; unsigned long next; int err = 0; p4d = p4d_offset(pgd, addr); do { next = p4d_addr_end(addr, end); if (p4d_none_or_clear_bad(p4d)) { if (walk->pte_hole) err = walk->pte_hole(addr, next, walk); if (err) break; continue; } if (walk->pmd_entry || walk->pte_entry) err = walk_pud_range(p4d, addr, next, walk); if (err) break; } while (p4d++, addr = next, addr != end); return err; }
static void __init kasan_early_p4d_populate(pgd_t *pgd, unsigned long addr, unsigned long end) { pgd_t pgd_entry; p4d_t *p4d, p4d_entry; unsigned long next; if (pgd_none(*pgd)) { pgd_entry = __pgd(_KERNPG_TABLE | __pa_nodebug(kasan_early_shadow_p4d)); set_pgd(pgd, pgd_entry); } p4d = early_p4d_offset(pgd, addr); do { next = p4d_addr_end(addr, end); if (!p4d_none(*p4d)) continue; p4d_entry = __p4d(_KERNPG_TABLE | __pa_nodebug(kasan_early_shadow_pud)); set_p4d(p4d, p4d_entry); } while (p4d++, addr = next, addr != end && p4d_none(*p4d)); }
static inline int ioremap_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end, phys_addr_t phys_addr, pgprot_t prot) { p4d_t *p4d; unsigned long next; phys_addr -= addr; p4d = p4d_alloc(&init_mm, pgd, addr); if (!p4d) return -ENOMEM; do { next = p4d_addr_end(addr, end); if (ioremap_p4d_enabled() && ((next - addr) == P4D_SIZE) && IS_ALIGNED(phys_addr + addr, P4D_SIZE)) { if (p4d_set_huge(p4d, phys_addr + addr, prot)) continue; } if (ioremap_pud_range(p4d, addr, next, phys_addr + addr, prot)) return -ENOMEM; } while (p4d++, addr = next, addr != end); return 0; }
static void __init kasan_populate_pgd(pgd_t *pgd, unsigned long addr, unsigned long end, int nid) { void *p; p4d_t *p4d; unsigned long next; if (pgd_none(*pgd)) { p = early_alloc(PAGE_SIZE, nid, true); pgd_populate(&init_mm, pgd, p); } p4d = p4d_offset(pgd, addr); do { next = p4d_addr_end(addr, end); kasan_populate_p4d(p4d, addr, next, nid); } while (p4d++, addr = next, addr != end); }