static pud_t *__init kasan_pud_offset(pgd_t *pgdp, unsigned long addr, int node, bool early) { if (pgd_none(READ_ONCE(*pgdp))) { phys_addr_t pud_phys = early ? __pa_symbol(kasan_early_shadow_pud) : kasan_alloc_zeroed_page(node); __pgd_populate(pgdp, pud_phys, PMD_TYPE_TABLE); } return early ? pud_offset_kimg(pgdp, addr) : pud_offset(pgdp, addr); }
static void alloc_init_pud(pgd_t *pgd, unsigned long addr, unsigned long end, phys_addr_t phys, pgprot_t prot, phys_addr_t (*pgtable_alloc)(void)) { pud_t *pud; unsigned long next; if (pgd_none(*pgd)) { phys_addr_t pud_phys; BUG_ON(!pgtable_alloc); pud_phys = pgtable_alloc(); __pgd_populate(pgd, pud_phys, PUD_TYPE_TABLE); } BUG_ON(pgd_bad(*pgd)); pud = pud_set_fixmap_offset(pgd, addr); do { next = pud_addr_end(addr, end); /* * For 4K granule only, attempt to put down a 1GB block */ if (use_1G_block(addr, next, phys) && block_mappings_allowed(pgtable_alloc)) { pud_t old_pud = *pud; pud_set_huge(pud, phys, prot); /* * If we have an old value for a pud, it will * be pointing to a pmd table that we no longer * need (from swapper_pg_dir). * * Look up the old pmd table and free it. */ if (!pud_none(old_pud)) { flush_tlb_all(); if (pud_table(old_pud)) { phys_addr_t table = pud_page_paddr(old_pud); if (!WARN_ON_ONCE(slab_is_available())) memblock_free(table, PAGE_SIZE); } } } else { alloc_init_pmd(pud, addr, next, phys, prot, pgtable_alloc); } phys += next - addr; } while (pud++, addr = next, addr != end); pud_clear_fixmap(); }
static void __init kasan_early_pud_populate(pgd_t *pgd, unsigned long addr, unsigned long end) { pud_t *pud; unsigned long next; if (pgd_none(*pgd)) __pgd_populate(pgd, __pa_symbol(kasan_zero_pud), PUD_TYPE_TABLE); pud = pud_offset_kimg(pgd, addr); do { next = pud_addr_end(addr, end); kasan_early_pmd_populate(pud, addr, next); } while (pud++, addr = next, addr != end && pud_none(*pud)); }