static int set_up_temporary_text_mapping(pgd_t *pgd) { pmd_t *pmd; pud_t *pud; p4d_t *p4d = NULL; pgprot_t pgtable_prot = __pgprot(_KERNPG_TABLE); pgprot_t pmd_text_prot = __pgprot(__PAGE_KERNEL_LARGE_EXEC); /* Filter out unsupported __PAGE_KERNEL* bits: */ pgprot_val(pmd_text_prot) &= __default_kernel_pte_mask; pgprot_val(pgtable_prot) &= __default_kernel_pte_mask; /* * The new mapping only has to cover the page containing the image * kernel's entry point (jump_address_phys), because the switch over to * it is carried out by relocated code running from a page allocated * specifically for this purpose and covered by the identity mapping, so * the temporary kernel text mapping is only needed for the final jump. * Moreover, in that mapping the virtual address of the image kernel's * entry point must be the same as its virtual address in the image * kernel (restore_jump_address), so the image kernel's * restore_registers() code doesn't find itself in a different area of * the virtual address space after switching over to the original page * tables used by the image kernel. */ if (pgtable_l5_enabled()) { p4d = (p4d_t *)get_safe_page(GFP_ATOMIC); if (!p4d) return -ENOMEM; } pud = (pud_t *)get_safe_page(GFP_ATOMIC); if (!pud) return -ENOMEM; pmd = (pmd_t *)get_safe_page(GFP_ATOMIC); if (!pmd) return -ENOMEM; set_pmd(pmd + pmd_index(restore_jump_address), __pmd((jump_address_phys & PMD_MASK) | pgprot_val(pmd_text_prot))); set_pud(pud + pud_index(restore_jump_address), __pud(__pa(pmd) | pgprot_val(pgtable_prot))); if (p4d) { p4d_t new_p4d = __p4d(__pa(pud) | pgprot_val(pgtable_prot)); pgd_t new_pgd = __pgd(__pa(p4d) | pgprot_val(pgtable_prot)); set_p4d(p4d + p4d_index(restore_jump_address), new_p4d); set_pgd(pgd + pgd_index(restore_jump_address), new_pgd); } else { /* No p4d for 4-level paging: point the pgd to the pud page table */ pgd_t new_pgd = __pgd(__pa(pud) | pgprot_val(pgtable_prot)); set_pgd(pgd + pgd_index(restore_jump_address), new_pgd); } return 0; }
static inline p4d_t *early_p4d_offset(pgd_t *pgd, unsigned long addr) { unsigned long p4d; if (!pgtable_l5_enabled()) return (p4d_t *)pgd; p4d = __pa_nodebug(pgd_val(*pgd)) & PTE_PFN_MASK; p4d += __START_KERNEL_map - phys_base; return (p4d_t *)p4d + p4d_index(addr); }
static void __meminit init_trampoline_pud(void) { pud_t *pud_page_tramp, *pud, *pud_tramp; p4d_t *p4d_page_tramp, *p4d, *p4d_tramp; unsigned long paddr, vaddr; pgd_t *pgd; pud_page_tramp = alloc_low_page(); /* * There are two mappings for the low 1MB area, the direct mapping * and the 1:1 mapping for the real mode trampoline: * * Direct mapping: virt_addr = phys_addr + PAGE_OFFSET * 1:1 mapping: virt_addr = phys_addr */ paddr = 0; vaddr = (unsigned long)__va(paddr); pgd = pgd_offset_k(vaddr); p4d = p4d_offset(pgd, vaddr); pud = pud_offset(p4d, vaddr); pud_tramp = pud_page_tramp + pud_index(paddr); *pud_tramp = *pud; if (pgtable_l5_enabled()) { p4d_page_tramp = alloc_low_page(); p4d_tramp = p4d_page_tramp + p4d_index(paddr); set_p4d(p4d_tramp, __p4d(_KERNPG_TABLE | __pa(pud_page_tramp))); set_pgd(&trampoline_pgd_entry, __pgd(_KERNPG_TABLE | __pa(p4d_page_tramp))); } else { set_pgd(&trampoline_pgd_entry, __pgd(_KERNPG_TABLE | __pa(pud_page_tramp))); } }
pgd_t * __init efi_call_phys_prolog(void) { unsigned long vaddr, addr_pgd, addr_p4d, addr_pud; pgd_t *save_pgd, *pgd_k, *pgd_efi; p4d_t *p4d, *p4d_k, *p4d_efi; pud_t *pud; int pgd; int n_pgds, i, j; if (!efi_enabled(EFI_OLD_MEMMAP)) { save_pgd = (pgd_t *)__read_cr3(); write_cr3((unsigned long)efi_scratch.efi_pgt); goto out; } early_code_mapping_set_exec(1); n_pgds = DIV_ROUND_UP((max_pfn << PAGE_SHIFT), PGDIR_SIZE); save_pgd = kmalloc_array(n_pgds, sizeof(*save_pgd), GFP_KERNEL); /* * Build 1:1 identity mapping for efi=old_map usage. Note that * PAGE_OFFSET is PGDIR_SIZE aligned when KASLR is disabled, while * it is PUD_SIZE ALIGNED with KASLR enabled. So for a given physical * address X, the pud_index(X) != pud_index(__va(X)), we can only copy * PUD entry of __va(X) to fill in pud entry of X to build 1:1 mapping. * This means here we can only reuse the PMD tables of the direct mapping. */ for (pgd = 0; pgd < n_pgds; pgd++) { addr_pgd = (unsigned long)(pgd * PGDIR_SIZE); vaddr = (unsigned long)__va(pgd * PGDIR_SIZE); pgd_efi = pgd_offset_k(addr_pgd); save_pgd[pgd] = *pgd_efi; p4d = p4d_alloc(&init_mm, pgd_efi, addr_pgd); if (!p4d) { pr_err("Failed to allocate p4d table!\n"); goto out; } for (i = 0; i < PTRS_PER_P4D; i++) { addr_p4d = addr_pgd + i * P4D_SIZE; p4d_efi = p4d + p4d_index(addr_p4d); pud = pud_alloc(&init_mm, p4d_efi, addr_p4d); if (!pud) { pr_err("Failed to allocate pud table!\n"); goto out; } for (j = 0; j < PTRS_PER_PUD; j++) { addr_pud = addr_p4d + j * PUD_SIZE; if (addr_pud > (max_pfn << PAGE_SHIFT)) break; vaddr = (unsigned long)__va(addr_pud); pgd_k = pgd_offset_k(vaddr); p4d_k = p4d_offset(pgd_k, vaddr); pud[j] = *pud_offset(p4d_k, vaddr); } } } out: __flush_tlb_all(); return save_pgd; }
/* * Add low kernel mappings for passing arguments to EFI functions. */ void efi_sync_low_kernel_mappings(void) { unsigned num_entries; pgd_t *pgd_k, *pgd_efi; p4d_t *p4d_k, *p4d_efi; pud_t *pud_k, *pud_efi; if (efi_enabled(EFI_OLD_MEMMAP)) return; /* * We can share all PGD entries apart from the one entry that * covers the EFI runtime mapping space. * * Make sure the EFI runtime region mappings are guaranteed to * only span a single PGD entry and that the entry also maps * other important kernel regions. */ BUILD_BUG_ON(pgd_index(EFI_VA_END) != pgd_index(MODULES_END)); BUILD_BUG_ON((EFI_VA_START & PGDIR_MASK) != (EFI_VA_END & PGDIR_MASK)); pgd_efi = efi_pgd + pgd_index(PAGE_OFFSET); pgd_k = pgd_offset_k(PAGE_OFFSET); num_entries = pgd_index(EFI_VA_END) - pgd_index(PAGE_OFFSET); memcpy(pgd_efi, pgd_k, sizeof(pgd_t) * num_entries); /* * As with PGDs, we share all P4D entries apart from the one entry * that covers the EFI runtime mapping space. */ BUILD_BUG_ON(p4d_index(EFI_VA_END) != p4d_index(MODULES_END)); BUILD_BUG_ON((EFI_VA_START & P4D_MASK) != (EFI_VA_END & P4D_MASK)); pgd_efi = efi_pgd + pgd_index(EFI_VA_END); pgd_k = pgd_offset_k(EFI_VA_END); p4d_efi = p4d_offset(pgd_efi, 0); p4d_k = p4d_offset(pgd_k, 0); num_entries = p4d_index(EFI_VA_END); memcpy(p4d_efi, p4d_k, sizeof(p4d_t) * num_entries); /* * We share all the PUD entries apart from those that map the * EFI regions. Copy around them. */ BUILD_BUG_ON((EFI_VA_START & ~PUD_MASK) != 0); BUILD_BUG_ON((EFI_VA_END & ~PUD_MASK) != 0); p4d_efi = p4d_offset(pgd_efi, EFI_VA_END); p4d_k = p4d_offset(pgd_k, EFI_VA_END); pud_efi = pud_offset(p4d_efi, 0); pud_k = pud_offset(p4d_k, 0); num_entries = pud_index(EFI_VA_END); memcpy(pud_efi, pud_k, sizeof(pud_t) * num_entries); pud_efi = pud_offset(p4d_efi, EFI_VA_START); pud_k = pud_offset(p4d_k, EFI_VA_START); num_entries = PTRS_PER_PUD - pud_index(EFI_VA_START); memcpy(pud_efi, pud_k, sizeof(pud_t) * num_entries); }