pgd_t * __init efi_call_phys_prolog(void) { unsigned long vaddress; pgd_t *save_pgd; int pgd; int n_pgds; if (!efi_enabled(EFI_OLD_MEMMAP)) { save_pgd = (pgd_t *)read_cr3(); write_cr3((unsigned long)efi_scratch.efi_pgt); goto out; } early_code_mapping_set_exec(1); n_pgds = DIV_ROUND_UP((max_pfn << PAGE_SHIFT), PGDIR_SIZE); save_pgd = kmalloc_array(n_pgds, sizeof(*save_pgd), GFP_KERNEL); for (pgd = 0; pgd < n_pgds; pgd++) { save_pgd[pgd] = *pgd_offset_k(pgd * PGDIR_SIZE); vaddress = (unsigned long)__va(pgd * PGDIR_SIZE); set_pgd(pgd_offset_k(pgd * PGDIR_SIZE), *pgd_offset_k(vaddress)); } out: __flush_tlb_all(); return save_pgd; }
void __init efi_call_phys_epilog(void) { /* * After the lock is released, the original page table is restored. */ set_pgd(pgd_offset_k(0x0UL), save_pgd); __flush_tlb_all(); local_irq_restore(efi_flags); early_code_mapping_set_exec(0); }
void __init efi_call_phys_epilog(void) { int pgd; int n_pgds = DIV_ROUND_UP((max_pfn << PAGE_SHIFT) , PGDIR_SIZE); for (pgd = 0; pgd < n_pgds; pgd++) set_pgd(pgd_offset_k(pgd * PGDIR_SIZE), save_pgd[pgd]); kfree(save_pgd); __flush_tlb_all(); local_irq_restore(efi_flags); early_code_mapping_set_exec(0); }
void __init efi_call_phys_prelog(void) { unsigned long vaddress; early_code_mapping_set_exec(1); local_irq_save(efi_flags); vaddress = (unsigned long)__va(0x0UL); save_pgd = *pgd_offset_k(0x0UL); set_pgd(pgd_offset_k(0x0UL), *pgd_offset_k(vaddress)); __flush_tlb_all(); }
void __init efi_call_phys_epilog(pgd_t *save_pgd) { /* * After the lock is released, the original page table is restored. */ int pgd_idx, i; int nr_pgds; pgd_t *pgd; p4d_t *p4d; pud_t *pud; if (!efi_enabled(EFI_OLD_MEMMAP)) { write_cr3((unsigned long)save_pgd); __flush_tlb_all(); return; } nr_pgds = DIV_ROUND_UP((max_pfn << PAGE_SHIFT) , PGDIR_SIZE); for (pgd_idx = 0; pgd_idx < nr_pgds; pgd_idx++) { pgd = pgd_offset_k(pgd_idx * PGDIR_SIZE); set_pgd(pgd_offset_k(pgd_idx * PGDIR_SIZE), save_pgd[pgd_idx]); if (!(pgd_val(*pgd) & _PAGE_PRESENT)) continue; for (i = 0; i < PTRS_PER_P4D; i++) { p4d = p4d_offset(pgd, pgd_idx * PGDIR_SIZE + i * P4D_SIZE); if (!(p4d_val(*p4d) & _PAGE_PRESENT)) continue; pud = (pud_t *)p4d_page_vaddr(*p4d); pud_free(&init_mm, pud); } p4d = (p4d_t *)pgd_page_vaddr(*pgd); p4d_free(&init_mm, p4d); } kfree(save_pgd); __flush_tlb_all(); early_code_mapping_set_exec(0); }
void __init efi_call_phys_epilog(void) { /* * After the lock is released, the original page table is restored. */ int pgd; int n_pgds = DIV_ROUND_UP((max_pfn << PAGE_SHIFT) , PGDIR_SIZE); if (!efi_enabled(EFI_OLD_MEMMAP)) return; for (pgd = 0; pgd < n_pgds; pgd++) set_pgd(pgd_offset_k(pgd * PGDIR_SIZE), save_pgd[pgd]); kfree(save_pgd); __flush_tlb_all(); local_irq_restore(efi_flags); early_code_mapping_set_exec(0); }
void __init efi_call_phys_prelog(void) { unsigned long vaddress; int pgd; int n_pgds; early_code_mapping_set_exec(1); local_irq_save(efi_flags); n_pgds = DIV_ROUND_UP((max_pfn << PAGE_SHIFT), PGDIR_SIZE); save_pgd = kmalloc(n_pgds * sizeof(pgd_t), GFP_KERNEL); for (pgd = 0; pgd < n_pgds; pgd++) { save_pgd[pgd] = *pgd_offset_k(pgd * PGDIR_SIZE); vaddress = (unsigned long)__va(pgd * PGDIR_SIZE); set_pgd(pgd_offset_k(pgd * PGDIR_SIZE), *pgd_offset_k(vaddress)); } __flush_tlb_all(); }
void __init efi_call_phys_epilog(void) { /* * After the lock is released, the original page table is restored. */ int pgd; int n_pgds = DIV_ROUND_UP((max_pfn << PAGE_SHIFT) , PGDIR_SIZE); for (pgd = 0; pgd < n_pgds; pgd++) set_pgd(pgd_offset_k(pgd * PGDIR_SIZE), save_pgd[pgd]); kfree(save_pgd); #ifdef CONFIG_PAX_PER_CPU_PGD load_cr3(get_cpu_pgd(smp_processor_id(), kernel)); #endif __flush_tlb_all(); local_irq_restore(efi_flags); early_code_mapping_set_exec(0); }
void __init efi_call_phys_epilog(pgd_t *save_pgd) { /* * After the lock is released, the original page table is restored. */ int pgd_idx; int nr_pgds; if (!efi_enabled(EFI_OLD_MEMMAP)) { write_cr3((unsigned long)save_pgd); __flush_tlb_all(); return; } nr_pgds = DIV_ROUND_UP((max_pfn << PAGE_SHIFT) , PGDIR_SIZE); for (pgd_idx = 0; pgd_idx < nr_pgds; pgd_idx++) set_pgd(pgd_offset_k(pgd_idx * PGDIR_SIZE), save_pgd[pgd_idx]); kfree(save_pgd); __flush_tlb_all(); early_code_mapping_set_exec(0); }
pgd_t * __init efi_call_phys_prolog(void) { unsigned long vaddr, addr_pgd, addr_p4d, addr_pud; pgd_t *save_pgd, *pgd_k, *pgd_efi; p4d_t *p4d, *p4d_k, *p4d_efi; pud_t *pud; int pgd; int n_pgds, i, j; if (!efi_enabled(EFI_OLD_MEMMAP)) { save_pgd = (pgd_t *)__read_cr3(); write_cr3((unsigned long)efi_scratch.efi_pgt); goto out; } early_code_mapping_set_exec(1); n_pgds = DIV_ROUND_UP((max_pfn << PAGE_SHIFT), PGDIR_SIZE); save_pgd = kmalloc_array(n_pgds, sizeof(*save_pgd), GFP_KERNEL); /* * Build 1:1 identity mapping for efi=old_map usage. Note that * PAGE_OFFSET is PGDIR_SIZE aligned when KASLR is disabled, while * it is PUD_SIZE ALIGNED with KASLR enabled. So for a given physical * address X, the pud_index(X) != pud_index(__va(X)), we can only copy * PUD entry of __va(X) to fill in pud entry of X to build 1:1 mapping. * This means here we can only reuse the PMD tables of the direct mapping. */ for (pgd = 0; pgd < n_pgds; pgd++) { addr_pgd = (unsigned long)(pgd * PGDIR_SIZE); vaddr = (unsigned long)__va(pgd * PGDIR_SIZE); pgd_efi = pgd_offset_k(addr_pgd); save_pgd[pgd] = *pgd_efi; p4d = p4d_alloc(&init_mm, pgd_efi, addr_pgd); if (!p4d) { pr_err("Failed to allocate p4d table!\n"); goto out; } for (i = 0; i < PTRS_PER_P4D; i++) { addr_p4d = addr_pgd + i * P4D_SIZE; p4d_efi = p4d + p4d_index(addr_p4d); pud = pud_alloc(&init_mm, p4d_efi, addr_p4d); if (!pud) { pr_err("Failed to allocate pud table!\n"); goto out; } for (j = 0; j < PTRS_PER_PUD; j++) { addr_pud = addr_p4d + j * PUD_SIZE; if (addr_pud > (max_pfn << PAGE_SHIFT)) break; vaddr = (unsigned long)__va(addr_pud); pgd_k = pgd_offset_k(vaddr); p4d_k = p4d_offset(pgd_k, vaddr); pud[j] = *pud_offset(p4d_k, vaddr); } } } out: __flush_tlb_all(); return save_pgd; }