static int set_up_temporary_text_mapping(pgd_t *pgd)
{
pmd_t *pmd;
pud_t *pud;
p4d_t *p4d = NULL;
pgprot_t pgtable_prot = __pgprot(_KERNPG_TABLE);
pgprot_t pmd_text_prot = __pgprot(__PAGE_KERNEL_LARGE_EXEC);
/* Filter out unsupported __PAGE_KERNEL* bits: */
pgprot_val(pmd_text_prot) &= __default_kernel_pte_mask;
pgprot_val(pgtable_prot) &= __default_kernel_pte_mask;
/*
* The new mapping only has to cover the page containing the image
* kernel's entry point (jump_address_phys), because the switch over to
* it is carried out by relocated code running from a page allocated
* specifically for this purpose and covered by the identity mapping, so
* the temporary kernel text mapping is only needed for the final jump.
* Moreover, in that mapping the virtual address of the image kernel's
* entry point must be the same as its virtual address in the image
* kernel (restore_jump_address), so the image kernel's
* restore_registers() code doesn't find itself in a different area of
* the virtual address space after switching over to the original page
* tables used by the image kernel.
*/
if (pgtable_l5_enabled()) {
p4d = (p4d_t *)get_safe_page(GFP_ATOMIC);
if (!p4d)
return -ENOMEM;
}
pud = (pud_t *)get_safe_page(GFP_ATOMIC);
if (!pud)
return -ENOMEM;
pmd = (pmd_t *)get_safe_page(GFP_ATOMIC);
if (!pmd)
return -ENOMEM;
set_pmd(pmd + pmd_index(restore_jump_address),
__pmd((jump_address_phys & PMD_MASK) | pgprot_val(pmd_text_prot)));
set_pud(pud + pud_index(restore_jump_address),
__pud(__pa(pmd) | pgprot_val(pgtable_prot)));
if (p4d) {
p4d_t new_p4d = __p4d(__pa(pud) | pgprot_val(pgtable_prot));
pgd_t new_pgd = __pgd(__pa(p4d) | pgprot_val(pgtable_prot));
set_p4d(p4d + p4d_index(restore_jump_address), new_p4d);
set_pgd(pgd + pgd_index(restore_jump_address), new_pgd);
} else {
/* No p4d for 4-level paging: point the pgd to the pud page table */
pgd_t new_pgd = __pgd(__pa(pud) | pgprot_val(pgtable_prot));
set_pgd(pgd + pgd_index(restore_jump_address), new_pgd);
}
return 0;
}
/*H:435
* And this is us, creating the new page directory. If we really do
* allocate a new one (and so the kernel parts are not there), we set
* blank_pgdir.
*/
static unsigned int new_pgdir(struct lg_cpu *cpu,
unsigned long gpgdir,
int *blank_pgdir)
{
unsigned int next;
#ifdef CONFIG_X86_PAE
pmd_t *pmd_table;
#endif
/*
* We pick one entry at random to throw out. Choosing the Least
* Recently Used might be better, but this is easy.
*/
next = random32() % ARRAY_SIZE(cpu->lg->pgdirs);
/* If it's never been allocated at all before, try now. */
if (!cpu->lg->pgdirs[next].pgdir) {
cpu->lg->pgdirs[next].pgdir =
(pgd_t *)get_zeroed_page(GFP_KERNEL);
/* If the allocation fails, just keep using the one we have */
if (!cpu->lg->pgdirs[next].pgdir)
next = cpu->cpu_pgd;
else {
#ifdef CONFIG_X86_PAE
/*
* In PAE mode, allocate a pmd page and populate the
* last pgd entry.
*/
pmd_table = (pmd_t *)get_zeroed_page(GFP_KERNEL);
if (!pmd_table) {
free_page((long)cpu->lg->pgdirs[next].pgdir);
set_pgd(cpu->lg->pgdirs[next].pgdir, __pgd(0));
next = cpu->cpu_pgd;
} else {
set_pgd(cpu->lg->pgdirs[next].pgdir +
SWITCHER_PGD_INDEX,
__pgd(__pa(pmd_table) | _PAGE_PRESENT));
/*
* This is a blank page, so there are no kernel
* mappings: caller must map the stack!
*/
*blank_pgdir = 1;
}
#else
*blank_pgdir = 1;
#endif
}
}
/* Record which Guest toplevel this shadows. */
cpu->lg->pgdirs[next].gpgdir = gpgdir;
/* Release all the non-kernel mappings. */
flush_user_mappings(cpu->lg, next);
return next;
}
pgd_t *pgd_alloc(struct mm_struct *mm)
{
int i;
pgd_t *pgd = kmem_cache_alloc(pgd_cache, GFP_KERNEL);
pgd_test_and_unpin(pgd);
if (PTRS_PER_PMD == 1 || !pgd)
return pgd;
for (i = 0; i < USER_PTRS_PER_PGD; ++i) {
pmd_t *pmd = kmem_cache_alloc(pmd_cache, GFP_KERNEL);
if (!pmd)
goto out_oom;
set_pgd(&pgd[i], __pgd(1 + __pa(pmd)));
}
if (!HAVE_SHARED_KERNEL_PMD) {
unsigned long flags;
for (i = USER_PTRS_PER_PGD; i < PTRS_PER_PGD; i++) {
pmd_t *pmd = kmem_cache_alloc(pmd_cache, GFP_KERNEL);
if (!pmd)
goto out_oom;
set_pgd(&pgd[i], __pgd(1 + __pa(pmd)));
}
spin_lock_irqsave(&pgd_lock, flags);
for (i = USER_PTRS_PER_PGD; i < PTRS_PER_PGD; i++) {
unsigned long v = (unsigned long)i << PGDIR_SHIFT;
pgd_t *kpgd = pgd_offset_k(v);
pud_t *kpud = pud_offset(kpgd, v);
pmd_t *kpmd = pmd_offset(kpud, v);
pmd_t *pmd = (void *)__va(pgd_val(pgd[i])-1);
memcpy(pmd, kpmd, PAGE_SIZE);
make_lowmem_page_readonly(
pmd, XENFEAT_writable_page_tables);
}
pgd_list_add(pgd);
spin_unlock_irqrestore(&pgd_lock, flags);
}
return pgd;
out_oom:
for (i--; i >= 0; i--)
kmem_cache_free(pmd_cache, (void *)__va(pgd_val(pgd[i])-1));
kmem_cache_free(pgd_cache, pgd);
return NULL;
}
static void __init kasan_early_p4d_populate(pgd_t *pgd,
unsigned long addr,
unsigned long end)
{
pgd_t pgd_entry;
p4d_t *p4d, p4d_entry;
unsigned long next;
if (pgd_none(*pgd)) {
pgd_entry = __pgd(_KERNPG_TABLE |
__pa_nodebug(kasan_early_shadow_p4d));
set_pgd(pgd, pgd_entry);
}
p4d = early_p4d_offset(pgd, addr);
do {
next = p4d_addr_end(addr, end);
if (!p4d_none(*p4d))
continue;
p4d_entry = __p4d(_KERNPG_TABLE |
__pa_nodebug(kasan_early_shadow_pud));
set_p4d(p4d, p4d_entry);
} while (p4d++, addr = next, addr != end && p4d_none(*p4d));
}
static int set_up_temporary_text_mapping(pgd_t *pgd)
{
pmd_t *pmd;
pud_t *pud;
/*
* The new mapping only has to cover the page containing the image
* kernel's entry point (jump_address_phys), because the switch over to
* it is carried out by relocated code running from a page allocated
* specifically for this purpose and covered by the identity mapping, so
* the temporary kernel text mapping is only needed for the final jump.
* Moreover, in that mapping the virtual address of the image kernel's
* entry point must be the same as its virtual address in the image
* kernel (restore_jump_address), so the image kernel's
* restore_registers() code doesn't find itself in a different area of
* the virtual address space after switching over to the original page
* tables used by the image kernel.
*/
pud = (pud_t *)get_safe_page(GFP_ATOMIC);
if (!pud)
return -ENOMEM;
pmd = (pmd_t *)get_safe_page(GFP_ATOMIC);
if (!pmd)
return -ENOMEM;
set_pmd(pmd + pmd_index(restore_jump_address),
__pmd((jump_address_phys & PMD_MASK) | __PAGE_KERNEL_LARGE_EXEC));
set_pud(pud + pud_index(restore_jump_address),
__pud(__pa(pmd) | _KERNPG_TABLE));
set_pgd(pgd + pgd_index(restore_jump_address),
__pgd(__pa(pud) | _KERNPG_TABLE));
return 0;
}
int kernel_ident_mapping_init(struct x86_mapping_info *info, pgd_t *pgd_page,
unsigned long pstart, unsigned long pend)
{
unsigned long addr = pstart + info->offset;
unsigned long end = pend + info->offset;
unsigned long next;
int result;
for (; addr < end; addr = next) {
pgd_t *pgd = pgd_page + pgd_index(addr);
pud_t *pud;
next = (addr & PGDIR_MASK) + PGDIR_SIZE;
if (next > end)
next = end;
if (pgd_present(*pgd)) {
pud = pud_offset(pgd, 0);
result = ident_pud_init(info, pud, addr, next);
if (result)
return result;
continue;
}
pud = (pud_t *)info->alloc_pgt_page(info->context);
if (!pud)
return -ENOMEM;
result = ident_pud_init(info, pud, addr, next);
if (result)
return result;
set_pgd(pgd, __pgd(__pa(pud) | _KERNPG_TABLE));
}
return 0;
}
/*
* Create PGD aligned trampoline table to allow real mode initialization
* of additional CPUs. Consume only 1 low memory page.
*/
void __meminit init_trampoline(void)
{
unsigned long paddr, paddr_next;
pgd_t *pgd;
pud_t *pud_page, *pud_page_tramp;
int i;
if (!kaslr_memory_enabled()) {
init_trampoline_default();
return;
}
pud_page_tramp = alloc_low_page();
paddr = 0;
pgd = pgd_offset_k((unsigned long)__va(paddr));
pud_page = (pud_t *) pgd_page_vaddr(*pgd);
for (i = pud_index(paddr); i < PTRS_PER_PUD; i++, paddr = paddr_next) {
pud_t *pud, *pud_tramp;
unsigned long vaddr = (unsigned long)__va(paddr);
pud_tramp = pud_page_tramp + pud_index(paddr);
pud = pud_page + pud_index(vaddr);
paddr_next = (paddr & PUD_MASK) + PUD_SIZE;
*pud_tramp = *pud;
}
set_pgd(&trampoline_pgd_entry,
__pgd(_KERNPG_TABLE | __pa(pud_page_tramp)));
}
/*
* set up paging
*/
void __init paging_init(void)
{
unsigned long zones_size[MAX_NR_ZONES] = {0,};
pte_t *ppte;
int loop;
/* main kernel space -> RAM mapping is handled as 1:1 transparent by
* the MMU */
memset(swapper_pg_dir, 0, sizeof(swapper_pg_dir));
memset(kernel_vmalloc_ptes, 0, sizeof(kernel_vmalloc_ptes));
/* load the VMALLOC area PTE table addresses into the kernel PGD */
ppte = kernel_vmalloc_ptes;
for (loop = VMALLOC_START / (PAGE_SIZE * PTRS_PER_PTE);
loop < VMALLOC_END / (PAGE_SIZE * PTRS_PER_PTE);
loop++
) {
set_pgd(swapper_pg_dir + loop, __pgd(__pa(ppte) | _PAGE_TABLE));
ppte += PAGE_SIZE / sizeof(pte_t);
}
/* declare the sizes of the RAM zones (only use the normal zone) */
zones_size[ZONE_NORMAL] =
contig_page_data.bdata->node_low_pfn -
contig_page_data.bdata->node_min_pfn;
/* pass the memory from the bootmem allocator to the main allocator */
free_area_init(zones_size);
__flush_tlb_all();
}
static void shmedia_mapioaddr(unsigned long pa, unsigned long va)
{
pgd_t *pgdp;
pmd_t *pmdp;
pte_t *ptep;
unsigned long flags = 1; /* 1 = CB0-1 device */
DEBUG_IOREMAP(("shmedia_mapiopage pa %08x va %08x\n", pa, va));
pgdp = pgd_offset_k(va);
if (pgd_none(*pgdp)) {
pmdp = alloc_bootmem_low_pages(PTRS_PER_PMD * sizeof(pmd_t));
if (pmdp == NULL) panic("No memory for pmd\n");
memset(pmdp, 0, PTRS_PER_PGD * sizeof(pmd_t));
set_pgd(pgdp, __pgd((unsigned long)pmdp | _KERNPG_TABLE));
}
pmdp = pmd_offset(pgdp, va);
if (pmd_none(*pmdp)) {
ptep = alloc_bootmem_low_pages(PTRS_PER_PTE * sizeof(pte_t));
if (ptep == NULL) panic("No memory for pte\n");
clear_page((void *)ptep);
set_pmd(pmdp, __pmd((unsigned long)ptep + _PAGE_TABLE));
}
ptep = pte_offset(pmdp, va);
set_pte(ptep, mk_pte_phys(pa, __pgprot(_PAGE_PRESENT |
_PAGE_READ | _PAGE_WRITE |
_PAGE_DIRTY | _PAGE_ACCESSED |_PAGE_SHARED | flags)));
}
pgd_t *pgd_alloc(struct mm_struct *mm)
{
int i;
pgd_t *pgd = kmem_cache_alloc(pgd_cache, GFP_KERNEL);
if (PTRS_PER_PMD == 1 || !pgd)
return pgd;
for (i = 0; i < USER_PTRS_PER_PGD; ++i) {
pmd_t *pmd = kmem_cache_alloc(pmd_cache, GFP_KERNEL);
if (!pmd)
goto out_oom;
paravirt_alloc_pd(__pa(pmd) >> PAGE_SHIFT);
set_pgd(&pgd[i], __pgd(1 + __pa(pmd)));
}
return pgd;
out_oom:
for (i--; i >= 0; i--) {
pgd_t pgdent = pgd[i];
void* pmd = (void *)__va(pgd_val(pgdent)-1);
paravirt_release_pd(__pa(pmd) >> PAGE_SHIFT);
kmem_cache_free(pmd_cache, pmd);
}
kmem_cache_free(pgd_cache, pgd);
return NULL;
}
pgd_t *pgd_alloc(struct mm_struct *mm)
{
int i;
pgd_t *pgd = quicklist_alloc(0, GFP_KERNEL, pgd_ctor);
if (PTRS_PER_PMD == 1 || !pgd)
return pgd;
for (i = 0; i < UNSHARED_PTRS_PER_PGD; ++i) {
pmd_t *pmd = pmd_cache_alloc(i);
if (!pmd)
goto out_oom;
paravirt_alloc_pd(__pa(pmd) >> PAGE_SHIFT);
set_pgd(&pgd[i], __pgd(1 + __pa(pmd)));
}
return pgd;
out_oom:
for (i--; i >= 0; i--) {
pgd_t pgdent = pgd[i];
void* pmd = (void *)__va(pgd_val(pgdent)-1);
paravirt_release_pd(__pa(pmd) >> PAGE_SHIFT);
pmd_cache_free(pmd, i);
}
quicklist_free(0, pgd_dtor, pgd);
return NULL;
}
static inline void resume_init_first_level_page_table(pgd_t *pg_dir)
{
#ifdef CONFIG_X86_PAE
int i;
/* Init entries of the first-level page table to the zero page */
for (i = 0; i < PTRS_PER_PGD; i++)
set_pgd(pg_dir + i,
__pgd(__pa(empty_zero_page) | _PAGE_PRESENT));
#endif
}
void zap_low_mappings (void)
{
int i;
save_pg_dir();
/*
* Zap initial low-memory mappings.
*
* Note that "pgd_clear()" doesn't do it for
* us, because pgd_clear() is a no-op on i386.
*/
for (i = 0; i < USER_PTRS_PER_PGD; i++)
#ifdef CONFIG_X86_PAE
set_pgd(swapper_pg_dir+i, __pgd(1 + __pa(empty_zero_page)));
#else
set_pgd(swapper_pg_dir+i, __pgd(0));
#endif
flush_tlb_all();
}
static void __init clear_pgds(unsigned long start,
unsigned long end)
{
/*
* Remove references to kasan page tables from
* swapper_pg_dir. pgd_clear() can't be used
* here because it's nop on 2,3-level pagetable setups
*/
for (; start < end; start += PGDIR_SIZE)
set_pgd(pgd_offset_k(start), __pgd(0));
}
static pud_t *fill_pud(pgd_t *pgd, unsigned long vaddr)
{
if (pgd_none(*pgd)) {
pud_t *pud = (pud_t *)get_zeroed_page(GFP_ATOMIC);
set_pgd(pgd, __pgd(_PAGE_TABLE | __pa(pud)));
if (pud != pud_offset(pgd, 0))
printk(KERN_ERR "EFI PAGETABLE BUG #00! %p <-> %p\n",
pud, pud_offset(pgd, 0));
}
return pud_offset(pgd, vaddr);
}
static void __meminit init_trampoline_pud(void)
{
pud_t *pud_page_tramp, *pud, *pud_tramp;
p4d_t *p4d_page_tramp, *p4d, *p4d_tramp;
unsigned long paddr, vaddr;
pgd_t *pgd;
pud_page_tramp = alloc_low_page();
/*
* There are two mappings for the low 1MB area, the direct mapping
* and the 1:1 mapping for the real mode trampoline:
*
* Direct mapping: virt_addr = phys_addr + PAGE_OFFSET
* 1:1 mapping: virt_addr = phys_addr
*/
paddr = 0;
vaddr = (unsigned long)__va(paddr);
pgd = pgd_offset_k(vaddr);
p4d = p4d_offset(pgd, vaddr);
pud = pud_offset(p4d, vaddr);
pud_tramp = pud_page_tramp + pud_index(paddr);
*pud_tramp = *pud;
if (pgtable_l5_enabled()) {
p4d_page_tramp = alloc_low_page();
p4d_tramp = p4d_page_tramp + p4d_index(paddr);
set_p4d(p4d_tramp,
__p4d(_KERNPG_TABLE | __pa(pud_page_tramp)));
set_pgd(&trampoline_pgd_entry,
__pgd(_KERNPG_TABLE | __pa(p4d_page_tramp)));
} else {
set_pgd(&trampoline_pgd_entry,
__pgd(_KERNPG_TABLE | __pa(pud_page_tramp)));
}
}
static void __init kasan_map_early_shadow(pgd_t *pgd)
{
int i;
unsigned long start = KASAN_SHADOW_START;
unsigned long end = KASAN_SHADOW_END;
for (i = pgd_index(start); start < end; i++) {
pgd[i] = __pgd(__pa_nodebug(kasan_zero_pud)
| _KERNPG_TABLE);
start += PGDIR_SIZE;
}
}
/*
* Creates a middle page table and puts a pointer to it in the
* given global directory entry. This only returns the gd entry
* in non-PAE compilation mode, since the middle layer is folded.
*/
static pmd_t * __init one_md_table_init(pgd_t *pgd)
{
pud_t *pud;
pmd_t *pmd_table;
#ifdef CONFIG_X86_PAE
if (!(pgd_val(*pgd) & _PAGE_PRESENT)) {
pmd_table = (pmd_t *) alloc_bootmem_low_pages(PAGE_SIZE);
paravirt_alloc_pmd(&init_mm, __pa(pmd_table) >> PAGE_SHIFT);
set_pgd(pgd, __pgd(__pa(pmd_table) | _PAGE_PRESENT));
pud = pud_offset(pgd, 0);
BUG_ON(pmd_table != pmd_offset(pud, 0));
}
/*
* set up paging
*/
void __init paging_init(void)
{
unsigned long zones_size[MAX_NR_ZONES] = {0,};
pte_t *ppte;
int loop;
/* main kernel space -> RAM mapping is handled as 1:1 transparent by
* the MMU */
memset(swapper_pg_dir, 0, sizeof(swapper_pg_dir));
memset(kernel_vmalloc_ptes, 0, sizeof(kernel_vmalloc_ptes));
/* load the VMALLOC area PTE table addresses into the kernel PGD */
ppte = kernel_vmalloc_ptes;
for (loop = VMALLOC_START / (PAGE_SIZE * PTRS_PER_PTE);
loop < VMALLOC_END / (PAGE_SIZE * PTRS_PER_PTE);
loop++
) {
set_pgd(swapper_pg_dir + loop, __pgd(__pa(ppte) | _PAGE_TABLE));
ppte += PAGE_SIZE / sizeof(pte_t);
}
/* declare the sizes of the RAM zones (only use the normal zone) */
zones_size[ZONE_NORMAL] =
contig_page_data.bdata->node_low_pfn -
contig_page_data.bdata->node_min_pfn;
/* pass the memory from the bootmem allocator to the main allocator */
free_area_init(zones_size);
#ifdef CONFIG_MN10300_HAS_ATOMIC_OPS_UNIT
/* The Atomic Operation Unit registers need to be mapped to userspace
* for all processes. The following uses vm_area_register_early() to
* reserve the first page of the vmalloc area and sets the pte for that
* page.
*
* glibc hardcodes this virtual mapping, so we're pretty much stuck with
* it from now on.
*/
user_iomap_vm.flags = VM_USERMAP;
user_iomap_vm.size = 1 << PAGE_SHIFT;
vm_area_register_early(&user_iomap_vm, PAGE_SIZE);
ppte = kernel_vmalloc_ptes;
set_pte(ppte, pfn_pte(USER_ATOMIC_OPS_PAGE_ADDR >> PAGE_SHIFT,
PAGE_USERIO));
#endif
local_flush_tlb_all();
}
static void release_pgd(pgd_t *spgd)
{
/* If the entry's not present, there's nothing to release. */
if (pgd_flags(*spgd) & _PAGE_PRESENT) {
unsigned int i;
pmd_t *pmdpage = __va(pgd_pfn(*spgd) << PAGE_SHIFT);
for (i = 0; i < PTRS_PER_PMD; i++)
release_pmd(&pmdpage[i]);
/* Now we can free the page of PMDs */
free_page((long)pmdpage);
/* And zero out the PGD entry so we never release it twice. */
set_pgd(spgd, __pgd(0));
}
}
static void __init pagetable_init (void)
{
unsigned long vaddr;
pgd_t *pgd_base = swapper_pg_dir;
#ifdef CONFIG_X86_PAE
int i;
/* Init entries of the first-level page table to the zero page */
for (i = 0; i < PTRS_PER_PGD; i++)
set_pgd(pgd_base + i, __pgd(__pa(empty_zero_page) | _PAGE_PRESENT));
#endif
/* Enable PSE if available */
if (cpu_has_pse) {
set_in_cr4(X86_CR4_PSE);
}
/* Enable PGE if available */
if (cpu_has_pge) {
set_in_cr4(X86_CR4_PGE);
__PAGE_KERNEL |= _PAGE_GLOBAL;
__PAGE_KERNEL_EXEC |= _PAGE_GLOBAL;
}
kernel_physical_mapping_init(pgd_base);
remap_numa_kva();
/*
* Fixed mappings, only the page table structure has to be
* created - mappings will be set by set_fixmap():
*/
vaddr = __fix_to_virt(__end_of_fixed_addresses - 1) & PMD_MASK;
page_table_range_init(vaddr, 0, pgd_base);
permanent_kmaps_init(pgd_base);
#ifdef CONFIG_X86_PAE
/*
* Add low memory identity-mappings - SMP needs it when
* starting up on an AP from real-mode. In the non-PAE
* case we already have these mappings through head.S.
* All user-space mappings are explicitly cleared after
* SMP startup.
*/
set_pgd(&pgd_base[0], pgd_base[USER_PTRS_PER_PGD]);
#endif
}
static void shmedia_mapioaddr(unsigned long pa, unsigned long va,
unsigned long flags)
{
pgd_t *pgdp;
pud_t *pudp;
pmd_t *pmdp;
pte_t *ptep, pte;
pgprot_t prot;
pr_debug("shmedia_mapiopage pa %08lx va %08lx\n", pa, va);
if (!flags)
flags = 1; /* 1 = CB0-1 device */
pgdp = pgd_offset_k(va);
if (pgd_none(*pgdp) || !pgd_present(*pgdp)) {
pudp = (pud_t *)sh64_get_page();
set_pgd(pgdp, __pgd((unsigned long)pudp | _KERNPG_TABLE));
}
pudp = pud_offset(pgdp, va);
if (pud_none(*pudp) || !pud_present(*pudp)) {
pmdp = (pmd_t *)sh64_get_page();
set_pud(pudp, __pud((unsigned long)pmdp | _KERNPG_TABLE));
}
pmdp = pmd_offset(pudp, va);
if (pmd_none(*pmdp) || !pmd_present(*pmdp)) {
ptep = (pte_t *)sh64_get_page();
set_pmd(pmdp, __pmd((unsigned long)ptep + _PAGE_TABLE));
}
prot = __pgprot(_PAGE_PRESENT | _PAGE_READ | _PAGE_WRITE |
_PAGE_DIRTY | _PAGE_ACCESSED | _PAGE_SHARED | flags);
pte = pfn_pte(pa >> PAGE_SHIFT, prot);
ptep = pte_offset_kernel(pmdp, va);
if (!pte_none(*ptep) &&
pte_val(*ptep) != pte_val(pte))
pte_ERROR(*ptep);
set_pte(ptep, pte);
flush_tlb_kernel_range(va, PAGE_SIZE);
}
/*
* Creates a middle page table and puts a pointer to it in the
* given global directory entry. This only returns the gd entry
* in non-PAE compilation mode, since the middle layer is folded.
*/
static pmd_t * __init one_md_table_init(pgd_t *pgd)
{
pud_t *pud;
pmd_t *pmd_table;
#ifdef CONFIG_X86_PAE
pmd_table = (pmd_t *) alloc_bootmem_low_pages(PAGE_SIZE);
set_pgd(pgd, __pgd(__pa(pmd_table) | _PAGE_PRESENT));
pud = pud_offset(pgd, 0);
if (pmd_table != pmd_offset(pud, 0))
BUG();
#else
pud = pud_offset(pgd, 0);
pmd_table = pmd_offset(pud, 0);
#endif
return pmd_table;
}
static void machine_kexec_page_table_set_one(
pgd_t *pgd, pmd_t *pmd, pte_t *pte,
unsigned long vaddr, unsigned long paddr)
{
pud_t *pud;
pgd += pgd_index(vaddr);
#ifdef CONFIG_X86_PAE
if (!(pgd_val(*pgd) & _PAGE_PRESENT))
set_pgd(pgd, __pgd(__pa(pmd) | _PAGE_PRESENT));
#endif
pud = pud_offset(pgd, vaddr);
pmd = pmd_offset(pud, vaddr);
if (!(pmd_val(*pmd) & _PAGE_PRESENT))
set_pmd(pmd, __pmd(__pa(pte) | _PAGE_TABLE));
pte = pte_offset_kernel(pmd, vaddr);
set_pte(pte, pfn_pte(paddr >> PAGE_SHIFT, PAGE_KERNEL_EXEC));
}
/*
* The VSYSCALL page is the only user-accessible page in the kernel address
* range. Normally, the kernel page tables can have _PAGE_USER clear, but
* the tables covering VSYSCALL_ADDR need _PAGE_USER set if vsyscalls
* are enabled.
*
* Some day we may create a "minimal" vsyscall mode in which we emulate
* vsyscalls but leave the page not present. If so, we skip calling
* this.
*/
void __init set_vsyscall_pgtable_user_bits(pgd_t *root)
{
pgd_t *pgd;
p4d_t *p4d;
pud_t *pud;
pmd_t *pmd;
pgd = pgd_offset_pgd(root, VSYSCALL_ADDR);
set_pgd(pgd, __pgd(pgd_val(*pgd) | _PAGE_USER));
p4d = p4d_offset(pgd, VSYSCALL_ADDR);
#if CONFIG_PGTABLE_LEVELS >= 5
set_p4d(p4d, __p4d(p4d_val(*p4d) | _PAGE_USER));
#endif
pud = pud_offset(p4d, VSYSCALL_ADDR);
set_pud(pud, __pud(pud_val(*pud) | _PAGE_USER));
pmd = pmd_offset(pud, VSYSCALL_ADDR);
set_pmd(pmd, __pmd(pmd_val(*pmd) | _PAGE_USER));
}
/*H:480
* (vi) Mapping the Switcher when the Guest is about to run.
*
* The Switcher and the two pages for this CPU need to be visible in the
* Guest (and not the pages for other CPUs). We have the appropriate PTE pages
* for each CPU already set up, we just need to hook them in now we know which
* Guest is about to run on this CPU.
*/
void map_switcher_in_guest(struct lg_cpu *cpu, struct lguest_pages *pages)
{
pte_t *switcher_pte_page = __this_cpu_read(switcher_pte_pages);
pte_t regs_pte;
#ifdef CONFIG_X86_PAE
pmd_t switcher_pmd;
pmd_t *pmd_table;
switcher_pmd = pfn_pmd(__pa(switcher_pte_page) >> PAGE_SHIFT,
PAGE_KERNEL_EXEC);
/* Figure out where the pmd page is, by reading the PGD, and converting
* it to a virtual address. */
pmd_table = __va(pgd_pfn(cpu->lg->
pgdirs[cpu->cpu_pgd].pgdir[SWITCHER_PGD_INDEX])
<< PAGE_SHIFT);
/* Now write it into the shadow page table. */
set_pmd(&pmd_table[SWITCHER_PMD_INDEX], switcher_pmd);
#else
pgd_t switcher_pgd;
/*
* Make the last PGD entry for this Guest point to the Switcher's PTE
* page for this CPU (with appropriate flags).
*/
switcher_pgd = __pgd(__pa(switcher_pte_page) | __PAGE_KERNEL_EXEC);
cpu->lg->pgdirs[cpu->cpu_pgd].pgdir[SWITCHER_PGD_INDEX] = switcher_pgd;
#endif
/*
* We also change the Switcher PTE page. When we're running the Guest,
* we want the Guest's "regs" page to appear where the first Switcher
* page for this CPU is. This is an optimization: when the Switcher
* saves the Guest registers, it saves them into the first page of this
* CPU's "struct lguest_pages": if we make sure the Guest's register
* page is already mapped there, we don't have to copy them out
* again.
*/
regs_pte = pfn_pte(__pa(cpu->regs_page) >> PAGE_SHIFT, PAGE_KERNEL);
set_pte(&switcher_pte_page[pte_index((unsigned long)pages)], regs_pte);
}
/*H:450
* If we chase down the release_pgd() code, the non-PAE version looks like
* this. The PAE version is almost identical, but instead of calling
* release_pte it calls release_pmd(), which looks much like this.
*/
static void release_pgd(pgd_t *spgd)
{
/* If the entry's not present, there's nothing to release. */
if (pgd_flags(*spgd) & _PAGE_PRESENT) {
unsigned int i;
/*
* Converting the pfn to find the actual PTE page is easy: turn
* the page number into a physical address, then convert to a
* virtual address (easy for kernel pages like this one).
*/
pte_t *ptepage = __va(pgd_pfn(*spgd) << PAGE_SHIFT);
/* For each entry in the page, we might need to release it. */
for (i = 0; i < PTRS_PER_PTE; i++)
release_pte(ptepage[i]);
/* Now we can free the page of PTEs */
free_page((long)ptepage);
/* And zero out the PGD entry so we never release it twice. */
*spgd = __pgd(0);
}
}
/*
* Create a middle page table on a resume-safe page and put a pointer to it in
* the given global directory entry. This only returns the gd entry
* in non-PAE compilation mode, since the middle layer is folded.
*/
static pmd_t *resume_one_md_table_init(pgd_t *pgd)
{
pud_t *pud;
pmd_t *pmd_table;
#ifdef CONFIG_X86_PAE
pmd_table = (pmd_t *)get_safe_page(GFP_ATOMIC);
if (!pmd_table)
return NULL;
set_pgd(pgd, __pgd(__pa(pmd_table) | _PAGE_PRESENT));
pud = pud_offset(pgd, 0);
BUG_ON(pmd_table != pmd_offset(pud, 0));
#else
pud = pud_offset(pgd, 0);
pmd_table = pmd_offset(pud, 0);
#endif
return pmd_table;
}
void __init init_espfix_bsp(void)
{
pgd_t *pgd_p;
pud_t *pud_p;
unsigned long index = pgd_index(ESPFIX_BASE_ADDR);
/* Install the espfix pud into the kernel page directory */
pgd_p = &init_level4_pgt[index];
pud_p = espfix_pud_page;
paravirt_alloc_pud(&init_mm, __pa(pud_p) >> PAGE_SHIFT);
set_pgd(pgd_p, __pgd(PGTABLE_PROT | __pa(pud_p)));
#ifdef CONFIG_PAX_PER_CPU_PGD
clone_pgd_range(get_cpu_pgd(0, kernel) + index, swapper_pg_dir + index, 1);
clone_pgd_range(get_cpu_pgd(0, user) + index, swapper_pg_dir + index, 1);
#endif
/* Randomize the locations */
init_espfix_random();
/* The rest is the same as for any other processor */
init_espfix_ap(0);
}
void __init
paging_init(void)
{
int i;
unsigned long zones_size[MAX_NR_ZONES];
printk("Setting up paging and the MMU.\n");
/* Clear out the init_mm.pgd that will contain the kernel's mappings. */
for(i = 0; i < PTRS_PER_PGD; i++)
swapper_pg_dir[i] = __pgd(0);
cris_mmu_init();
/*
* Initialize the bad page table and bad page to point to a couple of
* allocated pages.
*/
empty_zero_page = (unsigned long) alloc_bootmem_pages(PAGE_SIZE);
memset((void *) empty_zero_page, 0, PAGE_SIZE);
/* All pages are DMA'able in Etrax, so put all in the DMA'able zone. */
zones_size[0] = ((unsigned long) high_memory - PAGE_OFFSET) >> PAGE_SHIFT;
for (i = 1; i < MAX_NR_ZONES; i++)
zones_size[i] = 0;
/*
* Use free_area_init_node instead of free_area_init, because it is
* designed for systems where the DRAM starts at an address
* substantially higher than 0, like us (we start at PAGE_OFFSET). This
* saves space in the mem_map page array.
*/
free_area_init_node(0, zones_size, PAGE_OFFSET >> PAGE_SHIFT, 0);
mem_map = contig_page_data.node_mem_map;
}