static int __init dell_efi_quirk(const struct dmi_system_id *d)
{
u64 vaddr;
pgd_t *pgd;
pud_t *pud;
pmd_t *pmd;
pte_t *pte;
/*
* Some UEFI run time implementations (DELL) require physical page
* zero to be mapped. This location is used during EfiResetSystem
* when ResetType is EfiResetWarm (reboot=warm). UEFI writes to
* a BIOS physical address of 0x472 for the reboot mode. The reason
* for this hasn't been revealed by the UEFI developers.
*/
printk(KERN_INFO
"%s series board detected. Applying quirk for"
" page 0 UEFI firmware access.\n", d->ident);
vaddr = 0UL;
pgd = efi_pgd + pgd_index(vaddr);
pud = fill_pud(pgd, vaddr);
pmd = fill_pmd(pud, vaddr);
pte = fill_pte(pmd, vaddr);
set_pte(pte, pfn_pte(0UL, PAGE_KERNEL));
return 0;
}
/*
* Associate a large virtual page frame with a given physical page frame
* and protection flags for that frame. pfn is for the base of the page,
* vaddr is what the page gets mapped to - both must be properly aligned.
* The pmd must already be instantiated. Assumes PAE mode.
*/
void set_pmd_pfn(unsigned long vaddr, unsigned long pfn, pgprot_t flags)
{
pgd_t *pgd;
pud_t *pud;
pmd_t *pmd;
if (vaddr & (PMD_SIZE-1)) { /* vaddr is misaligned */
printk(KERN_WARNING "set_pmd_pfn: vaddr misaligned\n");
return; /* BUG(); */
}
if (pfn & (PTRS_PER_PTE-1)) { /* pfn is misaligned */
printk(KERN_WARNING "set_pmd_pfn: pfn misaligned\n");
return; /* BUG(); */
}
pgd = swapper_pg_dir + pgd_index(vaddr);
if (pgd_none(*pgd)) {
printk(KERN_WARNING "set_pmd_pfn: pgd_none\n");
return; /* BUG(); */
}
pud = pud_offset(pgd, vaddr);
pmd = pmd_offset(pud, vaddr);
set_pmd(pmd, pfn_pmd(pfn, flags));
/*
* It's enough to flush this one mapping.
* (PGE mappings get flushed as well)
*/
__flush_tlb_one(vaddr);
}
static inline pmd_t *vmalloc_sync_one(pgd_t *pgd, unsigned long address)
{
unsigned index = pgd_index(address);
pgd_t *pgd_k;
pud_t *pud, *pud_k;
pmd_t *pmd, *pmd_k;
pgd += index;
pgd_k = init_mm.pgd + index;
if (!pgd_present(*pgd_k))
return NULL;
pud = pud_offset(pgd, address);
pud_k = pud_offset(pgd_k, address);
if (!pud_present(*pud_k))
return NULL;
pmd = pmd_offset(pud, address);
pmd_k = pmd_offset(pud_k, address);
if (!pmd_present(*pmd_k))
return NULL;
if (!pmd_present(*pmd))
set_pmd(pmd, *pmd_k);
else
BUG_ON(pmd_ptfn(*pmd) != pmd_ptfn(*pmd_k));
return pmd_k;
}
/*
* Associate a virtual page frame with a given physical page frame
* and protection flags for that frame.
*/
static void set_pte_pfn_ma(unsigned long vaddr, unsigned long pfn,
pgprot_t flags)
{
pgd_t *pgd;
pud_t *pud;
pmd_t *pmd;
pte_t *pte;
pgd = swapper_pg_dir + pgd_index(vaddr);
if (pgd_none(*pgd)) {
BUG();
return;
}
pud = pud_offset(pgd, vaddr);
if (pud_none(*pud)) {
BUG();
return;
}
pmd = pmd_offset(pud, vaddr);
if (pmd_none(*pmd)) {
BUG();
return;
}
pte = pte_offset_kernel(pmd, vaddr);
/* <pfn,flags> stored as-is, to permit clearing entries */
set_pte(pte, pfn_pte_ma(pfn, flags));
/*
* It's enough to flush this one mapping.
* (PGE mappings get flushed as well)
*/
__flush_tlb_one(vaddr);
}
/**
* resume_map_numa_kva - add KVA mapping to the temporary page tables created
* during resume from hibernation
* @pgd_base - temporary resume page directory
*/
void resume_map_numa_kva(pgd_t *pgd_base)
{
int node;
for_each_online_node(node) {
unsigned long start_va, start_pfn, nr_pages, pfn;
start_va = (unsigned long)node_remap_start_vaddr[node];
start_pfn = node_remap_start_pfn[node];
nr_pages = (node_remap_end_vaddr[node] -
node_remap_start_vaddr[node]) >> PAGE_SHIFT;
printk(KERN_DEBUG "%s: node %d\n", __func__, node);
for (pfn = 0; pfn < nr_pages; pfn += PTRS_PER_PTE) {
unsigned long vaddr = start_va + (pfn << PAGE_SHIFT);
pgd_t *pgd = pgd_base + pgd_index(vaddr);
pud_t *pud = pud_offset(pgd, vaddr);
pmd_t *pmd = pmd_offset(pud, vaddr);
set_pmd(pmd, pfn_pmd(start_pfn + pfn,
PAGE_KERNEL_LARGE_EXEC));
printk(KERN_DEBUG "%s: %08lx -> pfn %08lx\n",
__func__, vaddr, start_pfn + pfn);
}
}
}
void
handle_mmu_bus_fault(struct pt_regs *regs)
{
int cause;
int select;
#ifdef DEBUG
int index;
int page_id;
int acc, inv;
#endif
pgd_t* pgd = (pgd_t*)per_cpu(current_pgd, smp_processor_id());
pmd_t *pmd;
pte_t pte;
int miss, we, writeac;
unsigned long address;
unsigned long flags;
cause = *R_MMU_CAUSE;
address = cause & PAGE_MASK; /* get faulting address */
select = *R_TLB_SELECT;
#ifdef DEBUG
page_id = IO_EXTRACT(R_MMU_CAUSE, page_id, cause);
acc = IO_EXTRACT(R_MMU_CAUSE, acc_excp, cause);
inv = IO_EXTRACT(R_MMU_CAUSE, inv_excp, cause);
index = IO_EXTRACT(R_TLB_SELECT, index, select);
#endif
miss = IO_EXTRACT(R_MMU_CAUSE, miss_excp, cause);
we = IO_EXTRACT(R_MMU_CAUSE, we_excp, cause);
writeac = IO_EXTRACT(R_MMU_CAUSE, wr_rd, cause);
D(printk("bus_fault from IRP 0x%lx: addr 0x%lx, miss %d, inv %d, we %d, acc %d, dx %d pid %d\n",
regs->irp, address, miss, inv, we, acc, index, page_id));
/* leave it to the MM system fault handler */
if (miss)
do_page_fault(address, regs, 0, writeac);
else
do_page_fault(address, regs, 1, we);
/* Reload TLB with new entry to avoid an extra miss exception.
* do_page_fault may have flushed the TLB so we have to restore
* the MMU registers.
*/
local_save_flags(flags);
local_irq_disable();
pmd = (pmd_t *)(pgd + pgd_index(address));
if (pmd_none(*pmd))
goto exit;
pte = *pte_offset_kernel(pmd, address);
if (!pte_present(pte))
goto exit;
*R_TLB_SELECT = select;
*R_TLB_HI = cause;
*R_TLB_LO = pte_val(pte);
exit:
local_irq_restore(flags);
}
void pgd_free(struct mm_struct *mm, pgd_t *pgd_base)
{
pgd_t *pgd;
pud_t *pud;
pmd_t *pmd;
pgtable_t pte;
if (!pgd_base)
return;
pgd = pgd_base + pgd_index(0);
if (pgd_none_or_clear_bad(pgd))
goto no_pgd;
pud = pud_offset(pgd, 0);
if (pud_none_or_clear_bad(pud))
goto no_pud;
pmd = pmd_offset(pud, 0);
if (pmd_none_or_clear_bad(pmd))
goto no_pmd;
pte = pmd_pgtable(*pmd);
pmd_clear(pmd);
pte_free(mm, pte);
no_pmd:
pud_clear(pud);
pmd_free(mm, pmd);
no_pud:
pgd_clear(pgd);
pud_free(mm, pud);
no_pgd:
#if defined(CONFIG_SYNO_ARMADA_ARCH)
#ifdef CONFIG_ARM_LPAE
/*
* Free modules/pkmap or identity pmd tables.
*/
for (pgd = pgd_base; pgd < pgd_base + PTRS_PER_PGD; pgd++) {
if (pgd_none_or_clear_bad(pgd))
continue;
if (pgd_val(*pgd) & L_PGD_SWAPPER)
continue;
pud = pud_offset(pgd, 0);
if (pud_none_or_clear_bad(pud))
continue;
pmd = pmd_offset(pud, 0);
pud_clear(pud);
pmd_free(mm, pmd);
pgd_clear(pgd);
pud_free(mm, pud);
}
#endif
__pgd_free(pgd_base);
#elif defined(CONFIG_SYNO_COMCERTO)
free_pages((unsigned long) pgd_base, get_order(16384));
#else
free_pages((unsigned long) pgd_base, 2);
#endif
}
unsigned long virtaddr_to_physaddr(struct mm_struct *mm, unsigned long vaddr)
{
pgd_t *pgd;
pud_t *pud;
pmd_t *pmd;
pte_t *pte;
unsigned long paddr = 0;
pgd = pgd_offset(mm, vaddr);
printk("pgd_val = 0x%lx\n", pgd_val(*pgd));
printk("pgd_index = %lu\n", pgd_index(vaddr));
if (pgd_none(*pgd)) {
printk("not mapped in pgd\n");
return INVALID_ADDR;
}
pud = pud_offset(pgd, vaddr);
printk("pud_val = 0x%lx\n", pud_val(*pud));
printk("pud_index = %lu\n", pud_index(vaddr));
if (pud_none(*pud)) {
printk("not mapped in pud\n");
return INVALID_ADDR;
}
pmd = pmd_offset(pud, vaddr);
printk("pmd_val = 0x%lx\n", pmd_val(*pmd));
printk("pmd_index = %lx\n", pmd_index(vaddr));
if(pmd_none(*pmd)){
printk("not mapped in pmd\n");
return INVALID_ADDR;
}
/*If pmd_large is true, represent pmd is the last level*/
if(pmd_large(*pmd)){
paddr = (pmd_val(*pmd) & PAGE_MASK);
paddr = paddr | (vaddr & ~PAGE_MASK);
return paddr;
}
/*Walk the forth level page table
** you may use PAGE_MASK = 0xfffffffffffff000 to help you get [0:11] bits
***/
else{
/* XXX: Need to implement */
pte = pte_offset_kernel(pmd, vaddr);
printk("pte_val = 0x%lx\n", pte_val(*pte));
printk("pte_index = %lx\n", pte_index(vaddr));
if(pte_none(*pte)){
printk("not mapped in pte\n");
return INVALID_ADDR;
}
paddr = (pte_val(*pte) & PAGE_MASK);
paddr = paddr | (vaddr & ~PAGE_MASK);
printk("paddr = %lx\n", paddr);
printk("__pa = %lx\n", __pa(vaddr)); /* magic macro in the kernel */
/* End of implement */
return paddr;
}
}
void pgd_free(struct mm_struct *mm, pgd_t *pgd_base)
{
pgd_t *pgd;
pud_t *pud;
pmd_t *pmd;
pgtable_t pte;
if (!pgd_base)
return;
pgd = pgd_base + pgd_index(0);
if (pgd_none_or_clear_bad(pgd))
goto no_pgd;
pud = pud_offset(pgd, 0);
if (pud_none_or_clear_bad(pud))
goto no_pud;
pmd = pmd_offset(pud, 0);
if (pmd_none_or_clear_bad(pmd))
goto no_pmd;
pte = pmd_pgtable(*pmd);
pmd_clear(pmd);
pte_free(mm, pte);
mm_dec_nr_ptes(mm);
no_pmd:
pud_clear(pud);
pmd_free(mm, pmd);
mm_dec_nr_pmds(mm);
no_pud:
pgd_clear(pgd);
pud_free(mm, pud);
no_pgd:
#ifdef CONFIG_ARM_LPAE
/*
* Free modules/pkmap or identity pmd tables.
*/
for (pgd = pgd_base; pgd < pgd_base + PTRS_PER_PGD; pgd++) {
if (pgd_none_or_clear_bad(pgd))
continue;
if (pgd_val(*pgd) & L_PGD_SWAPPER)
continue;
pud = pud_offset(pgd, 0);
if (pud_none_or_clear_bad(pud))
continue;
pmd = pmd_offset(pud, 0);
pud_clear(pud);
pmd_free(mm, pmd);
mm_dec_nr_pmds(mm);
pgd_clear(pgd);
pud_free(mm, pud);
}
#endif
__pgd_free(pgd_base);
}
/*
* for Xen extraction
*/
unsigned long long
kvtop_xen_x86_64(unsigned long kvaddr)
{
unsigned long long dirp, entry;
if (!is_xen_vaddr(kvaddr))
return NOT_PADDR;
if (is_xen_text(kvaddr))
return (unsigned long)kvaddr - XEN_VIRT_START + info->xen_phys_start;
if (is_direct(kvaddr))
return (unsigned long)kvaddr - DIRECTMAP_VIRT_START;
if ((dirp = kvtop_xen_x86_64(SYMBOL(pgd_l4))) == NOT_PADDR)
return NOT_PADDR;
dirp += pml4_index(kvaddr) * sizeof(unsigned long long);
if (!readmem(MADDR_XEN, dirp, &entry, sizeof(entry)))
return NOT_PADDR;
if (!(entry & _PAGE_PRESENT))
return NOT_PADDR;
dirp = entry & ENTRY_MASK;
dirp += pgd_index(kvaddr) * sizeof(unsigned long long);
if (!readmem(MADDR_XEN, dirp, &entry, sizeof(entry)))
return NOT_PADDR;
if (!(entry & _PAGE_PRESENT))
return NOT_PADDR;
dirp = entry & ENTRY_MASK;
dirp += pmd_index(kvaddr) * sizeof(unsigned long long);
if (!readmem(MADDR_XEN, dirp, &entry, sizeof(entry)))
return NOT_PADDR;
if (!(entry & _PAGE_PRESENT))
return NOT_PADDR;
if (entry & _PAGE_PSE) {
entry = (entry & ENTRY_MASK) + (kvaddr & ((1UL << PMD_SHIFT) - 1));
return entry;
}
dirp = entry & ENTRY_MASK;
dirp += pte_index(kvaddr) * sizeof(unsigned long long);
if (!readmem(MADDR_XEN, dirp, &entry, sizeof(entry)))
return NOT_PADDR;
if (!(entry & _PAGE_PRESENT)) {
return NOT_PADDR;
}
entry = (entry & ENTRY_MASK) + (kvaddr & ((1UL << PTE_SHIFT) - 1));
return entry;
}
void identity_mapping_del(pgd_t *pgd, unsigned long addr, unsigned long end)
{
unsigned long next;
pgd += pgd_index(addr);
do {
next = pgd_addr_end(addr, end);
idmap_del_pud(pgd, addr, next);
} while (pgd++, addr = next, addr != end);
}
static int __filemap_sync(struct vm_area_struct *vma, unsigned long address,
size_t size, unsigned int flags)
{
pgd_t *pgd;
unsigned long end = address + size;
unsigned long next;
int i;
int error = 0;
/* Aquire the lock early; it may be possible to avoid dropping
* and reaquiring it repeatedly.
*/
spin_lock(&vma->vm_mm->page_table_lock);
pgd = pgd_offset(vma->vm_mm, address);
flush_cache_range(vma, address, end);
/* For hugepages we can't go walking the page table normally,
* but that's ok, hugetlbfs is memory based, so we don't need
* to do anything more on an msync() */
if (is_vm_hugetlb_page(vma))
goto out;
if (address >= end)
BUG();
for (i = pgd_index(address); i <= pgd_index(end-1); i++) {
next = (address + PGDIR_SIZE) & PGDIR_MASK;
if (next <= address || next > end)
next = end;
error |= filemap_sync_pud_range(pgd, address, next, vma, flags);
address = next;
pgd++;
}
/*
* Why flush ? filemap_sync_pte already flushed the tlbs with the
* dirty bits.
*/
flush_tlb_range(vma, end - size, end);
out:
spin_unlock(&vma->vm_mm->page_table_lock);
return error;
}
/* Copy pages non-empty in from, and empty in to */
void
copy_pgtbl_range_nonzero(paddr_t pt_to, paddr_t pt_from,
unsigned long lower_addr, unsigned long size)
{
pgd_t *tpgd = ((pgd_t *)chal_pa2va((void*)pt_to)) + pgd_index(lower_addr);
pgd_t *fpgd = ((pgd_t *)chal_pa2va((void*)pt_from)) + pgd_index(lower_addr);
unsigned int span = hpage_index(size);
int i;
printk("Copying from %p:%d to %p.\n", fpgd, span, tpgd);
/* sizeof(pgd entry) is intended */
for (i = 0 ; i < span ; i++) {
if (!(pgd_val(tpgd[i]) & _PAGE_PRESENT)) {
if (pgd_val(fpgd[i]) & _PAGE_PRESENT) printk("\tcopying vaddr %lx.\n", lower_addr + i * HPAGE_SHIFT);
memcpy(&tpgd[i], &fpgd[i], sizeof(pgd_t));
}
}
}
/* Create a new PMD entry */
int __init early_make_pgtable(unsigned long address)
{
unsigned long physaddr = address - __PAGE_OFFSET;
unsigned long i;
pgdval_t pgd, *pgd_p;
pudval_t pud, *pud_p;
pmdval_t pmd, *pmd_p;
/* Invalid address or early pgt is done ? */
if (physaddr >= MAXMEM || read_cr3() != __pa(early_level4_pgt))
return -1;
again:
pgd_p = &early_level4_pgt[pgd_index(address)].pgd;
pgd = *pgd_p;
/*
* The use of __START_KERNEL_map rather than __PAGE_OFFSET here is
* critical -- __PAGE_OFFSET would point us back into the dynamic
* range and we might end up looping forever...
*/
if (pgd)
pud_p = (pudval_t *)((pgd & PTE_PFN_MASK) + __START_KERNEL_map - phys_base);
else {
if (next_early_pgt >= EARLY_DYNAMIC_PAGE_TABLES) {
reset_early_page_tables();
goto again;
}
pud_p = (pudval_t *)early_dynamic_pgts[next_early_pgt++];
for (i = 0; i < PTRS_PER_PUD; i++)
pud_p[i] = 0;
*pgd_p = (pgdval_t)pud_p - __START_KERNEL_map + phys_base + _KERNPG_TABLE;
}
pud_p += pud_index(address);
pud = *pud_p;
if (pud)
pmd_p = (pmdval_t *)((pud & PTE_PFN_MASK) + __START_KERNEL_map - phys_base);
else {
if (next_early_pgt >= EARLY_DYNAMIC_PAGE_TABLES) {
reset_early_page_tables();
goto again;
}
pmd_p = (pmdval_t *)early_dynamic_pgts[next_early_pgt++];
for (i = 0; i < PTRS_PER_PMD; i++)
pmd_p[i] = 0;
*pud_p = (pudval_t)pmd_p - __START_KERNEL_map + phys_base + _KERNPG_TABLE;
}
pmd = (physaddr & PMD_MASK) + early_pmd_flags;
pmd_p[pmd_index(address)] = pmd;
return 0;
}
static void unmap_range(struct kvm *kvm, pgd_t *pgdp,
unsigned long long start, u64 size)
{
pgd_t *pgd;
pud_t *pud;
pmd_t *pmd;
pte_t *pte;
unsigned long long addr = start, end = start + size;
u64 next;
while (addr < end) {
pgd = pgdp + pgd_index(addr);
pud = pud_offset(pgd, addr);
if (pud_none(*pud)) {
addr = kvm_pud_addr_end(addr, end);
continue;
}
if (pud_huge(*pud)) {
/*
* If we are dealing with a huge pud, just clear it and
* move on.
*/
clear_pud_entry(kvm, pud, addr);
addr = kvm_pud_addr_end(addr, end);
continue;
}
pmd = pmd_offset(pud, addr);
if (pmd_none(*pmd)) {
addr = kvm_pmd_addr_end(addr, end);
continue;
}
if (!kvm_pmd_huge(*pmd)) {
pte = pte_offset_kernel(pmd, addr);
clear_pte_entry(kvm, pte, addr);
next = addr + PAGE_SIZE;
}
/*
* If the pmd entry is to be cleared, walk back up the ladder
*/
if (kvm_pmd_huge(*pmd) || page_empty(pte)) {
clear_pmd_entry(kvm, pmd, addr);
next = kvm_pmd_addr_end(addr, end);
if (page_empty(pmd) && !page_empty(pud)) {
clear_pud_entry(kvm, pud, addr);
next = kvm_pud_addr_end(addr, end);
}
}
addr = next;
}
}
void
handle_mmu_bus_fault(struct pt_regs *regs)
{
int cause;
int select;
#ifdef DEBUG
int index;
int page_id;
int acc, inv;
#endif
pgd_t* pgd = (pgd_t*)per_cpu(current_pgd, smp_processor_id());
pmd_t *pmd;
pte_t pte;
int miss, we, writeac;
unsigned long address;
unsigned long flags;
cause = *R_MMU_CAUSE;
address = cause & PAGE_MASK;
select = *R_TLB_SELECT;
#ifdef DEBUG
page_id = IO_EXTRACT(R_MMU_CAUSE, page_id, cause);
acc = IO_EXTRACT(R_MMU_CAUSE, acc_excp, cause);
inv = IO_EXTRACT(R_MMU_CAUSE, inv_excp, cause);
index = IO_EXTRACT(R_TLB_SELECT, index, select);
#endif
miss = IO_EXTRACT(R_MMU_CAUSE, miss_excp, cause);
we = IO_EXTRACT(R_MMU_CAUSE, we_excp, cause);
writeac = IO_EXTRACT(R_MMU_CAUSE, wr_rd, cause);
D(printk("bus_fault from IRP 0x%lx: addr 0x%lx, miss %d, inv %d, we %d, acc %d, dx %d pid %d\n",
regs->irp, address, miss, inv, we, acc, index, page_id));
if (miss)
do_page_fault(address, regs, 0, writeac);
else
do_page_fault(address, regs, 1, we);
local_irq_save(flags);
pmd = (pmd_t *)(pgd + pgd_index(address));
if (pmd_none(*pmd))
goto exit;
pte = *pte_offset_kernel(pmd, address);
if (!pte_present(pte))
goto exit;
*R_TLB_SELECT = select;
*R_TLB_HI = cause;
*R_TLB_LO = pte_val(pte);
exit:
local_irq_restore(flags);
}
void
pgtbl_print_path(paddr_t pgtbl, unsigned long addr)
{
pgd_t *pt = ((pgd_t *)chal_pa2va((void*)pgtbl)) + pgd_index(addr);
pte_t *pe = pgtbl_lookup_address(pgtbl, addr);
printk("cos: addr %x, pgd entry - %x, pte entry - %x\n",
(unsigned int)addr, (unsigned int)pgd_val(*pt), (unsigned int)pte_val(*pe));
return;
}
int
chal_pgtbl_rem_middledir(paddr_t pt, unsigned long vaddr)
{
pgd_t *pgd = ((pgd_t *)chal_pa2va((void*)pt)) + pgd_index(vaddr);
unsigned long *page;
page = (unsigned long *)chal_pa2va((void*)(pgd->pgd & PTE_PFN_MASK));
pgd->pgd = 0;
chal_free_page(page);
return 0;
}
/*
* This maps the physical memory to kernel virtual address space, a total
* of max_low_pfn pages, by creating page tables starting from address
* PAGE_OFFSET. The page tables are allocated out of resume-safe pages.
*/
static int resume_physical_mapping_init(pgd_t *pgd_base)
{
unsigned long pfn;
pgd_t *pgd;
pmd_t *pmd;
pte_t *pte;
int pgd_idx, pmd_idx;
pgd_idx = pgd_index(PAGE_OFFSET);
pgd = pgd_base + pgd_idx;
pfn = 0;
for (; pgd_idx < PTRS_PER_PGD; pgd++, pgd_idx++) {
pmd = resume_one_md_table_init(pgd);
if (!pmd)
return -ENOMEM;
if (pfn >= max_low_pfn)
continue;
for (pmd_idx = 0; pmd_idx < PTRS_PER_PMD; pmd++, pmd_idx++) {
if (pfn >= max_low_pfn)
break;
/* Map with big pages if possible, otherwise create
* normal page tables.
* NOTE: We can mark everything as executable here
*/
if (cpu_has_pse) {
set_pmd(pmd, pfn_pmd(pfn, PAGE_KERNEL_LARGE_EXEC));
pfn += PTRS_PER_PTE;
} else {
pte_t *max_pte;
pte = resume_one_page_table_init(pmd);
if (!pte)
return -ENOMEM;
max_pte = pte + PTRS_PER_PTE;
for (; pte < max_pte; pte++, pfn++) {
if (pfn >= max_low_pfn)
break;
set_pte(pte, pfn_pte(pfn, PAGE_KERNEL_EXEC));
}
}
}
}
resume_map_numa_kva(pgd_base);
return 0;
}
/* allocate and link in a page middle directory */
int
chal_pgtbl_add_middledir(paddr_t pt, unsigned long vaddr)
{
pgd_t *pgd = ((pgd_t *)chal_pa2va((void*)pt)) + pgd_index(vaddr);
unsigned long *page;
page = chal_alloc_page(); /* zeroed */
if (!page) return -1;
pgd->pgd = (unsigned long)chal_va2pa(page) | _PAGE_PRESENT | _PAGE_RW | _PAGE_USER | _PAGE_ACCESSED;
return 0;
}
static void __init kasan_map_early_shadow(pgd_t *pgd)
{
int i;
unsigned long start = KASAN_SHADOW_START;
unsigned long end = KASAN_SHADOW_END;
for (i = pgd_index(start); start < end; i++) {
pgd[i] = __pgd(__pa_nodebug(kasan_zero_pud)
| _KERNPG_TABLE);
start += PGDIR_SIZE;
}
}
static void unmap_range(struct kvm *kvm, pgd_t *pgdp,
phys_addr_t start, u64 size)
{
pgd_t *pgd;
phys_addr_t addr = start, end = start + size;
phys_addr_t next;
pgd = pgdp + pgd_index(addr);
do {
next = kvm_pgd_addr_end(addr, end);
unmap_puds(kvm, pgd, addr, next);
} while (pgd++, addr = next, addr != end);
}
static void __init kasan_map_early_shadow(pgd_t *pgd)
{
/* See comment in kasan_init() */
unsigned long addr = KASAN_SHADOW_START & PGDIR_MASK;
unsigned long end = KASAN_SHADOW_END;
unsigned long next;
pgd += pgd_index(addr);
do {
next = pgd_addr_end(addr, end);
kasan_early_p4d_populate(pgd, addr, next);
} while (pgd++, addr = next, addr != end);
}
void __init init_espfix_bsp(void)
{
pgd_t *pgd_p;
pteval_t ptemask;
ptemask = __supported_pte_mask;
/* Install the espfix pud into the kernel page directory */
pgd_p = &init_level4_pgt[pgd_index(ESPFIX_BASE_ADDR)];
pgd_populate(&init_mm, pgd_p, (pud_t *)espfix_pud_page);
/* Randomize the locations */
init_espfix_random();
/* The rest is the same as for any other processor */
init_espfix_ap();
}
void pgd_free(struct mm_struct *mm, pgd_t *pgd_base)
{
unsigned long flags;
pgd_t *pgd;
pud_t *pud;
pmd_t *pmd;
pgtable_t pte;
if (!pgd_base)
return;
pgd = pgd_base + pgd_index(0);
if (pgd_none_or_clear_bad(pgd))
goto no_pgd;
pud = pud_offset(pgd + pgd_index(fcse_va_to_mva(mm, 0)), 0);
if (pud_none_or_clear_bad(pud))
goto no_pud;
pmd = pmd_offset(pud, 0);
if (pmd_none_or_clear_bad(pmd))
goto no_pmd;
pte = pmd_pgtable(*pmd);
pmd_clear(pmd);
pte_free(mm, pte);
no_pmd:
pud_clear(pud);
pmd_free(mm, pmd);
no_pud:
pgd_clear(pgd);
pud_free(mm, pud);
no_pgd:
pgd_list_lock(flags);
pgd_list_del(pgd);
pgd_list_unlock(flags);
free_pages((unsigned long) pgd_base, 2);
}
void vmalloc_sync_all(void)
{
#ifdef __tilegx__
/* Currently all L1 kernel pmd's are static and shared. */
BUILD_BUG_ON(pgd_index(VMALLOC_END - PAGE_SIZE) !=
pgd_index(VMALLOC_START));
#else
/*
* Note that races in the updates of insync and start aren't
* problematic: insync can only get set bits added, and updates to
* start are only improving performance (without affecting correctness
* if undone).
*/
static DECLARE_BITMAP(insync, PTRS_PER_PGD);
static unsigned long start = PAGE_OFFSET;
unsigned long address;
BUILD_BUG_ON(PAGE_OFFSET & ~PGDIR_MASK);
for (address = start; address >= PAGE_OFFSET; address += PGDIR_SIZE) {
if (!test_bit(pgd_index(address), insync)) {
unsigned long flags;
struct list_head *pos;
spin_lock_irqsave(&pgd_lock, flags);
list_for_each(pos, &pgd_list)
if (!vmalloc_sync_one(list_to_pgd(pos),
address)) {
/* Must be at first entry in list. */
BUG_ON(pos != pgd_list.next);
break;
}
spin_unlock_irqrestore(&pgd_lock, flags);
if (pos != pgd_list.next)
set_bit(pgd_index(address), insync);
}
if (address == start && test_bit(pgd_index(address), insync))
start = address + PGDIR_SIZE;
}
/*H:320
* The page table code is curly enough to need helper functions to keep it
* clear and clean. The kernel itself provides many of them; one advantage
* of insisting that the Guest and Host use the same CONFIG_PAE setting.
*
* There are two functions which return pointers to the shadow (aka "real")
* page tables.
*
* spgd_addr() takes the virtual address and returns a pointer to the top-level
* page directory entry (PGD) for that address. Since we keep track of several
* page tables, the "i" argument tells us which one we're interested in (it's
* usually the current one).
*/
static pgd_t *spgd_addr(struct lg_cpu *cpu, u32 i, unsigned long vaddr)
{
unsigned int index = pgd_index(vaddr);
#ifndef CONFIG_X86_PAE
/* We kill any Guest trying to touch the Switcher addresses. */
if (index >= SWITCHER_PGD_INDEX) {
kill_guest(cpu, "attempt to access switcher pages");
index = 0;
}
#endif
/* Return a pointer index'th pgd entry for the i'th page table. */
return &cpu->lg->pgdirs[i].pgdir[index];
}
void
chal_pgtbl_zero_range(paddr_t pt, unsigned long lower_addr, unsigned long size)
{
pgd_t *pgd = ((pgd_t *)chal_pa2va((void*)pt)) + pgd_index(lower_addr);
unsigned int span = hpage_index(size);
if (!(pgd_val(*pgd)) & _PAGE_PRESENT) {
printk("cos: BUG: nothing to copy from pgd @ %x.\n",
(unsigned int)lower_addr);
}
/* sizeof(pgd entry) is intended */
memset(pgd, 0, span*sizeof(pgd_t));
}
static void identity_mapping_add(pgd_t *pgd, unsigned long addr, unsigned long end)
{
unsigned long prot, next;
prot = PMD_TYPE_SECT | PMD_SECT_AP_WRITE | PMD_SECT_AF;
if (cpu_architecture() <= CPU_ARCH_ARMv5TEJ && !cpu_is_xscale())
prot |= PMD_BIT4;
pgd += pgd_index(addr);
do {
next = pgd_addr_end(addr, end);
idmap_add_pud(pgd, addr, next, prot);
} while (pgd++, addr = next, addr != end);
}
/*
* paging_init() sets up the page tables - note that all of lowmem is
* already mapped by head.S.
*/
void __init paging_init(void)
{
#ifdef CONFIG_HIGHMEM
unsigned long vaddr, end;
#endif
#ifdef __tilegx__
pud_t *pud;
#endif
pgd_t *pgd_base = swapper_pg_dir;
kernel_physical_mapping_init(pgd_base);
#ifdef CONFIG_HIGHMEM
/*
* Fixed mappings, only the page table structure has to be
* created - mappings will be set by set_fixmap():
*/
vaddr = __fix_to_virt(__end_of_fixed_addresses - 1) & PMD_MASK;
end = (FIXADDR_TOP + PMD_SIZE - 1) & PMD_MASK;
page_table_range_init(vaddr, end, pgd_base);
permanent_kmaps_init(pgd_base);
#endif
#ifdef __tilegx__
/*
* Since GX allocates just one pmd_t array worth of vmalloc space,
* we go ahead and allocate it statically here, then share it
* globally. As a result we don't have to worry about any task
* changing init_mm once we get up and running, and there's no
* need for e.g. vmalloc_sync_all().
*/
BUILD_BUG_ON(pgd_index(VMALLOC_START) != pgd_index(VMALLOC_END - 1));
pud = pud_offset(pgd_base + pgd_index(VMALLOC_START), VMALLOC_START);
assign_pmd(pud, alloc_pmd());
#endif
}