pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr, unsigned long sz) { pgd_t *pgdp; pud_t *pudp; pmd_t *pmdp = NULL; pgdp = pgd_offset(mm, addr); pudp = pud_alloc(mm, pgdp, addr); if (pudp) pmdp = pmd_alloc(mm, pudp, addr); return (pte_t *) pmdp; }
/* * Initialise the consistent memory allocation. */ static int __init consistent_init(void) { int ret = 0; pgd_t *pgd; pud_t *pud; pmd_t *pmd; pte_t *pte; int i = 0; unsigned long base = consistent_base; unsigned long num_ptes = (CONSISTENT_END - base) >> PMD_SHIFT; consistent_pte = kmalloc(num_ptes * sizeof(pte_t), GFP_KERNEL); if (!consistent_pte) { pr_err("%s: no memory\n", __func__); return -ENOMEM; } pr_debug("DMA memory: 0x%08lx - 0x%08lx:\n", base, CONSISTENT_END); consistent_head.vm_start = base; do { pgd = pgd_offset(&init_mm, base); pud = pud_alloc(&init_mm, pgd, base); if (!pud) { printk(KERN_ERR "%s: no pud tables\n", __func__); ret = -ENOMEM; break; } pmd = pmd_alloc(&init_mm, pud, base); if (!pmd) { printk(KERN_ERR "%s: no pmd tables\n", __func__); ret = -ENOMEM; break; } WARN_ON(!pmd_none(*pmd)); pte = pte_alloc_kernel(pmd, base); if (!pte) { printk(KERN_ERR "%s: no pte tables\n", __func__); ret = -ENOMEM; break; } consistent_pte[i++] = pte; base += PMD_SIZE; } while (base < CONSISTENT_END); return ret; }
static int init_stub_pte(struct mm_struct *mm, unsigned long proc, unsigned long kernel) { pgd_t *pgd; pud_t *pud; pmd_t *pmd; pte_t *pte; pgd = pgd_offset(mm, proc); pud = pud_alloc(mm, pgd, proc); if (!pud) goto out; pmd = pmd_alloc(mm, pud, proc); if (!pmd) goto out_pmd; pte = pte_alloc_map(mm, pmd, proc); if (!pte) goto out_pte; /* There's an interaction between the skas0 stub pages, stack * randomization, and the BUG at the end of exit_mmap. exit_mmap * checks that the number of page tables freed is the same as had * been allocated. If the stack is on the last page table page, * then the stack pte page will be freed, and if not, it won't. To * avoid having to know where the stack is, or if the process mapped * something at the top of its address space for some other reason, * we set TASK_SIZE to end at the start of the last page table. * This keeps exit_mmap off the last page, but introduces a leak * of that page. So, we hang onto it here and free it in * destroy_context_skas. */ mm->context.skas.last_page_table = pmd_page_kernel(*pmd); #ifdef CONFIG_3_LEVEL_PGTABLES mm->context.skas.last_pmd = (unsigned long) __va(pud_val(*pud)); #endif *pte = mk_pte(virt_to_page(kernel), __pgprot(_PAGE_PRESENT)); *pte = pte_mkexec(*pte); *pte = pte_wrprotect(*pte); return(0); out_pmd: pud_free(pud); out_pte: pmd_free(pmd); out: return(-ENOMEM); }
pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr) { pgd_t *pgd; pud_t *pud; pte_t *pte = NULL; pgd = pgd_offset(mm, addr); pud = pud_alloc(mm, pgd, addr); if (pud) pte = (pte_t *) pmd_alloc(mm, pud, addr); BUG_ON(pte && !pte_none(*pte) && !pte_huge(*pte)); return pte; }
pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr, unsigned long sz) { pgd_t *pgd; pud_t *pud; pte_t *pte = NULL; pgd = pgd_offset(mm, addr); pud = pud_alloc(mm, pgd, addr); if (pud) pte = (pte_t *)pmd_alloc(mm, pud, addr); return pte; }
/* * nid, region_start, and region_end are hints to try to place the page * table memory in the same node or region. */ static int __map_kernel_page(unsigned long ea, unsigned long pa, pgprot_t flags, unsigned int map_page_size, int nid, unsigned long region_start, unsigned long region_end) { unsigned long pfn = pa >> PAGE_SHIFT; pgd_t *pgdp; pud_t *pudp; pmd_t *pmdp; pte_t *ptep; /* * Make sure task size is correct as per the max adddr */ BUILD_BUG_ON(TASK_SIZE_USER64 > RADIX_PGTABLE_RANGE); if (unlikely(!slab_is_available())) return early_map_kernel_page(ea, pa, flags, map_page_size, nid, region_start, region_end); /* * Should make page table allocation functions be able to take a * node, so we can place kernel page tables on the right nodes after * boot. */ pgdp = pgd_offset_k(ea); pudp = pud_alloc(&init_mm, pgdp, ea); if (!pudp) return -ENOMEM; if (map_page_size == PUD_SIZE) { ptep = (pte_t *)pudp; goto set_the_pte; } pmdp = pmd_alloc(&init_mm, pudp, ea); if (!pmdp) return -ENOMEM; if (map_page_size == PMD_SIZE) { ptep = pmdp_ptep(pmdp); goto set_the_pte; } ptep = pte_alloc_kernel(pmdp, ea); if (!ptep) return -ENOMEM; set_the_pte: set_pte_at(&init_mm, ea, ptep, pfn_pte(pfn, flags)); smp_wmb(); return 0; }
static int vmap_pud_range(pgd_t *pgd, unsigned long addr, unsigned long end, pgprot_t prot, struct page **pages, int *nr) { pud_t *pud; unsigned long next; pud = pud_alloc(&init_mm, pgd, addr); if (!pud) return -ENOMEM; do { next = pud_addr_end(addr, end); if (vmap_pmd_range(pud, addr, next, prot, pages, nr)) return -ENOMEM; } while (pud++, addr = next, addr != end); return 0; }
static inline int ioremap_pud_range(pgd_t *pgd, unsigned long addr, unsigned long end, unsigned long phys_addr, pgprot_t prot) { pud_t *pud; unsigned long next; phys_addr -= addr; pud = pud_alloc(&init_mm, pgd, addr); if (!pud) return -ENOMEM; do { next = pud_addr_end(addr, end); if (ioremap_pmd_range(pud, addr, next, phys_addr + addr, prot)) return -ENOMEM; } while (pud++, addr = next, addr != end); return 0; }
static pmd_t *mm_alloc_pmd(struct mm_struct *mm, unsigned long address) { pgd_t *pgd; pud_t *pud; pmd_t *pmd = NULL; pgd = pgd_offset(mm, address); pud = pud_alloc(mm, pgd, address); if (pud) /* * Note that we didn't run this because the pmd was * missing, the *pmd may be already established and in * turn it may also be a trans_huge_pmd. */ pmd = pmd_alloc(mm, pud, address); return pmd; }
/* * Initialise the consistent memory allocation. */ static int __init consistent_init(void) { int ret = 0; pgd_t *pgd; pud_t *pud; pmd_t *pmd; pte_t *pte; int i = 0; u32 base = CONSISTENT_BASE; return 0; #if 0 do { pgd = pgd_offset(&init_mm, base); pud = pud_alloc(&init_mm, pgd, base); if (!pud) { printk(KERN_ERR "%s: no pud tables\n", __func__); ret = -ENOMEM; break; } pmd = pmd_alloc(&init_mm, pud, base); if (!pmd) { printk(KERN_ERR "%s: no pmd tables\n", __func__); ret = -ENOMEM; break; } WARN_ON(!pmd_none(*pmd)); pte = pte_alloc_kernel(pmd, base); if (!pte) { printk(KERN_ERR "%s: no pte tables\n", __func__); ret = -ENOMEM; break; } consistent_pte[i++] = pte; base += (1 << PGDIR_SHIFT); } while (base < CONSISTENT_END); return ret; #endif }
pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr, unsigned long sz) { pgd_t *pgd; pud_t *pud; pte_t *pte = NULL; pr_debug("%s: addr:0x%lx sz:0x%lx\n", __func__, addr, sz); pgd = pgd_offset(mm, addr); pud = pud_alloc(mm, pgd, addr); if (!pud) return NULL; if (sz == PUD_SIZE) { pte = (pte_t *)pud; } else if (sz == (PAGE_SIZE * CONT_PTES)) { pmd_t *pmd = pmd_alloc(mm, pud, addr); WARN_ON(addr & (sz - 1)); /* * Note that if this code were ever ported to the * 32-bit arm platform then it will cause trouble in * the case where CONFIG_HIGHPTE is set, since there * will be no pte_unmap() to correspond with this * pte_alloc_map(). */ pte = pte_alloc_map(mm, pmd, addr); } else if (sz == PMD_SIZE) { if (IS_ENABLED(CONFIG_ARCH_WANT_HUGE_PMD_SHARE) && pud_none(*pud)) pte = huge_pmd_share(mm, addr, pud); else pte = (pte_t *)pmd_alloc(mm, pud, addr); } else if (sz == (PMD_SIZE * CONT_PMDS)) { pmd_t *pmd; pmd = pmd_alloc(mm, pud, addr); WARN_ON(addr & (sz - 1)); return (pte_t *)pmd; } pr_debug("%s: addr:0x%lx sz:0x%lx ret pte=%p/0x%llx\n", __func__, addr, sz, pte, pte_val(*pte)); return pte; }
pte_t * huge_pte_alloc(struct mm_struct *mm, unsigned long addr, unsigned long sz) { unsigned long taddr = htlbpage_to_page(addr); pgd_t *pgd; pud_t *pud; pmd_t *pmd; pte_t *pte = NULL; pgd = pgd_offset(mm, taddr); pud = pud_alloc(mm, pgd, taddr); if (pud) { pmd = pmd_alloc(mm, pud, taddr); if (pmd) pte = pte_alloc_map(mm, pmd, taddr); } return pte; }
pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr, unsigned long sz) { pgd_t *pgd; pud_t *pud; pte_t *pte = NULL; /* We do not yet support multiple huge page sizes. */ BUG_ON(sz != PMD_SIZE); pgd = pgd_offset(mm, addr); pud = pud_alloc(mm, pgd, addr); if (pud) pte = (pte_t *) pmd_alloc(mm, pud, addr); BUG_ON(pte && !pte_none(*pte) && !pte_huge(*pte)); return pte; }
/* * map_kernel_page currently only called by __ioremap * map_kernel_page adds an entry to the ioremap page table * and adds an entry to the HPT, possibly bolting it */ int __ref map_kernel_page(unsigned long ea, unsigned long pa, pgprot_t prot) { pgd_t *pgdp; pud_t *pudp; pmd_t *pmdp; pte_t *ptep; BUILD_BUG_ON(TASK_SIZE_USER64 > PGTABLE_RANGE); if (slab_is_available()) { pgdp = pgd_offset_k(ea); pudp = pud_alloc(&init_mm, pgdp, ea); if (!pudp) return -ENOMEM; pmdp = pmd_alloc(&init_mm, pudp, ea); if (!pmdp) return -ENOMEM; ptep = pte_alloc_kernel(pmdp, ea); if (!ptep) return -ENOMEM; } else { pgdp = pgd_offset_k(ea); #ifndef __PAGETABLE_PUD_FOLDED if (pgd_none(*pgdp)) { pudp = early_alloc_pgtable(PUD_TABLE_SIZE); pgd_populate(&init_mm, pgdp, pudp); } #endif /* !__PAGETABLE_PUD_FOLDED */ pudp = pud_offset(pgdp, ea); if (pud_none(*pudp)) { pmdp = early_alloc_pgtable(PMD_TABLE_SIZE); pud_populate(&init_mm, pudp, pmdp); } pmdp = pmd_offset(pudp, ea); if (!pmd_present(*pmdp)) { ptep = early_alloc_pgtable(PAGE_SIZE); pmd_populate_kernel(&init_mm, pmdp, ptep); } ptep = pte_offset_kernel(pmdp, ea); } set_pte_at(&init_mm, ea, ptep, pfn_pte(pa >> PAGE_SHIFT, prot)); smp_wmb(); return 0; }
pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr) { pgd_t *pgd; pud_t *pud; pmd_t *pmd; pte_t *pte = NULL; pgd = pgd_offset(mm, addr); if (pgd) { pud = pud_alloc(mm, pgd, addr); if (pud) { pmd = pmd_alloc(mm, pud, addr); if (pmd) pte = pte_alloc_map(mm, pmd, addr); } } return pte; }
/* * Initialise the consistent memory allocation. */ static int __init consistent_init(void) { int ret = 0; pgd_t *pgd; pud_t *pud; pmd_t *pmd; pte_t *pte; int i = 0; u32 base = CONSISTENT_BASE; do { pgd = pgd_offset(&init_mm, base); pud = pud_alloc(&init_mm, pgd, base); if (!pud) { ; ret = -ENOMEM; break; } pmd = pmd_alloc(&init_mm, pud, base); if (!pmd) { ; ret = -ENOMEM; break; } WARN_ON(!pmd_none(*pmd)); pte = pte_alloc_kernel(pmd, base); if (!pte) { ; ret = -ENOMEM; break; } consistent_pte[i++] = pte; base += (1 << PGDIR_SHIFT); } while (base < CONSISTENT_END); return ret; }
static int remap_area_pages(unsigned long address, unsigned long phys_addr, unsigned long size, unsigned long flags) { pgd_t *dir; int error = 0; unsigned long end = address + size; BUG_ON(address >= end); phys_addr -= address; dir = pgd_offset_k(address); flush_cache_all(); do { pud_t *pud; pmd_t *pmd; error = -ENOMEM; pud = pud_alloc(&init_mm, dir, address); if (!pud) break; pmd = pmd_alloc(&init_mm, pud, address); if (!pmd) break; if (remap_area_pmd(pmd, address, end - address, phys_addr + address, flags)) break; error = 0; address = (address + PGDIR_SIZE) & PGDIR_MASK; dir++; } while (address && (address < end)); flush_tlb_all(); return error; }
pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr, unsigned long sz) { pgd_t *pgdp; pud_t *pudp; pmd_t *pmdp; pte_t *ptep = NULL; pgdp = pgd_offset(mm, addr); pudp = pud_alloc(mm, pgdp, addr); if (!pudp) return NULL; if (sz == PUD_SIZE) { ptep = (pte_t *)pudp; } else if (sz == (PAGE_SIZE * CONT_PTES)) { pmdp = pmd_alloc(mm, pudp, addr); WARN_ON(addr & (sz - 1)); /* * Note that if this code were ever ported to the * 32-bit arm platform then it will cause trouble in * the case where CONFIG_HIGHPTE is set, since there * will be no pte_unmap() to correspond with this * pte_alloc_map(). */ ptep = pte_alloc_map(mm, pmdp, addr); } else if (sz == PMD_SIZE) { if (IS_ENABLED(CONFIG_ARCH_WANT_HUGE_PMD_SHARE) && pud_none(READ_ONCE(*pudp))) ptep = huge_pmd_share(mm, addr, pudp); else ptep = (pte_t *)pmd_alloc(mm, pudp, addr); } else if (sz == (PMD_SIZE * CONT_PMDS)) { pmdp = pmd_alloc(mm, pudp, addr); WARN_ON(addr & (sz - 1)); return (pte_t *)pmdp; } return ptep; }
/* * map_kernel_page currently only called by __ioremap * map_kernel_page adds an entry to the ioremap page table * and adds an entry to the HPT, possibly bolting it */ int map_kernel_page(unsigned long ea, unsigned long pa, int flags) { pgd_t *pgdp; pud_t *pudp; pmd_t *pmdp; pte_t *ptep; if (slab_is_available()) { pgdp = pgd_offset_k(ea); pudp = pud_alloc(&init_mm, pgdp, ea); if (!pudp) return -ENOMEM; pmdp = pmd_alloc(&init_mm, pudp, ea); if (!pmdp) return -ENOMEM; ptep = pte_alloc_kernel(pmdp, ea); if (!ptep) return -ENOMEM; set_pte_at(&init_mm, ea, ptep, pfn_pte(pa >> PAGE_SHIFT, __pgprot(flags))); } else {
void radix__change_memory_range(unsigned long start, unsigned long end, unsigned long clear) { unsigned long idx; pgd_t *pgdp; pud_t *pudp; pmd_t *pmdp; pte_t *ptep; start = ALIGN_DOWN(start, PAGE_SIZE); end = PAGE_ALIGN(end); // aligns up pr_debug("Changing flags on range %lx-%lx removing 0x%lx\n", start, end, clear); for (idx = start; idx < end; idx += PAGE_SIZE) { pgdp = pgd_offset_k(idx); pudp = pud_alloc(&init_mm, pgdp, idx); if (!pudp) continue; if (pud_huge(*pudp)) { ptep = (pte_t *)pudp; goto update_the_pte; } pmdp = pmd_alloc(&init_mm, pudp, idx); if (!pmdp) continue; if (pmd_huge(*pmdp)) { ptep = pmdp_ptep(pmdp); goto update_the_pte; } ptep = pte_alloc_kernel(pmdp, idx); if (!ptep) continue; update_the_pte: radix__pte_update(&init_mm, idx, ptep, clear, 0, 0); } radix__flush_tlb_kernel_range(start, end); }
int oleole_get_gPTE_offset_with_alloc(struct mm_struct *mm, pte_t **result, unsigned long address) { pgd_t *pgd, pgd_v; pud_t *pud, pud_v; pmd_t *pmd, pmd_v; pte_t *pte; pgd = pgd_offset(mm, address); pgd_v = *pgd; if (pgd_none(pgd_v)) if (pud_alloc(mm, pgd, address) == NULL) return -ENOMEM; pud = pud_offset(pgd, address); pud_v = *pud; if (oleole_pud_none(pud_v)) if (oleole_pmd_alloc(pud)) return -ENOMEM; if (unlikely((pud_val(pud_v) & _PAGE_DEACTIVATED))) reactivate_pmd_table(pud); pmd = pmd_offset(pud, address); pmd_v = *pmd; if (oleole_pmd_none(pmd_v)) if (oleole_pte_alloc(pmd)) return -ENOMEM; if (unlikely((pmd_val(pmd_v) & _PAGE_DEACTIVATED))) reactivate_pte_table(pmd); pte = pte_offset_map(pmd, address); *result = pte; return 0; }
static int map_tboot_page(unsigned long vaddr, unsigned long pfn, pgprot_t prot) { pgd_t *pgd; pud_t *pud; pmd_t *pmd; pte_t *pte; pgd = pgd_offset(&tboot_mm, vaddr); pud = pud_alloc(&tboot_mm, pgd, vaddr); if (!pud) return -1; pmd = pmd_alloc(&tboot_mm, pud, vaddr); if (!pmd) return -1; pte = pte_alloc_map(&tboot_mm, pmd, vaddr); if (!pte) return -1; set_pte_at(&tboot_mm, vaddr, pte, pfn_pte(pfn, prot)); pte_unmap(pte); return 0; }
int io_remap_pfn_range(struct vm_area_struct *vma, unsigned long from, unsigned long pfn, unsigned long size, pgprot_t prot) { int error = 0; pgd_t * dir; unsigned long beg = from; unsigned long end = from + size; struct mm_struct *mm = vma->vm_mm; int space = GET_IOSPACE(pfn); unsigned long offset = GET_PFN(pfn) << PAGE_SHIFT; unsigned long phys_base; phys_base = offset | (((unsigned long) space) << 32UL); /* See comment in mm/memory.c remap_pfn_range */ vma->vm_flags |= VM_IO | VM_RESERVED | VM_PFNMAP; vma->vm_pgoff = phys_base >> PAGE_SHIFT; prot = __pgprot(pg_iobits); offset -= from; dir = pgd_offset(mm, from); flush_cache_range(vma, beg, end); while (from < end) { pud_t *pud = pud_alloc(mm, dir, from); error = -ENOMEM; if (!pud) break; error = io_remap_pud_range(mm, pud, from, end - from, offset + from, prot, space); if (error) break; from = (from + PGDIR_SIZE) & PGDIR_MASK; dir++; } flush_tlb_range(vma, beg, end); return error; }
pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr) { pgd_t *pgd; pud_t *pud; pmd_t *pmd; pte_t *pte = NULL; /* We must align the address, because our caller will run * set_huge_pte_at() on whatever we return, which writes out * all of the sub-ptes for the hugepage range. So we have * to give it the first such sub-pte. */ addr &= HPAGE_MASK; pgd = pgd_offset(mm, addr); pud = pud_alloc(mm, pgd, addr); if (pud) { pmd = pmd_alloc(mm, pud, addr); if (pmd) pte = pte_alloc_map(mm, pmd, addr); } return pte; }
pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr, unsigned long sz) { pgd_t *pgd; pud_t *pud; pmd_t *pmd; pte_t *pte = NULL; pgd = pgd_offset(mm, addr); pud = pud_alloc(mm, pgd, addr); if (pud) { pmd = pmd_alloc(mm, pud, addr); if (!pmd) return NULL; if (sz >= PMD_SIZE) pte = (pte_t *)pmd; else pte = pte_alloc_map(mm, pmd, addr); } return pte; }
static void set_pte_phys(unsigned long addr, unsigned long phys, pgprot_t prot) { pgd_t *pgd; pud_t *pud; pmd_t *pmd; pte_t *pte; pgd = pgd_offset_k(addr); if (pgd_none(*pgd)) { pgd_ERROR(*pgd); return; } pud = pud_alloc(NULL, pgd, addr); if (unlikely(!pud)) { pud_ERROR(*pud); return; } pmd = pmd_alloc(NULL, pud, addr); if (unlikely(!pmd)) { pmd_ERROR(*pmd); return; } pte = pte_offset_kernel(pmd, addr); if (!pte_none(*pte)) { pte_ERROR(*pte); return; } set_pte(pte, pfn_pte(phys >> PAGE_SHIFT, prot)); if (cached_to_uncached) flush_tlb_one(get_asid(), addr); }
/* * We need our own copy of the higher levels of the page tables * because we want to avoid inserting EFI region mappings (EFI_VA_END * to EFI_VA_START) into the standard kernel page tables. Everything * else can be shared, see efi_sync_low_kernel_mappings(). * * We don't want the pgd on the pgd_list and cannot use pgd_alloc() for the * allocation. */ int __init efi_alloc_page_tables(void) { pgd_t *pgd, *efi_pgd; p4d_t *p4d; pud_t *pud; gfp_t gfp_mask; if (efi_enabled(EFI_OLD_MEMMAP)) return 0; gfp_mask = GFP_KERNEL | __GFP_ZERO; efi_pgd = (pgd_t *)__get_free_pages(gfp_mask, PGD_ALLOCATION_ORDER); if (!efi_pgd) return -ENOMEM; pgd = efi_pgd + pgd_index(EFI_VA_END); p4d = p4d_alloc(&init_mm, pgd, EFI_VA_END); if (!p4d) { free_page((unsigned long)efi_pgd); return -ENOMEM; } pud = pud_alloc(&init_mm, p4d, EFI_VA_END); if (!pud) { if (pgtable_l5_enabled) free_page((unsigned long) pgd_page_vaddr(*pgd)); free_pages((unsigned long)efi_pgd, PGD_ALLOCATION_ORDER); return -ENOMEM; } efi_mm.pgd = efi_pgd; mm_init_cpumask(&efi_mm); init_new_context(NULL, &efi_mm); return 0; }
/* * need to get a 16k page for level 1 */ pgd_t *pgd_alloc(struct mm_struct *mm) { pgd_t *new_pgd, *init_pgd; pud_t *new_pud, *init_pud; pmd_t *new_pmd, *init_pmd; pte_t *new_pte, *init_pte; new_pgd = __pgd_alloc(); if (!new_pgd) goto no_pgd; memset(new_pgd, 0, USER_PTRS_PER_PGD * sizeof(pgd_t)); /* * Copy over the kernel and IO PGD entries */ init_pgd = pgd_offset_k(0); memcpy(new_pgd + USER_PTRS_PER_PGD, init_pgd + USER_PTRS_PER_PGD, (PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t)); clean_dcache_area(new_pgd, PTRS_PER_PGD * sizeof(pgd_t)); #ifdef CONFIG_ARM_LPAE /* * Allocate PMD table for modules and pkmap mappings. */ new_pud = pud_alloc(mm, new_pgd + pgd_index(MODULES_VADDR), MODULES_VADDR); if (!new_pud) goto no_pud; new_pmd = pmd_alloc(mm, new_pud, 0); if (!new_pmd) goto no_pmd; #endif if (!vectors_high()) { /* * On ARM, first page must always be allocated since it * contains the machine vectors. The vectors are always high * with LPAE. */ new_pud = pud_alloc(mm, new_pgd, 0); if (!new_pud) goto no_pud; new_pmd = pmd_alloc(mm, new_pud, 0); if (!new_pmd) goto no_pmd; new_pte = pte_alloc_map(mm, NULL, new_pmd, 0); if (!new_pte) goto no_pte; init_pud = pud_offset(init_pgd, 0); init_pmd = pmd_offset(init_pud, 0); init_pte = pte_offset_map(init_pmd, 0); set_pte_ext(new_pte, *init_pte, 0); pte_unmap(init_pte); pte_unmap(new_pte); } return new_pgd; no_pte: pmd_free(mm, new_pmd); no_pmd: pud_free(mm, new_pud); no_pud: __pgd_free(new_pgd); no_pgd: return NULL; }
void *nvmap_mmap(struct nvmap_handle_ref *ref) { struct nvmap_handle *h; pgprot_t prot; unsigned long adj_size; unsigned long offs; struct vm_struct *v; void *p; h = nvmap_handle_get(ref->handle); if (!h) return NULL; prot = nvmap_pgprot(h, pgprot_kernel); if (h->heap_pgalloc) return vm_map_ram(h->pgalloc.pages, h->size >> PAGE_SHIFT, -1, prot); /* carveout - explicitly map the pfns into a vmalloc area */ nvmap_usecount_inc(h); adj_size = h->carveout->base & ~PAGE_MASK; adj_size += h->size; adj_size = PAGE_ALIGN(adj_size); v = alloc_vm_area(adj_size); if (!v) { nvmap_usecount_dec(h); nvmap_handle_put(h); return NULL; } p = v->addr + (h->carveout->base & ~PAGE_MASK); for (offs = 0; offs < adj_size; offs += PAGE_SIZE) { unsigned long addr = (unsigned long) v->addr + offs; unsigned int pfn; pgd_t *pgd; pud_t *pud; pmd_t *pmd; pte_t *pte; pfn = __phys_to_pfn(h->carveout->base + offs); pgd = pgd_offset_k(addr); pud = pud_alloc(&init_mm, pgd, addr); if (!pud) break; pmd = pmd_alloc(&init_mm, pud, addr); if (!pmd) break; pte = pte_alloc_kernel(pmd, addr); if (!pte) break; set_pte_at(&init_mm, addr, pte, pfn_pte(pfn, prot)); flush_tlb_kernel_page(addr); } if (offs != adj_size) { free_vm_area(v); nvmap_usecount_dec(h); nvmap_handle_put(h); return NULL; } /* leave the handle ref count incremented by 1, so that * the handle will not be freed while the kernel mapping exists. * nvmap_handle_put will be called by unmapping this address */ return p; }
pgd_t *pgd_alloc(struct mm_struct *mm) { pgd_t *new_pgd, *init_pgd; pud_t *new_pud, *init_pud; pmd_t *new_pmd, *init_pmd; pte_t *new_pte, *init_pte; new_pgd = __pgd_alloc(); if (!new_pgd) goto no_pgd; memset(new_pgd, 0, USER_PTRS_PER_PGD * sizeof(pgd_t)); init_pgd = pgd_offset_k(0); memcpy(new_pgd + USER_PTRS_PER_PGD, init_pgd + USER_PTRS_PER_PGD, (PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t)); clean_dcache_area(new_pgd, PTRS_PER_PGD * sizeof(pgd_t)); #ifdef CONFIG_ARM_LPAE new_pud = pud_alloc(mm, new_pgd + pgd_index(MODULES_VADDR), MODULES_VADDR); if (!new_pud) goto no_pud; new_pmd = pmd_alloc(mm, new_pud, 0); if (!new_pmd) goto no_pmd; #endif if (!vectors_high()) { new_pud = pud_alloc(mm, new_pgd, 0); if (!new_pud) goto no_pud; new_pmd = pmd_alloc(mm, new_pud, 0); if (!new_pmd) goto no_pmd; new_pte = pte_alloc_map(mm, NULL, new_pmd, 0); if (!new_pte) goto no_pte; init_pud = pud_offset(init_pgd, 0); init_pmd = pmd_offset(init_pud, 0); init_pte = pte_offset_map(init_pmd, 0); set_pte_ext(new_pte, *init_pte, 0); pte_unmap(init_pte); pte_unmap(new_pte); } return new_pgd; no_pte: pmd_free(mm, new_pmd); no_pmd: pud_free(mm, new_pud); no_pud: __pgd_free(new_pgd); no_pgd: return NULL; }