static void __init kasan_early_p4d_populate(pgd_t *pgd, unsigned long addr, unsigned long end) { pgd_t pgd_entry; p4d_t *p4d, p4d_entry; unsigned long next; if (pgd_none(*pgd)) { pgd_entry = __pgd(_KERNPG_TABLE | __pa_nodebug(kasan_early_shadow_p4d)); set_pgd(pgd, pgd_entry); } p4d = early_p4d_offset(pgd, addr); do { next = p4d_addr_end(addr, end); if (!p4d_none(*p4d)) continue; p4d_entry = __p4d(_KERNPG_TABLE | __pa_nodebug(kasan_early_shadow_pud)); set_p4d(p4d, p4d_entry); } while (p4d++, addr = next, addr != end && p4d_none(*p4d)); }
static void sync_current_stack_to_mm(struct mm_struct *mm) { unsigned long sp = current_stack_pointer; pgd_t *pgd = pgd_offset(mm, sp); if (pgtable_l5_enabled()) { if (unlikely(pgd_none(*pgd))) { pgd_t *pgd_ref = pgd_offset_k(sp); set_pgd(pgd, *pgd_ref); } } else { /* * "pgd" is faked. The top level entries are "p4d"s, so sync * the p4d. This compiles to approximately the same code as * the 5-level case. */ p4d_t *p4d = p4d_offset(pgd, sp); if (unlikely(p4d_none(*p4d))) { pgd_t *pgd_ref = pgd_offset_k(sp); p4d_t *p4d_ref = p4d_offset(pgd_ref, sp); set_p4d(p4d, *p4d_ref); } } }
static int get_gate_page(struct mm_struct *mm, unsigned long address, unsigned int gup_flags, struct vm_area_struct **vma, struct page **page) { pgd_t *pgd; p4d_t *p4d; pud_t *pud; pmd_t *pmd; pte_t *pte; int ret = -EFAULT; /* user gate pages are read-only */ if (gup_flags & FOLL_WRITE) return -EFAULT; if (address > TASK_SIZE) pgd = pgd_offset_k(address); else pgd = pgd_offset_gate(mm, address); BUG_ON(pgd_none(*pgd)); p4d = p4d_offset(pgd, address); BUG_ON(p4d_none(*p4d)); pud = pud_offset(p4d, address); BUG_ON(pud_none(*pud)); pmd = pmd_offset(pud, address); if (pmd_none(*pmd)) return -EFAULT; VM_BUG_ON(pmd_trans_huge(*pmd)); pte = pte_offset_map(pmd, address); if (pte_none(*pte)) goto unmap; *vma = get_gate_vma(mm); if (!page) goto out; *page = vm_normal_page(*vma, address, *pte); if (!*page) { if ((gup_flags & FOLL_DUMP) || !is_zero_pfn(pte_pfn(*pte))) goto unmap; *page = pte_page(*pte); } get_page(*page); out: ret = 0; unmap: pte_unmap(pte); return ret; }
static void __init kasan_populate_p4d(p4d_t *p4d, unsigned long addr, unsigned long end, int nid) { pud_t *pud; unsigned long next; if (p4d_none(*p4d)) { void *p = early_alloc(PAGE_SIZE, nid, true); p4d_populate(&init_mm, p4d, p); } pud = pud_offset(p4d, addr); do { next = pud_addr_end(addr, end); if (!pud_large(*pud)) kasan_populate_pud(pud, addr, next, nid); } while (pud++, addr = next, addr != end); }
/** * follow_page_mask - look up a page descriptor from a user-virtual address * @vma: vm_area_struct mapping @address * @address: virtual address to look up * @flags: flags modifying lookup behaviour * @page_mask: on output, *page_mask is set according to the size of the page * * @flags can have FOLL_ flags set, defined in <linux/mm.h> * * Returns the mapped (struct page *), %NULL if no mapping exists, or * an error pointer if there is a mapping to something not represented * by a page descriptor (see also vm_normal_page()). */ struct page *follow_page_mask(struct vm_area_struct *vma, unsigned long address, unsigned int flags, unsigned int *page_mask) { pgd_t *pgd; p4d_t *p4d; pud_t *pud; pmd_t *pmd; spinlock_t *ptl; struct page *page; struct mm_struct *mm = vma->vm_mm; *page_mask = 0; page = follow_huge_addr(mm, address, flags & FOLL_WRITE); if (!IS_ERR(page)) { BUG_ON(flags & FOLL_GET); return page; } pgd = pgd_offset(mm, address); if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd))) return no_page_table(vma, flags); p4d = p4d_offset(pgd, address); if (p4d_none(*p4d)) return no_page_table(vma, flags); BUILD_BUG_ON(p4d_huge(*p4d)); if (unlikely(p4d_bad(*p4d))) return no_page_table(vma, flags); pud = pud_offset(p4d, address); if (pud_none(*pud)) return no_page_table(vma, flags); if (pud_huge(*pud) && vma->vm_flags & VM_HUGETLB) { page = follow_huge_pud(mm, address, pud, flags); if (page) return page; return no_page_table(vma, flags); } if (pud_devmap(*pud)) { ptl = pud_lock(mm, pud); page = follow_devmap_pud(vma, address, pud, flags); spin_unlock(ptl); if (page) return page; } if (unlikely(pud_bad(*pud))) return no_page_table(vma, flags); pmd = pmd_offset(pud, address); if (pmd_none(*pmd)) return no_page_table(vma, flags); if (pmd_huge(*pmd) && vma->vm_flags & VM_HUGETLB) { page = follow_huge_pmd(mm, address, pmd, flags); if (page) return page; return no_page_table(vma, flags); } if (pmd_devmap(*pmd)) { ptl = pmd_lock(mm, pmd); page = follow_devmap_pmd(vma, address, pmd, flags); spin_unlock(ptl); if (page) return page; } if (likely(!pmd_trans_huge(*pmd))) return follow_page_pte(vma, address, pmd, flags); if ((flags & FOLL_NUMA) && pmd_protnone(*pmd)) return no_page_table(vma, flags); ptl = pmd_lock(mm, pmd); if (unlikely(!pmd_trans_huge(*pmd))) { spin_unlock(ptl); return follow_page_pte(vma, address, pmd, flags); } if (flags & FOLL_SPLIT) { int ret; page = pmd_page(*pmd); if (is_huge_zero_page(page)) { spin_unlock(ptl); ret = 0; split_huge_pmd(vma, pmd, address); if (pmd_trans_unstable(pmd)) ret = -EBUSY; } else { get_page(page); spin_unlock(ptl); lock_page(page); ret = split_huge_page(page); unlock_page(page); put_page(page); if (pmd_none(*pmd)) return no_page_table(vma, flags); } return ret ? ERR_PTR(ret) : follow_page_pte(vma, address, pmd, flags); } page = follow_trans_huge_pmd(vma, address, pmd, flags); spin_unlock(ptl); *page_mask = HPAGE_PMD_NR - 1; return page; }
/* * map any virtual address of the current process to its * physical one. */ static unsigned long long any_v2p(unsigned long long vaddr) { pgd_t *pgd = pgd_offset(current->mm, vaddr); #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 12, 0) p4d_t *p4d; #endif pud_t *pud; pmd_t *pmd; pte_t *pte; /* to lock the page */ struct page *pg; unsigned long long paddr; if (bad_address(pgd)) { printk(KERN_ALERT "[nskk] Alert: bad address of pgd %p\n", pgd); goto bad; } if (!pgd_present(*pgd)) { printk(KERN_ALERT "[nskk] Alert: pgd not present %lu\n", *pgd); goto out; } #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 12, 0) p4d = p4d_offset(pgd, vaddr); if (p4d_none(*p4d)) return 0; pud = pud_offset(p4d, vaddr); #else pud = pud_offset(pgd, vaddr); #endif if (bad_address(pud)) { printk(KERN_ALERT "[nskk] Alert: bad address of pud %p\n", pud); goto bad; } if (!pud_present(*pud) || pud_large(*pud)) { printk(KERN_ALERT "[nskk] Alert: pud not present %lu\n", *pud); goto out; } pmd = pmd_offset(pud, vaddr); if (bad_address(pmd)) { printk(KERN_ALERT "[nskk] Alert: bad address of pmd %p\n", pmd); goto bad; } if (!pmd_present(*pmd) || pmd_large(*pmd)) { printk(KERN_ALERT "[nskk] Alert: pmd not present %lu\n", *pmd); goto out; } pte = pte_offset_kernel(pmd, vaddr); if (bad_address(pte)) { printk(KERN_ALERT "[nskk] Alert: bad address of pte %p\n", pte); goto bad; } if (!pte_present(*pte)) { printk(KERN_ALERT "[nskk] Alert: pte not present %lu\n", *pte); goto out; } pg = pte_page(*pte); #if 1 paddr = (pte_val(*pte) & PHYSICAL_PAGE_MASK) | (vaddr&(PAGE_SIZE-1)); #else pte->pte |= _PAGE_RW; // | _PAGE_USER; paddr = pte_val(*pte); #endif out: return paddr; bad: printk(KERN_ALERT "[nskk] Alert: Bad address\n"); return 0; }