/* * When built with CONFIG_DEBUG_PAGEALLOC and CONFIG_HIBERNATION, this function * is used to determine if a linear map page has been marked as not-valid by * CONFIG_DEBUG_PAGEALLOC. Walk the page table and check the PTE_VALID bit. * This is based on kern_addr_valid(), which almost does what we need. * * Because this is only called on the kernel linear map, p?d_sect() implies * p?d_present(). When debug_pagealloc is enabled, sections mappings are * disabled. */ bool kernel_page_present(struct page *page) { pgd_t *pgd; pud_t *pud; pmd_t *pmd; pte_t *pte; unsigned long addr = (unsigned long)page_address(page); pgd = pgd_offset_k(addr); if (pgd_none(*pgd)) return false; pud = pud_offset(pgd, addr); if (pud_none(*pud)) return false; if (pud_sect(*pud)) return true; pmd = pmd_offset(pud, addr); if (pmd_none(*pmd)) return false; if (pmd_sect(*pmd)) return true; pte = pte_offset_kernel(pmd, addr); return pte_valid(*pte); }
/* * Changing some bits of contiguous entries requires us to follow a * Break-Before-Make approach, breaking the whole contiguous set * before we can change any entries. See ARM DDI 0487A.k_iss10775, * "Misprogramming of the Contiguous bit", page D4-1762. * * This helper performs the break step. */ static pte_t get_clear_flush(struct mm_struct *mm, unsigned long addr, pte_t *ptep, unsigned long pgsize, unsigned long ncontig) { struct vm_area_struct vma = { .vm_mm = mm }; pte_t orig_pte = huge_ptep_get(ptep); bool valid = pte_valid(orig_pte); unsigned long i, saddr = addr; for (i = 0; i < ncontig; i++, addr += pgsize, ptep++) { pte_t pte = ptep_get_and_clear(mm, addr, ptep); /* * If HW_AFDBM is enabled, then the HW could turn on * the dirty bit for any page in the set, so check * them all. All hugetlb entries are already young. */ if (pte_dirty(pte)) orig_pte = pte_mkdirty(orig_pte); } if (valid) flush_tlb_range(&vma, saddr, addr); return orig_pte; }
/* * When built with CONFIG_DEBUG_PAGEALLOC and CONFIG_HIBERNATION, this function * is used to determine if a linear map page has been marked as not-valid by * CONFIG_DEBUG_PAGEALLOC. Walk the page table and check the PTE_VALID bit. * This is based on kern_addr_valid(), which almost does what we need. * * Because this is only called on the kernel linear map, p?d_sect() implies * p?d_present(). When debug_pagealloc is enabled, sections mappings are * disabled. */ bool kernel_page_present(struct page *page) { pgd_t *pgdp; pud_t *pudp, pud; pmd_t *pmdp, pmd; pte_t *ptep; unsigned long addr = (unsigned long)page_address(page); pgdp = pgd_offset_k(addr); if (pgd_none(READ_ONCE(*pgdp))) return false; pudp = pud_offset(pgdp, addr); pud = READ_ONCE(*pudp); if (pud_none(pud)) return false; if (pud_sect(pud)) return true; pmdp = pmd_offset(pudp, addr); pmd = READ_ONCE(*pmdp); if (pmd_none(pmd)) return false; if (pmd_sect(pmd)) return true; ptep = pte_offset_kernel(pmdp, addr); return pte_valid(READ_ONCE(*ptep)); }
/* * Changing some bits of contiguous entries requires us to follow a * Break-Before-Make approach, breaking the whole contiguous set * before we can change any entries. See ARM DDI 0487A.k_iss10775, * "Misprogramming of the Contiguous bit", page D4-1762. * * This helper performs the break step. */ static pte_t get_clear_flush(struct mm_struct *mm, unsigned long addr, pte_t *ptep, unsigned long pgsize, unsigned long ncontig) { pte_t orig_pte = huge_ptep_get(ptep); bool valid = pte_valid(orig_pte); unsigned long i, saddr = addr; for (i = 0; i < ncontig; i++, addr += pgsize, ptep++) { pte_t pte = ptep_get_and_clear(mm, addr, ptep); /* * If HW_AFDBM is enabled, then the HW could turn on * the dirty or accessed bit for any page in the set, * so check them all. */ if (pte_dirty(pte)) orig_pte = pte_mkdirty(orig_pte); if (pte_young(pte)) orig_pte = pte_mkyoung(orig_pte); } if (valid) { struct vm_area_struct vma = TLB_FLUSH_VMA(mm, 0); flush_tlb_range(&vma, saddr, addr); } return orig_pte; }
static void _copy_pte(pte_t *dst_ptep, pte_t *src_ptep, unsigned long addr) { pte_t pte = READ_ONCE(*src_ptep); if (pte_valid(pte)) { /* * Resume will overwrite areas that may be marked * read only (code, rodata). Clear the RDONLY bit from * the temporary mappings we use during restore. */ set_pte(dst_ptep, pte_mkwrite(pte)); } else if (debug_pagealloc_enabled() && !pte_none(pte)) { /* * debug_pagealloc will removed the PTE_VALID bit if * the page isn't in use by the resume kernel. It may have * been in use by the original kernel, in which case we need * to put it back in our copy to do the restore. * * Before marking this entry valid, check the pfn should * be mapped. */ BUG_ON(!pfn_valid(pte_pfn(pte))); set_pte(dst_ptep, pte_mkpresent(pte_mkwrite(pte))); } }