static inline void unswap_pgd(struct vm_area_struct * vma, pgd_t *dir,
unsigned long address, unsigned long size,
unsigned long entry, unsigned long page
/* , int isswap */)
{
pmd_t * pmd;
unsigned long offset, end;
if (pgd_none(*dir))
return;
if (pgd_bad(*dir)) {
printk("unswap_pgd: bad pgd (%08lx)\n", pgd_val(*dir));
pgd_clear(dir);
return;
}
pmd = pmd_offset(dir, address);
offset = address & PGDIR_MASK;
address &= ~PGDIR_MASK;
end = address + size;
if (end > PGDIR_SIZE)
end = PGDIR_SIZE;
do {
unswap_pmd(vma, pmd, address, end - address, offset, entry,
page /* , isswap */);
address = (address + PMD_SIZE) & PMD_MASK;
pmd++;
} while (address < end);
}
static inline pte_t *get_one_pte(struct mm_struct *mm, unsigned long addr)
{
pgd_t * pgd;
pmd_t * pmd;
pte_t * pte = NULL;
pgd = pgd_offset(mm, addr);
if (pgd_none(*pgd))
goto end;
if (pgd_bad(*pgd)) {
pgd_ERROR(*pgd);
pgd_clear(pgd);
goto end;
}
pmd = pmd_offset(pgd, addr);
if (pmd_none(*pmd))
goto end;
if (pmd_bad(*pmd)) {
pmd_ERROR(*pmd);
pmd_clear(pmd);
goto end;
}
pte = pte_offset(pmd, addr);
if (pte_none(*pte))
pte = NULL;
end:
return pte;
}
static inline pte_t *get_one_pte(struct mm_struct *mm, unsigned long addr)
{
pgd_t * pgd;
pmd_t * pmd;
pte_t * pte = NULL;
pgd = pgd_offset(mm, addr);
if (pgd_none(*pgd))
goto end;
if (pgd_bad(*pgd)) {
printk("move_one_page: bad source pgd (%08lx)\n", pgd_val(*pgd));
pgd_clear(pgd);
goto end;
}
pmd = pmd_offset(pgd, addr);
if (pmd_none(*pmd))
goto end;
if (pmd_bad(*pmd)) {
printk("move_one_page: bad source pmd (%08lx)\n", pmd_val(*pmd));
pmd_clear(pmd);
goto end;
}
pte = pte_offset(pmd, addr);
if (pte_none(*pte))
pte = NULL;
end:
return pte;
}
void pgd_free(struct mm_struct *mm, pgd_t *pgd_base)
{
pgd_t *pgd;
pud_t *pud;
pmd_t *pmd;
pgtable_t pte;
if (!pgd_base)
return;
pgd = pgd_base + pgd_index(0);
if (pgd_none_or_clear_bad(pgd))
goto no_pgd;
pud = pud_offset(pgd, 0);
if (pud_none_or_clear_bad(pud))
goto no_pud;
pmd = pmd_offset(pud, 0);
if (pmd_none_or_clear_bad(pmd))
goto no_pmd;
pte = pmd_pgtable(*pmd);
pmd_clear(pmd);
pte_free(mm, pte);
no_pmd:
pud_clear(pud);
pmd_free(mm, pmd);
no_pud:
pgd_clear(pgd);
pud_free(mm, pud);
no_pgd:
free_pages((unsigned long) pgd_base, 2);
}
static inline int zap_pmd_range(mmu_gather_t *tlb, pgd_t * dir, unsigned long address, unsigned long size)
{
pmd_t * pmd;
unsigned long end;
int freed;
if (pgd_none(*dir))
return 0;
if (pgd_bad(*dir)) {
pgd_ERROR(*dir);
pgd_clear(dir);
return 0;
}
pmd = pmd_offset(dir, address);
end = address + size;
if (end > ((address + PGDIR_SIZE) & PGDIR_MASK))
end = ((address + PGDIR_SIZE) & PGDIR_MASK);
freed = 0;
do {
freed += zap_pte_range(tlb, pmd, address, end - address);
address = (address + PMD_SIZE) & PMD_MASK;
pmd++;
} while (address < end);
return freed;
}
static inline void remove_mapping_pmd_range (pgd_t *pgd, unsigned long address,
unsigned long size)
{
pmd_t *pmd;
unsigned long end;
if (pgd_none (*pgd))
return;
if (pgd_bad (*pgd)) {
printk ("remove_graphics_pmd_range: bad pgd (%08lx)\n",
pgd_val (*pgd));
pgd_clear (pgd);
return;
}
pmd = pmd_offset (pgd, address);
address &= ~PGDIR_MASK;
end = address + size;
if (end > PGDIR_SIZE)
end = PGDIR_SIZE;
do {
remove_mapping_pte_range (pmd, address, end - address);
address = (address + PMD_SIZE) & PMD_MASK;
pmd++;
} while (address < end);
}
static inline int swap_out_pgd(struct mm_struct * mm, struct vm_area_struct * vma, pgd_t *dir, unsigned long address, unsigned long end, int gfp_mask)
{
pmd_t * pmd;
unsigned long pgd_end;
if (pgd_none(*dir))
return 0;
if (pgd_bad(*dir)) {
pgd_ERROR(*dir);
pgd_clear(dir);
return 0;
}
pmd = pmd_offset(dir, address);
pgd_end = (address + PGDIR_SIZE) & PGDIR_MASK;
if (pgd_end && (end > pgd_end))
end = pgd_end;
do {
int result = swap_out_pmd(mm, vma, pmd, address, end, gfp_mask);
if (result)
return result;
if (!mm->swap_cnt)
return 0;
address = (address + PMD_SIZE) & PMD_MASK;
pmd++;
} while (address && (address < end));
return 0;
}
/* mm->page_table_lock is held. mmap_sem is not held */
static inline int swap_out_pgd(struct mm_struct * mm, struct vm_area_struct * vma, pgd_t *dir, unsigned long address, unsigned long end, int count, zone_t * classzone)
{
pmd_t * pmd;
unsigned long pgd_end;
if (pgd_none(*dir))
return count;
if (pgd_bad(*dir)) {
pgd_ERROR(*dir);
pgd_clear(dir);
return count;
}
pmd = pmd_offset(dir, address);
pgd_end = (address + PGDIR_SIZE) & PGDIR_MASK;
if (pgd_end && (end > pgd_end))
end = pgd_end;
do {
count = swap_out_pmd(mm, vma, pmd, address, end, count, classzone);
if (!count)
break;
address = (address + PMD_SIZE) & PMD_MASK;
pmd++;
} while (address && (address < end));
return count;
}
static bool kvm_mips_flush_gpa_pgd(pgd_t *pgd, unsigned long start_gpa,
unsigned long end_gpa)
{
pud_t *pud;
unsigned long end = ~0ul;
int i_min = pgd_index(start_gpa);
int i_max = pgd_index(end_gpa);
bool safe_to_remove = (i_min == 0 && i_max == PTRS_PER_PGD - 1);
int i;
for (i = i_min; i <= i_max; ++i, start_gpa = 0) {
if (!pgd_present(pgd[i]))
continue;
pud = pud_offset(pgd + i, 0);
if (i == i_max)
end = end_gpa;
if (kvm_mips_flush_gpa_pud(pud, start_gpa, end)) {
pgd_clear(pgd + i);
pud_free(NULL, pud);
} else {
safe_to_remove = false;
}
}
return safe_to_remove;
}
/*
* This routine gets a long from any process space by following the page
* tables. NOTE! You should check that the long isn't on a page boundary,
* and that it is in the task area before calling this: this routine does
* no checking.
*/
static unsigned long get_long(struct vm_area_struct * vma, unsigned long addr)
{
pgd_t * pgdir;
pte_t * pgtable;
unsigned long page;
repeat:
pgdir = PAGE_DIR_OFFSET(vma->vm_mm, addr);
if (pgd_none(*pgdir)) {
do_no_page(vma, addr, 0);
goto repeat;
}
if (pgd_bad(*pgdir)) {
printk("ptrace: bad page directory %08lx\n", pgd_val(*pgdir));
pgd_clear(pgdir);
return 0;
}
pgtable = (pte_t *) (PAGE_PTR(addr) + pgd_page(*pgdir));
if (!pte_present(*pgtable)) {
do_no_page(vma, addr, 0);
goto repeat;
}
page = pte_page(*pgtable);
/* this is a hack for non-kernel-mapped video buffers and similar */
if (page >= high_memory)
return 0;
page += addr & ~PAGE_MASK;
return *(unsigned long *) page;
}
static inline int unuse_pgd(struct vm_area_struct * vma, pgd_t *dir,
unsigned long address, unsigned long size,
unsigned int type, unsigned long page)
{
pmd_t * pmd;
unsigned long offset, end;
if (pgd_none(*dir))
return 0;
if (pgd_bad(*dir)) {
printk("unuse_pgd: bad pgd (%08lx)\n", pgd_val(*dir));
pgd_clear(dir);
return 0;
}
pmd = pmd_offset(dir, address);
offset = address & PGDIR_MASK;
address &= ~PGDIR_MASK;
end = address + size;
if (end > PGDIR_SIZE)
end = PGDIR_SIZE;
do {
if (unuse_pmd(vma, pmd, address, end - address, offset, type, page))
return 1;
address = (address + PMD_SIZE) & PMD_MASK;
pmd++;
} while (address < end);
return 0;
}
static unsigned long get_phys_addr(struct task_struct * p, unsigned long ptr)
{
pgd_t *page_dir;
pmd_t *page_middle;
pte_t pte;
if (!p || !p->mm || ptr >= TASK_SIZE)
return 0;
page_dir = pgd_offset(p->mm,ptr);
if (pgd_none(*page_dir))
return 0;
if (pgd_bad(*page_dir)) {
printk("bad page directory entry %08lx\n", pgd_val(*page_dir));
pgd_clear(page_dir);
return 0;
}
page_middle = pmd_offset(page_dir,ptr);
if (pmd_none(*page_middle))
return 0;
if (pmd_bad(*page_middle)) {
printk("bad page middle entry %08lx\n", pmd_val(*page_middle));
pmd_clear(page_middle);
return 0;
}
pte = *pte_offset(page_middle,ptr);
if (!pte_present(pte))
return 0;
return pte_page(pte) + (ptr & ~PAGE_MASK);
}
/* mmlist_lock and vma->vm_mm->page_table_lock are held */
static inline void unuse_pgd(struct vm_area_struct * vma, pgd_t *dir,
unsigned long address, unsigned long size,
swp_entry_t entry, struct page* page)
{
pmd_t * pmd;
unsigned long offset, end;
if (pgd_none(*dir))
return;
if (pgd_bad(*dir)) {
pgd_ERROR(*dir);
pgd_clear(dir);
return;
}
pmd = pmd_offset(dir, address);
offset = address & PGDIR_MASK;
address &= ~PGDIR_MASK;
end = address + size;
if (end > PGDIR_SIZE)
end = PGDIR_SIZE;
if (address >= end)
BUG();
do {
unuse_pmd(vma, pmd, address, end - address, offset, entry,
page);
address = (address + PMD_SIZE) & PMD_MASK;
pmd++;
} while (address && (address < end));
}
static inline void iterate_pmd(pgd_t * dir, unsigned long address,
unsigned long size, pte_iterator_t op,
unsigned long arg)
{
pmd_t *pmd;
unsigned long end;
if (pgd_none(*dir))
return;
if (pgd_bad(*dir)) {
pgd_ERROR(*dir);
pgd_clear(dir);
return;
}
pmd = pmd_offset(dir, address);
address &= ~PGDIR_MASK;
end = address + size;
if (end > PGDIR_SIZE)
end = PGDIR_SIZE;
do {
iterate_pte(pmd, address, end - address, op, arg);
address = (address + PMD_SIZE) & PMD_MASK;
pmd++;
} while (address < end);
}
static inline int filemap_sync_pmd_range(pgd_t * pgd,
unsigned long address, unsigned long end,
struct vm_area_struct *vma, unsigned int flags)
{
pmd_t * pmd;
int error;
if (pgd_none(*pgd))
return 0;
if (pgd_bad(*pgd)) {
pgd_ERROR(*pgd);
pgd_clear(pgd);
return 0;
}
pmd = pmd_offset(pgd, address);
if ((address & PGDIR_MASK) != (end & PGDIR_MASK))
end = (address & PGDIR_MASK) + PGDIR_SIZE;
error = 0;
do {
error |= filemap_sync_pte_range(pmd, address, end, vma, flags);
address = (address + PMD_SIZE) & PMD_MASK;
pmd++;
} while (address && (address < end));
return error;
}
/*
* This function zeroes out partial mmap'ed pages at truncation time..
*/
static void partial_clear(struct vm_area_struct *vma, unsigned long address)
{
pgd_t *page_dir;
pmd_t *page_middle;
pte_t *page_table, pte;
page_dir = pgd_offset(vma->vm_mm, address);
if (pgd_none(*page_dir))
return;
if (pgd_bad(*page_dir)) {
printk("bad page table directory entry %p:[%lx]\n", page_dir, pgd_val(*page_dir));
pgd_clear(page_dir);
return;
}
page_middle = pmd_offset(page_dir, address);
if (pmd_none(*page_middle))
return;
if (pmd_bad(*page_middle)) {
printk("bad page table directory entry %p:[%lx]\n", page_dir, pgd_val(*page_dir));
pmd_clear(page_middle);
return;
}
page_table = pte_offset(page_middle, address);
pte = *page_table;
if (!pte_present(pte))
return;
flush_cache_page(vma, address);
address &= ~PAGE_MASK;
address += pte_page(pte);
if (address >= high_memory)
return;
memset((void *) address, 0, PAGE_SIZE - (address & ~PAGE_MASK));
flush_page_to_ram(pte_page(pte));
}
static inline int copy_pmd_range(pgd_t *dst_pgd, pgd_t *src_pgd, unsigned long address, unsigned long size, int cow)
{
pmd_t * src_pmd, * dst_pmd;
unsigned long end;
int error = 0;
if (pgd_none(*src_pgd))
return 0;
if (pgd_bad(*src_pgd)) {
printk("copy_pmd_range: bad pgd (%08lx)\n", pgd_val(*src_pgd));
pgd_clear(src_pgd);
return 0;
}
src_pmd = pmd_offset(src_pgd, address);
if (pgd_none(*dst_pgd)) {
if (!pmd_alloc(dst_pgd, 0))
return -ENOMEM;
}
dst_pmd = pmd_offset(dst_pgd, address);
address &= ~PGDIR_MASK;
end = address + size;
if (end > PGDIR_SIZE)
end = PGDIR_SIZE;
do {
error = copy_pte_range(dst_pmd++, src_pmd++, address, end - address, cow);
if (error)
break;
address = (address + PMD_SIZE) & PMD_MASK;
} while (address < end);
return error;
}
static void clear_pgd_entry(struct kvm *kvm, pgd_t *pgd, phys_addr_t addr)
{
pud_t *pud_table __maybe_unused = pud_offset(pgd, 0);
pgd_clear(pgd);
kvm_tlb_flush_vmid_ipa(kvm, addr);
pud_free(NULL, pud_table);
put_page(virt_to_page(pgd));
}
static int mem_write(struct inode * inode, struct file * file,char * buf, int count)
{
pgd_t *page_dir;
pmd_t *page_middle;
pte_t pte;
char * page;
struct task_struct * tsk;
unsigned long addr;
char *tmp;
int i;
if (count < 0)
return -EINVAL;
addr = file->f_pos;
tsk = get_task(inode->i_ino >> 16);
if (!tsk)
return -ESRCH;
tmp = buf;
while (count > 0) {
if (current->signal & ~current->blocked)
break;
page_dir = pgd_offset(tsk,addr);
if (pgd_none(*page_dir))
break;
if (pgd_bad(*page_dir)) {
printk("Bad page dir entry %08lx\n", pgd_val(*page_dir));
pgd_clear(page_dir);
break;
}
page_middle = pmd_offset(page_dir,addr);
if (pmd_none(*page_middle))
break;
if (pmd_bad(*page_middle)) {
printk("Bad page middle entry %08lx\n", pmd_val(*page_middle));
pmd_clear(page_middle);
break;
}
pte = *pte_offset(page_middle,addr);
if (!pte_present(pte))
break;
if (!pte_write(pte))
break;
page = (char *) pte_page(pte) + (addr & ~PAGE_MASK);
i = PAGE_SIZE-(addr & ~PAGE_MASK);
if (i > count)
i = count;
memcpy_fromfs(page, tmp, i);
addr += i;
tmp += i;
count -= i;
}
file->f_pos = addr;
if (tmp != buf)
return tmp-buf;
if (current->signal & ~current->blocked)
return -ERESTARTSYS;
return 0;
}
static inline void free_one_pgd(pgd_t * dir)
{
int j;
pmd_t * pmd;
if (pgd_none(*dir))
return;
if (pgd_bad(*dir)) {
printk("free_one_pgd: bad directory entry %08lx\n", pgd_val(*dir));
pgd_clear(dir);
return;
}
pmd = pmd_offset(dir, 0);
pgd_clear(dir);
for (j = 0; j < PTRS_PER_PMD ; j++)
free_one_pmd(pmd+j);
pmd_free(pmd);
}
static inline void free_one_pgd(pgd_t * dir)
{
int j;
pmd_t * pmd;
if (pgd_none(*dir))
return;
if (pgd_bad(*dir)) {
pgd_ERROR(*dir);
pgd_clear(dir);
return;
}
pmd = pmd_offset(dir, 0);
pgd_clear(dir);
for (j = 0; j < PTRS_PER_PMD ; j++)
free_one_pmd(pmd+j);
pmd_free(pmd);
}
/*
* This routine gets a long from any process space by following the page
* tables. NOTE! You should check that the long isn't on a page boundary,
* and that it is in the task area before calling this: this routine does
* no checking.
*/
static unsigned long get_long(struct task_struct * tsk,
struct vm_area_struct * vma, unsigned long addr)
{
pgd_t * pgdir;
pmd_t * pgmiddle;
pte_t * pgtable;
unsigned long page;
int fault;
repeat:
pgdir = pgd_offset(vma->vm_mm, addr);
if (pgd_none(*pgdir)) {
fault = handle_mm_fault(tsk, vma, addr, 0);
if (fault > 0)
goto repeat;
if (fault < 0)
force_sig(SIGKILL, tsk);
return 0;
}
if (pgd_bad(*pgdir)) {
printk("ptrace[1]: bad page directory %lx\n", pgd_val(*pgdir));
pgd_clear(pgdir);
return 0;
}
pgmiddle = pmd_offset(pgdir, addr);
if (pmd_none(*pgmiddle)) {
fault = handle_mm_fault(tsk, vma, addr, 0);
if (fault > 0)
goto repeat;
if (fault < 0)
force_sig(SIGKILL, tsk);
return 0;
}
if (pmd_bad(*pgmiddle)) {
printk("ptrace[3]: bad pmd %lx\n", pmd_val(*pgmiddle));
pmd_clear(pgmiddle);
return 0;
}
pgtable = pte_offset(pgmiddle, addr);
if (!pte_present(*pgtable)) {
fault = handle_mm_fault(tsk, vma, addr, 0);
if (fault > 0)
goto repeat;
if (fault < 0)
force_sig(SIGKILL, tsk);
return 0;
}
page = pte_page(*pgtable);
/* this is a hack for non-kernel-mapped video buffers and similar */
if (MAP_NR(page) >= max_mapnr)
return 0;
page += addr & ~PAGE_MASK;
return *(unsigned long *) page;
}
/*
* This routine puts a long into any process space by following the page
* tables. NOTE! You should check that the long isn't on a page boundary,
* and that it is in the task area before calling this: this routine does
* no checking.
*
* Now keeps R/W state of page so that a text page stays readonly
* even if a debugger scribbles breakpoints into it. -M.U-
*/
static void put_long(struct task_struct * tsk, struct vm_area_struct * vma, unsigned long addr,
unsigned long data)
{
pgd_t *pgdir;
pmd_t *pgmiddle;
pte_t *pgtable;
unsigned long page;
repeat:
pgdir = pgd_offset(vma->vm_mm, addr);
if (!pgd_present(*pgdir)) {
do_no_page(tsk, vma, addr, 1);
goto repeat;
}
if (pgd_bad(*pgdir)) {
printk("ptrace: bad page directory %08lx\n", pgd_val(*pgdir));
pgd_clear(pgdir);
return;
}
pgmiddle = pmd_offset(pgdir,addr);
if (pmd_none(*pgmiddle)) {
do_no_page(tsk, vma, addr, 1);
goto repeat;
}
if (pmd_bad(*pgmiddle)) {
printk("ptrace: bad page directory %08lx\n",
pmd_val(*pgmiddle));
pmd_clear(pgmiddle);
return;
}
pgtable = pte_offset(pgmiddle, addr);
if (!pte_present(*pgtable)) {
do_no_page(tsk, vma, addr, 1);
goto repeat;
}
page = pte_page(*pgtable);
if (!pte_write(*pgtable)) {
do_wp_page(tsk, vma, addr, 2);
goto repeat;
}
/* this is a hack for non-kernel-mapped video buffers and similar */
if (page < high_memory) {
*(unsigned long *) (page + (addr & ~PAGE_MASK)) = data;
flush_page_to_ram (page);
}
/* we're bypassing pagetables, so we have to set the dirty bit ourselves */
/* this should also re-instate whatever read-only mode there was before */
*pgtable = pte_mkdirty(mk_pte(page, vma->vm_page_prot));
flush_tlb_all();
}
/*
* __iounmap unmaps nearly everything, so be careful
* it doesn't free currently pointer/page tables anymore but it
* wans't used anyway and might be added later.
*/
void __iounmap(void *addr, unsigned long size)
{
unsigned long virtaddr = (unsigned long)addr;
pgd_t *pgd_dir;
pmd_t *pmd_dir;
pte_t *pte_dir;
while ((long)size > 0) {
pgd_dir = pgd_offset_k(virtaddr);
if (pgd_bad(*pgd_dir)) {
printk("iounmap: bad pgd(%08lx)\n", pgd_val(*pgd_dir));
pgd_clear(pgd_dir);
return;
}
pmd_dir = pmd_offset(pgd_dir, virtaddr);
if (CPU_IS_020_OR_030) {
int pmd_off = (virtaddr/PTRTREESIZE) & 15;
int pmd_type = pmd_dir->pmd[pmd_off] & _DESCTYPE_MASK;
if (pmd_type == _PAGE_PRESENT) {
pmd_dir->pmd[pmd_off] = 0;
virtaddr += PTRTREESIZE;
size -= PTRTREESIZE;
continue;
} else if (pmd_type == 0)
continue;
}
if (pmd_bad(*pmd_dir)) {
printk("iounmap: bad pmd (%08lx)\n", pmd_val(*pmd_dir));
pmd_clear(pmd_dir);
return;
}
pte_dir = pte_offset_kernel(pmd_dir, virtaddr);
pte_val(*pte_dir) = 0;
virtaddr += PAGE_SIZE;
size -= PAGE_SIZE;
}
flush_tlb_all();
}
void oleolevm_free_pgd_range(struct mmu_gather *tlb,
struct vm_area_struct *vma,
unsigned long addr, unsigned long end,
unsigned long floor,
unsigned long ceiling)
{
unsigned long flags;
oleole_guest_system_t *gsys;
addr &= PMD_MASK;
for ( ; addr < end ; addr += PMD_SIZE) {
pgd_t *pgd;
pud_t *pud;
struct page *page;
pgd = pgd_offset(tlb->mm, addr);
if (oleole_pgd_none_or_clear_bad(pgd))
continue;
free_pud_range(pgd);
pud = pud_offset(pgd, 0);
page = virt_to_page(pud);
__free_page(page);
pgd_clear(pgd);
}
/* */
gsys = vma->vm_private_data;
if (gsys) {
spin_lock_irqsave(&gsys->lock, flags);
vma->vm_private_data = NULL;
gsys->vma = NULL;
spin_unlock_irqrestore(&gsys->lock, flags);
}
tlb->need_flush = 1;
}
/*
* This routine puts a long into any process space by following the page
* tables. NOTE! You should check that the long isn't on a page boundary,
* and that it is in the task area before calling this: this routine does
* no checking.
*
* Now keeps R/W state of page so that a text page stays readonly
* even if a debugger scribbles breakpoints into it. -M.U-
*/
static void put_long(struct vm_area_struct * vma, unsigned long addr,
unsigned long data)
{
pgd_t *pgdir;
pte_t *pgtable;
unsigned long page;
repeat:
pgdir = PAGE_DIR_OFFSET(vma->vm_task, addr);
if (!pgd_present(*pgdir)) {
do_no_page(vma, addr, 1);
goto repeat;
}
if (pgd_bad(*pgdir)) {
printk("ptrace: bad page directory %08lx\n", pgd_val(*pgdir));
pgd_clear(pgdir);
return;
}
pgtable = (pte_t *) (PAGE_PTR(addr) + pgd_page(*pgdir));
if (!pte_present(*pgtable)) {
do_no_page(vma, addr, 1);
goto repeat;
}
page = pte_page(*pgtable);
if (!pte_write(*pgtable)) {
do_wp_page(vma, addr, 1);
goto repeat;
}
/* this is a hack for non-kernel-mapped video buffers and similar */
if (page < high_memory) {
page += addr & ~PAGE_MASK;
*(unsigned long *) page = data;
}
/* we're bypassing pagetables, so we have to set the dirty bit ourselves */
/* this should also re-instate whatever read-only mode there was before */
*pgtable = pte_mkdirty(mk_pte(page, vma->vm_page_prot));
invalidate();
}
void pgd_free(struct mm_struct *mm, pgd_t *pgd_base)
{
unsigned long flags;
pgd_t *pgd;
pud_t *pud;
pmd_t *pmd;
pgtable_t pte;
if (!pgd_base)
return;
pgd = pgd_base + pgd_index(0);
if (pgd_none_or_clear_bad(pgd))
goto no_pgd;
pud = pud_offset(pgd + pgd_index(fcse_va_to_mva(mm, 0)), 0);
if (pud_none_or_clear_bad(pud))
goto no_pud;
pmd = pmd_offset(pud, 0);
if (pmd_none_or_clear_bad(pmd))
goto no_pmd;
pte = pmd_pgtable(*pmd);
pmd_clear(pmd);
pte_free(mm, pte);
no_pmd:
pud_clear(pud);
pmd_free(mm, pmd);
no_pud:
pgd_clear(pgd);
pud_free(mm, pud);
no_pgd:
pgd_list_lock(flags);
pgd_list_del(pgd);
pgd_list_unlock(flags);
free_pages((unsigned long) pgd_base, 2);
}
static inline void free_area_pmd(pgd_t * dir, unsigned long address, unsigned long size)
{
pmd_t * pmd;
unsigned long end;
if (pgd_none(*dir))
return;
if (pgd_bad(*dir)) {
printk("free_area_pmd: bad pgd (%08lx)\n", pgd_val(*dir));
pgd_clear(dir);
return;
}
pmd = pmd_offset(dir, address);
address &= ~PGDIR_MASK;
end = address + size;
if (end > PGDIR_SIZE)
end = PGDIR_SIZE;
while (address < end) {
free_area_pte(pmd, address, end - address);
address = (address + PMD_SIZE) & PMD_MASK;
pmd++;
}
}
static void __init clear_pgds(unsigned long start,
unsigned long end)
{
pgd_t *pgd;
/* See comment in kasan_init() */
unsigned long pgd_end = end & PGDIR_MASK;
for (; start < pgd_end; start += PGDIR_SIZE) {
pgd = pgd_offset_k(start);
/*
* With folded p4d, pgd_clear() is nop, use p4d_clear()
* instead.
*/
if (pgtable_l5_enabled())
pgd_clear(pgd);
else
p4d_clear(p4d_offset(pgd, start));
}
pgd = pgd_offset_k(start);
for (; start < end; start += P4D_SIZE)
p4d_clear(p4d_offset(pgd, start));
}
static inline void statm_pmd_range(pgd_t * pgd, unsigned long address, unsigned long size,
int * pages, int * shared, int * dirty, int * total)
{
pmd_t * pmd;
unsigned long end;
if (pgd_none(*pgd))
return;
if (pgd_bad(*pgd)) {
printk("statm_pmd_range: bad pgd (%08lx)\n", pgd_val(*pgd));
pgd_clear(pgd);
return;
}
pmd = pmd_offset(pgd, address);
address &= ~PGDIR_MASK;
end = address + size;
if (end > PGDIR_SIZE)
end = PGDIR_SIZE;
do {
statm_pte_range(pmd, address, end - address, pages, shared, dirty, total);
address = (address + PMD_SIZE) & PMD_MASK;
pmd++;
} while (address < end);
}
