static int move_page_tables(struct mm_struct * mm,
unsigned long new_addr, unsigned long old_addr, unsigned long len)
{
unsigned long offset = len;
flush_cache_range(mm, old_addr, old_addr + len);
flush_tlb_range(mm, old_addr, old_addr + len);
/*
* This is not the clever way to do this, but we're taking the
* easy way out on the assumption that most remappings will be
* only a few pages.. This also makes error recovery easier.
*/
while (offset) {
offset -= PAGE_SIZE;
if (move_one_page(mm, old_addr + offset, new_addr + offset))
goto oops_we_failed;
}
return 0;
/*
* Ok, the move failed because we didn't have enough pages for
* the new page table tree. This is unlikely, but we have to
* take the possibility into account. In that case we just move
* all the pages back (this will work, because we still have
* the old page tables)
*/
oops_we_failed:
flush_cache_range(mm, new_addr, new_addr + len);
while ((offset += PAGE_SIZE) < len)
move_one_page(mm, new_addr + offset, old_addr + offset);
zap_page_range(mm, new_addr, new_addr + len);
flush_tlb_range(mm, new_addr, new_addr + len);
return -1;
}
/*
* copy one vm_area from one task to the other. Assumes the page tables
* already present in the new task to be cleared in the whole range
* covered by this vma.
*/
int copy_page_range(struct mm_struct *dst, struct mm_struct *src,
struct vm_area_struct *vma)
{
pgd_t * src_pgd, * dst_pgd;
unsigned long address = vma->vm_start;
unsigned long end = vma->vm_end;
int error = 0, cow;
cow = (vma->vm_flags & (VM_SHARED | VM_WRITE)) == VM_WRITE;
src_pgd = pgd_offset(src, address);
dst_pgd = pgd_offset(dst, address);
flush_cache_range(src, vma->vm_start, vma->vm_end);
flush_cache_range(dst, vma->vm_start, vma->vm_end);
while (address < end) {
error = copy_pmd_range(dst_pgd++, src_pgd++, address, end - address, cow);
if (error)
break;
address = (address + PGDIR_SIZE) & PGDIR_MASK;
#ifdef CONFIG_BESTA
if (!address) break; /* unsigned overflow */
#endif
}
/* Note that the src ptes get c-o-w treatment, so they change too. */
flush_tlb_range(src, vma->vm_start, vma->vm_end);
flush_tlb_range(dst, vma->vm_start, vma->vm_end);
return error;
}
static void vmtruncate_list(struct vm_area_struct *mpnt,
unsigned long pgoff, unsigned long partial)
{
do {
struct mm_struct *mm = mpnt->vm_mm;
unsigned long start = mpnt->vm_start;
unsigned long end = mpnt->vm_end;
unsigned long len = end - start;
unsigned long diff;
/* mapping wholly truncated? */
if (mpnt->vm_pgoff >= pgoff) {
flush_cache_range(mm, start, end);
zap_page_range(mm, start, len);
flush_tlb_range(mm, start, end);
continue;
}
/* mapping wholly unaffected? */
len = len >> PAGE_SHIFT;
diff = pgoff - mpnt->vm_pgoff;
if (diff >= len)
continue;
/* Ok, partially affected.. */
start += diff << PAGE_SHIFT;
len = (len - diff) << PAGE_SHIFT;
flush_cache_range(mm, start, end);
zap_page_range(mm, start, len);
flush_tlb_range(mm, start, end);
} while ((mpnt = mpnt->vm_next_share) != NULL);
}
int zeromap_page_range(unsigned long address, unsigned long size, pgprot_t prot)
{
int error = 0;
pgd_t * dir;
unsigned long beg = address;
unsigned long end = address + size;
pte_t zero_pte;
zero_pte = pte_wrprotect(mk_pte(ZERO_PAGE, prot));
dir = pgd_offset(current->mm, address);
flush_cache_range(current->mm, beg, end);
while (address < end) {
pmd_t *pmd = pmd_alloc(dir, address);
error = -ENOMEM;
if (!pmd)
break;
error = zeromap_pmd_range(pmd, address, end - address, zero_pte);
if (error)
break;
address = (address + PGDIR_SIZE) & PGDIR_MASK;
#ifdef CONFIG_BESTA
if (!address) break; /* unsigned overflow */
#endif
dir++;
}
flush_tlb_range(current->mm, beg, end);
return error;
}
// init_mm.page_table_lock must be held before calling!
void pram_writeable(void * vaddr, unsigned long size, int rw)
{
unsigned long addr = (unsigned long)vaddr & PAGE_MASK;
unsigned long end = (unsigned long)vaddr + size;
unsigned long start = addr;
do {
pram_page_writeable(addr, rw);
addr += PAGE_SIZE;
} while (addr && (addr < end));
/*
* FIXME: can't use flush_tlb_page/range() until these
* routines support flushing memory regions owned by
* init_mm (so far only PPC versions).
*/
#if 0
if (end <= start + PAGE_SIZE)
flush_tlb_page(find_vma(&init_mm,start), start);
else
flush_tlb_range(&init_mm, start, end);
#else
flush_tlb_all();
#endif
}
int remap_page_range(unsigned long from, unsigned long offset, unsigned long size, pgprot_t prot)
{
int error = 0;
pgd_t * dir;
unsigned long beg = from;
unsigned long end = from + size;
offset -= from;
dir = pgd_offset(current->mm, from);
flush_cache_range(current->mm, beg, end);
while (from < end) {
pmd_t *pmd = pmd_alloc(dir, from);
error = -ENOMEM;
if (!pmd)
break;
error = remap_pmd_range(pmd, from, end - from, offset + from, prot);
if (error)
break;
from = (from + PGDIR_SIZE) & PGDIR_MASK;
#ifdef CONFIG_BESTA
if (!from) break; /* unsigned overflow */
#endif
dir++;
}
flush_tlb_range(current->mm, beg, end);
return error;
}
void pmdp_invalidate(struct vm_area_struct *vma, unsigned long address,
pmd_t *pmdp)
{
pmd_t entry = *pmdp;
set_pmd_at(vma->vm_mm, address, pmdp, pmd_mknotpresent(entry));
flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
}
void flush_tlb_mm(struct mm_struct *mm)
{
unsigned long shift, base = 0;
#ifdef ARM_PID_RELOC
struct vm_area_struct *vma;
shift = 25;
base = ((unsigned long)mm->context.pid << shift);
#else
shift = TASK_SHIFT;
#endif
#if defined(CONFIG_CELL)
okl4_unmap_page_size(&mm->context, base, shift);
#elif defined(CONFIG_IGUANA)
{
L4_Fpage_t fpage;
fpage = L4_FpageLog2(base, shift);
eas_unmap(mm->context.eas, fpage);
}
#endif
#ifdef ARM_PID_RELOC
/* Walk through the list of VMAs and flush those
* that are outside the PID relocation region
*/
vma = mm->mmap;
while(vma) {
if (vma->vm_start >= 0x2000000UL)
flush_tlb_range(vma, vma->vm_start, vma->vm_end);
vma = vma->vm_next;
}
#endif
}
int io_remap_page_range(unsigned long from, unsigned long offset, unsigned long size, pgprot_t prot, int space)
{
int error = 0;
pgd_t * dir;
unsigned long beg = from;
unsigned long end = from + size;
prot = __pgprot(pg_iobits);
offset -= from;
dir = pgd_offset(current->mm, from);
flush_cache_range(current->mm, beg, end);
while (from < end) {
pmd_t *pmd = pmd_alloc(dir, from);
error = -ENOMEM;
if (!pmd)
break;
error = io_remap_pmd_range(pmd, from, end - from, offset + from, prot, space);
if (error)
break;
from = (from + PGDIR_SIZE) & PGDIR_MASK;
dir++;
}
flush_tlb_range(current->mm, beg, end);
return error;
}
int
vmap_page_range (unsigned long from, unsigned long size, unsigned long vaddr)
{
int error = 0;
pgd_t * dir;
unsigned long beg = from;
unsigned long end = from + size;
vaddr -= from;
dir = pgd_offset(current->mm, from);
flush_cache_range(current->mm, beg, end);
while (from < end) {
pmd_t *pmd = pmd_alloc(dir, from);
error = -ENOMEM;
if (!pmd)
break;
error = vmap_pmd_range(pmd, from, end - from, vaddr + from);
if (error)
break;
from = (from + PGDIR_SIZE) & PGDIR_MASK;
dir++;
}
flush_tlb_range(current->mm, beg, end);
return error;
}
/*
* Changing some bits of contiguous entries requires us to follow a
* Break-Before-Make approach, breaking the whole contiguous set
* before we can change any entries. See ARM DDI 0487A.k_iss10775,
* "Misprogramming of the Contiguous bit", page D4-1762.
*
* This helper performs the break step.
*/
static pte_t get_clear_flush(struct mm_struct *mm,
unsigned long addr,
pte_t *ptep,
unsigned long pgsize,
unsigned long ncontig)
{
struct vm_area_struct vma = { .vm_mm = mm };
pte_t orig_pte = huge_ptep_get(ptep);
bool valid = pte_valid(orig_pte);
unsigned long i, saddr = addr;
for (i = 0; i < ncontig; i++, addr += pgsize, ptep++) {
pte_t pte = ptep_get_and_clear(mm, addr, ptep);
/*
* If HW_AFDBM is enabled, then the HW could turn on
* the dirty bit for any page in the set, so check
* them all. All hugetlb entries are already young.
*/
if (pte_dirty(pte))
orig_pte = pte_mkdirty(orig_pte);
}
if (valid)
flush_tlb_range(&vma, saddr, addr);
return orig_pte;
}
/*
* Changing some bits of contiguous entries requires us to follow a
* Break-Before-Make approach, breaking the whole contiguous set
* before we can change any entries. See ARM DDI 0487A.k_iss10775,
* "Misprogramming of the Contiguous bit", page D4-1762.
*
* This helper performs the break step.
*/
static pte_t get_clear_flush(struct mm_struct *mm,
unsigned long addr,
pte_t *ptep,
unsigned long pgsize,
unsigned long ncontig)
{
pte_t orig_pte = huge_ptep_get(ptep);
bool valid = pte_valid(orig_pte);
unsigned long i, saddr = addr;
for (i = 0; i < ncontig; i++, addr += pgsize, ptep++) {
pte_t pte = ptep_get_and_clear(mm, addr, ptep);
/*
* If HW_AFDBM is enabled, then the HW could turn on
* the dirty or accessed bit for any page in the set,
* so check them all.
*/
if (pte_dirty(pte))
orig_pte = pte_mkdirty(orig_pte);
if (pte_young(pte))
orig_pte = pte_mkyoung(orig_pte);
}
if (valid) {
struct vm_area_struct vma = TLB_FLUSH_VMA(mm, 0);
flush_tlb_range(&vma, saddr, addr);
}
return orig_pte;
}
int io_remap_page_range(struct vm_area_struct *vma, unsigned long from, unsigned long offset, unsigned long size, pgprot_t prot, int space)
{
int error = 0;
pgd_t * dir;
unsigned long beg = from;
unsigned long end = from + size;
struct mm_struct *mm = vma->vm_mm;
prot = __pgprot(pg_iobits);
offset -= from;
dir = pgd_offset(mm, from);
flush_cache_range(vma, beg, end);
spin_lock(&mm->page_table_lock);
while (from < end) {
pmd_t *pmd = pmd_alloc(current->mm, dir, from);
error = -ENOMEM;
if (!pmd)
break;
error = io_remap_pmd_range(pmd, from, end - from, offset + from, prot, space);
if (error)
break;
from = (from + PGDIR_SIZE) & PGDIR_MASK;
dir++;
}
spin_unlock(&mm->page_table_lock);
flush_tlb_range(vma, beg, end);
return error;
}
/* Note: this is only safe if the mm semaphore is held when called. */
int remap_page_range(unsigned long from, unsigned long phys_addr, unsigned long size, pgprot_t prot)
{
int error = 0;
pgd_t * dir;
unsigned long beg = from;
unsigned long end = from + size;
phys_addr -= from;
dir = pgd_offset(current->mm, from);
flush_cache_range(current->mm, beg, end);
if (from >= end)
BUG();
do {
pmd_t *pmd = pmd_alloc(dir, from);
error = -ENOMEM;
if (!pmd)
break;
error = remap_pmd_range(pmd, from, end - from, phys_addr + from, prot);
if (error)
break;
from = (from + PGDIR_SIZE) & PGDIR_MASK;
dir++;
} while (from && (from < end));
flush_tlb_range(current->mm, beg, end);
return error;
}
int zeromap_page_range(unsigned long address, unsigned long size, pgprot_t prot)
{
int error = 0;
pgd_t * dir;
unsigned long beg = address;
unsigned long end = address + size;
dir = pgd_offset(current->mm, address);
flush_cache_range(current->mm, beg, end);
if (address >= end)
BUG();
do {
pmd_t *pmd = pmd_alloc(dir, address);
error = -ENOMEM;
if (!pmd)
break;
error = zeromap_pmd_range(pmd, address, end - address, prot);
if (error)
break;
address = (address + PGDIR_SIZE) & PGDIR_MASK;
dir++;
} while (address && (address < end));
flush_tlb_range(current->mm, beg, end);
return error;
}
int io_remap_pfn_range(struct vm_area_struct *vma, unsigned long from,
unsigned long pfn, unsigned long size, pgprot_t prot)
{
int error = 0;
pgd_t * dir;
unsigned long beg = from;
unsigned long end = from + size;
struct mm_struct *mm = vma->vm_mm;
int space = GET_IOSPACE(pfn);
unsigned long offset = GET_PFN(pfn) << PAGE_SHIFT;
/* See comment in mm/memory.c remap_pfn_range */
vma->vm_flags |= VM_IO | VM_RESERVED | VM_PFNMAP;
vma->vm_pgoff = (offset >> PAGE_SHIFT) |
((unsigned long)space << 28UL);
offset -= from;
dir = pgd_offset(mm, from);
flush_cache_range(vma, beg, end);
while (from < end) {
pmd_t *pmd = pmd_alloc(mm, dir, from);
error = -ENOMEM;
if (!pmd)
break;
error = io_remap_pmd_range(mm, pmd, from, end - from, offset + from, prot, space);
if (error)
break;
from = (from + PGDIR_SIZE) & PGDIR_MASK;
dir++;
}
flush_tlb_range(vma, beg, end);
return error;
}
static int filemap_sync(struct vm_area_struct * vma, unsigned long address,
size_t size, unsigned int flags)
{
pgd_t * dir;
unsigned long end = address + size;
int error = 0;
/* Aquire the lock early; it may be possible to avoid dropping
* and reaquiring it repeatedly.
*/
spin_lock(&vma->vm_mm->page_table_lock);
dir = pgd_offset(vma->vm_mm, address);
flush_cache_range(vma, address, end);
if (address >= end)
BUG();
do {
error |= filemap_sync_pmd_range(dir, address, end, vma, flags);
address = (address + PGDIR_SIZE) & PGDIR_MASK;
dir++;
} while (address && (address < end));
flush_tlb_range(vma, end - size, end);
spin_unlock(&vma->vm_mm->page_table_lock);
return error;
}
void pmdp_splitting_flush(struct vm_area_struct *vma, unsigned long address,
pmd_t *pmdp)
{
pmd_t pmd = pmd_mksplitting(*pmdp);
VM_BUG_ON(address & ~HPAGE_PMD_MASK);
set_pmd_at(vma->vm_mm, address, pmdp, pmd);
/* tlb flush only to serialize against gup-fast */
flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
}
pmd_t pmdp_clear_flush(struct vm_area_struct *vma, unsigned long address,
pmd_t *pmdp)
{
pmd_t pmd;
VM_BUG_ON(address & ~HPAGE_PMD_MASK);
pmd = pmdp_get_and_clear(vma->vm_mm, address, pmdp);
flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
return pmd;
}
pmd_t pmdp_splitting_flush(struct vm_area_struct *vma, unsigned long address,
pmd_t *pmdp)
{
pmd_t pmd = pmd_mksplitting(*pmdp);
VM_BUG_ON(address & ~HPAGE_PMD_MASK);
set_pmd_at(vma->vm_mm, address, pmdp, pmd);
flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
}
void pmdp_invalidate(struct vm_area_struct *vma, unsigned long address,
pmd_t *pmdp)
{
pmd_t entry = *pmdp;
pmd_val(entry) &= ~_PAGE_VALID;
set_pmd_at(vma->vm_mm, address, pmdp, entry);
flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
}
void
kernel_set_cachemode(unsigned long vaddr, unsigned long size, int what)
{
switch (what) {
case IOMAP_FULL_CACHING:
iterate_pages(vaddr, size, set_cached, 0);
flush_tlb_range(&init_mm, vaddr, size);
break;
case IOMAP_NOCACHE_SER:
iterate_pages(vaddr, size, set_uncached, 0);
flush_tlb_range(&init_mm, vaddr, size);
break;
default:
printk(KERN_CRIT
"kernel_set_cachemode mode %d not understood\n",
what);
break;
}
}
/*
* We use this to invalidate a pmdp entry before switching from a
* hugepte to regular pmd entry.
*/
void pmdp_invalidate(struct vm_area_struct *vma, unsigned long address,
pmd_t *pmdp)
{
pmd_hugepage_update(vma->vm_mm, address, pmdp, _PAGE_PRESENT, 0);
flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
/*
* This ensures that generic code that rely on IRQ disabling
* to prevent a parallel THP split work as expected.
*/
kick_all_cpus_sync();
}
/*
* For fun, we are using the MMU for this.
*/
static inline size_t read_zero_pagealigned(char * buf, size_t size)
{
struct mm_struct *mm;
struct vm_area_struct * vma;
unsigned long addr=(unsigned long)buf;
mm = current->mm;
/* Oops, this was forgotten before. -ben */
down(&mm->mmap_sem);
/* For private mappings, just map in zero pages. */
for (vma = find_vma(mm, addr); vma; vma = vma->vm_next) {
unsigned long count;
if (vma->vm_start > addr || (vma->vm_flags & VM_WRITE) == 0)
goto out_up;
if (vma->vm_flags & VM_SHARED)
break;
count = vma->vm_end - addr;
if (count > size)
count = size;
flush_cache_range(mm, addr, addr + count);
zap_page_range(mm, addr, count);
zeromap_page_range(addr, count, PAGE_COPY);
flush_tlb_range(mm, addr, addr + count);
size -= count;
buf += count;
addr += count;
if (size == 0)
goto out_up;
}
up(&mm->mmap_sem);
/* The shared case is hard. Let's do the conventional zeroing. */
do {
unsigned long unwritten = clear_user(buf, PAGE_SIZE);
if (unwritten)
return size + unwritten - PAGE_SIZE;
if (current->need_resched)
schedule();
buf += PAGE_SIZE;
size -= PAGE_SIZE;
} while (size);
return size;
out_up:
up(&mm->mmap_sem);
return size;
}
int pmdp_clear_flush_young(struct vm_area_struct *vma,
unsigned long address, pmd_t *pmdp)
{
int young;
#ifndef CONFIG_TRANSPARENT_HUGEPAGE
BUG();
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
VM_BUG_ON(address & ~HPAGE_PMD_MASK);
young = pmdp_test_and_clear_young(vma, address, pmdp);
if (young)
flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
return young;
}
/*
* Changing some bits of contiguous entries requires us to follow a
* Break-Before-Make approach, breaking the whole contiguous set
* before we can change any entries. See ARM DDI 0487A.k_iss10775,
* "Misprogramming of the Contiguous bit", page D4-1762.
*
* This helper performs the break step for use cases where the
* original pte is not needed.
*/
static void clear_flush(struct mm_struct *mm,
unsigned long addr,
pte_t *ptep,
unsigned long pgsize,
unsigned long ncontig)
{
struct vm_area_struct vma = { .vm_mm = mm };
unsigned long i, saddr = addr;
for (i = 0; i < ncontig; i++, addr += pgsize, ptep++)
pte_clear(mm, addr, ptep);
flush_tlb_range(&vma, saddr, addr);
}
int pmdp_set_access_flags(struct vm_area_struct *vma,
unsigned long address, pmd_t *pmdp,
pmd_t entry, int dirty)
{
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
int changed = !pmd_same(*pmdp, entry);
VM_BUG_ON(address & ~HPAGE_PMD_MASK);
if (changed) {
set_pmd_at(vma->vm_mm, address, pmdp, entry);
flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
}
return changed;
#else /* CONFIG_TRANSPARENT_HUGEPAGE */
BUG();
return 0;
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
}
/*
* This routine is called from the page fault handler to remove a
* range of active mappings at this point
*/
void
remove_mapping (struct task_struct *task, unsigned long start, unsigned long end)
{
unsigned long beg = start;
pgd_t *dir;
down (&task->mm->mmap_sem);
dir = pgd_offset (task->mm, start);
flush_cache_range (task->mm, beg, end);
while (start < end){
remove_mapping_pmd_range (dir, start, end - start);
start = (start + PGDIR_SIZE) & PGDIR_MASK;
dir++;
}
flush_tlb_range (task->mm, beg, end);
up (&task->mm->mmap_sem);
}
pmd_t pmdp_collapse_flush(struct vm_area_struct *vma, unsigned long address,
pmd_t *pmdp)
{
/*
* pmd and hugepage pte format are same. So we could
* use the same function.
*/
pmd_t pmd;
VM_BUG_ON(address & ~HPAGE_PMD_MASK);
VM_BUG_ON(pmd_trans_huge(*pmdp));
pmd = pmdp_huge_get_and_clear(vma->vm_mm, address, pmdp);
/* collapse entails shooting down ptes not pmd */
flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
return pmd;
}
/*
* remove user pages in a given range.
*/
int zap_page_range(struct mm_struct *mm, unsigned long address, unsigned long size)
{
pgd_t * dir;
unsigned long end = address + size;
dir = pgd_offset(mm, address);
flush_cache_range(mm, end - size, end);
while (address < end) {
zap_pmd_range(dir, address, end - address);
address = (address + PGDIR_SIZE) & PGDIR_MASK;
#ifdef CONFIG_BESTA
if (!address) break; /* unsigned overflow */
#endif
dir++;
}
flush_tlb_range(mm, end - size, end);
return 0;
}