static void set_pte_phys(unsigned long addr, unsigned long phys, pgprot_t prot)
{
pgd_t *pgd;
pud_t *pud;
pmd_t *pmd;
pte_t *pte;
pgd = pgd_offset_k(addr);
if (pgd_none(*pgd)) {
pgd_ERROR(*pgd);
return;
}
pud = pud_alloc(NULL, pgd, addr);
if (unlikely(!pud)) {
pud_ERROR(*pud);
return;
}
pmd = pmd_alloc(NULL, pud, addr);
if (unlikely(!pmd)) {
pmd_ERROR(*pmd);
return;
}
pte = pte_offset_kernel(pmd, addr);
if (!pte_none(*pte)) {
pte_ERROR(*pte);
return;
}
set_pte(pte, pfn_pte(phys >> PAGE_SHIFT, prot));
local_flush_tlb_one(get_asid(), addr);
}
void free_pgd_slow(pgd_t *pgd)
{
pmd_t *pmd;
pte_t *pte;
if (!pgd)
return;
/* pgd is always present and good */
pmd = (pmd_t *)pgd;
if (pmd_none(*pmd))
goto free;
if (pmd_bad(*pmd)) {
pmd_ERROR(*pmd);
pmd_clear(pmd);
goto free;
}
pte = pte_offset(pmd, 0);
pmd_clear(pmd);
pte_free(pte);
pmd_free(pmd);
free:
free_pages((unsigned long) pgd, 2);
}
void free_pgd_slow(struct mm_struct *mm, pgd_t *pgd)
{
pmd_t *pmd;
pgtable_t pte;
if (!pgd)
return;
/* pgd is always present and good */
pmd = pmd_off(pgd, 0);
if (pmd_none(*pmd))
goto free;
if (pmd_bad(*pmd)) {
pmd_ERROR(*pmd);
pmd_clear(pmd);
goto free;
}
pte = pmd_pgtable(*pmd);
pmd_clear(pmd);
pte_free(mm, pte);
pmd_free(mm, pmd);
free:
free_pages((unsigned long) pgd, 2);
}
/* Unmap a kernel mapping if it exists. This is useful to avoid prefetches
from the CPU leading to inconsistent cache lines. address and size
must be aligned to 2MB boundaries.
Does nothing when the mapping doesn't exist. */
void __init clear_kernel_mapping(unsigned long address, unsigned long size)
{
unsigned long end = address + size;
BUG_ON(address & ~LARGE_PAGE_MASK);
BUG_ON(size & ~LARGE_PAGE_MASK);
for (; address < end; address += LARGE_PAGE_SIZE) {
pgd_t *pgd = pgd_offset_k(address);
pud_t *pud;
pmd_t *pmd;
if (pgd_none(*pgd))
continue;
pud = pud_offset(pgd, address);
if (pud_none(*pud))
continue;
pmd = pmd_offset(pud, address);
if (!pmd || pmd_none(*pmd))
continue;
if (0 == (pmd_val(*pmd) & _PAGE_PSE)) {
/* Could handle this, but it should not happen currently. */
printk(KERN_ERR
"clear_kernel_mapping: mapping has been split. will leak memory\n");
pmd_ERROR(*pmd);
}
set_pmd(pmd, __pmd(0));
}
__flush_tlb_all();
}
static pte_t *__get_pte_phys(unsigned long addr)
{
pgd_t *pgd;
pud_t *pud;
pmd_t *pmd;
pgd = pgd_offset_k(addr);
if (pgd_none(*pgd)) {
pgd_ERROR(*pgd);
return NULL;
}
pud = pud_alloc(NULL, pgd, addr);
if (unlikely(!pud)) {
pud_ERROR(*pud);
return NULL;
}
pmd = pmd_alloc(NULL, pud, addr);
if (unlikely(!pmd)) {
pmd_ERROR(*pmd);
return NULL;
}
return pte_offset_kernel(pmd, addr);
}
static inline void iterate_pte(pmd_t * pmd, unsigned long address,
unsigned long size, pte_iterator_t op,
unsigned long arg)
{
pte_t *pte;
unsigned long end;
if (pmd_none(*pmd))
return;
if (pmd_bad(*pmd)) {
pmd_ERROR(*pmd);
pmd_clear(pmd);
return;
}
pte = pte_offset(pmd, address);
address &= ~PMD_MASK;
end = address + size;
if (end > PMD_SIZE)
end = PMD_SIZE;
do {
op(pte, arg);
address += PAGE_SIZE;
pte++;
} while (address < end);
}
static int filemap_sync_pte_range(pmd_t * pmd,
unsigned long address, unsigned long end,
struct vm_area_struct *vma, unsigned int flags)
{
pte_t *pte;
int error;
if (pmd_none(*pmd))
return 0;
if (pmd_bad(*pmd)) {
pmd_ERROR(*pmd);
pmd_clear(pmd);
return 0;
}
pte = pte_offset_map(pmd, address);
if ((address & PMD_MASK) != (end & PMD_MASK))
end = (address & PMD_MASK) + PMD_SIZE;
error = 0;
do {
error |= filemap_sync_pte(pte, vma, address, flags);
address += PAGE_SIZE;
pte++;
} while (address && (address < end));
pte_unmap(pte - 1);
return error;
}
static inline int zap_pte_range(struct mm_struct *mm, pmd_t * pmd, unsigned long address, unsigned long size)
{
pte_t * pte;
int freed;
if (pmd_none(*pmd))
return 0;
if (pmd_bad(*pmd)) {
pmd_ERROR(*pmd);
pmd_clear(pmd);
return 0;
}
pte = pte_offset(pmd, address);
address &= ~PMD_MASK;
if (address + size > PMD_SIZE)
size = PMD_SIZE - address;
size >>= PAGE_SHIFT;
freed = 0;
for (;;) {
pte_t page;
if (!size)
break;
page = ptep_get_and_clear(pte);
pte++;
size--;
if (pte_none(page))
continue;
freed += free_pte(page);
}
return freed;
}
/* mmlist_lock and vma->vm_mm->page_table_lock are held */
static inline void unuse_pmd(struct vm_area_struct * vma, pmd_t *dir,
unsigned long address, unsigned long size, unsigned long offset,
swp_entry_t entry, struct page* page)
{
pte_t * pte;
unsigned long end;
if (pmd_none(*dir))
return;
if (pmd_bad(*dir)) {
pmd_ERROR(*dir);
pmd_clear(dir);
return;
}
pte = pte_offset(dir, address);
offset += address & PMD_MASK;
address &= ~PMD_MASK;
end = address + size;
if (end > PMD_SIZE)
end = PMD_SIZE;
do {
unuse_pte(vma, offset+address-vma->vm_start, pte, entry, page);
address += PAGE_SIZE;
pte++;
} while (address && (address < end));
}
void free_pgd_slow(struct mm_struct *mm, pgd_t *pgd)
{
unsigned long flags;
pmd_t *pmd;
pgtable_t pte;
if (!pgd)
return;
/* pgd is always present and good */
pmd = pmd_off(pgd + pgd_index(fcse_va_to_mva(mm, 0)), 0);
if (pmd_none(*pmd))
goto free;
if (pmd_bad(*pmd)) {
pmd_ERROR(*pmd);
pmd_clear(pmd);
goto free;
}
pte = pmd_pgtable(*pmd);
pmd_clear(pmd);
pte_free(mm, pte);
pmd_free(mm, pmd);
free:
pgd_list_lock(flags);
pgd_list_del(pgd);
pgd_list_unlock(flags);
free_pages((unsigned long) pgd, 2);
}
static inline pte_t *get_one_pte(struct mm_struct *mm, unsigned long addr)
{
pgd_t * pgd;
pmd_t * pmd;
pte_t * pte = NULL;
pgd = pgd_offset(mm, addr);
if (pgd_none(*pgd))
goto end;
if (pgd_bad(*pgd)) {
pgd_ERROR(*pgd);
pgd_clear(pgd);
goto end;
}
pmd = pmd_offset(pgd, addr);
if (pmd_none(*pmd))
goto end;
if (pmd_bad(*pmd)) {
pmd_ERROR(*pmd);
pmd_clear(pmd);
goto end;
}
pte = pte_offset(pmd, addr);
if (pte_none(*pte))
pte = NULL;
end:
return pte;
}
static inline int swap_out_pmd(struct mm_struct * mm, struct vm_area_struct * vma, pmd_t *dir, unsigned long address, unsigned long end, int gfp_mask)
{
pte_t * pte;
unsigned long pmd_end;
if (pmd_none(*dir))
return 0;
if (pmd_bad(*dir)) {
pmd_ERROR(*dir);
pmd_clear(dir);
return 0;
}
pte = pte_offset(dir, address);
pmd_end = (address + PMD_SIZE) & PMD_MASK;
if (end > pmd_end)
end = pmd_end;
do {
int result;
mm->swap_address = address + PAGE_SIZE;
result = try_to_swap_out(mm, vma, address, pte, gfp_mask);
if (result)
return result;
if (!mm->swap_cnt)
return 0;
address += PAGE_SIZE;
pte++;
} while (address && (address < end));
return 0;
}
/* mmlist_lock and vma->vm_mm->page_table_lock are held */
static inline void unuse_pmd(struct vm_area_struct * vma, pmd_t *dir,
unsigned long address, unsigned long size, unsigned long offset,
swp_entry_t entry, struct page* page)
{
struct pte_chain * pte_chain = NULL;
pte_t *pte, *mapping;
unsigned long end;
if (pmd_none(*dir))
return;
if (pmd_bad(*dir)) {
pmd_ERROR(*dir);
pmd_clear(dir);
return;
}
mapping = pte = pte_offset_map(dir, address);
offset += address & PMD_MASK;
address &= ~PMD_MASK;
end = address + size;
if (end > PMD_SIZE)
end = PMD_SIZE;
do {
/*
* FIXME: handle pte_chain_alloc() failures
*/
if (pte_chain == NULL)
pte_chain = pte_chain_alloc(GFP_ATOMIC);
unuse_pte(vma, offset+address-vma->vm_start,
pte, entry, page, &pte_chain);
address += PAGE_SIZE;
pte++;
} while (address && (address < end));
pte_unmap(mapping);
pte_chain_free(pte_chain);
}
/*
* Note: this doesn't free the actual pages themselves. That
* has been handled earlier when unmapping all the memory regions.
*/
static inline void free_one_pmd(pmd_t * dir)
{
pte_t * pte;
if (pmd_none(*dir))
return;
if (pmd_bad(*dir)) {
pmd_ERROR(*dir);
pmd_clear(dir);
return;
}
pte = pte_offset(dir, 0);
pmd_clear(dir);
pte_free(pte);
}
/* mm->page_table_lock is held. mmap_sem is not held */
static inline int swap_out_pmd(struct mm_struct * mm, struct vm_area_struct * vma, pmd_t *dir, unsigned long address, unsigned long end, int count, zone_t * classzone)
{
pte_t * pte;
unsigned long pmd_end;
DEFINE_LOCK_COUNT();
if (pmd_none(*dir))
return count;
if (pmd_bad(*dir)) {
pmd_ERROR(*dir);
pmd_clear(dir);
return count;
}
pte = pte_offset(dir, address);
pmd_end = (address + PMD_SIZE) & PMD_MASK;
if (end > pmd_end)
end = pmd_end;
do {
if (pte_present(*pte)) {
struct page *page = pte_page(*pte);
if (VALID_PAGE(page) && !PageReserved(page)) {
count -= try_to_swap_out(mm, vma, address, pte, page, classzone);
if (!count) {
address += PAGE_SIZE;
break;
}
/* we reach this with a lock depth of 1 or 2 */
#if 0
if (TEST_LOCK_COUNT(4)) {
if (conditional_schedule_needed())
return count;
RESET_LOCK_COUNT();
}
#endif
}
}
address += PAGE_SIZE;
pte++;
} while (address && (address < end));
mm->swap_address = address;
return count;
}
/* mm->page_table_lock is held. mmap_sem is not held */
static inline int swap_out_pmd(struct mm_struct * mm, struct vm_area_struct * vma, pmd_t *dir, unsigned long address, unsigned long end, int count, zone_t * classzone)
{
pte_t * pte;
unsigned long pmd_end;
if (pmd_none(*dir))
return count;
if (pmd_bad(*dir)) {
pmd_ERROR(*dir);
pmd_clear(dir);
return count;
}
pte = pte_offset(dir, address);
pmd_end = (address + PMD_SIZE) & PMD_MASK;
if (end > pmd_end)
end = pmd_end;
do {
if (pte_present(*pte)) {
struct page *page = pte_page(*pte);
if (VALID_PAGE(page) && !PageReserved(page)) {
count -= try_to_swap_out(mm, vma, address, pte, page, classzone);
if (!count) {
address += PAGE_SIZE;
break;
}
}
}
address += PAGE_SIZE;
pte++;
} while (address && (address < end));
mm->swap_address = address;
return count;
}
static inline int zap_pte_range(mmu_gather_t *tlb, pmd_t * pmd, unsigned long address, unsigned long size)
{
unsigned long offset;
pte_t * ptep;
int freed = 0;
if (pmd_none(*pmd))
return 0;
if (pmd_bad(*pmd)) {
pmd_ERROR(*pmd);
pmd_clear(pmd);
return 0;
}
ptep = pte_offset(pmd, address);
offset = address & ~PMD_MASK;
if (offset + size > PMD_SIZE)
size = PMD_SIZE - offset;
size &= PAGE_MASK;
for (offset=0; offset < size; ptep++, offset += PAGE_SIZE) {
pte_t pte = *ptep;
if (pte_none(pte))
continue;
if (pte_present(pte)) {
struct page *page = pte_page(pte);
if (VALID_PAGE(page) && !PageReserved(page))
freed ++;
/* This will eventually call __free_pte on the pte. */
tlb_remove_page(tlb, ptep, address + offset);
} else {
free_swap_and_cache(pte_to_swp_entry(pte));
pte_clear(ptep);
}
}
return freed;
}
/*
* copy one vm_area from one task to the other. Assumes the page tables
* already present in the new task to be cleared in the whole range
* covered by this vma.
*
* 08Jan98 Merged into one routine from several inline routines to reduce
* variable count and make things faster. -jj
*/
int copy_page_range(struct mm_struct *dst, struct mm_struct *src,
struct vm_area_struct *vma)
{
pgd_t * src_pgd, * dst_pgd;
unsigned long address = vma->vm_start;
unsigned long end = vma->vm_end;
unsigned long cow = (vma->vm_flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE;
src_pgd = pgd_offset(src, address)-1;
dst_pgd = pgd_offset(dst, address)-1;
for (;;) {
pmd_t * src_pmd, * dst_pmd;
src_pgd++; dst_pgd++;
/* copy_pmd_range */
if (pgd_none(*src_pgd))
goto skip_copy_pmd_range;
if (pgd_bad(*src_pgd)) {
pgd_ERROR(*src_pgd);
pgd_clear(src_pgd);
skip_copy_pmd_range: address = (address + PGDIR_SIZE) & PGDIR_MASK;
if (!address || (address >= end))
goto out;
continue;
}
if (pgd_none(*dst_pgd)) {
if (!pmd_alloc(dst_pgd, 0))
goto nomem;
}
src_pmd = pmd_offset(src_pgd, address);
dst_pmd = pmd_offset(dst_pgd, address);
do {
pte_t * src_pte, * dst_pte;
/* copy_pte_range */
if (pmd_none(*src_pmd))
goto skip_copy_pte_range;
if (pmd_bad(*src_pmd)) {
pmd_ERROR(*src_pmd);
pmd_clear(src_pmd);
skip_copy_pte_range: address = (address + PMD_SIZE) & PMD_MASK;
if (address >= end)
goto out;
goto cont_copy_pmd_range;
}
if (pmd_none(*dst_pmd)) {
if (!pte_alloc(dst_pmd, 0))
goto nomem;
}
src_pte = pte_offset(src_pmd, address);
dst_pte = pte_offset(dst_pmd, address);
do {
pte_t pte = *src_pte;
struct page *ptepage;
/* copy_one_pte */
if (pte_none(pte))
goto cont_copy_pte_range_noset;
if (!pte_present(pte)) {
swap_duplicate(pte_to_swp_entry(pte));
goto cont_copy_pte_range;
}
ptepage = pte_page(pte);
if ((!VALID_PAGE(ptepage)) ||
PageReserved(ptepage))
goto cont_copy_pte_range;
/* If it's a COW mapping, write protect it both in the parent and the child */
if (cow) {
ptep_set_wrprotect(src_pte);
pte = *src_pte;
}
/* If it's a shared mapping, mark it clean in the child */
if (vma->vm_flags & VM_SHARED)
pte = pte_mkclean(pte);
pte = pte_mkold(pte);
get_page(ptepage);
cont_copy_pte_range: set_pte(dst_pte, pte);
cont_copy_pte_range_noset: address += PAGE_SIZE;
if (address >= end)
goto out;
src_pte++;
dst_pte++;
} while ((unsigned long)src_pte & PTE_TABLE_MASK);
cont_copy_pmd_range: src_pmd++;
dst_pmd++;
} while ((unsigned long)src_pmd & PMD_TABLE_MASK);
}
out:
return 0;
nomem:
return -ENOMEM;
}
void pmd_clear_bad(pmd_t *pmd)
{
pmd_ERROR(*pmd);
pmd_clear(pmd);
}