pgd_t *get_pgd_slow(struct mm_struct *mm)
{
pgd_t *new_pgd, *init_pgd;
pmd_t *new_pmd, *init_pmd;
pte_t *new_pte, *init_pte;
new_pgd = alloc_pgd_table(GFP_KERNEL);
if (!new_pgd)
goto no_pgd;
/*
* This lock is here just to satisfy pmd_alloc and pte_lock
*/
spin_lock(&mm->page_table_lock);
/*
* On ARM, first page must always be allocated since it contains
* the machine vectors.
*/
new_pmd = pmd_alloc(mm, new_pgd, 0);
if (!new_pmd)
goto no_pmd;
new_pte = pte_alloc(mm, new_pmd, 0);
if (!new_pte)
goto no_pte;
init_pgd = pgd_offset_k(0);
init_pmd = pmd_offset(init_pgd, 0);
init_pte = pte_offset(init_pmd, 0);
set_pte(new_pte, *init_pte);
/*
* most of the page table entries are zeroed
* wne the table is created.
*/
memcpy(new_pgd + USER_PTRS_PER_PGD, init_pgd + USER_PTRS_PER_PGD,
(PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t));
spin_unlock(&mm->page_table_lock);
/* update MEMC tables */
cpu_memc_update_all(new_pgd);
return new_pgd;
no_pte:
spin_unlock(&mm->page_table_lock);
pmd_free(new_pmd);
free_pgd_slow(new_pgd);
return NULL;
no_pmd:
spin_unlock(&mm->page_table_lock);
free_pgd_slow(new_pgd);
return NULL;
no_pgd:
return NULL;
}
static inline int remap_area_pmd(pmd_t * pmd, unsigned long address, unsigned long size,
unsigned long phys_addr, unsigned long flags)
{
unsigned long end;
address &= ~PGDIR_MASK;
end = address + size;
if (end > PGDIR_SIZE)
end = PGDIR_SIZE;
DEBUG_IOREMAP(("%s: pmd 0x%08x, addr. 0x%08x, phys. 0x%08x, end 0x%08x pmd 0x%08x\n", \
pmd, address, phys_addr, end));
phys_addr -= address;
do {
DEBUG_IOREMAP(("%s: pmd 0x%08x, addr. 0x%08x, phys. 0x%08x, end 0x%08x pmd 0x%08x\n", \
__FUNCTION__, pmd, address, phys_addr, end));
pte_t * pte = pte_alloc(&init_mm, pmd, address);
if (!pte) {
return -ENOMEM;
}
remap_area_pte(pte, address, end - address, address + phys_addr, flags);
address = (address + PMD_SIZE) & PMD_MASK;
pmd++;
DEBUG_IOREMAP(("remap_area_pmd address is 0x%08x phys_addr is 0x%08x end 0x%08x\n", address,phys_addr,end));
} while (address && (address < end));
return 0;
}
/*
* This routine is used to map in a page into an address space: needed by
* execve() for the initial stack and environment pages.
*/
unsigned long put_dirty_page(struct task_struct * tsk, unsigned long page, unsigned long address)
{
pgd_t * pgd;
pmd_t * pmd;
pte_t * pte;
if (page >= high_memory)
printk("put_dirty_page: trying to put page %08lx at %08lx\n",page,address);
if (mem_map[MAP_NR(page)].count != 1)
printk("mem_map disagrees with %08lx at %08lx\n",page,address);
pgd = pgd_offset(tsk->mm,address);
pmd = pmd_alloc(pgd, address);
if (!pmd) {
free_page(page);
oom(tsk);
return 0;
}
pte = pte_alloc(pmd, address);
if (!pte) {
free_page(page);
oom(tsk);
return 0;
}
if (!pte_none(*pte)) {
printk("put_dirty_page: page already exists\n");
free_page(page);
return 0;
}
flush_page_to_ram(page);
set_pte(pte, pte_mkwrite(pte_mkdirty(mk_pte(page, PAGE_COPY))));
/* no need for invalidate */
return page;
}
static inline int copy_pte_range(pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long address, unsigned long size, int cow)
{
pte_t * src_pte, * dst_pte;
unsigned long end;
if (pmd_none(*src_pmd))
return 0;
if (pmd_bad(*src_pmd)) {
printk("copy_pte_range: bad pmd (%08lx)\n", pmd_val(*src_pmd));
pmd_clear(src_pmd);
return 0;
}
src_pte = pte_offset(src_pmd, address);
if (pmd_none(*dst_pmd)) {
if (!pte_alloc(dst_pmd, 0))
return -ENOMEM;
}
dst_pte = pte_offset(dst_pmd, address);
address &= ~PMD_MASK;
end = address + size;
if (end >= PMD_SIZE)
end = PMD_SIZE;
do {
/* I would like to switch arguments here, to make it
* consistent with copy_xxx_range and memcpy syntax.
*/
copy_one_pte(src_pte++, dst_pte++, cow);
address += PAGE_SIZE;
} while (address < end);
return 0;
}
static pteval_t *get_pte(pgd_t *pgtable, uintptr_t vaddr)
{
pgd_t *pgd = pgd_offset(pgtable, vaddr);
p4d_t *p4d = p4d_alloc(pgd, vaddr);
pud_t *pud = pud_alloc(p4d, vaddr);
pmd_t *pmd = pmd_alloc(pud, vaddr);
pte_t *pte = pte_alloc(pmd, vaddr);
return &pte_val(*pte);
}
static inline pte_t *alloc_one_pte(struct mm_struct *mm, unsigned long addr)
{
pmd_t * pmd;
pte_t * pte = NULL;
pmd = pmd_alloc(pgd_offset(mm, addr), addr);
if (pmd)
pte = pte_alloc(pmd, addr);
return pte;
}
static int __init minicache_init(void)
{
pgd_t *pgd;
pmd_t *pmd;
pgd = pgd_offset_k(minicache_address);
pmd = pmd_alloc(&init_mm, pgd, minicache_address);
if (!pmd)
BUG();
minicache_pte = pte_alloc(&init_mm, pmd, minicache_address);
if (!minicache_pte)
BUG();
return 0;
}
/*
* By the time we get here, we already hold the mm semaphore
*/
int handle_mm_fault(struct mm_struct *mm, struct vm_area_struct * vma,
unsigned long address, int write_access)
{
int ret = -1;
pgd_t *pgd;
pmd_t *pmd;
pgd = pgd_offset(mm, address);
pmd = pmd_alloc(pgd, address);
if (pmd) {
pte_t * pte = pte_alloc(pmd, address);
if (pte)
ret = handle_pte_fault(mm, vma, address, write_access, pte);
}
return ret;
}
static inline int zeromap_pmd_range(pmd_t * pmd, unsigned long address, unsigned long size, pte_t zero_pte)
{
unsigned long end;
address &= ~PGDIR_MASK;
end = address + size;
if (end > PGDIR_SIZE)
end = PGDIR_SIZE;
do {
pte_t * pte = pte_alloc(pmd, address);
if (!pte)
return -ENOMEM;
zeromap_pte_range(pte, address, end - address, zero_pte);
address = (address + PMD_SIZE) & PMD_MASK;
pmd++;
} while (address < end);
return 0;
}
void handle_mm_fault(struct vm_area_struct *vma, unsigned long address,int write_access)
{
pgd_t * pgd;
pmd_t * pmd;
pte_t * pte;
pgd = pgd_offset(vma->vm_task, address);
pmd = pmd_alloc(pgd,address);
if(!pmd)
goto no_memory;
pte = pte_alloc(pmd, address);
if(!pte)
goto no_memory;
handle_pte_fault(vma,address,pte,write_access);
no_memory:
oom();
}
static inline int zeromap_pmd_range(struct mm_struct *mm, pmd_t * pmd, unsigned long address,
unsigned long size, pgprot_t prot)
{
unsigned long end;
address &= ~PGDIR_MASK;
end = address + size;
if (end > PGDIR_SIZE)
end = PGDIR_SIZE;
do {
pte_t * pte = pte_alloc(mm, pmd, address);
if (!pte)
return -ENOMEM;
zeromap_pte_range(mm, pte, address, end - address, prot);
address = (address + PMD_SIZE) & PMD_MASK;
pmd++;
} while (address && (address < end));
return 0;
}
static inline int
vmap_pmd_range (pmd_t *pmd, unsigned long address, unsigned long size, unsigned long vaddr)
{
unsigned long end;
address &= ~PGDIR_MASK;
end = address + size;
if (end > PGDIR_SIZE)
end = PGDIR_SIZE;
vaddr -= address;
do {
pte_t * pte = pte_alloc(current->mm, pmd, address);
if (!pte)
return -ENOMEM;
vmap_pte_range(pte, address, end - address, address + vaddr);
address = (address + PMD_SIZE) & PMD_MASK;
pmd++;
} while (address < end);
return 0;
}
static inline int io_remap_pmd_range(pmd_t * pmd, unsigned long address, unsigned long size,
unsigned long offset, pgprot_t prot, int space)
{
unsigned long end;
address &= ~PGDIR_MASK;
end = address + size;
if (end > PGDIR_SIZE)
end = PGDIR_SIZE;
offset -= address;
do {
pte_t * pte = pte_alloc(current->mm, pmd, address);
if (!pte)
return -ENOMEM;
io_remap_pte_range(pte, address, end - address, address + offset, prot, space);
address = (address + PMD_SIZE) & PMD_MASK;
pmd++;
} while (address < end);
return 0;
}
static inline int remap_pmd_range(pmd_t * pmd, unsigned long address, unsigned long size,
unsigned long phys_addr, pgprot_t prot)
{
unsigned long end;
address &= ~PGDIR_MASK;
end = address + size;
if (end > PGDIR_SIZE)
end = PGDIR_SIZE;
phys_addr -= address;
do {
pte_t * pte = pte_alloc(pmd, address);
if (!pte)
return -ENOMEM;
remap_pte_range(pte, address, end - address, address + phys_addr, prot);
address = (address + PMD_SIZE) & PMD_MASK;
pmd++;
} while (address && (address < end));
return 0;
}
void handle_mm_fault(struct vm_area_struct * vma, unsigned long address,
int write_access)
{
pgd_t *pgd;
pmd_t *pmd;
pte_t *pte;
pgd = pgd_offset(vma->vm_mm, address);
pmd = pmd_alloc(pgd, address);
if (!pmd)
goto no_memory;
pte = pte_alloc(pmd, address);
if (!pte)
goto no_memory;
handle_pte_fault(vma, address, write_access, pte);
update_mmu_cache(vma, address, *pte);
return;
no_memory:
oom(current);
}
pgd_t *get_pgd_slow(void)
{
pgd_t *pgd = (pgd_t *)alloc_pgd_table(GFP_KERNEL);
pmd_t *new_pmd;
if (pgd) {
pgd_t *init = pgd_offset(&init_mm, 0);
memzero(pgd, USER_PTRS_PER_PGD * sizeof(pgd_t));
memcpy(pgd + USER_PTRS_PER_PGD, init + USER_PTRS_PER_PGD,
(PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t));
/*
* On ARM, first page must always be allocated
*/
if (!pmd_alloc(pgd, 0))
goto nomem;
else {
pmd_t *old_pmd = pmd_offset(init, 0);
new_pmd = pmd_offset(pgd, 0);
if (!pte_alloc(new_pmd, 0))
goto nomem_pmd;
else {
pte_t *new_pte = pte_offset(new_pmd, 0);
pte_t *old_pte = pte_offset(old_pmd, 0);
set_pte (new_pte, *old_pte);
}
}
/* update MEMC tables */
cpu_memc_update_all(pgd);
}
return pgd;
nomem_pmd:
pmd_free(new_pmd);
nomem:
free_pgd_slow(pgd);
return NULL;
}
int
map_page(unsigned long va, unsigned long pa, int flags)
{
pmd_t *pd;
pte_t *pg;
int err = -ENOMEM;
spin_lock(&init_mm.page_table_lock);
/* Use upper 10 bits of VA to index the first level map */
pd = pmd_offset(pgd_offset_k(va), va);
/* Use middle 10 bits of VA to index the second-level map */
pg = pte_alloc(&init_mm, pd, va);
if (pg != 0) {
err = 0;
set_pte(pg, mk_pte_phys(pa & PAGE_MASK, __pgprot(flags)));
if (mem_init_done)
flush_HPTE(0, va, pg);
}
spin_unlock(&init_mm.page_table_lock);
return err;
}
static inline int remap_area_pmd(pmd_t * pmd, unsigned long address, unsigned long size,
unsigned long phys_addr, unsigned long flags)
{
unsigned long end;
address &= ~PGDIR_MASK;
end = address + size;
if (end > PGDIR_SIZE)
end = PGDIR_SIZE;
phys_addr -= address;
if (address >= end)
BUG();
do {
pte_t * pte = pte_alloc(&init_mm, pmd, address);
if (!pte)
return -ENOMEM;
remap_area_pte(pte, address, end - address, address + phys_addr, flags);
address = (address + PMD_SIZE) & PMD_MASK;
pmd++;
} while (address && (address < end));
return 0;
}
static inline pte_t* get_empty_pgtable(struct task_struct *tsk, unsigned long address)
{
pgd_t *pgd;
pmd_t *pmd;
pte_t *pte;
pgd = pgd_offset(tsk,address);
pmd = pmd_alloc(pgd,address);
if(!pmd)
{
oom();
return NULL;
}
pte = pte_alloc(pmd, address);
if(!pte)
{
oom();
return NULL;
}
return NULL;
}
/*
* By the time we get here, we already hold the mm semaphore
*/
int handle_mm_fault(struct mm_struct *mm, struct vm_area_struct * vma,
unsigned long address, int write_access)
{
pgd_t *pgd;
pmd_t *pmd;
current->state = TASK_RUNNING;
pgd = pgd_offset(mm, address);
/*
* We need the page table lock to synchronize with kswapd
* and the SMP-safe atomic PTE updates.
*/
spin_lock(&mm->page_table_lock);
pmd = pmd_alloc(mm, pgd, address);
if (pmd) {
pte_t * pte = pte_alloc(mm, pmd, address);
if (pte)
return handle_pte_fault(mm, vma, address, write_access, pte);
}
spin_unlock(&mm->page_table_lock);
return -1;
}
static int fault_in_page(int taskid,
struct vm_area_struct *vma,
unsigned long address, int write)
{
static unsigned last_address;
static int last_task, loop_counter;
struct task_struct *tsk = task[taskid];
pgd_t *pgd;
pmd_t *pmd;
pte_t *pte;
if (!tsk || !tsk->mm)
return 1;
if (!vma || (write && !(vma->vm_flags & VM_WRITE)))
goto bad_area;
if (vma->vm_start > address)
goto bad_area;
if (address == last_address && taskid == last_task) {
loop_counter++;
} else {
loop_counter = 0;
last_address = address;
last_task = taskid;
}
if (loop_counter == WRITE_LIMIT && !write) {
printk("MSC bug? setting write request\n");
stats.errors++;
write = 1;
}
if (loop_counter == LOOP_LIMIT) {
printk("MSC bug? failing request\n");
stats.errors++;
return 1;
}
pgd = pgd_offset(vma->vm_mm, address);
pmd = pmd_alloc(pgd,address);
if(!pmd)
goto no_memory;
pte = pte_alloc(pmd, address);
if(!pte)
goto no_memory;
if(!pte_present(*pte)) {
handle_mm_fault(tsk, vma, address, write);
goto finish_up;
}
set_pte(pte, pte_mkyoung(*pte));
flush_tlb_page(vma, address);
if(!write)
goto finish_up;
if(pte_write(*pte)) {
set_pte(pte, pte_mkdirty(*pte));
flush_tlb_page(vma, address);
goto finish_up;
}
handle_mm_fault(tsk, vma, address, write);
/* Fall through for do_wp_page */
finish_up:
stats.success++;
return 0;
no_memory:
stats.failure++;
oom(tsk);
return 1;
bad_area:
stats.failure++;
tsk->tss.sig_address = address;
tsk->tss.sig_desc = SUBSIG_NOMAPPING;
send_sig(SIGSEGV, tsk, 1);
return 1;
}
/*
* do_no_page() tries to create a new page mapping. It aggressively
* tries to share with existing pages, but makes a separate copy if
* the "write_access" parameter is true in order to avoid the next
* page fault.
*
* As this is called only for pages that do not currently exist, we
* do not need to flush old virtual caches or the TLB.
*/
void do_no_page(struct task_struct * tsk, struct vm_area_struct * vma,
unsigned long address, int write_access)
{
pgd_t * pgd;
pmd_t * pmd;
pte_t * page_table;
pte_t entry;
unsigned long page;
pgd = pgd_offset(tsk->mm, address);
pmd = pmd_alloc(pgd, address);
if (!pmd)
goto no_memory;
page_table = pte_alloc(pmd, address);
if (!page_table)
goto no_memory;
entry = *page_table;
if (pte_present(entry))
goto is_present;
if (!pte_none(entry))
goto swap_page;
address &= PAGE_MASK;
if (!vma->vm_ops || !vma->vm_ops->nopage)
goto anonymous_page;
/*
* The third argument is "no_share", which tells the low-level code
* to copy, not share the page even if sharing is possible. It's
* essentially an early COW detection
*/
page = vma->vm_ops->nopage(vma, address,
(vma->vm_flags & VM_SHARED)?0:write_access);
if (!page)
goto sigbus;
++tsk->maj_flt;
++vma->vm_mm->rss;
/*
* This silly early PAGE_DIRTY setting removes a race
* due to the bad i386 page protection. But it's valid
* for other architectures too.
*
* Note that if write_access is true, we either now have
* a exclusive copy of the page, or this is a shared mapping,
* so we can make it writable and dirty to avoid having to
* handle that later.
*/
flush_page_to_ram(page);
entry = mk_pte(page, vma->vm_page_prot);
if (write_access) {
entry = pte_mkwrite(pte_mkdirty(entry));
} else if (mem_map[MAP_NR(page)].count > 1 && !(vma->vm_flags & VM_SHARED))
entry = pte_wrprotect(entry);
put_page(page_table, entry);
/* no need to invalidate: a not-present page shouldn't be cached */
return;
anonymous_page:
entry = pte_wrprotect(mk_pte(ZERO_PAGE, vma->vm_page_prot));
if (write_access) {
unsigned long page = __get_free_page(GFP_KERNEL);
if (!page)
goto sigbus;
memset((void *) page, 0, PAGE_SIZE);
entry = pte_mkwrite(pte_mkdirty(mk_pte(page, vma->vm_page_prot)));
vma->vm_mm->rss++;
tsk->min_flt++;
flush_page_to_ram(page);
}
put_page(page_table, entry);
return;
sigbus:
force_sig(SIGBUS, current);
put_page(page_table, BAD_PAGE);
/* no need to invalidate, wasn't present */
return;
swap_page:
do_swap_page(tsk, vma, address, page_table, entry, write_access);
return;
no_memory:
oom(tsk);
is_present:
return;
}
/*
* need to get a 16k page for level 1
*/
pgd_t *get_pgd_slow(struct mm_struct *mm)
{
pgd_t *new_pgd, *init_pgd;
pmd_t *new_pmd, *init_pmd;
pte_t *new_pte, *init_pte;
new_pgd = (pgd_t *)__get_free_pages(GFP_KERNEL, 2);
if (!new_pgd)
goto no_pgd;
memzero(new_pgd, FIRST_KERNEL_PGD_NR * sizeof(pgd_t));
init_pgd = pgd_offset_k(0);
if (vectors_base() == 0) {
init_pmd = pmd_offset(init_pgd, 0);
init_pte = pte_offset(init_pmd, 0);
/*
* This lock is here just to satisfy pmd_alloc and pte_lock
*/
spin_lock(&mm->page_table_lock);
/*
* On ARM, first page must always be allocated since it
* contains the machine vectors.
*/
new_pmd = pmd_alloc(mm, new_pgd, 0);
if (!new_pmd)
goto no_pmd;
new_pte = pte_alloc(mm, new_pmd, 0);
if (!new_pte)
goto no_pte;
set_pte(new_pte, *init_pte);
spin_unlock(&mm->page_table_lock);
}
/*
* Copy over the kernel and IO PGD entries
*/
memcpy(new_pgd + FIRST_KERNEL_PGD_NR, init_pgd + FIRST_KERNEL_PGD_NR,
(PTRS_PER_PGD - FIRST_KERNEL_PGD_NR) * sizeof(pgd_t));
/*
* FIXME: this should not be necessary
*/
clean_cache_area(new_pgd, PTRS_PER_PGD * sizeof(pgd_t));
return new_pgd;
no_pte:
spin_unlock(&mm->page_table_lock);
pmd_free(new_pmd);
free_pages((unsigned long)new_pgd, 2);
return NULL;
no_pmd:
spin_unlock(&mm->page_table_lock);
free_pages((unsigned long)new_pgd, 2);
return NULL;
no_pgd:
return NULL;
}
/*
* This isn't really reliable by any means..
*/
int mem_mmap(struct inode * inode, struct file * file,
struct vm_area_struct * vma)
{
struct task_struct *tsk;
pgd_t *src_dir, *dest_dir;
pmd_t *src_middle, *dest_middle;
pte_t *src_table, *dest_table;
unsigned long stmp, dtmp, mapnr;
struct vm_area_struct *src_vma = NULL;
/* Get the source's task information */
tsk = get_task(inode->i_ino >> 16);
if (!tsk)
return -ESRCH;
/* Ensure that we have a valid source area. (Has to be mmap'ed and
have valid page information.) We can't map shared memory at the
moment because working out the vm_area_struct & nattach stuff isn't
worth it. */
src_vma = tsk->mm->mmap;
stmp = vma->vm_offset;
while (stmp < vma->vm_offset + (vma->vm_end - vma->vm_start)) {
while (src_vma && stmp > src_vma->vm_end)
src_vma = src_vma->vm_next;
if (!src_vma || (src_vma->vm_flags & VM_SHM))
return -EINVAL;
src_dir = pgd_offset(tsk->mm, stmp);
if (pgd_none(*src_dir))
return -EINVAL;
if (pgd_bad(*src_dir)) {
printk("Bad source page dir entry %08lx\n", pgd_val(*src_dir));
return -EINVAL;
}
src_middle = pmd_offset(src_dir, stmp);
if (pmd_none(*src_middle))
return -EINVAL;
if (pmd_bad(*src_middle)) {
printk("Bad source page middle entry %08lx\n", pmd_val(*src_middle));
return -EINVAL;
}
src_table = pte_offset(src_middle, stmp);
if (pte_none(*src_table))
return -EINVAL;
if (stmp < src_vma->vm_start) {
if (!(src_vma->vm_flags & VM_GROWSDOWN))
return -EINVAL;
if (src_vma->vm_end - stmp > current->rlim[RLIMIT_STACK].rlim_cur)
return -EINVAL;
}
stmp += PAGE_SIZE;
}
src_vma = tsk->mm->mmap;
stmp = vma->vm_offset;
dtmp = vma->vm_start;
flush_cache_range(vma->vm_mm, vma->vm_start, vma->vm_end);
flush_cache_range(src_vma->vm_mm, src_vma->vm_start, src_vma->vm_end);
while (dtmp < vma->vm_end) {
while (src_vma && stmp > src_vma->vm_end)
src_vma = src_vma->vm_next;
src_dir = pgd_offset(tsk->mm, stmp);
src_middle = pmd_offset(src_dir, stmp);
src_table = pte_offset(src_middle, stmp);
dest_dir = pgd_offset(current->mm, dtmp);
dest_middle = pmd_alloc(dest_dir, dtmp);
if (!dest_middle)
return -ENOMEM;
dest_table = pte_alloc(dest_middle, dtmp);
if (!dest_table)
return -ENOMEM;
if (!pte_present(*src_table))
do_no_page(tsk, src_vma, stmp, 1);
if ((vma->vm_flags & VM_WRITE) && !pte_write(*src_table))
do_wp_page(tsk, src_vma, stmp, 1);
set_pte(src_table, pte_mkdirty(*src_table));
set_pte(dest_table, *src_table);
mapnr = MAP_NR(pte_page(*src_table));
if (mapnr < MAP_NR(high_memory))
mem_map[mapnr].count++;
stmp += PAGE_SIZE;
dtmp += PAGE_SIZE;
}
flush_tlb_range(vma->vm_mm, vma->vm_start, vma->vm_end);
flush_tlb_range(src_vma->vm_mm, src_vma->vm_start, src_vma->vm_end);
return 0;
}
/*
* copy one vm_area from one task to the other. Assumes the page tables
* already present in the new task to be cleared in the whole range
* covered by this vma.
*
* 08Jan98 Merged into one routine from several inline routines to reduce
* variable count and make things faster. -jj
*/
int copy_page_range(struct mm_struct *dst, struct mm_struct *src,
struct vm_area_struct *vma)
{
pgd_t * src_pgd, * dst_pgd;
unsigned long address = vma->vm_start;
unsigned long end = vma->vm_end;
unsigned long cow = (vma->vm_flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE;
src_pgd = pgd_offset(src, address)-1;
dst_pgd = pgd_offset(dst, address)-1;
for (;;) {
pmd_t * src_pmd, * dst_pmd;
src_pgd++; dst_pgd++;
/* copy_pmd_range */
if (pgd_none(*src_pgd))
goto skip_copy_pmd_range;
if (pgd_bad(*src_pgd)) {
pgd_ERROR(*src_pgd);
pgd_clear(src_pgd);
skip_copy_pmd_range: address = (address + PGDIR_SIZE) & PGDIR_MASK;
if (!address || (address >= end))
goto out;
continue;
}
if (pgd_none(*dst_pgd)) {
if (!pmd_alloc(dst_pgd, 0))
goto nomem;
}
src_pmd = pmd_offset(src_pgd, address);
dst_pmd = pmd_offset(dst_pgd, address);
do {
pte_t * src_pte, * dst_pte;
/* copy_pte_range */
if (pmd_none(*src_pmd))
goto skip_copy_pte_range;
if (pmd_bad(*src_pmd)) {
pmd_ERROR(*src_pmd);
pmd_clear(src_pmd);
skip_copy_pte_range: address = (address + PMD_SIZE) & PMD_MASK;
if (address >= end)
goto out;
goto cont_copy_pmd_range;
}
if (pmd_none(*dst_pmd)) {
if (!pte_alloc(dst_pmd, 0))
goto nomem;
}
src_pte = pte_offset(src_pmd, address);
dst_pte = pte_offset(dst_pmd, address);
do {
pte_t pte = *src_pte;
struct page *ptepage;
/* copy_one_pte */
if (pte_none(pte))
goto cont_copy_pte_range_noset;
if (!pte_present(pte)) {
swap_duplicate(pte_to_swp_entry(pte));
goto cont_copy_pte_range;
}
ptepage = pte_page(pte);
if ((!VALID_PAGE(ptepage)) ||
PageReserved(ptepage))
goto cont_copy_pte_range;
/* If it's a COW mapping, write protect it both in the parent and the child */
if (cow) {
ptep_set_wrprotect(src_pte);
pte = *src_pte;
}
/* If it's a shared mapping, mark it clean in the child */
if (vma->vm_flags & VM_SHARED)
pte = pte_mkclean(pte);
pte = pte_mkold(pte);
get_page(ptepage);
cont_copy_pte_range: set_pte(dst_pte, pte);
cont_copy_pte_range_noset: address += PAGE_SIZE;
if (address >= end)
goto out;
src_pte++;
dst_pte++;
} while ((unsigned long)src_pte & PTE_TABLE_MASK);
cont_copy_pmd_range: src_pmd++;
dst_pmd++;
} while ((unsigned long)src_pmd & PMD_TABLE_MASK);
}
out:
return 0;
nomem:
return -ENOMEM;
}