pgd_t *get_pgd_slow(struct mm_struct *mm) { pgd_t *new_pgd, *init_pgd; pmd_t *new_pmd, *init_pmd; pte_t *new_pte, *init_pte; new_pgd = alloc_pgd_table(GFP_KERNEL); if (!new_pgd) goto no_pgd; /* * This lock is here just to satisfy pmd_alloc and pte_lock */ spin_lock(&mm->page_table_lock); /* * On ARM, first page must always be allocated since it contains * the machine vectors. */ new_pmd = pmd_alloc(mm, new_pgd, 0); if (!new_pmd) goto no_pmd; new_pte = pte_alloc(mm, new_pmd, 0); if (!new_pte) goto no_pte; init_pgd = pgd_offset_k(0); init_pmd = pmd_offset(init_pgd, 0); init_pte = pte_offset(init_pmd, 0); set_pte(new_pte, *init_pte); /* * most of the page table entries are zeroed * wne the table is created. */ memcpy(new_pgd + USER_PTRS_PER_PGD, init_pgd + USER_PTRS_PER_PGD, (PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t)); spin_unlock(&mm->page_table_lock); /* update MEMC tables */ cpu_memc_update_all(new_pgd); return new_pgd; no_pte: spin_unlock(&mm->page_table_lock); pmd_free(new_pmd); free_pgd_slow(new_pgd); return NULL; no_pmd: spin_unlock(&mm->page_table_lock); free_pgd_slow(new_pgd); return NULL; no_pgd: return NULL; }
static inline int remap_area_pmd(pmd_t * pmd, unsigned long address, unsigned long size, unsigned long phys_addr, unsigned long flags) { unsigned long end; address &= ~PGDIR_MASK; end = address + size; if (end > PGDIR_SIZE) end = PGDIR_SIZE; DEBUG_IOREMAP(("%s: pmd 0x%08x, addr. 0x%08x, phys. 0x%08x, end 0x%08x pmd 0x%08x\n", \ pmd, address, phys_addr, end)); phys_addr -= address; do { DEBUG_IOREMAP(("%s: pmd 0x%08x, addr. 0x%08x, phys. 0x%08x, end 0x%08x pmd 0x%08x\n", \ __FUNCTION__, pmd, address, phys_addr, end)); pte_t * pte = pte_alloc(&init_mm, pmd, address); if (!pte) { return -ENOMEM; } remap_area_pte(pte, address, end - address, address + phys_addr, flags); address = (address + PMD_SIZE) & PMD_MASK; pmd++; DEBUG_IOREMAP(("remap_area_pmd address is 0x%08x phys_addr is 0x%08x end 0x%08x\n", address,phys_addr,end)); } while (address && (address < end)); return 0; }
/* * This routine is used to map in a page into an address space: needed by * execve() for the initial stack and environment pages. */ unsigned long put_dirty_page(struct task_struct * tsk, unsigned long page, unsigned long address) { pgd_t * pgd; pmd_t * pmd; pte_t * pte; if (page >= high_memory) printk("put_dirty_page: trying to put page %08lx at %08lx\n",page,address); if (mem_map[MAP_NR(page)].count != 1) printk("mem_map disagrees with %08lx at %08lx\n",page,address); pgd = pgd_offset(tsk->mm,address); pmd = pmd_alloc(pgd, address); if (!pmd) { free_page(page); oom(tsk); return 0; } pte = pte_alloc(pmd, address); if (!pte) { free_page(page); oom(tsk); return 0; } if (!pte_none(*pte)) { printk("put_dirty_page: page already exists\n"); free_page(page); return 0; } flush_page_to_ram(page); set_pte(pte, pte_mkwrite(pte_mkdirty(mk_pte(page, PAGE_COPY)))); /* no need for invalidate */ return page; }
static inline int copy_pte_range(pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long address, unsigned long size, int cow) { pte_t * src_pte, * dst_pte; unsigned long end; if (pmd_none(*src_pmd)) return 0; if (pmd_bad(*src_pmd)) { printk("copy_pte_range: bad pmd (%08lx)\n", pmd_val(*src_pmd)); pmd_clear(src_pmd); return 0; } src_pte = pte_offset(src_pmd, address); if (pmd_none(*dst_pmd)) { if (!pte_alloc(dst_pmd, 0)) return -ENOMEM; } dst_pte = pte_offset(dst_pmd, address); address &= ~PMD_MASK; end = address + size; if (end >= PMD_SIZE) end = PMD_SIZE; do { /* I would like to switch arguments here, to make it * consistent with copy_xxx_range and memcpy syntax. */ copy_one_pte(src_pte++, dst_pte++, cow); address += PAGE_SIZE; } while (address < end); return 0; }
static pteval_t *get_pte(pgd_t *pgtable, uintptr_t vaddr) { pgd_t *pgd = pgd_offset(pgtable, vaddr); p4d_t *p4d = p4d_alloc(pgd, vaddr); pud_t *pud = pud_alloc(p4d, vaddr); pmd_t *pmd = pmd_alloc(pud, vaddr); pte_t *pte = pte_alloc(pmd, vaddr); return &pte_val(*pte); }
static inline pte_t *alloc_one_pte(struct mm_struct *mm, unsigned long addr) { pmd_t * pmd; pte_t * pte = NULL; pmd = pmd_alloc(pgd_offset(mm, addr), addr); if (pmd) pte = pte_alloc(pmd, addr); return pte; }
static int __init minicache_init(void) { pgd_t *pgd; pmd_t *pmd; pgd = pgd_offset_k(minicache_address); pmd = pmd_alloc(&init_mm, pgd, minicache_address); if (!pmd) BUG(); minicache_pte = pte_alloc(&init_mm, pmd, minicache_address); if (!minicache_pte) BUG(); return 0; }
/* * By the time we get here, we already hold the mm semaphore */ int handle_mm_fault(struct mm_struct *mm, struct vm_area_struct * vma, unsigned long address, int write_access) { int ret = -1; pgd_t *pgd; pmd_t *pmd; pgd = pgd_offset(mm, address); pmd = pmd_alloc(pgd, address); if (pmd) { pte_t * pte = pte_alloc(pmd, address); if (pte) ret = handle_pte_fault(mm, vma, address, write_access, pte); } return ret; }
static inline int zeromap_pmd_range(pmd_t * pmd, unsigned long address, unsigned long size, pte_t zero_pte) { unsigned long end; address &= ~PGDIR_MASK; end = address + size; if (end > PGDIR_SIZE) end = PGDIR_SIZE; do { pte_t * pte = pte_alloc(pmd, address); if (!pte) return -ENOMEM; zeromap_pte_range(pte, address, end - address, zero_pte); address = (address + PMD_SIZE) & PMD_MASK; pmd++; } while (address < end); return 0; }
void handle_mm_fault(struct vm_area_struct *vma, unsigned long address,int write_access) { pgd_t * pgd; pmd_t * pmd; pte_t * pte; pgd = pgd_offset(vma->vm_task, address); pmd = pmd_alloc(pgd,address); if(!pmd) goto no_memory; pte = pte_alloc(pmd, address); if(!pte) goto no_memory; handle_pte_fault(vma,address,pte,write_access); no_memory: oom(); }
static inline int zeromap_pmd_range(struct mm_struct *mm, pmd_t * pmd, unsigned long address, unsigned long size, pgprot_t prot) { unsigned long end; address &= ~PGDIR_MASK; end = address + size; if (end > PGDIR_SIZE) end = PGDIR_SIZE; do { pte_t * pte = pte_alloc(mm, pmd, address); if (!pte) return -ENOMEM; zeromap_pte_range(mm, pte, address, end - address, prot); address = (address + PMD_SIZE) & PMD_MASK; pmd++; } while (address && (address < end)); return 0; }
static inline int vmap_pmd_range (pmd_t *pmd, unsigned long address, unsigned long size, unsigned long vaddr) { unsigned long end; address &= ~PGDIR_MASK; end = address + size; if (end > PGDIR_SIZE) end = PGDIR_SIZE; vaddr -= address; do { pte_t * pte = pte_alloc(current->mm, pmd, address); if (!pte) return -ENOMEM; vmap_pte_range(pte, address, end - address, address + vaddr); address = (address + PMD_SIZE) & PMD_MASK; pmd++; } while (address < end); return 0; }
static inline int io_remap_pmd_range(pmd_t * pmd, unsigned long address, unsigned long size, unsigned long offset, pgprot_t prot, int space) { unsigned long end; address &= ~PGDIR_MASK; end = address + size; if (end > PGDIR_SIZE) end = PGDIR_SIZE; offset -= address; do { pte_t * pte = pte_alloc(current->mm, pmd, address); if (!pte) return -ENOMEM; io_remap_pte_range(pte, address, end - address, address + offset, prot, space); address = (address + PMD_SIZE) & PMD_MASK; pmd++; } while (address < end); return 0; }
static inline int remap_pmd_range(pmd_t * pmd, unsigned long address, unsigned long size, unsigned long phys_addr, pgprot_t prot) { unsigned long end; address &= ~PGDIR_MASK; end = address + size; if (end > PGDIR_SIZE) end = PGDIR_SIZE; phys_addr -= address; do { pte_t * pte = pte_alloc(pmd, address); if (!pte) return -ENOMEM; remap_pte_range(pte, address, end - address, address + phys_addr, prot); address = (address + PMD_SIZE) & PMD_MASK; pmd++; } while (address && (address < end)); return 0; }
void handle_mm_fault(struct vm_area_struct * vma, unsigned long address, int write_access) { pgd_t *pgd; pmd_t *pmd; pte_t *pte; pgd = pgd_offset(vma->vm_mm, address); pmd = pmd_alloc(pgd, address); if (!pmd) goto no_memory; pte = pte_alloc(pmd, address); if (!pte) goto no_memory; handle_pte_fault(vma, address, write_access, pte); update_mmu_cache(vma, address, *pte); return; no_memory: oom(current); }
pgd_t *get_pgd_slow(void) { pgd_t *pgd = (pgd_t *)alloc_pgd_table(GFP_KERNEL); pmd_t *new_pmd; if (pgd) { pgd_t *init = pgd_offset(&init_mm, 0); memzero(pgd, USER_PTRS_PER_PGD * sizeof(pgd_t)); memcpy(pgd + USER_PTRS_PER_PGD, init + USER_PTRS_PER_PGD, (PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t)); /* * On ARM, first page must always be allocated */ if (!pmd_alloc(pgd, 0)) goto nomem; else { pmd_t *old_pmd = pmd_offset(init, 0); new_pmd = pmd_offset(pgd, 0); if (!pte_alloc(new_pmd, 0)) goto nomem_pmd; else { pte_t *new_pte = pte_offset(new_pmd, 0); pte_t *old_pte = pte_offset(old_pmd, 0); set_pte (new_pte, *old_pte); } } /* update MEMC tables */ cpu_memc_update_all(pgd); } return pgd; nomem_pmd: pmd_free(new_pmd); nomem: free_pgd_slow(pgd); return NULL; }
int map_page(unsigned long va, unsigned long pa, int flags) { pmd_t *pd; pte_t *pg; int err = -ENOMEM; spin_lock(&init_mm.page_table_lock); /* Use upper 10 bits of VA to index the first level map */ pd = pmd_offset(pgd_offset_k(va), va); /* Use middle 10 bits of VA to index the second-level map */ pg = pte_alloc(&init_mm, pd, va); if (pg != 0) { err = 0; set_pte(pg, mk_pte_phys(pa & PAGE_MASK, __pgprot(flags))); if (mem_init_done) flush_HPTE(0, va, pg); } spin_unlock(&init_mm.page_table_lock); return err; }
static inline int remap_area_pmd(pmd_t * pmd, unsigned long address, unsigned long size, unsigned long phys_addr, unsigned long flags) { unsigned long end; address &= ~PGDIR_MASK; end = address + size; if (end > PGDIR_SIZE) end = PGDIR_SIZE; phys_addr -= address; if (address >= end) BUG(); do { pte_t * pte = pte_alloc(&init_mm, pmd, address); if (!pte) return -ENOMEM; remap_area_pte(pte, address, end - address, address + phys_addr, flags); address = (address + PMD_SIZE) & PMD_MASK; pmd++; } while (address && (address < end)); return 0; }
static inline pte_t* get_empty_pgtable(struct task_struct *tsk, unsigned long address) { pgd_t *pgd; pmd_t *pmd; pte_t *pte; pgd = pgd_offset(tsk,address); pmd = pmd_alloc(pgd,address); if(!pmd) { oom(); return NULL; } pte = pte_alloc(pmd, address); if(!pte) { oom(); return NULL; } return NULL; }
/* * By the time we get here, we already hold the mm semaphore */ int handle_mm_fault(struct mm_struct *mm, struct vm_area_struct * vma, unsigned long address, int write_access) { pgd_t *pgd; pmd_t *pmd; current->state = TASK_RUNNING; pgd = pgd_offset(mm, address); /* * We need the page table lock to synchronize with kswapd * and the SMP-safe atomic PTE updates. */ spin_lock(&mm->page_table_lock); pmd = pmd_alloc(mm, pgd, address); if (pmd) { pte_t * pte = pte_alloc(mm, pmd, address); if (pte) return handle_pte_fault(mm, vma, address, write_access, pte); } spin_unlock(&mm->page_table_lock); return -1; }
static int fault_in_page(int taskid, struct vm_area_struct *vma, unsigned long address, int write) { static unsigned last_address; static int last_task, loop_counter; struct task_struct *tsk = task[taskid]; pgd_t *pgd; pmd_t *pmd; pte_t *pte; if (!tsk || !tsk->mm) return 1; if (!vma || (write && !(vma->vm_flags & VM_WRITE))) goto bad_area; if (vma->vm_start > address) goto bad_area; if (address == last_address && taskid == last_task) { loop_counter++; } else { loop_counter = 0; last_address = address; last_task = taskid; } if (loop_counter == WRITE_LIMIT && !write) { printk("MSC bug? setting write request\n"); stats.errors++; write = 1; } if (loop_counter == LOOP_LIMIT) { printk("MSC bug? failing request\n"); stats.errors++; return 1; } pgd = pgd_offset(vma->vm_mm, address); pmd = pmd_alloc(pgd,address); if(!pmd) goto no_memory; pte = pte_alloc(pmd, address); if(!pte) goto no_memory; if(!pte_present(*pte)) { handle_mm_fault(tsk, vma, address, write); goto finish_up; } set_pte(pte, pte_mkyoung(*pte)); flush_tlb_page(vma, address); if(!write) goto finish_up; if(pte_write(*pte)) { set_pte(pte, pte_mkdirty(*pte)); flush_tlb_page(vma, address); goto finish_up; } handle_mm_fault(tsk, vma, address, write); /* Fall through for do_wp_page */ finish_up: stats.success++; return 0; no_memory: stats.failure++; oom(tsk); return 1; bad_area: stats.failure++; tsk->tss.sig_address = address; tsk->tss.sig_desc = SUBSIG_NOMAPPING; send_sig(SIGSEGV, tsk, 1); return 1; }
/* * do_no_page() tries to create a new page mapping. It aggressively * tries to share with existing pages, but makes a separate copy if * the "write_access" parameter is true in order to avoid the next * page fault. * * As this is called only for pages that do not currently exist, we * do not need to flush old virtual caches or the TLB. */ void do_no_page(struct task_struct * tsk, struct vm_area_struct * vma, unsigned long address, int write_access) { pgd_t * pgd; pmd_t * pmd; pte_t * page_table; pte_t entry; unsigned long page; pgd = pgd_offset(tsk->mm, address); pmd = pmd_alloc(pgd, address); if (!pmd) goto no_memory; page_table = pte_alloc(pmd, address); if (!page_table) goto no_memory; entry = *page_table; if (pte_present(entry)) goto is_present; if (!pte_none(entry)) goto swap_page; address &= PAGE_MASK; if (!vma->vm_ops || !vma->vm_ops->nopage) goto anonymous_page; /* * The third argument is "no_share", which tells the low-level code * to copy, not share the page even if sharing is possible. It's * essentially an early COW detection */ page = vma->vm_ops->nopage(vma, address, (vma->vm_flags & VM_SHARED)?0:write_access); if (!page) goto sigbus; ++tsk->maj_flt; ++vma->vm_mm->rss; /* * This silly early PAGE_DIRTY setting removes a race * due to the bad i386 page protection. But it's valid * for other architectures too. * * Note that if write_access is true, we either now have * a exclusive copy of the page, or this is a shared mapping, * so we can make it writable and dirty to avoid having to * handle that later. */ flush_page_to_ram(page); entry = mk_pte(page, vma->vm_page_prot); if (write_access) { entry = pte_mkwrite(pte_mkdirty(entry)); } else if (mem_map[MAP_NR(page)].count > 1 && !(vma->vm_flags & VM_SHARED)) entry = pte_wrprotect(entry); put_page(page_table, entry); /* no need to invalidate: a not-present page shouldn't be cached */ return; anonymous_page: entry = pte_wrprotect(mk_pte(ZERO_PAGE, vma->vm_page_prot)); if (write_access) { unsigned long page = __get_free_page(GFP_KERNEL); if (!page) goto sigbus; memset((void *) page, 0, PAGE_SIZE); entry = pte_mkwrite(pte_mkdirty(mk_pte(page, vma->vm_page_prot))); vma->vm_mm->rss++; tsk->min_flt++; flush_page_to_ram(page); } put_page(page_table, entry); return; sigbus: force_sig(SIGBUS, current); put_page(page_table, BAD_PAGE); /* no need to invalidate, wasn't present */ return; swap_page: do_swap_page(tsk, vma, address, page_table, entry, write_access); return; no_memory: oom(tsk); is_present: return; }
/* * need to get a 16k page for level 1 */ pgd_t *get_pgd_slow(struct mm_struct *mm) { pgd_t *new_pgd, *init_pgd; pmd_t *new_pmd, *init_pmd; pte_t *new_pte, *init_pte; new_pgd = (pgd_t *)__get_free_pages(GFP_KERNEL, 2); if (!new_pgd) goto no_pgd; memzero(new_pgd, FIRST_KERNEL_PGD_NR * sizeof(pgd_t)); init_pgd = pgd_offset_k(0); if (vectors_base() == 0) { init_pmd = pmd_offset(init_pgd, 0); init_pte = pte_offset(init_pmd, 0); /* * This lock is here just to satisfy pmd_alloc and pte_lock */ spin_lock(&mm->page_table_lock); /* * On ARM, first page must always be allocated since it * contains the machine vectors. */ new_pmd = pmd_alloc(mm, new_pgd, 0); if (!new_pmd) goto no_pmd; new_pte = pte_alloc(mm, new_pmd, 0); if (!new_pte) goto no_pte; set_pte(new_pte, *init_pte); spin_unlock(&mm->page_table_lock); } /* * Copy over the kernel and IO PGD entries */ memcpy(new_pgd + FIRST_KERNEL_PGD_NR, init_pgd + FIRST_KERNEL_PGD_NR, (PTRS_PER_PGD - FIRST_KERNEL_PGD_NR) * sizeof(pgd_t)); /* * FIXME: this should not be necessary */ clean_cache_area(new_pgd, PTRS_PER_PGD * sizeof(pgd_t)); return new_pgd; no_pte: spin_unlock(&mm->page_table_lock); pmd_free(new_pmd); free_pages((unsigned long)new_pgd, 2); return NULL; no_pmd: spin_unlock(&mm->page_table_lock); free_pages((unsigned long)new_pgd, 2); return NULL; no_pgd: return NULL; }
/* * This isn't really reliable by any means.. */ int mem_mmap(struct inode * inode, struct file * file, struct vm_area_struct * vma) { struct task_struct *tsk; pgd_t *src_dir, *dest_dir; pmd_t *src_middle, *dest_middle; pte_t *src_table, *dest_table; unsigned long stmp, dtmp, mapnr; struct vm_area_struct *src_vma = NULL; /* Get the source's task information */ tsk = get_task(inode->i_ino >> 16); if (!tsk) return -ESRCH; /* Ensure that we have a valid source area. (Has to be mmap'ed and have valid page information.) We can't map shared memory at the moment because working out the vm_area_struct & nattach stuff isn't worth it. */ src_vma = tsk->mm->mmap; stmp = vma->vm_offset; while (stmp < vma->vm_offset + (vma->vm_end - vma->vm_start)) { while (src_vma && stmp > src_vma->vm_end) src_vma = src_vma->vm_next; if (!src_vma || (src_vma->vm_flags & VM_SHM)) return -EINVAL; src_dir = pgd_offset(tsk->mm, stmp); if (pgd_none(*src_dir)) return -EINVAL; if (pgd_bad(*src_dir)) { printk("Bad source page dir entry %08lx\n", pgd_val(*src_dir)); return -EINVAL; } src_middle = pmd_offset(src_dir, stmp); if (pmd_none(*src_middle)) return -EINVAL; if (pmd_bad(*src_middle)) { printk("Bad source page middle entry %08lx\n", pmd_val(*src_middle)); return -EINVAL; } src_table = pte_offset(src_middle, stmp); if (pte_none(*src_table)) return -EINVAL; if (stmp < src_vma->vm_start) { if (!(src_vma->vm_flags & VM_GROWSDOWN)) return -EINVAL; if (src_vma->vm_end - stmp > current->rlim[RLIMIT_STACK].rlim_cur) return -EINVAL; } stmp += PAGE_SIZE; } src_vma = tsk->mm->mmap; stmp = vma->vm_offset; dtmp = vma->vm_start; flush_cache_range(vma->vm_mm, vma->vm_start, vma->vm_end); flush_cache_range(src_vma->vm_mm, src_vma->vm_start, src_vma->vm_end); while (dtmp < vma->vm_end) { while (src_vma && stmp > src_vma->vm_end) src_vma = src_vma->vm_next; src_dir = pgd_offset(tsk->mm, stmp); src_middle = pmd_offset(src_dir, stmp); src_table = pte_offset(src_middle, stmp); dest_dir = pgd_offset(current->mm, dtmp); dest_middle = pmd_alloc(dest_dir, dtmp); if (!dest_middle) return -ENOMEM; dest_table = pte_alloc(dest_middle, dtmp); if (!dest_table) return -ENOMEM; if (!pte_present(*src_table)) do_no_page(tsk, src_vma, stmp, 1); if ((vma->vm_flags & VM_WRITE) && !pte_write(*src_table)) do_wp_page(tsk, src_vma, stmp, 1); set_pte(src_table, pte_mkdirty(*src_table)); set_pte(dest_table, *src_table); mapnr = MAP_NR(pte_page(*src_table)); if (mapnr < MAP_NR(high_memory)) mem_map[mapnr].count++; stmp += PAGE_SIZE; dtmp += PAGE_SIZE; } flush_tlb_range(vma->vm_mm, vma->vm_start, vma->vm_end); flush_tlb_range(src_vma->vm_mm, src_vma->vm_start, src_vma->vm_end); return 0; }
/* * copy one vm_area from one task to the other. Assumes the page tables * already present in the new task to be cleared in the whole range * covered by this vma. * * 08Jan98 Merged into one routine from several inline routines to reduce * variable count and make things faster. -jj */ int copy_page_range(struct mm_struct *dst, struct mm_struct *src, struct vm_area_struct *vma) { pgd_t * src_pgd, * dst_pgd; unsigned long address = vma->vm_start; unsigned long end = vma->vm_end; unsigned long cow = (vma->vm_flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE; src_pgd = pgd_offset(src, address)-1; dst_pgd = pgd_offset(dst, address)-1; for (;;) { pmd_t * src_pmd, * dst_pmd; src_pgd++; dst_pgd++; /* copy_pmd_range */ if (pgd_none(*src_pgd)) goto skip_copy_pmd_range; if (pgd_bad(*src_pgd)) { pgd_ERROR(*src_pgd); pgd_clear(src_pgd); skip_copy_pmd_range: address = (address + PGDIR_SIZE) & PGDIR_MASK; if (!address || (address >= end)) goto out; continue; } if (pgd_none(*dst_pgd)) { if (!pmd_alloc(dst_pgd, 0)) goto nomem; } src_pmd = pmd_offset(src_pgd, address); dst_pmd = pmd_offset(dst_pgd, address); do { pte_t * src_pte, * dst_pte; /* copy_pte_range */ if (pmd_none(*src_pmd)) goto skip_copy_pte_range; if (pmd_bad(*src_pmd)) { pmd_ERROR(*src_pmd); pmd_clear(src_pmd); skip_copy_pte_range: address = (address + PMD_SIZE) & PMD_MASK; if (address >= end) goto out; goto cont_copy_pmd_range; } if (pmd_none(*dst_pmd)) { if (!pte_alloc(dst_pmd, 0)) goto nomem; } src_pte = pte_offset(src_pmd, address); dst_pte = pte_offset(dst_pmd, address); do { pte_t pte = *src_pte; struct page *ptepage; /* copy_one_pte */ if (pte_none(pte)) goto cont_copy_pte_range_noset; if (!pte_present(pte)) { swap_duplicate(pte_to_swp_entry(pte)); goto cont_copy_pte_range; } ptepage = pte_page(pte); if ((!VALID_PAGE(ptepage)) || PageReserved(ptepage)) goto cont_copy_pte_range; /* If it's a COW mapping, write protect it both in the parent and the child */ if (cow) { ptep_set_wrprotect(src_pte); pte = *src_pte; } /* If it's a shared mapping, mark it clean in the child */ if (vma->vm_flags & VM_SHARED) pte = pte_mkclean(pte); pte = pte_mkold(pte); get_page(ptepage); cont_copy_pte_range: set_pte(dst_pte, pte); cont_copy_pte_range_noset: address += PAGE_SIZE; if (address >= end) goto out; src_pte++; dst_pte++; } while ((unsigned long)src_pte & PTE_TABLE_MASK); cont_copy_pmd_range: src_pmd++; dst_pmd++; } while ((unsigned long)src_pmd & PMD_TABLE_MASK); } out: return 0; nomem: return -ENOMEM; }