/*
* These routines also need to handle stuff like marking pages dirty
* and/or accessed for architectures that don't do it in hardware (most
* RISC architectures). The early dirtying is also good on the i386.
*
* There is also a hook called "update_mmu_cache()" that architectures
* with external mmu caches can use to update those (ie the Sparc or
* PowerPC hashed page tables that act as extended TLBs).
*
* Note the "page_table_lock". It is to protect against kswapd removing
* pages from under us. Note that kswapd only ever _removes_ pages, never
* adds them. As such, once we have noticed that the page is not present,
* we can drop the lock early.
*
* The adding of pages is protected by the MM semaphore (which we hold),
* so we don't need to worry about a page being suddenly been added into
* our VM.
*/
static inline int handle_pte_fault(struct mm_struct *mm,
struct vm_area_struct * vma, unsigned long address,
int write_access, pte_t * pte)
{
pte_t entry;
/*
* We need the page table lock to synchronize with kswapd
* and the SMP-safe atomic PTE updates.
*/
spin_lock(&mm->page_table_lock);
entry = *pte;
if (!pte_present(entry)) {
/*
* If it truly wasn't present, we know that kswapd
* and the PTE updates will not touch it later. So
* drop the lock.
*/
spin_unlock(&mm->page_table_lock);
if (pte_none(entry))
return do_no_page(mm, vma, address, write_access, pte);
return do_swap_page(mm, vma, address, pte, pte_to_swp_entry(entry), write_access);
}
if (write_access) {
if (!pte_write(entry))
return do_wp_page(mm, vma, address, pte, entry);
entry = pte_mkdirty(entry);
}
entry = pte_mkyoung(entry);
establish_pte(vma, address, pte, entry);
spin_unlock(&mm->page_table_lock);
return 1;
}
/*
* This routine handles present pages, when users try to write
* to a shared page. It is done by copying the page to a new address
* and decrementing the shared-page counter for the old page.
*
* Goto-purists beware: the only reason for goto's here is that it results
* in better assembly code.. The "default" path will see no jumps at all.
*
* Note that this routine assumes that the protection checks have been
* done by the caller (the low-level page fault routine in most cases).
* Thus we can safely just mark it writable once we've done any necessary
* COW.
*
* We also mark the page dirty at this point even though the page will
* change only once the write actually happens. This avoids a few races,
* and potentially makes it more efficient.
*
* We hold the mm semaphore and the page_table_lock on entry and exit
* with the page_table_lock released.
*/
static int do_wp_page(struct mm_struct *mm, struct vm_area_struct * vma,
unsigned long address, pte_t *page_table, pte_t pte)
{
struct page *old_page, *new_page;
old_page = pte_page(pte);
if (!VALID_PAGE(old_page))
goto bad_wp_page;
if (!TryLockPage(old_page)) {
int reuse = can_share_swap_page(old_page);
unlock_page(old_page);
if (reuse) {
#ifndef CONFIG_SUPERH
/* Not needed for VIPT cache */
flush_cache_page(vma, address);
#endif
establish_pte(vma, address, page_table, pte_mkyoung(pte_mkdirty(pte_mkwrite(pte))));
spin_unlock(&mm->page_table_lock);
return 1; /* Minor fault */
}
}
/*
* Ok, we need to copy. Oh, well..
*/
page_cache_get(old_page);
spin_unlock(&mm->page_table_lock);
new_page = alloc_page(GFP_HIGHUSER);
if (!new_page)
goto no_mem;
copy_cow_page(old_page,new_page,address);
/*
* Re-check the pte - we dropped the lock
*/
spin_lock(&mm->page_table_lock);
if (pte_same(*page_table, pte)) {
if (PageReserved(old_page))
++mm->rss;
break_cow(vma, new_page, address, page_table);
lru_cache_add(new_page);
/* Free the old page.. */
new_page = old_page;
}
spin_unlock(&mm->page_table_lock);
page_cache_release(new_page);
page_cache_release(old_page);
return 1; /* Minor fault */
bad_wp_page:
spin_unlock(&mm->page_table_lock);
printk("do_wp_page: bogus page at address %08lx (page 0x%lx)\n",address,(unsigned long)old_page);
return -1;
no_mem:
page_cache_release(old_page);
return -1;
}
/*
* We hold the mm semaphore for reading and vma->vm_mm->page_table_lock
*/
static inline void break_cow(struct vm_area_struct * vma, struct page * new_page, unsigned long address,
pte_t *page_table)
{
flush_page_to_ram(new_page);
flush_cache_page(vma, address);
establish_pte(vma, address, page_table, pte_mkwrite(pte_mkdirty(mk_pte(new_page, vma->vm_page_prot))));
}
/*
* We hold the mm semaphore for reading and vma->vm_mm->page_table_lock
*/
static inline void break_cow(struct vm_area_struct * vma, struct page * new_page, unsigned long address,
pte_t *page_table)
{
flush_page_to_ram(new_page);
#ifndef CONFIG_SUPERH
/* Not needed for VIPT cache (need better API for caches) */
flush_cache_page(vma, address);
#endif
establish_pte(vma, address, page_table, pte_mkwrite(pte_mkdirty(mk_pte(new_page, vma->vm_page_prot))));
}
/*
* This routine handles present pages, when users try to write
* to a shared page. It is done by copying the page to a new address
* and decrementing the shared-page counter for the old page.
*
* Goto-purists beware: the only reason for goto's here is that it results
* in better assembly code.. The "default" path will see no jumps at all.
*
* Note that this routine assumes that the protection checks have been
* done by the caller (the low-level page fault routine in most cases).
* Thus we can safely just mark it writable once we've done any necessary
* COW.
*
* We also mark the page dirty at this point even though the page will
* change only once the write actually happens. This avoids a few races,
* and potentially makes it more efficient.
*
* We enter with the page table read-lock held, and need to exit without
* it.
*/
static int do_wp_page(struct mm_struct *mm, struct vm_area_struct * vma,
unsigned long address, pte_t *page_table, pte_t pte)
{
struct page *old_page, *new_page;
old_page = pte_page(pte);
if (!VALID_PAGE(old_page))
goto bad_wp_page;
/*
* We can avoid the copy if:
* - we're the only user (count == 1)
* - the only other user is the swap cache,
* and the only swap cache user is itself,
* in which case we can just continue to
* use the same swap cache (it will be
* marked dirty).
*/
switch (page_count(old_page)) {
case 2:
/*
* Lock the page so that no one can look it up from
* the swap cache, grab a reference and start using it.
* Can not do lock_page, holding page_table_lock.
*/
if (!PageSwapCache(old_page) || TryLockPage(old_page))
break;
if (is_page_shared(old_page)) {
UnlockPage(old_page);
break;
}
UnlockPage(old_page);
/* FallThrough */
case 1:
flush_cache_page(vma, address);
establish_pte(vma, address, page_table, pte_mkyoung(pte_mkdirty(pte_mkwrite(pte))));
spin_unlock(&mm->page_table_lock);
return 1; /* Minor fault */
}
/*
* Ok, we need to copy. Oh, well..
*/
spin_unlock(&mm->page_table_lock);
new_page = page_cache_alloc();
if (!new_page)
return -1;
spin_lock(&mm->page_table_lock);
/*
* Re-check the pte - we dropped the lock
*/
if (pte_same(*page_table, pte)) {
if (PageReserved(old_page))
++mm->rss;
break_cow(vma, old_page, new_page, address, page_table);
/* Free the old page.. */
new_page = old_page;
}
spin_unlock(&mm->page_table_lock);
page_cache_release(new_page);
return 1; /* Minor fault */
bad_wp_page:
spin_unlock(&mm->page_table_lock);
printk("do_wp_page: bogus page at address %08lx (page 0x%lx)\n",address,(unsigned long)old_page);
return -1;
}
