int access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len, int write)
{
struct mm_struct *mm;
struct vm_area_struct *vma;
struct page *page;
void *old_buf = buf;
/* Worry about races with exit() */
task_lock(tsk);
mm = tsk->mm;
if (!tsk->task_dumpable || (&init_mm == mm))
mm = NULL;
if (mm)
atomic_inc(&mm->mm_users);
task_unlock(tsk);
if (!mm)
return 0;
down_read(&mm->mmap_sem);
/* ignore errors, just check how much was sucessfully transfered */
while (len) {
int bytes, ret, offset;
void *maddr;
ret = get_user_pages(current, mm, addr, 1,
write, 1, &page, &vma);
if (ret <= 0)
break;
bytes = len;
offset = addr & (PAGE_SIZE-1);
if (bytes > PAGE_SIZE-offset)
bytes = PAGE_SIZE-offset;
flush_cache_page(vma, addr);
maddr = kmap(page);
if (write) {
memcpy(maddr + offset, buf, bytes);
#ifdef CONFIG_SUPERH
flush_dcache_page(page);
#endif
flush_page_to_ram(page);
flush_icache_user_range(vma, page, addr, len);
} else {
memcpy(buf, maddr + offset, bytes);
flush_page_to_ram(page);
}
kunmap(page);
put_page(page);
len -= bytes;
buf += bytes;
addr += bytes;
}
up_read(&mm->mmap_sem);
mmput(mm);
return buf - old_buf;
}
/*
* This function zeroes out partial mmap'ed pages at truncation time..
*/
static void partial_clear(struct vm_area_struct *vma, unsigned long address)
{
pgd_t *page_dir;
pmd_t *page_middle;
pte_t *page_table, pte;
page_dir = pgd_offset(vma->vm_mm, address);
if (pgd_none(*page_dir))
return;
if (pgd_bad(*page_dir)) {
printk("bad page table directory entry %p:[%lx]\n", page_dir, pgd_val(*page_dir));
pgd_clear(page_dir);
return;
}
page_middle = pmd_offset(page_dir, address);
if (pmd_none(*page_middle))
return;
if (pmd_bad(*page_middle)) {
printk("bad page table directory entry %p:[%lx]\n", page_dir, pgd_val(*page_dir));
pmd_clear(page_middle);
return;
}
page_table = pte_offset(page_middle, address);
pte = *page_table;
if (!pte_present(pte))
return;
flush_cache_page(vma, address);
address &= ~PAGE_MASK;
address += pte_page(pte);
if (address >= high_memory)
return;
memset((void *) address, 0, PAGE_SIZE - (address & ~PAGE_MASK));
flush_page_to_ram(pte_page(pte));
}
/*
* Trying to stop swapping from a file is fraught with races, so
* we repeat quite a bit here when we have to pause. swapoff()
* isn't exactly timing-critical, so who cares (but this is /really/
* inefficient, ugh).
*
* We return 1 after having slept, which makes the process start over
* from the beginning for this process..
*/
static inline int unuse_pte(struct vm_area_struct * vma, unsigned long address,
pte_t *dir, unsigned int type, unsigned long page)
{
pte_t pte = *dir;
if (pte_none(pte))
return 0;
if (pte_present(pte)) {
unsigned long page_nr = MAP_NR(pte_page(pte));
if (page_nr >= MAP_NR(high_memory))
return 0;
if (!in_swap_cache(page_nr))
return 0;
if (SWP_TYPE(in_swap_cache(page_nr)) != type)
return 0;
delete_from_swap_cache(page_nr);
set_pte(dir, pte_mkdirty(pte));
return 0;
}
if (SWP_TYPE(pte_val(pte)) != type)
return 0;
read_swap_page(pte_val(pte), (char *) page);
#if 0 /* Is this really needed here, hasn't it been solved elsewhere? */
flush_page_to_ram(page);
#endif
if (pte_val(*dir) != pte_val(pte)) {
free_page(page);
return 1;
}
set_pte(dir, pte_mkwrite(pte_mkdirty(mk_pte(page, vma->vm_page_prot))));
flush_tlb_page(vma, address);
++vma->vm_mm->rss;
swap_free(pte_val(pte));
return 1;
}
/*
* This routine is used to map in a page into an address space: needed by
* execve() for the initial stack and environment pages.
*/
unsigned long put_dirty_page(struct task_struct * tsk, unsigned long page, unsigned long address)
{
pgd_t * pgd;
pmd_t * pmd;
pte_t * pte;
if (page >= high_memory)
printk("put_dirty_page: trying to put page %08lx at %08lx\n",page,address);
if (mem_map[MAP_NR(page)].count != 1)
printk("mem_map disagrees with %08lx at %08lx\n",page,address);
pgd = pgd_offset(tsk->mm,address);
pmd = pmd_alloc(pgd, address);
if (!pmd) {
free_page(page);
oom(tsk);
return 0;
}
pte = pte_alloc(pmd, address);
if (!pte) {
free_page(page);
oom(tsk);
return 0;
}
if (!pte_none(*pte)) {
printk("put_dirty_page: page already exists\n");
free_page(page);
return 0;
}
flush_page_to_ram(page);
set_pte(pte, pte_mkwrite(pte_mkdirty(mk_pte(page, PAGE_COPY))));
/* no need for invalidate */
return page;
}
/*
* We hold the mm semaphore for reading and vma->vm_mm->page_table_lock
*/
static inline void break_cow(struct vm_area_struct * vma, struct page * new_page, unsigned long address,
pte_t *page_table)
{
flush_page_to_ram(new_page);
flush_cache_page(vma, address);
establish_pte(vma, address, page_table, pte_mkwrite(pte_mkdirty(mk_pte(new_page, vma->vm_page_prot))));
}
/*
* We hold the mm semaphore for reading and vma->vm_mm->page_table_lock
*/
static inline void break_cow(struct vm_area_struct * vma, struct page * new_page, unsigned long address,
pte_t *page_table)
{
flush_page_to_ram(new_page);
#ifndef CONFIG_SUPERH
/* Not needed for VIPT cache (need better API for caches) */
flush_cache_page(vma, address);
#endif
establish_pte(vma, address, page_table, pte_mkwrite(pte_mkdirty(mk_pte(new_page, vma->vm_page_prot))));
}
struct page *
tas_vma_nopage (struct vm_area_struct * area,
unsigned long address, int write_access)
{
if ( address != _TAS_START_MAGIC
|| area->vm_start != _TAS_START_MAGIC
|| area->vm_pgoff != 0 )
return 0;
get_page(tas_code_buffer);
flush_page_to_ram(tas_code_buffer);
return tas_code_buffer;
}
/*
* This routine puts a long into any process space by following the page
* tables. NOTE! You should check that the long isn't on a page boundary,
* and that it is in the task area before calling this: this routine does
* no checking.
*
* Now keeps R/W state of page so that a text page stays readonly
* even if a debugger scribbles breakpoints into it. -M.U-
*/
static void put_long(struct task_struct * tsk, struct vm_area_struct * vma, unsigned long addr,
unsigned long data)
{
pgd_t *pgdir;
pmd_t *pgmiddle;
pte_t *pgtable;
unsigned long page;
repeat:
pgdir = pgd_offset(vma->vm_mm, addr);
if (!pgd_present(*pgdir)) {
do_no_page(tsk, vma, addr, 1);
goto repeat;
}
if (pgd_bad(*pgdir)) {
printk("ptrace: bad page directory %08lx\n", pgd_val(*pgdir));
pgd_clear(pgdir);
return;
}
pgmiddle = pmd_offset(pgdir,addr);
if (pmd_none(*pgmiddle)) {
do_no_page(tsk, vma, addr, 1);
goto repeat;
}
if (pmd_bad(*pgmiddle)) {
printk("ptrace: bad page directory %08lx\n",
pmd_val(*pgmiddle));
pmd_clear(pgmiddle);
return;
}
pgtable = pte_offset(pgmiddle, addr);
if (!pte_present(*pgtable)) {
do_no_page(tsk, vma, addr, 1);
goto repeat;
}
page = pte_page(*pgtable);
if (!pte_write(*pgtable)) {
do_wp_page(tsk, vma, addr, 2);
goto repeat;
}
/* this is a hack for non-kernel-mapped video buffers and similar */
if (page < high_memory) {
*(unsigned long *) (page + (addr & ~PAGE_MASK)) = data;
flush_page_to_ram (page);
}
/* we're bypassing pagetables, so we have to set the dirty bit ourselves */
/* this should also re-instate whatever read-only mode there was before */
*pgtable = pte_mkdirty(mk_pte(page, vma->vm_page_prot));
flush_tlb_all();
}
static inline void do_swap_page(struct task_struct * tsk,
struct vm_area_struct * vma, unsigned long address,
pte_t * page_table, pte_t entry, int write_access)
{
pte_t page;
if (!vma->vm_ops || !vma->vm_ops->swapin) {
swap_in(tsk, vma, page_table, pte_val(entry), write_access);
flush_page_to_ram(pte_page(*page_table));
return;
}
page = vma->vm_ops->swapin(vma, address - vma->vm_start + vma->vm_offset, pte_val(entry));
if (pte_val(*page_table) != pte_val(entry)) {
free_page(pte_page(page));
return;
}
if (mem_map[MAP_NR(pte_page(page))].count > 1 && !(vma->vm_flags & VM_SHARED))
page = pte_wrprotect(page);
++vma->vm_mm->rss;
++tsk->maj_flt;
flush_page_to_ram(pte_page(page));
set_pte(page_table, page);
return;
}
/*
* This only needs the MM semaphore
*/
static int do_anonymous_page(struct mm_struct * mm, struct vm_area_struct * vma, pte_t *page_table, int write_access, unsigned long addr)
{
struct page *page = NULL;
pte_t entry = pte_wrprotect(mk_pte(ZERO_PAGE(addr), vma->vm_page_prot));
if (write_access) {
page = alloc_page(GFP_HIGHUSER);
if (!page)
return -1;
clear_user_highpage(page, addr);
entry = pte_mkwrite(pte_mkdirty(mk_pte(page, vma->vm_page_prot)));
mm->rss++;
flush_page_to_ram(page);
}
set_pte(page_table, entry);
/* No need to invalidate - it was non-present before */
update_mmu_cache(vma, addr, entry);
return 1; /* Minor fault */
}
/*
* do_no_page() tries to create a new page mapping. It aggressively
* tries to share with existing pages, but makes a separate copy if
* the "write_access" parameter is true in order to avoid the next
* page fault.
*
* As this is called only for pages that do not currently exist, we
* do not need to flush old virtual caches or the TLB.
*
* This is called with the MM semaphore held.
*/
static int do_no_page(struct mm_struct * mm, struct vm_area_struct * vma,
unsigned long address, int write_access, pte_t *page_table)
{
struct page * new_page;
pte_t entry;
if (!vma->vm_ops || !vma->vm_ops->nopage)
return do_anonymous_page(mm, vma, page_table, write_access, address);
/*
* The third argument is "no_share", which tells the low-level code
* to copy, not share the page even if sharing is possible. It's
* essentially an early COW detection.
*/
new_page = vma->vm_ops->nopage(vma, address & PAGE_MASK, (vma->vm_flags & VM_SHARED)?0:write_access);
if (new_page == NULL) /* no page was available -- SIGBUS */
return 0;
if (new_page == NOPAGE_OOM)
return -1;
++mm->rss;
/*
* This silly early PAGE_DIRTY setting removes a race
* due to the bad i386 page protection. But it's valid
* for other architectures too.
*
* Note that if write_access is true, we either now have
* an exclusive copy of the page, or this is a shared mapping,
* so we can make it writable and dirty to avoid having to
* handle that later.
*/
flush_page_to_ram(new_page);
flush_icache_page(vma, new_page);
entry = mk_pte(new_page, vma->vm_page_prot);
if (write_access) {
entry = pte_mkwrite(pte_mkdirty(entry));
} else if (page_count(new_page) > 1 &&
!(vma->vm_flags & VM_SHARED))
entry = pte_wrprotect(entry);
set_pte(page_table, entry);
/* no need to invalidate: a not-present page shouldn't be cached */
update_mmu_cache(vma, address, entry);
return 2; /* Major fault */
}
/*
* We are called with the MM semaphore and page_table_lock
* spinlock held to protect against concurrent faults in
* multithreaded programs.
*/
static int do_anonymous_page(struct mm_struct * mm, struct vm_area_struct * vma, pte_t *page_table, int write_access, unsigned long addr)
{
pte_t entry;
/* Read-only mapping of ZERO_PAGE. */
entry = pte_wrprotect(mk_pte(ZERO_PAGE(addr), vma->vm_page_prot));
/* ..except if it's a write access */
if (write_access) {
struct page *page;
/* Allocate our own private page. */
spin_unlock(&mm->page_table_lock);
page = alloc_page(GFP_HIGHUSER);
if (!page)
goto no_mem;
clear_user_highpage(page, addr);
spin_lock(&mm->page_table_lock);
if (!pte_none(*page_table)) {
page_cache_release(page);
spin_unlock(&mm->page_table_lock);
return 1;
}
mm->rss++;
flush_page_to_ram(page);
entry = pte_mkwrite(pte_mkdirty(mk_pte(page, vma->vm_page_prot)));
lru_cache_add(page);
mark_page_accessed(page);
}
set_pte(page_table, entry);
/* No need to invalidate - it was non-present before */
update_mmu_cache(vma, addr, entry);
spin_unlock(&mm->page_table_lock);
return 1; /* Minor fault */
no_mem:
return -1;
}
static int do_swap_page(struct mm_struct * mm,
struct vm_area_struct * vma, unsigned long address,
pte_t * page_table, swp_entry_t entry, int write_access)
{
struct page *page = lookup_swap_cache(entry);
pte_t pte;
if (!page) {
lock_kernel();
swapin_readahead(entry);
page = read_swap_cache(entry);
unlock_kernel();
if (!page)
return -1;
flush_page_to_ram(page);
flush_icache_page(vma, page);
}
mm->rss++;
pte = mk_pte(page, vma->vm_page_prot);
/*
* Freeze the "shared"ness of the page, ie page_count + swap_count.
* Must lock page before transferring our swap count to already
* obtained page count.
*/
lock_page(page);
swap_free(entry);
if (write_access && !is_page_shared(page))
pte = pte_mkwrite(pte_mkdirty(pte));
UnlockPage(page);
set_pte(page_table, pte);
/* No need to invalidate - it was non-present before */
update_mmu_cache(vma, address, pte);
return 1; /* Minor fault */
}
/*
* We completely avoid races by reading each swap page in advance,
* and then search for the process using it. All the necessary
* page table adjustments can then be made atomically.
*/
static int try_to_unuse(unsigned int type)
{
struct swap_info_struct * si = &swap_info[type];
struct mm_struct *start_mm;
unsigned short *swap_map;
unsigned short swcount;
struct page *page;
swp_entry_t entry;
int i = 0;
int retval = 0;
int reset_overflow = 0;
/*
* When searching mms for an entry, a good strategy is to
* start at the first mm we freed the previous entry from
* (though actually we don't notice whether we or coincidence
* freed the entry). Initialize this start_mm with a hold.
*
* A simpler strategy would be to start at the last mm we
* freed the previous entry from; but that would take less
* advantage of mmlist ordering (now preserved by swap_out()),
* which clusters forked address spaces together, most recent
* child immediately after parent. If we race with dup_mmap(),
* we very much want to resolve parent before child, otherwise
* we may miss some entries: using last mm would invert that.
*/
start_mm = &init_mm;
atomic_inc(&init_mm.mm_users);
/*
* Keep on scanning until all entries have gone. Usually,
* one pass through swap_map is enough, but not necessarily:
* mmput() removes mm from mmlist before exit_mmap() and its
* zap_page_range(). That's not too bad, those entries are
* on their way out, and handled faster there than here.
* do_munmap() behaves similarly, taking the range out of mm's
* vma list before zap_page_range(). But unfortunately, when
* unmapping a part of a vma, it takes the whole out first,
* then reinserts what's left after (might even reschedule if
* open() method called) - so swap entries may be invisible
* to swapoff for a while, then reappear - but that is rare.
*/
while ((i = find_next_to_unuse(si, i))) {
/*
* Get a page for the entry, using the existing swap
* cache page if there is one. Otherwise, get a clean
* page and read the swap into it.
*/
swap_map = &si->swap_map[i];
entry = SWP_ENTRY(type, i);
page = read_swap_cache_async(entry);
if (!page) {
/*
* Either swap_duplicate() failed because entry
* has been freed independently, and will not be
* reused since sys_swapoff() already disabled
* allocation from here, or alloc_page() failed.
*/
if (!*swap_map)
continue;
retval = -ENOMEM;
break;
}
/*
* Don't hold on to start_mm if it looks like exiting.
*/
if (atomic_read(&start_mm->mm_users) == 1) {
mmput(start_mm);
start_mm = &init_mm;
atomic_inc(&init_mm.mm_users);
}
/*
* Wait for and lock page. When do_swap_page races with
* try_to_unuse, do_swap_page can handle the fault much
* faster than try_to_unuse can locate the entry. This
* apparently redundant "wait_on_page" lets try_to_unuse
* defer to do_swap_page in such a case - in some tests,
* do_swap_page and try_to_unuse repeatedly compete.
*/
wait_on_page(page);
lock_page(page);
/*
* Remove all references to entry, without blocking.
* Whenever we reach init_mm, there's no address space
* to search, but use it as a reminder to search shmem.
*/
swcount = *swap_map;
if (swcount > 1) {
flush_page_to_ram(page);
if (start_mm == &init_mm)
shmem_unuse(entry, page);
else
unuse_process(start_mm, entry, page);
}
if (*swap_map > 1) {
int set_start_mm = (*swap_map >= swcount);
struct list_head *p = &start_mm->mmlist;
struct mm_struct *new_start_mm = start_mm;
struct mm_struct *mm;
spin_lock(&mmlist_lock);
while (*swap_map > 1 &&
(p = p->next) != &start_mm->mmlist) {
mm = list_entry(p, struct mm_struct, mmlist);
swcount = *swap_map;
if (mm == &init_mm) {
set_start_mm = 1;
shmem_unuse(entry, page);
} else
unuse_process(mm, entry, page);
if (set_start_mm && *swap_map < swcount) {
new_start_mm = mm;
set_start_mm = 0;
}
}
atomic_inc(&new_start_mm->mm_users);
spin_unlock(&mmlist_lock);
mmput(start_mm);
start_mm = new_start_mm;
}
/*
* How could swap count reach 0x7fff when the maximum
* pid is 0x7fff, and there's no way to repeat a swap
* page within an mm (except in shmem, where it's the
* shared object which takes the reference count)?
* We believe SWAP_MAP_MAX cannot occur in Linux 2.4.
*
* If that's wrong, then we should worry more about
* exit_mmap() and do_munmap() cases described above:
* we might be resetting SWAP_MAP_MAX too early here.
* We know "Undead"s can happen, they're okay, so don't
* report them; but do report if we reset SWAP_MAP_MAX.
*/
if (*swap_map == SWAP_MAP_MAX) {
swap_list_lock();
swap_device_lock(si);
nr_swap_pages++;
*swap_map = 1;
swap_device_unlock(si);
swap_list_unlock();
reset_overflow = 1;
}
/*
* If a reference remains (rare), we would like to leave
* the page in the swap cache; but try_to_swap_out could
* then re-duplicate the entry once we drop page lock,
* so we might loop indefinitely; also, that page could
* not be swapped out to other storage meanwhile. So:
* delete from cache even if there's another reference,
* after ensuring that the data has been saved to disk -
* since if the reference remains (rarer), it will be
* read from disk into another page. Splitting into two
* pages would be incorrect if swap supported "shared
* private" pages, but they are handled by tmpfs files.
* Note shmem_unuse already deleted its from swap cache.
*/
if ((*swap_map > 1) && PageDirty(page) && PageSwapCache(page)) {
rw_swap_page(WRITE, page);
lock_page(page);
}
if (PageSwapCache(page))
delete_from_swap_cache(page);
/*
* So we could skip searching mms once swap count went
* to 1, we did not mark any present ptes as dirty: must
* mark page dirty so try_to_swap_out will preserve it.
*/
SetPageDirty(page);
UnlockPage(page);
page_cache_release(page);
/*
* Make sure that we aren't completely killing
* interactive performance. Interruptible check on
* signal_pending() would be nice, but changes the spec?
*/
if (current->need_resched)
schedule();
}
/*
* do_no_page() tries to create a new page mapping. It aggressively
* tries to share with existing pages, but makes a separate copy if
* the "write_access" parameter is true in order to avoid the next
* page fault.
*
* As this is called only for pages that do not currently exist, we
* do not need to flush old virtual caches or the TLB.
*
* This is called with the MM semaphore held and the page table
* spinlock held. Exit with the spinlock released.
*/
static int do_no_page(struct mm_struct * mm, struct vm_area_struct * vma,
unsigned long address, int write_access, pte_t *page_table)
{
struct page * new_page;
pte_t entry;
if (!vma->vm_ops || !vma->vm_ops->nopage)
return do_anonymous_page(mm, vma, page_table, write_access, address);
spin_unlock(&mm->page_table_lock);
new_page = vma->vm_ops->nopage(vma, address & PAGE_MASK, 0);
if (new_page == NULL) /* no page was available -- SIGBUS */
return 0;
if (new_page == NOPAGE_OOM)
return -1;
/*
* Should we do an early C-O-W break?
*/
if (write_access && !(vma->vm_flags & VM_SHARED)) {
struct page * page = alloc_page(GFP_HIGHUSER);
if (!page) {
page_cache_release(new_page);
return -1;
}
copy_highpage(page, new_page);
page_cache_release(new_page);
lru_cache_add(page);
new_page = page;
}
spin_lock(&mm->page_table_lock);
/*
* This silly early PAGE_DIRTY setting removes a race
* due to the bad i386 page protection. But it's valid
* for other architectures too.
*
* Note that if write_access is true, we either now have
* an exclusive copy of the page, or this is a shared mapping,
* so we can make it writable and dirty to avoid having to
* handle that later.
*/
/* Only go through if we didn't race with anybody else... */
if (pte_none(*page_table)) {
++mm->rss;
flush_page_to_ram(new_page);
flush_icache_page(vma, new_page);
entry = mk_pte(new_page, vma->vm_page_prot);
if (write_access)
entry = pte_mkwrite(pte_mkdirty(entry));
set_pte(page_table, entry);
} else {
/* One of our sibling threads was faster, back out. */
page_cache_release(new_page);
spin_unlock(&mm->page_table_lock);
return 1;
}
/* no need to invalidate: a not-present page shouldn't be cached */
update_mmu_cache(vma, address, entry);
spin_unlock(&mm->page_table_lock);
return 2; /* Major fault */
}
/*
* This routine handles present pages, when users try to write
* to a shared page. It is done by copying the page to a new address
* and decrementing the shared-page counter for the old page.
*
* Goto-purists beware: the only reason for goto's here is that it results
* in better assembly code.. The "default" path will see no jumps at all.
*
* Note that this routine assumes that the protection checks have been
* done by the caller (the low-level page fault routine in most cases).
* Thus we can safely just mark it writable once we've done any necessary
* COW.
*
* We also mark the page dirty at this point even though the page will
* change only once the write actually happens. This avoids a few races,
* and potentially makes it more efficient.
*/
void do_wp_page(struct task_struct * tsk, struct vm_area_struct * vma,
unsigned long address, int write_access)
{
pgd_t *page_dir;
pmd_t *page_middle;
pte_t *page_table, pte;
unsigned long old_page, new_page;
new_page = __get_free_page(GFP_KERNEL);
page_dir = pgd_offset(vma->vm_mm, address);
if (pgd_none(*page_dir))
goto end_wp_page;
if (pgd_bad(*page_dir))
goto bad_wp_pagedir;
page_middle = pmd_offset(page_dir, address);
if (pmd_none(*page_middle))
goto end_wp_page;
if (pmd_bad(*page_middle))
goto bad_wp_pagemiddle;
page_table = pte_offset(page_middle, address);
pte = *page_table;
if (!pte_present(pte))
goto end_wp_page;
if (pte_write(pte))
goto end_wp_page;
old_page = pte_page(pte);
if (old_page >= high_memory)
goto bad_wp_page;
tsk->min_flt++;
/*
* Do we need to copy?
*/
if (mem_map[MAP_NR(old_page)].count != 1) {
if (new_page) {
if (PageReserved(mem_map + MAP_NR(old_page)))
++vma->vm_mm->rss;
copy_page(old_page,new_page);
flush_page_to_ram(old_page);
flush_page_to_ram(new_page);
flush_cache_page(vma, address);
set_pte(page_table, pte_mkwrite(pte_mkdirty(mk_pte(new_page, vma->vm_page_prot))));
free_page(old_page);
flush_tlb_page(vma, address);
return;
}
flush_cache_page(vma, address);
set_pte(page_table, BAD_PAGE);
flush_tlb_page(vma, address);
free_page(old_page);
oom(tsk);
return;
}
flush_cache_page(vma, address);
set_pte(page_table, pte_mkdirty(pte_mkwrite(pte)));
flush_tlb_page(vma, address);
if (new_page)
free_page(new_page);
return;
bad_wp_page:
printk("do_wp_page: bogus page at address %08lx (%08lx)\n",address,old_page);
send_sig(SIGKILL, tsk, 1);
goto end_wp_page;
bad_wp_pagemiddle:
printk("do_wp_page: bogus page-middle at address %08lx (%08lx)\n", address, pmd_val(*page_middle));
send_sig(SIGKILL, tsk, 1);
goto end_wp_page;
bad_wp_pagedir:
printk("do_wp_page: bogus page-dir entry at address %08lx (%08lx)\n", address, pgd_val(*page_dir));
send_sig(SIGKILL, tsk, 1);
end_wp_page:
if (new_page)
free_page(new_page);
return;
}
static lpae_t mfn_to_p2m_entry(unsigned long mfn, unsigned int mattr,
p2m_type_t t)
{
paddr_t pa = ((paddr_t) mfn) << PAGE_SHIFT;
/* xn and write bit will be defined in the switch */
lpae_t e = (lpae_t) {
.p2m.af = 1,
.p2m.sh = LPAE_SH_OUTER,
.p2m.read = 1,
.p2m.mattr = mattr,
.p2m.table = 1,
.p2m.valid = 1,
.p2m.type = t,
};
BUILD_BUG_ON(p2m_max_real_type > (1 << 4));
switch (t)
{
case p2m_ram_rw:
e.p2m.xn = 0;
e.p2m.write = 1;
break;
case p2m_ram_ro:
e.p2m.xn = 0;
e.p2m.write = 0;
break;
case p2m_map_foreign:
case p2m_grant_map_rw:
case p2m_mmio_direct:
e.p2m.xn = 1;
e.p2m.write = 1;
break;
case p2m_grant_map_ro:
case p2m_invalid:
e.p2m.xn = 1;
e.p2m.write = 0;
break;
case p2m_max_real_type:
BUG();
break;
}
ASSERT(!(pa & ~PAGE_MASK));
ASSERT(!(pa & ~PADDR_MASK));
e.bits |= pa;
return e;
}
/* Allocate a new page table page and hook it in via the given entry */
static int p2m_create_table(struct domain *d,
lpae_t *entry)
{
struct p2m_domain *p2m = &d->arch.p2m;
struct page_info *page;
void *p;
lpae_t pte;
BUG_ON(entry->p2m.valid);
page = alloc_domheap_page(NULL, 0);
if ( page == NULL )
return -ENOMEM;
page_list_add(page, &p2m->pages);
p = __map_domain_page(page);
clear_page(p);
unmap_domain_page(p);
pte = mfn_to_p2m_entry(page_to_mfn(page), MATTR_MEM, p2m_invalid);
write_pte(entry, pte);
return 0;
}
enum p2m_operation {
INSERT,
ALLOCATE,
REMOVE,
RELINQUISH,
CACHEFLUSH,
};
static int apply_p2m_changes(struct domain *d,
enum p2m_operation op,
paddr_t start_gpaddr,
paddr_t end_gpaddr,
paddr_t maddr,
int mattr,
p2m_type_t t)
{
int rc;
struct p2m_domain *p2m = &d->arch.p2m;
lpae_t *first = NULL, *second = NULL, *third = NULL;
paddr_t addr;
unsigned long cur_first_page = ~0,
cur_first_offset = ~0,
cur_second_offset = ~0;
unsigned long count = 0;
unsigned int flush = 0;
bool_t populate = (op == INSERT || op == ALLOCATE);
lpae_t pte;
spin_lock(&p2m->lock);
if ( d != current->domain )
p2m_load_VTTBR(d);
addr = start_gpaddr;
while ( addr < end_gpaddr )
{
if ( cur_first_page != p2m_first_level_index(addr) )
{
if ( first ) unmap_domain_page(first);
first = p2m_map_first(p2m, addr);
if ( !first )
{
rc = -EINVAL;
goto out;
}
cur_first_page = p2m_first_level_index(addr);
}
if ( !first[first_table_offset(addr)].p2m.valid )
{
if ( !populate )
{
addr = (addr + FIRST_SIZE) & FIRST_MASK;
continue;
}
rc = p2m_create_table(d, &first[first_table_offset(addr)]);
if ( rc < 0 )
{
printk("p2m_populate_ram: L1 failed\n");
goto out;
}
}
BUG_ON(!first[first_table_offset(addr)].p2m.valid);
if ( cur_first_offset != first_table_offset(addr) )
{
if (second) unmap_domain_page(second);
second = map_domain_page(first[first_table_offset(addr)].p2m.base);
cur_first_offset = first_table_offset(addr);
}
/* else: second already valid */
if ( !second[second_table_offset(addr)].p2m.valid )
{
if ( !populate )
{
addr = (addr + SECOND_SIZE) & SECOND_MASK;
continue;
}
rc = p2m_create_table(d, &second[second_table_offset(addr)]);
if ( rc < 0 ) {
printk("p2m_populate_ram: L2 failed\n");
goto out;
}
}
BUG_ON(!second[second_table_offset(addr)].p2m.valid);
if ( cur_second_offset != second_table_offset(addr) )
{
/* map third level */
if (third) unmap_domain_page(third);
third = map_domain_page(second[second_table_offset(addr)].p2m.base);
cur_second_offset = second_table_offset(addr);
}
pte = third[third_table_offset(addr)];
flush |= pte.p2m.valid;
/* TODO: Handle other p2m type
*
* It's safe to do the put_page here because page_alloc will
* flush the TLBs if the page is reallocated before the end of
* this loop.
*/
if ( pte.p2m.valid && p2m_is_foreign(pte.p2m.type) )
{
unsigned long mfn = pte.p2m.base;
ASSERT(mfn_valid(mfn));
put_page(mfn_to_page(mfn));
}
/* Allocate a new RAM page and attach */
switch (op) {
case ALLOCATE:
{
struct page_info *page;
ASSERT(!pte.p2m.valid);
rc = -ENOMEM;
page = alloc_domheap_page(d, 0);
if ( page == NULL ) {
printk("p2m_populate_ram: failed to allocate page\n");
goto out;
}
pte = mfn_to_p2m_entry(page_to_mfn(page), mattr, t);
write_pte(&third[third_table_offset(addr)], pte);
}
break;
case INSERT:
{
pte = mfn_to_p2m_entry(maddr >> PAGE_SHIFT, mattr, t);
write_pte(&third[third_table_offset(addr)], pte);
maddr += PAGE_SIZE;
}
break;
case RELINQUISH:
case REMOVE:
{
if ( !pte.p2m.valid )
{
count++;
break;
}
count += 0x10;
memset(&pte, 0x00, sizeof(pte));
write_pte(&third[third_table_offset(addr)], pte);
count++;
}
break;
case CACHEFLUSH:
{
if ( !pte.p2m.valid || !p2m_is_ram(pte.p2m.type) )
break;
flush_page_to_ram(pte.p2m.base);
}
break;
}
/* Preempt every 2MiB (mapped) or 32 MiB (unmapped) - arbitrary */
if ( op == RELINQUISH && count >= 0x2000 )
{
if ( hypercall_preempt_check() )
{
p2m->lowest_mapped_gfn = addr >> PAGE_SHIFT;
rc = -EAGAIN;
goto out;
}
count = 0;
}
/* Got the next page */
addr += PAGE_SIZE;
}
if ( flush )
{
/* At the beginning of the function, Xen is updating VTTBR
* with the domain where the mappings are created. In this
* case it's only necessary to flush TLBs on every CPUs with
* the current VMID (our domain).
*/
flush_tlb();
}
if ( op == ALLOCATE || op == INSERT )
{
unsigned long sgfn = paddr_to_pfn(start_gpaddr);
unsigned long egfn = paddr_to_pfn(end_gpaddr);
p2m->max_mapped_gfn = MAX(p2m->max_mapped_gfn, egfn);
p2m->lowest_mapped_gfn = MIN(p2m->lowest_mapped_gfn, sgfn);
}
rc = 0;
out:
if (third) unmap_domain_page(third);
if (second) unmap_domain_page(second);
if (first) unmap_domain_page(first);
if ( d != current->domain )
p2m_load_VTTBR(current->domain);
spin_unlock(&p2m->lock);
return rc;
}
/*
* do_no_page() tries to create a new page mapping. It aggressively
* tries to share with existing pages, but makes a separate copy if
* the "write_access" parameter is true in order to avoid the next
* page fault.
*
* As this is called only for pages that do not currently exist, we
* do not need to flush old virtual caches or the TLB.
*/
void do_no_page(struct task_struct * tsk, struct vm_area_struct * vma,
unsigned long address, int write_access)
{
pgd_t * pgd;
pmd_t * pmd;
pte_t * page_table;
pte_t entry;
unsigned long page;
pgd = pgd_offset(tsk->mm, address);
pmd = pmd_alloc(pgd, address);
if (!pmd)
goto no_memory;
page_table = pte_alloc(pmd, address);
if (!page_table)
goto no_memory;
entry = *page_table;
if (pte_present(entry))
goto is_present;
if (!pte_none(entry))
goto swap_page;
address &= PAGE_MASK;
if (!vma->vm_ops || !vma->vm_ops->nopage)
goto anonymous_page;
/*
* The third argument is "no_share", which tells the low-level code
* to copy, not share the page even if sharing is possible. It's
* essentially an early COW detection
*/
page = vma->vm_ops->nopage(vma, address,
(vma->vm_flags & VM_SHARED)?0:write_access);
if (!page)
goto sigbus;
++tsk->maj_flt;
++vma->vm_mm->rss;
/*
* This silly early PAGE_DIRTY setting removes a race
* due to the bad i386 page protection. But it's valid
* for other architectures too.
*
* Note that if write_access is true, we either now have
* a exclusive copy of the page, or this is a shared mapping,
* so we can make it writable and dirty to avoid having to
* handle that later.
*/
flush_page_to_ram(page);
entry = mk_pte(page, vma->vm_page_prot);
if (write_access) {
entry = pte_mkwrite(pte_mkdirty(entry));
} else if (mem_map[MAP_NR(page)].count > 1 && !(vma->vm_flags & VM_SHARED))
entry = pte_wrprotect(entry);
put_page(page_table, entry);
/* no need to invalidate: a not-present page shouldn't be cached */
return;
anonymous_page:
entry = pte_wrprotect(mk_pte(ZERO_PAGE, vma->vm_page_prot));
if (write_access) {
unsigned long page = __get_free_page(GFP_KERNEL);
if (!page)
goto sigbus;
memset((void *) page, 0, PAGE_SIZE);
entry = pte_mkwrite(pte_mkdirty(mk_pte(page, vma->vm_page_prot)));
vma->vm_mm->rss++;
tsk->min_flt++;
flush_page_to_ram(page);
}
put_page(page_table, entry);
return;
sigbus:
force_sig(SIGBUS, current);
put_page(page_table, BAD_PAGE);
/* no need to invalidate, wasn't present */
return;
swap_page:
do_swap_page(tsk, vma, address, page_table, entry, write_access);
return;
no_memory:
oom(tsk);
is_present:
return;
}
/*
* Actual dumper
*
* This is a two-pass process; first we find the offsets of the bits,
* and then they are actually written out. If we run out of core limit
* we just truncate.
*/
static int elf_core_dump(long signr, struct pt_regs * regs, struct file * file)
{
int has_dumped = 0;
mm_segment_t fs;
int segs;
size_t size = 0;
int i;
struct vm_area_struct *vma;
struct elfhdr elf;
off_t offset = 0, dataoff;
unsigned long limit = current->rlim[RLIMIT_CORE].rlim_cur;
int numnote = 4;
struct memelfnote notes[4];
struct elf_prstatus prstatus; /* NT_PRSTATUS */
elf_fpregset_t fpu; /* NT_PRFPREG */
struct elf_prpsinfo psinfo; /* NT_PRPSINFO */
/* first copy the parameters from user space */
memset(&psinfo, 0, sizeof(psinfo));
{
unsigned int i, len;
len = current->mm->arg_end - current->mm->arg_start;
if (len >= ELF_PRARGSZ)
len = ELF_PRARGSZ-1;
copy_from_user(&psinfo.pr_psargs,
(const char *)current->mm->arg_start, len);
for(i = 0; i < len; i++)
if (psinfo.pr_psargs[i] == 0)
psinfo.pr_psargs[i] = ' ';
psinfo.pr_psargs[len] = 0;
}
memset(&prstatus, 0, sizeof(prstatus));
/*
* This transfers the registers from regs into the standard
* coredump arrangement, whatever that is.
*/
#ifdef ELF_CORE_COPY_REGS
ELF_CORE_COPY_REGS(prstatus.pr_reg, regs)
#else
if (sizeof(elf_gregset_t) != sizeof(struct pt_regs))
{
printk("sizeof(elf_gregset_t) (%ld) != sizeof(struct pt_regs) (%ld)\n",
(long)sizeof(elf_gregset_t), (long)sizeof(struct pt_regs));
}
else
*(struct pt_regs *)&prstatus.pr_reg = *regs;
#endif
/* now stop all vm operations */
down_write(¤t->mm->mmap_sem);
segs = current->mm->map_count;
#ifdef DEBUG
printk("elf_core_dump: %d segs %lu limit\n", segs, limit);
#endif
/* Set up header */
memcpy(elf.e_ident, ELFMAG, SELFMAG);
elf.e_ident[EI_CLASS] = ELF_CLASS;
elf.e_ident[EI_DATA] = ELF_DATA;
elf.e_ident[EI_VERSION] = EV_CURRENT;
memset(elf.e_ident+EI_PAD, 0, EI_NIDENT-EI_PAD);
elf.e_type = ET_CORE;
elf.e_machine = ELF_ARCH;
elf.e_version = EV_CURRENT;
elf.e_entry = 0;
elf.e_phoff = sizeof(elf);
elf.e_shoff = 0;
#ifdef ELF_CORE_EFLAGS
elf.e_flags = ELF_CORE_EFLAGS;
#else
elf.e_flags = 0;
#endif
elf.e_ehsize = sizeof(elf);
elf.e_phentsize = sizeof(struct elf_phdr);
elf.e_phnum = segs+1; /* Include notes */
elf.e_shentsize = 0;
elf.e_shnum = 0;
elf.e_shstrndx = 0;
fs = get_fs();
set_fs(KERNEL_DS);
has_dumped = 1;
current->flags |= PF_DUMPCORE;
DUMP_WRITE(&elf, sizeof(elf));
offset += sizeof(elf); /* Elf header */
offset += (segs+1) * sizeof(struct elf_phdr); /* Program headers */
/*
* Set up the notes in similar form to SVR4 core dumps made
* with info from their /proc.
*/
notes[0].name = "CORE";
notes[0].type = NT_PRSTATUS;
notes[0].datasz = sizeof(prstatus);
notes[0].data = &prstatus;
prstatus.pr_info.si_signo = prstatus.pr_cursig = signr;
prstatus.pr_sigpend = current->pending.signal.sig[0];
prstatus.pr_sighold = current->blocked.sig[0];
psinfo.pr_pid = prstatus.pr_pid = current->pid;
psinfo.pr_ppid = prstatus.pr_ppid = current->p_pptr->pid;
psinfo.pr_pgrp = prstatus.pr_pgrp = current->pgrp;
psinfo.pr_sid = prstatus.pr_sid = current->session;
prstatus.pr_utime.tv_sec = CT_TO_SECS(current->times.tms_utime);
prstatus.pr_utime.tv_usec = CT_TO_USECS(current->times.tms_utime);
prstatus.pr_stime.tv_sec = CT_TO_SECS(current->times.tms_stime);
prstatus.pr_stime.tv_usec = CT_TO_USECS(current->times.tms_stime);
prstatus.pr_cutime.tv_sec = CT_TO_SECS(current->times.tms_cutime);
prstatus.pr_cutime.tv_usec = CT_TO_USECS(current->times.tms_cutime);
prstatus.pr_cstime.tv_sec = CT_TO_SECS(current->times.tms_cstime);
prstatus.pr_cstime.tv_usec = CT_TO_USECS(current->times.tms_cstime);
#ifdef DEBUG
dump_regs("Passed in regs", (elf_greg_t *)regs);
dump_regs("prstatus regs", (elf_greg_t *)&prstatus.pr_reg);
#endif
notes[1].name = "CORE";
notes[1].type = NT_PRPSINFO;
notes[1].datasz = sizeof(psinfo);
notes[1].data = &psinfo;
i = current->state ? ffz(~current->state) + 1 : 0;
psinfo.pr_state = i;
psinfo.pr_sname = (i < 0 || i > 5) ? '.' : "RSDZTD"[i];
psinfo.pr_zomb = psinfo.pr_sname == 'Z';
psinfo.pr_nice = task_nice(current);
psinfo.pr_flag = current->flags;
psinfo.pr_uid = NEW_TO_OLD_UID(current->uid);
psinfo.pr_gid = NEW_TO_OLD_GID(current->gid);
strncpy(psinfo.pr_fname, current->comm, sizeof(psinfo.pr_fname));
notes[2].name = "CORE";
notes[2].type = NT_TASKSTRUCT;
notes[2].datasz = sizeof(*current);
notes[2].data = current;
/* Try to dump the FPU. */
prstatus.pr_fpvalid = dump_fpu (regs, &fpu);
if (!prstatus.pr_fpvalid)
{
numnote--;
}
else
{
notes[3].name = "CORE";
notes[3].type = NT_PRFPREG;
notes[3].datasz = sizeof(fpu);
notes[3].data = &fpu;
}
/* Write notes phdr entry */
{
struct elf_phdr phdr;
int sz = 0;
for(i = 0; i < numnote; i++)
sz += notesize(¬es[i]);
phdr.p_type = PT_NOTE;
phdr.p_offset = offset;
phdr.p_vaddr = 0;
phdr.p_paddr = 0;
phdr.p_filesz = sz;
phdr.p_memsz = 0;
phdr.p_flags = 0;
phdr.p_align = 0;
offset += phdr.p_filesz;
DUMP_WRITE(&phdr, sizeof(phdr));
}
/* Page-align dumped data */
dataoff = offset = roundup(offset, ELF_EXEC_PAGESIZE);
/* Write program headers for segments dump */
for(vma = current->mm->mmap; vma != NULL; vma = vma->vm_next) {
struct elf_phdr phdr;
size_t sz;
sz = vma->vm_end - vma->vm_start;
phdr.p_type = PT_LOAD;
phdr.p_offset = offset;
phdr.p_vaddr = vma->vm_start;
phdr.p_paddr = 0;
phdr.p_filesz = maydump(vma) ? sz : 0;
phdr.p_memsz = sz;
offset += phdr.p_filesz;
phdr.p_flags = vma->vm_flags & VM_READ ? PF_R : 0;
if (vma->vm_flags & VM_WRITE) phdr.p_flags |= PF_W;
if (vma->vm_flags & VM_EXEC) phdr.p_flags |= PF_X;
phdr.p_align = ELF_EXEC_PAGESIZE;
DUMP_WRITE(&phdr, sizeof(phdr));
}
for(i = 0; i < numnote; i++)
if (!writenote(¬es[i], file))
goto end_coredump;
DUMP_SEEK(dataoff);
for(vma = current->mm->mmap; vma != NULL; vma = vma->vm_next) {
unsigned long addr;
if (!maydump(vma))
continue;
#ifdef DEBUG
printk("elf_core_dump: writing %08lx-%08lx\n", vma->vm_start, vma->vm_end);
#endif
for (addr = vma->vm_start;
addr < vma->vm_end;
addr += PAGE_SIZE) {
struct page* page;
struct vm_area_struct *vma;
if (get_user_pages(current, current->mm, addr, 1, 0, 1,
&page, &vma) <= 0) {
DUMP_SEEK (file->f_pos + PAGE_SIZE);
} else {
if (page == ZERO_PAGE(addr)) {
DUMP_SEEK (file->f_pos + PAGE_SIZE);
} else {
void *kaddr;
flush_cache_page(vma, addr);
kaddr = kmap(page);
DUMP_WRITE(kaddr, PAGE_SIZE);
flush_page_to_ram(page);
kunmap(page);
}
put_page(page);
}
}
}
if ((off_t) file->f_pos != offset) {
/* Sanity check */
printk("elf_core_dump: file->f_pos (%ld) != offset (%ld)\n",
(off_t) file->f_pos, offset);
}
end_coredump:
set_fs(fs);
up_write(¤t->mm->mmap_sem);
return has_dumped;
}
/*
* We hold the mm semaphore and the page_table_lock on entry and
* should release the pagetable lock on exit..
*/
static int do_swap_page(struct mm_struct * mm,
struct vm_area_struct * vma, unsigned long address,
pte_t * page_table, pte_t orig_pte, int write_access)
{
struct page *page;
swp_entry_t entry = pte_to_swp_entry(orig_pte);
pte_t pte;
int ret = 1;
spin_unlock(&mm->page_table_lock);
page = lookup_swap_cache(entry);
if (!page) {
swapin_readahead(entry);
page = read_swap_cache_async(entry);
if (!page) {
/*
* Back out if somebody else faulted in this pte while
* we released the page table lock.
*/
int retval;
spin_lock(&mm->page_table_lock);
retval = pte_same(*page_table, orig_pte) ? -1 : 1;
spin_unlock(&mm->page_table_lock);
return retval;
}
/* Had to read the page from swap area: Major fault */
ret = 2;
}
lock_page(page);
/*
* Back out if somebody else faulted in this pte while we
* released the page table lock.
*/
spin_lock(&mm->page_table_lock);
if (!pte_same(*page_table, orig_pte)) {
spin_unlock(&mm->page_table_lock);
unlock_page(page);
page_cache_release(page);
return 1;
}
/* The page isn't present yet, go ahead with the fault. */
swap_free(entry);
if (vm_swap_full())
remove_exclusive_swap_page(page);
mm->rss++;
pte = mk_pte(page, vma->vm_page_prot);
if (write_access && can_share_swap_page(page))
pte = pte_mkdirty(pte_mkwrite(pte));
unlock_page(page);
flush_page_to_ram(page);
flush_icache_page(vma, page);
set_pte(page_table, pte);
/* No need to invalidate - it was non-present before */
update_mmu_cache(vma, address, pte);
spin_unlock(&mm->page_table_lock);
return ret;
}