static int
pin_page_for_write(const void __user *_addr, pte_t **ptep, spinlock_t **ptlp)
{
unsigned long addr = (unsigned long)_addr;
pgd_t *pgd;
pmd_t *pmd;
pte_t *pte;
pud_t *pud;
spinlock_t *ptl;
pgd = pgd_offset(current->mm, addr);
if (unlikely(pgd_none(*pgd) || pgd_bad(*pgd)))
return 0;
pud = pud_offset(pgd, addr);
if (unlikely(pud_none(*pud) || pud_bad(*pud)))
return 0;
pmd = pmd_offset(pud, addr);
if (unlikely(pmd_none(*pmd) || pmd_bad(*pmd)))
return 0;
pte = pte_offset_map_lock(current->mm, pmd, addr, &ptl);
if (unlikely(!pte_present(*pte) || !pte_young(*pte) ||
!pte_write(*pte) || !pte_dirty(*pte))) {
pte_unmap_unlock(pte, ptl);
return 0;
}
*ptep = pte;
*ptlp = ptl;
return 1;
}
/*
* No need to decide whether this PTE shares the swap entry with others,
* just let do_wp_page work it out if a write is requested later - to
* force COW, vm_page_prot omits write permission from any private vma.
*/
static int unuse_pte(struct vm_area_struct *vma, pmd_t *pmd,
unsigned long addr, swp_entry_t entry, struct page *page)
{
spinlock_t *ptl;
pte_t *pte;
int ret = 1;
if (mem_cgroup_charge(page, vma->vm_mm, GFP_KERNEL))
ret = -ENOMEM;
pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
if (unlikely(!pte_same(*pte, swp_entry_to_pte(entry)))) {
if (ret > 0)
mem_cgroup_uncharge_page(page);
ret = 0;
goto out;
}
inc_mm_counter(vma->vm_mm, anon_rss);
get_page(page);
set_pte_at(vma->vm_mm, addr, pte,
pte_mkold(mk_pte(page, vma->vm_page_prot)));
page_add_anon_rmap(page, vma, addr);
swap_free(entry);
/*
* Move the page to the active list so it is not
* immediately swapped out again after swapon.
*/
activate_page(page);
out:
pte_unmap_unlock(pte, ptl);
return ret;
}
static int smaps_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,
struct mm_walk *walk)
{
struct vm_area_struct *vma = walk->vma;
pte_t *pte;
spinlock_t *ptl;
if (pmd_trans_huge_lock(pmd, vma, &ptl) == 1) {
smaps_pmd_entry(pmd, addr, walk);
spin_unlock(ptl);
return 0;
}
if (pmd_trans_unstable(pmd))
return 0;
/*
* The mmap_sem held all the way back in m_start() is what
* keeps khugepaged out of here and from collapsing things
* in here.
*/
pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
for (; addr != end; pte++, addr += PAGE_SIZE)
smaps_pte_entry(pte, addr, walk);
pte_unmap_unlock(pte - 1, ptl);
cond_resched();
return 0;
}
static struct page* my_follow_page(struct vm_area_struct *vma, unsigned long addr) {
pud_t *pud = NULL;
pmd_t *pmd = NULL;
pgd_t *pgd = NULL;
pte_t *pte = NULL;
spinlock_t *ptl = NULL;
struct page* page = NULL;
struct mm_struct *mm = current->mm;
pgd = pgd_offset(current->mm, addr);
if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd))) {
goto out;
}
pud = pud_offset(pgd, addr);
if (pud_none(*pud) || unlikely(pud_bad(*pud))) {
goto out;
}
printk("aaaa\n");
pmd = pmd_offset(pud, addr);
if (pmd_none(*pmd) || unlikely(pmd_bad(*pmd))) {
goto out;
}
pte = pte_offset_map_lock(current->mm, pmd, addr, &ptl);
printk("bbbb\n");
if (!pte) goto out;
printk("cccc\n");
if (!pte_present(*pte)) goto unlock;
page = pfn_to_page(pte_pfn(*pte));
if (!page) goto unlock;
get_page(page);
unlock:
pte_unmap_unlock(pte, ptl);
out:
return page;
}
static int mfill_zeropage_pte(struct mm_struct *dst_mm,
pmd_t *dst_pmd,
struct vm_area_struct *dst_vma,
unsigned long dst_addr)
{
pte_t _dst_pte, *dst_pte;
spinlock_t *ptl;
int ret;
pgoff_t offset, max_off;
struct inode *inode;
_dst_pte = pte_mkspecial(pfn_pte(my_zero_pfn(dst_addr),
dst_vma->vm_page_prot));
dst_pte = pte_offset_map_lock(dst_mm, dst_pmd, dst_addr, &ptl);
if (dst_vma->vm_file) {
/* the shmem MAP_PRIVATE case requires checking the i_size */
inode = dst_vma->vm_file->f_inode;
offset = linear_page_index(dst_vma, dst_addr);
max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
ret = -EFAULT;
if (unlikely(offset >= max_off))
goto out_unlock;
}
ret = -EEXIST;
if (!pte_none(*dst_pte))
goto out_unlock;
set_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte);
/* No need to invalidate - it was non-present before */
update_mmu_cache(dst_vma, dst_addr, dst_pte);
ret = 0;
out_unlock:
pte_unmap_unlock(dst_pte, ptl);
return ret;
}
static int clear_refs_pte_range(pmd_t *pmd, unsigned long addr,
unsigned long end, struct mm_walk *walk)
{
struct clear_refs_private *cp = walk->private;
struct vm_area_struct *vma = walk->vma;
pte_t *pte, ptent;
spinlock_t *ptl;
struct page *page;
if (pmd_trans_huge_lock(pmd, vma, &ptl) == 1) {
if (cp->type == CLEAR_REFS_SOFT_DIRTY) {
clear_soft_dirty_pmd(vma, addr, pmd);
goto out;
}
page = pmd_page(*pmd);
/* Clear accessed and referenced bits. */
pmdp_test_and_clear_young(vma, addr, pmd);
ClearPageReferenced(page);
out:
spin_unlock(ptl);
return 0;
}
if (pmd_trans_unstable(pmd))
return 0;
pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
for (; addr != end; pte++, addr += PAGE_SIZE) {
ptent = *pte;
if (cp->type == CLEAR_REFS_SOFT_DIRTY) {
clear_soft_dirty(vma, addr, pte);
continue;
}
if (!pte_present(ptent))
continue;
page = vm_normal_page(vma, addr, ptent);
if (!page)
continue;
/* Clear accessed and referenced bits. */
ptep_test_and_clear_young(vma, addr, pte);
ClearPageReferenced(page);
}
pte_unmap_unlock(pte - 1, ptl);
cond_resched();
return 0;
}
static void mincore_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
unsigned long addr, unsigned long end,
unsigned char *vec)
{
unsigned long next;
spinlock_t *ptl;
pte_t *ptep;
ptep = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
do {
pte_t pte = *ptep;
pgoff_t pgoff;
next = addr + PAGE_SIZE;
if (pte_none(pte))
mincore_unmapped_range(vma, addr, next, vec);
else if (pte_present(pte))
*vec = 1;
else if (pte_file(pte)) {
pgoff = pte_to_pgoff(pte);
*vec = mincore_page(vma->vm_file->f_mapping, pgoff);
} else { /* pte is a swap entry */
swp_entry_t entry = pte_to_swp_entry(pte);
if (is_migration_entry(entry)) {
/* migration entries are always uptodate */
*vec = 1;
} else {
#ifdef CONFIG_SWAP
pgoff = entry.val;
*vec = mincore_page(&swapper_space, pgoff);
#else
WARN_ON(1);
*vec = 1;
#endif
}
}
vec++;
} while (ptep++, addr = next, addr != end);
pte_unmap_unlock(ptep - 1, ptl);
}
inline int make_page_cow(struct mm_struct *mm, struct vm_area_struct *vma,
unsigned long addr)
{
pgd_t *pgd;
pud_t *pud;
pmd_t *pmd;
pte_t *pte;
spinlock_t *ptl;
pgd = pgd_offset(mm, addr);
if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd)))
goto no_page;
pud = pud_offset(pgd, addr);
if (pud_none(*pud) || unlikely(pud_bad(*pud)))
goto no_page;
pmd = pmd_offset(pud, addr);
if (pmd_none(*pmd) || unlikely(pmd_bad(*pmd)))
goto no_page;
BUG_ON(pmd_trans_huge(*pmd));
pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
if (!pte_present(*pte)) {
spin_unlock(ptl);
goto no_page;
}
ptep_set_wrprotect(mm, addr, pte);
spin_unlock(ptl);
#if !defined(CONFIG_GRAPHENE_BULK_IPC) && LINUX_VERSION_CODE >= KERNEL_VERSION(3, 2, 0)
my_flush_tlb_page(vma, addr);
#else
flush_tlb_page(vma, addr);
#endif
DEBUG("make page COW at %lx\n", addr);
return 0;
no_page:
return -EFAULT;
}
static int mfill_zeropage_pte(struct mm_struct *dst_mm,
pmd_t *dst_pmd,
struct vm_area_struct *dst_vma,
unsigned long dst_addr)
{
pte_t _dst_pte, *dst_pte;
spinlock_t *ptl;
int ret;
_dst_pte = pte_mkspecial(pfn_pte(my_zero_pfn(dst_addr),
dst_vma->vm_page_prot));
ret = -EEXIST;
dst_pte = pte_offset_map_lock(dst_mm, dst_pmd, dst_addr, &ptl);
if (!pte_none(*dst_pte))
goto out_unlock;
set_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte);
/* No need to invalidate - it was non-present before */
update_mmu_cache(dst_vma, dst_addr, dst_pte);
ret = 0;
out_unlock:
pte_unmap_unlock(dst_pte, ptl);
return ret;
}
static struct page* mtest_seek_page(struct vm_area_struct *vma, unsigned long addr)
{
pgd_t *pgd; //top level page table
pud_t *pud; //second level page table
pmd_t *pmd; //third level page table
pte_t *pte; //last level page table
spinlock_t *ptl;
struct page *page = NULL;
struct mm_struct *mm = vma->vm_mm;
pgd = pgd_offset(mm, addr);
if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd))) return NULL;
pud = pud_offset(pgd, addr);
if (pud_none(*pud) || unlikely(pud_bad(*pud))) return NULL;
pmd = pmd_offset(pud, addr);
if (pmd_none(*pmd) || unlikely(pmd_bad(*pmd))) return NULL;
pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
if (!pte) return NULL;
if (!pte_present(*pte)){
pte_unmap_unlock(pte, ptl);
return NULL;
}
page = pfn_to_page(pte_pfn(*pte));
if (!page){
pte_unmap_unlock(pte, ptl);
return NULL;
}
get_page(page);
pte_unmap_unlock(pte, ptl);
return page;
}
static unsigned long msync_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
unsigned long addr, unsigned long end)
{
pte_t *pte;
spinlock_t *ptl;
int progress = 0;
unsigned long ret = 0;
again:
pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
do {
struct page *page;
if (progress >= 64) {
progress = 0;
if (need_resched() || need_lockbreak(ptl))
break;
}
progress++;
if (!pte_present(*pte))
continue;
if (!pte_maybe_dirty(*pte))
continue;
page = vm_normal_page(vma, addr, *pte);
if (!page)
continue;
if (ptep_clear_flush_dirty(vma, addr, pte) ||
page_test_and_clear_dirty(page))
ret += set_page_dirty(page);
progress += 3;
} while (pte++, addr += PAGE_SIZE, addr != end);
pte_unmap_unlock(pte - 1, ptl);
cond_resched();
if (addr != end)
goto again;
return ret;
}
static int unuse_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
unsigned long addr, unsigned long end,
swp_entry_t entry, struct page *page)
{
pte_t swp_pte = swp_entry_to_pte(entry);
pte_t *pte;
spinlock_t *ptl;
int found = 0;
pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
do {
/*
* swapoff spends a _lot_ of time in this loop!
* Test inline before going to call unuse_pte.
*/
if (unlikely(pte_same(*pte, swp_pte))) {
unuse_pte(vma, pte++, addr, entry, page);
found = 1;
break;
}
} while (pte++, addr += PAGE_SIZE, addr != end);
pte_unmap_unlock(pte - 1, ptl);
return found;
}
/*pgtable sequential scan and count for __access_bits.*/
static int scan_pgtable(void)
{
pgd_t *pgd = NULL;
pud_t *pud = NULL;
pmd_t *pmd = NULL;
pte_t *ptep, pte;
spinlock_t *ptl;
struct mm_struct *mm;
struct vm_area_struct *vma;
unsigned long start = 0; /*the start of address.*/
unsigned long end = 0; /*the end of address.*/
unsigned long address = 0; /* the address of vma.*/
int number_hotpages = 0; /* the number of hot pages */
int number_vpages = 0;
int cycle_index = 0; /* the loop counter, which denotes ITERATIONS. */
/* the array that records the number of hot page in every cycle */
int hot_page[ITERATIONS];
int number_current_pg = 0;
int pg_count = 0;
int j = 0;
int times = 0; /* records reuse time*/
/* some variables that describe page "heat" */
int hig = 0;
int mid = 0;
int low = 0;
int llow = 0;
int lllow = 0;
int llllow = 0;
int all_pages = 0;/* the total number of pages */
/*the average number of hot pages in each iteration.*/
long avg_hotpage=0;
/*the total number of memory accesses across all pages*/
long num_access=0;
/* avg utilization of each page */
int avg_page_utilization = 0;
/*get the handle of current running benchmark.*/
struct task_struct *bench_process = get_current_process();
if(bench_process == NULL)
{
printk("sysmon: get no process handle in scan_pgtable function...exit&trying again...\n");
return 0;
}
else /* get the process*/
mm = bench_process->mm;
if(mm == NULL)
{
printk("sysmon: error mm is NULL, return back & trying...\n");
return 0;
}
for(j = 0; j < PAGE_ALL; j++)
page_heat[j] = -1;
for(j = 0; j < ITERATIONS; j++)
{
hot_page[j] = 0;
reuse_time[j] = 0;
dirty_page[j] = 0;
}
/*yanghao*/
times = 0;
for(cycle_index = 0; cycle_index < ITERATIONS; cycle_index++)
{
number_hotpages = 0;
/*scan each vma*/
for(vma = mm->mmap; vma; vma = vma->vm_next)
{
start = vma->vm_start;
end = vma->vm_end;
mm = vma->vm_mm;
/*in each vma, we check all pages*/
for(address = start; address < end; address += PAGE_SIZE)
{
/*scan page table for each page in this VMA*/
pgd = pgd_offset(mm, address);
if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd)))
continue;
pud = pud_offset(pgd, address);
if (pud_none(*pud) || unlikely(pud_bad(*pud)))
continue;
pmd = pmd_offset(pud, address);
if (pmd_none(*pmd) || unlikely(pmd_bad(*pmd)))
continue;
ptep = pte_offset_map_lock(mm, pmd, address, &ptl);
pte = *ptep;
if(pte_present(pte))
{
if(pte_young(pte)) /*hot page*/
{
/*re-set and clear _access_bits to 0*/
pte = pte_mkold(pte);
set_pte_at(mm, address, ptep, pte);
/*yanghao:re-set and clear _dirty_bits to 0*/
pte = pte_mkclean(pte);
set_pte_at(mm, address, ptep, pte);
}
}
else /*no page pte_none*/
{
pte_unmap_unlock(ptep, ptl);
continue;
}
pte_unmap_unlock(ptep, ptl);
page_counts++;
}
}
/*count the number of hot pages*/
if(bench_process == NULL)
{
printk("sysmon: get no process handle in scan_pgtable function...exit&trying again...\n");
return 0;
}
else /*get the process*/
mm = bench_process->mm;
if(mm == NULL)
{
printk("sysmon: error mm is NULL, return back & trying...\n");
return 0;
}
number_vpages = 0;
sampling_interval = page_counts / 250; /*yanghao:*/
page_counts = 0;
for(vma = mm->mmap; vma; vma = vma->vm_next)
{
start = vma->vm_start;
end = vma->vm_end;
/*scan each page in this VMA*/
mm = vma->vm_mm;
pg_count = 0;
for(address = start; address < end; address += PAGE_SIZE)
{
/*scan page table for each page in this VMA*/
pgd = pgd_offset(mm, address);
if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd)))
continue;
pud = pud_offset(pgd, address);
if (pud_none(*pud) || unlikely(pud_bad(*pud)))
continue;
pmd = pmd_offset(pud, address);
if (pmd_none(*pmd) || unlikely(pmd_bad(*pmd)))
continue;
ptep = pte_offset_map_lock(mm, pmd, address, &ptl);
pte = *ptep;
if(pte_present(pte))
{
if(pte_young(pte)) /* hot pages*/
{
number_current_pg = pg_count + number_vpages;
page_heat[number_current_pg]++;
hot_page[cycle_index]++;
/*yanghao:*/
if (page_counts == random_page)
{
times++;
if (pte_dirty(pte))
dirty_page[cycle_index] = 1;
}
}
else
{
if (page_counts == random_page)
reuse_time[times]++;
}
}
pg_count++;
pte_unmap_unlock(ptep, ptl);
page_counts++;
}
number_vpages += (int)(end - start)/PAGE_SIZE;
}
}
/*yanghao:cal. the No. of random_page*/
random_page += sampling_interval;
if(random_page >= page_counts)
random_page=page_counts / 300;
/*****************************OUTPUT************************************/
for(j = 0; j < PAGE_ALL; j++)
{
if(page_heat[j] < VH && page_heat[j] > H)
hig++;
if(page_heat[j] > M && page_heat[j] <= H)
mid++;
if(page_heat[j] <= M && page_heat[j] > L)
low++;
if(page_heat[j] > VL_MAX && page_heat[j] <= L)
llow++;
if(page_heat[j] > VL_MIN && page_heat[j] <= VL_MAX)
lllow++;
if(page_heat[j] >= 0 && page_heat[j] <= VL_MIN)
llllow++;
if(page_heat[j] > -1)
all_pages++;
}
/*the values reflect the accessing frequency of each physical page.*/
printk("[LOG: after sampling (%d loops) ...] ",ITERATIONS);
printk("the values denote the physical page accessing frequence.\n");
printk("-->hig (150,200) is %d. Indicating the number of re-used pages is high.\n",hig);
printk("-->mid (100,150] is %d.\n",mid);
printk("-->low (64,100] is %d.\n",low);
printk("-->llow (10,64] is %d. In locality,no too many re-used pages.\n",llow);
printk("-->lllow (5,10] is %d.\n",lllow);
printk("-->llllow [1,5] is %d.\n",llllow);
for(j = 0;j < ITERATIONS; j++)
avg_hotpage += hot_page[j];
avg_hotpage /= (j+1);
/*
* new step@20140704
* (1)the different phases of memory utilization
* (2)the avg. page accessing utilization
* (3)memory pages layout and spectrum
*/
for(j = 0; j < PAGE_ALL; j++)
if(page_heat[j] > -1) /*the page that is accessed at least once.*/
num_access += (page_heat[j] + 1);
printk("the total number of memory accesses is %ld, the average is %ld\n",
num_access, num_access / ITERATIONS);
avg_page_utilization = num_access / all_pages;
printk("Avg hot pages num is %ld, all used pages num is %d, avg utilization of each page is %d\n",
avg_hotpage, all_pages, avg_page_utilization);
/*yanghao:print the information about reuse-distance*/
if ((times == 0) && (reuse_time[0] ==0))
printk("the page No.%d is not available.",random_page);
else
{
if ((times == 0) && (reuse_time[0] == 0))
printk("the page No.%d was not used in this 200 loops.",random_page);
else
{
if (times < ITERATIONS)
times++;
printk("the reusetime of page No.%d is:",random_page);
for (j = 0; j < times; j++)
printk("%d ",reuse_time[j]);
printk("\n");
printk("the total number of the digit above denotes the sum that page NO.%d be accessd in %d loops.\n",
random_page,ITERATIONS);
printk("each digit means the sum loops that between current loop and the last loop.\n");
}
}
printk("\n\n");
return 1;
}
static struct page *follow_page_pte(struct vm_area_struct *vma,
unsigned long address, pmd_t *pmd, unsigned int flags)
{
struct mm_struct *mm = vma->vm_mm;
struct dev_pagemap *pgmap = NULL;
struct page *page;
spinlock_t *ptl;
pte_t *ptep, pte;
retry:
if (unlikely(pmd_bad(*pmd)))
return no_page_table(vma, flags);
ptep = pte_offset_map_lock(mm, pmd, address, &ptl);
pte = *ptep;
if (!pte_present(pte)) {
swp_entry_t entry;
/*
* KSM's break_ksm() relies upon recognizing a ksm page
* even while it is being migrated, so for that case we
* need migration_entry_wait().
*/
if (likely(!(flags & FOLL_MIGRATION)))
goto no_page;
if (pte_none(pte))
goto no_page;
entry = pte_to_swp_entry(pte);
if (!is_migration_entry(entry))
goto no_page;
pte_unmap_unlock(ptep, ptl);
migration_entry_wait(mm, pmd, address);
goto retry;
}
if ((flags & FOLL_NUMA) && pte_protnone(pte))
goto no_page;
if ((flags & FOLL_WRITE) && !can_follow_write_pte(pte, flags)) {
pte_unmap_unlock(ptep, ptl);
return NULL;
}
page = vm_normal_page(vma, address, pte);
if (!page && pte_devmap(pte) && (flags & FOLL_GET)) {
/*
* Only return device mapping pages in the FOLL_GET case since
* they are only valid while holding the pgmap reference.
*/
pgmap = get_dev_pagemap(pte_pfn(pte), NULL);
if (pgmap)
page = pte_page(pte);
else
goto no_page;
} else if (unlikely(!page)) {
if (flags & FOLL_DUMP) {
/* Avoid special (like zero) pages in core dumps */
page = ERR_PTR(-EFAULT);
goto out;
}
if (is_zero_pfn(pte_pfn(pte))) {
page = pte_page(pte);
} else {
int ret;
ret = follow_pfn_pte(vma, address, ptep, flags);
page = ERR_PTR(ret);
goto out;
}
}
if (flags & FOLL_SPLIT && PageTransCompound(page)) {
int ret;
get_page(page);
pte_unmap_unlock(ptep, ptl);
lock_page(page);
ret = split_huge_page(page);
unlock_page(page);
put_page(page);
if (ret)
return ERR_PTR(ret);
goto retry;
}
if (flags & FOLL_GET) {
get_page(page);
/* drop the pgmap reference now that we hold the page */
if (pgmap) {
put_dev_pagemap(pgmap);
pgmap = NULL;
}
}
if (flags & FOLL_TOUCH) {
if ((flags & FOLL_WRITE) &&
!pte_dirty(pte) && !PageDirty(page))
set_page_dirty(page);
/*
* pte_mkyoung() would be more correct here, but atomic care
* is needed to avoid losing the dirty bit: it is easier to use
* mark_page_accessed().
*/
mark_page_accessed(page);
}
if ((flags & FOLL_MLOCK) && (vma->vm_flags & VM_LOCKED)) {
/* Do not mlock pte-mapped THP */
if (PageTransCompound(page))
goto out;
/*
* The preliminary mapping check is mainly to avoid the
* pointless overhead of lock_page on the ZERO_PAGE
* which might bounce very badly if there is contention.
*
* If the page is already locked, we don't need to
* handle it now - vmscan will handle it later if and
* when it attempts to reclaim the page.
*/
if (page->mapping && trylock_page(page)) {
lru_add_drain(); /* push cached pages to LRU */
/*
* Because we lock page here, and migration is
* blocked by the pte's page reference, and we
* know the page is still mapped, we don't even
* need to check for file-cache page truncation.
*/
mlock_vma_page(page);
unlock_page(page);
}
}
out:
pte_unmap_unlock(ptep, ptl);
return page;
no_page:
pte_unmap_unlock(ptep, ptl);
if (!pte_none(pte))
return NULL;
return no_page_table(vma, flags);
}
static void fill_page_bit_map(struct mm_struct *mm,
unsigned long addr, unsigned long nr_pages,
unsigned long page_bit_map[PAGE_BITS])
{
int i = 0;
DEBUG("GIPC_SEND fill_page_bit_map %lx - %lx\n",
addr, addr + (nr_pages << PAGE_SHIFT));
do {
struct vm_area_struct *vma;
pgd_t *pgd;
pud_t *pud;
pmd_t *pmd;
pte_t *pte;
spinlock_t *ptl;
bool has_page = false;
vma = find_vma(mm, addr);
if (!vma)
goto next;
BUG_ON(vma->vm_flags & VM_HUGETLB);
pgd = pgd_offset(mm, addr);
if (pgd_none(*pgd) || pgd_bad(*pgd))
goto next;
pud = pud_offset(pgd, addr);
if (pud_none(*pud) || pud_bad(*pud))
goto next;
pmd = pmd_offset(pud, addr);
if (pmd_none(*pmd))
goto next;
if (unlikely(pmd_trans_huge(*pmd))) {
has_page = true;
goto next;
}
if (pmd_bad(*pmd))
goto next;
pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
if (pte_none(*pte))
goto next_locked;
if (unlikely(!pte_present(*pte)) && pte_file(*pte))
goto next_locked;
has_page = true;
next_locked:
spin_unlock(ptl);
next:
if (has_page) {
DEBUG("found a page at %lx\n", addr);
set_bit(i, page_bit_map);
} else {
clear_bit(i, page_bit_map);
}
} while (i++, addr += PAGE_SIZE, i < nr_pages);
}
static int mcopy_atomic_pte(struct mm_struct *dst_mm,
pmd_t *dst_pmd,
struct vm_area_struct *dst_vma,
unsigned long dst_addr,
unsigned long src_addr,
struct page **pagep)
{
struct mem_cgroup *memcg;
pte_t _dst_pte, *dst_pte;
spinlock_t *ptl;
void *page_kaddr;
int ret;
struct page *page;
if (!*pagep) {
ret = -ENOMEM;
page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, dst_vma, dst_addr);
if (!page)
goto out;
page_kaddr = kmap_atomic(page);
ret = copy_from_user(page_kaddr,
(const void __user *) src_addr,
PAGE_SIZE);
kunmap_atomic(page_kaddr);
/* fallback to copy_from_user outside mmap_sem */
if (unlikely(ret)) {
ret = -EFAULT;
*pagep = page;
/* don't free the page */
goto out;
}
} else {
page = *pagep;
*pagep = NULL;
}
/*
* The memory barrier inside __SetPageUptodate makes sure that
* preceeding stores to the page contents become visible before
* the set_pte_at() write.
*/
__SetPageUptodate(page);
ret = -ENOMEM;
if (mem_cgroup_try_charge(page, dst_mm, GFP_KERNEL, &memcg, false))
goto out_release;
_dst_pte = mk_pte(page, dst_vma->vm_page_prot);
if (dst_vma->vm_flags & VM_WRITE)
_dst_pte = pte_mkwrite(pte_mkdirty(_dst_pte));
ret = -EEXIST;
dst_pte = pte_offset_map_lock(dst_mm, dst_pmd, dst_addr, &ptl);
if (!pte_none(*dst_pte))
goto out_release_uncharge_unlock;
inc_mm_counter(dst_mm, MM_ANONPAGES);
page_add_new_anon_rmap(page, dst_vma, dst_addr, false);
mem_cgroup_commit_charge(page, memcg, false, false);
lru_cache_add_active_or_unevictable(page, dst_vma);
set_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte);
/* No need to invalidate - it was non-present before */
update_mmu_cache(dst_vma, dst_addr, dst_pte);
pte_unmap_unlock(dst_pte, ptl);
ret = 0;
out:
return ret;
out_release_uncharge_unlock:
pte_unmap_unlock(dst_pte, ptl);
mem_cgroup_cancel_charge(page, memcg, false);
out_release:
page_cache_release(page);
goto out;
}
static int
pin_page_for_write(const void __user *_addr, pte_t **ptep, spinlock_t **ptlp)
{
unsigned long addr = (unsigned long)_addr;
pgd_t *pgd;
pmd_t *pmd;
pte_t *pte;
pud_t *pud;
spinlock_t *ptl;
pgd = pgd_offset(current->mm, addr);
if (unlikely(pgd_none(*pgd) || pgd_bad(*pgd)))
return 0;
pud = pud_offset(pgd, addr);
if (unlikely(pud_none(*pud) || pud_bad(*pud)))
return 0;
pmd = pmd_offset(pud, addr);
if (unlikely(pmd_none(*pmd)))
return 0;
/*
* A pmd can be bad if it refers to a HugeTLB or THP page.
*
* Both THP and HugeTLB pages have the same pmd layout
* and should not be manipulated by the pte functions.
*
* Lock the page table for the destination and check
* to see that it's still huge and whether or not we will
* need to fault on write, or if we have a splitting THP.
*/
if (unlikely(pmd_thp_or_huge(*pmd))) {
ptl = ¤t->mm->page_table_lock;
spin_lock(ptl);
if (unlikely(!pmd_thp_or_huge(*pmd)
|| pmd_hugewillfault(*pmd)
|| pmd_trans_splitting(*pmd))) {
spin_unlock(ptl);
return 0;
}
*ptep = NULL;
*ptlp = ptl;
return 1;
}
if (unlikely(pmd_bad(*pmd)))
return 0;
pte = pte_offset_map_lock(current->mm, pmd, addr, &ptl);
if (unlikely(!pte_present(*pte) || !pte_young(*pte) ||
!pte_write(*pte) || !pte_dirty(*pte))) {
pte_unmap_unlock(pte, ptl);
return 0;
}
*ptep = pte;
*ptlp = ptl;
return 1;
}
static struct page *follow_page_pte(struct vm_area_struct *vma,
unsigned long address, pmd_t *pmd, unsigned int flags)
{
struct mm_struct *mm = vma->vm_mm;
struct page *page;
spinlock_t *ptl;
pte_t *ptep, pte;
retry:
if (unlikely(pmd_bad(*pmd)))
return no_page_table(vma, flags);
ptep = pte_offset_map_lock(mm, pmd, address, &ptl);
pte = *ptep;
if (!pte_present(pte)) {
swp_entry_t entry;
/*
* KSM's break_ksm() relies upon recognizing a ksm page
* even while it is being migrated, so for that case we
* need migration_entry_wait().
*/
if (likely(!(flags & FOLL_MIGRATION)))
goto no_page;
if (pte_none(pte) || pte_file(pte))
goto no_page;
entry = pte_to_swp_entry(pte);
if (!is_migration_entry(entry))
goto no_page;
pte_unmap_unlock(ptep, ptl);
migration_entry_wait(mm, pmd, address);
goto retry;
}
if ((flags & FOLL_NUMA) && pte_numa(pte))
goto no_page;
if ((flags & FOLL_WRITE) && !pte_write(pte)) {
pte_unmap_unlock(ptep, ptl);
return NULL;
}
page = vm_normal_page(vma, address, pte);
if (unlikely(!page)) {
if ((flags & FOLL_DUMP) ||
!is_zero_pfn(pte_pfn(pte)))
goto bad_page;
page = pte_page(pte);
}
if (flags & FOLL_GET)
get_page_foll(page);
if (flags & FOLL_TOUCH) {
if ((flags & FOLL_WRITE) &&
!pte_dirty(pte) && !PageDirty(page))
set_page_dirty(page);
/*
* pte_mkyoung() would be more correct here, but atomic care
* is needed to avoid losing the dirty bit: it is easier to use
* mark_page_accessed().
*/
mark_page_accessed(page);
}
if ((flags & FOLL_MLOCK) && (vma->vm_flags & VM_LOCKED)) {
/*
* The preliminary mapping check is mainly to avoid the
* pointless overhead of lock_page on the ZERO_PAGE
* which might bounce very badly if there is contention.
*
* If the page is already locked, we don't need to
* handle it now - vmscan will handle it later if and
* when it attempts to reclaim the page.
*/
if (page->mapping && trylock_page(page)) {
lru_add_drain(); /* push cached pages to LRU */
/*
* Because we lock page here, and migration is
* blocked by the pte's page reference, and we
* know the page is still mapped, we don't even
* need to check for file-cache page truncation.
*/
mlock_vma_page(page);
unlock_page(page);
}
}
pte_unmap_unlock(ptep, ptl);
return page;
bad_page:
pte_unmap_unlock(ptep, ptl);
return ERR_PTR(-EFAULT);
no_page:
pte_unmap_unlock(ptep, ptl);
if (!pte_none(pte))
return NULL;
return no_page_table(vma, flags);
}
