void __flush_anon_page(struct page *page, unsigned long vmaddr)
{
#ifdef CONFIG_RALINK_SOC
if (!PageHighMem(page)) {
unsigned long addr = (unsigned long) page_address(page);
if (pages_do_alias(addr, vmaddr & PAGE_MASK)) {
if (page_mapped(page) && !Page_dcache_dirty(page)) {
void *kaddr;
kaddr = kmap_coherent(page, vmaddr);
flush_data_cache_page((unsigned long)kaddr);
kunmap_coherent();
} else {
flush_data_cache_page(addr);
ClearPageDcacheDirty(page);
}
}
} else {
void *laddr = lowmem_page_address(page);
if (pages_do_alias((unsigned long)laddr, vmaddr & PAGE_MASK)) {
if (page_mapped(page) && !Page_dcache_dirty(page)) {
void *kaddr;
kaddr = kmap_coherent(page, vmaddr);
flush_data_cache_page((unsigned long)kaddr);
kunmap_coherent();
} else {
void *kaddr;
kaddr = kmap_atomic(page, KM_PTE1);
flush_data_cache_page((unsigned long)kaddr);
kunmap_atomic(kaddr, KM_PTE1);
ClearPageDcacheDirty(page);
}
}
}
#else
unsigned long addr = (unsigned long) page_address(page);
if (pages_do_alias(addr, vmaddr)) {
if (page_mapped(page) && !Page_dcache_dirty(page)) {
void *kaddr;
kaddr = kmap_coherent(page, vmaddr);
flush_data_cache_page((unsigned long)kaddr);
kunmap_coherent();
} else
flush_data_cache_page(addr);
}
#endif
}
/**
* invalidate_inode_pages2 - remove all unmapped pages from an address_space
* @mapping - the address_space
*
* invalidate_inode_pages2() is like truncate_inode_pages(), except for the case
* where the page is seen to be mapped into process pagetables. In that case,
* the page is marked clean but is left attached to its address_space.
*
* The page is also marked not uptodate so that a subsequent pagefault will
* perform I/O to bringthe page's contents back into sync with its backing
* store.
*
* FIXME: invalidate_inode_pages2() is probably trivially livelockable.
*/
void invalidate_inode_pages2(struct address_space *mapping)
{
struct pagevec pvec;
pgoff_t next = 0;
int i;
pagevec_init(&pvec, 0);
while (pagevec_lookup(&pvec, mapping, next, PAGEVEC_SIZE)) {
for (i = 0; i < pagevec_count(&pvec); i++) {
struct page *page = pvec.pages[i];
lock_page(page);
if (page->mapping == mapping) { /* truncate race? */
wait_on_page_writeback(page);
next = page->index + 1;
if (page_mapped(page)) {
clear_page_dirty(page);
ClearPageUptodate(page);
} else {
if (!invalidate_complete_page(mapping,
page)) {
clear_page_dirty(page);
ClearPageUptodate(page);
}
}
}
unlock_page(page);
}
pagevec_release(&pvec);
cond_resched();
}
}
/*
* Check that TLB entries with kernel ASID (1) have kernel VMA (>= TASK_SIZE),
* and TLB entries with user ASID (>=4) have VMA < TASK_SIZE.
*
* Check that valid TLB entries either have the same PA as the PTE, or PTE is
* marked as non-present. Non-present PTE and the page with non-zero refcount
* and zero mapcount is normal for batched TLB flush operation. Zero refcount
* means that the page was freed prematurely. Non-zero mapcount is unusual,
* but does not necessary means an error, thus marked as suspicious.
*/
static int check_tlb_entry(unsigned w, unsigned e, bool dtlb)
{
unsigned tlbidx = w | (e << PAGE_SHIFT);
unsigned r0 = dtlb ?
read_dtlb_virtual(tlbidx) : read_itlb_virtual(tlbidx);
unsigned vpn = (r0 & PAGE_MASK) | (e << PAGE_SHIFT);
unsigned pte = get_pte_for_vaddr(vpn);
unsigned mm_asid = (get_rasid_register() >> 8) & ASID_MASK;
unsigned tlb_asid = r0 & ASID_MASK;
bool kernel = tlb_asid == 1;
int rc = 0;
if (tlb_asid > 0 && ((vpn < TASK_SIZE) == kernel)) {
pr_err("%cTLB: way: %u, entry: %u, VPN %08x in %s PTE\n",
dtlb ? 'D' : 'I', w, e, vpn,
kernel ? "kernel" : "user");
rc |= TLB_INSANE;
}
if (tlb_asid == mm_asid) {
unsigned r1 = dtlb ? read_dtlb_translation(tlbidx) :
read_itlb_translation(tlbidx);
if ((pte ^ r1) & PAGE_MASK) {
pr_err("%cTLB: way: %u, entry: %u, mapping: %08x->%08x, PTE: %08x\n",
dtlb ? 'D' : 'I', w, e, r0, r1, pte);
if (pte == 0 || !pte_present(__pte(pte))) {
struct page *p = pfn_to_page(r1 >> PAGE_SHIFT);
pr_err("page refcount: %d, mapcount: %d\n",
page_count(p),
page_mapcount(p));
if (!page_count(p))
rc |= TLB_INSANE;
else if (page_mapped(p))
rc |= TLB_SUSPICIOUS;
} else {
void copy_user_highpage(struct page *to, struct page *from,
unsigned long vaddr, struct vm_area_struct *vma)
{
void *vfrom, *vto;
vto = kmap_atomic(to);
if (boot_cpu_data.dcache.n_aliases && page_mapped(from) &&
test_bit(PG_dcache_clean, &from->flags)) {
vfrom = kmap_coherent(from, vaddr);
copy_page(vto, vfrom);
kunmap_coherent(vfrom);
} else {
vfrom = kmap_atomic(from);
copy_page(vto, vfrom);
kunmap_atomic(vfrom);
}
if (pages_do_alias((unsigned long)vto, vaddr & PAGE_MASK) ||
(vma->vm_flags & VM_EXEC))
__flush_purge_region(vto, PAGE_SIZE);
kunmap_atomic(vto);
/* Make sure this page is cleared on other CPU's too before using it */
smp_wmb();
}
void __update_cache(struct vm_area_struct *vma, unsigned long address,
pte_t pte)
{
struct page *page;
unsigned long pfn, addr;
int exec = (vma->vm_flags & VM_EXEC) && !cpu_has_ic_fills_f_dc;
pfn = pte_pfn(pte);
if (unlikely(!pfn_valid(pfn))) {
wmb();
return;
}
page = pfn_to_page(pfn);
if (page_mapped(page) && Page_dcache_dirty(page)) {
void *kaddr = NULL;
if (PageHighMem(page)) {
addr = (unsigned long)kmap_atomic(page);
kaddr = (void *)addr;
} else
addr = (unsigned long) page_address(page);
if (exec || (cpu_has_dc_aliases &&
pages_do_alias(addr, address & PAGE_MASK))) {
flush_data_cache_page(addr);
ClearPageDcacheDirty(page);
}
if (kaddr)
kunmap_atomic((void *)kaddr);
}
wmb(); /* finish any outstanding arch cache flushes before ret to user */
}
/**
* free_swap_cache:page从交换区高速缓存中删除
*/
static inline void free_swap_cache(struct page *page)
{
if (PageSwapCache(page) && !page_mapped(page) && trylock_page(page)) {
try_to_free_swap(page);
unlock_page(page);
}
}
int fetch_page(void *addr) {
struct filemap *fm;
int rc;
fm = (struct filemap *) olock(virt2pfn(addr), OBJECT_FILEMAP);
if (!fm) return -EBADF;
rc = wait_for_object(fm, INFINITE);
if (rc < 0) {
orel(fm);
return rc;
}
if (!page_mapped(addr)) {
rc = fetch_file_page(fm, addr);
if (rc < 0) {
unlock_filemap(fm);
orel(fm);
return rc;
}
}
rc = unlock_filemap(fm);
if (rc < 0) return rc;
orel(fm);
return 0;
}
/*
* Handle cache congruency of kernel and userspace mappings of page when kernel
* writes-to/reads-from
*
* The idea is to defer flushing of kernel mapping after a WRITE, possible if:
* -dcache is NOT aliasing, hence any U/K-mappings of page are congruent
* -U-mapping doesn't exist yet for page (finalised in update_mmu_cache)
* -In SMP, if hardware caches are coherent
*
* There's a corollary case, where kernel READs from a userspace mapped page.
* If the U-mapping is not congruent to to K-mapping, former needs flushing.
*/
void flush_dcache_page(struct page *page)
{
struct address_space *mapping;
if (!cache_is_vipt_aliasing()) {
clear_bit(PG_dc_clean, &page->flags);
return;
}
/* don't handle anon pages here */
mapping = page_mapping(page);
if (!mapping)
return;
/*
* pagecache page, file not yet mapped to userspace
* Make a note that K-mapping is dirty
*/
if (!mapping_mapped(mapping)) {
clear_bit(PG_dc_clean, &page->flags);
} else if (page_mapped(page)) {
/* kernel reading from page with U-mapping */
void *paddr = page_address(page);
unsigned long vaddr = page->index << PAGE_CACHE_SHIFT;
if (addr_not_cache_congruent(paddr, vaddr))
__flush_dcache_page(paddr, vaddr);
}
}
int truncate_inode_page(struct address_space *mapping, struct page *page)
{
if (page_mapped(page)) {
unmap_mapping_range(mapping,
(loff_t)page->index << PAGE_CACHE_SHIFT,
PAGE_CACHE_SIZE, 0);
}
return truncate_complete_page(mapping, page);
}
/**
* __replace_page - replace page in vma by new page.
* based on replace_page in mm/ksm.c
*
* @vma: vma that holds the pte pointing to page
* @addr: address the old @page is mapped at
* @page: the cowed page we are replacing by kpage
* @kpage: the modified page we replace page by
*
* Returns 0 on success, -EFAULT on failure.
*/
static int __replace_page(struct vm_area_struct *vma, unsigned long addr,
struct page *old_page, struct page *new_page)
{
struct mm_struct *mm = vma->vm_mm;
spinlock_t *ptl;
pte_t *ptep;
int err;
/* For mmu_notifiers */
const unsigned long mmun_start = addr;
const unsigned long mmun_end = addr + PAGE_SIZE;
struct mem_cgroup *memcg;
err = mem_cgroup_try_charge(new_page, vma->vm_mm, GFP_KERNEL, &memcg,
false);
if (err)
return err;
/* For try_to_free_swap() and munlock_vma_page() below */
lock_page(old_page);
mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
err = -EAGAIN;
ptep = page_check_address(old_page, mm, addr, &ptl, 0);
if (!ptep) {
mem_cgroup_cancel_charge(new_page, memcg, false);
goto unlock;
}
get_page(new_page);
page_add_new_anon_rmap(new_page, vma, addr, false);
mem_cgroup_commit_charge(new_page, memcg, false, false);
lru_cache_add_active_or_unevictable(new_page, vma);
if (!PageAnon(old_page)) {
dec_mm_counter(mm, mm_counter_file(old_page));
inc_mm_counter(mm, MM_ANONPAGES);
}
flush_cache_page(vma, addr, pte_pfn(*ptep));
ptep_clear_flush_notify(vma, addr, ptep);
set_pte_at_notify(mm, addr, ptep, mk_pte(new_page, vma->vm_page_prot));
page_remove_rmap(old_page, false);
if (!page_mapped(old_page))
try_to_free_swap(old_page);
pte_unmap_unlock(ptep, ptl);
if (vma->vm_flags & VM_LOCKED)
munlock_vma_page(old_page);
put_page(old_page);
err = 0;
unlock:
mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
unlock_page(old_page);
return err;
}
/*
* Safely invalidate one page from its pagecache mapping.
* It only drops clean, unused pages. The page must be locked.
*
* Returns 1 if the page is successfully invalidated, otherwise 0.
*/
int invalidate_inode_page(struct page *page)
{
struct address_space *mapping = page_mapping(page);
if (!mapping)
return 0;
if (PageDirty(page) || PageWriteback(page))
return 0;
if (page_mapped(page))
return 0;
return invalidate_complete_page(mapping, page);
}
static int tux3_set_page_dirty_bug(struct page *page)
{
/* See comment of tux3_set_page_dirty() */
ClearPageReclaim(page);
assert(0);
/* This page should not be mmapped */
assert(!page_mapped(page));
/* This page should be dirty already, otherwise we will lost data. */
assert(PageDirty(page));
return 0;
}
int truncate_inode_page(struct address_space *mapping, struct page *page)
{
loff_t holelen;
VM_BUG_ON_PAGE(PageTail(page), page);
holelen = PageTransHuge(page) ? HPAGE_PMD_SIZE : PAGE_SIZE;
if (page_mapped(page)) {
unmap_mapping_range(mapping,
(loff_t)page->index << PAGE_SHIFT,
holelen, 0);
}
return truncate_complete_page(mapping, page);
}
void copy_from_user_page(struct vm_area_struct *vma, struct page *page,
unsigned long vaddr, void *dst, const void *src,
unsigned long len)
{
if (boot_cpu_data.dcache.n_aliases && page_mapped(page) &&
test_bit(PG_dcache_clean, &page->flags)) {
void *vfrom = kmap_coherent(page, vaddr) + (vaddr & ~PAGE_MASK);
memcpy(dst, vfrom, len);
kunmap_coherent(vfrom);
} else {
memcpy(dst, src, len);
if (boot_cpu_data.dcache.n_aliases)
clear_bit(PG_dcache_clean, &page->flags);
}
}
void __flush_anon_page(struct page *page, unsigned long vmaddr)
{
unsigned long addr = (unsigned long) page_address(page);
if (pages_do_alias(addr, vmaddr)) {
if (page_mapped(page) && !Page_dcache_dirty(page)) {
void *kaddr;
kaddr = kmap_coherent(page, vmaddr);
flush_data_cache_page((unsigned long)kaddr);
kunmap_coherent();
} else
flush_data_cache_page(addr);
}
}
/*
* If the page can not be invalidated, it is moved to the
* inactive list to speed up its reclaim. It is moved to the
* head of the list, rather than the tail, to give the flusher
* threads some time to write it out, as this is much more
* effective than the single-page writeout from reclaim.
*
* If the page isn't page_mapped and dirty/writeback, the page
* could reclaim asap using PG_reclaim.
*
* 1. active, mapped page -> none
* 2. active, dirty/writeback page -> inactive, head, PG_reclaim
* 3. inactive, mapped page -> none
* 4. inactive, dirty/writeback page -> inactive, head, PG_reclaim
* 5. inactive, clean -> inactive, tail
* 6. Others -> none
*
* In 4, why it moves inactive's head, the VM expects the page would
* be write it out by flusher threads as this is much more effective
* than the single-page writeout from reclaim.
*/
static void lru_deactivate_fn(struct page *page, void *arg)
{
int lru, file;
bool active;
struct zone *zone = page_zone(page);
if (!PageLRU(page))
return;
if (PageUnevictable(page))
return;
/* Some processes are using the page */
if (page_mapped(page))
return;
active = PageActive(page);
file = page_is_file_cache(page);
lru = page_lru_base_type(page);
del_page_from_lru_list(zone, page, lru + active);
ClearPageActive(page);
ClearPageReferenced(page);
add_page_to_lru_list(zone, page, lru);
if (PageWriteback(page) || PageDirty(page)) {
/*
* PG_reclaim could be raced with end_page_writeback
* It can make readahead confusing. But race window
* is _really_ small and it's non-critical problem.
*/
SetPageReclaim(page);
} else {
struct lruvec *lruvec;
/*
* The page's writeback ends up during pagevec
* We moves tha page into tail of inactive.
*/
lruvec = mem_cgroup_lru_move_lists(zone, page, lru, lru);
list_move_tail(&page->lru, &lruvec->lists[lru]);
__count_vm_event(PGROTATED);
}
if (active)
__count_vm_event(PGDEACTIVATE);
update_page_reclaim_stat(zone, page, file, 0);
}
void __flush_anon_page(struct page *page, unsigned long vmaddr)
{
unsigned long addr = (unsigned long) page_address(page);
if (pages_do_alias(addr, vmaddr)) {
if (boot_cpu_data.dcache.n_aliases && page_mapped(page) &&
test_bit(PG_dcache_clean, &page->flags)) {
void *kaddr;
kaddr = kmap_coherent(page, vmaddr);
/* XXX.. For now kunmap_coherent() does a purge */
/* __flush_purge_region((void *)kaddr, PAGE_SIZE); */
kunmap_coherent(kaddr);
} else
__flush_purge_region((void *)addr, PAGE_SIZE);
}
}
void copy_to_user_page(struct vm_area_struct *vma, struct page *page,
unsigned long vaddr, void *dst, const void *src,
unsigned long len)
{
if (boot_cpu_data.dcache.n_aliases && page_mapped(page) &&
test_bit(PG_dcache_clean, &page->flags)) {
void *vto = kmap_coherent(page, vaddr) + (vaddr & ~PAGE_MASK);
memcpy(vto, src, len);
kunmap_coherent(vto);
} else {
memcpy(dst, src, len);
if (boot_cpu_data.dcache.n_aliases)
clear_bit(PG_dcache_clean, &page->flags);
}
if (vma->vm_flags & VM_EXEC)
flush_cache_page(vma, vaddr, page_to_pfn(page));
}
void copy_user_highpage(struct page *to, struct page *from,
unsigned long u_vaddr, struct vm_area_struct *vma)
{
void *kfrom = page_address(from);
void *kto = page_address(to);
int clean_src_k_mappings = 0;
/*
* If SRC page was already mapped in userspace AND it's U-mapping is
* not congruent with K-mapping, sync former to physical page so that
* K-mapping in memcpy below, sees the right data
*
* Note that while @u_vaddr refers to DST page's userspace vaddr, it is
* equally valid for SRC page as well
*/
if (page_mapped(from) && addr_not_cache_congruent(kfrom, u_vaddr)) {
__flush_dcache_page(kfrom, u_vaddr);
clean_src_k_mappings = 1;
}
copy_page(kto, kfrom);
/*
* Mark DST page K-mapping as dirty for a later finalization by
* update_mmu_cache(). Although the finalization could have been done
* here as well (given that both vaddr/paddr are available).
* But update_mmu_cache() already has code to do that for other
* non copied user pages (e.g. read faults which wire in pagecache page
* directly).
*/
clear_bit(PG_dc_clean, &to->flags);
/*
* if SRC was already usermapped and non-congruent to kernel mapping
* sync the kernel mapping back to physical page
*/
if (clean_src_k_mappings) {
__flush_dcache_page(kfrom, kfrom);
set_bit(PG_dc_clean, &from->flags);
} else {
clear_bit(PG_dc_clean, &from->flags);
}
}
void __flush_dcache_page(struct page *page)
{
#ifdef CONFIG_RALINK_SOC
void *addr;
if (page_mapping(page) && !page_mapped(page)) {
SetPageDcacheDirty(page);
return;
}
#else
struct address_space *mapping = page_mapping(page);
unsigned long addr;
if (PageHighMem(page))
return;
if (mapping && !mapping_mapped(mapping)) {
SetPageDcacheDirty(page);
return;
}
#endif
/*
* We could delay the flush for the !page_mapping case too. But that
* case is for exec env/arg pages and those are %99 certainly going to
* get faulted into the tlb (and thus flushed) anyways.
*/
#ifdef CONFIG_RALINK_SOC
if (PageHighMem(page)) {
addr = kmap_atomic(page, KM_PTE1);
flush_data_cache_page((unsigned long)addr);
kunmap_atomic(addr, KM_PTE1);
} else {
addr = (void *) page_address(page);
flush_data_cache_page((unsigned long)addr);
}
ClearPageDcacheDirty(page);
#else
addr = (unsigned long) page_address(page);
flush_data_cache_page(addr);
#endif
}
static void drm_ttm_free_alloced_pages(struct drm_ttm *ttm)
{
int i;
struct drm_buffer_manager *bm = &ttm->dev->bm;
struct page **cur_page;
for (i = 0; i < ttm->num_pages; ++i) {
cur_page = ttm->pages + i;
if (*cur_page) {
#if (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,15))
ClearPageReserved(*cur_page);
#endif
if (page_count(*cur_page) != 1)
DRM_ERROR("Erroneous page count. Leaking pages.\n");
if (page_mapped(*cur_page))
DRM_ERROR("Erroneous map count. Leaking page mappings.\n");
__free_page(*cur_page);
--bm->cur_pages;
}
}
}
/*
* Delete a page from the page cache and free it. Caller has to make
* sure the page is locked and that nobody else uses it - or that usage
* is safe. The caller must hold the mapping's tree_lock and
* mem_cgroup_begin_page_stat().
*/
void __delete_from_page_cache(struct page *page, void *shadow,
struct mem_cgroup *memcg)
{
struct address_space *mapping = page->mapping;
trace_mm_filemap_delete_from_page_cache(page);
/*
* if we're uptodate, flush out into the cleancache, otherwise
* invalidate any existing cleancache entries. We can't leave
* stale data around in the cleancache once our page is gone
*/
if (PageUptodate(page) && PageMappedToDisk(page))
cleancache_put_page(page);
else
cleancache_invalidate_page(mapping, page);
page_cache_tree_delete(mapping, page, shadow);
page->mapping = NULL;
/* Leave page->index set: truncation lookup relies upon it */
/* hugetlb pages do not participate in page cache accounting. */
if (!PageHuge(page))
__dec_zone_page_state(page, NR_FILE_PAGES);
if (PageSwapBacked(page))
__dec_zone_page_state(page, NR_SHMEM);
BUG_ON(page_mapped(page));
/*
* At this point page must be either written or cleaned by truncate.
* Dirty page here signals a bug and loss of unwritten data.
*
* This fixes dirty accounting after removing the page entirely but
* leaves PageDirty set: it has no effect for truncated page and
* anyway will be cleared before returning page into buddy allocator.
*/
if (WARN_ON_ONCE(PageDirty(page)))
account_page_cleaned(page, mapping, memcg,
inode_to_wb(mapping->host));
}
int vmprotect(void *addr, unsigned long size, int protect) {
int pages = PAGES(size);
int i;
char *vaddr;
unsigned long flags;
if (size == 0) return 0;
addr = (void *) PAGEADDR(addr);
if (!valid_range(addr, size)) return -EINVAL;
flags = pte_flags_from_protect(protect);
if (flags == 0xFFFFFFFF) return -EINVAL;
vaddr = (char *) addr;
for (i = 0; i < pages; i++) {
if (page_mapped(vaddr)) {
set_page_flags(vaddr, (get_page_flags(vaddr) & ~PT_PROTECTMASK) | flags);
}
vaddr += PAGESIZE;
}
return 0;
}
void __delete_from_page_cache(struct page *page)
{
struct address_space *mapping = page->mapping;
if (PageUptodate(page) && PageMappedToDisk(page))
cleancache_put_page(page);
else
cleancache_invalidate_page(mapping, page);
radix_tree_delete(&mapping->page_tree, page->index);
page->mapping = NULL;
mapping->nrpages--;
__dec_zone_page_state(page, NR_FILE_PAGES);
if (PageSwapBacked(page))
__dec_zone_page_state(page, NR_SHMEM);
BUG_ON(page_mapped(page));
if (PageDirty(page) && mapping_cap_account_dirty(mapping)) {
dec_zone_page_state(page, NR_FILE_DIRTY);
dec_bdi_stat(mapping->backing_dev_info, BDI_RECLAIMABLE);
}
}
/**
* invalidate_mapping_pages - Invalidate all the unlocked pages of one inode
* @mapping: the address_space which holds the pages to invalidate
* @start: the offset 'from' which to invalidate
* @end: the offset 'to' which to invalidate (inclusive)
*
* This function only removes the unlocked pages, if you want to
* remove all the pages of one inode, you must call truncate_inode_pages.
*
* invalidate_mapping_pages() will not block on IO activity. It will not
* invalidate pages which are dirty, locked, under writeback or mapped into
* pagetables.
*/
unsigned long invalidate_mapping_pages(struct address_space *mapping,
pgoff_t start, pgoff_t end)
{
struct pagevec pvec;
pgoff_t next = start;
unsigned long ret = 0;
int i;
pagevec_init(&pvec, 0);
while (next <= end &&
pagevec_lookup(&pvec, mapping, next, PAGEVEC_SIZE)) {
for (i = 0; i < pagevec_count(&pvec); i++) {
struct page *page = pvec.pages[i];
if (TestSetPageLocked(page)) {
next++;
continue;
}
if (page->index > next)
next = page->index;
next++;
if (PageDirty(page) || PageWriteback(page))
goto unlock;
if (page_mapped(page))
goto unlock;
ret += invalidate_complete_page(mapping, page);
unlock:
unlock_page(page);
if (next > end)
break;
}
pagevec_release(&pvec);
cond_resched();
}
return ret;
}
void __flush_dcache_page(struct page *page)
{
void *addr;
if (page_mapping(page) && !page_mapped(page)) {
SetPageDcacheDirty(page);
return;
}
/*
* We could delay the flush for the !page_mapping case too. But that
* case is for exec env/arg pages and those are %99 certainly going to
* get faulted into the tlb (and thus flushed) anyways.
*/
if (PageHighMem(page)) {
addr = kmap_atomic(page);
flush_data_cache_page((unsigned long)addr);
kunmap_atomic(addr);
} else {
addr = (void *) page_address(page);
flush_data_cache_page((unsigned long)addr);
}
ClearPageDcacheDirty(page);
}
/**
* invalidate_inode_pages2_range - remove range of pages from an address_space
* @mapping: the address_space
* @start: the page offset 'from' which to invalidate
* @end: the page offset 'to' which to invalidate (inclusive)
*
* Any pages which are found to be mapped into pagetables are unmapped prior to
* invalidation.
*
* Returns -EBUSY if any pages could not be invalidated.
*/
int invalidate_inode_pages2_range(struct address_space *mapping,
pgoff_t start, pgoff_t end)
{
pgoff_t indices[PAGEVEC_SIZE];
struct pagevec pvec;
pgoff_t index;
int i;
int ret = 0;
int ret2 = 0;
int did_range_unmap = 0;
cleancache_invalidate_inode(mapping);
pagevec_init(&pvec, 0);
index = start;
while (index <= end && pagevec_lookup_entries(&pvec, mapping, index,
min(end - index, (pgoff_t)PAGEVEC_SIZE - 1) + 1,
indices)) {
for (i = 0; i < pagevec_count(&pvec); i++) {
struct page *page = pvec.pages[i];
/* We rely upon deletion not changing page->index */
index = indices[i];
if (index > end)
break;
if (radix_tree_exceptional_entry(page)) {
clear_exceptional_entry(mapping, index, page);
continue;
}
lock_page(page);
WARN_ON(page->index != index);
if (page->mapping != mapping) {
unlock_page(page);
continue;
}
wait_on_page_writeback(page);
if (page_mapped(page)) {
if (!did_range_unmap) {
/*
* Zap the rest of the file in one hit.
*/
unmap_mapping_range(mapping,
(loff_t)index << PAGE_CACHE_SHIFT,
(loff_t)(1 + end - index)
<< PAGE_CACHE_SHIFT,
0);
did_range_unmap = 1;
} else {
/*
* Just zap this page
*/
unmap_mapping_range(mapping,
(loff_t)index << PAGE_CACHE_SHIFT,
PAGE_CACHE_SIZE, 0);
}
}
BUG_ON(page_mapped(page));
ret2 = do_launder_page(mapping, page);
if (ret2 == 0) {
if (!invalidate_complete_page2(mapping, page))
ret2 = -EBUSY;
}
if (ret2 < 0)
ret = ret2;
unlock_page(page);
}
pagevec_remove_exceptionals(&pvec);
pagevec_release(&pvec);
cond_resched();
index++;
}
cleancache_invalidate_inode(mapping);
return ret;
}
void *vmalloc(void *addr, unsigned long size, int type, int protect, unsigned long tag, int *rc) {
int pages = PAGES(size);
unsigned long flags = pte_flags_from_protect(protect);
int i;
if (rc) *rc = 0;
if (size == 0) {
if (rc) *rc = -EINVAL;
return NULL;
}
if ((type & MEM_COMMIT) != 0 && flags == 0xFFFFFFFF) {
if (rc) *rc = -EINVAL;
return NULL;
}
addr = (void *) PAGEADDR(addr);
if (!addr && (type & MEM_COMMIT) != 0) type |= MEM_RESERVE;
if (!tag) tag = 'VM';
if (type & MEM_RESERVE) {
if (addr == NULL) {
if (type & MEM_ALIGN64K) {
addr = (void *) PTOB(rmap_alloc_align(vmap, pages, 64 * 1024 / PAGESIZE));
} else {
addr = (void *) PTOB(rmap_alloc(vmap, pages));
}
if (addr == NULL) {
if (rc) *rc = -ENOMEM;
return NULL;
}
} else {
if (rmap_reserve(vmap, BTOP(addr), pages)) {
if (rc) *rc = -ENOMEM;
return NULL;
}
}
} else {
if (!valid_range(addr, size)) {
if (rc) *rc = -EFAULT;
return NULL;
}
}
if (type & MEM_COMMIT) {
char *vaddr;
unsigned long pfn;
vaddr = (char *) addr;
for (i = 0; i < pages; i++) {
if (page_mapped(vaddr)) {
set_page_flags(vaddr, flags | PT_PRESENT);
} else {
pfn = alloc_pageframe(tag);
if (pfn == 0xFFFFFFFF) {
if (rc) *rc = -ENOMEM;
return NULL;
}
map_page(vaddr, pfn, flags | PT_PRESENT);
memset(vaddr, 0, PAGESIZE);
}
vaddr += PAGESIZE;
}
}
return addr;
}
u64 stable_page_flags(struct page *page)
{
u64 k;
u64 u;
/*
* pseudo flag: KPF_NOPAGE
* it differentiates a memory hole from a page with no flags
*/
if (!page)
return 1 << KPF_NOPAGE;
k = page->flags;
u = 0;
/*
* pseudo flags for the well known (anonymous) memory mapped pages
*
* Note that page->_mapcount is overloaded in SLOB/SLUB/SLQB, so the
* simple test in page_mapped() is not enough.
*/
if (!PageSlab(page) && page_mapped(page))
u |= 1 << KPF_MMAP;
if (PageAnon(page))
u |= 1 << KPF_ANON;
if (PageKsm(page))
u |= 1 << KPF_KSM;
/*
* compound pages: export both head/tail info
* they together define a compound page's start/end pos and order
*/
if (PageHead(page))
u |= 1 << KPF_COMPOUND_HEAD;
if (PageTail(page))
u |= 1 << KPF_COMPOUND_TAIL;
if (PageHuge(page))
u |= 1 << KPF_HUGE;
else if (PageTransCompound(page))
u |= 1 << KPF_THP;
/*
* Caveats on high order pages: page->_count will only be set
* -1 on the head page; SLUB/SLQB do the same for PG_slab;
* SLOB won't set PG_slab at all on compound pages.
*/
if (PageBuddy(page))
u |= 1 << KPF_BUDDY;
u |= kpf_copy_bit(k, KPF_LOCKED, PG_locked);
u |= kpf_copy_bit(k, KPF_SLAB, PG_slab);
u |= kpf_copy_bit(k, KPF_ERROR, PG_error);
u |= kpf_copy_bit(k, KPF_DIRTY, PG_dirty);
u |= kpf_copy_bit(k, KPF_UPTODATE, PG_uptodate);
u |= kpf_copy_bit(k, KPF_WRITEBACK, PG_writeback);
u |= kpf_copy_bit(k, KPF_LRU, PG_lru);
u |= kpf_copy_bit(k, KPF_REFERENCED, PG_referenced);
u |= kpf_copy_bit(k, KPF_ACTIVE, PG_active);
u |= kpf_copy_bit(k, KPF_RECLAIM, PG_reclaim);
u |= kpf_copy_bit(k, KPF_SWAPCACHE, PG_swapcache);
u |= kpf_copy_bit(k, KPF_SWAPBACKED, PG_swapbacked);
u |= kpf_copy_bit(k, KPF_UNEVICTABLE, PG_unevictable);
u |= kpf_copy_bit(k, KPF_MLOCKED, PG_mlocked);
#ifdef CONFIG_MEMORY_FAILURE
u |= kpf_copy_bit(k, KPF_HWPOISON, PG_hwpoison);
#endif
#ifdef CONFIG_ARCH_USES_PG_UNCACHED
u |= kpf_copy_bit(k, KPF_UNCACHED, PG_uncached);
#endif
u |= kpf_copy_bit(k, KPF_RESERVED, PG_reserved);
u |= kpf_copy_bit(k, KPF_MAPPEDTODISK, PG_mappedtodisk);
u |= kpf_copy_bit(k, KPF_PRIVATE, PG_private);
u |= kpf_copy_bit(k, KPF_PRIVATE_2, PG_private_2);
u |= kpf_copy_bit(k, KPF_OWNER_PRIVATE, PG_owner_priv_1);
u |= kpf_copy_bit(k, KPF_ARCH, PG_arch_1);
return u;
};
/*
* Must not be called with IRQs off. This should only be used on the
* slow path.
*
* Copy a foreign granted page to local memory.
*/
int gnttab_copy_grant_page(grant_ref_t ref, struct page **pagep)
{
struct gnttab_unmap_and_replace unmap;
mmu_update_t mmu;
struct page *page;
struct page *new_page;
void *new_addr;
void *addr;
paddr_t pfn;
maddr_t mfn;
maddr_t new_mfn;
int err;
page = *pagep;
if (!get_page_unless_zero(page))
return -ENOENT;
err = -ENOMEM;
new_page = alloc_page(GFP_ATOMIC | __GFP_NOWARN);
if (!new_page)
goto out;
new_addr = page_address(new_page);
addr = page_address(page);
copy_page(new_addr, addr);
pfn = page_to_pfn(page);
mfn = pfn_to_mfn(pfn);
new_mfn = virt_to_mfn(new_addr);
write_seqlock_bh(&gnttab_dma_lock);
/* Make seq visible before checking page_mapped. */
smp_mb();
/* Has the page been DMA-mapped? */
if (unlikely(page_mapped(page))) {
write_sequnlock_bh(&gnttab_dma_lock);
put_page(new_page);
err = -EBUSY;
goto out;
}
if (!xen_feature(XENFEAT_auto_translated_physmap))
set_phys_to_machine(pfn, new_mfn);
gnttab_set_replace_op(&unmap, (unsigned long)addr,
(unsigned long)new_addr, ref);
err = HYPERVISOR_grant_table_op(GNTTABOP_unmap_and_replace,
&unmap, 1);
BUG_ON(err);
BUG_ON(unmap.status != GNTST_okay);
write_sequnlock_bh(&gnttab_dma_lock);
if (!xen_feature(XENFEAT_auto_translated_physmap)) {
set_phys_to_machine(page_to_pfn(new_page), INVALID_P2M_ENTRY);
mmu.ptr = (new_mfn << PAGE_SHIFT) | MMU_MACHPHYS_UPDATE;
mmu.val = pfn;
err = HYPERVISOR_mmu_update(&mmu, 1, NULL, DOMID_SELF);
BUG_ON(err);
}
new_page->mapping = page->mapping;
new_page->index = page->index;
set_bit(PG_foreign, &new_page->flags);
if (PageReserved(page))
SetPageReserved(new_page);
*pagep = new_page;
SetPageForeign(page, gnttab_page_free);
page->mapping = NULL;
out:
put_page(page);
return err;
}
