pte_t swp_entry_to_pte (swp_entry_t entry)
{
swp_entry_t arch_entry;
arch_entry = (swp_entry_t){mk_swap_pte (swp_offset (entry)).pte};
__BUG_ON ((unsigned long) pte_file ((pte_t) {arch_entry.val}));
return (pte_t) {arch_entry.val};
}
static void mincore_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
unsigned long addr, unsigned long end,
unsigned char *vec)
{
unsigned long next;
spinlock_t *ptl;
pte_t *ptep;
ptep = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
do {
pte_t pte = *ptep;
pgoff_t pgoff;
next = addr + PAGE_SIZE;
if (pte_none(pte))
mincore_unmapped_range(vma, addr, next, vec);
else if (pte_present(pte))
*vec = 1;
else if (pte_file(pte)) {
pgoff = pte_to_pgoff(pte);
*vec = mincore_page(vma->vm_file->f_mapping, pgoff);
} else { /* pte is a swap entry */
swp_entry_t entry = pte_to_swp_entry(pte);
if (is_migration_entry(entry)) {
/* migration entries are always uptodate */
*vec = 1;
} else {
#ifdef CONFIG_SWAP
pgoff = entry.val;
*vec = mincore_page(&swapper_space, pgoff);
#else
WARN_ON(1);
*vec = 1;
#endif
}
}
vec++;
} while (ptep++, addr = next, addr != end);
pte_unmap_unlock(ptep - 1, ptl);
}
/* Update the home of a PTE if necessary (can also be used for a pgprot_t). */
pte_t pte_set_home(pte_t pte, int home)
{
/* Check for non-linear file mapping "PTEs" and pass them through. */
if (pte_file(pte))
return pte;
#if CHIP_HAS_MMIO()
/* Check for MMIO mappings and pass them through. */
if (hv_pte_get_mode(pte) == HV_PTE_MODE_MMIO)
return pte;
#endif
/*
* Only immutable pages get NC mappings. If we have a
* non-coherent PTE, but the underlying page is not
* immutable, it's likely the result of a forced
* caching setting running up against ptrace setting
* the page to be writable underneath. In this case,
* just keep the PTE coherent.
*/
if (hv_pte_get_nc(pte) && home != PAGE_HOME_IMMUTABLE) {
pte = hv_pte_clear_nc(pte);
pr_err("non-immutable page incoherently referenced: %#llx\n",
pte.val);
}
switch (home) {
case PAGE_HOME_UNCACHED:
pte = hv_pte_set_mode(pte, HV_PTE_MODE_UNCACHED);
break;
case PAGE_HOME_INCOHERENT:
pte = hv_pte_set_mode(pte, HV_PTE_MODE_CACHE_NO_L3);
break;
case PAGE_HOME_IMMUTABLE:
/*
* We could home this page anywhere, since it's immutable,
* but by default just home it to follow "hash_default".
*/
BUG_ON(hv_pte_get_writable(pte));
if (pte_get_forcecache(pte)) {
/* Upgrade "force any cpu" to "No L3" for immutable. */
if (hv_pte_get_mode(pte) == HV_PTE_MODE_CACHE_TILE_L3
&& pte_get_anyhome(pte)) {
pte = hv_pte_set_mode(pte,
HV_PTE_MODE_CACHE_NO_L3);
}
} else
#if CHIP_HAS_CBOX_HOME_MAP()
if (hash_default)
pte = hv_pte_set_mode(pte, HV_PTE_MODE_CACHE_HASH_L3);
else
#endif
pte = hv_pte_set_mode(pte, HV_PTE_MODE_CACHE_NO_L3);
pte = hv_pte_set_nc(pte);
break;
#if CHIP_HAS_CBOX_HOME_MAP()
case PAGE_HOME_HASH:
pte = hv_pte_set_mode(pte, HV_PTE_MODE_CACHE_HASH_L3);
break;
#endif
default:
BUG_ON(home < 0 || home >= NR_CPUS ||
!cpu_is_valid_lotar(home));
pte = hv_pte_set_mode(pte, HV_PTE_MODE_CACHE_TILE_L3);
pte = set_remote_cache_cpu(pte, home);
break;
}
#if CHIP_HAS_NC_AND_NOALLOC_BITS()
if (noallocl2)
pte = hv_pte_set_no_alloc_l2(pte);
/* Simplify "no local and no l3" to "uncached" */
if (hv_pte_get_no_alloc_l2(pte) && hv_pte_get_no_alloc_l1(pte) &&
hv_pte_get_mode(pte) == HV_PTE_MODE_CACHE_NO_L3) {
pte = hv_pte_set_mode(pte, HV_PTE_MODE_UNCACHED);
}
#endif
/* Checking this case here gives a better panic than from the hv. */
BUG_ON(hv_pte_get_mode(pte) == 0);
return pte;
}
static struct page *follow_page_pte(struct vm_area_struct *vma,
unsigned long address, pmd_t *pmd, unsigned int flags)
{
struct mm_struct *mm = vma->vm_mm;
struct page *page;
spinlock_t *ptl;
pte_t *ptep, pte;
retry:
if (unlikely(pmd_bad(*pmd)))
return no_page_table(vma, flags);
ptep = pte_offset_map_lock(mm, pmd, address, &ptl);
pte = *ptep;
if (!pte_present(pte)) {
swp_entry_t entry;
/*
* KSM's break_ksm() relies upon recognizing a ksm page
* even while it is being migrated, so for that case we
* need migration_entry_wait().
*/
if (likely(!(flags & FOLL_MIGRATION)))
goto no_page;
if (pte_none(pte) || pte_file(pte))
goto no_page;
entry = pte_to_swp_entry(pte);
if (!is_migration_entry(entry))
goto no_page;
pte_unmap_unlock(ptep, ptl);
migration_entry_wait(mm, pmd, address);
goto retry;
}
if ((flags & FOLL_NUMA) && pte_numa(pte))
goto no_page;
if ((flags & FOLL_WRITE) && !pte_write(pte)) {
pte_unmap_unlock(ptep, ptl);
return NULL;
}
page = vm_normal_page(vma, address, pte);
if (unlikely(!page)) {
if ((flags & FOLL_DUMP) ||
!is_zero_pfn(pte_pfn(pte)))
goto bad_page;
page = pte_page(pte);
}
if (flags & FOLL_GET)
get_page_foll(page);
if (flags & FOLL_TOUCH) {
if ((flags & FOLL_WRITE) &&
!pte_dirty(pte) && !PageDirty(page))
set_page_dirty(page);
/*
* pte_mkyoung() would be more correct here, but atomic care
* is needed to avoid losing the dirty bit: it is easier to use
* mark_page_accessed().
*/
mark_page_accessed(page);
}
if ((flags & FOLL_MLOCK) && (vma->vm_flags & VM_LOCKED)) {
/*
* The preliminary mapping check is mainly to avoid the
* pointless overhead of lock_page on the ZERO_PAGE
* which might bounce very badly if there is contention.
*
* If the page is already locked, we don't need to
* handle it now - vmscan will handle it later if and
* when it attempts to reclaim the page.
*/
if (page->mapping && trylock_page(page)) {
lru_add_drain(); /* push cached pages to LRU */
/*
* Because we lock page here, and migration is
* blocked by the pte's page reference, and we
* know the page is still mapped, we don't even
* need to check for file-cache page truncation.
*/
mlock_vma_page(page);
unlock_page(page);
}
}
pte_unmap_unlock(ptep, ptl);
return page;
bad_page:
pte_unmap_unlock(ptep, ptl);
return ERR_PTR(-EFAULT);
no_page:
pte_unmap_unlock(ptep, ptl);
if (!pte_none(pte))
return NULL;
return no_page_table(vma, flags);
}
static void fill_page_bit_map(struct mm_struct *mm,
unsigned long addr, unsigned long nr_pages,
unsigned long page_bit_map[PAGE_BITS])
{
int i = 0;
DEBUG("GIPC_SEND fill_page_bit_map %lx - %lx\n",
addr, addr + (nr_pages << PAGE_SHIFT));
do {
struct vm_area_struct *vma;
pgd_t *pgd;
pud_t *pud;
pmd_t *pmd;
pte_t *pte;
spinlock_t *ptl;
bool has_page = false;
vma = find_vma(mm, addr);
if (!vma)
goto next;
BUG_ON(vma->vm_flags & VM_HUGETLB);
pgd = pgd_offset(mm, addr);
if (pgd_none(*pgd) || pgd_bad(*pgd))
goto next;
pud = pud_offset(pgd, addr);
if (pud_none(*pud) || pud_bad(*pud))
goto next;
pmd = pmd_offset(pud, addr);
if (pmd_none(*pmd))
goto next;
if (unlikely(pmd_trans_huge(*pmd))) {
has_page = true;
goto next;
}
if (pmd_bad(*pmd))
goto next;
pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
if (pte_none(*pte))
goto next_locked;
if (unlikely(!pte_present(*pte)) && pte_file(*pte))
goto next_locked;
has_page = true;
next_locked:
spin_unlock(ptl);
next:
if (has_page) {
DEBUG("found a page at %lx\n", addr);
set_bit(i, page_bit_map);
} else {
clear_bit(i, page_bit_map);
}
} while (i++, addr += PAGE_SIZE, i < nr_pages);
}