/*
* page not present ... go through shm_pages
*/
static unsigned long shm_nopage(struct vm_area_struct * shmd, unsigned long address, int no_share)
{
pte_t pte;
struct shmid_kernel *shp;
unsigned int id, idx;
id = SWP_OFFSET(shmd->vm_pte) & SHM_ID_MASK;
idx = (address - shmd->vm_start + shmd->vm_offset) >> PAGE_SHIFT;
#ifdef DEBUG_SHM
if (id > max_shmid) {
printk ("shm_nopage: id=%d too big. proc mem corrupted\n", id);
return 0;
}
#endif
shp = shm_segs[id];
#ifdef DEBUG_SHM
if (shp == IPC_UNUSED || shp == IPC_NOID) {
printk ("shm_nopage: id=%d invalid. Race.\n", id);
return 0;
}
#endif
/* This can occur on a remap */
if (idx >= shp->shm_npages) {
return 0;
}
pte = __pte(shp->shm_pages[idx]);
if (!pte_present(pte)) {
unsigned long page = get_free_page(GFP_USER);
if (!page)
return -1;
pte = __pte(shp->shm_pages[idx]);
if (pte_present(pte)) {
free_page (page); /* doesn't sleep */
goto done;
}
if (!pte_none(pte)) {
rw_swap_page_nocache(READ, pte_val(pte), (char *)page);
pte = __pte(shp->shm_pages[idx]);
if (pte_present(pte)) {
free_page (page); /* doesn't sleep */
goto done;
}
swap_free(pte_val(pte));
shm_swp--;
}
shm_rss++;
pte = pte_mkdirty(mk_pte(page, PAGE_SHARED));
shp->shm_pages[idx] = pte_val(pte);
} else
--current->maj_flt; /* was incremented in do_no_page */
done: /* pte_val(pte) == shp->shm_pages[idx] */
current->min_flt++;
atomic_inc(&mem_map[MAP_NR(pte_page(pte))].count);
return pte_page(pte);
}
/*H:481
* This clears the Switcher mappings for cpu #i.
*/
static void remove_switcher_percpu_map(struct lg_cpu *cpu, unsigned int i)
{
unsigned long base = switcher_addr + PAGE_SIZE + i * PAGE_SIZE*2;
pte_t *pte;
/* Clear the mappings for both pages. */
pte = find_spte(cpu, base, false, 0, 0);
release_pte(*pte);
set_pte(pte, __pte(0));
pte = find_spte(cpu, base + PAGE_SIZE, false, 0, 0);
release_pte(*pte);
set_pte(pte, __pte(0));
}
static pte_t set_pte_filter(pte_t pte, unsigned long addr)
{
pte = __pte(pte_val(pte) & ~_PAGE_HPTEFLAGS);
if (pte_looks_normal(pte) && !(cpu_has_feature(CPU_FTR_COHERENT_ICACHE) ||
cpu_has_feature(CPU_FTR_NOEXECUTE))) {
struct page *pg = maybe_pte_to_page(pte);
if (!pg)
return pte;
if (!test_bit(PG_arch_1, &pg->flags)) {
#ifdef CONFIG_8xx
/* On 8xx, cache control instructions (particularly
* "dcbst" from flush_dcache_icache) fault as write
* operation if there is an unpopulated TLB entry
* for the address in question. To workaround that,
* we invalidate the TLB here, thus avoiding dcbst
* misbehaviour.
*/
/* 8xx doesn't care about PID, size or ind args */
_tlbil_va(addr, 0, 0, 0);
#endif /* CONFIG_8xx */
flush_dcache_icache_page(pg);
set_bit(PG_arch_1, &pg->flags);
}
}
return pte;
}
void grants_init(void)
{
unsigned long frames[NR_GRANT_PAGES];
gnttab_setup_table_t op;
op.dom = DOMID_SELF;
op.nr_frames = NR_GRANT_PAGES;
set_xen_guest_handle(op.frame_list, frames);
int rs = HYPERVISOR_grant_table_op(GNTTABOP_setup_table, &op, 1);
if (rs < 0)
fatal_error("grants_init: setup_table failed: %d\n", rs);
for (int i = NR_GRANT_ENTRIES-1; i >= NR_RESERVED_ENTRIES; i--)
{
free_list[i] = free_entry;
free_entry = i;
}
grant_entries = mm_alloc_pages(NR_GRANT_PAGES);
if (grant_entries == 0)
fatal_error("grants_init: grant entries page allocation failed\n");
for (int i = 0; i < NR_GRANT_PAGES; i++)
{
unsigned long ma_grant_table = frames[i] << PAGE_SHIFT;
rs = HYPERVISOR_update_va_mapping((unsigned long)grant_entries + i*PAGE_SIZE,
__pte(ma_grant_table | 7), UVMF_INVLPG);
if (rs < 0)
fatal_error("grants_init: update mapping failed: %d\n", rs);
}
}
/* Main kernel entry point, called by trampoline */
void start_kernel(start_info_t * start_info)
{
/* Define hypervisor upcall entry points */
HYPERVISOR_set_callbacks(
FLAT_KERNEL_CS, (unsigned long)hypervisor_callback,
FLAT_KERNEL_CS, (unsigned long)failsafe_callback);
/* Map the shared info page */
HYPERVISOR_update_va_mapping((unsigned long) shared_info,
__pte(start_info->shared_info),
UVMF_INVLPG);
/* Initialise the console */
console_init(start_info);
/* Write a message to check that it worked */
console_write("Hello world!\n\r");
console_write("Xen magic string: ");
console_write(start_info->magic);
console_write("\n\r");
/* Set up the XenStore driver */
xenstore_init(start_info);
/* Test the store */
xenstore_test();
/* Flush the console buffer */
console_flush();
/* Exit, since we don't know how to do anything else */
}
static int unmap_pte_fn(pte_t *pte, struct page *pmd_page,
unsigned long addr, void *data)
{
set_pte_at(&init_mm, addr, pte, __pte(0));
return 0;
}
/*
* Handle i/d cache flushing, called from set_pte_at() or ptep_set_access_flags()
*/
static pte_t do_dcache_icache_coherency(pte_t pte, unsigned long addr)
{
unsigned long pfn = pte_pfn(pte);
struct page *page;
if (unlikely(!pfn_valid(pfn)))
return pte;
page = pfn_to_page(pfn);
#ifdef CONFIG_8xx
/* On 8xx, cache control instructions (particularly
* "dcbst" from flush_dcache_icache) fault as write
* operation if there is an unpopulated TLB entry
* for the address in question. To workaround that,
* we invalidate the TLB here, thus avoiding dcbst
* misbehaviour.
*/
_tlbil_va(addr, 0 /* 8xx doesn't care about PID */);
#endif
if (!PageReserved(page) && !test_bit(PG_arch_1, &page->flags)) {
pr_devel("do_dcache_icache_coherency... flushing\n");
flush_dcache_icache_page(page);
set_bit(PG_arch_1, &page->flags);
}
else
pr_devel("do_dcache_icache_coherency... already clean\n");
return __pte(pte_val(pte) | _PAGE_HWEXEC);
}
/*
* Update the MMU hash table to correspond with a change to
* a Linux PTE. If wrprot is true, it is permissible to
* change the existing HPTE to read-only rather than removing it
* (if we remove it we should clear the _PTE_HPTEFLAGS bits).
*/
void hpte_update(struct mm_struct *mm, unsigned long addr,
unsigned long pte, int wrprot)
{
int i;
unsigned long context = 0;
struct ppc64_tlb_batch *batch = &__get_cpu_var(ppc64_tlb_batch);
if (REGION_ID(addr) == USER_REGION_ID)
context = mm->context.id;
i = batch->index;
/*
* This can happen when we are in the middle of a TLB batch and
* we encounter memory pressure (eg copy_page_range when it tries
* to allocate a new pte). If we have to reclaim memory and end
* up scanning and resetting referenced bits then our batch context
* will change mid stream.
*/
if (i != 0 && (context != batch->context ||
batch->large != pte_huge(pte))) {
flush_tlb_pending();
i = 0;
}
if (i == 0) {
batch->context = context;
batch->mm = mm;
batch->large = pte_huge(pte);
}
batch->pte[i] = __pte(pte);
batch->addr[i] = addr;
batch->index = ++i;
if (i >= PPC64_TLB_BATCH_NR)
flush_tlb_pending();
}
void
iounmap_atomic(void __iomem *kvaddr)
{
unsigned long vaddr = (unsigned long) kvaddr & PAGE_MASK;
if (vaddr >= __fix_to_virt(FIX_KMAP_END) &&
vaddr <= __fix_to_virt(FIX_KMAP_BEGIN)) {
int idx, type;
type = kmap_atomic_idx();
idx = type + KM_TYPE_NR * smp_processor_id();
#ifdef CONFIG_DEBUG_HIGHMEM
WARN_ON_ONCE(vaddr != __fix_to_virt(FIX_KMAP_BEGIN + idx));
#endif
/*
* Force other mappings to Oops if they'll try to access this
* pte without first remap it. Keeping stale mappings around
* is a bad idea also, in case the page changes cacheability
* attributes or becomes a protected page in a hypervisor.
*/
#ifdef CONFIG_PREEMPT_RT_FULL
current->kmap_pte[type] = __pte(0);
#endif
kpte_clear_flush(kmap_pte-idx, vaddr);
kmap_atomic_idx_pop();
}
pagefault_enable();
preempt_enable();
}
static int set_up_temporary_text_mapping(pgd_t *pgd_base)
{
pgd_t *pgd;
pmd_t *pmd;
pte_t *pte;
pgd = pgd_base + pgd_index(restore_jump_address);
pmd = resume_one_md_table_init(pgd);
if (!pmd)
return -ENOMEM;
if (boot_cpu_has(X86_FEATURE_PSE)) {
set_pmd(pmd + pmd_index(restore_jump_address),
__pmd((jump_address_phys & PMD_MASK) | pgprot_val(PAGE_KERNEL_LARGE_EXEC)));
} else {
pte = resume_one_page_table_init(pmd);
if (!pte)
return -ENOMEM;
set_pte(pte + pte_index(restore_jump_address),
__pte((jump_address_phys & PAGE_MASK) | pgprot_val(PAGE_KERNEL_EXEC)));
}
return 0;
}
static pte_t set_pte_filter(pte_t pte, unsigned long addr)
{
struct page *pg;
if (!(pte_val(pte) & _PAGE_EXEC) || !pte_looks_normal(pte))
return pte;
pg = maybe_pte_to_page(pte);
if (unlikely(!pg))
return pte;
if (test_bit(PG_arch_1, &pg->flags))
return pte;
if (is_exec_fault()) {
flush_dcache_icache_page(pg);
set_bit(PG_arch_1, &pg->flags);
return pte;
}
return __pte(pte_val(pte) & ~_PAGE_EXEC);
}
static pte_t set_access_flags_filter(pte_t pte, struct vm_area_struct *vma,
int dirty)
{
struct page *pg;
if (dirty || (pte_val(pte) & _PAGE_EXEC) || !is_exec_fault())
return pte;
#ifdef CONFIG_DEBUG_VM
if (WARN_ON(!(vma->vm_flags & VM_EXEC)))
return pte;
#endif
pg = maybe_pte_to_page(pte);
if (unlikely(!pg))
goto bail;
if (test_bit(PG_arch_1, &pg->flags))
goto bail;
flush_dcache_icache_page(pg);
set_bit(PG_arch_1, &pg->flags);
bail:
return __pte(pte_val(pte) | _PAGE_EXEC);
}
/*
* Check that TLB entries with kernel ASID (1) have kernel VMA (>= TASK_SIZE),
* and TLB entries with user ASID (>=4) have VMA < TASK_SIZE.
*
* Check that valid TLB entries either have the same PA as the PTE, or PTE is
* marked as non-present. Non-present PTE and the page with non-zero refcount
* and zero mapcount is normal for batched TLB flush operation. Zero refcount
* means that the page was freed prematurely. Non-zero mapcount is unusual,
* but does not necessary means an error, thus marked as suspicious.
*/
static int check_tlb_entry(unsigned w, unsigned e, bool dtlb)
{
unsigned tlbidx = w | (e << PAGE_SHIFT);
unsigned r0 = dtlb ?
read_dtlb_virtual(tlbidx) : read_itlb_virtual(tlbidx);
unsigned vpn = (r0 & PAGE_MASK) | (e << PAGE_SHIFT);
unsigned pte = get_pte_for_vaddr(vpn);
unsigned mm_asid = (get_rasid_register() >> 8) & ASID_MASK;
unsigned tlb_asid = r0 & ASID_MASK;
bool kernel = tlb_asid == 1;
int rc = 0;
if (tlb_asid > 0 && ((vpn < TASK_SIZE) == kernel)) {
pr_err("%cTLB: way: %u, entry: %u, VPN %08x in %s PTE\n",
dtlb ? 'D' : 'I', w, e, vpn,
kernel ? "kernel" : "user");
rc |= TLB_INSANE;
}
if (tlb_asid == mm_asid) {
unsigned r1 = dtlb ? read_dtlb_translation(tlbidx) :
read_itlb_translation(tlbidx);
if ((pte ^ r1) & PAGE_MASK) {
pr_err("%cTLB: way: %u, entry: %u, mapping: %08x->%08x, PTE: %08x\n",
dtlb ? 'D' : 'I', w, e, r0, r1, pte);
if (pte == 0 || !pte_present(__pte(pte))) {
struct page *p = pfn_to_page(r1 >> PAGE_SHIFT);
pr_err("page refcount: %d, mapcount: %d\n",
page_count(p),
page_mapcount(p));
if (!page_count(p))
rc |= TLB_INSANE;
else if (page_mapped(p))
rc |= TLB_SUSPICIOUS;
} else {
pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
pte_t *ptep)
{
pte_t entry;
int i;
unsigned long nptes;
entry = *ptep;
if (pte_present(entry))
mm->context.huge_pte_count--;
addr &= HPAGE_MASK;
nptes = 1 << HUGETLB_PAGE_ORDER;
for (i = 0; i < nptes; i++) {
*ptep = __pte(0UL);
addr += PAGE_SIZE;
ptep++;
}
/* Issue TLB flush at REAL_HPAGE_SIZE boundaries */
addr -= REAL_HPAGE_SIZE;
ptep -= nptes / 2;
maybe_tlb_batch_add(mm, addr, ptep, entry, 0);
addr -= REAL_HPAGE_SIZE;
ptep -= nptes / 2;
maybe_tlb_batch_add(mm, addr, ptep, entry, 0);
return entry;
}
/* Embedded type MMU with HW exec support. This is a bit more complicated
* as we don't have two bits to spare for _PAGE_EXEC and _PAGE_HWEXEC so
* instead we "filter out" the exec permission for non clean pages.
*/
static pte_t set_pte_filter(pte_t pte)
{
struct page *pg;
/* No exec permission in the first place, move on */
if (!(pte_val(pte) & _PAGE_EXEC) || !pte_looks_normal(pte))
return pte;
/* If you set _PAGE_EXEC on weird pages you're on your own */
pg = maybe_pte_to_page(pte);
if (unlikely(!pg))
return pte;
/* If the page clean, we move on */
if (test_bit(PG_arch_1, &pg->flags))
return pte;
/* If it's an exec fault, we flush the cache and make it clean */
if (is_exec_fault()) {
flush_dcache_icache_page(pg);
set_bit(PG_arch_1, &pg->flags);
return pte;
}
/* Else, we filter out _PAGE_EXEC */
return __pte(pte_val(pte) & ~_PAGE_EXEC);
}
void init_kio(void)
{
#if XCHAL_HAVE_PTP_MMU && XCHAL_HAVE_SPANNING_WAY && defined(CONFIG_OF)
/*
* Update the IO area mapping in case xtensa_kio_paddr has changed
*/
write_dtlb_entry(__pte(xtensa_kio_paddr + CA_WRITEBACK),
XCHAL_KIO_CACHED_VADDR + 6);
write_itlb_entry(__pte(xtensa_kio_paddr + CA_WRITEBACK),
XCHAL_KIO_CACHED_VADDR + 6);
write_dtlb_entry(__pte(xtensa_kio_paddr + CA_BYPASS),
XCHAL_KIO_BYPASS_VADDR + 6);
write_itlb_entry(__pte(xtensa_kio_paddr + CA_BYPASS),
XCHAL_KIO_BYPASS_VADDR + 6);
#endif
}
static int relocate_restore_code(void)
{
pgd_t *pgd;
pud_t *pud;
relocated_restore_code = get_safe_page(GFP_ATOMIC);
if (!relocated_restore_code)
return -ENOMEM;
memcpy((void *)relocated_restore_code, &core_restore_code, PAGE_SIZE);
/* Make the page containing the relocated code executable */
pgd = (pgd_t *)__va(read_cr3()) + pgd_index(relocated_restore_code);
pud = pud_offset(pgd, relocated_restore_code);
if (pud_large(*pud)) {
set_pud(pud, __pud(pud_val(*pud) & ~_PAGE_NX));
} else {
pmd_t *pmd = pmd_offset(pud, relocated_restore_code);
if (pmd_large(*pmd)) {
set_pmd(pmd, __pmd(pmd_val(*pmd) & ~_PAGE_NX));
} else {
pte_t *pte = pte_offset_kernel(pmd, relocated_restore_code);
set_pte(pte, __pte(pte_val(*pte) & ~_PAGE_NX));
}
}
__flush_tlb_all();
return 0;
}
/*
* Unmapping vs dcache management:
*
* If a guest maps certain memory pages as uncached, all writes will
* bypass the data cache and go directly to RAM. However, the CPUs
* can still speculate reads (not writes) and fill cache lines with
* data.
*
* Those cache lines will be *clean* cache lines though, so a
* clean+invalidate operation is equivalent to an invalidate
* operation, because no cache lines are marked dirty.
*
* Those clean cache lines could be filled prior to an uncached write
* by the guest, and the cache coherent IO subsystem would therefore
* end up writing old data to disk.
*
* This is why right after unmapping a page/section and invalidating
* the corresponding TLBs, we call kvm_flush_dcache_p*() to make sure
* the IO subsystem will never hit in the cache.
*/
static void unmap_ptes(struct kvm *kvm, pmd_t *pmd,
phys_addr_t addr, phys_addr_t end)
{
phys_addr_t start_addr = addr;
pte_t *pte, *start_pte;
start_pte = pte = pte_offset_kernel(pmd, addr);
do {
if (!pte_none(*pte)) {
pte_t old_pte = *pte;
kvm_set_pte(pte, __pte(0));
kvm_tlb_flush_vmid_ipa(kvm, addr);
/* No need to invalidate the cache for device mappings */
if ((pte_val(old_pte) & PAGE_S2_DEVICE) != PAGE_S2_DEVICE)
kvm_flush_dcache_pte(old_pte);
put_page(virt_to_page(pte));
}
} while (pte++, addr += PAGE_SIZE, addr != end);
if (kvm_pte_table_empty(kvm, start_pte))
clear_pmd_entry(kvm, pmd, start_addr);
}
void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
pte_t *ptep, pte_t entry)
{
unsigned int i, nptes, orig_shift, shift;
unsigned long size;
pte_t orig;
size = huge_tte_to_size(entry);
shift = size >= HPAGE_SIZE ? PMD_SHIFT : PAGE_SHIFT;
nptes = size >> shift;
if (!pte_present(*ptep) && pte_present(entry))
mm->context.hugetlb_pte_count += nptes;
addr &= ~(size - 1);
orig = *ptep;
orig_shift = pte_none(orig) ? PAGE_SHIFT : huge_tte_to_shift(orig);
for (i = 0; i < nptes; i++)
ptep[i] = __pte(pte_val(entry) + (i << shift));
maybe_tlb_batch_add(mm, addr, ptep, orig, 0, orig_shift);
/* An HPAGE_SIZE'ed page is composed of two REAL_HPAGE_SIZE'ed pages */
if (size == HPAGE_SIZE)
maybe_tlb_batch_add(mm, addr + REAL_HPAGE_SIZE, ptep, orig, 0,
orig_shift);
}
void __init kasan_early_init(void)
{
int i;
pteval_t pte_val = __pa_nodebug(kasan_early_shadow_page) |
__PAGE_KERNEL | _PAGE_ENC;
pmdval_t pmd_val = __pa_nodebug(kasan_early_shadow_pte) | _KERNPG_TABLE;
pudval_t pud_val = __pa_nodebug(kasan_early_shadow_pmd) | _KERNPG_TABLE;
p4dval_t p4d_val = __pa_nodebug(kasan_early_shadow_pud) | _KERNPG_TABLE;
/* Mask out unsupported __PAGE_KERNEL bits: */
pte_val &= __default_kernel_pte_mask;
pmd_val &= __default_kernel_pte_mask;
pud_val &= __default_kernel_pte_mask;
p4d_val &= __default_kernel_pte_mask;
for (i = 0; i < PTRS_PER_PTE; i++)
kasan_early_shadow_pte[i] = __pte(pte_val);
for (i = 0; i < PTRS_PER_PMD; i++)
kasan_early_shadow_pmd[i] = __pmd(pmd_val);
for (i = 0; i < PTRS_PER_PUD; i++)
kasan_early_shadow_pud[i] = __pud(pud_val);
for (i = 0; pgtable_l5_enabled() && i < PTRS_PER_P4D; i++)
kasan_early_shadow_p4d[i] = __p4d(p4d_val);
kasan_map_early_shadow(early_top_pgt);
kasan_map_early_shadow(init_top_pgt);
}
void __kunmap_atomic(void *kvaddr)
{
unsigned long vaddr = (unsigned long) kvaddr & PAGE_MASK;
int idx, type;
if (kvaddr >= (void *)FIXADDR_START) {
type = kmap_atomic_idx();
idx = type + KM_TYPE_NR * smp_processor_id();
if (cache_is_vivt())
__cpuc_flush_dcache_area((void *)vaddr, PAGE_SIZE);
#ifdef CONFIG_DEBUG_HIGHMEM
BUG_ON(vaddr != __fix_to_virt(FIX_KMAP_BEGIN + idx));
set_pte_ext(TOP_PTE(vaddr), __pte(0), 0);
local_flush_tlb_kernel_page(vaddr);
#else
(void) idx; /* to kill a warning */
#endif
kmap_atomic_idx_pop();
} else if (vaddr >= PKMAP_ADDR(0) && vaddr < PKMAP_ADDR(LAST_PKMAP)) {
/* this address was obtained through kmap_high_get() */
kunmap_high(pte_page(pkmap_page_table[PKMAP_NR(vaddr)]));
}
pagefault_enable();
}
static void kmap_remove_unused_cpu(int cpu)
{
int start_idx, idx, type;
int need_flush = 0;
pagefault_disable();
type = kmap_atomic_idx();
start_idx = type + 1 + KM_TYPE_NR * cpu;
for (idx = start_idx; idx < KM_TYPE_NR + KM_TYPE_NR * cpu; idx++) {
unsigned long vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx);
pte_t ptep;
ptep = get_top_pte(vaddr);
if (ptep) {
set_top_pte(vaddr, __pte(0));
need_flush = 1;
}
}
pagefault_enable();
/* flush the caches and tlb if required */
if (need_flush) {
local_flush_tlb_all();
flush_cache_all();
}
}
static void clear_pte_entry(struct kvm *kvm, pte_t *pte, phys_addr_t addr)
{
if (pte_present(*pte)) {
kvm_set_pte(pte, __pte(0));
put_page(virt_to_page(pte));
kvm_tlb_flush_vmid_ipa(kvm, addr);
}
}
void kmemcheck_hide_pages(struct page *p, unsigned int n)
{
unsigned int i;
for (i = 0; i < n; ++i) {
unsigned long address;
pte_t *pte;
unsigned int level;
address = (unsigned long) page_address(&p[i]);
pte = lookup_address(address, &level);
BUG_ON(!pte);
BUG_ON(level != PG_LEVEL_4K);
set_pte(pte, __pte(pte_val(*pte) & ~_PAGE_PRESENT));
set_pte(pte, __pte(pte_val(*pte) | _PAGE_HIDDEN));
__flush_tlb_one(address);
}
}
void set_pmd_at(struct mm_struct *mm, unsigned long addr,
pmd_t *pmdp, pmd_t pmd)
{
pmd_t orig = *pmdp;
*pmdp = pmd;
if (mm == &init_mm)
return;
if ((pmd_val(pmd) ^ pmd_val(orig)) & _PAGE_PMD_HUGE) {
/*
* Note that this routine only sets pmds for THP pages.
* Hugetlb pages are handled elsewhere. We need to check
* for huge zero page. Huge zero pages are like hugetlb
* pages in that there is no RSS, but there is the need
* for TSB entries. So, huge zero page counts go into
* hugetlb_pte_count.
*/
if (pmd_val(pmd) & _PAGE_PMD_HUGE) {
if (is_huge_zero_page(pmd_page(pmd)))
mm->context.hugetlb_pte_count++;
else
mm->context.thp_pte_count++;
} else {
if (is_huge_zero_page(pmd_page(orig)))
mm->context.hugetlb_pte_count--;
else
mm->context.thp_pte_count--;
}
/* Do not try to allocate the TSB hash table if we
* don't have one already. We have various locks held
* and thus we'll end up doing a GFP_KERNEL allocation
* in an atomic context.
*
* Instead, we let the first TLB miss on a hugepage
* take care of this.
*/
}
if (!pmd_none(orig)) {
addr &= HPAGE_MASK;
if (pmd_trans_huge(orig)) {
pte_t orig_pte = __pte(pmd_val(orig));
bool exec = pte_exec(orig_pte);
tlb_batch_add_one(mm, addr, exec, REAL_HPAGE_SHIFT);
tlb_batch_add_one(mm, addr + REAL_HPAGE_SIZE, exec,
REAL_HPAGE_SHIFT);
} else {
tlb_batch_pmd_scan(mm, addr, orig);
}
}
}
/*
* Update the MMU hash table to correspond with a change to
* a Linux PTE. If wrprot is true, it is permissible to
* change the existing HPTE to read-only rather than removing it
* (if we remove it we should clear the _PTE_HPTEFLAGS bits).
*/
void hpte_update(struct mm_struct *mm, unsigned long addr,
pte_t *ptep, unsigned long pte, int huge)
{
struct ppc64_tlb_batch *batch = &__get_cpu_var(ppc64_tlb_batch);
unsigned long vsid;
unsigned int psize;
int i;
i = batch->index;
/* We mask the address for the base page size. Huge pages will
* have applied their own masking already
*/
addr &= PAGE_MASK;
/* Get page size (maybe move back to caller) */
if (huge) {
#ifdef CONFIG_HUGETLB_PAGE
psize = mmu_huge_psize;
#else
BUG();
psize = pte_pagesize_index(pte); /* shutup gcc */
#endif
} else
psize = pte_pagesize_index(pte);
/*
* This can happen when we are in the middle of a TLB batch and
* we encounter memory pressure (eg copy_page_range when it tries
* to allocate a new pte). If we have to reclaim memory and end
* up scanning and resetting referenced bits then our batch context
* will change mid stream.
*
* We also need to ensure only one page size is present in a given
* batch
*/
if (i != 0 && (mm != batch->mm || batch->psize != psize)) {
flush_tlb_pending();
i = 0;
}
if (i == 0) {
batch->mm = mm;
batch->psize = psize;
}
if (!is_kernel_addr(addr)) {
vsid = get_vsid(mm->context.id, addr);
WARN_ON(vsid == 0);
} else
vsid = get_kernel_vsid(addr);
batch->vaddr[i] = (vsid << 28 ) | (addr & 0x0fffffff);
batch->pte[i] = __real_pte(__pte(pte), ptep);
batch->index = ++i;
if (i >= PPC64_TLB_BATCH_NR)
flush_tlb_pending();
}
int shm_swap (int prio, int gfp_mask)
{
pte_t page;
struct shmid_kernel *shp;
unsigned long swap_nr;
unsigned long id, idx;
int loop = 0;
int counter;
counter = shm_rss >> prio;
if (!counter || !(swap_nr = get_swap_page()))
return 0;
check_id:
shp = shm_segs[swap_id];
if (shp == IPC_UNUSED || shp == IPC_NOID || shp->u.shm_perm.mode & SHM_LOCKED ) {
next_id:
swap_idx = 0;
if (++swap_id > max_shmid) {
swap_id = 0;
if (loop)
goto failed;
loop = 1;
}
goto check_id;
}
id = swap_id;
check_table:
idx = swap_idx++;
if (idx >= shp->shm_npages)
goto next_id;
page = __pte(shp->shm_pages[idx]);
if (!pte_present(page))
goto check_table;
if ((gfp_mask & __GFP_DMA) && !PageDMA(&mem_map[MAP_NR(pte_page(page))]))
goto check_table;
swap_attempts++;
if (--counter < 0) { /* failed */
failed:
swap_free (swap_nr);
return 0;
}
if (atomic_read(&mem_map[MAP_NR(pte_page(page))].count) != 1)
goto check_table;
shp->shm_pages[idx] = swap_nr;
rw_swap_page_nocache (WRITE, swap_nr, (char *) pte_page(page));
free_page(pte_page(page));
swap_successes++;
shm_swp++;
shm_rss--;
return 1;
}
static
shared_info_t *map_shared_info(unsigned long pa)
{
if ( HYPERVISOR_update_va_mapping(
(unsigned long)shared_info, __pte(pa | 7), UVMF_INVLPG) )
{
printk("Failed to map shared_info!!\n");
do_exit();
}
return (shared_info_t *)shared_info;
}
static inline void l4x_cpu_set_pte_ext(pte_t *pteptr, pte_t pteval,
unsigned int ext)
{
if (pte_present(*pteptr)) {
if (pteval == __pte(0))
l4x_pte_clear(NULL, 0, *pteptr);
else
pte_val(pteval) = l4x_set_pte(NULL, 0, *pteptr, pteval);
}
*pteptr = pteval;
}
void __init kasan_init(void)
{
int i;
#ifdef CONFIG_KASAN_INLINE
register_die_notifier(&kasan_die_notifier);
#endif
memcpy(early_level4_pgt, init_level4_pgt, sizeof(early_level4_pgt));
load_cr3(early_level4_pgt);
__flush_tlb_all();
clear_pgds(KASAN_SHADOW_START, KASAN_SHADOW_END);
kasan_populate_zero_shadow((void *)KASAN_SHADOW_START,
kasan_mem_to_shadow((void *)PAGE_OFFSET));
for (i = 0; i < E820_X_MAX; i++) {
if (pfn_mapped[i].end == 0)
break;
if (map_range(&pfn_mapped[i]))
panic("kasan: unable to allocate shadow!");
}
kasan_populate_zero_shadow(
kasan_mem_to_shadow((void *)PAGE_OFFSET + MAXMEM),
kasan_mem_to_shadow((void *)__START_KERNEL_map));
vmemmap_populate((unsigned long)kasan_mem_to_shadow(_stext),
(unsigned long)kasan_mem_to_shadow(_end),
NUMA_NO_NODE);
kasan_populate_zero_shadow(kasan_mem_to_shadow((void *)MODULES_END),
(void *)KASAN_SHADOW_END);
load_cr3(init_level4_pgt);
__flush_tlb_all();
/*
* kasan_zero_page has been used as early shadow memory, thus it may
* contain some garbage. Now we can clear and write protect it, since
* after the TLB flush no one should write to it.
*/
memset(kasan_zero_page, 0, PAGE_SIZE);
for (i = 0; i < PTRS_PER_PTE; i++) {
pte_t pte = __pte(__pa(kasan_zero_page) | __PAGE_KERNEL_RO);
set_pte(&kasan_zero_pte[i], pte);
}
/* Flush TLBs again to be sure that write protection applied. */
__flush_tlb_all();
init_task.kasan_depth = 0;
pr_info("KernelAddressSanitizer initialized\n");
}
