/*
* Handle i/d cache flushing, called from set_pte_at() or ptep_set_access_flags()
*/
static pte_t do_dcache_icache_coherency(pte_t pte, unsigned long addr)
{
unsigned long pfn = pte_pfn(pte);
struct page *page;
if (unlikely(!pfn_valid(pfn)))
return pte;
page = pfn_to_page(pfn);
#ifdef CONFIG_8xx
/* On 8xx, cache control instructions (particularly
* "dcbst" from flush_dcache_icache) fault as write
* operation if there is an unpopulated TLB entry
* for the address in question. To workaround that,
* we invalidate the TLB here, thus avoiding dcbst
* misbehaviour.
*/
_tlbil_va(addr, 0 /* 8xx doesn't care about PID */);
#endif
if (!PageReserved(page) && !test_bit(PG_arch_1, &page->flags)) {
pr_devel("do_dcache_icache_coherency... flushing\n");
flush_dcache_icache_page(page);
set_bit(PG_arch_1, &page->flags);
}
else
pr_devel("do_dcache_icache_coherency... already clean\n");
return __pte(pte_val(pte) | _PAGE_HWEXEC);
}
/* Embedded type MMU with HW exec support. This is a bit more complicated
* as we don't have two bits to spare for _PAGE_EXEC and _PAGE_HWEXEC so
* instead we "filter out" the exec permission for non clean pages.
*/
static pte_t set_pte_filter(pte_t pte)
{
struct page *pg;
/* No exec permission in the first place, move on */
if (!(pte_val(pte) & _PAGE_EXEC) || !pte_looks_normal(pte))
return pte;
/* If you set _PAGE_EXEC on weird pages you're on your own */
pg = maybe_pte_to_page(pte);
if (unlikely(!pg))
return pte;
/* If the page clean, we move on */
if (test_bit(PG_arch_1, &pg->flags))
return pte;
/* If it's an exec fault, we flush the cache and make it clean */
if (is_exec_fault()) {
flush_dcache_icache_page(pg);
set_bit(PG_arch_1, &pg->flags);
return pte;
}
/* Else, we filter out _PAGE_EXEC */
return __pte(pte_val(pte) & ~_PAGE_EXEC);
}
static pte_t set_pte_filter(pte_t pte, unsigned long addr)
{
struct page *pg;
if (!(pte_val(pte) & _PAGE_EXEC) || !pte_looks_normal(pte))
return pte;
pg = maybe_pte_to_page(pte);
if (unlikely(!pg))
return pte;
if (test_bit(PG_arch_1, &pg->flags))
return pte;
if (is_exec_fault()) {
flush_dcache_icache_page(pg);
set_bit(PG_arch_1, &pg->flags);
return pte;
}
return __pte(pte_val(pte) & ~_PAGE_EXEC);
}
static pte_t set_access_flags_filter(pte_t pte, struct vm_area_struct *vma,
int dirty)
{
struct page *pg;
if (dirty || (pte_val(pte) & _PAGE_EXEC) || !is_exec_fault())
return pte;
#ifdef CONFIG_DEBUG_VM
if (WARN_ON(!(vma->vm_flags & VM_EXEC)))
return pte;
#endif
pg = maybe_pte_to_page(pte);
if (unlikely(!pg))
goto bail;
if (test_bit(PG_arch_1, &pg->flags))
goto bail;
flush_dcache_icache_page(pg);
set_bit(PG_arch_1, &pg->flags);
bail:
return __pte(pte_val(pte) | _PAGE_EXEC);
}
static pte_t set_pte_filter(pte_t pte, unsigned long addr)
{
pte = __pte(pte_val(pte) & ~_PAGE_HPTEFLAGS);
if (pte_looks_normal(pte) && !(cpu_has_feature(CPU_FTR_COHERENT_ICACHE) ||
cpu_has_feature(CPU_FTR_NOEXECUTE))) {
struct page *pg = maybe_pte_to_page(pte);
if (!pg)
return pte;
if (!test_bit(PG_arch_1, &pg->flags)) {
#ifdef CONFIG_8xx
/* On 8xx, cache control instructions (particularly
* "dcbst" from flush_dcache_icache) fault as write
* operation if there is an unpopulated TLB entry
* for the address in question. To workaround that,
* we invalidate the TLB here, thus avoiding dcbst
* misbehaviour.
*/
/* 8xx doesn't care about PID, size or ind args */
_tlbil_va(addr, 0, 0, 0);
#endif /* CONFIG_8xx */
flush_dcache_icache_page(pg);
set_bit(PG_arch_1, &pg->flags);
}
}
return pte;
}
/*
* This is called at the end of handling a user page fault, when the
* fault has been handled by updating a PTE in the linux page tables.
* We use it to preload an HPTE into the hash table corresponding to
* the updated linux PTE.
*
* This must always be called with the pte lock held.
*/
void update_mmu_cache(struct vm_area_struct *vma, unsigned long address,
pte_t pte)
{
#ifdef CONFIG_PPC_STD_MMU
unsigned long access = 0, trap;
#endif
unsigned long pfn = pte_pfn(pte);
/* handle i-cache coherency */
if (!cpu_has_feature(CPU_FTR_COHERENT_ICACHE) &&
!cpu_has_feature(CPU_FTR_NOEXECUTE) &&
pfn_valid(pfn)) {
struct page *page = pfn_to_page(pfn);
#ifdef CONFIG_8xx
/* On 8xx, cache control instructions (particularly
* "dcbst" from flush_dcache_icache) fault as write
* operation if there is an unpopulated TLB entry
* for the address in question. To workaround that,
* we invalidate the TLB here, thus avoiding dcbst
* misbehaviour.
*/
_tlbie(address);
#endif
if (!PageReserved(page)
&& !test_bit(PG_arch_1, &page->flags)) {
if (vma->vm_mm == current->active_mm) {
__flush_dcache_icache((void *) address);
} else
flush_dcache_icache_page(page);
set_bit(PG_arch_1, &page->flags);
}
}
#ifdef CONFIG_PPC_STD_MMU
/* We only want HPTEs for linux PTEs that have _PAGE_ACCESSED set */
if (!pte_young(pte) || address >= TASK_SIZE)
return;
/* We try to figure out if we are coming from an instruction
* access fault and pass that down to __hash_page so we avoid
* double-faulting on execution of fresh text. We have to test
* for regs NULL since init will get here first thing at boot
*
* We also avoid filling the hash if not coming from a fault
*/
if (current->thread.regs == NULL)
return;
trap = TRAP(current->thread.regs);
if (trap == 0x400)
access |= _PAGE_EXEC;
else if (trap != 0x300)
return;
hash_preload(vma->vm_mm, address, access, trap);
#endif /* CONFIG_PPC_STD_MMU */
}
static pte_t set_pte_filter(pte_t pte)
{
pte = __pte(pte_val(pte) & ~_PAGE_HPTEFLAGS);
if (pte_looks_normal(pte) && !(cpu_has_feature(CPU_FTR_COHERENT_ICACHE) ||
cpu_has_feature(CPU_FTR_NOEXECUTE))) {
struct page *pg = maybe_pte_to_page(pte);
if (!pg)
return pte;
if (!test_bit(PG_arch_1, &pg->flags)) {
flush_dcache_icache_page(pg);
set_bit(PG_arch_1, &pg->flags);
}
}
return pte;
}
/*
* Handle i/d cache flushing, called from set_pte_at() or ptep_set_access_flags()
*/
static pte_t do_dcache_icache_coherency(pte_t pte)
{
unsigned long pfn = pte_pfn(pte);
struct page *page;
if (unlikely(!pfn_valid(pfn)))
return pte;
page = pfn_to_page(pfn);
if (!PageReserved(page) && !test_bit(PG_arch_1, &page->flags)) {
pr_debug("do_dcache_icache_coherency... flushing\n");
flush_dcache_icache_page(page);
set_bit(PG_arch_1, &page->flags);
}
else
pr_debug("do_dcache_icache_coherency... already clean\n");
return __pte(pte_val(pte) | _PAGE_HWEXEC);
}
static pte_t set_pte_filter(pte_t pte, unsigned long addr)
{
pte = __pte(pte_val(pte) & ~_PAGE_HPTEFLAGS);
if (pte_looks_normal(pte) && !(cpu_has_feature(CPU_FTR_COHERENT_ICACHE) ||
cpu_has_feature(CPU_FTR_NOEXECUTE))) {
struct page *pg = maybe_pte_to_page(pte);
if (!pg)
return pte;
if (!test_bit(PG_arch_1, &pg->flags)) {
#ifdef CONFIG_8xx
_tlbil_va(addr, 0, 0, 0);
#endif
flush_dcache_icache_page(pg);
set_bit(PG_arch_1, &pg->flags);
}
}
return pte;
}
static pte_t set_access_flags_filter(pte_t pte, struct vm_area_struct *vma,
int dirty)
{
struct page *pg;
/* So here, we only care about exec faults, as we use them
* to recover lost _PAGE_EXEC and perform I$/D$ coherency
* if necessary. Also if _PAGE_EXEC is already set, same deal,
* we just bail out
*/
if (dirty || (pte_val(pte) & _PAGE_EXEC) || !is_exec_fault())
return pte;
#ifdef CONFIG_DEBUG_VM
/* So this is an exec fault, _PAGE_EXEC is not set. If it was
* an error we would have bailed out earlier in do_page_fault()
* but let's make sure of it
*/
if (WARN_ON(!(vma->vm_flags & VM_EXEC)))
return pte;
#endif /* CONFIG_DEBUG_VM */
/* If you set _PAGE_EXEC on weird pages you're on your own */
pg = maybe_pte_to_page(pte);
if (unlikely(!pg))
goto bail;
/* If the page is already clean, we move on */
if (test_bit(PG_arch_1, &pg->flags))
goto bail;
/* Clean the page and set PG_arch_1 */
flush_dcache_icache_page(pg);
set_bit(PG_arch_1, &pg->flags);
bail:
return __pte(pte_val(pte) | _PAGE_EXEC);
}