static inline void _tlbie_va_range(unsigned long start, unsigned long end, unsigned long pid, unsigned long page_size, unsigned long psize, bool also_pwc) { asm volatile("ptesync": : :"memory"); if (also_pwc) __tlbie_pid(pid, RIC_FLUSH_PWC); __tlbie_va_range(start, end, pid, page_size, psize); fixup_tlbie(); asm volatile("eieio; tlbsync; ptesync": : :"memory"); }
static inline void __radix__flush_tlb_range(struct mm_struct *mm, unsigned long start, unsigned long end, bool flush_all_sizes) { unsigned long pid; unsigned int page_shift = mmu_psize_defs[mmu_virtual_psize].shift; unsigned long page_size = 1UL << page_shift; unsigned long nr_pages = (end - start) >> page_shift; bool local, full; pid = mm->context.id; if (unlikely(pid == MMU_NO_CONTEXT)) return; preempt_disable(); smp_mb(); /* see radix__flush_tlb_mm */ if (!mm_is_thread_local(mm)) { if (unlikely(mm_is_singlethreaded(mm))) { if (end != TLB_FLUSH_ALL) { exit_flush_lazy_tlbs(mm); goto is_local; } } local = false; full = (end == TLB_FLUSH_ALL || nr_pages > tlb_single_page_flush_ceiling); } else { is_local: local = true; full = (end == TLB_FLUSH_ALL || nr_pages > tlb_local_single_page_flush_ceiling); } if (full) { if (local) { _tlbiel_pid(pid, RIC_FLUSH_TLB); } else { if (mm_needs_flush_escalation(mm)) _tlbie_pid(pid, RIC_FLUSH_ALL); else _tlbie_pid(pid, RIC_FLUSH_TLB); } } else { bool hflush = flush_all_sizes; bool gflush = flush_all_sizes; unsigned long hstart, hend; unsigned long gstart, gend; if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) hflush = true; if (hflush) { hstart = (start + PMD_SIZE - 1) & PMD_MASK; hend = end & PMD_MASK; if (hstart == hend) hflush = false; } if (gflush) { gstart = (start + PUD_SIZE - 1) & PUD_MASK; gend = end & PUD_MASK; if (gstart == gend) gflush = false; } asm volatile("ptesync": : :"memory"); if (local) { __tlbiel_va_range(start, end, pid, page_size, mmu_virtual_psize); if (hflush) __tlbiel_va_range(hstart, hend, pid, PMD_SIZE, MMU_PAGE_2M); if (gflush) __tlbiel_va_range(gstart, gend, pid, PUD_SIZE, MMU_PAGE_1G); asm volatile("ptesync": : :"memory"); } else { __tlbie_va_range(start, end, pid, page_size, mmu_virtual_psize); if (hflush) __tlbie_va_range(hstart, hend, pid, PMD_SIZE, MMU_PAGE_2M); if (gflush) __tlbie_va_range(gstart, gend, pid, PUD_SIZE, MMU_PAGE_1G); fixup_tlbie(); asm volatile("eieio; tlbsync; ptesync": : :"memory"); } }
void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) { struct mm_struct *mm = vma->vm_mm; unsigned long pid; unsigned int page_shift = mmu_psize_defs[mmu_virtual_psize].shift; unsigned long page_size = 1UL << page_shift; unsigned long nr_pages = (end - start) >> page_shift; bool local, full; #ifdef CONFIG_HUGETLB_PAGE if (is_vm_hugetlb_page(vma)) return radix__flush_hugetlb_tlb_range(vma, start, end); #endif pid = mm->context.id; if (unlikely(pid == MMU_NO_CONTEXT)) return; preempt_disable(); if (mm_is_thread_local(mm)) { local = true; full = (end == TLB_FLUSH_ALL || nr_pages > tlb_local_single_page_flush_ceiling); } else { local = false; full = (end == TLB_FLUSH_ALL || nr_pages > tlb_single_page_flush_ceiling); } if (full) { if (local) { _tlbiel_pid(pid, RIC_FLUSH_TLB); } else { if (mm_needs_flush_escalation(mm)) _tlbie_pid(pid, RIC_FLUSH_ALL); else _tlbie_pid(pid, RIC_FLUSH_TLB); } } else { bool hflush = false; unsigned long hstart, hend; #ifdef CONFIG_TRANSPARENT_HUGEPAGE hstart = (start + HPAGE_PMD_SIZE - 1) >> HPAGE_PMD_SHIFT; hend = end >> HPAGE_PMD_SHIFT; if (hstart < hend) { hstart <<= HPAGE_PMD_SHIFT; hend <<= HPAGE_PMD_SHIFT; hflush = true; } #endif asm volatile("ptesync": : :"memory"); if (local) { __tlbiel_va_range(start, end, pid, page_size, mmu_virtual_psize); if (hflush) __tlbiel_va_range(hstart, hend, pid, HPAGE_PMD_SIZE, MMU_PAGE_2M); asm volatile("ptesync": : :"memory"); } else { __tlbie_va_range(start, end, pid, page_size, mmu_virtual_psize); if (hflush) __tlbie_va_range(hstart, hend, pid, HPAGE_PMD_SIZE, MMU_PAGE_2M); fixup_tlbie(); asm volatile("eieio; tlbsync; ptesync": : :"memory"); } }