void flush_tlb_pmd_range(struct mm_struct *mm, pmd_t *pmd, unsigned long addr) { pte_t *pte; pte_t *start_pte; unsigned long flags; addr = _ALIGN_DOWN(addr, PMD_SIZE); /* Note: Normally, we should only ever use a batch within a * PTE locked section. This violates the rule, but will work * since we don't actually modify the PTEs, we just flush the * hash while leaving the PTEs intact (including their reference * to being hashed). This is not the most performance oriented * way to do things but is fine for our needs here. */ local_irq_save(flags); arch_enter_lazy_mmu_mode(); start_pte = pte_offset_map(pmd, addr); for (pte = start_pte; pte < start_pte + PTRS_PER_PTE; pte++) { unsigned long pteval = pte_val(*pte); if (pteval & _PAGE_HASHPTE) hpte_need_flush(mm, addr, pte, pteval, 0); addr += PAGE_SIZE; } arch_leave_lazy_mmu_mode(); local_irq_restore(flags); }
void __flush_hash_table_range(struct mm_struct *mm, unsigned long start, unsigned long end) { unsigned long flags; start = _ALIGN_DOWN(start, PAGE_SIZE); end = _ALIGN_UP(end, PAGE_SIZE); BUG_ON(!mm->pgd); /* Note: Normally, we should only ever use a batch within a * PTE locked section. This violates the rule, but will work * since we don't actually modify the PTEs, we just flush the * hash while leaving the PTEs intact (including their reference * to being hashed). This is not the most performance oriented * way to do things but is fine for our needs here. */ local_irq_save(flags); arch_enter_lazy_mmu_mode(); for (; start < end; start += PAGE_SIZE) { pte_t *ptep = find_linux_pte(mm->pgd, start); unsigned long pte; if (ptep == NULL) continue; pte = pte_val(*ptep); if (!(pte & _PAGE_HASHPTE)) continue; hpte_need_flush(mm, start, ptep, pte, 0); } arch_leave_lazy_mmu_mode(); local_irq_restore(flags); }
int gnttab_unmap_refs(struct gnttab_unmap_grant_ref *unmap_ops, struct gnttab_map_grant_ref *kmap_ops, struct page **pages, unsigned int count) { int i, ret; bool lazy = false; ret = HYPERVISOR_grant_table_op(GNTTABOP_unmap_grant_ref, unmap_ops, count); if (ret) return ret; if (xen_feature(XENFEAT_auto_translated_physmap)) return ret; if (!in_interrupt() && paravirt_get_lazy_mode() == PARAVIRT_LAZY_NONE) { arch_enter_lazy_mmu_mode(); lazy = true; } for (i = 0; i < count; i++) { ret = m2p_remove_override(pages[i], kmap_ops ? &kmap_ops[i] : NULL); if (ret) return ret; } if (lazy) arch_leave_lazy_mmu_mode(); return ret; }
int set_foreign_p2m_mapping(struct gnttab_map_grant_ref *map_ops, struct gnttab_map_grant_ref *kmap_ops, struct page **pages, unsigned int count) { int i, ret = 0; bool lazy = false; pte_t *pte; if (xen_feature(XENFEAT_auto_translated_physmap)) return 0; if (kmap_ops && !in_interrupt() && paravirt_get_lazy_mode() == PARAVIRT_LAZY_NONE) { arch_enter_lazy_mmu_mode(); lazy = true; } for (i = 0; i < count; i++) { unsigned long mfn, pfn; /* Do not add to override if the map failed. */ if (map_ops[i].status) continue; if (map_ops[i].flags & GNTMAP_contains_pte) { pte = (pte_t *)(mfn_to_virt(PFN_DOWN(map_ops[i].host_addr)) + (map_ops[i].host_addr & ~PAGE_MASK)); mfn = pte_mfn(*pte); } else { mfn = PFN_DOWN(map_ops[i].dev_bus_addr); } pfn = page_to_pfn(pages[i]); WARN_ON(PagePrivate(pages[i])); SetPagePrivate(pages[i]); set_page_private(pages[i], mfn); pages[i]->index = pfn_to_mfn(pfn); if (unlikely(!set_phys_to_machine(pfn, FOREIGN_FRAME(mfn)))) { ret = -ENOMEM; goto out; } if (kmap_ops) { ret = m2p_add_override(mfn, pages[i], &kmap_ops[i]); if (ret) goto out; } } out: if (lazy) arch_leave_lazy_mmu_mode(); return ret; }
void paravirt_start_context_switch(struct task_struct *prev) { BUG_ON(preemptible()); if (this_cpu_read(paravirt_lazy_mode) == PARAVIRT_LAZY_MMU) { arch_leave_lazy_mmu_mode(); set_ti_thread_flag(task_thread_info(prev), TIF_LAZY_MMU_UPDATES); } enter_lazy(PARAVIRT_LAZY_CPU); }
void paravirt_flush_lazy_mmu(void) { preempt_disable(); if (paravirt_get_lazy_mode() == PARAVIRT_LAZY_MMU) { arch_leave_lazy_mmu_mode(); arch_enter_lazy_mmu_mode(); } preempt_enable(); }
void arch_flush_lazy_mmu_mode(void) { preempt_disable(); if (paravirt_get_lazy_mode() == PARAVIRT_LAZY_MMU) { WARN_ON(preempt_count() == 1); arch_leave_lazy_mmu_mode(); arch_enter_lazy_mmu_mode(); } preempt_enable(); }
int gnttab_map_refs(struct gnttab_map_grant_ref *map_ops, struct gnttab_map_grant_ref *kmap_ops, struct page **pages, unsigned int count) { int i, ret; bool lazy = false; pte_t *pte; unsigned long mfn; ret = HYPERVISOR_grant_table_op(GNTTABOP_map_grant_ref, map_ops, count); if (ret) return ret; /* Retry eagain maps */ for (i = 0; i < count; i++) if (map_ops[i].status == GNTST_eagain) gnttab_retry_eagain_gop(GNTTABOP_map_grant_ref, map_ops + i, &map_ops[i].status, __func__); if (xen_feature(XENFEAT_auto_translated_physmap)) return ret; if (!in_interrupt() && paravirt_get_lazy_mode() == PARAVIRT_LAZY_NONE) { arch_enter_lazy_mmu_mode(); lazy = true; } for (i = 0; i < count; i++) { /* Do not add to override if the map failed. */ if (map_ops[i].status) continue; if (map_ops[i].flags & GNTMAP_contains_pte) { pte = (pte_t *) (mfn_to_virt(PFN_DOWN(map_ops[i].host_addr)) + (map_ops[i].host_addr & ~PAGE_MASK)); mfn = pte_mfn(*pte); } else { mfn = PFN_DOWN(map_ops[i].dev_bus_addr); } ret = m2p_add_override(mfn, pages[i], kmap_ops ? &kmap_ops[i] : NULL); if (ret) goto out; } out: if (lazy) arch_leave_lazy_mmu_mode(); return ret; }
int clear_foreign_p2m_mapping(struct gnttab_unmap_grant_ref *unmap_ops, struct gnttab_map_grant_ref *kmap_ops, struct page **pages, unsigned int count) { int i, ret = 0; bool lazy = false; if (xen_feature(XENFEAT_auto_translated_physmap)) return 0; if (kmap_ops && !in_interrupt() && paravirt_get_lazy_mode() == PARAVIRT_LAZY_NONE) { arch_enter_lazy_mmu_mode(); lazy = true; } for (i = 0; i < count; i++) { unsigned long mfn = __pfn_to_mfn(page_to_pfn(pages[i])); unsigned long pfn = page_to_pfn(pages[i]); if (mfn == INVALID_P2M_ENTRY || !(mfn & FOREIGN_FRAME_BIT)) { ret = -EINVAL; goto out; } set_page_private(pages[i], INVALID_P2M_ENTRY); WARN_ON(!PagePrivate(pages[i])); ClearPagePrivate(pages[i]); set_phys_to_machine(pfn, pages[i]->index); if (kmap_ops) ret = m2p_remove_override(pages[i], &kmap_ops[i], mfn); if (ret) goto out; } out: if (lazy) arch_leave_lazy_mmu_mode(); return ret; }