/* * Associate a virtual page frame with a given physical page frame * and protection flags for that frame. */ static void set_pte_pfn(unsigned long vaddr, unsigned long pfn, pgprot_t flags) { pgd_t *pgd; pud_t *pud; pmd_t *pmd; pte_t *pte; pgd = swapper_pg_dir + pgd_index(vaddr); if (pgd_none(*pgd)) { BUG(); return; } pud = pud_offset(pgd, vaddr); if (pud_none(*pud)) { BUG(); return; } pmd = pmd_offset(pud, vaddr); if (pmd_none(*pmd)) { BUG(); return; } pte = pte_offset_kernel(pmd, vaddr); /* <pfn,flags> stored as-is, to permit clearing entries */ set_pte(pte, pfn_pte(pfn, flags)); /* * It's enough to flush this one mapping. * This appears conservative since it is only called * from __set_fixmap. */ local_flush_tlb_page(NULL, vaddr, PAGE_SIZE); }
void __kunmap_atomic(void *kvaddr) { unsigned long vaddr = (unsigned long) kvaddr & PAGE_MASK; int type; if (vaddr < __fix_to_virt(FIX_KMAP_END)) { pagefault_enable(); return; } type = kmap_atomic_idx(); #ifdef CONFIG_DEBUG_HIGHMEM { unsigned int idx; idx = type + KM_TYPE_NR * smp_processor_id(); BUG_ON(vaddr != __fix_to_virt(FIX_KMAP_BEGIN + idx)); /* * force other mappings to Oops if they'll try to access * this pte without first remap it */ pte_clear(&init_mm, vaddr, kmap_pte-idx); local_flush_tlb_page(NULL, vaddr); } #endif kmap_atomic_idx_pop(); pagefault_enable(); }
/** * smp_flush_tlb - Callback to invalidate the TLB. * @unused: Callback context (ignored). */ void smp_flush_tlb(void *unused) { unsigned long cpu_id; cpu_id = get_cpu(); if (!cpumask_test_cpu(cpu_id, &flush_cpumask)) /* This was a BUG() but until someone can quote me the line * from the intel manual that guarantees an IPI to multiple * CPUs is retried _only_ on the erroring CPUs its staying as a * return * * BUG(); */ goto out; if (flush_va == FLUSH_ALL) local_flush_tlb(); else local_flush_tlb_page(flush_mm, flush_va); smp_mb__before_clear_bit(); cpumask_clear_cpu(cpu_id, &flush_cpumask); smp_mb__after_clear_bit(); out: put_cpu(); }
void flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr) { if (tlb_ops_need_broadcast()) { struct tlb_args ta; ta.ta_vma = vma; ta.ta_start = uaddr; on_each_cpu_mask(ipi_flush_tlb_page, &ta, 1, mm_cpumask(vma->vm_mm)); } else local_flush_tlb_page(vma, uaddr); }
void smp_flush_tlb_page(struct vm_area_struct *vma, unsigned long page) { struct mm_struct *mm = vma->vm_mm; if(mm->context != NO_CONTEXT) { if(mm->cpu_vm_mask != (1 << smp_processor_id())) xc2((smpfunc_t) BTFIXUP_CALL(local_flush_tlb_page), (unsigned long) vma, page); local_flush_tlb_page(vma, page); } }
void smp_flush_tlb_page(struct vm_area_struct *vma, unsigned long page) { struct mm_struct *mm = vma->vm_mm; if(mm->context != NO_CONTEXT) { cpumask_t cpu_mask = mm->cpu_vm_mask; cpu_clear(smp_processor_id(), cpu_mask); if (!cpus_empty(cpu_mask)) xc2((smpfunc_t) BTFIXUP_CALL(local_flush_tlb_page), (unsigned long) vma, page); local_flush_tlb_page(vma, page); } }
/** * flush_tlb_page - Invalidate TLB of page * @vma: The VM context to invalidate the page for. * @va: The virtual address of the page to invalidate. */ void flush_tlb_page(struct vm_area_struct *vma, unsigned long va) { struct mm_struct *mm = vma->vm_mm; cpumask_t cpu_mask; preempt_disable(); cpumask_copy(&cpu_mask, mm_cpumask(mm)); cpumask_clear_cpu(smp_processor_id(), &cpu_mask); local_flush_tlb_page(mm, va); if (!cpumask_empty(&cpu_mask)) flush_tlb_others(cpu_mask, mm, va); preempt_enable(); }
void flush_tlb_page(struct vm_area_struct *vma, unsigned long page) { if ((atomic_read(&vma->vm_mm->mm_users) != 1) || (current->mm != vma->vm_mm)) { struct flush_tlb_data fd; fd.vma = vma; fd.addr1 = page; smp_call_function(flush_tlb_page_ipi, (void *)&fd, 1, 1); } else { int i; for (i = 0; i < smp_num_cpus; i++) if (smp_processor_id() != i) cpu_context(i, vma->vm_mm) = 0; } local_flush_tlb_page(vma, page); }
void flush_tlb_page(struct vm_area_struct *vma, unsigned long page) { preempt_disable(); if ((atomic_read(&vma->vm_mm->mm_users) != 1) || (current->mm != vma->vm_mm)) { struct flush_tlb_data fd; fd.vma = vma; fd.addr1 = page; smp_on_other_tlbs(flush_tlb_page_ipi, (void *)&fd); } else { int i; for (i = 0; i < num_online_cpus(); i++) if (smp_processor_id() != i) cpu_context(i, vma->vm_mm) = 0; } local_flush_tlb_page(vma, page); preempt_enable(); }
void flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr) { if (IS_ENABLED(CONFIG_L4)) { l4x_unmap_sync_mm(vma->vm_mm); l4x_unmap_page(vma->vm_mm, uaddr); return; } if (tlb_ops_need_broadcast()) { struct tlb_args ta; ta.ta_vma = vma; ta.ta_start = uaddr; on_each_cpu_mask(mm_cpumask(vma->vm_mm), ipi_flush_tlb_page, &ta, 1); } else local_flush_tlb_page(vma, uaddr); broadcast_tlb_mm_a15_erratum(vma->vm_mm); }
/* * The use of kmap_atomic/kunmap_atomic is discouraged - kmap/kunmap * gives a more generic (and caching) interface. But kmap_atomic can * be used in IRQ contexts, so in some (very limited) cases we need * it. */ void *kmap_atomic_prot(struct page *page, pgprot_t prot) { unsigned long vaddr; int idx, type; /* even !CONFIG_PREEMPT needs this, for in_atomic in do_page_fault */ pagefault_disable(); if (!PageHighMem(page)) return page_address(page); type = kmap_atomic_idx_push(); idx = type + KM_TYPE_NR*smp_processor_id(); vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx); #ifdef CONFIG_DEBUG_HIGHMEM BUG_ON(!pte_none(*(kmap_pte-idx))); #endif __set_pte_at(&init_mm, vaddr, kmap_pte-idx, mk_pte(page, prot), 1); local_flush_tlb_page(NULL, vaddr); return (void*) vaddr; }
void kunmap_atomic_notypecheck(void *kvaddr, enum km_type type) { #ifdef CONFIG_DEBUG_HIGHMEM unsigned long vaddr = (unsigned long) kvaddr & PAGE_MASK; enum fixed_addresses idx = type + KM_TYPE_NR*smp_processor_id(); if (vaddr < __fix_to_virt(FIX_KMAP_END)) { pagefault_enable(); return; } BUG_ON(vaddr != __fix_to_virt(FIX_KMAP_BEGIN + idx)); /* * force other mappings to Oops if they'll try to access * this pte without first remap it */ pte_clear(&init_mm, vaddr, kmap_pte-idx); local_flush_tlb_page(NULL, vaddr); #endif pagefault_enable(); }
/* * Copy memory by briefly enabling incoherent cacheline-at-a-time mode. * * We set up our own source and destination PTEs that we fully control. * This is the only way to guarantee that we don't race with another * thread that is modifying the PTE; we can't afford to try the * copy_{to,from}_user() technique of catching the interrupt, since * we must run with interrupts disabled to avoid the risk of some * other code seeing the incoherent data in our cache. (Recall that * our cache is indexed by PA, so even if the other code doesn't use * our kmap_atomic virtual addresses, they'll still hit in cache using * the normal VAs that aren't supposed to hit in cache.) */ static void memcpy_multicache(void *dest, const void *source, pte_t dst_pte, pte_t src_pte, int len) { int idx; unsigned long flags, newsrc, newdst; pmd_t *pmdp; pte_t *ptep; int type0, type1; int cpu = get_cpu(); /* * Disable interrupts so that we don't recurse into memcpy() * in an interrupt handler, nor accidentally reference * the PA of the source from an interrupt routine. Also * notify the simulator that we're playing games so we don't * generate spurious coherency warnings. */ local_irq_save(flags); sim_allow_multiple_caching(1); /* Set up the new dest mapping */ type0 = kmap_atomic_idx_push(); idx = FIX_KMAP_BEGIN + (KM_TYPE_NR * cpu) + type0; newdst = __fix_to_virt(idx) + ((unsigned long)dest & (PAGE_SIZE-1)); pmdp = pmd_offset(pud_offset(pgd_offset_k(newdst), newdst), newdst); ptep = pte_offset_kernel(pmdp, newdst); if (pte_val(*ptep) != pte_val(dst_pte)) { set_pte(ptep, dst_pte); local_flush_tlb_page(NULL, newdst, PAGE_SIZE); } /* Set up the new source mapping */ type1 = kmap_atomic_idx_push(); idx += (type0 - type1); src_pte = hv_pte_set_nc(src_pte); src_pte = hv_pte_clear_writable(src_pte); /* be paranoid */ newsrc = __fix_to_virt(idx) + ((unsigned long)source & (PAGE_SIZE-1)); pmdp = pmd_offset(pud_offset(pgd_offset_k(newsrc), newsrc), newsrc); ptep = pte_offset_kernel(pmdp, newsrc); __set_pte(ptep, src_pte); /* set_pte() would be confused by this */ local_flush_tlb_page(NULL, newsrc, PAGE_SIZE); /* Actually move the data. */ __memcpy_asm((void *)newdst, (const void *)newsrc, len); /* * Remap the source as locally-cached and not OLOC'ed so that * we can inval without also invaling the remote cpu's cache. * This also avoids known errata with inv'ing cacheable oloc data. */ src_pte = hv_pte_set_mode(src_pte, HV_PTE_MODE_CACHE_NO_L3); src_pte = hv_pte_set_writable(src_pte); /* need write access for inv */ __set_pte(ptep, src_pte); /* set_pte() would be confused by this */ local_flush_tlb_page(NULL, newsrc, PAGE_SIZE); /* * Do the actual invalidation, covering the full L2 cache line * at the end since __memcpy_asm() is somewhat aggressive. */ __inv_buffer((void *)newsrc, len); /* * We're done: notify the simulator that all is back to normal, * and re-enable interrupts and pre-emption. */ kmap_atomic_idx_pop(); kmap_atomic_idx_pop(); sim_allow_multiple_caching(0); local_irq_restore(flags); put_cpu(); }
static void flush_tlb_page_ipi(void *info) { struct flush_tlb_data *fd = (struct flush_tlb_data *)info; local_flush_tlb_page(fd->vma, fd->addr1); }
static void memcpy_multicache(void *dest, const void *source, pte_t dst_pte, pte_t src_pte, int len) { int idx; unsigned long flags, newsrc, newdst; pmd_t *pmdp; pte_t *ptep; int type0, type1; int cpu = get_cpu(); /* */ local_irq_save(flags); sim_allow_multiple_caching(1); /* */ type0 = kmap_atomic_idx_push(); idx = FIX_KMAP_BEGIN + (KM_TYPE_NR * cpu) + type0; newdst = __fix_to_virt(idx) + ((unsigned long)dest & (PAGE_SIZE-1)); pmdp = pmd_offset(pud_offset(pgd_offset_k(newdst), newdst), newdst); ptep = pte_offset_kernel(pmdp, newdst); if (pte_val(*ptep) != pte_val(dst_pte)) { set_pte(ptep, dst_pte); local_flush_tlb_page(NULL, newdst, PAGE_SIZE); } /* */ type1 = kmap_atomic_idx_push(); idx += (type0 - type1); src_pte = hv_pte_set_nc(src_pte); src_pte = hv_pte_clear_writable(src_pte); /* */ newsrc = __fix_to_virt(idx) + ((unsigned long)source & (PAGE_SIZE-1)); pmdp = pmd_offset(pud_offset(pgd_offset_k(newsrc), newsrc), newsrc); ptep = pte_offset_kernel(pmdp, newsrc); __set_pte(ptep, src_pte); /* */ local_flush_tlb_page(NULL, newsrc, PAGE_SIZE); /* */ __memcpy_asm((void *)newdst, (const void *)newsrc, len); /* */ src_pte = hv_pte_set_mode(src_pte, HV_PTE_MODE_CACHE_NO_L3); src_pte = hv_pte_set_writable(src_pte); /* */ __set_pte(ptep, src_pte); /* */ local_flush_tlb_page(NULL, newsrc, PAGE_SIZE); /* */ __inv_buffer((void *)newsrc, len); /* */ kmap_atomic_idx_pop(); kmap_atomic_idx_pop(); sim_allow_multiple_caching(0); local_irq_restore(flags); put_cpu(); }
static inline void ipi_flush_tlb_page(void *arg) { struct tlb_args *ta = (struct tlb_args *)arg; local_flush_tlb_page(ta->ta_vma, ta->ta_start); }