void flush_tlb_mm(struct mm_struct *mm)
{
if (tlb_ops_need_broadcast())
on_each_cpu_mask(ipi_flush_tlb_mm, mm, 1, mm_cpumask(mm));
else
local_flush_tlb_mm(mm);
}
void flush_tlb_mm(struct mm_struct *mm)
{
if (tlb_ops_need_broadcast())
on_each_cpu_mask(mm_cpumask(mm), ipi_flush_tlb_mm, mm, 1);
else
local_flush_tlb_mm(mm);
broadcast_tlb_mm_a15_erratum(mm);
}
void
smp_flush_tlb_mm (struct mm_struct *mm)
{
local_flush_tlb_mm(mm);
/* this happens for the common case of a single-threaded fork(): */
if (likely(mm == current->active_mm && atomic_read(&mm->mm_users) == 1))
return;
smp_call_function((void (*)(void *))local_flush_tlb_mm, mm, 1, 1);
}
void smp_flush_tlb_mm(struct mm_struct *mm)
{
if(mm->context != NO_CONTEXT) {
if(mm->cpu_vm_mask != (1 << smp_processor_id())) {
xc1((smpfunc_t) BTFIXUP_CALL(local_flush_tlb_mm), (unsigned long) mm);
if(atomic_read(&mm->mm_users) == 1 && current->active_mm == mm)
mm->cpu_vm_mask = (1 << smp_processor_id());
}
local_flush_tlb_mm(mm);
}
}
void flush_tlb_mm(struct mm_struct *mm)
{
if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) {
smp_call_function(flush_tlb_mm_ipi, (void *)mm, 1, 1);
} else {
int i;
for (i = 0; i < smp_num_cpus; i++)
if (smp_processor_id() != i)
cpu_context(i, mm) = 0;
}
local_flush_tlb_mm(mm);
}
void smp_flush_tlb_mm(struct mm_struct *mm)
{
if(mm->context != NO_CONTEXT) {
cpumask_t cpu_mask = mm->cpu_vm_mask;
cpu_clear(smp_processor_id(), cpu_mask);
if (!cpus_empty(cpu_mask)) {
xc1((smpfunc_t) BTFIXUP_CALL(local_flush_tlb_mm), (unsigned long) mm);
if(atomic_read(&mm->mm_users) == 1 && current->active_mm == mm)
mm->cpu_vm_mask = cpumask_of_cpu(smp_processor_id());
}
local_flush_tlb_mm(mm);
}
}
void flush_tlb_mm(struct mm_struct *mm)
{
if (IS_ENABLED(CONFIG_L4)) {
l4x_unmap_sync_mm(mm);
l4x_del_task(mm);
return;
}
if (tlb_ops_need_broadcast())
on_each_cpu_mask(mm_cpumask(mm), ipi_flush_tlb_mm, mm, 1);
else
local_flush_tlb_mm(mm);
broadcast_tlb_mm_a15_erratum(mm);
}
void flush_tlb_mm(struct mm_struct *mm)
{
preempt_disable();
if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) {
smp_on_other_tlbs(flush_tlb_mm_ipi, (void *)mm);
} else {
int i;
for (i = 0; i < num_online_cpus(); i++)
if (smp_processor_id() != i)
cpu_context(i, mm) = 0;
}
local_flush_tlb_mm(mm);
preempt_enable();
}
/*
* The following tlb flush calls are invoked when old translations are
* being torn down, or pte attributes are changing. For single threaded
* address spaces, a new context is obtained on the current cpu, and tlb
* context on other cpus are invalidated to force a new context allocation
* at switch_mm time, should the mm ever be used on other cpus. For
* multithreaded address spaces, intercpu interrupts have to be sent.
* Another case where intercpu interrupts are required is when the target
* mm might be active on another cpu (eg debuggers doing the flushes on
* behalf of debugees, kswapd stealing pages from another process etc).
* Kanoj 07/00.
*/
void flush_tlb_mm(struct mm_struct *mm)
{
preempt_disable();
if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) {
smp_call_function(flush_tlb_mm_ipi, (void *)mm, 1);
} else {
int i;
for_each_online_cpu(i)
if (smp_processor_id() != i)
cpu_context(i, mm) = 0;
}
local_flush_tlb_mm(mm);
preempt_enable();
}
void flush_tlb_mm(struct mm_struct *mm)
{
preempt_disable();
if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) {
smp_on_other_tlbs(flush_tlb_mm_ipi, mm);
} else {
cpumask_t mask = cpu_online_map;
unsigned int cpu;
cpu_clear(smp_processor_id(), mask);
for_each_cpu_mask(cpu, mask)
if (cpu_context(cpu, mm))
cpu_context(cpu, mm) = 0;
}
local_flush_tlb_mm(mm);
preempt_enable();
}
/* Note that this will also be called on SMP if all other CPUs are
* offlined, which means that it may be called for cpu != 0. For
* this to work, we somewhat assume that CPUs that are onlined
* come up with a fully clean TLB (or are cleaned when offlined)
*/
static unsigned int steal_context_up(unsigned int id)
{
struct mm_struct *mm;
int cpu = smp_processor_id();
/* Pick up the victim mm */
mm = context_mm[id];
pr_hardcont(" | steal %d from 0x%p", id, mm);
/* Flush the TLB for that context */
local_flush_tlb_mm(mm);
/* Mark this mm has having no context anymore */
mm->context.id = MMU_NO_CONTEXT;
/* XXX This clear should ultimately be part of local_flush_tlb_mm */
__clear_bit(id, stale_map[cpu]);
return id;
}
static unsigned int steal_context_up(unsigned int id)
{
struct mm_struct *mm;
int cpu = smp_processor_id();
mm = context_mm[id];
pr_hardcont(" | steal %d from 0x%p", id, mm);
local_flush_tlb_mm(mm);
mm->context.id = MMU_NO_CONTEXT;
__clear_bit(id, stale_map[cpu]);
return id;
}
void local_flush_tlb_range(struct vm_area_struct *vma,
unsigned long start, unsigned long end)
{
int cpu = smp_processor_id();
struct mm_struct *mm = vma->vm_mm;
unsigned long flags;
if (mm->context.asid[cpu] == NO_CONTEXT)
return;
#if 0
printk("[tlbrange<%02lx,%08lx,%08lx>]\n",
(unsigned long)mm->context.asid[cpu], start, end);
#endif
local_irq_save(flags);
if (end-start + (PAGE_SIZE-1) <= _TLB_ENTRIES << PAGE_SHIFT) {
int oldpid = get_rasid_register();
set_rasid_register(ASID_INSERT(mm->context.asid[cpu]));
start &= PAGE_MASK;
if (vma->vm_flags & VM_EXEC)
while(start < end) {
invalidate_itlb_mapping(start);
invalidate_dtlb_mapping(start);
start += PAGE_SIZE;
}
else
while(start < end) {
invalidate_dtlb_mapping(start);
start += PAGE_SIZE;
}
set_rasid_register(oldpid);
} else {
local_flush_tlb_mm(mm);
}
local_irq_restore(flags);
}
/*
* Flush a Range of TLB entries for userland.
* @start is inclusive, while @end is exclusive
* Difference between this and Kernel Range Flush is
* -Here the fastest way (if range is too large) is to move to next ASID
* without doing any explicit Shootdown
* -In case of kernel Flush, entry has to be shot down explictly
*/
void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
unsigned long end)
{
const unsigned int cpu = smp_processor_id();
unsigned long flags;
/* If range @start to @end is more than 32 TLB entries deep,
* its better to move to a new ASID rather than searching for
* individual entries and then shooting them down
*
* The calc above is rough, doesn't account for unaligned parts,
* since this is heuristics based anyways
*/
if (unlikely((end - start) >= PAGE_SIZE * 32)) {
local_flush_tlb_mm(vma->vm_mm);
return;
}
/*
* @start moved to page start: this alone suffices for checking
* loop end condition below, w/o need for aligning @end to end
* e.g. 2000 to 4001 will anyhow loop twice
*/
start &= PAGE_MASK;
local_irq_save(flags);
if (asid_mm(vma->vm_mm, cpu) != MM_CTXT_NO_ASID) {
while (start < end) {
tlb_entry_erase(start | hw_pid(vma->vm_mm, cpu));
start += PAGE_SIZE;
}
}
utlb_invalidate();
local_irq_restore(flags);
}
void switch_mmu_context(struct mm_struct *prev, struct mm_struct *next)
{
unsigned int id, cpu = smp_processor_id();
unsigned long *map;
/* No lockless fast path .. yet */
atomic_spin_lock(&context_lock);
#ifndef DEBUG_STEAL_ONLY
pr_devel("[%d] activating context for mm @%p, active=%d, id=%d\n",
cpu, next, next->context.active, next->context.id);
#endif
#ifdef CONFIG_SMP
/* Mark us active and the previous one not anymore */
next->context.active++;
if (prev) {
#ifndef DEBUG_STEAL_ONLY
pr_devel(" old context %p active was: %d\n",
prev, prev->context.active);
#endif
WARN_ON(prev->context.active < 1);
prev->context.active--;
}
again:
#endif /* CONFIG_SMP */
/* If we already have a valid assigned context, skip all that */
id = next->context.id;
if (likely(id != MMU_NO_CONTEXT))
goto ctxt_ok;
/* We really don't have a context, let's try to acquire one */
id = next_context;
if (id > last_context)
id = first_context;
map = context_map;
/* No more free contexts, let's try to steal one */
if (nr_free_contexts == 0) {
#ifdef CONFIG_SMP
if (num_online_cpus() > 1) {
id = steal_context_smp(id);
if (id == MMU_NO_CONTEXT)
goto again;
goto stolen;
}
#endif /* CONFIG_SMP */
id = steal_context_up(id);
goto stolen;
}
nr_free_contexts--;
/* We know there's at least one free context, try to find it */
while (__test_and_set_bit(id, map)) {
id = find_next_zero_bit(map, last_context+1, id);
if (id > last_context)
id = first_context;
}
stolen:
next_context = id + 1;
context_mm[id] = next;
next->context.id = id;
#ifndef DEBUG_STEAL_ONLY
pr_devel("[%d] picked up new id %d, nrf is now %d\n",
cpu, id, nr_free_contexts);
#endif
context_check_map();
ctxt_ok:
/* If that context got marked stale on this CPU, then flush the
* local TLB for it and unmark it before we use it
*/
if (test_bit(id, stale_map[cpu])) {
pr_devel("[%d] flushing stale context %d for mm @%p !\n",
cpu, id, next);
local_flush_tlb_mm(next);
/* XXX This clear should ultimately be part of local_flush_tlb_mm */
__clear_bit(id, stale_map[cpu]);
}
/* Flick the MMU and release lock */
set_context(id, next->pgd);
atomic_spin_unlock(&context_lock);
}
static void flush_tlb_mm_ipi(void *mm)
{
local_flush_tlb_mm((struct mm_struct *)mm);
}
void switch_mmu_context(struct mm_struct *prev, struct mm_struct *next)
{
unsigned int i, id, cpu = smp_processor_id();
unsigned long *map;
raw_spin_lock(&context_lock);
pr_hard("[%d] activating context for mm @%p, active=%d, id=%d",
cpu, next, next->context.active, next->context.id);
#ifdef CONFIG_SMP
next->context.active++;
if (prev) {
pr_hardcont(" (old=0x%p a=%d)", prev, prev->context.active);
WARN_ON(prev->context.active < 1);
prev->context.active--;
}
again:
#endif
id = next->context.id;
if (likely(id != MMU_NO_CONTEXT)) {
#ifdef DEBUG_MAP_CONSISTENCY
if (context_mm[id] != next)
pr_err("MMU: mm 0x%p has id %d but context_mm[%d] says 0x%p\n",
next, id, id, context_mm[id]);
#endif
goto ctxt_ok;
}
id = next_context;
if (id > last_context)
id = first_context;
map = context_map;
if (nr_free_contexts == 0) {
#ifdef CONFIG_SMP
if (num_online_cpus() > 1) {
id = steal_context_smp(id);
if (id == MMU_NO_CONTEXT)
goto again;
goto stolen;
}
#endif
id = steal_context_up(id);
goto stolen;
}
nr_free_contexts--;
while (__test_and_set_bit(id, map)) {
id = find_next_zero_bit(map, last_context+1, id);
if (id > last_context)
id = first_context;
}
stolen:
next_context = id + 1;
context_mm[id] = next;
next->context.id = id;
pr_hardcont(" | new id=%d,nrf=%d", id, nr_free_contexts);
context_check_map();
ctxt_ok:
if (test_bit(id, stale_map[cpu])) {
pr_hardcont(" | stale flush %d [%d..%d]",
id, cpu_first_thread_sibling(cpu),
cpu_last_thread_sibling(cpu));
local_flush_tlb_mm(next);
for (i = cpu_first_thread_sibling(cpu);
i <= cpu_last_thread_sibling(cpu); i++) {
__clear_bit(id, stale_map[i]);
}
}
pr_hardcont(" -> %d\n", id);
set_context(id, next->pgd);
raw_spin_unlock(&context_lock);
}
static inline void ipi_flush_tlb_mm(void *arg)
{
struct mm_struct *mm = (struct mm_struct *)arg;
local_flush_tlb_mm(mm);
}
void switch_mmu_context(struct mm_struct *prev, struct mm_struct *next,
struct task_struct *tsk)
{
unsigned int i, id, cpu = smp_processor_id();
unsigned long *map;
/* No lockless fast path .. yet */
raw_spin_lock(&context_lock);
pr_hard("[%d] activating context for mm @%p, active=%d, id=%d",
cpu, next, next->context.active, next->context.id);
#ifdef CONFIG_SMP
/* Mark us active and the previous one not anymore */
next->context.active++;
if (prev) {
pr_hardcont(" (old=0x%p a=%d)", prev, prev->context.active);
WARN_ON(prev->context.active < 1);
prev->context.active--;
}
again:
#endif /* CONFIG_SMP */
/* If we already have a valid assigned context, skip all that */
id = next->context.id;
if (likely(id != MMU_NO_CONTEXT)) {
#ifdef DEBUG_MAP_CONSISTENCY
if (context_mm[id] != next)
pr_err("MMU: mm 0x%p has id %d but context_mm[%d] says 0x%p\n",
next, id, id, context_mm[id]);
#endif
goto ctxt_ok;
}
/* We really don't have a context, let's try to acquire one */
id = next_context;
if (id > last_context)
id = first_context;
map = context_map;
/* No more free contexts, let's try to steal one */
if (nr_free_contexts == 0) {
#ifdef CONFIG_SMP
if (num_online_cpus() > 1) {
id = steal_context_smp(id);
if (id == MMU_NO_CONTEXT)
goto again;
goto stolen;
}
#endif /* CONFIG_SMP */
if (no_selective_tlbil)
id = steal_all_contexts();
else
id = steal_context_up(id);
goto stolen;
}
nr_free_contexts--;
/* We know there's at least one free context, try to find it */
while (__test_and_set_bit(id, map)) {
id = find_next_zero_bit(map, last_context+1, id);
if (id > last_context)
id = first_context;
}
stolen:
next_context = id + 1;
context_mm[id] = next;
next->context.id = id;
pr_hardcont(" | new id=%d,nrf=%d", id, nr_free_contexts);
context_check_map();
ctxt_ok:
/* If that context got marked stale on this CPU, then flush the
* local TLB for it and unmark it before we use it
*/
if (test_bit(id, stale_map[cpu])) {
pr_hardcont(" | stale flush %d [%d..%d]",
id, cpu_first_thread_sibling(cpu),
cpu_last_thread_sibling(cpu));
local_flush_tlb_mm(next);
/* XXX This clear should ultimately be part of local_flush_tlb_mm */
for (i = cpu_first_thread_sibling(cpu);
i <= cpu_last_thread_sibling(cpu); i++) {
if (stale_map[i])
__clear_bit(id, stale_map[i]);
}
}
/* Flick the MMU and release lock */
pr_hardcont(" -> %d\n", id);
set_context(id, next->pgd);
raw_spin_unlock(&context_lock);
}
