void _arch_exit_mmap(struct mm_struct *mm)
{
struct task_struct *tsk = current;
task_lock(tsk);
/*
* We aggressively remove defunct pgd from cr3. We execute unmap_vmas()
* *much* faster this way, as no tlb flushes means bigger wrpt batches.
*/
if (tsk->active_mm == mm) {
tsk->active_mm = &init_mm;
atomic_inc(&init_mm.mm_count);
switch_mm(mm, &init_mm, tsk);
atomic_dec(&mm->mm_count);
BUG_ON(atomic_read(&mm->mm_count) == 0);
}
task_unlock(tsk);
if (test_bit(PG_pinned, &virt_to_page(mm->pgd)->flags) &&
(atomic_read(&mm->mm_count) == 1))
mm_unpin(mm);
}
/*
* Makes the calling thread switch to/from efi_mm context. Can be used
* for SetVirtualAddressMap() i.e. current->active_mm == init_mm as well
* as during efi runtime calls i.e current->active_mm == current_mm.
* We are not mm_dropping()/mm_grabbing() any mm, because we are not
* losing/creating any references.
*/
void efi_switch_mm(struct mm_struct *mm)
{
task_lock(current);
efi_scratch.prev_mm = current->active_mm;
current->active_mm = mm;
switch_mm(efi_scratch.prev_mm, mm, NULL);
task_unlock(current);
}
/**
* Switch to the new mm_struct and restore register/stack state to next.
* @prev: previously running (current) task
* @next: task to switch to
*/
static inline void __attribute__((always_inline)) context_switch(struct task_struct *prev, struct task_struct *next) {
struct mm_struct *mm, *prev_mm;
mm = next->mm;
prev_mm = prev->mm;
switch_mm(prev_mm, mm);
switch_to(prev, next);
/* next is now the current task */
}
/* Ensure we run on the idle task page tables so that we will
switch page tables before running user space. This is needed
on architectures with separate kernel and user page tables
because the user page table pointer is not saved/restored. */
static void switch_idle_mm(void)
{
struct mm_struct *mm = current->active_mm;
if (mm == &init_mm)
return;
atomic_inc(&init_mm.mm_count);
switch_mm(mm, &init_mm, current);
current->active_mm = &init_mm;
mmdrop(mm);
}
static inline void
context_switch(struct rq *rq, struct task_struct *prev,
struct task_struct *next)
{
struct mm_struct *mm, *oldmm;
prepare_task_switch(rq, prev, next);
mm = next->mm;
oldmm = prev->active_mm;
/*
* For paravirt, this is coupled with an exit in switch_to to
* combine the page table reload and the switch backend into
* one hypercall.
*/
arch_start_context_switch(prev);
if (!mm) {
next->active_mm = oldmm;
atomic_inc(&oldmm->mm_count);
enter_lazy_tlb(oldmm, next);
} else
switch_mm(oldmm, mm, next);
if (!prev->mm) {
prev->active_mm = NULL;
rq->prev_mm = oldmm;
}
/*
* Since the runqueue lock will be released by the next
* task (which is an invalid locking op but in the case
* of the scheduler it's an obvious special-case), so we
* do an early lockdep release here:
*/
#ifndef __ARCH_WANT_UNLOCKED_CTXSW
spin_release(&rq->lock.dep_map, 1, _THIS_IP_);
#endif
/* Here we just switch the register state and the stack. */
switch_to(prev, next, prev);
barrier();
/*
* this_rq must be evaluated again because prev may have moved
* CPUs since it called schedule(), thus the 'rq' on its stack
* frame will be invalid.
*/
finish_task_switch(this_rq(), prev);
}
/*
* use_mm
* Makes the calling kernel thread take on the specified
* mm context.
* Called by the retry thread execute retries within the
* iocb issuer's mm context, so that copy_from/to_user
* operations work seamlessly for aio.
* (Note: this routine is intended to be called only
* from a kernel thread context)
*/
void use_mm(struct mm_struct *mm)
{
struct mm_struct *active_mm;
struct task_struct *tsk = current;
task_lock(tsk);
active_mm = tsk->active_mm;
if (active_mm != mm) {
atomic_inc(&mm->mm_count);
tsk->active_mm = mm;
}
tsk->mm = mm;
switch_mm(active_mm, mm, tsk);
task_unlock(tsk);
if (active_mm != mm)
mmdrop(active_mm);
}
void leave_mm(int cpu)
{
struct mm_struct *loaded_mm = this_cpu_read(cpu_tlbstate.loaded_mm);
/*
* It's plausible that we're in lazy TLB mode while our mm is init_mm.
* If so, our callers still expect us to flush the TLB, but there
* aren't any user TLB entries in init_mm to worry about.
*
* This needs to happen before any other sanity checks due to
* intel_idle's shenanigans.
*/
if (loaded_mm == &init_mm)
return;
/* Warn if we're not lazy. */
WARN_ON(!this_cpu_read(cpu_tlbstate.is_lazy));
switch_mm(NULL, &init_mm, NULL);
}
/*
* enter_lazy_tlb() is a hint from the scheduler that we are entering a
* kernel thread or other context without an mm. Acceptable implementations
* include doing nothing whatsoever, switching to init_mm, or various clever
* lazy tricks to try to minimize TLB flushes.
*
* The scheduler reserves the right to call enter_lazy_tlb() several times
* in a row. It will notify us that we're going back to a real mm by
* calling switch_mm_irqs_off().
*/
void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk)
{
if (this_cpu_read(cpu_tlbstate.loaded_mm) == &init_mm)
return;
if (static_branch_unlikely(&tlb_use_lazy_mode)) {
/*
* There's a significant optimization that may be possible
* here. We have accurate enough TLB flush tracking that we
* don't need to maintain coherence of TLB per se when we're
* lazy. We do, however, need to maintain coherence of
* paging-structure caches. We could, in principle, leave our
* old mm loaded and only switch to init_mm when
* tlb_remove_page() happens.
*/
this_cpu_write(cpu_tlbstate.is_lazy, true);
} else {
switch_mm(NULL, &init_mm, NULL);
}
}
static void do_exit_flush_lazy_tlb(void *arg)
{
struct mm_struct *mm = arg;
unsigned long pid = mm->context.id;
if (current->mm == mm)
return; /* Local CPU */
if (current->active_mm == mm) {
/*
* Must be a kernel thread because sender is single-threaded.
*/
BUG_ON(current->mm);
mmgrab(&init_mm);
switch_mm(mm, &init_mm, current);
current->active_mm = &init_mm;
mmdrop(mm);
}
_tlbiel_pid(pid, RIC_FLUSH_ALL);
}