/*
* uvm_emap_switch: if the CPU is 'behind' the LWP in emap visibility,
* perform TLB flush and thus update the local view. Main purpose is
* to handle kernel preemption, while emap is in use.
*
* => called from mi_switch(), when LWP returns after block or preempt.
*/
void
uvm_emap_switch(lwp_t *l)
{
struct uvm_cpu *ucpu;
u_int curgen, gen;
KASSERT(kpreempt_disabled());
/* If LWP did not use emap, then nothing to do. */
if (__predict_true(l->l_emap_gen == UVM_EMAP_INACTIVE)) {
return;
}
/*
* No need to synchronise if generation number of current CPU is
* newer than the number of this LWP.
*
* This test assumes two's complement arithmetic and allows
* ~2B missed updates before it will produce bad results.
*/
ucpu = curcpu()->ci_data.cpu_uvm;
curgen = ucpu->emap_gen;
gen = l->l_emap_gen;
if (__predict_true((signed int)(curgen - gen) >= 0)) {
return;
}
/*
* See comments in uvm_emap_consume() about memory
* barriers and race conditions.
*/
curgen = uvm_emap_gen_return();
pmap_emap_sync(false);
ucpu->emap_gen = curgen;
}
/*
* uvm_emap_consume: update the current CPU and LWP to the given generation
* of the emap. In a case of LWP migration to a different CPU after block
* or preempt, uvm_emap_switch() will synchronise.
*
* => may be called from both interrupt and thread context.
*/
void
uvm_emap_consume(u_int gen)
{
struct cpu_info *ci;
struct uvm_cpu *ucpu;
lwp_t *l = curlwp;
u_int curgen;
if (gen == UVM_EMAP_INACTIVE) {
return;
}
/*
* No need to synchronise if generation number of current CPU is
* newer than the number of this LWP.
*
* This test assumes two's complement arithmetic and allows
* ~2B missed updates before it will produce bad results.
*/
KPREEMPT_DISABLE(l);
ci = l->l_cpu;
ucpu = ci->ci_data.cpu_uvm;
if (__predict_true((signed int)(ucpu->emap_gen - gen) >= 0)) {
l->l_emap_gen = ucpu->emap_gen;
KPREEMPT_ENABLE(l);
return;
}
/*
* Record the current generation _before_ issuing the TLB flush.
* No need for a memory barrier before, as reading a stale value
* for uvm_emap_gen is not a problem.
*
* pmap_emap_sync() must implicitly perform a full memory barrier,
* which prevents us from fetching a value from after the TLB flush
* has occurred (which would be bad).
*
* We can race with an interrupt on the current CPU updating the
* counter to a newer value. This could cause us to set a stale
* value into ucpu->emap_gen, overwriting a newer update from the
* interrupt. However, it does not matter since:
* (1) Interrupts always run to completion or block.
* (2) Interrupts will only ever install a newer value and,
* (3) We will roll the value forward later.
*/
curgen = uvm_emap_gen_return();
pmap_emap_sync(true);
ucpu->emap_gen = curgen;
l->l_emap_gen = curgen;
KASSERT((signed int)(curgen - gen) >= 0);
KPREEMPT_ENABLE(l);
}
static inline void
pmap_tlb_invalidate(const pmap_tlb_packet_t *tp)
{
int i;
/* Find out what we need to invalidate. */
if (tp->tp_count == (uint16_t)-1) {
u_int egen = uvm_emap_gen_return();
if (tp->tp_pte & PG_G) {
/* Invalidating user and kernel TLB entries. */
tlbflushg();
} else {
/* Invalidating user TLB entries only. */
tlbflush();
}
uvm_emap_update(egen);
} else {
/* Invalidating a single page or a range of pages. */
for (i = tp->tp_count - 1; i >= 0; i--) {
pmap_update_pg(tp->tp_va[i]);
}
}
}
