/** Process TLB Refill Exception.
*
* @param istate Interrupted register context.
*
*/
void tlb_refill(istate_t *istate)
{
uintptr_t badvaddr = cp0_badvaddr_read();
mutex_lock(&AS->lock);
asid_t asid = AS->asid;
mutex_unlock(&AS->lock);
int pfrc;
pte_t *pte = find_mapping_and_check(badvaddr, PF_ACCESS_READ,
istate, &pfrc);
if (!pte) {
switch (pfrc) {
case AS_PF_FAULT:
goto fail;
break;
case AS_PF_DEFER:
/*
* The page fault came during copy_from_uspace()
* or copy_to_uspace().
*/
return;
default:
panic("Unexpected pfrc (%d).", pfrc);
}
}
/*
* Record access to PTE.
*/
pte->a = 1;
entry_lo_t lo;
entry_hi_t hi;
tlb_prepare_entry_hi(&hi, asid, badvaddr);
tlb_prepare_entry_lo(&lo, pte->g, pte->p, pte->d, pte->c,
pte->frame);
/*
* New entry is to be inserted into TLB
*/
cp0_entry_hi_write(hi.value);
if ((badvaddr / PAGE_SIZE) % 2 == 0) {
cp0_entry_lo0_write(lo.value);
cp0_entry_lo1_write(0);
} else {
cp0_entry_lo0_write(0);
cp0_entry_lo1_write(lo.value);
}
cp0_pagemask_write(TLB_PAGE_MASK_16K);
tlbwr();
return;
fail:
tlb_refill_fail(istate);
}
/** Invalidate all TLB entries belonging to specified address space.
*
* @param asid Address space identifier.
*
*/
void tlb_invalidate_asid(asid_t asid)
{
ASSERT(asid != ASID_INVALID);
entry_hi_t hi_save;
hi_save.value = cp0_entry_hi_read();
ipl_t ipl = interrupts_disable();
for (unsigned int i = 0; i < TLB_ENTRY_COUNT; i++) {
cp0_index_write(i);
tlbr();
entry_hi_t hi;
hi.value = cp0_entry_hi_read();
if (hi.asid == asid) {
entry_lo_t lo0;
lo0.value = cp0_entry_lo0_read();
entry_lo_t lo1;
lo1.value = cp0_entry_lo1_read();
lo0.v = 0;
lo1.v = 0;
cp0_entry_lo0_write(lo0.value);
cp0_entry_lo1_write(lo1.value);
tlbwi();
}
}
interrupts_restore(ipl);
cp0_entry_hi_write(hi_save.value);
}
/** Invalidate all not wired TLB entries. */
void tlb_invalidate_all(void)
{
entry_hi_t hi_save;
hi_save.value = cp0_entry_hi_read();
ipl_t ipl = interrupts_disable();
for (unsigned int i = TLB_WIRED; i < TLB_ENTRY_COUNT; i++) {
cp0_index_write(i);
tlbr();
entry_lo_t lo0;
lo0.value = cp0_entry_lo0_read();
entry_lo_t lo1;
lo1.value = cp0_entry_lo1_read();
lo0.v = 0;
lo1.v = 0;
cp0_entry_lo0_write(lo0.value);
cp0_entry_lo1_write(lo1.value);
tlbwi();
}
interrupts_restore(ipl);
cp0_entry_hi_write(hi_save.value);
}
/** Process TLB Refill Exception.
*
* @param istate Interrupted register context.
*
*/
void tlb_refill(istate_t *istate)
{
uintptr_t badvaddr = cp0_badvaddr_read();
mutex_lock(&AS->lock);
asid_t asid = AS->asid;
mutex_unlock(&AS->lock);
pte_t *pte = find_mapping_and_check(badvaddr, PF_ACCESS_READ, istate);
if (pte) {
/*
* Record access to PTE.
*/
pte->a = 1;
entry_lo_t lo;
entry_hi_t hi;
tlb_prepare_entry_hi(&hi, asid, badvaddr);
tlb_prepare_entry_lo(&lo, pte->g, pte->p, pte->d, pte->c,
pte->frame);
/*
* New entry is to be inserted into TLB
*/
cp0_entry_hi_write(hi.value);
if ((badvaddr / PAGE_SIZE) % 2 == 0) {
cp0_entry_lo0_write(lo.value);
cp0_entry_lo1_write(0);
} else {
cp0_entry_lo0_write(0);
cp0_entry_lo1_write(lo.value);
}
cp0_pagemask_write(TLB_PAGE_MASK_16K);
tlbwr();
}
}
/** Process TLB Modified Exception.
*
* @param istate Interrupted register context.
*
*/
void tlb_modified(istate_t *istate)
{
uintptr_t badvaddr = cp0_badvaddr_read();
/*
* Locate the faulting entry in TLB.
*/
entry_hi_t hi;
hi.value = cp0_entry_hi_read();
tlb_prepare_entry_hi(&hi, hi.asid, badvaddr);
cp0_entry_hi_write(hi.value);
tlbp();
tlb_index_t index;
index.value = cp0_index_read();
ASSERT(!index.p);
pte_t *pte = find_mapping_and_check(badvaddr, PF_ACCESS_WRITE, istate);
if (pte) {
/*
* Read the faulting TLB entry.
*/
tlbr();
/*
* Record access and write to PTE.
*/
pte->a = 1;
pte->d = 1;
entry_lo_t lo;
tlb_prepare_entry_lo(&lo, pte->g, pte->p, pte->w, pte->c,
pte->frame);
/*
* The entry is to be updated in TLB.
*/
if ((badvaddr / PAGE_SIZE) % 2 == 0)
cp0_entry_lo0_write(lo.value);
else
cp0_entry_lo1_write(lo.value);
cp0_pagemask_write(TLB_PAGE_MASK_16K);
tlbwi();
}
}
/** Invalidate TLB entries for specified page range belonging to specified
* address space.
*
* @param asid Address space identifier.
* @param page First page whose TLB entry is to be invalidated.
* @param cnt Number of entries to invalidate.
*
*/
void tlb_invalidate_pages(asid_t asid, uintptr_t page, size_t cnt)
{
if (asid == ASID_INVALID)
return;
entry_hi_t hi_save;
hi_save.value = cp0_entry_hi_read();
ipl_t ipl = interrupts_disable();
for (unsigned int i = 0; i < cnt + 1; i += 2) {
entry_hi_t hi;
hi.value = 0;
tlb_prepare_entry_hi(&hi, asid, page + i * PAGE_SIZE);
cp0_entry_hi_write(hi.value);
tlbp();
tlb_index_t index;
index.value = cp0_index_read();
if (!index.p) {
/*
* Entry was found, index register contains valid
* index.
*/
tlbr();
entry_lo_t lo0;
lo0.value = cp0_entry_lo0_read();
entry_lo_t lo1;
lo1.value = cp0_entry_lo1_read();
lo0.v = 0;
lo1.v = 0;
cp0_entry_lo0_write(lo0.value);
cp0_entry_lo1_write(lo1.value);
tlbwi();
}
}
interrupts_restore(ipl);
cp0_entry_hi_write(hi_save.value);
}
/** Initialize TLB.
*
* Invalidate all entries and mark wired entries.
*
*/
void tlb_arch_init(void)
{
cp0_pagemask_write(TLB_PAGE_MASK_16K);
cp0_entry_hi_write(0);
cp0_entry_lo0_write(0);
cp0_entry_lo1_write(0);
/* Clear and initialize TLB. */
for (unsigned int i = 0; i < TLB_ENTRY_COUNT; i++) {
cp0_index_write(i);
tlbwi();
}
/*
* The kernel is going to make use of some wired
* entries (e.g. mapping kernel stacks in kseg3).
*/
cp0_wired_write(TLB_WIRED);
}
/** Process TLB Modified Exception.
*
* @param istate Interrupted register context.
*
*/
void tlb_modified(istate_t *istate)
{
uintptr_t badvaddr = cp0_badvaddr_read();
/*
* Locate the faulting entry in TLB.
*/
entry_hi_t hi;
hi.value = cp0_entry_hi_read();
tlb_prepare_entry_hi(&hi, hi.asid, badvaddr);
cp0_entry_hi_write(hi.value);
tlbp();
tlb_index_t index;
index.value = cp0_index_read();
/*
* Fail if the entry is not in TLB.
*/
if (index.p) {
printf("TLB entry not found.\n");
goto fail;
}
int pfrc;
pte_t *pte = find_mapping_and_check(badvaddr, PF_ACCESS_WRITE,
istate, &pfrc);
if (!pte) {
switch (pfrc) {
case AS_PF_FAULT:
goto fail;
break;
case AS_PF_DEFER:
/*
* The page fault came during copy_from_uspace()
* or copy_to_uspace().
*/
return;
default:
panic("Unexpected pfrc (%d).", pfrc);
}
}
/*
* Read the faulting TLB entry.
*/
tlbr();
/*
* Record access and write to PTE.
*/
pte->a = 1;
pte->d = 1;
entry_lo_t lo;
tlb_prepare_entry_lo(&lo, pte->g, pte->p, pte->w, pte->c,
pte->frame);
/*
* The entry is to be updated in TLB.
*/
if ((badvaddr / PAGE_SIZE) % 2 == 0)
cp0_entry_lo0_write(lo.value);
else
cp0_entry_lo1_write(lo.value);
cp0_pagemask_write(TLB_PAGE_MASK_16K);
tlbwi();
return;
fail:
tlb_modified_fail(istate);
}