/*
* We will need multiple versions of update_mmu_cache(), one that just
* updates the TLB with the new pte(s), and another which also checks
* for the R4k "end of page" hardware bug and does the needy.
*/
void __update_tlb(struct vm_area_struct * vma, unsigned long address, pte_t pte)
{
unsigned long flags;
pgd_t *pgdp;
pmd_t *pmdp;
pte_t *ptep;
int pid;
/*
* Handle debugger faulting in for debugee.
*/
if (current->active_mm != vma->vm_mm)
return;
pid = read_c0_entryhi() & cpu_asid_mask(¤t_cpu_data);
local_irq_save(flags);
address &= PAGE_MASK;
write_c0_vaddr(address);
write_c0_entryhi(pid);
pgdp = pgd_offset(vma->vm_mm, address);
pmdp = pmd_offset(pgdp, address);
ptep = pte_offset_map(pmdp, address);
tlb_probe();
write_c0_entrylo(pte_val(*ptep++) >> 6);
tlb_write();
write_c0_entryhi(pid);
local_irq_restore(flags);
}
void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
{
int cpu = smp_processor_id();
unsigned long flags;
int oldpid, newpid;
signed long idx;
if (!cpu_context(cpu, vma->vm_mm))
return;
newpid = cpu_asid(cpu, vma->vm_mm);
page &= PAGE_MASK;
local_irq_save(flags);
oldpid = read_c0_entryhi();
write_c0_vaddr(page);
write_c0_entryhi(newpid);
tlb_probe();
idx = read_c0_tlbset();
if (idx < 0)
goto finish;
write_c0_entrylo(0);
write_c0_entryhi(CKSEG0 + (idx << (PAGE_SHIFT + 1)));
tlb_write();
finish:
write_c0_entryhi(oldpid);
local_irq_restore(flags);
}
void
vm_tlbflush(vaddr_t target)
{
int spl;
int index;
spl = splhigh();
index = tlb_probe(target & PAGE_FRAME, 0);
if (index >=0)
tlb_write(TLBHI_INVALID(index), TLBLO_INVALID(), index);
splx(spl);
}
void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
unsigned long end)
{
struct mm_struct *mm = vma->vm_mm;
int cpu = smp_processor_id();
if (cpu_context(cpu, mm) != 0) {
unsigned long size, flags;
unsigned long config6_flags;
ENTER_CRITICAL(flags);
disable_pgwalker(config6_flags);
size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
size = (size + 1) >> 1;
if (size <= current_cpu_data.tlbsize/2) {
int oldpid = read_c0_entryhi();
int newpid = cpu_asid(cpu, mm);
start &= (PAGE_MASK << 1);
end += ((PAGE_SIZE << 1) - 1);
end &= (PAGE_MASK << 1);
while (start < end) {
int idx;
write_c0_entryhi(start | newpid);
start += (PAGE_SIZE << 1);
mtc0_tlbw_hazard();
tlb_probe();
tlb_probe_hazard();
idx = read_c0_index();
write_c0_entrylo0(0);
write_c0_entrylo1(0);
if (idx < 0)
continue;
/* Make sure all entries differ. */
#ifndef CONFIG_NLM_VMIPS
write_c0_entryhi(UNIQUE_ENTRYHI(idx));
#else
__write_64bit_c0_register($10, 0, (UNIQUE_VMIPS_ENTRYHI(idx)));
#endif
mtc0_tlbw_hazard();
tlb_write_indexed();
}
tlbw_use_hazard();
write_c0_entryhi(oldpid);
} else {
drop_mmu_context(mm, cpu);
}
FLUSH_ITLB;
enable_pgwalker(config6_flags);
EXIT_CRITICAL(flags);
}
void freePage(struct page* page) {
if(page->pt_state == PT_STATE_MAPPED) {
coremap_freeuserpages(page->pt_pagebase * PAGE_SIZE);
int spl = splhigh();
int tlbpos = tlb_probe(page->pt_pagebase * PAGE_SIZE, 0);
if (tlbpos >= 0) {
tlb_write(TLBHI_INVALID(tlbpos), TLBLO_INVALID(), tlbpos);
}
splx(spl);
} else if(page->pt_state == PT_STATE_MAPPED) {
swapfree(page);
}
}
/*
* mmu_map: Enter a translation into the MMU. (This is the end result
* of fault handling.)
*
* Synchronization: Takes coremap_spinlock. Does not block.
*/
void
mmu_map(struct addrspace *as, vaddr_t va, paddr_t pa, int writable)
{
int tlbix;
uint32_t ehi, elo;
unsigned cmix;
KASSERT(pa/PAGE_SIZE >= base_coremap_page);
KASSERT(pa/PAGE_SIZE - base_coremap_page < num_coremap_entries);
spinlock_acquire(&coremap_spinlock);
KASSERT(as == curcpu->c_vm.cvm_lastas);
cmix = PADDR_TO_COREMAP(pa);
KASSERT(cmix < num_coremap_entries);
/* Page must be pinned. */
KASSERT(coremap[cmix].cm_pinned);
tlbix = tlb_probe(va, 0);
if (tlbix < 0) {
KASSERT(coremap[cmix].cm_tlbix == -1);
KASSERT(coremap[cmix].cm_cpunum == 0);
tlbix = mipstlb_getslot();
KASSERT(tlbix>=0 && tlbix<NUM_TLB);
coremap[cmix].cm_tlbix = tlbix;
coremap[cmix].cm_cpunum = curcpu->c_number;
DEBUG(DB_TLB, "... pa 0x%05lx <-> tlb %d\n",
(unsigned long) COREMAP_TO_PADDR(cmix), tlbix);
}
else {
KASSERT(tlbix>=0 && tlbix<NUM_TLB);
KASSERT(coremap[cmix].cm_tlbix == tlbix);
KASSERT(coremap[cmix].cm_cpunum == curcpu->c_number);
}
ehi = va & TLBHI_VPAGE;
elo = (pa & TLBLO_PPAGE) | TLBLO_VALID;
if (writable) {
elo |= TLBLO_DIRTY;
}
tlb_write(ehi, elo, tlbix);
/* Unpin the page. */
coremap[cmix].cm_pinned = 0;
wchan_wakeall(coremap_pinchan);
spinlock_release(&coremap_spinlock);
}
void local_flush_tlb_range(struct mm_struct *mm, unsigned long start,
unsigned long end)
{
if (CPU_CONTEXT(smp_processor_id(), mm) != 0) {
unsigned long flags;
int size;
#ifdef DEBUG_TLB
printk("[tlbrange<%02x,%08lx,%08lx>]", (mm->context & 0xff),
start, end);
#endif
__save_and_cli(flags);
size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
size = (size + 1) >> 1;
if(size <= mips_cpu.tlbsize/2) {
int oldpid = (get_entryhi() & 0xff);
int newpid = (CPU_CONTEXT(smp_processor_id(), mm) &
0xff);
start &= (PAGE_MASK << 1);
end += ((PAGE_SIZE << 1) - 1);
end &= (PAGE_MASK << 1);
while(start < end) {
int idx;
set_entryhi(start | newpid);
start += (PAGE_SIZE << 1);
BARRIER;
tlb_probe();
BARRIER;
idx = get_index();
set_entrylo0(0);
set_entrylo1(0);
if(idx < 0)
continue;
/* Make sure all entries differ. */
set_entryhi(KSEG0+idx*0x2000);
BARRIER;
tlb_write_indexed();
BARRIER;
}
set_entryhi(oldpid);
} else {
get_new_mmu_context(mm, smp_processor_id());
if (mm == current->active_mm)
set_entryhi(CPU_CONTEXT(smp_processor_id(),
mm) & 0xff);
}
__restore_flags(flags);
}
void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
unsigned long end)
{
struct mm_struct *mm = vma->vm_mm;
int cpu = smp_processor_id();
unsigned long flags;
int oldpid, newpid, size;
if (!cpu_context(cpu, mm))
return;
size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
size = (size + 1) >> 1;
local_irq_save(flags);
if (size > TFP_TLB_SIZE / 2) {
drop_mmu_context(mm, cpu);
goto out_restore;
}
oldpid = read_c0_entryhi();
newpid = cpu_asid(cpu, mm);
write_c0_entrylo(0);
start &= PAGE_MASK;
end += (PAGE_SIZE - 1);
end &= PAGE_MASK;
while (start < end) {
signed long idx;
write_c0_vaddr(start);
write_c0_entryhi(start);
start += PAGE_SIZE;
tlb_probe();
idx = read_c0_tlbset();
if (idx < 0)
continue;
write_c0_entryhi(CKSEG0 + (idx << (PAGE_SHIFT + 1)));
tlb_write();
}
write_c0_entryhi(oldpid);
out_restore:
local_irq_restore(flags);
}
void local_flush_tlb_range(struct mm_struct *mm, unsigned long start,
unsigned long end)
{
int cpu = smp_processor_id();
if (cpu_context(cpu, mm) != 0) {
unsigned long flags;
int size;
#ifdef DEBUG_TLB
printk("[tlbrange<%02x,%08lx,%08lx>]", (mm->context & ASID_MASK),
start, end);
#endif
local_irq_save(flags);
size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
size = (size + 1) >> 1;
if(size <= current_cpu_data.tlbsize/2) {
int oldpid = read_c0_entryhi() & ASID_MASK;
int newpid = cpu_asid(cpu, mm);
start &= (PAGE_MASK << 1);
end += ((PAGE_SIZE << 1) - 1);
end &= (PAGE_MASK << 1);
while(start < end) {
int idx;
write_c0_entryhi(start | newpid);
start += (PAGE_SIZE << 1);
BARRIER;
tlb_probe();
BARRIER;
idx = read_c0_index();
write_c0_entrylo0(0);
write_c0_entrylo1(0);
if(idx < 0)
continue;
/* Make sure all entries differ. */
write_c0_entryhi(XKPHYS+idx*0x2000);
BARRIER;
tlb_write_indexed();
BARRIER;
}
write_c0_entryhi(oldpid);
} else {
drop_mmu_context(mm, cpu);
}
local_irq_restore(flags);
}
void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
unsigned long end)
{
struct mm_struct *mm = vma->vm_mm;
int cpu = smp_processor_id();
if (cpu_context(cpu, mm) != 0) {
unsigned long size, flags;
local_irq_save(flags);
start = round_down(start, PAGE_SIZE << 1);
end = round_up(end, PAGE_SIZE << 1);
size = (end - start) >> (PAGE_SHIFT + 1);
if (size <= (current_cpu_data.tlbsizeftlbsets ?
current_cpu_data.tlbsize / 8 :
current_cpu_data.tlbsize / 2)) {
int oldpid = read_c0_entryhi();
int newpid = cpu_asid(cpu, mm);
htw_stop();
while (start < end) {
int idx;
write_c0_entryhi(start | newpid);
start += (PAGE_SIZE << 1);
mtc0_tlbw_hazard();
tlb_probe();
tlb_probe_hazard();
idx = read_c0_index();
write_c0_entrylo0(0);
write_c0_entrylo1(0);
if (idx < 0)
continue;
/* Make sure all entries differ. */
write_c0_entryhi(UNIQUE_ENTRYHI(idx));
mtc0_tlbw_hazard();
tlb_write_indexed();
}
tlbw_use_hazard();
write_c0_entryhi(oldpid);
htw_start();
} else {
drop_mmu_context(mm, cpu);
}
flush_micro_tlb();
local_irq_restore(flags);
}
static void refill_tbl_to(struct km_walk_ctx * ctx, unsigned int asid, int write, int pos)
{
unsigned long entry, oldl1, oldl2;
unsigned long G_FLAG;
int idx;
int oldpid;
/* Just test ASID consistency: Current ASID must equal to Given ASID, kernel process do not obay this rule. */
oldpid = read_c0_entryhi();
/* Entry HI */
asid = asid & CPU_PAGE_FALG_ASID_MASK;
entry = get_vpn2(ctx->current_virtual_address);
entry |= asid;
write_c0_entryhi(entry);
mtc0_tlbw_hazard();
tlb_probe();
tlb_probe_hazard();
idx = read_c0_index();
oldl1 = read_c0_entrylo0();
oldl2 = read_c0_entrylo1();
/* Add the G_FLAG if ASID == 0, because the entry is from kernel and shared by all process */
G_FLAG = (ctx->mem == &kp_get_system()->mem_ctx)? 1 : 0;
/* Entry Low0 and Low1 */
WRITE_LO;
/* Write by type, the write is random if the TLB entry is flushed for R/W flags changing */
mtc0_tlbw_hazard();
if (unlikely(idx < 0))
tlb_write_random();
else
{
if (write == 2)
{
printk("Write is forced index for %x, pos %d, idx %d,asid %d, %x %x.\n", ctx->current_virtual_address, pos, idx, asid, oldl1, oldl2);
}
tlb_write_indexed();
}
tlbw_use_hazard();
/* Sanity: Just test ASID consistency: Current ASID must equal to Given ASID, kernel process do not obey this rule. */
if ((oldpid & 0xff) != (asid & 0xff) && asid != 0/*kernel asid*/)
printk("Why old = %x, asid = %x. ", oldpid, asid);
}
void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
unsigned long end)
{
struct mm_struct *mm = vma->vm_mm;
int cpu = smp_processor_id();
if (cpu_context(cpu, mm) != 0) {
unsigned long flags;
int size;
ENTER_CRITICAL(flags);
size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
size = (size + 1) >> 1;
local_irq_save(flags);
if (size <= current_cpu_data.tlbsize/2) {
int oldpid = read_c0_entryhi();
int newpid = cpu_asid(cpu, mm);
start &= (PAGE_MASK << 1);
end += ((PAGE_SIZE << 1) - 1);
end &= (PAGE_MASK << 1);
while (start < end) {
int idx;
write_c0_entryhi(start | newpid);
start += (PAGE_SIZE << 1);
mtc0_tlbw_hazard();
tlb_probe();
tlb_probe_hazard();
idx = read_c0_index();
write_c0_entrylo0(0);
write_c0_entrylo1(0);
if (idx < 0)
continue;
/* Make sure all entries differ. */
write_c0_entryhi(UNIQUE_ENTRYHI(idx));
mtc0_tlbw_hazard();
tlb_write_indexed();
}
tlbw_use_hazard();
write_c0_entryhi(oldpid);
} else {
drop_mmu_context(mm, cpu);
}
EXIT_CRITICAL(flags);
}
static void swaponepagein(int idx, struct addrspace* as) {
(void) as;
// 3. clear their tlb entries if present TODO
cm_setEntryDirtyState(COREMAP(idx),true);
struct page* pg = findPageFromCoreMap(COREMAP(idx), idx);
int spl = splhigh();
int tlbpos = tlb_probe(pg->pt_pagebase * PAGE_SIZE, 0);
if (tlbpos >= 0) {
tlb_write(TLBHI_INVALID(tlbpos), TLBLO_INVALID(), tlbpos);
} else {
//kprintf("was not on tlb\n");
}
splx(spl);
int swapPageindex = getOneSwapPage();
kprintf("Swap in :\tswap= %x,\tpage=%x \n",swapPageindex,pg->pt_virtbase);
//kprintf("Swap in page Vaddr = %x\n", pg->pt_virtbase);
struct iovec iov;
struct uio kuio;
iov.iov_kbase = (void*) PADDR_TO_KVADDR(cm_getEntryPaddr(idx));
iov.iov_len = PAGE_SIZE; // length of the memory space
kuio.uio_iov = &iov;
kuio.uio_iovcnt = 1;
kuio.uio_resid = PAGE_SIZE; // amount to write to the file
kuio.uio_space = NULL;
kuio.uio_offset = swap_map[swapPageindex].se_paddr * PAGE_SIZE;
kuio.uio_segflg = UIO_SYSSPACE;
kuio.uio_rw = UIO_WRITE;
//kprintf("before write \n");
// 4. write them to disk
spinlock_release(&coremap_lock);
int result = VOP_WRITE(swap_vnode, &kuio);
spinlock_acquire(&coremap_lock);
if (result) {
// release lock on the vnode
panic("WRITE FAILED!\n");
return;
}
cm_setEntryDirtyState(COREMAP(idx),false);
//kprintf("write complete\n");
pg->pt_state = PT_STATE_SWAPPED;
pg->pt_pagebase = swap_map[swapPageindex].se_paddr;
}
/**
@brief Flush memory range
If the memory range is too big, we flush all entries with this ASID
*/
void local_flush_tlb_range(unsigned int asid, unsigned long start, unsigned long end)
{
unsigned long size, flags;
ENTER_CRITICAL(flags);
size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
size = (size + 1) >> 1;
if (size <= current_cpu_data.tlbsize / 2)
{
int oldpid = read_c0_entryhi();
int newpid = asid;
start &= (PAGE_MASK << 1);
end += ((PAGE_SIZE << 1) - 1);
end &= (PAGE_MASK << 1);
while (start < end) {
int idx;
write_c0_entryhi(start | newpid);
start += (PAGE_SIZE << 1);
mtc0_tlbw_hazard();
tlb_probe();
tlb_probe_hazard();
idx = read_c0_index();
write_c0_entrylo0(0);
write_c0_entrylo1(0);
if (idx < 0)
continue;
/* Make sure all entries differ. */
write_c0_entryhi(UNIQUE_ENTRYHI(idx));
mtc0_tlbw_hazard();
tlb_write_indexed();
}
tlbw_use_hazard();
write_c0_entryhi(oldpid);
}
else
local_flush_asid(asid);
FLUSH_ITLB;
EXIT_CRITICAL(flags);
}
void local_flush_tlb_range(struct mm_struct *mm, unsigned long start,
unsigned long end)
{
int cpu = smp_processor_id();
if (cpu_context(cpu, mm) != 0) {
unsigned long flags;
int size;
#ifdef DEBUG_TLB
printk("[tlbrange<%02x,%08lx,%08lx>]",
(mm->context & ASID_MASK), start, end);
#endif
local_irq_save(flags);
size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
size = (size + 1) >> 1;
if (size <= NTLB_ENTRIES_HALF) {
int oldpid = (read_c0_entryhi() & ASID_MASK);
int newpid = (cpu_context(smp_processor_id(), mm)
& ASID_MASK);
start &= (PAGE_MASK << 1);
end += ((PAGE_SIZE << 1) - 1);
end &= (PAGE_MASK << 1);
while(start < end) {
int idx;
write_c0_entryhi(start | newpid);
start += (PAGE_SIZE << 1);
tlb_probe();
idx = read_c0_index();
write_c0_entrylo0(0);
write_c0_entrylo1(0);
write_c0_entryhi(KSEG0);
if(idx < 0)
continue;
tlb_write_indexed();
}
write_c0_entryhi(oldpid);
} else {
drop_mmu_context(mm, cpu);
}
local_irq_restore(flags);
}
void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
unsigned long end)
{
struct mm_struct *mm = vma->vm_mm;
int cpu = smp_processor_id();
if (cpu_context(cpu, mm) != 0) {
unsigned long flags;
int size;
#ifdef DEBUG_TLB
printk("[tlbrange<%lu,0x%08lx,0x%08lx>]",
cpu_context(cpu, mm) & ASID_MASK, start, end);
#endif
local_irq_save(flags);
size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
if (size <= current_cpu_data.tlbsize) {
int oldpid = read_c0_entryhi() & ASID_MASK;
int newpid = cpu_context(cpu, mm) & ASID_MASK;
start &= PAGE_MASK;
end += PAGE_SIZE - 1;
end &= PAGE_MASK;
while (start < end) {
int idx;
write_c0_entryhi(start | newpid);
start += PAGE_SIZE; /* BARRIER */
tlb_probe();
idx = read_c0_index();
write_c0_entrylo0(0);
write_c0_entryhi(KSEG0);
if (idx < 0) /* BARRIER */
continue;
tlb_write_indexed();
}
write_c0_entryhi(oldpid);
} else {
drop_mmu_context(mm, cpu);
}
local_irq_restore(flags);
}
void local_flush_tlb_range(struct mm_struct *mm, unsigned long start,
unsigned long end)
{
if (mm->context != 0) {
unsigned long flags;
int size;
#ifdef DEBUG_TLB
printk("[tlbrange<%lu,0x%08lx,0x%08lx>]",
(mm->context & 0xfc0), start, end);
#endif
save_and_cli(flags);
size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
if (size <= mips_cpu.tlbsize) {
int oldpid = (get_entryhi() & 0xfc0);
int newpid = (mm->context & 0xfc0);
start &= PAGE_MASK;
end += (PAGE_SIZE - 1);
end &= PAGE_MASK;
while (start < end) {
int idx;
set_entryhi(start | newpid);
start += PAGE_SIZE;
tlb_probe();
idx = get_index();
set_entrylo0(0);
set_entryhi(KSEG0);
if (idx < 0)
continue;
tlb_write_indexed();
}
set_entryhi(oldpid);
} else {
get_new_mmu_context(mm, smp_processor_id());
if (mm == current->active_mm)
set_entryhi(mm->context & 0xfc0);
}
restore_flags(flags);
}
void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
unsigned long end)
{
struct mm_struct *mm = vma->vm_mm;
unsigned long vma_mm_context = mm->context;
if (mm->context != 0) {
unsigned long flags;
int size;
local_irq_save(flags);
size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
if (size <= TLBSIZE) {
int oldpid = pevn_get() & ASID_MASK;
int newpid = vma_mm_context & ASID_MASK;
start &= PAGE_MASK;
end += (PAGE_SIZE - 1);
end &= PAGE_MASK;
while (start < end) {
int idx;
pevn_set(start | newpid);
start += PAGE_SIZE;
barrier();
tlb_probe();
idx = tlbpt_get();
pectx_set(0);
pevn_set(KSEG1);
if (idx < 0)
continue;
tlb_write_indexed();
}
pevn_set(oldpid);
} else {
/* Bigger than TLBSIZE, get new ASID directly */
get_new_mmu_context(mm);
if (mm == current->active_mm)
pevn_set(vma_mm_context & ASID_MASK);
}
local_irq_restore(flags);
}
void
dump_tlb_addr(unsigned long addr)
{
unsigned long flags, oldpid;
int index;
local_irq_save(flags);
oldpid = read_c0_entryhi() & 0xff;
write_c0_entryhi((addr & PAGE_MASK) | oldpid);
tlb_probe();
index = read_c0_index();
write_c0_entryhi(oldpid);
local_irq_restore(flags);
if (index < 0) {
printk("No entry for address 0x%08lx in TLB\n", addr);
return;
}
printk("Entry %d maps address 0x%08lx\n", index, addr);
dump_tlb(index, index);
}
void
dump_tlb_addr(unsigned long addr)
{
unsigned int flags, oldpid;
int index;
__save_and_cli(flags);
oldpid = get_entryhi() & 0xff;
set_entryhi((addr & PAGE_MASK) | oldpid);
tlb_probe();
index = get_index();
set_entryhi(oldpid);
__restore_flags(flags);
if (index < 0) {
printk("No entry for address 0x%08lx in TLB\n", addr);
return;
}
printk("Entry %d maps address 0x%08lx\n", index, addr);
dump_tlb(index, index);
}
/* Usable for KV1 addresses only! */
void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
{
unsigned long flags;
int size;
size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
size = (size + 1) >> 1;
if (size > TFP_TLB_SIZE / 2) {
local_flush_tlb_all();
return;
}
local_irq_save(flags);
write_c0_entrylo(0);
start &= PAGE_MASK;
end += (PAGE_SIZE - 1);
end &= PAGE_MASK;
while (start < end) {
signed long idx;
write_c0_vaddr(start);
write_c0_entryhi(start);
start += PAGE_SIZE;
tlb_probe();
idx = read_c0_tlbset();
if (idx < 0)
continue;
write_c0_entryhi(CKSEG0 + (idx << (PAGE_SHIFT + 1)));
tlb_write();
}
local_irq_restore(flags);
}
/*
* tlb_unmap: Searches the TLB for a vaddr translation and invalidates
* it if it exists.
*
* Synchronization: assumes we hold coremap_spinlock. Does not block.
*/
static
void
tlb_unmap(vaddr_t va)
{
int i;
uint32_t elo = 0, ehi = 0;
KASSERT(spinlock_do_i_hold(&coremap_spinlock));
KASSERT(va < MIPS_KSEG0);
i = tlb_probe(va & PAGE_FRAME,0);
if (i < 0) {
return;
}
tlb_read(&ehi, &elo, i);
KASSERT(elo & TLBLO_VALID);
DEBUG(DB_TLB, "invalidating tlb slot %d (va: 0x%x)\n", i, va);
tlb_invalidate(i);
}
void vm_tlbshootdown(const struct tlbshootdown *ts)
{
//(void)ts;
int tlb_entry, spl;
/* Disable interrupts on this CPU while frobbing the TLB. */
bool lock = get_coremap_spinlock();
spl = splhigh();
// Probe TLB so see if particular VA is present.
tlb_entry = tlb_probe(VA_TO_VPF(ts->ts_vaddr), 0);
if(tlb_entry < 0) {
// No entry found, so shoot down succeeded
splx(spl);
return;
}
// Invalidate the particular TLB entry
tlb_write(TLBHI_INVALID(tlb_entry), TLBLO_INVALID(), tlb_entry);
splx(spl);
release_coremap_spinlock(lock);
}
// handle page faults
int vm_fault(int faulttype, vaddr_t faultaddress) {
(void)faulttype;
// (void)faultaddress;
uint32_t tlbhi;
uint32_t tlblo;
if (curthread->t_addrspace == NULL) // kernel has page faulted, so return EFAULT, which will cause a panic (as it should)
return EFAULT;
faultaddress &= PAGE_FRAME; // page-align the fault address
struct page* pg = as_fault(curthread->t_addrspace,faultaddress);
if (pg==NULL){
return EFAULT;
}
spinlock_acquire(&pg->pg_lock);
int stat = pg->status;
spinlock_release(&pg->pg_lock);
if (stat==NOT_ALLOCD) {
int err = page_alloc(pg);
if (err)
return err;
}
KASSERT((pg->ram_addr&PAGE_FRAME)==pg->ram_addr);
KASSERT(pg->status==IN_MEM);
spinlock_acquire(&pg->pg_lock);
pg->is_dirty = 1;
tlblo = (pg->ram_addr & TLBLO_PPAGE) | TLBLO_VALID | TLBLO_DIRTY;
spinlock_release(&pg->pg_lock);
tlbhi = faultaddress & TLBHI_VPAGE;
spinlock_acquire(&tlb_lock);; // only one thread should be messing with the TLB at a time
//int probe = tlb_probe(tlbhi,0);
//if (probe<0)
tlb_random(tlbhi,tlblo);
//else
// tlb_write(tlbhi,tlblo,probe);
int probe = tlb_probe(tlbhi,0);
KASSERT(probe>=0);
spinlock_release(&tlb_lock);
return 0;
}
/*
* Handle a TLB miss exception for a page marked as able to trigger the
* end-of-page errata.
* Returns nonzero if the exception has been completely serviced, and no
* further processing in the trap handler is necessary.
*/
int
eop_tlb_miss_handler(struct trap_frame *trapframe, struct cpu_info *ci,
struct proc *p)
{
struct pcb *pcb;
vaddr_t va, faultva;
struct vmspace *vm;
vm_map_t map;
pmap_t pmap;
pt_entry_t *pte, entry;
int onfault;
u_long asid;
uint i, npairs;
int64_t tlbidx;
/*
* Check for a valid pte with the `special' bit set (PG_SP)
* in order to apply the end-of-page errata workaround.
*/
vm = p->p_vmspace;
map = &vm->vm_map;
faultva = trunc_page((vaddr_t)trapframe->badvaddr);
pmap = map->pmap;
pte = pmap_segmap(pmap, faultva);
if (pte == NULL)
return 0;
pte += uvtopte(faultva);
entry = *pte;
if ((entry & PG_SP) == 0)
return 0;
pcb = &p->p_addr->u_pcb;
asid = pmap->pm_asid[ci->ci_cpuid].pma_asid << PG_ASID_SHIFT;
/*
* For now, only allow one EOP vulnerable page to get a wired TLB
* entry. We will aggressively attempt to recycle the wired TLB
* entries created for that purpose, as soon as we are no longer
* needing the EOP page resident in the TLB.
*/
/*
* Figure out how many pages to wire in the TLB.
*/
if ((faultva & PG_ODDPG) != 0) {
/* odd page: need two pairs */
npairs = 2;
} else {
/* even page: only need one pair */
npairs = 1;
}
/*
* Fault-in the next page.
*/
va = faultva + PAGE_SIZE;
pte = pmap_segmap(pmap, va);
if (pte != NULL)
pte += uvtopte(va);
if (pte == NULL || (*pte & PG_V) == 0) {
onfault = pcb->pcb_onfault;
pcb->pcb_onfault = 0;
KERNEL_LOCK();
(void)uvm_fault(map, va, 0, PROT_READ | PROT_EXEC);
KERNEL_UNLOCK();
pcb->pcb_onfault = onfault;
}
/*
* Clear possible TLB entries for the pages we're about to wire.
*/
for (i = npairs, va = faultva & PG_HVPN; i != 0;
i--, va += 2 * PAGE_SIZE) {
tlbidx = tlb_probe(va | asid);
if (tlbidx >= 0)
tlb_update_indexed(CKSEG0_BASE, PG_NV, PG_NV, tlbidx);
}
/*
* Reserve the extra wired TLB, and fill them with the existing ptes.
*/
tlb_set_wired((UPAGES / 2) + npairs);
for (i = 0, va = faultva & PG_HVPN; i != npairs;
i++, va += 2 * PAGE_SIZE) {
pte = pmap_segmap(pmap, va);
if (pte == NULL)
tlb_update_indexed(va | asid,
PG_NV, PG_NV, (UPAGES / 2) + i);
else {
pte += uvtopte(va);
tlb_update_indexed(va | asid,
pte[0], pte[1], (UPAGES / 2) + i);
}
}
/*
* Save the base address of the EOP vulnerable page, to be able to
* figure out when the wired entry is no longer necessary.
*/
pcb->pcb_nwired = npairs;
pcb->pcb_wiredva = faultva & PG_HVPN;
pcb->pcb_wiredpc = faultva;
return 1;
}
static int dec_kn01_be_backend(struct pt_regs *regs, int is_fixup, int invoker)
{
volatile u32 *kn01_erraddr = (void *)CKSEG1ADDR(KN01_SLOT_BASE +
KN01_ERRADDR);
static const char excstr[] = "exception";
static const char intstr[] = "interrupt";
static const char cpustr[] = "CPU";
static const char mreadstr[] = "memory read";
static const char readstr[] = "read";
static const char writestr[] = "write";
static const char timestr[] = "timeout";
static const char paritystr[] = "parity error";
int data = regs->cp0_cause & 4;
unsigned int __user *pc = (unsigned int __user *)regs->cp0_epc +
((regs->cp0_cause & CAUSEF_BD) != 0);
union mips_instruction insn;
unsigned long entrylo, offset;
long asid, entryhi, vaddr;
const char *kind, *agent, *cycle, *event;
unsigned long address;
u32 erraddr = *kn01_erraddr;
int action = MIPS_BE_FATAL;
/* Ack ASAP, so that any subsequent errors get caught. */
dec_kn01_be_ack();
kind = invoker ? intstr : excstr;
agent = cpustr;
if (invoker)
address = erraddr;
else {
/* Bloody hardware doesn't record the address for reads... */
if (data) {
/* This never faults. */
__get_user(insn.word, pc);
vaddr = regs->regs[insn.i_format.rs] +
insn.i_format.simmediate;
} else
vaddr = (long)pc;
if (KSEGX(vaddr) == CKSEG0 || KSEGX(vaddr) == CKSEG1)
address = CPHYSADDR(vaddr);
else {
/* Peek at what physical address the CPU used. */
asid = read_c0_entryhi();
entryhi = asid & (PAGE_SIZE - 1);
entryhi |= vaddr & ~(PAGE_SIZE - 1);
write_c0_entryhi(entryhi);
BARRIER;
tlb_probe();
/* No need to check for presence. */
tlb_read();
entrylo = read_c0_entrylo0();
write_c0_entryhi(asid);
offset = vaddr & (PAGE_SIZE - 1);
address = (entrylo & ~(PAGE_SIZE - 1)) | offset;
}
}
/* Treat low 256MB as memory, high -- as I/O. */
if (address < 0x10000000) {
cycle = mreadstr;
event = paritystr;
} else {
cycle = invoker ? writestr : readstr;
event = timestr;
}
if (is_fixup)
action = MIPS_BE_FIXUP;
if (action != MIPS_BE_FIXUP)
printk(KERN_ALERT "Bus error %s: %s %s %s at %#010lx\n",
kind, agent, cycle, event, address);
return action;
}
int
vm_fault(int faulttype, vaddr_t faultaddress)
{
struct pte *target;
uint32_t tlbhi, tlblo;
int spl;
struct addrspace* as = curproc->p_addrspace;
int permission = as_check_region(as, faultaddress);
if (permission < 0 // check if not in region
&& as_check_stack(as,faultaddress) // check if not in stack
&& as_check_heap(as,faultaddress)) // check if not in heap
return EFAULT;
if(permission<0)
permission = READ | WRITE;
target = pte_get(as,faultaddress & PAGE_FRAME);
if(target==NULL) {
target = pt_alloc_page(as,faultaddress & PAGE_FRAME);
}
// Lock pagetable entry
if(swap_enabled == true)
lock_acquire(target->pte_lock);
if(target->in_memory == 0) { // Page is allocated but not in memory.
target = pt_load_page(as, faultaddress & PAGE_FRAME); // pt_alloc_page creates the page table entry if neccessary and also
// allocates it using coremap.
}
KASSERT(target->in_memory != 0);
KASSERT(target->paddr != 0);
KASSERT(target->paddr != 1);
tlbhi = faultaddress & PAGE_FRAME;
tlblo = (target->paddr & PAGE_FRAME) | TLBLO_VALID;
/* Adding CPU index to corresponding TLB entry */
coremap[PADDR_TO_CM(target->paddr)].cpu = curcpu->c_number;
coremap[PADDR_TO_CM(target->paddr)].page = target;
coremap[PADDR_TO_CM(target->paddr)].accessed = 1;
int index;
spl = splhigh();
// TODO permissions
//kprintf(" \n %x - %x \n",tlbhi, tlblo);
switch (faulttype) {
case VM_FAULT_READ:
case VM_FAULT_WRITE:
//index = PADDR_TO_CM(target->paddr);
//coremap[index].state = DIRTY;
tlb_random(tlbhi, tlblo);
break;
case VM_FAULT_READONLY:
tlblo |= TLBLO_DIRTY;
// TODO: Change physical page's state to DIRTY.
index = PADDR_TO_CM(target->paddr);
//KASSERT(coremap[index].state!=FIXED);
//KASSERT(coremap[index].state!=VICTIM);
KASSERT(target->in_memory != 0); // Someone swapped me out. Synchronization is broken.
//KASSERT(coremap[index].as ==as);
coremap[index].state = DIRTY; // Set it to dirty!
index = tlb_probe(faultaddress & PAGE_FRAME, 0);
tlb_write(tlbhi, tlblo, index);
}
splx(spl);
if(swap_enabled == true)
lock_release(target->pte_lock);
return 0;
}
static void swapin(int npages, struct addrspace* as) {
// 1. check if coremap lock is already held, else acquire it
if (swap_state == SWAP_STATE_NOSWAP) {
panic("Attempting to swap in when no swap disk is found!\n");
}
lock_acquire(swap_lock);
// 2. select a bunch of non kernel pages.
unsigned int i;
for (i = 0; i < page_count; i++) {
int i_idx = SWAP_IDX(i + swap_prev_write_idx);
if (cm_isEntryUsed(COREMAP(i_idx)) && cm_getEntryAddrspaceIdent(COREMAP(i_idx)) != NULL) {
struct page* pg = NULL;
if (cm_getEntryAddrspaceIdent(COREMAP(i_idx)) == as) {
pg = findPageFromCoreMap(COREMAP(i_idx), i_idx);
int spl = splhigh();
int tlbpos = tlb_probe(pg->pt_virtbase * PAGE_SIZE, 0);
splx(spl);
if (tlbpos >= 0) {
//kprintf("tlb hit at %x %d\n",pg->pt_pagebase, tlbpos);
continue;
}
}
int j = 0;
int flag = 1;
for (j = 0; j < npages; j++) {
int j_idx = SWAP_IDX(i + j + swap_prev_write_idx);
if (j_idx >= (int)page_count) {
kprintf("page count greater than i+j\n");
flag = 0;
break;
}
if (cm_isEntryUsed(
COREMAP(j_idx)) && cm_getEntryAddrspaceIdent(COREMAP(j_idx)) == NULL) {
kprintf("page used by kernel\n");
flag = 0;
break;
}
}
if (flag == 1) {
for (j = 0; j < npages; j++) {
int j_idx = SWAP_IDX(i + j + swap_prev_write_idx);
if (cm_isEntryUsed(COREMAP(j_idx)) && cm_getEntryAddrspaceIdent(COREMAP(j_idx)) != NULL) {
swaponepagein(j_idx, as);
cm_setEntryUseState(COREMAP(j_idx), false);
cm_setEntryDirtyState(COREMAP(j_idx), false);
// let the address space identifier be NULL initially
cm_setEntryAddrspaceIdent(COREMAP(j_idx), NULL);
coremap_pages_free++;
swap_prev_write_idx = j_idx + 1;
/*if(pg != NULL) {
kprintf("swapped in page was = %x\n", pg->pt_virtbase);
} else {
kprintf("page was nill\n");
}*/
} else {
kprintf("How did this get approved?\n");
}
}
//spinlock_acquire(&coremap_lock);
lock_release(swap_lock);
return;
}
}
}
panic("Out of pages to swap out!\n");
//spinlock_acquire(&coremap_lock);
lock_release(swap_lock);
// 2.5 Maintain a index of last page that was swapped in so that you swap in the one after that
// 5. free lock if you had acquired it in this method
}