int
mmuwalk(PTE* pml4, uintptr_t va, int level, PTE** ret,
uint64_t (*alloc)(usize))
{
int l;
uintmem pa;
PTE *pte;
Mpl pl;
pl = splhi();
if(DBGFLG > 1)
DBG("mmuwalk%d: va %#p level %d\n", machp()->machno, va, level);
pte = &pml4[PTLX(va, 3)];
for(l = 3; l >= 0; l--) {
if(l == level)
break;
if(!(*pte & PteP)) {
if(alloc == nil)
break;
pa = alloc(PTSZ);
if(pa == ~0)
return -1;
memset(UINT2PTR(KADDR(pa)), 0, PTSZ);
*pte = pa|PteRW|PteP;
}
else if(*pte & PtePS)
break;
pte = UINT2PTR(KADDR(PPN(*pte)));
pte += PTLX(va, l-1);
}
*ret = pte;
splx(pl);
return l;
}
/* virtual address fmt */
static int
fmtW(Fmt *f)
{
uint64_t va;
va = va_arg(f->args, uint64_t);
return fmtprint(f, "%#llx=0x[%llx][%llx][%llx][%llx][%llx]", va,
PTLX(va, 3), PTLX(va, 2), PTLX(va, 1), PTLX(va, 0),
va & ((1<<PGSHFT)-1));
}
static void
mmuzap(void)
{
uintptr *pte;
u64int w;
int i, x;
pte = m->pml4;
pte[PTLX(KMAP, 3)] = 0;
/* common case */
pte[PTLX(UTZERO, 3)] = 0;
pte[PTLX(TSTKTOP, 3)] = 0;
m->mmumap[PTLX(UTZERO, 3)/MAPBITS] &= ~(1ull<<(PTLX(UTZERO, 3)%MAPBITS));
m->mmumap[PTLX(TSTKTOP, 3)/MAPBITS] &= ~(1ull<<(PTLX(TSTKTOP, 3)%MAPBITS));
for(i = 0; i < nelem(m->mmumap); pte += MAPBITS, i++){
if((w = m->mmumap[i]) == 0)
continue;
m->mmumap[i] = 0;
for(x = 0; w != 0; w >>= 1, x++){
if(w & 1)
pte[x] = 0;
}
}
}
uintptr*
mmuwalk(uintptr* table, uintptr va, int level, int create)
{
uintptr pte;
int i, x;
x = PTLX(va, 3);
for(i = 2; i >= level; i--){
pte = table[x];
if(pte & PTEVALID){
if(pte & PTESIZE)
return 0;
table = KADDR(PPN(pte));
} else {
if(!create)
return 0;
table = mmucreate(table, va, i, x);
}
x = PTLX(va, i);
}
return &table[x];
}
void
mmuswitch(Proc *proc)
{
MMU *p;
mmuzap();
if(proc->newtlb){
mmufree(proc);
proc->newtlb = 0;
}
if((p = proc->kmaphead) != nil)
m->pml4[PTLX(KMAP, 3)] = PADDR(p->page) | PTEWRITE|PTEVALID;
for(p = proc->mmuhead; p != nil && p->level == PML4E; p = p->next){
m->mmumap[p->index/MAPBITS] |= 1ull<<(p->index%MAPBITS);
m->pml4[p->index] = PADDR(p->page) | PTEUSER|PTEWRITE|PTEVALID;
}
taskswitch((uintptr)proc->kstack+KSTACK);
}
/*
* pg->pgszi indicates the page size in machp()->pgsz[] used for the mapping.
* For the user, it can be either 2*MiB or 1*GiB pages.
* For 2*MiB pages, we use three levels, not four.
* For 1*GiB pages, we use two levels.
*/
void
mmuput(uintptr_t va, Page *pg, uint attr)
{
Proc *up = externup();
int lvl, user, x, pgsz;
PTE *pte;
Page *page, *prev;
Mpl pl;
uintmem pa, ppn;
char buf[80];
ppn = 0;
pa = pg->pa;
if(pa == 0)
panic("mmuput: zero pa");
if(DBGFLG) {
snprint(buf, sizeof buf, "cpu%d: up %#p mmuput %#p %#P %#ux\n",
machp()->machno, up, va, pa, attr);
print("%s", buf);
}
assert(pg->pgszi >= 0);
pgsz = sys->pgsz[pg->pgszi];
if(pa & (pgsz-1))
panic("mmuput: pa offset non zero: %#ullx\n", pa);
pa |= pteflags(attr);
pl = splhi();
if(DBGFLG)
mmuptpcheck(up);
user = (va < KZERO);
x = PTLX(va, 3);
pte = UINT2PTR(machp()->MMU.pml4->va);
pte += x;
prev = machp()->MMU.pml4;
for(lvl = 3; lvl >= 0; lvl--) {
if(user) {
if(pgsz == 2*MiB && lvl == 1) /* use 2M */
break;
if(pgsz == 1ull*GiB && lvl == 2) /* use 1G */
break;
}
for(page = up->MMU.mmuptp[lvl]; page != nil; page = page->next)
if(page->prev == prev && page->daddr == x) {
if(*pte == 0) {
print("mmu: jmk and nemo had fun\n");
*pte = PPN(page->pa)|PteU|PteRW|PteP;
}
break;
}
if(page == nil) {
if(up->MMU.mmuptp[0] == nil)
page = mmuptpalloc();
else {
page = up->MMU.mmuptp[0];
up->MMU.mmuptp[0] = page->next;
}
page->daddr = x;
page->next = up->MMU.mmuptp[lvl];
up->MMU.mmuptp[lvl] = page;
page->prev = prev;
*pte = PPN(page->pa)|PteU|PteRW|PteP;
if(lvl == 3 && x >= machp()->MMU.pml4->daddr)
machp()->MMU.pml4->daddr = x+1;
}
x = PTLX(va, lvl-1);
ppn = PPN(*pte);
if(ppn == 0)
panic("mmuput: ppn=0 l%d pte %#p = %#P\n", lvl, pte, *pte);
pte = UINT2PTR(KADDR(ppn));
pte += x;
prev = page;
}
if(DBGFLG)
checkpte(ppn, pte);
*pte = pa|PteU;
if(user)
switch(pgsz) {
case 2*MiB:
case 1*GiB:
*pte |= PtePS;
break;
default:
panic("mmuput: user pages must be 2M or 1G");
}
splx(pl);
if(DBGFLG) {
snprint(buf, sizeof buf, "cpu%d: up %#p new pte %#p = %#llux\n",
machp()->machno, up, pte, pte?*pte:~0);
print("%s", buf);
}
invlpg(va); /* only if old entry valid? */
}