void
mmurelease(Proc* proc)
{
Page *page, *next;
mmuptpfree(proc, 0);
for(page = proc->MMU.mmuptp[0]; page != nil; page = next) {
next = page->next;
if(--page->ref)
panic("mmurelease: page->ref %d\n", page->ref);
lock(&mmuptpfreelist.l);
page->next = mmuptpfreelist.next;
mmuptpfreelist.next = page;
mmuptpfreelist.ref++;
page->prev = nil;
unlock(&mmuptpfreelist.l);
}
if(proc->MMU.mmuptp[0] && pga.rend.l.p)
wakeup(&pga.rend);
proc->MMU.mmuptp[0] = nil;
tssrsp0(machp(), STACKALIGN(machp()->stack+MACHSTKSZ));
cr3put(machp()->MMU.pml4->pa);
}
void
mmuswitch(Proc* proc)
{
PTE *pte;
Page *page;
Mpl pl;
pl = splhi();
if(proc->newtlb) {
/*
* NIX: We cannot clear our page tables if they are going to
* be used in the AC
*/
if(proc->ac == nil)
mmuptpfree(proc, 1);
proc->newtlb = 0;
}
if(machp()->MMU.pml4->daddr) {
memset(UINT2PTR(machp()->MMU.pml4->va), 0, machp()->MMU.pml4->daddr*sizeof(PTE));
machp()->MMU.pml4->daddr = 0;
}
pte = UINT2PTR(machp()->MMU.pml4->va);
for(page = proc->MMU.mmuptp[3]; page != nil; page = page->next) {
pte[page->daddr] = PPN(page->pa)|PteU|PteRW|PteP;
if(page->daddr >= machp()->MMU.pml4->daddr)
machp()->MMU.pml4->daddr = page->daddr+1;
page->prev = machp()->MMU.pml4;
}
tssrsp0(machp(), STACKALIGN(PTR2UINT(proc->kstack+KSTACK)));
cr3put(machp()->MMU.pml4->pa);
splx(pl);
}
void
acmmuswitch(void)
{
extern Page mach0pml4;
DBG("acmmuswitch mpl4 %#p mach0pml4 %#p m0pml4 %#p\n", machp()->pml4->pa, mach0pml4.pa, sys->machptr[0]->pml4->pa);
cr3put(machp()->pml4->pa);
}
void
mmuflushtlb(uint64_t u)
{
machp()->tlbpurge++;
if(machp()->MMU.pml4->daddr) {
memset(UINT2PTR(machp()->MMU.pml4->va), 0, machp()->MMU.pml4->daddr*sizeof(PTE));
machp()->MMU.pml4->daddr = 0;
}
cr3put(machp()->MMU.pml4->pa);
}
/*
* Move the current process to an application core.
* This is performed at the end of execac(), and
* we pretend to be returning to user-space, but instead we
* dispatch the process to another core.
* 1. We do the final bookkeeping that syscall() would do after
* a return from sysexec(), because we are not returning.
* 2. We dispatch the process to an AC using an ICC.
*
* This function won't return unless the process is reclaimed back
* to the time-sharing core, and is the handler for the process
* to deal with traps and system calls until the process dies.
*
* Remember that this function is the "line" between user and kernel
* space, it's not expected to raise|handle any error.
*
* We install a safety error label, just in case we raise errors,
* which we shouldn't. (noerrorsleft knows that for exotic processes
* there is an error label pushed by us).
*/
void
runacore(void)
{
Proc *up = externup();
Ureg *ureg;
void (*fn)(void);
int rc, flush, s;
char *n;
uint64_t t1;
if(waserror())
panic("runacore: error: %s\n", up->errstr);
ureg = up->dbgreg;
fakeretfromsyscall(ureg);
fpusysrfork(ureg);
procpriority(up, PriKproc, 1);
rc = runac(up->ac, actouser, 1, nil, 0);
procpriority(up, PriNormal, 0);
for(;;){
t1 = fastticks(nil);
flush = 0;
fn = nil;
switch(rc){
case ICCTRAP:
s = splhi();
machp()->MMU.cr2 = up->ac->MMU.cr2;
DBG("runacore: trap %llu cr2 %#llx ureg %#p\n",
ureg->type, machp()->MMU.cr2, ureg);
switch(ureg->type){
case IdtIPI:
if(up->procctl || up->nnote)
notify(up->dbgreg);
if(up->ac == nil)
goto ToTC;
kexit(up->dbgreg);
break;
case IdtNM:
case IdtMF:
case IdtXF:
/* these are handled in the AC;
* If we get here, they left in m->NIX.icc->data
* a note to be posted to the process.
* Post it, and make the vector a NOP.
*/
n = up->ac->NIX.icc->note;
if(n != nil)
postnote(up, 1, n, NDebug);
ureg->type = IdtIPI; /* NOP */
break;
default:
cr3put(machp()->MMU.pml4->pa);
if(0 && ureg->type == IdtPF){
print("before PF:\n");
print("AC:\n");
dumpptepg(4, up->ac->MMU.pml4->pa);
print("\n%s:\n", rolename[NIXTC]);
dumpptepg(4, machp()->MMU.pml4->pa);
}
trap(ureg);
}
splx(s);
flush = 1;
fn = actrapret;
break;
case ICCSYSCALL:
DBG("runacore: syscall ax %#llx ureg %#p\n",
ureg->ax, ureg);
cr3put(machp()->MMU.pml4->pa);
//syscall(ureg->ax, ureg);
flush = 1;
fn = acsysret;
if(0)
if(up->nqtrap > 2 || up->nsyscall > 1)
goto ToTC;
if(up->ac == nil)
goto ToTC;
break;
default:
panic("runacore: unexpected rc = %d", rc);
}
up->tctime += fastticks2us(fastticks(nil) - t1);
procpriority(up, PriExtra, 1);
rc = runac(up->ac, fn, flush, nil, 0);
procpriority(up, PriNormal, 0);
}
ToTC:
/*
* to procctl, then syscall, to
* be back in the TC
*/
DBG("runacore: up %#p: return\n", up);
}
void
mmuinit(void)
{
uint8_t *p;
Page *page;
uint64_t o, pa, r, sz;
archmmu();
DBG("mach%d: %#p pml4 %#p npgsz %d\n", machp()->machno, machp(), machp()->MMU.pml4, sys->npgsz);
if(machp()->machno != 0) {
/* NIX: KLUDGE: Has to go when each mach is using
* its own page table
*/
p = UINT2PTR(machp()->stack);
p += MACHSTKSZ;
memmove(p, UINT2PTR(mach0pml4.va), PTSZ);
machp()->MMU.pml4 = &machp()->MMU.pml4kludge;
machp()->MMU.pml4->va = PTR2UINT(p);
machp()->MMU.pml4->pa = PADDR(p);
machp()->MMU.pml4->daddr = mach0pml4.daddr; /* # of user mappings in pml4 */
r = rdmsr(Efer);
r |= Nxe;
wrmsr(Efer, r);
cr3put(machp()->MMU.pml4->pa);
DBG("m %#p pml4 %#p\n", machp(), machp()->MMU.pml4);
return;
}
page = &mach0pml4;
page->pa = cr3get();
page->va = PTR2UINT(KADDR(page->pa));
machp()->MMU.pml4 = page;
r = rdmsr(Efer);
r |= Nxe;
wrmsr(Efer, r);
/*
* Set up the various kernel memory allocator limits:
* pmstart/pmend bound the unused physical memory;
* vmstart/vmend bound the total possible virtual memory
* used by the kernel;
* vmunused is the highest virtual address currently mapped
* and used by the kernel;
* vmunmapped is the highest virtual address currently
* mapped by the kernel.
* Vmunused can be bumped up to vmunmapped before more
* physical memory needs to be allocated and mapped.
*
* This is set up here so meminit can map appropriately.
*/
o = sys->pmstart;
sz = ROUNDUP(o, 4*MiB) - o;
pa = asmalloc(0, sz, 1, 0);
if(pa != o)
panic("mmuinit: pa %#llux memstart %#llux\n", pa, o);
sys->pmstart += sz;
sys->vmstart = KSEG0;
sys->vmunused = sys->vmstart + ROUNDUP(o, 4*KiB);
sys->vmunmapped = sys->vmstart + o + sz;
sys->vmend = sys->vmstart + TMFM;
print("mmuinit: vmstart %#p vmunused %#p vmunmapped %#p vmend %#p\n",
sys->vmstart, sys->vmunused, sys->vmunmapped, sys->vmend);
/*
* Set up the map for PD entry access by inserting
* the relevant PDP entry into the PD. It's equivalent
* to PADDR(sys->pd)|PteRW|PteP.
*
*/
sys->pd[PDX(PDMAP)] = sys->pdp[PDPX(PDMAP)] & ~(PteD|PteA);
print("sys->pd %#p %#p\n", sys->pd[PDX(PDMAP)], sys->pdp[PDPX(PDMAP)]);
assert((pdeget(PDMAP) & ~(PteD|PteA)) == (PADDR(sys->pd)|PteRW|PteP));
dumpmmuwalk(KZERO);
mmuphysaddr(PTR2UINT(end));
}
