/* from 386 */ void* vmap(uintptr pa, usize size) { uintptr pae, va; usize o, osize; /* * XXX - replace with new vm stuff. * Crock after crock - the first 4MB is mapped with 2MB pages * so catch that and return good values because the current mmukmap * will fail. */ if(pa+size < 4*MiB) return UINT2PTR(kseg0|pa); osize = size; o = pa & (BY2PG-1); pa -= o; size += o; size = ROUNDUP(size, BY2PG); va = kseg0|pa; pae = mmukmap(va, pa, size); if(pae == 0 || pae-size != pa) panic("vmap(%#p, %ld) called from %#p: mmukmap fails %#p", pa+o, osize, getcallerpc(&pa), pae); return UINT2PTR(va+o); }
void* xspanalloc(ulong size, int align, ulong span) { uintptr a, v, t; a = PTR2UINT(xalloc(size+align+span)); if(a == 0) panic("xspanalloc: %lud %d %lux\n", size, align, span); if(span > 2) { v = (a + span) & ~(span-1); t = v - a; if(t > 0) xhole(PADDR(UINT2PTR(a)), t); t = a + span - v; if(t > 0) xhole(PADDR(UINT2PTR(v+size+align)), t); } else v = a; if(align > 1) v = (v + align) & ~(align-1); return (void*)v; }
static void mmuptpfree(Proc* proc, int clear) { int l; PTE *pte; Page **last, *page; for(l = 1; l < 4; l++) { last = &proc->MMU.mmuptp[l]; if(*last == nil) continue; for(page = *last; page != nil; page = page->next) { //what is right here? 2 or 1? if(l <= 2 && clear) memset(UINT2PTR(page->va), 0, PTSZ); pte = UINT2PTR(page->prev->va); pte[page->daddr] = 0; last = &page->next; } *last = proc->MMU.mmuptp[0]; proc->MMU.mmuptp[0] = proc->MMU.mmuptp[l]; proc->MMU.mmuptp[l] = nil; } machp()->MMU.pml4->daddr = 0; }
/* tear down the identity map we created in assembly. ONLY do this after all the * APs have started up (and you know they've done so. But you must do it BEFORE * you create address spaces for procs, i.e. userinit() */ static void teardownidmap(Mach *m) { int i; uintptr_t va = 0; PTE *p; /* loop on the level 2 because we should not assume we know * how many there are But stop after 1G no matter what, and * report if there were that many, as that is odd. */ for(i = 0; i < 512; i++, va += BIGPGSZ) { if (mmuwalk(UINT2PTR(m->pml4->va), va, 1, &p, nil) != 1) break; if (! *p) break; iprint("teardown: va %p, pte %p\n", (void *)va, p); *p = 0; } iprint("Teardown: zapped %d PML1 entries\n", i); for(i = 2; i < 4; i++) { if (mmuwalk(UINT2PTR(m->pml4->va), 0, i, &p, nil) != i) { iprint("weird; 0 not mapped at %d\n", i); continue; } iprint("teardown: zap %p at level %d\n", p, i); if (p) *p = 0; } }
int mmuwalk(PTE* pml4, uintptr_t va, int level, PTE** ret, uint64_t (*alloc)(usize)) { int l; uintmem pa; PTE *pte; Mpl pl; pl = splhi(); if(DBGFLG > 1) DBG("mmuwalk%d: va %#p level %d\n", machp()->machno, va, level); pte = &pml4[PTLX(va, 3)]; for(l = 3; l >= 0; l--) { if(l == level) break; if(!(*pte & PteP)) { if(alloc == nil) break; pa = alloc(PTSZ); if(pa == ~0) return -1; memset(UINT2PTR(KADDR(pa)), 0, PTSZ); *pte = pa|PteRW|PteP; } else if(*pte & PtePS) break; pte = UINT2PTR(KADDR(PPN(*pte))); pte += PTLX(va, l-1); } *ret = pte; splx(pl); return l; }
void mmuswitch(Proc* proc) { PTE *pte; Page *page; Mpl pl; pl = splhi(); if(proc->newtlb) { /* * NIX: We cannot clear our page tables if they are going to * be used in the AC */ if(proc->ac == nil) mmuptpfree(proc, 1); proc->newtlb = 0; } if(machp()->MMU.pml4->daddr) { memset(UINT2PTR(machp()->MMU.pml4->va), 0, machp()->MMU.pml4->daddr*sizeof(PTE)); machp()->MMU.pml4->daddr = 0; } pte = UINT2PTR(machp()->MMU.pml4->va); for(page = proc->MMU.mmuptp[3]; page != nil; page = page->next) { pte[page->daddr] = PPN(page->pa)|PteU|PteRW|PteP; if(page->daddr >= machp()->MMU.pml4->daddr) machp()->MMU.pml4->daddr = page->daddr+1; page->prev = machp()->MMU.pml4; } tssrsp0(machp(), STACKALIGN(PTR2UINT(proc->kstack+KSTACK))); cr3put(machp()->MMU.pml4->pa); splx(pl); }
void* asmbootalloc(usize size) { uintptr_t va; assert(sys->vmunused+size <= sys->vmunmapped); va = sys->vmunused; sys->vmunused += size; memset(UINT2PTR(va), 0, size); return UINT2PTR(va); }
static void bootargs(uintptr base) { int i; ulong ssize; char **av, *p; /* * Push the boot args onto the stack. * The initial value of the user stack must be such * that the total used is larger than the maximum size * of the argument list checked in syscall. */ i = oargblen+1; p = UINT2PTR(STACKALIGN(base + BY2PG - sizeof(Tos) - i)); memmove(p, oargb, i); /* * Now push the argv pointers. * The code jumped to by touser in lproc.s expects arguments * main(char* argv0, ...) * and calls * startboot("/boot/boot", &argv0) * not the usual (int argc, char* argv[]) */ av = (char**)(p - (oargc+1)*sizeof(char*)); ssize = base + BY2PG - PTR2UINT(av); for(i = 0; i < oargc; i++) *av++ = (oargv[i] - oargb) + (p - base) + (USTKTOP - BY2PG); *av = nil; sp = USTKTOP - ssize; }
int main(int argc, char * argv[]) { pthread_t id[4]; int i; intptr_t result = 0; /* Create a few threads and then exit. */ for (i = 0; i < 4; i++) { assert(pthread_create(&id[i], NULL, func, UINT2PTR(i)) == 0); } /* * Let threads exit before we join them. * We should still retrieve the exit code for the threads. */ Sleep(1000); for (i = 0; i < 4; i++) { assert(pthread_join(id[i], (void **) &result) == 0); assert((int) result == i); } /* Success. */ return 0; }
void bootargs(uintptr base) { int i; ulong ssize; char **av, *p; /* * Push the boot args onto the stack. * Make sure the validaddr check in syscall won't fail * because there are fewer than the maximum number of * args by subtracting sizeof(up->arg). */ i = oargblen+1; p = UINT2PTR(STACKALIGN(base + BIGPGSZ - sizeof(up->arg) - i)); memmove(p, oargb, i); /* * Now push argc and the argv pointers. * This isn't strictly correct as the code jumped to by * touser in init9.[cs] calls startboot (port/initcode.c) which * expects arguments * startboot(char* argv0, char* argv[]) * not the usual (int argc, char* argv[]), but argv0 is * unused so it doesn't matter (at the moment...). */ av = (char**)(p - (oargc+2)*sizeof(char*)); ssize = base + BIGPGSZ - PTR2UINT(av); *av++ = (char*)oargc; for(i = 0; i < oargc; i++) *av++ = (oargv[i] - oargb) + (p - base) + (USTKTOP - BIGPGSZ); *av = nil; sp = USTKTOP - ssize; }
int main(int argc, char * argv[]) { pthread_t id[NUMTHREADS]; int i; /* Create a few threads and then exit. */ for (i = 0; i < NUMTHREADS; i++) { assert(pthread_create(&id[i], NULL, func, UINT2PTR(i)) == 0); } /* Some threads will finish before they are detached, some after. */ Sleep(NUMTHREADS/2 * 10 + 50); for (i = 0; i < NUMTHREADS; i++) { assert(pthread_detach(id[i]) == 0); } Sleep(NUMTHREADS * 10 + 100); /* * Check that all threads are now invalid. * This relies on unique thread IDs - e.g. works with * pthreads-w32 or Solaris, but may not work for Linux, BSD etc. */ for (i = 0; i < NUMTHREADS; i++) { assert(pthread_kill(id[i], 0) == ESRCH); } /* Success. */ return 0; }
void* sysexecregs(uintptr_t entry, uint32_t ssize, void *tos) { Proc *up = externup(); uintptr_t *sp; Ureg *ureg; // We made sure it was correctly aligned in sysexecstack, above. if (ssize & 0xf) { print("your stack is wrong: stacksize is not 16-byte aligned: %d\n", ssize); panic("misaligned stack in sysexecregs"); } sp = (uintptr_t*)(USTKTOP - ssize); ureg = up->dbgreg; ureg->sp = PTR2UINT(sp); ureg->ip = entry; ureg->type = 64; /* fiction for acid */ ureg->dx = (uintptr_t)tos; /* * return the address of kernel/user shared data * (e.g. clock stuff) */ return UINT2PTR(USTKTOP-sizeof(Tos)); }
uintmem mmuphysaddr(uintptr_t va) { int l; PTE *pte; uintmem mask, pa; /* * Given a VA, find the PA. * This is probably not the right interface, * but will do as an experiment. Usual * question, should va be void* or uintptr? */ l = mmuwalk(UINT2PTR(machp()->MMU.pml4->va), va, 0, &pte, nil); DBG("physaddr: va %#p l %d\n", va, l); if(l < 0) return ~0; mask = PGLSZ(l)-1; pa = (*pte & ~mask) + (va & mask); DBG("physaddr: l %d va %#p pa %#llux\n", l, va, pa); return pa; }
/* * run an arbitrary function with arbitrary args on an ap core * first argument is always pml4 for process * make a field and a struct for the args cache line. * * Returns the return-code for the ICC or -1 if the process was * interrupted while issuing the ICC. */ int runac(Mach *mp, APfunc func, int flushtlb, void *a, int32_t n) { Proc *up = externup(); uint8_t *dpg, *spg; if (n > sizeof(mp->NIX.icc->data)) panic("runac: args too long"); if(mp->online == 0) panic("Bad core"); if(mp->proc != nil && mp->proc != up) panic("runapfunc: mach is busy with another proc?"); memmove(mp->NIX.icc->data, a, n); if(flushtlb){ DBG("runac flushtlb: cppml4 %#p %#p\n", mp->MMU.pml4->pa, machp()->MMU.pml4->pa); dpg = UINT2PTR(mp->MMU.pml4->va); spg = UINT2PTR(machp()->MMU.pml4->va); /* We should copy less: * memmove(dgp, spg, machp()->MMU.pml4->daddr * sizeof(PTE)); */ memmove(dpg, spg, PTSZ); if(0){ print("runac: upac pml4 %#p\n", up->ac->MMU.pml4->pa); dumpptepg(4, up->ac->MMU.pml4->pa); } } mp->NIX.icc->flushtlb = flushtlb; mp->NIX.icc->rc = ICCOK; DBG("runac: exotic proc on cpu%d\n", mp->machno); if(waserror()){ qunlock(&up->debug); nexterror(); } qlock(&up->debug); up->nicc++; up->state = Exotic; up->psstate = 0; qunlock(&up->debug); poperror(); mfence(); mp->NIX.icc->fn = func; sched(); return mp->NIX.icc->rc; }
int xmerge(void *vp, void *vq) { Xhdr *p, *q; p = UINT2PTR((PTR2UINT(vp) - offsetof(Xhdr, data[0]))); q = UINT2PTR((PTR2UINT(vq) - offsetof(Xhdr, data[0]))); if(p->magix != Magichole || q->magix != Magichole) { xsummary(); panic("xmerge(%#p, %#p) bad magic %#lux, %#lux\n", vp, vq, p->magix, q->magix); } if((uchar*)p+p->size == (uchar*)q) { p->size += q->size; return 1; } return 0; }
void mmuflushtlb(uint64_t u) { machp()->tlbpurge++; if(machp()->MMU.pml4->daddr) { memset(UINT2PTR(machp()->MMU.pml4->va), 0, machp()->MMU.pml4->daddr*sizeof(PTE)); machp()->MMU.pml4->daddr = 0; } cr3put(machp()->MMU.pml4->pa); }
void debugtouser(void *va) { Mach *m = machp(); uintptr_t uva = (uintptr_t) va; PTE *pte, *pml4; pml4 = UINT2PTR(m->pml4->va); mmuwalk(pml4, uva, 0, &pte, nil); iprint("va %p m %p m>pml4 %p m->pml4->va %p pml4 %p PTE 0x%lx\n", va, m, m->pml4, m->pml4->va, (void *)pml4, *pte); }
void xfree(void *p) { Xhdr *x; x = UINT2PTR((PTR2UINT(p) - offsetof(Xhdr, data[0]))); if(x->magix != Magichole) { xsummary(); panic("xfree(%#p) %#ux != %#lux", p, Magichole, x->magix); } xhole(PADDR(x), x->size); }
void* alloczio(Segment *s, int32_t len) { Zseg *zs; uintptr_t va; zs = &s->zseg; va = zmapalloc(zs->map, len); if(va == 0ULL) error("kernel zero copy segment exhausted"); return UINT2PTR(va); }
/* * KSEG0 maps low memory. * KSEG2 maps almost all memory, but starting at an address determined * by the address space map (see asm.c). * Thus, almost everything in physical memory is already mapped, but * there are things that fall in the gap * (acpi tables, device memory-mapped registers, etc.) * for those things, we also want to disable caching. * vmap() is required to access them. */ void* vmap(uintptr_t pa, usize size) { Proc *up = externup(); uintptr_t va; usize o, sz; DBG("vmap(%#p, %lud) pc=%#p\n", pa, size, getcallerpc(&pa)); if(machp()->machno != 0) panic("vmap"); /* * This is incomplete; the checks are not comprehensive * enough. * Sometimes the request is for an already-mapped piece * of low memory, in which case just return a good value * and hope that a corresponding vunmap of the address * will have the same address. * To do this properly will require keeping track of the * mappings; perhaps something like kmap, but kmap probably * can't be used early enough for some of the uses. */ if(pa+size < 1ull*MiB) return KADDR(pa); if(pa < 1ull*MiB) return nil; /* * Might be asking for less than a page. * This should have a smaller granularity if * the page size is large. */ o = pa & ((1<<PGSHFT)-1); pa -= o; sz = ROUNDUP(size+o, PGSZ); if(pa == 0){ print("vmap(0, %lud) pc=%#p\n", size, getcallerpc(&pa)); return nil; } ilock(&vmaplock); if((va = vmapalloc(sz)) == 0 || pdmap(pa, PtePCD|PteRW, va, sz) < 0){ iunlock(&vmaplock); return nil; } iunlock(&vmaplock); DBG("vmap(%#p, %lud) => %#p\n", pa+o, size, va+o); return UINT2PTR(va + o); }
static Page* mmuptpalloc(void) { void* va; Page *page; /* * Do not really need a whole Page structure, * but it makes testing this out a lot easier. * Could keep a cache and free excess. * Have to maintain any fiction for pexit? */ lock(&mmuptpfreelist.l); if((page = mmuptpfreelist.next) != nil) { mmuptpfreelist.next = page->next; mmuptpfreelist.ref--; unlock(&mmuptpfreelist.l); if(page->ref++ != 0) panic("mmuptpalloc ref\n"); page->prev = page->next = nil; memset(UINT2PTR(page->va), 0, PTSZ); if(page->pa == 0) panic("mmuptpalloc: free page with pa == 0"); return page; } unlock(&mmuptpfreelist.l); if((page = malloc(sizeof(Page))) == nil) { print("mmuptpalloc Page\n"); return nil; } if((va = mallocalign(PTSZ, PTSZ, 0, 0)) == nil) { print("mmuptpalloc va\n"); free(page); return nil; } page->va = PTR2UINT(va); page->pa = PADDR(va); page->ref = 1; if(page->pa == 0) panic("mmuptpalloc: no pa"); return page; }
void* KADDR(uintptr_t pa) { uint8_t* va; va = UINT2PTR(pa); if(pa < TMFM) { km++; return KSEG0+va; } assert(pa < KSEG2); k2++; return KSEG2+va; }
void dumpmmuwalk(uint64_t addr) { int l; PTE *pte, *pml4; pml4 = UINT2PTR(machp()->MMU.pml4->va); if((l = mmuwalk(pml4, addr, 3, &pte, nil)) >= 0) print("cpu%d: mmu l%d pte %#p = %llux\n", machp()->machno, l, pte, *pte); if((l = mmuwalk(pml4, addr, 2, &pte, nil)) >= 0) print("cpu%d: mmu l%d pte %#p = %llux\n", machp()->machno, l, pte, *pte); if((l = mmuwalk(pml4, addr, 1, &pte, nil)) >= 0) print("cpu%d: mmu l%d pte %#p = %llux\n", machp()->machno, l, pte, *pte); if((l = mmuwalk(pml4, addr, 0, &pte, nil)) >= 0) print("cpu%d: mmu l%d pte %#p = %llux\n", machp()->machno, l, pte, *pte); }
static void mmul2empty(Proc* proc, int clear) { PTE *l1; Page **l2, *page; l1 = m->mmul1; l2 = &proc->mmul2; for(page = *l2; page != nil; page = page->next){ if(clear) memset(UINT2PTR(page->va), 0, BY2PG); l1[page->daddr] = Fault; l2 = &page->next; } *l2 = proc->mmul2cache; proc->mmul2cache = proc->mmul2; proc->mmul2 = nil; }
static void checkpte(uintmem ppn, void *a) { Proc *up = externup(); int l; PTE *pte, *pml4; uint64_t addr; char buf[240], *s; addr = PTR2UINT(a); pml4 = UINT2PTR(machp()->pml4->va); pte = 0; s = buf; *s = 0; if((l = mmuwalk(pml4, addr, 3, &pte, nil)) < 0 || (*pte&PteP) == 0) goto Panic; s = seprint(buf, buf+sizeof buf, "check3: l%d pte %#p = %llux\n", l, pte, pte?*pte:~0); if((l = mmuwalk(pml4, addr, 2, &pte, nil)) < 0 || (*pte&PteP) == 0) goto Panic; s = seprint(s, buf+sizeof buf, "check2: l%d pte %#p = %llux\n", l, pte, pte?*pte:~0); if(*pte&PtePS) return; if((l = mmuwalk(pml4, addr, 1, &pte, nil)) < 0 || (*pte&PteP) == 0) goto Panic; seprint(s, buf+sizeof buf, "check1: l%d pte %#p = %llux\n", l, pte, pte?*pte:~0); return; Panic: seprint(s, buf+sizeof buf, "checkpte: l%d addr %#p ppn %#ullx kaddr %#p pte %#p = %llux", l, a, ppn, KADDR(ppn), pte, pte?*pte:~0); print("%s\n", buf); seprint(buf, buf+sizeof buf, "start %#ullx unused %#ullx" " unmap %#ullx end %#ullx\n", sys->vmstart, sys->vmunused, sys->vmunmapped, sys->vmend); panic("%s", buf); }
void* sysexecregs(uintptr entry, ulong ssize, ulong nargs) { uintptr *sp; Ureg *ureg; sp = (uintptr*)(USTKTOP - ssize); *--sp = nargs; ureg = up->dbgreg; ureg->sp = PTR2UINT(sp); ureg->ip = entry; ureg->type = 64; /* fiction for acid */ /* * return the address of kernel/user shared data * (e.g. clock stuff) */ return UINT2PTR(USTKTOP-sizeof(Tos)); }
int main(int argc, char * argv[]) { pthread_t id[4]; int i; intptr_t result = 0; /* Create a few threads and then exit. */ for (i = 0; i < 4; i++) { assert(pthread_create(&id[i], NULL, func, UINT2PTR(i)) == 0); } for (i = 0; i < 4; i++) { assert(pthread_join(id[i], (void **) &result) == 0); assert((int) result == i); } /* Success. */ return 0; }
static void bootargs(uintptr base) { int i; ulong ssize; char **av, *p; /* * Push the boot args onto the stack. * The initial value of the user stack must be such * that the total used is larger than the maximum size * of the argument list checked in syscall. */ i = oargblen+1; p = UINT2PTR(STACKALIGN(base + BY2PG - Ustkheadroom - i)); memmove(p, oargb, i); /* * Now push argc and the argv pointers. * This isn't strictly correct as the code jumped to by * touser in init9.s calls startboot (port/initcode.c) which * expects arguments * startboot(char *argv0, char **argv) * not the usual (int argc, char* argv[]), but argv0 is * unused so it doesn't matter (at the moment...). */ av = (char**)(p - (oargc+2)*sizeof(char*)); ssize = base + BY2PG - PTR2UINT(av); *av++ = (char*)oargc; for(i = 0; i < oargc; i++) *av++ = (oargv[i] - oargb) + (p - base) + (USTKTOP - BY2PG); *av = nil; /* * Leave space for the return PC of the * caller of initcode. */ sp = USTKTOP - ssize - sizeof(void*); }
void dumpptepg(int lvl, uintptr_t pa) { PTE *pte; int tab, i; tab = 4 - lvl; pte = UINT2PTR(KADDR(pa)); for(i = 0; i < PTSZ/sizeof(PTE); i++) if(pte[i] & PteP) { tabs(tab); print("l%d %#p[%#05x]: %#ullx\n", lvl, pa, i, pte[i]); /* skip kernel mappings */ if((pte[i]&PteU) == 0) { tabs(tab+1); print("...kern...\n"); continue; } if(lvl > 2) dumpptepg(lvl-1, PPN(pte[i])); } }
void userinit(void) { Proc *p; Segment *s; KMap *k; Page *pg; p = newproc(); p->pgrp = newpgrp(); p->egrp = smalloc(sizeof(Egrp)); p->egrp->ref = 1; p->fgrp = dupfgrp(nil); p->rgrp = newrgrp(); p->procmode = 0640; kstrdup(&eve, ""); kstrdup(&p->text, "*init*"); kstrdup(&p->user, eve); /* * Kernel Stack * * N.B. make sure there's enough space for syscall to check * for valid args and * space for gotolabel's return PC * AMD64 stack must be quad-aligned. */ p->sched.pc = PTR2UINT(init0); p->sched.sp = PTR2UINT(p->kstack+KSTACK-sizeof(up->arg)-sizeof(uintptr)); p->sched.sp = STACKALIGN(p->sched.sp); /* * User Stack * * Technically, newpage can't be called here because it * should only be called when in a user context as it may * try to sleep if there are no pages available, but that * shouldn't be the case here. */ s = newseg(SG_STACK, USTKTOP-USTKSIZE, USTKSIZE/BIGPGSZ); p->seg[SSEG] = s; pg = newpage(1, 0, USTKTOP-BIGPGSZ, BIGPGSZ, -1); segpage(s, pg); k = kmap(pg); bootargs(VA(k)); kunmap(k); /* * Text */ s = newseg(SG_TEXT, UTZERO, 1); s->flushme++; p->seg[TSEG] = s; pg = newpage(1, 0, UTZERO, BIGPGSZ, -1); memset(pg->cachectl, PG_TXTFLUSH, sizeof(pg->cachectl)); segpage(s, pg); k = kmap(s->map[0]->pages[0]); memmove(UINT2PTR(VA(k)), initcode, sizeof initcode); kunmap(k); ready(p); }