/*
* Grow the GDT.
*/
void
gdt_grow(int which)
{
size_t old_len, new_len;
CPU_INFO_ITERATOR cii;
struct cpu_info *ci;
struct vm_page *pg;
vaddr_t va;
old_len = gdt_size[which] * sizeof(gdt[0]);
gdt_size[which] <<= 1;
new_len = old_len << 1;
#ifdef XEN
if (which != 0) {
size_t max_len = MAXGDTSIZ * sizeof(gdt[0]);
if (old_len == 0) {
gdt_size[which] = MINGDTSIZ;
new_len = gdt_size[which] * sizeof(gdt[0]);
}
for(va = (vaddr_t)(cpu_info_primary.ci_gdt) + old_len + max_len;
va < (vaddr_t)(cpu_info_primary.ci_gdt) + new_len + max_len;
va += PAGE_SIZE) {
while ((pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO))
== NULL) {
uvm_wait("gdt_grow");
}
pmap_kenter_pa(va, VM_PAGE_TO_PHYS(pg),
VM_PROT_READ | VM_PROT_WRITE);
}
return;
}
#endif
for (CPU_INFO_FOREACH(cii, ci)) {
for (va = (vaddr_t)(ci->ci_gdt) + old_len;
va < (vaddr_t)(ci->ci_gdt) + new_len;
va += PAGE_SIZE) {
while ((pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO)) ==
NULL) {
uvm_wait("gdt_grow");
}
pmap_kenter_pa(va, VM_PAGE_TO_PHYS(pg),
VM_PROT_READ | VM_PROT_WRITE);
}
}
pmap_update(pmap_kernel());
}
/*
* Allocate shadow GDT for a slave CPU.
*/
void
gdt_alloc_cpu(struct cpu_info *ci)
{
int max_len = MAXGDTSIZ * sizeof(gdt[0]);
int min_len = MINGDTSIZ * sizeof(gdt[0]);
struct vm_page *pg;
vaddr_t va;
ci->ci_gdt = (union descriptor *)uvm_km_alloc(kernel_map, max_len,
0, UVM_KMF_VAONLY);
for (va = (vaddr_t)ci->ci_gdt; va < (vaddr_t)ci->ci_gdt + min_len;
va += PAGE_SIZE) {
while ((pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO))
== NULL) {
uvm_wait("gdt_alloc_cpu");
}
pmap_kenter_pa(va, VM_PAGE_TO_PHYS(pg),
VM_PROT_READ | VM_PROT_WRITE);
}
pmap_update(pmap_kernel());
memset(ci->ci_gdt, 0, min_len);
memcpy(ci->ci_gdt, gdt, gdt_count[0] * sizeof(gdt[0]));
setsegment(&ci->ci_gdt[GCPU_SEL].sd, ci, 0xfffff,
SDT_MEMRWA, SEL_KPL, 1, 1);
}
/*
* Initialize the GDT.
*/
void
gdt_init(void)
{
char *old_gdt;
struct vm_page *pg;
vaddr_t va;
struct cpu_info *ci = &cpu_info_primary;
gdt_next = 0;
gdt_free = GNULL_SEL;
old_gdt = gdtstore;
gdtstore = (char *)uvm_km_valloc(kernel_map, MAXGDTSIZ);
for (va = (vaddr_t)gdtstore; va < (vaddr_t)gdtstore + MAXGDTSIZ;
va += PAGE_SIZE) {
pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO);
if (pg == NULL) {
panic("gdt_init: no pages");
}
pmap_kenter_pa(va, VM_PAGE_TO_PHYS(pg),
VM_PROT_READ | VM_PROT_WRITE);
}
bcopy(old_gdt, gdtstore, DYNSEL_START);
ci->ci_gdt = gdtstore;
set_sys_segment(GDT_ADDR_SYS(gdtstore, GLDT_SEL), ldtstore,
LDT_SIZE - 1, SDT_SYSLDT, SEL_KPL, 0);
gdt_init_cpu(ci);
}
/*
* Initialize the GDT subsystem. Called from autoconf().
*/
void
gdt_init()
{
size_t max_len, min_len;
struct vm_page *pg;
vaddr_t va;
struct cpu_info *ci = &cpu_info_primary;
simple_lock_init(&gdt_simplelock);
lockinit(&gdt_lock_store, PZERO, "gdtlck", 0, 0);
max_len = MAXGDTSIZ * sizeof(union descriptor);
min_len = MINGDTSIZ * sizeof(union descriptor);
gdt_size = MINGDTSIZ;
gdt_count = NGDT;
gdt_next = NGDT;
gdt_free = GNULL_SEL;
gdt = (union descriptor *)uvm_km_valloc(kernel_map, max_len);
for (va = (vaddr_t)gdt; va < (vaddr_t)gdt + min_len; va += PAGE_SIZE) {
pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO);
if (pg == NULL)
panic("gdt_init: no pages");
pmap_kenter_pa(va, VM_PAGE_TO_PHYS(pg),
VM_PROT_READ | VM_PROT_WRITE);
}
bcopy(bootstrap_gdt, gdt, NGDT * sizeof(union descriptor));
ci->ci_gdt = gdt;
setsegment(&ci->ci_gdt[GCPU_SEL].sd, ci, sizeof(struct cpu_info)-1,
SDT_MEMRWA, SEL_KPL, 0, 0);
gdt_init_cpu(ci);
}
/*
* Grow the GDT.
*/
void
gdt_grow()
{
size_t old_len, new_len;
CPU_INFO_ITERATOR cii;
struct cpu_info *ci;
struct vm_page *pg;
vaddr_t va;
old_len = gdt_size * sizeof(union descriptor);
gdt_size <<= 1;
new_len = old_len << 1;
CPU_INFO_FOREACH(cii, ci) {
for (va = (vaddr_t)(ci->ci_gdt) + old_len;
va < (vaddr_t)(ci->ci_gdt) + new_len;
va += PAGE_SIZE) {
while (
(pg =
uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO)) ==
NULL) {
uvm_wait("gdt_grow");
}
pmap_kenter_pa(va, VM_PAGE_TO_PHYS(pg),
VM_PROT_READ | VM_PROT_WRITE);
}
}
}
/*
* Initialize the GDT subsystem. Called from autoconf().
*/
void
gdt_init(void)
{
struct vm_page *pg;
vaddr_t va;
struct cpu_info *ci = &cpu_info_primary;
gdt_next = NGDT;
gdt_free = GNULL_SEL;
gdt = (union descriptor *)uvm_km_valloc(kernel_map, MAXGDTSIZ);
for (va = (vaddr_t)gdt; va < (vaddr_t)gdt + MAXGDTSIZ;
va += PAGE_SIZE) {
pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO);
if (pg == NULL)
panic("gdt_init: no pages");
pmap_kenter_pa(va, VM_PAGE_TO_PHYS(pg),
PROT_READ | PROT_WRITE);
}
bcopy(bootstrap_gdt, gdt, NGDT * sizeof(union descriptor));
ci->ci_gdt = gdt;
setsegment(&ci->ci_gdt[GCPU_SEL].sd, ci, sizeof(struct cpu_info)-1,
SDT_MEMRWA, SEL_KPL, 0, 0);
gdt_init_cpu(ci);
}
void
rumptest_busypage()
{
struct lwp *newl;
int rv;
cv_init(&tcv, "napina");
uobj = uao_create(1, 0);
mutex_enter(uobj->vmobjlock);
testpg = uvm_pagealloc(uobj, 0, NULL, 0);
mutex_exit(uobj->vmobjlock);
if (testpg == NULL)
panic("couldn't create vm page");
rv = kthread_create(PRI_NONE, KTHREAD_MUSTJOIN | KTHREAD_MPSAFE, NULL,
thread, NULL, &newl, "jointest");
if (rv)
panic("thread creation failed: %d", rv);
mutex_enter(uobj->vmobjlock);
while (!threadrun)
cv_wait(&tcv, uobj->vmobjlock);
uvm_page_unbusy(&testpg, 1);
mutex_exit(uobj->vmobjlock);
rv = kthread_join(newl);
if (rv)
panic("thread join failed: %d", rv);
}
int
uvm_loanbreak_anon(struct vm_anon *anon, struct uvm_object *uobj)
{
struct vm_page *pg;
KASSERT(mutex_owned(anon->an_lock));
KASSERT(uobj == NULL || mutex_owned(uobj->vmobjlock));
/* get new un-owned replacement page */
pg = uvm_pagealloc(NULL, 0, NULL, 0);
if (pg == NULL) {
return ENOMEM;
}
/* copy old -> new */
uvm_pagecopy(anon->an_page, pg);
/* force reload */
pmap_page_protect(anon->an_page, VM_PROT_NONE);
mutex_enter(&uvm_pageqlock); /* KILL loan */
anon->an_page->uanon = NULL;
/* in case we owned */
anon->an_page->pqflags &= ~PQ_ANON;
if (uobj) {
/* if we were receiver of loan */
anon->an_page->loan_count--;
} else {
/*
* we were the lender (A->K); need to remove the page from
* pageq's.
*/
uvm_pagedequeue(anon->an_page);
}
if (uobj) {
mutex_exit(uobj->vmobjlock);
}
/* install new page in anon */
anon->an_page = pg;
pg->uanon = anon;
pg->pqflags |= PQ_ANON;
uvm_pageactivate(pg);
mutex_exit(&uvm_pageqlock);
pg->flags &= ~(PG_BUSY|PG_FAKE);
UVM_PAGE_OWN(pg, NULL);
/* done! */
return 0;
}
/*
* Initialize the GDT subsystem. Called from autoconf().
*/
void
gdt_init()
{
size_t max_len, min_len;
union descriptor *old_gdt;
struct vm_page *pg;
vaddr_t va;
struct cpu_info *ci = &cpu_info_primary;
mutex_init(&gdt_lock_store, MUTEX_DEFAULT, IPL_NONE);
max_len = MAXGDTSIZ * sizeof(gdt[0]);
min_len = MINGDTSIZ * sizeof(gdt[0]);
gdt_size[0] = MINGDTSIZ;
gdt_count[0] = NGDT;
gdt_next[0] = NGDT;
gdt_free[0] = GNULL_SEL;
#ifdef XEN
max_len = max_len * 2;
gdt_size[1] = 0;
gdt_count[1] = MAXGDTSIZ;
gdt_next[1] = MAXGDTSIZ;
gdt_free[1] = GNULL_SEL;
#endif
old_gdt = gdt;
gdt = (union descriptor *)uvm_km_alloc(kernel_map, max_len,
0, UVM_KMF_VAONLY);
for (va = (vaddr_t)gdt; va < (vaddr_t)gdt + min_len; va += PAGE_SIZE) {
pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO);
if (pg == NULL) {
panic("gdt_init: no pages");
}
pmap_kenter_pa(va, VM_PAGE_TO_PHYS(pg),
VM_PROT_READ | VM_PROT_WRITE);
}
pmap_update(pmap_kernel());
memcpy(gdt, old_gdt, NGDT * sizeof(gdt[0]));
ci->ci_gdt = gdt;
setsegment(&ci->ci_gdt[GCPU_SEL].sd, ci, 0xfffff,
SDT_MEMRWA, SEL_KPL, 1, 1);
gdt_init_cpu(ci);
}
/*
* Allocate shadow GDT for a slave cpu.
*/
void
gdt_alloc_cpu(struct cpu_info *ci)
{
struct vm_page *pg;
vaddr_t va;
ci->ci_gdt = (char *)uvm_km_valloc(kernel_map, MAXGDTSIZ);
uvm_map_pageable(kernel_map, (vaddr_t)ci->ci_gdt,
(vaddr_t)ci->ci_gdt + MAXGDTSIZ, FALSE, FALSE);
for (va = (vaddr_t)ci->ci_gdt; va < (vaddr_t)ci->ci_gdt + MAXGDTSIZ;
va += PAGE_SIZE) {
pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO);
if (pg == NULL)
panic("gdt_init: no pages");
pmap_kenter_pa(va, VM_PAGE_TO_PHYS(pg),
VM_PROT_READ | VM_PROT_WRITE);
}
bzero(ci->ci_gdt, MAXGDTSIZ);
bcopy(gdtstore, ci->ci_gdt, MAXGDTSIZ);
}
/* ARGSUSED */
vaddr_t
uvm_km_alloc_poolpage1(struct vm_map *map, struct uvm_object *obj,
boolean_t waitok)
{
#if defined(__HAVE_PMAP_DIRECT)
struct vm_page *pg;
vaddr_t va;
again:
pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_USERESERVE);
if (__predict_false(pg == NULL)) {
if (waitok) {
uvm_wait("plpg");
goto again;
} else
return (0);
}
va = pmap_map_direct(pg);
if (__predict_false(va == 0))
uvm_pagefree(pg);
return (va);
#else
vaddr_t va;
int s;
/*
* NOTE: We may be called with a map that doesn't require splvm
* protection (e.g. kernel_map). However, it does not hurt to
* go to splvm in this case (since unprotected maps will never be
* accessed in interrupt context).
*
* XXX We may want to consider changing the interface to this
* XXX function.
*/
s = splvm();
va = uvm_km_kmemalloc(map, obj, PAGE_SIZE, waitok ? 0 : UVM_KMF_NOWAIT);
splx(s);
return (va);
#endif /* __HAVE_PMAP_DIRECT */
}
/*
* Allocate shadow GDT for a slave cpu.
*/
void
gdt_alloc_cpu(struct cpu_info *ci)
{
struct vm_page *pg;
vaddr_t va;
ci->ci_gdt = (union descriptor *)uvm_km_valloc(kernel_map, MAXGDTSIZ);
uvm_map_pageable(kernel_map, (vaddr_t)ci->ci_gdt,
(vaddr_t)ci->ci_gdt + MAXGDTSIZ, FALSE, FALSE);
for (va = (vaddr_t)ci->ci_gdt; va < (vaddr_t)ci->ci_gdt + MAXGDTSIZ;
va += PAGE_SIZE) {
pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO);
if (pg == NULL)
panic("gdt_init: no pages");
pmap_kenter_pa(va, VM_PAGE_TO_PHYS(pg),
PROT_READ | PROT_WRITE);
}
bzero(ci->ci_gdt, MAXGDTSIZ);
bcopy(gdt, ci->ci_gdt, MAXGDTSIZ);
setsegment(&ci->ci_gdt[GCPU_SEL].sd, ci, sizeof(struct cpu_info)-1,
SDT_MEMRWA, SEL_KPL, 0, 0);
}
vaddr_t
uvm_km_kmemalloc(struct vm_map *map, struct uvm_object *obj, vsize_t size,
int flags)
{
vaddr_t kva, loopva;
voff_t offset;
struct vm_page *pg;
int mapflags;
UVMHIST_FUNC("uvm_km_kmemalloc"); UVMHIST_CALLED(maphist);
UVMHIST_LOG(maphist," (map=%p, obj=%p, size=0x%lx, flags=%d)",
map, obj, size, flags);
KASSERT(vm_map_pmap(map) == pmap_kernel());
/*
* we cannot yet make pmap_enter() not sleep
* and thus demand that we are called with NOWAIT in that case
*/
KASSERT(!((flags & UVM_KMF_NOWAIT) && obj));
/*
* setup for call
*/
mapflags = flags & UVM_KMF_NOWAIT? UVM_FLAG_NOWAIT : 0;
mapflags |= flags & UVM_KMF_TRYLOCK;
size = round_page(size);
kva = vm_map_min(map); /* hint */
/*
* allocate some virtual space
*/
if (__predict_false(uvm_map(map, &kva, size, obj, UVM_UNKNOWN_OFFSET,
0, UVM_MAPFLAG(UVM_PROT_RW, UVM_PROT_RW, UVM_INH_NONE,
UVM_ADV_RANDOM, mapflags)) != 0)) {
UVMHIST_LOG(maphist, "<- done (no VM)",0,0,0,0);
return(0);
}
/*
* if all we wanted was VA, return now
*/
if (flags & UVM_KMF_VALLOC) {
UVMHIST_LOG(maphist,"<- done valloc (kva=0x%lx)", kva,0,0,0);
return(kva);
}
/*
* recover object offset from virtual address
*/
if (obj != NULL)
offset = kva - vm_map_min(kernel_map);
else
offset = 0;
UVMHIST_LOG(maphist, " kva=0x%lx, offset=0x%lx", kva, offset,0,0);
/*
* now allocate and map in the memory... note that we are the only ones
* whom should ever get a handle on this area of VM.
*/
loopva = kva;
while (loopva != kva + size) {
pg = uvm_pagealloc(obj, offset, NULL, 0);
if (pg) {
atomic_clearbits_int(&pg->pg_flags, PG_BUSY);
UVM_PAGE_OWN(pg, NULL);
}
if (__predict_false(pg == NULL)) {
if ((flags & UVM_KMF_NOWAIT) ||
((flags & UVM_KMF_CANFAIL) &&
uvmexp.swpgonly == uvmexp.swpages)) {
/* free everything! */
uvm_unmap(map, kva, kva + size);
return (0);
} else {
uvm_wait("km_getwait2"); /* sleep here */
continue;
}
}
/*
* map it in: note that we call pmap_enter with the map and
* object unlocked in case we are kmem_map.
*
* pager mappings that must not sleep here will incidently
* be installed using pmap_kenter_pa() and thus not sleep!
*/
if (obj == NULL) {
pmap_kenter_pa(loopva, VM_PAGE_TO_PHYS(pg),
UVM_PROT_RW);
} else {
pmap_enter(map->pmap, loopva, VM_PAGE_TO_PHYS(pg),
UVM_PROT_RW,
PMAP_WIRED | VM_PROT_READ | VM_PROT_WRITE);
}
loopva += PAGE_SIZE;
offset += PAGE_SIZE;
}
pmap_update(pmap_kernel());
UVMHIST_LOG(maphist,"<- done (kva=0x%lx)", kva,0,0,0);
return(kva);
}
/*
* cpu_startup: allocate memory for variable-sized tables,
* initialize cpu, and do autoconfiguration.
*
* This is called early in init_main.c:main(), after the
* kernel memory allocator is ready for use, but before
* the creation of processes 1,2, and mountroot, etc.
*/
void
cpu_startup()
{
caddr_t v;
int sz, i;
vsize_t size;
int base, residual;
vaddr_t minaddr, maxaddr;
char pbuf[9];
/*
* Initialize message buffer (for kernel printf).
* This is put in physical pages four through seven
* so it will always be in the same place after a
* reboot. (physical pages 0-3 are reserved by the PROM
* for its vector table and other stuff.)
* Its mapping was prepared in pmap_bootstrap().
* Also, offset some to avoid PROM scribbles.
*/
v = (caddr_t) (NBPG * 4);
msgbufaddr = (caddr_t)(v + MSGBUFOFF);
initmsgbuf(msgbufaddr, MSGBUFSIZE);
#ifdef DDB
{
extern int end[];
extern char *esym;
ddb_init(end[0], end + 1, (int*)esym);
}
#endif /* DDB */
/*
* Good {morning,afternoon,evening,night}.
*/
printf(version);
identifycpu();
fputype = FPU_NONE;
#ifdef FPU_EMULATE
printf("fpu: emulator\n");
#else
printf("fpu: no math support\n");
#endif
format_bytes(pbuf, sizeof(pbuf), ctob(physmem));
printf("total memory = %s\n", pbuf);
/*
* XXX fredette - we force a small number of buffers
* to help me debug this on my low-memory machine.
* this should go away at some point, allowing the
* normal automatic buffer-sizing to happen.
*/
bufpages = 37;
/*
* Get scratch page for dumpsys().
*/
if ((dumppage = uvm_km_alloc(kernel_map, NBPG)) == 0)
panic("startup: alloc dumppage");
/*
* Find out how much space we need, allocate it,
* and then give everything true virtual addresses.
*/
sz = (int)allocsys(NULL, NULL);
if ((v = (caddr_t)uvm_km_alloc(kernel_map, round_page(sz))) == 0)
panic("startup: no room for tables");
if (allocsys(v, NULL) - v != sz)
panic("startup: table size inconsistency");
/*
* Now allocate buffers proper. They are different than the above
* in that they usually occupy more virtual memory than physical.
*/
size = MAXBSIZE * nbuf;
if (uvm_map(kernel_map, (vaddr_t *) &buffers, round_page(size),
NULL, UVM_UNKNOWN_OFFSET, 0,
UVM_MAPFLAG(UVM_PROT_NONE, UVM_PROT_NONE, UVM_INH_NONE,
UVM_ADV_NORMAL, 0)) != 0)
panic("startup: cannot allocate VM for buffers");
minaddr = (vaddr_t)buffers;
if ((bufpages / nbuf) >= btoc(MAXBSIZE)) {
/* don't want to alloc more physical mem than needed */
bufpages = btoc(MAXBSIZE) * nbuf;
}
base = bufpages / nbuf;
residual = bufpages % nbuf;
for (i = 0; i < nbuf; i++) {
vsize_t curbufsize;
vaddr_t curbuf;
struct vm_page *pg;
/*
* Each buffer has MAXBSIZE bytes of VM space allocated. Of
* that MAXBSIZE space, we allocate and map (base+1) pages
* for the first "residual" buffers, and then we allocate
* "base" pages for the rest.
*/
curbuf = (vaddr_t) buffers + (i * MAXBSIZE);
curbufsize = NBPG * ((i < residual) ? (base+1) : base);
while (curbufsize) {
pg = uvm_pagealloc(NULL, 0, NULL, 0);
if (pg == NULL)
panic("cpu_startup: not enough memory for "
"buffer cache");
pmap_kenter_pa(curbuf, VM_PAGE_TO_PHYS(pg),
VM_PROT_READ|VM_PROT_WRITE);
curbuf += PAGE_SIZE;
curbufsize -= PAGE_SIZE;
}
}
pmap_update(pmap_kernel());
/*
* Allocate a submap for exec arguments. This map effectively
* limits the number of processes exec'ing at any time.
*/
exec_map = uvm_km_suballoc(kernel_map, &minaddr, &maxaddr,
NCARGS, VM_MAP_PAGEABLE, FALSE, NULL);
/*
* We don't use a submap for physio, and use a separate map
* for DVMA allocations. Our vmapbuf just maps pages into
* the kernel map (any kernel mapping is OK) and then the
* device drivers clone the kernel mappings into DVMA space.
*/
/*
* Finally, allocate mbuf cluster submap.
*/
mb_map = uvm_km_suballoc(kernel_map, &minaddr, &maxaddr,
nmbclusters * mclbytes, VM_MAP_INTRSAFE,
FALSE, NULL);
format_bytes(pbuf, sizeof(pbuf), ptoa(uvmexp.free));
printf("avail memory = %s\n", pbuf);
format_bytes(pbuf, sizeof(pbuf), bufpages * NBPG);
printf("using %d buffers containing %s of memory\n", nbuf, pbuf);
/*
* Allocate a virtual page (for use by /dev/mem)
* This page is handed to pmap_enter() therefore
* it has to be in the normal kernel VA range.
*/
vmmap = uvm_km_valloc_wait(kernel_map, NBPG);
/*
* Allocate dma map for devices on the bus.
*/
dvmamap = extent_create("dvmamap",
DVMA_MAP_BASE, DVMA_MAP_BASE + DVMA_MAP_AVAIL,
M_DEVBUF, 0, 0, EX_NOWAIT);
if (dvmamap == NULL)
panic("unable to allocate DVMA map");
/*
* Set up CPU-specific registers, cache, etc.
*/
initcpu();
/*
* Set up buffers, so they can be used to read disk labels.
*/
bufinit();
}
static int
uvm_loanzero(struct uvm_faultinfo *ufi, void ***output, int flags)
{
struct vm_anon *anon;
struct vm_page *pg;
struct vm_amap *amap = ufi->entry->aref.ar_amap;
UVMHIST_FUNC(__func__); UVMHIST_CALLED(loanhist);
again:
mutex_enter(&uvm_loanzero_object.vmobjlock);
/*
* first, get ahold of our single zero page.
*/
if (__predict_false((pg =
TAILQ_FIRST(&uvm_loanzero_object.memq)) == NULL)) {
while ((pg = uvm_pagealloc(&uvm_loanzero_object, 0, NULL,
UVM_PGA_ZERO)) == NULL) {
mutex_exit(&uvm_loanzero_object.vmobjlock);
uvmfault_unlockall(ufi, amap, NULL, NULL);
uvm_wait("loanzero");
if (!uvmfault_relock(ufi)) {
return (0);
}
if (amap) {
amap_lock(amap);
}
goto again;
}
/* got a zero'd page. */
pg->flags &= ~(PG_WANTED|PG_BUSY|PG_FAKE);
pg->flags |= PG_RDONLY;
mutex_enter(&uvm_pageqlock);
uvm_pageactivate(pg);
mutex_exit(&uvm_pageqlock);
UVM_PAGE_OWN(pg, NULL);
}
if ((flags & UVM_LOAN_TOANON) == 0) { /* loaning to kernel-page */
mutex_enter(&uvm_pageqlock);
pg->loan_count++;
mutex_exit(&uvm_pageqlock);
mutex_exit(&uvm_loanzero_object.vmobjlock);
**output = pg;
(*output)++;
return (1);
}
/*
* loaning to an anon. check to see if there is already an anon
* associated with this page. if so, then just return a reference
* to this object.
*/
if (pg->uanon) {
anon = pg->uanon;
mutex_enter(&anon->an_lock);
anon->an_ref++;
mutex_exit(&anon->an_lock);
mutex_exit(&uvm_loanzero_object.vmobjlock);
**output = anon;
(*output)++;
return (1);
}
/*
* need to allocate a new anon
*/
anon = uvm_analloc();
if (anon == NULL) {
/* out of swap causes us to fail */
mutex_exit(&uvm_loanzero_object.vmobjlock);
uvmfault_unlockall(ufi, amap, NULL, NULL);
return (-1);
}
anon->an_page = pg;
pg->uanon = anon;
mutex_enter(&uvm_pageqlock);
pg->loan_count++;
uvm_pageactivate(pg);
mutex_exit(&uvm_pageqlock);
mutex_exit(&anon->an_lock);
mutex_exit(&uvm_loanzero_object.vmobjlock);
**output = anon;
(*output)++;
return (1);
}
/*
* uvm_loanbreak: break loan on a uobj page
*
* => called with uobj locked
* => the page should be busy
* => return value:
* newly allocated page if succeeded
*/
struct vm_page *
uvm_loanbreak(struct vm_page *uobjpage)
{
struct vm_page *pg;
#ifdef DIAGNOSTIC
struct uvm_object *uobj = uobjpage->uobject;
#endif
KASSERT(uobj != NULL);
KASSERT(mutex_owned(&uobj->vmobjlock));
KASSERT(uobjpage->flags & PG_BUSY);
/* alloc new un-owned page */
pg = uvm_pagealloc(NULL, 0, NULL, 0);
if (pg == NULL)
return NULL;
/*
* copy the data from the old page to the new
* one and clear the fake flags on the new page (keep it busy).
* force a reload of the old page by clearing it from all
* pmaps.
* transfer dirtiness of the old page to the new page.
* then lock the page queues to rename the pages.
*/
uvm_pagecopy(uobjpage, pg); /* old -> new */
pg->flags &= ~PG_FAKE;
pmap_page_protect(uobjpage, VM_PROT_NONE);
if ((uobjpage->flags & PG_CLEAN) != 0 && !pmap_clear_modify(uobjpage)) {
pmap_clear_modify(pg);
pg->flags |= PG_CLEAN;
} else {
/* uvm_pagecopy marked it dirty */
KASSERT((pg->flags & PG_CLEAN) == 0);
/* a object with a dirty page should be dirty. */
KASSERT(!UVM_OBJ_IS_CLEAN(uobj));
}
if (uobjpage->flags & PG_WANTED)
wakeup(uobjpage);
/* uobj still locked */
uobjpage->flags &= ~(PG_WANTED|PG_BUSY);
UVM_PAGE_OWN(uobjpage, NULL);
mutex_enter(&uvm_pageqlock);
/*
* replace uobjpage with new page.
*/
uvm_pagereplace(uobjpage, pg);
/*
* if the page is no longer referenced by
* an anon (i.e. we are breaking an O->K
* loan), then remove it from any pageq's.
*/
if (uobjpage->uanon == NULL)
uvm_pagedequeue(uobjpage);
/*
* at this point we have absolutely no
* control over uobjpage
*/
/* install new page */
uvm_pageactivate(pg);
mutex_exit(&uvm_pageqlock);
/*
* done! loan is broken and "pg" is
* PG_BUSY. it can now replace uobjpage.
*/
return pg;
}
int
uvmfault_anonget(struct uvm_faultinfo *ufi, struct vm_amap *amap,
struct vm_anon *anon)
{
boolean_t we_own; /* we own anon's page? */
boolean_t locked; /* did we relock? */
struct vm_page *pg;
int result;
result = 0; /* XXX shut up gcc */
uvmexp.fltanget++;
/* bump rusage counters */
if (anon->an_page)
curproc->p_ru.ru_minflt++;
else
curproc->p_ru.ru_majflt++;
/*
* loop until we get it, or fail.
*/
while (1) {
we_own = FALSE; /* TRUE if we set PG_BUSY on a page */
pg = anon->an_page;
/*
* if there is a resident page and it is loaned, then anon
* may not own it. call out to uvm_anon_lockpage() to ensure
* the real owner of the page has been identified and locked.
*/
if (pg && pg->loan_count)
pg = uvm_anon_lockloanpg(anon);
/*
* page there? make sure it is not busy/released.
*/
if (pg) {
/*
* at this point, if the page has a uobject [meaning
* we have it on loan], then that uobject is locked
* by us! if the page is busy, we drop all the
* locks (including uobject) and try again.
*/
if ((pg->pg_flags & (PG_BUSY|PG_RELEASED)) == 0) {
return (VM_PAGER_OK);
}
atomic_setbits_int(&pg->pg_flags, PG_WANTED);
uvmexp.fltpgwait++;
/*
* the last unlock must be an atomic unlock+wait on
* the owner of page
*/
if (pg->uobject) { /* owner is uobject ? */
uvmfault_unlockall(ufi, amap, NULL, anon);
UVM_UNLOCK_AND_WAIT(pg,
&pg->uobject->vmobjlock,
FALSE, "anonget1",0);
} else {
/* anon owns page */
uvmfault_unlockall(ufi, amap, NULL, NULL);
UVM_UNLOCK_AND_WAIT(pg,&anon->an_lock,0,
"anonget2",0);
}
/* ready to relock and try again */
} else {
/*
* no page, we must try and bring it in.
*/
pg = uvm_pagealloc(NULL, 0, anon, 0);
if (pg == NULL) { /* out of RAM. */
uvmfault_unlockall(ufi, amap, NULL, anon);
uvmexp.fltnoram++;
uvm_wait("flt_noram1");
/* ready to relock and try again */
} else {
/* we set the PG_BUSY bit */
we_own = TRUE;
uvmfault_unlockall(ufi, amap, NULL, anon);
/*
* we are passing a PG_BUSY+PG_FAKE+PG_CLEAN
* page into the uvm_swap_get function with
* all data structures unlocked. note that
* it is ok to read an_swslot here because
* we hold PG_BUSY on the page.
*/
uvmexp.pageins++;
result = uvm_swap_get(pg, anon->an_swslot,
PGO_SYNCIO);
/*
* we clean up after the i/o below in the
* "we_own" case
*/
/* ready to relock and try again */
}
}
/*
* now relock and try again
*/
locked = uvmfault_relock(ufi);
if (locked || we_own)
simple_lock(&anon->an_lock);
/*
* if we own the page (i.e. we set PG_BUSY), then we need
* to clean up after the I/O. there are three cases to
* consider:
* [1] page released during I/O: free anon and ReFault.
* [2] I/O not OK. free the page and cause the fault
* to fail.
* [3] I/O OK! activate the page and sync with the
* non-we_own case (i.e. drop anon lock if not locked).
*/
if (we_own) {
if (pg->pg_flags & PG_WANTED) {
/* still holding object lock */
wakeup(pg);
}
/* un-busy! */
atomic_clearbits_int(&pg->pg_flags,
PG_WANTED|PG_BUSY|PG_FAKE);
UVM_PAGE_OWN(pg, NULL);
/*
* if we were RELEASED during I/O, then our anon is
* no longer part of an amap. we need to free the
* anon and try again.
*/
if (pg->pg_flags & PG_RELEASED) {
pmap_page_protect(pg, VM_PROT_NONE);
simple_unlock(&anon->an_lock);
uvm_anfree(anon); /* frees page for us */
if (locked)
uvmfault_unlockall(ufi, amap, NULL,
NULL);
uvmexp.fltpgrele++;
return (VM_PAGER_REFAULT); /* refault! */
}
if (result != VM_PAGER_OK) {
KASSERT(result != VM_PAGER_PEND);
/* remove page from anon */
anon->an_page = NULL;
/*
* remove the swap slot from the anon
* and mark the anon as having no real slot.
* don't free the swap slot, thus preventing
* it from being used again.
*/
uvm_swap_markbad(anon->an_swslot, 1);
anon->an_swslot = SWSLOT_BAD;
/*
* note: page was never !PG_BUSY, so it
* can't be mapped and thus no need to
* pmap_page_protect it...
*/
uvm_lock_pageq();
uvm_pagefree(pg);
uvm_unlock_pageq();
if (locked)
uvmfault_unlockall(ufi, amap, NULL,
anon);
else
simple_unlock(&anon->an_lock);
return (VM_PAGER_ERROR);
}
/*
* must be OK, clear modify (already PG_CLEAN)
* and activate
*/
pmap_clear_modify(pg);
uvm_lock_pageq();
uvm_pageactivate(pg);
uvm_unlock_pageq();
if (!locked)
simple_unlock(&anon->an_lock);
}
/*
* we were not able to relock. restart fault.
*/
if (!locked)
return (VM_PAGER_REFAULT);
/*
* verify no one has touched the amap and moved the anon on us.
*/
if (ufi != NULL &&
amap_lookup(&ufi->entry->aref,
ufi->orig_rvaddr - ufi->entry->start) != anon) {
uvmfault_unlockall(ufi, amap, NULL, anon);
return (VM_PAGER_REFAULT);
}
/*
* try it again!
*/
uvmexp.fltanretry++;
continue;
} /* while (1) */
/*NOTREACHED*/
}
void
cpu_startup()
{
caddr_t v;
int sz, i;
vsize_t size;
int base, residual;
vaddr_t minaddr, maxaddr, uarea_pages;
/*
* Initialize error message buffer (at end of core).
* avail_end was pre-decremented in luna88k_bootstrap() to compensate.
*/
for (i = 0; i < btoc(MSGBUFSIZE); i++)
pmap_kenter_pa((paddr_t)msgbufp + i * NBPG,
avail_end + i * NBPG, VM_PROT_READ | VM_PROT_WRITE);
pmap_update(pmap_kernel());
initmsgbuf((caddr_t)msgbufp, round_page(MSGBUFSIZE));
/* Determine the machine type from FUSE ROM data */
get_fuse_rom_data();
if (strncmp(fuse_rom_data, "MNAME=LUNA88K+", 14) == 0) {
machtype = LUNA_88K2;
}
/* Determine the 'auto-boot' device from NVRAM data */
get_nvram_data();
get_autoboot_device();
/*
* Good {morning,afternoon,evening,night}.
*/
printf(version);
identifycpu();
printf("real mem = %d\n", ctob(physmem));
/*
* Check front DIP switch setting
*/
printf("dipsw = 0x%x\n", dipswitch);
/* Check DIP switch 1 - 1 */
if ((0x8000 & dipswitch) == 0) {
boothowto |= RB_SINGLE;
}
/* Check DIP switch 1 - 3 */
if ((0x2000 & dipswitch) == 0) {
boothowto |= RB_ASKNAME;
}
/* Check DIP switch 1 - 4 */
if ((0x1000 & dipswitch) == 0) {
boothowto |= RB_CONFIG;
}
/*
* Check frame buffer depth.
*/
switch (hwplanebits) {
case 0: /* No frame buffer */
case 1:
case 4:
case 8:
break;
default:
printf("unexpected frame buffer depth = %d\n", hwplanebits);
hwplanebits = 0;
break;
}
#if 0 /* just for test */
/*
* Get boot arguments
*/
{
char buf[256];
char **p = (volatile char **)0x00001120;
strncpy(buf, *p, 256);
if (buf[255] != '\0')
buf[255] = '\0';
printf("boot arg: (0x%x) %s\n", *p, buf);
}
#endif
/*
* Find out how much space we need, allocate it,
* and then give everything true virtual addresses.
*/
sz = (int)allocsys((caddr_t)0);
if ((v = (caddr_t)uvm_km_zalloc(kernel_map, round_page(sz))) == 0)
panic("startup: no room for tables");
if (allocsys(v) - v != sz)
panic("startup: table size inconsistency");
/*
* Grab UADDR virtual address
*/
uarea_pages = UADDR;
uvm_map(kernel_map, (vaddr_t *)&uarea_pages, USPACE,
NULL, UVM_UNKNOWN_OFFSET, 0,
UVM_MAPFLAG(UVM_PROT_NONE, UVM_PROT_NONE, UVM_INH_NONE,
UVM_ADV_NORMAL, 0));
if (uarea_pages != UADDR)
panic("uarea_pages %lx: UADDR not free", uarea_pages);
/*
* Grab the OBIO space that we hardwired in pmap_bootstrap
*/
obiova = OBIO_START;
uvm_map(kernel_map, (vaddr_t *)&obiova, OBIO_SIZE,
NULL, UVM_UNKNOWN_OFFSET, 0,
UVM_MAPFLAG(UVM_PROT_NONE, UVM_PROT_NONE, UVM_INH_NONE,
UVM_ADV_NORMAL, 0));
if (obiova != OBIO_START)
panic("obiova %lx: OBIO not free", obiova);
/*
* Now allocate buffers proper. They are different than the above
* in that they usually occupy more virtual memory than physical.
*/
size = MAXBSIZE * nbuf;
if (uvm_map(kernel_map, (vaddr_t *) &buffers, round_page(size),
NULL, UVM_UNKNOWN_OFFSET, 0, UVM_MAPFLAG(UVM_PROT_NONE,
UVM_PROT_NONE, UVM_INH_NONE, UVM_ADV_NORMAL, 0)))
panic("cpu_startup: cannot allocate VM for buffers");
minaddr = (vaddr_t)buffers;
if ((bufpages / nbuf) >= btoc(MAXBSIZE)) {
/* don't want to alloc more physical mem than needed */
bufpages = btoc(MAXBSIZE) * nbuf;
}
base = bufpages / nbuf;
residual = bufpages % nbuf;
for (i = 0; i < nbuf; i++) {
vsize_t curbufsize;
vaddr_t curbuf;
struct vm_page *pg;
/*
* Each buffer has MAXBSIZE bytes of VM space allocated. Of
* that MAXBSIZE space, we allocate and map (base+1) pages
* for the first "residual" buffers, and then we allocate
* "base" pages for the rest.
*/
curbuf = (vaddr_t)buffers + (i * MAXBSIZE);
curbufsize = PAGE_SIZE * ((i < residual) ? (base+1) : base);
while (curbufsize) {
pg = uvm_pagealloc(NULL, 0, NULL, 0);
if (pg == NULL)
panic("cpu_startup: not enough memory for "
"buffer cache");
pmap_kenter_pa(curbuf, VM_PAGE_TO_PHYS(pg),
VM_PROT_READ | VM_PROT_WRITE);
curbuf += PAGE_SIZE;
curbufsize -= PAGE_SIZE;
}
}
pmap_update(pmap_kernel());
/*
* Allocate a submap for exec arguments. This map effectively
* limits the number of processes exec'ing at any time.
*/
exec_map = uvm_km_suballoc(kernel_map, &minaddr, &maxaddr,
16 * NCARGS, VM_MAP_PAGEABLE, FALSE, NULL);
/*
* Allocate map for physio.
*/
phys_map = uvm_km_suballoc(kernel_map, &minaddr, &maxaddr,
VM_PHYS_SIZE, 0, FALSE, NULL);
printf("avail mem = %ld (%d pages)\n", ptoa(uvmexp.free), uvmexp.free);
printf("using %d buffers containing %d bytes of memory\n", nbuf,
bufpages * PAGE_SIZE);
/*
* Set up buffers, so they can be used to read disk labels.
*/
bufinit();
/*
* Initialize the autovectored interrupt list.
*/
isrinit();
/*
* Configure the system.
*/
if (boothowto & RB_CONFIG) {
#ifdef BOOT_CONFIG
user_config();
#else
printf("kernel does not support -c; continuing..\n");
#endif
}
/*
* Say hello to the world on LCD.
*/
greeting();
}
/*
* Allocate memory for variable-sized tables,
*/
void
cpu_startup()
{
unsigned i;
int base, residual;
vaddr_t minaddr, maxaddr;
vsize_t size;
char pbuf[9];
/*
* Good {morning,afternoon,evening,night}.
*/
printf(version);
printf("%s\n", cpu_model);
format_bytes(pbuf, sizeof(pbuf), ctob(physmem));
printf("%s memory", pbuf);
/*
* Allocate virtual address space for file I/O buffers.
* Note they are different than the array of headers, 'buf',
* and usually occupy more virtual memory than physical.
*/
size = MAXBSIZE * nbuf;
if (uvm_map(kernel_map, (vaddr_t *)&buffers, round_page(size),
NULL, UVM_UNKNOWN_OFFSET, 0, UVM_MAPFLAG(UVM_PROT_NONE,
UVM_PROT_NONE, UVM_INH_NONE, UVM_ADV_NORMAL, 0)) != 0)
panic("startup: cannot allocate VM for buffers");
minaddr = (vaddr_t)buffers;
base = bufpages / nbuf;
residual = bufpages % nbuf;
for (i = 0; i < nbuf; i++) {
vsize_t curbufsize;
vaddr_t curbuf;
struct vm_page *pg;
/*
* Each buffer has MAXBSIZE bytes of VM space allocated. Of
* that MAXBSIZE space, we allocate and map (base+1) pages
* for the first "residual" buffers, and then we allocate
* "base" pages for the rest.
*/
curbuf = (vaddr_t) buffers + (i * MAXBSIZE);
curbufsize = NBPG * ((i < residual) ? (base + 1) : base);
while (curbufsize) {
pg = uvm_pagealloc(NULL, 0, NULL, 0);
if (pg == NULL)
panic("cpu_startup: not enough memory for "
"buffer cache");
pmap_kenter_pa(curbuf, VM_PAGE_TO_PHYS(pg),
VM_PROT_READ|VM_PROT_WRITE);
curbuf += PAGE_SIZE;
curbufsize -= PAGE_SIZE;
}
}
pmap_update(pmap_kernel());
/*
* Allocate a submap for exec arguments. This map effectively
* limits the number of processes exec'ing at any time.
*/
exec_map = uvm_km_suballoc(kernel_map, &minaddr, &maxaddr,
16 * NCARGS, VM_MAP_PAGEABLE, FALSE, NULL);
/*
* Allocate a submap for physio.
*/
phys_map = uvm_km_suballoc(kernel_map, &minaddr, &maxaddr,
VM_PHYS_SIZE, 0, FALSE, NULL);
/*
* (No need to allocate an mbuf cluster submap. Mbuf clusters
* are allocated via the pool allocator, and we use KSEG to
* map those pages.)
*/
format_bytes(pbuf, sizeof(pbuf), ptoa(uvmexp.free));
printf(", %s free", pbuf);
format_bytes(pbuf, sizeof(pbuf), bufpages * NBPG);
printf(", %s in %d buffers\n", pbuf, nbuf);
/*
* Set up buffers, so they can be used to read disk labels.
*/
bufinit();
}
vaddr_t
uvm_km_alloc1(struct vm_map *map, vsize_t size, vsize_t align, boolean_t zeroit)
{
vaddr_t kva, loopva;
voff_t offset;
struct vm_page *pg;
UVMHIST_FUNC("uvm_km_alloc1"); UVMHIST_CALLED(maphist);
UVMHIST_LOG(maphist,"(map=%p, size=0x%lx)", map, size,0,0);
KASSERT(vm_map_pmap(map) == pmap_kernel());
size = round_page(size);
kva = vm_map_min(map); /* hint */
/*
* allocate some virtual space
*/
if (__predict_false(uvm_map(map, &kva, size, uvm.kernel_object,
UVM_UNKNOWN_OFFSET, align, UVM_MAPFLAG(UVM_PROT_ALL, UVM_PROT_ALL,
UVM_INH_NONE, UVM_ADV_RANDOM, 0)) != 0)) {
UVMHIST_LOG(maphist,"<- done (no VM)",0,0,0,0);
return(0);
}
/*
* recover object offset from virtual address
*/
offset = kva - vm_map_min(kernel_map);
UVMHIST_LOG(maphist," kva=0x%lx, offset=0x%lx", kva, offset,0,0);
/*
* now allocate the memory. we must be careful about released pages.
*/
loopva = kva;
while (size) {
simple_lock(&uvm.kernel_object->vmobjlock);
pg = uvm_pagelookup(uvm.kernel_object, offset);
/*
* if we found a page in an unallocated region, it must be
* released
*/
if (pg) {
if ((pg->pg_flags & PG_RELEASED) == 0)
panic("uvm_km_alloc1: non-released page");
atomic_setbits_int(&pg->pg_flags, PG_WANTED);
UVM_UNLOCK_AND_WAIT(pg, &uvm.kernel_object->vmobjlock,
FALSE, "km_alloc", 0);
continue; /* retry */
}
/* allocate ram */
pg = uvm_pagealloc(uvm.kernel_object, offset, NULL, 0);
if (pg) {
atomic_clearbits_int(&pg->pg_flags, PG_BUSY);
UVM_PAGE_OWN(pg, NULL);
}
simple_unlock(&uvm.kernel_object->vmobjlock);
if (__predict_false(pg == NULL)) {
if (curproc == uvm.pagedaemon_proc) {
/*
* It is unfeasible for the page daemon to
* sleep for memory, so free what we have
* allocated and fail.
*/
uvm_unmap(map, kva, loopva - kva);
return (NULL);
} else {
uvm_wait("km_alloc1w"); /* wait for memory */
continue;
}
}
/*
* map it in; note we're never called with an intrsafe
* object, so we always use regular old pmap_enter().
*/
pmap_enter(map->pmap, loopva, VM_PAGE_TO_PHYS(pg),
UVM_PROT_ALL, PMAP_WIRED | VM_PROT_READ | VM_PROT_WRITE);
loopva += PAGE_SIZE;
offset += PAGE_SIZE;
size -= PAGE_SIZE;
}
pmap_update(map->pmap);
/*
* zero on request (note that "size" is now zero due to the above loop
* so we need to subtract kva from loopva to reconstruct the size).
*/
if (zeroit)
memset((caddr_t)kva, 0, loopva - kva);
UVMHIST_LOG(maphist,"<- done (kva=0x%lx)", kva,0,0,0);
return(kva);
}
/*
* Machine-dependent startup code
*/
void
cpu_startup()
{
unsigned i;
caddr_t v;
long sz;
int base, residual;
#ifdef DEBUG
extern int pmapdebug;
int opmapdebug = pmapdebug;
#endif
vaddr_t minaddr, maxaddr;
vsize_t size;
extern struct user *proc0paddr;
#ifdef DEBUG
pmapdebug = 0;
#endif
proc0.p_addr = proc0paddr;
/*
* Good {morning,afternoon,evening,night}.
*/
printf(version);
/*identifycpu();*/
printf("total memory = %d\n", physmem * PAGE_SIZE);
/*
* Find out how much space we need, allocate it,
* and then give everything true virtual addresses.
*/
sz = (long)allocsys(NULL);
if ((v = (caddr_t)uvm_km_alloc(kernel_map, round_page(sz))) == 0)
panic("startup: no room for %lx bytes of tables", sz);
if (allocsys(v) - v != sz)
panic("startup: table size inconsistency");
/*
* allocate virtual and physical memory for the buffers.
*/
size = MAXBSIZE * nbuf; /* # bytes for buffers */
/* allocate VM for buffers... area is not managed by VM system */
if (uvm_map(kernel_map, (vaddr_t *) &buffers, round_page(size),
NULL, UVM_UNKNOWN_OFFSET,
UVM_MAPFLAG(UVM_PROT_NONE, UVM_PROT_NONE, UVM_INH_NONE,
UVM_ADV_NORMAL, 0)) != 0)
panic("cpu_startup: cannot allocate VM for buffers");
minaddr = (vaddr_t) buffers;
if ((bufpages / nbuf) >= btoc(MAXBSIZE)) {
bufpages = btoc(MAXBSIZE) * nbuf; /* do not overallocate RAM */
}
base = bufpages / nbuf;
residual = bufpages % nbuf;
/* now allocate RAM for buffers */
for (i = 0 ; i < nbuf ; i++) {
vaddr_t curbuf;
vsize_t curbufsize;
struct vm_page *pg;
/*
* each buffer has MAXBSIZE bytes of VM space allocated. of
* that MAXBSIZE space we allocate and map (base+1) pages
* for the first "residual" buffers, and then we allocate
* "base" pages for the rest.
*/
curbuf = (vaddr_t) buffers + (i * MAXBSIZE);
curbufsize = NBPG * ((i < residual) ? (base+1) : base);
while (curbufsize) {
pg = uvm_pagealloc(NULL, 0, NULL, 0);
if (pg == NULL)
panic("cpu_startup: "
"not enough RAM for buffer cache");
pmap_enter(kernel_map->pmap, curbuf,
VM_PAGE_TO_PHYS(pg), VM_PROT_READ|VM_PROT_WRITE,
VM_PROT_READ|VM_PROT_WRITE|PMAP_WIRED);
curbuf += PAGE_SIZE;
curbufsize -= PAGE_SIZE;
}
}
pmap_update();
/*
* Allocate a submap for exec arguments. This map effectively
* limits the number of processes exec'ing at any time.
*/
exec_map = uvm_km_suballoc(kernel_map, &minaddr, &maxaddr,
16*NCARGS, VM_MAP_PAGEABLE, FALSE, NULL);
/*
* Finally, allocate mbuf cluster submap.
*/
mb_map = uvm_km_suballoc(kernel_map, &minaddr, &maxaddr,
VM_MBUF_SIZE, VM_MAP_INTRSAFE, FALSE, NULL);
#ifdef DEBUG
pmapdebug = opmapdebug;
#endif
printf("avail memory = %ld\n", (long)uvmexp.free * PAGE_SIZE);
printf("using %d buffers containing %ld of memory\n", nbuf,
(long)bufpages * PAGE_SIZE);
/*
* Set up buffers, so they can be used to read disk labels.
*/
bufinit();
#if 0
pmap_redzone();
#endif
}
/*
* cpu_startup: allocate memory for variable-sized tables,
* initialize cpu, and do autoconfiguration.
*/
void
cpu_startup()
{
extern char *kernel_text, *etext;
unsigned i;
caddr_t v;
int base, residual;
u_quad_t vmememsize;
vaddr_t minaddr, maxaddr;
vsize_t size;
#ifdef DEBUG
extern int pmapdebug;
int opmapdebug = pmapdebug;
pmapdebug = 0;
#endif
/*
* Initialize the kernel crash dump header.
*/
cpu_init_kcore_hdr();
/*
* Good {morning,afternoon,evening,night}.
*/
printf(version);
identifycpu();
printf("real mem = %d", ctob(physmem));
for (vmememsize = 0, i = 1; i < mem_cluster_cnt; i++)
vmememsize += mem_clusters[i].size;
if (vmememsize != 0)
printf(" (%qu on-board, %qu VMEbus)",
mem_clusters[0].size, vmememsize);
printf("\n");
/*
* Find out how much space we need, allocate it,
* and then give everything true virtual addresses.
*/
size = (vsize_t)allocsys((caddr_t)0);
if ((v = (caddr_t)uvm_km_zalloc(kernel_map, round_page(size))) == 0)
panic("startup: no room for tables");
if ((allocsys(v) - v) != size)
panic("startup: talbe size inconsistency");
/*
* Now allocate buffers proper. They are different than the above
* in that they usually occupy more virtual memory than physical.
*/
size = MAXBSIZE * nbuf;
if (uvm_map(kernel_map, (vaddr_t *) &buffers, round_page(size),
NULL, UVM_UNKNOWN_OFFSET,
UVM_MAPFLAG(UVM_PROT_NONE, UVM_PROT_NONE, UVM_INH_NONE,
UVM_ADV_NORMAL, 0)) != KERN_SUCCESS)
panic("startup: cannot allocate VM for buffers");
minaddr = (vaddr_t)buffers;
base = bufpages / nbuf;
residual = bufpages % nbuf;
for (i = 0; i < nbuf; i++) {
vsize_t curbufsize;
vaddr_t curbuf;
struct vm_page *pg;
/*
* Each buffer has MAXBSIZE bytes of VM space allocated. Of
* that MAXBSIZE space, we allocate and map (base+1) pages
* for the first "residual" buffers, and then we allocate
* "base" pages for the rest.
*/
curbuf = (vaddr_t) buffers + (i * MAXBSIZE);
curbufsize = CLBYTES * ((i < residual) ? (base+1) : base);
while (curbufsize) {
pg = uvm_pagealloc(NULL, 0, NULL, 0);
if (pg == NULL)
panic("cpu_startup: not enough memory for "
"buffer cache");
#ifdef PMAP_NEW
pmap_kenter_pgs(curbuf, &pg, 1);
#else
pmap_enter(kernel_map->pmap, curbuf,
VM_PAGE_TO_PHYS(pg), VM_PROT_READ|VM_PROT_WRITE,
TRUE, VM_PROT_READ|VM_PROT_WRITE);
#endif
curbuf += PAGE_SIZE;
curbufsize -= PAGE_SIZE;
}
}
/*
* Allocate a submap for exec arguments. This map effectively
* limits the number of processes exec'ing at any time.
*/
exec_map = uvm_km_suballoc(kernel_map, &minaddr, &maxaddr,
16*NCARGS, TRUE, FALSE, NULL);
/*
* Allocate a submap for physio
*/
phys_map = uvm_km_suballoc(kernel_map, &minaddr, &maxaddr,
VM_PHYS_SIZE, TRUE, FALSE, NULL);
/*
* Finally, allocate mbuf cluster submap.
*/
mb_map = uvm_km_suballoc(kernel_map, &minaddr, &maxaddr,
VM_MBUF_SIZE, FALSE, FALSE, NULL);
/*
* Initialize callouts
*/
callfree = callout;
for (i = 1; i < ncallout; i++)
callout[i-1].c_next = &callout[i];
callout[i-1].c_next = NULL;
#ifdef DEBUG
pmapdebug = opmapdebug;
#endif
printf("avail mem = %ld\n", ptoa(uvmexp.free));
printf("using %d buffers containing %d bytes of memory\n",
nbuf, bufpages * CLBYTES);
/*
* Tell the VM system that the area before the text segment
* is invalid.
*
* XXX Should just change KERNBASE and VM_MIN_KERNEL_ADDRESS,
* XXX but not right now.
*/
if (uvm_map_protect(kernel_map, 0, round_page(&kernel_text),
UVM_PROT_NONE, TRUE) != KERN_SUCCESS)
panic("can't mark pre-text pages off-limits");
/*
* Tell the VM system that writing to the kernel text isn't allowed.
* If we don't, we might end up COW'ing the text segment!
*/
if (uvm_map_protect(kernel_map, trunc_page(&kernel_text),
round_page(&etext), UVM_PROT_READ|UVM_PROT_EXEC, TRUE)
!= KERN_SUCCESS)
panic("can't protect kernel text");
/*
* Set up CPU-specific registers, cache, etc.
*/
initcpu();
/*
* Set up buffers, so they can be used to read disk labels.
*/
bufinit();
}