/*
* Grow the GDT.
*/
void
gdt_grow(int which)
{
size_t old_len, new_len;
CPU_INFO_ITERATOR cii;
struct cpu_info *ci;
struct vm_page *pg;
vaddr_t va;
old_len = gdt_size[which] * sizeof(gdt[0]);
gdt_size[which] <<= 1;
new_len = old_len << 1;
#ifdef XEN
if (which != 0) {
size_t max_len = MAXGDTSIZ * sizeof(gdt[0]);
if (old_len == 0) {
gdt_size[which] = MINGDTSIZ;
new_len = gdt_size[which] * sizeof(gdt[0]);
}
for(va = (vaddr_t)(cpu_info_primary.ci_gdt) + old_len + max_len;
va < (vaddr_t)(cpu_info_primary.ci_gdt) + new_len + max_len;
va += PAGE_SIZE) {
while ((pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO))
== NULL) {
uvm_wait("gdt_grow");
}
pmap_kenter_pa(va, VM_PAGE_TO_PHYS(pg),
VM_PROT_READ | VM_PROT_WRITE);
}
return;
}
#endif
for (CPU_INFO_FOREACH(cii, ci)) {
for (va = (vaddr_t)(ci->ci_gdt) + old_len;
va < (vaddr_t)(ci->ci_gdt) + new_len;
va += PAGE_SIZE) {
while ((pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO)) ==
NULL) {
uvm_wait("gdt_grow");
}
pmap_kenter_pa(va, VM_PAGE_TO_PHYS(pg),
VM_PROT_READ | VM_PROT_WRITE);
}
}
pmap_update(pmap_kernel());
}
/*
* Grow the GDT.
*/
void
gdt_grow()
{
size_t old_len, new_len;
CPU_INFO_ITERATOR cii;
struct cpu_info *ci;
struct vm_page *pg;
vaddr_t va;
old_len = gdt_size * sizeof(union descriptor);
gdt_size <<= 1;
new_len = old_len << 1;
CPU_INFO_FOREACH(cii, ci) {
for (va = (vaddr_t)(ci->ci_gdt) + old_len;
va < (vaddr_t)(ci->ci_gdt) + new_len;
va += PAGE_SIZE) {
while (
(pg =
uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO)) ==
NULL) {
uvm_wait("gdt_grow");
}
pmap_kenter_pa(va, VM_PAGE_TO_PHYS(pg),
VM_PROT_READ | VM_PROT_WRITE);
}
}
}
/*
* Allocate shadow GDT for a slave CPU.
*/
void
gdt_alloc_cpu(struct cpu_info *ci)
{
int max_len = MAXGDTSIZ * sizeof(gdt[0]);
int min_len = MINGDTSIZ * sizeof(gdt[0]);
struct vm_page *pg;
vaddr_t va;
ci->ci_gdt = (union descriptor *)uvm_km_alloc(kernel_map, max_len,
0, UVM_KMF_VAONLY);
for (va = (vaddr_t)ci->ci_gdt; va < (vaddr_t)ci->ci_gdt + min_len;
va += PAGE_SIZE) {
while ((pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO))
== NULL) {
uvm_wait("gdt_alloc_cpu");
}
pmap_kenter_pa(va, VM_PAGE_TO_PHYS(pg),
VM_PROT_READ | VM_PROT_WRITE);
}
pmap_update(pmap_kernel());
memset(ci->ci_gdt, 0, min_len);
memcpy(ci->ci_gdt, gdt, gdt_count[0] * sizeof(gdt[0]));
setsegment(&ci->ci_gdt[GCPU_SEL].sd, ci, 0xfffff,
SDT_MEMRWA, SEL_KPL, 1, 1);
}
/*
* uvmfault_amapcopy: clear "needs_copy" in a map.
*
* => called with VM data structures unlocked (usually, see below)
* => we get a write lock on the maps and clear needs_copy for a VA
* => if we are out of RAM we sleep (waiting for more)
*/
static void
uvmfault_amapcopy(struct uvm_faultinfo *ufi)
{
/* while we haven't done the job */
while (1) {
/* no mapping? give up. */
if (uvmfault_lookup(ufi, TRUE) == FALSE)
return;
/* copy if needed. */
if (UVM_ET_ISNEEDSCOPY(ufi->entry))
amap_copy(ufi->map, ufi->entry, M_NOWAIT, TRUE,
ufi->orig_rvaddr, ufi->orig_rvaddr + 1);
/* didn't work? must be out of RAM. unlock and sleep. */
if (UVM_ET_ISNEEDSCOPY(ufi->entry)) {
uvmfault_unlockmaps(ufi, TRUE);
uvm_wait("fltamapcopy");
continue;
}
/* got it! unlock and return*/
uvmfault_unlockmaps(ufi, TRUE);
return;
}
/*NOTREACHED*/
}
/* ARGSUSED */
vaddr_t
uvm_km_alloc_poolpage1(struct vm_map *map, struct uvm_object *obj,
boolean_t waitok)
{
#if defined(__HAVE_PMAP_DIRECT)
struct vm_page *pg;
vaddr_t va;
again:
pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_USERESERVE);
if (__predict_false(pg == NULL)) {
if (waitok) {
uvm_wait("plpg");
goto again;
} else
return (0);
}
va = pmap_map_direct(pg);
if (__predict_false(va == 0))
uvm_pagefree(pg);
return (va);
#else
vaddr_t va;
int s;
/*
* NOTE: We may be called with a map that doesn't require splvm
* protection (e.g. kernel_map). However, it does not hurt to
* go to splvm in this case (since unprotected maps will never be
* accessed in interrupt context).
*
* XXX We may want to consider changing the interface to this
* XXX function.
*/
s = splvm();
va = uvm_km_kmemalloc(map, obj, PAGE_SIZE, waitok ? 0 : UVM_KMF_NOWAIT);
splx(s);
return (va);
#endif /* __HAVE_PMAP_DIRECT */
}
static int
uvm_loanzero(struct uvm_faultinfo *ufi, void ***output, int flags)
{
struct vm_anon *anon;
struct vm_page *pg;
struct vm_amap *amap = ufi->entry->aref.ar_amap;
UVMHIST_FUNC(__func__); UVMHIST_CALLED(loanhist);
again:
mutex_enter(&uvm_loanzero_object.vmobjlock);
/*
* first, get ahold of our single zero page.
*/
if (__predict_false((pg =
TAILQ_FIRST(&uvm_loanzero_object.memq)) == NULL)) {
while ((pg = uvm_pagealloc(&uvm_loanzero_object, 0, NULL,
UVM_PGA_ZERO)) == NULL) {
mutex_exit(&uvm_loanzero_object.vmobjlock);
uvmfault_unlockall(ufi, amap, NULL, NULL);
uvm_wait("loanzero");
if (!uvmfault_relock(ufi)) {
return (0);
}
if (amap) {
amap_lock(amap);
}
goto again;
}
/* got a zero'd page. */
pg->flags &= ~(PG_WANTED|PG_BUSY|PG_FAKE);
pg->flags |= PG_RDONLY;
mutex_enter(&uvm_pageqlock);
uvm_pageactivate(pg);
mutex_exit(&uvm_pageqlock);
UVM_PAGE_OWN(pg, NULL);
}
if ((flags & UVM_LOAN_TOANON) == 0) { /* loaning to kernel-page */
mutex_enter(&uvm_pageqlock);
pg->loan_count++;
mutex_exit(&uvm_pageqlock);
mutex_exit(&uvm_loanzero_object.vmobjlock);
**output = pg;
(*output)++;
return (1);
}
/*
* loaning to an anon. check to see if there is already an anon
* associated with this page. if so, then just return a reference
* to this object.
*/
if (pg->uanon) {
anon = pg->uanon;
mutex_enter(&anon->an_lock);
anon->an_ref++;
mutex_exit(&anon->an_lock);
mutex_exit(&uvm_loanzero_object.vmobjlock);
**output = anon;
(*output)++;
return (1);
}
/*
* need to allocate a new anon
*/
anon = uvm_analloc();
if (anon == NULL) {
/* out of swap causes us to fail */
mutex_exit(&uvm_loanzero_object.vmobjlock);
uvmfault_unlockall(ufi, amap, NULL, NULL);
return (-1);
}
anon->an_page = pg;
pg->uanon = anon;
mutex_enter(&uvm_pageqlock);
pg->loan_count++;
uvm_pageactivate(pg);
mutex_exit(&uvm_pageqlock);
mutex_exit(&anon->an_lock);
mutex_exit(&uvm_loanzero_object.vmobjlock);
**output = anon;
(*output)++;
return (1);
}
int
uvmfault_anonget(struct uvm_faultinfo *ufi, struct vm_amap *amap,
struct vm_anon *anon)
{
boolean_t we_own; /* we own anon's page? */
boolean_t locked; /* did we relock? */
struct vm_page *pg;
int result;
result = 0; /* XXX shut up gcc */
uvmexp.fltanget++;
/* bump rusage counters */
if (anon->an_page)
curproc->p_ru.ru_minflt++;
else
curproc->p_ru.ru_majflt++;
/*
* loop until we get it, or fail.
*/
while (1) {
we_own = FALSE; /* TRUE if we set PG_BUSY on a page */
pg = anon->an_page;
/*
* if there is a resident page and it is loaned, then anon
* may not own it. call out to uvm_anon_lockpage() to ensure
* the real owner of the page has been identified and locked.
*/
if (pg && pg->loan_count)
pg = uvm_anon_lockloanpg(anon);
/*
* page there? make sure it is not busy/released.
*/
if (pg) {
/*
* at this point, if the page has a uobject [meaning
* we have it on loan], then that uobject is locked
* by us! if the page is busy, we drop all the
* locks (including uobject) and try again.
*/
if ((pg->pg_flags & (PG_BUSY|PG_RELEASED)) == 0) {
return (VM_PAGER_OK);
}
atomic_setbits_int(&pg->pg_flags, PG_WANTED);
uvmexp.fltpgwait++;
/*
* the last unlock must be an atomic unlock+wait on
* the owner of page
*/
if (pg->uobject) { /* owner is uobject ? */
uvmfault_unlockall(ufi, amap, NULL, anon);
UVM_UNLOCK_AND_WAIT(pg,
&pg->uobject->vmobjlock,
FALSE, "anonget1",0);
} else {
/* anon owns page */
uvmfault_unlockall(ufi, amap, NULL, NULL);
UVM_UNLOCK_AND_WAIT(pg,&anon->an_lock,0,
"anonget2",0);
}
/* ready to relock and try again */
} else {
/*
* no page, we must try and bring it in.
*/
pg = uvm_pagealloc(NULL, 0, anon, 0);
if (pg == NULL) { /* out of RAM. */
uvmfault_unlockall(ufi, amap, NULL, anon);
uvmexp.fltnoram++;
uvm_wait("flt_noram1");
/* ready to relock and try again */
} else {
/* we set the PG_BUSY bit */
we_own = TRUE;
uvmfault_unlockall(ufi, amap, NULL, anon);
/*
* we are passing a PG_BUSY+PG_FAKE+PG_CLEAN
* page into the uvm_swap_get function with
* all data structures unlocked. note that
* it is ok to read an_swslot here because
* we hold PG_BUSY on the page.
*/
uvmexp.pageins++;
result = uvm_swap_get(pg, anon->an_swslot,
PGO_SYNCIO);
/*
* we clean up after the i/o below in the
* "we_own" case
*/
/* ready to relock and try again */
}
}
/*
* now relock and try again
*/
locked = uvmfault_relock(ufi);
if (locked || we_own)
simple_lock(&anon->an_lock);
/*
* if we own the page (i.e. we set PG_BUSY), then we need
* to clean up after the I/O. there are three cases to
* consider:
* [1] page released during I/O: free anon and ReFault.
* [2] I/O not OK. free the page and cause the fault
* to fail.
* [3] I/O OK! activate the page and sync with the
* non-we_own case (i.e. drop anon lock if not locked).
*/
if (we_own) {
if (pg->pg_flags & PG_WANTED) {
/* still holding object lock */
wakeup(pg);
}
/* un-busy! */
atomic_clearbits_int(&pg->pg_flags,
PG_WANTED|PG_BUSY|PG_FAKE);
UVM_PAGE_OWN(pg, NULL);
/*
* if we were RELEASED during I/O, then our anon is
* no longer part of an amap. we need to free the
* anon and try again.
*/
if (pg->pg_flags & PG_RELEASED) {
pmap_page_protect(pg, VM_PROT_NONE);
simple_unlock(&anon->an_lock);
uvm_anfree(anon); /* frees page for us */
if (locked)
uvmfault_unlockall(ufi, amap, NULL,
NULL);
uvmexp.fltpgrele++;
return (VM_PAGER_REFAULT); /* refault! */
}
if (result != VM_PAGER_OK) {
KASSERT(result != VM_PAGER_PEND);
/* remove page from anon */
anon->an_page = NULL;
/*
* remove the swap slot from the anon
* and mark the anon as having no real slot.
* don't free the swap slot, thus preventing
* it from being used again.
*/
uvm_swap_markbad(anon->an_swslot, 1);
anon->an_swslot = SWSLOT_BAD;
/*
* note: page was never !PG_BUSY, so it
* can't be mapped and thus no need to
* pmap_page_protect it...
*/
uvm_lock_pageq();
uvm_pagefree(pg);
uvm_unlock_pageq();
if (locked)
uvmfault_unlockall(ufi, amap, NULL,
anon);
else
simple_unlock(&anon->an_lock);
return (VM_PAGER_ERROR);
}
/*
* must be OK, clear modify (already PG_CLEAN)
* and activate
*/
pmap_clear_modify(pg);
uvm_lock_pageq();
uvm_pageactivate(pg);
uvm_unlock_pageq();
if (!locked)
simple_unlock(&anon->an_lock);
}
/*
* we were not able to relock. restart fault.
*/
if (!locked)
return (VM_PAGER_REFAULT);
/*
* verify no one has touched the amap and moved the anon on us.
*/
if (ufi != NULL &&
amap_lookup(&ufi->entry->aref,
ufi->orig_rvaddr - ufi->entry->start) != anon) {
uvmfault_unlockall(ufi, amap, NULL, anon);
return (VM_PAGER_REFAULT);
}
/*
* try it again!
*/
uvmexp.fltanretry++;
continue;
} /* while (1) */
/*NOTREACHED*/
}
int
uvm_objwire(struct uvm_object *uobj, off_t start, off_t end,
struct pglist *pageq)
{
int i, npages, error;
struct vm_page *pgs[FETCH_PAGECOUNT];
off_t offset = start, left;
left = (end - start) >> PAGE_SHIFT;
mtx_enter(&uobj->vmobjlock);
while (left) {
npages = MIN(FETCH_PAGECOUNT, left);
/* Get the pages */
memset(pgs, 0, sizeof(pgs));
error = (*uobj->pgops->pgo_get)(uobj, offset, pgs, &npages, 0,
VM_PROT_READ | VM_PROT_WRITE, UVM_ADV_SEQUENTIAL,
PGO_ALLPAGES | PGO_SYNCIO);
if (error)
goto error;
mtx_enter(&uobj->vmobjlock);
for (i = 0; i < npages; i++) {
KASSERT(pgs[i] != NULL);
KASSERT(!(pgs[i]->pg_flags & PG_RELEASED));
#if 0
/*
* Loan break
*/
if (pgs[i]->loan_count) {
while (pgs[i]->loan_count) {
pg = uvm_loanbreak(pgs[i]);
if (!pg) {
mtx_leave(&uobj->vmobjlock);
uvm_wait("uobjwirepg");
mtx_enter(&uobj->vmobjlock);
continue;
}
}
pgs[i] = pg;
}
#endif
if (pgs[i]->pg_flags & PQ_AOBJ) {
atomic_clearbits_int(&pgs[i]->pg_flags,
PG_CLEAN);
uao_dropswap(uobj, i);
}
}
/* Wire the pages */
uvm_lock_pageq();
for (i = 0; i < npages; i++) {
uvm_pagewire(pgs[i]);
if (pageq != NULL)
TAILQ_INSERT_TAIL(pageq, pgs[i], pageq);
}
uvm_unlock_pageq();
/* Unbusy the pages */
uvm_page_unbusy(pgs, npages);
left -= npages;
offset += (off_t)npages << PAGE_SHIFT;
}
mtx_leave(&uobj->vmobjlock);
return 0;
error:
/* Unwire the pages which have been wired */
uvm_objunwire(uobj, start, offset);
return error;
}
vaddr_t
uvm_km_alloc1(struct vm_map *map, vsize_t size, vsize_t align, boolean_t zeroit)
{
vaddr_t kva, loopva;
voff_t offset;
struct vm_page *pg;
UVMHIST_FUNC("uvm_km_alloc1"); UVMHIST_CALLED(maphist);
UVMHIST_LOG(maphist,"(map=%p, size=0x%lx)", map, size,0,0);
KASSERT(vm_map_pmap(map) == pmap_kernel());
size = round_page(size);
kva = vm_map_min(map); /* hint */
/*
* allocate some virtual space
*/
if (__predict_false(uvm_map(map, &kva, size, uvm.kernel_object,
UVM_UNKNOWN_OFFSET, align, UVM_MAPFLAG(UVM_PROT_ALL, UVM_PROT_ALL,
UVM_INH_NONE, UVM_ADV_RANDOM, 0)) != 0)) {
UVMHIST_LOG(maphist,"<- done (no VM)",0,0,0,0);
return(0);
}
/*
* recover object offset from virtual address
*/
offset = kva - vm_map_min(kernel_map);
UVMHIST_LOG(maphist," kva=0x%lx, offset=0x%lx", kva, offset,0,0);
/*
* now allocate the memory. we must be careful about released pages.
*/
loopva = kva;
while (size) {
simple_lock(&uvm.kernel_object->vmobjlock);
pg = uvm_pagelookup(uvm.kernel_object, offset);
/*
* if we found a page in an unallocated region, it must be
* released
*/
if (pg) {
if ((pg->pg_flags & PG_RELEASED) == 0)
panic("uvm_km_alloc1: non-released page");
atomic_setbits_int(&pg->pg_flags, PG_WANTED);
UVM_UNLOCK_AND_WAIT(pg, &uvm.kernel_object->vmobjlock,
FALSE, "km_alloc", 0);
continue; /* retry */
}
/* allocate ram */
pg = uvm_pagealloc(uvm.kernel_object, offset, NULL, 0);
if (pg) {
atomic_clearbits_int(&pg->pg_flags, PG_BUSY);
UVM_PAGE_OWN(pg, NULL);
}
simple_unlock(&uvm.kernel_object->vmobjlock);
if (__predict_false(pg == NULL)) {
if (curproc == uvm.pagedaemon_proc) {
/*
* It is unfeasible for the page daemon to
* sleep for memory, so free what we have
* allocated and fail.
*/
uvm_unmap(map, kva, loopva - kva);
return (NULL);
} else {
uvm_wait("km_alloc1w"); /* wait for memory */
continue;
}
}
/*
* map it in; note we're never called with an intrsafe
* object, so we always use regular old pmap_enter().
*/
pmap_enter(map->pmap, loopva, VM_PAGE_TO_PHYS(pg),
UVM_PROT_ALL, PMAP_WIRED | VM_PROT_READ | VM_PROT_WRITE);
loopva += PAGE_SIZE;
offset += PAGE_SIZE;
size -= PAGE_SIZE;
}
pmap_update(map->pmap);
/*
* zero on request (note that "size" is now zero due to the above loop
* so we need to subtract kva from loopva to reconstruct the size).
*/
if (zeroit)
memset((caddr_t)kva, 0, loopva - kva);
UVMHIST_LOG(maphist,"<- done (kva=0x%lx)", kva,0,0,0);
return(kva);
}
vaddr_t
uvm_km_kmemalloc(struct vm_map *map, struct uvm_object *obj, vsize_t size,
int flags)
{
vaddr_t kva, loopva;
voff_t offset;
struct vm_page *pg;
int mapflags;
UVMHIST_FUNC("uvm_km_kmemalloc"); UVMHIST_CALLED(maphist);
UVMHIST_LOG(maphist," (map=%p, obj=%p, size=0x%lx, flags=%d)",
map, obj, size, flags);
KASSERT(vm_map_pmap(map) == pmap_kernel());
/*
* we cannot yet make pmap_enter() not sleep
* and thus demand that we are called with NOWAIT in that case
*/
KASSERT(!((flags & UVM_KMF_NOWAIT) && obj));
/*
* setup for call
*/
mapflags = flags & UVM_KMF_NOWAIT? UVM_FLAG_NOWAIT : 0;
mapflags |= flags & UVM_KMF_TRYLOCK;
size = round_page(size);
kva = vm_map_min(map); /* hint */
/*
* allocate some virtual space
*/
if (__predict_false(uvm_map(map, &kva, size, obj, UVM_UNKNOWN_OFFSET,
0, UVM_MAPFLAG(UVM_PROT_RW, UVM_PROT_RW, UVM_INH_NONE,
UVM_ADV_RANDOM, mapflags)) != 0)) {
UVMHIST_LOG(maphist, "<- done (no VM)",0,0,0,0);
return(0);
}
/*
* if all we wanted was VA, return now
*/
if (flags & UVM_KMF_VALLOC) {
UVMHIST_LOG(maphist,"<- done valloc (kva=0x%lx)", kva,0,0,0);
return(kva);
}
/*
* recover object offset from virtual address
*/
if (obj != NULL)
offset = kva - vm_map_min(kernel_map);
else
offset = 0;
UVMHIST_LOG(maphist, " kva=0x%lx, offset=0x%lx", kva, offset,0,0);
/*
* now allocate and map in the memory... note that we are the only ones
* whom should ever get a handle on this area of VM.
*/
loopva = kva;
while (loopva != kva + size) {
pg = uvm_pagealloc(obj, offset, NULL, 0);
if (pg) {
atomic_clearbits_int(&pg->pg_flags, PG_BUSY);
UVM_PAGE_OWN(pg, NULL);
}
if (__predict_false(pg == NULL)) {
if ((flags & UVM_KMF_NOWAIT) ||
((flags & UVM_KMF_CANFAIL) &&
uvmexp.swpgonly == uvmexp.swpages)) {
/* free everything! */
uvm_unmap(map, kva, kva + size);
return (0);
} else {
uvm_wait("km_getwait2"); /* sleep here */
continue;
}
}
/*
* map it in: note that we call pmap_enter with the map and
* object unlocked in case we are kmem_map.
*
* pager mappings that must not sleep here will incidently
* be installed using pmap_kenter_pa() and thus not sleep!
*/
if (obj == NULL) {
pmap_kenter_pa(loopva, VM_PAGE_TO_PHYS(pg),
UVM_PROT_RW);
} else {
pmap_enter(map->pmap, loopva, VM_PAGE_TO_PHYS(pg),
UVM_PROT_RW,
PMAP_WIRED | VM_PROT_READ | VM_PROT_WRITE);
}
loopva += PAGE_SIZE;
offset += PAGE_SIZE;
}
pmap_update(pmap_kernel());
UVMHIST_LOG(maphist,"<- done (kva=0x%lx)", kva,0,0,0);
return(kva);
}
