/* * Grow the GDT. */ void gdt_grow(int which) { size_t old_len, new_len; CPU_INFO_ITERATOR cii; struct cpu_info *ci; struct vm_page *pg; vaddr_t va; old_len = gdt_size[which] * sizeof(gdt[0]); gdt_size[which] <<= 1; new_len = old_len << 1; #ifdef XEN if (which != 0) { size_t max_len = MAXGDTSIZ * sizeof(gdt[0]); if (old_len == 0) { gdt_size[which] = MINGDTSIZ; new_len = gdt_size[which] * sizeof(gdt[0]); } for(va = (vaddr_t)(cpu_info_primary.ci_gdt) + old_len + max_len; va < (vaddr_t)(cpu_info_primary.ci_gdt) + new_len + max_len; va += PAGE_SIZE) { while ((pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO)) == NULL) { uvm_wait("gdt_grow"); } pmap_kenter_pa(va, VM_PAGE_TO_PHYS(pg), VM_PROT_READ | VM_PROT_WRITE); } return; } #endif for (CPU_INFO_FOREACH(cii, ci)) { for (va = (vaddr_t)(ci->ci_gdt) + old_len; va < (vaddr_t)(ci->ci_gdt) + new_len; va += PAGE_SIZE) { while ((pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO)) == NULL) { uvm_wait("gdt_grow"); } pmap_kenter_pa(va, VM_PAGE_TO_PHYS(pg), VM_PROT_READ | VM_PROT_WRITE); } } pmap_update(pmap_kernel()); }
/* * Grow the GDT. */ void gdt_grow() { size_t old_len, new_len; CPU_INFO_ITERATOR cii; struct cpu_info *ci; struct vm_page *pg; vaddr_t va; old_len = gdt_size * sizeof(union descriptor); gdt_size <<= 1; new_len = old_len << 1; CPU_INFO_FOREACH(cii, ci) { for (va = (vaddr_t)(ci->ci_gdt) + old_len; va < (vaddr_t)(ci->ci_gdt) + new_len; va += PAGE_SIZE) { while ( (pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO)) == NULL) { uvm_wait("gdt_grow"); } pmap_kenter_pa(va, VM_PAGE_TO_PHYS(pg), VM_PROT_READ | VM_PROT_WRITE); } } }
/* * Allocate shadow GDT for a slave CPU. */ void gdt_alloc_cpu(struct cpu_info *ci) { int max_len = MAXGDTSIZ * sizeof(gdt[0]); int min_len = MINGDTSIZ * sizeof(gdt[0]); struct vm_page *pg; vaddr_t va; ci->ci_gdt = (union descriptor *)uvm_km_alloc(kernel_map, max_len, 0, UVM_KMF_VAONLY); for (va = (vaddr_t)ci->ci_gdt; va < (vaddr_t)ci->ci_gdt + min_len; va += PAGE_SIZE) { while ((pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO)) == NULL) { uvm_wait("gdt_alloc_cpu"); } pmap_kenter_pa(va, VM_PAGE_TO_PHYS(pg), VM_PROT_READ | VM_PROT_WRITE); } pmap_update(pmap_kernel()); memset(ci->ci_gdt, 0, min_len); memcpy(ci->ci_gdt, gdt, gdt_count[0] * sizeof(gdt[0])); setsegment(&ci->ci_gdt[GCPU_SEL].sd, ci, 0xfffff, SDT_MEMRWA, SEL_KPL, 1, 1); }
/* * uvmfault_amapcopy: clear "needs_copy" in a map. * * => called with VM data structures unlocked (usually, see below) * => we get a write lock on the maps and clear needs_copy for a VA * => if we are out of RAM we sleep (waiting for more) */ static void uvmfault_amapcopy(struct uvm_faultinfo *ufi) { /* while we haven't done the job */ while (1) { /* no mapping? give up. */ if (uvmfault_lookup(ufi, TRUE) == FALSE) return; /* copy if needed. */ if (UVM_ET_ISNEEDSCOPY(ufi->entry)) amap_copy(ufi->map, ufi->entry, M_NOWAIT, TRUE, ufi->orig_rvaddr, ufi->orig_rvaddr + 1); /* didn't work? must be out of RAM. unlock and sleep. */ if (UVM_ET_ISNEEDSCOPY(ufi->entry)) { uvmfault_unlockmaps(ufi, TRUE); uvm_wait("fltamapcopy"); continue; } /* got it! unlock and return*/ uvmfault_unlockmaps(ufi, TRUE); return; } /*NOTREACHED*/ }
/* ARGSUSED */ vaddr_t uvm_km_alloc_poolpage1(struct vm_map *map, struct uvm_object *obj, boolean_t waitok) { #if defined(__HAVE_PMAP_DIRECT) struct vm_page *pg; vaddr_t va; again: pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_USERESERVE); if (__predict_false(pg == NULL)) { if (waitok) { uvm_wait("plpg"); goto again; } else return (0); } va = pmap_map_direct(pg); if (__predict_false(va == 0)) uvm_pagefree(pg); return (va); #else vaddr_t va; int s; /* * NOTE: We may be called with a map that doesn't require splvm * protection (e.g. kernel_map). However, it does not hurt to * go to splvm in this case (since unprotected maps will never be * accessed in interrupt context). * * XXX We may want to consider changing the interface to this * XXX function. */ s = splvm(); va = uvm_km_kmemalloc(map, obj, PAGE_SIZE, waitok ? 0 : UVM_KMF_NOWAIT); splx(s); return (va); #endif /* __HAVE_PMAP_DIRECT */ }
static int uvm_loanzero(struct uvm_faultinfo *ufi, void ***output, int flags) { struct vm_anon *anon; struct vm_page *pg; struct vm_amap *amap = ufi->entry->aref.ar_amap; UVMHIST_FUNC(__func__); UVMHIST_CALLED(loanhist); again: mutex_enter(&uvm_loanzero_object.vmobjlock); /* * first, get ahold of our single zero page. */ if (__predict_false((pg = TAILQ_FIRST(&uvm_loanzero_object.memq)) == NULL)) { while ((pg = uvm_pagealloc(&uvm_loanzero_object, 0, NULL, UVM_PGA_ZERO)) == NULL) { mutex_exit(&uvm_loanzero_object.vmobjlock); uvmfault_unlockall(ufi, amap, NULL, NULL); uvm_wait("loanzero"); if (!uvmfault_relock(ufi)) { return (0); } if (amap) { amap_lock(amap); } goto again; } /* got a zero'd page. */ pg->flags &= ~(PG_WANTED|PG_BUSY|PG_FAKE); pg->flags |= PG_RDONLY; mutex_enter(&uvm_pageqlock); uvm_pageactivate(pg); mutex_exit(&uvm_pageqlock); UVM_PAGE_OWN(pg, NULL); } if ((flags & UVM_LOAN_TOANON) == 0) { /* loaning to kernel-page */ mutex_enter(&uvm_pageqlock); pg->loan_count++; mutex_exit(&uvm_pageqlock); mutex_exit(&uvm_loanzero_object.vmobjlock); **output = pg; (*output)++; return (1); } /* * loaning to an anon. check to see if there is already an anon * associated with this page. if so, then just return a reference * to this object. */ if (pg->uanon) { anon = pg->uanon; mutex_enter(&anon->an_lock); anon->an_ref++; mutex_exit(&anon->an_lock); mutex_exit(&uvm_loanzero_object.vmobjlock); **output = anon; (*output)++; return (1); } /* * need to allocate a new anon */ anon = uvm_analloc(); if (anon == NULL) { /* out of swap causes us to fail */ mutex_exit(&uvm_loanzero_object.vmobjlock); uvmfault_unlockall(ufi, amap, NULL, NULL); return (-1); } anon->an_page = pg; pg->uanon = anon; mutex_enter(&uvm_pageqlock); pg->loan_count++; uvm_pageactivate(pg); mutex_exit(&uvm_pageqlock); mutex_exit(&anon->an_lock); mutex_exit(&uvm_loanzero_object.vmobjlock); **output = anon; (*output)++; return (1); }
int uvmfault_anonget(struct uvm_faultinfo *ufi, struct vm_amap *amap, struct vm_anon *anon) { boolean_t we_own; /* we own anon's page? */ boolean_t locked; /* did we relock? */ struct vm_page *pg; int result; result = 0; /* XXX shut up gcc */ uvmexp.fltanget++; /* bump rusage counters */ if (anon->an_page) curproc->p_ru.ru_minflt++; else curproc->p_ru.ru_majflt++; /* * loop until we get it, or fail. */ while (1) { we_own = FALSE; /* TRUE if we set PG_BUSY on a page */ pg = anon->an_page; /* * if there is a resident page and it is loaned, then anon * may not own it. call out to uvm_anon_lockpage() to ensure * the real owner of the page has been identified and locked. */ if (pg && pg->loan_count) pg = uvm_anon_lockloanpg(anon); /* * page there? make sure it is not busy/released. */ if (pg) { /* * at this point, if the page has a uobject [meaning * we have it on loan], then that uobject is locked * by us! if the page is busy, we drop all the * locks (including uobject) and try again. */ if ((pg->pg_flags & (PG_BUSY|PG_RELEASED)) == 0) { return (VM_PAGER_OK); } atomic_setbits_int(&pg->pg_flags, PG_WANTED); uvmexp.fltpgwait++; /* * the last unlock must be an atomic unlock+wait on * the owner of page */ if (pg->uobject) { /* owner is uobject ? */ uvmfault_unlockall(ufi, amap, NULL, anon); UVM_UNLOCK_AND_WAIT(pg, &pg->uobject->vmobjlock, FALSE, "anonget1",0); } else { /* anon owns page */ uvmfault_unlockall(ufi, amap, NULL, NULL); UVM_UNLOCK_AND_WAIT(pg,&anon->an_lock,0, "anonget2",0); } /* ready to relock and try again */ } else { /* * no page, we must try and bring it in. */ pg = uvm_pagealloc(NULL, 0, anon, 0); if (pg == NULL) { /* out of RAM. */ uvmfault_unlockall(ufi, amap, NULL, anon); uvmexp.fltnoram++; uvm_wait("flt_noram1"); /* ready to relock and try again */ } else { /* we set the PG_BUSY bit */ we_own = TRUE; uvmfault_unlockall(ufi, amap, NULL, anon); /* * we are passing a PG_BUSY+PG_FAKE+PG_CLEAN * page into the uvm_swap_get function with * all data structures unlocked. note that * it is ok to read an_swslot here because * we hold PG_BUSY on the page. */ uvmexp.pageins++; result = uvm_swap_get(pg, anon->an_swslot, PGO_SYNCIO); /* * we clean up after the i/o below in the * "we_own" case */ /* ready to relock and try again */ } } /* * now relock and try again */ locked = uvmfault_relock(ufi); if (locked || we_own) simple_lock(&anon->an_lock); /* * if we own the page (i.e. we set PG_BUSY), then we need * to clean up after the I/O. there are three cases to * consider: * [1] page released during I/O: free anon and ReFault. * [2] I/O not OK. free the page and cause the fault * to fail. * [3] I/O OK! activate the page and sync with the * non-we_own case (i.e. drop anon lock if not locked). */ if (we_own) { if (pg->pg_flags & PG_WANTED) { /* still holding object lock */ wakeup(pg); } /* un-busy! */ atomic_clearbits_int(&pg->pg_flags, PG_WANTED|PG_BUSY|PG_FAKE); UVM_PAGE_OWN(pg, NULL); /* * if we were RELEASED during I/O, then our anon is * no longer part of an amap. we need to free the * anon and try again. */ if (pg->pg_flags & PG_RELEASED) { pmap_page_protect(pg, VM_PROT_NONE); simple_unlock(&anon->an_lock); uvm_anfree(anon); /* frees page for us */ if (locked) uvmfault_unlockall(ufi, amap, NULL, NULL); uvmexp.fltpgrele++; return (VM_PAGER_REFAULT); /* refault! */ } if (result != VM_PAGER_OK) { KASSERT(result != VM_PAGER_PEND); /* remove page from anon */ anon->an_page = NULL; /* * remove the swap slot from the anon * and mark the anon as having no real slot. * don't free the swap slot, thus preventing * it from being used again. */ uvm_swap_markbad(anon->an_swslot, 1); anon->an_swslot = SWSLOT_BAD; /* * note: page was never !PG_BUSY, so it * can't be mapped and thus no need to * pmap_page_protect it... */ uvm_lock_pageq(); uvm_pagefree(pg); uvm_unlock_pageq(); if (locked) uvmfault_unlockall(ufi, amap, NULL, anon); else simple_unlock(&anon->an_lock); return (VM_PAGER_ERROR); } /* * must be OK, clear modify (already PG_CLEAN) * and activate */ pmap_clear_modify(pg); uvm_lock_pageq(); uvm_pageactivate(pg); uvm_unlock_pageq(); if (!locked) simple_unlock(&anon->an_lock); } /* * we were not able to relock. restart fault. */ if (!locked) return (VM_PAGER_REFAULT); /* * verify no one has touched the amap and moved the anon on us. */ if (ufi != NULL && amap_lookup(&ufi->entry->aref, ufi->orig_rvaddr - ufi->entry->start) != anon) { uvmfault_unlockall(ufi, amap, NULL, anon); return (VM_PAGER_REFAULT); } /* * try it again! */ uvmexp.fltanretry++; continue; } /* while (1) */ /*NOTREACHED*/ }
int uvm_objwire(struct uvm_object *uobj, off_t start, off_t end, struct pglist *pageq) { int i, npages, error; struct vm_page *pgs[FETCH_PAGECOUNT]; off_t offset = start, left; left = (end - start) >> PAGE_SHIFT; mtx_enter(&uobj->vmobjlock); while (left) { npages = MIN(FETCH_PAGECOUNT, left); /* Get the pages */ memset(pgs, 0, sizeof(pgs)); error = (*uobj->pgops->pgo_get)(uobj, offset, pgs, &npages, 0, VM_PROT_READ | VM_PROT_WRITE, UVM_ADV_SEQUENTIAL, PGO_ALLPAGES | PGO_SYNCIO); if (error) goto error; mtx_enter(&uobj->vmobjlock); for (i = 0; i < npages; i++) { KASSERT(pgs[i] != NULL); KASSERT(!(pgs[i]->pg_flags & PG_RELEASED)); #if 0 /* * Loan break */ if (pgs[i]->loan_count) { while (pgs[i]->loan_count) { pg = uvm_loanbreak(pgs[i]); if (!pg) { mtx_leave(&uobj->vmobjlock); uvm_wait("uobjwirepg"); mtx_enter(&uobj->vmobjlock); continue; } } pgs[i] = pg; } #endif if (pgs[i]->pg_flags & PQ_AOBJ) { atomic_clearbits_int(&pgs[i]->pg_flags, PG_CLEAN); uao_dropswap(uobj, i); } } /* Wire the pages */ uvm_lock_pageq(); for (i = 0; i < npages; i++) { uvm_pagewire(pgs[i]); if (pageq != NULL) TAILQ_INSERT_TAIL(pageq, pgs[i], pageq); } uvm_unlock_pageq(); /* Unbusy the pages */ uvm_page_unbusy(pgs, npages); left -= npages; offset += (off_t)npages << PAGE_SHIFT; } mtx_leave(&uobj->vmobjlock); return 0; error: /* Unwire the pages which have been wired */ uvm_objunwire(uobj, start, offset); return error; }
vaddr_t uvm_km_alloc1(struct vm_map *map, vsize_t size, vsize_t align, boolean_t zeroit) { vaddr_t kva, loopva; voff_t offset; struct vm_page *pg; UVMHIST_FUNC("uvm_km_alloc1"); UVMHIST_CALLED(maphist); UVMHIST_LOG(maphist,"(map=%p, size=0x%lx)", map, size,0,0); KASSERT(vm_map_pmap(map) == pmap_kernel()); size = round_page(size); kva = vm_map_min(map); /* hint */ /* * allocate some virtual space */ if (__predict_false(uvm_map(map, &kva, size, uvm.kernel_object, UVM_UNKNOWN_OFFSET, align, UVM_MAPFLAG(UVM_PROT_ALL, UVM_PROT_ALL, UVM_INH_NONE, UVM_ADV_RANDOM, 0)) != 0)) { UVMHIST_LOG(maphist,"<- done (no VM)",0,0,0,0); return(0); } /* * recover object offset from virtual address */ offset = kva - vm_map_min(kernel_map); UVMHIST_LOG(maphist," kva=0x%lx, offset=0x%lx", kva, offset,0,0); /* * now allocate the memory. we must be careful about released pages. */ loopva = kva; while (size) { simple_lock(&uvm.kernel_object->vmobjlock); pg = uvm_pagelookup(uvm.kernel_object, offset); /* * if we found a page in an unallocated region, it must be * released */ if (pg) { if ((pg->pg_flags & PG_RELEASED) == 0) panic("uvm_km_alloc1: non-released page"); atomic_setbits_int(&pg->pg_flags, PG_WANTED); UVM_UNLOCK_AND_WAIT(pg, &uvm.kernel_object->vmobjlock, FALSE, "km_alloc", 0); continue; /* retry */ } /* allocate ram */ pg = uvm_pagealloc(uvm.kernel_object, offset, NULL, 0); if (pg) { atomic_clearbits_int(&pg->pg_flags, PG_BUSY); UVM_PAGE_OWN(pg, NULL); } simple_unlock(&uvm.kernel_object->vmobjlock); if (__predict_false(pg == NULL)) { if (curproc == uvm.pagedaemon_proc) { /* * It is unfeasible for the page daemon to * sleep for memory, so free what we have * allocated and fail. */ uvm_unmap(map, kva, loopva - kva); return (NULL); } else { uvm_wait("km_alloc1w"); /* wait for memory */ continue; } } /* * map it in; note we're never called with an intrsafe * object, so we always use regular old pmap_enter(). */ pmap_enter(map->pmap, loopva, VM_PAGE_TO_PHYS(pg), UVM_PROT_ALL, PMAP_WIRED | VM_PROT_READ | VM_PROT_WRITE); loopva += PAGE_SIZE; offset += PAGE_SIZE; size -= PAGE_SIZE; } pmap_update(map->pmap); /* * zero on request (note that "size" is now zero due to the above loop * so we need to subtract kva from loopva to reconstruct the size). */ if (zeroit) memset((caddr_t)kva, 0, loopva - kva); UVMHIST_LOG(maphist,"<- done (kva=0x%lx)", kva,0,0,0); return(kva); }
vaddr_t uvm_km_kmemalloc(struct vm_map *map, struct uvm_object *obj, vsize_t size, int flags) { vaddr_t kva, loopva; voff_t offset; struct vm_page *pg; int mapflags; UVMHIST_FUNC("uvm_km_kmemalloc"); UVMHIST_CALLED(maphist); UVMHIST_LOG(maphist," (map=%p, obj=%p, size=0x%lx, flags=%d)", map, obj, size, flags); KASSERT(vm_map_pmap(map) == pmap_kernel()); /* * we cannot yet make pmap_enter() not sleep * and thus demand that we are called with NOWAIT in that case */ KASSERT(!((flags & UVM_KMF_NOWAIT) && obj)); /* * setup for call */ mapflags = flags & UVM_KMF_NOWAIT? UVM_FLAG_NOWAIT : 0; mapflags |= flags & UVM_KMF_TRYLOCK; size = round_page(size); kva = vm_map_min(map); /* hint */ /* * allocate some virtual space */ if (__predict_false(uvm_map(map, &kva, size, obj, UVM_UNKNOWN_OFFSET, 0, UVM_MAPFLAG(UVM_PROT_RW, UVM_PROT_RW, UVM_INH_NONE, UVM_ADV_RANDOM, mapflags)) != 0)) { UVMHIST_LOG(maphist, "<- done (no VM)",0,0,0,0); return(0); } /* * if all we wanted was VA, return now */ if (flags & UVM_KMF_VALLOC) { UVMHIST_LOG(maphist,"<- done valloc (kva=0x%lx)", kva,0,0,0); return(kva); } /* * recover object offset from virtual address */ if (obj != NULL) offset = kva - vm_map_min(kernel_map); else offset = 0; UVMHIST_LOG(maphist, " kva=0x%lx, offset=0x%lx", kva, offset,0,0); /* * now allocate and map in the memory... note that we are the only ones * whom should ever get a handle on this area of VM. */ loopva = kva; while (loopva != kva + size) { pg = uvm_pagealloc(obj, offset, NULL, 0); if (pg) { atomic_clearbits_int(&pg->pg_flags, PG_BUSY); UVM_PAGE_OWN(pg, NULL); } if (__predict_false(pg == NULL)) { if ((flags & UVM_KMF_NOWAIT) || ((flags & UVM_KMF_CANFAIL) && uvmexp.swpgonly == uvmexp.swpages)) { /* free everything! */ uvm_unmap(map, kva, kva + size); return (0); } else { uvm_wait("km_getwait2"); /* sleep here */ continue; } } /* * map it in: note that we call pmap_enter with the map and * object unlocked in case we are kmem_map. * * pager mappings that must not sleep here will incidently * be installed using pmap_kenter_pa() and thus not sleep! */ if (obj == NULL) { pmap_kenter_pa(loopva, VM_PAGE_TO_PHYS(pg), UVM_PROT_RW); } else { pmap_enter(map->pmap, loopva, VM_PAGE_TO_PHYS(pg), UVM_PROT_RW, PMAP_WIRED | VM_PROT_READ | VM_PROT_WRITE); } loopva += PAGE_SIZE; offset += PAGE_SIZE; } pmap_update(pmap_kernel()); UVMHIST_LOG(maphist,"<- done (kva=0x%lx)", kva,0,0,0); return(kva); }