static __inline void uvmfault_anonflush(struct vm_anon **anons, int n) { int lcv; struct vm_page *pg; for (lcv = 0 ; lcv < n ; lcv++) { if (anons[lcv] == NULL) continue; simple_lock(&anons[lcv]->an_lock); pg = anons[lcv]->an_page; if (pg && (pg->pg_flags & PG_BUSY) == 0 && pg->loan_count == 0) { uvm_lock_pageq(); if (pg->wire_count == 0) { #ifdef UBC pmap_clear_reference(pg); #else pmap_page_protect(pg, VM_PROT_NONE); #endif uvm_pagedeactivate(pg); } uvm_unlock_pageq(); } simple_unlock(&anons[lcv]->an_lock); } }
/* * uvm_km_pgremove: remove pages from a kernel uvm_object. * * => when you unmap a part of anonymous kernel memory you want to toss * the pages right away. (this gets called from uvm_unmap_...). */ void uvm_km_pgremove(struct uvm_object *uobj, vaddr_t start, vaddr_t end) { struct vm_page *pp; voff_t curoff; UVMHIST_FUNC("uvm_km_pgremove"); UVMHIST_CALLED(maphist); KASSERT(uobj->pgops == &aobj_pager); for (curoff = start ; curoff < end ; curoff += PAGE_SIZE) { pp = uvm_pagelookup(uobj, curoff); if (pp == NULL) continue; UVMHIST_LOG(maphist," page %p, busy=%ld", pp, pp->pg_flags & PG_BUSY, 0, 0); if (pp->pg_flags & PG_BUSY) { /* owner must check for this when done */ atomic_setbits_int(&pp->pg_flags, PG_RELEASED); } else { /* free the swap slot... */ uao_dropswap(uobj, curoff >> PAGE_SHIFT); /* * ...and free the page; note it may be on the * active or inactive queues. */ uvm_lock_pageq(); uvm_pagefree(pp); uvm_unlock_pageq(); } } }
int uvm_objwire(struct uvm_object *uobj, voff_t start, voff_t end, struct pglist *pageq) { int i, npages, left, error; struct vm_page *pgs[FETCH_PAGECOUNT]; voff_t offset = start; left = (end - start) >> PAGE_SHIFT; while (left) { npages = MIN(FETCH_PAGECOUNT, left); /* Get the pages */ memset(pgs, 0, sizeof(pgs)); error = (*uobj->pgops->pgo_get)(uobj, offset, pgs, &npages, 0, PROT_READ | PROT_WRITE, MADV_SEQUENTIAL, PGO_ALLPAGES | PGO_SYNCIO); if (error) goto error; for (i = 0; i < npages; i++) { KASSERT(pgs[i] != NULL); KASSERT(!(pgs[i]->pg_flags & PG_RELEASED)); if (pgs[i]->pg_flags & PQ_AOBJ) { atomic_clearbits_int(&pgs[i]->pg_flags, PG_CLEAN); uao_dropswap(uobj, i); } } /* Wire the pages */ uvm_lock_pageq(); for (i = 0; i < npages; i++) { uvm_pagewire(pgs[i]); if (pageq != NULL) TAILQ_INSERT_TAIL(pageq, pgs[i], pageq); } uvm_unlock_pageq(); /* Unbusy the pages */ uvm_page_unbusy(pgs, npages); left -= npages; offset += (voff_t)npages << PAGE_SHIFT; } return 0; error: /* Unwire the pages which have been wired */ uvm_objunwire(uobj, start, offset); return error; }
void uvm_objunwire(struct uvm_object *uobj, voff_t start, voff_t end) { struct vm_page *pg; off_t offset; uvm_lock_pageq(); for (offset = start; offset < end; offset += PAGE_SIZE) { pg = uvm_pagelookup(uobj, offset); KASSERT(pg != NULL); KASSERT(!(pg->pg_flags & PG_RELEASED)); uvm_pageunwire(pg); } uvm_unlock_pageq(); }
/* * uvmfault_anonflush: try and deactivate pages in specified anons * * => does not have to deactivate page if it is busy */ static __inline void uvmfault_anonflush(struct vm_anon **anons, int n) { int lcv; struct vm_page *pg; for (lcv = 0 ; lcv < n ; lcv++) { if (anons[lcv] == NULL) continue; mtx_enter(&anons[lcv]->an_lock); pg = anons[lcv]->an_page; if (pg && (pg->pg_flags & PG_BUSY) == 0 && pg->loan_count == 0) { uvm_lock_pageq(); if (pg->wire_count == 0) uvm_pagedeactivate(pg); uvm_unlock_pageq(); } mtx_leave(&anons[lcv]->an_lock); } }
int uvmfault_anonget(struct uvm_faultinfo *ufi, struct vm_amap *amap, struct vm_anon *anon) { boolean_t we_own; /* we own anon's page? */ boolean_t locked; /* did we relock? */ struct vm_page *pg; int result; result = 0; /* XXX shut up gcc */ uvmexp.fltanget++; /* bump rusage counters */ if (anon->an_page) curproc->p_ru.ru_minflt++; else curproc->p_ru.ru_majflt++; /* * loop until we get it, or fail. */ while (1) { we_own = FALSE; /* TRUE if we set PG_BUSY on a page */ pg = anon->an_page; /* * if there is a resident page and it is loaned, then anon * may not own it. call out to uvm_anon_lockpage() to ensure * the real owner of the page has been identified and locked. */ if (pg && pg->loan_count) pg = uvm_anon_lockloanpg(anon); /* * page there? make sure it is not busy/released. */ if (pg) { /* * at this point, if the page has a uobject [meaning * we have it on loan], then that uobject is locked * by us! if the page is busy, we drop all the * locks (including uobject) and try again. */ if ((pg->pg_flags & (PG_BUSY|PG_RELEASED)) == 0) { return (VM_PAGER_OK); } atomic_setbits_int(&pg->pg_flags, PG_WANTED); uvmexp.fltpgwait++; /* * the last unlock must be an atomic unlock+wait on * the owner of page */ if (pg->uobject) { /* owner is uobject ? */ uvmfault_unlockall(ufi, amap, NULL, anon); UVM_UNLOCK_AND_WAIT(pg, &pg->uobject->vmobjlock, FALSE, "anonget1",0); } else { /* anon owns page */ uvmfault_unlockall(ufi, amap, NULL, NULL); UVM_UNLOCK_AND_WAIT(pg,&anon->an_lock,0, "anonget2",0); } /* ready to relock and try again */ } else { /* * no page, we must try and bring it in. */ pg = uvm_pagealloc(NULL, 0, anon, 0); if (pg == NULL) { /* out of RAM. */ uvmfault_unlockall(ufi, amap, NULL, anon); uvmexp.fltnoram++; uvm_wait("flt_noram1"); /* ready to relock and try again */ } else { /* we set the PG_BUSY bit */ we_own = TRUE; uvmfault_unlockall(ufi, amap, NULL, anon); /* * we are passing a PG_BUSY+PG_FAKE+PG_CLEAN * page into the uvm_swap_get function with * all data structures unlocked. note that * it is ok to read an_swslot here because * we hold PG_BUSY on the page. */ uvmexp.pageins++; result = uvm_swap_get(pg, anon->an_swslot, PGO_SYNCIO); /* * we clean up after the i/o below in the * "we_own" case */ /* ready to relock and try again */ } } /* * now relock and try again */ locked = uvmfault_relock(ufi); if (locked || we_own) simple_lock(&anon->an_lock); /* * if we own the page (i.e. we set PG_BUSY), then we need * to clean up after the I/O. there are three cases to * consider: * [1] page released during I/O: free anon and ReFault. * [2] I/O not OK. free the page and cause the fault * to fail. * [3] I/O OK! activate the page and sync with the * non-we_own case (i.e. drop anon lock if not locked). */ if (we_own) { if (pg->pg_flags & PG_WANTED) { /* still holding object lock */ wakeup(pg); } /* un-busy! */ atomic_clearbits_int(&pg->pg_flags, PG_WANTED|PG_BUSY|PG_FAKE); UVM_PAGE_OWN(pg, NULL); /* * if we were RELEASED during I/O, then our anon is * no longer part of an amap. we need to free the * anon and try again. */ if (pg->pg_flags & PG_RELEASED) { pmap_page_protect(pg, VM_PROT_NONE); simple_unlock(&anon->an_lock); uvm_anfree(anon); /* frees page for us */ if (locked) uvmfault_unlockall(ufi, amap, NULL, NULL); uvmexp.fltpgrele++; return (VM_PAGER_REFAULT); /* refault! */ } if (result != VM_PAGER_OK) { KASSERT(result != VM_PAGER_PEND); /* remove page from anon */ anon->an_page = NULL; /* * remove the swap slot from the anon * and mark the anon as having no real slot. * don't free the swap slot, thus preventing * it from being used again. */ uvm_swap_markbad(anon->an_swslot, 1); anon->an_swslot = SWSLOT_BAD; /* * note: page was never !PG_BUSY, so it * can't be mapped and thus no need to * pmap_page_protect it... */ uvm_lock_pageq(); uvm_pagefree(pg); uvm_unlock_pageq(); if (locked) uvmfault_unlockall(ufi, amap, NULL, anon); else simple_unlock(&anon->an_lock); return (VM_PAGER_ERROR); } /* * must be OK, clear modify (already PG_CLEAN) * and activate */ pmap_clear_modify(pg); uvm_lock_pageq(); uvm_pageactivate(pg); uvm_unlock_pageq(); if (!locked) simple_unlock(&anon->an_lock); } /* * we were not able to relock. restart fault. */ if (!locked) return (VM_PAGER_REFAULT); /* * verify no one has touched the amap and moved the anon on us. */ if (ufi != NULL && amap_lookup(&ufi->entry->aref, ufi->orig_rvaddr - ufi->entry->start) != anon) { uvmfault_unlockall(ufi, amap, NULL, anon); return (VM_PAGER_REFAULT); } /* * try it again! */ uvmexp.fltanretry++; continue; } /* while (1) */ /*NOTREACHED*/ }
int uvm_objwire(struct uvm_object *uobj, off_t start, off_t end, struct pglist *pageq) { int i, npages, error; struct vm_page *pgs[FETCH_PAGECOUNT]; off_t offset = start, left; left = (end - start) >> PAGE_SHIFT; mtx_enter(&uobj->vmobjlock); while (left) { npages = MIN(FETCH_PAGECOUNT, left); /* Get the pages */ memset(pgs, 0, sizeof(pgs)); error = (*uobj->pgops->pgo_get)(uobj, offset, pgs, &npages, 0, VM_PROT_READ | VM_PROT_WRITE, UVM_ADV_SEQUENTIAL, PGO_ALLPAGES | PGO_SYNCIO); if (error) goto error; mtx_enter(&uobj->vmobjlock); for (i = 0; i < npages; i++) { KASSERT(pgs[i] != NULL); KASSERT(!(pgs[i]->pg_flags & PG_RELEASED)); #if 0 /* * Loan break */ if (pgs[i]->loan_count) { while (pgs[i]->loan_count) { pg = uvm_loanbreak(pgs[i]); if (!pg) { mtx_leave(&uobj->vmobjlock); uvm_wait("uobjwirepg"); mtx_enter(&uobj->vmobjlock); continue; } } pgs[i] = pg; } #endif if (pgs[i]->pg_flags & PQ_AOBJ) { atomic_clearbits_int(&pgs[i]->pg_flags, PG_CLEAN); uao_dropswap(uobj, i); } } /* Wire the pages */ uvm_lock_pageq(); for (i = 0; i < npages; i++) { uvm_pagewire(pgs[i]); if (pageq != NULL) TAILQ_INSERT_TAIL(pageq, pgs[i], pageq); } uvm_unlock_pageq(); /* Unbusy the pages */ uvm_page_unbusy(pgs, npages); left -= npages; offset += (off_t)npages << PAGE_SHIFT; } mtx_leave(&uobj->vmobjlock); return 0; error: /* Unwire the pages which have been wired */ uvm_objunwire(uobj, start, offset); return error; }