/*
* uvm_loanbreak: break loan on a uobj page
*
* => called with uobj locked
* => the page should be busy
* => return value:
* newly allocated page if succeeded
*/
struct vm_page *
uvm_loanbreak(struct vm_page *uobjpage)
{
struct vm_page *pg;
#ifdef DIAGNOSTIC
struct uvm_object *uobj = uobjpage->uobject;
#endif
KASSERT(uobj != NULL);
KASSERT(mutex_owned(&uobj->vmobjlock));
KASSERT(uobjpage->flags & PG_BUSY);
/* alloc new un-owned page */
pg = uvm_pagealloc(NULL, 0, NULL, 0);
if (pg == NULL)
return NULL;
/*
* copy the data from the old page to the new
* one and clear the fake flags on the new page (keep it busy).
* force a reload of the old page by clearing it from all
* pmaps.
* transfer dirtiness of the old page to the new page.
* then lock the page queues to rename the pages.
*/
uvm_pagecopy(uobjpage, pg); /* old -> new */
pg->flags &= ~PG_FAKE;
pmap_page_protect(uobjpage, VM_PROT_NONE);
if ((uobjpage->flags & PG_CLEAN) != 0 && !pmap_clear_modify(uobjpage)) {
pmap_clear_modify(pg);
pg->flags |= PG_CLEAN;
} else {
/* uvm_pagecopy marked it dirty */
KASSERT((pg->flags & PG_CLEAN) == 0);
/* a object with a dirty page should be dirty. */
KASSERT(!UVM_OBJ_IS_CLEAN(uobj));
}
if (uobjpage->flags & PG_WANTED)
wakeup(uobjpage);
/* uobj still locked */
uobjpage->flags &= ~(PG_WANTED|PG_BUSY);
UVM_PAGE_OWN(uobjpage, NULL);
mutex_enter(&uvm_pageqlock);
/*
* replace uobjpage with new page.
*/
uvm_pagereplace(uobjpage, pg);
/*
* if the page is no longer referenced by
* an anon (i.e. we are breaking an O->K
* loan), then remove it from any pageq's.
*/
if (uobjpage->uanon == NULL)
uvm_pagedequeue(uobjpage);
/*
* at this point we have absolutely no
* control over uobjpage
*/
/* install new page */
uvm_pageactivate(pg);
mutex_exit(&uvm_pageqlock);
/*
* done! loan is broken and "pg" is
* PG_BUSY. it can now replace uobjpage.
*/
return pg;
}
int
uvmfault_anonget(struct uvm_faultinfo *ufi, struct vm_amap *amap,
struct vm_anon *anon)
{
boolean_t we_own; /* we own anon's page? */
boolean_t locked; /* did we relock? */
struct vm_page *pg;
int result;
result = 0; /* XXX shut up gcc */
uvmexp.fltanget++;
/* bump rusage counters */
if (anon->an_page)
curproc->p_ru.ru_minflt++;
else
curproc->p_ru.ru_majflt++;
/*
* loop until we get it, or fail.
*/
while (1) {
we_own = FALSE; /* TRUE if we set PG_BUSY on a page */
pg = anon->an_page;
/*
* if there is a resident page and it is loaned, then anon
* may not own it. call out to uvm_anon_lockpage() to ensure
* the real owner of the page has been identified and locked.
*/
if (pg && pg->loan_count)
pg = uvm_anon_lockloanpg(anon);
/*
* page there? make sure it is not busy/released.
*/
if (pg) {
/*
* at this point, if the page has a uobject [meaning
* we have it on loan], then that uobject is locked
* by us! if the page is busy, we drop all the
* locks (including uobject) and try again.
*/
if ((pg->pg_flags & (PG_BUSY|PG_RELEASED)) == 0) {
return (VM_PAGER_OK);
}
atomic_setbits_int(&pg->pg_flags, PG_WANTED);
uvmexp.fltpgwait++;
/*
* the last unlock must be an atomic unlock+wait on
* the owner of page
*/
if (pg->uobject) { /* owner is uobject ? */
uvmfault_unlockall(ufi, amap, NULL, anon);
UVM_UNLOCK_AND_WAIT(pg,
&pg->uobject->vmobjlock,
FALSE, "anonget1",0);
} else {
/* anon owns page */
uvmfault_unlockall(ufi, amap, NULL, NULL);
UVM_UNLOCK_AND_WAIT(pg,&anon->an_lock,0,
"anonget2",0);
}
/* ready to relock and try again */
} else {
/*
* no page, we must try and bring it in.
*/
pg = uvm_pagealloc(NULL, 0, anon, 0);
if (pg == NULL) { /* out of RAM. */
uvmfault_unlockall(ufi, amap, NULL, anon);
uvmexp.fltnoram++;
uvm_wait("flt_noram1");
/* ready to relock and try again */
} else {
/* we set the PG_BUSY bit */
we_own = TRUE;
uvmfault_unlockall(ufi, amap, NULL, anon);
/*
* we are passing a PG_BUSY+PG_FAKE+PG_CLEAN
* page into the uvm_swap_get function with
* all data structures unlocked. note that
* it is ok to read an_swslot here because
* we hold PG_BUSY on the page.
*/
uvmexp.pageins++;
result = uvm_swap_get(pg, anon->an_swslot,
PGO_SYNCIO);
/*
* we clean up after the i/o below in the
* "we_own" case
*/
/* ready to relock and try again */
}
}
/*
* now relock and try again
*/
locked = uvmfault_relock(ufi);
if (locked || we_own)
simple_lock(&anon->an_lock);
/*
* if we own the page (i.e. we set PG_BUSY), then we need
* to clean up after the I/O. there are three cases to
* consider:
* [1] page released during I/O: free anon and ReFault.
* [2] I/O not OK. free the page and cause the fault
* to fail.
* [3] I/O OK! activate the page and sync with the
* non-we_own case (i.e. drop anon lock if not locked).
*/
if (we_own) {
if (pg->pg_flags & PG_WANTED) {
/* still holding object lock */
wakeup(pg);
}
/* un-busy! */
atomic_clearbits_int(&pg->pg_flags,
PG_WANTED|PG_BUSY|PG_FAKE);
UVM_PAGE_OWN(pg, NULL);
/*
* if we were RELEASED during I/O, then our anon is
* no longer part of an amap. we need to free the
* anon and try again.
*/
if (pg->pg_flags & PG_RELEASED) {
pmap_page_protect(pg, VM_PROT_NONE);
simple_unlock(&anon->an_lock);
uvm_anfree(anon); /* frees page for us */
if (locked)
uvmfault_unlockall(ufi, amap, NULL,
NULL);
uvmexp.fltpgrele++;
return (VM_PAGER_REFAULT); /* refault! */
}
if (result != VM_PAGER_OK) {
KASSERT(result != VM_PAGER_PEND);
/* remove page from anon */
anon->an_page = NULL;
/*
* remove the swap slot from the anon
* and mark the anon as having no real slot.
* don't free the swap slot, thus preventing
* it from being used again.
*/
uvm_swap_markbad(anon->an_swslot, 1);
anon->an_swslot = SWSLOT_BAD;
/*
* note: page was never !PG_BUSY, so it
* can't be mapped and thus no need to
* pmap_page_protect it...
*/
uvm_lock_pageq();
uvm_pagefree(pg);
uvm_unlock_pageq();
if (locked)
uvmfault_unlockall(ufi, amap, NULL,
anon);
else
simple_unlock(&anon->an_lock);
return (VM_PAGER_ERROR);
}
/*
* must be OK, clear modify (already PG_CLEAN)
* and activate
*/
pmap_clear_modify(pg);
uvm_lock_pageq();
uvm_pageactivate(pg);
uvm_unlock_pageq();
if (!locked)
simple_unlock(&anon->an_lock);
}
/*
* we were not able to relock. restart fault.
*/
if (!locked)
return (VM_PAGER_REFAULT);
/*
* verify no one has touched the amap and moved the anon on us.
*/
if (ufi != NULL &&
amap_lookup(&ufi->entry->aref,
ufi->orig_rvaddr - ufi->entry->start) != anon) {
uvmfault_unlockall(ufi, amap, NULL, anon);
return (VM_PAGER_REFAULT);
}
/*
* try it again!
*/
uvmexp.fltanretry++;
continue;
} /* while (1) */
/*NOTREACHED*/
}
