static int
uvm_loanuobj(struct uvm_faultinfo *ufi, void ***output, int flags, vaddr_t va)
{
struct vm_amap *amap = ufi->entry->aref.ar_amap;
struct uvm_object *uobj = ufi->entry->object.uvm_obj;
struct vm_page *pg;
struct vm_anon *anon;
int error, npages;
bool locked;
UVMHIST_FUNC(__func__); UVMHIST_CALLED(loanhist);
/*
* first we must make sure the page is resident.
*
* XXXCDC: duplicate code with uvm_fault().
*/
mutex_enter(&uobj->vmobjlock);
if (uobj->pgops->pgo_get) { /* try locked pgo_get */
npages = 1;
pg = NULL;
error = (*uobj->pgops->pgo_get)(uobj,
va - ufi->entry->start + ufi->entry->offset,
&pg, &npages, 0, VM_PROT_READ, MADV_NORMAL, PGO_LOCKED);
} else {
error = EIO; /* must have pgo_get op */
}
/*
* check the result of the locked pgo_get. if there is a problem,
* then we fail the loan.
*/
if (error && error != EBUSY) {
uvmfault_unlockall(ufi, amap, uobj, NULL);
return (-1);
}
/*
* if we need to unlock for I/O, do so now.
*/
if (error == EBUSY) {
uvmfault_unlockall(ufi, amap, NULL, NULL);
/* locked: uobj */
npages = 1;
error = (*uobj->pgops->pgo_get)(uobj,
va - ufi->entry->start + ufi->entry->offset,
&pg, &npages, 0, VM_PROT_READ, MADV_NORMAL, PGO_SYNCIO);
/* locked: <nothing> */
if (error) {
if (error == EAGAIN) {
tsleep(&lbolt, PVM, "fltagain2", 0);
return (0);
}
return (-1);
}
/*
* pgo_get was a success. attempt to relock everything.
*/
locked = uvmfault_relock(ufi);
if (locked && amap)
amap_lock(amap);
uobj = pg->uobject;
mutex_enter(&uobj->vmobjlock);
/*
* verify that the page has not be released and re-verify
* that amap slot is still free. if there is a problem we
* drop our lock (thus force a lookup refresh/retry).
*/
if ((pg->flags & PG_RELEASED) != 0 ||
(locked && amap && amap_lookup(&ufi->entry->aref,
ufi->orig_rvaddr - ufi->entry->start))) {
if (locked)
uvmfault_unlockall(ufi, amap, NULL, NULL);
locked = false;
}
/*
* didn't get the lock? release the page and retry.
*/
if (locked == false) {
if (pg->flags & PG_WANTED) {
wakeup(pg);
}
if (pg->flags & PG_RELEASED) {
mutex_enter(&uvm_pageqlock);
uvm_pagefree(pg);
mutex_exit(&uvm_pageqlock);
mutex_exit(&uobj->vmobjlock);
return (0);
}
mutex_enter(&uvm_pageqlock);
uvm_pageactivate(pg);
mutex_exit(&uvm_pageqlock);
pg->flags &= ~(PG_BUSY|PG_WANTED);
UVM_PAGE_OWN(pg, NULL);
mutex_exit(&uobj->vmobjlock);
return (0);
}
}
KASSERT(uobj == pg->uobject);
/*
* at this point we have the page we want ("pg") marked PG_BUSY for us
* and we have all data structures locked. do the loanout. page can
* not be PG_RELEASED (we caught this above).
*/
if ((flags & UVM_LOAN_TOANON) == 0) {
if (uvm_loanpage(&pg, 1)) {
uvmfault_unlockall(ufi, amap, uobj, NULL);
return (-1);
}
mutex_exit(&uobj->vmobjlock);
**output = pg;
(*output)++;
return (1);
}
/*
* must be a loan to an anon. check to see if there is already
* an anon associated with this page. if so, then just return
* a reference to this object. the page should already be
* mapped read-only because it is already on loan.
*/
if (pg->uanon) {
anon = pg->uanon;
mutex_enter(&anon->an_lock);
anon->an_ref++;
mutex_exit(&anon->an_lock);
if (pg->flags & PG_WANTED) {
wakeup(pg);
}
pg->flags &= ~(PG_WANTED|PG_BUSY);
UVM_PAGE_OWN(pg, NULL);
mutex_exit(&uobj->vmobjlock);
**output = anon;
(*output)++;
return (1);
}
/*
* need to allocate a new anon
*/
anon = uvm_analloc();
if (anon == NULL) {
goto fail;
}
anon->an_page = pg;
pg->uanon = anon;
mutex_enter(&uvm_pageqlock);
if (pg->wire_count > 0) {
mutex_exit(&uvm_pageqlock);
UVMHIST_LOG(loanhist, "wired %p", pg,0,0,0);
pg->uanon = NULL;
anon->an_page = NULL;
anon->an_ref--;
mutex_exit(&anon->an_lock);
uvm_anfree(anon);
goto fail;
}
if (pg->loan_count == 0) {
pmap_page_protect(pg, VM_PROT_READ);
}
pg->loan_count++;
uvm_pageactivate(pg);
mutex_exit(&uvm_pageqlock);
if (pg->flags & PG_WANTED) {
wakeup(pg);
}
pg->flags &= ~(PG_WANTED|PG_BUSY);
UVM_PAGE_OWN(pg, NULL);
mutex_exit(&uobj->vmobjlock);
mutex_exit(&anon->an_lock);
**output = anon;
(*output)++;
return (1);
fail:
UVMHIST_LOG(loanhist, "fail", 0,0,0,0);
/*
* unlock everything and bail out.
*/
if (pg->flags & PG_WANTED) {
wakeup(pg);
}
pg->flags &= ~(PG_WANTED|PG_BUSY);
UVM_PAGE_OWN(pg, NULL);
uvmfault_unlockall(ufi, amap, uobj, NULL);
return (-1);
}
static inline int
uvm_loanentry(struct uvm_faultinfo *ufi, void ***output, int flags)
{
vaddr_t curaddr = ufi->orig_rvaddr;
vsize_t togo = ufi->size;
struct vm_aref *aref = &ufi->entry->aref;
struct uvm_object *uobj = ufi->entry->object.uvm_obj;
struct vm_anon *anon;
int rv, result = 0;
UVMHIST_FUNC(__func__); UVMHIST_CALLED(loanhist);
/*
* lock us the rest of the way down (we unlock before return)
*/
if (aref->ar_amap)
amap_lock(aref->ar_amap);
/*
* loop until done
*/
while (togo) {
/*
* find the page we want. check the anon layer first.
*/
if (aref->ar_amap) {
anon = amap_lookup(aref, curaddr - ufi->entry->start);
} else {
anon = NULL;
}
/* locked: map, amap, uobj */
if (anon) {
rv = uvm_loananon(ufi, output, flags, anon);
} else if (uobj) {
rv = uvm_loanuobj(ufi, output, flags, curaddr);
} else if (UVM_ET_ISCOPYONWRITE(ufi->entry)) {
rv = uvm_loanzero(ufi, output, flags);
} else {
uvmfault_unlockall(ufi, aref->ar_amap, uobj, NULL);
rv = -1;
}
/* locked: if (rv > 0) => map, amap, uobj [o.w. unlocked] */
KASSERT(rv > 0 || aref->ar_amap == NULL ||
!mutex_owned(&aref->ar_amap->am_l));
KASSERT(rv > 0 || uobj == NULL ||
!mutex_owned(&uobj->vmobjlock));
/* total failure */
if (rv < 0) {
UVMHIST_LOG(loanhist, "failure %d", rv, 0,0,0);
return (-1);
}
/* relock failed, need to do another lookup */
if (rv == 0) {
UVMHIST_LOG(loanhist, "relock failure %d", result
,0,0,0);
return (result);
}
/*
* got it... advance to next page
*/
result++;
togo -= PAGE_SIZE;
curaddr += PAGE_SIZE;
}
/*
* unlock what we locked, unlock the maps and return
*/
if (aref->ar_amap)
amap_unlock(aref->ar_amap);
uvmfault_unlockmaps(ufi, false);
UVMHIST_LOG(loanhist, "done %d", result, 0,0,0);
return (result);
}
/* ARGSUSED */
int
sys_mincore(struct lwp *l, const struct sys_mincore_args *uap,
register_t *retval)
{
/* {
syscallarg(void *) addr;
syscallarg(size_t) len;
syscallarg(char *) vec;
} */
struct proc *p = l->l_proc;
struct vm_page *pg;
char *vec, pgi;
struct uvm_object *uobj;
struct vm_amap *amap;
struct vm_anon *anon;
struct vm_map_entry *entry;
vaddr_t start, end, lim;
struct vm_map *map;
vsize_t len;
int error = 0, npgs;
map = &p->p_vmspace->vm_map;
start = (vaddr_t)SCARG(uap, addr);
len = SCARG(uap, len);
vec = SCARG(uap, vec);
if (start & PAGE_MASK)
return EINVAL;
len = round_page(len);
end = start + len;
if (end <= start)
return EINVAL;
/*
* Lock down vec, so our returned status isn't outdated by
* storing the status byte for a page.
*/
npgs = len >> PAGE_SHIFT;
error = uvm_vslock(p->p_vmspace, vec, npgs, VM_PROT_WRITE);
if (error) {
return error;
}
vm_map_lock_read(map);
if (uvm_map_lookup_entry(map, start, &entry) == false) {
error = ENOMEM;
goto out;
}
for (/* nothing */;
entry != &map->header && entry->start < end;
entry = entry->next) {
KASSERT(!UVM_ET_ISSUBMAP(entry));
KASSERT(start >= entry->start);
/* Make sure there are no holes. */
if (entry->end < end &&
(entry->next == &map->header ||
entry->next->start > entry->end)) {
error = ENOMEM;
goto out;
}
lim = end < entry->end ? end : entry->end;
/*
* Special case for objects with no "real" pages. Those
* are always considered resident (mapped devices).
*/
if (UVM_ET_ISOBJ(entry)) {
KASSERT(!UVM_OBJ_IS_KERN_OBJECT(entry->object.uvm_obj));
if (UVM_OBJ_IS_DEVICE(entry->object.uvm_obj)) {
for (/* nothing */; start < lim;
start += PAGE_SIZE, vec++)
subyte(vec, 1);
continue;
}
}
amap = entry->aref.ar_amap; /* upper layer */
uobj = entry->object.uvm_obj; /* lower layer */
if (amap != NULL)
amap_lock(amap);
if (uobj != NULL)
mutex_enter(uobj->vmobjlock);
for (/* nothing */; start < lim; start += PAGE_SIZE, vec++) {
pgi = 0;
if (amap != NULL) {
/* Check the upper layer first. */
anon = amap_lookup(&entry->aref,
start - entry->start);
/* Don't need to lock anon here. */
if (anon != NULL && anon->an_page != NULL) {
/*
* Anon has the page for this entry
* offset.
*/
pgi = 1;
}
}
if (uobj != NULL && pgi == 0) {
/* Check the lower layer. */
pg = uvm_pagelookup(uobj,
entry->offset + (start - entry->start));
if (pg != NULL) {
/*
* Object has the page for this entry
* offset.
*/
pgi = 1;
}
}
(void) subyte(vec, pgi);
}
if (uobj != NULL)
mutex_exit(uobj->vmobjlock);
if (amap != NULL)
amap_unlock(amap);
}
out:
vm_map_unlock_read(map);
uvm_vsunlock(p->p_vmspace, SCARG(uap, vec), npgs);
return error;
}
int
uvmfault_anonget(struct uvm_faultinfo *ufi, struct vm_amap *amap,
struct vm_anon *anon)
{
boolean_t we_own; /* we own anon's page? */
boolean_t locked; /* did we relock? */
struct vm_page *pg;
int result;
result = 0; /* XXX shut up gcc */
uvmexp.fltanget++;
/* bump rusage counters */
if (anon->an_page)
curproc->p_ru.ru_minflt++;
else
curproc->p_ru.ru_majflt++;
/*
* loop until we get it, or fail.
*/
while (1) {
we_own = FALSE; /* TRUE if we set PG_BUSY on a page */
pg = anon->an_page;
/*
* if there is a resident page and it is loaned, then anon
* may not own it. call out to uvm_anon_lockpage() to ensure
* the real owner of the page has been identified and locked.
*/
if (pg && pg->loan_count)
pg = uvm_anon_lockloanpg(anon);
/*
* page there? make sure it is not busy/released.
*/
if (pg) {
/*
* at this point, if the page has a uobject [meaning
* we have it on loan], then that uobject is locked
* by us! if the page is busy, we drop all the
* locks (including uobject) and try again.
*/
if ((pg->pg_flags & (PG_BUSY|PG_RELEASED)) == 0) {
return (VM_PAGER_OK);
}
atomic_setbits_int(&pg->pg_flags, PG_WANTED);
uvmexp.fltpgwait++;
/*
* the last unlock must be an atomic unlock+wait on
* the owner of page
*/
if (pg->uobject) { /* owner is uobject ? */
uvmfault_unlockall(ufi, amap, NULL, anon);
UVM_UNLOCK_AND_WAIT(pg,
&pg->uobject->vmobjlock,
FALSE, "anonget1",0);
} else {
/* anon owns page */
uvmfault_unlockall(ufi, amap, NULL, NULL);
UVM_UNLOCK_AND_WAIT(pg,&anon->an_lock,0,
"anonget2",0);
}
/* ready to relock and try again */
} else {
/*
* no page, we must try and bring it in.
*/
pg = uvm_pagealloc(NULL, 0, anon, 0);
if (pg == NULL) { /* out of RAM. */
uvmfault_unlockall(ufi, amap, NULL, anon);
uvmexp.fltnoram++;
uvm_wait("flt_noram1");
/* ready to relock and try again */
} else {
/* we set the PG_BUSY bit */
we_own = TRUE;
uvmfault_unlockall(ufi, amap, NULL, anon);
/*
* we are passing a PG_BUSY+PG_FAKE+PG_CLEAN
* page into the uvm_swap_get function with
* all data structures unlocked. note that
* it is ok to read an_swslot here because
* we hold PG_BUSY on the page.
*/
uvmexp.pageins++;
result = uvm_swap_get(pg, anon->an_swslot,
PGO_SYNCIO);
/*
* we clean up after the i/o below in the
* "we_own" case
*/
/* ready to relock and try again */
}
}
/*
* now relock and try again
*/
locked = uvmfault_relock(ufi);
if (locked || we_own)
simple_lock(&anon->an_lock);
/*
* if we own the page (i.e. we set PG_BUSY), then we need
* to clean up after the I/O. there are three cases to
* consider:
* [1] page released during I/O: free anon and ReFault.
* [2] I/O not OK. free the page and cause the fault
* to fail.
* [3] I/O OK! activate the page and sync with the
* non-we_own case (i.e. drop anon lock if not locked).
*/
if (we_own) {
if (pg->pg_flags & PG_WANTED) {
/* still holding object lock */
wakeup(pg);
}
/* un-busy! */
atomic_clearbits_int(&pg->pg_flags,
PG_WANTED|PG_BUSY|PG_FAKE);
UVM_PAGE_OWN(pg, NULL);
/*
* if we were RELEASED during I/O, then our anon is
* no longer part of an amap. we need to free the
* anon and try again.
*/
if (pg->pg_flags & PG_RELEASED) {
pmap_page_protect(pg, VM_PROT_NONE);
simple_unlock(&anon->an_lock);
uvm_anfree(anon); /* frees page for us */
if (locked)
uvmfault_unlockall(ufi, amap, NULL,
NULL);
uvmexp.fltpgrele++;
return (VM_PAGER_REFAULT); /* refault! */
}
if (result != VM_PAGER_OK) {
KASSERT(result != VM_PAGER_PEND);
/* remove page from anon */
anon->an_page = NULL;
/*
* remove the swap slot from the anon
* and mark the anon as having no real slot.
* don't free the swap slot, thus preventing
* it from being used again.
*/
uvm_swap_markbad(anon->an_swslot, 1);
anon->an_swslot = SWSLOT_BAD;
/*
* note: page was never !PG_BUSY, so it
* can't be mapped and thus no need to
* pmap_page_protect it...
*/
uvm_lock_pageq();
uvm_pagefree(pg);
uvm_unlock_pageq();
if (locked)
uvmfault_unlockall(ufi, amap, NULL,
anon);
else
simple_unlock(&anon->an_lock);
return (VM_PAGER_ERROR);
}
/*
* must be OK, clear modify (already PG_CLEAN)
* and activate
*/
pmap_clear_modify(pg);
uvm_lock_pageq();
uvm_pageactivate(pg);
uvm_unlock_pageq();
if (!locked)
simple_unlock(&anon->an_lock);
}
/*
* we were not able to relock. restart fault.
*/
if (!locked)
return (VM_PAGER_REFAULT);
/*
* verify no one has touched the amap and moved the anon on us.
*/
if (ufi != NULL &&
amap_lookup(&ufi->entry->aref,
ufi->orig_rvaddr - ufi->entry->start) != anon) {
uvmfault_unlockall(ufi, amap, NULL, anon);
return (VM_PAGER_REFAULT);
}
/*
* try it again!
*/
uvmexp.fltanretry++;
continue;
} /* while (1) */
/*NOTREACHED*/
}
/* ARGSUSED */
int
sys_mincore(struct proc *p, void *v, register_t *retval)
{
struct sys_mincore_args /* {
syscallarg(void *) addr;
syscallarg(size_t) len;
syscallarg(char *) vec;
} */ *uap = v;
vm_page_t m;
char *vec, pgi;
struct uvm_object *uobj;
struct vm_amap *amap;
struct vm_anon *anon;
vm_map_entry_t entry;
vaddr_t start, end, lim;
vm_map_t map;
vsize_t len, npgs;
int error = 0;
map = &p->p_vmspace->vm_map;
start = (vaddr_t)SCARG(uap, addr);
len = SCARG(uap, len);
vec = SCARG(uap, vec);
if (start & PAGE_MASK)
return (EINVAL);
len = round_page(len);
end = start + len;
if (end <= start)
return (EINVAL);
npgs = len >> PAGE_SHIFT;
/*
* Lock down vec, so our returned status isn't outdated by
* storing the status byte for a page.
*/
if ((error = uvm_vslock(p, vec, npgs, VM_PROT_WRITE)) != 0)
return (error);
vm_map_lock_read(map);
if (uvm_map_lookup_entry(map, start, &entry) == FALSE) {
error = ENOMEM;
goto out;
}
for (/* nothing */;
entry != &map->header && entry->start < end;
entry = entry->next) {
KASSERT(!UVM_ET_ISSUBMAP(entry));
KASSERT(start >= entry->start);
/* Make sure there are no holes. */
if (entry->end < end &&
(entry->next == &map->header ||
entry->next->start > entry->end)) {
error = ENOMEM;
goto out;
}
lim = end < entry->end ? end : entry->end;
/*
* Special case for objects with no "real" pages. Those
* are always considered resident (mapped devices).
*/
if (UVM_ET_ISOBJ(entry)) {
KASSERT(!UVM_OBJ_IS_KERN_OBJECT(entry->object.uvm_obj));
if (entry->object.uvm_obj->pgops->pgo_releasepg
== NULL) {
pgi = 1;
for (/* nothing */; start < lim;
start += PAGE_SIZE, vec++)
copyout(&pgi, vec, sizeof(char));
continue;
}
}
amap = entry->aref.ar_amap; /* top layer */
uobj = entry->object.uvm_obj; /* bottom layer */
if (uobj != NULL)
simple_lock(&uobj->vmobjlock);
for (/* nothing */; start < lim; start += PAGE_SIZE, vec++) {
pgi = 0;
if (amap != NULL) {
/* Check the top layer first. */
anon = amap_lookup(&entry->aref,
start - entry->start);
/* Don't need to lock anon here. */
if (anon != NULL && anon->an_page != NULL) {
/*
* Anon has the page for this entry
* offset.
*/
pgi = 1;
}
}
if (uobj != NULL && pgi == 0) {
/* Check the bottom layer. */
m = uvm_pagelookup(uobj,
entry->offset + (start - entry->start));
if (m != NULL) {
/*
* Object has the page for this entry
* offset.
*/
pgi = 1;
}
}
copyout(&pgi, vec, sizeof(char));
}
if (uobj != NULL)
simple_unlock(&uobj->vmobjlock);
}
out:
vm_map_unlock_read(map);
uvm_vsunlock(p, SCARG(uap, vec), npgs);
return (error);
}
