/* * uvmfault_amapcopy: clear "needs_copy" in a map. * * => called with VM data structures unlocked (usually, see below) * => we get a write lock on the maps and clear needs_copy for a VA * => if we are out of RAM we sleep (waiting for more) */ static void uvmfault_amapcopy(struct uvm_faultinfo *ufi) { /* while we haven't done the job */ while (1) { /* no mapping? give up. */ if (uvmfault_lookup(ufi, TRUE) == FALSE) return; /* copy if needed. */ if (UVM_ET_ISNEEDSCOPY(ufi->entry)) amap_copy(ufi->map, ufi->entry, M_NOWAIT, TRUE, ufi->orig_rvaddr, ufi->orig_rvaddr + 1); /* didn't work? must be out of RAM. unlock and sleep. */ if (UVM_ET_ISNEEDSCOPY(ufi->entry)) { uvmfault_unlockmaps(ufi, TRUE); uvm_wait("fltamapcopy"); continue; } /* got it! unlock and return*/ uvmfault_unlockmaps(ufi, TRUE); return; } /*NOTREACHED*/ }
int uvm_loan(struct vm_map *map, vaddr_t start, vsize_t len, void *v, int flags) { struct uvm_faultinfo ufi; void **result, **output; int rv, error; UVMHIST_FUNC(__func__); UVMHIST_CALLED(loanhist); /* * ensure that one and only one of the flags is set */ KASSERT(((flags & UVM_LOAN_TOANON) == 0) ^ ((flags & UVM_LOAN_TOPAGE) == 0)); KASSERT((map->flags & VM_MAP_INTRSAFE) == 0); /* * "output" is a pointer to the current place to put the loaned page. */ result = v; output = &result[0]; /* start at the beginning ... */ /* * while we've got pages to do */ while (len > 0) { /* * fill in params for a call to uvmfault_lookup */ ufi.orig_map = map; ufi.orig_rvaddr = start; ufi.orig_size = len; /* * do the lookup, the only time this will fail is if we hit on * an unmapped region (an error) */ if (!uvmfault_lookup(&ufi, false)) { error = ENOENT; goto fail; } /* * map now locked. now do the loanout... */ rv = uvm_loanentry(&ufi, &output, flags); if (rv < 0) { /* all unlocked due to error */ error = EINVAL; goto fail; } /* * done! the map is unlocked. advance, if possible. * * XXXCDC: could be recoded to hold the map lock with * smarter code (but it only happens on map entry * boundaries, so it isn't that bad). */ if (rv) { rv <<= PAGE_SHIFT; len -= rv; start += rv; } } UVMHIST_LOG(loanhist, "success", 0,0,0,0); return 0; fail: /* * failed to complete loans. drop any loans and return failure code. * map is already unlocked. */ if (output - result) { if (flags & UVM_LOAN_TOANON) { uvm_unloananon((struct vm_anon **)result, output - result); } else { uvm_unloanpage((struct vm_page **)result, output - result); } } UVMHIST_LOG(loanhist, "error %d", error,0,0,0); return (error); }
int uvm_fault(vm_map_t orig_map, vaddr_t vaddr, vm_fault_t fault_type, vm_prot_t access_type) { struct uvm_faultinfo ufi; vm_prot_t enter_prot; boolean_t wired, narrow, promote, locked, shadowed; int npages, nback, nforw, centeridx, result, lcv, gotpages; vaddr_t startva, currva; voff_t uoff; paddr_t pa; struct vm_amap *amap; struct uvm_object *uobj; struct vm_anon *anons_store[UVM_MAXRANGE], **anons, *anon, *oanon; struct vm_page *pages[UVM_MAXRANGE], *pg, *uobjpage; anon = NULL; pg = NULL; uvmexp.faults++; /* XXX: locking? */ /* * init the IN parameters in the ufi */ ufi.orig_map = orig_map; ufi.orig_rvaddr = trunc_page(vaddr); ufi.orig_size = PAGE_SIZE; /* can't get any smaller than this */ if (fault_type == VM_FAULT_WIRE) narrow = TRUE; /* don't look for neighborhood * pages on wire */ else narrow = FALSE; /* normal fault */ /* * "goto ReFault" means restart the page fault from ground zero. */ ReFault: /* * lookup and lock the maps */ if (uvmfault_lookup(&ufi, FALSE) == FALSE) { return (EFAULT); } /* locked: maps(read) */ #ifdef DIAGNOSTIC if ((ufi.map->flags & VM_MAP_PAGEABLE) == 0) panic("uvm_fault: fault on non-pageable map (%p, 0x%lx)", ufi.map, vaddr); #endif /* * check protection */ if ((ufi.entry->protection & access_type) != access_type) { uvmfault_unlockmaps(&ufi, FALSE); return (EACCES); } /* * "enter_prot" is the protection we want to enter the page in at. * for certain pages (e.g. copy-on-write pages) this protection can * be more strict than ufi.entry->protection. "wired" means either * the entry is wired or we are fault-wiring the pg. */ enter_prot = ufi.entry->protection; wired = VM_MAPENT_ISWIRED(ufi.entry) || (fault_type == VM_FAULT_WIRE); if (wired) access_type = enter_prot; /* full access for wired */ /* * handle "needs_copy" case. if we need to copy the amap we will * have to drop our readlock and relock it with a write lock. (we * need a write lock to change anything in a map entry [e.g. * needs_copy]). */ if (UVM_ET_ISNEEDSCOPY(ufi.entry)) { if ((access_type & VM_PROT_WRITE) || (ufi.entry->object.uvm_obj == NULL)) { /* need to clear */ uvmfault_unlockmaps(&ufi, FALSE); uvmfault_amapcopy(&ufi); uvmexp.fltamcopy++; goto ReFault; } else { /* * ensure that we pmap_enter page R/O since * needs_copy is still true */ enter_prot &= ~VM_PROT_WRITE; } } /* * identify the players */ amap = ufi.entry->aref.ar_amap; /* top layer */ uobj = ufi.entry->object.uvm_obj; /* bottom layer */ /* * check for a case 0 fault. if nothing backing the entry then * error now. */ if (amap == NULL && uobj == NULL) { uvmfault_unlockmaps(&ufi, FALSE); return (EFAULT); } /* * establish range of interest based on advice from mapper * and then clip to fit map entry. note that we only want * to do this the first time through the fault. if we * ReFault we will disable this by setting "narrow" to true. */ if (narrow == FALSE) { /* wide fault (!narrow) */ KASSERT(uvmadvice[ufi.entry->advice].advice == ufi.entry->advice); nback = min(uvmadvice[ufi.entry->advice].nback, (ufi.orig_rvaddr - ufi.entry->start) >> PAGE_SHIFT); startva = ufi.orig_rvaddr - (nback << PAGE_SHIFT); nforw = min(uvmadvice[ufi.entry->advice].nforw, ((ufi.entry->end - ufi.orig_rvaddr) >> PAGE_SHIFT) - 1); /* * note: "-1" because we don't want to count the * faulting page as forw */ npages = nback + nforw + 1; centeridx = nback; narrow = TRUE; /* ensure only once per-fault */ } else {