/*
* Release a page busied for a getpages operation. The page may have become
* wired (typically due to being used by the buffer cache) or otherwise been
* soft-busied and cannot be freed in that case. A held page can still be
* freed.
*/
void
vnode_pager_freepage(vm_page_t m)
{
if (m->busy || m->wire_count || (m->flags & PG_NEED_COMMIT)) {
vm_page_activate(m);
vm_page_wakeup(m);
} else {
vm_page_free(m);
}
}
int
vm_fault_hold(vm_map_t map, vm_offset_t vaddr, vm_prot_t fault_type,
int fault_flags, vm_page_t *m_hold)
{
vm_prot_t prot;
long ahead, behind;
int alloc_req, era, faultcount, nera, reqpage, result;
boolean_t growstack, is_first_object_locked, wired;
int map_generation;
vm_object_t next_object;
vm_page_t marray[VM_FAULT_READ_MAX];
int hardfault;
struct faultstate fs;
struct vnode *vp;
int locked, error;
hardfault = 0;
growstack = TRUE;
PCPU_INC(cnt.v_vm_faults);
fs.vp = NULL;
faultcount = reqpage = 0;
RetryFault:;
/*
* Find the backing store object and offset into it to begin the
* search.
*/
fs.map = map;
result = vm_map_lookup(&fs.map, vaddr, fault_type, &fs.entry,
&fs.first_object, &fs.first_pindex, &prot, &wired);
if (result != KERN_SUCCESS) {
if (growstack && result == KERN_INVALID_ADDRESS &&
map != kernel_map) {
result = vm_map_growstack(curproc, vaddr);
if (result != KERN_SUCCESS)
return (KERN_FAILURE);
growstack = FALSE;
goto RetryFault;
}
return (result);
}
map_generation = fs.map->timestamp;
if (fs.entry->eflags & MAP_ENTRY_NOFAULT) {
panic("vm_fault: fault on nofault entry, addr: %lx",
(u_long)vaddr);
}
/*
* Make a reference to this object to prevent its disposal while we
* are messing with it. Once we have the reference, the map is free
* to be diddled. Since objects reference their shadows (and copies),
* they will stay around as well.
*
* Bump the paging-in-progress count to prevent size changes (e.g.
* truncation operations) during I/O. This must be done after
* obtaining the vnode lock in order to avoid possible deadlocks.
*/
VM_OBJECT_WLOCK(fs.first_object);
vm_object_reference_locked(fs.first_object);
vm_object_pip_add(fs.first_object, 1);
fs.lookup_still_valid = TRUE;
if (wired)
fault_type = prot | (fault_type & VM_PROT_COPY);
fs.first_m = NULL;
/*
* Search for the page at object/offset.
*/
fs.object = fs.first_object;
fs.pindex = fs.first_pindex;
while (TRUE) {
/*
* If the object is dead, we stop here
*/
if (fs.object->flags & OBJ_DEAD) {
unlock_and_deallocate(&fs);
return (KERN_PROTECTION_FAILURE);
}
/*
* See if page is resident
*/
fs.m = vm_page_lookup(fs.object, fs.pindex);
if (fs.m != NULL) {
/*
* check for page-based copy on write.
* We check fs.object == fs.first_object so
* as to ensure the legacy COW mechanism is
* used when the page in question is part of
* a shadow object. Otherwise, vm_page_cowfault()
* removes the page from the backing object,
* which is not what we want.
*/
vm_page_lock(fs.m);
if ((fs.m->cow) &&
(fault_type & VM_PROT_WRITE) &&
(fs.object == fs.first_object)) {
vm_page_cowfault(fs.m);
unlock_and_deallocate(&fs);
goto RetryFault;
}
/*
* Wait/Retry if the page is busy. We have to do this
* if the page is busy via either VPO_BUSY or
* vm_page_t->busy because the vm_pager may be using
* vm_page_t->busy for pageouts ( and even pageins if
* it is the vnode pager ), and we could end up trying
* to pagein and pageout the same page simultaneously.
*
* We can theoretically allow the busy case on a read
* fault if the page is marked valid, but since such
* pages are typically already pmap'd, putting that
* special case in might be more effort then it is
* worth. We cannot under any circumstances mess
* around with a vm_page_t->busy page except, perhaps,
* to pmap it.
*/
if ((fs.m->oflags & VPO_BUSY) || fs.m->busy) {
/*
* Reference the page before unlocking and
* sleeping so that the page daemon is less
* likely to reclaim it.
*/
vm_page_aflag_set(fs.m, PGA_REFERENCED);
vm_page_unlock(fs.m);
if (fs.object != fs.first_object) {
if (!VM_OBJECT_TRYWLOCK(
fs.first_object)) {
VM_OBJECT_WUNLOCK(fs.object);
VM_OBJECT_WLOCK(fs.first_object);
VM_OBJECT_WLOCK(fs.object);
}
vm_page_lock(fs.first_m);
vm_page_free(fs.first_m);
vm_page_unlock(fs.first_m);
vm_object_pip_wakeup(fs.first_object);
VM_OBJECT_WUNLOCK(fs.first_object);
fs.first_m = NULL;
}
unlock_map(&fs);
if (fs.m == vm_page_lookup(fs.object,
fs.pindex)) {
vm_page_sleep_if_busy(fs.m, TRUE,
"vmpfw");
}
vm_object_pip_wakeup(fs.object);
VM_OBJECT_WUNLOCK(fs.object);
PCPU_INC(cnt.v_intrans);
vm_object_deallocate(fs.first_object);
goto RetryFault;
}
vm_page_remque(fs.m);
vm_page_unlock(fs.m);
/*
* Mark page busy for other processes, and the
* pagedaemon. If it still isn't completely valid
* (readable), jump to readrest, else break-out ( we
* found the page ).
*/
vm_page_busy(fs.m);
if (fs.m->valid != VM_PAGE_BITS_ALL)
goto readrest;
break;
}
/*
* Page is not resident, If this is the search termination
* or the pager might contain the page, allocate a new page.
*/
if (TRYPAGER || fs.object == fs.first_object) {
if (fs.pindex >= fs.object->size) {
unlock_and_deallocate(&fs);
return (KERN_PROTECTION_FAILURE);
}
/*
* Allocate a new page for this object/offset pair.
*
* Unlocked read of the p_flag is harmless. At
* worst, the P_KILLED might be not observed
* there, and allocation can fail, causing
* restart and new reading of the p_flag.
*/
fs.m = NULL;
if (!vm_page_count_severe() || P_KILLED(curproc)) {
#if VM_NRESERVLEVEL > 0
if ((fs.object->flags & OBJ_COLORED) == 0) {
fs.object->flags |= OBJ_COLORED;
fs.object->pg_color = atop(vaddr) -
fs.pindex;
}
#endif
alloc_req = P_KILLED(curproc) ?
VM_ALLOC_SYSTEM : VM_ALLOC_NORMAL;
if (fs.object->type != OBJT_VNODE &&
fs.object->backing_object == NULL)
alloc_req |= VM_ALLOC_ZERO;
fs.m = vm_page_alloc(fs.object, fs.pindex,
alloc_req);
}
if (fs.m == NULL) {
unlock_and_deallocate(&fs);
VM_WAITPFAULT;
goto RetryFault;
} else if (fs.m->valid == VM_PAGE_BITS_ALL)
break;
}
readrest:
/*
* We have found a valid page or we have allocated a new page.
* The page thus may not be valid or may not be entirely
* valid.
*
* Attempt to fault-in the page if there is a chance that the
* pager has it, and potentially fault in additional pages
* at the same time.
*/
if (TRYPAGER) {
int rv;
u_char behavior = vm_map_entry_behavior(fs.entry);
if (behavior == MAP_ENTRY_BEHAV_RANDOM ||
P_KILLED(curproc)) {
behind = 0;
ahead = 0;
} else if (behavior == MAP_ENTRY_BEHAV_SEQUENTIAL) {
behind = 0;
ahead = atop(fs.entry->end - vaddr) - 1;
if (ahead > VM_FAULT_READ_AHEAD_MAX)
ahead = VM_FAULT_READ_AHEAD_MAX;
if (fs.pindex == fs.entry->next_read)
vm_fault_cache_behind(&fs,
VM_FAULT_READ_MAX);
} else {
/*
* If this is a sequential page fault, then
* arithmetically increase the number of pages
* in the read-ahead window. Otherwise, reset
* the read-ahead window to its smallest size.
*/
behind = atop(vaddr - fs.entry->start);
if (behind > VM_FAULT_READ_BEHIND)
behind = VM_FAULT_READ_BEHIND;
ahead = atop(fs.entry->end - vaddr) - 1;
era = fs.entry->read_ahead;
if (fs.pindex == fs.entry->next_read) {
nera = era + behind;
if (nera > VM_FAULT_READ_AHEAD_MAX)
nera = VM_FAULT_READ_AHEAD_MAX;
behind = 0;
if (ahead > nera)
ahead = nera;
if (era == VM_FAULT_READ_AHEAD_MAX)
vm_fault_cache_behind(&fs,
VM_FAULT_CACHE_BEHIND);
} else if (ahead > VM_FAULT_READ_AHEAD_MIN)
ahead = VM_FAULT_READ_AHEAD_MIN;
if (era != ahead)
fs.entry->read_ahead = ahead;
}
/*
* Call the pager to retrieve the data, if any, after
* releasing the lock on the map. We hold a ref on
* fs.object and the pages are VPO_BUSY'd.
*/
unlock_map(&fs);
if (fs.object->type == OBJT_VNODE) {
vp = fs.object->handle;
if (vp == fs.vp)
goto vnode_locked;
else if (fs.vp != NULL) {
vput(fs.vp);
fs.vp = NULL;
}
locked = VOP_ISLOCKED(vp);
if (locked != LK_EXCLUSIVE)
locked = LK_SHARED;
/* Do not sleep for vnode lock while fs.m is busy */
error = vget(vp, locked | LK_CANRECURSE |
LK_NOWAIT, curthread);
if (error != 0) {
vhold(vp);
release_page(&fs);
unlock_and_deallocate(&fs);
error = vget(vp, locked | LK_RETRY |
LK_CANRECURSE, curthread);
vdrop(vp);
fs.vp = vp;
KASSERT(error == 0,
("vm_fault: vget failed"));
goto RetryFault;
}
fs.vp = vp;
}
vnode_locked:
KASSERT(fs.vp == NULL || !fs.map->system_map,
("vm_fault: vnode-backed object mapped by system map"));
/*
* now we find out if any other pages should be paged
* in at this time this routine checks to see if the
* pages surrounding this fault reside in the same
* object as the page for this fault. If they do,
* then they are faulted in also into the object. The
* array "marray" returned contains an array of
* vm_page_t structs where one of them is the
* vm_page_t passed to the routine. The reqpage
* return value is the index into the marray for the
* vm_page_t passed to the routine.
*
* fs.m plus the additional pages are VPO_BUSY'd.
*/
faultcount = vm_fault_additional_pages(
fs.m, behind, ahead, marray, &reqpage);
rv = faultcount ?
vm_pager_get_pages(fs.object, marray, faultcount,
reqpage) : VM_PAGER_FAIL;
if (rv == VM_PAGER_OK) {
/*
* Found the page. Leave it busy while we play
* with it.
*/
/*
* Relookup in case pager changed page. Pager
* is responsible for disposition of old page
* if moved.
*/
fs.m = vm_page_lookup(fs.object, fs.pindex);
if (!fs.m) {
unlock_and_deallocate(&fs);
goto RetryFault;
}
hardfault++;
break; /* break to PAGE HAS BEEN FOUND */
}
/*
* Remove the bogus page (which does not exist at this
* object/offset); before doing so, we must get back
* our object lock to preserve our invariant.
*
* Also wake up any other process that may want to bring
* in this page.
*
* If this is the top-level object, we must leave the
* busy page to prevent another process from rushing
* past us, and inserting the page in that object at
* the same time that we are.
*/
if (rv == VM_PAGER_ERROR)
printf("vm_fault: pager read error, pid %d (%s)\n",
curproc->p_pid, curproc->p_comm);
/*
* Data outside the range of the pager or an I/O error
*/
/*
* XXX - the check for kernel_map is a kludge to work
* around having the machine panic on a kernel space
* fault w/ I/O error.
*/
if (((fs.map != kernel_map) && (rv == VM_PAGER_ERROR)) ||
(rv == VM_PAGER_BAD)) {
vm_page_lock(fs.m);
vm_page_free(fs.m);
vm_page_unlock(fs.m);
fs.m = NULL;
unlock_and_deallocate(&fs);
return ((rv == VM_PAGER_ERROR) ? KERN_FAILURE : KERN_PROTECTION_FAILURE);
}
if (fs.object != fs.first_object) {
vm_page_lock(fs.m);
vm_page_free(fs.m);
vm_page_unlock(fs.m);
fs.m = NULL;
/*
* XXX - we cannot just fall out at this
* point, m has been freed and is invalid!
*/
}
}
/*
* We get here if the object has default pager (or unwiring)
* or the pager doesn't have the page.
*/
if (fs.object == fs.first_object)
fs.first_m = fs.m;
/*
* Move on to the next object. Lock the next object before
* unlocking the current one.
*/
fs.pindex += OFF_TO_IDX(fs.object->backing_object_offset);
next_object = fs.object->backing_object;
if (next_object == NULL) {
/*
* If there's no object left, fill the page in the top
* object with zeros.
*/
if (fs.object != fs.first_object) {
vm_object_pip_wakeup(fs.object);
VM_OBJECT_WUNLOCK(fs.object);
fs.object = fs.first_object;
fs.pindex = fs.first_pindex;
fs.m = fs.first_m;
VM_OBJECT_WLOCK(fs.object);
}
fs.first_m = NULL;
/*
* Zero the page if necessary and mark it valid.
*/
if ((fs.m->flags & PG_ZERO) == 0) {
pmap_zero_page(fs.m);
} else {
PCPU_INC(cnt.v_ozfod);
}
PCPU_INC(cnt.v_zfod);
fs.m->valid = VM_PAGE_BITS_ALL;
break; /* break to PAGE HAS BEEN FOUND */
} else {
KASSERT(fs.object != next_object,
("object loop %p", next_object));
VM_OBJECT_WLOCK(next_object);
vm_object_pip_add(next_object, 1);
if (fs.object != fs.first_object)
vm_object_pip_wakeup(fs.object);
VM_OBJECT_WUNLOCK(fs.object);
fs.object = next_object;
}
}
KASSERT((fs.m->oflags & VPO_BUSY) != 0,
("vm_fault: not busy after main loop"));
/*
* PAGE HAS BEEN FOUND. [Loop invariant still holds -- the object lock
* is held.]
*/
/*
* If the page is being written, but isn't already owned by the
* top-level object, we have to copy it into a new page owned by the
* top-level object.
*/
if (fs.object != fs.first_object) {
/*
* We only really need to copy if we want to write it.
*/
if ((fault_type & (VM_PROT_COPY | VM_PROT_WRITE)) != 0) {
/*
* This allows pages to be virtually copied from a
* backing_object into the first_object, where the
* backing object has no other refs to it, and cannot
* gain any more refs. Instead of a bcopy, we just
* move the page from the backing object to the
* first object. Note that we must mark the page
* dirty in the first object so that it will go out
* to swap when needed.
*/
is_first_object_locked = FALSE;
if (
/*
* Only one shadow object
*/
(fs.object->shadow_count == 1) &&
/*
* No COW refs, except us
*/
(fs.object->ref_count == 1) &&
/*
* No one else can look this object up
*/
(fs.object->handle == NULL) &&
/*
* No other ways to look the object up
*/
((fs.object->type == OBJT_DEFAULT) ||
(fs.object->type == OBJT_SWAP)) &&
(is_first_object_locked = VM_OBJECT_TRYWLOCK(fs.first_object)) &&
/*
* We don't chase down the shadow chain
*/
fs.object == fs.first_object->backing_object) {
/*
* get rid of the unnecessary page
*/
vm_page_lock(fs.first_m);
vm_page_free(fs.first_m);
vm_page_unlock(fs.first_m);
/*
* grab the page and put it into the
* process'es object. The page is
* automatically made dirty.
*/
vm_page_lock(fs.m);
vm_page_rename(fs.m, fs.first_object, fs.first_pindex);
vm_page_unlock(fs.m);
vm_page_busy(fs.m);
fs.first_m = fs.m;
fs.m = NULL;
PCPU_INC(cnt.v_cow_optim);
} else {
/*
* Oh, well, lets copy it.
*/
pmap_copy_page(fs.m, fs.first_m);
fs.first_m->valid = VM_PAGE_BITS_ALL;
if (wired && (fault_flags &
VM_FAULT_CHANGE_WIRING) == 0) {
vm_page_lock(fs.first_m);
vm_page_wire(fs.first_m);
vm_page_unlock(fs.first_m);
vm_page_lock(fs.m);
vm_page_unwire(fs.m, FALSE);
vm_page_unlock(fs.m);
}
/*
* We no longer need the old page or object.
*/
release_page(&fs);
}
/*
* fs.object != fs.first_object due to above
* conditional
*/
vm_object_pip_wakeup(fs.object);
VM_OBJECT_WUNLOCK(fs.object);
/*
* Only use the new page below...
*/
fs.object = fs.first_object;
fs.pindex = fs.first_pindex;
fs.m = fs.first_m;
if (!is_first_object_locked)
VM_OBJECT_WLOCK(fs.object);
PCPU_INC(cnt.v_cow_faults);
curthread->td_cow++;
} else {
prot &= ~VM_PROT_WRITE;
}
}
/*
* We must verify that the maps have not changed since our last
* lookup.
*/
if (!fs.lookup_still_valid) {
vm_object_t retry_object;
vm_pindex_t retry_pindex;
vm_prot_t retry_prot;
if (!vm_map_trylock_read(fs.map)) {
release_page(&fs);
unlock_and_deallocate(&fs);
goto RetryFault;
}
fs.lookup_still_valid = TRUE;
if (fs.map->timestamp != map_generation) {
result = vm_map_lookup_locked(&fs.map, vaddr, fault_type,
&fs.entry, &retry_object, &retry_pindex, &retry_prot, &wired);
/*
* If we don't need the page any longer, put it on the inactive
* list (the easiest thing to do here). If no one needs it,
* pageout will grab it eventually.
*/
if (result != KERN_SUCCESS) {
release_page(&fs);
unlock_and_deallocate(&fs);
/*
* If retry of map lookup would have blocked then
* retry fault from start.
*/
if (result == KERN_FAILURE)
goto RetryFault;
return (result);
}
if ((retry_object != fs.first_object) ||
(retry_pindex != fs.first_pindex)) {
release_page(&fs);
unlock_and_deallocate(&fs);
goto RetryFault;
}
/*
* Check whether the protection has changed or the object has
* been copied while we left the map unlocked. Changing from
* read to write permission is OK - we leave the page
* write-protected, and catch the write fault. Changing from
* write to read permission means that we can't mark the page
* write-enabled after all.
*/
prot &= retry_prot;
}
}
/*
* If the page was filled by a pager, update the map entry's
* last read offset. Since the pager does not return the
* actual set of pages that it read, this update is based on
* the requested set. Typically, the requested and actual
* sets are the same.
*
* XXX The following assignment modifies the map
* without holding a write lock on it.
*/
if (hardfault)
fs.entry->next_read = fs.pindex + faultcount - reqpage;
if ((prot & VM_PROT_WRITE) != 0 ||
(fault_flags & VM_FAULT_DIRTY) != 0) {
vm_object_set_writeable_dirty(fs.object);
/*
* If this is a NOSYNC mmap we do not want to set VPO_NOSYNC
* if the page is already dirty to prevent data written with
* the expectation of being synced from not being synced.
* Likewise if this entry does not request NOSYNC then make
* sure the page isn't marked NOSYNC. Applications sharing
* data should use the same flags to avoid ping ponging.
*/
if (fs.entry->eflags & MAP_ENTRY_NOSYNC) {
if (fs.m->dirty == 0)
fs.m->oflags |= VPO_NOSYNC;
} else {
fs.m->oflags &= ~VPO_NOSYNC;
}
/*
* If the fault is a write, we know that this page is being
* written NOW so dirty it explicitly to save on
* pmap_is_modified() calls later.
*
* Also tell the backing pager, if any, that it should remove
* any swap backing since the page is now dirty.
*/
if (((fault_type & VM_PROT_WRITE) != 0 &&
(fault_flags & VM_FAULT_CHANGE_WIRING) == 0) ||
(fault_flags & VM_FAULT_DIRTY) != 0) {
vm_page_dirty(fs.m);
vm_pager_page_unswapped(fs.m);
}
}
/*
* Page had better still be busy
*/
KASSERT(fs.m->oflags & VPO_BUSY,
("vm_fault: page %p not busy!", fs.m));
/*
* Page must be completely valid or it is not fit to
* map into user space. vm_pager_get_pages() ensures this.
*/
KASSERT(fs.m->valid == VM_PAGE_BITS_ALL,
("vm_fault: page %p partially invalid", fs.m));
VM_OBJECT_WUNLOCK(fs.object);
/*
* Put this page into the physical map. We had to do the unlock above
* because pmap_enter() may sleep. We don't put the page
* back on the active queue until later so that the pageout daemon
* won't find it (yet).
*/
pmap_enter(fs.map->pmap, vaddr, fault_type, fs.m, prot, wired);
if ((fault_flags & VM_FAULT_CHANGE_WIRING) == 0 && wired == 0)
vm_fault_prefault(fs.map->pmap, vaddr, fs.entry);
VM_OBJECT_WLOCK(fs.object);
vm_page_lock(fs.m);
/*
* If the page is not wired down, then put it where the pageout daemon
* can find it.
*/
if (fault_flags & VM_FAULT_CHANGE_WIRING) {
if (wired)
vm_page_wire(fs.m);
else
vm_page_unwire(fs.m, 1);
} else
vm_page_activate(fs.m);
if (m_hold != NULL) {
*m_hold = fs.m;
vm_page_hold(fs.m);
}
vm_page_unlock(fs.m);
vm_page_wakeup(fs.m);
/*
* Unlock everything, and return
*/
unlock_and_deallocate(&fs);
if (hardfault) {
PCPU_INC(cnt.v_io_faults);
curthread->td_ru.ru_majflt++;
} else
curthread->td_ru.ru_minflt++;
return (KERN_SUCCESS);
}
/*
* Routine:
* vm_fault_copy_entry
* Function:
* Create new shadow object backing dst_entry with private copy of
* all underlying pages. When src_entry is equal to dst_entry,
* function implements COW for wired-down map entry. Otherwise,
* it forks wired entry into dst_map.
*
* In/out conditions:
* The source and destination maps must be locked for write.
* The source map entry must be wired down (or be a sharing map
* entry corresponding to a main map entry that is wired down).
*/
void
vm_fault_copy_entry(vm_map_t dst_map, vm_map_t src_map,
vm_map_entry_t dst_entry, vm_map_entry_t src_entry,
vm_ooffset_t *fork_charge)
{
vm_object_t backing_object, dst_object, object, src_object;
vm_pindex_t dst_pindex, pindex, src_pindex;
vm_prot_t access, prot;
vm_offset_t vaddr;
vm_page_t dst_m;
vm_page_t src_m;
boolean_t src_readonly, upgrade;
#ifdef lint
src_map++;
#endif /* lint */
upgrade = src_entry == dst_entry;
src_object = src_entry->object.vm_object;
src_pindex = OFF_TO_IDX(src_entry->offset);
src_readonly = (src_entry->protection & VM_PROT_WRITE) == 0;
/*
* Create the top-level object for the destination entry. (Doesn't
* actually shadow anything - we copy the pages directly.)
*/
dst_object = vm_object_allocate(OBJT_DEFAULT,
OFF_TO_IDX(dst_entry->end - dst_entry->start));
#if VM_NRESERVLEVEL > 0
dst_object->flags |= OBJ_COLORED;
dst_object->pg_color = atop(dst_entry->start);
#endif
VM_OBJECT_WLOCK(dst_object);
KASSERT(upgrade || dst_entry->object.vm_object == NULL,
("vm_fault_copy_entry: vm_object not NULL"));
dst_entry->object.vm_object = dst_object;
dst_entry->offset = 0;
dst_object->charge = dst_entry->end - dst_entry->start;
if (fork_charge != NULL) {
KASSERT(dst_entry->cred == NULL,
("vm_fault_copy_entry: leaked swp charge"));
dst_object->cred = curthread->td_ucred;
crhold(dst_object->cred);
*fork_charge += dst_object->charge;
} else {
dst_object->cred = dst_entry->cred;
dst_entry->cred = NULL;
}
access = prot = dst_entry->protection;
/*
* If not an upgrade, then enter the mappings in the pmap as
* read and/or execute accesses. Otherwise, enter them as
* write accesses.
*
* A writeable large page mapping is only created if all of
* the constituent small page mappings are modified. Marking
* PTEs as modified on inception allows promotion to happen
* without taking potentially large number of soft faults.
*/
if (!upgrade)
access &= ~VM_PROT_WRITE;
/*
* Loop through all of the virtual pages within the entry's
* range, copying each page from the source object to the
* destination object. Since the source is wired, those pages
* must exist. In contrast, the destination is pageable.
* Since the destination object does share any backing storage
* with the source object, all of its pages must be dirtied,
* regardless of whether they can be written.
*/
for (vaddr = dst_entry->start, dst_pindex = 0;
vaddr < dst_entry->end;
vaddr += PAGE_SIZE, dst_pindex++) {
/*
* Allocate a page in the destination object.
*/
do {
dst_m = vm_page_alloc(dst_object, dst_pindex,
VM_ALLOC_NORMAL);
if (dst_m == NULL) {
VM_OBJECT_WUNLOCK(dst_object);
VM_WAIT;
VM_OBJECT_WLOCK(dst_object);
}
} while (dst_m == NULL);
/*
* Find the page in the source object, and copy it in.
* (Because the source is wired down, the page will be in
* memory.)
*/
VM_OBJECT_WLOCK(src_object);
object = src_object;
pindex = src_pindex + dst_pindex;
while ((src_m = vm_page_lookup(object, pindex)) == NULL &&
src_readonly &&
(backing_object = object->backing_object) != NULL) {
/*
* Allow fallback to backing objects if we are reading.
*/
VM_OBJECT_WLOCK(backing_object);
pindex += OFF_TO_IDX(object->backing_object_offset);
VM_OBJECT_WUNLOCK(object);
object = backing_object;
}
if (src_m == NULL)
panic("vm_fault_copy_wired: page missing");
pmap_copy_page(src_m, dst_m);
VM_OBJECT_WUNLOCK(object);
dst_m->valid = VM_PAGE_BITS_ALL;
dst_m->dirty = VM_PAGE_BITS_ALL;
VM_OBJECT_WUNLOCK(dst_object);
/*
* Enter it in the pmap. If a wired, copy-on-write
* mapping is being replaced by a write-enabled
* mapping, then wire that new mapping.
*/
pmap_enter(dst_map->pmap, vaddr, access, dst_m, prot, upgrade);
/*
* Mark it no longer busy, and put it on the active list.
*/
VM_OBJECT_WLOCK(dst_object);
if (upgrade) {
vm_page_lock(src_m);
vm_page_unwire(src_m, 0);
vm_page_unlock(src_m);
vm_page_lock(dst_m);
vm_page_wire(dst_m);
vm_page_unlock(dst_m);
} else {
vm_page_lock(dst_m);
vm_page_activate(dst_m);
vm_page_unlock(dst_m);
}
vm_page_wakeup(dst_m);
}
VM_OBJECT_WUNLOCK(dst_object);
if (upgrade) {
dst_entry->eflags &= ~(MAP_ENTRY_COW | MAP_ENTRY_NEEDS_COPY);
vm_object_deallocate(src_object);
}
}
int
exec_map_first_page(struct image_params *imgp)
{
int rv, i, after, initial_pagein;
vm_page_t ma[VM_INITIAL_PAGEIN];
vm_object_t object;
if (imgp->firstpage != NULL)
exec_unmap_first_page(imgp);
object = imgp->vp->v_object;
if (object == NULL)
return (EACCES);
VM_OBJECT_WLOCK(object);
#if VM_NRESERVLEVEL > 0
vm_object_color(object, 0);
#endif
ma[0] = vm_page_grab(object, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOBUSY);
if (ma[0]->valid != VM_PAGE_BITS_ALL) {
vm_page_xbusy(ma[0]);
if (!vm_pager_has_page(object, 0, NULL, &after)) {
vm_page_lock(ma[0]);
vm_page_free(ma[0]);
vm_page_unlock(ma[0]);
VM_OBJECT_WUNLOCK(object);
return (EIO);
}
initial_pagein = min(after, VM_INITIAL_PAGEIN);
KASSERT(initial_pagein <= object->size,
("%s: initial_pagein %d object->size %ju",
__func__, initial_pagein, (uintmax_t )object->size));
for (i = 1; i < initial_pagein; i++) {
if ((ma[i] = vm_page_next(ma[i - 1])) != NULL) {
if (ma[i]->valid)
break;
if (!vm_page_tryxbusy(ma[i]))
break;
} else {
ma[i] = vm_page_alloc(object, i,
VM_ALLOC_NORMAL);
if (ma[i] == NULL)
break;
}
}
initial_pagein = i;
rv = vm_pager_get_pages(object, ma, initial_pagein, NULL, NULL);
if (rv != VM_PAGER_OK) {
for (i = 0; i < initial_pagein; i++) {
vm_page_lock(ma[i]);
vm_page_free(ma[i]);
vm_page_unlock(ma[i]);
}
VM_OBJECT_WUNLOCK(object);
return (EIO);
}
vm_page_xunbusy(ma[0]);
for (i = 1; i < initial_pagein; i++)
vm_page_readahead_finish(ma[i]);
}
vm_page_lock(ma[0]);
vm_page_hold(ma[0]);
vm_page_activate(ma[0]);
vm_page_unlock(ma[0]);
VM_OBJECT_WUNLOCK(object);
imgp->firstpage = sf_buf_alloc(ma[0], 0);
imgp->image_header = (char *)sf_buf_kva(imgp->firstpage);
return (0);
}
/*
* vm_pageout_scan does the dirty work for the pageout daemon.
*/
void
vm_pageout_scan()
{
register vm_page_t m, next;
register int page_shortage;
register int s;
register int pages_freed;
int free;
vm_object_t object;
/*
* Only continue when we want more pages to be "free"
*/
cnt.v_rev++;
s = splimp();
simple_lock(&vm_page_queue_free_lock);
free = cnt.v_free_count;
simple_unlock(&vm_page_queue_free_lock);
splx(s);
if (free < cnt.v_free_target) {
swapout_threads();
/*
* Be sure the pmap system is updated so
* we can scan the inactive queue.
*/
pmap_update();
}
/*
* Acquire the resident page system lock,
* as we may be changing what's resident quite a bit.
*/
vm_page_lock_queues();
/*
* Start scanning the inactive queue for pages we can free.
* We keep scanning until we have enough free pages or
* we have scanned through the entire queue. If we
* encounter dirty pages, we start cleaning them.
*/
pages_freed = 0;
for (m = vm_page_queue_inactive.tqh_first; m != NULL; m = next) {
s = splimp();
simple_lock(&vm_page_queue_free_lock);
free = cnt.v_free_count;
simple_unlock(&vm_page_queue_free_lock);
splx(s);
if (free >= cnt.v_free_target)
break;
cnt.v_scan++;
next = m->pageq.tqe_next;
/*
* If the page has been referenced, move it back to the
* active queue.
*/
if (pmap_is_referenced(VM_PAGE_TO_PHYS(m))) {
vm_page_activate(m);
cnt.v_reactivated++;
continue;
}
/*
* If the page is clean, free it up.
*/
if (m->flags & PG_CLEAN) {
object = m->object;
if (vm_object_lock_try(object)) {
pmap_page_protect(VM_PAGE_TO_PHYS(m),
VM_PROT_NONE);
vm_page_free(m);
pages_freed++;
cnt.v_dfree++;
vm_object_unlock(object);
}
continue;
}
/*
* If the page is dirty but already being washed, skip it.
*/
if ((m->flags & PG_LAUNDRY) == 0)
continue;
/*
* Otherwise the page is dirty and still in the laundry,
* so we start the cleaning operation and remove it from
* the laundry.
*/
object = m->object;
if (!vm_object_lock_try(object))
continue;
cnt.v_pageouts++;
#ifdef CLUSTERED_PAGEOUT
if (object->pager &&
vm_pager_cancluster(object->pager, PG_CLUSTERPUT))
vm_pageout_cluster(m, object);
else
#endif
vm_pageout_page(m, object);
thread_wakeup((int) object);
vm_object_unlock(object);
/*
* Former next page may no longer even be on the inactive
* queue (due to potential blocking in the pager with the
* queues unlocked). If it isn't, we just start over.
*/
if (next && (next->flags & PG_INACTIVE) == 0)
next = vm_page_queue_inactive.tqh_first;
}
/*
* Compute the page shortage. If we are still very low on memory
* be sure that we will move a minimal amount of pages from active
* to inactive.
*/
page_shortage = cnt.v_inactive_target - cnt.v_inactive_count;
if (page_shortage <= 0 && pages_freed == 0)
page_shortage = 1;
while (page_shortage > 0) {
/*
* Move some more pages from active to inactive.
*/
if ((m = vm_page_queue_active.tqh_first) == NULL)
break;
vm_page_deactivate(m);
page_shortage--;
}
vm_page_unlock_queues();
}
/*
* Vnode op for VM getpages.
* Wish wish .... get rid from multiple IO routines
*
* nwfs_getpages(struct vnode *a_vp, vm_page_t *a_m, int a_count,
* int a_reqpage, vm_ooffset_t a_offset)
*/
int
nwfs_getpages(struct vop_getpages_args *ap)
{
#ifndef NWFS_RWCACHE
return vnode_pager_generic_getpages(ap->a_vp, ap->a_m, ap->a_count,
ap->a_reqpage, ap->a_seqaccess);
#else
int i, error, npages;
size_t nextoff, toff;
size_t count;
size_t size;
struct uio uio;
struct iovec iov;
vm_offset_t kva;
struct buf *bp;
struct vnode *vp;
struct thread *td = curthread; /* XXX */
struct ucred *cred;
struct nwmount *nmp;
struct nwnode *np;
vm_page_t *pages;
KKASSERT(td->td_proc);
cred = td->td_proc->p_ucred;
vp = ap->a_vp;
np = VTONW(vp);
nmp = VFSTONWFS(vp->v_mount);
pages = ap->a_m;
count = (size_t)ap->a_count;
if (vp->v_object == NULL) {
kprintf("nwfs_getpages: called with non-merged cache vnode??\n");
return VM_PAGER_ERROR;
}
bp = getpbuf_kva(&nwfs_pbuf_freecnt);
npages = btoc(count);
kva = (vm_offset_t) bp->b_data;
pmap_qenter(kva, pages, npages);
iov.iov_base = (caddr_t) kva;
iov.iov_len = count;
uio.uio_iov = &iov;
uio.uio_iovcnt = 1;
uio.uio_offset = IDX_TO_OFF(pages[0]->pindex);
uio.uio_resid = count;
uio.uio_segflg = UIO_SYSSPACE;
uio.uio_rw = UIO_READ;
uio.uio_td = td;
error = ncp_read(NWFSTOCONN(nmp), &np->n_fh, &uio,cred);
pmap_qremove(kva, npages);
relpbuf(bp, &nwfs_pbuf_freecnt);
if (error && (uio.uio_resid == count)) {
kprintf("nwfs_getpages: error %d\n",error);
for (i = 0; i < npages; i++) {
if (ap->a_reqpage != i)
vnode_pager_freepage(pages[i]);
}
return VM_PAGER_ERROR;
}
size = count - uio.uio_resid;
for (i = 0, toff = 0; i < npages; i++, toff = nextoff) {
vm_page_t m;
nextoff = toff + PAGE_SIZE;
m = pages[i];
m->flags &= ~PG_ZERO;
/*
* NOTE: pmap dirty bit should have already been cleared.
* We do not clear it here.
*/
if (nextoff <= size) {
m->valid = VM_PAGE_BITS_ALL;
m->dirty = 0;
} else {
int nvalid = ((size + DEV_BSIZE - 1) - toff) &
~(DEV_BSIZE - 1);
vm_page_set_validclean(m, 0, nvalid);
}
if (i != ap->a_reqpage) {
/*
* Whether or not to leave the page activated is up in
* the air, but we should put the page on a page queue
* somewhere (it already is in the object). Result:
* It appears that emperical results show that
* deactivating pages is best.
*/
/*
* Just in case someone was asking for this page we
* now tell them that it is ok to use.
*/
if (!error) {
if (m->flags & PG_REFERENCED)
vm_page_activate(m);
else
vm_page_deactivate(m);
vm_page_wakeup(m);
} else {
vnode_pager_freepage(m);
}
}
}
return 0;
#endif /* NWFS_RWCACHE */
}
/*
* Vnode op for VM getpages.
* Wish wish .... get rid from multiple IO routines
*
* smbfs_getpages(struct vnode *a_vp, vm_page_t *a_m, int a_count,
* int a_reqpage, vm_ooffset_t a_offset)
*/
int
smbfs_getpages(struct vop_getpages_args *ap)
{
#ifdef SMBFS_RWGENERIC
return vop_stdgetpages(ap);
#else
int i, error, npages;
int doclose;
size_t size, toff, nextoff, count;
struct uio uio;
struct iovec iov;
vm_offset_t kva;
struct buf *bp;
struct vnode *vp;
struct thread *td = curthread; /* XXX */
struct ucred *cred;
struct smbmount *smp;
struct smbnode *np;
struct smb_cred scred;
vm_page_t *pages;
KKASSERT(td->td_proc);
vp = ap->a_vp;
cred = td->td_proc->p_ucred;
np = VTOSMB(vp);
smp = VFSTOSMBFS(vp->v_mount);
pages = ap->a_m;
count = (size_t)ap->a_count;
if (vp->v_object == NULL) {
kprintf("smbfs_getpages: called with non-merged cache vnode??\n");
return VM_PAGER_ERROR;
}
smb_makescred(&scred, td, cred);
bp = getpbuf_kva(&smbfs_pbuf_freecnt);
npages = btoc(count);
kva = (vm_offset_t) bp->b_data;
pmap_qenter(kva, pages, npages);
iov.iov_base = (caddr_t) kva;
iov.iov_len = count;
uio.uio_iov = &iov;
uio.uio_iovcnt = 1;
uio.uio_offset = IDX_TO_OFF(pages[0]->pindex);
uio.uio_resid = count;
uio.uio_segflg = UIO_SYSSPACE;
uio.uio_rw = UIO_READ;
uio.uio_td = td;
/*
* This is kinda nasty. Since smbfs is physically closing the
* fid on close(), we have to reopen it if necessary. There are
* other races here too, such as if another process opens the same
* file while we are blocked in read. XXX
*/
error = 0;
doclose = 0;
if (np->n_opencount == 0) {
error = smbfs_smb_open(np, SMB_AM_OPENREAD, &scred);
if (error == 0)
doclose = 1;
}
if (error == 0)
error = smb_read(smp->sm_share, np->n_fid, &uio, &scred);
if (doclose)
smbfs_smb_close(smp->sm_share, np->n_fid, NULL, &scred);
pmap_qremove(kva, npages);
relpbuf(bp, &smbfs_pbuf_freecnt);
if (error && (uio.uio_resid == count)) {
kprintf("smbfs_getpages: error %d\n",error);
for (i = 0; i < npages; i++) {
if (ap->a_reqpage != i)
vnode_pager_freepage(pages[i]);
}
return VM_PAGER_ERROR;
}
size = count - uio.uio_resid;
for (i = 0, toff = 0; i < npages; i++, toff = nextoff) {
vm_page_t m;
nextoff = toff + PAGE_SIZE;
m = pages[i];
m->flags &= ~PG_ZERO;
/*
* NOTE: pmap dirty bit should have already been cleared.
* We do not clear it here.
*/
if (nextoff <= size) {
m->valid = VM_PAGE_BITS_ALL;
m->dirty = 0;
} else {
int nvalid = ((size + DEV_BSIZE - 1) - toff) &
~(DEV_BSIZE - 1);
vm_page_set_validclean(m, 0, nvalid);
}
if (i != ap->a_reqpage) {
/*
* Whether or not to leave the page activated is up in
* the air, but we should put the page on a page queue
* somewhere (it already is in the object). Result:
* It appears that emperical results show that
* deactivating pages is best.
*/
/*
* Just in case someone was asking for this page we
* now tell them that it is ok to use.
*/
if (!error) {
if (m->flags & PG_REFERENCED)
vm_page_activate(m);
else
vm_page_deactivate(m);
vm_page_wakeup(m);
} else {
vnode_pager_freepage(m);
}
}
}
return 0;
#endif /* SMBFS_RWGENERIC */
}
/*
struct vnop_getpages_args {
struct vnode *a_vp;
vm_page_t *a_m;
int a_count;
int a_reqpage;
vm_ooffset_t a_offset;
};
*/
static int
fuse_vnop_getpages(struct vop_getpages_args *ap)
{
int i, error, nextoff, size, toff, count, npages;
struct uio uio;
struct iovec iov;
vm_offset_t kva;
struct buf *bp;
struct vnode *vp;
struct thread *td;
struct ucred *cred;
vm_page_t *pages;
FS_DEBUG2G("heh\n");
vp = ap->a_vp;
KASSERT(vp->v_object, ("objectless vp passed to getpages"));
td = curthread; /* XXX */
cred = curthread->td_ucred; /* XXX */
pages = ap->a_m;
count = ap->a_count;
if (!fsess_opt_mmap(vnode_mount(vp))) {
FS_DEBUG("called on non-cacheable vnode??\n");
return (VM_PAGER_ERROR);
}
npages = btoc(count);
/*
* If the requested page is partially valid, just return it and
* allow the pager to zero-out the blanks. Partially valid pages
* can only occur at the file EOF.
*/
VM_OBJECT_LOCK(vp->v_object);
fuse_vm_page_lock_queues();
if (pages[ap->a_reqpage]->valid != 0) {
for (i = 0; i < npages; ++i) {
if (i != ap->a_reqpage) {
fuse_vm_page_lock(pages[i]);
vm_page_free(pages[i]);
fuse_vm_page_unlock(pages[i]);
}
}
fuse_vm_page_unlock_queues();
VM_OBJECT_UNLOCK(vp->v_object);
return 0;
}
fuse_vm_page_unlock_queues();
VM_OBJECT_UNLOCK(vp->v_object);
/*
* We use only the kva address for the buffer, but this is extremely
* convienient and fast.
*/
bp = getpbuf(&fuse_pbuf_freecnt);
kva = (vm_offset_t)bp->b_data;
pmap_qenter(kva, pages, npages);
PCPU_INC(cnt.v_vnodein);
PCPU_ADD(cnt.v_vnodepgsin, npages);
iov.iov_base = (caddr_t)kva;
iov.iov_len = count;
uio.uio_iov = &iov;
uio.uio_iovcnt = 1;
uio.uio_offset = IDX_TO_OFF(pages[0]->pindex);
uio.uio_resid = count;
uio.uio_segflg = UIO_SYSSPACE;
uio.uio_rw = UIO_READ;
uio.uio_td = td;
error = fuse_io_dispatch(vp, &uio, IO_DIRECT, cred);
pmap_qremove(kva, npages);
relpbuf(bp, &fuse_pbuf_freecnt);
if (error && (uio.uio_resid == count)) {
FS_DEBUG("error %d\n", error);
VM_OBJECT_LOCK(vp->v_object);
fuse_vm_page_lock_queues();
for (i = 0; i < npages; ++i) {
if (i != ap->a_reqpage) {
fuse_vm_page_lock(pages[i]);
vm_page_free(pages[i]);
fuse_vm_page_unlock(pages[i]);
}
}
fuse_vm_page_unlock_queues();
VM_OBJECT_UNLOCK(vp->v_object);
return VM_PAGER_ERROR;
}
/*
* Calculate the number of bytes read and validate only that number
* of bytes. Note that due to pending writes, size may be 0. This
* does not mean that the remaining data is invalid!
*/
size = count - uio.uio_resid;
VM_OBJECT_LOCK(vp->v_object);
fuse_vm_page_lock_queues();
for (i = 0, toff = 0; i < npages; i++, toff = nextoff) {
vm_page_t m;
nextoff = toff + PAGE_SIZE;
m = pages[i];
if (nextoff <= size) {
/*
* Read operation filled an entire page
*/
m->valid = VM_PAGE_BITS_ALL;
KASSERT(m->dirty == 0,
("fuse_getpages: page %p is dirty", m));
} else if (size > toff) {
/*
* Read operation filled a partial page.
*/
m->valid = 0;
vm_page_set_valid_range(m, 0, size - toff);
KASSERT(m->dirty == 0,
("fuse_getpages: page %p is dirty", m));
} else {
/*
* Read operation was short. If no error occured
* we may have hit a zero-fill section. We simply
* leave valid set to 0.
*/
;
}
if (i != ap->a_reqpage) {
/*
* Whether or not to leave the page activated is up in
* the air, but we should put the page on a page queue
* somewhere (it already is in the object). Result:
* It appears that emperical results show that
* deactivating pages is best.
*/
/*
* Just in case someone was asking for this page we
* now tell them that it is ok to use.
*/
if (!error) {
if (m->oflags & VPO_WANTED) {
fuse_vm_page_lock(m);
vm_page_activate(m);
fuse_vm_page_unlock(m);
} else {
fuse_vm_page_lock(m);
vm_page_deactivate(m);
fuse_vm_page_unlock(m);
}
vm_page_wakeup(m);
} else {
fuse_vm_page_lock(m);
vm_page_free(m);
fuse_vm_page_unlock(m);
}
}
}
fuse_vm_page_unlock_queues();
VM_OBJECT_UNLOCK(vp->v_object);
return 0;
}
/*
* This is now called from local media FS's to operate against their
* own vnodes if they fail to implement VOP_GETPAGES.
*
* With all the caching local media devices do these days there is really
* very little point to attempting to restrict the I/O size to contiguous
* blocks on-disk, especially if our caller thinks we need all the specified
* pages. Just construct and issue a READ.
*/
int
vnode_pager_generic_getpages(struct vnode *vp, vm_page_t *mpp, int bytecount,
int reqpage, int seqaccess)
{
struct iovec aiov;
struct uio auio;
off_t foff;
int error;
int count;
int i;
int ioflags;
/*
* Do not do anything if the vnode is bad.
*/
if (vp->v_mount == NULL)
return VM_PAGER_BAD;
/*
* Calculate the number of pages. Since we are paging in whole
* pages, adjust bytecount to be an integral multiple of the page
* size. It will be clipped to the file EOF later on.
*/
bytecount = round_page(bytecount);
count = bytecount / PAGE_SIZE;
/*
* We could check m[reqpage]->valid here and shortcut the operation,
* but doing so breaks read-ahead. Instead assume that the VM
* system has already done at least the check, don't worry about
* any races, and issue the VOP_READ to allow read-ahead to function.
*
* This keeps the pipeline full for I/O bound sequentially scanned
* mmap()'s
*/
/* don't shortcut */
/*
* Discard pages past the file EOF. If the requested page is past
* the file EOF we just leave its valid bits set to 0, the caller
* expects to maintain ownership of the requested page. If the
* entire range is past file EOF discard everything and generate
* a pagein error.
*/
foff = IDX_TO_OFF(mpp[0]->pindex);
if (foff >= vp->v_filesize) {
for (i = 0; i < count; i++) {
if (i != reqpage)
vnode_pager_freepage(mpp[i]);
}
return VM_PAGER_ERROR;
}
if (foff + bytecount > vp->v_filesize) {
bytecount = vp->v_filesize - foff;
i = round_page(bytecount) / PAGE_SIZE;
while (count > i) {
--count;
if (count != reqpage)
vnode_pager_freepage(mpp[count]);
}
}
/*
* The size of the transfer is bytecount. bytecount will be an
* integral multiple of the page size unless it has been clipped
* to the file EOF. The transfer cannot exceed the file EOF.
*
* When dealing with real devices we must round-up to the device
* sector size.
*/
if (vp->v_type == VBLK || vp->v_type == VCHR) {
int secmask = vp->v_rdev->si_bsize_phys - 1;
KASSERT(secmask < PAGE_SIZE, ("vnode_pager_generic_getpages: sector size %d too large", secmask + 1));
bytecount = (bytecount + secmask) & ~secmask;
}
/*
* Severe hack to avoid deadlocks with the buffer cache
*/
for (i = 0; i < count; ++i) {
vm_page_t mt = mpp[i];
vm_page_io_start(mt);
vm_page_wakeup(mt);
}
/*
* Issue the I/O with some read-ahead if bytecount > PAGE_SIZE
*/
ioflags = IO_VMIO;
if (seqaccess)
ioflags |= IO_SEQMAX << IO_SEQSHIFT;
aiov.iov_base = NULL;
aiov.iov_len = bytecount;
auio.uio_iov = &aiov;
auio.uio_iovcnt = 1;
auio.uio_offset = foff;
auio.uio_segflg = UIO_NOCOPY;
auio.uio_rw = UIO_READ;
auio.uio_resid = bytecount;
auio.uio_td = NULL;
mycpu->gd_cnt.v_vnodein++;
mycpu->gd_cnt.v_vnodepgsin += count;
error = VOP_READ(vp, &auio, ioflags, proc0.p_ucred);
/*
* Severe hack to avoid deadlocks with the buffer cache
*/
for (i = 0; i < count; ++i) {
vm_page_busy_wait(mpp[i], FALSE, "getpgs");
vm_page_io_finish(mpp[i]);
}
/*
* Calculate the actual number of bytes read and clean up the
* page list.
*/
bytecount -= auio.uio_resid;
for (i = 0; i < count; ++i) {
vm_page_t mt = mpp[i];
if (i != reqpage) {
if (error == 0 && mt->valid) {
if (mt->flags & PG_REFERENCED)
vm_page_activate(mt);
else
vm_page_deactivate(mt);
vm_page_wakeup(mt);
} else {
vnode_pager_freepage(mt);
}
} else if (mt->valid == 0) {
if (error == 0) {
kprintf("page failed but no I/O error page "
"%p object %p pindex %d\n",
mt, mt->object, (int) mt->pindex);
/* whoops, something happened */
error = EINVAL;
}
} else if (mt->valid != VM_PAGE_BITS_ALL) {
/*
* Zero-extend the requested page if necessary (if
* the filesystem is using a small block size).
*/
vm_page_zero_invalid(mt, TRUE);
}
}
if (error) {
kprintf("vnode_pager_getpage: I/O read error\n");
}
return (error ? VM_PAGER_ERROR : VM_PAGER_OK);
}
/*
* spec_getpages() - get pages associated with device vnode.
*
* Note that spec_read and spec_write do not use the buffer cache, so we
* must fully implement getpages here.
*/
static int
devfs_spec_getpages(struct vop_getpages_args *ap)
{
vm_offset_t kva;
int error;
int i, pcount, size;
struct buf *bp;
vm_page_t m;
vm_ooffset_t offset;
int toff, nextoff, nread;
struct vnode *vp = ap->a_vp;
int blksiz;
int gotreqpage;
error = 0;
pcount = round_page(ap->a_count) / PAGE_SIZE;
/*
* Calculate the offset of the transfer and do sanity check.
*/
offset = IDX_TO_OFF(ap->a_m[0]->pindex) + ap->a_offset;
/*
* Round up physical size for real devices. We cannot round using
* v_mount's block size data because v_mount has nothing to do with
* the device. i.e. it's usually '/dev'. We need the physical block
* size for the device itself.
*
* We can't use v_rdev->si_mountpoint because it only exists when the
* block device is mounted. However, we can use v_rdev.
*/
if (vn_isdisk(vp, NULL))
blksiz = vp->v_rdev->si_bsize_phys;
else
blksiz = DEV_BSIZE;
size = (ap->a_count + blksiz - 1) & ~(blksiz - 1);
bp = getpbuf_kva(NULL);
kva = (vm_offset_t)bp->b_data;
/*
* Map the pages to be read into the kva.
*/
pmap_qenter(kva, ap->a_m, pcount);
/* Build a minimal buffer header. */
bp->b_cmd = BUF_CMD_READ;
bp->b_bcount = size;
bp->b_resid = 0;
bsetrunningbufspace(bp, size);
bp->b_bio1.bio_offset = offset;
bp->b_bio1.bio_done = devfs_spec_getpages_iodone;
mycpu->gd_cnt.v_vnodein++;
mycpu->gd_cnt.v_vnodepgsin += pcount;
/* Do the input. */
vn_strategy(ap->a_vp, &bp->b_bio1);
crit_enter();
/* We definitely need to be at splbio here. */
while (bp->b_cmd != BUF_CMD_DONE)
tsleep(bp, 0, "spread", 0);
crit_exit();
if (bp->b_flags & B_ERROR) {
if (bp->b_error)
error = bp->b_error;
else
error = EIO;
}
/*
* If EOF is encountered we must zero-extend the result in order
* to ensure that the page does not contain garabge. When no
* error occurs, an early EOF is indicated if b_bcount got truncated.
* b_resid is relative to b_bcount and should be 0, but some devices
* might indicate an EOF with b_resid instead of truncating b_bcount.
*/
nread = bp->b_bcount - bp->b_resid;
if (nread < ap->a_count)
bzero((caddr_t)kva + nread, ap->a_count - nread);
pmap_qremove(kva, pcount);
gotreqpage = 0;
for (i = 0, toff = 0; i < pcount; i++, toff = nextoff) {
nextoff = toff + PAGE_SIZE;
m = ap->a_m[i];
m->flags &= ~PG_ZERO;
/*
* NOTE: vm_page_undirty/clear_dirty etc do not clear the
* pmap modified bit. pmap modified bit should have
* already been cleared.
*/
if (nextoff <= nread) {
m->valid = VM_PAGE_BITS_ALL;
vm_page_undirty(m);
} else if (toff < nread) {
/*
* Since this is a VM request, we have to supply the
* unaligned offset to allow vm_page_set_valid()
* to zero sub-DEV_BSIZE'd portions of the page.
*/
vm_page_set_valid(m, 0, nread - toff);
vm_page_clear_dirty_end_nonincl(m, 0, nread - toff);
} else {
m->valid = 0;
vm_page_undirty(m);
}
if (i != ap->a_reqpage) {
/*
* Just in case someone was asking for this page we
* now tell them that it is ok to use.
*/
if (!error || (m->valid == VM_PAGE_BITS_ALL)) {
if (m->valid) {
if (m->flags & PG_REFERENCED) {
vm_page_activate(m);
} else {
vm_page_deactivate(m);
}
vm_page_wakeup(m);
} else {
vm_page_free(m);
}
} else {
vm_page_free(m);
}
} else if (m->valid) {
gotreqpage = 1;
/*
* Since this is a VM request, we need to make the
* entire page presentable by zeroing invalid sections.
*/
if (m->valid != VM_PAGE_BITS_ALL)
vm_page_zero_invalid(m, FALSE);
}
}
if (!gotreqpage) {
m = ap->a_m[ap->a_reqpage];
devfs_debug(DEVFS_DEBUG_WARNING,
"spec_getpages:(%s) I/O read failure: (error=%d) bp %p vp %p\n",
devtoname(vp->v_rdev), error, bp, bp->b_vp);
devfs_debug(DEVFS_DEBUG_WARNING,
" size: %d, resid: %d, a_count: %d, valid: 0x%x\n",
size, bp->b_resid, ap->a_count, m->valid);
devfs_debug(DEVFS_DEBUG_WARNING,
" nread: %d, reqpage: %d, pindex: %lu, pcount: %d\n",
nread, ap->a_reqpage, (u_long)m->pindex, pcount);
/*
* Free the buffer header back to the swap buffer pool.
*/
relpbuf(bp, NULL);
return VM_PAGER_ERROR;
}
/*
* Free the buffer header back to the swap buffer pool.
*/
relpbuf(bp, NULL);
if (DEVFS_NODE(ap->a_vp))
nanotime(&DEVFS_NODE(ap->a_vp)->mtime);
return VM_PAGER_OK;
}
