/*
* Identify the physical page mapped at the given kernel virtual
* address. Insert this physical page into the given address space at
* the given virtual address, replacing the physical page, if any,
* that already exists there.
*/
static int
vm_pgmoveco(vm_map_t mapa, vm_offset_t kaddr, vm_offset_t uaddr)
{
vm_map_t map = mapa;
vm_page_t kern_pg, user_pg;
vm_object_t uobject;
vm_map_entry_t entry;
vm_pindex_t upindex;
vm_prot_t prot;
boolean_t wired;
KASSERT((uaddr & PAGE_MASK) == 0,
("vm_pgmoveco: uaddr is not page aligned"));
/*
* Herein the physical page is validated and dirtied. It is
* unwired in sf_buf_mext().
*/
kern_pg = PHYS_TO_VM_PAGE(vtophys(kaddr));
kern_pg->valid = VM_PAGE_BITS_ALL;
KASSERT(kern_pg->queue == PQ_NONE && kern_pg->wire_count == 1,
("vm_pgmoveco: kern_pg is not correctly wired"));
if ((vm_map_lookup(&map, uaddr,
VM_PROT_WRITE, &entry, &uobject,
&upindex, &prot, &wired)) != KERN_SUCCESS) {
return(EFAULT);
}
VM_OBJECT_LOCK(uobject);
retry:
if ((user_pg = vm_page_lookup(uobject, upindex)) != NULL) {
if (vm_page_sleep_if_busy(user_pg, TRUE, "vm_pgmoveco"))
goto retry;
vm_page_lock_queues();
pmap_remove_all(user_pg);
vm_page_free(user_pg);
} else {
/*
* Even if a physical page does not exist in the
* object chain's first object, a physical page from a
* backing object may be mapped read only.
*/
if (uobject->backing_object != NULL)
pmap_remove(map->pmap, uaddr, uaddr + PAGE_SIZE);
vm_page_lock_queues();
}
vm_page_insert(kern_pg, uobject, upindex);
vm_page_dirty(kern_pg);
vm_page_unlock_queues();
VM_OBJECT_UNLOCK(uobject);
vm_map_lookup_done(map, entry);
return(KERN_SUCCESS);
}
void
cdev_pager_free_page(vm_object_t object, vm_page_t m)
{
VM_OBJECT_ASSERT_WLOCKED(object);
if (object->type == OBJT_MGTDEVICE) {
KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("unmanaged %p", m));
pmap_remove_all(m);
vm_page_lock(m);
vm_page_remove(m);
vm_page_unlock(m);
} else if (object->type == OBJT_DEVICE)
dev_pager_free_page(object, m);
}
/*
* Speed up the reclamation of up to "distance" pages that precede the
* faulting pindex within the first object of the shadow chain.
*/
static void
vm_fault_cache_behind(const struct faultstate *fs, int distance)
{
vm_object_t first_object, object;
vm_page_t m, m_prev;
vm_pindex_t pindex;
object = fs->object;
VM_OBJECT_ASSERT_WLOCKED(object);
first_object = fs->first_object;
if (first_object != object) {
if (!VM_OBJECT_TRYWLOCK(first_object)) {
VM_OBJECT_WUNLOCK(object);
VM_OBJECT_WLOCK(first_object);
VM_OBJECT_WLOCK(object);
}
}
/* Neither fictitious nor unmanaged pages can be cached. */
if ((first_object->flags & (OBJ_FICTITIOUS | OBJ_UNMANAGED)) == 0) {
if (fs->first_pindex < distance)
pindex = 0;
else
pindex = fs->first_pindex - distance;
if (pindex < OFF_TO_IDX(fs->entry->offset))
pindex = OFF_TO_IDX(fs->entry->offset);
m = first_object != object ? fs->first_m : fs->m;
KASSERT((m->oflags & VPO_BUSY) != 0,
("vm_fault_cache_behind: page %p is not busy", m));
m_prev = vm_page_prev(m);
while ((m = m_prev) != NULL && m->pindex >= pindex &&
m->valid == VM_PAGE_BITS_ALL) {
m_prev = vm_page_prev(m);
if (m->busy != 0 || (m->oflags & VPO_BUSY) != 0)
continue;
vm_page_lock(m);
if (m->hold_count == 0 && m->wire_count == 0) {
pmap_remove_all(m);
vm_page_aflag_clear(m, PGA_REFERENCED);
if (m->dirty != 0)
vm_page_deactivate(m);
else
vm_page_cache(m);
}
vm_page_unlock(m);
}
}
if (first_object != object)
VM_OBJECT_WUNLOCK(first_object);
}
/*
* Speed up the reclamation of up to "distance" pages that precede the
* faulting pindex within the first object of the shadow chain.
*/
static void
vm_fault_cache_behind(const struct faultstate *fs, int distance)
{
vm_object_t first_object, object;
vm_page_t m, m_prev;
vm_pindex_t pindex;
object = fs->object;
VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
first_object = fs->first_object;
if (first_object != object) {
if (!VM_OBJECT_TRYLOCK(first_object)) {
VM_OBJECT_UNLOCK(object);
VM_OBJECT_LOCK(first_object);
VM_OBJECT_LOCK(object);
}
}
if (first_object->type != OBJT_DEVICE &&
first_object->type != OBJT_PHYS && first_object->type != OBJT_SG) {
if (fs->first_pindex < distance)
pindex = 0;
else
pindex = fs->first_pindex - distance;
if (pindex < OFF_TO_IDX(fs->entry->offset))
pindex = OFF_TO_IDX(fs->entry->offset);
m = first_object != object ? fs->first_m : fs->m;
KASSERT((m->oflags & VPO_BUSY) != 0,
("vm_fault_cache_behind: page %p is not busy", m));
m_prev = vm_page_prev(m);
while ((m = m_prev) != NULL && m->pindex >= pindex &&
m->valid == VM_PAGE_BITS_ALL) {
m_prev = vm_page_prev(m);
if (m->busy != 0 || (m->oflags & VPO_BUSY) != 0)
continue;
vm_page_lock(m);
if (m->hold_count == 0 && m->wire_count == 0) {
pmap_remove_all(m);
vm_page_aflag_clear(m, PGA_REFERENCED);
if (m->dirty != 0)
vm_page_deactivate(m);
else
vm_page_cache(m);
}
vm_page_unlock(m);
}
}
if (first_object != object)
VM_OBJECT_UNLOCK(first_object);
}
