int
acpi_sleep_machdep(struct acpi_softc *sc, int state)
{
ACPI_STATUS status;
vm_offset_t oldphys;
struct pmap *pm;
vm_page_t page;
static vm_page_t opage = NULL;
int ret = 0;
int pteobj_allocated = 0;
u_long ef;
struct proc *p;
if (sc->acpi_wakeaddr == 0) {
return (0);
}
AcpiSetFirmwareWakingVector(sc->acpi_wakephys);
ef = read_eflags();
disable_intr();
/* Create Identity Mapping */
if ((p = curproc) == NULL)
p = &proc0;
pm = vmspace_pmap(p->p_vmspace);
if (pm->pm_pteobj == NULL) {
pm->pm_pteobj = vm_object_allocate(OBJT_DEFAULT, PTDPTDI + 1);
pteobj_allocated = 1;
}
oldphys = pmap_extract(pm, sc->acpi_wakephys);
if (oldphys) {
opage = PHYS_TO_VM_PAGE(oldphys);
}
page = PHYS_TO_VM_PAGE(sc->acpi_wakephys);
pmap_enter(pm, sc->acpi_wakephys, page,
VM_PROT_READ | VM_PROT_WRITE | VM_PROT_EXECUTE, 1);
ret_addr = 0;
if (acpi_savecpu()) {
/* Execute Sleep */
p_gdt = (struct region_descriptor *)(sc->acpi_wakeaddr + physical_gdt);
p_gdt->rd_limit = r_gdt.rd_limit;
p_gdt->rd_base = vtophys(r_gdt.rd_base);
WAKECODE_FIXUP(physical_esp, u_int32_t, vtophys(r_esp));
WAKECODE_FIXUP(previous_cr0, u_int32_t, r_cr0);
WAKECODE_FIXUP(previous_cr2, u_int32_t, r_cr2);
WAKECODE_FIXUP(previous_cr3, u_int32_t, r_cr3);
WAKECODE_FIXUP(previous_cr4, u_int32_t, r_cr4);
WAKECODE_FIXUP(previous_tr, u_int16_t, r_tr);
WAKECODE_BCOPY(previous_gdt, struct region_descriptor, r_gdt);
WAKECODE_FIXUP(previous_ldt, u_int16_t, r_ldt);
WAKECODE_BCOPY(previous_idt, struct region_descriptor, r_idt);
WAKECODE_FIXUP(where_to_recover, void, acpi_restorecpu);
WAKECODE_FIXUP(previous_ds, u_int16_t, r_ds);
WAKECODE_FIXUP(previous_es, u_int16_t, r_es);
WAKECODE_FIXUP(previous_fs, u_int16_t, r_fs);
WAKECODE_FIXUP(previous_gs, u_int16_t, r_gs);
WAKECODE_FIXUP(previous_ss, u_int16_t, r_ss);
if (acpi_get_verbose(sc)) {
acpi_printcpu();
}
wbinvd();
if (state == ACPI_STATE_S4 && sc->acpi_s4bios) {
status = AcpiEnterSleepStateS4Bios();
} else {
status = AcpiEnterSleepState(state);
}
if (status != AE_OK) {
device_printf(sc->acpi_dev,
"AcpiEnterSleepState failed - %s\n",
AcpiFormatException(status));
ret = -1;
goto out;
}
for (;;) ;
} else {
/* Execute Wakeup */
#if 0
initializecpu();
#endif
icu_reinit();
if (acpi_get_verbose(sc)) {
acpi_savecpu();
acpi_printcpu();
}
}
out:
vm_page_lock_queues();
pmap_remove(pm, sc->acpi_wakephys, sc->acpi_wakephys + PAGE_SIZE);
vm_page_unlock_queues();
if (opage) {
pmap_enter(pm, sc->acpi_wakephys, page,
VM_PROT_READ | VM_PROT_WRITE, 0);
}
if (pteobj_allocated) {
vm_object_deallocate(pm->pm_pteobj);
pm->pm_pteobj = NULL;
}
write_eflags(ef);
return (ret);
}
int
memory_object_control_uiomove(
memory_object_control_t control,
memory_object_offset_t offset,
void * uio,
int start_offset,
int io_requested,
int mark_dirty,
int take_reference)
{
vm_object_t object;
vm_page_t dst_page;
int xsize;
int retval = 0;
int cur_run;
int cur_needed;
int i;
int orig_offset;
boolean_t make_lru = FALSE;
vm_page_t page_run[MAX_RUN];
object = memory_object_control_to_vm_object(control);
if (object == VM_OBJECT_NULL) {
return (0);
}
assert(!object->internal);
vm_object_lock(object);
if (mark_dirty && object->copy != VM_OBJECT_NULL) {
/*
* We can't modify the pages without honoring
* copy-on-write obligations first, so fall off
* this optimized path and fall back to the regular
* path.
*/
vm_object_unlock(object);
return 0;
}
orig_offset = start_offset;
while (io_requested && retval == 0) {
cur_needed = (start_offset + io_requested + (PAGE_SIZE - 1)) / PAGE_SIZE;
if (cur_needed > MAX_RUN)
cur_needed = MAX_RUN;
for (cur_run = 0; cur_run < cur_needed; ) {
if ((dst_page = vm_page_lookup(object, offset)) == VM_PAGE_NULL)
break;
/*
* Sync up on getting the busy bit
*/
if ((dst_page->busy || dst_page->cleaning)) {
/*
* someone else is playing with the page... if we've
* already collected pages into this run, go ahead
* and process now, we can't block on this
* page while holding other pages in the BUSY state
* otherwise we will wait
*/
if (cur_run)
break;
PAGE_SLEEP(object, dst_page, THREAD_UNINT);
continue;
}
/*
* this routine is only called when copying
* to/from real files... no need to consider
* encrypted swap pages
*/
assert(!dst_page->encrypted);
if (mark_dirty) {
dst_page->dirty = TRUE;
if (dst_page->cs_validated) {
/*
* CODE SIGNING:
* We're modifying a code-signed
* page: assume that it is now tainted.
*/
dst_page->cs_tainted = TRUE;
vm_cs_tainted_forces++;
}
}
dst_page->busy = TRUE;
page_run[cur_run++] = dst_page;
offset += PAGE_SIZE_64;
}
if (cur_run == 0)
/*
* we hit a 'hole' in the cache
* we bail at this point
* we'll unlock the object below
*/
break;
vm_object_unlock(object);
for (i = 0; i < cur_run; i++) {
dst_page = page_run[i];
if ((xsize = PAGE_SIZE - start_offset) > io_requested)
xsize = io_requested;
if ( (retval = uiomove64((addr64_t)(((addr64_t)(dst_page->phys_page) << 12) + start_offset), xsize, uio)) )
break;
io_requested -= xsize;
start_offset = 0;
}
vm_object_lock(object);
/*
* if we have more than 1 page to work on
* in the current run, or the original request
* started at offset 0 of the page, or we're
* processing multiple batches, we will move
* the pages to the tail of the inactive queue
* to implement an LRU for read/write accesses
*
* the check for orig_offset == 0 is there to
* mitigate the cost of small (< page_size) requests
* to the same page (this way we only move it once)
*/
if (take_reference && (cur_run > 1 || orig_offset == 0)) {
vm_page_lockspin_queues();
make_lru = TRUE;
}
for (i = 0; i < cur_run; i++) {
dst_page = page_run[i];
/*
* someone is explicitly referencing this page...
* update clustered and speculative state
*
*/
VM_PAGE_CONSUME_CLUSTERED(dst_page);
if (make_lru == TRUE)
vm_page_lru(dst_page);
PAGE_WAKEUP_DONE(dst_page);
}
if (make_lru == TRUE) {
vm_page_unlock_queues();
make_lru = FALSE;
}
orig_offset = 0;
}
vm_object_unlock(object);
return (retval);
}
/*
* This routine takes a user's map, array of pages, number of pages, and flags
* and then does the following:
* - validate that the user has access to those pages (flags indicates read
* or write) - if not fail
* - validate that count is enough to hold range number of pages - if not fail
* - fault in any non-resident pages
* - if the user is doing a read force a write fault for any COWed pages
* - if the user is doing a read mark all pages as dirty
* - hold all pages
*/
int
vm_fault_hold_user_pages(vm_map_t map, vm_offset_t addr, vm_page_t *mp,
int count, vm_prot_t prot)
{
vm_offset_t end, va;
int faults, rv;
pmap_t pmap;
vm_page_t m, *pages;
pmap = vm_map_pmap(map);
pages = mp;
addr &= ~PAGE_MASK;
/*
* Check that virtual address range is legal
* This check is somewhat bogus as on some architectures kernel
* and user do not share VA - however, it appears that all FreeBSD
* architectures define it
*/
end = addr + (count * PAGE_SIZE);
if (end > VM_MAXUSER_ADDRESS) {
log(LOG_WARNING, "bad address passed to vm_fault_hold_user_pages");
return (EFAULT);
}
/*
* First optimistically assume that all pages are resident
* (and R/W if for write) if so just mark pages as held (and
* dirty if for write) and return
*/
vm_page_lock_queues();
for (pages = mp, faults = 0, va = addr; va < end;
va += PAGE_SIZE, pages++) {
/*
* page queue mutex is recursable so this is OK
* it would be really nice if we had an unlocked
* version of this so we were only acquiring the
* pmap lock 1 time as opposed to potentially
* many dozens of times
*/
*pages = m = pmap_extract_and_hold(pmap, va, prot);
if (m == NULL) {
faults++;
continue;
}
/*
* Preemptively mark dirty - the pages
* will never have the modified bit set if
* they are only changed via DMA
*/
if (prot & VM_PROT_WRITE)
vm_page_dirty(m);
}
vm_page_unlock_queues();
if (faults == 0)
return (0);
/*
* Pages either have insufficient permissions or are not present
* trigger a fault where neccessary
*
*/
rv = 0;
for (pages = mp, va = addr; va < end; va += PAGE_SIZE, pages++) {
/*
* Account for a very narrow race where the page may be
* taken away from us before it is held
*/
while (*pages == NULL) {
rv = vm_fault(map, va, prot,
(prot & VM_PROT_WRITE) ? VM_FAULT_DIRTY : VM_FAULT_NORMAL);
if (rv)
goto error;
*pages = pmap_extract_and_hold(pmap, va, prot);
}
}
return (0);
error:
log(LOG_WARNING,
"vm_fault bad return rv=%d va=0x%zx\n", rv, va);
vm_page_lock_queues();
for (pages = mp, va = addr; va < end; va += PAGE_SIZE, pages++)
if (*pages) {
vm_page_unhold(*pages);
*pages = NULL;
}
vm_page_unlock_queues();
return (EFAULT);
}
/*
* vm_pageout_scan does the dirty work for the pageout daemon.
*/
void
vm_pageout_scan()
{
register vm_page_t m, next;
register int page_shortage;
register int s;
register int pages_freed;
int free;
vm_object_t object;
/*
* Only continue when we want more pages to be "free"
*/
cnt.v_rev++;
s = splimp();
simple_lock(&vm_page_queue_free_lock);
free = cnt.v_free_count;
simple_unlock(&vm_page_queue_free_lock);
splx(s);
if (free < cnt.v_free_target) {
swapout_threads();
/*
* Be sure the pmap system is updated so
* we can scan the inactive queue.
*/
pmap_update();
}
/*
* Acquire the resident page system lock,
* as we may be changing what's resident quite a bit.
*/
vm_page_lock_queues();
/*
* Start scanning the inactive queue for pages we can free.
* We keep scanning until we have enough free pages or
* we have scanned through the entire queue. If we
* encounter dirty pages, we start cleaning them.
*/
pages_freed = 0;
for (m = vm_page_queue_inactive.tqh_first; m != NULL; m = next) {
s = splimp();
simple_lock(&vm_page_queue_free_lock);
free = cnt.v_free_count;
simple_unlock(&vm_page_queue_free_lock);
splx(s);
if (free >= cnt.v_free_target)
break;
cnt.v_scan++;
next = m->pageq.tqe_next;
/*
* If the page has been referenced, move it back to the
* active queue.
*/
if (pmap_is_referenced(VM_PAGE_TO_PHYS(m))) {
vm_page_activate(m);
cnt.v_reactivated++;
continue;
}
/*
* If the page is clean, free it up.
*/
if (m->flags & PG_CLEAN) {
object = m->object;
if (vm_object_lock_try(object)) {
pmap_page_protect(VM_PAGE_TO_PHYS(m),
VM_PROT_NONE);
vm_page_free(m);
pages_freed++;
cnt.v_dfree++;
vm_object_unlock(object);
}
continue;
}
/*
* If the page is dirty but already being washed, skip it.
*/
if ((m->flags & PG_LAUNDRY) == 0)
continue;
/*
* Otherwise the page is dirty and still in the laundry,
* so we start the cleaning operation and remove it from
* the laundry.
*/
object = m->object;
if (!vm_object_lock_try(object))
continue;
cnt.v_pageouts++;
#ifdef CLUSTERED_PAGEOUT
if (object->pager &&
vm_pager_cancluster(object->pager, PG_CLUSTERPUT))
vm_pageout_cluster(m, object);
else
#endif
vm_pageout_page(m, object);
thread_wakeup((int) object);
vm_object_unlock(object);
/*
* Former next page may no longer even be on the inactive
* queue (due to potential blocking in the pager with the
* queues unlocked). If it isn't, we just start over.
*/
if (next && (next->flags & PG_INACTIVE) == 0)
next = vm_page_queue_inactive.tqh_first;
}
/*
* Compute the page shortage. If we are still very low on memory
* be sure that we will move a minimal amount of pages from active
* to inactive.
*/
page_shortage = cnt.v_inactive_target - cnt.v_inactive_count;
if (page_shortage <= 0 && pages_freed == 0)
page_shortage = 1;
while (page_shortage > 0) {
/*
* Move some more pages from active to inactive.
*/
if ((m = vm_page_queue_active.tqh_first) == NULL)
break;
vm_page_deactivate(m);
page_shortage--;
}
vm_page_unlock_queues();
}
int
memory_object_control_uiomove(
memory_object_control_t control,
memory_object_offset_t offset,
void * uio,
int start_offset,
int io_requested,
int mark_dirty,
int take_reference)
{
vm_object_t object;
vm_page_t dst_page;
int xsize;
int retval = 0;
int cur_run;
int cur_needed;
int i;
int orig_offset;
vm_page_t page_run[MAX_RUN];
object = memory_object_control_to_vm_object(control);
if (object == VM_OBJECT_NULL) {
return (0);
}
assert(!object->internal);
vm_object_lock(object);
if (mark_dirty && object->copy != VM_OBJECT_NULL) {
/*
* We can't modify the pages without honoring
* copy-on-write obligations first, so fall off
* this optimized path and fall back to the regular
* path.
*/
vm_object_unlock(object);
return 0;
}
orig_offset = start_offset;
while (io_requested && retval == 0) {
cur_needed = (start_offset + io_requested + (PAGE_SIZE - 1)) / PAGE_SIZE;
if (cur_needed > MAX_RUN)
cur_needed = MAX_RUN;
for (cur_run = 0; cur_run < cur_needed; ) {
if ((dst_page = vm_page_lookup(object, offset)) == VM_PAGE_NULL)
break;
/*
* if we're in this routine, we are inside a filesystem's
* locking model, so we don't ever want to wait for pages that have
* list_req_pending == TRUE since it means that the
* page is a candidate for some type of I/O operation,
* but that it has not yet been gathered into a UPL...
* this implies that it is still outside the domain
* of the filesystem and that whoever is responsible for
* grabbing it into a UPL may be stuck behind the filesystem
* lock this thread owns, or trying to take a lock exclusively
* and waiting for the readers to drain from a rw lock...
* if we block in those cases, we will deadlock
*/
if (dst_page->list_req_pending) {
if (dst_page->absent) {
/*
* this is the list_req_pending | absent | busy case
* which originates from vm_fault_page... we want
* to fall out of the fast path and go back
* to the caller which will gather this page
* into a UPL and issue the I/O if no one
* else beats us to it
*/
break;
}
if (dst_page->pageout || dst_page->cleaning) {
/*
* this is the list_req_pending | pageout | busy case
* or the list_req_pending | cleaning case...
* which originate from the pageout_scan and
* msync worlds for the pageout case and the hibernate
* pre-cleaning world for the cleaning case...
* we need to reset the state of this page to indicate
* it should stay in the cache marked dirty... nothing else we
* can do at this point... we can't block on it, we can't busy
* it and we can't clean it from this routine.
*/
vm_page_lockspin_queues();
vm_pageout_queue_steal(dst_page, TRUE);
vm_page_deactivate(dst_page);
vm_page_unlock_queues();
}
/*
* this is the list_req_pending | cleaning case...
* we can go ahead and deal with this page since
* its ok for us to mark this page busy... if a UPL
* tries to gather this page, it will block until the
* busy is cleared, thus allowing us safe use of the page
* when we're done with it, we will clear busy and wake
* up anyone waiting on it, thus allowing the UPL creation
* to finish
*/
} else if (dst_page->busy || dst_page->cleaning) {
/*
* someone else is playing with the page... if we've
* already collected pages into this run, go ahead
* and process now, we can't block on this
* page while holding other pages in the BUSY state
* otherwise we will wait
*/
if (cur_run)
break;
PAGE_SLEEP(object, dst_page, THREAD_UNINT);
continue;
}
/*
* this routine is only called when copying
* to/from real files... no need to consider
* encrypted swap pages
*/
assert(!dst_page->encrypted);
if (mark_dirty) {
dst_page->dirty = TRUE;
if (dst_page->cs_validated &&
!dst_page->cs_tainted) {
/*
* CODE SIGNING:
* We're modifying a code-signed
* page: force revalidate
*/
dst_page->cs_validated = FALSE;
#if DEVELOPMENT || DEBUG
vm_cs_validated_resets++;
#endif
pmap_disconnect(dst_page->phys_page);
}
}
dst_page->busy = TRUE;
page_run[cur_run++] = dst_page;
offset += PAGE_SIZE_64;
}
if (cur_run == 0)
/*
* we hit a 'hole' in the cache or
* a page we don't want to try to handle,
* so bail at this point
* we'll unlock the object below
*/
break;
vm_object_unlock(object);
for (i = 0; i < cur_run; i++) {
dst_page = page_run[i];
if ((xsize = PAGE_SIZE - start_offset) > io_requested)
xsize = io_requested;
if ( (retval = uiomove64((addr64_t)(((addr64_t)(dst_page->phys_page) << 12) + start_offset), xsize, uio)) )
break;
io_requested -= xsize;
start_offset = 0;
}
vm_object_lock(object);
/*
* if we have more than 1 page to work on
* in the current run, or the original request
* started at offset 0 of the page, or we're
* processing multiple batches, we will move
* the pages to the tail of the inactive queue
* to implement an LRU for read/write accesses
*
* the check for orig_offset == 0 is there to
* mitigate the cost of small (< page_size) requests
* to the same page (this way we only move it once)
*/
if (take_reference && (cur_run > 1 || orig_offset == 0)) {
vm_page_lockspin_queues();
for (i = 0; i < cur_run; i++)
vm_page_lru(page_run[i]);
vm_page_unlock_queues();
}
for (i = 0; i < cur_run; i++) {
dst_page = page_run[i];
/*
* someone is explicitly referencing this page...
* update clustered and speculative state
*
*/
VM_PAGE_CONSUME_CLUSTERED(dst_page);
PAGE_WAKEUP_DONE(dst_page);
}
orig_offset = 0;
}
vm_object_unlock(object);
return (retval);
}
DECLHIDDEN(int) rtR0MemObjNativeFree(RTR0MEMOBJ pMem)
{
PRTR0MEMOBJFREEBSD pMemFreeBSD = (PRTR0MEMOBJFREEBSD)pMem;
int rc;
switch (pMemFreeBSD->Core.enmType)
{
case RTR0MEMOBJTYPE_PAGE:
case RTR0MEMOBJTYPE_LOW:
case RTR0MEMOBJTYPE_CONT:
rc = vm_map_remove(kernel_map,
(vm_offset_t)pMemFreeBSD->Core.pv,
(vm_offset_t)pMemFreeBSD->Core.pv + pMemFreeBSD->Core.cb);
AssertMsg(rc == KERN_SUCCESS, ("%#x", rc));
break;
case RTR0MEMOBJTYPE_LOCK:
{
vm_map_t pMap = kernel_map;
if (pMemFreeBSD->Core.u.Lock.R0Process != NIL_RTR0PROCESS)
pMap = &((struct proc *)pMemFreeBSD->Core.u.Lock.R0Process)->p_vmspace->vm_map;
rc = vm_map_unwire(pMap,
(vm_offset_t)pMemFreeBSD->Core.pv,
(vm_offset_t)pMemFreeBSD->Core.pv + pMemFreeBSD->Core.cb,
VM_MAP_WIRE_SYSTEM | VM_MAP_WIRE_NOHOLES);
AssertMsg(rc == KERN_SUCCESS, ("%#x", rc));
break;
}
case RTR0MEMOBJTYPE_RES_VIRT:
{
vm_map_t pMap = kernel_map;
if (pMemFreeBSD->Core.u.ResVirt.R0Process != NIL_RTR0PROCESS)
pMap = &((struct proc *)pMemFreeBSD->Core.u.ResVirt.R0Process)->p_vmspace->vm_map;
rc = vm_map_remove(pMap,
(vm_offset_t)pMemFreeBSD->Core.pv,
(vm_offset_t)pMemFreeBSD->Core.pv + pMemFreeBSD->Core.cb);
AssertMsg(rc == KERN_SUCCESS, ("%#x", rc));
break;
}
case RTR0MEMOBJTYPE_MAPPING:
{
vm_map_t pMap = kernel_map;
if (pMemFreeBSD->Core.u.Mapping.R0Process != NIL_RTR0PROCESS)
pMap = &((struct proc *)pMemFreeBSD->Core.u.Mapping.R0Process)->p_vmspace->vm_map;
rc = vm_map_remove(pMap,
(vm_offset_t)pMemFreeBSD->Core.pv,
(vm_offset_t)pMemFreeBSD->Core.pv + pMemFreeBSD->Core.cb);
AssertMsg(rc == KERN_SUCCESS, ("%#x", rc));
break;
}
case RTR0MEMOBJTYPE_PHYS:
case RTR0MEMOBJTYPE_PHYS_NC:
{
#if __FreeBSD_version >= 1000030
VM_OBJECT_WLOCK(pMemFreeBSD->pObject);
#else
VM_OBJECT_LOCK(pMemFreeBSD->pObject);
#endif
vm_page_t pPage = vm_page_find_least(pMemFreeBSD->pObject, 0);
vm_page_lock_queues();
for (vm_page_t pPage = vm_page_find_least(pMemFreeBSD->pObject, 0);
pPage != NULL;
pPage = vm_page_next(pPage))
{
vm_page_unwire(pPage, 0);
}
vm_page_unlock_queues();
#if __FreeBSD_version >= 1000030
VM_OBJECT_WUNLOCK(pMemFreeBSD->pObject);
#else
VM_OBJECT_UNLOCK(pMemFreeBSD->pObject);
#endif
vm_object_deallocate(pMemFreeBSD->pObject);
break;
}
default:
AssertMsgFailed(("enmType=%d\n", pMemFreeBSD->Core.enmType));
return VERR_INTERNAL_ERROR;
}
return VINF_SUCCESS;
}
static int
shm_dotruncate(struct shmfd *shmfd, off_t length)
{
vm_object_t object;
vm_page_t m;
vm_pindex_t nobjsize;
vm_ooffset_t delta;
object = shmfd->shm_object;
VM_OBJECT_LOCK(object);
if (length == shmfd->shm_size) {
VM_OBJECT_UNLOCK(object);
return (0);
}
nobjsize = OFF_TO_IDX(length + PAGE_MASK);
/* Are we shrinking? If so, trim the end. */
if (length < shmfd->shm_size) {
delta = ptoa(object->size - nobjsize);
/* Toss in memory pages. */
if (nobjsize < object->size)
vm_object_page_remove(object, nobjsize, object->size,
FALSE);
/* Toss pages from swap. */
if (object->type == OBJT_SWAP)
swap_pager_freespace(object, nobjsize, delta);
/* Free the swap accounted for shm */
swap_release_by_uid(delta, object->uip);
object->charge -= delta;
/*
* If the last page is partially mapped, then zero out
* the garbage at the end of the page. See comments
* in vnode_pager_setsize() for more details.
*
* XXXJHB: This handles in memory pages, but what about
* a page swapped out to disk?
*/
if ((length & PAGE_MASK) &&
(m = vm_page_lookup(object, OFF_TO_IDX(length))) != NULL &&
m->valid != 0) {
int base = (int)length & PAGE_MASK;
int size = PAGE_SIZE - base;
pmap_zero_page_area(m, base, size);
/*
* Update the valid bits to reflect the blocks that
* have been zeroed. Some of these valid bits may
* have already been set.
*/
vm_page_set_valid(m, base, size);
/*
* Round "base" to the next block boundary so that the
* dirty bit for a partially zeroed block is not
* cleared.
*/
base = roundup2(base, DEV_BSIZE);
vm_page_lock_queues();
vm_page_clear_dirty(m, base, PAGE_SIZE - base);
vm_page_unlock_queues();
} else if ((length & PAGE_MASK) &&
__predict_false(object->cache != NULL)) {
vm_page_cache_free(object, OFF_TO_IDX(length),
nobjsize);
}
} else {
/* Attempt to reserve the swap */
delta = ptoa(nobjsize - object->size);
if (!swap_reserve_by_uid(delta, object->uip)) {
VM_OBJECT_UNLOCK(object);
return (ENOMEM);
}
object->charge += delta;
}
shmfd->shm_size = length;
mtx_lock(&shm_timestamp_lock);
vfs_timestamp(&shmfd->shm_ctime);
shmfd->shm_mtime = shmfd->shm_ctime;
mtx_unlock(&shm_timestamp_lock);
object->size = nobjsize;
VM_OBJECT_UNLOCK(object);
return (0);
}
int
proc_rwmem(struct proc *p, struct uio *uio)
{
struct vmspace *vm;
vm_map_t map;
vm_object_t object = NULL;
vm_offset_t pageno = 0; /* page number */
vm_prot_t reqprot;
vm_offset_t kva;
int error, writing;
GIANT_REQUIRED;
/*
* if the vmspace is in the midst of being deallocated or the
* process is exiting, don't try to grab anything. The page table
* usage in that process can be messed up.
*/
vm = p->p_vmspace;
if ((p->p_flag & P_WEXIT))
return (EFAULT);
if (vm->vm_refcnt < 1)
return (EFAULT);
++vm->vm_refcnt;
/*
* The map we want...
*/
map = &vm->vm_map;
writing = uio->uio_rw == UIO_WRITE;
reqprot = writing ? (VM_PROT_WRITE | VM_PROT_OVERRIDE_WRITE) :
VM_PROT_READ;
kva = kmem_alloc_pageable(kernel_map, PAGE_SIZE);
/*
* Only map in one page at a time. We don't have to, but it
* makes things easier. This way is trivial - right?
*/
do {
vm_map_t tmap;
vm_offset_t uva;
int page_offset; /* offset into page */
vm_map_entry_t out_entry;
vm_prot_t out_prot;
boolean_t wired;
vm_pindex_t pindex;
u_int len;
vm_page_t m;
object = NULL;
uva = (vm_offset_t)uio->uio_offset;
/*
* Get the page number of this segment.
*/
pageno = trunc_page(uva);
page_offset = uva - pageno;
/*
* How many bytes to copy
*/
len = min(PAGE_SIZE - page_offset, uio->uio_resid);
/*
* Fault the page on behalf of the process
*/
error = vm_fault(map, pageno, reqprot, VM_FAULT_NORMAL);
if (error) {
error = EFAULT;
break;
}
/*
* Now we need to get the page. out_entry, out_prot, wired,
* and single_use aren't used. One would think the vm code
* would be a *bit* nicer... We use tmap because
* vm_map_lookup() can change the map argument.
*/
tmap = map;
error = vm_map_lookup(&tmap, pageno, reqprot, &out_entry,
&object, &pindex, &out_prot, &wired);
if (error) {
error = EFAULT;
/*
* Make sure that there is no residue in 'object' from
* an error return on vm_map_lookup.
*/
object = NULL;
break;
}
m = vm_page_lookup(object, pindex);
/* Allow fallback to backing objects if we are reading */
while (m == NULL && !writing && object->backing_object) {
pindex += OFF_TO_IDX(object->backing_object_offset);
object = object->backing_object;
m = vm_page_lookup(object, pindex);
}
if (m == NULL) {
error = EFAULT;
/*
* Make sure that there is no residue in 'object' from
* an error return on vm_map_lookup.
*/
object = NULL;
vm_map_lookup_done(tmap, out_entry);
break;
}
/*
* Wire the page into memory
*/
vm_page_lock_queues();
vm_page_wire(m);
vm_page_unlock_queues();
/*
* We're done with tmap now.
* But reference the object first, so that we won't loose
* it.
*/
vm_object_reference(object);
vm_map_lookup_done(tmap, out_entry);
pmap_qenter(kva, &m, 1);
/*
* Now do the i/o move.
*/
error = uiomove((caddr_t)(kva + page_offset), len, uio);
pmap_qremove(kva, 1);
/*
* release the page and the object
*/
vm_page_lock_queues();
vm_page_unwire(m, 1);
vm_page_unlock_queues();
vm_object_deallocate(object);
object = NULL;
} while (error == 0 && uio->uio_resid > 0);
if (object)
vm_object_deallocate(object);
kmem_free(kernel_map, kva, PAGE_SIZE);
vmspace_free(vm);
return (error);
}
int
socow_setup(struct mbuf *m0, struct uio *uio)
{
struct sf_buf *sf;
vm_page_t pp;
struct iovec *iov;
struct vmspace *vmspace;
struct vm_map *map;
vm_offset_t offset, uva;
socow_stats.attempted++;
vmspace = curproc->p_vmspace;
map = &vmspace->vm_map;
uva = (vm_offset_t) uio->uio_iov->iov_base;
offset = uva & PAGE_MASK;
/*
* Verify that access to the given address is allowed from user-space.
*/
if (vm_fault_quick((caddr_t)uva, VM_PROT_READ) < 0)
return (0);
/*
* verify page is mapped & not already wired for i/o
*/
pp = pmap_extract_and_hold(map->pmap, uva, VM_PROT_READ);
if (pp == NULL) {
socow_stats.fail_not_mapped++;
return(0);
}
/*
* set up COW
*/
vm_page_lock_queues();
if (vm_page_cowsetup(pp) != 0) {
vm_page_unhold(pp);
vm_page_unlock_queues();
return (0);
}
/*
* wire the page for I/O
*/
vm_page_wire(pp);
vm_page_unhold(pp);
vm_page_unlock_queues();
/*
* Allocate an sf buf
*/
sf = sf_buf_alloc(pp, SFB_CATCH);
if (!sf) {
vm_page_lock_queues();
vm_page_cowclear(pp);
vm_page_unwire(pp, 0);
/*
* Check for the object going away on us. This can
* happen since we don't hold a reference to it.
* If so, we're responsible for freeing the page.
*/
if (pp->wire_count == 0 && pp->object == NULL)
vm_page_free(pp);
vm_page_unlock_queues();
socow_stats.fail_sf_buf++;
return(0);
}
/*
* attach to mbuf
*/
MEXTADD(m0, sf_buf_kva(sf), PAGE_SIZE, socow_iodone,
(void*)sf_buf_kva(sf), sf, M_RDONLY, EXT_SFBUF);
m0->m_len = PAGE_SIZE - offset;
m0->m_data = (caddr_t)sf_buf_kva(sf) + offset;
socow_stats.success++;
iov = uio->uio_iov;
iov->iov_base = (char *)iov->iov_base + m0->m_len;
iov->iov_len -= m0->m_len;
uio->uio_resid -= m0->m_len;
uio->uio_offset += m0->m_len;
if (iov->iov_len == 0) {
uio->uio_iov++;
uio->uio_iovcnt--;
}
return(m0->m_len);
}
static __inline void ma_vm_page_unlock_queues(void) { vm_page_unlock_queues(); };