void
post_execve(struct thread *td, int error, struct vmspace *oldvmspace)
{
struct proc *p;
KASSERT(td == curthread, ("non-current thread %p", td));
p = td->td_proc;
if ((p->p_flag & P_HADTHREADS) != 0) {
PROC_LOCK(p);
/*
* If success, we upgrade to SINGLE_EXIT state to
* force other threads to suicide.
*/
if (error == 0)
thread_single(p, SINGLE_EXIT);
else
thread_single_end(p, SINGLE_BOUNDARY);
PROC_UNLOCK(p);
}
if ((td->td_pflags & TDP_EXECVMSPC) != 0) {
KASSERT(p->p_vmspace != oldvmspace,
("oldvmspace still used"));
vmspace_free(oldvmspace);
td->td_pflags &= ~TDP_EXECVMSPC;
}
}
int f () {
vmspace_t vms;
// CHECK: init: 0
kprintf("init: %d\n", vmspace_init(&vms, 0xC1000000, 0x1C000000));
// CHECK: alloc1: dcfea000
kprintf("alloc1: %x\n", vmspace_alloc(&vms, 0x1000, 0));
// CHECK: alloc2: dcfe8000
kprintf("alloc2: %x\n", vmspace_alloc(&vms, 0x1000, 0));
// CHECK: alloc3: dcfe9000
kprintf("alloc3: %x\n", vmspace_alloc(&vms, 0x1000, 0));
// CHECK: alloc4: dcfe0000
kprintf("alloc4: %x\n", vmspace_alloc(&vms, 0x1000, 0));
// CHECK: alloc5: dcfe1000
kprintf("alloc5: %x\n", vmspace_alloc(&vms, 0x1000, 0));
// CHECK: alloc6: dcfe2000
kprintf("alloc6: %x\n", vmspace_alloc(&vms, 0x1000, 0));
// CHECK: alloc7: dcfc0000
kprintf("alloc7: %x\n", vmspace_alloc(&vms, 0x10000, 0));
vmspace_free(&vms, 0x1000, 0xdcfe2000, 0);
// CHECK: alloc8: dcfe2000
kprintf("alloc8: %x\n", vmspace_alloc(&vms, 0x1000, 0));
// If we free everything we just allocated, and then allocate
// them again, we can check buddies were correctly merged
// by observing that the allocations return the same values
// in the same order.
vmspace_free(&vms, 0x1000, 0xdcfea000, 0);
vmspace_free(&vms, 0x1000, 0xdcfe8000, 0);
vmspace_free(&vms, 0x1000, 0xdcfe9000, 0);
vmspace_free(&vms, 0x1000, 0xdcfe0000, 0);
vmspace_free(&vms, 0x1000, 0xdcfe1000, 0);
vmspace_free(&vms, 0x1000, 0xdcfe2000, 0);
vmspace_free(&vms, 0x10000, 0xdcfc0000, 0);
// CHECK: alloc1: dcfea000
kprintf("alloc1: %x\n", vmspace_alloc(&vms, 0x1000, 0));
// CHECK: alloc2: dcfe8000
kprintf("alloc2: %x\n", vmspace_alloc(&vms, 0x1000, 0));
// CHECK: alloc3: dcfe9000
kprintf("alloc3: %x\n", vmspace_alloc(&vms, 0x1000, 0));
// CHECK: alloc4: dcfe0000
kprintf("alloc4: %x\n", vmspace_alloc(&vms, 0x1000, 0));
// CHECK: alloc5: dcfe1000
kprintf("alloc5: %x\n", vmspace_alloc(&vms, 0x1000, 0));
// CHECK: alloc6: dcfe2000
kprintf("alloc6: %x\n", vmspace_alloc(&vms, 0x1000, 0));
// CHECK: alloc7: dcfc0000
kprintf("alloc7: %x\n", vmspace_alloc(&vms, 0x10000, 0));
// CHECK-NOT: Page fault
uintptr_t *addr = (uintptr_t*)vmspace_alloc(&vms, 0x1000, 1);
*addr = 0x42;
return 0;
}
void kfree(void *p) {
p = (void*) ((uintptr_t)p - sizeof(uintptr_t));
uintptr_t *ptr = (uintptr_t*)p;
unsigned l2 = ptr[0] & 0xFF;
unsigned canary = ptr[0] >> 8;
assert(canary == KMALLOC_CANARY && "Heap corruption!");
if (l2 <= MAX_CACHESZ_LOG2)
slab_cache_free(&caches[l2-MIN_CACHESZ_LOG2], p);
else
vmspace_free(&kernel_vmspace, (1U << l2), (uintptr_t)p, 1);
}
void debugger_getprocs_callback(struct allocation_t* ref)
{
if (!ref)
return;
struct message_t* message = __get(ref);
if (!message)
return;
// Only handle requests
if (message->header.request != 1)
goto cleanup;
int(*_sx_slock)(struct sx *sx, int opts, const char *file, int line) = kdlsym(_sx_slock);
void(*_sx_sunlock)(struct sx *sx, const char *file, int line) = kdlsym(_sx_sunlock);
void(*_mtx_lock_flags)(struct mtx *m, int opts, const char *file, int line) = kdlsym(_mtx_lock_flags);
void(*_mtx_unlock_flags)(struct mtx *m, int opts, const char *file, int line) = kdlsym(_mtx_unlock_flags);
struct sx* allproclock = (struct sx*)kdlsym(allproc_lock);
struct proclist* allproc = (struct proclist*)*(uint64_t*)kdlsym(allproc);
void(*vmspace_free)(struct vmspace *) = kdlsym(vmspace_free);
struct vmspace* (*vmspace_acquire_ref)(struct proc *) = kdlsym(vmspace_acquire_ref);
void(*_vm_map_lock_read)(vm_map_t map, const char *file, int line) = kdlsym(_vm_map_lock_read);
void(*_vm_map_unlock_read)(vm_map_t map, const char *file, int line) = kdlsym(_vm_map_unlock_read);
uint64_t procCount = 0;
struct proc* p = NULL;
struct debugger_getprocs_t getproc = { 0 };
sx_slock(allproclock);
FOREACH_PROC_IN_SYSTEM(p)
{
PROC_LOCK(p);
// Zero out our process information
kmemset(&getproc, 0, sizeof(getproc));
// Get the vm map
struct vmspace* vm = vmspace_acquire_ref(p);
vm_map_t map = &p->p_vmspace->vm_map;
vm_map_lock_read(map);
struct vm_map_entry* entry = map->header.next;
// Copy over all of the address information
getproc.process_id = p->p_pid;
getproc.text_address = (uint64_t)entry->start;
getproc.text_size = (uint64_t)entry->end - entry->start;
getproc.data_address = (uint64_t)p->p_vmspace->vm_daddr;
getproc.data_size = p->p_vmspace->vm_dsize;
// Copy over the name and path
kmemcpy(getproc.process_name, p->p_comm, sizeof(getproc.process_name));
kmemcpy(getproc.path, p->p_elfpath, sizeof(getproc.path));
// Write it back to the PC
kwrite(message->socket, &getproc, sizeof(getproc));
procCount++;
// Free the vmmap
vm_map_unlock_read(map);
vmspace_free(vm);
PROC_UNLOCK(p);
}
sx_sunlock(allproclock);
// Send finalizer, because f**k this shit
kmemset(&getproc, 0xDD, sizeof(getproc));
kwrite(message->socket, &getproc, sizeof(getproc));
cleanup:
__dec(ref);
}
int
fork1(struct thread *td, struct fork_req *fr)
{
struct proc *p1, *newproc;
struct thread *td2;
struct vmspace *vm2;
struct file *fp_procdesc;
vm_ooffset_t mem_charged;
int error, nprocs_new, ok;
static int curfail;
static struct timeval lastfail;
int flags, pages;
flags = fr->fr_flags;
pages = fr->fr_pages;
if ((flags & RFSTOPPED) != 0)
MPASS(fr->fr_procp != NULL && fr->fr_pidp == NULL);
else
MPASS(fr->fr_procp == NULL);
/* Check for the undefined or unimplemented flags. */
if ((flags & ~(RFFLAGS | RFTSIGFLAGS(RFTSIGMASK))) != 0)
return (EINVAL);
/* Signal value requires RFTSIGZMB. */
if ((flags & RFTSIGFLAGS(RFTSIGMASK)) != 0 && (flags & RFTSIGZMB) == 0)
return (EINVAL);
/* Can't copy and clear. */
if ((flags & (RFFDG|RFCFDG)) == (RFFDG|RFCFDG))
return (EINVAL);
/* Check the validity of the signal number. */
if ((flags & RFTSIGZMB) != 0 && (u_int)RFTSIGNUM(flags) > _SIG_MAXSIG)
return (EINVAL);
if ((flags & RFPROCDESC) != 0) {
/* Can't not create a process yet get a process descriptor. */
if ((flags & RFPROC) == 0)
return (EINVAL);
/* Must provide a place to put a procdesc if creating one. */
if (fr->fr_pd_fd == NULL)
return (EINVAL);
/* Check if we are using supported flags. */
if ((fr->fr_pd_flags & ~PD_ALLOWED_AT_FORK) != 0)
return (EINVAL);
}
p1 = td->td_proc;
/*
* Here we don't create a new process, but we divorce
* certain parts of a process from itself.
*/
if ((flags & RFPROC) == 0) {
if (fr->fr_procp != NULL)
*fr->fr_procp = NULL;
else if (fr->fr_pidp != NULL)
*fr->fr_pidp = 0;
return (fork_norfproc(td, flags));
}
fp_procdesc = NULL;
newproc = NULL;
vm2 = NULL;
/*
* Increment the nprocs resource before allocations occur.
* Although process entries are dynamically created, we still
* keep a global limit on the maximum number we will
* create. There are hard-limits as to the number of processes
* that can run, established by the KVA and memory usage for
* the process data.
*
* Don't allow a nonprivileged user to use the last ten
* processes; don't let root exceed the limit.
*/
nprocs_new = atomic_fetchadd_int(&nprocs, 1) + 1;
if ((nprocs_new >= maxproc - 10 && priv_check_cred(td->td_ucred,
PRIV_MAXPROC, 0) != 0) || nprocs_new >= maxproc) {
error = EAGAIN;
sx_xlock(&allproc_lock);
if (ppsratecheck(&lastfail, &curfail, 1)) {
printf("maxproc limit exceeded by uid %u (pid %d); "
"see tuning(7) and login.conf(5)\n",
td->td_ucred->cr_ruid, p1->p_pid);
}
sx_xunlock(&allproc_lock);
goto fail2;
}
/*
* If required, create a process descriptor in the parent first; we
* will abandon it if something goes wrong. We don't finit() until
* later.
*/
if (flags & RFPROCDESC) {
error = procdesc_falloc(td, &fp_procdesc, fr->fr_pd_fd,
fr->fr_pd_flags, fr->fr_pd_fcaps);
if (error != 0)
goto fail2;
}
mem_charged = 0;
if (pages == 0)
pages = kstack_pages;
/* Allocate new proc. */
newproc = uma_zalloc(proc_zone, M_WAITOK);
td2 = FIRST_THREAD_IN_PROC(newproc);
if (td2 == NULL) {
td2 = thread_alloc(pages);
if (td2 == NULL) {
error = ENOMEM;
goto fail2;
}
proc_linkup(newproc, td2);
} else {
if (td2->td_kstack == 0 || td2->td_kstack_pages != pages) {
if (td2->td_kstack != 0)
vm_thread_dispose(td2);
if (!thread_alloc_stack(td2, pages)) {
error = ENOMEM;
goto fail2;
}
}
}
if ((flags & RFMEM) == 0) {
vm2 = vmspace_fork(p1->p_vmspace, &mem_charged);
if (vm2 == NULL) {
error = ENOMEM;
goto fail2;
}
if (!swap_reserve(mem_charged)) {
/*
* The swap reservation failed. The accounting
* from the entries of the copied vm2 will be
* subtracted in vmspace_free(), so force the
* reservation there.
*/
swap_reserve_force(mem_charged);
error = ENOMEM;
goto fail2;
}
} else
vm2 = NULL;
/*
* XXX: This is ugly; when we copy resource usage, we need to bump
* per-cred resource counters.
*/
proc_set_cred_init(newproc, crhold(td->td_ucred));
/*
* Initialize resource accounting for the child process.
*/
error = racct_proc_fork(p1, newproc);
if (error != 0) {
error = EAGAIN;
goto fail1;
}
#ifdef MAC
mac_proc_init(newproc);
#endif
newproc->p_klist = knlist_alloc(&newproc->p_mtx);
STAILQ_INIT(&newproc->p_ktr);
/* We have to lock the process tree while we look for a pid. */
sx_slock(&proctree_lock);
sx_xlock(&allproc_lock);
/*
* Increment the count of procs running with this uid. Don't allow
* a nonprivileged user to exceed their current limit.
*
* XXXRW: Can we avoid privilege here if it's not needed?
*/
error = priv_check_cred(td->td_ucred, PRIV_PROC_LIMIT, 0);
if (error == 0)
ok = chgproccnt(td->td_ucred->cr_ruidinfo, 1, 0);
else {
ok = chgproccnt(td->td_ucred->cr_ruidinfo, 1,
lim_cur(td, RLIMIT_NPROC));
}
if (ok) {
do_fork(td, fr, newproc, td2, vm2, fp_procdesc);
return (0);
}
error = EAGAIN;
sx_sunlock(&proctree_lock);
sx_xunlock(&allproc_lock);
#ifdef MAC
mac_proc_destroy(newproc);
#endif
racct_proc_exit(newproc);
fail1:
crfree(newproc->p_ucred);
newproc->p_ucred = NULL;
fail2:
if (vm2 != NULL)
vmspace_free(vm2);
uma_zfree(proc_zone, newproc);
if ((flags & RFPROCDESC) != 0 && fp_procdesc != NULL) {
fdclose(td, fp_procdesc, *fr->fr_pd_fd);
fdrop(fp_procdesc, td);
}
atomic_add_int(&nprocs, -1);
pause("fork", hz / 2);
return (error);
}
/*
* The map entries can *almost* be read with programs like cat. However,
* large maps need special programs to read. It is not easy to implement
* a program that can sense the required size of the buffer, and then
* subsequently do a read with the appropriate size. This operation cannot
* be atomic. The best that we can do is to allow the program to do a read
* with an arbitrarily large buffer, and return as much as we can. We can
* return an error code if the buffer is too small (EFBIG), then the program
* can try a bigger buffer.
*/
int
procfs_doprocmap(PFS_FILL_ARGS)
{
struct vmspace *vm;
vm_map_t map;
vm_map_entry_t entry, tmp_entry;
struct vnode *vp;
char *fullpath, *freepath;
struct uidinfo *uip;
int error, vfslocked;
unsigned int last_timestamp;
#ifdef COMPAT_FREEBSD32
int wrap32 = 0;
#endif
PROC_LOCK(p);
error = p_candebug(td, p);
PROC_UNLOCK(p);
if (error)
return (error);
if (uio->uio_rw != UIO_READ)
return (EOPNOTSUPP);
#ifdef COMPAT_FREEBSD32
if (curproc->p_sysent->sv_flags & SV_ILP32) {
if (!(p->p_sysent->sv_flags & SV_ILP32))
return (EOPNOTSUPP);
wrap32 = 1;
}
#endif
vm = vmspace_acquire_ref(p);
if (vm == NULL)
return (ESRCH);
map = &vm->vm_map;
vm_map_lock_read(map);
for (entry = map->header.next; entry != &map->header;
entry = entry->next) {
vm_object_t obj, tobj, lobj;
int ref_count, shadow_count, flags;
vm_offset_t e_start, e_end, addr;
int resident, privateresident;
char *type;
vm_eflags_t e_eflags;
vm_prot_t e_prot;
if (entry->eflags & MAP_ENTRY_IS_SUB_MAP)
continue;
e_eflags = entry->eflags;
e_prot = entry->protection;
e_start = entry->start;
e_end = entry->end;
privateresident = 0;
obj = entry->object.vm_object;
if (obj != NULL) {
VM_OBJECT_LOCK(obj);
if (obj->shadow_count == 1)
privateresident = obj->resident_page_count;
}
uip = (entry->uip) ? entry->uip : (obj ? obj->uip : NULL);
resident = 0;
addr = entry->start;
while (addr < entry->end) {
if (pmap_extract(map->pmap, addr))
resident++;
addr += PAGE_SIZE;
}
for (lobj = tobj = obj; tobj; tobj = tobj->backing_object) {
if (tobj != obj)
VM_OBJECT_LOCK(tobj);
if (lobj != obj)
VM_OBJECT_UNLOCK(lobj);
lobj = tobj;
}
last_timestamp = map->timestamp;
vm_map_unlock_read(map);
freepath = NULL;
fullpath = "-";
if (lobj) {
switch (lobj->type) {
default:
case OBJT_DEFAULT:
type = "default";
vp = NULL;
break;
case OBJT_VNODE:
type = "vnode";
vp = lobj->handle;
vref(vp);
break;
case OBJT_SWAP:
type = "swap";
vp = NULL;
break;
case OBJT_SG:
case OBJT_DEVICE:
type = "device";
vp = NULL;
break;
}
if (lobj != obj)
VM_OBJECT_UNLOCK(lobj);
flags = obj->flags;
ref_count = obj->ref_count;
shadow_count = obj->shadow_count;
VM_OBJECT_UNLOCK(obj);
if (vp != NULL) {
vn_fullpath(td, vp, &fullpath, &freepath);
vfslocked = VFS_LOCK_GIANT(vp->v_mount);
vrele(vp);
VFS_UNLOCK_GIANT(vfslocked);
}
} else {
type = "none";
flags = 0;
ref_count = 0;
shadow_count = 0;
}
/*
* format:
* start, end, resident, private resident, cow, access, type,
* charged, charged uid.
*/
error = sbuf_printf(sb,
"0x%lx 0x%lx %d %d %p %s%s%s %d %d 0x%x %s %s %s %s %s %d\n",
(u_long)e_start, (u_long)e_end,
resident, privateresident,
#ifdef COMPAT_FREEBSD32
wrap32 ? NULL : obj, /* Hide 64 bit value */
#else
obj,
#endif
(e_prot & VM_PROT_READ)?"r":"-",
(e_prot & VM_PROT_WRITE)?"w":"-",
(e_prot & VM_PROT_EXECUTE)?"x":"-",
ref_count, shadow_count, flags,
(e_eflags & MAP_ENTRY_COW)?"COW":"NCOW",
(e_eflags & MAP_ENTRY_NEEDS_COPY)?"NC":"NNC",
type, fullpath,
uip ? "CH":"NCH", uip ? uip->ui_uid : -1);
if (freepath != NULL)
free(freepath, M_TEMP);
vm_map_lock_read(map);
if (error == -1) {
error = 0;
break;
}
if (last_timestamp != map->timestamp) {
/*
* Look again for the entry because the map was
* modified while it was unlocked. Specifically,
* the entry may have been clipped, merged, or deleted.
*/
vm_map_lookup_entry(map, e_end - 1, &tmp_entry);
entry = tmp_entry;
}
}
vm_map_unlock_read(map);
vmspace_free(vm);
return (error);
}
static void destroy(slab_cache_t *c, slab_footer_t *f) {
vmspace_free(c->vms, SLAB_SIZE, START_FOR_FOOTER(f), /*free_phys=*/1);
}
/*
* Original vm_pageout_oom, will be called if LRU pageout_oom will fail
*/
static void
original_vm_pageout_oom(int shortage)
{
struct proc *p, *bigproc;
vm_offset_t size, bigsize;
struct thread *td;
struct vmspace *vm;
/*
* We keep the process bigproc locked once we find it to keep anyone
* from messing with it; however, there is a possibility of
* deadlock if process B is bigproc and one of it's child processes
* attempts to propagate a signal to B while we are waiting for A's
* lock while walking this list. To avoid this, we don't block on
* the process lock but just skip a process if it is already locked.
*/
bigproc = NULL;
bigsize = 0;
sx_slock(&allproc_lock);
FOREACH_PROC_IN_SYSTEM(p) {
int breakout;
if (PROC_TRYLOCK(p) == 0)
continue;
/*
* If this is a system, protected or killed process, skip it.
*/
if (p->p_state != PRS_NORMAL ||
(p->p_flag & (P_INEXEC | P_PROTECTED | P_SYSTEM)) ||
(p->p_pid == 1) || P_KILLED(p) ||
((p->p_pid < 48) && (swap_pager_avail != 0))) {
PROC_UNLOCK(p);
continue;
}
/*
* If the process is in a non-running type state,
* don't touch it. Check all the threads individually.
*/
breakout = 0;
FOREACH_THREAD_IN_PROC(p, td) {
thread_lock(td);
if (!TD_ON_RUNQ(td) &&
!TD_IS_RUNNING(td) &&
!TD_IS_SLEEPING(td)) {
thread_unlock(td);
breakout = 1;
break;
}
thread_unlock(td);
}
if (breakout) {
PROC_UNLOCK(p);
continue;
}
/*
* get the process size
*/
vm = vmspace_acquire_ref(p);
if (vm == NULL) {
PROC_UNLOCK(p);
continue;
}
if (!vm_map_trylock_read(&vm->vm_map)) {
vmspace_free(vm);
PROC_UNLOCK(p);
continue;
}
size = vmspace_swap_count(vm);
vm_map_unlock_read(&vm->vm_map);
if (shortage == VM_OOM_MEM)
size += vmspace_resident_count(vm);
vmspace_free(vm);
/*
* if the this process is bigger than the biggest one
* remember it.
*/
if (size > bigsize) {
if (bigproc != NULL)
PROC_UNLOCK(bigproc);
bigproc = p;
bigsize = size;
} else
PROC_UNLOCK(p);
}
void
ept_vmspace_free(struct vmspace *vmspace)
{
vmspace_free(vmspace);
}
void
svm_npt_free(struct vmspace *vmspace)
{
vmspace_free(vmspace);
}
int
proc_rwmem(struct proc *p, struct uio *uio)
{
struct vmspace *vm;
vm_map_t map;
vm_object_t object = NULL;
vm_offset_t pageno = 0; /* page number */
vm_prot_t reqprot;
vm_offset_t kva;
int error, writing;
GIANT_REQUIRED;
/*
* if the vmspace is in the midst of being deallocated or the
* process is exiting, don't try to grab anything. The page table
* usage in that process can be messed up.
*/
vm = p->p_vmspace;
if ((p->p_flag & P_WEXIT))
return (EFAULT);
if (vm->vm_refcnt < 1)
return (EFAULT);
++vm->vm_refcnt;
/*
* The map we want...
*/
map = &vm->vm_map;
writing = uio->uio_rw == UIO_WRITE;
reqprot = writing ? (VM_PROT_WRITE | VM_PROT_OVERRIDE_WRITE) :
VM_PROT_READ;
kva = kmem_alloc_pageable(kernel_map, PAGE_SIZE);
/*
* Only map in one page at a time. We don't have to, but it
* makes things easier. This way is trivial - right?
*/
do {
vm_map_t tmap;
vm_offset_t uva;
int page_offset; /* offset into page */
vm_map_entry_t out_entry;
vm_prot_t out_prot;
boolean_t wired;
vm_pindex_t pindex;
u_int len;
vm_page_t m;
object = NULL;
uva = (vm_offset_t)uio->uio_offset;
/*
* Get the page number of this segment.
*/
pageno = trunc_page(uva);
page_offset = uva - pageno;
/*
* How many bytes to copy
*/
len = min(PAGE_SIZE - page_offset, uio->uio_resid);
/*
* Fault the page on behalf of the process
*/
error = vm_fault(map, pageno, reqprot, VM_FAULT_NORMAL);
if (error) {
error = EFAULT;
break;
}
/*
* Now we need to get the page. out_entry, out_prot, wired,
* and single_use aren't used. One would think the vm code
* would be a *bit* nicer... We use tmap because
* vm_map_lookup() can change the map argument.
*/
tmap = map;
error = vm_map_lookup(&tmap, pageno, reqprot, &out_entry,
&object, &pindex, &out_prot, &wired);
if (error) {
error = EFAULT;
/*
* Make sure that there is no residue in 'object' from
* an error return on vm_map_lookup.
*/
object = NULL;
break;
}
m = vm_page_lookup(object, pindex);
/* Allow fallback to backing objects if we are reading */
while (m == NULL && !writing && object->backing_object) {
pindex += OFF_TO_IDX(object->backing_object_offset);
object = object->backing_object;
m = vm_page_lookup(object, pindex);
}
if (m == NULL) {
error = EFAULT;
/*
* Make sure that there is no residue in 'object' from
* an error return on vm_map_lookup.
*/
object = NULL;
vm_map_lookup_done(tmap, out_entry);
break;
}
/*
* Wire the page into memory
*/
vm_page_lock_queues();
vm_page_wire(m);
vm_page_unlock_queues();
/*
* We're done with tmap now.
* But reference the object first, so that we won't loose
* it.
*/
vm_object_reference(object);
vm_map_lookup_done(tmap, out_entry);
pmap_qenter(kva, &m, 1);
/*
* Now do the i/o move.
*/
error = uiomove((caddr_t)(kva + page_offset), len, uio);
pmap_qremove(kva, 1);
/*
* release the page and the object
*/
vm_page_lock_queues();
vm_page_unwire(m, 1);
vm_page_unlock_queues();
vm_object_deallocate(object);
object = NULL;
} while (error == 0 && uio->uio_resid > 0);
if (object)
vm_object_deallocate(object);
kmem_free(kernel_map, kva, PAGE_SIZE);
vmspace_free(vm);
return (error);
}
