void
uvm_deallocate(struct vm_map *map, vaddr_t start, vsize_t size)
{
if (size == 0)
return;
uvm_unmap(map, trunc_page(start), round_page(start + size));
}
void
uvm_deallocate(struct vm_map *map, vaddr_t start, vsize_t size)
{
if (map == NULL)
panic("uvm_deallocate with null map");
if (size == 0)
return;
uvm_unmap(map, trunc_page(start), round_page(start+size));
}
/*
* Common function for unmapping DMA-safe memory. May be called by
* bus-specific DMA memory unmapping functions.
*/
void
_bus_dmamem_unmap(bus_dma_tag_t t, void *kva, size_t size)
{
#ifdef DIAGNOSTIC
if ((u_long)kva & PAGE_MASK)
panic("_bus_dmamem_unmap");
#endif
size = m68k_round_page(size);
uvm_unmap(kernel_map, (vaddr_t)kva, (vaddr_t)kva + size);
}
int
grfunmap(dev_t dev, struct macfb_softc *sc, void *addr, struct proc *p)
{
vm_size_t size;
addr = (char*)addr - sc->sc_dc->dc_offset;
if (addr <= 0)
return (-1);
size = m68k_round_page(sc->sc_dc->dc_offset + sc->sc_dc->dc_size);
uvm_unmap(&p->p_vmspace->vm_map, (vaddr_t)addr, (vaddr_t)addr + size);
return 0;
}
int
exec_sigcode_map(struct proc *p, struct emul *e)
{
vsize_t sz;
sz = (vaddr_t)e->e_esigcode - (vaddr_t)e->e_sigcode;
/*
* If we don't have a sigobject for this emulation, create one.
*
* sigobject is an anonymous memory object (just like SYSV shared
* memory) that we keep a permanent reference to and that we map
* in all processes that need this sigcode. The creation is simple,
* we create an object, add a permanent reference to it, map it in
* kernel space, copy out the sigcode to it and unmap it.
* Then we map it with PROT_READ|PROT_EXEC into the process just
* the way sys_mmap would map it.
*/
if (e->e_sigobject == NULL) {
vaddr_t va;
int r;
e->e_sigobject = uao_create(sz, 0);
uao_reference(e->e_sigobject); /* permanent reference */
va = vm_map_min(kernel_map); /* hint */
if ((r = uvm_map(kernel_map, &va, round_page(sz), e->e_sigobject,
0, 0, UVM_MAPFLAG(UVM_PROT_RW, UVM_PROT_RW,
UVM_INH_SHARE, UVM_ADV_RANDOM, 0)))) {
uao_detach(e->e_sigobject);
return (ENOMEM);
}
memcpy((void *)va, e->e_sigcode, sz);
uvm_unmap(kernel_map, va, va + round_page(sz));
}
/* Just a hint to uvm_mmap where to put it. */
p->p_sigcode = uvm_map_hint(p, VM_PROT_READ|VM_PROT_EXECUTE);
uao_reference(e->e_sigobject);
if (uvm_map(&p->p_vmspace->vm_map, &p->p_sigcode, round_page(sz),
e->e_sigobject, 0, 0, UVM_MAPFLAG(UVM_PROT_RX, UVM_PROT_RX,
UVM_INH_SHARE, UVM_ADV_RANDOM, 0))) {
uao_detach(e->e_sigobject);
return (ENOMEM);
}
return (0);
}
int
exec_sigcode_map(struct process *pr, struct emul *e)
{
vsize_t sz;
sz = (vaddr_t)e->e_esigcode - (vaddr_t)e->e_sigcode;
/*
* If we don't have a sigobject for this emulation, create one.
*
* sigobject is an anonymous memory object (just like SYSV shared
* memory) that we keep a permanent reference to and that we map
* in all processes that need this sigcode. The creation is simple,
* we create an object, add a permanent reference to it, map it in
* kernel space, copy out the sigcode to it and unmap it.
* Then we map it with PROT_READ|PROT_EXEC into the process just
* the way sys_mmap would map it.
*/
if (e->e_sigobject == NULL) {
vaddr_t va;
int r;
e->e_sigobject = uao_create(sz, 0);
uao_reference(e->e_sigobject); /* permanent reference */
if ((r = uvm_map(kernel_map, &va, round_page(sz), e->e_sigobject,
0, 0, UVM_MAPFLAG(PROT_READ | PROT_WRITE, PROT_READ | PROT_WRITE,
MAP_INHERIT_SHARE, MADV_RANDOM, 0)))) {
uao_detach(e->e_sigobject);
return (ENOMEM);
}
memcpy((void *)va, e->e_sigcode, sz);
uvm_unmap(kernel_map, va, va + round_page(sz));
}
pr->ps_sigcode = 0; /* no hint */
uao_reference(e->e_sigobject);
if (uvm_map(&pr->ps_vmspace->vm_map, &pr->ps_sigcode, round_page(sz),
e->e_sigobject, 0, 0, UVM_MAPFLAG(PROT_READ | PROT_EXEC,
PROT_READ | PROT_WRITE | PROT_EXEC, MAP_INHERIT_COPY,
MADV_RANDOM, UVM_FLAG_COPYONW))) {
uao_detach(e->e_sigobject);
return (ENOMEM);
}
return (0);
}
/*
* Machine-dependent startup code
*/
void
cpu_startup()
{
#ifdef DEBUG
extern int pmapdebug;
int opmapdebug = pmapdebug;
#endif
vaddr_t minaddr, maxaddr;
paddr_t msgbufpa;
extern struct user *proc0paddr;
#ifdef DEBUG
pmapdebug = 0;
#endif
if (CPU_ISSUN4M)
stackgap_random = STACKGAP_RANDOM_SUN4M;
/*
* Re-map the message buffer from its temporary address
* at KERNBASE to MSGBUF_VA.
*/
/* Get physical address of the message buffer */
pmap_extract(pmap_kernel(), (vaddr_t)KERNBASE, &msgbufpa);
/* Invalidate the current mapping at KERNBASE. */
pmap_kremove((vaddr_t)KERNBASE, PAGE_SIZE);
pmap_update(pmap_kernel());
/* Enter the new mapping */
pmap_map(MSGBUF_VA, msgbufpa, msgbufpa + PAGE_SIZE,
PROT_READ | PROT_WRITE);
/* Re-initialize the message buffer. */
initmsgbuf((caddr_t)(MSGBUF_VA + (CPU_ISSUN4 ? 4096 : 0)), MSGBUFSIZE);
proc0.p_addr = proc0paddr;
/*
* Good {morning,afternoon,evening,night}.
*/
printf(version);
/*identifycpu();*/
printf("real mem = %lu (%luMB)\n", ptoa(physmem),
ptoa(physmem)/1024/1024);
/*
* uvm_km_init() has allocated all the virtual memory below the
* end of the kernel image. If VM_MIN_KERNEL_ADDRESS is below
* KERNBASE, we need to reclaim that range.
*/
if (vm_min_kernel_address < (vaddr_t)KERNBASE) {
uvm_unmap(kernel_map, vm_min_kernel_address, (vaddr_t)KERNBASE);
}
/*
* Allocate a submap for exec arguments. This map effectively
* limits the number of processes exec'ing at any time.
*/
minaddr = vm_map_min(kernel_map);
exec_map = uvm_km_suballoc(kernel_map, &minaddr, &maxaddr,
16*NCARGS, VM_MAP_PAGEABLE, FALSE, NULL);
/*
* Set up userland PIE limits. PIE is disabled on sun4/4c/4e due
* to the limited address space.
*/
if (CPU_ISSUN4M) {
vm_pie_max_addr = VM_MAXUSER_ADDRESS / 4;
}
dvma_init();
#ifdef DEBUG
pmapdebug = opmapdebug;
#endif
printf("avail mem = %lu (%luMB)\n", ptoa(uvmexp.free),
ptoa(uvmexp.free)/1024/1024);
/*
* Set up buffers, so they can be used to read disk labels.
*/
bufinit();
}
int
uvm_mmap(struct vm_map *map, vaddr_t *addr, vsize_t size, vm_prot_t prot,
vm_prot_t maxprot, int flags, int advice, struct uvm_object *uobj,
voff_t foff, vsize_t locklimit)
{
vaddr_t align = 0;
int error;
uvm_flag_t uvmflag = 0;
/*
* check params
*/
if (size == 0)
return 0;
if (foff & PAGE_MASK)
return EINVAL;
if ((prot & maxprot) != prot)
return EINVAL;
/*
* for non-fixed mappings, round off the suggested address.
* for fixed mappings, check alignment and zap old mappings.
*/
if ((flags & MAP_FIXED) == 0) {
*addr = round_page(*addr);
} else {
if (*addr & PAGE_MASK)
return EINVAL;
uvmflag |= UVM_FLAG_FIXED;
(void) uvm_unmap(map, *addr, *addr + size);
}
/*
* Try to see if any requested alignment can even be attemped.
* Make sure we can express the alignment (asking for a >= 4GB
* alignment on an ILP32 architecure make no sense) and the
* alignment is at least for a page sized quanitiy. If the
* request was for a fixed mapping, make sure supplied address
* adheres to the request alignment.
*/
align = (flags & MAP_ALIGNMENT_MASK) >> MAP_ALIGNMENT_SHIFT;
if (align) {
if (align >= sizeof(vaddr_t) * NBBY)
return EINVAL;
align = 1L << align;
if (align < PAGE_SIZE)
return EINVAL;
if (align >= vm_map_max(map))
return ENOMEM;
if (flags & MAP_FIXED) {
if ((*addr & (align-1)) != 0)
return EINVAL;
align = 0;
}
}
/*
* check resource limits
*/
if (!VM_MAP_IS_KERNEL(map) &&
(((rlim_t)curproc->p_vmspace->vm_map.size + (rlim_t)size) >
curproc->p_rlimit[RLIMIT_AS].rlim_cur))
return ENOMEM;
/*
* handle anon vs. non-anon mappings. for non-anon mappings attach
* to underlying vm object.
*/
if (flags & MAP_ANON) {
KASSERT(uobj == NULL);
foff = UVM_UNKNOWN_OFFSET;
if ((flags & MAP_SHARED) == 0)
/* XXX: defer amap create */
uvmflag |= UVM_FLAG_COPYONW;
else
/* shared: create amap now */
uvmflag |= UVM_FLAG_OVERLAY;
} else {
KASSERT(uobj != NULL);
if ((flags & MAP_SHARED) == 0) {
uvmflag |= UVM_FLAG_COPYONW;
}
}
uvmflag = UVM_MAPFLAG(prot, maxprot,
(flags & MAP_SHARED) ? UVM_INH_SHARE : UVM_INH_COPY, advice,
uvmflag);
error = uvm_map(map, addr, size, uobj, foff, align, uvmflag);
if (error) {
if (uobj)
uobj->pgops->pgo_detach(uobj);
return error;
}
/*
* POSIX 1003.1b -- if our address space was configured
* to lock all future mappings, wire the one we just made.
*
* Also handle the MAP_WIRED flag here.
*/
if (prot == VM_PROT_NONE) {
/*
* No more work to do in this case.
*/
return 0;
}
if ((flags & MAP_WIRED) != 0 || (map->flags & VM_MAP_WIREFUTURE) != 0) {
vm_map_lock(map);
if (atop(size) + uvmexp.wired > uvmexp.wiredmax ||
(locklimit != 0 &&
size + ptoa(pmap_wired_count(vm_map_pmap(map))) >
locklimit)) {
vm_map_unlock(map);
uvm_unmap(map, *addr, *addr + size);
return ENOMEM;
}
/*
* uvm_map_pageable() always returns the map unlocked.
*/
error = uvm_map_pageable(map, *addr, *addr + size,
false, UVM_LK_ENTER);
if (error) {
uvm_unmap(map, *addr, *addr + size);
return error;
}
return 0;
}
return 0;
}
vaddr_t
uvm_km_alloc1(struct vm_map *map, vsize_t size, vsize_t align, boolean_t zeroit)
{
vaddr_t kva, loopva;
voff_t offset;
struct vm_page *pg;
UVMHIST_FUNC("uvm_km_alloc1"); UVMHIST_CALLED(maphist);
UVMHIST_LOG(maphist,"(map=%p, size=0x%lx)", map, size,0,0);
KASSERT(vm_map_pmap(map) == pmap_kernel());
size = round_page(size);
kva = vm_map_min(map); /* hint */
/*
* allocate some virtual space
*/
if (__predict_false(uvm_map(map, &kva, size, uvm.kernel_object,
UVM_UNKNOWN_OFFSET, align, UVM_MAPFLAG(UVM_PROT_ALL, UVM_PROT_ALL,
UVM_INH_NONE, UVM_ADV_RANDOM, 0)) != 0)) {
UVMHIST_LOG(maphist,"<- done (no VM)",0,0,0,0);
return(0);
}
/*
* recover object offset from virtual address
*/
offset = kva - vm_map_min(kernel_map);
UVMHIST_LOG(maphist," kva=0x%lx, offset=0x%lx", kva, offset,0,0);
/*
* now allocate the memory. we must be careful about released pages.
*/
loopva = kva;
while (size) {
simple_lock(&uvm.kernel_object->vmobjlock);
pg = uvm_pagelookup(uvm.kernel_object, offset);
/*
* if we found a page in an unallocated region, it must be
* released
*/
if (pg) {
if ((pg->pg_flags & PG_RELEASED) == 0)
panic("uvm_km_alloc1: non-released page");
atomic_setbits_int(&pg->pg_flags, PG_WANTED);
UVM_UNLOCK_AND_WAIT(pg, &uvm.kernel_object->vmobjlock,
FALSE, "km_alloc", 0);
continue; /* retry */
}
/* allocate ram */
pg = uvm_pagealloc(uvm.kernel_object, offset, NULL, 0);
if (pg) {
atomic_clearbits_int(&pg->pg_flags, PG_BUSY);
UVM_PAGE_OWN(pg, NULL);
}
simple_unlock(&uvm.kernel_object->vmobjlock);
if (__predict_false(pg == NULL)) {
if (curproc == uvm.pagedaemon_proc) {
/*
* It is unfeasible for the page daemon to
* sleep for memory, so free what we have
* allocated and fail.
*/
uvm_unmap(map, kva, loopva - kva);
return (NULL);
} else {
uvm_wait("km_alloc1w"); /* wait for memory */
continue;
}
}
/*
* map it in; note we're never called with an intrsafe
* object, so we always use regular old pmap_enter().
*/
pmap_enter(map->pmap, loopva, VM_PAGE_TO_PHYS(pg),
UVM_PROT_ALL, PMAP_WIRED | VM_PROT_READ | VM_PROT_WRITE);
loopva += PAGE_SIZE;
offset += PAGE_SIZE;
size -= PAGE_SIZE;
}
pmap_update(map->pmap);
/*
* zero on request (note that "size" is now zero due to the above loop
* so we need to subtract kva from loopva to reconstruct the size).
*/
if (zeroit)
memset((caddr_t)kva, 0, loopva - kva);
UVMHIST_LOG(maphist,"<- done (kva=0x%lx)", kva,0,0,0);
return(kva);
}
void
uvm_km_free(struct vm_map *map, vaddr_t addr, vsize_t size)
{
uvm_unmap(map, trunc_page(addr), round_page(addr+size));
}
vaddr_t
uvm_km_kmemalloc(struct vm_map *map, struct uvm_object *obj, vsize_t size,
int flags)
{
vaddr_t kva, loopva;
voff_t offset;
struct vm_page *pg;
int mapflags;
UVMHIST_FUNC("uvm_km_kmemalloc"); UVMHIST_CALLED(maphist);
UVMHIST_LOG(maphist," (map=%p, obj=%p, size=0x%lx, flags=%d)",
map, obj, size, flags);
KASSERT(vm_map_pmap(map) == pmap_kernel());
/*
* we cannot yet make pmap_enter() not sleep
* and thus demand that we are called with NOWAIT in that case
*/
KASSERT(!((flags & UVM_KMF_NOWAIT) && obj));
/*
* setup for call
*/
mapflags = flags & UVM_KMF_NOWAIT? UVM_FLAG_NOWAIT : 0;
mapflags |= flags & UVM_KMF_TRYLOCK;
size = round_page(size);
kva = vm_map_min(map); /* hint */
/*
* allocate some virtual space
*/
if (__predict_false(uvm_map(map, &kva, size, obj, UVM_UNKNOWN_OFFSET,
0, UVM_MAPFLAG(UVM_PROT_RW, UVM_PROT_RW, UVM_INH_NONE,
UVM_ADV_RANDOM, mapflags)) != 0)) {
UVMHIST_LOG(maphist, "<- done (no VM)",0,0,0,0);
return(0);
}
/*
* if all we wanted was VA, return now
*/
if (flags & UVM_KMF_VALLOC) {
UVMHIST_LOG(maphist,"<- done valloc (kva=0x%lx)", kva,0,0,0);
return(kva);
}
/*
* recover object offset from virtual address
*/
if (obj != NULL)
offset = kva - vm_map_min(kernel_map);
else
offset = 0;
UVMHIST_LOG(maphist, " kva=0x%lx, offset=0x%lx", kva, offset,0,0);
/*
* now allocate and map in the memory... note that we are the only ones
* whom should ever get a handle on this area of VM.
*/
loopva = kva;
while (loopva != kva + size) {
pg = uvm_pagealloc(obj, offset, NULL, 0);
if (pg) {
atomic_clearbits_int(&pg->pg_flags, PG_BUSY);
UVM_PAGE_OWN(pg, NULL);
}
if (__predict_false(pg == NULL)) {
if ((flags & UVM_KMF_NOWAIT) ||
((flags & UVM_KMF_CANFAIL) &&
uvmexp.swpgonly == uvmexp.swpages)) {
/* free everything! */
uvm_unmap(map, kva, kva + size);
return (0);
} else {
uvm_wait("km_getwait2"); /* sleep here */
continue;
}
}
/*
* map it in: note that we call pmap_enter with the map and
* object unlocked in case we are kmem_map.
*
* pager mappings that must not sleep here will incidently
* be installed using pmap_kenter_pa() and thus not sleep!
*/
if (obj == NULL) {
pmap_kenter_pa(loopva, VM_PAGE_TO_PHYS(pg),
UVM_PROT_RW);
} else {
pmap_enter(map->pmap, loopva, VM_PAGE_TO_PHYS(pg),
UVM_PROT_RW,
PMAP_WIRED | VM_PROT_READ | VM_PROT_WRITE);
}
loopva += PAGE_SIZE;
offset += PAGE_SIZE;
}
pmap_update(pmap_kernel());
UVMHIST_LOG(maphist,"<- done (kva=0x%lx)", kva,0,0,0);
return(kva);
}
int
uvm_mmap(vm_map_t map, vaddr_t *addr, vsize_t size, vm_prot_t prot,
vm_prot_t maxprot, int flags, caddr_t handle, voff_t foff,
vsize_t locklimit, struct proc *p)
{
struct uvm_object *uobj;
struct vnode *vp;
int error;
int advice = UVM_ADV_NORMAL;
uvm_flag_t uvmflag = 0;
vsize_t align = 0; /* userland page size */
/*
* check params
*/
if (size == 0)
return(0);
if (foff & PAGE_MASK)
return(EINVAL);
if ((prot & maxprot) != prot)
return(EINVAL);
/*
* for non-fixed mappings, round off the suggested address.
* for fixed mappings, check alignment and zap old mappings.
*/
if ((flags & MAP_FIXED) == 0) {
*addr = round_page(*addr); /* round */
} else {
if (*addr & PAGE_MASK)
return(EINVAL);
uvmflag |= UVM_FLAG_FIXED;
uvm_unmap_p(map, *addr, *addr + size, p); /* zap! */
}
/*
* handle anon vs. non-anon mappings. for non-anon mappings attach
* to underlying vm object.
*/
if (flags & MAP_ANON) {
if ((flags & MAP_FIXED) == 0 && size >= __LDPGSZ)
align = __LDPGSZ;
foff = UVM_UNKNOWN_OFFSET;
uobj = NULL;
if ((flags & MAP_SHARED) == 0)
/* XXX: defer amap create */
uvmflag |= UVM_FLAG_COPYONW;
else
/* shared: create amap now */
uvmflag |= UVM_FLAG_OVERLAY;
} else {
vp = (struct vnode *) handle; /* get vnode */
if (vp->v_type != VCHR) {
uobj = uvn_attach((void *) vp, (flags & MAP_SHARED) ?
maxprot : (maxprot & ~VM_PROT_WRITE));
if (uobj) {
assert((void*)uobj == vp);
if (flags & MAP_DENYWRITE)
uvmflag |= UVM_FLAG_DENYWRITE;
if ((flags & MAP_SHARED)
&& (maxprot & VM_PROT_WRITE))
uvmflag |= UVM_FLAG_WRITECOUNT;
}
#ifndef UBC
/*
* XXXCDC: hack from old code
* don't allow vnodes which have been mapped
* shared-writeable to persist [forces them to be
* flushed out when last reference goes].
* XXXCDC: interesting side effect: avoids a bug.
* note that in WRITE [ufs_readwrite.c] that we
* allocate buffer, uncache, and then do the write.
* the problem with this is that if the uncache causes
* VM data to be flushed to the same area of the file
* we are writing to... in that case we've got the
* buffer locked and our process goes to sleep forever.
*
* XXXCDC: checking maxprot protects us from the
* "persistbug" program but this is not a long term
* solution.
*
* XXXCDC: we don't bother calling uncache with the vp
* VOP_LOCKed since we know that we are already
* holding a valid reference to the uvn (from the
* uvn_attach above), and thus it is impossible for
* the uncache to kill the uvn and trigger I/O.
*/
if (flags & MAP_SHARED) {
if ((prot & VM_PROT_WRITE) ||
(maxprot & VM_PROT_WRITE)) {
uvm_vnp_uncache(vp);
}
}
#else
/* XXX for now, attach doesn't gain a ref */
VREF(vp);
#endif
} else {
uobj = udv_attach((void *) &vp->v_rdev,
(flags & MAP_SHARED) ? maxprot :
(maxprot & ~VM_PROT_WRITE), foff, size);
/*
* XXX Some devices don't like to be mapped with
* XXX PROT_EXEC, but we don't really have a
* XXX better way of handling this, right now
*/
if (uobj == NULL && (prot & PROT_EXEC) == 0) {
maxprot &= ~VM_PROT_EXECUTE;
uobj = udv_attach((void *) &vp->v_rdev,
(flags & MAP_SHARED) ? maxprot :
(maxprot & ~VM_PROT_WRITE), foff, size);
}
advice = UVM_ADV_RANDOM;
}
if (uobj == NULL)
return((vp->v_type == VREG) ? ENOMEM : EINVAL);
if ((flags & MAP_SHARED) == 0)
uvmflag |= UVM_FLAG_COPYONW;
}
/*
* set up mapping flags
*/
uvmflag = UVM_MAPFLAG(prot, maxprot,
(flags & MAP_SHARED) ? UVM_INH_SHARE : UVM_INH_COPY,
advice, uvmflag);
error = uvm_map_p(map, addr, size, uobj, foff, align, uvmflag, p);
if (error == 0) {
/*
* POSIX 1003.1b -- if our address space was configured
* to lock all future mappings, wire the one we just made.
*/
if (prot == VM_PROT_NONE) {
/*
* No more work to do in this case.
*/
return (0);
}
vm_map_lock(map);
if (map->flags & VM_MAP_WIREFUTURE) {
if ((atop(size) + uvmexp.wired) > uvmexp.wiredmax
#ifdef pmap_wired_count
|| (locklimit != 0 && (size +
ptoa(pmap_wired_count(vm_map_pmap(map)))) >
locklimit)
#endif
) {
error = ENOMEM;
vm_map_unlock(map);
/* unmap the region! */
uvm_unmap(map, *addr, *addr + size);
goto bad;
}
/*
* uvm_map_pageable() always returns the map
* unlocked.
*/
error = uvm_map_pageable(map, *addr, *addr + size,
FALSE, UVM_LK_ENTER);
if (error != 0) {
/* unmap the region! */
uvm_unmap(map, *addr, *addr + size);
goto bad;
}
return (0);
}
vm_map_unlock(map);
return (0);
}
/*
* errors: first detach from the uobj, if any.
*/
if (uobj)
uobj->pgops->pgo_detach(uobj);
bad:
return (error);
}
