/*
* kmap_alloc_wait:
*
* Allocates pageable memory from a sub-map of the kernel. If the submap
* has no room, the caller sleeps waiting for more memory in the submap.
*
* This routine may block.
*/
vm_offset_t
kmap_alloc_wait(vm_map_t map, vm_size_t size)
{
vm_offset_t addr;
size = round_page(size);
if (!swap_reserve(size))
return (0);
for (;;) {
/*
* To make this work for more than one map, use the map's lock
* to lock out sleepers/wakers.
*/
vm_map_lock(map);
if (vm_map_findspace(map, vm_map_min(map), size, &addr) == 0)
break;
/* no space now; see if we can ever get space */
if (vm_map_max(map) - vm_map_min(map) < size) {
vm_map_unlock(map);
swap_release(size);
return (0);
}
map->needs_wakeup = TRUE;
vm_map_unlock_and_wait(map, 0);
}
vm_map_insert(map, NULL, 0, addr, addr + size, VM_PROT_ALL,
VM_PROT_ALL, MAP_ACC_CHARGED);
vm_map_unlock(map);
return (addr);
}
/*
* No requirements.
*/
int
kernacc(c_caddr_t addr, int len, int rw)
{
boolean_t rv;
vm_offset_t saddr, eaddr;
vm_prot_t prot;
KASSERT((rw & (~VM_PROT_ALL)) == 0,
("illegal ``rw'' argument to kernacc (%x)", rw));
/*
* The globaldata space is not part of the kernel_map proper,
* check access separately.
*/
if (is_globaldata_space((vm_offset_t)addr, (vm_offset_t)(addr + len)))
return (TRUE);
/*
* Nominal kernel memory access - check access via kernel_map.
*/
if ((vm_offset_t)addr + len > vm_map_max(&kernel_map) ||
(vm_offset_t)addr + len < (vm_offset_t)addr) {
return (FALSE);
}
prot = rw;
saddr = trunc_page((vm_offset_t)addr);
eaddr = round_page((vm_offset_t)addr + len);
rv = vm_map_check_protection(&kernel_map, saddr, eaddr, prot, FALSE);
return (rv == TRUE);
}
/*
* tunable_mbinit() has to be run before any mbuf allocations are done.
*/
static void
tunable_mbinit(void *dummy)
{
#ifndef __rtems__
quad_t realmem;
/*
* The default limit for all mbuf related memory is 1/2 of all
* available kernel memory (physical or kmem).
* At most it can be 3/4 of available kernel memory.
*/
realmem = qmin((quad_t)physmem * PAGE_SIZE,
vm_map_max(kmem_map) - vm_map_min(kmem_map));
maxmbufmem = realmem / 2;
TUNABLE_QUAD_FETCH("kern.ipc.maxmbufmem", &maxmbufmem);
if (maxmbufmem > realmem / 4 * 3)
maxmbufmem = realmem / 4 * 3;
#else /* __rtems__ */
maxmbufmem = rtems_bsd_get_allocator_domain_size(
RTEMS_BSD_ALLOCATOR_DOMAIN_MBUF);
#endif /* __rtems__ */
TUNABLE_INT_FETCH("kern.ipc.nmbclusters", &nmbclusters);
if (nmbclusters == 0)
nmbclusters = maxmbufmem / MCLBYTES / 4;
TUNABLE_INT_FETCH("kern.ipc.nmbjumbop", &nmbjumbop);
if (nmbjumbop == 0)
nmbjumbop = maxmbufmem / MJUMPAGESIZE / 4;
TUNABLE_INT_FETCH("kern.ipc.nmbjumbo9", &nmbjumbo9);
if (nmbjumbo9 == 0)
nmbjumbo9 = maxmbufmem / MJUM9BYTES / 6;
TUNABLE_INT_FETCH("kern.ipc.nmbjumbo16", &nmbjumbo16);
if (nmbjumbo16 == 0)
nmbjumbo16 = maxmbufmem / MJUM16BYTES / 6;
/*
* We need at least as many mbufs as we have clusters of
* the various types added together.
*/
TUNABLE_INT_FETCH("kern.ipc.nmbufs", &nmbufs);
if (nmbufs < nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16)
nmbufs = lmax(maxmbufmem / MSIZE / 5,
nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16);
}
/*
* tunable_mbinit() has to be run before any mbuf allocations are done.
*/
static void
tunable_mbinit(void *dummy)
{
quad_t realmem, maxmbufmem;
/*
* The default limit for all mbuf related memory is 1/2 of all
* available kernel memory (physical or kmem).
* At most it can be 3/4 of available kernel memory.
*/
realmem = qmin((quad_t)physmem * PAGE_SIZE,
vm_map_max(kernel_map) - vm_map_min(kernel_map));
maxmbufmem = realmem / 2;
TUNABLE_QUAD_FETCH("kern.maxmbufmem", &maxmbufmem);
if (maxmbufmem > realmem / 4 * 3)
maxmbufmem = realmem / 4 * 3;
TUNABLE_INT_FETCH("kern.ipc.nmbclusters", &nmbclusters);
if (nmbclusters == 0)
nmbclusters = maxmbufmem / MCLBYTES / 4;
TUNABLE_INT_FETCH("kern.ipc.nmbjumbop", &nmbjumbop);
if (nmbjumbop == 0)
nmbjumbop = maxmbufmem / MJUMPAGESIZE / 4;
TUNABLE_INT_FETCH("kern.ipc.nmbjumbo9", &nmbjumbo9);
if (nmbjumbo9 == 0)
nmbjumbo9 = maxmbufmem / MJUM9BYTES / 6;
TUNABLE_INT_FETCH("kern.ipc.nmbjumbo16", &nmbjumbo16);
if (nmbjumbo16 == 0)
nmbjumbo16 = maxmbufmem / MJUM16BYTES / 6;
/*
* We need at least as many mbufs as we have clusters of
* the various types added together.
*/
TUNABLE_INT_FETCH("kern.ipc.nmbufs", &nmbufs);
if (nmbufs < nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16)
nmbufs = lmax(maxmbufmem / MSIZE / 5,
nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16);
}
static int
range_test(struct vm_map *map, vaddr_t addr, vsize_t size, bool ismmap)
{
vaddr_t vm_min_address = vm_map_min(map);
vaddr_t vm_max_address = vm_map_max(map);
vaddr_t eaddr = addr + size;
int res = 0;
if (addr < vm_min_address)
return EINVAL;
if (eaddr > vm_max_address)
return ismmap ? EFBIG : EINVAL;
if (addr > eaddr) /* no wrapping! */
return ismmap ? EOVERFLOW : EINVAL;
#ifdef MD_MMAP_RANGE_TEST
res = MD_MMAP_RANGE_TEST(addr, eaddr);
#endif
return res;
}
/*
* Return a fudged value to be used for vm_kmem_size for allocating
* the kmem_map. The memguard memory will be a submap.
*/
unsigned long
memguard_fudge(unsigned long km_size, const struct vm_map *parent_map)
{
u_long mem_pgs, parent_size;
vm_memguard_divisor = 10;
/* CTFLAG_RDTUN doesn't work during the early boot process. */
TUNABLE_INT_FETCH("vm.memguard.divisor", &vm_memguard_divisor);
parent_size = vm_map_max(parent_map) - vm_map_min(parent_map) +
PAGE_SIZE;
/* Pick a conservative value if provided value sucks. */
if ((vm_memguard_divisor <= 0) ||
((parent_size / vm_memguard_divisor) == 0))
vm_memguard_divisor = 10;
/*
* Limit consumption of physical pages to
* 1/vm_memguard_divisor of system memory. If the KVA is
* smaller than this then the KVA limit comes into play first.
* This prevents memguard's page promotions from completely
* using up memory, since most malloc(9) calls are sub-page.
*/
mem_pgs = vm_cnt.v_page_count;
memguard_physlimit = (mem_pgs / vm_memguard_divisor) * PAGE_SIZE;
/*
* We want as much KVA as we can take safely. Use at most our
* allotted fraction of the parent map's size. Limit this to
* twice the physical memory to avoid using too much memory as
* pagetable pages (size must be multiple of PAGE_SIZE).
*/
memguard_mapsize = round_page(parent_size / vm_memguard_divisor);
if (memguard_mapsize / (2 * PAGE_SIZE) > mem_pgs)
memguard_mapsize = mem_pgs * 2 * PAGE_SIZE;
if (km_size + memguard_mapsize > parent_size)
memguard_mapsize = 0;
return (km_size + memguard_mapsize);
}
vaddr_t
uvm_km_valloc_prefer_wait(struct vm_map *map, vsize_t size, voff_t prefer)
{
vaddr_t kva;
UVMHIST_FUNC("uvm_km_valloc_prefer_wait"); UVMHIST_CALLED(maphist);
UVMHIST_LOG(maphist, "(map=%p, size=0x%lx)", map, size, 0,0);
KASSERT(vm_map_pmap(map) == pmap_kernel());
size = round_page(size);
if (size > vm_map_max(map) - vm_map_min(map))
return(0);
while (1) {
kva = vm_map_min(map); /* hint */
/*
* allocate some virtual space. will be demand filled
* by kernel_object.
*/
if (__predict_true(uvm_map(map, &kva, size, uvm.kernel_object,
prefer, 0, UVM_MAPFLAG(UVM_PROT_ALL,
UVM_PROT_ALL, UVM_INH_NONE, UVM_ADV_RANDOM, 0)) == 0)) {
UVMHIST_LOG(maphist,"<- done (kva=0x%lx)", kva,0,0,0);
return(kva);
}
/*
* failed. sleep for a while (on map)
*/
UVMHIST_LOG(maphist,"<<<sleeping>>>",0,0,0,0);
tsleep((caddr_t)map, PVM, "vallocwait", 0);
}
/*NOTREACHED*/
}
/*
* Hold each of the physical pages that are mapped by the specified range of
* virtual addresses, ["addr", "addr" + "len"), if those mappings are valid
* and allow the specified types of access, "prot". If all of the implied
* pages are successfully held, then the number of held pages is returned
* together with pointers to those pages in the array "ma". However, if any
* of the pages cannot be held, -1 is returned.
*/
int
vm_fault_quick_hold_pages(vm_map_t map, vm_offset_t addr, vm_size_t len,
vm_prot_t prot, vm_page_t *ma, int max_count)
{
vm_offset_t end, va;
vm_page_t *mp;
int count;
boolean_t pmap_failed;
if (len == 0)
return (0);
end = round_page(addr + len);
addr = trunc_page(addr);
/*
* Check for illegal addresses.
*/
if (addr < vm_map_min(map) || addr > end || end > vm_map_max(map))
return (-1);
count = howmany(end - addr, PAGE_SIZE);
if (count > max_count)
panic("vm_fault_quick_hold_pages: count > max_count");
/*
* Most likely, the physical pages are resident in the pmap, so it is
* faster to try pmap_extract_and_hold() first.
*/
pmap_failed = FALSE;
for (mp = ma, va = addr; va < end; mp++, va += PAGE_SIZE) {
*mp = pmap_extract_and_hold(map->pmap, va, prot);
if (*mp == NULL)
pmap_failed = TRUE;
else if ((prot & VM_PROT_WRITE) != 0 &&
(*mp)->dirty != VM_PAGE_BITS_ALL) {
/*
* Explicitly dirty the physical page. Otherwise, the
* caller's changes may go unnoticed because they are
* performed through an unmanaged mapping or by a DMA
* operation.
*
* The object lock is not held here.
* See vm_page_clear_dirty_mask().
*/
vm_page_dirty(*mp);
}
}
if (pmap_failed) {
/*
* One or more pages could not be held by the pmap. Either no
* page was mapped at the specified virtual address or that
* mapping had insufficient permissions. Attempt to fault in
* and hold these pages.
*/
for (mp = ma, va = addr; va < end; mp++, va += PAGE_SIZE)
if (*mp == NULL && vm_fault_hold(map, va, prot,
VM_FAULT_NORMAL, mp) != KERN_SUCCESS)
goto error;
}
return (count);
error:
for (mp = ma; mp < ma + count; mp++)
if (*mp != NULL) {
vm_page_lock(*mp);
vm_page_unhold(*mp);
vm_page_unlock(*mp);
}
return (-1);
}
/*
* Destroy old address space, and allocate a new stack.
* The new stack is only sgrowsiz large because it is grown
* automatically on a page fault.
*/
int
exec_new_vmspace(struct image_params *imgp, struct sysentvec *sv)
{
int error;
struct proc *p = imgp->proc;
struct vmspace *vmspace = p->p_vmspace;
vm_object_t obj;
struct rlimit rlim_stack;
vm_offset_t sv_minuser, stack_addr;
vm_map_t map;
u_long ssiz;
imgp->vmspace_destroyed = 1;
imgp->sysent = sv;
/* May be called with Giant held */
EVENTHANDLER_DIRECT_INVOKE(process_exec, p, imgp);
/*
* Blow away entire process VM, if address space not shared,
* otherwise, create a new VM space so that other threads are
* not disrupted
*/
map = &vmspace->vm_map;
if (map_at_zero)
sv_minuser = sv->sv_minuser;
else
sv_minuser = MAX(sv->sv_minuser, PAGE_SIZE);
if (vmspace->vm_refcnt == 1 && vm_map_min(map) == sv_minuser &&
vm_map_max(map) == sv->sv_maxuser &&
cpu_exec_vmspace_reuse(p, map)) {
shmexit(vmspace);
pmap_remove_pages(vmspace_pmap(vmspace));
vm_map_remove(map, vm_map_min(map), vm_map_max(map));
/*
* An exec terminates mlockall(MCL_FUTURE), ASLR state
* must be re-evaluated.
*/
vm_map_lock(map);
vm_map_modflags(map, 0, MAP_WIREFUTURE | MAP_ASLR |
MAP_ASLR_IGNSTART);
vm_map_unlock(map);
} else {
error = vmspace_exec(p, sv_minuser, sv->sv_maxuser);
if (error)
return (error);
vmspace = p->p_vmspace;
map = &vmspace->vm_map;
}
map->flags |= imgp->map_flags;
/* Map a shared page */
obj = sv->sv_shared_page_obj;
if (obj != NULL) {
vm_object_reference(obj);
error = vm_map_fixed(map, obj, 0,
sv->sv_shared_page_base, sv->sv_shared_page_len,
VM_PROT_READ | VM_PROT_EXECUTE,
VM_PROT_READ | VM_PROT_EXECUTE,
MAP_INHERIT_SHARE | MAP_ACC_NO_CHARGE);
if (error != KERN_SUCCESS) {
vm_object_deallocate(obj);
return (vm_mmap_to_errno(error));
}
}
/* Allocate a new stack */
if (imgp->stack_sz != 0) {
ssiz = trunc_page(imgp->stack_sz);
PROC_LOCK(p);
lim_rlimit_proc(p, RLIMIT_STACK, &rlim_stack);
PROC_UNLOCK(p);
if (ssiz > rlim_stack.rlim_max)
ssiz = rlim_stack.rlim_max;
if (ssiz > rlim_stack.rlim_cur) {
rlim_stack.rlim_cur = ssiz;
kern_setrlimit(curthread, RLIMIT_STACK, &rlim_stack);
}
} else if (sv->sv_maxssiz != NULL) {
ssiz = *sv->sv_maxssiz;
} else {
ssiz = maxssiz;
}
stack_addr = sv->sv_usrstack - ssiz;
error = vm_map_stack(map, stack_addr, (vm_size_t)ssiz,
obj != NULL && imgp->stack_prot != 0 ? imgp->stack_prot :
sv->sv_stackprot, VM_PROT_ALL, MAP_STACK_GROWS_DOWN);
if (error != KERN_SUCCESS)
return (vm_mmap_to_errno(error));
/*
* vm_ssize and vm_maxsaddr are somewhat antiquated concepts, but they
* are still used to enforce the stack rlimit on the process stack.
*/
vmspace->vm_ssize = sgrowsiz >> PAGE_SHIFT;
vmspace->vm_maxsaddr = (char *)stack_addr;
return (0);
}
/*
* Machine-dependent startup code
*/
void
cpu_startup()
{
caddr_t v;
int sz;
#ifdef DEBUG
extern int pmapdebug;
int opmapdebug = pmapdebug;
#endif
vaddr_t minaddr, maxaddr;
extern struct user *proc0paddr;
#ifdef DEBUG
pmapdebug = 0;
#endif
if (CPU_ISSUN4M) {
extern int stackgap_random;
stackgap_random = STACKGAP_RANDOM_SUN4M;
}
/*
* fix message buffer mapping, note phys addr of msgbuf is 0
*/
pmap_map(MSGBUF_VA, 0, MSGBUFSIZE, VM_PROT_READ|VM_PROT_WRITE);
initmsgbuf((caddr_t)(MSGBUF_VA + (CPU_ISSUN4 ? 4096 : 0)), MSGBUFSIZE);
proc0.p_addr = proc0paddr;
/*
* Good {morning,afternoon,evening,night}.
*/
printf(version);
/*identifycpu();*/
printf("real mem = %u (%uMB)\n", ptoa(physmem),
ptoa(physmem)/1024/1024);
/*
* Find out how much space we need, allocate it,
* and then give everything true virtual addresses.
*/
sz = (int)allocsys((caddr_t)0);
if ((v = (caddr_t)uvm_km_alloc(kernel_map, round_page(sz))) == 0)
panic("startup: no room for tables");
if (allocsys(v) - v != sz)
panic("startup: table size inconsistency");
/*
* Determine how many buffers to allocate.
* We allocate bufcachepercent% of memory for buffer space.
*/
if (bufpages == 0)
bufpages = physmem * bufcachepercent / 100;
/* Restrict to at most 25% filled kvm */
if (bufpages >
(VM_MAX_KERNEL_ADDRESS-VM_MIN_KERNEL_ADDRESS) / PAGE_SIZE / 4)
bufpages = (VM_MAX_KERNEL_ADDRESS-VM_MIN_KERNEL_ADDRESS) /
PAGE_SIZE / 4;
/*
* Allocate a submap for exec arguments. This map effectively
* limits the number of processes exec'ing at any time.
*/
minaddr = vm_map_min(kernel_map);
exec_map = uvm_km_suballoc(kernel_map, &minaddr, &maxaddr,
16*NCARGS, VM_MAP_PAGEABLE, FALSE, NULL);
/*
* Allocate a map for physio. Others use a submap of the kernel
* map, but we want one completely separate, even though it uses
* the same pmap.
*/
dvma_base = CPU_ISSUN4M ? DVMA4M_BASE : DVMA_BASE;
dvma_end = CPU_ISSUN4M ? DVMA4M_END : DVMA_END;
#if defined(SUN4M)
if (CPU_ISSUN4M) {
/*
* The DVMA space we want partially overrides kernel_map.
* Allocate it in kernel_map as well to prevent it from being
* used for other things.
*/
if (uvm_map(kernel_map, &dvma_base,
vm_map_max(kernel_map) - dvma_base,
NULL, UVM_UNKNOWN_OFFSET, 0,
UVM_MAPFLAG(UVM_PROT_NONE, UVM_PROT_NONE, UVM_INH_NONE,
UVM_ADV_NORMAL, 0)))
panic("startup: can not steal dvma map");
}
#endif
phys_map = uvm_map_create(pmap_kernel(), dvma_base, dvma_end,
VM_MAP_INTRSAFE);
if (phys_map == NULL)
panic("unable to create DVMA map");
/*
* Allocate DVMA space and dump into a privately managed
* resource map for double mappings which is usable from
* interrupt contexts.
*/
if (uvm_km_valloc_wait(phys_map, (dvma_end-dvma_base)) != dvma_base)
panic("unable to allocate from DVMA map");
dvmamap_extent = extent_create("dvmamap", dvma_base, dvma_end,
M_DEVBUF, NULL, 0, EX_NOWAIT);
if (dvmamap_extent == 0)
panic("unable to allocate extent for dvma");
#ifdef DEBUG
pmapdebug = opmapdebug;
#endif
printf("avail mem = %lu (%luMB)\n", ptoa(uvmexp.free),
ptoa(uvmexp.free)/1024/1024);
/*
* Set up buffers, so they can be used to read disk labels.
*/
bufinit();
/* Early interrupt handlers initialization */
intr_init();
}
/**
* Worker for the two virtual address space reservers.
*
* We're leaning on the examples provided by mmap and vm_mmap in vm_mmap.c here.
*/
static int rtR0MemObjNativeReserveInMap(PPRTR0MEMOBJINTERNAL ppMem, void *pvFixed, size_t cb, size_t uAlignment, RTR0PROCESS R0Process, vm_map_t pMap)
{
int rc;
/*
* The pvFixed address range must be within the VM space when specified.
*/
if ( pvFixed != (void *)-1
&& ( (vm_offset_t)pvFixed < vm_map_min(pMap)
|| (vm_offset_t)pvFixed + cb > vm_map_max(pMap)))
return VERR_INVALID_PARAMETER;
/*
* Check that the specified alignment is supported.
*/
if (uAlignment > PAGE_SIZE)
return VERR_NOT_SUPPORTED;
/*
* Create the object.
*/
PRTR0MEMOBJFREEBSD pMemFreeBSD = (PRTR0MEMOBJFREEBSD)rtR0MemObjNew(sizeof(*pMemFreeBSD), RTR0MEMOBJTYPE_RES_VIRT, NULL, cb);
if (!pMemFreeBSD)
return VERR_NO_MEMORY;
vm_offset_t MapAddress = pvFixed != (void *)-1
? (vm_offset_t)pvFixed
: vm_map_min(pMap);
if (pvFixed != (void *)-1)
vm_map_remove(pMap,
MapAddress,
MapAddress + cb);
rc = vm_map_find(pMap, /* map */
NULL, /* object */
0, /* offset */
&MapAddress, /* addr (IN/OUT) */
cb, /* length */
#if __FreeBSD_version >= 1000055
0, /* max addr */
#endif
pvFixed == (void *)-1 ? VMFS_ANY_SPACE : VMFS_NO_SPACE,
/* find_space */
VM_PROT_NONE, /* protection */
VM_PROT_ALL, /* max(_prot) ?? */
0); /* cow (copy-on-write) */
if (rc == KERN_SUCCESS)
{
if (R0Process != NIL_RTR0PROCESS)
{
rc = vm_map_inherit(pMap,
MapAddress,
MapAddress + cb,
VM_INHERIT_SHARE);
AssertMsg(rc == KERN_SUCCESS, ("%#x\n", rc));
}
pMemFreeBSD->Core.pv = (void *)MapAddress;
pMemFreeBSD->Core.u.ResVirt.R0Process = R0Process;
*ppMem = &pMemFreeBSD->Core;
return VINF_SUCCESS;
}
rc = VERR_NO_MEMORY; /** @todo fix translation (borrow from darwin) */
rtR0MemObjDelete(&pMemFreeBSD->Core);
return rc;
}
static int
copyio(int copy_type, user_addr_t user_addr, char *kernel_addr,
vm_size_t nbytes, vm_size_t *lencopied, int use_kernel_map)
{
thread_t thread;
pmap_t pmap;
vm_size_t bytes_copied;
int error = 0;
boolean_t istate = FALSE;
boolean_t recursive_CopyIOActive;
#if KDEBUG
int debug_type = 0xeff70010;
debug_type += (copy_type << 2);
#endif
thread = current_thread();
KERNEL_DEBUG(debug_type | DBG_FUNC_START,
(unsigned)(user_addr >> 32), (unsigned)user_addr,
nbytes, thread->machine.copyio_state, 0);
if (nbytes == 0)
goto out;
pmap = thread->map->pmap;
if ((copy_type != COPYINPHYS) && (copy_type != COPYOUTPHYS) && ((vm_offset_t)kernel_addr < VM_MIN_KERNEL_AND_KEXT_ADDRESS)) {
panic("Invalid copy parameter, copy type: %d, kernel address: %p", copy_type, kernel_addr);
}
/* Sanity and security check for addresses to/from a user */
if (((pmap != kernel_pmap) && (use_kernel_map == 0)) &&
((nbytes && (user_addr+nbytes <= user_addr)) || ((user_addr + nbytes) > vm_map_max(thread->map)))) {
error = EFAULT;
goto out;
}
/*
* If the no_shared_cr3 boot-arg is set (true), the kernel runs on
* its own pmap and cr3 rather than the user's -- so that wild accesses
* from kernel or kexts can be trapped. So, during copyin and copyout,
* we need to switch back to the user's map/cr3. The thread is flagged
* "CopyIOActive" at this time so that if the thread is pre-empted,
* we will later restore the correct cr3.
*/
recursive_CopyIOActive = thread->machine.specFlags & CopyIOActive;
thread->machine.specFlags |= CopyIOActive;
user_access_enable();
if (no_shared_cr3) {
istate = ml_set_interrupts_enabled(FALSE);
if (get_cr3_base() != pmap->pm_cr3)
set_cr3_raw(pmap->pm_cr3);
}
/*
* Ensure that we're running on the target thread's cr3.
*/
if ((pmap != kernel_pmap) && !use_kernel_map &&
(get_cr3_base() != pmap->pm_cr3)) {
panic("copyio(%d,%p,%p,%ld,%p,%d) cr3 is %p expects %p",
copy_type, (void *)user_addr, kernel_addr, nbytes, lencopied, use_kernel_map,
(void *) get_cr3_raw(), (void *) pmap->pm_cr3);
}
if (no_shared_cr3)
(void) ml_set_interrupts_enabled(istate);
KERNEL_DEBUG(0xeff70044 | DBG_FUNC_NONE, (unsigned)user_addr,
(unsigned)kernel_addr, nbytes, 0, 0);
switch (copy_type) {
case COPYIN:
error = _bcopy((const void *) user_addr,
kernel_addr,
nbytes);
break;
case COPYOUT:
error = _bcopy(kernel_addr,
(void *) user_addr,
nbytes);
break;
case COPYINPHYS:
error = _bcopy((const void *) user_addr,
PHYSMAP_PTOV(kernel_addr),
nbytes);
break;
case COPYOUTPHYS:
error = _bcopy((const void *) PHYSMAP_PTOV(kernel_addr),
(void *) user_addr,
nbytes);
break;
case COPYINSTR:
error = _bcopystr((const void *) user_addr,
kernel_addr,
(int) nbytes,
&bytes_copied);
/*
* lencopied should be updated on success
* or ENAMETOOLONG... but not EFAULT
*/
if (error != EFAULT)
*lencopied = bytes_copied;
if (error) {
#if KDEBUG
nbytes = *lencopied;
#endif
break;
}
if (*(kernel_addr + bytes_copied - 1) == 0) {
/*
* we found a NULL terminator... we're done
*/
#if KDEBUG
nbytes = *lencopied;
#endif
break;
} else {
/*
* no more room in the buffer and we haven't
* yet come across a NULL terminator
*/
#if KDEBUG
nbytes = *lencopied;
#endif
error = ENAMETOOLONG;
break;
}
break;
}
user_access_disable();
if (!recursive_CopyIOActive) {
thread->machine.specFlags &= ~CopyIOActive;
}
if (no_shared_cr3) {
istate = ml_set_interrupts_enabled(FALSE);
if (get_cr3_raw() != kernel_pmap->pm_cr3)
set_cr3_raw(kernel_pmap->pm_cr3);
(void) ml_set_interrupts_enabled(istate);
}
out:
KERNEL_DEBUG(debug_type | DBG_FUNC_END, (unsigned)user_addr,
(unsigned)kernel_addr, (unsigned)nbytes, error, 0);
return (error);
}
vm_offset_t
get_map_max(
vm_map_t map)
{
return(vm_map_max(map));
}
vm_offset_t max_valid_stack_address(void)
{
return vm_map_max(kernel_map);
}
int
uvm_mmap(struct vm_map *map, vaddr_t *addr, vsize_t size, vm_prot_t prot,
vm_prot_t maxprot, int flags, int advice, struct uvm_object *uobj,
voff_t foff, vsize_t locklimit)
{
vaddr_t align = 0;
int error;
uvm_flag_t uvmflag = 0;
/*
* check params
*/
if (size == 0)
return 0;
if (foff & PAGE_MASK)
return EINVAL;
if ((prot & maxprot) != prot)
return EINVAL;
/*
* for non-fixed mappings, round off the suggested address.
* for fixed mappings, check alignment and zap old mappings.
*/
if ((flags & MAP_FIXED) == 0) {
*addr = round_page(*addr);
} else {
if (*addr & PAGE_MASK)
return EINVAL;
uvmflag |= UVM_FLAG_FIXED;
(void) uvm_unmap(map, *addr, *addr + size);
}
/*
* Try to see if any requested alignment can even be attemped.
* Make sure we can express the alignment (asking for a >= 4GB
* alignment on an ILP32 architecure make no sense) and the
* alignment is at least for a page sized quanitiy. If the
* request was for a fixed mapping, make sure supplied address
* adheres to the request alignment.
*/
align = (flags & MAP_ALIGNMENT_MASK) >> MAP_ALIGNMENT_SHIFT;
if (align) {
if (align >= sizeof(vaddr_t) * NBBY)
return EINVAL;
align = 1L << align;
if (align < PAGE_SIZE)
return EINVAL;
if (align >= vm_map_max(map))
return ENOMEM;
if (flags & MAP_FIXED) {
if ((*addr & (align-1)) != 0)
return EINVAL;
align = 0;
}
}
/*
* check resource limits
*/
if (!VM_MAP_IS_KERNEL(map) &&
(((rlim_t)curproc->p_vmspace->vm_map.size + (rlim_t)size) >
curproc->p_rlimit[RLIMIT_AS].rlim_cur))
return ENOMEM;
/*
* handle anon vs. non-anon mappings. for non-anon mappings attach
* to underlying vm object.
*/
if (flags & MAP_ANON) {
KASSERT(uobj == NULL);
foff = UVM_UNKNOWN_OFFSET;
if ((flags & MAP_SHARED) == 0)
/* XXX: defer amap create */
uvmflag |= UVM_FLAG_COPYONW;
else
/* shared: create amap now */
uvmflag |= UVM_FLAG_OVERLAY;
} else {
KASSERT(uobj != NULL);
if ((flags & MAP_SHARED) == 0) {
uvmflag |= UVM_FLAG_COPYONW;
}
}
uvmflag = UVM_MAPFLAG(prot, maxprot,
(flags & MAP_SHARED) ? UVM_INH_SHARE : UVM_INH_COPY, advice,
uvmflag);
error = uvm_map(map, addr, size, uobj, foff, align, uvmflag);
if (error) {
if (uobj)
uobj->pgops->pgo_detach(uobj);
return error;
}
/*
* POSIX 1003.1b -- if our address space was configured
* to lock all future mappings, wire the one we just made.
*
* Also handle the MAP_WIRED flag here.
*/
if (prot == VM_PROT_NONE) {
/*
* No more work to do in this case.
*/
return 0;
}
if ((flags & MAP_WIRED) != 0 || (map->flags & VM_MAP_WIREFUTURE) != 0) {
vm_map_lock(map);
if (atop(size) + uvmexp.wired > uvmexp.wiredmax ||
(locklimit != 0 &&
size + ptoa(pmap_wired_count(vm_map_pmap(map))) >
locklimit)) {
vm_map_unlock(map);
uvm_unmap(map, *addr, *addr + size);
return ENOMEM;
}
/*
* uvm_map_pageable() always returns the map unlocked.
*/
error = uvm_map_pageable(map, *addr, *addr + size,
false, UVM_LK_ENTER);
if (error) {
uvm_unmap(map, *addr, *addr + size);
return error;
}
return 0;
}
return 0;
}
void
uvm_init(void)
{
vaddr_t kvm_start, kvm_end;
/*
* step 0: ensure that the hardware set the page size
*/
if (uvmexp.pagesize == 0) {
panic("uvm_init: page size not set");
}
/*
* step 1: set up stats.
*/
averunnable.fscale = FSCALE;
/*
* step 2: init the page sub-system. this includes allocating the
* vm_page structures, and setting up all the page queues (and
* locks). available memory will be put in the "free" queue.
* kvm_start and kvm_end will be set to the area of kernel virtual
* memory which is available for general use.
*/
uvm_page_init(&kvm_start, &kvm_end);
/*
* step 3: init the map sub-system. allocates the static pool of
* vm_map_entry structures that are used for "special" kernel maps
* (e.g. kernel_map, kmem_map, etc...).
*/
uvm_map_init();
/*
* step 4: setup the kernel's virtual memory data structures. this
* includes setting up the kernel_map/kernel_object and the kmem_map/
* kmem_object.
*/
uvm_km_init(kvm_start, kvm_end);
/*
* step 5: init the pmap module. the pmap module is free to allocate
* memory for its private use (e.g. pvlists).
*/
pmap_init();
/*
* step 6: init the kernel memory allocator. after this call the
* kernel memory allocator (malloc) can be used.
*/
kmeminit();
/*
* step 6.5: init the dma allocator, which is backed by pools.
*/
dma_alloc_init();
/*
* step 7: init all pagers and the pager_map.
*/
uvm_pager_init();
/*
* step 8: init anonymous memory system
*/
amap_init(); /* init amap module */
/*
* step 9: init uvm_km_page allocator memory.
*/
uvm_km_page_init();
/*
* the VM system is now up! now that malloc is up we can
* enable paging of kernel objects.
*/
uao_create(VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS,
UAO_FLAG_KERNSWAP);
/*
* reserve some unmapped space for malloc/pool use after free usage
*/
#ifdef DEADBEEF0
kvm_start = trunc_page(DEADBEEF0) - PAGE_SIZE;
if (uvm_map(kernel_map, &kvm_start, 3 * PAGE_SIZE,
NULL, UVM_UNKNOWN_OFFSET, 0, UVM_MAPFLAG(UVM_PROT_NONE,
UVM_PROT_NONE, UVM_INH_NONE, UVM_ADV_RANDOM, UVM_FLAG_FIXED)))
panic("uvm_init: cannot reserve dead beef @0x%x", DEADBEEF0);
#endif
#ifdef DEADBEEF1
kvm_start = trunc_page(DEADBEEF1) - PAGE_SIZE;
if (uvm_map(kernel_map, &kvm_start, 3 * PAGE_SIZE,
NULL, UVM_UNKNOWN_OFFSET, 0, UVM_MAPFLAG(UVM_PROT_NONE,
UVM_PROT_NONE, UVM_INH_NONE, UVM_ADV_RANDOM, UVM_FLAG_FIXED)))
panic("uvm_init: cannot reserve dead beef @0x%x", DEADBEEF1);
#endif
/*
* init anonymous memory systems
*/
uvm_anon_init();
#ifndef SMALL_KERNEL
/*
* Switch kernel and kmem_map over to a best-fit allocator,
* instead of walking the tree.
*/
uvm_map_set_uaddr(kernel_map, &kernel_map->uaddr_any[3],
uaddr_bestfit_create(vm_map_min(kernel_map),
vm_map_max(kernel_map)));
uvm_map_set_uaddr(kmem_map, &kmem_map->uaddr_any[3],
uaddr_bestfit_create(vm_map_min(kmem_map),
vm_map_max(kmem_map)));
#endif /* !SMALL_KERNEL */
}
