/*
* Map an IO request into kernel virtual address space.
*/
void
vmapbuf(struct buf *bp, vsize_t len)
{
vaddr_t faddr, taddr, off;
paddr_t pa;
#ifdef DIAGNOSTIC
if (!(bp->b_flags & B_PHYS))
panic("vmapbuf");
#endif
faddr = trunc_page((vaddr_t)(bp->b_saveaddr = bp->b_data));
off = (vaddr_t)bp->b_data - faddr;
len = round_page(off + len);
taddr = uvm_km_valloc_wait(phys_map, len);
bp->b_data = (caddr_t)(taddr + off);
for (; len > 0; len -= NBPG) {
pmap_extract(vm_map_pmap(&bp->b_proc->p_vmspace->vm_map),
faddr, &pa);
pmap_enter(vm_map_pmap(phys_map), taddr, pa,
VM_PROT_READ | VM_PROT_WRITE, PMAP_WIRED);
faddr += NBPG;
taddr += NBPG;
}
pmap_update(vm_map_pmap(phys_map));
}
/*
* Map an IO request into kernel virtual address space.
*/
int
vmapbuf(struct buf *bp, vsize_t len)
{
vaddr_t uva, kva;
paddr_t pa;
vsize_t size, off;
int npf;
struct pmap *upmap, *kpmap;
#ifdef DIAGNOSTIC
if ((bp->b_flags & B_PHYS) == 0)
panic("vmapbuf");
#endif
upmap = vm_map_pmap(&bp->b_proc->p_vmspace->vm_map);
kpmap = vm_map_pmap(phys_map);
bp->b_saveaddr = bp->b_data;
uva = trunc_page((vaddr_t)bp->b_data);
off = (vaddr_t)bp->b_data - uva;
size = round_page(off + len);
kva = uvm_km_alloc(phys_map, len, 0, UVM_KMF_VAONLY | UVM_KMF_WAITVA);
bp->b_data = (void *)(kva + off);
npf = btoc(size);
while (npf--) {
if (pmap_extract(upmap, uva, &pa) == false)
panic("vmapbuf: null page frame");
pmap_enter(kpmap, kva, pa,
VM_PROT_READ | VM_PROT_WRITE, PMAP_WIRED);
uva += PAGE_SIZE;
kva += PAGE_SIZE;
}
pmap_update(kpmap);
return 0;
}
/* This code was originally stolen from the alpha port. */
int
vmapbuf(struct buf *bp, vsize_t len)
{
vaddr_t faddr, taddr, off;
paddr_t pa;
struct proc *p;
vm_prot_t prot;
if ((bp->b_flags & B_PHYS) == 0)
panic("vmapbuf");
p = bp->b_proc;
bp->b_saveaddr = bp->b_data;
faddr = trunc_page((vaddr_t)bp->b_data);
off = (vaddr_t)bp->b_data - faddr;
len = round_page(off + len);
taddr = uvm_km_alloc(phys_map, len, 0, UVM_KMF_VAONLY | UVM_KMF_WAITVA);
bp->b_data = (void *)(taddr + off);
len = atop(len);
prot = bp->b_flags & B_READ ? VM_PROT_READ | VM_PROT_WRITE :
VM_PROT_READ;
while (len--) {
if (pmap_extract(vm_map_pmap(&p->p_vmspace->vm_map), faddr,
&pa) == false)
panic("vmapbuf: null page frame");
pmap_enter(vm_map_pmap(phys_map), taddr, trunc_page(pa),
prot, prot | PMAP_WIRED);
faddr += PAGE_SIZE;
taddr += PAGE_SIZE;
}
pmap_update(vm_map_pmap(phys_map));
return 0;
}
vaddr_t
uvm_km_valloc_align(struct vm_map *map, vsize_t size, vsize_t align)
{
vaddr_t kva;
UVMHIST_FUNC("uvm_km_valloc"); UVMHIST_CALLED(maphist);
UVMHIST_LOG(maphist, "(map=%p, size=0x%lx)", map, size, 0,0);
KASSERT(vm_map_pmap(map) == pmap_kernel());
size = round_page(size);
kva = vm_map_min(map); /* hint */
/*
* allocate some virtual space. will be demand filled by kernel_object.
*/
if (__predict_false(uvm_map(map, &kva, size, uvm.kernel_object,
UVM_UNKNOWN_OFFSET, align, UVM_MAPFLAG(UVM_PROT_ALL, UVM_PROT_ALL,
UVM_INH_NONE, UVM_ADV_RANDOM, 0)) != 0)) {
UVMHIST_LOG(maphist, "<- done (no VM)", 0,0,0,0);
return(0);
}
UVMHIST_LOG(maphist, "<- done (kva=0x%lx)", kva,0,0,0);
return(kva);
}
/*
* vm_fault_unwire:
*
* Unwire a range of virtual addresses in a map.
*/
void
vm_fault_unwire(vm_map_t map, vm_offset_t start, vm_offset_t end,
boolean_t fictitious)
{
vm_paddr_t pa;
vm_offset_t va;
vm_page_t m;
pmap_t pmap;
pmap = vm_map_pmap(map);
/*
* Since the pages are wired down, we must be able to get their
* mappings from the physical map system.
*/
for (va = start; va < end; va += PAGE_SIZE) {
pa = pmap_extract(pmap, va);
if (pa != 0) {
pmap_change_wiring(pmap, va, FALSE);
if (!fictitious) {
m = PHYS_TO_VM_PAGE(pa);
vm_page_lock(m);
vm_page_unwire(m, TRUE);
vm_page_unlock(m);
}
}
}
}
/*
* Map an IO request into kernel virtual address space.
*/
void
vmapbuf(struct buf *bp, vsize_t len)
{
struct pmap *pm = vm_map_pmap(&bp->b_proc->p_vmspace->vm_map);
vaddr_t kva, uva;
vsize_t size, off;
#ifdef DIAGNOSTIC
if ((bp->b_flags & B_PHYS) == 0)
panic("vmapbuf");
#endif
bp->b_saveaddr = bp->b_data;
uva = trunc_page((vaddr_t)bp->b_data);
off = (vaddr_t)bp->b_data - uva;
size = round_page(off + len);
kva = uvm_km_valloc_prefer_wait(phys_map, size, uva);
bp->b_data = (caddr_t)(kva + off);
while (size > 0) {
paddr_t pa;
if (pmap_extract(pm, uva, &pa) == FALSE)
panic("vmapbuf: null page frame");
else
pmap_kenter_pa(kva, pa, UVM_PROT_RW);
uva += PAGE_SIZE;
kva += PAGE_SIZE;
size -= PAGE_SIZE;
}
pmap_update(pmap_kernel());
}
/*
* Map a user I/O request into kernel virtual address space.
* Note: the pages are already locked by uvm_vslock(), so we
* do not need to pass an access_type to pmap_enter().
*/
int
vmapbuf(struct buf *bp, vsize_t len)
{
struct pmap *upmap;
vaddr_t uva; /* User VA (map from) */
vaddr_t kva; /* Kernel VA (new to) */
paddr_t pa; /* physical address */
vsize_t off;
if ((bp->b_flags & B_PHYS) == 0)
panic("vmapbuf");
bp->b_saveaddr = bp->b_data;
uva = trunc_page((vaddr_t)bp->b_data);
off = (vaddr_t)bp->b_data - uva;
len = round_page(off + len);
kva = uvm_km_alloc(kernel_map, len, 0, UVM_KMF_VAONLY | UVM_KMF_WAITVA);
bp->b_data = (void *)(kva + off);
upmap = vm_map_pmap(&bp->b_proc->p_vmspace->vm_map);
do {
if (pmap_extract(upmap, uva, &pa) == FALSE)
panic("vmapbuf: null page frame");
/* Now map the page into kernel space. */
pmap_kenter_pa(kva, pa, VM_PROT_READ | VM_PROT_WRITE, 0);
uva += PAGE_SIZE;
kva += PAGE_SIZE;
len -= PAGE_SIZE;
} while (len);
pmap_update(pmap_kernel());
return 0;
}
/*
* Unmap a previously-mapped user I/O request.
*/
void
vunmapbuf(struct buf *bp, vsize_t len)
{
vaddr_t addr, off;
if ((bp->b_flags & B_PHYS) == 0)
panic("vunmapbuf");
addr = trunc_page((vaddr_t)bp->b_data);
off = (vaddr_t)bp->b_data - addr;
len = round_page(off + len);
pmap_remove(vm_map_pmap(phys_map), addr, addr + len);
pmap_update(vm_map_pmap(phys_map));
uvm_km_free(phys_map, addr, len, UVM_KMF_VAONLY);
bp->b_data = bp->b_saveaddr;
bp->b_saveaddr = NULL;
}
/*
* Map a user I/O request into kernel virtual address space.
*/
int
vmapbuf(struct buf *bp, vsize_t len)
{
vaddr_t kva; /* Kernel VA (new to) */
if ((bp->b_flags & B_PHYS) == 0)
panic("vmapbuf");
vaddr_t uva = mips_trunc_page(bp->b_data);
const vaddr_t off = (vaddr_t)bp->b_data - uva;
len = mips_round_page(off + len);
kva = uvm_km_alloc(phys_map, len, atop(uva) & uvmexp.colormask,
UVM_KMF_VAONLY | UVM_KMF_WAITVA | UVM_KMF_COLORMATCH);
KASSERT((atop(kva ^ uva) & uvmexp.colormask) == 0);
bp->b_saveaddr = bp->b_data;
bp->b_data = (void *)(kva + off);
struct pmap * const upmap = vm_map_pmap(&bp->b_proc->p_vmspace->vm_map);
do {
paddr_t pa; /* physical address */
if (pmap_extract(upmap, uva, &pa) == false)
panic("vmapbuf: null page frame");
pmap_kenter_pa(kva, pa, VM_PROT_READ | VM_PROT_WRITE,
PMAP_WIRED);
uva += PAGE_SIZE;
kva += PAGE_SIZE;
len -= PAGE_SIZE;
} while (len);
pmap_update(pmap_kernel());
return 0;
}
/*
* Map a user I/O request into kernel virtual address space.
* Note: the pages are already locked by uvm_vslock(), so we
* do not need to pass an access_type to pmap_enter().
*/
void
vmapbuf(struct buf *bp, vsize_t len)
{
vaddr_t faddr, taddr, off;
paddr_t fpa;
if ((bp->b_flags & B_PHYS) == 0)
panic("vmapbuf");
faddr = trunc_page((vaddr_t)(bp->b_saveaddr = bp->b_data));
off = (vaddr_t)bp->b_data - faddr;
len = round_page(off + len);
taddr= uvm_km_valloc_wait(phys_map, len);
bp->b_data = (caddr_t)(taddr + off);
/*
* The region is locked, so we expect that pmap_pte() will return
* non-NULL.
* XXX: unwise to expect this in a multithreaded environment.
* anything can happen to a pmap between the time we lock a
* region, release the pmap lock, and then relock it for
* the pmap_extract().
*
* no need to flush TLB since we expect nothing to be mapped
* where we we just allocated (TLB will be flushed when our
* mapping is removed).
*/
while (len) {
(void) pmap_extract(vm_map_pmap(&bp->b_proc->p_vmspace->vm_map),
faddr, &fpa);
pmap_kenter_pa(taddr, fpa, PROT_READ | PROT_WRITE);
faddr += PAGE_SIZE;
taddr += PAGE_SIZE;
len -= PAGE_SIZE;
}
}
/*
* uvm_km_suballoc: allocate a submap in the kernel map. once a submap
* is allocated all references to that area of VM must go through it. this
* allows the locking of VAs in kernel_map to be broken up into regions.
*
* => if `fixed' is true, *min specifies where the region described
* by the submap must start
* => if submap is non NULL we use that as the submap, otherwise we
* alloc a new map
*/
struct vm_map *
uvm_km_suballoc(struct vm_map *map, vaddr_t *min, vaddr_t *max, vsize_t size,
int flags, boolean_t fixed, struct vm_map *submap)
{
int mapflags = UVM_FLAG_NOMERGE | (fixed ? UVM_FLAG_FIXED : 0);
size = round_page(size); /* round up to pagesize */
/*
* first allocate a blank spot in the parent map
*/
if (uvm_map(map, min, size, NULL, UVM_UNKNOWN_OFFSET, 0,
UVM_MAPFLAG(UVM_PROT_ALL, UVM_PROT_ALL, UVM_INH_NONE,
UVM_ADV_RANDOM, mapflags)) != 0) {
panic("uvm_km_suballoc: unable to allocate space in parent map");
}
/*
* set VM bounds (min is filled in by uvm_map)
*/
*max = *min + size;
/*
* add references to pmap and create or init the submap
*/
pmap_reference(vm_map_pmap(map));
if (submap == NULL) {
submap = uvm_map_create(vm_map_pmap(map), *min, *max, flags);
if (submap == NULL)
panic("uvm_km_suballoc: unable to create submap");
} else {
uvm_map_setup(submap, *min, *max, flags);
submap->pmap = vm_map_pmap(map);
}
/*
* now let uvm_map_submap plug in it...
*/
if (uvm_map_submap(map, *min, *max, submap) != 0)
panic("uvm_km_suballoc: submap allocation failed");
return(submap);
}
int
mappedcopyout(void *f, void *t, size_t count)
{
void *fromp = f, *top = t;
vaddr_t kva;
paddr_t upa;
size_t len;
int off, alignable;
pmap_t upmap;
#define CADDR2 caddr1
#ifdef DEBUG
if (mappedcopydebug & MDB_COPYOUT)
printf("mappedcopyout(%p, %p, %lu), pid %d\n",
fromp, top, (u_long)count, curproc->p_pid);
mappedcopyoutcount++;
#endif
if (CADDR2 == 0)
CADDR2 = (void *) uvm_km_alloc(kernel_map, PAGE_SIZE, 0,
UVM_KMF_VAONLY);
kva = (vaddr_t) CADDR2;
off = (int)((u_long)top & PAGE_MASK);
alignable = (off == ((u_long)fromp & PAGE_MASK));
upmap = vm_map_pmap(&curproc->p_vmspace->vm_map);
while (count > 0) {
/*
* First access of a page, use subyte to make sure
* page is faulted in and write access allowed.
*/
if (subyte(top, *((char *)fromp)) == -1)
return EFAULT;
/*
* Map in the page and memcpy data out to it
*/
if (pmap_extract(upmap, trunc_page((vaddr_t)top), &upa)
== false)
panic("mappedcopyout: null page frame");
len = min(count, (PAGE_SIZE - off));
pmap_enter(pmap_kernel(), kva, upa,
VM_PROT_READ|VM_PROT_WRITE,
VM_PROT_READ|VM_PROT_WRITE|PMAP_WIRED);
pmap_update(pmap_kernel());
if (len == PAGE_SIZE && alignable && off == 0)
copypage(fromp, (void *)kva);
else
memcpy((void *)(kva + off), fromp, len);
fromp += len;
top += len;
count -= len;
off = 0;
}
pmap_remove(pmap_kernel(), kva, kva + PAGE_SIZE);
pmap_update(pmap_kernel());
return 0;
#undef CADDR2
}
/*
* Free the io map addresses associated with this IO operation.
*/
void
vunmapbuf(struct buf *bp, vsize_t len)
{
vaddr_t addr, off;
#ifdef DIAGNOSTIC
if (!(bp->b_flags & B_PHYS))
panic("vunmapbuf");
#endif
addr = trunc_page((vaddr_t)bp->b_data);
off = (vaddr_t)bp->b_data - addr;
len = round_page(off + len);
pmap_remove(vm_map_pmap(phys_map), addr, addr + len);
pmap_update(vm_map_pmap(phys_map));
uvm_km_free_wakeup(phys_map, addr, len);
bp->b_data = bp->b_saveaddr;
bp->b_saveaddr = 0;
}
void
kmem_io_map_deallocate(
vm_map_t map,
vm_offset_t addr,
vm_size_t size)
{
/*
* Remove the mappings. The pmap_remove is needed.
*/
pmap_remove(vm_map_pmap(map), addr, addr + size);
vm_map_remove(map, addr, addr + size);
}
void
kmem_submap(
vm_map_t map,
vm_map_t parent,
vm_offset_t *min,
vm_offset_t *max,
vm_size_t size,
boolean_t pageable)
{
vm_offset_t addr;
kern_return_t kr;
size = round_page(size);
/*
* Need reference on submap object because it is internal
* to the vm_system. vm_object_enter will never be called
* on it (usual source of reference for vm_map_enter).
*/
vm_object_reference(vm_submap_object);
addr = vm_map_min(parent);
kr = vm_map_enter(parent, &addr, size,
(vm_offset_t) 0, TRUE,
vm_submap_object, (vm_offset_t) 0, FALSE,
VM_PROT_DEFAULT, VM_PROT_ALL, VM_INHERIT_DEFAULT);
if (kr != KERN_SUCCESS)
panic("kmem_submap");
pmap_reference(vm_map_pmap(parent));
vm_map_setup(map, vm_map_pmap(parent), addr, addr + size, pageable);
kr = vm_map_submap(parent, addr, addr + size, map);
if (kr != KERN_SUCCESS)
panic("kmem_submap");
*min = addr;
*max = addr + size;
}
/*
* Define the code needed before returning to user mode, for
* trap and syscall.
*/
void
userret(struct proc *p)
{
int sig;
/* Do any deferred user pmap operations. */
PMAP_USERRET(vm_map_pmap(&p->p_vmspace->vm_map));
/* take pending signals */
while ((sig = CURSIG(p)) != 0)
postsig(sig);
curcpu()->ci_schedstate.spc_curpriority = p->p_priority = p->p_usrpri;
}
/*
* Map an IO request into kernel virtual address space.
*/
void
vmapbuf(struct buf *bp, vsize_t len)
{
vaddr_t uva, kva;
paddr_t pa;
vsize_t size, off;
int npf;
struct proc *p;
struct vm_map *map;
struct pmap *upmap, *kpmap;
#ifdef DIAGNOSTIC
if ((bp->b_flags & B_PHYS) == 0)
panic("vmapbuf");
#endif
p = bp->b_proc;
map = &p->p_vmspace->vm_map;
upmap = vm_map_pmap(map);
kpmap = vm_map_pmap(phys_map);
bp->b_saveaddr = bp->b_data;
uva = trunc_page((vaddr_t)bp->b_data);
off = (vaddr_t)bp->b_data - uva;
size = round_page(off + len);
kva = uvm_km_valloc_prefer_wait(phys_map, size, uva);
bp->b_data = (caddr_t)(kva + off);
npf = btoc(size);
while (npf--) {
if (pmap_extract(upmap, uva, &pa) == FALSE)
panic("vmapbuf: null page frame");
pmap_enter(kpmap, kva, pa,
VM_PROT_READ | VM_PROT_WRITE, PMAP_WIRED);
uva += PAGE_SIZE;
kva += PAGE_SIZE;
}
pmap_update(kpmap);
}
/*
* Map a user I/O request into kernel virtual address space.
* Note: the pages are already locked by uvm_vslock(), so we
* do not need to pass an access_type to pmap_enter().
*/
int
vmapbuf(struct buf *bp, vsize_t len)
{
struct pmap *upmap, *kpmap __unused;
vaddr_t uva; /* User VA (map from) */
vaddr_t kva; /* Kernel VA (new to) */
paddr_t pa; /* physical address */
vsize_t off;
if ((bp->b_flags & B_PHYS) == 0)
panic("vmapbuf");
uva = m68k_trunc_page(bp->b_saveaddr = bp->b_data);
off = (vaddr_t)bp->b_data - uva;
len = m68k_round_page(off + len);
kva = uvm_km_alloc(phys_map, len, 0, UVM_KMF_VAONLY | UVM_KMF_WAITVA);
bp->b_data = (void *)(kva + off);
upmap = vm_map_pmap(&bp->b_proc->p_vmspace->vm_map);
kpmap = vm_map_pmap(phys_map);
do {
if (pmap_extract(upmap, uva, &pa) == false)
panic("vmapbuf: null page frame");
#ifdef M68K_VAC
pmap_enter(kpmap, kva, pa, VM_PROT_READ | VM_PROT_WRITE,
PMAP_WIRED);
#else
pmap_kenter_pa(kva, pa, VM_PROT_READ | VM_PROT_WRITE, 0);
#endif
uva += PAGE_SIZE;
kva += PAGE_SIZE;
len -= PAGE_SIZE;
} while (len);
pmap_update(kpmap);
return 0;
}
int copyoutmap(
vm_map_t map,
char *fromaddr,
char *toaddr,
int length)
{
if (vm_map_pmap(map) == kernel_pmap) {
/* assume a correct copy */
memcpy(toaddr, fromaddr, length);
return 0;
}
if (current_map() == map)
return copyout(fromaddr, toaddr, length);
return 1;
}
int
sys_mlock(struct lwp *l, const struct sys_mlock_args *uap, register_t *retval)
{
/* {
syscallarg(const void *) addr;
syscallarg(size_t) len;
} */
struct proc *p = l->l_proc;
vaddr_t addr;
vsize_t size, pageoff;
int error;
/*
* extract syscall args from uap
*/
addr = (vaddr_t)SCARG(uap, addr);
size = (vsize_t)SCARG(uap, len);
/*
* align the address to a page boundary and adjust the size accordingly
*/
pageoff = (addr & PAGE_MASK);
addr -= pageoff;
size += pageoff;
size = (vsize_t)round_page(size);
error = range_test(&p->p_vmspace->vm_map, addr, size, false);
if (error)
return ENOMEM;
if (atop(size) + uvmexp.wired > uvmexp.wiredmax)
return EAGAIN;
if (size + ptoa(pmap_wired_count(vm_map_pmap(&p->p_vmspace->vm_map))) >
p->p_rlimit[RLIMIT_MEMLOCK].rlim_cur)
return EAGAIN;
error = uvm_map_pageable(&p->p_vmspace->vm_map, addr, addr+size, false,
0);
if (error == EFAULT)
error = ENOMEM;
return error;
}
/*
* Free the io map PTEs associated with this IO operation.
* We also invalidate the TLB entries and restore the original b_addr.
*/
void
vunmapbuf(struct buf *bp, vsize_t len)
{
vaddr_t addr, off;
pmap_t kpmap;
if ((bp->b_flags & B_PHYS) == 0)
panic("vunmapbuf");
addr = trunc_page((vaddr_t)bp->b_data);
off = (vaddr_t)bp->b_data - addr;
len = round_page(off + len);
kpmap = vm_map_pmap(phys_map);
pmap_remove(kpmap, addr, addr + len);
pmap_update(kpmap);
uvm_km_free_wakeup(phys_map, addr, len);
bp->b_data = bp->b_saveaddr;
bp->b_saveaddr = 0;
}
/*
* mlock system call handler
*
* mlock_args(const void *addr, size_t len)
*
* No requirements
*/
int
sys_mlock(struct mlock_args *uap)
{
vm_offset_t addr;
vm_offset_t tmpaddr;
vm_size_t size, pageoff;
struct thread *td = curthread;
struct proc *p = td->td_proc;
int error;
addr = (vm_offset_t) uap->addr;
size = uap->len;
pageoff = (addr & PAGE_MASK);
addr -= pageoff;
size += pageoff;
size = (vm_size_t) round_page(size);
if (size < uap->len) /* wrap */
return(EINVAL);
tmpaddr = addr + size; /* workaround gcc4 opt */
if (tmpaddr < addr) /* wrap */
return (EINVAL);
if (atop(size) + vmstats.v_wire_count > vm_page_max_wired)
return (EAGAIN);
/*
* We do not need to synchronize against other threads updating ucred;
* they update p->ucred, which is synchronized into td_ucred ourselves.
*/
#ifdef pmap_wired_count
if (size + ptoa(pmap_wired_count(vm_map_pmap(&p->p_vmspace->vm_map))) >
p->p_rlimit[RLIMIT_MEMLOCK].rlim_cur) {
return (ENOMEM);
}
#else
error = priv_check_cred(td->td_ucred, PRIV_ROOT, 0);
if (error) {
return (error);
}
#endif
error = vm_map_unwire(&p->p_vmspace->vm_map, addr, addr + size, FALSE);
return (error == KERN_SUCCESS ? 0 : ENOMEM);
}
/*
* Map a user I/O request into kernel virtual address space.
* Note: the pages are already locked by uvm_vslock(), so we
* do not need to pass an access_type to pmap_enter().
*/
int
vmapbuf(struct buf *bp, vsize_t len)
{
vaddr_t faddr, taddr, off;
paddr_t fpa;
#ifdef PMAP_DEBUG
if (pmap_debug_level > 0)
printf("vmapbuf: bp=%08x buf=%08x len=%08x\n", (u_int)bp,
(u_int)bp->b_data, (u_int)len);
#endif /* PMAP_DEBUG */
if ((bp->b_flags & B_PHYS) == 0)
panic("vmapbuf");
bp->b_saveaddr = bp->b_data;
faddr = trunc_page((vaddr_t)bp->b_data);
off = (vaddr_t)bp->b_data - faddr;
len = round_page(off + len);
taddr = uvm_km_alloc(phys_map, len, atop(faddr) & uvmexp.colormask,
UVM_KMF_VAONLY | UVM_KMF_WAITVA | UVM_KMF_COLORMATCH);
bp->b_data = (void *)(taddr + off);
/*
* The region is locked, so we expect that pmap_pte() will return
* non-NULL.
*/
while (len) {
(void) pmap_extract(vm_map_pmap(&bp->b_proc->p_vmspace->vm_map),
faddr, &fpa);
pmap_enter(pmap_kernel(), taddr, fpa,
VM_PROT_READ|VM_PROT_WRITE, VM_PROT_READ|VM_PROT_WRITE|PMAP_WIRED);
faddr += PAGE_SIZE;
taddr += PAGE_SIZE;
len -= PAGE_SIZE;
}
pmap_update(pmap_kernel());
return 0;
}
/*
* Map a range of user addresses into the kernel.
*/
vaddr_t
vmaprange(struct proc *p, vaddr_t uaddr, vsize_t len, int prot)
{
vaddr_t faddr, taddr, kaddr;
vsize_t off;
paddr_t pa;
faddr = trunc_page(uaddr);
off = uaddr - faddr;
len = round_page(off + len);
taddr = uvm_km_alloc(phys_map, len, 0, UVM_KMF_VAONLY | UVM_KMF_WAITVA);
kaddr = taddr + off;
for (; len > 0; len -= PAGE_SIZE) {
(void) pmap_extract(vm_map_pmap(&p->p_vmspace->vm_map),
faddr, &pa);
pmap_kenter_pa(taddr, pa, prot, 0);
faddr += PAGE_SIZE;
taddr += PAGE_SIZE;
}
return (kaddr);
}
/*
* Unmap IO request from the kernel virtual address space.
*/
void
vunmapbuf(struct buf *bp, vsize_t len)
{
struct pmap *pmap;
vaddr_t kva;
vsize_t off;
#ifdef DIAGNOSTIC
if ((bp->b_flags & B_PHYS) == 0)
panic("vunmapbuf");
#endif
kva = trunc_page((vaddr_t)bp->b_data);
off = (vaddr_t)bp->b_data - kva;
len = round_page(off + len);
pmap = vm_map_pmap(phys_map);
pmap_remove(pmap, kva, kva + len);
pmap_update(pmap);
uvm_km_free(phys_map, kva, len, UVM_KMF_VAONLY);
bp->b_data = bp->b_saveaddr;
bp->b_saveaddr = NULL;
}
int
sys_mlock(struct proc *p, void *v, register_t *retval)
{
struct sys_mlock_args /* {
syscallarg(const void *) addr;
syscallarg(size_t) len;
} */ *uap = v;
vaddr_t addr;
vsize_t size, pageoff;
int error;
/*
* extract syscall args from uap
*/
addr = (vaddr_t)SCARG(uap, addr);
size = (vsize_t)SCARG(uap, len);
/*
* align the address to a page boundary and adjust the size accordingly
*/
ALIGN_ADDR(addr, size, pageoff);
if (addr > SIZE_MAX - size)
return (EINVAL); /* disallow wrap-around. */
if (atop(size) + uvmexp.wired > uvmexp.wiredmax)
return (EAGAIN);
#ifdef pmap_wired_count
if (size + ptoa(pmap_wired_count(vm_map_pmap(&p->p_vmspace->vm_map))) >
p->p_rlimit[RLIMIT_MEMLOCK].rlim_cur)
return (EAGAIN);
#else
if ((error = suser(p, 0)) != 0)
return (error);
#endif
error = uvm_map_pageable(&p->p_vmspace->vm_map, addr, addr+size, FALSE,
0);
return (error == 0 ? 0 : ENOMEM);
}
/*
* Unmap a previously-mapped user I/O request.
*/
void
vunmapbuf(struct buf *bp, vsize_t len)
{
vaddr_t kva;
vsize_t off;
if ((bp->b_flags & B_PHYS) == 0)
panic("vunmapbuf");
kva = m68k_trunc_page(bp->b_data);
off = (vaddr_t)bp->b_data - kva;
len = m68k_round_page(off + len);
#ifdef M68K_VAC
pmap_remove(vm_map_pmap(phys_map), kva, kva + len);
#else
pmap_kremove(kva, len);
#endif
pmap_update(pmap_kernel());
uvm_km_free(phys_map, kva, len, UVM_KMF_VAONLY);
bp->b_data = bp->b_saveaddr;
bp->b_saveaddr = 0;
}
/*
* kmem_suballoc:
*
* Allocates a map to manage a subrange
* of the kernel virtual address space.
*
* Arguments are as follows:
*
* parent Map to take range from
* min, max Returned endpoints of map
* size Size of range to find
* superpage_align Request that min is superpage aligned
*/
vm_map_t
kmem_suballoc(vm_map_t parent, vm_offset_t *min, vm_offset_t *max,
vm_size_t size, boolean_t superpage_align)
{
int ret;
vm_map_t result;
size = round_page(size);
*min = vm_map_min(parent);
ret = vm_map_find(parent, NULL, 0, min, size, 0, superpage_align ?
VMFS_SUPER_SPACE : VMFS_ANY_SPACE, VM_PROT_ALL, VM_PROT_ALL,
MAP_ACC_NO_CHARGE);
if (ret != KERN_SUCCESS)
panic("kmem_suballoc: bad status return of %d", ret);
*max = *min + size;
result = vm_map_create(vm_map_pmap(parent), *min, *max);
if (result == NULL)
panic("kmem_suballoc: cannot create submap");
if (vm_map_submap(parent, *min, *max, result) != KERN_SUCCESS)
panic("kmem_suballoc: unable to change range to submap");
return (result);
}
vaddr_t
uvm_km_valloc_prefer_wait(struct vm_map *map, vsize_t size, voff_t prefer)
{
vaddr_t kva;
UVMHIST_FUNC("uvm_km_valloc_prefer_wait"); UVMHIST_CALLED(maphist);
UVMHIST_LOG(maphist, "(map=%p, size=0x%lx)", map, size, 0,0);
KASSERT(vm_map_pmap(map) == pmap_kernel());
size = round_page(size);
if (size > vm_map_max(map) - vm_map_min(map))
return(0);
while (1) {
kva = vm_map_min(map); /* hint */
/*
* allocate some virtual space. will be demand filled
* by kernel_object.
*/
if (__predict_true(uvm_map(map, &kva, size, uvm.kernel_object,
prefer, 0, UVM_MAPFLAG(UVM_PROT_ALL,
UVM_PROT_ALL, UVM_INH_NONE, UVM_ADV_RANDOM, 0)) == 0)) {
UVMHIST_LOG(maphist,"<- done (kva=0x%lx)", kva,0,0,0);
return(kva);
}
/*
* failed. sleep for a while (on map)
*/
UVMHIST_LOG(maphist,"<<<sleeping>>>",0,0,0,0);
tsleep((caddr_t)map, PVM, "vallocwait", 0);
}
/*NOTREACHED*/
}
/*
* Map a user I/O request into kernel virtual address space.
* Note: these pages have already been locked by uvm_vslock.
*/
int
vmapbuf(struct buf *bp, vsize_t len)
{
vaddr_t faddr, taddr;
vsize_t off;
paddr_t pa;
int prot = VM_PROT_READ | ((bp->b_flags & B_READ) ? VM_PROT_WRITE : 0);
#ifdef DIAGNOSTIC
if (!(bp->b_flags & B_PHYS))
panic("vmapbuf");
#endif
/*
* XXX Reimplement this with vmaprange (on at least PPC_IBM4XX CPUs).
*/
bp->b_saveaddr = bp->b_data;
faddr = trunc_page((vaddr_t)bp->b_saveaddr);
off = (vaddr_t)bp->b_data - faddr;
len = round_page(off + len);
taddr = uvm_km_alloc(phys_map, len, 0, UVM_KMF_VAONLY | UVM_KMF_WAITVA);
bp->b_data = (void *)(taddr + off);
for (; len > 0; len -= PAGE_SIZE) {
(void) pmap_extract(vm_map_pmap(&bp->b_proc->p_vmspace->vm_map),
faddr, &pa);
/*
* Use pmap_enter so the referenced and modified bits are
* appropriately set.
*/
pmap_kenter_pa(taddr, pa, prot, 0);
faddr += PAGE_SIZE;
taddr += PAGE_SIZE;
}
pmap_update(pmap_kernel());
return 0;
}