/* --------------------------------------------------------------------- */
static int
tmpfs_nocacheread(vm_object_t tobj, vm_pindex_t idx,
vm_offset_t offset, size_t tlen, struct uio *uio)
{
vm_page_t m;
int error, rv;
VM_OBJECT_LOCK(tobj);
m = vm_page_grab(tobj, idx, VM_ALLOC_WIRED |
VM_ALLOC_NORMAL | VM_ALLOC_RETRY);
if (m->valid != VM_PAGE_BITS_ALL) {
if (vm_pager_has_page(tobj, idx, NULL, NULL)) {
rv = vm_pager_get_pages(tobj, &m, 1, 0);
if (rv != VM_PAGER_OK) {
vm_page_lock(m);
vm_page_free(m);
vm_page_unlock(m);
VM_OBJECT_UNLOCK(tobj);
return (EIO);
}
} else
vm_page_zero_invalid(m, TRUE);
}
VM_OBJECT_UNLOCK(tobj);
error = uiomove_fromphys(&m, offset, tlen, uio);
VM_OBJECT_LOCK(tobj);
vm_page_lock(m);
vm_page_unwire(m, TRUE);
vm_page_unlock(m);
vm_page_wakeup(m);
VM_OBJECT_UNLOCK(tobj);
return (error);
}
static vm_page_t
efi_1t1_page(void)
{
return (vm_page_grab(obj_1t1_pt, efi_1t1_idx++, VM_ALLOC_NOBUSY |
VM_ALLOC_WIRED | VM_ALLOC_ZERO));
}
/* --------------------------------------------------------------------- */
static int
tmpfs_nocacheread(vm_object_t tobj, vm_pindex_t idx,
vm_offset_t offset, size_t tlen, struct uio *uio)
{
vm_page_t m;
int error;
VM_OBJECT_LOCK(tobj);
vm_object_pip_add(tobj, 1);
m = vm_page_grab(tobj, idx, VM_ALLOC_WIRED |
VM_ALLOC_ZERO | VM_ALLOC_NORMAL | VM_ALLOC_RETRY);
if (m->valid != VM_PAGE_BITS_ALL) {
if (vm_pager_has_page(tobj, idx, NULL, NULL)) {
error = vm_pager_get_pages(tobj, &m, 1, 0);
if (error != 0) {
printf("tmpfs get pages from pager error [read]\n");
goto out;
}
} else
vm_page_zero_invalid(m, TRUE);
}
VM_OBJECT_UNLOCK(tobj);
error = uiomove_fromphys(&m, offset, tlen, uio);
VM_OBJECT_LOCK(tobj);
out:
vm_page_lock(m);
vm_page_unwire(m, TRUE);
vm_page_unlock(m);
vm_page_wakeup(m);
vm_object_pip_subtract(tobj, 1);
VM_OBJECT_UNLOCK(tobj);
return (error);
}
vm_page_t
shmem_read_mapping_page(vm_object_t object, vm_pindex_t pindex)
{
vm_page_t m;
int rv;
VM_OBJECT_LOCK_ASSERT_OWNED(object);
m = vm_page_grab(object, pindex, VM_ALLOC_NORMAL | VM_ALLOC_RETRY);
if (m->valid != VM_PAGE_BITS_ALL) {
if (vm_pager_has_page(object, pindex)) {
rv = vm_pager_get_page(object, &m, 1);
m = vm_page_lookup(object, pindex);
if (m == NULL)
return ERR_PTR(-ENOMEM);
if (rv != VM_PAGER_OK) {
vm_page_free(m);
return ERR_PTR(-ENOMEM);
}
} else {
pmap_zero_page(VM_PAGE_TO_PHYS(m));
m->valid = VM_PAGE_BITS_ALL;
m->dirty = 0;
}
}
vm_page_wire(m);
vm_page_wakeup(m);
return (m);
}
static vm_page_t
efi_1t1_page(vm_pindex_t idx)
{
return (vm_page_grab(obj_1t1_pt, idx, VM_ALLOC_NOBUSY |
VM_ALLOC_WIRED | VM_ALLOC_ZERO));
}
void
himem_reserve(
int npages)
{
register int i = 0;
vm_page_t free_head = VM_PAGE_NULL;
vm_page_t low;
hil_t hil;
spl_t ipl;
extern vm_offset_t avail_end;
if (avail_end <= HIGH_MEM)
return;
hil = (hil_t)kalloc(npages*sizeof(struct himem_link));
if (hil == (hil_t)0)
panic("himem_reserve: kalloc failed\n");
for (i=0; i < npages-1; i++)
(hil+i)->next = hil+i+1;
/*
* This is the only way of getting low physical pages
* wtithout changing VM internals
*/
for (i=0; i != npages;) {
if ((low = vm_page_grab()) == VM_PAGE_NULL)
panic("No low memory pages for himem\n");
vm_page_gobble(low); /* mark as consumed internally */
if (_high_mem_page(low->phys_addr)) {
low->pageq.next = (queue_entry_t)free_head;
free_head = low;
} else {
(hil+i)->low_page = low->phys_addr;
i++;
}
}
for (low = free_head; low; low = free_head) {
free_head = (vm_page_t) low->pageq.next;
VM_PAGE_FREE(low);
}
ipl = splhi();
simple_lock(&hil_lock);
(hil+npages-1)->next = hil_head;
hil_head = hil;
simple_unlock(&hil_lock);
splx(ipl);
}
int ttm_tt_swapin(struct ttm_tt *ttm)
{
vm_object_t obj;
vm_page_t from_page, to_page;
int i, ret, rv;
obj = ttm->swap_storage;
VM_OBJECT_LOCK(obj);
vm_object_pip_add(obj, 1);
for (i = 0; i < ttm->num_pages; ++i) {
from_page = vm_page_grab(obj, i, VM_ALLOC_NORMAL |
VM_ALLOC_RETRY);
if (from_page->valid != VM_PAGE_BITS_ALL) {
if (vm_pager_has_page(obj, i)) {
rv = vm_pager_get_page(obj, &from_page, 1);
if (rv != VM_PAGER_OK) {
vm_page_free(from_page);
ret = -EIO;
goto err_ret;
}
} else {
vm_page_zero_invalid(from_page, TRUE);
}
}
to_page = ttm->pages[i];
if (unlikely(to_page == NULL)) {
ret = -ENOMEM;
vm_page_wakeup(from_page);
goto err_ret;
}
pmap_copy_page(VM_PAGE_TO_PHYS(from_page),
VM_PAGE_TO_PHYS(to_page));
vm_page_wakeup(from_page);
}
vm_object_pip_wakeup(obj);
VM_OBJECT_UNLOCK(obj);
if (!(ttm->page_flags & TTM_PAGE_FLAG_PERSISTENT_SWAP))
vm_object_deallocate(obj);
ttm->swap_storage = NULL;
ttm->page_flags &= ~TTM_PAGE_FLAG_SWAPPED;
return (0);
err_ret:
vm_object_pip_wakeup(obj);
VM_OBJECT_UNLOCK(obj);
return (ret);
}
static int
spigen_mmap_single(struct cdev *cdev, vm_ooffset_t *offset,
vm_size_t size, struct vm_object **object, int nprot)
{
device_t dev = cdev->si_drv1;
struct spigen_softc *sc = device_get_softc(dev);
vm_page_t *m;
size_t n, pages;
if (size == 0 ||
(nprot & (PROT_EXEC | PROT_READ | PROT_WRITE))
!= (PROT_READ | PROT_WRITE))
return (EINVAL);
size = roundup2(size, PAGE_SIZE);
pages = size / PAGE_SIZE;
mtx_lock(&sc->sc_mtx);
if (sc->sc_mmap_buffer != NULL) {
mtx_unlock(&sc->sc_mtx);
return (EBUSY);
} else if (size > sc->sc_command_length_max + sc->sc_data_length_max) {
mtx_unlock(&sc->sc_mtx);
return (E2BIG);
}
sc->sc_mmap_buffer_size = size;
*offset = 0;
sc->sc_mmap_buffer = *object = vm_pager_allocate(OBJT_PHYS, 0, size,
nprot, *offset, curthread->td_ucred);
m = malloc(sizeof(*m) * pages, M_TEMP, M_WAITOK);
VM_OBJECT_WLOCK(*object);
vm_object_reference_locked(*object); // kernel and userland both
for (n = 0; n < pages; n++) {
m[n] = vm_page_grab(*object, n,
VM_ALLOC_NOBUSY | VM_ALLOC_ZERO | VM_ALLOC_WIRED);
m[n]->valid = VM_PAGE_BITS_ALL;
}
VM_OBJECT_WUNLOCK(*object);
sc->sc_mmap_kvaddr = kva_alloc(size);
pmap_qenter(sc->sc_mmap_kvaddr, m, pages);
free(m, M_TEMP);
mtx_unlock(&sc->sc_mtx);
if (*object == NULL)
return (EINVAL);
return (0);
}
int ttm_tt_swapout(struct ttm_tt *ttm, vm_object_t persistent_swap_storage)
{
vm_object_t obj;
vm_page_t from_page, to_page;
int i;
BUG_ON(ttm->state != tt_unbound && ttm->state != tt_unpopulated);
BUG_ON(ttm->caching_state != tt_cached);
if (!persistent_swap_storage) {
obj = swap_pager_alloc(NULL,
IDX_TO_OFF(ttm->num_pages), VM_PROT_DEFAULT, 0);
if (obj == NULL) {
pr_err("Failed allocating swap storage\n");
return (-ENOMEM);
}
} else
obj = persistent_swap_storage;
VM_OBJECT_LOCK(obj);
vm_object_pip_add(obj, 1);
for (i = 0; i < ttm->num_pages; ++i) {
from_page = ttm->pages[i];
if (unlikely(from_page == NULL))
continue;
to_page = vm_page_grab(obj, i, VM_ALLOC_NORMAL |
VM_ALLOC_RETRY);
pmap_copy_page(VM_PAGE_TO_PHYS(from_page),
VM_PAGE_TO_PHYS(to_page));
to_page->valid = VM_PAGE_BITS_ALL;
vm_page_dirty(to_page);
vm_page_wakeup(to_page);
}
vm_object_pip_wakeup(obj);
VM_OBJECT_UNLOCK(obj);
ttm->bdev->driver->ttm_tt_unpopulate(ttm);
ttm->swap_storage = obj;
ttm->page_flags |= TTM_PAGE_FLAG_SWAPPED;
if (persistent_swap_storage)
ttm->page_flags |= TTM_PAGE_FLAG_PERSISTENT_SWAP;
return 0;
}
int ttm_tt_swapout(struct ttm_tt *ttm, vm_object_t persistent_swap_storage)
{
vm_object_t obj;
vm_page_t from_page, to_page;
int i;
MPASS(ttm->state == tt_unbound || ttm->state == tt_unpopulated);
MPASS(ttm->caching_state == tt_cached);
if (persistent_swap_storage == NULL) {
obj = vm_pager_allocate(OBJT_SWAP, NULL,
IDX_TO_OFF(ttm->num_pages), VM_PROT_DEFAULT, 0,
curthread->td_ucred);
if (obj == NULL) {
printf("[TTM] Failed allocating swap storage\n");
return (-ENOMEM);
}
} else
obj = persistent_swap_storage;
VM_OBJECT_WLOCK(obj);
vm_object_pip_add(obj, 1);
for (i = 0; i < ttm->num_pages; ++i) {
from_page = ttm->pages[i];
if (unlikely(from_page == NULL))
continue;
to_page = vm_page_grab(obj, i, VM_ALLOC_NORMAL);
pmap_copy_page(from_page, to_page);
to_page->valid = VM_PAGE_BITS_ALL;
vm_page_dirty(to_page);
vm_page_xunbusy(to_page);
}
vm_object_pip_wakeup(obj);
VM_OBJECT_WUNLOCK(obj);
ttm->bdev->driver->ttm_tt_unpopulate(ttm);
ttm->swap_storage = obj;
ttm->page_flags |= TTM_PAGE_FLAG_SWAPPED;
if (persistent_swap_storage != NULL)
ttm->page_flags |= TTM_PAGE_FLAG_PERSISTENT_SWAP;
return (0);
}
int
exec_map_first_page(struct image_params *imgp)
{
int rv, i, after, initial_pagein;
vm_page_t ma[VM_INITIAL_PAGEIN];
vm_object_t object;
if (imgp->firstpage != NULL)
exec_unmap_first_page(imgp);
object = imgp->vp->v_object;
if (object == NULL)
return (EACCES);
VM_OBJECT_WLOCK(object);
#if VM_NRESERVLEVEL > 0
vm_object_color(object, 0);
#endif
ma[0] = vm_page_grab(object, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOBUSY);
if (ma[0]->valid != VM_PAGE_BITS_ALL) {
vm_page_xbusy(ma[0]);
if (!vm_pager_has_page(object, 0, NULL, &after)) {
vm_page_lock(ma[0]);
vm_page_free(ma[0]);
vm_page_unlock(ma[0]);
VM_OBJECT_WUNLOCK(object);
return (EIO);
}
initial_pagein = min(after, VM_INITIAL_PAGEIN);
KASSERT(initial_pagein <= object->size,
("%s: initial_pagein %d object->size %ju",
__func__, initial_pagein, (uintmax_t )object->size));
for (i = 1; i < initial_pagein; i++) {
if ((ma[i] = vm_page_next(ma[i - 1])) != NULL) {
if (ma[i]->valid)
break;
if (!vm_page_tryxbusy(ma[i]))
break;
} else {
ma[i] = vm_page_alloc(object, i,
VM_ALLOC_NORMAL);
if (ma[i] == NULL)
break;
}
}
initial_pagein = i;
rv = vm_pager_get_pages(object, ma, initial_pagein, NULL, NULL);
if (rv != VM_PAGER_OK) {
for (i = 0; i < initial_pagein; i++) {
vm_page_lock(ma[i]);
vm_page_free(ma[i]);
vm_page_unlock(ma[i]);
}
VM_OBJECT_WUNLOCK(object);
return (EIO);
}
vm_page_xunbusy(ma[0]);
for (i = 1; i < initial_pagein; i++)
vm_page_readahead_finish(ma[i]);
}
vm_page_lock(ma[0]);
vm_page_hold(ma[0]);
vm_page_activate(ma[0]);
vm_page_unlock(ma[0]);
VM_OBJECT_WUNLOCK(object);
imgp->firstpage = sf_buf_alloc(ma[0], 0);
imgp->image_header = (char *)sf_buf_kva(imgp->firstpage);
return (0);
}
static int
tmpfs_mappedwrite(vm_object_t vobj, vm_object_t tobj, size_t len, struct uio *uio)
{
vm_pindex_t idx;
vm_page_t vpg, tpg;
vm_offset_t offset;
off_t addr;
size_t tlen;
int error, rv;
error = 0;
addr = uio->uio_offset;
idx = OFF_TO_IDX(addr);
offset = addr & PAGE_MASK;
tlen = MIN(PAGE_SIZE - offset, len);
if ((vobj == NULL) ||
(vobj->resident_page_count == 0 && vobj->cache == NULL)) {
vpg = NULL;
goto nocache;
}
VM_OBJECT_LOCK(vobj);
lookupvpg:
if (((vpg = vm_page_lookup(vobj, idx)) != NULL) &&
vm_page_is_valid(vpg, offset, tlen)) {
if ((vpg->oflags & VPO_BUSY) != 0) {
/*
* Reference the page before unlocking and sleeping so
* that the page daemon is less likely to reclaim it.
*/
vm_page_reference(vpg);
vm_page_sleep(vpg, "tmfsmw");
goto lookupvpg;
}
vm_page_busy(vpg);
vm_page_undirty(vpg);
VM_OBJECT_UNLOCK(vobj);
error = uiomove_fromphys(&vpg, offset, tlen, uio);
} else {
if (__predict_false(vobj->cache != NULL))
vm_page_cache_free(vobj, idx, idx + 1);
VM_OBJECT_UNLOCK(vobj);
vpg = NULL;
}
nocache:
VM_OBJECT_LOCK(tobj);
tpg = vm_page_grab(tobj, idx, VM_ALLOC_WIRED |
VM_ALLOC_NORMAL | VM_ALLOC_RETRY);
if (tpg->valid != VM_PAGE_BITS_ALL) {
if (vm_pager_has_page(tobj, idx, NULL, NULL)) {
rv = vm_pager_get_pages(tobj, &tpg, 1, 0);
if (rv != VM_PAGER_OK) {
vm_page_lock(tpg);
vm_page_free(tpg);
vm_page_unlock(tpg);
error = EIO;
goto out;
}
} else
vm_page_zero_invalid(tpg, TRUE);
}
VM_OBJECT_UNLOCK(tobj);
if (vpg == NULL)
error = uiomove_fromphys(&tpg, offset, tlen, uio);
else {
KASSERT(vpg->valid == VM_PAGE_BITS_ALL, ("parts of vpg invalid"));
pmap_copy_page(vpg, tpg);
}
VM_OBJECT_LOCK(tobj);
if (error == 0) {
KASSERT(tpg->valid == VM_PAGE_BITS_ALL,
("parts of tpg invalid"));
vm_page_dirty(tpg);
}
vm_page_lock(tpg);
vm_page_unwire(tpg, TRUE);
vm_page_unlock(tpg);
vm_page_wakeup(tpg);
out:
VM_OBJECT_UNLOCK(tobj);
if (vpg != NULL) {
VM_OBJECT_LOCK(vobj);
vm_page_wakeup(vpg);
VM_OBJECT_UNLOCK(vobj);
}
return (error);
}
static int
uiomove_object_page(vm_object_t obj, size_t len, struct uio *uio)
{
vm_page_t m;
vm_pindex_t idx;
size_t tlen;
int error, offset, rv;
idx = OFF_TO_IDX(uio->uio_offset);
offset = uio->uio_offset & PAGE_MASK;
tlen = MIN(PAGE_SIZE - offset, len);
VM_OBJECT_WLOCK(obj);
/*
* Parallel reads of the page content from disk are prevented
* by exclusive busy.
*
* Although the tmpfs vnode lock is held here, it is
* nonetheless safe to sleep waiting for a free page. The
* pageout daemon does not need to acquire the tmpfs vnode
* lock to page out tobj's pages because tobj is a OBJT_SWAP
* type object.
*/
m = vm_page_grab(obj, idx, VM_ALLOC_NORMAL);
if (m->valid != VM_PAGE_BITS_ALL) {
if (vm_pager_has_page(obj, idx, NULL, NULL)) {
rv = vm_pager_get_pages(obj, &m, 1, 0);
m = vm_page_lookup(obj, idx);
if (m == NULL) {
printf(
"uiomove_object: vm_obj %p idx %jd null lookup rv %d\n",
obj, idx, rv);
VM_OBJECT_WUNLOCK(obj);
return (EIO);
}
if (rv != VM_PAGER_OK) {
printf(
"uiomove_object: vm_obj %p idx %jd valid %x pager error %d\n",
obj, idx, m->valid, rv);
vm_page_lock(m);
vm_page_free(m);
vm_page_unlock(m);
VM_OBJECT_WUNLOCK(obj);
return (EIO);
}
} else
vm_page_zero_invalid(m, TRUE);
}
vm_page_xunbusy(m);
vm_page_lock(m);
vm_page_hold(m);
vm_page_unlock(m);
VM_OBJECT_WUNLOCK(obj);
error = uiomove_fromphys(&m, offset, tlen, uio);
if (uio->uio_rw == UIO_WRITE && error == 0) {
VM_OBJECT_WLOCK(obj);
vm_page_dirty(m);
VM_OBJECT_WUNLOCK(obj);
}
vm_page_lock(m);
vm_page_unhold(m);
if (m->queue == PQ_NONE) {
vm_page_deactivate(m);
} else {
/* Requeue to maintain LRU ordering. */
vm_page_requeue(m);
}
vm_page_unlock(m);
return (error);
}
!vm_pager_has_page(obj, idx, NULL, NULL)) {
VM_OBJECT_WUNLOCK(obj);
return (uiomove(__DECONST(void *, zero_region), tlen, uio));
}
/*
* Parallel reads of the page content from disk are prevented
* by exclusive busy.
*
* Although the tmpfs vnode lock is held here, it is
* nonetheless safe to sleep waiting for a free page. The
* pageout daemon does not need to acquire the tmpfs vnode
* lock to page out tobj's pages because tobj is a OBJT_SWAP
* type object.
*/
m = vm_page_grab(obj, idx, VM_ALLOC_NORMAL);
if (m->valid != VM_PAGE_BITS_ALL) {
if (vm_pager_has_page(obj, idx, NULL, NULL)) {
rv = vm_pager_get_pages(obj, &m, 1, 0);
if (rv != VM_PAGER_OK) {
printf(
"uiomove_object: vm_obj %p idx %jd valid %x pager error %d\n",
obj, idx, m->valid, rv);
vm_page_lock(m);
vm_page_free(m);
vm_page_unlock(m);
VM_OBJECT_WUNLOCK(obj);
return (EIO);
}
} else
vm_page_zero_invalid(m, TRUE);
shared_page_write(res, size, data);
sx_xunlock(&shared_page_alloc_sx);
return (res);
}
static void
shared_page_init(void *dummy __unused)
{
vm_page_t m;
vm_offset_t addr;
sx_init(&shared_page_alloc_sx, "shpsx");
shared_page_obj = vm_pager_allocate(OBJT_PHYS, 0, PAGE_SIZE,
VM_PROT_DEFAULT, 0, NULL);
VM_OBJECT_LOCK(shared_page_obj);
m = vm_page_grab(shared_page_obj, 0, VM_ALLOC_RETRY | VM_ALLOC_NOBUSY |
VM_ALLOC_ZERO);
m->valid = VM_PAGE_BITS_ALL;
VM_OBJECT_UNLOCK(shared_page_obj);
addr = kmem_alloc_nofault(kernel_map, PAGE_SIZE);
pmap_qenter(addr, &m, 1);
shared_page_mapping = (char *)addr;
}
SYSINIT(shp, SI_SUB_EXEC, SI_ORDER_FIRST, (sysinit_cfunc_t)shared_page_init,
NULL);
static void
timehands_update(struct sysentvec *sv)
{
struct vdso_timehands th;
struct vdso_timekeep *tk;
!vm_pager_has_page(obj, idx, NULL, NULL)) {
VM_OBJECT_WUNLOCK(obj);
return (uiomove(__DECONST(void *, zero_region), tlen, uio));
}
/*
* Parallel reads of the page content from disk are prevented
* by exclusive busy.
*
* Although the tmpfs vnode lock is held here, it is
* nonetheless safe to sleep waiting for a free page. The
* pageout daemon does not need to acquire the tmpfs vnode
* lock to page out tobj's pages because tobj is a OBJT_SWAP
* type object.
*/
m = vm_page_grab(obj, idx, VM_ALLOC_NORMAL | VM_ALLOC_NOBUSY);
if (m->valid != VM_PAGE_BITS_ALL) {
vm_page_xbusy(m);
if (vm_pager_has_page(obj, idx, NULL, NULL)) {
rv = vm_pager_get_pages(obj, &m, 1, NULL, NULL);
if (rv != VM_PAGER_OK) {
printf(
"uiomove_object: vm_obj %p idx %jd valid %x pager error %d\n",
obj, idx, m->valid, rv);
vm_page_lock(m);
vm_page_free(m);
vm_page_unlock(m);
VM_OBJECT_WUNLOCK(obj);
return (EIO);
}
} else
kern_return_t
kernel_memory_allocate(
register vm_map_t map,
register vm_offset_t *addrp,
register vm_size_t size,
register vm_offset_t mask,
int flags)
{
vm_object_t object;
vm_object_offset_t offset;
vm_object_offset_t pg_offset;
vm_map_entry_t entry;
vm_map_offset_t map_addr, fill_start;
vm_map_offset_t map_mask;
vm_map_size_t map_size, fill_size;
kern_return_t kr;
vm_page_t mem;
vm_page_t guard_page_list = NULL;
vm_page_t wired_page_list = NULL;
int guard_page_count = 0;
int wired_page_count = 0;
int i;
int vm_alloc_flags;
if (! vm_kernel_ready) {
panic("kernel_memory_allocate: VM is not ready");
}
if (size == 0) {
*addrp = 0;
return KERN_INVALID_ARGUMENT;
}
map_size = vm_map_round_page(size);
map_mask = (vm_map_offset_t) mask;
vm_alloc_flags = 0;
/*
* limit the size of a single extent of wired memory
* to try and limit the damage to the system if
* too many pages get wired down
*/
if (map_size > (1 << 30)) {
return KERN_RESOURCE_SHORTAGE;
}
/*
* Guard pages:
*
* Guard pages are implemented as ficticious pages. By placing guard pages
* on either end of a stack, they can help detect cases where a thread walks
* off either end of its stack. They are allocated and set up here and attempts
* to access those pages are trapped in vm_fault_page().
*
* The map_size we were passed may include extra space for
* guard pages. If those were requested, then back it out of fill_size
* since vm_map_find_space() takes just the actual size not including
* guard pages. Similarly, fill_start indicates where the actual pages
* will begin in the range.
*/
fill_start = 0;
fill_size = map_size;
if (flags & KMA_GUARD_FIRST) {
vm_alloc_flags |= VM_FLAGS_GUARD_BEFORE;
fill_start += PAGE_SIZE_64;
fill_size -= PAGE_SIZE_64;
if (map_size < fill_start + fill_size) {
/* no space for a guard page */
*addrp = 0;
return KERN_INVALID_ARGUMENT;
}
guard_page_count++;
}
if (flags & KMA_GUARD_LAST) {
vm_alloc_flags |= VM_FLAGS_GUARD_AFTER;
fill_size -= PAGE_SIZE_64;
if (map_size <= fill_start + fill_size) {
/* no space for a guard page */
*addrp = 0;
return KERN_INVALID_ARGUMENT;
}
guard_page_count++;
}
wired_page_count = (int) (fill_size / PAGE_SIZE_64);
assert(wired_page_count * PAGE_SIZE_64 == fill_size);
for (i = 0; i < guard_page_count; i++) {
for (;;) {
mem = vm_page_grab_guard();
if (mem != VM_PAGE_NULL)
break;
if (flags & KMA_NOPAGEWAIT) {
kr = KERN_RESOURCE_SHORTAGE;
goto out;
}
vm_page_more_fictitious();
}
mem->pageq.next = (queue_entry_t)guard_page_list;
guard_page_list = mem;
}
for (i = 0; i < wired_page_count; i++) {
uint64_t unavailable;
for (;;) {
if (flags & KMA_LOMEM)
mem = vm_page_grablo();
else
mem = vm_page_grab();
if (mem != VM_PAGE_NULL)
break;
if (flags & KMA_NOPAGEWAIT) {
kr = KERN_RESOURCE_SHORTAGE;
goto out;
}
if ((flags & KMA_LOMEM) && (vm_lopage_needed == TRUE)) {
kr = KERN_RESOURCE_SHORTAGE;
goto out;
}
unavailable = (vm_page_wire_count + vm_page_free_target) * PAGE_SIZE;
if (unavailable > max_mem || map_size > (max_mem - unavailable)) {
kr = KERN_RESOURCE_SHORTAGE;
goto out;
}
VM_PAGE_WAIT();
}
mem->pageq.next = (queue_entry_t)wired_page_list;
wired_page_list = mem;
}
/*
* Allocate a new object (if necessary). We must do this before
* locking the map, or risk deadlock with the default pager.
*/
if ((flags & KMA_KOBJECT) != 0) {
object = kernel_object;
vm_object_reference(object);
} else {
object = vm_object_allocate(map_size);
}
kr = vm_map_find_space(map, &map_addr,
fill_size, map_mask,
vm_alloc_flags, &entry);
if (KERN_SUCCESS != kr) {
vm_object_deallocate(object);
goto out;
}
entry->object.vm_object = object;
entry->offset = offset = (object == kernel_object) ?
map_addr : 0;
entry->wired_count++;
if (flags & KMA_PERMANENT)
entry->permanent = TRUE;
if (object != kernel_object)
vm_object_reference(object);
vm_object_lock(object);
vm_map_unlock(map);
pg_offset = 0;
if (fill_start) {
if (guard_page_list == NULL)
panic("kernel_memory_allocate: guard_page_list == NULL");
mem = guard_page_list;
guard_page_list = (vm_page_t)mem->pageq.next;
mem->pageq.next = NULL;
vm_page_insert(mem, object, offset + pg_offset);
mem->busy = FALSE;
pg_offset += PAGE_SIZE_64;
}
for (pg_offset = fill_start; pg_offset < fill_start + fill_size; pg_offset += PAGE_SIZE_64) {
if (wired_page_list == NULL)
panic("kernel_memory_allocate: wired_page_list == NULL");
mem = wired_page_list;
wired_page_list = (vm_page_t)mem->pageq.next;
mem->pageq.next = NULL;
mem->wire_count++;
vm_page_insert(mem, object, offset + pg_offset);
mem->busy = FALSE;
mem->pmapped = TRUE;
mem->wpmapped = TRUE;
PMAP_ENTER(kernel_pmap, map_addr + pg_offset, mem,
VM_PROT_READ | VM_PROT_WRITE, object->wimg_bits & VM_WIMG_MASK, TRUE);
if (flags & KMA_NOENCRYPT) {
bzero(CAST_DOWN(void *, (map_addr + pg_offset)), PAGE_SIZE);
pmap_set_noencrypt(mem->phys_page);
}
}
