/* Purge VM for a file when its callback is revoked.
*
* Locking: No lock is held, not even the global lock.
*/
void
osi_VM_FlushPages(struct vcache *avc, afs_ucred_t *credp)
{
struct vnode *vp;
struct vm_object *obj;
vp = AFSTOV(avc);
ASSERT_VOP_LOCKED(vp, __func__);
if (VOP_GETVOBJECT(vp, &obj) == 0) {
VM_OBJECT_LOCK(obj);
vm_object_page_remove(obj, 0, 0, FALSE);
VM_OBJECT_UNLOCK(obj);
}
osi_vinvalbuf(vp, 0, 0, 0);
}
/* Purge VM for a file when its callback is revoked.
*
* Locking: No lock is held, not even the global lock.
*/
void
osi_VM_FlushPages(struct vcache *avc, afs_ucred_t *credp)
{
struct vnode *vp;
struct vm_object *obj;
vp = AFSTOV(avc);
ASSERT_VOP_LOCKED(vp, __func__);
obj = vp->v_object;
if (obj != NULL) {
AFS_VM_OBJECT_WLOCK(obj);
vm_object_page_remove(obj, 0, 0, FALSE);
AFS_VM_OBJECT_WUNLOCK(obj);
}
osi_vinvalbuf(vp, 0, 0, 0);
}
/* Try to discard pages, in order to recycle a vcache entry.
*
* We also make some sanity checks: ref count, open count, held locks.
*
* We also do some non-VM-related chores, such as releasing the cred pointer
* (for AIX and Solaris) and releasing the gnode (for AIX).
*
* Locking: afs_xvcache lock is held. If it is dropped and re-acquired,
* *slept should be set to warn the caller.
*
* Formerly, afs_xvcache was dropped and re-acquired for Solaris, but now it
* is not dropped and re-acquired for any platform. It may be that *slept is
* therefore obsolescent.
*
* OSF/1 Locking: VN_LOCK has been called.
* We do not lock the vnode here, but instead require that it be exclusive
* locked by code calling osi_VM_StoreAllSegments directly, or scheduling it
* from the bqueue - Matt
* Maybe better to just call vnode_pager_setsize()?
*/
int
osi_VM_FlushVCache(struct vcache *avc, int *slept)
{
struct vm_object *obj;
struct vnode *vp;
if (VREFCOUNT(avc) > 1)
return EBUSY;
if (avc->opens)
return EBUSY;
/* if a lock is held, give up */
if (CheckLock(&avc->lock))
return EBUSY;
return(0);
AFS_GUNLOCK();
vp = AFSTOV(avc);
#ifndef AFS_FBSD70_ENV
lock_vnode(vp);
#endif
if (VOP_GETVOBJECT(vp, &obj) == 0) {
VM_OBJECT_LOCK(obj);
vm_object_page_remove(obj, 0, 0, FALSE);
#if 1
if (obj->ref_count == 0) {
simple_lock(&vp->v_interlock);
vgonel(vp, curthread);
vp->v_tag = VT_AFS;
SetAfsVnode(vp);
}
#endif
VM_OBJECT_UNLOCK(obj);
}
#ifndef AFS_FBSD70_ENV
unlock_vnode(vp);
#endif
AFS_GLOCK();
return 0;
}
/*
* Lets the VM system know about a change in size for a file.
* We adjust our own internal size and flush any cached pages in
* the associated object that are affected by the size change.
*
* Note: this routine may be invoked as a result of a pager put
* operation (possibly at object termination time), so we must be careful.
*/
void
vnode_pager_setsize(struct vnode *vp, vm_ooffset_t nsize)
{
vm_object_t object;
vm_page_t m;
vm_pindex_t nobjsize;
if ((object = vp->v_object) == NULL)
return;
/* ASSERT_VOP_ELOCKED(vp, "vnode_pager_setsize and not locked vnode"); */
VM_OBJECT_WLOCK(object);
if (object->type == OBJT_DEAD) {
VM_OBJECT_WUNLOCK(object);
return;
}
KASSERT(object->type == OBJT_VNODE,
("not vnode-backed object %p", object));
if (nsize == object->un_pager.vnp.vnp_size) {
/*
* Hasn't changed size
*/
VM_OBJECT_WUNLOCK(object);
return;
}
nobjsize = OFF_TO_IDX(nsize + PAGE_MASK);
if (nsize < object->un_pager.vnp.vnp_size) {
/*
* File has shrunk. Toss any cached pages beyond the new EOF.
*/
if (nobjsize < object->size)
vm_object_page_remove(object, nobjsize, object->size,
0);
/*
* this gets rid of garbage at the end of a page that is now
* only partially backed by the vnode.
*
* XXX for some reason (I don't know yet), if we take a
* completely invalid page and mark it partially valid
* it can screw up NFS reads, so we don't allow the case.
*/
if ((nsize & PAGE_MASK) &&
(m = vm_page_lookup(object, OFF_TO_IDX(nsize))) != NULL &&
m->valid != 0) {
int base = (int)nsize & PAGE_MASK;
int size = PAGE_SIZE - base;
/*
* Clear out partial-page garbage in case
* the page has been mapped.
*/
pmap_zero_page_area(m, base, size);
/*
* Update the valid bits to reflect the blocks that
* have been zeroed. Some of these valid bits may
* have already been set.
*/
vm_page_set_valid_range(m, base, size);
/*
* Round "base" to the next block boundary so that the
* dirty bit for a partially zeroed block is not
* cleared.
*/
base = roundup2(base, DEV_BSIZE);
/*
* Clear out partial-page dirty bits.
*
* note that we do not clear out the valid
* bits. This would prevent bogus_page
* replacement from working properly.
*/
vm_page_clear_dirty(m, base, PAGE_SIZE - base);
}
}
object->un_pager.vnp.vnp_size = nsize;
object->size = nobjsize;
VM_OBJECT_WUNLOCK(object);
}
static int
shm_dotruncate(struct shmfd *shmfd, off_t length)
{
vm_object_t object;
vm_page_t m, ma[1];
vm_pindex_t idx, nobjsize;
vm_ooffset_t delta;
int base, rv;
object = shmfd->shm_object;
VM_OBJECT_LOCK(object);
if (length == shmfd->shm_size) {
VM_OBJECT_UNLOCK(object);
return (0);
}
nobjsize = OFF_TO_IDX(length + PAGE_MASK);
/* Are we shrinking? If so, trim the end. */
if (length < shmfd->shm_size) {
/*
* Disallow any requests to shrink the size if this
* object is mapped into the kernel.
*/
if (shmfd->shm_kmappings > 0) {
VM_OBJECT_UNLOCK(object);
return (EBUSY);
}
/*
* Zero the truncated part of the last page.
*/
base = length & PAGE_MASK;
if (base != 0) {
idx = OFF_TO_IDX(length);
retry:
m = vm_page_lookup(object, idx);
if (m != NULL) {
if ((m->oflags & VPO_BUSY) != 0 ||
m->busy != 0) {
vm_page_sleep(m, "shmtrc");
goto retry;
}
} else if (vm_pager_has_page(object, idx, NULL, NULL)) {
m = vm_page_alloc(object, idx, VM_ALLOC_NORMAL);
if (m == NULL) {
VM_OBJECT_UNLOCK(object);
VM_WAIT;
VM_OBJECT_LOCK(object);
goto retry;
} else if (m->valid != VM_PAGE_BITS_ALL) {
ma[0] = m;
rv = vm_pager_get_pages(object, ma, 1,
0);
m = vm_page_lookup(object, idx);
} else
/* A cached page was reactivated. */
rv = VM_PAGER_OK;
vm_page_lock(m);
if (rv == VM_PAGER_OK) {
vm_page_deactivate(m);
vm_page_unlock(m);
vm_page_wakeup(m);
} else {
vm_page_free(m);
vm_page_unlock(m);
VM_OBJECT_UNLOCK(object);
return (EIO);
}
}
if (m != NULL) {
pmap_zero_page_area(m, base, PAGE_SIZE - base);
KASSERT(m->valid == VM_PAGE_BITS_ALL,
("shm_dotruncate: page %p is invalid", m));
vm_page_dirty(m);
vm_pager_page_unswapped(m);
}
}
delta = ptoa(object->size - nobjsize);
/* Toss in memory pages. */
if (nobjsize < object->size)
vm_object_page_remove(object, nobjsize, object->size,
0);
/* Toss pages from swap. */
if (object->type == OBJT_SWAP)
swap_pager_freespace(object, nobjsize, delta);
/* Free the swap accounted for shm */
swap_release_by_cred(delta, object->cred);
object->charge -= delta;
} else {
/* Attempt to reserve the swap */
delta = ptoa(nobjsize - object->size);
if (!swap_reserve_by_cred(delta, object->cred)) {
VM_OBJECT_UNLOCK(object);
return (ENOMEM);
}
object->charge += delta;
}
shmfd->shm_size = length;
mtx_lock(&shm_timestamp_lock);
vfs_timestamp(&shmfd->shm_ctime);
shmfd->shm_mtime = shmfd->shm_ctime;
mtx_unlock(&shm_timestamp_lock);
object->size = nobjsize;
VM_OBJECT_UNLOCK(object);
return (0);
}
static int
shm_dotruncate(struct shmfd *shmfd, off_t length)
{
vm_object_t object;
vm_page_t m;
vm_pindex_t nobjsize;
vm_ooffset_t delta;
object = shmfd->shm_object;
VM_OBJECT_LOCK(object);
if (length == shmfd->shm_size) {
VM_OBJECT_UNLOCK(object);
return (0);
}
nobjsize = OFF_TO_IDX(length + PAGE_MASK);
/* Are we shrinking? If so, trim the end. */
if (length < shmfd->shm_size) {
delta = ptoa(object->size - nobjsize);
/* Toss in memory pages. */
if (nobjsize < object->size)
vm_object_page_remove(object, nobjsize, object->size,
FALSE);
/* Toss pages from swap. */
if (object->type == OBJT_SWAP)
swap_pager_freespace(object, nobjsize, delta);
/* Free the swap accounted for shm */
swap_release_by_uid(delta, object->uip);
object->charge -= delta;
/*
* If the last page is partially mapped, then zero out
* the garbage at the end of the page. See comments
* in vnode_pager_setsize() for more details.
*
* XXXJHB: This handles in memory pages, but what about
* a page swapped out to disk?
*/
if ((length & PAGE_MASK) &&
(m = vm_page_lookup(object, OFF_TO_IDX(length))) != NULL &&
m->valid != 0) {
int base = (int)length & PAGE_MASK;
int size = PAGE_SIZE - base;
pmap_zero_page_area(m, base, size);
/*
* Update the valid bits to reflect the blocks that
* have been zeroed. Some of these valid bits may
* have already been set.
*/
vm_page_set_valid(m, base, size);
/*
* Round "base" to the next block boundary so that the
* dirty bit for a partially zeroed block is not
* cleared.
*/
base = roundup2(base, DEV_BSIZE);
vm_page_lock_queues();
vm_page_clear_dirty(m, base, PAGE_SIZE - base);
vm_page_unlock_queues();
} else if ((length & PAGE_MASK) &&
__predict_false(object->cache != NULL)) {
vm_page_cache_free(object, OFF_TO_IDX(length),
nobjsize);
}
} else {
/* Attempt to reserve the swap */
delta = ptoa(nobjsize - object->size);
if (!swap_reserve_by_uid(delta, object->uip)) {
VM_OBJECT_UNLOCK(object);
return (ENOMEM);
}
object->charge += delta;
}
shmfd->shm_size = length;
mtx_lock(&shm_timestamp_lock);
vfs_timestamp(&shmfd->shm_ctime);
shmfd->shm_mtime = shmfd->shm_ctime;
mtx_unlock(&shm_timestamp_lock);
object->size = nobjsize;
VM_OBJECT_UNLOCK(object);
return (0);
}
/*
* Lets the VM system know about a change in size for a file.
* We adjust our own internal size and flush any cached pages in
* the associated object that are affected by the size change.
*
* NOTE: This routine may be invoked as a result of a pager put
* operation (possibly at object termination time), so we must be careful.
*
* NOTE: vp->v_filesize is initialized to NOOFFSET (-1), be sure that
* we do not blow up on the case. nsize will always be >= 0, however.
*/
void
vnode_pager_setsize(struct vnode *vp, vm_ooffset_t nsize)
{
vm_pindex_t nobjsize;
vm_pindex_t oobjsize;
vm_object_t object;
object = vp->v_object;
if (object == NULL)
return;
vm_object_hold(object);
KKASSERT(vp->v_object == object);
/*
* Hasn't changed size
*/
if (nsize == vp->v_filesize) {
vm_object_drop(object);
return;
}
/*
* Has changed size. Adjust the VM object's size and v_filesize
* before we start scanning pages to prevent new pages from being
* allocated during the scan.
*/
nobjsize = OFF_TO_IDX(nsize + PAGE_MASK);
oobjsize = object->size;
object->size = nobjsize;
/*
* File has shrunk. Toss any cached pages beyond the new EOF.
*/
if (nsize < vp->v_filesize) {
vp->v_filesize = nsize;
if (nobjsize < oobjsize) {
vm_object_page_remove(object, nobjsize, oobjsize,
FALSE);
}
/*
* This gets rid of garbage at the end of a page that is now
* only partially backed by the vnode. Since we are setting
* the entire page valid & clean after we are done we have
* to be sure that the portion of the page within the file
* bounds is already valid. If it isn't then making it
* valid would create a corrupt block.
*/
if (nsize & PAGE_MASK) {
vm_offset_t kva;
vm_page_t m;
m = vm_page_lookup_busy_wait(object, OFF_TO_IDX(nsize),
TRUE, "vsetsz");
if (m && m->valid) {
int base = (int)nsize & PAGE_MASK;
int size = PAGE_SIZE - base;
struct lwbuf *lwb;
struct lwbuf lwb_cache;
/*
* Clear out partial-page garbage in case
* the page has been mapped.
*
* This is byte aligned.
*/
lwb = lwbuf_alloc(m, &lwb_cache);
kva = lwbuf_kva(lwb);
bzero((caddr_t)kva + base, size);
lwbuf_free(lwb);
/*
* XXX work around SMP data integrity race
* by unmapping the page from user processes.
* The garbage we just cleared may be mapped
* to a user process running on another cpu
* and this code is not running through normal
* I/O channels which handle SMP issues for
* us, so unmap page to synchronize all cpus.
*
* XXX should vm_pager_unmap_page() have
* dealt with this?
*/
vm_page_protect(m, VM_PROT_NONE);
/*
* Clear out partial-page dirty bits. This
* has the side effect of setting the valid
* bits, but that is ok. There are a bunch
* of places in the VM system where we expected
* m->dirty == VM_PAGE_BITS_ALL. The file EOF
* case is one of them. If the page is still
* partially dirty, make it fully dirty.
*
* NOTE: We do not clear out the valid
* bits. This would prevent bogus_page
* replacement from working properly.
*
* NOTE: We do not want to clear the dirty
* bit for a partial DEV_BSIZE'd truncation!
* This is DEV_BSIZE aligned!
*/
vm_page_clear_dirty_beg_nonincl(m, base, size);
if (m->dirty != 0)
m->dirty = VM_PAGE_BITS_ALL;
vm_page_wakeup(m);
} else if (m) {
vm_page_wakeup(m);
}
}
} else {
vp->v_filesize = nsize;
}
vm_object_drop(object);
}
kern_return_t
kmem_alloc_contig(
vm_map_t map,
vm_offset_t *addrp,
vm_size_t size,
vm_offset_t mask,
ppnum_t max_pnum,
ppnum_t pnum_mask,
int flags)
{
vm_object_t object;
vm_object_offset_t offset;
vm_map_offset_t map_addr;
vm_map_offset_t map_mask;
vm_map_size_t map_size, i;
vm_map_entry_t entry;
vm_page_t m, pages;
kern_return_t kr;
if (map == VM_MAP_NULL || (flags & ~(KMA_KOBJECT | KMA_LOMEM | KMA_NOPAGEWAIT)))
return KERN_INVALID_ARGUMENT;
if (size == 0) {
*addrp = 0;
return KERN_INVALID_ARGUMENT;
}
map_size = vm_map_round_page(size);
map_mask = (vm_map_offset_t)mask;
/*
* Allocate a new object (if necessary) and the reference we
* will be donating to the map entry. We must do this before
* locking the map, or risk deadlock with the default pager.
*/
if ((flags & KMA_KOBJECT) != 0) {
object = kernel_object;
vm_object_reference(object);
} else {
object = vm_object_allocate(map_size);
}
kr = vm_map_find_space(map, &map_addr, map_size, map_mask, 0, &entry);
if (KERN_SUCCESS != kr) {
vm_object_deallocate(object);
return kr;
}
entry->object.vm_object = object;
entry->offset = offset = (object == kernel_object) ?
map_addr : 0;
/* Take an extra object ref in case the map entry gets deleted */
vm_object_reference(object);
vm_map_unlock(map);
kr = cpm_allocate(CAST_DOWN(vm_size_t, map_size), &pages, max_pnum, pnum_mask, FALSE, flags);
if (kr != KERN_SUCCESS) {
vm_map_remove(map, vm_map_trunc_page(map_addr),
vm_map_round_page(map_addr + map_size), 0);
vm_object_deallocate(object);
*addrp = 0;
return kr;
}
vm_object_lock(object);
for (i = 0; i < map_size; i += PAGE_SIZE) {
m = pages;
pages = NEXT_PAGE(m);
*(NEXT_PAGE_PTR(m)) = VM_PAGE_NULL;
m->busy = FALSE;
vm_page_insert(m, object, offset + i);
}
vm_object_unlock(object);
if ((kr = vm_map_wire(map, vm_map_trunc_page(map_addr),
vm_map_round_page(map_addr + map_size), VM_PROT_DEFAULT, FALSE))
!= KERN_SUCCESS) {
if (object == kernel_object) {
vm_object_lock(object);
vm_object_page_remove(object, offset, offset + map_size);
vm_object_unlock(object);
}
vm_map_remove(map, vm_map_trunc_page(map_addr),
vm_map_round_page(map_addr + map_size), 0);
vm_object_deallocate(object);
return kr;
}
vm_object_deallocate(object);
if (object == kernel_object)
vm_map_simplify(map, map_addr);
*addrp = (vm_offset_t) map_addr;
assert((vm_map_offset_t) *addrp == map_addr);
return KERN_SUCCESS;
}
/*
* Resizes the aobj associated to the regular file pointed to by vp to
* the size newsize. 'vp' must point to a vnode that represents a regular
* file. 'newsize' must be positive.
*
* pass trivial as 1 when buf content will be overwritten, otherwise set 0
* to be zero filled.
*
* Returns zero on success or an appropriate error code on failure.
*
* Caller must hold the node exclusively locked.
*/
int
tmpfs_reg_resize(struct vnode *vp, off_t newsize, int trivial)
{
int error;
vm_pindex_t newpages, oldpages;
struct tmpfs_mount *tmp;
struct tmpfs_node *node;
off_t oldsize;
#ifdef INVARIANTS
KKASSERT(vp->v_type == VREG);
KKASSERT(newsize >= 0);
#endif
node = VP_TO_TMPFS_NODE(vp);
tmp = VFS_TO_TMPFS(vp->v_mount);
/*
* Convert the old and new sizes to the number of pages needed to
* store them. It may happen that we do not need to do anything
* because the last allocated page can accommodate the change on
* its own.
*/
oldsize = node->tn_size;
oldpages = round_page64(oldsize) / PAGE_SIZE;
KKASSERT(oldpages == node->tn_reg.tn_aobj_pages);
newpages = round_page64(newsize) / PAGE_SIZE;
if (newpages > oldpages &&
tmp->tm_pages_used + newpages - oldpages > tmp->tm_pages_max) {
error = ENOSPC;
goto out;
}
node->tn_reg.tn_aobj_pages = newpages;
node->tn_size = newsize;
if (newpages != oldpages)
atomic_add_long(&tmp->tm_pages_used, (newpages - oldpages));
/*
* When adjusting the vnode filesize and its VM object we must
* also adjust our backing VM object (aobj). The blocksize
* used must match the block sized we use for the buffer cache.
*
* The backing VM object may contain VM pages as well as swap
* assignments if we previously renamed main object pages into
* it during deactivation.
*/
if (newsize < oldsize) {
vm_pindex_t osize;
vm_pindex_t nsize;
vm_object_t aobj;
error = nvtruncbuf(vp, newsize, TMPFS_BLKSIZE, -1, 0);
aobj = node->tn_reg.tn_aobj;
if (aobj) {
osize = aobj->size;
nsize = vp->v_object->size;
if (nsize < osize) {
aobj->size = osize;
swap_pager_freespace(aobj, nsize,
osize - nsize);
vm_object_page_remove(aobj, nsize, osize,
FALSE);
}
}
} else {
vm_object_t aobj;
error = nvextendbuf(vp, oldsize, newsize,
TMPFS_BLKSIZE, TMPFS_BLKSIZE,
-1, -1, trivial);
aobj = node->tn_reg.tn_aobj;
if (aobj)
aobj->size = vp->v_object->size;
}
out:
return error;
}
