/*
struct vnop_putpages_args {
struct vnode *a_vp;
vm_page_t *a_m;
int a_count;
int a_sync;
int *a_rtvals;
vm_ooffset_t a_offset;
};
*/
static int
fuse_vnop_putpages(struct vop_putpages_args *ap)
{
struct uio uio;
struct iovec iov;
vm_offset_t kva;
struct buf *bp;
int i, error, npages, count;
off_t offset;
int *rtvals;
struct vnode *vp;
struct thread *td;
struct ucred *cred;
vm_page_t *pages;
vm_ooffset_t fsize;
FS_DEBUG2G("heh\n");
vp = ap->a_vp;
KASSERT(vp->v_object, ("objectless vp passed to putpages"));
fsize = vp->v_object->un_pager.vnp.vnp_size;
td = curthread; /* XXX */
cred = curthread->td_ucred; /* XXX */
pages = ap->a_m;
count = ap->a_count;
rtvals = ap->a_rtvals;
npages = btoc(count);
offset = IDX_TO_OFF(pages[0]->pindex);
if (!fsess_opt_mmap(vnode_mount(vp))) {
FS_DEBUG("called on non-cacheable vnode??\n");
}
for (i = 0; i < npages; i++)
rtvals[i] = VM_PAGER_AGAIN;
/*
* When putting pages, do not extend file past EOF.
*/
if (offset + count > fsize) {
count = fsize - offset;
if (count < 0)
count = 0;
}
/*
* We use only the kva address for the buffer, but this is extremely
* convenient and fast.
*/
bp = getpbuf(&fuse_pbuf_freecnt);
kva = (vm_offset_t)bp->b_data;
pmap_qenter(kva, pages, npages);
PCPU_INC(cnt.v_vnodeout);
PCPU_ADD(cnt.v_vnodepgsout, count);
iov.iov_base = (caddr_t)kva;
iov.iov_len = count;
uio.uio_iov = &iov;
uio.uio_iovcnt = 1;
uio.uio_offset = offset;
uio.uio_resid = count;
uio.uio_segflg = UIO_SYSSPACE;
uio.uio_rw = UIO_WRITE;
uio.uio_td = td;
error = fuse_io_dispatch(vp, &uio, IO_DIRECT, cred);
pmap_qremove(kva, npages);
relpbuf(bp, &fuse_pbuf_freecnt);
if (!error) {
int nwritten = round_page(count - uio.uio_resid) / PAGE_SIZE;
for (i = 0; i < nwritten; i++) {
rtvals[i] = VM_PAGER_OK;
VM_OBJECT_WLOCK(pages[i]->object);
vm_page_undirty(pages[i]);
VM_OBJECT_WUNLOCK(pages[i]->object);
}
}
return rtvals[0];
}
/*
* If blocks are contiguous on disk, use this to provide clustered
* read ahead. We will read as many blocks as possible sequentially
* and then parcel them up into logical blocks in the buffer hash table.
*/
static struct buf *
cluster_rbuild(struct vnode *vp, u_quad_t filesize, daddr_t lbn,
daddr_t blkno, long size, int run, int gbflags, struct buf *fbp)
{
struct buf *bp, *tbp;
daddr_t bn;
off_t off;
long tinc, tsize;
int i, inc, j, k, toff;
KASSERT(size == vp->v_mount->mnt_stat.f_iosize,
("cluster_rbuild: size %ld != f_iosize %jd\n",
size, (intmax_t)vp->v_mount->mnt_stat.f_iosize));
/*
* avoid a division
*/
while ((u_quad_t) size * (lbn + run) > filesize) {
--run;
}
if (fbp) {
tbp = fbp;
tbp->b_iocmd = BIO_READ;
} else {
tbp = getblk(vp, lbn, size, 0, 0, gbflags);
if (tbp->b_flags & B_CACHE)
return tbp;
tbp->b_flags |= B_ASYNC | B_RAM;
tbp->b_iocmd = BIO_READ;
}
tbp->b_blkno = blkno;
if( (tbp->b_flags & B_MALLOC) ||
((tbp->b_flags & B_VMIO) == 0) || (run <= 1) )
return tbp;
bp = trypbuf(&cluster_pbuf_freecnt);
if (bp == NULL)
return tbp;
/*
* We are synthesizing a buffer out of vm_page_t's, but
* if the block size is not page aligned then the starting
* address may not be either. Inherit the b_data offset
* from the original buffer.
*/
bp->b_flags = B_ASYNC | B_CLUSTER | B_VMIO;
if ((gbflags & GB_UNMAPPED) != 0) {
bp->b_data = unmapped_buf;
} else {
bp->b_data = (char *)((vm_offset_t)bp->b_data |
((vm_offset_t)tbp->b_data & PAGE_MASK));
}
bp->b_iocmd = BIO_READ;
bp->b_iodone = cluster_callback;
bp->b_blkno = blkno;
bp->b_lblkno = lbn;
bp->b_offset = tbp->b_offset;
KASSERT(bp->b_offset != NOOFFSET, ("cluster_rbuild: no buffer offset"));
pbgetvp(vp, bp);
TAILQ_INIT(&bp->b_cluster.cluster_head);
bp->b_bcount = 0;
bp->b_bufsize = 0;
bp->b_npages = 0;
inc = btodb(size);
for (bn = blkno, i = 0; i < run; ++i, bn += inc) {
if (i == 0) {
VM_OBJECT_WLOCK(tbp->b_bufobj->bo_object);
vfs_drain_busy_pages(tbp);
vm_object_pip_add(tbp->b_bufobj->bo_object,
tbp->b_npages);
for (k = 0; k < tbp->b_npages; k++)
vm_page_sbusy(tbp->b_pages[k]);
VM_OBJECT_WUNLOCK(tbp->b_bufobj->bo_object);
} else {
if ((bp->b_npages * PAGE_SIZE) +
round_page(size) > vp->v_mount->mnt_iosize_max) {
break;
}
tbp = getblk(vp, lbn + i, size, 0, 0, GB_LOCK_NOWAIT |
(gbflags & GB_UNMAPPED));
/* Don't wait around for locked bufs. */
if (tbp == NULL)
break;
/*
* Stop scanning if the buffer is fully valid
* (marked B_CACHE), or locked (may be doing a
* background write), or if the buffer is not
* VMIO backed. The clustering code can only deal
* with VMIO-backed buffers. The bo lock is not
* required for the BKGRDINPROG check since it
* can not be set without the buf lock.
*/
if ((tbp->b_vflags & BV_BKGRDINPROG) ||
(tbp->b_flags & B_CACHE) ||
(tbp->b_flags & B_VMIO) == 0) {
bqrelse(tbp);
break;
}
/*
* The buffer must be completely invalid in order to
* take part in the cluster. If it is partially valid
* then we stop.
*/
off = tbp->b_offset;
tsize = size;
VM_OBJECT_WLOCK(tbp->b_bufobj->bo_object);
for (j = 0; tsize > 0; j++) {
toff = off & PAGE_MASK;
tinc = tsize;
if (toff + tinc > PAGE_SIZE)
tinc = PAGE_SIZE - toff;
VM_OBJECT_ASSERT_WLOCKED(tbp->b_pages[j]->object);
if ((tbp->b_pages[j]->valid &
vm_page_bits(toff, tinc)) != 0)
break;
if (vm_page_xbusied(tbp->b_pages[j]))
break;
vm_object_pip_add(tbp->b_bufobj->bo_object, 1);
vm_page_sbusy(tbp->b_pages[j]);
off += tinc;
tsize -= tinc;
}
if (tsize > 0) {
clean_sbusy:
vm_object_pip_add(tbp->b_bufobj->bo_object, -j);
for (k = 0; k < j; k++)
vm_page_sunbusy(tbp->b_pages[k]);
VM_OBJECT_WUNLOCK(tbp->b_bufobj->bo_object);
bqrelse(tbp);
break;
}
VM_OBJECT_WUNLOCK(tbp->b_bufobj->bo_object);
/*
* Set a read-ahead mark as appropriate
*/
if ((fbp && (i == 1)) || (i == (run - 1)))
tbp->b_flags |= B_RAM;
/*
* Set the buffer up for an async read (XXX should
* we do this only if we do not wind up brelse()ing?).
* Set the block number if it isn't set, otherwise
* if it is make sure it matches the block number we
* expect.
*/
tbp->b_flags |= B_ASYNC;
tbp->b_iocmd = BIO_READ;
if (tbp->b_blkno == tbp->b_lblkno) {
tbp->b_blkno = bn;
} else if (tbp->b_blkno != bn) {
VM_OBJECT_WLOCK(tbp->b_bufobj->bo_object);
goto clean_sbusy;
}
}
/*
* XXX fbp from caller may not be B_ASYNC, but we are going
* to biodone() it in cluster_callback() anyway
*/
BUF_KERNPROC(tbp);
TAILQ_INSERT_TAIL(&bp->b_cluster.cluster_head,
tbp, b_cluster.cluster_entry);
VM_OBJECT_WLOCK(tbp->b_bufobj->bo_object);
for (j = 0; j < tbp->b_npages; j += 1) {
vm_page_t m;
m = tbp->b_pages[j];
if ((bp->b_npages == 0) ||
(bp->b_pages[bp->b_npages-1] != m)) {
bp->b_pages[bp->b_npages] = m;
bp->b_npages++;
}
if (m->valid == VM_PAGE_BITS_ALL)
tbp->b_pages[j] = bogus_page;
}
VM_OBJECT_WUNLOCK(tbp->b_bufobj->bo_object);
/*
* Don't inherit tbp->b_bufsize as it may be larger due to
* a non-page-aligned size. Instead just aggregate using
* 'size'.
*/
if (tbp->b_bcount != size)
printf("warning: tbp->b_bcount wrong %ld vs %ld\n", tbp->b_bcount, size);
if (tbp->b_bufsize != size)
printf("warning: tbp->b_bufsize wrong %ld vs %ld\n", tbp->b_bufsize, size);
bp->b_bcount += size;
bp->b_bufsize += size;
}
/*
* Fully valid pages in the cluster are already good and do not need
* to be re-read from disk. Replace the page with bogus_page
*/
VM_OBJECT_WLOCK(bp->b_bufobj->bo_object);
for (j = 0; j < bp->b_npages; j++) {
VM_OBJECT_ASSERT_WLOCKED(bp->b_pages[j]->object);
if (bp->b_pages[j]->valid == VM_PAGE_BITS_ALL)
bp->b_pages[j] = bogus_page;
}
VM_OBJECT_WUNLOCK(bp->b_bufobj->bo_object);
if (bp->b_bufsize > bp->b_kvasize)
panic("cluster_rbuild: b_bufsize(%ld) > b_kvasize(%d)\n",
bp->b_bufsize, bp->b_kvasize);
if (buf_mapped(bp)) {
pmap_qenter(trunc_page((vm_offset_t) bp->b_data),
(vm_page_t *)bp->b_pages, bp->b_npages);
}
return (bp);
}
static int
vnode_pager_generic_getpages_done(struct buf *bp)
{
vm_object_t object;
off_t tfoff, nextoff;
int i, error;
error = (bp->b_ioflags & BIO_ERROR) != 0 ? EIO : 0;
object = bp->b_vp->v_object;
if (error == 0 && bp->b_bcount != bp->b_npages * PAGE_SIZE) {
if (!buf_mapped(bp)) {
bp->b_data = bp->b_kvabase;
pmap_qenter((vm_offset_t)bp->b_data, bp->b_pages,
bp->b_npages);
}
bzero(bp->b_data + bp->b_bcount,
PAGE_SIZE * bp->b_npages - bp->b_bcount);
}
if (buf_mapped(bp)) {
pmap_qremove((vm_offset_t)bp->b_data, bp->b_npages);
bp->b_data = unmapped_buf;
}
VM_OBJECT_WLOCK(object);
for (i = 0, tfoff = IDX_TO_OFF(bp->b_pages[0]->pindex);
i < bp->b_npages; i++, tfoff = nextoff) {
vm_page_t mt;
nextoff = tfoff + PAGE_SIZE;
mt = bp->b_pages[i];
if (nextoff <= object->un_pager.vnp.vnp_size) {
/*
* Read filled up entire page.
*/
mt->valid = VM_PAGE_BITS_ALL;
KASSERT(mt->dirty == 0,
("%s: page %p is dirty", __func__, mt));
KASSERT(!pmap_page_is_mapped(mt),
("%s: page %p is mapped", __func__, mt));
} else {
/*
* Read did not fill up entire page.
*
* Currently we do not set the entire page valid,
* we just try to clear the piece that we couldn't
* read.
*/
vm_page_set_valid_range(mt, 0,
object->un_pager.vnp.vnp_size - tfoff);
KASSERT((mt->dirty & vm_page_bits(0,
object->un_pager.vnp.vnp_size - tfoff)) == 0,
("%s: page %p is dirty", __func__, mt));
}
if (i != bp->b_pager.pg_reqpage)
vm_page_readahead_finish(mt);
}
VM_OBJECT_WUNLOCK(object);
if (error != 0)
printf("%s: I/O read error %d\n", __func__, error);
return (error);
}
/*
struct vnop_getpages_args {
struct vnode *a_vp;
vm_page_t *a_m;
int a_count;
int a_reqpage;
};
*/
static int
fuse_vnop_getpages(struct vop_getpages_args *ap)
{
int i, error, nextoff, size, toff, count, npages;
struct uio uio;
struct iovec iov;
vm_offset_t kva;
struct buf *bp;
struct vnode *vp;
struct thread *td;
struct ucred *cred;
vm_page_t *pages;
FS_DEBUG2G("heh\n");
vp = ap->a_vp;
KASSERT(vp->v_object, ("objectless vp passed to getpages"));
td = curthread; /* XXX */
cred = curthread->td_ucred; /* XXX */
pages = ap->a_m;
npages = ap->a_count;
if (!fsess_opt_mmap(vnode_mount(vp))) {
FS_DEBUG("called on non-cacheable vnode??\n");
return (VM_PAGER_ERROR);
}
/*
* If the last page is partially valid, just return it and allow
* the pager to zero-out the blanks. Partially valid pages can
* only occur at the file EOF.
*
* XXXGL: is that true for FUSE, which is a local filesystem,
* but still somewhat disconnected from the kernel?
*/
VM_OBJECT_WLOCK(vp->v_object);
if (pages[npages - 1]->valid != 0 && --npages == 0)
goto out;
VM_OBJECT_WUNLOCK(vp->v_object);
/*
* We use only the kva address for the buffer, but this is extremely
* convenient and fast.
*/
bp = getpbuf(&fuse_pbuf_freecnt);
kva = (vm_offset_t)bp->b_data;
pmap_qenter(kva, pages, npages);
PCPU_INC(cnt.v_vnodein);
PCPU_ADD(cnt.v_vnodepgsin, npages);
count = npages << PAGE_SHIFT;
iov.iov_base = (caddr_t)kva;
iov.iov_len = count;
uio.uio_iov = &iov;
uio.uio_iovcnt = 1;
uio.uio_offset = IDX_TO_OFF(pages[0]->pindex);
uio.uio_resid = count;
uio.uio_segflg = UIO_SYSSPACE;
uio.uio_rw = UIO_READ;
uio.uio_td = td;
error = fuse_io_dispatch(vp, &uio, IO_DIRECT, cred);
pmap_qremove(kva, npages);
relpbuf(bp, &fuse_pbuf_freecnt);
if (error && (uio.uio_resid == count)) {
FS_DEBUG("error %d\n", error);
return VM_PAGER_ERROR;
}
/*
* Calculate the number of bytes read and validate only that number
* of bytes. Note that due to pending writes, size may be 0. This
* does not mean that the remaining data is invalid!
*/
size = count - uio.uio_resid;
VM_OBJECT_WLOCK(vp->v_object);
fuse_vm_page_lock_queues();
for (i = 0, toff = 0; i < npages; i++, toff = nextoff) {
vm_page_t m;
nextoff = toff + PAGE_SIZE;
m = pages[i];
if (nextoff <= size) {
/*
* Read operation filled an entire page
*/
m->valid = VM_PAGE_BITS_ALL;
KASSERT(m->dirty == 0,
("fuse_getpages: page %p is dirty", m));
} else if (size > toff) {
/*
* Read operation filled a partial page.
*/
m->valid = 0;
vm_page_set_valid_range(m, 0, size - toff);
KASSERT(m->dirty == 0,
("fuse_getpages: page %p is dirty", m));
} else {
/*
* Read operation was short. If no error occurred
* we may have hit a zero-fill section. We simply
* leave valid set to 0.
*/
;
}
}
fuse_vm_page_unlock_queues();
out:
VM_OBJECT_WUNLOCK(vp->v_object);
if (ap->a_rbehind)
*ap->a_rbehind = 0;
if (ap->a_rahead)
*ap->a_rahead = 0;
return (VM_PAGER_OK);
}
/*
* small block filesystem vnode pager input
*/
static int
vnode_pager_input_smlfs(vm_object_t object, vm_page_t m)
{
struct vnode *vp;
struct bufobj *bo;
struct buf *bp;
struct sf_buf *sf;
daddr_t fileaddr;
vm_offset_t bsize;
vm_page_bits_t bits;
int error, i;
error = 0;
vp = object->handle;
if (vp->v_iflag & VI_DOOMED)
return VM_PAGER_BAD;
bsize = vp->v_mount->mnt_stat.f_iosize;
VOP_BMAP(vp, 0, &bo, 0, NULL, NULL);
sf = sf_buf_alloc(m, 0);
for (i = 0; i < PAGE_SIZE / bsize; i++) {
vm_ooffset_t address;
bits = vm_page_bits(i * bsize, bsize);
if (m->valid & bits)
continue;
address = IDX_TO_OFF(m->pindex) + i * bsize;
if (address >= object->un_pager.vnp.vnp_size) {
fileaddr = -1;
} else {
error = vnode_pager_addr(vp, address, &fileaddr, NULL);
if (error)
break;
}
if (fileaddr != -1) {
bp = getpbuf(&vnode_pbuf_freecnt);
/* build a minimal buffer header */
bp->b_iocmd = BIO_READ;
bp->b_iodone = bdone;
KASSERT(bp->b_rcred == NOCRED, ("leaking read ucred"));
KASSERT(bp->b_wcred == NOCRED, ("leaking write ucred"));
bp->b_rcred = crhold(curthread->td_ucred);
bp->b_wcred = crhold(curthread->td_ucred);
bp->b_data = (caddr_t)sf_buf_kva(sf) + i * bsize;
bp->b_blkno = fileaddr;
pbgetbo(bo, bp);
bp->b_vp = vp;
bp->b_bcount = bsize;
bp->b_bufsize = bsize;
bp->b_runningbufspace = bp->b_bufsize;
atomic_add_long(&runningbufspace, bp->b_runningbufspace);
/* do the input */
bp->b_iooffset = dbtob(bp->b_blkno);
bstrategy(bp);
bwait(bp, PVM, "vnsrd");
if ((bp->b_ioflags & BIO_ERROR) != 0)
error = EIO;
/*
* free the buffer header back to the swap buffer pool
*/
bp->b_vp = NULL;
pbrelbo(bp);
relpbuf(bp, &vnode_pbuf_freecnt);
if (error)
break;
} else
bzero((caddr_t)sf_buf_kva(sf) + i * bsize, bsize);
KASSERT((m->dirty & bits) == 0,
("vnode_pager_input_smlfs: page %p is dirty", m));
VM_OBJECT_WLOCK(object);
m->valid |= bits;
VM_OBJECT_WUNLOCK(object);
}
sf_buf_free(sf);
if (error) {
return VM_PAGER_ERROR;
}
return VM_PAGER_OK;
}
/*
* This is now called from local media FS's to operate against their
* own vnodes if they fail to implement VOP_GETPAGES.
*/
int
vnode_pager_generic_getpages(struct vnode *vp, vm_page_t *m, int bytecount,
int reqpage, vop_getpages_iodone_t iodone, void *arg)
{
vm_object_t object;
struct bufobj *bo;
struct buf *bp;
daddr_t firstaddr, reqblock;
off_t foff, pib;
int pbefore, pafter, i, size, bsize, first, last, *freecnt;
int count, error, before, after, secmask;
KASSERT(vp->v_type != VCHR && vp->v_type != VBLK,
("vnode_pager_generic_getpages does not support devices"));
if (vp->v_iflag & VI_DOOMED)
return (VM_PAGER_BAD);
object = vp->v_object;
count = bytecount / PAGE_SIZE;
bsize = vp->v_mount->mnt_stat.f_iosize;
/*
* Synchronous and asynchronous paging operations use different
* free pbuf counters. This is done to avoid asynchronous requests
* to consume all pbufs.
* Allocate the pbuf at the very beginning of the function, so that
* if we are low on certain kind of pbufs don't even proceed to BMAP,
* but sleep.
*/
freecnt = iodone != NULL ?
&vnode_async_pbuf_freecnt : &vnode_pbuf_freecnt;
bp = getpbuf(freecnt);
/*
* Get the underlying device blocks for the file with VOP_BMAP().
* If the file system doesn't support VOP_BMAP, use old way of
* getting pages via VOP_READ.
*/
error = VOP_BMAP(vp, IDX_TO_OFF(m[reqpage]->pindex) / bsize, &bo,
&reqblock, &after, &before);
if (error == EOPNOTSUPP) {
relpbuf(bp, freecnt);
VM_OBJECT_WLOCK(object);
for (i = 0; i < count; i++)
if (i != reqpage) {
vm_page_lock(m[i]);
vm_page_free(m[i]);
vm_page_unlock(m[i]);
}
PCPU_INC(cnt.v_vnodein);
PCPU_INC(cnt.v_vnodepgsin);
error = vnode_pager_input_old(object, m[reqpage]);
VM_OBJECT_WUNLOCK(object);
return (error);
} else if (error != 0) {
relpbuf(bp, freecnt);
vm_pager_free_nonreq(object, m, reqpage, count, FALSE);
return (VM_PAGER_ERROR);
/*
* If the blocksize is smaller than a page size, then use
* special small filesystem code.
*/
} else if ((PAGE_SIZE / bsize) > 1) {
relpbuf(bp, freecnt);
vm_pager_free_nonreq(object, m, reqpage, count, FALSE);
PCPU_INC(cnt.v_vnodein);
PCPU_INC(cnt.v_vnodepgsin);
return (vnode_pager_input_smlfs(object, m[reqpage]));
}
/*
* Since the caller has busied the requested page, that page's valid
* field will not be changed by other threads.
*/
vm_page_assert_xbusied(m[reqpage]);
/*
* If we have a completely valid page available to us, we can
* clean up and return. Otherwise we have to re-read the
* media.
*/
if (m[reqpage]->valid == VM_PAGE_BITS_ALL) {
relpbuf(bp, freecnt);
vm_pager_free_nonreq(object, m, reqpage, count, FALSE);
return (VM_PAGER_OK);
} else if (reqblock == -1) {
relpbuf(bp, freecnt);
pmap_zero_page(m[reqpage]);
KASSERT(m[reqpage]->dirty == 0,
("vnode_pager_generic_getpages: page %p is dirty", m));
VM_OBJECT_WLOCK(object);
m[reqpage]->valid = VM_PAGE_BITS_ALL;
vm_pager_free_nonreq(object, m, reqpage, count, TRUE);
VM_OBJECT_WUNLOCK(object);
return (VM_PAGER_OK);
} else if (m[reqpage]->valid != 0) {
VM_OBJECT_WLOCK(object);
m[reqpage]->valid = 0;
VM_OBJECT_WUNLOCK(object);
}
pib = IDX_TO_OFF(m[reqpage]->pindex) % bsize;
pbefore = ((daddr_t)before * bsize + pib) / PAGE_SIZE;
pafter = ((daddr_t)(after + 1) * bsize - pib) / PAGE_SIZE - 1;
first = reqpage < pbefore ? 0 : reqpage - pbefore;
last = reqpage + pafter >= count ? count - 1 : reqpage + pafter;
if (first > 0 || last + 1 < count) {
VM_OBJECT_WLOCK(object);
for (i = 0; i < first; i++) {
vm_page_lock(m[i]);
vm_page_free(m[i]);
vm_page_unlock(m[i]);
}
for (i = last + 1; i < count; i++) {
vm_page_lock(m[i]);
vm_page_free(m[i]);
vm_page_unlock(m[i]);
}
VM_OBJECT_WUNLOCK(object);
}
/*
* here on direct device I/O
*/
firstaddr = reqblock;
firstaddr += pib / DEV_BSIZE;
firstaddr -= IDX_TO_OFF(reqpage - first) / DEV_BSIZE;
/*
* The first and last page have been calculated now, move
* input pages to be zero based, and adjust the count.
*/
m += first;
reqpage -= first;
count = last - first + 1;
/*
* calculate the file virtual address for the transfer
*/
foff = IDX_TO_OFF(m[0]->pindex);
/*
* calculate the size of the transfer
*/
size = count * PAGE_SIZE;
KASSERT(count > 0, ("zero count"));
if ((foff + size) > object->un_pager.vnp.vnp_size)
size = object->un_pager.vnp.vnp_size - foff;
KASSERT(size > 0, ("zero size"));
/*
* round up physical size for real devices.
*/
secmask = bo->bo_bsize - 1;
KASSERT(secmask < PAGE_SIZE && secmask > 0,
("vnode_pager_generic_getpages: sector size %d too large",
secmask + 1));
size = (size + secmask) & ~secmask;
/*
* and map the pages to be read into the kva, if the filesystem
* requires mapped buffers.
*/
if ((vp->v_mount->mnt_kern_flag & MNTK_UNMAPPED_BUFS) != 0 &&
unmapped_buf_allowed) {
bp->b_data = unmapped_buf;
bp->b_offset = 0;
} else {
bp->b_data = bp->b_kvabase;
pmap_qenter((vm_offset_t)bp->b_data, m, count);
}
/* build a minimal buffer header */
bp->b_iocmd = BIO_READ;
KASSERT(bp->b_rcred == NOCRED, ("leaking read ucred"));
KASSERT(bp->b_wcred == NOCRED, ("leaking write ucred"));
bp->b_rcred = crhold(curthread->td_ucred);
bp->b_wcred = crhold(curthread->td_ucred);
bp->b_blkno = firstaddr;
pbgetbo(bo, bp);
bp->b_vp = vp;
bp->b_bcount = size;
bp->b_bufsize = size;
bp->b_runningbufspace = bp->b_bufsize;
for (i = 0; i < count; i++)
bp->b_pages[i] = m[i];
bp->b_npages = count;
bp->b_pager.pg_reqpage = reqpage;
atomic_add_long(&runningbufspace, bp->b_runningbufspace);
PCPU_INC(cnt.v_vnodein);
PCPU_ADD(cnt.v_vnodepgsin, count);
/* do the input */
bp->b_iooffset = dbtob(bp->b_blkno);
if (iodone != NULL) { /* async */
bp->b_pager.pg_iodone = iodone;
bp->b_caller1 = arg;
bp->b_iodone = vnode_pager_generic_getpages_done_async;
bp->b_flags |= B_ASYNC;
BUF_KERNPROC(bp);
bstrategy(bp);
/* Good bye! */
} else {
bp->b_iodone = bdone;
bstrategy(bp);
bwait(bp, PVM, "vnread");
error = vnode_pager_generic_getpages_done(bp);
for (i = 0; i < bp->b_npages; i++)
bp->b_pages[i] = NULL;
bp->b_vp = NULL;
pbrelbo(bp);
relpbuf(bp, &vnode_pbuf_freecnt);
}
return (error != 0 ? VM_PAGER_ERROR : VM_PAGER_OK);
}
/*
* Lets the VM system know about a change in size for a file.
* We adjust our own internal size and flush any cached pages in
* the associated object that are affected by the size change.
*
* Note: this routine may be invoked as a result of a pager put
* operation (possibly at object termination time), so we must be careful.
*/
void
vnode_pager_setsize(struct vnode *vp, vm_ooffset_t nsize)
{
vm_object_t object;
vm_page_t m;
vm_pindex_t nobjsize;
if ((object = vp->v_object) == NULL)
return;
/* ASSERT_VOP_ELOCKED(vp, "vnode_pager_setsize and not locked vnode"); */
VM_OBJECT_WLOCK(object);
if (object->type == OBJT_DEAD) {
VM_OBJECT_WUNLOCK(object);
return;
}
KASSERT(object->type == OBJT_VNODE,
("not vnode-backed object %p", object));
if (nsize == object->un_pager.vnp.vnp_size) {
/*
* Hasn't changed size
*/
VM_OBJECT_WUNLOCK(object);
return;
}
nobjsize = OFF_TO_IDX(nsize + PAGE_MASK);
if (nsize < object->un_pager.vnp.vnp_size) {
/*
* File has shrunk. Toss any cached pages beyond the new EOF.
*/
if (nobjsize < object->size)
vm_object_page_remove(object, nobjsize, object->size,
0);
/*
* this gets rid of garbage at the end of a page that is now
* only partially backed by the vnode.
*
* XXX for some reason (I don't know yet), if we take a
* completely invalid page and mark it partially valid
* it can screw up NFS reads, so we don't allow the case.
*/
if ((nsize & PAGE_MASK) &&
(m = vm_page_lookup(object, OFF_TO_IDX(nsize))) != NULL &&
m->valid != 0) {
int base = (int)nsize & PAGE_MASK;
int size = PAGE_SIZE - base;
/*
* Clear out partial-page garbage in case
* the page has been mapped.
*/
pmap_zero_page_area(m, base, size);
/*
* Update the valid bits to reflect the blocks that
* have been zeroed. Some of these valid bits may
* have already been set.
*/
vm_page_set_valid_range(m, base, size);
/*
* Round "base" to the next block boundary so that the
* dirty bit for a partially zeroed block is not
* cleared.
*/
base = roundup2(base, DEV_BSIZE);
/*
* Clear out partial-page dirty bits.
*
* note that we do not clear out the valid
* bits. This would prevent bogus_page
* replacement from working properly.
*/
vm_page_clear_dirty(m, base, PAGE_SIZE - base);
} else if ((nsize & PAGE_MASK) &&
vm_page_is_cached(object, OFF_TO_IDX(nsize))) {
vm_page_cache_free(object, OFF_TO_IDX(nsize),
nobjsize);
}
}
object->un_pager.vnp.vnp_size = nsize;
object->size = nobjsize;
VM_OBJECT_WUNLOCK(object);
}
static boolean_t
vnode_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before,
int *after)
{
struct vnode *vp = object->handle;
daddr_t bn;
int err;
daddr_t reqblock;
int poff;
int bsize;
int pagesperblock, blocksperpage;
VM_OBJECT_ASSERT_WLOCKED(object);
/*
* If no vp or vp is doomed or marked transparent to VM, we do not
* have the page.
*/
if (vp == NULL || vp->v_iflag & VI_DOOMED)
return FALSE;
/*
* If the offset is beyond end of file we do
* not have the page.
*/
if (IDX_TO_OFF(pindex) >= object->un_pager.vnp.vnp_size)
return FALSE;
bsize = vp->v_mount->mnt_stat.f_iosize;
pagesperblock = bsize / PAGE_SIZE;
blocksperpage = 0;
if (pagesperblock > 0) {
reqblock = pindex / pagesperblock;
} else {
blocksperpage = (PAGE_SIZE / bsize);
reqblock = pindex * blocksperpage;
}
VM_OBJECT_WUNLOCK(object);
err = VOP_BMAP(vp, reqblock, NULL, &bn, after, before);
VM_OBJECT_WLOCK(object);
if (err)
return TRUE;
if (bn == -1)
return FALSE;
if (pagesperblock > 0) {
poff = pindex - (reqblock * pagesperblock);
if (before) {
*before *= pagesperblock;
*before += poff;
}
if (after) {
/*
* The BMAP vop can report a partial block in the
* 'after', but must not report blocks after EOF.
* Assert the latter, and truncate 'after' in case
* of the former.
*/
KASSERT((reqblock + *after) * pagesperblock <
roundup2(object->size, pagesperblock),
("%s: reqblock %jd after %d size %ju", __func__,
(intmax_t )reqblock, *after,
(uintmax_t )object->size));
*after *= pagesperblock;
*after += pagesperblock - (poff + 1);
if (pindex + *after >= object->size)
*after = object->size - 1 - pindex;
}
} else {
if (before) {
*before /= blocksperpage;
}
if (after) {
*after /= blocksperpage;
}
}
return TRUE;
}
/*
* Allocate (or lookup) pager for a vnode.
* Handle is a vnode pointer.
*
* MPSAFE
*/
vm_object_t
vnode_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot,
vm_ooffset_t offset, struct ucred *cred)
{
vm_object_t object;
struct vnode *vp;
/*
* Pageout to vnode, no can do yet.
*/
if (handle == NULL)
return (NULL);
vp = (struct vnode *) handle;
/*
* If the object is being terminated, wait for it to
* go away.
*/
retry:
while ((object = vp->v_object) != NULL) {
VM_OBJECT_WLOCK(object);
if ((object->flags & OBJ_DEAD) == 0)
break;
vm_object_set_flag(object, OBJ_DISCONNECTWNT);
VM_OBJECT_SLEEP(object, object, PDROP | PVM, "vadead", 0);
}
KASSERT(vp->v_usecount != 0, ("vnode_pager_alloc: no vnode reference"));
if (object == NULL) {
/*
* Add an object of the appropriate size
*/
object = vm_object_allocate(OBJT_VNODE, OFF_TO_IDX(round_page(size)));
object->un_pager.vnp.vnp_size = size;
object->un_pager.vnp.writemappings = 0;
object->handle = handle;
VI_LOCK(vp);
if (vp->v_object != NULL) {
/*
* Object has been created while we were sleeping
*/
VI_UNLOCK(vp);
VM_OBJECT_WLOCK(object);
KASSERT(object->ref_count == 1,
("leaked ref %p %d", object, object->ref_count));
object->type = OBJT_DEAD;
object->ref_count = 0;
VM_OBJECT_WUNLOCK(object);
vm_object_destroy(object);
goto retry;
}
vp->v_object = object;
VI_UNLOCK(vp);
} else {
object->ref_count++;
#if VM_NRESERVLEVEL > 0
vm_object_color(object, 0);
#endif
VM_OBJECT_WUNLOCK(object);
}
vref(vp);
return (object);
}
/*
* This is now called from local media FS's to operate against their
* own vnodes if they fail to implement VOP_PUTPAGES.
*
* This is typically called indirectly via the pageout daemon and
* clustering has already typically occured, so in general we ask the
* underlying filesystem to write the data out asynchronously rather
* then delayed.
*/
int
vnode_pager_generic_putpages(struct vnode *vp, vm_page_t *ma, int bytecount,
int flags, int *rtvals)
{
int i;
vm_object_t object;
vm_page_t m;
int count;
int maxsize, ncount;
vm_ooffset_t poffset;
struct uio auio;
struct iovec aiov;
int error;
int ioflags;
int ppscheck = 0;
static struct timeval lastfail;
static int curfail;
object = vp->v_object;
count = bytecount / PAGE_SIZE;
for (i = 0; i < count; i++)
rtvals[i] = VM_PAGER_ERROR;
if ((int64_t)ma[0]->pindex < 0) {
printf("vnode_pager_putpages: attempt to write meta-data!!! -- 0x%lx(%lx)\n",
(long)ma[0]->pindex, (u_long)ma[0]->dirty);
rtvals[0] = VM_PAGER_BAD;
return VM_PAGER_BAD;
}
maxsize = count * PAGE_SIZE;
ncount = count;
poffset = IDX_TO_OFF(ma[0]->pindex);
/*
* If the page-aligned write is larger then the actual file we
* have to invalidate pages occuring beyond the file EOF. However,
* there is an edge case where a file may not be page-aligned where
* the last page is partially invalid. In this case the filesystem
* may not properly clear the dirty bits for the entire page (which
* could be VM_PAGE_BITS_ALL due to the page having been mmap()d).
* With the page locked we are free to fix-up the dirty bits here.
*
* We do not under any circumstances truncate the valid bits, as
* this will screw up bogus page replacement.
*/
VM_OBJECT_WLOCK(object);
if (maxsize + poffset > object->un_pager.vnp.vnp_size) {
if (object->un_pager.vnp.vnp_size > poffset) {
int pgoff;
maxsize = object->un_pager.vnp.vnp_size - poffset;
ncount = btoc(maxsize);
if ((pgoff = (int)maxsize & PAGE_MASK) != 0) {
/*
* If the object is locked and the following
* conditions hold, then the page's dirty
* field cannot be concurrently changed by a
* pmap operation.
*/
m = ma[ncount - 1];
vm_page_assert_sbusied(m);
KASSERT(!pmap_page_is_write_mapped(m),
("vnode_pager_generic_putpages: page %p is not read-only", m));
vm_page_clear_dirty(m, pgoff, PAGE_SIZE -
pgoff);
}
} else {
maxsize = 0;
ncount = 0;
}
if (ncount < count) {
for (i = ncount; i < count; i++) {
rtvals[i] = VM_PAGER_BAD;
}
}
}
VM_OBJECT_WUNLOCK(object);
/*
* pageouts are already clustered, use IO_ASYNC to force a bawrite()
* rather then a bdwrite() to prevent paging I/O from saturating
* the buffer cache. Dummy-up the sequential heuristic to cause
* large ranges to cluster. If neither IO_SYNC or IO_ASYNC is set,
* the system decides how to cluster.
*/
ioflags = IO_VMIO;
if (flags & (VM_PAGER_PUT_SYNC | VM_PAGER_PUT_INVAL))
ioflags |= IO_SYNC;
else if ((flags & VM_PAGER_CLUSTER_OK) == 0)
ioflags |= IO_ASYNC;
ioflags |= (flags & VM_PAGER_PUT_INVAL) ? IO_INVAL: 0;
ioflags |= IO_SEQMAX << IO_SEQSHIFT;
aiov.iov_base = (caddr_t) 0;
aiov.iov_len = maxsize;
auio.uio_iov = &aiov;
auio.uio_iovcnt = 1;
auio.uio_offset = poffset;
auio.uio_segflg = UIO_NOCOPY;
auio.uio_rw = UIO_WRITE;
auio.uio_resid = maxsize;
auio.uio_td = (struct thread *) 0;
error = VOP_WRITE(vp, &auio, ioflags, curthread->td_ucred);
PCPU_INC(cnt.v_vnodeout);
PCPU_ADD(cnt.v_vnodepgsout, ncount);
if (error) {
if ((ppscheck = ppsratecheck(&lastfail, &curfail, 1)))
printf("vnode_pager_putpages: I/O error %d\n", error);
}
if (auio.uio_resid) {
if (ppscheck || ppsratecheck(&lastfail, &curfail, 1))
printf("vnode_pager_putpages: residual I/O %zd at %lu\n",
auio.uio_resid, (u_long)ma[0]->pindex);
}
for (i = 0; i < ncount; i++) {
rtvals[i] = VM_PAGER_OK;
}
return rtvals[0];
}
/*
* This is now called from local media FS's to operate against their
* own vnodes if they fail to implement VOP_GETPAGES.
*/
int
vnode_pager_generic_getpages(struct vnode *vp, vm_page_t *m, int count,
int *a_rbehind, int *a_rahead, vop_getpages_iodone_t iodone, void *arg)
{
vm_object_t object;
struct bufobj *bo;
struct buf *bp;
off_t foff;
#ifdef INVARIANTS
off_t blkno0;
#endif
int bsize, pagesperblock, *freecnt;
int error, before, after, rbehind, rahead, poff, i;
int bytecount, secmask;
KASSERT(vp->v_type != VCHR && vp->v_type != VBLK,
("%s does not support devices", __func__));
if (vp->v_iflag & VI_DOOMED)
return (VM_PAGER_BAD);
object = vp->v_object;
foff = IDX_TO_OFF(m[0]->pindex);
bsize = vp->v_mount->mnt_stat.f_iosize;
pagesperblock = bsize / PAGE_SIZE;
KASSERT(foff < object->un_pager.vnp.vnp_size,
("%s: page %p offset beyond vp %p size", __func__, m[0], vp));
KASSERT(count <= sizeof(bp->b_pages),
("%s: requested %d pages", __func__, count));
/*
* The last page has valid blocks. Invalid part can only
* exist at the end of file, and the page is made fully valid
* by zeroing in vm_pager_get_pages().
*/
if (m[count - 1]->valid != 0 && --count == 0) {
if (iodone != NULL)
iodone(arg, m, 1, 0);
return (VM_PAGER_OK);
}
/*
* Synchronous and asynchronous paging operations use different
* free pbuf counters. This is done to avoid asynchronous requests
* to consume all pbufs.
* Allocate the pbuf at the very beginning of the function, so that
* if we are low on certain kind of pbufs don't even proceed to BMAP,
* but sleep.
*/
freecnt = iodone != NULL ?
&vnode_async_pbuf_freecnt : &vnode_pbuf_freecnt;
bp = getpbuf(freecnt);
/*
* Get the underlying device blocks for the file with VOP_BMAP().
* If the file system doesn't support VOP_BMAP, use old way of
* getting pages via VOP_READ.
*/
error = VOP_BMAP(vp, foff / bsize, &bo, &bp->b_blkno, &after, &before);
if (error == EOPNOTSUPP) {
relpbuf(bp, freecnt);
VM_OBJECT_WLOCK(object);
for (i = 0; i < count; i++) {
PCPU_INC(cnt.v_vnodein);
PCPU_INC(cnt.v_vnodepgsin);
error = vnode_pager_input_old(object, m[i]);
if (error)
break;
}
VM_OBJECT_WUNLOCK(object);
return (error);
} else if (error != 0) {
relpbuf(bp, freecnt);
return (VM_PAGER_ERROR);
}
/*
* If the file system supports BMAP, but blocksize is smaller
* than a page size, then use special small filesystem code.
*/
if (pagesperblock == 0) {
relpbuf(bp, freecnt);
for (i = 0; i < count; i++) {
PCPU_INC(cnt.v_vnodein);
PCPU_INC(cnt.v_vnodepgsin);
error = vnode_pager_input_smlfs(object, m[i]);
if (error)
break;
}
return (error);
}
/*
* A sparse file can be encountered only for a single page request,
* which may not be preceded by call to vm_pager_haspage().
*/
if (bp->b_blkno == -1) {
KASSERT(count == 1,
("%s: array[%d] request to a sparse file %p", __func__,
count, vp));
relpbuf(bp, freecnt);
pmap_zero_page(m[0]);
KASSERT(m[0]->dirty == 0, ("%s: page %p is dirty",
__func__, m[0]));
VM_OBJECT_WLOCK(object);
m[0]->valid = VM_PAGE_BITS_ALL;
VM_OBJECT_WUNLOCK(object);
return (VM_PAGER_OK);
}
#ifdef INVARIANTS
blkno0 = bp->b_blkno;
#endif
bp->b_blkno += (foff % bsize) / DEV_BSIZE;
/* Recalculate blocks available after/before to pages. */
poff = (foff % bsize) / PAGE_SIZE;
before *= pagesperblock;
before += poff;
after *= pagesperblock;
after += pagesperblock - (poff + 1);
if (m[0]->pindex + after >= object->size)
after = object->size - 1 - m[0]->pindex;
KASSERT(count <= after + 1, ("%s: %d pages asked, can do only %d",
__func__, count, after + 1));
after -= count - 1;
/* Trim requested rbehind/rahead to possible values. */
rbehind = a_rbehind ? *a_rbehind : 0;
rahead = a_rahead ? *a_rahead : 0;
rbehind = min(rbehind, before);
rbehind = min(rbehind, m[0]->pindex);
rahead = min(rahead, after);
rahead = min(rahead, object->size - m[count - 1]->pindex);
/*
* Check that total amount of pages fit into buf. Trim rbehind and
* rahead evenly if not.
*/
if (rbehind + rahead + count > nitems(bp->b_pages)) {
int trim, sum;
trim = rbehind + rahead + count - nitems(bp->b_pages) + 1;
sum = rbehind + rahead;
if (rbehind == before) {
/* Roundup rbehind trim to block size. */
rbehind -= roundup(trim * rbehind / sum, pagesperblock);
if (rbehind < 0)
rbehind = 0;
} else
rbehind -= trim * rbehind / sum;
rahead -= trim * rahead / sum;
}
KASSERT(rbehind + rahead + count <= nitems(bp->b_pages),
("%s: behind %d ahead %d count %d", __func__,
rbehind, rahead, count));
/*
* Fill in the bp->b_pages[] array with requested and optional
* read behind or read ahead pages. Read behind pages are looked
* up in a backward direction, down to a first cached page. Same
* for read ahead pages, but there is no need to shift the array
* in case of encountering a cached page.
*/
i = bp->b_npages = 0;
if (rbehind) {
vm_pindex_t startpindex, tpindex;
vm_page_t p;
VM_OBJECT_WLOCK(object);
startpindex = m[0]->pindex - rbehind;
if ((p = TAILQ_PREV(m[0], pglist, listq)) != NULL &&
p->pindex >= startpindex)
startpindex = p->pindex + 1;
/* tpindex is unsigned; beware of numeric underflow. */
for (tpindex = m[0]->pindex - 1;
tpindex >= startpindex && tpindex < m[0]->pindex;
tpindex--, i++) {
p = vm_page_alloc(object, tpindex, VM_ALLOC_NORMAL);
if (p == NULL) {
/* Shift the array. */
for (int j = 0; j < i; j++)
bp->b_pages[j] = bp->b_pages[j +
tpindex + 1 - startpindex];
break;
}
bp->b_pages[tpindex - startpindex] = p;
}
bp->b_pgbefore = i;
bp->b_npages += i;
bp->b_blkno -= IDX_TO_OFF(i) / DEV_BSIZE;
} else
bp->b_pgbefore = 0;
/* Requested pages. */
for (int j = 0; j < count; j++, i++)
bp->b_pages[i] = m[j];
bp->b_npages += count;
if (rahead) {
vm_pindex_t endpindex, tpindex;
vm_page_t p;
if (!VM_OBJECT_WOWNED(object))
VM_OBJECT_WLOCK(object);
endpindex = m[count - 1]->pindex + rahead + 1;
if ((p = TAILQ_NEXT(m[count - 1], listq)) != NULL &&
p->pindex < endpindex)
endpindex = p->pindex;
if (endpindex > object->size)
endpindex = object->size;
for (tpindex = m[count - 1]->pindex + 1;
tpindex < endpindex; i++, tpindex++) {
p = vm_page_alloc(object, tpindex, VM_ALLOC_NORMAL);
if (p == NULL)
break;
bp->b_pages[i] = p;
}
bp->b_pgafter = i - bp->b_npages;
bp->b_npages = i;
} else
bp->b_pgafter = 0;
if (VM_OBJECT_WOWNED(object))
VM_OBJECT_WUNLOCK(object);
/* Report back actual behind/ahead read. */
if (a_rbehind)
*a_rbehind = bp->b_pgbefore;
if (a_rahead)
*a_rahead = bp->b_pgafter;
#ifdef INVARIANTS
KASSERT(bp->b_npages <= nitems(bp->b_pages),
("%s: buf %p overflowed", __func__, bp));
for (int j = 1, prev = 1; j < bp->b_npages; j++) {
if (bp->b_pages[j] == bogus_page)
continue;
KASSERT(bp->b_pages[j]->pindex - bp->b_pages[prev]->pindex ==
j - prev, ("%s: pages array not consecutive, bp %p",
__func__, bp));
prev = j;
}
#endif
/*
* Recalculate first offset and bytecount with regards to read behind.
* Truncate bytecount to vnode real size and round up physical size
* for real devices.
*/
foff = IDX_TO_OFF(bp->b_pages[0]->pindex);
bytecount = bp->b_npages << PAGE_SHIFT;
if ((foff + bytecount) > object->un_pager.vnp.vnp_size)
bytecount = object->un_pager.vnp.vnp_size - foff;
secmask = bo->bo_bsize - 1;
KASSERT(secmask < PAGE_SIZE && secmask > 0,
("%s: sector size %d too large", __func__, secmask + 1));
bytecount = (bytecount + secmask) & ~secmask;
/*
* And map the pages to be read into the kva, if the filesystem
* requires mapped buffers.
*/
if ((vp->v_mount->mnt_kern_flag & MNTK_UNMAPPED_BUFS) != 0 &&
unmapped_buf_allowed) {
bp->b_data = unmapped_buf;
bp->b_offset = 0;
} else {
bp->b_data = bp->b_kvabase;
pmap_qenter((vm_offset_t)bp->b_data, bp->b_pages, bp->b_npages);
}
/* Build a minimal buffer header. */
bp->b_iocmd = BIO_READ;
KASSERT(bp->b_rcred == NOCRED, ("leaking read ucred"));
KASSERT(bp->b_wcred == NOCRED, ("leaking write ucred"));
bp->b_rcred = crhold(curthread->td_ucred);
bp->b_wcred = crhold(curthread->td_ucred);
pbgetbo(bo, bp);
bp->b_vp = vp;
bp->b_bcount = bp->b_bufsize = bp->b_runningbufspace = bytecount;
bp->b_iooffset = dbtob(bp->b_blkno);
KASSERT(IDX_TO_OFF(m[0]->pindex - bp->b_pages[0]->pindex) ==
(blkno0 - bp->b_blkno) * DEV_BSIZE +
IDX_TO_OFF(m[0]->pindex) % bsize,
("wrong offsets bsize %d m[0] %ju b_pages[0] %ju "
"blkno0 %ju b_blkno %ju", bsize,
(uintmax_t)m[0]->pindex, (uintmax_t)bp->b_pages[0]->pindex,
(uintmax_t)blkno0, (uintmax_t)bp->b_blkno));
atomic_add_long(&runningbufspace, bp->b_runningbufspace);
PCPU_INC(cnt.v_vnodein);
PCPU_ADD(cnt.v_vnodepgsin, bp->b_npages);
if (iodone != NULL) { /* async */
bp->b_pgiodone = iodone;
bp->b_caller1 = arg;
bp->b_iodone = vnode_pager_generic_getpages_done_async;
bp->b_flags |= B_ASYNC;
BUF_KERNPROC(bp);
bstrategy(bp);
return (VM_PAGER_OK);
} else {
bp->b_iodone = bdone;
bstrategy(bp);
bwait(bp, PVM, "vnread");
error = vnode_pager_generic_getpages_done(bp);
for (i = 0; i < bp->b_npages; i++)
bp->b_pages[i] = NULL;
bp->b_vp = NULL;
pbrelbo(bp);
relpbuf(bp, &vnode_pbuf_freecnt);
return (error != 0 ? VM_PAGER_ERROR : VM_PAGER_OK);
}
}
