void
vnode_pager_undirty_pages(vm_page_t *ma, int *rtvals, int written)
{
vm_object_t obj;
int i, pos;
if (written == 0)
return;
obj = ma[0]->object;
VM_OBJECT_WLOCK(obj);
for (i = 0, pos = 0; pos < written; i++, pos += PAGE_SIZE) {
if (pos < trunc_page(written)) {
rtvals[i] = VM_PAGER_OK;
vm_page_undirty(ma[i]);
} else {
/* Partially written page. */
rtvals[i] = VM_PAGER_AGAIN;
vm_page_clear_dirty(ma[i], 0, written & PAGE_MASK);
}
}
VM_OBJECT_WUNLOCK(obj);
}
static int
tmpfs_mappedwrite(vm_object_t vobj, vm_object_t tobj, size_t len, struct uio *uio)
{
vm_pindex_t idx;
vm_page_t vpg, tpg;
vm_offset_t offset;
off_t addr;
size_t tlen;
int error, rv;
error = 0;
addr = uio->uio_offset;
idx = OFF_TO_IDX(addr);
offset = addr & PAGE_MASK;
tlen = MIN(PAGE_SIZE - offset, len);
if ((vobj == NULL) ||
(vobj->resident_page_count == 0 && vobj->cache == NULL)) {
vpg = NULL;
goto nocache;
}
VM_OBJECT_LOCK(vobj);
lookupvpg:
if (((vpg = vm_page_lookup(vobj, idx)) != NULL) &&
vm_page_is_valid(vpg, offset, tlen)) {
if ((vpg->oflags & VPO_BUSY) != 0) {
/*
* Reference the page before unlocking and sleeping so
* that the page daemon is less likely to reclaim it.
*/
vm_page_reference(vpg);
vm_page_sleep(vpg, "tmfsmw");
goto lookupvpg;
}
vm_page_busy(vpg);
vm_page_undirty(vpg);
VM_OBJECT_UNLOCK(vobj);
error = uiomove_fromphys(&vpg, offset, tlen, uio);
} else {
if (__predict_false(vobj->cache != NULL))
vm_page_cache_free(vobj, idx, idx + 1);
VM_OBJECT_UNLOCK(vobj);
vpg = NULL;
}
nocache:
VM_OBJECT_LOCK(tobj);
tpg = vm_page_grab(tobj, idx, VM_ALLOC_WIRED |
VM_ALLOC_NORMAL | VM_ALLOC_RETRY);
if (tpg->valid != VM_PAGE_BITS_ALL) {
if (vm_pager_has_page(tobj, idx, NULL, NULL)) {
rv = vm_pager_get_pages(tobj, &tpg, 1, 0);
if (rv != VM_PAGER_OK) {
vm_page_lock(tpg);
vm_page_free(tpg);
vm_page_unlock(tpg);
error = EIO;
goto out;
}
} else
vm_page_zero_invalid(tpg, TRUE);
}
VM_OBJECT_UNLOCK(tobj);
if (vpg == NULL)
error = uiomove_fromphys(&tpg, offset, tlen, uio);
else {
KASSERT(vpg->valid == VM_PAGE_BITS_ALL, ("parts of vpg invalid"));
pmap_copy_page(vpg, tpg);
}
VM_OBJECT_LOCK(tobj);
if (error == 0) {
KASSERT(tpg->valid == VM_PAGE_BITS_ALL,
("parts of tpg invalid"));
vm_page_dirty(tpg);
}
vm_page_lock(tpg);
vm_page_unwire(tpg, TRUE);
vm_page_unlock(tpg);
vm_page_wakeup(tpg);
out:
VM_OBJECT_UNLOCK(tobj);
if (vpg != NULL) {
VM_OBJECT_LOCK(vobj);
vm_page_wakeup(vpg);
VM_OBJECT_UNLOCK(vobj);
}
return (error);
}
/*
struct vnop_putpages_args {
struct vnode *a_vp;
vm_page_t *a_m;
int a_count;
int a_sync;
int *a_rtvals;
vm_ooffset_t a_offset;
};
*/
static int
fuse_vnop_putpages(struct vop_putpages_args *ap)
{
struct uio uio;
struct iovec iov;
vm_offset_t kva;
struct buf *bp;
int i, error, npages, count;
off_t offset;
int *rtvals;
struct vnode *vp;
struct thread *td;
struct ucred *cred;
vm_page_t *pages;
vm_ooffset_t fsize;
FS_DEBUG2G("heh\n");
vp = ap->a_vp;
KASSERT(vp->v_object, ("objectless vp passed to putpages"));
fsize = vp->v_object->un_pager.vnp.vnp_size;
td = curthread; /* XXX */
cred = curthread->td_ucred; /* XXX */
pages = ap->a_m;
count = ap->a_count;
rtvals = ap->a_rtvals;
npages = btoc(count);
offset = IDX_TO_OFF(pages[0]->pindex);
if (!fsess_opt_mmap(vnode_mount(vp))) {
FS_DEBUG("called on non-cacheable vnode??\n");
}
for (i = 0; i < npages; i++)
rtvals[i] = VM_PAGER_AGAIN;
/*
* When putting pages, do not extend file past EOF.
*/
if (offset + count > fsize) {
count = fsize - offset;
if (count < 0)
count = 0;
}
/*
* We use only the kva address for the buffer, but this is extremely
* convienient and fast.
*/
bp = getpbuf(&fuse_pbuf_freecnt);
kva = (vm_offset_t)bp->b_data;
pmap_qenter(kva, pages, npages);
PCPU_INC(cnt.v_vnodeout);
PCPU_ADD(cnt.v_vnodepgsout, count);
iov.iov_base = (caddr_t)kva;
iov.iov_len = count;
uio.uio_iov = &iov;
uio.uio_iovcnt = 1;
uio.uio_offset = offset;
uio.uio_resid = count;
uio.uio_segflg = UIO_SYSSPACE;
uio.uio_rw = UIO_WRITE;
uio.uio_td = td;
error = fuse_io_dispatch(vp, &uio, IO_DIRECT, cred);
pmap_qremove(kva, npages);
relpbuf(bp, &fuse_pbuf_freecnt);
if (!error) {
int nwritten = round_page(count - uio.uio_resid) / PAGE_SIZE;
for (i = 0; i < nwritten; i++) {
rtvals[i] = VM_PAGER_OK;
VM_OBJECT_WLOCK(pages[i]->object);
vm_page_undirty(pages[i]);
VM_OBJECT_WUNLOCK(pages[i]->object);
}
}
return rtvals[0];
}
/*
* Vnode op for VM putpages.
* possible bug: all IO done in sync mode
* Note that vop_close always invalidate pages before close, so it's
* not necessary to open vnode.
*
* nwfs_putpages(struct vnode *a_vp, vm_page_t *a_m, int a_count,
* int a_sync, int *a_rtvals, vm_ooffset_t a_offset)
*/
int
nwfs_putpages(struct vop_putpages_args *ap)
{
int error;
struct thread *td = curthread; /* XXX */
struct vnode *vp = ap->a_vp;
struct ucred *cred;
#ifndef NWFS_RWCACHE
KKASSERT(td->td_proc);
cred = td->td_proc->p_ucred; /* XXX */
VOP_OPEN(vp, FWRITE, cred, NULL);
error = vnode_pager_generic_putpages(ap->a_vp, ap->a_m, ap->a_count,
ap->a_sync, ap->a_rtvals);
VOP_CLOSE(vp, FWRITE, cred);
return error;
#else
struct uio uio;
struct iovec iov;
vm_offset_t kva;
struct buf *bp;
int i, npages, count;
int *rtvals;
struct nwmount *nmp;
struct nwnode *np;
vm_page_t *pages;
KKASSERT(td->td_proc);
cred = td->td_proc->p_ucred; /* XXX */
/* VOP_OPEN(vp, FWRITE, cred, NULL);*/
np = VTONW(vp);
nmp = VFSTONWFS(vp->v_mount);
pages = ap->a_m;
count = ap->a_count;
rtvals = ap->a_rtvals;
npages = btoc(count);
for (i = 0; i < npages; i++) {
rtvals[i] = VM_PAGER_AGAIN;
}
bp = getpbuf_kva(&nwfs_pbuf_freecnt);
kva = (vm_offset_t) bp->b_data;
pmap_qenter(kva, pages, npages);
iov.iov_base = (caddr_t) kva;
iov.iov_len = count;
uio.uio_iov = &iov;
uio.uio_iovcnt = 1;
uio.uio_offset = IDX_TO_OFF(pages[0]->pindex);
uio.uio_resid = count;
uio.uio_segflg = UIO_SYSSPACE;
uio.uio_rw = UIO_WRITE;
uio.uio_td = td;
NCPVNDEBUG("ofs=%d,resid=%d\n",(int)uio.uio_offset, uio.uio_resid);
error = ncp_write(NWFSTOCONN(nmp), &np->n_fh, &uio, cred);
/* VOP_CLOSE(vp, FWRITE, cred);*/
NCPVNDEBUG("paged write done: %d\n", error);
pmap_qremove(kva, npages);
relpbuf(bp, &nwfs_pbuf_freecnt);
if (!error) {
int nwritten = round_page(count - uio.uio_resid) / PAGE_SIZE;
for (i = 0; i < nwritten; i++) {
rtvals[i] = VM_PAGER_OK;
vm_page_undirty(pages[i]);
}
}
return rtvals[0];
#endif /* NWFS_RWCACHE */
}
/*
* Vnode op for VM putpages.
* possible bug: all IO done in sync mode
* Note that vop_close always invalidate pages before close, so it's
* not necessary to open vnode.
*
* smbfs_putpages(struct vnode *a_vp, vm_page_t *a_m, int a_count, int a_sync,
* int *a_rtvals, vm_ooffset_t a_offset)
*/
int
smbfs_putpages(struct vop_putpages_args *ap)
{
int error;
struct vnode *vp = ap->a_vp;
struct thread *td = curthread; /* XXX */
struct ucred *cred;
#ifdef SMBFS_RWGENERIC
KKASSERT(td->td_proc);
cred = td->td_proc->p_ucred;
VOP_OPEN(vp, FWRITE, cred, NULL);
error = vop_stdputpages(ap);
VOP_CLOSE(vp, FWRITE, cred);
return error;
#else
struct uio uio;
struct iovec iov;
vm_offset_t kva;
struct buf *bp;
int i, npages, count;
int doclose;
int *rtvals;
struct smbmount *smp;
struct smbnode *np;
struct smb_cred scred;
vm_page_t *pages;
KKASSERT(td->td_proc);
cred = td->td_proc->p_ucred;
/* VOP_OPEN(vp, FWRITE, cred, NULL);*/
np = VTOSMB(vp);
smp = VFSTOSMBFS(vp->v_mount);
pages = ap->a_m;
count = ap->a_count;
rtvals = ap->a_rtvals;
npages = btoc(count);
for (i = 0; i < npages; i++) {
rtvals[i] = VM_PAGER_AGAIN;
}
bp = getpbuf_kva(&smbfs_pbuf_freecnt);
kva = (vm_offset_t) bp->b_data;
pmap_qenter(kva, pages, npages);
iov.iov_base = (caddr_t) kva;
iov.iov_len = count;
uio.uio_iov = &iov;
uio.uio_iovcnt = 1;
uio.uio_offset = IDX_TO_OFF(pages[0]->pindex);
uio.uio_resid = count;
uio.uio_segflg = UIO_SYSSPACE;
uio.uio_rw = UIO_WRITE;
uio.uio_td = td;
SMBVDEBUG("ofs=%d,resid=%d\n",(int)uio.uio_offset, uio.uio_resid);
smb_makescred(&scred, td, cred);
/*
* This is kinda nasty. Since smbfs is physically closing the
* fid on close(), we have to reopen it if necessary. There are
* other races here too, such as if another process opens the same
* file while we are blocked in read, or the file is open read-only
* XXX
*/
error = 0;
doclose = 0;
if (np->n_opencount == 0) {
error = smbfs_smb_open(np, SMB_AM_OPENRW, &scred);
if (error == 0)
doclose = 1;
}
if (error == 0)
error = smb_write(smp->sm_share, np->n_fid, &uio, &scred);
if (doclose)
smbfs_smb_close(smp->sm_share, np->n_fid, NULL, &scred);
/* VOP_CLOSE(vp, FWRITE, cred);*/
SMBVDEBUG("paged write done: %d\n", error);
pmap_qremove(kva, npages);
relpbuf(bp, &smbfs_pbuf_freecnt);
if (!error) {
int nwritten = round_page(count - uio.uio_resid) / PAGE_SIZE;
for (i = 0; i < nwritten; i++) {
rtvals[i] = VM_PAGER_OK;
vm_page_undirty(pages[i]);
}
}
return rtvals[0];
#endif /* SMBFS_RWGENERIC */
}
/*
* This is now called from local media FS's to operate against their
* own vnodes if they fail to implement VOP_PUTPAGES.
*
* This is typically called indirectly via the pageout daemon and
* clustering has already typically occured, so in general we ask the
* underlying filesystem to write the data out asynchronously rather
* then delayed.
*/
int
vnode_pager_generic_putpages(struct vnode *vp, vm_page_t *m, int bytecount,
int flags, int *rtvals)
{
int i;
int maxsize, ncount, count;
vm_ooffset_t poffset;
struct uio auio;
struct iovec aiov;
int error;
int ioflags;
count = bytecount / PAGE_SIZE;
for (i = 0; i < count; i++)
rtvals[i] = VM_PAGER_AGAIN;
if ((int) m[0]->pindex < 0) {
kprintf("vnode_pager_putpages: attempt to write meta-data!!! -- 0x%lx(%x)\n",
(long)m[0]->pindex, m[0]->dirty);
rtvals[0] = VM_PAGER_BAD;
return VM_PAGER_BAD;
}
maxsize = count * PAGE_SIZE;
ncount = count;
poffset = IDX_TO_OFF(m[0]->pindex);
/*
* If the page-aligned write is larger then the actual file we
* have to invalidate pages occuring beyond the file EOF.
*
* If the file EOF resides in the middle of a page we still clear
* all of that page's dirty bits later on. If we didn't it would
* endlessly re-write.
*
* We do not under any circumstances truncate the valid bits, as
* this will screw up bogus page replacement.
*
* The caller has already read-protected the pages. The VFS must
* use the buffer cache to wrap the pages. The pages might not
* be immediately flushed by the buffer cache but once under its
* control the pages themselves can wind up being marked clean
* and their covering buffer cache buffer can be marked dirty.
*/
if (poffset + maxsize > vp->v_filesize) {
if (poffset < vp->v_filesize) {
maxsize = vp->v_filesize - poffset;
ncount = btoc(maxsize);
} else {
maxsize = 0;
ncount = 0;
}
if (ncount < count) {
for (i = ncount; i < count; i++) {
rtvals[i] = VM_PAGER_BAD;
}
}
}
/*
* pageouts are already clustered, use IO_ASYNC to force a bawrite()
* rather then a bdwrite() to prevent paging I/O from saturating
* the buffer cache. Dummy-up the sequential heuristic to cause
* large ranges to cluster. If neither IO_SYNC or IO_ASYNC is set,
* the system decides how to cluster.
*/
ioflags = IO_VMIO;
if (flags & (VM_PAGER_PUT_SYNC | VM_PAGER_PUT_INVAL))
ioflags |= IO_SYNC;
else if ((flags & VM_PAGER_CLUSTER_OK) == 0)
ioflags |= IO_ASYNC;
ioflags |= (flags & VM_PAGER_PUT_INVAL) ? IO_INVAL: 0;
ioflags |= IO_SEQMAX << IO_SEQSHIFT;
aiov.iov_base = (caddr_t) 0;
aiov.iov_len = maxsize;
auio.uio_iov = &aiov;
auio.uio_iovcnt = 1;
auio.uio_offset = poffset;
auio.uio_segflg = UIO_NOCOPY;
auio.uio_rw = UIO_WRITE;
auio.uio_resid = maxsize;
auio.uio_td = NULL;
error = VOP_WRITE(vp, &auio, ioflags, proc0.p_ucred);
mycpu->gd_cnt.v_vnodeout++;
mycpu->gd_cnt.v_vnodepgsout += ncount;
if (error) {
krateprintf(&vbadrate,
"vnode_pager_putpages: I/O error %d\n", error);
}
if (auio.uio_resid) {
krateprintf(&vresrate,
"vnode_pager_putpages: residual I/O %zd at %lu\n",
auio.uio_resid, (u_long)m[0]->pindex);
}
if (error == 0) {
for (i = 0; i < ncount; i++) {
rtvals[i] = VM_PAGER_OK;
vm_page_undirty(m[i]);
}
}
return rtvals[0];
}
/*
* spec_getpages() - get pages associated with device vnode.
*
* Note that spec_read and spec_write do not use the buffer cache, so we
* must fully implement getpages here.
*/
static int
devfs_spec_getpages(struct vop_getpages_args *ap)
{
vm_offset_t kva;
int error;
int i, pcount, size;
struct buf *bp;
vm_page_t m;
vm_ooffset_t offset;
int toff, nextoff, nread;
struct vnode *vp = ap->a_vp;
int blksiz;
int gotreqpage;
error = 0;
pcount = round_page(ap->a_count) / PAGE_SIZE;
/*
* Calculate the offset of the transfer and do sanity check.
*/
offset = IDX_TO_OFF(ap->a_m[0]->pindex) + ap->a_offset;
/*
* Round up physical size for real devices. We cannot round using
* v_mount's block size data because v_mount has nothing to do with
* the device. i.e. it's usually '/dev'. We need the physical block
* size for the device itself.
*
* We can't use v_rdev->si_mountpoint because it only exists when the
* block device is mounted. However, we can use v_rdev.
*/
if (vn_isdisk(vp, NULL))
blksiz = vp->v_rdev->si_bsize_phys;
else
blksiz = DEV_BSIZE;
size = (ap->a_count + blksiz - 1) & ~(blksiz - 1);
bp = getpbuf_kva(NULL);
kva = (vm_offset_t)bp->b_data;
/*
* Map the pages to be read into the kva.
*/
pmap_qenter(kva, ap->a_m, pcount);
/* Build a minimal buffer header. */
bp->b_cmd = BUF_CMD_READ;
bp->b_bcount = size;
bp->b_resid = 0;
bsetrunningbufspace(bp, size);
bp->b_bio1.bio_offset = offset;
bp->b_bio1.bio_done = devfs_spec_getpages_iodone;
mycpu->gd_cnt.v_vnodein++;
mycpu->gd_cnt.v_vnodepgsin += pcount;
/* Do the input. */
vn_strategy(ap->a_vp, &bp->b_bio1);
crit_enter();
/* We definitely need to be at splbio here. */
while (bp->b_cmd != BUF_CMD_DONE)
tsleep(bp, 0, "spread", 0);
crit_exit();
if (bp->b_flags & B_ERROR) {
if (bp->b_error)
error = bp->b_error;
else
error = EIO;
}
/*
* If EOF is encountered we must zero-extend the result in order
* to ensure that the page does not contain garabge. When no
* error occurs, an early EOF is indicated if b_bcount got truncated.
* b_resid is relative to b_bcount and should be 0, but some devices
* might indicate an EOF with b_resid instead of truncating b_bcount.
*/
nread = bp->b_bcount - bp->b_resid;
if (nread < ap->a_count)
bzero((caddr_t)kva + nread, ap->a_count - nread);
pmap_qremove(kva, pcount);
gotreqpage = 0;
for (i = 0, toff = 0; i < pcount; i++, toff = nextoff) {
nextoff = toff + PAGE_SIZE;
m = ap->a_m[i];
m->flags &= ~PG_ZERO;
/*
* NOTE: vm_page_undirty/clear_dirty etc do not clear the
* pmap modified bit. pmap modified bit should have
* already been cleared.
*/
if (nextoff <= nread) {
m->valid = VM_PAGE_BITS_ALL;
vm_page_undirty(m);
} else if (toff < nread) {
/*
* Since this is a VM request, we have to supply the
* unaligned offset to allow vm_page_set_valid()
* to zero sub-DEV_BSIZE'd portions of the page.
*/
vm_page_set_valid(m, 0, nread - toff);
vm_page_clear_dirty_end_nonincl(m, 0, nread - toff);
} else {
m->valid = 0;
vm_page_undirty(m);
}
if (i != ap->a_reqpage) {
/*
* Just in case someone was asking for this page we
* now tell them that it is ok to use.
*/
if (!error || (m->valid == VM_PAGE_BITS_ALL)) {
if (m->valid) {
if (m->flags & PG_REFERENCED) {
vm_page_activate(m);
} else {
vm_page_deactivate(m);
}
vm_page_wakeup(m);
} else {
vm_page_free(m);
}
} else {
vm_page_free(m);
}
} else if (m->valid) {
gotreqpage = 1;
/*
* Since this is a VM request, we need to make the
* entire page presentable by zeroing invalid sections.
*/
if (m->valid != VM_PAGE_BITS_ALL)
vm_page_zero_invalid(m, FALSE);
}
}
if (!gotreqpage) {
m = ap->a_m[ap->a_reqpage];
devfs_debug(DEVFS_DEBUG_WARNING,
"spec_getpages:(%s) I/O read failure: (error=%d) bp %p vp %p\n",
devtoname(vp->v_rdev), error, bp, bp->b_vp);
devfs_debug(DEVFS_DEBUG_WARNING,
" size: %d, resid: %d, a_count: %d, valid: 0x%x\n",
size, bp->b_resid, ap->a_count, m->valid);
devfs_debug(DEVFS_DEBUG_WARNING,
" nread: %d, reqpage: %d, pindex: %lu, pcount: %d\n",
nread, ap->a_reqpage, (u_long)m->pindex, pcount);
/*
* Free the buffer header back to the swap buffer pool.
*/
relpbuf(bp, NULL);
return VM_PAGER_ERROR;
}
/*
* Free the buffer header back to the swap buffer pool.
*/
relpbuf(bp, NULL);
if (DEVFS_NODE(ap->a_vp))
nanotime(&DEVFS_NODE(ap->a_vp)->mtime);
return VM_PAGER_OK;
}