void vnode_pager_undirty_pages(vm_page_t *ma, int *rtvals, int written) { vm_object_t obj; int i, pos; if (written == 0) return; obj = ma[0]->object; VM_OBJECT_WLOCK(obj); for (i = 0, pos = 0; pos < written; i++, pos += PAGE_SIZE) { if (pos < trunc_page(written)) { rtvals[i] = VM_PAGER_OK; vm_page_undirty(ma[i]); } else { /* Partially written page. */ rtvals[i] = VM_PAGER_AGAIN; vm_page_clear_dirty(ma[i], 0, written & PAGE_MASK); } } VM_OBJECT_WUNLOCK(obj); }
static int tmpfs_mappedwrite(vm_object_t vobj, vm_object_t tobj, size_t len, struct uio *uio) { vm_pindex_t idx; vm_page_t vpg, tpg; vm_offset_t offset; off_t addr; size_t tlen; int error, rv; error = 0; addr = uio->uio_offset; idx = OFF_TO_IDX(addr); offset = addr & PAGE_MASK; tlen = MIN(PAGE_SIZE - offset, len); if ((vobj == NULL) || (vobj->resident_page_count == 0 && vobj->cache == NULL)) { vpg = NULL; goto nocache; } VM_OBJECT_LOCK(vobj); lookupvpg: if (((vpg = vm_page_lookup(vobj, idx)) != NULL) && vm_page_is_valid(vpg, offset, tlen)) { if ((vpg->oflags & VPO_BUSY) != 0) { /* * Reference the page before unlocking and sleeping so * that the page daemon is less likely to reclaim it. */ vm_page_reference(vpg); vm_page_sleep(vpg, "tmfsmw"); goto lookupvpg; } vm_page_busy(vpg); vm_page_undirty(vpg); VM_OBJECT_UNLOCK(vobj); error = uiomove_fromphys(&vpg, offset, tlen, uio); } else { if (__predict_false(vobj->cache != NULL)) vm_page_cache_free(vobj, idx, idx + 1); VM_OBJECT_UNLOCK(vobj); vpg = NULL; } nocache: VM_OBJECT_LOCK(tobj); tpg = vm_page_grab(tobj, idx, VM_ALLOC_WIRED | VM_ALLOC_NORMAL | VM_ALLOC_RETRY); if (tpg->valid != VM_PAGE_BITS_ALL) { if (vm_pager_has_page(tobj, idx, NULL, NULL)) { rv = vm_pager_get_pages(tobj, &tpg, 1, 0); if (rv != VM_PAGER_OK) { vm_page_lock(tpg); vm_page_free(tpg); vm_page_unlock(tpg); error = EIO; goto out; } } else vm_page_zero_invalid(tpg, TRUE); } VM_OBJECT_UNLOCK(tobj); if (vpg == NULL) error = uiomove_fromphys(&tpg, offset, tlen, uio); else { KASSERT(vpg->valid == VM_PAGE_BITS_ALL, ("parts of vpg invalid")); pmap_copy_page(vpg, tpg); } VM_OBJECT_LOCK(tobj); if (error == 0) { KASSERT(tpg->valid == VM_PAGE_BITS_ALL, ("parts of tpg invalid")); vm_page_dirty(tpg); } vm_page_lock(tpg); vm_page_unwire(tpg, TRUE); vm_page_unlock(tpg); vm_page_wakeup(tpg); out: VM_OBJECT_UNLOCK(tobj); if (vpg != NULL) { VM_OBJECT_LOCK(vobj); vm_page_wakeup(vpg); VM_OBJECT_UNLOCK(vobj); } return (error); }
/* struct vnop_putpages_args { struct vnode *a_vp; vm_page_t *a_m; int a_count; int a_sync; int *a_rtvals; vm_ooffset_t a_offset; }; */ static int fuse_vnop_putpages(struct vop_putpages_args *ap) { struct uio uio; struct iovec iov; vm_offset_t kva; struct buf *bp; int i, error, npages, count; off_t offset; int *rtvals; struct vnode *vp; struct thread *td; struct ucred *cred; vm_page_t *pages; vm_ooffset_t fsize; FS_DEBUG2G("heh\n"); vp = ap->a_vp; KASSERT(vp->v_object, ("objectless vp passed to putpages")); fsize = vp->v_object->un_pager.vnp.vnp_size; td = curthread; /* XXX */ cred = curthread->td_ucred; /* XXX */ pages = ap->a_m; count = ap->a_count; rtvals = ap->a_rtvals; npages = btoc(count); offset = IDX_TO_OFF(pages[0]->pindex); if (!fsess_opt_mmap(vnode_mount(vp))) { FS_DEBUG("called on non-cacheable vnode??\n"); } for (i = 0; i < npages; i++) rtvals[i] = VM_PAGER_AGAIN; /* * When putting pages, do not extend file past EOF. */ if (offset + count > fsize) { count = fsize - offset; if (count < 0) count = 0; } /* * We use only the kva address for the buffer, but this is extremely * convienient and fast. */ bp = getpbuf(&fuse_pbuf_freecnt); kva = (vm_offset_t)bp->b_data; pmap_qenter(kva, pages, npages); PCPU_INC(cnt.v_vnodeout); PCPU_ADD(cnt.v_vnodepgsout, count); iov.iov_base = (caddr_t)kva; iov.iov_len = count; uio.uio_iov = &iov; uio.uio_iovcnt = 1; uio.uio_offset = offset; uio.uio_resid = count; uio.uio_segflg = UIO_SYSSPACE; uio.uio_rw = UIO_WRITE; uio.uio_td = td; error = fuse_io_dispatch(vp, &uio, IO_DIRECT, cred); pmap_qremove(kva, npages); relpbuf(bp, &fuse_pbuf_freecnt); if (!error) { int nwritten = round_page(count - uio.uio_resid) / PAGE_SIZE; for (i = 0; i < nwritten; i++) { rtvals[i] = VM_PAGER_OK; VM_OBJECT_WLOCK(pages[i]->object); vm_page_undirty(pages[i]); VM_OBJECT_WUNLOCK(pages[i]->object); } } return rtvals[0]; }
/* * Vnode op for VM putpages. * possible bug: all IO done in sync mode * Note that vop_close always invalidate pages before close, so it's * not necessary to open vnode. * * nwfs_putpages(struct vnode *a_vp, vm_page_t *a_m, int a_count, * int a_sync, int *a_rtvals, vm_ooffset_t a_offset) */ int nwfs_putpages(struct vop_putpages_args *ap) { int error; struct thread *td = curthread; /* XXX */ struct vnode *vp = ap->a_vp; struct ucred *cred; #ifndef NWFS_RWCACHE KKASSERT(td->td_proc); cred = td->td_proc->p_ucred; /* XXX */ VOP_OPEN(vp, FWRITE, cred, NULL); error = vnode_pager_generic_putpages(ap->a_vp, ap->a_m, ap->a_count, ap->a_sync, ap->a_rtvals); VOP_CLOSE(vp, FWRITE, cred); return error; #else struct uio uio; struct iovec iov; vm_offset_t kva; struct buf *bp; int i, npages, count; int *rtvals; struct nwmount *nmp; struct nwnode *np; vm_page_t *pages; KKASSERT(td->td_proc); cred = td->td_proc->p_ucred; /* XXX */ /* VOP_OPEN(vp, FWRITE, cred, NULL);*/ np = VTONW(vp); nmp = VFSTONWFS(vp->v_mount); pages = ap->a_m; count = ap->a_count; rtvals = ap->a_rtvals; npages = btoc(count); for (i = 0; i < npages; i++) { rtvals[i] = VM_PAGER_AGAIN; } bp = getpbuf_kva(&nwfs_pbuf_freecnt); kva = (vm_offset_t) bp->b_data; pmap_qenter(kva, pages, npages); iov.iov_base = (caddr_t) kva; iov.iov_len = count; uio.uio_iov = &iov; uio.uio_iovcnt = 1; uio.uio_offset = IDX_TO_OFF(pages[0]->pindex); uio.uio_resid = count; uio.uio_segflg = UIO_SYSSPACE; uio.uio_rw = UIO_WRITE; uio.uio_td = td; NCPVNDEBUG("ofs=%d,resid=%d\n",(int)uio.uio_offset, uio.uio_resid); error = ncp_write(NWFSTOCONN(nmp), &np->n_fh, &uio, cred); /* VOP_CLOSE(vp, FWRITE, cred);*/ NCPVNDEBUG("paged write done: %d\n", error); pmap_qremove(kva, npages); relpbuf(bp, &nwfs_pbuf_freecnt); if (!error) { int nwritten = round_page(count - uio.uio_resid) / PAGE_SIZE; for (i = 0; i < nwritten; i++) { rtvals[i] = VM_PAGER_OK; vm_page_undirty(pages[i]); } } return rtvals[0]; #endif /* NWFS_RWCACHE */ }
/* * Vnode op for VM putpages. * possible bug: all IO done in sync mode * Note that vop_close always invalidate pages before close, so it's * not necessary to open vnode. * * smbfs_putpages(struct vnode *a_vp, vm_page_t *a_m, int a_count, int a_sync, * int *a_rtvals, vm_ooffset_t a_offset) */ int smbfs_putpages(struct vop_putpages_args *ap) { int error; struct vnode *vp = ap->a_vp; struct thread *td = curthread; /* XXX */ struct ucred *cred; #ifdef SMBFS_RWGENERIC KKASSERT(td->td_proc); cred = td->td_proc->p_ucred; VOP_OPEN(vp, FWRITE, cred, NULL); error = vop_stdputpages(ap); VOP_CLOSE(vp, FWRITE, cred); return error; #else struct uio uio; struct iovec iov; vm_offset_t kva; struct buf *bp; int i, npages, count; int doclose; int *rtvals; struct smbmount *smp; struct smbnode *np; struct smb_cred scred; vm_page_t *pages; KKASSERT(td->td_proc); cred = td->td_proc->p_ucred; /* VOP_OPEN(vp, FWRITE, cred, NULL);*/ np = VTOSMB(vp); smp = VFSTOSMBFS(vp->v_mount); pages = ap->a_m; count = ap->a_count; rtvals = ap->a_rtvals; npages = btoc(count); for (i = 0; i < npages; i++) { rtvals[i] = VM_PAGER_AGAIN; } bp = getpbuf_kva(&smbfs_pbuf_freecnt); kva = (vm_offset_t) bp->b_data; pmap_qenter(kva, pages, npages); iov.iov_base = (caddr_t) kva; iov.iov_len = count; uio.uio_iov = &iov; uio.uio_iovcnt = 1; uio.uio_offset = IDX_TO_OFF(pages[0]->pindex); uio.uio_resid = count; uio.uio_segflg = UIO_SYSSPACE; uio.uio_rw = UIO_WRITE; uio.uio_td = td; SMBVDEBUG("ofs=%d,resid=%d\n",(int)uio.uio_offset, uio.uio_resid); smb_makescred(&scred, td, cred); /* * This is kinda nasty. Since smbfs is physically closing the * fid on close(), we have to reopen it if necessary. There are * other races here too, such as if another process opens the same * file while we are blocked in read, or the file is open read-only * XXX */ error = 0; doclose = 0; if (np->n_opencount == 0) { error = smbfs_smb_open(np, SMB_AM_OPENRW, &scred); if (error == 0) doclose = 1; } if (error == 0) error = smb_write(smp->sm_share, np->n_fid, &uio, &scred); if (doclose) smbfs_smb_close(smp->sm_share, np->n_fid, NULL, &scred); /* VOP_CLOSE(vp, FWRITE, cred);*/ SMBVDEBUG("paged write done: %d\n", error); pmap_qremove(kva, npages); relpbuf(bp, &smbfs_pbuf_freecnt); if (!error) { int nwritten = round_page(count - uio.uio_resid) / PAGE_SIZE; for (i = 0; i < nwritten; i++) { rtvals[i] = VM_PAGER_OK; vm_page_undirty(pages[i]); } } return rtvals[0]; #endif /* SMBFS_RWGENERIC */ }
/* * This is now called from local media FS's to operate against their * own vnodes if they fail to implement VOP_PUTPAGES. * * This is typically called indirectly via the pageout daemon and * clustering has already typically occured, so in general we ask the * underlying filesystem to write the data out asynchronously rather * then delayed. */ int vnode_pager_generic_putpages(struct vnode *vp, vm_page_t *m, int bytecount, int flags, int *rtvals) { int i; int maxsize, ncount, count; vm_ooffset_t poffset; struct uio auio; struct iovec aiov; int error; int ioflags; count = bytecount / PAGE_SIZE; for (i = 0; i < count; i++) rtvals[i] = VM_PAGER_AGAIN; if ((int) m[0]->pindex < 0) { kprintf("vnode_pager_putpages: attempt to write meta-data!!! -- 0x%lx(%x)\n", (long)m[0]->pindex, m[0]->dirty); rtvals[0] = VM_PAGER_BAD; return VM_PAGER_BAD; } maxsize = count * PAGE_SIZE; ncount = count; poffset = IDX_TO_OFF(m[0]->pindex); /* * If the page-aligned write is larger then the actual file we * have to invalidate pages occuring beyond the file EOF. * * If the file EOF resides in the middle of a page we still clear * all of that page's dirty bits later on. If we didn't it would * endlessly re-write. * * We do not under any circumstances truncate the valid bits, as * this will screw up bogus page replacement. * * The caller has already read-protected the pages. The VFS must * use the buffer cache to wrap the pages. The pages might not * be immediately flushed by the buffer cache but once under its * control the pages themselves can wind up being marked clean * and their covering buffer cache buffer can be marked dirty. */ if (poffset + maxsize > vp->v_filesize) { if (poffset < vp->v_filesize) { maxsize = vp->v_filesize - poffset; ncount = btoc(maxsize); } else { maxsize = 0; ncount = 0; } if (ncount < count) { for (i = ncount; i < count; i++) { rtvals[i] = VM_PAGER_BAD; } } } /* * pageouts are already clustered, use IO_ASYNC to force a bawrite() * rather then a bdwrite() to prevent paging I/O from saturating * the buffer cache. Dummy-up the sequential heuristic to cause * large ranges to cluster. If neither IO_SYNC or IO_ASYNC is set, * the system decides how to cluster. */ ioflags = IO_VMIO; if (flags & (VM_PAGER_PUT_SYNC | VM_PAGER_PUT_INVAL)) ioflags |= IO_SYNC; else if ((flags & VM_PAGER_CLUSTER_OK) == 0) ioflags |= IO_ASYNC; ioflags |= (flags & VM_PAGER_PUT_INVAL) ? IO_INVAL: 0; ioflags |= IO_SEQMAX << IO_SEQSHIFT; aiov.iov_base = (caddr_t) 0; aiov.iov_len = maxsize; auio.uio_iov = &aiov; auio.uio_iovcnt = 1; auio.uio_offset = poffset; auio.uio_segflg = UIO_NOCOPY; auio.uio_rw = UIO_WRITE; auio.uio_resid = maxsize; auio.uio_td = NULL; error = VOP_WRITE(vp, &auio, ioflags, proc0.p_ucred); mycpu->gd_cnt.v_vnodeout++; mycpu->gd_cnt.v_vnodepgsout += ncount; if (error) { krateprintf(&vbadrate, "vnode_pager_putpages: I/O error %d\n", error); } if (auio.uio_resid) { krateprintf(&vresrate, "vnode_pager_putpages: residual I/O %zd at %lu\n", auio.uio_resid, (u_long)m[0]->pindex); } if (error == 0) { for (i = 0; i < ncount; i++) { rtvals[i] = VM_PAGER_OK; vm_page_undirty(m[i]); } } return rtvals[0]; }
/* * spec_getpages() - get pages associated with device vnode. * * Note that spec_read and spec_write do not use the buffer cache, so we * must fully implement getpages here. */ static int devfs_spec_getpages(struct vop_getpages_args *ap) { vm_offset_t kva; int error; int i, pcount, size; struct buf *bp; vm_page_t m; vm_ooffset_t offset; int toff, nextoff, nread; struct vnode *vp = ap->a_vp; int blksiz; int gotreqpage; error = 0; pcount = round_page(ap->a_count) / PAGE_SIZE; /* * Calculate the offset of the transfer and do sanity check. */ offset = IDX_TO_OFF(ap->a_m[0]->pindex) + ap->a_offset; /* * Round up physical size for real devices. We cannot round using * v_mount's block size data because v_mount has nothing to do with * the device. i.e. it's usually '/dev'. We need the physical block * size for the device itself. * * We can't use v_rdev->si_mountpoint because it only exists when the * block device is mounted. However, we can use v_rdev. */ if (vn_isdisk(vp, NULL)) blksiz = vp->v_rdev->si_bsize_phys; else blksiz = DEV_BSIZE; size = (ap->a_count + blksiz - 1) & ~(blksiz - 1); bp = getpbuf_kva(NULL); kva = (vm_offset_t)bp->b_data; /* * Map the pages to be read into the kva. */ pmap_qenter(kva, ap->a_m, pcount); /* Build a minimal buffer header. */ bp->b_cmd = BUF_CMD_READ; bp->b_bcount = size; bp->b_resid = 0; bsetrunningbufspace(bp, size); bp->b_bio1.bio_offset = offset; bp->b_bio1.bio_done = devfs_spec_getpages_iodone; mycpu->gd_cnt.v_vnodein++; mycpu->gd_cnt.v_vnodepgsin += pcount; /* Do the input. */ vn_strategy(ap->a_vp, &bp->b_bio1); crit_enter(); /* We definitely need to be at splbio here. */ while (bp->b_cmd != BUF_CMD_DONE) tsleep(bp, 0, "spread", 0); crit_exit(); if (bp->b_flags & B_ERROR) { if (bp->b_error) error = bp->b_error; else error = EIO; } /* * If EOF is encountered we must zero-extend the result in order * to ensure that the page does not contain garabge. When no * error occurs, an early EOF is indicated if b_bcount got truncated. * b_resid is relative to b_bcount and should be 0, but some devices * might indicate an EOF with b_resid instead of truncating b_bcount. */ nread = bp->b_bcount - bp->b_resid; if (nread < ap->a_count) bzero((caddr_t)kva + nread, ap->a_count - nread); pmap_qremove(kva, pcount); gotreqpage = 0; for (i = 0, toff = 0; i < pcount; i++, toff = nextoff) { nextoff = toff + PAGE_SIZE; m = ap->a_m[i]; m->flags &= ~PG_ZERO; /* * NOTE: vm_page_undirty/clear_dirty etc do not clear the * pmap modified bit. pmap modified bit should have * already been cleared. */ if (nextoff <= nread) { m->valid = VM_PAGE_BITS_ALL; vm_page_undirty(m); } else if (toff < nread) { /* * Since this is a VM request, we have to supply the * unaligned offset to allow vm_page_set_valid() * to zero sub-DEV_BSIZE'd portions of the page. */ vm_page_set_valid(m, 0, nread - toff); vm_page_clear_dirty_end_nonincl(m, 0, nread - toff); } else { m->valid = 0; vm_page_undirty(m); } if (i != ap->a_reqpage) { /* * Just in case someone was asking for this page we * now tell them that it is ok to use. */ if (!error || (m->valid == VM_PAGE_BITS_ALL)) { if (m->valid) { if (m->flags & PG_REFERENCED) { vm_page_activate(m); } else { vm_page_deactivate(m); } vm_page_wakeup(m); } else { vm_page_free(m); } } else { vm_page_free(m); } } else if (m->valid) { gotreqpage = 1; /* * Since this is a VM request, we need to make the * entire page presentable by zeroing invalid sections. */ if (m->valid != VM_PAGE_BITS_ALL) vm_page_zero_invalid(m, FALSE); } } if (!gotreqpage) { m = ap->a_m[ap->a_reqpage]; devfs_debug(DEVFS_DEBUG_WARNING, "spec_getpages:(%s) I/O read failure: (error=%d) bp %p vp %p\n", devtoname(vp->v_rdev), error, bp, bp->b_vp); devfs_debug(DEVFS_DEBUG_WARNING, " size: %d, resid: %d, a_count: %d, valid: 0x%x\n", size, bp->b_resid, ap->a_count, m->valid); devfs_debug(DEVFS_DEBUG_WARNING, " nread: %d, reqpage: %d, pindex: %lu, pcount: %d\n", nread, ap->a_reqpage, (u_long)m->pindex, pcount); /* * Free the buffer header back to the swap buffer pool. */ relpbuf(bp, NULL); return VM_PAGER_ERROR; } /* * Free the buffer header back to the swap buffer pool. */ relpbuf(bp, NULL); if (DEVFS_NODE(ap->a_vp)) nanotime(&DEVFS_NODE(ap->a_vp)->mtime); return VM_PAGER_OK; }