/*
* Leffler, et al., says on p. 231:
* "The minphys() routine is called by physio() to adjust the
* size of each I/O transfer before the latter is passed to
* the strategy routine..."
*
* so, just adjust the buffer's count accounting to MAXPHYS here,
* and return the new count;
*/
u_int
minphys(struct buf *bp)
{
buf_setcount(bp, min(MAXPHYS, buf_count(bp)));
return buf_count(bp);
}
static void
vdev_disk_io_start(zio_t *zio)
{
vdev_t *vd = zio->io_vd;
vdev_disk_t *dvd = vd->vdev_tsd;
struct buf *bp;
vfs_context_t context;
int flags, error = 0;
/*
* If the vdev is closed, it's likely in the REMOVED or FAULTED state.
* Nothing to be done here but return failure.
*/
if (dvd == NULL || (dvd->vd_offline) || dvd->vd_devvp == NULL) {
zio->io_error = ENXIO;
zio_interrupt(zio);
return;
}
switch (zio->io_type) {
case ZIO_TYPE_IOCTL:
if (!vdev_readable(vd)) {
zio->io_error = SET_ERROR(ENXIO);
zio_interrupt(zio);
return;
}
switch (zio->io_cmd) {
case DKIOCFLUSHWRITECACHE:
if (zfs_nocacheflush)
break;
if (vd->vdev_nowritecache) {
zio->io_error = SET_ERROR(ENOTSUP);
break;
}
context = vfs_context_create(spl_vfs_context_kernel());
error = VNOP_IOCTL(dvd->vd_devvp, DKIOCSYNCHRONIZECACHE,
NULL, FWRITE, context);
(void) vfs_context_rele(context);
if (error == 0)
vdev_disk_ioctl_done(zio, error);
else
error = ENOTSUP;
if (error == 0) {
/*
* The ioctl will be done asychronously,
* and will call vdev_disk_ioctl_done()
* upon completion.
*/
return;
} else if (error == ENOTSUP || error == ENOTTY) {
/*
* If we get ENOTSUP or ENOTTY, we know that
* no future attempts will ever succeed.
* In this case we set a persistent bit so
* that we don't bother with the ioctl in the
* future.
*/
vd->vdev_nowritecache = B_TRUE;
}
zio->io_error = error;
break;
default:
zio->io_error = SET_ERROR(ENOTSUP);
} /* io_cmd */
zio_execute(zio);
return;
case ZIO_TYPE_WRITE:
if (zio->io_priority == ZIO_PRIORITY_SYNC_WRITE)
flags = B_WRITE;
else
flags = B_WRITE | B_ASYNC;
break;
case ZIO_TYPE_READ:
if (zio->io_priority == ZIO_PRIORITY_SYNC_READ)
flags = B_READ;
else
flags = B_READ | B_ASYNC;
break;
default:
zio->io_error = SET_ERROR(ENOTSUP);
zio_interrupt(zio);
return;
} /* io_type */
ASSERT(zio->io_type == ZIO_TYPE_READ || zio->io_type == ZIO_TYPE_WRITE);
/* Stop OSX from also caching our data */
flags |= B_NOCACHE;
if (zio->io_flags & ZIO_FLAG_FAILFAST)
flags |= B_FAILFAST;
zio->io_target_timestamp = zio_handle_io_delay(zio);
bp = buf_alloc(dvd->vd_devvp);
ASSERT(bp != NULL);
ASSERT(zio->io_data != NULL);
ASSERT(zio->io_size != 0);
buf_setflags(bp, flags);
buf_setcount(bp, zio->io_size);
buf_setdataptr(bp, (uintptr_t)zio->io_data);
/*
* Map offset to blcknumber, based on physical block number.
* (512, 4096, ..). If we fail to map, default back to
* standard 512. lbtodb() is fixed at 512.
*/
buf_setblkno(bp, zio->io_offset >> dvd->vd_ashift);
buf_setlblkno(bp, zio->io_offset >> dvd->vd_ashift);
buf_setsize(bp, zio->io_size);
if (buf_setcallback(bp, vdev_disk_io_intr, zio) != 0)
panic("vdev_disk_io_start: buf_setcallback failed\n");
if (zio->io_type == ZIO_TYPE_WRITE) {
vnode_startwrite(dvd->vd_devvp);
}
error = VNOP_STRATEGY(bp);
ASSERT(error == 0);
if (error) {
zio->io_error = error;
zio_interrupt(zio);
return;
}
}
static void mdevstrategy(struct buf *bp) {
unsigned int left, lop, csize;
vm_offset_t vaddr, blkoff;
int devid;
addr64_t paddr, fvaddr;
ppnum_t pp;
devid = minor(buf_device(bp)); /* Get minor device number */
if ((mdev[devid].mdFlags & mdInited) == 0) { /* Have we actually been defined yet? */
buf_seterror(bp, ENXIO);
buf_biodone(bp);
return;
}
buf_setresid(bp, buf_count(bp)); /* Set byte count */
blkoff = buf_blkno(bp) * mdev[devid].mdSecsize; /* Get offset into file */
/*
* Note that reading past end is an error, but reading at end is an EOF. For these
* we just return with resid == count.
*/
if (blkoff >= (mdev[devid].mdSize << 12)) { /* Are they trying to read/write at/after end? */
if(blkoff != (mdev[devid].mdSize << 12)) { /* Are we trying to read after EOF? */
buf_seterror(bp, EINVAL); /* Yeah, this is an error */
}
buf_biodone(bp); /* Return */
return;
}
if ((blkoff + buf_count(bp)) > (mdev[devid].mdSize << 12)) { /* Will this read go past end? */
buf_setcount(bp, ((mdev[devid].mdSize << 12) - blkoff)); /* Yes, trim to max */
}
/*
* make sure the buffer's data area is
* accessible
*/
if (buf_map(bp, (caddr_t *)&vaddr))
panic("ramstrategy: buf_map failed\n");
fvaddr = (mdev[devid].mdBase << 12) + blkoff; /* Point to offset into ram disk */
if (buf_flags(bp) & B_READ) { /* Is this a read? */
if(!(mdev[devid].mdFlags & mdPhys)) { /* Physical mapped disk? */
bcopy((void *)((uintptr_t)fvaddr),
(void *)vaddr, (size_t)buf_count(bp)); /* This is virtual, just get the data */
}
else {
left = buf_count(bp); /* Init the amount left to copy */
while(left) { /* Go until it is all copied */
lop = min((4096 - (vaddr & 4095)), (4096 - (fvaddr & 4095))); /* Get smallest amount left on sink and source */
csize = min(lop, left); /* Don't move more than we need to */
pp = pmap_find_phys(kernel_pmap, (addr64_t)((uintptr_t)vaddr)); /* Get the sink physical address */
if(!pp) { /* Not found, what gives? */
panic("mdevstrategy: sink address %016llX not mapped\n", (addr64_t)((uintptr_t)vaddr));
}
paddr = (addr64_t)(((addr64_t)pp << 12) | (addr64_t)(vaddr & 4095)); /* Get actual address */
bcopy_phys(fvaddr, paddr, csize); /* Copy this on in */
mapping_set_mod(paddr >> 12); /* Make sure we know that it is modified */
left = left - csize; /* Calculate what is left */
vaddr = vaddr + csize; /* Move to next sink address */
fvaddr = fvaddr + csize; /* Bump to next physical address */
}
}
}
else { /* This is a write */
if(!(mdev[devid].mdFlags & mdPhys)) { /* Physical mapped disk? */
static void
vnstrategy(struct buf *bp)
{
struct vn_softc *vn;
int error = 0;
long sz; /* in sc_secsize chunks */
daddr64_t blk_num;
struct vnode * shadow_vp = NULL;
struct vnode * vp = NULL;
struct vfs_context context;
vn = vn_table + vnunit(buf_device(bp));
if ((vn->sc_flags & VNF_INITED) == 0) {
error = ENXIO;
goto done;
}
context.vc_thread = current_thread();
context.vc_ucred = vn->sc_cred;
buf_setresid(bp, buf_count(bp));
/*
* Check for required alignment. Transfers must be a valid
* multiple of the sector size.
*/
blk_num = buf_blkno(bp);
if (buf_count(bp) % vn->sc_secsize != 0) {
error = EINVAL;
goto done;
}
sz = howmany(buf_count(bp), vn->sc_secsize);
/*
* If out of bounds return an error. If at the EOF point,
* simply read or write less.
*/
if (blk_num >= 0 && (u_int64_t)blk_num >= vn->sc_size) {
if (blk_num > 0 && (u_int64_t)blk_num > vn->sc_size) {
error = EINVAL;
}
goto done;
}
/*
* If the request crosses EOF, truncate the request.
*/
if ((blk_num + sz) > 0 && ((u_int64_t)(blk_num + sz)) > vn->sc_size) {
buf_setcount(bp, (vn->sc_size - blk_num) * vn->sc_secsize);
buf_setresid(bp, buf_count(bp));
}
vp = vn->sc_vp;
if (vp == NULL) {
error = ENXIO;
goto done;
}
error = vnode_getwithvid(vp, vn->sc_vid);
if (error != 0) {
/* the vnode is no longer available, abort */
error = ENXIO;
vnclear(vn, &context);
goto done;
}
shadow_vp = vn->sc_shadow_vp;
if (shadow_vp != NULL) {
error = vnode_getwithvid(shadow_vp,
vn->sc_shadow_vid);
if (error != 0) {
/* the vnode is no longer available, abort */
error = ENXIO;
vnode_put(vn->sc_vp);
vnclear(vn, &context);
goto done;
}
}
error = vn_readwrite_io(vn, bp, &context);
vnode_put(vp);
if (shadow_vp != NULL) {
vnode_put(shadow_vp);
}
done:
if (error) {
buf_seterror(bp, error);
}
buf_biodone(bp);
return;
}
static int
vdev_disk_io_start(zio_t *zio)
{
vdev_t *vd = zio->io_vd;
vdev_disk_t *dvd = vd->vdev_tsd;
struct buf *bp;
vfs_context_t context;
int flags, error = 0;
if (zio->io_type == ZIO_TYPE_IOCTL) {
zio_vdev_io_bypass(zio);
/* XXPOLICY */
if (vdev_is_dead(vd)) {
zio->io_error = ENXIO;
//zio_next_stage_async(zio);
return (ZIO_PIPELINE_CONTINUE);
//return;
}
switch (zio->io_cmd) {
case DKIOCFLUSHWRITECACHE:
if (zfs_nocacheflush)
break;
if (vd->vdev_nowritecache) {
zio->io_error = SET_ERROR(ENOTSUP);
break;
}
context = vfs_context_create((vfs_context_t)0);
error = VNOP_IOCTL(dvd->vd_devvp, DKIOCSYNCHRONIZECACHE, NULL, FWRITE, context);
(void) vfs_context_rele(context);
if (error == 0)
vdev_disk_ioctl_done(zio, error);
else
error = ENOTSUP;
if (error == 0) {
/*
* The ioctl will be done asychronously,
* and will call vdev_disk_ioctl_done()
* upon completion.
*/
return ZIO_PIPELINE_STOP;
} else if (error == ENOTSUP || error == ENOTTY) {
/*
* If we get ENOTSUP or ENOTTY, we know that
* no future attempts will ever succeed.
* In this case we set a persistent bit so
* that we don't bother with the ioctl in the
* future.
*/
vd->vdev_nowritecache = B_TRUE;
}
zio->io_error = error;
break;
default:
zio->io_error = SET_ERROR(ENOTSUP);
}
//zio_next_stage_async(zio);
return (ZIO_PIPELINE_CONTINUE);
}
if (zio->io_type == ZIO_TYPE_READ && vdev_cache_read(zio) == 0)
return (ZIO_PIPELINE_STOP);
// return;
if ((zio = vdev_queue_io(zio)) == NULL)
return (ZIO_PIPELINE_CONTINUE);
// return;
flags = (zio->io_type == ZIO_TYPE_READ ? B_READ : B_WRITE);
//flags |= B_NOCACHE;
if (zio->io_flags & ZIO_FLAG_FAILFAST)
flags |= B_FAILFAST;
/*
* Check the state of this device to see if it has been offlined or
* is in an error state. If the device was offlined or closed,
* dvd will be NULL and buf_alloc below will fail
*/
//error = vdev_is_dead(vd) ? ENXIO : vdev_error_inject(vd, zio);
if (vdev_is_dead(vd)) {
error = ENXIO;
}
if (error) {
zio->io_error = error;
//zio_next_stage_async(zio);
return (ZIO_PIPELINE_CONTINUE);
}
bp = buf_alloc(dvd->vd_devvp);
ASSERT(bp != NULL);
ASSERT(zio->io_data != NULL);
ASSERT(zio->io_size != 0);
buf_setflags(bp, flags);
buf_setcount(bp, zio->io_size);
buf_setdataptr(bp, (uintptr_t)zio->io_data);
if (dvd->vd_ashift) {
buf_setlblkno(bp, zio->io_offset>>dvd->vd_ashift);
buf_setblkno(bp, zio->io_offset>>dvd->vd_ashift);
} else {
int
physio( void (*f_strategy)(buf_t),
buf_t bp,
dev_t dev,
int flags,
u_int (*f_minphys)(buf_t),
struct uio *uio,
int blocksize)
{
struct proc *p = current_proc();
int error, i, buf_allocated, todo, iosize;
int orig_bflags = 0;
int64_t done;
error = 0;
flags &= B_READ | B_WRITE;
buf_allocated = 0;
/*
* [check user read/write access to the data buffer]
*
* Check each iov one by one. Note that we know if we're reading or
* writing, so we ignore the uio's rw parameter. Also note that if
* we're doing a read, that's a *write* to user-space.
*/
for (i = 0; i < uio->uio_iovcnt; i++) {
if (UIO_SEG_IS_USER_SPACE(uio->uio_segflg)) {
user_addr_t base;
user_size_t len;
if (uio_getiov(uio, i, &base, &len) ||
!useracc(base,
len,
(flags == B_READ) ? B_WRITE : B_READ))
return (EFAULT);
}
}
/*
* Make sure we have a buffer, creating one if necessary.
*/
if (bp == NULL) {
bp = buf_alloc((vnode_t)0);
buf_allocated = 1;
} else
orig_bflags = buf_flags(bp);
/*
* at this point we should have a buffer
* that is marked BL_BUSY... we either
* acquired it via buf_alloc, or it was
* passed into us... if it was passed
* in, it needs to already be owned by
* the caller (i.e. BL_BUSY is set)
*/
assert(bp->b_lflags & BL_BUSY);
/*
* [set up the fixed part of the buffer for a transfer]
*/
bp->b_dev = dev;
bp->b_proc = p;
/*
* [mark the buffer busy for physical I/O]
* (i.e. set B_PHYS (because it's an I/O to user
* memory, and B_RAW, because B_RAW is to be
* "Set by physio for raw transfers.", in addition
* to the read/write flag.)
*/
buf_setflags(bp, B_PHYS | B_RAW);
/*
* [while there is data to transfer and no I/O error]
* Note that I/O errors are handled with a 'goto' at the bottom
* of the 'while' loop.
*/
while (uio_resid(uio) > 0) {
if ( (iosize = uio_curriovlen(uio)) > MAXPHYSIO_WIRED)
iosize = MAXPHYSIO_WIRED;
/*
* make sure we're set to issue a fresh I/O
* in the right direction
*/
buf_reset(bp, flags);
/* [set up the buffer for a maximum-sized transfer] */
buf_setblkno(bp, uio_offset(uio) / blocksize);
buf_setcount(bp, iosize);
buf_setdataptr(bp, (uintptr_t)CAST_DOWN(caddr_t, uio_curriovbase(uio)));
/*
* [call f_minphys to bound the tranfer size]
* and remember the amount of data to transfer,
* for later comparison.
*/
(*f_minphys)(bp);
todo = buf_count(bp);
/*
* [lock the part of the user address space involved
* in the transfer]
*/
if(UIO_SEG_IS_USER_SPACE(uio->uio_segflg)) {
error = vslock(CAST_USER_ADDR_T(buf_dataptr(bp)),
(user_size_t)todo);
if (error)
goto done;
}
/* [call f_strategy to start the transfer] */
(*f_strategy)(bp);
/* [wait for the transfer to complete] */
error = (int)buf_biowait(bp);
/*
* [unlock the part of the address space previously
* locked]
*/
if(UIO_SEG_IS_USER_SPACE(uio->uio_segflg))
vsunlock(CAST_USER_ADDR_T(buf_dataptr(bp)),
(user_size_t)todo,
(flags & B_READ));
/*
* [deduct the transfer size from the total number
* of data to transfer]
*/
done = buf_count(bp) - buf_resid(bp);
uio_update(uio, done);
/*
* Now, check for an error.
* Also, handle weird end-of-disk semantics.
*/
if (error || done < todo)
goto done;
}
done:
if (buf_allocated)
buf_free(bp);
else
buf_setflags(bp, orig_bflags);
return (error);
}
__private_extern__
int
fuse_internal_strategy(vnode_t vp, buf_t bp)
{
size_t biosize;
size_t chunksize;
size_t respsize;
int mapped = FALSE;
int mode;
int op;
int vtype = vnode_vtype(vp);
int err = 0;
caddr_t bufdat;
off_t left;
off_t offset;
int32_t bflags = buf_flags(bp);
fufh_type_t fufh_type;
struct fuse_dispatcher fdi;
struct fuse_data *data;
struct fuse_vnode_data *fvdat = VTOFUD(vp);
struct fuse_filehandle *fufh = NULL;
mount_t mp = vnode_mount(vp);
data = fuse_get_mpdata(mp);
biosize = data->blocksize;
if (!(vtype == VREG || vtype == VDIR)) {
return ENOTSUP;
}
if (bflags & B_READ) {
mode = FREAD;
fufh_type = FUFH_RDONLY; /* FUFH_RDWR will also do */
} else {
mode = FWRITE;
fufh_type = FUFH_WRONLY; /* FUFH_RDWR will also do */
}
if (fvdat->flag & FN_CREATING) {
fuse_lck_mtx_lock(fvdat->createlock);
if (fvdat->flag & FN_CREATING) {
(void)fuse_msleep(fvdat->creator, fvdat->createlock,
PDROP | PINOD | PCATCH, "fuse_internal_strategy",
NULL);
} else {
fuse_lck_mtx_unlock(fvdat->createlock);
}
}
fufh = &(fvdat->fufh[fufh_type]);
if (!FUFH_IS_VALID(fufh)) {
fufh_type = FUFH_RDWR;
fufh = &(fvdat->fufh[fufh_type]);
if (!FUFH_IS_VALID(fufh)) {
fufh = NULL;
} else {
/* We've successfully fallen back to FUFH_RDWR. */
}
}
if (!fufh) {
if (mode == FREAD) {
fufh_type = FUFH_RDONLY;
} else {
fufh_type = FUFH_RDWR;
}
/*
* Lets NOT do the filehandle preflight check here.
*/
err = fuse_filehandle_get(vp, NULL, fufh_type, 0 /* mode */);
if (!err) {
fufh = &(fvdat->fufh[fufh_type]);
FUFH_AUX_INC(fufh);
/* We've created a NEW fufh of type fufh_type. open_count is 1. */
}
} else { /* good fufh */
FUSE_OSAddAtomic(1, (SInt32 *)&fuse_fh_reuse_count);
/* We're using an existing fufh of type fufh_type. */
}
if (err) {
/* A more typical error case. */
if ((err == ENOTCONN) || fuse_isdeadfs(vp)) {
buf_seterror(bp, EIO);
buf_biodone(bp);
return EIO;
}
IOLog("MacFUSE: strategy failed to get fh "
"(vtype=%d, fufh_type=%d, err=%d)\n", vtype, fufh_type, err);
if (!vfs_issynchronous(mp)) {
IOLog("MacFUSE: asynchronous write failed!\n");
}
buf_seterror(bp, EIO);
buf_biodone(bp);
return EIO;
}
if (!fufh) {
panic("MacFUSE: tried everything but still no fufh");
/* NOTREACHED */
}
#define B_INVAL 0x00040000 /* Does not contain valid info. */
#define B_ERROR 0x00080000 /* I/O error occurred. */
if (bflags & B_INVAL) {
IOLog("MacFUSE: buffer does not contain valid information\n");
}
if (bflags & B_ERROR) {
IOLog("MacFUSE: an I/O error has occured\n");
}
if (buf_count(bp) == 0) {
return 0;
}
fdisp_init(&fdi, 0);
if (mode == FREAD) {
struct fuse_read_in *fri;
buf_setresid(bp, buf_count(bp));
offset = (off_t)((off_t)buf_blkno(bp) * biosize);
if (offset >= fvdat->filesize) {
/* Trying to read at/after EOF? */
if (offset != fvdat->filesize) {
/* Trying to read after EOF? */
buf_seterror(bp, EINVAL);
}
buf_biodone(bp);
return 0;
}
/* Note that we just made sure that offset < fvdat->filesize. */
if ((offset + buf_count(bp)) > fvdat->filesize) {
/* Trimming read */
buf_setcount(bp, (uint32_t)(fvdat->filesize - offset));
}
if (buf_map(bp, &bufdat)) {
IOLog("MacFUSE: failed to map buffer in strategy\n");
return EFAULT;
} else {
mapped = TRUE;
}
while (buf_resid(bp) > 0) {
chunksize = min((size_t)buf_resid(bp), data->iosize);
fdi.iosize = sizeof(*fri);
op = FUSE_READ;
if (vtype == VDIR) {
op = FUSE_READDIR;
}
fdisp_make_vp(&fdi, op, vp, (vfs_context_t)0);
fri = fdi.indata;
fri->fh = fufh->fh_id;
/*
* Historical note:
*
* fri->offset = ((off_t)(buf_blkno(bp))) * biosize;
*
* This wasn't being incremented!?
*/
fri->offset = offset;
fri->size = (typeof(fri->size))chunksize;
fdi.tick->tk_aw_type = FT_A_BUF;
fdi.tick->tk_aw_bufdata = bufdat;
if ((err = fdisp_wait_answ(&fdi))) {
/* There was a problem with reading. */
goto out;
}
respsize = fdi.tick->tk_aw_bufsize;
if (respsize < 0) { /* Cannot really happen... */
err = EIO;
goto out;
}
buf_setresid(bp, (uint32_t)(buf_resid(bp) - respsize));
bufdat += respsize;
offset += respsize;
/* Did we hit EOF before being done? */
if ((respsize == 0) && (buf_resid(bp) > 0)) {
/*
* Historical note:
* If we don't get enough data, just fill the rest with zeros.
* In NFS context, this would mean a hole in the file.
*/
/* Zero-pad the incomplete buffer. */
bzero(bufdat, buf_resid(bp));
buf_setresid(bp, 0);
break;
}
} /* while (buf_resid(bp) > 0) */
} else {
/* write */
struct fuse_write_in *fwi;
struct fuse_write_out *fwo;
int merr = 0;
off_t diff;
if (buf_map(bp, &bufdat)) {
IOLog("MacFUSE: failed to map buffer in strategy\n");
return EFAULT;
} else {
mapped = TRUE;
}
/* Write begin */
buf_setresid(bp, buf_count(bp));
offset = (off_t)((off_t)buf_blkno(bp) * biosize);
/* XXX: TBD -- Check here for extension (writing past end) */
left = buf_count(bp);
while (left) {
fdi.iosize = sizeof(*fwi);
op = FUSE_WRITE;
fdisp_make_vp(&fdi, op, vp, (vfs_context_t)0);
chunksize = min((size_t)left, data->iosize);
fwi = fdi.indata;
fwi->fh = fufh->fh_id;
fwi->offset = offset;
fwi->size = (typeof(fwi->size))chunksize;
fdi.tick->tk_ms_type = FT_M_BUF;
fdi.tick->tk_ms_bufdata = bufdat;
fdi.tick->tk_ms_bufsize = chunksize;
/* About to write <chunksize> at <offset> */
if ((err = fdisp_wait_answ(&fdi))) {
merr = 1;
break;
}
fwo = fdi.answ;
diff = chunksize - fwo->size;
if (diff < 0) {
err = EINVAL;
break;
}
left -= fwo->size;
bufdat += fwo->size;
offset += fwo->size;
buf_setresid(bp, buf_resid(bp) - fwo->size);
}
if (merr) {
goto out;
}
}
if (fdi.tick) {
fuse_ticket_drop(fdi.tick);
} else {
/* No ticket upon leaving */
}
out:
if (err) {
buf_seterror(bp, err);
}
if (mapped == TRUE) {
buf_unmap(bp);
}
buf_biodone(bp);
return err;
}
