void
vdev_queue_io_done(zio_t *zio)
{
vdev_queue_t *vq = &zio->io_vd->vdev_queue;
if (zio_injection_enabled)
delay(SEC_TO_TICK(zio_handle_io_delay(zio)));
mutex_enter(&vq->vq_lock);
vdev_queue_pending_remove(vq, zio);
vq->vq_io_complete_ts = gethrtime();
for (int i = 0; i < zfs_vdev_ramp_rate; i++) {
zio_t *nio = vdev_queue_io_to_issue(vq, zfs_vdev_max_pending);
if (nio == NULL)
break;
mutex_exit(&vq->vq_lock);
if (nio->io_done == vdev_queue_agg_io_done) {
zio_nowait(nio);
} else {
zio_vdev_io_reissue(nio);
zio_execute(nio);
}
mutex_enter(&vq->vq_lock);
}
mutex_exit(&vq->vq_lock);
}
static void
vdev_disk_io_start(zio_t *zio)
{
vdev_t *vd = zio->io_vd;
vdev_disk_t *dvd = vd->vdev_tsd;
struct buf *bp;
vfs_context_t context;
int flags, error = 0;
/*
* If the vdev is closed, it's likely in the REMOVED or FAULTED state.
* Nothing to be done here but return failure.
*/
if (dvd == NULL || (dvd->vd_offline) || dvd->vd_devvp == NULL) {
zio->io_error = ENXIO;
zio_interrupt(zio);
return;
}
switch (zio->io_type) {
case ZIO_TYPE_IOCTL:
if (!vdev_readable(vd)) {
zio->io_error = SET_ERROR(ENXIO);
zio_interrupt(zio);
return;
}
switch (zio->io_cmd) {
case DKIOCFLUSHWRITECACHE:
if (zfs_nocacheflush)
break;
if (vd->vdev_nowritecache) {
zio->io_error = SET_ERROR(ENOTSUP);
break;
}
context = vfs_context_create(spl_vfs_context_kernel());
error = VNOP_IOCTL(dvd->vd_devvp, DKIOCSYNCHRONIZECACHE,
NULL, FWRITE, context);
(void) vfs_context_rele(context);
if (error == 0)
vdev_disk_ioctl_done(zio, error);
else
error = ENOTSUP;
if (error == 0) {
/*
* The ioctl will be done asychronously,
* and will call vdev_disk_ioctl_done()
* upon completion.
*/
return;
} else if (error == ENOTSUP || error == ENOTTY) {
/*
* If we get ENOTSUP or ENOTTY, we know that
* no future attempts will ever succeed.
* In this case we set a persistent bit so
* that we don't bother with the ioctl in the
* future.
*/
vd->vdev_nowritecache = B_TRUE;
}
zio->io_error = error;
break;
default:
zio->io_error = SET_ERROR(ENOTSUP);
} /* io_cmd */
zio_execute(zio);
return;
case ZIO_TYPE_WRITE:
if (zio->io_priority == ZIO_PRIORITY_SYNC_WRITE)
flags = B_WRITE;
else
flags = B_WRITE | B_ASYNC;
break;
case ZIO_TYPE_READ:
if (zio->io_priority == ZIO_PRIORITY_SYNC_READ)
flags = B_READ;
else
flags = B_READ | B_ASYNC;
break;
default:
zio->io_error = SET_ERROR(ENOTSUP);
zio_interrupt(zio);
return;
} /* io_type */
ASSERT(zio->io_type == ZIO_TYPE_READ || zio->io_type == ZIO_TYPE_WRITE);
/* Stop OSX from also caching our data */
flags |= B_NOCACHE;
if (zio->io_flags & ZIO_FLAG_FAILFAST)
flags |= B_FAILFAST;
zio->io_target_timestamp = zio_handle_io_delay(zio);
bp = buf_alloc(dvd->vd_devvp);
ASSERT(bp != NULL);
ASSERT(zio->io_data != NULL);
ASSERT(zio->io_size != 0);
buf_setflags(bp, flags);
buf_setcount(bp, zio->io_size);
buf_setdataptr(bp, (uintptr_t)zio->io_data);
/*
* Map offset to blcknumber, based on physical block number.
* (512, 4096, ..). If we fail to map, default back to
* standard 512. lbtodb() is fixed at 512.
*/
buf_setblkno(bp, zio->io_offset >> dvd->vd_ashift);
buf_setlblkno(bp, zio->io_offset >> dvd->vd_ashift);
buf_setsize(bp, zio->io_size);
if (buf_setcallback(bp, vdev_disk_io_intr, zio) != 0)
panic("vdev_disk_io_start: buf_setcallback failed\n");
if (zio->io_type == ZIO_TYPE_WRITE) {
vnode_startwrite(dvd->vd_devvp);
}
error = VNOP_STRATEGY(bp);
ASSERT(error == 0);
if (error) {
zio->io_error = error;
zio_interrupt(zio);
return;
}
}