static void virtio_blk_set_config(VirtIODevice *vdev, const uint8_t *config)
{
VirtIOBlock *s = VIRTIO_BLK(vdev);
struct virtio_blk_config blkcfg;
memcpy(&blkcfg, config, sizeof(blkcfg));
bdrv_set_enable_write_cache(s->bs, blkcfg.wce != 0);
}
static void virtio_blk_device_realize(DeviceState *dev, Error **errp)
{
VirtIODevice *vdev = VIRTIO_DEVICE(dev);
VirtIOBlock *s = VIRTIO_BLK(dev);
VirtIOBlkConf *blk = &(s->blk);
#ifdef CONFIG_VIRTIO_BLK_DATA_PLANE
Error *err = NULL;
#endif
static int virtio_blk_id;
if (!blk->conf.bs) {
error_setg(errp, "drive property not set");
return;
}
if (!bdrv_is_inserted(blk->conf.bs)) {
error_setg(errp, "Device needs media, but drive is empty");
return;
}
blkconf_serial(&blk->conf, &blk->serial);
s->original_wce = bdrv_enable_write_cache(blk->conf.bs);
if (blkconf_geometry(&blk->conf, NULL, 65535, 255, 255) < 0) {
error_setg(errp, "Error setting geometry");
return;
}
virtio_init(vdev, "virtio-blk", VIRTIO_ID_BLOCK,
sizeof(struct virtio_blk_config));
s->bs = blk->conf.bs;
s->conf = &blk->conf;
s->rq = NULL;
s->sector_mask = (s->conf->logical_block_size / BDRV_SECTOR_SIZE) - 1;
s->vq = virtio_add_queue(vdev, 128, virtio_blk_handle_output);
s->complete_request = virtio_blk_complete_request;
#ifdef CONFIG_VIRTIO_BLK_DATA_PLANE
virtio_blk_data_plane_create(vdev, blk, &s->dataplane, &err);
if (err != NULL) {
error_propagate(errp, err);
virtio_cleanup(vdev);
return;
}
s->migration_state_notifier.notify = virtio_blk_migration_state_changed;
add_migration_state_change_notifier(&s->migration_state_notifier);
#endif
s->change = qemu_add_vm_change_state_handler(virtio_blk_dma_restart_cb, s);
register_savevm(dev, "virtio-blk", virtio_blk_id++, 2,
virtio_blk_save, virtio_blk_load, s);
bdrv_set_dev_ops(s->bs, &virtio_block_ops, s);
bdrv_set_guest_block_size(s->bs, s->conf->logical_block_size);
bdrv_iostatus_enable(s->bs);
add_boot_device_path(s->conf->bootindex, dev, "/disk@0,0");
}
static int virtio_blk_device_init(VirtIODevice *vdev)
{
DeviceState *qdev = DEVICE(vdev);
VirtIOBlock *s = VIRTIO_BLK(vdev);
VirtIOBlkConf *blk = &(s->blk);
static int virtio_blk_id;
if (!blk->conf.bs) {
error_report("drive property not set");
return -1;
}
if (!bdrv_is_inserted(blk->conf.bs)) {
error_report("Device needs media, but drive is empty");
return -1;
}
blkconf_serial(&blk->conf, &blk->serial);
if (blkconf_geometry(&blk->conf, NULL, 65535, 255, 255) < 0) {
return -1;
}
virtio_init(vdev, "virtio-blk", VIRTIO_ID_BLOCK,
sizeof(struct virtio_blk_config));
vdev->get_config = virtio_blk_update_config;
vdev->set_config = virtio_blk_set_config;
vdev->get_features = virtio_blk_get_features;
vdev->set_status = virtio_blk_set_status;
vdev->reset = virtio_blk_reset;
s->bs = blk->conf.bs;
s->conf = &blk->conf;
memcpy(&(s->blk), blk, sizeof(struct VirtIOBlkConf));
s->rq = NULL;
s->sector_mask = (s->conf->logical_block_size / BDRV_SECTOR_SIZE) - 1;
s->vq = virtio_add_queue(vdev, 128, virtio_blk_handle_output);
#ifdef CONFIG_VIRTIO_BLK_DATA_PLANE
if (!virtio_blk_data_plane_create(vdev, blk, &s->dataplane)) {
virtio_common_cleanup(vdev);
return -1;
}
#endif
s->change = qemu_add_vm_change_state_handler(virtio_blk_dma_restart_cb, s);
register_savevm(qdev, "virtio-blk", virtio_blk_id++, 2,
virtio_blk_save, virtio_blk_load, s);
bdrv_set_dev_ops(s->bs, &virtio_block_ops, s);
bdrv_set_buffer_alignment(s->bs, s->conf->logical_block_size);
bdrv_iostatus_enable(s->bs);
add_boot_device_path(s->conf->bootindex, qdev, "/disk@0,0");
return 0;
}
/* Context: QEMU global mutex held */
void virtio_blk_data_plane_start(VirtIOBlockDataPlane *s)
{
BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(s->vdev)));
VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
VirtIOBlock *vblk = VIRTIO_BLK(s->vdev);
int r;
if (vblk->dataplane_started || s->starting) {
return;
}
s->starting = true;
s->vq = virtio_get_queue(s->vdev, 0);
/* Set up guest notifier (irq) */
r = k->set_guest_notifiers(qbus->parent, 1, true);
if (r != 0) {
fprintf(stderr, "virtio-blk failed to set guest notifier (%d), "
"ensure -enable-kvm is set\n", r);
goto fail_guest_notifiers;
}
s->guest_notifier = virtio_queue_get_guest_notifier(s->vq);
/* Set up virtqueue notify */
r = k->set_host_notifier(qbus->parent, 0, true);
if (r != 0) {
fprintf(stderr, "virtio-blk failed to set host notifier (%d)\n", r);
goto fail_host_notifier;
}
s->starting = false;
vblk->dataplane_started = true;
trace_virtio_blk_data_plane_start(s);
blk_set_aio_context(s->conf->conf.blk, s->ctx);
/* Kick right away to begin processing requests already in vring */
event_notifier_set(virtio_queue_get_host_notifier(s->vq));
/* Get this show started by hooking up our callbacks */
aio_context_acquire(s->ctx);
virtio_queue_aio_set_host_notifier_handler(s->vq, s->ctx,
virtio_blk_data_plane_handle_output);
aio_context_release(s->ctx);
return;
fail_host_notifier:
k->set_guest_notifiers(qbus->parent, 1, false);
fail_guest_notifiers:
vblk->dataplane_disabled = true;
s->starting = false;
vblk->dataplane_started = true;
}
static void virtio_blk_save_device(VirtIODevice *vdev, QEMUFile *f)
{
VirtIOBlock *s = VIRTIO_BLK(vdev);
VirtIOBlockReq *req = s->rq;
while (req) {
qemu_put_sbyte(f, 1);
qemu_put_buffer(f, (unsigned char *)req->elem,
sizeof(VirtQueueElement));
req = req->next;
}
qemu_put_sbyte(f, 0);
}
static int virtio_blk_device_exit(DeviceState *dev)
{
VirtIODevice *vdev = VIRTIO_DEVICE(dev);
VirtIOBlock *s = VIRTIO_BLK(dev);
#ifdef CONFIG_VIRTIO_BLK_DATA_PLANE
virtio_blk_data_plane_destroy(s->dataplane);
s->dataplane = NULL;
#endif
qemu_del_vm_change_state_handler(s->change);
unregister_savevm(dev, "virtio-blk", s);
blockdev_mark_auto_del(s->bs);
virtio_common_cleanup(vdev);
return 0;
}
static void virtio_blk_device_unrealize(DeviceState *dev, Error **errp)
{
VirtIODevice *vdev = VIRTIO_DEVICE(dev);
VirtIOBlock *s = VIRTIO_BLK(dev);
#ifdef CONFIG_VIRTIO_BLK_DATA_PLANE
remove_migration_state_change_notifier(&s->migration_state_notifier);
virtio_blk_data_plane_destroy(s->dataplane);
s->dataplane = NULL;
#endif
qemu_del_vm_change_state_handler(s->change);
unregister_savevm(dev, "virtio-blk", s);
blockdev_mark_auto_del(s->bs);
virtio_cleanup(vdev);
}
static void handle_notify(EventNotifier *e)
{
VirtIOBlockDataPlane *s = container_of(e, VirtIOBlockDataPlane,
host_notifier);
VirtIOBlock *vblk = VIRTIO_BLK(s->vdev);
event_notifier_test_and_clear(&s->host_notifier);
blk_io_plug(s->conf->conf.blk);
for (;;) {
MultiReqBuffer mrb = {};
int ret;
/* Disable guest->host notifies to avoid unnecessary vmexits */
vring_disable_notification(s->vdev, &s->vring);
for (;;) {
VirtIOBlockReq *req = virtio_blk_alloc_request(vblk);
ret = vring_pop(s->vdev, &s->vring, &req->elem);
if (ret < 0) {
virtio_blk_free_request(req);
break; /* no more requests */
}
trace_virtio_blk_data_plane_process_request(s, req->elem.out_num,
req->elem.in_num,
req->elem.index);
virtio_blk_handle_request(req, &mrb);
}
if (mrb.num_reqs) {
virtio_blk_submit_multireq(s->conf->conf.blk, &mrb);
}
if (likely(ret == -EAGAIN)) { /* vring emptied */
/* Re-enable guest->host notifies and stop processing the vring.
* But if the guest has snuck in more descriptors, keep processing.
*/
if (vring_enable_notification(s->vdev, &s->vring)) {
break;
}
} else { /* fatal error */
break;
}
}
blk_io_unplug(s->conf->conf.blk);
}
static void virtio_blk_reset(VirtIODevice *vdev)
{
#ifdef CONFIG_VIRTIO_BLK_DATA_PLANE
VirtIOBlock *s = VIRTIO_BLK(vdev);
if (s->dataplane) {
virtio_blk_data_plane_stop(s->dataplane);
}
#endif
/*
* This should cancel pending requests, but can't do nicely until there
* are per-device request lists.
*/
bdrv_drain_all();
}
static void virtio_blk_reset(VirtIODevice *vdev)
{
VirtIOBlock *s = VIRTIO_BLK(vdev);
#ifdef CONFIG_VIRTIO_BLK_DATA_PLANE
if (s->dataplane) {
virtio_blk_data_plane_stop(s->dataplane);
}
#endif
/*
* This should cancel pending requests, but can't do nicely until there
* are per-device request lists.
*/
bdrv_drain_all();
bdrv_set_enable_write_cache(s->bs, s->original_wce);
}
/* Context: QEMU global mutex held */
void virtio_blk_data_plane_stop(VirtIOBlockDataPlane *s)
{
BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(s->vdev)));
VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
VirtIOBlock *vblk = VIRTIO_BLK(s->vdev);
unsigned i;
unsigned nvqs = s->conf->num_queues;
if (!vblk->dataplane_started || s->stopping) {
return;
}
/* Better luck next time. */
if (vblk->dataplane_disabled) {
vblk->dataplane_disabled = false;
vblk->dataplane_started = false;
return;
}
s->stopping = true;
trace_virtio_blk_data_plane_stop(s);
aio_context_acquire(s->ctx);
/* Stop notifications for new requests from guest */
for (i = 0; i < nvqs; i++) {
VirtQueue *vq = virtio_get_queue(s->vdev, i);
virtio_queue_aio_set_host_notifier_handler(vq, s->ctx, NULL);
}
/* Drain and switch bs back to the QEMU main loop */
blk_set_aio_context(s->conf->conf.blk, qemu_get_aio_context());
aio_context_release(s->ctx);
for (i = 0; i < nvqs; i++) {
virtio_bus_set_host_notifier(VIRTIO_BUS(qbus), i, false);
}
/* Clean up guest notifier (irq) */
k->set_guest_notifiers(qbus->parent, nvqs, false);
vblk->dataplane_started = false;
s->stopping = false;
}
static void virtio_blk_set_status(VirtIODevice *vdev, uint8_t status)
{
VirtIOBlock *s = VIRTIO_BLK(vdev);
uint32_t features;
#ifdef CONFIG_VIRTIO_BLK_DATA_PLANE
if (s->dataplane && !(status & (VIRTIO_CONFIG_S_DRIVER |
VIRTIO_CONFIG_S_DRIVER_OK))) {
virtio_blk_data_plane_stop(s->dataplane);
}
#endif
if (!(status & VIRTIO_CONFIG_S_DRIVER_OK)) {
return;
}
features = vdev->guest_features;
bdrv_set_enable_write_cache(s->bs, !!(features & (1 << VIRTIO_BLK_F_WCE)));
}
/* Context: QEMU global mutex held */
void virtio_blk_data_plane_stop(VirtIOBlockDataPlane *s)
{
BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(s->vdev)));
VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
VirtIOBlock *vblk = VIRTIO_BLK(s->vdev);
/* Better luck next time. */
if (s->disabled) {
s->disabled = false;
return;
}
if (!s->started || s->stopping) {
return;
}
s->stopping = true;
vblk->complete_request = s->saved_complete_request;
trace_virtio_blk_data_plane_stop(s);
aio_context_acquire(s->ctx);
/* Stop notifications for new requests from guest */
aio_set_event_notifier(s->ctx, &s->host_notifier, true, NULL);
/* Drain and switch bs back to the QEMU main loop */
blk_set_aio_context(s->conf->conf.blk, qemu_get_aio_context());
aio_context_release(s->ctx);
/* Sync vring state back to virtqueue so that non-dataplane request
* processing can continue when we disable the host notifier below.
*/
vring_teardown(&s->vring, s->vdev, 0);
k->set_host_notifier(qbus->parent, 0, false);
/* Clean up guest notifier (irq) */
k->set_guest_notifiers(qbus->parent, 1, false);
s->started = false;
s->stopping = false;
}
static int virtio_blk_load_device(VirtIODevice *vdev, QEMUFile *f,
int version_id)
{
VirtIOBlock *s = VIRTIO_BLK(vdev);
while (qemu_get_sbyte(f)) {
VirtIOBlockReq *req = virtio_blk_alloc_request(s);
qemu_get_buffer(f, (unsigned char *)req->elem,
sizeof(VirtQueueElement));
req->next = s->rq;
s->rq = req;
virtqueue_map_sg(req->elem->in_sg, req->elem->in_addr,
req->elem->in_num, 1);
virtqueue_map_sg(req->elem->out_sg, req->elem->out_addr,
req->elem->out_num, 0);
}
return 0;
}
static uint32_t virtio_blk_get_features(VirtIODevice *vdev, uint32_t features)
{
VirtIOBlock *s = VIRTIO_BLK(vdev);
features |= (1 << VIRTIO_BLK_F_SEG_MAX);
features |= (1 << VIRTIO_BLK_F_GEOMETRY);
features |= (1 << VIRTIO_BLK_F_TOPOLOGY);
features |= (1 << VIRTIO_BLK_F_BLK_SIZE);
features |= (1 << VIRTIO_BLK_F_SCSI);
if (s->blk.config_wce) {
features |= (1 << VIRTIO_BLK_F_CONFIG_WCE);
}
if (bdrv_enable_write_cache(s->bs))
features |= (1 << VIRTIO_BLK_F_WCE);
if (bdrv_is_read_only(s->bs))
features |= 1 << VIRTIO_BLK_F_RO;
return features;
}
static void virtio_blk_set_status(VirtIODevice *vdev, uint8_t status)
{
VirtIOBlock *s = VIRTIO_BLK(vdev);
uint32_t features;
#ifdef CONFIG_VIRTIO_BLK_DATA_PLANE
if (s->dataplane && !(status & (VIRTIO_CONFIG_S_DRIVER |
VIRTIO_CONFIG_S_DRIVER_OK))) {
virtio_blk_data_plane_stop(s->dataplane);
}
#endif
if (!(status & VIRTIO_CONFIG_S_DRIVER_OK)) {
return;
}
features = vdev->guest_features;
/* A guest that supports VIRTIO_BLK_F_CONFIG_WCE must be able to send
* cache flushes. Thus, the "auto writethrough" behavior is never
* necessary for guests that support the VIRTIO_BLK_F_CONFIG_WCE feature.
* Leaving it enabled would break the following sequence:
*
* Guest started with "-drive cache=writethrough"
* Guest sets status to 0
* Guest sets DRIVER bit in status field
* Guest reads host features (WCE=0, CONFIG_WCE=1)
* Guest writes guest features (WCE=0, CONFIG_WCE=1)
* Guest writes 1 to the WCE configuration field (writeback mode)
* Guest sets DRIVER_OK bit in status field
*
* s->bs would erroneously be placed in writethrough mode.
*/
if (!(features & (1 << VIRTIO_BLK_F_CONFIG_WCE))) {
aio_context_acquire(bdrv_get_aio_context(s->bs));
bdrv_set_enable_write_cache(s->bs,
!!(features & (1 << VIRTIO_BLK_F_WCE)));
aio_context_release(bdrv_get_aio_context(s->bs));
}
}
/* coalesce internal state, copy to pci i/o region 0
*/
static void virtio_blk_update_config(VirtIODevice *vdev, uint8_t *config)
{
VirtIOBlock *s = VIRTIO_BLK(vdev);
struct virtio_blk_config blkcfg;
uint64_t capacity;
int blk_size = s->conf->logical_block_size;
bdrv_get_geometry(s->bs, &capacity);
memset(&blkcfg, 0, sizeof(blkcfg));
virtio_stq_p(vdev, &blkcfg.capacity, capacity);
virtio_stl_p(vdev, &blkcfg.seg_max, 128 - 2);
virtio_stw_p(vdev, &blkcfg.cylinders, s->conf->cyls);
virtio_stl_p(vdev, &blkcfg.blk_size, blk_size);
virtio_stw_p(vdev, &blkcfg.min_io_size, s->conf->min_io_size / blk_size);
virtio_stw_p(vdev, &blkcfg.opt_io_size, s->conf->opt_io_size / blk_size);
blkcfg.heads = s->conf->heads;
/*
* We must ensure that the block device capacity is a multiple of
* the logical block size. If that is not the case, let's use
* sector_mask to adopt the geometry to have a correct picture.
* For those devices where the capacity is ok for the given geometry
* we don't touch the sector value of the geometry, since some devices
* (like s390 dasd) need a specific value. Here the capacity is already
* cyls*heads*secs*blk_size and the sector value is not block size
* divided by 512 - instead it is the amount of blk_size blocks
* per track (cylinder).
*/
if (bdrv_getlength(s->bs) / s->conf->heads / s->conf->secs % blk_size) {
blkcfg.sectors = s->conf->secs & ~s->sector_mask;
} else {
blkcfg.sectors = s->conf->secs;
}
blkcfg.size_max = 0;
blkcfg.physical_block_exp = get_physical_block_exp(s->conf);
blkcfg.alignment_offset = 0;
blkcfg.wce = bdrv_enable_write_cache(s->bs);
memcpy(config, &blkcfg, sizeof(struct virtio_blk_config));
}
static void virtio_blk_handle_output(VirtIODevice *vdev, VirtQueue *vq)
{
VirtIOBlock *s = VIRTIO_BLK(vdev);
VirtIOBlockReq *req;
MultiReqBuffer mrb = {
.num_writes = 0,
};
#ifdef CONFIG_VIRTIO_BLK_DATA_PLANE
/* Some guests kick before setting VIRTIO_CONFIG_S_DRIVER_OK so start
* dataplane here instead of waiting for .set_status().
*/
if (s->dataplane) {
virtio_blk_data_plane_start(s->dataplane);
return;
}
#endif
while ((req = virtio_blk_get_request(s))) {
virtio_blk_handle_request(req, &mrb);
}
virtio_submit_multiwrite(s->bs, &mrb);
/*
* FIXME: Want to check for completions before returning to guest mode,
* so cached reads and writes are reported as quickly as possible. But
* that should be done in the generic block layer.
*/
}
static void virtio_blk_dma_restart_bh(void *opaque)
{
VirtIOBlock *s = opaque;
VirtIOBlockReq *req = s->rq;
MultiReqBuffer mrb = {
.num_writes = 0,
};
qemu_bh_delete(s->bh);
s->bh = NULL;
s->rq = NULL;
while (req) {
virtio_blk_handle_request(req, &mrb);
req = req->next;
}
virtio_submit_multiwrite(s->bs, &mrb);
}
static void virtio_blk_dma_restart_cb(void *opaque, int running,
RunState state)
{
VirtIOBlock *s = opaque;
if (!running) {
return;
}
if (!s->bh) {
s->bh = aio_bh_new(bdrv_get_aio_context(s->blk.conf.bs),
virtio_blk_dma_restart_bh, s);
qemu_bh_schedule(s->bh);
}
}
void virtio_blk_set_conf(DeviceState *dev, VirtIOBlkConf *blk)
{
VirtIOBlock *s = VIRTIO_BLK(dev);
memcpy(&(s->blk), blk, sizeof(struct VirtIOBlkConf));
}
static void handle_notify(EventNotifier *e)
{
VirtIOBlockDataPlane *s = container_of(e, VirtIOBlockDataPlane,
host_notifier);
VirtIOBlock *vblk = VIRTIO_BLK(s->vdev);
event_notifier_test_and_clear(&s->host_notifier);
blk_io_plug(s->conf->conf.blk);
for (;;) {
MultiReqBuffer mrb = {
.num_writes = 0,
};
int ret;
/* Disable guest->host notifies to avoid unnecessary vmexits */
vring_disable_notification(s->vdev, &s->vring);
for (;;) {
VirtIOBlockReq *req = virtio_blk_alloc_request(vblk);
ret = vring_pop(s->vdev, &s->vring, &req->elem);
if (ret < 0) {
virtio_blk_free_request(req);
break; /* no more requests */
}
trace_virtio_blk_data_plane_process_request(s, req->elem.out_num,
req->elem.in_num,
req->elem.index);
virtio_blk_handle_request(req, &mrb);
}
virtio_submit_multiwrite(s->conf->conf.blk, &mrb);
if (likely(ret == -EAGAIN)) { /* vring emptied */
/* Re-enable guest->host notifies and stop processing the vring.
* But if the guest has snuck in more descriptors, keep processing.
*/
if (vring_enable_notification(s->vdev, &s->vring)) {
break;
}
} else { /* fatal error */
break;
}
}
blk_io_unplug(s->conf->conf.blk);
}
/* Context: QEMU global mutex held */
void virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *conf,
VirtIOBlockDataPlane **dataplane,
Error **errp)
{
VirtIOBlockDataPlane *s;
Error *local_err = NULL;
BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(vdev)));
VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
*dataplane = NULL;
if (!conf->data_plane && !conf->iothread) {
return;
}
/* Don't try if transport does not support notifiers. */
if (!k->set_guest_notifiers || !k->set_host_notifier) {
error_setg(errp,
"device is incompatible with x-data-plane "
"(transport does not support notifiers)");
return;
}
/* If dataplane is (re-)enabled while the guest is running there could be
* block jobs that can conflict.
*/
if (blk_op_is_blocked(conf->conf.blk, BLOCK_OP_TYPE_DATAPLANE,
&local_err)) {
error_setg(errp, "cannot start dataplane thread: %s",
error_get_pretty(local_err));
error_free(local_err);
return;
}
s = g_new0(VirtIOBlockDataPlane, 1);
s->vdev = vdev;
s->conf = conf;
if (conf->iothread) {
s->iothread = conf->iothread;
object_ref(OBJECT(s->iothread));
} else {
/* Create per-device IOThread if none specified. This is for
* x-data-plane option compatibility. If x-data-plane is removed we
* can drop this.
*/
object_initialize(&s->internal_iothread_obj,
sizeof(s->internal_iothread_obj),
TYPE_IOTHREAD);
user_creatable_complete(OBJECT(&s->internal_iothread_obj), &error_abort);
s->iothread = &s->internal_iothread_obj;
}
s->ctx = iothread_get_aio_context(s->iothread);
s->bh = aio_bh_new(s->ctx, notify_guest_bh, s);
error_setg(&s->blocker, "block device is in use by data plane");
blk_op_block_all(conf->conf.blk, s->blocker);
blk_op_unblock(conf->conf.blk, BLOCK_OP_TYPE_RESIZE, s->blocker);
blk_op_unblock(conf->conf.blk, BLOCK_OP_TYPE_DRIVE_DEL, s->blocker);
*dataplane = s;
}
