static void virtio_blk_rw_complete(void *opaque, int ret)
{
VirtIOBlockReq *next = opaque;
while (next) {
VirtIOBlockReq *req = next;
next = req->mr_next;
trace_virtio_blk_rw_complete(req, ret);
if (req->qiov.nalloc != -1) {
/* If nalloc is != 1 req->qiov is a local copy of the original
* external iovec. It was allocated in submit_merged_requests
* to be able to merge requests. */
qemu_iovec_destroy(&req->qiov);
}
if (ret) {
int p = virtio_ldl_p(VIRTIO_DEVICE(req->dev), &req->out.type);
bool is_read = !(p & VIRTIO_BLK_T_OUT);
if (virtio_blk_handle_rw_error(req, -ret, is_read)) {
continue;
}
}
virtio_blk_req_complete(req, VIRTIO_BLK_S_OK);
block_acct_done(blk_get_stats(req->dev->blk), &req->acct);
virtio_blk_free_request(req);
}
}
static void virtio_blk_flush_complete(void *opaque, int ret)
{
VirtIOBlockReq *req = opaque;
if (ret) {
if (virtio_blk_handle_rw_error(req, -ret, 0)) {
return;
}
}
virtio_blk_req_complete(req, VIRTIO_BLK_S_OK);
block_acct_done(blk_get_stats(req->dev->blk), &req->acct);
virtio_blk_free_request(req);
}
static int virtio_blk_handle_rw_error(VirtIOBlockReq *req, int error,
bool is_read)
{
BlockErrorAction action = blk_get_error_action(req->dev->blk,
is_read, error);
VirtIOBlock *s = req->dev;
if (action == BLOCK_ERROR_ACTION_STOP) {
req->next = s->rq;
s->rq = req;
} else if (action == BLOCK_ERROR_ACTION_REPORT) {
virtio_blk_req_complete(req, VIRTIO_BLK_S_IOERR);
block_acct_done(blk_get_stats(s->blk), &req->acct);
virtio_blk_free_request(req);
}
blk_error_action(s->blk, action, is_read, error);
return action != BLOCK_ERROR_ACTION_IGNORE;
}
static void virtio_blk_flush_complete(void *opaque, int ret)
{
VirtIOBlockReq *req = opaque;
VirtIOBlock *s = req->dev;
aio_context_acquire(blk_get_aio_context(s->conf.conf.blk));
if (ret) {
if (virtio_blk_handle_rw_error(req, -ret, 0)) {
goto out;
}
}
virtio_blk_req_complete(req, VIRTIO_BLK_S_OK);
block_acct_done(blk_get_stats(req->dev->blk), &req->acct);
virtio_blk_free_request(req);
out:
aio_context_release(blk_get_aio_context(s->conf.conf.blk));
}
static void virtio_blk_rw_complete(void *opaque, int ret)
{
VirtIOBlockReq *next = opaque;
while (next) {
VirtIOBlockReq *req = next;
next = req->mr_next;
trace_virtio_blk_rw_complete(req, ret);
if (req->qiov.nalloc != -1) {
/* If nalloc is != 1 req->qiov is a local copy of the original
* external iovec. It was allocated in submit_merged_requests
* to be able to merge requests. */
qemu_iovec_destroy(&req->qiov);
}
if (ret) {
int p = virtio_ldl_p(VIRTIO_DEVICE(req->dev), &req->out.type);
bool is_read = !(p & VIRTIO_BLK_T_OUT);
/* Note that memory may be dirtied on read failure. If the
* virtio request is not completed here, as is the case for
* BLOCK_ERROR_ACTION_STOP, the memory may not be copied
* correctly during live migration. While this is ugly,
* it is acceptable because the device is free to write to
* the memory until the request is completed (which will
* happen on the other side of the migration).
*/
if (virtio_blk_handle_rw_error(req, -ret, is_read)) {
/* Break the link in case the next request is added to the
* restart queue and is going to be parsed from the ring again.
*/
req->mr_next = NULL;
continue;
}
}
virtio_blk_req_complete(req, VIRTIO_BLK_S_OK);
block_acct_done(blk_get_stats(req->dev->blk), &req->acct);
virtio_blk_free_request(req);
}
}
static void virtio_blk_rw_complete(void *opaque, int ret)
{
VirtIOBlockReq *next = opaque;
VirtIOBlock *s = next->dev;
VirtIODevice *vdev = VIRTIO_DEVICE(s);
aio_context_acquire(blk_get_aio_context(s->conf.conf.blk));
while (next) {
VirtIOBlockReq *req = next;
next = req->mr_next;
trace_virtio_blk_rw_complete(vdev, req, ret);
if (req->qiov.nalloc != -1) {
/* If nalloc is != -1 req->qiov is a local copy of the original
* external iovec. It was allocated in submit_requests to be
* able to merge requests. */
qemu_iovec_destroy(&req->qiov);
}
if (ret) {
int p = virtio_ldl_p(VIRTIO_DEVICE(req->dev), &req->out.type);
bool is_read = !(p & VIRTIO_BLK_T_OUT);
/* Note that memory may be dirtied on read failure. If the
* virtio request is not completed here, as is the case for
* BLOCK_ERROR_ACTION_STOP, the memory may not be copied
* correctly during live migration. While this is ugly,
* it is acceptable because the device is free to write to
* the memory until the request is completed (which will
* happen on the other side of the migration).
*/
if (virtio_blk_handle_rw_error(req, -ret, is_read, true)) {
continue;
}
}
virtio_blk_req_complete(req, VIRTIO_BLK_S_OK);
block_acct_done(blk_get_stats(req->dev->blk), &req->acct);
virtio_blk_free_request(req);
}
aio_context_release(blk_get_aio_context(s->conf.conf.blk));
}
static void virtio_blk_discard_write_zeroes_complete(void *opaque, int ret)
{
VirtIOBlockReq *req = opaque;
VirtIOBlock *s = req->dev;
bool is_write_zeroes = (virtio_ldl_p(VIRTIO_DEVICE(s), &req->out.type) &
~VIRTIO_BLK_T_BARRIER) == VIRTIO_BLK_T_WRITE_ZEROES;
aio_context_acquire(blk_get_aio_context(s->conf.conf.blk));
if (ret) {
if (virtio_blk_handle_rw_error(req, -ret, false, is_write_zeroes)) {
goto out;
}
}
virtio_blk_req_complete(req, VIRTIO_BLK_S_OK);
if (is_write_zeroes) {
block_acct_done(blk_get_stats(s->blk), &req->acct);
}
virtio_blk_free_request(req);
out:
aio_context_release(blk_get_aio_context(s->conf.conf.blk));
}
static void qemu_aio_complete(void *opaque, int ret)
{
struct ioreq *ioreq = opaque;
struct XenBlkDev *blkdev = ioreq->blkdev;
struct XenDevice *xendev = &blkdev->xendev;
aio_context_acquire(blkdev->ctx);
if (ret != 0) {
xen_pv_printf(xendev, 0, "%s I/O error\n",
ioreq->req.operation == BLKIF_OP_READ ? "read" : "write");
ioreq->aio_errors++;
}
ioreq->aio_inflight--;
if (ioreq->presync) {
ioreq->presync = 0;
ioreq_runio_qemu_aio(ioreq);
goto done;
}
if (ioreq->aio_inflight > 0) {
goto done;
}
switch (ioreq->req.operation) {
case BLKIF_OP_READ:
/* in case of failure ioreq->aio_errors is increased */
if (ret == 0) {
ioreq_grant_copy(ioreq);
}
qemu_vfree(ioreq->buf);
break;
case BLKIF_OP_WRITE:
case BLKIF_OP_FLUSH_DISKCACHE:
if (!ioreq->req.nr_segments) {
break;
}
qemu_vfree(ioreq->buf);
break;
default:
break;
}
ioreq->status = ioreq->aio_errors ? BLKIF_RSP_ERROR : BLKIF_RSP_OKAY;
ioreq_finish(ioreq);
switch (ioreq->req.operation) {
case BLKIF_OP_WRITE:
case BLKIF_OP_FLUSH_DISKCACHE:
if (!ioreq->req.nr_segments) {
break;
}
case BLKIF_OP_READ:
if (ioreq->status == BLKIF_RSP_OKAY) {
block_acct_done(blk_get_stats(blkdev->blk), &ioreq->acct);
} else {
block_acct_failed(blk_get_stats(blkdev->blk), &ioreq->acct);
}
break;
case BLKIF_OP_DISCARD:
default:
break;
}
qemu_bh_schedule(blkdev->bh);
done:
aio_context_release(blkdev->ctx);
}
static void xen_block_complete_aio(void *opaque, int ret)
{
XenBlockRequest *request = opaque;
XenBlockDataPlane *dataplane = request->dataplane;
aio_context_acquire(dataplane->ctx);
if (ret != 0) {
error_report("%s I/O error",
request->req.operation == BLKIF_OP_READ ?
"read" : "write");
request->aio_errors++;
}
request->aio_inflight--;
if (request->presync) {
request->presync = 0;
xen_block_do_aio(request);
goto done;
}
if (request->aio_inflight > 0) {
goto done;
}
switch (request->req.operation) {
case BLKIF_OP_READ:
/* in case of failure request->aio_errors is increased */
if (ret == 0) {
xen_block_copy_request(request);
}
break;
case BLKIF_OP_WRITE:
case BLKIF_OP_FLUSH_DISKCACHE:
default:
break;
}
request->status = request->aio_errors ? BLKIF_RSP_ERROR : BLKIF_RSP_OKAY;
xen_block_finish_request(request);
switch (request->req.operation) {
case BLKIF_OP_WRITE:
case BLKIF_OP_FLUSH_DISKCACHE:
if (!request->req.nr_segments) {
break;
}
/* fall through */
case BLKIF_OP_READ:
if (request->status == BLKIF_RSP_OKAY) {
block_acct_done(blk_get_stats(dataplane->blk), &request->acct);
} else {
block_acct_failed(blk_get_stats(dataplane->blk), &request->acct);
}
break;
case BLKIF_OP_DISCARD:
default:
break;
}
if (xen_block_send_response(request)) {
Error *local_err = NULL;
xen_device_notify_event_channel(dataplane->xendev,
dataplane->event_channel,
&local_err);
if (local_err) {
error_report_err(local_err);
}
}
xen_block_release_request(request);
qemu_bh_schedule(dataplane->bh);
done:
aio_context_release(dataplane->ctx);
}
