static void qemu_aio_complete(void *opaque, int ret)
{
struct ioreq *ioreq = opaque;
if (ret != 0) {
xen_be_printf(&ioreq->blkdev->xendev, 0, "%s I/O error\n",
ioreq->req.operation == BLKIF_OP_READ ? "read" : "write");
ioreq->aio_errors++;
}
ioreq->aio_inflight--;
if (ioreq->presync) {
ioreq->presync = 0;
ioreq_runio_qemu_aio(ioreq);
return;
}
if (ioreq->aio_inflight > 0) {
return;
}
if (ioreq->postsync) {
ioreq->postsync = 0;
ioreq->aio_inflight++;
bdrv_aio_flush(ioreq->blkdev->bs, qemu_aio_complete, ioreq);
return;
}
ioreq->status = ioreq->aio_errors ? BLKIF_RSP_ERROR : BLKIF_RSP_OKAY;
ioreq_unmap(ioreq);
ioreq_finish(ioreq);
bdrv_acct_done(ioreq->blkdev->bs, &ioreq->acct);
qemu_bh_schedule(ioreq->blkdev->bh);
}
static int ioreq_runio_qemu_aio(struct ioreq *ioreq)
{
struct XenBlkDev *blkdev = ioreq->blkdev;
if (ioreq->req.nr_segments && ioreq_map(ioreq) == -1) {
goto err_no_map;
}
ioreq->aio_inflight++;
if (ioreq->presync) {
bdrv_flush(blkdev->bs); /* FIXME: aio_flush() ??? */
}
switch (ioreq->req.operation) {
case BLKIF_OP_READ:
bdrv_acct_start(blkdev->bs, &ioreq->acct, ioreq->v.size, BDRV_ACCT_READ);
ioreq->aio_inflight++;
bdrv_aio_readv(blkdev->bs, ioreq->start / BLOCK_SIZE,
&ioreq->v, ioreq->v.size / BLOCK_SIZE,
qemu_aio_complete, ioreq);
break;
case BLKIF_OP_WRITE:
case BLKIF_OP_WRITE_BARRIER:
if (!ioreq->req.nr_segments) {
break;
}
bdrv_acct_start(blkdev->bs, &ioreq->acct, ioreq->v.size, BDRV_ACCT_WRITE);
ioreq->aio_inflight++;
bdrv_aio_writev(blkdev->bs, ioreq->start / BLOCK_SIZE,
&ioreq->v, ioreq->v.size / BLOCK_SIZE,
qemu_aio_complete, ioreq);
break;
default:
/* unknown operation (shouldn't happen -- parse catches this) */
goto err;
}
if (ioreq->postsync) {
bdrv_flush(blkdev->bs); /* FIXME: aio_flush() ??? */
}
qemu_aio_complete(ioreq, 0);
return 0;
err:
ioreq_unmap(ioreq);
err_no_map:
ioreq_finish(ioreq);
ioreq->status = BLKIF_RSP_ERROR;
return -1;
}
static void qemu_aio_complete(void *opaque, int ret)
{
struct ioreq *ioreq = opaque;
if (ret != 0) {
xen_be_printf(&ioreq->blkdev->xendev, 0, "%s I/O error\n",
ioreq->req.operation == BLKIF_OP_READ ? "read" : "write");
ioreq->aio_errors++;
}
ioreq->aio_inflight--;
if (ioreq->aio_inflight > 0)
return;
ioreq->status = ioreq->aio_errors ? BLKIF_RSP_ERROR : BLKIF_RSP_OKAY;
ioreq_unmap(ioreq);
ioreq_finish(ioreq);
qemu_bh_schedule(ioreq->blkdev->bh);
}
static int ioreq_runio_qemu_sync(struct ioreq *ioreq)
{
struct XenBlkDev *blkdev = ioreq->blkdev;
int i, rc, len = 0;
off_t pos;
if (ioreq->req.nr_segments && ioreq_map(ioreq) == -1)
goto err;
if (ioreq->presync)
bdrv_flush(blkdev->bs);
switch (ioreq->req.operation) {
case BLKIF_OP_READ:
pos = ioreq->start;
for (i = 0; i < ioreq->v.niov; i++) {
rc = bdrv_read(blkdev->bs, pos / BLOCK_SIZE,
ioreq->v.iov[i].iov_base,
ioreq->v.iov[i].iov_len / BLOCK_SIZE);
if (rc != 0) {
xen_be_printf(&blkdev->xendev, 0, "rd I/O error (%p, len %zd)\n",
ioreq->v.iov[i].iov_base,
ioreq->v.iov[i].iov_len);
goto err;
}
len += ioreq->v.iov[i].iov_len;
pos += ioreq->v.iov[i].iov_len;
}
break;
case BLKIF_OP_WRITE:
case BLKIF_OP_WRITE_BARRIER:
if (!ioreq->req.nr_segments)
break;
pos = ioreq->start;
for (i = 0; i < ioreq->v.niov; i++) {
rc = bdrv_write(blkdev->bs, pos / BLOCK_SIZE,
ioreq->v.iov[i].iov_base,
ioreq->v.iov[i].iov_len / BLOCK_SIZE);
if (rc != 0) {
xen_be_printf(&blkdev->xendev, 0, "wr I/O error (%p, len %zd)\n",
ioreq->v.iov[i].iov_base,
ioreq->v.iov[i].iov_len);
goto err;
}
len += ioreq->v.iov[i].iov_len;
pos += ioreq->v.iov[i].iov_len;
}
break;
default:
/* unknown operation (shouldn't happen -- parse catches this) */
goto err;
}
if (ioreq->postsync)
bdrv_flush(blkdev->bs);
ioreq->status = BLKIF_RSP_OKAY;
ioreq_unmap(ioreq);
ioreq_finish(ioreq);
return 0;
err:
ioreq->status = BLKIF_RSP_ERROR;
return -1;
}
static int ioreq_runio_qemu_aio(struct ioreq *ioreq)
{
struct XenBlkDev *blkdev = ioreq->blkdev;
ioreq->buf = qemu_memalign(XC_PAGE_SIZE, ioreq->size);
if (ioreq->req.nr_segments &&
(ioreq->req.operation == BLKIF_OP_WRITE ||
ioreq->req.operation == BLKIF_OP_FLUSH_DISKCACHE) &&
ioreq_grant_copy(ioreq)) {
qemu_vfree(ioreq->buf);
goto err;
}
ioreq->aio_inflight++;
if (ioreq->presync) {
blk_aio_flush(ioreq->blkdev->blk, qemu_aio_complete, ioreq);
return 0;
}
switch (ioreq->req.operation) {
case BLKIF_OP_READ:
qemu_iovec_add(&ioreq->v, ioreq->buf, ioreq->size);
block_acct_start(blk_get_stats(blkdev->blk), &ioreq->acct,
ioreq->v.size, BLOCK_ACCT_READ);
ioreq->aio_inflight++;
blk_aio_preadv(blkdev->blk, ioreq->start, &ioreq->v, 0,
qemu_aio_complete, ioreq);
break;
case BLKIF_OP_WRITE:
case BLKIF_OP_FLUSH_DISKCACHE:
if (!ioreq->req.nr_segments) {
break;
}
qemu_iovec_add(&ioreq->v, ioreq->buf, ioreq->size);
block_acct_start(blk_get_stats(blkdev->blk), &ioreq->acct,
ioreq->v.size,
ioreq->req.operation == BLKIF_OP_WRITE ?
BLOCK_ACCT_WRITE : BLOCK_ACCT_FLUSH);
ioreq->aio_inflight++;
blk_aio_pwritev(blkdev->blk, ioreq->start, &ioreq->v, 0,
qemu_aio_complete, ioreq);
break;
case BLKIF_OP_DISCARD:
{
struct blkif_request_discard *req = (void *)&ioreq->req;
if (!blk_split_discard(ioreq, req->sector_number, req->nr_sectors)) {
goto err;
}
break;
}
default:
/* unknown operation (shouldn't happen -- parse catches this) */
goto err;
}
qemu_aio_complete(ioreq, 0);
return 0;
err:
ioreq_finish(ioreq);
ioreq->status = BLKIF_RSP_ERROR;
return -1;
}
static void qemu_aio_complete(void *opaque, int ret)
{
struct ioreq *ioreq = opaque;
struct XenBlkDev *blkdev = ioreq->blkdev;
struct XenDevice *xendev = &blkdev->xendev;
aio_context_acquire(blkdev->ctx);
if (ret != 0) {
xen_pv_printf(xendev, 0, "%s I/O error\n",
ioreq->req.operation == BLKIF_OP_READ ? "read" : "write");
ioreq->aio_errors++;
}
ioreq->aio_inflight--;
if (ioreq->presync) {
ioreq->presync = 0;
ioreq_runio_qemu_aio(ioreq);
goto done;
}
if (ioreq->aio_inflight > 0) {
goto done;
}
switch (ioreq->req.operation) {
case BLKIF_OP_READ:
/* in case of failure ioreq->aio_errors is increased */
if (ret == 0) {
ioreq_grant_copy(ioreq);
}
qemu_vfree(ioreq->buf);
break;
case BLKIF_OP_WRITE:
case BLKIF_OP_FLUSH_DISKCACHE:
if (!ioreq->req.nr_segments) {
break;
}
qemu_vfree(ioreq->buf);
break;
default:
break;
}
ioreq->status = ioreq->aio_errors ? BLKIF_RSP_ERROR : BLKIF_RSP_OKAY;
ioreq_finish(ioreq);
switch (ioreq->req.operation) {
case BLKIF_OP_WRITE:
case BLKIF_OP_FLUSH_DISKCACHE:
if (!ioreq->req.nr_segments) {
break;
}
case BLKIF_OP_READ:
if (ioreq->status == BLKIF_RSP_OKAY) {
block_acct_done(blk_get_stats(blkdev->blk), &ioreq->acct);
} else {
block_acct_failed(blk_get_stats(blkdev->blk), &ioreq->acct);
}
break;
case BLKIF_OP_DISCARD:
default:
break;
}
qemu_bh_schedule(blkdev->bh);
done:
aio_context_release(blkdev->ctx);
}
