static enum handler_return virtio_block_irq_driver_callback(struct virtio_device *dev, uint ring, const struct vring_used_elem *e)
{
struct virtio_block_dev *bdev = (struct virtio_block_dev *)dev->priv;
LTRACEF("dev %p, ring %u, e %p, id %u, len %u\n", dev, ring, e, e->id, e->len);
/* parse our descriptor chain, add back to the free queue */
uint16_t i = e->id;
for (;;) {
int next;
struct vring_desc *desc = virtio_desc_index_to_desc(dev, ring, i);
//virtio_dump_desc(desc);
if (desc->flags & VRING_DESC_F_NEXT) {
next = desc->next;
} else {
/* end of chain */
next = -1;
}
virtio_free_desc(dev, ring, i);
if (next < 0)
break;
i = next;
}
/* signal our event */
event_signal(&bdev->io_event, false);
return INT_RESCHEDULE;
}
static status_t send_command_response(struct virtio_gpu_dev *gdev, const void *cmd, size_t cmd_len, void **_res, size_t res_len)
{
DEBUG_ASSERT(gdev);
DEBUG_ASSERT(cmd);
DEBUG_ASSERT(_res);
DEBUG_ASSERT(cmd_len + res_len < PAGE_SIZE);
LTRACEF("gdev %p, cmd %p, cmd_len %zu, res %p, res_len %zu\n", gdev, cmd, cmd_len, _res, res_len);
uint16_t i;
struct vring_desc *desc = virtio_alloc_desc_chain(gdev->dev, 0, 2, &i);
DEBUG_ASSERT(desc);
memcpy(gdev->gpu_request, cmd, cmd_len);
desc->addr = gdev->gpu_request_phys;
desc->len = cmd_len;
desc->flags |= VRING_DESC_F_NEXT;
/* set the second descriptor to the response with the write bit set */
desc = virtio_desc_index_to_desc(gdev->dev, 0, desc->next);
DEBUG_ASSERT(desc);
void *res = (void *)((uint8_t *)gdev->gpu_request + cmd_len);
*_res = res;
paddr_t res_phys = gdev->gpu_request_phys + cmd_len;
memset(res, 0, res_len);
desc->addr = res_phys;
desc->len = res_len;
desc->flags = VRING_DESC_F_WRITE;
/* submit the transfer */
virtio_submit_chain(gdev->dev, 0, i);
/* kick it off */
virtio_kick(gdev->dev, 0);
/* wait for result */
event_wait(&gdev->io_event);
return NO_ERROR;
}
ssize_t virtio_block_read_write(struct virtio_device *dev, void *buf, off_t offset, size_t len, bool write)
{
struct virtio_block_dev *bdev = (struct virtio_block_dev *)dev->priv;
uint16_t i;
struct vring_desc *desc;
paddr_t pa;
vaddr_t va = (vaddr_t)buf;
LTRACEF("dev %p, buf %p, offset 0x%llx, len %zu\n", dev, buf, offset, len);
mutex_acquire(&bdev->lock);
/* set up the request */
bdev->blk_req->type = write ? VIRTIO_BLK_T_OUT : VIRTIO_BLK_T_IN;
bdev->blk_req->ioprio = 0;
bdev->blk_req->sector = offset / 512;
LTRACEF("blk_req type %u ioprio %u sector %llu\n",
bdev->blk_req->type, bdev->blk_req->ioprio, bdev->blk_req->sector);
/* put together a transfer */
desc = virtio_alloc_desc_chain(dev, 0, 3, &i);
LTRACEF("after alloc chain desc %p, i %u\n", desc, i);
// XXX not cache safe.
// At the moment only tested on arm qemu, which doesn't emulate cache.
/* set up the descriptor pointing to the head */
desc->addr = bdev->blk_req_phys;
desc->len = sizeof(struct virtio_blk_req);
desc->flags |= VRING_DESC_F_NEXT;
/* set up the descriptor pointing to the buffer */
desc = virtio_desc_index_to_desc(dev, 0, desc->next);
#if WITH_KERNEL_VM
/* translate the first buffer */
arch_mmu_query(va, &pa, NULL);
desc->addr = (uint64_t)pa;
/* desc->len is filled in below */
#else
desc->addr = (uint64_t)(uintptr_t)buf;
desc->len = len;
#endif
desc->flags |= write ? 0 : VRING_DESC_F_WRITE; /* mark buffer as write-only if its a block read */
desc->flags |= VRING_DESC_F_NEXT;
#if WITH_KERNEL_VM
/* see if we need to add more descriptors due to scatter gather */
paddr_t next_pa = PAGE_ALIGN(pa + 1);
desc->len = MIN(next_pa - pa, len);
LTRACEF("first descriptor va 0x%lx desc->addr 0x%llx desc->len %u\n", va, desc->addr, desc->len);
len -= desc->len;
while (len > 0) {
/* amount of source buffer handled by this iteration of the loop */
size_t len_tohandle = MIN(len, PAGE_SIZE);
/* translate the next page in the buffer */
va = PAGE_ALIGN(va + 1);
arch_mmu_query(va, &pa, NULL);
LTRACEF("va now 0x%lx, pa 0x%lx, next_pa 0x%lx, remaining len %zu\n", va, pa, next_pa, len);
/* is the new translated physical address contiguous to the last one? */
if (next_pa == pa) {
LTRACEF("extending last one by %zu bytes\n", len_tohandle);
desc->len += len_tohandle;
} else {
uint16_t next_i = virtio_alloc_desc(dev, 0);
struct vring_desc *next_desc = virtio_desc_index_to_desc(dev, 0, next_i);
DEBUG_ASSERT(next_desc);
LTRACEF("doesn't extend, need new desc, allocated desc %i (%p)\n", next_i, next_desc);
/* fill this descriptor in and put it after the last one but before the response descriptor */
next_desc->addr = (uint64_t)pa;
next_desc->len = len_tohandle;
next_desc->flags = write ? 0 : VRING_DESC_F_WRITE; /* mark buffer as write-only if its a block read */
next_desc->flags |= VRING_DESC_F_NEXT;
next_desc->next = desc->next;
desc->next = next_i;
desc = next_desc;
}
len -= len_tohandle;
next_pa += PAGE_SIZE;
}
#endif
/* set up the descriptor pointing to the response */
desc = virtio_desc_index_to_desc(dev, 0, desc->next);
desc->addr = bdev->blk_response_phys;
desc->len = 1;
desc->flags = VRING_DESC_F_WRITE;
/* submit the transfer */
virtio_submit_chain(dev, 0, i);
/* kick it off */
virtio_kick(dev, 0);
/* wait for the transfer to complete */
event_wait(&bdev->io_event);
LTRACEF("status 0x%hhx\n", bdev->blk_response);
mutex_release(&bdev->lock);
return len;
}
ssize_t virtio_block_read_write(struct virtio_device *dev, void *buf, off_t offset, size_t len, bool write)
{
struct virtio_block_dev *bdev = (struct virtio_block_dev *)dev->priv;
uint16_t i;
struct vring_desc *desc;
paddr_t pa;
LTRACEF("dev %p, buf %p, offset 0x%llx, len %zu\n", dev, buf, offset, len);
mutex_acquire(&bdev->lock);
/* set up the request */
bdev->blk_req->type = write ? VIRTIO_BLK_T_OUT : VIRTIO_BLK_T_IN;
bdev->blk_req->ioprio = 0;
bdev->blk_req->sector = offset / 512;
LTRACEF("blk_req type %u ioprio %u sector %llu\n",
bdev->blk_req->type, bdev->blk_req->ioprio, bdev->blk_req->sector);
/* put together a transfer */
desc = virtio_alloc_desc_chain(dev, 0, 3, &i);
LTRACEF("after alloc chain desc %p, i %u\n", desc, i);
// XXX not cache safe.
// At the moment only tested on arm qemu, which doesn't emulate cache.
/* set up the descriptor pointing to the head */
desc->addr = bdev->blk_req_phys;
desc->len = sizeof(struct virtio_blk_req);
desc->flags |= VRING_DESC_F_NEXT;
/* set up the descriptor pointing to the buffer */
desc = virtio_desc_index_to_desc(dev, 0, desc->next);
#if WITH_KERNEL_VM
// XXX handle bufs that cross page boundaries
arch_mmu_query((vaddr_t)buf, &pa, NULL);
desc->addr = (uint64_t)pa;
#else
desc->addr = (uint64_t)(uintptr_t)buf;
#endif
desc->len = len;
desc->flags |= write ? 0 : VRING_DESC_F_WRITE; /* mark buffer as write-only if its a block read */
desc->flags |= VRING_DESC_F_NEXT;
/* set up the descriptor pointing to the response */
desc = virtio_desc_index_to_desc(dev, 0, desc->next);
desc->addr = bdev->blk_response_phys;
desc->len = 1;
desc->flags = VRING_DESC_F_WRITE;
/* submit the transfer */
virtio_submit_chain(dev, 0, i);
/* kick it off */
virtio_kick(dev, 0);
/* wait for the transfer to complete */
event_wait(&bdev->io_event);
LTRACEF("status 0x%hhx\n", bdev->blk_response);
mutex_release(&bdev->lock);
return len;
}
