/*
* This routine finds the first virtqueue described in the configuration of
* this device and sets it up.
*/
static struct virtqueue *kvm_find_vq(struct virtio_device *vdev,
unsigned index,
void (*callback)(struct virtqueue *vq),
const char *name)
{
struct kvm_device *kdev = to_kvmdev(vdev);
struct kvm_vqconfig *config;
struct virtqueue *vq;
int err;
if (index >= kdev->desc->num_vq)
return ERR_PTR(-ENOENT);
if (!name)
return NULL;
config = kvm_vq_config(kdev->desc)+index;
err = vmem_add_mapping(config->address,
vring_size(config->num,
KVM_S390_VIRTIO_RING_ALIGN));
if (err)
goto out;
vq = vring_new_virtqueue(index, config->num, KVM_S390_VIRTIO_RING_ALIGN,
vdev, true, (void *) config->address,
kvm_notify, callback, name);
if (!vq) {
err = -ENOMEM;
goto unmap;
}
/*
* register a callback token
* The host will sent this via the external interrupt parameter
*/
config->token = (u64) vq;
vq->priv = config;
return vq;
unmap:
vmem_remove_mapping(config->address,
vring_size(config->num,
KVM_S390_VIRTIO_RING_ALIGN));
out:
return ERR_PTR(err);
}
static int zynq_rpmsg_virtio_find_vqs(struct virtio_device *vdev,
unsigned nvqs, struct virtqueue *vqs[],
vq_callback_t *callbacks[],
const char *names[])
{
int i;
struct zynq_rpmsg_vring *local_vring;
void *vring_va;
int size;
/* Skip through the vrings. */
for (i = 0; i < nvqs; i++) {
local_vring = &(zynq_rpmsg_p->vrings[i]);
local_vring->len = zynq_rpmsg_p->num_descs;
size = vring_size(zynq_rpmsg_p->num_descs,
zynq_rpmsg_p->align);
/* Allocate non-cacheable memory for the vring. */
local_vring->va = dma_alloc_coherent
(&(zynq_rpmsg_platform->dev),
size, &(local_vring->dma), GFP_KERNEL);
vring_va = local_vring->va;
memset(vring_va, 0, size);
local_vring->vq = vring_new_virtqueue(i,
zynq_rpmsg_p->num_descs,
zynq_rpmsg_p->align, vdev,
false, vring_va,
zynq_rpmsg_virtio_notify,
callbacks[i], names[i]);
vqs[i] = local_vring->vq;
}
return 0;
}
/*
* This routine finds the Nth virtqueue described in the configuration of
* this device and sets it up.
*
* This is kind of an ugly duckling. It'd be nicer to have a standard
* representation of a virtqueue in the configuration space, but it seems that
* everyone wants to do it differently. The KVM coders want the Guest to
* allocate its own pages and tell the Host where they are, but for lguest it's
* simpler for the Host to simply tell us where the pages are.
*/
static struct virtqueue *lg_find_vq(struct virtio_device *vdev,
unsigned index,
void (*callback)(struct virtqueue *vq),
const char *name)
{
struct lguest_device *ldev = to_lgdev(vdev);
struct lguest_vq_info *lvq;
struct virtqueue *vq;
int err;
if (!name)
return NULL;
/* We must have this many virtqueues. */
if (index >= ldev->desc->num_vq)
return ERR_PTR(-ENOENT);
lvq = kmalloc(sizeof(*lvq), GFP_KERNEL);
if (!lvq)
return ERR_PTR(-ENOMEM);
/*
* Make a copy of the "struct lguest_vqconfig" entry, which sits after
* the descriptor. We need a copy because the config space might not
* be aligned correctly.
*/
memcpy(&lvq->config, lg_vq(ldev->desc)+index, sizeof(lvq->config));
printk("Mapping virtqueue %i addr %lx\n", index,
(unsigned long)lvq->config.pfn << PAGE_SHIFT);
/* Figure out how many pages the ring will take, and map that memory */
lvq->pages = lguest_map((unsigned long)lvq->config.pfn << PAGE_SHIFT,
DIV_ROUND_UP(vring_size(lvq->config.num,
LGUEST_VRING_ALIGN),
PAGE_SIZE));
if (!lvq->pages) {
err = -ENOMEM;
goto free_lvq;
}
/*
* OK, tell virtio_ring.c to set up a virtqueue now we know its size
* and we've got a pointer to its pages. Note that we set weak_barriers
* to 'true': the host just a(nother) SMP CPU, so we only need inter-cpu
* barriers.
*/
vq = vring_new_virtqueue(index, lvq->config.num, LGUEST_VRING_ALIGN, vdev,
true, lvq->pages, lg_notify, callback, name);
if (!vq) {
err = -ENOMEM;
goto unmap;
}
/* Make sure the interrupt is allocated. */
err = lguest_setup_irq(lvq->config.irq);
if (err)
goto destroy_vring;
/*
* Tell the interrupt for this virtqueue to go to the virtio_ring
* interrupt handler.
*
* FIXME: We used to have a flag for the Host to tell us we could use
* the interrupt as a source of randomness: it'd be nice to have that
* back.
*/
err = request_irq(lvq->config.irq, vring_interrupt, IRQF_SHARED,
dev_name(&vdev->dev), vq);
if (err)
goto free_desc;
/*
* Last of all we hook up our 'struct lguest_vq_info" to the
* virtqueue's priv pointer.
*/
vq->priv = lvq;
return vq;
free_desc:
irq_free_desc(lvq->config.irq);
destroy_vring:
vring_del_virtqueue(vq);
unmap:
lguest_unmap(lvq->pages);
free_lvq:
kfree(lvq);
return ERR_PTR(err);
}
static struct virtqueue *vm_setup_vq(struct virtio_device *vdev, unsigned index,
void (*callback)(struct virtqueue *vq),
const char *name)
{
struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
struct virtio_mmio_vq_info *info;
struct virtqueue *vq;
unsigned long flags, size;
int err;
/* Select the queue we're interested in */
writel(index, vm_dev->base + VIRTIO_MMIO_QUEUE_SEL);
/* Queue shouldn't already be set up. */
if (readl(vm_dev->base + VIRTIO_MMIO_QUEUE_PFN)) {
err = -ENOENT;
goto error_available;
}
/* Allocate and fill out our active queue description */
info = kmalloc(sizeof(*info), GFP_KERNEL);
if (!info) {
err = -ENOMEM;
goto error_kmalloc;
}
info->queue_index = index;
/* Allocate pages for the queue - start with a queue as big as
* possible (limited by maximum size allowed by device), drop down
* to a minimal size, just big enough to fit descriptor table
* and two rings (which makes it "alignment_size * 2")
*/
info->num = readl(vm_dev->base + VIRTIO_MMIO_QUEUE_NUM_MAX);
while (1) {
size = PAGE_ALIGN(vring_size(info->num,
VIRTIO_MMIO_VRING_ALIGN));
/* Already smallest possible allocation? */
if (size <= VIRTIO_MMIO_VRING_ALIGN * 2) {
err = -ENOMEM;
goto error_alloc_pages;
}
info->queue = alloc_pages_exact(size, GFP_KERNEL | __GFP_ZERO);
if (info->queue)
break;
info->num /= 2;
}
/* Activate the queue */
writel(info->num, vm_dev->base + VIRTIO_MMIO_QUEUE_NUM);
writel(VIRTIO_MMIO_VRING_ALIGN,
vm_dev->base + VIRTIO_MMIO_QUEUE_ALIGN);
writel(virt_to_phys(info->queue) >> PAGE_SHIFT,
vm_dev->base + VIRTIO_MMIO_QUEUE_PFN);
/* Create the vring */
vq = vring_new_virtqueue(info->num, VIRTIO_MMIO_VRING_ALIGN, vdev,
true, info->queue, vm_notify, callback, name);
if (!vq) {
err = -ENOMEM;
goto error_new_virtqueue;
}
vq->priv = info;
info->vq = vq;
spin_lock_irqsave(&vm_dev->lock, flags);
list_add(&info->node, &vm_dev->virtqueues);
spin_unlock_irqrestore(&vm_dev->lock, flags);
return vq;
error_new_virtqueue:
writel(0, vm_dev->base + VIRTIO_MMIO_QUEUE_PFN);
free_pages_exact(info->queue, size);
error_alloc_pages:
kfree(info);
error_kmalloc:
error_available:
return ERR_PTR(err);
}
void vhost_vq_setup(struct vdev_info *dev, struct vq_info *info)
{
struct vhost_vring_state state = { .index = info->idx };
struct vhost_vring_file file = { .index = info->idx };
unsigned long long features = dev->vdev.features[0];
struct vhost_vring_addr addr = {
.index = info->idx,
.desc_user_addr = (uint64_t)(unsigned long)info->vring.desc,
.avail_user_addr = (uint64_t)(unsigned long)info->vring.avail,
.used_user_addr = (uint64_t)(unsigned long)info->vring.used,
};
int r;
r = ioctl(dev->control, VHOST_SET_FEATURES, &features);
assert(r >= 0);
state.num = info->vring.num;
r = ioctl(dev->control, VHOST_SET_VRING_NUM, &state);
assert(r >= 0);
state.num = 0;
r = ioctl(dev->control, VHOST_SET_VRING_BASE, &state);
assert(r >= 0);
r = ioctl(dev->control, VHOST_SET_VRING_ADDR, &addr);
assert(r >= 0);
file.fd = info->kick;
r = ioctl(dev->control, VHOST_SET_VRING_KICK, &file);
assert(r >= 0);
file.fd = info->call;
r = ioctl(dev->control, VHOST_SET_VRING_CALL, &file);
assert(r >= 0);
}
static void vq_info_add(struct vdev_info *dev, int num)
{
struct vq_info *info = &dev->vqs[dev->nvqs];
int r;
info->idx = dev->nvqs;
info->kick = eventfd(0, EFD_NONBLOCK);
info->call = eventfd(0, EFD_NONBLOCK);
r = posix_memalign(&info->ring, 4096, vring_size(num, 4096));
assert(r >= 0);
memset(info->ring, 0, vring_size(num, 4096));
vring_init(&info->vring, num, info->ring, 4096);
info->vq = vring_new_virtqueue(info->vring.num, 4096, &dev->vdev, info->ring,
vq_notify, vq_callback, "test");
assert(info->vq);
info->vq->priv = info;
vhost_vq_setup(dev, info);
dev->fds[info->idx].fd = info->call;
dev->fds[info->idx].events = POLLIN;
dev->nvqs++;
}
static void vdev_info_init(struct vdev_info* dev, unsigned long long features)
{
int r;
memset(dev, 0, sizeof *dev);
dev->vdev.features[0] = features;
dev->vdev.features[1] = features >> 32;
dev->buf_size = 1024;
dev->buf = malloc(dev->buf_size);
assert(dev->buf);
dev->control = open("/dev/vhost-test", O_RDWR);
assert(dev->control >= 0);
r = ioctl(dev->control, VHOST_SET_OWNER, NULL);
assert(r >= 0);
dev->mem = malloc(offsetof(struct vhost_memory, regions) +
sizeof dev->mem->regions[0]);
assert(dev->mem);
memset(dev->mem, 0, offsetof(struct vhost_memory, regions) +
sizeof dev->mem->regions[0]);
dev->mem->nregions = 1;
dev->mem->regions[0].guest_phys_addr = (long)dev->buf;
dev->mem->regions[0].userspace_addr = (long)dev->buf;
dev->mem->regions[0].memory_size = dev->buf_size;
r = ioctl(dev->control, VHOST_SET_MEM_TABLE, dev->mem);
assert(r >= 0);
}
/* TODO: this is pretty bad: we get a cache line bounce
* for the wait queue on poll and another one on read,
* plus the read which is there just to clear the
* current state. */
static void wait_for_interrupt(struct vdev_info *dev)
{
int i;
unsigned long long val;
poll(dev->fds, dev->nvqs, -1);
for (i = 0; i < dev->nvqs; ++i)
if (dev->fds[i].revents & POLLIN) {
read(dev->fds[i].fd, &val, sizeof val);
}
}
static void run_test(struct vdev_info *dev, struct vq_info *vq, int bufs)
{
struct scatterlist sl;
long started = 0, completed = 0;
long completed_before;
int r, test = 1;
unsigned len;
long long spurious = 0;
r = ioctl(dev->control, VHOST_TEST_RUN, &test);
assert(r >= 0);
for (;;) {
virtqueue_disable_cb(vq->vq);
completed_before = completed;
do {
if (started < bufs) {
sg_init_one(&sl, dev->buf, dev->buf_size);
r = virtqueue_add_buf(vq->vq, &sl, 1, 0,
dev->buf + started);
if (likely(r >= 0)) {
++started;
virtqueue_kick(vq->vq);
}
} else
r = -1;
/* Flush out completed bufs if any */
if (virtqueue_get_buf(vq->vq, &len)) {
++completed;
r = 0;
}
} while (r >= 0);
if (completed == completed_before)
++spurious;
assert(completed <= bufs);
assert(started <= bufs);
if (completed == bufs)
break;
if (virtqueue_enable_cb(vq->vq)) {
wait_for_interrupt(dev);
}
}
test = 0;
r = ioctl(dev->control, VHOST_TEST_RUN, &test);
assert(r >= 0);
fprintf(stderr, "spurious wakeus: 0x%llx\n", spurious);
}
const char optstring[] = "h";
const struct option longopts[] = {
{
.name = "help",
.val = 'h',
},
{
.name = "event-idx",
/*
* This routine will assign vring's allocated in host/io memory. Code in
* virtio_ring.c however continues to access this io memory as if it were local
* memory without io accessors.
*/
static struct virtqueue *mic_find_vq(struct virtio_device *vdev,
unsigned index,
void (*callback)(struct virtqueue *vq),
const char *name)
{
struct mic_vdev *mvdev = to_micvdev(vdev);
struct mic_vqconfig __iomem *vqconfig;
struct mic_vqconfig config;
struct virtqueue *vq;
void __iomem *va;
struct _mic_vring_info __iomem *info;
void *used;
int vr_size, _vr_size, err, magic;
struct vring *vr;
u8 type = ioread8(&mvdev->desc->type);
if (index >= ioread8(&mvdev->desc->num_vq))
return ERR_PTR(-ENOENT);
if (!name)
return ERR_PTR(-ENOENT);
/* First assign the vring's allocated in host memory */
vqconfig = mic_vq_config(mvdev->desc) + index;
memcpy_fromio(&config, vqconfig, sizeof(config));
_vr_size = vring_size(le16_to_cpu(config.num), MIC_VIRTIO_RING_ALIGN);
vr_size = PAGE_ALIGN(_vr_size + sizeof(struct _mic_vring_info));
va = mic_card_map(mvdev->mdev, le64_to_cpu(config.address), vr_size);
if (!va)
return ERR_PTR(-ENOMEM);
mvdev->vr[index] = va;
memset_io(va, 0x0, _vr_size);
vq = vring_new_virtqueue(index, le16_to_cpu(config.num),
MIC_VIRTIO_RING_ALIGN, vdev, false,
(void __force *)va, mic_notify, callback,
name);
if (!vq) {
err = -ENOMEM;
goto unmap;
}
info = va + _vr_size;
magic = ioread32(&info->magic);
if (WARN(magic != MIC_MAGIC + type + index, "magic mismatch")) {
err = -EIO;
goto unmap;
}
/* Allocate and reassign used ring now */
mvdev->used_size[index] = PAGE_ALIGN(sizeof(__u16) * 3 +
sizeof(struct vring_used_elem) *
le16_to_cpu(config.num));
used = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
get_order(mvdev->used_size[index]));
if (!used) {
err = -ENOMEM;
dev_err(mic_dev(mvdev), "%s %d err %d\n",
__func__, __LINE__, err);
goto del_vq;
}
iowrite64(virt_to_phys(used), &vqconfig->used_address);
/*
* To reassign the used ring here we are directly accessing
* struct vring_virtqueue which is a private data structure
* in virtio_ring.c. At the minimum, a BUILD_BUG_ON() in
* vring_new_virtqueue() would ensure that
* (&vq->vring == (struct vring *) (&vq->vq + 1));
*/
vr = (struct vring *)(vq + 1);
vr->used = used;
vq->priv = mvdev;
return vq;
del_vq:
vring_del_virtqueue(vq);
unmap:
mic_card_unmap(mvdev->mdev, mvdev->vr[index]);
return ERR_PTR(err);
}
/*
* This routine finds the first virtqueue described in the configuration of
* this device and sets it up.
*/
static struct virtqueue *kvm_find_vq(struct virtio_device *vdev,
unsigned index,
void (*callback)(struct virtqueue *vq),
const char *name)
{
struct kvm_device *kdev = to_kvmdev(vdev);
struct kvm_vqinfo *vqi;
struct kvm_vqconfig *config;
struct virtqueue *vq;
long irq;
int err = -EINVAL;
if (index >= kdev->desc->num_vq)
return ERR_PTR(-ENOENT);
vqi = kzalloc(sizeof(*vqi), GFP_KERNEL);
if (!vqi)
return ERR_PTR(-ENOMEM);
config = kvm_vq_config(kdev->desc)+index;
vqi->config = config;
vqi->pages = generic_remap_prot(config->pa,
vring_size(config->num,
KVM_TILE_VIRTIO_RING_ALIGN),
0, io_prot());
if (!vqi->pages) {
err = -ENOMEM;
goto out;
}
vq = vring_new_virtqueue(config->num, KVM_TILE_VIRTIO_RING_ALIGN,
vdev, 0, vqi->pages,
kvm_notify, callback, name);
if (!vq) {
err = -ENOMEM;
goto unmap;
}
/*
* Trigger the IPI interrupt in SW way.
* TODO: We do not need to create one irq for each vq. A bit wasteful.
*/
irq = create_irq();
if (irq < 0) {
err = -ENXIO;
goto del_virtqueue;
}
tile_irq_activate(irq, TILE_IRQ_SW_CLEAR);
if (request_irq(irq, vring_interrupt, 0, dev_name(&vdev->dev), vq)) {
err = -ENXIO;
destroy_irq(irq);
goto del_virtqueue;
}
config->irq = irq;
vq->priv = vqi;
return vq;
del_virtqueue:
vring_del_virtqueue(vq);
unmap:
vunmap(vqi->pages);
out:
return ERR_PTR(err);
}
