/* * This routine finds the first virtqueue described in the configuration of * this device and sets it up. */ static struct virtqueue *kvm_find_vq(struct virtio_device *vdev, unsigned index, void (*callback)(struct virtqueue *vq), const char *name) { struct kvm_device *kdev = to_kvmdev(vdev); struct kvm_vqconfig *config; struct virtqueue *vq; int err; if (index >= kdev->desc->num_vq) return ERR_PTR(-ENOENT); if (!name) return NULL; config = kvm_vq_config(kdev->desc)+index; err = vmem_add_mapping(config->address, vring_size(config->num, KVM_S390_VIRTIO_RING_ALIGN)); if (err) goto out; vq = vring_new_virtqueue(index, config->num, KVM_S390_VIRTIO_RING_ALIGN, vdev, true, (void *) config->address, kvm_notify, callback, name); if (!vq) { err = -ENOMEM; goto unmap; } /* * register a callback token * The host will sent this via the external interrupt parameter */ config->token = (u64) vq; vq->priv = config; return vq; unmap: vmem_remove_mapping(config->address, vring_size(config->num, KVM_S390_VIRTIO_RING_ALIGN)); out: return ERR_PTR(err); }
static int zynq_rpmsg_virtio_find_vqs(struct virtio_device *vdev, unsigned nvqs, struct virtqueue *vqs[], vq_callback_t *callbacks[], const char *names[]) { int i; struct zynq_rpmsg_vring *local_vring; void *vring_va; int size; /* Skip through the vrings. */ for (i = 0; i < nvqs; i++) { local_vring = &(zynq_rpmsg_p->vrings[i]); local_vring->len = zynq_rpmsg_p->num_descs; size = vring_size(zynq_rpmsg_p->num_descs, zynq_rpmsg_p->align); /* Allocate non-cacheable memory for the vring. */ local_vring->va = dma_alloc_coherent (&(zynq_rpmsg_platform->dev), size, &(local_vring->dma), GFP_KERNEL); vring_va = local_vring->va; memset(vring_va, 0, size); local_vring->vq = vring_new_virtqueue(i, zynq_rpmsg_p->num_descs, zynq_rpmsg_p->align, vdev, false, vring_va, zynq_rpmsg_virtio_notify, callbacks[i], names[i]); vqs[i] = local_vring->vq; } return 0; }
/* * This routine finds the Nth virtqueue described in the configuration of * this device and sets it up. * * This is kind of an ugly duckling. It'd be nicer to have a standard * representation of a virtqueue in the configuration space, but it seems that * everyone wants to do it differently. The KVM coders want the Guest to * allocate its own pages and tell the Host where they are, but for lguest it's * simpler for the Host to simply tell us where the pages are. */ static struct virtqueue *lg_find_vq(struct virtio_device *vdev, unsigned index, void (*callback)(struct virtqueue *vq), const char *name) { struct lguest_device *ldev = to_lgdev(vdev); struct lguest_vq_info *lvq; struct virtqueue *vq; int err; if (!name) return NULL; /* We must have this many virtqueues. */ if (index >= ldev->desc->num_vq) return ERR_PTR(-ENOENT); lvq = kmalloc(sizeof(*lvq), GFP_KERNEL); if (!lvq) return ERR_PTR(-ENOMEM); /* * Make a copy of the "struct lguest_vqconfig" entry, which sits after * the descriptor. We need a copy because the config space might not * be aligned correctly. */ memcpy(&lvq->config, lg_vq(ldev->desc)+index, sizeof(lvq->config)); printk("Mapping virtqueue %i addr %lx\n", index, (unsigned long)lvq->config.pfn << PAGE_SHIFT); /* Figure out how many pages the ring will take, and map that memory */ lvq->pages = lguest_map((unsigned long)lvq->config.pfn << PAGE_SHIFT, DIV_ROUND_UP(vring_size(lvq->config.num, LGUEST_VRING_ALIGN), PAGE_SIZE)); if (!lvq->pages) { err = -ENOMEM; goto free_lvq; } /* * OK, tell virtio_ring.c to set up a virtqueue now we know its size * and we've got a pointer to its pages. Note that we set weak_barriers * to 'true': the host just a(nother) SMP CPU, so we only need inter-cpu * barriers. */ vq = vring_new_virtqueue(index, lvq->config.num, LGUEST_VRING_ALIGN, vdev, true, lvq->pages, lg_notify, callback, name); if (!vq) { err = -ENOMEM; goto unmap; } /* Make sure the interrupt is allocated. */ err = lguest_setup_irq(lvq->config.irq); if (err) goto destroy_vring; /* * Tell the interrupt for this virtqueue to go to the virtio_ring * interrupt handler. * * FIXME: We used to have a flag for the Host to tell us we could use * the interrupt as a source of randomness: it'd be nice to have that * back. */ err = request_irq(lvq->config.irq, vring_interrupt, IRQF_SHARED, dev_name(&vdev->dev), vq); if (err) goto free_desc; /* * Last of all we hook up our 'struct lguest_vq_info" to the * virtqueue's priv pointer. */ vq->priv = lvq; return vq; free_desc: irq_free_desc(lvq->config.irq); destroy_vring: vring_del_virtqueue(vq); unmap: lguest_unmap(lvq->pages); free_lvq: kfree(lvq); return ERR_PTR(err); }
static struct virtqueue *vm_setup_vq(struct virtio_device *vdev, unsigned index, void (*callback)(struct virtqueue *vq), const char *name) { struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev); struct virtio_mmio_vq_info *info; struct virtqueue *vq; unsigned long flags, size; int err; /* Select the queue we're interested in */ writel(index, vm_dev->base + VIRTIO_MMIO_QUEUE_SEL); /* Queue shouldn't already be set up. */ if (readl(vm_dev->base + VIRTIO_MMIO_QUEUE_PFN)) { err = -ENOENT; goto error_available; } /* Allocate and fill out our active queue description */ info = kmalloc(sizeof(*info), GFP_KERNEL); if (!info) { err = -ENOMEM; goto error_kmalloc; } info->queue_index = index; /* Allocate pages for the queue - start with a queue as big as * possible (limited by maximum size allowed by device), drop down * to a minimal size, just big enough to fit descriptor table * and two rings (which makes it "alignment_size * 2") */ info->num = readl(vm_dev->base + VIRTIO_MMIO_QUEUE_NUM_MAX); while (1) { size = PAGE_ALIGN(vring_size(info->num, VIRTIO_MMIO_VRING_ALIGN)); /* Already smallest possible allocation? */ if (size <= VIRTIO_MMIO_VRING_ALIGN * 2) { err = -ENOMEM; goto error_alloc_pages; } info->queue = alloc_pages_exact(size, GFP_KERNEL | __GFP_ZERO); if (info->queue) break; info->num /= 2; } /* Activate the queue */ writel(info->num, vm_dev->base + VIRTIO_MMIO_QUEUE_NUM); writel(VIRTIO_MMIO_VRING_ALIGN, vm_dev->base + VIRTIO_MMIO_QUEUE_ALIGN); writel(virt_to_phys(info->queue) >> PAGE_SHIFT, vm_dev->base + VIRTIO_MMIO_QUEUE_PFN); /* Create the vring */ vq = vring_new_virtqueue(info->num, VIRTIO_MMIO_VRING_ALIGN, vdev, true, info->queue, vm_notify, callback, name); if (!vq) { err = -ENOMEM; goto error_new_virtqueue; } vq->priv = info; info->vq = vq; spin_lock_irqsave(&vm_dev->lock, flags); list_add(&info->node, &vm_dev->virtqueues); spin_unlock_irqrestore(&vm_dev->lock, flags); return vq; error_new_virtqueue: writel(0, vm_dev->base + VIRTIO_MMIO_QUEUE_PFN); free_pages_exact(info->queue, size); error_alloc_pages: kfree(info); error_kmalloc: error_available: return ERR_PTR(err); }
void vhost_vq_setup(struct vdev_info *dev, struct vq_info *info) { struct vhost_vring_state state = { .index = info->idx }; struct vhost_vring_file file = { .index = info->idx }; unsigned long long features = dev->vdev.features[0]; struct vhost_vring_addr addr = { .index = info->idx, .desc_user_addr = (uint64_t)(unsigned long)info->vring.desc, .avail_user_addr = (uint64_t)(unsigned long)info->vring.avail, .used_user_addr = (uint64_t)(unsigned long)info->vring.used, }; int r; r = ioctl(dev->control, VHOST_SET_FEATURES, &features); assert(r >= 0); state.num = info->vring.num; r = ioctl(dev->control, VHOST_SET_VRING_NUM, &state); assert(r >= 0); state.num = 0; r = ioctl(dev->control, VHOST_SET_VRING_BASE, &state); assert(r >= 0); r = ioctl(dev->control, VHOST_SET_VRING_ADDR, &addr); assert(r >= 0); file.fd = info->kick; r = ioctl(dev->control, VHOST_SET_VRING_KICK, &file); assert(r >= 0); file.fd = info->call; r = ioctl(dev->control, VHOST_SET_VRING_CALL, &file); assert(r >= 0); } static void vq_info_add(struct vdev_info *dev, int num) { struct vq_info *info = &dev->vqs[dev->nvqs]; int r; info->idx = dev->nvqs; info->kick = eventfd(0, EFD_NONBLOCK); info->call = eventfd(0, EFD_NONBLOCK); r = posix_memalign(&info->ring, 4096, vring_size(num, 4096)); assert(r >= 0); memset(info->ring, 0, vring_size(num, 4096)); vring_init(&info->vring, num, info->ring, 4096); info->vq = vring_new_virtqueue(info->vring.num, 4096, &dev->vdev, info->ring, vq_notify, vq_callback, "test"); assert(info->vq); info->vq->priv = info; vhost_vq_setup(dev, info); dev->fds[info->idx].fd = info->call; dev->fds[info->idx].events = POLLIN; dev->nvqs++; } static void vdev_info_init(struct vdev_info* dev, unsigned long long features) { int r; memset(dev, 0, sizeof *dev); dev->vdev.features[0] = features; dev->vdev.features[1] = features >> 32; dev->buf_size = 1024; dev->buf = malloc(dev->buf_size); assert(dev->buf); dev->control = open("/dev/vhost-test", O_RDWR); assert(dev->control >= 0); r = ioctl(dev->control, VHOST_SET_OWNER, NULL); assert(r >= 0); dev->mem = malloc(offsetof(struct vhost_memory, regions) + sizeof dev->mem->regions[0]); assert(dev->mem); memset(dev->mem, 0, offsetof(struct vhost_memory, regions) + sizeof dev->mem->regions[0]); dev->mem->nregions = 1; dev->mem->regions[0].guest_phys_addr = (long)dev->buf; dev->mem->regions[0].userspace_addr = (long)dev->buf; dev->mem->regions[0].memory_size = dev->buf_size; r = ioctl(dev->control, VHOST_SET_MEM_TABLE, dev->mem); assert(r >= 0); } /* TODO: this is pretty bad: we get a cache line bounce * for the wait queue on poll and another one on read, * plus the read which is there just to clear the * current state. */ static void wait_for_interrupt(struct vdev_info *dev) { int i; unsigned long long val; poll(dev->fds, dev->nvqs, -1); for (i = 0; i < dev->nvqs; ++i) if (dev->fds[i].revents & POLLIN) { read(dev->fds[i].fd, &val, sizeof val); } } static void run_test(struct vdev_info *dev, struct vq_info *vq, int bufs) { struct scatterlist sl; long started = 0, completed = 0; long completed_before; int r, test = 1; unsigned len; long long spurious = 0; r = ioctl(dev->control, VHOST_TEST_RUN, &test); assert(r >= 0); for (;;) { virtqueue_disable_cb(vq->vq); completed_before = completed; do { if (started < bufs) { sg_init_one(&sl, dev->buf, dev->buf_size); r = virtqueue_add_buf(vq->vq, &sl, 1, 0, dev->buf + started); if (likely(r >= 0)) { ++started; virtqueue_kick(vq->vq); } } else r = -1; /* Flush out completed bufs if any */ if (virtqueue_get_buf(vq->vq, &len)) { ++completed; r = 0; } } while (r >= 0); if (completed == completed_before) ++spurious; assert(completed <= bufs); assert(started <= bufs); if (completed == bufs) break; if (virtqueue_enable_cb(vq->vq)) { wait_for_interrupt(dev); } } test = 0; r = ioctl(dev->control, VHOST_TEST_RUN, &test); assert(r >= 0); fprintf(stderr, "spurious wakeus: 0x%llx\n", spurious); } const char optstring[] = "h"; const struct option longopts[] = { { .name = "help", .val = 'h', }, { .name = "event-idx",
/* * This routine will assign vring's allocated in host/io memory. Code in * virtio_ring.c however continues to access this io memory as if it were local * memory without io accessors. */ static struct virtqueue *mic_find_vq(struct virtio_device *vdev, unsigned index, void (*callback)(struct virtqueue *vq), const char *name) { struct mic_vdev *mvdev = to_micvdev(vdev); struct mic_vqconfig __iomem *vqconfig; struct mic_vqconfig config; struct virtqueue *vq; void __iomem *va; struct _mic_vring_info __iomem *info; void *used; int vr_size, _vr_size, err, magic; struct vring *vr; u8 type = ioread8(&mvdev->desc->type); if (index >= ioread8(&mvdev->desc->num_vq)) return ERR_PTR(-ENOENT); if (!name) return ERR_PTR(-ENOENT); /* First assign the vring's allocated in host memory */ vqconfig = mic_vq_config(mvdev->desc) + index; memcpy_fromio(&config, vqconfig, sizeof(config)); _vr_size = vring_size(le16_to_cpu(config.num), MIC_VIRTIO_RING_ALIGN); vr_size = PAGE_ALIGN(_vr_size + sizeof(struct _mic_vring_info)); va = mic_card_map(mvdev->mdev, le64_to_cpu(config.address), vr_size); if (!va) return ERR_PTR(-ENOMEM); mvdev->vr[index] = va; memset_io(va, 0x0, _vr_size); vq = vring_new_virtqueue(index, le16_to_cpu(config.num), MIC_VIRTIO_RING_ALIGN, vdev, false, (void __force *)va, mic_notify, callback, name); if (!vq) { err = -ENOMEM; goto unmap; } info = va + _vr_size; magic = ioread32(&info->magic); if (WARN(magic != MIC_MAGIC + type + index, "magic mismatch")) { err = -EIO; goto unmap; } /* Allocate and reassign used ring now */ mvdev->used_size[index] = PAGE_ALIGN(sizeof(__u16) * 3 + sizeof(struct vring_used_elem) * le16_to_cpu(config.num)); used = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, get_order(mvdev->used_size[index])); if (!used) { err = -ENOMEM; dev_err(mic_dev(mvdev), "%s %d err %d\n", __func__, __LINE__, err); goto del_vq; } iowrite64(virt_to_phys(used), &vqconfig->used_address); /* * To reassign the used ring here we are directly accessing * struct vring_virtqueue which is a private data structure * in virtio_ring.c. At the minimum, a BUILD_BUG_ON() in * vring_new_virtqueue() would ensure that * (&vq->vring == (struct vring *) (&vq->vq + 1)); */ vr = (struct vring *)(vq + 1); vr->used = used; vq->priv = mvdev; return vq; del_vq: vring_del_virtqueue(vq); unmap: mic_card_unmap(mvdev->mdev, mvdev->vr[index]); return ERR_PTR(err); }
/* * This routine finds the first virtqueue described in the configuration of * this device and sets it up. */ static struct virtqueue *kvm_find_vq(struct virtio_device *vdev, unsigned index, void (*callback)(struct virtqueue *vq), const char *name) { struct kvm_device *kdev = to_kvmdev(vdev); struct kvm_vqinfo *vqi; struct kvm_vqconfig *config; struct virtqueue *vq; long irq; int err = -EINVAL; if (index >= kdev->desc->num_vq) return ERR_PTR(-ENOENT); vqi = kzalloc(sizeof(*vqi), GFP_KERNEL); if (!vqi) return ERR_PTR(-ENOMEM); config = kvm_vq_config(kdev->desc)+index; vqi->config = config; vqi->pages = generic_remap_prot(config->pa, vring_size(config->num, KVM_TILE_VIRTIO_RING_ALIGN), 0, io_prot()); if (!vqi->pages) { err = -ENOMEM; goto out; } vq = vring_new_virtqueue(config->num, KVM_TILE_VIRTIO_RING_ALIGN, vdev, 0, vqi->pages, kvm_notify, callback, name); if (!vq) { err = -ENOMEM; goto unmap; } /* * Trigger the IPI interrupt in SW way. * TODO: We do not need to create one irq for each vq. A bit wasteful. */ irq = create_irq(); if (irq < 0) { err = -ENXIO; goto del_virtqueue; } tile_irq_activate(irq, TILE_IRQ_SW_CLEAR); if (request_irq(irq, vring_interrupt, 0, dev_name(&vdev->dev), vq)) { err = -ENXIO; destroy_irq(irq); goto del_virtqueue; } config->irq = irq; vq->priv = vqi; return vq; del_virtqueue: vring_del_virtqueue(vq); unmap: vunmap(vqi->pages); out: return ERR_PTR(err); }