static void virtballoon_remove(struct virtio_device *vdev) { struct virtio_balloon *vb = vdev->priv; if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_DEFLATE_ON_OOM)) virtio_balloon_unregister_shrinker(vb); spin_lock_irq(&vb->stop_update_lock); vb->stop_update = true; spin_unlock_irq(&vb->stop_update_lock); cancel_work_sync(&vb->update_balloon_size_work); cancel_work_sync(&vb->update_balloon_stats_work); if (virtio_has_feature(vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) { cancel_work_sync(&vb->report_free_page_work); destroy_workqueue(vb->balloon_wq); } remove_common(vb); #ifdef CONFIG_BALLOON_COMPACTION if (vb->vb_dev_info.inode) iput(vb->vb_dev_info.inode); kern_unmount(balloon_mnt); #endif kfree(vb); }
static int init_vqs(struct virtio_balloon *vb) { struct virtqueue *vqs[3]; vq_callback_t *callbacks[] = { balloon_ack, balloon_ack, stats_request }; const char *names[] = { "inflate", "deflate", "stats" }; int err, nvqs; /* * We expect two virtqueues: inflate and deflate, and * optionally stat. */ nvqs = virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_STATS_VQ) ? 3 : 2; err = vb->vdev->config->find_vqs(vb->vdev, nvqs, vqs, callbacks, names); if (err) return err; vb->inflate_vq = vqs[0]; vb->deflate_vq = vqs[1]; if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_STATS_VQ)) { struct scatterlist sg; vb->stats_vq = vqs[2]; /* * Prime this virtqueue with one buffer so the hypervisor can * use it to signal us later. */ sg_init_one(&sg, vb->stats, sizeof vb->stats); if (virtqueue_add_outbuf(vb->stats_vq, &sg, 1, vb, GFP_KERNEL) < 0) BUG(); virtqueue_kick(vb->stats_vq); } return 0; }
static int rpmsg_dev_remove(struct device *dev) { struct rpmsg_channel *rpdev = to_rpmsg_channel(dev); struct rpmsg_driver *rpdrv = to_rpmsg_driver(rpdev->dev.driver); struct virtproc_info *vrp = rpdev->vrp; int err = 0; /* tell remote processor's name service we're removing this channel */ if (rpdev->announce && virtio_has_feature(vrp->vdev, VIRTIO_RPMSG_F_NS)) { struct rpmsg_ns_msg nsm; strncpy(nsm.name, rpdev->id.name, RPMSG_NAME_SIZE); nsm.addr = rpdev->src; nsm.flags = RPMSG_NS_DESTROY; err = rpmsg_sendto(rpdev, &nsm, sizeof(nsm), RPMSG_NS_ADDR); if (err) dev_err(dev, "failed to announce service %d\n", err); } rpdrv->remove(rpdev); rpmsg_destroy_ept(rpdev->ept); return err; }
static void fill_balloon(struct virtio_balloon *vb, size_t num) { struct balloon_dev_info *vb_dev_info = &vb->vb_dev_info; /* We can only do one array worth at a time. */ num = min(num, ARRAY_SIZE(vb->pfns)); mutex_lock(&vb->balloon_lock); for (vb->num_pfns = 0; vb->num_pfns < num; vb->num_pfns += VIRTIO_BALLOON_PAGES_PER_PAGE) { struct page *page = balloon_page_enqueue(vb_dev_info); if (!page) { dev_info_ratelimited(&vb->vdev->dev, "Out of puff! Can't get %u pages\n", VIRTIO_BALLOON_PAGES_PER_PAGE); /* Sleep for at least 1/5 of a second before retry. */ msleep(200); break; } set_page_pfns(vb->pfns + vb->num_pfns, page); vb->num_pages += VIRTIO_BALLOON_PAGES_PER_PAGE; if (!virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_DEFLATE_ON_OOM)) adjust_managed_page_count(page, -1); } /* Did we get any? */ if (vb->num_pfns != 0) tell_host(vb, vb->inflate_vq); mutex_unlock(&vb->balloon_lock); }
static void update_balloon_size(struct virtio_balloon *vb) { u32 actual = vb->num_pages; /* Legacy balloon config space is LE, unlike all other devices. */ if (!virtio_has_feature(vb->vdev, VIRTIO_F_VERSION_1)) actual = (__force u32)cpu_to_le32(actual); virtio_cwrite(vb->vdev, struct virtio_balloon_config, actual, &actual); }
static int virtinput_probe(struct virtio_device *vdev) { struct virtio_input *vi; unsigned long flags; size_t size; int abs, err; if (!virtio_has_feature(vdev, VIRTIO_F_VERSION_1)) return -ENODEV; vi = kzalloc(sizeof(*vi), GFP_KERNEL); if (!vi) return -ENOMEM; vdev->priv = vi; vi->vdev = vdev; spin_lock_init(&vi->lock); err = virtinput_init_vqs(vi); if (err) goto err_init_vq; vi->idev = input_allocate_device(); if (!vi->idev) { err = -ENOMEM; goto err_input_alloc; } input_set_drvdata(vi->idev, vi); size = virtinput_cfg_select(vi, VIRTIO_INPUT_CFG_ID_NAME, 0); virtio_cread_bytes(vi->vdev, offsetof(struct virtio_input_config, u.string), vi->name, min(size, sizeof(vi->name))); size = virtinput_cfg_select(vi, VIRTIO_INPUT_CFG_ID_SERIAL, 0); virtio_cread_bytes(vi->vdev, offsetof(struct virtio_input_config, u.string), vi->serial, min(size, sizeof(vi->serial))); snprintf(vi->phys, sizeof(vi->phys), "virtio%d/input0", vdev->index); vi->idev->name = vi->name; vi->idev->phys = vi->phys; vi->idev->uniq = vi->serial; size = virtinput_cfg_select(vi, VIRTIO_INPUT_CFG_ID_DEVIDS, 0); if (size >= sizeof(struct virtio_input_devids)) { virtio_cread(vi->vdev, struct virtio_input_config, u.ids.bustype, &vi->idev->id.bustype); virtio_cread(vi->vdev, struct virtio_input_config, u.ids.vendor, &vi->idev->id.vendor); virtio_cread(vi->vdev, struct virtio_input_config, u.ids.product, &vi->idev->id.product); virtio_cread(vi->vdev, struct virtio_input_config, u.ids.version, &vi->idev->id.version); } else {
/* We provide getgeo only to please some old bootloader/partitioning tools */ static int virtblk_getgeo(struct block_device *bd, struct hd_geometry *geo) { struct virtio_blk *vblk = bd->bd_disk->private_data; /* see if the host passed in geometry config */ if (virtio_has_feature(vblk->vdev, VIRTIO_BLK_F_GEOMETRY)) { virtio_cread(vblk->vdev, struct virtio_blk_config, geometry.cylinders, &geo->cylinders); virtio_cread(vblk->vdev, struct virtio_blk_config, geometry.heads, &geo->heads); virtio_cread(vblk->vdev, struct virtio_blk_config, geometry.sectors, &geo->sectors); } else {
static void release_pages_balloon(struct virtio_balloon *vb) { unsigned int i; /* Find pfns pointing at start of each page, get pages and free them. */ for (i = 0; i < vb->num_pfns; i += VIRTIO_BALLOON_PAGES_PER_PAGE) { struct page *page = balloon_pfn_to_page(vb->pfns[i]); if (!virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_DEFLATE_ON_OOM)) adjust_managed_page_count(page, 1); put_page(page); /* balloon reference */ } }
static void release_pages_balloon(struct virtio_balloon *vb, struct list_head *pages) { struct page *page, *next; list_for_each_entry_safe(page, next, pages, lru) { if (!virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_DEFLATE_ON_OOM)) adjust_managed_page_count(page, 1); list_del(&page->lru); put_page(page); /* balloon reference */ } }
struct virtqueue *vring_new_virtqueue(unsigned int num, unsigned int vring_align, struct virtio_device *vdev, void *pages, void (*notify)(struct virtqueue *), void (*callback)(struct virtqueue *), const char *name) { struct vring_virtqueue *vq; unsigned int i; /* We assume num is a power of 2. */ if (num & (num - 1)) { dev_warn(&vdev->dev, "Bad virtqueue length %u\n", num); return NULL; } vq = kmalloc(sizeof(*vq) + sizeof(void *)*num, GFP_KERNEL); if (!vq) return NULL; vring_init(&vq->vring, num, pages, vring_align); vq->vq.callback = callback; vq->vq.vdev = vdev; vq->vq.name = name; vq->notify = notify; vq->broken = false; vq->last_used_idx = 0; vq->num_added = 0; list_add_tail(&vq->vq.list, &vdev->vqs); #ifdef DEBUG vq->in_use = false; #endif vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC); /* No callback? Tell other side not to bother us. */ if (!callback) vq->vring.avail->flags |= VRING_AVAIL_F_NO_INTERRUPT; /* Put everything in free lists. */ vq->num_free = num; vq->free_head = 0; for (i = 0; i < num-1; i++) { vq->vring.desc[i].next = i+1; vq->data[i] = NULL; } vq->data[i] = NULL; return &vq->vq; }
static unsigned fill_balloon(struct virtio_balloon *vb, size_t num) { unsigned num_allocated_pages; unsigned num_pfns; struct page *page; LIST_HEAD(pages); /* We can only do one array worth at a time. */ num = min(num, ARRAY_SIZE(vb->pfns)); for (num_pfns = 0; num_pfns < num; num_pfns += VIRTIO_BALLOON_PAGES_PER_PAGE) { struct page *page = balloon_page_alloc(); if (!page) { dev_info_ratelimited(&vb->vdev->dev, "Out of puff! Can't get %u pages\n", VIRTIO_BALLOON_PAGES_PER_PAGE); /* Sleep for at least 1/5 of a second before retry. */ msleep(200); break; } balloon_page_push(&pages, page); } mutex_lock(&vb->balloon_lock); vb->num_pfns = 0; while ((page = balloon_page_pop(&pages))) { balloon_page_enqueue(&vb->vb_dev_info, page); set_page_pfns(vb, vb->pfns + vb->num_pfns, page); vb->num_pages += VIRTIO_BALLOON_PAGES_PER_PAGE; if (!virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_DEFLATE_ON_OOM)) adjust_managed_page_count(page, -1); vb->num_pfns += VIRTIO_BALLOON_PAGES_PER_PAGE; } num_allocated_pages = vb->num_pfns; /* Did we get any? */ if (vb->num_pfns != 0) tell_host(vb, vb->inflate_vq); mutex_unlock(&vb->balloon_lock); return num_allocated_pages; }
void virtio_queue_set_notification(VirtQueue *vq, int enable) { vq->notification = enable; if (virtio_has_feature(vq->vdev, VIRTIO_RING_F_EVENT_IDX)) { vring_set_avail_event(vq, vring_avail_idx(vq)); } else if (enable) { vring_used_flags_unset_bit(vq, VRING_USED_F_NO_NOTIFY); } else { vring_used_flags_set_bit(vq, VRING_USED_F_NO_NOTIFY); } if (enable) { /* Expose avail event/used flags before caller checks the avail idx. */ smp_mb(); } }
static int virtblk_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd, unsigned long data) { struct gendisk *disk = bdev->bd_disk; struct virtio_blk *vblk = disk->private_data; /* * Only allow the generic SCSI ioctls if the host can support it. */ if (!virtio_has_feature(vblk->vdev, VIRTIO_BLK_F_SCSI)) return -ENOTTY; return scsi_cmd_blk_ioctl(bdev, mode, cmd, (void __user *)data); }
static inline s64 towards_target(struct virtio_balloon *vb) { s64 target; u32 num_pages; virtio_cread(vb->vdev, struct virtio_balloon_config, num_pages, &num_pages); /* Legacy balloon config space is LE, unlike all other devices. */ if (!virtio_has_feature(vb->vdev, VIRTIO_F_VERSION_1)) num_pages = le32_to_cpu((__force __le32)num_pages); target = num_pages; return target - vb->num_pages; }
static void virtio_balloon_device_realize(DeviceState *dev, Error **errp) { VirtIODevice *vdev = VIRTIO_DEVICE(dev); VirtIOBalloon *s = VIRTIO_BALLOON(dev); int ret; virtio_init(vdev, "virtio-balloon", VIRTIO_ID_BALLOON, sizeof(struct virtio_balloon_config)); ret = qemu_add_balloon_handler(virtio_balloon_to_target, virtio_balloon_stat, s); if (ret < 0) { error_setg(errp, "Only one balloon device is supported"); virtio_cleanup(vdev); return; } s->ivq = virtio_add_queue(vdev, 128, virtio_balloon_handle_output); s->dvq = virtio_add_queue(vdev, 128, virtio_balloon_handle_output); s->svq = virtio_add_queue(vdev, 128, virtio_balloon_receive_stats); if (virtio_has_feature(s->host_features, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) { s->free_page_vq = virtio_add_queue(vdev, VIRTQUEUE_MAX_SIZE, virtio_balloon_handle_free_page_vq); s->free_page_report_status = FREE_PAGE_REPORT_S_STOP; s->free_page_report_cmd_id = VIRTIO_BALLOON_FREE_PAGE_REPORT_CMD_ID_MIN; s->free_page_report_notify.notify = virtio_balloon_free_page_report_notify; precopy_add_notifier(&s->free_page_report_notify); if (s->iothread) { object_ref(OBJECT(s->iothread)); s->free_page_bh = aio_bh_new(iothread_get_aio_context(s->iothread), virtio_ballloon_get_free_page_hints, s); qemu_mutex_init(&s->free_page_lock); qemu_cond_init(&s->free_page_cond); s->block_iothread = false; } else { /* Simply disable this feature if the iothread wasn't created. */ s->host_features &= ~(1 << VIRTIO_BALLOON_F_FREE_PAGE_HINT); virtio_error(vdev, "iothread is missing"); } } reset_stats(s); }
/* * virtballoon_oom_notify - release pages when system is under severe * memory pressure (called from out_of_memory()) * @self : notifier block struct * @dummy: not used * @parm : returned - number of freed pages * * The balancing of memory by use of the virtio balloon should not cause * the termination of processes while there are pages in the balloon. * If virtio balloon manages to release some memory, it will make the * system return and retry the allocation that forced the OOM killer * to run. */ static int virtballoon_oom_notify(struct notifier_block *self, unsigned long dummy, void *parm) { struct virtio_balloon *vb; unsigned long *freed; unsigned num_freed_pages; vb = container_of(self, struct virtio_balloon, nb); if (!virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_DEFLATE_ON_OOM)) return NOTIFY_OK; freed = parm; num_freed_pages = leak_balloon(vb, oom_pages); update_balloon_size(vb); *freed += num_freed_pages; return NOTIFY_OK; }
static unsigned long virtio_balloon_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc) { unsigned long pages_to_free, pages_freed = 0; struct virtio_balloon *vb = container_of(shrinker, struct virtio_balloon, shrinker); pages_to_free = sc->nr_to_scan * VIRTIO_BALLOON_PAGES_PER_PAGE; if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) pages_freed = shrink_free_pages(vb, pages_to_free); if (pages_freed >= pages_to_free) return pages_freed; pages_freed += shrink_balloon_pages(vb, pages_to_free - pages_freed); return pages_freed; }
static int rpmsg_dev_probe(struct device *dev) { struct rpmsg_channel *rpdev = to_rpmsg_channel(dev); struct rpmsg_driver *rpdrv = to_rpmsg_driver(rpdev->dev.driver); struct virtproc_info *vrp = rpdev->vrp; struct rpmsg_endpoint *ept; int err; ept = rpmsg_create_ept(rpdev, rpdrv->callback, NULL, rpdev->src); if (!ept) { dev_err(dev, "failed to create endpoint\n"); err = -ENOMEM; goto out; } rpdev->ept = ept; rpdev->src = ept->addr; err = rpdrv->probe(rpdev); if (err) { dev_err(dev, "%s: failed: %d\n", __func__, err); rpmsg_destroy_ept(ept); goto out; } /* need to tell remote processor's name service about this channel ? */ if (rpdev->announce && virtio_has_feature(vrp->vdev, VIRTIO_RPMSG_F_NS)) { struct rpmsg_ns_msg nsm; strncpy(nsm.name, rpdev->id.name, RPMSG_NAME_SIZE); nsm.addr = rpdev->src; nsm.flags = RPMSG_NS_CREATE; err = rpmsg_sendto(rpdev, &nsm, sizeof(nsm), RPMSG_NS_ADDR); if (err) dev_err(dev, "failed to announce service %d\n", err); } out: return err; }
static int virtio_finalize_features(struct virtio_device *dev) { int ret = dev->config->finalize_features(dev); unsigned status; if (ret) return ret; if (!virtio_has_feature(dev, VIRTIO_F_VERSION_1)) return 0; add_status(dev, VIRTIO_CONFIG_S_FEATURES_OK); status = dev->config->get_status(dev); if (!(status & VIRTIO_CONFIG_S_FEATURES_OK)) { dev_err(&dev->dev, "virtio: device refuses features: %x\n", status); return -ENODEV; } return 0; }
static int vhost_net_set_vnet_endian(VirtIODevice *dev, NetClientState *peer, bool set) { int r = 0; if (virtio_has_feature(dev, VIRTIO_F_VERSION_1) || (virtio_legacy_is_cross_endian(dev) && !virtio_is_big_endian(dev))) { r = qemu_set_vnet_le(peer, set); if (r) { error_report("backend does not support LE vnet headers"); } } else if (virtio_legacy_is_cross_endian(dev)) { r = qemu_set_vnet_be(peer, set); if (r) { error_report("backend does not support BE vnet headers"); } } return r; }
/* * Allocate/free a vq. */ struct virtqueue * virtio_alloc_vq(struct virtio_softc *sc, unsigned int index, unsigned int size, unsigned int indirect_num, const char *name) { int vq_size, allocsize1, allocsize2, allocsize = 0; int ret; unsigned int ncookies; size_t len; struct virtqueue *vq; ddi_put16(sc->sc_ioh, /* LINTED E_BAD_PTR_CAST_ALIGN */ (uint16_t *)(sc->sc_io_addr + VIRTIO_CONFIG_QUEUE_SELECT), index); vq_size = ddi_get16(sc->sc_ioh, /* LINTED E_BAD_PTR_CAST_ALIGN */ (uint16_t *)(sc->sc_io_addr + VIRTIO_CONFIG_QUEUE_SIZE)); if (vq_size == 0) { dev_err(sc->sc_dev, CE_WARN, "virtqueue dest not exist, index %d for %s\n", index, name); goto out; } vq = kmem_zalloc(sizeof (struct virtqueue), KM_SLEEP); /* size 0 => use native vq size, good for receive queues. */ if (size) vq_size = MIN(vq_size, size); /* allocsize1: descriptor table + avail ring + pad */ allocsize1 = VIRTQUEUE_ALIGN(sizeof (struct vring_desc) * vq_size + sizeof (struct vring_avail) + sizeof (uint16_t) * vq_size); /* allocsize2: used ring + pad */ allocsize2 = VIRTQUEUE_ALIGN(sizeof (struct vring_used) + sizeof (struct vring_used_elem) * vq_size); allocsize = allocsize1 + allocsize2; ret = ddi_dma_alloc_handle(sc->sc_dev, &virtio_vq_dma_attr, DDI_DMA_SLEEP, NULL, &vq->vq_dma_handle); if (ret != DDI_SUCCESS) { dev_err(sc->sc_dev, CE_WARN, "Failed to allocate dma handle for vq %d", index); goto out_alloc_handle; } ret = ddi_dma_mem_alloc(vq->vq_dma_handle, allocsize, &virtio_vq_devattr, DDI_DMA_CONSISTENT, DDI_DMA_SLEEP, NULL, (caddr_t *)&vq->vq_vaddr, &len, &vq->vq_dma_acch); if (ret != DDI_SUCCESS) { dev_err(sc->sc_dev, CE_WARN, "Failed to allocate dma memory for vq %d", index); goto out_alloc; } ret = ddi_dma_addr_bind_handle(vq->vq_dma_handle, NULL, (caddr_t)vq->vq_vaddr, len, DDI_DMA_RDWR | DDI_DMA_CONSISTENT, DDI_DMA_SLEEP, NULL, &vq->vq_dma_cookie, &ncookies); if (ret != DDI_DMA_MAPPED) { dev_err(sc->sc_dev, CE_WARN, "Failed to bind dma memory for vq %d", index); goto out_bind; } /* We asked for a single segment */ ASSERT(ncookies == 1); /* and page-ligned buffers. */ ASSERT(vq->vq_dma_cookie.dmac_laddress % VIRTIO_PAGE_SIZE == 0); (void) memset(vq->vq_vaddr, 0, allocsize); /* Make sure all zeros hit the buffer before we point the host to it */ membar_producer(); /* set the vq address */ ddi_put32(sc->sc_ioh, /* LINTED E_BAD_PTR_CAST_ALIGN */ (uint32_t *)(sc->sc_io_addr + VIRTIO_CONFIG_QUEUE_ADDRESS), (vq->vq_dma_cookie.dmac_laddress / VIRTIO_PAGE_SIZE)); /* remember addresses and offsets for later use */ vq->vq_owner = sc; vq->vq_num = vq_size; vq->vq_index = index; vq->vq_descs = vq->vq_vaddr; vq->vq_availoffset = sizeof (struct vring_desc)*vq_size; vq->vq_avail = (void *)(((char *)vq->vq_descs) + vq->vq_availoffset); vq->vq_usedoffset = allocsize1; vq->vq_used = (void *)(((char *)vq->vq_descs) + vq->vq_usedoffset); ASSERT(indirect_num == 0 || virtio_has_feature(sc, VIRTIO_F_RING_INDIRECT_DESC)); vq->vq_indirect_num = indirect_num; /* free slot management */ vq->vq_entries = kmem_zalloc(sizeof (struct vq_entry) * vq_size, KM_SLEEP); ret = virtio_init_vq(sc, vq); if (ret) goto out_init; dev_debug(sc->sc_dev, CE_NOTE, "Allocated %d entries for vq %d:%s (%d indirect descs)", vq_size, index, name, indirect_num * vq_size); return (vq); out_init: kmem_free(vq->vq_entries, sizeof (struct vq_entry) * vq_size); (void) ddi_dma_unbind_handle(vq->vq_dma_handle); out_bind: ddi_dma_mem_free(&vq->vq_dma_acch); out_alloc: ddi_dma_free_handle(&vq->vq_dma_handle); out_alloc_handle: kmem_free(vq, sizeof (struct virtqueue)); out: return (NULL); }
static int rpmsg_probe(struct virtio_device *vdev) { vq_callback_t *vq_cbs[] = { rpmsg_recv_done, rpmsg_xmit_done }; const char *names[] = { "input", "output" }; struct virtqueue *vqs[2]; struct virtproc_info *vrp; void *addr; int err, i, num_bufs, buf_size, total_buf_size; struct rpmsg_channel_info *ch; vrp = kzalloc(sizeof(*vrp), GFP_KERNEL); if (!vrp) return -ENOMEM; vrp->vdev = vdev; idr_init(&vrp->endpoints); spin_lock_init(&vrp->endpoints_lock); mutex_init(&vrp->svq_lock); mutex_init(&vrp->rm_lock); init_waitqueue_head(&vrp->sendq); /* We expect two virtqueues, rx and tx (in this order) */ err = vdev->config->find_vqs(vdev, 2, vqs, vq_cbs, names); if (err) goto free_vi; vrp->rvq = vqs[0]; vrp->svq = vqs[1]; /* Platform must supply pre-allocated uncached buffers for now */ vdev->config->get(vdev, VPROC_BUF_ADDR, &addr, sizeof(addr)); vdev->config->get(vdev, VPROC_BUF_NUM, &num_bufs, sizeof(num_bufs)); vdev->config->get(vdev, VPROC_BUF_SZ, &buf_size, sizeof(buf_size)); total_buf_size = num_bufs * buf_size; dev_dbg(&vdev->dev, "%d buffers, size %d, addr 0x%x, total 0x%x\n", num_bufs, buf_size, (unsigned int) addr, total_buf_size); vrp->num_bufs = num_bufs; vrp->buf_size = buf_size; vrp->rbufs = addr; vrp->sbufs = addr + total_buf_size / 2; /* simulated addr base to make virt_to_page happy */ vdev->config->get(vdev, VPROC_SIM_BASE, &vrp->sim_base, sizeof(vrp->sim_base)); /* set up the receive buffers */ for (i = 0; i < num_bufs / 2; i++) { struct scatterlist sg; void *tmpaddr = vrp->rbufs + i * buf_size; void *simaddr = vrp->sim_base + i * buf_size; sg_init_one(&sg, simaddr, buf_size); err = virtqueue_add_buf_gfp(vrp->rvq, &sg, 0, 1, tmpaddr, GFP_KERNEL); WARN_ON(err < 0); /* sanity check; this can't really happen */ } /* tell the remote processor it can start sending data */ virtqueue_kick(vrp->rvq); /* suppress "tx-complete" interrupts */ virtqueue_disable_cb(vrp->svq); vdev->priv = vrp; dev_info(&vdev->dev, "rpmsg backend virtproc probed successfully\n"); /* if supported by the remote processor, enable the name service */ if (virtio_has_feature(vdev, VIRTIO_RPMSG_F_NS)) { vrp->ns_ept = __rpmsg_create_ept(vrp, NULL, rpmsg_ns_cb, vrp, RPMSG_NS_ADDR); if (!vrp->ns_ept) { dev_err(&vdev->dev, "failed to create the ns ept\n"); err = -ENOMEM; goto vqs_del; } } /* look for platform-specific static channels */ vdev->config->get(vdev, VPROC_STATIC_CHANNELS, &ch, sizeof(ch)); for (i = 0; ch && ch[i].name[0]; i++) rpmsg_create_channel(vrp, &ch[i]); return 0; vqs_del: vdev->config->del_vqs(vrp->vdev); free_vi: kfree(vrp); return err; }
static int rpmsg_probe(struct virtio_device *vdev) { vq_callback_t *vq_cbs[] = { rpmsg_recv_done, rpmsg_xmit_done }; const char *names[] = { "input", "output" }; struct virtqueue *vqs[2]; struct virtproc_info *vrp; void *bufs_va; int err = 0, i; vrp = kzalloc(sizeof(*vrp), GFP_KERNEL); if (!vrp) return -ENOMEM; vrp->vdev = vdev; idr_init(&vrp->endpoints); mutex_init(&vrp->endpoints_lock); mutex_init(&vrp->tx_lock); init_waitqueue_head(&vrp->sendq); err = vdev->config->find_vqs(vdev, 2, vqs, vq_cbs, names); if (err) goto free_vrp; vrp->rvq = vqs[0]; vrp->svq = vqs[1]; bufs_va = dma_alloc_coherent(vdev->dev.parent, RPMSG_TOTAL_BUF_SPACE, &vrp->bufs_dma, GFP_KERNEL); if (!bufs_va) goto vqs_del; dev_dbg(&vdev->dev, "buffers: va %p, dma 0x%llx\n", bufs_va, (unsigned long long)vrp->bufs_dma); vrp->rbufs = bufs_va; vrp->sbufs = bufs_va + RPMSG_TOTAL_BUF_SPACE / 2; for (i = 0; i < RPMSG_NUM_BUFS / 2; i++) { struct scatterlist sg; void *cpu_addr = vrp->rbufs + i * RPMSG_BUF_SIZE; sg_init_one(&sg, cpu_addr, RPMSG_BUF_SIZE); err = virtqueue_add_buf(vrp->rvq, &sg, 0, 1, cpu_addr, GFP_KERNEL); WARN_ON(err < 0); } virtqueue_disable_cb(vrp->svq); vdev->priv = vrp; if (virtio_has_feature(vdev, VIRTIO_RPMSG_F_NS)) { vrp->ns_ept = __rpmsg_create_ept(vrp, NULL, rpmsg_ns_cb, vrp, RPMSG_NS_ADDR); if (!vrp->ns_ept) { dev_err(&vdev->dev, "failed to create the ns ept\n"); err = -ENOMEM; goto free_coherent; } } virtqueue_kick(vrp->rvq); dev_info(&vdev->dev, "rpmsg host is online\n"); return 0; free_coherent: dma_free_coherent(vdev->dev.parent, RPMSG_TOTAL_BUF_SPACE, bufs_va, vrp->bufs_dma); vqs_del: vdev->config->del_vqs(vrp->vdev); free_vrp: kfree(vrp); return err; }
static int virtio_blk_handle_scsi_req(VirtIOBlockReq *req) { int status = VIRTIO_BLK_S_OK; struct virtio_scsi_inhdr *scsi = NULL; VirtIODevice *vdev = VIRTIO_DEVICE(req->dev); VirtQueueElement *elem = &req->elem; VirtIOBlock *blk = req->dev; #ifdef __linux__ int i; VirtIOBlockIoctlReq *ioctl_req; BlockAIOCB *acb; #endif /* * We require at least one output segment each for the virtio_blk_outhdr * and the SCSI command block. * * We also at least require the virtio_blk_inhdr, the virtio_scsi_inhdr * and the sense buffer pointer in the input segments. */ if (elem->out_num < 2 || elem->in_num < 3) { status = VIRTIO_BLK_S_IOERR; goto fail; } /* * The scsi inhdr is placed in the second-to-last input segment, just * before the regular inhdr. */ scsi = (void *)elem->in_sg[elem->in_num - 2].iov_base; if (!virtio_has_feature(blk->host_features, VIRTIO_BLK_F_SCSI)) { status = VIRTIO_BLK_S_UNSUPP; goto fail; } /* * No support for bidirection commands yet. */ if (elem->out_num > 2 && elem->in_num > 3) { status = VIRTIO_BLK_S_UNSUPP; goto fail; } #ifdef __linux__ ioctl_req = g_new0(VirtIOBlockIoctlReq, 1); ioctl_req->req = req; ioctl_req->hdr.interface_id = 'S'; ioctl_req->hdr.cmd_len = elem->out_sg[1].iov_len; ioctl_req->hdr.cmdp = elem->out_sg[1].iov_base; ioctl_req->hdr.dxfer_len = 0; if (elem->out_num > 2) { /* * If there are more than the minimally required 2 output segments * there is write payload starting from the third iovec. */ ioctl_req->hdr.dxfer_direction = SG_DXFER_TO_DEV; ioctl_req->hdr.iovec_count = elem->out_num - 2; for (i = 0; i < ioctl_req->hdr.iovec_count; i++) { ioctl_req->hdr.dxfer_len += elem->out_sg[i + 2].iov_len; } ioctl_req->hdr.dxferp = elem->out_sg + 2; } else if (elem->in_num > 3) { /* * If we have more than 3 input segments the guest wants to actually * read data. */ ioctl_req->hdr.dxfer_direction = SG_DXFER_FROM_DEV; ioctl_req->hdr.iovec_count = elem->in_num - 3; for (i = 0; i < ioctl_req->hdr.iovec_count; i++) { ioctl_req->hdr.dxfer_len += elem->in_sg[i].iov_len; } ioctl_req->hdr.dxferp = elem->in_sg; } else { /* * Some SCSI commands don't actually transfer any data. */ ioctl_req->hdr.dxfer_direction = SG_DXFER_NONE; } ioctl_req->hdr.sbp = elem->in_sg[elem->in_num - 3].iov_base; ioctl_req->hdr.mx_sb_len = elem->in_sg[elem->in_num - 3].iov_len; acb = blk_aio_ioctl(blk->blk, SG_IO, &ioctl_req->hdr, virtio_blk_ioctl_complete, ioctl_req); if (!acb) { g_free(ioctl_req); status = VIRTIO_BLK_S_UNSUPP; goto fail; } return -EINPROGRESS; #else abort(); #endif fail: /* Just put anything nonzero so that the ioctl fails in the guest. */ if (scsi) { virtio_stl_p(vdev, &scsi->errors, 255); } return status; }
static int rpmsg_probe(struct virtio_device *vdev) { vq_callback_t *vq_cbs[] = { rpmsg_recv_done, rpmsg_xmit_done }; const char *names[] = { "input", "output" }; struct virtqueue *vqs[2]; struct virtproc_info *vrp; void *bufs_va; int err = 0, i, vproc_id; vrp = kzalloc(sizeof(*vrp), GFP_KERNEL); if (!vrp) return -ENOMEM; vrp->vdev = vdev; idr_init(&vrp->endpoints); mutex_init(&vrp->endpoints_lock); mutex_init(&vrp->tx_lock); init_waitqueue_head(&vrp->sendq); if (!idr_pre_get(&vprocs, GFP_KERNEL)) goto free_vrp; mutex_lock(&vprocs_mutex); err = idr_get_new(&vprocs, vrp, &vproc_id); mutex_unlock(&vprocs_mutex); if (err) { dev_err(&vdev->dev, "idr_get_new failed: %d\n", err); goto free_vrp; } vrp->id = vproc_id; /* We expect two virtqueues, rx and tx (and in this order) */ err = vdev->config->find_vqs(vdev, 2, vqs, vq_cbs, names); if (err) goto rem_idr; vrp->rvq = vqs[0]; vrp->svq = vqs[1]; /* allocate coherent memory for the buffers */ bufs_va = dma_alloc_coherent(vdev->dev.parent->parent, RPMSG_TOTAL_BUF_SPACE, &vrp->bufs_dma, GFP_KERNEL); if (!bufs_va) goto vqs_del; dev_dbg(&vdev->dev, "buffers: va %p, dma 0x%llx\n", bufs_va, (unsigned long long)vrp->bufs_dma); /* half of the buffers is dedicated for RX */ vrp->rbufs = bufs_va; /* and half is dedicated for TX */ vrp->sbufs = bufs_va + RPMSG_TOTAL_BUF_SPACE / 2; /* set up the receive buffers */ for (i = 0; i < RPMSG_NUM_BUFS / 2; i++) { struct scatterlist sg; void *cpu_addr = vrp->rbufs + i * RPMSG_BUF_SIZE; sg_init_one(&sg, cpu_addr, RPMSG_BUF_SIZE); err = virtqueue_add_buf(vrp->rvq, &sg, 0, 1, cpu_addr, GFP_KERNEL); WARN_ON(err); /* sanity check; this can't really happen */ } /* suppress "tx-complete" interrupts */ virtqueue_disable_cb(vrp->svq); vdev->priv = vrp; /* if supported by the remote processor, enable the name service */ if (virtio_has_feature(vdev, VIRTIO_RPMSG_F_NS)) { /* a dedicated endpoint handles the name service msgs */ vrp->ns_ept = __rpmsg_create_ept(vrp, NULL, rpmsg_ns_cb, vrp, RPMSG_NS_ADDR); if (!vrp->ns_ept) { dev_err(&vdev->dev, "failed to create the ns ept\n"); err = -ENOMEM; goto free_coherent; } } /* tell the remote processor it can start sending messages */ virtqueue_kick(vrp->rvq); dev_info(&vdev->dev, "rpmsg host is online\n"); return 0; free_coherent: dma_free_coherent(vdev->dev.parent->parent, RPMSG_TOTAL_BUF_SPACE, bufs_va, vrp->bufs_dma); vqs_del: vdev->config->del_vqs(vrp->vdev); rem_idr: mutex_lock(&vprocs_mutex); idr_remove(&vprocs, vproc_id); mutex_unlock(&vprocs_mutex); free_vrp: kfree(vrp); return err; }
static int vhost_user_set_log_base(struct vhost_dev *dev, uint64_t base, struct vhost_log *log) { int fds[VHOST_MEMORY_MAX_NREGIONS]; size_t fd_num = 0; bool shmfd = virtio_has_feature(dev->protocol_features, VHOST_USER_PROTOCOL_F_LOG_SHMFD); VhostUserMsg msg = { .request = VHOST_USER_SET_LOG_BASE, .flags = VHOST_USER_VERSION, .payload.u64 = base, .size = sizeof(msg.payload.u64), }; if (shmfd && log->fd != -1) { fds[fd_num++] = log->fd; } vhost_user_write(dev, &msg, fds, fd_num); if (shmfd) { msg.size = 0; if (vhost_user_read(dev, &msg) < 0) { return 0; } if (msg.request != VHOST_USER_SET_LOG_BASE) { error_report("Received unexpected msg type. " "Expected %d received %d", VHOST_USER_SET_LOG_BASE, msg.request); return -1; } } return 0; } static int vhost_user_set_mem_table(struct vhost_dev *dev, struct vhost_memory *mem) { int fds[VHOST_MEMORY_MAX_NREGIONS]; int i, fd; size_t fd_num = 0; VhostUserMsg msg = { .request = VHOST_USER_SET_MEM_TABLE, .flags = VHOST_USER_VERSION, }; for (i = 0; i < dev->mem->nregions; ++i) { struct vhost_memory_region *reg = dev->mem->regions + i; ram_addr_t ram_addr; assert((uintptr_t)reg->userspace_addr == reg->userspace_addr); qemu_ram_addr_from_host((void *)(uintptr_t)reg->userspace_addr, &ram_addr); fd = qemu_get_ram_fd(ram_addr); if (fd > 0) { msg.payload.memory.regions[fd_num].userspace_addr = reg->userspace_addr; msg.payload.memory.regions[fd_num].memory_size = reg->memory_size; msg.payload.memory.regions[fd_num].guest_phys_addr = reg->guest_phys_addr; msg.payload.memory.regions[fd_num].mmap_offset = reg->userspace_addr - (uintptr_t) qemu_get_ram_block_host_ptr(ram_addr); assert(fd_num < VHOST_MEMORY_MAX_NREGIONS); fds[fd_num++] = fd; } } msg.payload.memory.nregions = fd_num; if (!fd_num) { error_report("Failed initializing vhost-user memory map, " "consider using -object memory-backend-file share=on"); return -1; } msg.size = sizeof(msg.payload.memory.nregions); msg.size += sizeof(msg.payload.memory.padding); msg.size += fd_num * sizeof(VhostUserMemoryRegion); vhost_user_write(dev, &msg, fds, fd_num); return 0; }
static int virtballoon_probe(struct virtio_device *vdev) { struct virtio_balloon *vb; __u32 poison_val; int err; if (!vdev->config->get) { dev_err(&vdev->dev, "%s failure: config access disabled\n", __func__); return -EINVAL; } vdev->priv = vb = kzalloc(sizeof(*vb), GFP_KERNEL); if (!vb) { err = -ENOMEM; goto out; } INIT_WORK(&vb->update_balloon_stats_work, update_balloon_stats_func); INIT_WORK(&vb->update_balloon_size_work, update_balloon_size_func); spin_lock_init(&vb->stop_update_lock); mutex_init(&vb->balloon_lock); init_waitqueue_head(&vb->acked); vb->vdev = vdev; balloon_devinfo_init(&vb->vb_dev_info); err = init_vqs(vb); if (err) goto out_free_vb; #ifdef CONFIG_BALLOON_COMPACTION balloon_mnt = kern_mount(&balloon_fs); if (IS_ERR(balloon_mnt)) { err = PTR_ERR(balloon_mnt); goto out_del_vqs; } vb->vb_dev_info.migratepage = virtballoon_migratepage; vb->vb_dev_info.inode = alloc_anon_inode(balloon_mnt->mnt_sb); if (IS_ERR(vb->vb_dev_info.inode)) { err = PTR_ERR(vb->vb_dev_info.inode); kern_unmount(balloon_mnt); goto out_del_vqs; } vb->vb_dev_info.inode->i_mapping->a_ops = &balloon_aops; #endif if (virtio_has_feature(vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) { /* * There is always one entry reserved for cmd id, so the ring * size needs to be at least two to report free page hints. */ if (virtqueue_get_vring_size(vb->free_page_vq) < 2) { err = -ENOSPC; goto out_del_vqs; } vb->balloon_wq = alloc_workqueue("balloon-wq", WQ_FREEZABLE | WQ_CPU_INTENSIVE, 0); if (!vb->balloon_wq) { err = -ENOMEM; goto out_del_vqs; } INIT_WORK(&vb->report_free_page_work, report_free_page_func); vb->cmd_id_received = VIRTIO_BALLOON_CMD_ID_STOP; vb->cmd_id_active = cpu_to_virtio32(vb->vdev, VIRTIO_BALLOON_CMD_ID_STOP); vb->cmd_id_stop = cpu_to_virtio32(vb->vdev, VIRTIO_BALLOON_CMD_ID_STOP); vb->num_free_page_blocks = 0; spin_lock_init(&vb->free_page_list_lock); INIT_LIST_HEAD(&vb->free_page_list); if (virtio_has_feature(vdev, VIRTIO_BALLOON_F_PAGE_POISON)) { memset(&poison_val, PAGE_POISON, sizeof(poison_val)); virtio_cwrite(vb->vdev, struct virtio_balloon_config, poison_val, &poison_val); } } /* * We continue to use VIRTIO_BALLOON_F_DEFLATE_ON_OOM to decide if a * shrinker needs to be registered to relieve memory pressure. */ if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_DEFLATE_ON_OOM)) { err = virtio_balloon_register_shrinker(vb); if (err) goto out_del_balloon_wq; } virtio_device_ready(vdev); if (towards_target(vb)) virtballoon_changed(vdev); return 0; out_del_balloon_wq: if (virtio_has_feature(vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) destroy_workqueue(vb->balloon_wq); out_del_vqs: vdev->config->del_vqs(vdev); out_free_vb: kfree(vb); out: return err; }
static int init_vqs(struct virtio_balloon *vb) { struct virtqueue *vqs[VIRTIO_BALLOON_VQ_MAX]; vq_callback_t *callbacks[VIRTIO_BALLOON_VQ_MAX]; const char *names[VIRTIO_BALLOON_VQ_MAX]; int err; /* * Inflateq and deflateq are used unconditionally. The names[] * will be NULL if the related feature is not enabled, which will * cause no allocation for the corresponding virtqueue in find_vqs. */ callbacks[VIRTIO_BALLOON_VQ_INFLATE] = balloon_ack; names[VIRTIO_BALLOON_VQ_INFLATE] = "inflate"; callbacks[VIRTIO_BALLOON_VQ_DEFLATE] = balloon_ack; names[VIRTIO_BALLOON_VQ_DEFLATE] = "deflate"; names[VIRTIO_BALLOON_VQ_STATS] = NULL; names[VIRTIO_BALLOON_VQ_FREE_PAGE] = NULL; if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_STATS_VQ)) { names[VIRTIO_BALLOON_VQ_STATS] = "stats"; callbacks[VIRTIO_BALLOON_VQ_STATS] = stats_request; } if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) { names[VIRTIO_BALLOON_VQ_FREE_PAGE] = "free_page_vq"; callbacks[VIRTIO_BALLOON_VQ_FREE_PAGE] = NULL; } err = vb->vdev->config->find_vqs(vb->vdev, VIRTIO_BALLOON_VQ_MAX, vqs, callbacks, names, NULL, NULL); if (err) return err; vb->inflate_vq = vqs[VIRTIO_BALLOON_VQ_INFLATE]; vb->deflate_vq = vqs[VIRTIO_BALLOON_VQ_DEFLATE]; if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_STATS_VQ)) { struct scatterlist sg; unsigned int num_stats; vb->stats_vq = vqs[VIRTIO_BALLOON_VQ_STATS]; /* * Prime this virtqueue with one buffer so the hypervisor can * use it to signal us later (it can't be broken yet!). */ num_stats = update_balloon_stats(vb); sg_init_one(&sg, vb->stats, sizeof(vb->stats[0]) * num_stats); err = virtqueue_add_outbuf(vb->stats_vq, &sg, 1, vb, GFP_KERNEL); if (err) { dev_warn(&vb->vdev->dev, "%s: add stat_vq failed\n", __func__); return err; } virtqueue_kick(vb->stats_vq); } if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) vb->free_page_vq = vqs[VIRTIO_BALLOON_VQ_FREE_PAGE]; return 0; }
static int rpmsg_probe(struct virtio_device *vdev) { vq_callback_t *vq_cbs[] = { rpmsg_recv_done, rpmsg_xmit_done }; const char *names[] = { "input", "output" }; struct virtqueue *vqs[2]; struct virtproc_info *vrp; struct rproc *vrp_rproc; void *bufs_va; void *cpu_addr; /* buffer virtual address */ void *cpu_addr_dma; /* buffer DMA address' virutal address conversion */ void *rbufs_guest_addr_kva; int err = 0, i; size_t total_buf_space; vrp = kzalloc(sizeof(*vrp), GFP_KERNEL); if (!vrp) return -ENOMEM; vrp->vdev = vdev; idr_init(&vrp->endpoints); mutex_init(&vrp->endpoints_lock); mutex_init(&vrp->tx_lock); init_waitqueue_head(&vrp->sendq); /* We expect two virtqueues, rx and tx (and in this order) */ err = vdev->config->find_vqs(vdev, 2, vqs, vq_cbs, names); if (err) goto free_vrp; vrp->rvq = vqs[0]; vrp->svq = vqs[1]; /* we expect symmetric tx/rx vrings */ WARN_ON(virtqueue_get_vring_size(vrp->rvq) != virtqueue_get_vring_size(vrp->svq)); /* we need less buffers if vrings are small */ if (virtqueue_get_vring_size(vrp->rvq) < MAX_RPMSG_NUM_BUFS / 2) vrp->num_bufs = virtqueue_get_vring_size(vrp->rvq) * 2; else vrp->num_bufs = MAX_RPMSG_NUM_BUFS; total_buf_space = vrp->num_bufs * RPMSG_BUF_SIZE; /* allocate coherent memory for the buffers */ bufs_va = dma_alloc_coherent(vdev->dev.parent->parent, total_buf_space, &vrp->bufs_dma, GFP_KERNEL); if (!bufs_va) { err = -ENOMEM; goto vqs_del; } dev_dbg(&vdev->dev, "buffers: va %p, dma 0x%llx\n", bufs_va, (unsigned long long)vrp->bufs_dma); /* half of the buffers is dedicated for RX */ vrp->rbufs = bufs_va; /* and half is dedicated for TX */ vrp->sbufs = bufs_va + total_buf_space / 2; vrp_rproc = vdev_to_rproc(vdev); rbufs_guest_addr_kva = vrp->rbufs; if (vrp_rproc->ops->kva_to_guest_addr_kva) { rbufs_guest_addr_kva = vrp_rproc->ops->kva_to_guest_addr_kva(vrp_rproc, vrp->rbufs, vrp->rvq); } /* set up the receive buffers */ for (i = 0; i < vrp->num_bufs / 2; i++) { struct scatterlist sg; cpu_addr = vrp->rbufs + i * RPMSG_BUF_SIZE; cpu_addr_dma = rbufs_guest_addr_kva + i*RPMSG_BUF_SIZE; sg_init_one(&sg, cpu_addr_dma, RPMSG_BUF_SIZE); err = virtqueue_add_inbuf(vrp->rvq, &sg, 1, cpu_addr, GFP_KERNEL); WARN_ON(err); /* sanity check; this can't really happen */ } /* suppress "tx-complete" interrupts */ virtqueue_disable_cb(vrp->svq); vdev->priv = vrp; /* if supported by the remote processor, enable the name service */ if (virtio_has_feature(vdev, VIRTIO_RPMSG_F_NS)) { /* a dedicated endpoint handles the name service msgs */ vrp->ns_ept = __rpmsg_create_ept(vrp, NULL, rpmsg_ns_cb, vrp, RPMSG_NS_ADDR); if (!vrp->ns_ept) { dev_err(&vdev->dev, "failed to create the ns ept\n"); err = -ENOMEM; goto free_coherent; } } /* tell the remote processor it can start sending messages */ virtqueue_kick(vrp->rvq); dev_info(&vdev->dev, "rpmsg host is online\n"); return 0; free_coherent: dma_free_coherent(vdev->dev.parent->parent, total_buf_space, bufs_va, vrp->bufs_dma); vqs_del: vdev->config->del_vqs(vrp->vdev); free_vrp: kfree(vrp); return err; }
static int rpmsg_probe(struct virtio_device *vdev) { vq_callback_t *vq_cbs[] = { rpmsg_recv_done, rpmsg_xmit_done }; static const char * const names[] = { "input", "output" }; struct virtqueue *vqs[2]; struct virtproc_info *vrp; void *bufs_va; int err = 0, i; size_t total_buf_space; bool notify; vrp = kzalloc(sizeof(*vrp), GFP_KERNEL); if (!vrp) return -ENOMEM; vrp->vdev = vdev; idr_init(&vrp->endpoints); mutex_init(&vrp->endpoints_lock); mutex_init(&vrp->tx_lock); init_waitqueue_head(&vrp->sendq); /* We expect two virtqueues, rx and tx (and in this order) */ err = vdev->config->find_vqs(vdev, 2, vqs, vq_cbs, names); if (err) goto free_vrp; vrp->rvq = vqs[0]; vrp->svq = vqs[1]; /* we expect symmetric tx/rx vrings */ WARN_ON(virtqueue_get_vring_size(vrp->rvq) != virtqueue_get_vring_size(vrp->svq)); /* we need less buffers if vrings are small */ if (virtqueue_get_vring_size(vrp->rvq) < MAX_RPMSG_NUM_BUFS / 2) vrp->num_bufs = virtqueue_get_vring_size(vrp->rvq) * 2; else vrp->num_bufs = MAX_RPMSG_NUM_BUFS; total_buf_space = vrp->num_bufs * RPMSG_BUF_SIZE; /* allocate coherent memory for the buffers */ bufs_va = dma_alloc_coherent(vdev->dev.parent->parent, total_buf_space, &vrp->bufs_dma, GFP_KERNEL); if (!bufs_va) { err = -ENOMEM; goto vqs_del; } dev_dbg(&vdev->dev, "buffers: va %p, dma 0x%llx\n", bufs_va, (unsigned long long)vrp->bufs_dma); /* half of the buffers is dedicated for RX */ vrp->rbufs = bufs_va; /* and half is dedicated for TX */ vrp->sbufs = bufs_va + total_buf_space / 2; /* set up the receive buffers */ for (i = 0; i < vrp->num_bufs / 2; i++) { struct scatterlist sg; void *cpu_addr = vrp->rbufs + i * RPMSG_BUF_SIZE; rpmsg_msg_sg_init(vrp, &sg, cpu_addr, RPMSG_BUF_SIZE); err = rpmsg_virtqueue_add_inbuf(vrp->rvq, &sg, 1, cpu_addr, GFP_KERNEL); WARN_ON(err); /* sanity check; this can't really happen */ } /* suppress "tx-complete" interrupts */ virtqueue_disable_cb(vrp->svq); vdev->priv = vrp; /* if supported by the remote processor, enable the name service */ if (virtio_has_feature(vdev, VIRTIO_RPMSG_F_NS)) { /* a dedicated endpoint handles the name service msgs */ vrp->ns_ept = __rpmsg_create_ept(vrp, NULL, rpmsg_ns_cb, vrp, RPMSG_NS_ADDR); if (!vrp->ns_ept) { dev_err(&vdev->dev, "failed to create the ns ept\n"); err = -ENOMEM; goto free_coherent; } } /* * Prepare to kick but don't notify yet - we can't do this before * device is ready. */ notify = virtqueue_kick_prepare(vrp->rvq); /* From this point on, we can notify and get callbacks. */ virtio_device_ready(vdev); /* tell the remote processor it can start sending messages */ /* * this might be concurrent with callbacks, but we are only * doing notify, not a full kick here, so that's ok. */ if (notify) virtqueue_notify(vrp->rvq); dev_info(&vdev->dev, "rpmsg host is online\n"); return 0; free_coherent: dma_free_coherent(vdev->dev.parent->parent, total_buf_space, bufs_va, vrp->bufs_dma); vqs_del: vdev->config->del_vqs(vrp->vdev); free_vrp: kfree(vrp); return err; }