static void virtballoon_remove(struct virtio_device *vdev)
{
struct virtio_balloon *vb = vdev->priv;
if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_DEFLATE_ON_OOM))
virtio_balloon_unregister_shrinker(vb);
spin_lock_irq(&vb->stop_update_lock);
vb->stop_update = true;
spin_unlock_irq(&vb->stop_update_lock);
cancel_work_sync(&vb->update_balloon_size_work);
cancel_work_sync(&vb->update_balloon_stats_work);
if (virtio_has_feature(vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) {
cancel_work_sync(&vb->report_free_page_work);
destroy_workqueue(vb->balloon_wq);
}
remove_common(vb);
#ifdef CONFIG_BALLOON_COMPACTION
if (vb->vb_dev_info.inode)
iput(vb->vb_dev_info.inode);
kern_unmount(balloon_mnt);
#endif
kfree(vb);
}
static int init_vqs(struct virtio_balloon *vb)
{
struct virtqueue *vqs[3];
vq_callback_t *callbacks[] = { balloon_ack, balloon_ack, stats_request };
const char *names[] = { "inflate", "deflate", "stats" };
int err, nvqs;
/*
* We expect two virtqueues: inflate and deflate, and
* optionally stat.
*/
nvqs = virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_STATS_VQ) ? 3 : 2;
err = vb->vdev->config->find_vqs(vb->vdev, nvqs, vqs, callbacks, names);
if (err)
return err;
vb->inflate_vq = vqs[0];
vb->deflate_vq = vqs[1];
if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_STATS_VQ)) {
struct scatterlist sg;
vb->stats_vq = vqs[2];
/*
* Prime this virtqueue with one buffer so the hypervisor can
* use it to signal us later.
*/
sg_init_one(&sg, vb->stats, sizeof vb->stats);
if (virtqueue_add_outbuf(vb->stats_vq, &sg, 1, vb, GFP_KERNEL)
< 0)
BUG();
virtqueue_kick(vb->stats_vq);
}
return 0;
}
static int rpmsg_dev_remove(struct device *dev)
{
struct rpmsg_channel *rpdev = to_rpmsg_channel(dev);
struct rpmsg_driver *rpdrv = to_rpmsg_driver(rpdev->dev.driver);
struct virtproc_info *vrp = rpdev->vrp;
int err = 0;
/* tell remote processor's name service we're removing this channel */
if (rpdev->announce &&
virtio_has_feature(vrp->vdev, VIRTIO_RPMSG_F_NS)) {
struct rpmsg_ns_msg nsm;
strncpy(nsm.name, rpdev->id.name, RPMSG_NAME_SIZE);
nsm.addr = rpdev->src;
nsm.flags = RPMSG_NS_DESTROY;
err = rpmsg_sendto(rpdev, &nsm, sizeof(nsm), RPMSG_NS_ADDR);
if (err)
dev_err(dev, "failed to announce service %d\n", err);
}
rpdrv->remove(rpdev);
rpmsg_destroy_ept(rpdev->ept);
return err;
}
static void fill_balloon(struct virtio_balloon *vb, size_t num)
{
struct balloon_dev_info *vb_dev_info = &vb->vb_dev_info;
/* We can only do one array worth at a time. */
num = min(num, ARRAY_SIZE(vb->pfns));
mutex_lock(&vb->balloon_lock);
for (vb->num_pfns = 0; vb->num_pfns < num;
vb->num_pfns += VIRTIO_BALLOON_PAGES_PER_PAGE) {
struct page *page = balloon_page_enqueue(vb_dev_info);
if (!page) {
dev_info_ratelimited(&vb->vdev->dev,
"Out of puff! Can't get %u pages\n",
VIRTIO_BALLOON_PAGES_PER_PAGE);
/* Sleep for at least 1/5 of a second before retry. */
msleep(200);
break;
}
set_page_pfns(vb->pfns + vb->num_pfns, page);
vb->num_pages += VIRTIO_BALLOON_PAGES_PER_PAGE;
if (!virtio_has_feature(vb->vdev,
VIRTIO_BALLOON_F_DEFLATE_ON_OOM))
adjust_managed_page_count(page, -1);
}
/* Did we get any? */
if (vb->num_pfns != 0)
tell_host(vb, vb->inflate_vq);
mutex_unlock(&vb->balloon_lock);
}
static void update_balloon_size(struct virtio_balloon *vb)
{
u32 actual = vb->num_pages;
/* Legacy balloon config space is LE, unlike all other devices. */
if (!virtio_has_feature(vb->vdev, VIRTIO_F_VERSION_1))
actual = (__force u32)cpu_to_le32(actual);
virtio_cwrite(vb->vdev, struct virtio_balloon_config, actual,
&actual);
}
static int virtinput_probe(struct virtio_device *vdev)
{
struct virtio_input *vi;
unsigned long flags;
size_t size;
int abs, err;
if (!virtio_has_feature(vdev, VIRTIO_F_VERSION_1))
return -ENODEV;
vi = kzalloc(sizeof(*vi), GFP_KERNEL);
if (!vi)
return -ENOMEM;
vdev->priv = vi;
vi->vdev = vdev;
spin_lock_init(&vi->lock);
err = virtinput_init_vqs(vi);
if (err)
goto err_init_vq;
vi->idev = input_allocate_device();
if (!vi->idev) {
err = -ENOMEM;
goto err_input_alloc;
}
input_set_drvdata(vi->idev, vi);
size = virtinput_cfg_select(vi, VIRTIO_INPUT_CFG_ID_NAME, 0);
virtio_cread_bytes(vi->vdev, offsetof(struct virtio_input_config,
u.string),
vi->name, min(size, sizeof(vi->name)));
size = virtinput_cfg_select(vi, VIRTIO_INPUT_CFG_ID_SERIAL, 0);
virtio_cread_bytes(vi->vdev, offsetof(struct virtio_input_config,
u.string),
vi->serial, min(size, sizeof(vi->serial)));
snprintf(vi->phys, sizeof(vi->phys),
"virtio%d/input0", vdev->index);
vi->idev->name = vi->name;
vi->idev->phys = vi->phys;
vi->idev->uniq = vi->serial;
size = virtinput_cfg_select(vi, VIRTIO_INPUT_CFG_ID_DEVIDS, 0);
if (size >= sizeof(struct virtio_input_devids)) {
virtio_cread(vi->vdev, struct virtio_input_config,
u.ids.bustype, &vi->idev->id.bustype);
virtio_cread(vi->vdev, struct virtio_input_config,
u.ids.vendor, &vi->idev->id.vendor);
virtio_cread(vi->vdev, struct virtio_input_config,
u.ids.product, &vi->idev->id.product);
virtio_cread(vi->vdev, struct virtio_input_config,
u.ids.version, &vi->idev->id.version);
} else {
/* We provide getgeo only to please some old bootloader/partitioning tools */
static int virtblk_getgeo(struct block_device *bd, struct hd_geometry *geo)
{
struct virtio_blk *vblk = bd->bd_disk->private_data;
/* see if the host passed in geometry config */
if (virtio_has_feature(vblk->vdev, VIRTIO_BLK_F_GEOMETRY)) {
virtio_cread(vblk->vdev, struct virtio_blk_config,
geometry.cylinders, &geo->cylinders);
virtio_cread(vblk->vdev, struct virtio_blk_config,
geometry.heads, &geo->heads);
virtio_cread(vblk->vdev, struct virtio_blk_config,
geometry.sectors, &geo->sectors);
} else {
static void release_pages_balloon(struct virtio_balloon *vb)
{
unsigned int i;
/* Find pfns pointing at start of each page, get pages and free them. */
for (i = 0; i < vb->num_pfns; i += VIRTIO_BALLOON_PAGES_PER_PAGE) {
struct page *page = balloon_pfn_to_page(vb->pfns[i]);
if (!virtio_has_feature(vb->vdev,
VIRTIO_BALLOON_F_DEFLATE_ON_OOM))
adjust_managed_page_count(page, 1);
put_page(page); /* balloon reference */
}
}
static void release_pages_balloon(struct virtio_balloon *vb,
struct list_head *pages)
{
struct page *page, *next;
list_for_each_entry_safe(page, next, pages, lru) {
if (!virtio_has_feature(vb->vdev,
VIRTIO_BALLOON_F_DEFLATE_ON_OOM))
adjust_managed_page_count(page, 1);
list_del(&page->lru);
put_page(page); /* balloon reference */
}
}
struct virtqueue *vring_new_virtqueue(unsigned int num,
unsigned int vring_align,
struct virtio_device *vdev,
void *pages,
void (*notify)(struct virtqueue *),
void (*callback)(struct virtqueue *),
const char *name)
{
struct vring_virtqueue *vq;
unsigned int i;
/* We assume num is a power of 2. */
if (num & (num - 1)) {
dev_warn(&vdev->dev, "Bad virtqueue length %u\n", num);
return NULL;
}
vq = kmalloc(sizeof(*vq) + sizeof(void *)*num, GFP_KERNEL);
if (!vq)
return NULL;
vring_init(&vq->vring, num, pages, vring_align);
vq->vq.callback = callback;
vq->vq.vdev = vdev;
vq->vq.name = name;
vq->notify = notify;
vq->broken = false;
vq->last_used_idx = 0;
vq->num_added = 0;
list_add_tail(&vq->vq.list, &vdev->vqs);
#ifdef DEBUG
vq->in_use = false;
#endif
vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC);
/* No callback? Tell other side not to bother us. */
if (!callback)
vq->vring.avail->flags |= VRING_AVAIL_F_NO_INTERRUPT;
/* Put everything in free lists. */
vq->num_free = num;
vq->free_head = 0;
for (i = 0; i < num-1; i++) {
vq->vring.desc[i].next = i+1;
vq->data[i] = NULL;
}
vq->data[i] = NULL;
return &vq->vq;
}
static unsigned fill_balloon(struct virtio_balloon *vb, size_t num)
{
unsigned num_allocated_pages;
unsigned num_pfns;
struct page *page;
LIST_HEAD(pages);
/* We can only do one array worth at a time. */
num = min(num, ARRAY_SIZE(vb->pfns));
for (num_pfns = 0; num_pfns < num;
num_pfns += VIRTIO_BALLOON_PAGES_PER_PAGE) {
struct page *page = balloon_page_alloc();
if (!page) {
dev_info_ratelimited(&vb->vdev->dev,
"Out of puff! Can't get %u pages\n",
VIRTIO_BALLOON_PAGES_PER_PAGE);
/* Sleep for at least 1/5 of a second before retry. */
msleep(200);
break;
}
balloon_page_push(&pages, page);
}
mutex_lock(&vb->balloon_lock);
vb->num_pfns = 0;
while ((page = balloon_page_pop(&pages))) {
balloon_page_enqueue(&vb->vb_dev_info, page);
set_page_pfns(vb, vb->pfns + vb->num_pfns, page);
vb->num_pages += VIRTIO_BALLOON_PAGES_PER_PAGE;
if (!virtio_has_feature(vb->vdev,
VIRTIO_BALLOON_F_DEFLATE_ON_OOM))
adjust_managed_page_count(page, -1);
vb->num_pfns += VIRTIO_BALLOON_PAGES_PER_PAGE;
}
num_allocated_pages = vb->num_pfns;
/* Did we get any? */
if (vb->num_pfns != 0)
tell_host(vb, vb->inflate_vq);
mutex_unlock(&vb->balloon_lock);
return num_allocated_pages;
}
void virtio_queue_set_notification(VirtQueue *vq, int enable)
{
vq->notification = enable;
if (virtio_has_feature(vq->vdev, VIRTIO_RING_F_EVENT_IDX)) {
vring_set_avail_event(vq, vring_avail_idx(vq));
} else if (enable) {
vring_used_flags_unset_bit(vq, VRING_USED_F_NO_NOTIFY);
} else {
vring_used_flags_set_bit(vq, VRING_USED_F_NO_NOTIFY);
}
if (enable) {
/* Expose avail event/used flags before caller checks the avail idx. */
smp_mb();
}
}
static int virtblk_ioctl(struct block_device *bdev, fmode_t mode,
unsigned int cmd, unsigned long data)
{
struct gendisk *disk = bdev->bd_disk;
struct virtio_blk *vblk = disk->private_data;
/*
* Only allow the generic SCSI ioctls if the host can support it.
*/
if (!virtio_has_feature(vblk->vdev, VIRTIO_BLK_F_SCSI))
return -ENOTTY;
return scsi_cmd_blk_ioctl(bdev, mode, cmd,
(void __user *)data);
}
static inline s64 towards_target(struct virtio_balloon *vb)
{
s64 target;
u32 num_pages;
virtio_cread(vb->vdev, struct virtio_balloon_config, num_pages,
&num_pages);
/* Legacy balloon config space is LE, unlike all other devices. */
if (!virtio_has_feature(vb->vdev, VIRTIO_F_VERSION_1))
num_pages = le32_to_cpu((__force __le32)num_pages);
target = num_pages;
return target - vb->num_pages;
}
static void virtio_balloon_device_realize(DeviceState *dev, Error **errp)
{
VirtIODevice *vdev = VIRTIO_DEVICE(dev);
VirtIOBalloon *s = VIRTIO_BALLOON(dev);
int ret;
virtio_init(vdev, "virtio-balloon", VIRTIO_ID_BALLOON,
sizeof(struct virtio_balloon_config));
ret = qemu_add_balloon_handler(virtio_balloon_to_target,
virtio_balloon_stat, s);
if (ret < 0) {
error_setg(errp, "Only one balloon device is supported");
virtio_cleanup(vdev);
return;
}
s->ivq = virtio_add_queue(vdev, 128, virtio_balloon_handle_output);
s->dvq = virtio_add_queue(vdev, 128, virtio_balloon_handle_output);
s->svq = virtio_add_queue(vdev, 128, virtio_balloon_receive_stats);
if (virtio_has_feature(s->host_features,
VIRTIO_BALLOON_F_FREE_PAGE_HINT)) {
s->free_page_vq = virtio_add_queue(vdev, VIRTQUEUE_MAX_SIZE,
virtio_balloon_handle_free_page_vq);
s->free_page_report_status = FREE_PAGE_REPORT_S_STOP;
s->free_page_report_cmd_id =
VIRTIO_BALLOON_FREE_PAGE_REPORT_CMD_ID_MIN;
s->free_page_report_notify.notify =
virtio_balloon_free_page_report_notify;
precopy_add_notifier(&s->free_page_report_notify);
if (s->iothread) {
object_ref(OBJECT(s->iothread));
s->free_page_bh = aio_bh_new(iothread_get_aio_context(s->iothread),
virtio_ballloon_get_free_page_hints, s);
qemu_mutex_init(&s->free_page_lock);
qemu_cond_init(&s->free_page_cond);
s->block_iothread = false;
} else {
/* Simply disable this feature if the iothread wasn't created. */
s->host_features &= ~(1 << VIRTIO_BALLOON_F_FREE_PAGE_HINT);
virtio_error(vdev, "iothread is missing");
}
}
reset_stats(s);
}
/*
* virtballoon_oom_notify - release pages when system is under severe
* memory pressure (called from out_of_memory())
* @self : notifier block struct
* @dummy: not used
* @parm : returned - number of freed pages
*
* The balancing of memory by use of the virtio balloon should not cause
* the termination of processes while there are pages in the balloon.
* If virtio balloon manages to release some memory, it will make the
* system return and retry the allocation that forced the OOM killer
* to run.
*/
static int virtballoon_oom_notify(struct notifier_block *self,
unsigned long dummy, void *parm)
{
struct virtio_balloon *vb;
unsigned long *freed;
unsigned num_freed_pages;
vb = container_of(self, struct virtio_balloon, nb);
if (!virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_DEFLATE_ON_OOM))
return NOTIFY_OK;
freed = parm;
num_freed_pages = leak_balloon(vb, oom_pages);
update_balloon_size(vb);
*freed += num_freed_pages;
return NOTIFY_OK;
}
static unsigned long virtio_balloon_shrinker_scan(struct shrinker *shrinker,
struct shrink_control *sc)
{
unsigned long pages_to_free, pages_freed = 0;
struct virtio_balloon *vb = container_of(shrinker,
struct virtio_balloon, shrinker);
pages_to_free = sc->nr_to_scan * VIRTIO_BALLOON_PAGES_PER_PAGE;
if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT))
pages_freed = shrink_free_pages(vb, pages_to_free);
if (pages_freed >= pages_to_free)
return pages_freed;
pages_freed += shrink_balloon_pages(vb, pages_to_free - pages_freed);
return pages_freed;
}
static int rpmsg_dev_probe(struct device *dev)
{
struct rpmsg_channel *rpdev = to_rpmsg_channel(dev);
struct rpmsg_driver *rpdrv = to_rpmsg_driver(rpdev->dev.driver);
struct virtproc_info *vrp = rpdev->vrp;
struct rpmsg_endpoint *ept;
int err;
ept = rpmsg_create_ept(rpdev, rpdrv->callback, NULL, rpdev->src);
if (!ept) {
dev_err(dev, "failed to create endpoint\n");
err = -ENOMEM;
goto out;
}
rpdev->ept = ept;
rpdev->src = ept->addr;
err = rpdrv->probe(rpdev);
if (err) {
dev_err(dev, "%s: failed: %d\n", __func__, err);
rpmsg_destroy_ept(ept);
goto out;
}
/* need to tell remote processor's name service about this channel ? */
if (rpdev->announce &&
virtio_has_feature(vrp->vdev, VIRTIO_RPMSG_F_NS)) {
struct rpmsg_ns_msg nsm;
strncpy(nsm.name, rpdev->id.name, RPMSG_NAME_SIZE);
nsm.addr = rpdev->src;
nsm.flags = RPMSG_NS_CREATE;
err = rpmsg_sendto(rpdev, &nsm, sizeof(nsm), RPMSG_NS_ADDR);
if (err)
dev_err(dev, "failed to announce service %d\n", err);
}
out:
return err;
}
static int virtio_finalize_features(struct virtio_device *dev)
{
int ret = dev->config->finalize_features(dev);
unsigned status;
if (ret)
return ret;
if (!virtio_has_feature(dev, VIRTIO_F_VERSION_1))
return 0;
add_status(dev, VIRTIO_CONFIG_S_FEATURES_OK);
status = dev->config->get_status(dev);
if (!(status & VIRTIO_CONFIG_S_FEATURES_OK)) {
dev_err(&dev->dev, "virtio: device refuses features: %x\n",
status);
return -ENODEV;
}
return 0;
}
static int vhost_net_set_vnet_endian(VirtIODevice *dev, NetClientState *peer,
bool set)
{
int r = 0;
if (virtio_has_feature(dev, VIRTIO_F_VERSION_1) ||
(virtio_legacy_is_cross_endian(dev) && !virtio_is_big_endian(dev))) {
r = qemu_set_vnet_le(peer, set);
if (r) {
error_report("backend does not support LE vnet headers");
}
} else if (virtio_legacy_is_cross_endian(dev)) {
r = qemu_set_vnet_be(peer, set);
if (r) {
error_report("backend does not support BE vnet headers");
}
}
return r;
}
/*
* Allocate/free a vq.
*/
struct virtqueue *
virtio_alloc_vq(struct virtio_softc *sc, unsigned int index, unsigned int size,
unsigned int indirect_num, const char *name)
{
int vq_size, allocsize1, allocsize2, allocsize = 0;
int ret;
unsigned int ncookies;
size_t len;
struct virtqueue *vq;
ddi_put16(sc->sc_ioh,
/* LINTED E_BAD_PTR_CAST_ALIGN */
(uint16_t *)(sc->sc_io_addr + VIRTIO_CONFIG_QUEUE_SELECT), index);
vq_size = ddi_get16(sc->sc_ioh,
/* LINTED E_BAD_PTR_CAST_ALIGN */
(uint16_t *)(sc->sc_io_addr + VIRTIO_CONFIG_QUEUE_SIZE));
if (vq_size == 0) {
dev_err(sc->sc_dev, CE_WARN,
"virtqueue dest not exist, index %d for %s\n", index, name);
goto out;
}
vq = kmem_zalloc(sizeof (struct virtqueue), KM_SLEEP);
/* size 0 => use native vq size, good for receive queues. */
if (size)
vq_size = MIN(vq_size, size);
/* allocsize1: descriptor table + avail ring + pad */
allocsize1 = VIRTQUEUE_ALIGN(sizeof (struct vring_desc) * vq_size +
sizeof (struct vring_avail) + sizeof (uint16_t) * vq_size);
/* allocsize2: used ring + pad */
allocsize2 = VIRTQUEUE_ALIGN(sizeof (struct vring_used) +
sizeof (struct vring_used_elem) * vq_size);
allocsize = allocsize1 + allocsize2;
ret = ddi_dma_alloc_handle(sc->sc_dev, &virtio_vq_dma_attr,
DDI_DMA_SLEEP, NULL, &vq->vq_dma_handle);
if (ret != DDI_SUCCESS) {
dev_err(sc->sc_dev, CE_WARN,
"Failed to allocate dma handle for vq %d", index);
goto out_alloc_handle;
}
ret = ddi_dma_mem_alloc(vq->vq_dma_handle, allocsize,
&virtio_vq_devattr, DDI_DMA_CONSISTENT, DDI_DMA_SLEEP, NULL,
(caddr_t *)&vq->vq_vaddr, &len, &vq->vq_dma_acch);
if (ret != DDI_SUCCESS) {
dev_err(sc->sc_dev, CE_WARN,
"Failed to allocate dma memory for vq %d", index);
goto out_alloc;
}
ret = ddi_dma_addr_bind_handle(vq->vq_dma_handle, NULL,
(caddr_t)vq->vq_vaddr, len, DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
DDI_DMA_SLEEP, NULL, &vq->vq_dma_cookie, &ncookies);
if (ret != DDI_DMA_MAPPED) {
dev_err(sc->sc_dev, CE_WARN,
"Failed to bind dma memory for vq %d", index);
goto out_bind;
}
/* We asked for a single segment */
ASSERT(ncookies == 1);
/* and page-ligned buffers. */
ASSERT(vq->vq_dma_cookie.dmac_laddress % VIRTIO_PAGE_SIZE == 0);
(void) memset(vq->vq_vaddr, 0, allocsize);
/* Make sure all zeros hit the buffer before we point the host to it */
membar_producer();
/* set the vq address */
ddi_put32(sc->sc_ioh,
/* LINTED E_BAD_PTR_CAST_ALIGN */
(uint32_t *)(sc->sc_io_addr + VIRTIO_CONFIG_QUEUE_ADDRESS),
(vq->vq_dma_cookie.dmac_laddress / VIRTIO_PAGE_SIZE));
/* remember addresses and offsets for later use */
vq->vq_owner = sc;
vq->vq_num = vq_size;
vq->vq_index = index;
vq->vq_descs = vq->vq_vaddr;
vq->vq_availoffset = sizeof (struct vring_desc)*vq_size;
vq->vq_avail = (void *)(((char *)vq->vq_descs) + vq->vq_availoffset);
vq->vq_usedoffset = allocsize1;
vq->vq_used = (void *)(((char *)vq->vq_descs) + vq->vq_usedoffset);
ASSERT(indirect_num == 0 ||
virtio_has_feature(sc, VIRTIO_F_RING_INDIRECT_DESC));
vq->vq_indirect_num = indirect_num;
/* free slot management */
vq->vq_entries = kmem_zalloc(sizeof (struct vq_entry) * vq_size,
KM_SLEEP);
ret = virtio_init_vq(sc, vq);
if (ret)
goto out_init;
dev_debug(sc->sc_dev, CE_NOTE,
"Allocated %d entries for vq %d:%s (%d indirect descs)",
vq_size, index, name, indirect_num * vq_size);
return (vq);
out_init:
kmem_free(vq->vq_entries, sizeof (struct vq_entry) * vq_size);
(void) ddi_dma_unbind_handle(vq->vq_dma_handle);
out_bind:
ddi_dma_mem_free(&vq->vq_dma_acch);
out_alloc:
ddi_dma_free_handle(&vq->vq_dma_handle);
out_alloc_handle:
kmem_free(vq, sizeof (struct virtqueue));
out:
return (NULL);
}
static int rpmsg_probe(struct virtio_device *vdev)
{
vq_callback_t *vq_cbs[] = { rpmsg_recv_done, rpmsg_xmit_done };
const char *names[] = { "input", "output" };
struct virtqueue *vqs[2];
struct virtproc_info *vrp;
void *addr;
int err, i, num_bufs, buf_size, total_buf_size;
struct rpmsg_channel_info *ch;
vrp = kzalloc(sizeof(*vrp), GFP_KERNEL);
if (!vrp)
return -ENOMEM;
vrp->vdev = vdev;
idr_init(&vrp->endpoints);
spin_lock_init(&vrp->endpoints_lock);
mutex_init(&vrp->svq_lock);
mutex_init(&vrp->rm_lock);
init_waitqueue_head(&vrp->sendq);
/* We expect two virtqueues, rx and tx (in this order) */
err = vdev->config->find_vqs(vdev, 2, vqs, vq_cbs, names);
if (err)
goto free_vi;
vrp->rvq = vqs[0];
vrp->svq = vqs[1];
/* Platform must supply pre-allocated uncached buffers for now */
vdev->config->get(vdev, VPROC_BUF_ADDR, &addr, sizeof(addr));
vdev->config->get(vdev, VPROC_BUF_NUM, &num_bufs,
sizeof(num_bufs));
vdev->config->get(vdev, VPROC_BUF_SZ, &buf_size, sizeof(buf_size));
total_buf_size = num_bufs * buf_size;
dev_dbg(&vdev->dev, "%d buffers, size %d, addr 0x%x, total 0x%x\n",
num_bufs, buf_size, (unsigned int) addr, total_buf_size);
vrp->num_bufs = num_bufs;
vrp->buf_size = buf_size;
vrp->rbufs = addr;
vrp->sbufs = addr + total_buf_size / 2;
/* simulated addr base to make virt_to_page happy */
vdev->config->get(vdev, VPROC_SIM_BASE, &vrp->sim_base,
sizeof(vrp->sim_base));
/* set up the receive buffers */
for (i = 0; i < num_bufs / 2; i++) {
struct scatterlist sg;
void *tmpaddr = vrp->rbufs + i * buf_size;
void *simaddr = vrp->sim_base + i * buf_size;
sg_init_one(&sg, simaddr, buf_size);
err = virtqueue_add_buf_gfp(vrp->rvq, &sg, 0, 1, tmpaddr,
GFP_KERNEL);
WARN_ON(err < 0); /* sanity check; this can't really happen */
}
/* tell the remote processor it can start sending data */
virtqueue_kick(vrp->rvq);
/* suppress "tx-complete" interrupts */
virtqueue_disable_cb(vrp->svq);
vdev->priv = vrp;
dev_info(&vdev->dev, "rpmsg backend virtproc probed successfully\n");
/* if supported by the remote processor, enable the name service */
if (virtio_has_feature(vdev, VIRTIO_RPMSG_F_NS)) {
vrp->ns_ept = __rpmsg_create_ept(vrp, NULL, rpmsg_ns_cb,
vrp, RPMSG_NS_ADDR);
if (!vrp->ns_ept) {
dev_err(&vdev->dev, "failed to create the ns ept\n");
err = -ENOMEM;
goto vqs_del;
}
}
/* look for platform-specific static channels */
vdev->config->get(vdev, VPROC_STATIC_CHANNELS, &ch, sizeof(ch));
for (i = 0; ch && ch[i].name[0]; i++)
rpmsg_create_channel(vrp, &ch[i]);
return 0;
vqs_del:
vdev->config->del_vqs(vrp->vdev);
free_vi:
kfree(vrp);
return err;
}
static int rpmsg_probe(struct virtio_device *vdev)
{
vq_callback_t *vq_cbs[] = { rpmsg_recv_done, rpmsg_xmit_done };
const char *names[] = { "input", "output" };
struct virtqueue *vqs[2];
struct virtproc_info *vrp;
void *bufs_va;
int err = 0, i;
vrp = kzalloc(sizeof(*vrp), GFP_KERNEL);
if (!vrp)
return -ENOMEM;
vrp->vdev = vdev;
idr_init(&vrp->endpoints);
mutex_init(&vrp->endpoints_lock);
mutex_init(&vrp->tx_lock);
init_waitqueue_head(&vrp->sendq);
err = vdev->config->find_vqs(vdev, 2, vqs, vq_cbs, names);
if (err)
goto free_vrp;
vrp->rvq = vqs[0];
vrp->svq = vqs[1];
bufs_va = dma_alloc_coherent(vdev->dev.parent, RPMSG_TOTAL_BUF_SPACE,
&vrp->bufs_dma, GFP_KERNEL);
if (!bufs_va)
goto vqs_del;
dev_dbg(&vdev->dev, "buffers: va %p, dma 0x%llx\n", bufs_va,
(unsigned long long)vrp->bufs_dma);
vrp->rbufs = bufs_va;
vrp->sbufs = bufs_va + RPMSG_TOTAL_BUF_SPACE / 2;
for (i = 0; i < RPMSG_NUM_BUFS / 2; i++) {
struct scatterlist sg;
void *cpu_addr = vrp->rbufs + i * RPMSG_BUF_SIZE;
sg_init_one(&sg, cpu_addr, RPMSG_BUF_SIZE);
err = virtqueue_add_buf(vrp->rvq, &sg, 0, 1, cpu_addr,
GFP_KERNEL);
WARN_ON(err < 0);
}
virtqueue_disable_cb(vrp->svq);
vdev->priv = vrp;
if (virtio_has_feature(vdev, VIRTIO_RPMSG_F_NS)) {
vrp->ns_ept = __rpmsg_create_ept(vrp, NULL, rpmsg_ns_cb,
vrp, RPMSG_NS_ADDR);
if (!vrp->ns_ept) {
dev_err(&vdev->dev, "failed to create the ns ept\n");
err = -ENOMEM;
goto free_coherent;
}
}
virtqueue_kick(vrp->rvq);
dev_info(&vdev->dev, "rpmsg host is online\n");
return 0;
free_coherent:
dma_free_coherent(vdev->dev.parent, RPMSG_TOTAL_BUF_SPACE, bufs_va,
vrp->bufs_dma);
vqs_del:
vdev->config->del_vqs(vrp->vdev);
free_vrp:
kfree(vrp);
return err;
}
static int virtio_blk_handle_scsi_req(VirtIOBlockReq *req)
{
int status = VIRTIO_BLK_S_OK;
struct virtio_scsi_inhdr *scsi = NULL;
VirtIODevice *vdev = VIRTIO_DEVICE(req->dev);
VirtQueueElement *elem = &req->elem;
VirtIOBlock *blk = req->dev;
#ifdef __linux__
int i;
VirtIOBlockIoctlReq *ioctl_req;
BlockAIOCB *acb;
#endif
/*
* We require at least one output segment each for the virtio_blk_outhdr
* and the SCSI command block.
*
* We also at least require the virtio_blk_inhdr, the virtio_scsi_inhdr
* and the sense buffer pointer in the input segments.
*/
if (elem->out_num < 2 || elem->in_num < 3) {
status = VIRTIO_BLK_S_IOERR;
goto fail;
}
/*
* The scsi inhdr is placed in the second-to-last input segment, just
* before the regular inhdr.
*/
scsi = (void *)elem->in_sg[elem->in_num - 2].iov_base;
if (!virtio_has_feature(blk->host_features, VIRTIO_BLK_F_SCSI)) {
status = VIRTIO_BLK_S_UNSUPP;
goto fail;
}
/*
* No support for bidirection commands yet.
*/
if (elem->out_num > 2 && elem->in_num > 3) {
status = VIRTIO_BLK_S_UNSUPP;
goto fail;
}
#ifdef __linux__
ioctl_req = g_new0(VirtIOBlockIoctlReq, 1);
ioctl_req->req = req;
ioctl_req->hdr.interface_id = 'S';
ioctl_req->hdr.cmd_len = elem->out_sg[1].iov_len;
ioctl_req->hdr.cmdp = elem->out_sg[1].iov_base;
ioctl_req->hdr.dxfer_len = 0;
if (elem->out_num > 2) {
/*
* If there are more than the minimally required 2 output segments
* there is write payload starting from the third iovec.
*/
ioctl_req->hdr.dxfer_direction = SG_DXFER_TO_DEV;
ioctl_req->hdr.iovec_count = elem->out_num - 2;
for (i = 0; i < ioctl_req->hdr.iovec_count; i++) {
ioctl_req->hdr.dxfer_len += elem->out_sg[i + 2].iov_len;
}
ioctl_req->hdr.dxferp = elem->out_sg + 2;
} else if (elem->in_num > 3) {
/*
* If we have more than 3 input segments the guest wants to actually
* read data.
*/
ioctl_req->hdr.dxfer_direction = SG_DXFER_FROM_DEV;
ioctl_req->hdr.iovec_count = elem->in_num - 3;
for (i = 0; i < ioctl_req->hdr.iovec_count; i++) {
ioctl_req->hdr.dxfer_len += elem->in_sg[i].iov_len;
}
ioctl_req->hdr.dxferp = elem->in_sg;
} else {
/*
* Some SCSI commands don't actually transfer any data.
*/
ioctl_req->hdr.dxfer_direction = SG_DXFER_NONE;
}
ioctl_req->hdr.sbp = elem->in_sg[elem->in_num - 3].iov_base;
ioctl_req->hdr.mx_sb_len = elem->in_sg[elem->in_num - 3].iov_len;
acb = blk_aio_ioctl(blk->blk, SG_IO, &ioctl_req->hdr,
virtio_blk_ioctl_complete, ioctl_req);
if (!acb) {
g_free(ioctl_req);
status = VIRTIO_BLK_S_UNSUPP;
goto fail;
}
return -EINPROGRESS;
#else
abort();
#endif
fail:
/* Just put anything nonzero so that the ioctl fails in the guest. */
if (scsi) {
virtio_stl_p(vdev, &scsi->errors, 255);
}
return status;
}
static int rpmsg_probe(struct virtio_device *vdev)
{
vq_callback_t *vq_cbs[] = { rpmsg_recv_done, rpmsg_xmit_done };
const char *names[] = { "input", "output" };
struct virtqueue *vqs[2];
struct virtproc_info *vrp;
void *bufs_va;
int err = 0, i, vproc_id;
vrp = kzalloc(sizeof(*vrp), GFP_KERNEL);
if (!vrp)
return -ENOMEM;
vrp->vdev = vdev;
idr_init(&vrp->endpoints);
mutex_init(&vrp->endpoints_lock);
mutex_init(&vrp->tx_lock);
init_waitqueue_head(&vrp->sendq);
if (!idr_pre_get(&vprocs, GFP_KERNEL))
goto free_vrp;
mutex_lock(&vprocs_mutex);
err = idr_get_new(&vprocs, vrp, &vproc_id);
mutex_unlock(&vprocs_mutex);
if (err) {
dev_err(&vdev->dev, "idr_get_new failed: %d\n", err);
goto free_vrp;
}
vrp->id = vproc_id;
/* We expect two virtqueues, rx and tx (and in this order) */
err = vdev->config->find_vqs(vdev, 2, vqs, vq_cbs, names);
if (err)
goto rem_idr;
vrp->rvq = vqs[0];
vrp->svq = vqs[1];
/* allocate coherent memory for the buffers */
bufs_va = dma_alloc_coherent(vdev->dev.parent->parent,
RPMSG_TOTAL_BUF_SPACE,
&vrp->bufs_dma, GFP_KERNEL);
if (!bufs_va)
goto vqs_del;
dev_dbg(&vdev->dev, "buffers: va %p, dma 0x%llx\n", bufs_va,
(unsigned long long)vrp->bufs_dma);
/* half of the buffers is dedicated for RX */
vrp->rbufs = bufs_va;
/* and half is dedicated for TX */
vrp->sbufs = bufs_va + RPMSG_TOTAL_BUF_SPACE / 2;
/* set up the receive buffers */
for (i = 0; i < RPMSG_NUM_BUFS / 2; i++) {
struct scatterlist sg;
void *cpu_addr = vrp->rbufs + i * RPMSG_BUF_SIZE;
sg_init_one(&sg, cpu_addr, RPMSG_BUF_SIZE);
err = virtqueue_add_buf(vrp->rvq, &sg, 0, 1, cpu_addr,
GFP_KERNEL);
WARN_ON(err); /* sanity check; this can't really happen */
}
/* suppress "tx-complete" interrupts */
virtqueue_disable_cb(vrp->svq);
vdev->priv = vrp;
/* if supported by the remote processor, enable the name service */
if (virtio_has_feature(vdev, VIRTIO_RPMSG_F_NS)) {
/* a dedicated endpoint handles the name service msgs */
vrp->ns_ept = __rpmsg_create_ept(vrp, NULL, rpmsg_ns_cb,
vrp, RPMSG_NS_ADDR);
if (!vrp->ns_ept) {
dev_err(&vdev->dev, "failed to create the ns ept\n");
err = -ENOMEM;
goto free_coherent;
}
}
/* tell the remote processor it can start sending messages */
virtqueue_kick(vrp->rvq);
dev_info(&vdev->dev, "rpmsg host is online\n");
return 0;
free_coherent:
dma_free_coherent(vdev->dev.parent->parent, RPMSG_TOTAL_BUF_SPACE,
bufs_va, vrp->bufs_dma);
vqs_del:
vdev->config->del_vqs(vrp->vdev);
rem_idr:
mutex_lock(&vprocs_mutex);
idr_remove(&vprocs, vproc_id);
mutex_unlock(&vprocs_mutex);
free_vrp:
kfree(vrp);
return err;
}
static int vhost_user_set_log_base(struct vhost_dev *dev, uint64_t base,
struct vhost_log *log)
{
int fds[VHOST_MEMORY_MAX_NREGIONS];
size_t fd_num = 0;
bool shmfd = virtio_has_feature(dev->protocol_features,
VHOST_USER_PROTOCOL_F_LOG_SHMFD);
VhostUserMsg msg = {
.request = VHOST_USER_SET_LOG_BASE,
.flags = VHOST_USER_VERSION,
.payload.u64 = base,
.size = sizeof(msg.payload.u64),
};
if (shmfd && log->fd != -1) {
fds[fd_num++] = log->fd;
}
vhost_user_write(dev, &msg, fds, fd_num);
if (shmfd) {
msg.size = 0;
if (vhost_user_read(dev, &msg) < 0) {
return 0;
}
if (msg.request != VHOST_USER_SET_LOG_BASE) {
error_report("Received unexpected msg type. "
"Expected %d received %d",
VHOST_USER_SET_LOG_BASE, msg.request);
return -1;
}
}
return 0;
}
static int vhost_user_set_mem_table(struct vhost_dev *dev,
struct vhost_memory *mem)
{
int fds[VHOST_MEMORY_MAX_NREGIONS];
int i, fd;
size_t fd_num = 0;
VhostUserMsg msg = {
.request = VHOST_USER_SET_MEM_TABLE,
.flags = VHOST_USER_VERSION,
};
for (i = 0; i < dev->mem->nregions; ++i) {
struct vhost_memory_region *reg = dev->mem->regions + i;
ram_addr_t ram_addr;
assert((uintptr_t)reg->userspace_addr == reg->userspace_addr);
qemu_ram_addr_from_host((void *)(uintptr_t)reg->userspace_addr,
&ram_addr);
fd = qemu_get_ram_fd(ram_addr);
if (fd > 0) {
msg.payload.memory.regions[fd_num].userspace_addr = reg->userspace_addr;
msg.payload.memory.regions[fd_num].memory_size = reg->memory_size;
msg.payload.memory.regions[fd_num].guest_phys_addr = reg->guest_phys_addr;
msg.payload.memory.regions[fd_num].mmap_offset = reg->userspace_addr -
(uintptr_t) qemu_get_ram_block_host_ptr(ram_addr);
assert(fd_num < VHOST_MEMORY_MAX_NREGIONS);
fds[fd_num++] = fd;
}
}
msg.payload.memory.nregions = fd_num;
if (!fd_num) {
error_report("Failed initializing vhost-user memory map, "
"consider using -object memory-backend-file share=on");
return -1;
}
msg.size = sizeof(msg.payload.memory.nregions);
msg.size += sizeof(msg.payload.memory.padding);
msg.size += fd_num * sizeof(VhostUserMemoryRegion);
vhost_user_write(dev, &msg, fds, fd_num);
return 0;
}
static int virtballoon_probe(struct virtio_device *vdev)
{
struct virtio_balloon *vb;
__u32 poison_val;
int err;
if (!vdev->config->get) {
dev_err(&vdev->dev, "%s failure: config access disabled\n",
__func__);
return -EINVAL;
}
vdev->priv = vb = kzalloc(sizeof(*vb), GFP_KERNEL);
if (!vb) {
err = -ENOMEM;
goto out;
}
INIT_WORK(&vb->update_balloon_stats_work, update_balloon_stats_func);
INIT_WORK(&vb->update_balloon_size_work, update_balloon_size_func);
spin_lock_init(&vb->stop_update_lock);
mutex_init(&vb->balloon_lock);
init_waitqueue_head(&vb->acked);
vb->vdev = vdev;
balloon_devinfo_init(&vb->vb_dev_info);
err = init_vqs(vb);
if (err)
goto out_free_vb;
#ifdef CONFIG_BALLOON_COMPACTION
balloon_mnt = kern_mount(&balloon_fs);
if (IS_ERR(balloon_mnt)) {
err = PTR_ERR(balloon_mnt);
goto out_del_vqs;
}
vb->vb_dev_info.migratepage = virtballoon_migratepage;
vb->vb_dev_info.inode = alloc_anon_inode(balloon_mnt->mnt_sb);
if (IS_ERR(vb->vb_dev_info.inode)) {
err = PTR_ERR(vb->vb_dev_info.inode);
kern_unmount(balloon_mnt);
goto out_del_vqs;
}
vb->vb_dev_info.inode->i_mapping->a_ops = &balloon_aops;
#endif
if (virtio_has_feature(vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) {
/*
* There is always one entry reserved for cmd id, so the ring
* size needs to be at least two to report free page hints.
*/
if (virtqueue_get_vring_size(vb->free_page_vq) < 2) {
err = -ENOSPC;
goto out_del_vqs;
}
vb->balloon_wq = alloc_workqueue("balloon-wq",
WQ_FREEZABLE | WQ_CPU_INTENSIVE, 0);
if (!vb->balloon_wq) {
err = -ENOMEM;
goto out_del_vqs;
}
INIT_WORK(&vb->report_free_page_work, report_free_page_func);
vb->cmd_id_received = VIRTIO_BALLOON_CMD_ID_STOP;
vb->cmd_id_active = cpu_to_virtio32(vb->vdev,
VIRTIO_BALLOON_CMD_ID_STOP);
vb->cmd_id_stop = cpu_to_virtio32(vb->vdev,
VIRTIO_BALLOON_CMD_ID_STOP);
vb->num_free_page_blocks = 0;
spin_lock_init(&vb->free_page_list_lock);
INIT_LIST_HEAD(&vb->free_page_list);
if (virtio_has_feature(vdev, VIRTIO_BALLOON_F_PAGE_POISON)) {
memset(&poison_val, PAGE_POISON, sizeof(poison_val));
virtio_cwrite(vb->vdev, struct virtio_balloon_config,
poison_val, &poison_val);
}
}
/*
* We continue to use VIRTIO_BALLOON_F_DEFLATE_ON_OOM to decide if a
* shrinker needs to be registered to relieve memory pressure.
*/
if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_DEFLATE_ON_OOM)) {
err = virtio_balloon_register_shrinker(vb);
if (err)
goto out_del_balloon_wq;
}
virtio_device_ready(vdev);
if (towards_target(vb))
virtballoon_changed(vdev);
return 0;
out_del_balloon_wq:
if (virtio_has_feature(vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT))
destroy_workqueue(vb->balloon_wq);
out_del_vqs:
vdev->config->del_vqs(vdev);
out_free_vb:
kfree(vb);
out:
return err;
}
static int init_vqs(struct virtio_balloon *vb)
{
struct virtqueue *vqs[VIRTIO_BALLOON_VQ_MAX];
vq_callback_t *callbacks[VIRTIO_BALLOON_VQ_MAX];
const char *names[VIRTIO_BALLOON_VQ_MAX];
int err;
/*
* Inflateq and deflateq are used unconditionally. The names[]
* will be NULL if the related feature is not enabled, which will
* cause no allocation for the corresponding virtqueue in find_vqs.
*/
callbacks[VIRTIO_BALLOON_VQ_INFLATE] = balloon_ack;
names[VIRTIO_BALLOON_VQ_INFLATE] = "inflate";
callbacks[VIRTIO_BALLOON_VQ_DEFLATE] = balloon_ack;
names[VIRTIO_BALLOON_VQ_DEFLATE] = "deflate";
names[VIRTIO_BALLOON_VQ_STATS] = NULL;
names[VIRTIO_BALLOON_VQ_FREE_PAGE] = NULL;
if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_STATS_VQ)) {
names[VIRTIO_BALLOON_VQ_STATS] = "stats";
callbacks[VIRTIO_BALLOON_VQ_STATS] = stats_request;
}
if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) {
names[VIRTIO_BALLOON_VQ_FREE_PAGE] = "free_page_vq";
callbacks[VIRTIO_BALLOON_VQ_FREE_PAGE] = NULL;
}
err = vb->vdev->config->find_vqs(vb->vdev, VIRTIO_BALLOON_VQ_MAX,
vqs, callbacks, names, NULL, NULL);
if (err)
return err;
vb->inflate_vq = vqs[VIRTIO_BALLOON_VQ_INFLATE];
vb->deflate_vq = vqs[VIRTIO_BALLOON_VQ_DEFLATE];
if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_STATS_VQ)) {
struct scatterlist sg;
unsigned int num_stats;
vb->stats_vq = vqs[VIRTIO_BALLOON_VQ_STATS];
/*
* Prime this virtqueue with one buffer so the hypervisor can
* use it to signal us later (it can't be broken yet!).
*/
num_stats = update_balloon_stats(vb);
sg_init_one(&sg, vb->stats, sizeof(vb->stats[0]) * num_stats);
err = virtqueue_add_outbuf(vb->stats_vq, &sg, 1, vb,
GFP_KERNEL);
if (err) {
dev_warn(&vb->vdev->dev, "%s: add stat_vq failed\n",
__func__);
return err;
}
virtqueue_kick(vb->stats_vq);
}
if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT))
vb->free_page_vq = vqs[VIRTIO_BALLOON_VQ_FREE_PAGE];
return 0;
}
static int rpmsg_probe(struct virtio_device *vdev)
{
vq_callback_t *vq_cbs[] = { rpmsg_recv_done, rpmsg_xmit_done };
const char *names[] = { "input", "output" };
struct virtqueue *vqs[2];
struct virtproc_info *vrp;
struct rproc *vrp_rproc;
void *bufs_va;
void *cpu_addr; /* buffer virtual address */
void *cpu_addr_dma; /* buffer DMA address' virutal address conversion */
void *rbufs_guest_addr_kva;
int err = 0, i;
size_t total_buf_space;
vrp = kzalloc(sizeof(*vrp), GFP_KERNEL);
if (!vrp)
return -ENOMEM;
vrp->vdev = vdev;
idr_init(&vrp->endpoints);
mutex_init(&vrp->endpoints_lock);
mutex_init(&vrp->tx_lock);
init_waitqueue_head(&vrp->sendq);
/* We expect two virtqueues, rx and tx (and in this order) */
err = vdev->config->find_vqs(vdev, 2, vqs, vq_cbs, names);
if (err)
goto free_vrp;
vrp->rvq = vqs[0];
vrp->svq = vqs[1];
/* we expect symmetric tx/rx vrings */
WARN_ON(virtqueue_get_vring_size(vrp->rvq) !=
virtqueue_get_vring_size(vrp->svq));
/* we need less buffers if vrings are small */
if (virtqueue_get_vring_size(vrp->rvq) < MAX_RPMSG_NUM_BUFS / 2)
vrp->num_bufs = virtqueue_get_vring_size(vrp->rvq) * 2;
else
vrp->num_bufs = MAX_RPMSG_NUM_BUFS;
total_buf_space = vrp->num_bufs * RPMSG_BUF_SIZE;
/* allocate coherent memory for the buffers */
bufs_va = dma_alloc_coherent(vdev->dev.parent->parent,
total_buf_space, &vrp->bufs_dma,
GFP_KERNEL);
if (!bufs_va) {
err = -ENOMEM;
goto vqs_del;
}
dev_dbg(&vdev->dev, "buffers: va %p, dma 0x%llx\n", bufs_va,
(unsigned long long)vrp->bufs_dma);
/* half of the buffers is dedicated for RX */
vrp->rbufs = bufs_va;
/* and half is dedicated for TX */
vrp->sbufs = bufs_va + total_buf_space / 2;
vrp_rproc = vdev_to_rproc(vdev);
rbufs_guest_addr_kva = vrp->rbufs;
if (vrp_rproc->ops->kva_to_guest_addr_kva) {
rbufs_guest_addr_kva = vrp_rproc->ops->kva_to_guest_addr_kva(vrp_rproc, vrp->rbufs, vrp->rvq);
}
/* set up the receive buffers */
for (i = 0; i < vrp->num_bufs / 2; i++) {
struct scatterlist sg;
cpu_addr = vrp->rbufs + i * RPMSG_BUF_SIZE;
cpu_addr_dma = rbufs_guest_addr_kva + i*RPMSG_BUF_SIZE;
sg_init_one(&sg, cpu_addr_dma, RPMSG_BUF_SIZE);
err = virtqueue_add_inbuf(vrp->rvq, &sg, 1, cpu_addr,
GFP_KERNEL);
WARN_ON(err); /* sanity check; this can't really happen */
}
/* suppress "tx-complete" interrupts */
virtqueue_disable_cb(vrp->svq);
vdev->priv = vrp;
/* if supported by the remote processor, enable the name service */
if (virtio_has_feature(vdev, VIRTIO_RPMSG_F_NS)) {
/* a dedicated endpoint handles the name service msgs */
vrp->ns_ept = __rpmsg_create_ept(vrp, NULL, rpmsg_ns_cb,
vrp, RPMSG_NS_ADDR);
if (!vrp->ns_ept) {
dev_err(&vdev->dev, "failed to create the ns ept\n");
err = -ENOMEM;
goto free_coherent;
}
}
/* tell the remote processor it can start sending messages */
virtqueue_kick(vrp->rvq);
dev_info(&vdev->dev, "rpmsg host is online\n");
return 0;
free_coherent:
dma_free_coherent(vdev->dev.parent->parent, total_buf_space,
bufs_va, vrp->bufs_dma);
vqs_del:
vdev->config->del_vqs(vrp->vdev);
free_vrp:
kfree(vrp);
return err;
}
static int rpmsg_probe(struct virtio_device *vdev)
{
vq_callback_t *vq_cbs[] = { rpmsg_recv_done, rpmsg_xmit_done };
static const char * const names[] = { "input", "output" };
struct virtqueue *vqs[2];
struct virtproc_info *vrp;
void *bufs_va;
int err = 0, i;
size_t total_buf_space;
bool notify;
vrp = kzalloc(sizeof(*vrp), GFP_KERNEL);
if (!vrp)
return -ENOMEM;
vrp->vdev = vdev;
idr_init(&vrp->endpoints);
mutex_init(&vrp->endpoints_lock);
mutex_init(&vrp->tx_lock);
init_waitqueue_head(&vrp->sendq);
/* We expect two virtqueues, rx and tx (and in this order) */
err = vdev->config->find_vqs(vdev, 2, vqs, vq_cbs, names);
if (err)
goto free_vrp;
vrp->rvq = vqs[0];
vrp->svq = vqs[1];
/* we expect symmetric tx/rx vrings */
WARN_ON(virtqueue_get_vring_size(vrp->rvq) !=
virtqueue_get_vring_size(vrp->svq));
/* we need less buffers if vrings are small */
if (virtqueue_get_vring_size(vrp->rvq) < MAX_RPMSG_NUM_BUFS / 2)
vrp->num_bufs = virtqueue_get_vring_size(vrp->rvq) * 2;
else
vrp->num_bufs = MAX_RPMSG_NUM_BUFS;
total_buf_space = vrp->num_bufs * RPMSG_BUF_SIZE;
/* allocate coherent memory for the buffers */
bufs_va = dma_alloc_coherent(vdev->dev.parent->parent,
total_buf_space, &vrp->bufs_dma,
GFP_KERNEL);
if (!bufs_va) {
err = -ENOMEM;
goto vqs_del;
}
dev_dbg(&vdev->dev, "buffers: va %p, dma 0x%llx\n", bufs_va,
(unsigned long long)vrp->bufs_dma);
/* half of the buffers is dedicated for RX */
vrp->rbufs = bufs_va;
/* and half is dedicated for TX */
vrp->sbufs = bufs_va + total_buf_space / 2;
/* set up the receive buffers */
for (i = 0; i < vrp->num_bufs / 2; i++) {
struct scatterlist sg;
void *cpu_addr = vrp->rbufs + i * RPMSG_BUF_SIZE;
rpmsg_msg_sg_init(vrp, &sg, cpu_addr, RPMSG_BUF_SIZE);
err = rpmsg_virtqueue_add_inbuf(vrp->rvq, &sg, 1, cpu_addr,
GFP_KERNEL);
WARN_ON(err); /* sanity check; this can't really happen */
}
/* suppress "tx-complete" interrupts */
virtqueue_disable_cb(vrp->svq);
vdev->priv = vrp;
/* if supported by the remote processor, enable the name service */
if (virtio_has_feature(vdev, VIRTIO_RPMSG_F_NS)) {
/* a dedicated endpoint handles the name service msgs */
vrp->ns_ept = __rpmsg_create_ept(vrp, NULL, rpmsg_ns_cb,
vrp, RPMSG_NS_ADDR);
if (!vrp->ns_ept) {
dev_err(&vdev->dev, "failed to create the ns ept\n");
err = -ENOMEM;
goto free_coherent;
}
}
/*
* Prepare to kick but don't notify yet - we can't do this before
* device is ready.
*/
notify = virtqueue_kick_prepare(vrp->rvq);
/* From this point on, we can notify and get callbacks. */
virtio_device_ready(vdev);
/* tell the remote processor it can start sending messages */
/*
* this might be concurrent with callbacks, but we are only
* doing notify, not a full kick here, so that's ok.
*/
if (notify)
virtqueue_notify(vrp->rvq);
dev_info(&vdev->dev, "rpmsg host is online\n");
return 0;
free_coherent:
dma_free_coherent(vdev->dev.parent->parent, total_buf_space,
bufs_va, vrp->bufs_dma);
vqs_del:
vdev->config->del_vqs(vrp->vdev);
free_vrp:
kfree(vrp);
return err;
}
