static void virtinput_queue_evtbuf(struct virtio_input *vi, struct virtio_input_event *evtbuf) { struct scatterlist sg[1]; sg_init_one(sg, evtbuf, sizeof(*evtbuf)); virtqueue_add_inbuf(vi->evt, sg, 1, evtbuf, GFP_ATOMIC); }
/* The host will fill any buffer we give it with sweet, sweet randomness. */ static void register_buffer(struct virtrng_info *vi, u8 *buf, size_t size) { struct scatterlist sg; sg_init_one(&sg, buf, size); /* There should always be room for one buffer. */ virtqueue_add_inbuf(vi->vq, &sg, 1, buf, GFP_KERNEL); virtqueue_kick(vi->vq); }
static int get_free_page_and_send(struct virtio_balloon *vb) { struct virtqueue *vq = vb->free_page_vq; struct page *page; struct scatterlist sg; int err, unused; void *p; /* Detach all the used buffers from the vq */ while (virtqueue_get_buf(vq, &unused)) ; page = alloc_pages(VIRTIO_BALLOON_FREE_PAGE_ALLOC_FLAG, VIRTIO_BALLOON_FREE_PAGE_ORDER); /* * When the allocation returns NULL, it indicates that we have got all * the possible free pages, so return -EINTR to stop. */ if (!page) return -EINTR; p = page_address(page); sg_init_one(&sg, p, VIRTIO_BALLOON_FREE_PAGE_SIZE); /* There is always 1 entry reserved for the cmd id to use. */ if (vq->num_free > 1) { err = virtqueue_add_inbuf(vq, &sg, 1, p, GFP_KERNEL); if (unlikely(err)) { free_pages((unsigned long)p, VIRTIO_BALLOON_FREE_PAGE_ORDER); return err; } virtqueue_kick(vq); spin_lock_irq(&vb->free_page_list_lock); balloon_page_push(&vb->free_page_list, page); vb->num_free_page_blocks++; spin_unlock_irq(&vb->free_page_list_lock); } else { /* * The vq has no available entry to add this page block, so * just free it. */ free_pages((unsigned long)p, VIRTIO_BALLOON_FREE_PAGE_ORDER); } return 0; }
static int rpmsg_probe(struct virtio_device *vdev) { vq_callback_t *vq_cbs[] = { rpmsg_recv_done, rpmsg_xmit_done }; const char *names[] = { "input", "output" }; struct virtqueue *vqs[2]; struct virtproc_info *vrp; void *bufs_va; int err = 0, i; vrp = kzalloc(sizeof(*vrp), GFP_KERNEL); if (!vrp) return -ENOMEM; vrp->vdev = vdev; idr_init(&vrp->endpoints); mutex_init(&vrp->endpoints_lock); mutex_init(&vrp->tx_lock); init_waitqueue_head(&vrp->sendq); /* We expect two virtqueues, rx and tx (and in this order) */ err = vdev->config->find_vqs(vdev, 2, vqs, vq_cbs, names); if (err) goto free_vrp; vrp->rvq = vqs[0]; vrp->svq = vqs[1]; /* allocate coherent memory for the buffers */ bufs_va = dma_alloc_coherent(vdev->dev.parent->parent, RPMSG_TOTAL_BUF_SPACE, &vrp->bufs_dma, GFP_KERNEL); if (!bufs_va) { err = -ENOMEM; goto vqs_del; } dev_dbg(&vdev->dev, "buffers: va %p, dma 0x%llx\n", bufs_va, (unsigned long long)vrp->bufs_dma); /* half of the buffers is dedicated for RX */ vrp->rbufs = bufs_va; /* and half is dedicated for TX */ vrp->sbufs = bufs_va + RPMSG_TOTAL_BUF_SPACE / 2; /* set up the receive buffers */ for (i = 0; i < RPMSG_NUM_BUFS / 2; i++) { struct scatterlist sg; void *cpu_addr = vrp->rbufs + i * RPMSG_BUF_SIZE; sg_init_one(&sg, cpu_addr, RPMSG_BUF_SIZE); err = virtqueue_add_inbuf(vrp->rvq, &sg, 1, cpu_addr, GFP_KERNEL); WARN_ON(err); /* sanity check; this can't really happen */ } /* suppress "tx-complete" interrupts */ virtqueue_disable_cb(vrp->svq); vdev->priv = vrp; /* if supported by the remote processor, enable the name service */ if (virtio_has_feature(vdev, VIRTIO_RPMSG_F_NS)) { /* a dedicated endpoint handles the name service msgs */ vrp->ns_ept = __rpmsg_create_ept(vrp, NULL, rpmsg_ns_cb, vrp, RPMSG_NS_ADDR); if (!vrp->ns_ept) { dev_err(&vdev->dev, "failed to create the ns ept\n"); err = -ENOMEM; goto free_coherent; } } /* tell the remote processor it can start sending messages */ virtqueue_kick(vrp->rvq); dev_info(&vdev->dev, "rpmsg host is online\n"); return 0; free_coherent: dma_free_coherent(vdev->dev.parent->parent, RPMSG_TOTAL_BUF_SPACE, bufs_va, vrp->bufs_dma); vqs_del: vdev->config->del_vqs(vrp->vdev); free_vrp: kfree(vrp); return err; }
static int rpmsg_recv_single(struct virtproc_info *vrp, struct device *dev, struct rpmsg_hdr *msg, unsigned int len) { struct rpmsg_endpoint *ept; struct scatterlist sg; int err; dev_dbg(dev, "From: 0x%x, To: 0x%x, Len: %d, Flags: %d, Reserved: %d\n", msg->src, msg->dst, msg->len, msg->flags, msg->reserved); print_hex_dump(KERN_DEBUG, "rpmsg_virtio RX: ", DUMP_PREFIX_NONE, 16, 1, msg, sizeof(*msg) + msg->len, true); /* * We currently use fixed-sized buffers, so trivially sanitize * the reported payload length. */ if (len > RPMSG_BUF_SIZE || msg->len > (len - sizeof(struct rpmsg_hdr))) { dev_warn(dev, "inbound msg too big: (%d, %d)\n", len, msg->len); return -EINVAL; } /* use the dst addr to fetch the callback of the appropriate user */ mutex_lock(&vrp->endpoints_lock); ept = idr_find(&vrp->endpoints, msg->dst); /* let's make sure no one deallocates ept while we use it */ if (ept) kref_get(&ept->refcount); mutex_unlock(&vrp->endpoints_lock); if (ept) { /* make sure ept->cb doesn't go away while we use it */ mutex_lock(&ept->cb_lock); if (ept->cb) ept->cb(ept->rpdev, msg->data, msg->len, ept->priv, msg->src); mutex_unlock(&ept->cb_lock); /* farewell, ept, we don't need you anymore */ kref_put(&ept->refcount, __ept_release); } else dev_warn(dev, "msg received with no recipient\n"); /* publish the real size of the buffer */ sg_init_one(&sg, msg, RPMSG_BUF_SIZE); /* add the buffer back to the remote processor's virtqueue */ err = virtqueue_add_inbuf(vrp->rvq, &sg, 1, msg, GFP_KERNEL); if (err < 0) { dev_err(dev, "failed to add a virtqueue buffer: %d\n", err); return err; } return 0; }
static int rpmsg_probe(struct virtio_device *vdev) { vq_callback_t *vq_cbs[] = { rpmsg_recv_done, rpmsg_xmit_done }; static const char * const names[] = { "input", "output" }; struct virtqueue *vqs[2]; struct virtproc_info *vrp; void *bufs_va; int err = 0, i; size_t total_buf_space; bool notify; vrp = kzalloc(sizeof(*vrp), GFP_KERNEL); if (!vrp) return -ENOMEM; vrp->vdev = vdev; idr_init(&vrp->endpoints); mutex_init(&vrp->endpoints_lock); mutex_init(&vrp->tx_lock); init_waitqueue_head(&vrp->sendq); /* We expect two virtqueues, rx and tx (and in this order) */ err = vdev->config->find_vqs(vdev, 2, vqs, vq_cbs, names); if (err) goto free_vrp; vrp->rvq = vqs[0]; vrp->svq = vqs[1]; /* we expect symmetric tx/rx vrings */ WARN_ON(virtqueue_get_vring_size(vrp->rvq) != virtqueue_get_vring_size(vrp->svq)); /* we need less buffers if vrings are small */ if (virtqueue_get_vring_size(vrp->rvq) < MAX_RPMSG_NUM_BUFS / 2) vrp->num_bufs = virtqueue_get_vring_size(vrp->rvq) * 2; else vrp->num_bufs = MAX_RPMSG_NUM_BUFS; total_buf_space = vrp->num_bufs * RPMSG_BUF_SIZE; /* allocate coherent memory for the buffers */ bufs_va = dma_alloc_coherent(vdev->dev.parent->parent, total_buf_space, &vrp->bufs_dma, GFP_KERNEL); if (!bufs_va) { err = -ENOMEM; goto vqs_del; } dev_dbg(&vdev->dev, "buffers: va %p, dma 0x%llx\n", bufs_va, (unsigned long long)vrp->bufs_dma); /* half of the buffers is dedicated for RX */ vrp->rbufs = bufs_va; /* and half is dedicated for TX */ vrp->sbufs = bufs_va + total_buf_space / 2; /* set up the receive buffers */ for (i = 0; i < vrp->num_bufs / 2; i++) { struct scatterlist sg; void *cpu_addr = vrp->rbufs + i * RPMSG_BUF_SIZE; sg_init_one(&sg, cpu_addr, RPMSG_BUF_SIZE); err = virtqueue_add_inbuf(vrp->rvq, &sg, 1, cpu_addr, GFP_KERNEL); WARN_ON(err); /* sanity check; this can't really happen */ } /* suppress "tx-complete" interrupts */ virtqueue_disable_cb(vrp->svq); vdev->priv = vrp; /* if supported by the remote processor, enable the name service */ if (virtio_has_feature(vdev, VIRTIO_RPMSG_F_NS)) { /* a dedicated endpoint handles the name service msgs */ vrp->ns_ept = __rpmsg_create_ept(vrp, NULL, rpmsg_ns_cb, vrp, RPMSG_NS_ADDR); if (!vrp->ns_ept) { dev_err(&vdev->dev, "failed to create the ns ept\n"); err = -ENOMEM; goto free_coherent; } } /* * Prepare to kick but don't notify yet - we can't do this before * device is ready. */ notify = virtqueue_kick_prepare(vrp->rvq); /* From this point on, we can notify and get callbacks. */ virtio_device_ready(vdev); /* tell the remote processor it can start sending messages */ /* * this might be concurrent with callbacks, but we are only * doing notify, not a full kick here, so that's ok. */ if (notify) virtqueue_notify(vrp->rvq); dev_info(&vdev->dev, "rpmsg host is online\n"); return 0; free_coherent: dma_free_coherent(vdev->dev.parent->parent, total_buf_space, bufs_va, vrp->bufs_dma); vqs_del: vdev->config->del_vqs(vrp->vdev); free_vrp: kfree(vrp); return err; }