static void virtinput_queue_evtbuf(struct virtio_input *vi,
struct virtio_input_event *evtbuf)
{
struct scatterlist sg[1];
sg_init_one(sg, evtbuf, sizeof(*evtbuf));
virtqueue_add_inbuf(vi->evt, sg, 1, evtbuf, GFP_ATOMIC);
}
/* The host will fill any buffer we give it with sweet, sweet randomness. */
static void register_buffer(struct virtrng_info *vi, u8 *buf, size_t size)
{
struct scatterlist sg;
sg_init_one(&sg, buf, size);
/* There should always be room for one buffer. */
virtqueue_add_inbuf(vi->vq, &sg, 1, buf, GFP_KERNEL);
virtqueue_kick(vi->vq);
}
static int get_free_page_and_send(struct virtio_balloon *vb)
{
struct virtqueue *vq = vb->free_page_vq;
struct page *page;
struct scatterlist sg;
int err, unused;
void *p;
/* Detach all the used buffers from the vq */
while (virtqueue_get_buf(vq, &unused))
;
page = alloc_pages(VIRTIO_BALLOON_FREE_PAGE_ALLOC_FLAG,
VIRTIO_BALLOON_FREE_PAGE_ORDER);
/*
* When the allocation returns NULL, it indicates that we have got all
* the possible free pages, so return -EINTR to stop.
*/
if (!page)
return -EINTR;
p = page_address(page);
sg_init_one(&sg, p, VIRTIO_BALLOON_FREE_PAGE_SIZE);
/* There is always 1 entry reserved for the cmd id to use. */
if (vq->num_free > 1) {
err = virtqueue_add_inbuf(vq, &sg, 1, p, GFP_KERNEL);
if (unlikely(err)) {
free_pages((unsigned long)p,
VIRTIO_BALLOON_FREE_PAGE_ORDER);
return err;
}
virtqueue_kick(vq);
spin_lock_irq(&vb->free_page_list_lock);
balloon_page_push(&vb->free_page_list, page);
vb->num_free_page_blocks++;
spin_unlock_irq(&vb->free_page_list_lock);
} else {
/*
* The vq has no available entry to add this page block, so
* just free it.
*/
free_pages((unsigned long)p, VIRTIO_BALLOON_FREE_PAGE_ORDER);
}
return 0;
}
static int rpmsg_probe(struct virtio_device *vdev)
{
vq_callback_t *vq_cbs[] = { rpmsg_recv_done, rpmsg_xmit_done };
const char *names[] = { "input", "output" };
struct virtqueue *vqs[2];
struct virtproc_info *vrp;
void *bufs_va;
int err = 0, i;
vrp = kzalloc(sizeof(*vrp), GFP_KERNEL);
if (!vrp)
return -ENOMEM;
vrp->vdev = vdev;
idr_init(&vrp->endpoints);
mutex_init(&vrp->endpoints_lock);
mutex_init(&vrp->tx_lock);
init_waitqueue_head(&vrp->sendq);
/* We expect two virtqueues, rx and tx (and in this order) */
err = vdev->config->find_vqs(vdev, 2, vqs, vq_cbs, names);
if (err)
goto free_vrp;
vrp->rvq = vqs[0];
vrp->svq = vqs[1];
/* allocate coherent memory for the buffers */
bufs_va = dma_alloc_coherent(vdev->dev.parent->parent,
RPMSG_TOTAL_BUF_SPACE,
&vrp->bufs_dma, GFP_KERNEL);
if (!bufs_va) {
err = -ENOMEM;
goto vqs_del;
}
dev_dbg(&vdev->dev, "buffers: va %p, dma 0x%llx\n", bufs_va,
(unsigned long long)vrp->bufs_dma);
/* half of the buffers is dedicated for RX */
vrp->rbufs = bufs_va;
/* and half is dedicated for TX */
vrp->sbufs = bufs_va + RPMSG_TOTAL_BUF_SPACE / 2;
/* set up the receive buffers */
for (i = 0; i < RPMSG_NUM_BUFS / 2; i++) {
struct scatterlist sg;
void *cpu_addr = vrp->rbufs + i * RPMSG_BUF_SIZE;
sg_init_one(&sg, cpu_addr, RPMSG_BUF_SIZE);
err = virtqueue_add_inbuf(vrp->rvq, &sg, 1, cpu_addr,
GFP_KERNEL);
WARN_ON(err); /* sanity check; this can't really happen */
}
/* suppress "tx-complete" interrupts */
virtqueue_disable_cb(vrp->svq);
vdev->priv = vrp;
/* if supported by the remote processor, enable the name service */
if (virtio_has_feature(vdev, VIRTIO_RPMSG_F_NS)) {
/* a dedicated endpoint handles the name service msgs */
vrp->ns_ept = __rpmsg_create_ept(vrp, NULL, rpmsg_ns_cb,
vrp, RPMSG_NS_ADDR);
if (!vrp->ns_ept) {
dev_err(&vdev->dev, "failed to create the ns ept\n");
err = -ENOMEM;
goto free_coherent;
}
}
/* tell the remote processor it can start sending messages */
virtqueue_kick(vrp->rvq);
dev_info(&vdev->dev, "rpmsg host is online\n");
return 0;
free_coherent:
dma_free_coherent(vdev->dev.parent->parent, RPMSG_TOTAL_BUF_SPACE,
bufs_va, vrp->bufs_dma);
vqs_del:
vdev->config->del_vqs(vrp->vdev);
free_vrp:
kfree(vrp);
return err;
}
static int rpmsg_recv_single(struct virtproc_info *vrp, struct device *dev,
struct rpmsg_hdr *msg, unsigned int len)
{
struct rpmsg_endpoint *ept;
struct scatterlist sg;
int err;
dev_dbg(dev, "From: 0x%x, To: 0x%x, Len: %d, Flags: %d, Reserved: %d\n",
msg->src, msg->dst, msg->len,
msg->flags, msg->reserved);
print_hex_dump(KERN_DEBUG, "rpmsg_virtio RX: ", DUMP_PREFIX_NONE, 16, 1,
msg, sizeof(*msg) + msg->len, true);
/*
* We currently use fixed-sized buffers, so trivially sanitize
* the reported payload length.
*/
if (len > RPMSG_BUF_SIZE ||
msg->len > (len - sizeof(struct rpmsg_hdr))) {
dev_warn(dev, "inbound msg too big: (%d, %d)\n", len, msg->len);
return -EINVAL;
}
/* use the dst addr to fetch the callback of the appropriate user */
mutex_lock(&vrp->endpoints_lock);
ept = idr_find(&vrp->endpoints, msg->dst);
/* let's make sure no one deallocates ept while we use it */
if (ept)
kref_get(&ept->refcount);
mutex_unlock(&vrp->endpoints_lock);
if (ept) {
/* make sure ept->cb doesn't go away while we use it */
mutex_lock(&ept->cb_lock);
if (ept->cb)
ept->cb(ept->rpdev, msg->data, msg->len, ept->priv,
msg->src);
mutex_unlock(&ept->cb_lock);
/* farewell, ept, we don't need you anymore */
kref_put(&ept->refcount, __ept_release);
} else
dev_warn(dev, "msg received with no recipient\n");
/* publish the real size of the buffer */
sg_init_one(&sg, msg, RPMSG_BUF_SIZE);
/* add the buffer back to the remote processor's virtqueue */
err = virtqueue_add_inbuf(vrp->rvq, &sg, 1, msg, GFP_KERNEL);
if (err < 0) {
dev_err(dev, "failed to add a virtqueue buffer: %d\n", err);
return err;
}
return 0;
}
static int rpmsg_probe(struct virtio_device *vdev)
{
vq_callback_t *vq_cbs[] = { rpmsg_recv_done, rpmsg_xmit_done };
static const char * const names[] = { "input", "output" };
struct virtqueue *vqs[2];
struct virtproc_info *vrp;
void *bufs_va;
int err = 0, i;
size_t total_buf_space;
bool notify;
vrp = kzalloc(sizeof(*vrp), GFP_KERNEL);
if (!vrp)
return -ENOMEM;
vrp->vdev = vdev;
idr_init(&vrp->endpoints);
mutex_init(&vrp->endpoints_lock);
mutex_init(&vrp->tx_lock);
init_waitqueue_head(&vrp->sendq);
/* We expect two virtqueues, rx and tx (and in this order) */
err = vdev->config->find_vqs(vdev, 2, vqs, vq_cbs, names);
if (err)
goto free_vrp;
vrp->rvq = vqs[0];
vrp->svq = vqs[1];
/* we expect symmetric tx/rx vrings */
WARN_ON(virtqueue_get_vring_size(vrp->rvq) !=
virtqueue_get_vring_size(vrp->svq));
/* we need less buffers if vrings are small */
if (virtqueue_get_vring_size(vrp->rvq) < MAX_RPMSG_NUM_BUFS / 2)
vrp->num_bufs = virtqueue_get_vring_size(vrp->rvq) * 2;
else
vrp->num_bufs = MAX_RPMSG_NUM_BUFS;
total_buf_space = vrp->num_bufs * RPMSG_BUF_SIZE;
/* allocate coherent memory for the buffers */
bufs_va = dma_alloc_coherent(vdev->dev.parent->parent,
total_buf_space, &vrp->bufs_dma,
GFP_KERNEL);
if (!bufs_va) {
err = -ENOMEM;
goto vqs_del;
}
dev_dbg(&vdev->dev, "buffers: va %p, dma 0x%llx\n", bufs_va,
(unsigned long long)vrp->bufs_dma);
/* half of the buffers is dedicated for RX */
vrp->rbufs = bufs_va;
/* and half is dedicated for TX */
vrp->sbufs = bufs_va + total_buf_space / 2;
/* set up the receive buffers */
for (i = 0; i < vrp->num_bufs / 2; i++) {
struct scatterlist sg;
void *cpu_addr = vrp->rbufs + i * RPMSG_BUF_SIZE;
sg_init_one(&sg, cpu_addr, RPMSG_BUF_SIZE);
err = virtqueue_add_inbuf(vrp->rvq, &sg, 1, cpu_addr,
GFP_KERNEL);
WARN_ON(err); /* sanity check; this can't really happen */
}
/* suppress "tx-complete" interrupts */
virtqueue_disable_cb(vrp->svq);
vdev->priv = vrp;
/* if supported by the remote processor, enable the name service */
if (virtio_has_feature(vdev, VIRTIO_RPMSG_F_NS)) {
/* a dedicated endpoint handles the name service msgs */
vrp->ns_ept = __rpmsg_create_ept(vrp, NULL, rpmsg_ns_cb,
vrp, RPMSG_NS_ADDR);
if (!vrp->ns_ept) {
dev_err(&vdev->dev, "failed to create the ns ept\n");
err = -ENOMEM;
goto free_coherent;
}
}
/*
* Prepare to kick but don't notify yet - we can't do this before
* device is ready.
*/
notify = virtqueue_kick_prepare(vrp->rvq);
/* From this point on, we can notify and get callbacks. */
virtio_device_ready(vdev);
/* tell the remote processor it can start sending messages */
/*
* this might be concurrent with callbacks, but we are only
* doing notify, not a full kick here, so that's ok.
*/
if (notify)
virtqueue_notify(vrp->rvq);
dev_info(&vdev->dev, "rpmsg host is online\n");
return 0;
free_coherent:
dma_free_coherent(vdev->dev.parent->parent, total_buf_space,
bufs_va, vrp->bufs_dma);
vqs_del:
vdev->config->del_vqs(vrp->vdev);
free_vrp:
kfree(vrp);
return err;
}
