static void rpmsg_recv_done(struct virtqueue *rvq)
{
struct rpmsg_hdr *msg;
unsigned int len;
struct rpmsg_endpoint *ept;
struct scatterlist sg;
unsigned long offset;
void *sim_addr;
struct virtproc_info *vrp = rvq->vdev->priv;
struct device *dev = &rvq->vdev->dev;
int err;
/* if fail to acquire the lock, remove stage is happening, then just return */
if(!mutex_trylock(&vrp->rm_lock))
return;
/* make sure the descriptors are updated before reading */
rmb();
msg = virtqueue_get_buf(rvq, &len);
if (!msg) {
dev_err(dev, "uhm, incoming signal, but no used buffer ?\n");
return;
}
dev_dbg(dev, "From: 0x%x, To: 0x%x, Len: %d, Flags: %d, Unused: %d\n",
msg->src, msg->dst, msg->len,
msg->flags, msg->unused);
#if 0
print_hex_dump(KERN_DEBUG, "rpmsg_virtio RX: ", DUMP_PREFIX_NONE, 16, 1,
msg, sizeof(*msg) + msg->len, true);
#endif
/* fetch the callback of the appropriate user */
spin_lock(&vrp->endpoints_lock);
ept = idr_find(&vrp->endpoints, msg->dst);
spin_unlock(&vrp->endpoints_lock);
if (ept && ept->cb)
ept->cb(ept->rpdev, msg->data, msg->len, ept->priv, msg->src);
else
dev_warn(dev, "msg received with no recepient\n");
/* add the buffer back to the remote processor's virtqueue */
offset = ((unsigned long) msg) - ((unsigned long) vrp->rbufs);
sim_addr = vrp->sim_base + offset;
sg_init_one(&sg, sim_addr, sizeof(*msg) + len);
err = virtqueue_add_buf_gfp(vrp->rvq, &sg, 0, 1, msg, GFP_KERNEL);
if (err < 0) {
dev_err(dev, "failed to add a virtqueue buffer: %d\n", err);
return;
}
/* descriptors must be written before kicking remote processor */
wmb();
/* tell the remote processor we added another available rx buffer */
virtqueue_kick(vrp->rvq);
mutex_unlock(&vrp->rm_lock);
}
/* called when an rx buffer is used, and it's time to digest a message */
static void rpmsg_recv_done(struct virtqueue *rvq)
{
struct rpmsg_hdr *msg;
unsigned int len;
struct rpmsg_endpoint *ept;
struct scatterlist sg;
struct virtproc_info *vrp = rvq->vdev->priv;
struct device *dev = &rvq->vdev->dev;
int err;
msg = virtqueue_get_buf(rvq, &len);
if (!msg) {
dev_err(dev, "uhm, incoming signal, but no used buffer ?\n");
return;
}
dev_dbg(dev, "From: 0x%x, To: 0x%x, Len: %d, Flags: %d, Reserved: %d\n",
msg->src, msg->dst, msg->len,
msg->flags, msg->reserved);
print_hex_dump(KERN_DEBUG, "rpmsg_virtio RX: ", DUMP_PREFIX_NONE, 16, 1,
msg, sizeof(*msg) + msg->len, true);
/* use the dst addr to fetch the callback of the appropriate user */
mutex_lock(&vrp->endpoints_lock);
ept = idr_find(&vrp->endpoints, msg->dst);
mutex_unlock(&vrp->endpoints_lock);
if (ept && ept->cb)
ept->cb(ept->rpdev, msg->data, msg->len, ept->priv, msg->src);
else
dev_warn(dev, "msg received with no recepient\n");
sg_init_one(&sg, msg, sizeof(*msg) + len);
/* add the buffer back to the remote processor's virtqueue */
err = virtqueue_add_buf_gfp(vrp->rvq, &sg, 0, 1, msg, GFP_KERNEL);
if (err < 0) {
dev_err(dev, "failed to add a virtqueue buffer: %d\n", err);
return;
}
/* tell the remote processor we added another available rx buffer */
virtqueue_kick(vrp->rvq);
}
static int rpmsg_probe(struct virtio_device *vdev)
{
vq_callback_t *vq_cbs[] = { rpmsg_recv_done, rpmsg_xmit_done };
const char *names[] = { "input", "output" };
struct virtqueue *vqs[2];
struct virtproc_info *vrp;
void *addr;
int err, i, num_bufs, buf_size, total_buf_size;
struct rpmsg_channel_info *ch;
vrp = kzalloc(sizeof(*vrp), GFP_KERNEL);
if (!vrp)
return -ENOMEM;
vrp->vdev = vdev;
idr_init(&vrp->endpoints);
spin_lock_init(&vrp->endpoints_lock);
mutex_init(&vrp->svq_lock);
mutex_init(&vrp->rm_lock);
init_waitqueue_head(&vrp->sendq);
/* We expect two virtqueues, rx and tx (in this order) */
err = vdev->config->find_vqs(vdev, 2, vqs, vq_cbs, names);
if (err)
goto free_vi;
vrp->rvq = vqs[0];
vrp->svq = vqs[1];
/* Platform must supply pre-allocated uncached buffers for now */
vdev->config->get(vdev, VPROC_BUF_ADDR, &addr, sizeof(addr));
vdev->config->get(vdev, VPROC_BUF_NUM, &num_bufs,
sizeof(num_bufs));
vdev->config->get(vdev, VPROC_BUF_SZ, &buf_size, sizeof(buf_size));
total_buf_size = num_bufs * buf_size;
dev_dbg(&vdev->dev, "%d buffers, size %d, addr 0x%x, total 0x%x\n",
num_bufs, buf_size, (unsigned int) addr, total_buf_size);
vrp->num_bufs = num_bufs;
vrp->buf_size = buf_size;
vrp->rbufs = addr;
vrp->sbufs = addr + total_buf_size / 2;
/* simulated addr base to make virt_to_page happy */
vdev->config->get(vdev, VPROC_SIM_BASE, &vrp->sim_base,
sizeof(vrp->sim_base));
/* set up the receive buffers */
for (i = 0; i < num_bufs / 2; i++) {
struct scatterlist sg;
void *tmpaddr = vrp->rbufs + i * buf_size;
void *simaddr = vrp->sim_base + i * buf_size;
sg_init_one(&sg, simaddr, buf_size);
err = virtqueue_add_buf_gfp(vrp->rvq, &sg, 0, 1, tmpaddr,
GFP_KERNEL);
WARN_ON(err < 0); /* sanity check; this can't really happen */
}
/* tell the remote processor it can start sending data */
virtqueue_kick(vrp->rvq);
/* suppress "tx-complete" interrupts */
virtqueue_disable_cb(vrp->svq);
vdev->priv = vrp;
dev_info(&vdev->dev, "rpmsg backend virtproc probed successfully\n");
/* if supported by the remote processor, enable the name service */
if (virtio_has_feature(vdev, VIRTIO_RPMSG_F_NS)) {
vrp->ns_ept = __rpmsg_create_ept(vrp, NULL, rpmsg_ns_cb,
vrp, RPMSG_NS_ADDR);
if (!vrp->ns_ept) {
dev_err(&vdev->dev, "failed to create the ns ept\n");
err = -ENOMEM;
goto vqs_del;
}
}
/* look for platform-specific static channels */
vdev->config->get(vdev, VPROC_STATIC_CHANNELS, &ch, sizeof(ch));
for (i = 0; ch && ch[i].name[0]; i++)
rpmsg_create_channel(vrp, &ch[i]);
return 0;
vqs_del:
vdev->config->del_vqs(vrp->vdev);
free_vi:
kfree(vrp);
return err;
}
/* XXX: the blocking 'wait' mechanism hasn't been tested yet */
int rpmsg_send_offchannel_raw(struct rpmsg_channel *rpdev, u32 src, u32 dst,
void *data, int len, bool wait)
{
struct virtproc_info *vrp = rpdev->vrp;
struct device *dev = &rpdev->dev;
struct scatterlist sg;
struct rpmsg_hdr *msg;
int err;
unsigned long offset;
void *sim_addr;
if (src == RPMSG_ADDR_ANY || dst == RPMSG_ADDR_ANY) {
dev_err(dev, "invalid addr (src 0x%x, dst 0x%x)\n", src, dst);
return -EINVAL;
}
/* the payload's size is currently limited */
if (len > vrp->buf_size - sizeof(struct rpmsg_hdr)) {
dev_err(dev, "message is too big (%d)\n", len);
return -EMSGSIZE;
}
/*
* protect svq from simultaneous concurrent manipulations,
* and serialize the sending of messages
*/
if (mutex_lock_interruptible(&vrp->svq_lock))
return -ERESTARTSYS;
/* grab a buffer */
msg = get_a_buf(vrp);
if (!msg && !wait) {
err = -ENOMEM;
goto out;
}
/* no free buffer ? wait for one (but bail after 15 seconds) */
if (!msg) {
/* enable "tx-complete" interrupts before dozing off */
virtqueue_enable_cb(vrp->svq);
/*
* sleep until a free buffer is available or 15 secs elapse.
* the timeout period is not configurable because frankly
* i don't see why drivers need to deal with that.
* if later this happens to be required, it'd be easy to add.
*/
err = wait_event_interruptible_timeout(vrp->sendq,
(msg = get_a_buf(vrp)),
msecs_to_jiffies(15000));
/* on success, suppress "tx-complete" interrupts again */
virtqueue_disable_cb(vrp->svq);
if (err < 0) {
err = -ERESTARTSYS;
goto out;
}
if (!msg) {
dev_err(dev, "timeout waiting for buffer\n");
err = -ETIMEDOUT;
goto out;
}
}
msg->len = len;
msg->flags = 0;
msg->src = src;
msg->dst = dst;
msg->unused = 0;
memcpy(msg->data, data, len);
dev_dbg(dev, "TX From 0x%x, To 0x%x, Len %d, Flags %d, Unused %d\n",
msg->src, msg->dst, msg->len,
msg->flags, msg->unused);
#if 0
print_hex_dump(KERN_DEBUG, "rpmsg_virtio TX: ", DUMP_PREFIX_NONE, 16, 1,
msg, sizeof(*msg) + msg->len, true);
#endif
offset = ((unsigned long) msg) - ((unsigned long) vrp->rbufs);
sim_addr = vrp->sim_base + offset;
sg_init_one(&sg, sim_addr, sizeof(*msg) + len);
/* add message to the remote processor's virtqueue */
err = virtqueue_add_buf_gfp(vrp->svq, &sg, 1, 0, msg, GFP_KERNEL);
if (err < 0) {
dev_err(dev, "virtqueue_add_buf_gfp failed: %d\n", err);
goto out;
}
/* descriptors must be written before kicking remote processor */
wmb();
/* tell the remote processor it has a pending message to read */
virtqueue_kick(vrp->svq);
err = 0;
out:
mutex_unlock(&vrp->svq_lock);
return err;
}
static int rpmsg_probe(struct virtio_device *vdev)
{
vq_callback_t *vq_cbs[] = { rpmsg_recv_done, rpmsg_xmit_done };
const char *names[] = { "input", "output" };
struct virtqueue *vqs[2];
struct virtproc_info *vrp;
void *bufs_va;
int err = 0, i;
vrp = kzalloc(sizeof(*vrp), GFP_KERNEL);
if (!vrp)
return -ENOMEM;
vrp->vdev = vdev;
idr_init(&vrp->endpoints);
mutex_init(&vrp->endpoints_lock);
mutex_init(&vrp->tx_lock);
init_waitqueue_head(&vrp->sendq);
/* We expect two virtqueues, rx and tx (and in this order) */
err = vdev->config->find_vqs(vdev, 2, vqs, vq_cbs, names);
if (err)
goto free_vrp;
vrp->rvq = vqs[0];
vrp->svq = vqs[1];
/* allocate coherent memory for the buffers */
bufs_va = dma_alloc_coherent(vdev->dev.parent, RPMSG_TOTAL_BUF_SPACE,
&vrp->bufs_dma, GFP_KERNEL);
if (!bufs_va)
goto vqs_del;
dev_dbg(&vdev->dev, "buffers: va %p, dma 0x%x\n", bufs_va,
vrp->bufs_dma);
/* half of the buffers is dedicated for RX */
vrp->rbufs = bufs_va;
/* and half is dedicated for TX */
vrp->sbufs = bufs_va + RPMSG_TOTAL_BUF_SPACE / 2;
/* set up the receive buffers */
for (i = 0; i < RPMSG_NUM_BUFS / 2; i++) {
struct scatterlist sg;
void *cpu_addr = vrp->rbufs + i * RPMSG_BUF_SIZE;
sg_init_one(&sg, cpu_addr, RPMSG_BUF_SIZE);
err = virtqueue_add_buf_gfp(vrp->rvq, &sg, 0, 1, cpu_addr,
GFP_KERNEL);
WARN_ON(err < 0); /* sanity check; this can't really happen */
}
/* suppress "tx-complete" interrupts */
virtqueue_disable_cb(vrp->svq);
vdev->priv = vrp;
/* if supported by the remote processor, enable the name service */
if (virtio_has_feature(vdev, VIRTIO_RPMSG_F_NS)) {
/* a dedicated endpoint handles the name service msgs */
vrp->ns_ept = __rpmsg_create_ept(vrp, NULL, rpmsg_ns_cb,
vrp, RPMSG_NS_ADDR);
if (!vrp->ns_ept) {
dev_err(&vdev->dev, "failed to create the ns ept\n");
err = -ENOMEM;
goto free_coherent;
}
}
/* tell the remote processor it can start sending messages */
virtqueue_kick(vrp->rvq);
dev_info(&vdev->dev, "rpmsg host is online\n");
return 0;
free_coherent:
dma_free_coherent(vdev->dev.parent, RPMSG_TOTAL_BUF_SPACE, bufs_va,
vrp->bufs_dma);
vqs_del:
vdev->config->del_vqs(vrp->vdev);
free_vrp:
kfree(vrp);
return err;
}
/**
* rpmsg_send_offchannel_raw() - send a message across to the remote processor
* @rpdev: the rpmsg channel
* @src: source address
* @dst: destination address
* @data: payload of message
* @len: length of payload
* @wait: indicates whether caller should block in case no TX buffers available
*
* This function is the base implementation for all of the rpmsg sending API.
*
* It will send @data of length @len to @dst, and say it's from @src. The
* message will be sent to the remote processor which the @rpdev channel
* belongs to.
*
* The message is sent using one of the TX buffers that are available for
* communication with this remote processor.
*
* If @wait is true, the caller will be blocked until either a TX buffer is
* available, or 15 seconds elapses (we don't want callers to
* sleep indefinitely due to misbehaving remote processors), and in that
* case -ERESTARTSYS is returned. The number '15' itself was picked
* arbitrarily; there's little point in asking drivers to provide a timeout
* value themselves.
*
* Otherwise, if @wait is false, and there are no TX buffers available,
* the function will immediately fail, and -ENOMEM will be returned.
*
* Normally drivers shouldn't use this function directly; instead, drivers
* should use the appropriate rpmsg_{try}send{to, _offchannel} API
* (see include/linux/rpmsg.h).
*
* Returns 0 on success and an appropriate error value on failure.
*/
int rpmsg_send_offchannel_raw(struct rpmsg_channel *rpdev, u32 src, u32 dst,
void *data, int len, bool wait)
{
struct virtproc_info *vrp = rpdev->vrp;
struct device *dev = &rpdev->dev;
struct scatterlist sg;
struct rpmsg_hdr *msg;
int err;
/* bcasting isn't allowed */
if (src == RPMSG_ADDR_ANY || dst == RPMSG_ADDR_ANY) {
dev_err(dev, "invalid addr (src 0x%x, dst 0x%x)\n", src, dst);
return -EINVAL;
}
/*
* We currently use fixed-sized buffers, and therefore the payload
* length is limited.
*
* One of the possible improvements here is either to support
* user-provided buffers (and then we can also support zero-copy
* messaging), or to improve the buffer allocator, to support
* variable-length buffer sizes.
*/
if (len > RPMSG_BUF_SIZE - sizeof(struct rpmsg_hdr)) {
dev_err(dev, "message is too big (%d)\n", len);
return -EMSGSIZE;
}
/* grab a buffer */
msg = get_a_tx_buf(vrp);
if (!msg && !wait)
return -ENOMEM;
/* no free buffer ? wait for one (but bail after 15 seconds) */
while (!msg) {
/* enable "tx-complete" interrupts, if not already enabled */
rpmsg_upref_sleepers(vrp);
/*
* sleep until a free buffer is available or 15 secs elapse.
* the timeout period is not configurable because there's
* little point in asking drivers to specify that.
* if later this happens to be required, it'd be easy to add.
*/
err = wait_event_interruptible_timeout(vrp->sendq,
(msg = get_a_tx_buf(vrp)),
msecs_to_jiffies(15000));
/* disable "tx-complete" interrupts if we're the last sleeper */
rpmsg_downref_sleepers(vrp);
/* timeout ? */
if (!err) {
dev_err(dev, "timeout waiting for a tx buffer\n");
return -ERESTARTSYS;
}
}
msg->len = len;
msg->flags = 0;
msg->src = src;
msg->dst = dst;
msg->reserved = 0;
memcpy(msg->data, data, len);
dev_dbg(dev, "TX From 0x%x, To 0x%x, Len %d, Flags %d, Reserved %d\n",
msg->src, msg->dst, msg->len,
msg->flags, msg->reserved);
print_hex_dump(KERN_DEBUG, "rpmsg_virtio TX: ", DUMP_PREFIX_NONE, 16, 1,
msg, sizeof(*msg) + msg->len, true);
sg_init_one(&sg, msg, sizeof(*msg) + len);
mutex_lock(&vrp->tx_lock);
/* add message to the remote processor's virtqueue */
err = virtqueue_add_buf_gfp(vrp->svq, &sg, 1, 0, msg, GFP_KERNEL);
if (err < 0) {
/*
* need to reclaim the buffer here, otherwise it's lost
* (memory won't leak, but rpmsg won't use it again for TX).
* this will wait for a buffer management overhaul.
*/
dev_err(dev, "virtqueue_add_buf_gfp failed: %d\n", err);
goto out;
}
/* tell the remote processor it has a pending message to read */
virtqueue_kick(vrp->svq);
err = 0;
out:
mutex_unlock(&vrp->tx_lock);
return err;
}
