/*
* match for a vop device with a specific desc pointer
*/
static int vop_match_desc(struct device *dev, void *data)
{
struct virtio_device *_dev = dev_to_virtio(dev);
struct _vop_vdev *vdev = to_vopvdev(_dev);
return vdev->desc == (void __iomem *)data;
}
static int vop_finalize_features(struct virtio_device *vdev)
{
unsigned int i, bits;
struct mic_device_desc __iomem *desc = to_vopvdev(vdev)->desc;
u8 feature_len = ioread8(&desc->feature_len);
/* Second half of bitmap is features we accept. */
u8 __iomem *out_features =
_vop_vq_features(desc) + feature_len;
/* Give virtio_ring a chance to accept features. */
vring_transport_features(vdev);
/* Give virtio_vop a chance to accept features. */
vop_transport_features(vdev);
memset_io(out_features, 0, feature_len);
bits = min_t(unsigned, feature_len,
sizeof(vdev->features)) * 8;
for (i = 0; i < bits; i++) {
if (__virtio_test_bit(vdev, i))
iowrite8(ioread8(&out_features[i / 8]) | (1 << (i % 8)),
&out_features[i / 8]);
}
return 0;
}
static void vop_set(struct virtio_device *vdev, unsigned int offset,
const void *buf, unsigned len)
{
struct mic_device_desc __iomem *desc = to_vopvdev(vdev)->desc;
if (offset + len > ioread8(&desc->config_len))
return;
memcpy_toio(_vop_vq_configspace(desc) + offset, buf, len);
}
static void vop_reset(struct virtio_device *dev)
{
struct _vop_vdev *vdev = to_vopvdev(dev);
dev_dbg(_vop_dev(vdev), "%s: virtio id %d\n",
__func__, dev->id.device);
vop_reset_inform_host(dev);
complete_all(&vdev->reset_done);
}
static void vop_set_status(struct virtio_device *dev, u8 status)
{
struct _vop_vdev *vdev = to_vopvdev(dev);
struct vop_device *vpdev = vdev->vpdev;
if (!status)
return;
iowrite8(status, &vdev->desc->status);
vpdev->hw_ops->send_intr(vpdev, vdev->c2h_vdev_db);
}
static void vop_del_vqs(struct virtio_device *dev)
{
struct _vop_vdev *vdev = to_vopvdev(dev);
struct virtqueue *vq, *n;
int idx = 0;
dev_dbg(_vop_dev(vdev), "%s\n", __func__);
list_for_each_entry_safe(vq, n, &dev->vqs, list)
vop_del_vq(vq, idx++);
}
static int vop_find_vqs(struct virtio_device *dev, unsigned nvqs,
struct virtqueue *vqs[],
vq_callback_t *callbacks[],
const char * const names[], const bool *ctx,
struct irq_affinity *desc)
{
struct _vop_vdev *vdev = to_vopvdev(dev);
struct vop_device *vpdev = vdev->vpdev;
struct mic_device_ctrl __iomem *dc = vdev->dc;
int i, err, retry, queue_idx = 0;
/* We must have this many virtqueues. */
if (nvqs > ioread8(&vdev->desc->num_vq))
return -ENOENT;
for (i = 0; i < nvqs; ++i) {
if (!names[i]) {
vqs[i] = NULL;
continue;
}
dev_dbg(_vop_dev(vdev), "%s: %d: %s\n",
__func__, i, names[i]);
vqs[i] = vop_find_vq(dev, queue_idx++, callbacks[i], names[i],
ctx ? ctx[i] : false);
if (IS_ERR(vqs[i])) {
err = PTR_ERR(vqs[i]);
goto error;
}
}
iowrite8(1, &dc->used_address_updated);
/*
* Send an interrupt to the host to inform it that used
* rings have been re-assigned.
*/
vpdev->hw_ops->send_intr(vpdev, vdev->c2h_vdev_db);
for (retry = 100; --retry;) {
if (!ioread8(&dc->used_address_updated))
break;
msleep(100);
}
dev_dbg(_vop_dev(vdev), "%s: retry: %d\n", __func__, retry);
if (!retry) {
err = -ENODEV;
goto error;
}
return 0;
error:
vop_del_vqs(dev);
return err;
}
static void vop_del_vq(struct virtqueue *vq, int n)
{
struct _vop_vdev *vdev = to_vopvdev(vq->vdev);
struct vring *vr = (struct vring *)(vq + 1);
struct vop_device *vpdev = vdev->vpdev;
dma_unmap_single(&vpdev->dev, vdev->used[n],
vdev->used_size[n], DMA_BIDIRECTIONAL);
free_pages((unsigned long)vr->used, get_order(vdev->used_size[n]));
vring_del_virtqueue(vq);
vpdev->hw_ops->iounmap(vpdev, vdev->vr[n]);
vdev->vr[n] = NULL;
}
/* This gets the device's feature bits. */
static u64 vop_get_features(struct virtio_device *vdev)
{
unsigned int i, bits;
u32 features = 0;
struct mic_device_desc __iomem *desc = to_vopvdev(vdev)->desc;
u8 __iomem *in_features = _vop_vq_features(desc);
int feature_len = ioread8(&desc->feature_len);
bits = min_t(unsigned, feature_len, sizeof(vdev->features)) * 8;
for (i = 0; i < bits; i++)
if (ioread8(&in_features[i / 8]) & (BIT(i % 8)))
features |= BIT(i);
return features;
}
/* Inform host on a virtio device reset and wait for ack from host */
static void vop_reset_inform_host(struct virtio_device *dev)
{
struct _vop_vdev *vdev = to_vopvdev(dev);
struct mic_device_ctrl __iomem *dc = vdev->dc;
struct vop_device *vpdev = vdev->vpdev;
int retry;
iowrite8(0, &dc->host_ack);
iowrite8(1, &dc->vdev_reset);
vpdev->hw_ops->send_intr(vpdev, vdev->c2h_vdev_db);
/* Wait till host completes all card accesses and acks the reset */
for (retry = 100; retry--;) {
if (ioread8(&dc->host_ack))
break;
msleep(100);
};
dev_dbg(_vop_dev(vdev), "%s: retry: %d\n", __func__, retry);
/* Reset status to 0 in case we timed out */
iowrite8(0, &vdev->desc->status);
}
/*
* This routine will assign vring's allocated in host/io memory. Code in
* virtio_ring.c however continues to access this io memory as if it were local
* memory without io accessors.
*/
static struct virtqueue *vop_find_vq(struct virtio_device *dev,
unsigned index,
void (*callback)(struct virtqueue *vq),
const char *name)
{
struct _vop_vdev *vdev = to_vopvdev(dev);
struct vop_device *vpdev = vdev->vpdev;
struct mic_vqconfig __iomem *vqconfig;
struct mic_vqconfig config;
struct virtqueue *vq;
void __iomem *va;
struct _mic_vring_info __iomem *info;
void *used;
int vr_size, _vr_size, err, magic;
struct vring *vr;
u8 type = ioread8(&vdev->desc->type);
if (index >= ioread8(&vdev->desc->num_vq))
return ERR_PTR(-ENOENT);
if (!name)
return ERR_PTR(-ENOENT);
/* First assign the vring's allocated in host memory */
vqconfig = _vop_vq_config(vdev->desc) + index;
memcpy_fromio(&config, vqconfig, sizeof(config));
_vr_size = vring_size(le16_to_cpu(config.num), MIC_VIRTIO_RING_ALIGN);
vr_size = PAGE_ALIGN(_vr_size + sizeof(struct _mic_vring_info));
va = vpdev->hw_ops->ioremap(vpdev, le64_to_cpu(config.address),
vr_size);
if (!va)
return ERR_PTR(-ENOMEM);
vdev->vr[index] = va;
memset_io(va, 0x0, _vr_size);
vq = vring_new_virtqueue(
index,
le16_to_cpu(config.num), MIC_VIRTIO_RING_ALIGN,
dev,
false,
(void __force *)va, vop_notify, callback, name);
if (!vq) {
err = -ENOMEM;
goto unmap;
}
info = va + _vr_size;
magic = ioread32(&info->magic);
if (WARN(magic != MIC_MAGIC + type + index, "magic mismatch")) {
err = -EIO;
goto unmap;
}
/* Allocate and reassign used ring now */
vdev->used_size[index] = PAGE_ALIGN(sizeof(__u16) * 3 +
sizeof(struct vring_used_elem) *
le16_to_cpu(config.num));
used = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
get_order(vdev->used_size[index]));
if (!used) {
err = -ENOMEM;
dev_err(_vop_dev(vdev), "%s %d err %d\n",
__func__, __LINE__, err);
goto del_vq;
}
vdev->used[index] = dma_map_single(&vpdev->dev, used,
vdev->used_size[index],
DMA_BIDIRECTIONAL);
if (dma_mapping_error(&vpdev->dev, vdev->used[index])) {
err = -ENOMEM;
dev_err(_vop_dev(vdev), "%s %d err %d\n",
__func__, __LINE__, err);
goto free_used;
}
writeq(vdev->used[index], &vqconfig->used_address);
/*
* To reassign the used ring here we are directly accessing
* struct vring_virtqueue which is a private data structure
* in virtio_ring.c. At the minimum, a BUILD_BUG_ON() in
* vring_new_virtqueue() would ensure that
* (&vq->vring == (struct vring *) (&vq->vq + 1));
*/
vr = (struct vring *)(vq + 1);
vr->used = used;
vq->priv = vdev;
return vq;
free_used:
free_pages((unsigned long)used,
get_order(vdev->used_size[index]));
del_vq:
vring_del_virtqueue(vq);
unmap:
vpdev->hw_ops->iounmap(vpdev, vdev->vr[index]);
return ERR_PTR(err);
}
/*
* The operations to get and set the status word just access the status
* field of the device descriptor. set_status also interrupts the host
* to tell about status changes.
*/
static u8 vop_get_status(struct virtio_device *vdev)
{
return ioread8(&to_vopvdev(vdev)->desc->status);
}