[deliverable/linux.git] / drivers / virtio / virtio_mmio.c

/*
 * Virtio memory mapped device driver
 *
 * Copyright 2011, ARM Ltd.
 *
 * This module allows virtio devices to be used over a virtual, memory mapped
 * platform device.
 *
 * The guest device(s) may be instantiated in one of three equivalent ways:
 *
 * 1. Static platform device in board's code, eg.:
 *
 *	static struct platform_device v2m_virtio_device = {
 *		.name = "virtio-mmio",
 *		.id = -1,
 *		.num_resources = 2,
 *		.resource = (struct resource []) {
 *			{
 *				.start = 0x1001e000,
 *				.end = 0x1001e0ff,
 *				.flags = IORESOURCE_MEM,
 *			}, {
 *				.start = 42 + 32,
 *				.end = 42 + 32,
 *				.flags = IORESOURCE_IRQ,
 *			},
 *		}
 *	};
 *
 * 2. Device Tree node, eg.:
 *
 *		virtio_block@1e000 {
 *			compatible = "virtio,mmio";
 *			reg = <0x1e000 0x100>;
 *			interrupts = <42>;
 *		}
 *
 * 3. Kernel module (or command line) parameter. Can be used more than once -
 *    one device will be created for each one. Syntax:
 *
 *		[virtio_mmio.]device=<size>@<baseaddr>:<irq>[:<id>]
 *    where:
 *		<size>     := size (can use standard suffixes like K, M or G)
 *		<baseaddr> := physical base address
 *		<irq>      := interrupt number (as passed to request_irq())
 *		<id>       := (optional) platform device id
 *    eg.:
 *		virtio_mmio.device=0x100@0x100b0000:48 \
 *				virtio_mmio.device=1K@0x1001e000:74
 *
 *
 *
 * Registers layout (all 32-bit wide):
 *
 * offset d. name             description
 * ------ -- ---------------- -----------------
 *
 * 0x000  R  MagicValue       Magic value "virt"
 * 0x004  R  Version          Device version (current max. 1)
 * 0x008  R  DeviceID         Virtio device ID
 * 0x00c  R  VendorID         Virtio vendor ID
 *
 * 0x010  R  HostFeatures     Features supported by the host
 * 0x014  W  HostFeaturesSel  Set of host features to access via HostFeatures
 *
 * 0x020  W  GuestFeatures    Features activated by the guest
 * 0x024  W  GuestFeaturesSel Set of activated features to set via GuestFeatures
 * 0x028  W  GuestPageSize    Size of guest's memory page in bytes
 *
 * 0x030  W  QueueSel         Queue selector
 * 0x034  R  QueueNumMax      Maximum size of the currently selected queue
 * 0x038  W  QueueNum         Queue size for the currently selected queue
 * 0x03c  W  QueueAlign       Used Ring alignment for the current queue
 * 0x040  RW QueuePFN         PFN for the currently selected queue
 *
 * 0x050  W  QueueNotify      Queue notifier
 * 0x060  R  InterruptStatus  Interrupt status register
 * 0x064  W  InterruptACK     Interrupt acknowledge register
 * 0x070  RW Status           Device status register
 *
 * 0x100+ RW                  Device-specific configuration space
 *
 * Based on Virtio PCI driver by Anthony Liguori, copyright IBM Corp. 2007
 *
 * This work is licensed under the terms of the GNU GPL, version 2 or later.
 * See the COPYING file in the top-level directory.
 */

#define pr_fmt(fmt) "virtio-mmio: " fmt

#include <linux/highmem.h>
#include <linux/interrupt.h>
#include <linux/io.h>
#include <linux/list.h>
#include <linux/module.h>
#include <linux/platform_device.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <linux/virtio.h>
#include <linux/virtio_config.h>
#include <linux/virtio_mmio.h>
#include <linux/virtio_ring.h>


/* The alignment to use between consumer and producer parts of vring.
 * Currently hardcoded to the page size. */
#define VIRTIO_MMIO_VRING_ALIGN		PAGE_SIZE


#define to_virtio_mmio_device(_plat_dev) \
	container_of(_plat_dev, struct virtio_mmio_device, vdev)

struct virtio_mmio_device {
	struct virtio_device vdev;
	struct platform_device *pdev;

	void __iomem *base;
	unsigned long version;

	/* a list of queues so we can dispatch IRQs */
	spinlock_t lock;
	struct list_head virtqueues;
};

struct virtio_mmio_vq_info {
	/* the actual virtqueue */
	struct virtqueue *vq;

	/* the number of entries in the queue */
	unsigned int num;

	/* the virtual address of the ring queue */
	void *queue;

	/* the list node for the virtqueues list */
	struct list_head node;
};


/* Configuration interface */

static u32 vm_get_features(struct virtio_device *vdev)
{
	struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);

	/* TODO: Features > 32 bits */
	writel(0, vm_dev->base + VIRTIO_MMIO_HOST_FEATURES_SEL);

	return readl(vm_dev->base + VIRTIO_MMIO_HOST_FEATURES);
}

static void vm_finalize_features(struct virtio_device *vdev)
{
	struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
	int i;

	/* Give virtio_ring a chance to accept features. */
	vring_transport_features(vdev);

	for (i = 0; i < ARRAY_SIZE(vdev->features); i++) {
		writel(i, vm_dev->base + VIRTIO_MMIO_GUEST_FEATURES_SEL);
		writel(vdev->features[i],
				vm_dev->base + VIRTIO_MMIO_GUEST_FEATURES);
	}
}

static void vm_get(struct virtio_device *vdev, unsigned offset,
		   void *buf, unsigned len)
{
	struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
	u8 *ptr = buf;
	int i;

	for (i = 0; i < len; i++)
		ptr[i] = readb(vm_dev->base + VIRTIO_MMIO_CONFIG + offset + i);
}

static void vm_set(struct virtio_device *vdev, unsigned offset,
		   const void *buf, unsigned len)
{
	struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
	const u8 *ptr = buf;
	int i;

	for (i = 0; i < len; i++)
		writeb(ptr[i], vm_dev->base + VIRTIO_MMIO_CONFIG + offset + i);
}

static u8 vm_get_status(struct virtio_device *vdev)
{
	struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);

	return readl(vm_dev->base + VIRTIO_MMIO_STATUS) & 0xff;
}

static void vm_set_status(struct virtio_device *vdev, u8 status)
{
	struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);

	/* We should never be setting status to 0. */
	BUG_ON(status == 0);

	writel(status, vm_dev->base + VIRTIO_MMIO_STATUS);
}

static void vm_reset(struct virtio_device *vdev)
{
	struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);

	/* 0 status means a reset. */
	writel(0, vm_dev->base + VIRTIO_MMIO_STATUS);
}


/* Transport interface */

/* the notify function used when creating a virt queue */
static bool vm_notify(struct virtqueue *vq)
{
	struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vq->vdev);

	/* We write the queue's selector into the notification register to
	 * signal the other end */
	writel(vq->index, vm_dev->base + VIRTIO_MMIO_QUEUE_NOTIFY);
	return true;
}

/* Notify all virtqueues on an interrupt. */
static irqreturn_t vm_interrupt(int irq, void *opaque)
{
	struct virtio_mmio_device *vm_dev = opaque;
	struct virtio_mmio_vq_info *info;
	struct virtio_driver *vdrv = container_of(vm_dev->vdev.dev.driver,
			struct virtio_driver, driver);
	unsigned long status;
	unsigned long flags;
	irqreturn_t ret = IRQ_NONE;

	/* Read and acknowledge interrupts */
	status = readl(vm_dev->base + VIRTIO_MMIO_INTERRUPT_STATUS);
	writel(status, vm_dev->base + VIRTIO_MMIO_INTERRUPT_ACK);

	if (unlikely(status & VIRTIO_MMIO_INT_CONFIG)
			&& vdrv && vdrv->config_changed) {
		vdrv->config_changed(&vm_dev->vdev);
		ret = IRQ_HANDLED;
	}

	if (likely(status & VIRTIO_MMIO_INT_VRING)) {
		spin_lock_irqsave(&vm_dev->lock, flags);
		list_for_each_entry(info, &vm_dev->virtqueues, node)
			ret |= vring_interrupt(irq, info->vq);
		spin_unlock_irqrestore(&vm_dev->lock, flags);
	}

	return ret;
}


static void vm_del_vq(struct virtqueue *vq)
{
	struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vq->vdev);
	struct virtio_mmio_vq_info *info = vq->priv;
	unsigned long flags, size;
	unsigned int index = vq->index;

	spin_lock_irqsave(&vm_dev->lock, flags);
	list_del(&info->node);
	spin_unlock_irqrestore(&vm_dev->lock, flags);

	vring_del_virtqueue(vq);

	/* Select and deactivate the queue */
	writel(index, vm_dev->base + VIRTIO_MMIO_QUEUE_SEL);
	writel(0, vm_dev->base + VIRTIO_MMIO_QUEUE_PFN);

	size = PAGE_ALIGN(vring_size(info->num, VIRTIO_MMIO_VRING_ALIGN));
	free_pages_exact(info->queue, size);
	kfree(info);
}

static void vm_del_vqs(struct virtio_device *vdev)
{
	struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
	struct virtqueue *vq, *n;

	list_for_each_entry_safe(vq, n, &vdev->vqs, list)
		vm_del_vq(vq);

	free_irq(platform_get_irq(vm_dev->pdev, 0), vm_dev);
}


static struct virtqueue *vm_setup_vq(struct virtio_device *vdev, unsigned index,
				  void (*callback)(struct virtqueue *vq),
				  const char *name)
{
	struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
	struct virtio_mmio_vq_info *info;
	struct virtqueue *vq;
	unsigned long flags, size;
	int err;

	if (!name)
		return NULL;

	/* Select the queue we're interested in */
	writel(index, vm_dev->base + VIRTIO_MMIO_QUEUE_SEL);

	/* Queue shouldn't already be set up. */
	if (readl(vm_dev->base + VIRTIO_MMIO_QUEUE_PFN)) {
		err = -ENOENT;
		goto error_available;
	}

	/* Allocate and fill out our active queue description */
	info = kmalloc(sizeof(*info), GFP_KERNEL);
	if (!info) {
		err = -ENOMEM;
		goto error_kmalloc;
	}

	/* Allocate pages for the queue - start with a queue as big as
	 * possible (limited by maximum size allowed by device), drop down
	 * to a minimal size, just big enough to fit descriptor table
	 * and two rings (which makes it "alignment_size * 2")
	 */
	info->num = readl(vm_dev->base + VIRTIO_MMIO_QUEUE_NUM_MAX);

	/* If the device reports a 0 entry queue, we won't be able to
	 * use it to perform I/O, and vring_new_virtqueue() can't create
	 * empty queues anyway, so don't bother to set up the device.
	 */
	if (info->num == 0) {
		err = -ENOENT;
		goto error_alloc_pages;
	}

	while (1) {
		size = PAGE_ALIGN(vring_size(info->num,
				VIRTIO_MMIO_VRING_ALIGN));
		/* Did the last iter shrink the queue below minimum size? */
		if (size < VIRTIO_MMIO_VRING_ALIGN * 2) {
			err = -ENOMEM;
			goto error_alloc_pages;
		}

		info->queue = alloc_pages_exact(size, GFP_KERNEL | __GFP_ZERO);
		if (info->queue)
			break;

		info->num /= 2;
	}

	/* Activate the queue */
	writel(info->num, vm_dev->base + VIRTIO_MMIO_QUEUE_NUM);
	writel(VIRTIO_MMIO_VRING_ALIGN,
			vm_dev->base + VIRTIO_MMIO_QUEUE_ALIGN);
	writel(virt_to_phys(info->queue) >> PAGE_SHIFT,
			vm_dev->base + VIRTIO_MMIO_QUEUE_PFN);

	/* Create the vring */
	vq = vring_new_virtqueue(index, info->num, VIRTIO_MMIO_VRING_ALIGN, vdev,
				 true, info->queue, vm_notify, callback, name);
	if (!vq) {
		err = -ENOMEM;
		goto error_new_virtqueue;
	}

	vq->priv = info;
	info->vq = vq;

	spin_lock_irqsave(&vm_dev->lock, flags);
	list_add(&info->node, &vm_dev->virtqueues);
	spin_unlock_irqrestore(&vm_dev->lock, flags);

	return vq;

error_new_virtqueue:
	writel(0, vm_dev->base + VIRTIO_MMIO_QUEUE_PFN);
	free_pages_exact(info->queue, size);
error_alloc_pages:
	kfree(info);
error_kmalloc:
error_available:
	return ERR_PTR(err);
}

static int vm_find_vqs(struct virtio_device *vdev, unsigned nvqs,
		       struct virtqueue *vqs[],
		       vq_callback_t *callbacks[],
		       const char *names[])
{
	struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
	unsigned int irq = platform_get_irq(vm_dev->pdev, 0);
	int i, err;

	err = request_irq(irq, vm_interrupt, IRQF_SHARED,
			dev_name(&vdev->dev), vm_dev);
	if (err)
		return err;

	for (i = 0; i < nvqs; ++i) {
		vqs[i] = vm_setup_vq(vdev, i, callbacks[i], names[i]);
		if (IS_ERR(vqs[i])) {
			vm_del_vqs(vdev);
			return PTR_ERR(vqs[i]);
		}
	}

	return 0;
}

static const char *vm_bus_name(struct virtio_device *vdev)
{
	struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);

	return vm_dev->pdev->name;
}

static const struct virtio_config_ops virtio_mmio_config_ops = {
	.get		= vm_get,
	.set		= vm_set,
	.get_status	= vm_get_status,
	.set_status	= vm_set_status,
	.reset		= vm_reset,
	.find_vqs	= vm_find_vqs,
	.del_vqs	= vm_del_vqs,
	.get_features	= vm_get_features,
	.finalize_features = vm_finalize_features,
	.bus_name	= vm_bus_name,
};


/* Platform device */

static int virtio_mmio_probe(struct platform_device *pdev)
{
	struct virtio_mmio_device *vm_dev;
	struct resource *mem;
	unsigned long magic;

	mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
	if (!mem)
		return -EINVAL;

	if (!devm_request_mem_region(&pdev->dev, mem->start,
			resource_size(mem), pdev->name))
		return -EBUSY;

	vm_dev = devm_kzalloc(&pdev->dev, sizeof(*vm_dev), GFP_KERNEL);
	if (!vm_dev)
		return  -ENOMEM;

	vm_dev->vdev.dev.parent = &pdev->dev;
	vm_dev->vdev.config = &virtio_mmio_config_ops;
	vm_dev->pdev = pdev;
	INIT_LIST_HEAD(&vm_dev->virtqueues);
	spin_lock_init(&vm_dev->lock);

	vm_dev->base = devm_ioremap(&pdev->dev, mem->start, resource_size(mem));
	if (vm_dev->base == NULL)
		return -EFAULT;

	/* Check magic value */
	magic = readl(vm_dev->base + VIRTIO_MMIO_MAGIC_VALUE);
	if (magic != ('v' | 'i' << 8 | 'r' << 16 | 't' << 24)) {
		dev_warn(&pdev->dev, "Wrong magic value 0x%08lx!\n", magic);
		return -ENODEV;
	}

	/* Check device version */
	vm_dev->version = readl(vm_dev->base + VIRTIO_MMIO_VERSION);
	if (vm_dev->version != 1) {
		dev_err(&pdev->dev, "Version %ld not supported!\n",
				vm_dev->version);
		return -ENXIO;
	}

	vm_dev->vdev.id.device = readl(vm_dev->base + VIRTIO_MMIO_DEVICE_ID);
	vm_dev->vdev.id.vendor = readl(vm_dev->base + VIRTIO_MMIO_VENDOR_ID);

	writel(PAGE_SIZE, vm_dev->base + VIRTIO_MMIO_GUEST_PAGE_SIZE);

	platform_set_drvdata(pdev, vm_dev);

	return register_virtio_device(&vm_dev->vdev);
}

static int virtio_mmio_remove(struct platform_device *pdev)
{
	struct virtio_mmio_device *vm_dev = platform_get_drvdata(pdev);

	unregister_virtio_device(&vm_dev->vdev);

	return 0;
}


/* Devices list parameter */

#if defined(CONFIG_VIRTIO_MMIO_CMDLINE_DEVICES)

static struct device vm_cmdline_parent = {
	.init_name = "virtio-mmio-cmdline",
};

static int vm_cmdline_parent_registered;
static int vm_cmdline_id;

static int vm_cmdline_set(const char *device,
		const struct kernel_param *kp)
{
	int err;
	struct resource resources[2] = {};
	char *str;
	long long int base, size;
	unsigned int irq;
	int processed, consumed = 0;
	struct platform_device *pdev;

	/* Consume "size" part of the command line parameter */
	size = memparse(device, &str);

	/* Get "@<base>:<irq>[:<id>]" chunks */
	processed = sscanf(str, "@%lli:%u%n:%d%n",
			&base, &irq, &consumed,
			&vm_cmdline_id, &consumed);

	/*
	 * sscanf() must processes at least 2 chunks; also there
	 * must be no extra characters after the last chunk, so
	 * str[consumed] must be '\0'
	 */
	if (processed < 2 || str[consumed])
		return -EINVAL;

	resources[0].flags = IORESOURCE_MEM;
	resources[0].start = base;
	resources[0].end = base + size - 1;

	resources[1].flags = IORESOURCE_IRQ;
	resources[1].start = resources[1].end = irq;

	if (!vm_cmdline_parent_registered) {
		err = device_register(&vm_cmdline_parent);
		if (err) {
			pr_err("Failed to register parent device!\n");
			return err;
		}
		vm_cmdline_parent_registered = 1;
	}

	pr_info("Registering device virtio-mmio.%d at 0x%llx-0x%llx, IRQ %d.\n",
		       vm_cmdline_id,
		       (unsigned long long)resources[0].start,
		       (unsigned long long)resources[0].end,
		       (int)resources[1].start);

	pdev = platform_device_register_resndata(&vm_cmdline_parent,
			"virtio-mmio", vm_cmdline_id++,
			resources, ARRAY_SIZE(resources), NULL, 0);
	if (IS_ERR(pdev))
		return PTR_ERR(pdev);

	return 0;
}

static int vm_cmdline_get_device(struct device *dev, void *data)
{
	char *buffer = data;
	unsigned int len = strlen(buffer);
	struct platform_device *pdev = to_platform_device(dev);

	snprintf(buffer + len, PAGE_SIZE - len, "0x%llx@0x%llx:%llu:%d\n",
			pdev->resource[0].end - pdev->resource[0].start + 1ULL,
			(unsigned long long)pdev->resource[0].start,
			(unsigned long long)pdev->resource[1].start,
			pdev->id);
	return 0;
}

static int vm_cmdline_get(char *buffer, const struct kernel_param *kp)
{
	buffer[0] = '\0';
	device_for_each_child(&vm_cmdline_parent, buffer,
			vm_cmdline_get_device);
	return strlen(buffer) + 1;
}

static struct kernel_param_ops vm_cmdline_param_ops = {
	.set = vm_cmdline_set,
	.get = vm_cmdline_get,
};

device_param_cb(device, &vm_cmdline_param_ops, NULL, S_IRUSR);

static int vm_unregister_cmdline_device(struct device *dev,
		void *data)
{
	platform_device_unregister(to_platform_device(dev));

	return 0;
}

static void vm_unregister_cmdline_devices(void)
{
	if (vm_cmdline_parent_registered) {
		device_for_each_child(&vm_cmdline_parent, NULL,
				vm_unregister_cmdline_device);
		device_unregister(&vm_cmdline_parent);
		vm_cmdline_parent_registered = 0;
	}
}

#else

static void vm_unregister_cmdline_devices(void)
{
}

#endif

/* Platform driver */

static struct of_device_id virtio_mmio_match[] = {
	{ .compatible = "virtio,mmio", },
	{},
};
MODULE_DEVICE_TABLE(of, virtio_mmio_match);

static struct platform_driver virtio_mmio_driver = {
	.probe		= virtio_mmio_probe,
	.remove		= virtio_mmio_remove,
	.driver		= {
		.name	= "virtio-mmio",
		.owner	= THIS_MODULE,
		.of_match_table	= virtio_mmio_match,
	},
};

static int __init virtio_mmio_init(void)
{
	return platform_driver_register(&virtio_mmio_driver);
}

static void __exit virtio_mmio_exit(void)
{
	platform_driver_unregister(&virtio_mmio_driver);
	vm_unregister_cmdline_devices();
}

module_init(virtio_mmio_init);
module_exit(virtio_mmio_exit);

MODULE_AUTHOR("Pawel Moll <pawel.moll@arm.com>");
MODULE_DESCRIPTION("Platform bus driver for memory mapped virtio devices");
MODULE_LICENSE("GPL");
Commit	Line	Data
edfd52e6 PM	1	/*
	2	* Virtio memory mapped device driver
	3	*
	4	* Copyright 2011, ARM Ltd.
	5	*
	6	* This module allows virtio devices to be used over a virtual, memory mapped
	7	* platform device.
	8	*
81a054ce PM	9	* The guest device(s) may be instantiated in one of three equivalent ways:
	10	*
	11	* 1. Static platform device in board's code, eg.:
	12	*
	13	* static struct platform_device v2m_virtio_device = {
	14	* .name = "virtio-mmio",
	15	* .id = -1,
	16	* .num_resources = 2,
	17	* .resource = (struct resource []) {
	18	* {
	19	* .start = 0x1001e000,
	20	* .end = 0x1001e0ff,
	21	* .flags = IORESOURCE_MEM,
	22	* }, {
	23	* .start = 42 + 32,
	24	* .end = 42 + 32,
	25	* .flags = IORESOURCE_IRQ,
	26	* },
	27	* }
	28	* };
	29	*
	30	* 2. Device Tree node, eg.:
	31	*
	32	* virtio_block@1e000 {
	33	* compatible = "virtio,mmio";
	34	* reg = <0x1e000 0x100>;
	35	* interrupts = <42>;
	36	* }
	37	*
	38	* 3. Kernel module (or command line) parameter. Can be used more than once -
	39	* one device will be created for each one. Syntax:
	40	*
	41	* [virtio_mmio.]device=<size>@<baseaddr>:<irq>[:<id>]
	42	* where:
	43	* <size> := size (can use standard suffixes like K, M or G)
	44	* <baseaddr> := physical base address
	45	* <irq> := interrupt number (as passed to request_irq())
	46	* <id> := (optional) platform device id
	47	* eg.:
	48	* virtio_mmio.device=0x100@0x100b0000:48 \
	49	* virtio_mmio.device=1K@0x1001e000:74
	50	*
	51	*
	52	*
edfd52e6 PM	53	* Registers layout (all 32-bit wide):
	54	*
	55	* offset d. name description
	56	* ------ -- ---------------- -----------------
	57	*
	58	* 0x000 R MagicValue Magic value "virt"
	59	* 0x004 R Version Device version (current max. 1)
	60	* 0x008 R DeviceID Virtio device ID
	61	* 0x00c R VendorID Virtio vendor ID
	62	*
	63	* 0x010 R HostFeatures Features supported by the host
	64	* 0x014 W HostFeaturesSel Set of host features to access via HostFeatures
	65	*
	66	* 0x020 W GuestFeatures Features activated by the guest
	67	* 0x024 W GuestFeaturesSel Set of activated features to set via GuestFeatures
	68	* 0x028 W GuestPageSize Size of guest's memory page in bytes
	69	*
	70	* 0x030 W QueueSel Queue selector
	71	* 0x034 R QueueNumMax Maximum size of the currently selected queue
	72	* 0x038 W QueueNum Queue size for the currently selected queue
	73	* 0x03c W QueueAlign Used Ring alignment for the current queue
	74	* 0x040 RW QueuePFN PFN for the currently selected queue
	75	*
	76	* 0x050 W QueueNotify Queue notifier
	77	* 0x060 R InterruptStatus Interrupt status register
0d34cc2d	78	* 0x064 W InterruptACK Interrupt acknowledge register
edfd52e6 PM	79	* 0x070 RW Status Device status register
	80	*
	81	* 0x100+ RW Device-specific configuration space
	82	*
	83	* Based on Virtio PCI driver by Anthony Liguori, copyright IBM Corp. 2007
	84	*
	85	* This work is licensed under the terms of the GNU GPL, version 2 or later.
	86	* See the COPYING file in the top-level directory.
	87	*/
	88
81a054ce PM	89	#define pr_fmt(fmt) "virtio-mmio: " fmt
81a054ce PM	90
edfd52e6 PM	91	#include <linux/highmem.h>
	92	#include <linux/interrupt.h>
	93	#include <linux/io.h>
	94	#include <linux/list.h>
	95	#include <linux/module.h>
	96	#include <linux/platform_device.h>
	97	#include <linux/slab.h>
	98	#include <linux/spinlock.h>
	99	#include <linux/virtio.h>
	100	#include <linux/virtio_config.h>
	101	#include <linux/virtio_mmio.h>
	102	#include <linux/virtio_ring.h>
	103
	104
	105
	106	/* The alignment to use between consumer and producer parts of vring.
	107	* Currently hardcoded to the page size. */
	108	#define VIRTIO_MMIO_VRING_ALIGN PAGE_SIZE
	109
	110
	111
	112	#define to_virtio_mmio_device(_plat_dev) \
	113	container_of(_plat_dev, struct virtio_mmio_device, vdev)
	114
	115	struct virtio_mmio_device {
	116	struct virtio_device vdev;
	117	struct platform_device *pdev;
	118
	119	void __iomem *base;
	120	unsigned long version;
	121
	122	/* a list of queues so we can dispatch IRQs */
	123	spinlock_t lock;
	124	struct list_head virtqueues;
	125	};
	126
	127	struct virtio_mmio_vq_info {
	128	/* the actual virtqueue */
	129	struct virtqueue *vq;
	130
	131	/* the number of entries in the queue */
	132	unsigned int num;
	133
edfd52e6 PM	134	/* the virtual address of the ring queue */
	135	void *queue;
	136
	137	/* the list node for the virtqueues list */
	138	struct list_head node;
	139	};
	140
	141
	142
	143	/* Configuration interface */
	144
	145	static u32 vm_get_features(struct virtio_device *vdev)
	146	{
	147	struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
	148
	149	/* TODO: Features > 32 bits */
	150	writel(0, vm_dev->base + VIRTIO_MMIO_HOST_FEATURES_SEL);
	151
	152	return readl(vm_dev->base + VIRTIO_MMIO_HOST_FEATURES);
	153	}
	154
	155	static void vm_finalize_features(struct virtio_device *vdev)
	156	{
	157	struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
	158	int i;
	159
	160	/* Give virtio_ring a chance to accept features. */
	161	vring_transport_features(vdev);
	162
	163	for (i = 0; i < ARRAY_SIZE(vdev->features); i++) {
fe1a7fe2	164	writel(i, vm_dev->base + VIRTIO_MMIO_GUEST_FEATURES_SEL);
edfd52e6 PM	165	writel(vdev->features[i],
	166	vm_dev->base + VIRTIO_MMIO_GUEST_FEATURES);
	167	}
	168	}
	169
	170	static void vm_get(struct virtio_device *vdev, unsigned offset,
	171	void *buf, unsigned len)
	172	{
	173	struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
	174	u8 *ptr = buf;
	175	int i;
	176
	177	for (i = 0; i < len; i++)
	178	ptr[i] = readb(vm_dev->base + VIRTIO_MMIO_CONFIG + offset + i);
	179	}
	180
	181	static void vm_set(struct virtio_device *vdev, unsigned offset,
	182	const void *buf, unsigned len)
	183	{
	184	struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
	185	const u8 *ptr = buf;
	186	int i;
	187
	188	for (i = 0; i < len; i++)
	189	writeb(ptr[i], vm_dev->base + VIRTIO_MMIO_CONFIG + offset + i);
	190	}
	191
	192	static u8 vm_get_status(struct virtio_device *vdev)
	193	{
	194	struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
	195
	196	return readl(vm_dev->base + VIRTIO_MMIO_STATUS) & 0xff;
	197	}
	198
	199	static void vm_set_status(struct virtio_device *vdev, u8 status)
	200	{
	201	struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
	202
	203	/* We should never be setting status to 0. */
	204	BUG_ON(status == 0);
	205
	206	writel(status, vm_dev->base + VIRTIO_MMIO_STATUS);
	207	}
	208
	209	static void vm_reset(struct virtio_device *vdev)
	210	{
	211	struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
	212
	213	/* 0 status means a reset. */
	214	writel(0, vm_dev->base + VIRTIO_MMIO_STATUS);
	215	}
	216
	217
	218
	219	/* Transport interface */
	220
	221	/* the notify function used when creating a virt queue */
46f9c2b9	222	static bool vm_notify(struct virtqueue *vq)
edfd52e6 PM	223	{
edfd52e6 PM	224	struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vq->vdev);
edfd52e6 PM	225
	226	/* We write the queue's selector into the notification register to
	227	* signal the other end */
06ca287d	228	writel(vq->index, vm_dev->base + VIRTIO_MMIO_QUEUE_NOTIFY);
46f9c2b9	229	return true;
edfd52e6 PM	230	}
	231
	232	/* Notify all virtqueues on an interrupt. */
	233	static irqreturn_t vm_interrupt(int irq, void *opaque)
	234	{
	235	struct virtio_mmio_device *vm_dev = opaque;
	236	struct virtio_mmio_vq_info *info;
	237	struct virtio_driver *vdrv = container_of(vm_dev->vdev.dev.driver,
	238	struct virtio_driver, driver);
	239	unsigned long status;
	240	unsigned long flags;
	241	irqreturn_t ret = IRQ_NONE;
	242
	243	/* Read and acknowledge interrupts */
	244	status = readl(vm_dev->base + VIRTIO_MMIO_INTERRUPT_STATUS);
	245	writel(status, vm_dev->base + VIRTIO_MMIO_INTERRUPT_ACK);
	246
	247	if (unlikely(status & VIRTIO_MMIO_INT_CONFIG)
	248	&& vdrv && vdrv->config_changed) {
	249	vdrv->config_changed(&vm_dev->vdev);
	250	ret = IRQ_HANDLED;
	251	}
	252
	253	if (likely(status & VIRTIO_MMIO_INT_VRING)) {
	254	spin_lock_irqsave(&vm_dev->lock, flags);
	255	list_for_each_entry(info, &vm_dev->virtqueues, node)
	256	ret \|= vring_interrupt(irq, info->vq);
	257	spin_unlock_irqrestore(&vm_dev->lock, flags);
	258	}
	259
	260	return ret;
	261	}
	262
	263
	264
	265	static void vm_del_vq(struct virtqueue *vq)
	266	{
	267	struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vq->vdev);
	268	struct virtio_mmio_vq_info *info = vq->priv;
	269	unsigned long flags, size;
06ca287d	270	unsigned int index = vq->index;
edfd52e6 PM	271
	272	spin_lock_irqsave(&vm_dev->lock, flags);
	273	list_del(&info->node);
	274	spin_unlock_irqrestore(&vm_dev->lock, flags);
	275
	276	vring_del_virtqueue(vq);
	277
	278	/* Select and deactivate the queue */
17bb6d40	279	writel(index, vm_dev->base + VIRTIO_MMIO_QUEUE_SEL);
edfd52e6 PM	280	writel(0, vm_dev->base + VIRTIO_MMIO_QUEUE_PFN);
	281
	282	size = PAGE_ALIGN(vring_size(info->num, VIRTIO_MMIO_VRING_ALIGN));
	283	free_pages_exact(info->queue, size);
	284	kfree(info);
	285	}
	286
	287	static void vm_del_vqs(struct virtio_device *vdev)
	288	{
	289	struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
	290	struct virtqueue vq, n;
	291
	292	list_for_each_entry_safe(vq, n, &vdev->vqs, list)
	293	vm_del_vq(vq);
	294
	295	free_irq(platform_get_irq(vm_dev->pdev, 0), vm_dev);
	296	}
	297
	298
	299
	300	static struct virtqueue vm_setup_vq(struct virtio_device vdev, unsigned index,
	301	void (callback)(struct virtqueue vq),
	302	const char *name)
	303	{
	304	struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
	305	struct virtio_mmio_vq_info *info;
	306	struct virtqueue *vq;
	307	unsigned long flags, size;
	308	int err;
	309
6457f126 MT	310	if (!name)
	311	return NULL;
	312
edfd52e6 PM	313	/* Select the queue we're interested in */
	314	writel(index, vm_dev->base + VIRTIO_MMIO_QUEUE_SEL);
	315
	316	/* Queue shouldn't already be set up. */
	317	if (readl(vm_dev->base + VIRTIO_MMIO_QUEUE_PFN)) {
	318	err = -ENOENT;
	319	goto error_available;
	320	}
	321
	322	/* Allocate and fill out our active queue description */
	323	info = kmalloc(sizeof(*info), GFP_KERNEL);
	324	if (!info) {
	325	err = -ENOMEM;
	326	goto error_kmalloc;
	327	}
edfd52e6 PM	328
	329	/* Allocate pages for the queue - start with a queue as big as
	330	* possible (limited by maximum size allowed by device), drop down
	331	* to a minimal size, just big enough to fit descriptor table
	332	* and two rings (which makes it "alignment_size * 2")
	333	*/
	334	info->num = readl(vm_dev->base + VIRTIO_MMIO_QUEUE_NUM_MAX);
d78b519f BF	335
	336	/* If the device reports a 0 entry queue, we won't be able to
	337	* use it to perform I/O, and vring_new_virtqueue() can't create
	338	* empty queues anyway, so don't bother to set up the device.
	339	*/
	340	if (info->num == 0) {
	341	err = -ENOENT;
	342	goto error_alloc_pages;
	343	}
	344
edfd52e6 PM	345	while (1) {
	346	size = PAGE_ALIGN(vring_size(info->num,
	347	VIRTIO_MMIO_VRING_ALIGN));
3850d29f BF	348	/* Did the last iter shrink the queue below minimum size? */
3850d29f BF	349	if (size < VIRTIO_MMIO_VRING_ALIGN * 2) {
edfd52e6 PM	350	err = -ENOMEM;
	351	goto error_alloc_pages;
	352	}
	353
	354	info->queue = alloc_pages_exact(size, GFP_KERNEL \| __GFP_ZERO);
	355	if (info->queue)
	356	break;
	357
	358	info->num /= 2;
	359	}
	360
	361	/* Activate the queue */
	362	writel(info->num, vm_dev->base + VIRTIO_MMIO_QUEUE_NUM);
	363	writel(VIRTIO_MMIO_VRING_ALIGN,
	364	vm_dev->base + VIRTIO_MMIO_QUEUE_ALIGN);
	365	writel(virt_to_phys(info->queue) >> PAGE_SHIFT,
	366	vm_dev->base + VIRTIO_MMIO_QUEUE_PFN);
	367
	368	/* Create the vring */
17bb6d40	369	vq = vring_new_virtqueue(index, info->num, VIRTIO_MMIO_VRING_ALIGN, vdev,
7b21e34f	370	true, info->queue, vm_notify, callback, name);
edfd52e6 PM	371	if (!vq) {
	372	err = -ENOMEM;
	373	goto error_new_virtqueue;
	374	}
	375
	376	vq->priv = info;
	377	info->vq = vq;
	378
	379	spin_lock_irqsave(&vm_dev->lock, flags);
	380	list_add(&info->node, &vm_dev->virtqueues);
	381	spin_unlock_irqrestore(&vm_dev->lock, flags);
	382
	383	return vq;
	384
	385	error_new_virtqueue:
	386	writel(0, vm_dev->base + VIRTIO_MMIO_QUEUE_PFN);
	387	free_pages_exact(info->queue, size);
	388	error_alloc_pages:
	389	kfree(info);
	390	error_kmalloc:
	391	error_available:
	392	return ERR_PTR(err);
	393	}
	394
	395	static int vm_find_vqs(struct virtio_device *vdev, unsigned nvqs,
	396	struct virtqueue *vqs[],
	397	vq_callback_t *callbacks[],
	398	const char *names[])
	399	{
	400	struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
	401	unsigned int irq = platform_get_irq(vm_dev->pdev, 0);
	402	int i, err;
	403
	404	err = request_irq(irq, vm_interrupt, IRQF_SHARED,
	405	dev_name(&vdev->dev), vm_dev);
	406	if (err)
	407	return err;
	408
	409	for (i = 0; i < nvqs; ++i) {
	410	vqs[i] = vm_setup_vq(vdev, i, callbacks[i], names[i]);
	411	if (IS_ERR(vqs[i])) {
	412	vm_del_vqs(vdev);
	413	return PTR_ERR(vqs[i]);
	414	}
	415	}
	416
	417	return 0;
	418	}
	419
66846048 RJ	420	static const char vm_bus_name(struct virtio_device vdev)
	421	{
	422	struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
edfd52e6	423
66846048 RJ	424	return vm_dev->pdev->name;
66846048 RJ	425	}
edfd52e6	426
93503932	427	static const struct virtio_config_ops virtio_mmio_config_ops = {
edfd52e6 PM	428	.get = vm_get,
	429	.set = vm_set,
	430	.get_status = vm_get_status,
	431	.set_status = vm_set_status,
	432	.reset = vm_reset,
	433	.find_vqs = vm_find_vqs,
	434	.del_vqs = vm_del_vqs,
	435	.get_features = vm_get_features,
	436	.finalize_features = vm_finalize_features,
66846048	437	.bus_name = vm_bus_name,
edfd52e6 PM	438	};
	439
	440
	441
	442	/* Platform device */
	443
8590dbc7	444	static int virtio_mmio_probe(struct platform_device *pdev)
edfd52e6 PM	445	{
	446	struct virtio_mmio_device *vm_dev;
	447	struct resource *mem;
	448	unsigned long magic;
	449
	450	mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
	451	if (!mem)
	452	return -EINVAL;
	453
	454	if (!devm_request_mem_region(&pdev->dev, mem->start,
	455	resource_size(mem), pdev->name))
	456	return -EBUSY;
	457
	458	vm_dev = devm_kzalloc(&pdev->dev, sizeof(*vm_dev), GFP_KERNEL);
	459	if (!vm_dev)
	460	return -ENOMEM;
	461
	462	vm_dev->vdev.dev.parent = &pdev->dev;
	463	vm_dev->vdev.config = &virtio_mmio_config_ops;
	464	vm_dev->pdev = pdev;
	465	INIT_LIST_HEAD(&vm_dev->virtqueues);
	466	spin_lock_init(&vm_dev->lock);
	467
	468	vm_dev->base = devm_ioremap(&pdev->dev, mem->start, resource_size(mem));
	469	if (vm_dev->base == NULL)
	470	return -EFAULT;
	471
	472	/* Check magic value */
	473	magic = readl(vm_dev->base + VIRTIO_MMIO_MAGIC_VALUE);
4ae85370	474	if (magic != ('v' \| 'i' << 8 \| 'r' << 16 \| 't' << 24)) {
edfd52e6 PM	475	dev_warn(&pdev->dev, "Wrong magic value 0x%08lx!\n", magic);
	476	return -ENODEV;
	477	}
	478
	479	/* Check device version */
	480	vm_dev->version = readl(vm_dev->base + VIRTIO_MMIO_VERSION);
	481	if (vm_dev->version != 1) {
	482	dev_err(&pdev->dev, "Version %ld not supported!\n",
	483	vm_dev->version);
	484	return -ENXIO;
	485	}
	486
	487	vm_dev->vdev.id.device = readl(vm_dev->base + VIRTIO_MMIO_DEVICE_ID);
	488	vm_dev->vdev.id.vendor = readl(vm_dev->base + VIRTIO_MMIO_VENDOR_ID);
	489
	490	writel(PAGE_SIZE, vm_dev->base + VIRTIO_MMIO_GUEST_PAGE_SIZE);
	491
	492	platform_set_drvdata(pdev, vm_dev);
	493
	494	return register_virtio_device(&vm_dev->vdev);
	495	}
	496
8590dbc7	497	static int virtio_mmio_remove(struct platform_device *pdev)
edfd52e6 PM	498	{
	499	struct virtio_mmio_device *vm_dev = platform_get_drvdata(pdev);
	500
	501	unregister_virtio_device(&vm_dev->vdev);
	502
	503	return 0;
	504	}
	505
	506
	507
81a054ce PM	508	/* Devices list parameter */
	509
	510	#if defined(CONFIG_VIRTIO_MMIO_CMDLINE_DEVICES)
	511
	512	static struct device vm_cmdline_parent = {
	513	.init_name = "virtio-mmio-cmdline",
	514	};
	515
	516	static int vm_cmdline_parent_registered;
	517	static int vm_cmdline_id;
	518
	519	static int vm_cmdline_set(const char *device,
	520	const struct kernel_param *kp)
	521	{
	522	int err;
	523	struct resource resources[2] = {};
	524	char *str;
40f9938c PM	525	long long int base, size;
40f9938c PM	526	unsigned int irq;
81a054ce PM	527	int processed, consumed = 0;
	528	struct platform_device *pdev;
	529
40f9938c PM	530	/* Consume "size" part of the command line parameter */
40f9938c PM	531	size = memparse(device, &str);
81a054ce	532
40f9938c	533	/* Get "@<base>:<irq>[:<id>]" chunks */
81a054ce	534	processed = sscanf(str, "@%lli:%u%n:%d%n",
40f9938c	535	&base, &irq, &consumed,
81a054ce PM	536	&vm_cmdline_id, &consumed);
81a054ce PM	537
40f9938c PM	538	/*
	539	* sscanf() must processes at least 2 chunks; also there
	540	* must be no extra characters after the last chunk, so
	541	* str[consumed] must be '\0'
	542	*/
	543	if (processed < 2 \|\| str[consumed])
81a054ce PM	544	return -EINVAL;
81a054ce PM	545
40f9938c	546	resources[0].flags = IORESOURCE_MEM;
81a054ce	547	resources[0].start = base;
40f9938c PM	548	resources[0].end = base + size - 1;
	549
	550	resources[1].flags = IORESOURCE_IRQ;
	551	resources[1].start = resources[1].end = irq;
81a054ce PM	552
	553	if (!vm_cmdline_parent_registered) {
	554	err = device_register(&vm_cmdline_parent);
	555	if (err) {
	556	pr_err("Failed to register parent device!\n");
	557	return err;
	558	}
	559	vm_cmdline_parent_registered = 1;
	560	}
	561
	562	pr_info("Registering device virtio-mmio.%d at 0x%llx-0x%llx, IRQ %d.\n",
	563	vm_cmdline_id,
	564	(unsigned long long)resources[0].start,
	565	(unsigned long long)resources[0].end,
	566	(int)resources[1].start);
	567
	568	pdev = platform_device_register_resndata(&vm_cmdline_parent,
	569	"virtio-mmio", vm_cmdline_id++,
	570	resources, ARRAY_SIZE(resources), NULL, 0);
	571	if (IS_ERR(pdev))
	572	return PTR_ERR(pdev);
	573
	574	return 0;
	575	}
	576
	577	static int vm_cmdline_get_device(struct device dev, void data)
	578	{
	579	char *buffer = data;
	580	unsigned int len = strlen(buffer);
	581	struct platform_device *pdev = to_platform_device(dev);
	582
	583	snprintf(buffer + len, PAGE_SIZE - len, "0x%llx@0x%llx:%llu:%d\n",
	584	pdev->resource[0].end - pdev->resource[0].start + 1ULL,
	585	(unsigned long long)pdev->resource[0].start,
	586	(unsigned long long)pdev->resource[1].start,
	587	pdev->id);
	588	return 0;
	589	}
	590
	591	static int vm_cmdline_get(char buffer, const struct kernel_param kp)
	592	{
	593	buffer[0] = '\0';
	594	device_for_each_child(&vm_cmdline_parent, buffer,
	595	vm_cmdline_get_device);
	596	return strlen(buffer) + 1;
	597	}
	598
	599	static struct kernel_param_ops vm_cmdline_param_ops = {
	600	.set = vm_cmdline_set,
	601	.get = vm_cmdline_get,
	602	};
	603
	604	device_param_cb(device, &vm_cmdline_param_ops, NULL, S_IRUSR);
	605
	606	static int vm_unregister_cmdline_device(struct device *dev,
	607	void *data)
	608	{
	609	platform_device_unregister(to_platform_device(dev));
	610
	611	return 0;
	612	}
	613
	614	static void vm_unregister_cmdline_devices(void)
	615	{
616	if (vm_cmdline_parent_registered) {
617	device_for_each_child(&vm_cmdline_parent, NULL,
618	vm_unregister_cmdline_device);
619	device_unregister(&vm_cmdline_parent);
620	vm_cmdline_parent_registered = 0;
621	}
622	}
623
624	#else
625
626	static void vm_unregister_cmdline_devices(void)
627	{
628	}
629
630	#endif
631
edfd52e6 PM	632	/* Platform driver */
	633
	634	static struct of_device_id virtio_mmio_match[] = {
	635	{ .compatible = "virtio,mmio", },
	636	{},
	637	};
	638	MODULE_DEVICE_TABLE(of, virtio_mmio_match);
	639
	640	static struct platform_driver virtio_mmio_driver = {
	641	.probe = virtio_mmio_probe,
8590dbc7	642	.remove = virtio_mmio_remove,
edfd52e6 PM	643	.driver = {
	644	.name = "virtio-mmio",
	645	.owner = THIS_MODULE,
	646	.of_match_table = virtio_mmio_match,
	647	},
	648	};
	649
	650	static int __init virtio_mmio_init(void)
	651	{
	652	return platform_driver_register(&virtio_mmio_driver);
	653	}
	654
	655	static void __exit virtio_mmio_exit(void)
	656	{
	657	platform_driver_unregister(&virtio_mmio_driver);
81a054ce	658	vm_unregister_cmdline_devices();
edfd52e6 PM	659	}
	660
	661	module_init(virtio_mmio_init);
	662	module_exit(virtio_mmio_exit);
	663
	664	MODULE_AUTHOR("Pawel Moll <pawel.moll@arm.com>");
	665	MODULE_DESCRIPTION("Platform bus driver for memory mapped virtio devices");
	666	MODULE_LICENSE("GPL");