[deliverable/linux.git] / drivers / iommu / iommu.c

/*
 * Copyright (C) 2007-2008 Advanced Micro Devices, Inc.
 * Author: Joerg Roedel <joerg.roedel@amd.com>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License version 2 as published
 * by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
 */

#define pr_fmt(fmt)    "%s: " fmt, __func__

#include <linux/device.h>
#include <linux/kernel.h>
#include <linux/bug.h>
#include <linux/types.h>
#include <linux/module.h>
#include <linux/slab.h>
#include <linux/errno.h>
#include <linux/iommu.h>

static ssize_t show_iommu_group(struct device *dev,
				struct device_attribute *attr, char *buf)
{
	unsigned int groupid;

	if (iommu_device_group(dev, &groupid))
		return 0;

	return sprintf(buf, "%u", groupid);
}
static DEVICE_ATTR(iommu_group, S_IRUGO, show_iommu_group, NULL);

static int add_iommu_group(struct device *dev, void *data)
{
	unsigned int groupid;

	if (iommu_device_group(dev, &groupid) == 0)
		return device_create_file(dev, &dev_attr_iommu_group);

	return 0;
}

static int remove_iommu_group(struct device *dev)
{
	unsigned int groupid;

	if (iommu_device_group(dev, &groupid) == 0)
		device_remove_file(dev, &dev_attr_iommu_group);

	return 0;
}

static int iommu_device_notifier(struct notifier_block *nb,
				 unsigned long action, void *data)
{
	struct device *dev = data;

	if (action == BUS_NOTIFY_ADD_DEVICE)
		return add_iommu_group(dev, NULL);
	else if (action == BUS_NOTIFY_DEL_DEVICE)
		return remove_iommu_group(dev);

	return 0;
}

static struct notifier_block iommu_device_nb = {
	.notifier_call = iommu_device_notifier,
};

static void iommu_bus_init(struct bus_type *bus, struct iommu_ops *ops)
{
	bus_register_notifier(bus, &iommu_device_nb);
	bus_for_each_dev(bus, NULL, NULL, add_iommu_group);
}

/**
 * bus_set_iommu - set iommu-callbacks for the bus
 * @bus: bus.
 * @ops: the callbacks provided by the iommu-driver
 *
 * This function is called by an iommu driver to set the iommu methods
 * used for a particular bus. Drivers for devices on that bus can use
 * the iommu-api after these ops are registered.
 * This special function is needed because IOMMUs are usually devices on
 * the bus itself, so the iommu drivers are not initialized when the bus
 * is set up. With this function the iommu-driver can set the iommu-ops
 * afterwards.
 */
int bus_set_iommu(struct bus_type *bus, struct iommu_ops *ops)
{
	if (bus->iommu_ops != NULL)
		return -EBUSY;

	bus->iommu_ops = ops;

	/* Do IOMMU specific setup for this bus-type */
	iommu_bus_init(bus, ops);

	return 0;
}
EXPORT_SYMBOL_GPL(bus_set_iommu);

bool iommu_present(struct bus_type *bus)
{
	return bus->iommu_ops != NULL;
}
EXPORT_SYMBOL_GPL(iommu_present);

/**
 * iommu_set_fault_handler() - set a fault handler for an iommu domain
 * @domain: iommu domain
 * @handler: fault handler
 * @token: user data, will be passed back to the fault handler
 *
 * This function should be used by IOMMU users which want to be notified
 * whenever an IOMMU fault happens.
 *
 * The fault handler itself should return 0 on success, and an appropriate
 * error code otherwise.
 */
void iommu_set_fault_handler(struct iommu_domain *domain,
					iommu_fault_handler_t handler,
					void *token)
{
	BUG_ON(!domain);

	domain->handler = handler;
	domain->handler_token = token;
}
EXPORT_SYMBOL_GPL(iommu_set_fault_handler);

struct iommu_domain *iommu_domain_alloc(struct bus_type *bus)
{
	struct iommu_domain *domain;
	int ret;

	if (bus == NULL || bus->iommu_ops == NULL)
		return NULL;

	domain = kzalloc(sizeof(*domain), GFP_KERNEL);
	if (!domain)
		return NULL;

	domain->ops = bus->iommu_ops;

	ret = domain->ops->domain_init(domain);
	if (ret)
		goto out_free;

	return domain;

out_free:
	kfree(domain);

	return NULL;
}
EXPORT_SYMBOL_GPL(iommu_domain_alloc);

void iommu_domain_free(struct iommu_domain *domain)
{
	if (likely(domain->ops->domain_destroy != NULL))
		domain->ops->domain_destroy(domain);

	kfree(domain);
}
EXPORT_SYMBOL_GPL(iommu_domain_free);

int iommu_attach_device(struct iommu_domain *domain, struct device *dev)
{
	if (unlikely(domain->ops->attach_dev == NULL))
		return -ENODEV;

	return domain->ops->attach_dev(domain, dev);
}
EXPORT_SYMBOL_GPL(iommu_attach_device);

void iommu_detach_device(struct iommu_domain *domain, struct device *dev)
{
	if (unlikely(domain->ops->detach_dev == NULL))
		return;

	domain->ops->detach_dev(domain, dev);
}
EXPORT_SYMBOL_GPL(iommu_detach_device);

phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain,
			       unsigned long iova)
{
	if (unlikely(domain->ops->iova_to_phys == NULL))
		return 0;

	return domain->ops->iova_to_phys(domain, iova);
}
EXPORT_SYMBOL_GPL(iommu_iova_to_phys);

int iommu_domain_has_cap(struct iommu_domain *domain,
			 unsigned long cap)
{
	if (unlikely(domain->ops->domain_has_cap == NULL))
		return 0;

	return domain->ops->domain_has_cap(domain, cap);
}
EXPORT_SYMBOL_GPL(iommu_domain_has_cap);

int iommu_map(struct iommu_domain *domain, unsigned long iova,
	      phys_addr_t paddr, size_t size, int prot)
{
	unsigned long orig_iova = iova;
	unsigned int min_pagesz;
	size_t orig_size = size;
	int ret = 0;

	if (unlikely(domain->ops->map == NULL))
		return -ENODEV;

	/* find out the minimum page size supported */
	min_pagesz = 1 << __ffs(domain->ops->pgsize_bitmap);

	/*
	 * both the virtual address and the physical one, as well as
	 * the size of the mapping, must be aligned (at least) to the
	 * size of the smallest page supported by the hardware
	 */
	if (!IS_ALIGNED(iova | paddr | size, min_pagesz)) {
		pr_err("unaligned: iova 0x%lx pa 0x%lx size 0x%lx min_pagesz "
			"0x%x\n", iova, (unsigned long)paddr,
			(unsigned long)size, min_pagesz);
		return -EINVAL;
	}

	pr_debug("map: iova 0x%lx pa 0x%lx size 0x%lx\n", iova,
				(unsigned long)paddr, (unsigned long)size);

	while (size) {
		unsigned long pgsize, addr_merge = iova | paddr;
		unsigned int pgsize_idx;

		/* Max page size that still fits into 'size' */
		pgsize_idx = __fls(size);

		/* need to consider alignment requirements ? */
		if (likely(addr_merge)) {
			/* Max page size allowed by both iova and paddr */
			unsigned int align_pgsize_idx = __ffs(addr_merge);

			pgsize_idx = min(pgsize_idx, align_pgsize_idx);
		}

		/* build a mask of acceptable page sizes */
		pgsize = (1UL << (pgsize_idx + 1)) - 1;

		/* throw away page sizes not supported by the hardware */
		pgsize &= domain->ops->pgsize_bitmap;

		/* make sure we're still sane */
		BUG_ON(!pgsize);

		/* pick the biggest page */
		pgsize_idx = __fls(pgsize);
		pgsize = 1UL << pgsize_idx;

		pr_debug("mapping: iova 0x%lx pa 0x%lx pgsize %lu\n", iova,
					(unsigned long)paddr, pgsize);

		ret = domain->ops->map(domain, iova, paddr, pgsize, prot);
		if (ret)
			break;

		iova += pgsize;
		paddr += pgsize;
		size -= pgsize;
	}

	/* unroll mapping in case something went wrong */
	if (ret)
		iommu_unmap(domain, orig_iova, orig_size - size);

	return ret;
}
EXPORT_SYMBOL_GPL(iommu_map);

size_t iommu_unmap(struct iommu_domain *domain, unsigned long iova, size_t size)
{
	size_t unmapped_page, unmapped = 0;
	unsigned int min_pagesz;

	if (unlikely(domain->ops->unmap == NULL))
		return -ENODEV;

	/* find out the minimum page size supported */
	min_pagesz = 1 << __ffs(domain->ops->pgsize_bitmap);

	/*
	 * The virtual address, as well as the size of the mapping, must be
	 * aligned (at least) to the size of the smallest page supported
	 * by the hardware
	 */
	if (!IS_ALIGNED(iova | size, min_pagesz)) {
		pr_err("unaligned: iova 0x%lx size 0x%lx min_pagesz 0x%x\n",
					iova, (unsigned long)size, min_pagesz);
		return -EINVAL;
	}

	pr_debug("unmap this: iova 0x%lx size 0x%lx\n", iova,
							(unsigned long)size);

	/*
	 * Keep iterating until we either unmap 'size' bytes (or more)
	 * or we hit an area that isn't mapped.
	 */
	while (unmapped < size) {
		size_t left = size - unmapped;

		unmapped_page = domain->ops->unmap(domain, iova, left);
		if (!unmapped_page)
			break;

		pr_debug("unmapped: iova 0x%lx size %lx\n", iova,
					(unsigned long)unmapped_page);

		iova += unmapped_page;
		unmapped += unmapped_page;
	}

	return unmapped;
}
EXPORT_SYMBOL_GPL(iommu_unmap);

int iommu_device_group(struct device *dev, unsigned int *groupid)
{
	if (iommu_present(dev->bus) && dev->bus->iommu_ops->device_group)
		return dev->bus->iommu_ops->device_group(dev, groupid);

	return -ENODEV;
}
EXPORT_SYMBOL_GPL(iommu_device_group);
Commit	Line	Data
fc2100eb JR	1	/*
	2	* Copyright (C) 2007-2008 Advanced Micro Devices, Inc.
	3	* Author: Joerg Roedel <joerg.roedel@amd.com>
	4	*
	5	* This program is free software; you can redistribute it and/or modify it
	6	* under the terms of the GNU General Public License version 2 as published
	7	* by the Free Software Foundation.
	8	*
	9	* This program is distributed in the hope that it will be useful,
	10	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	11	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	12	* GNU General Public License for more details.
	13	*
	14	* You should have received a copy of the GNU General Public License
	15	* along with this program; if not, write to the Free Software
	16	* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
	17	*/
	18
7d3002cc OBC	19	#define pr_fmt(fmt) "%s: " fmt, __func__
7d3002cc OBC	20
905d66c1	21	#include <linux/device.h>
40998188	22	#include <linux/kernel.h>
fc2100eb JR	23	#include <linux/bug.h>
fc2100eb JR	24	#include <linux/types.h>
60db4027 AM	25	#include <linux/module.h>
60db4027 AM	26	#include <linux/slab.h>
fc2100eb JR	27	#include <linux/errno.h>
	28	#include <linux/iommu.h>
	29
1460432c AW	30	static ssize_t show_iommu_group(struct device *dev,
	31	struct device_attribute attr, char buf)
	32	{
	33	unsigned int groupid;
	34
	35	if (iommu_device_group(dev, &groupid))
	36	return 0;
	37
	38	return sprintf(buf, "%u", groupid);
	39	}
	40	static DEVICE_ATTR(iommu_group, S_IRUGO, show_iommu_group, NULL);
	41
	42	static int add_iommu_group(struct device dev, void data)
	43	{
	44	unsigned int groupid;
	45
	46	if (iommu_device_group(dev, &groupid) == 0)
	47	return device_create_file(dev, &dev_attr_iommu_group);
	48
	49	return 0;
	50	}
	51
	52	static int remove_iommu_group(struct device *dev)
	53	{
	54	unsigned int groupid;
	55
	56	if (iommu_device_group(dev, &groupid) == 0)
	57	device_remove_file(dev, &dev_attr_iommu_group);
	58
	59	return 0;
	60	}
	61
	62	static int iommu_device_notifier(struct notifier_block *nb,
	63	unsigned long action, void *data)
	64	{
	65	struct device *dev = data;
	66
	67	if (action == BUS_NOTIFY_ADD_DEVICE)
	68	return add_iommu_group(dev, NULL);
	69	else if (action == BUS_NOTIFY_DEL_DEVICE)
	70	return remove_iommu_group(dev);
	71
	72	return 0;
	73	}
	74
	75	static struct notifier_block iommu_device_nb = {
	76	.notifier_call = iommu_device_notifier,
	77	};
	78
ff21776d JR	79	static void iommu_bus_init(struct bus_type bus, struct iommu_ops ops)
ff21776d JR	80	{
1460432c AW	81	bus_register_notifier(bus, &iommu_device_nb);
1460432c AW	82	bus_for_each_dev(bus, NULL, NULL, add_iommu_group);
ff21776d	83	}
fc2100eb	84
ff21776d JR	85	/**
	86	* bus_set_iommu - set iommu-callbacks for the bus
	87	* @bus: bus.
	88	* @ops: the callbacks provided by the iommu-driver
	89	*
	90	* This function is called by an iommu driver to set the iommu methods
	91	* used for a particular bus. Drivers for devices on that bus can use
	92	* the iommu-api after these ops are registered.
	93	* This special function is needed because IOMMUs are usually devices on
	94	* the bus itself, so the iommu drivers are not initialized when the bus
	95	* is set up. With this function the iommu-driver can set the iommu-ops
	96	* afterwards.
	97	*/
	98	int bus_set_iommu(struct bus_type bus, struct iommu_ops ops)
fc2100eb	99	{
ff21776d JR	100	if (bus->iommu_ops != NULL)
ff21776d JR	101	return -EBUSY;
fc2100eb	102
ff21776d JR	103	bus->iommu_ops = ops;
	104
	105	/* Do IOMMU specific setup for this bus-type */
	106	iommu_bus_init(bus, ops);
fc2100eb	107
ff21776d	108	return 0;
fc2100eb	109	}
ff21776d	110	EXPORT_SYMBOL_GPL(bus_set_iommu);
fc2100eb	111
a1b60c1c	112	bool iommu_present(struct bus_type *bus)
fc2100eb	113	{
94441c3b	114	return bus->iommu_ops != NULL;
fc2100eb	115	}
a1b60c1c	116	EXPORT_SYMBOL_GPL(iommu_present);
fc2100eb	117
4f3f8d9d OBC	118	/**
	119	* iommu_set_fault_handler() - set a fault handler for an iommu domain
	120	* @domain: iommu domain
	121	* @handler: fault handler
77ca2332	122	* @token: user data, will be passed back to the fault handler
0ed6d2d2 OBC	123	*
	124	* This function should be used by IOMMU users which want to be notified
	125	* whenever an IOMMU fault happens.
	126	*
	127	* The fault handler itself should return 0 on success, and an appropriate
	128	* error code otherwise.
4f3f8d9d OBC	129	*/
4f3f8d9d OBC	130	void iommu_set_fault_handler(struct iommu_domain *domain,
77ca2332 OBC	131	iommu_fault_handler_t handler,
77ca2332 OBC	132	void *token)
4f3f8d9d OBC	133	{
	134	BUG_ON(!domain);
	135
	136	domain->handler = handler;
77ca2332	137	domain->handler_token = token;
4f3f8d9d	138	}
30bd918c	139	EXPORT_SYMBOL_GPL(iommu_set_fault_handler);
4f3f8d9d	140
905d66c1	141	struct iommu_domain iommu_domain_alloc(struct bus_type bus)
fc2100eb JR	142	{
	143	struct iommu_domain *domain;
	144	int ret;
	145
94441c3b	146	if (bus == NULL \|\| bus->iommu_ops == NULL)
905d66c1 JR	147	return NULL;
905d66c1 JR	148
8bd6960c	149	domain = kzalloc(sizeof(*domain), GFP_KERNEL);
fc2100eb JR	150	if (!domain)
	151	return NULL;
	152
94441c3b	153	domain->ops = bus->iommu_ops;
905d66c1	154
94441c3b	155	ret = domain->ops->domain_init(domain);
fc2100eb JR	156	if (ret)
	157	goto out_free;
	158
	159	return domain;
	160
	161	out_free:
	162	kfree(domain);
	163
	164	return NULL;
	165	}
	166	EXPORT_SYMBOL_GPL(iommu_domain_alloc);
	167
	168	void iommu_domain_free(struct iommu_domain *domain)
	169	{
e5aa7f00 JR	170	if (likely(domain->ops->domain_destroy != NULL))
	171	domain->ops->domain_destroy(domain);
	172
fc2100eb JR	173	kfree(domain);
	174	}
	175	EXPORT_SYMBOL_GPL(iommu_domain_free);
	176
	177	int iommu_attach_device(struct iommu_domain domain, struct device dev)
	178	{
e5aa7f00 JR	179	if (unlikely(domain->ops->attach_dev == NULL))
	180	return -ENODEV;
	181
	182	return domain->ops->attach_dev(domain, dev);
fc2100eb JR	183	}
	184	EXPORT_SYMBOL_GPL(iommu_attach_device);
	185
	186	void iommu_detach_device(struct iommu_domain domain, struct device dev)
	187	{
e5aa7f00 JR	188	if (unlikely(domain->ops->detach_dev == NULL))
	189	return;
	190
	191	domain->ops->detach_dev(domain, dev);
fc2100eb JR	192	}
	193	EXPORT_SYMBOL_GPL(iommu_detach_device);
	194
fc2100eb JR	195	phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain,
	196	unsigned long iova)
	197	{
e5aa7f00 JR	198	if (unlikely(domain->ops->iova_to_phys == NULL))
	199	return 0;
	200
	201	return domain->ops->iova_to_phys(domain, iova);
fc2100eb JR	202	}
fc2100eb JR	203	EXPORT_SYMBOL_GPL(iommu_iova_to_phys);
dbb9fd86 SY	204
	205	int iommu_domain_has_cap(struct iommu_domain *domain,
	206	unsigned long cap)
	207	{
e5aa7f00 JR	208	if (unlikely(domain->ops->domain_has_cap == NULL))
	209	return 0;
	210
	211	return domain->ops->domain_has_cap(domain, cap);
dbb9fd86 SY	212	}
dbb9fd86 SY	213	EXPORT_SYMBOL_GPL(iommu_domain_has_cap);
cefc53c7 JR	214
cefc53c7 JR	215	int iommu_map(struct iommu_domain *domain, unsigned long iova,
7d3002cc	216	phys_addr_t paddr, size_t size, int prot)
cefc53c7	217	{
7d3002cc OBC	218	unsigned long orig_iova = iova;
	219	unsigned int min_pagesz;
	220	size_t orig_size = size;
	221	int ret = 0;
cefc53c7	222
e5aa7f00 JR	223	if (unlikely(domain->ops->map == NULL))
e5aa7f00 JR	224	return -ENODEV;
cefc53c7	225
7d3002cc OBC	226	/* find out the minimum page size supported */
	227	min_pagesz = 1 << __ffs(domain->ops->pgsize_bitmap);
	228
	229	/*
	230	* both the virtual address and the physical one, as well as
	231	* the size of the mapping, must be aligned (at least) to the
	232	* size of the smallest page supported by the hardware
	233	*/
	234	if (!IS_ALIGNED(iova \| paddr \| size, min_pagesz)) {
	235	pr_err("unaligned: iova 0x%lx pa 0x%lx size 0x%lx min_pagesz "
	236	"0x%x\n", iova, (unsigned long)paddr,
	237	(unsigned long)size, min_pagesz);
	238	return -EINVAL;
	239	}
	240
	241	pr_debug("map: iova 0x%lx pa 0x%lx size 0x%lx\n", iova,
	242	(unsigned long)paddr, (unsigned long)size);
	243
	244	while (size) {
	245	unsigned long pgsize, addr_merge = iova \| paddr;
	246	unsigned int pgsize_idx;
	247
	248	/* Max page size that still fits into 'size' */
	249	pgsize_idx = __fls(size);
	250
	251	/* need to consider alignment requirements ? */
	252	if (likely(addr_merge)) {
	253	/* Max page size allowed by both iova and paddr */
	254	unsigned int align_pgsize_idx = __ffs(addr_merge);
	255
	256	pgsize_idx = min(pgsize_idx, align_pgsize_idx);
	257	}
	258
	259	/* build a mask of acceptable page sizes */
	260	pgsize = (1UL << (pgsize_idx + 1)) - 1;
	261
	262	/* throw away page sizes not supported by the hardware */
	263	pgsize &= domain->ops->pgsize_bitmap;
cefc53c7	264
7d3002cc OBC	265	/* make sure we're still sane */
7d3002cc OBC	266	BUG_ON(!pgsize);
cefc53c7	267
7d3002cc OBC	268	/* pick the biggest page */
	269	pgsize_idx = __fls(pgsize);
	270	pgsize = 1UL << pgsize_idx;
	271
	272	pr_debug("mapping: iova 0x%lx pa 0x%lx pgsize %lu\n", iova,
	273	(unsigned long)paddr, pgsize);
	274
	275	ret = domain->ops->map(domain, iova, paddr, pgsize, prot);
	276	if (ret)
	277	break;
	278
	279	iova += pgsize;
	280	paddr += pgsize;
	281	size -= pgsize;
	282	}
	283
	284	/* unroll mapping in case something went wrong */
	285	if (ret)
	286	iommu_unmap(domain, orig_iova, orig_size - size);
	287
	288	return ret;
cefc53c7 JR	289	}
	290	EXPORT_SYMBOL_GPL(iommu_map);
	291
7d3002cc	292	size_t iommu_unmap(struct iommu_domain *domain, unsigned long iova, size_t size)
cefc53c7	293	{
7d3002cc OBC	294	size_t unmapped_page, unmapped = 0;
7d3002cc OBC	295	unsigned int min_pagesz;
cefc53c7	296
e5aa7f00 JR	297	if (unlikely(domain->ops->unmap == NULL))
	298	return -ENODEV;
	299
7d3002cc OBC	300	/* find out the minimum page size supported */
	301	min_pagesz = 1 << __ffs(domain->ops->pgsize_bitmap);
	302
	303	/*
	304	* The virtual address, as well as the size of the mapping, must be
	305	* aligned (at least) to the size of the smallest page supported
	306	* by the hardware
	307	*/
	308	if (!IS_ALIGNED(iova \| size, min_pagesz)) {
	309	pr_err("unaligned: iova 0x%lx size 0x%lx min_pagesz 0x%x\n",
	310	iova, (unsigned long)size, min_pagesz);
	311	return -EINVAL;
	312	}
	313
	314	pr_debug("unmap this: iova 0x%lx size 0x%lx\n", iova,
	315	(unsigned long)size);
	316
	317	/*
	318	* Keep iterating until we either unmap 'size' bytes (or more)
	319	* or we hit an area that isn't mapped.
	320	*/
	321	while (unmapped < size) {
	322	size_t left = size - unmapped;
	323
	324	unmapped_page = domain->ops->unmap(domain, iova, left);
	325	if (!unmapped_page)
	326	break;
	327
	328	pr_debug("unmapped: iova 0x%lx size %lx\n", iova,
	329	(unsigned long)unmapped_page);
	330
	331	iova += unmapped_page;
	332	unmapped += unmapped_page;
	333	}
	334
	335	return unmapped;
cefc53c7 JR	336	}
cefc53c7 JR	337	EXPORT_SYMBOL_GPL(iommu_unmap);
1460432c AW	338
	339	int iommu_device_group(struct device dev, unsigned int groupid)
	340	{
	341	if (iommu_present(dev->bus) && dev->bus->iommu_ops->device_group)
	342	return dev->bus->iommu_ops->device_group(dev, groupid);
	343
	344	return -ENODEV;
	345	}
	346	EXPORT_SYMBOL_GPL(iommu_device_group);