[deliverable/linux.git] / drivers / vfio / vfio_iommu_spapr_tce.c

/*
 * VFIO: IOMMU DMA mapping support for TCE on POWER
 *
 * Copyright (C) 2013 IBM Corp.  All rights reserved.
 *     Author: Alexey Kardashevskiy <aik@ozlabs.ru>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 * Derived from original vfio_iommu_type1.c:
 * Copyright (C) 2012 Red Hat, Inc.  All rights reserved.
 *     Author: Alex Williamson <alex.williamson@redhat.com>
 */

#include <linux/module.h>
#include <linux/pci.h>
#include <linux/slab.h>
#include <linux/uaccess.h>
#include <linux/err.h>
#include <linux/vfio.h>
#include <asm/iommu.h>
#include <asm/tce.h>

#define DRIVER_VERSION  "0.1"
#define DRIVER_AUTHOR   "aik@ozlabs.ru"
#define DRIVER_DESC     "VFIO IOMMU SPAPR TCE"

static void tce_iommu_detach_group(void *iommu_data,
		struct iommu_group *iommu_group);

/*
 * VFIO IOMMU fd for SPAPR_TCE IOMMU implementation
 *
 * This code handles mapping and unmapping of user data buffers
 * into DMA'ble space using the IOMMU
 */

/*
 * The container descriptor supports only a single group per container.
 * Required by the API as the container is not supplied with the IOMMU group
 * at the moment of initialization.
 */
struct tce_container {
	struct mutex lock;
	struct iommu_table *tbl;
	bool enabled;
};

static int tce_iommu_enable(struct tce_container *container)
{
	int ret = 0;
	unsigned long locked, lock_limit, npages;
	struct iommu_table *tbl = container->tbl;

	if (!container->tbl)
		return -ENXIO;

	if (!current->mm)
		return -ESRCH; /* process exited */

	if (container->enabled)
		return -EBUSY;

	/*
	 * When userspace pages are mapped into the IOMMU, they are effectively
	 * locked memory, so, theoretically, we need to update the accounting
	 * of locked pages on each map and unmap.  For powerpc, the map unmap
	 * paths can be very hot, though, and the accounting would kill
	 * performance, especially since it would be difficult to impossible
	 * to handle the accounting in real mode only.
	 *
	 * To address that, rather than precisely accounting every page, we
	 * instead account for a worst case on locked memory when the iommu is
	 * enabled and disabled.  The worst case upper bound on locked memory
	 * is the size of the whole iommu window, which is usually relatively
	 * small (compared to total memory sizes) on POWER hardware.
	 *
	 * Also we don't have a nice way to fail on H_PUT_TCE due to ulimits,
	 * that would effectively kill the guest at random points, much better
	 * enforcing the limit based on the max that the guest can map.
	 */
	down_write(&current->mm->mmap_sem);
	npages = (tbl->it_size << IOMMU_PAGE_SHIFT_4K) >> PAGE_SHIFT;
	locked = current->mm->locked_vm + npages;
	lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
	if (locked > lock_limit && !capable(CAP_IPC_LOCK)) {
		pr_warn("RLIMIT_MEMLOCK (%ld) exceeded\n",
				rlimit(RLIMIT_MEMLOCK));
		ret = -ENOMEM;
	} else {

		current->mm->locked_vm += npages;
		container->enabled = true;
	}
	up_write(&current->mm->mmap_sem);

	return ret;
}

static void tce_iommu_disable(struct tce_container *container)
{
	if (!container->enabled)
		return;

	container->enabled = false;

	if (!container->tbl || !current->mm)
		return;

	down_write(&current->mm->mmap_sem);
	current->mm->locked_vm -= (container->tbl->it_size <<
			IOMMU_PAGE_SHIFT_4K) >> PAGE_SHIFT;
	up_write(&current->mm->mmap_sem);
}

static void *tce_iommu_open(unsigned long arg)
{
	struct tce_container *container;

	if (arg != VFIO_SPAPR_TCE_IOMMU) {
		pr_err("tce_vfio: Wrong IOMMU type\n");
		return ERR_PTR(-EINVAL);
	}

	container = kzalloc(sizeof(*container), GFP_KERNEL);
	if (!container)
		return ERR_PTR(-ENOMEM);

	mutex_init(&container->lock);

	return container;
}

static void tce_iommu_release(void *iommu_data)
{
	struct tce_container *container = iommu_data;

	WARN_ON(container->tbl && !container->tbl->it_group);
	tce_iommu_disable(container);

	if (container->tbl && container->tbl->it_group)
		tce_iommu_detach_group(iommu_data, container->tbl->it_group);

	mutex_destroy(&container->lock);

	kfree(container);
}

static long tce_iommu_ioctl(void *iommu_data,
				 unsigned int cmd, unsigned long arg)
{
	struct tce_container *container = iommu_data;
	unsigned long minsz;
	long ret;

	switch (cmd) {
	case VFIO_CHECK_EXTENSION:
		return (arg == VFIO_SPAPR_TCE_IOMMU) ? 1 : 0;

	case VFIO_IOMMU_SPAPR_TCE_GET_INFO: {
		struct vfio_iommu_spapr_tce_info info;
		struct iommu_table *tbl = container->tbl;

		if (WARN_ON(!tbl))
			return -ENXIO;

		minsz = offsetofend(struct vfio_iommu_spapr_tce_info,
				dma32_window_size);

		if (copy_from_user(&info, (void __user *)arg, minsz))
			return -EFAULT;

		if (info.argsz < minsz)
			return -EINVAL;

		info.dma32_window_start = tbl->it_offset << IOMMU_PAGE_SHIFT_4K;
		info.dma32_window_size = tbl->it_size << IOMMU_PAGE_SHIFT_4K;
		info.flags = 0;

		if (copy_to_user((void __user *)arg, &info, minsz))
			return -EFAULT;

		return 0;
	}
	case VFIO_IOMMU_MAP_DMA: {
		struct vfio_iommu_type1_dma_map param;
		struct iommu_table *tbl = container->tbl;
		unsigned long tce, i;

		if (!tbl)
			return -ENXIO;

		BUG_ON(!tbl->it_group);

		minsz = offsetofend(struct vfio_iommu_type1_dma_map, size);

		if (copy_from_user(&param, (void __user *)arg, minsz))
			return -EFAULT;

		if (param.argsz < minsz)
			return -EINVAL;

		if (param.flags & ~(VFIO_DMA_MAP_FLAG_READ |
				VFIO_DMA_MAP_FLAG_WRITE))
			return -EINVAL;

		if ((param.size & ~IOMMU_PAGE_MASK_4K) ||
				(param.vaddr & ~IOMMU_PAGE_MASK_4K))
			return -EINVAL;

		/* iova is checked by the IOMMU API */
		tce = param.vaddr;
		if (param.flags & VFIO_DMA_MAP_FLAG_READ)
			tce |= TCE_PCI_READ;
		if (param.flags & VFIO_DMA_MAP_FLAG_WRITE)
			tce |= TCE_PCI_WRITE;

		ret = iommu_tce_put_param_check(tbl, param.iova, tce);
		if (ret)
			return ret;

		for (i = 0; i < (param.size >> IOMMU_PAGE_SHIFT_4K); ++i) {
			ret = iommu_put_tce_user_mode(tbl,
					(param.iova >> IOMMU_PAGE_SHIFT_4K) + i,
					tce);
			if (ret)
				break;
			tce += IOMMU_PAGE_SIZE_4K;
		}
		if (ret)
			iommu_clear_tces_and_put_pages(tbl,
					param.iova >> IOMMU_PAGE_SHIFT_4K, i);

		iommu_flush_tce(tbl);

		return ret;
	}
	case VFIO_IOMMU_UNMAP_DMA: {
		struct vfio_iommu_type1_dma_unmap param;
		struct iommu_table *tbl = container->tbl;

		if (WARN_ON(!tbl))
			return -ENXIO;

		minsz = offsetofend(struct vfio_iommu_type1_dma_unmap,
				size);

		if (copy_from_user(&param, (void __user *)arg, minsz))
			return -EFAULT;

		if (param.argsz < minsz)
			return -EINVAL;

		/* No flag is supported now */
		if (param.flags)
			return -EINVAL;

		if (param.size & ~IOMMU_PAGE_MASK_4K)
			return -EINVAL;

		ret = iommu_tce_clear_param_check(tbl, param.iova, 0,
				param.size >> IOMMU_PAGE_SHIFT_4K);
		if (ret)
			return ret;

		ret = iommu_clear_tces_and_put_pages(tbl,
				param.iova >> IOMMU_PAGE_SHIFT_4K,
				param.size >> IOMMU_PAGE_SHIFT_4K);
		iommu_flush_tce(tbl);

		return ret;
	}
	case VFIO_IOMMU_ENABLE:
		mutex_lock(&container->lock);
		ret = tce_iommu_enable(container);
		mutex_unlock(&container->lock);
		return ret;


	case VFIO_IOMMU_DISABLE:
		mutex_lock(&container->lock);
		tce_iommu_disable(container);
		mutex_unlock(&container->lock);
		return 0;
	}

	return -ENOTTY;
}

static int tce_iommu_attach_group(void *iommu_data,
		struct iommu_group *iommu_group)
{
	int ret;
	struct tce_container *container = iommu_data;
	struct iommu_table *tbl = iommu_group_get_iommudata(iommu_group);

	BUG_ON(!tbl);
	mutex_lock(&container->lock);

	/* pr_debug("tce_vfio: Attaching group #%u to iommu %p\n",
			iommu_group_id(iommu_group), iommu_group); */
	if (container->tbl) {
		pr_warn("tce_vfio: Only one group per IOMMU container is allowed, existing id=%d, attaching id=%d\n",
				iommu_group_id(container->tbl->it_group),
				iommu_group_id(iommu_group));
		ret = -EBUSY;
	} else if (container->enabled) {
		pr_err("tce_vfio: attaching group #%u to enabled container\n",
				iommu_group_id(iommu_group));
		ret = -EBUSY;
	} else {
		ret = iommu_take_ownership(tbl);
		if (!ret)
			container->tbl = tbl;
	}

	mutex_unlock(&container->lock);

	return ret;
}

static void tce_iommu_detach_group(void *iommu_data,
		struct iommu_group *iommu_group)
{
	struct tce_container *container = iommu_data;
	struct iommu_table *tbl = iommu_group_get_iommudata(iommu_group);

	BUG_ON(!tbl);
	mutex_lock(&container->lock);
	if (tbl != container->tbl) {
		pr_warn("tce_vfio: detaching group #%u, expected group is #%u\n",
				iommu_group_id(iommu_group),
				iommu_group_id(tbl->it_group));
	} else {
		if (container->enabled) {
			pr_warn("tce_vfio: detaching group #%u from enabled container, forcing disable\n",
					iommu_group_id(tbl->it_group));
			tce_iommu_disable(container);
		}

		/* pr_debug("tce_vfio: detaching group #%u from iommu %p\n",
				iommu_group_id(iommu_group), iommu_group); */
		container->tbl = NULL;
		iommu_release_ownership(tbl);
	}
	mutex_unlock(&container->lock);
}

const struct vfio_iommu_driver_ops tce_iommu_driver_ops = {
	.name		= "iommu-vfio-powerpc",
	.owner		= THIS_MODULE,
	.open		= tce_iommu_open,
	.release	= tce_iommu_release,
	.ioctl		= tce_iommu_ioctl,
	.attach_group	= tce_iommu_attach_group,
	.detach_group	= tce_iommu_detach_group,
};

static int __init tce_iommu_init(void)
{
	return vfio_register_iommu_driver(&tce_iommu_driver_ops);
}

static void __exit tce_iommu_cleanup(void)
{
	vfio_unregister_iommu_driver(&tce_iommu_driver_ops);
}

module_init(tce_iommu_init);
module_exit(tce_iommu_cleanup);

MODULE_VERSION(DRIVER_VERSION);
MODULE_LICENSE("GPL v2");
MODULE_AUTHOR(DRIVER_AUTHOR);
MODULE_DESCRIPTION(DRIVER_DESC);
Commit	Line	Data
5ffd229c AK	1	/*
	2	* VFIO: IOMMU DMA mapping support for TCE on POWER
	3	*
	4	* Copyright (C) 2013 IBM Corp. All rights reserved.
	5	* Author: Alexey Kardashevskiy <aik@ozlabs.ru>
	6	*
	7	* This program is free software; you can redistribute it and/or modify
	8	* it under the terms of the GNU General Public License version 2 as
	9	* published by the Free Software Foundation.
	10	*
	11	* Derived from original vfio_iommu_type1.c:
	12	* Copyright (C) 2012 Red Hat, Inc. All rights reserved.
	13	* Author: Alex Williamson <alex.williamson@redhat.com>
	14	*/
	15
	16	#include <linux/module.h>
	17	#include <linux/pci.h>
	18	#include <linux/slab.h>
	19	#include <linux/uaccess.h>
	20	#include <linux/err.h>
	21	#include <linux/vfio.h>
	22	#include <asm/iommu.h>
	23	#include <asm/tce.h>
	24
	25	#define DRIVER_VERSION "0.1"
	26	#define DRIVER_AUTHOR "aik@ozlabs.ru"
	27	#define DRIVER_DESC "VFIO IOMMU SPAPR TCE"
	28
	29	static void tce_iommu_detach_group(void *iommu_data,
	30	struct iommu_group *iommu_group);
	31
	32	/*
	33	* VFIO IOMMU fd for SPAPR_TCE IOMMU implementation
	34	*
	35	* This code handles mapping and unmapping of user data buffers
	36	* into DMA'ble space using the IOMMU
	37	*/
	38
	39	/*
	40	* The container descriptor supports only a single group per container.
	41	* Required by the API as the container is not supplied with the IOMMU group
	42	* at the moment of initialization.
	43	*/
	44	struct tce_container {
	45	struct mutex lock;
	46	struct iommu_table *tbl;
	47	bool enabled;
	48	};
	49
	50	static int tce_iommu_enable(struct tce_container *container)
	51	{
	52	int ret = 0;
	53	unsigned long locked, lock_limit, npages;
	54	struct iommu_table *tbl = container->tbl;
	55
	56	if (!container->tbl)
	57	return -ENXIO;
	58
	59	if (!current->mm)
	60	return -ESRCH; /* process exited */
	61
	62	if (container->enabled)
	63	return -EBUSY;
	64
65	/*
66	* When userspace pages are mapped into the IOMMU, they are effectively
67	* locked memory, so, theoretically, we need to update the accounting
68	* of locked pages on each map and unmap. For powerpc, the map unmap
69	* paths can be very hot, though, and the accounting would kill
70	* performance, especially since it would be difficult to impossible
71	* to handle the accounting in real mode only.
72	*
73	* To address that, rather than precisely accounting every page, we
74	* instead account for a worst case on locked memory when the iommu is
75	* enabled and disabled. The worst case upper bound on locked memory
76	* is the size of the whole iommu window, which is usually relatively
77	* small (compared to total memory sizes) on POWER hardware.
78	*
79	* Also we don't have a nice way to fail on H_PUT_TCE due to ulimits,
80	* that would effectively kill the guest at random points, much better
81	* enforcing the limit based on the max that the guest can map.
82	*/
83	down_write(&current->mm->mmap_sem);
e589a440	84	npages = (tbl->it_size << IOMMU_PAGE_SHIFT_4K) >> PAGE_SHIFT;
5ffd229c AK	85	locked = current->mm->locked_vm + npages;
	86	lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
	87	if (locked > lock_limit && !capable(CAP_IPC_LOCK)) {
	88	pr_warn("RLIMIT_MEMLOCK (%ld) exceeded\n",
	89	rlimit(RLIMIT_MEMLOCK));
	90	ret = -ENOMEM;
	91	} else {
	92
	93	current->mm->locked_vm += npages;
	94	container->enabled = true;
	95	}
	96	up_write(&current->mm->mmap_sem);
	97
	98	return ret;
	99	}
	100
	101	static void tce_iommu_disable(struct tce_container *container)
	102	{
	103	if (!container->enabled)
	104	return;
	105
	106	container->enabled = false;
	107
	108	if (!container->tbl \|\| !current->mm)
	109	return;
	110
	111	down_write(&current->mm->mmap_sem);
	112	current->mm->locked_vm -= (container->tbl->it_size <<
e589a440	113	IOMMU_PAGE_SHIFT_4K) >> PAGE_SHIFT;
5ffd229c AK	114	up_write(&current->mm->mmap_sem);
	115	}
	116
	117	static void *tce_iommu_open(unsigned long arg)
	118	{
	119	struct tce_container *container;
	120
	121	if (arg != VFIO_SPAPR_TCE_IOMMU) {
	122	pr_err("tce_vfio: Wrong IOMMU type\n");
	123	return ERR_PTR(-EINVAL);
	124	}
	125
	126	container = kzalloc(sizeof(*container), GFP_KERNEL);
	127	if (!container)
	128	return ERR_PTR(-ENOMEM);
	129
	130	mutex_init(&container->lock);
	131
	132	return container;
	133	}
	134
	135	static void tce_iommu_release(void *iommu_data)
	136	{
	137	struct tce_container *container = iommu_data;
	138
	139	WARN_ON(container->tbl && !container->tbl->it_group);
	140	tce_iommu_disable(container);
	141
	142	if (container->tbl && container->tbl->it_group)
	143	tce_iommu_detach_group(iommu_data, container->tbl->it_group);
	144
	145	mutex_destroy(&container->lock);
	146
	147	kfree(container);
	148	}
	149
	150	static long tce_iommu_ioctl(void *iommu_data,
	151	unsigned int cmd, unsigned long arg)
	152	{
	153	struct tce_container *container = iommu_data;
	154	unsigned long minsz;
	155	long ret;
	156
	157	switch (cmd) {
	158	case VFIO_CHECK_EXTENSION:
	159	return (arg == VFIO_SPAPR_TCE_IOMMU) ? 1 : 0;
	160
	161	case VFIO_IOMMU_SPAPR_TCE_GET_INFO: {
	162	struct vfio_iommu_spapr_tce_info info;
	163	struct iommu_table *tbl = container->tbl;
	164
	165	if (WARN_ON(!tbl))
	166	return -ENXIO;
	167
	168	minsz = offsetofend(struct vfio_iommu_spapr_tce_info,
	169	dma32_window_size);
	170
	171	if (copy_from_user(&info, (void __user *)arg, minsz))
	172	return -EFAULT;
	173
	174	if (info.argsz < minsz)
	175	return -EINVAL;
	176
e589a440 AP	177	info.dma32_window_start = tbl->it_offset << IOMMU_PAGE_SHIFT_4K;
e589a440 AP	178	info.dma32_window_size = tbl->it_size << IOMMU_PAGE_SHIFT_4K;
5ffd229c AK	179	info.flags = 0;
	180
	181	if (copy_to_user((void __user *)arg, &info, minsz))
	182	return -EFAULT;
	183
	184	return 0;
	185	}
	186	case VFIO_IOMMU_MAP_DMA: {
	187	struct vfio_iommu_type1_dma_map param;
	188	struct iommu_table *tbl = container->tbl;
	189	unsigned long tce, i;
	190
	191	if (!tbl)
	192	return -ENXIO;
	193
	194	BUG_ON(!tbl->it_group);
	195
	196	minsz = offsetofend(struct vfio_iommu_type1_dma_map, size);
	197
	198	if (copy_from_user(&param, (void __user *)arg, minsz))
	199	return -EFAULT;
	200
	201	if (param.argsz < minsz)
	202	return -EINVAL;
	203
	204	if (param.flags & ~(VFIO_DMA_MAP_FLAG_READ \|
	205	VFIO_DMA_MAP_FLAG_WRITE))
	206	return -EINVAL;
	207
e589a440 AP	208	if ((param.size & ~IOMMU_PAGE_MASK_4K) \|\|
e589a440 AP	209	(param.vaddr & ~IOMMU_PAGE_MASK_4K))
5ffd229c AK	210	return -EINVAL;
	211
	212	/* iova is checked by the IOMMU API */
	213	tce = param.vaddr;
	214	if (param.flags & VFIO_DMA_MAP_FLAG_READ)
	215	tce \|= TCE_PCI_READ;
	216	if (param.flags & VFIO_DMA_MAP_FLAG_WRITE)
	217	tce \|= TCE_PCI_WRITE;
	218
	219	ret = iommu_tce_put_param_check(tbl, param.iova, tce);
	220	if (ret)
	221	return ret;
	222
e589a440	223	for (i = 0; i < (param.size >> IOMMU_PAGE_SHIFT_4K); ++i) {
5ffd229c	224	ret = iommu_put_tce_user_mode(tbl,
e589a440	225	(param.iova >> IOMMU_PAGE_SHIFT_4K) + i,
5ffd229c AK	226	tce);
	227	if (ret)
	228	break;
e589a440	229	tce += IOMMU_PAGE_SIZE_4K;
5ffd229c AK	230	}
	231	if (ret)
	232	iommu_clear_tces_and_put_pages(tbl,
e589a440	233	param.iova >> IOMMU_PAGE_SHIFT_4K, i);
5ffd229c AK	234
	235	iommu_flush_tce(tbl);
	236
	237	return ret;
	238	}
	239	case VFIO_IOMMU_UNMAP_DMA: {
	240	struct vfio_iommu_type1_dma_unmap param;
	241	struct iommu_table *tbl = container->tbl;
	242
	243	if (WARN_ON(!tbl))
	244	return -ENXIO;
	245
	246	minsz = offsetofend(struct vfio_iommu_type1_dma_unmap,
	247	size);
	248
	249	if (copy_from_user(&param, (void __user *)arg, minsz))
	250	return -EFAULT;
	251
	252	if (param.argsz < minsz)
	253	return -EINVAL;
	254
	255	/* No flag is supported now */
	256	if (param.flags)
	257	return -EINVAL;
	258
e589a440	259	if (param.size & ~IOMMU_PAGE_MASK_4K)
5ffd229c AK	260	return -EINVAL;
	261
	262	ret = iommu_tce_clear_param_check(tbl, param.iova, 0,
e589a440	263	param.size >> IOMMU_PAGE_SHIFT_4K);
5ffd229c AK	264	if (ret)
	265	return ret;
	266
	267	ret = iommu_clear_tces_and_put_pages(tbl,
e589a440 AP	268	param.iova >> IOMMU_PAGE_SHIFT_4K,
e589a440 AP	269	param.size >> IOMMU_PAGE_SHIFT_4K);
5ffd229c AK	270	iommu_flush_tce(tbl);
	271
	272	return ret;
	273	}
	274	case VFIO_IOMMU_ENABLE:
	275	mutex_lock(&container->lock);
	276	ret = tce_iommu_enable(container);
	277	mutex_unlock(&container->lock);
	278	return ret;
	279
	280
	281	case VFIO_IOMMU_DISABLE:
	282	mutex_lock(&container->lock);
	283	tce_iommu_disable(container);
	284	mutex_unlock(&container->lock);
	285	return 0;
	286	}
	287
	288	return -ENOTTY;
	289	}
	290
	291	static int tce_iommu_attach_group(void *iommu_data,
	292	struct iommu_group *iommu_group)
	293	{
	294	int ret;
	295	struct tce_container *container = iommu_data;
	296	struct iommu_table *tbl = iommu_group_get_iommudata(iommu_group);
	297
	298	BUG_ON(!tbl);
	299	mutex_lock(&container->lock);
	300
	301	/* pr_debug("tce_vfio: Attaching group #%u to iommu %p\n",
	302	iommu_group_id(iommu_group), iommu_group); */
	303	if (container->tbl) {
	304	pr_warn("tce_vfio: Only one group per IOMMU container is allowed, existing id=%d, attaching id=%d\n",
	305	iommu_group_id(container->tbl->it_group),
	306	iommu_group_id(iommu_group));
	307	ret = -EBUSY;
	308	} else if (container->enabled) {
	309	pr_err("tce_vfio: attaching group #%u to enabled container\n",
	310	iommu_group_id(iommu_group));
	311	ret = -EBUSY;
	312	} else {
	313	ret = iommu_take_ownership(tbl);
	314	if (!ret)
	315	container->tbl = tbl;
	316	}
	317
	318	mutex_unlock(&container->lock);
	319
	320	return ret;
	321	}
	322
	323	static void tce_iommu_detach_group(void *iommu_data,
	324	struct iommu_group *iommu_group)
	325	{
	326	struct tce_container *container = iommu_data;
	327	struct iommu_table *tbl = iommu_group_get_iommudata(iommu_group);
	328
	329	BUG_ON(!tbl);
	330	mutex_lock(&container->lock);
	331	if (tbl != container->tbl) {
	332	pr_warn("tce_vfio: detaching group #%u, expected group is #%u\n",
	333	iommu_group_id(iommu_group),
334	iommu_group_id(tbl->it_group));
335	} else {
336	if (container->enabled) {
337	pr_warn("tce_vfio: detaching group #%u from enabled container, forcing disable\n",
338	iommu_group_id(tbl->it_group));
339	tce_iommu_disable(container);
340	}
341
342	/* pr_debug("tce_vfio: detaching group #%u from iommu %p\n",
343	iommu_group_id(iommu_group), iommu_group); */
344	container->tbl = NULL;
345	iommu_release_ownership(tbl);
346	}
347	mutex_unlock(&container->lock);
348	}
349
350	const struct vfio_iommu_driver_ops tce_iommu_driver_ops = {
351	.name = "iommu-vfio-powerpc",
352	.owner = THIS_MODULE,
353	.open = tce_iommu_open,
354	.release = tce_iommu_release,
355	.ioctl = tce_iommu_ioctl,
356	.attach_group = tce_iommu_attach_group,
357	.detach_group = tce_iommu_detach_group,
358	};
359
360	static int __init tce_iommu_init(void)
361	{
362	return vfio_register_iommu_driver(&tce_iommu_driver_ops);
363	}
364
365	static void __exit tce_iommu_cleanup(void)
366	{
367	vfio_unregister_iommu_driver(&tce_iommu_driver_ops);
368	}
369
370	module_init(tce_iommu_init);
371	module_exit(tce_iommu_cleanup);
372
373	MODULE_VERSION(DRIVER_VERSION);
374	MODULE_LICENSE("GPL v2");
375	MODULE_AUTHOR(DRIVER_AUTHOR);
376	MODULE_DESCRIPTION(DRIVER_DESC);
377