4 * Copyright (C) 2012 Red Hat, Inc. All rights reserved.
5 * Author: Alex Williamson <alex.williamson@redhat.com>
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
11 * Derived from original vfio:
12 * Copyright 2010 Cisco Systems, Inc. All rights reserved.
13 * Author: Tom Lyon, pugs@cisco.com
16 #include <linux/cdev.h>
17 #include <linux/compat.h>
18 #include <linux/device.h>
19 #include <linux/file.h>
20 #include <linux/anon_inodes.h>
22 #include <linux/idr.h>
23 #include <linux/iommu.h>
24 #include <linux/list.h>
25 #include <linux/miscdevice.h>
26 #include <linux/module.h>
27 #include <linux/mutex.h>
28 #include <linux/rwsem.h>
29 #include <linux/sched.h>
30 #include <linux/slab.h>
31 #include <linux/stat.h>
32 #include <linux/string.h>
33 #include <linux/uaccess.h>
34 #include <linux/vfio.h>
35 #include <linux/wait.h>
37 #define DRIVER_VERSION "0.3"
38 #define DRIVER_AUTHOR "Alex Williamson <alex.williamson@redhat.com>"
39 #define DRIVER_DESC "VFIO - User Level meta-driver"
43 struct list_head iommu_drivers_list
;
44 struct mutex iommu_drivers_lock
;
45 struct list_head group_list
;
47 struct mutex group_lock
;
48 struct cdev group_cdev
;
50 wait_queue_head_t release_q
;
53 struct vfio_iommu_driver
{
54 const struct vfio_iommu_driver_ops
*ops
;
55 struct list_head vfio_next
;
58 struct vfio_container
{
60 struct list_head group_list
;
61 struct rw_semaphore group_lock
;
62 struct vfio_iommu_driver
*iommu_driver
;
66 struct vfio_unbound_dev
{
68 struct list_head unbound_next
;
74 atomic_t container_users
;
75 struct iommu_group
*iommu_group
;
76 struct vfio_container
*container
;
77 struct list_head device_list
;
78 struct mutex device_lock
;
80 struct notifier_block nb
;
81 struct list_head vfio_next
;
82 struct list_head container_next
;
83 struct list_head unbound_list
;
84 struct mutex unbound_lock
;
91 const struct vfio_device_ops
*ops
;
92 struct vfio_group
*group
;
93 struct list_head group_next
;
98 * IOMMU driver registration
100 int vfio_register_iommu_driver(const struct vfio_iommu_driver_ops
*ops
)
102 struct vfio_iommu_driver
*driver
, *tmp
;
104 driver
= kzalloc(sizeof(*driver
), GFP_KERNEL
);
110 mutex_lock(&vfio
.iommu_drivers_lock
);
112 /* Check for duplicates */
113 list_for_each_entry(tmp
, &vfio
.iommu_drivers_list
, vfio_next
) {
114 if (tmp
->ops
== ops
) {
115 mutex_unlock(&vfio
.iommu_drivers_lock
);
121 list_add(&driver
->vfio_next
, &vfio
.iommu_drivers_list
);
123 mutex_unlock(&vfio
.iommu_drivers_lock
);
127 EXPORT_SYMBOL_GPL(vfio_register_iommu_driver
);
129 void vfio_unregister_iommu_driver(const struct vfio_iommu_driver_ops
*ops
)
131 struct vfio_iommu_driver
*driver
;
133 mutex_lock(&vfio
.iommu_drivers_lock
);
134 list_for_each_entry(driver
, &vfio
.iommu_drivers_list
, vfio_next
) {
135 if (driver
->ops
== ops
) {
136 list_del(&driver
->vfio_next
);
137 mutex_unlock(&vfio
.iommu_drivers_lock
);
142 mutex_unlock(&vfio
.iommu_drivers_lock
);
144 EXPORT_SYMBOL_GPL(vfio_unregister_iommu_driver
);
147 * Group minor allocation/free - both called with vfio.group_lock held
149 static int vfio_alloc_group_minor(struct vfio_group
*group
)
151 return idr_alloc(&vfio
.group_idr
, group
, 0, MINORMASK
+ 1, GFP_KERNEL
);
154 static void vfio_free_group_minor(int minor
)
156 idr_remove(&vfio
.group_idr
, minor
);
159 static int vfio_iommu_group_notifier(struct notifier_block
*nb
,
160 unsigned long action
, void *data
);
161 static void vfio_group_get(struct vfio_group
*group
);
164 * Container objects - containers are created when /dev/vfio/vfio is
165 * opened, but their lifecycle extends until the last user is done, so
166 * it's freed via kref. Must support container/group/device being
167 * closed in any order.
169 static void vfio_container_get(struct vfio_container
*container
)
171 kref_get(&container
->kref
);
174 static void vfio_container_release(struct kref
*kref
)
176 struct vfio_container
*container
;
177 container
= container_of(kref
, struct vfio_container
, kref
);
182 static void vfio_container_put(struct vfio_container
*container
)
184 kref_put(&container
->kref
, vfio_container_release
);
187 static void vfio_group_unlock_and_free(struct vfio_group
*group
)
189 mutex_unlock(&vfio
.group_lock
);
191 * Unregister outside of lock. A spurious callback is harmless now
192 * that the group is no longer in vfio.group_list.
194 iommu_group_unregister_notifier(group
->iommu_group
, &group
->nb
);
199 * Group objects - create, release, get, put, search
201 static struct vfio_group
*vfio_create_group(struct iommu_group
*iommu_group
)
203 struct vfio_group
*group
, *tmp
;
207 group
= kzalloc(sizeof(*group
), GFP_KERNEL
);
209 return ERR_PTR(-ENOMEM
);
211 kref_init(&group
->kref
);
212 INIT_LIST_HEAD(&group
->device_list
);
213 mutex_init(&group
->device_lock
);
214 INIT_LIST_HEAD(&group
->unbound_list
);
215 mutex_init(&group
->unbound_lock
);
216 atomic_set(&group
->container_users
, 0);
217 atomic_set(&group
->opened
, 0);
218 group
->iommu_group
= iommu_group
;
220 group
->nb
.notifier_call
= vfio_iommu_group_notifier
;
223 * blocking notifiers acquire a rwsem around registering and hold
224 * it around callback. Therefore, need to register outside of
225 * vfio.group_lock to avoid A-B/B-A contention. Our callback won't
226 * do anything unless it can find the group in vfio.group_list, so
227 * no harm in registering early.
229 ret
= iommu_group_register_notifier(iommu_group
, &group
->nb
);
235 mutex_lock(&vfio
.group_lock
);
237 /* Did we race creating this group? */
238 list_for_each_entry(tmp
, &vfio
.group_list
, vfio_next
) {
239 if (tmp
->iommu_group
== iommu_group
) {
241 vfio_group_unlock_and_free(group
);
246 minor
= vfio_alloc_group_minor(group
);
248 vfio_group_unlock_and_free(group
);
249 return ERR_PTR(minor
);
252 dev
= device_create(vfio
.class, NULL
,
253 MKDEV(MAJOR(vfio
.group_devt
), minor
),
254 group
, "%d", iommu_group_id(iommu_group
));
256 vfio_free_group_minor(minor
);
257 vfio_group_unlock_and_free(group
);
258 return (struct vfio_group
*)dev
; /* ERR_PTR */
261 group
->minor
= minor
;
264 list_add(&group
->vfio_next
, &vfio
.group_list
);
266 mutex_unlock(&vfio
.group_lock
);
271 /* called with vfio.group_lock held */
272 static void vfio_group_release(struct kref
*kref
)
274 struct vfio_group
*group
= container_of(kref
, struct vfio_group
, kref
);
275 struct vfio_unbound_dev
*unbound
, *tmp
;
276 struct iommu_group
*iommu_group
= group
->iommu_group
;
278 WARN_ON(!list_empty(&group
->device_list
));
280 list_for_each_entry_safe(unbound
, tmp
,
281 &group
->unbound_list
, unbound_next
) {
282 list_del(&unbound
->unbound_next
);
286 device_destroy(vfio
.class, MKDEV(MAJOR(vfio
.group_devt
), group
->minor
));
287 list_del(&group
->vfio_next
);
288 vfio_free_group_minor(group
->minor
);
289 vfio_group_unlock_and_free(group
);
290 iommu_group_put(iommu_group
);
293 static void vfio_group_put(struct vfio_group
*group
)
295 kref_put_mutex(&group
->kref
, vfio_group_release
, &vfio
.group_lock
);
298 /* Assume group_lock or group reference is held */
299 static void vfio_group_get(struct vfio_group
*group
)
301 kref_get(&group
->kref
);
305 * Not really a try as we will sleep for mutex, but we need to make
306 * sure the group pointer is valid under lock and get a reference.
308 static struct vfio_group
*vfio_group_try_get(struct vfio_group
*group
)
310 struct vfio_group
*target
= group
;
312 mutex_lock(&vfio
.group_lock
);
313 list_for_each_entry(group
, &vfio
.group_list
, vfio_next
) {
314 if (group
== target
) {
315 vfio_group_get(group
);
316 mutex_unlock(&vfio
.group_lock
);
320 mutex_unlock(&vfio
.group_lock
);
326 struct vfio_group
*vfio_group_get_from_iommu(struct iommu_group
*iommu_group
)
328 struct vfio_group
*group
;
330 mutex_lock(&vfio
.group_lock
);
331 list_for_each_entry(group
, &vfio
.group_list
, vfio_next
) {
332 if (group
->iommu_group
== iommu_group
) {
333 vfio_group_get(group
);
334 mutex_unlock(&vfio
.group_lock
);
338 mutex_unlock(&vfio
.group_lock
);
343 static struct vfio_group
*vfio_group_get_from_minor(int minor
)
345 struct vfio_group
*group
;
347 mutex_lock(&vfio
.group_lock
);
348 group
= idr_find(&vfio
.group_idr
, minor
);
350 mutex_unlock(&vfio
.group_lock
);
353 vfio_group_get(group
);
354 mutex_unlock(&vfio
.group_lock
);
360 * Device objects - create, release, get, put, search
363 struct vfio_device
*vfio_group_create_device(struct vfio_group
*group
,
365 const struct vfio_device_ops
*ops
,
368 struct vfio_device
*device
;
370 device
= kzalloc(sizeof(*device
), GFP_KERNEL
);
372 return ERR_PTR(-ENOMEM
);
374 kref_init(&device
->kref
);
376 device
->group
= group
;
378 device
->device_data
= device_data
;
379 dev_set_drvdata(dev
, device
);
381 /* No need to get group_lock, caller has group reference */
382 vfio_group_get(group
);
384 mutex_lock(&group
->device_lock
);
385 list_add(&device
->group_next
, &group
->device_list
);
386 mutex_unlock(&group
->device_lock
);
391 static void vfio_device_release(struct kref
*kref
)
393 struct vfio_device
*device
= container_of(kref
,
394 struct vfio_device
, kref
);
395 struct vfio_group
*group
= device
->group
;
397 list_del(&device
->group_next
);
398 mutex_unlock(&group
->device_lock
);
400 dev_set_drvdata(device
->dev
, NULL
);
404 /* vfio_del_group_dev may be waiting for this device */
405 wake_up(&vfio
.release_q
);
408 /* Device reference always implies a group reference */
409 void vfio_device_put(struct vfio_device
*device
)
411 struct vfio_group
*group
= device
->group
;
412 kref_put_mutex(&device
->kref
, vfio_device_release
, &group
->device_lock
);
413 vfio_group_put(group
);
415 EXPORT_SYMBOL_GPL(vfio_device_put
);
417 static void vfio_device_get(struct vfio_device
*device
)
419 vfio_group_get(device
->group
);
420 kref_get(&device
->kref
);
423 static struct vfio_device
*vfio_group_get_device(struct vfio_group
*group
,
426 struct vfio_device
*device
;
428 mutex_lock(&group
->device_lock
);
429 list_for_each_entry(device
, &group
->device_list
, group_next
) {
430 if (device
->dev
== dev
) {
431 vfio_device_get(device
);
432 mutex_unlock(&group
->device_lock
);
436 mutex_unlock(&group
->device_lock
);
441 * Whitelist some drivers that we know are safe (no dma) or just sit on
442 * a device. It's not always practical to leave a device within a group
443 * driverless as it could get re-bound to something unsafe.
445 static const char * const vfio_driver_whitelist
[] = { "pci-stub", "pcieport" };
447 static bool vfio_whitelisted_driver(struct device_driver
*drv
)
451 for (i
= 0; i
< ARRAY_SIZE(vfio_driver_whitelist
); i
++) {
452 if (!strcmp(drv
->name
, vfio_driver_whitelist
[i
]))
460 * A vfio group is viable for use by userspace if all devices are in
461 * one of the following states:
463 * - bound to a vfio driver
464 * - bound to a whitelisted driver
466 * We use two methods to determine whether a device is bound to a vfio
467 * driver. The first is to test whether the device exists in the vfio
468 * group. The second is to test if the device exists on the group
469 * unbound_list, indicating it's in the middle of transitioning from
470 * a vfio driver to driver-less.
472 static int vfio_dev_viable(struct device
*dev
, void *data
)
474 struct vfio_group
*group
= data
;
475 struct vfio_device
*device
;
476 struct device_driver
*drv
= ACCESS_ONCE(dev
->driver
);
477 struct vfio_unbound_dev
*unbound
;
480 mutex_lock(&group
->unbound_lock
);
481 list_for_each_entry(unbound
, &group
->unbound_list
, unbound_next
) {
482 if (dev
== unbound
->dev
) {
487 mutex_unlock(&group
->unbound_lock
);
489 if (!ret
|| !drv
|| vfio_whitelisted_driver(drv
))
492 device
= vfio_group_get_device(group
, dev
);
494 vfio_device_put(device
);
502 * Async device support
504 static int vfio_group_nb_add_dev(struct vfio_group
*group
, struct device
*dev
)
506 struct vfio_device
*device
;
508 /* Do we already know about it? We shouldn't */
509 device
= vfio_group_get_device(group
, dev
);
510 if (WARN_ON_ONCE(device
)) {
511 vfio_device_put(device
);
515 /* Nothing to do for idle groups */
516 if (!atomic_read(&group
->container_users
))
519 /* TODO Prevent device auto probing */
520 WARN("Device %s added to live group %d!\n", dev_name(dev
),
521 iommu_group_id(group
->iommu_group
));
526 static int vfio_group_nb_verify(struct vfio_group
*group
, struct device
*dev
)
528 /* We don't care what happens when the group isn't in use */
529 if (!atomic_read(&group
->container_users
))
532 return vfio_dev_viable(dev
, group
);
535 static int vfio_iommu_group_notifier(struct notifier_block
*nb
,
536 unsigned long action
, void *data
)
538 struct vfio_group
*group
= container_of(nb
, struct vfio_group
, nb
);
539 struct device
*dev
= data
;
540 struct vfio_unbound_dev
*unbound
;
543 * Need to go through a group_lock lookup to get a reference or we
544 * risk racing a group being removed. Ignore spurious notifies.
546 group
= vfio_group_try_get(group
);
551 case IOMMU_GROUP_NOTIFY_ADD_DEVICE
:
552 vfio_group_nb_add_dev(group
, dev
);
554 case IOMMU_GROUP_NOTIFY_DEL_DEVICE
:
556 * Nothing to do here. If the device is in use, then the
557 * vfio sub-driver should block the remove callback until
558 * it is unused. If the device is unused or attached to a
559 * stub driver, then it should be released and we don't
560 * care that it will be going away.
563 case IOMMU_GROUP_NOTIFY_BIND_DRIVER
:
564 pr_debug("%s: Device %s, group %d binding to driver\n",
565 __func__
, dev_name(dev
),
566 iommu_group_id(group
->iommu_group
));
568 case IOMMU_GROUP_NOTIFY_BOUND_DRIVER
:
569 pr_debug("%s: Device %s, group %d bound to driver %s\n",
570 __func__
, dev_name(dev
),
571 iommu_group_id(group
->iommu_group
), dev
->driver
->name
);
572 BUG_ON(vfio_group_nb_verify(group
, dev
));
574 case IOMMU_GROUP_NOTIFY_UNBIND_DRIVER
:
575 pr_debug("%s: Device %s, group %d unbinding from driver %s\n",
576 __func__
, dev_name(dev
),
577 iommu_group_id(group
->iommu_group
), dev
->driver
->name
);
579 case IOMMU_GROUP_NOTIFY_UNBOUND_DRIVER
:
580 pr_debug("%s: Device %s, group %d unbound from driver\n",
581 __func__
, dev_name(dev
),
582 iommu_group_id(group
->iommu_group
));
584 * XXX An unbound device in a live group is ok, but we'd
585 * really like to avoid the above BUG_ON by preventing other
586 * drivers from binding to it. Once that occurs, we have to
587 * stop the system to maintain isolation. At a minimum, we'd
588 * want a toggle to disable driver auto probe for this device.
591 mutex_lock(&group
->unbound_lock
);
592 list_for_each_entry(unbound
,
593 &group
->unbound_list
, unbound_next
) {
594 if (dev
== unbound
->dev
) {
595 list_del(&unbound
->unbound_next
);
600 mutex_unlock(&group
->unbound_lock
);
604 vfio_group_put(group
);
611 int vfio_add_group_dev(struct device
*dev
,
612 const struct vfio_device_ops
*ops
, void *device_data
)
614 struct iommu_group
*iommu_group
;
615 struct vfio_group
*group
;
616 struct vfio_device
*device
;
618 iommu_group
= iommu_group_get(dev
);
622 group
= vfio_group_get_from_iommu(iommu_group
);
624 group
= vfio_create_group(iommu_group
);
626 iommu_group_put(iommu_group
);
627 return PTR_ERR(group
);
631 * A found vfio_group already holds a reference to the
632 * iommu_group. A created vfio_group keeps the reference.
634 iommu_group_put(iommu_group
);
637 device
= vfio_group_get_device(group
, dev
);
639 WARN(1, "Device %s already exists on group %d\n",
640 dev_name(dev
), iommu_group_id(iommu_group
));
641 vfio_device_put(device
);
642 vfio_group_put(group
);
646 device
= vfio_group_create_device(group
, dev
, ops
, device_data
);
647 if (IS_ERR(device
)) {
648 vfio_group_put(group
);
649 return PTR_ERR(device
);
653 * Drop all but the vfio_device reference. The vfio_device holds
654 * a reference to the vfio_group, which holds a reference to the
657 vfio_group_put(group
);
661 EXPORT_SYMBOL_GPL(vfio_add_group_dev
);
664 * Get a reference to the vfio_device for a device. Even if the
665 * caller thinks they own the device, they could be racing with a
666 * release call path, so we can't trust drvdata for the shortcut.
667 * Go the long way around, from the iommu_group to the vfio_group
668 * to the vfio_device.
670 struct vfio_device
*vfio_device_get_from_dev(struct device
*dev
)
672 struct iommu_group
*iommu_group
;
673 struct vfio_group
*group
;
674 struct vfio_device
*device
;
676 iommu_group
= iommu_group_get(dev
);
680 group
= vfio_group_get_from_iommu(iommu_group
);
681 iommu_group_put(iommu_group
);
685 device
= vfio_group_get_device(group
, dev
);
686 vfio_group_put(group
);
690 EXPORT_SYMBOL_GPL(vfio_device_get_from_dev
);
693 * Caller must hold a reference to the vfio_device
695 void *vfio_device_data(struct vfio_device
*device
)
697 return device
->device_data
;
699 EXPORT_SYMBOL_GPL(vfio_device_data
);
701 /* Given a referenced group, check if it contains the device */
702 static bool vfio_dev_present(struct vfio_group
*group
, struct device
*dev
)
704 struct vfio_device
*device
;
706 device
= vfio_group_get_device(group
, dev
);
710 vfio_device_put(device
);
715 * Decrement the device reference count and wait for the device to be
716 * removed. Open file descriptors for the device... */
717 void *vfio_del_group_dev(struct device
*dev
)
719 struct vfio_device
*device
= dev_get_drvdata(dev
);
720 struct vfio_group
*group
= device
->group
;
721 void *device_data
= device
->device_data
;
722 struct vfio_unbound_dev
*unbound
;
725 bool interrupted
= false;
728 * The group exists so long as we have a device reference. Get
729 * a group reference and use it to scan for the device going away.
731 vfio_group_get(group
);
734 * When the device is removed from the group, the group suddenly
735 * becomes non-viable; the device has a driver (until the unbind
736 * completes), but it's not present in the group. This is bad news
737 * for any external users that need to re-acquire a group reference
738 * in order to match and release their existing reference. To
739 * solve this, we track such devices on the unbound_list to bridge
740 * the gap until they're fully unbound.
742 unbound
= kzalloc(sizeof(*unbound
), GFP_KERNEL
);
745 mutex_lock(&group
->unbound_lock
);
746 list_add(&unbound
->unbound_next
, &group
->unbound_list
);
747 mutex_unlock(&group
->unbound_lock
);
751 vfio_device_put(device
);
754 * If the device is still present in the group after the above
755 * 'put', then it is in use and we need to request it from the
756 * bus driver. The driver may in turn need to request the
757 * device from the user. We send the request on an arbitrary
758 * interval with counter to allow the driver to take escalating
759 * measures to release the device if it has the ability to do so.
762 device
= vfio_group_get_device(group
, dev
);
766 if (device
->ops
->request
)
767 device
->ops
->request(device_data
, i
++);
769 vfio_device_put(device
);
772 ret
= wait_event_timeout(vfio
.release_q
,
773 !vfio_dev_present(group
, dev
), HZ
* 10);
775 ret
= wait_event_interruptible_timeout(vfio
.release_q
,
776 !vfio_dev_present(group
, dev
), HZ
* 10);
777 if (ret
== -ERESTARTSYS
) {
780 "Device is currently in use, task"
782 "blocked until device is released",
783 current
->comm
, task_pid_nr(current
));
788 vfio_group_put(group
);
792 EXPORT_SYMBOL_GPL(vfio_del_group_dev
);
795 * VFIO base fd, /dev/vfio/vfio
797 static long vfio_ioctl_check_extension(struct vfio_container
*container
,
800 struct vfio_iommu_driver
*driver
;
803 down_read(&container
->group_lock
);
805 driver
= container
->iommu_driver
;
808 /* No base extensions yet */
811 * If no driver is set, poll all registered drivers for
812 * extensions and return the first positive result. If
813 * a driver is already set, further queries will be passed
814 * only to that driver.
817 mutex_lock(&vfio
.iommu_drivers_lock
);
818 list_for_each_entry(driver
, &vfio
.iommu_drivers_list
,
820 if (!try_module_get(driver
->ops
->owner
))
823 ret
= driver
->ops
->ioctl(NULL
,
824 VFIO_CHECK_EXTENSION
,
826 module_put(driver
->ops
->owner
);
830 mutex_unlock(&vfio
.iommu_drivers_lock
);
832 ret
= driver
->ops
->ioctl(container
->iommu_data
,
833 VFIO_CHECK_EXTENSION
, arg
);
836 up_read(&container
->group_lock
);
841 /* hold write lock on container->group_lock */
842 static int __vfio_container_attach_groups(struct vfio_container
*container
,
843 struct vfio_iommu_driver
*driver
,
846 struct vfio_group
*group
;
849 list_for_each_entry(group
, &container
->group_list
, container_next
) {
850 ret
= driver
->ops
->attach_group(data
, group
->iommu_group
);
858 list_for_each_entry_continue_reverse(group
, &container
->group_list
,
860 driver
->ops
->detach_group(data
, group
->iommu_group
);
866 static long vfio_ioctl_set_iommu(struct vfio_container
*container
,
869 struct vfio_iommu_driver
*driver
;
872 down_write(&container
->group_lock
);
875 * The container is designed to be an unprivileged interface while
876 * the group can be assigned to specific users. Therefore, only by
877 * adding a group to a container does the user get the privilege of
878 * enabling the iommu, which may allocate finite resources. There
879 * is no unset_iommu, but by removing all the groups from a container,
880 * the container is deprivileged and returns to an unset state.
882 if (list_empty(&container
->group_list
) || container
->iommu_driver
) {
883 up_write(&container
->group_lock
);
887 mutex_lock(&vfio
.iommu_drivers_lock
);
888 list_for_each_entry(driver
, &vfio
.iommu_drivers_list
, vfio_next
) {
891 if (!try_module_get(driver
->ops
->owner
))
895 * The arg magic for SET_IOMMU is the same as CHECK_EXTENSION,
896 * so test which iommu driver reported support for this
897 * extension and call open on them. We also pass them the
898 * magic, allowing a single driver to support multiple
899 * interfaces if they'd like.
901 if (driver
->ops
->ioctl(NULL
, VFIO_CHECK_EXTENSION
, arg
) <= 0) {
902 module_put(driver
->ops
->owner
);
906 /* module reference holds the driver we're working on */
907 mutex_unlock(&vfio
.iommu_drivers_lock
);
909 data
= driver
->ops
->open(arg
);
912 module_put(driver
->ops
->owner
);
913 goto skip_drivers_unlock
;
916 ret
= __vfio_container_attach_groups(container
, driver
, data
);
918 container
->iommu_driver
= driver
;
919 container
->iommu_data
= data
;
921 driver
->ops
->release(data
);
922 module_put(driver
->ops
->owner
);
925 goto skip_drivers_unlock
;
928 mutex_unlock(&vfio
.iommu_drivers_lock
);
930 up_write(&container
->group_lock
);
935 static long vfio_fops_unl_ioctl(struct file
*filep
,
936 unsigned int cmd
, unsigned long arg
)
938 struct vfio_container
*container
= filep
->private_data
;
939 struct vfio_iommu_driver
*driver
;
947 case VFIO_GET_API_VERSION
:
948 ret
= VFIO_API_VERSION
;
950 case VFIO_CHECK_EXTENSION
:
951 ret
= vfio_ioctl_check_extension(container
, arg
);
954 ret
= vfio_ioctl_set_iommu(container
, arg
);
957 down_read(&container
->group_lock
);
959 driver
= container
->iommu_driver
;
960 data
= container
->iommu_data
;
962 if (driver
) /* passthrough all unrecognized ioctls */
963 ret
= driver
->ops
->ioctl(data
, cmd
, arg
);
965 up_read(&container
->group_lock
);
972 static long vfio_fops_compat_ioctl(struct file
*filep
,
973 unsigned int cmd
, unsigned long arg
)
975 arg
= (unsigned long)compat_ptr(arg
);
976 return vfio_fops_unl_ioctl(filep
, cmd
, arg
);
978 #endif /* CONFIG_COMPAT */
980 static int vfio_fops_open(struct inode
*inode
, struct file
*filep
)
982 struct vfio_container
*container
;
984 container
= kzalloc(sizeof(*container
), GFP_KERNEL
);
988 INIT_LIST_HEAD(&container
->group_list
);
989 init_rwsem(&container
->group_lock
);
990 kref_init(&container
->kref
);
992 filep
->private_data
= container
;
997 static int vfio_fops_release(struct inode
*inode
, struct file
*filep
)
999 struct vfio_container
*container
= filep
->private_data
;
1001 filep
->private_data
= NULL
;
1003 vfio_container_put(container
);
1009 * Once an iommu driver is set, we optionally pass read/write/mmap
1010 * on to the driver, allowing management interfaces beyond ioctl.
1012 static ssize_t
vfio_fops_read(struct file
*filep
, char __user
*buf
,
1013 size_t count
, loff_t
*ppos
)
1015 struct vfio_container
*container
= filep
->private_data
;
1016 struct vfio_iommu_driver
*driver
;
1017 ssize_t ret
= -EINVAL
;
1019 down_read(&container
->group_lock
);
1021 driver
= container
->iommu_driver
;
1022 if (likely(driver
&& driver
->ops
->read
))
1023 ret
= driver
->ops
->read(container
->iommu_data
,
1026 up_read(&container
->group_lock
);
1031 static ssize_t
vfio_fops_write(struct file
*filep
, const char __user
*buf
,
1032 size_t count
, loff_t
*ppos
)
1034 struct vfio_container
*container
= filep
->private_data
;
1035 struct vfio_iommu_driver
*driver
;
1036 ssize_t ret
= -EINVAL
;
1038 down_read(&container
->group_lock
);
1040 driver
= container
->iommu_driver
;
1041 if (likely(driver
&& driver
->ops
->write
))
1042 ret
= driver
->ops
->write(container
->iommu_data
,
1045 up_read(&container
->group_lock
);
1050 static int vfio_fops_mmap(struct file
*filep
, struct vm_area_struct
*vma
)
1052 struct vfio_container
*container
= filep
->private_data
;
1053 struct vfio_iommu_driver
*driver
;
1056 down_read(&container
->group_lock
);
1058 driver
= container
->iommu_driver
;
1059 if (likely(driver
&& driver
->ops
->mmap
))
1060 ret
= driver
->ops
->mmap(container
->iommu_data
, vma
);
1062 up_read(&container
->group_lock
);
1067 static const struct file_operations vfio_fops
= {
1068 .owner
= THIS_MODULE
,
1069 .open
= vfio_fops_open
,
1070 .release
= vfio_fops_release
,
1071 .read
= vfio_fops_read
,
1072 .write
= vfio_fops_write
,
1073 .unlocked_ioctl
= vfio_fops_unl_ioctl
,
1074 #ifdef CONFIG_COMPAT
1075 .compat_ioctl
= vfio_fops_compat_ioctl
,
1077 .mmap
= vfio_fops_mmap
,
1081 * VFIO Group fd, /dev/vfio/$GROUP
1083 static void __vfio_group_unset_container(struct vfio_group
*group
)
1085 struct vfio_container
*container
= group
->container
;
1086 struct vfio_iommu_driver
*driver
;
1088 down_write(&container
->group_lock
);
1090 driver
= container
->iommu_driver
;
1092 driver
->ops
->detach_group(container
->iommu_data
,
1093 group
->iommu_group
);
1095 group
->container
= NULL
;
1096 list_del(&group
->container_next
);
1098 /* Detaching the last group deprivileges a container, remove iommu */
1099 if (driver
&& list_empty(&container
->group_list
)) {
1100 driver
->ops
->release(container
->iommu_data
);
1101 module_put(driver
->ops
->owner
);
1102 container
->iommu_driver
= NULL
;
1103 container
->iommu_data
= NULL
;
1106 up_write(&container
->group_lock
);
1108 vfio_container_put(container
);
1112 * VFIO_GROUP_UNSET_CONTAINER should fail if there are other users or
1113 * if there was no container to unset. Since the ioctl is called on
1114 * the group, we know that still exists, therefore the only valid
1115 * transition here is 1->0.
1117 static int vfio_group_unset_container(struct vfio_group
*group
)
1119 int users
= atomic_cmpxchg(&group
->container_users
, 1, 0);
1126 __vfio_group_unset_container(group
);
1132 * When removing container users, anything that removes the last user
1133 * implicitly removes the group from the container. That is, if the
1134 * group file descriptor is closed, as well as any device file descriptors,
1135 * the group is free.
1137 static void vfio_group_try_dissolve_container(struct vfio_group
*group
)
1139 if (0 == atomic_dec_if_positive(&group
->container_users
))
1140 __vfio_group_unset_container(group
);
1143 static int vfio_group_set_container(struct vfio_group
*group
, int container_fd
)
1146 struct vfio_container
*container
;
1147 struct vfio_iommu_driver
*driver
;
1150 if (atomic_read(&group
->container_users
))
1153 f
= fdget(container_fd
);
1157 /* Sanity check, is this really our fd? */
1158 if (f
.file
->f_op
!= &vfio_fops
) {
1163 container
= f
.file
->private_data
;
1164 WARN_ON(!container
); /* fget ensures we don't race vfio_release */
1166 down_write(&container
->group_lock
);
1168 driver
= container
->iommu_driver
;
1170 ret
= driver
->ops
->attach_group(container
->iommu_data
,
1171 group
->iommu_group
);
1176 group
->container
= container
;
1177 list_add(&group
->container_next
, &container
->group_list
);
1179 /* Get a reference on the container and mark a user within the group */
1180 vfio_container_get(container
);
1181 atomic_inc(&group
->container_users
);
1184 up_write(&container
->group_lock
);
1189 static bool vfio_group_viable(struct vfio_group
*group
)
1191 return (iommu_group_for_each_dev(group
->iommu_group
,
1192 group
, vfio_dev_viable
) == 0);
1195 static const struct file_operations vfio_device_fops
;
1197 static int vfio_group_get_device_fd(struct vfio_group
*group
, char *buf
)
1199 struct vfio_device
*device
;
1203 if (0 == atomic_read(&group
->container_users
) ||
1204 !group
->container
->iommu_driver
|| !vfio_group_viable(group
))
1207 mutex_lock(&group
->device_lock
);
1208 list_for_each_entry(device
, &group
->device_list
, group_next
) {
1209 if (strcmp(dev_name(device
->dev
), buf
))
1212 ret
= device
->ops
->open(device
->device_data
);
1216 * We can't use anon_inode_getfd() because we need to modify
1217 * the f_mode flags directly to allow more than just ioctls
1219 ret
= get_unused_fd_flags(O_CLOEXEC
);
1221 device
->ops
->release(device
->device_data
);
1225 filep
= anon_inode_getfile("[vfio-device]", &vfio_device_fops
,
1227 if (IS_ERR(filep
)) {
1229 ret
= PTR_ERR(filep
);
1230 device
->ops
->release(device
->device_data
);
1235 * TODO: add an anon_inode interface to do this.
1236 * Appears to be missing by lack of need rather than
1237 * explicitly prevented. Now there's need.
1239 filep
->f_mode
|= (FMODE_LSEEK
| FMODE_PREAD
| FMODE_PWRITE
);
1241 vfio_device_get(device
);
1242 atomic_inc(&group
->container_users
);
1244 fd_install(ret
, filep
);
1247 mutex_unlock(&group
->device_lock
);
1252 static long vfio_group_fops_unl_ioctl(struct file
*filep
,
1253 unsigned int cmd
, unsigned long arg
)
1255 struct vfio_group
*group
= filep
->private_data
;
1259 case VFIO_GROUP_GET_STATUS
:
1261 struct vfio_group_status status
;
1262 unsigned long minsz
;
1264 minsz
= offsetofend(struct vfio_group_status
, flags
);
1266 if (copy_from_user(&status
, (void __user
*)arg
, minsz
))
1269 if (status
.argsz
< minsz
)
1274 if (vfio_group_viable(group
))
1275 status
.flags
|= VFIO_GROUP_FLAGS_VIABLE
;
1277 if (group
->container
)
1278 status
.flags
|= VFIO_GROUP_FLAGS_CONTAINER_SET
;
1280 if (copy_to_user((void __user
*)arg
, &status
, minsz
))
1286 case VFIO_GROUP_SET_CONTAINER
:
1290 if (get_user(fd
, (int __user
*)arg
))
1296 ret
= vfio_group_set_container(group
, fd
);
1299 case VFIO_GROUP_UNSET_CONTAINER
:
1300 ret
= vfio_group_unset_container(group
);
1302 case VFIO_GROUP_GET_DEVICE_FD
:
1306 buf
= strndup_user((const char __user
*)arg
, PAGE_SIZE
);
1308 return PTR_ERR(buf
);
1310 ret
= vfio_group_get_device_fd(group
, buf
);
1319 #ifdef CONFIG_COMPAT
1320 static long vfio_group_fops_compat_ioctl(struct file
*filep
,
1321 unsigned int cmd
, unsigned long arg
)
1323 arg
= (unsigned long)compat_ptr(arg
);
1324 return vfio_group_fops_unl_ioctl(filep
, cmd
, arg
);
1326 #endif /* CONFIG_COMPAT */
1328 static int vfio_group_fops_open(struct inode
*inode
, struct file
*filep
)
1330 struct vfio_group
*group
;
1333 group
= vfio_group_get_from_minor(iminor(inode
));
1337 /* Do we need multiple instances of the group open? Seems not. */
1338 opened
= atomic_cmpxchg(&group
->opened
, 0, 1);
1340 vfio_group_put(group
);
1344 /* Is something still in use from a previous open? */
1345 if (group
->container
) {
1346 atomic_dec(&group
->opened
);
1347 vfio_group_put(group
);
1351 filep
->private_data
= group
;
1356 static int vfio_group_fops_release(struct inode
*inode
, struct file
*filep
)
1358 struct vfio_group
*group
= filep
->private_data
;
1360 filep
->private_data
= NULL
;
1362 vfio_group_try_dissolve_container(group
);
1364 atomic_dec(&group
->opened
);
1366 vfio_group_put(group
);
1371 static const struct file_operations vfio_group_fops
= {
1372 .owner
= THIS_MODULE
,
1373 .unlocked_ioctl
= vfio_group_fops_unl_ioctl
,
1374 #ifdef CONFIG_COMPAT
1375 .compat_ioctl
= vfio_group_fops_compat_ioctl
,
1377 .open
= vfio_group_fops_open
,
1378 .release
= vfio_group_fops_release
,
1384 static int vfio_device_fops_release(struct inode
*inode
, struct file
*filep
)
1386 struct vfio_device
*device
= filep
->private_data
;
1388 device
->ops
->release(device
->device_data
);
1390 vfio_group_try_dissolve_container(device
->group
);
1392 vfio_device_put(device
);
1397 static long vfio_device_fops_unl_ioctl(struct file
*filep
,
1398 unsigned int cmd
, unsigned long arg
)
1400 struct vfio_device
*device
= filep
->private_data
;
1402 if (unlikely(!device
->ops
->ioctl
))
1405 return device
->ops
->ioctl(device
->device_data
, cmd
, arg
);
1408 static ssize_t
vfio_device_fops_read(struct file
*filep
, char __user
*buf
,
1409 size_t count
, loff_t
*ppos
)
1411 struct vfio_device
*device
= filep
->private_data
;
1413 if (unlikely(!device
->ops
->read
))
1416 return device
->ops
->read(device
->device_data
, buf
, count
, ppos
);
1419 static ssize_t
vfio_device_fops_write(struct file
*filep
,
1420 const char __user
*buf
,
1421 size_t count
, loff_t
*ppos
)
1423 struct vfio_device
*device
= filep
->private_data
;
1425 if (unlikely(!device
->ops
->write
))
1428 return device
->ops
->write(device
->device_data
, buf
, count
, ppos
);
1431 static int vfio_device_fops_mmap(struct file
*filep
, struct vm_area_struct
*vma
)
1433 struct vfio_device
*device
= filep
->private_data
;
1435 if (unlikely(!device
->ops
->mmap
))
1438 return device
->ops
->mmap(device
->device_data
, vma
);
1441 #ifdef CONFIG_COMPAT
1442 static long vfio_device_fops_compat_ioctl(struct file
*filep
,
1443 unsigned int cmd
, unsigned long arg
)
1445 arg
= (unsigned long)compat_ptr(arg
);
1446 return vfio_device_fops_unl_ioctl(filep
, cmd
, arg
);
1448 #endif /* CONFIG_COMPAT */
1450 static const struct file_operations vfio_device_fops
= {
1451 .owner
= THIS_MODULE
,
1452 .release
= vfio_device_fops_release
,
1453 .read
= vfio_device_fops_read
,
1454 .write
= vfio_device_fops_write
,
1455 .unlocked_ioctl
= vfio_device_fops_unl_ioctl
,
1456 #ifdef CONFIG_COMPAT
1457 .compat_ioctl
= vfio_device_fops_compat_ioctl
,
1459 .mmap
= vfio_device_fops_mmap
,
1463 * External user API, exported by symbols to be linked dynamically.
1465 * The protocol includes:
1466 * 1. do normal VFIO init operation:
1467 * - opening a new container;
1468 * - attaching group(s) to it;
1469 * - setting an IOMMU driver for a container.
1470 * When IOMMU is set for a container, all groups in it are
1471 * considered ready to use by an external user.
1473 * 2. User space passes a group fd to an external user.
1474 * The external user calls vfio_group_get_external_user()
1476 * - the group is initialized;
1477 * - IOMMU is set for it.
1478 * If both checks passed, vfio_group_get_external_user()
1479 * increments the container user counter to prevent
1480 * the VFIO group from disposal before KVM exits.
1482 * 3. The external user calls vfio_external_user_iommu_id()
1483 * to know an IOMMU ID.
1485 * 4. When the external KVM finishes, it calls
1486 * vfio_group_put_external_user() to release the VFIO group.
1487 * This call decrements the container user counter.
1489 struct vfio_group
*vfio_group_get_external_user(struct file
*filep
)
1491 struct vfio_group
*group
= filep
->private_data
;
1493 if (filep
->f_op
!= &vfio_group_fops
)
1494 return ERR_PTR(-EINVAL
);
1496 if (!atomic_inc_not_zero(&group
->container_users
))
1497 return ERR_PTR(-EINVAL
);
1499 if (!group
->container
->iommu_driver
||
1500 !vfio_group_viable(group
)) {
1501 atomic_dec(&group
->container_users
);
1502 return ERR_PTR(-EINVAL
);
1505 vfio_group_get(group
);
1509 EXPORT_SYMBOL_GPL(vfio_group_get_external_user
);
1511 void vfio_group_put_external_user(struct vfio_group
*group
)
1513 vfio_group_put(group
);
1514 vfio_group_try_dissolve_container(group
);
1516 EXPORT_SYMBOL_GPL(vfio_group_put_external_user
);
1518 int vfio_external_user_iommu_id(struct vfio_group
*group
)
1520 return iommu_group_id(group
->iommu_group
);
1522 EXPORT_SYMBOL_GPL(vfio_external_user_iommu_id
);
1524 long vfio_external_check_extension(struct vfio_group
*group
, unsigned long arg
)
1526 return vfio_ioctl_check_extension(group
->container
, arg
);
1528 EXPORT_SYMBOL_GPL(vfio_external_check_extension
);
1531 * Module/class support
1533 static char *vfio_devnode(struct device
*dev
, umode_t
*mode
)
1535 return kasprintf(GFP_KERNEL
, "vfio/%s", dev_name(dev
));
1538 static struct miscdevice vfio_dev
= {
1539 .minor
= VFIO_MINOR
,
1542 .nodename
= "vfio/vfio",
1543 .mode
= S_IRUGO
| S_IWUGO
,
1546 static int __init
vfio_init(void)
1550 idr_init(&vfio
.group_idr
);
1551 mutex_init(&vfio
.group_lock
);
1552 mutex_init(&vfio
.iommu_drivers_lock
);
1553 INIT_LIST_HEAD(&vfio
.group_list
);
1554 INIT_LIST_HEAD(&vfio
.iommu_drivers_list
);
1555 init_waitqueue_head(&vfio
.release_q
);
1557 ret
= misc_register(&vfio_dev
);
1559 pr_err("vfio: misc device register failed\n");
1563 /* /dev/vfio/$GROUP */
1564 vfio
.class = class_create(THIS_MODULE
, "vfio");
1565 if (IS_ERR(vfio
.class)) {
1566 ret
= PTR_ERR(vfio
.class);
1570 vfio
.class->devnode
= vfio_devnode
;
1572 ret
= alloc_chrdev_region(&vfio
.group_devt
, 0, MINORMASK
, "vfio");
1574 goto err_alloc_chrdev
;
1576 cdev_init(&vfio
.group_cdev
, &vfio_group_fops
);
1577 ret
= cdev_add(&vfio
.group_cdev
, vfio
.group_devt
, MINORMASK
);
1581 pr_info(DRIVER_DESC
" version: " DRIVER_VERSION
"\n");
1584 * Attempt to load known iommu-drivers. This gives us a working
1585 * environment without the user needing to explicitly load iommu
1588 request_module_nowait("vfio_iommu_type1");
1589 request_module_nowait("vfio_iommu_spapr_tce");
1594 unregister_chrdev_region(vfio
.group_devt
, MINORMASK
);
1596 class_destroy(vfio
.class);
1599 misc_deregister(&vfio_dev
);
1603 static void __exit
vfio_cleanup(void)
1605 WARN_ON(!list_empty(&vfio
.group_list
));
1607 idr_destroy(&vfio
.group_idr
);
1608 cdev_del(&vfio
.group_cdev
);
1609 unregister_chrdev_region(vfio
.group_devt
, MINORMASK
);
1610 class_destroy(vfio
.class);
1612 misc_deregister(&vfio_dev
);
1615 module_init(vfio_init
);
1616 module_exit(vfio_cleanup
);
1618 MODULE_VERSION(DRIVER_VERSION
);
1619 MODULE_LICENSE("GPL v2");
1620 MODULE_AUTHOR(DRIVER_AUTHOR
);
1621 MODULE_DESCRIPTION(DRIVER_DESC
);
1622 MODULE_ALIAS_MISCDEV(VFIO_MINOR
);
1623 MODULE_ALIAS("devname:vfio/vfio");