Commit | Line | Data |
---|---|---|
721eecbf GH |
1 | /* |
2 | * kvm eventfd support - use eventfd objects to signal various KVM events | |
3 | * | |
4 | * Copyright 2009 Novell. All Rights Reserved. | |
221d059d | 5 | * Copyright 2010 Red Hat, Inc. and/or its affiliates. |
721eecbf GH |
6 | * |
7 | * Author: | |
8 | * Gregory Haskins <ghaskins@novell.com> | |
9 | * | |
10 | * This file is free software; you can redistribute it and/or modify | |
11 | * it under the terms of version 2 of the GNU General Public License | |
12 | * as published by the Free Software Foundation. | |
13 | * | |
14 | * This program is distributed in the hope that it will be useful, | |
15 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
16 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
17 | * GNU General Public License for more details. | |
18 | * | |
19 | * You should have received a copy of the GNU General Public License | |
20 | * along with this program; if not, write to the Free Software Foundation, | |
21 | * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA. | |
22 | */ | |
23 | ||
24 | #include <linux/kvm_host.h> | |
d34e6b17 | 25 | #include <linux/kvm.h> |
721eecbf GH |
26 | #include <linux/workqueue.h> |
27 | #include <linux/syscalls.h> | |
28 | #include <linux/wait.h> | |
29 | #include <linux/poll.h> | |
30 | #include <linux/file.h> | |
31 | #include <linux/list.h> | |
32 | #include <linux/eventfd.h> | |
d34e6b17 | 33 | #include <linux/kernel.h> |
5a0e3ad6 | 34 | #include <linux/slab.h> |
d34e6b17 GH |
35 | |
36 | #include "iodev.h" | |
721eecbf | 37 | |
914daba8 | 38 | #ifdef __KVM_HAVE_IOAPIC |
721eecbf GH |
39 | /* |
40 | * -------------------------------------------------------------------- | |
41 | * irqfd: Allows an fd to be used to inject an interrupt to the guest | |
42 | * | |
43 | * Credit goes to Avi Kivity for the original idea. | |
44 | * -------------------------------------------------------------------- | |
45 | */ | |
46 | ||
7a84428a AW |
47 | /* |
48 | * Resampling irqfds are a special variety of irqfds used to emulate | |
49 | * level triggered interrupts. The interrupt is asserted on eventfd | |
50 | * trigger. On acknowledgement through the irq ack notifier, the | |
51 | * interrupt is de-asserted and userspace is notified through the | |
52 | * resamplefd. All resamplers on the same gsi are de-asserted | |
53 | * together, so we don't need to track the state of each individual | |
54 | * user. We can also therefore share the same irq source ID. | |
55 | */ | |
56 | struct _irqfd_resampler { | |
57 | struct kvm *kvm; | |
58 | /* | |
59 | * List of resampling struct _irqfd objects sharing this gsi. | |
60 | * RCU list modified under kvm->irqfds.resampler_lock | |
61 | */ | |
62 | struct list_head list; | |
63 | struct kvm_irq_ack_notifier notifier; | |
64 | /* | |
65 | * Entry in list of kvm->irqfd.resampler_list. Use for sharing | |
66 | * resamplers among irqfds on the same gsi. | |
67 | * Accessed and modified under kvm->irqfds.resampler_lock | |
68 | */ | |
69 | struct list_head link; | |
70 | }; | |
71 | ||
721eecbf | 72 | struct _irqfd { |
bd2b53b2 MT |
73 | /* Used for MSI fast-path */ |
74 | struct kvm *kvm; | |
75 | wait_queue_t wait; | |
76 | /* Update side is protected by irqfds.lock */ | |
77 | struct kvm_kernel_irq_routing_entry __rcu *irq_entry; | |
78 | /* Used for level IRQ fast-path */ | |
79 | int gsi; | |
80 | struct work_struct inject; | |
7a84428a AW |
81 | /* The resampler used by this irqfd (resampler-only) */ |
82 | struct _irqfd_resampler *resampler; | |
83 | /* Eventfd notified on resample (resampler-only) */ | |
84 | struct eventfd_ctx *resamplefd; | |
85 | /* Entry in list of irqfds for a resampler (resampler-only) */ | |
86 | struct list_head resampler_link; | |
bd2b53b2 MT |
87 | /* Used for setup/shutdown */ |
88 | struct eventfd_ctx *eventfd; | |
89 | struct list_head list; | |
90 | poll_table pt; | |
91 | struct work_struct shutdown; | |
721eecbf GH |
92 | }; |
93 | ||
94 | static struct workqueue_struct *irqfd_cleanup_wq; | |
95 | ||
96 | static void | |
97 | irqfd_inject(struct work_struct *work) | |
98 | { | |
99 | struct _irqfd *irqfd = container_of(work, struct _irqfd, inject); | |
100 | struct kvm *kvm = irqfd->kvm; | |
101 | ||
7a84428a AW |
102 | if (!irqfd->resampler) { |
103 | kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID, irqfd->gsi, 1); | |
104 | kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID, irqfd->gsi, 0); | |
105 | } else | |
106 | kvm_set_irq(kvm, KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID, | |
107 | irqfd->gsi, 1); | |
108 | } | |
109 | ||
110 | /* | |
111 | * Since resampler irqfds share an IRQ source ID, we de-assert once | |
112 | * then notify all of the resampler irqfds using this GSI. We can't | |
113 | * do multiple de-asserts or we risk racing with incoming re-asserts. | |
114 | */ | |
115 | static void | |
116 | irqfd_resampler_ack(struct kvm_irq_ack_notifier *kian) | |
117 | { | |
118 | struct _irqfd_resampler *resampler; | |
119 | struct _irqfd *irqfd; | |
120 | ||
121 | resampler = container_of(kian, struct _irqfd_resampler, notifier); | |
122 | ||
123 | kvm_set_irq(resampler->kvm, KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID, | |
124 | resampler->notifier.gsi, 0); | |
125 | ||
126 | rcu_read_lock(); | |
127 | ||
128 | list_for_each_entry_rcu(irqfd, &resampler->list, resampler_link) | |
129 | eventfd_signal(irqfd->resamplefd, 1); | |
130 | ||
131 | rcu_read_unlock(); | |
132 | } | |
133 | ||
134 | static void | |
135 | irqfd_resampler_shutdown(struct _irqfd *irqfd) | |
136 | { | |
137 | struct _irqfd_resampler *resampler = irqfd->resampler; | |
138 | struct kvm *kvm = resampler->kvm; | |
139 | ||
140 | mutex_lock(&kvm->irqfds.resampler_lock); | |
141 | ||
142 | list_del_rcu(&irqfd->resampler_link); | |
143 | synchronize_rcu(); | |
144 | ||
145 | if (list_empty(&resampler->list)) { | |
146 | list_del(&resampler->link); | |
147 | kvm_unregister_irq_ack_notifier(kvm, &resampler->notifier); | |
148 | kvm_set_irq(kvm, KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID, | |
149 | resampler->notifier.gsi, 0); | |
150 | kfree(resampler); | |
151 | } | |
152 | ||
153 | mutex_unlock(&kvm->irqfds.resampler_lock); | |
721eecbf GH |
154 | } |
155 | ||
156 | /* | |
157 | * Race-free decouple logic (ordering is critical) | |
158 | */ | |
159 | static void | |
160 | irqfd_shutdown(struct work_struct *work) | |
161 | { | |
162 | struct _irqfd *irqfd = container_of(work, struct _irqfd, shutdown); | |
b6a114d2 | 163 | u64 cnt; |
721eecbf GH |
164 | |
165 | /* | |
166 | * Synchronize with the wait-queue and unhook ourselves to prevent | |
167 | * further events. | |
168 | */ | |
b6a114d2 | 169 | eventfd_ctx_remove_wait_queue(irqfd->eventfd, &irqfd->wait, &cnt); |
721eecbf GH |
170 | |
171 | /* | |
172 | * We know no new events will be scheduled at this point, so block | |
173 | * until all previously outstanding events have completed | |
174 | */ | |
43829731 | 175 | flush_work(&irqfd->inject); |
721eecbf | 176 | |
7a84428a AW |
177 | if (irqfd->resampler) { |
178 | irqfd_resampler_shutdown(irqfd); | |
179 | eventfd_ctx_put(irqfd->resamplefd); | |
180 | } | |
181 | ||
721eecbf GH |
182 | /* |
183 | * It is now safe to release the object's resources | |
184 | */ | |
185 | eventfd_ctx_put(irqfd->eventfd); | |
186 | kfree(irqfd); | |
187 | } | |
188 | ||
189 | ||
190 | /* assumes kvm->irqfds.lock is held */ | |
191 | static bool | |
192 | irqfd_is_active(struct _irqfd *irqfd) | |
193 | { | |
194 | return list_empty(&irqfd->list) ? false : true; | |
195 | } | |
196 | ||
197 | /* | |
198 | * Mark the irqfd as inactive and schedule it for removal | |
199 | * | |
200 | * assumes kvm->irqfds.lock is held | |
201 | */ | |
202 | static void | |
203 | irqfd_deactivate(struct _irqfd *irqfd) | |
204 | { | |
205 | BUG_ON(!irqfd_is_active(irqfd)); | |
206 | ||
207 | list_del_init(&irqfd->list); | |
208 | ||
209 | queue_work(irqfd_cleanup_wq, &irqfd->shutdown); | |
210 | } | |
211 | ||
212 | /* | |
213 | * Called with wqh->lock held and interrupts disabled | |
214 | */ | |
215 | static int | |
216 | irqfd_wakeup(wait_queue_t *wait, unsigned mode, int sync, void *key) | |
217 | { | |
218 | struct _irqfd *irqfd = container_of(wait, struct _irqfd, wait); | |
219 | unsigned long flags = (unsigned long)key; | |
bd2b53b2 MT |
220 | struct kvm_kernel_irq_routing_entry *irq; |
221 | struct kvm *kvm = irqfd->kvm; | |
721eecbf | 222 | |
bd2b53b2 MT |
223 | if (flags & POLLIN) { |
224 | rcu_read_lock(); | |
225 | irq = rcu_dereference(irqfd->irq_entry); | |
721eecbf | 226 | /* An event has been signaled, inject an interrupt */ |
bd2b53b2 MT |
227 | if (irq) |
228 | kvm_set_msi(irq, kvm, KVM_USERSPACE_IRQ_SOURCE_ID, 1); | |
229 | else | |
230 | schedule_work(&irqfd->inject); | |
231 | rcu_read_unlock(); | |
232 | } | |
721eecbf GH |
233 | |
234 | if (flags & POLLHUP) { | |
235 | /* The eventfd is closing, detach from KVM */ | |
721eecbf GH |
236 | unsigned long flags; |
237 | ||
238 | spin_lock_irqsave(&kvm->irqfds.lock, flags); | |
239 | ||
240 | /* | |
241 | * We must check if someone deactivated the irqfd before | |
242 | * we could acquire the irqfds.lock since the item is | |
243 | * deactivated from the KVM side before it is unhooked from | |
244 | * the wait-queue. If it is already deactivated, we can | |
245 | * simply return knowing the other side will cleanup for us. | |
246 | * We cannot race against the irqfd going away since the | |
247 | * other side is required to acquire wqh->lock, which we hold | |
248 | */ | |
249 | if (irqfd_is_active(irqfd)) | |
250 | irqfd_deactivate(irqfd); | |
251 | ||
252 | spin_unlock_irqrestore(&kvm->irqfds.lock, flags); | |
253 | } | |
254 | ||
255 | return 0; | |
256 | } | |
257 | ||
258 | static void | |
259 | irqfd_ptable_queue_proc(struct file *file, wait_queue_head_t *wqh, | |
260 | poll_table *pt) | |
261 | { | |
262 | struct _irqfd *irqfd = container_of(pt, struct _irqfd, pt); | |
721eecbf GH |
263 | add_wait_queue(wqh, &irqfd->wait); |
264 | } | |
265 | ||
bd2b53b2 MT |
266 | /* Must be called under irqfds.lock */ |
267 | static void irqfd_update(struct kvm *kvm, struct _irqfd *irqfd, | |
268 | struct kvm_irq_routing_table *irq_rt) | |
269 | { | |
270 | struct kvm_kernel_irq_routing_entry *e; | |
bd2b53b2 MT |
271 | |
272 | if (irqfd->gsi >= irq_rt->nr_rt_entries) { | |
273 | rcu_assign_pointer(irqfd->irq_entry, NULL); | |
274 | return; | |
275 | } | |
276 | ||
b67bfe0d | 277 | hlist_for_each_entry(e, &irq_rt->map[irqfd->gsi], link) { |
bd2b53b2 MT |
278 | /* Only fast-path MSI. */ |
279 | if (e->type == KVM_IRQ_ROUTING_MSI) | |
280 | rcu_assign_pointer(irqfd->irq_entry, e); | |
281 | else | |
282 | rcu_assign_pointer(irqfd->irq_entry, NULL); | |
283 | } | |
284 | } | |
285 | ||
721eecbf | 286 | static int |
d4db2935 | 287 | kvm_irqfd_assign(struct kvm *kvm, struct kvm_irqfd *args) |
721eecbf | 288 | { |
bd2b53b2 | 289 | struct kvm_irq_routing_table *irq_rt; |
f1d1c309 | 290 | struct _irqfd *irqfd, *tmp; |
721eecbf | 291 | struct file *file = NULL; |
7a84428a | 292 | struct eventfd_ctx *eventfd = NULL, *resamplefd = NULL; |
721eecbf GH |
293 | int ret; |
294 | unsigned int events; | |
295 | ||
296 | irqfd = kzalloc(sizeof(*irqfd), GFP_KERNEL); | |
297 | if (!irqfd) | |
298 | return -ENOMEM; | |
299 | ||
300 | irqfd->kvm = kvm; | |
d4db2935 | 301 | irqfd->gsi = args->gsi; |
721eecbf GH |
302 | INIT_LIST_HEAD(&irqfd->list); |
303 | INIT_WORK(&irqfd->inject, irqfd_inject); | |
304 | INIT_WORK(&irqfd->shutdown, irqfd_shutdown); | |
305 | ||
d4db2935 | 306 | file = eventfd_fget(args->fd); |
721eecbf GH |
307 | if (IS_ERR(file)) { |
308 | ret = PTR_ERR(file); | |
309 | goto fail; | |
310 | } | |
311 | ||
312 | eventfd = eventfd_ctx_fileget(file); | |
313 | if (IS_ERR(eventfd)) { | |
314 | ret = PTR_ERR(eventfd); | |
315 | goto fail; | |
316 | } | |
317 | ||
318 | irqfd->eventfd = eventfd; | |
319 | ||
7a84428a AW |
320 | if (args->flags & KVM_IRQFD_FLAG_RESAMPLE) { |
321 | struct _irqfd_resampler *resampler; | |
322 | ||
323 | resamplefd = eventfd_ctx_fdget(args->resamplefd); | |
324 | if (IS_ERR(resamplefd)) { | |
325 | ret = PTR_ERR(resamplefd); | |
326 | goto fail; | |
327 | } | |
328 | ||
329 | irqfd->resamplefd = resamplefd; | |
330 | INIT_LIST_HEAD(&irqfd->resampler_link); | |
331 | ||
332 | mutex_lock(&kvm->irqfds.resampler_lock); | |
333 | ||
334 | list_for_each_entry(resampler, | |
49f8a1a5 | 335 | &kvm->irqfds.resampler_list, link) { |
7a84428a AW |
336 | if (resampler->notifier.gsi == irqfd->gsi) { |
337 | irqfd->resampler = resampler; | |
338 | break; | |
339 | } | |
340 | } | |
341 | ||
342 | if (!irqfd->resampler) { | |
343 | resampler = kzalloc(sizeof(*resampler), GFP_KERNEL); | |
344 | if (!resampler) { | |
345 | ret = -ENOMEM; | |
346 | mutex_unlock(&kvm->irqfds.resampler_lock); | |
347 | goto fail; | |
348 | } | |
349 | ||
350 | resampler->kvm = kvm; | |
351 | INIT_LIST_HEAD(&resampler->list); | |
352 | resampler->notifier.gsi = irqfd->gsi; | |
353 | resampler->notifier.irq_acked = irqfd_resampler_ack; | |
354 | INIT_LIST_HEAD(&resampler->link); | |
355 | ||
356 | list_add(&resampler->link, &kvm->irqfds.resampler_list); | |
357 | kvm_register_irq_ack_notifier(kvm, | |
358 | &resampler->notifier); | |
359 | irqfd->resampler = resampler; | |
360 | } | |
361 | ||
362 | list_add_rcu(&irqfd->resampler_link, &irqfd->resampler->list); | |
363 | synchronize_rcu(); | |
364 | ||
365 | mutex_unlock(&kvm->irqfds.resampler_lock); | |
366 | } | |
367 | ||
721eecbf GH |
368 | /* |
369 | * Install our own custom wake-up handling so we are notified via | |
370 | * a callback whenever someone signals the underlying eventfd | |
371 | */ | |
372 | init_waitqueue_func_entry(&irqfd->wait, irqfd_wakeup); | |
373 | init_poll_funcptr(&irqfd->pt, irqfd_ptable_queue_proc); | |
374 | ||
f1d1c309 MT |
375 | spin_lock_irq(&kvm->irqfds.lock); |
376 | ||
377 | ret = 0; | |
378 | list_for_each_entry(tmp, &kvm->irqfds.items, list) { | |
379 | if (irqfd->eventfd != tmp->eventfd) | |
380 | continue; | |
381 | /* This fd is used for another irq already. */ | |
382 | ret = -EBUSY; | |
383 | spin_unlock_irq(&kvm->irqfds.lock); | |
384 | goto fail; | |
385 | } | |
386 | ||
bd2b53b2 MT |
387 | irq_rt = rcu_dereference_protected(kvm->irq_routing, |
388 | lockdep_is_held(&kvm->irqfds.lock)); | |
389 | irqfd_update(kvm, irqfd, irq_rt); | |
390 | ||
721eecbf GH |
391 | events = file->f_op->poll(file, &irqfd->pt); |
392 | ||
721eecbf | 393 | list_add_tail(&irqfd->list, &kvm->irqfds.items); |
721eecbf GH |
394 | |
395 | /* | |
396 | * Check if there was an event already pending on the eventfd | |
397 | * before we registered, and trigger it as if we didn't miss it. | |
398 | */ | |
399 | if (events & POLLIN) | |
400 | schedule_work(&irqfd->inject); | |
401 | ||
6bbfb265 MT |
402 | spin_unlock_irq(&kvm->irqfds.lock); |
403 | ||
721eecbf GH |
404 | /* |
405 | * do not drop the file until the irqfd is fully initialized, otherwise | |
406 | * we might race against the POLLHUP | |
407 | */ | |
408 | fput(file); | |
409 | ||
410 | return 0; | |
411 | ||
412 | fail: | |
7a84428a AW |
413 | if (irqfd->resampler) |
414 | irqfd_resampler_shutdown(irqfd); | |
415 | ||
416 | if (resamplefd && !IS_ERR(resamplefd)) | |
417 | eventfd_ctx_put(resamplefd); | |
418 | ||
721eecbf GH |
419 | if (eventfd && !IS_ERR(eventfd)) |
420 | eventfd_ctx_put(eventfd); | |
421 | ||
6223011f | 422 | if (!IS_ERR(file)) |
721eecbf GH |
423 | fput(file); |
424 | ||
425 | kfree(irqfd); | |
426 | return ret; | |
427 | } | |
914daba8 | 428 | #endif |
721eecbf GH |
429 | |
430 | void | |
d34e6b17 | 431 | kvm_eventfd_init(struct kvm *kvm) |
721eecbf | 432 | { |
914daba8 | 433 | #ifdef __KVM_HAVE_IOAPIC |
721eecbf GH |
434 | spin_lock_init(&kvm->irqfds.lock); |
435 | INIT_LIST_HEAD(&kvm->irqfds.items); | |
7a84428a AW |
436 | INIT_LIST_HEAD(&kvm->irqfds.resampler_list); |
437 | mutex_init(&kvm->irqfds.resampler_lock); | |
914daba8 | 438 | #endif |
d34e6b17 | 439 | INIT_LIST_HEAD(&kvm->ioeventfds); |
721eecbf GH |
440 | } |
441 | ||
914daba8 | 442 | #ifdef __KVM_HAVE_IOAPIC |
721eecbf GH |
443 | /* |
444 | * shutdown any irqfd's that match fd+gsi | |
445 | */ | |
446 | static int | |
d4db2935 | 447 | kvm_irqfd_deassign(struct kvm *kvm, struct kvm_irqfd *args) |
721eecbf GH |
448 | { |
449 | struct _irqfd *irqfd, *tmp; | |
450 | struct eventfd_ctx *eventfd; | |
451 | ||
d4db2935 | 452 | eventfd = eventfd_ctx_fdget(args->fd); |
721eecbf GH |
453 | if (IS_ERR(eventfd)) |
454 | return PTR_ERR(eventfd); | |
455 | ||
456 | spin_lock_irq(&kvm->irqfds.lock); | |
457 | ||
458 | list_for_each_entry_safe(irqfd, tmp, &kvm->irqfds.items, list) { | |
d4db2935 | 459 | if (irqfd->eventfd == eventfd && irqfd->gsi == args->gsi) { |
bd2b53b2 MT |
460 | /* |
461 | * This rcu_assign_pointer is needed for when | |
c8ce057e MT |
462 | * another thread calls kvm_irq_routing_update before |
463 | * we flush workqueue below (we synchronize with | |
464 | * kvm_irq_routing_update using irqfds.lock). | |
bd2b53b2 MT |
465 | * It is paired with synchronize_rcu done by caller |
466 | * of that function. | |
467 | */ | |
468 | rcu_assign_pointer(irqfd->irq_entry, NULL); | |
721eecbf | 469 | irqfd_deactivate(irqfd); |
bd2b53b2 | 470 | } |
721eecbf GH |
471 | } |
472 | ||
473 | spin_unlock_irq(&kvm->irqfds.lock); | |
474 | eventfd_ctx_put(eventfd); | |
475 | ||
476 | /* | |
477 | * Block until we know all outstanding shutdown jobs have completed | |
478 | * so that we guarantee there will not be any more interrupts on this | |
479 | * gsi once this deassign function returns. | |
480 | */ | |
481 | flush_workqueue(irqfd_cleanup_wq); | |
482 | ||
483 | return 0; | |
484 | } | |
485 | ||
486 | int | |
d4db2935 | 487 | kvm_irqfd(struct kvm *kvm, struct kvm_irqfd *args) |
721eecbf | 488 | { |
7a84428a | 489 | if (args->flags & ~(KVM_IRQFD_FLAG_DEASSIGN | KVM_IRQFD_FLAG_RESAMPLE)) |
326cf033 AW |
490 | return -EINVAL; |
491 | ||
d4db2935 AW |
492 | if (args->flags & KVM_IRQFD_FLAG_DEASSIGN) |
493 | return kvm_irqfd_deassign(kvm, args); | |
721eecbf | 494 | |
d4db2935 | 495 | return kvm_irqfd_assign(kvm, args); |
721eecbf GH |
496 | } |
497 | ||
498 | /* | |
499 | * This function is called as the kvm VM fd is being released. Shutdown all | |
500 | * irqfds that still remain open | |
501 | */ | |
502 | void | |
503 | kvm_irqfd_release(struct kvm *kvm) | |
504 | { | |
505 | struct _irqfd *irqfd, *tmp; | |
506 | ||
507 | spin_lock_irq(&kvm->irqfds.lock); | |
508 | ||
509 | list_for_each_entry_safe(irqfd, tmp, &kvm->irqfds.items, list) | |
510 | irqfd_deactivate(irqfd); | |
511 | ||
512 | spin_unlock_irq(&kvm->irqfds.lock); | |
513 | ||
514 | /* | |
515 | * Block until we know all outstanding shutdown jobs have completed | |
516 | * since we do not take a kvm* reference. | |
517 | */ | |
518 | flush_workqueue(irqfd_cleanup_wq); | |
519 | ||
520 | } | |
521 | ||
bd2b53b2 MT |
522 | /* |
523 | * Change irq_routing and irqfd. | |
524 | * Caller must invoke synchronize_rcu afterwards. | |
525 | */ | |
526 | void kvm_irq_routing_update(struct kvm *kvm, | |
527 | struct kvm_irq_routing_table *irq_rt) | |
528 | { | |
529 | struct _irqfd *irqfd; | |
530 | ||
531 | spin_lock_irq(&kvm->irqfds.lock); | |
532 | ||
533 | rcu_assign_pointer(kvm->irq_routing, irq_rt); | |
534 | ||
535 | list_for_each_entry(irqfd, &kvm->irqfds.items, list) | |
536 | irqfd_update(kvm, irqfd, irq_rt); | |
537 | ||
538 | spin_unlock_irq(&kvm->irqfds.lock); | |
539 | } | |
540 | ||
721eecbf GH |
541 | /* |
542 | * create a host-wide workqueue for issuing deferred shutdown requests | |
543 | * aggregated from all vm* instances. We need our own isolated single-thread | |
544 | * queue to prevent deadlock against flushing the normal work-queue. | |
545 | */ | |
546 | static int __init irqfd_module_init(void) | |
547 | { | |
548 | irqfd_cleanup_wq = create_singlethread_workqueue("kvm-irqfd-cleanup"); | |
549 | if (!irqfd_cleanup_wq) | |
550 | return -ENOMEM; | |
551 | ||
552 | return 0; | |
553 | } | |
554 | ||
555 | static void __exit irqfd_module_exit(void) | |
556 | { | |
557 | destroy_workqueue(irqfd_cleanup_wq); | |
558 | } | |
559 | ||
560 | module_init(irqfd_module_init); | |
561 | module_exit(irqfd_module_exit); | |
914daba8 | 562 | #endif |
d34e6b17 GH |
563 | |
564 | /* | |
565 | * -------------------------------------------------------------------- | |
566 | * ioeventfd: translate a PIO/MMIO memory write to an eventfd signal. | |
567 | * | |
568 | * userspace can register a PIO/MMIO address with an eventfd for receiving | |
569 | * notification when the memory has been touched. | |
570 | * -------------------------------------------------------------------- | |
571 | */ | |
572 | ||
573 | struct _ioeventfd { | |
574 | struct list_head list; | |
575 | u64 addr; | |
576 | int length; | |
577 | struct eventfd_ctx *eventfd; | |
578 | u64 datamatch; | |
579 | struct kvm_io_device dev; | |
580 | bool wildcard; | |
581 | }; | |
582 | ||
583 | static inline struct _ioeventfd * | |
584 | to_ioeventfd(struct kvm_io_device *dev) | |
585 | { | |
586 | return container_of(dev, struct _ioeventfd, dev); | |
587 | } | |
588 | ||
589 | static void | |
590 | ioeventfd_release(struct _ioeventfd *p) | |
591 | { | |
592 | eventfd_ctx_put(p->eventfd); | |
593 | list_del(&p->list); | |
594 | kfree(p); | |
595 | } | |
596 | ||
597 | static bool | |
598 | ioeventfd_in_range(struct _ioeventfd *p, gpa_t addr, int len, const void *val) | |
599 | { | |
600 | u64 _val; | |
601 | ||
602 | if (!(addr == p->addr && len == p->length)) | |
603 | /* address-range must be precise for a hit */ | |
604 | return false; | |
605 | ||
606 | if (p->wildcard) | |
607 | /* all else equal, wildcard is always a hit */ | |
608 | return true; | |
609 | ||
610 | /* otherwise, we have to actually compare the data */ | |
611 | ||
612 | BUG_ON(!IS_ALIGNED((unsigned long)val, len)); | |
613 | ||
614 | switch (len) { | |
615 | case 1: | |
616 | _val = *(u8 *)val; | |
617 | break; | |
618 | case 2: | |
619 | _val = *(u16 *)val; | |
620 | break; | |
621 | case 4: | |
622 | _val = *(u32 *)val; | |
623 | break; | |
624 | case 8: | |
625 | _val = *(u64 *)val; | |
626 | break; | |
627 | default: | |
628 | return false; | |
629 | } | |
630 | ||
631 | return _val == p->datamatch ? true : false; | |
632 | } | |
633 | ||
634 | /* MMIO/PIO writes trigger an event if the addr/val match */ | |
635 | static int | |
636 | ioeventfd_write(struct kvm_io_device *this, gpa_t addr, int len, | |
637 | const void *val) | |
638 | { | |
639 | struct _ioeventfd *p = to_ioeventfd(this); | |
640 | ||
641 | if (!ioeventfd_in_range(p, addr, len, val)) | |
642 | return -EOPNOTSUPP; | |
643 | ||
644 | eventfd_signal(p->eventfd, 1); | |
645 | return 0; | |
646 | } | |
647 | ||
648 | /* | |
649 | * This function is called as KVM is completely shutting down. We do not | |
650 | * need to worry about locking just nuke anything we have as quickly as possible | |
651 | */ | |
652 | static void | |
653 | ioeventfd_destructor(struct kvm_io_device *this) | |
654 | { | |
655 | struct _ioeventfd *p = to_ioeventfd(this); | |
656 | ||
657 | ioeventfd_release(p); | |
658 | } | |
659 | ||
660 | static const struct kvm_io_device_ops ioeventfd_ops = { | |
661 | .write = ioeventfd_write, | |
662 | .destructor = ioeventfd_destructor, | |
663 | }; | |
664 | ||
665 | /* assumes kvm->slots_lock held */ | |
666 | static bool | |
667 | ioeventfd_check_collision(struct kvm *kvm, struct _ioeventfd *p) | |
668 | { | |
669 | struct _ioeventfd *_p; | |
670 | ||
671 | list_for_each_entry(_p, &kvm->ioeventfds, list) | |
672 | if (_p->addr == p->addr && _p->length == p->length && | |
673 | (_p->wildcard || p->wildcard || | |
674 | _p->datamatch == p->datamatch)) | |
675 | return true; | |
676 | ||
677 | return false; | |
678 | } | |
679 | ||
680 | static int | |
681 | kvm_assign_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args) | |
682 | { | |
683 | int pio = args->flags & KVM_IOEVENTFD_FLAG_PIO; | |
e93f8a0f | 684 | enum kvm_bus bus_idx = pio ? KVM_PIO_BUS : KVM_MMIO_BUS; |
d34e6b17 GH |
685 | struct _ioeventfd *p; |
686 | struct eventfd_ctx *eventfd; | |
687 | int ret; | |
688 | ||
689 | /* must be natural-word sized */ | |
690 | switch (args->len) { | |
691 | case 1: | |
692 | case 2: | |
693 | case 4: | |
694 | case 8: | |
695 | break; | |
696 | default: | |
697 | return -EINVAL; | |
698 | } | |
699 | ||
700 | /* check for range overflow */ | |
701 | if (args->addr + args->len < args->addr) | |
702 | return -EINVAL; | |
703 | ||
704 | /* check for extra flags that we don't understand */ | |
705 | if (args->flags & ~KVM_IOEVENTFD_VALID_FLAG_MASK) | |
706 | return -EINVAL; | |
707 | ||
708 | eventfd = eventfd_ctx_fdget(args->fd); | |
709 | if (IS_ERR(eventfd)) | |
710 | return PTR_ERR(eventfd); | |
711 | ||
712 | p = kzalloc(sizeof(*p), GFP_KERNEL); | |
713 | if (!p) { | |
714 | ret = -ENOMEM; | |
715 | goto fail; | |
716 | } | |
717 | ||
718 | INIT_LIST_HEAD(&p->list); | |
719 | p->addr = args->addr; | |
720 | p->length = args->len; | |
721 | p->eventfd = eventfd; | |
722 | ||
723 | /* The datamatch feature is optional, otherwise this is a wildcard */ | |
724 | if (args->flags & KVM_IOEVENTFD_FLAG_DATAMATCH) | |
725 | p->datamatch = args->datamatch; | |
726 | else | |
727 | p->wildcard = true; | |
728 | ||
79fac95e | 729 | mutex_lock(&kvm->slots_lock); |
d34e6b17 | 730 | |
25985edc | 731 | /* Verify that there isn't a match already */ |
d34e6b17 GH |
732 | if (ioeventfd_check_collision(kvm, p)) { |
733 | ret = -EEXIST; | |
734 | goto unlock_fail; | |
735 | } | |
736 | ||
737 | kvm_iodevice_init(&p->dev, &ioeventfd_ops); | |
738 | ||
743eeb0b SL |
739 | ret = kvm_io_bus_register_dev(kvm, bus_idx, p->addr, p->length, |
740 | &p->dev); | |
d34e6b17 GH |
741 | if (ret < 0) |
742 | goto unlock_fail; | |
743 | ||
744 | list_add_tail(&p->list, &kvm->ioeventfds); | |
745 | ||
79fac95e | 746 | mutex_unlock(&kvm->slots_lock); |
d34e6b17 GH |
747 | |
748 | return 0; | |
749 | ||
750 | unlock_fail: | |
79fac95e | 751 | mutex_unlock(&kvm->slots_lock); |
d34e6b17 GH |
752 | |
753 | fail: | |
754 | kfree(p); | |
755 | eventfd_ctx_put(eventfd); | |
756 | ||
757 | return ret; | |
758 | } | |
759 | ||
760 | static int | |
761 | kvm_deassign_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args) | |
762 | { | |
763 | int pio = args->flags & KVM_IOEVENTFD_FLAG_PIO; | |
e93f8a0f | 764 | enum kvm_bus bus_idx = pio ? KVM_PIO_BUS : KVM_MMIO_BUS; |
d34e6b17 GH |
765 | struct _ioeventfd *p, *tmp; |
766 | struct eventfd_ctx *eventfd; | |
767 | int ret = -ENOENT; | |
768 | ||
769 | eventfd = eventfd_ctx_fdget(args->fd); | |
770 | if (IS_ERR(eventfd)) | |
771 | return PTR_ERR(eventfd); | |
772 | ||
79fac95e | 773 | mutex_lock(&kvm->slots_lock); |
d34e6b17 GH |
774 | |
775 | list_for_each_entry_safe(p, tmp, &kvm->ioeventfds, list) { | |
776 | bool wildcard = !(args->flags & KVM_IOEVENTFD_FLAG_DATAMATCH); | |
777 | ||
778 | if (p->eventfd != eventfd || | |
779 | p->addr != args->addr || | |
780 | p->length != args->len || | |
781 | p->wildcard != wildcard) | |
782 | continue; | |
783 | ||
784 | if (!p->wildcard && p->datamatch != args->datamatch) | |
785 | continue; | |
786 | ||
e93f8a0f | 787 | kvm_io_bus_unregister_dev(kvm, bus_idx, &p->dev); |
d34e6b17 GH |
788 | ioeventfd_release(p); |
789 | ret = 0; | |
790 | break; | |
791 | } | |
792 | ||
79fac95e | 793 | mutex_unlock(&kvm->slots_lock); |
d34e6b17 GH |
794 | |
795 | eventfd_ctx_put(eventfd); | |
796 | ||
797 | return ret; | |
798 | } | |
799 | ||
800 | int | |
801 | kvm_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args) | |
802 | { | |
803 | if (args->flags & KVM_IOEVENTFD_FLAG_DEASSIGN) | |
804 | return kvm_deassign_ioeventfd(kvm, args); | |
805 | ||
806 | return kvm_assign_ioeventfd(kvm, args); | |
807 | } |