2 * Kernel-based Virtual Machine driver for Linux
4 * This module enables machines with Intel VT-x extensions to run virtual
5 * machines without emulation or binary translation.
7 * Copyright (C) 2006 Qumranet, Inc.
10 * Avi Kivity <avi@qumranet.com>
11 * Yaniv Kamay <yaniv@qumranet.com>
13 * This work is licensed under the terms of the GNU GPL, version 2. See
14 * the COPYING file in the top-level directory.
20 #include "x86_emulate.h"
21 #include "segment_descriptor.h"
24 #include <linux/kvm.h>
25 #include <linux/module.h>
26 #include <linux/errno.h>
27 #include <linux/percpu.h>
28 #include <linux/gfp.h>
30 #include <linux/miscdevice.h>
31 #include <linux/vmalloc.h>
32 #include <linux/reboot.h>
33 #include <linux/debugfs.h>
34 #include <linux/highmem.h>
35 #include <linux/file.h>
36 #include <linux/sysdev.h>
37 #include <linux/cpu.h>
38 #include <linux/sched.h>
39 #include <linux/cpumask.h>
40 #include <linux/smp.h>
41 #include <linux/anon_inodes.h>
42 #include <linux/profile.h>
43 #include <linux/kvm_para.h>
44 #include <linux/pagemap.h>
46 #include <asm/processor.h>
49 #include <asm/uaccess.h>
52 MODULE_AUTHOR("Qumranet");
53 MODULE_LICENSE("GPL");
55 static DEFINE_SPINLOCK(kvm_lock
);
56 static LIST_HEAD(vm_list
);
58 static cpumask_t cpus_hardware_enabled
;
60 struct kvm_x86_ops
*kvm_x86_ops
;
61 struct kmem_cache
*kvm_vcpu_cache
;
62 EXPORT_SYMBOL_GPL(kvm_vcpu_cache
);
64 static __read_mostly
struct preempt_ops kvm_preempt_ops
;
66 #define STAT_OFFSET(x) offsetof(struct kvm_vcpu, stat.x)
68 static struct kvm_stats_debugfs_item
{
71 struct dentry
*dentry
;
72 } debugfs_entries
[] = {
73 { "pf_fixed", STAT_OFFSET(pf_fixed
) },
74 { "pf_guest", STAT_OFFSET(pf_guest
) },
75 { "tlb_flush", STAT_OFFSET(tlb_flush
) },
76 { "invlpg", STAT_OFFSET(invlpg
) },
77 { "exits", STAT_OFFSET(exits
) },
78 { "io_exits", STAT_OFFSET(io_exits
) },
79 { "mmio_exits", STAT_OFFSET(mmio_exits
) },
80 { "signal_exits", STAT_OFFSET(signal_exits
) },
81 { "irq_window", STAT_OFFSET(irq_window_exits
) },
82 { "halt_exits", STAT_OFFSET(halt_exits
) },
83 { "halt_wakeup", STAT_OFFSET(halt_wakeup
) },
84 { "request_irq", STAT_OFFSET(request_irq_exits
) },
85 { "irq_exits", STAT_OFFSET(irq_exits
) },
86 { "light_exits", STAT_OFFSET(light_exits
) },
87 { "efer_reload", STAT_OFFSET(efer_reload
) },
91 static struct dentry
*debugfs_dir
;
93 #define CR0_RESERVED_BITS \
94 (~(unsigned long)(X86_CR0_PE | X86_CR0_MP | X86_CR0_EM | X86_CR0_TS \
95 | X86_CR0_ET | X86_CR0_NE | X86_CR0_WP | X86_CR0_AM \
96 | X86_CR0_NW | X86_CR0_CD | X86_CR0_PG))
97 #define CR4_RESERVED_BITS \
98 (~(unsigned long)(X86_CR4_VME | X86_CR4_PVI | X86_CR4_TSD | X86_CR4_DE\
99 | X86_CR4_PSE | X86_CR4_PAE | X86_CR4_MCE \
100 | X86_CR4_PGE | X86_CR4_PCE | X86_CR4_OSFXSR \
101 | X86_CR4_OSXMMEXCPT | X86_CR4_VMXE))
103 #define CR8_RESERVED_BITS (~(unsigned long)X86_CR8_TPR)
104 #define EFER_RESERVED_BITS 0xfffffffffffff2fe
107 /* LDT or TSS descriptor in the GDT. 16 bytes. */
108 struct segment_descriptor_64
{
109 struct segment_descriptor s
;
116 static long kvm_vcpu_ioctl(struct file
*file
, unsigned int ioctl
,
119 unsigned long segment_base(u16 selector
)
121 struct descriptor_table gdt
;
122 struct segment_descriptor
*d
;
123 unsigned long table_base
;
129 asm("sgdt %0" : "=m"(gdt
));
130 table_base
= gdt
.base
;
132 if (selector
& 4) { /* from ldt */
135 asm("sldt %0" : "=g"(ldt_selector
));
136 table_base
= segment_base(ldt_selector
);
138 d
= (struct segment_descriptor
*)(table_base
+ (selector
& ~7));
139 v
= d
->base_low
| ((unsigned long)d
->base_mid
<< 16) |
140 ((unsigned long)d
->base_high
<< 24);
142 if (d
->system
== 0 && (d
->type
== 2 || d
->type
== 9 || d
->type
== 11))
143 v
|= ((unsigned long) \
144 ((struct segment_descriptor_64
*)d
)->base_higher
) << 32;
148 EXPORT_SYMBOL_GPL(segment_base
);
150 static inline int valid_vcpu(int n
)
152 return likely(n
>= 0 && n
< KVM_MAX_VCPUS
);
155 void kvm_load_guest_fpu(struct kvm_vcpu
*vcpu
)
157 if (!vcpu
->fpu_active
|| vcpu
->guest_fpu_loaded
)
160 vcpu
->guest_fpu_loaded
= 1;
161 fx_save(&vcpu
->host_fx_image
);
162 fx_restore(&vcpu
->guest_fx_image
);
164 EXPORT_SYMBOL_GPL(kvm_load_guest_fpu
);
166 void kvm_put_guest_fpu(struct kvm_vcpu
*vcpu
)
168 if (!vcpu
->guest_fpu_loaded
)
171 vcpu
->guest_fpu_loaded
= 0;
172 fx_save(&vcpu
->guest_fx_image
);
173 fx_restore(&vcpu
->host_fx_image
);
175 EXPORT_SYMBOL_GPL(kvm_put_guest_fpu
);
178 * Switches to specified vcpu, until a matching vcpu_put()
180 void vcpu_load(struct kvm_vcpu
*vcpu
)
184 mutex_lock(&vcpu
->mutex
);
186 preempt_notifier_register(&vcpu
->preempt_notifier
);
187 kvm_arch_vcpu_load(vcpu
, cpu
);
191 void vcpu_put(struct kvm_vcpu
*vcpu
)
194 kvm_arch_vcpu_put(vcpu
);
195 preempt_notifier_unregister(&vcpu
->preempt_notifier
);
197 mutex_unlock(&vcpu
->mutex
);
200 static void ack_flush(void *_completed
)
204 void kvm_flush_remote_tlbs(struct kvm
*kvm
)
208 struct kvm_vcpu
*vcpu
;
211 for (i
= 0; i
< KVM_MAX_VCPUS
; ++i
) {
212 vcpu
= kvm
->vcpus
[i
];
215 if (test_and_set_bit(KVM_REQ_TLB_FLUSH
, &vcpu
->requests
))
218 if (cpu
!= -1 && cpu
!= raw_smp_processor_id())
221 smp_call_function_mask(cpus
, ack_flush
, NULL
, 1);
224 int kvm_vcpu_init(struct kvm_vcpu
*vcpu
, struct kvm
*kvm
, unsigned id
)
229 mutex_init(&vcpu
->mutex
);
231 vcpu
->mmu
.root_hpa
= INVALID_PAGE
;
234 if (!irqchip_in_kernel(kvm
) || id
== 0)
235 vcpu
->mp_state
= VCPU_MP_STATE_RUNNABLE
;
237 vcpu
->mp_state
= VCPU_MP_STATE_UNINITIALIZED
;
238 init_waitqueue_head(&vcpu
->wq
);
240 page
= alloc_page(GFP_KERNEL
| __GFP_ZERO
);
245 vcpu
->run
= page_address(page
);
247 page
= alloc_page(GFP_KERNEL
| __GFP_ZERO
);
252 vcpu
->pio_data
= page_address(page
);
254 r
= kvm_mmu_create(vcpu
);
256 goto fail_free_pio_data
;
258 if (irqchip_in_kernel(kvm
)) {
259 r
= kvm_create_lapic(vcpu
);
261 goto fail_mmu_destroy
;
267 kvm_mmu_destroy(vcpu
);
269 free_page((unsigned long)vcpu
->pio_data
);
271 free_page((unsigned long)vcpu
->run
);
275 EXPORT_SYMBOL_GPL(kvm_vcpu_init
);
277 void kvm_vcpu_uninit(struct kvm_vcpu
*vcpu
)
279 kvm_free_lapic(vcpu
);
280 kvm_mmu_destroy(vcpu
);
281 free_page((unsigned long)vcpu
->pio_data
);
282 free_page((unsigned long)vcpu
->run
);
284 EXPORT_SYMBOL_GPL(kvm_vcpu_uninit
);
286 static struct kvm
*kvm_create_vm(void)
288 struct kvm
*kvm
= kzalloc(sizeof(struct kvm
), GFP_KERNEL
);
291 return ERR_PTR(-ENOMEM
);
293 kvm_io_bus_init(&kvm
->pio_bus
);
294 mutex_init(&kvm
->lock
);
295 INIT_LIST_HEAD(&kvm
->active_mmu_pages
);
296 kvm_io_bus_init(&kvm
->mmio_bus
);
297 spin_lock(&kvm_lock
);
298 list_add(&kvm
->vm_list
, &vm_list
);
299 spin_unlock(&kvm_lock
);
303 static void kvm_free_kernel_physmem(struct kvm_memory_slot
*free
)
307 for (i
= 0; i
< free
->npages
; ++i
)
308 if (free
->phys_mem
[i
])
309 __free_page(free
->phys_mem
[i
]);
313 * Free any memory in @free but not in @dont.
315 static void kvm_free_physmem_slot(struct kvm_memory_slot
*free
,
316 struct kvm_memory_slot
*dont
)
318 if (!dont
|| free
->phys_mem
!= dont
->phys_mem
)
319 if (free
->phys_mem
) {
320 if (!free
->user_alloc
)
321 kvm_free_kernel_physmem(free
);
322 vfree(free
->phys_mem
);
324 if (!dont
|| free
->rmap
!= dont
->rmap
)
327 if (!dont
|| free
->dirty_bitmap
!= dont
->dirty_bitmap
)
328 vfree(free
->dirty_bitmap
);
330 free
->phys_mem
= NULL
;
332 free
->dirty_bitmap
= NULL
;
335 static void kvm_free_physmem(struct kvm
*kvm
)
339 for (i
= 0; i
< kvm
->nmemslots
; ++i
)
340 kvm_free_physmem_slot(&kvm
->memslots
[i
], NULL
);
343 static void free_pio_guest_pages(struct kvm_vcpu
*vcpu
)
347 for (i
= 0; i
< ARRAY_SIZE(vcpu
->pio
.guest_pages
); ++i
)
348 if (vcpu
->pio
.guest_pages
[i
]) {
349 kvm_release_page(vcpu
->pio
.guest_pages
[i
]);
350 vcpu
->pio
.guest_pages
[i
] = NULL
;
354 static void kvm_unload_vcpu_mmu(struct kvm_vcpu
*vcpu
)
357 kvm_mmu_unload(vcpu
);
361 static void kvm_free_vcpus(struct kvm
*kvm
)
366 * Unpin any mmu pages first.
368 for (i
= 0; i
< KVM_MAX_VCPUS
; ++i
)
370 kvm_unload_vcpu_mmu(kvm
->vcpus
[i
]);
371 for (i
= 0; i
< KVM_MAX_VCPUS
; ++i
) {
373 kvm_x86_ops
->vcpu_free(kvm
->vcpus
[i
]);
374 kvm
->vcpus
[i
] = NULL
;
380 static void kvm_destroy_vm(struct kvm
*kvm
)
382 spin_lock(&kvm_lock
);
383 list_del(&kvm
->vm_list
);
384 spin_unlock(&kvm_lock
);
385 kvm_io_bus_destroy(&kvm
->pio_bus
);
386 kvm_io_bus_destroy(&kvm
->mmio_bus
);
390 kvm_free_physmem(kvm
);
394 static int kvm_vm_release(struct inode
*inode
, struct file
*filp
)
396 struct kvm
*kvm
= filp
->private_data
;
402 static void inject_gp(struct kvm_vcpu
*vcpu
)
404 kvm_x86_ops
->inject_gp(vcpu
, 0);
408 * Load the pae pdptrs. Return true is they are all valid.
410 static int load_pdptrs(struct kvm_vcpu
*vcpu
, unsigned long cr3
)
412 gfn_t pdpt_gfn
= cr3
>> PAGE_SHIFT
;
413 unsigned offset
= ((cr3
& (PAGE_SIZE
-1)) >> 5) << 2;
416 u64 pdpte
[ARRAY_SIZE(vcpu
->pdptrs
)];
418 mutex_lock(&vcpu
->kvm
->lock
);
419 ret
= kvm_read_guest_page(vcpu
->kvm
, pdpt_gfn
, pdpte
,
420 offset
* sizeof(u64
), sizeof(pdpte
));
425 for (i
= 0; i
< ARRAY_SIZE(pdpte
); ++i
) {
426 if ((pdpte
[i
] & 1) && (pdpte
[i
] & 0xfffffff0000001e6ull
)) {
433 memcpy(vcpu
->pdptrs
, pdpte
, sizeof(vcpu
->pdptrs
));
435 mutex_unlock(&vcpu
->kvm
->lock
);
440 void set_cr0(struct kvm_vcpu
*vcpu
, unsigned long cr0
)
442 if (cr0
& CR0_RESERVED_BITS
) {
443 printk(KERN_DEBUG
"set_cr0: 0x%lx #GP, reserved bits 0x%lx\n",
449 if ((cr0
& X86_CR0_NW
) && !(cr0
& X86_CR0_CD
)) {
450 printk(KERN_DEBUG
"set_cr0: #GP, CD == 0 && NW == 1\n");
455 if ((cr0
& X86_CR0_PG
) && !(cr0
& X86_CR0_PE
)) {
456 printk(KERN_DEBUG
"set_cr0: #GP, set PG flag "
457 "and a clear PE flag\n");
462 if (!is_paging(vcpu
) && (cr0
& X86_CR0_PG
)) {
464 if ((vcpu
->shadow_efer
& EFER_LME
)) {
468 printk(KERN_DEBUG
"set_cr0: #GP, start paging "
469 "in long mode while PAE is disabled\n");
473 kvm_x86_ops
->get_cs_db_l_bits(vcpu
, &cs_db
, &cs_l
);
475 printk(KERN_DEBUG
"set_cr0: #GP, start paging "
476 "in long mode while CS.L == 1\n");
483 if (is_pae(vcpu
) && !load_pdptrs(vcpu
, vcpu
->cr3
)) {
484 printk(KERN_DEBUG
"set_cr0: #GP, pdptrs "
492 kvm_x86_ops
->set_cr0(vcpu
, cr0
);
495 mutex_lock(&vcpu
->kvm
->lock
);
496 kvm_mmu_reset_context(vcpu
);
497 mutex_unlock(&vcpu
->kvm
->lock
);
500 EXPORT_SYMBOL_GPL(set_cr0
);
502 void lmsw(struct kvm_vcpu
*vcpu
, unsigned long msw
)
504 set_cr0(vcpu
, (vcpu
->cr0
& ~0x0ful
) | (msw
& 0x0f));
506 EXPORT_SYMBOL_GPL(lmsw
);
508 void set_cr4(struct kvm_vcpu
*vcpu
, unsigned long cr4
)
510 if (cr4
& CR4_RESERVED_BITS
) {
511 printk(KERN_DEBUG
"set_cr4: #GP, reserved bits\n");
516 if (is_long_mode(vcpu
)) {
517 if (!(cr4
& X86_CR4_PAE
)) {
518 printk(KERN_DEBUG
"set_cr4: #GP, clearing PAE while "
523 } else if (is_paging(vcpu
) && !is_pae(vcpu
) && (cr4
& X86_CR4_PAE
)
524 && !load_pdptrs(vcpu
, vcpu
->cr3
)) {
525 printk(KERN_DEBUG
"set_cr4: #GP, pdptrs reserved bits\n");
530 if (cr4
& X86_CR4_VMXE
) {
531 printk(KERN_DEBUG
"set_cr4: #GP, setting VMXE\n");
535 kvm_x86_ops
->set_cr4(vcpu
, cr4
);
537 mutex_lock(&vcpu
->kvm
->lock
);
538 kvm_mmu_reset_context(vcpu
);
539 mutex_unlock(&vcpu
->kvm
->lock
);
541 EXPORT_SYMBOL_GPL(set_cr4
);
543 void set_cr3(struct kvm_vcpu
*vcpu
, unsigned long cr3
)
545 if (is_long_mode(vcpu
)) {
546 if (cr3
& CR3_L_MODE_RESERVED_BITS
) {
547 printk(KERN_DEBUG
"set_cr3: #GP, reserved bits\n");
553 if (cr3
& CR3_PAE_RESERVED_BITS
) {
555 "set_cr3: #GP, reserved bits\n");
559 if (is_paging(vcpu
) && !load_pdptrs(vcpu
, cr3
)) {
560 printk(KERN_DEBUG
"set_cr3: #GP, pdptrs "
567 * We don't check reserved bits in nonpae mode, because
568 * this isn't enforced, and VMware depends on this.
572 mutex_lock(&vcpu
->kvm
->lock
);
574 * Does the new cr3 value map to physical memory? (Note, we
575 * catch an invalid cr3 even in real-mode, because it would
576 * cause trouble later on when we turn on paging anyway.)
578 * A real CPU would silently accept an invalid cr3 and would
579 * attempt to use it - with largely undefined (and often hard
580 * to debug) behavior on the guest side.
582 if (unlikely(!gfn_to_memslot(vcpu
->kvm
, cr3
>> PAGE_SHIFT
)))
586 vcpu
->mmu
.new_cr3(vcpu
);
588 mutex_unlock(&vcpu
->kvm
->lock
);
590 EXPORT_SYMBOL_GPL(set_cr3
);
592 void set_cr8(struct kvm_vcpu
*vcpu
, unsigned long cr8
)
594 if (cr8
& CR8_RESERVED_BITS
) {
595 printk(KERN_DEBUG
"set_cr8: #GP, reserved bits 0x%lx\n", cr8
);
599 if (irqchip_in_kernel(vcpu
->kvm
))
600 kvm_lapic_set_tpr(vcpu
, cr8
);
604 EXPORT_SYMBOL_GPL(set_cr8
);
606 unsigned long get_cr8(struct kvm_vcpu
*vcpu
)
608 if (irqchip_in_kernel(vcpu
->kvm
))
609 return kvm_lapic_get_cr8(vcpu
);
613 EXPORT_SYMBOL_GPL(get_cr8
);
615 u64
kvm_get_apic_base(struct kvm_vcpu
*vcpu
)
617 if (irqchip_in_kernel(vcpu
->kvm
))
618 return vcpu
->apic_base
;
620 return vcpu
->apic_base
;
622 EXPORT_SYMBOL_GPL(kvm_get_apic_base
);
624 void kvm_set_apic_base(struct kvm_vcpu
*vcpu
, u64 data
)
626 /* TODO: reserve bits check */
627 if (irqchip_in_kernel(vcpu
->kvm
))
628 kvm_lapic_set_base(vcpu
, data
);
630 vcpu
->apic_base
= data
;
632 EXPORT_SYMBOL_GPL(kvm_set_apic_base
);
634 void fx_init(struct kvm_vcpu
*vcpu
)
636 unsigned after_mxcsr_mask
;
638 /* Initialize guest FPU by resetting ours and saving into guest's */
640 fx_save(&vcpu
->host_fx_image
);
642 fx_save(&vcpu
->guest_fx_image
);
643 fx_restore(&vcpu
->host_fx_image
);
646 vcpu
->cr0
|= X86_CR0_ET
;
647 after_mxcsr_mask
= offsetof(struct i387_fxsave_struct
, st_space
);
648 vcpu
->guest_fx_image
.mxcsr
= 0x1f80;
649 memset((void *)&vcpu
->guest_fx_image
+ after_mxcsr_mask
,
650 0, sizeof(struct i387_fxsave_struct
) - after_mxcsr_mask
);
652 EXPORT_SYMBOL_GPL(fx_init
);
655 * Allocate some memory and give it an address in the guest physical address
658 * Discontiguous memory is allowed, mostly for framebuffers.
660 static int kvm_vm_ioctl_set_memory_region(struct kvm
*kvm
,
662 kvm_userspace_memory_region
*mem
,
667 unsigned long npages
;
669 struct kvm_memory_slot
*memslot
;
670 struct kvm_memory_slot old
, new;
673 /* General sanity checks */
674 if (mem
->memory_size
& (PAGE_SIZE
- 1))
676 if (mem
->guest_phys_addr
& (PAGE_SIZE
- 1))
678 if (mem
->slot
>= KVM_MEMORY_SLOTS
)
680 if (mem
->guest_phys_addr
+ mem
->memory_size
< mem
->guest_phys_addr
)
683 memslot
= &kvm
->memslots
[mem
->slot
];
684 base_gfn
= mem
->guest_phys_addr
>> PAGE_SHIFT
;
685 npages
= mem
->memory_size
>> PAGE_SHIFT
;
688 mem
->flags
&= ~KVM_MEM_LOG_DIRTY_PAGES
;
690 mutex_lock(&kvm
->lock
);
692 new = old
= *memslot
;
694 new.base_gfn
= base_gfn
;
696 new.flags
= mem
->flags
;
698 /* Disallow changing a memory slot's size. */
700 if (npages
&& old
.npages
&& npages
!= old
.npages
)
703 /* Check for overlaps */
705 for (i
= 0; i
< KVM_MEMORY_SLOTS
; ++i
) {
706 struct kvm_memory_slot
*s
= &kvm
->memslots
[i
];
710 if (!((base_gfn
+ npages
<= s
->base_gfn
) ||
711 (base_gfn
>= s
->base_gfn
+ s
->npages
)))
715 /* Deallocate if slot is being removed */
719 /* Free page dirty bitmap if unneeded */
720 if (!(new.flags
& KVM_MEM_LOG_DIRTY_PAGES
))
721 new.dirty_bitmap
= NULL
;
725 /* Allocate if a slot is being created */
726 if (npages
&& !new.phys_mem
) {
727 new.phys_mem
= vmalloc(npages
* sizeof(struct page
*));
732 new.rmap
= vmalloc(npages
* sizeof(struct page
*));
737 memset(new.phys_mem
, 0, npages
* sizeof(struct page
*));
738 memset(new.rmap
, 0, npages
* sizeof(*new.rmap
));
741 new.userspace_addr
= mem
->userspace_addr
;
743 for (i
= 0; i
< npages
; ++i
) {
744 new.phys_mem
[i
] = alloc_page(GFP_HIGHUSER
746 if (!new.phys_mem
[i
])
752 /* Allocate page dirty bitmap if needed */
753 if ((new.flags
& KVM_MEM_LOG_DIRTY_PAGES
) && !new.dirty_bitmap
) {
754 unsigned dirty_bytes
= ALIGN(npages
, BITS_PER_LONG
) / 8;
756 new.dirty_bitmap
= vmalloc(dirty_bytes
);
757 if (!new.dirty_bitmap
)
759 memset(new.dirty_bitmap
, 0, dirty_bytes
);
762 if (mem
->slot
>= kvm
->nmemslots
)
763 kvm
->nmemslots
= mem
->slot
+ 1;
765 if (!kvm
->n_requested_mmu_pages
) {
766 unsigned int n_pages
;
769 n_pages
= npages
* KVM_PERMILLE_MMU_PAGES
/ 1000;
770 kvm_mmu_change_mmu_pages(kvm
, kvm
->n_alloc_mmu_pages
+
773 unsigned int nr_mmu_pages
;
775 n_pages
= old
.npages
* KVM_PERMILLE_MMU_PAGES
/ 1000;
776 nr_mmu_pages
= kvm
->n_alloc_mmu_pages
- n_pages
;
777 nr_mmu_pages
= max(nr_mmu_pages
,
778 (unsigned int) KVM_MIN_ALLOC_MMU_PAGES
);
779 kvm_mmu_change_mmu_pages(kvm
, nr_mmu_pages
);
785 kvm_mmu_slot_remove_write_access(kvm
, mem
->slot
);
786 kvm_flush_remote_tlbs(kvm
);
788 mutex_unlock(&kvm
->lock
);
790 kvm_free_physmem_slot(&old
, &new);
794 mutex_unlock(&kvm
->lock
);
795 kvm_free_physmem_slot(&new, &old
);
800 static int kvm_vm_ioctl_set_nr_mmu_pages(struct kvm
*kvm
,
801 u32 kvm_nr_mmu_pages
)
803 if (kvm_nr_mmu_pages
< KVM_MIN_ALLOC_MMU_PAGES
)
806 mutex_lock(&kvm
->lock
);
808 kvm_mmu_change_mmu_pages(kvm
, kvm_nr_mmu_pages
);
809 kvm
->n_requested_mmu_pages
= kvm_nr_mmu_pages
;
811 mutex_unlock(&kvm
->lock
);
815 static int kvm_vm_ioctl_get_nr_mmu_pages(struct kvm
*kvm
)
817 return kvm
->n_alloc_mmu_pages
;
821 * Get (and clear) the dirty memory log for a memory slot.
823 static int kvm_vm_ioctl_get_dirty_log(struct kvm
*kvm
,
824 struct kvm_dirty_log
*log
)
826 struct kvm_memory_slot
*memslot
;
829 unsigned long any
= 0;
831 mutex_lock(&kvm
->lock
);
834 if (log
->slot
>= KVM_MEMORY_SLOTS
)
837 memslot
= &kvm
->memslots
[log
->slot
];
839 if (!memslot
->dirty_bitmap
)
842 n
= ALIGN(memslot
->npages
, BITS_PER_LONG
) / 8;
844 for (i
= 0; !any
&& i
< n
/sizeof(long); ++i
)
845 any
= memslot
->dirty_bitmap
[i
];
848 if (copy_to_user(log
->dirty_bitmap
, memslot
->dirty_bitmap
, n
))
851 /* If nothing is dirty, don't bother messing with page tables. */
853 kvm_mmu_slot_remove_write_access(kvm
, log
->slot
);
854 kvm_flush_remote_tlbs(kvm
);
855 memset(memslot
->dirty_bitmap
, 0, n
);
861 mutex_unlock(&kvm
->lock
);
866 * Set a new alias region. Aliases map a portion of physical memory into
867 * another portion. This is useful for memory windows, for example the PC
870 static int kvm_vm_ioctl_set_memory_alias(struct kvm
*kvm
,
871 struct kvm_memory_alias
*alias
)
874 struct kvm_mem_alias
*p
;
877 /* General sanity checks */
878 if (alias
->memory_size
& (PAGE_SIZE
- 1))
880 if (alias
->guest_phys_addr
& (PAGE_SIZE
- 1))
882 if (alias
->slot
>= KVM_ALIAS_SLOTS
)
884 if (alias
->guest_phys_addr
+ alias
->memory_size
885 < alias
->guest_phys_addr
)
887 if (alias
->target_phys_addr
+ alias
->memory_size
888 < alias
->target_phys_addr
)
891 mutex_lock(&kvm
->lock
);
893 p
= &kvm
->aliases
[alias
->slot
];
894 p
->base_gfn
= alias
->guest_phys_addr
>> PAGE_SHIFT
;
895 p
->npages
= alias
->memory_size
>> PAGE_SHIFT
;
896 p
->target_gfn
= alias
->target_phys_addr
>> PAGE_SHIFT
;
898 for (n
= KVM_ALIAS_SLOTS
; n
> 0; --n
)
899 if (kvm
->aliases
[n
- 1].npages
)
903 kvm_mmu_zap_all(kvm
);
905 mutex_unlock(&kvm
->lock
);
913 static int kvm_vm_ioctl_get_irqchip(struct kvm
*kvm
, struct kvm_irqchip
*chip
)
918 switch (chip
->chip_id
) {
919 case KVM_IRQCHIP_PIC_MASTER
:
920 memcpy(&chip
->chip
.pic
,
921 &pic_irqchip(kvm
)->pics
[0],
922 sizeof(struct kvm_pic_state
));
924 case KVM_IRQCHIP_PIC_SLAVE
:
925 memcpy(&chip
->chip
.pic
,
926 &pic_irqchip(kvm
)->pics
[1],
927 sizeof(struct kvm_pic_state
));
929 case KVM_IRQCHIP_IOAPIC
:
930 memcpy(&chip
->chip
.ioapic
,
932 sizeof(struct kvm_ioapic_state
));
941 static int kvm_vm_ioctl_set_irqchip(struct kvm
*kvm
, struct kvm_irqchip
*chip
)
946 switch (chip
->chip_id
) {
947 case KVM_IRQCHIP_PIC_MASTER
:
948 memcpy(&pic_irqchip(kvm
)->pics
[0],
950 sizeof(struct kvm_pic_state
));
952 case KVM_IRQCHIP_PIC_SLAVE
:
953 memcpy(&pic_irqchip(kvm
)->pics
[1],
955 sizeof(struct kvm_pic_state
));
957 case KVM_IRQCHIP_IOAPIC
:
958 memcpy(ioapic_irqchip(kvm
),
960 sizeof(struct kvm_ioapic_state
));
966 kvm_pic_update_irq(pic_irqchip(kvm
));
970 int is_error_page(struct page
*page
)
972 return page
== bad_page
;
974 EXPORT_SYMBOL_GPL(is_error_page
);
976 gfn_t
unalias_gfn(struct kvm
*kvm
, gfn_t gfn
)
979 struct kvm_mem_alias
*alias
;
981 for (i
= 0; i
< kvm
->naliases
; ++i
) {
982 alias
= &kvm
->aliases
[i
];
983 if (gfn
>= alias
->base_gfn
984 && gfn
< alias
->base_gfn
+ alias
->npages
)
985 return alias
->target_gfn
+ gfn
- alias
->base_gfn
;
990 static struct kvm_memory_slot
*__gfn_to_memslot(struct kvm
*kvm
, gfn_t gfn
)
994 for (i
= 0; i
< kvm
->nmemslots
; ++i
) {
995 struct kvm_memory_slot
*memslot
= &kvm
->memslots
[i
];
997 if (gfn
>= memslot
->base_gfn
998 && gfn
< memslot
->base_gfn
+ memslot
->npages
)
1004 struct kvm_memory_slot
*gfn_to_memslot(struct kvm
*kvm
, gfn_t gfn
)
1006 gfn
= unalias_gfn(kvm
, gfn
);
1007 return __gfn_to_memslot(kvm
, gfn
);
1010 struct page
*gfn_to_page(struct kvm
*kvm
, gfn_t gfn
)
1012 struct kvm_memory_slot
*slot
;
1014 gfn
= unalias_gfn(kvm
, gfn
);
1015 slot
= __gfn_to_memslot(kvm
, gfn
);
1020 if (slot
->user_alloc
) {
1021 struct page
*page
[1];
1024 down_read(¤t
->mm
->mmap_sem
);
1025 npages
= get_user_pages(current
, current
->mm
,
1026 slot
->userspace_addr
1027 + (gfn
- slot
->base_gfn
) * PAGE_SIZE
, 1,
1029 up_read(¤t
->mm
->mmap_sem
);
1036 get_page(slot
->phys_mem
[gfn
- slot
->base_gfn
]);
1037 return slot
->phys_mem
[gfn
- slot
->base_gfn
];
1039 EXPORT_SYMBOL_GPL(gfn_to_page
);
1041 void kvm_release_page(struct page
*page
)
1043 if (!PageReserved(page
))
1047 EXPORT_SYMBOL_GPL(kvm_release_page
);
1049 static int next_segment(unsigned long len
, int offset
)
1051 if (len
> PAGE_SIZE
- offset
)
1052 return PAGE_SIZE
- offset
;
1057 int kvm_read_guest_page(struct kvm
*kvm
, gfn_t gfn
, void *data
, int offset
,
1063 page
= gfn_to_page(kvm
, gfn
);
1064 if (is_error_page(page
)) {
1065 kvm_release_page(page
);
1068 page_virt
= kmap_atomic(page
, KM_USER0
);
1070 memcpy(data
, page_virt
+ offset
, len
);
1072 kunmap_atomic(page_virt
, KM_USER0
);
1073 kvm_release_page(page
);
1076 EXPORT_SYMBOL_GPL(kvm_read_guest_page
);
1078 int kvm_read_guest(struct kvm
*kvm
, gpa_t gpa
, void *data
, unsigned long len
)
1080 gfn_t gfn
= gpa
>> PAGE_SHIFT
;
1082 int offset
= offset_in_page(gpa
);
1085 while ((seg
= next_segment(len
, offset
)) != 0) {
1086 ret
= kvm_read_guest_page(kvm
, gfn
, data
, offset
, seg
);
1096 EXPORT_SYMBOL_GPL(kvm_read_guest
);
1098 int kvm_write_guest_page(struct kvm
*kvm
, gfn_t gfn
, const void *data
,
1099 int offset
, int len
)
1104 page
= gfn_to_page(kvm
, gfn
);
1105 if (is_error_page(page
)) {
1106 kvm_release_page(page
);
1109 page_virt
= kmap_atomic(page
, KM_USER0
);
1111 memcpy(page_virt
+ offset
, data
, len
);
1113 kunmap_atomic(page_virt
, KM_USER0
);
1114 mark_page_dirty(kvm
, gfn
);
1115 kvm_release_page(page
);
1118 EXPORT_SYMBOL_GPL(kvm_write_guest_page
);
1120 int kvm_write_guest(struct kvm
*kvm
, gpa_t gpa
, const void *data
,
1123 gfn_t gfn
= gpa
>> PAGE_SHIFT
;
1125 int offset
= offset_in_page(gpa
);
1128 while ((seg
= next_segment(len
, offset
)) != 0) {
1129 ret
= kvm_write_guest_page(kvm
, gfn
, data
, offset
, seg
);
1140 int kvm_clear_guest_page(struct kvm
*kvm
, gfn_t gfn
, int offset
, int len
)
1145 page
= gfn_to_page(kvm
, gfn
);
1146 if (is_error_page(page
)) {
1147 kvm_release_page(page
);
1150 page_virt
= kmap_atomic(page
, KM_USER0
);
1152 memset(page_virt
+ offset
, 0, len
);
1154 kunmap_atomic(page_virt
, KM_USER0
);
1155 kvm_release_page(page
);
1158 EXPORT_SYMBOL_GPL(kvm_clear_guest_page
);
1160 int kvm_clear_guest(struct kvm
*kvm
, gpa_t gpa
, unsigned long len
)
1162 gfn_t gfn
= gpa
>> PAGE_SHIFT
;
1164 int offset
= offset_in_page(gpa
);
1167 while ((seg
= next_segment(len
, offset
)) != 0) {
1168 ret
= kvm_clear_guest_page(kvm
, gfn
, offset
, seg
);
1177 EXPORT_SYMBOL_GPL(kvm_clear_guest
);
1179 /* WARNING: Does not work on aliased pages. */
1180 void mark_page_dirty(struct kvm
*kvm
, gfn_t gfn
)
1182 struct kvm_memory_slot
*memslot
;
1184 memslot
= __gfn_to_memslot(kvm
, gfn
);
1185 if (memslot
&& memslot
->dirty_bitmap
) {
1186 unsigned long rel_gfn
= gfn
- memslot
->base_gfn
;
1189 if (!test_bit(rel_gfn
, memslot
->dirty_bitmap
))
1190 set_bit(rel_gfn
, memslot
->dirty_bitmap
);
1194 int emulator_read_std(unsigned long addr
,
1197 struct kvm_vcpu
*vcpu
)
1202 gpa_t gpa
= vcpu
->mmu
.gva_to_gpa(vcpu
, addr
);
1203 unsigned offset
= addr
& (PAGE_SIZE
-1);
1204 unsigned tocopy
= min(bytes
, (unsigned)PAGE_SIZE
- offset
);
1207 if (gpa
== UNMAPPED_GVA
)
1208 return X86EMUL_PROPAGATE_FAULT
;
1209 ret
= kvm_read_guest(vcpu
->kvm
, gpa
, data
, tocopy
);
1211 return X86EMUL_UNHANDLEABLE
;
1218 return X86EMUL_CONTINUE
;
1220 EXPORT_SYMBOL_GPL(emulator_read_std
);
1222 static int emulator_write_std(unsigned long addr
,
1225 struct kvm_vcpu
*vcpu
)
1227 pr_unimpl(vcpu
, "emulator_write_std: addr %lx n %d\n", addr
, bytes
);
1228 return X86EMUL_UNHANDLEABLE
;
1232 * Only apic need an MMIO device hook, so shortcut now..
1234 static struct kvm_io_device
*vcpu_find_pervcpu_dev(struct kvm_vcpu
*vcpu
,
1237 struct kvm_io_device
*dev
;
1240 dev
= &vcpu
->apic
->dev
;
1241 if (dev
->in_range(dev
, addr
))
1247 static struct kvm_io_device
*vcpu_find_mmio_dev(struct kvm_vcpu
*vcpu
,
1250 struct kvm_io_device
*dev
;
1252 dev
= vcpu_find_pervcpu_dev(vcpu
, addr
);
1254 dev
= kvm_io_bus_find_dev(&vcpu
->kvm
->mmio_bus
, addr
);
1258 static struct kvm_io_device
*vcpu_find_pio_dev(struct kvm_vcpu
*vcpu
,
1261 return kvm_io_bus_find_dev(&vcpu
->kvm
->pio_bus
, addr
);
1264 static int emulator_read_emulated(unsigned long addr
,
1267 struct kvm_vcpu
*vcpu
)
1269 struct kvm_io_device
*mmio_dev
;
1272 if (vcpu
->mmio_read_completed
) {
1273 memcpy(val
, vcpu
->mmio_data
, bytes
);
1274 vcpu
->mmio_read_completed
= 0;
1275 return X86EMUL_CONTINUE
;
1276 } else if (emulator_read_std(addr
, val
, bytes
, vcpu
)
1277 == X86EMUL_CONTINUE
)
1278 return X86EMUL_CONTINUE
;
1280 gpa
= vcpu
->mmu
.gva_to_gpa(vcpu
, addr
);
1281 if (gpa
== UNMAPPED_GVA
)
1282 return X86EMUL_PROPAGATE_FAULT
;
1285 * Is this MMIO handled locally?
1287 mmio_dev
= vcpu_find_mmio_dev(vcpu
, gpa
);
1289 kvm_iodevice_read(mmio_dev
, gpa
, bytes
, val
);
1290 return X86EMUL_CONTINUE
;
1293 vcpu
->mmio_needed
= 1;
1294 vcpu
->mmio_phys_addr
= gpa
;
1295 vcpu
->mmio_size
= bytes
;
1296 vcpu
->mmio_is_write
= 0;
1298 return X86EMUL_UNHANDLEABLE
;
1301 static int emulator_write_phys(struct kvm_vcpu
*vcpu
, gpa_t gpa
,
1302 const void *val
, int bytes
)
1306 ret
= kvm_write_guest(vcpu
->kvm
, gpa
, val
, bytes
);
1309 kvm_mmu_pte_write(vcpu
, gpa
, val
, bytes
);
1313 static int emulator_write_emulated_onepage(unsigned long addr
,
1316 struct kvm_vcpu
*vcpu
)
1318 struct kvm_io_device
*mmio_dev
;
1319 gpa_t gpa
= vcpu
->mmu
.gva_to_gpa(vcpu
, addr
);
1321 if (gpa
== UNMAPPED_GVA
) {
1322 kvm_x86_ops
->inject_page_fault(vcpu
, addr
, 2);
1323 return X86EMUL_PROPAGATE_FAULT
;
1326 if (emulator_write_phys(vcpu
, gpa
, val
, bytes
))
1327 return X86EMUL_CONTINUE
;
1330 * Is this MMIO handled locally?
1332 mmio_dev
= vcpu_find_mmio_dev(vcpu
, gpa
);
1334 kvm_iodevice_write(mmio_dev
, gpa
, bytes
, val
);
1335 return X86EMUL_CONTINUE
;
1338 vcpu
->mmio_needed
= 1;
1339 vcpu
->mmio_phys_addr
= gpa
;
1340 vcpu
->mmio_size
= bytes
;
1341 vcpu
->mmio_is_write
= 1;
1342 memcpy(vcpu
->mmio_data
, val
, bytes
);
1344 return X86EMUL_CONTINUE
;
1347 int emulator_write_emulated(unsigned long addr
,
1350 struct kvm_vcpu
*vcpu
)
1352 /* Crossing a page boundary? */
1353 if (((addr
+ bytes
- 1) ^ addr
) & PAGE_MASK
) {
1356 now
= -addr
& ~PAGE_MASK
;
1357 rc
= emulator_write_emulated_onepage(addr
, val
, now
, vcpu
);
1358 if (rc
!= X86EMUL_CONTINUE
)
1364 return emulator_write_emulated_onepage(addr
, val
, bytes
, vcpu
);
1366 EXPORT_SYMBOL_GPL(emulator_write_emulated
);
1368 static int emulator_cmpxchg_emulated(unsigned long addr
,
1372 struct kvm_vcpu
*vcpu
)
1374 static int reported
;
1378 printk(KERN_WARNING
"kvm: emulating exchange as write\n");
1380 return emulator_write_emulated(addr
, new, bytes
, vcpu
);
1383 static unsigned long get_segment_base(struct kvm_vcpu
*vcpu
, int seg
)
1385 return kvm_x86_ops
->get_segment_base(vcpu
, seg
);
1388 int emulate_invlpg(struct kvm_vcpu
*vcpu
, gva_t address
)
1390 return X86EMUL_CONTINUE
;
1393 int emulate_clts(struct kvm_vcpu
*vcpu
)
1395 kvm_x86_ops
->set_cr0(vcpu
, vcpu
->cr0
& ~X86_CR0_TS
);
1396 return X86EMUL_CONTINUE
;
1399 int emulator_get_dr(struct x86_emulate_ctxt
*ctxt
, int dr
, unsigned long *dest
)
1401 struct kvm_vcpu
*vcpu
= ctxt
->vcpu
;
1405 *dest
= kvm_x86_ops
->get_dr(vcpu
, dr
);
1406 return X86EMUL_CONTINUE
;
1408 pr_unimpl(vcpu
, "%s: unexpected dr %u\n", __FUNCTION__
, dr
);
1409 return X86EMUL_UNHANDLEABLE
;
1413 int emulator_set_dr(struct x86_emulate_ctxt
*ctxt
, int dr
, unsigned long value
)
1415 unsigned long mask
= (ctxt
->mode
== X86EMUL_MODE_PROT64
) ? ~0ULL : ~0U;
1418 kvm_x86_ops
->set_dr(ctxt
->vcpu
, dr
, value
& mask
, &exception
);
1420 /* FIXME: better handling */
1421 return X86EMUL_UNHANDLEABLE
;
1423 return X86EMUL_CONTINUE
;
1426 void kvm_report_emulation_failure(struct kvm_vcpu
*vcpu
, const char *context
)
1428 static int reported
;
1430 unsigned long rip
= vcpu
->rip
;
1431 unsigned long rip_linear
;
1433 rip_linear
= rip
+ get_segment_base(vcpu
, VCPU_SREG_CS
);
1438 emulator_read_std(rip_linear
, (void *)opcodes
, 4, vcpu
);
1440 printk(KERN_ERR
"emulation failed (%s) rip %lx %02x %02x %02x %02x\n",
1441 context
, rip
, opcodes
[0], opcodes
[1], opcodes
[2], opcodes
[3]);
1444 EXPORT_SYMBOL_GPL(kvm_report_emulation_failure
);
1446 struct x86_emulate_ops emulate_ops
= {
1447 .read_std
= emulator_read_std
,
1448 .write_std
= emulator_write_std
,
1449 .read_emulated
= emulator_read_emulated
,
1450 .write_emulated
= emulator_write_emulated
,
1451 .cmpxchg_emulated
= emulator_cmpxchg_emulated
,
1454 int emulate_instruction(struct kvm_vcpu
*vcpu
,
1455 struct kvm_run
*run
,
1462 vcpu
->mmio_fault_cr2
= cr2
;
1463 kvm_x86_ops
->cache_regs(vcpu
);
1465 vcpu
->mmio_is_write
= 0;
1466 vcpu
->pio
.string
= 0;
1470 kvm_x86_ops
->get_cs_db_l_bits(vcpu
, &cs_db
, &cs_l
);
1472 vcpu
->emulate_ctxt
.vcpu
= vcpu
;
1473 vcpu
->emulate_ctxt
.eflags
= kvm_x86_ops
->get_rflags(vcpu
);
1474 vcpu
->emulate_ctxt
.cr2
= cr2
;
1475 vcpu
->emulate_ctxt
.mode
=
1476 (vcpu
->emulate_ctxt
.eflags
& X86_EFLAGS_VM
)
1477 ? X86EMUL_MODE_REAL
: cs_l
1478 ? X86EMUL_MODE_PROT64
: cs_db
1479 ? X86EMUL_MODE_PROT32
: X86EMUL_MODE_PROT16
;
1481 if (vcpu
->emulate_ctxt
.mode
== X86EMUL_MODE_PROT64
) {
1482 vcpu
->emulate_ctxt
.cs_base
= 0;
1483 vcpu
->emulate_ctxt
.ds_base
= 0;
1484 vcpu
->emulate_ctxt
.es_base
= 0;
1485 vcpu
->emulate_ctxt
.ss_base
= 0;
1487 vcpu
->emulate_ctxt
.cs_base
=
1488 get_segment_base(vcpu
, VCPU_SREG_CS
);
1489 vcpu
->emulate_ctxt
.ds_base
=
1490 get_segment_base(vcpu
, VCPU_SREG_DS
);
1491 vcpu
->emulate_ctxt
.es_base
=
1492 get_segment_base(vcpu
, VCPU_SREG_ES
);
1493 vcpu
->emulate_ctxt
.ss_base
=
1494 get_segment_base(vcpu
, VCPU_SREG_SS
);
1497 vcpu
->emulate_ctxt
.gs_base
=
1498 get_segment_base(vcpu
, VCPU_SREG_GS
);
1499 vcpu
->emulate_ctxt
.fs_base
=
1500 get_segment_base(vcpu
, VCPU_SREG_FS
);
1502 r
= x86_decode_insn(&vcpu
->emulate_ctxt
, &emulate_ops
);
1504 if (kvm_mmu_unprotect_page_virt(vcpu
, cr2
))
1505 return EMULATE_DONE
;
1506 return EMULATE_FAIL
;
1510 r
= x86_emulate_insn(&vcpu
->emulate_ctxt
, &emulate_ops
);
1512 if (vcpu
->pio
.string
)
1513 return EMULATE_DO_MMIO
;
1515 if ((r
|| vcpu
->mmio_is_write
) && run
) {
1516 run
->exit_reason
= KVM_EXIT_MMIO
;
1517 run
->mmio
.phys_addr
= vcpu
->mmio_phys_addr
;
1518 memcpy(run
->mmio
.data
, vcpu
->mmio_data
, 8);
1519 run
->mmio
.len
= vcpu
->mmio_size
;
1520 run
->mmio
.is_write
= vcpu
->mmio_is_write
;
1524 if (kvm_mmu_unprotect_page_virt(vcpu
, cr2
))
1525 return EMULATE_DONE
;
1526 if (!vcpu
->mmio_needed
) {
1527 kvm_report_emulation_failure(vcpu
, "mmio");
1528 return EMULATE_FAIL
;
1530 return EMULATE_DO_MMIO
;
1533 kvm_x86_ops
->decache_regs(vcpu
);
1534 kvm_x86_ops
->set_rflags(vcpu
, vcpu
->emulate_ctxt
.eflags
);
1536 if (vcpu
->mmio_is_write
) {
1537 vcpu
->mmio_needed
= 0;
1538 return EMULATE_DO_MMIO
;
1541 return EMULATE_DONE
;
1543 EXPORT_SYMBOL_GPL(emulate_instruction
);
1546 * The vCPU has executed a HLT instruction with in-kernel mode enabled.
1548 static void kvm_vcpu_block(struct kvm_vcpu
*vcpu
)
1550 DECLARE_WAITQUEUE(wait
, current
);
1552 add_wait_queue(&vcpu
->wq
, &wait
);
1555 * We will block until either an interrupt or a signal wakes us up
1557 while (!kvm_cpu_has_interrupt(vcpu
)
1558 && !signal_pending(current
)
1559 && vcpu
->mp_state
!= VCPU_MP_STATE_RUNNABLE
1560 && vcpu
->mp_state
!= VCPU_MP_STATE_SIPI_RECEIVED
) {
1561 set_current_state(TASK_INTERRUPTIBLE
);
1567 __set_current_state(TASK_RUNNING
);
1568 remove_wait_queue(&vcpu
->wq
, &wait
);
1571 int kvm_emulate_halt(struct kvm_vcpu
*vcpu
)
1573 ++vcpu
->stat
.halt_exits
;
1574 if (irqchip_in_kernel(vcpu
->kvm
)) {
1575 vcpu
->mp_state
= VCPU_MP_STATE_HALTED
;
1576 kvm_vcpu_block(vcpu
);
1577 if (vcpu
->mp_state
!= VCPU_MP_STATE_RUNNABLE
)
1581 vcpu
->run
->exit_reason
= KVM_EXIT_HLT
;
1585 EXPORT_SYMBOL_GPL(kvm_emulate_halt
);
1587 int kvm_emulate_hypercall(struct kvm_vcpu
*vcpu
)
1589 unsigned long nr
, a0
, a1
, a2
, a3
, ret
;
1591 kvm_x86_ops
->cache_regs(vcpu
);
1593 nr
= vcpu
->regs
[VCPU_REGS_RAX
];
1594 a0
= vcpu
->regs
[VCPU_REGS_RBX
];
1595 a1
= vcpu
->regs
[VCPU_REGS_RCX
];
1596 a2
= vcpu
->regs
[VCPU_REGS_RDX
];
1597 a3
= vcpu
->regs
[VCPU_REGS_RSI
];
1599 if (!is_long_mode(vcpu
)) {
1612 vcpu
->regs
[VCPU_REGS_RAX
] = ret
;
1613 kvm_x86_ops
->decache_regs(vcpu
);
1616 EXPORT_SYMBOL_GPL(kvm_emulate_hypercall
);
1618 int kvm_fix_hypercall(struct kvm_vcpu
*vcpu
)
1620 char instruction
[3];
1623 mutex_lock(&vcpu
->kvm
->lock
);
1626 * Blow out the MMU to ensure that no other VCPU has an active mapping
1627 * to ensure that the updated hypercall appears atomically across all
1630 kvm_mmu_zap_all(vcpu
->kvm
);
1632 kvm_x86_ops
->cache_regs(vcpu
);
1633 kvm_x86_ops
->patch_hypercall(vcpu
, instruction
);
1634 if (emulator_write_emulated(vcpu
->rip
, instruction
, 3, vcpu
)
1635 != X86EMUL_CONTINUE
)
1638 mutex_unlock(&vcpu
->kvm
->lock
);
1643 static u64
mk_cr_64(u64 curr_cr
, u32 new_val
)
1645 return (curr_cr
& ~((1ULL << 32) - 1)) | new_val
;
1648 void realmode_lgdt(struct kvm_vcpu
*vcpu
, u16 limit
, unsigned long base
)
1650 struct descriptor_table dt
= { limit
, base
};
1652 kvm_x86_ops
->set_gdt(vcpu
, &dt
);
1655 void realmode_lidt(struct kvm_vcpu
*vcpu
, u16 limit
, unsigned long base
)
1657 struct descriptor_table dt
= { limit
, base
};
1659 kvm_x86_ops
->set_idt(vcpu
, &dt
);
1662 void realmode_lmsw(struct kvm_vcpu
*vcpu
, unsigned long msw
,
1663 unsigned long *rflags
)
1666 *rflags
= kvm_x86_ops
->get_rflags(vcpu
);
1669 unsigned long realmode_get_cr(struct kvm_vcpu
*vcpu
, int cr
)
1671 kvm_x86_ops
->decache_cr4_guest_bits(vcpu
);
1682 vcpu_printf(vcpu
, "%s: unexpected cr %u\n", __FUNCTION__
, cr
);
1687 void realmode_set_cr(struct kvm_vcpu
*vcpu
, int cr
, unsigned long val
,
1688 unsigned long *rflags
)
1692 set_cr0(vcpu
, mk_cr_64(vcpu
->cr0
, val
));
1693 *rflags
= kvm_x86_ops
->get_rflags(vcpu
);
1702 set_cr4(vcpu
, mk_cr_64(vcpu
->cr4
, val
));
1705 vcpu_printf(vcpu
, "%s: unexpected cr %u\n", __FUNCTION__
, cr
);
1709 int kvm_get_msr_common(struct kvm_vcpu
*vcpu
, u32 msr
, u64
*pdata
)
1714 case 0xc0010010: /* SYSCFG */
1715 case 0xc0010015: /* HWCR */
1716 case MSR_IA32_PLATFORM_ID
:
1717 case MSR_IA32_P5_MC_ADDR
:
1718 case MSR_IA32_P5_MC_TYPE
:
1719 case MSR_IA32_MC0_CTL
:
1720 case MSR_IA32_MCG_STATUS
:
1721 case MSR_IA32_MCG_CAP
:
1722 case MSR_IA32_MC0_MISC
:
1723 case MSR_IA32_MC0_MISC
+4:
1724 case MSR_IA32_MC0_MISC
+8:
1725 case MSR_IA32_MC0_MISC
+12:
1726 case MSR_IA32_MC0_MISC
+16:
1727 case MSR_IA32_UCODE_REV
:
1728 case MSR_IA32_PERF_STATUS
:
1729 case MSR_IA32_EBL_CR_POWERON
:
1730 /* MTRR registers */
1732 case 0x200 ... 0x2ff:
1735 case 0xcd: /* fsb frequency */
1738 case MSR_IA32_APICBASE
:
1739 data
= kvm_get_apic_base(vcpu
);
1741 case MSR_IA32_MISC_ENABLE
:
1742 data
= vcpu
->ia32_misc_enable_msr
;
1744 #ifdef CONFIG_X86_64
1746 data
= vcpu
->shadow_efer
;
1750 pr_unimpl(vcpu
, "unhandled rdmsr: 0x%x\n", msr
);
1756 EXPORT_SYMBOL_GPL(kvm_get_msr_common
);
1759 * Reads an msr value (of 'msr_index') into 'pdata'.
1760 * Returns 0 on success, non-0 otherwise.
1761 * Assumes vcpu_load() was already called.
1763 int kvm_get_msr(struct kvm_vcpu
*vcpu
, u32 msr_index
, u64
*pdata
)
1765 return kvm_x86_ops
->get_msr(vcpu
, msr_index
, pdata
);
1768 #ifdef CONFIG_X86_64
1770 static void set_efer(struct kvm_vcpu
*vcpu
, u64 efer
)
1772 if (efer
& EFER_RESERVED_BITS
) {
1773 printk(KERN_DEBUG
"set_efer: 0x%llx #GP, reserved bits\n",
1780 && (vcpu
->shadow_efer
& EFER_LME
) != (efer
& EFER_LME
)) {
1781 printk(KERN_DEBUG
"set_efer: #GP, change LME while paging\n");
1786 kvm_x86_ops
->set_efer(vcpu
, efer
);
1789 efer
|= vcpu
->shadow_efer
& EFER_LMA
;
1791 vcpu
->shadow_efer
= efer
;
1796 int kvm_set_msr_common(struct kvm_vcpu
*vcpu
, u32 msr
, u64 data
)
1799 #ifdef CONFIG_X86_64
1801 set_efer(vcpu
, data
);
1804 case MSR_IA32_MC0_STATUS
:
1805 pr_unimpl(vcpu
, "%s: MSR_IA32_MC0_STATUS 0x%llx, nop\n",
1806 __FUNCTION__
, data
);
1808 case MSR_IA32_MCG_STATUS
:
1809 pr_unimpl(vcpu
, "%s: MSR_IA32_MCG_STATUS 0x%llx, nop\n",
1810 __FUNCTION__
, data
);
1812 case MSR_IA32_UCODE_REV
:
1813 case MSR_IA32_UCODE_WRITE
:
1814 case 0x200 ... 0x2ff: /* MTRRs */
1816 case MSR_IA32_APICBASE
:
1817 kvm_set_apic_base(vcpu
, data
);
1819 case MSR_IA32_MISC_ENABLE
:
1820 vcpu
->ia32_misc_enable_msr
= data
;
1823 pr_unimpl(vcpu
, "unhandled wrmsr: 0x%x\n", msr
);
1828 EXPORT_SYMBOL_GPL(kvm_set_msr_common
);
1831 * Writes msr value into into the appropriate "register".
1832 * Returns 0 on success, non-0 otherwise.
1833 * Assumes vcpu_load() was already called.
1835 int kvm_set_msr(struct kvm_vcpu
*vcpu
, u32 msr_index
, u64 data
)
1837 return kvm_x86_ops
->set_msr(vcpu
, msr_index
, data
);
1840 void kvm_resched(struct kvm_vcpu
*vcpu
)
1842 if (!need_resched())
1846 EXPORT_SYMBOL_GPL(kvm_resched
);
1848 void kvm_emulate_cpuid(struct kvm_vcpu
*vcpu
)
1852 struct kvm_cpuid_entry
*e
, *best
;
1854 kvm_x86_ops
->cache_regs(vcpu
);
1855 function
= vcpu
->regs
[VCPU_REGS_RAX
];
1856 vcpu
->regs
[VCPU_REGS_RAX
] = 0;
1857 vcpu
->regs
[VCPU_REGS_RBX
] = 0;
1858 vcpu
->regs
[VCPU_REGS_RCX
] = 0;
1859 vcpu
->regs
[VCPU_REGS_RDX
] = 0;
1861 for (i
= 0; i
< vcpu
->cpuid_nent
; ++i
) {
1862 e
= &vcpu
->cpuid_entries
[i
];
1863 if (e
->function
== function
) {
1868 * Both basic or both extended?
1870 if (((e
->function
^ function
) & 0x80000000) == 0)
1871 if (!best
|| e
->function
> best
->function
)
1875 vcpu
->regs
[VCPU_REGS_RAX
] = best
->eax
;
1876 vcpu
->regs
[VCPU_REGS_RBX
] = best
->ebx
;
1877 vcpu
->regs
[VCPU_REGS_RCX
] = best
->ecx
;
1878 vcpu
->regs
[VCPU_REGS_RDX
] = best
->edx
;
1880 kvm_x86_ops
->decache_regs(vcpu
);
1881 kvm_x86_ops
->skip_emulated_instruction(vcpu
);
1883 EXPORT_SYMBOL_GPL(kvm_emulate_cpuid
);
1885 static int pio_copy_data(struct kvm_vcpu
*vcpu
)
1887 void *p
= vcpu
->pio_data
;
1890 int nr_pages
= vcpu
->pio
.guest_pages
[1] ? 2 : 1;
1892 q
= vmap(vcpu
->pio
.guest_pages
, nr_pages
, VM_READ
|VM_WRITE
,
1895 free_pio_guest_pages(vcpu
);
1898 q
+= vcpu
->pio
.guest_page_offset
;
1899 bytes
= vcpu
->pio
.size
* vcpu
->pio
.cur_count
;
1901 memcpy(q
, p
, bytes
);
1903 memcpy(p
, q
, bytes
);
1904 q
-= vcpu
->pio
.guest_page_offset
;
1906 free_pio_guest_pages(vcpu
);
1910 static int complete_pio(struct kvm_vcpu
*vcpu
)
1912 struct kvm_pio_request
*io
= &vcpu
->pio
;
1916 kvm_x86_ops
->cache_regs(vcpu
);
1920 memcpy(&vcpu
->regs
[VCPU_REGS_RAX
], vcpu
->pio_data
,
1924 r
= pio_copy_data(vcpu
);
1926 kvm_x86_ops
->cache_regs(vcpu
);
1933 delta
*= io
->cur_count
;
1935 * The size of the register should really depend on
1936 * current address size.
1938 vcpu
->regs
[VCPU_REGS_RCX
] -= delta
;
1944 vcpu
->regs
[VCPU_REGS_RDI
] += delta
;
1946 vcpu
->regs
[VCPU_REGS_RSI
] += delta
;
1949 kvm_x86_ops
->decache_regs(vcpu
);
1951 io
->count
-= io
->cur_count
;
1957 static void kernel_pio(struct kvm_io_device
*pio_dev
,
1958 struct kvm_vcpu
*vcpu
,
1961 /* TODO: String I/O for in kernel device */
1963 mutex_lock(&vcpu
->kvm
->lock
);
1965 kvm_iodevice_read(pio_dev
, vcpu
->pio
.port
,
1969 kvm_iodevice_write(pio_dev
, vcpu
->pio
.port
,
1972 mutex_unlock(&vcpu
->kvm
->lock
);
1975 static void pio_string_write(struct kvm_io_device
*pio_dev
,
1976 struct kvm_vcpu
*vcpu
)
1978 struct kvm_pio_request
*io
= &vcpu
->pio
;
1979 void *pd
= vcpu
->pio_data
;
1982 mutex_lock(&vcpu
->kvm
->lock
);
1983 for (i
= 0; i
< io
->cur_count
; i
++) {
1984 kvm_iodevice_write(pio_dev
, io
->port
,
1989 mutex_unlock(&vcpu
->kvm
->lock
);
1992 int kvm_emulate_pio(struct kvm_vcpu
*vcpu
, struct kvm_run
*run
, int in
,
1993 int size
, unsigned port
)
1995 struct kvm_io_device
*pio_dev
;
1997 vcpu
->run
->exit_reason
= KVM_EXIT_IO
;
1998 vcpu
->run
->io
.direction
= in
? KVM_EXIT_IO_IN
: KVM_EXIT_IO_OUT
;
1999 vcpu
->run
->io
.size
= vcpu
->pio
.size
= size
;
2000 vcpu
->run
->io
.data_offset
= KVM_PIO_PAGE_OFFSET
* PAGE_SIZE
;
2001 vcpu
->run
->io
.count
= vcpu
->pio
.count
= vcpu
->pio
.cur_count
= 1;
2002 vcpu
->run
->io
.port
= vcpu
->pio
.port
= port
;
2004 vcpu
->pio
.string
= 0;
2006 vcpu
->pio
.guest_page_offset
= 0;
2009 kvm_x86_ops
->cache_regs(vcpu
);
2010 memcpy(vcpu
->pio_data
, &vcpu
->regs
[VCPU_REGS_RAX
], 4);
2011 kvm_x86_ops
->decache_regs(vcpu
);
2013 kvm_x86_ops
->skip_emulated_instruction(vcpu
);
2015 pio_dev
= vcpu_find_pio_dev(vcpu
, port
);
2017 kernel_pio(pio_dev
, vcpu
, vcpu
->pio_data
);
2023 EXPORT_SYMBOL_GPL(kvm_emulate_pio
);
2025 int kvm_emulate_pio_string(struct kvm_vcpu
*vcpu
, struct kvm_run
*run
, int in
,
2026 int size
, unsigned long count
, int down
,
2027 gva_t address
, int rep
, unsigned port
)
2029 unsigned now
, in_page
;
2033 struct kvm_io_device
*pio_dev
;
2035 vcpu
->run
->exit_reason
= KVM_EXIT_IO
;
2036 vcpu
->run
->io
.direction
= in
? KVM_EXIT_IO_IN
: KVM_EXIT_IO_OUT
;
2037 vcpu
->run
->io
.size
= vcpu
->pio
.size
= size
;
2038 vcpu
->run
->io
.data_offset
= KVM_PIO_PAGE_OFFSET
* PAGE_SIZE
;
2039 vcpu
->run
->io
.count
= vcpu
->pio
.count
= vcpu
->pio
.cur_count
= count
;
2040 vcpu
->run
->io
.port
= vcpu
->pio
.port
= port
;
2042 vcpu
->pio
.string
= 1;
2043 vcpu
->pio
.down
= down
;
2044 vcpu
->pio
.guest_page_offset
= offset_in_page(address
);
2045 vcpu
->pio
.rep
= rep
;
2048 kvm_x86_ops
->skip_emulated_instruction(vcpu
);
2053 in_page
= PAGE_SIZE
- offset_in_page(address
);
2055 in_page
= offset_in_page(address
) + size
;
2056 now
= min(count
, (unsigned long)in_page
/ size
);
2059 * String I/O straddles page boundary. Pin two guest pages
2060 * so that we satisfy atomicity constraints. Do just one
2061 * transaction to avoid complexity.
2068 * String I/O in reverse. Yuck. Kill the guest, fix later.
2070 pr_unimpl(vcpu
, "guest string pio down\n");
2074 vcpu
->run
->io
.count
= now
;
2075 vcpu
->pio
.cur_count
= now
;
2077 if (vcpu
->pio
.cur_count
== vcpu
->pio
.count
)
2078 kvm_x86_ops
->skip_emulated_instruction(vcpu
);
2080 for (i
= 0; i
< nr_pages
; ++i
) {
2081 mutex_lock(&vcpu
->kvm
->lock
);
2082 page
= gva_to_page(vcpu
, address
+ i
* PAGE_SIZE
);
2083 vcpu
->pio
.guest_pages
[i
] = page
;
2084 mutex_unlock(&vcpu
->kvm
->lock
);
2087 free_pio_guest_pages(vcpu
);
2092 pio_dev
= vcpu_find_pio_dev(vcpu
, port
);
2093 if (!vcpu
->pio
.in
) {
2094 /* string PIO write */
2095 ret
= pio_copy_data(vcpu
);
2096 if (ret
>= 0 && pio_dev
) {
2097 pio_string_write(pio_dev
, vcpu
);
2099 if (vcpu
->pio
.count
== 0)
2103 pr_unimpl(vcpu
, "no string pio read support yet, "
2104 "port %x size %d count %ld\n",
2109 EXPORT_SYMBOL_GPL(kvm_emulate_pio_string
);
2112 * Check if userspace requested an interrupt window, and that the
2113 * interrupt window is open.
2115 * No need to exit to userspace if we already have an interrupt queued.
2117 static int dm_request_for_irq_injection(struct kvm_vcpu
*vcpu
,
2118 struct kvm_run
*kvm_run
)
2120 return (!vcpu
->irq_summary
&&
2121 kvm_run
->request_interrupt_window
&&
2122 vcpu
->interrupt_window_open
&&
2123 (kvm_x86_ops
->get_rflags(vcpu
) & X86_EFLAGS_IF
));
2126 static void post_kvm_run_save(struct kvm_vcpu
*vcpu
,
2127 struct kvm_run
*kvm_run
)
2129 kvm_run
->if_flag
= (kvm_x86_ops
->get_rflags(vcpu
) & X86_EFLAGS_IF
) != 0;
2130 kvm_run
->cr8
= get_cr8(vcpu
);
2131 kvm_run
->apic_base
= kvm_get_apic_base(vcpu
);
2132 if (irqchip_in_kernel(vcpu
->kvm
))
2133 kvm_run
->ready_for_interrupt_injection
= 1;
2135 kvm_run
->ready_for_interrupt_injection
=
2136 (vcpu
->interrupt_window_open
&&
2137 vcpu
->irq_summary
== 0);
2140 static int __vcpu_run(struct kvm_vcpu
*vcpu
, struct kvm_run
*kvm_run
)
2144 if (unlikely(vcpu
->mp_state
== VCPU_MP_STATE_SIPI_RECEIVED
)) {
2145 pr_debug("vcpu %d received sipi with vector # %x\n",
2146 vcpu
->vcpu_id
, vcpu
->sipi_vector
);
2147 kvm_lapic_reset(vcpu
);
2148 kvm_x86_ops
->vcpu_reset(vcpu
);
2149 vcpu
->mp_state
= VCPU_MP_STATE_RUNNABLE
;
2153 if (vcpu
->guest_debug
.enabled
)
2154 kvm_x86_ops
->guest_debug_pre(vcpu
);
2157 r
= kvm_mmu_reload(vcpu
);
2161 kvm_inject_pending_timer_irqs(vcpu
);
2165 kvm_x86_ops
->prepare_guest_switch(vcpu
);
2166 kvm_load_guest_fpu(vcpu
);
2168 local_irq_disable();
2170 if (signal_pending(current
)) {
2174 kvm_run
->exit_reason
= KVM_EXIT_INTR
;
2175 ++vcpu
->stat
.signal_exits
;
2179 if (irqchip_in_kernel(vcpu
->kvm
))
2180 kvm_x86_ops
->inject_pending_irq(vcpu
);
2181 else if (!vcpu
->mmio_read_completed
)
2182 kvm_x86_ops
->inject_pending_vectors(vcpu
, kvm_run
);
2184 vcpu
->guest_mode
= 1;
2188 if (test_and_clear_bit(KVM_REQ_TLB_FLUSH
, &vcpu
->requests
))
2189 kvm_x86_ops
->tlb_flush(vcpu
);
2191 kvm_x86_ops
->run(vcpu
, kvm_run
);
2193 vcpu
->guest_mode
= 0;
2199 * We must have an instruction between local_irq_enable() and
2200 * kvm_guest_exit(), so the timer interrupt isn't delayed by
2201 * the interrupt shadow. The stat.exits increment will do nicely.
2202 * But we need to prevent reordering, hence this barrier():
2211 * Profile KVM exit RIPs:
2213 if (unlikely(prof_on
== KVM_PROFILING
)) {
2214 kvm_x86_ops
->cache_regs(vcpu
);
2215 profile_hit(KVM_PROFILING
, (void *)vcpu
->rip
);
2218 r
= kvm_x86_ops
->handle_exit(kvm_run
, vcpu
);
2221 if (dm_request_for_irq_injection(vcpu
, kvm_run
)) {
2223 kvm_run
->exit_reason
= KVM_EXIT_INTR
;
2224 ++vcpu
->stat
.request_irq_exits
;
2227 if (!need_resched()) {
2228 ++vcpu
->stat
.light_exits
;
2239 post_kvm_run_save(vcpu
, kvm_run
);
2245 static int kvm_vcpu_ioctl_run(struct kvm_vcpu
*vcpu
, struct kvm_run
*kvm_run
)
2252 if (unlikely(vcpu
->mp_state
== VCPU_MP_STATE_UNINITIALIZED
)) {
2253 kvm_vcpu_block(vcpu
);
2258 if (vcpu
->sigset_active
)
2259 sigprocmask(SIG_SETMASK
, &vcpu
->sigset
, &sigsaved
);
2261 /* re-sync apic's tpr */
2262 if (!irqchip_in_kernel(vcpu
->kvm
))
2263 set_cr8(vcpu
, kvm_run
->cr8
);
2265 if (vcpu
->pio
.cur_count
) {
2266 r
= complete_pio(vcpu
);
2271 if (vcpu
->mmio_needed
) {
2272 memcpy(vcpu
->mmio_data
, kvm_run
->mmio
.data
, 8);
2273 vcpu
->mmio_read_completed
= 1;
2274 vcpu
->mmio_needed
= 0;
2275 r
= emulate_instruction(vcpu
, kvm_run
,
2276 vcpu
->mmio_fault_cr2
, 0, 1);
2277 if (r
== EMULATE_DO_MMIO
) {
2279 * Read-modify-write. Back to userspace.
2286 if (kvm_run
->exit_reason
== KVM_EXIT_HYPERCALL
) {
2287 kvm_x86_ops
->cache_regs(vcpu
);
2288 vcpu
->regs
[VCPU_REGS_RAX
] = kvm_run
->hypercall
.ret
;
2289 kvm_x86_ops
->decache_regs(vcpu
);
2292 r
= __vcpu_run(vcpu
, kvm_run
);
2295 if (vcpu
->sigset_active
)
2296 sigprocmask(SIG_SETMASK
, &sigsaved
, NULL
);
2302 static int kvm_vcpu_ioctl_get_regs(struct kvm_vcpu
*vcpu
,
2303 struct kvm_regs
*regs
)
2307 kvm_x86_ops
->cache_regs(vcpu
);
2309 regs
->rax
= vcpu
->regs
[VCPU_REGS_RAX
];
2310 regs
->rbx
= vcpu
->regs
[VCPU_REGS_RBX
];
2311 regs
->rcx
= vcpu
->regs
[VCPU_REGS_RCX
];
2312 regs
->rdx
= vcpu
->regs
[VCPU_REGS_RDX
];
2313 regs
->rsi
= vcpu
->regs
[VCPU_REGS_RSI
];
2314 regs
->rdi
= vcpu
->regs
[VCPU_REGS_RDI
];
2315 regs
->rsp
= vcpu
->regs
[VCPU_REGS_RSP
];
2316 regs
->rbp
= vcpu
->regs
[VCPU_REGS_RBP
];
2317 #ifdef CONFIG_X86_64
2318 regs
->r8
= vcpu
->regs
[VCPU_REGS_R8
];
2319 regs
->r9
= vcpu
->regs
[VCPU_REGS_R9
];
2320 regs
->r10
= vcpu
->regs
[VCPU_REGS_R10
];
2321 regs
->r11
= vcpu
->regs
[VCPU_REGS_R11
];
2322 regs
->r12
= vcpu
->regs
[VCPU_REGS_R12
];
2323 regs
->r13
= vcpu
->regs
[VCPU_REGS_R13
];
2324 regs
->r14
= vcpu
->regs
[VCPU_REGS_R14
];
2325 regs
->r15
= vcpu
->regs
[VCPU_REGS_R15
];
2328 regs
->rip
= vcpu
->rip
;
2329 regs
->rflags
= kvm_x86_ops
->get_rflags(vcpu
);
2332 * Don't leak debug flags in case they were set for guest debugging
2334 if (vcpu
->guest_debug
.enabled
&& vcpu
->guest_debug
.singlestep
)
2335 regs
->rflags
&= ~(X86_EFLAGS_TF
| X86_EFLAGS_RF
);
2342 static int kvm_vcpu_ioctl_set_regs(struct kvm_vcpu
*vcpu
,
2343 struct kvm_regs
*regs
)
2347 vcpu
->regs
[VCPU_REGS_RAX
] = regs
->rax
;
2348 vcpu
->regs
[VCPU_REGS_RBX
] = regs
->rbx
;
2349 vcpu
->regs
[VCPU_REGS_RCX
] = regs
->rcx
;
2350 vcpu
->regs
[VCPU_REGS_RDX
] = regs
->rdx
;
2351 vcpu
->regs
[VCPU_REGS_RSI
] = regs
->rsi
;
2352 vcpu
->regs
[VCPU_REGS_RDI
] = regs
->rdi
;
2353 vcpu
->regs
[VCPU_REGS_RSP
] = regs
->rsp
;
2354 vcpu
->regs
[VCPU_REGS_RBP
] = regs
->rbp
;
2355 #ifdef CONFIG_X86_64
2356 vcpu
->regs
[VCPU_REGS_R8
] = regs
->r8
;
2357 vcpu
->regs
[VCPU_REGS_R9
] = regs
->r9
;
2358 vcpu
->regs
[VCPU_REGS_R10
] = regs
->r10
;
2359 vcpu
->regs
[VCPU_REGS_R11
] = regs
->r11
;
2360 vcpu
->regs
[VCPU_REGS_R12
] = regs
->r12
;
2361 vcpu
->regs
[VCPU_REGS_R13
] = regs
->r13
;
2362 vcpu
->regs
[VCPU_REGS_R14
] = regs
->r14
;
2363 vcpu
->regs
[VCPU_REGS_R15
] = regs
->r15
;
2366 vcpu
->rip
= regs
->rip
;
2367 kvm_x86_ops
->set_rflags(vcpu
, regs
->rflags
);
2369 kvm_x86_ops
->decache_regs(vcpu
);
2376 static void get_segment(struct kvm_vcpu
*vcpu
,
2377 struct kvm_segment
*var
, int seg
)
2379 return kvm_x86_ops
->get_segment(vcpu
, var
, seg
);
2382 static int kvm_vcpu_ioctl_get_sregs(struct kvm_vcpu
*vcpu
,
2383 struct kvm_sregs
*sregs
)
2385 struct descriptor_table dt
;
2390 get_segment(vcpu
, &sregs
->cs
, VCPU_SREG_CS
);
2391 get_segment(vcpu
, &sregs
->ds
, VCPU_SREG_DS
);
2392 get_segment(vcpu
, &sregs
->es
, VCPU_SREG_ES
);
2393 get_segment(vcpu
, &sregs
->fs
, VCPU_SREG_FS
);
2394 get_segment(vcpu
, &sregs
->gs
, VCPU_SREG_GS
);
2395 get_segment(vcpu
, &sregs
->ss
, VCPU_SREG_SS
);
2397 get_segment(vcpu
, &sregs
->tr
, VCPU_SREG_TR
);
2398 get_segment(vcpu
, &sregs
->ldt
, VCPU_SREG_LDTR
);
2400 kvm_x86_ops
->get_idt(vcpu
, &dt
);
2401 sregs
->idt
.limit
= dt
.limit
;
2402 sregs
->idt
.base
= dt
.base
;
2403 kvm_x86_ops
->get_gdt(vcpu
, &dt
);
2404 sregs
->gdt
.limit
= dt
.limit
;
2405 sregs
->gdt
.base
= dt
.base
;
2407 kvm_x86_ops
->decache_cr4_guest_bits(vcpu
);
2408 sregs
->cr0
= vcpu
->cr0
;
2409 sregs
->cr2
= vcpu
->cr2
;
2410 sregs
->cr3
= vcpu
->cr3
;
2411 sregs
->cr4
= vcpu
->cr4
;
2412 sregs
->cr8
= get_cr8(vcpu
);
2413 sregs
->efer
= vcpu
->shadow_efer
;
2414 sregs
->apic_base
= kvm_get_apic_base(vcpu
);
2416 if (irqchip_in_kernel(vcpu
->kvm
)) {
2417 memset(sregs
->interrupt_bitmap
, 0,
2418 sizeof sregs
->interrupt_bitmap
);
2419 pending_vec
= kvm_x86_ops
->get_irq(vcpu
);
2420 if (pending_vec
>= 0)
2421 set_bit(pending_vec
,
2422 (unsigned long *)sregs
->interrupt_bitmap
);
2424 memcpy(sregs
->interrupt_bitmap
, vcpu
->irq_pending
,
2425 sizeof sregs
->interrupt_bitmap
);
2432 static void set_segment(struct kvm_vcpu
*vcpu
,
2433 struct kvm_segment
*var
, int seg
)
2435 return kvm_x86_ops
->set_segment(vcpu
, var
, seg
);
2438 static int kvm_vcpu_ioctl_set_sregs(struct kvm_vcpu
*vcpu
,
2439 struct kvm_sregs
*sregs
)
2441 int mmu_reset_needed
= 0;
2442 int i
, pending_vec
, max_bits
;
2443 struct descriptor_table dt
;
2447 dt
.limit
= sregs
->idt
.limit
;
2448 dt
.base
= sregs
->idt
.base
;
2449 kvm_x86_ops
->set_idt(vcpu
, &dt
);
2450 dt
.limit
= sregs
->gdt
.limit
;
2451 dt
.base
= sregs
->gdt
.base
;
2452 kvm_x86_ops
->set_gdt(vcpu
, &dt
);
2454 vcpu
->cr2
= sregs
->cr2
;
2455 mmu_reset_needed
|= vcpu
->cr3
!= sregs
->cr3
;
2456 vcpu
->cr3
= sregs
->cr3
;
2458 set_cr8(vcpu
, sregs
->cr8
);
2460 mmu_reset_needed
|= vcpu
->shadow_efer
!= sregs
->efer
;
2461 #ifdef CONFIG_X86_64
2462 kvm_x86_ops
->set_efer(vcpu
, sregs
->efer
);
2464 kvm_set_apic_base(vcpu
, sregs
->apic_base
);
2466 kvm_x86_ops
->decache_cr4_guest_bits(vcpu
);
2468 mmu_reset_needed
|= vcpu
->cr0
!= sregs
->cr0
;
2469 vcpu
->cr0
= sregs
->cr0
;
2470 kvm_x86_ops
->set_cr0(vcpu
, sregs
->cr0
);
2472 mmu_reset_needed
|= vcpu
->cr4
!= sregs
->cr4
;
2473 kvm_x86_ops
->set_cr4(vcpu
, sregs
->cr4
);
2474 if (!is_long_mode(vcpu
) && is_pae(vcpu
))
2475 load_pdptrs(vcpu
, vcpu
->cr3
);
2477 if (mmu_reset_needed
)
2478 kvm_mmu_reset_context(vcpu
);
2480 if (!irqchip_in_kernel(vcpu
->kvm
)) {
2481 memcpy(vcpu
->irq_pending
, sregs
->interrupt_bitmap
,
2482 sizeof vcpu
->irq_pending
);
2483 vcpu
->irq_summary
= 0;
2484 for (i
= 0; i
< ARRAY_SIZE(vcpu
->irq_pending
); ++i
)
2485 if (vcpu
->irq_pending
[i
])
2486 __set_bit(i
, &vcpu
->irq_summary
);
2488 max_bits
= (sizeof sregs
->interrupt_bitmap
) << 3;
2489 pending_vec
= find_first_bit(
2490 (const unsigned long *)sregs
->interrupt_bitmap
,
2492 /* Only pending external irq is handled here */
2493 if (pending_vec
< max_bits
) {
2494 kvm_x86_ops
->set_irq(vcpu
, pending_vec
);
2495 pr_debug("Set back pending irq %d\n",
2500 set_segment(vcpu
, &sregs
->cs
, VCPU_SREG_CS
);
2501 set_segment(vcpu
, &sregs
->ds
, VCPU_SREG_DS
);
2502 set_segment(vcpu
, &sregs
->es
, VCPU_SREG_ES
);
2503 set_segment(vcpu
, &sregs
->fs
, VCPU_SREG_FS
);
2504 set_segment(vcpu
, &sregs
->gs
, VCPU_SREG_GS
);
2505 set_segment(vcpu
, &sregs
->ss
, VCPU_SREG_SS
);
2507 set_segment(vcpu
, &sregs
->tr
, VCPU_SREG_TR
);
2508 set_segment(vcpu
, &sregs
->ldt
, VCPU_SREG_LDTR
);
2515 void kvm_get_cs_db_l_bits(struct kvm_vcpu
*vcpu
, int *db
, int *l
)
2517 struct kvm_segment cs
;
2519 get_segment(vcpu
, &cs
, VCPU_SREG_CS
);
2523 EXPORT_SYMBOL_GPL(kvm_get_cs_db_l_bits
);
2526 * Translate a guest virtual address to a guest physical address.
2528 static int kvm_vcpu_ioctl_translate(struct kvm_vcpu
*vcpu
,
2529 struct kvm_translation
*tr
)
2531 unsigned long vaddr
= tr
->linear_address
;
2535 mutex_lock(&vcpu
->kvm
->lock
);
2536 gpa
= vcpu
->mmu
.gva_to_gpa(vcpu
, vaddr
);
2537 tr
->physical_address
= gpa
;
2538 tr
->valid
= gpa
!= UNMAPPED_GVA
;
2541 mutex_unlock(&vcpu
->kvm
->lock
);
2547 static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu
*vcpu
,
2548 struct kvm_interrupt
*irq
)
2550 if (irq
->irq
< 0 || irq
->irq
>= 256)
2552 if (irqchip_in_kernel(vcpu
->kvm
))
2556 set_bit(irq
->irq
, vcpu
->irq_pending
);
2557 set_bit(irq
->irq
/ BITS_PER_LONG
, &vcpu
->irq_summary
);
2564 static int kvm_vcpu_ioctl_debug_guest(struct kvm_vcpu
*vcpu
,
2565 struct kvm_debug_guest
*dbg
)
2571 r
= kvm_x86_ops
->set_guest_debug(vcpu
, dbg
);
2578 static struct page
*kvm_vcpu_nopage(struct vm_area_struct
*vma
,
2579 unsigned long address
,
2582 struct kvm_vcpu
*vcpu
= vma
->vm_file
->private_data
;
2583 unsigned long pgoff
;
2586 pgoff
= ((address
- vma
->vm_start
) >> PAGE_SHIFT
) + vma
->vm_pgoff
;
2588 page
= virt_to_page(vcpu
->run
);
2589 else if (pgoff
== KVM_PIO_PAGE_OFFSET
)
2590 page
= virt_to_page(vcpu
->pio_data
);
2592 return NOPAGE_SIGBUS
;
2595 *type
= VM_FAULT_MINOR
;
2600 static struct vm_operations_struct kvm_vcpu_vm_ops
= {
2601 .nopage
= kvm_vcpu_nopage
,
2604 static int kvm_vcpu_mmap(struct file
*file
, struct vm_area_struct
*vma
)
2606 vma
->vm_ops
= &kvm_vcpu_vm_ops
;
2610 static int kvm_vcpu_release(struct inode
*inode
, struct file
*filp
)
2612 struct kvm_vcpu
*vcpu
= filp
->private_data
;
2614 fput(vcpu
->kvm
->filp
);
2618 static struct file_operations kvm_vcpu_fops
= {
2619 .release
= kvm_vcpu_release
,
2620 .unlocked_ioctl
= kvm_vcpu_ioctl
,
2621 .compat_ioctl
= kvm_vcpu_ioctl
,
2622 .mmap
= kvm_vcpu_mmap
,
2626 * Allocates an inode for the vcpu.
2628 static int create_vcpu_fd(struct kvm_vcpu
*vcpu
)
2631 struct inode
*inode
;
2634 r
= anon_inode_getfd(&fd
, &inode
, &file
,
2635 "kvm-vcpu", &kvm_vcpu_fops
, vcpu
);
2638 atomic_inc(&vcpu
->kvm
->filp
->f_count
);
2643 * Creates some virtual cpus. Good luck creating more than one.
2645 static int kvm_vm_ioctl_create_vcpu(struct kvm
*kvm
, int n
)
2648 struct kvm_vcpu
*vcpu
;
2653 vcpu
= kvm_x86_ops
->vcpu_create(kvm
, n
);
2655 return PTR_ERR(vcpu
);
2657 preempt_notifier_init(&vcpu
->preempt_notifier
, &kvm_preempt_ops
);
2659 /* We do fxsave: this must be aligned. */
2660 BUG_ON((unsigned long)&vcpu
->host_fx_image
& 0xF);
2663 r
= kvm_mmu_setup(vcpu
);
2668 mutex_lock(&kvm
->lock
);
2669 if (kvm
->vcpus
[n
]) {
2671 mutex_unlock(&kvm
->lock
);
2674 kvm
->vcpus
[n
] = vcpu
;
2675 mutex_unlock(&kvm
->lock
);
2677 /* Now it's all set up, let userspace reach it */
2678 r
= create_vcpu_fd(vcpu
);
2684 mutex_lock(&kvm
->lock
);
2685 kvm
->vcpus
[n
] = NULL
;
2686 mutex_unlock(&kvm
->lock
);
2690 kvm_mmu_unload(vcpu
);
2694 kvm_x86_ops
->vcpu_free(vcpu
);
2698 static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu
*vcpu
, sigset_t
*sigset
)
2701 sigdelsetmask(sigset
, sigmask(SIGKILL
)|sigmask(SIGSTOP
));
2702 vcpu
->sigset_active
= 1;
2703 vcpu
->sigset
= *sigset
;
2705 vcpu
->sigset_active
= 0;
2710 * fxsave fpu state. Taken from x86_64/processor.h. To be killed when
2711 * we have asm/x86/processor.h
2722 u32 st_space
[32]; /* 8*16 bytes for each FP-reg = 128 bytes */
2723 #ifdef CONFIG_X86_64
2724 u32 xmm_space
[64]; /* 16*16 bytes for each XMM-reg = 256 bytes */
2726 u32 xmm_space
[32]; /* 8*16 bytes for each XMM-reg = 128 bytes */
2730 static int kvm_vcpu_ioctl_get_fpu(struct kvm_vcpu
*vcpu
, struct kvm_fpu
*fpu
)
2732 struct fxsave
*fxsave
= (struct fxsave
*)&vcpu
->guest_fx_image
;
2736 memcpy(fpu
->fpr
, fxsave
->st_space
, 128);
2737 fpu
->fcw
= fxsave
->cwd
;
2738 fpu
->fsw
= fxsave
->swd
;
2739 fpu
->ftwx
= fxsave
->twd
;
2740 fpu
->last_opcode
= fxsave
->fop
;
2741 fpu
->last_ip
= fxsave
->rip
;
2742 fpu
->last_dp
= fxsave
->rdp
;
2743 memcpy(fpu
->xmm
, fxsave
->xmm_space
, sizeof fxsave
->xmm_space
);
2750 static int kvm_vcpu_ioctl_set_fpu(struct kvm_vcpu
*vcpu
, struct kvm_fpu
*fpu
)
2752 struct fxsave
*fxsave
= (struct fxsave
*)&vcpu
->guest_fx_image
;
2756 memcpy(fxsave
->st_space
, fpu
->fpr
, 128);
2757 fxsave
->cwd
= fpu
->fcw
;
2758 fxsave
->swd
= fpu
->fsw
;
2759 fxsave
->twd
= fpu
->ftwx
;
2760 fxsave
->fop
= fpu
->last_opcode
;
2761 fxsave
->rip
= fpu
->last_ip
;
2762 fxsave
->rdp
= fpu
->last_dp
;
2763 memcpy(fxsave
->xmm_space
, fpu
->xmm
, sizeof fxsave
->xmm_space
);
2770 static long kvm_vcpu_ioctl(struct file
*filp
,
2771 unsigned int ioctl
, unsigned long arg
)
2773 struct kvm_vcpu
*vcpu
= filp
->private_data
;
2774 void __user
*argp
= (void __user
*)arg
;
2782 r
= kvm_vcpu_ioctl_run(vcpu
, vcpu
->run
);
2784 case KVM_GET_REGS
: {
2785 struct kvm_regs kvm_regs
;
2787 memset(&kvm_regs
, 0, sizeof kvm_regs
);
2788 r
= kvm_vcpu_ioctl_get_regs(vcpu
, &kvm_regs
);
2792 if (copy_to_user(argp
, &kvm_regs
, sizeof kvm_regs
))
2797 case KVM_SET_REGS
: {
2798 struct kvm_regs kvm_regs
;
2801 if (copy_from_user(&kvm_regs
, argp
, sizeof kvm_regs
))
2803 r
= kvm_vcpu_ioctl_set_regs(vcpu
, &kvm_regs
);
2809 case KVM_GET_SREGS
: {
2810 struct kvm_sregs kvm_sregs
;
2812 memset(&kvm_sregs
, 0, sizeof kvm_sregs
);
2813 r
= kvm_vcpu_ioctl_get_sregs(vcpu
, &kvm_sregs
);
2817 if (copy_to_user(argp
, &kvm_sregs
, sizeof kvm_sregs
))
2822 case KVM_SET_SREGS
: {
2823 struct kvm_sregs kvm_sregs
;
2826 if (copy_from_user(&kvm_sregs
, argp
, sizeof kvm_sregs
))
2828 r
= kvm_vcpu_ioctl_set_sregs(vcpu
, &kvm_sregs
);
2834 case KVM_TRANSLATE
: {
2835 struct kvm_translation tr
;
2838 if (copy_from_user(&tr
, argp
, sizeof tr
))
2840 r
= kvm_vcpu_ioctl_translate(vcpu
, &tr
);
2844 if (copy_to_user(argp
, &tr
, sizeof tr
))
2849 case KVM_INTERRUPT
: {
2850 struct kvm_interrupt irq
;
2853 if (copy_from_user(&irq
, argp
, sizeof irq
))
2855 r
= kvm_vcpu_ioctl_interrupt(vcpu
, &irq
);
2861 case KVM_DEBUG_GUEST
: {
2862 struct kvm_debug_guest dbg
;
2865 if (copy_from_user(&dbg
, argp
, sizeof dbg
))
2867 r
= kvm_vcpu_ioctl_debug_guest(vcpu
, &dbg
);
2873 case KVM_SET_SIGNAL_MASK
: {
2874 struct kvm_signal_mask __user
*sigmask_arg
= argp
;
2875 struct kvm_signal_mask kvm_sigmask
;
2876 sigset_t sigset
, *p
;
2881 if (copy_from_user(&kvm_sigmask
, argp
,
2882 sizeof kvm_sigmask
))
2885 if (kvm_sigmask
.len
!= sizeof sigset
)
2888 if (copy_from_user(&sigset
, sigmask_arg
->sigset
,
2893 r
= kvm_vcpu_ioctl_set_sigmask(vcpu
, &sigset
);
2899 memset(&fpu
, 0, sizeof fpu
);
2900 r
= kvm_vcpu_ioctl_get_fpu(vcpu
, &fpu
);
2904 if (copy_to_user(argp
, &fpu
, sizeof fpu
))
2913 if (copy_from_user(&fpu
, argp
, sizeof fpu
))
2915 r
= kvm_vcpu_ioctl_set_fpu(vcpu
, &fpu
);
2922 r
= kvm_arch_vcpu_ioctl(filp
, ioctl
, arg
);
2928 static long kvm_vm_ioctl(struct file
*filp
,
2929 unsigned int ioctl
, unsigned long arg
)
2931 struct kvm
*kvm
= filp
->private_data
;
2932 void __user
*argp
= (void __user
*)arg
;
2936 case KVM_CREATE_VCPU
:
2937 r
= kvm_vm_ioctl_create_vcpu(kvm
, arg
);
2941 case KVM_SET_MEMORY_REGION
: {
2942 struct kvm_memory_region kvm_mem
;
2943 struct kvm_userspace_memory_region kvm_userspace_mem
;
2946 if (copy_from_user(&kvm_mem
, argp
, sizeof kvm_mem
))
2948 kvm_userspace_mem
.slot
= kvm_mem
.slot
;
2949 kvm_userspace_mem
.flags
= kvm_mem
.flags
;
2950 kvm_userspace_mem
.guest_phys_addr
= kvm_mem
.guest_phys_addr
;
2951 kvm_userspace_mem
.memory_size
= kvm_mem
.memory_size
;
2952 r
= kvm_vm_ioctl_set_memory_region(kvm
, &kvm_userspace_mem
, 0);
2957 case KVM_SET_USER_MEMORY_REGION
: {
2958 struct kvm_userspace_memory_region kvm_userspace_mem
;
2961 if (copy_from_user(&kvm_userspace_mem
, argp
,
2962 sizeof kvm_userspace_mem
))
2965 r
= kvm_vm_ioctl_set_memory_region(kvm
, &kvm_userspace_mem
, 1);
2970 case KVM_SET_NR_MMU_PAGES
:
2971 r
= kvm_vm_ioctl_set_nr_mmu_pages(kvm
, arg
);
2975 case KVM_GET_NR_MMU_PAGES
:
2976 r
= kvm_vm_ioctl_get_nr_mmu_pages(kvm
);
2978 case KVM_GET_DIRTY_LOG
: {
2979 struct kvm_dirty_log log
;
2982 if (copy_from_user(&log
, argp
, sizeof log
))
2984 r
= kvm_vm_ioctl_get_dirty_log(kvm
, &log
);
2989 case KVM_SET_MEMORY_ALIAS
: {
2990 struct kvm_memory_alias alias
;
2993 if (copy_from_user(&alias
, argp
, sizeof alias
))
2995 r
= kvm_vm_ioctl_set_memory_alias(kvm
, &alias
);
3000 case KVM_CREATE_IRQCHIP
:
3002 kvm
->vpic
= kvm_create_pic(kvm
);
3004 r
= kvm_ioapic_init(kvm
);
3013 case KVM_IRQ_LINE
: {
3014 struct kvm_irq_level irq_event
;
3017 if (copy_from_user(&irq_event
, argp
, sizeof irq_event
))
3019 if (irqchip_in_kernel(kvm
)) {
3020 mutex_lock(&kvm
->lock
);
3021 if (irq_event
.irq
< 16)
3022 kvm_pic_set_irq(pic_irqchip(kvm
),
3025 kvm_ioapic_set_irq(kvm
->vioapic
,
3028 mutex_unlock(&kvm
->lock
);
3033 case KVM_GET_IRQCHIP
: {
3034 /* 0: PIC master, 1: PIC slave, 2: IOAPIC */
3035 struct kvm_irqchip chip
;
3038 if (copy_from_user(&chip
, argp
, sizeof chip
))
3041 if (!irqchip_in_kernel(kvm
))
3043 r
= kvm_vm_ioctl_get_irqchip(kvm
, &chip
);
3047 if (copy_to_user(argp
, &chip
, sizeof chip
))
3052 case KVM_SET_IRQCHIP
: {
3053 /* 0: PIC master, 1: PIC slave, 2: IOAPIC */
3054 struct kvm_irqchip chip
;
3057 if (copy_from_user(&chip
, argp
, sizeof chip
))
3060 if (!irqchip_in_kernel(kvm
))
3062 r
= kvm_vm_ioctl_set_irqchip(kvm
, &chip
);
3075 static struct page
*kvm_vm_nopage(struct vm_area_struct
*vma
,
3076 unsigned long address
,
3079 struct kvm
*kvm
= vma
->vm_file
->private_data
;
3080 unsigned long pgoff
;
3083 pgoff
= ((address
- vma
->vm_start
) >> PAGE_SHIFT
) + vma
->vm_pgoff
;
3084 page
= gfn_to_page(kvm
, pgoff
);
3085 if (is_error_page(page
)) {
3086 kvm_release_page(page
);
3087 return NOPAGE_SIGBUS
;
3090 *type
= VM_FAULT_MINOR
;
3095 static struct vm_operations_struct kvm_vm_vm_ops
= {
3096 .nopage
= kvm_vm_nopage
,
3099 static int kvm_vm_mmap(struct file
*file
, struct vm_area_struct
*vma
)
3101 vma
->vm_ops
= &kvm_vm_vm_ops
;
3105 static struct file_operations kvm_vm_fops
= {
3106 .release
= kvm_vm_release
,
3107 .unlocked_ioctl
= kvm_vm_ioctl
,
3108 .compat_ioctl
= kvm_vm_ioctl
,
3109 .mmap
= kvm_vm_mmap
,
3112 static int kvm_dev_ioctl_create_vm(void)
3115 struct inode
*inode
;
3119 kvm
= kvm_create_vm();
3121 return PTR_ERR(kvm
);
3122 r
= anon_inode_getfd(&fd
, &inode
, &file
, "kvm-vm", &kvm_vm_fops
, kvm
);
3124 kvm_destroy_vm(kvm
);
3133 static long kvm_dev_ioctl(struct file
*filp
,
3134 unsigned int ioctl
, unsigned long arg
)
3136 void __user
*argp
= (void __user
*)arg
;
3140 case KVM_GET_API_VERSION
:
3144 r
= KVM_API_VERSION
;
3150 r
= kvm_dev_ioctl_create_vm();
3152 case KVM_CHECK_EXTENSION
: {
3153 int ext
= (long)argp
;
3156 case KVM_CAP_IRQCHIP
:
3158 case KVM_CAP_MMU_SHADOW_CACHE_CONTROL
:
3159 case KVM_CAP_USER_MEMORY
:
3168 case KVM_GET_VCPU_MMAP_SIZE
:
3175 return kvm_arch_dev_ioctl(filp
, ioctl
, arg
);
3181 static struct file_operations kvm_chardev_ops
= {
3182 .unlocked_ioctl
= kvm_dev_ioctl
,
3183 .compat_ioctl
= kvm_dev_ioctl
,
3186 static struct miscdevice kvm_dev
= {
3193 * Make sure that a cpu that is being hot-unplugged does not have any vcpus
3196 static void decache_vcpus_on_cpu(int cpu
)
3199 struct kvm_vcpu
*vcpu
;
3202 spin_lock(&kvm_lock
);
3203 list_for_each_entry(vm
, &vm_list
, vm_list
)
3204 for (i
= 0; i
< KVM_MAX_VCPUS
; ++i
) {
3205 vcpu
= vm
->vcpus
[i
];
3209 * If the vcpu is locked, then it is running on some
3210 * other cpu and therefore it is not cached on the
3213 * If it's not locked, check the last cpu it executed
3216 if (mutex_trylock(&vcpu
->mutex
)) {
3217 if (vcpu
->cpu
== cpu
) {
3218 kvm_x86_ops
->vcpu_decache(vcpu
);
3221 mutex_unlock(&vcpu
->mutex
);
3224 spin_unlock(&kvm_lock
);
3227 static void hardware_enable(void *junk
)
3229 int cpu
= raw_smp_processor_id();
3231 if (cpu_isset(cpu
, cpus_hardware_enabled
))
3233 cpu_set(cpu
, cpus_hardware_enabled
);
3234 kvm_x86_ops
->hardware_enable(NULL
);
3237 static void hardware_disable(void *junk
)
3239 int cpu
= raw_smp_processor_id();
3241 if (!cpu_isset(cpu
, cpus_hardware_enabled
))
3243 cpu_clear(cpu
, cpus_hardware_enabled
);
3244 decache_vcpus_on_cpu(cpu
);
3245 kvm_x86_ops
->hardware_disable(NULL
);
3248 static int kvm_cpu_hotplug(struct notifier_block
*notifier
, unsigned long val
,
3255 case CPU_DYING_FROZEN
:
3256 printk(KERN_INFO
"kvm: disabling virtualization on CPU%d\n",
3258 hardware_disable(NULL
);
3260 case CPU_UP_CANCELED
:
3261 case CPU_UP_CANCELED_FROZEN
:
3262 printk(KERN_INFO
"kvm: disabling virtualization on CPU%d\n",
3264 smp_call_function_single(cpu
, hardware_disable
, NULL
, 0, 1);
3267 case CPU_ONLINE_FROZEN
:
3268 printk(KERN_INFO
"kvm: enabling virtualization on CPU%d\n",
3270 smp_call_function_single(cpu
, hardware_enable
, NULL
, 0, 1);
3276 static int kvm_reboot(struct notifier_block
*notifier
, unsigned long val
,
3279 if (val
== SYS_RESTART
) {
3281 * Some (well, at least mine) BIOSes hang on reboot if
3284 printk(KERN_INFO
"kvm: exiting hardware virtualization\n");
3285 on_each_cpu(hardware_disable
, NULL
, 0, 1);
3290 static struct notifier_block kvm_reboot_notifier
= {
3291 .notifier_call
= kvm_reboot
,
3295 void kvm_io_bus_init(struct kvm_io_bus
*bus
)
3297 memset(bus
, 0, sizeof(*bus
));
3300 void kvm_io_bus_destroy(struct kvm_io_bus
*bus
)
3304 for (i
= 0; i
< bus
->dev_count
; i
++) {
3305 struct kvm_io_device
*pos
= bus
->devs
[i
];
3307 kvm_iodevice_destructor(pos
);
3311 struct kvm_io_device
*kvm_io_bus_find_dev(struct kvm_io_bus
*bus
, gpa_t addr
)
3315 for (i
= 0; i
< bus
->dev_count
; i
++) {
3316 struct kvm_io_device
*pos
= bus
->devs
[i
];
3318 if (pos
->in_range(pos
, addr
))
3325 void kvm_io_bus_register_dev(struct kvm_io_bus
*bus
, struct kvm_io_device
*dev
)
3327 BUG_ON(bus
->dev_count
> (NR_IOBUS_DEVS
-1));
3329 bus
->devs
[bus
->dev_count
++] = dev
;
3332 static struct notifier_block kvm_cpu_notifier
= {
3333 .notifier_call
= kvm_cpu_hotplug
,
3334 .priority
= 20, /* must be > scheduler priority */
3337 static u64
stat_get(void *_offset
)
3339 unsigned offset
= (long)_offset
;
3342 struct kvm_vcpu
*vcpu
;
3345 spin_lock(&kvm_lock
);
3346 list_for_each_entry(kvm
, &vm_list
, vm_list
)
3347 for (i
= 0; i
< KVM_MAX_VCPUS
; ++i
) {
3348 vcpu
= kvm
->vcpus
[i
];
3350 total
+= *(u32
*)((void *)vcpu
+ offset
);
3352 spin_unlock(&kvm_lock
);
3356 DEFINE_SIMPLE_ATTRIBUTE(stat_fops
, stat_get
, NULL
, "%llu\n");
3358 static __init
void kvm_init_debug(void)
3360 struct kvm_stats_debugfs_item
*p
;
3362 debugfs_dir
= debugfs_create_dir("kvm", NULL
);
3363 for (p
= debugfs_entries
; p
->name
; ++p
)
3364 p
->dentry
= debugfs_create_file(p
->name
, 0444, debugfs_dir
,
3365 (void *)(long)p
->offset
,
3369 static void kvm_exit_debug(void)
3371 struct kvm_stats_debugfs_item
*p
;
3373 for (p
= debugfs_entries
; p
->name
; ++p
)
3374 debugfs_remove(p
->dentry
);
3375 debugfs_remove(debugfs_dir
);
3378 static int kvm_suspend(struct sys_device
*dev
, pm_message_t state
)
3380 hardware_disable(NULL
);
3384 static int kvm_resume(struct sys_device
*dev
)
3386 hardware_enable(NULL
);
3390 static struct sysdev_class kvm_sysdev_class
= {
3392 .suspend
= kvm_suspend
,
3393 .resume
= kvm_resume
,
3396 static struct sys_device kvm_sysdev
= {
3398 .cls
= &kvm_sysdev_class
,
3401 struct page
*bad_page
;
3404 struct kvm_vcpu
*preempt_notifier_to_vcpu(struct preempt_notifier
*pn
)
3406 return container_of(pn
, struct kvm_vcpu
, preempt_notifier
);
3409 static void kvm_sched_in(struct preempt_notifier
*pn
, int cpu
)
3411 struct kvm_vcpu
*vcpu
= preempt_notifier_to_vcpu(pn
);
3413 kvm_x86_ops
->vcpu_load(vcpu
, cpu
);
3416 static void kvm_sched_out(struct preempt_notifier
*pn
,
3417 struct task_struct
*next
)
3419 struct kvm_vcpu
*vcpu
= preempt_notifier_to_vcpu(pn
);
3421 kvm_x86_ops
->vcpu_put(vcpu
);
3424 int kvm_init_x86(struct kvm_x86_ops
*ops
, unsigned int vcpu_size
,
3425 struct module
*module
)
3431 printk(KERN_ERR
"kvm: already loaded the other module\n");
3435 if (!ops
->cpu_has_kvm_support()) {
3436 printk(KERN_ERR
"kvm: no hardware support\n");
3439 if (ops
->disabled_by_bios()) {
3440 printk(KERN_ERR
"kvm: disabled by bios\n");
3446 r
= kvm_x86_ops
->hardware_setup();
3450 for_each_online_cpu(cpu
) {
3451 smp_call_function_single(cpu
,
3452 kvm_x86_ops
->check_processor_compatibility
,
3458 on_each_cpu(hardware_enable
, NULL
, 0, 1);
3459 r
= register_cpu_notifier(&kvm_cpu_notifier
);
3462 register_reboot_notifier(&kvm_reboot_notifier
);
3464 r
= sysdev_class_register(&kvm_sysdev_class
);
3468 r
= sysdev_register(&kvm_sysdev
);
3472 /* A kmem cache lets us meet the alignment requirements of fx_save. */
3473 kvm_vcpu_cache
= kmem_cache_create("kvm_vcpu", vcpu_size
,
3474 __alignof__(struct kvm_vcpu
), 0, 0);
3475 if (!kvm_vcpu_cache
) {
3480 kvm_chardev_ops
.owner
= module
;
3482 r
= misc_register(&kvm_dev
);
3484 printk(KERN_ERR
"kvm: misc device register failed\n");
3488 kvm_preempt_ops
.sched_in
= kvm_sched_in
;
3489 kvm_preempt_ops
.sched_out
= kvm_sched_out
;
3491 kvm_mmu_set_nonpresent_ptes(0ull, 0ull);
3496 kmem_cache_destroy(kvm_vcpu_cache
);
3498 sysdev_unregister(&kvm_sysdev
);
3500 sysdev_class_unregister(&kvm_sysdev_class
);
3502 unregister_reboot_notifier(&kvm_reboot_notifier
);
3503 unregister_cpu_notifier(&kvm_cpu_notifier
);
3505 on_each_cpu(hardware_disable
, NULL
, 0, 1);
3507 kvm_x86_ops
->hardware_unsetup();
3512 EXPORT_SYMBOL_GPL(kvm_init_x86
);
3514 void kvm_exit_x86(void)
3516 misc_deregister(&kvm_dev
);
3517 kmem_cache_destroy(kvm_vcpu_cache
);
3518 sysdev_unregister(&kvm_sysdev
);
3519 sysdev_class_unregister(&kvm_sysdev_class
);
3520 unregister_reboot_notifier(&kvm_reboot_notifier
);
3521 unregister_cpu_notifier(&kvm_cpu_notifier
);
3522 on_each_cpu(hardware_disable
, NULL
, 0, 1);
3523 kvm_x86_ops
->hardware_unsetup();
3526 EXPORT_SYMBOL_GPL(kvm_exit_x86
);
3528 static __init
int kvm_init(void)
3532 r
= kvm_mmu_module_init();
3540 bad_page
= alloc_page(GFP_KERNEL
| __GFP_ZERO
);
3542 if (bad_page
== NULL
) {
3551 kvm_mmu_module_exit();
3556 static __exit
void kvm_exit(void)
3559 __free_page(bad_page
);
3560 kvm_mmu_module_exit();
3563 module_init(kvm_init
)
3564 module_exit(kvm_exit
)