3 * Local APIC virtualization
5 * Copyright (C) 2006 Qumranet, Inc.
6 * Copyright (C) 2007 Novell
7 * Copyright (C) 2007 Intel
8 * Copyright 2009 Red Hat, Inc. and/or its affiliates.
11 * Dor Laor <dor.laor@qumranet.com>
12 * Gregory Haskins <ghaskins@novell.com>
13 * Yaozu (Eddie) Dong <eddie.dong@intel.com>
15 * Based on Xen 3.1 code, Copyright (c) 2004, Intel Corporation.
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
21 #include <linux/kvm_host.h>
22 #include <linux/kvm.h>
24 #include <linux/highmem.h>
25 #include <linux/smp.h>
26 #include <linux/hrtimer.h>
28 #include <linux/module.h>
29 #include <linux/math64.h>
30 #include <linux/slab.h>
31 #include <asm/processor.h>
34 #include <asm/current.h>
35 #include <asm/apicdef.h>
36 #include <linux/atomic.h>
37 #include "kvm_cache_regs.h"
43 #define mod_64(x, y) ((x) - (y) * div64_u64(x, y))
45 #define mod_64(x, y) ((x) % (y))
53 #define APIC_BUS_CYCLE_NS 1
55 /* #define apic_debug(fmt,arg...) printk(KERN_WARNING fmt,##arg) */
56 #define apic_debug(fmt, arg...)
58 #define APIC_LVT_NUM 6
59 /* 14 is the version for Xeon and Pentium 8.4.8*/
60 #define APIC_VERSION (0x14UL | ((APIC_LVT_NUM - 1) << 16))
61 #define LAPIC_MMIO_LENGTH (1 << 12)
62 /* followed define is not in apicdef.h */
63 #define APIC_SHORT_MASK 0xc0000
64 #define APIC_DEST_NOSHORT 0x0
65 #define APIC_DEST_MASK 0x800
66 #define MAX_APIC_VECTOR 256
68 #define VEC_POS(v) ((v) & (32 - 1))
69 #define REG_POS(v) (((v) >> 5) << 4)
71 static unsigned int min_timer_period_us
= 500;
72 module_param(min_timer_period_us
, uint
, S_IRUGO
| S_IWUSR
);
74 static inline u32
apic_get_reg(struct kvm_lapic
*apic
, int reg_off
)
76 return *((u32
*) (apic
->regs
+ reg_off
));
79 static inline void apic_set_reg(struct kvm_lapic
*apic
, int reg_off
, u32 val
)
81 *((u32
*) (apic
->regs
+ reg_off
)) = val
;
84 static inline int apic_test_and_set_vector(int vec
, void *bitmap
)
86 return test_and_set_bit(VEC_POS(vec
), (bitmap
) + REG_POS(vec
));
89 static inline int apic_test_and_clear_vector(int vec
, void *bitmap
)
91 return test_and_clear_bit(VEC_POS(vec
), (bitmap
) + REG_POS(vec
));
94 static inline void apic_set_vector(int vec
, void *bitmap
)
96 set_bit(VEC_POS(vec
), (bitmap
) + REG_POS(vec
));
99 static inline void apic_clear_vector(int vec
, void *bitmap
)
101 clear_bit(VEC_POS(vec
), (bitmap
) + REG_POS(vec
));
104 static inline int apic_hw_enabled(struct kvm_lapic
*apic
)
106 return (apic
)->vcpu
->arch
.apic_base
& MSR_IA32_APICBASE_ENABLE
;
109 static inline int apic_sw_enabled(struct kvm_lapic
*apic
)
111 return apic_get_reg(apic
, APIC_SPIV
) & APIC_SPIV_APIC_ENABLED
;
114 static inline int apic_enabled(struct kvm_lapic
*apic
)
116 return apic_sw_enabled(apic
) && apic_hw_enabled(apic
);
120 (APIC_LVT_MASKED | APIC_SEND_PENDING | APIC_VECTOR_MASK)
123 (LVT_MASK | APIC_MODE_MASK | APIC_INPUT_POLARITY | \
124 APIC_LVT_REMOTE_IRR | APIC_LVT_LEVEL_TRIGGER)
126 static inline int kvm_apic_id(struct kvm_lapic
*apic
)
128 return (apic_get_reg(apic
, APIC_ID
) >> 24) & 0xff;
131 static inline int apic_lvt_enabled(struct kvm_lapic
*apic
, int lvt_type
)
133 return !(apic_get_reg(apic
, lvt_type
) & APIC_LVT_MASKED
);
136 static inline int apic_lvt_vector(struct kvm_lapic
*apic
, int lvt_type
)
138 return apic_get_reg(apic
, lvt_type
) & APIC_VECTOR_MASK
;
141 static inline int apic_lvtt_period(struct kvm_lapic
*apic
)
143 return apic_get_reg(apic
, APIC_LVTT
) & APIC_LVT_TIMER_PERIODIC
;
146 static inline int apic_lvt_nmi_mode(u32 lvt_val
)
148 return (lvt_val
& (APIC_MODE_MASK
| APIC_LVT_MASKED
)) == APIC_DM_NMI
;
151 void kvm_apic_set_version(struct kvm_vcpu
*vcpu
)
153 struct kvm_lapic
*apic
= vcpu
->arch
.apic
;
154 struct kvm_cpuid_entry2
*feat
;
155 u32 v
= APIC_VERSION
;
157 if (!irqchip_in_kernel(vcpu
->kvm
))
160 feat
= kvm_find_cpuid_entry(apic
->vcpu
, 0x1, 0);
161 if (feat
&& (feat
->ecx
& (1 << (X86_FEATURE_X2APIC
& 31))))
162 v
|= APIC_LVR_DIRECTED_EOI
;
163 apic_set_reg(apic
, APIC_LVR
, v
);
166 static inline int apic_x2apic_mode(struct kvm_lapic
*apic
)
168 return apic
->vcpu
->arch
.apic_base
& X2APIC_ENABLE
;
171 static unsigned int apic_lvt_mask
[APIC_LVT_NUM
] = {
172 LVT_MASK
| APIC_LVT_TIMER_PERIODIC
, /* LVTT */
173 LVT_MASK
| APIC_MODE_MASK
, /* LVTTHMR */
174 LVT_MASK
| APIC_MODE_MASK
, /* LVTPC */
175 LINT_MASK
, LINT_MASK
, /* LVT0-1 */
176 LVT_MASK
/* LVTERR */
179 static int find_highest_vector(void *bitmap
)
182 int word_offset
= MAX_APIC_VECTOR
>> 5;
184 while ((word_offset
!= 0) && (word
[(--word_offset
) << 2] == 0))
187 if (likely(!word_offset
&& !word
[0]))
190 return fls(word
[word_offset
<< 2]) - 1 + (word_offset
<< 5);
193 static inline int apic_test_and_set_irr(int vec
, struct kvm_lapic
*apic
)
195 apic
->irr_pending
= true;
196 return apic_test_and_set_vector(vec
, apic
->regs
+ APIC_IRR
);
199 static inline int apic_search_irr(struct kvm_lapic
*apic
)
201 return find_highest_vector(apic
->regs
+ APIC_IRR
);
204 static inline int apic_find_highest_irr(struct kvm_lapic
*apic
)
208 if (!apic
->irr_pending
)
211 result
= apic_search_irr(apic
);
212 ASSERT(result
== -1 || result
>= 16);
217 static inline void apic_clear_irr(int vec
, struct kvm_lapic
*apic
)
219 apic
->irr_pending
= false;
220 apic_clear_vector(vec
, apic
->regs
+ APIC_IRR
);
221 if (apic_search_irr(apic
) != -1)
222 apic
->irr_pending
= true;
225 int kvm_lapic_find_highest_irr(struct kvm_vcpu
*vcpu
)
227 struct kvm_lapic
*apic
= vcpu
->arch
.apic
;
230 /* This may race with setting of irr in __apic_accept_irq() and
231 * value returned may be wrong, but kvm_vcpu_kick() in __apic_accept_irq
232 * will cause vmexit immediately and the value will be recalculated
233 * on the next vmentry.
237 highest_irr
= apic_find_highest_irr(apic
);
242 static int __apic_accept_irq(struct kvm_lapic
*apic
, int delivery_mode
,
243 int vector
, int level
, int trig_mode
);
245 int kvm_apic_set_irq(struct kvm_vcpu
*vcpu
, struct kvm_lapic_irq
*irq
)
247 struct kvm_lapic
*apic
= vcpu
->arch
.apic
;
249 return __apic_accept_irq(apic
, irq
->delivery_mode
, irq
->vector
,
250 irq
->level
, irq
->trig_mode
);
253 static inline int apic_find_highest_isr(struct kvm_lapic
*apic
)
257 result
= find_highest_vector(apic
->regs
+ APIC_ISR
);
258 ASSERT(result
== -1 || result
>= 16);
263 static void apic_update_ppr(struct kvm_lapic
*apic
)
265 u32 tpr
, isrv
, ppr
, old_ppr
;
268 old_ppr
= apic_get_reg(apic
, APIC_PROCPRI
);
269 tpr
= apic_get_reg(apic
, APIC_TASKPRI
);
270 isr
= apic_find_highest_isr(apic
);
271 isrv
= (isr
!= -1) ? isr
: 0;
273 if ((tpr
& 0xf0) >= (isrv
& 0xf0))
278 apic_debug("vlapic %p, ppr 0x%x, isr 0x%x, isrv 0x%x",
279 apic
, ppr
, isr
, isrv
);
281 if (old_ppr
!= ppr
) {
282 apic_set_reg(apic
, APIC_PROCPRI
, ppr
);
284 kvm_make_request(KVM_REQ_EVENT
, apic
->vcpu
);
288 static void apic_set_tpr(struct kvm_lapic
*apic
, u32 tpr
)
290 apic_set_reg(apic
, APIC_TASKPRI
, tpr
);
291 apic_update_ppr(apic
);
294 int kvm_apic_match_physical_addr(struct kvm_lapic
*apic
, u16 dest
)
296 return dest
== 0xff || kvm_apic_id(apic
) == dest
;
299 int kvm_apic_match_logical_addr(struct kvm_lapic
*apic
, u8 mda
)
304 if (apic_x2apic_mode(apic
)) {
305 logical_id
= apic_get_reg(apic
, APIC_LDR
);
306 return logical_id
& mda
;
309 logical_id
= GET_APIC_LOGICAL_ID(apic_get_reg(apic
, APIC_LDR
));
311 switch (apic_get_reg(apic
, APIC_DFR
)) {
313 if (logical_id
& mda
)
316 case APIC_DFR_CLUSTER
:
317 if (((logical_id
>> 4) == (mda
>> 0x4))
318 && (logical_id
& mda
& 0xf))
322 apic_debug("Bad DFR vcpu %d: %08x\n",
323 apic
->vcpu
->vcpu_id
, apic_get_reg(apic
, APIC_DFR
));
330 int kvm_apic_match_dest(struct kvm_vcpu
*vcpu
, struct kvm_lapic
*source
,
331 int short_hand
, int dest
, int dest_mode
)
334 struct kvm_lapic
*target
= vcpu
->arch
.apic
;
336 apic_debug("target %p, source %p, dest 0x%x, "
337 "dest_mode 0x%x, short_hand 0x%x\n",
338 target
, source
, dest
, dest_mode
, short_hand
);
341 switch (short_hand
) {
342 case APIC_DEST_NOSHORT
:
345 result
= kvm_apic_match_physical_addr(target
, dest
);
348 result
= kvm_apic_match_logical_addr(target
, dest
);
351 result
= (target
== source
);
353 case APIC_DEST_ALLINC
:
356 case APIC_DEST_ALLBUT
:
357 result
= (target
!= source
);
360 apic_debug("kvm: apic: Bad dest shorthand value %x\n",
369 * Add a pending IRQ into lapic.
370 * Return 1 if successfully added and 0 if discarded.
372 static int __apic_accept_irq(struct kvm_lapic
*apic
, int delivery_mode
,
373 int vector
, int level
, int trig_mode
)
376 struct kvm_vcpu
*vcpu
= apic
->vcpu
;
378 switch (delivery_mode
) {
380 vcpu
->arch
.apic_arb_prio
++;
382 /* FIXME add logic for vcpu on reset */
383 if (unlikely(!apic_enabled(apic
)))
387 apic_debug("level trig mode for vector %d", vector
);
388 apic_set_vector(vector
, apic
->regs
+ APIC_TMR
);
390 apic_clear_vector(vector
, apic
->regs
+ APIC_TMR
);
392 result
= !apic_test_and_set_irr(vector
, apic
);
393 trace_kvm_apic_accept_irq(vcpu
->vcpu_id
, delivery_mode
,
394 trig_mode
, vector
, !result
);
397 apic_debug("level trig mode repeatedly for "
398 "vector %d", vector
);
402 kvm_make_request(KVM_REQ_EVENT
, vcpu
);
407 apic_debug("Ignoring delivery mode 3\n");
411 apic_debug("Ignoring guest SMI\n");
416 kvm_inject_nmi(vcpu
);
423 vcpu
->arch
.mp_state
= KVM_MP_STATE_INIT_RECEIVED
;
424 kvm_make_request(KVM_REQ_EVENT
, vcpu
);
427 apic_debug("Ignoring de-assert INIT to vcpu %d\n",
432 case APIC_DM_STARTUP
:
433 apic_debug("SIPI to vcpu %d vector 0x%02x\n",
434 vcpu
->vcpu_id
, vector
);
435 if (vcpu
->arch
.mp_state
== KVM_MP_STATE_INIT_RECEIVED
) {
437 vcpu
->arch
.sipi_vector
= vector
;
438 vcpu
->arch
.mp_state
= KVM_MP_STATE_SIPI_RECEIVED
;
439 kvm_make_request(KVM_REQ_EVENT
, vcpu
);
446 * Should only be called by kvm_apic_local_deliver() with LVT0,
447 * before NMI watchdog was enabled. Already handled by
448 * kvm_apic_accept_pic_intr().
453 printk(KERN_ERR
"TODO: unsupported delivery mode %x\n",
460 int kvm_apic_compare_prio(struct kvm_vcpu
*vcpu1
, struct kvm_vcpu
*vcpu2
)
462 return vcpu1
->arch
.apic_arb_prio
- vcpu2
->arch
.apic_arb_prio
;
465 static void apic_set_eoi(struct kvm_lapic
*apic
)
467 int vector
= apic_find_highest_isr(apic
);
470 * Not every write EOI will has corresponding ISR,
471 * one example is when Kernel check timer on setup_IO_APIC
476 apic_clear_vector(vector
, apic
->regs
+ APIC_ISR
);
477 apic_update_ppr(apic
);
479 if (apic_test_and_clear_vector(vector
, apic
->regs
+ APIC_TMR
))
480 trigger_mode
= IOAPIC_LEVEL_TRIG
;
482 trigger_mode
= IOAPIC_EDGE_TRIG
;
483 if (!(apic_get_reg(apic
, APIC_SPIV
) & APIC_SPIV_DIRECTED_EOI
))
484 kvm_ioapic_update_eoi(apic
->vcpu
->kvm
, vector
, trigger_mode
);
485 kvm_make_request(KVM_REQ_EVENT
, apic
->vcpu
);
488 static void apic_send_ipi(struct kvm_lapic
*apic
)
490 u32 icr_low
= apic_get_reg(apic
, APIC_ICR
);
491 u32 icr_high
= apic_get_reg(apic
, APIC_ICR2
);
492 struct kvm_lapic_irq irq
;
494 irq
.vector
= icr_low
& APIC_VECTOR_MASK
;
495 irq
.delivery_mode
= icr_low
& APIC_MODE_MASK
;
496 irq
.dest_mode
= icr_low
& APIC_DEST_MASK
;
497 irq
.level
= icr_low
& APIC_INT_ASSERT
;
498 irq
.trig_mode
= icr_low
& APIC_INT_LEVELTRIG
;
499 irq
.shorthand
= icr_low
& APIC_SHORT_MASK
;
500 if (apic_x2apic_mode(apic
))
501 irq
.dest_id
= icr_high
;
503 irq
.dest_id
= GET_APIC_DEST_FIELD(icr_high
);
505 trace_kvm_apic_ipi(icr_low
, irq
.dest_id
);
507 apic_debug("icr_high 0x%x, icr_low 0x%x, "
508 "short_hand 0x%x, dest 0x%x, trig_mode 0x%x, level 0x%x, "
509 "dest_mode 0x%x, delivery_mode 0x%x, vector 0x%x\n",
510 icr_high
, icr_low
, irq
.shorthand
, irq
.dest_id
,
511 irq
.trig_mode
, irq
.level
, irq
.dest_mode
, irq
.delivery_mode
,
514 kvm_irq_delivery_to_apic(apic
->vcpu
->kvm
, apic
, &irq
);
517 static u32
apic_get_tmcct(struct kvm_lapic
*apic
)
523 ASSERT(apic
!= NULL
);
525 /* if initial count is 0, current count should also be 0 */
526 if (apic_get_reg(apic
, APIC_TMICT
) == 0)
529 remaining
= hrtimer_get_remaining(&apic
->lapic_timer
.timer
);
530 if (ktime_to_ns(remaining
) < 0)
531 remaining
= ktime_set(0, 0);
533 ns
= mod_64(ktime_to_ns(remaining
), apic
->lapic_timer
.period
);
534 tmcct
= div64_u64(ns
,
535 (APIC_BUS_CYCLE_NS
* apic
->divide_count
));
540 static void __report_tpr_access(struct kvm_lapic
*apic
, bool write
)
542 struct kvm_vcpu
*vcpu
= apic
->vcpu
;
543 struct kvm_run
*run
= vcpu
->run
;
545 kvm_make_request(KVM_REQ_REPORT_TPR_ACCESS
, vcpu
);
546 run
->tpr_access
.rip
= kvm_rip_read(vcpu
);
547 run
->tpr_access
.is_write
= write
;
550 static inline void report_tpr_access(struct kvm_lapic
*apic
, bool write
)
552 if (apic
->vcpu
->arch
.tpr_access_reporting
)
553 __report_tpr_access(apic
, write
);
556 static u32
__apic_read(struct kvm_lapic
*apic
, unsigned int offset
)
560 if (offset
>= LAPIC_MMIO_LENGTH
)
565 if (apic_x2apic_mode(apic
))
566 val
= kvm_apic_id(apic
);
568 val
= kvm_apic_id(apic
) << 24;
571 apic_debug("Access APIC ARBPRI register which is for P6\n");
574 case APIC_TMCCT
: /* Timer CCR */
575 val
= apic_get_tmcct(apic
);
579 report_tpr_access(apic
, false);
582 apic_update_ppr(apic
);
583 val
= apic_get_reg(apic
, offset
);
590 static inline struct kvm_lapic
*to_lapic(struct kvm_io_device
*dev
)
592 return container_of(dev
, struct kvm_lapic
, dev
);
595 static int apic_reg_read(struct kvm_lapic
*apic
, u32 offset
, int len
,
598 unsigned char alignment
= offset
& 0xf;
600 /* this bitmask has a bit cleared for each reserver register */
601 static const u64 rmask
= 0x43ff01ffffffe70cULL
;
603 if ((alignment
+ len
) > 4) {
604 apic_debug("KVM_APIC_READ: alignment error %x %d\n",
609 if (offset
> 0x3f0 || !(rmask
& (1ULL << (offset
>> 4)))) {
610 apic_debug("KVM_APIC_READ: read reserved register %x\n",
615 result
= __apic_read(apic
, offset
& ~0xf);
617 trace_kvm_apic_read(offset
, result
);
623 memcpy(data
, (char *)&result
+ alignment
, len
);
626 printk(KERN_ERR
"Local APIC read with len = %x, "
627 "should be 1,2, or 4 instead\n", len
);
633 static int apic_mmio_in_range(struct kvm_lapic
*apic
, gpa_t addr
)
635 return apic_hw_enabled(apic
) &&
636 addr
>= apic
->base_address
&&
637 addr
< apic
->base_address
+ LAPIC_MMIO_LENGTH
;
640 static int apic_mmio_read(struct kvm_io_device
*this,
641 gpa_t address
, int len
, void *data
)
643 struct kvm_lapic
*apic
= to_lapic(this);
644 u32 offset
= address
- apic
->base_address
;
646 if (!apic_mmio_in_range(apic
, address
))
649 apic_reg_read(apic
, offset
, len
, data
);
654 static void update_divide_count(struct kvm_lapic
*apic
)
656 u32 tmp1
, tmp2
, tdcr
;
658 tdcr
= apic_get_reg(apic
, APIC_TDCR
);
660 tmp2
= ((tmp1
& 0x3) | ((tmp1
& 0x8) >> 1)) + 1;
661 apic
->divide_count
= 0x1 << (tmp2
& 0x7);
663 apic_debug("timer divide count is 0x%x\n",
667 static void start_apic_timer(struct kvm_lapic
*apic
)
669 ktime_t now
= apic
->lapic_timer
.timer
.base
->get_time();
671 apic
->lapic_timer
.period
= (u64
)apic_get_reg(apic
, APIC_TMICT
) *
672 APIC_BUS_CYCLE_NS
* apic
->divide_count
;
673 atomic_set(&apic
->lapic_timer
.pending
, 0);
675 if (!apic
->lapic_timer
.period
)
678 * Do not allow the guest to program periodic timers with small
679 * interval, since the hrtimers are not throttled by the host
682 if (apic_lvtt_period(apic
)) {
683 s64 min_period
= min_timer_period_us
* 1000LL;
685 if (apic
->lapic_timer
.period
< min_period
) {
687 "kvm: vcpu %i: requested %lld ns "
688 "lapic timer period limited to %lld ns\n",
689 apic
->vcpu
->vcpu_id
, apic
->lapic_timer
.period
,
691 apic
->lapic_timer
.period
= min_period
;
695 hrtimer_start(&apic
->lapic_timer
.timer
,
696 ktime_add_ns(now
, apic
->lapic_timer
.period
),
699 apic_debug("%s: bus cycle is %" PRId64
"ns, now 0x%016"
701 "timer initial count 0x%x, period %lldns, "
702 "expire @ 0x%016" PRIx64
".\n", __func__
,
703 APIC_BUS_CYCLE_NS
, ktime_to_ns(now
),
704 apic_get_reg(apic
, APIC_TMICT
),
705 apic
->lapic_timer
.period
,
706 ktime_to_ns(ktime_add_ns(now
,
707 apic
->lapic_timer
.period
)));
710 static void apic_manage_nmi_watchdog(struct kvm_lapic
*apic
, u32 lvt0_val
)
712 int nmi_wd_enabled
= apic_lvt_nmi_mode(apic_get_reg(apic
, APIC_LVT0
));
714 if (apic_lvt_nmi_mode(lvt0_val
)) {
715 if (!nmi_wd_enabled
) {
716 apic_debug("Receive NMI setting on APIC_LVT0 "
717 "for cpu %d\n", apic
->vcpu
->vcpu_id
);
718 apic
->vcpu
->kvm
->arch
.vapics_in_nmi_mode
++;
720 } else if (nmi_wd_enabled
)
721 apic
->vcpu
->kvm
->arch
.vapics_in_nmi_mode
--;
724 static int apic_reg_write(struct kvm_lapic
*apic
, u32 reg
, u32 val
)
728 trace_kvm_apic_write(reg
, val
);
731 case APIC_ID
: /* Local APIC ID */
732 if (!apic_x2apic_mode(apic
))
733 apic_set_reg(apic
, APIC_ID
, val
);
739 report_tpr_access(apic
, true);
740 apic_set_tpr(apic
, val
& 0xff);
748 if (!apic_x2apic_mode(apic
))
749 apic_set_reg(apic
, APIC_LDR
, val
& APIC_LDR_MASK
);
755 if (!apic_x2apic_mode(apic
))
756 apic_set_reg(apic
, APIC_DFR
, val
| 0x0FFFFFFF);
763 if (apic_get_reg(apic
, APIC_LVR
) & APIC_LVR_DIRECTED_EOI
)
764 mask
|= APIC_SPIV_DIRECTED_EOI
;
765 apic_set_reg(apic
, APIC_SPIV
, val
& mask
);
766 if (!(val
& APIC_SPIV_APIC_ENABLED
)) {
770 for (i
= 0; i
< APIC_LVT_NUM
; i
++) {
771 lvt_val
= apic_get_reg(apic
,
772 APIC_LVTT
+ 0x10 * i
);
773 apic_set_reg(apic
, APIC_LVTT
+ 0x10 * i
,
774 lvt_val
| APIC_LVT_MASKED
);
776 atomic_set(&apic
->lapic_timer
.pending
, 0);
782 /* No delay here, so we always clear the pending bit */
783 apic_set_reg(apic
, APIC_ICR
, val
& ~(1 << 12));
788 if (!apic_x2apic_mode(apic
))
790 apic_set_reg(apic
, APIC_ICR2
, val
);
794 apic_manage_nmi_watchdog(apic
, val
);
800 /* TODO: Check vector */
801 if (!apic_sw_enabled(apic
))
802 val
|= APIC_LVT_MASKED
;
804 val
&= apic_lvt_mask
[(reg
- APIC_LVTT
) >> 4];
805 apic_set_reg(apic
, reg
, val
);
810 hrtimer_cancel(&apic
->lapic_timer
.timer
);
811 apic_set_reg(apic
, APIC_TMICT
, val
);
812 start_apic_timer(apic
);
817 apic_debug("KVM_WRITE:TDCR %x\n", val
);
818 apic_set_reg(apic
, APIC_TDCR
, val
);
819 update_divide_count(apic
);
823 if (apic_x2apic_mode(apic
) && val
!= 0) {
824 apic_debug("KVM_WRITE:ESR not zero %x\n", val
);
830 if (apic_x2apic_mode(apic
)) {
831 apic_reg_write(apic
, APIC_ICR
, 0x40000 | (val
& 0xff));
840 apic_debug("Local APIC Write to read-only register %x\n", reg
);
844 static int apic_mmio_write(struct kvm_io_device
*this,
845 gpa_t address
, int len
, const void *data
)
847 struct kvm_lapic
*apic
= to_lapic(this);
848 unsigned int offset
= address
- apic
->base_address
;
851 if (!apic_mmio_in_range(apic
, address
))
855 * APIC register must be aligned on 128-bits boundary.
856 * 32/64/128 bits registers must be accessed thru 32 bits.
859 if (len
!= 4 || (offset
& 0xf)) {
860 /* Don't shout loud, $infamous_os would cause only noise. */
861 apic_debug("apic write: bad size=%d %lx\n", len
, (long)address
);
867 /* too common printing */
868 if (offset
!= APIC_EOI
)
869 apic_debug("%s: offset 0x%x with length 0x%x, and value is "
870 "0x%x\n", __func__
, offset
, len
, val
);
872 apic_reg_write(apic
, offset
& 0xff0, val
);
877 void kvm_lapic_set_eoi(struct kvm_vcpu
*vcpu
)
879 struct kvm_lapic
*apic
= vcpu
->arch
.apic
;
882 apic_reg_write(vcpu
->arch
.apic
, APIC_EOI
, 0);
884 EXPORT_SYMBOL_GPL(kvm_lapic_set_eoi
);
886 void kvm_free_lapic(struct kvm_vcpu
*vcpu
)
888 if (!vcpu
->arch
.apic
)
891 hrtimer_cancel(&vcpu
->arch
.apic
->lapic_timer
.timer
);
893 if (vcpu
->arch
.apic
->regs
)
894 free_page((unsigned long)vcpu
->arch
.apic
->regs
);
896 kfree(vcpu
->arch
.apic
);
900 *----------------------------------------------------------------------
902 *----------------------------------------------------------------------
905 void kvm_lapic_set_tpr(struct kvm_vcpu
*vcpu
, unsigned long cr8
)
907 struct kvm_lapic
*apic
= vcpu
->arch
.apic
;
911 apic_set_tpr(apic
, ((cr8
& 0x0f) << 4)
912 | (apic_get_reg(apic
, APIC_TASKPRI
) & 4));
915 u64
kvm_lapic_get_cr8(struct kvm_vcpu
*vcpu
)
917 struct kvm_lapic
*apic
= vcpu
->arch
.apic
;
922 tpr
= (u64
) apic_get_reg(apic
, APIC_TASKPRI
);
924 return (tpr
& 0xf0) >> 4;
927 void kvm_lapic_set_base(struct kvm_vcpu
*vcpu
, u64 value
)
929 struct kvm_lapic
*apic
= vcpu
->arch
.apic
;
932 value
|= MSR_IA32_APICBASE_BSP
;
933 vcpu
->arch
.apic_base
= value
;
937 if (!kvm_vcpu_is_bsp(apic
->vcpu
))
938 value
&= ~MSR_IA32_APICBASE_BSP
;
940 vcpu
->arch
.apic_base
= value
;
941 if (apic_x2apic_mode(apic
)) {
942 u32 id
= kvm_apic_id(apic
);
943 u32 ldr
= ((id
& ~0xf) << 16) | (1 << (id
& 0xf));
944 apic_set_reg(apic
, APIC_LDR
, ldr
);
946 apic
->base_address
= apic
->vcpu
->arch
.apic_base
&
947 MSR_IA32_APICBASE_BASE
;
949 /* with FSB delivery interrupt, we can restart APIC functionality */
950 apic_debug("apic base msr is 0x%016" PRIx64
", and base address is "
951 "0x%lx.\n", apic
->vcpu
->arch
.apic_base
, apic
->base_address
);
955 void kvm_lapic_reset(struct kvm_vcpu
*vcpu
)
957 struct kvm_lapic
*apic
;
960 apic_debug("%s\n", __func__
);
963 apic
= vcpu
->arch
.apic
;
964 ASSERT(apic
!= NULL
);
966 /* Stop the timer in case it's a reset to an active apic */
967 hrtimer_cancel(&apic
->lapic_timer
.timer
);
969 apic_set_reg(apic
, APIC_ID
, vcpu
->vcpu_id
<< 24);
970 kvm_apic_set_version(apic
->vcpu
);
972 for (i
= 0; i
< APIC_LVT_NUM
; i
++)
973 apic_set_reg(apic
, APIC_LVTT
+ 0x10 * i
, APIC_LVT_MASKED
);
974 apic_set_reg(apic
, APIC_LVT0
,
975 SET_APIC_DELIVERY_MODE(0, APIC_MODE_EXTINT
));
977 apic_set_reg(apic
, APIC_DFR
, 0xffffffffU
);
978 apic_set_reg(apic
, APIC_SPIV
, 0xff);
979 apic_set_reg(apic
, APIC_TASKPRI
, 0);
980 apic_set_reg(apic
, APIC_LDR
, 0);
981 apic_set_reg(apic
, APIC_ESR
, 0);
982 apic_set_reg(apic
, APIC_ICR
, 0);
983 apic_set_reg(apic
, APIC_ICR2
, 0);
984 apic_set_reg(apic
, APIC_TDCR
, 0);
985 apic_set_reg(apic
, APIC_TMICT
, 0);
986 for (i
= 0; i
< 8; i
++) {
987 apic_set_reg(apic
, APIC_IRR
+ 0x10 * i
, 0);
988 apic_set_reg(apic
, APIC_ISR
+ 0x10 * i
, 0);
989 apic_set_reg(apic
, APIC_TMR
+ 0x10 * i
, 0);
991 apic
->irr_pending
= false;
992 update_divide_count(apic
);
993 atomic_set(&apic
->lapic_timer
.pending
, 0);
994 if (kvm_vcpu_is_bsp(vcpu
))
995 vcpu
->arch
.apic_base
|= MSR_IA32_APICBASE_BSP
;
996 apic_update_ppr(apic
);
998 vcpu
->arch
.apic_arb_prio
= 0;
1000 apic_debug(KERN_INFO
"%s: vcpu=%p, id=%d, base_msr="
1001 "0x%016" PRIx64
", base_address=0x%0lx.\n", __func__
,
1002 vcpu
, kvm_apic_id(apic
),
1003 vcpu
->arch
.apic_base
, apic
->base_address
);
1006 bool kvm_apic_present(struct kvm_vcpu
*vcpu
)
1008 return vcpu
->arch
.apic
&& apic_hw_enabled(vcpu
->arch
.apic
);
1011 int kvm_lapic_enabled(struct kvm_vcpu
*vcpu
)
1013 return kvm_apic_present(vcpu
) && apic_sw_enabled(vcpu
->arch
.apic
);
1017 *----------------------------------------------------------------------
1019 *----------------------------------------------------------------------
1022 static bool lapic_is_periodic(struct kvm_timer
*ktimer
)
1024 struct kvm_lapic
*apic
= container_of(ktimer
, struct kvm_lapic
,
1026 return apic_lvtt_period(apic
);
1029 int apic_has_pending_timer(struct kvm_vcpu
*vcpu
)
1031 struct kvm_lapic
*lapic
= vcpu
->arch
.apic
;
1033 if (lapic
&& apic_enabled(lapic
) && apic_lvt_enabled(lapic
, APIC_LVTT
))
1034 return atomic_read(&lapic
->lapic_timer
.pending
);
1039 static int kvm_apic_local_deliver(struct kvm_lapic
*apic
, int lvt_type
)
1041 u32 reg
= apic_get_reg(apic
, lvt_type
);
1042 int vector
, mode
, trig_mode
;
1044 if (apic_hw_enabled(apic
) && !(reg
& APIC_LVT_MASKED
)) {
1045 vector
= reg
& APIC_VECTOR_MASK
;
1046 mode
= reg
& APIC_MODE_MASK
;
1047 trig_mode
= reg
& APIC_LVT_LEVEL_TRIGGER
;
1048 return __apic_accept_irq(apic
, mode
, vector
, 1, trig_mode
);
1053 void kvm_apic_nmi_wd_deliver(struct kvm_vcpu
*vcpu
)
1055 struct kvm_lapic
*apic
= vcpu
->arch
.apic
;
1058 kvm_apic_local_deliver(apic
, APIC_LVT0
);
1061 static struct kvm_timer_ops lapic_timer_ops
= {
1062 .is_periodic
= lapic_is_periodic
,
1065 static const struct kvm_io_device_ops apic_mmio_ops
= {
1066 .read
= apic_mmio_read
,
1067 .write
= apic_mmio_write
,
1070 int kvm_create_lapic(struct kvm_vcpu
*vcpu
)
1072 struct kvm_lapic
*apic
;
1074 ASSERT(vcpu
!= NULL
);
1075 apic_debug("apic_init %d\n", vcpu
->vcpu_id
);
1077 apic
= kzalloc(sizeof(*apic
), GFP_KERNEL
);
1081 vcpu
->arch
.apic
= apic
;
1083 apic
->regs
= (void *)get_zeroed_page(GFP_KERNEL
);
1085 printk(KERN_ERR
"malloc apic regs error for vcpu %x\n",
1087 goto nomem_free_apic
;
1091 hrtimer_init(&apic
->lapic_timer
.timer
, CLOCK_MONOTONIC
,
1093 apic
->lapic_timer
.timer
.function
= kvm_timer_fn
;
1094 apic
->lapic_timer
.t_ops
= &lapic_timer_ops
;
1095 apic
->lapic_timer
.kvm
= vcpu
->kvm
;
1096 apic
->lapic_timer
.vcpu
= vcpu
;
1098 apic
->base_address
= APIC_DEFAULT_PHYS_BASE
;
1099 vcpu
->arch
.apic_base
= APIC_DEFAULT_PHYS_BASE
;
1101 kvm_lapic_reset(vcpu
);
1102 kvm_iodevice_init(&apic
->dev
, &apic_mmio_ops
);
1111 int kvm_apic_has_interrupt(struct kvm_vcpu
*vcpu
)
1113 struct kvm_lapic
*apic
= vcpu
->arch
.apic
;
1116 if (!apic
|| !apic_enabled(apic
))
1119 apic_update_ppr(apic
);
1120 highest_irr
= apic_find_highest_irr(apic
);
1121 if ((highest_irr
== -1) ||
1122 ((highest_irr
& 0xF0) <= apic_get_reg(apic
, APIC_PROCPRI
)))
1127 int kvm_apic_accept_pic_intr(struct kvm_vcpu
*vcpu
)
1129 u32 lvt0
= apic_get_reg(vcpu
->arch
.apic
, APIC_LVT0
);
1132 if (!apic_hw_enabled(vcpu
->arch
.apic
))
1134 if ((lvt0
& APIC_LVT_MASKED
) == 0 &&
1135 GET_APIC_DELIVERY_MODE(lvt0
) == APIC_MODE_EXTINT
)
1140 void kvm_inject_apic_timer_irqs(struct kvm_vcpu
*vcpu
)
1142 struct kvm_lapic
*apic
= vcpu
->arch
.apic
;
1144 if (apic
&& atomic_read(&apic
->lapic_timer
.pending
) > 0) {
1145 if (kvm_apic_local_deliver(apic
, APIC_LVTT
))
1146 atomic_dec(&apic
->lapic_timer
.pending
);
1150 int kvm_get_apic_interrupt(struct kvm_vcpu
*vcpu
)
1152 int vector
= kvm_apic_has_interrupt(vcpu
);
1153 struct kvm_lapic
*apic
= vcpu
->arch
.apic
;
1158 apic_set_vector(vector
, apic
->regs
+ APIC_ISR
);
1159 apic_update_ppr(apic
);
1160 apic_clear_irr(vector
, apic
);
1164 void kvm_apic_post_state_restore(struct kvm_vcpu
*vcpu
)
1166 struct kvm_lapic
*apic
= vcpu
->arch
.apic
;
1168 apic
->base_address
= vcpu
->arch
.apic_base
&
1169 MSR_IA32_APICBASE_BASE
;
1170 kvm_apic_set_version(vcpu
);
1172 apic_update_ppr(apic
);
1173 hrtimer_cancel(&apic
->lapic_timer
.timer
);
1174 update_divide_count(apic
);
1175 start_apic_timer(apic
);
1176 apic
->irr_pending
= true;
1177 kvm_make_request(KVM_REQ_EVENT
, vcpu
);
1180 void __kvm_migrate_apic_timer(struct kvm_vcpu
*vcpu
)
1182 struct kvm_lapic
*apic
= vcpu
->arch
.apic
;
1183 struct hrtimer
*timer
;
1188 timer
= &apic
->lapic_timer
.timer
;
1189 if (hrtimer_cancel(timer
))
1190 hrtimer_start_expires(timer
, HRTIMER_MODE_ABS
);
1193 void kvm_lapic_sync_from_vapic(struct kvm_vcpu
*vcpu
)
1198 if (!irqchip_in_kernel(vcpu
->kvm
) || !vcpu
->arch
.apic
->vapic_addr
)
1201 vapic
= kmap_atomic(vcpu
->arch
.apic
->vapic_page
, KM_USER0
);
1202 data
= *(u32
*)(vapic
+ offset_in_page(vcpu
->arch
.apic
->vapic_addr
));
1203 kunmap_atomic(vapic
, KM_USER0
);
1205 apic_set_tpr(vcpu
->arch
.apic
, data
& 0xff);
1208 void kvm_lapic_sync_to_vapic(struct kvm_vcpu
*vcpu
)
1211 int max_irr
, max_isr
;
1212 struct kvm_lapic
*apic
;
1215 if (!irqchip_in_kernel(vcpu
->kvm
) || !vcpu
->arch
.apic
->vapic_addr
)
1218 apic
= vcpu
->arch
.apic
;
1219 tpr
= apic_get_reg(apic
, APIC_TASKPRI
) & 0xff;
1220 max_irr
= apic_find_highest_irr(apic
);
1223 max_isr
= apic_find_highest_isr(apic
);
1226 data
= (tpr
& 0xff) | ((max_isr
& 0xf0) << 8) | (max_irr
<< 24);
1228 vapic
= kmap_atomic(vcpu
->arch
.apic
->vapic_page
, KM_USER0
);
1229 *(u32
*)(vapic
+ offset_in_page(vcpu
->arch
.apic
->vapic_addr
)) = data
;
1230 kunmap_atomic(vapic
, KM_USER0
);
1233 void kvm_lapic_set_vapic_addr(struct kvm_vcpu
*vcpu
, gpa_t vapic_addr
)
1235 if (!irqchip_in_kernel(vcpu
->kvm
))
1238 vcpu
->arch
.apic
->vapic_addr
= vapic_addr
;
1241 int kvm_x2apic_msr_write(struct kvm_vcpu
*vcpu
, u32 msr
, u64 data
)
1243 struct kvm_lapic
*apic
= vcpu
->arch
.apic
;
1244 u32 reg
= (msr
- APIC_BASE_MSR
) << 4;
1246 if (!irqchip_in_kernel(vcpu
->kvm
) || !apic_x2apic_mode(apic
))
1249 /* if this is ICR write vector before command */
1251 apic_reg_write(apic
, APIC_ICR2
, (u32
)(data
>> 32));
1252 return apic_reg_write(apic
, reg
, (u32
)data
);
1255 int kvm_x2apic_msr_read(struct kvm_vcpu
*vcpu
, u32 msr
, u64
*data
)
1257 struct kvm_lapic
*apic
= vcpu
->arch
.apic
;
1258 u32 reg
= (msr
- APIC_BASE_MSR
) << 4, low
, high
= 0;
1260 if (!irqchip_in_kernel(vcpu
->kvm
) || !apic_x2apic_mode(apic
))
1263 if (apic_reg_read(apic
, reg
, 4, &low
))
1266 apic_reg_read(apic
, APIC_ICR2
, 4, &high
);
1268 *data
= (((u64
)high
) << 32) | low
;
1273 int kvm_hv_vapic_msr_write(struct kvm_vcpu
*vcpu
, u32 reg
, u64 data
)
1275 struct kvm_lapic
*apic
= vcpu
->arch
.apic
;
1277 if (!irqchip_in_kernel(vcpu
->kvm
))
1280 /* if this is ICR write vector before command */
1281 if (reg
== APIC_ICR
)
1282 apic_reg_write(apic
, APIC_ICR2
, (u32
)(data
>> 32));
1283 return apic_reg_write(apic
, reg
, (u32
)data
);
1286 int kvm_hv_vapic_msr_read(struct kvm_vcpu
*vcpu
, u32 reg
, u64
*data
)
1288 struct kvm_lapic
*apic
= vcpu
->arch
.apic
;
1291 if (!irqchip_in_kernel(vcpu
->kvm
))
1294 if (apic_reg_read(apic
, reg
, 4, &low
))
1296 if (reg
== APIC_ICR
)
1297 apic_reg_read(apic
, APIC_ICR2
, 4, &high
);
1299 *data
= (((u64
)high
) << 32) | low
;