1 #ifndef __ASM_PARAVIRT_H
2 #define __ASM_PARAVIRT_H
3 /* Various instructions on x86 need to be replaced for
4 * para-virtualization: those hooks are defined here. */
5 #include <linux/stringify.h>
9 /* These are the most performance critical ops, so we want to be able to patch
11 #define PARAVIRT_IRQ_DISABLE 0
12 #define PARAVIRT_IRQ_ENABLE 1
13 #define PARAVIRT_RESTORE_FLAGS 2
14 #define PARAVIRT_SAVE_FLAGS 3
15 #define PARAVIRT_SAVE_FLAGS_IRQ_DISABLE 4
16 #define PARAVIRT_INTERRUPT_RETURN 5
17 #define PARAVIRT_STI_SYSEXIT 6
19 /* Bitmask of what can be clobbered: usually at least eax. */
27 #include <linux/types.h>
30 struct Xgt_desc_struct
;
36 unsigned int kernel_rpl
;
41 * Patch may replace one of the defined code sequences with arbitrary
42 * code, subject to the same register constraints. This generally
43 * means the code is not free to clobber any registers other than EAX.
44 * The patch function should return the number of bytes of code
45 * generated, as we nop pad the rest in generic code.
47 unsigned (*patch
)(u8 type
, u16 clobber
, void *firstinsn
, unsigned len
);
49 void (*arch_setup
)(void);
50 char *(*memory_setup
)(void);
51 void (*init_IRQ
)(void);
55 unsigned long (*get_wallclock
)(void);
56 int (*set_wallclock
)(unsigned long);
57 void (*time_init
)(void);
59 void (*cpuid
)(unsigned int *eax
, unsigned int *ebx
,
60 unsigned int *ecx
, unsigned int *edx
);
62 unsigned long (*get_debugreg
)(int regno
);
63 void (*set_debugreg
)(int regno
, unsigned long value
);
67 unsigned long (*read_cr0
)(void);
68 void (*write_cr0
)(unsigned long);
70 unsigned long (*read_cr2
)(void);
71 void (*write_cr2
)(unsigned long);
73 unsigned long (*read_cr3
)(void);
74 void (*write_cr3
)(unsigned long);
76 unsigned long (*read_cr4_safe
)(void);
77 unsigned long (*read_cr4
)(void);
78 void (*write_cr4
)(unsigned long);
80 unsigned long (*save_fl
)(void);
81 void (*restore_fl
)(unsigned long);
82 void (*irq_disable
)(void);
83 void (*irq_enable
)(void);
84 void (*safe_halt
)(void);
88 /* err = 0/-EFAULT. wrmsr returns 0/-EFAULT. */
89 u64 (*read_msr
)(unsigned int msr
, int *err
);
90 int (*write_msr
)(unsigned int msr
, u64 val
);
92 u64 (*read_tsc
)(void);
93 u64 (*read_pmc
)(void);
94 u64 (*get_scheduled_cycles
)(void);
95 unsigned long (*get_cpu_khz
)(void);
97 void (*load_tr_desc
)(void);
98 void (*load_gdt
)(const struct Xgt_desc_struct
*);
99 void (*load_idt
)(const struct Xgt_desc_struct
*);
100 void (*store_gdt
)(struct Xgt_desc_struct
*);
101 void (*store_idt
)(struct Xgt_desc_struct
*);
102 void (*set_ldt
)(const void *desc
, unsigned entries
);
103 unsigned long (*store_tr
)(void);
104 void (*load_tls
)(struct thread_struct
*t
, unsigned int cpu
);
105 void (*write_ldt_entry
)(struct desc_struct
*,
106 int entrynum
, u32 low
, u32 high
);
107 void (*write_gdt_entry
)(struct desc_struct
*,
108 int entrynum
, u32 low
, u32 high
);
109 void (*write_idt_entry
)(struct desc_struct
*,
110 int entrynum
, u32 low
, u32 high
);
111 void (*load_esp0
)(struct tss_struct
*tss
, struct thread_struct
*t
);
113 void (*set_iopl_mask
)(unsigned mask
);
115 void (*io_delay
)(void);
117 #ifdef CONFIG_X86_LOCAL_APIC
118 void (*apic_write
)(unsigned long reg
, unsigned long v
);
119 void (*apic_write_atomic
)(unsigned long reg
, unsigned long v
);
120 unsigned long (*apic_read
)(unsigned long reg
);
121 void (*setup_boot_clock
)(void);
122 void (*setup_secondary_clock
)(void);
125 void (*flush_tlb_user
)(void);
126 void (*flush_tlb_kernel
)(void);
127 void (*flush_tlb_single
)(u32 addr
);
129 void (*map_pt_hook
)(int type
, pte_t
*va
, u32 pfn
);
131 void (*alloc_pt
)(u32 pfn
);
132 void (*alloc_pd
)(u32 pfn
);
133 void (*alloc_pd_clone
)(u32 pfn
, u32 clonepfn
, u32 start
, u32 count
);
134 void (*release_pt
)(u32 pfn
);
135 void (*release_pd
)(u32 pfn
);
137 void (*set_pte
)(pte_t
*ptep
, pte_t pteval
);
138 void (*set_pte_at
)(struct mm_struct
*mm
, unsigned long addr
, pte_t
*ptep
, pte_t pteval
);
139 void (*set_pmd
)(pmd_t
*pmdp
, pmd_t pmdval
);
140 void (*pte_update
)(struct mm_struct
*mm
, unsigned long addr
, pte_t
*ptep
);
141 void (*pte_update_defer
)(struct mm_struct
*mm
, unsigned long addr
, pte_t
*ptep
);
143 pte_t (*ptep_get_and_clear
)(pte_t
*ptep
);
145 #ifdef CONFIG_X86_PAE
146 void (*set_pte_atomic
)(pte_t
*ptep
, pte_t pteval
);
147 void (*set_pte_present
)(struct mm_struct
*mm
, unsigned long addr
, pte_t
*ptep
, pte_t pte
);
148 void (*set_pud
)(pud_t
*pudp
, pud_t pudval
);
149 void (*pte_clear
)(struct mm_struct
*mm
, unsigned long addr
, pte_t
*ptep
);
150 void (*pmd_clear
)(pmd_t
*pmdp
);
152 unsigned long long (*pte_val
)(pte_t
);
153 unsigned long long (*pmd_val
)(pmd_t
);
154 unsigned long long (*pgd_val
)(pgd_t
);
156 pte_t (*make_pte
)(unsigned long long pte
);
157 pmd_t (*make_pmd
)(unsigned long long pmd
);
158 pgd_t (*make_pgd
)(unsigned long long pgd
);
160 unsigned long (*pte_val
)(pte_t
);
161 unsigned long (*pgd_val
)(pgd_t
);
163 pte_t (*make_pte
)(unsigned long pte
);
164 pgd_t (*make_pgd
)(unsigned long pgd
);
167 void (*set_lazy_mode
)(int mode
);
169 /* These two are jmp to, not actually called. */
170 void (*irq_enable_sysexit
)(void);
173 void (*startup_ipi_hook
)(int phys_apicid
, unsigned long start_eip
, unsigned long start_esp
);
176 /* Mark a paravirt probe function. */
177 #define paravirt_probe(fn) \
178 static asmlinkage void (*__paravirtprobe_##fn)(void) __attribute_used__ \
179 __attribute__((__section__(".paravirtprobe"))) = fn
181 extern struct paravirt_ops paravirt_ops
;
183 #define paravirt_enabled() (paravirt_ops.paravirt_enabled)
185 static inline void load_esp0(struct tss_struct
*tss
,
186 struct thread_struct
*thread
)
188 paravirt_ops
.load_esp0(tss
, thread
);
191 #define ARCH_SETUP paravirt_ops.arch_setup();
192 static inline unsigned long get_wallclock(void)
194 return paravirt_ops
.get_wallclock();
197 static inline int set_wallclock(unsigned long nowtime
)
199 return paravirt_ops
.set_wallclock(nowtime
);
202 static inline void (*choose_time_init(void))(void)
204 return paravirt_ops
.time_init
;
207 /* The paravirtualized CPUID instruction. */
208 static inline void __cpuid(unsigned int *eax
, unsigned int *ebx
,
209 unsigned int *ecx
, unsigned int *edx
)
211 paravirt_ops
.cpuid(eax
, ebx
, ecx
, edx
);
215 * These special macros can be used to get or set a debugging register
217 #define get_debugreg(var, reg) var = paravirt_ops.get_debugreg(reg)
218 #define set_debugreg(val, reg) paravirt_ops.set_debugreg(reg, val)
220 #define clts() paravirt_ops.clts()
222 #define read_cr0() paravirt_ops.read_cr0()
223 #define write_cr0(x) paravirt_ops.write_cr0(x)
225 #define read_cr2() paravirt_ops.read_cr2()
226 #define write_cr2(x) paravirt_ops.write_cr2(x)
228 #define read_cr3() paravirt_ops.read_cr3()
229 #define write_cr3(x) paravirt_ops.write_cr3(x)
231 #define read_cr4() paravirt_ops.read_cr4()
232 #define read_cr4_safe(x) paravirt_ops.read_cr4_safe()
233 #define write_cr4(x) paravirt_ops.write_cr4(x)
235 #define raw_ptep_get_and_clear(xp) (paravirt_ops.ptep_get_and_clear(xp))
237 static inline void raw_safe_halt(void)
239 paravirt_ops
.safe_halt();
242 static inline void halt(void)
244 paravirt_ops
.safe_halt();
246 #define wbinvd() paravirt_ops.wbinvd()
248 #define get_kernel_rpl() (paravirt_ops.kernel_rpl)
250 /* These should all do BUG_ON(_err), but our headers are too tangled. */
251 #define rdmsr(msr,val1,val2) do { \
253 u64 _l = paravirt_ops.read_msr(msr,&_err); \
258 #define wrmsr(msr,val1,val2) do { \
259 u64 _l = ((u64)(val2) << 32) | (val1); \
260 paravirt_ops.write_msr((msr), _l); \
263 #define rdmsrl(msr,val) do { \
265 val = paravirt_ops.read_msr((msr),&_err); \
268 #define wrmsrl(msr,val) (paravirt_ops.write_msr((msr),(val)))
269 #define wrmsr_safe(msr,a,b) ({ \
270 u64 _l = ((u64)(b) << 32) | (a); \
271 paravirt_ops.write_msr((msr),_l); \
274 /* rdmsr with exception handling */
275 #define rdmsr_safe(msr,a,b) ({ \
277 u64 _l = paravirt_ops.read_msr(msr,&_err); \
282 #define rdtsc(low,high) do { \
283 u64 _l = paravirt_ops.read_tsc(); \
288 #define rdtscl(low) do { \
289 u64 _l = paravirt_ops.read_tsc(); \
293 #define rdtscll(val) (val = paravirt_ops.read_tsc())
295 #define get_scheduled_cycles(val) (val = paravirt_ops.get_scheduled_cycles())
296 #define calculate_cpu_khz() (paravirt_ops.get_cpu_khz())
298 #define write_tsc(val1,val2) wrmsr(0x10, val1, val2)
300 #define rdpmc(counter,low,high) do { \
301 u64 _l = paravirt_ops.read_pmc(); \
306 #define load_TR_desc() (paravirt_ops.load_tr_desc())
307 #define load_gdt(dtr) (paravirt_ops.load_gdt(dtr))
308 #define load_idt(dtr) (paravirt_ops.load_idt(dtr))
309 #define set_ldt(addr, entries) (paravirt_ops.set_ldt((addr), (entries)))
310 #define store_gdt(dtr) (paravirt_ops.store_gdt(dtr))
311 #define store_idt(dtr) (paravirt_ops.store_idt(dtr))
312 #define store_tr(tr) ((tr) = paravirt_ops.store_tr())
313 #define load_TLS(t,cpu) (paravirt_ops.load_tls((t),(cpu)))
314 #define write_ldt_entry(dt, entry, low, high) \
315 (paravirt_ops.write_ldt_entry((dt), (entry), (low), (high)))
316 #define write_gdt_entry(dt, entry, low, high) \
317 (paravirt_ops.write_gdt_entry((dt), (entry), (low), (high)))
318 #define write_idt_entry(dt, entry, low, high) \
319 (paravirt_ops.write_idt_entry((dt), (entry), (low), (high)))
320 #define set_iopl_mask(mask) (paravirt_ops.set_iopl_mask(mask))
322 #define __pte(x) paravirt_ops.make_pte(x)
323 #define __pgd(x) paravirt_ops.make_pgd(x)
325 #define pte_val(x) paravirt_ops.pte_val(x)
326 #define pgd_val(x) paravirt_ops.pgd_val(x)
328 #ifdef CONFIG_X86_PAE
329 #define __pmd(x) paravirt_ops.make_pmd(x)
330 #define pmd_val(x) paravirt_ops.pmd_val(x)
333 /* The paravirtualized I/O functions */
334 static inline void slow_down_io(void) {
335 paravirt_ops
.io_delay();
336 #ifdef REALLY_SLOW_IO
337 paravirt_ops
.io_delay();
338 paravirt_ops
.io_delay();
339 paravirt_ops
.io_delay();
343 #ifdef CONFIG_X86_LOCAL_APIC
345 * Basic functions accessing APICs.
347 static inline void apic_write(unsigned long reg
, unsigned long v
)
349 paravirt_ops
.apic_write(reg
,v
);
352 static inline void apic_write_atomic(unsigned long reg
, unsigned long v
)
354 paravirt_ops
.apic_write_atomic(reg
,v
);
357 static inline unsigned long apic_read(unsigned long reg
)
359 return paravirt_ops
.apic_read(reg
);
362 static inline void setup_boot_clock(void)
364 paravirt_ops
.setup_boot_clock();
367 static inline void setup_secondary_clock(void)
369 paravirt_ops
.setup_secondary_clock();
375 static inline void startup_ipi_hook(int phys_apicid
, unsigned long start_eip
,
376 unsigned long start_esp
)
378 return paravirt_ops
.startup_ipi_hook(phys_apicid
, start_eip
, start_esp
);
382 #define __flush_tlb() paravirt_ops.flush_tlb_user()
383 #define __flush_tlb_global() paravirt_ops.flush_tlb_kernel()
384 #define __flush_tlb_single(addr) paravirt_ops.flush_tlb_single(addr)
386 #define paravirt_map_pt_hook(type, va, pfn) paravirt_ops.map_pt_hook(type, va, pfn)
388 #define paravirt_alloc_pt(pfn) paravirt_ops.alloc_pt(pfn)
389 #define paravirt_release_pt(pfn) paravirt_ops.release_pt(pfn)
391 #define paravirt_alloc_pd(pfn) paravirt_ops.alloc_pd(pfn)
392 #define paravirt_alloc_pd_clone(pfn, clonepfn, start, count) \
393 paravirt_ops.alloc_pd_clone(pfn, clonepfn, start, count)
394 #define paravirt_release_pd(pfn) paravirt_ops.release_pd(pfn)
396 static inline void set_pte(pte_t
*ptep
, pte_t pteval
)
398 paravirt_ops
.set_pte(ptep
, pteval
);
401 static inline void set_pte_at(struct mm_struct
*mm
, unsigned long addr
,
402 pte_t
*ptep
, pte_t pteval
)
404 paravirt_ops
.set_pte_at(mm
, addr
, ptep
, pteval
);
407 static inline void set_pmd(pmd_t
*pmdp
, pmd_t pmdval
)
409 paravirt_ops
.set_pmd(pmdp
, pmdval
);
412 static inline void pte_update(struct mm_struct
*mm
, u32 addr
, pte_t
*ptep
)
414 paravirt_ops
.pte_update(mm
, addr
, ptep
);
417 static inline void pte_update_defer(struct mm_struct
*mm
, u32 addr
, pte_t
*ptep
)
419 paravirt_ops
.pte_update_defer(mm
, addr
, ptep
);
422 #ifdef CONFIG_X86_PAE
423 static inline void set_pte_atomic(pte_t
*ptep
, pte_t pteval
)
425 paravirt_ops
.set_pte_atomic(ptep
, pteval
);
428 static inline void set_pte_present(struct mm_struct
*mm
, unsigned long addr
, pte_t
*ptep
, pte_t pte
)
430 paravirt_ops
.set_pte_present(mm
, addr
, ptep
, pte
);
433 static inline void set_pud(pud_t
*pudp
, pud_t pudval
)
435 paravirt_ops
.set_pud(pudp
, pudval
);
438 static inline void pte_clear(struct mm_struct
*mm
, unsigned long addr
, pte_t
*ptep
)
440 paravirt_ops
.pte_clear(mm
, addr
, ptep
);
443 static inline void pmd_clear(pmd_t
*pmdp
)
445 paravirt_ops
.pmd_clear(pmdp
);
449 /* Lazy mode for batching updates / context switch */
450 #define PARAVIRT_LAZY_NONE 0
451 #define PARAVIRT_LAZY_MMU 1
452 #define PARAVIRT_LAZY_CPU 2
453 #define PARAVIRT_LAZY_FLUSH 3
455 #define __HAVE_ARCH_ENTER_LAZY_CPU_MODE
456 #define arch_enter_lazy_cpu_mode() paravirt_ops.set_lazy_mode(PARAVIRT_LAZY_CPU)
457 #define arch_leave_lazy_cpu_mode() paravirt_ops.set_lazy_mode(PARAVIRT_LAZY_NONE)
458 #define arch_flush_lazy_cpu_mode() paravirt_ops.set_lazy_mode(PARAVIRT_LAZY_FLUSH)
460 #define __HAVE_ARCH_ENTER_LAZY_MMU_MODE
461 #define arch_enter_lazy_mmu_mode() paravirt_ops.set_lazy_mode(PARAVIRT_LAZY_MMU)
462 #define arch_leave_lazy_mmu_mode() paravirt_ops.set_lazy_mode(PARAVIRT_LAZY_NONE)
463 #define arch_flush_lazy_mmu_mode() paravirt_ops.set_lazy_mode(PARAVIRT_LAZY_FLUSH)
465 void _paravirt_nop(void);
466 #define paravirt_nop ((void *)_paravirt_nop)
468 /* These all sit in the .parainstructions section to tell us what to patch. */
469 struct paravirt_patch
{
470 u8
*instr
; /* original instructions */
471 u8 instrtype
; /* type of this instruction */
472 u8 len
; /* length of original instruction */
473 u16 clobbers
; /* what registers you may clobber */
476 #define paravirt_alt(insn_string, typenum, clobber) \
477 "771:\n\t" insn_string "\n" "772:\n" \
478 ".pushsection .parainstructions,\"a\"\n" \
480 " .byte " __stringify(typenum) "\n" \
481 " .byte 772b-771b\n" \
482 " .short " __stringify(clobber) "\n" \
485 static inline unsigned long __raw_local_save_flags(void)
489 __asm__
__volatile__(paravirt_alt( "pushl %%ecx; pushl %%edx;"
491 "popl %%edx; popl %%ecx",
492 PARAVIRT_SAVE_FLAGS
, CLBR_NONE
)
493 : "=a"(f
): "m"(paravirt_ops
.save_fl
)
498 static inline void raw_local_irq_restore(unsigned long f
)
500 __asm__
__volatile__(paravirt_alt( "pushl %%ecx; pushl %%edx;"
502 "popl %%edx; popl %%ecx",
503 PARAVIRT_RESTORE_FLAGS
, CLBR_EAX
)
504 : "=a"(f
) : "m" (paravirt_ops
.restore_fl
), "0"(f
)
508 static inline void raw_local_irq_disable(void)
510 __asm__
__volatile__(paravirt_alt( "pushl %%ecx; pushl %%edx;"
512 "popl %%edx; popl %%ecx",
513 PARAVIRT_IRQ_DISABLE
, CLBR_EAX
)
514 : : "m" (paravirt_ops
.irq_disable
)
515 : "memory", "eax", "cc");
518 static inline void raw_local_irq_enable(void)
520 __asm__
__volatile__(paravirt_alt( "pushl %%ecx; pushl %%edx;"
522 "popl %%edx; popl %%ecx",
523 PARAVIRT_IRQ_ENABLE
, CLBR_EAX
)
524 : : "m" (paravirt_ops
.irq_enable
)
525 : "memory", "eax", "cc");
528 static inline unsigned long __raw_local_irq_save(void)
532 __asm__
__volatile__(paravirt_alt( "pushl %%ecx; pushl %%edx;"
533 "call *%1; pushl %%eax;"
534 "call *%2; popl %%eax;"
535 "popl %%edx; popl %%ecx",
536 PARAVIRT_SAVE_FLAGS_IRQ_DISABLE
,
539 : "m" (paravirt_ops
.save_fl
),
540 "m" (paravirt_ops
.irq_disable
)
545 #define CLI_STRING paravirt_alt("pushl %%ecx; pushl %%edx;" \
546 "call *paravirt_ops+%c[irq_disable];" \
547 "popl %%edx; popl %%ecx", \
548 PARAVIRT_IRQ_DISABLE, CLBR_EAX)
550 #define STI_STRING paravirt_alt("pushl %%ecx; pushl %%edx;" \
551 "call *paravirt_ops+%c[irq_enable];" \
552 "popl %%edx; popl %%ecx", \
553 PARAVIRT_IRQ_ENABLE, CLBR_EAX)
554 #define CLI_STI_CLOBBERS , "%eax"
555 #define CLI_STI_INPUT_ARGS \
557 [irq_disable] "i" (offsetof(struct paravirt_ops, irq_disable)), \
558 [irq_enable] "i" (offsetof(struct paravirt_ops, irq_enable))
560 #else /* __ASSEMBLY__ */
562 #define PARA_PATCH(ptype, clobbers, ops) \
566 .pushsection .parainstructions,"a"; \
573 #define INTERRUPT_RETURN \
574 PARA_PATCH(PARAVIRT_INTERRUPT_RETURN, CLBR_ANY, \
575 jmp *%cs:paravirt_ops+PARAVIRT_iret)
577 #define DISABLE_INTERRUPTS(clobbers) \
578 PARA_PATCH(PARAVIRT_IRQ_DISABLE, clobbers, \
579 pushl %ecx; pushl %edx; \
580 call *paravirt_ops+PARAVIRT_irq_disable; \
581 popl %edx; popl %ecx) \
583 #define ENABLE_INTERRUPTS(clobbers) \
584 PARA_PATCH(PARAVIRT_IRQ_ENABLE, clobbers, \
585 pushl %ecx; pushl %edx; \
586 call *%cs:paravirt_ops+PARAVIRT_irq_enable; \
587 popl %edx; popl %ecx)
589 #define ENABLE_INTERRUPTS_SYSEXIT \
590 PARA_PATCH(PARAVIRT_STI_SYSEXIT, CLBR_ANY, \
591 jmp *%cs:paravirt_ops+PARAVIRT_irq_enable_sysexit)
593 #define GET_CR0_INTO_EAX \
594 call *paravirt_ops+PARAVIRT_read_cr0
596 #endif /* __ASSEMBLY__ */
597 #endif /* CONFIG_PARAVIRT */
598 #endif /* __ASM_PARAVIRT_H */