Commit | Line | Data |
---|---|---|
d3561b7f RR |
1 | /* Paravirtualization interfaces |
2 | Copyright (C) 2006 Rusty Russell IBM Corporation | |
3 | ||
4 | This program is free software; you can redistribute it and/or modify | |
5 | it under the terms of the GNU General Public License as published by | |
6 | the Free Software Foundation; either version 2 of the License, or | |
7 | (at your option) any later version. | |
8 | ||
9 | This program is distributed in the hope that it will be useful, | |
10 | but WITHOUT ANY WARRANTY; without even the implied warranty of | |
11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
12 | GNU General Public License for more details. | |
13 | ||
14 | You should have received a copy of the GNU General Public License | |
15 | along with this program; if not, write to the Free Software | |
16 | Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA | |
17 | */ | |
18 | #include <linux/errno.h> | |
19 | #include <linux/module.h> | |
20 | #include <linux/efi.h> | |
21 | #include <linux/bcd.h> | |
c9ccf30d | 22 | #include <linux/start_kernel.h> |
d3561b7f RR |
23 | |
24 | #include <asm/bug.h> | |
25 | #include <asm/paravirt.h> | |
26 | #include <asm/desc.h> | |
27 | #include <asm/setup.h> | |
28 | #include <asm/arch_hooks.h> | |
29 | #include <asm/time.h> | |
30 | #include <asm/irq.h> | |
31 | #include <asm/delay.h> | |
13623d79 RR |
32 | #include <asm/fixmap.h> |
33 | #include <asm/apic.h> | |
d3561b7f RR |
34 | |
35 | /* nop stub */ | |
36 | static void native_nop(void) | |
37 | { | |
38 | } | |
39 | ||
40 | static void __init default_banner(void) | |
41 | { | |
42 | printk(KERN_INFO "Booting paravirtualized kernel on %s\n", | |
43 | paravirt_ops.name); | |
44 | } | |
45 | ||
46 | char *memory_setup(void) | |
47 | { | |
48 | return paravirt_ops.memory_setup(); | |
49 | } | |
50 | ||
139ec7c4 RR |
51 | /* Simple instruction patching code. */ |
52 | #define DEF_NATIVE(name, code) \ | |
53 | extern const char start_##name[], end_##name[]; \ | |
54 | asm("start_" #name ": " code "; end_" #name ":") | |
55 | DEF_NATIVE(cli, "cli"); | |
56 | DEF_NATIVE(sti, "sti"); | |
57 | DEF_NATIVE(popf, "push %eax; popf"); | |
58 | DEF_NATIVE(pushf, "pushf; pop %eax"); | |
59 | DEF_NATIVE(pushf_cli, "pushf; pop %eax; cli"); | |
60 | DEF_NATIVE(iret, "iret"); | |
61 | DEF_NATIVE(sti_sysexit, "sti; sysexit"); | |
62 | ||
63 | static const struct native_insns | |
64 | { | |
65 | const char *start, *end; | |
66 | } native_insns[] = { | |
67 | [PARAVIRT_IRQ_DISABLE] = { start_cli, end_cli }, | |
68 | [PARAVIRT_IRQ_ENABLE] = { start_sti, end_sti }, | |
69 | [PARAVIRT_RESTORE_FLAGS] = { start_popf, end_popf }, | |
70 | [PARAVIRT_SAVE_FLAGS] = { start_pushf, end_pushf }, | |
71 | [PARAVIRT_SAVE_FLAGS_IRQ_DISABLE] = { start_pushf_cli, end_pushf_cli }, | |
72 | [PARAVIRT_INTERRUPT_RETURN] = { start_iret, end_iret }, | |
73 | [PARAVIRT_STI_SYSEXIT] = { start_sti_sysexit, end_sti_sysexit }, | |
74 | }; | |
75 | ||
76 | static unsigned native_patch(u8 type, u16 clobbers, void *insns, unsigned len) | |
77 | { | |
78 | unsigned int insn_len; | |
79 | ||
80 | /* Don't touch it if we don't have a replacement */ | |
81 | if (type >= ARRAY_SIZE(native_insns) || !native_insns[type].start) | |
82 | return len; | |
83 | ||
84 | insn_len = native_insns[type].end - native_insns[type].start; | |
85 | ||
86 | /* Similarly if we can't fit replacement. */ | |
87 | if (len < insn_len) | |
88 | return len; | |
89 | ||
90 | memcpy(insns, native_insns[type].start, insn_len); | |
91 | return insn_len; | |
92 | } | |
93 | ||
d3561b7f RR |
94 | static fastcall unsigned long native_get_debugreg(int regno) |
95 | { | |
96 | unsigned long val = 0; /* Damn you, gcc! */ | |
97 | ||
98 | switch (regno) { | |
99 | case 0: | |
100 | asm("movl %%db0, %0" :"=r" (val)); break; | |
101 | case 1: | |
102 | asm("movl %%db1, %0" :"=r" (val)); break; | |
103 | case 2: | |
104 | asm("movl %%db2, %0" :"=r" (val)); break; | |
105 | case 3: | |
106 | asm("movl %%db3, %0" :"=r" (val)); break; | |
107 | case 6: | |
108 | asm("movl %%db6, %0" :"=r" (val)); break; | |
109 | case 7: | |
110 | asm("movl %%db7, %0" :"=r" (val)); break; | |
111 | default: | |
112 | BUG(); | |
113 | } | |
114 | return val; | |
115 | } | |
116 | ||
117 | static fastcall void native_set_debugreg(int regno, unsigned long value) | |
118 | { | |
119 | switch (regno) { | |
120 | case 0: | |
121 | asm("movl %0,%%db0" : /* no output */ :"r" (value)); | |
122 | break; | |
123 | case 1: | |
124 | asm("movl %0,%%db1" : /* no output */ :"r" (value)); | |
125 | break; | |
126 | case 2: | |
127 | asm("movl %0,%%db2" : /* no output */ :"r" (value)); | |
128 | break; | |
129 | case 3: | |
130 | asm("movl %0,%%db3" : /* no output */ :"r" (value)); | |
131 | break; | |
132 | case 6: | |
133 | asm("movl %0,%%db6" : /* no output */ :"r" (value)); | |
134 | break; | |
135 | case 7: | |
136 | asm("movl %0,%%db7" : /* no output */ :"r" (value)); | |
137 | break; | |
138 | default: | |
139 | BUG(); | |
140 | } | |
141 | } | |
142 | ||
143 | void init_IRQ(void) | |
144 | { | |
145 | paravirt_ops.init_IRQ(); | |
146 | } | |
147 | ||
148 | static fastcall void native_clts(void) | |
149 | { | |
150 | asm volatile ("clts"); | |
151 | } | |
152 | ||
153 | static fastcall unsigned long native_read_cr0(void) | |
154 | { | |
155 | unsigned long val; | |
156 | asm volatile("movl %%cr0,%0\n\t" :"=r" (val)); | |
157 | return val; | |
158 | } | |
159 | ||
160 | static fastcall void native_write_cr0(unsigned long val) | |
161 | { | |
162 | asm volatile("movl %0,%%cr0": :"r" (val)); | |
163 | } | |
164 | ||
165 | static fastcall unsigned long native_read_cr2(void) | |
166 | { | |
167 | unsigned long val; | |
168 | asm volatile("movl %%cr2,%0\n\t" :"=r" (val)); | |
169 | return val; | |
170 | } | |
171 | ||
172 | static fastcall void native_write_cr2(unsigned long val) | |
173 | { | |
174 | asm volatile("movl %0,%%cr2": :"r" (val)); | |
175 | } | |
176 | ||
177 | static fastcall unsigned long native_read_cr3(void) | |
178 | { | |
179 | unsigned long val; | |
180 | asm volatile("movl %%cr3,%0\n\t" :"=r" (val)); | |
181 | return val; | |
182 | } | |
183 | ||
184 | static fastcall void native_write_cr3(unsigned long val) | |
185 | { | |
186 | asm volatile("movl %0,%%cr3": :"r" (val)); | |
187 | } | |
188 | ||
189 | static fastcall unsigned long native_read_cr4(void) | |
190 | { | |
191 | unsigned long val; | |
192 | asm volatile("movl %%cr4,%0\n\t" :"=r" (val)); | |
193 | return val; | |
194 | } | |
195 | ||
196 | static fastcall unsigned long native_read_cr4_safe(void) | |
197 | { | |
198 | unsigned long val; | |
199 | /* This could fault if %cr4 does not exist */ | |
200 | asm("1: movl %%cr4, %0 \n" | |
201 | "2: \n" | |
202 | ".section __ex_table,\"a\" \n" | |
203 | ".long 1b,2b \n" | |
204 | ".previous \n" | |
205 | : "=r" (val): "0" (0)); | |
206 | return val; | |
207 | } | |
208 | ||
209 | static fastcall void native_write_cr4(unsigned long val) | |
210 | { | |
211 | asm volatile("movl %0,%%cr4": :"r" (val)); | |
212 | } | |
213 | ||
214 | static fastcall unsigned long native_save_fl(void) | |
215 | { | |
216 | unsigned long f; | |
217 | asm volatile("pushfl ; popl %0":"=g" (f): /* no input */); | |
218 | return f; | |
219 | } | |
220 | ||
221 | static fastcall void native_restore_fl(unsigned long f) | |
222 | { | |
223 | asm volatile("pushl %0 ; popfl": /* no output */ | |
224 | :"g" (f) | |
225 | :"memory", "cc"); | |
226 | } | |
227 | ||
228 | static fastcall void native_irq_disable(void) | |
229 | { | |
230 | asm volatile("cli": : :"memory"); | |
231 | } | |
232 | ||
233 | static fastcall void native_irq_enable(void) | |
234 | { | |
235 | asm volatile("sti": : :"memory"); | |
236 | } | |
237 | ||
238 | static fastcall void native_safe_halt(void) | |
239 | { | |
240 | asm volatile("sti; hlt": : :"memory"); | |
241 | } | |
242 | ||
243 | static fastcall void native_halt(void) | |
244 | { | |
245 | asm volatile("hlt": : :"memory"); | |
246 | } | |
247 | ||
248 | static fastcall void native_wbinvd(void) | |
249 | { | |
250 | asm volatile("wbinvd": : :"memory"); | |
251 | } | |
252 | ||
253 | static fastcall unsigned long long native_read_msr(unsigned int msr, int *err) | |
254 | { | |
255 | unsigned long long val; | |
256 | ||
257 | asm volatile("2: rdmsr ; xorl %0,%0\n" | |
258 | "1:\n\t" | |
259 | ".section .fixup,\"ax\"\n\t" | |
260 | "3: movl %3,%0 ; jmp 1b\n\t" | |
261 | ".previous\n\t" | |
262 | ".section __ex_table,\"a\"\n" | |
263 | " .align 4\n\t" | |
264 | " .long 2b,3b\n\t" | |
265 | ".previous" | |
266 | : "=r" (*err), "=A" (val) | |
267 | : "c" (msr), "i" (-EFAULT)); | |
268 | ||
269 | return val; | |
270 | } | |
271 | ||
272 | static fastcall int native_write_msr(unsigned int msr, unsigned long long val) | |
273 | { | |
274 | int err; | |
275 | asm volatile("2: wrmsr ; xorl %0,%0\n" | |
276 | "1:\n\t" | |
277 | ".section .fixup,\"ax\"\n\t" | |
278 | "3: movl %4,%0 ; jmp 1b\n\t" | |
279 | ".previous\n\t" | |
280 | ".section __ex_table,\"a\"\n" | |
281 | " .align 4\n\t" | |
282 | " .long 2b,3b\n\t" | |
283 | ".previous" | |
284 | : "=a" (err) | |
285 | : "c" (msr), "0" ((u32)val), "d" ((u32)(val>>32)), | |
286 | "i" (-EFAULT)); | |
287 | return err; | |
288 | } | |
289 | ||
290 | static fastcall unsigned long long native_read_tsc(void) | |
291 | { | |
292 | unsigned long long val; | |
293 | asm volatile("rdtsc" : "=A" (val)); | |
294 | return val; | |
295 | } | |
296 | ||
297 | static fastcall unsigned long long native_read_pmc(void) | |
298 | { | |
299 | unsigned long long val; | |
300 | asm volatile("rdpmc" : "=A" (val)); | |
301 | return val; | |
302 | } | |
303 | ||
304 | static fastcall void native_load_tr_desc(void) | |
305 | { | |
306 | asm volatile("ltr %w0"::"q" (GDT_ENTRY_TSS*8)); | |
307 | } | |
308 | ||
309 | static fastcall void native_load_gdt(const struct Xgt_desc_struct *dtr) | |
310 | { | |
311 | asm volatile("lgdt %0"::"m" (*dtr)); | |
312 | } | |
313 | ||
314 | static fastcall void native_load_idt(const struct Xgt_desc_struct *dtr) | |
315 | { | |
316 | asm volatile("lidt %0"::"m" (*dtr)); | |
317 | } | |
318 | ||
319 | static fastcall void native_store_gdt(struct Xgt_desc_struct *dtr) | |
320 | { | |
321 | asm ("sgdt %0":"=m" (*dtr)); | |
322 | } | |
323 | ||
324 | static fastcall void native_store_idt(struct Xgt_desc_struct *dtr) | |
325 | { | |
326 | asm ("sidt %0":"=m" (*dtr)); | |
327 | } | |
328 | ||
329 | static fastcall unsigned long native_store_tr(void) | |
330 | { | |
331 | unsigned long tr; | |
332 | asm ("str %0":"=r" (tr)); | |
333 | return tr; | |
334 | } | |
335 | ||
336 | static fastcall void native_load_tls(struct thread_struct *t, unsigned int cpu) | |
337 | { | |
338 | #define C(i) get_cpu_gdt_table(cpu)[GDT_ENTRY_TLS_MIN + i] = t->tls_array[i] | |
339 | C(0); C(1); C(2); | |
340 | #undef C | |
341 | } | |
342 | ||
343 | static inline void native_write_dt_entry(void *dt, int entry, u32 entry_low, u32 entry_high) | |
344 | { | |
345 | u32 *lp = (u32 *)((char *)dt + entry*8); | |
346 | lp[0] = entry_low; | |
347 | lp[1] = entry_high; | |
348 | } | |
349 | ||
350 | static fastcall void native_write_ldt_entry(void *dt, int entrynum, u32 low, u32 high) | |
351 | { | |
352 | native_write_dt_entry(dt, entrynum, low, high); | |
353 | } | |
354 | ||
355 | static fastcall void native_write_gdt_entry(void *dt, int entrynum, u32 low, u32 high) | |
356 | { | |
357 | native_write_dt_entry(dt, entrynum, low, high); | |
358 | } | |
359 | ||
360 | static fastcall void native_write_idt_entry(void *dt, int entrynum, u32 low, u32 high) | |
361 | { | |
362 | native_write_dt_entry(dt, entrynum, low, high); | |
363 | } | |
364 | ||
365 | static fastcall void native_load_esp0(struct tss_struct *tss, | |
366 | struct thread_struct *thread) | |
367 | { | |
368 | tss->esp0 = thread->esp0; | |
369 | ||
370 | /* This can only happen when SEP is enabled, no need to test "SEP"arately */ | |
371 | if (unlikely(tss->ss1 != thread->sysenter_cs)) { | |
372 | tss->ss1 = thread->sysenter_cs; | |
373 | wrmsr(MSR_IA32_SYSENTER_CS, thread->sysenter_cs, 0); | |
374 | } | |
375 | } | |
376 | ||
377 | static fastcall void native_io_delay(void) | |
378 | { | |
379 | asm volatile("outb %al,$0x80"); | |
380 | } | |
381 | ||
382 | /* These are in entry.S */ | |
383 | extern fastcall void native_iret(void); | |
384 | extern fastcall void native_irq_enable_sysexit(void); | |
385 | ||
386 | static int __init print_banner(void) | |
387 | { | |
388 | paravirt_ops.banner(); | |
389 | return 0; | |
390 | } | |
391 | core_initcall(print_banner); | |
392 | ||
c9ccf30d RR |
393 | /* We simply declare start_kernel to be the paravirt probe of last resort. */ |
394 | paravirt_probe(start_kernel); | |
395 | ||
d3561b7f RR |
396 | struct paravirt_ops paravirt_ops = { |
397 | .name = "bare hardware", | |
398 | .paravirt_enabled = 0, | |
399 | .kernel_rpl = 0, | |
400 | ||
139ec7c4 | 401 | .patch = native_patch, |
d3561b7f RR |
402 | .banner = default_banner, |
403 | .arch_setup = native_nop, | |
404 | .memory_setup = machine_specific_memory_setup, | |
405 | .get_wallclock = native_get_wallclock, | |
406 | .set_wallclock = native_set_wallclock, | |
407 | .time_init = time_init_hook, | |
408 | .init_IRQ = native_init_IRQ, | |
409 | ||
410 | .cpuid = native_cpuid, | |
411 | .get_debugreg = native_get_debugreg, | |
412 | .set_debugreg = native_set_debugreg, | |
413 | .clts = native_clts, | |
414 | .read_cr0 = native_read_cr0, | |
415 | .write_cr0 = native_write_cr0, | |
416 | .read_cr2 = native_read_cr2, | |
417 | .write_cr2 = native_write_cr2, | |
418 | .read_cr3 = native_read_cr3, | |
419 | .write_cr3 = native_write_cr3, | |
420 | .read_cr4 = native_read_cr4, | |
421 | .read_cr4_safe = native_read_cr4_safe, | |
422 | .write_cr4 = native_write_cr4, | |
423 | .save_fl = native_save_fl, | |
424 | .restore_fl = native_restore_fl, | |
425 | .irq_disable = native_irq_disable, | |
426 | .irq_enable = native_irq_enable, | |
427 | .safe_halt = native_safe_halt, | |
428 | .halt = native_halt, | |
429 | .wbinvd = native_wbinvd, | |
430 | .read_msr = native_read_msr, | |
431 | .write_msr = native_write_msr, | |
432 | .read_tsc = native_read_tsc, | |
433 | .read_pmc = native_read_pmc, | |
434 | .load_tr_desc = native_load_tr_desc, | |
435 | .set_ldt = native_set_ldt, | |
436 | .load_gdt = native_load_gdt, | |
437 | .load_idt = native_load_idt, | |
438 | .store_gdt = native_store_gdt, | |
439 | .store_idt = native_store_idt, | |
440 | .store_tr = native_store_tr, | |
441 | .load_tls = native_load_tls, | |
442 | .write_ldt_entry = native_write_ldt_entry, | |
443 | .write_gdt_entry = native_write_gdt_entry, | |
444 | .write_idt_entry = native_write_idt_entry, | |
445 | .load_esp0 = native_load_esp0, | |
446 | ||
447 | .set_iopl_mask = native_set_iopl_mask, | |
448 | .io_delay = native_io_delay, | |
449 | .const_udelay = __const_udelay, | |
450 | ||
13623d79 RR |
451 | #ifdef CONFIG_X86_LOCAL_APIC |
452 | .apic_write = native_apic_write, | |
453 | .apic_write_atomic = native_apic_write_atomic, | |
454 | .apic_read = native_apic_read, | |
455 | #endif | |
456 | ||
d3561b7f RR |
457 | .irq_enable_sysexit = native_irq_enable_sysexit, |
458 | .iret = native_iret, | |
459 | }; | |
460 | EXPORT_SYMBOL(paravirt_ops); |