| 1 | /* Paravirtualization interfaces |
| 2 | Copyright (C) 2006 Rusty Russell IBM Corporation |
| 3 | |
| 4 | This program is free software; you can redistribute it and/or modify |
| 5 | it under the terms of the GNU General Public License as published by |
| 6 | the Free Software Foundation; either version 2 of the License, or |
| 7 | (at your option) any later version. |
| 8 | |
| 9 | This program is distributed in the hope that it will be useful, |
| 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 12 | GNU General Public License for more details. |
| 13 | |
| 14 | You should have received a copy of the GNU General Public License |
| 15 | along with this program; if not, write to the Free Software |
| 16 | Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA |
| 17 | */ |
| 18 | #include <linux/errno.h> |
| 19 | #include <linux/module.h> |
| 20 | #include <linux/efi.h> |
| 21 | #include <linux/bcd.h> |
| 22 | #include <linux/start_kernel.h> |
| 23 | |
| 24 | #include <asm/bug.h> |
| 25 | #include <asm/paravirt.h> |
| 26 | #include <asm/desc.h> |
| 27 | #include <asm/setup.h> |
| 28 | #include <asm/arch_hooks.h> |
| 29 | #include <asm/time.h> |
| 30 | #include <asm/irq.h> |
| 31 | #include <asm/delay.h> |
| 32 | |
| 33 | /* nop stub */ |
| 34 | static void native_nop(void) |
| 35 | { |
| 36 | } |
| 37 | |
| 38 | static void __init default_banner(void) |
| 39 | { |
| 40 | printk(KERN_INFO "Booting paravirtualized kernel on %s\n", |
| 41 | paravirt_ops.name); |
| 42 | } |
| 43 | |
| 44 | char *memory_setup(void) |
| 45 | { |
| 46 | return paravirt_ops.memory_setup(); |
| 47 | } |
| 48 | |
| 49 | /* Simple instruction patching code. */ |
| 50 | #define DEF_NATIVE(name, code) \ |
| 51 | extern const char start_##name[], end_##name[]; \ |
| 52 | asm("start_" #name ": " code "; end_" #name ":") |
| 53 | DEF_NATIVE(cli, "cli"); |
| 54 | DEF_NATIVE(sti, "sti"); |
| 55 | DEF_NATIVE(popf, "push %eax; popf"); |
| 56 | DEF_NATIVE(pushf, "pushf; pop %eax"); |
| 57 | DEF_NATIVE(pushf_cli, "pushf; pop %eax; cli"); |
| 58 | DEF_NATIVE(iret, "iret"); |
| 59 | DEF_NATIVE(sti_sysexit, "sti; sysexit"); |
| 60 | |
| 61 | static const struct native_insns |
| 62 | { |
| 63 | const char *start, *end; |
| 64 | } native_insns[] = { |
| 65 | [PARAVIRT_IRQ_DISABLE] = { start_cli, end_cli }, |
| 66 | [PARAVIRT_IRQ_ENABLE] = { start_sti, end_sti }, |
| 67 | [PARAVIRT_RESTORE_FLAGS] = { start_popf, end_popf }, |
| 68 | [PARAVIRT_SAVE_FLAGS] = { start_pushf, end_pushf }, |
| 69 | [PARAVIRT_SAVE_FLAGS_IRQ_DISABLE] = { start_pushf_cli, end_pushf_cli }, |
| 70 | [PARAVIRT_INTERRUPT_RETURN] = { start_iret, end_iret }, |
| 71 | [PARAVIRT_STI_SYSEXIT] = { start_sti_sysexit, end_sti_sysexit }, |
| 72 | }; |
| 73 | |
| 74 | static unsigned native_patch(u8 type, u16 clobbers, void *insns, unsigned len) |
| 75 | { |
| 76 | unsigned int insn_len; |
| 77 | |
| 78 | /* Don't touch it if we don't have a replacement */ |
| 79 | if (type >= ARRAY_SIZE(native_insns) || !native_insns[type].start) |
| 80 | return len; |
| 81 | |
| 82 | insn_len = native_insns[type].end - native_insns[type].start; |
| 83 | |
| 84 | /* Similarly if we can't fit replacement. */ |
| 85 | if (len < insn_len) |
| 86 | return len; |
| 87 | |
| 88 | memcpy(insns, native_insns[type].start, insn_len); |
| 89 | return insn_len; |
| 90 | } |
| 91 | |
| 92 | static fastcall unsigned long native_get_debugreg(int regno) |
| 93 | { |
| 94 | unsigned long val = 0; /* Damn you, gcc! */ |
| 95 | |
| 96 | switch (regno) { |
| 97 | case 0: |
| 98 | asm("movl %%db0, %0" :"=r" (val)); break; |
| 99 | case 1: |
| 100 | asm("movl %%db1, %0" :"=r" (val)); break; |
| 101 | case 2: |
| 102 | asm("movl %%db2, %0" :"=r" (val)); break; |
| 103 | case 3: |
| 104 | asm("movl %%db3, %0" :"=r" (val)); break; |
| 105 | case 6: |
| 106 | asm("movl %%db6, %0" :"=r" (val)); break; |
| 107 | case 7: |
| 108 | asm("movl %%db7, %0" :"=r" (val)); break; |
| 109 | default: |
| 110 | BUG(); |
| 111 | } |
| 112 | return val; |
| 113 | } |
| 114 | |
| 115 | static fastcall void native_set_debugreg(int regno, unsigned long value) |
| 116 | { |
| 117 | switch (regno) { |
| 118 | case 0: |
| 119 | asm("movl %0,%%db0" : /* no output */ :"r" (value)); |
| 120 | break; |
| 121 | case 1: |
| 122 | asm("movl %0,%%db1" : /* no output */ :"r" (value)); |
| 123 | break; |
| 124 | case 2: |
| 125 | asm("movl %0,%%db2" : /* no output */ :"r" (value)); |
| 126 | break; |
| 127 | case 3: |
| 128 | asm("movl %0,%%db3" : /* no output */ :"r" (value)); |
| 129 | break; |
| 130 | case 6: |
| 131 | asm("movl %0,%%db6" : /* no output */ :"r" (value)); |
| 132 | break; |
| 133 | case 7: |
| 134 | asm("movl %0,%%db7" : /* no output */ :"r" (value)); |
| 135 | break; |
| 136 | default: |
| 137 | BUG(); |
| 138 | } |
| 139 | } |
| 140 | |
| 141 | void init_IRQ(void) |
| 142 | { |
| 143 | paravirt_ops.init_IRQ(); |
| 144 | } |
| 145 | |
| 146 | static fastcall void native_clts(void) |
| 147 | { |
| 148 | asm volatile ("clts"); |
| 149 | } |
| 150 | |
| 151 | static fastcall unsigned long native_read_cr0(void) |
| 152 | { |
| 153 | unsigned long val; |
| 154 | asm volatile("movl %%cr0,%0\n\t" :"=r" (val)); |
| 155 | return val; |
| 156 | } |
| 157 | |
| 158 | static fastcall void native_write_cr0(unsigned long val) |
| 159 | { |
| 160 | asm volatile("movl %0,%%cr0": :"r" (val)); |
| 161 | } |
| 162 | |
| 163 | static fastcall unsigned long native_read_cr2(void) |
| 164 | { |
| 165 | unsigned long val; |
| 166 | asm volatile("movl %%cr2,%0\n\t" :"=r" (val)); |
| 167 | return val; |
| 168 | } |
| 169 | |
| 170 | static fastcall void native_write_cr2(unsigned long val) |
| 171 | { |
| 172 | asm volatile("movl %0,%%cr2": :"r" (val)); |
| 173 | } |
| 174 | |
| 175 | static fastcall unsigned long native_read_cr3(void) |
| 176 | { |
| 177 | unsigned long val; |
| 178 | asm volatile("movl %%cr3,%0\n\t" :"=r" (val)); |
| 179 | return val; |
| 180 | } |
| 181 | |
| 182 | static fastcall void native_write_cr3(unsigned long val) |
| 183 | { |
| 184 | asm volatile("movl %0,%%cr3": :"r" (val)); |
| 185 | } |
| 186 | |
| 187 | static fastcall unsigned long native_read_cr4(void) |
| 188 | { |
| 189 | unsigned long val; |
| 190 | asm volatile("movl %%cr4,%0\n\t" :"=r" (val)); |
| 191 | return val; |
| 192 | } |
| 193 | |
| 194 | static fastcall unsigned long native_read_cr4_safe(void) |
| 195 | { |
| 196 | unsigned long val; |
| 197 | /* This could fault if %cr4 does not exist */ |
| 198 | asm("1: movl %%cr4, %0 \n" |
| 199 | "2: \n" |
| 200 | ".section __ex_table,\"a\" \n" |
| 201 | ".long 1b,2b \n" |
| 202 | ".previous \n" |
| 203 | : "=r" (val): "0" (0)); |
| 204 | return val; |
| 205 | } |
| 206 | |
| 207 | static fastcall void native_write_cr4(unsigned long val) |
| 208 | { |
| 209 | asm volatile("movl %0,%%cr4": :"r" (val)); |
| 210 | } |
| 211 | |
| 212 | static fastcall unsigned long native_save_fl(void) |
| 213 | { |
| 214 | unsigned long f; |
| 215 | asm volatile("pushfl ; popl %0":"=g" (f): /* no input */); |
| 216 | return f; |
| 217 | } |
| 218 | |
| 219 | static fastcall void native_restore_fl(unsigned long f) |
| 220 | { |
| 221 | asm volatile("pushl %0 ; popfl": /* no output */ |
| 222 | :"g" (f) |
| 223 | :"memory", "cc"); |
| 224 | } |
| 225 | |
| 226 | static fastcall void native_irq_disable(void) |
| 227 | { |
| 228 | asm volatile("cli": : :"memory"); |
| 229 | } |
| 230 | |
| 231 | static fastcall void native_irq_enable(void) |
| 232 | { |
| 233 | asm volatile("sti": : :"memory"); |
| 234 | } |
| 235 | |
| 236 | static fastcall void native_safe_halt(void) |
| 237 | { |
| 238 | asm volatile("sti; hlt": : :"memory"); |
| 239 | } |
| 240 | |
| 241 | static fastcall void native_halt(void) |
| 242 | { |
| 243 | asm volatile("hlt": : :"memory"); |
| 244 | } |
| 245 | |
| 246 | static fastcall void native_wbinvd(void) |
| 247 | { |
| 248 | asm volatile("wbinvd": : :"memory"); |
| 249 | } |
| 250 | |
| 251 | static fastcall unsigned long long native_read_msr(unsigned int msr, int *err) |
| 252 | { |
| 253 | unsigned long long val; |
| 254 | |
| 255 | asm volatile("2: rdmsr ; xorl %0,%0\n" |
| 256 | "1:\n\t" |
| 257 | ".section .fixup,\"ax\"\n\t" |
| 258 | "3: movl %3,%0 ; jmp 1b\n\t" |
| 259 | ".previous\n\t" |
| 260 | ".section __ex_table,\"a\"\n" |
| 261 | " .align 4\n\t" |
| 262 | " .long 2b,3b\n\t" |
| 263 | ".previous" |
| 264 | : "=r" (*err), "=A" (val) |
| 265 | : "c" (msr), "i" (-EFAULT)); |
| 266 | |
| 267 | return val; |
| 268 | } |
| 269 | |
| 270 | static fastcall int native_write_msr(unsigned int msr, unsigned long long val) |
| 271 | { |
| 272 | int err; |
| 273 | asm volatile("2: wrmsr ; xorl %0,%0\n" |
| 274 | "1:\n\t" |
| 275 | ".section .fixup,\"ax\"\n\t" |
| 276 | "3: movl %4,%0 ; jmp 1b\n\t" |
| 277 | ".previous\n\t" |
| 278 | ".section __ex_table,\"a\"\n" |
| 279 | " .align 4\n\t" |
| 280 | " .long 2b,3b\n\t" |
| 281 | ".previous" |
| 282 | : "=a" (err) |
| 283 | : "c" (msr), "0" ((u32)val), "d" ((u32)(val>>32)), |
| 284 | "i" (-EFAULT)); |
| 285 | return err; |
| 286 | } |
| 287 | |
| 288 | static fastcall unsigned long long native_read_tsc(void) |
| 289 | { |
| 290 | unsigned long long val; |
| 291 | asm volatile("rdtsc" : "=A" (val)); |
| 292 | return val; |
| 293 | } |
| 294 | |
| 295 | static fastcall unsigned long long native_read_pmc(void) |
| 296 | { |
| 297 | unsigned long long val; |
| 298 | asm volatile("rdpmc" : "=A" (val)); |
| 299 | return val; |
| 300 | } |
| 301 | |
| 302 | static fastcall void native_load_tr_desc(void) |
| 303 | { |
| 304 | asm volatile("ltr %w0"::"q" (GDT_ENTRY_TSS*8)); |
| 305 | } |
| 306 | |
| 307 | static fastcall void native_load_gdt(const struct Xgt_desc_struct *dtr) |
| 308 | { |
| 309 | asm volatile("lgdt %0"::"m" (*dtr)); |
| 310 | } |
| 311 | |
| 312 | static fastcall void native_load_idt(const struct Xgt_desc_struct *dtr) |
| 313 | { |
| 314 | asm volatile("lidt %0"::"m" (*dtr)); |
| 315 | } |
| 316 | |
| 317 | static fastcall void native_store_gdt(struct Xgt_desc_struct *dtr) |
| 318 | { |
| 319 | asm ("sgdt %0":"=m" (*dtr)); |
| 320 | } |
| 321 | |
| 322 | static fastcall void native_store_idt(struct Xgt_desc_struct *dtr) |
| 323 | { |
| 324 | asm ("sidt %0":"=m" (*dtr)); |
| 325 | } |
| 326 | |
| 327 | static fastcall unsigned long native_store_tr(void) |
| 328 | { |
| 329 | unsigned long tr; |
| 330 | asm ("str %0":"=r" (tr)); |
| 331 | return tr; |
| 332 | } |
| 333 | |
| 334 | static fastcall void native_load_tls(struct thread_struct *t, unsigned int cpu) |
| 335 | { |
| 336 | #define C(i) get_cpu_gdt_table(cpu)[GDT_ENTRY_TLS_MIN + i] = t->tls_array[i] |
| 337 | C(0); C(1); C(2); |
| 338 | #undef C |
| 339 | } |
| 340 | |
| 341 | static inline void native_write_dt_entry(void *dt, int entry, u32 entry_low, u32 entry_high) |
| 342 | { |
| 343 | u32 *lp = (u32 *)((char *)dt + entry*8); |
| 344 | lp[0] = entry_low; |
| 345 | lp[1] = entry_high; |
| 346 | } |
| 347 | |
| 348 | static fastcall void native_write_ldt_entry(void *dt, int entrynum, u32 low, u32 high) |
| 349 | { |
| 350 | native_write_dt_entry(dt, entrynum, low, high); |
| 351 | } |
| 352 | |
| 353 | static fastcall void native_write_gdt_entry(void *dt, int entrynum, u32 low, u32 high) |
| 354 | { |
| 355 | native_write_dt_entry(dt, entrynum, low, high); |
| 356 | } |
| 357 | |
| 358 | static fastcall void native_write_idt_entry(void *dt, int entrynum, u32 low, u32 high) |
| 359 | { |
| 360 | native_write_dt_entry(dt, entrynum, low, high); |
| 361 | } |
| 362 | |
| 363 | static fastcall void native_load_esp0(struct tss_struct *tss, |
| 364 | struct thread_struct *thread) |
| 365 | { |
| 366 | tss->esp0 = thread->esp0; |
| 367 | |
| 368 | /* This can only happen when SEP is enabled, no need to test "SEP"arately */ |
| 369 | if (unlikely(tss->ss1 != thread->sysenter_cs)) { |
| 370 | tss->ss1 = thread->sysenter_cs; |
| 371 | wrmsr(MSR_IA32_SYSENTER_CS, thread->sysenter_cs, 0); |
| 372 | } |
| 373 | } |
| 374 | |
| 375 | static fastcall void native_io_delay(void) |
| 376 | { |
| 377 | asm volatile("outb %al,$0x80"); |
| 378 | } |
| 379 | |
| 380 | /* These are in entry.S */ |
| 381 | extern fastcall void native_iret(void); |
| 382 | extern fastcall void native_irq_enable_sysexit(void); |
| 383 | |
| 384 | static int __init print_banner(void) |
| 385 | { |
| 386 | paravirt_ops.banner(); |
| 387 | return 0; |
| 388 | } |
| 389 | core_initcall(print_banner); |
| 390 | |
| 391 | /* We simply declare start_kernel to be the paravirt probe of last resort. */ |
| 392 | paravirt_probe(start_kernel); |
| 393 | |
| 394 | struct paravirt_ops paravirt_ops = { |
| 395 | .name = "bare hardware", |
| 396 | .paravirt_enabled = 0, |
| 397 | .kernel_rpl = 0, |
| 398 | |
| 399 | .patch = native_patch, |
| 400 | .banner = default_banner, |
| 401 | .arch_setup = native_nop, |
| 402 | .memory_setup = machine_specific_memory_setup, |
| 403 | .get_wallclock = native_get_wallclock, |
| 404 | .set_wallclock = native_set_wallclock, |
| 405 | .time_init = time_init_hook, |
| 406 | .init_IRQ = native_init_IRQ, |
| 407 | |
| 408 | .cpuid = native_cpuid, |
| 409 | .get_debugreg = native_get_debugreg, |
| 410 | .set_debugreg = native_set_debugreg, |
| 411 | .clts = native_clts, |
| 412 | .read_cr0 = native_read_cr0, |
| 413 | .write_cr0 = native_write_cr0, |
| 414 | .read_cr2 = native_read_cr2, |
| 415 | .write_cr2 = native_write_cr2, |
| 416 | .read_cr3 = native_read_cr3, |
| 417 | .write_cr3 = native_write_cr3, |
| 418 | .read_cr4 = native_read_cr4, |
| 419 | .read_cr4_safe = native_read_cr4_safe, |
| 420 | .write_cr4 = native_write_cr4, |
| 421 | .save_fl = native_save_fl, |
| 422 | .restore_fl = native_restore_fl, |
| 423 | .irq_disable = native_irq_disable, |
| 424 | .irq_enable = native_irq_enable, |
| 425 | .safe_halt = native_safe_halt, |
| 426 | .halt = native_halt, |
| 427 | .wbinvd = native_wbinvd, |
| 428 | .read_msr = native_read_msr, |
| 429 | .write_msr = native_write_msr, |
| 430 | .read_tsc = native_read_tsc, |
| 431 | .read_pmc = native_read_pmc, |
| 432 | .load_tr_desc = native_load_tr_desc, |
| 433 | .set_ldt = native_set_ldt, |
| 434 | .load_gdt = native_load_gdt, |
| 435 | .load_idt = native_load_idt, |
| 436 | .store_gdt = native_store_gdt, |
| 437 | .store_idt = native_store_idt, |
| 438 | .store_tr = native_store_tr, |
| 439 | .load_tls = native_load_tls, |
| 440 | .write_ldt_entry = native_write_ldt_entry, |
| 441 | .write_gdt_entry = native_write_gdt_entry, |
| 442 | .write_idt_entry = native_write_idt_entry, |
| 443 | .load_esp0 = native_load_esp0, |
| 444 | |
| 445 | .set_iopl_mask = native_set_iopl_mask, |
| 446 | .io_delay = native_io_delay, |
| 447 | .const_udelay = __const_udelay, |
| 448 | |
| 449 | .irq_enable_sysexit = native_irq_enable_sysexit, |
| 450 | .iret = native_iret, |
| 451 | }; |
| 452 | EXPORT_SYMBOL(paravirt_ops); |