projects / deliverable / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
arm/xen: fix SMP guests boot
[deliverable/linux.git] / arch / powerpc / kvm / book3s_hv_rmhandlers.S
1 /*
2 * This program is free software; you can redistribute it and/or modify
3 * it under the terms of the GNU General Public License, version 2, as
4 * published by the Free Software Foundation.
5 *
6 * This program is distributed in the hope that it will be useful,
7 * but WITHOUT ANY WARRANTY; without even the implied warranty of
8 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
9 * GNU General Public License for more details.
10 *
11 * Copyright 2011 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
12 *
13 * Derived from book3s_rmhandlers.S and other files, which are:
14 *
15 * Copyright SUSE Linux Products GmbH 2009
16 *
17 * Authors: Alexander Graf <agraf@suse.de>
18 */
19
20 #include <asm/ppc_asm.h>
21 #include <asm/kvm_asm.h>
22 #include <asm/reg.h>
23 #include <asm/mmu.h>
24 #include <asm/page.h>
25 #include <asm/ptrace.h>
26 #include <asm/hvcall.h>
27 #include <asm/asm-offsets.h>
28 #include <asm/exception-64s.h>
29 #include <asm/kvm_book3s_asm.h>
30 #include <asm/book3s/64/mmu-hash.h>
31 #include <asm/tm.h>
32 #include <asm/opal.h>
33
34 #define VCPU_GPRS_TM(reg) (((reg) * ULONG_SIZE) + VCPU_GPR_TM)
35
36 /* Values in HSTATE_NAPPING(r13) */
37 #define NAPPING_CEDE 1
38 #define NAPPING_NOVCPU 2
39
40 /*
41 * Call kvmppc_hv_entry in real mode.
42 * Must be called with interrupts hard-disabled.
43 *
44 * Input Registers:
45 *
46 * LR = return address to continue at after eventually re-enabling MMU
47 */
48 _GLOBAL_TOC(kvmppc_hv_entry_trampoline)
49 mflr r0
50 std r0, PPC_LR_STKOFF(r1)
51 stdu r1, -112(r1)
52 mfmsr r10
53 LOAD_REG_ADDR(r5, kvmppc_call_hv_entry)
54 li r0,MSR_RI
55 andc r0,r10,r0
56 li r6,MSR_IR | MSR_DR
57 andc r6,r10,r6
58 mtmsrd r0,1 /* clear RI in MSR */
59 mtsrr0 r5
60 mtsrr1 r6
61 RFI
62
63 kvmppc_call_hv_entry:
64 ld r4, HSTATE_KVM_VCPU(r13)
65 bl kvmppc_hv_entry
66
67 /* Back from guest - restore host state and return to caller */
68
69 BEGIN_FTR_SECTION
70 /* Restore host DABR and DABRX */
71 ld r5,HSTATE_DABR(r13)
72 li r6,7
73 mtspr SPRN_DABR,r5
74 mtspr SPRN_DABRX,r6
75 END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
76
77 /* Restore SPRG3 */
78 ld r3,PACA_SPRG_VDSO(r13)
79 mtspr SPRN_SPRG_VDSO_WRITE,r3
80
81 /* Reload the host's PMU registers */
82 ld r3, PACALPPACAPTR(r13) /* is the host using the PMU? */
83 lbz r4, LPPACA_PMCINUSE(r3)
84 cmpwi r4, 0
85 beq 23f /* skip if not */
86 BEGIN_FTR_SECTION
87 ld r3, HSTATE_MMCR0(r13)
88 andi. r4, r3, MMCR0_PMAO_SYNC | MMCR0_PMAO
89 cmpwi r4, MMCR0_PMAO
90 beql kvmppc_fix_pmao
91 END_FTR_SECTION_IFSET(CPU_FTR_PMAO_BUG)
92 lwz r3, HSTATE_PMC1(r13)
93 lwz r4, HSTATE_PMC2(r13)
94 lwz r5, HSTATE_PMC3(r13)
95 lwz r6, HSTATE_PMC4(r13)
96 lwz r8, HSTATE_PMC5(r13)
97 lwz r9, HSTATE_PMC6(r13)
98 mtspr SPRN_PMC1, r3
99 mtspr SPRN_PMC2, r4
100 mtspr SPRN_PMC3, r5
101 mtspr SPRN_PMC4, r6
102 mtspr SPRN_PMC5, r8
103 mtspr SPRN_PMC6, r9
104 ld r3, HSTATE_MMCR0(r13)
105 ld r4, HSTATE_MMCR1(r13)
106 ld r5, HSTATE_MMCRA(r13)
107 ld r6, HSTATE_SIAR(r13)
108 ld r7, HSTATE_SDAR(r13)
109 mtspr SPRN_MMCR1, r4
110 mtspr SPRN_MMCRA, r5
111 mtspr SPRN_SIAR, r6
112 mtspr SPRN_SDAR, r7
113 BEGIN_FTR_SECTION
114 ld r8, HSTATE_MMCR2(r13)
115 ld r9, HSTATE_SIER(r13)
116 mtspr SPRN_MMCR2, r8
117 mtspr SPRN_SIER, r9
118 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
119 mtspr SPRN_MMCR0, r3
120 isync
121 23:
122
123 /*
124 * Reload DEC. HDEC interrupts were disabled when
125 * we reloaded the host's LPCR value.
126 */
127 ld r3, HSTATE_DECEXP(r13)
128 mftb r4
129 subf r4, r4, r3
130 mtspr SPRN_DEC, r4
131
132 /* hwthread_req may have got set by cede or no vcpu, so clear it */
133 li r0, 0
134 stb r0, HSTATE_HWTHREAD_REQ(r13)
135
136 /*
137 * For external and machine check interrupts, we need
138 * to call the Linux handler to process the interrupt.
139 * We do that by jumping to absolute address 0x500 for
140 * external interrupts, or the machine_check_fwnmi label
141 * for machine checks (since firmware might have patched
142 * the vector area at 0x200). The [h]rfid at the end of the
143 * handler will return to the book3s_hv_interrupts.S code.
144 * For other interrupts we do the rfid to get back
145 * to the book3s_hv_interrupts.S code here.
146 */
147 ld r8, 112+PPC_LR_STKOFF(r1)
148 addi r1, r1, 112
149 ld r7, HSTATE_HOST_MSR(r13)
150
151 cmpwi cr1, r12, BOOK3S_INTERRUPT_MACHINE_CHECK
152 cmpwi r12, BOOK3S_INTERRUPT_EXTERNAL
153 beq 11f
154 cmpwi r12, BOOK3S_INTERRUPT_H_DOORBELL
155 beq 15f /* Invoke the H_DOORBELL handler */
156 cmpwi cr2, r12, BOOK3S_INTERRUPT_HMI
157 beq cr2, 14f /* HMI check */
158
159 /* RFI into the highmem handler, or branch to interrupt handler */
160 mfmsr r6
161 li r0, MSR_RI
162 andc r6, r6, r0
163 mtmsrd r6, 1 /* Clear RI in MSR */
164 mtsrr0 r8
165 mtsrr1 r7
166 beq cr1, 13f /* machine check */
167 RFI
168
169 /* On POWER7, we have external interrupts set to use HSRR0/1 */
170 11: mtspr SPRN_HSRR0, r8
171 mtspr SPRN_HSRR1, r7
172 ba 0x500
173
174 13: b machine_check_fwnmi
175
176 14: mtspr SPRN_HSRR0, r8
177 mtspr SPRN_HSRR1, r7
178 b hmi_exception_after_realmode
179
180 15: mtspr SPRN_HSRR0, r8
181 mtspr SPRN_HSRR1, r7
182 ba 0xe80
183
184 kvmppc_primary_no_guest:
185 /* We handle this much like a ceded vcpu */
186 /* put the HDEC into the DEC, since HDEC interrupts don't wake us */
187 mfspr r3, SPRN_HDEC
188 mtspr SPRN_DEC, r3
189 /*
190 * Make sure the primary has finished the MMU switch.
191 * We should never get here on a secondary thread, but
192 * check it for robustness' sake.
193 */
194 ld r5, HSTATE_KVM_VCORE(r13)
195 65: lbz r0, VCORE_IN_GUEST(r5)
196 cmpwi r0, 0
197 beq 65b
198 /* Set LPCR. */
199 ld r8,VCORE_LPCR(r5)
200 mtspr SPRN_LPCR,r8
201 isync
202 /* set our bit in napping_threads */
203 ld r5, HSTATE_KVM_VCORE(r13)
204 lbz r7, HSTATE_PTID(r13)
205 li r0, 1
206 sld r0, r0, r7
207 addi r6, r5, VCORE_NAPPING_THREADS
208 1: lwarx r3, 0, r6
209 or r3, r3, r0
210 stwcx. r3, 0, r6
211 bne 1b
212 /* order napping_threads update vs testing entry_exit_map */
213 isync
214 li r12, 0
215 lwz r7, VCORE_ENTRY_EXIT(r5)
216 cmpwi r7, 0x100
217 bge kvm_novcpu_exit /* another thread already exiting */
218 li r3, NAPPING_NOVCPU
219 stb r3, HSTATE_NAPPING(r13)
220
221 li r3, 0 /* Don't wake on privileged (OS) doorbell */
222 b kvm_do_nap
223
224 kvm_novcpu_wakeup:
225 ld r1, HSTATE_HOST_R1(r13)
226 ld r5, HSTATE_KVM_VCORE(r13)
227 li r0, 0
228 stb r0, HSTATE_NAPPING(r13)
229
230 /* check the wake reason */
231 bl kvmppc_check_wake_reason
232
233 /* see if any other thread is already exiting */
234 lwz r0, VCORE_ENTRY_EXIT(r5)
235 cmpwi r0, 0x100
236 bge kvm_novcpu_exit
237
238 /* clear our bit in napping_threads */
239 lbz r7, HSTATE_PTID(r13)
240 li r0, 1
241 sld r0, r0, r7
242 addi r6, r5, VCORE_NAPPING_THREADS
243 4: lwarx r7, 0, r6
244 andc r7, r7, r0
245 stwcx. r7, 0, r6
246 bne 4b
247
248 /* See if the wake reason means we need to exit */
249 cmpdi r3, 0
250 bge kvm_novcpu_exit
251
252 /* See if our timeslice has expired (HDEC is negative) */
253 mfspr r0, SPRN_HDEC
254 li r12, BOOK3S_INTERRUPT_HV_DECREMENTER
255 cmpwi r0, 0
256 blt kvm_novcpu_exit
257
258 /* Got an IPI but other vcpus aren't yet exiting, must be a latecomer */
259 ld r4, HSTATE_KVM_VCPU(r13)
260 cmpdi r4, 0
261 beq kvmppc_primary_no_guest
262
263 #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
264 addi r3, r4, VCPU_TB_RMENTRY
265 bl kvmhv_start_timing
266 #endif
267 b kvmppc_got_guest
268
269 kvm_novcpu_exit:
270 #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
271 ld r4, HSTATE_KVM_VCPU(r13)
272 cmpdi r4, 0
273 beq 13f
274 addi r3, r4, VCPU_TB_RMEXIT
275 bl kvmhv_accumulate_time
276 #endif
277 13: mr r3, r12
278 stw r12, 112-4(r1)
279 bl kvmhv_commence_exit
280 nop
281 lwz r12, 112-4(r1)
282 b kvmhv_switch_to_host
283
284 /*
285 * We come in here when wakened from nap mode.
286 * Relocation is off and most register values are lost.
287 * r13 points to the PACA.
288 */
289 .globl kvm_start_guest
290 kvm_start_guest:
291
292 /* Set runlatch bit the minute you wake up from nap */
293 mfspr r0, SPRN_CTRLF
294 ori r0, r0, 1
295 mtspr SPRN_CTRLT, r0
296
297 ld r2,PACATOC(r13)
298
299 li r0,KVM_HWTHREAD_IN_KVM
300 stb r0,HSTATE_HWTHREAD_STATE(r13)
301
302 /* NV GPR values from power7_idle() will no longer be valid */
303 li r0,1
304 stb r0,PACA_NAPSTATELOST(r13)
305
306 /* were we napping due to cede? */
307 lbz r0,HSTATE_NAPPING(r13)
308 cmpwi r0,NAPPING_CEDE
309 beq kvm_end_cede
310 cmpwi r0,NAPPING_NOVCPU
311 beq kvm_novcpu_wakeup
312
313 ld r1,PACAEMERGSP(r13)
314 subi r1,r1,STACK_FRAME_OVERHEAD
315
316 /*
317 * We weren't napping due to cede, so this must be a secondary
318 * thread being woken up to run a guest, or being woken up due
319 * to a stray IPI. (Or due to some machine check or hypervisor
320 * maintenance interrupt while the core is in KVM.)
321 */
322
323 /* Check the wake reason in SRR1 to see why we got here */
324 bl kvmppc_check_wake_reason
325 cmpdi r3, 0
326 bge kvm_no_guest
327
328 /* get vcore pointer, NULL if we have nothing to run */
329 ld r5,HSTATE_KVM_VCORE(r13)
330 cmpdi r5,0
331 /* if we have no vcore to run, go back to sleep */
332 beq kvm_no_guest
333
334 kvm_secondary_got_guest:
335
336 /* Set HSTATE_DSCR(r13) to something sensible */
337 ld r6, PACA_DSCR_DEFAULT(r13)
338 std r6, HSTATE_DSCR(r13)
339
340 /* On thread 0 of a subcore, set HDEC to max */
341 lbz r4, HSTATE_PTID(r13)
342 cmpwi r4, 0
343 bne 63f
344 lis r6, 0x7fff
345 ori r6, r6, 0xffff
346 mtspr SPRN_HDEC, r6
347 /* and set per-LPAR registers, if doing dynamic micro-threading */
348 ld r6, HSTATE_SPLIT_MODE(r13)
349 cmpdi r6, 0
350 beq 63f
351 ld r0, KVM_SPLIT_RPR(r6)
352 mtspr SPRN_RPR, r0
353 ld r0, KVM_SPLIT_PMMAR(r6)
354 mtspr SPRN_PMMAR, r0
355 ld r0, KVM_SPLIT_LDBAR(r6)
356 mtspr SPRN_LDBAR, r0
357 isync
358 63:
359 /* Order load of vcpu after load of vcore */
360 lwsync
361 ld r4, HSTATE_KVM_VCPU(r13)
362 bl kvmppc_hv_entry
363
364 /* Back from the guest, go back to nap */
365 /* Clear our vcpu and vcore pointers so we don't come back in early */
366 li r0, 0
367 std r0, HSTATE_KVM_VCPU(r13)
368 /*
369 * Once we clear HSTATE_KVM_VCORE(r13), the code in
370 * kvmppc_run_core() is going to assume that all our vcpu
371 * state is visible in memory. This lwsync makes sure
372 * that that is true.
373 */
374 lwsync
375 std r0, HSTATE_KVM_VCORE(r13)
376
377 /*
378 * All secondaries exiting guest will fall through this path.
379 * Before proceeding, just check for HMI interrupt and
380 * invoke opal hmi handler. By now we are sure that the
381 * primary thread on this core/subcore has already made partition
382 * switch/TB resync and we are good to call opal hmi handler.
383 */
384 cmpwi r12, BOOK3S_INTERRUPT_HMI
385 bne kvm_no_guest
386
387 li r3,0 /* NULL argument */
388 bl hmi_exception_realmode
389 /*
390 * At this point we have finished executing in the guest.
391 * We need to wait for hwthread_req to become zero, since
392 * we may not turn on the MMU while hwthread_req is non-zero.
393 * While waiting we also need to check if we get given a vcpu to run.
394 */
395 kvm_no_guest:
396 lbz r3, HSTATE_HWTHREAD_REQ(r13)
397 cmpwi r3, 0
398 bne 53f
399 HMT_MEDIUM
400 li r0, KVM_HWTHREAD_IN_KERNEL
401 stb r0, HSTATE_HWTHREAD_STATE(r13)
402 /* need to recheck hwthread_req after a barrier, to avoid race */
403 sync
404 lbz r3, HSTATE_HWTHREAD_REQ(r13)
405 cmpwi r3, 0
406 bne 54f
407 /*
408 * We jump to pnv_wakeup_loss, which will return to the caller
409 * of power7_nap in the powernv cpu offline loop. The value we
410 * put in r3 becomes the return value for power7_nap.
411 */
412 li r3, LPCR_PECE0
413 mfspr r4, SPRN_LPCR
414 rlwimi r4, r3, 0, LPCR_PECE0 | LPCR_PECE1
415 mtspr SPRN_LPCR, r4
416 li r3, 0
417 b pnv_wakeup_loss
418
419 53: HMT_LOW
420 ld r5, HSTATE_KVM_VCORE(r13)
421 cmpdi r5, 0
422 bne 60f
423 ld r3, HSTATE_SPLIT_MODE(r13)
424 cmpdi r3, 0
425 beq kvm_no_guest
426 lbz r0, KVM_SPLIT_DO_NAP(r3)
427 cmpwi r0, 0
428 beq kvm_no_guest
429 HMT_MEDIUM
430 b kvm_unsplit_nap
431 60: HMT_MEDIUM
432 b kvm_secondary_got_guest
433
434 54: li r0, KVM_HWTHREAD_IN_KVM
435 stb r0, HSTATE_HWTHREAD_STATE(r13)
436 b kvm_no_guest
437
438 /*
439 * Here the primary thread is trying to return the core to
440 * whole-core mode, so we need to nap.
441 */
442 kvm_unsplit_nap:
443 /*
444 * When secondaries are napping in kvm_unsplit_nap() with
445 * hwthread_req = 1, HMI goes ignored even though subcores are
446 * already exited the guest. Hence HMI keeps waking up secondaries
447 * from nap in a loop and secondaries always go back to nap since
448 * no vcore is assigned to them. This makes impossible for primary
449 * thread to get hold of secondary threads resulting into a soft
450 * lockup in KVM path.
451 *
452 * Let us check if HMI is pending and handle it before we go to nap.
453 */
454 cmpwi r12, BOOK3S_INTERRUPT_HMI
455 bne 55f
456 li r3, 0 /* NULL argument */
457 bl hmi_exception_realmode
458 55:
459 /*
460 * Ensure that secondary doesn't nap when it has
461 * its vcore pointer set.
462 */
463 sync /* matches smp_mb() before setting split_info.do_nap */
464 ld r0, HSTATE_KVM_VCORE(r13)
465 cmpdi r0, 0
466 bne kvm_no_guest
467 /* clear any pending message */
468 BEGIN_FTR_SECTION
469 lis r6, (PPC_DBELL_SERVER << (63-36))@h
470 PPC_MSGCLR(6)
471 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
472 /* Set kvm_split_mode.napped[tid] = 1 */
473 ld r3, HSTATE_SPLIT_MODE(r13)
474 li r0, 1
475 lhz r4, PACAPACAINDEX(r13)
476 clrldi r4, r4, 61 /* micro-threading => P8 => 8 threads/core */
477 addi r4, r4, KVM_SPLIT_NAPPED
478 stbx r0, r3, r4
479 /* Check the do_nap flag again after setting napped[] */
480 sync
481 lbz r0, KVM_SPLIT_DO_NAP(r3)
482 cmpwi r0, 0
483 beq 57f
484 li r3, (LPCR_PECEDH | LPCR_PECE0) >> 4
485 mfspr r4, SPRN_LPCR
486 rlwimi r4, r3, 4, (LPCR_PECEDP | LPCR_PECEDH | LPCR_PECE0 | LPCR_PECE1)
487 mtspr SPRN_LPCR, r4
488 isync
489 std r0, HSTATE_SCRATCH0(r13)
490 ptesync
491 ld r0, HSTATE_SCRATCH0(r13)
492 1: cmpd r0, r0
493 bne 1b
494 nap
495 b .
496
497 57: li r0, 0
498 stbx r0, r3, r4
499 b kvm_no_guest
500
501 /******************************************************************************
502 * *
503 * Entry code *
504 * *
505 *****************************************************************************/
506
507 .global kvmppc_hv_entry
508 kvmppc_hv_entry:
509
510 /* Required state:
511 *
512 * R4 = vcpu pointer (or NULL)
513 * MSR = ~IR|DR
514 * R13 = PACA
515 * R1 = host R1
516 * R2 = TOC
517 * all other volatile GPRS = free
518 */
519 mflr r0
520 std r0, PPC_LR_STKOFF(r1)
521 stdu r1, -112(r1)
522
523 /* Save R1 in the PACA */
524 std r1, HSTATE_HOST_R1(r13)
525
526 li r6, KVM_GUEST_MODE_HOST_HV
527 stb r6, HSTATE_IN_GUEST(r13)
528
529 #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
530 /* Store initial timestamp */
531 cmpdi r4, 0
532 beq 1f
533 addi r3, r4, VCPU_TB_RMENTRY
534 bl kvmhv_start_timing
535 1:
536 #endif
537 /* Clear out SLB */
538 li r6,0
539 slbmte r6,r6
540 slbia
541 ptesync
542
543 /*
544 * POWER7/POWER8 host -> guest partition switch code.
545 * We don't have to lock against concurrent tlbies,
546 * but we do have to coordinate across hardware threads.
547 */
548 /* Set bit in entry map iff exit map is zero. */
549 ld r5, HSTATE_KVM_VCORE(r13)
550 li r7, 1
551 lbz r6, HSTATE_PTID(r13)
552 sld r7, r7, r6
553 addi r9, r5, VCORE_ENTRY_EXIT
554 21: lwarx r3, 0, r9
555 cmpwi r3, 0x100 /* any threads starting to exit? */
556 bge secondary_too_late /* if so we're too late to the party */
557 or r3, r3, r7
558 stwcx. r3, 0, r9
559 bne 21b
560
561 /* Primary thread switches to guest partition. */
562 ld r9,VCORE_KVM(r5) /* pointer to struct kvm */
563 cmpwi r6,0
564 bne 10f
565 ld r6,KVM_SDR1(r9)
566 lwz r7,KVM_LPID(r9)
567 li r0,LPID_RSVD /* switch to reserved LPID */
568 mtspr SPRN_LPID,r0
569 ptesync
570 mtspr SPRN_SDR1,r6 /* switch to partition page table */
571 mtspr SPRN_LPID,r7
572 isync
573
574 /* See if we need to flush the TLB */
575 lhz r6,PACAPACAINDEX(r13) /* test_bit(cpu, need_tlb_flush) */
576 clrldi r7,r6,64-6 /* extract bit number (6 bits) */
577 srdi r6,r6,6 /* doubleword number */
578 sldi r6,r6,3 /* address offset */
579 add r6,r6,r9
580 addi r6,r6,KVM_NEED_FLUSH /* dword in kvm->arch.need_tlb_flush */
581 li r0,1
582 sld r0,r0,r7
583 ld r7,0(r6)
584 and. r7,r7,r0
585 beq 22f
586 23: ldarx r7,0,r6 /* if set, clear the bit */
587 andc r7,r7,r0
588 stdcx. r7,0,r6
589 bne 23b
590 /* Flush the TLB of any entries for this LPID */
591 /* use arch 2.07S as a proxy for POWER8 */
592 BEGIN_FTR_SECTION
593 li r6,512 /* POWER8 has 512 sets */
594 FTR_SECTION_ELSE
595 li r6,128 /* POWER7 has 128 sets */
596 ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_207S)
597 mtctr r6
598 li r7,0x800 /* IS field = 0b10 */
599 ptesync
600 28: tlbiel r7
601 addi r7,r7,0x1000
602 bdnz 28b
603 ptesync
604
605 /* Add timebase offset onto timebase */
606 22: ld r8,VCORE_TB_OFFSET(r5)
607 cmpdi r8,0
608 beq 37f
609 mftb r6 /* current host timebase */
610 add r8,r8,r6
611 mtspr SPRN_TBU40,r8 /* update upper 40 bits */
612 mftb r7 /* check if lower 24 bits overflowed */
613 clrldi r6,r6,40
614 clrldi r7,r7,40
615 cmpld r7,r6
616 bge 37f
617 addis r8,r8,0x100 /* if so, increment upper 40 bits */
618 mtspr SPRN_TBU40,r8
619
620 /* Load guest PCR value to select appropriate compat mode */
621 37: ld r7, VCORE_PCR(r5)
622 cmpdi r7, 0
623 beq 38f
624 mtspr SPRN_PCR, r7
625 38:
626
627 BEGIN_FTR_SECTION
628 /* DPDES is shared between threads */
629 ld r8, VCORE_DPDES(r5)
630 mtspr SPRN_DPDES, r8
631 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
632
633 /* Mark the subcore state as inside guest */
634 bl kvmppc_subcore_enter_guest
635 nop
636 ld r5, HSTATE_KVM_VCORE(r13)
637 ld r4, HSTATE_KVM_VCPU(r13)
638 li r0,1
639 stb r0,VCORE_IN_GUEST(r5) /* signal secondaries to continue */
640
641 /* Do we have a guest vcpu to run? */
642 10: cmpdi r4, 0
643 beq kvmppc_primary_no_guest
644 kvmppc_got_guest:
645
646 /* Load up guest SLB entries */
647 lwz r5,VCPU_SLB_MAX(r4)
648 cmpwi r5,0
649 beq 9f
650 mtctr r5
651 addi r6,r4,VCPU_SLB
652 1: ld r8,VCPU_SLB_E(r6)
653 ld r9,VCPU_SLB_V(r6)
654 slbmte r9,r8
655 addi r6,r6,VCPU_SLB_SIZE
656 bdnz 1b
657 9:
658 /* Increment yield count if they have a VPA */
659 ld r3, VCPU_VPA(r4)
660 cmpdi r3, 0
661 beq 25f
662 li r6, LPPACA_YIELDCOUNT
663 LWZX_BE r5, r3, r6
664 addi r5, r5, 1
665 STWX_BE r5, r3, r6
666 li r6, 1
667 stb r6, VCPU_VPA_DIRTY(r4)
668 25:
669
670 /* Save purr/spurr */
671 mfspr r5,SPRN_PURR
672 mfspr r6,SPRN_SPURR
673 std r5,HSTATE_PURR(r13)
674 std r6,HSTATE_SPURR(r13)
675 ld r7,VCPU_PURR(r4)
676 ld r8,VCPU_SPURR(r4)
677 mtspr SPRN_PURR,r7
678 mtspr SPRN_SPURR,r8
679
680 BEGIN_FTR_SECTION
681 /* Set partition DABR */
682 /* Do this before re-enabling PMU to avoid P7 DABR corruption bug */
683 lwz r5,VCPU_DABRX(r4)
684 ld r6,VCPU_DABR(r4)
685 mtspr SPRN_DABRX,r5
686 mtspr SPRN_DABR,r6
687 isync
688 END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
689
690 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
691 BEGIN_FTR_SECTION
692 bl kvmppc_restore_tm
693 END_FTR_SECTION_IFSET(CPU_FTR_TM)
694 #endif
695
696 /* Load guest PMU registers */
697 /* R4 is live here (vcpu pointer) */
698 li r3, 1
699 sldi r3, r3, 31 /* MMCR0_FC (freeze counters) bit */
700 mtspr SPRN_MMCR0, r3 /* freeze all counters, disable ints */
701 isync
702 BEGIN_FTR_SECTION
703 ld r3, VCPU_MMCR(r4)
704 andi. r5, r3, MMCR0_PMAO_SYNC | MMCR0_PMAO
705 cmpwi r5, MMCR0_PMAO
706 beql kvmppc_fix_pmao
707 END_FTR_SECTION_IFSET(CPU_FTR_PMAO_BUG)
708 lwz r3, VCPU_PMC(r4) /* always load up guest PMU registers */
709 lwz r5, VCPU_PMC + 4(r4) /* to prevent information leak */
710 lwz r6, VCPU_PMC + 8(r4)
711 lwz r7, VCPU_PMC + 12(r4)
712 lwz r8, VCPU_PMC + 16(r4)
713 lwz r9, VCPU_PMC + 20(r4)
714 mtspr SPRN_PMC1, r3
715 mtspr SPRN_PMC2, r5
716 mtspr SPRN_PMC3, r6
717 mtspr SPRN_PMC4, r7
718 mtspr SPRN_PMC5, r8
719 mtspr SPRN_PMC6, r9
720 ld r3, VCPU_MMCR(r4)
721 ld r5, VCPU_MMCR + 8(r4)
722 ld r6, VCPU_MMCR + 16(r4)
723 ld r7, VCPU_SIAR(r4)
724 ld r8, VCPU_SDAR(r4)
725 mtspr SPRN_MMCR1, r5
726 mtspr SPRN_MMCRA, r6
727 mtspr SPRN_SIAR, r7
728 mtspr SPRN_SDAR, r8
729 BEGIN_FTR_SECTION
730 ld r5, VCPU_MMCR + 24(r4)
731 ld r6, VCPU_SIER(r4)
732 lwz r7, VCPU_PMC + 24(r4)
733 lwz r8, VCPU_PMC + 28(r4)
734 ld r9, VCPU_MMCR + 32(r4)
735 mtspr SPRN_MMCR2, r5
736 mtspr SPRN_SIER, r6
737 mtspr SPRN_SPMC1, r7
738 mtspr SPRN_SPMC2, r8
739 mtspr SPRN_MMCRS, r9
740 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
741 mtspr SPRN_MMCR0, r3
742 isync
743
744 /* Load up FP, VMX and VSX registers */
745 bl kvmppc_load_fp
746
747 ld r14, VCPU_GPR(R14)(r4)
748 ld r15, VCPU_GPR(R15)(r4)
749 ld r16, VCPU_GPR(R16)(r4)
750 ld r17, VCPU_GPR(R17)(r4)
751 ld r18, VCPU_GPR(R18)(r4)
752 ld r19, VCPU_GPR(R19)(r4)
753 ld r20, VCPU_GPR(R20)(r4)
754 ld r21, VCPU_GPR(R21)(r4)
755 ld r22, VCPU_GPR(R22)(r4)
756 ld r23, VCPU_GPR(R23)(r4)
757 ld r24, VCPU_GPR(R24)(r4)
758 ld r25, VCPU_GPR(R25)(r4)
759 ld r26, VCPU_GPR(R26)(r4)
760 ld r27, VCPU_GPR(R27)(r4)
761 ld r28, VCPU_GPR(R28)(r4)
762 ld r29, VCPU_GPR(R29)(r4)
763 ld r30, VCPU_GPR(R30)(r4)
764 ld r31, VCPU_GPR(R31)(r4)
765
766 /* Switch DSCR to guest value */
767 ld r5, VCPU_DSCR(r4)
768 mtspr SPRN_DSCR, r5
769
770 BEGIN_FTR_SECTION
771 /* Skip next section on POWER7 */
772 b 8f
773 END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
774 /* Load up POWER8-specific registers */
775 ld r5, VCPU_IAMR(r4)
776 lwz r6, VCPU_PSPB(r4)
777 ld r7, VCPU_FSCR(r4)
778 mtspr SPRN_IAMR, r5
779 mtspr SPRN_PSPB, r6
780 mtspr SPRN_FSCR, r7
781 ld r5, VCPU_DAWR(r4)
782 ld r6, VCPU_DAWRX(r4)
783 ld r7, VCPU_CIABR(r4)
784 ld r8, VCPU_TAR(r4)
785 mtspr SPRN_DAWR, r5
786 mtspr SPRN_DAWRX, r6
787 mtspr SPRN_CIABR, r7
788 mtspr SPRN_TAR, r8
789 ld r5, VCPU_IC(r4)
790 ld r6, VCPU_VTB(r4)
791 mtspr SPRN_IC, r5
792 mtspr SPRN_VTB, r6
793 ld r8, VCPU_EBBHR(r4)
794 mtspr SPRN_EBBHR, r8
795 ld r5, VCPU_EBBRR(r4)
796 ld r6, VCPU_BESCR(r4)
797 ld r7, VCPU_CSIGR(r4)
798 ld r8, VCPU_TACR(r4)
799 mtspr SPRN_EBBRR, r5
800 mtspr SPRN_BESCR, r6
801 mtspr SPRN_CSIGR, r7
802 mtspr SPRN_TACR, r8
803 ld r5, VCPU_TCSCR(r4)
804 ld r6, VCPU_ACOP(r4)
805 lwz r7, VCPU_GUEST_PID(r4)
806 ld r8, VCPU_WORT(r4)
807 mtspr SPRN_TCSCR, r5
808 mtspr SPRN_ACOP, r6
809 mtspr SPRN_PID, r7
810 mtspr SPRN_WORT, r8
811 8:
812
813 /*
814 * Set the decrementer to the guest decrementer.
815 */
816 ld r8,VCPU_DEC_EXPIRES(r4)
817 /* r8 is a host timebase value here, convert to guest TB */
818 ld r5,HSTATE_KVM_VCORE(r13)
819 ld r6,VCORE_TB_OFFSET(r5)
820 add r8,r8,r6
821 mftb r7
822 subf r3,r7,r8
823 mtspr SPRN_DEC,r3
824 stw r3,VCPU_DEC(r4)
825
826 ld r5, VCPU_SPRG0(r4)
827 ld r6, VCPU_SPRG1(r4)
828 ld r7, VCPU_SPRG2(r4)
829 ld r8, VCPU_SPRG3(r4)
830 mtspr SPRN_SPRG0, r5
831 mtspr SPRN_SPRG1, r6
832 mtspr SPRN_SPRG2, r7
833 mtspr SPRN_SPRG3, r8
834
835 /* Load up DAR and DSISR */
836 ld r5, VCPU_DAR(r4)
837 lwz r6, VCPU_DSISR(r4)
838 mtspr SPRN_DAR, r5
839 mtspr SPRN_DSISR, r6
840
841 /* Restore AMR and UAMOR, set AMOR to all 1s */
842 ld r5,VCPU_AMR(r4)
843 ld r6,VCPU_UAMOR(r4)
844 li r7,-1
845 mtspr SPRN_AMR,r5
846 mtspr SPRN_UAMOR,r6
847 mtspr SPRN_AMOR,r7
848
849 /* Restore state of CTRL run bit; assume 1 on entry */
850 lwz r5,VCPU_CTRL(r4)
851 andi. r5,r5,1
852 bne 4f
853 mfspr r6,SPRN_CTRLF
854 clrrdi r6,r6,1
855 mtspr SPRN_CTRLT,r6
856 4:
857 /* Secondary threads wait for primary to have done partition switch */
858 ld r5, HSTATE_KVM_VCORE(r13)
859 lbz r6, HSTATE_PTID(r13)
860 cmpwi r6, 0
861 beq 21f
862 lbz r0, VCORE_IN_GUEST(r5)
863 cmpwi r0, 0
864 bne 21f
865 HMT_LOW
866 20: lwz r3, VCORE_ENTRY_EXIT(r5)
867 cmpwi r3, 0x100
868 bge no_switch_exit
869 lbz r0, VCORE_IN_GUEST(r5)
870 cmpwi r0, 0
871 beq 20b
872 HMT_MEDIUM
873 21:
874 /* Set LPCR. */
875 ld r8,VCORE_LPCR(r5)
876 mtspr SPRN_LPCR,r8
877 isync
878
879 /* Check if HDEC expires soon */
880 mfspr r3, SPRN_HDEC
881 cmpwi r3, 512 /* 1 microsecond */
882 blt hdec_soon
883
884 ld r6, VCPU_CTR(r4)
885 ld r7, VCPU_XER(r4)
886
887 mtctr r6
888 mtxer r7
889
890 kvmppc_cede_reentry: /* r4 = vcpu, r13 = paca */
891 ld r10, VCPU_PC(r4)
892 ld r11, VCPU_MSR(r4)
893 ld r6, VCPU_SRR0(r4)
894 ld r7, VCPU_SRR1(r4)
895 mtspr SPRN_SRR0, r6
896 mtspr SPRN_SRR1, r7
897
898 deliver_guest_interrupt:
899 /* r11 = vcpu->arch.msr & ~MSR_HV */
900 rldicl r11, r11, 63 - MSR_HV_LG, 1
901 rotldi r11, r11, 1 + MSR_HV_LG
902 ori r11, r11, MSR_ME
903
904 /* Check if we can deliver an external or decrementer interrupt now */
905 ld r0, VCPU_PENDING_EXC(r4)
906 rldicl r0, r0, 64 - BOOK3S_IRQPRIO_EXTERNAL_LEVEL, 63
907 cmpdi cr1, r0, 0
908 andi. r8, r11, MSR_EE
909 mfspr r8, SPRN_LPCR
910 /* Insert EXTERNAL_LEVEL bit into LPCR at the MER bit position */
911 rldimi r8, r0, LPCR_MER_SH, 63 - LPCR_MER_SH
912 mtspr SPRN_LPCR, r8
913 isync
914 beq 5f
915 li r0, BOOK3S_INTERRUPT_EXTERNAL
916 bne cr1, 12f
917 mfspr r0, SPRN_DEC
918 cmpwi r0, 0
919 li r0, BOOK3S_INTERRUPT_DECREMENTER
920 bge 5f
921
922 12: mtspr SPRN_SRR0, r10
923 mr r10,r0
924 mtspr SPRN_SRR1, r11
925 mr r9, r4
926 bl kvmppc_msr_interrupt
927 5:
928
929 /*
930 * Required state:
931 * R4 = vcpu
932 * R10: value for HSRR0
933 * R11: value for HSRR1
934 * R13 = PACA
935 */
936 fast_guest_return:
937 li r0,0
938 stb r0,VCPU_CEDED(r4) /* cancel cede */
939 mtspr SPRN_HSRR0,r10
940 mtspr SPRN_HSRR1,r11
941
942 /* Activate guest mode, so faults get handled by KVM */
943 li r9, KVM_GUEST_MODE_GUEST_HV
944 stb r9, HSTATE_IN_GUEST(r13)
945
946 #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
947 /* Accumulate timing */
948 addi r3, r4, VCPU_TB_GUEST
949 bl kvmhv_accumulate_time
950 #endif
951
952 /* Enter guest */
953
954 BEGIN_FTR_SECTION
955 ld r5, VCPU_CFAR(r4)
956 mtspr SPRN_CFAR, r5
957 END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
958 BEGIN_FTR_SECTION
959 ld r0, VCPU_PPR(r4)
960 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
961
962 ld r5, VCPU_LR(r4)
963 lwz r6, VCPU_CR(r4)
964 mtlr r5
965 mtcr r6
966
967 ld r1, VCPU_GPR(R1)(r4)
968 ld r2, VCPU_GPR(R2)(r4)
969 ld r3, VCPU_GPR(R3)(r4)
970 ld r5, VCPU_GPR(R5)(r4)
971 ld r6, VCPU_GPR(R6)(r4)
972 ld r7, VCPU_GPR(R7)(r4)
973 ld r8, VCPU_GPR(R8)(r4)
974 ld r9, VCPU_GPR(R9)(r4)
975 ld r10, VCPU_GPR(R10)(r4)
976 ld r11, VCPU_GPR(R11)(r4)
977 ld r12, VCPU_GPR(R12)(r4)
978 ld r13, VCPU_GPR(R13)(r4)
979
980 BEGIN_FTR_SECTION
981 mtspr SPRN_PPR, r0
982 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
983 ld r0, VCPU_GPR(R0)(r4)
984 ld r4, VCPU_GPR(R4)(r4)
985
986 hrfid
987 b .
988
989 secondary_too_late:
990 li r12, 0
991 cmpdi r4, 0
992 beq 11f
993 stw r12, VCPU_TRAP(r4)
994 #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
995 addi r3, r4, VCPU_TB_RMEXIT
996 bl kvmhv_accumulate_time
997 #endif
998 11: b kvmhv_switch_to_host
999
1000 no_switch_exit:
1001 HMT_MEDIUM
1002 li r12, 0
1003 b 12f
1004 hdec_soon:
1005 li r12, BOOK3S_INTERRUPT_HV_DECREMENTER
1006 12: stw r12, VCPU_TRAP(r4)
1007 mr r9, r4
1008 #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
1009 addi r3, r4, VCPU_TB_RMEXIT
1010 bl kvmhv_accumulate_time
1011 #endif
1012 b guest_exit_cont
1013
1014 /******************************************************************************
1015 * *
1016 * Exit code *
1017 * *
1018 *****************************************************************************/
1019
1020 /*
1021 * We come here from the first-level interrupt handlers.
1022 */
1023 .globl kvmppc_interrupt_hv
1024 kvmppc_interrupt_hv:
1025 /*
1026 * Register contents:
1027 * R12 = interrupt vector
1028 * R13 = PACA
1029 * guest CR, R12 saved in shadow VCPU SCRATCH1/0
1030 * guest R13 saved in SPRN_SCRATCH0
1031 */
1032 std r9, HSTATE_SCRATCH2(r13)
1033
1034 lbz r9, HSTATE_IN_GUEST(r13)
1035 cmpwi r9, KVM_GUEST_MODE_HOST_HV
1036 beq kvmppc_bad_host_intr
1037 #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
1038 cmpwi r9, KVM_GUEST_MODE_GUEST
1039 ld r9, HSTATE_SCRATCH2(r13)
1040 beq kvmppc_interrupt_pr
1041 #endif
1042 /* We're now back in the host but in guest MMU context */
1043 li r9, KVM_GUEST_MODE_HOST_HV
1044 stb r9, HSTATE_IN_GUEST(r13)
1045
1046 ld r9, HSTATE_KVM_VCPU(r13)
1047
1048 /* Save registers */
1049
1050 std r0, VCPU_GPR(R0)(r9)
1051 std r1, VCPU_GPR(R1)(r9)
1052 std r2, VCPU_GPR(R2)(r9)
1053 std r3, VCPU_GPR(R3)(r9)
1054 std r4, VCPU_GPR(R4)(r9)
1055 std r5, VCPU_GPR(R5)(r9)
1056 std r6, VCPU_GPR(R6)(r9)
1057 std r7, VCPU_GPR(R7)(r9)
1058 std r8, VCPU_GPR(R8)(r9)
1059 ld r0, HSTATE_SCRATCH2(r13)
1060 std r0, VCPU_GPR(R9)(r9)
1061 std r10, VCPU_GPR(R10)(r9)
1062 std r11, VCPU_GPR(R11)(r9)
1063 ld r3, HSTATE_SCRATCH0(r13)
1064 lwz r4, HSTATE_SCRATCH1(r13)
1065 std r3, VCPU_GPR(R12)(r9)
1066 stw r4, VCPU_CR(r9)
1067 BEGIN_FTR_SECTION
1068 ld r3, HSTATE_CFAR(r13)
1069 std r3, VCPU_CFAR(r9)
1070 END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
1071 BEGIN_FTR_SECTION
1072 ld r4, HSTATE_PPR(r13)
1073 std r4, VCPU_PPR(r9)
1074 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
1075
1076 /* Restore R1/R2 so we can handle faults */
1077 ld r1, HSTATE_HOST_R1(r13)
1078 ld r2, PACATOC(r13)
1079
1080 mfspr r10, SPRN_SRR0
1081 mfspr r11, SPRN_SRR1
1082 std r10, VCPU_SRR0(r9)
1083 std r11, VCPU_SRR1(r9)
1084 andi. r0, r12, 2 /* need to read HSRR0/1? */
1085 beq 1f
1086 mfspr r10, SPRN_HSRR0
1087 mfspr r11, SPRN_HSRR1
1088 clrrdi r12, r12, 2
1089 1: std r10, VCPU_PC(r9)
1090 std r11, VCPU_MSR(r9)
1091
1092 GET_SCRATCH0(r3)
1093 mflr r4
1094 std r3, VCPU_GPR(R13)(r9)
1095 std r4, VCPU_LR(r9)
1096
1097 stw r12,VCPU_TRAP(r9)
1098
1099 #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
1100 addi r3, r9, VCPU_TB_RMINTR
1101 mr r4, r9
1102 bl kvmhv_accumulate_time
1103 ld r5, VCPU_GPR(R5)(r9)
1104 ld r6, VCPU_GPR(R6)(r9)
1105 ld r7, VCPU_GPR(R7)(r9)
1106 ld r8, VCPU_GPR(R8)(r9)
1107 #endif
1108
1109 /* Save HEIR (HV emulation assist reg) in emul_inst
1110 if this is an HEI (HV emulation interrupt, e40) */
1111 li r3,KVM_INST_FETCH_FAILED
1112 stw r3,VCPU_LAST_INST(r9)
1113 cmpwi r12,BOOK3S_INTERRUPT_H_EMUL_ASSIST
1114 bne 11f
1115 mfspr r3,SPRN_HEIR
1116 11: stw r3,VCPU_HEIR(r9)
1117
1118 /* these are volatile across C function calls */
1119 mfctr r3
1120 mfxer r4
1121 std r3, VCPU_CTR(r9)
1122 std r4, VCPU_XER(r9)
1123
1124 /* If this is a page table miss then see if it's theirs or ours */
1125 cmpwi r12, BOOK3S_INTERRUPT_H_DATA_STORAGE
1126 beq kvmppc_hdsi
1127 cmpwi r12, BOOK3S_INTERRUPT_H_INST_STORAGE
1128 beq kvmppc_hisi
1129
1130 /* See if this is a leftover HDEC interrupt */
1131 cmpwi r12,BOOK3S_INTERRUPT_HV_DECREMENTER
1132 bne 2f
1133 mfspr r3,SPRN_HDEC
1134 cmpwi r3,0
1135 mr r4,r9
1136 bge fast_guest_return
1137 2:
1138 /* See if this is an hcall we can handle in real mode */
1139 cmpwi r12,BOOK3S_INTERRUPT_SYSCALL
1140 beq hcall_try_real_mode
1141
1142 /* Hypervisor doorbell - exit only if host IPI flag set */
1143 cmpwi r12, BOOK3S_INTERRUPT_H_DOORBELL
1144 bne 3f
1145 lbz r0, HSTATE_HOST_IPI(r13)
1146 cmpwi r0, 0
1147 beq 4f
1148 b guest_exit_cont
1149 3:
1150 /* External interrupt ? */
1151 cmpwi r12, BOOK3S_INTERRUPT_EXTERNAL
1152 bne+ guest_exit_cont
1153
1154 /* External interrupt, first check for host_ipi. If this is
1155 * set, we know the host wants us out so let's do it now
1156 */
1157 bl kvmppc_read_intr
1158 cmpdi r3, 0
1159 bgt guest_exit_cont
1160
1161 /* Check if any CPU is heading out to the host, if so head out too */
1162 4: ld r5, HSTATE_KVM_VCORE(r13)
1163 lwz r0, VCORE_ENTRY_EXIT(r5)
1164 cmpwi r0, 0x100
1165 mr r4, r9
1166 blt deliver_guest_interrupt
1167
1168 guest_exit_cont: /* r9 = vcpu, r12 = trap, r13 = paca */
1169 /* Save more register state */
1170 mfdar r6
1171 mfdsisr r7
1172 std r6, VCPU_DAR(r9)
1173 stw r7, VCPU_DSISR(r9)
1174 /* don't overwrite fault_dar/fault_dsisr if HDSI */
1175 cmpwi r12,BOOK3S_INTERRUPT_H_DATA_STORAGE
1176 beq mc_cont
1177 std r6, VCPU_FAULT_DAR(r9)
1178 stw r7, VCPU_FAULT_DSISR(r9)
1179
1180 /* See if it is a machine check */
1181 cmpwi r12, BOOK3S_INTERRUPT_MACHINE_CHECK
1182 beq machine_check_realmode
1183 mc_cont:
1184 #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
1185 addi r3, r9, VCPU_TB_RMEXIT
1186 mr r4, r9
1187 bl kvmhv_accumulate_time
1188 #endif
1189
1190 mr r3, r12
1191 /* Increment exit count, poke other threads to exit */
1192 bl kvmhv_commence_exit
1193 nop
1194 ld r9, HSTATE_KVM_VCPU(r13)
1195 lwz r12, VCPU_TRAP(r9)
1196
1197 /* Stop others sending VCPU interrupts to this physical CPU */
1198 li r0, -1
1199 stw r0, VCPU_CPU(r9)
1200 stw r0, VCPU_THREAD_CPU(r9)
1201
1202 /* Save guest CTRL register, set runlatch to 1 */
1203 mfspr r6,SPRN_CTRLF
1204 stw r6,VCPU_CTRL(r9)
1205 andi. r0,r6,1
1206 bne 4f
1207 ori r6,r6,1
1208 mtspr SPRN_CTRLT,r6
1209 4:
1210 /* Read the guest SLB and save it away */
1211 lwz r0,VCPU_SLB_NR(r9) /* number of entries in SLB */
1212 mtctr r0
1213 li r6,0
1214 addi r7,r9,VCPU_SLB
1215 li r5,0
1216 1: slbmfee r8,r6
1217 andis. r0,r8,SLB_ESID_V@h
1218 beq 2f
1219 add r8,r8,r6 /* put index in */
1220 slbmfev r3,r6
1221 std r8,VCPU_SLB_E(r7)
1222 std r3,VCPU_SLB_V(r7)
1223 addi r7,r7,VCPU_SLB_SIZE
1224 addi r5,r5,1
1225 2: addi r6,r6,1
1226 bdnz 1b
1227 stw r5,VCPU_SLB_MAX(r9)
1228
1229 /*
1230 * Save the guest PURR/SPURR
1231 */
1232 mfspr r5,SPRN_PURR
1233 mfspr r6,SPRN_SPURR
1234 ld r7,VCPU_PURR(r9)
1235 ld r8,VCPU_SPURR(r9)
1236 std r5,VCPU_PURR(r9)
1237 std r6,VCPU_SPURR(r9)
1238 subf r5,r7,r5
1239 subf r6,r8,r6
1240
1241 /*
1242 * Restore host PURR/SPURR and add guest times
1243 * so that the time in the guest gets accounted.
1244 */
1245 ld r3,HSTATE_PURR(r13)
1246 ld r4,HSTATE_SPURR(r13)
1247 add r3,r3,r5
1248 add r4,r4,r6
1249 mtspr SPRN_PURR,r3
1250 mtspr SPRN_SPURR,r4
1251
1252 /* Save DEC */
1253 mfspr r5,SPRN_DEC
1254 mftb r6
1255 extsw r5,r5
1256 add r5,r5,r6
1257 /* r5 is a guest timebase value here, convert to host TB */
1258 ld r3,HSTATE_KVM_VCORE(r13)
1259 ld r4,VCORE_TB_OFFSET(r3)
1260 subf r5,r4,r5
1261 std r5,VCPU_DEC_EXPIRES(r9)
1262
1263 BEGIN_FTR_SECTION
1264 b 8f
1265 END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
1266 /* Save POWER8-specific registers */
1267 mfspr r5, SPRN_IAMR
1268 mfspr r6, SPRN_PSPB
1269 mfspr r7, SPRN_FSCR
1270 std r5, VCPU_IAMR(r9)
1271 stw r6, VCPU_PSPB(r9)
1272 std r7, VCPU_FSCR(r9)
1273 mfspr r5, SPRN_IC
1274 mfspr r6, SPRN_VTB
1275 mfspr r7, SPRN_TAR
1276 std r5, VCPU_IC(r9)
1277 std r6, VCPU_VTB(r9)
1278 std r7, VCPU_TAR(r9)
1279 mfspr r8, SPRN_EBBHR
1280 std r8, VCPU_EBBHR(r9)
1281 mfspr r5, SPRN_EBBRR
1282 mfspr r6, SPRN_BESCR
1283 mfspr r7, SPRN_CSIGR
1284 mfspr r8, SPRN_TACR
1285 std r5, VCPU_EBBRR(r9)
1286 std r6, VCPU_BESCR(r9)
1287 std r7, VCPU_CSIGR(r9)
1288 std r8, VCPU_TACR(r9)
1289 mfspr r5, SPRN_TCSCR
1290 mfspr r6, SPRN_ACOP
1291 mfspr r7, SPRN_PID
1292 mfspr r8, SPRN_WORT
1293 std r5, VCPU_TCSCR(r9)
1294 std r6, VCPU_ACOP(r9)
1295 stw r7, VCPU_GUEST_PID(r9)
1296 std r8, VCPU_WORT(r9)
1297 /*
1298 * Restore various registers to 0, where non-zero values
1299 * set by the guest could disrupt the host.
1300 */
1301 li r0, 0
1302 mtspr SPRN_IAMR, r0
1303 mtspr SPRN_CIABR, r0
1304 mtspr SPRN_DAWRX, r0
1305 mtspr SPRN_TCSCR, r0
1306 mtspr SPRN_WORT, r0
1307 /* Set MMCRS to 1<<31 to freeze and disable the SPMC counters */
1308 li r0, 1
1309 sldi r0, r0, 31
1310 mtspr SPRN_MMCRS, r0
1311 8:
1312
1313 /* Save and reset AMR and UAMOR before turning on the MMU */
1314 mfspr r5,SPRN_AMR
1315 mfspr r6,SPRN_UAMOR
1316 std r5,VCPU_AMR(r9)
1317 std r6,VCPU_UAMOR(r9)
1318 li r6,0
1319 mtspr SPRN_AMR,r6
1320
1321 /* Switch DSCR back to host value */
1322 mfspr r8, SPRN_DSCR
1323 ld r7, HSTATE_DSCR(r13)
1324 std r8, VCPU_DSCR(r9)
1325 mtspr SPRN_DSCR, r7
1326
1327 /* Save non-volatile GPRs */
1328 std r14, VCPU_GPR(R14)(r9)
1329 std r15, VCPU_GPR(R15)(r9)
1330 std r16, VCPU_GPR(R16)(r9)
1331 std r17, VCPU_GPR(R17)(r9)
1332 std r18, VCPU_GPR(R18)(r9)
1333 std r19, VCPU_GPR(R19)(r9)
1334 std r20, VCPU_GPR(R20)(r9)
1335 std r21, VCPU_GPR(R21)(r9)
1336 std r22, VCPU_GPR(R22)(r9)
1337 std r23, VCPU_GPR(R23)(r9)
1338 std r24, VCPU_GPR(R24)(r9)
1339 std r25, VCPU_GPR(R25)(r9)
1340 std r26, VCPU_GPR(R26)(r9)
1341 std r27, VCPU_GPR(R27)(r9)
1342 std r28, VCPU_GPR(R28)(r9)
1343 std r29, VCPU_GPR(R29)(r9)
1344 std r30, VCPU_GPR(R30)(r9)
1345 std r31, VCPU_GPR(R31)(r9)
1346
1347 /* Save SPRGs */
1348 mfspr r3, SPRN_SPRG0
1349 mfspr r4, SPRN_SPRG1
1350 mfspr r5, SPRN_SPRG2
1351 mfspr r6, SPRN_SPRG3
1352 std r3, VCPU_SPRG0(r9)
1353 std r4, VCPU_SPRG1(r9)
1354 std r5, VCPU_SPRG2(r9)
1355 std r6, VCPU_SPRG3(r9)
1356
1357 /* save FP state */
1358 mr r3, r9
1359 bl kvmppc_save_fp
1360
1361 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1362 BEGIN_FTR_SECTION
1363 bl kvmppc_save_tm
1364 END_FTR_SECTION_IFSET(CPU_FTR_TM)
1365 #endif
1366
1367 /* Increment yield count if they have a VPA */
1368 ld r8, VCPU_VPA(r9) /* do they have a VPA? */
1369 cmpdi r8, 0
1370 beq 25f
1371 li r4, LPPACA_YIELDCOUNT
1372 LWZX_BE r3, r8, r4
1373 addi r3, r3, 1
1374 STWX_BE r3, r8, r4
1375 li r3, 1
1376 stb r3, VCPU_VPA_DIRTY(r9)
1377 25:
1378 /* Save PMU registers if requested */
1379 /* r8 and cr0.eq are live here */
1380 BEGIN_FTR_SECTION
1381 /*
1382 * POWER8 seems to have a hardware bug where setting
1383 * MMCR0[PMAE] along with MMCR0[PMC1CE] and/or MMCR0[PMCjCE]
1384 * when some counters are already negative doesn't seem
1385 * to cause a performance monitor alert (and hence interrupt).
1386 * The effect of this is that when saving the PMU state,
1387 * if there is no PMU alert pending when we read MMCR0
1388 * before freezing the counters, but one becomes pending
1389 * before we read the counters, we lose it.
1390 * To work around this, we need a way to freeze the counters
1391 * before reading MMCR0. Normally, freezing the counters
1392 * is done by writing MMCR0 (to set MMCR0[FC]) which
1393 * unavoidably writes MMCR0[PMA0] as well. On POWER8,
1394 * we can also freeze the counters using MMCR2, by writing
1395 * 1s to all the counter freeze condition bits (there are
1396 * 9 bits each for 6 counters).
1397 */
1398 li r3, -1 /* set all freeze bits */
1399 clrrdi r3, r3, 10
1400 mfspr r10, SPRN_MMCR2
1401 mtspr SPRN_MMCR2, r3
1402 isync
1403 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
1404 li r3, 1
1405 sldi r3, r3, 31 /* MMCR0_FC (freeze counters) bit */
1406 mfspr r4, SPRN_MMCR0 /* save MMCR0 */
1407 mtspr SPRN_MMCR0, r3 /* freeze all counters, disable ints */
1408 mfspr r6, SPRN_MMCRA
1409 /* Clear MMCRA in order to disable SDAR updates */
1410 li r7, 0
1411 mtspr SPRN_MMCRA, r7
1412 isync
1413 beq 21f /* if no VPA, save PMU stuff anyway */
1414 lbz r7, LPPACA_PMCINUSE(r8)
1415 cmpwi r7, 0 /* did they ask for PMU stuff to be saved? */
1416 bne 21f
1417 std r3, VCPU_MMCR(r9) /* if not, set saved MMCR0 to FC */
1418 b 22f
1419 21: mfspr r5, SPRN_MMCR1
1420 mfspr r7, SPRN_SIAR
1421 mfspr r8, SPRN_SDAR
1422 std r4, VCPU_MMCR(r9)
1423 std r5, VCPU_MMCR + 8(r9)
1424 std r6, VCPU_MMCR + 16(r9)
1425 BEGIN_FTR_SECTION
1426 std r10, VCPU_MMCR + 24(r9)
1427 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
1428 std r7, VCPU_SIAR(r9)
1429 std r8, VCPU_SDAR(r9)
1430 mfspr r3, SPRN_PMC1
1431 mfspr r4, SPRN_PMC2
1432 mfspr r5, SPRN_PMC3
1433 mfspr r6, SPRN_PMC4
1434 mfspr r7, SPRN_PMC5
1435 mfspr r8, SPRN_PMC6
1436 stw r3, VCPU_PMC(r9)
1437 stw r4, VCPU_PMC + 4(r9)
1438 stw r5, VCPU_PMC + 8(r9)
1439 stw r6, VCPU_PMC + 12(r9)
1440 stw r7, VCPU_PMC + 16(r9)
1441 stw r8, VCPU_PMC + 20(r9)
1442 BEGIN_FTR_SECTION
1443 mfspr r5, SPRN_SIER
1444 mfspr r6, SPRN_SPMC1
1445 mfspr r7, SPRN_SPMC2
1446 mfspr r8, SPRN_MMCRS
1447 std r5, VCPU_SIER(r9)
1448 stw r6, VCPU_PMC + 24(r9)
1449 stw r7, VCPU_PMC + 28(r9)
1450 std r8, VCPU_MMCR + 32(r9)
1451 lis r4, 0x8000
1452 mtspr SPRN_MMCRS, r4
1453 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
1454 22:
1455 /* Clear out SLB */
1456 li r5,0
1457 slbmte r5,r5
1458 slbia
1459 ptesync
1460
1461 /*
1462 * POWER7/POWER8 guest -> host partition switch code.
1463 * We don't have to lock against tlbies but we do
1464 * have to coordinate the hardware threads.
1465 */
1466 kvmhv_switch_to_host:
1467 /* Secondary threads wait for primary to do partition switch */
1468 ld r5,HSTATE_KVM_VCORE(r13)
1469 ld r4,VCORE_KVM(r5) /* pointer to struct kvm */
1470 lbz r3,HSTATE_PTID(r13)
1471 cmpwi r3,0
1472 beq 15f
1473 HMT_LOW
1474 13: lbz r3,VCORE_IN_GUEST(r5)
1475 cmpwi r3,0
1476 bne 13b
1477 HMT_MEDIUM
1478 b 16f
1479
1480 /* Primary thread waits for all the secondaries to exit guest */
1481 15: lwz r3,VCORE_ENTRY_EXIT(r5)
1482 rlwinm r0,r3,32-8,0xff
1483 clrldi r3,r3,56
1484 cmpw r3,r0
1485 bne 15b
1486 isync
1487
1488 /* Did we actually switch to the guest at all? */
1489 lbz r6, VCORE_IN_GUEST(r5)
1490 cmpwi r6, 0
1491 beq 19f
1492
1493 /* Primary thread switches back to host partition */
1494 ld r6,KVM_HOST_SDR1(r4)
1495 lwz r7,KVM_HOST_LPID(r4)
1496 li r8,LPID_RSVD /* switch to reserved LPID */
1497 mtspr SPRN_LPID,r8
1498 ptesync
1499 mtspr SPRN_SDR1,r6 /* switch to partition page table */
1500 mtspr SPRN_LPID,r7
1501 isync
1502
1503 BEGIN_FTR_SECTION
1504 /* DPDES is shared between threads */
1505 mfspr r7, SPRN_DPDES
1506 std r7, VCORE_DPDES(r5)
1507 /* clear DPDES so we don't get guest doorbells in the host */
1508 li r8, 0
1509 mtspr SPRN_DPDES, r8
1510 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
1511
1512 /* If HMI, call kvmppc_realmode_hmi_handler() */
1513 cmpwi r12, BOOK3S_INTERRUPT_HMI
1514 bne 27f
1515 bl kvmppc_realmode_hmi_handler
1516 nop
1517 li r12, BOOK3S_INTERRUPT_HMI
1518 /*
1519 * At this point kvmppc_realmode_hmi_handler would have resync-ed
1520 * the TB. Hence it is not required to subtract guest timebase
1521 * offset from timebase. So, skip it.
1522 *
1523 * Also, do not call kvmppc_subcore_exit_guest() because it has
1524 * been invoked as part of kvmppc_realmode_hmi_handler().
1525 */
1526 b 30f
1527
1528 27:
1529 /* Subtract timebase offset from timebase */
1530 ld r8,VCORE_TB_OFFSET(r5)
1531 cmpdi r8,0
1532 beq 17f
1533 mftb r6 /* current guest timebase */
1534 subf r8,r8,r6
1535 mtspr SPRN_TBU40,r8 /* update upper 40 bits */
1536 mftb r7 /* check if lower 24 bits overflowed */
1537 clrldi r6,r6,40
1538 clrldi r7,r7,40
1539 cmpld r7,r6
1540 bge 17f
1541 addis r8,r8,0x100 /* if so, increment upper 40 bits */
1542 mtspr SPRN_TBU40,r8
1543
1544 17: bl kvmppc_subcore_exit_guest
1545 nop
1546 30: ld r5,HSTATE_KVM_VCORE(r13)
1547 ld r4,VCORE_KVM(r5) /* pointer to struct kvm */
1548
1549 /* Reset PCR */
1550 ld r0, VCORE_PCR(r5)
1551 cmpdi r0, 0
1552 beq 18f
1553 li r0, 0
1554 mtspr SPRN_PCR, r0
1555 18:
1556 /* Signal secondary CPUs to continue */
1557 stb r0,VCORE_IN_GUEST(r5)
1558 19: lis r8,0x7fff /* MAX_INT@h */
1559 mtspr SPRN_HDEC,r8
1560
1561 16: ld r8,KVM_HOST_LPCR(r4)
1562 mtspr SPRN_LPCR,r8
1563 isync
1564
1565 /* load host SLB entries */
1566 ld r8,PACA_SLBSHADOWPTR(r13)
1567
1568 .rept SLB_NUM_BOLTED
1569 li r3, SLBSHADOW_SAVEAREA
1570 LDX_BE r5, r8, r3
1571 addi r3, r3, 8
1572 LDX_BE r6, r8, r3
1573 andis. r7,r5,SLB_ESID_V@h
1574 beq 1f
1575 slbmte r6,r5
1576 1: addi r8,r8,16
1577 .endr
1578
1579 #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
1580 /* Finish timing, if we have a vcpu */
1581 ld r4, HSTATE_KVM_VCPU(r13)
1582 cmpdi r4, 0
1583 li r3, 0
1584 beq 2f
1585 bl kvmhv_accumulate_time
1586 2:
1587 #endif
1588 /* Unset guest mode */
1589 li r0, KVM_GUEST_MODE_NONE
1590 stb r0, HSTATE_IN_GUEST(r13)
1591
1592 ld r0, 112+PPC_LR_STKOFF(r1)
1593 addi r1, r1, 112
1594 mtlr r0
1595 blr
1596
1597 /*
1598 * Check whether an HDSI is an HPTE not found fault or something else.
1599 * If it is an HPTE not found fault that is due to the guest accessing
1600 * a page that they have mapped but which we have paged out, then
1601 * we continue on with the guest exit path. In all other cases,
1602 * reflect the HDSI to the guest as a DSI.
1603 */
1604 kvmppc_hdsi:
1605 mfspr r4, SPRN_HDAR
1606 mfspr r6, SPRN_HDSISR
1607 /* HPTE not found fault or protection fault? */
1608 andis. r0, r6, (DSISR_NOHPTE | DSISR_PROTFAULT)@h
1609 beq 1f /* if not, send it to the guest */
1610 andi. r0, r11, MSR_DR /* data relocation enabled? */
1611 beq 3f
1612 clrrdi r0, r4, 28
1613 PPC_SLBFEE_DOT(R5, R0) /* if so, look up SLB */
1614 li r0, BOOK3S_INTERRUPT_DATA_SEGMENT
1615 bne 7f /* if no SLB entry found */
1616 4: std r4, VCPU_FAULT_DAR(r9)
1617 stw r6, VCPU_FAULT_DSISR(r9)
1618
1619 /* Search the hash table. */
1620 mr r3, r9 /* vcpu pointer */
1621 li r7, 1 /* data fault */
1622 bl kvmppc_hpte_hv_fault
1623 ld r9, HSTATE_KVM_VCPU(r13)
1624 ld r10, VCPU_PC(r9)
1625 ld r11, VCPU_MSR(r9)
1626 li r12, BOOK3S_INTERRUPT_H_DATA_STORAGE
1627 cmpdi r3, 0 /* retry the instruction */
1628 beq 6f
1629 cmpdi r3, -1 /* handle in kernel mode */
1630 beq guest_exit_cont
1631 cmpdi r3, -2 /* MMIO emulation; need instr word */
1632 beq 2f
1633
1634 /* Synthesize a DSI (or DSegI) for the guest */
1635 ld r4, VCPU_FAULT_DAR(r9)
1636 mr r6, r3
1637 1: li r0, BOOK3S_INTERRUPT_DATA_STORAGE
1638 mtspr SPRN_DSISR, r6
1639 7: mtspr SPRN_DAR, r4
1640 mtspr SPRN_SRR0, r10
1641 mtspr SPRN_SRR1, r11
1642 mr r10, r0
1643 bl kvmppc_msr_interrupt
1644 fast_interrupt_c_return:
1645 6: ld r7, VCPU_CTR(r9)
1646 ld r8, VCPU_XER(r9)
1647 mtctr r7
1648 mtxer r8
1649 mr r4, r9
1650 b fast_guest_return
1651
1652 3: ld r5, VCPU_KVM(r9) /* not relocated, use VRMA */
1653 ld r5, KVM_VRMA_SLB_V(r5)
1654 b 4b
1655
1656 /* If this is for emulated MMIO, load the instruction word */
1657 2: li r8, KVM_INST_FETCH_FAILED /* In case lwz faults */
1658
1659 /* Set guest mode to 'jump over instruction' so if lwz faults
1660 * we'll just continue at the next IP. */
1661 li r0, KVM_GUEST_MODE_SKIP
1662 stb r0, HSTATE_IN_GUEST(r13)
1663
1664 /* Do the access with MSR:DR enabled */
1665 mfmsr r3
1666 ori r4, r3, MSR_DR /* Enable paging for data */
1667 mtmsrd r4
1668 lwz r8, 0(r10)
1669 mtmsrd r3
1670
1671 /* Store the result */
1672 stw r8, VCPU_LAST_INST(r9)
1673
1674 /* Unset guest mode. */
1675 li r0, KVM_GUEST_MODE_HOST_HV
1676 stb r0, HSTATE_IN_GUEST(r13)
1677 b guest_exit_cont
1678
1679 /*
1680 * Similarly for an HISI, reflect it to the guest as an ISI unless
1681 * it is an HPTE not found fault for a page that we have paged out.
1682 */
1683 kvmppc_hisi:
1684 andis. r0, r11, SRR1_ISI_NOPT@h
1685 beq 1f
1686 andi. r0, r11, MSR_IR /* instruction relocation enabled? */
1687 beq 3f
1688 clrrdi r0, r10, 28
1689 PPC_SLBFEE_DOT(R5, R0) /* if so, look up SLB */
1690 li r0, BOOK3S_INTERRUPT_INST_SEGMENT
1691 bne 7f /* if no SLB entry found */
1692 4:
1693 /* Search the hash table. */
1694 mr r3, r9 /* vcpu pointer */
1695 mr r4, r10
1696 mr r6, r11
1697 li r7, 0 /* instruction fault */
1698 bl kvmppc_hpte_hv_fault
1699 ld r9, HSTATE_KVM_VCPU(r13)
1700 ld r10, VCPU_PC(r9)
1701 ld r11, VCPU_MSR(r9)
1702 li r12, BOOK3S_INTERRUPT_H_INST_STORAGE
1703 cmpdi r3, 0 /* retry the instruction */
1704 beq fast_interrupt_c_return
1705 cmpdi r3, -1 /* handle in kernel mode */
1706 beq guest_exit_cont
1707
1708 /* Synthesize an ISI (or ISegI) for the guest */
1709 mr r11, r3
1710 1: li r0, BOOK3S_INTERRUPT_INST_STORAGE
1711 7: mtspr SPRN_SRR0, r10
1712 mtspr SPRN_SRR1, r11
1713 mr r10, r0
1714 bl kvmppc_msr_interrupt
1715 b fast_interrupt_c_return
1716
1717 3: ld r6, VCPU_KVM(r9) /* not relocated, use VRMA */
1718 ld r5, KVM_VRMA_SLB_V(r6)
1719 b 4b
1720
1721 /*
1722 * Try to handle an hcall in real mode.
1723 * Returns to the guest if we handle it, or continues on up to
1724 * the kernel if we can't (i.e. if we don't have a handler for
1725 * it, or if the handler returns H_TOO_HARD).
1726 *
1727 * r5 - r8 contain hcall args,
1728 * r9 = vcpu, r10 = pc, r11 = msr, r12 = trap, r13 = paca
1729 */
1730 hcall_try_real_mode:
1731 ld r3,VCPU_GPR(R3)(r9)
1732 andi. r0,r11,MSR_PR
1733 /* sc 1 from userspace - reflect to guest syscall */
1734 bne sc_1_fast_return
1735 clrrdi r3,r3,2
1736 cmpldi r3,hcall_real_table_end - hcall_real_table
1737 bge guest_exit_cont
1738 /* See if this hcall is enabled for in-kernel handling */
1739 ld r4, VCPU_KVM(r9)
1740 srdi r0, r3, 8 /* r0 = (r3 / 4) >> 6 */
1741 sldi r0, r0, 3 /* index into kvm->arch.enabled_hcalls[] */
1742 add r4, r4, r0
1743 ld r0, KVM_ENABLED_HCALLS(r4)
1744 rlwinm r4, r3, 32-2, 0x3f /* r4 = (r3 / 4) & 0x3f */
1745 srd r0, r0, r4
1746 andi. r0, r0, 1
1747 beq guest_exit_cont
1748 /* Get pointer to handler, if any, and call it */
1749 LOAD_REG_ADDR(r4, hcall_real_table)
1750 lwax r3,r3,r4
1751 cmpwi r3,0
1752 beq guest_exit_cont
1753 add r12,r3,r4
1754 mtctr r12
1755 mr r3,r9 /* get vcpu pointer */
1756 ld r4,VCPU_GPR(R4)(r9)
1757 bctrl
1758 cmpdi r3,H_TOO_HARD
1759 beq hcall_real_fallback
1760 ld r4,HSTATE_KVM_VCPU(r13)
1761 std r3,VCPU_GPR(R3)(r4)
1762 ld r10,VCPU_PC(r4)
1763 ld r11,VCPU_MSR(r4)
1764 b fast_guest_return
1765
1766 sc_1_fast_return:
1767 mtspr SPRN_SRR0,r10
1768 mtspr SPRN_SRR1,r11
1769 li r10, BOOK3S_INTERRUPT_SYSCALL
1770 bl kvmppc_msr_interrupt
1771 mr r4,r9
1772 b fast_guest_return
1773
1774 /* We've attempted a real mode hcall, but it's punted it back
1775 * to userspace. We need to restore some clobbered volatiles
1776 * before resuming the pass-it-to-qemu path */
1777 hcall_real_fallback:
1778 li r12,BOOK3S_INTERRUPT_SYSCALL
1779 ld r9, HSTATE_KVM_VCPU(r13)
1780
1781 b guest_exit_cont
1782
1783 .globl hcall_real_table
1784 hcall_real_table:
1785 .long 0 /* 0 - unused */
1786 .long DOTSYM(kvmppc_h_remove) - hcall_real_table
1787 .long DOTSYM(kvmppc_h_enter) - hcall_real_table
1788 .long DOTSYM(kvmppc_h_read) - hcall_real_table
1789 .long DOTSYM(kvmppc_h_clear_mod) - hcall_real_table
1790 .long DOTSYM(kvmppc_h_clear_ref) - hcall_real_table
1791 .long DOTSYM(kvmppc_h_protect) - hcall_real_table
1792 .long DOTSYM(kvmppc_h_get_tce) - hcall_real_table
1793 .long DOTSYM(kvmppc_rm_h_put_tce) - hcall_real_table
1794 .long 0 /* 0x24 - H_SET_SPRG0 */
1795 .long DOTSYM(kvmppc_h_set_dabr) - hcall_real_table
1796 .long 0 /* 0x2c */
1797 .long 0 /* 0x30 */
1798 .long 0 /* 0x34 */
1799 .long 0 /* 0x38 */
1800 .long 0 /* 0x3c */
1801 .long 0 /* 0x40 */
1802 .long 0 /* 0x44 */
1803 .long 0 /* 0x48 */
1804 .long 0 /* 0x4c */
1805 .long 0 /* 0x50 */
1806 .long 0 /* 0x54 */
1807 .long 0 /* 0x58 */
1808 .long 0 /* 0x5c */
1809 .long 0 /* 0x60 */
1810 #ifdef CONFIG_KVM_XICS
1811 .long DOTSYM(kvmppc_rm_h_eoi) - hcall_real_table
1812 .long DOTSYM(kvmppc_rm_h_cppr) - hcall_real_table
1813 .long DOTSYM(kvmppc_rm_h_ipi) - hcall_real_table
1814 .long 0 /* 0x70 - H_IPOLL */
1815 .long DOTSYM(kvmppc_rm_h_xirr) - hcall_real_table
1816 #else
1817 .long 0 /* 0x64 - H_EOI */
1818 .long 0 /* 0x68 - H_CPPR */
1819 .long 0 /* 0x6c - H_IPI */
1820 .long 0 /* 0x70 - H_IPOLL */
1821 .long 0 /* 0x74 - H_XIRR */
1822 #endif
1823 .long 0 /* 0x78 */
1824 .long 0 /* 0x7c */
1825 .long 0 /* 0x80 */
1826 .long 0 /* 0x84 */
1827 .long 0 /* 0x88 */
1828 .long 0 /* 0x8c */
1829 .long 0 /* 0x90 */
1830 .long 0 /* 0x94 */
1831 .long 0 /* 0x98 */
1832 .long 0 /* 0x9c */
1833 .long 0 /* 0xa0 */
1834 .long 0 /* 0xa4 */
1835 .long 0 /* 0xa8 */
1836 .long 0 /* 0xac */
1837 .long 0 /* 0xb0 */
1838 .long 0 /* 0xb4 */
1839 .long 0 /* 0xb8 */
1840 .long 0 /* 0xbc */
1841 .long 0 /* 0xc0 */
1842 .long 0 /* 0xc4 */
1843 .long 0 /* 0xc8 */
1844 .long 0 /* 0xcc */
1845 .long 0 /* 0xd0 */
1846 .long 0 /* 0xd4 */
1847 .long 0 /* 0xd8 */
1848 .long 0 /* 0xdc */
1849 .long DOTSYM(kvmppc_h_cede) - hcall_real_table
1850 .long DOTSYM(kvmppc_rm_h_confer) - hcall_real_table
1851 .long 0 /* 0xe8 */
1852 .long 0 /* 0xec */
1853 .long 0 /* 0xf0 */
1854 .long 0 /* 0xf4 */
1855 .long 0 /* 0xf8 */
1856 .long 0 /* 0xfc */
1857 .long 0 /* 0x100 */
1858 .long 0 /* 0x104 */
1859 .long 0 /* 0x108 */
1860 .long 0 /* 0x10c */
1861 .long 0 /* 0x110 */
1862 .long 0 /* 0x114 */
1863 .long 0 /* 0x118 */
1864 .long 0 /* 0x11c */
1865 .long 0 /* 0x120 */
1866 .long DOTSYM(kvmppc_h_bulk_remove) - hcall_real_table
1867 .long 0 /* 0x128 */
1868 .long 0 /* 0x12c */
1869 .long 0 /* 0x130 */
1870 .long DOTSYM(kvmppc_h_set_xdabr) - hcall_real_table
1871 .long DOTSYM(kvmppc_rm_h_stuff_tce) - hcall_real_table
1872 .long DOTSYM(kvmppc_rm_h_put_tce_indirect) - hcall_real_table
1873 .long 0 /* 0x140 */
1874 .long 0 /* 0x144 */
1875 .long 0 /* 0x148 */
1876 .long 0 /* 0x14c */
1877 .long 0 /* 0x150 */
1878 .long 0 /* 0x154 */
1879 .long 0 /* 0x158 */
1880 .long 0 /* 0x15c */
1881 .long 0 /* 0x160 */
1882 .long 0 /* 0x164 */
1883 .long 0 /* 0x168 */
1884 .long 0 /* 0x16c */
1885 .long 0 /* 0x170 */
1886 .long 0 /* 0x174 */
1887 .long 0 /* 0x178 */
1888 .long 0 /* 0x17c */
1889 .long 0 /* 0x180 */
1890 .long 0 /* 0x184 */
1891 .long 0 /* 0x188 */
1892 .long 0 /* 0x18c */
1893 .long 0 /* 0x190 */
1894 .long 0 /* 0x194 */
1895 .long 0 /* 0x198 */
1896 .long 0 /* 0x19c */
1897 .long 0 /* 0x1a0 */
1898 .long 0 /* 0x1a4 */
1899 .long 0 /* 0x1a8 */
1900 .long 0 /* 0x1ac */
1901 .long 0 /* 0x1b0 */
1902 .long 0 /* 0x1b4 */
1903 .long 0 /* 0x1b8 */
1904 .long 0 /* 0x1bc */
1905 .long 0 /* 0x1c0 */
1906 .long 0 /* 0x1c4 */
1907 .long 0 /* 0x1c8 */
1908 .long 0 /* 0x1cc */
1909 .long 0 /* 0x1d0 */
1910 .long 0 /* 0x1d4 */
1911 .long 0 /* 0x1d8 */
1912 .long 0 /* 0x1dc */
1913 .long 0 /* 0x1e0 */
1914 .long 0 /* 0x1e4 */
1915 .long 0 /* 0x1e8 */
1916 .long 0 /* 0x1ec */
1917 .long 0 /* 0x1f0 */
1918 .long 0 /* 0x1f4 */
1919 .long 0 /* 0x1f8 */
1920 .long 0 /* 0x1fc */
1921 .long 0 /* 0x200 */
1922 .long 0 /* 0x204 */
1923 .long 0 /* 0x208 */
1924 .long 0 /* 0x20c */
1925 .long 0 /* 0x210 */
1926 .long 0 /* 0x214 */
1927 .long 0 /* 0x218 */
1928 .long 0 /* 0x21c */
1929 .long 0 /* 0x220 */
1930 .long 0 /* 0x224 */
1931 .long 0 /* 0x228 */
1932 .long 0 /* 0x22c */
1933 .long 0 /* 0x230 */
1934 .long 0 /* 0x234 */
1935 .long 0 /* 0x238 */
1936 .long 0 /* 0x23c */
1937 .long 0 /* 0x240 */
1938 .long 0 /* 0x244 */
1939 .long 0 /* 0x248 */
1940 .long 0 /* 0x24c */
1941 .long 0 /* 0x250 */
1942 .long 0 /* 0x254 */
1943 .long 0 /* 0x258 */
1944 .long 0 /* 0x25c */
1945 .long 0 /* 0x260 */
1946 .long 0 /* 0x264 */
1947 .long 0 /* 0x268 */
1948 .long 0 /* 0x26c */
1949 .long 0 /* 0x270 */
1950 .long 0 /* 0x274 */
1951 .long 0 /* 0x278 */
1952 .long 0 /* 0x27c */
1953 .long 0 /* 0x280 */
1954 .long 0 /* 0x284 */
1955 .long 0 /* 0x288 */
1956 .long 0 /* 0x28c */
1957 .long 0 /* 0x290 */
1958 .long 0 /* 0x294 */
1959 .long 0 /* 0x298 */
1960 .long 0 /* 0x29c */
1961 .long 0 /* 0x2a0 */
1962 .long 0 /* 0x2a4 */
1963 .long 0 /* 0x2a8 */
1964 .long 0 /* 0x2ac */
1965 .long 0 /* 0x2b0 */
1966 .long 0 /* 0x2b4 */
1967 .long 0 /* 0x2b8 */
1968 .long 0 /* 0x2bc */
1969 .long 0 /* 0x2c0 */
1970 .long 0 /* 0x2c4 */
1971 .long 0 /* 0x2c8 */
1972 .long 0 /* 0x2cc */
1973 .long 0 /* 0x2d0 */
1974 .long 0 /* 0x2d4 */
1975 .long 0 /* 0x2d8 */
1976 .long 0 /* 0x2dc */
1977 .long 0 /* 0x2e0 */
1978 .long 0 /* 0x2e4 */
1979 .long 0 /* 0x2e8 */
1980 .long 0 /* 0x2ec */
1981 .long 0 /* 0x2f0 */
1982 .long 0 /* 0x2f4 */
1983 .long 0 /* 0x2f8 */
1984 .long 0 /* 0x2fc */
1985 .long DOTSYM(kvmppc_h_random) - hcall_real_table
1986 .globl hcall_real_table_end
1987 hcall_real_table_end:
1988
1989 _GLOBAL(kvmppc_h_set_xdabr)
1990 andi. r0, r5, DABRX_USER | DABRX_KERNEL
1991 beq 6f
1992 li r0, DABRX_USER | DABRX_KERNEL | DABRX_BTI
1993 andc. r0, r5, r0
1994 beq 3f
1995 6: li r3, H_PARAMETER
1996 blr
1997
1998 _GLOBAL(kvmppc_h_set_dabr)
1999 li r5, DABRX_USER | DABRX_KERNEL
2000 3:
2001 BEGIN_FTR_SECTION
2002 b 2f
2003 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
2004 std r4,VCPU_DABR(r3)
2005 stw r5, VCPU_DABRX(r3)
2006 mtspr SPRN_DABRX, r5
2007 /* Work around P7 bug where DABR can get corrupted on mtspr */
2008 1: mtspr SPRN_DABR,r4
2009 mfspr r5, SPRN_DABR
2010 cmpd r4, r5
2011 bne 1b
2012 isync
2013 li r3,0
2014 blr
2015
2016 /* Emulate H_SET_DABR/X on P8 for the sake of compat mode guests */
2017 2: rlwimi r5, r4, 5, DAWRX_DR | DAWRX_DW
2018 rlwimi r5, r4, 2, DAWRX_WT
2019 clrrdi r4, r4, 3
2020 std r4, VCPU_DAWR(r3)
2021 std r5, VCPU_DAWRX(r3)
2022 mtspr SPRN_DAWR, r4
2023 mtspr SPRN_DAWRX, r5
2024 li r3, 0
2025 blr
2026
2027 _GLOBAL(kvmppc_h_cede) /* r3 = vcpu pointer, r11 = msr, r13 = paca */
2028 ori r11,r11,MSR_EE
2029 std r11,VCPU_MSR(r3)
2030 li r0,1
2031 stb r0,VCPU_CEDED(r3)
2032 sync /* order setting ceded vs. testing prodded */
2033 lbz r5,VCPU_PRODDED(r3)
2034 cmpwi r5,0
2035 bne kvm_cede_prodded
2036 li r12,0 /* set trap to 0 to say hcall is handled */
2037 stw r12,VCPU_TRAP(r3)
2038 li r0,H_SUCCESS
2039 std r0,VCPU_GPR(R3)(r3)
2040
2041 /*
2042 * Set our bit in the bitmask of napping threads unless all the
2043 * other threads are already napping, in which case we send this
2044 * up to the host.
2045 */
2046 ld r5,HSTATE_KVM_VCORE(r13)
2047 lbz r6,HSTATE_PTID(r13)
2048 lwz r8,VCORE_ENTRY_EXIT(r5)
2049 clrldi r8,r8,56
2050 li r0,1
2051 sld r0,r0,r6
2052 addi r6,r5,VCORE_NAPPING_THREADS
2053 31: lwarx r4,0,r6
2054 or r4,r4,r0
2055 cmpw r4,r8
2056 beq kvm_cede_exit
2057 stwcx. r4,0,r6
2058 bne 31b
2059 /* order napping_threads update vs testing entry_exit_map */
2060 isync
2061 li r0,NAPPING_CEDE
2062 stb r0,HSTATE_NAPPING(r13)
2063 lwz r7,VCORE_ENTRY_EXIT(r5)
2064 cmpwi r7,0x100
2065 bge 33f /* another thread already exiting */
2066
2067 /*
2068 * Although not specifically required by the architecture, POWER7
2069 * preserves the following registers in nap mode, even if an SMT mode
2070 * switch occurs: SLB entries, PURR, SPURR, AMOR, UAMOR, AMR, SPRG0-3,
2071 * DAR, DSISR, DABR, DABRX, DSCR, PMCx, MMCRx, SIAR, SDAR.
2072 */
2073 /* Save non-volatile GPRs */
2074 std r14, VCPU_GPR(R14)(r3)
2075 std r15, VCPU_GPR(R15)(r3)
2076 std r16, VCPU_GPR(R16)(r3)
2077 std r17, VCPU_GPR(R17)(r3)
2078 std r18, VCPU_GPR(R18)(r3)
2079 std r19, VCPU_GPR(R19)(r3)
2080 std r20, VCPU_GPR(R20)(r3)
2081 std r21, VCPU_GPR(R21)(r3)
2082 std r22, VCPU_GPR(R22)(r3)
2083 std r23, VCPU_GPR(R23)(r3)
2084 std r24, VCPU_GPR(R24)(r3)
2085 std r25, VCPU_GPR(R25)(r3)
2086 std r26, VCPU_GPR(R26)(r3)
2087 std r27, VCPU_GPR(R27)(r3)
2088 std r28, VCPU_GPR(R28)(r3)
2089 std r29, VCPU_GPR(R29)(r3)
2090 std r30, VCPU_GPR(R30)(r3)
2091 std r31, VCPU_GPR(R31)(r3)
2092
2093 /* save FP state */
2094 bl kvmppc_save_fp
2095
2096 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
2097 BEGIN_FTR_SECTION
2098 ld r9, HSTATE_KVM_VCPU(r13)
2099 bl kvmppc_save_tm
2100 END_FTR_SECTION_IFSET(CPU_FTR_TM)
2101 #endif
2102
2103 /*
2104 * Set DEC to the smaller of DEC and HDEC, so that we wake
2105 * no later than the end of our timeslice (HDEC interrupts
2106 * don't wake us from nap).
2107 */
2108 mfspr r3, SPRN_DEC
2109 mfspr r4, SPRN_HDEC
2110 mftb r5
2111 cmpw r3, r4
2112 ble 67f
2113 mtspr SPRN_DEC, r4
2114 67:
2115 /* save expiry time of guest decrementer */
2116 extsw r3, r3
2117 add r3, r3, r5
2118 ld r4, HSTATE_KVM_VCPU(r13)
2119 ld r5, HSTATE_KVM_VCORE(r13)
2120 ld r6, VCORE_TB_OFFSET(r5)
2121 subf r3, r6, r3 /* convert to host TB value */
2122 std r3, VCPU_DEC_EXPIRES(r4)
2123
2124 #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
2125 ld r4, HSTATE_KVM_VCPU(r13)
2126 addi r3, r4, VCPU_TB_CEDE
2127 bl kvmhv_accumulate_time
2128 #endif
2129
2130 lis r3, LPCR_PECEDP@h /* Do wake on privileged doorbell */
2131
2132 /*
2133 * Take a nap until a decrementer or external or doobell interrupt
2134 * occurs, with PECE1 and PECE0 set in LPCR.
2135 * On POWER8, set PECEDH, and if we are ceding, also set PECEDP.
2136 * Also clear the runlatch bit before napping.
2137 */
2138 kvm_do_nap:
2139 mfspr r0, SPRN_CTRLF
2140 clrrdi r0, r0, 1
2141 mtspr SPRN_CTRLT, r0
2142
2143 li r0,1
2144 stb r0,HSTATE_HWTHREAD_REQ(r13)
2145 mfspr r5,SPRN_LPCR
2146 ori r5,r5,LPCR_PECE0 | LPCR_PECE1
2147 BEGIN_FTR_SECTION
2148 ori r5, r5, LPCR_PECEDH
2149 rlwimi r5, r3, 0, LPCR_PECEDP
2150 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
2151 mtspr SPRN_LPCR,r5
2152 isync
2153 li r0, 0
2154 std r0, HSTATE_SCRATCH0(r13)
2155 ptesync
2156 ld r0, HSTATE_SCRATCH0(r13)
2157 1: cmpd r0, r0
2158 bne 1b
2159 nap
2160 b .
2161
2162 33: mr r4, r3
2163 li r3, 0
2164 li r12, 0
2165 b 34f
2166
2167 kvm_end_cede:
2168 /* get vcpu pointer */
2169 ld r4, HSTATE_KVM_VCPU(r13)
2170
2171 /* Woken by external or decrementer interrupt */
2172 ld r1, HSTATE_HOST_R1(r13)
2173
2174 #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
2175 addi r3, r4, VCPU_TB_RMINTR
2176 bl kvmhv_accumulate_time
2177 #endif
2178
2179 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
2180 BEGIN_FTR_SECTION
2181 bl kvmppc_restore_tm
2182 END_FTR_SECTION_IFSET(CPU_FTR_TM)
2183 #endif
2184
2185 /* load up FP state */
2186 bl kvmppc_load_fp
2187
2188 /* Restore guest decrementer */
2189 ld r3, VCPU_DEC_EXPIRES(r4)
2190 ld r5, HSTATE_KVM_VCORE(r13)
2191 ld r6, VCORE_TB_OFFSET(r5)
2192 add r3, r3, r6 /* convert host TB to guest TB value */
2193 mftb r7
2194 subf r3, r7, r3
2195 mtspr SPRN_DEC, r3
2196
2197 /* Load NV GPRS */
2198 ld r14, VCPU_GPR(R14)(r4)
2199 ld r15, VCPU_GPR(R15)(r4)
2200 ld r16, VCPU_GPR(R16)(r4)
2201 ld r17, VCPU_GPR(R17)(r4)
2202 ld r18, VCPU_GPR(R18)(r4)
2203 ld r19, VCPU_GPR(R19)(r4)
2204 ld r20, VCPU_GPR(R20)(r4)
2205 ld r21, VCPU_GPR(R21)(r4)
2206 ld r22, VCPU_GPR(R22)(r4)
2207 ld r23, VCPU_GPR(R23)(r4)
2208 ld r24, VCPU_GPR(R24)(r4)
2209 ld r25, VCPU_GPR(R25)(r4)
2210 ld r26, VCPU_GPR(R26)(r4)
2211 ld r27, VCPU_GPR(R27)(r4)
2212 ld r28, VCPU_GPR(R28)(r4)
2213 ld r29, VCPU_GPR(R29)(r4)
2214 ld r30, VCPU_GPR(R30)(r4)
2215 ld r31, VCPU_GPR(R31)(r4)
2216
2217 /* Check the wake reason in SRR1 to see why we got here */
2218 bl kvmppc_check_wake_reason
2219
2220 /* clear our bit in vcore->napping_threads */
2221 34: ld r5,HSTATE_KVM_VCORE(r13)
2222 lbz r7,HSTATE_PTID(r13)
2223 li r0,1
2224 sld r0,r0,r7
2225 addi r6,r5,VCORE_NAPPING_THREADS
2226 32: lwarx r7,0,r6
2227 andc r7,r7,r0
2228 stwcx. r7,0,r6
2229 bne 32b
2230 li r0,0
2231 stb r0,HSTATE_NAPPING(r13)
2232
2233 /* See if the wake reason means we need to exit */
2234 stw r12, VCPU_TRAP(r4)
2235 mr r9, r4
2236 cmpdi r3, 0
2237 bgt guest_exit_cont
2238
2239 /* see if any other thread is already exiting */
2240 lwz r0,VCORE_ENTRY_EXIT(r5)
2241 cmpwi r0,0x100
2242 bge guest_exit_cont
2243
2244 b kvmppc_cede_reentry /* if not go back to guest */
2245
2246 /* cede when already previously prodded case */
2247 kvm_cede_prodded:
2248 li r0,0
2249 stb r0,VCPU_PRODDED(r3)
2250 sync /* order testing prodded vs. clearing ceded */
2251 stb r0,VCPU_CEDED(r3)
2252 li r3,H_SUCCESS
2253 blr
2254
2255 /* we've ceded but we want to give control to the host */
2256 kvm_cede_exit:
2257 ld r9, HSTATE_KVM_VCPU(r13)
2258 b guest_exit_cont
2259
2260 /* Try to handle a machine check in real mode */
2261 machine_check_realmode:
2262 mr r3, r9 /* get vcpu pointer */
2263 bl kvmppc_realmode_machine_check
2264 nop
2265 ld r9, HSTATE_KVM_VCPU(r13)
2266 li r12, BOOK3S_INTERRUPT_MACHINE_CHECK
2267 /*
2268 * Deliver unhandled/fatal (e.g. UE) MCE errors to guest through
2269 * machine check interrupt (set HSRR0 to 0x200). And for handled
2270 * errors (no-fatal), just go back to guest execution with current
2271 * HSRR0 instead of exiting guest. This new approach will inject
2272 * machine check to guest for fatal error causing guest to crash.
2273 *
2274 * The old code used to return to host for unhandled errors which
2275 * was causing guest to hang with soft lockups inside guest and
2276 * makes it difficult to recover guest instance.
2277 *
2278 * if we receive machine check with MSR(RI=0) then deliver it to
2279 * guest as machine check causing guest to crash.
2280 */
2281 ld r11, VCPU_MSR(r9)
2282 rldicl. r0, r11, 64-MSR_HV_LG, 63 /* check if it happened in HV mode */
2283 bne mc_cont /* if so, exit to host */
2284 andi. r10, r11, MSR_RI /* check for unrecoverable exception */
2285 beq 1f /* Deliver a machine check to guest */
2286 ld r10, VCPU_PC(r9)
2287 cmpdi r3, 0 /* Did we handle MCE ? */
2288 bne 2f /* Continue guest execution. */
2289 /* If not, deliver a machine check. SRR0/1 are already set */
2290 1: li r10, BOOK3S_INTERRUPT_MACHINE_CHECK
2291 bl kvmppc_msr_interrupt
2292 2: b fast_interrupt_c_return
2293
2294 /*
2295 * Check the reason we woke from nap, and take appropriate action.
2296 * Returns (in r3):
2297 * 0 if nothing needs to be done
2298 * 1 if something happened that needs to be handled by the host
2299 * -1 if there was a guest wakeup (IPI or msgsnd)
2300 *
2301 * Also sets r12 to the interrupt vector for any interrupt that needs
2302 * to be handled now by the host (0x500 for external interrupt), or zero.
2303 * Modifies r0, r6, r7, r8.
2304 */
2305 kvmppc_check_wake_reason:
2306 mfspr r6, SPRN_SRR1
2307 BEGIN_FTR_SECTION
2308 rlwinm r6, r6, 45-31, 0xf /* extract wake reason field (P8) */
2309 FTR_SECTION_ELSE
2310 rlwinm r6, r6, 45-31, 0xe /* P7 wake reason field is 3 bits */
2311 ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_207S)
2312 cmpwi r6, 8 /* was it an external interrupt? */
2313 li r12, BOOK3S_INTERRUPT_EXTERNAL
2314 beq kvmppc_read_intr /* if so, see what it was */
2315 li r3, 0
2316 li r12, 0
2317 cmpwi r6, 6 /* was it the decrementer? */
2318 beq 0f
2319 BEGIN_FTR_SECTION
2320 cmpwi r6, 5 /* privileged doorbell? */
2321 beq 0f
2322 cmpwi r6, 3 /* hypervisor doorbell? */
2323 beq 3f
2324 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
2325 cmpwi r6, 0xa /* Hypervisor maintenance ? */
2326 beq 4f
2327 li r3, 1 /* anything else, return 1 */
2328 0: blr
2329
2330 /* hypervisor doorbell */
2331 3: li r12, BOOK3S_INTERRUPT_H_DOORBELL
2332
2333 /*
2334 * Clear the doorbell as we will invoke the handler
2335 * explicitly in the guest exit path.
2336 */
2337 lis r6, (PPC_DBELL_SERVER << (63-36))@h
2338 PPC_MSGCLR(6)
2339 /* see if it's a host IPI */
2340 li r3, 1
2341 lbz r0, HSTATE_HOST_IPI(r13)
2342 cmpwi r0, 0
2343 bnelr
2344 /* if not, return -1 */
2345 li r3, -1
2346 blr
2347
2348 /* Woken up due to Hypervisor maintenance interrupt */
2349 4: li r12, BOOK3S_INTERRUPT_HMI
2350 li r3, 1
2351 blr
2352
2353 /*
2354 * Determine what sort of external interrupt is pending (if any).
2355 * Returns:
2356 * 0 if no interrupt is pending
2357 * 1 if an interrupt is pending that needs to be handled by the host
2358 * -1 if there was a guest wakeup IPI (which has now been cleared)
2359 * Modifies r0, r6, r7, r8, returns value in r3.
2360 */
2361 kvmppc_read_intr:
2362 /* see if a host IPI is pending */
2363 li r3, 1
2364 lbz r0, HSTATE_HOST_IPI(r13)
2365 cmpwi r0, 0
2366 bne 1f
2367
2368 /* Now read the interrupt from the ICP */
2369 ld r6, HSTATE_XICS_PHYS(r13)
2370 li r7, XICS_XIRR
2371 cmpdi r6, 0
2372 beq- 1f
2373 lwzcix r0, r6, r7
2374 /*
2375 * Save XIRR for later. Since we get in in reverse endian on LE
2376 * systems, save it byte reversed and fetch it back in host endian.
2377 */
2378 li r3, HSTATE_SAVED_XIRR
2379 STWX_BE r0, r3, r13
2380 #ifdef __LITTLE_ENDIAN__
2381 lwz r3, HSTATE_SAVED_XIRR(r13)
2382 #else
2383 mr r3, r0
2384 #endif
2385 rlwinm. r3, r3, 0, 0xffffff
2386 sync
2387 beq 1f /* if nothing pending in the ICP */
2388
2389 /* We found something in the ICP...
2390 *
2391 * If it's not an IPI, stash it in the PACA and return to
2392 * the host, we don't (yet) handle directing real external
2393 * interrupts directly to the guest
2394 */
2395 cmpwi r3, XICS_IPI /* if there is, is it an IPI? */
2396 bne 42f
2397
2398 /* It's an IPI, clear the MFRR and EOI it */
2399 li r3, 0xff
2400 li r8, XICS_MFRR
2401 stbcix r3, r6, r8 /* clear the IPI */
2402 stwcix r0, r6, r7 /* EOI it */
2403 sync
2404
2405 /* We need to re-check host IPI now in case it got set in the
2406 * meantime. If it's clear, we bounce the interrupt to the
2407 * guest
2408 */
2409 lbz r0, HSTATE_HOST_IPI(r13)
2410 cmpwi r0, 0
2411 bne- 43f
2412
2413 /* OK, it's an IPI for us */
2414 li r12, 0
2415 li r3, -1
2416 1: blr
2417
2418 42: /* It's not an IPI and it's for the host. We saved a copy of XIRR in
2419 * the PACA earlier, it will be picked up by the host ICP driver
2420 */
2421 li r3, 1
2422 b 1b
2423
2424 43: /* We raced with the host, we need to resend that IPI, bummer */
2425 li r0, IPI_PRIORITY
2426 stbcix r0, r6, r8 /* set the IPI */
2427 sync
2428 li r3, 1
2429 b 1b
2430
2431 /*
2432 * Save away FP, VMX and VSX registers.
2433 * r3 = vcpu pointer
2434 * N.B. r30 and r31 are volatile across this function,
2435 * thus it is not callable from C.
2436 */
2437 kvmppc_save_fp:
2438 mflr r30
2439 mr r31,r3
2440 mfmsr r5
2441 ori r8,r5,MSR_FP
2442 #ifdef CONFIG_ALTIVEC
2443 BEGIN_FTR_SECTION
2444 oris r8,r8,MSR_VEC@h
2445 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
2446 #endif
2447 #ifdef CONFIG_VSX
2448 BEGIN_FTR_SECTION
2449 oris r8,r8,MSR_VSX@h
2450 END_FTR_SECTION_IFSET(CPU_FTR_VSX)
2451 #endif
2452 mtmsrd r8
2453 addi r3,r3,VCPU_FPRS
2454 bl store_fp_state
2455 #ifdef CONFIG_ALTIVEC
2456 BEGIN_FTR_SECTION
2457 addi r3,r31,VCPU_VRS
2458 bl store_vr_state
2459 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
2460 #endif
2461 mfspr r6,SPRN_VRSAVE
2462 stw r6,VCPU_VRSAVE(r31)
2463 mtlr r30
2464 blr
2465
2466 /*
2467 * Load up FP, VMX and VSX registers
2468 * r4 = vcpu pointer
2469 * N.B. r30 and r31 are volatile across this function,
2470 * thus it is not callable from C.
2471 */
2472 kvmppc_load_fp:
2473 mflr r30
2474 mr r31,r4
2475 mfmsr r9
2476 ori r8,r9,MSR_FP
2477 #ifdef CONFIG_ALTIVEC
2478 BEGIN_FTR_SECTION
2479 oris r8,r8,MSR_VEC@h
2480 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
2481 #endif
2482 #ifdef CONFIG_VSX
2483 BEGIN_FTR_SECTION
2484 oris r8,r8,MSR_VSX@h
2485 END_FTR_SECTION_IFSET(CPU_FTR_VSX)
2486 #endif
2487 mtmsrd r8
2488 addi r3,r4,VCPU_FPRS
2489 bl load_fp_state
2490 #ifdef CONFIG_ALTIVEC
2491 BEGIN_FTR_SECTION
2492 addi r3,r31,VCPU_VRS
2493 bl load_vr_state
2494 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
2495 #endif
2496 lwz r7,VCPU_VRSAVE(r31)
2497 mtspr SPRN_VRSAVE,r7
2498 mtlr r30
2499 mr r4,r31
2500 blr
2501
2502 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
2503 /*
2504 * Save transactional state and TM-related registers.
2505 * Called with r9 pointing to the vcpu struct.
2506 * This can modify all checkpointed registers, but
2507 * restores r1, r2 and r9 (vcpu pointer) before exit.
2508 */
2509 kvmppc_save_tm:
2510 mflr r0
2511 std r0, PPC_LR_STKOFF(r1)
2512
2513 /* Turn on TM. */
2514 mfmsr r8
2515 li r0, 1
2516 rldimi r8, r0, MSR_TM_LG, 63-MSR_TM_LG
2517 mtmsrd r8
2518
2519 ld r5, VCPU_MSR(r9)
2520 rldicl. r5, r5, 64 - MSR_TS_S_LG, 62
2521 beq 1f /* TM not active in guest. */
2522
2523 std r1, HSTATE_HOST_R1(r13)
2524 li r3, TM_CAUSE_KVM_RESCHED
2525
2526 /* Clear the MSR RI since r1, r13 are all going to be foobar. */
2527 li r5, 0
2528 mtmsrd r5, 1
2529
2530 /* All GPRs are volatile at this point. */
2531 TRECLAIM(R3)
2532
2533 /* Temporarily store r13 and r9 so we have some regs to play with */
2534 SET_SCRATCH0(r13)
2535 GET_PACA(r13)
2536 std r9, PACATMSCRATCH(r13)
2537 ld r9, HSTATE_KVM_VCPU(r13)
2538
2539 /* Get a few more GPRs free. */
2540 std r29, VCPU_GPRS_TM(29)(r9)
2541 std r30, VCPU_GPRS_TM(30)(r9)
2542 std r31, VCPU_GPRS_TM(31)(r9)
2543
2544 /* Save away PPR and DSCR soon so don't run with user values. */
2545 mfspr r31, SPRN_PPR
2546 HMT_MEDIUM
2547 mfspr r30, SPRN_DSCR
2548 ld r29, HSTATE_DSCR(r13)
2549 mtspr SPRN_DSCR, r29
2550
2551 /* Save all but r9, r13 & r29-r31 */
2552 reg = 0
2553 .rept 29
2554 .if (reg != 9) && (reg != 13)
2555 std reg, VCPU_GPRS_TM(reg)(r9)
2556 .endif
2557 reg = reg + 1
2558 .endr
2559 /* ... now save r13 */
2560 GET_SCRATCH0(r4)
2561 std r4, VCPU_GPRS_TM(13)(r9)
2562 /* ... and save r9 */
2563 ld r4, PACATMSCRATCH(r13)
2564 std r4, VCPU_GPRS_TM(9)(r9)
2565
2566 /* Reload stack pointer and TOC. */
2567 ld r1, HSTATE_HOST_R1(r13)
2568 ld r2, PACATOC(r13)
2569
2570 /* Set MSR RI now we have r1 and r13 back. */
2571 li r5, MSR_RI
2572 mtmsrd r5, 1
2573
2574 /* Save away checkpinted SPRs. */
2575 std r31, VCPU_PPR_TM(r9)
2576 std r30, VCPU_DSCR_TM(r9)
2577 mflr r5
2578 mfcr r6
2579 mfctr r7
2580 mfspr r8, SPRN_AMR
2581 mfspr r10, SPRN_TAR
2582 std r5, VCPU_LR_TM(r9)
2583 stw r6, VCPU_CR_TM(r9)
2584 std r7, VCPU_CTR_TM(r9)
2585 std r8, VCPU_AMR_TM(r9)
2586 std r10, VCPU_TAR_TM(r9)
2587
2588 /* Restore r12 as trap number. */
2589 lwz r12, VCPU_TRAP(r9)
2590
2591 /* Save FP/VSX. */
2592 addi r3, r9, VCPU_FPRS_TM
2593 bl store_fp_state
2594 addi r3, r9, VCPU_VRS_TM
2595 bl store_vr_state
2596 mfspr r6, SPRN_VRSAVE
2597 stw r6, VCPU_VRSAVE_TM(r9)
2598 1:
2599 /*
2600 * We need to save these SPRs after the treclaim so that the software
2601 * error code is recorded correctly in the TEXASR. Also the user may
2602 * change these outside of a transaction, so they must always be
2603 * context switched.
2604 */
2605 mfspr r5, SPRN_TFHAR
2606 mfspr r6, SPRN_TFIAR
2607 mfspr r7, SPRN_TEXASR
2608 std r5, VCPU_TFHAR(r9)
2609 std r6, VCPU_TFIAR(r9)
2610 std r7, VCPU_TEXASR(r9)
2611
2612 ld r0, PPC_LR_STKOFF(r1)
2613 mtlr r0
2614 blr
2615
2616 /*
2617 * Restore transactional state and TM-related registers.
2618 * Called with r4 pointing to the vcpu struct.
2619 * This potentially modifies all checkpointed registers.
2620 * It restores r1, r2, r4 from the PACA.
2621 */
2622 kvmppc_restore_tm:
2623 mflr r0
2624 std r0, PPC_LR_STKOFF(r1)
2625
2626 /* Turn on TM/FP/VSX/VMX so we can restore them. */
2627 mfmsr r5
2628 li r6, MSR_TM >> 32
2629 sldi r6, r6, 32
2630 or r5, r5, r6
2631 ori r5, r5, MSR_FP
2632 oris r5, r5, (MSR_VEC | MSR_VSX)@h
2633 mtmsrd r5
2634
2635 /*
2636 * The user may change these outside of a transaction, so they must
2637 * always be context switched.
2638 */
2639 ld r5, VCPU_TFHAR(r4)
2640 ld r6, VCPU_TFIAR(r4)
2641 ld r7, VCPU_TEXASR(r4)
2642 mtspr SPRN_TFHAR, r5
2643 mtspr SPRN_TFIAR, r6
2644 mtspr SPRN_TEXASR, r7
2645
2646 ld r5, VCPU_MSR(r4)
2647 rldicl. r5, r5, 64 - MSR_TS_S_LG, 62
2648 beqlr /* TM not active in guest */
2649 std r1, HSTATE_HOST_R1(r13)
2650
2651 /* Make sure the failure summary is set, otherwise we'll program check
2652 * when we trechkpt. It's possible that this might have been not set
2653 * on a kvmppc_set_one_reg() call but we shouldn't let this crash the
2654 * host.
2655 */
2656 oris r7, r7, (TEXASR_FS)@h
2657 mtspr SPRN_TEXASR, r7
2658
2659 /*
2660 * We need to load up the checkpointed state for the guest.
2661 * We need to do this early as it will blow away any GPRs, VSRs and
2662 * some SPRs.
2663 */
2664
2665 mr r31, r4
2666 addi r3, r31, VCPU_FPRS_TM
2667 bl load_fp_state
2668 addi r3, r31, VCPU_VRS_TM
2669 bl load_vr_state
2670 mr r4, r31
2671 lwz r7, VCPU_VRSAVE_TM(r4)
2672 mtspr SPRN_VRSAVE, r7
2673
2674 ld r5, VCPU_LR_TM(r4)
2675 lwz r6, VCPU_CR_TM(r4)
2676 ld r7, VCPU_CTR_TM(r4)
2677 ld r8, VCPU_AMR_TM(r4)
2678 ld r9, VCPU_TAR_TM(r4)
2679 mtlr r5
2680 mtcr r6
2681 mtctr r7
2682 mtspr SPRN_AMR, r8
2683 mtspr SPRN_TAR, r9
2684
2685 /*
2686 * Load up PPR and DSCR values but don't put them in the actual SPRs
2687 * till the last moment to avoid running with userspace PPR and DSCR for
2688 * too long.
2689 */
2690 ld r29, VCPU_DSCR_TM(r4)
2691 ld r30, VCPU_PPR_TM(r4)
2692
2693 std r2, PACATMSCRATCH(r13) /* Save TOC */
2694
2695 /* Clear the MSR RI since r1, r13 are all going to be foobar. */
2696 li r5, 0
2697 mtmsrd r5, 1
2698
2699 /* Load GPRs r0-r28 */
2700 reg = 0
2701 .rept 29
2702 ld reg, VCPU_GPRS_TM(reg)(r31)
2703 reg = reg + 1
2704 .endr
2705
2706 mtspr SPRN_DSCR, r29
2707 mtspr SPRN_PPR, r30
2708
2709 /* Load final GPRs */
2710 ld 29, VCPU_GPRS_TM(29)(r31)
2711 ld 30, VCPU_GPRS_TM(30)(r31)
2712 ld 31, VCPU_GPRS_TM(31)(r31)
2713
2714 /* TM checkpointed state is now setup. All GPRs are now volatile. */
2715 TRECHKPT
2716
2717 /* Now let's get back the state we need. */
2718 HMT_MEDIUM
2719 GET_PACA(r13)
2720 ld r29, HSTATE_DSCR(r13)
2721 mtspr SPRN_DSCR, r29
2722 ld r4, HSTATE_KVM_VCPU(r13)
2723 ld r1, HSTATE_HOST_R1(r13)
2724 ld r2, PACATMSCRATCH(r13)
2725
2726 /* Set the MSR RI since we have our registers back. */
2727 li r5, MSR_RI
2728 mtmsrd r5, 1
2729
2730 ld r0, PPC_LR_STKOFF(r1)
2731 mtlr r0
2732 blr
2733 #endif
2734
2735 /*
2736 * We come here if we get any exception or interrupt while we are
2737 * executing host real mode code while in guest MMU context.
2738 * For now just spin, but we should do something better.
2739 */
2740 kvmppc_bad_host_intr:
2741 b .
2742
2743 /*
2744 * This mimics the MSR transition on IRQ delivery. The new guest MSR is taken
2745 * from VCPU_INTR_MSR and is modified based on the required TM state changes.
2746 * r11 has the guest MSR value (in/out)
2747 * r9 has a vcpu pointer (in)
2748 * r0 is used as a scratch register
2749 */
2750 kvmppc_msr_interrupt:
2751 rldicl r0, r11, 64 - MSR_TS_S_LG, 62
2752 cmpwi r0, 2 /* Check if we are in transactional state.. */
2753 ld r11, VCPU_INTR_MSR(r9)
2754 bne 1f
2755 /* ... if transactional, change to suspended */
2756 li r0, 1
2757 1: rldimi r11, r0, MSR_TS_S_LG, 63 - MSR_TS_T_LG
2758 blr
2759
2760 /*
2761 * This works around a hardware bug on POWER8E processors, where
2762 * writing a 1 to the MMCR0[PMAO] bit doesn't generate a
2763 * performance monitor interrupt. Instead, when we need to have
2764 * an interrupt pending, we have to arrange for a counter to overflow.
2765 */
2766 kvmppc_fix_pmao:
2767 li r3, 0
2768 mtspr SPRN_MMCR2, r3
2769 lis r3, (MMCR0_PMXE | MMCR0_FCECE)@h
2770 ori r3, r3, MMCR0_PMCjCE | MMCR0_C56RUN
2771 mtspr SPRN_MMCR0, r3
2772 lis r3, 0x7fff
2773 ori r3, r3, 0xffff
2774 mtspr SPRN_PMC6, r3
2775 isync
2776 blr
2777
2778 #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
2779 /*
2780 * Start timing an activity
2781 * r3 = pointer to time accumulation struct, r4 = vcpu
2782 */
2783 kvmhv_start_timing:
2784 ld r5, HSTATE_KVM_VCORE(r13)
2785 lbz r6, VCORE_IN_GUEST(r5)
2786 cmpwi r6, 0
2787 beq 5f /* if in guest, need to */
2788 ld r6, VCORE_TB_OFFSET(r5) /* subtract timebase offset */
2789 5: mftb r5
2790 subf r5, r6, r5
2791 std r3, VCPU_CUR_ACTIVITY(r4)
2792 std r5, VCPU_ACTIVITY_START(r4)
2793 blr
2794
2795 /*
2796 * Accumulate time to one activity and start another.
2797 * r3 = pointer to new time accumulation struct, r4 = vcpu
2798 */
2799 kvmhv_accumulate_time:
2800 ld r5, HSTATE_KVM_VCORE(r13)
2801 lbz r8, VCORE_IN_GUEST(r5)
2802 cmpwi r8, 0
2803 beq 4f /* if in guest, need to */
2804 ld r8, VCORE_TB_OFFSET(r5) /* subtract timebase offset */
2805 4: ld r5, VCPU_CUR_ACTIVITY(r4)
2806 ld r6, VCPU_ACTIVITY_START(r4)
2807 std r3, VCPU_CUR_ACTIVITY(r4)
2808 mftb r7
2809 subf r7, r8, r7
2810 std r7, VCPU_ACTIVITY_START(r4)
2811 cmpdi r5, 0
2812 beqlr
2813 subf r3, r6, r7
2814 ld r8, TAS_SEQCOUNT(r5)
2815 cmpdi r8, 0
2816 addi r8, r8, 1
2817 std r8, TAS_SEQCOUNT(r5)
2818 lwsync
2819 ld r7, TAS_TOTAL(r5)
2820 add r7, r7, r3
2821 std r7, TAS_TOTAL(r5)
2822 ld r6, TAS_MIN(r5)
2823 ld r7, TAS_MAX(r5)
2824 beq 3f
2825 cmpd r3, r6
2826 bge 1f
2827 3: std r3, TAS_MIN(r5)
2828 1: cmpd r3, r7
2829 ble 2f
2830 std r3, TAS_MAX(r5)
2831 2: lwsync
2832 addi r8, r8, 1
2833 std r8, TAS_SEQCOUNT(r5)
2834 blr
2835 #endif
Linux kernel - Deliverables
This page took 0.144699 seconds and 5 git commands to generate.