powerpc: Macros for saving/restore PPR
[deliverable/linux.git] / arch / powerpc / kernel / entry_64.S
1 /*
2 * PowerPC version
3 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
4 * Rewritten by Cort Dougan (cort@cs.nmt.edu) for PReP
5 * Copyright (C) 1996 Cort Dougan <cort@cs.nmt.edu>
6 * Adapted for Power Macintosh by Paul Mackerras.
7 * Low-level exception handlers and MMU support
8 * rewritten by Paul Mackerras.
9 * Copyright (C) 1996 Paul Mackerras.
10 * MPC8xx modifications Copyright (C) 1997 Dan Malek (dmalek@jlc.net).
11 *
12 * This file contains the system call entry code, context switch
13 * code, and exception/interrupt return code for PowerPC.
14 *
15 * This program is free software; you can redistribute it and/or
16 * modify it under the terms of the GNU General Public License
17 * as published by the Free Software Foundation; either version
18 * 2 of the License, or (at your option) any later version.
19 */
20
21 #include <linux/errno.h>
22 #include <asm/unistd.h>
23 #include <asm/processor.h>
24 #include <asm/page.h>
25 #include <asm/mmu.h>
26 #include <asm/thread_info.h>
27 #include <asm/ppc_asm.h>
28 #include <asm/asm-offsets.h>
29 #include <asm/cputable.h>
30 #include <asm/firmware.h>
31 #include <asm/bug.h>
32 #include <asm/ptrace.h>
33 #include <asm/irqflags.h>
34 #include <asm/ftrace.h>
35 #include <asm/hw_irq.h>
36
37 /*
38 * System calls.
39 */
40 .section ".toc","aw"
41 .SYS_CALL_TABLE:
42 .tc .sys_call_table[TC],.sys_call_table
43
44 /* This value is used to mark exception frames on the stack. */
45 exception_marker:
46 .tc ID_EXC_MARKER[TC],STACK_FRAME_REGS_MARKER
47
48 .section ".text"
49 .align 7
50
51 #undef SHOW_SYSCALLS
52
53 .globl system_call_common
54 system_call_common:
55 andi. r10,r12,MSR_PR
56 mr r10,r1
57 addi r1,r1,-INT_FRAME_SIZE
58 beq- 1f
59 ld r1,PACAKSAVE(r13)
60 1: std r10,0(r1)
61 std r11,_NIP(r1)
62 std r12,_MSR(r1)
63 std r0,GPR0(r1)
64 std r10,GPR1(r1)
65 beq 2f /* if from kernel mode */
66 ACCOUNT_CPU_USER_ENTRY(r10, r11)
67 2: std r2,GPR2(r1)
68 std r3,GPR3(r1)
69 mfcr r2
70 std r4,GPR4(r1)
71 std r5,GPR5(r1)
72 std r6,GPR6(r1)
73 std r7,GPR7(r1)
74 std r8,GPR8(r1)
75 li r11,0
76 std r11,GPR9(r1)
77 std r11,GPR10(r1)
78 std r11,GPR11(r1)
79 std r11,GPR12(r1)
80 std r11,_XER(r1)
81 std r11,_CTR(r1)
82 std r9,GPR13(r1)
83 mflr r10
84 /*
85 * This clears CR0.SO (bit 28), which is the error indication on
86 * return from this system call.
87 */
88 rldimi r2,r11,28,(63-28)
89 li r11,0xc01
90 std r10,_LINK(r1)
91 std r11,_TRAP(r1)
92 std r3,ORIG_GPR3(r1)
93 std r2,_CCR(r1)
94 ld r2,PACATOC(r13)
95 addi r9,r1,STACK_FRAME_OVERHEAD
96 ld r11,exception_marker@toc(r2)
97 std r11,-16(r9) /* "regshere" marker */
98 #if defined(CONFIG_VIRT_CPU_ACCOUNTING) && defined(CONFIG_PPC_SPLPAR)
99 BEGIN_FW_FTR_SECTION
100 beq 33f
101 /* if from user, see if there are any DTL entries to process */
102 ld r10,PACALPPACAPTR(r13) /* get ptr to VPA */
103 ld r11,PACA_DTL_RIDX(r13) /* get log read index */
104 ld r10,LPPACA_DTLIDX(r10) /* get log write index */
105 cmpd cr1,r11,r10
106 beq+ cr1,33f
107 bl .accumulate_stolen_time
108 REST_GPR(0,r1)
109 REST_4GPRS(3,r1)
110 REST_2GPRS(7,r1)
111 addi r9,r1,STACK_FRAME_OVERHEAD
112 33:
113 END_FW_FTR_SECTION_IFSET(FW_FEATURE_SPLPAR)
114 #endif /* CONFIG_VIRT_CPU_ACCOUNTING && CONFIG_PPC_SPLPAR */
115
116 /*
117 * A syscall should always be called with interrupts enabled
118 * so we just unconditionally hard-enable here. When some kind
119 * of irq tracing is used, we additionally check that condition
120 * is correct
121 */
122 #if defined(CONFIG_TRACE_IRQFLAGS) && defined(CONFIG_BUG)
123 lbz r10,PACASOFTIRQEN(r13)
124 xori r10,r10,1
125 1: tdnei r10,0
126 EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,BUGFLAG_WARNING
127 #endif
128
129 #ifdef CONFIG_PPC_BOOK3E
130 wrteei 1
131 #else
132 ld r11,PACAKMSR(r13)
133 ori r11,r11,MSR_EE
134 mtmsrd r11,1
135 #endif /* CONFIG_PPC_BOOK3E */
136
137 /* We do need to set SOFTE in the stack frame or the return
138 * from interrupt will be painful
139 */
140 li r10,1
141 std r10,SOFTE(r1)
142
143 #ifdef SHOW_SYSCALLS
144 bl .do_show_syscall
145 REST_GPR(0,r1)
146 REST_4GPRS(3,r1)
147 REST_2GPRS(7,r1)
148 addi r9,r1,STACK_FRAME_OVERHEAD
149 #endif
150 CURRENT_THREAD_INFO(r11, r1)
151 ld r10,TI_FLAGS(r11)
152 andi. r11,r10,_TIF_SYSCALL_T_OR_A
153 bne- syscall_dotrace
154 .Lsyscall_dotrace_cont:
155 cmpldi 0,r0,NR_syscalls
156 bge- syscall_enosys
157
158 system_call: /* label this so stack traces look sane */
159 /*
160 * Need to vector to 32 Bit or default sys_call_table here,
161 * based on caller's run-mode / personality.
162 */
163 ld r11,.SYS_CALL_TABLE@toc(2)
164 andi. r10,r10,_TIF_32BIT
165 beq 15f
166 addi r11,r11,8 /* use 32-bit syscall entries */
167 clrldi r3,r3,32
168 clrldi r4,r4,32
169 clrldi r5,r5,32
170 clrldi r6,r6,32
171 clrldi r7,r7,32
172 clrldi r8,r8,32
173 15:
174 slwi r0,r0,4
175 ldx r10,r11,r0 /* Fetch system call handler [ptr] */
176 mtctr r10
177 bctrl /* Call handler */
178
179 syscall_exit:
180 std r3,RESULT(r1)
181 #ifdef SHOW_SYSCALLS
182 bl .do_show_syscall_exit
183 ld r3,RESULT(r1)
184 #endif
185 CURRENT_THREAD_INFO(r12, r1)
186
187 ld r8,_MSR(r1)
188 #ifdef CONFIG_PPC_BOOK3S
189 /* No MSR:RI on BookE */
190 andi. r10,r8,MSR_RI
191 beq- unrecov_restore
192 #endif
193 /*
194 * Disable interrupts so current_thread_info()->flags can't change,
195 * and so that we don't get interrupted after loading SRR0/1.
196 */
197 #ifdef CONFIG_PPC_BOOK3E
198 wrteei 0
199 #else
200 ld r10,PACAKMSR(r13)
201 /*
202 * For performance reasons we clear RI the same time that we
203 * clear EE. We only need to clear RI just before we restore r13
204 * below, but batching it with EE saves us one expensive mtmsrd call.
205 * We have to be careful to restore RI if we branch anywhere from
206 * here (eg syscall_exit_work).
207 */
208 li r9,MSR_RI
209 andc r11,r10,r9
210 mtmsrd r11,1
211 #endif /* CONFIG_PPC_BOOK3E */
212
213 ld r9,TI_FLAGS(r12)
214 li r11,-_LAST_ERRNO
215 andi. r0,r9,(_TIF_SYSCALL_T_OR_A|_TIF_SINGLESTEP|_TIF_USER_WORK_MASK|_TIF_PERSYSCALL_MASK)
216 bne- syscall_exit_work
217 cmpld r3,r11
218 ld r5,_CCR(r1)
219 bge- syscall_error
220 .Lsyscall_error_cont:
221 ld r7,_NIP(r1)
222 BEGIN_FTR_SECTION
223 stdcx. r0,0,r1 /* to clear the reservation */
224 END_FTR_SECTION_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
225 andi. r6,r8,MSR_PR
226 ld r4,_LINK(r1)
227
228 beq- 1f
229 ACCOUNT_CPU_USER_EXIT(r11, r12)
230 ld r13,GPR13(r1) /* only restore r13 if returning to usermode */
231 1: ld r2,GPR2(r1)
232 ld r1,GPR1(r1)
233 mtlr r4
234 mtcr r5
235 mtspr SPRN_SRR0,r7
236 mtspr SPRN_SRR1,r8
237 RFI
238 b . /* prevent speculative execution */
239
240 syscall_error:
241 oris r5,r5,0x1000 /* Set SO bit in CR */
242 neg r3,r3
243 std r5,_CCR(r1)
244 b .Lsyscall_error_cont
245
246 /* Traced system call support */
247 syscall_dotrace:
248 bl .save_nvgprs
249 addi r3,r1,STACK_FRAME_OVERHEAD
250 bl .do_syscall_trace_enter
251 /*
252 * Restore argument registers possibly just changed.
253 * We use the return value of do_syscall_trace_enter
254 * for the call number to look up in the table (r0).
255 */
256 mr r0,r3
257 ld r3,GPR3(r1)
258 ld r4,GPR4(r1)
259 ld r5,GPR5(r1)
260 ld r6,GPR6(r1)
261 ld r7,GPR7(r1)
262 ld r8,GPR8(r1)
263 addi r9,r1,STACK_FRAME_OVERHEAD
264 CURRENT_THREAD_INFO(r10, r1)
265 ld r10,TI_FLAGS(r10)
266 b .Lsyscall_dotrace_cont
267
268 syscall_enosys:
269 li r3,-ENOSYS
270 b syscall_exit
271
272 syscall_exit_work:
273 #ifdef CONFIG_PPC_BOOK3S
274 mtmsrd r10,1 /* Restore RI */
275 #endif
276 /* If TIF_RESTOREALL is set, don't scribble on either r3 or ccr.
277 If TIF_NOERROR is set, just save r3 as it is. */
278
279 andi. r0,r9,_TIF_RESTOREALL
280 beq+ 0f
281 REST_NVGPRS(r1)
282 b 2f
283 0: cmpld r3,r11 /* r10 is -LAST_ERRNO */
284 blt+ 1f
285 andi. r0,r9,_TIF_NOERROR
286 bne- 1f
287 ld r5,_CCR(r1)
288 neg r3,r3
289 oris r5,r5,0x1000 /* Set SO bit in CR */
290 std r5,_CCR(r1)
291 1: std r3,GPR3(r1)
292 2: andi. r0,r9,(_TIF_PERSYSCALL_MASK)
293 beq 4f
294
295 /* Clear per-syscall TIF flags if any are set. */
296
297 li r11,_TIF_PERSYSCALL_MASK
298 addi r12,r12,TI_FLAGS
299 3: ldarx r10,0,r12
300 andc r10,r10,r11
301 stdcx. r10,0,r12
302 bne- 3b
303 subi r12,r12,TI_FLAGS
304
305 4: /* Anything else left to do? */
306 andi. r0,r9,(_TIF_SYSCALL_T_OR_A|_TIF_SINGLESTEP)
307 beq .ret_from_except_lite
308
309 /* Re-enable interrupts */
310 #ifdef CONFIG_PPC_BOOK3E
311 wrteei 1
312 #else
313 ld r10,PACAKMSR(r13)
314 ori r10,r10,MSR_EE
315 mtmsrd r10,1
316 #endif /* CONFIG_PPC_BOOK3E */
317
318 bl .save_nvgprs
319 addi r3,r1,STACK_FRAME_OVERHEAD
320 bl .do_syscall_trace_leave
321 b .ret_from_except
322
323 /* Save non-volatile GPRs, if not already saved. */
324 _GLOBAL(save_nvgprs)
325 ld r11,_TRAP(r1)
326 andi. r0,r11,1
327 beqlr-
328 SAVE_NVGPRS(r1)
329 clrrdi r0,r11,1
330 std r0,_TRAP(r1)
331 blr
332
333
334 /*
335 * The sigsuspend and rt_sigsuspend system calls can call do_signal
336 * and thus put the process into the stopped state where we might
337 * want to examine its user state with ptrace. Therefore we need
338 * to save all the nonvolatile registers (r14 - r31) before calling
339 * the C code. Similarly, fork, vfork and clone need the full
340 * register state on the stack so that it can be copied to the child.
341 */
342
343 _GLOBAL(ppc_fork)
344 bl .save_nvgprs
345 bl .sys_fork
346 b syscall_exit
347
348 _GLOBAL(ppc_vfork)
349 bl .save_nvgprs
350 bl .sys_vfork
351 b syscall_exit
352
353 _GLOBAL(ppc_clone)
354 bl .save_nvgprs
355 bl .sys_clone
356 b syscall_exit
357
358 _GLOBAL(ppc32_swapcontext)
359 bl .save_nvgprs
360 bl .compat_sys_swapcontext
361 b syscall_exit
362
363 _GLOBAL(ppc64_swapcontext)
364 bl .save_nvgprs
365 bl .sys_swapcontext
366 b syscall_exit
367
368 _GLOBAL(ret_from_fork)
369 bl .schedule_tail
370 REST_NVGPRS(r1)
371 li r3,0
372 b syscall_exit
373
374 _GLOBAL(ret_from_kernel_thread)
375 bl .schedule_tail
376 REST_NVGPRS(r1)
377 li r3,0
378 std r3,0(r1)
379 ld r14, 0(r14)
380 mtlr r14
381 mr r3,r15
382 blrl
383 li r3,0
384 b syscall_exit
385
386 .section ".toc","aw"
387 DSCR_DEFAULT:
388 .tc dscr_default[TC],dscr_default
389
390 .section ".text"
391
392 /*
393 * This routine switches between two different tasks. The process
394 * state of one is saved on its kernel stack. Then the state
395 * of the other is restored from its kernel stack. The memory
396 * management hardware is updated to the second process's state.
397 * Finally, we can return to the second process, via ret_from_except.
398 * On entry, r3 points to the THREAD for the current task, r4
399 * points to the THREAD for the new task.
400 *
401 * Note: there are two ways to get to the "going out" portion
402 * of this code; either by coming in via the entry (_switch)
403 * or via "fork" which must set up an environment equivalent
404 * to the "_switch" path. If you change this you'll have to change
405 * the fork code also.
406 *
407 * The code which creates the new task context is in 'copy_thread'
408 * in arch/powerpc/kernel/process.c
409 */
410 .align 7
411 _GLOBAL(_switch)
412 mflr r0
413 std r0,16(r1)
414 stdu r1,-SWITCH_FRAME_SIZE(r1)
415 /* r3-r13 are caller saved -- Cort */
416 SAVE_8GPRS(14, r1)
417 SAVE_10GPRS(22, r1)
418 mflr r20 /* Return to switch caller */
419 mfmsr r22
420 li r0, MSR_FP
421 #ifdef CONFIG_VSX
422 BEGIN_FTR_SECTION
423 oris r0,r0,MSR_VSX@h /* Disable VSX */
424 END_FTR_SECTION_IFSET(CPU_FTR_VSX)
425 #endif /* CONFIG_VSX */
426 #ifdef CONFIG_ALTIVEC
427 BEGIN_FTR_SECTION
428 oris r0,r0,MSR_VEC@h /* Disable altivec */
429 mfspr r24,SPRN_VRSAVE /* save vrsave register value */
430 std r24,THREAD_VRSAVE(r3)
431 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
432 #endif /* CONFIG_ALTIVEC */
433 #ifdef CONFIG_PPC64
434 BEGIN_FTR_SECTION
435 mfspr r25,SPRN_DSCR
436 std r25,THREAD_DSCR(r3)
437 END_FTR_SECTION_IFSET(CPU_FTR_DSCR)
438 #endif
439 and. r0,r0,r22
440 beq+ 1f
441 andc r22,r22,r0
442 MTMSRD(r22)
443 isync
444 1: std r20,_NIP(r1)
445 mfcr r23
446 std r23,_CCR(r1)
447 std r1,KSP(r3) /* Set old stack pointer */
448
449 #ifdef CONFIG_SMP
450 /* We need a sync somewhere here to make sure that if the
451 * previous task gets rescheduled on another CPU, it sees all
452 * stores it has performed on this one.
453 */
454 sync
455 #endif /* CONFIG_SMP */
456
457 /*
458 * If we optimise away the clear of the reservation in system
459 * calls because we know the CPU tracks the address of the
460 * reservation, then we need to clear it here to cover the
461 * case that the kernel context switch path has no larx
462 * instructions.
463 */
464 BEGIN_FTR_SECTION
465 ldarx r6,0,r1
466 END_FTR_SECTION_IFSET(CPU_FTR_STCX_CHECKS_ADDRESS)
467
468 addi r6,r4,-THREAD /* Convert THREAD to 'current' */
469 std r6,PACACURRENT(r13) /* Set new 'current' */
470
471 ld r8,KSP(r4) /* new stack pointer */
472 #ifdef CONFIG_PPC_BOOK3S
473 BEGIN_FTR_SECTION
474 BEGIN_FTR_SECTION_NESTED(95)
475 clrrdi r6,r8,28 /* get its ESID */
476 clrrdi r9,r1,28 /* get current sp ESID */
477 FTR_SECTION_ELSE_NESTED(95)
478 clrrdi r6,r8,40 /* get its 1T ESID */
479 clrrdi r9,r1,40 /* get current sp 1T ESID */
480 ALT_MMU_FTR_SECTION_END_NESTED_IFCLR(MMU_FTR_1T_SEGMENT, 95)
481 FTR_SECTION_ELSE
482 b 2f
483 ALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_SLB)
484 clrldi. r0,r6,2 /* is new ESID c00000000? */
485 cmpd cr1,r6,r9 /* or is new ESID the same as current ESID? */
486 cror eq,4*cr1+eq,eq
487 beq 2f /* if yes, don't slbie it */
488
489 /* Bolt in the new stack SLB entry */
490 ld r7,KSP_VSID(r4) /* Get new stack's VSID */
491 oris r0,r6,(SLB_ESID_V)@h
492 ori r0,r0,(SLB_NUM_BOLTED-1)@l
493 BEGIN_FTR_SECTION
494 li r9,MMU_SEGSIZE_1T /* insert B field */
495 oris r6,r6,(MMU_SEGSIZE_1T << SLBIE_SSIZE_SHIFT)@h
496 rldimi r7,r9,SLB_VSID_SSIZE_SHIFT,0
497 END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT)
498
499 /* Update the last bolted SLB. No write barriers are needed
500 * here, provided we only update the current CPU's SLB shadow
501 * buffer.
502 */
503 ld r9,PACA_SLBSHADOWPTR(r13)
504 li r12,0
505 std r12,SLBSHADOW_STACKESID(r9) /* Clear ESID */
506 std r7,SLBSHADOW_STACKVSID(r9) /* Save VSID */
507 std r0,SLBSHADOW_STACKESID(r9) /* Save ESID */
508
509 /* No need to check for MMU_FTR_NO_SLBIE_B here, since when
510 * we have 1TB segments, the only CPUs known to have the errata
511 * only support less than 1TB of system memory and we'll never
512 * actually hit this code path.
513 */
514
515 slbie r6
516 slbie r6 /* Workaround POWER5 < DD2.1 issue */
517 slbmte r7,r0
518 isync
519 2:
520 #endif /* !CONFIG_PPC_BOOK3S */
521
522 CURRENT_THREAD_INFO(r7, r8) /* base of new stack */
523 /* Note: this uses SWITCH_FRAME_SIZE rather than INT_FRAME_SIZE
524 because we don't need to leave the 288-byte ABI gap at the
525 top of the kernel stack. */
526 addi r7,r7,THREAD_SIZE-SWITCH_FRAME_SIZE
527
528 mr r1,r8 /* start using new stack pointer */
529 std r7,PACAKSAVE(r13)
530
531 #ifdef CONFIG_ALTIVEC
532 BEGIN_FTR_SECTION
533 ld r0,THREAD_VRSAVE(r4)
534 mtspr SPRN_VRSAVE,r0 /* if G4, restore VRSAVE reg */
535 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
536 #endif /* CONFIG_ALTIVEC */
537 #ifdef CONFIG_PPC64
538 BEGIN_FTR_SECTION
539 lwz r6,THREAD_DSCR_INHERIT(r4)
540 ld r7,DSCR_DEFAULT@toc(2)
541 ld r0,THREAD_DSCR(r4)
542 cmpwi r6,0
543 bne 1f
544 ld r0,0(r7)
545 1: cmpd r0,r25
546 beq 2f
547 mtspr SPRN_DSCR,r0
548 2:
549 END_FTR_SECTION_IFSET(CPU_FTR_DSCR)
550 #endif
551
552 ld r6,_CCR(r1)
553 mtcrf 0xFF,r6
554
555 /* r3-r13 are destroyed -- Cort */
556 REST_8GPRS(14, r1)
557 REST_10GPRS(22, r1)
558
559 /* convert old thread to its task_struct for return value */
560 addi r3,r3,-THREAD
561 ld r7,_NIP(r1) /* Return to _switch caller in new task */
562 mtlr r7
563 addi r1,r1,SWITCH_FRAME_SIZE
564 blr
565
566 .align 7
567 _GLOBAL(ret_from_except)
568 ld r11,_TRAP(r1)
569 andi. r0,r11,1
570 bne .ret_from_except_lite
571 REST_NVGPRS(r1)
572
573 _GLOBAL(ret_from_except_lite)
574 /*
575 * Disable interrupts so that current_thread_info()->flags
576 * can't change between when we test it and when we return
577 * from the interrupt.
578 */
579 #ifdef CONFIG_PPC_BOOK3E
580 wrteei 0
581 #else
582 ld r10,PACAKMSR(r13) /* Get kernel MSR without EE */
583 mtmsrd r10,1 /* Update machine state */
584 #endif /* CONFIG_PPC_BOOK3E */
585
586 CURRENT_THREAD_INFO(r9, r1)
587 ld r3,_MSR(r1)
588 ld r4,TI_FLAGS(r9)
589 andi. r3,r3,MSR_PR
590 beq resume_kernel
591
592 /* Check current_thread_info()->flags */
593 andi. r0,r4,_TIF_USER_WORK_MASK
594 beq restore
595
596 andi. r0,r4,_TIF_NEED_RESCHED
597 beq 1f
598 bl .restore_interrupts
599 bl .schedule
600 b .ret_from_except_lite
601
602 1: bl .save_nvgprs
603 bl .restore_interrupts
604 addi r3,r1,STACK_FRAME_OVERHEAD
605 bl .do_notify_resume
606 b .ret_from_except
607
608 resume_kernel:
609 /* check current_thread_info, _TIF_EMULATE_STACK_STORE */
610 CURRENT_THREAD_INFO(r9, r1)
611 ld r8,TI_FLAGS(r9)
612 andis. r8,r8,_TIF_EMULATE_STACK_STORE@h
613 beq+ 1f
614
615 addi r8,r1,INT_FRAME_SIZE /* Get the kprobed function entry */
616
617 lwz r3,GPR1(r1)
618 subi r3,r3,INT_FRAME_SIZE /* dst: Allocate a trampoline exception frame */
619 mr r4,r1 /* src: current exception frame */
620 mr r1,r3 /* Reroute the trampoline frame to r1 */
621
622 /* Copy from the original to the trampoline. */
623 li r5,INT_FRAME_SIZE/8 /* size: INT_FRAME_SIZE */
624 li r6,0 /* start offset: 0 */
625 mtctr r5
626 2: ldx r0,r6,r4
627 stdx r0,r6,r3
628 addi r6,r6,8
629 bdnz 2b
630
631 /* Do real store operation to complete stwu */
632 lwz r5,GPR1(r1)
633 std r8,0(r5)
634
635 /* Clear _TIF_EMULATE_STACK_STORE flag */
636 lis r11,_TIF_EMULATE_STACK_STORE@h
637 addi r5,r9,TI_FLAGS
638 ldarx r4,0,r5
639 andc r4,r4,r11
640 stdcx. r4,0,r5
641 bne- 0b
642 1:
643
644 #ifdef CONFIG_PREEMPT
645 /* Check if we need to preempt */
646 andi. r0,r4,_TIF_NEED_RESCHED
647 beq+ restore
648 /* Check that preempt_count() == 0 and interrupts are enabled */
649 lwz r8,TI_PREEMPT(r9)
650 cmpwi cr1,r8,0
651 ld r0,SOFTE(r1)
652 cmpdi r0,0
653 crandc eq,cr1*4+eq,eq
654 bne restore
655
656 /*
657 * Here we are preempting the current task. We want to make
658 * sure we are soft-disabled first
659 */
660 SOFT_DISABLE_INTS(r3,r4)
661 1: bl .preempt_schedule_irq
662
663 /* Re-test flags and eventually loop */
664 CURRENT_THREAD_INFO(r9, r1)
665 ld r4,TI_FLAGS(r9)
666 andi. r0,r4,_TIF_NEED_RESCHED
667 bne 1b
668 #endif /* CONFIG_PREEMPT */
669
670 .globl fast_exc_return_irq
671 fast_exc_return_irq:
672 restore:
673 /*
674 * This is the main kernel exit path. First we check if we
675 * are about to re-enable interrupts
676 */
677 ld r5,SOFTE(r1)
678 lbz r6,PACASOFTIRQEN(r13)
679 cmpwi cr0,r5,0
680 beq restore_irq_off
681
682 /* We are enabling, were we already enabled ? Yes, just return */
683 cmpwi cr0,r6,1
684 beq cr0,do_restore
685
686 /*
687 * We are about to soft-enable interrupts (we are hard disabled
688 * at this point). We check if there's anything that needs to
689 * be replayed first.
690 */
691 lbz r0,PACAIRQHAPPENED(r13)
692 cmpwi cr0,r0,0
693 bne- restore_check_irq_replay
694
695 /*
696 * Get here when nothing happened while soft-disabled, just
697 * soft-enable and move-on. We will hard-enable as a side
698 * effect of rfi
699 */
700 restore_no_replay:
701 TRACE_ENABLE_INTS
702 li r0,1
703 stb r0,PACASOFTIRQEN(r13);
704
705 /*
706 * Final return path. BookE is handled in a different file
707 */
708 do_restore:
709 #ifdef CONFIG_PPC_BOOK3E
710 b .exception_return_book3e
711 #else
712 /*
713 * Clear the reservation. If we know the CPU tracks the address of
714 * the reservation then we can potentially save some cycles and use
715 * a larx. On POWER6 and POWER7 this is significantly faster.
716 */
717 BEGIN_FTR_SECTION
718 stdcx. r0,0,r1 /* to clear the reservation */
719 FTR_SECTION_ELSE
720 ldarx r4,0,r1
721 ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
722
723 /*
724 * Some code path such as load_up_fpu or altivec return directly
725 * here. They run entirely hard disabled and do not alter the
726 * interrupt state. They also don't use lwarx/stwcx. and thus
727 * are known not to leave dangling reservations.
728 */
729 .globl fast_exception_return
730 fast_exception_return:
731 ld r3,_MSR(r1)
732 ld r4,_CTR(r1)
733 ld r0,_LINK(r1)
734 mtctr r4
735 mtlr r0
736 ld r4,_XER(r1)
737 mtspr SPRN_XER,r4
738
739 REST_8GPRS(5, r1)
740
741 andi. r0,r3,MSR_RI
742 beq- unrecov_restore
743
744 /*
745 * Clear RI before restoring r13. If we are returning to
746 * userspace and we take an exception after restoring r13,
747 * we end up corrupting the userspace r13 value.
748 */
749 ld r4,PACAKMSR(r13) /* Get kernel MSR without EE */
750 andc r4,r4,r0 /* r0 contains MSR_RI here */
751 mtmsrd r4,1
752
753 /*
754 * r13 is our per cpu area, only restore it if we are returning to
755 * userspace the value stored in the stack frame may belong to
756 * another CPU.
757 */
758 andi. r0,r3,MSR_PR
759 beq 1f
760 ACCOUNT_CPU_USER_EXIT(r2, r4)
761 REST_GPR(13, r1)
762 1:
763 mtspr SPRN_SRR1,r3
764
765 ld r2,_CCR(r1)
766 mtcrf 0xFF,r2
767 ld r2,_NIP(r1)
768 mtspr SPRN_SRR0,r2
769
770 ld r0,GPR0(r1)
771 ld r2,GPR2(r1)
772 ld r3,GPR3(r1)
773 ld r4,GPR4(r1)
774 ld r1,GPR1(r1)
775
776 rfid
777 b . /* prevent speculative execution */
778
779 #endif /* CONFIG_PPC_BOOK3E */
780
781 /*
782 * We are returning to a context with interrupts soft disabled.
783 *
784 * However, we may also about to hard enable, so we need to
785 * make sure that in this case, we also clear PACA_IRQ_HARD_DIS
786 * or that bit can get out of sync and bad things will happen
787 */
788 restore_irq_off:
789 ld r3,_MSR(r1)
790 lbz r7,PACAIRQHAPPENED(r13)
791 andi. r0,r3,MSR_EE
792 beq 1f
793 rlwinm r7,r7,0,~PACA_IRQ_HARD_DIS
794 stb r7,PACAIRQHAPPENED(r13)
795 1: li r0,0
796 stb r0,PACASOFTIRQEN(r13);
797 TRACE_DISABLE_INTS
798 b do_restore
799
800 /*
801 * Something did happen, check if a re-emit is needed
802 * (this also clears paca->irq_happened)
803 */
804 restore_check_irq_replay:
805 /* XXX: We could implement a fast path here where we check
806 * for irq_happened being just 0x01, in which case we can
807 * clear it and return. That means that we would potentially
808 * miss a decrementer having wrapped all the way around.
809 *
810 * Still, this might be useful for things like hash_page
811 */
812 bl .__check_irq_replay
813 cmpwi cr0,r3,0
814 beq restore_no_replay
815
816 /*
817 * We need to re-emit an interrupt. We do so by re-using our
818 * existing exception frame. We first change the trap value,
819 * but we need to ensure we preserve the low nibble of it
820 */
821 ld r4,_TRAP(r1)
822 clrldi r4,r4,60
823 or r4,r4,r3
824 std r4,_TRAP(r1)
825
826 /*
827 * Then find the right handler and call it. Interrupts are
828 * still soft-disabled and we keep them that way.
829 */
830 cmpwi cr0,r3,0x500
831 bne 1f
832 addi r3,r1,STACK_FRAME_OVERHEAD;
833 bl .do_IRQ
834 b .ret_from_except
835 1: cmpwi cr0,r3,0x900
836 bne 1f
837 addi r3,r1,STACK_FRAME_OVERHEAD;
838 bl .timer_interrupt
839 b .ret_from_except
840 #ifdef CONFIG_PPC_DOORBELL
841 1:
842 #ifdef CONFIG_PPC_BOOK3E
843 cmpwi cr0,r3,0x280
844 #else
845 BEGIN_FTR_SECTION
846 cmpwi cr0,r3,0xe80
847 FTR_SECTION_ELSE
848 cmpwi cr0,r3,0xa00
849 ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE)
850 #endif /* CONFIG_PPC_BOOK3E */
851 bne 1f
852 addi r3,r1,STACK_FRAME_OVERHEAD;
853 bl .doorbell_exception
854 b .ret_from_except
855 #endif /* CONFIG_PPC_DOORBELL */
856 1: b .ret_from_except /* What else to do here ? */
857
858 unrecov_restore:
859 addi r3,r1,STACK_FRAME_OVERHEAD
860 bl .unrecoverable_exception
861 b unrecov_restore
862
863 #ifdef CONFIG_PPC_RTAS
864 /*
865 * On CHRP, the Run-Time Abstraction Services (RTAS) have to be
866 * called with the MMU off.
867 *
868 * In addition, we need to be in 32b mode, at least for now.
869 *
870 * Note: r3 is an input parameter to rtas, so don't trash it...
871 */
872 _GLOBAL(enter_rtas)
873 mflr r0
874 std r0,16(r1)
875 stdu r1,-RTAS_FRAME_SIZE(r1) /* Save SP and create stack space. */
876
877 /* Because RTAS is running in 32b mode, it clobbers the high order half
878 * of all registers that it saves. We therefore save those registers
879 * RTAS might touch to the stack. (r0, r3-r13 are caller saved)
880 */
881 SAVE_GPR(2, r1) /* Save the TOC */
882 SAVE_GPR(13, r1) /* Save paca */
883 SAVE_8GPRS(14, r1) /* Save the non-volatiles */
884 SAVE_10GPRS(22, r1) /* ditto */
885
886 mfcr r4
887 std r4,_CCR(r1)
888 mfctr r5
889 std r5,_CTR(r1)
890 mfspr r6,SPRN_XER
891 std r6,_XER(r1)
892 mfdar r7
893 std r7,_DAR(r1)
894 mfdsisr r8
895 std r8,_DSISR(r1)
896
897 /* Temporary workaround to clear CR until RTAS can be modified to
898 * ignore all bits.
899 */
900 li r0,0
901 mtcr r0
902
903 #ifdef CONFIG_BUG
904 /* There is no way it is acceptable to get here with interrupts enabled,
905 * check it with the asm equivalent of WARN_ON
906 */
907 lbz r0,PACASOFTIRQEN(r13)
908 1: tdnei r0,0
909 EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,BUGFLAG_WARNING
910 #endif
911
912 /* Hard-disable interrupts */
913 mfmsr r6
914 rldicl r7,r6,48,1
915 rotldi r7,r7,16
916 mtmsrd r7,1
917
918 /* Unfortunately, the stack pointer and the MSR are also clobbered,
919 * so they are saved in the PACA which allows us to restore
920 * our original state after RTAS returns.
921 */
922 std r1,PACAR1(r13)
923 std r6,PACASAVEDMSR(r13)
924
925 /* Setup our real return addr */
926 LOAD_REG_ADDR(r4,.rtas_return_loc)
927 clrldi r4,r4,2 /* convert to realmode address */
928 mtlr r4
929
930 li r0,0
931 ori r0,r0,MSR_EE|MSR_SE|MSR_BE|MSR_RI
932 andc r0,r6,r0
933
934 li r9,1
935 rldicr r9,r9,MSR_SF_LG,(63-MSR_SF_LG)
936 ori r9,r9,MSR_IR|MSR_DR|MSR_FE0|MSR_FE1|MSR_FP|MSR_RI
937 andc r6,r0,r9
938 sync /* disable interrupts so SRR0/1 */
939 mtmsrd r0 /* don't get trashed */
940
941 LOAD_REG_ADDR(r4, rtas)
942 ld r5,RTASENTRY(r4) /* get the rtas->entry value */
943 ld r4,RTASBASE(r4) /* get the rtas->base value */
944
945 mtspr SPRN_SRR0,r5
946 mtspr SPRN_SRR1,r6
947 rfid
948 b . /* prevent speculative execution */
949
950 _STATIC(rtas_return_loc)
951 /* relocation is off at this point */
952 GET_PACA(r4)
953 clrldi r4,r4,2 /* convert to realmode address */
954
955 bcl 20,31,$+4
956 0: mflr r3
957 ld r3,(1f-0b)(r3) /* get &.rtas_restore_regs */
958
959 mfmsr r6
960 li r0,MSR_RI
961 andc r6,r6,r0
962 sync
963 mtmsrd r6
964
965 ld r1,PACAR1(r4) /* Restore our SP */
966 ld r4,PACASAVEDMSR(r4) /* Restore our MSR */
967
968 mtspr SPRN_SRR0,r3
969 mtspr SPRN_SRR1,r4
970 rfid
971 b . /* prevent speculative execution */
972
973 .align 3
974 1: .llong .rtas_restore_regs
975
976 _STATIC(rtas_restore_regs)
977 /* relocation is on at this point */
978 REST_GPR(2, r1) /* Restore the TOC */
979 REST_GPR(13, r1) /* Restore paca */
980 REST_8GPRS(14, r1) /* Restore the non-volatiles */
981 REST_10GPRS(22, r1) /* ditto */
982
983 GET_PACA(r13)
984
985 ld r4,_CCR(r1)
986 mtcr r4
987 ld r5,_CTR(r1)
988 mtctr r5
989 ld r6,_XER(r1)
990 mtspr SPRN_XER,r6
991 ld r7,_DAR(r1)
992 mtdar r7
993 ld r8,_DSISR(r1)
994 mtdsisr r8
995
996 addi r1,r1,RTAS_FRAME_SIZE /* Unstack our frame */
997 ld r0,16(r1) /* get return address */
998
999 mtlr r0
1000 blr /* return to caller */
1001
1002 #endif /* CONFIG_PPC_RTAS */
1003
1004 _GLOBAL(enter_prom)
1005 mflr r0
1006 std r0,16(r1)
1007 stdu r1,-PROM_FRAME_SIZE(r1) /* Save SP and create stack space */
1008
1009 /* Because PROM is running in 32b mode, it clobbers the high order half
1010 * of all registers that it saves. We therefore save those registers
1011 * PROM might touch to the stack. (r0, r3-r13 are caller saved)
1012 */
1013 SAVE_GPR(2, r1)
1014 SAVE_GPR(13, r1)
1015 SAVE_8GPRS(14, r1)
1016 SAVE_10GPRS(22, r1)
1017 mfcr r10
1018 mfmsr r11
1019 std r10,_CCR(r1)
1020 std r11,_MSR(r1)
1021
1022 /* Get the PROM entrypoint */
1023 mtlr r4
1024
1025 /* Switch MSR to 32 bits mode
1026 */
1027 #ifdef CONFIG_PPC_BOOK3E
1028 rlwinm r11,r11,0,1,31
1029 mtmsr r11
1030 #else /* CONFIG_PPC_BOOK3E */
1031 mfmsr r11
1032 li r12,1
1033 rldicr r12,r12,MSR_SF_LG,(63-MSR_SF_LG)
1034 andc r11,r11,r12
1035 li r12,1
1036 rldicr r12,r12,MSR_ISF_LG,(63-MSR_ISF_LG)
1037 andc r11,r11,r12
1038 mtmsrd r11
1039 #endif /* CONFIG_PPC_BOOK3E */
1040 isync
1041
1042 /* Enter PROM here... */
1043 blrl
1044
1045 /* Just make sure that r1 top 32 bits didn't get
1046 * corrupt by OF
1047 */
1048 rldicl r1,r1,0,32
1049
1050 /* Restore the MSR (back to 64 bits) */
1051 ld r0,_MSR(r1)
1052 MTMSRD(r0)
1053 isync
1054
1055 /* Restore other registers */
1056 REST_GPR(2, r1)
1057 REST_GPR(13, r1)
1058 REST_8GPRS(14, r1)
1059 REST_10GPRS(22, r1)
1060 ld r4,_CCR(r1)
1061 mtcr r4
1062
1063 addi r1,r1,PROM_FRAME_SIZE
1064 ld r0,16(r1)
1065 mtlr r0
1066 blr
1067
1068 #ifdef CONFIG_FUNCTION_TRACER
1069 #ifdef CONFIG_DYNAMIC_FTRACE
1070 _GLOBAL(mcount)
1071 _GLOBAL(_mcount)
1072 blr
1073
1074 _GLOBAL(ftrace_caller)
1075 /* Taken from output of objdump from lib64/glibc */
1076 mflr r3
1077 ld r11, 0(r1)
1078 stdu r1, -112(r1)
1079 std r3, 128(r1)
1080 ld r4, 16(r11)
1081 subi r3, r3, MCOUNT_INSN_SIZE
1082 .globl ftrace_call
1083 ftrace_call:
1084 bl ftrace_stub
1085 nop
1086 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1087 .globl ftrace_graph_call
1088 ftrace_graph_call:
1089 b ftrace_graph_stub
1090 _GLOBAL(ftrace_graph_stub)
1091 #endif
1092 ld r0, 128(r1)
1093 mtlr r0
1094 addi r1, r1, 112
1095 _GLOBAL(ftrace_stub)
1096 blr
1097 #else
1098 _GLOBAL(mcount)
1099 blr
1100
1101 _GLOBAL(_mcount)
1102 /* Taken from output of objdump from lib64/glibc */
1103 mflr r3
1104 ld r11, 0(r1)
1105 stdu r1, -112(r1)
1106 std r3, 128(r1)
1107 ld r4, 16(r11)
1108
1109 subi r3, r3, MCOUNT_INSN_SIZE
1110 LOAD_REG_ADDR(r5,ftrace_trace_function)
1111 ld r5,0(r5)
1112 ld r5,0(r5)
1113 mtctr r5
1114 bctrl
1115 nop
1116
1117
1118 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1119 b ftrace_graph_caller
1120 #endif
1121 ld r0, 128(r1)
1122 mtlr r0
1123 addi r1, r1, 112
1124 _GLOBAL(ftrace_stub)
1125 blr
1126
1127 #endif /* CONFIG_DYNAMIC_FTRACE */
1128
1129 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1130 _GLOBAL(ftrace_graph_caller)
1131 /* load r4 with local address */
1132 ld r4, 128(r1)
1133 subi r4, r4, MCOUNT_INSN_SIZE
1134
1135 /* get the parent address */
1136 ld r11, 112(r1)
1137 addi r3, r11, 16
1138
1139 bl .prepare_ftrace_return
1140 nop
1141
1142 ld r0, 128(r1)
1143 mtlr r0
1144 addi r1, r1, 112
1145 blr
1146
1147 _GLOBAL(return_to_handler)
1148 /* need to save return values */
1149 std r4, -24(r1)
1150 std r3, -16(r1)
1151 std r31, -8(r1)
1152 mr r31, r1
1153 stdu r1, -112(r1)
1154
1155 bl .ftrace_return_to_handler
1156 nop
1157
1158 /* return value has real return address */
1159 mtlr r3
1160
1161 ld r1, 0(r1)
1162 ld r4, -24(r1)
1163 ld r3, -16(r1)
1164 ld r31, -8(r1)
1165
1166 /* Jump back to real return address */
1167 blr
1168
1169 _GLOBAL(mod_return_to_handler)
1170 /* need to save return values */
1171 std r4, -32(r1)
1172 std r3, -24(r1)
1173 /* save TOC */
1174 std r2, -16(r1)
1175 std r31, -8(r1)
1176 mr r31, r1
1177 stdu r1, -112(r1)
1178
1179 /*
1180 * We are in a module using the module's TOC.
1181 * Switch to our TOC to run inside the core kernel.
1182 */
1183 ld r2, PACATOC(r13)
1184
1185 bl .ftrace_return_to_handler
1186 nop
1187
1188 /* return value has real return address */
1189 mtlr r3
1190
1191 ld r1, 0(r1)
1192 ld r4, -32(r1)
1193 ld r3, -24(r1)
1194 ld r2, -16(r1)
1195 ld r31, -8(r1)
1196
1197 /* Jump back to real return address */
1198 blr
1199 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
1200 #endif /* CONFIG_FUNCTION_TRACER */
This page took 0.189612 seconds and 6 git commands to generate.