projects / deliverable / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
powerpc/tm: Abort syscalls in active transactions
[deliverable/linux.git] / arch / powerpc / kernel / entry_64.S
1 /*
2 * PowerPC version
3 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
4 * Rewritten by Cort Dougan (cort@cs.nmt.edu) for PReP
5 * Copyright (C) 1996 Cort Dougan <cort@cs.nmt.edu>
6 * Adapted for Power Macintosh by Paul Mackerras.
7 * Low-level exception handlers and MMU support
8 * rewritten by Paul Mackerras.
9 * Copyright (C) 1996 Paul Mackerras.
10 * MPC8xx modifications Copyright (C) 1997 Dan Malek (dmalek@jlc.net).
11 *
12 * This file contains the system call entry code, context switch
13 * code, and exception/interrupt return code for PowerPC.
14 *
15 * This program is free software; you can redistribute it and/or
16 * modify it under the terms of the GNU General Public License
17 * as published by the Free Software Foundation; either version
18 * 2 of the License, or (at your option) any later version.
19 */
20
21 #include <linux/errno.h>
22 #include <asm/unistd.h>
23 #include <asm/processor.h>
24 #include <asm/page.h>
25 #include <asm/mmu.h>
26 #include <asm/thread_info.h>
27 #include <asm/ppc_asm.h>
28 #include <asm/asm-offsets.h>
29 #include <asm/cputable.h>
30 #include <asm/firmware.h>
31 #include <asm/bug.h>
32 #include <asm/ptrace.h>
33 #include <asm/irqflags.h>
34 #include <asm/ftrace.h>
35 #include <asm/hw_irq.h>
36 #include <asm/context_tracking.h>
37 #include <asm/tm.h>
38
39 /*
40 * System calls.
41 */
42 .section ".toc","aw"
43 SYS_CALL_TABLE:
44 .tc sys_call_table[TC],sys_call_table
45
46 /* This value is used to mark exception frames on the stack. */
47 exception_marker:
48 .tc ID_EXC_MARKER[TC],STACK_FRAME_REGS_MARKER
49
50 .section ".text"
51 .align 7
52
53 .globl system_call_common
54 system_call_common:
55 andi. r10,r12,MSR_PR
56 mr r10,r1
57 addi r1,r1,-INT_FRAME_SIZE
58 beq- 1f
59 ld r1,PACAKSAVE(r13)
60 1: std r10,0(r1)
61 std r11,_NIP(r1)
62 std r12,_MSR(r1)
63 std r0,GPR0(r1)
64 std r10,GPR1(r1)
65 beq 2f /* if from kernel mode */
66 ACCOUNT_CPU_USER_ENTRY(r10, r11)
67 2: std r2,GPR2(r1)
68 std r3,GPR3(r1)
69 mfcr r2
70 std r4,GPR4(r1)
71 std r5,GPR5(r1)
72 std r6,GPR6(r1)
73 std r7,GPR7(r1)
74 std r8,GPR8(r1)
75 li r11,0
76 std r11,GPR9(r1)
77 std r11,GPR10(r1)
78 std r11,GPR11(r1)
79 std r11,GPR12(r1)
80 std r11,_XER(r1)
81 std r11,_CTR(r1)
82 std r9,GPR13(r1)
83 mflr r10
84 /*
85 * This clears CR0.SO (bit 28), which is the error indication on
86 * return from this system call.
87 */
88 rldimi r2,r11,28,(63-28)
89 li r11,0xc01
90 std r10,_LINK(r1)
91 std r11,_TRAP(r1)
92 std r3,ORIG_GPR3(r1)
93 std r2,_CCR(r1)
94 ld r2,PACATOC(r13)
95 addi r9,r1,STACK_FRAME_OVERHEAD
96 ld r11,exception_marker@toc(r2)
97 std r11,-16(r9) /* "regshere" marker */
98 #if defined(CONFIG_VIRT_CPU_ACCOUNTING_NATIVE) && defined(CONFIG_PPC_SPLPAR)
99 BEGIN_FW_FTR_SECTION
100 beq 33f
101 /* if from user, see if there are any DTL entries to process */
102 ld r10,PACALPPACAPTR(r13) /* get ptr to VPA */
103 ld r11,PACA_DTL_RIDX(r13) /* get log read index */
104 addi r10,r10,LPPACA_DTLIDX
105 LDX_BE r10,0,r10 /* get log write index */
106 cmpd cr1,r11,r10
107 beq+ cr1,33f
108 bl accumulate_stolen_time
109 REST_GPR(0,r1)
110 REST_4GPRS(3,r1)
111 REST_2GPRS(7,r1)
112 addi r9,r1,STACK_FRAME_OVERHEAD
113 33:
114 END_FW_FTR_SECTION_IFSET(FW_FEATURE_SPLPAR)
115 #endif /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE && CONFIG_PPC_SPLPAR */
116
117 /*
118 * A syscall should always be called with interrupts enabled
119 * so we just unconditionally hard-enable here. When some kind
120 * of irq tracing is used, we additionally check that condition
121 * is correct
122 */
123 #if defined(CONFIG_TRACE_IRQFLAGS) && defined(CONFIG_BUG)
124 lbz r10,PACASOFTIRQEN(r13)
125 xori r10,r10,1
126 1: tdnei r10,0
127 EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,BUGFLAG_WARNING
128 #endif
129
130 #ifdef CONFIG_PPC_BOOK3E
131 wrteei 1
132 #else
133 ld r11,PACAKMSR(r13)
134 ori r11,r11,MSR_EE
135 mtmsrd r11,1
136 #endif /* CONFIG_PPC_BOOK3E */
137
138 /* We do need to set SOFTE in the stack frame or the return
139 * from interrupt will be painful
140 */
141 li r10,1
142 std r10,SOFTE(r1)
143
144 CURRENT_THREAD_INFO(r11, r1)
145 ld r10,TI_FLAGS(r11)
146 andi. r11,r10,_TIF_SYSCALL_DOTRACE
147 bne syscall_dotrace
148 .Lsyscall_dotrace_cont:
149 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
150 BEGIN_FTR_SECTION
151 b 1f
152 END_FTR_SECTION_IFCLR(CPU_FTR_TM)
153 extrdi. r11, r12, 1, (63-MSR_TS_T_LG) /* transaction active? */
154 beq+ 1f
155
156 /* Doom the transaction and don't perform the syscall: */
157 mfmsr r11
158 li r12, 1
159 rldimi r11, r12, MSR_TM_LG, 63-MSR_TM_LG
160 mtmsrd r11, 0
161 li r11, (TM_CAUSE_SYSCALL|TM_CAUSE_PERSISTENT)
162 TABORT(R11)
163
164 b .Lsyscall_exit
165 1:
166 #endif
167 cmpldi 0,r0,NR_syscalls
168 bge- syscall_enosys
169
170 system_call: /* label this so stack traces look sane */
171 /*
172 * Need to vector to 32 Bit or default sys_call_table here,
173 * based on caller's run-mode / personality.
174 */
175 ld r11,SYS_CALL_TABLE@toc(2)
176 andi. r10,r10,_TIF_32BIT
177 beq 15f
178 addi r11,r11,8 /* use 32-bit syscall entries */
179 clrldi r3,r3,32
180 clrldi r4,r4,32
181 clrldi r5,r5,32
182 clrldi r6,r6,32
183 clrldi r7,r7,32
184 clrldi r8,r8,32
185 15:
186 slwi r0,r0,4
187 ldx r12,r11,r0 /* Fetch system call handler [ptr] */
188 mtctr r12
189 bctrl /* Call handler */
190
191 .Lsyscall_exit:
192 std r3,RESULT(r1)
193 CURRENT_THREAD_INFO(r12, r1)
194
195 ld r8,_MSR(r1)
196 #ifdef CONFIG_PPC_BOOK3S
197 /* No MSR:RI on BookE */
198 andi. r10,r8,MSR_RI
199 beq- unrecov_restore
200 #endif
201 /*
202 * Disable interrupts so current_thread_info()->flags can't change,
203 * and so that we don't get interrupted after loading SRR0/1.
204 */
205 #ifdef CONFIG_PPC_BOOK3E
206 wrteei 0
207 #else
208 ld r10,PACAKMSR(r13)
209 /*
210 * For performance reasons we clear RI the same time that we
211 * clear EE. We only need to clear RI just before we restore r13
212 * below, but batching it with EE saves us one expensive mtmsrd call.
213 * We have to be careful to restore RI if we branch anywhere from
214 * here (eg syscall_exit_work).
215 */
216 li r9,MSR_RI
217 andc r11,r10,r9
218 mtmsrd r11,1
219 #endif /* CONFIG_PPC_BOOK3E */
220
221 ld r9,TI_FLAGS(r12)
222 li r11,-_LAST_ERRNO
223 andi. r0,r9,(_TIF_SYSCALL_DOTRACE|_TIF_SINGLESTEP|_TIF_USER_WORK_MASK|_TIF_PERSYSCALL_MASK)
224 bne- syscall_exit_work
225 cmpld r3,r11
226 ld r5,_CCR(r1)
227 bge- syscall_error
228 .Lsyscall_error_cont:
229 ld r7,_NIP(r1)
230 BEGIN_FTR_SECTION
231 stdcx. r0,0,r1 /* to clear the reservation */
232 END_FTR_SECTION_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
233 andi. r6,r8,MSR_PR
234 ld r4,_LINK(r1)
235
236 beq- 1f
237 ACCOUNT_CPU_USER_EXIT(r11, r12)
238 HMT_MEDIUM_LOW_HAS_PPR
239 ld r13,GPR13(r1) /* only restore r13 if returning to usermode */
240 1: ld r2,GPR2(r1)
241 ld r1,GPR1(r1)
242 mtlr r4
243 mtcr r5
244 mtspr SPRN_SRR0,r7
245 mtspr SPRN_SRR1,r8
246 RFI
247 b . /* prevent speculative execution */
248
249 syscall_error:
250 oris r5,r5,0x1000 /* Set SO bit in CR */
251 neg r3,r3
252 std r5,_CCR(r1)
253 b .Lsyscall_error_cont
254
255 /* Traced system call support */
256 syscall_dotrace:
257 bl save_nvgprs
258 addi r3,r1,STACK_FRAME_OVERHEAD
259 bl do_syscall_trace_enter
260 /*
261 * Restore argument registers possibly just changed.
262 * We use the return value of do_syscall_trace_enter
263 * for the call number to look up in the table (r0).
264 */
265 mr r0,r3
266 ld r3,GPR3(r1)
267 ld r4,GPR4(r1)
268 ld r5,GPR5(r1)
269 ld r6,GPR6(r1)
270 ld r7,GPR7(r1)
271 ld r8,GPR8(r1)
272 addi r9,r1,STACK_FRAME_OVERHEAD
273 CURRENT_THREAD_INFO(r10, r1)
274 ld r10,TI_FLAGS(r10)
275 b .Lsyscall_dotrace_cont
276
277 syscall_enosys:
278 li r3,-ENOSYS
279 b .Lsyscall_exit
280
281 syscall_exit_work:
282 #ifdef CONFIG_PPC_BOOK3S
283 mtmsrd r10,1 /* Restore RI */
284 #endif
285 /* If TIF_RESTOREALL is set, don't scribble on either r3 or ccr.
286 If TIF_NOERROR is set, just save r3 as it is. */
287
288 andi. r0,r9,_TIF_RESTOREALL
289 beq+ 0f
290 REST_NVGPRS(r1)
291 b 2f
292 0: cmpld r3,r11 /* r10 is -LAST_ERRNO */
293 blt+ 1f
294 andi. r0,r9,_TIF_NOERROR
295 bne- 1f
296 ld r5,_CCR(r1)
297 neg r3,r3
298 oris r5,r5,0x1000 /* Set SO bit in CR */
299 std r5,_CCR(r1)
300 1: std r3,GPR3(r1)
301 2: andi. r0,r9,(_TIF_PERSYSCALL_MASK)
302 beq 4f
303
304 /* Clear per-syscall TIF flags if any are set. */
305
306 li r11,_TIF_PERSYSCALL_MASK
307 addi r12,r12,TI_FLAGS
308 3: ldarx r10,0,r12
309 andc r10,r10,r11
310 stdcx. r10,0,r12
311 bne- 3b
312 subi r12,r12,TI_FLAGS
313
314 4: /* Anything else left to do? */
315 SET_DEFAULT_THREAD_PPR(r3, r10) /* Set thread.ppr = 3 */
316 andi. r0,r9,(_TIF_SYSCALL_DOTRACE|_TIF_SINGLESTEP)
317 beq ret_from_except_lite
318
319 /* Re-enable interrupts */
320 #ifdef CONFIG_PPC_BOOK3E
321 wrteei 1
322 #else
323 ld r10,PACAKMSR(r13)
324 ori r10,r10,MSR_EE
325 mtmsrd r10,1
326 #endif /* CONFIG_PPC_BOOK3E */
327
328 bl save_nvgprs
329 addi r3,r1,STACK_FRAME_OVERHEAD
330 bl do_syscall_trace_leave
331 b ret_from_except
332
333 /* Save non-volatile GPRs, if not already saved. */
334 _GLOBAL(save_nvgprs)
335 ld r11,_TRAP(r1)
336 andi. r0,r11,1
337 beqlr-
338 SAVE_NVGPRS(r1)
339 clrrdi r0,r11,1
340 std r0,_TRAP(r1)
341 blr
342
343
344 /*
345 * The sigsuspend and rt_sigsuspend system calls can call do_signal
346 * and thus put the process into the stopped state where we might
347 * want to examine its user state with ptrace. Therefore we need
348 * to save all the nonvolatile registers (r14 - r31) before calling
349 * the C code. Similarly, fork, vfork and clone need the full
350 * register state on the stack so that it can be copied to the child.
351 */
352
353 _GLOBAL(ppc_fork)
354 bl save_nvgprs
355 bl sys_fork
356 b .Lsyscall_exit
357
358 _GLOBAL(ppc_vfork)
359 bl save_nvgprs
360 bl sys_vfork
361 b .Lsyscall_exit
362
363 _GLOBAL(ppc_clone)
364 bl save_nvgprs
365 bl sys_clone
366 b .Lsyscall_exit
367
368 _GLOBAL(ppc32_swapcontext)
369 bl save_nvgprs
370 bl compat_sys_swapcontext
371 b .Lsyscall_exit
372
373 _GLOBAL(ppc64_swapcontext)
374 bl save_nvgprs
375 bl sys_swapcontext
376 b .Lsyscall_exit
377
378 _GLOBAL(ppc_switch_endian)
379 bl save_nvgprs
380 bl sys_switch_endian
381 b .Lsyscall_exit
382
383 _GLOBAL(ret_from_fork)
384 bl schedule_tail
385 REST_NVGPRS(r1)
386 li r3,0
387 b .Lsyscall_exit
388
389 _GLOBAL(ret_from_kernel_thread)
390 bl schedule_tail
391 REST_NVGPRS(r1)
392 mtlr r14
393 mr r3,r15
394 #if defined(_CALL_ELF) && _CALL_ELF == 2
395 mr r12,r14
396 #endif
397 blrl
398 li r3,0
399 b .Lsyscall_exit
400
401 /*
402 * This routine switches between two different tasks. The process
403 * state of one is saved on its kernel stack. Then the state
404 * of the other is restored from its kernel stack. The memory
405 * management hardware is updated to the second process's state.
406 * Finally, we can return to the second process, via ret_from_except.
407 * On entry, r3 points to the THREAD for the current task, r4
408 * points to the THREAD for the new task.
409 *
410 * Note: there are two ways to get to the "going out" portion
411 * of this code; either by coming in via the entry (_switch)
412 * or via "fork" which must set up an environment equivalent
413 * to the "_switch" path. If you change this you'll have to change
414 * the fork code also.
415 *
416 * The code which creates the new task context is in 'copy_thread'
417 * in arch/powerpc/kernel/process.c
418 */
419 .align 7
420 _GLOBAL(_switch)
421 mflr r0
422 std r0,16(r1)
423 stdu r1,-SWITCH_FRAME_SIZE(r1)
424 /* r3-r13 are caller saved -- Cort */
425 SAVE_8GPRS(14, r1)
426 SAVE_10GPRS(22, r1)
427 mflr r20 /* Return to switch caller */
428 mfmsr r22
429 li r0, MSR_FP
430 #ifdef CONFIG_VSX
431 BEGIN_FTR_SECTION
432 oris r0,r0,MSR_VSX@h /* Disable VSX */
433 END_FTR_SECTION_IFSET(CPU_FTR_VSX)
434 #endif /* CONFIG_VSX */
435 #ifdef CONFIG_ALTIVEC
436 BEGIN_FTR_SECTION
437 oris r0,r0,MSR_VEC@h /* Disable altivec */
438 mfspr r24,SPRN_VRSAVE /* save vrsave register value */
439 std r24,THREAD_VRSAVE(r3)
440 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
441 #endif /* CONFIG_ALTIVEC */
442 and. r0,r0,r22
443 beq+ 1f
444 andc r22,r22,r0
445 MTMSRD(r22)
446 isync
447 1: std r20,_NIP(r1)
448 mfcr r23
449 std r23,_CCR(r1)
450 std r1,KSP(r3) /* Set old stack pointer */
451
452 #ifdef CONFIG_PPC_BOOK3S_64
453 BEGIN_FTR_SECTION
454 /* Event based branch registers */
455 mfspr r0, SPRN_BESCR
456 std r0, THREAD_BESCR(r3)
457 mfspr r0, SPRN_EBBHR
458 std r0, THREAD_EBBHR(r3)
459 mfspr r0, SPRN_EBBRR
460 std r0, THREAD_EBBRR(r3)
461 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
462 #endif
463
464 #ifdef CONFIG_SMP
465 /* We need a sync somewhere here to make sure that if the
466 * previous task gets rescheduled on another CPU, it sees all
467 * stores it has performed on this one.
468 */
469 sync
470 #endif /* CONFIG_SMP */
471
472 /*
473 * If we optimise away the clear of the reservation in system
474 * calls because we know the CPU tracks the address of the
475 * reservation, then we need to clear it here to cover the
476 * case that the kernel context switch path has no larx
477 * instructions.
478 */
479 BEGIN_FTR_SECTION
480 ldarx r6,0,r1
481 END_FTR_SECTION_IFSET(CPU_FTR_STCX_CHECKS_ADDRESS)
482
483 #ifdef CONFIG_PPC_BOOK3S
484 /* Cancel all explict user streams as they will have no use after context
485 * switch and will stop the HW from creating streams itself
486 */
487 DCBT_STOP_ALL_STREAM_IDS(r6)
488 #endif
489
490 addi r6,r4,-THREAD /* Convert THREAD to 'current' */
491 std r6,PACACURRENT(r13) /* Set new 'current' */
492
493 ld r8,KSP(r4) /* new stack pointer */
494 #ifdef CONFIG_PPC_BOOK3S
495 BEGIN_FTR_SECTION
496 clrrdi r6,r8,28 /* get its ESID */
497 clrrdi r9,r1,28 /* get current sp ESID */
498 FTR_SECTION_ELSE
499 clrrdi r6,r8,40 /* get its 1T ESID */
500 clrrdi r9,r1,40 /* get current sp 1T ESID */
501 ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_1T_SEGMENT)
502 clrldi. r0,r6,2 /* is new ESID c00000000? */
503 cmpd cr1,r6,r9 /* or is new ESID the same as current ESID? */
504 cror eq,4*cr1+eq,eq
505 beq 2f /* if yes, don't slbie it */
506
507 /* Bolt in the new stack SLB entry */
508 ld r7,KSP_VSID(r4) /* Get new stack's VSID */
509 oris r0,r6,(SLB_ESID_V)@h
510 ori r0,r0,(SLB_NUM_BOLTED-1)@l
511 BEGIN_FTR_SECTION
512 li r9,MMU_SEGSIZE_1T /* insert B field */
513 oris r6,r6,(MMU_SEGSIZE_1T << SLBIE_SSIZE_SHIFT)@h
514 rldimi r7,r9,SLB_VSID_SSIZE_SHIFT,0
515 END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT)
516
517 /* Update the last bolted SLB. No write barriers are needed
518 * here, provided we only update the current CPU's SLB shadow
519 * buffer.
520 */
521 ld r9,PACA_SLBSHADOWPTR(r13)
522 li r12,0
523 std r12,SLBSHADOW_STACKESID(r9) /* Clear ESID */
524 li r12,SLBSHADOW_STACKVSID
525 STDX_BE r7,r12,r9 /* Save VSID */
526 li r12,SLBSHADOW_STACKESID
527 STDX_BE r0,r12,r9 /* Save ESID */
528
529 /* No need to check for MMU_FTR_NO_SLBIE_B here, since when
530 * we have 1TB segments, the only CPUs known to have the errata
531 * only support less than 1TB of system memory and we'll never
532 * actually hit this code path.
533 */
534
535 slbie r6
536 slbie r6 /* Workaround POWER5 < DD2.1 issue */
537 slbmte r7,r0
538 isync
539 2:
540 #endif /* !CONFIG_PPC_BOOK3S */
541
542 CURRENT_THREAD_INFO(r7, r8) /* base of new stack */
543 /* Note: this uses SWITCH_FRAME_SIZE rather than INT_FRAME_SIZE
544 because we don't need to leave the 288-byte ABI gap at the
545 top of the kernel stack. */
546 addi r7,r7,THREAD_SIZE-SWITCH_FRAME_SIZE
547
548 mr r1,r8 /* start using new stack pointer */
549 std r7,PACAKSAVE(r13)
550
551 #ifdef CONFIG_PPC_BOOK3S_64
552 BEGIN_FTR_SECTION
553 /* Event based branch registers */
554 ld r0, THREAD_BESCR(r4)
555 mtspr SPRN_BESCR, r0
556 ld r0, THREAD_EBBHR(r4)
557 mtspr SPRN_EBBHR, r0
558 ld r0, THREAD_EBBRR(r4)
559 mtspr SPRN_EBBRR, r0
560
561 ld r0,THREAD_TAR(r4)
562 mtspr SPRN_TAR,r0
563 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
564 #endif
565
566 #ifdef CONFIG_ALTIVEC
567 BEGIN_FTR_SECTION
568 ld r0,THREAD_VRSAVE(r4)
569 mtspr SPRN_VRSAVE,r0 /* if G4, restore VRSAVE reg */
570 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
571 #endif /* CONFIG_ALTIVEC */
572 #ifdef CONFIG_PPC64
573 BEGIN_FTR_SECTION
574 lwz r6,THREAD_DSCR_INHERIT(r4)
575 ld r0,THREAD_DSCR(r4)
576 cmpwi r6,0
577 bne 1f
578 ld r0,PACA_DSCR(r13)
579 1:
580 BEGIN_FTR_SECTION_NESTED(70)
581 mfspr r8, SPRN_FSCR
582 rldimi r8, r6, FSCR_DSCR_LG, (63 - FSCR_DSCR_LG)
583 mtspr SPRN_FSCR, r8
584 END_FTR_SECTION_NESTED(CPU_FTR_ARCH_207S, CPU_FTR_ARCH_207S, 70)
585 cmpd r0,r25
586 beq 2f
587 mtspr SPRN_DSCR,r0
588 2:
589 END_FTR_SECTION_IFSET(CPU_FTR_DSCR)
590 #endif
591
592 ld r6,_CCR(r1)
593 mtcrf 0xFF,r6
594
595 /* r3-r13 are destroyed -- Cort */
596 REST_8GPRS(14, r1)
597 REST_10GPRS(22, r1)
598
599 /* convert old thread to its task_struct for return value */
600 addi r3,r3,-THREAD
601 ld r7,_NIP(r1) /* Return to _switch caller in new task */
602 mtlr r7
603 addi r1,r1,SWITCH_FRAME_SIZE
604 blr
605
606 .align 7
607 _GLOBAL(ret_from_except)
608 ld r11,_TRAP(r1)
609 andi. r0,r11,1
610 bne ret_from_except_lite
611 REST_NVGPRS(r1)
612
613 _GLOBAL(ret_from_except_lite)
614 /*
615 * Disable interrupts so that current_thread_info()->flags
616 * can't change between when we test it and when we return
617 * from the interrupt.
618 */
619 #ifdef CONFIG_PPC_BOOK3E
620 wrteei 0
621 #else
622 ld r10,PACAKMSR(r13) /* Get kernel MSR without EE */
623 mtmsrd r10,1 /* Update machine state */
624 #endif /* CONFIG_PPC_BOOK3E */
625
626 CURRENT_THREAD_INFO(r9, r1)
627 ld r3,_MSR(r1)
628 #ifdef CONFIG_PPC_BOOK3E
629 ld r10,PACACURRENT(r13)
630 #endif /* CONFIG_PPC_BOOK3E */
631 ld r4,TI_FLAGS(r9)
632 andi. r3,r3,MSR_PR
633 beq resume_kernel
634 #ifdef CONFIG_PPC_BOOK3E
635 lwz r3,(THREAD+THREAD_DBCR0)(r10)
636 #endif /* CONFIG_PPC_BOOK3E */
637
638 /* Check current_thread_info()->flags */
639 andi. r0,r4,_TIF_USER_WORK_MASK
640 #ifdef CONFIG_PPC_BOOK3E
641 bne 1f
642 /*
643 * Check to see if the dbcr0 register is set up to debug.
644 * Use the internal debug mode bit to do this.
645 */
646 andis. r0,r3,DBCR0_IDM@h
647 beq restore
648 mfmsr r0
649 rlwinm r0,r0,0,~MSR_DE /* Clear MSR.DE */
650 mtmsr r0
651 mtspr SPRN_DBCR0,r3
652 li r10, -1
653 mtspr SPRN_DBSR,r10
654 b restore
655 #else
656 beq restore
657 #endif
658 1: andi. r0,r4,_TIF_NEED_RESCHED
659 beq 2f
660 bl restore_interrupts
661 SCHEDULE_USER
662 b ret_from_except_lite
663 2:
664 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
665 andi. r0,r4,_TIF_USER_WORK_MASK & ~_TIF_RESTORE_TM
666 bne 3f /* only restore TM if nothing else to do */
667 addi r3,r1,STACK_FRAME_OVERHEAD
668 bl restore_tm_state
669 b restore
670 3:
671 #endif
672 bl save_nvgprs
673 /*
674 * Use a non volatile GPR to save and restore our thread_info flags
675 * across the call to restore_interrupts.
676 */
677 mr r30,r4
678 bl restore_interrupts
679 mr r4,r30
680 addi r3,r1,STACK_FRAME_OVERHEAD
681 bl do_notify_resume
682 b ret_from_except
683
684 resume_kernel:
685 /* check current_thread_info, _TIF_EMULATE_STACK_STORE */
686 andis. r8,r4,_TIF_EMULATE_STACK_STORE@h
687 beq+ 1f
688
689 addi r8,r1,INT_FRAME_SIZE /* Get the kprobed function entry */
690
691 lwz r3,GPR1(r1)
692 subi r3,r3,INT_FRAME_SIZE /* dst: Allocate a trampoline exception frame */
693 mr r4,r1 /* src: current exception frame */
694 mr r1,r3 /* Reroute the trampoline frame to r1 */
695
696 /* Copy from the original to the trampoline. */
697 li r5,INT_FRAME_SIZE/8 /* size: INT_FRAME_SIZE */
698 li r6,0 /* start offset: 0 */
699 mtctr r5
700 2: ldx r0,r6,r4
701 stdx r0,r6,r3
702 addi r6,r6,8
703 bdnz 2b
704
705 /* Do real store operation to complete stwu */
706 lwz r5,GPR1(r1)
707 std r8,0(r5)
708
709 /* Clear _TIF_EMULATE_STACK_STORE flag */
710 lis r11,_TIF_EMULATE_STACK_STORE@h
711 addi r5,r9,TI_FLAGS
712 0: ldarx r4,0,r5
713 andc r4,r4,r11
714 stdcx. r4,0,r5
715 bne- 0b
716 1:
717
718 #ifdef CONFIG_PREEMPT
719 /* Check if we need to preempt */
720 andi. r0,r4,_TIF_NEED_RESCHED
721 beq+ restore
722 /* Check that preempt_count() == 0 and interrupts are enabled */
723 lwz r8,TI_PREEMPT(r9)
724 cmpwi cr1,r8,0
725 ld r0,SOFTE(r1)
726 cmpdi r0,0
727 crandc eq,cr1*4+eq,eq
728 bne restore
729
730 /*
731 * Here we are preempting the current task. We want to make
732 * sure we are soft-disabled first and reconcile irq state.
733 */
734 RECONCILE_IRQ_STATE(r3,r4)
735 1: bl preempt_schedule_irq
736
737 /* Re-test flags and eventually loop */
738 CURRENT_THREAD_INFO(r9, r1)
739 ld r4,TI_FLAGS(r9)
740 andi. r0,r4,_TIF_NEED_RESCHED
741 bne 1b
742
743 /*
744 * arch_local_irq_restore() from preempt_schedule_irq above may
745 * enable hard interrupt but we really should disable interrupts
746 * when we return from the interrupt, and so that we don't get
747 * interrupted after loading SRR0/1.
748 */
749 #ifdef CONFIG_PPC_BOOK3E
750 wrteei 0
751 #else
752 ld r10,PACAKMSR(r13) /* Get kernel MSR without EE */
753 mtmsrd r10,1 /* Update machine state */
754 #endif /* CONFIG_PPC_BOOK3E */
755 #endif /* CONFIG_PREEMPT */
756
757 .globl fast_exc_return_irq
758 fast_exc_return_irq:
759 restore:
760 /*
761 * This is the main kernel exit path. First we check if we
762 * are about to re-enable interrupts
763 */
764 ld r5,SOFTE(r1)
765 lbz r6,PACASOFTIRQEN(r13)
766 cmpwi cr0,r5,0
767 beq restore_irq_off
768
769 /* We are enabling, were we already enabled ? Yes, just return */
770 cmpwi cr0,r6,1
771 beq cr0,do_restore
772
773 /*
774 * We are about to soft-enable interrupts (we are hard disabled
775 * at this point). We check if there's anything that needs to
776 * be replayed first.
777 */
778 lbz r0,PACAIRQHAPPENED(r13)
779 cmpwi cr0,r0,0
780 bne- restore_check_irq_replay
781
782 /*
783 * Get here when nothing happened while soft-disabled, just
784 * soft-enable and move-on. We will hard-enable as a side
785 * effect of rfi
786 */
787 restore_no_replay:
788 TRACE_ENABLE_INTS
789 li r0,1
790 stb r0,PACASOFTIRQEN(r13);
791
792 /*
793 * Final return path. BookE is handled in a different file
794 */
795 do_restore:
796 #ifdef CONFIG_PPC_BOOK3E
797 b exception_return_book3e
798 #else
799 /*
800 * Clear the reservation. If we know the CPU tracks the address of
801 * the reservation then we can potentially save some cycles and use
802 * a larx. On POWER6 and POWER7 this is significantly faster.
803 */
804 BEGIN_FTR_SECTION
805 stdcx. r0,0,r1 /* to clear the reservation */
806 FTR_SECTION_ELSE
807 ldarx r4,0,r1
808 ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
809
810 /*
811 * Some code path such as load_up_fpu or altivec return directly
812 * here. They run entirely hard disabled and do not alter the
813 * interrupt state. They also don't use lwarx/stwcx. and thus
814 * are known not to leave dangling reservations.
815 */
816 .globl fast_exception_return
817 fast_exception_return:
818 ld r3,_MSR(r1)
819 ld r4,_CTR(r1)
820 ld r0,_LINK(r1)
821 mtctr r4
822 mtlr r0
823 ld r4,_XER(r1)
824 mtspr SPRN_XER,r4
825
826 REST_8GPRS(5, r1)
827
828 andi. r0,r3,MSR_RI
829 beq- unrecov_restore
830
831 /* Load PPR from thread struct before we clear MSR:RI */
832 BEGIN_FTR_SECTION
833 ld r2,PACACURRENT(r13)
834 ld r2,TASKTHREADPPR(r2)
835 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
836
837 /*
838 * Clear RI before restoring r13. If we are returning to
839 * userspace and we take an exception after restoring r13,
840 * we end up corrupting the userspace r13 value.
841 */
842 ld r4,PACAKMSR(r13) /* Get kernel MSR without EE */
843 andc r4,r4,r0 /* r0 contains MSR_RI here */
844 mtmsrd r4,1
845
846 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
847 /* TM debug */
848 std r3, PACATMSCRATCH(r13) /* Stash returned-to MSR */
849 #endif
850 /*
851 * r13 is our per cpu area, only restore it if we are returning to
852 * userspace the value stored in the stack frame may belong to
853 * another CPU.
854 */
855 andi. r0,r3,MSR_PR
856 beq 1f
857 BEGIN_FTR_SECTION
858 mtspr SPRN_PPR,r2 /* Restore PPR */
859 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
860 ACCOUNT_CPU_USER_EXIT(r2, r4)
861 REST_GPR(13, r1)
862 1:
863 mtspr SPRN_SRR1,r3
864
865 ld r2,_CCR(r1)
866 mtcrf 0xFF,r2
867 ld r2,_NIP(r1)
868 mtspr SPRN_SRR0,r2
869
870 ld r0,GPR0(r1)
871 ld r2,GPR2(r1)
872 ld r3,GPR3(r1)
873 ld r4,GPR4(r1)
874 ld r1,GPR1(r1)
875
876 rfid
877 b . /* prevent speculative execution */
878
879 #endif /* CONFIG_PPC_BOOK3E */
880
881 /*
882 * We are returning to a context with interrupts soft disabled.
883 *
884 * However, we may also about to hard enable, so we need to
885 * make sure that in this case, we also clear PACA_IRQ_HARD_DIS
886 * or that bit can get out of sync and bad things will happen
887 */
888 restore_irq_off:
889 ld r3,_MSR(r1)
890 lbz r7,PACAIRQHAPPENED(r13)
891 andi. r0,r3,MSR_EE
892 beq 1f
893 rlwinm r7,r7,0,~PACA_IRQ_HARD_DIS
894 stb r7,PACAIRQHAPPENED(r13)
895 1: li r0,0
896 stb r0,PACASOFTIRQEN(r13);
897 TRACE_DISABLE_INTS
898 b do_restore
899
900 /*
901 * Something did happen, check if a re-emit is needed
902 * (this also clears paca->irq_happened)
903 */
904 restore_check_irq_replay:
905 /* XXX: We could implement a fast path here where we check
906 * for irq_happened being just 0x01, in which case we can
907 * clear it and return. That means that we would potentially
908 * miss a decrementer having wrapped all the way around.
909 *
910 * Still, this might be useful for things like hash_page
911 */
912 bl __check_irq_replay
913 cmpwi cr0,r3,0
914 beq restore_no_replay
915
916 /*
917 * We need to re-emit an interrupt. We do so by re-using our
918 * existing exception frame. We first change the trap value,
919 * but we need to ensure we preserve the low nibble of it
920 */
921 ld r4,_TRAP(r1)
922 clrldi r4,r4,60
923 or r4,r4,r3
924 std r4,_TRAP(r1)
925
926 /*
927 * Then find the right handler and call it. Interrupts are
928 * still soft-disabled and we keep them that way.
929 */
930 cmpwi cr0,r3,0x500
931 bne 1f
932 addi r3,r1,STACK_FRAME_OVERHEAD;
933 bl do_IRQ
934 b ret_from_except
935 1: cmpwi cr0,r3,0xe60
936 bne 1f
937 addi r3,r1,STACK_FRAME_OVERHEAD;
938 bl handle_hmi_exception
939 b ret_from_except
940 1: cmpwi cr0,r3,0x900
941 bne 1f
942 addi r3,r1,STACK_FRAME_OVERHEAD;
943 bl timer_interrupt
944 b ret_from_except
945 #ifdef CONFIG_PPC_DOORBELL
946 1:
947 #ifdef CONFIG_PPC_BOOK3E
948 cmpwi cr0,r3,0x280
949 #else
950 BEGIN_FTR_SECTION
951 cmpwi cr0,r3,0xe80
952 FTR_SECTION_ELSE
953 cmpwi cr0,r3,0xa00
954 ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE)
955 #endif /* CONFIG_PPC_BOOK3E */
956 bne 1f
957 addi r3,r1,STACK_FRAME_OVERHEAD;
958 bl doorbell_exception
959 b ret_from_except
960 #endif /* CONFIG_PPC_DOORBELL */
961 1: b ret_from_except /* What else to do here ? */
962
963 unrecov_restore:
964 addi r3,r1,STACK_FRAME_OVERHEAD
965 bl unrecoverable_exception
966 b unrecov_restore
967
968 #ifdef CONFIG_PPC_RTAS
969 /*
970 * On CHRP, the Run-Time Abstraction Services (RTAS) have to be
971 * called with the MMU off.
972 *
973 * In addition, we need to be in 32b mode, at least for now.
974 *
975 * Note: r3 is an input parameter to rtas, so don't trash it...
976 */
977 _GLOBAL(enter_rtas)
978 mflr r0
979 std r0,16(r1)
980 stdu r1,-RTAS_FRAME_SIZE(r1) /* Save SP and create stack space. */
981
982 /* Because RTAS is running in 32b mode, it clobbers the high order half
983 * of all registers that it saves. We therefore save those registers
984 * RTAS might touch to the stack. (r0, r3-r13 are caller saved)
985 */
986 SAVE_GPR(2, r1) /* Save the TOC */
987 SAVE_GPR(13, r1) /* Save paca */
988 SAVE_8GPRS(14, r1) /* Save the non-volatiles */
989 SAVE_10GPRS(22, r1) /* ditto */
990
991 mfcr r4
992 std r4,_CCR(r1)
993 mfctr r5
994 std r5,_CTR(r1)
995 mfspr r6,SPRN_XER
996 std r6,_XER(r1)
997 mfdar r7
998 std r7,_DAR(r1)
999 mfdsisr r8
1000 std r8,_DSISR(r1)
1001
1002 /* Temporary workaround to clear CR until RTAS can be modified to
1003 * ignore all bits.
1004 */
1005 li r0,0
1006 mtcr r0
1007
1008 #ifdef CONFIG_BUG
1009 /* There is no way it is acceptable to get here with interrupts enabled,
1010 * check it with the asm equivalent of WARN_ON
1011 */
1012 lbz r0,PACASOFTIRQEN(r13)
1013 1: tdnei r0,0
1014 EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,BUGFLAG_WARNING
1015 #endif
1016
1017 /* Hard-disable interrupts */
1018 mfmsr r6
1019 rldicl r7,r6,48,1
1020 rotldi r7,r7,16
1021 mtmsrd r7,1
1022
1023 /* Unfortunately, the stack pointer and the MSR are also clobbered,
1024 * so they are saved in the PACA which allows us to restore
1025 * our original state after RTAS returns.
1026 */
1027 std r1,PACAR1(r13)
1028 std r6,PACASAVEDMSR(r13)
1029
1030 /* Setup our real return addr */
1031 LOAD_REG_ADDR(r4,rtas_return_loc)
1032 clrldi r4,r4,2 /* convert to realmode address */
1033 mtlr r4
1034
1035 li r0,0
1036 ori r0,r0,MSR_EE|MSR_SE|MSR_BE|MSR_RI
1037 andc r0,r6,r0
1038
1039 li r9,1
1040 rldicr r9,r9,MSR_SF_LG,(63-MSR_SF_LG)
1041 ori r9,r9,MSR_IR|MSR_DR|MSR_FE0|MSR_FE1|MSR_FP|MSR_RI|MSR_LE
1042 andc r6,r0,r9
1043 sync /* disable interrupts so SRR0/1 */
1044 mtmsrd r0 /* don't get trashed */
1045
1046 LOAD_REG_ADDR(r4, rtas)
1047 ld r5,RTASENTRY(r4) /* get the rtas->entry value */
1048 ld r4,RTASBASE(r4) /* get the rtas->base value */
1049
1050 mtspr SPRN_SRR0,r5
1051 mtspr SPRN_SRR1,r6
1052 rfid
1053 b . /* prevent speculative execution */
1054
1055 rtas_return_loc:
1056 FIXUP_ENDIAN
1057
1058 /* relocation is off at this point */
1059 GET_PACA(r4)
1060 clrldi r4,r4,2 /* convert to realmode address */
1061
1062 bcl 20,31,$+4
1063 0: mflr r3
1064 ld r3,(1f-0b)(r3) /* get &rtas_restore_regs */
1065
1066 mfmsr r6
1067 li r0,MSR_RI
1068 andc r6,r6,r0
1069 sync
1070 mtmsrd r6
1071
1072 ld r1,PACAR1(r4) /* Restore our SP */
1073 ld r4,PACASAVEDMSR(r4) /* Restore our MSR */
1074
1075 mtspr SPRN_SRR0,r3
1076 mtspr SPRN_SRR1,r4
1077 rfid
1078 b . /* prevent speculative execution */
1079
1080 .align 3
1081 1: .llong rtas_restore_regs
1082
1083 rtas_restore_regs:
1084 /* relocation is on at this point */
1085 REST_GPR(2, r1) /* Restore the TOC */
1086 REST_GPR(13, r1) /* Restore paca */
1087 REST_8GPRS(14, r1) /* Restore the non-volatiles */
1088 REST_10GPRS(22, r1) /* ditto */
1089
1090 GET_PACA(r13)
1091
1092 ld r4,_CCR(r1)
1093 mtcr r4
1094 ld r5,_CTR(r1)
1095 mtctr r5
1096 ld r6,_XER(r1)
1097 mtspr SPRN_XER,r6
1098 ld r7,_DAR(r1)
1099 mtdar r7
1100 ld r8,_DSISR(r1)
1101 mtdsisr r8
1102
1103 addi r1,r1,RTAS_FRAME_SIZE /* Unstack our frame */
1104 ld r0,16(r1) /* get return address */
1105
1106 mtlr r0
1107 blr /* return to caller */
1108
1109 #endif /* CONFIG_PPC_RTAS */
1110
1111 _GLOBAL(enter_prom)
1112 mflr r0
1113 std r0,16(r1)
1114 stdu r1,-PROM_FRAME_SIZE(r1) /* Save SP and create stack space */
1115
1116 /* Because PROM is running in 32b mode, it clobbers the high order half
1117 * of all registers that it saves. We therefore save those registers
1118 * PROM might touch to the stack. (r0, r3-r13 are caller saved)
1119 */
1120 SAVE_GPR(2, r1)
1121 SAVE_GPR(13, r1)
1122 SAVE_8GPRS(14, r1)
1123 SAVE_10GPRS(22, r1)
1124 mfcr r10
1125 mfmsr r11
1126 std r10,_CCR(r1)
1127 std r11,_MSR(r1)
1128
1129 /* Put PROM address in SRR0 */
1130 mtsrr0 r4
1131
1132 /* Setup our trampoline return addr in LR */
1133 bcl 20,31,$+4
1134 0: mflr r4
1135 addi r4,r4,(1f - 0b)
1136 mtlr r4
1137
1138 /* Prepare a 32-bit mode big endian MSR
1139 */
1140 #ifdef CONFIG_PPC_BOOK3E
1141 rlwinm r11,r11,0,1,31
1142 mtsrr1 r11
1143 rfi
1144 #else /* CONFIG_PPC_BOOK3E */
1145 LOAD_REG_IMMEDIATE(r12, MSR_SF | MSR_ISF | MSR_LE)
1146 andc r11,r11,r12
1147 mtsrr1 r11
1148 rfid
1149 #endif /* CONFIG_PPC_BOOK3E */
1150
1151 1: /* Return from OF */
1152 FIXUP_ENDIAN
1153
1154 /* Just make sure that r1 top 32 bits didn't get
1155 * corrupt by OF
1156 */
1157 rldicl r1,r1,0,32
1158
1159 /* Restore the MSR (back to 64 bits) */
1160 ld r0,_MSR(r1)
1161 MTMSRD(r0)
1162 isync
1163
1164 /* Restore other registers */
1165 REST_GPR(2, r1)
1166 REST_GPR(13, r1)
1167 REST_8GPRS(14, r1)
1168 REST_10GPRS(22, r1)
1169 ld r4,_CCR(r1)
1170 mtcr r4
1171
1172 addi r1,r1,PROM_FRAME_SIZE
1173 ld r0,16(r1)
1174 mtlr r0
1175 blr
1176
1177 #ifdef CONFIG_FUNCTION_TRACER
1178 #ifdef CONFIG_DYNAMIC_FTRACE
1179 _GLOBAL(mcount)
1180 _GLOBAL(_mcount)
1181 blr
1182
1183 _GLOBAL_TOC(ftrace_caller)
1184 /* Taken from output of objdump from lib64/glibc */
1185 mflr r3
1186 ld r11, 0(r1)
1187 stdu r1, -112(r1)
1188 std r3, 128(r1)
1189 ld r4, 16(r11)
1190 subi r3, r3, MCOUNT_INSN_SIZE
1191 .globl ftrace_call
1192 ftrace_call:
1193 bl ftrace_stub
1194 nop
1195 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1196 .globl ftrace_graph_call
1197 ftrace_graph_call:
1198 b ftrace_graph_stub
1199 _GLOBAL(ftrace_graph_stub)
1200 #endif
1201 ld r0, 128(r1)
1202 mtlr r0
1203 addi r1, r1, 112
1204 _GLOBAL(ftrace_stub)
1205 blr
1206 #else
1207 _GLOBAL_TOC(_mcount)
1208 /* Taken from output of objdump from lib64/glibc */
1209 mflr r3
1210 ld r11, 0(r1)
1211 stdu r1, -112(r1)
1212 std r3, 128(r1)
1213 ld r4, 16(r11)
1214
1215 subi r3, r3, MCOUNT_INSN_SIZE
1216 LOAD_REG_ADDR(r5,ftrace_trace_function)
1217 ld r5,0(r5)
1218 ld r5,0(r5)
1219 mtctr r5
1220 bctrl
1221 nop
1222
1223
1224 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1225 b ftrace_graph_caller
1226 #endif
1227 ld r0, 128(r1)
1228 mtlr r0
1229 addi r1, r1, 112
1230 _GLOBAL(ftrace_stub)
1231 blr
1232
1233 #endif /* CONFIG_DYNAMIC_FTRACE */
1234
1235 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1236 _GLOBAL(ftrace_graph_caller)
1237 /* load r4 with local address */
1238 ld r4, 128(r1)
1239 subi r4, r4, MCOUNT_INSN_SIZE
1240
1241 /* Grab the LR out of the caller stack frame */
1242 ld r11, 112(r1)
1243 ld r3, 16(r11)
1244
1245 bl prepare_ftrace_return
1246 nop
1247
1248 /*
1249 * prepare_ftrace_return gives us the address we divert to.
1250 * Change the LR in the callers stack frame to this.
1251 */
1252 ld r11, 112(r1)
1253 std r3, 16(r11)
1254
1255 ld r0, 128(r1)
1256 mtlr r0
1257 addi r1, r1, 112
1258 blr
1259
1260 _GLOBAL(return_to_handler)
1261 /* need to save return values */
1262 std r4, -32(r1)
1263 std r3, -24(r1)
1264 /* save TOC */
1265 std r2, -16(r1)
1266 std r31, -8(r1)
1267 mr r31, r1
1268 stdu r1, -112(r1)
1269
1270 /*
1271 * We might be called from a module.
1272 * Switch to our TOC to run inside the core kernel.
1273 */
1274 ld r2, PACATOC(r13)
1275
1276 bl ftrace_return_to_handler
1277 nop
1278
1279 /* return value has real return address */
1280 mtlr r3
1281
1282 ld r1, 0(r1)
1283 ld r4, -32(r1)
1284 ld r3, -24(r1)
1285 ld r2, -16(r1)
1286 ld r31, -8(r1)
1287
1288 /* Jump back to real return address */
1289 blr
1290 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
1291 #endif /* CONFIG_FUNCTION_TRACER */
Linux kernel - Deliverables
This page took 0.060656 seconds and 5 git commands to generate.