1 /*
2 * PowerPC version
3 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
4 * Rewritten by Cort Dougan (cort@cs.nmt.edu) for PReP
5 * Copyright (C) 1996 Cort Dougan <cort@cs.nmt.edu>
6 * Adapted for Power Macintosh by Paul Mackerras.
7 * Low-level exception handlers and MMU support
8 * rewritten by Paul Mackerras.
9 * Copyright (C) 1996 Paul Mackerras.
10 * MPC8xx modifications Copyright (C) 1997 Dan Malek (dmalek@jlc.net).
11 *
12 * This file contains the system call entry code, context switch
13 * code, and exception/interrupt return code for PowerPC.
14 *
15 * This program is free software; you can redistribute it and/or
16 * modify it under the terms of the GNU General Public License
17 * as published by the Free Software Foundation; either version
18 * 2 of the License, or (at your option) any later version.
19 */
21 #include <linux/errno.h>
22 #include <asm/unistd.h>
23 #include <asm/processor.h>
24 #include <asm/page.h>
25 #include <asm/mmu.h>
26 #include <asm/thread_info.h>
27 #include <asm/ppc_asm.h>
28 #include <asm/asm-offsets.h>
29 #include <asm/cputable.h>
30 #include <asm/firmware.h>
31 #include <asm/bug.h>
32 #include <asm/ptrace.h>
33 #include <asm/irqflags.h>
34 #include <asm/ftrace.h>
35 #include <asm/hw_irq.h>
37 /*
38 * System calls.
39 */
40 .section ".toc","aw"
41 .SYS_CALL_TABLE:
42 .tc .sys_call_table[TC],.sys_call_table
44 /* This value is used to mark exception frames on the stack. */
45 exception_marker:
46 .tc ID_EXC_MARKER[TC],STACK_FRAME_REGS_MARKER
48 .section ".text"
49 .align 7
51 #undef SHOW_SYSCALLS
53 .globl system_call_common
54 system_call_common:
55 andi. r10,r12,MSR_PR
56 mr r10,r1
57 addi r1,r1,-INT_FRAME_SIZE
58 beq- 1f
59 ld r1,PACAKSAVE(r13)
60 1: std r10,0(r1)
61 std r11,_NIP(r1)
62 std r12,_MSR(r1)
63 std r0,GPR0(r1)
64 std r10,GPR1(r1)
65 ACCOUNT_CPU_USER_ENTRY(r10, r11)
66 std r2,GPR2(r1)
67 std r3,GPR3(r1)
68 mfcr r2
69 std r4,GPR4(r1)
70 std r5,GPR5(r1)
71 std r6,GPR6(r1)
72 std r7,GPR7(r1)
73 std r8,GPR8(r1)
74 li r11,0
75 std r11,GPR9(r1)
76 std r11,GPR10(r1)
77 std r11,GPR11(r1)
78 std r11,GPR12(r1)
79 std r11,_XER(r1)
80 std r11,_CTR(r1)
81 std r9,GPR13(r1)
82 mflr r10
83 /*
84 * This clears CR0.SO (bit 28), which is the error indication on
85 * return from this system call.
86 */
87 rldimi r2,r11,28,(63-28)
88 li r11,0xc01
89 std r10,_LINK(r1)
90 std r11,_TRAP(r1)
91 std r3,ORIG_GPR3(r1)
92 std r2,_CCR(r1)
93 ld r2,PACATOC(r13)
94 addi r9,r1,STACK_FRAME_OVERHEAD
95 ld r11,exception_marker@toc(r2)
96 std r11,-16(r9) /* "regshere" marker */
97 #if defined(CONFIG_VIRT_CPU_ACCOUNTING) && defined(CONFIG_PPC_SPLPAR)
98 BEGIN_FW_FTR_SECTION
99 beq 33f
100 /* if from user, see if there are any DTL entries to process */
101 ld r10,PACALPPACAPTR(r13) /* get ptr to VPA */
102 ld r11,PACA_DTL_RIDX(r13) /* get log read index */
103 ld r10,LPPACA_DTLIDX(r10) /* get log write index */
104 cmpd cr1,r11,r10
105 beq+ cr1,33f
106 bl .accumulate_stolen_time
107 REST_GPR(0,r1)
108 REST_4GPRS(3,r1)
109 REST_2GPRS(7,r1)
110 addi r9,r1,STACK_FRAME_OVERHEAD
111 33:
112 END_FW_FTR_SECTION_IFSET(FW_FEATURE_SPLPAR)
113 #endif /* CONFIG_VIRT_CPU_ACCOUNTING && CONFIG_PPC_SPLPAR */
115 /*
116 * A syscall should always be called with interrupts enabled
117 * so we just unconditionally hard-enable here. When some kind
118 * of irq tracing is used, we additionally check that condition
119 * is correct
120 */
121 #if defined(CONFIG_TRACE_IRQFLAGS) && defined(CONFIG_BUG)
122 lbz r10,PACASOFTIRQEN(r13)
123 xori r10,r10,1
124 1: tdnei r10,0
125 EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,BUGFLAG_WARNING
126 #endif
128 #ifdef CONFIG_PPC_BOOK3E
129 wrteei 1
130 #else
131 ld r11,PACAKMSR(r13)
132 ori r11,r11,MSR_EE
133 mtmsrd r11,1
134 #endif /* CONFIG_PPC_BOOK3E */
136 /* We do need to set SOFTE in the stack frame or the return
137 * from interrupt will be painful
138 */
139 li r10,1
140 std r10,SOFTE(r1)
142 #ifdef SHOW_SYSCALLS
143 bl .do_show_syscall
144 REST_GPR(0,r1)
145 REST_4GPRS(3,r1)
146 REST_2GPRS(7,r1)
147 addi r9,r1,STACK_FRAME_OVERHEAD
148 #endif
149 CURRENT_THREAD_INFO(r11, r1)
150 ld r10,TI_FLAGS(r11)
151 andi. r11,r10,_TIF_SYSCALL_T_OR_A
152 bne- syscall_dotrace
153 .Lsyscall_dotrace_cont:
154 cmpldi 0,r0,NR_syscalls
155 bge- syscall_enosys
157 system_call: /* label this so stack traces look sane */
158 /*
159 * Need to vector to 32 Bit or default sys_call_table here,
160 * based on caller's run-mode / personality.
161 */
162 ld r11,.SYS_CALL_TABLE@toc(2)
163 andi. r10,r10,_TIF_32BIT
164 beq 15f
165 addi r11,r11,8 /* use 32-bit syscall entries */
166 clrldi r3,r3,32
167 clrldi r4,r4,32
168 clrldi r5,r5,32
169 clrldi r6,r6,32
170 clrldi r7,r7,32
171 clrldi r8,r8,32
172 15:
173 slwi r0,r0,4
174 ldx r10,r11,r0 /* Fetch system call handler [ptr] */
175 mtctr r10
176 bctrl /* Call handler */
178 syscall_exit:
179 std r3,RESULT(r1)
180 #ifdef SHOW_SYSCALLS
181 bl .do_show_syscall_exit
182 ld r3,RESULT(r1)
183 #endif
184 CURRENT_THREAD_INFO(r12, r1)
186 ld r8,_MSR(r1)
187 #ifdef CONFIG_PPC_BOOK3S
188 /* No MSR:RI on BookE */
189 andi. r10,r8,MSR_RI
190 beq- unrecov_restore
191 #endif
192 /*
193 * Disable interrupts so current_thread_info()->flags can't change,
194 * and so that we don't get interrupted after loading SRR0/1.
195 */
196 #ifdef CONFIG_PPC_BOOK3E
197 wrteei 0
198 #else
199 ld r10,PACAKMSR(r13)
200 /*
201 * For performance reasons we clear RI the same time that we
202 * clear EE. We only need to clear RI just before we restore r13
203 * below, but batching it with EE saves us one expensive mtmsrd call.
204 * We have to be careful to restore RI if we branch anywhere from
205 * here (eg syscall_exit_work).
206 */
207 li r9,MSR_RI
208 andc r11,r10,r9
209 mtmsrd r11,1
210 #endif /* CONFIG_PPC_BOOK3E */
212 ld r9,TI_FLAGS(r12)
213 li r11,-_LAST_ERRNO
214 andi. r0,r9,(_TIF_SYSCALL_T_OR_A|_TIF_SINGLESTEP|_TIF_USER_WORK_MASK|_TIF_PERSYSCALL_MASK)
215 bne- syscall_exit_work
216 cmpld r3,r11
217 ld r5,_CCR(r1)
218 bge- syscall_error
219 .Lsyscall_error_cont:
220 ld r7,_NIP(r1)
221 BEGIN_FTR_SECTION
222 stdcx. r0,0,r1 /* to clear the reservation */
223 END_FTR_SECTION_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
224 andi. r6,r8,MSR_PR
225 ld r4,_LINK(r1)
227 beq- 1f
228 ACCOUNT_CPU_USER_EXIT(r11, r12)
229 ld r13,GPR13(r1) /* only restore r13 if returning to usermode */
230 1: ld r2,GPR2(r1)
231 ld r1,GPR1(r1)
232 mtlr r4
233 mtcr r5
234 mtspr SPRN_SRR0,r7
235 mtspr SPRN_SRR1,r8
236 RFI
237 b . /* prevent speculative execution */
239 syscall_error:
240 oris r5,r5,0x1000 /* Set SO bit in CR */
241 neg r3,r3
242 std r5,_CCR(r1)
243 b .Lsyscall_error_cont
245 /* Traced system call support */
246 syscall_dotrace:
247 bl .save_nvgprs
248 addi r3,r1,STACK_FRAME_OVERHEAD
249 bl .do_syscall_trace_enter
250 /*
251 * Restore argument registers possibly just changed.
252 * We use the return value of do_syscall_trace_enter
253 * for the call number to look up in the table (r0).
254 */
255 mr r0,r3
256 ld r3,GPR3(r1)
257 ld r4,GPR4(r1)
258 ld r5,GPR5(r1)
259 ld r6,GPR6(r1)
260 ld r7,GPR7(r1)
261 ld r8,GPR8(r1)
262 addi r9,r1,STACK_FRAME_OVERHEAD
263 CURRENT_THREAD_INFO(r10, r1)
264 ld r10,TI_FLAGS(r10)
265 b .Lsyscall_dotrace_cont
267 syscall_enosys:
268 li r3,-ENOSYS
269 b syscall_exit
271 syscall_exit_work:
272 #ifdef CONFIG_PPC_BOOK3S
273 mtmsrd r10,1 /* Restore RI */
274 #endif
275 /* If TIF_RESTOREALL is set, don't scribble on either r3 or ccr.
276 If TIF_NOERROR is set, just save r3 as it is. */
278 andi. r0,r9,_TIF_RESTOREALL
279 beq+ 0f
280 REST_NVGPRS(r1)
281 b 2f
282 0: cmpld r3,r11 /* r10 is -LAST_ERRNO */
283 blt+ 1f
284 andi. r0,r9,_TIF_NOERROR
285 bne- 1f
286 ld r5,_CCR(r1)
287 neg r3,r3
288 oris r5,r5,0x1000 /* Set SO bit in CR */
289 std r5,_CCR(r1)
290 1: std r3,GPR3(r1)
291 2: andi. r0,r9,(_TIF_PERSYSCALL_MASK)
292 beq 4f
294 /* Clear per-syscall TIF flags if any are set. */
296 li r11,_TIF_PERSYSCALL_MASK
297 addi r12,r12,TI_FLAGS
298 3: ldarx r10,0,r12
299 andc r10,r10,r11
300 stdcx. r10,0,r12
301 bne- 3b
302 subi r12,r12,TI_FLAGS
304 4: /* Anything else left to do? */
305 andi. r0,r9,(_TIF_SYSCALL_T_OR_A|_TIF_SINGLESTEP)
306 beq .ret_from_except_lite
308 /* Re-enable interrupts */
309 #ifdef CONFIG_PPC_BOOK3E
310 wrteei 1
311 #else
312 ld r10,PACAKMSR(r13)
313 ori r10,r10,MSR_EE
314 mtmsrd r10,1
315 #endif /* CONFIG_PPC_BOOK3E */
317 bl .save_nvgprs
318 addi r3,r1,STACK_FRAME_OVERHEAD
319 bl .do_syscall_trace_leave
320 b .ret_from_except
322 /* Save non-volatile GPRs, if not already saved. */
323 _GLOBAL(save_nvgprs)
324 ld r11,_TRAP(r1)
325 andi. r0,r11,1
326 beqlr-
327 SAVE_NVGPRS(r1)
328 clrrdi r0,r11,1
329 std r0,_TRAP(r1)
330 blr
333 /*
334 * The sigsuspend and rt_sigsuspend system calls can call do_signal
335 * and thus put the process into the stopped state where we might
336 * want to examine its user state with ptrace. Therefore we need
337 * to save all the nonvolatile registers (r14 - r31) before calling
338 * the C code. Similarly, fork, vfork and clone need the full
339 * register state on the stack so that it can be copied to the child.
340 */
342 _GLOBAL(ppc_fork)
343 bl .save_nvgprs
344 bl .sys_fork
345 b syscall_exit
347 _GLOBAL(ppc_vfork)
348 bl .save_nvgprs
349 bl .sys_vfork
350 b syscall_exit
352 _GLOBAL(ppc_clone)
353 bl .save_nvgprs
354 bl .sys_clone
355 b syscall_exit
357 _GLOBAL(ppc32_swapcontext)
358 bl .save_nvgprs
359 bl .compat_sys_swapcontext
360 b syscall_exit
362 _GLOBAL(ppc64_swapcontext)
363 bl .save_nvgprs
364 bl .sys_swapcontext
365 b syscall_exit
367 _GLOBAL(ret_from_fork)
368 bl .schedule_tail
369 REST_NVGPRS(r1)
370 li r3,0
371 b syscall_exit
373 _GLOBAL(ret_from_kernel_thread)
374 bl .schedule_tail
375 REST_NVGPRS(r1)
376 REST_GPR(2,r1)
377 li r3,0
378 std r3,0(r1)
379 mtlr r14
380 mr r3,r15
381 blrl
382 li r3,0
383 b .do_exit # no return
385 _GLOBAL(__ret_from_kernel_execve)
386 addi r1,r3,-STACK_FRAME_OVERHEAD
387 li r10,1
388 std r10,SOFTE(r1)
389 b syscall_exit
391 .section ".toc","aw"
392 DSCR_DEFAULT:
393 .tc dscr_default[TC],dscr_default
395 .section ".text"
397 /*
398 * This routine switches between two different tasks. The process
399 * state of one is saved on its kernel stack. Then the state
400 * of the other is restored from its kernel stack. The memory
401 * management hardware is updated to the second process's state.
402 * Finally, we can return to the second process, via ret_from_except.
403 * On entry, r3 points to the THREAD for the current task, r4
404 * points to the THREAD for the new task.
405 *
406 * Note: there are two ways to get to the "going out" portion
407 * of this code; either by coming in via the entry (_switch)
408 * or via "fork" which must set up an environment equivalent
409 * to the "_switch" path. If you change this you'll have to change
410 * the fork code also.
411 *
412 * The code which creates the new task context is in 'copy_thread'
413 * in arch/powerpc/kernel/process.c
414 */
415 .align 7
416 _GLOBAL(_switch)
417 mflr r0
418 std r0,16(r1)
419 stdu r1,-SWITCH_FRAME_SIZE(r1)
420 /* r3-r13 are caller saved -- Cort */
421 SAVE_8GPRS(14, r1)
422 SAVE_10GPRS(22, r1)
423 mflr r20 /* Return to switch caller */
424 mfmsr r22
425 li r0, MSR_FP
426 #ifdef CONFIG_VSX
427 BEGIN_FTR_SECTION
428 oris r0,r0,MSR_VSX@h /* Disable VSX */
429 END_FTR_SECTION_IFSET(CPU_FTR_VSX)
430 #endif /* CONFIG_VSX */
431 #ifdef CONFIG_ALTIVEC
432 BEGIN_FTR_SECTION
433 oris r0,r0,MSR_VEC@h /* Disable altivec */
434 mfspr r24,SPRN_VRSAVE /* save vrsave register value */
435 std r24,THREAD_VRSAVE(r3)
436 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
437 #endif /* CONFIG_ALTIVEC */
438 #ifdef CONFIG_PPC64
439 BEGIN_FTR_SECTION
440 mfspr r25,SPRN_DSCR
441 std r25,THREAD_DSCR(r3)
442 END_FTR_SECTION_IFSET(CPU_FTR_DSCR)
443 #endif
444 and. r0,r0,r22
445 beq+ 1f
446 andc r22,r22,r0
447 MTMSRD(r22)
448 isync
449 1: std r20,_NIP(r1)
450 mfcr r23
451 std r23,_CCR(r1)
452 std r1,KSP(r3) /* Set old stack pointer */
454 #ifdef CONFIG_SMP
455 /* We need a sync somewhere here to make sure that if the
456 * previous task gets rescheduled on another CPU, it sees all
457 * stores it has performed on this one.
458 */
459 sync
460 #endif /* CONFIG_SMP */
462 /*
463 * If we optimise away the clear of the reservation in system
464 * calls because we know the CPU tracks the address of the
465 * reservation, then we need to clear it here to cover the
466 * case that the kernel context switch path has no larx
467 * instructions.
468 */
469 BEGIN_FTR_SECTION
470 ldarx r6,0,r1
471 END_FTR_SECTION_IFSET(CPU_FTR_STCX_CHECKS_ADDRESS)
473 addi r6,r4,-THREAD /* Convert THREAD to 'current' */
474 std r6,PACACURRENT(r13) /* Set new 'current' */
476 ld r8,KSP(r4) /* new stack pointer */
477 #ifdef CONFIG_PPC_BOOK3S
478 BEGIN_FTR_SECTION
479 BEGIN_FTR_SECTION_NESTED(95)
480 clrrdi r6,r8,28 /* get its ESID */
481 clrrdi r9,r1,28 /* get current sp ESID */
482 FTR_SECTION_ELSE_NESTED(95)
483 clrrdi r6,r8,40 /* get its 1T ESID */
484 clrrdi r9,r1,40 /* get current sp 1T ESID */
485 ALT_MMU_FTR_SECTION_END_NESTED_IFCLR(MMU_FTR_1T_SEGMENT, 95)
486 FTR_SECTION_ELSE
487 b 2f
488 ALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_SLB)
489 clrldi. r0,r6,2 /* is new ESID c00000000? */
490 cmpd cr1,r6,r9 /* or is new ESID the same as current ESID? */
491 cror eq,4*cr1+eq,eq
492 beq 2f /* if yes, don't slbie it */
494 /* Bolt in the new stack SLB entry */
495 ld r7,KSP_VSID(r4) /* Get new stack's VSID */
496 oris r0,r6,(SLB_ESID_V)@h
497 ori r0,r0,(SLB_NUM_BOLTED-1)@l
498 BEGIN_FTR_SECTION
499 li r9,MMU_SEGSIZE_1T /* insert B field */
500 oris r6,r6,(MMU_SEGSIZE_1T << SLBIE_SSIZE_SHIFT)@h
501 rldimi r7,r9,SLB_VSID_SSIZE_SHIFT,0
502 END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT)
504 /* Update the last bolted SLB. No write barriers are needed
505 * here, provided we only update the current CPU's SLB shadow
506 * buffer.
507 */
508 ld r9,PACA_SLBSHADOWPTR(r13)
509 li r12,0
510 std r12,SLBSHADOW_STACKESID(r9) /* Clear ESID */
511 std r7,SLBSHADOW_STACKVSID(r9) /* Save VSID */
512 std r0,SLBSHADOW_STACKESID(r9) /* Save ESID */
514 /* No need to check for MMU_FTR_NO_SLBIE_B here, since when
515 * we have 1TB segments, the only CPUs known to have the errata
516 * only support less than 1TB of system memory and we'll never
517 * actually hit this code path.
518 */
520 slbie r6
521 slbie r6 /* Workaround POWER5 < DD2.1 issue */
522 slbmte r7,r0
523 isync
524 2:
525 #endif /* !CONFIG_PPC_BOOK3S */
527 CURRENT_THREAD_INFO(r7, r8) /* base of new stack */
528 /* Note: this uses SWITCH_FRAME_SIZE rather than INT_FRAME_SIZE
529 because we don't need to leave the 288-byte ABI gap at the
530 top of the kernel stack. */
531 addi r7,r7,THREAD_SIZE-SWITCH_FRAME_SIZE
533 mr r1,r8 /* start using new stack pointer */
534 std r7,PACAKSAVE(r13)
536 #ifdef CONFIG_ALTIVEC
537 BEGIN_FTR_SECTION
538 ld r0,THREAD_VRSAVE(r4)
539 mtspr SPRN_VRSAVE,r0 /* if G4, restore VRSAVE reg */
540 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
541 #endif /* CONFIG_ALTIVEC */
542 #ifdef CONFIG_PPC64
543 BEGIN_FTR_SECTION
544 lwz r6,THREAD_DSCR_INHERIT(r4)
545 ld r7,DSCR_DEFAULT@toc(2)
546 ld r0,THREAD_DSCR(r4)
547 cmpwi r6,0
548 bne 1f
549 ld r0,0(r7)
550 1: cmpd r0,r25
551 beq 2f
552 mtspr SPRN_DSCR,r0
553 2:
554 END_FTR_SECTION_IFSET(CPU_FTR_DSCR)
555 #endif
557 ld r6,_CCR(r1)
558 mtcrf 0xFF,r6
560 /* r3-r13 are destroyed -- Cort */
561 REST_8GPRS(14, r1)
562 REST_10GPRS(22, r1)
564 /* convert old thread to its task_struct for return value */
565 addi r3,r3,-THREAD
566 ld r7,_NIP(r1) /* Return to _switch caller in new task */
567 mtlr r7
568 addi r1,r1,SWITCH_FRAME_SIZE
569 blr
571 .align 7
572 _GLOBAL(ret_from_except)
573 ld r11,_TRAP(r1)
574 andi. r0,r11,1
575 bne .ret_from_except_lite
576 REST_NVGPRS(r1)
578 _GLOBAL(ret_from_except_lite)
579 /*
580 * Disable interrupts so that current_thread_info()->flags
581 * can't change between when we test it and when we return
582 * from the interrupt.
583 */
584 #ifdef CONFIG_PPC_BOOK3E
585 wrteei 0
586 #else
587 ld r10,PACAKMSR(r13) /* Get kernel MSR without EE */
588 mtmsrd r10,1 /* Update machine state */
589 #endif /* CONFIG_PPC_BOOK3E */
591 CURRENT_THREAD_INFO(r9, r1)
592 ld r3,_MSR(r1)
593 ld r4,TI_FLAGS(r9)
594 andi. r3,r3,MSR_PR
595 beq resume_kernel
597 /* Check current_thread_info()->flags */
598 andi. r0,r4,_TIF_USER_WORK_MASK
599 beq restore
601 andi. r0,r4,_TIF_NEED_RESCHED
602 beq 1f
603 bl .restore_interrupts
604 bl .schedule
605 b .ret_from_except_lite
607 1: bl .save_nvgprs
608 bl .restore_interrupts
609 addi r3,r1,STACK_FRAME_OVERHEAD
610 bl .do_notify_resume
611 b .ret_from_except
613 resume_kernel:
614 /* check current_thread_info, _TIF_EMULATE_STACK_STORE */
615 CURRENT_THREAD_INFO(r9, r1)
616 ld r8,TI_FLAGS(r9)
617 andis. r8,r8,_TIF_EMULATE_STACK_STORE@h
618 beq+ 1f
620 addi r8,r1,INT_FRAME_SIZE /* Get the kprobed function entry */
622 lwz r3,GPR1(r1)
623 subi r3,r3,INT_FRAME_SIZE /* dst: Allocate a trampoline exception frame */
624 mr r4,r1 /* src: current exception frame */
625 mr r1,r3 /* Reroute the trampoline frame to r1 */
627 /* Copy from the original to the trampoline. */
628 li r5,INT_FRAME_SIZE/8 /* size: INT_FRAME_SIZE */
629 li r6,0 /* start offset: 0 */
630 mtctr r5
631 2: ldx r0,r6,r4
632 stdx r0,r6,r3
633 addi r6,r6,8
634 bdnz 2b
636 /* Do real store operation to complete stwu */
637 lwz r5,GPR1(r1)
638 std r8,0(r5)
640 /* Clear _TIF_EMULATE_STACK_STORE flag */
641 lis r11,_TIF_EMULATE_STACK_STORE@h
642 addi r5,r9,TI_FLAGS
643 ldarx r4,0,r5
644 andc r4,r4,r11
645 stdcx. r4,0,r5
646 bne- 0b
647 1:
649 #ifdef CONFIG_PREEMPT
650 /* Check if we need to preempt */
651 andi. r0,r4,_TIF_NEED_RESCHED
652 beq+ restore
653 /* Check that preempt_count() == 0 and interrupts are enabled */
654 lwz r8,TI_PREEMPT(r9)
655 cmpwi cr1,r8,0
656 ld r0,SOFTE(r1)
657 cmpdi r0,0
658 crandc eq,cr1*4+eq,eq
659 bne restore
661 /*
662 * Here we are preempting the current task. We want to make
663 * sure we are soft-disabled first
664 */
665 SOFT_DISABLE_INTS(r3,r4)
666 1: bl .preempt_schedule_irq
668 /* Re-test flags and eventually loop */
669 CURRENT_THREAD_INFO(r9, r1)
670 ld r4,TI_FLAGS(r9)
671 andi. r0,r4,_TIF_NEED_RESCHED
672 bne 1b
673 #endif /* CONFIG_PREEMPT */
675 .globl fast_exc_return_irq
676 fast_exc_return_irq:
677 restore:
678 /*
679 * This is the main kernel exit path. First we check if we
680 * are about to re-enable interrupts
681 */
682 ld r5,SOFTE(r1)
683 lbz r6,PACASOFTIRQEN(r13)
684 cmpwi cr0,r5,0
685 beq restore_irq_off
687 /* We are enabling, were we already enabled ? Yes, just return */
688 cmpwi cr0,r6,1
689 beq cr0,do_restore
691 /*
692 * We are about to soft-enable interrupts (we are hard disabled
693 * at this point). We check if there's anything that needs to
694 * be replayed first.
695 */
696 lbz r0,PACAIRQHAPPENED(r13)
697 cmpwi cr0,r0,0
698 bne- restore_check_irq_replay
700 /*
701 * Get here when nothing happened while soft-disabled, just
702 * soft-enable and move-on. We will hard-enable as a side
703 * effect of rfi
704 */
705 restore_no_replay:
706 TRACE_ENABLE_INTS
707 li r0,1
708 stb r0,PACASOFTIRQEN(r13);
710 /*
711 * Final return path. BookE is handled in a different file
712 */
713 do_restore:
714 #ifdef CONFIG_PPC_BOOK3E
715 b .exception_return_book3e
716 #else
717 /*
718 * Clear the reservation. If we know the CPU tracks the address of
719 * the reservation then we can potentially save some cycles and use
720 * a larx. On POWER6 and POWER7 this is significantly faster.
721 */
722 BEGIN_FTR_SECTION
723 stdcx. r0,0,r1 /* to clear the reservation */
724 FTR_SECTION_ELSE
725 ldarx r4,0,r1
726 ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
728 /*
729 * Some code path such as load_up_fpu or altivec return directly
730 * here. They run entirely hard disabled and do not alter the
731 * interrupt state. They also don't use lwarx/stwcx. and thus
732 * are known not to leave dangling reservations.
733 */
734 .globl fast_exception_return
735 fast_exception_return:
736 ld r3,_MSR(r1)
737 ld r4,_CTR(r1)
738 ld r0,_LINK(r1)
739 mtctr r4
740 mtlr r0
741 ld r4,_XER(r1)
742 mtspr SPRN_XER,r4
744 REST_8GPRS(5, r1)
746 andi. r0,r3,MSR_RI
747 beq- unrecov_restore
749 /*
750 * Clear RI before restoring r13. If we are returning to
751 * userspace and we take an exception after restoring r13,
752 * we end up corrupting the userspace r13 value.
753 */
754 ld r4,PACAKMSR(r13) /* Get kernel MSR without EE */
755 andc r4,r4,r0 /* r0 contains MSR_RI here */
756 mtmsrd r4,1
758 /*
759 * r13 is our per cpu area, only restore it if we are returning to
760 * userspace the value stored in the stack frame may belong to
761 * another CPU.
762 */
763 andi. r0,r3,MSR_PR
764 beq 1f
765 ACCOUNT_CPU_USER_EXIT(r2, r4)
766 REST_GPR(13, r1)
767 1:
768 mtspr SPRN_SRR1,r3
770 ld r2,_CCR(r1)
771 mtcrf 0xFF,r2
772 ld r2,_NIP(r1)
773 mtspr SPRN_SRR0,r2
775 ld r0,GPR0(r1)
776 ld r2,GPR2(r1)
777 ld r3,GPR3(r1)
778 ld r4,GPR4(r1)
779 ld r1,GPR1(r1)
781 rfid
782 b . /* prevent speculative execution */
784 #endif /* CONFIG_PPC_BOOK3E */
786 /*
787 * We are returning to a context with interrupts soft disabled.
788 *
789 * However, we may also about to hard enable, so we need to
790 * make sure that in this case, we also clear PACA_IRQ_HARD_DIS
791 * or that bit can get out of sync and bad things will happen
792 */
793 restore_irq_off:
794 ld r3,_MSR(r1)
795 lbz r7,PACAIRQHAPPENED(r13)
796 andi. r0,r3,MSR_EE
797 beq 1f
798 rlwinm r7,r7,0,~PACA_IRQ_HARD_DIS
799 stb r7,PACAIRQHAPPENED(r13)
800 1: li r0,0
801 stb r0,PACASOFTIRQEN(r13);
802 TRACE_DISABLE_INTS
803 b do_restore
805 /*
806 * Something did happen, check if a re-emit is needed
807 * (this also clears paca->irq_happened)
808 */
809 restore_check_irq_replay:
810 /* XXX: We could implement a fast path here where we check
811 * for irq_happened being just 0x01, in which case we can
812 * clear it and return. That means that we would potentially
813 * miss a decrementer having wrapped all the way around.
814 *
815 * Still, this might be useful for things like hash_page
816 */
817 bl .__check_irq_replay
818 cmpwi cr0,r3,0
819 beq restore_no_replay
821 /*
822 * We need to re-emit an interrupt. We do so by re-using our
823 * existing exception frame. We first change the trap value,
824 * but we need to ensure we preserve the low nibble of it
825 */
826 ld r4,_TRAP(r1)
827 clrldi r4,r4,60
828 or r4,r4,r3
829 std r4,_TRAP(r1)
831 /*
832 * Then find the right handler and call it. Interrupts are
833 * still soft-disabled and we keep them that way.
834 */
835 cmpwi cr0,r3,0x500
836 bne 1f
837 addi r3,r1,STACK_FRAME_OVERHEAD;
838 bl .do_IRQ
839 b .ret_from_except
840 1: cmpwi cr0,r3,0x900
841 bne 1f
842 addi r3,r1,STACK_FRAME_OVERHEAD;
843 bl .timer_interrupt
844 b .ret_from_except
845 #ifdef CONFIG_PPC_BOOK3E
846 1: cmpwi cr0,r3,0x280
847 bne 1f
848 addi r3,r1,STACK_FRAME_OVERHEAD;
849 bl .doorbell_exception
850 b .ret_from_except
851 #endif /* CONFIG_PPC_BOOK3E */
852 1: b .ret_from_except /* What else to do here ? */
854 unrecov_restore:
855 addi r3,r1,STACK_FRAME_OVERHEAD
856 bl .unrecoverable_exception
857 b unrecov_restore
859 #ifdef CONFIG_PPC_RTAS
860 /*
861 * On CHRP, the Run-Time Abstraction Services (RTAS) have to be
862 * called with the MMU off.
863 *
864 * In addition, we need to be in 32b mode, at least for now.
865 *
866 * Note: r3 is an input parameter to rtas, so don't trash it...
867 */
868 _GLOBAL(enter_rtas)
869 mflr r0
870 std r0,16(r1)
871 stdu r1,-RTAS_FRAME_SIZE(r1) /* Save SP and create stack space. */
873 /* Because RTAS is running in 32b mode, it clobbers the high order half
874 * of all registers that it saves. We therefore save those registers
875 * RTAS might touch to the stack. (r0, r3-r13 are caller saved)
876 */
877 SAVE_GPR(2, r1) /* Save the TOC */
878 SAVE_GPR(13, r1) /* Save paca */
879 SAVE_8GPRS(14, r1) /* Save the non-volatiles */
880 SAVE_10GPRS(22, r1) /* ditto */
882 mfcr r4
883 std r4,_CCR(r1)
884 mfctr r5
885 std r5,_CTR(r1)
886 mfspr r6,SPRN_XER
887 std r6,_XER(r1)
888 mfdar r7
889 std r7,_DAR(r1)
890 mfdsisr r8
891 std r8,_DSISR(r1)
893 /* Temporary workaround to clear CR until RTAS can be modified to
894 * ignore all bits.
895 */
896 li r0,0
897 mtcr r0
899 #ifdef CONFIG_BUG
900 /* There is no way it is acceptable to get here with interrupts enabled,
901 * check it with the asm equivalent of WARN_ON
902 */
903 lbz r0,PACASOFTIRQEN(r13)
904 1: tdnei r0,0
905 EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,BUGFLAG_WARNING
906 #endif
908 /* Hard-disable interrupts */
909 mfmsr r6
910 rldicl r7,r6,48,1
911 rotldi r7,r7,16
912 mtmsrd r7,1
914 /* Unfortunately, the stack pointer and the MSR are also clobbered,
915 * so they are saved in the PACA which allows us to restore
916 * our original state after RTAS returns.
917 */
918 std r1,PACAR1(r13)
919 std r6,PACASAVEDMSR(r13)
921 /* Setup our real return addr */
922 LOAD_REG_ADDR(r4,.rtas_return_loc)
923 clrldi r4,r4,2 /* convert to realmode address */
924 mtlr r4
926 li r0,0
927 ori r0,r0,MSR_EE|MSR_SE|MSR_BE|MSR_RI
928 andc r0,r6,r0
930 li r9,1
931 rldicr r9,r9,MSR_SF_LG,(63-MSR_SF_LG)
932 ori r9,r9,MSR_IR|MSR_DR|MSR_FE0|MSR_FE1|MSR_FP|MSR_RI
933 andc r6,r0,r9
934 sync /* disable interrupts so SRR0/1 */
935 mtmsrd r0 /* don't get trashed */
937 LOAD_REG_ADDR(r4, rtas)
938 ld r5,RTASENTRY(r4) /* get the rtas->entry value */
939 ld r4,RTASBASE(r4) /* get the rtas->base value */
941 mtspr SPRN_SRR0,r5
942 mtspr SPRN_SRR1,r6
943 rfid
944 b . /* prevent speculative execution */
946 _STATIC(rtas_return_loc)
947 /* relocation is off at this point */
948 GET_PACA(r4)
949 clrldi r4,r4,2 /* convert to realmode address */
951 bcl 20,31,$+4
952 0: mflr r3
953 ld r3,(1f-0b)(r3) /* get &.rtas_restore_regs */
955 mfmsr r6
956 li r0,MSR_RI
957 andc r6,r6,r0
958 sync
959 mtmsrd r6
961 ld r1,PACAR1(r4) /* Restore our SP */
962 ld r4,PACASAVEDMSR(r4) /* Restore our MSR */
964 mtspr SPRN_SRR0,r3
965 mtspr SPRN_SRR1,r4
966 rfid
967 b . /* prevent speculative execution */
969 .align 3
970 1: .llong .rtas_restore_regs
972 _STATIC(rtas_restore_regs)
973 /* relocation is on at this point */
974 REST_GPR(2, r1) /* Restore the TOC */
975 REST_GPR(13, r1) /* Restore paca */
976 REST_8GPRS(14, r1) /* Restore the non-volatiles */
977 REST_10GPRS(22, r1) /* ditto */
979 GET_PACA(r13)
981 ld r4,_CCR(r1)
982 mtcr r4
983 ld r5,_CTR(r1)
984 mtctr r5
985 ld r6,_XER(r1)
986 mtspr SPRN_XER,r6
987 ld r7,_DAR(r1)
988 mtdar r7
989 ld r8,_DSISR(r1)
990 mtdsisr r8
992 addi r1,r1,RTAS_FRAME_SIZE /* Unstack our frame */
993 ld r0,16(r1) /* get return address */
995 mtlr r0
996 blr /* return to caller */
998 #endif /* CONFIG_PPC_RTAS */
1000 _GLOBAL(enter_prom)
1001 mflr r0
1002 std r0,16(r1)
1003 stdu r1,-PROM_FRAME_SIZE(r1) /* Save SP and create stack space */
1005 /* Because PROM is running in 32b mode, it clobbers the high order half
1006 * of all registers that it saves. We therefore save those registers
1007 * PROM might touch to the stack. (r0, r3-r13 are caller saved)
1008 */
1009 SAVE_GPR(2, r1)
1010 SAVE_GPR(13, r1)
1011 SAVE_8GPRS(14, r1)
1012 SAVE_10GPRS(22, r1)
1013 mfcr r10
1014 mfmsr r11
1015 std r10,_CCR(r1)
1016 std r11,_MSR(r1)
1018 /* Get the PROM entrypoint */
1019 mtlr r4
1021 /* Switch MSR to 32 bits mode
1022 */
1023 #ifdef CONFIG_PPC_BOOK3E
1024 rlwinm r11,r11,0,1,31
1025 mtmsr r11
1026 #else /* CONFIG_PPC_BOOK3E */
1027 mfmsr r11
1028 li r12,1
1029 rldicr r12,r12,MSR_SF_LG,(63-MSR_SF_LG)
1030 andc r11,r11,r12
1031 li r12,1
1032 rldicr r12,r12,MSR_ISF_LG,(63-MSR_ISF_LG)
1033 andc r11,r11,r12
1034 mtmsrd r11
1035 #endif /* CONFIG_PPC_BOOK3E */
1036 isync
1038 /* Enter PROM here... */
1039 blrl
1041 /* Just make sure that r1 top 32 bits didn't get
1042 * corrupt by OF
1043 */
1044 rldicl r1,r1,0,32
1046 /* Restore the MSR (back to 64 bits) */
1047 ld r0,_MSR(r1)
1048 MTMSRD(r0)
1049 isync
1051 /* Restore other registers */
1052 REST_GPR(2, r1)
1053 REST_GPR(13, r1)
1054 REST_8GPRS(14, r1)
1055 REST_10GPRS(22, r1)
1056 ld r4,_CCR(r1)
1057 mtcr r4
1059 addi r1,r1,PROM_FRAME_SIZE
1060 ld r0,16(r1)
1061 mtlr r0
1062 blr
1064 #ifdef CONFIG_FUNCTION_TRACER
1065 #ifdef CONFIG_DYNAMIC_FTRACE
1066 _GLOBAL(mcount)
1067 _GLOBAL(_mcount)
1068 blr
1070 _GLOBAL(ftrace_caller)
1071 /* Taken from output of objdump from lib64/glibc */
1072 mflr r3
1073 ld r11, 0(r1)
1074 stdu r1, -112(r1)
1075 std r3, 128(r1)
1076 ld r4, 16(r11)
1077 subi r3, r3, MCOUNT_INSN_SIZE
1078 .globl ftrace_call
1079 ftrace_call:
1080 bl ftrace_stub
1081 nop
1082 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1083 .globl ftrace_graph_call
1084 ftrace_graph_call:
1085 b ftrace_graph_stub
1086 _GLOBAL(ftrace_graph_stub)
1087 #endif
1088 ld r0, 128(r1)
1089 mtlr r0
1090 addi r1, r1, 112
1091 _GLOBAL(ftrace_stub)
1092 blr
1093 #else
1094 _GLOBAL(mcount)
1095 blr
1097 _GLOBAL(_mcount)
1098 /* Taken from output of objdump from lib64/glibc */
1099 mflr r3
1100 ld r11, 0(r1)
1101 stdu r1, -112(r1)
1102 std r3, 128(r1)
1103 ld r4, 16(r11)
1105 subi r3, r3, MCOUNT_INSN_SIZE
1106 LOAD_REG_ADDR(r5,ftrace_trace_function)
1107 ld r5,0(r5)
1108 ld r5,0(r5)
1109 mtctr r5
1110 bctrl
1111 nop
1114 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1115 b ftrace_graph_caller
1116 #endif
1117 ld r0, 128(r1)
1118 mtlr r0
1119 addi r1, r1, 112
1120 _GLOBAL(ftrace_stub)
1121 blr
1123 #endif /* CONFIG_DYNAMIC_FTRACE */
1125 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1126 _GLOBAL(ftrace_graph_caller)
1127 /* load r4 with local address */
1128 ld r4, 128(r1)
1129 subi r4, r4, MCOUNT_INSN_SIZE
1131 /* get the parent address */
1132 ld r11, 112(r1)
1133 addi r3, r11, 16
1135 bl .prepare_ftrace_return
1136 nop
1138 ld r0, 128(r1)
1139 mtlr r0
1140 addi r1, r1, 112
1141 blr
1143 _GLOBAL(return_to_handler)
1144 /* need to save return values */
1145 std r4, -24(r1)
1146 std r3, -16(r1)
1147 std r31, -8(r1)
1148 mr r31, r1
1149 stdu r1, -112(r1)
1151 bl .ftrace_return_to_handler
1152 nop
1154 /* return value has real return address */
1155 mtlr r3
1157 ld r1, 0(r1)
1158 ld r4, -24(r1)
1159 ld r3, -16(r1)
1160 ld r31, -8(r1)
1162 /* Jump back to real return address */
1163 blr
1165 _GLOBAL(mod_return_to_handler)
1166 /* need to save return values */
1167 std r4, -32(r1)
1168 std r3, -24(r1)
1169 /* save TOC */
1170 std r2, -16(r1)
1171 std r31, -8(r1)
1172 mr r31, r1
1173 stdu r1, -112(r1)
1175 /*
1176 * We are in a module using the module's TOC.
1177 * Switch to our TOC to run inside the core kernel.
1178 */
1179 ld r2, PACATOC(r13)
1181 bl .ftrace_return_to_handler
1182 nop
1184 /* return value has real return address */
1185 mtlr r3
1187 ld r1, 0(r1)
1188 ld r4, -32(r1)
1189 ld r3, -24(r1)
1190 ld r2, -16(r1)
1191 ld r31, -8(r1)
1193 /* Jump back to real return address */
1194 blr
1195 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
1196 #endif /* CONFIG_FUNCTION_TRACER */