1/* SPDX-License-Identifier: GPL-2.0-or-later */ 2/* 3 * Split from ftrace_64.S 4 */ 5 6#include <linux/magic.h> 7#include <asm/ppc_asm.h> 8#include <asm/asm-offsets.h> 9#include <asm/ftrace.h> 10#include <asm/ppc-opcode.h> 11#include <asm/export.h> 12#include <asm/thread_info.h> 13#include <asm/bug.h> 14#include <asm/ptrace.h> 15 16/* 17 * 18 * ftrace_caller()/ftrace_regs_caller() is the function that replaces _mcount() 19 * when ftrace is active. 20 * 21 * We arrive here after a function A calls function B, and we are the trace 22 * function for B. When we enter r1 points to A's stack frame, B has not yet 23 * had a chance to allocate one yet. 24 * 25 * Additionally r2 may point either to the TOC for A, or B, depending on 26 * whether B did a TOC setup sequence before calling us. 27 * 28 * On entry the LR points back to the _mcount() call site, and r0 holds the 29 * saved LR as it was on entry to B, ie. the original return address at the 30 * call site in A. 31 * 32 * Our job is to save the register state into a struct pt_regs (on the stack) 33 * and then arrange for the ftrace function to be called. 34 */ 35_GLOBAL(ftrace_regs_caller) 36 /* Save the original return address in A's stack frame */ 37 std r0,LRSAVE(r1) 38 39 /* Create our stack frame + pt_regs */ 40 stdu r1,-SWITCH_FRAME_SIZE(r1) 41 42 /* Save all gprs to pt_regs */ 43 SAVE_GPR(0, r1) 44 SAVE_10GPRS(2, r1) 45 46 /* Ok to continue? */ 47 lbz r3, PACA_FTRACE_ENABLED(r13) 48 cmpdi r3, 0 49 beq ftrace_no_trace 50 51 SAVE_10GPRS(12, r1) 52 SAVE_10GPRS(22, r1) 53 54 /* Save previous stack pointer (r1) */ 55 addi r8, r1, SWITCH_FRAME_SIZE 56 std r8, GPR1(r1) 57 58 /* Load special regs for save below */ 59 mfmsr r8 60 mfctr r9 61 mfxer r10 62 mfcr r11 63 64 /* Get the _mcount() call site out of LR */ 65 mflr r7 66 /* Save it as pt_regs->nip */ 67 std r7, _NIP(r1) 68 /* Save the read LR in pt_regs->link */ 69 std r0, _LINK(r1) 70 71 /* Save callee's TOC in the ABI compliant location */ 72 std r2, 24(r1) 73 ld r2,PACATOC(r13) /* get kernel TOC in r2 */ 74 75 addis r3,r2,function_trace_op@toc@ha 76 addi r3,r3,function_trace_op@toc@l 77 ld r5,0(r3) 78 79#ifdef CONFIG_LIVEPATCH 80 mr r14,r7 /* remember old NIP */ 81#endif 82 /* Calculate ip from nip-4 into r3 for call below */ 83 subi r3, r7, MCOUNT_INSN_SIZE 84 85 /* Put the original return address in r4 as parent_ip */ 86 mr r4, r0 87 88 /* Save special regs */ 89 std r8, _MSR(r1) 90 std r9, _CTR(r1) 91 std r10, _XER(r1) 92 std r11, _CCR(r1) 93 94 /* Load &pt_regs in r6 for call below */ 95 addi r6, r1 ,STACK_FRAME_OVERHEAD 96 97 /* ftrace_call(r3, r4, r5, r6) */ 98.globl ftrace_regs_call 99ftrace_regs_call: 100 bl ftrace_stub 101 nop 102 103 /* Load ctr with the possibly modified NIP */ 104 ld r3, _NIP(r1) 105 mtctr r3 106#ifdef CONFIG_LIVEPATCH 107 cmpd r14, r3 /* has NIP been altered? */ 108#endif 109 110 /* Restore gprs */ 111 REST_GPR(0,r1) 112 REST_10GPRS(2,r1) 113 REST_10GPRS(12,r1) 114 REST_10GPRS(22,r1) 115 116 /* Restore possibly modified LR */ 117 ld r0, _LINK(r1) 118 mtlr r0 119 120 /* Restore callee's TOC */ 121 ld r2, 24(r1) 122 123 /* Pop our stack frame */ 124 addi r1, r1, SWITCH_FRAME_SIZE 125 126#ifdef CONFIG_LIVEPATCH 127 /* Based on the cmpd above, if the NIP was altered handle livepatch */ 128 bne- livepatch_handler 129#endif 130 131ftrace_caller_common: 132#ifdef CONFIG_FUNCTION_GRAPH_TRACER 133.globl ftrace_graph_call 134ftrace_graph_call: 135 b ftrace_graph_stub 136_GLOBAL(ftrace_graph_stub) 137#endif 138 139 bctr /* jump after _mcount site */ 140 141_GLOBAL(ftrace_stub) 142 blr 143 144ftrace_no_trace: 145 mflr r3 146 mtctr r3 147 REST_GPR(3, r1) 148 addi r1, r1, SWITCH_FRAME_SIZE 149 mtlr r0 150 bctr 151 152_GLOBAL(ftrace_caller) 153 /* Save the original return address in A's stack frame */ 154 std r0, LRSAVE(r1) 155 156 /* Create our stack frame + pt_regs */ 157 stdu r1, -SWITCH_FRAME_SIZE(r1) 158 159 /* Save all gprs to pt_regs */ 160 SAVE_8GPRS(3, r1) 161 162 lbz r3, PACA_FTRACE_ENABLED(r13) 163 cmpdi r3, 0 164 beq ftrace_no_trace 165 166 /* Get the _mcount() call site out of LR */ 167 mflr r7 168 std r7, _NIP(r1) 169 170 /* Save callee's TOC in the ABI compliant location */ 171 std r2, 24(r1) 172 ld r2, PACATOC(r13) /* get kernel TOC in r2 */ 173 174 addis r3, r2, function_trace_op@toc@ha 175 addi r3, r3, function_trace_op@toc@l 176 ld r5, 0(r3) 177 178 /* Calculate ip from nip-4 into r3 for call below */ 179 subi r3, r7, MCOUNT_INSN_SIZE 180 181 /* Put the original return address in r4 as parent_ip */ 182 mr r4, r0 183 184 /* Set pt_regs to NULL */ 185 li r6, 0 186 187 /* ftrace_call(r3, r4, r5, r6) */ 188.globl ftrace_call 189ftrace_call: 190 bl ftrace_stub 191 nop 192 193 ld r3, _NIP(r1) 194 mtctr r3 195 196 /* Restore gprs */ 197 REST_8GPRS(3,r1) 198 199 /* Restore callee's TOC */ 200 ld r2, 24(r1) 201 202 /* Pop our stack frame */ 203 addi r1, r1, SWITCH_FRAME_SIZE 204 205 /* Reload original LR */ 206 ld r0, LRSAVE(r1) 207 mtlr r0 208 209 /* Handle function_graph or go back */ 210 b ftrace_caller_common 211 212#ifdef CONFIG_LIVEPATCH 213 /* 214 * This function runs in the mcount context, between two functions. As 215 * such it can only clobber registers which are volatile and used in 216 * function linkage. 217 * 218 * We get here when a function A, calls another function B, but B has 219 * been live patched with a new function C. 220 * 221 * On entry: 222 * - we have no stack frame and can not allocate one 223 * - LR points back to the original caller (in A) 224 * - CTR holds the new NIP in C 225 * - r0, r11 & r12 are free 226 */ 227livepatch_handler: 228 ld r12, PACA_THREAD_INFO(r13) 229 230 /* Allocate 3 x 8 bytes */ 231 ld r11, TI_livepatch_sp(r12) 232 addi r11, r11, 24 233 std r11, TI_livepatch_sp(r12) 234 235 /* Save toc & real LR on livepatch stack */ 236 std r2, -24(r11) 237 mflr r12 238 std r12, -16(r11) 239 240 /* Store stack end marker */ 241 lis r12, STACK_END_MAGIC@h 242 ori r12, r12, STACK_END_MAGIC@l 243 std r12, -8(r11) 244 245 /* Put ctr in r12 for global entry and branch there */ 246 mfctr r12 247 bctrl 248 249 /* 250 * Now we are returning from the patched function to the original 251 * caller A. We are free to use r11, r12 and we can use r2 until we 252 * restore it. 253 */ 254 255 ld r12, PACA_THREAD_INFO(r13) 256 257 ld r11, TI_livepatch_sp(r12) 258 259 /* Check stack marker hasn't been trashed */ 260 lis r2, STACK_END_MAGIC@h 261 ori r2, r2, STACK_END_MAGIC@l 262 ld r12, -8(r11) 2631: tdne r12, r2 264 EMIT_BUG_ENTRY 1b, __FILE__, __LINE__ - 1, 0 265 266 /* Restore LR & toc from livepatch stack */ 267 ld r12, -16(r11) 268 mtlr r12 269 ld r2, -24(r11) 270 271 /* Pop livepatch stack frame */ 272 ld r12, PACA_THREAD_INFO(r13) 273 subi r11, r11, 24 274 std r11, TI_livepatch_sp(r12) 275 276 /* Return to original caller of live patched function */ 277 blr 278#endif /* CONFIG_LIVEPATCH */ 279 280#ifdef CONFIG_FUNCTION_GRAPH_TRACER 281_GLOBAL(ftrace_graph_caller) 282 stdu r1, -112(r1) 283 /* with -mprofile-kernel, parameter regs are still alive at _mcount */ 284 std r10, 104(r1) 285 std r9, 96(r1) 286 std r8, 88(r1) 287 std r7, 80(r1) 288 std r6, 72(r1) 289 std r5, 64(r1) 290 std r4, 56(r1) 291 std r3, 48(r1) 292 293 /* Save callee's TOC in the ABI compliant location */ 294 std r2, 24(r1) 295 ld r2, PACATOC(r13) /* get kernel TOC in r2 */ 296 297 addi r5, r1, 112 298 mfctr r4 /* ftrace_caller has moved local addr here */ 299 std r4, 40(r1) 300 mflr r3 /* ftrace_caller has restored LR from stack */ 301 subi r4, r4, MCOUNT_INSN_SIZE 302 303 bl prepare_ftrace_return 304 nop 305 306 /* 307 * prepare_ftrace_return gives us the address we divert to. 308 * Change the LR to this. 309 */ 310 mtlr r3 311 312 ld r0, 40(r1) 313 mtctr r0 314 ld r10, 104(r1) 315 ld r9, 96(r1) 316 ld r8, 88(r1) 317 ld r7, 80(r1) 318 ld r6, 72(r1) 319 ld r5, 64(r1) 320 ld r4, 56(r1) 321 ld r3, 48(r1) 322 323 /* Restore callee's TOC */ 324 ld r2, 24(r1) 325 326 addi r1, r1, 112 327 mflr r0 328 std r0, LRSAVE(r1) 329 bctr 330#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 331