Commit | Line | Data |
---|---|---|
fad57feb | 1 | /* |
7780b6a2 | 2 | * Copyright (C) 2008 Matt Fleming <matt@console-pimps.org> |
b5cfeac9 | 3 | * Copyright (C) 2008 Paul Mundt <lethal@linux-sh.org> |
fad57feb MF |
4 | * |
5 | * Code for replacing ftrace calls with jumps. | |
6 | * | |
7 | * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com> | |
8 | * | |
9 | * Thanks goes to Ingo Molnar, for suggesting the idea. | |
10 | * Mathieu Desnoyers, for suggesting postponing the modifications. | |
11 | * Arjan van de Ven, for keeping me straight, and explaining to me | |
12 | * the dangers of modifying code on the run. | |
13 | */ | |
14 | #include <linux/uaccess.h> | |
15 | #include <linux/ftrace.h> | |
16 | #include <linux/string.h> | |
17 | #include <linux/init.h> | |
18 | #include <linux/io.h> | |
327933f5 | 19 | #include <linux/kernel.h> |
fad57feb MF |
20 | #include <asm/ftrace.h> |
21 | #include <asm/cacheflush.h> | |
c652d780 MF |
22 | #include <asm/unistd.h> |
23 | #include <trace/syscall.h> | |
fad57feb | 24 | |
327933f5 | 25 | #ifdef CONFIG_DYNAMIC_FTRACE |
fad57feb MF |
26 | static unsigned char ftrace_replaced_code[MCOUNT_INSN_SIZE]; |
27 | ||
9e28c46b MF |
28 | static unsigned char ftrace_nop[4]; |
29 | /* | |
30 | * If we're trying to nop out a call to a function, we instead | |
31 | * place a call to the address after the memory table. | |
32 | * | |
33 | * 8c011060 <a>: | |
34 | * 8c011060: 02 d1 mov.l 8c01106c <a+0xc>,r1 | |
35 | * 8c011062: 22 4f sts.l pr,@-r15 | |
36 | * 8c011064: 02 c7 mova 8c011070 <a+0x10>,r0 | |
37 | * 8c011066: 2b 41 jmp @r1 | |
38 | * 8c011068: 2a 40 lds r0,pr | |
39 | * 8c01106a: 09 00 nop | |
40 | * 8c01106c: 68 24 .word 0x2468 <--- ip | |
41 | * 8c01106e: 1d 8c .word 0x8c1d | |
42 | * 8c011070: 26 4f lds.l @r15+,pr <--- ip + MCOUNT_INSN_SIZE | |
43 | * | |
44 | * We write 0x8c011070 to 0x8c01106c so that on entry to a() we branch | |
45 | * past the _mcount call and continue executing code like normal. | |
46 | */ | |
47 | static unsigned char *ftrace_nop_replace(unsigned long ip) | |
fad57feb | 48 | { |
9e28c46b | 49 | __raw_writel(ip + MCOUNT_INSN_SIZE, ftrace_nop); |
fad57feb MF |
50 | return ftrace_nop; |
51 | } | |
52 | ||
9e28c46b | 53 | static unsigned char *ftrace_call_replace(unsigned long ip, unsigned long addr) |
fad57feb MF |
54 | { |
55 | /* Place the address in the memory table. */ | |
9e28c46b | 56 | __raw_writel(addr, ftrace_replaced_code); |
fad57feb MF |
57 | |
58 | /* | |
59 | * No locking needed, this must be called via kstop_machine | |
60 | * which in essence is like running on a uniprocessor machine. | |
61 | */ | |
62 | return ftrace_replaced_code; | |
63 | } | |
64 | ||
e4b053d9 PM |
65 | /* |
66 | * Modifying code must take extra care. On an SMP machine, if | |
67 | * the code being modified is also being executed on another CPU | |
68 | * that CPU will have undefined results and possibly take a GPF. | |
69 | * We use kstop_machine to stop other CPUS from exectuing code. | |
70 | * But this does not stop NMIs from happening. We still need | |
71 | * to protect against that. We separate out the modification of | |
72 | * the code to take care of this. | |
73 | * | |
74 | * Two buffers are added: An IP buffer and a "code" buffer. | |
75 | * | |
76 | * 1) Put the instruction pointer into the IP buffer | |
77 | * and the new code into the "code" buffer. | |
78 | * 2) Wait for any running NMIs to finish and set a flag that says | |
79 | * we are modifying code, it is done in an atomic operation. | |
80 | * 3) Write the code | |
81 | * 4) clear the flag. | |
82 | * 5) Wait for any running NMIs to finish. | |
83 | * | |
84 | * If an NMI is executed, the first thing it does is to call | |
85 | * "ftrace_nmi_enter". This will check if the flag is set to write | |
86 | * and if it is, it will write what is in the IP and "code" buffers. | |
87 | * | |
88 | * The trick is, it does not matter if everyone is writing the same | |
89 | * content to the code location. Also, if a CPU is executing code | |
90 | * it is OK to write to that code location if the contents being written | |
91 | * are the same as what exists. | |
92 | */ | |
93 | #define MOD_CODE_WRITE_FLAG (1 << 31) /* set when NMI should do the write */ | |
94 | static atomic_t nmi_running = ATOMIC_INIT(0); | |
95 | static int mod_code_status; /* holds return value of text write */ | |
96 | static void *mod_code_ip; /* holds the IP to write to */ | |
97 | static void *mod_code_newcode; /* holds the text to write to the IP */ | |
98 | ||
99 | static unsigned nmi_wait_count; | |
100 | static atomic_t nmi_update_count = ATOMIC_INIT(0); | |
101 | ||
102 | int ftrace_arch_read_dyn_info(char *buf, int size) | |
103 | { | |
104 | int r; | |
105 | ||
106 | r = snprintf(buf, size, "%u %u", | |
107 | nmi_wait_count, | |
108 | atomic_read(&nmi_update_count)); | |
109 | return r; | |
110 | } | |
111 | ||
112 | static void clear_mod_flag(void) | |
113 | { | |
114 | int old = atomic_read(&nmi_running); | |
115 | ||
116 | for (;;) { | |
117 | int new = old & ~MOD_CODE_WRITE_FLAG; | |
118 | ||
119 | if (old == new) | |
120 | break; | |
121 | ||
122 | old = atomic_cmpxchg(&nmi_running, old, new); | |
123 | } | |
124 | } | |
125 | ||
126 | static void ftrace_mod_code(void) | |
127 | { | |
128 | /* | |
129 | * Yes, more than one CPU process can be writing to mod_code_status. | |
130 | * (and the code itself) | |
131 | * But if one were to fail, then they all should, and if one were | |
132 | * to succeed, then they all should. | |
133 | */ | |
134 | mod_code_status = probe_kernel_write(mod_code_ip, mod_code_newcode, | |
135 | MCOUNT_INSN_SIZE); | |
136 | ||
137 | /* if we fail, then kill any new writers */ | |
138 | if (mod_code_status) | |
139 | clear_mod_flag(); | |
140 | } | |
141 | ||
7b2c8625 | 142 | void arch_ftrace_nmi_enter(void) |
e4b053d9 PM |
143 | { |
144 | if (atomic_inc_return(&nmi_running) & MOD_CODE_WRITE_FLAG) { | |
145 | smp_rmb(); | |
146 | ftrace_mod_code(); | |
147 | atomic_inc(&nmi_update_count); | |
148 | } | |
149 | /* Must have previous changes seen before executions */ | |
150 | smp_mb(); | |
151 | } | |
152 | ||
7b2c8625 | 153 | void arch_ftrace_nmi_exit(void) |
e4b053d9 PM |
154 | { |
155 | /* Finish all executions before clearing nmi_running */ | |
156 | smp_mb(); | |
157 | atomic_dec(&nmi_running); | |
158 | } | |
159 | ||
160 | static void wait_for_nmi_and_set_mod_flag(void) | |
161 | { | |
162 | if (!atomic_cmpxchg(&nmi_running, 0, MOD_CODE_WRITE_FLAG)) | |
163 | return; | |
164 | ||
165 | do { | |
166 | cpu_relax(); | |
167 | } while (atomic_cmpxchg(&nmi_running, 0, MOD_CODE_WRITE_FLAG)); | |
168 | ||
169 | nmi_wait_count++; | |
170 | } | |
171 | ||
172 | static void wait_for_nmi(void) | |
173 | { | |
174 | if (!atomic_read(&nmi_running)) | |
175 | return; | |
176 | ||
177 | do { | |
178 | cpu_relax(); | |
179 | } while (atomic_read(&nmi_running)); | |
180 | ||
181 | nmi_wait_count++; | |
182 | } | |
183 | ||
184 | static int | |
185 | do_ftrace_mod_code(unsigned long ip, void *new_code) | |
186 | { | |
187 | mod_code_ip = (void *)ip; | |
188 | mod_code_newcode = new_code; | |
189 | ||
190 | /* The buffers need to be visible before we let NMIs write them */ | |
191 | smp_mb(); | |
192 | ||
193 | wait_for_nmi_and_set_mod_flag(); | |
194 | ||
195 | /* Make sure all running NMIs have finished before we write the code */ | |
196 | smp_mb(); | |
197 | ||
198 | ftrace_mod_code(); | |
199 | ||
200 | /* Make sure the write happens before clearing the bit */ | |
201 | smp_mb(); | |
202 | ||
203 | clear_mod_flag(); | |
204 | wait_for_nmi(); | |
205 | ||
206 | return mod_code_status; | |
207 | } | |
208 | ||
9e28c46b | 209 | static int ftrace_modify_code(unsigned long ip, unsigned char *old_code, |
fad57feb MF |
210 | unsigned char *new_code) |
211 | { | |
212 | unsigned char replaced[MCOUNT_INSN_SIZE]; | |
213 | ||
214 | /* | |
5243238a LB |
215 | * Note: |
216 | * We are paranoid about modifying text, as if a bug was to happen, it | |
217 | * could cause us to read or write to someplace that could cause harm. | |
218 | * Carefully read and modify the code with probe_kernel_*(), and make | |
219 | * sure what we read is what we expected it to be before modifying it. | |
fad57feb MF |
220 | */ |
221 | ||
fad57feb MF |
222 | /* read the text we want to modify */ |
223 | if (probe_kernel_read(replaced, (void *)ip, MCOUNT_INSN_SIZE)) | |
224 | return -EFAULT; | |
225 | ||
226 | /* Make sure it is what we expect it to be */ | |
227 | if (memcmp(replaced, old_code, MCOUNT_INSN_SIZE) != 0) | |
228 | return -EINVAL; | |
229 | ||
230 | /* replace the text with the new text */ | |
e4b053d9 | 231 | if (do_ftrace_mod_code(ip, new_code)) |
fad57feb MF |
232 | return -EPERM; |
233 | ||
234 | flush_icache_range(ip, ip + MCOUNT_INSN_SIZE); | |
235 | ||
236 | return 0; | |
237 | } | |
238 | ||
239 | int ftrace_update_ftrace_func(ftrace_func_t func) | |
240 | { | |
9e28c46b | 241 | unsigned long ip = (unsigned long)(&ftrace_call) + MCOUNT_INSN_OFFSET; |
fad57feb MF |
242 | unsigned char old[MCOUNT_INSN_SIZE], *new; |
243 | ||
9e28c46b | 244 | memcpy(old, (unsigned char *)ip, MCOUNT_INSN_SIZE); |
fad57feb MF |
245 | new = ftrace_call_replace(ip, (unsigned long)func); |
246 | ||
9e28c46b | 247 | return ftrace_modify_code(ip, old, new); |
fad57feb MF |
248 | } |
249 | ||
b5cfeac9 PM |
250 | int ftrace_make_nop(struct module *mod, |
251 | struct dyn_ftrace *rec, unsigned long addr) | |
252 | { | |
253 | unsigned char *new, *old; | |
254 | unsigned long ip = rec->ip; | |
255 | ||
256 | old = ftrace_call_replace(ip, addr); | |
9e28c46b | 257 | new = ftrace_nop_replace(ip); |
b5cfeac9 PM |
258 | |
259 | return ftrace_modify_code(rec->ip, old, new); | |
260 | } | |
261 | ||
262 | int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) | |
263 | { | |
264 | unsigned char *new, *old; | |
265 | unsigned long ip = rec->ip; | |
266 | ||
9e28c46b | 267 | old = ftrace_nop_replace(ip); |
b5cfeac9 PM |
268 | new = ftrace_call_replace(ip, addr); |
269 | ||
270 | return ftrace_modify_code(rec->ip, old, new); | |
271 | } | |
272 | ||
3a36cb11 | 273 | int __init ftrace_dyn_arch_init(void) |
fad57feb | 274 | { |
fad57feb MF |
275 | return 0; |
276 | } | |
327933f5 MF |
277 | #endif /* CONFIG_DYNAMIC_FTRACE */ |
278 | ||
279 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER | |
280 | #ifdef CONFIG_DYNAMIC_FTRACE | |
281 | extern void ftrace_graph_call(void); | |
282 | ||
283 | static int ftrace_mod(unsigned long ip, unsigned long old_addr, | |
284 | unsigned long new_addr) | |
285 | { | |
286 | unsigned char code[MCOUNT_INSN_SIZE]; | |
287 | ||
288 | if (probe_kernel_read(code, (void *)ip, MCOUNT_INSN_SIZE)) | |
289 | return -EFAULT; | |
290 | ||
291 | if (old_addr != __raw_readl((unsigned long *)code)) | |
292 | return -EINVAL; | |
293 | ||
294 | __raw_writel(new_addr, ip); | |
295 | return 0; | |
296 | } | |
297 | ||
298 | int ftrace_enable_ftrace_graph_caller(void) | |
299 | { | |
300 | unsigned long ip, old_addr, new_addr; | |
301 | ||
302 | ip = (unsigned long)(&ftrace_graph_call) + GRAPH_INSN_OFFSET; | |
303 | old_addr = (unsigned long)(&skip_trace); | |
304 | new_addr = (unsigned long)(&ftrace_graph_caller); | |
305 | ||
306 | return ftrace_mod(ip, old_addr, new_addr); | |
307 | } | |
308 | ||
309 | int ftrace_disable_ftrace_graph_caller(void) | |
310 | { | |
311 | unsigned long ip, old_addr, new_addr; | |
312 | ||
313 | ip = (unsigned long)(&ftrace_graph_call) + GRAPH_INSN_OFFSET; | |
314 | old_addr = (unsigned long)(&ftrace_graph_caller); | |
315 | new_addr = (unsigned long)(&skip_trace); | |
316 | ||
317 | return ftrace_mod(ip, old_addr, new_addr); | |
318 | } | |
319 | #endif /* CONFIG_DYNAMIC_FTRACE */ | |
320 | ||
321 | /* | |
322 | * Hook the return address and push it in the stack of return addrs | |
323 | * in the current thread info. | |
324 | * | |
325 | * This is the main routine for the function graph tracer. The function | |
326 | * graph tracer essentially works like this: | |
327 | * | |
328 | * parent is the stack address containing self_addr's return address. | |
329 | * We pull the real return address out of parent and store it in | |
330 | * current's ret_stack. Then, we replace the return address on the stack | |
331 | * with the address of return_to_handler. self_addr is the function that | |
332 | * called mcount. | |
333 | * | |
334 | * When self_addr returns, it will jump to return_to_handler which calls | |
335 | * ftrace_return_to_handler. ftrace_return_to_handler will pull the real | |
336 | * return address off of current's ret_stack and jump to it. | |
337 | */ | |
338 | void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr) | |
339 | { | |
340 | unsigned long old; | |
341 | int faulted, err; | |
342 | struct ftrace_graph_ent trace; | |
343 | unsigned long return_hooker = (unsigned long)&return_to_handler; | |
344 | ||
7fa322db SRRH |
345 | if (unlikely(ftrace_graph_is_dead())) |
346 | return; | |
347 | ||
327933f5 MF |
348 | if (unlikely(atomic_read(¤t->tracing_graph_pause))) |
349 | return; | |
350 | ||
351 | /* | |
352 | * Protect against fault, even if it shouldn't | |
353 | * happen. This tool is too much intrusive to | |
354 | * ignore such a protection. | |
355 | */ | |
356 | __asm__ __volatile__( | |
357 | "1: \n\t" | |
358 | "mov.l @%2, %0 \n\t" | |
359 | "2: \n\t" | |
360 | "mov.l %3, @%2 \n\t" | |
361 | "mov #0, %1 \n\t" | |
362 | "3: \n\t" | |
363 | ".section .fixup, \"ax\" \n\t" | |
364 | "4: \n\t" | |
365 | "mov.l 5f, %0 \n\t" | |
366 | "jmp @%0 \n\t" | |
367 | " mov #1, %1 \n\t" | |
368 | ".balign 4 \n\t" | |
369 | "5: .long 3b \n\t" | |
370 | ".previous \n\t" | |
371 | ".section __ex_table,\"a\" \n\t" | |
372 | ".long 1b, 4b \n\t" | |
373 | ".long 2b, 4b \n\t" | |
374 | ".previous \n\t" | |
375 | : "=&r" (old), "=r" (faulted) | |
376 | : "r" (parent), "r" (return_hooker) | |
377 | ); | |
378 | ||
379 | if (unlikely(faulted)) { | |
380 | ftrace_graph_stop(); | |
381 | WARN_ON(1); | |
382 | return; | |
383 | } | |
384 | ||
9a7c348b | 385 | err = ftrace_push_return_trace(old, self_addr, &trace.depth, 0, NULL); |
327933f5 MF |
386 | if (err == -EBUSY) { |
387 | __raw_writel(old, parent); | |
388 | return; | |
389 | } | |
390 | ||
391 | trace.func = self_addr; | |
392 | ||
393 | /* Only trace if the calling function expects to */ | |
394 | if (!ftrace_graph_entry(&trace)) { | |
395 | current->curr_ret_stack--; | |
396 | __raw_writel(old, parent); | |
397 | } | |
398 | } | |
399 | #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ |