Commit | Line | Data |
---|---|---|
e5a81b62 SR |
1 | /* |
2 | * Copyright (C) 2008 Steven Rostedt <srostedt@redhat.com> | |
3 | * | |
4 | */ | |
5 | #include <linux/stacktrace.h> | |
6 | #include <linux/kallsyms.h> | |
7 | #include <linux/seq_file.h> | |
8 | #include <linux/spinlock.h> | |
9 | #include <linux/uaccess.h> | |
10 | #include <linux/debugfs.h> | |
11 | #include <linux/ftrace.h> | |
12 | #include <linux/module.h> | |
f38f1d2a | 13 | #include <linux/sysctl.h> |
e5a81b62 SR |
14 | #include <linux/init.h> |
15 | #include <linux/fs.h> | |
38628078 | 16 | #include <linux/magic.h> |
762e1207 SR |
17 | |
18 | #include <asm/setup.h> | |
19 | ||
e5a81b62 SR |
20 | #include "trace.h" |
21 | ||
22 | #define STACK_TRACE_ENTRIES 500 | |
23 | ||
d4ecbfc4 | 24 | #ifdef CC_USING_FENTRY |
4df29712 | 25 | # define fentry 1 |
d4ecbfc4 | 26 | #else |
4df29712 | 27 | # define fentry 0 |
d4ecbfc4 SRRH |
28 | #endif |
29 | ||
1b6cced6 SR |
30 | static unsigned long stack_dump_trace[STACK_TRACE_ENTRIES+1] = |
31 | { [0 ... (STACK_TRACE_ENTRIES)] = ULONG_MAX }; | |
32 | static unsigned stack_dump_index[STACK_TRACE_ENTRIES]; | |
33 | ||
4df29712 SRRH |
34 | /* |
35 | * Reserve one entry for the passed in ip. This will allow | |
36 | * us to remove most or all of the stack size overhead | |
37 | * added by the stack tracer itself. | |
38 | */ | |
e5a81b62 | 39 | static struct stack_trace max_stack_trace = { |
4df29712 SRRH |
40 | .max_entries = STACK_TRACE_ENTRIES - 1, |
41 | .entries = &stack_dump_trace[1], | |
e5a81b62 SR |
42 | }; |
43 | ||
44 | static unsigned long max_stack_size; | |
445c8951 | 45 | static arch_spinlock_t max_stack_lock = |
edc35bd7 | 46 | (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED; |
e5a81b62 | 47 | |
e5a81b62 | 48 | static DEFINE_PER_CPU(int, trace_active); |
f38f1d2a SR |
49 | static DEFINE_MUTEX(stack_sysctl_mutex); |
50 | ||
51 | int stack_tracer_enabled; | |
52 | static int last_stack_tracer_enabled; | |
e5a81b62 | 53 | |
e3172181 MK |
54 | static inline void print_max_stack(void) |
55 | { | |
56 | long i; | |
57 | int size; | |
58 | ||
59 | pr_emerg(" Depth Size Location (%d entries)\n" | |
60 | " ----- ---- --------\n", | |
61 | max_stack_trace.nr_entries - 1); | |
62 | ||
63 | for (i = 0; i < max_stack_trace.nr_entries; i++) { | |
64 | if (stack_dump_trace[i] == ULONG_MAX) | |
65 | break; | |
66 | if (i+1 == max_stack_trace.nr_entries || | |
67 | stack_dump_trace[i+1] == ULONG_MAX) | |
68 | size = stack_dump_index[i]; | |
69 | else | |
70 | size = stack_dump_index[i] - stack_dump_index[i+1]; | |
71 | ||
72 | pr_emerg("%3ld) %8d %5d %pS\n", i, stack_dump_index[i], | |
73 | size, (void *)stack_dump_trace[i]); | |
74 | } | |
75 | } | |
76 | ||
87889501 | 77 | static inline void |
d4ecbfc4 | 78 | check_stack(unsigned long ip, unsigned long *stack) |
e5a81b62 | 79 | { |
e3172181 | 80 | unsigned long this_size, flags; unsigned long *p, *top, *start; |
4df29712 SRRH |
81 | static int tracer_frame; |
82 | int frame_size = ACCESS_ONCE(tracer_frame); | |
1b6cced6 | 83 | int i; |
e5a81b62 | 84 | |
87889501 | 85 | this_size = ((unsigned long)stack) & (THREAD_SIZE-1); |
e5a81b62 | 86 | this_size = THREAD_SIZE - this_size; |
4df29712 SRRH |
87 | /* Remove the frame of the tracer */ |
88 | this_size -= frame_size; | |
e5a81b62 SR |
89 | |
90 | if (this_size <= max_stack_size) | |
91 | return; | |
92 | ||
81520a1b | 93 | /* we do not handle interrupt stacks yet */ |
87889501 | 94 | if (!object_is_on_stack(stack)) |
81520a1b SR |
95 | return; |
96 | ||
a5e25883 | 97 | local_irq_save(flags); |
0199c4e6 | 98 | arch_spin_lock(&max_stack_lock); |
e5a81b62 | 99 | |
4df29712 SRRH |
100 | /* In case another CPU set the tracer_frame on us */ |
101 | if (unlikely(!frame_size)) | |
102 | this_size -= tracer_frame; | |
103 | ||
e5a81b62 SR |
104 | /* a race could have already updated it */ |
105 | if (this_size <= max_stack_size) | |
106 | goto out; | |
107 | ||
108 | max_stack_size = this_size; | |
109 | ||
7eea4fce JW |
110 | max_stack_trace.nr_entries = 0; |
111 | ||
112 | if (using_ftrace_ops_list_func()) | |
113 | max_stack_trace.skip = 4; | |
114 | else | |
115 | max_stack_trace.skip = 3; | |
e5a81b62 SR |
116 | |
117 | save_stack_trace(&max_stack_trace); | |
118 | ||
d4ecbfc4 | 119 | /* |
4df29712 SRRH |
120 | * Add the passed in ip from the function tracer. |
121 | * Searching for this on the stack will skip over | |
122 | * most of the overhead from the stack tracer itself. | |
d4ecbfc4 | 123 | */ |
4df29712 SRRH |
124 | stack_dump_trace[0] = ip; |
125 | max_stack_trace.nr_entries++; | |
d4ecbfc4 | 126 | |
1b6cced6 SR |
127 | /* |
128 | * Now find where in the stack these are. | |
129 | */ | |
130 | i = 0; | |
87889501 | 131 | start = stack; |
1b6cced6 SR |
132 | top = (unsigned long *) |
133 | (((unsigned long)start & ~(THREAD_SIZE-1)) + THREAD_SIZE); | |
134 | ||
135 | /* | |
136 | * Loop through all the entries. One of the entries may | |
137 | * for some reason be missed on the stack, so we may | |
138 | * have to account for them. If they are all there, this | |
139 | * loop will only happen once. This code only takes place | |
140 | * on a new max, so it is far from a fast path. | |
141 | */ | |
142 | while (i < max_stack_trace.nr_entries) { | |
0a37119d | 143 | int found = 0; |
1b6cced6 SR |
144 | |
145 | stack_dump_index[i] = this_size; | |
146 | p = start; | |
147 | ||
148 | for (; p < top && i < max_stack_trace.nr_entries; p++) { | |
149 | if (*p == stack_dump_trace[i]) { | |
150 | this_size = stack_dump_index[i++] = | |
151 | (top - p) * sizeof(unsigned long); | |
0a37119d | 152 | found = 1; |
1b6cced6 SR |
153 | /* Start the search from here */ |
154 | start = p + 1; | |
4df29712 SRRH |
155 | /* |
156 | * We do not want to show the overhead | |
157 | * of the stack tracer stack in the | |
158 | * max stack. If we haven't figured | |
159 | * out what that is, then figure it out | |
160 | * now. | |
161 | */ | |
162 | if (unlikely(!tracer_frame) && i == 1) { | |
163 | tracer_frame = (p - stack) * | |
164 | sizeof(unsigned long); | |
165 | max_stack_size -= tracer_frame; | |
166 | } | |
1b6cced6 SR |
167 | } |
168 | } | |
169 | ||
0a37119d SR |
170 | if (!found) |
171 | i++; | |
1b6cced6 SR |
172 | } |
173 | ||
e3172181 MK |
174 | if ((current != &init_task && |
175 | *(end_of_stack(current)) != STACK_END_MAGIC)) { | |
176 | print_max_stack(); | |
177 | BUG(); | |
178 | } | |
179 | ||
e5a81b62 | 180 | out: |
0199c4e6 | 181 | arch_spin_unlock(&max_stack_lock); |
a5e25883 | 182 | local_irq_restore(flags); |
e5a81b62 SR |
183 | } |
184 | ||
185 | static void | |
a1e2e31d SR |
186 | stack_trace_call(unsigned long ip, unsigned long parent_ip, |
187 | struct ftrace_ops *op, struct pt_regs *pt_regs) | |
e5a81b62 | 188 | { |
87889501 | 189 | unsigned long stack; |
5168ae50 | 190 | int cpu; |
e5a81b62 | 191 | |
5168ae50 | 192 | preempt_disable_notrace(); |
e5a81b62 SR |
193 | |
194 | cpu = raw_smp_processor_id(); | |
195 | /* no atomic needed, we only modify this variable by this cpu */ | |
196 | if (per_cpu(trace_active, cpu)++ != 0) | |
197 | goto out; | |
198 | ||
4df29712 SRRH |
199 | /* |
200 | * When fentry is used, the traced function does not get | |
201 | * its stack frame set up, and we lose the parent. | |
202 | * The ip is pretty useless because the function tracer | |
203 | * was called before that function set up its stack frame. | |
204 | * In this case, we use the parent ip. | |
205 | * | |
206 | * By adding the return address of either the parent ip | |
207 | * or the current ip we can disregard most of the stack usage | |
208 | * caused by the stack tracer itself. | |
209 | * | |
210 | * The function tracer always reports the address of where the | |
211 | * mcount call was, but the stack will hold the return address. | |
212 | */ | |
213 | if (fentry) | |
214 | ip = parent_ip; | |
215 | else | |
216 | ip += MCOUNT_INSN_SIZE; | |
217 | ||
218 | check_stack(ip, &stack); | |
e5a81b62 SR |
219 | |
220 | out: | |
221 | per_cpu(trace_active, cpu)--; | |
222 | /* prevent recursion in schedule */ | |
5168ae50 | 223 | preempt_enable_notrace(); |
e5a81b62 SR |
224 | } |
225 | ||
226 | static struct ftrace_ops trace_ops __read_mostly = | |
227 | { | |
228 | .func = stack_trace_call, | |
4740974a | 229 | .flags = FTRACE_OPS_FL_RECURSION_SAFE, |
e5a81b62 SR |
230 | }; |
231 | ||
232 | static ssize_t | |
233 | stack_max_size_read(struct file *filp, char __user *ubuf, | |
234 | size_t count, loff_t *ppos) | |
235 | { | |
236 | unsigned long *ptr = filp->private_data; | |
237 | char buf[64]; | |
238 | int r; | |
239 | ||
240 | r = snprintf(buf, sizeof(buf), "%ld\n", *ptr); | |
241 | if (r > sizeof(buf)) | |
242 | r = sizeof(buf); | |
243 | return simple_read_from_buffer(ubuf, count, ppos, buf, r); | |
244 | } | |
245 | ||
246 | static ssize_t | |
247 | stack_max_size_write(struct file *filp, const char __user *ubuf, | |
248 | size_t count, loff_t *ppos) | |
249 | { | |
250 | long *ptr = filp->private_data; | |
251 | unsigned long val, flags; | |
e5a81b62 | 252 | int ret; |
4f48f8b7 | 253 | int cpu; |
e5a81b62 | 254 | |
22fe9b54 PH |
255 | ret = kstrtoul_from_user(ubuf, count, 10, &val); |
256 | if (ret) | |
e5a81b62 SR |
257 | return ret; |
258 | ||
a5e25883 | 259 | local_irq_save(flags); |
4f48f8b7 LJ |
260 | |
261 | /* | |
262 | * In case we trace inside arch_spin_lock() or after (NMI), | |
263 | * we will cause circular lock, so we also need to increase | |
264 | * the percpu trace_active here. | |
265 | */ | |
266 | cpu = smp_processor_id(); | |
267 | per_cpu(trace_active, cpu)++; | |
268 | ||
0199c4e6 | 269 | arch_spin_lock(&max_stack_lock); |
e5a81b62 | 270 | *ptr = val; |
0199c4e6 | 271 | arch_spin_unlock(&max_stack_lock); |
4f48f8b7 LJ |
272 | |
273 | per_cpu(trace_active, cpu)--; | |
a5e25883 | 274 | local_irq_restore(flags); |
e5a81b62 SR |
275 | |
276 | return count; | |
277 | } | |
278 | ||
f38f1d2a | 279 | static const struct file_operations stack_max_size_fops = { |
e5a81b62 SR |
280 | .open = tracing_open_generic, |
281 | .read = stack_max_size_read, | |
282 | .write = stack_max_size_write, | |
6038f373 | 283 | .llseek = default_llseek, |
e5a81b62 SR |
284 | }; |
285 | ||
286 | static void * | |
2fc5f0cf | 287 | __next(struct seq_file *m, loff_t *pos) |
e5a81b62 | 288 | { |
2fc5f0cf | 289 | long n = *pos - 1; |
e5a81b62 | 290 | |
2fc5f0cf | 291 | if (n >= max_stack_trace.nr_entries || stack_dump_trace[n] == ULONG_MAX) |
e5a81b62 SR |
292 | return NULL; |
293 | ||
2fc5f0cf | 294 | m->private = (void *)n; |
1b6cced6 | 295 | return &m->private; |
e5a81b62 SR |
296 | } |
297 | ||
2fc5f0cf LZ |
298 | static void * |
299 | t_next(struct seq_file *m, void *v, loff_t *pos) | |
e5a81b62 | 300 | { |
2fc5f0cf LZ |
301 | (*pos)++; |
302 | return __next(m, pos); | |
303 | } | |
e5a81b62 | 304 | |
2fc5f0cf LZ |
305 | static void *t_start(struct seq_file *m, loff_t *pos) |
306 | { | |
4f48f8b7 LJ |
307 | int cpu; |
308 | ||
e5a81b62 | 309 | local_irq_disable(); |
4f48f8b7 LJ |
310 | |
311 | cpu = smp_processor_id(); | |
312 | per_cpu(trace_active, cpu)++; | |
313 | ||
0199c4e6 | 314 | arch_spin_lock(&max_stack_lock); |
e5a81b62 | 315 | |
522a110b LW |
316 | if (*pos == 0) |
317 | return SEQ_START_TOKEN; | |
318 | ||
2fc5f0cf | 319 | return __next(m, pos); |
e5a81b62 SR |
320 | } |
321 | ||
322 | static void t_stop(struct seq_file *m, void *p) | |
323 | { | |
4f48f8b7 LJ |
324 | int cpu; |
325 | ||
0199c4e6 | 326 | arch_spin_unlock(&max_stack_lock); |
4f48f8b7 LJ |
327 | |
328 | cpu = smp_processor_id(); | |
329 | per_cpu(trace_active, cpu)--; | |
330 | ||
e5a81b62 SR |
331 | local_irq_enable(); |
332 | } | |
333 | ||
1b6cced6 | 334 | static int trace_lookup_stack(struct seq_file *m, long i) |
e5a81b62 | 335 | { |
1b6cced6 | 336 | unsigned long addr = stack_dump_trace[i]; |
e5a81b62 | 337 | |
151772db | 338 | return seq_printf(m, "%pS\n", (void *)addr); |
e5a81b62 SR |
339 | } |
340 | ||
e447e1df SR |
341 | static void print_disabled(struct seq_file *m) |
342 | { | |
343 | seq_puts(m, "#\n" | |
344 | "# Stack tracer disabled\n" | |
345 | "#\n" | |
346 | "# To enable the stack tracer, either add 'stacktrace' to the\n" | |
347 | "# kernel command line\n" | |
348 | "# or 'echo 1 > /proc/sys/kernel/stack_tracer_enabled'\n" | |
349 | "#\n"); | |
350 | } | |
351 | ||
e5a81b62 SR |
352 | static int t_show(struct seq_file *m, void *v) |
353 | { | |
522a110b | 354 | long i; |
1b6cced6 SR |
355 | int size; |
356 | ||
522a110b | 357 | if (v == SEQ_START_TOKEN) { |
eb1871f3 | 358 | seq_printf(m, " Depth Size Location" |
1b6cced6 | 359 | " (%d entries)\n" |
eb1871f3 | 360 | " ----- ---- --------\n", |
083a63b4 | 361 | max_stack_trace.nr_entries - 1); |
e447e1df SR |
362 | |
363 | if (!stack_tracer_enabled && !max_stack_size) | |
364 | print_disabled(m); | |
365 | ||
1b6cced6 SR |
366 | return 0; |
367 | } | |
e5a81b62 | 368 | |
522a110b LW |
369 | i = *(long *)v; |
370 | ||
1b6cced6 SR |
371 | if (i >= max_stack_trace.nr_entries || |
372 | stack_dump_trace[i] == ULONG_MAX) | |
e5a81b62 SR |
373 | return 0; |
374 | ||
1b6cced6 SR |
375 | if (i+1 == max_stack_trace.nr_entries || |
376 | stack_dump_trace[i+1] == ULONG_MAX) | |
377 | size = stack_dump_index[i]; | |
378 | else | |
379 | size = stack_dump_index[i] - stack_dump_index[i+1]; | |
380 | ||
381 | seq_printf(m, "%3ld) %8d %5d ", i, stack_dump_index[i], size); | |
382 | ||
383 | trace_lookup_stack(m, i); | |
e5a81b62 SR |
384 | |
385 | return 0; | |
386 | } | |
387 | ||
f38f1d2a | 388 | static const struct seq_operations stack_trace_seq_ops = { |
e5a81b62 SR |
389 | .start = t_start, |
390 | .next = t_next, | |
391 | .stop = t_stop, | |
392 | .show = t_show, | |
393 | }; | |
394 | ||
395 | static int stack_trace_open(struct inode *inode, struct file *file) | |
396 | { | |
d8cc1ab7 | 397 | return seq_open(file, &stack_trace_seq_ops); |
e5a81b62 SR |
398 | } |
399 | ||
f38f1d2a | 400 | static const struct file_operations stack_trace_fops = { |
e5a81b62 SR |
401 | .open = stack_trace_open, |
402 | .read = seq_read, | |
403 | .llseek = seq_lseek, | |
d8cc1ab7 | 404 | .release = seq_release, |
e5a81b62 SR |
405 | }; |
406 | ||
d2d45c7a SR |
407 | static int |
408 | stack_trace_filter_open(struct inode *inode, struct file *file) | |
409 | { | |
410 | return ftrace_regex_open(&trace_ops, FTRACE_ITER_FILTER, | |
411 | inode, file); | |
412 | } | |
413 | ||
414 | static const struct file_operations stack_trace_filter_fops = { | |
415 | .open = stack_trace_filter_open, | |
416 | .read = seq_read, | |
417 | .write = ftrace_filter_write, | |
098c879e | 418 | .llseek = tracing_lseek, |
d2d45c7a SR |
419 | .release = ftrace_regex_release, |
420 | }; | |
421 | ||
f38f1d2a SR |
422 | int |
423 | stack_trace_sysctl(struct ctl_table *table, int write, | |
8d65af78 | 424 | void __user *buffer, size_t *lenp, |
f38f1d2a SR |
425 | loff_t *ppos) |
426 | { | |
427 | int ret; | |
428 | ||
429 | mutex_lock(&stack_sysctl_mutex); | |
430 | ||
8d65af78 | 431 | ret = proc_dointvec(table, write, buffer, lenp, ppos); |
f38f1d2a SR |
432 | |
433 | if (ret || !write || | |
a32c7765 | 434 | (last_stack_tracer_enabled == !!stack_tracer_enabled)) |
f38f1d2a SR |
435 | goto out; |
436 | ||
a32c7765 | 437 | last_stack_tracer_enabled = !!stack_tracer_enabled; |
f38f1d2a SR |
438 | |
439 | if (stack_tracer_enabled) | |
440 | register_ftrace_function(&trace_ops); | |
441 | else | |
442 | unregister_ftrace_function(&trace_ops); | |
443 | ||
444 | out: | |
445 | mutex_unlock(&stack_sysctl_mutex); | |
446 | return ret; | |
447 | } | |
448 | ||
762e1207 SR |
449 | static char stack_trace_filter_buf[COMMAND_LINE_SIZE+1] __initdata; |
450 | ||
f38f1d2a SR |
451 | static __init int enable_stacktrace(char *str) |
452 | { | |
762e1207 SR |
453 | if (strncmp(str, "_filter=", 8) == 0) |
454 | strncpy(stack_trace_filter_buf, str+8, COMMAND_LINE_SIZE); | |
455 | ||
e05a43b7 SR |
456 | stack_tracer_enabled = 1; |
457 | last_stack_tracer_enabled = 1; | |
f38f1d2a SR |
458 | return 1; |
459 | } | |
460 | __setup("stacktrace", enable_stacktrace); | |
461 | ||
e5a81b62 SR |
462 | static __init int stack_trace_init(void) |
463 | { | |
464 | struct dentry *d_tracer; | |
e5a81b62 SR |
465 | |
466 | d_tracer = tracing_init_dentry(); | |
ed6f1c99 NK |
467 | if (!d_tracer) |
468 | return 0; | |
e5a81b62 | 469 | |
5452af66 FW |
470 | trace_create_file("stack_max_size", 0644, d_tracer, |
471 | &max_stack_size, &stack_max_size_fops); | |
e5a81b62 | 472 | |
5452af66 FW |
473 | trace_create_file("stack_trace", 0444, d_tracer, |
474 | NULL, &stack_trace_fops); | |
e5a81b62 | 475 | |
d2d45c7a SR |
476 | trace_create_file("stack_trace_filter", 0444, d_tracer, |
477 | NULL, &stack_trace_filter_fops); | |
478 | ||
762e1207 SR |
479 | if (stack_trace_filter_buf[0]) |
480 | ftrace_set_early_filter(&trace_ops, stack_trace_filter_buf, 1); | |
481 | ||
e05a43b7 | 482 | if (stack_tracer_enabled) |
f38f1d2a | 483 | register_ftrace_function(&trace_ops); |
e5a81b62 SR |
484 | |
485 | return 0; | |
486 | } | |
487 | ||
488 | device_initcall(stack_trace_init); |