3 * Function graph tracer.
4 * Copyright (c) 2008-2009 Frederic Weisbecker <fweisbec@gmail.com>
5 * Mostly borrowed from function tracer which
6 * is Copyright (c) Steven Rostedt <srostedt@redhat.com>
9 #include <linux/uaccess.h>
10 #include <linux/ftrace.h>
11 #include <linux/interrupt.h>
12 #include <linux/slab.h>
16 #include "trace_output.h"
18 static bool kill_ftrace_graph
;
21 * ftrace_graph_is_dead - returns true if ftrace_graph_stop() was called
23 * ftrace_graph_stop() is called when a severe error is detected in
24 * the function graph tracing. This function is called by the critical
25 * paths of function graph to keep those paths from doing any more harm.
27 bool ftrace_graph_is_dead(void)
29 return kill_ftrace_graph
;
33 * ftrace_graph_stop - set to permanently disable function graph tracincg
35 * In case of an error int function graph tracing, this is called
36 * to try to keep function graph tracing from causing any more harm.
37 * Usually this is pretty severe and this is called to try to at least
38 * get a warning out to the user.
40 void ftrace_graph_stop(void)
42 kill_ftrace_graph
= true;
45 /* When set, irq functions will be ignored */
46 static int ftrace_graph_skip_irqs
;
48 struct fgraph_cpu_data
{
53 unsigned long enter_funcs
[FTRACE_RETFUNC_DEPTH
];
57 struct fgraph_cpu_data __percpu
*cpu_data
;
59 /* Place to preserve last processed entry. */
60 struct ftrace_graph_ent_entry ent
;
61 struct ftrace_graph_ret_entry ret
;
66 #define TRACE_GRAPH_INDENT 2
68 static unsigned int max_depth
;
70 static struct tracer_opt trace_opts
[] = {
71 /* Display overruns? (for self-debug purpose) */
72 { TRACER_OPT(funcgraph
-overrun
, TRACE_GRAPH_PRINT_OVERRUN
) },
74 { TRACER_OPT(funcgraph
-cpu
, TRACE_GRAPH_PRINT_CPU
) },
75 /* Display Overhead ? */
76 { TRACER_OPT(funcgraph
-overhead
, TRACE_GRAPH_PRINT_OVERHEAD
) },
77 /* Display proc name/pid */
78 { TRACER_OPT(funcgraph
-proc
, TRACE_GRAPH_PRINT_PROC
) },
79 /* Display duration of execution */
80 { TRACER_OPT(funcgraph
-duration
, TRACE_GRAPH_PRINT_DURATION
) },
81 /* Display absolute time of an entry */
82 { TRACER_OPT(funcgraph
-abstime
, TRACE_GRAPH_PRINT_ABS_TIME
) },
83 /* Display interrupts */
84 { TRACER_OPT(funcgraph
-irqs
, TRACE_GRAPH_PRINT_IRQS
) },
85 /* Display function name after trailing } */
86 { TRACER_OPT(funcgraph
-tail
, TRACE_GRAPH_PRINT_TAIL
) },
87 /* Include sleep time (scheduled out) between entry and return */
88 { TRACER_OPT(sleep
-time
, TRACE_GRAPH_SLEEP_TIME
) },
89 /* Include time within nested functions */
90 { TRACER_OPT(graph
-time
, TRACE_GRAPH_GRAPH_TIME
) },
94 static struct tracer_flags tracer_flags
= {
95 /* Don't display overruns, proc, or tail by default */
96 .val
= TRACE_GRAPH_PRINT_CPU
| TRACE_GRAPH_PRINT_OVERHEAD
|
97 TRACE_GRAPH_PRINT_DURATION
| TRACE_GRAPH_PRINT_IRQS
|
98 TRACE_GRAPH_SLEEP_TIME
| TRACE_GRAPH_GRAPH_TIME
,
102 static struct trace_array
*graph_array
;
105 * DURATION column is being also used to display IRQ signs,
106 * following values are used by print_graph_irq and others
107 * to fill in space into DURATION column.
110 FLAGS_FILL_FULL
= 1 << TRACE_GRAPH_PRINT_FILL_SHIFT
,
111 FLAGS_FILL_START
= 2 << TRACE_GRAPH_PRINT_FILL_SHIFT
,
112 FLAGS_FILL_END
= 3 << TRACE_GRAPH_PRINT_FILL_SHIFT
,
116 print_graph_duration(struct trace_array
*tr
, unsigned long long duration
,
117 struct trace_seq
*s
, u32 flags
);
119 /* Add a function return address to the trace stack on thread info.*/
121 ftrace_push_return_trace(unsigned long ret
, unsigned long func
, int *depth
,
122 unsigned long frame_pointer
, unsigned long *retp
)
124 unsigned long long calltime
;
127 if (unlikely(ftrace_graph_is_dead()))
130 if (!current
->ret_stack
)
134 * We must make sure the ret_stack is tested before we read
139 /* The return trace stack is full */
140 if (current
->curr_ret_stack
== FTRACE_RETFUNC_DEPTH
- 1) {
141 atomic_inc(¤t
->trace_overrun
);
146 * The curr_ret_stack is an index to ftrace return stack of
147 * current task. Its value should be in [0, FTRACE_RETFUNC_
148 * DEPTH) when the function graph tracer is used. To support
149 * filtering out specific functions, it makes the index
150 * negative by subtracting huge value (FTRACE_NOTRACE_DEPTH)
151 * so when it sees a negative index the ftrace will ignore
152 * the record. And the index gets recovered when returning
153 * from the filtered function by adding the FTRACE_NOTRACE_
154 * DEPTH and then it'll continue to record functions normally.
156 * The curr_ret_stack is initialized to -1 and get increased
157 * in this function. So it can be less than -1 only if it was
158 * filtered out via ftrace_graph_notrace_addr() which can be
159 * set from set_graph_notrace file in tracefs by user.
161 if (current
->curr_ret_stack
< -1)
164 calltime
= trace_clock_local();
166 index
= ++current
->curr_ret_stack
;
167 if (ftrace_graph_notrace_addr(func
))
168 current
->curr_ret_stack
-= FTRACE_NOTRACE_DEPTH
;
170 current
->ret_stack
[index
].ret
= ret
;
171 current
->ret_stack
[index
].func
= func
;
172 current
->ret_stack
[index
].calltime
= calltime
;
173 current
->ret_stack
[index
].subtime
= 0;
174 #ifdef HAVE_FUNCTION_GRAPH_FP_TEST
175 current
->ret_stack
[index
].fp
= frame_pointer
;
177 #ifdef HAVE_FUNCTION_GRAPH_RET_ADDR_PTR
178 current
->ret_stack
[index
].retp
= retp
;
180 *depth
= current
->curr_ret_stack
;
185 /* Retrieve a function return address to the trace stack on thread info.*/
187 ftrace_pop_return_trace(struct ftrace_graph_ret
*trace
, unsigned long *ret
,
188 unsigned long frame_pointer
)
192 index
= current
->curr_ret_stack
;
195 * A negative index here means that it's just returned from a
196 * notrace'd function. Recover index to get an original
197 * return address. See ftrace_push_return_trace().
199 * TODO: Need to check whether the stack gets corrupted.
202 index
+= FTRACE_NOTRACE_DEPTH
;
204 if (unlikely(index
< 0 || index
>= FTRACE_RETFUNC_DEPTH
)) {
207 /* Might as well panic, otherwise we have no where to go */
208 *ret
= (unsigned long)panic
;
212 #ifdef HAVE_FUNCTION_GRAPH_FP_TEST
214 * The arch may choose to record the frame pointer used
215 * and check it here to make sure that it is what we expect it
216 * to be. If gcc does not set the place holder of the return
217 * address in the frame pointer, and does a copy instead, then
218 * the function graph trace will fail. This test detects this
221 * Currently, x86_32 with optimize for size (-Os) makes the latest
224 * Note, -mfentry does not use frame pointers, and this test
225 * is not needed if CC_USING_FENTRY is set.
227 if (unlikely(current
->ret_stack
[index
].fp
!= frame_pointer
)) {
229 WARN(1, "Bad frame pointer: expected %lx, received %lx\n"
230 " from func %ps return to %lx\n",
231 current
->ret_stack
[index
].fp
,
233 (void *)current
->ret_stack
[index
].func
,
234 current
->ret_stack
[index
].ret
);
235 *ret
= (unsigned long)panic
;
240 *ret
= current
->ret_stack
[index
].ret
;
241 trace
->func
= current
->ret_stack
[index
].func
;
242 trace
->calltime
= current
->ret_stack
[index
].calltime
;
243 trace
->overrun
= atomic_read(¤t
->trace_overrun
);
244 trace
->depth
= index
;
248 * Send the trace to the ring-buffer.
249 * @return the original return address.
251 unsigned long ftrace_return_to_handler(unsigned long frame_pointer
)
253 struct ftrace_graph_ret trace
;
256 ftrace_pop_return_trace(&trace
, &ret
, frame_pointer
);
257 trace
.rettime
= trace_clock_local();
259 current
->curr_ret_stack
--;
261 * The curr_ret_stack can be less than -1 only if it was
262 * filtered out and it's about to return from the function.
263 * Recover the index and continue to trace normal functions.
265 if (current
->curr_ret_stack
< -1) {
266 current
->curr_ret_stack
+= FTRACE_NOTRACE_DEPTH
;
271 * The trace should run after decrementing the ret counter
272 * in case an interrupt were to come in. We don't want to
273 * lose the interrupt if max_depth is set.
275 ftrace_graph_return(&trace
);
277 if (unlikely(!ret
)) {
280 /* Might as well panic. What else to do? */
281 ret
= (unsigned long)panic
;
288 * ftrace_graph_ret_addr - convert a potentially modified stack return address
289 * to its original value
291 * This function can be called by stack unwinding code to convert a found stack
292 * return address ('ret') to its original value, in case the function graph
293 * tracer has modified it to be 'return_to_handler'. If the address hasn't
294 * been modified, the unchanged value of 'ret' is returned.
296 * 'idx' is a state variable which should be initialized by the caller to zero
297 * before the first call.
299 * 'retp' is a pointer to the return address on the stack. It's ignored if
300 * the arch doesn't have HAVE_FUNCTION_GRAPH_RET_ADDR_PTR defined.
302 #ifdef HAVE_FUNCTION_GRAPH_RET_ADDR_PTR
303 unsigned long ftrace_graph_ret_addr(struct task_struct
*task
, int *idx
,
304 unsigned long ret
, unsigned long *retp
)
306 int index
= task
->curr_ret_stack
;
309 if (ret
!= (unsigned long)return_to_handler
)
313 index
+= FTRACE_NOTRACE_DEPTH
;
318 for (i
= 0; i
<= index
; i
++)
319 if (task
->ret_stack
[i
].retp
== retp
)
320 return task
->ret_stack
[i
].ret
;
324 #else /* !HAVE_FUNCTION_GRAPH_RET_ADDR_PTR */
325 unsigned long ftrace_graph_ret_addr(struct task_struct
*task
, int *idx
,
326 unsigned long ret
, unsigned long *retp
)
330 if (ret
!= (unsigned long)return_to_handler
)
333 task_idx
= task
->curr_ret_stack
;
335 if (!task
->ret_stack
|| task_idx
< *idx
)
341 return task
->ret_stack
[task_idx
].ret
;
343 #endif /* HAVE_FUNCTION_GRAPH_RET_ADDR_PTR */
345 int __trace_graph_entry(struct trace_array
*tr
,
346 struct ftrace_graph_ent
*trace
,
350 struct trace_event_call
*call
= &event_funcgraph_entry
;
351 struct ring_buffer_event
*event
;
352 struct ring_buffer
*buffer
= tr
->trace_buffer
.buffer
;
353 struct ftrace_graph_ent_entry
*entry
;
355 event
= trace_buffer_lock_reserve(buffer
, TRACE_GRAPH_ENT
,
356 sizeof(*entry
), flags
, pc
);
359 entry
= ring_buffer_event_data(event
);
360 entry
->graph_ent
= *trace
;
361 if (!call_filter_check_discard(call
, entry
, buffer
, event
))
362 __buffer_unlock_commit(buffer
, event
);
367 static inline int ftrace_graph_ignore_irqs(void)
369 if (!ftrace_graph_skip_irqs
|| trace_recursion_test(TRACE_IRQ_BIT
))
375 int trace_graph_entry(struct ftrace_graph_ent
*trace
)
377 struct trace_array
*tr
= graph_array
;
378 struct trace_array_cpu
*data
;
385 if (!ftrace_trace_task(tr
))
388 /* trace it when it is-nested-in or is a function enabled. */
389 if ((!(trace
->depth
|| ftrace_graph_addr(trace
->func
)) ||
390 ftrace_graph_ignore_irqs()) || (trace
->depth
< 0) ||
391 (max_depth
&& trace
->depth
>= max_depth
))
395 * Do not trace a function if it's filtered by set_graph_notrace.
396 * Make the index of ret stack negative to indicate that it should
397 * ignore further functions. But it needs its own ret stack entry
398 * to recover the original index in order to continue tracing after
399 * returning from the function.
401 if (ftrace_graph_notrace_addr(trace
->func
))
405 * Stop here if tracing_threshold is set. We only write function return
406 * events to the ring buffer.
411 local_irq_save(flags
);
412 cpu
= raw_smp_processor_id();
413 data
= per_cpu_ptr(tr
->trace_buffer
.data
, cpu
);
414 disabled
= atomic_inc_return(&data
->disabled
);
415 if (likely(disabled
== 1)) {
416 pc
= preempt_count();
417 ret
= __trace_graph_entry(tr
, trace
, flags
, pc
);
422 atomic_dec(&data
->disabled
);
423 local_irq_restore(flags
);
429 __trace_graph_function(struct trace_array
*tr
,
430 unsigned long ip
, unsigned long flags
, int pc
)
432 u64 time
= trace_clock_local();
433 struct ftrace_graph_ent ent
= {
437 struct ftrace_graph_ret ret
= {
444 __trace_graph_entry(tr
, &ent
, flags
, pc
);
445 __trace_graph_return(tr
, &ret
, flags
, pc
);
449 trace_graph_function(struct trace_array
*tr
,
450 unsigned long ip
, unsigned long parent_ip
,
451 unsigned long flags
, int pc
)
453 __trace_graph_function(tr
, ip
, flags
, pc
);
456 void __trace_graph_return(struct trace_array
*tr
,
457 struct ftrace_graph_ret
*trace
,
461 struct trace_event_call
*call
= &event_funcgraph_exit
;
462 struct ring_buffer_event
*event
;
463 struct ring_buffer
*buffer
= tr
->trace_buffer
.buffer
;
464 struct ftrace_graph_ret_entry
*entry
;
466 event
= trace_buffer_lock_reserve(buffer
, TRACE_GRAPH_RET
,
467 sizeof(*entry
), flags
, pc
);
470 entry
= ring_buffer_event_data(event
);
472 if (!call_filter_check_discard(call
, entry
, buffer
, event
))
473 __buffer_unlock_commit(buffer
, event
);
476 void trace_graph_return(struct ftrace_graph_ret
*trace
)
478 struct trace_array
*tr
= graph_array
;
479 struct trace_array_cpu
*data
;
485 local_irq_save(flags
);
486 cpu
= raw_smp_processor_id();
487 data
= per_cpu_ptr(tr
->trace_buffer
.data
, cpu
);
488 disabled
= atomic_inc_return(&data
->disabled
);
489 if (likely(disabled
== 1)) {
490 pc
= preempt_count();
491 __trace_graph_return(tr
, trace
, flags
, pc
);
493 atomic_dec(&data
->disabled
);
494 local_irq_restore(flags
);
497 void set_graph_array(struct trace_array
*tr
)
501 /* Make graph_array visible before we start tracing */
506 static void trace_graph_thresh_return(struct ftrace_graph_ret
*trace
)
508 if (tracing_thresh
&&
509 (trace
->rettime
- trace
->calltime
< tracing_thresh
))
512 trace_graph_return(trace
);
515 static int graph_trace_init(struct trace_array
*tr
)
521 ret
= register_ftrace_graph(&trace_graph_thresh_return
,
524 ret
= register_ftrace_graph(&trace_graph_return
,
528 tracing_start_cmdline_record();
533 static void graph_trace_reset(struct trace_array
*tr
)
535 tracing_stop_cmdline_record();
536 unregister_ftrace_graph();
539 static int graph_trace_update_thresh(struct trace_array
*tr
)
541 graph_trace_reset(tr
);
542 return graph_trace_init(tr
);
545 static int max_bytes_for_cpu
;
547 static void print_graph_cpu(struct trace_seq
*s
, int cpu
)
550 * Start with a space character - to make it stand out
551 * to the right a bit when trace output is pasted into
554 trace_seq_printf(s
, " %*d) ", max_bytes_for_cpu
, cpu
);
557 #define TRACE_GRAPH_PROCINFO_LENGTH 14
559 static void print_graph_proc(struct trace_seq
*s
, pid_t pid
)
561 char comm
[TASK_COMM_LEN
];
562 /* sign + log10(MAX_INT) + '\0' */
568 trace_find_cmdline(pid
, comm
);
570 sprintf(pid_str
, "%d", pid
);
572 /* 1 stands for the "-" character */
573 len
= strlen(comm
) + strlen(pid_str
) + 1;
575 if (len
< TRACE_GRAPH_PROCINFO_LENGTH
)
576 spaces
= TRACE_GRAPH_PROCINFO_LENGTH
- len
;
578 /* First spaces to align center */
579 for (i
= 0; i
< spaces
/ 2; i
++)
580 trace_seq_putc(s
, ' ');
582 trace_seq_printf(s
, "%s-%s", comm
, pid_str
);
584 /* Last spaces to align center */
585 for (i
= 0; i
< spaces
- (spaces
/ 2); i
++)
586 trace_seq_putc(s
, ' ');
590 static void print_graph_lat_fmt(struct trace_seq
*s
, struct trace_entry
*entry
)
592 trace_seq_putc(s
, ' ');
593 trace_print_lat_fmt(s
, entry
);
596 /* If the pid changed since the last trace, output this event */
598 verif_pid(struct trace_seq
*s
, pid_t pid
, int cpu
, struct fgraph_data
*data
)
606 last_pid
= &(per_cpu_ptr(data
->cpu_data
, cpu
)->last_pid
);
608 if (*last_pid
== pid
)
611 prev_pid
= *last_pid
;
617 * Context-switch trace line:
619 ------------------------------------------
620 | 1) migration/0--1 => sshd-1755
621 ------------------------------------------
624 trace_seq_puts(s
, " ------------------------------------------\n");
625 print_graph_cpu(s
, cpu
);
626 print_graph_proc(s
, prev_pid
);
627 trace_seq_puts(s
, " => ");
628 print_graph_proc(s
, pid
);
629 trace_seq_puts(s
, "\n ------------------------------------------\n\n");
632 static struct ftrace_graph_ret_entry
*
633 get_return_for_leaf(struct trace_iterator
*iter
,
634 struct ftrace_graph_ent_entry
*curr
)
636 struct fgraph_data
*data
= iter
->private;
637 struct ring_buffer_iter
*ring_iter
= NULL
;
638 struct ring_buffer_event
*event
;
639 struct ftrace_graph_ret_entry
*next
;
642 * If the previous output failed to write to the seq buffer,
643 * then we just reuse the data from before.
645 if (data
&& data
->failed
) {
650 ring_iter
= trace_buffer_iter(iter
, iter
->cpu
);
652 /* First peek to compare current entry and the next one */
654 event
= ring_buffer_iter_peek(ring_iter
, NULL
);
657 * We need to consume the current entry to see
660 ring_buffer_consume(iter
->trace_buffer
->buffer
, iter
->cpu
,
662 event
= ring_buffer_peek(iter
->trace_buffer
->buffer
, iter
->cpu
,
669 next
= ring_buffer_event_data(event
);
673 * Save current and next entries for later reference
674 * if the output fails.
678 * If the next event is not a return type, then
679 * we only care about what type it is. Otherwise we can
680 * safely copy the entire event.
682 if (next
->ent
.type
== TRACE_GRAPH_RET
)
685 data
->ret
.ent
.type
= next
->ent
.type
;
689 if (next
->ent
.type
!= TRACE_GRAPH_RET
)
692 if (curr
->ent
.pid
!= next
->ent
.pid
||
693 curr
->graph_ent
.func
!= next
->ret
.func
)
696 /* this is a leaf, now advance the iterator */
698 ring_buffer_read(ring_iter
, NULL
);
703 static void print_graph_abs_time(u64 t
, struct trace_seq
*s
)
705 unsigned long usecs_rem
;
707 usecs_rem
= do_div(t
, NSEC_PER_SEC
);
710 trace_seq_printf(s
, "%5lu.%06lu | ",
711 (unsigned long)t
, usecs_rem
);
715 print_graph_irq(struct trace_iterator
*iter
, unsigned long addr
,
716 enum trace_type type
, int cpu
, pid_t pid
, u32 flags
)
718 struct trace_array
*tr
= iter
->tr
;
719 struct trace_seq
*s
= &iter
->seq
;
720 struct trace_entry
*ent
= iter
->ent
;
722 if (addr
< (unsigned long)__irqentry_text_start
||
723 addr
>= (unsigned long)__irqentry_text_end
)
726 if (tr
->trace_flags
& TRACE_ITER_CONTEXT_INFO
) {
728 if (flags
& TRACE_GRAPH_PRINT_ABS_TIME
)
729 print_graph_abs_time(iter
->ts
, s
);
732 if (flags
& TRACE_GRAPH_PRINT_CPU
)
733 print_graph_cpu(s
, cpu
);
736 if (flags
& TRACE_GRAPH_PRINT_PROC
) {
737 print_graph_proc(s
, pid
);
738 trace_seq_puts(s
, " | ");
742 if (tr
->trace_flags
& TRACE_ITER_LATENCY_FMT
)
743 print_graph_lat_fmt(s
, ent
);
747 print_graph_duration(tr
, 0, s
, flags
| FLAGS_FILL_START
);
749 if (type
== TRACE_GRAPH_ENT
)
750 trace_seq_puts(s
, "==========>");
752 trace_seq_puts(s
, "<==========");
754 print_graph_duration(tr
, 0, s
, flags
| FLAGS_FILL_END
);
755 trace_seq_putc(s
, '\n');
759 trace_print_graph_duration(unsigned long long duration
, struct trace_seq
*s
)
761 unsigned long nsecs_rem
= do_div(duration
, 1000);
762 /* log10(ULONG_MAX) + '\0' */
768 sprintf(usecs_str
, "%lu", (unsigned long) duration
);
771 trace_seq_printf(s
, "%s", usecs_str
);
773 len
= strlen(usecs_str
);
775 /* Print nsecs (we don't want to exceed 7 numbers) */
777 size_t slen
= min_t(size_t, sizeof(nsecs_str
), 8UL - len
);
779 snprintf(nsecs_str
, slen
, "%03lu", nsecs_rem
);
780 trace_seq_printf(s
, ".%s", nsecs_str
);
781 len
+= strlen(nsecs_str
) + 1;
784 trace_seq_puts(s
, " us ");
786 /* Print remaining spaces to fit the row's width */
787 for (i
= len
; i
< 8; i
++)
788 trace_seq_putc(s
, ' ');
792 print_graph_duration(struct trace_array
*tr
, unsigned long long duration
,
793 struct trace_seq
*s
, u32 flags
)
795 if (!(flags
& TRACE_GRAPH_PRINT_DURATION
) ||
796 !(tr
->trace_flags
& TRACE_ITER_CONTEXT_INFO
))
799 /* No real adata, just filling the column with spaces */
800 switch (flags
& TRACE_GRAPH_PRINT_FILL_MASK
) {
801 case FLAGS_FILL_FULL
:
802 trace_seq_puts(s
, " | ");
804 case FLAGS_FILL_START
:
805 trace_seq_puts(s
, " ");
808 trace_seq_puts(s
, " |");
812 /* Signal a overhead of time execution to the output */
813 if (flags
& TRACE_GRAPH_PRINT_OVERHEAD
)
814 trace_seq_printf(s
, "%c ", trace_find_mark(duration
));
816 trace_seq_puts(s
, " ");
818 trace_print_graph_duration(duration
, s
);
819 trace_seq_puts(s
, "| ");
822 /* Case of a leaf function on its call entry */
823 static enum print_line_t
824 print_graph_entry_leaf(struct trace_iterator
*iter
,
825 struct ftrace_graph_ent_entry
*entry
,
826 struct ftrace_graph_ret_entry
*ret_entry
,
827 struct trace_seq
*s
, u32 flags
)
829 struct fgraph_data
*data
= iter
->private;
830 struct trace_array
*tr
= iter
->tr
;
831 struct ftrace_graph_ret
*graph_ret
;
832 struct ftrace_graph_ent
*call
;
833 unsigned long long duration
;
836 graph_ret
= &ret_entry
->ret
;
837 call
= &entry
->graph_ent
;
838 duration
= graph_ret
->rettime
- graph_ret
->calltime
;
841 struct fgraph_cpu_data
*cpu_data
;
844 cpu_data
= per_cpu_ptr(data
->cpu_data
, cpu
);
847 * Comments display at + 1 to depth. Since
848 * this is a leaf function, keep the comments
849 * equal to this depth.
851 cpu_data
->depth
= call
->depth
- 1;
853 /* No need to keep this function around for this depth */
854 if (call
->depth
< FTRACE_RETFUNC_DEPTH
)
855 cpu_data
->enter_funcs
[call
->depth
] = 0;
858 /* Overhead and duration */
859 print_graph_duration(tr
, duration
, s
, flags
);
862 for (i
= 0; i
< call
->depth
* TRACE_GRAPH_INDENT
; i
++)
863 trace_seq_putc(s
, ' ');
865 trace_seq_printf(s
, "%ps();\n", (void *)call
->func
);
867 return trace_handle_return(s
);
870 static enum print_line_t
871 print_graph_entry_nested(struct trace_iterator
*iter
,
872 struct ftrace_graph_ent_entry
*entry
,
873 struct trace_seq
*s
, int cpu
, u32 flags
)
875 struct ftrace_graph_ent
*call
= &entry
->graph_ent
;
876 struct fgraph_data
*data
= iter
->private;
877 struct trace_array
*tr
= iter
->tr
;
881 struct fgraph_cpu_data
*cpu_data
;
884 cpu_data
= per_cpu_ptr(data
->cpu_data
, cpu
);
885 cpu_data
->depth
= call
->depth
;
887 /* Save this function pointer to see if the exit matches */
888 if (call
->depth
< FTRACE_RETFUNC_DEPTH
)
889 cpu_data
->enter_funcs
[call
->depth
] = call
->func
;
893 print_graph_duration(tr
, 0, s
, flags
| FLAGS_FILL_FULL
);
896 for (i
= 0; i
< call
->depth
* TRACE_GRAPH_INDENT
; i
++)
897 trace_seq_putc(s
, ' ');
899 trace_seq_printf(s
, "%ps() {\n", (void *)call
->func
);
901 if (trace_seq_has_overflowed(s
))
902 return TRACE_TYPE_PARTIAL_LINE
;
905 * we already consumed the current entry to check the next one
906 * and see if this is a leaf.
908 return TRACE_TYPE_NO_CONSUME
;
912 print_graph_prologue(struct trace_iterator
*iter
, struct trace_seq
*s
,
913 int type
, unsigned long addr
, u32 flags
)
915 struct fgraph_data
*data
= iter
->private;
916 struct trace_entry
*ent
= iter
->ent
;
917 struct trace_array
*tr
= iter
->tr
;
921 verif_pid(s
, ent
->pid
, cpu
, data
);
925 print_graph_irq(iter
, addr
, type
, cpu
, ent
->pid
, flags
);
927 if (!(tr
->trace_flags
& TRACE_ITER_CONTEXT_INFO
))
931 if (flags
& TRACE_GRAPH_PRINT_ABS_TIME
)
932 print_graph_abs_time(iter
->ts
, s
);
935 if (flags
& TRACE_GRAPH_PRINT_CPU
)
936 print_graph_cpu(s
, cpu
);
939 if (flags
& TRACE_GRAPH_PRINT_PROC
) {
940 print_graph_proc(s
, ent
->pid
);
941 trace_seq_puts(s
, " | ");
945 if (tr
->trace_flags
& TRACE_ITER_LATENCY_FMT
)
946 print_graph_lat_fmt(s
, ent
);
952 * Entry check for irq code
955 * - we are inside irq code
956 * - we just entered irq code
959 * - funcgraph-interrupts option is set
960 * - we are not inside irq code
963 check_irq_entry(struct trace_iterator
*iter
, u32 flags
,
964 unsigned long addr
, int depth
)
968 struct fgraph_data
*data
= iter
->private;
971 * If we are either displaying irqs, or we got called as
972 * a graph event and private data does not exist,
973 * then we bypass the irq check.
975 if ((flags
& TRACE_GRAPH_PRINT_IRQS
) ||
979 depth_irq
= &(per_cpu_ptr(data
->cpu_data
, cpu
)->depth_irq
);
982 * We are inside the irq code
987 if ((addr
< (unsigned long)__irqentry_text_start
) ||
988 (addr
>= (unsigned long)__irqentry_text_end
))
992 * We are entering irq code.
999 * Return check for irq code
1002 * - we are inside irq code
1003 * - we just left irq code
1006 * - funcgraph-interrupts option is set
1007 * - we are not inside irq code
1010 check_irq_return(struct trace_iterator
*iter
, u32 flags
, int depth
)
1012 int cpu
= iter
->cpu
;
1014 struct fgraph_data
*data
= iter
->private;
1017 * If we are either displaying irqs, or we got called as
1018 * a graph event and private data does not exist,
1019 * then we bypass the irq check.
1021 if ((flags
& TRACE_GRAPH_PRINT_IRQS
) ||
1025 depth_irq
= &(per_cpu_ptr(data
->cpu_data
, cpu
)->depth_irq
);
1028 * We are not inside the irq code.
1030 if (*depth_irq
== -1)
1034 * We are inside the irq code, and this is returning entry.
1035 * Let's not trace it and clear the entry depth, since
1036 * we are out of irq code.
1038 * This condition ensures that we 'leave the irq code' once
1039 * we are out of the entry depth. Thus protecting us from
1040 * the RETURN entry loss.
1042 if (*depth_irq
>= depth
) {
1048 * We are inside the irq code, and this is not the entry.
1053 static enum print_line_t
1054 print_graph_entry(struct ftrace_graph_ent_entry
*field
, struct trace_seq
*s
,
1055 struct trace_iterator
*iter
, u32 flags
)
1057 struct fgraph_data
*data
= iter
->private;
1058 struct ftrace_graph_ent
*call
= &field
->graph_ent
;
1059 struct ftrace_graph_ret_entry
*leaf_ret
;
1060 static enum print_line_t ret
;
1061 int cpu
= iter
->cpu
;
1063 if (check_irq_entry(iter
, flags
, call
->func
, call
->depth
))
1064 return TRACE_TYPE_HANDLED
;
1066 print_graph_prologue(iter
, s
, TRACE_GRAPH_ENT
, call
->func
, flags
);
1068 leaf_ret
= get_return_for_leaf(iter
, field
);
1070 ret
= print_graph_entry_leaf(iter
, field
, leaf_ret
, s
, flags
);
1072 ret
= print_graph_entry_nested(iter
, field
, s
, cpu
, flags
);
1076 * If we failed to write our output, then we need to make
1077 * note of it. Because we already consumed our entry.
1089 static enum print_line_t
1090 print_graph_return(struct ftrace_graph_ret
*trace
, struct trace_seq
*s
,
1091 struct trace_entry
*ent
, struct trace_iterator
*iter
,
1094 unsigned long long duration
= trace
->rettime
- trace
->calltime
;
1095 struct fgraph_data
*data
= iter
->private;
1096 struct trace_array
*tr
= iter
->tr
;
1097 pid_t pid
= ent
->pid
;
1098 int cpu
= iter
->cpu
;
1102 if (check_irq_return(iter
, flags
, trace
->depth
))
1103 return TRACE_TYPE_HANDLED
;
1106 struct fgraph_cpu_data
*cpu_data
;
1107 int cpu
= iter
->cpu
;
1109 cpu_data
= per_cpu_ptr(data
->cpu_data
, cpu
);
1112 * Comments display at + 1 to depth. This is the
1113 * return from a function, we now want the comments
1114 * to display at the same level of the bracket.
1116 cpu_data
->depth
= trace
->depth
- 1;
1118 if (trace
->depth
< FTRACE_RETFUNC_DEPTH
) {
1119 if (cpu_data
->enter_funcs
[trace
->depth
] != trace
->func
)
1121 cpu_data
->enter_funcs
[trace
->depth
] = 0;
1125 print_graph_prologue(iter
, s
, 0, 0, flags
);
1127 /* Overhead and duration */
1128 print_graph_duration(tr
, duration
, s
, flags
);
1131 for (i
= 0; i
< trace
->depth
* TRACE_GRAPH_INDENT
; i
++)
1132 trace_seq_putc(s
, ' ');
1135 * If the return function does not have a matching entry,
1136 * then the entry was lost. Instead of just printing
1137 * the '}' and letting the user guess what function this
1138 * belongs to, write out the function name. Always do
1139 * that if the funcgraph-tail option is enabled.
1141 if (func_match
&& !(flags
& TRACE_GRAPH_PRINT_TAIL
))
1142 trace_seq_puts(s
, "}\n");
1144 trace_seq_printf(s
, "} /* %ps */\n", (void *)trace
->func
);
1147 if (flags
& TRACE_GRAPH_PRINT_OVERRUN
)
1148 trace_seq_printf(s
, " (Overruns: %lu)\n",
1151 print_graph_irq(iter
, trace
->func
, TRACE_GRAPH_RET
,
1154 return trace_handle_return(s
);
1157 static enum print_line_t
1158 print_graph_comment(struct trace_seq
*s
, struct trace_entry
*ent
,
1159 struct trace_iterator
*iter
, u32 flags
)
1161 struct trace_array
*tr
= iter
->tr
;
1162 unsigned long sym_flags
= (tr
->trace_flags
& TRACE_ITER_SYM_MASK
);
1163 struct fgraph_data
*data
= iter
->private;
1164 struct trace_event
*event
;
1170 depth
= per_cpu_ptr(data
->cpu_data
, iter
->cpu
)->depth
;
1172 print_graph_prologue(iter
, s
, 0, 0, flags
);
1175 print_graph_duration(tr
, 0, s
, flags
| FLAGS_FILL_FULL
);
1179 for (i
= 0; i
< (depth
+ 1) * TRACE_GRAPH_INDENT
; i
++)
1180 trace_seq_putc(s
, ' ');
1183 trace_seq_puts(s
, "/* ");
1185 switch (iter
->ent
->type
) {
1187 ret
= trace_print_bprintk_msg_only(iter
);
1188 if (ret
!= TRACE_TYPE_HANDLED
)
1192 ret
= trace_print_printk_msg_only(iter
);
1193 if (ret
!= TRACE_TYPE_HANDLED
)
1197 event
= ftrace_find_event(ent
->type
);
1199 return TRACE_TYPE_UNHANDLED
;
1201 ret
= event
->funcs
->trace(iter
, sym_flags
, event
);
1202 if (ret
!= TRACE_TYPE_HANDLED
)
1206 if (trace_seq_has_overflowed(s
))
1209 /* Strip ending newline */
1210 if (s
->buffer
[s
->seq
.len
- 1] == '\n') {
1211 s
->buffer
[s
->seq
.len
- 1] = '\0';
1215 trace_seq_puts(s
, " */\n");
1217 return trace_handle_return(s
);
1222 print_graph_function_flags(struct trace_iterator
*iter
, u32 flags
)
1224 struct ftrace_graph_ent_entry
*field
;
1225 struct fgraph_data
*data
= iter
->private;
1226 struct trace_entry
*entry
= iter
->ent
;
1227 struct trace_seq
*s
= &iter
->seq
;
1228 int cpu
= iter
->cpu
;
1231 if (data
&& per_cpu_ptr(data
->cpu_data
, cpu
)->ignore
) {
1232 per_cpu_ptr(data
->cpu_data
, cpu
)->ignore
= 0;
1233 return TRACE_TYPE_HANDLED
;
1237 * If the last output failed, there's a possibility we need
1238 * to print out the missing entry which would never go out.
1240 if (data
&& data
->failed
) {
1242 iter
->cpu
= data
->cpu
;
1243 ret
= print_graph_entry(field
, s
, iter
, flags
);
1244 if (ret
== TRACE_TYPE_HANDLED
&& iter
->cpu
!= cpu
) {
1245 per_cpu_ptr(data
->cpu_data
, iter
->cpu
)->ignore
= 1;
1246 ret
= TRACE_TYPE_NO_CONSUME
;
1252 switch (entry
->type
) {
1253 case TRACE_GRAPH_ENT
: {
1255 * print_graph_entry() may consume the current event,
1256 * thus @field may become invalid, so we need to save it.
1257 * sizeof(struct ftrace_graph_ent_entry) is very small,
1258 * it can be safely saved at the stack.
1260 struct ftrace_graph_ent_entry saved
;
1261 trace_assign_type(field
, entry
);
1263 return print_graph_entry(&saved
, s
, iter
, flags
);
1265 case TRACE_GRAPH_RET
: {
1266 struct ftrace_graph_ret_entry
*field
;
1267 trace_assign_type(field
, entry
);
1268 return print_graph_return(&field
->ret
, s
, entry
, iter
, flags
);
1272 /* dont trace stack and functions as comments */
1273 return TRACE_TYPE_UNHANDLED
;
1276 return print_graph_comment(s
, entry
, iter
, flags
);
1279 return TRACE_TYPE_HANDLED
;
1282 static enum print_line_t
1283 print_graph_function(struct trace_iterator
*iter
)
1285 return print_graph_function_flags(iter
, tracer_flags
.val
);
1288 static enum print_line_t
1289 print_graph_function_event(struct trace_iterator
*iter
, int flags
,
1290 struct trace_event
*event
)
1292 return print_graph_function(iter
);
1295 static void print_lat_header(struct seq_file
*s
, u32 flags
)
1297 static const char spaces
[] = " " /* 16 spaces */
1299 " "; /* 17 spaces */
1302 if (flags
& TRACE_GRAPH_PRINT_ABS_TIME
)
1304 if (flags
& TRACE_GRAPH_PRINT_CPU
)
1306 if (flags
& TRACE_GRAPH_PRINT_PROC
)
1309 seq_printf(s
, "#%.*s _-----=> irqs-off \n", size
, spaces
);
1310 seq_printf(s
, "#%.*s / _----=> need-resched \n", size
, spaces
);
1311 seq_printf(s
, "#%.*s| / _---=> hardirq/softirq \n", size
, spaces
);
1312 seq_printf(s
, "#%.*s|| / _--=> preempt-depth \n", size
, spaces
);
1313 seq_printf(s
, "#%.*s||| / \n", size
, spaces
);
1316 static void __print_graph_headers_flags(struct trace_array
*tr
,
1317 struct seq_file
*s
, u32 flags
)
1319 int lat
= tr
->trace_flags
& TRACE_ITER_LATENCY_FMT
;
1322 print_lat_header(s
, flags
);
1326 if (flags
& TRACE_GRAPH_PRINT_ABS_TIME
)
1327 seq_puts(s
, " TIME ");
1328 if (flags
& TRACE_GRAPH_PRINT_CPU
)
1329 seq_puts(s
, " CPU");
1330 if (flags
& TRACE_GRAPH_PRINT_PROC
)
1331 seq_puts(s
, " TASK/PID ");
1333 seq_puts(s
, "||||");
1334 if (flags
& TRACE_GRAPH_PRINT_DURATION
)
1335 seq_puts(s
, " DURATION ");
1336 seq_puts(s
, " FUNCTION CALLS\n");
1340 if (flags
& TRACE_GRAPH_PRINT_ABS_TIME
)
1342 if (flags
& TRACE_GRAPH_PRINT_CPU
)
1344 if (flags
& TRACE_GRAPH_PRINT_PROC
)
1345 seq_puts(s
, " | | ");
1347 seq_puts(s
, "||||");
1348 if (flags
& TRACE_GRAPH_PRINT_DURATION
)
1349 seq_puts(s
, " | | ");
1350 seq_puts(s
, " | | | |\n");
1353 static void print_graph_headers(struct seq_file
*s
)
1355 print_graph_headers_flags(s
, tracer_flags
.val
);
1358 void print_graph_headers_flags(struct seq_file
*s
, u32 flags
)
1360 struct trace_iterator
*iter
= s
->private;
1361 struct trace_array
*tr
= iter
->tr
;
1363 if (!(tr
->trace_flags
& TRACE_ITER_CONTEXT_INFO
))
1366 if (tr
->trace_flags
& TRACE_ITER_LATENCY_FMT
) {
1367 /* print nothing if the buffers are empty */
1368 if (trace_empty(iter
))
1371 print_trace_header(s
, iter
);
1374 __print_graph_headers_flags(tr
, s
, flags
);
1377 void graph_trace_open(struct trace_iterator
*iter
)
1379 /* pid and depth on the last trace processed */
1380 struct fgraph_data
*data
;
1384 iter
->private = NULL
;
1386 /* We can be called in atomic context via ftrace_dump() */
1387 gfpflags
= (in_atomic() || irqs_disabled()) ? GFP_ATOMIC
: GFP_KERNEL
;
1389 data
= kzalloc(sizeof(*data
), gfpflags
);
1393 data
->cpu_data
= alloc_percpu_gfp(struct fgraph_cpu_data
, gfpflags
);
1394 if (!data
->cpu_data
)
1397 for_each_possible_cpu(cpu
) {
1398 pid_t
*pid
= &(per_cpu_ptr(data
->cpu_data
, cpu
)->last_pid
);
1399 int *depth
= &(per_cpu_ptr(data
->cpu_data
, cpu
)->depth
);
1400 int *ignore
= &(per_cpu_ptr(data
->cpu_data
, cpu
)->ignore
);
1401 int *depth_irq
= &(per_cpu_ptr(data
->cpu_data
, cpu
)->depth_irq
);
1409 iter
->private = data
;
1416 pr_warn("function graph tracer: not enough memory\n");
1419 void graph_trace_close(struct trace_iterator
*iter
)
1421 struct fgraph_data
*data
= iter
->private;
1424 free_percpu(data
->cpu_data
);
1430 func_graph_set_flag(struct trace_array
*tr
, u32 old_flags
, u32 bit
, int set
)
1432 if (bit
== TRACE_GRAPH_PRINT_IRQS
)
1433 ftrace_graph_skip_irqs
= !set
;
1435 if (bit
== TRACE_GRAPH_SLEEP_TIME
)
1436 ftrace_graph_sleep_time_control(set
);
1438 if (bit
== TRACE_GRAPH_GRAPH_TIME
)
1439 ftrace_graph_graph_time_control(set
);
1444 static struct trace_event_functions graph_functions
= {
1445 .trace
= print_graph_function_event
,
1448 static struct trace_event graph_trace_entry_event
= {
1449 .type
= TRACE_GRAPH_ENT
,
1450 .funcs
= &graph_functions
,
1453 static struct trace_event graph_trace_ret_event
= {
1454 .type
= TRACE_GRAPH_RET
,
1455 .funcs
= &graph_functions
1458 static struct tracer graph_trace __tracer_data
= {
1459 .name
= "function_graph",
1460 .update_thresh
= graph_trace_update_thresh
,
1461 .open
= graph_trace_open
,
1462 .pipe_open
= graph_trace_open
,
1463 .close
= graph_trace_close
,
1464 .pipe_close
= graph_trace_close
,
1465 .init
= graph_trace_init
,
1466 .reset
= graph_trace_reset
,
1467 .print_line
= print_graph_function
,
1468 .print_header
= print_graph_headers
,
1469 .flags
= &tracer_flags
,
1470 .set_flag
= func_graph_set_flag
,
1471 #ifdef CONFIG_FTRACE_SELFTEST
1472 .selftest
= trace_selftest_startup_function_graph
,
1478 graph_depth_write(struct file
*filp
, const char __user
*ubuf
, size_t cnt
,
1484 ret
= kstrtoul_from_user(ubuf
, cnt
, 10, &val
);
1496 graph_depth_read(struct file
*filp
, char __user
*ubuf
, size_t cnt
,
1499 char buf
[15]; /* More than enough to hold UINT_MAX + "\n"*/
1502 n
= sprintf(buf
, "%d\n", max_depth
);
1504 return simple_read_from_buffer(ubuf
, cnt
, ppos
, buf
, n
);
1507 static const struct file_operations graph_depth_fops
= {
1508 .open
= tracing_open_generic
,
1509 .write
= graph_depth_write
,
1510 .read
= graph_depth_read
,
1511 .llseek
= generic_file_llseek
,
1514 static __init
int init_graph_tracefs(void)
1516 struct dentry
*d_tracer
;
1518 d_tracer
= tracing_init_dentry();
1519 if (IS_ERR(d_tracer
))
1522 trace_create_file("max_graph_depth", 0644, d_tracer
,
1523 NULL
, &graph_depth_fops
);
1527 fs_initcall(init_graph_tracefs
);
1529 static __init
int init_graph_trace(void)
1531 max_bytes_for_cpu
= snprintf(NULL
, 0, "%d", nr_cpu_ids
- 1);
1533 if (!register_trace_event(&graph_trace_entry_event
)) {
1534 pr_warn("Warning: could not register graph trace events\n");
1538 if (!register_trace_event(&graph_trace_ret_event
)) {
1539 pr_warn("Warning: could not register graph trace events\n");
1543 return register_tracer(&graph_trace
);
1546 core_initcall(init_graph_trace
);