4 * Copyright (C) 2008 Steven Rostedt <srostedt@redhat.com>
6 #include <linux/kallsyms.h>
7 #include <linux/seq_file.h>
8 #include <linux/spinlock.h>
9 #include <linux/irqflags.h>
10 #include <linux/debugfs.h>
11 #include <linux/uaccess.h>
12 #include <linux/module.h>
13 #include <linux/ftrace.h>
14 #include <linux/hash.h>
16 #include <asm/local.h>
19 #include "trace_stat.h"
20 #include "trace_output.h"
22 #ifdef CONFIG_BRANCH_TRACER
24 static struct tracer branch_trace
;
25 static int branch_tracing_enabled __read_mostly
;
26 static DEFINE_MUTEX(branch_tracing_mutex
);
28 static struct trace_array
*branch_tracer
;
31 probe_likely_condition(struct ftrace_branch_data
*f
, int val
, int expect
)
33 struct trace_array
*tr
= branch_tracer
;
34 struct ring_buffer_event
*event
;
35 struct trace_branch
*entry
;
36 unsigned long flags
, irq_flags
;
41 * I would love to save just the ftrace_likely_data pointer, but
42 * this code can also be used by modules. Ugly things can happen
43 * if the module is unloaded, and then we go and read the
44 * pointer. This is slower, but much safer.
50 local_irq_save(flags
);
51 cpu
= raw_smp_processor_id();
52 if (atomic_inc_return(&tr
->data
[cpu
]->disabled
) != 1)
55 event
= ring_buffer_lock_reserve(tr
->buffer
, sizeof(*entry
),
61 entry
= ring_buffer_event_data(event
);
62 tracing_generic_entry_update(&entry
->ent
, flags
, pc
);
63 entry
->ent
.type
= TRACE_BRANCH
;
65 /* Strip off the path, only save the file */
66 p
= f
->file
+ strlen(f
->file
);
67 while (p
>= f
->file
&& *p
!= '/')
71 strncpy(entry
->func
, f
->func
, TRACE_FUNC_SIZE
);
72 strncpy(entry
->file
, p
, TRACE_FILE_SIZE
);
73 entry
->func
[TRACE_FUNC_SIZE
] = 0;
74 entry
->file
[TRACE_FILE_SIZE
] = 0;
75 entry
->line
= f
->line
;
76 entry
->correct
= val
== expect
;
78 ring_buffer_unlock_commit(tr
->buffer
, event
, irq_flags
);
81 atomic_dec(&tr
->data
[cpu
]->disabled
);
82 local_irq_restore(flags
);
86 void trace_likely_condition(struct ftrace_branch_data
*f
, int val
, int expect
)
88 if (!branch_tracing_enabled
)
91 probe_likely_condition(f
, val
, expect
);
94 int enable_branch_tracing(struct trace_array
*tr
)
98 mutex_lock(&branch_tracing_mutex
);
101 * Must be seen before enabling. The reader is a condition
102 * where we do not need a matching rmb()
105 branch_tracing_enabled
++;
106 mutex_unlock(&branch_tracing_mutex
);
111 void disable_branch_tracing(void)
113 mutex_lock(&branch_tracing_mutex
);
115 if (!branch_tracing_enabled
)
118 branch_tracing_enabled
--;
121 mutex_unlock(&branch_tracing_mutex
);
124 static void start_branch_trace(struct trace_array
*tr
)
126 enable_branch_tracing(tr
);
129 static void stop_branch_trace(struct trace_array
*tr
)
131 disable_branch_tracing();
134 static int branch_trace_init(struct trace_array
*tr
)
136 tracing_reset_online_cpus(tr
);
137 start_branch_trace(tr
);
141 static void branch_trace_reset(struct trace_array
*tr
)
143 stop_branch_trace(tr
);
147 trace_print_print(struct trace_seq
*s
, struct trace_entry
*entry
, int flags
)
149 struct print_entry
*field
;
151 trace_assign_type(field
, entry
);
153 if (seq_print_ip_sym(s
, field
->ip
, flags
))
156 if (trace_seq_printf(s
, ": %s", field
->buf
))
160 return TRACE_TYPE_PARTIAL_LINE
;
164 trace_branch_print(struct trace_seq
*s
, struct trace_entry
*entry
, int flags
)
166 struct trace_branch
*field
;
168 trace_assign_type(field
, entry
);
170 if (trace_seq_printf(s
, "[%s] %s:%s:%d\n",
171 field
->correct
? " ok " : " MISS ",
175 return TRACE_TYPE_PARTIAL_LINE
;
181 static struct trace_event trace_branch_event
= {
182 .type
= TRACE_BRANCH
,
183 .trace
= trace_branch_print
,
184 .latency_trace
= trace_branch_print
,
185 .raw
= trace_nop_print
,
186 .hex
= trace_nop_print
,
187 .binary
= trace_nop_print
,
190 static struct tracer branch_trace __read_mostly
=
193 .init
= branch_trace_init
,
194 .reset
= branch_trace_reset
,
195 #ifdef CONFIG_FTRACE_SELFTEST
196 .selftest
= trace_selftest_startup_branch
,
197 #endif /* CONFIG_FTRACE_SELFTEST */
200 __init
static int init_branch_tracer(void)
204 ret
= register_ftrace_event(&trace_branch_event
);
206 printk(KERN_WARNING
"Warning: could not register "
210 return register_tracer(&branch_trace
);
212 device_initcall(init_branch_tracer
);
216 void trace_likely_condition(struct ftrace_branch_data
*f
, int val
, int expect
)
219 #endif /* CONFIG_BRANCH_TRACER */
221 void ftrace_likely_update(struct ftrace_branch_data
*f
, int val
, int expect
)
224 * I would love to have a trace point here instead, but the
225 * trace point code is so inundated with unlikely and likely
226 * conditions that the recursive nightmare that exists is too
227 * much to try to get working. At least for now.
229 trace_likely_condition(f
, val
, expect
);
231 /* FIXME: Make this atomic! */
237 EXPORT_SYMBOL(ftrace_likely_update
);
239 extern unsigned long __start_annotated_branch_profile
[];
240 extern unsigned long __stop_annotated_branch_profile
[];
242 static int annotated_branch_stat_headers(struct seq_file
*m
)
244 seq_printf(m
, " correct incorrect %% ");
245 seq_printf(m
, " Function "
247 " ------- --------- - "
253 static inline long get_incorrect_percent(struct ftrace_branch_data
*p
)
258 percent
= p
->incorrect
* 100;
259 percent
/= p
->correct
+ p
->incorrect
;
261 percent
= p
->incorrect
? 100 : -1;
266 static int branch_stat_show(struct seq_file
*m
, void *v
)
268 struct ftrace_branch_data
*p
= v
;
272 /* Only print the file, not the path */
273 f
= p
->file
+ strlen(p
->file
);
274 while (f
>= p
->file
&& *f
!= '/')
279 * The miss is overlayed on correct, and hit on incorrect.
281 percent
= get_incorrect_percent(p
);
283 seq_printf(m
, "%8lu %8lu ", p
->correct
, p
->incorrect
);
285 seq_printf(m
, " X ");
287 seq_printf(m
, "%3ld ", percent
);
288 seq_printf(m
, "%-30.30s %-20.20s %d\n", p
->func
, f
, p
->line
);
292 static void *annotated_branch_stat_start(void)
294 return __start_annotated_branch_profile
;
298 annotated_branch_stat_next(void *v
, int idx
)
300 struct ftrace_branch_data
*p
= v
;
304 if ((void *)p
>= (void *)__stop_annotated_branch_profile
)
310 static int annotated_branch_stat_cmp(void *p1
, void *p2
)
312 struct ftrace_branch_data
*a
= p1
;
313 struct ftrace_branch_data
*b
= p2
;
315 long percent_a
, percent_b
;
317 percent_a
= get_incorrect_percent(a
);
318 percent_b
= get_incorrect_percent(b
);
320 if (percent_a
< percent_b
)
322 if (percent_a
> percent_b
)
328 static struct tracer_stat annotated_branch_stats
= {
329 .name
= "branch_annotated",
330 .stat_start
= annotated_branch_stat_start
,
331 .stat_next
= annotated_branch_stat_next
,
332 .stat_cmp
= annotated_branch_stat_cmp
,
333 .stat_headers
= annotated_branch_stat_headers
,
334 .stat_show
= branch_stat_show
337 __init
static int init_annotated_branch_stats(void)
341 ret
= register_stat_tracer(&annotated_branch_stats
);
343 printk(KERN_WARNING
"Warning: could not register "
344 "annotated branches stats\n");
349 fs_initcall(init_annotated_branch_stats
);
351 #ifdef CONFIG_PROFILE_ALL_BRANCHES
353 extern unsigned long __start_branch_profile
[];
354 extern unsigned long __stop_branch_profile
[];
356 static int all_branch_stat_headers(struct seq_file
*m
)
358 seq_printf(m
, " miss hit %% ");
359 seq_printf(m
, " Function "
361 " ------- --------- - "
367 static void *all_branch_stat_start(void)
369 return __start_branch_profile
;
373 all_branch_stat_next(void *v
, int idx
)
375 struct ftrace_branch_data
*p
= v
;
379 if ((void *)p
>= (void *)__stop_branch_profile
)
385 static struct tracer_stat all_branch_stats
= {
386 .name
= "branch_all",
387 .stat_start
= all_branch_stat_start
,
388 .stat_next
= all_branch_stat_next
,
389 .stat_headers
= all_branch_stat_headers
,
390 .stat_show
= branch_stat_show
393 __init
static int all_annotated_branch_stats(void)
397 ret
= register_stat_tracer(&all_branch_stats
);
399 printk(KERN_WARNING
"Warning: could not register "
400 "all branches stats\n");
405 fs_initcall(all_annotated_branch_stats
);
406 #endif /* CONFIG_PROFILE_ALL_BRANCHES */