2 * ring buffer based function tracer
4 * Copyright (C) 2007-2012 Steven Rostedt <srostedt@redhat.com>
5 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
7 * Originally taken from the RT patch by:
8 * Arnaldo Carvalho de Melo <acme@redhat.com>
10 * Based on code from the latency_tracer, that is:
11 * Copyright (C) 2004-2006 Ingo Molnar
12 * Copyright (C) 2004 Nadia Yvette Chambers
14 #include <linux/ring_buffer.h>
15 #include <generated/utsrelease.h>
16 #include <linux/stacktrace.h>
17 #include <linux/writeback.h>
18 #include <linux/kallsyms.h>
19 #include <linux/seq_file.h>
20 #include <linux/notifier.h>
21 #include <linux/irqflags.h>
22 #include <linux/debugfs.h>
23 #include <linux/tracefs.h>
24 #include <linux/pagemap.h>
25 #include <linux/hardirq.h>
26 #include <linux/linkage.h>
27 #include <linux/uaccess.h>
28 #include <linux/kprobes.h>
29 #include <linux/ftrace.h>
30 #include <linux/module.h>
31 #include <linux/percpu.h>
32 #include <linux/splice.h>
33 #include <linux/kdebug.h>
34 #include <linux/string.h>
35 #include <linux/mount.h>
36 #include <linux/rwsem.h>
37 #include <linux/slab.h>
38 #include <linux/ctype.h>
39 #include <linux/init.h>
40 #include <linux/poll.h>
41 #include <linux/nmi.h>
43 #include <linux/sched/rt.h>
46 #include "trace_output.h"
49 * On boot up, the ring buffer is set to the minimum size, so that
50 * we do not waste memory on systems that are not using tracing.
52 bool ring_buffer_expanded
;
55 * We need to change this state when a selftest is running.
56 * A selftest will lurk into the ring-buffer to count the
57 * entries inserted during the selftest although some concurrent
58 * insertions into the ring-buffer such as trace_printk could occurred
59 * at the same time, giving false positive or negative results.
61 static bool __read_mostly tracing_selftest_running
;
64 * If a tracer is running, we do not want to run SELFTEST.
66 bool __read_mostly tracing_selftest_disabled
;
68 /* Pipe tracepoints to printk */
69 struct trace_iterator
*tracepoint_print_iter
;
70 int tracepoint_printk
;
72 /* For tracers that don't implement custom flags */
73 static struct tracer_opt dummy_tracer_opt
[] = {
78 dummy_set_flag(struct trace_array
*tr
, u32 old_flags
, u32 bit
, int set
)
84 * To prevent the comm cache from being overwritten when no
85 * tracing is active, only save the comm when a trace event
88 static DEFINE_PER_CPU(bool, trace_cmdline_save
);
91 * Kill all tracing for good (never come back).
92 * It is initialized to 1 but will turn to zero if the initialization
93 * of the tracer is successful. But that is the only place that sets
96 static int tracing_disabled
= 1;
98 cpumask_var_t __read_mostly tracing_buffer_mask
;
101 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
103 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
104 * is set, then ftrace_dump is called. This will output the contents
105 * of the ftrace buffers to the console. This is very useful for
106 * capturing traces that lead to crashes and outputing it to a
109 * It is default off, but you can enable it with either specifying
110 * "ftrace_dump_on_oops" in the kernel command line, or setting
111 * /proc/sys/kernel/ftrace_dump_on_oops
112 * Set 1 if you want to dump buffers of all CPUs
113 * Set 2 if you want to dump the buffer of the CPU that triggered oops
116 enum ftrace_dump_mode ftrace_dump_on_oops
;
118 /* When set, tracing will stop when a WARN*() is hit */
119 int __disable_trace_on_warning
;
121 #ifdef CONFIG_TRACE_ENUM_MAP_FILE
122 /* Map of enums to their values, for "enum_map" file */
123 struct trace_enum_map_head
{
125 unsigned long length
;
128 union trace_enum_map_item
;
130 struct trace_enum_map_tail
{
132 * "end" is first and points to NULL as it must be different
133 * than "mod" or "enum_string"
135 union trace_enum_map_item
*next
;
136 const char *end
; /* points to NULL */
139 static DEFINE_MUTEX(trace_enum_mutex
);
142 * The trace_enum_maps are saved in an array with two extra elements,
143 * one at the beginning, and one at the end. The beginning item contains
144 * the count of the saved maps (head.length), and the module they
145 * belong to if not built in (head.mod). The ending item contains a
146 * pointer to the next array of saved enum_map items.
148 union trace_enum_map_item
{
149 struct trace_enum_map map
;
150 struct trace_enum_map_head head
;
151 struct trace_enum_map_tail tail
;
154 static union trace_enum_map_item
*trace_enum_maps
;
155 #endif /* CONFIG_TRACE_ENUM_MAP_FILE */
157 static int tracing_set_tracer(struct trace_array
*tr
, const char *buf
);
159 #define MAX_TRACER_SIZE 100
160 static char bootup_tracer_buf
[MAX_TRACER_SIZE
] __initdata
;
161 static char *default_bootup_tracer
;
163 static bool allocate_snapshot
;
165 static int __init
set_cmdline_ftrace(char *str
)
167 strlcpy(bootup_tracer_buf
, str
, MAX_TRACER_SIZE
);
168 default_bootup_tracer
= bootup_tracer_buf
;
169 /* We are using ftrace early, expand it */
170 ring_buffer_expanded
= true;
173 __setup("ftrace=", set_cmdline_ftrace
);
175 static int __init
set_ftrace_dump_on_oops(char *str
)
177 if (*str
++ != '=' || !*str
) {
178 ftrace_dump_on_oops
= DUMP_ALL
;
182 if (!strcmp("orig_cpu", str
)) {
183 ftrace_dump_on_oops
= DUMP_ORIG
;
189 __setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops
);
191 static int __init
stop_trace_on_warning(char *str
)
193 if ((strcmp(str
, "=0") != 0 && strcmp(str
, "=off") != 0))
194 __disable_trace_on_warning
= 1;
197 __setup("traceoff_on_warning", stop_trace_on_warning
);
199 static int __init
boot_alloc_snapshot(char *str
)
201 allocate_snapshot
= true;
202 /* We also need the main ring buffer expanded */
203 ring_buffer_expanded
= true;
206 __setup("alloc_snapshot", boot_alloc_snapshot
);
209 static char trace_boot_options_buf
[MAX_TRACER_SIZE
] __initdata
;
211 static int __init
set_trace_boot_options(char *str
)
213 strlcpy(trace_boot_options_buf
, str
, MAX_TRACER_SIZE
);
216 __setup("trace_options=", set_trace_boot_options
);
218 static char trace_boot_clock_buf
[MAX_TRACER_SIZE
] __initdata
;
219 static char *trace_boot_clock __initdata
;
221 static int __init
set_trace_boot_clock(char *str
)
223 strlcpy(trace_boot_clock_buf
, str
, MAX_TRACER_SIZE
);
224 trace_boot_clock
= trace_boot_clock_buf
;
227 __setup("trace_clock=", set_trace_boot_clock
);
229 static int __init
set_tracepoint_printk(char *str
)
231 if ((strcmp(str
, "=0") != 0 && strcmp(str
, "=off") != 0))
232 tracepoint_printk
= 1;
235 __setup("tp_printk", set_tracepoint_printk
);
237 unsigned long long ns2usecs(cycle_t nsec
)
244 /* trace_flags holds trace_options default values */
245 #define TRACE_DEFAULT_FLAGS \
246 (FUNCTION_DEFAULT_FLAGS | \
247 TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK | \
248 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | \
249 TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE | \
250 TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS)
252 /* trace_options that are only supported by global_trace */
253 #define TOP_LEVEL_TRACE_FLAGS (TRACE_ITER_PRINTK | \
254 TRACE_ITER_PRINTK_MSGONLY | TRACE_ITER_RECORD_CMD)
256 /* trace_flags that are default zero for instances */
257 #define ZEROED_TRACE_FLAGS \
258 TRACE_ITER_EVENT_FORK
261 * The global_trace is the descriptor that holds the tracing
262 * buffers for the live tracing. For each CPU, it contains
263 * a link list of pages that will store trace entries. The
264 * page descriptor of the pages in the memory is used to hold
265 * the link list by linking the lru item in the page descriptor
266 * to each of the pages in the buffer per CPU.
268 * For each active CPU there is a data field that holds the
269 * pages for the buffer for that CPU. Each CPU has the same number
270 * of pages allocated for its buffer.
272 static struct trace_array global_trace
= {
273 .trace_flags
= TRACE_DEFAULT_FLAGS
,
276 LIST_HEAD(ftrace_trace_arrays
);
278 int trace_array_get(struct trace_array
*this_tr
)
280 struct trace_array
*tr
;
283 mutex_lock(&trace_types_lock
);
284 list_for_each_entry(tr
, &ftrace_trace_arrays
, list
) {
291 mutex_unlock(&trace_types_lock
);
296 static void __trace_array_put(struct trace_array
*this_tr
)
298 WARN_ON(!this_tr
->ref
);
302 void trace_array_put(struct trace_array
*this_tr
)
304 mutex_lock(&trace_types_lock
);
305 __trace_array_put(this_tr
);
306 mutex_unlock(&trace_types_lock
);
309 int call_filter_check_discard(struct trace_event_call
*call
, void *rec
,
310 struct ring_buffer
*buffer
,
311 struct ring_buffer_event
*event
)
313 if (unlikely(call
->flags
& TRACE_EVENT_FL_FILTERED
) &&
314 !filter_match_preds(call
->filter
, rec
)) {
315 ring_buffer_discard_commit(buffer
, event
);
322 static cycle_t
buffer_ftrace_now(struct trace_buffer
*buf
, int cpu
)
326 /* Early boot up does not have a buffer yet */
328 return trace_clock_local();
330 ts
= ring_buffer_time_stamp(buf
->buffer
, cpu
);
331 ring_buffer_normalize_time_stamp(buf
->buffer
, cpu
, &ts
);
336 cycle_t
ftrace_now(int cpu
)
338 return buffer_ftrace_now(&global_trace
.trace_buffer
, cpu
);
342 * tracing_is_enabled - Show if global_trace has been disabled
344 * Shows if the global trace has been enabled or not. It uses the
345 * mirror flag "buffer_disabled" to be used in fast paths such as for
346 * the irqsoff tracer. But it may be inaccurate due to races. If you
347 * need to know the accurate state, use tracing_is_on() which is a little
348 * slower, but accurate.
350 int tracing_is_enabled(void)
353 * For quick access (irqsoff uses this in fast path), just
354 * return the mirror variable of the state of the ring buffer.
355 * It's a little racy, but we don't really care.
358 return !global_trace
.buffer_disabled
;
362 * trace_buf_size is the size in bytes that is allocated
363 * for a buffer. Note, the number of bytes is always rounded
366 * This number is purposely set to a low number of 16384.
367 * If the dump on oops happens, it will be much appreciated
368 * to not have to wait for all that output. Anyway this can be
369 * boot time and run time configurable.
371 #define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
373 static unsigned long trace_buf_size
= TRACE_BUF_SIZE_DEFAULT
;
375 /* trace_types holds a link list of available tracers. */
376 static struct tracer
*trace_types __read_mostly
;
379 * trace_types_lock is used to protect the trace_types list.
381 DEFINE_MUTEX(trace_types_lock
);
384 * serialize the access of the ring buffer
386 * ring buffer serializes readers, but it is low level protection.
387 * The validity of the events (which returns by ring_buffer_peek() ..etc)
388 * are not protected by ring buffer.
390 * The content of events may become garbage if we allow other process consumes
391 * these events concurrently:
392 * A) the page of the consumed events may become a normal page
393 * (not reader page) in ring buffer, and this page will be rewrited
394 * by events producer.
395 * B) The page of the consumed events may become a page for splice_read,
396 * and this page will be returned to system.
398 * These primitives allow multi process access to different cpu ring buffer
401 * These primitives don't distinguish read-only and read-consume access.
402 * Multi read-only access are also serialized.
406 static DECLARE_RWSEM(all_cpu_access_lock
);
407 static DEFINE_PER_CPU(struct mutex
, cpu_access_lock
);
409 static inline void trace_access_lock(int cpu
)
411 if (cpu
== RING_BUFFER_ALL_CPUS
) {
412 /* gain it for accessing the whole ring buffer. */
413 down_write(&all_cpu_access_lock
);
415 /* gain it for accessing a cpu ring buffer. */
417 /* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */
418 down_read(&all_cpu_access_lock
);
420 /* Secondly block other access to this @cpu ring buffer. */
421 mutex_lock(&per_cpu(cpu_access_lock
, cpu
));
425 static inline void trace_access_unlock(int cpu
)
427 if (cpu
== RING_BUFFER_ALL_CPUS
) {
428 up_write(&all_cpu_access_lock
);
430 mutex_unlock(&per_cpu(cpu_access_lock
, cpu
));
431 up_read(&all_cpu_access_lock
);
435 static inline void trace_access_lock_init(void)
439 for_each_possible_cpu(cpu
)
440 mutex_init(&per_cpu(cpu_access_lock
, cpu
));
445 static DEFINE_MUTEX(access_lock
);
447 static inline void trace_access_lock(int cpu
)
450 mutex_lock(&access_lock
);
453 static inline void trace_access_unlock(int cpu
)
456 mutex_unlock(&access_lock
);
459 static inline void trace_access_lock_init(void)
465 #ifdef CONFIG_STACKTRACE
466 static void __ftrace_trace_stack(struct ring_buffer
*buffer
,
468 int skip
, int pc
, struct pt_regs
*regs
);
469 static inline void ftrace_trace_stack(struct trace_array
*tr
,
470 struct ring_buffer
*buffer
,
472 int skip
, int pc
, struct pt_regs
*regs
);
475 static inline void __ftrace_trace_stack(struct ring_buffer
*buffer
,
477 int skip
, int pc
, struct pt_regs
*regs
)
480 static inline void ftrace_trace_stack(struct trace_array
*tr
,
481 struct ring_buffer
*buffer
,
483 int skip
, int pc
, struct pt_regs
*regs
)
489 static void tracer_tracing_on(struct trace_array
*tr
)
491 if (tr
->trace_buffer
.buffer
)
492 ring_buffer_record_on(tr
->trace_buffer
.buffer
);
494 * This flag is looked at when buffers haven't been allocated
495 * yet, or by some tracers (like irqsoff), that just want to
496 * know if the ring buffer has been disabled, but it can handle
497 * races of where it gets disabled but we still do a record.
498 * As the check is in the fast path of the tracers, it is more
499 * important to be fast than accurate.
501 tr
->buffer_disabled
= 0;
502 /* Make the flag seen by readers */
507 * tracing_on - enable tracing buffers
509 * This function enables tracing buffers that may have been
510 * disabled with tracing_off.
512 void tracing_on(void)
514 tracer_tracing_on(&global_trace
);
516 EXPORT_SYMBOL_GPL(tracing_on
);
519 * __trace_puts - write a constant string into the trace buffer.
520 * @ip: The address of the caller
521 * @str: The constant string to write
522 * @size: The size of the string.
524 int __trace_puts(unsigned long ip
, const char *str
, int size
)
526 struct ring_buffer_event
*event
;
527 struct ring_buffer
*buffer
;
528 struct print_entry
*entry
;
529 unsigned long irq_flags
;
533 if (!(global_trace
.trace_flags
& TRACE_ITER_PRINTK
))
536 pc
= preempt_count();
538 if (unlikely(tracing_selftest_running
|| tracing_disabled
))
541 alloc
= sizeof(*entry
) + size
+ 2; /* possible \n added */
543 local_save_flags(irq_flags
);
544 buffer
= global_trace
.trace_buffer
.buffer
;
545 event
= trace_buffer_lock_reserve(buffer
, TRACE_PRINT
, alloc
,
550 entry
= ring_buffer_event_data(event
);
553 memcpy(&entry
->buf
, str
, size
);
555 /* Add a newline if necessary */
556 if (entry
->buf
[size
- 1] != '\n') {
557 entry
->buf
[size
] = '\n';
558 entry
->buf
[size
+ 1] = '\0';
560 entry
->buf
[size
] = '\0';
562 __buffer_unlock_commit(buffer
, event
);
563 ftrace_trace_stack(&global_trace
, buffer
, irq_flags
, 4, pc
, NULL
);
567 EXPORT_SYMBOL_GPL(__trace_puts
);
570 * __trace_bputs - write the pointer to a constant string into trace buffer
571 * @ip: The address of the caller
572 * @str: The constant string to write to the buffer to
574 int __trace_bputs(unsigned long ip
, const char *str
)
576 struct ring_buffer_event
*event
;
577 struct ring_buffer
*buffer
;
578 struct bputs_entry
*entry
;
579 unsigned long irq_flags
;
580 int size
= sizeof(struct bputs_entry
);
583 if (!(global_trace
.trace_flags
& TRACE_ITER_PRINTK
))
586 pc
= preempt_count();
588 if (unlikely(tracing_selftest_running
|| tracing_disabled
))
591 local_save_flags(irq_flags
);
592 buffer
= global_trace
.trace_buffer
.buffer
;
593 event
= trace_buffer_lock_reserve(buffer
, TRACE_BPUTS
, size
,
598 entry
= ring_buffer_event_data(event
);
602 __buffer_unlock_commit(buffer
, event
);
603 ftrace_trace_stack(&global_trace
, buffer
, irq_flags
, 4, pc
, NULL
);
607 EXPORT_SYMBOL_GPL(__trace_bputs
);
609 #ifdef CONFIG_TRACER_SNAPSHOT
611 * trace_snapshot - take a snapshot of the current buffer.
613 * This causes a swap between the snapshot buffer and the current live
614 * tracing buffer. You can use this to take snapshots of the live
615 * trace when some condition is triggered, but continue to trace.
617 * Note, make sure to allocate the snapshot with either
618 * a tracing_snapshot_alloc(), or by doing it manually
619 * with: echo 1 > /sys/kernel/debug/tracing/snapshot
621 * If the snapshot buffer is not allocated, it will stop tracing.
622 * Basically making a permanent snapshot.
624 void tracing_snapshot(void)
626 struct trace_array
*tr
= &global_trace
;
627 struct tracer
*tracer
= tr
->current_trace
;
631 internal_trace_puts("*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n");
632 internal_trace_puts("*** snapshot is being ignored ***\n");
636 if (!tr
->allocated_snapshot
) {
637 internal_trace_puts("*** SNAPSHOT NOT ALLOCATED ***\n");
638 internal_trace_puts("*** stopping trace here! ***\n");
643 /* Note, snapshot can not be used when the tracer uses it */
644 if (tracer
->use_max_tr
) {
645 internal_trace_puts("*** LATENCY TRACER ACTIVE ***\n");
646 internal_trace_puts("*** Can not use snapshot (sorry) ***\n");
650 local_irq_save(flags
);
651 update_max_tr(tr
, current
, smp_processor_id());
652 local_irq_restore(flags
);
654 EXPORT_SYMBOL_GPL(tracing_snapshot
);
656 static int resize_buffer_duplicate_size(struct trace_buffer
*trace_buf
,
657 struct trace_buffer
*size_buf
, int cpu_id
);
658 static void set_buffer_entries(struct trace_buffer
*buf
, unsigned long val
);
660 static int alloc_snapshot(struct trace_array
*tr
)
664 if (!tr
->allocated_snapshot
) {
666 /* allocate spare buffer */
667 ret
= resize_buffer_duplicate_size(&tr
->max_buffer
,
668 &tr
->trace_buffer
, RING_BUFFER_ALL_CPUS
);
672 tr
->allocated_snapshot
= true;
678 static void free_snapshot(struct trace_array
*tr
)
681 * We don't free the ring buffer. instead, resize it because
682 * The max_tr ring buffer has some state (e.g. ring->clock) and
683 * we want preserve it.
685 ring_buffer_resize(tr
->max_buffer
.buffer
, 1, RING_BUFFER_ALL_CPUS
);
686 set_buffer_entries(&tr
->max_buffer
, 1);
687 tracing_reset_online_cpus(&tr
->max_buffer
);
688 tr
->allocated_snapshot
= false;
692 * tracing_alloc_snapshot - allocate snapshot buffer.
694 * This only allocates the snapshot buffer if it isn't already
695 * allocated - it doesn't also take a snapshot.
697 * This is meant to be used in cases where the snapshot buffer needs
698 * to be set up for events that can't sleep but need to be able to
699 * trigger a snapshot.
701 int tracing_alloc_snapshot(void)
703 struct trace_array
*tr
= &global_trace
;
706 ret
= alloc_snapshot(tr
);
711 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot
);
714 * trace_snapshot_alloc - allocate and take a snapshot of the current buffer.
716 * This is similar to trace_snapshot(), but it will allocate the
717 * snapshot buffer if it isn't already allocated. Use this only
718 * where it is safe to sleep, as the allocation may sleep.
720 * This causes a swap between the snapshot buffer and the current live
721 * tracing buffer. You can use this to take snapshots of the live
722 * trace when some condition is triggered, but continue to trace.
724 void tracing_snapshot_alloc(void)
728 ret
= tracing_alloc_snapshot();
734 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc
);
736 void tracing_snapshot(void)
738 WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used");
740 EXPORT_SYMBOL_GPL(tracing_snapshot
);
741 int tracing_alloc_snapshot(void)
743 WARN_ONCE(1, "Snapshot feature not enabled, but snapshot allocation used");
746 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot
);
747 void tracing_snapshot_alloc(void)
752 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc
);
753 #endif /* CONFIG_TRACER_SNAPSHOT */
755 static void tracer_tracing_off(struct trace_array
*tr
)
757 if (tr
->trace_buffer
.buffer
)
758 ring_buffer_record_off(tr
->trace_buffer
.buffer
);
760 * This flag is looked at when buffers haven't been allocated
761 * yet, or by some tracers (like irqsoff), that just want to
762 * know if the ring buffer has been disabled, but it can handle
763 * races of where it gets disabled but we still do a record.
764 * As the check is in the fast path of the tracers, it is more
765 * important to be fast than accurate.
767 tr
->buffer_disabled
= 1;
768 /* Make the flag seen by readers */
773 * tracing_off - turn off tracing buffers
775 * This function stops the tracing buffers from recording data.
776 * It does not disable any overhead the tracers themselves may
777 * be causing. This function simply causes all recording to
778 * the ring buffers to fail.
780 void tracing_off(void)
782 tracer_tracing_off(&global_trace
);
784 EXPORT_SYMBOL_GPL(tracing_off
);
786 void disable_trace_on_warning(void)
788 if (__disable_trace_on_warning
)
793 * tracer_tracing_is_on - show real state of ring buffer enabled
794 * @tr : the trace array to know if ring buffer is enabled
796 * Shows real state of the ring buffer if it is enabled or not.
798 static int tracer_tracing_is_on(struct trace_array
*tr
)
800 if (tr
->trace_buffer
.buffer
)
801 return ring_buffer_record_is_on(tr
->trace_buffer
.buffer
);
802 return !tr
->buffer_disabled
;
806 * tracing_is_on - show state of ring buffers enabled
808 int tracing_is_on(void)
810 return tracer_tracing_is_on(&global_trace
);
812 EXPORT_SYMBOL_GPL(tracing_is_on
);
814 static int __init
set_buf_size(char *str
)
816 unsigned long buf_size
;
820 buf_size
= memparse(str
, &str
);
821 /* nr_entries can not be zero */
824 trace_buf_size
= buf_size
;
827 __setup("trace_buf_size=", set_buf_size
);
829 static int __init
set_tracing_thresh(char *str
)
831 unsigned long threshold
;
836 ret
= kstrtoul(str
, 0, &threshold
);
839 tracing_thresh
= threshold
* 1000;
842 __setup("tracing_thresh=", set_tracing_thresh
);
844 unsigned long nsecs_to_usecs(unsigned long nsecs
)
850 * TRACE_FLAGS is defined as a tuple matching bit masks with strings.
851 * It uses C(a, b) where 'a' is the enum name and 'b' is the string that
852 * matches it. By defining "C(a, b) b", TRACE_FLAGS becomes a list
853 * of strings in the order that the enums were defined.
858 /* These must match the bit postions in trace_iterator_flags */
859 static const char *trace_options
[] = {
867 int in_ns
; /* is this clock in nanoseconds? */
869 { trace_clock_local
, "local", 1 },
870 { trace_clock_global
, "global", 1 },
871 { trace_clock_counter
, "counter", 0 },
872 { trace_clock_jiffies
, "uptime", 0 },
873 { trace_clock
, "perf", 1 },
874 { ktime_get_mono_fast_ns
, "mono", 1 },
875 { ktime_get_raw_fast_ns
, "mono_raw", 1 },
880 * trace_parser_get_init - gets the buffer for trace parser
882 int trace_parser_get_init(struct trace_parser
*parser
, int size
)
884 memset(parser
, 0, sizeof(*parser
));
886 parser
->buffer
= kmalloc(size
, GFP_KERNEL
);
895 * trace_parser_put - frees the buffer for trace parser
897 void trace_parser_put(struct trace_parser
*parser
)
899 kfree(parser
->buffer
);
903 * trace_get_user - reads the user input string separated by space
904 * (matched by isspace(ch))
906 * For each string found the 'struct trace_parser' is updated,
907 * and the function returns.
909 * Returns number of bytes read.
911 * See kernel/trace/trace.h for 'struct trace_parser' details.
913 int trace_get_user(struct trace_parser
*parser
, const char __user
*ubuf
,
914 size_t cnt
, loff_t
*ppos
)
921 trace_parser_clear(parser
);
923 ret
= get_user(ch
, ubuf
++);
931 * The parser is not finished with the last write,
932 * continue reading the user input without skipping spaces.
935 /* skip white space */
936 while (cnt
&& isspace(ch
)) {
937 ret
= get_user(ch
, ubuf
++);
944 /* only spaces were written */
954 /* read the non-space input */
955 while (cnt
&& !isspace(ch
)) {
956 if (parser
->idx
< parser
->size
- 1)
957 parser
->buffer
[parser
->idx
++] = ch
;
962 ret
= get_user(ch
, ubuf
++);
969 /* We either got finished input or we have to wait for another call. */
971 parser
->buffer
[parser
->idx
] = 0;
972 parser
->cont
= false;
973 } else if (parser
->idx
< parser
->size
- 1) {
975 parser
->buffer
[parser
->idx
++] = ch
;
988 /* TODO add a seq_buf_to_buffer() */
989 static ssize_t
trace_seq_to_buffer(struct trace_seq
*s
, void *buf
, size_t cnt
)
993 if (trace_seq_used(s
) <= s
->seq
.readpos
)
996 len
= trace_seq_used(s
) - s
->seq
.readpos
;
999 memcpy(buf
, s
->buffer
+ s
->seq
.readpos
, cnt
);
1001 s
->seq
.readpos
+= cnt
;
1005 unsigned long __read_mostly tracing_thresh
;
1007 #ifdef CONFIG_TRACER_MAX_TRACE
1009 * Copy the new maximum trace into the separate maximum-trace
1010 * structure. (this way the maximum trace is permanently saved,
1011 * for later retrieval via /sys/kernel/debug/tracing/latency_trace)
1014 __update_max_tr(struct trace_array
*tr
, struct task_struct
*tsk
, int cpu
)
1016 struct trace_buffer
*trace_buf
= &tr
->trace_buffer
;
1017 struct trace_buffer
*max_buf
= &tr
->max_buffer
;
1018 struct trace_array_cpu
*data
= per_cpu_ptr(trace_buf
->data
, cpu
);
1019 struct trace_array_cpu
*max_data
= per_cpu_ptr(max_buf
->data
, cpu
);
1022 max_buf
->time_start
= data
->preempt_timestamp
;
1024 max_data
->saved_latency
= tr
->max_latency
;
1025 max_data
->critical_start
= data
->critical_start
;
1026 max_data
->critical_end
= data
->critical_end
;
1028 memcpy(max_data
->comm
, tsk
->comm
, TASK_COMM_LEN
);
1029 max_data
->pid
= tsk
->pid
;
1031 * If tsk == current, then use current_uid(), as that does not use
1032 * RCU. The irq tracer can be called out of RCU scope.
1035 max_data
->uid
= current_uid();
1037 max_data
->uid
= task_uid(tsk
);
1039 max_data
->nice
= tsk
->static_prio
- 20 - MAX_RT_PRIO
;
1040 max_data
->policy
= tsk
->policy
;
1041 max_data
->rt_priority
= tsk
->rt_priority
;
1043 /* record this tasks comm */
1044 tracing_record_cmdline(tsk
);
1048 * update_max_tr - snapshot all trace buffers from global_trace to max_tr
1050 * @tsk: the task with the latency
1051 * @cpu: The cpu that initiated the trace.
1053 * Flip the buffers between the @tr and the max_tr and record information
1054 * about which task was the cause of this latency.
1057 update_max_tr(struct trace_array
*tr
, struct task_struct
*tsk
, int cpu
)
1059 struct ring_buffer
*buf
;
1064 WARN_ON_ONCE(!irqs_disabled());
1066 if (!tr
->allocated_snapshot
) {
1067 /* Only the nop tracer should hit this when disabling */
1068 WARN_ON_ONCE(tr
->current_trace
!= &nop_trace
);
1072 arch_spin_lock(&tr
->max_lock
);
1074 buf
= tr
->trace_buffer
.buffer
;
1075 tr
->trace_buffer
.buffer
= tr
->max_buffer
.buffer
;
1076 tr
->max_buffer
.buffer
= buf
;
1078 __update_max_tr(tr
, tsk
, cpu
);
1079 arch_spin_unlock(&tr
->max_lock
);
1083 * update_max_tr_single - only copy one trace over, and reset the rest
1085 * @tsk - task with the latency
1086 * @cpu - the cpu of the buffer to copy.
1088 * Flip the trace of a single CPU buffer between the @tr and the max_tr.
1091 update_max_tr_single(struct trace_array
*tr
, struct task_struct
*tsk
, int cpu
)
1098 WARN_ON_ONCE(!irqs_disabled());
1099 if (!tr
->allocated_snapshot
) {
1100 /* Only the nop tracer should hit this when disabling */
1101 WARN_ON_ONCE(tr
->current_trace
!= &nop_trace
);
1105 arch_spin_lock(&tr
->max_lock
);
1107 ret
= ring_buffer_swap_cpu(tr
->max_buffer
.buffer
, tr
->trace_buffer
.buffer
, cpu
);
1109 if (ret
== -EBUSY
) {
1111 * We failed to swap the buffer due to a commit taking
1112 * place on this CPU. We fail to record, but we reset
1113 * the max trace buffer (no one writes directly to it)
1114 * and flag that it failed.
1116 trace_array_printk_buf(tr
->max_buffer
.buffer
, _THIS_IP_
,
1117 "Failed to swap buffers due to commit in progress\n");
1120 WARN_ON_ONCE(ret
&& ret
!= -EAGAIN
&& ret
!= -EBUSY
);
1122 __update_max_tr(tr
, tsk
, cpu
);
1123 arch_spin_unlock(&tr
->max_lock
);
1125 #endif /* CONFIG_TRACER_MAX_TRACE */
1127 static int wait_on_pipe(struct trace_iterator
*iter
, bool full
)
1129 /* Iterators are static, they should be filled or empty */
1130 if (trace_buffer_iter(iter
, iter
->cpu_file
))
1133 return ring_buffer_wait(iter
->trace_buffer
->buffer
, iter
->cpu_file
,
1137 #ifdef CONFIG_FTRACE_STARTUP_TEST
1138 static int run_tracer_selftest(struct tracer
*type
)
1140 struct trace_array
*tr
= &global_trace
;
1141 struct tracer
*saved_tracer
= tr
->current_trace
;
1144 if (!type
->selftest
|| tracing_selftest_disabled
)
1148 * Run a selftest on this tracer.
1149 * Here we reset the trace buffer, and set the current
1150 * tracer to be this tracer. The tracer can then run some
1151 * internal tracing to verify that everything is in order.
1152 * If we fail, we do not register this tracer.
1154 tracing_reset_online_cpus(&tr
->trace_buffer
);
1156 tr
->current_trace
= type
;
1158 #ifdef CONFIG_TRACER_MAX_TRACE
1159 if (type
->use_max_tr
) {
1160 /* If we expanded the buffers, make sure the max is expanded too */
1161 if (ring_buffer_expanded
)
1162 ring_buffer_resize(tr
->max_buffer
.buffer
, trace_buf_size
,
1163 RING_BUFFER_ALL_CPUS
);
1164 tr
->allocated_snapshot
= true;
1168 /* the test is responsible for initializing and enabling */
1169 pr_info("Testing tracer %s: ", type
->name
);
1170 ret
= type
->selftest(type
, tr
);
1171 /* the test is responsible for resetting too */
1172 tr
->current_trace
= saved_tracer
;
1174 printk(KERN_CONT
"FAILED!\n");
1175 /* Add the warning after printing 'FAILED' */
1179 /* Only reset on passing, to avoid touching corrupted buffers */
1180 tracing_reset_online_cpus(&tr
->trace_buffer
);
1182 #ifdef CONFIG_TRACER_MAX_TRACE
1183 if (type
->use_max_tr
) {
1184 tr
->allocated_snapshot
= false;
1186 /* Shrink the max buffer again */
1187 if (ring_buffer_expanded
)
1188 ring_buffer_resize(tr
->max_buffer
.buffer
, 1,
1189 RING_BUFFER_ALL_CPUS
);
1193 printk(KERN_CONT
"PASSED\n");
1197 static inline int run_tracer_selftest(struct tracer
*type
)
1201 #endif /* CONFIG_FTRACE_STARTUP_TEST */
1203 static void add_tracer_options(struct trace_array
*tr
, struct tracer
*t
);
1205 static void __init
apply_trace_boot_options(void);
1208 * register_tracer - register a tracer with the ftrace system.
1209 * @type - the plugin for the tracer
1211 * Register a new plugin tracer.
1213 int __init
register_tracer(struct tracer
*type
)
1219 pr_info("Tracer must have a name\n");
1223 if (strlen(type
->name
) >= MAX_TRACER_SIZE
) {
1224 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE
);
1228 mutex_lock(&trace_types_lock
);
1230 tracing_selftest_running
= true;
1232 for (t
= trace_types
; t
; t
= t
->next
) {
1233 if (strcmp(type
->name
, t
->name
) == 0) {
1235 pr_info("Tracer %s already registered\n",
1242 if (!type
->set_flag
)
1243 type
->set_flag
= &dummy_set_flag
;
1245 /*allocate a dummy tracer_flags*/
1246 type
->flags
= kmalloc(sizeof(*type
->flags
), GFP_KERNEL
);
1251 type
->flags
->val
= 0;
1252 type
->flags
->opts
= dummy_tracer_opt
;
1254 if (!type
->flags
->opts
)
1255 type
->flags
->opts
= dummy_tracer_opt
;
1257 /* store the tracer for __set_tracer_option */
1258 type
->flags
->trace
= type
;
1260 ret
= run_tracer_selftest(type
);
1264 type
->next
= trace_types
;
1266 add_tracer_options(&global_trace
, type
);
1269 tracing_selftest_running
= false;
1270 mutex_unlock(&trace_types_lock
);
1272 if (ret
|| !default_bootup_tracer
)
1275 if (strncmp(default_bootup_tracer
, type
->name
, MAX_TRACER_SIZE
))
1278 printk(KERN_INFO
"Starting tracer '%s'\n", type
->name
);
1279 /* Do we want this tracer to start on bootup? */
1280 tracing_set_tracer(&global_trace
, type
->name
);
1281 default_bootup_tracer
= NULL
;
1283 apply_trace_boot_options();
1285 /* disable other selftests, since this will break it. */
1286 tracing_selftest_disabled
= true;
1287 #ifdef CONFIG_FTRACE_STARTUP_TEST
1288 printk(KERN_INFO
"Disabling FTRACE selftests due to running tracer '%s'\n",
1296 void tracing_reset(struct trace_buffer
*buf
, int cpu
)
1298 struct ring_buffer
*buffer
= buf
->buffer
;
1303 ring_buffer_record_disable(buffer
);
1305 /* Make sure all commits have finished */
1306 synchronize_sched();
1307 ring_buffer_reset_cpu(buffer
, cpu
);
1309 ring_buffer_record_enable(buffer
);
1312 void tracing_reset_online_cpus(struct trace_buffer
*buf
)
1314 struct ring_buffer
*buffer
= buf
->buffer
;
1320 ring_buffer_record_disable(buffer
);
1322 /* Make sure all commits have finished */
1323 synchronize_sched();
1325 buf
->time_start
= buffer_ftrace_now(buf
, buf
->cpu
);
1327 for_each_online_cpu(cpu
)
1328 ring_buffer_reset_cpu(buffer
, cpu
);
1330 ring_buffer_record_enable(buffer
);
1333 /* Must have trace_types_lock held */
1334 void tracing_reset_all_online_cpus(void)
1336 struct trace_array
*tr
;
1338 list_for_each_entry(tr
, &ftrace_trace_arrays
, list
) {
1339 tracing_reset_online_cpus(&tr
->trace_buffer
);
1340 #ifdef CONFIG_TRACER_MAX_TRACE
1341 tracing_reset_online_cpus(&tr
->max_buffer
);
1346 #define SAVED_CMDLINES_DEFAULT 128
1347 #define NO_CMDLINE_MAP UINT_MAX
1348 static arch_spinlock_t trace_cmdline_lock
= __ARCH_SPIN_LOCK_UNLOCKED
;
1349 struct saved_cmdlines_buffer
{
1350 unsigned map_pid_to_cmdline
[PID_MAX_DEFAULT
+1];
1351 unsigned *map_cmdline_to_pid
;
1352 unsigned cmdline_num
;
1354 char *saved_cmdlines
;
1356 static struct saved_cmdlines_buffer
*savedcmd
;
1358 /* temporary disable recording */
1359 static atomic_t trace_record_cmdline_disabled __read_mostly
;
1361 static inline char *get_saved_cmdlines(int idx
)
1363 return &savedcmd
->saved_cmdlines
[idx
* TASK_COMM_LEN
];
1366 static inline void set_cmdline(int idx
, const char *cmdline
)
1368 memcpy(get_saved_cmdlines(idx
), cmdline
, TASK_COMM_LEN
);
1371 static int allocate_cmdlines_buffer(unsigned int val
,
1372 struct saved_cmdlines_buffer
*s
)
1374 s
->map_cmdline_to_pid
= kmalloc(val
* sizeof(*s
->map_cmdline_to_pid
),
1376 if (!s
->map_cmdline_to_pid
)
1379 s
->saved_cmdlines
= kmalloc(val
* TASK_COMM_LEN
, GFP_KERNEL
);
1380 if (!s
->saved_cmdlines
) {
1381 kfree(s
->map_cmdline_to_pid
);
1386 s
->cmdline_num
= val
;
1387 memset(&s
->map_pid_to_cmdline
, NO_CMDLINE_MAP
,
1388 sizeof(s
->map_pid_to_cmdline
));
1389 memset(s
->map_cmdline_to_pid
, NO_CMDLINE_MAP
,
1390 val
* sizeof(*s
->map_cmdline_to_pid
));
1395 static int trace_create_savedcmd(void)
1399 savedcmd
= kmalloc(sizeof(*savedcmd
), GFP_KERNEL
);
1403 ret
= allocate_cmdlines_buffer(SAVED_CMDLINES_DEFAULT
, savedcmd
);
1413 int is_tracing_stopped(void)
1415 return global_trace
.stop_count
;
1419 * tracing_start - quick start of the tracer
1421 * If tracing is enabled but was stopped by tracing_stop,
1422 * this will start the tracer back up.
1424 void tracing_start(void)
1426 struct ring_buffer
*buffer
;
1427 unsigned long flags
;
1429 if (tracing_disabled
)
1432 raw_spin_lock_irqsave(&global_trace
.start_lock
, flags
);
1433 if (--global_trace
.stop_count
) {
1434 if (global_trace
.stop_count
< 0) {
1435 /* Someone screwed up their debugging */
1437 global_trace
.stop_count
= 0;
1442 /* Prevent the buffers from switching */
1443 arch_spin_lock(&global_trace
.max_lock
);
1445 buffer
= global_trace
.trace_buffer
.buffer
;
1447 ring_buffer_record_enable(buffer
);
1449 #ifdef CONFIG_TRACER_MAX_TRACE
1450 buffer
= global_trace
.max_buffer
.buffer
;
1452 ring_buffer_record_enable(buffer
);
1455 arch_spin_unlock(&global_trace
.max_lock
);
1458 raw_spin_unlock_irqrestore(&global_trace
.start_lock
, flags
);
1461 static void tracing_start_tr(struct trace_array
*tr
)
1463 struct ring_buffer
*buffer
;
1464 unsigned long flags
;
1466 if (tracing_disabled
)
1469 /* If global, we need to also start the max tracer */
1470 if (tr
->flags
& TRACE_ARRAY_FL_GLOBAL
)
1471 return tracing_start();
1473 raw_spin_lock_irqsave(&tr
->start_lock
, flags
);
1475 if (--tr
->stop_count
) {
1476 if (tr
->stop_count
< 0) {
1477 /* Someone screwed up their debugging */
1484 buffer
= tr
->trace_buffer
.buffer
;
1486 ring_buffer_record_enable(buffer
);
1489 raw_spin_unlock_irqrestore(&tr
->start_lock
, flags
);
1493 * tracing_stop - quick stop of the tracer
1495 * Light weight way to stop tracing. Use in conjunction with
1498 void tracing_stop(void)
1500 struct ring_buffer
*buffer
;
1501 unsigned long flags
;
1503 raw_spin_lock_irqsave(&global_trace
.start_lock
, flags
);
1504 if (global_trace
.stop_count
++)
1507 /* Prevent the buffers from switching */
1508 arch_spin_lock(&global_trace
.max_lock
);
1510 buffer
= global_trace
.trace_buffer
.buffer
;
1512 ring_buffer_record_disable(buffer
);
1514 #ifdef CONFIG_TRACER_MAX_TRACE
1515 buffer
= global_trace
.max_buffer
.buffer
;
1517 ring_buffer_record_disable(buffer
);
1520 arch_spin_unlock(&global_trace
.max_lock
);
1523 raw_spin_unlock_irqrestore(&global_trace
.start_lock
, flags
);
1526 static void tracing_stop_tr(struct trace_array
*tr
)
1528 struct ring_buffer
*buffer
;
1529 unsigned long flags
;
1531 /* If global, we need to also stop the max tracer */
1532 if (tr
->flags
& TRACE_ARRAY_FL_GLOBAL
)
1533 return tracing_stop();
1535 raw_spin_lock_irqsave(&tr
->start_lock
, flags
);
1536 if (tr
->stop_count
++)
1539 buffer
= tr
->trace_buffer
.buffer
;
1541 ring_buffer_record_disable(buffer
);
1544 raw_spin_unlock_irqrestore(&tr
->start_lock
, flags
);
1547 void trace_stop_cmdline_recording(void);
1549 static int trace_save_cmdline(struct task_struct
*tsk
)
1553 if (!tsk
->pid
|| unlikely(tsk
->pid
> PID_MAX_DEFAULT
))
1557 * It's not the end of the world if we don't get
1558 * the lock, but we also don't want to spin
1559 * nor do we want to disable interrupts,
1560 * so if we miss here, then better luck next time.
1562 if (!arch_spin_trylock(&trace_cmdline_lock
))
1565 idx
= savedcmd
->map_pid_to_cmdline
[tsk
->pid
];
1566 if (idx
== NO_CMDLINE_MAP
) {
1567 idx
= (savedcmd
->cmdline_idx
+ 1) % savedcmd
->cmdline_num
;
1570 * Check whether the cmdline buffer at idx has a pid
1571 * mapped. We are going to overwrite that entry so we
1572 * need to clear the map_pid_to_cmdline. Otherwise we
1573 * would read the new comm for the old pid.
1575 pid
= savedcmd
->map_cmdline_to_pid
[idx
];
1576 if (pid
!= NO_CMDLINE_MAP
)
1577 savedcmd
->map_pid_to_cmdline
[pid
] = NO_CMDLINE_MAP
;
1579 savedcmd
->map_cmdline_to_pid
[idx
] = tsk
->pid
;
1580 savedcmd
->map_pid_to_cmdline
[tsk
->pid
] = idx
;
1582 savedcmd
->cmdline_idx
= idx
;
1585 set_cmdline(idx
, tsk
->comm
);
1587 arch_spin_unlock(&trace_cmdline_lock
);
1592 static void __trace_find_cmdline(int pid
, char comm
[])
1597 strcpy(comm
, "<idle>");
1601 if (WARN_ON_ONCE(pid
< 0)) {
1602 strcpy(comm
, "<XXX>");
1606 if (pid
> PID_MAX_DEFAULT
) {
1607 strcpy(comm
, "<...>");
1611 map
= savedcmd
->map_pid_to_cmdline
[pid
];
1612 if (map
!= NO_CMDLINE_MAP
)
1613 strcpy(comm
, get_saved_cmdlines(map
));
1615 strcpy(comm
, "<...>");
1618 void trace_find_cmdline(int pid
, char comm
[])
1621 arch_spin_lock(&trace_cmdline_lock
);
1623 __trace_find_cmdline(pid
, comm
);
1625 arch_spin_unlock(&trace_cmdline_lock
);
1629 void tracing_record_cmdline(struct task_struct
*tsk
)
1631 if (atomic_read(&trace_record_cmdline_disabled
) || !tracing_is_on())
1634 if (!__this_cpu_read(trace_cmdline_save
))
1637 if (trace_save_cmdline(tsk
))
1638 __this_cpu_write(trace_cmdline_save
, false);
1642 tracing_generic_entry_update(struct trace_entry
*entry
, unsigned long flags
,
1645 struct task_struct
*tsk
= current
;
1647 entry
->preempt_count
= pc
& 0xff;
1648 entry
->pid
= (tsk
) ? tsk
->pid
: 0;
1650 #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
1651 (irqs_disabled_flags(flags
) ? TRACE_FLAG_IRQS_OFF
: 0) |
1653 TRACE_FLAG_IRQS_NOSUPPORT
|
1655 ((pc
& NMI_MASK
) ? TRACE_FLAG_NMI
: 0) |
1656 ((pc
& HARDIRQ_MASK
) ? TRACE_FLAG_HARDIRQ
: 0) |
1657 ((pc
& SOFTIRQ_MASK
) ? TRACE_FLAG_SOFTIRQ
: 0) |
1658 (tif_need_resched() ? TRACE_FLAG_NEED_RESCHED
: 0) |
1659 (test_preempt_need_resched() ? TRACE_FLAG_PREEMPT_RESCHED
: 0);
1661 EXPORT_SYMBOL_GPL(tracing_generic_entry_update
);
1663 struct ring_buffer_event
*
1664 trace_buffer_lock_reserve(struct ring_buffer
*buffer
,
1667 unsigned long flags
, int pc
)
1669 struct ring_buffer_event
*event
;
1671 event
= ring_buffer_lock_reserve(buffer
, len
);
1672 if (event
!= NULL
) {
1673 struct trace_entry
*ent
= ring_buffer_event_data(event
);
1675 tracing_generic_entry_update(ent
, flags
, pc
);
1683 __buffer_unlock_commit(struct ring_buffer
*buffer
, struct ring_buffer_event
*event
)
1685 __this_cpu_write(trace_cmdline_save
, true);
1686 ring_buffer_unlock_commit(buffer
, event
);
1689 void trace_buffer_unlock_commit(struct trace_array
*tr
,
1690 struct ring_buffer
*buffer
,
1691 struct ring_buffer_event
*event
,
1692 unsigned long flags
, int pc
)
1694 __buffer_unlock_commit(buffer
, event
);
1696 ftrace_trace_stack(tr
, buffer
, flags
, 6, pc
, NULL
);
1697 ftrace_trace_userstack(buffer
, flags
, pc
);
1700 static struct ring_buffer
*temp_buffer
;
1702 struct ring_buffer_event
*
1703 trace_event_buffer_lock_reserve(struct ring_buffer
**current_rb
,
1704 struct trace_event_file
*trace_file
,
1705 int type
, unsigned long len
,
1706 unsigned long flags
, int pc
)
1708 struct ring_buffer_event
*entry
;
1710 *current_rb
= trace_file
->tr
->trace_buffer
.buffer
;
1711 entry
= trace_buffer_lock_reserve(*current_rb
,
1712 type
, len
, flags
, pc
);
1714 * If tracing is off, but we have triggers enabled
1715 * we still need to look at the event data. Use the temp_buffer
1716 * to store the trace event for the tigger to use. It's recusive
1717 * safe and will not be recorded anywhere.
1719 if (!entry
&& trace_file
->flags
& EVENT_FILE_FL_TRIGGER_COND
) {
1720 *current_rb
= temp_buffer
;
1721 entry
= trace_buffer_lock_reserve(*current_rb
,
1722 type
, len
, flags
, pc
);
1726 EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve
);
1728 struct ring_buffer_event
*
1729 trace_current_buffer_lock_reserve(struct ring_buffer
**current_rb
,
1730 int type
, unsigned long len
,
1731 unsigned long flags
, int pc
)
1733 *current_rb
= global_trace
.trace_buffer
.buffer
;
1734 return trace_buffer_lock_reserve(*current_rb
,
1735 type
, len
, flags
, pc
);
1737 EXPORT_SYMBOL_GPL(trace_current_buffer_lock_reserve
);
1739 void trace_buffer_unlock_commit_regs(struct trace_array
*tr
,
1740 struct ring_buffer
*buffer
,
1741 struct ring_buffer_event
*event
,
1742 unsigned long flags
, int pc
,
1743 struct pt_regs
*regs
)
1745 __buffer_unlock_commit(buffer
, event
);
1747 ftrace_trace_stack(tr
, buffer
, flags
, 0, pc
, regs
);
1748 ftrace_trace_userstack(buffer
, flags
, pc
);
1751 void trace_current_buffer_discard_commit(struct ring_buffer
*buffer
,
1752 struct ring_buffer_event
*event
)
1754 ring_buffer_discard_commit(buffer
, event
);
1756 EXPORT_SYMBOL_GPL(trace_current_buffer_discard_commit
);
1759 trace_function(struct trace_array
*tr
,
1760 unsigned long ip
, unsigned long parent_ip
, unsigned long flags
,
1763 struct trace_event_call
*call
= &event_function
;
1764 struct ring_buffer
*buffer
= tr
->trace_buffer
.buffer
;
1765 struct ring_buffer_event
*event
;
1766 struct ftrace_entry
*entry
;
1768 event
= trace_buffer_lock_reserve(buffer
, TRACE_FN
, sizeof(*entry
),
1772 entry
= ring_buffer_event_data(event
);
1774 entry
->parent_ip
= parent_ip
;
1776 if (!call_filter_check_discard(call
, entry
, buffer
, event
))
1777 __buffer_unlock_commit(buffer
, event
);
1780 #ifdef CONFIG_STACKTRACE
1782 #define FTRACE_STACK_MAX_ENTRIES (PAGE_SIZE / sizeof(unsigned long))
1783 struct ftrace_stack
{
1784 unsigned long calls
[FTRACE_STACK_MAX_ENTRIES
];
1787 static DEFINE_PER_CPU(struct ftrace_stack
, ftrace_stack
);
1788 static DEFINE_PER_CPU(int, ftrace_stack_reserve
);
1790 static void __ftrace_trace_stack(struct ring_buffer
*buffer
,
1791 unsigned long flags
,
1792 int skip
, int pc
, struct pt_regs
*regs
)
1794 struct trace_event_call
*call
= &event_kernel_stack
;
1795 struct ring_buffer_event
*event
;
1796 struct stack_entry
*entry
;
1797 struct stack_trace trace
;
1799 int size
= FTRACE_STACK_ENTRIES
;
1801 trace
.nr_entries
= 0;
1805 * Since events can happen in NMIs there's no safe way to
1806 * use the per cpu ftrace_stacks. We reserve it and if an interrupt
1807 * or NMI comes in, it will just have to use the default
1808 * FTRACE_STACK_SIZE.
1810 preempt_disable_notrace();
1812 use_stack
= __this_cpu_inc_return(ftrace_stack_reserve
);
1814 * We don't need any atomic variables, just a barrier.
1815 * If an interrupt comes in, we don't care, because it would
1816 * have exited and put the counter back to what we want.
1817 * We just need a barrier to keep gcc from moving things
1821 if (use_stack
== 1) {
1822 trace
.entries
= this_cpu_ptr(ftrace_stack
.calls
);
1823 trace
.max_entries
= FTRACE_STACK_MAX_ENTRIES
;
1826 save_stack_trace_regs(regs
, &trace
);
1828 save_stack_trace(&trace
);
1830 if (trace
.nr_entries
> size
)
1831 size
= trace
.nr_entries
;
1833 /* From now on, use_stack is a boolean */
1836 size
*= sizeof(unsigned long);
1838 event
= trace_buffer_lock_reserve(buffer
, TRACE_STACK
,
1839 sizeof(*entry
) + size
, flags
, pc
);
1842 entry
= ring_buffer_event_data(event
);
1844 memset(&entry
->caller
, 0, size
);
1847 memcpy(&entry
->caller
, trace
.entries
,
1848 trace
.nr_entries
* sizeof(unsigned long));
1850 trace
.max_entries
= FTRACE_STACK_ENTRIES
;
1851 trace
.entries
= entry
->caller
;
1853 save_stack_trace_regs(regs
, &trace
);
1855 save_stack_trace(&trace
);
1858 entry
->size
= trace
.nr_entries
;
1860 if (!call_filter_check_discard(call
, entry
, buffer
, event
))
1861 __buffer_unlock_commit(buffer
, event
);
1864 /* Again, don't let gcc optimize things here */
1866 __this_cpu_dec(ftrace_stack_reserve
);
1867 preempt_enable_notrace();
1871 static inline void ftrace_trace_stack(struct trace_array
*tr
,
1872 struct ring_buffer
*buffer
,
1873 unsigned long flags
,
1874 int skip
, int pc
, struct pt_regs
*regs
)
1876 if (!(tr
->trace_flags
& TRACE_ITER_STACKTRACE
))
1879 __ftrace_trace_stack(buffer
, flags
, skip
, pc
, regs
);
1882 void __trace_stack(struct trace_array
*tr
, unsigned long flags
, int skip
,
1885 __ftrace_trace_stack(tr
->trace_buffer
.buffer
, flags
, skip
, pc
, NULL
);
1889 * trace_dump_stack - record a stack back trace in the trace buffer
1890 * @skip: Number of functions to skip (helper handlers)
1892 void trace_dump_stack(int skip
)
1894 unsigned long flags
;
1896 if (tracing_disabled
|| tracing_selftest_running
)
1899 local_save_flags(flags
);
1902 * Skip 3 more, seems to get us at the caller of
1906 __ftrace_trace_stack(global_trace
.trace_buffer
.buffer
,
1907 flags
, skip
, preempt_count(), NULL
);
1910 static DEFINE_PER_CPU(int, user_stack_count
);
1913 ftrace_trace_userstack(struct ring_buffer
*buffer
, unsigned long flags
, int pc
)
1915 struct trace_event_call
*call
= &event_user_stack
;
1916 struct ring_buffer_event
*event
;
1917 struct userstack_entry
*entry
;
1918 struct stack_trace trace
;
1920 if (!(global_trace
.trace_flags
& TRACE_ITER_USERSTACKTRACE
))
1924 * NMIs can not handle page faults, even with fix ups.
1925 * The save user stack can (and often does) fault.
1927 if (unlikely(in_nmi()))
1931 * prevent recursion, since the user stack tracing may
1932 * trigger other kernel events.
1935 if (__this_cpu_read(user_stack_count
))
1938 __this_cpu_inc(user_stack_count
);
1940 event
= trace_buffer_lock_reserve(buffer
, TRACE_USER_STACK
,
1941 sizeof(*entry
), flags
, pc
);
1943 goto out_drop_count
;
1944 entry
= ring_buffer_event_data(event
);
1946 entry
->tgid
= current
->tgid
;
1947 memset(&entry
->caller
, 0, sizeof(entry
->caller
));
1949 trace
.nr_entries
= 0;
1950 trace
.max_entries
= FTRACE_STACK_ENTRIES
;
1952 trace
.entries
= entry
->caller
;
1954 save_stack_trace_user(&trace
);
1955 if (!call_filter_check_discard(call
, entry
, buffer
, event
))
1956 __buffer_unlock_commit(buffer
, event
);
1959 __this_cpu_dec(user_stack_count
);
1965 static void __trace_userstack(struct trace_array
*tr
, unsigned long flags
)
1967 ftrace_trace_userstack(tr
, flags
, preempt_count());
1971 #endif /* CONFIG_STACKTRACE */
1973 /* created for use with alloc_percpu */
1974 struct trace_buffer_struct
{
1975 char buffer
[TRACE_BUF_SIZE
];
1978 static struct trace_buffer_struct
*trace_percpu_buffer
;
1979 static struct trace_buffer_struct
*trace_percpu_sirq_buffer
;
1980 static struct trace_buffer_struct
*trace_percpu_irq_buffer
;
1981 static struct trace_buffer_struct
*trace_percpu_nmi_buffer
;
1984 * The buffer used is dependent on the context. There is a per cpu
1985 * buffer for normal context, softirq contex, hard irq context and
1986 * for NMI context. Thise allows for lockless recording.
1988 * Note, if the buffers failed to be allocated, then this returns NULL
1990 static char *get_trace_buf(void)
1992 struct trace_buffer_struct
*percpu_buffer
;
1995 * If we have allocated per cpu buffers, then we do not
1996 * need to do any locking.
1999 percpu_buffer
= trace_percpu_nmi_buffer
;
2001 percpu_buffer
= trace_percpu_irq_buffer
;
2002 else if (in_softirq())
2003 percpu_buffer
= trace_percpu_sirq_buffer
;
2005 percpu_buffer
= trace_percpu_buffer
;
2010 return this_cpu_ptr(&percpu_buffer
->buffer
[0]);
2013 static int alloc_percpu_trace_buffer(void)
2015 struct trace_buffer_struct
*buffers
;
2016 struct trace_buffer_struct
*sirq_buffers
;
2017 struct trace_buffer_struct
*irq_buffers
;
2018 struct trace_buffer_struct
*nmi_buffers
;
2020 buffers
= alloc_percpu(struct trace_buffer_struct
);
2024 sirq_buffers
= alloc_percpu(struct trace_buffer_struct
);
2028 irq_buffers
= alloc_percpu(struct trace_buffer_struct
);
2032 nmi_buffers
= alloc_percpu(struct trace_buffer_struct
);
2036 trace_percpu_buffer
= buffers
;
2037 trace_percpu_sirq_buffer
= sirq_buffers
;
2038 trace_percpu_irq_buffer
= irq_buffers
;
2039 trace_percpu_nmi_buffer
= nmi_buffers
;
2044 free_percpu(irq_buffers
);
2046 free_percpu(sirq_buffers
);
2048 free_percpu(buffers
);
2050 WARN(1, "Could not allocate percpu trace_printk buffer");
2054 static int buffers_allocated
;
2056 void trace_printk_init_buffers(void)
2058 if (buffers_allocated
)
2061 if (alloc_percpu_trace_buffer())
2064 /* trace_printk() is for debug use only. Don't use it in production. */
2067 pr_warn("**********************************************************\n");
2068 pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
2070 pr_warn("** trace_printk() being used. Allocating extra memory. **\n");
2072 pr_warn("** This means that this is a DEBUG kernel and it is **\n");
2073 pr_warn("** unsafe for production use. **\n");
2075 pr_warn("** If you see this message and you are not debugging **\n");
2076 pr_warn("** the kernel, report this immediately to your vendor! **\n");
2078 pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
2079 pr_warn("**********************************************************\n");
2081 /* Expand the buffers to set size */
2082 tracing_update_buffers();
2084 buffers_allocated
= 1;
2087 * trace_printk_init_buffers() can be called by modules.
2088 * If that happens, then we need to start cmdline recording
2089 * directly here. If the global_trace.buffer is already
2090 * allocated here, then this was called by module code.
2092 if (global_trace
.trace_buffer
.buffer
)
2093 tracing_start_cmdline_record();
2096 void trace_printk_start_comm(void)
2098 /* Start tracing comms if trace printk is set */
2099 if (!buffers_allocated
)
2101 tracing_start_cmdline_record();
2104 static void trace_printk_start_stop_comm(int enabled
)
2106 if (!buffers_allocated
)
2110 tracing_start_cmdline_record();
2112 tracing_stop_cmdline_record();
2116 * trace_vbprintk - write binary msg to tracing buffer
2119 int trace_vbprintk(unsigned long ip
, const char *fmt
, va_list args
)
2121 struct trace_event_call
*call
= &event_bprint
;
2122 struct ring_buffer_event
*event
;
2123 struct ring_buffer
*buffer
;
2124 struct trace_array
*tr
= &global_trace
;
2125 struct bprint_entry
*entry
;
2126 unsigned long flags
;
2128 int len
= 0, size
, pc
;
2130 if (unlikely(tracing_selftest_running
|| tracing_disabled
))
2133 /* Don't pollute graph traces with trace_vprintk internals */
2134 pause_graph_tracing();
2136 pc
= preempt_count();
2137 preempt_disable_notrace();
2139 tbuffer
= get_trace_buf();
2145 len
= vbin_printf((u32
*)tbuffer
, TRACE_BUF_SIZE
/sizeof(int), fmt
, args
);
2147 if (len
> TRACE_BUF_SIZE
/sizeof(int) || len
< 0)
2150 local_save_flags(flags
);
2151 size
= sizeof(*entry
) + sizeof(u32
) * len
;
2152 buffer
= tr
->trace_buffer
.buffer
;
2153 event
= trace_buffer_lock_reserve(buffer
, TRACE_BPRINT
, size
,
2157 entry
= ring_buffer_event_data(event
);
2161 memcpy(entry
->buf
, tbuffer
, sizeof(u32
) * len
);
2162 if (!call_filter_check_discard(call
, entry
, buffer
, event
)) {
2163 __buffer_unlock_commit(buffer
, event
);
2164 ftrace_trace_stack(tr
, buffer
, flags
, 6, pc
, NULL
);
2168 preempt_enable_notrace();
2169 unpause_graph_tracing();
2173 EXPORT_SYMBOL_GPL(trace_vbprintk
);
2176 __trace_array_vprintk(struct ring_buffer
*buffer
,
2177 unsigned long ip
, const char *fmt
, va_list args
)
2179 struct trace_event_call
*call
= &event_print
;
2180 struct ring_buffer_event
*event
;
2181 int len
= 0, size
, pc
;
2182 struct print_entry
*entry
;
2183 unsigned long flags
;
2186 if (tracing_disabled
|| tracing_selftest_running
)
2189 /* Don't pollute graph traces with trace_vprintk internals */
2190 pause_graph_tracing();
2192 pc
= preempt_count();
2193 preempt_disable_notrace();
2196 tbuffer
= get_trace_buf();
2202 len
= vscnprintf(tbuffer
, TRACE_BUF_SIZE
, fmt
, args
);
2204 local_save_flags(flags
);
2205 size
= sizeof(*entry
) + len
+ 1;
2206 event
= trace_buffer_lock_reserve(buffer
, TRACE_PRINT
, size
,
2210 entry
= ring_buffer_event_data(event
);
2213 memcpy(&entry
->buf
, tbuffer
, len
+ 1);
2214 if (!call_filter_check_discard(call
, entry
, buffer
, event
)) {
2215 __buffer_unlock_commit(buffer
, event
);
2216 ftrace_trace_stack(&global_trace
, buffer
, flags
, 6, pc
, NULL
);
2219 preempt_enable_notrace();
2220 unpause_graph_tracing();
2225 int trace_array_vprintk(struct trace_array
*tr
,
2226 unsigned long ip
, const char *fmt
, va_list args
)
2228 return __trace_array_vprintk(tr
->trace_buffer
.buffer
, ip
, fmt
, args
);
2231 int trace_array_printk(struct trace_array
*tr
,
2232 unsigned long ip
, const char *fmt
, ...)
2237 if (!(global_trace
.trace_flags
& TRACE_ITER_PRINTK
))
2241 ret
= trace_array_vprintk(tr
, ip
, fmt
, ap
);
2246 int trace_array_printk_buf(struct ring_buffer
*buffer
,
2247 unsigned long ip
, const char *fmt
, ...)
2252 if (!(global_trace
.trace_flags
& TRACE_ITER_PRINTK
))
2256 ret
= __trace_array_vprintk(buffer
, ip
, fmt
, ap
);
2261 int trace_vprintk(unsigned long ip
, const char *fmt
, va_list args
)
2263 return trace_array_vprintk(&global_trace
, ip
, fmt
, args
);
2265 EXPORT_SYMBOL_GPL(trace_vprintk
);
2267 static void trace_iterator_increment(struct trace_iterator
*iter
)
2269 struct ring_buffer_iter
*buf_iter
= trace_buffer_iter(iter
, iter
->cpu
);
2273 ring_buffer_read(buf_iter
, NULL
);
2276 static struct trace_entry
*
2277 peek_next_entry(struct trace_iterator
*iter
, int cpu
, u64
*ts
,
2278 unsigned long *lost_events
)
2280 struct ring_buffer_event
*event
;
2281 struct ring_buffer_iter
*buf_iter
= trace_buffer_iter(iter
, cpu
);
2284 event
= ring_buffer_iter_peek(buf_iter
, ts
);
2286 event
= ring_buffer_peek(iter
->trace_buffer
->buffer
, cpu
, ts
,
2290 iter
->ent_size
= ring_buffer_event_length(event
);
2291 return ring_buffer_event_data(event
);
2297 static struct trace_entry
*
2298 __find_next_entry(struct trace_iterator
*iter
, int *ent_cpu
,
2299 unsigned long *missing_events
, u64
*ent_ts
)
2301 struct ring_buffer
*buffer
= iter
->trace_buffer
->buffer
;
2302 struct trace_entry
*ent
, *next
= NULL
;
2303 unsigned long lost_events
= 0, next_lost
= 0;
2304 int cpu_file
= iter
->cpu_file
;
2305 u64 next_ts
= 0, ts
;
2311 * If we are in a per_cpu trace file, don't bother by iterating over
2312 * all cpu and peek directly.
2314 if (cpu_file
> RING_BUFFER_ALL_CPUS
) {
2315 if (ring_buffer_empty_cpu(buffer
, cpu_file
))
2317 ent
= peek_next_entry(iter
, cpu_file
, ent_ts
, missing_events
);
2319 *ent_cpu
= cpu_file
;
2324 for_each_tracing_cpu(cpu
) {
2326 if (ring_buffer_empty_cpu(buffer
, cpu
))
2329 ent
= peek_next_entry(iter
, cpu
, &ts
, &lost_events
);
2332 * Pick the entry with the smallest timestamp:
2334 if (ent
&& (!next
|| ts
< next_ts
)) {
2338 next_lost
= lost_events
;
2339 next_size
= iter
->ent_size
;
2343 iter
->ent_size
= next_size
;
2346 *ent_cpu
= next_cpu
;
2352 *missing_events
= next_lost
;
2357 /* Find the next real entry, without updating the iterator itself */
2358 struct trace_entry
*trace_find_next_entry(struct trace_iterator
*iter
,
2359 int *ent_cpu
, u64
*ent_ts
)
2361 return __find_next_entry(iter
, ent_cpu
, NULL
, ent_ts
);
2364 /* Find the next real entry, and increment the iterator to the next entry */
2365 void *trace_find_next_entry_inc(struct trace_iterator
*iter
)
2367 iter
->ent
= __find_next_entry(iter
, &iter
->cpu
,
2368 &iter
->lost_events
, &iter
->ts
);
2371 trace_iterator_increment(iter
);
2373 return iter
->ent
? iter
: NULL
;
2376 static void trace_consume(struct trace_iterator
*iter
)
2378 ring_buffer_consume(iter
->trace_buffer
->buffer
, iter
->cpu
, &iter
->ts
,
2379 &iter
->lost_events
);
2382 static void *s_next(struct seq_file
*m
, void *v
, loff_t
*pos
)
2384 struct trace_iterator
*iter
= m
->private;
2388 WARN_ON_ONCE(iter
->leftover
);
2392 /* can't go backwards */
2397 ent
= trace_find_next_entry_inc(iter
);
2401 while (ent
&& iter
->idx
< i
)
2402 ent
= trace_find_next_entry_inc(iter
);
2409 void tracing_iter_reset(struct trace_iterator
*iter
, int cpu
)
2411 struct ring_buffer_event
*event
;
2412 struct ring_buffer_iter
*buf_iter
;
2413 unsigned long entries
= 0;
2416 per_cpu_ptr(iter
->trace_buffer
->data
, cpu
)->skipped_entries
= 0;
2418 buf_iter
= trace_buffer_iter(iter
, cpu
);
2422 ring_buffer_iter_reset(buf_iter
);
2425 * We could have the case with the max latency tracers
2426 * that a reset never took place on a cpu. This is evident
2427 * by the timestamp being before the start of the buffer.
2429 while ((event
= ring_buffer_iter_peek(buf_iter
, &ts
))) {
2430 if (ts
>= iter
->trace_buffer
->time_start
)
2433 ring_buffer_read(buf_iter
, NULL
);
2436 per_cpu_ptr(iter
->trace_buffer
->data
, cpu
)->skipped_entries
= entries
;
2440 * The current tracer is copied to avoid a global locking
2443 static void *s_start(struct seq_file
*m
, loff_t
*pos
)
2445 struct trace_iterator
*iter
= m
->private;
2446 struct trace_array
*tr
= iter
->tr
;
2447 int cpu_file
= iter
->cpu_file
;
2453 * copy the tracer to avoid using a global lock all around.
2454 * iter->trace is a copy of current_trace, the pointer to the
2455 * name may be used instead of a strcmp(), as iter->trace->name
2456 * will point to the same string as current_trace->name.
2458 mutex_lock(&trace_types_lock
);
2459 if (unlikely(tr
->current_trace
&& iter
->trace
->name
!= tr
->current_trace
->name
))
2460 *iter
->trace
= *tr
->current_trace
;
2461 mutex_unlock(&trace_types_lock
);
2463 #ifdef CONFIG_TRACER_MAX_TRACE
2464 if (iter
->snapshot
&& iter
->trace
->use_max_tr
)
2465 return ERR_PTR(-EBUSY
);
2468 if (!iter
->snapshot
)
2469 atomic_inc(&trace_record_cmdline_disabled
);
2471 if (*pos
!= iter
->pos
) {
2476 if (cpu_file
== RING_BUFFER_ALL_CPUS
) {
2477 for_each_tracing_cpu(cpu
)
2478 tracing_iter_reset(iter
, cpu
);
2480 tracing_iter_reset(iter
, cpu_file
);
2483 for (p
= iter
; p
&& l
< *pos
; p
= s_next(m
, p
, &l
))
2488 * If we overflowed the seq_file before, then we want
2489 * to just reuse the trace_seq buffer again.
2495 p
= s_next(m
, p
, &l
);
2499 trace_event_read_lock();
2500 trace_access_lock(cpu_file
);
2504 static void s_stop(struct seq_file
*m
, void *p
)
2506 struct trace_iterator
*iter
= m
->private;
2508 #ifdef CONFIG_TRACER_MAX_TRACE
2509 if (iter
->snapshot
&& iter
->trace
->use_max_tr
)
2513 if (!iter
->snapshot
)
2514 atomic_dec(&trace_record_cmdline_disabled
);
2516 trace_access_unlock(iter
->cpu_file
);
2517 trace_event_read_unlock();
2521 get_total_entries(struct trace_buffer
*buf
,
2522 unsigned long *total
, unsigned long *entries
)
2524 unsigned long count
;
2530 for_each_tracing_cpu(cpu
) {
2531 count
= ring_buffer_entries_cpu(buf
->buffer
, cpu
);
2533 * If this buffer has skipped entries, then we hold all
2534 * entries for the trace and we need to ignore the
2535 * ones before the time stamp.
2537 if (per_cpu_ptr(buf
->data
, cpu
)->skipped_entries
) {
2538 count
-= per_cpu_ptr(buf
->data
, cpu
)->skipped_entries
;
2539 /* total is the same as the entries */
2543 ring_buffer_overrun_cpu(buf
->buffer
, cpu
);
2548 static void print_lat_help_header(struct seq_file
*m
)
2550 seq_puts(m
, "# _------=> CPU# \n"
2551 "# / _-----=> irqs-off \n"
2552 "# | / _----=> need-resched \n"
2553 "# || / _---=> hardirq/softirq \n"
2554 "# ||| / _--=> preempt-depth \n"
2556 "# cmd pid ||||| time | caller \n"
2557 "# \\ / ||||| \\ | / \n");
2560 static void print_event_info(struct trace_buffer
*buf
, struct seq_file
*m
)
2562 unsigned long total
;
2563 unsigned long entries
;
2565 get_total_entries(buf
, &total
, &entries
);
2566 seq_printf(m
, "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n",
2567 entries
, total
, num_online_cpus());
2571 static void print_func_help_header(struct trace_buffer
*buf
, struct seq_file
*m
)
2573 print_event_info(buf
, m
);
2574 seq_puts(m
, "# TASK-PID CPU# TIMESTAMP FUNCTION\n"
2578 static void print_func_help_header_irq(struct trace_buffer
*buf
, struct seq_file
*m
)
2580 print_event_info(buf
, m
);
2581 seq_puts(m
, "# _-----=> irqs-off\n"
2582 "# / _----=> need-resched\n"
2583 "# | / _---=> hardirq/softirq\n"
2584 "# || / _--=> preempt-depth\n"
2586 "# TASK-PID CPU# |||| TIMESTAMP FUNCTION\n"
2587 "# | | | |||| | |\n");
2591 print_trace_header(struct seq_file
*m
, struct trace_iterator
*iter
)
2593 unsigned long sym_flags
= (global_trace
.trace_flags
& TRACE_ITER_SYM_MASK
);
2594 struct trace_buffer
*buf
= iter
->trace_buffer
;
2595 struct trace_array_cpu
*data
= per_cpu_ptr(buf
->data
, buf
->cpu
);
2596 struct tracer
*type
= iter
->trace
;
2597 unsigned long entries
;
2598 unsigned long total
;
2599 const char *name
= "preemption";
2603 get_total_entries(buf
, &total
, &entries
);
2605 seq_printf(m
, "# %s latency trace v1.1.5 on %s\n",
2607 seq_puts(m
, "# -----------------------------------"
2608 "---------------------------------\n");
2609 seq_printf(m
, "# latency: %lu us, #%lu/%lu, CPU#%d |"
2610 " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
2611 nsecs_to_usecs(data
->saved_latency
),
2615 #if defined(CONFIG_PREEMPT_NONE)
2617 #elif defined(CONFIG_PREEMPT_VOLUNTARY)
2619 #elif defined(CONFIG_PREEMPT)
2624 /* These are reserved for later use */
2627 seq_printf(m
, " #P:%d)\n", num_online_cpus());
2631 seq_puts(m
, "# -----------------\n");
2632 seq_printf(m
, "# | task: %.16s-%d "
2633 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
2634 data
->comm
, data
->pid
,
2635 from_kuid_munged(seq_user_ns(m
), data
->uid
), data
->nice
,
2636 data
->policy
, data
->rt_priority
);
2637 seq_puts(m
, "# -----------------\n");
2639 if (data
->critical_start
) {
2640 seq_puts(m
, "# => started at: ");
2641 seq_print_ip_sym(&iter
->seq
, data
->critical_start
, sym_flags
);
2642 trace_print_seq(m
, &iter
->seq
);
2643 seq_puts(m
, "\n# => ended at: ");
2644 seq_print_ip_sym(&iter
->seq
, data
->critical_end
, sym_flags
);
2645 trace_print_seq(m
, &iter
->seq
);
2646 seq_puts(m
, "\n#\n");
2652 static void test_cpu_buff_start(struct trace_iterator
*iter
)
2654 struct trace_seq
*s
= &iter
->seq
;
2655 struct trace_array
*tr
= iter
->tr
;
2657 if (!(tr
->trace_flags
& TRACE_ITER_ANNOTATE
))
2660 if (!(iter
->iter_flags
& TRACE_FILE_ANNOTATE
))
2663 if (iter
->started
&& cpumask_test_cpu(iter
->cpu
, iter
->started
))
2666 if (per_cpu_ptr(iter
->trace_buffer
->data
, iter
->cpu
)->skipped_entries
)
2670 cpumask_set_cpu(iter
->cpu
, iter
->started
);
2672 /* Don't print started cpu buffer for the first entry of the trace */
2674 trace_seq_printf(s
, "##### CPU %u buffer started ####\n",
2678 static enum print_line_t
print_trace_fmt(struct trace_iterator
*iter
)
2680 struct trace_array
*tr
= iter
->tr
;
2681 struct trace_seq
*s
= &iter
->seq
;
2682 unsigned long sym_flags
= (tr
->trace_flags
& TRACE_ITER_SYM_MASK
);
2683 struct trace_entry
*entry
;
2684 struct trace_event
*event
;
2688 test_cpu_buff_start(iter
);
2690 event
= ftrace_find_event(entry
->type
);
2692 if (tr
->trace_flags
& TRACE_ITER_CONTEXT_INFO
) {
2693 if (iter
->iter_flags
& TRACE_FILE_LAT_FMT
)
2694 trace_print_lat_context(iter
);
2696 trace_print_context(iter
);
2699 if (trace_seq_has_overflowed(s
))
2700 return TRACE_TYPE_PARTIAL_LINE
;
2703 return event
->funcs
->trace(iter
, sym_flags
, event
);
2705 trace_seq_printf(s
, "Unknown type %d\n", entry
->type
);
2707 return trace_handle_return(s
);
2710 static enum print_line_t
print_raw_fmt(struct trace_iterator
*iter
)
2712 struct trace_array
*tr
= iter
->tr
;
2713 struct trace_seq
*s
= &iter
->seq
;
2714 struct trace_entry
*entry
;
2715 struct trace_event
*event
;
2719 if (tr
->trace_flags
& TRACE_ITER_CONTEXT_INFO
)
2720 trace_seq_printf(s
, "%d %d %llu ",
2721 entry
->pid
, iter
->cpu
, iter
->ts
);
2723 if (trace_seq_has_overflowed(s
))
2724 return TRACE_TYPE_PARTIAL_LINE
;
2726 event
= ftrace_find_event(entry
->type
);
2728 return event
->funcs
->raw(iter
, 0, event
);
2730 trace_seq_printf(s
, "%d ?\n", entry
->type
);
2732 return trace_handle_return(s
);
2735 static enum print_line_t
print_hex_fmt(struct trace_iterator
*iter
)
2737 struct trace_array
*tr
= iter
->tr
;
2738 struct trace_seq
*s
= &iter
->seq
;
2739 unsigned char newline
= '\n';
2740 struct trace_entry
*entry
;
2741 struct trace_event
*event
;
2745 if (tr
->trace_flags
& TRACE_ITER_CONTEXT_INFO
) {
2746 SEQ_PUT_HEX_FIELD(s
, entry
->pid
);
2747 SEQ_PUT_HEX_FIELD(s
, iter
->cpu
);
2748 SEQ_PUT_HEX_FIELD(s
, iter
->ts
);
2749 if (trace_seq_has_overflowed(s
))
2750 return TRACE_TYPE_PARTIAL_LINE
;
2753 event
= ftrace_find_event(entry
->type
);
2755 enum print_line_t ret
= event
->funcs
->hex(iter
, 0, event
);
2756 if (ret
!= TRACE_TYPE_HANDLED
)
2760 SEQ_PUT_FIELD(s
, newline
);
2762 return trace_handle_return(s
);
2765 static enum print_line_t
print_bin_fmt(struct trace_iterator
*iter
)
2767 struct trace_array
*tr
= iter
->tr
;
2768 struct trace_seq
*s
= &iter
->seq
;
2769 struct trace_entry
*entry
;
2770 struct trace_event
*event
;
2774 if (tr
->trace_flags
& TRACE_ITER_CONTEXT_INFO
) {
2775 SEQ_PUT_FIELD(s
, entry
->pid
);
2776 SEQ_PUT_FIELD(s
, iter
->cpu
);
2777 SEQ_PUT_FIELD(s
, iter
->ts
);
2778 if (trace_seq_has_overflowed(s
))
2779 return TRACE_TYPE_PARTIAL_LINE
;
2782 event
= ftrace_find_event(entry
->type
);
2783 return event
? event
->funcs
->binary(iter
, 0, event
) :
2787 int trace_empty(struct trace_iterator
*iter
)
2789 struct ring_buffer_iter
*buf_iter
;
2792 /* If we are looking at one CPU buffer, only check that one */
2793 if (iter
->cpu_file
!= RING_BUFFER_ALL_CPUS
) {
2794 cpu
= iter
->cpu_file
;
2795 buf_iter
= trace_buffer_iter(iter
, cpu
);
2797 if (!ring_buffer_iter_empty(buf_iter
))
2800 if (!ring_buffer_empty_cpu(iter
->trace_buffer
->buffer
, cpu
))
2806 for_each_tracing_cpu(cpu
) {
2807 buf_iter
= trace_buffer_iter(iter
, cpu
);
2809 if (!ring_buffer_iter_empty(buf_iter
))
2812 if (!ring_buffer_empty_cpu(iter
->trace_buffer
->buffer
, cpu
))
2820 /* Called with trace_event_read_lock() held. */
2821 enum print_line_t
print_trace_line(struct trace_iterator
*iter
)
2823 struct trace_array
*tr
= iter
->tr
;
2824 unsigned long trace_flags
= tr
->trace_flags
;
2825 enum print_line_t ret
;
2827 if (iter
->lost_events
) {
2828 trace_seq_printf(&iter
->seq
, "CPU:%d [LOST %lu EVENTS]\n",
2829 iter
->cpu
, iter
->lost_events
);
2830 if (trace_seq_has_overflowed(&iter
->seq
))
2831 return TRACE_TYPE_PARTIAL_LINE
;
2834 if (iter
->trace
&& iter
->trace
->print_line
) {
2835 ret
= iter
->trace
->print_line(iter
);
2836 if (ret
!= TRACE_TYPE_UNHANDLED
)
2840 if (iter
->ent
->type
== TRACE_BPUTS
&&
2841 trace_flags
& TRACE_ITER_PRINTK
&&
2842 trace_flags
& TRACE_ITER_PRINTK_MSGONLY
)
2843 return trace_print_bputs_msg_only(iter
);
2845 if (iter
->ent
->type
== TRACE_BPRINT
&&
2846 trace_flags
& TRACE_ITER_PRINTK
&&
2847 trace_flags
& TRACE_ITER_PRINTK_MSGONLY
)
2848 return trace_print_bprintk_msg_only(iter
);
2850 if (iter
->ent
->type
== TRACE_PRINT
&&
2851 trace_flags
& TRACE_ITER_PRINTK
&&
2852 trace_flags
& TRACE_ITER_PRINTK_MSGONLY
)
2853 return trace_print_printk_msg_only(iter
);
2855 if (trace_flags
& TRACE_ITER_BIN
)
2856 return print_bin_fmt(iter
);
2858 if (trace_flags
& TRACE_ITER_HEX
)
2859 return print_hex_fmt(iter
);
2861 if (trace_flags
& TRACE_ITER_RAW
)
2862 return print_raw_fmt(iter
);
2864 return print_trace_fmt(iter
);
2867 void trace_latency_header(struct seq_file
*m
)
2869 struct trace_iterator
*iter
= m
->private;
2870 struct trace_array
*tr
= iter
->tr
;
2872 /* print nothing if the buffers are empty */
2873 if (trace_empty(iter
))
2876 if (iter
->iter_flags
& TRACE_FILE_LAT_FMT
)
2877 print_trace_header(m
, iter
);
2879 if (!(tr
->trace_flags
& TRACE_ITER_VERBOSE
))
2880 print_lat_help_header(m
);
2883 void trace_default_header(struct seq_file
*m
)
2885 struct trace_iterator
*iter
= m
->private;
2886 struct trace_array
*tr
= iter
->tr
;
2887 unsigned long trace_flags
= tr
->trace_flags
;
2889 if (!(trace_flags
& TRACE_ITER_CONTEXT_INFO
))
2892 if (iter
->iter_flags
& TRACE_FILE_LAT_FMT
) {
2893 /* print nothing if the buffers are empty */
2894 if (trace_empty(iter
))
2896 print_trace_header(m
, iter
);
2897 if (!(trace_flags
& TRACE_ITER_VERBOSE
))
2898 print_lat_help_header(m
);
2900 if (!(trace_flags
& TRACE_ITER_VERBOSE
)) {
2901 if (trace_flags
& TRACE_ITER_IRQ_INFO
)
2902 print_func_help_header_irq(iter
->trace_buffer
, m
);
2904 print_func_help_header(iter
->trace_buffer
, m
);
2909 static void test_ftrace_alive(struct seq_file
*m
)
2911 if (!ftrace_is_dead())
2913 seq_puts(m
, "# WARNING: FUNCTION TRACING IS CORRUPTED\n"
2914 "# MAY BE MISSING FUNCTION EVENTS\n");
2917 #ifdef CONFIG_TRACER_MAX_TRACE
2918 static void show_snapshot_main_help(struct seq_file
*m
)
2920 seq_puts(m
, "# echo 0 > snapshot : Clears and frees snapshot buffer\n"
2921 "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
2922 "# Takes a snapshot of the main buffer.\n"
2923 "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate or free)\n"
2924 "# (Doesn't have to be '2' works with any number that\n"
2925 "# is not a '0' or '1')\n");
2928 static void show_snapshot_percpu_help(struct seq_file
*m
)
2930 seq_puts(m
, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n");
2931 #ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
2932 seq_puts(m
, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
2933 "# Takes a snapshot of the main buffer for this cpu.\n");
2935 seq_puts(m
, "# echo 1 > snapshot : Not supported with this kernel.\n"
2936 "# Must use main snapshot file to allocate.\n");
2938 seq_puts(m
, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n"
2939 "# (Doesn't have to be '2' works with any number that\n"
2940 "# is not a '0' or '1')\n");
2943 static void print_snapshot_help(struct seq_file
*m
, struct trace_iterator
*iter
)
2945 if (iter
->tr
->allocated_snapshot
)
2946 seq_puts(m
, "#\n# * Snapshot is allocated *\n#\n");
2948 seq_puts(m
, "#\n# * Snapshot is freed *\n#\n");
2950 seq_puts(m
, "# Snapshot commands:\n");
2951 if (iter
->cpu_file
== RING_BUFFER_ALL_CPUS
)
2952 show_snapshot_main_help(m
);
2954 show_snapshot_percpu_help(m
);
2957 /* Should never be called */
2958 static inline void print_snapshot_help(struct seq_file
*m
, struct trace_iterator
*iter
) { }
2961 static int s_show(struct seq_file
*m
, void *v
)
2963 struct trace_iterator
*iter
= v
;
2966 if (iter
->ent
== NULL
) {
2968 seq_printf(m
, "# tracer: %s\n", iter
->trace
->name
);
2970 test_ftrace_alive(m
);
2972 if (iter
->snapshot
&& trace_empty(iter
))
2973 print_snapshot_help(m
, iter
);
2974 else if (iter
->trace
&& iter
->trace
->print_header
)
2975 iter
->trace
->print_header(m
);
2977 trace_default_header(m
);
2979 } else if (iter
->leftover
) {
2981 * If we filled the seq_file buffer earlier, we
2982 * want to just show it now.
2984 ret
= trace_print_seq(m
, &iter
->seq
);
2986 /* ret should this time be zero, but you never know */
2987 iter
->leftover
= ret
;
2990 print_trace_line(iter
);
2991 ret
= trace_print_seq(m
, &iter
->seq
);
2993 * If we overflow the seq_file buffer, then it will
2994 * ask us for this data again at start up.
2996 * ret is 0 if seq_file write succeeded.
2999 iter
->leftover
= ret
;
3006 * Should be used after trace_array_get(), trace_types_lock
3007 * ensures that i_cdev was already initialized.
3009 static inline int tracing_get_cpu(struct inode
*inode
)
3011 if (inode
->i_cdev
) /* See trace_create_cpu_file() */
3012 return (long)inode
->i_cdev
- 1;
3013 return RING_BUFFER_ALL_CPUS
;
3016 static const struct seq_operations tracer_seq_ops
= {
3023 static struct trace_iterator
*
3024 __tracing_open(struct inode
*inode
, struct file
*file
, bool snapshot
)
3026 struct trace_array
*tr
= inode
->i_private
;
3027 struct trace_iterator
*iter
;
3030 if (tracing_disabled
)
3031 return ERR_PTR(-ENODEV
);
3033 iter
= __seq_open_private(file
, &tracer_seq_ops
, sizeof(*iter
));
3035 return ERR_PTR(-ENOMEM
);
3037 iter
->buffer_iter
= kcalloc(nr_cpu_ids
, sizeof(*iter
->buffer_iter
),
3039 if (!iter
->buffer_iter
)
3043 * We make a copy of the current tracer to avoid concurrent
3044 * changes on it while we are reading.
3046 mutex_lock(&trace_types_lock
);
3047 iter
->trace
= kzalloc(sizeof(*iter
->trace
), GFP_KERNEL
);
3051 *iter
->trace
= *tr
->current_trace
;
3053 if (!zalloc_cpumask_var(&iter
->started
, GFP_KERNEL
))
3058 #ifdef CONFIG_TRACER_MAX_TRACE
3059 /* Currently only the top directory has a snapshot */
3060 if (tr
->current_trace
->print_max
|| snapshot
)
3061 iter
->trace_buffer
= &tr
->max_buffer
;
3064 iter
->trace_buffer
= &tr
->trace_buffer
;
3065 iter
->snapshot
= snapshot
;
3067 iter
->cpu_file
= tracing_get_cpu(inode
);
3068 mutex_init(&iter
->mutex
);
3070 /* Notify the tracer early; before we stop tracing. */
3071 if (iter
->trace
&& iter
->trace
->open
)
3072 iter
->trace
->open(iter
);
3074 /* Annotate start of buffers if we had overruns */
3075 if (ring_buffer_overruns(iter
->trace_buffer
->buffer
))
3076 iter
->iter_flags
|= TRACE_FILE_ANNOTATE
;
3078 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
3079 if (trace_clocks
[tr
->clock_id
].in_ns
)
3080 iter
->iter_flags
|= TRACE_FILE_TIME_IN_NS
;
3082 /* stop the trace while dumping if we are not opening "snapshot" */
3083 if (!iter
->snapshot
)
3084 tracing_stop_tr(tr
);
3086 if (iter
->cpu_file
== RING_BUFFER_ALL_CPUS
) {
3087 for_each_tracing_cpu(cpu
) {
3088 iter
->buffer_iter
[cpu
] =
3089 ring_buffer_read_prepare(iter
->trace_buffer
->buffer
, cpu
);
3091 ring_buffer_read_prepare_sync();
3092 for_each_tracing_cpu(cpu
) {
3093 ring_buffer_read_start(iter
->buffer_iter
[cpu
]);
3094 tracing_iter_reset(iter
, cpu
);
3097 cpu
= iter
->cpu_file
;
3098 iter
->buffer_iter
[cpu
] =
3099 ring_buffer_read_prepare(iter
->trace_buffer
->buffer
, cpu
);
3100 ring_buffer_read_prepare_sync();
3101 ring_buffer_read_start(iter
->buffer_iter
[cpu
]);
3102 tracing_iter_reset(iter
, cpu
);
3105 mutex_unlock(&trace_types_lock
);
3110 mutex_unlock(&trace_types_lock
);
3112 kfree(iter
->buffer_iter
);
3114 seq_release_private(inode
, file
);
3115 return ERR_PTR(-ENOMEM
);
3118 int tracing_open_generic(struct inode
*inode
, struct file
*filp
)
3120 if (tracing_disabled
)
3123 filp
->private_data
= inode
->i_private
;
3127 bool tracing_is_disabled(void)
3129 return (tracing_disabled
) ? true: false;
3133 * Open and update trace_array ref count.
3134 * Must have the current trace_array passed to it.
3136 static int tracing_open_generic_tr(struct inode
*inode
, struct file
*filp
)
3138 struct trace_array
*tr
= inode
->i_private
;
3140 if (tracing_disabled
)
3143 if (trace_array_get(tr
) < 0)
3146 filp
->private_data
= inode
->i_private
;
3151 static int tracing_release(struct inode
*inode
, struct file
*file
)
3153 struct trace_array
*tr
= inode
->i_private
;
3154 struct seq_file
*m
= file
->private_data
;
3155 struct trace_iterator
*iter
;
3158 if (!(file
->f_mode
& FMODE_READ
)) {
3159 trace_array_put(tr
);
3163 /* Writes do not use seq_file */
3165 mutex_lock(&trace_types_lock
);
3167 for_each_tracing_cpu(cpu
) {
3168 if (iter
->buffer_iter
[cpu
])
3169 ring_buffer_read_finish(iter
->buffer_iter
[cpu
]);
3172 if (iter
->trace
&& iter
->trace
->close
)
3173 iter
->trace
->close(iter
);
3175 if (!iter
->snapshot
)
3176 /* reenable tracing if it was previously enabled */
3177 tracing_start_tr(tr
);
3179 __trace_array_put(tr
);
3181 mutex_unlock(&trace_types_lock
);
3183 mutex_destroy(&iter
->mutex
);
3184 free_cpumask_var(iter
->started
);
3186 kfree(iter
->buffer_iter
);
3187 seq_release_private(inode
, file
);
3192 static int tracing_release_generic_tr(struct inode
*inode
, struct file
*file
)
3194 struct trace_array
*tr
= inode
->i_private
;
3196 trace_array_put(tr
);
3200 static int tracing_single_release_tr(struct inode
*inode
, struct file
*file
)
3202 struct trace_array
*tr
= inode
->i_private
;
3204 trace_array_put(tr
);
3206 return single_release(inode
, file
);
3209 static int tracing_open(struct inode
*inode
, struct file
*file
)
3211 struct trace_array
*tr
= inode
->i_private
;
3212 struct trace_iterator
*iter
;
3215 if (trace_array_get(tr
) < 0)
3218 /* If this file was open for write, then erase contents */
3219 if ((file
->f_mode
& FMODE_WRITE
) && (file
->f_flags
& O_TRUNC
)) {
3220 int cpu
= tracing_get_cpu(inode
);
3222 if (cpu
== RING_BUFFER_ALL_CPUS
)
3223 tracing_reset_online_cpus(&tr
->trace_buffer
);
3225 tracing_reset(&tr
->trace_buffer
, cpu
);
3228 if (file
->f_mode
& FMODE_READ
) {
3229 iter
= __tracing_open(inode
, file
, false);
3231 ret
= PTR_ERR(iter
);
3232 else if (tr
->trace_flags
& TRACE_ITER_LATENCY_FMT
)
3233 iter
->iter_flags
|= TRACE_FILE_LAT_FMT
;
3237 trace_array_put(tr
);
3243 * Some tracers are not suitable for instance buffers.
3244 * A tracer is always available for the global array (toplevel)
3245 * or if it explicitly states that it is.
3248 trace_ok_for_array(struct tracer
*t
, struct trace_array
*tr
)
3250 return (tr
->flags
& TRACE_ARRAY_FL_GLOBAL
) || t
->allow_instances
;
3253 /* Find the next tracer that this trace array may use */
3254 static struct tracer
*
3255 get_tracer_for_array(struct trace_array
*tr
, struct tracer
*t
)
3257 while (t
&& !trace_ok_for_array(t
, tr
))
3264 t_next(struct seq_file
*m
, void *v
, loff_t
*pos
)
3266 struct trace_array
*tr
= m
->private;
3267 struct tracer
*t
= v
;
3272 t
= get_tracer_for_array(tr
, t
->next
);
3277 static void *t_start(struct seq_file
*m
, loff_t
*pos
)
3279 struct trace_array
*tr
= m
->private;
3283 mutex_lock(&trace_types_lock
);
3285 t
= get_tracer_for_array(tr
, trace_types
);
3286 for (; t
&& l
< *pos
; t
= t_next(m
, t
, &l
))
3292 static void t_stop(struct seq_file
*m
, void *p
)
3294 mutex_unlock(&trace_types_lock
);
3297 static int t_show(struct seq_file
*m
, void *v
)
3299 struct tracer
*t
= v
;
3304 seq_puts(m
, t
->name
);
3313 static const struct seq_operations show_traces_seq_ops
= {
3320 static int show_traces_open(struct inode
*inode
, struct file
*file
)
3322 struct trace_array
*tr
= inode
->i_private
;
3326 if (tracing_disabled
)
3329 ret
= seq_open(file
, &show_traces_seq_ops
);
3333 m
= file
->private_data
;
3340 tracing_write_stub(struct file
*filp
, const char __user
*ubuf
,
3341 size_t count
, loff_t
*ppos
)
3346 loff_t
tracing_lseek(struct file
*file
, loff_t offset
, int whence
)
3350 if (file
->f_mode
& FMODE_READ
)
3351 ret
= seq_lseek(file
, offset
, whence
);
3353 file
->f_pos
= ret
= 0;
3358 static const struct file_operations tracing_fops
= {
3359 .open
= tracing_open
,
3361 .write
= tracing_write_stub
,
3362 .llseek
= tracing_lseek
,
3363 .release
= tracing_release
,
3366 static const struct file_operations show_traces_fops
= {
3367 .open
= show_traces_open
,
3369 .release
= seq_release
,
3370 .llseek
= seq_lseek
,
3374 * The tracer itself will not take this lock, but still we want
3375 * to provide a consistent cpumask to user-space:
3377 static DEFINE_MUTEX(tracing_cpumask_update_lock
);
3380 * Temporary storage for the character representation of the
3381 * CPU bitmask (and one more byte for the newline):
3383 static char mask_str
[NR_CPUS
+ 1];
3386 tracing_cpumask_read(struct file
*filp
, char __user
*ubuf
,
3387 size_t count
, loff_t
*ppos
)
3389 struct trace_array
*tr
= file_inode(filp
)->i_private
;
3392 mutex_lock(&tracing_cpumask_update_lock
);
3394 len
= snprintf(mask_str
, count
, "%*pb\n",
3395 cpumask_pr_args(tr
->tracing_cpumask
));
3400 count
= simple_read_from_buffer(ubuf
, count
, ppos
, mask_str
, NR_CPUS
+1);
3403 mutex_unlock(&tracing_cpumask_update_lock
);
3409 tracing_cpumask_write(struct file
*filp
, const char __user
*ubuf
,
3410 size_t count
, loff_t
*ppos
)
3412 struct trace_array
*tr
= file_inode(filp
)->i_private
;
3413 cpumask_var_t tracing_cpumask_new
;
3416 if (!alloc_cpumask_var(&tracing_cpumask_new
, GFP_KERNEL
))
3419 err
= cpumask_parse_user(ubuf
, count
, tracing_cpumask_new
);
3423 mutex_lock(&tracing_cpumask_update_lock
);
3425 local_irq_disable();
3426 arch_spin_lock(&tr
->max_lock
);
3427 for_each_tracing_cpu(cpu
) {
3429 * Increase/decrease the disabled counter if we are
3430 * about to flip a bit in the cpumask:
3432 if (cpumask_test_cpu(cpu
, tr
->tracing_cpumask
) &&
3433 !cpumask_test_cpu(cpu
, tracing_cpumask_new
)) {
3434 atomic_inc(&per_cpu_ptr(tr
->trace_buffer
.data
, cpu
)->disabled
);
3435 ring_buffer_record_disable_cpu(tr
->trace_buffer
.buffer
, cpu
);
3437 if (!cpumask_test_cpu(cpu
, tr
->tracing_cpumask
) &&
3438 cpumask_test_cpu(cpu
, tracing_cpumask_new
)) {
3439 atomic_dec(&per_cpu_ptr(tr
->trace_buffer
.data
, cpu
)->disabled
);
3440 ring_buffer_record_enable_cpu(tr
->trace_buffer
.buffer
, cpu
);
3443 arch_spin_unlock(&tr
->max_lock
);
3446 cpumask_copy(tr
->tracing_cpumask
, tracing_cpumask_new
);
3448 mutex_unlock(&tracing_cpumask_update_lock
);
3449 free_cpumask_var(tracing_cpumask_new
);
3454 free_cpumask_var(tracing_cpumask_new
);
3459 static const struct file_operations tracing_cpumask_fops
= {
3460 .open
= tracing_open_generic_tr
,
3461 .read
= tracing_cpumask_read
,
3462 .write
= tracing_cpumask_write
,
3463 .release
= tracing_release_generic_tr
,
3464 .llseek
= generic_file_llseek
,
3467 static int tracing_trace_options_show(struct seq_file
*m
, void *v
)
3469 struct tracer_opt
*trace_opts
;
3470 struct trace_array
*tr
= m
->private;
3474 mutex_lock(&trace_types_lock
);
3475 tracer_flags
= tr
->current_trace
->flags
->val
;
3476 trace_opts
= tr
->current_trace
->flags
->opts
;
3478 for (i
= 0; trace_options
[i
]; i
++) {
3479 if (tr
->trace_flags
& (1 << i
))
3480 seq_printf(m
, "%s\n", trace_options
[i
]);
3482 seq_printf(m
, "no%s\n", trace_options
[i
]);
3485 for (i
= 0; trace_opts
[i
].name
; i
++) {
3486 if (tracer_flags
& trace_opts
[i
].bit
)
3487 seq_printf(m
, "%s\n", trace_opts
[i
].name
);
3489 seq_printf(m
, "no%s\n", trace_opts
[i
].name
);
3491 mutex_unlock(&trace_types_lock
);
3496 static int __set_tracer_option(struct trace_array
*tr
,
3497 struct tracer_flags
*tracer_flags
,
3498 struct tracer_opt
*opts
, int neg
)
3500 struct tracer
*trace
= tracer_flags
->trace
;
3503 ret
= trace
->set_flag(tr
, tracer_flags
->val
, opts
->bit
, !neg
);
3508 tracer_flags
->val
&= ~opts
->bit
;
3510 tracer_flags
->val
|= opts
->bit
;
3514 /* Try to assign a tracer specific option */
3515 static int set_tracer_option(struct trace_array
*tr
, char *cmp
, int neg
)
3517 struct tracer
*trace
= tr
->current_trace
;
3518 struct tracer_flags
*tracer_flags
= trace
->flags
;
3519 struct tracer_opt
*opts
= NULL
;
3522 for (i
= 0; tracer_flags
->opts
[i
].name
; i
++) {
3523 opts
= &tracer_flags
->opts
[i
];
3525 if (strcmp(cmp
, opts
->name
) == 0)
3526 return __set_tracer_option(tr
, trace
->flags
, opts
, neg
);
3532 /* Some tracers require overwrite to stay enabled */
3533 int trace_keep_overwrite(struct tracer
*tracer
, u32 mask
, int set
)
3535 if (tracer
->enabled
&& (mask
& TRACE_ITER_OVERWRITE
) && !set
)
3541 int set_tracer_flag(struct trace_array
*tr
, unsigned int mask
, int enabled
)
3543 /* do nothing if flag is already set */
3544 if (!!(tr
->trace_flags
& mask
) == !!enabled
)
3547 /* Give the tracer a chance to approve the change */
3548 if (tr
->current_trace
->flag_changed
)
3549 if (tr
->current_trace
->flag_changed(tr
, mask
, !!enabled
))
3553 tr
->trace_flags
|= mask
;
3555 tr
->trace_flags
&= ~mask
;
3557 if (mask
== TRACE_ITER_RECORD_CMD
)
3558 trace_event_enable_cmd_record(enabled
);
3560 if (mask
== TRACE_ITER_EVENT_FORK
)
3561 trace_event_follow_fork(tr
, enabled
);
3563 if (mask
== TRACE_ITER_OVERWRITE
) {
3564 ring_buffer_change_overwrite(tr
->trace_buffer
.buffer
, enabled
);
3565 #ifdef CONFIG_TRACER_MAX_TRACE
3566 ring_buffer_change_overwrite(tr
->max_buffer
.buffer
, enabled
);
3570 if (mask
== TRACE_ITER_PRINTK
) {
3571 trace_printk_start_stop_comm(enabled
);
3572 trace_printk_control(enabled
);
3578 static int trace_set_options(struct trace_array
*tr
, char *option
)
3584 size_t orig_len
= strlen(option
);
3586 cmp
= strstrip(option
);
3588 if (strncmp(cmp
, "no", 2) == 0) {
3593 mutex_lock(&trace_types_lock
);
3595 for (i
= 0; trace_options
[i
]; i
++) {
3596 if (strcmp(cmp
, trace_options
[i
]) == 0) {
3597 ret
= set_tracer_flag(tr
, 1 << i
, !neg
);
3602 /* If no option could be set, test the specific tracer options */
3603 if (!trace_options
[i
])
3604 ret
= set_tracer_option(tr
, cmp
, neg
);
3606 mutex_unlock(&trace_types_lock
);
3609 * If the first trailing whitespace is replaced with '\0' by strstrip,
3610 * turn it back into a space.
3612 if (orig_len
> strlen(option
))
3613 option
[strlen(option
)] = ' ';
3618 static void __init
apply_trace_boot_options(void)
3620 char *buf
= trace_boot_options_buf
;
3624 option
= strsep(&buf
, ",");
3630 trace_set_options(&global_trace
, option
);
3632 /* Put back the comma to allow this to be called again */
3639 tracing_trace_options_write(struct file
*filp
, const char __user
*ubuf
,
3640 size_t cnt
, loff_t
*ppos
)
3642 struct seq_file
*m
= filp
->private_data
;
3643 struct trace_array
*tr
= m
->private;
3647 if (cnt
>= sizeof(buf
))
3650 if (copy_from_user(buf
, ubuf
, cnt
))
3655 ret
= trace_set_options(tr
, buf
);
3664 static int tracing_trace_options_open(struct inode
*inode
, struct file
*file
)
3666 struct trace_array
*tr
= inode
->i_private
;
3669 if (tracing_disabled
)
3672 if (trace_array_get(tr
) < 0)
3675 ret
= single_open(file
, tracing_trace_options_show
, inode
->i_private
);
3677 trace_array_put(tr
);
3682 static const struct file_operations tracing_iter_fops
= {
3683 .open
= tracing_trace_options_open
,
3685 .llseek
= seq_lseek
,
3686 .release
= tracing_single_release_tr
,
3687 .write
= tracing_trace_options_write
,
3690 static const char readme_msg
[] =
3691 "tracing mini-HOWTO:\n\n"
3692 "# echo 0 > tracing_on : quick way to disable tracing\n"
3693 "# echo 1 > tracing_on : quick way to re-enable tracing\n\n"
3694 " Important files:\n"
3695 " trace\t\t\t- The static contents of the buffer\n"
3696 "\t\t\t To clear the buffer write into this file: echo > trace\n"
3697 " trace_pipe\t\t- A consuming read to see the contents of the buffer\n"
3698 " current_tracer\t- function and latency tracers\n"
3699 " available_tracers\t- list of configured tracers for current_tracer\n"
3700 " buffer_size_kb\t- view and modify size of per cpu buffer\n"
3701 " buffer_total_size_kb - view total size of all cpu buffers\n\n"
3702 " trace_clock\t\t-change the clock used to order events\n"
3703 " local: Per cpu clock but may not be synced across CPUs\n"
3704 " global: Synced across CPUs but slows tracing down.\n"
3705 " counter: Not a clock, but just an increment\n"
3706 " uptime: Jiffy counter from time of boot\n"
3707 " perf: Same clock that perf events use\n"
3708 #ifdef CONFIG_X86_64
3709 " x86-tsc: TSC cycle counter\n"
3711 "\n trace_marker\t\t- Writes into this file writes into the kernel buffer\n"
3712 " tracing_cpumask\t- Limit which CPUs to trace\n"
3713 " instances\t\t- Make sub-buffers with: mkdir instances/foo\n"
3714 "\t\t\t Remove sub-buffer with rmdir\n"
3715 " trace_options\t\t- Set format or modify how tracing happens\n"
3716 "\t\t\t Disable an option by adding a suffix 'no' to the\n"
3717 "\t\t\t option name\n"
3718 " saved_cmdlines_size\t- echo command number in here to store comm-pid list\n"
3719 #ifdef CONFIG_DYNAMIC_FTRACE
3720 "\n available_filter_functions - list of functions that can be filtered on\n"
3721 " set_ftrace_filter\t- echo function name in here to only trace these\n"
3722 "\t\t\t functions\n"
3723 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
3724 "\t modules: Can select a group via module\n"
3725 "\t Format: :mod:<module-name>\n"
3726 "\t example: echo :mod:ext3 > set_ftrace_filter\n"
3727 "\t triggers: a command to perform when function is hit\n"
3728 "\t Format: <function>:<trigger>[:count]\n"
3729 "\t trigger: traceon, traceoff\n"
3730 "\t\t enable_event:<system>:<event>\n"
3731 "\t\t disable_event:<system>:<event>\n"
3732 #ifdef CONFIG_STACKTRACE
3735 #ifdef CONFIG_TRACER_SNAPSHOT
3740 "\t example: echo do_fault:traceoff > set_ftrace_filter\n"
3741 "\t echo do_trap:traceoff:3 > set_ftrace_filter\n"
3742 "\t The first one will disable tracing every time do_fault is hit\n"
3743 "\t The second will disable tracing at most 3 times when do_trap is hit\n"
3744 "\t The first time do trap is hit and it disables tracing, the\n"
3745 "\t counter will decrement to 2. If tracing is already disabled,\n"
3746 "\t the counter will not decrement. It only decrements when the\n"
3747 "\t trigger did work\n"
3748 "\t To remove trigger without count:\n"
3749 "\t echo '!<function>:<trigger> > set_ftrace_filter\n"
3750 "\t To remove trigger with a count:\n"
3751 "\t echo '!<function>:<trigger>:0 > set_ftrace_filter\n"
3752 " set_ftrace_notrace\t- echo function name in here to never trace.\n"
3753 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
3754 "\t modules: Can select a group via module command :mod:\n"
3755 "\t Does not accept triggers\n"
3756 #endif /* CONFIG_DYNAMIC_FTRACE */
3757 #ifdef CONFIG_FUNCTION_TRACER
3758 " set_ftrace_pid\t- Write pid(s) to only function trace those pids\n"
3761 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
3762 " set_graph_function\t- Trace the nested calls of a function (function_graph)\n"
3763 " set_graph_notrace\t- Do not trace the nested calls of a function (function_graph)\n"
3764 " max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n"
3766 #ifdef CONFIG_TRACER_SNAPSHOT
3767 "\n snapshot\t\t- Like 'trace' but shows the content of the static\n"
3768 "\t\t\t snapshot buffer. Read the contents for more\n"
3769 "\t\t\t information\n"
3771 #ifdef CONFIG_STACK_TRACER
3772 " stack_trace\t\t- Shows the max stack trace when active\n"
3773 " stack_max_size\t- Shows current max stack size that was traced\n"
3774 "\t\t\t Write into this file to reset the max size (trigger a\n"
3775 "\t\t\t new trace)\n"
3776 #ifdef CONFIG_DYNAMIC_FTRACE
3777 " stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace\n"
3780 #endif /* CONFIG_STACK_TRACER */
3781 " events/\t\t- Directory containing all trace event subsystems:\n"
3782 " enable\t\t- Write 0/1 to enable/disable tracing of all events\n"
3783 " events/<system>/\t- Directory containing all trace events for <system>:\n"
3784 " enable\t\t- Write 0/1 to enable/disable tracing of all <system>\n"
3786 " filter\t\t- If set, only events passing filter are traced\n"
3787 " events/<system>/<event>/\t- Directory containing control files for\n"
3789 " enable\t\t- Write 0/1 to enable/disable tracing of <event>\n"
3790 " filter\t\t- If set, only events passing filter are traced\n"
3791 " trigger\t\t- If set, a command to perform when event is hit\n"
3792 "\t Format: <trigger>[:count][if <filter>]\n"
3793 "\t trigger: traceon, traceoff\n"
3794 "\t enable_event:<system>:<event>\n"
3795 "\t disable_event:<system>:<event>\n"
3796 #ifdef CONFIG_HIST_TRIGGERS
3797 "\t enable_hist:<system>:<event>\n"
3798 "\t disable_hist:<system>:<event>\n"
3800 #ifdef CONFIG_STACKTRACE
3803 #ifdef CONFIG_TRACER_SNAPSHOT
3806 #ifdef CONFIG_HIST_TRIGGERS
3807 "\t\t hist (see below)\n"
3809 "\t example: echo traceoff > events/block/block_unplug/trigger\n"
3810 "\t echo traceoff:3 > events/block/block_unplug/trigger\n"
3811 "\t echo 'enable_event:kmem:kmalloc:3 if nr_rq > 1' > \\\n"
3812 "\t events/block/block_unplug/trigger\n"
3813 "\t The first disables tracing every time block_unplug is hit.\n"
3814 "\t The second disables tracing the first 3 times block_unplug is hit.\n"
3815 "\t The third enables the kmalloc event the first 3 times block_unplug\n"
3816 "\t is hit and has value of greater than 1 for the 'nr_rq' event field.\n"
3817 "\t Like function triggers, the counter is only decremented if it\n"
3818 "\t enabled or disabled tracing.\n"
3819 "\t To remove a trigger without a count:\n"
3820 "\t echo '!<trigger> > <system>/<event>/trigger\n"
3821 "\t To remove a trigger with a count:\n"
3822 "\t echo '!<trigger>:0 > <system>/<event>/trigger\n"
3823 "\t Filters can be ignored when removing a trigger.\n"
3824 #ifdef CONFIG_HIST_TRIGGERS
3825 " hist trigger\t- If set, event hits are aggregated into a hash table\n"
3826 "\t Format: hist:keys=<field1[,field2,...]>\n"
3827 "\t [:values=<field1[,field2,...]>]\n"
3828 "\t [:sort=<field1[,field2,...]>]\n"
3829 "\t [:size=#entries]\n"
3830 "\t [:pause][:continue][:clear]\n"
3831 "\t [:name=histname1]\n"
3832 "\t [if <filter>]\n\n"
3833 "\t When a matching event is hit, an entry is added to a hash\n"
3834 "\t table using the key(s) and value(s) named, and the value of a\n"
3835 "\t sum called 'hitcount' is incremented. Keys and values\n"
3836 "\t correspond to fields in the event's format description. Keys\n"
3837 "\t can be any field, or the special string 'stacktrace'.\n"
3838 "\t Compound keys consisting of up to two fields can be specified\n"
3839 "\t by the 'keys' keyword. Values must correspond to numeric\n"
3840 "\t fields. Sort keys consisting of up to two fields can be\n"
3841 "\t specified using the 'sort' keyword. The sort direction can\n"
3842 "\t be modified by appending '.descending' or '.ascending' to a\n"
3843 "\t sort field. The 'size' parameter can be used to specify more\n"
3844 "\t or fewer than the default 2048 entries for the hashtable size.\n"
3845 "\t If a hist trigger is given a name using the 'name' parameter,\n"
3846 "\t its histogram data will be shared with other triggers of the\n"
3847 "\t same name, and trigger hits will update this common data.\n\n"
3848 "\t Reading the 'hist' file for the event will dump the hash\n"
3849 "\t table in its entirety to stdout. If there are multiple hist\n"
3850 "\t triggers attached to an event, there will be a table for each\n"
3851 "\t trigger in the output. The table displayed for a named\n"
3852 "\t trigger will be the same as any other instance having the\n"
3853 "\t same name. The default format used to display a given field\n"
3854 "\t can be modified by appending any of the following modifiers\n"
3855 "\t to the field name, as applicable:\n\n"
3856 "\t .hex display a number as a hex value\n"
3857 "\t .sym display an address as a symbol\n"
3858 "\t .sym-offset display an address as a symbol and offset\n"
3859 "\t .execname display a common_pid as a program name\n"
3860 "\t .syscall display a syscall id as a syscall name\n\n"
3861 "\t .log2 display log2 value rather than raw number\n\n"
3862 "\t The 'pause' parameter can be used to pause an existing hist\n"
3863 "\t trigger or to start a hist trigger but not log any events\n"
3864 "\t until told to do so. 'continue' can be used to start or\n"
3865 "\t restart a paused hist trigger.\n\n"
3866 "\t The 'clear' parameter will clear the contents of a running\n"
3867 "\t hist trigger and leave its current paused/active state\n"
3869 "\t The enable_hist and disable_hist triggers can be used to\n"
3870 "\t have one event conditionally start and stop another event's\n"
3871 "\t already-attached hist trigger. The syntax is analagous to\n"
3872 "\t the enable_event and disable_event triggers.\n"
3877 tracing_readme_read(struct file
*filp
, char __user
*ubuf
,
3878 size_t cnt
, loff_t
*ppos
)
3880 return simple_read_from_buffer(ubuf
, cnt
, ppos
,
3881 readme_msg
, strlen(readme_msg
));
3884 static const struct file_operations tracing_readme_fops
= {
3885 .open
= tracing_open_generic
,
3886 .read
= tracing_readme_read
,
3887 .llseek
= generic_file_llseek
,
3890 static void *saved_cmdlines_next(struct seq_file
*m
, void *v
, loff_t
*pos
)
3892 unsigned int *ptr
= v
;
3894 if (*pos
|| m
->count
)
3899 for (; ptr
< &savedcmd
->map_cmdline_to_pid
[savedcmd
->cmdline_num
];
3901 if (*ptr
== -1 || *ptr
== NO_CMDLINE_MAP
)
3910 static void *saved_cmdlines_start(struct seq_file
*m
, loff_t
*pos
)
3916 arch_spin_lock(&trace_cmdline_lock
);
3918 v
= &savedcmd
->map_cmdline_to_pid
[0];
3920 v
= saved_cmdlines_next(m
, v
, &l
);
3928 static void saved_cmdlines_stop(struct seq_file
*m
, void *v
)
3930 arch_spin_unlock(&trace_cmdline_lock
);
3934 static int saved_cmdlines_show(struct seq_file
*m
, void *v
)
3936 char buf
[TASK_COMM_LEN
];
3937 unsigned int *pid
= v
;
3939 __trace_find_cmdline(*pid
, buf
);
3940 seq_printf(m
, "%d %s\n", *pid
, buf
);
3944 static const struct seq_operations tracing_saved_cmdlines_seq_ops
= {
3945 .start
= saved_cmdlines_start
,
3946 .next
= saved_cmdlines_next
,
3947 .stop
= saved_cmdlines_stop
,
3948 .show
= saved_cmdlines_show
,
3951 static int tracing_saved_cmdlines_open(struct inode
*inode
, struct file
*filp
)
3953 if (tracing_disabled
)
3956 return seq_open(filp
, &tracing_saved_cmdlines_seq_ops
);
3959 static const struct file_operations tracing_saved_cmdlines_fops
= {
3960 .open
= tracing_saved_cmdlines_open
,
3962 .llseek
= seq_lseek
,
3963 .release
= seq_release
,
3967 tracing_saved_cmdlines_size_read(struct file
*filp
, char __user
*ubuf
,
3968 size_t cnt
, loff_t
*ppos
)
3973 arch_spin_lock(&trace_cmdline_lock
);
3974 r
= scnprintf(buf
, sizeof(buf
), "%u\n", savedcmd
->cmdline_num
);
3975 arch_spin_unlock(&trace_cmdline_lock
);
3977 return simple_read_from_buffer(ubuf
, cnt
, ppos
, buf
, r
);
3980 static void free_saved_cmdlines_buffer(struct saved_cmdlines_buffer
*s
)
3982 kfree(s
->saved_cmdlines
);
3983 kfree(s
->map_cmdline_to_pid
);
3987 static int tracing_resize_saved_cmdlines(unsigned int val
)
3989 struct saved_cmdlines_buffer
*s
, *savedcmd_temp
;
3991 s
= kmalloc(sizeof(*s
), GFP_KERNEL
);
3995 if (allocate_cmdlines_buffer(val
, s
) < 0) {
4000 arch_spin_lock(&trace_cmdline_lock
);
4001 savedcmd_temp
= savedcmd
;
4003 arch_spin_unlock(&trace_cmdline_lock
);
4004 free_saved_cmdlines_buffer(savedcmd_temp
);
4010 tracing_saved_cmdlines_size_write(struct file
*filp
, const char __user
*ubuf
,
4011 size_t cnt
, loff_t
*ppos
)
4016 ret
= kstrtoul_from_user(ubuf
, cnt
, 10, &val
);
4020 /* must have at least 1 entry or less than PID_MAX_DEFAULT */
4021 if (!val
|| val
> PID_MAX_DEFAULT
)
4024 ret
= tracing_resize_saved_cmdlines((unsigned int)val
);
4033 static const struct file_operations tracing_saved_cmdlines_size_fops
= {
4034 .open
= tracing_open_generic
,
4035 .read
= tracing_saved_cmdlines_size_read
,
4036 .write
= tracing_saved_cmdlines_size_write
,
4039 #ifdef CONFIG_TRACE_ENUM_MAP_FILE
4040 static union trace_enum_map_item
*
4041 update_enum_map(union trace_enum_map_item
*ptr
)
4043 if (!ptr
->map
.enum_string
) {
4044 if (ptr
->tail
.next
) {
4045 ptr
= ptr
->tail
.next
;
4046 /* Set ptr to the next real item (skip head) */
4054 static void *enum_map_next(struct seq_file
*m
, void *v
, loff_t
*pos
)
4056 union trace_enum_map_item
*ptr
= v
;
4059 * Paranoid! If ptr points to end, we don't want to increment past it.
4060 * This really should never happen.
4062 ptr
= update_enum_map(ptr
);
4063 if (WARN_ON_ONCE(!ptr
))
4070 ptr
= update_enum_map(ptr
);
4075 static void *enum_map_start(struct seq_file
*m
, loff_t
*pos
)
4077 union trace_enum_map_item
*v
;
4080 mutex_lock(&trace_enum_mutex
);
4082 v
= trace_enum_maps
;
4086 while (v
&& l
< *pos
) {
4087 v
= enum_map_next(m
, v
, &l
);
4093 static void enum_map_stop(struct seq_file
*m
, void *v
)
4095 mutex_unlock(&trace_enum_mutex
);
4098 static int enum_map_show(struct seq_file
*m
, void *v
)
4100 union trace_enum_map_item
*ptr
= v
;
4102 seq_printf(m
, "%s %ld (%s)\n",
4103 ptr
->map
.enum_string
, ptr
->map
.enum_value
,
4109 static const struct seq_operations tracing_enum_map_seq_ops
= {
4110 .start
= enum_map_start
,
4111 .next
= enum_map_next
,
4112 .stop
= enum_map_stop
,
4113 .show
= enum_map_show
,
4116 static int tracing_enum_map_open(struct inode
*inode
, struct file
*filp
)
4118 if (tracing_disabled
)
4121 return seq_open(filp
, &tracing_enum_map_seq_ops
);
4124 static const struct file_operations tracing_enum_map_fops
= {
4125 .open
= tracing_enum_map_open
,
4127 .llseek
= seq_lseek
,
4128 .release
= seq_release
,
4131 static inline union trace_enum_map_item
*
4132 trace_enum_jmp_to_tail(union trace_enum_map_item
*ptr
)
4134 /* Return tail of array given the head */
4135 return ptr
+ ptr
->head
.length
+ 1;
4139 trace_insert_enum_map_file(struct module
*mod
, struct trace_enum_map
**start
,
4142 struct trace_enum_map
**stop
;
4143 struct trace_enum_map
**map
;
4144 union trace_enum_map_item
*map_array
;
4145 union trace_enum_map_item
*ptr
;
4150 * The trace_enum_maps contains the map plus a head and tail item,
4151 * where the head holds the module and length of array, and the
4152 * tail holds a pointer to the next list.
4154 map_array
= kmalloc(sizeof(*map_array
) * (len
+ 2), GFP_KERNEL
);
4156 pr_warn("Unable to allocate trace enum mapping\n");
4160 mutex_lock(&trace_enum_mutex
);
4162 if (!trace_enum_maps
)
4163 trace_enum_maps
= map_array
;
4165 ptr
= trace_enum_maps
;
4167 ptr
= trace_enum_jmp_to_tail(ptr
);
4168 if (!ptr
->tail
.next
)
4170 ptr
= ptr
->tail
.next
;
4173 ptr
->tail
.next
= map_array
;
4175 map_array
->head
.mod
= mod
;
4176 map_array
->head
.length
= len
;
4179 for (map
= start
; (unsigned long)map
< (unsigned long)stop
; map
++) {
4180 map_array
->map
= **map
;
4183 memset(map_array
, 0, sizeof(*map_array
));
4185 mutex_unlock(&trace_enum_mutex
);
4188 static void trace_create_enum_file(struct dentry
*d_tracer
)
4190 trace_create_file("enum_map", 0444, d_tracer
,
4191 NULL
, &tracing_enum_map_fops
);
4194 #else /* CONFIG_TRACE_ENUM_MAP_FILE */
4195 static inline void trace_create_enum_file(struct dentry
*d_tracer
) { }
4196 static inline void trace_insert_enum_map_file(struct module
*mod
,
4197 struct trace_enum_map
**start
, int len
) { }
4198 #endif /* !CONFIG_TRACE_ENUM_MAP_FILE */
4200 static void trace_insert_enum_map(struct module
*mod
,
4201 struct trace_enum_map
**start
, int len
)
4203 struct trace_enum_map
**map
;
4210 trace_event_enum_update(map
, len
);
4212 trace_insert_enum_map_file(mod
, start
, len
);
4216 tracing_set_trace_read(struct file
*filp
, char __user
*ubuf
,
4217 size_t cnt
, loff_t
*ppos
)
4219 struct trace_array
*tr
= filp
->private_data
;
4220 char buf
[MAX_TRACER_SIZE
+2];
4223 mutex_lock(&trace_types_lock
);
4224 r
= sprintf(buf
, "%s\n", tr
->current_trace
->name
);
4225 mutex_unlock(&trace_types_lock
);
4227 return simple_read_from_buffer(ubuf
, cnt
, ppos
, buf
, r
);
4230 int tracer_init(struct tracer
*t
, struct trace_array
*tr
)
4232 tracing_reset_online_cpus(&tr
->trace_buffer
);
4236 static void set_buffer_entries(struct trace_buffer
*buf
, unsigned long val
)
4240 for_each_tracing_cpu(cpu
)
4241 per_cpu_ptr(buf
->data
, cpu
)->entries
= val
;
4244 #ifdef CONFIG_TRACER_MAX_TRACE
4245 /* resize @tr's buffer to the size of @size_tr's entries */
4246 static int resize_buffer_duplicate_size(struct trace_buffer
*trace_buf
,
4247 struct trace_buffer
*size_buf
, int cpu_id
)
4251 if (cpu_id
== RING_BUFFER_ALL_CPUS
) {
4252 for_each_tracing_cpu(cpu
) {
4253 ret
= ring_buffer_resize(trace_buf
->buffer
,
4254 per_cpu_ptr(size_buf
->data
, cpu
)->entries
, cpu
);
4257 per_cpu_ptr(trace_buf
->data
, cpu
)->entries
=
4258 per_cpu_ptr(size_buf
->data
, cpu
)->entries
;
4261 ret
= ring_buffer_resize(trace_buf
->buffer
,
4262 per_cpu_ptr(size_buf
->data
, cpu_id
)->entries
, cpu_id
);
4264 per_cpu_ptr(trace_buf
->data
, cpu_id
)->entries
=
4265 per_cpu_ptr(size_buf
->data
, cpu_id
)->entries
;
4270 #endif /* CONFIG_TRACER_MAX_TRACE */
4272 static int __tracing_resize_ring_buffer(struct trace_array
*tr
,
4273 unsigned long size
, int cpu
)
4278 * If kernel or user changes the size of the ring buffer
4279 * we use the size that was given, and we can forget about
4280 * expanding it later.
4282 ring_buffer_expanded
= true;
4284 /* May be called before buffers are initialized */
4285 if (!tr
->trace_buffer
.buffer
)
4288 ret
= ring_buffer_resize(tr
->trace_buffer
.buffer
, size
, cpu
);
4292 #ifdef CONFIG_TRACER_MAX_TRACE
4293 if (!(tr
->flags
& TRACE_ARRAY_FL_GLOBAL
) ||
4294 !tr
->current_trace
->use_max_tr
)
4297 ret
= ring_buffer_resize(tr
->max_buffer
.buffer
, size
, cpu
);
4299 int r
= resize_buffer_duplicate_size(&tr
->trace_buffer
,
4300 &tr
->trace_buffer
, cpu
);
4303 * AARGH! We are left with different
4304 * size max buffer!!!!
4305 * The max buffer is our "snapshot" buffer.
4306 * When a tracer needs a snapshot (one of the
4307 * latency tracers), it swaps the max buffer
4308 * with the saved snap shot. We succeeded to
4309 * update the size of the main buffer, but failed to
4310 * update the size of the max buffer. But when we tried
4311 * to reset the main buffer to the original size, we
4312 * failed there too. This is very unlikely to
4313 * happen, but if it does, warn and kill all
4317 tracing_disabled
= 1;
4322 if (cpu
== RING_BUFFER_ALL_CPUS
)
4323 set_buffer_entries(&tr
->max_buffer
, size
);
4325 per_cpu_ptr(tr
->max_buffer
.data
, cpu
)->entries
= size
;
4328 #endif /* CONFIG_TRACER_MAX_TRACE */
4330 if (cpu
== RING_BUFFER_ALL_CPUS
)
4331 set_buffer_entries(&tr
->trace_buffer
, size
);
4333 per_cpu_ptr(tr
->trace_buffer
.data
, cpu
)->entries
= size
;
4338 static ssize_t
tracing_resize_ring_buffer(struct trace_array
*tr
,
4339 unsigned long size
, int cpu_id
)
4343 mutex_lock(&trace_types_lock
);
4345 if (cpu_id
!= RING_BUFFER_ALL_CPUS
) {
4346 /* make sure, this cpu is enabled in the mask */
4347 if (!cpumask_test_cpu(cpu_id
, tracing_buffer_mask
)) {
4353 ret
= __tracing_resize_ring_buffer(tr
, size
, cpu_id
);
4358 mutex_unlock(&trace_types_lock
);
4365 * tracing_update_buffers - used by tracing facility to expand ring buffers
4367 * To save on memory when the tracing is never used on a system with it
4368 * configured in. The ring buffers are set to a minimum size. But once
4369 * a user starts to use the tracing facility, then they need to grow
4370 * to their default size.
4372 * This function is to be called when a tracer is about to be used.
4374 int tracing_update_buffers(void)
4378 mutex_lock(&trace_types_lock
);
4379 if (!ring_buffer_expanded
)
4380 ret
= __tracing_resize_ring_buffer(&global_trace
, trace_buf_size
,
4381 RING_BUFFER_ALL_CPUS
);
4382 mutex_unlock(&trace_types_lock
);
4387 struct trace_option_dentry
;
4390 create_trace_option_files(struct trace_array
*tr
, struct tracer
*tracer
);
4393 * Used to clear out the tracer before deletion of an instance.
4394 * Must have trace_types_lock held.
4396 static void tracing_set_nop(struct trace_array
*tr
)
4398 if (tr
->current_trace
== &nop_trace
)
4401 tr
->current_trace
->enabled
--;
4403 if (tr
->current_trace
->reset
)
4404 tr
->current_trace
->reset(tr
);
4406 tr
->current_trace
= &nop_trace
;
4409 static void add_tracer_options(struct trace_array
*tr
, struct tracer
*t
)
4411 /* Only enable if the directory has been created already. */
4415 create_trace_option_files(tr
, t
);
4418 static int tracing_set_tracer(struct trace_array
*tr
, const char *buf
)
4421 #ifdef CONFIG_TRACER_MAX_TRACE
4426 mutex_lock(&trace_types_lock
);
4428 if (!ring_buffer_expanded
) {
4429 ret
= __tracing_resize_ring_buffer(tr
, trace_buf_size
,
4430 RING_BUFFER_ALL_CPUS
);
4436 for (t
= trace_types
; t
; t
= t
->next
) {
4437 if (strcmp(t
->name
, buf
) == 0)
4444 if (t
== tr
->current_trace
)
4447 /* Some tracers are only allowed for the top level buffer */
4448 if (!trace_ok_for_array(t
, tr
)) {
4453 /* If trace pipe files are being read, we can't change the tracer */
4454 if (tr
->current_trace
->ref
) {
4459 trace_branch_disable();
4461 tr
->current_trace
->enabled
--;
4463 if (tr
->current_trace
->reset
)
4464 tr
->current_trace
->reset(tr
);
4466 /* Current trace needs to be nop_trace before synchronize_sched */
4467 tr
->current_trace
= &nop_trace
;
4469 #ifdef CONFIG_TRACER_MAX_TRACE
4470 had_max_tr
= tr
->allocated_snapshot
;
4472 if (had_max_tr
&& !t
->use_max_tr
) {
4474 * We need to make sure that the update_max_tr sees that
4475 * current_trace changed to nop_trace to keep it from
4476 * swapping the buffers after we resize it.
4477 * The update_max_tr is called from interrupts disabled
4478 * so a synchronized_sched() is sufficient.
4480 synchronize_sched();
4485 #ifdef CONFIG_TRACER_MAX_TRACE
4486 if (t
->use_max_tr
&& !had_max_tr
) {
4487 ret
= alloc_snapshot(tr
);
4494 ret
= tracer_init(t
, tr
);
4499 tr
->current_trace
= t
;
4500 tr
->current_trace
->enabled
++;
4501 trace_branch_enable(tr
);
4503 mutex_unlock(&trace_types_lock
);
4509 tracing_set_trace_write(struct file
*filp
, const char __user
*ubuf
,
4510 size_t cnt
, loff_t
*ppos
)
4512 struct trace_array
*tr
= filp
->private_data
;
4513 char buf
[MAX_TRACER_SIZE
+1];
4520 if (cnt
> MAX_TRACER_SIZE
)
4521 cnt
= MAX_TRACER_SIZE
;
4523 if (copy_from_user(buf
, ubuf
, cnt
))
4528 /* strip ending whitespace. */
4529 for (i
= cnt
- 1; i
> 0 && isspace(buf
[i
]); i
--)
4532 err
= tracing_set_tracer(tr
, buf
);
4542 tracing_nsecs_read(unsigned long *ptr
, char __user
*ubuf
,
4543 size_t cnt
, loff_t
*ppos
)
4548 r
= snprintf(buf
, sizeof(buf
), "%ld\n",
4549 *ptr
== (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr
));
4550 if (r
> sizeof(buf
))
4552 return simple_read_from_buffer(ubuf
, cnt
, ppos
, buf
, r
);
4556 tracing_nsecs_write(unsigned long *ptr
, const char __user
*ubuf
,
4557 size_t cnt
, loff_t
*ppos
)
4562 ret
= kstrtoul_from_user(ubuf
, cnt
, 10, &val
);
4572 tracing_thresh_read(struct file
*filp
, char __user
*ubuf
,
4573 size_t cnt
, loff_t
*ppos
)
4575 return tracing_nsecs_read(&tracing_thresh
, ubuf
, cnt
, ppos
);
4579 tracing_thresh_write(struct file
*filp
, const char __user
*ubuf
,
4580 size_t cnt
, loff_t
*ppos
)
4582 struct trace_array
*tr
= filp
->private_data
;
4585 mutex_lock(&trace_types_lock
);
4586 ret
= tracing_nsecs_write(&tracing_thresh
, ubuf
, cnt
, ppos
);
4590 if (tr
->current_trace
->update_thresh
) {
4591 ret
= tr
->current_trace
->update_thresh(tr
);
4598 mutex_unlock(&trace_types_lock
);
4603 #ifdef CONFIG_TRACER_MAX_TRACE
4606 tracing_max_lat_read(struct file
*filp
, char __user
*ubuf
,
4607 size_t cnt
, loff_t
*ppos
)
4609 return tracing_nsecs_read(filp
->private_data
, ubuf
, cnt
, ppos
);
4613 tracing_max_lat_write(struct file
*filp
, const char __user
*ubuf
,
4614 size_t cnt
, loff_t
*ppos
)
4616 return tracing_nsecs_write(filp
->private_data
, ubuf
, cnt
, ppos
);
4621 static int tracing_open_pipe(struct inode
*inode
, struct file
*filp
)
4623 struct trace_array
*tr
= inode
->i_private
;
4624 struct trace_iterator
*iter
;
4627 if (tracing_disabled
)
4630 if (trace_array_get(tr
) < 0)
4633 mutex_lock(&trace_types_lock
);
4635 /* create a buffer to store the information to pass to userspace */
4636 iter
= kzalloc(sizeof(*iter
), GFP_KERNEL
);
4639 __trace_array_put(tr
);
4643 trace_seq_init(&iter
->seq
);
4644 iter
->trace
= tr
->current_trace
;
4646 if (!alloc_cpumask_var(&iter
->started
, GFP_KERNEL
)) {
4651 /* trace pipe does not show start of buffer */
4652 cpumask_setall(iter
->started
);
4654 if (tr
->trace_flags
& TRACE_ITER_LATENCY_FMT
)
4655 iter
->iter_flags
|= TRACE_FILE_LAT_FMT
;
4657 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
4658 if (trace_clocks
[tr
->clock_id
].in_ns
)
4659 iter
->iter_flags
|= TRACE_FILE_TIME_IN_NS
;
4662 iter
->trace_buffer
= &tr
->trace_buffer
;
4663 iter
->cpu_file
= tracing_get_cpu(inode
);
4664 mutex_init(&iter
->mutex
);
4665 filp
->private_data
= iter
;
4667 if (iter
->trace
->pipe_open
)
4668 iter
->trace
->pipe_open(iter
);
4670 nonseekable_open(inode
, filp
);
4672 tr
->current_trace
->ref
++;
4674 mutex_unlock(&trace_types_lock
);
4680 __trace_array_put(tr
);
4681 mutex_unlock(&trace_types_lock
);
4685 static int tracing_release_pipe(struct inode
*inode
, struct file
*file
)
4687 struct trace_iterator
*iter
= file
->private_data
;
4688 struct trace_array
*tr
= inode
->i_private
;
4690 mutex_lock(&trace_types_lock
);
4692 tr
->current_trace
->ref
--;
4694 if (iter
->trace
->pipe_close
)
4695 iter
->trace
->pipe_close(iter
);
4697 mutex_unlock(&trace_types_lock
);
4699 free_cpumask_var(iter
->started
);
4700 mutex_destroy(&iter
->mutex
);
4703 trace_array_put(tr
);
4709 trace_poll(struct trace_iterator
*iter
, struct file
*filp
, poll_table
*poll_table
)
4711 struct trace_array
*tr
= iter
->tr
;
4713 /* Iterators are static, they should be filled or empty */
4714 if (trace_buffer_iter(iter
, iter
->cpu_file
))
4715 return POLLIN
| POLLRDNORM
;
4717 if (tr
->trace_flags
& TRACE_ITER_BLOCK
)
4719 * Always select as readable when in blocking mode
4721 return POLLIN
| POLLRDNORM
;
4723 return ring_buffer_poll_wait(iter
->trace_buffer
->buffer
, iter
->cpu_file
,
4728 tracing_poll_pipe(struct file
*filp
, poll_table
*poll_table
)
4730 struct trace_iterator
*iter
= filp
->private_data
;
4732 return trace_poll(iter
, filp
, poll_table
);
4735 /* Must be called with iter->mutex held. */
4736 static int tracing_wait_pipe(struct file
*filp
)
4738 struct trace_iterator
*iter
= filp
->private_data
;
4741 while (trace_empty(iter
)) {
4743 if ((filp
->f_flags
& O_NONBLOCK
)) {
4748 * We block until we read something and tracing is disabled.
4749 * We still block if tracing is disabled, but we have never
4750 * read anything. This allows a user to cat this file, and
4751 * then enable tracing. But after we have read something,
4752 * we give an EOF when tracing is again disabled.
4754 * iter->pos will be 0 if we haven't read anything.
4756 if (!tracing_is_on() && iter
->pos
)
4759 mutex_unlock(&iter
->mutex
);
4761 ret
= wait_on_pipe(iter
, false);
4763 mutex_lock(&iter
->mutex
);
4776 tracing_read_pipe(struct file
*filp
, char __user
*ubuf
,
4777 size_t cnt
, loff_t
*ppos
)
4779 struct trace_iterator
*iter
= filp
->private_data
;
4782 /* return any leftover data */
4783 sret
= trace_seq_to_user(&iter
->seq
, ubuf
, cnt
);
4787 trace_seq_init(&iter
->seq
);
4790 * Avoid more than one consumer on a single file descriptor
4791 * This is just a matter of traces coherency, the ring buffer itself
4794 mutex_lock(&iter
->mutex
);
4795 if (iter
->trace
->read
) {
4796 sret
= iter
->trace
->read(iter
, filp
, ubuf
, cnt
, ppos
);
4802 sret
= tracing_wait_pipe(filp
);
4806 /* stop when tracing is finished */
4807 if (trace_empty(iter
)) {
4812 if (cnt
>= PAGE_SIZE
)
4813 cnt
= PAGE_SIZE
- 1;
4815 /* reset all but tr, trace, and overruns */
4816 memset(&iter
->seq
, 0,
4817 sizeof(struct trace_iterator
) -
4818 offsetof(struct trace_iterator
, seq
));
4819 cpumask_clear(iter
->started
);
4822 trace_event_read_lock();
4823 trace_access_lock(iter
->cpu_file
);
4824 while (trace_find_next_entry_inc(iter
) != NULL
) {
4825 enum print_line_t ret
;
4826 int save_len
= iter
->seq
.seq
.len
;
4828 ret
= print_trace_line(iter
);
4829 if (ret
== TRACE_TYPE_PARTIAL_LINE
) {
4830 /* don't print partial lines */
4831 iter
->seq
.seq
.len
= save_len
;
4834 if (ret
!= TRACE_TYPE_NO_CONSUME
)
4835 trace_consume(iter
);
4837 if (trace_seq_used(&iter
->seq
) >= cnt
)
4841 * Setting the full flag means we reached the trace_seq buffer
4842 * size and we should leave by partial output condition above.
4843 * One of the trace_seq_* functions is not used properly.
4845 WARN_ONCE(iter
->seq
.full
, "full flag set for trace type %d",
4848 trace_access_unlock(iter
->cpu_file
);
4849 trace_event_read_unlock();
4851 /* Now copy what we have to the user */
4852 sret
= trace_seq_to_user(&iter
->seq
, ubuf
, cnt
);
4853 if (iter
->seq
.seq
.readpos
>= trace_seq_used(&iter
->seq
))
4854 trace_seq_init(&iter
->seq
);
4857 * If there was nothing to send to user, in spite of consuming trace
4858 * entries, go back to wait for more entries.
4864 mutex_unlock(&iter
->mutex
);
4869 static void tracing_spd_release_pipe(struct splice_pipe_desc
*spd
,
4872 __free_page(spd
->pages
[idx
]);
4875 static const struct pipe_buf_operations tracing_pipe_buf_ops
= {
4877 .confirm
= generic_pipe_buf_confirm
,
4878 .release
= generic_pipe_buf_release
,
4879 .steal
= generic_pipe_buf_steal
,
4880 .get
= generic_pipe_buf_get
,
4884 tracing_fill_pipe_page(size_t rem
, struct trace_iterator
*iter
)
4890 /* Seq buffer is page-sized, exactly what we need. */
4892 save_len
= iter
->seq
.seq
.len
;
4893 ret
= print_trace_line(iter
);
4895 if (trace_seq_has_overflowed(&iter
->seq
)) {
4896 iter
->seq
.seq
.len
= save_len
;
4901 * This should not be hit, because it should only
4902 * be set if the iter->seq overflowed. But check it
4903 * anyway to be safe.
4905 if (ret
== TRACE_TYPE_PARTIAL_LINE
) {
4906 iter
->seq
.seq
.len
= save_len
;
4910 count
= trace_seq_used(&iter
->seq
) - save_len
;
4913 iter
->seq
.seq
.len
= save_len
;
4917 if (ret
!= TRACE_TYPE_NO_CONSUME
)
4918 trace_consume(iter
);
4920 if (!trace_find_next_entry_inc(iter
)) {
4930 static ssize_t
tracing_splice_read_pipe(struct file
*filp
,
4932 struct pipe_inode_info
*pipe
,
4936 struct page
*pages_def
[PIPE_DEF_BUFFERS
];
4937 struct partial_page partial_def
[PIPE_DEF_BUFFERS
];
4938 struct trace_iterator
*iter
= filp
->private_data
;
4939 struct splice_pipe_desc spd
= {
4941 .partial
= partial_def
,
4942 .nr_pages
= 0, /* This gets updated below. */
4943 .nr_pages_max
= PIPE_DEF_BUFFERS
,
4945 .ops
= &tracing_pipe_buf_ops
,
4946 .spd_release
= tracing_spd_release_pipe
,
4952 if (splice_grow_spd(pipe
, &spd
))
4955 mutex_lock(&iter
->mutex
);
4957 if (iter
->trace
->splice_read
) {
4958 ret
= iter
->trace
->splice_read(iter
, filp
,
4959 ppos
, pipe
, len
, flags
);
4964 ret
= tracing_wait_pipe(filp
);
4968 if (!iter
->ent
&& !trace_find_next_entry_inc(iter
)) {
4973 trace_event_read_lock();
4974 trace_access_lock(iter
->cpu_file
);
4976 /* Fill as many pages as possible. */
4977 for (i
= 0, rem
= len
; i
< spd
.nr_pages_max
&& rem
; i
++) {
4978 spd
.pages
[i
] = alloc_page(GFP_KERNEL
);
4982 rem
= tracing_fill_pipe_page(rem
, iter
);
4984 /* Copy the data into the page, so we can start over. */
4985 ret
= trace_seq_to_buffer(&iter
->seq
,
4986 page_address(spd
.pages
[i
]),
4987 trace_seq_used(&iter
->seq
));
4989 __free_page(spd
.pages
[i
]);
4992 spd
.partial
[i
].offset
= 0;
4993 spd
.partial
[i
].len
= trace_seq_used(&iter
->seq
);
4995 trace_seq_init(&iter
->seq
);
4998 trace_access_unlock(iter
->cpu_file
);
4999 trace_event_read_unlock();
5000 mutex_unlock(&iter
->mutex
);
5005 ret
= splice_to_pipe(pipe
, &spd
);
5009 splice_shrink_spd(&spd
);
5013 mutex_unlock(&iter
->mutex
);
5018 tracing_entries_read(struct file
*filp
, char __user
*ubuf
,
5019 size_t cnt
, loff_t
*ppos
)
5021 struct inode
*inode
= file_inode(filp
);
5022 struct trace_array
*tr
= inode
->i_private
;
5023 int cpu
= tracing_get_cpu(inode
);
5028 mutex_lock(&trace_types_lock
);
5030 if (cpu
== RING_BUFFER_ALL_CPUS
) {
5031 int cpu
, buf_size_same
;
5036 /* check if all cpu sizes are same */
5037 for_each_tracing_cpu(cpu
) {
5038 /* fill in the size from first enabled cpu */
5040 size
= per_cpu_ptr(tr
->trace_buffer
.data
, cpu
)->entries
;
5041 if (size
!= per_cpu_ptr(tr
->trace_buffer
.data
, cpu
)->entries
) {
5047 if (buf_size_same
) {
5048 if (!ring_buffer_expanded
)
5049 r
= sprintf(buf
, "%lu (expanded: %lu)\n",
5051 trace_buf_size
>> 10);
5053 r
= sprintf(buf
, "%lu\n", size
>> 10);
5055 r
= sprintf(buf
, "X\n");
5057 r
= sprintf(buf
, "%lu\n", per_cpu_ptr(tr
->trace_buffer
.data
, cpu
)->entries
>> 10);
5059 mutex_unlock(&trace_types_lock
);
5061 ret
= simple_read_from_buffer(ubuf
, cnt
, ppos
, buf
, r
);
5066 tracing_entries_write(struct file
*filp
, const char __user
*ubuf
,
5067 size_t cnt
, loff_t
*ppos
)
5069 struct inode
*inode
= file_inode(filp
);
5070 struct trace_array
*tr
= inode
->i_private
;
5074 ret
= kstrtoul_from_user(ubuf
, cnt
, 10, &val
);
5078 /* must have at least 1 entry */
5082 /* value is in KB */
5084 ret
= tracing_resize_ring_buffer(tr
, val
, tracing_get_cpu(inode
));
5094 tracing_total_entries_read(struct file
*filp
, char __user
*ubuf
,
5095 size_t cnt
, loff_t
*ppos
)
5097 struct trace_array
*tr
= filp
->private_data
;
5100 unsigned long size
= 0, expanded_size
= 0;
5102 mutex_lock(&trace_types_lock
);
5103 for_each_tracing_cpu(cpu
) {
5104 size
+= per_cpu_ptr(tr
->trace_buffer
.data
, cpu
)->entries
>> 10;
5105 if (!ring_buffer_expanded
)
5106 expanded_size
+= trace_buf_size
>> 10;
5108 if (ring_buffer_expanded
)
5109 r
= sprintf(buf
, "%lu\n", size
);
5111 r
= sprintf(buf
, "%lu (expanded: %lu)\n", size
, expanded_size
);
5112 mutex_unlock(&trace_types_lock
);
5114 return simple_read_from_buffer(ubuf
, cnt
, ppos
, buf
, r
);
5118 tracing_free_buffer_write(struct file
*filp
, const char __user
*ubuf
,
5119 size_t cnt
, loff_t
*ppos
)
5122 * There is no need to read what the user has written, this function
5123 * is just to make sure that there is no error when "echo" is used
5132 tracing_free_buffer_release(struct inode
*inode
, struct file
*filp
)
5134 struct trace_array
*tr
= inode
->i_private
;
5136 /* disable tracing ? */
5137 if (tr
->trace_flags
& TRACE_ITER_STOP_ON_FREE
)
5138 tracer_tracing_off(tr
);
5139 /* resize the ring buffer to 0 */
5140 tracing_resize_ring_buffer(tr
, 0, RING_BUFFER_ALL_CPUS
);
5142 trace_array_put(tr
);
5148 tracing_mark_write(struct file
*filp
, const char __user
*ubuf
,
5149 size_t cnt
, loff_t
*fpos
)
5151 unsigned long addr
= (unsigned long)ubuf
;
5152 struct trace_array
*tr
= filp
->private_data
;
5153 struct ring_buffer_event
*event
;
5154 struct ring_buffer
*buffer
;
5155 struct print_entry
*entry
;
5156 unsigned long irq_flags
;
5157 struct page
*pages
[2];
5167 if (tracing_disabled
)
5170 if (!(tr
->trace_flags
& TRACE_ITER_MARKERS
))
5173 if (cnt
> TRACE_BUF_SIZE
)
5174 cnt
= TRACE_BUF_SIZE
;
5177 * Userspace is injecting traces into the kernel trace buffer.
5178 * We want to be as non intrusive as possible.
5179 * To do so, we do not want to allocate any special buffers
5180 * or take any locks, but instead write the userspace data
5181 * straight into the ring buffer.
5183 * First we need to pin the userspace buffer into memory,
5184 * which, most likely it is, because it just referenced it.
5185 * But there's no guarantee that it is. By using get_user_pages_fast()
5186 * and kmap_atomic/kunmap_atomic() we can get access to the
5187 * pages directly. We then write the data directly into the
5190 BUILD_BUG_ON(TRACE_BUF_SIZE
>= PAGE_SIZE
);
5192 /* check if we cross pages */
5193 if ((addr
& PAGE_MASK
) != ((addr
+ cnt
) & PAGE_MASK
))
5196 offset
= addr
& (PAGE_SIZE
- 1);
5199 ret
= get_user_pages_fast(addr
, nr_pages
, 0, pages
);
5200 if (ret
< nr_pages
) {
5202 put_page(pages
[ret
]);
5207 for (i
= 0; i
< nr_pages
; i
++)
5208 map_page
[i
] = kmap_atomic(pages
[i
]);
5210 local_save_flags(irq_flags
);
5211 size
= sizeof(*entry
) + cnt
+ 2; /* possible \n added */
5212 buffer
= tr
->trace_buffer
.buffer
;
5213 event
= trace_buffer_lock_reserve(buffer
, TRACE_PRINT
, size
,
5214 irq_flags
, preempt_count());
5216 /* Ring buffer disabled, return as if not open for write */
5221 entry
= ring_buffer_event_data(event
);
5222 entry
->ip
= _THIS_IP_
;
5224 if (nr_pages
== 2) {
5225 len
= PAGE_SIZE
- offset
;
5226 memcpy(&entry
->buf
, map_page
[0] + offset
, len
);
5227 memcpy(&entry
->buf
[len
], map_page
[1], cnt
- len
);
5229 memcpy(&entry
->buf
, map_page
[0] + offset
, cnt
);
5231 if (entry
->buf
[cnt
- 1] != '\n') {
5232 entry
->buf
[cnt
] = '\n';
5233 entry
->buf
[cnt
+ 1] = '\0';
5235 entry
->buf
[cnt
] = '\0';
5237 __buffer_unlock_commit(buffer
, event
);
5244 for (i
= nr_pages
- 1; i
>= 0; i
--) {
5245 kunmap_atomic(map_page
[i
]);
5252 static int tracing_clock_show(struct seq_file
*m
, void *v
)
5254 struct trace_array
*tr
= m
->private;
5257 for (i
= 0; i
< ARRAY_SIZE(trace_clocks
); i
++)
5259 "%s%s%s%s", i
? " " : "",
5260 i
== tr
->clock_id
? "[" : "", trace_clocks
[i
].name
,
5261 i
== tr
->clock_id
? "]" : "");
5267 static int tracing_set_clock(struct trace_array
*tr
, const char *clockstr
)
5271 for (i
= 0; i
< ARRAY_SIZE(trace_clocks
); i
++) {
5272 if (strcmp(trace_clocks
[i
].name
, clockstr
) == 0)
5275 if (i
== ARRAY_SIZE(trace_clocks
))
5278 mutex_lock(&trace_types_lock
);
5282 ring_buffer_set_clock(tr
->trace_buffer
.buffer
, trace_clocks
[i
].func
);
5285 * New clock may not be consistent with the previous clock.
5286 * Reset the buffer so that it doesn't have incomparable timestamps.
5288 tracing_reset_online_cpus(&tr
->trace_buffer
);
5290 #ifdef CONFIG_TRACER_MAX_TRACE
5291 if (tr
->flags
& TRACE_ARRAY_FL_GLOBAL
&& tr
->max_buffer
.buffer
)
5292 ring_buffer_set_clock(tr
->max_buffer
.buffer
, trace_clocks
[i
].func
);
5293 tracing_reset_online_cpus(&tr
->max_buffer
);
5296 mutex_unlock(&trace_types_lock
);
5301 static ssize_t
tracing_clock_write(struct file
*filp
, const char __user
*ubuf
,
5302 size_t cnt
, loff_t
*fpos
)
5304 struct seq_file
*m
= filp
->private_data
;
5305 struct trace_array
*tr
= m
->private;
5307 const char *clockstr
;
5310 if (cnt
>= sizeof(buf
))
5313 if (copy_from_user(buf
, ubuf
, cnt
))
5318 clockstr
= strstrip(buf
);
5320 ret
= tracing_set_clock(tr
, clockstr
);
5329 static int tracing_clock_open(struct inode
*inode
, struct file
*file
)
5331 struct trace_array
*tr
= inode
->i_private
;
5334 if (tracing_disabled
)
5337 if (trace_array_get(tr
))
5340 ret
= single_open(file
, tracing_clock_show
, inode
->i_private
);
5342 trace_array_put(tr
);
5347 struct ftrace_buffer_info
{
5348 struct trace_iterator iter
;
5353 #ifdef CONFIG_TRACER_SNAPSHOT
5354 static int tracing_snapshot_open(struct inode
*inode
, struct file
*file
)
5356 struct trace_array
*tr
= inode
->i_private
;
5357 struct trace_iterator
*iter
;
5361 if (trace_array_get(tr
) < 0)
5364 if (file
->f_mode
& FMODE_READ
) {
5365 iter
= __tracing_open(inode
, file
, true);
5367 ret
= PTR_ERR(iter
);
5369 /* Writes still need the seq_file to hold the private data */
5371 m
= kzalloc(sizeof(*m
), GFP_KERNEL
);
5374 iter
= kzalloc(sizeof(*iter
), GFP_KERNEL
);
5382 iter
->trace_buffer
= &tr
->max_buffer
;
5383 iter
->cpu_file
= tracing_get_cpu(inode
);
5385 file
->private_data
= m
;
5389 trace_array_put(tr
);
5395 tracing_snapshot_write(struct file
*filp
, const char __user
*ubuf
, size_t cnt
,
5398 struct seq_file
*m
= filp
->private_data
;
5399 struct trace_iterator
*iter
= m
->private;
5400 struct trace_array
*tr
= iter
->tr
;
5404 ret
= tracing_update_buffers();
5408 ret
= kstrtoul_from_user(ubuf
, cnt
, 10, &val
);
5412 mutex_lock(&trace_types_lock
);
5414 if (tr
->current_trace
->use_max_tr
) {
5421 if (iter
->cpu_file
!= RING_BUFFER_ALL_CPUS
) {
5425 if (tr
->allocated_snapshot
)
5429 /* Only allow per-cpu swap if the ring buffer supports it */
5430 #ifndef CONFIG_RING_BUFFER_ALLOW_SWAP
5431 if (iter
->cpu_file
!= RING_BUFFER_ALL_CPUS
) {
5436 if (!tr
->allocated_snapshot
) {
5437 ret
= alloc_snapshot(tr
);
5441 local_irq_disable();
5442 /* Now, we're going to swap */
5443 if (iter
->cpu_file
== RING_BUFFER_ALL_CPUS
)
5444 update_max_tr(tr
, current
, smp_processor_id());
5446 update_max_tr_single(tr
, current
, iter
->cpu_file
);
5450 if (tr
->allocated_snapshot
) {
5451 if (iter
->cpu_file
== RING_BUFFER_ALL_CPUS
)
5452 tracing_reset_online_cpus(&tr
->max_buffer
);
5454 tracing_reset(&tr
->max_buffer
, iter
->cpu_file
);
5464 mutex_unlock(&trace_types_lock
);
5468 static int tracing_snapshot_release(struct inode
*inode
, struct file
*file
)
5470 struct seq_file
*m
= file
->private_data
;
5473 ret
= tracing_release(inode
, file
);
5475 if (file
->f_mode
& FMODE_READ
)
5478 /* If write only, the seq_file is just a stub */
5486 static int tracing_buffers_open(struct inode
*inode
, struct file
*filp
);
5487 static ssize_t
tracing_buffers_read(struct file
*filp
, char __user
*ubuf
,
5488 size_t count
, loff_t
*ppos
);
5489 static int tracing_buffers_release(struct inode
*inode
, struct file
*file
);
5490 static ssize_t
tracing_buffers_splice_read(struct file
*file
, loff_t
*ppos
,
5491 struct pipe_inode_info
*pipe
, size_t len
, unsigned int flags
);
5493 static int snapshot_raw_open(struct inode
*inode
, struct file
*filp
)
5495 struct ftrace_buffer_info
*info
;
5498 ret
= tracing_buffers_open(inode
, filp
);
5502 info
= filp
->private_data
;
5504 if (info
->iter
.trace
->use_max_tr
) {
5505 tracing_buffers_release(inode
, filp
);
5509 info
->iter
.snapshot
= true;
5510 info
->iter
.trace_buffer
= &info
->iter
.tr
->max_buffer
;
5515 #endif /* CONFIG_TRACER_SNAPSHOT */
5518 static const struct file_operations tracing_thresh_fops
= {
5519 .open
= tracing_open_generic
,
5520 .read
= tracing_thresh_read
,
5521 .write
= tracing_thresh_write
,
5522 .llseek
= generic_file_llseek
,
5525 #ifdef CONFIG_TRACER_MAX_TRACE
5526 static const struct file_operations tracing_max_lat_fops
= {
5527 .open
= tracing_open_generic
,
5528 .read
= tracing_max_lat_read
,
5529 .write
= tracing_max_lat_write
,
5530 .llseek
= generic_file_llseek
,
5534 static const struct file_operations set_tracer_fops
= {
5535 .open
= tracing_open_generic
,
5536 .read
= tracing_set_trace_read
,
5537 .write
= tracing_set_trace_write
,
5538 .llseek
= generic_file_llseek
,
5541 static const struct file_operations tracing_pipe_fops
= {
5542 .open
= tracing_open_pipe
,
5543 .poll
= tracing_poll_pipe
,
5544 .read
= tracing_read_pipe
,
5545 .splice_read
= tracing_splice_read_pipe
,
5546 .release
= tracing_release_pipe
,
5547 .llseek
= no_llseek
,
5550 static const struct file_operations tracing_entries_fops
= {
5551 .open
= tracing_open_generic_tr
,
5552 .read
= tracing_entries_read
,
5553 .write
= tracing_entries_write
,
5554 .llseek
= generic_file_llseek
,
5555 .release
= tracing_release_generic_tr
,
5558 static const struct file_operations tracing_total_entries_fops
= {
5559 .open
= tracing_open_generic_tr
,
5560 .read
= tracing_total_entries_read
,
5561 .llseek
= generic_file_llseek
,
5562 .release
= tracing_release_generic_tr
,
5565 static const struct file_operations tracing_free_buffer_fops
= {
5566 .open
= tracing_open_generic_tr
,
5567 .write
= tracing_free_buffer_write
,
5568 .release
= tracing_free_buffer_release
,
5571 static const struct file_operations tracing_mark_fops
= {
5572 .open
= tracing_open_generic_tr
,
5573 .write
= tracing_mark_write
,
5574 .llseek
= generic_file_llseek
,
5575 .release
= tracing_release_generic_tr
,
5578 static const struct file_operations trace_clock_fops
= {
5579 .open
= tracing_clock_open
,
5581 .llseek
= seq_lseek
,
5582 .release
= tracing_single_release_tr
,
5583 .write
= tracing_clock_write
,
5586 #ifdef CONFIG_TRACER_SNAPSHOT
5587 static const struct file_operations snapshot_fops
= {
5588 .open
= tracing_snapshot_open
,
5590 .write
= tracing_snapshot_write
,
5591 .llseek
= tracing_lseek
,
5592 .release
= tracing_snapshot_release
,
5595 static const struct file_operations snapshot_raw_fops
= {
5596 .open
= snapshot_raw_open
,
5597 .read
= tracing_buffers_read
,
5598 .release
= tracing_buffers_release
,
5599 .splice_read
= tracing_buffers_splice_read
,
5600 .llseek
= no_llseek
,
5603 #endif /* CONFIG_TRACER_SNAPSHOT */
5605 static int tracing_buffers_open(struct inode
*inode
, struct file
*filp
)
5607 struct trace_array
*tr
= inode
->i_private
;
5608 struct ftrace_buffer_info
*info
;
5611 if (tracing_disabled
)
5614 if (trace_array_get(tr
) < 0)
5617 info
= kzalloc(sizeof(*info
), GFP_KERNEL
);
5619 trace_array_put(tr
);
5623 mutex_lock(&trace_types_lock
);
5626 info
->iter
.cpu_file
= tracing_get_cpu(inode
);
5627 info
->iter
.trace
= tr
->current_trace
;
5628 info
->iter
.trace_buffer
= &tr
->trace_buffer
;
5630 /* Force reading ring buffer for first read */
5631 info
->read
= (unsigned int)-1;
5633 filp
->private_data
= info
;
5635 tr
->current_trace
->ref
++;
5637 mutex_unlock(&trace_types_lock
);
5639 ret
= nonseekable_open(inode
, filp
);
5641 trace_array_put(tr
);
5647 tracing_buffers_poll(struct file
*filp
, poll_table
*poll_table
)
5649 struct ftrace_buffer_info
*info
= filp
->private_data
;
5650 struct trace_iterator
*iter
= &info
->iter
;
5652 return trace_poll(iter
, filp
, poll_table
);
5656 tracing_buffers_read(struct file
*filp
, char __user
*ubuf
,
5657 size_t count
, loff_t
*ppos
)
5659 struct ftrace_buffer_info
*info
= filp
->private_data
;
5660 struct trace_iterator
*iter
= &info
->iter
;
5667 #ifdef CONFIG_TRACER_MAX_TRACE
5668 if (iter
->snapshot
&& iter
->tr
->current_trace
->use_max_tr
)
5673 info
->spare
= ring_buffer_alloc_read_page(iter
->trace_buffer
->buffer
,
5678 /* Do we have previous read data to read? */
5679 if (info
->read
< PAGE_SIZE
)
5683 trace_access_lock(iter
->cpu_file
);
5684 ret
= ring_buffer_read_page(iter
->trace_buffer
->buffer
,
5688 trace_access_unlock(iter
->cpu_file
);
5691 if (trace_empty(iter
)) {
5692 if ((filp
->f_flags
& O_NONBLOCK
))
5695 ret
= wait_on_pipe(iter
, false);
5706 size
= PAGE_SIZE
- info
->read
;
5710 ret
= copy_to_user(ubuf
, info
->spare
+ info
->read
, size
);
5722 static int tracing_buffers_release(struct inode
*inode
, struct file
*file
)
5724 struct ftrace_buffer_info
*info
= file
->private_data
;
5725 struct trace_iterator
*iter
= &info
->iter
;
5727 mutex_lock(&trace_types_lock
);
5729 iter
->tr
->current_trace
->ref
--;
5731 __trace_array_put(iter
->tr
);
5734 ring_buffer_free_read_page(iter
->trace_buffer
->buffer
, info
->spare
);
5737 mutex_unlock(&trace_types_lock
);
5743 struct ring_buffer
*buffer
;
5748 static void buffer_pipe_buf_release(struct pipe_inode_info
*pipe
,
5749 struct pipe_buffer
*buf
)
5751 struct buffer_ref
*ref
= (struct buffer_ref
*)buf
->private;
5756 ring_buffer_free_read_page(ref
->buffer
, ref
->page
);
5761 static void buffer_pipe_buf_get(struct pipe_inode_info
*pipe
,
5762 struct pipe_buffer
*buf
)
5764 struct buffer_ref
*ref
= (struct buffer_ref
*)buf
->private;
5769 /* Pipe buffer operations for a buffer. */
5770 static const struct pipe_buf_operations buffer_pipe_buf_ops
= {
5772 .confirm
= generic_pipe_buf_confirm
,
5773 .release
= buffer_pipe_buf_release
,
5774 .steal
= generic_pipe_buf_steal
,
5775 .get
= buffer_pipe_buf_get
,
5779 * Callback from splice_to_pipe(), if we need to release some pages
5780 * at the end of the spd in case we error'ed out in filling the pipe.
5782 static void buffer_spd_release(struct splice_pipe_desc
*spd
, unsigned int i
)
5784 struct buffer_ref
*ref
=
5785 (struct buffer_ref
*)spd
->partial
[i
].private;
5790 ring_buffer_free_read_page(ref
->buffer
, ref
->page
);
5792 spd
->partial
[i
].private = 0;
5796 tracing_buffers_splice_read(struct file
*file
, loff_t
*ppos
,
5797 struct pipe_inode_info
*pipe
, size_t len
,
5800 struct ftrace_buffer_info
*info
= file
->private_data
;
5801 struct trace_iterator
*iter
= &info
->iter
;
5802 struct partial_page partial_def
[PIPE_DEF_BUFFERS
];
5803 struct page
*pages_def
[PIPE_DEF_BUFFERS
];
5804 struct splice_pipe_desc spd
= {
5806 .partial
= partial_def
,
5807 .nr_pages_max
= PIPE_DEF_BUFFERS
,
5809 .ops
= &buffer_pipe_buf_ops
,
5810 .spd_release
= buffer_spd_release
,
5812 struct buffer_ref
*ref
;
5813 int entries
, size
, i
;
5816 #ifdef CONFIG_TRACER_MAX_TRACE
5817 if (iter
->snapshot
&& iter
->tr
->current_trace
->use_max_tr
)
5821 if (splice_grow_spd(pipe
, &spd
))
5824 if (*ppos
& (PAGE_SIZE
- 1))
5827 if (len
& (PAGE_SIZE
- 1)) {
5828 if (len
< PAGE_SIZE
)
5834 trace_access_lock(iter
->cpu_file
);
5835 entries
= ring_buffer_entries_cpu(iter
->trace_buffer
->buffer
, iter
->cpu_file
);
5837 for (i
= 0; i
< spd
.nr_pages_max
&& len
&& entries
; i
++, len
-= PAGE_SIZE
) {
5841 ref
= kzalloc(sizeof(*ref
), GFP_KERNEL
);
5848 ref
->buffer
= iter
->trace_buffer
->buffer
;
5849 ref
->page
= ring_buffer_alloc_read_page(ref
->buffer
, iter
->cpu_file
);
5856 r
= ring_buffer_read_page(ref
->buffer
, &ref
->page
,
5857 len
, iter
->cpu_file
, 1);
5859 ring_buffer_free_read_page(ref
->buffer
, ref
->page
);
5865 * zero out any left over data, this is going to
5868 size
= ring_buffer_page_len(ref
->page
);
5869 if (size
< PAGE_SIZE
)
5870 memset(ref
->page
+ size
, 0, PAGE_SIZE
- size
);
5872 page
= virt_to_page(ref
->page
);
5874 spd
.pages
[i
] = page
;
5875 spd
.partial
[i
].len
= PAGE_SIZE
;
5876 spd
.partial
[i
].offset
= 0;
5877 spd
.partial
[i
].private = (unsigned long)ref
;
5881 entries
= ring_buffer_entries_cpu(iter
->trace_buffer
->buffer
, iter
->cpu_file
);
5884 trace_access_unlock(iter
->cpu_file
);
5887 /* did we read anything? */
5888 if (!spd
.nr_pages
) {
5892 if ((file
->f_flags
& O_NONBLOCK
) || (flags
& SPLICE_F_NONBLOCK
))
5895 ret
= wait_on_pipe(iter
, true);
5902 ret
= splice_to_pipe(pipe
, &spd
);
5903 splice_shrink_spd(&spd
);
5908 static const struct file_operations tracing_buffers_fops
= {
5909 .open
= tracing_buffers_open
,
5910 .read
= tracing_buffers_read
,
5911 .poll
= tracing_buffers_poll
,
5912 .release
= tracing_buffers_release
,
5913 .splice_read
= tracing_buffers_splice_read
,
5914 .llseek
= no_llseek
,
5918 tracing_stats_read(struct file
*filp
, char __user
*ubuf
,
5919 size_t count
, loff_t
*ppos
)
5921 struct inode
*inode
= file_inode(filp
);
5922 struct trace_array
*tr
= inode
->i_private
;
5923 struct trace_buffer
*trace_buf
= &tr
->trace_buffer
;
5924 int cpu
= tracing_get_cpu(inode
);
5925 struct trace_seq
*s
;
5927 unsigned long long t
;
5928 unsigned long usec_rem
;
5930 s
= kmalloc(sizeof(*s
), GFP_KERNEL
);
5936 cnt
= ring_buffer_entries_cpu(trace_buf
->buffer
, cpu
);
5937 trace_seq_printf(s
, "entries: %ld\n", cnt
);
5939 cnt
= ring_buffer_overrun_cpu(trace_buf
->buffer
, cpu
);
5940 trace_seq_printf(s
, "overrun: %ld\n", cnt
);
5942 cnt
= ring_buffer_commit_overrun_cpu(trace_buf
->buffer
, cpu
);
5943 trace_seq_printf(s
, "commit overrun: %ld\n", cnt
);
5945 cnt
= ring_buffer_bytes_cpu(trace_buf
->buffer
, cpu
);
5946 trace_seq_printf(s
, "bytes: %ld\n", cnt
);
5948 if (trace_clocks
[tr
->clock_id
].in_ns
) {
5949 /* local or global for trace_clock */
5950 t
= ns2usecs(ring_buffer_oldest_event_ts(trace_buf
->buffer
, cpu
));
5951 usec_rem
= do_div(t
, USEC_PER_SEC
);
5952 trace_seq_printf(s
, "oldest event ts: %5llu.%06lu\n",
5955 t
= ns2usecs(ring_buffer_time_stamp(trace_buf
->buffer
, cpu
));
5956 usec_rem
= do_div(t
, USEC_PER_SEC
);
5957 trace_seq_printf(s
, "now ts: %5llu.%06lu\n", t
, usec_rem
);
5959 /* counter or tsc mode for trace_clock */
5960 trace_seq_printf(s
, "oldest event ts: %llu\n",
5961 ring_buffer_oldest_event_ts(trace_buf
->buffer
, cpu
));
5963 trace_seq_printf(s
, "now ts: %llu\n",
5964 ring_buffer_time_stamp(trace_buf
->buffer
, cpu
));
5967 cnt
= ring_buffer_dropped_events_cpu(trace_buf
->buffer
, cpu
);
5968 trace_seq_printf(s
, "dropped events: %ld\n", cnt
);
5970 cnt
= ring_buffer_read_events_cpu(trace_buf
->buffer
, cpu
);
5971 trace_seq_printf(s
, "read events: %ld\n", cnt
);
5973 count
= simple_read_from_buffer(ubuf
, count
, ppos
,
5974 s
->buffer
, trace_seq_used(s
));
5981 static const struct file_operations tracing_stats_fops
= {
5982 .open
= tracing_open_generic_tr
,
5983 .read
= tracing_stats_read
,
5984 .llseek
= generic_file_llseek
,
5985 .release
= tracing_release_generic_tr
,
5988 #ifdef CONFIG_DYNAMIC_FTRACE
5990 int __weak
ftrace_arch_read_dyn_info(char *buf
, int size
)
5996 tracing_read_dyn_info(struct file
*filp
, char __user
*ubuf
,
5997 size_t cnt
, loff_t
*ppos
)
5999 static char ftrace_dyn_info_buffer
[1024];
6000 static DEFINE_MUTEX(dyn_info_mutex
);
6001 unsigned long *p
= filp
->private_data
;
6002 char *buf
= ftrace_dyn_info_buffer
;
6003 int size
= ARRAY_SIZE(ftrace_dyn_info_buffer
);
6006 mutex_lock(&dyn_info_mutex
);
6007 r
= sprintf(buf
, "%ld ", *p
);
6009 r
+= ftrace_arch_read_dyn_info(buf
+r
, (size
-1)-r
);
6012 r
= simple_read_from_buffer(ubuf
, cnt
, ppos
, buf
, r
);
6014 mutex_unlock(&dyn_info_mutex
);
6019 static const struct file_operations tracing_dyn_info_fops
= {
6020 .open
= tracing_open_generic
,
6021 .read
= tracing_read_dyn_info
,
6022 .llseek
= generic_file_llseek
,
6024 #endif /* CONFIG_DYNAMIC_FTRACE */
6026 #if defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE)
6028 ftrace_snapshot(unsigned long ip
, unsigned long parent_ip
, void **data
)
6034 ftrace_count_snapshot(unsigned long ip
, unsigned long parent_ip
, void **data
)
6036 unsigned long *count
= (long *)data
;
6048 ftrace_snapshot_print(struct seq_file
*m
, unsigned long ip
,
6049 struct ftrace_probe_ops
*ops
, void *data
)
6051 long count
= (long)data
;
6053 seq_printf(m
, "%ps:", (void *)ip
);
6055 seq_puts(m
, "snapshot");
6058 seq_puts(m
, ":unlimited\n");
6060 seq_printf(m
, ":count=%ld\n", count
);
6065 static struct ftrace_probe_ops snapshot_probe_ops
= {
6066 .func
= ftrace_snapshot
,
6067 .print
= ftrace_snapshot_print
,
6070 static struct ftrace_probe_ops snapshot_count_probe_ops
= {
6071 .func
= ftrace_count_snapshot
,
6072 .print
= ftrace_snapshot_print
,
6076 ftrace_trace_snapshot_callback(struct ftrace_hash
*hash
,
6077 char *glob
, char *cmd
, char *param
, int enable
)
6079 struct ftrace_probe_ops
*ops
;
6080 void *count
= (void *)-1;
6084 /* hash funcs only work with set_ftrace_filter */
6088 ops
= param
? &snapshot_count_probe_ops
: &snapshot_probe_ops
;
6090 if (glob
[0] == '!') {
6091 unregister_ftrace_function_probe_func(glob
+1, ops
);
6098 number
= strsep(¶m
, ":");
6100 if (!strlen(number
))
6104 * We use the callback data field (which is a pointer)
6107 ret
= kstrtoul(number
, 0, (unsigned long *)&count
);
6112 ret
= register_ftrace_function_probe(glob
, ops
, count
);
6115 alloc_snapshot(&global_trace
);
6117 return ret
< 0 ? ret
: 0;
6120 static struct ftrace_func_command ftrace_snapshot_cmd
= {
6122 .func
= ftrace_trace_snapshot_callback
,
6125 static __init
int register_snapshot_cmd(void)
6127 return register_ftrace_command(&ftrace_snapshot_cmd
);
6130 static inline __init
int register_snapshot_cmd(void) { return 0; }
6131 #endif /* defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) */
6133 static struct dentry
*tracing_get_dentry(struct trace_array
*tr
)
6135 if (WARN_ON(!tr
->dir
))
6136 return ERR_PTR(-ENODEV
);
6138 /* Top directory uses NULL as the parent */
6139 if (tr
->flags
& TRACE_ARRAY_FL_GLOBAL
)
6142 /* All sub buffers have a descriptor */
6146 static struct dentry
*tracing_dentry_percpu(struct trace_array
*tr
, int cpu
)
6148 struct dentry
*d_tracer
;
6151 return tr
->percpu_dir
;
6153 d_tracer
= tracing_get_dentry(tr
);
6154 if (IS_ERR(d_tracer
))
6157 tr
->percpu_dir
= tracefs_create_dir("per_cpu", d_tracer
);
6159 WARN_ONCE(!tr
->percpu_dir
,
6160 "Could not create tracefs directory 'per_cpu/%d'\n", cpu
);
6162 return tr
->percpu_dir
;
6165 static struct dentry
*
6166 trace_create_cpu_file(const char *name
, umode_t mode
, struct dentry
*parent
,
6167 void *data
, long cpu
, const struct file_operations
*fops
)
6169 struct dentry
*ret
= trace_create_file(name
, mode
, parent
, data
, fops
);
6171 if (ret
) /* See tracing_get_cpu() */
6172 d_inode(ret
)->i_cdev
= (void *)(cpu
+ 1);
6177 tracing_init_tracefs_percpu(struct trace_array
*tr
, long cpu
)
6179 struct dentry
*d_percpu
= tracing_dentry_percpu(tr
, cpu
);
6180 struct dentry
*d_cpu
;
6181 char cpu_dir
[30]; /* 30 characters should be more than enough */
6186 snprintf(cpu_dir
, 30, "cpu%ld", cpu
);
6187 d_cpu
= tracefs_create_dir(cpu_dir
, d_percpu
);
6189 pr_warn("Could not create tracefs '%s' entry\n", cpu_dir
);
6193 /* per cpu trace_pipe */
6194 trace_create_cpu_file("trace_pipe", 0444, d_cpu
,
6195 tr
, cpu
, &tracing_pipe_fops
);
6198 trace_create_cpu_file("trace", 0644, d_cpu
,
6199 tr
, cpu
, &tracing_fops
);
6201 trace_create_cpu_file("trace_pipe_raw", 0444, d_cpu
,
6202 tr
, cpu
, &tracing_buffers_fops
);
6204 trace_create_cpu_file("stats", 0444, d_cpu
,
6205 tr
, cpu
, &tracing_stats_fops
);
6207 trace_create_cpu_file("buffer_size_kb", 0444, d_cpu
,
6208 tr
, cpu
, &tracing_entries_fops
);
6210 #ifdef CONFIG_TRACER_SNAPSHOT
6211 trace_create_cpu_file("snapshot", 0644, d_cpu
,
6212 tr
, cpu
, &snapshot_fops
);
6214 trace_create_cpu_file("snapshot_raw", 0444, d_cpu
,
6215 tr
, cpu
, &snapshot_raw_fops
);
6219 #ifdef CONFIG_FTRACE_SELFTEST
6220 /* Let selftest have access to static functions in this file */
6221 #include "trace_selftest.c"
6225 trace_options_read(struct file
*filp
, char __user
*ubuf
, size_t cnt
,
6228 struct trace_option_dentry
*topt
= filp
->private_data
;
6231 if (topt
->flags
->val
& topt
->opt
->bit
)
6236 return simple_read_from_buffer(ubuf
, cnt
, ppos
, buf
, 2);
6240 trace_options_write(struct file
*filp
, const char __user
*ubuf
, size_t cnt
,
6243 struct trace_option_dentry
*topt
= filp
->private_data
;
6247 ret
= kstrtoul_from_user(ubuf
, cnt
, 10, &val
);
6251 if (val
!= 0 && val
!= 1)
6254 if (!!(topt
->flags
->val
& topt
->opt
->bit
) != val
) {
6255 mutex_lock(&trace_types_lock
);
6256 ret
= __set_tracer_option(topt
->tr
, topt
->flags
,
6258 mutex_unlock(&trace_types_lock
);
6269 static const struct file_operations trace_options_fops
= {
6270 .open
= tracing_open_generic
,
6271 .read
= trace_options_read
,
6272 .write
= trace_options_write
,
6273 .llseek
= generic_file_llseek
,
6277 * In order to pass in both the trace_array descriptor as well as the index
6278 * to the flag that the trace option file represents, the trace_array
6279 * has a character array of trace_flags_index[], which holds the index
6280 * of the bit for the flag it represents. index[0] == 0, index[1] == 1, etc.
6281 * The address of this character array is passed to the flag option file
6282 * read/write callbacks.
6284 * In order to extract both the index and the trace_array descriptor,
6285 * get_tr_index() uses the following algorithm.
6289 * As the pointer itself contains the address of the index (remember
6292 * Then to get the trace_array descriptor, by subtracting that index
6293 * from the ptr, we get to the start of the index itself.
6295 * ptr - idx == &index[0]
6297 * Then a simple container_of() from that pointer gets us to the
6298 * trace_array descriptor.
6300 static void get_tr_index(void *data
, struct trace_array
**ptr
,
6301 unsigned int *pindex
)
6303 *pindex
= *(unsigned char *)data
;
6305 *ptr
= container_of(data
- *pindex
, struct trace_array
,
6310 trace_options_core_read(struct file
*filp
, char __user
*ubuf
, size_t cnt
,
6313 void *tr_index
= filp
->private_data
;
6314 struct trace_array
*tr
;
6318 get_tr_index(tr_index
, &tr
, &index
);
6320 if (tr
->trace_flags
& (1 << index
))
6325 return simple_read_from_buffer(ubuf
, cnt
, ppos
, buf
, 2);
6329 trace_options_core_write(struct file
*filp
, const char __user
*ubuf
, size_t cnt
,
6332 void *tr_index
= filp
->private_data
;
6333 struct trace_array
*tr
;
6338 get_tr_index(tr_index
, &tr
, &index
);
6340 ret
= kstrtoul_from_user(ubuf
, cnt
, 10, &val
);
6344 if (val
!= 0 && val
!= 1)
6347 mutex_lock(&trace_types_lock
);
6348 ret
= set_tracer_flag(tr
, 1 << index
, val
);
6349 mutex_unlock(&trace_types_lock
);
6359 static const struct file_operations trace_options_core_fops
= {
6360 .open
= tracing_open_generic
,
6361 .read
= trace_options_core_read
,
6362 .write
= trace_options_core_write
,
6363 .llseek
= generic_file_llseek
,
6366 struct dentry
*trace_create_file(const char *name
,
6368 struct dentry
*parent
,
6370 const struct file_operations
*fops
)
6374 ret
= tracefs_create_file(name
, mode
, parent
, data
, fops
);
6376 pr_warn("Could not create tracefs '%s' entry\n", name
);
6382 static struct dentry
*trace_options_init_dentry(struct trace_array
*tr
)
6384 struct dentry
*d_tracer
;
6389 d_tracer
= tracing_get_dentry(tr
);
6390 if (IS_ERR(d_tracer
))
6393 tr
->options
= tracefs_create_dir("options", d_tracer
);
6395 pr_warn("Could not create tracefs directory 'options'\n");
6403 create_trace_option_file(struct trace_array
*tr
,
6404 struct trace_option_dentry
*topt
,
6405 struct tracer_flags
*flags
,
6406 struct tracer_opt
*opt
)
6408 struct dentry
*t_options
;
6410 t_options
= trace_options_init_dentry(tr
);
6414 topt
->flags
= flags
;
6418 topt
->entry
= trace_create_file(opt
->name
, 0644, t_options
, topt
,
6419 &trace_options_fops
);
6424 create_trace_option_files(struct trace_array
*tr
, struct tracer
*tracer
)
6426 struct trace_option_dentry
*topts
;
6427 struct trace_options
*tr_topts
;
6428 struct tracer_flags
*flags
;
6429 struct tracer_opt
*opts
;
6436 flags
= tracer
->flags
;
6438 if (!flags
|| !flags
->opts
)
6442 * If this is an instance, only create flags for tracers
6443 * the instance may have.
6445 if (!trace_ok_for_array(tracer
, tr
))
6448 for (i
= 0; i
< tr
->nr_topts
; i
++) {
6449 /* Make sure there's no duplicate flags. */
6450 if (WARN_ON_ONCE(tr
->topts
[i
].tracer
->flags
== tracer
->flags
))
6456 for (cnt
= 0; opts
[cnt
].name
; cnt
++)
6459 topts
= kcalloc(cnt
+ 1, sizeof(*topts
), GFP_KERNEL
);
6463 tr_topts
= krealloc(tr
->topts
, sizeof(*tr
->topts
) * (tr
->nr_topts
+ 1),
6470 tr
->topts
= tr_topts
;
6471 tr
->topts
[tr
->nr_topts
].tracer
= tracer
;
6472 tr
->topts
[tr
->nr_topts
].topts
= topts
;
6475 for (cnt
= 0; opts
[cnt
].name
; cnt
++) {
6476 create_trace_option_file(tr
, &topts
[cnt
], flags
,
6478 WARN_ONCE(topts
[cnt
].entry
== NULL
,
6479 "Failed to create trace option: %s",
6484 static struct dentry
*
6485 create_trace_option_core_file(struct trace_array
*tr
,
6486 const char *option
, long index
)
6488 struct dentry
*t_options
;
6490 t_options
= trace_options_init_dentry(tr
);
6494 return trace_create_file(option
, 0644, t_options
,
6495 (void *)&tr
->trace_flags_index
[index
],
6496 &trace_options_core_fops
);
6499 static void create_trace_options_dir(struct trace_array
*tr
)
6501 struct dentry
*t_options
;
6502 bool top_level
= tr
== &global_trace
;
6505 t_options
= trace_options_init_dentry(tr
);
6509 for (i
= 0; trace_options
[i
]; i
++) {
6511 !((1 << i
) & TOP_LEVEL_TRACE_FLAGS
))
6512 create_trace_option_core_file(tr
, trace_options
[i
], i
);
6517 rb_simple_read(struct file
*filp
, char __user
*ubuf
,
6518 size_t cnt
, loff_t
*ppos
)
6520 struct trace_array
*tr
= filp
->private_data
;
6524 r
= tracer_tracing_is_on(tr
);
6525 r
= sprintf(buf
, "%d\n", r
);
6527 return simple_read_from_buffer(ubuf
, cnt
, ppos
, buf
, r
);
6531 rb_simple_write(struct file
*filp
, const char __user
*ubuf
,
6532 size_t cnt
, loff_t
*ppos
)
6534 struct trace_array
*tr
= filp
->private_data
;
6535 struct ring_buffer
*buffer
= tr
->trace_buffer
.buffer
;
6539 ret
= kstrtoul_from_user(ubuf
, cnt
, 10, &val
);
6544 mutex_lock(&trace_types_lock
);
6546 tracer_tracing_on(tr
);
6547 if (tr
->current_trace
->start
)
6548 tr
->current_trace
->start(tr
);
6550 tracer_tracing_off(tr
);
6551 if (tr
->current_trace
->stop
)
6552 tr
->current_trace
->stop(tr
);
6554 mutex_unlock(&trace_types_lock
);
6562 static const struct file_operations rb_simple_fops
= {
6563 .open
= tracing_open_generic_tr
,
6564 .read
= rb_simple_read
,
6565 .write
= rb_simple_write
,
6566 .release
= tracing_release_generic_tr
,
6567 .llseek
= default_llseek
,
6570 struct dentry
*trace_instance_dir
;
6573 init_tracer_tracefs(struct trace_array
*tr
, struct dentry
*d_tracer
);
6576 allocate_trace_buffer(struct trace_array
*tr
, struct trace_buffer
*buf
, int size
)
6578 enum ring_buffer_flags rb_flags
;
6580 rb_flags
= tr
->trace_flags
& TRACE_ITER_OVERWRITE
? RB_FL_OVERWRITE
: 0;
6584 buf
->buffer
= ring_buffer_alloc(size
, rb_flags
);
6588 buf
->data
= alloc_percpu(struct trace_array_cpu
);
6590 ring_buffer_free(buf
->buffer
);
6594 /* Allocate the first page for all buffers */
6595 set_buffer_entries(&tr
->trace_buffer
,
6596 ring_buffer_size(tr
->trace_buffer
.buffer
, 0));
6601 static int allocate_trace_buffers(struct trace_array
*tr
, int size
)
6605 ret
= allocate_trace_buffer(tr
, &tr
->trace_buffer
, size
);
6609 #ifdef CONFIG_TRACER_MAX_TRACE
6610 ret
= allocate_trace_buffer(tr
, &tr
->max_buffer
,
6611 allocate_snapshot
? size
: 1);
6613 ring_buffer_free(tr
->trace_buffer
.buffer
);
6614 free_percpu(tr
->trace_buffer
.data
);
6617 tr
->allocated_snapshot
= allocate_snapshot
;
6620 * Only the top level trace array gets its snapshot allocated
6621 * from the kernel command line.
6623 allocate_snapshot
= false;
6628 static void free_trace_buffer(struct trace_buffer
*buf
)
6631 ring_buffer_free(buf
->buffer
);
6633 free_percpu(buf
->data
);
6638 static void free_trace_buffers(struct trace_array
*tr
)
6643 free_trace_buffer(&tr
->trace_buffer
);
6645 #ifdef CONFIG_TRACER_MAX_TRACE
6646 free_trace_buffer(&tr
->max_buffer
);
6650 static void init_trace_flags_index(struct trace_array
*tr
)
6654 /* Used by the trace options files */
6655 for (i
= 0; i
< TRACE_FLAGS_MAX_SIZE
; i
++)
6656 tr
->trace_flags_index
[i
] = i
;
6659 static void __update_tracer_options(struct trace_array
*tr
)
6663 for (t
= trace_types
; t
; t
= t
->next
)
6664 add_tracer_options(tr
, t
);
6667 static void update_tracer_options(struct trace_array
*tr
)
6669 mutex_lock(&trace_types_lock
);
6670 __update_tracer_options(tr
);
6671 mutex_unlock(&trace_types_lock
);
6674 static int instance_mkdir(const char *name
)
6676 struct trace_array
*tr
;
6679 mutex_lock(&trace_types_lock
);
6682 list_for_each_entry(tr
, &ftrace_trace_arrays
, list
) {
6683 if (tr
->name
&& strcmp(tr
->name
, name
) == 0)
6688 tr
= kzalloc(sizeof(*tr
), GFP_KERNEL
);
6692 tr
->name
= kstrdup(name
, GFP_KERNEL
);
6696 if (!alloc_cpumask_var(&tr
->tracing_cpumask
, GFP_KERNEL
))
6699 tr
->trace_flags
= global_trace
.trace_flags
& ~ZEROED_TRACE_FLAGS
;
6701 cpumask_copy(tr
->tracing_cpumask
, cpu_all_mask
);
6703 raw_spin_lock_init(&tr
->start_lock
);
6705 tr
->max_lock
= (arch_spinlock_t
)__ARCH_SPIN_LOCK_UNLOCKED
;
6707 tr
->current_trace
= &nop_trace
;
6709 INIT_LIST_HEAD(&tr
->systems
);
6710 INIT_LIST_HEAD(&tr
->events
);
6712 if (allocate_trace_buffers(tr
, trace_buf_size
) < 0)
6715 tr
->dir
= tracefs_create_dir(name
, trace_instance_dir
);
6719 ret
= event_trace_add_tracer(tr
->dir
, tr
);
6721 tracefs_remove_recursive(tr
->dir
);
6725 init_tracer_tracefs(tr
, tr
->dir
);
6726 init_trace_flags_index(tr
);
6727 __update_tracer_options(tr
);
6729 list_add(&tr
->list
, &ftrace_trace_arrays
);
6731 mutex_unlock(&trace_types_lock
);
6736 free_trace_buffers(tr
);
6737 free_cpumask_var(tr
->tracing_cpumask
);
6742 mutex_unlock(&trace_types_lock
);
6748 static int instance_rmdir(const char *name
)
6750 struct trace_array
*tr
;
6755 mutex_lock(&trace_types_lock
);
6758 list_for_each_entry(tr
, &ftrace_trace_arrays
, list
) {
6759 if (tr
->name
&& strcmp(tr
->name
, name
) == 0) {
6768 if (tr
->ref
|| (tr
->current_trace
&& tr
->current_trace
->ref
))
6771 list_del(&tr
->list
);
6773 /* Disable all the flags that were enabled coming in */
6774 for (i
= 0; i
< TRACE_FLAGS_MAX_SIZE
; i
++) {
6775 if ((1 << i
) & ZEROED_TRACE_FLAGS
)
6776 set_tracer_flag(tr
, 1 << i
, 0);
6779 tracing_set_nop(tr
);
6780 event_trace_del_tracer(tr
);
6781 ftrace_destroy_function_files(tr
);
6782 tracefs_remove_recursive(tr
->dir
);
6783 free_trace_buffers(tr
);
6785 for (i
= 0; i
< tr
->nr_topts
; i
++) {
6786 kfree(tr
->topts
[i
].topts
);
6796 mutex_unlock(&trace_types_lock
);
6801 static __init
void create_trace_instances(struct dentry
*d_tracer
)
6803 trace_instance_dir
= tracefs_create_instance_dir("instances", d_tracer
,
6806 if (WARN_ON(!trace_instance_dir
))
6811 init_tracer_tracefs(struct trace_array
*tr
, struct dentry
*d_tracer
)
6815 trace_create_file("available_tracers", 0444, d_tracer
,
6816 tr
, &show_traces_fops
);
6818 trace_create_file("current_tracer", 0644, d_tracer
,
6819 tr
, &set_tracer_fops
);
6821 trace_create_file("tracing_cpumask", 0644, d_tracer
,
6822 tr
, &tracing_cpumask_fops
);
6824 trace_create_file("trace_options", 0644, d_tracer
,
6825 tr
, &tracing_iter_fops
);
6827 trace_create_file("trace", 0644, d_tracer
,
6830 trace_create_file("trace_pipe", 0444, d_tracer
,
6831 tr
, &tracing_pipe_fops
);
6833 trace_create_file("buffer_size_kb", 0644, d_tracer
,
6834 tr
, &tracing_entries_fops
);
6836 trace_create_file("buffer_total_size_kb", 0444, d_tracer
,
6837 tr
, &tracing_total_entries_fops
);
6839 trace_create_file("free_buffer", 0200, d_tracer
,
6840 tr
, &tracing_free_buffer_fops
);
6842 trace_create_file("trace_marker", 0220, d_tracer
,
6843 tr
, &tracing_mark_fops
);
6845 trace_create_file("trace_clock", 0644, d_tracer
, tr
,
6848 trace_create_file("tracing_on", 0644, d_tracer
,
6849 tr
, &rb_simple_fops
);
6851 create_trace_options_dir(tr
);
6853 #ifdef CONFIG_TRACER_MAX_TRACE
6854 trace_create_file("tracing_max_latency", 0644, d_tracer
,
6855 &tr
->max_latency
, &tracing_max_lat_fops
);
6858 if (ftrace_create_function_files(tr
, d_tracer
))
6859 WARN(1, "Could not allocate function filter files");
6861 #ifdef CONFIG_TRACER_SNAPSHOT
6862 trace_create_file("snapshot", 0644, d_tracer
,
6863 tr
, &snapshot_fops
);
6866 for_each_tracing_cpu(cpu
)
6867 tracing_init_tracefs_percpu(tr
, cpu
);
6871 static struct vfsmount
*trace_automount(void *ingore
)
6873 struct vfsmount
*mnt
;
6874 struct file_system_type
*type
;
6877 * To maintain backward compatibility for tools that mount
6878 * debugfs to get to the tracing facility, tracefs is automatically
6879 * mounted to the debugfs/tracing directory.
6881 type
= get_fs_type("tracefs");
6884 mnt
= vfs_kern_mount(type
, 0, "tracefs", NULL
);
6885 put_filesystem(type
);
6894 * tracing_init_dentry - initialize top level trace array
6896 * This is called when creating files or directories in the tracing
6897 * directory. It is called via fs_initcall() by any of the boot up code
6898 * and expects to return the dentry of the top level tracing directory.
6900 struct dentry
*tracing_init_dentry(void)
6902 struct trace_array
*tr
= &global_trace
;
6904 /* The top level trace array uses NULL as parent */
6908 if (WARN_ON(!tracefs_initialized()) ||
6909 (IS_ENABLED(CONFIG_DEBUG_FS
) &&
6910 WARN_ON(!debugfs_initialized())))
6911 return ERR_PTR(-ENODEV
);
6914 * As there may still be users that expect the tracing
6915 * files to exist in debugfs/tracing, we must automount
6916 * the tracefs file system there, so older tools still
6917 * work with the newer kerenl.
6919 tr
->dir
= debugfs_create_automount("tracing", NULL
,
6920 trace_automount
, NULL
);
6922 pr_warn_once("Could not create debugfs directory 'tracing'\n");
6923 return ERR_PTR(-ENOMEM
);
6929 extern struct trace_enum_map
*__start_ftrace_enum_maps
[];
6930 extern struct trace_enum_map
*__stop_ftrace_enum_maps
[];
6932 static void __init
trace_enum_init(void)
6936 len
= __stop_ftrace_enum_maps
- __start_ftrace_enum_maps
;
6937 trace_insert_enum_map(NULL
, __start_ftrace_enum_maps
, len
);
6940 #ifdef CONFIG_MODULES
6941 static void trace_module_add_enums(struct module
*mod
)
6943 if (!mod
->num_trace_enums
)
6947 * Modules with bad taint do not have events created, do
6948 * not bother with enums either.
6950 if (trace_module_has_bad_taint(mod
))
6953 trace_insert_enum_map(mod
, mod
->trace_enums
, mod
->num_trace_enums
);
6956 #ifdef CONFIG_TRACE_ENUM_MAP_FILE
6957 static void trace_module_remove_enums(struct module
*mod
)
6959 union trace_enum_map_item
*map
;
6960 union trace_enum_map_item
**last
= &trace_enum_maps
;
6962 if (!mod
->num_trace_enums
)
6965 mutex_lock(&trace_enum_mutex
);
6967 map
= trace_enum_maps
;
6970 if (map
->head
.mod
== mod
)
6972 map
= trace_enum_jmp_to_tail(map
);
6973 last
= &map
->tail
.next
;
6974 map
= map
->tail
.next
;
6979 *last
= trace_enum_jmp_to_tail(map
)->tail
.next
;
6982 mutex_unlock(&trace_enum_mutex
);
6985 static inline void trace_module_remove_enums(struct module
*mod
) { }
6986 #endif /* CONFIG_TRACE_ENUM_MAP_FILE */
6988 static int trace_module_notify(struct notifier_block
*self
,
6989 unsigned long val
, void *data
)
6991 struct module
*mod
= data
;
6994 case MODULE_STATE_COMING
:
6995 trace_module_add_enums(mod
);
6997 case MODULE_STATE_GOING
:
6998 trace_module_remove_enums(mod
);
7005 static struct notifier_block trace_module_nb
= {
7006 .notifier_call
= trace_module_notify
,
7009 #endif /* CONFIG_MODULES */
7011 static __init
int tracer_init_tracefs(void)
7013 struct dentry
*d_tracer
;
7015 trace_access_lock_init();
7017 d_tracer
= tracing_init_dentry();
7018 if (IS_ERR(d_tracer
))
7021 init_tracer_tracefs(&global_trace
, d_tracer
);
7023 trace_create_file("tracing_thresh", 0644, d_tracer
,
7024 &global_trace
, &tracing_thresh_fops
);
7026 trace_create_file("README", 0444, d_tracer
,
7027 NULL
, &tracing_readme_fops
);
7029 trace_create_file("saved_cmdlines", 0444, d_tracer
,
7030 NULL
, &tracing_saved_cmdlines_fops
);
7032 trace_create_file("saved_cmdlines_size", 0644, d_tracer
,
7033 NULL
, &tracing_saved_cmdlines_size_fops
);
7037 trace_create_enum_file(d_tracer
);
7039 #ifdef CONFIG_MODULES
7040 register_module_notifier(&trace_module_nb
);
7043 #ifdef CONFIG_DYNAMIC_FTRACE
7044 trace_create_file("dyn_ftrace_total_info", 0444, d_tracer
,
7045 &ftrace_update_tot_cnt
, &tracing_dyn_info_fops
);
7048 create_trace_instances(d_tracer
);
7050 update_tracer_options(&global_trace
);
7055 static int trace_panic_handler(struct notifier_block
*this,
7056 unsigned long event
, void *unused
)
7058 if (ftrace_dump_on_oops
)
7059 ftrace_dump(ftrace_dump_on_oops
);
7063 static struct notifier_block trace_panic_notifier
= {
7064 .notifier_call
= trace_panic_handler
,
7066 .priority
= 150 /* priority: INT_MAX >= x >= 0 */
7069 static int trace_die_handler(struct notifier_block
*self
,
7075 if (ftrace_dump_on_oops
)
7076 ftrace_dump(ftrace_dump_on_oops
);
7084 static struct notifier_block trace_die_notifier
= {
7085 .notifier_call
= trace_die_handler
,
7090 * printk is set to max of 1024, we really don't need it that big.
7091 * Nothing should be printing 1000 characters anyway.
7093 #define TRACE_MAX_PRINT 1000
7096 * Define here KERN_TRACE so that we have one place to modify
7097 * it if we decide to change what log level the ftrace dump
7100 #define KERN_TRACE KERN_EMERG
7103 trace_printk_seq(struct trace_seq
*s
)
7105 /* Probably should print a warning here. */
7106 if (s
->seq
.len
>= TRACE_MAX_PRINT
)
7107 s
->seq
.len
= TRACE_MAX_PRINT
;
7110 * More paranoid code. Although the buffer size is set to
7111 * PAGE_SIZE, and TRACE_MAX_PRINT is 1000, this is just
7112 * an extra layer of protection.
7114 if (WARN_ON_ONCE(s
->seq
.len
>= s
->seq
.size
))
7115 s
->seq
.len
= s
->seq
.size
- 1;
7117 /* should be zero ended, but we are paranoid. */
7118 s
->buffer
[s
->seq
.len
] = 0;
7120 printk(KERN_TRACE
"%s", s
->buffer
);
7125 void trace_init_global_iter(struct trace_iterator
*iter
)
7127 iter
->tr
= &global_trace
;
7128 iter
->trace
= iter
->tr
->current_trace
;
7129 iter
->cpu_file
= RING_BUFFER_ALL_CPUS
;
7130 iter
->trace_buffer
= &global_trace
.trace_buffer
;
7132 if (iter
->trace
&& iter
->trace
->open
)
7133 iter
->trace
->open(iter
);
7135 /* Annotate start of buffers if we had overruns */
7136 if (ring_buffer_overruns(iter
->trace_buffer
->buffer
))
7137 iter
->iter_flags
|= TRACE_FILE_ANNOTATE
;
7139 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
7140 if (trace_clocks
[iter
->tr
->clock_id
].in_ns
)
7141 iter
->iter_flags
|= TRACE_FILE_TIME_IN_NS
;
7144 void ftrace_dump(enum ftrace_dump_mode oops_dump_mode
)
7146 /* use static because iter can be a bit big for the stack */
7147 static struct trace_iterator iter
;
7148 static atomic_t dump_running
;
7149 struct trace_array
*tr
= &global_trace
;
7150 unsigned int old_userobj
;
7151 unsigned long flags
;
7154 /* Only allow one dump user at a time. */
7155 if (atomic_inc_return(&dump_running
) != 1) {
7156 atomic_dec(&dump_running
);
7161 * Always turn off tracing when we dump.
7162 * We don't need to show trace output of what happens
7163 * between multiple crashes.
7165 * If the user does a sysrq-z, then they can re-enable
7166 * tracing with echo 1 > tracing_on.
7170 local_irq_save(flags
);
7172 /* Simulate the iterator */
7173 trace_init_global_iter(&iter
);
7175 for_each_tracing_cpu(cpu
) {
7176 atomic_inc(&per_cpu_ptr(iter
.trace_buffer
->data
, cpu
)->disabled
);
7179 old_userobj
= tr
->trace_flags
& TRACE_ITER_SYM_USEROBJ
;
7181 /* don't look at user memory in panic mode */
7182 tr
->trace_flags
&= ~TRACE_ITER_SYM_USEROBJ
;
7184 switch (oops_dump_mode
) {
7186 iter
.cpu_file
= RING_BUFFER_ALL_CPUS
;
7189 iter
.cpu_file
= raw_smp_processor_id();
7194 printk(KERN_TRACE
"Bad dumping mode, switching to all CPUs dump\n");
7195 iter
.cpu_file
= RING_BUFFER_ALL_CPUS
;
7198 printk(KERN_TRACE
"Dumping ftrace buffer:\n");
7200 /* Did function tracer already get disabled? */
7201 if (ftrace_is_dead()) {
7202 printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
7203 printk("# MAY BE MISSING FUNCTION EVENTS\n");
7207 * We need to stop all tracing on all CPUS to read the
7208 * the next buffer. This is a bit expensive, but is
7209 * not done often. We fill all what we can read,
7210 * and then release the locks again.
7213 while (!trace_empty(&iter
)) {
7216 printk(KERN_TRACE
"---------------------------------\n");
7220 /* reset all but tr, trace, and overruns */
7221 memset(&iter
.seq
, 0,
7222 sizeof(struct trace_iterator
) -
7223 offsetof(struct trace_iterator
, seq
));
7224 iter
.iter_flags
|= TRACE_FILE_LAT_FMT
;
7227 if (trace_find_next_entry_inc(&iter
) != NULL
) {
7230 ret
= print_trace_line(&iter
);
7231 if (ret
!= TRACE_TYPE_NO_CONSUME
)
7232 trace_consume(&iter
);
7234 touch_nmi_watchdog();
7236 trace_printk_seq(&iter
.seq
);
7240 printk(KERN_TRACE
" (ftrace buffer empty)\n");
7242 printk(KERN_TRACE
"---------------------------------\n");
7245 tr
->trace_flags
|= old_userobj
;
7247 for_each_tracing_cpu(cpu
) {
7248 atomic_dec(&per_cpu_ptr(iter
.trace_buffer
->data
, cpu
)->disabled
);
7250 atomic_dec(&dump_running
);
7251 local_irq_restore(flags
);
7253 EXPORT_SYMBOL_GPL(ftrace_dump
);
7255 __init
static int tracer_alloc_buffers(void)
7261 * Make sure we don't accidently add more trace options
7262 * than we have bits for.
7264 BUILD_BUG_ON(TRACE_ITER_LAST_BIT
> TRACE_FLAGS_MAX_SIZE
);
7266 if (!alloc_cpumask_var(&tracing_buffer_mask
, GFP_KERNEL
))
7269 if (!alloc_cpumask_var(&global_trace
.tracing_cpumask
, GFP_KERNEL
))
7270 goto out_free_buffer_mask
;
7272 /* Only allocate trace_printk buffers if a trace_printk exists */
7273 if (__stop___trace_bprintk_fmt
!= __start___trace_bprintk_fmt
)
7274 /* Must be called before global_trace.buffer is allocated */
7275 trace_printk_init_buffers();
7277 /* To save memory, keep the ring buffer size to its minimum */
7278 if (ring_buffer_expanded
)
7279 ring_buf_size
= trace_buf_size
;
7283 cpumask_copy(tracing_buffer_mask
, cpu_possible_mask
);
7284 cpumask_copy(global_trace
.tracing_cpumask
, cpu_all_mask
);
7286 raw_spin_lock_init(&global_trace
.start_lock
);
7288 /* Used for event triggers */
7289 temp_buffer
= ring_buffer_alloc(PAGE_SIZE
, RB_FL_OVERWRITE
);
7291 goto out_free_cpumask
;
7293 if (trace_create_savedcmd() < 0)
7294 goto out_free_temp_buffer
;
7296 /* TODO: make the number of buffers hot pluggable with CPUS */
7297 if (allocate_trace_buffers(&global_trace
, ring_buf_size
) < 0) {
7298 printk(KERN_ERR
"tracer: failed to allocate ring buffer!\n");
7300 goto out_free_savedcmd
;
7303 if (global_trace
.buffer_disabled
)
7306 if (trace_boot_clock
) {
7307 ret
= tracing_set_clock(&global_trace
, trace_boot_clock
);
7309 pr_warn("Trace clock %s not defined, going back to default\n",
7314 * register_tracer() might reference current_trace, so it
7315 * needs to be set before we register anything. This is
7316 * just a bootstrap of current_trace anyway.
7318 global_trace
.current_trace
= &nop_trace
;
7320 global_trace
.max_lock
= (arch_spinlock_t
)__ARCH_SPIN_LOCK_UNLOCKED
;
7322 ftrace_init_global_array_ops(&global_trace
);
7324 init_trace_flags_index(&global_trace
);
7326 register_tracer(&nop_trace
);
7328 /* All seems OK, enable tracing */
7329 tracing_disabled
= 0;
7331 atomic_notifier_chain_register(&panic_notifier_list
,
7332 &trace_panic_notifier
);
7334 register_die_notifier(&trace_die_notifier
);
7336 global_trace
.flags
= TRACE_ARRAY_FL_GLOBAL
;
7338 INIT_LIST_HEAD(&global_trace
.systems
);
7339 INIT_LIST_HEAD(&global_trace
.events
);
7340 list_add(&global_trace
.list
, &ftrace_trace_arrays
);
7342 apply_trace_boot_options();
7344 register_snapshot_cmd();
7349 free_saved_cmdlines_buffer(savedcmd
);
7350 out_free_temp_buffer
:
7351 ring_buffer_free(temp_buffer
);
7353 free_cpumask_var(global_trace
.tracing_cpumask
);
7354 out_free_buffer_mask
:
7355 free_cpumask_var(tracing_buffer_mask
);
7360 void __init
trace_init(void)
7362 if (tracepoint_printk
) {
7363 tracepoint_print_iter
=
7364 kmalloc(sizeof(*tracepoint_print_iter
), GFP_KERNEL
);
7365 if (WARN_ON(!tracepoint_print_iter
))
7366 tracepoint_printk
= 0;
7368 tracer_alloc_buffers();
7372 __init
static int clear_boot_tracer(void)
7375 * The default tracer at boot buffer is an init section.
7376 * This function is called in lateinit. If we did not
7377 * find the boot tracer, then clear it out, to prevent
7378 * later registration from accessing the buffer that is
7379 * about to be freed.
7381 if (!default_bootup_tracer
)
7384 printk(KERN_INFO
"ftrace bootup tracer '%s' not registered.\n",
7385 default_bootup_tracer
);
7386 default_bootup_tracer
= NULL
;
7391 fs_initcall(tracer_init_tracefs
);
7392 late_initcall(clear_boot_tracer
);