Commit | Line | Data |
---|---|---|
36994e58 FW |
1 | /* |
2 | * Memory allocator tracing | |
3 | * | |
4 | * Copyright (C) 2008 Eduard - Gabriel Munteanu | |
5 | * Copyright (C) 2008 Pekka Enberg <penberg@cs.helsinki.fi> | |
6 | * Copyright (C) 2008 Frederic Weisbecker <fweisbec@gmail.com> | |
7 | */ | |
8 | ||
c826e3cd IM |
9 | #include <linux/tracepoint.h> |
10 | #include <linux/seq_file.h> | |
36994e58 | 11 | #include <linux/debugfs.h> |
c826e3cd | 12 | #include <linux/dcache.h> |
36994e58 | 13 | #include <linux/fs.h> |
c826e3cd | 14 | |
02af61bb | 15 | #include <linux/kmemtrace.h> |
36994e58 | 16 | |
36994e58 | 17 | #include "trace_output.h" |
c826e3cd | 18 | #include "trace.h" |
36994e58 FW |
19 | |
20 | /* Select an alternative, minimalistic output than the original one */ | |
21 | #define TRACE_KMEM_OPT_MINIMAL 0x1 | |
22 | ||
23 | static struct tracer_opt kmem_opts[] = { | |
24 | /* Default disable the minimalistic output */ | |
25 | { TRACER_OPT(kmem_minimalistic, TRACE_KMEM_OPT_MINIMAL) }, | |
26 | { } | |
27 | }; | |
28 | ||
29 | static struct tracer_flags kmem_tracer_flags = { | |
c826e3cd IM |
30 | .val = 0, |
31 | .opts = kmem_opts | |
36994e58 FW |
32 | }; |
33 | ||
36994e58 FW |
34 | static struct trace_array *kmemtrace_array; |
35 | ||
ca2b84cb EGM |
36 | /* Trace allocations */ |
37 | static inline void kmemtrace_alloc(enum kmemtrace_type_id type_id, | |
38 | unsigned long call_site, | |
39 | const void *ptr, | |
40 | size_t bytes_req, | |
41 | size_t bytes_alloc, | |
42 | gfp_t gfp_flags, | |
43 | int node) | |
44 | { | |
e1112b4d | 45 | struct ftrace_event_call *call = &event_kmem_alloc; |
ca2b84cb | 46 | struct trace_array *tr = kmemtrace_array; |
c826e3cd IM |
47 | struct kmemtrace_alloc_entry *entry; |
48 | struct ring_buffer_event *event; | |
ca2b84cb EGM |
49 | |
50 | event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry)); | |
51 | if (!event) | |
52 | return; | |
c826e3cd IM |
53 | |
54 | entry = ring_buffer_event_data(event); | |
ca2b84cb EGM |
55 | tracing_generic_entry_update(&entry->ent, 0, 0); |
56 | ||
c826e3cd IM |
57 | entry->ent.type = TRACE_KMEM_ALLOC; |
58 | entry->type_id = type_id; | |
59 | entry->call_site = call_site; | |
60 | entry->ptr = ptr; | |
61 | entry->bytes_req = bytes_req; | |
62 | entry->bytes_alloc = bytes_alloc; | |
63 | entry->gfp_flags = gfp_flags; | |
64 | entry->node = node; | |
ca2b84cb | 65 | |
eb02ce01 TZ |
66 | if (!filter_check_discard(call, entry, tr->buffer, event)) |
67 | ring_buffer_unlock_commit(tr->buffer, event); | |
ca2b84cb EGM |
68 | |
69 | trace_wake_up(); | |
70 | } | |
71 | ||
72 | static inline void kmemtrace_free(enum kmemtrace_type_id type_id, | |
73 | unsigned long call_site, | |
74 | const void *ptr) | |
75 | { | |
e1112b4d | 76 | struct ftrace_event_call *call = &event_kmem_free; |
ca2b84cb | 77 | struct trace_array *tr = kmemtrace_array; |
c826e3cd IM |
78 | struct kmemtrace_free_entry *entry; |
79 | struct ring_buffer_event *event; | |
ca2b84cb EGM |
80 | |
81 | event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry)); | |
82 | if (!event) | |
83 | return; | |
84 | entry = ring_buffer_event_data(event); | |
85 | tracing_generic_entry_update(&entry->ent, 0, 0); | |
86 | ||
c826e3cd IM |
87 | entry->ent.type = TRACE_KMEM_FREE; |
88 | entry->type_id = type_id; | |
89 | entry->call_site = call_site; | |
90 | entry->ptr = ptr; | |
ca2b84cb | 91 | |
eb02ce01 TZ |
92 | if (!filter_check_discard(call, entry, tr->buffer, event)) |
93 | ring_buffer_unlock_commit(tr->buffer, event); | |
ca2b84cb EGM |
94 | |
95 | trace_wake_up(); | |
96 | } | |
97 | ||
98 | static void kmemtrace_kmalloc(unsigned long call_site, | |
99 | const void *ptr, | |
100 | size_t bytes_req, | |
101 | size_t bytes_alloc, | |
102 | gfp_t gfp_flags) | |
103 | { | |
104 | kmemtrace_alloc(KMEMTRACE_TYPE_KMALLOC, call_site, ptr, | |
105 | bytes_req, bytes_alloc, gfp_flags, -1); | |
106 | } | |
107 | ||
108 | static void kmemtrace_kmem_cache_alloc(unsigned long call_site, | |
109 | const void *ptr, | |
110 | size_t bytes_req, | |
111 | size_t bytes_alloc, | |
112 | gfp_t gfp_flags) | |
113 | { | |
114 | kmemtrace_alloc(KMEMTRACE_TYPE_CACHE, call_site, ptr, | |
115 | bytes_req, bytes_alloc, gfp_flags, -1); | |
116 | } | |
117 | ||
118 | static void kmemtrace_kmalloc_node(unsigned long call_site, | |
119 | const void *ptr, | |
120 | size_t bytes_req, | |
121 | size_t bytes_alloc, | |
122 | gfp_t gfp_flags, | |
123 | int node) | |
124 | { | |
125 | kmemtrace_alloc(KMEMTRACE_TYPE_KMALLOC, call_site, ptr, | |
126 | bytes_req, bytes_alloc, gfp_flags, node); | |
127 | } | |
128 | ||
129 | static void kmemtrace_kmem_cache_alloc_node(unsigned long call_site, | |
130 | const void *ptr, | |
131 | size_t bytes_req, | |
132 | size_t bytes_alloc, | |
133 | gfp_t gfp_flags, | |
134 | int node) | |
135 | { | |
136 | kmemtrace_alloc(KMEMTRACE_TYPE_CACHE, call_site, ptr, | |
137 | bytes_req, bytes_alloc, gfp_flags, node); | |
138 | } | |
139 | ||
140 | static void kmemtrace_kfree(unsigned long call_site, const void *ptr) | |
141 | { | |
142 | kmemtrace_free(KMEMTRACE_TYPE_KMALLOC, call_site, ptr); | |
143 | } | |
144 | ||
145 | static void kmemtrace_kmem_cache_free(unsigned long call_site, const void *ptr) | |
146 | { | |
147 | kmemtrace_free(KMEMTRACE_TYPE_CACHE, call_site, ptr); | |
148 | } | |
149 | ||
150 | static int kmemtrace_start_probes(void) | |
151 | { | |
152 | int err; | |
153 | ||
154 | err = register_trace_kmalloc(kmemtrace_kmalloc); | |
155 | if (err) | |
156 | return err; | |
157 | err = register_trace_kmem_cache_alloc(kmemtrace_kmem_cache_alloc); | |
158 | if (err) | |
159 | return err; | |
160 | err = register_trace_kmalloc_node(kmemtrace_kmalloc_node); | |
161 | if (err) | |
162 | return err; | |
163 | err = register_trace_kmem_cache_alloc_node(kmemtrace_kmem_cache_alloc_node); | |
164 | if (err) | |
165 | return err; | |
166 | err = register_trace_kfree(kmemtrace_kfree); | |
167 | if (err) | |
168 | return err; | |
169 | err = register_trace_kmem_cache_free(kmemtrace_kmem_cache_free); | |
170 | ||
171 | return err; | |
172 | } | |
173 | ||
174 | static void kmemtrace_stop_probes(void) | |
175 | { | |
176 | unregister_trace_kmalloc(kmemtrace_kmalloc); | |
177 | unregister_trace_kmem_cache_alloc(kmemtrace_kmem_cache_alloc); | |
178 | unregister_trace_kmalloc_node(kmemtrace_kmalloc_node); | |
179 | unregister_trace_kmem_cache_alloc_node(kmemtrace_kmem_cache_alloc_node); | |
180 | unregister_trace_kfree(kmemtrace_kfree); | |
181 | unregister_trace_kmem_cache_free(kmemtrace_kmem_cache_free); | |
182 | } | |
183 | ||
36994e58 FW |
184 | static int kmem_trace_init(struct trace_array *tr) |
185 | { | |
36994e58 FW |
186 | kmemtrace_array = tr; |
187 | ||
76f0d073 | 188 | tracing_reset_online_cpus(tr); |
36994e58 | 189 | |
ca2b84cb | 190 | kmemtrace_start_probes(); |
36994e58 FW |
191 | |
192 | return 0; | |
193 | } | |
194 | ||
195 | static void kmem_trace_reset(struct trace_array *tr) | |
196 | { | |
ca2b84cb | 197 | kmemtrace_stop_probes(); |
36994e58 FW |
198 | } |
199 | ||
200 | static void kmemtrace_headers(struct seq_file *s) | |
201 | { | |
202 | /* Don't need headers for the original kmemtrace output */ | |
203 | if (!(kmem_tracer_flags.val & TRACE_KMEM_OPT_MINIMAL)) | |
204 | return; | |
205 | ||
206 | seq_printf(s, "#\n"); | |
207 | seq_printf(s, "# ALLOC TYPE REQ GIVEN FLAGS " | |
208 | " POINTER NODE CALLER\n"); | |
209 | seq_printf(s, "# FREE | | | | " | |
210 | " | | | |\n"); | |
211 | seq_printf(s, "# |\n\n"); | |
212 | } | |
213 | ||
214 | /* | |
42af9054 EGM |
215 | * The following functions give the original output from kmemtrace, |
216 | * plus the origin CPU, since reordering occurs in-kernel now. | |
36994e58 | 217 | */ |
42af9054 EGM |
218 | |
219 | #define KMEMTRACE_USER_ALLOC 0 | |
220 | #define KMEMTRACE_USER_FREE 1 | |
221 | ||
222 | struct kmemtrace_user_event { | |
c826e3cd IM |
223 | u8 event_id; |
224 | u8 type_id; | |
225 | u16 event_size; | |
226 | u32 cpu; | |
227 | u64 timestamp; | |
228 | unsigned long call_site; | |
229 | unsigned long ptr; | |
42af9054 EGM |
230 | }; |
231 | ||
232 | struct kmemtrace_user_event_alloc { | |
c826e3cd IM |
233 | size_t bytes_req; |
234 | size_t bytes_alloc; | |
235 | unsigned gfp_flags; | |
236 | int node; | |
42af9054 EGM |
237 | }; |
238 | ||
36994e58 | 239 | static enum print_line_t |
80098c20 | 240 | kmemtrace_print_alloc(struct trace_iterator *iter, int flags) |
ddc1637a LZ |
241 | { |
242 | struct trace_seq *s = &iter->seq; | |
243 | struct kmemtrace_alloc_entry *entry; | |
244 | int ret; | |
245 | ||
246 | trace_assign_type(entry, iter->ent); | |
247 | ||
248 | ret = trace_seq_printf(s, "type_id %d call_site %pF ptr %lu " | |
249 | "bytes_req %lu bytes_alloc %lu gfp_flags %lu node %d\n", | |
250 | entry->type_id, (void *)entry->call_site, (unsigned long)entry->ptr, | |
251 | (unsigned long)entry->bytes_req, (unsigned long)entry->bytes_alloc, | |
252 | (unsigned long)entry->gfp_flags, entry->node); | |
253 | ||
254 | if (!ret) | |
255 | return TRACE_TYPE_PARTIAL_LINE; | |
256 | return TRACE_TYPE_HANDLED; | |
257 | } | |
258 | ||
259 | static enum print_line_t | |
80098c20 | 260 | kmemtrace_print_free(struct trace_iterator *iter, int flags) |
36994e58 FW |
261 | { |
262 | struct trace_seq *s = &iter->seq; | |
ddc1637a LZ |
263 | struct kmemtrace_free_entry *entry; |
264 | int ret; | |
265 | ||
266 | trace_assign_type(entry, iter->ent); | |
267 | ||
268 | ret = trace_seq_printf(s, "type_id %d call_site %pF ptr %lu\n", | |
269 | entry->type_id, (void *)entry->call_site, | |
270 | (unsigned long)entry->ptr); | |
271 | ||
272 | if (!ret) | |
273 | return TRACE_TYPE_PARTIAL_LINE; | |
274 | return TRACE_TYPE_HANDLED; | |
275 | } | |
276 | ||
277 | static enum print_line_t | |
80098c20 | 278 | kmemtrace_print_alloc_user(struct trace_iterator *iter, int flags) |
ddc1637a LZ |
279 | { |
280 | struct trace_seq *s = &iter->seq; | |
281 | struct kmemtrace_alloc_entry *entry; | |
42af9054 | 282 | struct kmemtrace_user_event *ev; |
ddc1637a LZ |
283 | struct kmemtrace_user_event_alloc *ev_alloc; |
284 | ||
285 | trace_assign_type(entry, iter->ent); | |
36994e58 | 286 | |
42af9054 EGM |
287 | ev = trace_seq_reserve(s, sizeof(*ev)); |
288 | if (!ev) | |
289 | return TRACE_TYPE_PARTIAL_LINE; | |
c826e3cd IM |
290 | |
291 | ev->event_id = KMEMTRACE_USER_ALLOC; | |
292 | ev->type_id = entry->type_id; | |
293 | ev->event_size = sizeof(*ev) + sizeof(*ev_alloc); | |
294 | ev->cpu = iter->cpu; | |
295 | ev->timestamp = iter->ts; | |
296 | ev->call_site = entry->call_site; | |
297 | ev->ptr = (unsigned long)entry->ptr; | |
42af9054 EGM |
298 | |
299 | ev_alloc = trace_seq_reserve(s, sizeof(*ev_alloc)); | |
300 | if (!ev_alloc) | |
36994e58 | 301 | return TRACE_TYPE_PARTIAL_LINE; |
c826e3cd IM |
302 | |
303 | ev_alloc->bytes_req = entry->bytes_req; | |
304 | ev_alloc->bytes_alloc = entry->bytes_alloc; | |
305 | ev_alloc->gfp_flags = entry->gfp_flags; | |
306 | ev_alloc->node = entry->node; | |
36994e58 FW |
307 | |
308 | return TRACE_TYPE_HANDLED; | |
309 | } | |
310 | ||
311 | static enum print_line_t | |
80098c20 | 312 | kmemtrace_print_free_user(struct trace_iterator *iter, int flags) |
36994e58 FW |
313 | { |
314 | struct trace_seq *s = &iter->seq; | |
ddc1637a | 315 | struct kmemtrace_free_entry *entry; |
42af9054 | 316 | struct kmemtrace_user_event *ev; |
36994e58 | 317 | |
ddc1637a LZ |
318 | trace_assign_type(entry, iter->ent); |
319 | ||
42af9054 EGM |
320 | ev = trace_seq_reserve(s, sizeof(*ev)); |
321 | if (!ev) | |
36994e58 | 322 | return TRACE_TYPE_PARTIAL_LINE; |
c826e3cd IM |
323 | |
324 | ev->event_id = KMEMTRACE_USER_FREE; | |
325 | ev->type_id = entry->type_id; | |
326 | ev->event_size = sizeof(*ev); | |
327 | ev->cpu = iter->cpu; | |
328 | ev->timestamp = iter->ts; | |
329 | ev->call_site = entry->call_site; | |
330 | ev->ptr = (unsigned long)entry->ptr; | |
36994e58 FW |
331 | |
332 | return TRACE_TYPE_HANDLED; | |
333 | } | |
334 | ||
36994e58 FW |
335 | /* The two other following provide a more minimalistic output */ |
336 | static enum print_line_t | |
ddc1637a | 337 | kmemtrace_print_alloc_compress(struct trace_iterator *iter) |
36994e58 | 338 | { |
ddc1637a | 339 | struct kmemtrace_alloc_entry *entry; |
36994e58 FW |
340 | struct trace_seq *s = &iter->seq; |
341 | int ret; | |
342 | ||
ddc1637a LZ |
343 | trace_assign_type(entry, iter->ent); |
344 | ||
36994e58 FW |
345 | /* Alloc entry */ |
346 | ret = trace_seq_printf(s, " + "); | |
347 | if (!ret) | |
348 | return TRACE_TYPE_PARTIAL_LINE; | |
349 | ||
350 | /* Type */ | |
351 | switch (entry->type_id) { | |
352 | case KMEMTRACE_TYPE_KMALLOC: | |
353 | ret = trace_seq_printf(s, "K "); | |
354 | break; | |
355 | case KMEMTRACE_TYPE_CACHE: | |
356 | ret = trace_seq_printf(s, "C "); | |
357 | break; | |
358 | case KMEMTRACE_TYPE_PAGES: | |
359 | ret = trace_seq_printf(s, "P "); | |
360 | break; | |
361 | default: | |
362 | ret = trace_seq_printf(s, "? "); | |
363 | } | |
364 | ||
365 | if (!ret) | |
366 | return TRACE_TYPE_PARTIAL_LINE; | |
367 | ||
368 | /* Requested */ | |
ecf441b5 | 369 | ret = trace_seq_printf(s, "%4zu ", entry->bytes_req); |
36994e58 FW |
370 | if (!ret) |
371 | return TRACE_TYPE_PARTIAL_LINE; | |
372 | ||
373 | /* Allocated */ | |
ecf441b5 | 374 | ret = trace_seq_printf(s, "%4zu ", entry->bytes_alloc); |
36994e58 FW |
375 | if (!ret) |
376 | return TRACE_TYPE_PARTIAL_LINE; | |
377 | ||
378 | /* Flags | |
379 | * TODO: would be better to see the name of the GFP flag names | |
380 | */ | |
381 | ret = trace_seq_printf(s, "%08x ", entry->gfp_flags); | |
382 | if (!ret) | |
383 | return TRACE_TYPE_PARTIAL_LINE; | |
384 | ||
385 | /* Pointer to allocated */ | |
386 | ret = trace_seq_printf(s, "0x%tx ", (ptrdiff_t)entry->ptr); | |
387 | if (!ret) | |
388 | return TRACE_TYPE_PARTIAL_LINE; | |
389 | ||
6a167c65 FW |
390 | /* Node and call site*/ |
391 | ret = trace_seq_printf(s, "%4d %pf\n", entry->node, | |
392 | (void *)entry->call_site); | |
36994e58 FW |
393 | if (!ret) |
394 | return TRACE_TYPE_PARTIAL_LINE; | |
395 | ||
36994e58 FW |
396 | return TRACE_TYPE_HANDLED; |
397 | } | |
398 | ||
399 | static enum print_line_t | |
ddc1637a | 400 | kmemtrace_print_free_compress(struct trace_iterator *iter) |
36994e58 | 401 | { |
ddc1637a | 402 | struct kmemtrace_free_entry *entry; |
36994e58 FW |
403 | struct trace_seq *s = &iter->seq; |
404 | int ret; | |
405 | ||
ddc1637a LZ |
406 | trace_assign_type(entry, iter->ent); |
407 | ||
36994e58 FW |
408 | /* Free entry */ |
409 | ret = trace_seq_printf(s, " - "); | |
410 | if (!ret) | |
411 | return TRACE_TYPE_PARTIAL_LINE; | |
412 | ||
413 | /* Type */ | |
414 | switch (entry->type_id) { | |
415 | case KMEMTRACE_TYPE_KMALLOC: | |
416 | ret = trace_seq_printf(s, "K "); | |
417 | break; | |
418 | case KMEMTRACE_TYPE_CACHE: | |
419 | ret = trace_seq_printf(s, "C "); | |
420 | break; | |
421 | case KMEMTRACE_TYPE_PAGES: | |
422 | ret = trace_seq_printf(s, "P "); | |
423 | break; | |
424 | default: | |
425 | ret = trace_seq_printf(s, "? "); | |
426 | } | |
427 | ||
428 | if (!ret) | |
429 | return TRACE_TYPE_PARTIAL_LINE; | |
430 | ||
431 | /* Skip requested/allocated/flags */ | |
432 | ret = trace_seq_printf(s, " "); | |
433 | if (!ret) | |
434 | return TRACE_TYPE_PARTIAL_LINE; | |
435 | ||
436 | /* Pointer to allocated */ | |
437 | ret = trace_seq_printf(s, "0x%tx ", (ptrdiff_t)entry->ptr); | |
438 | if (!ret) | |
439 | return TRACE_TYPE_PARTIAL_LINE; | |
440 | ||
6a167c65 FW |
441 | /* Skip node and print call site*/ |
442 | ret = trace_seq_printf(s, " %pf\n", (void *)entry->call_site); | |
36994e58 FW |
443 | if (!ret) |
444 | return TRACE_TYPE_PARTIAL_LINE; | |
445 | ||
36994e58 FW |
446 | return TRACE_TYPE_HANDLED; |
447 | } | |
448 | ||
449 | static enum print_line_t kmemtrace_print_line(struct trace_iterator *iter) | |
450 | { | |
451 | struct trace_entry *entry = iter->ent; | |
452 | ||
ddc1637a LZ |
453 | if (!(kmem_tracer_flags.val & TRACE_KMEM_OPT_MINIMAL)) |
454 | return TRACE_TYPE_UNHANDLED; | |
36994e58 | 455 | |
ddc1637a LZ |
456 | switch (entry->type) { |
457 | case TRACE_KMEM_ALLOC: | |
458 | return kmemtrace_print_alloc_compress(iter); | |
459 | case TRACE_KMEM_FREE: | |
460 | return kmemtrace_print_free_compress(iter); | |
36994e58 FW |
461 | default: |
462 | return TRACE_TYPE_UNHANDLED; | |
463 | } | |
464 | } | |
465 | ||
ddc1637a LZ |
466 | static struct trace_event kmem_trace_alloc = { |
467 | .type = TRACE_KMEM_ALLOC, | |
80098c20 LZ |
468 | .trace = kmemtrace_print_alloc, |
469 | .binary = kmemtrace_print_alloc_user, | |
ddc1637a LZ |
470 | }; |
471 | ||
472 | static struct trace_event kmem_trace_free = { | |
473 | .type = TRACE_KMEM_FREE, | |
80098c20 LZ |
474 | .trace = kmemtrace_print_free, |
475 | .binary = kmemtrace_print_free_user, | |
ddc1637a LZ |
476 | }; |
477 | ||
36994e58 | 478 | static struct tracer kmem_tracer __read_mostly = { |
c826e3cd IM |
479 | .name = "kmemtrace", |
480 | .init = kmem_trace_init, | |
481 | .reset = kmem_trace_reset, | |
482 | .print_line = kmemtrace_print_line, | |
483 | .print_header = kmemtrace_headers, | |
484 | .flags = &kmem_tracer_flags | |
36994e58 FW |
485 | }; |
486 | ||
3e806802 IM |
487 | void kmemtrace_init(void) |
488 | { | |
489 | /* earliest opportunity to start kmem tracing */ | |
490 | } | |
491 | ||
36994e58 FW |
492 | static int __init init_kmem_tracer(void) |
493 | { | |
ddc1637a LZ |
494 | if (!register_ftrace_event(&kmem_trace_alloc)) { |
495 | pr_warning("Warning: could not register kmem events\n"); | |
496 | return 1; | |
497 | } | |
498 | ||
499 | if (!register_ftrace_event(&kmem_trace_free)) { | |
500 | pr_warning("Warning: could not register kmem events\n"); | |
501 | return 1; | |
502 | } | |
503 | ||
f9ac5a69 | 504 | if (register_tracer(&kmem_tracer) != 0) { |
ddc1637a LZ |
505 | pr_warning("Warning: could not register the kmem tracer\n"); |
506 | return 1; | |
507 | } | |
508 | ||
509 | return 0; | |
36994e58 | 510 | } |
36994e58 | 511 | device_initcall(init_kmem_tracer); |