2 * Copyright (C) 2008-2011 Mathieu Desnoyers
3 * Copyright (C) 2009 Pierre-Marc Fournier
5 * This library is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU Lesser General Public
7 * License as published by the Free Software Foundation;
8 * version 2.1 of the License.
10 * This library is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * Lesser General Public License for more details.
15 * You should have received a copy of the GNU Lesser General Public
16 * License along with this library; if not, write to the Free Software
17 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19 * Ported to userspace by Pierre-Marc Fournier.
24 #include <lttng/tracepoint.h>
27 #include <urcu/arch.h>
29 #include <urcu/hlist.h>
30 #include <urcu/uatomic.h>
31 #include <urcu/compiler.h>
34 #include <lttng/usterr-signal-safe.h>
35 #include "tracepoint-internal.h"
36 #include "ltt-tracer-core.h"
40 /* Set to 1 to enable tracepoint debug output */
41 static const int tracepoint_debug
;
42 static int initialized
;
43 static void (*new_tracepoint_cb
)(struct tracepoint
*);
45 /* libraries that contain tracepoints (struct tracepoint_lib) */
46 static CDS_LIST_HEAD(libs
);
49 * The UST lock protects the library tracepoints, the hash table, and
51 * All calls to the tracepoint API must be protected by the UST lock,
52 * excepts calls to tracepoint_register_lib and
53 * tracepoint_unregister_lib, which take the UST lock themselves.
57 * Tracepoint hash table, containing the active tracepoints.
58 * Protected by tracepoints_mutex.
60 #define TRACEPOINT_HASH_BITS 6
61 #define TRACEPOINT_TABLE_SIZE (1 << TRACEPOINT_HASH_BITS)
62 static struct cds_hlist_head tracepoint_table
[TRACEPOINT_TABLE_SIZE
];
64 static CDS_LIST_HEAD(old_probes
);
65 static int need_update
;
69 * It is used to to delay the free of multiple probes array until a quiescent
71 * Tracepoint entries modifications are protected by the tracepoints_mutex.
73 struct tracepoint_entry
{
74 struct cds_hlist_node hlist
;
75 struct tracepoint_probe
*probes
;
76 int refcount
; /* Number of times armed. 0 if disarmed. */
82 struct cds_list_head list
;
84 struct tracepoint_probe probes
[0];
87 static inline void *allocate_probes(int count
)
89 struct tp_probes
*p
= zmalloc(count
* sizeof(struct tracepoint_probe
)
90 + sizeof(struct tp_probes
));
91 return p
== NULL
? NULL
: p
->probes
;
94 static inline void release_probes(void *old
)
97 struct tp_probes
*tp_probes
= caa_container_of(old
,
98 struct tp_probes
, probes
[0]);
104 static void debug_print_probes(struct tracepoint_entry
*entry
)
108 if (!tracepoint_debug
|| !entry
->probes
)
111 for (i
= 0; entry
->probes
[i
].func
; i
++)
112 DBG("Probe %d : %p", i
, entry
->probes
[i
].func
);
116 tracepoint_entry_add_probe(struct tracepoint_entry
*entry
,
117 void *probe
, void *data
)
120 struct tracepoint_probe
*old
, *new;
124 debug_print_probes(entry
);
127 /* (N -> N+1), (N != 0, 1) probes */
128 for (nr_probes
= 0; old
[nr_probes
].func
; nr_probes
++)
129 if (old
[nr_probes
].func
== probe
&&
130 old
[nr_probes
].data
== data
)
131 return ERR_PTR(-EEXIST
);
133 /* + 2 : one for new probe, one for NULL func */
134 new = allocate_probes(nr_probes
+ 2);
136 return ERR_PTR(-ENOMEM
);
138 memcpy(new, old
, nr_probes
* sizeof(struct tracepoint_probe
));
139 new[nr_probes
].func
= probe
;
140 new[nr_probes
].data
= data
;
141 new[nr_probes
+ 1].func
= NULL
;
142 entry
->refcount
= nr_probes
+ 1;
144 debug_print_probes(entry
);
149 tracepoint_entry_remove_probe(struct tracepoint_entry
*entry
, void *probe
,
152 int nr_probes
= 0, nr_del
= 0, i
;
153 struct tracepoint_probe
*old
, *new;
158 return ERR_PTR(-ENOENT
);
160 debug_print_probes(entry
);
161 /* (N -> M), (N > 1, M >= 0) probes */
162 for (nr_probes
= 0; old
[nr_probes
].func
; nr_probes
++) {
164 (old
[nr_probes
].func
== probe
&&
165 old
[nr_probes
].data
== data
))
169 if (nr_probes
- nr_del
== 0) {
170 /* N -> 0, (N > 1) */
171 entry
->probes
= NULL
;
173 debug_print_probes(entry
);
177 /* N -> M, (N > 1, M > 0) */
179 new = allocate_probes(nr_probes
- nr_del
+ 1);
181 return ERR_PTR(-ENOMEM
);
182 for (i
= 0; old
[i
].func
; i
++)
184 (old
[i
].func
!= probe
|| old
[i
].data
!= data
))
186 new[nr_probes
- nr_del
].func
= NULL
;
187 entry
->refcount
= nr_probes
- nr_del
;
190 debug_print_probes(entry
);
195 * Get tracepoint if the tracepoint is present in the tracepoint hash table.
196 * Must be called with tracepoints_mutex held.
197 * Returns NULL if not present.
199 static struct tracepoint_entry
*get_tracepoint(const char *name
)
201 struct cds_hlist_head
*head
;
202 struct cds_hlist_node
*node
;
203 struct tracepoint_entry
*e
;
204 uint32_t hash
= jhash(name
, strlen(name
), 0);
206 head
= &tracepoint_table
[hash
& (TRACEPOINT_TABLE_SIZE
- 1)];
207 cds_hlist_for_each_entry(e
, node
, head
, hlist
) {
208 if (!strcmp(name
, e
->name
))
215 * Add the tracepoint to the tracepoint hash table. Must be called with
216 * tracepoints_mutex held.
218 static struct tracepoint_entry
*add_tracepoint(const char *name
)
220 struct cds_hlist_head
*head
;
221 struct cds_hlist_node
*node
;
222 struct tracepoint_entry
*e
;
223 size_t name_len
= strlen(name
) + 1;
224 uint32_t hash
= jhash(name
, name_len
-1, 0);
226 head
= &tracepoint_table
[hash
& (TRACEPOINT_TABLE_SIZE
- 1)];
227 cds_hlist_for_each_entry(e
, node
, head
, hlist
) {
228 if (!strcmp(name
, e
->name
)) {
229 DBG("tracepoint %s busy", name
);
230 return ERR_PTR(-EEXIST
); /* Already there */
234 * Using zmalloc here to allocate a variable length element. Could
235 * cause some memory fragmentation if overused.
237 e
= zmalloc(sizeof(struct tracepoint_entry
) + name_len
);
239 return ERR_PTR(-ENOMEM
);
240 memcpy(&e
->name
[0], name
, name_len
);
243 cds_hlist_add_head(&e
->hlist
, head
);
248 * Remove the tracepoint from the tracepoint hash table. Must be called with
251 static inline void remove_tracepoint(struct tracepoint_entry
*e
)
253 cds_hlist_del(&e
->hlist
);
258 * Sets the probe callback corresponding to one tracepoint.
260 static void set_tracepoint(struct tracepoint_entry
**entry
,
261 struct tracepoint
*elem
, int active
)
263 WARN_ON(strcmp((*entry
)->name
, elem
->name
) != 0);
266 * rcu_assign_pointer has a cmm_smp_wmb() which makes sure that the new
267 * probe callbacks array is consistent before setting a pointer to it.
268 * This array is referenced by __DO_TRACE from
269 * include/linux/tracepoints.h. A matching cmm_smp_read_barrier_depends()
272 rcu_assign_pointer(elem
->probes
, (*entry
)->probes
);
273 elem
->state
= active
;
277 * Disable a tracepoint and its probe callback.
278 * Note: only waiting an RCU period after setting elem->call to the empty
279 * function insures that the original callback is not used anymore. This insured
280 * by preempt_disable around the call site.
282 static void disable_tracepoint(struct tracepoint
*elem
)
285 rcu_assign_pointer(elem
->probes
, NULL
);
289 * tracepoint_update_probe_range - Update a probe range
290 * @begin: beginning of the range
291 * @end: end of the range
293 * Updates the probe callback corresponding to a range of tracepoints.
296 void tracepoint_update_probe_range(struct tracepoint
* const *begin
,
297 struct tracepoint
* const *end
)
299 struct tracepoint
* const *iter
;
300 struct tracepoint_entry
*mark_entry
;
302 for (iter
= begin
; iter
< end
; iter
++) {
304 continue; /* skip dummy */
305 if (!(*iter
)->name
) {
306 disable_tracepoint(*iter
);
309 mark_entry
= get_tracepoint((*iter
)->name
);
311 set_tracepoint(&mark_entry
, *iter
,
312 !!mark_entry
->refcount
);
314 disable_tracepoint(*iter
);
319 static void lib_update_tracepoints(void)
321 struct tracepoint_lib
*lib
;
323 cds_list_for_each_entry(lib
, &libs
, list
) {
324 tracepoint_update_probe_range(lib
->tracepoints_start
,
325 lib
->tracepoints_start
+ lib
->tracepoints_count
);
330 * Update probes, removing the faulty probes.
332 static void tracepoint_update_probes(void)
334 /* tracepoints registered from libraries and executable. */
335 lib_update_tracepoints();
338 static struct tracepoint_probe
*
339 tracepoint_add_probe(const char *name
, void *probe
, void *data
)
341 struct tracepoint_entry
*entry
;
342 struct tracepoint_probe
*old
;
344 entry
= get_tracepoint(name
);
346 entry
= add_tracepoint(name
);
348 return (struct tracepoint_probe
*)entry
;
350 old
= tracepoint_entry_add_probe(entry
, probe
, data
);
351 if (IS_ERR(old
) && !entry
->refcount
)
352 remove_tracepoint(entry
);
357 * __tracepoint_probe_register - Connect a probe to a tracepoint
358 * @name: tracepoint name
359 * @probe: probe handler
361 * Returns 0 if ok, error value on error.
362 * The probe address must at least be aligned on the architecture pointer size.
363 * Called with the UST lock held.
365 int __tracepoint_probe_register(const char *name
, void *probe
, void *data
)
369 old
= tracepoint_add_probe(name
, probe
, data
);
373 tracepoint_update_probes(); /* may update entry */
378 static void *tracepoint_remove_probe(const char *name
, void *probe
, void *data
)
380 struct tracepoint_entry
*entry
;
383 entry
= get_tracepoint(name
);
385 return ERR_PTR(-ENOENT
);
386 old
= tracepoint_entry_remove_probe(entry
, probe
, data
);
389 if (!entry
->refcount
)
390 remove_tracepoint(entry
);
395 * tracepoint_probe_unregister - Disconnect a probe from a tracepoint
396 * @name: tracepoint name
397 * @probe: probe function pointer
398 * @probe: probe data pointer
400 * Called with the UST lock held.
402 int __tracepoint_probe_unregister(const char *name
, void *probe
, void *data
)
406 old
= tracepoint_remove_probe(name
, probe
, data
);
410 tracepoint_update_probes(); /* may update entry */
415 static void tracepoint_add_old_probes(void *old
)
419 struct tp_probes
*tp_probes
= caa_container_of(old
,
420 struct tp_probes
, probes
[0]);
421 cds_list_add(&tp_probes
->u
.list
, &old_probes
);
426 * tracepoint_probe_register_noupdate - register a probe but not connect
427 * @name: tracepoint name
428 * @probe: probe handler
430 * caller must call tracepoint_probe_update_all()
431 * Called with the UST lock held.
433 int tracepoint_probe_register_noupdate(const char *name
, void *probe
,
438 old
= tracepoint_add_probe(name
, probe
, data
);
442 tracepoint_add_old_probes(old
);
447 * tracepoint_probe_unregister_noupdate - remove a probe but not disconnect
448 * @name: tracepoint name
449 * @probe: probe function pointer
451 * caller must call tracepoint_probe_update_all()
452 * Called with the UST lock held.
454 int tracepoint_probe_unregister_noupdate(const char *name
, void *probe
,
459 old
= tracepoint_remove_probe(name
, probe
, data
);
463 tracepoint_add_old_probes(old
);
468 * tracepoint_probe_update_all - update tracepoints
469 * Called with the UST lock held.
471 void tracepoint_probe_update_all(void)
473 CDS_LIST_HEAD(release_probes
);
474 struct tp_probes
*pos
, *next
;
479 if (!cds_list_empty(&old_probes
))
480 cds_list_replace_init(&old_probes
, &release_probes
);
483 tracepoint_update_probes();
484 cds_list_for_each_entry_safe(pos
, next
, &release_probes
, u
.list
) {
485 cds_list_del(&pos
->u
.list
);
492 * Returns 0 if current not found.
493 * Returns 1 if current found.
495 * Called with tracepoint mutex held
497 int lib_get_iter_tracepoints(struct tracepoint_iter
*iter
)
499 struct tracepoint_lib
*iter_lib
;
502 cds_list_for_each_entry(iter_lib
, &libs
, list
) {
503 if (iter_lib
< iter
->lib
)
505 else if (iter_lib
> iter
->lib
)
506 iter
->tracepoint
= NULL
;
507 found
= tracepoint_get_iter_range(&iter
->tracepoint
,
508 iter_lib
->tracepoints_start
,
509 iter_lib
->tracepoints_start
+ iter_lib
->tracepoints_count
);
511 iter
->lib
= iter_lib
;
519 * tracepoint_get_iter_range - Get a next tracepoint iterator given a range.
520 * @tracepoint: current tracepoints (in), next tracepoint (out)
521 * @begin: beginning of the range
522 * @end: end of the range
524 * Returns whether a next tracepoint has been found (1) or not (0).
525 * Will return the first tracepoint in the range if the input tracepoint is
527 * Called with tracepoint mutex held.
529 int tracepoint_get_iter_range(struct tracepoint
* const **tracepoint
,
530 struct tracepoint
* const *begin
, struct tracepoint
* const *end
)
532 if (!*tracepoint
&& begin
!= end
)
534 while (*tracepoint
>= begin
&& *tracepoint
< end
) {
536 (*tracepoint
)++; /* skip dummy */
544 * Called with tracepoint mutex held.
546 static void tracepoint_get_iter(struct tracepoint_iter
*iter
)
550 /* tracepoints in libs. */
551 found
= lib_get_iter_tracepoints(iter
);
553 tracepoint_iter_reset(iter
);
557 * Called with UST lock held.
559 void tracepoint_iter_start(struct tracepoint_iter
*iter
)
561 tracepoint_get_iter(iter
);
565 * Called with UST lock held.
567 void tracepoint_iter_next(struct tracepoint_iter
*iter
)
571 * iter->tracepoint may be invalid because we blindly incremented it.
572 * Make sure it is valid by marshalling on the tracepoints, getting the
573 * tracepoints from following modules if necessary.
575 tracepoint_get_iter(iter
);
579 * Called with UST lock held.
581 void tracepoint_iter_stop(struct tracepoint_iter
*iter
)
585 void tracepoint_iter_reset(struct tracepoint_iter
*iter
)
587 iter
->tracepoint
= NULL
;
590 void tracepoint_set_new_tracepoint_cb(void (*cb
)(struct tracepoint
*))
592 new_tracepoint_cb
= cb
;
595 static void new_tracepoints(struct tracepoint
* const *start
, struct tracepoint
* const *end
)
597 if (new_tracepoint_cb
) {
598 struct tracepoint
* const *t
;
600 for (t
= start
; t
< end
; t
++) {
602 new_tracepoint_cb(*t
);
607 int tracepoint_register_lib(struct tracepoint
* const *tracepoints_start
,
608 int tracepoints_count
)
610 struct tracepoint_lib
*pl
, *iter
;
614 pl
= (struct tracepoint_lib
*) zmalloc(sizeof(struct tracepoint_lib
));
616 pl
->tracepoints_start
= tracepoints_start
;
617 pl
->tracepoints_count
= tracepoints_count
;
621 * We sort the libs by struct lib pointer address.
623 cds_list_for_each_entry_reverse(iter
, &libs
, list
) {
624 BUG_ON(iter
== pl
); /* Should never be in the list twice */
626 /* We belong to the location right after iter. */
627 cds_list_add(&pl
->list
, &iter
->list
);
631 /* We should be added at the head of the list */
632 cds_list_add(&pl
->list
, &libs
);
634 new_tracepoints(tracepoints_start
, tracepoints_start
+ tracepoints_count
);
636 /* TODO: update just the loaded lib */
637 lib_update_tracepoints();
640 DBG("just registered a tracepoints section from %p and having %d tracepoints",
641 tracepoints_start
, tracepoints_count
);
646 int tracepoint_unregister_lib(struct tracepoint
* const *tracepoints_start
)
648 struct tracepoint_lib
*lib
;
651 cds_list_for_each_entry(lib
, &libs
, list
) {
652 if (lib
->tracepoints_start
== tracepoints_start
) {
653 struct tracepoint_lib
*lib2free
= lib
;
654 cds_list_del(&lib
->list
);
664 void init_tracepoint(void)
666 if (uatomic_xchg(&initialized
, 1) == 1)
671 void exit_tracepoint(void)