1 // SPDX-License-Identifier: MIT
3 * Copyright 2022 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
6 #include <side/trace.h>
13 /* Top 8 bits reserved for kernel tracer use. */
14 #if SIDE_BITS_PER_LONG == 64
15 # define SIDE_EVENT_ENABLED_KERNEL_MASK 0xFF00000000000000ULL
16 # define SIDE_EVENT_ENABLED_KERNEL_USER_EVENT_MASK 0x8000000000000000ULL
18 /* Allow 2^56 tracer references on an event. */
19 # define SIDE_EVENT_ENABLED_USER_MASK 0x00FFFFFFFFFFFFFFULL
21 # define SIDE_EVENT_ENABLED_KERNEL_MASK 0xFF000000UL
22 # define SIDE_EVENT_ENABLED_KERNEL_USER_EVENT_MASK 0x80000000UL
24 /* Allow 2^24 tracer references on an event. */
25 # define SIDE_EVENT_ENABLED_USER_MASK 0x00FFFFFFUL
28 struct side_events_register_handle
{
29 struct side_list_node node
;
30 struct side_event_description
**events
;
34 struct side_tracer_handle
{
35 struct side_list_node node
;
36 void (*cb
)(enum side_tracer_notification notif
,
37 struct side_event_description
**events
, uint32_t nr_events
, void *priv
);
41 struct side_statedump_request_handle
{
42 struct side_list_node node
;
46 struct side_callback
{
48 void (*call
)(const struct side_event_description
*desc
,
49 const struct side_arg_vec
*side_arg_vec
,
51 void (*call_variadic
)(const struct side_event_description
*desc
,
52 const struct side_arg_vec
*side_arg_vec
,
53 const struct side_arg_dynamic_struct
*var_struct
,
60 static struct side_rcu_gp_state rcu_gp
;
63 * Lazy initialization for early use within library constructors.
65 static bool initialized
;
67 * Do not register/unregister any more events after destructor.
69 static bool finalized
;
72 * Recursive mutex to allow tracer callbacks to use the side API.
74 static pthread_mutex_t side_lock
= PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP
;
76 static DEFINE_SIDE_LIST_HEAD(side_events_list
);
77 static DEFINE_SIDE_LIST_HEAD(side_tracer_list
);
78 static DEFINE_SIDE_LIST_HEAD(side_statedump_list
);
81 * Callback filter key for state dump.
83 static __thread
void *filter_key
;
86 * The empty callback has a NULL function callback pointer, which stops
87 * iteration on the array of callbacks immediately.
89 const char side_empty_callback
[sizeof(struct side_callback
)];
92 void _side_call(const struct side_event_state
*event_state
, const struct side_arg_vec
*side_arg_vec
, void *key
)
94 struct side_rcu_read_state rcu_read_state
;
95 const struct side_event_state_0
*es0
;
96 const struct side_callback
*side_cb
;
99 if (side_unlikely(finalized
))
101 if (side_unlikely(!initialized
))
103 if (side_unlikely(event_state
->version
!= 0))
105 es0
= side_container_of(event_state
, const struct side_event_state_0
, parent
);
106 assert(!(es0
->desc
->flags
& SIDE_EVENT_FLAG_VARIADIC
));
107 enabled
= __atomic_load_n(&es0
->enabled
, __ATOMIC_RELAXED
);
108 if (side_unlikely(enabled
& SIDE_EVENT_ENABLED_KERNEL_USER_EVENT_MASK
)) {
109 // TODO: call kernel write.
111 side_rcu_read_begin(&rcu_gp
, &rcu_read_state
);
112 for (side_cb
= side_rcu_dereference(es0
->callbacks
); side_cb
->u
.call
!= NULL
; side_cb
++) {
113 /* A NULL key is always a match. */
114 if (key
&& side_cb
->key
&& side_cb
->key
!= key
)
116 side_cb
->u
.call(es0
->desc
, side_arg_vec
, side_cb
->priv
);
118 side_rcu_read_end(&rcu_gp
, &rcu_read_state
);
121 void side_call(const struct side_event_state
*event_state
, const struct side_arg_vec
*side_arg_vec
)
123 _side_call(event_state
, side_arg_vec
, NULL
);
126 void side_statedump_call(const struct side_event_state
*event_state
, const struct side_arg_vec
*side_arg_vec
)
128 _side_call(event_state
, side_arg_vec
, filter_key
);
132 void _side_call_variadic(const struct side_event_state
*event_state
,
133 const struct side_arg_vec
*side_arg_vec
,
134 const struct side_arg_dynamic_struct
*var_struct
,
137 struct side_rcu_read_state rcu_read_state
;
138 const struct side_event_state_0
*es0
;
139 const struct side_callback
*side_cb
;
142 if (side_unlikely(finalized
))
144 if (side_unlikely(!initialized
))
146 if (side_unlikely(event_state
->version
!= 0))
148 es0
= side_container_of(event_state
, const struct side_event_state_0
, parent
);
149 assert(es0
->desc
->flags
& SIDE_EVENT_FLAG_VARIADIC
);
150 enabled
= __atomic_load_n(&es0
->enabled
, __ATOMIC_RELAXED
);
151 if (side_unlikely(enabled
& SIDE_EVENT_ENABLED_KERNEL_USER_EVENT_MASK
)) {
152 // TODO: call kernel write.
154 side_rcu_read_begin(&rcu_gp
, &rcu_read_state
);
155 for (side_cb
= side_rcu_dereference(es0
->callbacks
); side_cb
->u
.call_variadic
!= NULL
; side_cb
++) {
156 /* A NULL key is always a match. */
157 if (key
&& side_cb
->key
&& side_cb
->key
!= key
)
159 side_cb
->u
.call_variadic(es0
->desc
, side_arg_vec
, var_struct
, side_cb
->priv
);
161 side_rcu_read_end(&rcu_gp
, &rcu_read_state
);
164 void side_call_variadic(const struct side_event_state
*event_state
,
165 const struct side_arg_vec
*side_arg_vec
,
166 const struct side_arg_dynamic_struct
*var_struct
)
168 _side_call_variadic(event_state
, side_arg_vec
, var_struct
, NULL
);
171 void side_statedump_call_variadic(const struct side_event_state
*event_state
,
172 const struct side_arg_vec
*side_arg_vec
,
173 const struct side_arg_dynamic_struct
*var_struct
)
175 _side_call_variadic(event_state
, side_arg_vec
, var_struct
, filter_key
);
179 const struct side_callback
*side_tracer_callback_lookup(
180 const struct side_event_description
*desc
,
181 void *call
, void *priv
, void *key
)
183 struct side_event_state
*event_state
= side_ptr_get(desc
->state
);
184 const struct side_event_state_0
*es0
;
185 const struct side_callback
*cb
;
187 if (side_unlikely(event_state
->version
!= 0))
189 es0
= side_container_of(event_state
, const struct side_event_state_0
, parent
);
190 for (cb
= es0
->callbacks
; cb
->u
.call
!= NULL
; cb
++) {
191 if ((void *) cb
->u
.call
== call
&& cb
->priv
== priv
&& cb
->key
== key
)
198 int _side_tracer_callback_register(struct side_event_description
*desc
,
199 void *call
, void *priv
, void *key
)
201 struct side_event_state
*event_state
;
202 struct side_callback
*old_cb
, *new_cb
;
203 struct side_event_state_0
*es0
;
204 int ret
= SIDE_ERROR_OK
;
208 return SIDE_ERROR_INVAL
;
210 return SIDE_ERROR_EXITING
;
213 pthread_mutex_lock(&side_lock
);
214 event_state
= side_ptr_get(desc
->state
);
215 if (side_unlikely(event_state
->version
!= 0))
217 es0
= side_container_of(event_state
, struct side_event_state_0
, parent
);
218 old_nr_cb
= es0
->nr_callbacks
;
219 if (old_nr_cb
== UINT32_MAX
) {
220 ret
= SIDE_ERROR_INVAL
;
223 /* Reject duplicate (call, priv) tuples. */
224 if (side_tracer_callback_lookup(desc
, call
, priv
, key
)) {
225 ret
= SIDE_ERROR_EXIST
;
228 old_cb
= (struct side_callback
*) es0
->callbacks
;
229 /* old_nr_cb + 1 (new cb) + 1 (NULL) */
230 new_cb
= (struct side_callback
*) calloc(old_nr_cb
+ 2, sizeof(struct side_callback
));
232 ret
= SIDE_ERROR_NOMEM
;
235 memcpy(new_cb
, old_cb
, old_nr_cb
);
236 if (desc
->flags
& SIDE_EVENT_FLAG_VARIADIC
)
237 new_cb
[old_nr_cb
].u
.call_variadic
=
238 (side_tracer_callback_variadic_func
) call
;
240 new_cb
[old_nr_cb
].u
.call
=
241 (side_tracer_callback_func
) call
;
242 new_cb
[old_nr_cb
].priv
= priv
;
243 new_cb
[old_nr_cb
].key
= key
;
244 /* High order bits are already zeroed. */
245 side_rcu_assign_pointer(es0
->callbacks
, new_cb
);
246 side_rcu_wait_grace_period(&rcu_gp
);
250 /* Increment concurrently with kernel setting the top bits. */
252 (void) __atomic_add_fetch(&es0
->enabled
, 1, __ATOMIC_RELAXED
);
254 pthread_mutex_unlock(&side_lock
);
258 int side_tracer_callback_register(struct side_event_description
*desc
,
259 side_tracer_callback_func call
,
260 void *priv
, void *key
)
262 if (desc
->flags
& SIDE_EVENT_FLAG_VARIADIC
)
263 return SIDE_ERROR_INVAL
;
264 return _side_tracer_callback_register(desc
, (void *) call
, priv
, key
);
267 int side_tracer_callback_variadic_register(struct side_event_description
*desc
,
268 side_tracer_callback_variadic_func call_variadic
,
269 void *priv
, void *key
)
271 if (!(desc
->flags
& SIDE_EVENT_FLAG_VARIADIC
))
272 return SIDE_ERROR_INVAL
;
273 return _side_tracer_callback_register(desc
, (void *) call_variadic
, priv
, key
);
276 static int _side_tracer_callback_unregister(struct side_event_description
*desc
,
277 void *call
, void *priv
, void *key
)
279 struct side_event_state
*event_state
;
280 struct side_callback
*old_cb
, *new_cb
;
281 const struct side_callback
*cb_pos
;
282 struct side_event_state_0
*es0
;
284 int ret
= SIDE_ERROR_OK
;
288 return SIDE_ERROR_INVAL
;
290 return SIDE_ERROR_EXITING
;
293 pthread_mutex_lock(&side_lock
);
294 event_state
= side_ptr_get(desc
->state
);
295 if (side_unlikely(event_state
->version
!= 0))
297 es0
= side_container_of(event_state
, struct side_event_state_0
, parent
);
298 cb_pos
= side_tracer_callback_lookup(desc
, call
, priv
, key
);
300 ret
= SIDE_ERROR_NOENT
;
303 old_nr_cb
= es0
->nr_callbacks
;
304 old_cb
= (struct side_callback
*) es0
->callbacks
;
305 if (old_nr_cb
== 1) {
306 new_cb
= (struct side_callback
*) &side_empty_callback
;
308 pos_idx
= cb_pos
- es0
->callbacks
;
309 /* Remove entry at pos_idx. */
310 /* old_nr_cb - 1 (removed cb) + 1 (NULL) */
311 new_cb
= (struct side_callback
*) calloc(old_nr_cb
, sizeof(struct side_callback
));
313 ret
= SIDE_ERROR_NOMEM
;
316 memcpy(new_cb
, old_cb
, pos_idx
);
317 memcpy(&new_cb
[pos_idx
], &old_cb
[pos_idx
+ 1], old_nr_cb
- pos_idx
- 1);
319 /* High order bits are already zeroed. */
320 side_rcu_assign_pointer(es0
->callbacks
, new_cb
);
321 side_rcu_wait_grace_period(&rcu_gp
);
324 /* Decrement concurrently with kernel setting the top bits. */
326 (void) __atomic_add_fetch(&es0
->enabled
, -1, __ATOMIC_RELAXED
);
328 pthread_mutex_unlock(&side_lock
);
332 int side_tracer_callback_unregister(struct side_event_description
*desc
,
333 side_tracer_callback_func call
,
334 void *priv
, void *key
)
336 if (desc
->flags
& SIDE_EVENT_FLAG_VARIADIC
)
337 return SIDE_ERROR_INVAL
;
338 return _side_tracer_callback_unregister(desc
, (void *) call
, priv
, key
);
341 int side_tracer_callback_variadic_unregister(struct side_event_description
*desc
,
342 side_tracer_callback_variadic_func call_variadic
,
343 void *priv
, void *key
)
345 if (!(desc
->flags
& SIDE_EVENT_FLAG_VARIADIC
))
346 return SIDE_ERROR_INVAL
;
347 return _side_tracer_callback_unregister(desc
, (void *) call_variadic
, priv
, key
);
350 struct side_events_register_handle
*side_events_register(struct side_event_description
**events
, uint32_t nr_events
)
352 struct side_events_register_handle
*events_handle
= NULL
;
353 struct side_tracer_handle
*tracer_handle
;
359 events_handle
= (struct side_events_register_handle
*)
360 calloc(1, sizeof(struct side_events_register_handle
));
363 events_handle
->events
= events
;
364 events_handle
->nr_events
= nr_events
;
366 pthread_mutex_lock(&side_lock
);
367 side_list_insert_node_tail(&side_events_list
, &events_handle
->node
);
368 side_list_for_each_entry(tracer_handle
, &side_tracer_list
, node
) {
369 tracer_handle
->cb(SIDE_TRACER_NOTIFICATION_INSERT_EVENTS
,
370 events
, nr_events
, tracer_handle
->priv
);
372 pthread_mutex_unlock(&side_lock
);
373 //TODO: call event batch register ioctl
374 return events_handle
;
378 void side_event_remove_callbacks(struct side_event_description
*desc
)
380 struct side_event_state
*event_state
= side_ptr_get(desc
->state
);
381 struct side_event_state_0
*es0
;
382 struct side_callback
*old_cb
;
385 if (side_unlikely(event_state
->version
!= 0))
387 es0
= side_container_of(event_state
, struct side_event_state_0
, parent
);
388 nr_cb
= es0
->nr_callbacks
;
391 old_cb
= (struct side_callback
*) es0
->callbacks
;
392 (void) __atomic_add_fetch(&es0
->enabled
, -1, __ATOMIC_RELAXED
);
394 * Setting the state back to 0 cb and empty callbacks out of
395 * caution. This should not matter because instrumentation is
398 es0
->nr_callbacks
= 0;
399 side_rcu_assign_pointer(es0
->callbacks
, &side_empty_callback
);
401 * No need to wait for grace period because instrumentation is
408 * Unregister event handle. At this point, all side events in that
409 * handle should be unreachable.
411 void side_events_unregister(struct side_events_register_handle
*events_handle
)
413 struct side_tracer_handle
*tracer_handle
;
422 pthread_mutex_lock(&side_lock
);
423 side_list_remove_node(&events_handle
->node
);
424 side_list_for_each_entry(tracer_handle
, &side_tracer_list
, node
) {
425 tracer_handle
->cb(SIDE_TRACER_NOTIFICATION_REMOVE_EVENTS
,
426 events_handle
->events
, events_handle
->nr_events
,
427 tracer_handle
->priv
);
429 for (i
= 0; i
< events_handle
->nr_events
; i
++) {
430 struct side_event_description
*event
= events_handle
->events
[i
];
432 /* Skip NULL pointers */
435 side_event_remove_callbacks(event
);
437 pthread_mutex_unlock(&side_lock
);
438 //TODO: call event batch unregister ioctl
442 struct side_tracer_handle
*side_tracer_event_notification_register(
443 void (*cb
)(enum side_tracer_notification notif
,
444 struct side_event_description
**events
, uint32_t nr_events
, void *priv
),
447 struct side_tracer_handle
*tracer_handle
;
448 struct side_events_register_handle
*events_handle
;
454 tracer_handle
= (struct side_tracer_handle
*)
455 calloc(1, sizeof(struct side_tracer_handle
));
458 pthread_mutex_lock(&side_lock
);
459 tracer_handle
->cb
= cb
;
460 tracer_handle
->priv
= priv
;
461 side_list_insert_node_tail(&side_tracer_list
, &tracer_handle
->node
);
462 side_list_for_each_entry(events_handle
, &side_events_list
, node
) {
463 cb(SIDE_TRACER_NOTIFICATION_INSERT_EVENTS
,
464 events_handle
->events
, events_handle
->nr_events
, priv
);
466 pthread_mutex_unlock(&side_lock
);
467 return tracer_handle
;
470 void side_tracer_event_notification_unregister(struct side_tracer_handle
*tracer_handle
)
472 struct side_events_register_handle
*events_handle
;
478 pthread_mutex_lock(&side_lock
);
479 side_list_for_each_entry(events_handle
, &side_events_list
, node
) {
480 tracer_handle
->cb(SIDE_TRACER_NOTIFICATION_REMOVE_EVENTS
,
481 events_handle
->events
, events_handle
->nr_events
,
482 tracer_handle
->priv
);
484 side_list_remove_node(&tracer_handle
->node
);
485 pthread_mutex_unlock(&side_lock
);
489 struct side_statedump_request_handle
*side_statedump_request_notification_register(void (*statedump_cb
)(void))
491 struct side_statedump_request_handle
*handle
;
498 * The statedump request notification should not be registered
499 * from a notification callback.
501 assert(filter_key
== NULL
);
502 handle
= (struct side_statedump_request_handle
*)
503 calloc(1, sizeof(struct side_statedump_request_handle
));
506 pthread_mutex_lock(&side_lock
);
507 handle
->cb
= statedump_cb
;
508 side_list_insert_node_tail(&side_statedump_list
, &handle
->node
);
509 /* Invoke callback for all tracers. */
511 pthread_mutex_unlock(&side_lock
);
515 void side_statedump_request_notification_unregister(struct side_statedump_request_handle
*handle
)
521 assert(filter_key
== NULL
);
522 pthread_mutex_lock(&side_lock
);
523 side_list_remove_node(&handle
->node
);
524 pthread_mutex_unlock(&side_lock
);
528 void side_tracer_statedump_request(void *key
)
530 struct side_statedump_request_handle
*handle
;
532 /* Invoke the state dump callback specifically for the tracer key. */
534 pthread_mutex_lock(&side_lock
);
535 side_list_for_each_entry(handle
, &side_statedump_list
, node
)
537 pthread_mutex_unlock(&side_lock
);
545 side_rcu_gp_init(&rcu_gp
);
550 * side_exit() is executed from a library destructor. It can be called
551 * explicitly at application exit as well. Concurrent side API use is
552 * not expected at that point.
556 struct side_events_register_handle
*handle
, *tmp
;
560 side_list_for_each_entry_safe(handle
, tmp
, &side_events_list
, node
)
561 side_events_unregister(handle
);
562 side_rcu_gp_exit(&rcu_gp
);