From 0d747f9829ee2f9cb93b0675d9dc6139c9cffda2 Mon Sep 17 00:00:00 2001 From: Mathieu Desnoyers Date: Thu, 21 Sep 2023 12:49:27 +0100 Subject: [PATCH] Use event state as argument to call fast path Eliminate the need to load the event description cache lines (to load a pointer to the event state) on the fast path by passing a pointer to the event state as argument to the call. Signed-off-by: Mathieu Desnoyers --- include/side/trace.h | 31 ++++++++++++++++++------------- src/side.c | 16 ++++++---------- 2 files changed, 24 insertions(+), 23 deletions(-) diff --git a/include/side/trace.h b/include/side/trace.h index f4b1e0f..3d6b350 100644 --- a/include/side/trace.h +++ b/include/side/trace.h @@ -84,6 +84,7 @@ struct side_event_description; struct side_arg_dynamic_struct; struct side_events_register_handle; struct side_arg_variant; +struct side_event_state; enum side_type_label { /* Stack-copy basic types */ @@ -651,15 +652,6 @@ struct side_tracer_dynamic_struct_visitor_ctx { void *priv; /* Private tracer context. */ } SIDE_PACKED; -/* - * This structure is _not_ packed to allow atomic operations on its - * fields. - */ -struct side_event_state { - uintptr_t enabled; - const struct side_callback *callbacks; -}; - struct side_event_description { side_ptr_t(struct side_event_state) state; side_ptr_t(const char) provider_name; @@ -674,6 +666,16 @@ struct side_event_description { uint32_t nr_callbacks; } SIDE_PACKED; +/* + * This structure is _not_ packed to allow atomic operations on its + * fields. + */ +struct side_event_state { + uintptr_t enabled; + const struct side_callback *callbacks; + struct side_event_description *desc; +}; + /* Event and type attributes */ #define side_attr(_key, _value) \ @@ -1775,7 +1777,7 @@ struct side_event_description { .sav = SIDE_PTR_INIT(side_sav), \ .len = SIDE_ARRAY_SIZE(side_sav), \ }; \ - side_call(&(_identifier), &side_arg_vec); \ + side_call(&(side_event_state__##_identifier), &side_arg_vec); \ } #define side_event(_identifier, _sav) \ @@ -1796,7 +1798,7 @@ struct side_event_description { .len = SIDE_ARRAY_SIZE(side_fields), \ .nr_attr = SIDE_ARRAY_SIZE(SIDE_PARAM_SELECT_ARG1(_, ##_attr, side_attr_list())), \ }; \ - side_call_variadic(&(_identifier), &side_arg_vec, &var_struct); \ + side_call_variadic(&(side_event_state__##_identifier), &side_arg_vec, &var_struct); \ } #define side_event_variadic(_identifier, _sav, _var, _attr...) \ @@ -1804,10 +1806,13 @@ struct side_event_description { side_event_call_variadic(_identifier, SIDE_PARAM(_sav), SIDE_PARAM(_var), SIDE_PARAM_SELECT_ARG1(_, ##_attr, side_attr_list())) #define _side_define_event(_linkage, _identifier, _provider, _event, _loglevel, _fields, _flags, _attr...) \ + _linkage struct side_event_description __attribute__((section("side_event_description"))) \ + _identifier; \ _linkage struct side_event_state __attribute__((section("side_event_state"))) \ side_event_state__##_identifier = { \ .enabled = 0, \ .callbacks = &side_empty_callback, \ + .desc = &_identifier, \ }; \ _linkage struct side_event_description __attribute__((section("side_event_description"))) \ _identifier = { \ @@ -1860,9 +1865,9 @@ extern "C" { extern const struct side_callback side_empty_callback; -void side_call(const struct side_event_description *desc, +void side_call(const struct side_event_state *state, const struct side_arg_vec *side_arg_vec); -void side_call_variadic(const struct side_event_description *desc, +void side_call_variadic(const struct side_event_state *state, const struct side_arg_vec *side_arg_vec, const struct side_arg_dynamic_struct *var_struct); diff --git a/src/side.c b/src/side.c index ec5fdbf..54e0b6a 100644 --- a/src/side.c +++ b/src/side.c @@ -62,9 +62,8 @@ static DEFINE_SIDE_LIST_HEAD(side_tracer_list); */ const struct side_callback side_empty_callback = { }; -void side_call(const struct side_event_description *desc, const struct side_arg_vec *side_arg_vec) +void side_call(const struct side_event_state *event_state, const struct side_arg_vec *side_arg_vec) { - struct side_event_state *event_state; struct side_rcu_read_state rcu_read_state; const struct side_callback *side_cb; uintptr_t enabled; @@ -73,26 +72,24 @@ void side_call(const struct side_event_description *desc, const struct side_arg_ return; if (side_unlikely(!initialized)) side_init(); - if (side_unlikely(desc->flags & SIDE_EVENT_FLAG_VARIADIC)) { + if (side_unlikely(event_state->desc->flags & SIDE_EVENT_FLAG_VARIADIC)) { printf("ERROR: unexpected variadic event description\n"); abort(); } - event_state = side_ptr_get(desc->state); enabled = __atomic_load_n(&event_state->enabled, __ATOMIC_RELAXED); if (side_unlikely(enabled & SIDE_EVENT_ENABLED_KERNEL_USER_EVENT_MASK)) { // TODO: call kernel write. } side_rcu_read_begin(&rcu_gp, &rcu_read_state); for (side_cb = side_rcu_dereference(event_state->callbacks); side_cb->u.call != NULL; side_cb++) - side_cb->u.call(desc, side_arg_vec, side_cb->priv); + side_cb->u.call(event_state->desc, side_arg_vec, side_cb->priv); side_rcu_read_end(&rcu_gp, &rcu_read_state); } -void side_call_variadic(const struct side_event_description *desc, +void side_call_variadic(const struct side_event_state *event_state, const struct side_arg_vec *side_arg_vec, const struct side_arg_dynamic_struct *var_struct) { - struct side_event_state *event_state; struct side_rcu_read_state rcu_read_state; const struct side_callback *side_cb; uintptr_t enabled; @@ -101,18 +98,17 @@ void side_call_variadic(const struct side_event_description *desc, return; if (side_unlikely(!initialized)) side_init(); - if (side_unlikely(!(desc->flags & SIDE_EVENT_FLAG_VARIADIC))) { + if (side_unlikely(!(event_state->desc->flags & SIDE_EVENT_FLAG_VARIADIC))) { printf("ERROR: unexpected non-variadic event description\n"); abort(); } - event_state = side_ptr_get(desc->state); enabled = __atomic_load_n(&event_state->enabled, __ATOMIC_RELAXED); if (side_unlikely(enabled & SIDE_EVENT_ENABLED_KERNEL_USER_EVENT_MASK)) { // TODO: call kernel write. } side_rcu_read_begin(&rcu_gp, &rcu_read_state); for (side_cb = side_rcu_dereference(event_state->callbacks); side_cb->u.call_variadic != NULL; side_cb++) - side_cb->u.call_variadic(desc, side_arg_vec, var_struct, side_cb->priv); + side_cb->u.call_variadic(event_state->desc, side_arg_vec, var_struct, side_cb->priv); side_rcu_read_end(&rcu_gp, &rcu_read_state); } -- 2.34.1