Use event state as argument to call fast path

Eliminate the need to load the event description cache lines (to load a
pointer to the event state) on the fast path by passing a pointer to the
event state as argument to the call.

Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>

diff --git a/include/side/trace.h b/include/side/trace.h
index f4b1e0ff66c679385f1f9ed8625b2dc625c2b5dc..3d6b350709ab9cc57069dbc8bd9e92d00be94c50 100644 (file)
--- a/include/side/trace.h
+++ b/include/side/trace.h
@@ -84,6 +84,7 @@ struct side_event_description;
 struct side_arg_dynamic_struct;
 struct side_events_register_handle;
 struct side_arg_variant;
+struct side_event_state;
 
 enum side_type_label {
        /* Stack-copy basic types */
@@ -651,15 +652,6 @@ struct side_tracer_dynamic_struct_visitor_ctx {
        void *priv;             /* Private tracer context. */
 } SIDE_PACKED;
 
-/*
- * This structure is _not_ packed to allow atomic operations on its
- * fields.
- */
-struct side_event_state {
-       uintptr_t enabled;
-       const struct side_callback *callbacks;
-};
-
 struct side_event_description {
        side_ptr_t(struct side_event_state) state;
        side_ptr_t(const char) provider_name;
@@ -674,6 +666,16 @@ struct side_event_description {
        uint32_t nr_callbacks;
 } SIDE_PACKED;
 
+/*
+ * This structure is _not_ packed to allow atomic operations on its
+ * fields.
+ */
+struct side_event_state {
+       uintptr_t enabled;
+       const struct side_callback *callbacks;
+       struct side_event_description *desc;
+};
+
 /* Event and type attributes */
 
 #define side_attr(_key, _value)        \
@@ -1775,7 +1777,7 @@ struct side_event_description {
                        .sav = SIDE_PTR_INIT(side_sav), \
                        .len = SIDE_ARRAY_SIZE(side_sav), \
                }; \
-               side_call(&(_identifier), &side_arg_vec); \
+               side_call(&(side_event_state__##_identifier), &side_arg_vec); \
        }
 
 #define side_event(_identifier, _sav) \
@@ -1796,7 +1798,7 @@ struct side_event_description {
                        .len = SIDE_ARRAY_SIZE(side_fields), \
                        .nr_attr = SIDE_ARRAY_SIZE(SIDE_PARAM_SELECT_ARG1(_, ##_attr, side_attr_list())), \
                }; \
-               side_call_variadic(&(_identifier), &side_arg_vec, &var_struct); \
+               side_call_variadic(&(side_event_state__##_identifier), &side_arg_vec, &var_struct); \
        }
 
 #define side_event_variadic(_identifier, _sav, _var, _attr...) \
@@ -1804,10 +1806,13 @@ struct side_event_description {
                side_event_call_variadic(_identifier, SIDE_PARAM(_sav), SIDE_PARAM(_var), SIDE_PARAM_SELECT_ARG1(_, ##_attr, side_attr_list()))
 
 #define _side_define_event(_linkage, _identifier, _provider, _event, _loglevel, _fields, _flags, _attr...) \
+       _linkage struct side_event_description __attribute__((section("side_event_description"))) \
+                       _identifier; \
        _linkage struct side_event_state __attribute__((section("side_event_state"))) \
                        side_event_state__##_identifier = { \
                .enabled = 0, \
                .callbacks = &side_empty_callback, \
+               .desc = &_identifier, \
        }; \
        _linkage struct side_event_description __attribute__((section("side_event_description"))) \
                        _identifier = { \
@@ -1860,9 +1865,9 @@ extern "C" {
 
 extern const struct side_callback side_empty_callback;
 
-void side_call(const struct side_event_description *desc,
+void side_call(const struct side_event_state *state,
        const struct side_arg_vec *side_arg_vec);
-void side_call_variadic(const struct side_event_description *desc,
+void side_call_variadic(const struct side_event_state *state,
        const struct side_arg_vec *side_arg_vec,
        const struct side_arg_dynamic_struct *var_struct);
 

diff --git a/src/side.c b/src/side.c
index ec5fdbf81b367de829522435f1509a02e5c0b74b..54e0b6a9481420ada73c1d51028889d5effeea25 100644 (file)
--- a/src/side.c
+++ b/src/side.c
@@ -62,9 +62,8 @@ static DEFINE_SIDE_LIST_HEAD(side_tracer_list);
  */
 const struct side_callback side_empty_callback = { };
 
-void side_call(const struct side_event_description *desc, const struct side_arg_vec *side_arg_vec)
+void side_call(const struct side_event_state *event_state, const struct side_arg_vec *side_arg_vec)
 {
-       struct side_event_state *event_state;
        struct side_rcu_read_state rcu_read_state;
        const struct side_callback *side_cb;
        uintptr_t enabled;
@@ -73,26 +72,24 @@ void side_call(const struct side_event_description *desc, const struct side_arg_
                return;
        if (side_unlikely(!initialized))
                side_init();
-       if (side_unlikely(desc->flags & SIDE_EVENT_FLAG_VARIADIC)) {
+       if (side_unlikely(event_state->desc->flags & SIDE_EVENT_FLAG_VARIADIC)) {
                printf("ERROR: unexpected variadic event description\n");
                abort();
        }
-       event_state = side_ptr_get(desc->state);
        enabled = __atomic_load_n(&event_state->enabled, __ATOMIC_RELAXED);
        if (side_unlikely(enabled & SIDE_EVENT_ENABLED_KERNEL_USER_EVENT_MASK)) {
                // TODO: call kernel write.
        }
        side_rcu_read_begin(&rcu_gp, &rcu_read_state);
        for (side_cb = side_rcu_dereference(event_state->callbacks); side_cb->u.call != NULL; side_cb++)
-               side_cb->u.call(desc, side_arg_vec, side_cb->priv);
+               side_cb->u.call(event_state->desc, side_arg_vec, side_cb->priv);
        side_rcu_read_end(&rcu_gp, &rcu_read_state);
 }
 
-void side_call_variadic(const struct side_event_description *desc,
+void side_call_variadic(const struct side_event_state *event_state,
        const struct side_arg_vec *side_arg_vec,
        const struct side_arg_dynamic_struct *var_struct)
 {
-       struct side_event_state *event_state;
        struct side_rcu_read_state rcu_read_state;
        const struct side_callback *side_cb;
        uintptr_t enabled;
@@ -101,18 +98,17 @@ void side_call_variadic(const struct side_event_description *desc,
                return;
        if (side_unlikely(!initialized))
                side_init();
-       if (side_unlikely(!(desc->flags & SIDE_EVENT_FLAG_VARIADIC))) {
+       if (side_unlikely(!(event_state->desc->flags & SIDE_EVENT_FLAG_VARIADIC))) {
                printf("ERROR: unexpected non-variadic event description\n");
                abort();
        }
-       event_state = side_ptr_get(desc->state);
        enabled = __atomic_load_n(&event_state->enabled, __ATOMIC_RELAXED);
        if (side_unlikely(enabled & SIDE_EVENT_ENABLED_KERNEL_USER_EVENT_MASK)) {
                // TODO: call kernel write.
        }
        side_rcu_read_begin(&rcu_gp, &rcu_read_state);
        for (side_cb = side_rcu_dereference(event_state->callbacks); side_cb->u.call_variadic != NULL; side_cb++)
-               side_cb->u.call_variadic(desc, side_arg_vec, var_struct, side_cb->priv);
+               side_cb->u.call_variadic(event_state->desc, side_arg_vec, var_struct, side_cb->priv);
        side_rcu_read_end(&rcu_gp, &rcu_read_state);
 }