src/side.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright 2022 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   4  */
   5
   6 #include <side/trace.h>
   7 #include <string.h>
   8
   9 #include "tracer.h"
  10 #include "rcu.h"
  11
  12 /* Top 8 bits reserved for kernel tracer use. */
  13 #define SIDE_EVENT_ENABLED_KERNEL_MASK                  0xFF000000
  14 #define SIDE_EVENT_ENABLED_KERNEL_USER_EVENT_MASK       0x80000000
  15
  16 /* Allow 2^24 tracer callbacks to be registered on an event. */
  17 #define SIDE_EVENT_ENABLED_USER_MASK                    0x00FFFFFF
  18
  19 static struct side_rcu_gp_state rcu_gp;
  20
  21 /*
  22  * Lazy initialization for early use within library constructors.
  23  */
  24 static bool initialized;
  25
  26 static pthread_mutex_t side_lock = PTHREAD_MUTEX_INITIALIZER;
  27
  28 static
  29 void side_init(void)
  30         __attribute__((constructor));
  31
  32 /*
  33  * The empty callback has a NULL function callback pointer, which stops
  34  * iteration on the array of callbacks immediately.
  35  */
  36 const struct side_callback side_empty_callback;
  37
  38 void side_call(const struct side_event_description *desc, const struct side_arg_vec_description *sav_desc)
  39 {
  40         const struct side_callback *side_cb;
  41         unsigned int rcu_period;
  42         uint32_t enabled;
  43
  44         if (side_unlikely(!initialized))
  45                 side_init();
  46         if (side_unlikely(desc->flags & SIDE_EVENT_FLAG_VARIADIC)) {
  47                 printf("ERROR: unexpected variadic event description\n");
  48                 abort();
  49         }
  50         enabled = __atomic_load_n(desc->enabled, __ATOMIC_RELAXED);
  51         if (side_unlikely(enabled & SIDE_EVENT_ENABLED_KERNEL_USER_EVENT_MASK)) {
  52                 // TODO: call kernel write.
  53         }
  54         if (side_unlikely(!(enabled & SIDE_EVENT_ENABLED_USER_MASK)))
  55                 return;
  56
  57         //TODO: replace tracer_call by rcu iteration on list of registered callbacks
  58         tracer_call(desc, sav_desc, NULL);
  59
  60         rcu_period = side_rcu_read_begin(&rcu_gp);
  61         for (side_cb = side_rcu_dereference(desc->callbacks); side_cb->u.call != NULL; side_cb++)
  62                 side_cb->u.call(desc, sav_desc, side_cb->priv);
  63         side_rcu_read_end(&rcu_gp, rcu_period);
  64 }
  65
  66 void side_call_variadic(const struct side_event_description *desc,
  67         const struct side_arg_vec_description *sav_desc,
  68         const struct side_arg_dynamic_event_struct *var_struct)
  69 {
  70         const struct side_callback *side_cb;
  71         unsigned int rcu_period;
  72         uint32_t enabled;
  73
  74         if (side_unlikely(!initialized))
  75                 side_init();
  76         if (side_unlikely(!(desc->flags & SIDE_EVENT_FLAG_VARIADIC))) {
  77                 printf("ERROR: unexpected non-variadic event description\n");
  78                 abort();
  79         }
  80         enabled = __atomic_load_n(desc->enabled, __ATOMIC_RELAXED);
  81         if (side_unlikely(enabled & SIDE_EVENT_ENABLED_KERNEL_USER_EVENT_MASK)) {
  82                 // TODO: call kernel write.
  83         }
  84         if (side_unlikely(!(enabled & SIDE_EVENT_ENABLED_USER_MASK)))
  85                 return;
  86
  87         //TODO: replace tracer_call by rcu iteration on list of registered callbacks
  88         tracer_call_variadic(desc, sav_desc, var_struct, NULL);
  89
  90         rcu_period = side_rcu_read_begin(&rcu_gp);
  91         for (side_cb = side_rcu_dereference(desc->callbacks); side_cb->u.call_variadic != NULL; side_cb++)
  92                 side_cb->u.call_variadic(desc, sav_desc, var_struct, side_cb->priv);
  93         side_rcu_read_end(&rcu_gp, rcu_period);
  94 }
  95
  96 static
  97 const struct side_callback *side_tracer_callback_lookup(
  98                 const struct side_event_description *desc,
  99                 void (*call)(), void *priv)
 100 {
 101         const struct side_callback *cb;
 102
 103         for (cb = desc->callbacks; cb->u.call != NULL; cb++) {
 104                 if (cb->u.call == call && cb->priv == priv)
 105                         return cb;
 106         }
 107         return NULL;
 108 }
 109
 110 static
 111 int _side_tracer_callback_register(struct side_event_description *desc,
 112                 void (*call)(), void *priv)
 113 {
 114         struct side_callback *old_cb, *new_cb;
 115         int ret = SIDE_ERROR_OK;
 116         uint32_t old_nr_cb;
 117
 118         if (!call)
 119                 return SIDE_ERROR_INVAL;
 120         pthread_mutex_lock(&side_lock);
 121         old_nr_cb = *desc->enabled & SIDE_EVENT_ENABLED_USER_MASK;
 122         if (old_nr_cb == SIDE_EVENT_ENABLED_USER_MASK) {
 123                 ret = SIDE_ERROR_INVAL;
 124                 goto unlock;
 125         }
 126         /* Reject duplicate (call, priv) tuples. */
 127         if (side_tracer_callback_lookup(desc, call, priv)) {
 128                 ret = SIDE_ERROR_EXIST;
 129                 goto unlock;
 130         }
 131         old_cb = (struct side_callback *) desc->callbacks;
 132         /* old_nr_cb + 1 (new cb) + 1 (NULL) */
 133         new_cb = (struct side_callback *) calloc(old_nr_cb + 2, sizeof(struct side_callback));
 134         if (!new_cb) {
 135                 ret = SIDE_ERROR_NOMEM;
 136                 goto unlock;
 137         }
 138         memcpy(new_cb, old_cb, old_nr_cb);
 139         if (desc->flags & SIDE_EVENT_FLAG_VARIADIC)
 140                 new_cb[old_nr_cb].u.call_variadic = call;
 141         else
 142                 new_cb[old_nr_cb].u.call = call;
 143         new_cb[old_nr_cb].priv = priv;
 144         side_rcu_assign_pointer(desc->callbacks, new_cb);
 145         side_rcu_wait_grace_period(&rcu_gp);
 146         if (old_nr_cb)
 147                 free(old_cb);
 148         /* Increment concurrently with kernel setting the top bits. */
 149         (void) __atomic_add_fetch(desc->enabled, 1, __ATOMIC_RELAXED);
 150 unlock:
 151         pthread_mutex_unlock(&side_lock);
 152         return ret;
 153 }
 154
 155 int side_tracer_callback_register(struct side_event_description *desc,
 156                 void (*call)(const struct side_event_description *desc,
 157                         const struct side_arg_vec_description *sav_desc,
 158                         void *priv),
 159                 void *priv)
 160 {
 161         if (desc->flags & SIDE_EVENT_FLAG_VARIADIC)
 162                 return SIDE_ERROR_INVAL;
 163         return _side_tracer_callback_register(desc, call, priv);
 164 }
 165
 166 int side_tracer_callback_variadic_register(struct side_event_description *desc,
 167                 void (*call_variadic)(const struct side_event_description *desc,
 168                         const struct side_arg_vec_description *sav_desc,
 169                         const struct side_arg_dynamic_event_struct *var_struct,
 170                         void *priv),
 171                 void *priv)
 172 {
 173         if (!(desc->flags & SIDE_EVENT_FLAG_VARIADIC))
 174                 return SIDE_ERROR_INVAL;
 175         return _side_tracer_callback_register(desc, call_variadic, priv);
 176 }
 177
 178 int _side_tracer_callback_unregister(struct side_event_description *desc,
 179                 void (*call)(), void *priv)
 180 {
 181         struct side_callback *old_cb, *new_cb;
 182         const struct side_callback *cb_pos;
 183         uint32_t pos_idx;
 184         int ret = SIDE_ERROR_OK;
 185         uint32_t old_nr_cb;
 186
 187         if (!call)
 188                 return SIDE_ERROR_INVAL;
 189         pthread_mutex_lock(&side_lock);
 190         cb_pos = side_tracer_callback_lookup(desc, call, priv);
 191         if (!cb_pos) {
 192                 ret = SIDE_ERROR_NOENT;
 193                 goto unlock;
 194         }
 195         old_nr_cb = *desc->enabled & SIDE_EVENT_ENABLED_USER_MASK;
 196         old_cb = (struct side_callback *) desc->callbacks;
 197         if (old_nr_cb == 1) {
 198                 new_cb = (struct side_callback *) &side_empty_callback;
 199         } else {
 200                 pos_idx = cb_pos - desc->callbacks;
 201                 /* Remove entry at pos_idx. */
 202                 /* old_nr_cb - 1 (removed cb) + 1 (NULL) */
 203                 new_cb = (struct side_callback *) calloc(old_nr_cb, sizeof(struct side_callback));
 204                 if (!new_cb) {
 205                         ret = SIDE_ERROR_NOMEM;
 206                         goto unlock;
 207                 }
 208                 memcpy(new_cb, old_cb, pos_idx);
 209                 memcpy(&new_cb[pos_idx], &old_cb[pos_idx + 1], old_nr_cb - pos_idx - 1);
 210         }
 211         side_rcu_assign_pointer(desc->callbacks, new_cb);
 212         side_rcu_wait_grace_period(&rcu_gp);
 213         free(old_cb);
 214         /* Decrement concurrently with kernel setting the top bits. */
 215         (void) __atomic_add_fetch(desc->enabled, -1, __ATOMIC_RELAXED);
 216 unlock:
 217         pthread_mutex_unlock(&side_lock);
 218         return ret;
 219 }
 220
 221 int side_tracer_callback_unregister(struct side_event_description *desc,
 222                 void (*call)(const struct side_event_description *desc,
 223                         const struct side_arg_vec_description *sav_desc,
 224                         void *priv),
 225                 void *priv)
 226 {
 227         if (desc->flags & SIDE_EVENT_FLAG_VARIADIC)
 228                 return SIDE_ERROR_INVAL;
 229         return _side_tracer_callback_unregister(desc, call, priv);
 230 }
 231
 232 int side_tracer_callback_variadic_unregister(struct side_event_description *desc,
 233                 void (*call_variadic)(const struct side_event_description *desc,
 234                         const struct side_arg_vec_description *sav_desc,
 235                         const struct side_arg_dynamic_event_struct *var_struct,
 236                         void *priv),
 237                 void *priv)
 238 {
 239         if (!(desc->flags & SIDE_EVENT_FLAG_VARIADIC))
 240                 return SIDE_ERROR_INVAL;
 241         return _side_tracer_callback_unregister(desc, call_variadic, priv);
 242 }
 243
 244 static
 245 void side_init(void)
 246 {
 247         if (initialized)
 248                 return;
 249         side_rcu_gp_init(&rcu_gp);
 250         initialized = true;
 251 }