+static
+bool lttng_event_container_current_id_full(struct lttng_event_container *container)
+{
+ switch (container->type) {
+ case LTTNG_EVENT_CONTAINER_CHANNEL:
+ {
+ struct lttng_channel *channel = lttng_event_container_get_channel(container);
+
+ return channel->free_event_id == -1U;
+ }
+ case LTTNG_EVENT_CONTAINER_COUNTER:
+ {
+ struct lttng_counter *counter = lttng_event_container_get_counter(container);
+ size_t nr_dimensions, max_nr_elem;
+
+ if (lttng_counter_get_nr_dimensions(&counter->counter->config,
+ counter->counter, &nr_dimensions))
+ return true;
+ WARN_ON_ONCE(nr_dimensions != 1);
+ if (nr_dimensions != 1)
+ return true;
+ if (lttng_counter_get_max_nr_elem(&counter->counter->config,
+ counter->counter, &max_nr_elem))
+ return true;
+ return counter->free_index >= max_nr_elem;
+ }
+ default:
+ WARN_ON_ONCE(1);
+ return true;
+ }
+}
+
+
+static
+int lttng_event_container_allocate_id(struct lttng_event_container *container,
+ const char *key_string, size_t *id)
+{
+ struct lttng_session *session = container->session;
+ struct lttng_event *event;
+
+ if (key_string[0]) {
+ struct hlist_head *head;
+
+ head = utils_borrow_hash_table_bucket(session->events_key_ht.table,
+ LTTNG_EVENT_HT_SIZE, key_string);
+ lttng_hlist_for_each_entry(event, head, key_hlist) {
+ if (!strcmp(key_string, event->key)) {
+ /* Same key, use same id. */
+ *id = event->id;
+ return 0;
+ }
+ }
+ }
+
+ if (lttng_event_container_current_id_full(container)) {
+ return -EMFILE;
+ }
+
+ switch (container->type) {
+ case LTTNG_EVENT_CONTAINER_CHANNEL:
+ {
+ struct lttng_channel *channel = lttng_event_container_get_channel(container);
+ *id = channel->free_event_id++;
+ break;
+ }
+ case LTTNG_EVENT_CONTAINER_COUNTER:
+ {
+ struct lttng_counter *counter = lttng_event_container_get_counter(container);
+ *id = counter->free_index++;
+ break;
+ }
+ default:
+ WARN_ON_ONCE(1);
+ return 0;
+ }
+
+ return 0;
+}
+
+static
+int format_event_key(char *key_string, const struct lttng_counter_key *key,
+ const char *event_name)
+{
+ const struct lttng_counter_key_dimension *dim;
+ size_t i, left = LTTNG_KEY_TOKEN_STRING_LEN_MAX;
+
+ key_string[0] = '\0';
+ if (!key || !key->nr_dimensions)
+ return 0;
+ /* Currently event keys can only be specified on a single dimension. */
+ if (key->nr_dimensions != 1)
+ return -EINVAL;
+ dim = &key->key_dimensions[0];
+ for (i = 0; i < dim->nr_key_tokens; i++) {
+ const struct lttng_key_token *token = &dim->key_tokens[i];
+ size_t token_len;
+ const char *str;
+
+ switch (token->type) {
+ case LTTNG_KEY_TOKEN_STRING:
+ str = token->arg.string;
+ break;
+ case LTTNG_KEY_TOKEN_EVENT_NAME:
+ str = event_name;
+ break;
+ default:
+ return -EINVAL;
+ }
+ token_len = strlen(str);
+ if (token_len >= left)
+ return -EINVAL;
+ strcat(key_string, str);
+ left -= token_len;
+ }
+ return 0;
+}
+
+static
+bool match_event_token(struct lttng_event_container *container,
+ struct lttng_event *event, uint64_t token)
+{
+ if (container->coalesce_hits)
+ return true;
+ if (event->user_token == token)
+ return true;
+ return false;
+}
+
+static
+int lttng_counter_append_descriptor(struct lttng_counter *counter,
+ uint64_t user_token,
+ size_t index,
+ const char *key)
+{
+ struct lttng_counter_map *map = &counter->map;
+ struct lttng_counter_map_descriptor *last;
+ int ret = 0;
+
+ if (strlen(key) >= LTTNG_KERNEL_COUNTER_KEY_LEN) {
+ WARN_ON_ONCE(1);
+ return -EOVERFLOW;
+ }
+ mutex_lock(&map->lock);
+ if (map->nr_descriptors == map->alloc_len) {
+ struct lttng_counter_map_descriptor *new_table, *old_table;
+ size_t old_len = map->nr_descriptors;
+ size_t new_len = max_t(size_t, old_len + 1, map->alloc_len * 2);
+
+ old_table = map->descriptors;
+ new_table = lttng_kvzalloc(sizeof(struct lttng_counter_map_descriptor) * new_len,
+ GFP_KERNEL);
+ if (!new_table) {
+ ret = -ENOMEM;
+ goto unlock;
+ }
+
+ if (old_table)
+ memcpy(new_table, old_table, old_len * sizeof(struct lttng_counter_map_descriptor));
+
+ map->descriptors = new_table;
+ map->alloc_len = new_len;
+ lttng_kvfree(old_table);
+ }
+ last = &map->descriptors[map->nr_descriptors++];
+ last->user_token = user_token;
+ last->array_index = index;
+ strcpy(last->key, key);
+unlock:
+ mutex_unlock(&map->lock);
+ return ret;
+}
+
+/*
+ * Supports event creation while tracing session is active.
+ * Needs to be called with sessions mutex held.
+ */
+struct lttng_event *_lttng_event_create(struct lttng_event_container *container,
+ struct lttng_kernel_event *event_param,
+ const struct lttng_counter_key *key,
+ void *filter,
+ const struct lttng_event_desc *event_desc,
+ enum lttng_kernel_instrumentation itype,
+ uint64_t token)
+{
+ struct lttng_session *session;
+ struct lttng_event *event;
+ char event_name[LTTNG_KERNEL_SYM_NAME_LEN];
+ struct hlist_head *name_head, *key_head;
+ char key_string[LTTNG_KEY_TOKEN_STRING_LEN_MAX];
+ int ret;
+
+ session = container->session;
+ switch (itype) {
+ case LTTNG_KERNEL_TRACEPOINT:
+ if (strlen(event_desc->name) >= LTTNG_KERNEL_SYM_NAME_LEN) {
+ ret = -EINVAL;
+ goto type_error;
+ }
+ strcpy(event_name, event_desc->name);
+ break;
+ case LTTNG_KERNEL_KPROBE:
+ case LTTNG_KERNEL_UPROBE:
+ case LTTNG_KERNEL_SYSCALL:
+ if (strlen(event_param->name) >= LTTNG_KERNEL_SYM_NAME_LEN) {
+ ret = -EINVAL;
+ goto type_error;
+ }
+ strcpy(event_name, event_param->name);
+ break;
+ case LTTNG_KERNEL_KRETPROBE:
+ if (strlen(event_param->name) >= LTTNG_KERNEL_SYM_NAME_LEN) {
+ ret = -EINVAL;
+ goto type_error;
+ }
+ strcpy(event_name, event_param->name);
+ if (strlen(event_name) + strlen("_entry") >= LTTNG_KERNEL_SYM_NAME_LEN) {
+ ret = -EINVAL;
+ goto type_error;
+ }
+ strcat(event_name, "_entry");
+ break;
+ case LTTNG_KERNEL_FUNCTION: /* Fall-through. */
+ case LTTNG_KERNEL_NOOP: /* Fall-through. */
+ default:
+ WARN_ON_ONCE(1);
+ ret = -EINVAL;
+ goto type_error;
+ }
+
+ if (format_event_key(key_string, key, event_name)) {
+ ret = -EINVAL;
+ goto type_error;
+ }
+
+ name_head = utils_borrow_hash_table_bucket(session->events_name_ht.table,
+ LTTNG_EVENT_HT_SIZE, event_name);
+ lttng_hlist_for_each_entry(event, name_head, name_hlist) {
+ bool same_event = false, same_container = false, same_key = false,
+ same_token = false;
+
+ WARN_ON_ONCE(!event->desc);
+ if (event_desc) {
+ if (event->desc == event_desc)
+ same_event = true;
+ } else {
+ if (!strcmp(event_name, event->desc->name))
+ same_event = true;
+ }
+ if (container == event->container) {
+ same_container = true;
+ if (match_event_token(container, event, token))
+ same_token = true;
+ }
+ if (key_string[0] == '\0' || !strcmp(key_string, event->key))
+ same_key = true;
+ if (same_event && same_container && same_key && same_token) {
+ ret = -EEXIST;
+ goto exist;
+ }
+ }
+
+ event = kmem_cache_zalloc(event_cache, GFP_KERNEL);
+ if (!event) {
+ ret = -ENOMEM;
+ goto cache_error;
+ }
+ event->container = container;
+ event->filter = filter;
+ event->instrumentation = itype;
+ event->evtype = LTTNG_TYPE_EVENT;
+ if (!container->coalesce_hits)
+ event->user_token = token;
+ INIT_LIST_HEAD(&event->filter_bytecode_runtime_head);
+ INIT_LIST_HEAD(&event->enablers_ref_head);
+ if (lttng_event_container_allocate_id(container, key_string,
+ &event->id)) {
+ ret = -EMFILE;
+ goto full;
+ }
+ if (key_string[0]) {
+ key_head = utils_borrow_hash_table_bucket(session->events_key_ht.table,
+ LTTNG_EVENT_HT_SIZE, key_string);
+ hlist_add_head(&event->key_hlist, key_head);
+ }
+ strcpy(event->key, key_string);
+
+ switch (itype) {
+ case LTTNG_KERNEL_TRACEPOINT:
+ /* Event will be enabled by enabler sync. */
+ event->enabled = 0;
+ event->registered = 0;
+ event->desc = lttng_event_desc_get(event_name);
+ if (!event->desc) {
+ ret = -ENOENT;
+ goto register_error;
+ }
+ /* Populate lttng_event structure before event registration. */
+ smp_wmb();
+ break;
+ case LTTNG_KERNEL_KPROBE:
+ /*
+ * Needs to be explicitly enabled after creation, since
+ * we may want to apply filters.
+ */
+ event->enabled = 0;
+ event->registered = 1;
+ /*
+ * Populate lttng_event structure before event
+ * registration.
+ */
+ smp_wmb();
+ ret = lttng_kprobes_register_event(event_name,
+ event_param->u.kprobe.symbol_name,
+ event_param->u.kprobe.offset,
+ event_param->u.kprobe.addr,
+ event);
+ if (ret) {
+ ret = -EINVAL;
+ goto register_error;
+ }
+ event->u.kprobe.user_token = token;
+ ret = try_module_get(event->desc->owner);
+ WARN_ON_ONCE(!ret);
+
+ /* Append descriptor to counter. */
+ switch (container->type) {
+ case LTTNG_EVENT_CONTAINER_COUNTER:
+ {
+ struct lttng_counter *counter;
+ const char *name = "<UNKNOWN>";
+ int ret;
+
+ counter = lttng_event_container_get_counter(container);
+ if (event->key[0])
+ name = event->key;
+ else
+ name = event_name;
+ ret = lttng_counter_append_descriptor(counter,
+ token, event->id,
+ name);
+ if (ret) {
+ WARN_ON_ONCE(1);
+ }
+ break;
+ }
+ case LTTNG_EVENT_CONTAINER_CHANNEL:
+ default:
+ break;
+ }
+ break;
+ case LTTNG_KERNEL_KRETPROBE:
+ {
+ struct lttng_event *event_return;
+
+ /* kretprobe defines 2 events */
+ /*
+ * Needs to be explicitly enabled after creation, since
+ * we may want to apply filters.
+ */
+ event->enabled = 0;
+ event->registered = 1;
+ event->u.kretprobe.user_token = token;
+
+ /* Append descriptor to counter. */
+ switch (container->type) {
+ case LTTNG_EVENT_CONTAINER_COUNTER:
+ {
+ struct lttng_counter *counter;
+ const char *name = "<UNKNOWN>";
+ int ret;
+
+ counter = lttng_event_container_get_counter(container);
+ if (event->key[0])
+ name = event->key;
+ else
+ name = event_name;
+ ret = lttng_counter_append_descriptor(counter,
+ token, event->id,
+ name);
+ if (ret) {
+ WARN_ON_ONCE(1);
+ }
+ break;
+ }
+ case LTTNG_EVENT_CONTAINER_CHANNEL:
+ default:
+ break;
+ }
+
+ event_return =
+ kmem_cache_zalloc(event_cache, GFP_KERNEL);
+ if (!event_return) {
+ ret = -ENOMEM;
+ goto register_error;
+ }
+ event_return->container = container;
+ event_return->filter = filter;
+
+ strcpy(event_name, event_param->name);
+ if (strlen(event_name) + strlen("_return") >= LTTNG_KERNEL_SYM_NAME_LEN) {
+ ret = -EINVAL;
+ goto register_error;
+ }
+ strcat(event_name, "_return");
+ if (format_event_key(key_string, key, event_name)) {
+ ret = -EINVAL;
+ goto register_error;
+ }
+ if (lttng_event_container_allocate_id(container, key_string, &event_return->id)) {
+ kmem_cache_free(event_cache, event_return);
+ ret = -EMFILE;
+ goto register_error;
+ }
+ key_head = utils_borrow_hash_table_bucket(session->events_key_ht.table,
+ LTTNG_EVENT_HT_SIZE, key_string);
+ hlist_add_head(&event_return->key_hlist, key_head);
+ event_return->enabled = 0;
+ event_return->registered = 1;
+ event_return->instrumentation = itype;
+ INIT_LIST_HEAD(&event_return->filter_bytecode_runtime_head);
+ INIT_LIST_HEAD(&event_return->enablers_ref_head);
+ event_return->u.kretprobe.user_token = token;
+ strcpy(event_return->key, key_string);
+ /*
+ * Populate lttng_event structure before kretprobe registration.
+ */
+ smp_wmb();
+ ret = lttng_kretprobes_register(event_name,
+ event_param->u.kretprobe.symbol_name,
+ event_param->u.kretprobe.offset,
+ event_param->u.kretprobe.addr,
+ event, event_return);
+ if (ret) {
+ kmem_cache_free(event_cache, event_return);
+ ret = -EINVAL;
+ goto register_error;
+ }
+ /* Take 2 refs on the module: one per event. */
+ ret = try_module_get(event->desc->owner);
+ WARN_ON_ONCE(!ret);
+ ret = try_module_get(event->desc->owner);
+ WARN_ON_ONCE(!ret);
+
+ /* Append exit descriptor to counter. */
+ switch (container->type) {
+ case LTTNG_EVENT_CONTAINER_COUNTER:
+ {
+ struct lttng_counter *counter;
+ const char *name = "<UNKNOWN>";
+ int ret;
+
+ counter = lttng_event_container_get_counter(container);
+ if (event_return->key[0])
+ name = event_return->key;
+ else
+ name = event_name;
+ ret = lttng_counter_append_descriptor(counter,
+ token, event_return->id,
+ name);
+ if (ret) {
+ WARN_ON_ONCE(1);
+ }
+ break;
+ }
+ case LTTNG_EVENT_CONTAINER_CHANNEL:
+ default:
+ break;
+ }
+ switch (container->type) {
+ case LTTNG_EVENT_CONTAINER_CHANNEL:
+ ret = _lttng_event_metadata_statedump(session, event_return);
+ WARN_ON_ONCE(ret > 0);
+ if (ret) {
+ kmem_cache_free(event_cache, event_return);
+ module_put(event->desc->owner);
+ module_put(event->desc->owner);
+ goto statedump_error;
+ }
+ break;
+ case LTTNG_EVENT_CONTAINER_COUNTER:
+ default:
+ break;
+ }
+ list_add(&event_return->list, &session->events);
+ break;
+ }
+ case LTTNG_KERNEL_SYSCALL:
+ /*
+ * Needs to be explicitly enabled after creation, since
+ * we may want to apply filters.
+ */
+ event->enabled = 0;
+ event->registered = 0;
+ event->desc = event_desc;
+ switch (event_param->u.syscall.entryexit) {
+ case LTTNG_KERNEL_SYSCALL_ENTRYEXIT:
+ ret = -EINVAL;
+ goto register_error;
+ case LTTNG_KERNEL_SYSCALL_ENTRY:
+ event->u.syscall.entryexit = LTTNG_SYSCALL_ENTRY;
+ break;
+ case LTTNG_KERNEL_SYSCALL_EXIT:
+ event->u.syscall.entryexit = LTTNG_SYSCALL_EXIT;
+ break;
+ }
+ switch (event_param->u.syscall.abi) {
+ case LTTNG_KERNEL_SYSCALL_ABI_ALL:
+ ret = -EINVAL;
+ goto register_error;
+ case LTTNG_KERNEL_SYSCALL_ABI_NATIVE:
+ event->u.syscall.abi = LTTNG_SYSCALL_ABI_NATIVE;
+ break;
+ case LTTNG_KERNEL_SYSCALL_ABI_COMPAT:
+ event->u.syscall.abi = LTTNG_SYSCALL_ABI_COMPAT;
+ break;
+ }
+ if (!event->desc) {
+ ret = -EINVAL;
+ goto register_error;
+ }
+ break;
+ case LTTNG_KERNEL_UPROBE:
+ /*
+ * Needs to be explicitly enabled after creation, since
+ * we may want to apply filters.
+ */
+ event->enabled = 0;
+ event->registered = 1;
+ event->u.uprobe.user_token = token;
+
+ /*
+ * Populate lttng_event structure before event
+ * registration.
+ */
+ smp_wmb();
+
+ ret = lttng_uprobes_register_event(event_name,
+ event_param->u.uprobe.fd,
+ event);
+ if (ret)
+ goto register_error;
+ ret = try_module_get(event->desc->owner);
+ WARN_ON_ONCE(!ret);
+
+ /* Append descriptor to counter. */
+ switch (container->type) {
+ case LTTNG_EVENT_CONTAINER_COUNTER:
+ {
+ struct lttng_counter *counter;
+ const char *name = "<UNKNOWN>";
+ int ret;
+
+ counter = lttng_event_container_get_counter(container);
+ if (event->key[0])
+ name = event->key;
+ else
+ name = event_name;
+ ret = lttng_counter_append_descriptor(counter,
+ token, event->id,
+ name);
+ if (ret) {
+ WARN_ON_ONCE(1);
+ }
+ break;
+ }
+ case LTTNG_EVENT_CONTAINER_CHANNEL:
+ default:
+ break;
+ }
+ break;
+ case LTTNG_KERNEL_FUNCTION: /* Fall-through. */
+ case LTTNG_KERNEL_NOOP: /* Fall-through.*/
+ default:
+ WARN_ON_ONCE(1);
+ ret = -EINVAL;
+ goto register_error;
+ }
+ switch (container->type) {
+ case LTTNG_EVENT_CONTAINER_CHANNEL:
+ ret = _lttng_event_metadata_statedump(session, event);
+ WARN_ON_ONCE(ret > 0);
+ if (ret) {
+ goto statedump_error;
+ }
+ break;
+ case LTTNG_EVENT_CONTAINER_COUNTER:
+ default:
+ break;
+ }
+ hlist_add_head(&event->name_hlist, name_head);
+ list_add(&event->list, &session->events);
+ return event;
+
+statedump_error:
+ /* If a statedump error occurs, events will not be readable. */
+register_error:
+full:
+ kmem_cache_free(event_cache, event);
+cache_error:
+exist:
+type_error:
+ return ERR_PTR(ret);
+}
+
+struct lttng_event_notifier *_lttng_event_notifier_create(
+ const struct lttng_event_desc *event_desc,
+ uint64_t token, uint64_t error_counter_index,
+ struct lttng_event_notifier_group *event_notifier_group,
+ struct lttng_kernel_event_notifier *event_notifier_param,
+ void *filter, enum lttng_kernel_instrumentation itype)