X-Git-Url: http://git.efficios.com/?a=blobdiff_plain;f=lttng-events.c;h=73cb361081c5f93b21eae258d3279057a7cc0378;hb=refs%2Fheads%2Fsow-2019-0002-rev1;hp=f648feb67a64c8ab180c429ec77db907e2ca5f8e;hpb=9616f0bf9524fc6b8ad184e87f4935c97f78a8bc;p=deliverable%2Flttng-modules.git diff --git a/lttng-events.c b/lttng-events.c index f648feb6..73cb3610 100644 --- a/lttng-events.c +++ b/lttng-events.c @@ -1,23 +1,10 @@ -/* +/* SPDX-License-Identifier: (GPL-2.0 or LGPL-2.1) + * * lttng-events.c * * Holds LTTng per-session event registry. * * Copyright (C) 2010-2012 Mathieu Desnoyers - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; only - * version 2.1 of the License. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ /* @@ -36,39 +23,50 @@ #include #include #include -#include "wrapper/file.h" -#include +#include #include #include - -#include "wrapper/uuid.h" -#include "wrapper/vmalloc.h" /* for wrapper_vmalloc_sync_all() */ -#include "wrapper/random.h" -#include "wrapper/tracepoint.h" -#include "wrapper/list.h" -#include "lttng-kernel-version.h" -#include "lttng-events.h" -#include "lttng-tracer.h" -#include "lttng-abi-old.h" -#include "wrapper/vzalloc.h" +#include +#include + +#include /* for wrapper_vmalloc_sync_all() */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #define METADATA_CACHE_DEFAULT_SIZE 4096 static LIST_HEAD(sessions); +static LIST_HEAD(trigger_groups); static LIST_HEAD(lttng_transport_list); /* * Protect the sessions and metadata caches. */ static DEFINE_MUTEX(sessions_mutex); static struct kmem_cache *event_cache; +static struct kmem_cache *trigger_cache; -static void lttng_session_lazy_sync_enablers(struct lttng_session *session); -static void lttng_session_sync_enablers(struct lttng_session *session); -static void lttng_enabler_destroy(struct lttng_enabler *enabler); +static void lttng_session_lazy_sync_event_enablers(struct lttng_session *session); +static void lttng_session_sync_event_enablers(struct lttng_session *session); +static void lttng_event_enabler_destroy(struct lttng_event_enabler *event_enabler); +static void lttng_trigger_enabler_destroy(struct lttng_trigger_enabler *trigger_enabler); +static void lttng_trigger_group_sync_enablers(struct lttng_trigger_group *trigger_group); static void _lttng_event_destroy(struct lttng_event *event); +static void _lttng_trigger_destroy(struct lttng_trigger *trigger); static void _lttng_channel_destroy(struct lttng_channel *chan); static int _lttng_event_unregister(struct lttng_event *event); +static int _lttng_trigger_unregister(struct lttng_trigger *trigger); static int _lttng_event_metadata_statedump(struct lttng_session *session, struct lttng_channel *chan, @@ -77,10 +75,19 @@ static int _lttng_session_metadata_statedump(struct lttng_session *session); static void _lttng_metadata_channel_hangup(struct lttng_metadata_stream *stream); +static +int _lttng_field_statedump(struct lttng_session *session, + const struct lttng_event_field *field, + size_t nesting); void synchronize_trace(void) { +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5,1,0)) + synchronize_rcu(); +#else synchronize_sched(); +#endif + #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3,4,0)) #ifdef CONFIG_PREEMPT_RT_FULL synchronize_rcu(); @@ -102,6 +109,17 @@ void lttng_unlock_sessions(void) mutex_unlock(&sessions_mutex); } +static struct lttng_transport *lttng_transport_find(const char *name) +{ + struct lttng_transport *transport; + + list_for_each_entry(transport, <tng_transport_list, node) { + if (!strcmp(transport->name, name)) + return transport; + } + return NULL; +} + /* * Called with sessions lock held. */ @@ -123,7 +141,7 @@ struct lttng_session *lttng_session_create(void) int i; mutex_lock(&sessions_mutex); - session = kzalloc(sizeof(struct lttng_session), GFP_KERNEL); + session = lttng_kvzalloc(sizeof(struct lttng_session), GFP_KERNEL); if (!session) goto err; INIT_LIST_HEAD(&session->chan); @@ -134,7 +152,7 @@ struct lttng_session *lttng_session_create(void) GFP_KERNEL); if (!metadata_cache) goto err_free_session; - metadata_cache->data = lttng_vzalloc(METADATA_CACHE_DEFAULT_SIZE); + metadata_cache->data = vzalloc(METADATA_CACHE_DEFAULT_SIZE); if (!metadata_cache->data) goto err_free_cache; metadata_cache->cache_alloc = METADATA_CACHE_DEFAULT_SIZE; @@ -148,18 +166,87 @@ struct lttng_session *lttng_session_create(void) for (i = 0; i < LTTNG_EVENT_HT_SIZE; i++) INIT_HLIST_HEAD(&session->events_ht.table[i]); list_add(&session->list, &sessions); + session->pid_tracker.session = session; + session->pid_tracker.tracker_type = TRACKER_PID; + session->vpid_tracker.session = session; + session->vpid_tracker.tracker_type = TRACKER_VPID; + session->uid_tracker.session = session; + session->uid_tracker.tracker_type = TRACKER_UID; + session->vuid_tracker.session = session; + session->vuid_tracker.tracker_type = TRACKER_VUID; + session->gid_tracker.session = session; + session->gid_tracker.tracker_type = TRACKER_GID; + session->vgid_tracker.session = session; + session->vgid_tracker.tracker_type = TRACKER_VGID; mutex_unlock(&sessions_mutex); return session; err_free_cache: kfree(metadata_cache); err_free_session: - kfree(session); + lttng_kvfree(session); err: mutex_unlock(&sessions_mutex); return NULL; } +struct lttng_trigger_group *lttng_trigger_group_create(void) +{ + struct lttng_transport *transport = NULL; + struct lttng_trigger_group *trigger_group; + const char *transport_name = "relay-trigger"; + size_t subbuf_size = 4096; //TODO + size_t num_subbuf = 16; //TODO + unsigned int switch_timer_interval = 0; + unsigned int read_timer_interval = 0; + int i; + + mutex_lock(&sessions_mutex); + + transport = lttng_transport_find(transport_name); + if (!transport) { + printk(KERN_WARNING "LTTng transport %s not found\n", + transport_name); + goto notransport; + } + if (!try_module_get(transport->owner)) { + printk(KERN_WARNING "LTT : Can't lock transport module.\n"); + goto notransport; + } + + trigger_group = lttng_kvzalloc(sizeof(struct lttng_trigger_group), + GFP_KERNEL); + if (!trigger_group) + goto nomem; + + trigger_group->ops = &transport->ops; + trigger_group->chan = transport->ops.channel_create(transport_name, + trigger_group, NULL, subbuf_size, num_subbuf, + switch_timer_interval, read_timer_interval); + if (!trigger_group->chan) + goto create_error; + + trigger_group->transport = transport; + INIT_LIST_HEAD(&trigger_group->enablers_head); + INIT_LIST_HEAD(&trigger_group->triggers_head); + for (i = 0; i < LTTNG_TRIGGER_HT_SIZE; i++) + INIT_HLIST_HEAD(&trigger_group->triggers_ht.table[i]); + + list_add(&trigger_group->node, &trigger_groups); + mutex_unlock(&sessions_mutex); + + return trigger_group; + +create_error: + lttng_kvfree(trigger_group); +nomem: + if (transport) + module_put(transport->owner); +notransport: + mutex_unlock(&sessions_mutex); + return NULL; +} + void metadata_cache_destroy(struct kref *kref) { struct lttng_metadata_cache *cache = @@ -173,13 +260,13 @@ void lttng_session_destroy(struct lttng_session *session) struct lttng_channel *chan, *tmpchan; struct lttng_event *event, *tmpevent; struct lttng_metadata_stream *metadata_stream; - struct lttng_enabler *enabler, *tmpenabler; + struct lttng_event_enabler *event_enabler, *tmp_event_enabler; int ret; mutex_lock(&sessions_mutex); - ACCESS_ONCE(session->active) = 0; + WRITE_ONCE(session->active, 0); list_for_each_entry(chan, &session->chan, list) { - ret = lttng_syscalls_unregister(chan); + ret = lttng_syscalls_unregister_event(chan); WARN_ON(ret); } list_for_each_entry(event, &session->events, list) { @@ -187,9 +274,9 @@ void lttng_session_destroy(struct lttng_session *session) WARN_ON(ret); } synchronize_trace(); /* Wait for in-flight events to complete */ - list_for_each_entry_safe(enabler, tmpenabler, + list_for_each_entry_safe(event_enabler, tmp_event_enabler, &session->enablers_head, node) - lttng_enabler_destroy(enabler); + lttng_event_enabler_destroy(event_enabler); list_for_each_entry_safe(event, tmpevent, &session->events, list) _lttng_event_destroy(event); list_for_each_entry_safe(chan, tmpchan, &session->chan, list) { @@ -198,12 +285,65 @@ void lttng_session_destroy(struct lttng_session *session) } list_for_each_entry(metadata_stream, &session->metadata_cache->metadata_stream, list) _lttng_metadata_channel_hangup(metadata_stream); - if (session->pid_tracker) - lttng_pid_tracker_destroy(session->pid_tracker); + lttng_id_tracker_destroy(&session->pid_tracker, false); + lttng_id_tracker_destroy(&session->vpid_tracker, false); + lttng_id_tracker_destroy(&session->uid_tracker, false); + lttng_id_tracker_destroy(&session->vuid_tracker, false); + lttng_id_tracker_destroy(&session->gid_tracker, false); + lttng_id_tracker_destroy(&session->vgid_tracker, false); kref_put(&session->metadata_cache->refcount, metadata_cache_destroy); list_del(&session->list); mutex_unlock(&sessions_mutex); - kfree(session); + lttng_kvfree(session); +} + +void lttng_trigger_group_destroy(struct lttng_trigger_group *trigger_group) +{ + struct lttng_trigger_enabler *trigger_enabler, *tmp_trigger_enabler; + struct lttng_trigger *trigger, *tmptrigger; + int ret; + + if (!trigger_group) + return; + + mutex_lock(&sessions_mutex); + + ret = lttng_syscalls_unregister_trigger(trigger_group); + WARN_ON(ret); + + list_for_each_entry_safe(trigger, tmptrigger, + &trigger_group->triggers_head, list) { + ret = _lttng_trigger_unregister(trigger); + WARN_ON(ret); + } + + synchronize_trace(); /* Wait for in-flight triggers to complete */ + + irq_work_sync(&trigger_group->wakeup_pending); + + list_for_each_entry_safe(trigger_enabler, tmp_trigger_enabler, + &trigger_group->enablers_head, node) + lttng_trigger_enabler_destroy(trigger_enabler); + + list_for_each_entry_safe(trigger, tmptrigger, + &trigger_group->triggers_head, list) + _lttng_trigger_destroy(trigger); + + trigger_group->ops->channel_destroy(trigger_group->chan); + module_put(trigger_group->transport->owner); + list_del(&trigger_group->node); + mutex_unlock(&sessions_mutex); + lttng_kvfree(trigger_group); +} + +int lttng_session_statedump(struct lttng_session *session) +{ + int ret; + + mutex_lock(&sessions_mutex); + ret = lttng_statedump_start(session); + mutex_unlock(&sessions_mutex); + return ret; } int lttng_session_enable(struct lttng_session *session) @@ -220,6 +360,9 @@ int lttng_session_enable(struct lttng_session *session) /* Set transient enabler state to "enabled" */ session->tstate = 1; + /* We need to sync enablers with session before activation. */ + lttng_session_sync_event_enablers(session); + /* * Snapshot the number of events per channel to know the type of header * we need to use. @@ -233,19 +376,22 @@ int lttng_session_enable(struct lttng_session *session) chan->header_type = 2; /* large */ } - /* We need to sync enablers with session before activation. */ - lttng_session_sync_enablers(session); + /* Clear each stream's quiescent state. */ + list_for_each_entry(chan, &session->chan, list) { + if (chan->channel_type != METADATA_CHANNEL) + lib_ring_buffer_clear_quiescent_channel(chan->chan); + } - ACCESS_ONCE(session->active) = 1; - ACCESS_ONCE(session->been_active) = 1; + WRITE_ONCE(session->active, 1); + WRITE_ONCE(session->been_active, 1); ret = _lttng_session_metadata_statedump(session); if (ret) { - ACCESS_ONCE(session->active) = 0; + WRITE_ONCE(session->active, 0); goto end; } ret = lttng_statedump_start(session); if (ret) - ACCESS_ONCE(session->active) = 0; + WRITE_ONCE(session->active, 0); end: mutex_unlock(&sessions_mutex); return ret; @@ -254,17 +400,24 @@ end: int lttng_session_disable(struct lttng_session *session) { int ret = 0; + struct lttng_channel *chan; mutex_lock(&sessions_mutex); if (!session->active) { ret = -EBUSY; goto end; } - ACCESS_ONCE(session->active) = 0; + WRITE_ONCE(session->active, 0); /* Set transient enabler state to "disabled" */ session->tstate = 0; - lttng_session_sync_enablers(session); + lttng_session_sync_event_enablers(session); + + /* Set each stream's quiescent state. */ + list_for_each_entry(chan, &session->chan, list) { + if (chan->channel_type != METADATA_CHANNEL) + lib_ring_buffer_set_quiescent_channel(chan->chan); + } end: mutex_unlock(&sessions_mutex); return ret; @@ -310,8 +463,6 @@ end: return ret; } - - int lttng_channel_enable(struct lttng_channel *channel) { int ret = 0; @@ -327,9 +478,9 @@ int lttng_channel_enable(struct lttng_channel *channel) } /* Set transient enabler state to "enabled" */ channel->tstate = 1; - lttng_session_sync_enablers(channel->session); + lttng_session_sync_event_enablers(channel->session); /* Set atomically the state to "enabled" */ - ACCESS_ONCE(channel->enabled) = 1; + WRITE_ONCE(channel->enabled, 1); end: mutex_unlock(&sessions_mutex); return ret; @@ -349,10 +500,10 @@ int lttng_channel_disable(struct lttng_channel *channel) goto end; } /* Set atomically the state to "disabled" */ - ACCESS_ONCE(channel->enabled) = 0; + WRITE_ONCE(channel->enabled, 0); /* Set transient enabler state to "enabled" */ channel->tstate = 0; - lttng_session_sync_enablers(channel->session); + lttng_session_sync_event_enablers(channel->session); end: mutex_unlock(&sessions_mutex); return ret; @@ -378,8 +529,9 @@ int lttng_event_enable(struct lttng_event *event) break; case LTTNG_KERNEL_KPROBE: case LTTNG_KERNEL_FUNCTION: + case LTTNG_KERNEL_UPROBE: case LTTNG_KERNEL_NOOP: - ACCESS_ONCE(event->enabled) = 1; + WRITE_ONCE(event->enabled, 1); break; case LTTNG_KERNEL_KRETPROBE: ret = lttng_kretprobes_event_enable_state(event, 1); @@ -413,8 +565,9 @@ int lttng_event_disable(struct lttng_event *event) break; case LTTNG_KERNEL_KPROBE: case LTTNG_KERNEL_FUNCTION: + case LTTNG_KERNEL_UPROBE: case LTTNG_KERNEL_NOOP: - ACCESS_ONCE(event->enabled) = 0; + WRITE_ONCE(event->enabled, 0); break; case LTTNG_KERNEL_KRETPROBE: ret = lttng_kretprobes_event_enable_state(event, 0); @@ -428,15 +581,64 @@ end: return ret; } -static struct lttng_transport *lttng_transport_find(const char *name) +int lttng_trigger_enable(struct lttng_trigger *trigger) { - struct lttng_transport *transport; + int ret = 0; - list_for_each_entry(transport, <tng_transport_list, node) { - if (!strcmp(transport->name, name)) - return transport; + mutex_lock(&sessions_mutex); + if (trigger->enabled) { + ret = -EEXIST; + goto end; } - return NULL; + switch (trigger->instrumentation) { + case LTTNG_KERNEL_TRACEPOINT: + case LTTNG_KERNEL_SYSCALL: + ret = -EINVAL; + break; + case LTTNG_KERNEL_KPROBE: + case LTTNG_KERNEL_UPROBE: + WRITE_ONCE(trigger->enabled, 1); + break; + case LTTNG_KERNEL_FUNCTION: + case LTTNG_KERNEL_NOOP: + case LTTNG_KERNEL_KRETPROBE: + default: + WARN_ON_ONCE(1); + ret = -EINVAL; + } +end: + mutex_unlock(&sessions_mutex); + return ret; +} + +int lttng_trigger_disable(struct lttng_trigger *trigger) +{ + int ret = 0; + + mutex_lock(&sessions_mutex); + if (!trigger->enabled) { + ret = -EEXIST; + goto end; + } + switch (trigger->instrumentation) { + case LTTNG_KERNEL_TRACEPOINT: + case LTTNG_KERNEL_SYSCALL: + ret = -EINVAL; + break; + case LTTNG_KERNEL_KPROBE: + case LTTNG_KERNEL_UPROBE: + WRITE_ONCE(trigger->enabled, 0); + break; + case LTTNG_KERNEL_FUNCTION: + case LTTNG_KERNEL_NOOP: + case LTTNG_KERNEL_KRETPROBE: + default: + WARN_ON_ONCE(1); + ret = -EINVAL; + } +end: + mutex_unlock(&sessions_mutex); + return ret; } struct lttng_channel *lttng_channel_create(struct lttng_session *session, @@ -545,8 +747,6 @@ struct lttng_event *_lttng_event_create(struct lttng_channel *chan, struct lttng_event *event; const char *event_name; struct hlist_head *head; - size_t name_len; - uint32_t hash; int ret; if (chan->free_event_id == -1U) { @@ -559,6 +759,7 @@ struct lttng_event *_lttng_event_create(struct lttng_channel *chan, event_name = event_desc->name; break; case LTTNG_KERNEL_KPROBE: + case LTTNG_KERNEL_UPROBE: case LTTNG_KERNEL_KRETPROBE: case LTTNG_KERNEL_FUNCTION: case LTTNG_KERNEL_NOOP: @@ -570,9 +771,9 @@ struct lttng_event *_lttng_event_create(struct lttng_channel *chan, ret = -EINVAL; goto type_error; } - name_len = strlen(event_name); - hash = jhash(event_name, name_len, 0); - head = &session->events_ht.table[hash & (LTTNG_EVENT_HT_SIZE - 1)]; + + head = utils_borrow_hash_table_bucket(session->events_ht.table, + LTTNG_EVENT_HT_SIZE, event_name); lttng_hlist_for_each_entry(event, head, hlist) { WARN_ON_ONCE(!event->desc); if (!strncmp(event->desc->name, event_name, @@ -601,7 +802,7 @@ struct lttng_event *_lttng_event_create(struct lttng_channel *chan, /* Event will be enabled by enabler sync. */ event->enabled = 0; event->registered = 0; - event->desc = lttng_event_get(event_name); + event->desc = lttng_event_desc_get(event_name); if (!event->desc) { ret = -ENOENT; goto register_error; @@ -621,7 +822,7 @@ struct lttng_event *_lttng_event_create(struct lttng_channel *chan, * registration. */ smp_wmb(); - ret = lttng_kprobes_register(event_name, + ret = lttng_kprobes_register_event(event_name, event_param->u.kprobe.symbol_name, event_param->u.kprobe.offset, event_param->u.kprobe.addr, @@ -722,6 +923,28 @@ struct lttng_event *_lttng_event_create(struct lttng_channel *chan, goto register_error; } break; + case LTTNG_KERNEL_UPROBE: + /* + * Needs to be explicitly enabled after creation, since + * we may want to apply filters. + */ + event->enabled = 0; + event->registered = 1; + + /* + * Populate lttng_event structure before event + * registration. + */ + smp_wmb(); + + ret = lttng_uprobes_register_event(event_param->name, + event_param->u.uprobe.fd, + event); + if (ret) + goto register_error; + ret = try_module_get(event->desc->owner); + WARN_ON_ONCE(!ret); + break; default: WARN_ON_ONCE(1); ret = -EINVAL; @@ -747,78 +970,267 @@ full: return ERR_PTR(ret); } -struct lttng_event *lttng_event_create(struct lttng_channel *chan, - struct lttng_kernel_event *event_param, - void *filter, - const struct lttng_event_desc *event_desc, - enum lttng_kernel_instrumentation itype) +static +void lttng_trigger_send_notification(struct lttng_trigger *trigger) { - struct lttng_event *event; + struct lttng_trigger_group *trigger_group = trigger->group; + struct lib_ring_buffer_ctx ctx; + int ret; - mutex_lock(&sessions_mutex); - event = _lttng_event_create(chan, event_param, filter, event_desc, - itype); - mutex_unlock(&sessions_mutex); - return event; + if (unlikely(!READ_ONCE(trigger->enabled))) + return; + + lib_ring_buffer_ctx_init(&ctx, trigger_group->chan, NULL, sizeof(trigger->id), + lttng_alignof(trigger->id), -1); + ret = trigger_group->ops->event_reserve(&ctx, 0); + if (ret < 0) { + //TODO: error handling with counter maps + //silently drop for now. WARN_ON_ONCE(1); + return; + } + lib_ring_buffer_align_ctx(&ctx, lttng_alignof(trigger->id)); + trigger_group->ops->event_write(&ctx, &trigger->id, sizeof(trigger->id)); + trigger_group->ops->event_commit(&ctx); + irq_work_queue(&trigger_group->wakeup_pending); } -/* Only used for tracepoints for now. */ -static -void register_event(struct lttng_event *event) +struct lttng_trigger *_lttng_trigger_create( + const struct lttng_event_desc *event_desc, + uint64_t id, struct lttng_trigger_group *trigger_group, + struct lttng_kernel_trigger *trigger_param, void *filter, + enum lttng_kernel_instrumentation itype) { - const struct lttng_event_desc *desc; - int ret = -EINVAL; - - if (event->registered) - return; + struct lttng_trigger *trigger; + const char *event_name; + struct hlist_head *head; + int ret; - desc = event->desc; - switch (event->instrumentation) { + switch (itype) { case LTTNG_KERNEL_TRACEPOINT: - ret = lttng_wrapper_tracepoint_probe_register(desc->kname, - desc->probe_callback, - event); + event_name = event_desc->name; break; + case LTTNG_KERNEL_KPROBE: + case LTTNG_KERNEL_UPROBE: case LTTNG_KERNEL_SYSCALL: - ret = lttng_syscall_filter_enable(event->chan, - desc->name); + event_name = trigger_param->name; break; - case LTTNG_KERNEL_KPROBE: case LTTNG_KERNEL_KRETPROBE: case LTTNG_KERNEL_FUNCTION: case LTTNG_KERNEL_NOOP: - ret = 0; - break; default: WARN_ON_ONCE(1); + ret = -EINVAL; + goto type_error; } - if (!ret) - event->registered = 1; -} -/* - * Only used internally at session destruction. - */ -int _lttng_event_unregister(struct lttng_event *event) -{ - const struct lttng_event_desc *desc; - int ret = -EINVAL; + head = utils_borrow_hash_table_bucket(trigger_group->triggers_ht.table, + LTTNG_TRIGGER_HT_SIZE, event_name); + lttng_hlist_for_each_entry(trigger, head, hlist) { + WARN_ON_ONCE(!trigger->desc); + if (!strncmp(trigger->desc->name, event_name, + LTTNG_KERNEL_SYM_NAME_LEN - 1) + && trigger_group == trigger->group + && id == trigger->id) { + ret = -EEXIST; + goto exist; + } + } - if (!event->registered) - return 0; + trigger = kmem_cache_zalloc(trigger_cache, GFP_KERNEL); + if (!trigger) { + ret = -ENOMEM; + goto cache_error; + } + trigger->group = trigger_group; + trigger->id = id; + trigger->filter = filter; + trigger->instrumentation = itype; + trigger->evtype = LTTNG_TYPE_EVENT; + trigger->send_notification = lttng_trigger_send_notification; + INIT_LIST_HEAD(&trigger->bytecode_runtime_head); + INIT_LIST_HEAD(&trigger->enablers_ref_head); - desc = event->desc; - switch (event->instrumentation) { + switch (itype) { case LTTNG_KERNEL_TRACEPOINT: - ret = lttng_wrapper_tracepoint_probe_unregister(event->desc->kname, - event->desc->probe_callback, - event); + /* Event will be enabled by enabler sync. */ + trigger->enabled = 0; + trigger->registered = 0; + trigger->desc = lttng_event_desc_get(event_name); + if (!trigger->desc) { + ret = -ENOENT; + goto register_error; + } + /* Populate lttng_trigger structure before event registration. */ + smp_wmb(); break; case LTTNG_KERNEL_KPROBE: - lttng_kprobes_unregister(event); - ret = 0; + /* + * Needs to be explicitly enabled after creation, since + * we may want to apply filters. + */ + trigger->enabled = 0; + trigger->registered = 1; + /* + * Populate lttng_trigger structure before event + * registration. + */ + smp_wmb(); + ret = lttng_kprobes_register_trigger( + trigger_param->u.kprobe.symbol_name, + trigger_param->u.kprobe.offset, + trigger_param->u.kprobe.addr, + trigger); + if (ret) { + ret = -EINVAL; + goto register_error; + } + ret = try_module_get(trigger->desc->owner); + WARN_ON_ONCE(!ret); break; - case LTTNG_KERNEL_KRETPROBE: + case LTTNG_KERNEL_NOOP: + case LTTNG_KERNEL_SYSCALL: + /* + * Needs to be explicitly enabled after creation, since + * we may want to apply filters. + */ + trigger->enabled = 0; + trigger->registered = 0; + trigger->desc = event_desc; + if (!trigger->desc) { + ret = -EINVAL; + goto register_error; + } + break; + case LTTNG_KERNEL_UPROBE: + /* + * Needs to be explicitly enabled after creation, since + * we may want to apply filters. + */ + trigger->enabled = 0; + trigger->registered = 1; + + /* + * Populate lttng_trigger structure before trigger + * registration. + */ + smp_wmb(); + + ret = lttng_uprobes_register_trigger(trigger_param->name, + trigger_param->u.uprobe.fd, + trigger); + if (ret) + goto register_error; + ret = try_module_get(trigger->desc->owner); + WARN_ON_ONCE(!ret); + break; + case LTTNG_KERNEL_KRETPROBE: + case LTTNG_KERNEL_FUNCTION: + default: + WARN_ON_ONCE(1); + ret = -EINVAL; + goto register_error; + } + + list_add(&trigger->list, &trigger_group->triggers_head); + hlist_add_head(&trigger->hlist, head); + return trigger; + +register_error: + kmem_cache_free(trigger_cache, trigger); +cache_error: +exist: +type_error: + return ERR_PTR(ret); +} + +struct lttng_event *lttng_event_create(struct lttng_channel *chan, + struct lttng_kernel_event *event_param, + void *filter, + const struct lttng_event_desc *event_desc, + enum lttng_kernel_instrumentation itype) +{ + struct lttng_event *event; + + mutex_lock(&sessions_mutex); + event = _lttng_event_create(chan, event_param, filter, event_desc, + itype); + mutex_unlock(&sessions_mutex); + return event; +} + +struct lttng_trigger *lttng_trigger_create( + const struct lttng_event_desc *event_desc, + uint64_t id, struct lttng_trigger_group *trigger_group, + struct lttng_kernel_trigger *trigger_param, void *filter, + enum lttng_kernel_instrumentation itype) +{ + struct lttng_trigger *trigger; + + mutex_lock(&sessions_mutex); + trigger = _lttng_trigger_create(event_desc, id, trigger_group, + trigger_param, filter, itype); + mutex_unlock(&sessions_mutex); + return trigger; +} + +/* Only used for tracepoints for now. */ +static +void register_event(struct lttng_event *event) +{ + const struct lttng_event_desc *desc; + int ret = -EINVAL; + + if (event->registered) + return; + + desc = event->desc; + switch (event->instrumentation) { + case LTTNG_KERNEL_TRACEPOINT: + ret = lttng_wrapper_tracepoint_probe_register(desc->kname, + desc->probe_callback, + event); + break; + case LTTNG_KERNEL_SYSCALL: + ret = lttng_syscall_filter_enable_event(event->chan, + desc->name); + break; + case LTTNG_KERNEL_KPROBE: + case LTTNG_KERNEL_UPROBE: + case LTTNG_KERNEL_KRETPROBE: + case LTTNG_KERNEL_FUNCTION: + case LTTNG_KERNEL_NOOP: + ret = 0; + break; + default: + WARN_ON_ONCE(1); + } + if (!ret) + event->registered = 1; +} + +/* + * Only used internally at session destruction. + */ +int _lttng_event_unregister(struct lttng_event *event) +{ + const struct lttng_event_desc *desc; + int ret = -EINVAL; + + if (!event->registered) + return 0; + + desc = event->desc; + switch (event->instrumentation) { + case LTTNG_KERNEL_TRACEPOINT: + ret = lttng_wrapper_tracepoint_probe_unregister(event->desc->kname, + event->desc->probe_callback, + event); + break; + case LTTNG_KERNEL_KPROBE: + lttng_kprobes_unregister_event(event); + ret = 0; + break; + case LTTNG_KERNEL_KRETPROBE: lttng_kretprobes_unregister(event); ret = 0; break; @@ -827,12 +1239,16 @@ int _lttng_event_unregister(struct lttng_event *event) ret = 0; break; case LTTNG_KERNEL_SYSCALL: - ret = lttng_syscall_filter_disable(event->chan, + ret = lttng_syscall_filter_disable_event(event->chan, desc->name); break; case LTTNG_KERNEL_NOOP: ret = 0; break; + case LTTNG_KERNEL_UPROBE: + lttng_uprobes_unregister_event(event); + ret = 0; + break; default: WARN_ON_ONCE(1); } @@ -841,6 +1257,78 @@ int _lttng_event_unregister(struct lttng_event *event) return ret; } +/* Only used for tracepoints for now. */ +static +void register_trigger(struct lttng_trigger *trigger) +{ + const struct lttng_event_desc *desc; + int ret = -EINVAL; + + if (trigger->registered) + return; + + desc = trigger->desc; + switch (trigger->instrumentation) { + case LTTNG_KERNEL_TRACEPOINT: + ret = lttng_wrapper_tracepoint_probe_register(desc->kname, + desc->trigger_callback, + trigger); + break; + case LTTNG_KERNEL_SYSCALL: + ret = lttng_syscall_filter_enable_trigger(trigger); + break; + case LTTNG_KERNEL_KPROBE: + case LTTNG_KERNEL_UPROBE: + ret = 0; + break; + case LTTNG_KERNEL_KRETPROBE: + case LTTNG_KERNEL_FUNCTION: + case LTTNG_KERNEL_NOOP: + default: + WARN_ON_ONCE(1); + } + if (!ret) + trigger->registered = 1; +} + +static +int _lttng_trigger_unregister(struct lttng_trigger *trigger) +{ + const struct lttng_event_desc *desc; + int ret = -EINVAL; + + if (!trigger->registered) + return 0; + + desc = trigger->desc; + switch (trigger->instrumentation) { + case LTTNG_KERNEL_TRACEPOINT: + ret = lttng_wrapper_tracepoint_probe_unregister(trigger->desc->kname, + trigger->desc->trigger_callback, + trigger); + break; + case LTTNG_KERNEL_KPROBE: + lttng_kprobes_unregister_trigger(trigger); + ret = 0; + break; + case LTTNG_KERNEL_UPROBE: + lttng_uprobes_unregister_trigger(trigger); + ret = 0; + break; + case LTTNG_KERNEL_SYSCALL: + ret = lttng_syscall_filter_disable_trigger(trigger); + break; + case LTTNG_KERNEL_KRETPROBE: + case LTTNG_KERNEL_FUNCTION: + case LTTNG_KERNEL_NOOP: + default: + WARN_ON_ONCE(1); + } + if (!ret) + trigger->registered = 0; + return ret; +} + /* * Only used internally at session destruction. */ @@ -849,11 +1337,11 @@ void _lttng_event_destroy(struct lttng_event *event) { switch (event->instrumentation) { case LTTNG_KERNEL_TRACEPOINT: - lttng_event_put(event->desc); + lttng_event_desc_put(event->desc); break; case LTTNG_KERNEL_KPROBE: module_put(event->desc->owner); - lttng_kprobes_destroy_private(event); + lttng_kprobes_destroy_event_private(event); break; case LTTNG_KERNEL_KRETPROBE: module_put(event->desc->owner); @@ -866,6 +1354,10 @@ void _lttng_event_destroy(struct lttng_event *event) case LTTNG_KERNEL_NOOP: case LTTNG_KERNEL_SYSCALL: break; + case LTTNG_KERNEL_UPROBE: + module_put(event->desc->owner); + lttng_uprobes_destroy_event_private(event); + break; default: WARN_ON_ONCE(1); } @@ -874,91 +1366,115 @@ void _lttng_event_destroy(struct lttng_event *event) kmem_cache_free(event_cache, event); } -int lttng_session_track_pid(struct lttng_session *session, int pid) +/* + * Only used internally at session destruction. + */ +static +void _lttng_trigger_destroy(struct lttng_trigger *trigger) +{ + switch (trigger->instrumentation) { + case LTTNG_KERNEL_TRACEPOINT: + lttng_event_desc_put(trigger->desc); + break; + case LTTNG_KERNEL_KPROBE: + module_put(trigger->desc->owner); + lttng_kprobes_destroy_trigger_private(trigger); + break; + case LTTNG_KERNEL_NOOP: + case LTTNG_KERNEL_SYSCALL: + break; + case LTTNG_KERNEL_UPROBE: + module_put(trigger->desc->owner); + lttng_uprobes_destroy_trigger_private(trigger); + break; + case LTTNG_KERNEL_KRETPROBE: + case LTTNG_KERNEL_FUNCTION: + default: + WARN_ON_ONCE(1); + } + list_del(&trigger->list); + kmem_cache_free(trigger_cache, trigger); +} + +struct lttng_id_tracker *get_tracker(struct lttng_session *session, + enum tracker_type tracker_type) +{ + switch (tracker_type) { + case TRACKER_PID: + return &session->pid_tracker; + case TRACKER_VPID: + return &session->vpid_tracker; + case TRACKER_UID: + return &session->uid_tracker; + case TRACKER_VUID: + return &session->vuid_tracker; + case TRACKER_GID: + return &session->gid_tracker; + case TRACKER_VGID: + return &session->vgid_tracker; + default: + WARN_ON_ONCE(1); + return NULL; + } +} + +int lttng_session_track_id(struct lttng_session *session, + enum tracker_type tracker_type, int id) { + struct lttng_id_tracker *tracker; int ret; - if (pid < -1) + tracker = get_tracker(session, tracker_type); + if (!tracker) + return -EINVAL; + if (id < -1) return -EINVAL; mutex_lock(&sessions_mutex); - if (pid == -1) { - /* track all pids: destroy tracker. */ - if (session->pid_tracker) { - struct lttng_pid_tracker *lpf; - - lpf = session->pid_tracker; - rcu_assign_pointer(session->pid_tracker, NULL); - synchronize_trace(); - lttng_pid_tracker_destroy(lpf); - } + if (id == -1) { + /* track all ids: destroy tracker. */ + lttng_id_tracker_destroy(tracker, true); ret = 0; } else { - if (!session->pid_tracker) { - struct lttng_pid_tracker *lpf; - - lpf = lttng_pid_tracker_create(); - if (!lpf) { - ret = -ENOMEM; - goto unlock; - } - ret = lttng_pid_tracker_add(lpf, pid); - rcu_assign_pointer(session->pid_tracker, lpf); - } else { - ret = lttng_pid_tracker_add(session->pid_tracker, pid); - } + ret = lttng_id_tracker_add(tracker, id); } -unlock: mutex_unlock(&sessions_mutex); return ret; } -int lttng_session_untrack_pid(struct lttng_session *session, int pid) +int lttng_session_untrack_id(struct lttng_session *session, + enum tracker_type tracker_type, int id) { + struct lttng_id_tracker *tracker; int ret; - if (pid < -1) + tracker = get_tracker(session, tracker_type); + if (!tracker) + return -EINVAL; + if (id < -1) return -EINVAL; mutex_lock(&sessions_mutex); - if (pid == -1) { - /* untrack all pids: replace by empty tracker. */ - struct lttng_pid_tracker *old_lpf = session->pid_tracker; - struct lttng_pid_tracker *lpf; - - lpf = lttng_pid_tracker_create(); - if (!lpf) { - ret = -ENOMEM; - goto unlock; - } - rcu_assign_pointer(session->pid_tracker, lpf); - synchronize_trace(); - if (old_lpf) - lttng_pid_tracker_destroy(old_lpf); - ret = 0; + if (id == -1) { + /* untrack all ids: replace by empty tracker. */ + ret = lttng_id_tracker_empty_set(tracker); } else { - if (!session->pid_tracker) { - ret = -ENOENT; - goto unlock; - } - ret = lttng_pid_tracker_del(session->pid_tracker, pid); + ret = lttng_id_tracker_del(tracker, id); } -unlock: mutex_unlock(&sessions_mutex); return ret; } static -void *pid_list_start(struct seq_file *m, loff_t *pos) +void *id_list_start(struct seq_file *m, loff_t *pos) { - struct lttng_session *session = m->private; - struct lttng_pid_tracker *lpf; - struct lttng_pid_hash_node *e; + struct lttng_id_tracker *id_tracker = m->private; + struct lttng_id_tracker_rcu *id_tracker_p = id_tracker->p; + struct lttng_id_hash_node *e; int iter = 0, i; mutex_lock(&sessions_mutex); - lpf = session->pid_tracker; - if (lpf) { - for (i = 0; i < LTTNG_PID_TABLE_SIZE; i++) { - struct hlist_head *head = &lpf->pid_hash[i]; + if (id_tracker_p) { + for (i = 0; i < LTTNG_ID_TABLE_SIZE; i++) { + struct hlist_head *head = &id_tracker_p->id_hash[i]; lttng_hlist_for_each_entry(e, head, hlist) { if (iter++ >= *pos) @@ -966,9 +1482,9 @@ void *pid_list_start(struct seq_file *m, loff_t *pos) } } } else { - /* PID tracker disabled. */ + /* ID tracker disabled. */ if (iter >= *pos && iter == 0) { - return session; /* empty tracker */ + return id_tracker_p; /* empty tracker */ } iter++; } @@ -978,18 +1494,17 @@ void *pid_list_start(struct seq_file *m, loff_t *pos) /* Called with sessions_mutex held. */ static -void *pid_list_next(struct seq_file *m, void *p, loff_t *ppos) +void *id_list_next(struct seq_file *m, void *p, loff_t *ppos) { - struct lttng_session *session = m->private; - struct lttng_pid_tracker *lpf; - struct lttng_pid_hash_node *e; + struct lttng_id_tracker *id_tracker = m->private; + struct lttng_id_tracker_rcu *id_tracker_p = id_tracker->p; + struct lttng_id_hash_node *e; int iter = 0, i; (*ppos)++; - lpf = session->pid_tracker; - if (lpf) { - for (i = 0; i < LTTNG_PID_TABLE_SIZE; i++) { - struct hlist_head *head = &lpf->pid_hash[i]; + if (id_tracker_p) { + for (i = 0; i < LTTNG_ID_TABLE_SIZE; i++) { + struct hlist_head *head = &id_tracker_p->id_hash[i]; lttng_hlist_for_each_entry(e, head, hlist) { if (iter++ >= *ppos) @@ -997,9 +1512,9 @@ void *pid_list_next(struct seq_file *m, void *p, loff_t *ppos) } } } else { - /* PID tracker disabled. */ + /* ID tracker disabled. */ if (iter >= *ppos && iter == 0) - return session; /* empty tracker */ + return p; /* empty tracker */ iter++; } @@ -1008,67 +1523,91 @@ void *pid_list_next(struct seq_file *m, void *p, loff_t *ppos) } static -void pid_list_stop(struct seq_file *m, void *p) +void id_list_stop(struct seq_file *m, void *p) { mutex_unlock(&sessions_mutex); } static -int pid_list_show(struct seq_file *m, void *p) +int id_list_show(struct seq_file *m, void *p) { - int pid; + struct lttng_id_tracker *id_tracker = m->private; + struct lttng_id_tracker_rcu *id_tracker_p = id_tracker->p; + int id; - if (p == m->private) { + if (p == id_tracker_p) { /* Tracker disabled. */ - pid = -1; + id = -1; } else { - const struct lttng_pid_hash_node *e = p; + const struct lttng_id_hash_node *e = p; - pid = lttng_pid_tracker_get_node_pid(e); + id = lttng_id_tracker_get_node_id(e); + } + switch (id_tracker->tracker_type) { + case TRACKER_PID: + seq_printf(m, "process { pid = %d; };\n", id); + break; + case TRACKER_VPID: + seq_printf(m, "process { vpid = %d; };\n", id); + break; + case TRACKER_UID: + seq_printf(m, "user { uid = %d; };\n", id); + break; + case TRACKER_VUID: + seq_printf(m, "user { vuid = %d; };\n", id); + break; + case TRACKER_GID: + seq_printf(m, "group { gid = %d; };\n", id); + break; + case TRACKER_VGID: + seq_printf(m, "group { vgid = %d; };\n", id); + break; + default: + seq_printf(m, "UNKNOWN { field = %d };\n", id); } - seq_printf(m, "process { pid = %d; };\n", pid); return 0; } static -const struct seq_operations lttng_tracker_pids_list_seq_ops = { - .start = pid_list_start, - .next = pid_list_next, - .stop = pid_list_stop, - .show = pid_list_show, +const struct seq_operations lttng_tracker_ids_list_seq_ops = { + .start = id_list_start, + .next = id_list_next, + .stop = id_list_stop, + .show = id_list_show, }; static -int lttng_tracker_pids_list_open(struct inode *inode, struct file *file) +int lttng_tracker_ids_list_open(struct inode *inode, struct file *file) { - return seq_open(file, <tng_tracker_pids_list_seq_ops); + return seq_open(file, <tng_tracker_ids_list_seq_ops); } static -int lttng_tracker_pids_list_release(struct inode *inode, struct file *file) +int lttng_tracker_ids_list_release(struct inode *inode, struct file *file) { struct seq_file *m = file->private_data; - struct lttng_session *session = m->private; + struct lttng_id_tracker *id_tracker = m->private; int ret; - WARN_ON_ONCE(!session); + WARN_ON_ONCE(!id_tracker); ret = seq_release(inode, file); - if (!ret && session) - fput(session->file); + if (!ret) + fput(id_tracker->session->file); return ret; } -const struct file_operations lttng_tracker_pids_list_fops = { +const struct file_operations lttng_tracker_ids_list_fops = { .owner = THIS_MODULE, - .open = lttng_tracker_pids_list_open, + .open = lttng_tracker_ids_list_open, .read = seq_read, .llseek = seq_lseek, - .release = lttng_tracker_pids_list_release, + .release = lttng_tracker_ids_list_release, }; -int lttng_session_list_tracker_pids(struct lttng_session *session) +int lttng_session_list_tracker_ids(struct lttng_session *session, + enum tracker_type tracker_type) { - struct file *tracker_pids_list_file; + struct file *tracker_ids_list_file; struct seq_file *m; int file_fd, ret; @@ -1078,25 +1617,32 @@ int lttng_session_list_tracker_pids(struct lttng_session *session) goto fd_error; } - tracker_pids_list_file = anon_inode_getfile("[lttng_tracker_pids_list]", - <tng_tracker_pids_list_fops, + tracker_ids_list_file = anon_inode_getfile("[lttng_tracker_ids_list]", + <tng_tracker_ids_list_fops, NULL, O_RDWR); - if (IS_ERR(tracker_pids_list_file)) { - ret = PTR_ERR(tracker_pids_list_file); + if (IS_ERR(tracker_ids_list_file)) { + ret = PTR_ERR(tracker_ids_list_file); goto file_error; } - ret = lttng_tracker_pids_list_fops.open(NULL, tracker_pids_list_file); + if (!atomic_long_add_unless(&session->file->f_count, 1, LONG_MAX)) { + ret = -EOVERFLOW; + goto refcount_error; + } + ret = lttng_tracker_ids_list_fops.open(NULL, tracker_ids_list_file); if (ret < 0) goto open_error; - m = tracker_pids_list_file->private_data; - m->private = session; - fd_install(file_fd, tracker_pids_list_file); - atomic_long_inc(&session->file->f_count); + m = tracker_ids_list_file->private_data; + + m->private = get_tracker(session, tracker_type); + BUG_ON(!m->private); + fd_install(file_fd, tracker_ids_list_file); return file_fd; open_error: - fput(tracker_pids_list_file); + atomic_long_dec(&session->file->f_count); +refcount_error: + fput(tracker_ids_list_file); file_error: put_unused_fd(file_fd); fd_error: @@ -1107,11 +1653,11 @@ fd_error: * Enabler management. */ static -int lttng_match_enabler_wildcard(const char *desc_name, - const char *name) +int lttng_match_enabler_star_glob(const char *desc_name, + const char *pattern) { - /* Compare excluding final '*' */ - if (strncmp(desc_name, name, strlen(name) - 1)) + if (!strutils_star_glob_match(pattern, LTTNG_SIZE_MAX, + desc_name, LTTNG_SIZE_MAX)) return 0; return 1; } @@ -1125,7 +1671,6 @@ int lttng_match_enabler_name(const char *desc_name, return 1; } -static int lttng_desc_match_enabler(const struct lttng_event_desc *desc, struct lttng_enabler *enabler) { @@ -1155,10 +1700,10 @@ int lttng_desc_match_enabler(const struct lttng_event_desc *desc, WARN_ON_ONCE(1); return -EINVAL; } - switch (enabler->type) { - case LTTNG_ENABLER_WILDCARD: - return lttng_match_enabler_wildcard(desc_name, enabler_name); - case LTTNG_ENABLER_NAME: + switch (enabler->format_type) { + case LTTNG_ENABLER_FORMAT_STAR_GLOB: + return lttng_match_enabler_star_glob(desc_name, enabler_name); + case LTTNG_ENABLER_FORMAT_NAME: return lttng_match_enabler_name(desc_name, enabler_name); default: return -EINVAL; @@ -1166,26 +1711,46 @@ int lttng_desc_match_enabler(const struct lttng_event_desc *desc, } static -int lttng_event_match_enabler(struct lttng_event *event, - struct lttng_enabler *enabler) +int lttng_event_enabler_match_event(struct lttng_event_enabler *event_enabler, + struct lttng_event *event) +{ + struct lttng_enabler *base_enabler = lttng_event_enabler_as_enabler( + event_enabler); + + if (base_enabler->event_param.instrumentation != event->instrumentation) + return 0; + if (lttng_desc_match_enabler(event->desc, base_enabler) + && event->chan == event_enabler->chan) + return 1; + else + return 0; +} + +static +int lttng_trigger_enabler_match_trigger(struct lttng_trigger_enabler *trigger_enabler, + struct lttng_trigger *trigger) { - if (enabler->event_param.instrumentation != event->instrumentation) + struct lttng_enabler *base_enabler = lttng_trigger_enabler_as_enabler( + trigger_enabler); + + if (base_enabler->event_param.instrumentation != trigger->instrumentation) return 0; - if (lttng_desc_match_enabler(event->desc, enabler) - && event->chan == enabler->chan) + if (lttng_desc_match_enabler(trigger->desc, base_enabler) + && trigger->group == trigger_enabler->group + && trigger->id == trigger_enabler->id) return 1; else return 0; } static -struct lttng_enabler_ref *lttng_event_enabler_ref(struct lttng_event *event, +struct lttng_enabler_ref *lttng_enabler_ref( + struct list_head *enablers_ref_list, struct lttng_enabler *enabler) { struct lttng_enabler_ref *enabler_ref; - list_for_each_entry(enabler_ref, - &event->enablers_ref_head, node) { + list_for_each_entry(enabler_ref, enablers_ref_list, node) { if (enabler_ref->ref == enabler) return enabler_ref; } @@ -1193,9 +1758,9 @@ struct lttng_enabler_ref *lttng_event_enabler_ref(struct lttng_event *event, } static -void lttng_create_tracepoint_if_missing(struct lttng_enabler *enabler) +void lttng_create_tracepoint_event_if_missing(struct lttng_event_enabler *event_enabler) { - struct lttng_session *session = enabler->chan->session; + struct lttng_session *session = event_enabler->chan->session; struct lttng_probe_desc *probe_desc; const struct lttng_event_desc *desc; int i; @@ -1211,25 +1776,22 @@ void lttng_create_tracepoint_if_missing(struct lttng_enabler *enabler) for (i = 0; i < probe_desc->nr_events; i++) { int found = 0; struct hlist_head *head; - const char *event_name; - size_t name_len; - uint32_t hash; struct lttng_event *event; desc = probe_desc->event_desc[i]; - if (!lttng_desc_match_enabler(desc, enabler)) + if (!lttng_desc_match_enabler(desc, + lttng_event_enabler_as_enabler(event_enabler))) continue; - event_name = desc->name; - name_len = strlen(event_name); /* * Check if already created. */ - hash = jhash(event_name, name_len, 0); - head = &session->events_ht.table[hash & (LTTNG_EVENT_HT_SIZE - 1)]; + head = utils_borrow_hash_table_bucket( + session->events_ht.table, LTTNG_EVENT_HT_SIZE, + desc->name); lttng_hlist_for_each_entry(event, head, hlist) { if (event->desc == desc - && event->chan == enabler->chan) + && event->chan == event_enabler->chan) found = 1; } if (found) @@ -1239,7 +1801,7 @@ void lttng_create_tracepoint_if_missing(struct lttng_enabler *enabler) * We need to create an event for this * event probe. */ - event = _lttng_event_create(enabler->chan, + event = _lttng_event_create(event_enabler->chan, NULL, NULL, desc, LTTNG_KERNEL_TRACEPOINT); if (!event) { @@ -1251,28 +1813,93 @@ void lttng_create_tracepoint_if_missing(struct lttng_enabler *enabler) } static -void lttng_create_syscall_if_missing(struct lttng_enabler *enabler) +void lttng_create_tracepoint_trigger_if_missing(struct lttng_trigger_enabler *trigger_enabler) { - int ret; + struct lttng_trigger_group *trigger_group = trigger_enabler->group; + struct lttng_probe_desc *probe_desc; + const struct lttng_event_desc *desc; + int i; + struct list_head *probe_list; - ret = lttng_syscalls_register(enabler->chan, NULL); - WARN_ON_ONCE(ret); -} + probe_list = lttng_get_probe_list_head(); + /* + * For each probe event, if we find that a probe event matches + * our enabler, create an associated lttng_trigger if not + * already present. + */ + list_for_each_entry(probe_desc, probe_list, head) { + for (i = 0; i < probe_desc->nr_events; i++) { + int found = 0; + struct hlist_head *head; + struct lttng_trigger *trigger; -/* - * Create struct lttng_event if it is missing and present in the list of - * tracepoint probes. - * Should be called with sessions mutex held. - */ -static -void lttng_create_event_if_missing(struct lttng_enabler *enabler) -{ - switch (enabler->event_param.instrumentation) { + desc = probe_desc->event_desc[i]; + if (!lttng_desc_match_enabler(desc, + lttng_trigger_enabler_as_enabler(trigger_enabler))) + continue; + + /* + * Check if already created. + */ + head = utils_borrow_hash_table_bucket( + trigger_group->triggers_ht.table, + LTTNG_TRIGGER_HT_SIZE, desc->name); + lttng_hlist_for_each_entry(trigger, head, hlist) { + if (trigger->desc == desc + && trigger->id == trigger_enabler->id) + found = 1; + } + if (found) + continue; + + /* + * We need to create a trigger for this event probe. + */ + trigger = _lttng_trigger_create(desc, + trigger_enabler->id, trigger_group, NULL, NULL, + LTTNG_KERNEL_TRACEPOINT); + if (IS_ERR(trigger)) { + printk(KERN_INFO "Unable to create trigger %s\n", + probe_desc->event_desc[i]->name); + } + } + } +} + +static +void lttng_create_syscall_event_if_missing(struct lttng_event_enabler *event_enabler) +{ + int ret; + + ret = lttng_syscalls_register_event(event_enabler->chan, NULL); + WARN_ON_ONCE(ret); +} + +static +void lttng_create_syscall_trigger_if_missing(struct lttng_trigger_enabler *trigger_enabler) +{ + int ret; + + ret = lttng_syscalls_register_trigger(trigger_enabler, NULL); + WARN_ON_ONCE(ret); + ret = lttng_syscals_create_matching_triggers(trigger_enabler, NULL); + WARN_ON_ONCE(ret); +} + +/* + * Create struct lttng_event if it is missing and present in the list of + * tracepoint probes. + * Should be called with sessions mutex held. + */ +static +void lttng_create_event_if_missing(struct lttng_event_enabler *event_enabler) +{ + switch (event_enabler->base.event_param.instrumentation) { case LTTNG_KERNEL_TRACEPOINT: - lttng_create_tracepoint_if_missing(enabler); + lttng_create_tracepoint_event_if_missing(event_enabler); break; case LTTNG_KERNEL_SYSCALL: - lttng_create_syscall_if_missing(enabler); + lttng_create_syscall_event_if_missing(event_enabler); break; default: WARN_ON_ONCE(1); @@ -1281,35 +1908,36 @@ void lttng_create_event_if_missing(struct lttng_enabler *enabler) } /* - * Create events associated with an enabler (if not already present), + * Create events associated with an event_enabler (if not already present), * and add backward reference from the event to the enabler. * Should be called with sessions mutex held. */ static -int lttng_enabler_ref_events(struct lttng_enabler *enabler) +int lttng_event_enabler_ref_events(struct lttng_event_enabler *event_enabler) { - struct lttng_session *session = enabler->chan->session; + struct lttng_session *session = event_enabler->chan->session; struct lttng_event *event; /* First ensure that probe events are created for this enabler. */ - lttng_create_event_if_missing(enabler); + lttng_create_event_if_missing(event_enabler); - /* For each event matching enabler in session event list. */ + /* For each event matching event_enabler in session event list. */ list_for_each_entry(event, &session->events, list) { struct lttng_enabler_ref *enabler_ref; - if (!lttng_event_match_enabler(event, enabler)) + if (!lttng_event_enabler_match_event(event_enabler, event)) continue; - enabler_ref = lttng_event_enabler_ref(event, enabler); + enabler_ref = lttng_enabler_ref(&event->enablers_ref_head, + lttng_event_enabler_as_enabler(event_enabler)); if (!enabler_ref) { /* * If no backward ref, create it. - * Add backward ref from event to enabler. + * Add backward ref from event to event_enabler. */ enabler_ref = kzalloc(sizeof(*enabler_ref), GFP_KERNEL); if (!enabler_ref) return -ENOMEM; - enabler_ref->ref = enabler; + enabler_ref->ref = lttng_event_enabler_as_enabler(event_enabler); list_add(&enabler_ref->node, &event->enablers_ref_head); } @@ -1317,13 +1945,83 @@ int lttng_enabler_ref_events(struct lttng_enabler *enabler) /* * Link filter bytecodes if not linked yet. */ - lttng_enabler_event_link_bytecode(event, enabler); + lttng_enabler_link_bytecode(event->desc, + lttng_static_ctx, + &event->bytecode_runtime_head, + lttng_event_enabler_as_enabler(event_enabler)); /* TODO: merge event context. */ } return 0; } +/* + * Create struct lttng_trigger if it is missing and present in the list of + * tracepoint probes. + * Should be called with sessions mutex held. + */ +static +void lttng_create_trigger_if_missing(struct lttng_trigger_enabler *trigger_enabler) +{ + switch (trigger_enabler->base.event_param.instrumentation) { + case LTTNG_KERNEL_TRACEPOINT: + lttng_create_tracepoint_trigger_if_missing(trigger_enabler); + break; + case LTTNG_KERNEL_SYSCALL: + lttng_create_syscall_trigger_if_missing(trigger_enabler); + break; + default: + WARN_ON_ONCE(1); + break; + } +} + +/* + * Create triggers associated with a trigger enabler (if not already present). + */ +static +int lttng_trigger_enabler_ref_triggers(struct lttng_trigger_enabler *trigger_enabler) +{ + struct lttng_trigger_group *trigger_group = trigger_enabler->group; + struct lttng_trigger *trigger; + + /* First ensure that probe triggers are created for this enabler. */ + lttng_create_trigger_if_missing(trigger_enabler); + + /* Link the created trigger with its associated enabler. */ + list_for_each_entry(trigger, &trigger_group->triggers_head, list) { + struct lttng_enabler_ref *enabler_ref; + + if (!lttng_trigger_enabler_match_trigger(trigger_enabler, trigger)) + continue; + + enabler_ref = lttng_enabler_ref(&trigger->enablers_ref_head, + lttng_trigger_enabler_as_enabler(trigger_enabler)); + if (!enabler_ref) { + /* + * If no backward ref, create it. + * Add backward ref from trigger to enabler. + */ + enabler_ref = kzalloc(sizeof(*enabler_ref), GFP_KERNEL); + if (!enabler_ref) + return -ENOMEM; + + enabler_ref->ref = lttng_trigger_enabler_as_enabler( + trigger_enabler); + list_add(&enabler_ref->node, + &trigger->enablers_ref_head); + } + + /* + * Link filter bytecodes if not linked yet. + */ + lttng_enabler_link_bytecode(trigger->desc, + trigger_group->ctx, &trigger->bytecode_runtime_head, + lttng_trigger_enabler_as_enabler(trigger_enabler)); + } + return 0; +} + /* * Called at module load: connect the probe on all enablers matching * this event. @@ -1334,52 +2032,87 @@ int lttng_fix_pending_events(void) struct lttng_session *session; list_for_each_entry(session, &sessions, list) - lttng_session_lazy_sync_enablers(session); + lttng_session_lazy_sync_event_enablers(session); return 0; } -struct lttng_enabler *lttng_enabler_create(enum lttng_enabler_type type, +static bool lttng_trigger_group_has_active_triggers( + struct lttng_trigger_group *trigger_group) +{ + struct lttng_trigger_enabler *trigger_enabler; + + list_for_each_entry(trigger_enabler, &trigger_group->enablers_head, + node) { + if (trigger_enabler->base.enabled) + return true; + } + return false; +} + +bool lttng_trigger_active(void) +{ + struct lttng_trigger_group *trigger_group; + + list_for_each_entry(trigger_group, &trigger_groups, node) { + if (lttng_trigger_group_has_active_triggers(trigger_group)) + return true; + } + return false; +} + +int lttng_fix_pending_triggers(void) +{ + struct lttng_trigger_group *trigger_group; + + list_for_each_entry(trigger_group, &trigger_groups, node) + lttng_trigger_group_sync_enablers(trigger_group); + return 0; +} + +struct lttng_event_enabler *lttng_event_enabler_create( + enum lttng_enabler_format_type format_type, struct lttng_kernel_event *event_param, struct lttng_channel *chan) { - struct lttng_enabler *enabler; + struct lttng_event_enabler *event_enabler; - enabler = kzalloc(sizeof(*enabler), GFP_KERNEL); - if (!enabler) + event_enabler = kzalloc(sizeof(*event_enabler), GFP_KERNEL); + if (!event_enabler) return NULL; - enabler->type = type; - INIT_LIST_HEAD(&enabler->filter_bytecode_head); - memcpy(&enabler->event_param, event_param, - sizeof(enabler->event_param)); - enabler->chan = chan; + event_enabler->base.format_type = format_type; + INIT_LIST_HEAD(&event_enabler->base.filter_bytecode_head); + memcpy(&event_enabler->base.event_param, event_param, + sizeof(event_enabler->base.event_param)); + event_enabler->chan = chan; /* ctx left NULL */ - enabler->enabled = 0; - enabler->evtype = LTTNG_TYPE_ENABLER; + event_enabler->base.enabled = 0; + event_enabler->base.evtype = LTTNG_TYPE_ENABLER; mutex_lock(&sessions_mutex); - list_add(&enabler->node, &enabler->chan->session->enablers_head); - lttng_session_lazy_sync_enablers(enabler->chan->session); + list_add(&event_enabler->node, &event_enabler->chan->session->enablers_head); + lttng_session_lazy_sync_event_enablers(event_enabler->chan->session); mutex_unlock(&sessions_mutex); - return enabler; + return event_enabler; } -int lttng_enabler_enable(struct lttng_enabler *enabler) +int lttng_event_enabler_enable(struct lttng_event_enabler *event_enabler) { mutex_lock(&sessions_mutex); - enabler->enabled = 1; - lttng_session_lazy_sync_enablers(enabler->chan->session); + lttng_event_enabler_as_enabler(event_enabler)->enabled = 1; + lttng_session_lazy_sync_event_enablers(event_enabler->chan->session); mutex_unlock(&sessions_mutex); return 0; } -int lttng_enabler_disable(struct lttng_enabler *enabler) +int lttng_event_enabler_disable(struct lttng_event_enabler *event_enabler) { mutex_lock(&sessions_mutex); - enabler->enabled = 0; - lttng_session_lazy_sync_enablers(enabler->chan->session); + lttng_event_enabler_as_enabler(event_enabler)->enabled = 0; + lttng_session_lazy_sync_event_enablers(event_enabler->chan->session); mutex_unlock(&sessions_mutex); return 0; } +static int lttng_enabler_attach_bytecode(struct lttng_enabler *enabler, struct lttng_kernel_filter_bytecode __user *bytecode) { @@ -1398,11 +2131,12 @@ int lttng_enabler_attach_bytecode(struct lttng_enabler *enabler, sizeof(*bytecode) + bytecode_len); if (ret) goto error_free; + bytecode_node->enabler = enabler; /* Enforce length based on allocated size */ bytecode_node->bc.len = bytecode_len; list_add_tail(&bytecode_node->node, &enabler->filter_bytecode_head); - lttng_session_lazy_sync_enablers(enabler->chan->session); + return 0; error_free: @@ -1410,7 +2144,35 @@ error_free: return ret; } -int lttng_enabler_attach_context(struct lttng_enabler *enabler, +int lttng_event_enabler_attach_bytecode(struct lttng_event_enabler *event_enabler, + struct lttng_kernel_filter_bytecode __user *bytecode) +{ + int ret; + ret = lttng_enabler_attach_bytecode( + lttng_event_enabler_as_enabler(event_enabler), bytecode); + if (ret) + goto error; + + lttng_session_lazy_sync_event_enablers(event_enabler->chan->session); + return 0; + +error: + return ret; +} + +int lttng_event_add_callsite(struct lttng_event *event, + struct lttng_kernel_event_callsite __user *callsite) +{ + + switch (event->instrumentation) { + case LTTNG_KERNEL_UPROBE: + return lttng_uprobes_event_add_callsite(event, callsite); + default: + return -EINVAL; + } +} + +int lttng_event_enabler_attach_context(struct lttng_event_enabler *event_enabler, struct lttng_kernel_context *context_param) { return -ENOSYS; @@ -1426,27 +2188,132 @@ void lttng_enabler_destroy(struct lttng_enabler *enabler) &enabler->filter_bytecode_head, node) { kfree(filter_node); } +} + +static +void lttng_event_enabler_destroy(struct lttng_event_enabler *event_enabler) +{ + lttng_enabler_destroy(lttng_event_enabler_as_enabler(event_enabler)); /* Destroy contexts */ - lttng_destroy_context(enabler->ctx); + lttng_destroy_context(event_enabler->ctx); + + list_del(&event_enabler->node); + kfree(event_enabler); +} + +struct lttng_trigger_enabler *lttng_trigger_enabler_create( + struct lttng_trigger_group *trigger_group, + enum lttng_enabler_format_type format_type, + struct lttng_kernel_trigger *trigger_param) +{ + struct lttng_trigger_enabler *trigger_enabler; + + trigger_enabler = kzalloc(sizeof(*trigger_enabler), GFP_KERNEL); + if (!trigger_enabler) + return NULL; + + trigger_enabler->base.format_type = format_type; + INIT_LIST_HEAD(&trigger_enabler->base.filter_bytecode_head); - list_del(&enabler->node); - kfree(enabler); + trigger_enabler->id = trigger_param->id; + + memcpy(&trigger_enabler->base.event_param.name, trigger_param->name, + sizeof(trigger_enabler->base.event_param.name)); + trigger_enabler->base.event_param.instrumentation = trigger_param->instrumentation; + trigger_enabler->base.evtype = LTTNG_TYPE_ENABLER; + + trigger_enabler->base.enabled = 0; + trigger_enabler->group = trigger_group; + + mutex_lock(&sessions_mutex); + list_add(&trigger_enabler->node, &trigger_enabler->group->enablers_head); + lttng_trigger_group_sync_enablers(trigger_enabler->group); + + mutex_unlock(&sessions_mutex); + + return trigger_enabler; +} + +int lttng_trigger_enabler_enable(struct lttng_trigger_enabler *trigger_enabler) +{ + mutex_lock(&sessions_mutex); + lttng_trigger_enabler_as_enabler(trigger_enabler)->enabled = 1; + lttng_trigger_group_sync_enablers(trigger_enabler->group); + mutex_unlock(&sessions_mutex); + return 0; +} + +int lttng_trigger_enabler_disable(struct lttng_trigger_enabler *trigger_enabler) +{ + mutex_lock(&sessions_mutex); + lttng_trigger_enabler_as_enabler(trigger_enabler)->enabled = 0; + lttng_trigger_group_sync_enablers(trigger_enabler->group); + mutex_unlock(&sessions_mutex); + return 0; +} + +int lttng_trigger_enabler_attach_bytecode(struct lttng_trigger_enabler *trigger_enabler, + struct lttng_kernel_filter_bytecode __user *bytecode) +{ + int ret; + + ret = lttng_enabler_attach_bytecode( + lttng_trigger_enabler_as_enabler(trigger_enabler), bytecode); + if (ret) + goto error; + + lttng_trigger_group_sync_enablers(trigger_enabler->group); + return 0; + +error: + return ret; +} + +int lttng_trigger_add_callsite(struct lttng_trigger *trigger, + struct lttng_kernel_event_callsite __user *callsite) +{ + + switch (trigger->instrumentation) { + case LTTNG_KERNEL_UPROBE: + return lttng_uprobes_trigger_add_callsite(trigger, callsite); + default: + return -EINVAL; + } +} + +int lttng_trigger_enabler_attach_context(struct lttng_trigger_enabler *trigger_enabler, + struct lttng_kernel_context *context_param) +{ + return -ENOSYS; +} + +static +void lttng_trigger_enabler_destroy(struct lttng_trigger_enabler *trigger_enabler) +{ + if (!trigger_enabler) { + return; + } + + list_del(&trigger_enabler->node); + + lttng_enabler_destroy(lttng_trigger_enabler_as_enabler(trigger_enabler)); + kfree(trigger_enabler); } /* - * lttng_session_sync_enablers should be called just before starting a + * lttng_session_sync_event_enablers should be called just before starting a * session. * Should be called with sessions mutex held. */ static -void lttng_session_sync_enablers(struct lttng_session *session) +void lttng_session_sync_event_enablers(struct lttng_session *session) { - struct lttng_enabler *enabler; + struct lttng_event_enabler *event_enabler; struct lttng_event *event; - list_for_each_entry(enabler, &session->enablers_head, node) - lttng_enabler_ref_events(enabler); + list_for_each_entry(event_enabler, &session->enablers_head, node) + lttng_event_enabler_ref_events(event_enabler); /* * For each event, if at least one of its enablers is enabled, * and its channel and session transient states are enabled, we @@ -1480,7 +2347,7 @@ void lttng_session_sync_enablers(struct lttng_session *session) */ enabled = enabled && session->tstate && event->chan->tstate; - ACCESS_ONCE(event->enabled) = enabled; + WRITE_ONCE(event->enabled, enabled); /* * Sync tracepoint registration with event enabled * state. @@ -1518,12 +2385,79 @@ void lttng_session_sync_enablers(struct lttng_session *session) * Should be called with sessions mutex held. */ static -void lttng_session_lazy_sync_enablers(struct lttng_session *session) +void lttng_session_lazy_sync_event_enablers(struct lttng_session *session) { /* We can skip if session is not active */ if (!session->active) return; - lttng_session_sync_enablers(session); + lttng_session_sync_event_enablers(session); +} + +static +void lttng_trigger_group_sync_enablers(struct lttng_trigger_group *trigger_group) +{ + struct lttng_trigger_enabler *trigger_enabler; + struct lttng_trigger *trigger; + + list_for_each_entry(trigger_enabler, &trigger_group->enablers_head, node) + lttng_trigger_enabler_ref_triggers(trigger_enabler); + + /* + * For each trigger, if at least one of its enablers is enabled, + * we enable the trigger, else we disable it. + */ + list_for_each_entry(trigger, &trigger_group->triggers_head, list) { + struct lttng_enabler_ref *enabler_ref; + struct lttng_bytecode_runtime *runtime; + int enabled = 0, has_enablers_without_bytecode = 0; + + switch (trigger->instrumentation) { + case LTTNG_KERNEL_TRACEPOINT: + case LTTNG_KERNEL_SYSCALL: + /* Enable triggers */ + list_for_each_entry(enabler_ref, + &trigger->enablers_ref_head, node) { + if (enabler_ref->ref->enabled) { + enabled = 1; + break; + } + } + break; + default: + /* Not handled with sync. */ + continue; + } + + WRITE_ONCE(trigger->enabled, enabled); + /* + * Sync tracepoint registration with trigger enabled + * state. + */ + if (enabled) { + if (!trigger->registered) + register_trigger(trigger); + } else { + if (trigger->registered) + _lttng_trigger_unregister(trigger); + } + + /* Check if has enablers without bytecode enabled */ + list_for_each_entry(enabler_ref, + &trigger->enablers_ref_head, node) { + if (enabler_ref->ref->enabled + && list_empty(&enabler_ref->ref->filter_bytecode_head)) { + has_enablers_without_bytecode = 1; + break; + } + } + trigger->has_enablers_without_bytecode = + has_enablers_without_bytecode; + + /* Enable filters */ + list_for_each_entry(runtime, + &trigger->bytecode_runtime_head, node) + lttng_filter_sync_state(runtime); + } } /* @@ -1587,72 +2521,393 @@ int lttng_metadata_output_channel(struct lttng_metadata_stream *stream, stream->metadata_in += reserve_len; ret = reserve_len; -end: - mutex_unlock(&stream->metadata_cache->lock); +end: + mutex_unlock(&stream->metadata_cache->lock); + return ret; +} + +/* + * Write the metadata to the metadata cache. + * Must be called with sessions_mutex held. + * The metadata cache lock protects us from concurrent read access from + * thread outputting metadata content to ring buffer. + */ +int lttng_metadata_printf(struct lttng_session *session, + const char *fmt, ...) +{ + char *str; + size_t len; + va_list ap; + struct lttng_metadata_stream *stream; + + WARN_ON_ONCE(!READ_ONCE(session->active)); + + va_start(ap, fmt); + str = kvasprintf(GFP_KERNEL, fmt, ap); + va_end(ap); + if (!str) + return -ENOMEM; + + len = strlen(str); + mutex_lock(&session->metadata_cache->lock); + if (session->metadata_cache->metadata_written + len > + session->metadata_cache->cache_alloc) { + char *tmp_cache_realloc; + unsigned int tmp_cache_alloc_size; + + tmp_cache_alloc_size = max_t(unsigned int, + session->metadata_cache->cache_alloc + len, + session->metadata_cache->cache_alloc << 1); + tmp_cache_realloc = vzalloc(tmp_cache_alloc_size); + if (!tmp_cache_realloc) + goto err; + if (session->metadata_cache->data) { + memcpy(tmp_cache_realloc, + session->metadata_cache->data, + session->metadata_cache->cache_alloc); + vfree(session->metadata_cache->data); + } + + session->metadata_cache->cache_alloc = tmp_cache_alloc_size; + session->metadata_cache->data = tmp_cache_realloc; + } + memcpy(session->metadata_cache->data + + session->metadata_cache->metadata_written, + str, len); + session->metadata_cache->metadata_written += len; + mutex_unlock(&session->metadata_cache->lock); + kfree(str); + + list_for_each_entry(stream, &session->metadata_cache->metadata_stream, list) + wake_up_interruptible(&stream->read_wait); + + return 0; + +err: + mutex_unlock(&session->metadata_cache->lock); + kfree(str); + return -ENOMEM; +} + +static +int print_tabs(struct lttng_session *session, size_t nesting) +{ + size_t i; + + for (i = 0; i < nesting; i++) { + int ret; + + ret = lttng_metadata_printf(session, " "); + if (ret) { + return ret; + } + } + return 0; +} + +/* + * Must be called with sessions_mutex held. + */ +static +int _lttng_struct_type_statedump(struct lttng_session *session, + const struct lttng_type *type, + size_t nesting) +{ + int ret; + uint32_t i, nr_fields; + + ret = print_tabs(session, nesting); + if (ret) + return ret; + ret = lttng_metadata_printf(session, + "struct {\n"); + if (ret) + return ret; + nr_fields = type->u._struct.nr_fields; + for (i = 0; i < nr_fields; i++) { + const struct lttng_event_field *iter_field; + + iter_field = &type->u._struct.fields[i]; + ret = _lttng_field_statedump(session, iter_field, nesting + 1); + if (ret) + return ret; + } + ret = print_tabs(session, nesting); + if (ret) + return ret; + ret = lttng_metadata_printf(session, + "}"); + return ret; +} + +/* + * Must be called with sessions_mutex held. + */ +static +int _lttng_struct_statedump(struct lttng_session *session, + const struct lttng_event_field *field, + size_t nesting) +{ + int ret; + + ret = _lttng_struct_type_statedump(session, + &field->type, nesting); + if (ret) + return ret; + ret = lttng_metadata_printf(session, + "_%s;\n", + field->name); + return ret; +} + +/* + * Must be called with sessions_mutex held. + */ +static +int _lttng_variant_type_statedump(struct lttng_session *session, + const struct lttng_type *type, + size_t nesting) +{ + int ret; + uint32_t i, nr_choices; + + ret = print_tabs(session, nesting); + if (ret) + return ret; + ret = lttng_metadata_printf(session, + "variant <_%s> {\n", + type->u.variant.tag_name); + if (ret) + return ret; + nr_choices = type->u.variant.nr_choices; + for (i = 0; i < nr_choices; i++) { + const struct lttng_event_field *iter_field; + + iter_field = &type->u.variant.choices[i]; + ret = _lttng_field_statedump(session, iter_field, nesting + 1); + if (ret) + return ret; + } + ret = print_tabs(session, nesting); + if (ret) + return ret; + ret = lttng_metadata_printf(session, + "}"); + return ret; +} + +/* + * Must be called with sessions_mutex held. + */ +static +int _lttng_variant_statedump(struct lttng_session *session, + const struct lttng_event_field *field, + size_t nesting) +{ + int ret; + + ret = _lttng_variant_type_statedump(session, + &field->type, nesting); + if (ret) + return ret; + ret = lttng_metadata_printf(session, + "_%s;\n", + field->name); + return ret; +} + +/* + * Must be called with sessions_mutex held. + */ +static +int _lttng_array_compound_statedump(struct lttng_session *session, + const struct lttng_event_field *field, + size_t nesting) +{ + int ret; + const struct lttng_type *elem_type; + + /* Only array of structures and variants are currently supported. */ + elem_type = field->type.u.array_compound.elem_type; + switch (elem_type->atype) { + case atype_struct: + ret = _lttng_struct_type_statedump(session, elem_type, nesting); + if (ret) + return ret; + break; + case atype_variant: + ret = _lttng_variant_type_statedump(session, elem_type, nesting); + if (ret) + return ret; + break; + default: + return -EINVAL; + } + ret = lttng_metadata_printf(session, + " _%s[%u];\n", + field->name, + field->type.u.array_compound.length); return ret; } /* - * Write the metadata to the metadata cache. * Must be called with sessions_mutex held. - * The metadata cache lock protects us from concurrent read access from - * thread outputting metadata content to ring buffer. */ -int lttng_metadata_printf(struct lttng_session *session, - const char *fmt, ...) +static +int _lttng_sequence_compound_statedump(struct lttng_session *session, + const struct lttng_event_field *field, + size_t nesting) { - char *str; - size_t len; - va_list ap; - struct lttng_metadata_stream *stream; - - WARN_ON_ONCE(!ACCESS_ONCE(session->active)); + int ret; + const char *length_name; + const struct lttng_type *elem_type; - va_start(ap, fmt); - str = kvasprintf(GFP_KERNEL, fmt, ap); - va_end(ap); - if (!str) - return -ENOMEM; + length_name = field->type.u.sequence_compound.length_name; - len = strlen(str); - mutex_lock(&session->metadata_cache->lock); - if (session->metadata_cache->metadata_written + len > - session->metadata_cache->cache_alloc) { - char *tmp_cache_realloc; - unsigned int tmp_cache_alloc_size; + /* Only array of structures and variants are currently supported. */ + elem_type = field->type.u.sequence_compound.elem_type; + switch (elem_type->atype) { + case atype_struct: + ret = _lttng_struct_type_statedump(session, elem_type, nesting); + if (ret) + return ret; + break; + case atype_variant: + ret = _lttng_variant_type_statedump(session, elem_type, nesting); + if (ret) + return ret; + break; + default: + return -EINVAL; + } + ret = lttng_metadata_printf(session, + " _%s[ _%s ];\n", + field->name, + length_name); + return ret; +} - tmp_cache_alloc_size = max_t(unsigned int, - session->metadata_cache->cache_alloc + len, - session->metadata_cache->cache_alloc << 1); - tmp_cache_realloc = lttng_vzalloc(tmp_cache_alloc_size); - if (!tmp_cache_realloc) - goto err; - if (session->metadata_cache->data) { - memcpy(tmp_cache_realloc, - session->metadata_cache->data, - session->metadata_cache->cache_alloc); - vfree(session->metadata_cache->data); - } +/* + * Must be called with sessions_mutex held. + */ +static +int _lttng_enum_statedump(struct lttng_session *session, + const struct lttng_event_field *field, + size_t nesting) +{ + const struct lttng_enum_desc *enum_desc; + const struct lttng_integer_type *container_type; + int ret; + unsigned int i, nr_entries; - session->metadata_cache->cache_alloc = tmp_cache_alloc_size; - session->metadata_cache->data = tmp_cache_realloc; - } - memcpy(session->metadata_cache->data + - session->metadata_cache->metadata_written, - str, len); - session->metadata_cache->metadata_written += len; - mutex_unlock(&session->metadata_cache->lock); - kfree(str); + enum_desc = field->type.u.basic.enumeration.desc; + container_type = &field->type.u.basic.enumeration.container_type; + nr_entries = enum_desc->nr_entries; - list_for_each_entry(stream, &session->metadata_cache->metadata_stream, list) - wake_up_interruptible(&stream->read_wait); + ret = print_tabs(session, nesting); + if (ret) + goto end; + ret = lttng_metadata_printf(session, + "enum : integer { size = %u; align = %u; signed = %u; encoding = %s; base = %u;%s } {\n", + container_type->size, + container_type->alignment, + container_type->signedness, + (container_type->encoding == lttng_encode_none) + ? "none" + : (container_type->encoding == lttng_encode_UTF8) + ? "UTF8" + : "ASCII", + container_type->base, +#if __BYTE_ORDER == __BIG_ENDIAN + container_type->reverse_byte_order ? " byte_order = le;" : "" +#else + container_type->reverse_byte_order ? " byte_order = be;" : "" +#endif + ); + if (ret) + goto end; + /* Dump all entries */ + for (i = 0; i < nr_entries; i++) { + const struct lttng_enum_entry *entry = &enum_desc->entries[i]; + int j, len; - return 0; + ret = print_tabs(session, nesting + 1); + if (ret) + goto end; + ret = lttng_metadata_printf(session, + "\""); + if (ret) + goto end; + len = strlen(entry->string); + /* Escape the character '"' */ + for (j = 0; j < len; j++) { + char c = entry->string[j]; + + switch (c) { + case '"': + ret = lttng_metadata_printf(session, + "\\\""); + break; + case '\\': + ret = lttng_metadata_printf(session, + "\\\\"); + break; + default: + ret = lttng_metadata_printf(session, + "%c", c); + break; + } + if (ret) + goto end; + } + ret = lttng_metadata_printf(session, "\""); + if (ret) + goto end; -err: - mutex_unlock(&session->metadata_cache->lock); - kfree(str); - return -ENOMEM; + if (entry->options.is_auto) { + ret = lttng_metadata_printf(session, ",\n"); + if (ret) + goto end; + } else { + ret = lttng_metadata_printf(session, + " = "); + if (ret) + goto end; + if (entry->start.signedness) + ret = lttng_metadata_printf(session, + "%lld", (long long) entry->start.value); + else + ret = lttng_metadata_printf(session, + "%llu", entry->start.value); + if (ret) + goto end; + if (entry->start.signedness == entry->end.signedness && + entry->start.value + == entry->end.value) { + ret = lttng_metadata_printf(session, + ",\n"); + } else { + if (entry->end.signedness) { + ret = lttng_metadata_printf(session, + " ... %lld,\n", + (long long) entry->end.value); + } else { + ret = lttng_metadata_printf(session, + " ... %llu,\n", + entry->end.value); + } + } + if (ret) + goto end; + } + } + ret = print_tabs(session, nesting); + if (ret) + goto end; + ret = lttng_metadata_printf(session, "} _%s;\n", + field->name); +end: + return ret; } /* @@ -1660,14 +2915,18 @@ err: */ static int _lttng_field_statedump(struct lttng_session *session, - const struct lttng_event_field *field) + const struct lttng_event_field *field, + size_t nesting) { int ret = 0; switch (field->type.atype) { case atype_integer: + ret = print_tabs(session, nesting); + if (ret) + return ret; ret = lttng_metadata_printf(session, - " integer { size = %u; align = %u; signed = %u; encoding = %s; base = %u;%s } _%s;\n", + "integer { size = %u; align = %u; signed = %u; encoding = %s; base = %u;%s } _%s;\n", field->type.u.basic.integer.size, field->type.u.basic.integer.alignment, field->type.u.basic.integer.signedness, @@ -1677,7 +2936,7 @@ int _lttng_field_statedump(struct lttng_session *session, ? "UTF8" : "ASCII", field->type.u.basic.integer.base, -#ifdef __BIG_ENDIAN +#if __BYTE_ORDER == __BIG_ENDIAN field->type.u.basic.integer.reverse_byte_order ? " byte_order = le;" : "", #else field->type.u.basic.integer.reverse_byte_order ? " byte_order = be;" : "", @@ -1685,18 +2944,30 @@ int _lttng_field_statedump(struct lttng_session *session, field->name); break; case atype_enum: - ret = lttng_metadata_printf(session, - " %s _%s;\n", - field->type.u.basic.enumeration.name, - field->name); + ret = _lttng_enum_statedump(session, field, nesting); break; case atype_array: + case atype_array_bitfield: { const struct lttng_basic_type *elem_type; elem_type = &field->type.u.array.elem_type; + if (field->type.u.array.elem_alignment) { + ret = print_tabs(session, nesting); + if (ret) + return ret; + ret = lttng_metadata_printf(session, + "struct { } align(%u) _%s_padding;\n", + field->type.u.array.elem_alignment * CHAR_BIT, + field->name); + if (ret) + return ret; + } + ret = print_tabs(session, nesting); + if (ret) + return ret; ret = lttng_metadata_printf(session, - " integer { size = %u; align = %u; signed = %u; encoding = %s; base = %u;%s } _%s[%u];\n", + "integer { size = %u; align = %u; signed = %u; encoding = %s; base = %u;%s } _%s[%u];\n", elem_type->u.basic.integer.size, elem_type->u.basic.integer.alignment, elem_type->u.basic.integer.signedness, @@ -1706,7 +2977,7 @@ int _lttng_field_statedump(struct lttng_session *session, ? "UTF8" : "ASCII", elem_type->u.basic.integer.base, -#ifdef __BIG_ENDIAN +#if __BYTE_ORDER == __BIG_ENDIAN elem_type->u.basic.integer.reverse_byte_order ? " byte_order = le;" : "", #else elem_type->u.basic.integer.reverse_byte_order ? " byte_order = be;" : "", @@ -1715,14 +2986,18 @@ int _lttng_field_statedump(struct lttng_session *session, break; } case atype_sequence: + case atype_sequence_bitfield: { const struct lttng_basic_type *elem_type; const struct lttng_basic_type *length_type; elem_type = &field->type.u.sequence.elem_type; length_type = &field->type.u.sequence.length_type; + ret = print_tabs(session, nesting); + if (ret) + return ret; ret = lttng_metadata_printf(session, - " integer { size = %u; align = %u; signed = %u; encoding = %s; base = %u;%s } __%s_length;\n", + "integer { size = %u; align = %u; signed = %u; encoding = %s; base = %u;%s } __%s_length;\n", length_type->u.basic.integer.size, (unsigned int) length_type->u.basic.integer.alignment, length_type->u.basic.integer.signedness, @@ -1732,7 +3007,7 @@ int _lttng_field_statedump(struct lttng_session *session, ? "UTF8" : "ASCII"), length_type->u.basic.integer.base, -#ifdef __BIG_ENDIAN +#if __BYTE_ORDER == __BIG_ENDIAN length_type->u.basic.integer.reverse_byte_order ? " byte_order = le;" : "", #else length_type->u.basic.integer.reverse_byte_order ? " byte_order = be;" : "", @@ -1741,8 +3016,22 @@ int _lttng_field_statedump(struct lttng_session *session, if (ret) return ret; + if (field->type.u.sequence.elem_alignment) { + ret = print_tabs(session, nesting); + if (ret) + return ret; + ret = lttng_metadata_printf(session, + "struct { } align(%u) _%s_padding;\n", + field->type.u.sequence.elem_alignment * CHAR_BIT, + field->name); + if (ret) + return ret; + } + ret = print_tabs(session, nesting); + if (ret) + return ret; ret = lttng_metadata_printf(session, - " integer { size = %u; align = %u; signed = %u; encoding = %s; base = %u;%s } _%s[ __%s_length ];\n", + "integer { size = %u; align = %u; signed = %u; encoding = %s; base = %u;%s } _%s[ __%s_length ];\n", elem_type->u.basic.integer.size, (unsigned int) elem_type->u.basic.integer.alignment, elem_type->u.basic.integer.signedness, @@ -1752,7 +3041,7 @@ int _lttng_field_statedump(struct lttng_session *session, ? "UTF8" : "ASCII"), elem_type->u.basic.integer.base, -#ifdef __BIG_ENDIAN +#if __BYTE_ORDER == __BIG_ENDIAN elem_type->u.basic.integer.reverse_byte_order ? " byte_order = le;" : "", #else elem_type->u.basic.integer.reverse_byte_order ? " byte_order = be;" : "", @@ -1764,12 +3053,28 @@ int _lttng_field_statedump(struct lttng_session *session, case atype_string: /* Default encoding is UTF8 */ + ret = print_tabs(session, nesting); + if (ret) + return ret; ret = lttng_metadata_printf(session, - " string%s _%s;\n", + "string%s _%s;\n", field->type.u.basic.string.encoding == lttng_encode_ASCII ? " { encoding = ASCII; }" : "", field->name); break; + case atype_struct: + ret = _lttng_struct_statedump(session, field, nesting); + break; + case atype_array_compound: + ret = _lttng_array_compound_statedump(session, field, nesting); + break; + case atype_sequence_compound: + ret = _lttng_sequence_compound_statedump(session, field, nesting); + break; + case atype_variant: + ret = _lttng_variant_statedump(session, field, nesting); + break; + default: WARN_ON_ONCE(1); return -EINVAL; @@ -1789,7 +3094,7 @@ int _lttng_context_metadata_statedump(struct lttng_session *session, for (i = 0; i < ctx->nr_fields; i++) { const struct lttng_ctx_field *field = &ctx->fields[i]; - ret = _lttng_field_statedump(session, &field->event_field); + ret = _lttng_field_statedump(session, &field->event_field, 2); if (ret) return ret; } @@ -1807,7 +3112,7 @@ int _lttng_fields_metadata_statedump(struct lttng_session *session, for (i = 0; i < desc->nr_fields; i++) { const struct lttng_event_field *field = &desc->fields[i]; - ret = _lttng_field_statedump(session, field); + ret = _lttng_field_statedump(session, field, 2); if (ret) return ret; } @@ -1824,7 +3129,7 @@ int _lttng_event_metadata_statedump(struct lttng_session *session, { int ret = 0; - if (event->metadata_dumped || !ACCESS_ONCE(session->active)) + if (event->metadata_dumped || !READ_ONCE(session->active)) return 0; if (chan->channel_type == METADATA_CHANNEL) return 0; @@ -1891,7 +3196,7 @@ int _lttng_channel_metadata_statedump(struct lttng_session *session, { int ret = 0; - if (chan->metadata_dumped || !ACCESS_ONCE(session->active)) + if (chan->metadata_dumped || !READ_ONCE(session->active)) return 0; if (chan->channel_type == METADATA_CHANNEL) @@ -2002,28 +3307,94 @@ int _lttng_event_header_declare(struct lttng_session *session) * taken at start of trace. * Yes, this is only an approximation. Yes, we can (and will) do better * in future versions. + * This function may return a negative offset. It may happen if the + * system sets the REALTIME clock to 0 after boot. */ static -uint64_t measure_clock_offset(void) +int64_t measure_clock_offset(void) { - uint64_t offset, monotonic[2], realtime; + uint64_t monotonic_avg, monotonic[2], realtime; + uint64_t tcf = trace_clock_freq(); + int64_t offset; struct timespec rts = { 0, 0 }; unsigned long flags; /* Disable interrupts to increase correlation precision. */ local_irq_save(flags); monotonic[0] = trace_clock_read64(); - getnstimeofday(&rts); + getnstimeofday(&rts); monotonic[1] = trace_clock_read64(); local_irq_restore(flags); - offset = (monotonic[0] + monotonic[1]) >> 1; - realtime = (uint64_t) rts.tv_sec * NSEC_PER_SEC; - realtime += rts.tv_nsec; - offset = realtime - offset; + monotonic_avg = (monotonic[0] + monotonic[1]) >> 1; + realtime = (uint64_t) rts.tv_sec * tcf; + if (tcf == NSEC_PER_SEC) { + realtime += rts.tv_nsec; + } else { + uint64_t n = rts.tv_nsec * tcf; + + do_div(n, NSEC_PER_SEC); + realtime += n; + } + offset = (int64_t) realtime - monotonic_avg; return offset; } +static +int print_escaped_ctf_string(struct lttng_session *session, const char *string) +{ + int ret; + size_t i; + char cur; + + i = 0; + cur = string[i]; + while (cur != '\0') { + switch (cur) { + case '\n': + ret = lttng_metadata_printf(session, "%s", "\\n"); + break; + case '\\': + case '"': + ret = lttng_metadata_printf(session, "%c", '\\'); + if (ret) + goto error; + /* We still print the current char */ + /* Fallthrough */ + default: + ret = lttng_metadata_printf(session, "%c", cur); + break; + } + + if (ret) + goto error; + + cur = string[++i]; + } +error: + return ret; +} + +static +int print_metadata_escaped_field(struct lttng_session *session, const char *field, + const char *field_value) +{ + int ret; + + ret = lttng_metadata_printf(session, " %s = \"", field); + if (ret) + goto error; + + ret = print_escaped_ctf_string(session, field_value); + if (ret) + goto error; + + ret = lttng_metadata_printf(session, "\";\n"); + +error: + return ret; +} + /* * Output metadata into this session's metadata buffers. * Must be called with sessions_mutex held. @@ -2033,11 +3404,12 @@ int _lttng_session_metadata_statedump(struct lttng_session *session) { unsigned char *uuid_c = session->uuid.b; unsigned char uuid_s[37], clock_uuid_s[BOOT_ID_LEN]; + const char *product_uuid; struct lttng_channel *chan; struct lttng_event *event; int ret = 0; - if (!ACCESS_ONCE(session->active)) + if (!READ_ONCE(session->active)) return 0; if (session->metadata_dumped) goto skip_session; @@ -2079,7 +3451,7 @@ int _lttng_session_metadata_statedump(struct lttng_session *session) CTF_SPEC_MAJOR, CTF_SPEC_MINOR, uuid_s, -#ifdef __BIG_ENDIAN +#if __BYTE_ORDER == __BIG_ENDIAN "be" #else "le" @@ -2099,7 +3471,7 @@ int _lttng_session_metadata_statedump(struct lttng_session *session) " tracer_major = %d;\n" " tracer_minor = %d;\n" " tracer_patchlevel = %d;\n" - "};\n\n", + " trace_buffering_scheme = \"global\";\n", current->nsproxy->uts_ns->name.nodename, utsname()->sysname, utsname()->release, @@ -2111,10 +3483,34 @@ int _lttng_session_metadata_statedump(struct lttng_session *session) if (ret) goto end; + ret = print_metadata_escaped_field(session, "trace_name", session->name); + if (ret) + goto end; + ret = print_metadata_escaped_field(session, "trace_creation_datetime", + session->creation_time); + if (ret) + goto end; + + /* Add the product UUID to the 'env' section */ + product_uuid = dmi_get_system_info(DMI_PRODUCT_UUID); + if (product_uuid) { + ret = lttng_metadata_printf(session, + " product_uuid = \"%s\";\n", + product_uuid + ); + if (ret) + goto end; + } + + /* Close the 'env' section */ + ret = lttng_metadata_printf(session, "};\n\n"); + if (ret) + goto end; + ret = lttng_metadata_printf(session, "clock {\n" - " name = %s;\n", - "monotonic" + " name = \"%s\";\n", + trace_clock_name() ); if (ret) goto end; @@ -2129,13 +3525,14 @@ int _lttng_session_metadata_statedump(struct lttng_session *session) } ret = lttng_metadata_printf(session, - " description = \"Monotonic Clock\";\n" + " description = \"%s\";\n" " freq = %llu; /* Frequency, in Hz */\n" " /* clock value offset from Epoch is: offset * (1/freq) */\n" - " offset = %llu;\n" + " offset = %lld;\n" "};\n\n", + trace_clock_description(), (unsigned long long) trace_clock_freq(), - (unsigned long long) measure_clock_offset() + (long long) measure_clock_offset() ); if (ret) goto end; @@ -2143,20 +3540,23 @@ int _lttng_session_metadata_statedump(struct lttng_session *session) ret = lttng_metadata_printf(session, "typealias integer {\n" " size = 27; align = 1; signed = false;\n" - " map = clock.monotonic.value;\n" + " map = clock.%s.value;\n" "} := uint27_clock_monotonic_t;\n" "\n" "typealias integer {\n" " size = 32; align = %u; signed = false;\n" - " map = clock.monotonic.value;\n" + " map = clock.%s.value;\n" "} := uint32_clock_monotonic_t;\n" "\n" "typealias integer {\n" " size = 64; align = %u; signed = false;\n" - " map = clock.monotonic.value;\n" + " map = clock.%s.value;\n" "} := uint64_clock_monotonic_t;\n\n", + trace_clock_name(), lttng_alignof(uint32_t) * CHAR_BIT, - lttng_alignof(uint64_t) * CHAR_BIT + trace_clock_name(), + lttng_alignof(uint64_t) * CHAR_BIT, + trace_clock_name() ); if (ret) goto end; @@ -2225,6 +3625,133 @@ void lttng_transport_unregister(struct lttng_transport *transport) } EXPORT_SYMBOL_GPL(lttng_transport_unregister); +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0)) + +enum cpuhp_state lttng_hp_prepare; +enum cpuhp_state lttng_hp_online; + +static int lttng_hotplug_prepare(unsigned int cpu, struct hlist_node *node) +{ + struct lttng_cpuhp_node *lttng_node; + + lttng_node = container_of(node, struct lttng_cpuhp_node, node); + switch (lttng_node->component) { + case LTTNG_RING_BUFFER_FRONTEND: + return 0; + case LTTNG_RING_BUFFER_BACKEND: + return lttng_cpuhp_rb_backend_prepare(cpu, lttng_node); + case LTTNG_RING_BUFFER_ITER: + return 0; + case LTTNG_CONTEXT_PERF_COUNTERS: + return 0; + default: + return -EINVAL; + } +} + +static int lttng_hotplug_dead(unsigned int cpu, struct hlist_node *node) +{ + struct lttng_cpuhp_node *lttng_node; + + lttng_node = container_of(node, struct lttng_cpuhp_node, node); + switch (lttng_node->component) { + case LTTNG_RING_BUFFER_FRONTEND: + return lttng_cpuhp_rb_frontend_dead(cpu, lttng_node); + case LTTNG_RING_BUFFER_BACKEND: + return 0; + case LTTNG_RING_BUFFER_ITER: + return 0; + case LTTNG_CONTEXT_PERF_COUNTERS: + return lttng_cpuhp_perf_counter_dead(cpu, lttng_node); + default: + return -EINVAL; + } +} + +static int lttng_hotplug_online(unsigned int cpu, struct hlist_node *node) +{ + struct lttng_cpuhp_node *lttng_node; + + lttng_node = container_of(node, struct lttng_cpuhp_node, node); + switch (lttng_node->component) { + case LTTNG_RING_BUFFER_FRONTEND: + return lttng_cpuhp_rb_frontend_online(cpu, lttng_node); + case LTTNG_RING_BUFFER_BACKEND: + return 0; + case LTTNG_RING_BUFFER_ITER: + return lttng_cpuhp_rb_iter_online(cpu, lttng_node); + case LTTNG_CONTEXT_PERF_COUNTERS: + return lttng_cpuhp_perf_counter_online(cpu, lttng_node); + default: + return -EINVAL; + } +} + +static int lttng_hotplug_offline(unsigned int cpu, struct hlist_node *node) +{ + struct lttng_cpuhp_node *lttng_node; + + lttng_node = container_of(node, struct lttng_cpuhp_node, node); + switch (lttng_node->component) { + case LTTNG_RING_BUFFER_FRONTEND: + return lttng_cpuhp_rb_frontend_offline(cpu, lttng_node); + case LTTNG_RING_BUFFER_BACKEND: + return 0; + case LTTNG_RING_BUFFER_ITER: + return 0; + case LTTNG_CONTEXT_PERF_COUNTERS: + return 0; + default: + return -EINVAL; + } +} + +static int __init lttng_init_cpu_hotplug(void) +{ + int ret; + + ret = cpuhp_setup_state_multi(CPUHP_BP_PREPARE_DYN, "lttng:prepare", + lttng_hotplug_prepare, + lttng_hotplug_dead); + if (ret < 0) { + return ret; + } + lttng_hp_prepare = ret; + lttng_rb_set_hp_prepare(ret); + + ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, "lttng:online", + lttng_hotplug_online, + lttng_hotplug_offline); + if (ret < 0) { + cpuhp_remove_multi_state(lttng_hp_prepare); + lttng_hp_prepare = 0; + return ret; + } + lttng_hp_online = ret; + lttng_rb_set_hp_online(ret); + + return 0; +} + +static void __exit lttng_exit_cpu_hotplug(void) +{ + lttng_rb_set_hp_online(0); + cpuhp_remove_multi_state(lttng_hp_online); + lttng_rb_set_hp_prepare(0); + cpuhp_remove_multi_state(lttng_hp_prepare); +} + +#else /* #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0)) */ +static int lttng_init_cpu_hotplug(void) +{ + return 0; +} +static void lttng_exit_cpu_hotplug(void) +{ +} +#endif /* #else #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0)) */ + + static int __init lttng_events_init(void) { int ret; @@ -2233,6 +3760,12 @@ static int __init lttng_events_init(void) if (ret) return ret; ret = wrapper_get_pfnblock_flags_mask_init(); + if (ret) + return ret; + ret = wrapper_get_pageblock_flags_mask_init(); + if (ret) + return ret; + ret = lttng_probes_init(); if (ret) return ret; ret = lttng_context_init(); @@ -2244,7 +3777,12 @@ static int __init lttng_events_init(void) event_cache = KMEM_CACHE(lttng_event, 0); if (!event_cache) { ret = -ENOMEM; - goto error_kmem; + goto error_kmem_event; + } + trigger_cache = KMEM_CACHE(lttng_trigger, 0); + if (!trigger_cache) { + ret = -ENOMEM; + goto error_kmem_trigger; } ret = lttng_abi_init(); if (ret) @@ -2252,16 +3790,55 @@ static int __init lttng_events_init(void) ret = lttng_logger_init(); if (ret) goto error_logger; + ret = lttng_init_cpu_hotplug(); + if (ret) + goto error_hotplug; + printk(KERN_NOTICE "LTTng: Loaded modules v%s.%s.%s%s (%s)%s%s\n", + __stringify(LTTNG_MODULES_MAJOR_VERSION), + __stringify(LTTNG_MODULES_MINOR_VERSION), + __stringify(LTTNG_MODULES_PATCHLEVEL_VERSION), + LTTNG_MODULES_EXTRAVERSION, + LTTNG_VERSION_NAME, +#ifdef LTTNG_EXTRA_VERSION_GIT + LTTNG_EXTRA_VERSION_GIT[0] == '\0' ? "" : " - " LTTNG_EXTRA_VERSION_GIT, +#else + "", +#endif +#ifdef LTTNG_EXTRA_VERSION_NAME + LTTNG_EXTRA_VERSION_NAME[0] == '\0' ? "" : " - " LTTNG_EXTRA_VERSION_NAME); +#else + ""); +#endif return 0; +error_hotplug: + lttng_logger_exit(); error_logger: lttng_abi_exit(); error_abi: + kmem_cache_destroy(trigger_cache); +error_kmem_trigger: kmem_cache_destroy(event_cache); -error_kmem: +error_kmem_event: lttng_tracepoint_exit(); error_tp: lttng_context_exit(); + printk(KERN_NOTICE "LTTng: Failed to load modules v%s.%s.%s%s (%s)%s%s\n", + __stringify(LTTNG_MODULES_MAJOR_VERSION), + __stringify(LTTNG_MODULES_MINOR_VERSION), + __stringify(LTTNG_MODULES_PATCHLEVEL_VERSION), + LTTNG_MODULES_EXTRAVERSION, + LTTNG_VERSION_NAME, +#ifdef LTTNG_EXTRA_VERSION_GIT + LTTNG_EXTRA_VERSION_GIT[0] == '\0' ? "" : " - " LTTNG_EXTRA_VERSION_GIT, +#else + "", +#endif +#ifdef LTTNG_EXTRA_VERSION_NAME + LTTNG_EXTRA_VERSION_NAME[0] == '\0' ? "" : " - " LTTNG_EXTRA_VERSION_NAME); +#else + ""); +#endif return ret; } @@ -2271,20 +3848,45 @@ static void __exit lttng_events_exit(void) { struct lttng_session *session, *tmpsession; + lttng_exit_cpu_hotplug(); lttng_logger_exit(); lttng_abi_exit(); list_for_each_entry_safe(session, tmpsession, &sessions, list) lttng_session_destroy(session); kmem_cache_destroy(event_cache); + kmem_cache_destroy(trigger_cache); lttng_tracepoint_exit(); lttng_context_exit(); + printk(KERN_NOTICE "LTTng: Unloaded modules v%s.%s.%s%s (%s)%s%s\n", + __stringify(LTTNG_MODULES_MAJOR_VERSION), + __stringify(LTTNG_MODULES_MINOR_VERSION), + __stringify(LTTNG_MODULES_PATCHLEVEL_VERSION), + LTTNG_MODULES_EXTRAVERSION, + LTTNG_VERSION_NAME, +#ifdef LTTNG_EXTRA_VERSION_GIT + LTTNG_EXTRA_VERSION_GIT[0] == '\0' ? "" : " - " LTTNG_EXTRA_VERSION_GIT, +#else + "", +#endif +#ifdef LTTNG_EXTRA_VERSION_NAME + LTTNG_EXTRA_VERSION_NAME[0] == '\0' ? "" : " - " LTTNG_EXTRA_VERSION_NAME); +#else + ""); +#endif } module_exit(lttng_events_exit); +#include "extra_version/patches.i" +#ifdef LTTNG_EXTRA_VERSION_GIT +MODULE_INFO(extra_version_git, LTTNG_EXTRA_VERSION_GIT); +#endif +#ifdef LTTNG_EXTRA_VERSION_NAME +MODULE_INFO(extra_version_name, LTTNG_EXTRA_VERSION_NAME); +#endif MODULE_LICENSE("GPL and additional rights"); MODULE_AUTHOR("Mathieu Desnoyers "); -MODULE_DESCRIPTION("LTTng Events"); +MODULE_DESCRIPTION("LTTng tracer"); MODULE_VERSION(__stringify(LTTNG_MODULES_MAJOR_VERSION) "." __stringify(LTTNG_MODULES_MINOR_VERSION) "." __stringify(LTTNG_MODULES_PATCHLEVEL_VERSION)