From: Mathieu Desnoyers Date: Fri, 19 Mar 2021 15:27:50 +0000 (-0400) Subject: SoW-2020-0003: Trace Hit Counters X-Git-Url: http://git.efficios.com/?p=deliverable%2Flttng-modules.git;a=commitdiff_plain;h=refs%2Fheads%2Fsow-2020-0002-rev3 SoW-2020-0003: Trace Hit Counters Signed-off-by: Mathieu Desnoyers Change-Id: Ib7ab673fb05b4824d0c7ba2e163dc1e88b465ee9 --- diff --git a/DO-NOT-COMMIT.txt b/DO-NOT-COMMIT.txt new file mode 100644 index 00000000..00fd55cf --- /dev/null +++ b/DO-NOT-COMMIT.txt @@ -0,0 +1 @@ +Trace hit counter diff --git a/include/lttng/abi.h b/include/lttng/abi.h index 0cdd6534..2cac459a 100644 --- a/include/lttng/abi.h +++ b/include/lttng/abi.h @@ -147,6 +147,46 @@ struct lttng_kernel_event_notifier { } __attribute__((packed)); #define LTTNG_KERNEL_COUNTER_DIMENSION_MAX 4 +#define LTTNG_KERNEL_EVENT_NOTIFIER_NOTIFICATION_PADDING 32 +struct lttng_kernel_event_notifier_notification { + uint64_t token; + uint16_t capture_buf_size; + char padding[LTTNG_KERNEL_EVENT_NOTIFIER_NOTIFICATION_PADDING]; +} __attribute__((packed)); + +enum lttng_kernel_key_token_type { + LTTNG_KERNEL_KEY_TOKEN_STRING = 0, /* arg: string_ptr. */ + LTTNG_KERNEL_KEY_TOKEN_EVENT_NAME = 1, /* no arg. */ + LTTNG_KERNEL_KEY_TOKEN_PROVIDER_NAME = 2, /* no arg. */ +}; + +#define LTTNG_KERNEL_KEY_ARG_PADDING1 60 +#define LTTNG_KERNEL_KEY_TOKEN_STRING_LEN_MAX 256 +struct lttng_kernel_key_token { + uint32_t type; /* enum lttng_kernel_key_token_type */ + union { + uint64_t string_ptr; + char padding[LTTNG_KERNEL_KEY_ARG_PADDING1]; + } arg; +} __attribute__((packed)); + +#define LTTNG_KERNEL_NR_KEY_TOKEN 4 +struct lttng_kernel_counter_key_dimension { + uint32_t nr_key_tokens; + struct lttng_kernel_key_token key_tokens[LTTNG_KERNEL_NR_KEY_TOKEN]; +} __attribute__((packed)); + +struct lttng_kernel_counter_key { + uint32_t nr_dimensions; + struct lttng_kernel_counter_key_dimension key_dimensions[LTTNG_KERNEL_COUNTER_DIMENSION_MAX]; +} __attribute__((packed)); + +#define LTTNG_KERNEL_COUNTER_EVENT_PADDING1 16 +struct lttng_kernel_counter_event { + struct lttng_kernel_event event; + struct lttng_kernel_counter_key key; + char padding[LTTNG_KERNEL_COUNTER_EVENT_PADDING1]; +} __attribute__((packed)); enum lttng_kernel_counter_arithmetic { LTTNG_KERNEL_COUNTER_ARITHMETIC_MODULAR = 0, @@ -208,11 +248,17 @@ struct lttng_kernel_counter_clear { char padding[LTTNG_KERNEL_COUNTER_CLEAR_PADDING]; } __attribute__((packed)); -#define LTTNG_KERNEL_EVENT_NOTIFIER_NOTIFICATION_PADDING 32 -struct lttng_kernel_event_notifier_notification { - uint64_t token; - uint16_t capture_buf_size; - char padding[LTTNG_KERNEL_EVENT_NOTIFIER_NOTIFICATION_PADDING]; +#define LTTNG_KERNEL_COUNTER_KEY_LEN 256 +#define LTTNG_KERNEL_COUNTER_MAP_DESCRIPTOR_PADDING 32 +struct lttng_kernel_counter_map_descriptor { + uint64_t descriptor_index; /* input. [ 0 .. nr_descriptors - 1 ] */ + + uint32_t dimension; /* outputs */ + uint64_t array_index; + uint64_t user_token; + char key[LTTNG_KERNEL_COUNTER_KEY_LEN]; + + char padding[LTTNG_KERNEL_COUNTER_MAP_DESCRIPTOR_PADDING]; } __attribute__((packed)); struct lttng_kernel_tracer_version { @@ -385,6 +431,7 @@ struct lttng_kernel_tracker_args { #define LTTNG_KERNEL_STREAM _IO(0xF6, 0x62) #define LTTNG_KERNEL_EVENT \ _IOW(0xF6, 0x63, struct lttng_kernel_event) +/* LTTNG_KERNEL_SYSCALL_MASK applies to both channel and counter fds. */ #define LTTNG_KERNEL_SYSCALL_MASK \ _IOWR(0xF6, 0x64, struct lttng_kernel_syscall_mask) @@ -430,7 +477,12 @@ struct lttng_kernel_tracker_args { _IOWR(0xF6, 0xC1, struct lttng_kernel_counter_aggregate) #define LTTNG_KERNEL_COUNTER_CLEAR \ _IOW(0xF6, 0xC2, struct lttng_kernel_counter_clear) - +#define LTTNG_KERNEL_COUNTER_MAP_NR_DESCRIPTORS \ + _IOR(0xF6, 0xC3, uint64_t) +#define LTTNG_KERNEL_COUNTER_MAP_DESCRIPTOR \ + _IOWR(0xF6, 0xC4, struct lttng_kernel_counter_map_descriptor) +#define LTTNG_KERNEL_COUNTER_EVENT \ + _IOW(0xF6, 0xC5, struct lttng_kernel_counter_event) /* * LTTng-specific ioctls for the lib ringbuffer. diff --git a/include/lttng/event-notifier-notification.h b/include/lttng/event-notifier-notification.h index b044e2fa..160b2542 100644 --- a/include/lttng/event-notifier-notification.h +++ b/include/lttng/event-notifier-notification.h @@ -12,6 +12,7 @@ void lttng_event_notifier_notification_send(struct lttng_event_notifier *event_notifier, struct lttng_probe_ctx *lttng_probe_ctx, - const char *stack_data); + const char *stack_data, + struct lttng_kernel_notifier_ctx *notif_ctx); #endif /* _LTTNG_EVENT_NOTIFIER_NOTIFICATION_H */ diff --git a/include/lttng/events.h b/include/lttng/events.h index 6aa880f8..ebf981eb 100644 --- a/include/lttng/events.h +++ b/include/lttng/events.h @@ -208,7 +208,7 @@ struct lttng_ctx { struct lttng_event_desc { const char *name; /* lttng-modules name */ const char *kname; /* Linux kernel name (tracepoints) */ - void *probe_callback; + void *probe_callback; /* store-event and count-event probe */ const struct lttng_event_ctx *ctx; /* context */ const struct lttng_event_field *fields; /* event payload */ unsigned int nr_fields; @@ -298,11 +298,13 @@ struct lttng_uprobe_handler { struct lttng_kprobe { struct kprobe kp; char *symbol_name; + uint64_t user_token; }; struct lttng_uprobe { struct inode *inode; struct list_head head; + uint64_t user_token; }; enum lttng_syscall_entryexit { @@ -315,14 +317,47 @@ enum lttng_syscall_abi { LTTNG_SYSCALL_ABI_COMPAT, }; +struct lttng_syscall { + struct list_head node; /* chain registered syscall trigger */ + unsigned int syscall_id; + bool is_compat; +}; + +enum lttng_key_token_type { + LTTNG_KEY_TOKEN_STRING = 0, + LTTNG_KEY_TOKEN_EVENT_NAME = 1, +}; + +#define LTTNG_KEY_TOKEN_STRING_LEN_MAX LTTNG_KERNEL_KEY_TOKEN_STRING_LEN_MAX +struct lttng_key_token { + enum lttng_key_token_type type; + union { + char string[LTTNG_KEY_TOKEN_STRING_LEN_MAX]; + } arg; +}; + +#define LTTNG_NR_KEY_TOKEN LTTNG_KERNEL_NR_KEY_TOKEN +struct lttng_counter_key_dimension { + size_t nr_key_tokens; + struct lttng_key_token key_tokens[LTTNG_KERNEL_NR_KEY_TOKEN]; +}; + +#define LTTNG_COUNTER_DIMENSION_MAX LTTNG_KERNEL_COUNTER_DIMENSION_MAX +struct lttng_counter_key { + size_t nr_dimensions; + struct lttng_counter_key_dimension key_dimensions[LTTNG_COUNTER_DIMENSION_MAX]; +}; + +#define LTTNG_KEY_TOKEN_STRING_LEN_MAX LTTNG_KERNEL_KEY_TOKEN_STRING_LEN_MAX + /* * lttng_event structure is referred to by the tracing fast path. It must be * kept small. */ struct lttng_event { enum lttng_event_type evtype; /* First field. */ - unsigned int id; - struct lttng_channel *chan; + struct lttng_event_container *container; + size_t id; int enabled; const struct lttng_event_desc *desc; void *filter; @@ -333,6 +368,7 @@ struct lttng_event { struct { struct lttng_krp *lttng_krp; char *symbol_name; + uint64_t user_token; } kretprobe; struct lttng_uprobe uprobe; struct { @@ -346,11 +382,27 @@ struct lttng_event { /* Backward references: list of lttng_enabler_ref (ref to enablers) */ struct list_head enablers_ref_head; - struct hlist_node hlist; /* session ht of events */ + struct hlist_node name_hlist; /* session ht of events, per event name */ + struct hlist_node key_hlist; /* session ht of events, per key */ int registered; /* has reg'd tracepoint probe */ /* list of struct lttng_bytecode_runtime, sorted by seqnum */ struct list_head filter_bytecode_runtime_head; int has_enablers_without_bytecode; + + /* List node of actions to perform (for syscall events). */ + struct hlist_node action_list; + + char key[LTTNG_KEY_TOKEN_STRING_LEN_MAX]; + + /* + * For non-coalesce-hit event containers, each event is + * associated with a single event enabler user_token. + */ + uint64_t user_token; +}; + +struct lttng_kernel_notifier_ctx { + int eval_capture; }; // FIXME: Really similar to lttng_event above. Could those be merged ? @@ -385,10 +437,12 @@ struct lttng_event_notifier { size_t num_captures; struct list_head capture_bytecode_runtime_head; int has_enablers_without_bytecode; + int eval_capture; /* Should evaluate capture */ void (*send_notification)(struct lttng_event_notifier *event_notifier, struct lttng_probe_ctx *lttng_probe_ctx, - const char *interpreter_stack_data); + const char *interpreter_stack_data, + struct lttng_kernel_notifier_ctx *notif_ctx); struct lttng_event_notifier_group *group; /* Weak ref */ }; @@ -418,7 +472,8 @@ struct lttng_enabler { struct lttng_event_enabler { struct lttng_enabler base; struct list_head node; /* per-session list of enablers */ - struct lttng_channel *chan; + struct lttng_event_container *container; + struct lttng_counter_key key; /* * Unused, but kept around to make it explicit that the tracer can do * it. @@ -567,35 +622,60 @@ struct lttng_event_notifier_ht { struct hlist_head table[LTTNG_EVENT_NOTIFIER_HT_SIZE]; }; -struct lttng_channel { - unsigned int id; - struct channel *chan; /* Channel buffers */ +enum lttng_event_container_type { + LTTNG_EVENT_CONTAINER_CHANNEL, + LTTNG_EVENT_CONTAINER_COUNTER, +}; + +/* + * An event can be contained within either a channel or a counter. + */ +struct lttng_event_container { + enum lttng_event_container_type type; + + struct file *file; /* File associated to event container */ + struct lttng_session *session; /* Session containing the container */ int enabled; - struct lttng_ctx *ctx; - /* Event ID management */ - struct lttng_session *session; - struct file *file; /* File associated to channel */ - unsigned int free_event_id; /* Next event ID to allocate */ - struct list_head list; /* Channel list */ - struct lttng_channel_ops *ops; - struct lttng_transport *transport; - struct hlist_head *sc_table; /* for syscall tracing */ + struct hlist_head *sc_table; /* for syscall tracing */ struct hlist_head *compat_sc_table; struct hlist_head *sc_exit_table; /* for syscall exit tracing */ struct hlist_head *compat_sc_exit_table; - struct hlist_head sc_unknown; /* for unknown syscalls */ + + /* + * Combining all unknown syscall events works as long as they + * are only matched by "all" syscalls enablers, but will require + * a design change when we allow matching by syscall number, for + * instance by allocating sc_tables accomodating NR_syscalls + * entries. + */ + struct hlist_head sc_unknown; /* for unknown syscalls */ struct hlist_head sc_compat_unknown; struct hlist_head sc_exit_unknown; struct hlist_head compat_sc_exit_unknown; + struct lttng_syscall_filter *sc_filter; - int header_type; /* 0: unset, 1: compact, 2: large */ - enum channel_type channel_type; int syscall_all_entry; int syscall_all_exit; - unsigned int metadata_dumped:1, - sys_enter_registered:1, + unsigned int sys_enter_registered:1, sys_exit_registered:1, - tstate:1; /* Transient enable state */ + tstate:1; /* Transient enable state */ + bool coalesce_hits; +}; + +struct lttng_channel { + unsigned int id; + struct channel *chan; /* Channel buffers */ + struct lttng_ctx *ctx; + /* Event ID management */ + unsigned int free_event_id; /* Next event ID to allocate */ + struct list_head list; /* Channel list */ + struct lttng_channel_ops *ops; + struct lttng_transport *transport; + int header_type; /* 0: unset, 1: compact, 2: large */ + enum channel_type channel_type; + unsigned int metadata_dumped:1; + + struct lttng_event_container parent; }; struct lttng_metadata_stream { @@ -659,6 +739,7 @@ struct lttng_session { struct file *file; /* File associated to session */ struct list_head chan; /* Channel list head */ struct list_head events; /* Event list head */ + struct list_head counters; /* Counters list head */ struct list_head list; /* Session list */ unsigned int free_chan_id; /* Next chan ID to allocate */ uuid_le uuid; /* Trace session unique ID */ @@ -673,18 +754,38 @@ struct lttng_session { tstate:1; /* Transient enable state */ /* List of event enablers */ struct list_head enablers_head; - /* Hash table of events */ - struct lttng_event_ht events_ht; + /* Hash table of events indexed by event name */ + struct lttng_event_ht events_name_ht; + /* Hash table of events indexed by key */ + struct lttng_event_ht events_key_ht; char name[LTTNG_KERNEL_SESSION_NAME_LEN]; char creation_time[LTTNG_KERNEL_SESSION_CREATION_TIME_ISO8601_LEN]; }; +struct lttng_counter_map_descriptor { + uint64_t user_token; + size_t array_index; + char key[LTTNG_KERNEL_COUNTER_KEY_LEN]; +}; + +struct lttng_counter_map { + struct lttng_counter_map_descriptor *descriptors; + size_t nr_descriptors; + size_t alloc_len; + struct mutex lock; /* counter map lock */ +}; + struct lttng_counter { - struct file *file; /* File associated to counter. */ struct file *owner; struct lttng_counter_transport *transport; struct lib_counter *counter; struct lttng_counter_ops *ops; + struct list_head node; /* Counter list (in session) */ + + size_t free_index; /* Next index to allocate */ + struct lttng_counter_map map; + + struct lttng_event_container parent; }; struct lttng_event_notifier_group { @@ -737,6 +838,34 @@ struct lttng_metadata_cache { uint64_t version; /* Current version of the metadata */ }; +static inline +struct lttng_event_container *lttng_channel_get_event_container(struct lttng_channel *channel) +{ + return &channel->parent; +} + +static inline +struct lttng_event_container *lttng_counter_get_event_container(struct lttng_counter *counter) +{ + return &counter->parent; +} + +static inline +struct lttng_channel *lttng_event_container_get_channel(struct lttng_event_container *container) +{ + if (container->type != LTTNG_EVENT_CONTAINER_CHANNEL) + return NULL; + return container_of(container, struct lttng_channel, parent); +} + +static inline +struct lttng_counter *lttng_event_container_get_counter(struct lttng_event_container *container) +{ + if (container->type != LTTNG_EVENT_CONTAINER_COUNTER) + return NULL; + return container_of(container, struct lttng_counter, parent); +} + void lttng_lock_sessions(void); void lttng_unlock_sessions(void); @@ -745,7 +874,8 @@ struct list_head *lttng_get_probe_list_head(void); struct lttng_event_enabler *lttng_event_enabler_create( enum lttng_enabler_format_type format_type, struct lttng_kernel_event *event_param, - struct lttng_channel *chan); + const struct lttng_counter_key *key, + struct lttng_event_container *container); int lttng_event_enabler_enable(struct lttng_event_enabler *event_enabler); int lttng_event_enabler_disable(struct lttng_event_enabler *event_enabler); @@ -771,9 +901,6 @@ int lttng_session_metadata_regenerate(struct lttng_session *session); int lttng_session_statedump(struct lttng_session *session); void metadata_cache_destroy(struct kref *kref); -struct lttng_counter *lttng_kernel_counter_create( - const char *counter_transport_name, size_t number_dimensions, - const size_t *dimensions_sizes); int lttng_kernel_counter_read(struct lttng_counter *counter, const size_t *dimension_indexes, int32_t cpu, int64_t *val, bool *overflow, bool *underflow); @@ -788,8 +915,13 @@ struct lttng_event_notifier_group *lttng_event_notifier_group_create(void); int lttng_event_notifier_group_create_error_counter( struct file *event_notifier_group_file, const struct lttng_kernel_counter_conf *error_counter_conf); + void lttng_event_notifier_group_destroy( struct lttng_event_notifier_group *event_notifier_group); +int lttng_event_notifier_group_set_error_counter( + struct lttng_event_notifier_group *event_notifier_group, + const char *counter_transport_name, + size_t counter_len); struct lttng_channel *lttng_channel_create(struct lttng_session *session, const char *transport_name, @@ -798,27 +930,27 @@ struct lttng_channel *lttng_channel_create(struct lttng_session *session, unsigned int switch_timer_interval, unsigned int read_timer_interval, enum channel_type channel_type); -struct lttng_channel *lttng_global_channel_create(struct lttng_session *session, - int overwrite, void *buf_addr, - size_t subbuf_size, size_t num_subbuf, - unsigned int switch_timer_interval, - unsigned int read_timer_interval); void lttng_metadata_channel_destroy(struct lttng_channel *chan); -struct lttng_event *lttng_event_create(struct lttng_channel *chan, +struct lttng_event *lttng_event_create(struct lttng_event_container *container, struct lttng_kernel_event *event_param, + const struct lttng_counter_key *key, void *filter, const struct lttng_event_desc *event_desc, - enum lttng_kernel_instrumentation itype); -struct lttng_event *_lttng_event_create(struct lttng_channel *chan, + enum lttng_kernel_instrumentation itype, + uint64_t user_token); +struct lttng_event *_lttng_event_create(struct lttng_event_container *container, struct lttng_kernel_event *event_param, + const struct lttng_counter_key *key, void *filter, const struct lttng_event_desc *event_desc, - enum lttng_kernel_instrumentation itype); -struct lttng_event *lttng_event_compat_old_create(struct lttng_channel *chan, - struct lttng_kernel_old_event *old_event_param, - void *filter, - const struct lttng_event_desc *internal_desc); + enum lttng_kernel_instrumentation itype, + uint64_t user_token); +struct lttng_counter *lttng_session_create_counter( + struct lttng_session *session, + const char *counter_transport_name, + size_t number_dimensions, const size_t *dimensions_sizes, + bool coalesce_hits); struct lttng_event_notifier *lttng_event_notifier_create( const struct lttng_event_desc *event_notifier_desc, @@ -837,8 +969,8 @@ struct lttng_event_notifier *_lttng_event_notifier_create( void *filter, enum lttng_kernel_instrumentation itype); -int lttng_channel_enable(struct lttng_channel *channel); -int lttng_channel_disable(struct lttng_channel *channel); +int lttng_event_container_enable(struct lttng_event_container *container); +int lttng_event_container_disable(struct lttng_event_container *container); int lttng_event_enable(struct lttng_event *event); int lttng_event_disable(struct lttng_event *event); @@ -890,16 +1022,13 @@ int lttng_desc_match_enabler(const struct lttng_event_desc *desc, #if defined(CONFIG_HAVE_SYSCALL_TRACEPOINTS) int lttng_syscalls_register_event(struct lttng_event_enabler *event_enabler, void *filter); -int lttng_syscalls_unregister_channel(struct lttng_channel *chan); -int lttng_syscalls_destroy_event(struct lttng_channel *chan); -int lttng_syscall_filter_enable_event( - struct lttng_channel *chan, +int lttng_syscalls_unregister_event_container(struct lttng_event_container *event_container); +int lttng_syscalls_destroy_event_container(struct lttng_event_container *container); +int lttng_syscall_filter_enable_event(struct lttng_event_container *container, struct lttng_event *event); -int lttng_syscall_filter_disable_event( - struct lttng_channel *chan, +int lttng_syscall_filter_disable_event(struct lttng_event_container *container, struct lttng_event *event); - -long lttng_channel_syscall_mask(struct lttng_channel *channel, +long lttng_event_container_syscall_mask(struct lttng_event_container *container, struct lttng_kernel_syscall_mask __user *usyscall_mask); int lttng_syscalls_register_event_notifier( @@ -917,29 +1046,29 @@ static inline int lttng_syscalls_register_event( return -ENOSYS; } -static inline int lttng_syscalls_unregister_channel(struct lttng_channel *chan) +static inline int lttng_syscalls_unregister_event_container(struct lttng_event_container *event_container) { return 0; } -static inline int lttng_syscalls_destroy(struct lttng_channel *chan) +static inline int lttng_syscalls_destroy_event_container(struct lttng_event_container *event_container) { return 0; } -static inline int lttng_syscall_filter_enable_event(struct lttng_channel *chan, +static inline int lttng_syscall_filter_enable_event(struct lttng_event_container *event_container, struct lttng_event *event); { return -ENOSYS; } -static inline int lttng_syscall_filter_disable_event(struct lttng_channel *chan, +static inline int lttng_syscall_filter_disable_event(struct lttng_event_container *event_container, struct lttng_event *event); { return -ENOSYS; } -static inline long lttng_channel_syscall_mask(struct lttng_channel *channel, +static inline long lttng_channel_syscall_mask(struct lttng_event_container *event_container, struct lttng_kernel_syscall_mask __user *usyscall_mask) { return -ENOSYS; diff --git a/include/lttng/tracepoint-event-impl.h b/include/lttng/tracepoint-event-impl.h index 079d7d37..d659a6d6 100644 --- a/include/lttng/tracepoint-event-impl.h +++ b/include/lttng/tracepoint-event-impl.h @@ -496,32 +496,12 @@ void __event_notifier_template_proto___##_name(void); #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code_pre, _fields, _code_post) \ -static void __event_probe__##_name(void *__data, _proto); - -#undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS -#define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, _fields, _code_post) \ -static void __event_probe__##_name(void *__data); - -#include TRACE_INCLUDE(TRACE_INCLUDE_FILE) - -/* - * Stage 3.1 of the trace event_notifiers. - * - * Create event_notifier probe callback prototypes. - */ - -/* Reset all macros within TRACEPOINT_EVENT */ -#include - -#undef TP_PROTO -#define TP_PROTO(...) __VA_ARGS__ - -#undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE -#define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code_pre, _fields, _code_post) \ +static void __event_probe__##_name(void *__data, _proto); \ static void __event_notifier_probe__##_name(void *__data, _proto); #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, _fields, _code_post) \ +static void __event_probe__##_name(void *__data); \ static void __event_notifier_probe__##_name(void *__data); #include TRACE_INCLUDE(TRACE_INCLUDE_FILE) @@ -1193,11 +1173,9 @@ static void __event_probe__##_name(void *__data, _proto) \ .event_notifier = NULL, \ .interruptible = !irqs_disabled(), \ }; \ - struct lttng_channel *__chan = __event->chan; \ - struct lttng_session *__session = __chan->session; \ + struct lttng_event_container *__container = __event->container; \ + struct lttng_session *__session = __container->session; \ struct lib_ring_buffer_ctx __ctx; \ - ssize_t __event_len; \ - size_t __event_align; \ size_t __orig_dynamic_len_offset, __dynamic_len_idx __attribute__((unused)); \ union { \ size_t __dynamic_len_removed[ARRAY_SIZE(__event_fields___##_name)]; \ @@ -1213,7 +1191,7 @@ static void __event_probe__##_name(void *__data, _proto) \ return; \ if (unlikely(!LTTNG_READ_ONCE(__session->active))) \ return; \ - if (unlikely(!LTTNG_READ_ONCE(__chan->enabled))) \ + if (unlikely(!LTTNG_READ_ONCE(__container->enabled))) \ return; \ if (unlikely(!LTTNG_READ_ONCE(__event->enabled))) \ return; \ @@ -1258,19 +1236,37 @@ static void __event_probe__##_name(void *__data, _proto) \ if (likely(!__filter_record)) \ goto __post; \ } \ - __event_len = __event_get_size__##_name(tp_locvar, _args); \ - if (unlikely(__event_len < 0)) { \ - lib_ring_buffer_lost_event_too_big(__chan->chan); \ - goto __post; \ + switch (__container->type) { \ + case LTTNG_EVENT_CONTAINER_CHANNEL: \ + { \ + struct lttng_channel *__chan = lttng_event_container_get_channel(__container); \ + ssize_t __event_len; \ + size_t __event_align; \ + \ + __event_len = __event_get_size__##_name(tp_locvar, _args); \ + if (unlikely(__event_len < 0)) { \ + lib_ring_buffer_lost_event_too_big(__chan->chan); \ + goto __post; \ + } \ + __event_align = __event_get_align__##_name(tp_locvar, _args); \ + lib_ring_buffer_ctx_init(&__ctx, __chan->chan, &__lttng_probe_ctx, __event_len, \ + __event_align, -1); \ + __ret = __chan->ops->event_reserve(&__ctx, __event->id); \ + if (__ret < 0) \ + goto __post; \ + _fields \ + __chan->ops->event_commit(&__ctx); \ + break; \ + } \ + case LTTNG_EVENT_CONTAINER_COUNTER: \ + { \ + struct lttng_counter *__counter = lttng_event_container_get_counter(__container); \ + size_t __index = __event->id; \ + \ + (void) __counter->ops->counter_add(__counter->counter, &__index, 1); \ + break; \ + } \ } \ - __event_align = __event_get_align__##_name(tp_locvar, _args); \ - lib_ring_buffer_ctx_init(&__ctx, __chan->chan, &__lttng_probe_ctx, __event_len, \ - __event_align, -1); \ - __ret = __chan->ops->event_reserve(&__ctx, __event->id); \ - if (__ret < 0) \ - goto __post; \ - _fields \ - __chan->ops->event_commit(&__ctx); \ __post: \ _code_post \ barrier(); /* use before un-reserve. */ \ @@ -1289,11 +1285,8 @@ static void __event_probe__##_name(void *__data) \ .event_notifier = NULL, \ .interruptible = !irqs_disabled(), \ }; \ - struct lttng_channel *__chan = __event->chan; \ - struct lttng_session *__session = __chan->session; \ - struct lib_ring_buffer_ctx __ctx; \ - ssize_t __event_len; \ - size_t __event_align; \ + struct lttng_event_container *__container = __event->container; \ + struct lttng_session *__session = __container->session; \ size_t __orig_dynamic_len_offset, __dynamic_len_idx __attribute__((unused)); \ union { \ size_t __dynamic_len_removed[ARRAY_SIZE(__event_fields___##_name)]; \ @@ -1309,7 +1302,7 @@ static void __event_probe__##_name(void *__data) \ return; \ if (unlikely(!LTTNG_READ_ONCE(__session->active))) \ return; \ - if (unlikely(!LTTNG_READ_ONCE(__chan->enabled))) \ + if (unlikely(!LTTNG_READ_ONCE(__container->enabled))) \ return; \ if (unlikely(!LTTNG_READ_ONCE(__event->enabled))) \ return; \ @@ -1338,7 +1331,7 @@ static void __event_probe__##_name(void *__data) \ __orig_dynamic_len_offset = this_cpu_ptr(<tng_dynamic_len_stack)->offset; \ __dynamic_len_idx = __orig_dynamic_len_offset; \ _code_pre \ - if (unlikely(!list_empty(&__event->filter_bytecode_runtime_head))) { \ + if (unlikely(!list_empty(&__event->filter_bytecode_runtime_head))) { \ struct lttng_bytecode_runtime *bc_runtime; \ int __filter_record = __event->has_enablers_without_bytecode; \ \ @@ -1354,19 +1347,38 @@ static void __event_probe__##_name(void *__data) \ if (likely(!__filter_record)) \ goto __post; \ } \ - __event_len = __event_get_size__##_name(tp_locvar); \ - if (unlikely(__event_len < 0)) { \ - lib_ring_buffer_lost_event_too_big(__chan->chan); \ - goto __post; \ + switch (__container->type) { \ + case LTTNG_EVENT_CONTAINER_CHANNEL: \ + { \ + struct lttng_channel *__chan = lttng_event_container_get_channel(__container); \ + struct lib_ring_buffer_ctx __ctx; \ + ssize_t __event_len; \ + size_t __event_align; \ + \ + __event_len = __event_get_size__##_name(tp_locvar); \ + if (unlikely(__event_len < 0)) { \ + lib_ring_buffer_lost_event_too_big(__chan->chan); \ + goto __post; \ + } \ + __event_align = __event_get_align__##_name(tp_locvar); \ + lib_ring_buffer_ctx_init(&__ctx, __chan->chan, &__lttng_probe_ctx, __event_len, \ + __event_align, -1); \ + __ret = __chan->ops->event_reserve(&__ctx, __event->id); \ + if (__ret < 0) \ + goto __post; \ + _fields \ + __chan->ops->event_commit(&__ctx); \ + break; \ + } \ + case LTTNG_EVENT_CONTAINER_COUNTER: \ + { \ + struct lttng_counter *__counter = lttng_event_container_get_counter(__container); \ + size_t __index = __event->id; \ + \ + (void) __counter->ops->counter_add(__counter->counter, &__index, 1); \ + break; \ + } \ } \ - __event_align = __event_get_align__##_name(tp_locvar); \ - lib_ring_buffer_ctx_init(&__ctx, __chan->chan, &__lttng_probe_ctx, __event_len, \ - __event_align, -1); \ - __ret = __chan->ops->event_reserve(&__ctx, __event->id); \ - if (__ret < 0) \ - goto __post; \ - _fields \ - __chan->ops->event_commit(&__ctx); \ __post: \ _code_post \ barrier(); /* use before un-reserve. */ \ @@ -1378,6 +1390,7 @@ __post: \ #undef __get_dynamic_len + /* * Stage 6.1 of tracepoint generation: generate event notifier probes * @@ -1430,6 +1443,8 @@ static void __event_notifier_probe__##_name(void *__data, _proto) \ struct probe_local_vars __tp_locvar; \ struct probe_local_vars *tp_locvar __attribute__((unused)) = \ &__tp_locvar; \ + struct lttng_kernel_notifier_ctx __notif_ctx; \ + bool __interpreter_stack_prepared = false; \ \ if (unlikely(!READ_ONCE(__event_notifier->enabled))) \ return; \ @@ -1440,6 +1455,7 @@ static void __event_notifier_probe__##_name(void *__data, _proto) \ \ __event_prepare_interpreter_stack__##_name(__stackvar.__interpreter_stack_data, \ tp_locvar, _args); \ + __interpreter_stack_prepared = true; \ lttng_list_for_each_entry_rcu(bc_runtime, &__event_notifier->filter_bytecode_runtime_head, node) { \ if (unlikely(bc_runtime->interpreter_funcs.filter(bc_runtime, &__lttng_probe_ctx, \ __stackvar.__interpreter_stack_data) & LTTNG_INTERPRETER_RECORD_FLAG)) \ @@ -1449,14 +1465,16 @@ static void __event_notifier_probe__##_name(void *__data, _proto) \ goto __post; \ } \ \ - if (unlikely(!list_empty(&__event_notifier->capture_bytecode_runtime_head))) \ + __notif_ctx.eval_capture = LTTNG_READ_ONCE(__event_notifier->eval_capture); \ + if (unlikely(!__interpreter_stack_prepared && __notif_ctx.eval_capture)) \ __event_prepare_interpreter_stack__##_name( \ __stackvar.__interpreter_stack_data, \ tp_locvar, _args); \ \ __event_notifier->send_notification(__event_notifier, \ &__lttng_probe_ctx, \ - __stackvar.__interpreter_stack_data); \ + __stackvar.__interpreter_stack_data, \ + &__notif_ctx); \ \ __post: \ _code_post \ @@ -1481,6 +1499,8 @@ static void __event_notifier_probe__##_name(void *__data) \ struct probe_local_vars __tp_locvar; \ struct probe_local_vars *tp_locvar __attribute__((unused)) = \ &__tp_locvar; \ + struct lttng_kernel_notifier_ctx __notif_ctx; \ + bool __interpreter_stack_prepared = false; \ \ if (unlikely(!READ_ONCE(__event_notifier->enabled))) \ return; \ @@ -1491,6 +1511,7 @@ static void __event_notifier_probe__##_name(void *__data) \ \ __event_prepare_interpreter_stack__##_name(__stackvar.__interpreter_stack_data, \ tp_locvar); \ + __interpreter_stack_prepared = true; \ lttng_list_for_each_entry_rcu(bc_runtime, &__event_notifier->filter_bytecode_runtime_head, node) { \ if (unlikely(bc_runtime->interpreter_funcs.filter(bc_runtime, &__lttng_probe_ctx, \ __stackvar.__interpreter_stack_data) & LTTNG_INTERPRETER_RECORD_FLAG)) \ @@ -1500,14 +1521,16 @@ static void __event_notifier_probe__##_name(void *__data) \ goto __post; \ } \ \ - if (unlikely(!list_empty(&__event_notifier->capture_bytecode_runtime_head))) \ + __notif_ctx.eval_capture = LTTNG_READ_ONCE(__event_notifier->eval_capture); \ + if (unlikely(!__interpreter_stack_prepared && __notif_ctx.eval_capture)) \ __event_prepare_interpreter_stack__##_name( \ __stackvar.__interpreter_stack_data, \ tp_locvar); \ \ __event_notifier->send_notification(__event_notifier, \ &__lttng_probe_ctx, \ - __stackvar.__interpreter_stack_data); \ + __stackvar.__interpreter_stack_data, \ + &__notif_ctx); \ __post: \ _code_post \ return; \ diff --git a/src/lttng-abi.c b/src/lttng-abi.c index 4d053985..f65caf93 100644 --- a/src/lttng-abi.c +++ b/src/lttng-abi.c @@ -490,6 +490,7 @@ int lttng_abi_create_channel(struct file *session_file, const struct file_operations *fops = NULL; const char *transport_name; struct lttng_channel *chan; + struct lttng_event_container *container; struct file *chan_file; int chan_fd; int ret = 0; @@ -557,7 +558,8 @@ int lttng_abi_create_channel(struct file *session_file, ret = -EINVAL; goto chan_error; } - chan->file = chan_file; + container = lttng_channel_get_event_container(chan); + container->file = chan_file; chan_file->private_data = chan; fd_install(chan_fd, chan_file); @@ -607,6 +609,139 @@ int lttng_abi_session_set_creation_time(struct lttng_session *session, return 0; } +static +int lttng_abi_validate_event_param(struct lttng_kernel_event *event_param) +{ + /* Limit ABI to implemented features. */ + switch (event_param->instrumentation) { + case LTTNG_KERNEL_SYSCALL: + switch (event_param->u.syscall.entryexit) { + case LTTNG_KERNEL_SYSCALL_ENTRY: + case LTTNG_KERNEL_SYSCALL_EXIT: + case LTTNG_KERNEL_SYSCALL_ENTRYEXIT: + break; + default: + return -EINVAL; + } + switch (event_param->u.syscall.abi) { + case LTTNG_KERNEL_SYSCALL_ABI_ALL: + break; + default: + return -EINVAL; + } + switch (event_param->u.syscall.match) { + case LTTNG_KERNEL_SYSCALL_MATCH_NAME: + break; + default: + return -EINVAL; + } + break; + + case LTTNG_KERNEL_TRACEPOINT: /* Fallthrough */ + case LTTNG_KERNEL_KPROBE: /* Fallthrough */ + case LTTNG_KERNEL_KRETPROBE: /* Fallthrough */ + case LTTNG_KERNEL_NOOP: /* Fallthrough */ + case LTTNG_KERNEL_UPROBE: + break; + + case LTTNG_KERNEL_FUNCTION: /* Fallthrough */ + default: + return -EINVAL; + } + return 0; +} + +static +int lttng_abi_create_event(struct file *event_container_file, + struct lttng_event_container *container, + struct lttng_kernel_event *event_param, + const struct lttng_counter_key *key) +{ + int event_fd, ret; + struct file *event_file; + void *priv; + + event_param->name[LTTNG_KERNEL_SYM_NAME_LEN - 1] = '\0'; + switch (event_param->instrumentation) { + case LTTNG_KERNEL_KRETPROBE: + event_param->u.kretprobe.symbol_name[LTTNG_KERNEL_SYM_NAME_LEN - 1] = '\0'; + break; + case LTTNG_KERNEL_KPROBE: + event_param->u.kprobe.symbol_name[LTTNG_KERNEL_SYM_NAME_LEN - 1] = '\0'; + break; + case LTTNG_KERNEL_FUNCTION: + WARN_ON_ONCE(1); + /* Not implemented. */ + break; + default: + break; + } + event_fd = lttng_get_unused_fd(); + if (event_fd < 0) { + ret = event_fd; + goto fd_error; + } + event_file = anon_inode_getfile("[lttng_event]", + <tng_event_fops, + NULL, O_RDWR); + if (IS_ERR(event_file)) { + ret = PTR_ERR(event_file); + goto file_error; + } + /* The event holds a reference on the container */ + if (!atomic_long_add_unless(&event_container_file->f_count, 1, LONG_MAX)) { + ret = -EOVERFLOW; + goto refcount_error; + } + ret = lttng_abi_validate_event_param(event_param); + if (ret) + goto event_error; + if (event_param->instrumentation == LTTNG_KERNEL_TRACEPOINT + || event_param->instrumentation == LTTNG_KERNEL_SYSCALL) { + struct lttng_event_enabler *event_enabler; + + if (strutils_is_star_glob_pattern(event_param->name)) { + /* + * If the event name is a star globbing pattern, + * we create the special star globbing enabler. + */ + event_enabler = lttng_event_enabler_create(LTTNG_ENABLER_FORMAT_STAR_GLOB, + event_param, key, container); + } else { + event_enabler = lttng_event_enabler_create(LTTNG_ENABLER_FORMAT_NAME, + event_param, key, container); + } + priv = event_enabler; + } else { + struct lttng_event *event; + + /* + * We tolerate no failure path after event creation. It + * will stay invariant for the rest of the session. + */ + event = lttng_event_create(container, event_param, key, + NULL, NULL, + event_param->instrumentation, event_param->token); + if (IS_ERR(event)) { + ret = PTR_ERR(event); + goto event_error; + } + priv = event; + } + event_file->private_data = priv; + fd_install(event_fd, event_file); + return event_fd; + +event_error: + atomic_long_dec(&event_container_file->f_count); +refcount_error: + fput(event_file); +file_error: + put_unused_fd(event_fd); +fd_error: + return ret; +} + static int lttng_counter_release(struct inode *inode, struct file *file) { @@ -623,10 +758,67 @@ int lttng_counter_release(struct inode *inode, struct file *file) return 0; } +static +int copy_counter_key(struct lttng_counter_key *key, + const struct lttng_kernel_counter_key *ukey) +{ + size_t i, j, nr_dimensions; + + nr_dimensions = ukey->nr_dimensions; + if (nr_dimensions > LTTNG_COUNTER_DIMENSION_MAX) + return -EINVAL; + key->nr_dimensions = nr_dimensions; + for (i = 0; i < nr_dimensions; i++) { + const struct lttng_kernel_counter_key_dimension *udim = + &ukey->key_dimensions[i]; + struct lttng_counter_key_dimension *dim = + &key->key_dimensions[i]; + size_t nr_key_tokens; + + nr_key_tokens = udim->nr_key_tokens; + if (!nr_key_tokens || nr_key_tokens > LTTNG_NR_KEY_TOKEN) + return -EINVAL; + dim->nr_key_tokens = nr_key_tokens; + for (j = 0; j < nr_key_tokens; j++) { + const struct lttng_kernel_key_token *utoken = + &udim->key_tokens[j]; + struct lttng_key_token *token = + &dim->key_tokens[j]; + + switch (utoken->type) { + case LTTNG_KERNEL_KEY_TOKEN_STRING: + { + long ret; + + token->type = LTTNG_KEY_TOKEN_STRING; + ret = strncpy_from_user(token->arg.string, + (char __user *)(unsigned long)utoken->arg.string_ptr, + LTTNG_KEY_TOKEN_STRING_LEN_MAX); + if (ret < 0) + return -EFAULT; + if (!ret || ret == LTTNG_KEY_TOKEN_STRING_LEN_MAX) + return -EINVAL; + break; + } + case LTTNG_KERNEL_KEY_TOKEN_EVENT_NAME: + token->type = LTTNG_KEY_TOKEN_EVENT_NAME; + break; + case LTTNG_KERNEL_KEY_TOKEN_PROVIDER_NAME: + printk(KERN_ERR "LTTng: Provider name token not supported.\n"); + /* Fallthrough */ + default: + return -EINVAL; + } + } + } + return 0; +} + static long lttng_counter_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { struct lttng_counter *counter = file->private_data; + struct lttng_event_container *container = lttng_counter_get_event_container(counter); size_t indexes[LTTNG_KERNEL_COUNTER_DIMENSION_MAX] = { 0 }; int i; @@ -720,6 +912,89 @@ long lttng_counter_ioctl(struct file *file, unsigned int cmd, unsigned long arg) return lttng_kernel_counter_clear(counter, indexes); } + case LTTNG_KERNEL_COUNTER_EVENT: + { + struct lttng_kernel_counter_event *ucounter_event_param; + struct lttng_counter_key *key; + int ret; + + key = kzalloc(sizeof(*key), GFP_KERNEL); + if (!key) + return -ENOMEM; + ucounter_event_param = kzalloc(sizeof(*ucounter_event_param), GFP_KERNEL); + if (!ucounter_event_param) { + ret = -ENOMEM; + goto free_key; + } + if (copy_from_user(ucounter_event_param, + (struct lttng_kernel_counter_event __user *) arg, + sizeof(*ucounter_event_param))) { + ret = -EFAULT; + goto free_param; + } + ret = copy_counter_key(key, &ucounter_event_param->key); + if (ret) + goto free_param; + ret = lttng_abi_create_event(file, container, &ucounter_event_param->event, key); + free_param: + kfree(ucounter_event_param); + free_key: + kfree(key); + return ret; + } + case LTTNG_KERNEL_ENABLE: + return lttng_event_container_enable(container); + case LTTNG_KERNEL_DISABLE: + return lttng_event_container_disable(container); + case LTTNG_KERNEL_SYSCALL_MASK: + return lttng_event_container_syscall_mask(container, + (struct lttng_kernel_syscall_mask __user *) arg); + case LTTNG_KERNEL_COUNTER_MAP_NR_DESCRIPTORS: + { + uint64_t __user *user_nr_descriptors = (uint64_t __user *) arg; + uint64_t nr_descriptors; + + mutex_lock(&counter->map.lock); + nr_descriptors = counter->map.nr_descriptors; + mutex_unlock(&counter->map.lock); + return put_user(nr_descriptors, user_nr_descriptors); + } + case LTTNG_KERNEL_COUNTER_MAP_DESCRIPTOR: + { + struct lttng_kernel_counter_map_descriptor __user *user_descriptor = + (struct lttng_kernel_counter_map_descriptor __user *) arg; + struct lttng_kernel_counter_map_descriptor local_descriptor; + struct lttng_counter_map_descriptor *kernel_descriptor; + int ret; + + if (copy_from_user(&local_descriptor, user_descriptor, + sizeof(local_descriptor))) + return -EFAULT; + if (validate_zeroed_padding(local_descriptor.padding, + sizeof(local_descriptor.padding))) + return -EINVAL; + + mutex_lock(&counter->map.lock); + if (local_descriptor.descriptor_index >= counter->map.nr_descriptors) { + ret = -EOVERFLOW; + goto map_descriptor_error_unlock; + } + kernel_descriptor = &counter->map.descriptors[local_descriptor.descriptor_index]; + local_descriptor.user_token = kernel_descriptor->user_token; + local_descriptor.array_index = kernel_descriptor->array_index; + memcpy(local_descriptor.key, kernel_descriptor->key, LTTNG_KERNEL_COUNTER_KEY_LEN); + mutex_unlock(&counter->map.lock); + + if (copy_to_user(user_descriptor, &local_descriptor, + sizeof(local_descriptor))) + return -EFAULT; + + return 0; + + map_descriptor_error_unlock: + mutex_unlock(&counter->map.lock); + return ret; + } default: WARN_ON_ONCE(1); return -ENOSYS; @@ -735,6 +1010,91 @@ static const struct file_operations lttng_counter_fops = { #endif }; +static +long lttng_abi_session_create_counter( + struct lttng_session *session, + const struct lttng_kernel_counter_conf *counter_conf) +{ + int counter_fd, ret, i; + char *counter_transport_name; + struct lttng_event_container *container; + struct lttng_counter *counter; + struct file *counter_file; + size_t dimension_sizes[LTTNG_KERNEL_COUNTER_DIMENSION_MAX] = { 0 }; + size_t number_dimensions; + + counter_fd = lttng_get_unused_fd(); + if (counter_fd < 0) { + ret = counter_fd; + goto fd_error; + } + + counter_file = anon_inode_getfile("[lttng_counter]", + <tng_counter_fops, + NULL, O_RDONLY); + if (IS_ERR(counter_file)) { + ret = PTR_ERR(counter_file); + goto file_error; + } + + if (counter_conf->arithmetic != LTTNG_KERNEL_COUNTER_ARITHMETIC_MODULAR) { + printk(KERN_ERR "LTTng: Map: Error counter of the wrong type.\n"); + return -EINVAL; + } + + switch (counter_conf->bitness) { + case LTTNG_KERNEL_COUNTER_BITNESS_64: + counter_transport_name = "counter-per-cpu-64-modular"; + break; + case LTTNG_KERNEL_COUNTER_BITNESS_32: + counter_transport_name = "counter-per-cpu-32-modular"; + break; + default: + return -EINVAL; + } + + number_dimensions = (size_t) counter_conf->number_dimensions; + + for (i = 0; i < counter_conf->number_dimensions; i++) { + if (counter_conf->dimensions[i].has_underflow) + return -EINVAL; + if (counter_conf->dimensions[i].has_overflow) + return -EINVAL; + dimension_sizes[i] = counter_conf->dimensions[i].size; + } + + if (!atomic_long_add_unless(&session->file->f_count, 1, LONG_MAX)) { + ret = -EOVERFLOW; + goto refcount_error; + } + + counter = lttng_session_create_counter(session, + counter_transport_name, + number_dimensions, dimension_sizes, + counter_conf->coalesce_hits); + if (!counter) { + ret = -EINVAL; + goto counter_error; + } + + counter->owner = session->file; + container = lttng_counter_get_event_container(counter); + container->file = counter_file; + counter_file->private_data = counter; + + fd_install(counter_fd, counter_file); + + return counter_fd; + +counter_error: + atomic_long_dec(&session->file->f_count); +refcount_error: + fput(counter_file); +file_error: + put_unused_fd(counter_fd); +fd_error: + return ret; +} static enum tracker_type get_tracker_type(struct lttng_kernel_tracker_args *tracker) @@ -925,6 +1285,17 @@ long lttng_session_ioctl(struct file *file, unsigned int cmd, unsigned long arg) return -EFAULT; return lttng_abi_session_set_creation_time(session, &time); } + case LTTNG_KERNEL_COUNTER: + { + struct lttng_kernel_counter_conf ucounter_conf; + + if (copy_from_user(&ucounter_conf, + (struct lttng_kernel_counter_conf __user *) arg, + sizeof(ucounter_conf))) + return -EFAULT; + return lttng_abi_session_create_counter(session, + &ucounter_conf); + } default: return -ENOIOCTLCMD; } @@ -1555,7 +1926,7 @@ const struct file_operations lttng_metadata_ring_buffer_file_operations = { }; static -int lttng_abi_create_stream_fd(struct file *channel_file, void *stream_priv, +int lttng_abi_create_stream_fd(struct file *file, void *stream_priv, const struct file_operations *fops, const char *name) { int stream_fd, ret; @@ -1592,9 +1963,9 @@ fd_error: } static -int lttng_abi_open_stream(struct file *channel_file) +int lttng_abi_open_stream(struct file *file) { - struct lttng_channel *channel = channel_file->private_data; + struct lttng_channel *channel = file->private_data; struct lib_ring_buffer *buf; int ret; void *stream_priv; @@ -1604,7 +1975,7 @@ int lttng_abi_open_stream(struct file *channel_file) return -ENOENT; stream_priv = buf; - ret = lttng_abi_create_stream_fd(channel_file, stream_priv, + ret = lttng_abi_create_stream_fd(file, stream_priv, <tng_stream_ring_buffer_file_operations, "[lttng_stream]"); if (ret < 0) @@ -1618,10 +1989,11 @@ fd_error: } static -int lttng_abi_open_metadata_stream(struct file *channel_file) +int lttng_abi_open_metadata_stream(struct file *file) { - struct lttng_channel *channel = channel_file->private_data; - struct lttng_session *session = channel->session; + struct lttng_channel *channel = file->private_data; + struct lttng_event_container *container = lttng_channel_get_event_container(channel); + struct lttng_session *session = container->session; struct lib_ring_buffer *buf; int ret; struct lttng_metadata_stream *metadata_stream; @@ -1660,7 +2032,7 @@ int lttng_abi_open_metadata_stream(struct file *channel_file) goto kref_error; } - ret = lttng_abi_create_stream_fd(channel_file, stream_priv, + ret = lttng_abi_create_stream_fd(file, stream_priv, <tng_metadata_ring_buffer_file_operations, "[lttng_metadata_stream]"); if (ret < 0) @@ -1718,139 +2090,6 @@ refcount_error: return ret; } -static -int lttng_abi_validate_event_param(struct lttng_kernel_event *event_param) -{ - /* Limit ABI to implemented features. */ - switch (event_param->instrumentation) { - case LTTNG_KERNEL_SYSCALL: - switch (event_param->u.syscall.entryexit) { - case LTTNG_KERNEL_SYSCALL_ENTRY: - case LTTNG_KERNEL_SYSCALL_EXIT: - case LTTNG_KERNEL_SYSCALL_ENTRYEXIT: - break; - default: - return -EINVAL; - } - switch (event_param->u.syscall.abi) { - case LTTNG_KERNEL_SYSCALL_ABI_ALL: - break; - default: - return -EINVAL; - } - switch (event_param->u.syscall.match) { - case LTTNG_KERNEL_SYSCALL_MATCH_NAME: - break; - default: - return -EINVAL; - } - break; - - case LTTNG_KERNEL_TRACEPOINT: /* Fallthrough */ - case LTTNG_KERNEL_KPROBE: /* Fallthrough */ - case LTTNG_KERNEL_KRETPROBE: /* Fallthrough */ - case LTTNG_KERNEL_NOOP: /* Fallthrough */ - case LTTNG_KERNEL_UPROBE: - break; - - case LTTNG_KERNEL_FUNCTION: /* Fallthrough */ - default: - return -EINVAL; - } - return 0; -} - -static -int lttng_abi_create_event(struct file *channel_file, - struct lttng_kernel_event *event_param) -{ - struct lttng_channel *channel = channel_file->private_data; - int event_fd, ret; - struct file *event_file; - void *priv; - - event_param->name[LTTNG_KERNEL_SYM_NAME_LEN - 1] = '\0'; - switch (event_param->instrumentation) { - case LTTNG_KERNEL_KRETPROBE: - event_param->u.kretprobe.symbol_name[LTTNG_KERNEL_SYM_NAME_LEN - 1] = '\0'; - break; - case LTTNG_KERNEL_KPROBE: - event_param->u.kprobe.symbol_name[LTTNG_KERNEL_SYM_NAME_LEN - 1] = '\0'; - break; - case LTTNG_KERNEL_FUNCTION: - WARN_ON_ONCE(1); - /* Not implemented. */ - break; - default: - break; - } - event_fd = lttng_get_unused_fd(); - if (event_fd < 0) { - ret = event_fd; - goto fd_error; - } - event_file = anon_inode_getfile("[lttng_event]", - <tng_event_fops, - NULL, O_RDWR); - if (IS_ERR(event_file)) { - ret = PTR_ERR(event_file); - goto file_error; - } - /* The event holds a reference on the channel */ - if (!atomic_long_add_unless(&channel_file->f_count, 1, LONG_MAX)) { - ret = -EOVERFLOW; - goto refcount_error; - } - ret = lttng_abi_validate_event_param(event_param); - if (ret) - goto event_error; - if (event_param->instrumentation == LTTNG_KERNEL_TRACEPOINT - || event_param->instrumentation == LTTNG_KERNEL_SYSCALL) { - struct lttng_event_enabler *event_enabler; - - if (strutils_is_star_glob_pattern(event_param->name)) { - /* - * If the event name is a star globbing pattern, - * we create the special star globbing enabler. - */ - event_enabler = lttng_event_enabler_create(LTTNG_ENABLER_FORMAT_STAR_GLOB, - event_param, channel); - } else { - event_enabler = lttng_event_enabler_create(LTTNG_ENABLER_FORMAT_NAME, - event_param, channel); - } - priv = event_enabler; - } else { - struct lttng_event *event; - - /* - * We tolerate no failure path after event creation. It - * will stay invariant for the rest of the session. - */ - event = lttng_event_create(channel, event_param, - NULL, NULL, - event_param->instrumentation); - WARN_ON_ONCE(!event); - if (IS_ERR(event)) { - ret = PTR_ERR(event); - goto event_error; - } - priv = event; - } - event_file->private_data = priv; - fd_install(event_fd, event_file); - return event_fd; - -event_error: - atomic_long_dec(&channel_file->f_count); -refcount_error: - fput(event_file); -file_error: - put_unused_fd(event_fd); -fd_error: - return ret; -} - static long lttng_event_notifier_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { @@ -2078,7 +2317,8 @@ long lttng_abi_event_notifier_group_create_error_counter( int counter_fd, ret; char *counter_transport_name; size_t counter_len; - struct lttng_counter *counter = NULL; + struct lttng_counter *counter; + struct lttng_event_container *container; struct file *counter_file; struct lttng_event_notifier_group *event_notifier_group = (struct lttng_event_notifier_group *) event_notifier_group_file->private_data; @@ -2104,19 +2344,6 @@ long lttng_abi_event_notifier_group_create_error_counter( return -EINVAL; } - /* - * Lock sessions to provide mutual exclusion against concurrent - * modification of event_notifier group, which would result in - * overwriting the error counter if set concurrently. - */ - lttng_lock_sessions(); - - if (event_notifier_group->error_counter) { - printk(KERN_ERR "Error counter already created in event_notifier group\n"); - ret = -EBUSY; - goto fd_error; - } - counter_fd = lttng_get_unused_fd(); if (counter_fd < 0) { ret = counter_fd; @@ -2138,29 +2365,18 @@ long lttng_abi_event_notifier_group_create_error_counter( goto refcount_error; } - counter = lttng_kernel_counter_create(counter_transport_name, - 1, &counter_len); - if (!counter) { - ret = -EINVAL; + ret = lttng_event_notifier_group_set_error_counter(event_notifier_group, + counter_transport_name, counter_len); + if (ret) goto counter_error; - } - - event_notifier_group->error_counter_len = counter_len; - /* - * store-release to publish error counter matches load-acquire - * in record_error. Ensures the counter is created and the - * error_counter_len is set before they are used. - */ - lttng_smp_store_release(&event_notifier_group->error_counter, counter); - counter->file = counter_file; + counter = event_notifier_group->error_counter; + container = lttng_counter_get_event_container(counter); + container->file = counter_file; counter->owner = event_notifier_group->file; counter_file->private_data = counter; - /* Ownership transferred. */ - counter = NULL; fd_install(counter_fd, counter_file); - lttng_unlock_sessions(); return counter_fd; @@ -2171,7 +2387,6 @@ refcount_error: file_error: put_unused_fd(counter_fd); fd_error: - lttng_unlock_sessions(); return ret; } @@ -2257,6 +2472,7 @@ static long lttng_channel_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { struct lttng_channel *channel = file->private_data; + struct lttng_event_container *container = lttng_channel_get_event_container(channel); switch (cmd) { case LTTNG_KERNEL_OLD_STREAM: @@ -2319,7 +2535,7 @@ long lttng_channel_ioctl(struct file *file, unsigned int cmd, unsigned long arg) default: break; } - ret = lttng_abi_create_event(file, uevent_param); + ret = lttng_abi_create_event(file, container, uevent_param, NULL); old_event_error_free_old_param: kfree(old_uevent_param); @@ -2336,7 +2552,7 @@ old_event_end: (struct lttng_kernel_event __user *) arg, sizeof(uevent_param))) return -EFAULT; - return lttng_abi_create_event(file, &uevent_param); + return lttng_abi_create_event(file, container, &uevent_param, NULL); } case LTTNG_KERNEL_OLD_CONTEXT: { @@ -2379,7 +2595,7 @@ old_event_end: ret = lttng_abi_add_context(file, ucontext_param, - &channel->ctx, channel->session); + &channel->ctx, container->session); old_ctx_error_free_old_param: kfree(old_ucontext_param); @@ -2398,16 +2614,16 @@ old_ctx_end: return -EFAULT; return lttng_abi_add_context(file, &ucontext_param, - &channel->ctx, channel->session); + &channel->ctx, container->session); } case LTTNG_KERNEL_OLD_ENABLE: case LTTNG_KERNEL_ENABLE: - return lttng_channel_enable(channel); + return lttng_event_container_enable(container); case LTTNG_KERNEL_OLD_DISABLE: case LTTNG_KERNEL_DISABLE: - return lttng_channel_disable(channel); + return lttng_event_container_disable(container); case LTTNG_KERNEL_SYSCALL_MASK: - return lttng_channel_syscall_mask(channel, + return lttng_event_container_syscall_mask(container, (struct lttng_kernel_syscall_mask __user *) arg); default: return -ENOIOCTLCMD; @@ -2470,21 +2686,26 @@ unsigned int lttng_channel_poll(struct file *file, poll_table *wait) static int lttng_channel_release(struct inode *inode, struct file *file) { - struct lttng_channel *channel = file->private_data; + struct lttng_channel *chan = file->private_data; + + if (chan) { + struct lttng_event_container *container = lttng_channel_get_event_container(chan); - if (channel) - fput(channel->session->file); + fput(container->session->file); + } return 0; } static int lttng_metadata_channel_release(struct inode *inode, struct file *file) { - struct lttng_channel *channel = file->private_data; + struct lttng_channel *chan = file->private_data; + + if (chan) { + struct lttng_event_container *container = lttng_channel_get_event_container(chan); - if (channel) { - fput(channel->session->file); - lttng_metadata_channel_destroy(channel); + fput(container->session->file); + lttng_metadata_channel_destroy(chan); } return 0; @@ -2614,12 +2835,12 @@ int lttng_event_release(struct inode *inode, struct file *file) case LTTNG_TYPE_EVENT: event = file->private_data; if (event) - fput(event->chan->file); + fput(event->container->file); break; case LTTNG_TYPE_ENABLER: event_enabler = file->private_data; if (event_enabler) - fput(event_enabler->chan->file); + fput(event_enabler->container->file); break; default: WARN_ON_ONCE(1); diff --git a/src/lttng-event-notifier-notification.c b/src/lttng-event-notifier-notification.c index 52b5593a..b39bfaad 100644 --- a/src/lttng-event-notifier-notification.c +++ b/src/lttng-event-notifier-notification.c @@ -426,7 +426,8 @@ void notification_send(struct lttng_event_notifier_notification *notif, void lttng_event_notifier_notification_send(struct lttng_event_notifier *event_notifier, struct lttng_probe_ctx *lttng_probe_ctx, - const char *stack_data) + const char *stack_data, + struct lttng_kernel_notifier_ctx *notif_ctx) { struct lttng_event_notifier_notification notif = { 0 }; int ret; @@ -440,7 +441,7 @@ void lttng_event_notifier_notification_send(struct lttng_event_notifier *event_n goto end; } - if (unlikely(!list_empty(&event_notifier->capture_bytecode_runtime_head))) { + if (unlikely(notif_ctx->eval_capture)) { struct lttng_bytecode_runtime *capture_bc_runtime; /* @@ -449,7 +450,7 @@ void lttng_event_notifier_notification_send(struct lttng_event_notifier *event_n * `output` parameter to the capture buffer. If the interpreter * fails, append an empty capture to the buffer. */ - list_for_each_entry(capture_bc_runtime, + list_for_each_entry_rcu(capture_bc_runtime, &event_notifier->capture_bytecode_runtime_head, node) { struct lttng_interpreter_output output; diff --git a/src/lttng-events.c b/src/lttng-events.c index d819c9e2..d02c110e 100644 --- a/src/lttng-events.c +++ b/src/lttng-events.c @@ -34,6 +34,7 @@ #include #include #include +#include #include #include #include @@ -45,6 +46,7 @@ #include #include #include +#include #include #define METADATA_CACHE_DEFAULT_SIZE 4096 @@ -68,12 +70,12 @@ static void lttng_event_notifier_group_sync_enablers(struct lttng_event_notifier static void _lttng_event_destroy(struct lttng_event *event); static void _lttng_event_notifier_destroy(struct lttng_event_notifier *event_notifier); -static void _lttng_channel_destroy(struct lttng_channel *chan); +static void _lttng_channel_destroy(struct lttng_channel *channel); +static void _lttng_session_counter_destroy(struct lttng_counter *counter); static int _lttng_event_unregister(struct lttng_event *event); static int _lttng_event_notifier_unregister(struct lttng_event_notifier *event_notifier); static int _lttng_event_metadata_statedump(struct lttng_session *session, - struct lttng_channel *chan, struct lttng_event *event); static int _lttng_session_metadata_statedump(struct lttng_session *session); @@ -88,6 +90,27 @@ int _lttng_field_statedump(struct lttng_session *session, const struct lttng_event_field *field, size_t nesting); +static bool lttng_event_container_is_metadata_channel(struct lttng_event_container *container) +{ + switch (container->type) { + case LTTNG_EVENT_CONTAINER_CHANNEL: + { + struct lttng_channel *chan = lttng_event_container_get_channel(container); + + return chan->channel_type == METADATA_CHANNEL; + } + case LTTNG_EVENT_CONTAINER_COUNTER: + return false; + default: + return false; + } +} + +static bool lttng_event_within_metadata_channel(struct lttng_event *event) +{ + return lttng_event_container_is_metadata_channel(event->container); +} + void synchronize_trace(void) { #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(5,1,0)) @@ -154,6 +177,7 @@ struct lttng_session *lttng_session_create(void) goto err; INIT_LIST_HEAD(&session->chan); INIT_LIST_HEAD(&session->events); + INIT_LIST_HEAD(&session->counters); lttng_guid_gen(&session->uuid); metadata_cache = kzalloc(sizeof(struct lttng_metadata_cache), @@ -172,7 +196,9 @@ struct lttng_session *lttng_session_create(void) sizeof(metadata_cache->uuid)); INIT_LIST_HEAD(&session->enablers_head); for (i = 0; i < LTTNG_EVENT_HT_SIZE; i++) - INIT_HLIST_HEAD(&session->events_ht.table[i]); + INIT_HLIST_HEAD(&session->events_name_ht.table[i]); + for (i = 0; i < LTTNG_EVENT_HT_SIZE; i++) + INIT_HLIST_HEAD(&session->events_key_ht.table[i]); list_add(&session->list, &sessions); session->pid_tracker.session = session; session->pid_tracker.tracker_type = TRACKER_PID; @@ -210,12 +236,15 @@ struct lttng_counter_transport *lttng_counter_transport_find(const char *name) return NULL; } +static struct lttng_counter *lttng_kernel_counter_create( const char *counter_transport_name, - size_t number_dimensions, const size_t *dimensions_sizes) + size_t number_dimensions, const size_t *dimensions_sizes, + bool coalesce_hits) { - struct lttng_counter *counter = NULL; struct lttng_counter_transport *counter_transport = NULL; + struct lttng_counter *counter = NULL; + struct lttng_event_container *container; counter_transport = lttng_counter_transport_find(counter_transport_name); if (!counter_transport) { @@ -231,14 +260,18 @@ struct lttng_counter *lttng_kernel_counter_create( counter = lttng_kvzalloc(sizeof(struct lttng_counter), GFP_KERNEL); if (!counter) goto nomem; - + container = lttng_counter_get_event_container(counter); + container->type = LTTNG_EVENT_CONTAINER_COUNTER; + container->coalesce_hits = coalesce_hits; /* Create event notifier error counter. */ counter->ops = &counter_transport->ops; counter->transport = counter_transport; + mutex_init(&counter->map.lock); counter->counter = counter->ops->counter_create( number_dimensions, dimensions_sizes, 0); if (!counter->counter) { + printk(KERN_WARNING "LTTng: Error creating counter"); goto create_error; } @@ -253,6 +286,60 @@ notransport: return NULL; } +static +void lttng_kernel_counter_destroy(struct lttng_counter *counter) +{ + counter->ops->counter_destroy(counter->counter); + module_put(counter->transport->owner); + lttng_kvfree(counter->map.descriptors); + lttng_kvfree(counter); +} + +int lttng_event_notifier_group_set_error_counter( + struct lttng_event_notifier_group *event_notifier_group, + const char *counter_transport_name, + size_t counter_len) +{ + struct lttng_counter *counter; + int ret; + + /* + * Lock sessions to provide mutual exclusion against concurrent + * modification of trigger group, which would result in + * overwriting the error counter if set concurrently. + */ + mutex_lock(&sessions_mutex); + + if (event_notifier_group->error_counter) { + printk(KERN_ERR "Error counter already set in event notifier group\n"); + ret = -EBUSY; + goto error; + } + + counter = lttng_kernel_counter_create(counter_transport_name, + 1, &counter_len, false); + if (!counter) { + ret = -EINVAL; + goto error; + } + + event_notifier_group->error_counter_len = counter_len; + /* + * store-release to publish error counter matches load-acquire + * in record_error. Ensures the counter is created and the + * error_counter_len is set before they are used. + */ + lttng_smp_store_release(&event_notifier_group->error_counter, + counter); + + mutex_unlock(&sessions_mutex); + return 0; + +error: + mutex_unlock(&sessions_mutex); + return ret; +} + struct lttng_event_notifier_group *lttng_event_notifier_group_create(void) { struct lttng_transport *transport = NULL; @@ -318,6 +405,34 @@ notransport: return NULL; } +struct lttng_counter *lttng_session_create_counter( + struct lttng_session *session, + const char *counter_transport_name, + size_t number_dimensions, const size_t *dimensions_sizes, + bool coalesce_hits) +{ + struct lttng_counter *counter; + struct lttng_event_container *container; + + counter = lttng_kernel_counter_create(counter_transport_name, + number_dimensions, dimensions_sizes, + coalesce_hits); + if (!counter) { + goto counter_error; + } + container = lttng_counter_get_event_container(counter); + + mutex_lock(&sessions_mutex); + container->session = session; + list_add(&counter->node, &session->counters); + mutex_unlock(&sessions_mutex); + + return counter; + +counter_error: + return NULL; +} + void metadata_cache_destroy(struct kref *kref) { struct lttng_metadata_cache *cache = @@ -332,12 +447,17 @@ void lttng_session_destroy(struct lttng_session *session) struct lttng_event *event, *tmpevent; struct lttng_metadata_stream *metadata_stream; struct lttng_event_enabler *event_enabler, *tmp_event_enabler; + struct lttng_counter *counter, *tmpcounter; int ret; mutex_lock(&sessions_mutex); WRITE_ONCE(session->active, 0); list_for_each_entry(chan, &session->chan, list) { - ret = lttng_syscalls_unregister_channel(chan); + ret = lttng_syscalls_unregister_event_container(lttng_channel_get_event_container(chan)); + WARN_ON(ret); + } + list_for_each_entry(counter, &session->counters, node) { + ret = lttng_syscalls_unregister_event_container(lttng_counter_get_event_container(counter)); WARN_ON(ret); } list_for_each_entry(event, &session->events, list) { @@ -346,7 +466,11 @@ void lttng_session_destroy(struct lttng_session *session) } synchronize_trace(); /* Wait for in-flight events to complete */ list_for_each_entry(chan, &session->chan, list) { - ret = lttng_syscalls_destroy_event(chan); + ret = lttng_syscalls_destroy_event_container(lttng_channel_get_event_container(chan)); + WARN_ON(ret); + } + list_for_each_entry(counter, &session->counters, node) { + ret = lttng_syscalls_destroy_event_container(lttng_counter_get_event_container(counter)); WARN_ON(ret); } list_for_each_entry_safe(event_enabler, tmp_event_enabler, @@ -358,6 +482,8 @@ void lttng_session_destroy(struct lttng_session *session) BUG_ON(chan->channel_type == METADATA_CHANNEL); _lttng_channel_destroy(chan); } + list_for_each_entry_safe(counter, tmpcounter, &session->counters, node) + _lttng_session_counter_destroy(counter); mutex_lock(&session->metadata_cache->lock); list_for_each_entry(metadata_stream, &session->metadata_cache->metadata_stream, list) _lttng_metadata_channel_hangup(metadata_stream); @@ -413,9 +539,7 @@ void lttng_event_notifier_group_destroy( if (event_notifier_group->error_counter) { struct lttng_counter *error_counter = event_notifier_group->error_counter; - error_counter->ops->counter_destroy(error_counter->counter); - module_put(error_counter->transport->owner); - lttng_kvfree(error_counter); + lttng_kernel_counter_destroy(error_counter); event_notifier_group->error_counter = NULL; } @@ -554,47 +678,47 @@ end: return ret; } -int lttng_channel_enable(struct lttng_channel *channel) +int lttng_event_container_enable(struct lttng_event_container *container) { int ret = 0; mutex_lock(&sessions_mutex); - if (channel->channel_type == METADATA_CHANNEL) { + if (lttng_event_container_is_metadata_channel(container)) { ret = -EPERM; goto end; } - if (channel->enabled) { + if (container->enabled) { ret = -EEXIST; goto end; } /* Set transient enabler state to "enabled" */ - channel->tstate = 1; - lttng_session_sync_event_enablers(channel->session); + container->tstate = 1; + lttng_session_sync_event_enablers(container->session); /* Set atomically the state to "enabled" */ - WRITE_ONCE(channel->enabled, 1); + WRITE_ONCE(container->enabled, 1); end: mutex_unlock(&sessions_mutex); return ret; } -int lttng_channel_disable(struct lttng_channel *channel) +int lttng_event_container_disable(struct lttng_event_container *container) { int ret = 0; mutex_lock(&sessions_mutex); - if (channel->channel_type == METADATA_CHANNEL) { + if (lttng_event_container_is_metadata_channel(container)) { ret = -EPERM; goto end; } - if (!channel->enabled) { + if (!container->enabled) { ret = -EEXIST; goto end; } /* Set atomically the state to "disabled" */ - WRITE_ONCE(channel->enabled, 0); + WRITE_ONCE(container->enabled, 0); /* Set transient enabler state to "enabled" */ - channel->tstate = 0; - lttng_session_sync_event_enablers(channel->session); + container->tstate = 0; + lttng_session_sync_event_enablers(container->session); end: mutex_unlock(&sessions_mutex); return ret; @@ -605,7 +729,7 @@ int lttng_event_enable(struct lttng_event *event) int ret = 0; mutex_lock(&sessions_mutex); - if (event->chan->channel_type == METADATA_CHANNEL) { + if (lttng_event_within_metadata_channel(event)) { ret = -EPERM; goto end; } @@ -641,7 +765,7 @@ int lttng_event_disable(struct lttng_event *event) int ret = 0; mutex_lock(&sessions_mutex); - if (event->chan->channel_type == METADATA_CHANNEL) { + if (lttng_event_within_metadata_channel(event)) { ret = -EPERM; goto end; } @@ -740,7 +864,8 @@ struct lttng_channel *lttng_channel_create(struct lttng_session *session, unsigned int read_timer_interval, enum channel_type channel_type) { - struct lttng_channel *chan; + struct lttng_event_container *container; + struct lttng_channel *chan = NULL; struct lttng_transport *transport = NULL; mutex_lock(&sessions_mutex); @@ -756,10 +881,21 @@ struct lttng_channel *lttng_channel_create(struct lttng_session *session, printk(KERN_WARNING "LTTng: Can't lock transport module.\n"); goto notransport; } - chan = kzalloc(sizeof(struct lttng_channel), GFP_KERNEL); + chan = lttng_kvzalloc(sizeof(struct lttng_channel), GFP_KERNEL); if (!chan) goto nomem; - chan->session = session; + container = lttng_channel_get_event_container(chan); + container->type = LTTNG_EVENT_CONTAINER_CHANNEL; + container->session = session; + container->tstate = 1; + container->enabled = 1; + /* + * The ring buffer always coalesces hits from various event + * enablers matching a given event to a single event record within the + * ring buffer. + */ + container->coalesce_hits = true; + chan->id = session->free_chan_id++; chan->ops = &transport->ops; /* @@ -768,12 +904,10 @@ struct lttng_channel *lttng_channel_create(struct lttng_session *session, * should be already accessible. */ chan->chan = transport->ops.channel_create(transport_name, - chan, buf_addr, subbuf_size, num_subbuf, + container, buf_addr, subbuf_size, num_subbuf, switch_timer_interval, read_timer_interval); if (!chan->chan) goto create_error; - chan->tstate = 1; - chan->enabled = 1; chan->transport = transport; chan->channel_type = channel_type; list_add(&chan->list, &session->chan); @@ -781,7 +915,7 @@ struct lttng_channel *lttng_channel_create(struct lttng_session *session, return chan; create_error: - kfree(chan); + lttng_kvfree(chan); nomem: if (transport) module_put(transport->owner); @@ -791,6 +925,13 @@ active: return NULL; } +static +void _lttng_session_counter_destroy(struct lttng_counter *counter) +{ + list_del(&counter->node); + lttng_kernel_counter_destroy(counter); +} + /* * Only used internally at session destruction for per-cpu channels, and * when metadata channel is released. @@ -803,13 +944,12 @@ void _lttng_channel_destroy(struct lttng_channel *chan) module_put(chan->transport->owner); list_del(&chan->list); lttng_destroy_context(chan->ctx); - kfree(chan); + lttng_kvfree(chan); } void lttng_metadata_channel_destroy(struct lttng_channel *chan) { BUG_ON(chan->channel_type != METADATA_CHANNEL); - /* Protect the metadata cache with the sessions_mutex. */ mutex_lock(&sessions_mutex); _lttng_channel_destroy(chan); @@ -824,53 +964,262 @@ void _lttng_metadata_channel_hangup(struct lttng_metadata_stream *stream) wake_up_interruptible(&stream->read_wait); } +static +bool lttng_event_container_current_id_full(struct lttng_event_container *container) +{ + switch (container->type) { + case LTTNG_EVENT_CONTAINER_CHANNEL: + { + struct lttng_channel *channel = lttng_event_container_get_channel(container); + + return channel->free_event_id == -1U; + } + case LTTNG_EVENT_CONTAINER_COUNTER: + { + struct lttng_counter *counter = lttng_event_container_get_counter(container); + size_t nr_dimensions, max_nr_elem; + + if (lttng_counter_get_nr_dimensions(&counter->counter->config, + counter->counter, &nr_dimensions)) + return true; + WARN_ON_ONCE(nr_dimensions != 1); + if (nr_dimensions != 1) + return true; + if (lttng_counter_get_max_nr_elem(&counter->counter->config, + counter->counter, &max_nr_elem)) + return true; + return counter->free_index >= max_nr_elem; + } + default: + WARN_ON_ONCE(1); + return true; + } +} + + +static +int lttng_event_container_allocate_id(struct lttng_event_container *container, + const char *key_string, size_t *id) +{ + struct lttng_session *session = container->session; + struct lttng_event *event; + + if (key_string[0]) { + struct hlist_head *head; + + head = utils_borrow_hash_table_bucket(session->events_key_ht.table, + LTTNG_EVENT_HT_SIZE, key_string); + lttng_hlist_for_each_entry(event, head, key_hlist) { + if (!strcmp(key_string, event->key)) { + /* Same key, use same id. */ + *id = event->id; + return 0; + } + } + } + + if (lttng_event_container_current_id_full(container)) { + return -EMFILE; + } + + switch (container->type) { + case LTTNG_EVENT_CONTAINER_CHANNEL: + { + struct lttng_channel *channel = lttng_event_container_get_channel(container); + *id = channel->free_event_id++; + break; + } + case LTTNG_EVENT_CONTAINER_COUNTER: + { + struct lttng_counter *counter = lttng_event_container_get_counter(container); + *id = counter->free_index++; + break; + } + default: + WARN_ON_ONCE(1); + return 0; + } + + return 0; +} + +static +int format_event_key(char *key_string, const struct lttng_counter_key *key, + const char *event_name) +{ + const struct lttng_counter_key_dimension *dim; + size_t i, left = LTTNG_KEY_TOKEN_STRING_LEN_MAX; + + key_string[0] = '\0'; + if (!key || !key->nr_dimensions) + return 0; + /* Currently event keys can only be specified on a single dimension. */ + if (key->nr_dimensions != 1) + return -EINVAL; + dim = &key->key_dimensions[0]; + for (i = 0; i < dim->nr_key_tokens; i++) { + const struct lttng_key_token *token = &dim->key_tokens[i]; + size_t token_len; + const char *str; + + switch (token->type) { + case LTTNG_KEY_TOKEN_STRING: + str = token->arg.string; + break; + case LTTNG_KEY_TOKEN_EVENT_NAME: + str = event_name; + break; + default: + return -EINVAL; + } + token_len = strlen(str); + if (token_len >= left) + return -EINVAL; + strcat(key_string, str); + left -= token_len; + } + return 0; +} + +static +bool match_event_token(struct lttng_event_container *container, + struct lttng_event *event, uint64_t token) +{ + if (container->coalesce_hits) + return true; + if (event->user_token == token) + return true; + return false; +} + +static +int lttng_counter_append_descriptor(struct lttng_counter *counter, + uint64_t user_token, + size_t index, + const char *key) +{ + struct lttng_counter_map *map = &counter->map; + struct lttng_counter_map_descriptor *last; + int ret = 0; + + if (strlen(key) >= LTTNG_KERNEL_COUNTER_KEY_LEN) { + WARN_ON_ONCE(1); + return -EOVERFLOW; + } + mutex_lock(&map->lock); + if (map->nr_descriptors == map->alloc_len) { + struct lttng_counter_map_descriptor *new_table, *old_table; + size_t old_len = map->nr_descriptors; + size_t new_len = max_t(size_t, old_len + 1, map->alloc_len * 2); + + old_table = map->descriptors; + new_table = lttng_kvzalloc(sizeof(struct lttng_counter_map_descriptor) * new_len, + GFP_KERNEL); + if (!new_table) { + ret = -ENOMEM; + goto unlock; + } + + if (old_table) + memcpy(new_table, old_table, old_len * sizeof(struct lttng_counter_map_descriptor)); + + map->descriptors = new_table; + map->alloc_len = new_len; + lttng_kvfree(old_table); + } + last = &map->descriptors[map->nr_descriptors++]; + last->user_token = user_token; + last->array_index = index; + strcpy(last->key, key); +unlock: + mutex_unlock(&map->lock); + return ret; +} /* * Supports event creation while tracing session is active. * Needs to be called with sessions mutex held. */ -struct lttng_event *_lttng_event_create(struct lttng_channel *chan, +struct lttng_event *_lttng_event_create(struct lttng_event_container *container, struct lttng_kernel_event *event_param, + const struct lttng_counter_key *key, void *filter, const struct lttng_event_desc *event_desc, - enum lttng_kernel_instrumentation itype) + enum lttng_kernel_instrumentation itype, + uint64_t token) { - struct lttng_session *session = chan->session; + struct lttng_session *session; struct lttng_event *event; - const char *event_name; - struct hlist_head *head; + char event_name[LTTNG_KERNEL_SYM_NAME_LEN]; + struct hlist_head *name_head, *key_head; + char key_string[LTTNG_KEY_TOKEN_STRING_LEN_MAX]; int ret; - if (chan->free_event_id == -1U) { - ret = -EMFILE; - goto full; - } - + session = container->session; switch (itype) { case LTTNG_KERNEL_TRACEPOINT: - event_name = event_desc->name; + if (strlen(event_desc->name) >= LTTNG_KERNEL_SYM_NAME_LEN) { + ret = -EINVAL; + goto type_error; + } + strcpy(event_name, event_desc->name); break; case LTTNG_KERNEL_KPROBE: case LTTNG_KERNEL_UPROBE: - case LTTNG_KERNEL_KRETPROBE: - case LTTNG_KERNEL_NOOP: case LTTNG_KERNEL_SYSCALL: - event_name = event_param->name; + if (strlen(event_param->name) >= LTTNG_KERNEL_SYM_NAME_LEN) { + ret = -EINVAL; + goto type_error; + } + strcpy(event_name, event_param->name); + break; + case LTTNG_KERNEL_KRETPROBE: + if (strlen(event_param->name) >= LTTNG_KERNEL_SYM_NAME_LEN) { + ret = -EINVAL; + goto type_error; + } + strcpy(event_name, event_param->name); + if (strlen(event_name) + strlen("_entry") >= LTTNG_KERNEL_SYM_NAME_LEN) { + ret = -EINVAL; + goto type_error; + } + strcat(event_name, "_entry"); break; case LTTNG_KERNEL_FUNCTION: /* Fall-through. */ + case LTTNG_KERNEL_NOOP: /* Fall-through. */ default: WARN_ON_ONCE(1); ret = -EINVAL; goto type_error; } - head = utils_borrow_hash_table_bucket(session->events_ht.table, + if (format_event_key(key_string, key, event_name)) { + ret = -EINVAL; + goto type_error; + } + + name_head = utils_borrow_hash_table_bucket(session->events_name_ht.table, LTTNG_EVENT_HT_SIZE, event_name); - lttng_hlist_for_each_entry(event, head, hlist) { + lttng_hlist_for_each_entry(event, name_head, name_hlist) { + bool same_event = false, same_container = false, same_key = false, + same_token = false; + WARN_ON_ONCE(!event->desc); - if (!strncmp(event->desc->name, event_name, - LTTNG_KERNEL_SYM_NAME_LEN - 1) - && chan == event->chan) { + if (event_desc) { + if (event->desc == event_desc) + same_event = true; + } else { + if (!strcmp(event_name, event->desc->name)) + same_event = true; + } + if (container == event->container) { + same_container = true; + if (match_event_token(container, event, token)) + same_token = true; + } + if (key_string[0] == '\0' || !strcmp(key_string, event->key)) + same_key = true; + if (same_event && same_container && same_key && same_token) { ret = -EEXIST; goto exist; } @@ -881,13 +1230,25 @@ struct lttng_event *_lttng_event_create(struct lttng_channel *chan, ret = -ENOMEM; goto cache_error; } - event->chan = chan; + event->container = container; event->filter = filter; - event->id = chan->free_event_id++; event->instrumentation = itype; event->evtype = LTTNG_TYPE_EVENT; + if (!container->coalesce_hits) + event->user_token = token; INIT_LIST_HEAD(&event->filter_bytecode_runtime_head); INIT_LIST_HEAD(&event->enablers_ref_head); + if (lttng_event_container_allocate_id(container, key_string, + &event->id)) { + ret = -EMFILE; + goto full; + } + if (key_string[0]) { + key_head = utils_borrow_hash_table_bucket(session->events_key_ht.table, + LTTNG_EVENT_HT_SIZE, key_string); + hlist_add_head(&event->key_hlist, key_head); + } + strcpy(event->key, key_string); switch (itype) { case LTTNG_KERNEL_TRACEPOINT: @@ -923,8 +1284,35 @@ struct lttng_event *_lttng_event_create(struct lttng_channel *chan, ret = -EINVAL; goto register_error; } + event->u.kprobe.user_token = token; ret = try_module_get(event->desc->owner); WARN_ON_ONCE(!ret); + + /* Append descriptor to counter. */ + switch (container->type) { + case LTTNG_EVENT_CONTAINER_COUNTER: + { + struct lttng_counter *counter; + const char *name = ""; + int ret; + + counter = lttng_event_container_get_counter(container); + if (event->key[0]) + name = event->key; + else + name = event_name; + ret = lttng_counter_append_descriptor(counter, + token, event->id, + name); + if (ret) { + WARN_ON_ONCE(1); + } + break; + } + case LTTNG_EVENT_CONTAINER_CHANNEL: + default: + break; + } break; case LTTNG_KERNEL_KRETPROBE: { @@ -937,20 +1325,68 @@ struct lttng_event *_lttng_event_create(struct lttng_channel *chan, */ event->enabled = 0; event->registered = 1; + event->u.kretprobe.user_token = token; + + /* Append descriptor to counter. */ + switch (container->type) { + case LTTNG_EVENT_CONTAINER_COUNTER: + { + struct lttng_counter *counter; + const char *name = ""; + int ret; + + counter = lttng_event_container_get_counter(container); + if (event->key[0]) + name = event->key; + else + name = event_name; + ret = lttng_counter_append_descriptor(counter, + token, event->id, + name); + if (ret) { + WARN_ON_ONCE(1); + } + break; + } + case LTTNG_EVENT_CONTAINER_CHANNEL: + default: + break; + } + event_return = kmem_cache_zalloc(event_cache, GFP_KERNEL); if (!event_return) { ret = -ENOMEM; goto register_error; } - event_return->chan = chan; + event_return->container = container; event_return->filter = filter; - event_return->id = chan->free_event_id++; + + strcpy(event_name, event_param->name); + if (strlen(event_name) + strlen("_return") >= LTTNG_KERNEL_SYM_NAME_LEN) { + ret = -EINVAL; + goto register_error; + } + strcat(event_name, "_return"); + if (format_event_key(key_string, key, event_name)) { + ret = -EINVAL; + goto register_error; + } + if (lttng_event_container_allocate_id(container, key_string, &event_return->id)) { + kmem_cache_free(event_cache, event_return); + ret = -EMFILE; + goto register_error; + } + key_head = utils_borrow_hash_table_bucket(session->events_key_ht.table, + LTTNG_EVENT_HT_SIZE, key_string); + hlist_add_head(&event_return->key_hlist, key_head); event_return->enabled = 0; event_return->registered = 1; event_return->instrumentation = itype; INIT_LIST_HEAD(&event_return->filter_bytecode_runtime_head); INIT_LIST_HEAD(&event_return->enablers_ref_head); + event_return->u.kretprobe.user_token = token; + strcpy(event_return->key, key_string); /* * Populate lttng_event structure before kretprobe registration. */ @@ -970,19 +1406,50 @@ struct lttng_event *_lttng_event_create(struct lttng_channel *chan, WARN_ON_ONCE(!ret); ret = try_module_get(event->desc->owner); WARN_ON_ONCE(!ret); - ret = _lttng_event_metadata_statedump(chan->session, chan, - event_return); - WARN_ON_ONCE(ret > 0); - if (ret) { - kmem_cache_free(event_cache, event_return); - module_put(event->desc->owner); - module_put(event->desc->owner); - goto statedump_error; + + /* Append exit descriptor to counter. */ + switch (container->type) { + case LTTNG_EVENT_CONTAINER_COUNTER: + { + struct lttng_counter *counter; + const char *name = ""; + int ret; + + counter = lttng_event_container_get_counter(container); + if (event_return->key[0]) + name = event_return->key; + else + name = event_name; + ret = lttng_counter_append_descriptor(counter, + token, event_return->id, + name); + if (ret) { + WARN_ON_ONCE(1); + } + break; + } + case LTTNG_EVENT_CONTAINER_CHANNEL: + default: + break; } - list_add(&event_return->list, &chan->session->events); + switch (container->type) { + case LTTNG_EVENT_CONTAINER_CHANNEL: + ret = _lttng_event_metadata_statedump(session, event_return); + WARN_ON_ONCE(ret > 0); + if (ret) { + kmem_cache_free(event_cache, event_return); + module_put(event->desc->owner); + module_put(event->desc->owner); + goto statedump_error; + } + break; + case LTTNG_EVENT_CONTAINER_COUNTER: + default: + break; + } + list_add(&event_return->list, &session->events); break; } - case LTTNG_KERNEL_NOOP: case LTTNG_KERNEL_SYSCALL: /* * Needs to be explicitly enabled after creation, since @@ -1025,6 +1492,7 @@ struct lttng_event *_lttng_event_create(struct lttng_channel *chan, */ event->enabled = 0; event->registered = 1; + event->u.uprobe.user_token = token; /* * Populate lttng_event structure before event @@ -1032,37 +1500,71 @@ struct lttng_event *_lttng_event_create(struct lttng_channel *chan, */ smp_wmb(); - ret = lttng_uprobes_register_event(event_param->name, + ret = lttng_uprobes_register_event(event_name, event_param->u.uprobe.fd, event); if (ret) goto register_error; ret = try_module_get(event->desc->owner); WARN_ON_ONCE(!ret); + + /* Append descriptor to counter. */ + switch (container->type) { + case LTTNG_EVENT_CONTAINER_COUNTER: + { + struct lttng_counter *counter; + const char *name = ""; + int ret; + + counter = lttng_event_container_get_counter(container); + if (event->key[0]) + name = event->key; + else + name = event_name; + ret = lttng_counter_append_descriptor(counter, + token, event->id, + name); + if (ret) { + WARN_ON_ONCE(1); + } + break; + } + case LTTNG_EVENT_CONTAINER_CHANNEL: + default: + break; + } break; - case LTTNG_KERNEL_FUNCTION: /* Fall-through */ + case LTTNG_KERNEL_FUNCTION: /* Fall-through. */ + case LTTNG_KERNEL_NOOP: /* Fall-through.*/ default: WARN_ON_ONCE(1); ret = -EINVAL; goto register_error; } - ret = _lttng_event_metadata_statedump(chan->session, chan, event); - WARN_ON_ONCE(ret > 0); - if (ret) { - goto statedump_error; + switch (container->type) { + case LTTNG_EVENT_CONTAINER_CHANNEL: + ret = _lttng_event_metadata_statedump(session, event); + WARN_ON_ONCE(ret > 0); + if (ret) { + goto statedump_error; + } + break; + case LTTNG_EVENT_CONTAINER_COUNTER: + default: + break; } - hlist_add_head(&event->hlist, head); - list_add(&event->list, &chan->session->events); + hlist_add_head(&event->name_hlist, name_head); + list_add(&event->list, &session->events); return event; statedump_error: /* If a statedump error occurs, events will not be readable. */ register_error: +full: kmem_cache_free(event_cache, event); cache_error: exist: type_error: -full: return ERR_PTR(ret); } @@ -1297,17 +1799,19 @@ int lttng_kernel_counter_clear(struct lttng_counter *counter, return counter->ops->counter_clear(counter->counter, dim_indexes); } -struct lttng_event *lttng_event_create(struct lttng_channel *chan, +struct lttng_event *lttng_event_create(struct lttng_event_container *container, struct lttng_kernel_event *event_param, + const struct lttng_counter_key *key, void *filter, const struct lttng_event_desc *event_desc, - enum lttng_kernel_instrumentation itype) + enum lttng_kernel_instrumentation itype, + uint64_t token) { struct lttng_event *event; mutex_lock(&sessions_mutex); - event = _lttng_event_create(chan, event_param, filter, event_desc, - itype); + event = _lttng_event_create(container, event_param, key, filter, event_desc, + itype, token); mutex_unlock(&sessions_mutex); return event; } @@ -1343,11 +1847,10 @@ void register_event(struct lttng_event *event) switch (event->instrumentation) { case LTTNG_KERNEL_TRACEPOINT: ret = lttng_wrapper_tracepoint_probe_register(desc->kname, - desc->probe_callback, - event); + desc->probe_callback, event); break; case LTTNG_KERNEL_SYSCALL: - ret = lttng_syscall_filter_enable_event(event->chan, event); + ret = lttng_syscall_filter_enable_event(event->container, event); break; case LTTNG_KERNEL_KPROBE: case LTTNG_KERNEL_UPROBE: @@ -1378,8 +1881,7 @@ int _lttng_event_unregister(struct lttng_event *event) switch (event->instrumentation) { case LTTNG_KERNEL_TRACEPOINT: ret = lttng_wrapper_tracepoint_probe_unregister(event->desc->kname, - event->desc->probe_callback, - event); + event->desc->probe_callback, event); break; case LTTNG_KERNEL_KPROBE: lttng_kprobes_unregister_event(event); @@ -1390,7 +1892,7 @@ int _lttng_event_unregister(struct lttng_event *event) ret = 0; break; case LTTNG_KERNEL_SYSCALL: - ret = lttng_syscall_filter_disable_event(event->chan, event); + ret = lttng_syscall_filter_disable_event(event->container, event); break; case LTTNG_KERNEL_NOOP: ret = 0; @@ -1922,19 +2424,19 @@ int lttng_desc_match_enabler(const struct lttng_event_desc *desc, } static -int lttng_event_enabler_match_event(struct lttng_event_enabler *event_enabler, +bool lttng_event_enabler_match_event(struct lttng_event_enabler *event_enabler, struct lttng_event *event) { struct lttng_enabler *base_enabler = lttng_event_enabler_as_enabler( event_enabler); - if (base_enabler->event_param.instrumentation != event->instrumentation) - return 0; - if (lttng_desc_match_enabler(event->desc, base_enabler) - && event->chan == event_enabler->chan) - return 1; + if (base_enabler->event_param.instrumentation == event->instrumentation + && lttng_desc_match_enabler(event->desc, base_enabler) > 0 + && event->container == event_enabler->container + && match_event_token(event->container, event, event_enabler->base.user_token)) + return true; else - return 0; + return false; } static @@ -1946,7 +2448,7 @@ int lttng_event_notifier_enabler_match_event_notifier(struct lttng_event_notifie if (base_enabler->event_param.instrumentation != event_notifier->instrumentation) return 0; - if (lttng_desc_match_enabler(event_notifier->desc, base_enabler) + if (lttng_desc_match_enabler(event_notifier->desc, base_enabler) > 0 && event_notifier->group == event_notifier_enabler->group && event_notifier->user_token == event_notifier_enabler->base.user_token) return 1; @@ -1971,7 +2473,6 @@ struct lttng_enabler_ref *lttng_enabler_ref( static void lttng_create_tracepoint_event_if_missing(struct lttng_event_enabler *event_enabler) { - struct lttng_session *session = event_enabler->chan->session; struct lttng_probe_desc *probe_desc; const struct lttng_event_desc *desc; int i; @@ -1985,37 +2486,22 @@ void lttng_create_tracepoint_event_if_missing(struct lttng_event_enabler *event_ */ list_for_each_entry(probe_desc, probe_list, head) { for (i = 0; i < probe_desc->nr_events; i++) { - int found = 0; - struct hlist_head *head; struct lttng_event *event; desc = probe_desc->event_desc[i]; - if (!lttng_desc_match_enabler(desc, - lttng_event_enabler_as_enabler(event_enabler))) + if (lttng_desc_match_enabler(desc, + lttng_event_enabler_as_enabler(event_enabler)) <= 0) continue; - /* - * Check if already created. - */ - head = utils_borrow_hash_table_bucket( - session->events_ht.table, LTTNG_EVENT_HT_SIZE, - desc->name); - lttng_hlist_for_each_entry(event, head, hlist) { - if (event->desc == desc - && event->chan == event_enabler->chan) - found = 1; - } - if (found) + /* Try to create an event for this event probe. */ + event = _lttng_event_create(event_enabler->container, + NULL, &event_enabler->key, NULL, desc, + LTTNG_KERNEL_TRACEPOINT, + event_enabler->base.user_token); + /* Skip if event is already found. */ + if (IS_ERR(event) && PTR_ERR(event) == -EEXIST) continue; - - /* - * We need to create an event for this - * event probe. - */ - event = _lttng_event_create(event_enabler->chan, - NULL, NULL, desc, - LTTNG_KERNEL_TRACEPOINT); - if (!event) { + if (IS_ERR(event)) { printk(KERN_INFO "LTTng: Unable to create event %s\n", probe_desc->event_desc[i]->name); } @@ -2045,8 +2531,8 @@ void lttng_create_tracepoint_event_notifier_if_missing(struct lttng_event_notifi struct lttng_event_notifier *event_notifier; desc = probe_desc->event_desc[i]; - if (!lttng_desc_match_enabler(desc, - lttng_event_notifier_enabler_as_enabler(event_notifier_enabler))) + if (lttng_desc_match_enabler(desc, + lttng_event_notifier_enabler_as_enabler(event_notifier_enabler)) <= 0) continue; /* @@ -2128,8 +2614,8 @@ void lttng_create_event_if_missing(struct lttng_event_enabler *event_enabler) static int lttng_event_enabler_ref_events(struct lttng_event_enabler *event_enabler) { - struct lttng_channel *chan = event_enabler->chan; - struct lttng_session *session = event_enabler->chan->session; + struct lttng_event_container *container = event_enabler->container; + struct lttng_session *session = container->session; struct lttng_enabler *base_enabler = lttng_event_enabler_as_enabler(event_enabler); struct lttng_event *event; @@ -2141,10 +2627,10 @@ int lttng_event_enabler_ref_events(struct lttng_event_enabler *event_enabler) enum lttng_kernel_syscall_entryexit entryexit = base_enabler->event_param.u.syscall.entryexit; if (entryexit == LTTNG_KERNEL_SYSCALL_ENTRY || entryexit == LTTNG_KERNEL_SYSCALL_ENTRYEXIT) - WRITE_ONCE(chan->syscall_all_entry, enabled); + WRITE_ONCE(container->syscall_all_entry, enabled); if (entryexit == LTTNG_KERNEL_SYSCALL_EXIT || entryexit == LTTNG_KERNEL_SYSCALL_ENTRYEXIT) - WRITE_ONCE(chan->syscall_all_exit, enabled); + WRITE_ONCE(container->syscall_all_exit, enabled); } /* First ensure that probe events are created for this enabler. */ @@ -2169,6 +2655,32 @@ int lttng_event_enabler_ref_events(struct lttng_event_enabler *event_enabler) enabler_ref->ref = lttng_event_enabler_as_enabler(event_enabler); list_add(&enabler_ref->node, &event->enablers_ref_head); + /* Append descriptor to counter. */ + switch (container->type) { + case LTTNG_EVENT_CONTAINER_COUNTER: + { + struct lttng_counter *counter; + const char *name = ""; + int ret; + + counter = lttng_event_container_get_counter(container); + if (event->key[0]) + name = event->key; + else if (event->desc && event->desc->name) + name = event->desc->name; + ret = lttng_counter_append_descriptor(counter, + event_enabler->base.user_token, event->id, + name); + if (ret) { + WARN_ON_ONCE(1); + return ret; + } + break; + } + case LTTNG_EVENT_CONTAINER_CHANNEL: + default: + break; + } } /* @@ -2326,7 +2838,8 @@ int lttng_fix_pending_event_notifiers(void) struct lttng_event_enabler *lttng_event_enabler_create( enum lttng_enabler_format_type format_type, struct lttng_kernel_event *event_param, - struct lttng_channel *chan) + const struct lttng_counter_key *key, + struct lttng_event_container *container) { struct lttng_event_enabler *event_enabler; @@ -2337,13 +2850,16 @@ struct lttng_event_enabler *lttng_event_enabler_create( INIT_LIST_HEAD(&event_enabler->base.filter_bytecode_head); memcpy(&event_enabler->base.event_param, event_param, sizeof(event_enabler->base.event_param)); - event_enabler->chan = chan; + event_enabler->container = container; /* ctx left NULL */ event_enabler->base.enabled = 0; event_enabler->base.evtype = LTTNG_TYPE_ENABLER; + event_enabler->base.user_token = event_param->token; + if (key) + event_enabler->key = *key; mutex_lock(&sessions_mutex); - list_add(&event_enabler->node, &event_enabler->chan->session->enablers_head); - lttng_session_lazy_sync_event_enablers(event_enabler->chan->session); + list_add(&event_enabler->node, &event_enabler->container->session->enablers_head); + lttng_session_lazy_sync_event_enablers(event_enabler->container->session); mutex_unlock(&sessions_mutex); return event_enabler; } @@ -2352,7 +2868,7 @@ int lttng_event_enabler_enable(struct lttng_event_enabler *event_enabler) { mutex_lock(&sessions_mutex); lttng_event_enabler_as_enabler(event_enabler)->enabled = 1; - lttng_session_lazy_sync_event_enablers(event_enabler->chan->session); + lttng_session_lazy_sync_event_enablers(event_enabler->container->session); mutex_unlock(&sessions_mutex); return 0; } @@ -2361,7 +2877,7 @@ int lttng_event_enabler_disable(struct lttng_event_enabler *event_enabler) { mutex_lock(&sessions_mutex); lttng_event_enabler_as_enabler(event_enabler)->enabled = 0; - lttng_session_lazy_sync_event_enablers(event_enabler->chan->session); + lttng_session_lazy_sync_event_enablers(event_enabler->container->session); mutex_unlock(&sessions_mutex); return 0; } @@ -2408,7 +2924,7 @@ int lttng_event_enabler_attach_filter_bytecode(struct lttng_event_enabler *event if (ret) goto error; - lttng_session_lazy_sync_event_enablers(event_enabler->chan->session); + lttng_session_lazy_sync_event_enablers(event_enabler->container->session); return 0; error: @@ -2622,7 +3138,7 @@ void lttng_session_sync_event_enablers(struct lttng_session *session) lttng_event_enabler_ref_events(event_enabler); /* * For each event, if at least one of its enablers is enabled, - * and its channel and session transient states are enabled, we + * and its event container and session transient states are enabled, we * enable the event, else we disable it. */ list_for_each_entry(event, &session->events, list) { @@ -2648,10 +3164,10 @@ void lttng_session_sync_event_enablers(struct lttng_session *session) } /* * Enabled state is based on union of enablers, with - * intesection of session and channel transient enable + * intesection of session and event container transient enable * states. */ - enabled = enabled && session->tstate && event->chan->tstate; + enabled = enabled && session->tstate && event->container->tstate; WRITE_ONCE(event->enabled, enabled); /* @@ -2768,6 +3284,8 @@ void lttng_event_notifier_group_sync_enablers(struct lttng_event_notifier_group list_for_each_entry(runtime, &event_notifier->capture_bytecode_runtime_head, node) lttng_bytecode_capture_sync_state(runtime); + + WRITE_ONCE(event_notifier->eval_capture, !!event_notifier->num_captures); } } @@ -3498,11 +4016,13 @@ int _lttng_fields_metadata_statedump(struct lttng_session *session, */ static int _lttng_event_metadata_statedump(struct lttng_session *session, - struct lttng_channel *chan, struct lttng_event *event) { + struct lttng_channel *chan; int ret = 0; + WARN_ON_ONCE(event->container->type != LTTNG_EVENT_CONTAINER_CHANNEL); + chan = lttng_event_container_get_channel(event->container); if (event->metadata_dumped || !LTTNG_READ_ONCE(session->active)) return 0; if (chan->channel_type == METADATA_CHANNEL) @@ -3513,11 +4033,11 @@ int _lttng_event_metadata_statedump(struct lttng_session *session, ret = lttng_metadata_printf(session, "event {\n" " name = \"%s\";\n" - " id = %u;\n" + " id = %zu;\n" " stream_id = %u;\n", event->desc->name, event->id, - event->chan->id); + chan->id); if (ret) goto end; @@ -3973,7 +4493,10 @@ skip_session: } list_for_each_entry(event, &session->events, list) { - ret = _lttng_event_metadata_statedump(session, event->chan, event); + /* Skip counter container. */ + if (event->container->type != LTTNG_EVENT_CONTAINER_CHANNEL) + continue; + ret = _lttng_event_metadata_statedump(session, event); if (ret) goto end; } diff --git a/src/lttng-ring-buffer-client.h b/src/lttng-ring-buffer-client.h index 4f8699c0..c234e8ae 100644 --- a/src/lttng-ring-buffer-client.h +++ b/src/lttng-ring-buffer-client.h @@ -141,7 +141,8 @@ size_t record_header_size(const struct lib_ring_buffer_config *config, struct lib_ring_buffer_ctx *ctx, struct lttng_client_ctx *client_ctx) { - struct lttng_channel *lttng_chan = channel_get_private(chan); + struct lttng_event_container *container = channel_get_private(chan); + struct lttng_channel *lttng_chan = lttng_event_container_get_channel(container); struct lttng_probe_ctx *lttng_probe_ctx = ctx->priv; struct lttng_event *event = lttng_probe_ctx->event; size_t orig_offset = offset; @@ -212,7 +213,8 @@ void lttng_write_event_header(const struct lib_ring_buffer_config *config, struct lib_ring_buffer_ctx *ctx, uint32_t event_id) { - struct lttng_channel *lttng_chan = channel_get_private(ctx->chan); + struct lttng_event_container *container = channel_get_private(ctx->chan); + struct lttng_channel *lttng_chan = lttng_event_container_get_channel(container); struct lttng_probe_ctx *lttng_probe_ctx = ctx->priv; struct lttng_event *event = lttng_probe_ctx->event; @@ -264,7 +266,8 @@ void lttng_write_event_header_slow(const struct lib_ring_buffer_config *config, struct lib_ring_buffer_ctx *ctx, uint32_t event_id) { - struct lttng_channel *lttng_chan = channel_get_private(ctx->chan); + struct lttng_event_container *container = channel_get_private(ctx->chan); + struct lttng_channel *lttng_chan = lttng_event_container_get_channel(container); struct lttng_probe_ctx *lttng_probe_ctx = ctx->priv; struct lttng_event *event = lttng_probe_ctx->event; @@ -365,8 +368,9 @@ static void client_buffer_begin(struct lib_ring_buffer *buf, u64 tsc, (struct packet_header *) lib_ring_buffer_offset_address(&buf->backend, subbuf_idx * chan->backend.subbuf_size); - struct lttng_channel *lttng_chan = channel_get_private(chan); - struct lttng_session *session = lttng_chan->session; + struct lttng_event_container *container = channel_get_private(chan); + struct lttng_channel *lttng_chan = lttng_event_container_get_channel(container); + struct lttng_session *session = container->session; header->magic = CTF_MAGIC_NUMBER; memcpy(header->uuid, session->uuid.b, sizeof(session->uuid)); @@ -480,7 +484,8 @@ static int client_stream_id(const struct lib_ring_buffer_config *config, uint64_t *stream_id) { struct channel *chan = buf->backend.chan; - struct lttng_channel *lttng_chan = channel_get_private(chan); + struct lttng_event_container *container = channel_get_private(chan); + struct lttng_channel *lttng_chan = lttng_event_container_get_channel(container); *stream_id = lttng_chan->id; return 0; @@ -555,10 +560,10 @@ struct channel *_channel_create(const char *name, unsigned int switch_timer_interval, unsigned int read_timer_interval) { - struct lttng_channel *lttng_chan = priv; + struct lttng_event_container *container = priv; struct channel *chan; - chan = channel_create(&client_config, name, lttng_chan, buf_addr, + chan = channel_create(&client_config, name, container, buf_addr, subbuf_size, num_subbuf, switch_timer_interval, read_timer_interval); if (chan) { @@ -618,7 +623,8 @@ static int lttng_event_reserve(struct lib_ring_buffer_ctx *ctx, uint32_t event_id) { - struct lttng_channel *lttng_chan = channel_get_private(ctx->chan); + struct lttng_event_container *container = channel_get_private(ctx->chan); + struct lttng_channel *lttng_chan = lttng_event_container_get_channel(container); struct lttng_probe_ctx *lttng_probe_ctx = ctx->priv; struct lttng_event *event = lttng_probe_ctx->event; struct lttng_client_ctx client_ctx; diff --git a/src/lttng-ring-buffer-metadata-client.h b/src/lttng-ring-buffer-metadata-client.h index 6fa0c2b1..c36dbd51 100644 --- a/src/lttng-ring-buffer-metadata-client.h +++ b/src/lttng-ring-buffer-metadata-client.h @@ -242,11 +242,11 @@ struct channel *_channel_create(const char *name, unsigned int switch_timer_interval, unsigned int read_timer_interval) { - struct lttng_channel *lttng_chan = priv; + struct lttng_event_container *container = priv; struct channel *chan; chan = channel_create(&client_config, name, - lttng_chan->session->metadata_cache, buf_addr, + container->session->metadata_cache, buf_addr, subbuf_size, num_subbuf, switch_timer_interval, read_timer_interval); if (chan) { diff --git a/src/lttng-syscalls.c b/src/lttng-syscalls.c index 3fce0914..eb813147 100644 --- a/src/lttng-syscalls.c +++ b/src/lttng-syscalls.c @@ -639,33 +639,33 @@ void syscall_entry_event_notifier_call_func(struct hlist_head *dispatch_list, void syscall_entry_event_probe(void *__data, struct pt_regs *regs, long id) { - struct lttng_channel *chan = __data; + struct lttng_event_container *container = __data; struct hlist_head *action_list, *unknown_action_list; const struct trace_syscall_entry *table, *entry; size_t table_len; if (unlikely(in_compat_syscall())) { - struct lttng_syscall_filter *filter = chan->sc_filter; + struct lttng_syscall_filter *filter = container->sc_filter; if (id < 0 || id >= NR_compat_syscalls - || (!READ_ONCE(chan->syscall_all_entry) && !test_bit(id, filter->sc_compat_entry))) { + || (!READ_ONCE(container->syscall_all_entry) && !test_bit(id, filter->sc_compat_entry))) { /* System call filtered out. */ return; } table = compat_sc_table; table_len = ARRAY_SIZE(compat_sc_table); - unknown_action_list = &chan->sc_compat_unknown; + unknown_action_list = &container->sc_compat_unknown; } else { - struct lttng_syscall_filter *filter = chan->sc_filter; + struct lttng_syscall_filter *filter = container->sc_filter; if (id < 0 || id >= NR_syscalls - || (!READ_ONCE(chan->syscall_all_entry) && !test_bit(id, filter->sc_entry))) { + || (!READ_ONCE(container->syscall_all_entry) && !test_bit(id, filter->sc_entry))) { /* System call filtered out. */ return; } table = sc_table; table_len = ARRAY_SIZE(sc_table); - unknown_action_list = &chan->sc_unknown; + unknown_action_list = &container->sc_unknown; } if (unlikely(id < 0 || id >= table_len)) { syscall_entry_event_unknown(unknown_action_list, regs, id); @@ -679,9 +679,9 @@ void syscall_entry_event_probe(void *__data, struct pt_regs *regs, long id) } if (unlikely(in_compat_syscall())) { - action_list = &chan->compat_sc_table[id]; + action_list = &container->compat_sc_table[id]; } else { - action_list = &chan->sc_table[id]; + action_list = &container->sc_table[id]; } if (unlikely(hlist_empty(action_list))) return; @@ -875,7 +875,7 @@ void syscall_exit_call_func(struct hlist_head *action_list, void syscall_exit_event_probe(void *__data, struct pt_regs *regs, long ret) { - struct lttng_channel *chan = __data; + struct lttng_event_container *container = __data; struct hlist_head *action_list, *unknown_action_list; const struct trace_syscall_entry *table, *entry; size_t table_len; @@ -884,27 +884,27 @@ void syscall_exit_event_probe(void *__data, struct pt_regs *regs, long ret) id = syscall_get_nr(current, regs); if (unlikely(in_compat_syscall())) { - struct lttng_syscall_filter *filter = chan->sc_filter; + struct lttng_syscall_filter *filter = container->sc_filter; if (id < 0 || id >= NR_compat_syscalls - || (!READ_ONCE(chan->syscall_all_exit) && !test_bit(id, filter->sc_compat_exit))) { + || (!READ_ONCE(container->syscall_all_exit) && !test_bit(id, filter->sc_compat_exit))) { /* System call filtered out. */ return; } table = compat_sc_exit_table; table_len = ARRAY_SIZE(compat_sc_exit_table); - unknown_action_list = &chan->compat_sc_exit_unknown; + unknown_action_list = &container->compat_sc_exit_unknown; } else { - struct lttng_syscall_filter *filter = chan->sc_filter; + struct lttng_syscall_filter *filter = container->sc_filter; if (id < 0 || id >= NR_syscalls - || (!READ_ONCE(chan->syscall_all_exit) && !test_bit(id, filter->sc_exit))) { + || (!READ_ONCE(container->syscall_all_exit) && !test_bit(id, filter->sc_exit))) { /* System call filtered out. */ return; } table = sc_exit_table; table_len = ARRAY_SIZE(sc_exit_table); - unknown_action_list = &chan->sc_exit_unknown; + unknown_action_list = &container->sc_exit_unknown; } if (unlikely(id < 0 || id >= table_len)) { syscall_exit_event_unknown(unknown_action_list, regs, id, ret); @@ -918,9 +918,9 @@ void syscall_exit_event_probe(void *__data, struct pt_regs *regs, long ret) } if (unlikely(in_compat_syscall())) { - action_list = &chan->compat_sc_exit_table[id]; + action_list = &container->compat_sc_exit_table[id]; } else { - action_list = &chan->sc_exit_table[id]; + action_list = &container->sc_exit_table[id]; } if (unlikely(hlist_empty(action_list))) return; @@ -1102,11 +1102,10 @@ void syscall_exit_event_notifier_probe(void *__data, struct pt_regs *regs, */ static int lttng_create_syscall_event_if_missing(const struct trace_syscall_entry *table, size_t table_len, - struct hlist_head *chan_table, struct lttng_event_enabler *event_enabler, + struct hlist_head *container_table, struct lttng_event_enabler *event_enabler, void *filter, enum sc_type type) { - struct lttng_channel *chan = event_enabler->chan; - struct lttng_session *session = chan->session; + struct lttng_event_container *container = event_enabler->container; unsigned int i; /* Allocate events for each syscall matching enabler, insert into table */ @@ -1114,8 +1113,6 @@ int lttng_create_syscall_event_if_missing(const struct trace_syscall_entry *tabl const struct lttng_event_desc *desc = table[i].desc; struct lttng_kernel_event ev; struct lttng_event *event; - struct hlist_head *head; - bool found = false; if (!desc) { /* Unknown syscall */ @@ -1124,19 +1121,6 @@ int lttng_create_syscall_event_if_missing(const struct trace_syscall_entry *tabl if (lttng_desc_match_enabler(desc, lttng_event_enabler_as_enabler(event_enabler)) <= 0) continue; - /* - * Check if already created. - */ - head = utils_borrow_hash_table_bucket( - session->events_ht.table, LTTNG_EVENT_HT_SIZE, - desc->name); - lttng_hlist_for_each_entry(event, head, hlist) { - if (event->desc == desc - && event->chan == event_enabler->chan) - found = true; - } - if (found) - continue; /* We need to create an event for this syscall/enabler. */ memset(&ev, 0, sizeof(ev)); @@ -1161,9 +1145,12 @@ int lttng_create_syscall_event_if_missing(const struct trace_syscall_entry *tabl strncpy(ev.name, desc->name, LTTNG_KERNEL_SYM_NAME_LEN - 1); ev.name[LTTNG_KERNEL_SYM_NAME_LEN - 1] = '\0'; ev.instrumentation = LTTNG_KERNEL_SYSCALL; - event = _lttng_event_create(chan, &ev, filter, - desc, ev.instrumentation); - WARN_ON_ONCE(!event); + event = _lttng_event_create(container, &ev, &event_enabler->key, filter, + desc, ev.instrumentation, + event_enabler->base.user_token); + /* Skip if event is already found. */ + if (IS_ERR(event) && PTR_ERR(event) == -EEXIST) + continue; if (IS_ERR(event)) { /* * If something goes wrong in event registration @@ -1173,7 +1160,7 @@ int lttng_create_syscall_event_if_missing(const struct trace_syscall_entry *tabl */ return PTR_ERR(event); } - hlist_add_head(&event->u.syscall.node, &chan_table[i]); + hlist_add_head(&event->u.syscall.node, &container_table[i]); } return 0; } @@ -1183,46 +1170,46 @@ int lttng_create_syscall_event_if_missing(const struct trace_syscall_entry *tabl */ int lttng_syscalls_register_event(struct lttng_event_enabler *event_enabler, void *filter) { - struct lttng_channel *chan = event_enabler->chan; + struct lttng_event_container *container = event_enabler->container; struct lttng_kernel_event ev; int ret; wrapper_vmalloc_sync_mappings(); - if (!chan->sc_table) { + if (!container->sc_table) { /* create syscall table mapping syscall to events */ - chan->sc_table = kzalloc(sizeof(struct lttng_event *) + container->sc_table = kzalloc(sizeof(struct hlist_head) * ARRAY_SIZE(sc_table), GFP_KERNEL); - if (!chan->sc_table) + if (!container->sc_table) return -ENOMEM; } - if (!chan->sc_exit_table) { + if (!container->sc_exit_table) { /* create syscall table mapping syscall to events */ - chan->sc_exit_table = kzalloc(sizeof(struct lttng_event *) + container->sc_exit_table = kzalloc(sizeof(struct hlist_head) * ARRAY_SIZE(sc_exit_table), GFP_KERNEL); - if (!chan->sc_exit_table) + if (!container->sc_exit_table) return -ENOMEM; } - #ifdef CONFIG_COMPAT - if (!chan->compat_sc_table) { + if (!container->compat_sc_table) { /* create syscall table mapping compat syscall to events */ - chan->compat_sc_table = kzalloc(sizeof(struct lttng_event *) + container->compat_sc_table = kzalloc(sizeof(struct hlist_head) * ARRAY_SIZE(compat_sc_table), GFP_KERNEL); - if (!chan->compat_sc_table) + if (!container->compat_sc_table) return -ENOMEM; } - if (!chan->compat_sc_exit_table) { + if (!container->compat_sc_exit_table) { /* create syscall table mapping compat syscall to events */ - chan->compat_sc_exit_table = kzalloc(sizeof(struct lttng_event *) + container->compat_sc_exit_table = kzalloc(sizeof(struct hlist_head) * ARRAY_SIZE(compat_sc_exit_table), GFP_KERNEL); - if (!chan->compat_sc_exit_table) + if (!container->compat_sc_exit_table) return -ENOMEM; } #endif - if (hlist_empty(&chan->sc_unknown)) { + + { const struct lttng_event_desc *desc = &__event_desc___syscall_entry_unknown; struct lttng_event *event; @@ -1233,16 +1220,19 @@ int lttng_syscalls_register_event(struct lttng_event_enabler *event_enabler, voi ev.instrumentation = LTTNG_KERNEL_SYSCALL; ev.u.syscall.entryexit = LTTNG_KERNEL_SYSCALL_ENTRY; ev.u.syscall.abi = LTTNG_KERNEL_SYSCALL_ABI_NATIVE; - event = _lttng_event_create(chan, &ev, filter, desc, - ev.instrumentation); - WARN_ON_ONCE(!event); + event = _lttng_event_create(container, &ev, &event_enabler->key, filter, + desc, ev.instrumentation, + event_enabler->base.user_token); if (IS_ERR(event)) { - return PTR_ERR(event); + if (PTR_ERR(event) != -EEXIST) + return PTR_ERR(event); + /* Skip if event is already found. */ + } else { + hlist_add_head(&event->u.syscall.node, &container->sc_unknown); } - hlist_add_head(&event->u.syscall.node, &chan->sc_unknown); } - if (hlist_empty(&chan->sc_compat_unknown)) { + { const struct lttng_event_desc *desc = &__event_desc___compat_syscall_entry_unknown; struct lttng_event *event; @@ -1253,16 +1243,19 @@ int lttng_syscalls_register_event(struct lttng_event_enabler *event_enabler, voi ev.instrumentation = LTTNG_KERNEL_SYSCALL; ev.u.syscall.entryexit = LTTNG_KERNEL_SYSCALL_ENTRY; ev.u.syscall.abi = LTTNG_KERNEL_SYSCALL_ABI_COMPAT; - event = _lttng_event_create(chan, &ev, filter, desc, - ev.instrumentation); - WARN_ON_ONCE(!event); + event = _lttng_event_create(container, &ev, &event_enabler->key, filter, + desc, ev.instrumentation, + event_enabler->base.user_token); if (IS_ERR(event)) { - return PTR_ERR(event); + if (PTR_ERR(event) != -EEXIST) + return PTR_ERR(event); + /* Skip if event is already found. */ + } else { + hlist_add_head(&event->u.syscall.node, &container->sc_compat_unknown); } - hlist_add_head(&event->u.syscall.node, &chan->sc_compat_unknown); } - if (hlist_empty(&chan->compat_sc_exit_unknown)) { + { const struct lttng_event_desc *desc = &__event_desc___compat_syscall_exit_unknown; struct lttng_event *event; @@ -1273,16 +1266,19 @@ int lttng_syscalls_register_event(struct lttng_event_enabler *event_enabler, voi ev.instrumentation = LTTNG_KERNEL_SYSCALL; ev.u.syscall.entryexit = LTTNG_KERNEL_SYSCALL_EXIT; ev.u.syscall.abi = LTTNG_KERNEL_SYSCALL_ABI_COMPAT; - event = _lttng_event_create(chan, &ev, filter, desc, - ev.instrumentation); - WARN_ON_ONCE(!event); + event = _lttng_event_create(container, &ev, &event_enabler->key, + filter, desc, ev.instrumentation, + event_enabler->base.user_token); if (IS_ERR(event)) { - return PTR_ERR(event); + if (PTR_ERR(event) != -EEXIST) + return PTR_ERR(event); + /* Skip if event is already found. */ + } else { + hlist_add_head(&event->u.syscall.node, &container->compat_sc_exit_unknown); } - hlist_add_head(&event->u.syscall.node, &chan->compat_sc_exit_unknown); } - if (hlist_empty(&chan->sc_exit_unknown)) { + { const struct lttng_event_desc *desc = &__event_desc___syscall_exit_unknown; struct lttng_event *event; @@ -1293,64 +1289,67 @@ int lttng_syscalls_register_event(struct lttng_event_enabler *event_enabler, voi ev.instrumentation = LTTNG_KERNEL_SYSCALL; ev.u.syscall.entryexit = LTTNG_KERNEL_SYSCALL_EXIT; ev.u.syscall.abi = LTTNG_KERNEL_SYSCALL_ABI_NATIVE; - event = _lttng_event_create(chan, &ev, filter, desc, - ev.instrumentation); - WARN_ON_ONCE(!event); + event = _lttng_event_create(container, &ev, &event_enabler->key, filter, + desc, ev.instrumentation, + event_enabler->base.user_token); if (IS_ERR(event)) { - return PTR_ERR(event); + if (PTR_ERR(event) != -EEXIST) + return PTR_ERR(event); + /* Skip if event is already found. */ + } else { + hlist_add_head(&event->u.syscall.node, &container->sc_exit_unknown); } - hlist_add_head(&event->u.syscall.node, &chan->sc_exit_unknown); } ret = lttng_create_syscall_event_if_missing(sc_table, ARRAY_SIZE(sc_table), - chan->sc_table, event_enabler, filter, SC_TYPE_ENTRY); + container->sc_table, event_enabler, filter, SC_TYPE_ENTRY); if (ret) return ret; ret = lttng_create_syscall_event_if_missing(sc_exit_table, ARRAY_SIZE(sc_exit_table), - chan->sc_exit_table, event_enabler, filter, SC_TYPE_EXIT); + container->sc_exit_table, event_enabler, filter, SC_TYPE_EXIT); if (ret) return ret; #ifdef CONFIG_COMPAT ret = lttng_create_syscall_event_if_missing(compat_sc_table, ARRAY_SIZE(compat_sc_table), - chan->compat_sc_table, event_enabler, filter, + container->compat_sc_table, event_enabler, filter, SC_TYPE_COMPAT_ENTRY); if (ret) return ret; ret = lttng_create_syscall_event_if_missing(compat_sc_exit_table, ARRAY_SIZE(compat_sc_exit_table), - chan->compat_sc_exit_table, event_enabler, filter, + container->compat_sc_exit_table, event_enabler, filter, SC_TYPE_COMPAT_EXIT); if (ret) return ret; #endif - if (!chan->sc_filter) { - chan->sc_filter = kzalloc(sizeof(struct lttng_syscall_filter), + if (!container->sc_filter) { + container->sc_filter = kzalloc(sizeof(struct lttng_syscall_filter), GFP_KERNEL); - if (!chan->sc_filter) + if (!container->sc_filter) return -ENOMEM; } - if (!chan->sys_enter_registered) { + if (!container->sys_enter_registered) { ret = lttng_wrapper_tracepoint_probe_register("sys_enter", - (void *) syscall_entry_event_probe, chan); + (void *) syscall_entry_event_probe, container); if (ret) return ret; - chan->sys_enter_registered = 1; + container->sys_enter_registered = 1; } /* * We change the name of sys_exit tracepoint due to namespace * conflict with sys_exit syscall entry. */ - if (!chan->sys_exit_registered) { + if (!container->sys_exit_registered) { ret = lttng_wrapper_tracepoint_probe_register("sys_exit", - (void *) syscall_exit_event_probe, chan); + (void *) syscall_exit_event_probe, container); if (ret) { WARN_ON_ONCE(lttng_wrapper_tracepoint_probe_unregister("sys_enter", - (void *) syscall_entry_event_probe, chan)); + (void *) syscall_entry_event_probe, container)); return ret; } - chan->sys_exit_registered = 1; + container->sys_exit_registered = 1; } return ret; } @@ -1572,8 +1571,8 @@ static int create_matching_event_notifiers( continue; } - if (!lttng_desc_match_enabler(desc, - lttng_event_notifier_enabler_as_enabler(event_notifier_enabler))) + if (lttng_desc_match_enabler(desc, + lttng_event_notifier_enabler_as_enabler(event_notifier_enabler)) <= 0) continue; /* @@ -1628,7 +1627,6 @@ static int create_matching_event_notifiers( end: return ret; - } int lttng_syscals_create_matching_event_notifiers( @@ -1729,38 +1727,38 @@ int lttng_syscalls_unregister_event_notifier_group( return 0; } -int lttng_syscalls_unregister_channel(struct lttng_channel *chan) +int lttng_syscalls_unregister_event_container(struct lttng_event_container *container) { int ret; - if (!chan->sc_table) + if (!container->sc_table) return 0; - if (chan->sys_enter_registered) { + if (container->sys_enter_registered) { ret = lttng_wrapper_tracepoint_probe_unregister("sys_enter", - (void *) syscall_entry_event_probe, chan); + (void *) syscall_entry_event_probe, container); if (ret) return ret; - chan->sys_enter_registered = 0; + container->sys_enter_registered = 0; } - if (chan->sys_exit_registered) { + if (container->sys_exit_registered) { ret = lttng_wrapper_tracepoint_probe_unregister("sys_exit", - (void *) syscall_exit_event_probe, chan); + (void *) syscall_exit_event_probe, container); if (ret) return ret; - chan->sys_exit_registered = 0; + container->sys_exit_registered = 0; } return 0; } -int lttng_syscalls_destroy_event(struct lttng_channel *chan) +int lttng_syscalls_destroy_event_container(struct lttng_event_container *container) { - kfree(chan->sc_table); - kfree(chan->sc_exit_table); + kfree(container->sc_table); + kfree(container->sc_exit_table); #ifdef CONFIG_COMPAT - kfree(chan->compat_sc_table); - kfree(chan->compat_sc_exit_table); + kfree(container->compat_sc_table); + kfree(container->compat_sc_exit_table); #endif - kfree(chan->sc_filter); + kfree(container->sc_filter); return 0; } @@ -1965,12 +1963,12 @@ end: } int lttng_syscall_filter_enable_event( - struct lttng_channel *channel, + struct lttng_event_container *container, struct lttng_event *event) { WARN_ON_ONCE(event->instrumentation != LTTNG_KERNEL_SYSCALL); - return lttng_syscall_filter_enable(channel->sc_filter, + return lttng_syscall_filter_enable(container->sc_filter, event->desc->name, event->u.syscall.abi, event->u.syscall.entryexit); } @@ -2053,10 +2051,10 @@ int lttng_syscall_filter_disable_event_notifier( } int lttng_syscall_filter_disable_event( - struct lttng_channel *channel, + struct lttng_event_container *container, struct lttng_event *event) { - return lttng_syscall_filter_disable(channel->sc_filter, + return lttng_syscall_filter_disable(container->sc_filter, event->desc->name, event->u.syscall.abi, event->u.syscall.entryexit); } @@ -2175,7 +2173,7 @@ const struct file_operations lttng_syscall_list_fops = { /* * A syscall is enabled if it is traced for either entry or exit. */ -long lttng_channel_syscall_mask(struct lttng_channel *channel, +long lttng_event_container_syscall_mask(struct lttng_event_container *container, struct lttng_kernel_syscall_mask __user *usyscall_mask) { uint32_t len, sc_tables_len, bitmask_len; @@ -2195,14 +2193,14 @@ long lttng_channel_syscall_mask(struct lttng_channel *channel, tmp_mask = kzalloc(bitmask_len, GFP_KERNEL); if (!tmp_mask) return -ENOMEM; - filter = channel->sc_filter; + filter = container->sc_filter; for (bit = 0; bit < ARRAY_SIZE(sc_table); bit++) { char state; - if (channel->sc_table) { - if (!(READ_ONCE(channel->syscall_all_entry) - || READ_ONCE(channel->syscall_all_exit)) && filter) + if (container->sc_table) { + if (!(READ_ONCE(container->syscall_all_entry) + || READ_ONCE(container->syscall_all_exit)) && filter) state = test_bit(bit, filter->sc_entry) || test_bit(bit, filter->sc_exit); else @@ -2215,9 +2213,9 @@ long lttng_channel_syscall_mask(struct lttng_channel *channel, for (; bit < sc_tables_len; bit++) { char state; - if (channel->compat_sc_table) { - if (!(READ_ONCE(channel->syscall_all_entry) - || READ_ONCE(channel->syscall_all_exit)) && filter) + if (container->compat_sc_table) { + if (!(READ_ONCE(container->syscall_all_entry) + || READ_ONCE(container->syscall_all_exit)) && filter) state = test_bit(bit - ARRAY_SIZE(sc_table), filter->sc_compat_entry) || test_bit(bit - ARRAY_SIZE(sc_table), diff --git a/src/probes/lttng-kprobes.c b/src/probes/lttng-kprobes.c index 6824088c..b7d1c85d 100644 --- a/src/probes/lttng-kprobes.c +++ b/src/probes/lttng-kprobes.c @@ -26,26 +26,42 @@ int lttng_kprobes_event_handler_pre(struct kprobe *p, struct pt_regs *regs) .event = event, .interruptible = !lttng_regs_irqs_disabled(regs), }; - struct lttng_channel *chan = event->chan; - struct lib_ring_buffer_ctx ctx; + struct lttng_event_container *container = event->container; int ret; unsigned long data = (unsigned long) p->addr; - if (unlikely(!LTTNG_READ_ONCE(chan->session->active))) + if (unlikely(!LTTNG_READ_ONCE(container->session->active))) return 0; - if (unlikely(!LTTNG_READ_ONCE(chan->enabled))) + if (unlikely(!LTTNG_READ_ONCE(container->enabled))) return 0; if (unlikely(!LTTNG_READ_ONCE(event->enabled))) return 0; - lib_ring_buffer_ctx_init(&ctx, chan->chan, <tng_probe_ctx, sizeof(data), - lttng_alignof(data), -1); - ret = chan->ops->event_reserve(&ctx, event->id); - if (ret < 0) - return 0; - lib_ring_buffer_align_ctx(&ctx, lttng_alignof(data)); - chan->ops->event_write(&ctx, &data, sizeof(data)); - chan->ops->event_commit(&ctx); + switch (container->type) { + case LTTNG_EVENT_CONTAINER_CHANNEL: + { + struct lttng_channel *chan = lttng_event_container_get_channel(container); + struct lib_ring_buffer_ctx ctx; + + lib_ring_buffer_ctx_init(&ctx, chan->chan, <tng_probe_ctx, sizeof(data), + lttng_alignof(data), -1); + ret = chan->ops->event_reserve(&ctx, event->id); + if (ret < 0) + return 0; + lib_ring_buffer_align_ctx(&ctx, lttng_alignof(data)); + chan->ops->event_write(&ctx, &data, sizeof(data)); + chan->ops->event_commit(&ctx); + break; + } + case LTTNG_EVENT_CONTAINER_COUNTER: + { + struct lttng_counter *counter = lttng_event_container_get_counter(container); + size_t index = event->id; + + (void) counter->ops->counter_add(counter->counter, &index, 1); + break; + } + } return 0; } @@ -54,11 +70,13 @@ int lttng_kprobes_event_notifier_handler_pre(struct kprobe *p, struct pt_regs *r { struct lttng_event_notifier *event_notifier = container_of(p, struct lttng_event_notifier, u.kprobe.kp); + struct lttng_kernel_notifier_ctx notif_ctx; if (unlikely(!READ_ONCE(event_notifier->enabled))) return 0; - event_notifier->send_notification(event_notifier, NULL, NULL); + notif_ctx.eval_capture = LTTNG_READ_ONCE(event_notifier->eval_capture); + event_notifier->send_notification(event_notifier, NULL, NULL, ¬if_ctx); return 0; } diff --git a/src/probes/lttng-kretprobes.c b/src/probes/lttng-kretprobes.c index 24cb52e4..b3e9f627 100644 --- a/src/probes/lttng-kretprobes.c +++ b/src/probes/lttng-kretprobes.c @@ -43,32 +43,48 @@ int _lttng_kretprobes_handler(struct kretprobe_instance *krpi, .event = event, .interruptible = !lttng_regs_irqs_disabled(regs), }; - struct lttng_channel *chan = event->chan; - struct lib_ring_buffer_ctx ctx; + struct lttng_event_container *container = event->container; int ret; struct { unsigned long ip; unsigned long parent_ip; } payload; - if (unlikely(!LTTNG_READ_ONCE(chan->session->active))) + if (unlikely(!LTTNG_READ_ONCE(container->session->active))) return 0; - if (unlikely(!LTTNG_READ_ONCE(chan->enabled))) + if (unlikely(!LTTNG_READ_ONCE(container->enabled))) return 0; if (unlikely(!LTTNG_READ_ONCE(event->enabled))) return 0; - payload.ip = (unsigned long) lttng_get_kretprobe(krpi)->kp.addr; - payload.parent_ip = (unsigned long) krpi->ret_addr; + switch (container->type) { + case LTTNG_EVENT_CONTAINER_CHANNEL: + { + struct lttng_channel *chan = lttng_event_container_get_channel(container); + struct lib_ring_buffer_ctx ctx; - lib_ring_buffer_ctx_init(&ctx, chan->chan, <tng_probe_ctx, sizeof(payload), - lttng_alignof(payload), -1); - ret = chan->ops->event_reserve(&ctx, event->id); - if (ret < 0) - return 0; - lib_ring_buffer_align_ctx(&ctx, lttng_alignof(payload)); - chan->ops->event_write(&ctx, &payload, sizeof(payload)); - chan->ops->event_commit(&ctx); + payload.ip = (unsigned long) lttng_get_kretprobe(krpi)->kp.addr; + payload.parent_ip = (unsigned long) krpi->ret_addr; + + lib_ring_buffer_ctx_init(&ctx, chan->chan, <tng_probe_ctx, sizeof(payload), + lttng_alignof(payload), -1); + ret = chan->ops->event_reserve(&ctx, event->id); + if (ret < 0) + return 0; + lib_ring_buffer_align_ctx(&ctx, lttng_alignof(payload)); + chan->ops->event_write(&ctx, &payload, sizeof(payload)); + chan->ops->event_commit(&ctx); + break; + } + case LTTNG_EVENT_CONTAINER_COUNTER: + { + struct lttng_counter *counter = lttng_event_container_get_counter(container); + size_t index = event->id; + + (void) counter->ops->counter_add(counter->counter, &index, 1); + break; + } + } return 0; } diff --git a/src/probes/lttng-uprobes.c b/src/probes/lttng-uprobes.c index 031bfa7c..fbed39eb 100644 --- a/src/probes/lttng-uprobes.c +++ b/src/probes/lttng-uprobes.c @@ -32,34 +32,50 @@ int lttng_uprobes_event_handler_pre(struct uprobe_consumer *uc, struct pt_regs * .event = event, .interruptible = !lttng_regs_irqs_disabled(regs), }; - struct lttng_channel *chan = event->chan; - struct lib_ring_buffer_ctx ctx; + struct lttng_event_container *container = event->container; int ret; struct { unsigned long ip; } payload; - if (unlikely(!LTTNG_READ_ONCE(chan->session->active))) + if (unlikely(!LTTNG_READ_ONCE(container->session->active))) return 0; - if (unlikely(!LTTNG_READ_ONCE(chan->enabled))) + if (unlikely(!LTTNG_READ_ONCE(container->enabled))) return 0; if (unlikely(!LTTNG_READ_ONCE(event->enabled))) return 0; - lib_ring_buffer_ctx_init(&ctx, chan->chan, <tng_probe_ctx, - sizeof(payload), lttng_alignof(payload), -1); + switch (container->type) { + case LTTNG_EVENT_CONTAINER_CHANNEL: + { + struct lttng_channel *chan = lttng_event_container_get_channel(container); + struct lib_ring_buffer_ctx ctx; - ret = chan->ops->event_reserve(&ctx, event->id); - if (ret < 0) - return 0; + lib_ring_buffer_ctx_init(&ctx, chan->chan, <tng_probe_ctx, + sizeof(payload), lttng_alignof(payload), -1); + + ret = chan->ops->event_reserve(&ctx, event->id); + if (ret < 0) + return 0; + + /* Event payload. */ + payload.ip = (unsigned long)instruction_pointer(regs); - /* Event payload. */ - payload.ip = (unsigned long)instruction_pointer(regs); + lib_ring_buffer_align_ctx(&ctx, lttng_alignof(payload)); + chan->ops->event_write(&ctx, &payload, sizeof(payload)); + chan->ops->event_commit(&ctx); + break; + } + case LTTNG_EVENT_CONTAINER_COUNTER: + { + struct lttng_counter *counter = lttng_event_container_get_counter(container); + size_t index = event->id; - lib_ring_buffer_align_ctx(&ctx, lttng_alignof(payload)); - chan->ops->event_write(&ctx, &payload, sizeof(payload)); - chan->ops->event_commit(&ctx); + (void) counter->ops->counter_add(counter->counter, &index, 1); + break; + } + } return 0; } @@ -69,11 +85,13 @@ int lttng_uprobes_event_notifier_handler_pre(struct uprobe_consumer *uc, struct struct lttng_uprobe_handler *uprobe_handler = container_of(uc, struct lttng_uprobe_handler, up_consumer); struct lttng_event_notifier *event_notifier = uprobe_handler->u.event_notifier; + struct lttng_kernel_notifier_ctx notif_ctx; if (unlikely(!READ_ONCE(event_notifier->enabled))) return 0; - event_notifier->send_notification(event_notifier, NULL, NULL); + notif_ctx.eval_capture = LTTNG_READ_ONCE(event_notifier->eval_capture); + event_notifier->send_notification(event_notifier, NULL, NULL, ¬if_ctx); return 0; }