From db34449e0268698d2a21c68bfd96f62dc6e84c16 Mon Sep 17 00:00:00 2001 From: Michael Jeanson Date: Tue, 9 Feb 2021 11:04:25 -0500 Subject: [PATCH 01/16] fix: cast LTTNG_KERNEL_VERSION/LTTNG_LINUX_VERSION_CODE to uint64_t Cast our version macros to an unsigned 64bits value to prevent overflowing when we append distro specific version information. Signed-off-by: Michael Jeanson Signed-off-by: Mathieu Desnoyers Change-Id: Ia42a5dc0dfddf64515aea144283af5cc0c3b97e0 --- include/lttng/kernel-version.h | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/include/lttng/kernel-version.h b/include/lttng/kernel-version.h index a41829f0..747a674f 100644 --- a/include/lttng/kernel-version.h +++ b/include/lttng/kernel-version.h @@ -11,6 +11,7 @@ #define _LTTNG_KERNEL_VERSION_H #include +#include #include /* @@ -40,19 +41,22 @@ * of LINUX_VERSION_CODE from the kernel headers and allocate 16bits. * Otherwise, keep using the version code from the headers to minimise the * behavior change and avoid regressions. + * + * Cast the result to uint64_t to prevent overflowing when we append distro + * specific version information. */ #if (LTTNG_LINUX_PATCH >= 256) #define LTTNG_KERNEL_VERSION(a, b, c) \ - (((a) << 24) + ((b) << 16) + (c)) + ((((a) << 24) + ((b) << 16) + (c)) * 1ULL) #define LTTNG_LINUX_VERSION_CODE \ LTTNG_KERNEL_VERSION(LTTNG_LINUX_MAJOR, LTTNG_LINUX_MINOR, LTTNG_LINUX_PATCH) #else -#define LTTNG_KERNEL_VERSION(a, b, c) KERNEL_VERSION(a, b, c) -#define LTTNG_LINUX_VERSION_CODE LINUX_VERSION_CODE +#define LTTNG_KERNEL_VERSION(a, b, c) (KERNEL_VERSION(a, b, c) * 1ULL) +#define LTTNG_LINUX_VERSION_CODE (LINUX_VERSION_CODE * 1ULL) #endif -- 2.34.1 From b737968d5f0badac76586270e7df4ce6447adc5a Mon Sep 17 00:00:00 2001 From: Michael Jeanson Date: Mon, 8 Feb 2021 15:32:47 -0500 Subject: [PATCH 02/16] fix: RT_PATCH_VERSION is close to overflow We allocated only 8bits for RT_PATCH_VERSION in LTTNG_RT_VERSION_CODE, the current RT patch version for the 4.4 branch is currently 214 which is getting close to 256. Bump it to 16bits to avoid breakage in the future. Signed-off-by: Michael Jeanson Signed-off-by: Mathieu Desnoyers Change-Id: I0666bbe996854696ac98e025eb02e5fced0540b1 --- include/lttng/kernel-version.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/lttng/kernel-version.h b/include/lttng/kernel-version.h index 747a674f..88f621e9 100644 --- a/include/lttng/kernel-version.h +++ b/include/lttng/kernel-version.h @@ -166,11 +166,11 @@ /* RT patch */ #define LTTNG_RT_KERNEL_VERSION(a, b, c, d) \ - (((LTTNG_KERNEL_VERSION(a, b, c)) << 8) + (d)) + (((LTTNG_KERNEL_VERSION(a, b, c)) << 16) + (d)) #ifdef RT_PATCH_VERSION #define LTTNG_RT_VERSION_CODE \ - ((LTTNG_LINUX_VERSION_CODE << 8) + RT_PATCH_VERSION) + ((LTTNG_LINUX_VERSION_CODE << 16) + RT_PATCH_VERSION) #else #define LTTNG_RT_VERSION_CODE 0 #endif -- 2.34.1 From 297b2e1fa7a781674b2d48b1d101ee16ee281b9b Mon Sep 17 00:00:00 2001 From: Michael Jeanson Date: Tue, 9 Feb 2021 11:25:57 -0500 Subject: [PATCH 03/16] fix: Add one digit to SLES minor release version Signed-off-by: Michael Jeanson Signed-off-by: Mathieu Desnoyers Change-Id: Ia4b67b38377bdd997b754b92b09b96934940a013 --- include/lttng/kernel-version.h | 4 ++-- scripts/abi-sle-version.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/include/lttng/kernel-version.h b/include/lttng/kernel-version.h index 88f621e9..d3fce8e7 100644 --- a/include/lttng/kernel-version.h +++ b/include/lttng/kernel-version.h @@ -128,11 +128,11 @@ /* SUSE Linux enterprise */ #define LTTNG_SLE_KERNEL_VERSION(a, b, c, d, e, f) \ - (((LTTNG_KERNEL_VERSION(a, b, c)) * 10000000ULL) + ((d) * 10000) + ((e) * 100) + (f)) + (((LTTNG_KERNEL_VERSION(a, b, c)) * 100000000ULL) + ((d) * 100000) + ((e) * 100) + (f)) #ifdef SLE_API_VERSION #define LTTNG_SLE_VERSION_CODE \ - ((LTTNG_LINUX_VERSION_CODE * 10000000ULL) + SLE_API_VERSION) + ((LTTNG_LINUX_VERSION_CODE * 100000000ULL) + SLE_API_VERSION) #else #define LTTNG_SLE_VERSION_CODE 0 #endif diff --git a/scripts/abi-sle-version.sh b/scripts/abi-sle-version.sh index 0d7254ac..e079e065 100755 --- a/scripts/abi-sle-version.sh +++ b/scripts/abi-sle-version.sh @@ -38,6 +38,6 @@ if [ "x$SLE_RELEASE_PATCH" = "x" ]; then fi # Combine all update numbers into one -SLE_API_VERSION="$((SLE_RELEASE_MAJOR * 10000 + SLE_RELEASE_MINOR * 100 + SLE_RELEASE_PATCH))" +SLE_API_VERSION="$((SLE_RELEASE_MAJOR * 100000 + SLE_RELEASE_MINOR * 100 + SLE_RELEASE_PATCH))" echo ${SLE_API_VERSION} -- 2.34.1 From aec7897f7dd400daedf54d294b39dda24d9a549f Mon Sep 17 00:00:00 2001 From: Michael Jeanson Date: Tue, 9 Feb 2021 11:28:27 -0500 Subject: [PATCH 04/16] fix: Add one digit to RHEL major release version Signed-off-by: Michael Jeanson Signed-off-by: Mathieu Desnoyers Change-Id: I4e8b10ee985db59f7795e026486b3d47b76b5728 --- include/lttng/kernel-version.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/lttng/kernel-version.h b/include/lttng/kernel-version.h index d3fce8e7..d292c087 100644 --- a/include/lttng/kernel-version.h +++ b/include/lttng/kernel-version.h @@ -109,11 +109,11 @@ /* RHEL */ #define LTTNG_RHEL_KERNEL_VERSION(a, b, c, d, e, f) \ - (((LTTNG_KERNEL_VERSION(a, b, c)) * 10000000ULL) + ((d) * 10000) + ((e) * 100) + (f)) + (((LTTNG_KERNEL_VERSION(a, b, c)) * 100000000ULL) + ((d) * 10000) + ((e) * 100) + (f)) #ifdef RHEL_API_VERSION #define LTTNG_RHEL_VERSION_CODE \ - ((LTTNG_LINUX_VERSION_CODE * 10000000ULL) + RHEL_API_VERSION) + ((LTTNG_LINUX_VERSION_CODE * 100000000ULL) + RHEL_API_VERSION) #else #define LTTNG_RHEL_VERSION_CODE 0 #endif -- 2.34.1 From cc584d6692836e86ca05df7cf58a1873b11fca56 Mon Sep 17 00:00:00 2001 From: Mathieu Desnoyers Date: Fri, 5 Feb 2021 16:21:47 -0500 Subject: [PATCH 05/16] Fix: writeback: out-of-bound reads Use ctf_string rather than ctf_array_text for name fields, because the source strings are not guaranteed to be at least 32 bytes. Signed-off-by: Mathieu Desnoyers --- include/instrumentation/events/writeback.h | 76 +++++++++++----------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/include/instrumentation/events/writeback.h b/include/instrumentation/events/writeback.h index b6dfd4c0..3d5df91f 100644 --- a/include/instrumentation/events/writeback.h +++ b/include/instrumentation/events/writeback.h @@ -96,8 +96,8 @@ LTTNG_TRACEPOINT_EVENT(writeback_dirty_page, TP_PROTO(struct page *page, struct address_space *mapping), TP_ARGS(page, mapping), TP_FIELDS( - ctf_array_text(char, name, - mapping ? dev_name(lttng_inode_to_bdi(mapping->host)->dev) : "(unknown)", 32) + ctf_string(name, + mapping ? dev_name(lttng_inode_to_bdi(mapping->host)->dev) : "(unknown)") ctf_integer(unsigned long, ino, mapping ? mapping->host->i_ino : 0) ctf_integer(pgoff_t, index, page->index) ) @@ -108,9 +108,9 @@ LTTNG_TRACEPOINT_EVENT_CLASS(writeback_dirty_inode_template, TP_ARGS(inode, flags), TP_FIELDS( /* may be called for files on pseudo FSes w/ unregistered bdi */ - ctf_array_text(char, name, + ctf_string(name, lttng_inode_to_bdi(inode)->dev ? - dev_name(lttng_inode_to_bdi(inode)->dev) : "(unknown)", 32) + dev_name(lttng_inode_to_bdi(inode)->dev) : "(unknown)") ctf_integer(unsigned long, ino, inode->i_ino) ctf_integer(unsigned long, state, inode->i_state) ctf_integer(unsigned long, flags, flags) @@ -128,8 +128,8 @@ LTTNG_TRACEPOINT_EVENT_CLASS(writeback_write_inode_template, TP_PROTO(struct inode *inode, struct writeback_control *wbc), TP_ARGS(inode, wbc), TP_FIELDS( - ctf_array_text(char, name, - dev_name(lttng_inode_to_bdi(inode)->dev), 32) + ctf_string(name, + dev_name(lttng_inode_to_bdi(inode)->dev)) ctf_integer(unsigned long, ino, inode->i_ino) ctf_integer(int, sync_mode, wbc->sync_mode) ) @@ -148,8 +148,8 @@ LTTNG_TRACEPOINT_EVENT(writeback_dirty_page, TP_PROTO(struct page *page, struct address_space *mapping), TP_ARGS(page, mapping), TP_FIELDS( - ctf_array_text(char, name, - mapping ? dev_name(mapping->backing_dev_info->dev) : "(unknown)", 32) + ctf_string(name, + mapping ? dev_name(mapping->backing_dev_info->dev) : "(unknown)") ctf_integer(unsigned long, ino, mapping ? mapping->host->i_ino : 0) ctf_integer(pgoff_t, index, page->index) ) @@ -160,10 +160,10 @@ LTTNG_TRACEPOINT_EVENT_CLASS(writeback_dirty_inode_template, TP_ARGS(inode, flags), TP_FIELDS( /* may be called for files on pseudo FSes w/ unregistered bdi */ - ctf_array_text(char, name, + ctf_string(name, inode->i_mapping->backing_dev_info->dev ? dev_name(inode->i_mapping->backing_dev_info->dev) - : "(unknown)", 32) + : "(unknown)") ctf_integer(unsigned long, ino, inode->i_ino) ctf_integer(unsigned long, flags, flags) ) @@ -179,8 +179,8 @@ LTTNG_TRACEPOINT_EVENT_CLASS(writeback_write_inode_template, TP_PROTO(struct inode *inode, struct writeback_control *wbc), TP_ARGS(inode, wbc), TP_FIELDS( - ctf_array_text(char, name, - dev_name(inode->i_mapping->backing_dev_info->dev), 32) + ctf_string(name, + dev_name(inode->i_mapping->backing_dev_info->dev)) ctf_integer(unsigned long, ino, inode->i_ino) ctf_integer(int, sync_mode, wbc->sync_mode) ) @@ -201,8 +201,8 @@ LTTNG_TRACEPOINT_EVENT_CLASS(writeback_work_class, TP_PROTO(struct bdi_writeback *wb, struct wb_writeback_work *work), TP_ARGS(wb, work), TP_FIELDS( - ctf_array_text(char, name, wb->bdi->dev ? dev_name(wb->bdi->dev) : - "(unknown)", 32) + ctf_string(name, wb->bdi->dev ? dev_name(wb->bdi->dev) : + "(unknown)") ) ) @@ -212,8 +212,8 @@ LTTNG_TRACEPOINT_EVENT_CLASS(writeback_work_class, TP_PROTO(struct backing_dev_info *bdi, struct wb_writeback_work *work), TP_ARGS(bdi, work), TP_FIELDS( - ctf_array_text(char, name, bdi->dev ? dev_name(bdi->dev) : - "(unknown)", 32) + ctf_string(name, bdi->dev ? dev_name(bdi->dev) : + "(unknown)") ) ) @@ -223,9 +223,9 @@ LTTNG_TRACEPOINT_EVENT_CLASS(writeback_work_class, TP_PROTO(struct backing_dev_info *bdi, struct wb_writeback_work *work), TP_ARGS(bdi, work), TP_FIELDS( - ctf_array_text(char, name, + ctf_string(name, dev_name(bdi->dev ? bdi->dev : - default_backing_dev_info.dev), 32) + default_backing_dev_info.dev)) ) ) @@ -270,8 +270,8 @@ LTTNG_TRACEPOINT_EVENT_CLASS(writeback_class, TP_PROTO(struct bdi_writeback *wb), TP_ARGS(wb), TP_FIELDS( - ctf_array_text(char, name, - dev_name(wb->bdi->dev), 32) + ctf_string(name, + dev_name(wb->bdi->dev)) ) ) @@ -290,8 +290,8 @@ LTTNG_TRACEPOINT_EVENT(writeback_bdi_register, TP_PROTO(struct backing_dev_info *bdi), TP_ARGS(bdi), TP_FIELDS( - ctf_array_text(char, name, - dev_name(bdi->dev), 32) + ctf_string(name, + dev_name(bdi->dev)) ) ) @@ -301,8 +301,8 @@ LTTNG_TRACEPOINT_EVENT_CLASS(writeback_class, TP_PROTO(struct backing_dev_info *bdi), TP_ARGS(bdi), TP_FIELDS( - ctf_array_text(char, name, - dev_name(bdi->dev), 32) + ctf_string(name, + dev_name(bdi->dev)) ) ) @@ -341,7 +341,7 @@ LTTNG_TRACEPOINT_EVENT_MAP(balance_dirty_written, TP_ARGS(bdi, written), TP_FIELDS( - ctf_array_text(char, name, dev_name(bdi->dev), 32) + ctf_string(name, dev_name(bdi->dev)) ctf_integer(int, written, written) ) ) @@ -351,7 +351,7 @@ LTTNG_TRACEPOINT_EVENT_CLASS(writeback_wbc_class, TP_PROTO(struct writeback_control *wbc, struct backing_dev_info *bdi), TP_ARGS(wbc, bdi), TP_FIELDS( - ctf_array_text(char, name, dev_name(bdi->dev), 32) + ctf_string(name, dev_name(bdi->dev)) ctf_integer(long, nr_to_write, wbc->nr_to_write) ctf_integer(long, pages_skipped, wbc->pages_skipped) ctf_integer(int, sync_mode, wbc->sync_mode) @@ -399,7 +399,7 @@ LTTNG_TRACEPOINT_EVENT(writeback_queue_io, int moved), TP_ARGS(wb, work, dirtied_before, moved), TP_FIELDS( - ctf_array_text(char, name, dev_name(wb->bdi->dev), 32) + ctf_string(name, dev_name(wb->bdi->dev)) ctf_integer(unsigned long, older, dirtied_before) ctf_integer(int, moved, moved) ) @@ -411,7 +411,7 @@ LTTNG_TRACEPOINT_EVENT(writeback_queue_io, int moved), TP_ARGS(wb, work, moved), TP_FIELDS( - ctf_array_text(char, name, dev_name(wb->bdi->dev), 32) + ctf_string(name, dev_name(wb->bdi->dev)) ctf_integer(int, moved, moved) ) ) @@ -422,7 +422,7 @@ LTTNG_TRACEPOINT_EVENT(writeback_queue_io, int moved), TP_ARGS(wb, older_than_this, moved), TP_FIELDS( - ctf_array_text(char, name, dev_name(wb->bdi->dev), 32) + ctf_string(name, dev_name(wb->bdi->dev)) ctf_integer(unsigned long, older, older_than_this ? *older_than_this : 0) ctf_integer(long, age, @@ -524,7 +524,7 @@ LTTNG_TRACEPOINT_EVENT_MAP(bdi_dirty_ratelimit, TP_ARGS(wb, dirty_rate, task_ratelimit), TP_FIELDS( - ctf_array_text(char, bdi, dev_name(wb->bdi->dev), 32) + ctf_string(bdi, dev_name(wb->bdi->dev)) ctf_integer(unsigned long, write_bw, KBps(wb->bdi->wb.write_bandwidth)) ctf_integer(unsigned long, avg_write_bw, KBps(wb->bdi->wb.avg_write_bandwidth)) ctf_integer(unsigned long, dirty_rate, KBps(dirty_rate)) @@ -548,7 +548,7 @@ LTTNG_TRACEPOINT_EVENT_MAP(bdi_dirty_ratelimit, TP_ARGS(bdi, dirty_rate, task_ratelimit), TP_FIELDS( - ctf_array_text(char, bdi, dev_name(bdi->dev), 32) + ctf_string(bdi, dev_name(bdi->dev)) ctf_integer(unsigned long, write_bw, KBps(bdi->wb.write_bandwidth)) ctf_integer(unsigned long, avg_write_bw, KBps(bdi->wb.avg_write_bandwidth)) ctf_integer(unsigned long, dirty_rate, KBps(dirty_rate)) @@ -572,7 +572,7 @@ LTTNG_TRACEPOINT_EVENT_MAP(bdi_dirty_ratelimit, TP_ARGS(bdi, dirty_rate, task_ratelimit), TP_FIELDS( - ctf_array_text(char, bdi, dev_name(bdi->dev), 32) + ctf_string(bdi, dev_name(bdi->dev)) ctf_integer(unsigned long, write_bw, KBps(bdi->write_bandwidth)) ctf_integer(unsigned long, avg_write_bw, KBps(bdi->avg_write_bandwidth)) ctf_integer(unsigned long, dirty_rate, KBps(dirty_rate)) @@ -610,7 +610,7 @@ LTTNG_TRACEPOINT_EVENT_MAP(balance_dirty_pages, ), TP_FIELDS( - ctf_array_text(char, bdi, dev_name(wb->bdi->dev), 32) + ctf_string(bdi, dev_name(wb->bdi->dev)) ctf_integer(unsigned long, limit, global_dirty_limit) ctf_integer(unsigned long, setpoint, (global_dirty_limit + (thresh + bg_thresh) / 2) / 2) @@ -668,7 +668,7 @@ LTTNG_TRACEPOINT_EVENT_MAP(balance_dirty_pages, ), TP_FIELDS( - ctf_array_text(char, bdi, dev_name(bdi->dev), 32) + ctf_string(bdi, dev_name(bdi->dev)) ctf_integer(unsigned long, limit, global_dirty_limit) ctf_integer(unsigned long, setpoint, (global_dirty_limit + (thresh + bg_thresh) / 2) / 2) @@ -707,8 +707,8 @@ LTTNG_TRACEPOINT_EVENT(writeback_sb_inodes_requeue, TP_ARGS(inode), TP_FIELDS( - ctf_array_text(char, name, - dev_name(lttng_inode_to_bdi(inode)->dev), 32) + ctf_string(name, + dev_name(lttng_inode_to_bdi(inode)->dev)) ctf_integer(unsigned long, ino, inode->i_ino) ctf_integer(unsigned long, state, inode->i_state) ctf_integer(unsigned long, dirtied_when, inode->dirtied_when) @@ -753,8 +753,8 @@ LTTNG_TRACEPOINT_EVENT_CLASS(writeback_single_inode_template, TP_ARGS(inode, wbc, nr_to_write), TP_FIELDS( - ctf_array_text(char, name, - dev_name(lttng_inode_to_bdi(inode)->dev), 32) + ctf_string(name, + dev_name(lttng_inode_to_bdi(inode)->dev)) ctf_integer(unsigned long, ino, inode->i_ino) ctf_integer(unsigned long, state, inode->i_state) ctf_integer(unsigned long, dirtied_when, inode->dirtied_when) -- 2.34.1 From b1fdcb19f150295c9db01ac6a52b95e627cb8903 Mon Sep 17 00:00:00 2001 From: Mathieu Desnoyers Date: Wed, 10 Feb 2021 11:45:42 -0500 Subject: [PATCH 06/16] fix: memcg: fix a crash in wb_workfn when a device disappears (5.6) See upstream commit: commit 68f23b89067fdf187763e75a56087550624fdbee ("memcg: fix a crash in wb_workfn when a device disappears") It is currently backported into stable branches 5.4 and 5.5, but appears to be missing from the 4.4, 4.9, 4.14, 4.19 LTS branches. Implement our own lttng_bdi_dev_name wrapper to provide this fix on builds against stable kernels which do not have this fix. There is one user-visible change with this commit: for builds against kernels < 4.4.0, the writeback_work_class events did use the default_backing_dev_info to handle cases where the device is NULL, writing "default" into the trace. This behavior is now aligned to match what is done in kernels >= 4.4.0, which is to write "(unknown)" into the name field. Link: https://lore.kernel.org/r/537870616.15400.1612973059419.JavaMail.zimbra@efficios.com> Signed-off-by: Mathieu Desnoyers Change-Id: I0823643aa2f9d4c2b9f2005748a2adfd4457979a --- include/instrumentation/events/writeback.h | 94 ++++++++++------------ 1 file changed, 41 insertions(+), 53 deletions(-) diff --git a/include/instrumentation/events/writeback.h b/include/instrumentation/events/writeback.h index 3d5df91f..f24237b2 100644 --- a/include/instrumentation/events/writeback.h +++ b/include/instrumentation/events/writeback.h @@ -14,6 +14,22 @@ #ifndef _TRACE_WRITEBACK_DEF_ #define _TRACE_WRITEBACK_DEF_ +#if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(5,6,0) || \ + LTTNG_KERNEL_RANGE(5,5,3, 5,6,0) || \ + LTTNG_KERNEL_RANGE(5,4,19, 5,5,0)) +static inline const char *lttng_bdi_dev_name(struct backing_dev_info *bdi) +{ + return bdi_dev_name(bdi); +} +#else +static inline const char *lttng_bdi_dev_name(struct backing_dev_info *bdi) +{ + if (!bdi || !bdi->dev) + return "(unknown)"; + return dev_name(bdi->dev); +} +#endif + /* * Vanilla kernels before 4.0 do not implement inode_to_bdi * RHEL kernels before 3.10.0-327.10.1 do not implement inode_to_bdi @@ -96,8 +112,7 @@ LTTNG_TRACEPOINT_EVENT(writeback_dirty_page, TP_PROTO(struct page *page, struct address_space *mapping), TP_ARGS(page, mapping), TP_FIELDS( - ctf_string(name, - mapping ? dev_name(lttng_inode_to_bdi(mapping->host)->dev) : "(unknown)") + ctf_string(name, lttng_bdi_dev_name(mapping ? lttng_inode_to_bdi(mapping->host) : NULL)) ctf_integer(unsigned long, ino, mapping ? mapping->host->i_ino : 0) ctf_integer(pgoff_t, index, page->index) ) @@ -108,9 +123,7 @@ LTTNG_TRACEPOINT_EVENT_CLASS(writeback_dirty_inode_template, TP_ARGS(inode, flags), TP_FIELDS( /* may be called for files on pseudo FSes w/ unregistered bdi */ - ctf_string(name, - lttng_inode_to_bdi(inode)->dev ? - dev_name(lttng_inode_to_bdi(inode)->dev) : "(unknown)") + ctf_string(name, lttng_bdi_dev_name(lttng_inode_to_bdi(inode))) ctf_integer(unsigned long, ino, inode->i_ino) ctf_integer(unsigned long, state, inode->i_state) ctf_integer(unsigned long, flags, flags) @@ -128,8 +141,7 @@ LTTNG_TRACEPOINT_EVENT_CLASS(writeback_write_inode_template, TP_PROTO(struct inode *inode, struct writeback_control *wbc), TP_ARGS(inode, wbc), TP_FIELDS( - ctf_string(name, - dev_name(lttng_inode_to_bdi(inode)->dev)) + ctf_string(name, lttng_bdi_dev_name(lttng_inode_to_bdi(inode))) ctf_integer(unsigned long, ino, inode->i_ino) ctf_integer(int, sync_mode, wbc->sync_mode) ) @@ -148,8 +160,7 @@ LTTNG_TRACEPOINT_EVENT(writeback_dirty_page, TP_PROTO(struct page *page, struct address_space *mapping), TP_ARGS(page, mapping), TP_FIELDS( - ctf_string(name, - mapping ? dev_name(mapping->backing_dev_info->dev) : "(unknown)") + ctf_string(name, lttng_bdi_dev_name(mapping ? mapping->backing_dev_info : NULL)) ctf_integer(unsigned long, ino, mapping ? mapping->host->i_ino : 0) ctf_integer(pgoff_t, index, page->index) ) @@ -160,10 +171,7 @@ LTTNG_TRACEPOINT_EVENT_CLASS(writeback_dirty_inode_template, TP_ARGS(inode, flags), TP_FIELDS( /* may be called for files on pseudo FSes w/ unregistered bdi */ - ctf_string(name, - inode->i_mapping->backing_dev_info->dev ? - dev_name(inode->i_mapping->backing_dev_info->dev) - : "(unknown)") + ctf_string(name, lttng_bdi_dev_name(inode->i_mapping->backing_dev_info)) ctf_integer(unsigned long, ino, inode->i_ino) ctf_integer(unsigned long, flags, flags) ) @@ -179,8 +187,7 @@ LTTNG_TRACEPOINT_EVENT_CLASS(writeback_write_inode_template, TP_PROTO(struct inode *inode, struct writeback_control *wbc), TP_ARGS(inode, wbc), TP_FIELDS( - ctf_string(name, - dev_name(inode->i_mapping->backing_dev_info->dev)) + ctf_string(name, lttng_bdi_dev_name(inode->i_mapping->backing_dev_info)) ctf_integer(unsigned long, ino, inode->i_ino) ctf_integer(int, sync_mode, wbc->sync_mode) ) @@ -201,35 +208,21 @@ LTTNG_TRACEPOINT_EVENT_CLASS(writeback_work_class, TP_PROTO(struct bdi_writeback *wb, struct wb_writeback_work *work), TP_ARGS(wb, work), TP_FIELDS( - ctf_string(name, wb->bdi->dev ? dev_name(wb->bdi->dev) : - "(unknown)") + ctf_string(name, lttng_bdi_dev_name(wb->bdi)) ) ) -#elif (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(4,0,0)) - -LTTNG_TRACEPOINT_EVENT_CLASS(writeback_work_class, - TP_PROTO(struct backing_dev_info *bdi, struct wb_writeback_work *work), - TP_ARGS(bdi, work), - TP_FIELDS( - ctf_string(name, bdi->dev ? dev_name(bdi->dev) : - "(unknown)") - ) -) - -#else /* #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(4,0,0)) */ +#else LTTNG_TRACEPOINT_EVENT_CLASS(writeback_work_class, TP_PROTO(struct backing_dev_info *bdi, struct wb_writeback_work *work), TP_ARGS(bdi, work), TP_FIELDS( - ctf_string(name, - dev_name(bdi->dev ? bdi->dev : - default_backing_dev_info.dev)) + ctf_string(name, lttng_bdi_dev_name(bdi)) ) ) -#endif /* #else #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(4,0,0)) */ +#endif /* #else if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(4,3,0)) */ #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(4,3,0)) @@ -270,8 +263,7 @@ LTTNG_TRACEPOINT_EVENT_CLASS(writeback_class, TP_PROTO(struct bdi_writeback *wb), TP_ARGS(wb), TP_FIELDS( - ctf_string(name, - dev_name(wb->bdi->dev)) + ctf_string(name, lttng_bdi_dev_name(wb->bdi)) ) ) @@ -290,8 +282,7 @@ LTTNG_TRACEPOINT_EVENT(writeback_bdi_register, TP_PROTO(struct backing_dev_info *bdi), TP_ARGS(bdi), TP_FIELDS( - ctf_string(name, - dev_name(bdi->dev)) + ctf_string(name, lttng_bdi_dev_name(bdi)) ) ) @@ -301,8 +292,7 @@ LTTNG_TRACEPOINT_EVENT_CLASS(writeback_class, TP_PROTO(struct backing_dev_info *bdi), TP_ARGS(bdi), TP_FIELDS( - ctf_string(name, - dev_name(bdi->dev)) + ctf_string(name, lttng_bdi_dev_name(bdi)) ) ) @@ -341,7 +331,7 @@ LTTNG_TRACEPOINT_EVENT_MAP(balance_dirty_written, TP_ARGS(bdi, written), TP_FIELDS( - ctf_string(name, dev_name(bdi->dev)) + ctf_string(name, lttng_bdi_dev_name(bdi)) ctf_integer(int, written, written) ) ) @@ -351,7 +341,7 @@ LTTNG_TRACEPOINT_EVENT_CLASS(writeback_wbc_class, TP_PROTO(struct writeback_control *wbc, struct backing_dev_info *bdi), TP_ARGS(wbc, bdi), TP_FIELDS( - ctf_string(name, dev_name(bdi->dev)) + ctf_string(name, lttng_bdi_dev_name(bdi)) ctf_integer(long, nr_to_write, wbc->nr_to_write) ctf_integer(long, pages_skipped, wbc->pages_skipped) ctf_integer(int, sync_mode, wbc->sync_mode) @@ -399,7 +389,7 @@ LTTNG_TRACEPOINT_EVENT(writeback_queue_io, int moved), TP_ARGS(wb, work, dirtied_before, moved), TP_FIELDS( - ctf_string(name, dev_name(wb->bdi->dev)) + ctf_string(name, lttng_bdi_dev_name(wb->bdi)) ctf_integer(unsigned long, older, dirtied_before) ctf_integer(int, moved, moved) ) @@ -411,7 +401,7 @@ LTTNG_TRACEPOINT_EVENT(writeback_queue_io, int moved), TP_ARGS(wb, work, moved), TP_FIELDS( - ctf_string(name, dev_name(wb->bdi->dev)) + ctf_string(name, lttng_bdi_dev_name(wb->bdi)) ctf_integer(int, moved, moved) ) ) @@ -422,7 +412,7 @@ LTTNG_TRACEPOINT_EVENT(writeback_queue_io, int moved), TP_ARGS(wb, older_than_this, moved), TP_FIELDS( - ctf_string(name, dev_name(wb->bdi->dev)) + ctf_string(name, lttng_bdi_dev_name(wb->bdi)) ctf_integer(unsigned long, older, older_than_this ? *older_than_this : 0) ctf_integer(long, age, @@ -524,7 +514,7 @@ LTTNG_TRACEPOINT_EVENT_MAP(bdi_dirty_ratelimit, TP_ARGS(wb, dirty_rate, task_ratelimit), TP_FIELDS( - ctf_string(bdi, dev_name(wb->bdi->dev)) + ctf_string(bdi, lttng_bdi_dev_name(wb->bdi)) ctf_integer(unsigned long, write_bw, KBps(wb->bdi->wb.write_bandwidth)) ctf_integer(unsigned long, avg_write_bw, KBps(wb->bdi->wb.avg_write_bandwidth)) ctf_integer(unsigned long, dirty_rate, KBps(dirty_rate)) @@ -548,7 +538,7 @@ LTTNG_TRACEPOINT_EVENT_MAP(bdi_dirty_ratelimit, TP_ARGS(bdi, dirty_rate, task_ratelimit), TP_FIELDS( - ctf_string(bdi, dev_name(bdi->dev)) + ctf_string(bdi, lttng_bdi_dev_name(bdi)) ctf_integer(unsigned long, write_bw, KBps(bdi->wb.write_bandwidth)) ctf_integer(unsigned long, avg_write_bw, KBps(bdi->wb.avg_write_bandwidth)) ctf_integer(unsigned long, dirty_rate, KBps(dirty_rate)) @@ -572,7 +562,7 @@ LTTNG_TRACEPOINT_EVENT_MAP(bdi_dirty_ratelimit, TP_ARGS(bdi, dirty_rate, task_ratelimit), TP_FIELDS( - ctf_string(bdi, dev_name(bdi->dev)) + ctf_string(bdi, lttng_bdi_dev_name(bdi)) ctf_integer(unsigned long, write_bw, KBps(bdi->write_bandwidth)) ctf_integer(unsigned long, avg_write_bw, KBps(bdi->avg_write_bandwidth)) ctf_integer(unsigned long, dirty_rate, KBps(dirty_rate)) @@ -610,7 +600,7 @@ LTTNG_TRACEPOINT_EVENT_MAP(balance_dirty_pages, ), TP_FIELDS( - ctf_string(bdi, dev_name(wb->bdi->dev)) + ctf_string(bdi, lttng_bdi_dev_name(wb->bdi)) ctf_integer(unsigned long, limit, global_dirty_limit) ctf_integer(unsigned long, setpoint, (global_dirty_limit + (thresh + bg_thresh) / 2) / 2) @@ -668,7 +658,7 @@ LTTNG_TRACEPOINT_EVENT_MAP(balance_dirty_pages, ), TP_FIELDS( - ctf_string(bdi, dev_name(bdi->dev)) + ctf_string(bdi, lttng_bdi_dev_name(bdi)) ctf_integer(unsigned long, limit, global_dirty_limit) ctf_integer(unsigned long, setpoint, (global_dirty_limit + (thresh + bg_thresh) / 2) / 2) @@ -707,8 +697,7 @@ LTTNG_TRACEPOINT_EVENT(writeback_sb_inodes_requeue, TP_ARGS(inode), TP_FIELDS( - ctf_string(name, - dev_name(lttng_inode_to_bdi(inode)->dev)) + ctf_string(name, lttng_bdi_dev_name(lttng_inode_to_bdi(inode))) ctf_integer(unsigned long, ino, inode->i_ino) ctf_integer(unsigned long, state, inode->i_state) ctf_integer(unsigned long, dirtied_when, inode->dirtied_when) @@ -753,8 +742,7 @@ LTTNG_TRACEPOINT_EVENT_CLASS(writeback_single_inode_template, TP_ARGS(inode, wbc, nr_to_write), TP_FIELDS( - ctf_string(name, - dev_name(lttng_inode_to_bdi(inode)->dev)) + ctf_string(name, lttng_bdi_dev_name(lttng_inode_to_bdi(inode))) ctf_integer(unsigned long, ino, inode->i_ino) ctf_integer(unsigned long, state, inode->i_state) ctf_integer(unsigned long, dirtied_when, inode->dirtied_when) -- 2.34.1 From dbabd7296e9bcfb263f96e78c185f1eb7346ee53 Mon Sep 17 00:00:00 2001 From: Mathieu Desnoyers Date: Wed, 10 Feb 2021 12:33:38 -0500 Subject: [PATCH 07/16] Fix: do not use bdi_unknown_name symbol Use the GPL-exported bdi_dev_name introduced in kernel 5.7. Do not use static inline bdi_dev_name in prior kernels because it uses the bdi_unknown_name symbol which is not exported to GPL modules. Signed-off-by: Mathieu Desnoyers Change-Id: I8b4e4fd84ecacef7942b308e615ca88db8dce7b6 --- include/instrumentation/events/writeback.h | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/include/instrumentation/events/writeback.h b/include/instrumentation/events/writeback.h index f24237b2..f05708a8 100644 --- a/include/instrumentation/events/writeback.h +++ b/include/instrumentation/events/writeback.h @@ -14,9 +14,12 @@ #ifndef _TRACE_WRITEBACK_DEF_ #define _TRACE_WRITEBACK_DEF_ -#if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(5,6,0) || \ - LTTNG_KERNEL_RANGE(5,5,3, 5,6,0) || \ - LTTNG_KERNEL_RANGE(5,4,19, 5,5,0)) +/* + * Use the GPL-exported bdi_dev_name introduced in kernel 5.7. Do not use + * static inline bdi_dev_name in prior kernels because it uses the bdi_unknown_name + * symbol which is not exported to GPL modules. + */ +#if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(5,7,0)) static inline const char *lttng_bdi_dev_name(struct backing_dev_info *bdi) { return bdi_dev_name(bdi); -- 2.34.1 From 859ad5c708009adbd7457d9551b385c63aff5cec Mon Sep 17 00:00:00 2001 From: Michael Jeanson Date: Tue, 16 Feb 2021 18:08:19 -0500 Subject: [PATCH 08/16] fix: Adjust ranges for Ubuntu 5.8.0-44 kernel Signed-off-by: Michael Jeanson Signed-off-by: Mathieu Desnoyers Change-Id: I419904dc9da316b38c2c16a08b6c17625b19b305 --- include/instrumentation/events/btrfs.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/include/instrumentation/events/btrfs.h b/include/instrumentation/events/btrfs.h index 3e8c7e77..acab6503 100644 --- a/include/instrumentation/events/btrfs.h +++ b/include/instrumentation/events/btrfs.h @@ -1918,7 +1918,8 @@ LTTNG_TRACEPOINT_EVENT_INSTANCE(btrfs__reserved_extent, btrfs_reserved_extent_f #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(5,10,0) || \ LTTNG_KERNEL_RANGE(5,9,5, 5,10,0) || \ - LTTNG_KERNEL_RANGE(5,4,78, 5,5,0)) + LTTNG_KERNEL_RANGE(5,4,78, 5,5,0) || \ + LTTNG_UBUNTU_KERNEL_RANGE(5,8,18,44, 5,9,0,0)) LTTNG_TRACEPOINT_EVENT_MAP(find_free_extent, btrfs_find_free_extent, -- 2.34.1 From 966ad25338569b3afbc1e900ec791eba54c67c9b Mon Sep 17 00:00:00 2001 From: Mathieu Desnoyers Date: Wed, 3 Mar 2021 10:10:16 -0500 Subject: [PATCH 09/16] Fix: memory leaks on event destroy Both filter runtime and event enabler ref objects are owned by the event, but are not freed upon destruction of the event object, thus leaking memory. Signed-off-by: Mathieu Desnoyers Change-Id: Ice9b1c18b47584838aea2b965494d3c8391f4c84 --- include/lttng/events.h | 1 + src/lttng-events.c | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/include/lttng/events.h b/include/lttng/events.h index 51892335..e6572680 100644 --- a/include/lttng/events.h +++ b/include/lttng/events.h @@ -986,6 +986,7 @@ void lttng_enabler_link_bytecode(const struct lttng_event_desc *event_desc, struct lttng_ctx *ctx, struct list_head *instance_bytecode_runtime_head, struct list_head *enabler_bytecode_runtime_head); +void lttng_free_event_filter_runtime(struct lttng_event *event); int lttng_probes_init(void); diff --git a/src/lttng-events.c b/src/lttng-events.c index cb1eb5bf..3569be68 100644 --- a/src/lttng-events.c +++ b/src/lttng-events.c @@ -1485,6 +1485,8 @@ int _lttng_event_notifier_unregister( static void _lttng_event_destroy(struct lttng_event *event) { + struct lttng_enabler_ref *enabler_ref, *tmp_enabler_ref; + switch (event->instrumentation) { case LTTNG_KERNEL_TRACEPOINT: lttng_event_desc_put(event->desc); @@ -1510,6 +1512,11 @@ void _lttng_event_destroy(struct lttng_event *event) } list_del(&event->list); lttng_destroy_context(event->ctx); + lttng_free_event_filter_runtime(event); + /* Free event enabler refs */ + list_for_each_entry_safe(enabler_ref, tmp_enabler_ref, + &event->enablers_ref_head, node) + kfree(enabler_ref); kmem_cache_free(event_cache, event); } -- 2.34.1 From 3ef5cc037506f5bce971bd66ef2f77310eca6650 Mon Sep 17 00:00:00 2001 From: Mathieu Desnoyers Date: Wed, 3 Mar 2021 10:22:38 -0500 Subject: [PATCH 10/16] Fix: memory leaks on event notifier destroy Both filter runtime and event enabler ref objects are owned by the event notifier, but are not freed upon destruction of the event notifier object, thus leaking memory. Signed-off-by: Mathieu Desnoyers Change-Id: I511569f56a38f670549a93cb6179b77861245712 --- include/lttng/events.h | 1 + src/lttng-bytecode.c | 11 +++++++++++ src/lttng-events.c | 7 +++++++ 3 files changed, 19 insertions(+) diff --git a/include/lttng/events.h b/include/lttng/events.h index e6572680..6aa880f8 100644 --- a/include/lttng/events.h +++ b/include/lttng/events.h @@ -987,6 +987,7 @@ void lttng_enabler_link_bytecode(const struct lttng_event_desc *event_desc, struct list_head *instance_bytecode_runtime_head, struct list_head *enabler_bytecode_runtime_head); void lttng_free_event_filter_runtime(struct lttng_event *event); +void lttng_free_event_notifier_filter_runtime(struct lttng_event_notifier *event_notifier); int lttng_probes_init(void); diff --git a/src/lttng-bytecode.c b/src/lttng-bytecode.c index 5702929b..86ffe949 100644 --- a/src/lttng-bytecode.c +++ b/src/lttng-bytecode.c @@ -603,3 +603,14 @@ void lttng_free_event_filter_runtime(struct lttng_event *event) kfree(runtime); } } + +void lttng_free_event_notifier_filter_runtime(struct lttng_event_notifier *event_notifier) +{ + struct bytecode_runtime *runtime, *tmp; + + list_for_each_entry_safe(runtime, tmp, + &event_notifier->filter_bytecode_runtime_head, p.node) { + kfree(runtime->data); + kfree(runtime); + } +} diff --git a/src/lttng-events.c b/src/lttng-events.c index 3569be68..2bec7284 100644 --- a/src/lttng-events.c +++ b/src/lttng-events.c @@ -1526,6 +1526,8 @@ void _lttng_event_destroy(struct lttng_event *event) static void _lttng_event_notifier_destroy(struct lttng_event_notifier *event_notifier) { + struct lttng_enabler_ref *enabler_ref, *tmp_enabler_ref; + switch (event_notifier->instrumentation) { case LTTNG_KERNEL_TRACEPOINT: lttng_event_desc_put(event_notifier->desc); @@ -1547,6 +1549,11 @@ void _lttng_event_notifier_destroy(struct lttng_event_notifier *event_notifier) WARN_ON_ONCE(1); } list_del(&event_notifier->list); + lttng_free_event_notifier_filter_runtime(event_notifier); + /* Free event enabler refs */ + list_for_each_entry_safe(enabler_ref, tmp_enabler_ref, + &event_notifier->enablers_ref_head, node) + kfree(enabler_ref); kmem_cache_free(event_notifier_cache, event_notifier); } -- 2.34.1 From acbe9250b563528ba81088e3589847a2ab75d2f5 Mon Sep 17 00:00:00 2001 From: =?utf8?q?J=C3=A9r=C3=A9mie=20Galarneau?= Date: Wed, 3 Mar 2021 18:52:19 -0500 Subject: [PATCH 11/16] Fix: filter interpreter early-exits on uninitialized value MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit I observed that syscall filtering on string arguments wouldn't work on my development machines, both running 5.11.2-arch1-1 (Arch Linux). For instance, enabling the tracing of the `openat()` syscall with the 'filename == "/proc/cpuinfo"' filter would not produce events even though matching events were present in another session that had no filtering active. The same problem occurred with `execve()`. I tried a couple of kernel versions before (5.11.1 and 5.10.13, if memory serves me well) and I had the same problem. Meanwhile, I couldn't reproduce the problem on various Debian machines (the LTTng CI) nor on a fresh Ubuntu 20.04 with both the stock kernel and with an updated 5.11.2 kernel. I built the lttng-modules with the interpreter debugging printout and saw the following warning: LTTng: [debug bytecode in /home/jgalar/EfficiOS/src/lttng-modules/src/lttng-bytecode-interpreter.c:bytecode_interpret@1508] Bytecode warning: loading a NULL string. After a shedload (yes, a _shed_load) of digging, I figured that the problem was hidden in plain sight near that logging statement. In the `BYTECODE_OP_LOAD_FIELD_REF_USER_STRING` operation, the 'ax' register's 'user_str' is initialized with the stack value (the user space string's address in our case). However, a NULL check is performed against the register's 'str' member. I initialy suspected that both members would be part of the same union and alias each-other, but they are actually contiguous in a structure. On the unaffected machines, I could confirm that the `str` member was uninitialized to a non-zero value causing the condition to evaluate to false. Francis Deslauriers reproduced the problem by initializing the interpreter stack to zero. I am unsure of the exact kernel configuration option that reveals this issue on Arch Linux, but my kernel has the following option enabled: CONFIG_GCC_PLUGIN_STRUCTLEAK_BYREF_ALL: Zero-initialize any stack variables that may be passed by reference and had not already been explicitly initialized. This is intended to eliminate all classes of uninitialized stack variable exploits and information exposures. I have not tried to build without this enabled as, anyhow, this seems to be a legitimate issue. I have spotted what appears to be an identical problem in `BYTECODE_OP_LOAD_FIELD_REF_USER_SEQUENCE` and corrected it. However, I have not exercised that code path. The commit that introduced this problem is 5b4ad89. The debug print-out of the `BYTECODE_OP_LOAD_FIELD_REF_USER_STRING` operation is modified to print the user string (truncated to 31 chars). Signed-off-by: Jérémie Galarneau Signed-off-by: Mathieu Desnoyers Change-Id: I2da3c31b9e3ce0e1b164cf3d2711c0893cbec273 --- src/lttng-bytecode-interpreter.c | 41 ++++++++++++++++++++++++++++---- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/src/lttng-bytecode-interpreter.c b/src/lttng-bytecode-interpreter.c index e183e330..153aeee6 100644 --- a/src/lttng-bytecode-interpreter.c +++ b/src/lttng-bytecode-interpreter.c @@ -20,7 +20,7 @@ * to handle user-space read. */ static -char get_char(struct estack_entry *reg, size_t offset) +char get_char(const struct estack_entry *reg, size_t offset) { if (unlikely(offset >= reg->u.s.seq_len)) return '\0'; @@ -711,6 +711,39 @@ again: return LTTNG_INTERPRETER_RECORD_FLAG; } +#ifdef DEBUG + +#define DBG_USER_STR_CUTOFF 32 + +/* + * In debug mode, print user string (truncated, if necessary). + */ +static inline +void dbg_load_ref_user_str_printk(const struct estack_entry *user_str_reg) +{ + size_t pos = 0; + char last_char; + char user_str[DBG_USER_STR_CUTOFF]; + + pagefault_disable(); + do { + last_char = get_char(user_str_reg, pos); + user_str[pos] = last_char; + pos++; + } while (last_char != '\0' && pos < sizeof(user_str)); + pagefault_enable(); + + user_str[sizeof(user_str) - 1] = '\0'; + dbg_printk("load field ref user string: '%s%s'\n", user_str, + last_char != '\0' ? "[...]" : ""); +} +#else +static inline +void dbg_load_ref_user_str_printk(const struct estack_entry *user_str_reg) +{ +} +#endif + /* * Return 0 (discard), or raise the 0x1 flag (log event). * Currently, other flags are kept for future extensions and have no @@ -1504,7 +1537,7 @@ uint64_t bytecode_interpret(void *interpreter_data, estack_push(stack, top, ax, bx, ax_t, bx_t); estack_ax(stack, top)->u.s.user_str = *(const char * const *) &interpreter_stack_data[ref->offset]; - if (unlikely(!estack_ax(stack, top)->u.s.str)) { + if (unlikely(!estack_ax(stack, top)->u.s.user_str)) { dbg_printk("Bytecode warning: loading a NULL string.\n"); ret = -EINVAL; goto end; @@ -1514,7 +1547,7 @@ uint64_t bytecode_interpret(void *interpreter_data, ESTACK_STRING_LITERAL_TYPE_NONE; estack_ax(stack, top)->u.s.user = 1; estack_ax(stack, top)->type = REG_STRING; - dbg_printk("ref load string %s\n", estack_ax(stack, top)->u.s.str); + dbg_load_ref_user_str_printk(estack_ax(stack, top)); next_pc += sizeof(struct load_op) + sizeof(struct field_ref); PO; } @@ -1532,7 +1565,7 @@ uint64_t bytecode_interpret(void *interpreter_data, estack_ax(stack, top)->u.s.user_str = *(const char **) (&interpreter_stack_data[ref->offset + sizeof(unsigned long)]); - if (unlikely(!estack_ax(stack, top)->u.s.str)) { + if (unlikely(!estack_ax(stack, top)->u.s.user_str)) { dbg_printk("Bytecode warning: loading a NULL sequence.\n"); ret = -EINVAL; goto end; -- 2.34.1 From a8bc8ae5c932b5cd78c1ae22c6b400585097682e Mon Sep 17 00:00:00 2001 From: Michael Jeanson Date: Thu, 4 Mar 2021 16:50:12 -0500 Subject: [PATCH 12/16] fix: mm, tracing: record slab name for kmem_cache_free() (v5.12) See upstream commit: commit 3544de8ee6e4817278b15fe08658de49abf58954 Author: Jacob Wen Date: Wed Feb 24 12:00:55 2021 -0800 mm, tracing: record slab name for kmem_cache_free() Currently, a trace record generated by the RCU core is as below. ... kmem_cache_free: call_site=rcu_core+0x1fd/0x610 ptr=00000000f3b49a66 It doesn't tell us what the RCU core has freed. This patch adds the slab name to trace_kmem_cache_free(). The new format is as follows. ... kmem_cache_free: call_site=rcu_core+0x1fd/0x610 ptr=0000000037f79c8d name=dentry ... kmem_cache_free: call_site=rcu_core+0x1fd/0x610 ptr=00000000f78cb7b5 name=sock_inode_cache ... kmem_cache_free: call_site=rcu_core+0x1fd/0x610 ptr=0000000018768985 name=pool_workqueue ... kmem_cache_free: call_site=rcu_core+0x1fd/0x610 ptr=000000006a6cb484 name=radix_tree_node We can use it to understand what the RCU core is going to free. For example, some users maybe interested in when the RCU core starts freeing reclaimable slabs like dentry to reduce memory pressure. Link: https://lkml.kernel.org/r/20201216072804.8838-1-jian.w.wen@oracle.com Signed-off-by: Michael Jeanson Signed-off-by: Mathieu Desnoyers Change-Id: I1ee2fc476614cadcc8d3ac5d8feddc7910e1aa3a --- include/instrumentation/events/kmem.h | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/include/instrumentation/events/kmem.h b/include/instrumentation/events/kmem.h index 0a9ba17f..6cb8e480 100644 --- a/include/instrumentation/events/kmem.h +++ b/include/instrumentation/events/kmem.h @@ -87,6 +87,32 @@ LTTNG_TRACEPOINT_EVENT_INSTANCE(kmem_alloc_node, kmem_cache_alloc_node, TP_ARGS(call_site, ptr, bytes_req, bytes_alloc, gfp_flags, node) ) +#if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(5,12,0)) +LTTNG_TRACEPOINT_EVENT(kfree, + + TP_PROTO(unsigned long call_site, const void *ptr), + + TP_ARGS(call_site, ptr), + + TP_FIELDS( + ctf_integer_hex(unsigned long, call_site, call_site) + ctf_integer_hex(const void *, ptr, ptr) + ) +) + +LTTNG_TRACEPOINT_EVENT(kmem_cache_free, + + TP_PROTO(unsigned long call_site, const void *ptr, const char *name), + + TP_ARGS(call_site, ptr, name), + + TP_FIELDS( + ctf_integer_hex(unsigned long, call_site, call_site) + ctf_integer_hex(const void *, ptr, ptr) + ctf_string(name, name) + ) +) +#else LTTNG_TRACEPOINT_EVENT_CLASS(kmem_free, TP_PROTO(unsigned long call_site, const void *ptr), @@ -114,6 +140,7 @@ LTTNG_TRACEPOINT_EVENT_INSTANCE(kmem_free, kmem_cache_free, TP_ARGS(call_site, ptr) ) +#endif #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(3,3,0)) LTTNG_TRACEPOINT_EVENT_MAP(mm_page_free, kmem_mm_page_free, -- 2.34.1 From dd1ee9589c12b053ede875f28d6f235187dd8c4f Mon Sep 17 00:00:00 2001 From: Mathieu Desnoyers Date: Wed, 10 Mar 2021 16:23:13 -0500 Subject: [PATCH 13/16] counters: add coalesce_hits to ABI This will be required by an upcoming feature in a subsequent release, so plan ahead with a coalesce_hits field. It is currently unused by lttng-modules. Signed-off-by: Mathieu Desnoyers Change-Id: I0dd3a5c00a89c59111b723db8e39390a16764133 --- include/lttng/abi.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/include/lttng/abi.h b/include/lttng/abi.h index 86c0a3cf..0cdd6534 100644 --- a/include/lttng/abi.h +++ b/include/lttng/abi.h @@ -146,6 +146,8 @@ struct lttng_kernel_event_notifier { char padding[LTTNG_KERNEL_EVENT_NOTIFIER_PADDING]; } __attribute__((packed)); +#define LTTNG_KERNEL_COUNTER_DIMENSION_MAX 4 + enum lttng_kernel_counter_arithmetic { LTTNG_KERNEL_COUNTER_ARITHMETIC_MODULAR = 0, }; @@ -163,13 +165,15 @@ struct lttng_kernel_counter_dimension { uint8_t has_overflow; } __attribute__((packed)); -#define LTTNG_KERNEL_COUNTER_DIMENSION_MAX 4 +#define LTTNG_KERNEL_COUNTER_CONF_PADDING1 67 struct lttng_kernel_counter_conf { uint32_t arithmetic; /* enum lttng_kernel_counter_arithmetic */ uint32_t bitness; /* enum lttng_kernel_counter_bitness */ uint32_t number_dimensions; int64_t global_sum_step; struct lttng_kernel_counter_dimension dimensions[LTTNG_KERNEL_COUNTER_DIMENSION_MAX]; + uint8_t coalesce_hits; + char padding[LTTNG_KERNEL_COUNTER_CONF_PADDING1]; } __attribute__((packed)); struct lttng_kernel_counter_index { -- 2.34.1 From a345f62132ac5b75d752af0741ac82da291f176d Mon Sep 17 00:00:00 2001 From: Mathieu Desnoyers Date: Fri, 12 Mar 2021 09:36:46 -0500 Subject: [PATCH 14/16] Fix: bytecode linker: iteration on wrong list head lttng_enabler_link_bytecode() calls link_bytecode() passing an insertion location (insert_loc) within the list. This insert location is meant to be used as cursor position where to add the new element. However, bytecode_is_linked() uses it as iteration list head, and this is where things fall apart: it will thus consider the real list head as being a list node, and will erroneously think that it is contained within a struct lttng_bytecode_runtime, and thus try to perform possibly out-of-bound read or read garbage data for the comparison. It worked fine most of the time because in usual scenarios the insert location is the list head. It falls apart when many bytecodes are linked to a given event. Fixes: 2dfda770cc6 ("Decouple `struct lttng_event` from filter code") Signed-off-by: Mathieu Desnoyers Change-Id: I7463c7a9399b8f7f7d0e3d74e6427aae46cf56ff --- src/lttng-bytecode.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/lttng-bytecode.c b/src/lttng-bytecode.c index 86ffe949..343e5380 100644 --- a/src/lttng-bytecode.c +++ b/src/lttng-bytecode.c @@ -398,6 +398,7 @@ static int link_bytecode(const struct lttng_event_desc *event_desc, struct lttng_ctx *ctx, struct lttng_bytecode_node *bytecode, + struct list_head *bytecode_runtime_head, struct list_head *insert_loc) { int ret, offset, next_offset; @@ -407,7 +408,7 @@ int link_bytecode(const struct lttng_event_desc *event_desc, if (!bytecode) return 0; /* Bytecode already linked */ - if (bytecode_is_linked(bytecode, insert_loc)) + if (bytecode_is_linked(bytecode, bytecode_runtime_head)) return 0; dbg_printk("Linking...\n"); @@ -566,7 +567,7 @@ void lttng_enabler_link_bytecode(const struct lttng_event_desc *event_desc, insert_loc = instance_bytecode_head; add_within: dbg_printk("linking bytecode\n"); - ret = link_bytecode(event_desc, ctx, enabler_bc, insert_loc); + ret = link_bytecode(event_desc, ctx, enabler_bc, instance_bytecode_head, insert_loc); if (ret) { dbg_printk("[lttng filter] warning: cannot link event bytecode\n"); } -- 2.34.1 From 484ec2179e14ae9272a7ad3d1820c837136fd144 Mon Sep 17 00:00:00 2001 From: Francis Deslauriers Date: Wed, 17 Mar 2021 10:40:56 -0400 Subject: [PATCH 15/16] Fix: kretprobe: null ptr deref on session destroy The `filter_bytecode_runtime_head` list is currently not initialized for the return event of the kretprobe. This caused a kernel null ptr dereference when destroying a session. It can reproduced with the following commands: lttng create lttng enable-event -k --function=lttng_test_filter_event_write my_event lttng start lttng stop lttng destroy Signed-off-by: Francis Deslauriers Signed-off-by: Mathieu Desnoyers Change-Id: I1162ce8b10dd7237a26331531f048346b984eee7 --- src/lttng-events.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/lttng-events.c b/src/lttng-events.c index 2bec7284..d819c9e2 100644 --- a/src/lttng-events.c +++ b/src/lttng-events.c @@ -949,6 +949,8 @@ struct lttng_event *_lttng_event_create(struct lttng_channel *chan, event_return->enabled = 0; event_return->registered = 1; event_return->instrumentation = itype; + INIT_LIST_HEAD(&event_return->filter_bytecode_runtime_head); + INIT_LIST_HEAD(&event_return->enablers_ref_head); /* * Populate lttng_event structure before kretprobe registration. */ -- 2.34.1 From 0c023c019b5c133847347f3da005dbb3288312ab Mon Sep 17 00:00:00 2001 From: Mathieu Desnoyers Date: Fri, 19 Mar 2021 11:27:50 -0400 Subject: [PATCH 16/16] SoW-2020-0002: Trace Hit Counters Signed-off-by: Mathieu Desnoyers Change-Id: Ib7ab673fb05b4824d0c7ba2e163dc1e88b465ee9 --- DO-NOT-COMMIT.txt | 1 + include/lttng/abi.h | 64 +- include/lttng/event-notifier-notification.h | 3 +- include/lttng/events.h | 247 ++++-- include/lttng/tracepoint-event-impl.h | 147 ++-- src/lttng-abi.c | 601 ++++++++++----- src/lttng-event-notifier-notification.c | 7 +- src/lttng-events.c | 803 ++++++++++++++++---- src/lttng-ring-buffer-client.h | 24 +- src/lttng-ring-buffer-metadata-client.h | 4 +- src/lttng-syscalls.c | 240 +++--- src/probes/lttng-kprobes.c | 44 +- src/probes/lttng-kretprobes.c | 44 +- src/probes/lttng-uprobes.c | 48 +- 14 files changed, 1642 insertions(+), 635 deletions(-) create mode 100644 DO-NOT-COMMIT.txt diff --git a/DO-NOT-COMMIT.txt b/DO-NOT-COMMIT.txt new file mode 100644 index 00000000..00fd55cf --- /dev/null +++ b/DO-NOT-COMMIT.txt @@ -0,0 +1 @@ +Trace hit counter diff --git a/include/lttng/abi.h b/include/lttng/abi.h index 0cdd6534..2cac459a 100644 --- a/include/lttng/abi.h +++ b/include/lttng/abi.h @@ -147,6 +147,46 @@ struct lttng_kernel_event_notifier { } __attribute__((packed)); #define LTTNG_KERNEL_COUNTER_DIMENSION_MAX 4 +#define LTTNG_KERNEL_EVENT_NOTIFIER_NOTIFICATION_PADDING 32 +struct lttng_kernel_event_notifier_notification { + uint64_t token; + uint16_t capture_buf_size; + char padding[LTTNG_KERNEL_EVENT_NOTIFIER_NOTIFICATION_PADDING]; +} __attribute__((packed)); + +enum lttng_kernel_key_token_type { + LTTNG_KERNEL_KEY_TOKEN_STRING = 0, /* arg: string_ptr. */ + LTTNG_KERNEL_KEY_TOKEN_EVENT_NAME = 1, /* no arg. */ + LTTNG_KERNEL_KEY_TOKEN_PROVIDER_NAME = 2, /* no arg. */ +}; + +#define LTTNG_KERNEL_KEY_ARG_PADDING1 60 +#define LTTNG_KERNEL_KEY_TOKEN_STRING_LEN_MAX 256 +struct lttng_kernel_key_token { + uint32_t type; /* enum lttng_kernel_key_token_type */ + union { + uint64_t string_ptr; + char padding[LTTNG_KERNEL_KEY_ARG_PADDING1]; + } arg; +} __attribute__((packed)); + +#define LTTNG_KERNEL_NR_KEY_TOKEN 4 +struct lttng_kernel_counter_key_dimension { + uint32_t nr_key_tokens; + struct lttng_kernel_key_token key_tokens[LTTNG_KERNEL_NR_KEY_TOKEN]; +} __attribute__((packed)); + +struct lttng_kernel_counter_key { + uint32_t nr_dimensions; + struct lttng_kernel_counter_key_dimension key_dimensions[LTTNG_KERNEL_COUNTER_DIMENSION_MAX]; +} __attribute__((packed)); + +#define LTTNG_KERNEL_COUNTER_EVENT_PADDING1 16 +struct lttng_kernel_counter_event { + struct lttng_kernel_event event; + struct lttng_kernel_counter_key key; + char padding[LTTNG_KERNEL_COUNTER_EVENT_PADDING1]; +} __attribute__((packed)); enum lttng_kernel_counter_arithmetic { LTTNG_KERNEL_COUNTER_ARITHMETIC_MODULAR = 0, @@ -208,11 +248,17 @@ struct lttng_kernel_counter_clear { char padding[LTTNG_KERNEL_COUNTER_CLEAR_PADDING]; } __attribute__((packed)); -#define LTTNG_KERNEL_EVENT_NOTIFIER_NOTIFICATION_PADDING 32 -struct lttng_kernel_event_notifier_notification { - uint64_t token; - uint16_t capture_buf_size; - char padding[LTTNG_KERNEL_EVENT_NOTIFIER_NOTIFICATION_PADDING]; +#define LTTNG_KERNEL_COUNTER_KEY_LEN 256 +#define LTTNG_KERNEL_COUNTER_MAP_DESCRIPTOR_PADDING 32 +struct lttng_kernel_counter_map_descriptor { + uint64_t descriptor_index; /* input. [ 0 .. nr_descriptors - 1 ] */ + + uint32_t dimension; /* outputs */ + uint64_t array_index; + uint64_t user_token; + char key[LTTNG_KERNEL_COUNTER_KEY_LEN]; + + char padding[LTTNG_KERNEL_COUNTER_MAP_DESCRIPTOR_PADDING]; } __attribute__((packed)); struct lttng_kernel_tracer_version { @@ -385,6 +431,7 @@ struct lttng_kernel_tracker_args { #define LTTNG_KERNEL_STREAM _IO(0xF6, 0x62) #define LTTNG_KERNEL_EVENT \ _IOW(0xF6, 0x63, struct lttng_kernel_event) +/* LTTNG_KERNEL_SYSCALL_MASK applies to both channel and counter fds. */ #define LTTNG_KERNEL_SYSCALL_MASK \ _IOWR(0xF6, 0x64, struct lttng_kernel_syscall_mask) @@ -430,7 +477,12 @@ struct lttng_kernel_tracker_args { _IOWR(0xF6, 0xC1, struct lttng_kernel_counter_aggregate) #define LTTNG_KERNEL_COUNTER_CLEAR \ _IOW(0xF6, 0xC2, struct lttng_kernel_counter_clear) - +#define LTTNG_KERNEL_COUNTER_MAP_NR_DESCRIPTORS \ + _IOR(0xF6, 0xC3, uint64_t) +#define LTTNG_KERNEL_COUNTER_MAP_DESCRIPTOR \ + _IOWR(0xF6, 0xC4, struct lttng_kernel_counter_map_descriptor) +#define LTTNG_KERNEL_COUNTER_EVENT \ + _IOW(0xF6, 0xC5, struct lttng_kernel_counter_event) /* * LTTng-specific ioctls for the lib ringbuffer. diff --git a/include/lttng/event-notifier-notification.h b/include/lttng/event-notifier-notification.h index b044e2fa..160b2542 100644 --- a/include/lttng/event-notifier-notification.h +++ b/include/lttng/event-notifier-notification.h @@ -12,6 +12,7 @@ void lttng_event_notifier_notification_send(struct lttng_event_notifier *event_notifier, struct lttng_probe_ctx *lttng_probe_ctx, - const char *stack_data); + const char *stack_data, + struct lttng_kernel_notifier_ctx *notif_ctx); #endif /* _LTTNG_EVENT_NOTIFIER_NOTIFICATION_H */ diff --git a/include/lttng/events.h b/include/lttng/events.h index 6aa880f8..ebf981eb 100644 --- a/include/lttng/events.h +++ b/include/lttng/events.h @@ -208,7 +208,7 @@ struct lttng_ctx { struct lttng_event_desc { const char *name; /* lttng-modules name */ const char *kname; /* Linux kernel name (tracepoints) */ - void *probe_callback; + void *probe_callback; /* store-event and count-event probe */ const struct lttng_event_ctx *ctx; /* context */ const struct lttng_event_field *fields; /* event payload */ unsigned int nr_fields; @@ -298,11 +298,13 @@ struct lttng_uprobe_handler { struct lttng_kprobe { struct kprobe kp; char *symbol_name; + uint64_t user_token; }; struct lttng_uprobe { struct inode *inode; struct list_head head; + uint64_t user_token; }; enum lttng_syscall_entryexit { @@ -315,14 +317,47 @@ enum lttng_syscall_abi { LTTNG_SYSCALL_ABI_COMPAT, }; +struct lttng_syscall { + struct list_head node; /* chain registered syscall trigger */ + unsigned int syscall_id; + bool is_compat; +}; + +enum lttng_key_token_type { + LTTNG_KEY_TOKEN_STRING = 0, + LTTNG_KEY_TOKEN_EVENT_NAME = 1, +}; + +#define LTTNG_KEY_TOKEN_STRING_LEN_MAX LTTNG_KERNEL_KEY_TOKEN_STRING_LEN_MAX +struct lttng_key_token { + enum lttng_key_token_type type; + union { + char string[LTTNG_KEY_TOKEN_STRING_LEN_MAX]; + } arg; +}; + +#define LTTNG_NR_KEY_TOKEN LTTNG_KERNEL_NR_KEY_TOKEN +struct lttng_counter_key_dimension { + size_t nr_key_tokens; + struct lttng_key_token key_tokens[LTTNG_KERNEL_NR_KEY_TOKEN]; +}; + +#define LTTNG_COUNTER_DIMENSION_MAX LTTNG_KERNEL_COUNTER_DIMENSION_MAX +struct lttng_counter_key { + size_t nr_dimensions; + struct lttng_counter_key_dimension key_dimensions[LTTNG_COUNTER_DIMENSION_MAX]; +}; + +#define LTTNG_KEY_TOKEN_STRING_LEN_MAX LTTNG_KERNEL_KEY_TOKEN_STRING_LEN_MAX + /* * lttng_event structure is referred to by the tracing fast path. It must be * kept small. */ struct lttng_event { enum lttng_event_type evtype; /* First field. */ - unsigned int id; - struct lttng_channel *chan; + struct lttng_event_container *container; + size_t id; int enabled; const struct lttng_event_desc *desc; void *filter; @@ -333,6 +368,7 @@ struct lttng_event { struct { struct lttng_krp *lttng_krp; char *symbol_name; + uint64_t user_token; } kretprobe; struct lttng_uprobe uprobe; struct { @@ -346,11 +382,27 @@ struct lttng_event { /* Backward references: list of lttng_enabler_ref (ref to enablers) */ struct list_head enablers_ref_head; - struct hlist_node hlist; /* session ht of events */ + struct hlist_node name_hlist; /* session ht of events, per event name */ + struct hlist_node key_hlist; /* session ht of events, per key */ int registered; /* has reg'd tracepoint probe */ /* list of struct lttng_bytecode_runtime, sorted by seqnum */ struct list_head filter_bytecode_runtime_head; int has_enablers_without_bytecode; + + /* List node of actions to perform (for syscall events). */ + struct hlist_node action_list; + + char key[LTTNG_KEY_TOKEN_STRING_LEN_MAX]; + + /* + * For non-coalesce-hit event containers, each event is + * associated with a single event enabler user_token. + */ + uint64_t user_token; +}; + +struct lttng_kernel_notifier_ctx { + int eval_capture; }; // FIXME: Really similar to lttng_event above. Could those be merged ? @@ -385,10 +437,12 @@ struct lttng_event_notifier { size_t num_captures; struct list_head capture_bytecode_runtime_head; int has_enablers_without_bytecode; + int eval_capture; /* Should evaluate capture */ void (*send_notification)(struct lttng_event_notifier *event_notifier, struct lttng_probe_ctx *lttng_probe_ctx, - const char *interpreter_stack_data); + const char *interpreter_stack_data, + struct lttng_kernel_notifier_ctx *notif_ctx); struct lttng_event_notifier_group *group; /* Weak ref */ }; @@ -418,7 +472,8 @@ struct lttng_enabler { struct lttng_event_enabler { struct lttng_enabler base; struct list_head node; /* per-session list of enablers */ - struct lttng_channel *chan; + struct lttng_event_container *container; + struct lttng_counter_key key; /* * Unused, but kept around to make it explicit that the tracer can do * it. @@ -567,35 +622,60 @@ struct lttng_event_notifier_ht { struct hlist_head table[LTTNG_EVENT_NOTIFIER_HT_SIZE]; }; -struct lttng_channel { - unsigned int id; - struct channel *chan; /* Channel buffers */ +enum lttng_event_container_type { + LTTNG_EVENT_CONTAINER_CHANNEL, + LTTNG_EVENT_CONTAINER_COUNTER, +}; + +/* + * An event can be contained within either a channel or a counter. + */ +struct lttng_event_container { + enum lttng_event_container_type type; + + struct file *file; /* File associated to event container */ + struct lttng_session *session; /* Session containing the container */ int enabled; - struct lttng_ctx *ctx; - /* Event ID management */ - struct lttng_session *session; - struct file *file; /* File associated to channel */ - unsigned int free_event_id; /* Next event ID to allocate */ - struct list_head list; /* Channel list */ - struct lttng_channel_ops *ops; - struct lttng_transport *transport; - struct hlist_head *sc_table; /* for syscall tracing */ + struct hlist_head *sc_table; /* for syscall tracing */ struct hlist_head *compat_sc_table; struct hlist_head *sc_exit_table; /* for syscall exit tracing */ struct hlist_head *compat_sc_exit_table; - struct hlist_head sc_unknown; /* for unknown syscalls */ + + /* + * Combining all unknown syscall events works as long as they + * are only matched by "all" syscalls enablers, but will require + * a design change when we allow matching by syscall number, for + * instance by allocating sc_tables accomodating NR_syscalls + * entries. + */ + struct hlist_head sc_unknown; /* for unknown syscalls */ struct hlist_head sc_compat_unknown; struct hlist_head sc_exit_unknown; struct hlist_head compat_sc_exit_unknown; + struct lttng_syscall_filter *sc_filter; - int header_type; /* 0: unset, 1: compact, 2: large */ - enum channel_type channel_type; int syscall_all_entry; int syscall_all_exit; - unsigned int metadata_dumped:1, - sys_enter_registered:1, + unsigned int sys_enter_registered:1, sys_exit_registered:1, - tstate:1; /* Transient enable state */ + tstate:1; /* Transient enable state */ + bool coalesce_hits; +}; + +struct lttng_channel { + unsigned int id; + struct channel *chan; /* Channel buffers */ + struct lttng_ctx *ctx; + /* Event ID management */ + unsigned int free_event_id; /* Next event ID to allocate */ + struct list_head list; /* Channel list */ + struct lttng_channel_ops *ops; + struct lttng_transport *transport; + int header_type; /* 0: unset, 1: compact, 2: large */ + enum channel_type channel_type; + unsigned int metadata_dumped:1; + + struct lttng_event_container parent; }; struct lttng_metadata_stream { @@ -659,6 +739,7 @@ struct lttng_session { struct file *file; /* File associated to session */ struct list_head chan; /* Channel list head */ struct list_head events; /* Event list head */ + struct list_head counters; /* Counters list head */ struct list_head list; /* Session list */ unsigned int free_chan_id; /* Next chan ID to allocate */ uuid_le uuid; /* Trace session unique ID */ @@ -673,18 +754,38 @@ struct lttng_session { tstate:1; /* Transient enable state */ /* List of event enablers */ struct list_head enablers_head; - /* Hash table of events */ - struct lttng_event_ht events_ht; + /* Hash table of events indexed by event name */ + struct lttng_event_ht events_name_ht; + /* Hash table of events indexed by key */ + struct lttng_event_ht events_key_ht; char name[LTTNG_KERNEL_SESSION_NAME_LEN]; char creation_time[LTTNG_KERNEL_SESSION_CREATION_TIME_ISO8601_LEN]; }; +struct lttng_counter_map_descriptor { + uint64_t user_token; + size_t array_index; + char key[LTTNG_KERNEL_COUNTER_KEY_LEN]; +}; + +struct lttng_counter_map { + struct lttng_counter_map_descriptor *descriptors; + size_t nr_descriptors; + size_t alloc_len; + struct mutex lock; /* counter map lock */ +}; + struct lttng_counter { - struct file *file; /* File associated to counter. */ struct file *owner; struct lttng_counter_transport *transport; struct lib_counter *counter; struct lttng_counter_ops *ops; + struct list_head node; /* Counter list (in session) */ + + size_t free_index; /* Next index to allocate */ + struct lttng_counter_map map; + + struct lttng_event_container parent; }; struct lttng_event_notifier_group { @@ -737,6 +838,34 @@ struct lttng_metadata_cache { uint64_t version; /* Current version of the metadata */ }; +static inline +struct lttng_event_container *lttng_channel_get_event_container(struct lttng_channel *channel) +{ + return &channel->parent; +} + +static inline +struct lttng_event_container *lttng_counter_get_event_container(struct lttng_counter *counter) +{ + return &counter->parent; +} + +static inline +struct lttng_channel *lttng_event_container_get_channel(struct lttng_event_container *container) +{ + if (container->type != LTTNG_EVENT_CONTAINER_CHANNEL) + return NULL; + return container_of(container, struct lttng_channel, parent); +} + +static inline +struct lttng_counter *lttng_event_container_get_counter(struct lttng_event_container *container) +{ + if (container->type != LTTNG_EVENT_CONTAINER_COUNTER) + return NULL; + return container_of(container, struct lttng_counter, parent); +} + void lttng_lock_sessions(void); void lttng_unlock_sessions(void); @@ -745,7 +874,8 @@ struct list_head *lttng_get_probe_list_head(void); struct lttng_event_enabler *lttng_event_enabler_create( enum lttng_enabler_format_type format_type, struct lttng_kernel_event *event_param, - struct lttng_channel *chan); + const struct lttng_counter_key *key, + struct lttng_event_container *container); int lttng_event_enabler_enable(struct lttng_event_enabler *event_enabler); int lttng_event_enabler_disable(struct lttng_event_enabler *event_enabler); @@ -771,9 +901,6 @@ int lttng_session_metadata_regenerate(struct lttng_session *session); int lttng_session_statedump(struct lttng_session *session); void metadata_cache_destroy(struct kref *kref); -struct lttng_counter *lttng_kernel_counter_create( - const char *counter_transport_name, size_t number_dimensions, - const size_t *dimensions_sizes); int lttng_kernel_counter_read(struct lttng_counter *counter, const size_t *dimension_indexes, int32_t cpu, int64_t *val, bool *overflow, bool *underflow); @@ -788,8 +915,13 @@ struct lttng_event_notifier_group *lttng_event_notifier_group_create(void); int lttng_event_notifier_group_create_error_counter( struct file *event_notifier_group_file, const struct lttng_kernel_counter_conf *error_counter_conf); + void lttng_event_notifier_group_destroy( struct lttng_event_notifier_group *event_notifier_group); +int lttng_event_notifier_group_set_error_counter( + struct lttng_event_notifier_group *event_notifier_group, + const char *counter_transport_name, + size_t counter_len); struct lttng_channel *lttng_channel_create(struct lttng_session *session, const char *transport_name, @@ -798,27 +930,27 @@ struct lttng_channel *lttng_channel_create(struct lttng_session *session, unsigned int switch_timer_interval, unsigned int read_timer_interval, enum channel_type channel_type); -struct lttng_channel *lttng_global_channel_create(struct lttng_session *session, - int overwrite, void *buf_addr, - size_t subbuf_size, size_t num_subbuf, - unsigned int switch_timer_interval, - unsigned int read_timer_interval); void lttng_metadata_channel_destroy(struct lttng_channel *chan); -struct lttng_event *lttng_event_create(struct lttng_channel *chan, +struct lttng_event *lttng_event_create(struct lttng_event_container *container, struct lttng_kernel_event *event_param, + const struct lttng_counter_key *key, void *filter, const struct lttng_event_desc *event_desc, - enum lttng_kernel_instrumentation itype); -struct lttng_event *_lttng_event_create(struct lttng_channel *chan, + enum lttng_kernel_instrumentation itype, + uint64_t user_token); +struct lttng_event *_lttng_event_create(struct lttng_event_container *container, struct lttng_kernel_event *event_param, + const struct lttng_counter_key *key, void *filter, const struct lttng_event_desc *event_desc, - enum lttng_kernel_instrumentation itype); -struct lttng_event *lttng_event_compat_old_create(struct lttng_channel *chan, - struct lttng_kernel_old_event *old_event_param, - void *filter, - const struct lttng_event_desc *internal_desc); + enum lttng_kernel_instrumentation itype, + uint64_t user_token); +struct lttng_counter *lttng_session_create_counter( + struct lttng_session *session, + const char *counter_transport_name, + size_t number_dimensions, const size_t *dimensions_sizes, + bool coalesce_hits); struct lttng_event_notifier *lttng_event_notifier_create( const struct lttng_event_desc *event_notifier_desc, @@ -837,8 +969,8 @@ struct lttng_event_notifier *_lttng_event_notifier_create( void *filter, enum lttng_kernel_instrumentation itype); -int lttng_channel_enable(struct lttng_channel *channel); -int lttng_channel_disable(struct lttng_channel *channel); +int lttng_event_container_enable(struct lttng_event_container *container); +int lttng_event_container_disable(struct lttng_event_container *container); int lttng_event_enable(struct lttng_event *event); int lttng_event_disable(struct lttng_event *event); @@ -890,16 +1022,13 @@ int lttng_desc_match_enabler(const struct lttng_event_desc *desc, #if defined(CONFIG_HAVE_SYSCALL_TRACEPOINTS) int lttng_syscalls_register_event(struct lttng_event_enabler *event_enabler, void *filter); -int lttng_syscalls_unregister_channel(struct lttng_channel *chan); -int lttng_syscalls_destroy_event(struct lttng_channel *chan); -int lttng_syscall_filter_enable_event( - struct lttng_channel *chan, +int lttng_syscalls_unregister_event_container(struct lttng_event_container *event_container); +int lttng_syscalls_destroy_event_container(struct lttng_event_container *container); +int lttng_syscall_filter_enable_event(struct lttng_event_container *container, struct lttng_event *event); -int lttng_syscall_filter_disable_event( - struct lttng_channel *chan, +int lttng_syscall_filter_disable_event(struct lttng_event_container *container, struct lttng_event *event); - -long lttng_channel_syscall_mask(struct lttng_channel *channel, +long lttng_event_container_syscall_mask(struct lttng_event_container *container, struct lttng_kernel_syscall_mask __user *usyscall_mask); int lttng_syscalls_register_event_notifier( @@ -917,29 +1046,29 @@ static inline int lttng_syscalls_register_event( return -ENOSYS; } -static inline int lttng_syscalls_unregister_channel(struct lttng_channel *chan) +static inline int lttng_syscalls_unregister_event_container(struct lttng_event_container *event_container) { return 0; } -static inline int lttng_syscalls_destroy(struct lttng_channel *chan) +static inline int lttng_syscalls_destroy_event_container(struct lttng_event_container *event_container) { return 0; } -static inline int lttng_syscall_filter_enable_event(struct lttng_channel *chan, +static inline int lttng_syscall_filter_enable_event(struct lttng_event_container *event_container, struct lttng_event *event); { return -ENOSYS; } -static inline int lttng_syscall_filter_disable_event(struct lttng_channel *chan, +static inline int lttng_syscall_filter_disable_event(struct lttng_event_container *event_container, struct lttng_event *event); { return -ENOSYS; } -static inline long lttng_channel_syscall_mask(struct lttng_channel *channel, +static inline long lttng_channel_syscall_mask(struct lttng_event_container *event_container, struct lttng_kernel_syscall_mask __user *usyscall_mask) { return -ENOSYS; diff --git a/include/lttng/tracepoint-event-impl.h b/include/lttng/tracepoint-event-impl.h index 079d7d37..d659a6d6 100644 --- a/include/lttng/tracepoint-event-impl.h +++ b/include/lttng/tracepoint-event-impl.h @@ -496,32 +496,12 @@ void __event_notifier_template_proto___##_name(void); #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code_pre, _fields, _code_post) \ -static void __event_probe__##_name(void *__data, _proto); - -#undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS -#define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, _fields, _code_post) \ -static void __event_probe__##_name(void *__data); - -#include TRACE_INCLUDE(TRACE_INCLUDE_FILE) - -/* - * Stage 3.1 of the trace event_notifiers. - * - * Create event_notifier probe callback prototypes. - */ - -/* Reset all macros within TRACEPOINT_EVENT */ -#include - -#undef TP_PROTO -#define TP_PROTO(...) __VA_ARGS__ - -#undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE -#define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code_pre, _fields, _code_post) \ +static void __event_probe__##_name(void *__data, _proto); \ static void __event_notifier_probe__##_name(void *__data, _proto); #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, _fields, _code_post) \ +static void __event_probe__##_name(void *__data); \ static void __event_notifier_probe__##_name(void *__data); #include TRACE_INCLUDE(TRACE_INCLUDE_FILE) @@ -1193,11 +1173,9 @@ static void __event_probe__##_name(void *__data, _proto) \ .event_notifier = NULL, \ .interruptible = !irqs_disabled(), \ }; \ - struct lttng_channel *__chan = __event->chan; \ - struct lttng_session *__session = __chan->session; \ + struct lttng_event_container *__container = __event->container; \ + struct lttng_session *__session = __container->session; \ struct lib_ring_buffer_ctx __ctx; \ - ssize_t __event_len; \ - size_t __event_align; \ size_t __orig_dynamic_len_offset, __dynamic_len_idx __attribute__((unused)); \ union { \ size_t __dynamic_len_removed[ARRAY_SIZE(__event_fields___##_name)]; \ @@ -1213,7 +1191,7 @@ static void __event_probe__##_name(void *__data, _proto) \ return; \ if (unlikely(!LTTNG_READ_ONCE(__session->active))) \ return; \ - if (unlikely(!LTTNG_READ_ONCE(__chan->enabled))) \ + if (unlikely(!LTTNG_READ_ONCE(__container->enabled))) \ return; \ if (unlikely(!LTTNG_READ_ONCE(__event->enabled))) \ return; \ @@ -1258,19 +1236,37 @@ static void __event_probe__##_name(void *__data, _proto) \ if (likely(!__filter_record)) \ goto __post; \ } \ - __event_len = __event_get_size__##_name(tp_locvar, _args); \ - if (unlikely(__event_len < 0)) { \ - lib_ring_buffer_lost_event_too_big(__chan->chan); \ - goto __post; \ + switch (__container->type) { \ + case LTTNG_EVENT_CONTAINER_CHANNEL: \ + { \ + struct lttng_channel *__chan = lttng_event_container_get_channel(__container); \ + ssize_t __event_len; \ + size_t __event_align; \ + \ + __event_len = __event_get_size__##_name(tp_locvar, _args); \ + if (unlikely(__event_len < 0)) { \ + lib_ring_buffer_lost_event_too_big(__chan->chan); \ + goto __post; \ + } \ + __event_align = __event_get_align__##_name(tp_locvar, _args); \ + lib_ring_buffer_ctx_init(&__ctx, __chan->chan, &__lttng_probe_ctx, __event_len, \ + __event_align, -1); \ + __ret = __chan->ops->event_reserve(&__ctx, __event->id); \ + if (__ret < 0) \ + goto __post; \ + _fields \ + __chan->ops->event_commit(&__ctx); \ + break; \ + } \ + case LTTNG_EVENT_CONTAINER_COUNTER: \ + { \ + struct lttng_counter *__counter = lttng_event_container_get_counter(__container); \ + size_t __index = __event->id; \ + \ + (void) __counter->ops->counter_add(__counter->counter, &__index, 1); \ + break; \ + } \ } \ - __event_align = __event_get_align__##_name(tp_locvar, _args); \ - lib_ring_buffer_ctx_init(&__ctx, __chan->chan, &__lttng_probe_ctx, __event_len, \ - __event_align, -1); \ - __ret = __chan->ops->event_reserve(&__ctx, __event->id); \ - if (__ret < 0) \ - goto __post; \ - _fields \ - __chan->ops->event_commit(&__ctx); \ __post: \ _code_post \ barrier(); /* use before un-reserve. */ \ @@ -1289,11 +1285,8 @@ static void __event_probe__##_name(void *__data) \ .event_notifier = NULL, \ .interruptible = !irqs_disabled(), \ }; \ - struct lttng_channel *__chan = __event->chan; \ - struct lttng_session *__session = __chan->session; \ - struct lib_ring_buffer_ctx __ctx; \ - ssize_t __event_len; \ - size_t __event_align; \ + struct lttng_event_container *__container = __event->container; \ + struct lttng_session *__session = __container->session; \ size_t __orig_dynamic_len_offset, __dynamic_len_idx __attribute__((unused)); \ union { \ size_t __dynamic_len_removed[ARRAY_SIZE(__event_fields___##_name)]; \ @@ -1309,7 +1302,7 @@ static void __event_probe__##_name(void *__data) \ return; \ if (unlikely(!LTTNG_READ_ONCE(__session->active))) \ return; \ - if (unlikely(!LTTNG_READ_ONCE(__chan->enabled))) \ + if (unlikely(!LTTNG_READ_ONCE(__container->enabled))) \ return; \ if (unlikely(!LTTNG_READ_ONCE(__event->enabled))) \ return; \ @@ -1338,7 +1331,7 @@ static void __event_probe__##_name(void *__data) \ __orig_dynamic_len_offset = this_cpu_ptr(<tng_dynamic_len_stack)->offset; \ __dynamic_len_idx = __orig_dynamic_len_offset; \ _code_pre \ - if (unlikely(!list_empty(&__event->filter_bytecode_runtime_head))) { \ + if (unlikely(!list_empty(&__event->filter_bytecode_runtime_head))) { \ struct lttng_bytecode_runtime *bc_runtime; \ int __filter_record = __event->has_enablers_without_bytecode; \ \ @@ -1354,19 +1347,38 @@ static void __event_probe__##_name(void *__data) \ if (likely(!__filter_record)) \ goto __post; \ } \ - __event_len = __event_get_size__##_name(tp_locvar); \ - if (unlikely(__event_len < 0)) { \ - lib_ring_buffer_lost_event_too_big(__chan->chan); \ - goto __post; \ + switch (__container->type) { \ + case LTTNG_EVENT_CONTAINER_CHANNEL: \ + { \ + struct lttng_channel *__chan = lttng_event_container_get_channel(__container); \ + struct lib_ring_buffer_ctx __ctx; \ + ssize_t __event_len; \ + size_t __event_align; \ + \ + __event_len = __event_get_size__##_name(tp_locvar); \ + if (unlikely(__event_len < 0)) { \ + lib_ring_buffer_lost_event_too_big(__chan->chan); \ + goto __post; \ + } \ + __event_align = __event_get_align__##_name(tp_locvar); \ + lib_ring_buffer_ctx_init(&__ctx, __chan->chan, &__lttng_probe_ctx, __event_len, \ + __event_align, -1); \ + __ret = __chan->ops->event_reserve(&__ctx, __event->id); \ + if (__ret < 0) \ + goto __post; \ + _fields \ + __chan->ops->event_commit(&__ctx); \ + break; \ + } \ + case LTTNG_EVENT_CONTAINER_COUNTER: \ + { \ + struct lttng_counter *__counter = lttng_event_container_get_counter(__container); \ + size_t __index = __event->id; \ + \ + (void) __counter->ops->counter_add(__counter->counter, &__index, 1); \ + break; \ + } \ } \ - __event_align = __event_get_align__##_name(tp_locvar); \ - lib_ring_buffer_ctx_init(&__ctx, __chan->chan, &__lttng_probe_ctx, __event_len, \ - __event_align, -1); \ - __ret = __chan->ops->event_reserve(&__ctx, __event->id); \ - if (__ret < 0) \ - goto __post; \ - _fields \ - __chan->ops->event_commit(&__ctx); \ __post: \ _code_post \ barrier(); /* use before un-reserve. */ \ @@ -1378,6 +1390,7 @@ __post: \ #undef __get_dynamic_len + /* * Stage 6.1 of tracepoint generation: generate event notifier probes * @@ -1430,6 +1443,8 @@ static void __event_notifier_probe__##_name(void *__data, _proto) \ struct probe_local_vars __tp_locvar; \ struct probe_local_vars *tp_locvar __attribute__((unused)) = \ &__tp_locvar; \ + struct lttng_kernel_notifier_ctx __notif_ctx; \ + bool __interpreter_stack_prepared = false; \ \ if (unlikely(!READ_ONCE(__event_notifier->enabled))) \ return; \ @@ -1440,6 +1455,7 @@ static void __event_notifier_probe__##_name(void *__data, _proto) \ \ __event_prepare_interpreter_stack__##_name(__stackvar.__interpreter_stack_data, \ tp_locvar, _args); \ + __interpreter_stack_prepared = true; \ lttng_list_for_each_entry_rcu(bc_runtime, &__event_notifier->filter_bytecode_runtime_head, node) { \ if (unlikely(bc_runtime->interpreter_funcs.filter(bc_runtime, &__lttng_probe_ctx, \ __stackvar.__interpreter_stack_data) & LTTNG_INTERPRETER_RECORD_FLAG)) \ @@ -1449,14 +1465,16 @@ static void __event_notifier_probe__##_name(void *__data, _proto) \ goto __post; \ } \ \ - if (unlikely(!list_empty(&__event_notifier->capture_bytecode_runtime_head))) \ + __notif_ctx.eval_capture = LTTNG_READ_ONCE(__event_notifier->eval_capture); \ + if (unlikely(!__interpreter_stack_prepared && __notif_ctx.eval_capture)) \ __event_prepare_interpreter_stack__##_name( \ __stackvar.__interpreter_stack_data, \ tp_locvar, _args); \ \ __event_notifier->send_notification(__event_notifier, \ &__lttng_probe_ctx, \ - __stackvar.__interpreter_stack_data); \ + __stackvar.__interpreter_stack_data, \ + &__notif_ctx); \ \ __post: \ _code_post \ @@ -1481,6 +1499,8 @@ static void __event_notifier_probe__##_name(void *__data) \ struct probe_local_vars __tp_locvar; \ struct probe_local_vars *tp_locvar __attribute__((unused)) = \ &__tp_locvar; \ + struct lttng_kernel_notifier_ctx __notif_ctx; \ + bool __interpreter_stack_prepared = false; \ \ if (unlikely(!READ_ONCE(__event_notifier->enabled))) \ return; \ @@ -1491,6 +1511,7 @@ static void __event_notifier_probe__##_name(void *__data) \ \ __event_prepare_interpreter_stack__##_name(__stackvar.__interpreter_stack_data, \ tp_locvar); \ + __interpreter_stack_prepared = true; \ lttng_list_for_each_entry_rcu(bc_runtime, &__event_notifier->filter_bytecode_runtime_head, node) { \ if (unlikely(bc_runtime->interpreter_funcs.filter(bc_runtime, &__lttng_probe_ctx, \ __stackvar.__interpreter_stack_data) & LTTNG_INTERPRETER_RECORD_FLAG)) \ @@ -1500,14 +1521,16 @@ static void __event_notifier_probe__##_name(void *__data) \ goto __post; \ } \ \ - if (unlikely(!list_empty(&__event_notifier->capture_bytecode_runtime_head))) \ + __notif_ctx.eval_capture = LTTNG_READ_ONCE(__event_notifier->eval_capture); \ + if (unlikely(!__interpreter_stack_prepared && __notif_ctx.eval_capture)) \ __event_prepare_interpreter_stack__##_name( \ __stackvar.__interpreter_stack_data, \ tp_locvar); \ \ __event_notifier->send_notification(__event_notifier, \ &__lttng_probe_ctx, \ - __stackvar.__interpreter_stack_data); \ + __stackvar.__interpreter_stack_data, \ + &__notif_ctx); \ __post: \ _code_post \ return; \ diff --git a/src/lttng-abi.c b/src/lttng-abi.c index 4d053985..f65caf93 100644 --- a/src/lttng-abi.c +++ b/src/lttng-abi.c @@ -490,6 +490,7 @@ int lttng_abi_create_channel(struct file *session_file, const struct file_operations *fops = NULL; const char *transport_name; struct lttng_channel *chan; + struct lttng_event_container *container; struct file *chan_file; int chan_fd; int ret = 0; @@ -557,7 +558,8 @@ int lttng_abi_create_channel(struct file *session_file, ret = -EINVAL; goto chan_error; } - chan->file = chan_file; + container = lttng_channel_get_event_container(chan); + container->file = chan_file; chan_file->private_data = chan; fd_install(chan_fd, chan_file); @@ -607,6 +609,139 @@ int lttng_abi_session_set_creation_time(struct lttng_session *session, return 0; } +static +int lttng_abi_validate_event_param(struct lttng_kernel_event *event_param) +{ + /* Limit ABI to implemented features. */ + switch (event_param->instrumentation) { + case LTTNG_KERNEL_SYSCALL: + switch (event_param->u.syscall.entryexit) { + case LTTNG_KERNEL_SYSCALL_ENTRY: + case LTTNG_KERNEL_SYSCALL_EXIT: + case LTTNG_KERNEL_SYSCALL_ENTRYEXIT: + break; + default: + return -EINVAL; + } + switch (event_param->u.syscall.abi) { + case LTTNG_KERNEL_SYSCALL_ABI_ALL: + break; + default: + return -EINVAL; + } + switch (event_param->u.syscall.match) { + case LTTNG_KERNEL_SYSCALL_MATCH_NAME: + break; + default: + return -EINVAL; + } + break; + + case LTTNG_KERNEL_TRACEPOINT: /* Fallthrough */ + case LTTNG_KERNEL_KPROBE: /* Fallthrough */ + case LTTNG_KERNEL_KRETPROBE: /* Fallthrough */ + case LTTNG_KERNEL_NOOP: /* Fallthrough */ + case LTTNG_KERNEL_UPROBE: + break; + + case LTTNG_KERNEL_FUNCTION: /* Fallthrough */ + default: + return -EINVAL; + } + return 0; +} + +static +int lttng_abi_create_event(struct file *event_container_file, + struct lttng_event_container *container, + struct lttng_kernel_event *event_param, + const struct lttng_counter_key *key) +{ + int event_fd, ret; + struct file *event_file; + void *priv; + + event_param->name[LTTNG_KERNEL_SYM_NAME_LEN - 1] = '\0'; + switch (event_param->instrumentation) { + case LTTNG_KERNEL_KRETPROBE: + event_param->u.kretprobe.symbol_name[LTTNG_KERNEL_SYM_NAME_LEN - 1] = '\0'; + break; + case LTTNG_KERNEL_KPROBE: + event_param->u.kprobe.symbol_name[LTTNG_KERNEL_SYM_NAME_LEN - 1] = '\0'; + break; + case LTTNG_KERNEL_FUNCTION: + WARN_ON_ONCE(1); + /* Not implemented. */ + break; + default: + break; + } + event_fd = lttng_get_unused_fd(); + if (event_fd < 0) { + ret = event_fd; + goto fd_error; + } + event_file = anon_inode_getfile("[lttng_event]", + <tng_event_fops, + NULL, O_RDWR); + if (IS_ERR(event_file)) { + ret = PTR_ERR(event_file); + goto file_error; + } + /* The event holds a reference on the container */ + if (!atomic_long_add_unless(&event_container_file->f_count, 1, LONG_MAX)) { + ret = -EOVERFLOW; + goto refcount_error; + } + ret = lttng_abi_validate_event_param(event_param); + if (ret) + goto event_error; + if (event_param->instrumentation == LTTNG_KERNEL_TRACEPOINT + || event_param->instrumentation == LTTNG_KERNEL_SYSCALL) { + struct lttng_event_enabler *event_enabler; + + if (strutils_is_star_glob_pattern(event_param->name)) { + /* + * If the event name is a star globbing pattern, + * we create the special star globbing enabler. + */ + event_enabler = lttng_event_enabler_create(LTTNG_ENABLER_FORMAT_STAR_GLOB, + event_param, key, container); + } else { + event_enabler = lttng_event_enabler_create(LTTNG_ENABLER_FORMAT_NAME, + event_param, key, container); + } + priv = event_enabler; + } else { + struct lttng_event *event; + + /* + * We tolerate no failure path after event creation. It + * will stay invariant for the rest of the session. + */ + event = lttng_event_create(container, event_param, key, + NULL, NULL, + event_param->instrumentation, event_param->token); + if (IS_ERR(event)) { + ret = PTR_ERR(event); + goto event_error; + } + priv = event; + } + event_file->private_data = priv; + fd_install(event_fd, event_file); + return event_fd; + +event_error: + atomic_long_dec(&event_container_file->f_count); +refcount_error: + fput(event_file); +file_error: + put_unused_fd(event_fd); +fd_error: + return ret; +} + static int lttng_counter_release(struct inode *inode, struct file *file) { @@ -623,10 +758,67 @@ int lttng_counter_release(struct inode *inode, struct file *file) return 0; } +static +int copy_counter_key(struct lttng_counter_key *key, + const struct lttng_kernel_counter_key *ukey) +{ + size_t i, j, nr_dimensions; + + nr_dimensions = ukey->nr_dimensions; + if (nr_dimensions > LTTNG_COUNTER_DIMENSION_MAX) + return -EINVAL; + key->nr_dimensions = nr_dimensions; + for (i = 0; i < nr_dimensions; i++) { + const struct lttng_kernel_counter_key_dimension *udim = + &ukey->key_dimensions[i]; + struct lttng_counter_key_dimension *dim = + &key->key_dimensions[i]; + size_t nr_key_tokens; + + nr_key_tokens = udim->nr_key_tokens; + if (!nr_key_tokens || nr_key_tokens > LTTNG_NR_KEY_TOKEN) + return -EINVAL; + dim->nr_key_tokens = nr_key_tokens; + for (j = 0; j < nr_key_tokens; j++) { + const struct lttng_kernel_key_token *utoken = + &udim->key_tokens[j]; + struct lttng_key_token *token = + &dim->key_tokens[j]; + + switch (utoken->type) { + case LTTNG_KERNEL_KEY_TOKEN_STRING: + { + long ret; + + token->type = LTTNG_KEY_TOKEN_STRING; + ret = strncpy_from_user(token->arg.string, + (char __user *)(unsigned long)utoken->arg.string_ptr, + LTTNG_KEY_TOKEN_STRING_LEN_MAX); + if (ret < 0) + return -EFAULT; + if (!ret || ret == LTTNG_KEY_TOKEN_STRING_LEN_MAX) + return -EINVAL; + break; + } + case LTTNG_KERNEL_KEY_TOKEN_EVENT_NAME: + token->type = LTTNG_KEY_TOKEN_EVENT_NAME; + break; + case LTTNG_KERNEL_KEY_TOKEN_PROVIDER_NAME: + printk(KERN_ERR "LTTng: Provider name token not supported.\n"); + /* Fallthrough */ + default: + return -EINVAL; + } + } + } + return 0; +} + static long lttng_counter_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { struct lttng_counter *counter = file->private_data; + struct lttng_event_container *container = lttng_counter_get_event_container(counter); size_t indexes[LTTNG_KERNEL_COUNTER_DIMENSION_MAX] = { 0 }; int i; @@ -720,6 +912,89 @@ long lttng_counter_ioctl(struct file *file, unsigned int cmd, unsigned long arg) return lttng_kernel_counter_clear(counter, indexes); } + case LTTNG_KERNEL_COUNTER_EVENT: + { + struct lttng_kernel_counter_event *ucounter_event_param; + struct lttng_counter_key *key; + int ret; + + key = kzalloc(sizeof(*key), GFP_KERNEL); + if (!key) + return -ENOMEM; + ucounter_event_param = kzalloc(sizeof(*ucounter_event_param), GFP_KERNEL); + if (!ucounter_event_param) { + ret = -ENOMEM; + goto free_key; + } + if (copy_from_user(ucounter_event_param, + (struct lttng_kernel_counter_event __user *) arg, + sizeof(*ucounter_event_param))) { + ret = -EFAULT; + goto free_param; + } + ret = copy_counter_key(key, &ucounter_event_param->key); + if (ret) + goto free_param; + ret = lttng_abi_create_event(file, container, &ucounter_event_param->event, key); + free_param: + kfree(ucounter_event_param); + free_key: + kfree(key); + return ret; + } + case LTTNG_KERNEL_ENABLE: + return lttng_event_container_enable(container); + case LTTNG_KERNEL_DISABLE: + return lttng_event_container_disable(container); + case LTTNG_KERNEL_SYSCALL_MASK: + return lttng_event_container_syscall_mask(container, + (struct lttng_kernel_syscall_mask __user *) arg); + case LTTNG_KERNEL_COUNTER_MAP_NR_DESCRIPTORS: + { + uint64_t __user *user_nr_descriptors = (uint64_t __user *) arg; + uint64_t nr_descriptors; + + mutex_lock(&counter->map.lock); + nr_descriptors = counter->map.nr_descriptors; + mutex_unlock(&counter->map.lock); + return put_user(nr_descriptors, user_nr_descriptors); + } + case LTTNG_KERNEL_COUNTER_MAP_DESCRIPTOR: + { + struct lttng_kernel_counter_map_descriptor __user *user_descriptor = + (struct lttng_kernel_counter_map_descriptor __user *) arg; + struct lttng_kernel_counter_map_descriptor local_descriptor; + struct lttng_counter_map_descriptor *kernel_descriptor; + int ret; + + if (copy_from_user(&local_descriptor, user_descriptor, + sizeof(local_descriptor))) + return -EFAULT; + if (validate_zeroed_padding(local_descriptor.padding, + sizeof(local_descriptor.padding))) + return -EINVAL; + + mutex_lock(&counter->map.lock); + if (local_descriptor.descriptor_index >= counter->map.nr_descriptors) { + ret = -EOVERFLOW; + goto map_descriptor_error_unlock; + } + kernel_descriptor = &counter->map.descriptors[local_descriptor.descriptor_index]; + local_descriptor.user_token = kernel_descriptor->user_token; + local_descriptor.array_index = kernel_descriptor->array_index; + memcpy(local_descriptor.key, kernel_descriptor->key, LTTNG_KERNEL_COUNTER_KEY_LEN); + mutex_unlock(&counter->map.lock); + + if (copy_to_user(user_descriptor, &local_descriptor, + sizeof(local_descriptor))) + return -EFAULT; + + return 0; + + map_descriptor_error_unlock: + mutex_unlock(&counter->map.lock); + return ret; + } default: WARN_ON_ONCE(1); return -ENOSYS; @@ -735,6 +1010,91 @@ static const struct file_operations lttng_counter_fops = { #endif }; +static +long lttng_abi_session_create_counter( + struct lttng_session *session, + const struct lttng_kernel_counter_conf *counter_conf) +{ + int counter_fd, ret, i; + char *counter_transport_name; + struct lttng_event_container *container; + struct lttng_counter *counter; + struct file *counter_file; + size_t dimension_sizes[LTTNG_KERNEL_COUNTER_DIMENSION_MAX] = { 0 }; + size_t number_dimensions; + + counter_fd = lttng_get_unused_fd(); + if (counter_fd < 0) { + ret = counter_fd; + goto fd_error; + } + + counter_file = anon_inode_getfile("[lttng_counter]", + <tng_counter_fops, + NULL, O_RDONLY); + if (IS_ERR(counter_file)) { + ret = PTR_ERR(counter_file); + goto file_error; + } + + if (counter_conf->arithmetic != LTTNG_KERNEL_COUNTER_ARITHMETIC_MODULAR) { + printk(KERN_ERR "LTTng: Map: Error counter of the wrong type.\n"); + return -EINVAL; + } + + switch (counter_conf->bitness) { + case LTTNG_KERNEL_COUNTER_BITNESS_64: + counter_transport_name = "counter-per-cpu-64-modular"; + break; + case LTTNG_KERNEL_COUNTER_BITNESS_32: + counter_transport_name = "counter-per-cpu-32-modular"; + break; + default: + return -EINVAL; + } + + number_dimensions = (size_t) counter_conf->number_dimensions; + + for (i = 0; i < counter_conf->number_dimensions; i++) { + if (counter_conf->dimensions[i].has_underflow) + return -EINVAL; + if (counter_conf->dimensions[i].has_overflow) + return -EINVAL; + dimension_sizes[i] = counter_conf->dimensions[i].size; + } + + if (!atomic_long_add_unless(&session->file->f_count, 1, LONG_MAX)) { + ret = -EOVERFLOW; + goto refcount_error; + } + + counter = lttng_session_create_counter(session, + counter_transport_name, + number_dimensions, dimension_sizes, + counter_conf->coalesce_hits); + if (!counter) { + ret = -EINVAL; + goto counter_error; + } + + counter->owner = session->file; + container = lttng_counter_get_event_container(counter); + container->file = counter_file; + counter_file->private_data = counter; + + fd_install(counter_fd, counter_file); + + return counter_fd; + +counter_error: + atomic_long_dec(&session->file->f_count); +refcount_error: + fput(counter_file); +file_error: + put_unused_fd(counter_fd); +fd_error: + return ret; +} static enum tracker_type get_tracker_type(struct lttng_kernel_tracker_args *tracker) @@ -925,6 +1285,17 @@ long lttng_session_ioctl(struct file *file, unsigned int cmd, unsigned long arg) return -EFAULT; return lttng_abi_session_set_creation_time(session, &time); } + case LTTNG_KERNEL_COUNTER: + { + struct lttng_kernel_counter_conf ucounter_conf; + + if (copy_from_user(&ucounter_conf, + (struct lttng_kernel_counter_conf __user *) arg, + sizeof(ucounter_conf))) + return -EFAULT; + return lttng_abi_session_create_counter(session, + &ucounter_conf); + } default: return -ENOIOCTLCMD; } @@ -1555,7 +1926,7 @@ const struct file_operations lttng_metadata_ring_buffer_file_operations = { }; static -int lttng_abi_create_stream_fd(struct file *channel_file, void *stream_priv, +int lttng_abi_create_stream_fd(struct file *file, void *stream_priv, const struct file_operations *fops, const char *name) { int stream_fd, ret; @@ -1592,9 +1963,9 @@ fd_error: } static -int lttng_abi_open_stream(struct file *channel_file) +int lttng_abi_open_stream(struct file *file) { - struct lttng_channel *channel = channel_file->private_data; + struct lttng_channel *channel = file->private_data; struct lib_ring_buffer *buf; int ret; void *stream_priv; @@ -1604,7 +1975,7 @@ int lttng_abi_open_stream(struct file *channel_file) return -ENOENT; stream_priv = buf; - ret = lttng_abi_create_stream_fd(channel_file, stream_priv, + ret = lttng_abi_create_stream_fd(file, stream_priv, <tng_stream_ring_buffer_file_operations, "[lttng_stream]"); if (ret < 0) @@ -1618,10 +1989,11 @@ fd_error: } static -int lttng_abi_open_metadata_stream(struct file *channel_file) +int lttng_abi_open_metadata_stream(struct file *file) { - struct lttng_channel *channel = channel_file->private_data; - struct lttng_session *session = channel->session; + struct lttng_channel *channel = file->private_data; + struct lttng_event_container *container = lttng_channel_get_event_container(channel); + struct lttng_session *session = container->session; struct lib_ring_buffer *buf; int ret; struct lttng_metadata_stream *metadata_stream; @@ -1660,7 +2032,7 @@ int lttng_abi_open_metadata_stream(struct file *channel_file) goto kref_error; } - ret = lttng_abi_create_stream_fd(channel_file, stream_priv, + ret = lttng_abi_create_stream_fd(file, stream_priv, <tng_metadata_ring_buffer_file_operations, "[lttng_metadata_stream]"); if (ret < 0) @@ -1718,139 +2090,6 @@ refcount_error: return ret; } -static -int lttng_abi_validate_event_param(struct lttng_kernel_event *event_param) -{ - /* Limit ABI to implemented features. */ - switch (event_param->instrumentation) { - case LTTNG_KERNEL_SYSCALL: - switch (event_param->u.syscall.entryexit) { - case LTTNG_KERNEL_SYSCALL_ENTRY: - case LTTNG_KERNEL_SYSCALL_EXIT: - case LTTNG_KERNEL_SYSCALL_ENTRYEXIT: - break; - default: - return -EINVAL; - } - switch (event_param->u.syscall.abi) { - case LTTNG_KERNEL_SYSCALL_ABI_ALL: - break; - default: - return -EINVAL; - } - switch (event_param->u.syscall.match) { - case LTTNG_KERNEL_SYSCALL_MATCH_NAME: - break; - default: - return -EINVAL; - } - break; - - case LTTNG_KERNEL_TRACEPOINT: /* Fallthrough */ - case LTTNG_KERNEL_KPROBE: /* Fallthrough */ - case LTTNG_KERNEL_KRETPROBE: /* Fallthrough */ - case LTTNG_KERNEL_NOOP: /* Fallthrough */ - case LTTNG_KERNEL_UPROBE: - break; - - case LTTNG_KERNEL_FUNCTION: /* Fallthrough */ - default: - return -EINVAL; - } - return 0; -} - -static -int lttng_abi_create_event(struct file *channel_file, - struct lttng_kernel_event *event_param) -{ - struct lttng_channel *channel = channel_file->private_data; - int event_fd, ret; - struct file *event_file; - void *priv; - - event_param->name[LTTNG_KERNEL_SYM_NAME_LEN - 1] = '\0'; - switch (event_param->instrumentation) { - case LTTNG_KERNEL_KRETPROBE: - event_param->u.kretprobe.symbol_name[LTTNG_KERNEL_SYM_NAME_LEN - 1] = '\0'; - break; - case LTTNG_KERNEL_KPROBE: - event_param->u.kprobe.symbol_name[LTTNG_KERNEL_SYM_NAME_LEN - 1] = '\0'; - break; - case LTTNG_KERNEL_FUNCTION: - WARN_ON_ONCE(1); - /* Not implemented. */ - break; - default: - break; - } - event_fd = lttng_get_unused_fd(); - if (event_fd < 0) { - ret = event_fd; - goto fd_error; - } - event_file = anon_inode_getfile("[lttng_event]", - <tng_event_fops, - NULL, O_RDWR); - if (IS_ERR(event_file)) { - ret = PTR_ERR(event_file); - goto file_error; - } - /* The event holds a reference on the channel */ - if (!atomic_long_add_unless(&channel_file->f_count, 1, LONG_MAX)) { - ret = -EOVERFLOW; - goto refcount_error; - } - ret = lttng_abi_validate_event_param(event_param); - if (ret) - goto event_error; - if (event_param->instrumentation == LTTNG_KERNEL_TRACEPOINT - || event_param->instrumentation == LTTNG_KERNEL_SYSCALL) { - struct lttng_event_enabler *event_enabler; - - if (strutils_is_star_glob_pattern(event_param->name)) { - /* - * If the event name is a star globbing pattern, - * we create the special star globbing enabler. - */ - event_enabler = lttng_event_enabler_create(LTTNG_ENABLER_FORMAT_STAR_GLOB, - event_param, channel); - } else { - event_enabler = lttng_event_enabler_create(LTTNG_ENABLER_FORMAT_NAME, - event_param, channel); - } - priv = event_enabler; - } else { - struct lttng_event *event; - - /* - * We tolerate no failure path after event creation. It - * will stay invariant for the rest of the session. - */ - event = lttng_event_create(channel, event_param, - NULL, NULL, - event_param->instrumentation); - WARN_ON_ONCE(!event); - if (IS_ERR(event)) { - ret = PTR_ERR(event); - goto event_error; - } - priv = event; - } - event_file->private_data = priv; - fd_install(event_fd, event_file); - return event_fd; - -event_error: - atomic_long_dec(&channel_file->f_count); -refcount_error: - fput(event_file); -file_error: - put_unused_fd(event_fd); -fd_error: - return ret; -} - static long lttng_event_notifier_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { @@ -2078,7 +2317,8 @@ long lttng_abi_event_notifier_group_create_error_counter( int counter_fd, ret; char *counter_transport_name; size_t counter_len; - struct lttng_counter *counter = NULL; + struct lttng_counter *counter; + struct lttng_event_container *container; struct file *counter_file; struct lttng_event_notifier_group *event_notifier_group = (struct lttng_event_notifier_group *) event_notifier_group_file->private_data; @@ -2104,19 +2344,6 @@ long lttng_abi_event_notifier_group_create_error_counter( return -EINVAL; } - /* - * Lock sessions to provide mutual exclusion against concurrent - * modification of event_notifier group, which would result in - * overwriting the error counter if set concurrently. - */ - lttng_lock_sessions(); - - if (event_notifier_group->error_counter) { - printk(KERN_ERR "Error counter already created in event_notifier group\n"); - ret = -EBUSY; - goto fd_error; - } - counter_fd = lttng_get_unused_fd(); if (counter_fd < 0) { ret = counter_fd; @@ -2138,29 +2365,18 @@ long lttng_abi_event_notifier_group_create_error_counter( goto refcount_error; } - counter = lttng_kernel_counter_create(counter_transport_name, - 1, &counter_len); - if (!counter) { - ret = -EINVAL; + ret = lttng_event_notifier_group_set_error_counter(event_notifier_group, + counter_transport_name, counter_len); + if (ret) goto counter_error; - } - - event_notifier_group->error_counter_len = counter_len; - /* - * store-release to publish error counter matches load-acquire - * in record_error. Ensures the counter is created and the - * error_counter_len is set before they are used. - */ - lttng_smp_store_release(&event_notifier_group->error_counter, counter); - counter->file = counter_file; + counter = event_notifier_group->error_counter; + container = lttng_counter_get_event_container(counter); + container->file = counter_file; counter->owner = event_notifier_group->file; counter_file->private_data = counter; - /* Ownership transferred. */ - counter = NULL; fd_install(counter_fd, counter_file); - lttng_unlock_sessions(); return counter_fd; @@ -2171,7 +2387,6 @@ refcount_error: file_error: put_unused_fd(counter_fd); fd_error: - lttng_unlock_sessions(); return ret; } @@ -2257,6 +2472,7 @@ static long lttng_channel_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { struct lttng_channel *channel = file->private_data; + struct lttng_event_container *container = lttng_channel_get_event_container(channel); switch (cmd) { case LTTNG_KERNEL_OLD_STREAM: @@ -2319,7 +2535,7 @@ long lttng_channel_ioctl(struct file *file, unsigned int cmd, unsigned long arg) default: break; } - ret = lttng_abi_create_event(file, uevent_param); + ret = lttng_abi_create_event(file, container, uevent_param, NULL); old_event_error_free_old_param: kfree(old_uevent_param); @@ -2336,7 +2552,7 @@ old_event_end: (struct lttng_kernel_event __user *) arg, sizeof(uevent_param))) return -EFAULT; - return lttng_abi_create_event(file, &uevent_param); + return lttng_abi_create_event(file, container, &uevent_param, NULL); } case LTTNG_KERNEL_OLD_CONTEXT: { @@ -2379,7 +2595,7 @@ old_event_end: ret = lttng_abi_add_context(file, ucontext_param, - &channel->ctx, channel->session); + &channel->ctx, container->session); old_ctx_error_free_old_param: kfree(old_ucontext_param); @@ -2398,16 +2614,16 @@ old_ctx_end: return -EFAULT; return lttng_abi_add_context(file, &ucontext_param, - &channel->ctx, channel->session); + &channel->ctx, container->session); } case LTTNG_KERNEL_OLD_ENABLE: case LTTNG_KERNEL_ENABLE: - return lttng_channel_enable(channel); + return lttng_event_container_enable(container); case LTTNG_KERNEL_OLD_DISABLE: case LTTNG_KERNEL_DISABLE: - return lttng_channel_disable(channel); + return lttng_event_container_disable(container); case LTTNG_KERNEL_SYSCALL_MASK: - return lttng_channel_syscall_mask(channel, + return lttng_event_container_syscall_mask(container, (struct lttng_kernel_syscall_mask __user *) arg); default: return -ENOIOCTLCMD; @@ -2470,21 +2686,26 @@ unsigned int lttng_channel_poll(struct file *file, poll_table *wait) static int lttng_channel_release(struct inode *inode, struct file *file) { - struct lttng_channel *channel = file->private_data; + struct lttng_channel *chan = file->private_data; + + if (chan) { + struct lttng_event_container *container = lttng_channel_get_event_container(chan); - if (channel) - fput(channel->session->file); + fput(container->session->file); + } return 0; } static int lttng_metadata_channel_release(struct inode *inode, struct file *file) { - struct lttng_channel *channel = file->private_data; + struct lttng_channel *chan = file->private_data; + + if (chan) { + struct lttng_event_container *container = lttng_channel_get_event_container(chan); - if (channel) { - fput(channel->session->file); - lttng_metadata_channel_destroy(channel); + fput(container->session->file); + lttng_metadata_channel_destroy(chan); } return 0; @@ -2614,12 +2835,12 @@ int lttng_event_release(struct inode *inode, struct file *file) case LTTNG_TYPE_EVENT: event = file->private_data; if (event) - fput(event->chan->file); + fput(event->container->file); break; case LTTNG_TYPE_ENABLER: event_enabler = file->private_data; if (event_enabler) - fput(event_enabler->chan->file); + fput(event_enabler->container->file); break; default: WARN_ON_ONCE(1); diff --git a/src/lttng-event-notifier-notification.c b/src/lttng-event-notifier-notification.c index 52b5593a..b39bfaad 100644 --- a/src/lttng-event-notifier-notification.c +++ b/src/lttng-event-notifier-notification.c @@ -426,7 +426,8 @@ void notification_send(struct lttng_event_notifier_notification *notif, void lttng_event_notifier_notification_send(struct lttng_event_notifier *event_notifier, struct lttng_probe_ctx *lttng_probe_ctx, - const char *stack_data) + const char *stack_data, + struct lttng_kernel_notifier_ctx *notif_ctx) { struct lttng_event_notifier_notification notif = { 0 }; int ret; @@ -440,7 +441,7 @@ void lttng_event_notifier_notification_send(struct lttng_event_notifier *event_n goto end; } - if (unlikely(!list_empty(&event_notifier->capture_bytecode_runtime_head))) { + if (unlikely(notif_ctx->eval_capture)) { struct lttng_bytecode_runtime *capture_bc_runtime; /* @@ -449,7 +450,7 @@ void lttng_event_notifier_notification_send(struct lttng_event_notifier *event_n * `output` parameter to the capture buffer. If the interpreter * fails, append an empty capture to the buffer. */ - list_for_each_entry(capture_bc_runtime, + list_for_each_entry_rcu(capture_bc_runtime, &event_notifier->capture_bytecode_runtime_head, node) { struct lttng_interpreter_output output; diff --git a/src/lttng-events.c b/src/lttng-events.c index d819c9e2..d02c110e 100644 --- a/src/lttng-events.c +++ b/src/lttng-events.c @@ -34,6 +34,7 @@ #include #include #include +#include #include #include #include @@ -45,6 +46,7 @@ #include #include #include +#include #include #define METADATA_CACHE_DEFAULT_SIZE 4096 @@ -68,12 +70,12 @@ static void lttng_event_notifier_group_sync_enablers(struct lttng_event_notifier static void _lttng_event_destroy(struct lttng_event *event); static void _lttng_event_notifier_destroy(struct lttng_event_notifier *event_notifier); -static void _lttng_channel_destroy(struct lttng_channel *chan); +static void _lttng_channel_destroy(struct lttng_channel *channel); +static void _lttng_session_counter_destroy(struct lttng_counter *counter); static int _lttng_event_unregister(struct lttng_event *event); static int _lttng_event_notifier_unregister(struct lttng_event_notifier *event_notifier); static int _lttng_event_metadata_statedump(struct lttng_session *session, - struct lttng_channel *chan, struct lttng_event *event); static int _lttng_session_metadata_statedump(struct lttng_session *session); @@ -88,6 +90,27 @@ int _lttng_field_statedump(struct lttng_session *session, const struct lttng_event_field *field, size_t nesting); +static bool lttng_event_container_is_metadata_channel(struct lttng_event_container *container) +{ + switch (container->type) { + case LTTNG_EVENT_CONTAINER_CHANNEL: + { + struct lttng_channel *chan = lttng_event_container_get_channel(container); + + return chan->channel_type == METADATA_CHANNEL; + } + case LTTNG_EVENT_CONTAINER_COUNTER: + return false; + default: + return false; + } +} + +static bool lttng_event_within_metadata_channel(struct lttng_event *event) +{ + return lttng_event_container_is_metadata_channel(event->container); +} + void synchronize_trace(void) { #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(5,1,0)) @@ -154,6 +177,7 @@ struct lttng_session *lttng_session_create(void) goto err; INIT_LIST_HEAD(&session->chan); INIT_LIST_HEAD(&session->events); + INIT_LIST_HEAD(&session->counters); lttng_guid_gen(&session->uuid); metadata_cache = kzalloc(sizeof(struct lttng_metadata_cache), @@ -172,7 +196,9 @@ struct lttng_session *lttng_session_create(void) sizeof(metadata_cache->uuid)); INIT_LIST_HEAD(&session->enablers_head); for (i = 0; i < LTTNG_EVENT_HT_SIZE; i++) - INIT_HLIST_HEAD(&session->events_ht.table[i]); + INIT_HLIST_HEAD(&session->events_name_ht.table[i]); + for (i = 0; i < LTTNG_EVENT_HT_SIZE; i++) + INIT_HLIST_HEAD(&session->events_key_ht.table[i]); list_add(&session->list, &sessions); session->pid_tracker.session = session; session->pid_tracker.tracker_type = TRACKER_PID; @@ -210,12 +236,15 @@ struct lttng_counter_transport *lttng_counter_transport_find(const char *name) return NULL; } +static struct lttng_counter *lttng_kernel_counter_create( const char *counter_transport_name, - size_t number_dimensions, const size_t *dimensions_sizes) + size_t number_dimensions, const size_t *dimensions_sizes, + bool coalesce_hits) { - struct lttng_counter *counter = NULL; struct lttng_counter_transport *counter_transport = NULL; + struct lttng_counter *counter = NULL; + struct lttng_event_container *container; counter_transport = lttng_counter_transport_find(counter_transport_name); if (!counter_transport) { @@ -231,14 +260,18 @@ struct lttng_counter *lttng_kernel_counter_create( counter = lttng_kvzalloc(sizeof(struct lttng_counter), GFP_KERNEL); if (!counter) goto nomem; - + container = lttng_counter_get_event_container(counter); + container->type = LTTNG_EVENT_CONTAINER_COUNTER; + container->coalesce_hits = coalesce_hits; /* Create event notifier error counter. */ counter->ops = &counter_transport->ops; counter->transport = counter_transport; + mutex_init(&counter->map.lock); counter->counter = counter->ops->counter_create( number_dimensions, dimensions_sizes, 0); if (!counter->counter) { + printk(KERN_WARNING "LTTng: Error creating counter"); goto create_error; } @@ -253,6 +286,60 @@ notransport: return NULL; } +static +void lttng_kernel_counter_destroy(struct lttng_counter *counter) +{ + counter->ops->counter_destroy(counter->counter); + module_put(counter->transport->owner); + lttng_kvfree(counter->map.descriptors); + lttng_kvfree(counter); +} + +int lttng_event_notifier_group_set_error_counter( + struct lttng_event_notifier_group *event_notifier_group, + const char *counter_transport_name, + size_t counter_len) +{ + struct lttng_counter *counter; + int ret; + + /* + * Lock sessions to provide mutual exclusion against concurrent + * modification of trigger group, which would result in + * overwriting the error counter if set concurrently. + */ + mutex_lock(&sessions_mutex); + + if (event_notifier_group->error_counter) { + printk(KERN_ERR "Error counter already set in event notifier group\n"); + ret = -EBUSY; + goto error; + } + + counter = lttng_kernel_counter_create(counter_transport_name, + 1, &counter_len, false); + if (!counter) { + ret = -EINVAL; + goto error; + } + + event_notifier_group->error_counter_len = counter_len; + /* + * store-release to publish error counter matches load-acquire + * in record_error. Ensures the counter is created and the + * error_counter_len is set before they are used. + */ + lttng_smp_store_release(&event_notifier_group->error_counter, + counter); + + mutex_unlock(&sessions_mutex); + return 0; + +error: + mutex_unlock(&sessions_mutex); + return ret; +} + struct lttng_event_notifier_group *lttng_event_notifier_group_create(void) { struct lttng_transport *transport = NULL; @@ -318,6 +405,34 @@ notransport: return NULL; } +struct lttng_counter *lttng_session_create_counter( + struct lttng_session *session, + const char *counter_transport_name, + size_t number_dimensions, const size_t *dimensions_sizes, + bool coalesce_hits) +{ + struct lttng_counter *counter; + struct lttng_event_container *container; + + counter = lttng_kernel_counter_create(counter_transport_name, + number_dimensions, dimensions_sizes, + coalesce_hits); + if (!counter) { + goto counter_error; + } + container = lttng_counter_get_event_container(counter); + + mutex_lock(&sessions_mutex); + container->session = session; + list_add(&counter->node, &session->counters); + mutex_unlock(&sessions_mutex); + + return counter; + +counter_error: + return NULL; +} + void metadata_cache_destroy(struct kref *kref) { struct lttng_metadata_cache *cache = @@ -332,12 +447,17 @@ void lttng_session_destroy(struct lttng_session *session) struct lttng_event *event, *tmpevent; struct lttng_metadata_stream *metadata_stream; struct lttng_event_enabler *event_enabler, *tmp_event_enabler; + struct lttng_counter *counter, *tmpcounter; int ret; mutex_lock(&sessions_mutex); WRITE_ONCE(session->active, 0); list_for_each_entry(chan, &session->chan, list) { - ret = lttng_syscalls_unregister_channel(chan); + ret = lttng_syscalls_unregister_event_container(lttng_channel_get_event_container(chan)); + WARN_ON(ret); + } + list_for_each_entry(counter, &session->counters, node) { + ret = lttng_syscalls_unregister_event_container(lttng_counter_get_event_container(counter)); WARN_ON(ret); } list_for_each_entry(event, &session->events, list) { @@ -346,7 +466,11 @@ void lttng_session_destroy(struct lttng_session *session) } synchronize_trace(); /* Wait for in-flight events to complete */ list_for_each_entry(chan, &session->chan, list) { - ret = lttng_syscalls_destroy_event(chan); + ret = lttng_syscalls_destroy_event_container(lttng_channel_get_event_container(chan)); + WARN_ON(ret); + } + list_for_each_entry(counter, &session->counters, node) { + ret = lttng_syscalls_destroy_event_container(lttng_counter_get_event_container(counter)); WARN_ON(ret); } list_for_each_entry_safe(event_enabler, tmp_event_enabler, @@ -358,6 +482,8 @@ void lttng_session_destroy(struct lttng_session *session) BUG_ON(chan->channel_type == METADATA_CHANNEL); _lttng_channel_destroy(chan); } + list_for_each_entry_safe(counter, tmpcounter, &session->counters, node) + _lttng_session_counter_destroy(counter); mutex_lock(&session->metadata_cache->lock); list_for_each_entry(metadata_stream, &session->metadata_cache->metadata_stream, list) _lttng_metadata_channel_hangup(metadata_stream); @@ -413,9 +539,7 @@ void lttng_event_notifier_group_destroy( if (event_notifier_group->error_counter) { struct lttng_counter *error_counter = event_notifier_group->error_counter; - error_counter->ops->counter_destroy(error_counter->counter); - module_put(error_counter->transport->owner); - lttng_kvfree(error_counter); + lttng_kernel_counter_destroy(error_counter); event_notifier_group->error_counter = NULL; } @@ -554,47 +678,47 @@ end: return ret; } -int lttng_channel_enable(struct lttng_channel *channel) +int lttng_event_container_enable(struct lttng_event_container *container) { int ret = 0; mutex_lock(&sessions_mutex); - if (channel->channel_type == METADATA_CHANNEL) { + if (lttng_event_container_is_metadata_channel(container)) { ret = -EPERM; goto end; } - if (channel->enabled) { + if (container->enabled) { ret = -EEXIST; goto end; } /* Set transient enabler state to "enabled" */ - channel->tstate = 1; - lttng_session_sync_event_enablers(channel->session); + container->tstate = 1; + lttng_session_sync_event_enablers(container->session); /* Set atomically the state to "enabled" */ - WRITE_ONCE(channel->enabled, 1); + WRITE_ONCE(container->enabled, 1); end: mutex_unlock(&sessions_mutex); return ret; } -int lttng_channel_disable(struct lttng_channel *channel) +int lttng_event_container_disable(struct lttng_event_container *container) { int ret = 0; mutex_lock(&sessions_mutex); - if (channel->channel_type == METADATA_CHANNEL) { + if (lttng_event_container_is_metadata_channel(container)) { ret = -EPERM; goto end; } - if (!channel->enabled) { + if (!container->enabled) { ret = -EEXIST; goto end; } /* Set atomically the state to "disabled" */ - WRITE_ONCE(channel->enabled, 0); + WRITE_ONCE(container->enabled, 0); /* Set transient enabler state to "enabled" */ - channel->tstate = 0; - lttng_session_sync_event_enablers(channel->session); + container->tstate = 0; + lttng_session_sync_event_enablers(container->session); end: mutex_unlock(&sessions_mutex); return ret; @@ -605,7 +729,7 @@ int lttng_event_enable(struct lttng_event *event) int ret = 0; mutex_lock(&sessions_mutex); - if (event->chan->channel_type == METADATA_CHANNEL) { + if (lttng_event_within_metadata_channel(event)) { ret = -EPERM; goto end; } @@ -641,7 +765,7 @@ int lttng_event_disable(struct lttng_event *event) int ret = 0; mutex_lock(&sessions_mutex); - if (event->chan->channel_type == METADATA_CHANNEL) { + if (lttng_event_within_metadata_channel(event)) { ret = -EPERM; goto end; } @@ -740,7 +864,8 @@ struct lttng_channel *lttng_channel_create(struct lttng_session *session, unsigned int read_timer_interval, enum channel_type channel_type) { - struct lttng_channel *chan; + struct lttng_event_container *container; + struct lttng_channel *chan = NULL; struct lttng_transport *transport = NULL; mutex_lock(&sessions_mutex); @@ -756,10 +881,21 @@ struct lttng_channel *lttng_channel_create(struct lttng_session *session, printk(KERN_WARNING "LTTng: Can't lock transport module.\n"); goto notransport; } - chan = kzalloc(sizeof(struct lttng_channel), GFP_KERNEL); + chan = lttng_kvzalloc(sizeof(struct lttng_channel), GFP_KERNEL); if (!chan) goto nomem; - chan->session = session; + container = lttng_channel_get_event_container(chan); + container->type = LTTNG_EVENT_CONTAINER_CHANNEL; + container->session = session; + container->tstate = 1; + container->enabled = 1; + /* + * The ring buffer always coalesces hits from various event + * enablers matching a given event to a single event record within the + * ring buffer. + */ + container->coalesce_hits = true; + chan->id = session->free_chan_id++; chan->ops = &transport->ops; /* @@ -768,12 +904,10 @@ struct lttng_channel *lttng_channel_create(struct lttng_session *session, * should be already accessible. */ chan->chan = transport->ops.channel_create(transport_name, - chan, buf_addr, subbuf_size, num_subbuf, + container, buf_addr, subbuf_size, num_subbuf, switch_timer_interval, read_timer_interval); if (!chan->chan) goto create_error; - chan->tstate = 1; - chan->enabled = 1; chan->transport = transport; chan->channel_type = channel_type; list_add(&chan->list, &session->chan); @@ -781,7 +915,7 @@ struct lttng_channel *lttng_channel_create(struct lttng_session *session, return chan; create_error: - kfree(chan); + lttng_kvfree(chan); nomem: if (transport) module_put(transport->owner); @@ -791,6 +925,13 @@ active: return NULL; } +static +void _lttng_session_counter_destroy(struct lttng_counter *counter) +{ + list_del(&counter->node); + lttng_kernel_counter_destroy(counter); +} + /* * Only used internally at session destruction for per-cpu channels, and * when metadata channel is released. @@ -803,13 +944,12 @@ void _lttng_channel_destroy(struct lttng_channel *chan) module_put(chan->transport->owner); list_del(&chan->list); lttng_destroy_context(chan->ctx); - kfree(chan); + lttng_kvfree(chan); } void lttng_metadata_channel_destroy(struct lttng_channel *chan) { BUG_ON(chan->channel_type != METADATA_CHANNEL); - /* Protect the metadata cache with the sessions_mutex. */ mutex_lock(&sessions_mutex); _lttng_channel_destroy(chan); @@ -824,53 +964,262 @@ void _lttng_metadata_channel_hangup(struct lttng_metadata_stream *stream) wake_up_interruptible(&stream->read_wait); } +static +bool lttng_event_container_current_id_full(struct lttng_event_container *container) +{ + switch (container->type) { + case LTTNG_EVENT_CONTAINER_CHANNEL: + { + struct lttng_channel *channel = lttng_event_container_get_channel(container); + + return channel->free_event_id == -1U; + } + case LTTNG_EVENT_CONTAINER_COUNTER: + { + struct lttng_counter *counter = lttng_event_container_get_counter(container); + size_t nr_dimensions, max_nr_elem; + + if (lttng_counter_get_nr_dimensions(&counter->counter->config, + counter->counter, &nr_dimensions)) + return true; + WARN_ON_ONCE(nr_dimensions != 1); + if (nr_dimensions != 1) + return true; + if (lttng_counter_get_max_nr_elem(&counter->counter->config, + counter->counter, &max_nr_elem)) + return true; + return counter->free_index >= max_nr_elem; + } + default: + WARN_ON_ONCE(1); + return true; + } +} + + +static +int lttng_event_container_allocate_id(struct lttng_event_container *container, + const char *key_string, size_t *id) +{ + struct lttng_session *session = container->session; + struct lttng_event *event; + + if (key_string[0]) { + struct hlist_head *head; + + head = utils_borrow_hash_table_bucket(session->events_key_ht.table, + LTTNG_EVENT_HT_SIZE, key_string); + lttng_hlist_for_each_entry(event, head, key_hlist) { + if (!strcmp(key_string, event->key)) { + /* Same key, use same id. */ + *id = event->id; + return 0; + } + } + } + + if (lttng_event_container_current_id_full(container)) { + return -EMFILE; + } + + switch (container->type) { + case LTTNG_EVENT_CONTAINER_CHANNEL: + { + struct lttng_channel *channel = lttng_event_container_get_channel(container); + *id = channel->free_event_id++; + break; + } + case LTTNG_EVENT_CONTAINER_COUNTER: + { + struct lttng_counter *counter = lttng_event_container_get_counter(container); + *id = counter->free_index++; + break; + } + default: + WARN_ON_ONCE(1); + return 0; + } + + return 0; +} + +static +int format_event_key(char *key_string, const struct lttng_counter_key *key, + const char *event_name) +{ + const struct lttng_counter_key_dimension *dim; + size_t i, left = LTTNG_KEY_TOKEN_STRING_LEN_MAX; + + key_string[0] = '\0'; + if (!key || !key->nr_dimensions) + return 0; + /* Currently event keys can only be specified on a single dimension. */ + if (key->nr_dimensions != 1) + return -EINVAL; + dim = &key->key_dimensions[0]; + for (i = 0; i < dim->nr_key_tokens; i++) { + const struct lttng_key_token *token = &dim->key_tokens[i]; + size_t token_len; + const char *str; + + switch (token->type) { + case LTTNG_KEY_TOKEN_STRING: + str = token->arg.string; + break; + case LTTNG_KEY_TOKEN_EVENT_NAME: + str = event_name; + break; + default: + return -EINVAL; + } + token_len = strlen(str); + if (token_len >= left) + return -EINVAL; + strcat(key_string, str); + left -= token_len; + } + return 0; +} + +static +bool match_event_token(struct lttng_event_container *container, + struct lttng_event *event, uint64_t token) +{ + if (container->coalesce_hits) + return true; + if (event->user_token == token) + return true; + return false; +} + +static +int lttng_counter_append_descriptor(struct lttng_counter *counter, + uint64_t user_token, + size_t index, + const char *key) +{ + struct lttng_counter_map *map = &counter->map; + struct lttng_counter_map_descriptor *last; + int ret = 0; + + if (strlen(key) >= LTTNG_KERNEL_COUNTER_KEY_LEN) { + WARN_ON_ONCE(1); + return -EOVERFLOW; + } + mutex_lock(&map->lock); + if (map->nr_descriptors == map->alloc_len) { + struct lttng_counter_map_descriptor *new_table, *old_table; + size_t old_len = map->nr_descriptors; + size_t new_len = max_t(size_t, old_len + 1, map->alloc_len * 2); + + old_table = map->descriptors; + new_table = lttng_kvzalloc(sizeof(struct lttng_counter_map_descriptor) * new_len, + GFP_KERNEL); + if (!new_table) { + ret = -ENOMEM; + goto unlock; + } + + if (old_table) + memcpy(new_table, old_table, old_len * sizeof(struct lttng_counter_map_descriptor)); + + map->descriptors = new_table; + map->alloc_len = new_len; + lttng_kvfree(old_table); + } + last = &map->descriptors[map->nr_descriptors++]; + last->user_token = user_token; + last->array_index = index; + strcpy(last->key, key); +unlock: + mutex_unlock(&map->lock); + return ret; +} /* * Supports event creation while tracing session is active. * Needs to be called with sessions mutex held. */ -struct lttng_event *_lttng_event_create(struct lttng_channel *chan, +struct lttng_event *_lttng_event_create(struct lttng_event_container *container, struct lttng_kernel_event *event_param, + const struct lttng_counter_key *key, void *filter, const struct lttng_event_desc *event_desc, - enum lttng_kernel_instrumentation itype) + enum lttng_kernel_instrumentation itype, + uint64_t token) { - struct lttng_session *session = chan->session; + struct lttng_session *session; struct lttng_event *event; - const char *event_name; - struct hlist_head *head; + char event_name[LTTNG_KERNEL_SYM_NAME_LEN]; + struct hlist_head *name_head, *key_head; + char key_string[LTTNG_KEY_TOKEN_STRING_LEN_MAX]; int ret; - if (chan->free_event_id == -1U) { - ret = -EMFILE; - goto full; - } - + session = container->session; switch (itype) { case LTTNG_KERNEL_TRACEPOINT: - event_name = event_desc->name; + if (strlen(event_desc->name) >= LTTNG_KERNEL_SYM_NAME_LEN) { + ret = -EINVAL; + goto type_error; + } + strcpy(event_name, event_desc->name); break; case LTTNG_KERNEL_KPROBE: case LTTNG_KERNEL_UPROBE: - case LTTNG_KERNEL_KRETPROBE: - case LTTNG_KERNEL_NOOP: case LTTNG_KERNEL_SYSCALL: - event_name = event_param->name; + if (strlen(event_param->name) >= LTTNG_KERNEL_SYM_NAME_LEN) { + ret = -EINVAL; + goto type_error; + } + strcpy(event_name, event_param->name); + break; + case LTTNG_KERNEL_KRETPROBE: + if (strlen(event_param->name) >= LTTNG_KERNEL_SYM_NAME_LEN) { + ret = -EINVAL; + goto type_error; + } + strcpy(event_name, event_param->name); + if (strlen(event_name) + strlen("_entry") >= LTTNG_KERNEL_SYM_NAME_LEN) { + ret = -EINVAL; + goto type_error; + } + strcat(event_name, "_entry"); break; case LTTNG_KERNEL_FUNCTION: /* Fall-through. */ + case LTTNG_KERNEL_NOOP: /* Fall-through. */ default: WARN_ON_ONCE(1); ret = -EINVAL; goto type_error; } - head = utils_borrow_hash_table_bucket(session->events_ht.table, + if (format_event_key(key_string, key, event_name)) { + ret = -EINVAL; + goto type_error; + } + + name_head = utils_borrow_hash_table_bucket(session->events_name_ht.table, LTTNG_EVENT_HT_SIZE, event_name); - lttng_hlist_for_each_entry(event, head, hlist) { + lttng_hlist_for_each_entry(event, name_head, name_hlist) { + bool same_event = false, same_container = false, same_key = false, + same_token = false; + WARN_ON_ONCE(!event->desc); - if (!strncmp(event->desc->name, event_name, - LTTNG_KERNEL_SYM_NAME_LEN - 1) - && chan == event->chan) { + if (event_desc) { + if (event->desc == event_desc) + same_event = true; + } else { + if (!strcmp(event_name, event->desc->name)) + same_event = true; + } + if (container == event->container) { + same_container = true; + if (match_event_token(container, event, token)) + same_token = true; + } + if (key_string[0] == '\0' || !strcmp(key_string, event->key)) + same_key = true; + if (same_event && same_container && same_key && same_token) { ret = -EEXIST; goto exist; } @@ -881,13 +1230,25 @@ struct lttng_event *_lttng_event_create(struct lttng_channel *chan, ret = -ENOMEM; goto cache_error; } - event->chan = chan; + event->container = container; event->filter = filter; - event->id = chan->free_event_id++; event->instrumentation = itype; event->evtype = LTTNG_TYPE_EVENT; + if (!container->coalesce_hits) + event->user_token = token; INIT_LIST_HEAD(&event->filter_bytecode_runtime_head); INIT_LIST_HEAD(&event->enablers_ref_head); + if (lttng_event_container_allocate_id(container, key_string, + &event->id)) { + ret = -EMFILE; + goto full; + } + if (key_string[0]) { + key_head = utils_borrow_hash_table_bucket(session->events_key_ht.table, + LTTNG_EVENT_HT_SIZE, key_string); + hlist_add_head(&event->key_hlist, key_head); + } + strcpy(event->key, key_string); switch (itype) { case LTTNG_KERNEL_TRACEPOINT: @@ -923,8 +1284,35 @@ struct lttng_event *_lttng_event_create(struct lttng_channel *chan, ret = -EINVAL; goto register_error; } + event->u.kprobe.user_token = token; ret = try_module_get(event->desc->owner); WARN_ON_ONCE(!ret); + + /* Append descriptor to counter. */ + switch (container->type) { + case LTTNG_EVENT_CONTAINER_COUNTER: + { + struct lttng_counter *counter; + const char *name = ""; + int ret; + + counter = lttng_event_container_get_counter(container); + if (event->key[0]) + name = event->key; + else + name = event_name; + ret = lttng_counter_append_descriptor(counter, + token, event->id, + name); + if (ret) { + WARN_ON_ONCE(1); + } + break; + } + case LTTNG_EVENT_CONTAINER_CHANNEL: + default: + break; + } break; case LTTNG_KERNEL_KRETPROBE: { @@ -937,20 +1325,68 @@ struct lttng_event *_lttng_event_create(struct lttng_channel *chan, */ event->enabled = 0; event->registered = 1; + event->u.kretprobe.user_token = token; + + /* Append descriptor to counter. */ + switch (container->type) { + case LTTNG_EVENT_CONTAINER_COUNTER: + { + struct lttng_counter *counter; + const char *name = ""; + int ret; + + counter = lttng_event_container_get_counter(container); + if (event->key[0]) + name = event->key; + else + name = event_name; + ret = lttng_counter_append_descriptor(counter, + token, event->id, + name); + if (ret) { + WARN_ON_ONCE(1); + } + break; + } + case LTTNG_EVENT_CONTAINER_CHANNEL: + default: + break; + } + event_return = kmem_cache_zalloc(event_cache, GFP_KERNEL); if (!event_return) { ret = -ENOMEM; goto register_error; } - event_return->chan = chan; + event_return->container = container; event_return->filter = filter; - event_return->id = chan->free_event_id++; + + strcpy(event_name, event_param->name); + if (strlen(event_name) + strlen("_return") >= LTTNG_KERNEL_SYM_NAME_LEN) { + ret = -EINVAL; + goto register_error; + } + strcat(event_name, "_return"); + if (format_event_key(key_string, key, event_name)) { + ret = -EINVAL; + goto register_error; + } + if (lttng_event_container_allocate_id(container, key_string, &event_return->id)) { + kmem_cache_free(event_cache, event_return); + ret = -EMFILE; + goto register_error; + } + key_head = utils_borrow_hash_table_bucket(session->events_key_ht.table, + LTTNG_EVENT_HT_SIZE, key_string); + hlist_add_head(&event_return->key_hlist, key_head); event_return->enabled = 0; event_return->registered = 1; event_return->instrumentation = itype; INIT_LIST_HEAD(&event_return->filter_bytecode_runtime_head); INIT_LIST_HEAD(&event_return->enablers_ref_head); + event_return->u.kretprobe.user_token = token; + strcpy(event_return->key, key_string); /* * Populate lttng_event structure before kretprobe registration. */ @@ -970,19 +1406,50 @@ struct lttng_event *_lttng_event_create(struct lttng_channel *chan, WARN_ON_ONCE(!ret); ret = try_module_get(event->desc->owner); WARN_ON_ONCE(!ret); - ret = _lttng_event_metadata_statedump(chan->session, chan, - event_return); - WARN_ON_ONCE(ret > 0); - if (ret) { - kmem_cache_free(event_cache, event_return); - module_put(event->desc->owner); - module_put(event->desc->owner); - goto statedump_error; + + /* Append exit descriptor to counter. */ + switch (container->type) { + case LTTNG_EVENT_CONTAINER_COUNTER: + { + struct lttng_counter *counter; + const char *name = ""; + int ret; + + counter = lttng_event_container_get_counter(container); + if (event_return->key[0]) + name = event_return->key; + else + name = event_name; + ret = lttng_counter_append_descriptor(counter, + token, event_return->id, + name); + if (ret) { + WARN_ON_ONCE(1); + } + break; + } + case LTTNG_EVENT_CONTAINER_CHANNEL: + default: + break; } - list_add(&event_return->list, &chan->session->events); + switch (container->type) { + case LTTNG_EVENT_CONTAINER_CHANNEL: + ret = _lttng_event_metadata_statedump(session, event_return); + WARN_ON_ONCE(ret > 0); + if (ret) { + kmem_cache_free(event_cache, event_return); + module_put(event->desc->owner); + module_put(event->desc->owner); + goto statedump_error; + } + break; + case LTTNG_EVENT_CONTAINER_COUNTER: + default: + break; + } + list_add(&event_return->list, &session->events); break; } - case LTTNG_KERNEL_NOOP: case LTTNG_KERNEL_SYSCALL: /* * Needs to be explicitly enabled after creation, since @@ -1025,6 +1492,7 @@ struct lttng_event *_lttng_event_create(struct lttng_channel *chan, */ event->enabled = 0; event->registered = 1; + event->u.uprobe.user_token = token; /* * Populate lttng_event structure before event @@ -1032,37 +1500,71 @@ struct lttng_event *_lttng_event_create(struct lttng_channel *chan, */ smp_wmb(); - ret = lttng_uprobes_register_event(event_param->name, + ret = lttng_uprobes_register_event(event_name, event_param->u.uprobe.fd, event); if (ret) goto register_error; ret = try_module_get(event->desc->owner); WARN_ON_ONCE(!ret); + + /* Append descriptor to counter. */ + switch (container->type) { + case LTTNG_EVENT_CONTAINER_COUNTER: + { + struct lttng_counter *counter; + const char *name = ""; + int ret; + + counter = lttng_event_container_get_counter(container); + if (event->key[0]) + name = event->key; + else + name = event_name; + ret = lttng_counter_append_descriptor(counter, + token, event->id, + name); + if (ret) { + WARN_ON_ONCE(1); + } + break; + } + case LTTNG_EVENT_CONTAINER_CHANNEL: + default: + break; + } break; - case LTTNG_KERNEL_FUNCTION: /* Fall-through */ + case LTTNG_KERNEL_FUNCTION: /* Fall-through. */ + case LTTNG_KERNEL_NOOP: /* Fall-through.*/ default: WARN_ON_ONCE(1); ret = -EINVAL; goto register_error; } - ret = _lttng_event_metadata_statedump(chan->session, chan, event); - WARN_ON_ONCE(ret > 0); - if (ret) { - goto statedump_error; + switch (container->type) { + case LTTNG_EVENT_CONTAINER_CHANNEL: + ret = _lttng_event_metadata_statedump(session, event); + WARN_ON_ONCE(ret > 0); + if (ret) { + goto statedump_error; + } + break; + case LTTNG_EVENT_CONTAINER_COUNTER: + default: + break; } - hlist_add_head(&event->hlist, head); - list_add(&event->list, &chan->session->events); + hlist_add_head(&event->name_hlist, name_head); + list_add(&event->list, &session->events); return event; statedump_error: /* If a statedump error occurs, events will not be readable. */ register_error: +full: kmem_cache_free(event_cache, event); cache_error: exist: type_error: -full: return ERR_PTR(ret); } @@ -1297,17 +1799,19 @@ int lttng_kernel_counter_clear(struct lttng_counter *counter, return counter->ops->counter_clear(counter->counter, dim_indexes); } -struct lttng_event *lttng_event_create(struct lttng_channel *chan, +struct lttng_event *lttng_event_create(struct lttng_event_container *container, struct lttng_kernel_event *event_param, + const struct lttng_counter_key *key, void *filter, const struct lttng_event_desc *event_desc, - enum lttng_kernel_instrumentation itype) + enum lttng_kernel_instrumentation itype, + uint64_t token) { struct lttng_event *event; mutex_lock(&sessions_mutex); - event = _lttng_event_create(chan, event_param, filter, event_desc, - itype); + event = _lttng_event_create(container, event_param, key, filter, event_desc, + itype, token); mutex_unlock(&sessions_mutex); return event; } @@ -1343,11 +1847,10 @@ void register_event(struct lttng_event *event) switch (event->instrumentation) { case LTTNG_KERNEL_TRACEPOINT: ret = lttng_wrapper_tracepoint_probe_register(desc->kname, - desc->probe_callback, - event); + desc->probe_callback, event); break; case LTTNG_KERNEL_SYSCALL: - ret = lttng_syscall_filter_enable_event(event->chan, event); + ret = lttng_syscall_filter_enable_event(event->container, event); break; case LTTNG_KERNEL_KPROBE: case LTTNG_KERNEL_UPROBE: @@ -1378,8 +1881,7 @@ int _lttng_event_unregister(struct lttng_event *event) switch (event->instrumentation) { case LTTNG_KERNEL_TRACEPOINT: ret = lttng_wrapper_tracepoint_probe_unregister(event->desc->kname, - event->desc->probe_callback, - event); + event->desc->probe_callback, event); break; case LTTNG_KERNEL_KPROBE: lttng_kprobes_unregister_event(event); @@ -1390,7 +1892,7 @@ int _lttng_event_unregister(struct lttng_event *event) ret = 0; break; case LTTNG_KERNEL_SYSCALL: - ret = lttng_syscall_filter_disable_event(event->chan, event); + ret = lttng_syscall_filter_disable_event(event->container, event); break; case LTTNG_KERNEL_NOOP: ret = 0; @@ -1922,19 +2424,19 @@ int lttng_desc_match_enabler(const struct lttng_event_desc *desc, } static -int lttng_event_enabler_match_event(struct lttng_event_enabler *event_enabler, +bool lttng_event_enabler_match_event(struct lttng_event_enabler *event_enabler, struct lttng_event *event) { struct lttng_enabler *base_enabler = lttng_event_enabler_as_enabler( event_enabler); - if (base_enabler->event_param.instrumentation != event->instrumentation) - return 0; - if (lttng_desc_match_enabler(event->desc, base_enabler) - && event->chan == event_enabler->chan) - return 1; + if (base_enabler->event_param.instrumentation == event->instrumentation + && lttng_desc_match_enabler(event->desc, base_enabler) > 0 + && event->container == event_enabler->container + && match_event_token(event->container, event, event_enabler->base.user_token)) + return true; else - return 0; + return false; } static @@ -1946,7 +2448,7 @@ int lttng_event_notifier_enabler_match_event_notifier(struct lttng_event_notifie if (base_enabler->event_param.instrumentation != event_notifier->instrumentation) return 0; - if (lttng_desc_match_enabler(event_notifier->desc, base_enabler) + if (lttng_desc_match_enabler(event_notifier->desc, base_enabler) > 0 && event_notifier->group == event_notifier_enabler->group && event_notifier->user_token == event_notifier_enabler->base.user_token) return 1; @@ -1971,7 +2473,6 @@ struct lttng_enabler_ref *lttng_enabler_ref( static void lttng_create_tracepoint_event_if_missing(struct lttng_event_enabler *event_enabler) { - struct lttng_session *session = event_enabler->chan->session; struct lttng_probe_desc *probe_desc; const struct lttng_event_desc *desc; int i; @@ -1985,37 +2486,22 @@ void lttng_create_tracepoint_event_if_missing(struct lttng_event_enabler *event_ */ list_for_each_entry(probe_desc, probe_list, head) { for (i = 0; i < probe_desc->nr_events; i++) { - int found = 0; - struct hlist_head *head; struct lttng_event *event; desc = probe_desc->event_desc[i]; - if (!lttng_desc_match_enabler(desc, - lttng_event_enabler_as_enabler(event_enabler))) + if (lttng_desc_match_enabler(desc, + lttng_event_enabler_as_enabler(event_enabler)) <= 0) continue; - /* - * Check if already created. - */ - head = utils_borrow_hash_table_bucket( - session->events_ht.table, LTTNG_EVENT_HT_SIZE, - desc->name); - lttng_hlist_for_each_entry(event, head, hlist) { - if (event->desc == desc - && event->chan == event_enabler->chan) - found = 1; - } - if (found) + /* Try to create an event for this event probe. */ + event = _lttng_event_create(event_enabler->container, + NULL, &event_enabler->key, NULL, desc, + LTTNG_KERNEL_TRACEPOINT, + event_enabler->base.user_token); + /* Skip if event is already found. */ + if (IS_ERR(event) && PTR_ERR(event) == -EEXIST) continue; - - /* - * We need to create an event for this - * event probe. - */ - event = _lttng_event_create(event_enabler->chan, - NULL, NULL, desc, - LTTNG_KERNEL_TRACEPOINT); - if (!event) { + if (IS_ERR(event)) { printk(KERN_INFO "LTTng: Unable to create event %s\n", probe_desc->event_desc[i]->name); } @@ -2045,8 +2531,8 @@ void lttng_create_tracepoint_event_notifier_if_missing(struct lttng_event_notifi struct lttng_event_notifier *event_notifier; desc = probe_desc->event_desc[i]; - if (!lttng_desc_match_enabler(desc, - lttng_event_notifier_enabler_as_enabler(event_notifier_enabler))) + if (lttng_desc_match_enabler(desc, + lttng_event_notifier_enabler_as_enabler(event_notifier_enabler)) <= 0) continue; /* @@ -2128,8 +2614,8 @@ void lttng_create_event_if_missing(struct lttng_event_enabler *event_enabler) static int lttng_event_enabler_ref_events(struct lttng_event_enabler *event_enabler) { - struct lttng_channel *chan = event_enabler->chan; - struct lttng_session *session = event_enabler->chan->session; + struct lttng_event_container *container = event_enabler->container; + struct lttng_session *session = container->session; struct lttng_enabler *base_enabler = lttng_event_enabler_as_enabler(event_enabler); struct lttng_event *event; @@ -2141,10 +2627,10 @@ int lttng_event_enabler_ref_events(struct lttng_event_enabler *event_enabler) enum lttng_kernel_syscall_entryexit entryexit = base_enabler->event_param.u.syscall.entryexit; if (entryexit == LTTNG_KERNEL_SYSCALL_ENTRY || entryexit == LTTNG_KERNEL_SYSCALL_ENTRYEXIT) - WRITE_ONCE(chan->syscall_all_entry, enabled); + WRITE_ONCE(container->syscall_all_entry, enabled); if (entryexit == LTTNG_KERNEL_SYSCALL_EXIT || entryexit == LTTNG_KERNEL_SYSCALL_ENTRYEXIT) - WRITE_ONCE(chan->syscall_all_exit, enabled); + WRITE_ONCE(container->syscall_all_exit, enabled); } /* First ensure that probe events are created for this enabler. */ @@ -2169,6 +2655,32 @@ int lttng_event_enabler_ref_events(struct lttng_event_enabler *event_enabler) enabler_ref->ref = lttng_event_enabler_as_enabler(event_enabler); list_add(&enabler_ref->node, &event->enablers_ref_head); + /* Append descriptor to counter. */ + switch (container->type) { + case LTTNG_EVENT_CONTAINER_COUNTER: + { + struct lttng_counter *counter; + const char *name = ""; + int ret; + + counter = lttng_event_container_get_counter(container); + if (event->key[0]) + name = event->key; + else if (event->desc && event->desc->name) + name = event->desc->name; + ret = lttng_counter_append_descriptor(counter, + event_enabler->base.user_token, event->id, + name); + if (ret) { + WARN_ON_ONCE(1); + return ret; + } + break; + } + case LTTNG_EVENT_CONTAINER_CHANNEL: + default: + break; + } } /* @@ -2326,7 +2838,8 @@ int lttng_fix_pending_event_notifiers(void) struct lttng_event_enabler *lttng_event_enabler_create( enum lttng_enabler_format_type format_type, struct lttng_kernel_event *event_param, - struct lttng_channel *chan) + const struct lttng_counter_key *key, + struct lttng_event_container *container) { struct lttng_event_enabler *event_enabler; @@ -2337,13 +2850,16 @@ struct lttng_event_enabler *lttng_event_enabler_create( INIT_LIST_HEAD(&event_enabler->base.filter_bytecode_head); memcpy(&event_enabler->base.event_param, event_param, sizeof(event_enabler->base.event_param)); - event_enabler->chan = chan; + event_enabler->container = container; /* ctx left NULL */ event_enabler->base.enabled = 0; event_enabler->base.evtype = LTTNG_TYPE_ENABLER; + event_enabler->base.user_token = event_param->token; + if (key) + event_enabler->key = *key; mutex_lock(&sessions_mutex); - list_add(&event_enabler->node, &event_enabler->chan->session->enablers_head); - lttng_session_lazy_sync_event_enablers(event_enabler->chan->session); + list_add(&event_enabler->node, &event_enabler->container->session->enablers_head); + lttng_session_lazy_sync_event_enablers(event_enabler->container->session); mutex_unlock(&sessions_mutex); return event_enabler; } @@ -2352,7 +2868,7 @@ int lttng_event_enabler_enable(struct lttng_event_enabler *event_enabler) { mutex_lock(&sessions_mutex); lttng_event_enabler_as_enabler(event_enabler)->enabled = 1; - lttng_session_lazy_sync_event_enablers(event_enabler->chan->session); + lttng_session_lazy_sync_event_enablers(event_enabler->container->session); mutex_unlock(&sessions_mutex); return 0; } @@ -2361,7 +2877,7 @@ int lttng_event_enabler_disable(struct lttng_event_enabler *event_enabler) { mutex_lock(&sessions_mutex); lttng_event_enabler_as_enabler(event_enabler)->enabled = 0; - lttng_session_lazy_sync_event_enablers(event_enabler->chan->session); + lttng_session_lazy_sync_event_enablers(event_enabler->container->session); mutex_unlock(&sessions_mutex); return 0; } @@ -2408,7 +2924,7 @@ int lttng_event_enabler_attach_filter_bytecode(struct lttng_event_enabler *event if (ret) goto error; - lttng_session_lazy_sync_event_enablers(event_enabler->chan->session); + lttng_session_lazy_sync_event_enablers(event_enabler->container->session); return 0; error: @@ -2622,7 +3138,7 @@ void lttng_session_sync_event_enablers(struct lttng_session *session) lttng_event_enabler_ref_events(event_enabler); /* * For each event, if at least one of its enablers is enabled, - * and its channel and session transient states are enabled, we + * and its event container and session transient states are enabled, we * enable the event, else we disable it. */ list_for_each_entry(event, &session->events, list) { @@ -2648,10 +3164,10 @@ void lttng_session_sync_event_enablers(struct lttng_session *session) } /* * Enabled state is based on union of enablers, with - * intesection of session and channel transient enable + * intesection of session and event container transient enable * states. */ - enabled = enabled && session->tstate && event->chan->tstate; + enabled = enabled && session->tstate && event->container->tstate; WRITE_ONCE(event->enabled, enabled); /* @@ -2768,6 +3284,8 @@ void lttng_event_notifier_group_sync_enablers(struct lttng_event_notifier_group list_for_each_entry(runtime, &event_notifier->capture_bytecode_runtime_head, node) lttng_bytecode_capture_sync_state(runtime); + + WRITE_ONCE(event_notifier->eval_capture, !!event_notifier->num_captures); } } @@ -3498,11 +4016,13 @@ int _lttng_fields_metadata_statedump(struct lttng_session *session, */ static int _lttng_event_metadata_statedump(struct lttng_session *session, - struct lttng_channel *chan, struct lttng_event *event) { + struct lttng_channel *chan; int ret = 0; + WARN_ON_ONCE(event->container->type != LTTNG_EVENT_CONTAINER_CHANNEL); + chan = lttng_event_container_get_channel(event->container); if (event->metadata_dumped || !LTTNG_READ_ONCE(session->active)) return 0; if (chan->channel_type == METADATA_CHANNEL) @@ -3513,11 +4033,11 @@ int _lttng_event_metadata_statedump(struct lttng_session *session, ret = lttng_metadata_printf(session, "event {\n" " name = \"%s\";\n" - " id = %u;\n" + " id = %zu;\n" " stream_id = %u;\n", event->desc->name, event->id, - event->chan->id); + chan->id); if (ret) goto end; @@ -3973,7 +4493,10 @@ skip_session: } list_for_each_entry(event, &session->events, list) { - ret = _lttng_event_metadata_statedump(session, event->chan, event); + /* Skip counter container. */ + if (event->container->type != LTTNG_EVENT_CONTAINER_CHANNEL) + continue; + ret = _lttng_event_metadata_statedump(session, event); if (ret) goto end; } diff --git a/src/lttng-ring-buffer-client.h b/src/lttng-ring-buffer-client.h index 4f8699c0..c234e8ae 100644 --- a/src/lttng-ring-buffer-client.h +++ b/src/lttng-ring-buffer-client.h @@ -141,7 +141,8 @@ size_t record_header_size(const struct lib_ring_buffer_config *config, struct lib_ring_buffer_ctx *ctx, struct lttng_client_ctx *client_ctx) { - struct lttng_channel *lttng_chan = channel_get_private(chan); + struct lttng_event_container *container = channel_get_private(chan); + struct lttng_channel *lttng_chan = lttng_event_container_get_channel(container); struct lttng_probe_ctx *lttng_probe_ctx = ctx->priv; struct lttng_event *event = lttng_probe_ctx->event; size_t orig_offset = offset; @@ -212,7 +213,8 @@ void lttng_write_event_header(const struct lib_ring_buffer_config *config, struct lib_ring_buffer_ctx *ctx, uint32_t event_id) { - struct lttng_channel *lttng_chan = channel_get_private(ctx->chan); + struct lttng_event_container *container = channel_get_private(ctx->chan); + struct lttng_channel *lttng_chan = lttng_event_container_get_channel(container); struct lttng_probe_ctx *lttng_probe_ctx = ctx->priv; struct lttng_event *event = lttng_probe_ctx->event; @@ -264,7 +266,8 @@ void lttng_write_event_header_slow(const struct lib_ring_buffer_config *config, struct lib_ring_buffer_ctx *ctx, uint32_t event_id) { - struct lttng_channel *lttng_chan = channel_get_private(ctx->chan); + struct lttng_event_container *container = channel_get_private(ctx->chan); + struct lttng_channel *lttng_chan = lttng_event_container_get_channel(container); struct lttng_probe_ctx *lttng_probe_ctx = ctx->priv; struct lttng_event *event = lttng_probe_ctx->event; @@ -365,8 +368,9 @@ static void client_buffer_begin(struct lib_ring_buffer *buf, u64 tsc, (struct packet_header *) lib_ring_buffer_offset_address(&buf->backend, subbuf_idx * chan->backend.subbuf_size); - struct lttng_channel *lttng_chan = channel_get_private(chan); - struct lttng_session *session = lttng_chan->session; + struct lttng_event_container *container = channel_get_private(chan); + struct lttng_channel *lttng_chan = lttng_event_container_get_channel(container); + struct lttng_session *session = container->session; header->magic = CTF_MAGIC_NUMBER; memcpy(header->uuid, session->uuid.b, sizeof(session->uuid)); @@ -480,7 +484,8 @@ static int client_stream_id(const struct lib_ring_buffer_config *config, uint64_t *stream_id) { struct channel *chan = buf->backend.chan; - struct lttng_channel *lttng_chan = channel_get_private(chan); + struct lttng_event_container *container = channel_get_private(chan); + struct lttng_channel *lttng_chan = lttng_event_container_get_channel(container); *stream_id = lttng_chan->id; return 0; @@ -555,10 +560,10 @@ struct channel *_channel_create(const char *name, unsigned int switch_timer_interval, unsigned int read_timer_interval) { - struct lttng_channel *lttng_chan = priv; + struct lttng_event_container *container = priv; struct channel *chan; - chan = channel_create(&client_config, name, lttng_chan, buf_addr, + chan = channel_create(&client_config, name, container, buf_addr, subbuf_size, num_subbuf, switch_timer_interval, read_timer_interval); if (chan) { @@ -618,7 +623,8 @@ static int lttng_event_reserve(struct lib_ring_buffer_ctx *ctx, uint32_t event_id) { - struct lttng_channel *lttng_chan = channel_get_private(ctx->chan); + struct lttng_event_container *container = channel_get_private(ctx->chan); + struct lttng_channel *lttng_chan = lttng_event_container_get_channel(container); struct lttng_probe_ctx *lttng_probe_ctx = ctx->priv; struct lttng_event *event = lttng_probe_ctx->event; struct lttng_client_ctx client_ctx; diff --git a/src/lttng-ring-buffer-metadata-client.h b/src/lttng-ring-buffer-metadata-client.h index 6fa0c2b1..c36dbd51 100644 --- a/src/lttng-ring-buffer-metadata-client.h +++ b/src/lttng-ring-buffer-metadata-client.h @@ -242,11 +242,11 @@ struct channel *_channel_create(const char *name, unsigned int switch_timer_interval, unsigned int read_timer_interval) { - struct lttng_channel *lttng_chan = priv; + struct lttng_event_container *container = priv; struct channel *chan; chan = channel_create(&client_config, name, - lttng_chan->session->metadata_cache, buf_addr, + container->session->metadata_cache, buf_addr, subbuf_size, num_subbuf, switch_timer_interval, read_timer_interval); if (chan) { diff --git a/src/lttng-syscalls.c b/src/lttng-syscalls.c index 3fce0914..eb813147 100644 --- a/src/lttng-syscalls.c +++ b/src/lttng-syscalls.c @@ -639,33 +639,33 @@ void syscall_entry_event_notifier_call_func(struct hlist_head *dispatch_list, void syscall_entry_event_probe(void *__data, struct pt_regs *regs, long id) { - struct lttng_channel *chan = __data; + struct lttng_event_container *container = __data; struct hlist_head *action_list, *unknown_action_list; const struct trace_syscall_entry *table, *entry; size_t table_len; if (unlikely(in_compat_syscall())) { - struct lttng_syscall_filter *filter = chan->sc_filter; + struct lttng_syscall_filter *filter = container->sc_filter; if (id < 0 || id >= NR_compat_syscalls - || (!READ_ONCE(chan->syscall_all_entry) && !test_bit(id, filter->sc_compat_entry))) { + || (!READ_ONCE(container->syscall_all_entry) && !test_bit(id, filter->sc_compat_entry))) { /* System call filtered out. */ return; } table = compat_sc_table; table_len = ARRAY_SIZE(compat_sc_table); - unknown_action_list = &chan->sc_compat_unknown; + unknown_action_list = &container->sc_compat_unknown; } else { - struct lttng_syscall_filter *filter = chan->sc_filter; + struct lttng_syscall_filter *filter = container->sc_filter; if (id < 0 || id >= NR_syscalls - || (!READ_ONCE(chan->syscall_all_entry) && !test_bit(id, filter->sc_entry))) { + || (!READ_ONCE(container->syscall_all_entry) && !test_bit(id, filter->sc_entry))) { /* System call filtered out. */ return; } table = sc_table; table_len = ARRAY_SIZE(sc_table); - unknown_action_list = &chan->sc_unknown; + unknown_action_list = &container->sc_unknown; } if (unlikely(id < 0 || id >= table_len)) { syscall_entry_event_unknown(unknown_action_list, regs, id); @@ -679,9 +679,9 @@ void syscall_entry_event_probe(void *__data, struct pt_regs *regs, long id) } if (unlikely(in_compat_syscall())) { - action_list = &chan->compat_sc_table[id]; + action_list = &container->compat_sc_table[id]; } else { - action_list = &chan->sc_table[id]; + action_list = &container->sc_table[id]; } if (unlikely(hlist_empty(action_list))) return; @@ -875,7 +875,7 @@ void syscall_exit_call_func(struct hlist_head *action_list, void syscall_exit_event_probe(void *__data, struct pt_regs *regs, long ret) { - struct lttng_channel *chan = __data; + struct lttng_event_container *container = __data; struct hlist_head *action_list, *unknown_action_list; const struct trace_syscall_entry *table, *entry; size_t table_len; @@ -884,27 +884,27 @@ void syscall_exit_event_probe(void *__data, struct pt_regs *regs, long ret) id = syscall_get_nr(current, regs); if (unlikely(in_compat_syscall())) { - struct lttng_syscall_filter *filter = chan->sc_filter; + struct lttng_syscall_filter *filter = container->sc_filter; if (id < 0 || id >= NR_compat_syscalls - || (!READ_ONCE(chan->syscall_all_exit) && !test_bit(id, filter->sc_compat_exit))) { + || (!READ_ONCE(container->syscall_all_exit) && !test_bit(id, filter->sc_compat_exit))) { /* System call filtered out. */ return; } table = compat_sc_exit_table; table_len = ARRAY_SIZE(compat_sc_exit_table); - unknown_action_list = &chan->compat_sc_exit_unknown; + unknown_action_list = &container->compat_sc_exit_unknown; } else { - struct lttng_syscall_filter *filter = chan->sc_filter; + struct lttng_syscall_filter *filter = container->sc_filter; if (id < 0 || id >= NR_syscalls - || (!READ_ONCE(chan->syscall_all_exit) && !test_bit(id, filter->sc_exit))) { + || (!READ_ONCE(container->syscall_all_exit) && !test_bit(id, filter->sc_exit))) { /* System call filtered out. */ return; } table = sc_exit_table; table_len = ARRAY_SIZE(sc_exit_table); - unknown_action_list = &chan->sc_exit_unknown; + unknown_action_list = &container->sc_exit_unknown; } if (unlikely(id < 0 || id >= table_len)) { syscall_exit_event_unknown(unknown_action_list, regs, id, ret); @@ -918,9 +918,9 @@ void syscall_exit_event_probe(void *__data, struct pt_regs *regs, long ret) } if (unlikely(in_compat_syscall())) { - action_list = &chan->compat_sc_exit_table[id]; + action_list = &container->compat_sc_exit_table[id]; } else { - action_list = &chan->sc_exit_table[id]; + action_list = &container->sc_exit_table[id]; } if (unlikely(hlist_empty(action_list))) return; @@ -1102,11 +1102,10 @@ void syscall_exit_event_notifier_probe(void *__data, struct pt_regs *regs, */ static int lttng_create_syscall_event_if_missing(const struct trace_syscall_entry *table, size_t table_len, - struct hlist_head *chan_table, struct lttng_event_enabler *event_enabler, + struct hlist_head *container_table, struct lttng_event_enabler *event_enabler, void *filter, enum sc_type type) { - struct lttng_channel *chan = event_enabler->chan; - struct lttng_session *session = chan->session; + struct lttng_event_container *container = event_enabler->container; unsigned int i; /* Allocate events for each syscall matching enabler, insert into table */ @@ -1114,8 +1113,6 @@ int lttng_create_syscall_event_if_missing(const struct trace_syscall_entry *tabl const struct lttng_event_desc *desc = table[i].desc; struct lttng_kernel_event ev; struct lttng_event *event; - struct hlist_head *head; - bool found = false; if (!desc) { /* Unknown syscall */ @@ -1124,19 +1121,6 @@ int lttng_create_syscall_event_if_missing(const struct trace_syscall_entry *tabl if (lttng_desc_match_enabler(desc, lttng_event_enabler_as_enabler(event_enabler)) <= 0) continue; - /* - * Check if already created. - */ - head = utils_borrow_hash_table_bucket( - session->events_ht.table, LTTNG_EVENT_HT_SIZE, - desc->name); - lttng_hlist_for_each_entry(event, head, hlist) { - if (event->desc == desc - && event->chan == event_enabler->chan) - found = true; - } - if (found) - continue; /* We need to create an event for this syscall/enabler. */ memset(&ev, 0, sizeof(ev)); @@ -1161,9 +1145,12 @@ int lttng_create_syscall_event_if_missing(const struct trace_syscall_entry *tabl strncpy(ev.name, desc->name, LTTNG_KERNEL_SYM_NAME_LEN - 1); ev.name[LTTNG_KERNEL_SYM_NAME_LEN - 1] = '\0'; ev.instrumentation = LTTNG_KERNEL_SYSCALL; - event = _lttng_event_create(chan, &ev, filter, - desc, ev.instrumentation); - WARN_ON_ONCE(!event); + event = _lttng_event_create(container, &ev, &event_enabler->key, filter, + desc, ev.instrumentation, + event_enabler->base.user_token); + /* Skip if event is already found. */ + if (IS_ERR(event) && PTR_ERR(event) == -EEXIST) + continue; if (IS_ERR(event)) { /* * If something goes wrong in event registration @@ -1173,7 +1160,7 @@ int lttng_create_syscall_event_if_missing(const struct trace_syscall_entry *tabl */ return PTR_ERR(event); } - hlist_add_head(&event->u.syscall.node, &chan_table[i]); + hlist_add_head(&event->u.syscall.node, &container_table[i]); } return 0; } @@ -1183,46 +1170,46 @@ int lttng_create_syscall_event_if_missing(const struct trace_syscall_entry *tabl */ int lttng_syscalls_register_event(struct lttng_event_enabler *event_enabler, void *filter) { - struct lttng_channel *chan = event_enabler->chan; + struct lttng_event_container *container = event_enabler->container; struct lttng_kernel_event ev; int ret; wrapper_vmalloc_sync_mappings(); - if (!chan->sc_table) { + if (!container->sc_table) { /* create syscall table mapping syscall to events */ - chan->sc_table = kzalloc(sizeof(struct lttng_event *) + container->sc_table = kzalloc(sizeof(struct hlist_head) * ARRAY_SIZE(sc_table), GFP_KERNEL); - if (!chan->sc_table) + if (!container->sc_table) return -ENOMEM; } - if (!chan->sc_exit_table) { + if (!container->sc_exit_table) { /* create syscall table mapping syscall to events */ - chan->sc_exit_table = kzalloc(sizeof(struct lttng_event *) + container->sc_exit_table = kzalloc(sizeof(struct hlist_head) * ARRAY_SIZE(sc_exit_table), GFP_KERNEL); - if (!chan->sc_exit_table) + if (!container->sc_exit_table) return -ENOMEM; } - #ifdef CONFIG_COMPAT - if (!chan->compat_sc_table) { + if (!container->compat_sc_table) { /* create syscall table mapping compat syscall to events */ - chan->compat_sc_table = kzalloc(sizeof(struct lttng_event *) + container->compat_sc_table = kzalloc(sizeof(struct hlist_head) * ARRAY_SIZE(compat_sc_table), GFP_KERNEL); - if (!chan->compat_sc_table) + if (!container->compat_sc_table) return -ENOMEM; } - if (!chan->compat_sc_exit_table) { + if (!container->compat_sc_exit_table) { /* create syscall table mapping compat syscall to events */ - chan->compat_sc_exit_table = kzalloc(sizeof(struct lttng_event *) + container->compat_sc_exit_table = kzalloc(sizeof(struct hlist_head) * ARRAY_SIZE(compat_sc_exit_table), GFP_KERNEL); - if (!chan->compat_sc_exit_table) + if (!container->compat_sc_exit_table) return -ENOMEM; } #endif - if (hlist_empty(&chan->sc_unknown)) { + + { const struct lttng_event_desc *desc = &__event_desc___syscall_entry_unknown; struct lttng_event *event; @@ -1233,16 +1220,19 @@ int lttng_syscalls_register_event(struct lttng_event_enabler *event_enabler, voi ev.instrumentation = LTTNG_KERNEL_SYSCALL; ev.u.syscall.entryexit = LTTNG_KERNEL_SYSCALL_ENTRY; ev.u.syscall.abi = LTTNG_KERNEL_SYSCALL_ABI_NATIVE; - event = _lttng_event_create(chan, &ev, filter, desc, - ev.instrumentation); - WARN_ON_ONCE(!event); + event = _lttng_event_create(container, &ev, &event_enabler->key, filter, + desc, ev.instrumentation, + event_enabler->base.user_token); if (IS_ERR(event)) { - return PTR_ERR(event); + if (PTR_ERR(event) != -EEXIST) + return PTR_ERR(event); + /* Skip if event is already found. */ + } else { + hlist_add_head(&event->u.syscall.node, &container->sc_unknown); } - hlist_add_head(&event->u.syscall.node, &chan->sc_unknown); } - if (hlist_empty(&chan->sc_compat_unknown)) { + { const struct lttng_event_desc *desc = &__event_desc___compat_syscall_entry_unknown; struct lttng_event *event; @@ -1253,16 +1243,19 @@ int lttng_syscalls_register_event(struct lttng_event_enabler *event_enabler, voi ev.instrumentation = LTTNG_KERNEL_SYSCALL; ev.u.syscall.entryexit = LTTNG_KERNEL_SYSCALL_ENTRY; ev.u.syscall.abi = LTTNG_KERNEL_SYSCALL_ABI_COMPAT; - event = _lttng_event_create(chan, &ev, filter, desc, - ev.instrumentation); - WARN_ON_ONCE(!event); + event = _lttng_event_create(container, &ev, &event_enabler->key, filter, + desc, ev.instrumentation, + event_enabler->base.user_token); if (IS_ERR(event)) { - return PTR_ERR(event); + if (PTR_ERR(event) != -EEXIST) + return PTR_ERR(event); + /* Skip if event is already found. */ + } else { + hlist_add_head(&event->u.syscall.node, &container->sc_compat_unknown); } - hlist_add_head(&event->u.syscall.node, &chan->sc_compat_unknown); } - if (hlist_empty(&chan->compat_sc_exit_unknown)) { + { const struct lttng_event_desc *desc = &__event_desc___compat_syscall_exit_unknown; struct lttng_event *event; @@ -1273,16 +1266,19 @@ int lttng_syscalls_register_event(struct lttng_event_enabler *event_enabler, voi ev.instrumentation = LTTNG_KERNEL_SYSCALL; ev.u.syscall.entryexit = LTTNG_KERNEL_SYSCALL_EXIT; ev.u.syscall.abi = LTTNG_KERNEL_SYSCALL_ABI_COMPAT; - event = _lttng_event_create(chan, &ev, filter, desc, - ev.instrumentation); - WARN_ON_ONCE(!event); + event = _lttng_event_create(container, &ev, &event_enabler->key, + filter, desc, ev.instrumentation, + event_enabler->base.user_token); if (IS_ERR(event)) { - return PTR_ERR(event); + if (PTR_ERR(event) != -EEXIST) + return PTR_ERR(event); + /* Skip if event is already found. */ + } else { + hlist_add_head(&event->u.syscall.node, &container->compat_sc_exit_unknown); } - hlist_add_head(&event->u.syscall.node, &chan->compat_sc_exit_unknown); } - if (hlist_empty(&chan->sc_exit_unknown)) { + { const struct lttng_event_desc *desc = &__event_desc___syscall_exit_unknown; struct lttng_event *event; @@ -1293,64 +1289,67 @@ int lttng_syscalls_register_event(struct lttng_event_enabler *event_enabler, voi ev.instrumentation = LTTNG_KERNEL_SYSCALL; ev.u.syscall.entryexit = LTTNG_KERNEL_SYSCALL_EXIT; ev.u.syscall.abi = LTTNG_KERNEL_SYSCALL_ABI_NATIVE; - event = _lttng_event_create(chan, &ev, filter, desc, - ev.instrumentation); - WARN_ON_ONCE(!event); + event = _lttng_event_create(container, &ev, &event_enabler->key, filter, + desc, ev.instrumentation, + event_enabler->base.user_token); if (IS_ERR(event)) { - return PTR_ERR(event); + if (PTR_ERR(event) != -EEXIST) + return PTR_ERR(event); + /* Skip if event is already found. */ + } else { + hlist_add_head(&event->u.syscall.node, &container->sc_exit_unknown); } - hlist_add_head(&event->u.syscall.node, &chan->sc_exit_unknown); } ret = lttng_create_syscall_event_if_missing(sc_table, ARRAY_SIZE(sc_table), - chan->sc_table, event_enabler, filter, SC_TYPE_ENTRY); + container->sc_table, event_enabler, filter, SC_TYPE_ENTRY); if (ret) return ret; ret = lttng_create_syscall_event_if_missing(sc_exit_table, ARRAY_SIZE(sc_exit_table), - chan->sc_exit_table, event_enabler, filter, SC_TYPE_EXIT); + container->sc_exit_table, event_enabler, filter, SC_TYPE_EXIT); if (ret) return ret; #ifdef CONFIG_COMPAT ret = lttng_create_syscall_event_if_missing(compat_sc_table, ARRAY_SIZE(compat_sc_table), - chan->compat_sc_table, event_enabler, filter, + container->compat_sc_table, event_enabler, filter, SC_TYPE_COMPAT_ENTRY); if (ret) return ret; ret = lttng_create_syscall_event_if_missing(compat_sc_exit_table, ARRAY_SIZE(compat_sc_exit_table), - chan->compat_sc_exit_table, event_enabler, filter, + container->compat_sc_exit_table, event_enabler, filter, SC_TYPE_COMPAT_EXIT); if (ret) return ret; #endif - if (!chan->sc_filter) { - chan->sc_filter = kzalloc(sizeof(struct lttng_syscall_filter), + if (!container->sc_filter) { + container->sc_filter = kzalloc(sizeof(struct lttng_syscall_filter), GFP_KERNEL); - if (!chan->sc_filter) + if (!container->sc_filter) return -ENOMEM; } - if (!chan->sys_enter_registered) { + if (!container->sys_enter_registered) { ret = lttng_wrapper_tracepoint_probe_register("sys_enter", - (void *) syscall_entry_event_probe, chan); + (void *) syscall_entry_event_probe, container); if (ret) return ret; - chan->sys_enter_registered = 1; + container->sys_enter_registered = 1; } /* * We change the name of sys_exit tracepoint due to namespace * conflict with sys_exit syscall entry. */ - if (!chan->sys_exit_registered) { + if (!container->sys_exit_registered) { ret = lttng_wrapper_tracepoint_probe_register("sys_exit", - (void *) syscall_exit_event_probe, chan); + (void *) syscall_exit_event_probe, container); if (ret) { WARN_ON_ONCE(lttng_wrapper_tracepoint_probe_unregister("sys_enter", - (void *) syscall_entry_event_probe, chan)); + (void *) syscall_entry_event_probe, container)); return ret; } - chan->sys_exit_registered = 1; + container->sys_exit_registered = 1; } return ret; } @@ -1572,8 +1571,8 @@ static int create_matching_event_notifiers( continue; } - if (!lttng_desc_match_enabler(desc, - lttng_event_notifier_enabler_as_enabler(event_notifier_enabler))) + if (lttng_desc_match_enabler(desc, + lttng_event_notifier_enabler_as_enabler(event_notifier_enabler)) <= 0) continue; /* @@ -1628,7 +1627,6 @@ static int create_matching_event_notifiers( end: return ret; - } int lttng_syscals_create_matching_event_notifiers( @@ -1729,38 +1727,38 @@ int lttng_syscalls_unregister_event_notifier_group( return 0; } -int lttng_syscalls_unregister_channel(struct lttng_channel *chan) +int lttng_syscalls_unregister_event_container(struct lttng_event_container *container) { int ret; - if (!chan->sc_table) + if (!container->sc_table) return 0; - if (chan->sys_enter_registered) { + if (container->sys_enter_registered) { ret = lttng_wrapper_tracepoint_probe_unregister("sys_enter", - (void *) syscall_entry_event_probe, chan); + (void *) syscall_entry_event_probe, container); if (ret) return ret; - chan->sys_enter_registered = 0; + container->sys_enter_registered = 0; } - if (chan->sys_exit_registered) { + if (container->sys_exit_registered) { ret = lttng_wrapper_tracepoint_probe_unregister("sys_exit", - (void *) syscall_exit_event_probe, chan); + (void *) syscall_exit_event_probe, container); if (ret) return ret; - chan->sys_exit_registered = 0; + container->sys_exit_registered = 0; } return 0; } -int lttng_syscalls_destroy_event(struct lttng_channel *chan) +int lttng_syscalls_destroy_event_container(struct lttng_event_container *container) { - kfree(chan->sc_table); - kfree(chan->sc_exit_table); + kfree(container->sc_table); + kfree(container->sc_exit_table); #ifdef CONFIG_COMPAT - kfree(chan->compat_sc_table); - kfree(chan->compat_sc_exit_table); + kfree(container->compat_sc_table); + kfree(container->compat_sc_exit_table); #endif - kfree(chan->sc_filter); + kfree(container->sc_filter); return 0; } @@ -1965,12 +1963,12 @@ end: } int lttng_syscall_filter_enable_event( - struct lttng_channel *channel, + struct lttng_event_container *container, struct lttng_event *event) { WARN_ON_ONCE(event->instrumentation != LTTNG_KERNEL_SYSCALL); - return lttng_syscall_filter_enable(channel->sc_filter, + return lttng_syscall_filter_enable(container->sc_filter, event->desc->name, event->u.syscall.abi, event->u.syscall.entryexit); } @@ -2053,10 +2051,10 @@ int lttng_syscall_filter_disable_event_notifier( } int lttng_syscall_filter_disable_event( - struct lttng_channel *channel, + struct lttng_event_container *container, struct lttng_event *event) { - return lttng_syscall_filter_disable(channel->sc_filter, + return lttng_syscall_filter_disable(container->sc_filter, event->desc->name, event->u.syscall.abi, event->u.syscall.entryexit); } @@ -2175,7 +2173,7 @@ const struct file_operations lttng_syscall_list_fops = { /* * A syscall is enabled if it is traced for either entry or exit. */ -long lttng_channel_syscall_mask(struct lttng_channel *channel, +long lttng_event_container_syscall_mask(struct lttng_event_container *container, struct lttng_kernel_syscall_mask __user *usyscall_mask) { uint32_t len, sc_tables_len, bitmask_len; @@ -2195,14 +2193,14 @@ long lttng_channel_syscall_mask(struct lttng_channel *channel, tmp_mask = kzalloc(bitmask_len, GFP_KERNEL); if (!tmp_mask) return -ENOMEM; - filter = channel->sc_filter; + filter = container->sc_filter; for (bit = 0; bit < ARRAY_SIZE(sc_table); bit++) { char state; - if (channel->sc_table) { - if (!(READ_ONCE(channel->syscall_all_entry) - || READ_ONCE(channel->syscall_all_exit)) && filter) + if (container->sc_table) { + if (!(READ_ONCE(container->syscall_all_entry) + || READ_ONCE(container->syscall_all_exit)) && filter) state = test_bit(bit, filter->sc_entry) || test_bit(bit, filter->sc_exit); else @@ -2215,9 +2213,9 @@ long lttng_channel_syscall_mask(struct lttng_channel *channel, for (; bit < sc_tables_len; bit++) { char state; - if (channel->compat_sc_table) { - if (!(READ_ONCE(channel->syscall_all_entry) - || READ_ONCE(channel->syscall_all_exit)) && filter) + if (container->compat_sc_table) { + if (!(READ_ONCE(container->syscall_all_entry) + || READ_ONCE(container->syscall_all_exit)) && filter) state = test_bit(bit - ARRAY_SIZE(sc_table), filter->sc_compat_entry) || test_bit(bit - ARRAY_SIZE(sc_table), diff --git a/src/probes/lttng-kprobes.c b/src/probes/lttng-kprobes.c index 6824088c..b7d1c85d 100644 --- a/src/probes/lttng-kprobes.c +++ b/src/probes/lttng-kprobes.c @@ -26,26 +26,42 @@ int lttng_kprobes_event_handler_pre(struct kprobe *p, struct pt_regs *regs) .event = event, .interruptible = !lttng_regs_irqs_disabled(regs), }; - struct lttng_channel *chan = event->chan; - struct lib_ring_buffer_ctx ctx; + struct lttng_event_container *container = event->container; int ret; unsigned long data = (unsigned long) p->addr; - if (unlikely(!LTTNG_READ_ONCE(chan->session->active))) + if (unlikely(!LTTNG_READ_ONCE(container->session->active))) return 0; - if (unlikely(!LTTNG_READ_ONCE(chan->enabled))) + if (unlikely(!LTTNG_READ_ONCE(container->enabled))) return 0; if (unlikely(!LTTNG_READ_ONCE(event->enabled))) return 0; - lib_ring_buffer_ctx_init(&ctx, chan->chan, <tng_probe_ctx, sizeof(data), - lttng_alignof(data), -1); - ret = chan->ops->event_reserve(&ctx, event->id); - if (ret < 0) - return 0; - lib_ring_buffer_align_ctx(&ctx, lttng_alignof(data)); - chan->ops->event_write(&ctx, &data, sizeof(data)); - chan->ops->event_commit(&ctx); + switch (container->type) { + case LTTNG_EVENT_CONTAINER_CHANNEL: + { + struct lttng_channel *chan = lttng_event_container_get_channel(container); + struct lib_ring_buffer_ctx ctx; + + lib_ring_buffer_ctx_init(&ctx, chan->chan, <tng_probe_ctx, sizeof(data), + lttng_alignof(data), -1); + ret = chan->ops->event_reserve(&ctx, event->id); + if (ret < 0) + return 0; + lib_ring_buffer_align_ctx(&ctx, lttng_alignof(data)); + chan->ops->event_write(&ctx, &data, sizeof(data)); + chan->ops->event_commit(&ctx); + break; + } + case LTTNG_EVENT_CONTAINER_COUNTER: + { + struct lttng_counter *counter = lttng_event_container_get_counter(container); + size_t index = event->id; + + (void) counter->ops->counter_add(counter->counter, &index, 1); + break; + } + } return 0; } @@ -54,11 +70,13 @@ int lttng_kprobes_event_notifier_handler_pre(struct kprobe *p, struct pt_regs *r { struct lttng_event_notifier *event_notifier = container_of(p, struct lttng_event_notifier, u.kprobe.kp); + struct lttng_kernel_notifier_ctx notif_ctx; if (unlikely(!READ_ONCE(event_notifier->enabled))) return 0; - event_notifier->send_notification(event_notifier, NULL, NULL); + notif_ctx.eval_capture = LTTNG_READ_ONCE(event_notifier->eval_capture); + event_notifier->send_notification(event_notifier, NULL, NULL, ¬if_ctx); return 0; } diff --git a/src/probes/lttng-kretprobes.c b/src/probes/lttng-kretprobes.c index 24cb52e4..b3e9f627 100644 --- a/src/probes/lttng-kretprobes.c +++ b/src/probes/lttng-kretprobes.c @@ -43,32 +43,48 @@ int _lttng_kretprobes_handler(struct kretprobe_instance *krpi, .event = event, .interruptible = !lttng_regs_irqs_disabled(regs), }; - struct lttng_channel *chan = event->chan; - struct lib_ring_buffer_ctx ctx; + struct lttng_event_container *container = event->container; int ret; struct { unsigned long ip; unsigned long parent_ip; } payload; - if (unlikely(!LTTNG_READ_ONCE(chan->session->active))) + if (unlikely(!LTTNG_READ_ONCE(container->session->active))) return 0; - if (unlikely(!LTTNG_READ_ONCE(chan->enabled))) + if (unlikely(!LTTNG_READ_ONCE(container->enabled))) return 0; if (unlikely(!LTTNG_READ_ONCE(event->enabled))) return 0; - payload.ip = (unsigned long) lttng_get_kretprobe(krpi)->kp.addr; - payload.parent_ip = (unsigned long) krpi->ret_addr; + switch (container->type) { + case LTTNG_EVENT_CONTAINER_CHANNEL: + { + struct lttng_channel *chan = lttng_event_container_get_channel(container); + struct lib_ring_buffer_ctx ctx; - lib_ring_buffer_ctx_init(&ctx, chan->chan, <tng_probe_ctx, sizeof(payload), - lttng_alignof(payload), -1); - ret = chan->ops->event_reserve(&ctx, event->id); - if (ret < 0) - return 0; - lib_ring_buffer_align_ctx(&ctx, lttng_alignof(payload)); - chan->ops->event_write(&ctx, &payload, sizeof(payload)); - chan->ops->event_commit(&ctx); + payload.ip = (unsigned long) lttng_get_kretprobe(krpi)->kp.addr; + payload.parent_ip = (unsigned long) krpi->ret_addr; + + lib_ring_buffer_ctx_init(&ctx, chan->chan, <tng_probe_ctx, sizeof(payload), + lttng_alignof(payload), -1); + ret = chan->ops->event_reserve(&ctx, event->id); + if (ret < 0) + return 0; + lib_ring_buffer_align_ctx(&ctx, lttng_alignof(payload)); + chan->ops->event_write(&ctx, &payload, sizeof(payload)); + chan->ops->event_commit(&ctx); + break; + } + case LTTNG_EVENT_CONTAINER_COUNTER: + { + struct lttng_counter *counter = lttng_event_container_get_counter(container); + size_t index = event->id; + + (void) counter->ops->counter_add(counter->counter, &index, 1); + break; + } + } return 0; } diff --git a/src/probes/lttng-uprobes.c b/src/probes/lttng-uprobes.c index 031bfa7c..fbed39eb 100644 --- a/src/probes/lttng-uprobes.c +++ b/src/probes/lttng-uprobes.c @@ -32,34 +32,50 @@ int lttng_uprobes_event_handler_pre(struct uprobe_consumer *uc, struct pt_regs * .event = event, .interruptible = !lttng_regs_irqs_disabled(regs), }; - struct lttng_channel *chan = event->chan; - struct lib_ring_buffer_ctx ctx; + struct lttng_event_container *container = event->container; int ret; struct { unsigned long ip; } payload; - if (unlikely(!LTTNG_READ_ONCE(chan->session->active))) + if (unlikely(!LTTNG_READ_ONCE(container->session->active))) return 0; - if (unlikely(!LTTNG_READ_ONCE(chan->enabled))) + if (unlikely(!LTTNG_READ_ONCE(container->enabled))) return 0; if (unlikely(!LTTNG_READ_ONCE(event->enabled))) return 0; - lib_ring_buffer_ctx_init(&ctx, chan->chan, <tng_probe_ctx, - sizeof(payload), lttng_alignof(payload), -1); + switch (container->type) { + case LTTNG_EVENT_CONTAINER_CHANNEL: + { + struct lttng_channel *chan = lttng_event_container_get_channel(container); + struct lib_ring_buffer_ctx ctx; - ret = chan->ops->event_reserve(&ctx, event->id); - if (ret < 0) - return 0; + lib_ring_buffer_ctx_init(&ctx, chan->chan, <tng_probe_ctx, + sizeof(payload), lttng_alignof(payload), -1); + + ret = chan->ops->event_reserve(&ctx, event->id); + if (ret < 0) + return 0; + + /* Event payload. */ + payload.ip = (unsigned long)instruction_pointer(regs); - /* Event payload. */ - payload.ip = (unsigned long)instruction_pointer(regs); + lib_ring_buffer_align_ctx(&ctx, lttng_alignof(payload)); + chan->ops->event_write(&ctx, &payload, sizeof(payload)); + chan->ops->event_commit(&ctx); + break; + } + case LTTNG_EVENT_CONTAINER_COUNTER: + { + struct lttng_counter *counter = lttng_event_container_get_counter(container); + size_t index = event->id; - lib_ring_buffer_align_ctx(&ctx, lttng_alignof(payload)); - chan->ops->event_write(&ctx, &payload, sizeof(payload)); - chan->ops->event_commit(&ctx); + (void) counter->ops->counter_add(counter->counter, &index, 1); + break; + } + } return 0; } @@ -69,11 +85,13 @@ int lttng_uprobes_event_notifier_handler_pre(struct uprobe_consumer *uc, struct struct lttng_uprobe_handler *uprobe_handler = container_of(uc, struct lttng_uprobe_handler, up_consumer); struct lttng_event_notifier *event_notifier = uprobe_handler->u.event_notifier; + struct lttng_kernel_notifier_ctx notif_ctx; if (unlikely(!READ_ONCE(event_notifier->enabled))) return 0; - event_notifier->send_notification(event_notifier, NULL, NULL); + notif_ctx.eval_capture = LTTNG_READ_ONCE(event_notifier->eval_capture); + event_notifier->send_notification(event_notifier, NULL, NULL, ¬if_ctx); return 0; } -- 2.34.1