tracing: branch tracer, fix vdso crash

[deliverable/linux.git] / include / linux / compiler.h

diff --git a/include/linux/compiler.h b/include/linux/compiler.h
index 8322141ee480c802ee6919d0f13a86218e45bfc4..63b7d9089d6e1642a10481d319a960172c9c531e 100644 (file)
--- a/include/linux/compiler.h
+++ b/include/linux/compiler.h
@@ -44,6 +44,8 @@ extern void __chk_io_ptr(const volatile void __iomem *);
 # error Sorry, your compiler is too old/not recognized.
 #endif
 
+#define notrace __attribute__((no_instrument_function))
+
 /* Intel compiler defines __GNUC__. So we will overwrite implementations
  * coming from above header files here
  */
@@ -57,8 +59,69 @@ extern void __chk_io_ptr(const volatile void __iomem *);
  * specific implementations come from the above header files
  */
 
-#define likely(x)      __builtin_expect(!!(x), 1)
-#define unlikely(x)    __builtin_expect(!!(x), 0)
+/*
+ * Note: DISABLE_UNLIKELY_PROFILE can be used by special lowlevel code
+ * to disable branch tracing on a per file basis.
+ */
+#if defined(CONFIG_TRACE_UNLIKELY_PROFILE) && !defined(DISABLE_UNLIKELY_PROFILE)
+struct ftrace_likely_data {
+       const char *func;
+       const char *file;
+       unsigned line;
+       unsigned long correct;
+       unsigned long incorrect;
+};
+void ftrace_likely_update(struct ftrace_likely_data *f, int val, int expect);
+
+#define likely_notrace(x)      __builtin_expect(!!(x), 1)
+#define unlikely_notrace(x)    __builtin_expect(!!(x), 0)
+
+#define likely_check(x) ({                                             \
+                       int ______r;                                    \
+                       static struct ftrace_likely_data                \
+                               __attribute__((__aligned__(4)))         \
+                               __attribute__((section("_ftrace_likely"))) \
+                               ______f = {                             \
+                               .func = __func__,                       \
+                               .file = __FILE__,                       \
+                               .line = __LINE__,                       \
+                       };                                              \
+                       ______f.line = __LINE__;                        \
+                       ______r = likely_notrace(x);                    \
+                       ftrace_likely_update(&______f, ______r, 1);     \
+                       ______r;                                        \
+               })
+#define unlikely_check(x) ({                                           \
+                       int ______r;                                    \
+                       static struct ftrace_likely_data                \
+                               __attribute__((__aligned__(4)))         \
+                               __attribute__((section("_ftrace_unlikely"))) \
+                               ______f = {                             \
+                               .func = __func__,                       \
+                               .file = __FILE__,                       \
+                               .line = __LINE__,                       \
+                       };                                              \
+                       ______f.line = __LINE__;                        \
+                       ______r = unlikely_notrace(x);                  \
+                       ftrace_likely_update(&______f, ______r, 0);     \
+                       ______r;                                        \
+               })
+
+/*
+ * Using __builtin_constant_p(x) to ignore cases where the return
+ * value is always the same.  This idea is taken from a similar patch
+ * written by Daniel Walker.
+ */
+# ifndef likely
+#  define likely(x)    (__builtin_constant_p(x) ? !!(x) : likely_check(x))
+# endif
+# ifndef unlikely
+#  define unlikely(x)  (__builtin_constant_p(x) ? !!(x) : unlikely_check(x))
+# endif
+#else
+# define likely(x)     __builtin_expect(!!(x), 1)
+# define unlikely(x)   __builtin_expect(!!(x), 0)
+#endif
 
 /* Optimization barrier */
 #ifndef barrier