tree/tiny rcu: Add debug RCU head objects

Helps finding racy users of call_rcu(), which results in hangs because list
entries are overwritten and/or skipped.

Changelog since v4:
- Bissectability is now OK
- Now generate a WARN_ON_ONCE() for non-initialized rcu_head passed to
  call_rcu(). Statically initialized objects are detected with
  object_is_static().
- Rename rcu_head_init_on_stack to init_rcu_head_on_stack.
- Remove init_rcu_head() completely.

Changelog since v3:
- Include comments from Lai Jiangshan

This new patch version is based on the debugobjects with the newly introduced
"active state" tracker.

Non-initialized entries are all considered as "statically initialized". An
activation fixup (triggered by call_rcu()) takes care of performing the debug
object initialization without issuing any warning. Since we cannot increase the
size of struct rcu_head, I don't see much room to put an identifier for
statically initialized rcu_head structures. So for now, we have to live without
"activation without explicit init" detection. But the main purpose of this debug
option is to detect double-activations (double call_rcu() use of a rcu_head
before the callback is executed), which is correctly addressed here.

This also detects potential internal RCU callback corruption, which would cause
the callbacks to be executed twice.

Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
CC: David S. Miller <davem@davemloft.net>
CC: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
CC: akpm@linux-foundation.org
CC: mingo@elte.hu
CC: laijs@cn.fujitsu.com
CC: dipankar@in.ibm.com
CC: josh@joshtriplett.org
CC: dvhltc@us.ibm.com
CC: niv@us.ibm.com
CC: tglx@linutronix.de
CC: peterz@infradead.org
CC: rostedt@goodmis.org
CC: Valdis.Kletnieks@vt.edu
CC: dhowells@redhat.com
CC: eric.dumazet@gmail.com
CC: Alexey Dobriyan <adobriyan@gmail.com>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Reviewed-by: Lai Jiangshan <laijs@cn.fujitsu.com>

diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h
index b653b4aaa8a6332c8d783f2cd9afb7c8882028b4..2b7fc506e479a3a4ea19d9226c6733a72156a3f4 100644 (file)
--- a/include/linux/rcupdate.h
+++ b/include/linux/rcupdate.h
@@ -40,6 +40,7 @@
 #include <linux/seqlock.h>
 #include <linux/lockdep.h>
 #include <linux/completion.h>
+#include <linux/debugobjects.h>
 
 #ifdef CONFIG_RCU_TORTURE_TEST
 extern int rcutorture_runnable; /* for sysctl */
@@ -79,6 +80,16 @@ extern void rcu_init(void);
        (ptr)->next = NULL; (ptr)->func = NULL; \
 } while (0)
 
+/*
+ * init_rcu_head_on_stack()/destroy_rcu_head_on_stack() are needed for dynamic
+ * initialization and destruction of rcu_head on the stack. rcu_head structures
+ * allocated dynamically in the heap or defined statically don't need any
+ * initialization.
+ */
+#ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
+extern void init_rcu_head_on_stack(struct rcu_head *head);
+extern void destroy_rcu_head_on_stack(struct rcu_head *head);
+#else /* !CONFIG_DEBUG_OBJECTS_RCU_HEAD */
 static inline void init_rcu_head_on_stack(struct rcu_head *head)
 {
 }
@@ -86,6 +97,7 @@ static inline void init_rcu_head_on_stack(struct rcu_head *head)
 static inline void destroy_rcu_head_on_stack(struct rcu_head *head)
 {
 }
+#endif /* #else !CONFIG_DEBUG_OBJECTS_RCU_HEAD */
 
 #ifdef CONFIG_DEBUG_LOCK_ALLOC
 
@@ -517,4 +529,41 @@ extern void call_rcu(struct rcu_head *head,
 extern void call_rcu_bh(struct rcu_head *head,
                        void (*func)(struct rcu_head *head));
 
+/*
+ * debug_rcu_head_queue()/debug_rcu_head_unqueue() are used internally
+ * by call_rcu() and rcu callback execution, and are therefore not part of the
+ * RCU API. Leaving in rcupdate.h because they are used by all RCU flavors.
+ */
+
+#ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
+# define STATE_RCU_HEAD_READY  0
+# define STATE_RCU_HEAD_QUEUED 1
+
+extern struct debug_obj_descr rcuhead_debug_descr;
+
+static inline void debug_rcu_head_queue(struct rcu_head *head)
+{
+       debug_object_activate(head, &rcuhead_debug_descr);
+       debug_object_active_state(head, &rcuhead_debug_descr,
+                                 STATE_RCU_HEAD_READY,
+                                 STATE_RCU_HEAD_QUEUED);
+}
+
+static inline void debug_rcu_head_unqueue(struct rcu_head *head)
+{
+       debug_object_active_state(head, &rcuhead_debug_descr,
+                                 STATE_RCU_HEAD_QUEUED,
+                                 STATE_RCU_HEAD_READY);
+       debug_object_deactivate(head, &rcuhead_debug_descr);
+}
+#else  /* !CONFIG_DEBUG_OBJECTS_RCU_HEAD */
+static inline void debug_rcu_head_queue(struct rcu_head *head)
+{
+}
+
+static inline void debug_rcu_head_unqueue(struct rcu_head *head)
+{
+}
+#endif /* #else !CONFIG_DEBUG_OBJECTS_RCU_HEAD */
+
 #endif /* __LINUX_RCUPDATE_H */

diff --git a/kernel/rcupdate.c b/kernel/rcupdate.c
index 72a8dc9567f5825ee24bb52174b67953ca372b63..4d169835fb362dcd6eb52a916a0d8e85e48c8fb8 100644 (file)
--- a/kernel/rcupdate.c
+++ b/kernel/rcupdate.c
@@ -114,3 +114,163 @@ int rcu_my_thread_group_empty(void)
 }
 EXPORT_SYMBOL_GPL(rcu_my_thread_group_empty);
 #endif /* #ifdef CONFIG_PROVE_RCU */
+
+#ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
+static inline void debug_init_rcu_head(struct rcu_head *head)
+{
+       debug_object_init(head, &rcuhead_debug_descr);
+}
+
+static inline void debug_rcu_head_free(struct rcu_head *head)
+{
+       debug_object_free(head, &rcuhead_debug_descr);
+}
+
+/*
+ * fixup_init is called when:
+ * - an active object is initialized
+ */
+static int rcuhead_fixup_init(void *addr, enum debug_obj_state state)
+{
+       struct rcu_head *head = addr;
+
+       switch (state) {
+       case ODEBUG_STATE_ACTIVE:
+               /*
+                * Ensure that queued callbacks are all executed.
+                * If we detect that we are nested in a RCU read-side critical
+                * section, we should simply fail, otherwise we would deadlock.
+                */
+               if (rcu_preempt_depth() != 0 || preempt_count() != 0 ||
+                   irqs_disabled()) {
+                       WARN_ON(1);
+                       return 0;
+               }
+               rcu_barrier();
+               rcu_barrier_sched();
+               rcu_barrier_bh();
+               debug_object_init(head, &rcuhead_debug_descr);
+               return 1;
+       default:
+               return 0;
+       }
+}
+
+/*
+ * fixup_activate is called when:
+ * - an active object is activated
+ * - an unknown object is activated (might be a statically initialized object)
+ * Activation is performed internally by call_rcu().
+ */
+static int rcuhead_fixup_activate(void *addr, enum debug_obj_state state)
+{
+       struct rcu_head *head = addr;
+
+       switch (state) {
+
+       case ODEBUG_STATE_NOTAVAILABLE:
+               /*
+                * This is not really a fixup. We just make sure that it is
+                * tracked in the object tracker.
+                */
+               debug_object_init(head, &rcuhead_debug_descr);
+               debug_object_activate(head, &rcuhead_debug_descr);
+               return 0;
+
+       case ODEBUG_STATE_ACTIVE:
+               /*
+                * Ensure that queued callbacks are all executed.
+                * If we detect that we are nested in a RCU read-side critical
+                * section, we should simply fail, otherwise we would deadlock.
+                */
+               if (rcu_preempt_depth() != 0 || preempt_count() != 0 ||
+                   irqs_disabled()) {
+                       WARN_ON(1);
+                       return 0;
+               }
+               rcu_barrier();
+               rcu_barrier_sched();
+               rcu_barrier_bh();
+               debug_object_activate(head, &rcuhead_debug_descr);
+               return 1;
+       default:
+               return 0;
+       }
+}
+
+/*
+ * fixup_free is called when:
+ * - an active object is freed
+ */
+static int rcuhead_fixup_free(void *addr, enum debug_obj_state state)
+{
+       struct rcu_head *head = addr;
+
+       switch (state) {
+       case ODEBUG_STATE_ACTIVE:
+               /*
+                * Ensure that queued callbacks are all executed.
+                * If we detect that we are nested in a RCU read-side critical
+                * section, we should simply fail, otherwise we would deadlock.
+                */
+#ifndef CONFIG_PREEMPT
+               WARN_ON(1);
+               return 0;
+#else
+               if (rcu_preempt_depth() != 0 || preempt_count() != 0 ||
+                   irqs_disabled()) {
+                       WARN_ON(1);
+                       return 0;
+               }
+               rcu_barrier();
+               rcu_barrier_sched();
+               rcu_barrier_bh();
+               debug_object_free(head, &rcuhead_debug_descr);
+               return 1;
+#endif
+       default:
+               return 0;
+       }
+}
+
+/**
+ * init_rcu_head_on_stack() - initialize on-stack rcu_head for debugobjects
+ * @head: pointer to rcu_head structure to be initialized
+ *
+ * This function informs debugobjects of a new rcu_head structure that
+ * has been allocated as an auto variable on the stack.  This function
+ * is not required for rcu_head structures that are statically defined or
+ * that are dynamically allocated on the heap.  This function has no
+ * effect for !CONFIG_DEBUG_OBJECTS_RCU_HEAD kernel builds.
+ */
+void init_rcu_head_on_stack(struct rcu_head *head)
+{
+       debug_object_init_on_stack(head, &rcuhead_debug_descr);
+}
+EXPORT_SYMBOL_GPL(init_rcu_head_on_stack);
+
+/**
+ * destroy_rcu_head_on_stack() - destroy on-stack rcu_head for debugobjects
+ * @head: pointer to rcu_head structure to be initialized
+ *
+ * This function informs debugobjects that an on-stack rcu_head structure
+ * is about to go out of scope.  As with init_rcu_head_on_stack(), this
+ * function is not required for rcu_head structures that are statically
+ * defined or that are dynamically allocated on the heap.  Also as with
+ * init_rcu_head_on_stack(), this function has no effect for
+ * !CONFIG_DEBUG_OBJECTS_RCU_HEAD kernel builds.
+ */
+void destroy_rcu_head_on_stack(struct rcu_head *head)
+{
+       debug_object_free(head, &rcuhead_debug_descr);
+}
+EXPORT_SYMBOL_GPL(destroy_rcu_head_on_stack);
+
+struct debug_obj_descr rcuhead_debug_descr = {
+       .name = "rcu_head",
+       .fixup_init = rcuhead_fixup_init,
+       .fixup_activate = rcuhead_fixup_activate,
+       .fixup_free = rcuhead_fixup_free,
+};
+EXPORT_SYMBOL_GPL(rcuhead_debug_descr);
+#endif /* #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */

diff --git a/kernel/rcutiny.c b/kernel/rcutiny.c
index 38729d3cd23692887afac015b442c052a150f38c..196ec02f8be0c1ad884b7d69599005a6fc3a4429 100644 (file)
--- a/kernel/rcutiny.c
+++ b/kernel/rcutiny.c
@@ -169,6 +169,7 @@ static void __rcu_process_callbacks(struct rcu_ctrlblk *rcp)
        while (list) {
                next = list->next;
                prefetch(next);
+               debug_rcu_head_unqueue(list);
                list->func(list);
                list = next;
        }
@@ -211,6 +212,7 @@ static void __call_rcu(struct rcu_head *head,
 {
        unsigned long flags;
 
+       debug_rcu_head_queue(head);
        head->func = func;
        head->next = NULL;
 

diff --git a/kernel/rcutree.c b/kernel/rcutree.c
index d4437345706f8a539eee6dba201662fd791978ef..d5bc43976c5ad202fa41be0456797d40bdde0c0d 100644 (file)
--- a/kernel/rcutree.c
+++ b/kernel/rcutree.c
@@ -1112,6 +1112,7 @@ static void rcu_do_batch(struct rcu_state *rsp, struct rcu_data *rdp)
        while (list) {
                next = list->next;
                prefetch(next);
+               debug_rcu_head_unqueue(list);
                list->func(list);
                list = next;
                if (++count >= rdp->blimit)
@@ -1388,6 +1389,7 @@ __call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu),
        unsigned long flags;
        struct rcu_data *rdp;
 
+       debug_rcu_head_queue(head);
        head->func = func;
        head->next = NULL;
 

diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug
index e722e9d62221d40560176b5f8d2cabdcf74a5e2d..142faa2ec6658b2943ee1a771355eb22d73429d0 100644 (file)
--- a/lib/Kconfig.debug
+++ b/lib/Kconfig.debug
@@ -307,6 +307,12 @@ config DEBUG_OBJECTS_WORK
          work queue routines to track the life time of work objects and
          validate the work operations.
 
+config DEBUG_OBJECTS_RCU_HEAD
+       bool "Debug RCU callbacks objects"
+       depends on DEBUG_OBJECTS && PREEMPT
+       help
+         Enable this to turn on debugging of RCU list heads (call_rcu() usage).
+
 config DEBUG_OBJECTS_ENABLE_DEFAULT
        int "debug_objects bootup default value (0-1)"
         range 0 1