rcu: Permit dyntick-idle with callbacks pending

[deliverable/linux.git] / kernel / rcutree.c

diff --git a/kernel/rcutree.c b/kernel/rcutree.c
index 7fb8b0e608116e982a6d6506585b246567e3c997..bf085d7f6a3f37b6fafd39225e4826661c2ad0c5 100644 (file)
--- a/kernel/rcutree.c
+++ b/kernel/rcutree.c
@@ -448,6 +448,7 @@ static void rcu_idle_exit_common(struct rcu_dynticks *rdtp, long long oldval)
        /* CPUs seeing atomic_inc() must see later RCU read-side crit sects */
        smp_mb__after_atomic_inc();  /* See above. */
        WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks) & 0x1));
+       rcu_cleanup_after_idle(smp_processor_id());
        trace_rcu_dyntick("End", oldval, rdtp->dynticks_nesting);
        if (!is_idle_task(current)) {
                struct task_struct *idle = idle_task(smp_processor_id());
@@ -607,7 +608,7 @@ int rcu_is_cpu_rrupt_from_idle(void)
 static int dyntick_save_progress_counter(struct rcu_data *rdp)
 {
        rdp->dynticks_snap = atomic_add_return(0, &rdp->dynticks->dynticks);
-       return 0;
+       return (rdp->dynticks_snap & 0x1) == 0;
 }
 
 /*
@@ -1086,7 +1087,6 @@ static void rcu_report_qs_rsp(struct rcu_state *rsp, unsigned long flags)
         * callbacks are waiting on the grace period that just now
         * completed.
         */
-       rcu_schedule_wake_gp_end();
        if (*rdp->nxttail[RCU_WAIT_TAIL] == NULL) {
                raw_spin_unlock(&rnp->lock);     /* irqs remain disabled. */
 
@@ -1672,7 +1672,6 @@ static void rcu_process_callbacks(struct softirq_action *unused)
                                &__get_cpu_var(rcu_sched_data));
        __rcu_process_callbacks(&rcu_bh_state, &__get_cpu_var(rcu_bh_data));
        rcu_preempt_process_callbacks();
-       rcu_wake_cpus_for_gp_end();
        trace_rcu_utilization("End RCU core");
 }
 
@@ -2056,8 +2055,10 @@ rcu_init_percpu_data(int cpu, struct rcu_state *rsp, int preemptible)
        rdp->qlen_last_fqs_check = 0;
        rdp->n_force_qs_snap = rsp->n_force_qs;
        rdp->blimit = blimit;
-       WARN_ON_ONCE(rdp->dynticks->dynticks_nesting != DYNTICK_TASK_NESTING);
-       WARN_ON_ONCE((atomic_read(&rdp->dynticks->dynticks) & 0x1) != 1);
+       rdp->dynticks->dynticks_nesting = DYNTICK_TASK_NESTING;
+       atomic_set(&rdp->dynticks->dynticks,
+                  (atomic_read(&rdp->dynticks->dynticks) & ~0x1) + 1);
+       rcu_prepare_for_idle_init(cpu);
        raw_spin_unlock(&rnp->lock);            /* irqs remain disabled. */
 
        /*
@@ -2139,6 +2140,7 @@ static int __cpuinit rcu_cpu_notify(struct notifier_block *self,
                rcu_send_cbs_to_online(&rcu_bh_state);
                rcu_send_cbs_to_online(&rcu_sched_state);
                rcu_preempt_send_cbs_to_online();
+               rcu_cleanup_after_idle(cpu);
                break;
        case CPU_DEAD:
        case CPU_DEAD_FROZEN: