kernel/stop_machine.c

   1 /* Copyright 2008, 2005 Rusty Russell rusty@rustcorp.com.au IBM Corporation.
   2  * GPL v2 and any later version.
   3  */
   4 #include <linux/cpu.h>
   5 #include <linux/err.h>
   6 #include <linux/kthread.h>
   7 #include <linux/module.h>
   8 #include <linux/sched.h>
   9 #include <linux/stop_machine.h>
  10 #include <linux/syscalls.h>
  11 #include <linux/interrupt.h>
  12
  13 #include <asm/atomic.h>
  14 #include <asm/uaccess.h>
  15
  16 /* This controls the threads on each CPU. */
  17 enum stopmachine_state {
  18         /* Dummy starting state for thread. */
  19         STOPMACHINE_NONE,
  20         /* Awaiting everyone to be scheduled. */
  21         STOPMACHINE_PREPARE,
  22         /* Disable interrupts. */
  23         STOPMACHINE_DISABLE_IRQ,
  24         /* Run the function */
  25         STOPMACHINE_RUN,
  26         /* Exit */
  27         STOPMACHINE_EXIT,
  28 };
  29 static enum stopmachine_state state;
  30
  31 struct stop_machine_data {
  32         int (*fn)(void *);
  33         void *data;
  34         int fnret;
  35 };
  36
  37 /* Like num_online_cpus(), but hotplug cpu uses us, so we need this. */
  38 static unsigned int num_threads;
  39 static atomic_t thread_ack;
  40 static DEFINE_MUTEX(lock);
  41
  42 static struct workqueue_struct *stop_machine_wq;
  43 static struct stop_machine_data active, idle;
  44 static const cpumask_t *active_cpus;
  45 static void *stop_machine_work;
  46
  47 static void set_state(enum stopmachine_state newstate)
  48 {
  49         /* Reset ack counter. */
  50         atomic_set(&thread_ack, num_threads);
  51         smp_wmb();
  52         state = newstate;
  53 }
  54
  55 /* Last one to ack a state moves to the next state. */
  56 static void ack_state(void)
  57 {
  58         if (atomic_dec_and_test(&thread_ack))
  59                 set_state(state + 1);
  60 }
  61
  62 /* This is the actual function which stops the CPU. It runs
  63  * in the context of a dedicated stopmachine workqueue. */
  64 static void stop_cpu(struct work_struct *unused)
  65 {
  66         enum stopmachine_state curstate = STOPMACHINE_NONE;
  67         struct stop_machine_data *smdata = &idle;
  68         int cpu = smp_processor_id();
  69         int err;
  70
  71         if (!active_cpus) {
  72                 if (cpu == cpumask_first(cpu_online_mask))
  73                         smdata = &active;
  74         } else {
  75                 if (cpumask_test_cpu(cpu, active_cpus))
  76                         smdata = &active;
  77         }
  78         /* Simple state machine */
  79         do {
  80                 /* Chill out and ensure we re-read stopmachine_state. */
  81                 cpu_relax();
  82                 if (state != curstate) {
  83                         curstate = state;
  84                         switch (curstate) {
  85                         case STOPMACHINE_DISABLE_IRQ:
  86                                 local_irq_disable();
  87                                 hard_irq_disable();
  88                                 break;
  89                         case STOPMACHINE_RUN:
  90                                 /* On multiple CPUs only a single error code
  91                                  * is needed to tell that something failed. */
  92                                 err = smdata->fn(smdata->data);
  93                                 if (err)
  94                                         smdata->fnret = err;
  95                                 break;
  96                         default:
  97                                 break;
  98                         }
  99                         ack_state();
 100                 }
 101         } while (curstate != STOPMACHINE_EXIT);
 102
 103         local_irq_enable();
 104 }
 105
 106 /* Callback for CPUs which aren't supposed to do anything. */
 107 static int chill(void *unused)
 108 {
 109         return 0;
 110 }
 111
 112 int __stop_machine(int (*fn)(void *), void *data, const struct cpumask *cpus)
 113 {
 114         struct work_struct *sm_work;
 115         int i, ret;
 116
 117         /* Set up initial state. */
 118         mutex_lock(&lock);
 119         num_threads = num_online_cpus();
 120         active_cpus = cpus;
 121         active.fn = fn;
 122         active.data = data;
 123         active.fnret = 0;
 124         idle.fn = chill;
 125         idle.data = NULL;
 126
 127         set_state(STOPMACHINE_PREPARE);
 128
 129         /* Schedule the stop_cpu work on all cpus: hold this CPU so one
 130          * doesn't hit this CPU until we're ready. */
 131         get_cpu();
 132         for_each_online_cpu(i) {
 133                 sm_work = percpu_ptr(stop_machine_work, i);
 134                 INIT_WORK(sm_work, stop_cpu);
 135                 queue_work_on(i, stop_machine_wq, sm_work);
 136         }
 137         /* This will release the thread on our CPU. */
 138         put_cpu();
 139         flush_workqueue(stop_machine_wq);
 140         ret = active.fnret;
 141         mutex_unlock(&lock);
 142         return ret;
 143 }
 144
 145 int stop_machine(int (*fn)(void *), void *data, const struct cpumask *cpus)
 146 {
 147         int ret;
 148
 149         /* No CPUs can come up or down during this. */
 150         get_online_cpus();
 151         ret = __stop_machine(fn, data, cpus);
 152         put_online_cpus();
 153
 154         return ret;
 155 }
 156 EXPORT_SYMBOL_GPL(stop_machine);
 157
 158 static int __init stop_machine_init(void)
 159 {
 160         stop_machine_wq = create_rt_workqueue("kstop");
 161         stop_machine_work = alloc_percpu(struct work_struct);
 162         return 0;
 163 }
 164 core_initcall(stop_machine_init);