Commit | Line | Data |
---|---|---|
e5582ca2 RR |
1 | /* Copyright 2005 Rusty Russell rusty@rustcorp.com.au IBM Corporation. |
2 | * GPL v2 and any later version. | |
3 | */ | |
1da177e4 LT |
4 | #include <linux/cpu.h> |
5 | #include <linux/err.h> | |
ee527cd3 PB |
6 | #include <linux/kthread.h> |
7 | #include <linux/module.h> | |
8 | #include <linux/sched.h> | |
9 | #include <linux/stop_machine.h> | |
1da177e4 | 10 | #include <linux/syscalls.h> |
a12bb444 BH |
11 | #include <linux/interrupt.h> |
12 | ||
1da177e4 LT |
13 | #include <asm/atomic.h> |
14 | #include <asm/semaphore.h> | |
15 | #include <asm/uaccess.h> | |
16 | ||
17 | /* Since we effect priority and affinity (both of which are visible | |
18 | * to, and settable by outside processes) we do indirection via a | |
19 | * kthread. */ | |
20 | ||
21 | /* Thread to stop each CPU in user context. */ | |
22 | enum stopmachine_state { | |
23 | STOPMACHINE_WAIT, | |
24 | STOPMACHINE_PREPARE, | |
25 | STOPMACHINE_DISABLE_IRQ, | |
26 | STOPMACHINE_EXIT, | |
27 | }; | |
28 | ||
29 | static enum stopmachine_state stopmachine_state; | |
30 | static unsigned int stopmachine_num_threads; | |
31 | static atomic_t stopmachine_thread_ack; | |
32 | static DECLARE_MUTEX(stopmachine_mutex); | |
33 | ||
d8cb7c1d | 34 | static int stopmachine(void *cpu) |
1da177e4 LT |
35 | { |
36 | int irqs_disabled = 0; | |
37 | int prepared = 0; | |
38 | ||
d8cb7c1d AM |
39 | set_cpus_allowed(current, cpumask_of_cpu((int)(long)cpu)); |
40 | ||
1da177e4 | 41 | /* Ack: we are alive */ |
d59dd462 | 42 | smp_mb(); /* Theoretically the ack = 0 might not be on this CPU yet. */ |
1da177e4 LT |
43 | atomic_inc(&stopmachine_thread_ack); |
44 | ||
45 | /* Simple state machine */ | |
46 | while (stopmachine_state != STOPMACHINE_EXIT) { | |
47 | if (stopmachine_state == STOPMACHINE_DISABLE_IRQ | |
48 | && !irqs_disabled) { | |
49 | local_irq_disable(); | |
a12bb444 | 50 | hard_irq_disable(); |
1da177e4 LT |
51 | irqs_disabled = 1; |
52 | /* Ack: irqs disabled. */ | |
d59dd462 | 53 | smp_mb(); /* Must read state first. */ |
1da177e4 LT |
54 | atomic_inc(&stopmachine_thread_ack); |
55 | } else if (stopmachine_state == STOPMACHINE_PREPARE | |
56 | && !prepared) { | |
57 | /* Everyone is in place, hold CPU. */ | |
58 | preempt_disable(); | |
59 | prepared = 1; | |
d59dd462 | 60 | smp_mb(); /* Must read state first. */ |
1da177e4 LT |
61 | atomic_inc(&stopmachine_thread_ack); |
62 | } | |
63 | /* Yield in first stage: migration threads need to | |
64 | * help our sisters onto their CPUs. */ | |
65 | if (!prepared && !irqs_disabled) | |
66 | yield(); | |
67 | else | |
68 | cpu_relax(); | |
69 | } | |
70 | ||
71 | /* Ack: we are exiting. */ | |
d59dd462 | 72 | smp_mb(); /* Must read state first. */ |
1da177e4 LT |
73 | atomic_inc(&stopmachine_thread_ack); |
74 | ||
75 | if (irqs_disabled) | |
76 | local_irq_enable(); | |
77 | if (prepared) | |
78 | preempt_enable(); | |
79 | ||
80 | return 0; | |
81 | } | |
82 | ||
83 | /* Change the thread state */ | |
84 | static void stopmachine_set_state(enum stopmachine_state state) | |
85 | { | |
86 | atomic_set(&stopmachine_thread_ack, 0); | |
d59dd462 | 87 | smp_wmb(); |
1da177e4 LT |
88 | stopmachine_state = state; |
89 | while (atomic_read(&stopmachine_thread_ack) != stopmachine_num_threads) | |
90 | cpu_relax(); | |
91 | } | |
92 | ||
93 | static int stop_machine(void) | |
94 | { | |
d8cb7c1d | 95 | int i, ret = 0; |
1da177e4 LT |
96 | |
97 | atomic_set(&stopmachine_thread_ack, 0); | |
98 | stopmachine_num_threads = 0; | |
99 | stopmachine_state = STOPMACHINE_WAIT; | |
100 | ||
101 | for_each_online_cpu(i) { | |
39c715b7 | 102 | if (i == raw_smp_processor_id()) |
1da177e4 | 103 | continue; |
d8cb7c1d AM |
104 | ret = kernel_thread(stopmachine, (void *)(long)i,CLONE_KERNEL); |
105 | if (ret < 0) | |
1da177e4 LT |
106 | break; |
107 | stopmachine_num_threads++; | |
108 | } | |
109 | ||
110 | /* Wait for them all to come to life. */ | |
111 | while (atomic_read(&stopmachine_thread_ack) != stopmachine_num_threads) | |
112 | yield(); | |
113 | ||
114 | /* If some failed, kill them all. */ | |
115 | if (ret < 0) { | |
116 | stopmachine_set_state(STOPMACHINE_EXIT); | |
1da177e4 LT |
117 | return ret; |
118 | } | |
119 | ||
1da177e4 | 120 | /* Now they are all started, make them hold the CPUs, ready. */ |
4557398f | 121 | preempt_disable(); |
1da177e4 LT |
122 | stopmachine_set_state(STOPMACHINE_PREPARE); |
123 | ||
124 | /* Make them disable irqs. */ | |
4557398f | 125 | local_irq_disable(); |
a12bb444 | 126 | hard_irq_disable(); |
1da177e4 LT |
127 | stopmachine_set_state(STOPMACHINE_DISABLE_IRQ); |
128 | ||
129 | return 0; | |
130 | } | |
131 | ||
132 | static void restart_machine(void) | |
133 | { | |
134 | stopmachine_set_state(STOPMACHINE_EXIT); | |
135 | local_irq_enable(); | |
4557398f | 136 | preempt_enable_no_resched(); |
1da177e4 LT |
137 | } |
138 | ||
139 | struct stop_machine_data | |
140 | { | |
141 | int (*fn)(void *); | |
142 | void *data; | |
143 | struct completion done; | |
144 | }; | |
145 | ||
146 | static int do_stop(void *_smdata) | |
147 | { | |
148 | struct stop_machine_data *smdata = _smdata; | |
149 | int ret; | |
150 | ||
151 | ret = stop_machine(); | |
152 | if (ret == 0) { | |
153 | ret = smdata->fn(smdata->data); | |
154 | restart_machine(); | |
155 | } | |
156 | ||
157 | /* We're done: you can kthread_stop us now */ | |
158 | complete(&smdata->done); | |
159 | ||
160 | /* Wait for kthread_stop */ | |
161 | set_current_state(TASK_INTERRUPTIBLE); | |
162 | while (!kthread_should_stop()) { | |
163 | schedule(); | |
164 | set_current_state(TASK_INTERRUPTIBLE); | |
165 | } | |
166 | __set_current_state(TASK_RUNNING); | |
167 | return ret; | |
168 | } | |
169 | ||
170 | struct task_struct *__stop_machine_run(int (*fn)(void *), void *data, | |
171 | unsigned int cpu) | |
172 | { | |
173 | struct stop_machine_data smdata; | |
174 | struct task_struct *p; | |
175 | ||
176 | smdata.fn = fn; | |
177 | smdata.data = data; | |
178 | init_completion(&smdata.done); | |
179 | ||
180 | down(&stopmachine_mutex); | |
181 | ||
182 | /* If they don't care which CPU fn runs on, bind to any online one. */ | |
183 | if (cpu == NR_CPUS) | |
39c715b7 | 184 | cpu = raw_smp_processor_id(); |
1da177e4 LT |
185 | |
186 | p = kthread_create(do_stop, &smdata, "kstopmachine"); | |
187 | if (!IS_ERR(p)) { | |
85653af7 ST |
188 | struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 }; |
189 | ||
190 | /* One high-prio thread per cpu. We'll do this one. */ | |
191 | sched_setscheduler(p, SCHED_FIFO, ¶m); | |
1da177e4 LT |
192 | kthread_bind(p, cpu); |
193 | wake_up_process(p); | |
194 | wait_for_completion(&smdata.done); | |
195 | } | |
196 | up(&stopmachine_mutex); | |
197 | return p; | |
198 | } | |
199 | ||
200 | int stop_machine_run(int (*fn)(void *), void *data, unsigned int cpu) | |
201 | { | |
202 | struct task_struct *p; | |
203 | int ret; | |
204 | ||
205 | /* No CPUs can come up or down during this. */ | |
206 | lock_cpu_hotplug(); | |
207 | p = __stop_machine_run(fn, data, cpu); | |
208 | if (!IS_ERR(p)) | |
209 | ret = kthread_stop(p); | |
210 | else | |
211 | ret = PTR_ERR(p); | |
212 | unlock_cpu_hotplug(); | |
213 | ||
214 | return ret; | |
215 | } | |
ee527cd3 | 216 | EXPORT_SYMBOL_GPL(stop_machine_run); |