Commit | Line | Data |
---|---|---|
d405a98c SP |
1 | /* |
2 | * PowerNV cpuidle code | |
3 | * | |
4 | * Copyright 2015 IBM Corp. | |
5 | * | |
6 | * This program is free software; you can redistribute it and/or | |
7 | * modify it under the terms of the GNU General Public License | |
8 | * as published by the Free Software Foundation; either version | |
9 | * 2 of the License, or (at your option) any later version. | |
10 | */ | |
11 | ||
12 | #include <linux/types.h> | |
13 | #include <linux/mm.h> | |
14 | #include <linux/slab.h> | |
15 | #include <linux/of.h> | |
5703d2f4 SP |
16 | #include <linux/device.h> |
17 | #include <linux/cpu.h> | |
d405a98c SP |
18 | |
19 | #include <asm/firmware.h> | |
4bece972 | 20 | #include <asm/machdep.h> |
d405a98c SP |
21 | #include <asm/opal.h> |
22 | #include <asm/cputhreads.h> | |
23 | #include <asm/cpuidle.h> | |
24 | #include <asm/code-patching.h> | |
25 | #include <asm/smp.h> | |
26 | ||
27 | #include "powernv.h" | |
28 | #include "subcore.h" | |
29 | ||
bcef83a0 SP |
30 | /* Power ISA 3.0 allows for stop states 0x0 - 0xF */ |
31 | #define MAX_STOP_STATE 0xF | |
32 | ||
d405a98c SP |
33 | static u32 supported_cpuidle_states; |
34 | ||
bcef83a0 | 35 | static int pnv_save_sprs_for_deep_states(void) |
d405a98c SP |
36 | { |
37 | int cpu; | |
38 | int rc; | |
39 | ||
40 | /* | |
446957ba | 41 | * hid0, hid1, hid4, hid5, hmeer and lpcr values are symmetric across |
d405a98c | 42 | * all cpus at boot. Get these reg values of current cpu and use the |
446957ba | 43 | * same across all cpus. |
d405a98c SP |
44 | */ |
45 | uint64_t lpcr_val = mfspr(SPRN_LPCR) & ~(u64)LPCR_PECE1; | |
46 | uint64_t hid0_val = mfspr(SPRN_HID0); | |
47 | uint64_t hid1_val = mfspr(SPRN_HID1); | |
48 | uint64_t hid4_val = mfspr(SPRN_HID4); | |
49 | uint64_t hid5_val = mfspr(SPRN_HID5); | |
50 | uint64_t hmeer_val = mfspr(SPRN_HMEER); | |
51 | ||
52 | for_each_possible_cpu(cpu) { | |
53 | uint64_t pir = get_hard_smp_processor_id(cpu); | |
54 | uint64_t hsprg0_val = (uint64_t)&paca[cpu]; | |
55 | ||
bcef83a0 SP |
56 | if (!cpu_has_feature(CPU_FTR_ARCH_300)) { |
57 | /* | |
58 | * HSPRG0 is used to store the cpu's pointer to paca. | |
59 | * Hence last 3 bits are guaranteed to be 0. Program | |
60 | * slw to restore HSPRG0 with 63rd bit set, so that | |
61 | * when a thread wakes up at 0x100 we can use this bit | |
62 | * to distinguish between fastsleep and deep winkle. | |
63 | * This is not necessary with stop/psscr since PLS | |
64 | * field of psscr indicates which state we are waking | |
65 | * up from. | |
66 | */ | |
67 | hsprg0_val |= 1; | |
68 | } | |
d405a98c SP |
69 | rc = opal_slw_set_reg(pir, SPRN_HSPRG0, hsprg0_val); |
70 | if (rc != 0) | |
71 | return rc; | |
72 | ||
73 | rc = opal_slw_set_reg(pir, SPRN_LPCR, lpcr_val); | |
74 | if (rc != 0) | |
75 | return rc; | |
76 | ||
77 | /* HIDs are per core registers */ | |
78 | if (cpu_thread_in_core(cpu) == 0) { | |
79 | ||
80 | rc = opal_slw_set_reg(pir, SPRN_HMEER, hmeer_val); | |
81 | if (rc != 0) | |
82 | return rc; | |
83 | ||
84 | rc = opal_slw_set_reg(pir, SPRN_HID0, hid0_val); | |
85 | if (rc != 0) | |
86 | return rc; | |
87 | ||
88 | rc = opal_slw_set_reg(pir, SPRN_HID1, hid1_val); | |
89 | if (rc != 0) | |
90 | return rc; | |
91 | ||
92 | rc = opal_slw_set_reg(pir, SPRN_HID4, hid4_val); | |
93 | if (rc != 0) | |
94 | return rc; | |
95 | ||
96 | rc = opal_slw_set_reg(pir, SPRN_HID5, hid5_val); | |
97 | if (rc != 0) | |
98 | return rc; | |
99 | } | |
100 | } | |
101 | ||
102 | return 0; | |
103 | } | |
104 | ||
105 | static void pnv_alloc_idle_core_states(void) | |
106 | { | |
107 | int i, j; | |
108 | int nr_cores = cpu_nr_cores(); | |
109 | u32 *core_idle_state; | |
110 | ||
111 | /* | |
112 | * core_idle_state - First 8 bits track the idle state of each thread | |
113 | * of the core. The 8th bit is the lock bit. Initially all thread bits | |
114 | * are set. They are cleared when the thread enters deep idle state | |
115 | * like sleep and winkle. Initially the lock bit is cleared. | |
116 | * The lock bit has 2 purposes | |
117 | * a. While the first thread is restoring core state, it prevents | |
118 | * other threads in the core from switching to process context. | |
119 | * b. While the last thread in the core is saving the core state, it | |
120 | * prevents a different thread from waking up. | |
121 | */ | |
122 | for (i = 0; i < nr_cores; i++) { | |
123 | int first_cpu = i * threads_per_core; | |
124 | int node = cpu_to_node(first_cpu); | |
125 | ||
126 | core_idle_state = kmalloc_node(sizeof(u32), GFP_KERNEL, node); | |
127 | *core_idle_state = PNV_CORE_IDLE_THREAD_BITS; | |
128 | ||
129 | for (j = 0; j < threads_per_core; j++) { | |
130 | int cpu = first_cpu + j; | |
131 | ||
132 | paca[cpu].core_idle_state_ptr = core_idle_state; | |
133 | paca[cpu].thread_idle_state = PNV_THREAD_RUNNING; | |
134 | paca[cpu].thread_mask = 1 << j; | |
135 | } | |
136 | } | |
137 | ||
138 | update_subcore_sibling_mask(); | |
139 | ||
bcef83a0 SP |
140 | if (supported_cpuidle_states & OPAL_PM_LOSE_FULL_CONTEXT) |
141 | pnv_save_sprs_for_deep_states(); | |
d405a98c SP |
142 | } |
143 | ||
144 | u32 pnv_get_supported_cpuidle_states(void) | |
145 | { | |
146 | return supported_cpuidle_states; | |
147 | } | |
148 | EXPORT_SYMBOL_GPL(pnv_get_supported_cpuidle_states); | |
149 | ||
5703d2f4 SP |
150 | |
151 | static void pnv_fastsleep_workaround_apply(void *info) | |
152 | ||
153 | { | |
154 | int rc; | |
155 | int *err = info; | |
156 | ||
157 | rc = opal_config_cpu_idle_state(OPAL_CONFIG_IDLE_FASTSLEEP, | |
158 | OPAL_CONFIG_IDLE_APPLY); | |
159 | if (rc) | |
160 | *err = 1; | |
161 | } | |
162 | ||
163 | /* | |
164 | * Used to store fastsleep workaround state | |
165 | * 0 - Workaround applied/undone at fastsleep entry/exit path (Default) | |
166 | * 1 - Workaround applied once, never undone. | |
167 | */ | |
168 | static u8 fastsleep_workaround_applyonce; | |
169 | ||
170 | static ssize_t show_fastsleep_workaround_applyonce(struct device *dev, | |
171 | struct device_attribute *attr, char *buf) | |
172 | { | |
173 | return sprintf(buf, "%u\n", fastsleep_workaround_applyonce); | |
174 | } | |
175 | ||
176 | static ssize_t store_fastsleep_workaround_applyonce(struct device *dev, | |
177 | struct device_attribute *attr, const char *buf, | |
178 | size_t count) | |
179 | { | |
180 | cpumask_t primary_thread_mask; | |
181 | int err; | |
182 | u8 val; | |
183 | ||
184 | if (kstrtou8(buf, 0, &val) || val != 1) | |
185 | return -EINVAL; | |
186 | ||
187 | if (fastsleep_workaround_applyonce == 1) | |
188 | return count; | |
189 | ||
190 | /* | |
191 | * fastsleep_workaround_applyonce = 1 implies | |
192 | * fastsleep workaround needs to be left in 'applied' state on all | |
193 | * the cores. Do this by- | |
194 | * 1. Patching out the call to 'undo' workaround in fastsleep exit path | |
446957ba | 195 | * 2. Sending ipi to all the cores which have at least one online thread |
5703d2f4 SP |
196 | * 3. Patching out the call to 'apply' workaround in fastsleep entry |
197 | * path | |
198 | * There is no need to send ipi to cores which have all threads | |
199 | * offlined, as last thread of the core entering fastsleep or deeper | |
200 | * state would have applied workaround. | |
201 | */ | |
202 | err = patch_instruction( | |
203 | (unsigned int *)pnv_fastsleep_workaround_at_exit, | |
204 | PPC_INST_NOP); | |
205 | if (err) { | |
206 | pr_err("fastsleep_workaround_applyonce change failed while patching pnv_fastsleep_workaround_at_exit"); | |
207 | goto fail; | |
208 | } | |
209 | ||
210 | get_online_cpus(); | |
211 | primary_thread_mask = cpu_online_cores_map(); | |
212 | on_each_cpu_mask(&primary_thread_mask, | |
213 | pnv_fastsleep_workaround_apply, | |
214 | &err, 1); | |
215 | put_online_cpus(); | |
216 | if (err) { | |
217 | pr_err("fastsleep_workaround_applyonce change failed while running pnv_fastsleep_workaround_apply"); | |
218 | goto fail; | |
219 | } | |
220 | ||
221 | err = patch_instruction( | |
222 | (unsigned int *)pnv_fastsleep_workaround_at_entry, | |
223 | PPC_INST_NOP); | |
224 | if (err) { | |
225 | pr_err("fastsleep_workaround_applyonce change failed while patching pnv_fastsleep_workaround_at_entry"); | |
226 | goto fail; | |
227 | } | |
228 | ||
229 | fastsleep_workaround_applyonce = 1; | |
230 | ||
231 | return count; | |
232 | fail: | |
233 | return -EIO; | |
234 | } | |
235 | ||
236 | static DEVICE_ATTR(fastsleep_workaround_applyonce, 0600, | |
237 | show_fastsleep_workaround_applyonce, | |
238 | store_fastsleep_workaround_applyonce); | |
239 | ||
bcef83a0 SP |
240 | |
241 | /* | |
242 | * Used for ppc_md.power_save which needs a function with no parameters | |
243 | */ | |
244 | static void power9_idle(void) | |
d405a98c | 245 | { |
bcef83a0 SP |
246 | /* Requesting stop state 0 */ |
247 | power9_idle_stop(0); | |
248 | } | |
249 | /* | |
250 | * First deep stop state. Used to figure out when to save/restore | |
251 | * hypervisor context. | |
252 | */ | |
253 | u64 pnv_first_deep_stop_state = MAX_STOP_STATE; | |
d405a98c | 254 | |
c0691f9d SP |
255 | /* |
256 | * Deepest stop idle state. Used when a cpu is offlined | |
257 | */ | |
258 | u64 pnv_deepest_stop_state; | |
259 | ||
bcef83a0 SP |
260 | /* |
261 | * Power ISA 3.0 idle initialization. | |
262 | * | |
263 | * POWER ISA 3.0 defines a new SPR Processor stop Status and Control | |
264 | * Register (PSSCR) to control idle behavior. | |
265 | * | |
266 | * PSSCR layout: | |
267 | * ---------------------------------------------------------- | |
268 | * | PLS | /// | SD | ESL | EC | PSLL | /// | TR | MTL | RL | | |
269 | * ---------------------------------------------------------- | |
270 | * 0 4 41 42 43 44 48 54 56 60 | |
271 | * | |
272 | * PSSCR key fields: | |
273 | * Bits 0:3 - Power-Saving Level Status (PLS). This field indicates the | |
274 | * lowest power-saving state the thread entered since stop instruction was | |
275 | * last executed. | |
276 | * | |
277 | * Bit 41 - Status Disable(SD) | |
278 | * 0 - Shows PLS entries | |
279 | * 1 - PLS entries are all 0 | |
280 | * | |
281 | * Bit 42 - Enable State Loss | |
282 | * 0 - No state is lost irrespective of other fields | |
283 | * 1 - Allows state loss | |
284 | * | |
285 | * Bit 43 - Exit Criterion | |
286 | * 0 - Exit from power-save mode on any interrupt | |
287 | * 1 - Exit from power-save mode controlled by LPCR's PECE bits | |
288 | * | |
289 | * Bits 44:47 - Power-Saving Level Limit | |
290 | * This limits the power-saving level that can be entered into. | |
291 | * | |
292 | * Bits 60:63 - Requested Level | |
293 | * Used to specify which power-saving level must be entered on executing | |
294 | * stop instruction | |
295 | * | |
296 | * @np: /ibm,opal/power-mgt device node | |
297 | * @flags: cpu-idle-state-flags array | |
298 | * @dt_idle_states: Number of idle state entries | |
299 | * Returns 0 on success | |
300 | */ | |
301 | static int __init pnv_arch300_idle_init(struct device_node *np, u32 *flags, | |
302 | int dt_idle_states) | |
303 | { | |
304 | u64 *psscr_val = NULL; | |
305 | int rc = 0, i; | |
d405a98c | 306 | |
bcef83a0 SP |
307 | psscr_val = kcalloc(dt_idle_states, sizeof(*psscr_val), |
308 | GFP_KERNEL); | |
309 | if (!psscr_val) { | |
310 | rc = -1; | |
d405a98c | 311 | goto out; |
bcef83a0 SP |
312 | } |
313 | if (of_property_read_u64_array(np, | |
314 | "ibm,cpu-idle-state-psscr", | |
315 | psscr_val, dt_idle_states)) { | |
316 | pr_warn("cpuidle-powernv: missing ibm,cpu-idle-states-psscr in DT\n"); | |
317 | rc = -1; | |
d405a98c | 318 | goto out; |
bcef83a0 | 319 | } |
d405a98c | 320 | |
bcef83a0 | 321 | /* |
c0691f9d SP |
322 | * Set pnv_first_deep_stop_state and pnv_deepest_stop_state. |
323 | * pnv_first_deep_stop_state should be set to the first stop | |
324 | * level to cause hypervisor state loss. | |
325 | * pnv_deepest_stop_state should be set to the deepest stop | |
326 | * stop state. | |
bcef83a0 SP |
327 | */ |
328 | pnv_first_deep_stop_state = MAX_STOP_STATE; | |
329 | for (i = 0; i < dt_idle_states; i++) { | |
330 | u64 psscr_rl = psscr_val[i] & PSSCR_RL_MASK; | |
331 | ||
332 | if ((flags[i] & OPAL_PM_LOSE_FULL_CONTEXT) && | |
333 | (pnv_first_deep_stop_state > psscr_rl)) | |
334 | pnv_first_deep_stop_state = psscr_rl; | |
c0691f9d SP |
335 | |
336 | if (pnv_deepest_stop_state < psscr_rl) | |
337 | pnv_deepest_stop_state = psscr_rl; | |
bcef83a0 SP |
338 | } |
339 | ||
340 | out: | |
341 | kfree(psscr_val); | |
342 | return rc; | |
343 | } | |
344 | ||
345 | /* | |
346 | * Probe device tree for supported idle states | |
347 | */ | |
348 | static void __init pnv_probe_idle_states(void) | |
349 | { | |
350 | struct device_node *np; | |
351 | int dt_idle_states; | |
352 | u32 *flags = NULL; | |
353 | int i; | |
354 | ||
355 | np = of_find_node_by_path("/ibm,opal/power-mgt"); | |
356 | if (!np) { | |
d405a98c SP |
357 | pr_warn("opal: PowerMgmt Node not found\n"); |
358 | goto out; | |
359 | } | |
bcef83a0 | 360 | dt_idle_states = of_property_count_u32_elems(np, |
d405a98c SP |
361 | "ibm,cpu-idle-state-flags"); |
362 | if (dt_idle_states < 0) { | |
363 | pr_warn("cpuidle-powernv: no idle states found in the DT\n"); | |
364 | goto out; | |
365 | } | |
366 | ||
bcef83a0 SP |
367 | flags = kcalloc(dt_idle_states, sizeof(*flags), GFP_KERNEL); |
368 | ||
369 | if (of_property_read_u32_array(np, | |
d405a98c SP |
370 | "ibm,cpu-idle-state-flags", flags, dt_idle_states)) { |
371 | pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-flags in DT\n"); | |
bcef83a0 SP |
372 | goto out; |
373 | } | |
374 | ||
375 | if (cpu_has_feature(CPU_FTR_ARCH_300)) { | |
376 | if (pnv_arch300_idle_init(np, flags, dt_idle_states)) | |
377 | goto out; | |
d405a98c SP |
378 | } |
379 | ||
380 | for (i = 0; i < dt_idle_states; i++) | |
381 | supported_cpuidle_states |= flags[i]; | |
382 | ||
bcef83a0 SP |
383 | out: |
384 | kfree(flags); | |
385 | } | |
386 | static int __init pnv_init_idle_states(void) | |
387 | { | |
388 | ||
389 | supported_cpuidle_states = 0; | |
390 | ||
391 | if (cpuidle_disable != IDLE_NO_OVERRIDE) | |
392 | goto out; | |
393 | ||
394 | pnv_probe_idle_states(); | |
395 | ||
d405a98c SP |
396 | if (!(supported_cpuidle_states & OPAL_PM_SLEEP_ENABLED_ER1)) { |
397 | patch_instruction( | |
398 | (unsigned int *)pnv_fastsleep_workaround_at_entry, | |
399 | PPC_INST_NOP); | |
400 | patch_instruction( | |
401 | (unsigned int *)pnv_fastsleep_workaround_at_exit, | |
402 | PPC_INST_NOP); | |
5703d2f4 SP |
403 | } else { |
404 | /* | |
405 | * OPAL_PM_SLEEP_ENABLED_ER1 is set. It indicates that | |
406 | * workaround is needed to use fastsleep. Provide sysfs | |
407 | * control to choose how this workaround has to be applied. | |
408 | */ | |
409 | device_create_file(cpu_subsys.dev_root, | |
410 | &dev_attr_fastsleep_workaround_applyonce); | |
d405a98c | 411 | } |
5703d2f4 | 412 | |
d405a98c | 413 | pnv_alloc_idle_core_states(); |
5593e303 SP |
414 | |
415 | if (supported_cpuidle_states & OPAL_PM_NAP_ENABLED) | |
416 | ppc_md.power_save = power7_idle; | |
bcef83a0 SP |
417 | else if (supported_cpuidle_states & OPAL_PM_STOP_INST_FAST) |
418 | ppc_md.power_save = power9_idle; | |
419 | ||
d405a98c SP |
420 | out: |
421 | return 0; | |
422 | } | |
4bece972 | 423 | machine_subsys_initcall(powernv, pnv_init_idle_states); |