Commit | Line | Data |
---|---|---|
d405a98c SP |
1 | /* |
2 | * PowerNV cpuidle code | |
3 | * | |
4 | * Copyright 2015 IBM Corp. | |
5 | * | |
6 | * This program is free software; you can redistribute it and/or | |
7 | * modify it under the terms of the GNU General Public License | |
8 | * as published by the Free Software Foundation; either version | |
9 | * 2 of the License, or (at your option) any later version. | |
10 | */ | |
11 | ||
12 | #include <linux/types.h> | |
13 | #include <linux/mm.h> | |
14 | #include <linux/slab.h> | |
15 | #include <linux/of.h> | |
5703d2f4 SP |
16 | #include <linux/device.h> |
17 | #include <linux/cpu.h> | |
d405a98c SP |
18 | |
19 | #include <asm/firmware.h> | |
4bece972 | 20 | #include <asm/machdep.h> |
d405a98c SP |
21 | #include <asm/opal.h> |
22 | #include <asm/cputhreads.h> | |
23 | #include <asm/cpuidle.h> | |
24 | #include <asm/code-patching.h> | |
25 | #include <asm/smp.h> | |
26 | ||
27 | #include "powernv.h" | |
28 | #include "subcore.h" | |
29 | ||
30 | static u32 supported_cpuidle_states; | |
31 | ||
34852ed5 | 32 | static int pnv_save_sprs_for_winkle(void) |
d405a98c SP |
33 | { |
34 | int cpu; | |
35 | int rc; | |
36 | ||
37 | /* | |
446957ba | 38 | * hid0, hid1, hid4, hid5, hmeer and lpcr values are symmetric across |
d405a98c | 39 | * all cpus at boot. Get these reg values of current cpu and use the |
446957ba | 40 | * same across all cpus. |
d405a98c SP |
41 | */ |
42 | uint64_t lpcr_val = mfspr(SPRN_LPCR) & ~(u64)LPCR_PECE1; | |
43 | uint64_t hid0_val = mfspr(SPRN_HID0); | |
44 | uint64_t hid1_val = mfspr(SPRN_HID1); | |
45 | uint64_t hid4_val = mfspr(SPRN_HID4); | |
46 | uint64_t hid5_val = mfspr(SPRN_HID5); | |
47 | uint64_t hmeer_val = mfspr(SPRN_HMEER); | |
48 | ||
49 | for_each_possible_cpu(cpu) { | |
50 | uint64_t pir = get_hard_smp_processor_id(cpu); | |
51 | uint64_t hsprg0_val = (uint64_t)&paca[cpu]; | |
52 | ||
53 | /* | |
54 | * HSPRG0 is used to store the cpu's pointer to paca. Hence last | |
55 | * 3 bits are guaranteed to be 0. Program slw to restore HSPRG0 | |
56 | * with 63rd bit set, so that when a thread wakes up at 0x100 we | |
57 | * can use this bit to distinguish between fastsleep and | |
58 | * deep winkle. | |
59 | */ | |
60 | hsprg0_val |= 1; | |
61 | ||
62 | rc = opal_slw_set_reg(pir, SPRN_HSPRG0, hsprg0_val); | |
63 | if (rc != 0) | |
64 | return rc; | |
65 | ||
66 | rc = opal_slw_set_reg(pir, SPRN_LPCR, lpcr_val); | |
67 | if (rc != 0) | |
68 | return rc; | |
69 | ||
70 | /* HIDs are per core registers */ | |
71 | if (cpu_thread_in_core(cpu) == 0) { | |
72 | ||
73 | rc = opal_slw_set_reg(pir, SPRN_HMEER, hmeer_val); | |
74 | if (rc != 0) | |
75 | return rc; | |
76 | ||
77 | rc = opal_slw_set_reg(pir, SPRN_HID0, hid0_val); | |
78 | if (rc != 0) | |
79 | return rc; | |
80 | ||
81 | rc = opal_slw_set_reg(pir, SPRN_HID1, hid1_val); | |
82 | if (rc != 0) | |
83 | return rc; | |
84 | ||
85 | rc = opal_slw_set_reg(pir, SPRN_HID4, hid4_val); | |
86 | if (rc != 0) | |
87 | return rc; | |
88 | ||
89 | rc = opal_slw_set_reg(pir, SPRN_HID5, hid5_val); | |
90 | if (rc != 0) | |
91 | return rc; | |
92 | } | |
93 | } | |
94 | ||
95 | return 0; | |
96 | } | |
97 | ||
98 | static void pnv_alloc_idle_core_states(void) | |
99 | { | |
100 | int i, j; | |
101 | int nr_cores = cpu_nr_cores(); | |
102 | u32 *core_idle_state; | |
103 | ||
104 | /* | |
105 | * core_idle_state - First 8 bits track the idle state of each thread | |
106 | * of the core. The 8th bit is the lock bit. Initially all thread bits | |
107 | * are set. They are cleared when the thread enters deep idle state | |
108 | * like sleep and winkle. Initially the lock bit is cleared. | |
109 | * The lock bit has 2 purposes | |
110 | * a. While the first thread is restoring core state, it prevents | |
111 | * other threads in the core from switching to process context. | |
112 | * b. While the last thread in the core is saving the core state, it | |
113 | * prevents a different thread from waking up. | |
114 | */ | |
115 | for (i = 0; i < nr_cores; i++) { | |
116 | int first_cpu = i * threads_per_core; | |
117 | int node = cpu_to_node(first_cpu); | |
118 | ||
119 | core_idle_state = kmalloc_node(sizeof(u32), GFP_KERNEL, node); | |
120 | *core_idle_state = PNV_CORE_IDLE_THREAD_BITS; | |
121 | ||
122 | for (j = 0; j < threads_per_core; j++) { | |
123 | int cpu = first_cpu + j; | |
124 | ||
125 | paca[cpu].core_idle_state_ptr = core_idle_state; | |
126 | paca[cpu].thread_idle_state = PNV_THREAD_RUNNING; | |
127 | paca[cpu].thread_mask = 1 << j; | |
128 | } | |
129 | } | |
130 | ||
131 | update_subcore_sibling_mask(); | |
132 | ||
133 | if (supported_cpuidle_states & OPAL_PM_WINKLE_ENABLED) | |
134 | pnv_save_sprs_for_winkle(); | |
135 | } | |
136 | ||
137 | u32 pnv_get_supported_cpuidle_states(void) | |
138 | { | |
139 | return supported_cpuidle_states; | |
140 | } | |
141 | EXPORT_SYMBOL_GPL(pnv_get_supported_cpuidle_states); | |
142 | ||
5703d2f4 SP |
143 | |
144 | static void pnv_fastsleep_workaround_apply(void *info) | |
145 | ||
146 | { | |
147 | int rc; | |
148 | int *err = info; | |
149 | ||
150 | rc = opal_config_cpu_idle_state(OPAL_CONFIG_IDLE_FASTSLEEP, | |
151 | OPAL_CONFIG_IDLE_APPLY); | |
152 | if (rc) | |
153 | *err = 1; | |
154 | } | |
155 | ||
156 | /* | |
157 | * Used to store fastsleep workaround state | |
158 | * 0 - Workaround applied/undone at fastsleep entry/exit path (Default) | |
159 | * 1 - Workaround applied once, never undone. | |
160 | */ | |
161 | static u8 fastsleep_workaround_applyonce; | |
162 | ||
163 | static ssize_t show_fastsleep_workaround_applyonce(struct device *dev, | |
164 | struct device_attribute *attr, char *buf) | |
165 | { | |
166 | return sprintf(buf, "%u\n", fastsleep_workaround_applyonce); | |
167 | } | |
168 | ||
169 | static ssize_t store_fastsleep_workaround_applyonce(struct device *dev, | |
170 | struct device_attribute *attr, const char *buf, | |
171 | size_t count) | |
172 | { | |
173 | cpumask_t primary_thread_mask; | |
174 | int err; | |
175 | u8 val; | |
176 | ||
177 | if (kstrtou8(buf, 0, &val) || val != 1) | |
178 | return -EINVAL; | |
179 | ||
180 | if (fastsleep_workaround_applyonce == 1) | |
181 | return count; | |
182 | ||
183 | /* | |
184 | * fastsleep_workaround_applyonce = 1 implies | |
185 | * fastsleep workaround needs to be left in 'applied' state on all | |
186 | * the cores. Do this by- | |
187 | * 1. Patching out the call to 'undo' workaround in fastsleep exit path | |
446957ba | 188 | * 2. Sending ipi to all the cores which have at least one online thread |
5703d2f4 SP |
189 | * 3. Patching out the call to 'apply' workaround in fastsleep entry |
190 | * path | |
191 | * There is no need to send ipi to cores which have all threads | |
192 | * offlined, as last thread of the core entering fastsleep or deeper | |
193 | * state would have applied workaround. | |
194 | */ | |
195 | err = patch_instruction( | |
196 | (unsigned int *)pnv_fastsleep_workaround_at_exit, | |
197 | PPC_INST_NOP); | |
198 | if (err) { | |
199 | pr_err("fastsleep_workaround_applyonce change failed while patching pnv_fastsleep_workaround_at_exit"); | |
200 | goto fail; | |
201 | } | |
202 | ||
203 | get_online_cpus(); | |
204 | primary_thread_mask = cpu_online_cores_map(); | |
205 | on_each_cpu_mask(&primary_thread_mask, | |
206 | pnv_fastsleep_workaround_apply, | |
207 | &err, 1); | |
208 | put_online_cpus(); | |
209 | if (err) { | |
210 | pr_err("fastsleep_workaround_applyonce change failed while running pnv_fastsleep_workaround_apply"); | |
211 | goto fail; | |
212 | } | |
213 | ||
214 | err = patch_instruction( | |
215 | (unsigned int *)pnv_fastsleep_workaround_at_entry, | |
216 | PPC_INST_NOP); | |
217 | if (err) { | |
218 | pr_err("fastsleep_workaround_applyonce change failed while patching pnv_fastsleep_workaround_at_entry"); | |
219 | goto fail; | |
220 | } | |
221 | ||
222 | fastsleep_workaround_applyonce = 1; | |
223 | ||
224 | return count; | |
225 | fail: | |
226 | return -EIO; | |
227 | } | |
228 | ||
229 | static DEVICE_ATTR(fastsleep_workaround_applyonce, 0600, | |
230 | show_fastsleep_workaround_applyonce, | |
231 | store_fastsleep_workaround_applyonce); | |
232 | ||
d405a98c SP |
233 | static int __init pnv_init_idle_states(void) |
234 | { | |
235 | struct device_node *power_mgt; | |
236 | int dt_idle_states; | |
237 | u32 *flags; | |
238 | int i; | |
239 | ||
240 | supported_cpuidle_states = 0; | |
241 | ||
242 | if (cpuidle_disable != IDLE_NO_OVERRIDE) | |
243 | goto out; | |
244 | ||
e4d54f71 | 245 | if (!firmware_has_feature(FW_FEATURE_OPAL)) |
d405a98c SP |
246 | goto out; |
247 | ||
248 | power_mgt = of_find_node_by_path("/ibm,opal/power-mgt"); | |
249 | if (!power_mgt) { | |
250 | pr_warn("opal: PowerMgmt Node not found\n"); | |
251 | goto out; | |
252 | } | |
253 | dt_idle_states = of_property_count_u32_elems(power_mgt, | |
254 | "ibm,cpu-idle-state-flags"); | |
255 | if (dt_idle_states < 0) { | |
256 | pr_warn("cpuidle-powernv: no idle states found in the DT\n"); | |
257 | goto out; | |
258 | } | |
259 | ||
260 | flags = kzalloc(sizeof(*flags) * dt_idle_states, GFP_KERNEL); | |
261 | if (of_property_read_u32_array(power_mgt, | |
262 | "ibm,cpu-idle-state-flags", flags, dt_idle_states)) { | |
263 | pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-flags in DT\n"); | |
264 | goto out_free; | |
265 | } | |
266 | ||
267 | for (i = 0; i < dt_idle_states; i++) | |
268 | supported_cpuidle_states |= flags[i]; | |
269 | ||
270 | if (!(supported_cpuidle_states & OPAL_PM_SLEEP_ENABLED_ER1)) { | |
271 | patch_instruction( | |
272 | (unsigned int *)pnv_fastsleep_workaround_at_entry, | |
273 | PPC_INST_NOP); | |
274 | patch_instruction( | |
275 | (unsigned int *)pnv_fastsleep_workaround_at_exit, | |
276 | PPC_INST_NOP); | |
5703d2f4 SP |
277 | } else { |
278 | /* | |
279 | * OPAL_PM_SLEEP_ENABLED_ER1 is set. It indicates that | |
280 | * workaround is needed to use fastsleep. Provide sysfs | |
281 | * control to choose how this workaround has to be applied. | |
282 | */ | |
283 | device_create_file(cpu_subsys.dev_root, | |
284 | &dev_attr_fastsleep_workaround_applyonce); | |
d405a98c | 285 | } |
5703d2f4 | 286 | |
d405a98c | 287 | pnv_alloc_idle_core_states(); |
5593e303 SP |
288 | |
289 | if (supported_cpuidle_states & OPAL_PM_NAP_ENABLED) | |
290 | ppc_md.power_save = power7_idle; | |
d405a98c SP |
291 | out_free: |
292 | kfree(flags); | |
293 | out: | |
294 | return 0; | |
295 | } | |
4bece972 | 296 | machine_subsys_initcall(powernv, pnv_init_idle_states); |