Commit | Line | Data |
---|---|---|
ccf55117 AK |
1 | /* |
2 | * Copyright (c) 2014 Samsung Electronics Co., Ltd. | |
3 | * http://www.samsung.com | |
4 | * | |
5 | * arch/arm/mach-exynos/mcpm-exynos.c | |
6 | * | |
7 | * Based on arch/arm/mach-vexpress/dcscb.c | |
8 | * | |
9 | * This program is free software; you can redistribute it and/or modify | |
10 | * it under the terms of the GNU General Public License version 2 as | |
11 | * published by the Free Software Foundation. | |
12 | */ | |
13 | ||
14 | #include <linux/arm-cci.h> | |
15 | #include <linux/delay.h> | |
16 | #include <linux/io.h> | |
17 | #include <linux/of_address.h> | |
adc548d7 | 18 | #include <linux/syscore_ops.h> |
ccf55117 AK |
19 | |
20 | #include <asm/cputype.h> | |
21 | #include <asm/cp15.h> | |
22 | #include <asm/mcpm.h> | |
23 | ||
24 | #include "regs-pmu.h" | |
25 | #include "common.h" | |
26 | ||
27 | #define EXYNOS5420_CPUS_PER_CLUSTER 4 | |
28 | #define EXYNOS5420_NR_CLUSTERS 2 | |
ccf55117 | 29 | |
20fe6f98 AK |
30 | #define EXYNOS5420_ENABLE_AUTOMATIC_CORE_DOWN BIT(9) |
31 | #define EXYNOS5420_USE_ARM_CORE_DOWN_STATE BIT(29) | |
32 | #define EXYNOS5420_USE_L2_COMMON_UP_STATE BIT(30) | |
33 | ||
adc548d7 AK |
34 | static void __iomem *ns_sram_base_addr; |
35 | ||
ccf55117 AK |
36 | /* |
37 | * The common v7_exit_coherency_flush API could not be used because of the | |
38 | * Erratum 799270 workaround. This macro is the same as the common one (in | |
39 | * arch/arm/include/asm/cacheflush.h) except for the erratum handling. | |
40 | */ | |
41 | #define exynos_v7_exit_coherency_flush(level) \ | |
42 | asm volatile( \ | |
43 | "stmfd sp!, {fp, ip}\n\t"\ | |
44 | "mrc p15, 0, r0, c1, c0, 0 @ get SCTLR\n\t" \ | |
45 | "bic r0, r0, #"__stringify(CR_C)"\n\t" \ | |
46 | "mcr p15, 0, r0, c1, c0, 0 @ set SCTLR\n\t" \ | |
47 | "isb\n\t"\ | |
48 | "bl v7_flush_dcache_"__stringify(level)"\n\t" \ | |
ccf55117 AK |
49 | "mrc p15, 0, r0, c1, c0, 1 @ get ACTLR\n\t" \ |
50 | "bic r0, r0, #(1 << 6) @ disable local coherency\n\t" \ | |
51 | /* Dummy Load of a device register to avoid Erratum 799270 */ \ | |
52 | "ldr r4, [%0]\n\t" \ | |
53 | "and r4, r4, #0\n\t" \ | |
54 | "orr r0, r0, r4\n\t" \ | |
55 | "mcr p15, 0, r0, c1, c0, 1 @ set ACTLR\n\t" \ | |
56 | "isb\n\t" \ | |
57 | "dsb\n\t" \ | |
58 | "ldmfd sp!, {fp, ip}" \ | |
59 | : \ | |
2e94ac42 | 60 | : "Ir" (pmu_base_addr + S5P_INFORM0) \ |
ccf55117 AK |
61 | : "r0", "r1", "r2", "r3", "r4", "r5", "r6", "r7", \ |
62 | "r9", "r10", "lr", "memory") | |
63 | ||
64 | /* | |
65 | * We can't use regular spinlocks. In the switcher case, it is possible | |
66 | * for an outbound CPU to call power_down() after its inbound counterpart | |
67 | * is already live using the same logical CPU number which trips lockdep | |
68 | * debugging. | |
69 | */ | |
70 | static arch_spinlock_t exynos_mcpm_lock = __ARCH_SPIN_LOCK_UNLOCKED; | |
71 | static int | |
72 | cpu_use_count[EXYNOS5420_CPUS_PER_CLUSTER][EXYNOS5420_NR_CLUSTERS]; | |
73 | ||
74 | #define exynos_cluster_usecnt(cluster) \ | |
75 | (cpu_use_count[0][cluster] + \ | |
76 | cpu_use_count[1][cluster] + \ | |
77 | cpu_use_count[2][cluster] + \ | |
78 | cpu_use_count[3][cluster]) | |
79 | ||
80 | #define exynos_cluster_unused(cluster) !exynos_cluster_usecnt(cluster) | |
81 | ||
ccf55117 AK |
82 | static int exynos_power_up(unsigned int cpu, unsigned int cluster) |
83 | { | |
84 | unsigned int cpunr = cpu + (cluster * EXYNOS5420_CPUS_PER_CLUSTER); | |
ccf55117 AK |
85 | |
86 | pr_debug("%s: cpu %u cluster %u\n", __func__, cpu, cluster); | |
87 | if (cpu >= EXYNOS5420_CPUS_PER_CLUSTER || | |
88 | cluster >= EXYNOS5420_NR_CLUSTERS) | |
89 | return -EINVAL; | |
90 | ||
91 | /* | |
92 | * Since this is called with IRQs enabled, and no arch_spin_lock_irq | |
93 | * variant exists, we need to disable IRQs manually here. | |
94 | */ | |
95 | local_irq_disable(); | |
96 | arch_spin_lock(&exynos_mcpm_lock); | |
97 | ||
98 | cpu_use_count[cpu][cluster]++; | |
99 | if (cpu_use_count[cpu][cluster] == 1) { | |
100 | bool was_cluster_down = | |
101 | (exynos_cluster_usecnt(cluster) == 1); | |
102 | ||
103 | /* | |
104 | * Turn on the cluster (L2/COMMON) and then power on the | |
105 | * cores. | |
106 | */ | |
107 | if (was_cluster_down) | |
20fe6f98 | 108 | exynos_cluster_power_up(cluster); |
ccf55117 | 109 | |
20fe6f98 | 110 | exynos_cpu_power_up(cpunr); |
ccf55117 AK |
111 | } else if (cpu_use_count[cpu][cluster] != 2) { |
112 | /* | |
113 | * The only possible values are: | |
114 | * 0 = CPU down | |
115 | * 1 = CPU (still) up | |
116 | * 2 = CPU requested to be up before it had a chance | |
117 | * to actually make itself down. | |
118 | * Any other value is a bug. | |
119 | */ | |
120 | BUG(); | |
121 | } | |
122 | ||
123 | arch_spin_unlock(&exynos_mcpm_lock); | |
124 | local_irq_enable(); | |
125 | ||
20fe6f98 | 126 | return 0; |
ccf55117 AK |
127 | } |
128 | ||
129 | /* | |
130 | * NOTE: This function requires the stack data to be visible through power down | |
131 | * and can only be executed on processors like A15 and A7 that hit the cache | |
132 | * with the C bit clear in the SCTLR register. | |
133 | */ | |
134 | static void exynos_power_down(void) | |
135 | { | |
136 | unsigned int mpidr, cpu, cluster; | |
137 | bool last_man = false, skip_wfi = false; | |
138 | unsigned int cpunr; | |
139 | ||
140 | mpidr = read_cpuid_mpidr(); | |
141 | cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0); | |
142 | cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1); | |
143 | cpunr = cpu + (cluster * EXYNOS5420_CPUS_PER_CLUSTER); | |
144 | ||
145 | pr_debug("%s: cpu %u cluster %u\n", __func__, cpu, cluster); | |
146 | BUG_ON(cpu >= EXYNOS5420_CPUS_PER_CLUSTER || | |
147 | cluster >= EXYNOS5420_NR_CLUSTERS); | |
148 | ||
149 | __mcpm_cpu_going_down(cpu, cluster); | |
150 | ||
151 | arch_spin_lock(&exynos_mcpm_lock); | |
152 | BUG_ON(__mcpm_cluster_state(cluster) != CLUSTER_UP); | |
153 | cpu_use_count[cpu][cluster]--; | |
154 | if (cpu_use_count[cpu][cluster] == 0) { | |
155 | exynos_cpu_power_down(cpunr); | |
156 | ||
20fe6f98 AK |
157 | if (exynos_cluster_unused(cluster)) { |
158 | exynos_cluster_power_down(cluster); | |
ccf55117 | 159 | last_man = true; |
20fe6f98 | 160 | } |
ccf55117 AK |
161 | } else if (cpu_use_count[cpu][cluster] == 1) { |
162 | /* | |
163 | * A power_up request went ahead of us. | |
164 | * Even if we do not want to shut this CPU down, | |
165 | * the caller expects a certain state as if the WFI | |
166 | * was aborted. So let's continue with cache cleaning. | |
167 | */ | |
168 | skip_wfi = true; | |
169 | } else { | |
170 | BUG(); | |
171 | } | |
172 | ||
173 | if (last_man && __mcpm_outbound_enter_critical(cpu, cluster)) { | |
174 | arch_spin_unlock(&exynos_mcpm_lock); | |
175 | ||
af040ffc | 176 | if (read_cpuid_part() == ARM_CPU_PART_CORTEX_A15) { |
ccf55117 AK |
177 | /* |
178 | * On the Cortex-A15 we need to disable | |
179 | * L2 prefetching before flushing the cache. | |
180 | */ | |
181 | asm volatile( | |
182 | "mcr p15, 1, %0, c15, c0, 3\n\t" | |
183 | "isb\n\t" | |
184 | "dsb" | |
185 | : : "r" (0x400)); | |
186 | } | |
187 | ||
188 | /* Flush all cache levels for this cluster. */ | |
189 | exynos_v7_exit_coherency_flush(all); | |
190 | ||
191 | /* | |
192 | * Disable cluster-level coherency by masking | |
193 | * incoming snoops and DVM messages: | |
194 | */ | |
195 | cci_disable_port_by_cpu(mpidr); | |
196 | ||
197 | __mcpm_outbound_leave_critical(cluster, CLUSTER_DOWN); | |
198 | } else { | |
199 | arch_spin_unlock(&exynos_mcpm_lock); | |
200 | ||
201 | /* Disable and flush the local CPU cache. */ | |
202 | exynos_v7_exit_coherency_flush(louis); | |
203 | } | |
204 | ||
205 | __mcpm_cpu_down(cpu, cluster); | |
206 | ||
207 | /* Now we are prepared for power-down, do it: */ | |
208 | if (!skip_wfi) | |
209 | wfi(); | |
210 | ||
211 | /* Not dead at this point? Let our caller cope. */ | |
212 | } | |
213 | ||
7c5688e7 | 214 | static int exynos_wait_for_powerdown(unsigned int cpu, unsigned int cluster) |
ccf55117 AK |
215 | { |
216 | unsigned int tries = 100; | |
217 | unsigned int cpunr = cpu + (cluster * EXYNOS5420_CPUS_PER_CLUSTER); | |
218 | ||
219 | pr_debug("%s: cpu %u cluster %u\n", __func__, cpu, cluster); | |
220 | BUG_ON(cpu >= EXYNOS5420_CPUS_PER_CLUSTER || | |
221 | cluster >= EXYNOS5420_NR_CLUSTERS); | |
222 | ||
223 | /* Wait for the core state to be OFF */ | |
224 | while (tries--) { | |
225 | if (ACCESS_ONCE(cpu_use_count[cpu][cluster]) == 0) { | |
226 | if ((exynos_cpu_power_state(cpunr) == 0)) | |
227 | return 0; /* success: the CPU is halted */ | |
228 | } | |
229 | ||
230 | /* Otherwise, wait and retry: */ | |
231 | msleep(1); | |
232 | } | |
233 | ||
234 | return -ETIMEDOUT; /* timeout */ | |
235 | } | |
236 | ||
fc2cac41 CK |
237 | static void exynos_powered_up(void) |
238 | { | |
239 | unsigned int mpidr, cpu, cluster; | |
240 | ||
241 | mpidr = read_cpuid_mpidr(); | |
242 | cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0); | |
243 | cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1); | |
244 | ||
245 | arch_spin_lock(&exynos_mcpm_lock); | |
246 | if (cpu_use_count[cpu][cluster] == 0) | |
247 | cpu_use_count[cpu][cluster] = 1; | |
248 | arch_spin_unlock(&exynos_mcpm_lock); | |
249 | } | |
250 | ||
251 | static void exynos_suspend(u64 residency) | |
252 | { | |
253 | unsigned int mpidr, cpunr; | |
254 | ||
255 | exynos_power_down(); | |
256 | ||
257 | /* | |
258 | * Execution reaches here only if cpu did not power down. | |
259 | * Hence roll back the changes done in exynos_power_down function. | |
260 | * | |
261 | * CAUTION: "This function requires the stack data to be visible through | |
262 | * power down and can only be executed on processors like A15 and A7 | |
263 | * that hit the cache with the C bit clear in the SCTLR register." | |
264 | */ | |
265 | mpidr = read_cpuid_mpidr(); | |
266 | cpunr = exynos_pmu_cpunr(mpidr); | |
267 | ||
268 | exynos_cpu_power_up(cpunr); | |
269 | } | |
270 | ||
ccf55117 AK |
271 | static const struct mcpm_platform_ops exynos_power_ops = { |
272 | .power_up = exynos_power_up, | |
273 | .power_down = exynos_power_down, | |
7c5688e7 | 274 | .wait_for_powerdown = exynos_wait_for_powerdown, |
fc2cac41 CK |
275 | .suspend = exynos_suspend, |
276 | .powered_up = exynos_powered_up, | |
ccf55117 AK |
277 | }; |
278 | ||
279 | static void __init exynos_mcpm_usage_count_init(void) | |
280 | { | |
281 | unsigned int mpidr, cpu, cluster; | |
282 | ||
283 | mpidr = read_cpuid_mpidr(); | |
284 | cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0); | |
285 | cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1); | |
286 | ||
287 | pr_debug("%s: cpu %u cluster %u\n", __func__, cpu, cluster); | |
288 | BUG_ON(cpu >= EXYNOS5420_CPUS_PER_CLUSTER || | |
289 | cluster >= EXYNOS5420_NR_CLUSTERS); | |
290 | ||
291 | cpu_use_count[cpu][cluster] = 1; | |
292 | } | |
293 | ||
294 | /* | |
295 | * Enable cluster-level coherency, in preparation for turning on the MMU. | |
296 | */ | |
297 | static void __naked exynos_pm_power_up_setup(unsigned int affinity_level) | |
298 | { | |
299 | asm volatile ("\n" | |
300 | "cmp r0, #1\n" | |
301 | "bxne lr\n" | |
302 | "b cci_enable_port_for_self"); | |
303 | } | |
304 | ||
fbb04990 NP |
305 | static void __init exynos_cache_off(void) |
306 | { | |
af040ffc | 307 | if (read_cpuid_part() == ARM_CPU_PART_CORTEX_A15) { |
fbb04990 NP |
308 | /* disable L2 prefetching on the Cortex-A15 */ |
309 | asm volatile( | |
310 | "mcr p15, 1, %0, c15, c0, 3\n\t" | |
311 | "isb\n\t" | |
312 | "dsb" | |
313 | : : "r" (0x400)); | |
314 | } | |
315 | exynos_v7_exit_coherency_flush(all); | |
316 | } | |
317 | ||
f99acff1 AK |
318 | static const struct of_device_id exynos_dt_mcpm_match[] = { |
319 | { .compatible = "samsung,exynos5420" }, | |
320 | { .compatible = "samsung,exynos5800" }, | |
321 | {}, | |
322 | }; | |
323 | ||
adc548d7 AK |
324 | static void exynos_mcpm_setup_entry_point(void) |
325 | { | |
326 | /* | |
327 | * U-Boot SPL is hardcoded to jump to the start of ns_sram_base_addr | |
328 | * as part of secondary_cpu_start(). Let's redirect it to the | |
329 | * mcpm_entry_point(). This is done during both secondary boot-up as | |
330 | * well as system resume. | |
331 | */ | |
332 | __raw_writel(0xe59f0000, ns_sram_base_addr); /* ldr r0, [pc, #0] */ | |
333 | __raw_writel(0xe12fff10, ns_sram_base_addr + 4); /* bx r0 */ | |
334 | __raw_writel(virt_to_phys(mcpm_entry_point), ns_sram_base_addr + 8); | |
335 | } | |
336 | ||
337 | static struct syscore_ops exynos_mcpm_syscore_ops = { | |
338 | .resume = exynos_mcpm_setup_entry_point, | |
339 | }; | |
340 | ||
ccf55117 AK |
341 | static int __init exynos_mcpm_init(void) |
342 | { | |
343 | struct device_node *node; | |
20fe6f98 | 344 | unsigned int value, i; |
ccf55117 AK |
345 | int ret; |
346 | ||
f99acff1 | 347 | node = of_find_matching_node(NULL, exynos_dt_mcpm_match); |
ccf55117 AK |
348 | if (!node) |
349 | return -ENODEV; | |
350 | of_node_put(node); | |
351 | ||
352 | if (!cci_probed()) | |
353 | return -ENODEV; | |
354 | ||
355 | node = of_find_compatible_node(NULL, NULL, | |
356 | "samsung,exynos4210-sysram-ns"); | |
357 | if (!node) | |
358 | return -ENODEV; | |
359 | ||
360 | ns_sram_base_addr = of_iomap(node, 0); | |
361 | of_node_put(node); | |
362 | if (!ns_sram_base_addr) { | |
363 | pr_err("failed to map non-secure iRAM base address\n"); | |
364 | return -ENOMEM; | |
365 | } | |
366 | ||
367 | /* | |
368 | * To increase the stability of KFC reset we need to program | |
369 | * the PMU SPARE3 register | |
370 | */ | |
2e94ac42 | 371 | pmu_raw_writel(EXYNOS5420_SWRESET_KFC_SEL, S5P_PMU_SPARE3); |
ccf55117 AK |
372 | |
373 | exynos_mcpm_usage_count_init(); | |
374 | ||
375 | ret = mcpm_platform_register(&exynos_power_ops); | |
376 | if (!ret) | |
377 | ret = mcpm_sync_init(exynos_pm_power_up_setup); | |
fbb04990 NP |
378 | if (!ret) |
379 | ret = mcpm_loopback(exynos_cache_off); /* turn on the CCI */ | |
ccf55117 AK |
380 | if (ret) { |
381 | iounmap(ns_sram_base_addr); | |
382 | return ret; | |
383 | } | |
384 | ||
385 | mcpm_smp_set_ops(); | |
386 | ||
387 | pr_info("Exynos MCPM support installed\n"); | |
388 | ||
20fe6f98 AK |
389 | /* |
390 | * On Exynos5420/5800 for the A15 and A7 clusters: | |
391 | * | |
392 | * EXYNOS5420_ENABLE_AUTOMATIC_CORE_DOWN ensures that all the cores | |
393 | * in a cluster are turned off before turning off the cluster L2. | |
394 | * | |
395 | * EXYNOS5420_USE_ARM_CORE_DOWN_STATE ensures that a cores is powered | |
396 | * off before waking it up. | |
397 | * | |
398 | * EXYNOS5420_USE_L2_COMMON_UP_STATE ensures that cluster L2 will be | |
399 | * turned on before the first man is powered up. | |
400 | */ | |
401 | for (i = 0; i < EXYNOS5420_NR_CLUSTERS; i++) { | |
2e94ac42 | 402 | value = pmu_raw_readl(EXYNOS_COMMON_OPTION(i)); |
20fe6f98 AK |
403 | value |= EXYNOS5420_ENABLE_AUTOMATIC_CORE_DOWN | |
404 | EXYNOS5420_USE_ARM_CORE_DOWN_STATE | | |
405 | EXYNOS5420_USE_L2_COMMON_UP_STATE; | |
2e94ac42 | 406 | pmu_raw_writel(value, EXYNOS_COMMON_OPTION(i)); |
20fe6f98 AK |
407 | } |
408 | ||
adc548d7 | 409 | exynos_mcpm_setup_entry_point(); |
ccf55117 | 410 | |
adc548d7 | 411 | register_syscore_ops(&exynos_mcpm_syscore_ops); |
ccf55117 AK |
412 | |
413 | return ret; | |
414 | } | |
415 | ||
416 | early_initcall(exynos_mcpm_init); |