2 * arch/arm/mm/cache-l2x0.c - L210/L220 cache controller support
4 * Copyright (C) 2007 ARM Limited
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19 #include <linux/cpu.h>
20 #include <linux/err.h>
21 #include <linux/init.h>
22 #include <linux/smp.h>
23 #include <linux/spinlock.h>
24 #include <linux/log2.h>
27 #include <linux/of_address.h>
29 #include <asm/cacheflush.h>
31 #include <asm/cputype.h>
32 #include <asm/hardware/cache-l2x0.h>
33 #include "cache-tauros3.h"
34 #include "cache-aurora-l2.h"
36 struct l2c_init_data
{
40 void (*of_parse
)(const struct device_node
*, u32
*, u32
*);
41 void (*enable
)(void __iomem
*, u32
, unsigned);
42 void (*fixup
)(void __iomem
*, u32
, struct outer_cache_fns
*);
43 void (*save
)(void __iomem
*);
44 struct outer_cache_fns outer_cache
;
47 #define CACHE_LINE_SIZE 32
49 static void __iomem
*l2x0_base
;
50 static DEFINE_RAW_SPINLOCK(l2x0_lock
);
51 static u32 l2x0_way_mask
; /* Bitmask of active ways */
53 static unsigned long sync_reg_offset
= L2X0_CACHE_SYNC
;
55 struct l2x0_regs l2x0_saved_regs
;
58 * Common code for all cache controllers.
60 static inline void l2c_wait_mask(void __iomem
*reg
, unsigned long mask
)
62 /* wait for cache operation by line or way to complete */
63 while (readl_relaxed(reg
) & mask
)
68 * By default, we write directly to secure registers. Platforms must
69 * override this if they are running non-secure.
71 static void l2c_write_sec(unsigned long val
, void __iomem
*base
, unsigned reg
)
73 if (val
== readl_relaxed(base
+ reg
))
75 if (outer_cache
.write_sec
)
76 outer_cache
.write_sec(val
, reg
);
78 writel_relaxed(val
, base
+ reg
);
82 * This should only be called when we have a requirement that the
83 * register be written due to a work-around, as platforms running
84 * in non-secure mode may not be able to access this register.
86 static inline void l2c_set_debug(void __iomem
*base
, unsigned long val
)
88 l2c_write_sec(val
, base
, L2X0_DEBUG_CTRL
);
91 static void __l2c_op_way(void __iomem
*reg
)
93 writel_relaxed(l2x0_way_mask
, reg
);
94 l2c_wait_mask(reg
, l2x0_way_mask
);
97 static inline void l2c_unlock(void __iomem
*base
, unsigned num
)
101 for (i
= 0; i
< num
; i
++) {
102 writel_relaxed(0, base
+ L2X0_LOCKDOWN_WAY_D_BASE
+
103 i
* L2X0_LOCKDOWN_STRIDE
);
104 writel_relaxed(0, base
+ L2X0_LOCKDOWN_WAY_I_BASE
+
105 i
* L2X0_LOCKDOWN_STRIDE
);
110 * Enable the L2 cache controller. This function must only be
111 * called when the cache controller is known to be disabled.
113 static void l2c_enable(void __iomem
*base
, u32 aux
, unsigned num_lock
)
117 l2c_write_sec(aux
, base
, L2X0_AUX_CTRL
);
119 l2c_unlock(base
, num_lock
);
121 local_irq_save(flags
);
122 __l2c_op_way(base
+ L2X0_INV_WAY
);
123 writel_relaxed(0, base
+ sync_reg_offset
);
124 l2c_wait_mask(base
+ sync_reg_offset
, 1);
125 local_irq_restore(flags
);
127 l2c_write_sec(L2X0_CTRL_EN
, base
, L2X0_CTRL
);
130 static void l2c_disable(void)
132 void __iomem
*base
= l2x0_base
;
134 outer_cache
.flush_all();
135 l2c_write_sec(0, base
, L2X0_CTRL
);
139 #ifdef CONFIG_CACHE_PL310
140 static inline void cache_wait(void __iomem
*reg
, unsigned long mask
)
142 /* cache operations by line are atomic on PL310 */
145 #define cache_wait l2c_wait_mask
148 static inline void cache_sync(void)
150 void __iomem
*base
= l2x0_base
;
152 writel_relaxed(0, base
+ sync_reg_offset
);
153 cache_wait(base
+ L2X0_CACHE_SYNC
, 1);
156 #if defined(CONFIG_PL310_ERRATA_588369) || defined(CONFIG_PL310_ERRATA_727915)
157 static inline void debug_writel(unsigned long val
)
159 l2c_set_debug(l2x0_base
, val
);
162 /* Optimised out for non-errata case */
163 static inline void debug_writel(unsigned long val
)
168 static void l2x0_cache_sync(void)
172 raw_spin_lock_irqsave(&l2x0_lock
, flags
);
174 raw_spin_unlock_irqrestore(&l2x0_lock
, flags
);
177 static void __l2x0_flush_all(void)
180 __l2c_op_way(l2x0_base
+ L2X0_CLEAN_INV_WAY
);
185 static void l2x0_flush_all(void)
190 raw_spin_lock_irqsave(&l2x0_lock
, flags
);
192 raw_spin_unlock_irqrestore(&l2x0_lock
, flags
);
195 static void l2x0_disable(void)
199 raw_spin_lock_irqsave(&l2x0_lock
, flags
);
201 l2c_write_sec(0, l2x0_base
, L2X0_CTRL
);
203 raw_spin_unlock_irqrestore(&l2x0_lock
, flags
);
206 static void l2c_save(void __iomem
*base
)
208 l2x0_saved_regs
.aux_ctrl
= readl_relaxed(l2x0_base
+ L2X0_AUX_CTRL
);
212 * L2C-210 specific code.
214 * The L2C-2x0 PA, set/way and sync operations are atomic, but we must
215 * ensure that no background operation is running. The way operations
216 * are all background tasks.
218 * While a background operation is in progress, any new operation is
219 * ignored (unspecified whether this causes an error.) Thankfully, not
222 * Never has a different sync register other than L2X0_CACHE_SYNC, but
223 * we use sync_reg_offset here so we can share some of this with L2C-310.
225 static void __l2c210_cache_sync(void __iomem
*base
)
227 writel_relaxed(0, base
+ sync_reg_offset
);
230 static void __l2c210_op_pa_range(void __iomem
*reg
, unsigned long start
,
233 while (start
< end
) {
234 writel_relaxed(start
, reg
);
235 start
+= CACHE_LINE_SIZE
;
239 static void l2c210_inv_range(unsigned long start
, unsigned long end
)
241 void __iomem
*base
= l2x0_base
;
243 if (start
& (CACHE_LINE_SIZE
- 1)) {
244 start
&= ~(CACHE_LINE_SIZE
- 1);
245 writel_relaxed(start
, base
+ L2X0_CLEAN_INV_LINE_PA
);
246 start
+= CACHE_LINE_SIZE
;
249 if (end
& (CACHE_LINE_SIZE
- 1)) {
250 end
&= ~(CACHE_LINE_SIZE
- 1);
251 writel_relaxed(end
, base
+ L2X0_CLEAN_INV_LINE_PA
);
254 __l2c210_op_pa_range(base
+ L2X0_INV_LINE_PA
, start
, end
);
255 __l2c210_cache_sync(base
);
258 static void l2c210_clean_range(unsigned long start
, unsigned long end
)
260 void __iomem
*base
= l2x0_base
;
262 start
&= ~(CACHE_LINE_SIZE
- 1);
263 __l2c210_op_pa_range(base
+ L2X0_CLEAN_LINE_PA
, start
, end
);
264 __l2c210_cache_sync(base
);
267 static void l2c210_flush_range(unsigned long start
, unsigned long end
)
269 void __iomem
*base
= l2x0_base
;
271 start
&= ~(CACHE_LINE_SIZE
- 1);
272 __l2c210_op_pa_range(base
+ L2X0_CLEAN_INV_LINE_PA
, start
, end
);
273 __l2c210_cache_sync(base
);
276 static void l2c210_flush_all(void)
278 void __iomem
*base
= l2x0_base
;
280 BUG_ON(!irqs_disabled());
282 __l2c_op_way(base
+ L2X0_CLEAN_INV_WAY
);
283 __l2c210_cache_sync(base
);
286 static void l2c210_sync(void)
288 __l2c210_cache_sync(l2x0_base
);
291 static void l2c210_resume(void)
293 void __iomem
*base
= l2x0_base
;
295 if (!(readl_relaxed(base
+ L2X0_CTRL
) & L2X0_CTRL_EN
))
296 l2c_enable(base
, l2x0_saved_regs
.aux_ctrl
, 1);
299 static const struct l2c_init_data l2c210_data __initconst
= {
303 .enable
= l2c_enable
,
306 .inv_range
= l2c210_inv_range
,
307 .clean_range
= l2c210_clean_range
,
308 .flush_range
= l2c210_flush_range
,
309 .flush_all
= l2c210_flush_all
,
310 .disable
= l2c_disable
,
312 .resume
= l2c210_resume
,
317 * L2C-220 specific code.
319 * All operations are background operations: they have to be waited for.
320 * Conflicting requests generate a slave error (which will cause an
321 * imprecise abort.) Never uses sync_reg_offset, so we hard-code the
322 * sync register here.
324 * However, we can re-use the l2c210_resume call.
326 static inline void __l2c220_cache_sync(void __iomem
*base
)
328 writel_relaxed(0, base
+ L2X0_CACHE_SYNC
);
329 l2c_wait_mask(base
+ L2X0_CACHE_SYNC
, 1);
332 static void l2c220_op_way(void __iomem
*base
, unsigned reg
)
336 raw_spin_lock_irqsave(&l2x0_lock
, flags
);
337 __l2c_op_way(base
+ reg
);
338 __l2c220_cache_sync(base
);
339 raw_spin_unlock_irqrestore(&l2x0_lock
, flags
);
342 static unsigned long l2c220_op_pa_range(void __iomem
*reg
, unsigned long start
,
343 unsigned long end
, unsigned long flags
)
345 raw_spinlock_t
*lock
= &l2x0_lock
;
347 while (start
< end
) {
348 unsigned long blk_end
= start
+ min(end
- start
, 4096UL);
350 while (start
< blk_end
) {
351 l2c_wait_mask(reg
, 1);
352 writel_relaxed(start
, reg
);
353 start
+= CACHE_LINE_SIZE
;
357 raw_spin_unlock_irqrestore(lock
, flags
);
358 raw_spin_lock_irqsave(lock
, flags
);
365 static void l2c220_inv_range(unsigned long start
, unsigned long end
)
367 void __iomem
*base
= l2x0_base
;
370 raw_spin_lock_irqsave(&l2x0_lock
, flags
);
371 if ((start
| end
) & (CACHE_LINE_SIZE
- 1)) {
372 if (start
& (CACHE_LINE_SIZE
- 1)) {
373 start
&= ~(CACHE_LINE_SIZE
- 1);
374 writel_relaxed(start
, base
+ L2X0_CLEAN_INV_LINE_PA
);
375 start
+= CACHE_LINE_SIZE
;
378 if (end
& (CACHE_LINE_SIZE
- 1)) {
379 end
&= ~(CACHE_LINE_SIZE
- 1);
380 l2c_wait_mask(base
+ L2X0_CLEAN_INV_LINE_PA
, 1);
381 writel_relaxed(end
, base
+ L2X0_CLEAN_INV_LINE_PA
);
385 flags
= l2c220_op_pa_range(base
+ L2X0_INV_LINE_PA
,
387 l2c_wait_mask(base
+ L2X0_INV_LINE_PA
, 1);
388 __l2c220_cache_sync(base
);
389 raw_spin_unlock_irqrestore(&l2x0_lock
, flags
);
392 static void l2c220_clean_range(unsigned long start
, unsigned long end
)
394 void __iomem
*base
= l2x0_base
;
397 start
&= ~(CACHE_LINE_SIZE
- 1);
398 if ((end
- start
) >= l2x0_size
) {
399 l2c220_op_way(base
, L2X0_CLEAN_WAY
);
403 raw_spin_lock_irqsave(&l2x0_lock
, flags
);
404 flags
= l2c220_op_pa_range(base
+ L2X0_CLEAN_LINE_PA
,
406 l2c_wait_mask(base
+ L2X0_CLEAN_INV_LINE_PA
, 1);
407 __l2c220_cache_sync(base
);
408 raw_spin_unlock_irqrestore(&l2x0_lock
, flags
);
411 static void l2c220_flush_range(unsigned long start
, unsigned long end
)
413 void __iomem
*base
= l2x0_base
;
416 start
&= ~(CACHE_LINE_SIZE
- 1);
417 if ((end
- start
) >= l2x0_size
) {
418 l2c220_op_way(base
, L2X0_CLEAN_INV_WAY
);
422 raw_spin_lock_irqsave(&l2x0_lock
, flags
);
423 flags
= l2c220_op_pa_range(base
+ L2X0_CLEAN_INV_LINE_PA
,
425 l2c_wait_mask(base
+ L2X0_CLEAN_INV_LINE_PA
, 1);
426 __l2c220_cache_sync(base
);
427 raw_spin_unlock_irqrestore(&l2x0_lock
, flags
);
430 static void l2c220_flush_all(void)
432 l2c220_op_way(l2x0_base
, L2X0_CLEAN_INV_WAY
);
435 static void l2c220_sync(void)
439 raw_spin_lock_irqsave(&l2x0_lock
, flags
);
440 __l2c220_cache_sync(l2x0_base
);
441 raw_spin_unlock_irqrestore(&l2x0_lock
, flags
);
444 static void l2c220_enable(void __iomem
*base
, u32 aux
, unsigned num_lock
)
447 * Always enable non-secure access to the lockdown registers -
448 * we write to them as part of the L2C enable sequence so they
449 * need to be accessible.
451 aux
|= L220_AUX_CTRL_NS_LOCKDOWN
;
453 l2c_enable(base
, aux
, num_lock
);
456 static const struct l2c_init_data l2c220_data
= {
460 .enable
= l2c220_enable
,
463 .inv_range
= l2c220_inv_range
,
464 .clean_range
= l2c220_clean_range
,
465 .flush_range
= l2c220_flush_range
,
466 .flush_all
= l2c220_flush_all
,
467 .disable
= l2c_disable
,
469 .resume
= l2c210_resume
,
474 * L2C-310 specific code.
476 * Very similar to L2C-210, the PA, set/way and sync operations are atomic,
477 * and the way operations are all background tasks. However, issuing an
478 * operation while a background operation is in progress results in a
479 * SLVERR response. We can reuse:
481 * __l2c210_cache_sync (using sync_reg_offset)
483 * l2c210_inv_range (if 588369 is not applicable)
485 * l2c210_flush_range (if 588369 is not applicable)
486 * l2c210_flush_all (if 727915 is not applicable)
489 * 588369: PL310 R0P0->R1P0, fixed R2P0.
490 * Affects: all clean+invalidate operations
491 * clean and invalidate skips the invalidate step, so we need to issue
492 * separate operations. We also require the above debug workaround
493 * enclosing this code fragment on affected parts. On unaffected parts,
494 * we must not use this workaround without the debug register writes
495 * to avoid exposing a problem similar to 727915.
497 * 727915: PL310 R2P0->R3P0, fixed R3P1.
498 * Affects: clean+invalidate by way
499 * clean and invalidate by way runs in the background, and a store can
500 * hit the line between the clean operation and invalidate operation,
501 * resulting in the store being lost.
503 * 752271: PL310 R3P0->R3P1-50REL0, fixed R3P2.
504 * Affects: 8x64-bit (double fill) line fetches
505 * double fill line fetches can fail to cause dirty data to be evicted
506 * from the cache before the new data overwrites the second line.
508 * 753970: PL310 R3P0, fixed R3P1.
510 * prevents merging writes after the sync operation, until another L2C
511 * operation is performed (or a number of other conditions.)
513 * 769419: PL310 R0P0->R3P1, fixed R3P2.
514 * Affects: store buffer
515 * store buffer is not automatically drained.
517 static void l2c310_inv_range_erratum(unsigned long start
, unsigned long end
)
519 void __iomem
*base
= l2x0_base
;
521 if ((start
| end
) & (CACHE_LINE_SIZE
- 1)) {
524 /* Erratum 588369 for both clean+invalidate operations */
525 raw_spin_lock_irqsave(&l2x0_lock
, flags
);
526 l2c_set_debug(base
, 0x03);
528 if (start
& (CACHE_LINE_SIZE
- 1)) {
529 start
&= ~(CACHE_LINE_SIZE
- 1);
530 writel_relaxed(start
, base
+ L2X0_CLEAN_LINE_PA
);
531 writel_relaxed(start
, base
+ L2X0_INV_LINE_PA
);
532 start
+= CACHE_LINE_SIZE
;
535 if (end
& (CACHE_LINE_SIZE
- 1)) {
536 end
&= ~(CACHE_LINE_SIZE
- 1);
537 writel_relaxed(end
, base
+ L2X0_CLEAN_LINE_PA
);
538 writel_relaxed(end
, base
+ L2X0_INV_LINE_PA
);
541 l2c_set_debug(base
, 0x00);
542 raw_spin_unlock_irqrestore(&l2x0_lock
, flags
);
545 __l2c210_op_pa_range(base
+ L2X0_INV_LINE_PA
, start
, end
);
546 __l2c210_cache_sync(base
);
549 static void l2c310_flush_range_erratum(unsigned long start
, unsigned long end
)
551 raw_spinlock_t
*lock
= &l2x0_lock
;
553 void __iomem
*base
= l2x0_base
;
555 raw_spin_lock_irqsave(lock
, flags
);
556 while (start
< end
) {
557 unsigned long blk_end
= start
+ min(end
- start
, 4096UL);
559 l2c_set_debug(base
, 0x03);
560 while (start
< blk_end
) {
561 writel_relaxed(start
, base
+ L2X0_CLEAN_LINE_PA
);
562 writel_relaxed(start
, base
+ L2X0_INV_LINE_PA
);
563 start
+= CACHE_LINE_SIZE
;
565 l2c_set_debug(base
, 0x00);
568 raw_spin_unlock_irqrestore(lock
, flags
);
569 raw_spin_lock_irqsave(lock
, flags
);
572 raw_spin_unlock_irqrestore(lock
, flags
);
573 __l2c210_cache_sync(base
);
576 static void l2c310_flush_all_erratum(void)
578 void __iomem
*base
= l2x0_base
;
581 raw_spin_lock_irqsave(&l2x0_lock
, flags
);
582 l2c_set_debug(base
, 0x03);
583 __l2c_op_way(base
+ L2X0_CLEAN_INV_WAY
);
584 l2c_set_debug(base
, 0x00);
585 __l2c210_cache_sync(base
);
586 raw_spin_unlock_irqrestore(&l2x0_lock
, flags
);
589 static void __init
l2c310_save(void __iomem
*base
)
595 l2x0_saved_regs
.tag_latency
= readl_relaxed(base
+
596 L310_TAG_LATENCY_CTRL
);
597 l2x0_saved_regs
.data_latency
= readl_relaxed(base
+
598 L310_DATA_LATENCY_CTRL
);
599 l2x0_saved_regs
.filter_end
= readl_relaxed(base
+
600 L310_ADDR_FILTER_END
);
601 l2x0_saved_regs
.filter_start
= readl_relaxed(base
+
602 L310_ADDR_FILTER_START
);
604 revision
= readl_relaxed(base
+ L2X0_CACHE_ID
) &
605 L2X0_CACHE_ID_RTL_MASK
;
607 /* From r2p0, there is Prefetch offset/control register */
608 if (revision
>= L310_CACHE_ID_RTL_R2P0
)
609 l2x0_saved_regs
.prefetch_ctrl
= readl_relaxed(base
+
612 /* From r3p0, there is Power control register */
613 if (revision
>= L310_CACHE_ID_RTL_R3P0
)
614 l2x0_saved_regs
.pwr_ctrl
= readl_relaxed(base
+
618 static void l2c310_resume(void)
620 void __iomem
*base
= l2x0_base
;
622 if (!(readl_relaxed(base
+ L2X0_CTRL
) & L2X0_CTRL_EN
)) {
625 /* restore pl310 setup */
626 writel_relaxed(l2x0_saved_regs
.tag_latency
,
627 base
+ L310_TAG_LATENCY_CTRL
);
628 writel_relaxed(l2x0_saved_regs
.data_latency
,
629 base
+ L310_DATA_LATENCY_CTRL
);
630 writel_relaxed(l2x0_saved_regs
.filter_end
,
631 base
+ L310_ADDR_FILTER_END
);
632 writel_relaxed(l2x0_saved_regs
.filter_start
,
633 base
+ L310_ADDR_FILTER_START
);
635 revision
= readl_relaxed(base
+ L2X0_CACHE_ID
) &
636 L2X0_CACHE_ID_RTL_MASK
;
638 if (revision
>= L310_CACHE_ID_RTL_R2P0
)
639 l2c_write_sec(l2x0_saved_regs
.prefetch_ctrl
, base
,
641 if (revision
>= L310_CACHE_ID_RTL_R3P0
)
642 l2c_write_sec(l2x0_saved_regs
.pwr_ctrl
, base
,
645 l2c_enable(base
, l2x0_saved_regs
.aux_ctrl
, 8);
647 /* Re-enable full-line-of-zeros for Cortex-A9 */
648 if (l2x0_saved_regs
.aux_ctrl
& L310_AUX_CTRL_FULL_LINE_ZERO
)
649 set_auxcr(get_auxcr() | BIT(3) | BIT(2) | BIT(1));
653 static int l2c310_cpu_enable_flz(struct notifier_block
*nb
, unsigned long act
, void *data
)
655 switch (act
& ~CPU_TASKS_FROZEN
) {
657 set_auxcr(get_auxcr() | BIT(3) | BIT(2) | BIT(1));
660 set_auxcr(get_auxcr() & ~(BIT(3) | BIT(2) | BIT(1)));
666 static void __init
l2c310_enable(void __iomem
*base
, u32 aux
, unsigned num_lock
)
668 unsigned rev
= readl_relaxed(base
+ L2X0_CACHE_ID
) & L2X0_CACHE_ID_RTL_MASK
;
669 bool cortex_a9
= read_cpuid_part() == ARM_CPU_PART_CORTEX_A9
;
671 if (rev
>= L310_CACHE_ID_RTL_R2P0
) {
673 aux
|= L310_AUX_CTRL_EARLY_BRESP
;
674 pr_info("L2C-310 enabling early BRESP for Cortex-A9\n");
675 } else if (aux
& L310_AUX_CTRL_EARLY_BRESP
) {
676 pr_warn("L2C-310 early BRESP only supported with Cortex-A9\n");
677 aux
&= ~L310_AUX_CTRL_EARLY_BRESP
;
682 u32 aux_cur
= readl_relaxed(base
+ L2X0_AUX_CTRL
);
683 u32 acr
= get_auxcr();
685 pr_debug("Cortex-A9 ACR=0x%08x\n", acr
);
687 if (acr
& BIT(3) && !(aux_cur
& L310_AUX_CTRL_FULL_LINE_ZERO
))
688 pr_err("L2C-310: full line of zeros enabled in Cortex-A9 but not L2C-310 - invalid\n");
690 if (aux
& L310_AUX_CTRL_FULL_LINE_ZERO
&& !(acr
& BIT(3)))
691 pr_err("L2C-310: enabling full line of zeros but not enabled in Cortex-A9\n");
693 if (!(aux
& L310_AUX_CTRL_FULL_LINE_ZERO
) && !outer_cache
.write_sec
) {
694 aux
|= L310_AUX_CTRL_FULL_LINE_ZERO
;
695 pr_info("L2C-310 full line of zeros enabled for Cortex-A9\n");
697 } else if (aux
& (L310_AUX_CTRL_FULL_LINE_ZERO
| L310_AUX_CTRL_EARLY_BRESP
)) {
698 pr_err("L2C-310: disabling Cortex-A9 specific feature bits\n");
699 aux
&= ~(L310_AUX_CTRL_FULL_LINE_ZERO
| L310_AUX_CTRL_EARLY_BRESP
);
702 if (aux
& (L310_AUX_CTRL_DATA_PREFETCH
| L310_AUX_CTRL_INSTR_PREFETCH
)) {
703 u32 prefetch
= readl_relaxed(base
+ L310_PREFETCH_CTRL
);
705 pr_info("L2C-310 %s%s prefetch enabled, offset %u lines\n",
706 aux
& L310_AUX_CTRL_INSTR_PREFETCH
? "I" : "",
707 aux
& L310_AUX_CTRL_DATA_PREFETCH
? "D" : "",
708 1 + (prefetch
& L310_PREFETCH_CTRL_OFFSET_MASK
));
711 /* r3p0 or later has power control register */
712 if (rev
>= L310_CACHE_ID_RTL_R3P0
) {
715 l2c_write_sec(L310_DYNAMIC_CLK_GATING_EN
| L310_STNDBY_MODE_EN
,
716 base
, L310_POWER_CTRL
);
717 power_ctrl
= readl_relaxed(base
+ L310_POWER_CTRL
);
718 pr_info("L2C-310 dynamic clock gating %sabled, standby mode %sabled\n",
719 power_ctrl
& L310_DYNAMIC_CLK_GATING_EN
? "en" : "dis",
720 power_ctrl
& L310_STNDBY_MODE_EN
? "en" : "dis");
724 * Always enable non-secure access to the lockdown registers -
725 * we write to them as part of the L2C enable sequence so they
726 * need to be accessible.
728 aux
|= L310_AUX_CTRL_NS_LOCKDOWN
;
730 l2c_enable(base
, aux
, num_lock
);
732 if (aux
& L310_AUX_CTRL_FULL_LINE_ZERO
) {
733 set_auxcr(get_auxcr() | BIT(3) | BIT(2) | BIT(1));
734 cpu_notifier(l2c310_cpu_enable_flz
, 0);
738 static void __init
l2c310_fixup(void __iomem
*base
, u32 cache_id
,
739 struct outer_cache_fns
*fns
)
741 unsigned revision
= cache_id
& L2X0_CACHE_ID_RTL_MASK
;
742 const char *errata
[8];
745 if (IS_ENABLED(CONFIG_PL310_ERRATA_588369
) &&
746 revision
< L310_CACHE_ID_RTL_R2P0
&&
747 /* For bcm compatibility */
748 fns
->inv_range
== l2c210_inv_range
) {
749 fns
->inv_range
= l2c310_inv_range_erratum
;
750 fns
->flush_range
= l2c310_flush_range_erratum
;
751 errata
[n
++] = "588369";
754 if (IS_ENABLED(CONFIG_PL310_ERRATA_727915
) &&
755 revision
>= L310_CACHE_ID_RTL_R2P0
&&
756 revision
< L310_CACHE_ID_RTL_R3P1
) {
757 fns
->flush_all
= l2c310_flush_all_erratum
;
758 errata
[n
++] = "727915";
761 if (revision
>= L310_CACHE_ID_RTL_R3P0
&&
762 revision
< L310_CACHE_ID_RTL_R3P2
) {
763 u32 val
= readl_relaxed(base
+ L310_PREFETCH_CTRL
);
764 /* I don't think bit23 is required here... but iMX6 does so */
765 if (val
& (BIT(30) | BIT(23))) {
766 val
&= ~(BIT(30) | BIT(23));
767 l2c_write_sec(val
, base
, L310_PREFETCH_CTRL
);
768 errata
[n
++] = "752271";
772 if (IS_ENABLED(CONFIG_PL310_ERRATA_753970
) &&
773 revision
== L310_CACHE_ID_RTL_R3P0
) {
774 sync_reg_offset
= L2X0_DUMMY_REG
;
775 errata
[n
++] = "753970";
778 if (IS_ENABLED(CONFIG_PL310_ERRATA_769419
))
779 errata
[n
++] = "769419";
784 pr_info("L2C-310 errat%s", n
> 1 ? "a" : "um");
785 for (i
= 0; i
< n
; i
++)
786 pr_cont(" %s", errata
[i
]);
787 pr_cont(" enabled\n");
791 static void l2c310_disable(void)
794 * If full-line-of-zeros is enabled, we must first disable it in the
795 * Cortex-A9 auxiliary control register before disabling the L2 cache.
797 if (l2x0_saved_regs
.aux_ctrl
& L310_AUX_CTRL_FULL_LINE_ZERO
)
798 set_auxcr(get_auxcr() & ~(BIT(3) | BIT(2) | BIT(1)));
803 static const struct l2c_init_data l2c310_init_fns __initconst
= {
807 .enable
= l2c310_enable
,
808 .fixup
= l2c310_fixup
,
811 .inv_range
= l2c210_inv_range
,
812 .clean_range
= l2c210_clean_range
,
813 .flush_range
= l2c210_flush_range
,
814 .flush_all
= l2c210_flush_all
,
815 .disable
= l2c310_disable
,
817 .resume
= l2c310_resume
,
821 static void __init
__l2c_init(const struct l2c_init_data
*data
,
822 u32 aux_val
, u32 aux_mask
, u32 cache_id
)
824 struct outer_cache_fns fns
;
825 unsigned way_size_bits
, ways
;
829 * Sanity check the aux values. aux_mask is the bits we preserve
830 * from reading the hardware register, and aux_val is the bits we
833 if (aux_val
& aux_mask
)
834 pr_alert("L2C: platform provided aux values permit register corruption.\n");
836 old_aux
= aux
= readl_relaxed(l2x0_base
+ L2X0_AUX_CTRL
);
841 pr_warn("L2C: DT/platform modifies aux control register: 0x%08x -> 0x%08x\n",
844 /* Determine the number of ways */
845 switch (cache_id
& L2X0_CACHE_ID_PART_MASK
) {
846 case L2X0_CACHE_ID_PART_L310
:
847 if ((aux_val
| ~aux_mask
) & (L2C_AUX_CTRL_WAY_SIZE_MASK
| L310_AUX_CTRL_ASSOCIATIVITY_16
))
848 pr_warn("L2C: DT/platform tries to modify or specify cache size\n");
855 case L2X0_CACHE_ID_PART_L210
:
856 case L2X0_CACHE_ID_PART_L220
:
857 ways
= (aux
>> 13) & 0xf;
860 case AURORA_CACHE_ID
:
861 ways
= (aux
>> 13) & 0xf;
862 ways
= 2 << ((ways
+ 1) >> 2);
866 /* Assume unknown chips have 8 ways */
871 l2x0_way_mask
= (1 << ways
) - 1;
874 * way_size_0 is the size that a way_size value of zero would be
875 * given the calculation: way_size = way_size_0 << way_size_bits.
876 * So, if way_size_bits=0 is reserved, but way_size_bits=1 is 16k,
877 * then way_size_0 would be 8k.
879 * L2 cache size = number of ways * way size.
881 way_size_bits
= (aux
& L2C_AUX_CTRL_WAY_SIZE_MASK
) >>
882 L2C_AUX_CTRL_WAY_SIZE_SHIFT
;
883 l2x0_size
= ways
* (data
->way_size_0
<< way_size_bits
);
885 fns
= data
->outer_cache
;
886 fns
.write_sec
= outer_cache
.write_sec
;
888 data
->fixup(l2x0_base
, cache_id
, &fns
);
891 * Check if l2x0 controller is already enabled. If we are booting
892 * in non-secure mode accessing the below registers will fault.
894 if (!(readl_relaxed(l2x0_base
+ L2X0_CTRL
) & L2X0_CTRL_EN
))
895 data
->enable(l2x0_base
, aux
, data
->num_lock
);
900 * It is strange to save the register state before initialisation,
901 * but hey, this is what the DT implementations decided to do.
904 data
->save(l2x0_base
);
906 /* Re-read it in case some bits are reserved. */
907 aux
= readl_relaxed(l2x0_base
+ L2X0_AUX_CTRL
);
909 pr_info("%s cache controller enabled, %d ways, %d kB\n",
910 data
->type
, ways
, l2x0_size
>> 10);
911 pr_info("%s: CACHE_ID 0x%08x, AUX_CTRL 0x%08x\n",
912 data
->type
, cache_id
, aux
);
915 void __init
l2x0_init(void __iomem
*base
, u32 aux_val
, u32 aux_mask
)
917 const struct l2c_init_data
*data
;
922 cache_id
= readl_relaxed(base
+ L2X0_CACHE_ID
);
924 switch (cache_id
& L2X0_CACHE_ID_PART_MASK
) {
926 case L2X0_CACHE_ID_PART_L210
:
930 case L2X0_CACHE_ID_PART_L220
:
934 case L2X0_CACHE_ID_PART_L310
:
935 data
= &l2c310_init_fns
;
939 __l2c_init(data
, aux_val
, aux_mask
, cache_id
);
943 static int l2_wt_override
;
945 /* Aurora don't have the cache ID register available, so we have to
946 * pass it though the device tree */
947 static u32 cache_id_part_number_from_dt
;
950 * l2x0_cache_size_of_parse() - read cache size parameters from DT
951 * @np: the device tree node for the l2 cache
952 * @aux_val: pointer to machine-supplied auxilary register value, to
953 * be augmented by the call (bits to be set to 1)
954 * @aux_mask: pointer to machine-supplied auxilary register mask, to
955 * be augmented by the call (bits to be set to 0)
956 * @associativity: variable to return the calculated associativity in
957 * @max_way_size: the maximum size in bytes for the cache ways
959 static int __init
l2x0_cache_size_of_parse(const struct device_node
*np
,
960 u32
*aux_val
, u32
*aux_mask
,
964 u32 mask
= 0, val
= 0;
965 u32 cache_size
= 0, sets
= 0;
966 u32 way_size_bits
= 1;
971 of_property_read_u32(np
, "cache-size", &cache_size
);
972 of_property_read_u32(np
, "cache-sets", &sets
);
973 of_property_read_u32(np
, "cache-block-size", &block_size
);
974 of_property_read_u32(np
, "cache-line-size", &line_size
);
976 if (!cache_size
|| !sets
)
979 /* All these l2 caches have the same line = block size actually */
982 /* If linesize if not given, it is equal to blocksize */
983 line_size
= block_size
;
985 /* Fall back to known size */
986 pr_warn("L2C OF: no cache block/line size given: "
987 "falling back to default size %d bytes\n",
989 line_size
= CACHE_LINE_SIZE
;
993 if (line_size
!= CACHE_LINE_SIZE
)
994 pr_warn("L2C OF: DT supplied line size %d bytes does "
995 "not match hardware line size of %d bytes\n",
1001 * set size = cache size / sets
1002 * ways = cache size / (sets * line size)
1003 * way size = cache size / (cache size / (sets * line size))
1004 * way size = sets * line size
1005 * associativity = ways = cache size / way size
1007 way_size
= sets
* line_size
;
1008 *associativity
= cache_size
/ way_size
;
1010 if (way_size
> max_way_size
) {
1011 pr_err("L2C OF: set size %dKB is too large\n", way_size
);
1015 pr_info("L2C OF: override cache size: %d bytes (%dKB)\n",
1016 cache_size
, cache_size
>> 10);
1017 pr_info("L2C OF: override line size: %d bytes\n", line_size
);
1018 pr_info("L2C OF: override way size: %d bytes (%dKB)\n",
1019 way_size
, way_size
>> 10);
1020 pr_info("L2C OF: override associativity: %d\n", *associativity
);
1023 * Calculates the bits 17:19 to set for way size:
1024 * 512KB -> 6, 256KB -> 5, ... 16KB -> 1
1026 way_size_bits
= ilog2(way_size
>> 10) - 3;
1027 if (way_size_bits
< 1 || way_size_bits
> 6) {
1028 pr_err("L2C OF: cache way size illegal: %dKB is not mapped\n",
1033 mask
|= L2C_AUX_CTRL_WAY_SIZE_MASK
;
1034 val
|= (way_size_bits
<< L2C_AUX_CTRL_WAY_SIZE_SHIFT
);
1043 static void __init
l2x0_of_parse(const struct device_node
*np
,
1044 u32
*aux_val
, u32
*aux_mask
)
1046 u32 data
[2] = { 0, 0 };
1049 u32 val
= 0, mask
= 0;
1053 of_property_read_u32(np
, "arm,tag-latency", &tag
);
1055 mask
|= L2X0_AUX_CTRL_TAG_LATENCY_MASK
;
1056 val
|= (tag
- 1) << L2X0_AUX_CTRL_TAG_LATENCY_SHIFT
;
1059 of_property_read_u32_array(np
, "arm,data-latency",
1060 data
, ARRAY_SIZE(data
));
1061 if (data
[0] && data
[1]) {
1062 mask
|= L2X0_AUX_CTRL_DATA_RD_LATENCY_MASK
|
1063 L2X0_AUX_CTRL_DATA_WR_LATENCY_MASK
;
1064 val
|= ((data
[0] - 1) << L2X0_AUX_CTRL_DATA_RD_LATENCY_SHIFT
) |
1065 ((data
[1] - 1) << L2X0_AUX_CTRL_DATA_WR_LATENCY_SHIFT
);
1068 of_property_read_u32(np
, "arm,dirty-latency", &dirty
);
1070 mask
|= L2X0_AUX_CTRL_DIRTY_LATENCY_MASK
;
1071 val
|= (dirty
- 1) << L2X0_AUX_CTRL_DIRTY_LATENCY_SHIFT
;
1074 ret
= l2x0_cache_size_of_parse(np
, aux_val
, aux_mask
, &assoc
, SZ_256K
);
1079 pr_err("l2x0 of: cache setting yield too high associativity\n");
1080 pr_err("l2x0 of: %d calculated, max 8\n", assoc
);
1082 mask
|= L2X0_AUX_CTRL_ASSOC_MASK
;
1083 val
|= (assoc
<< L2X0_AUX_CTRL_ASSOC_SHIFT
);
1091 static const struct l2c_init_data of_l2c210_data __initconst
= {
1093 .way_size_0
= SZ_8K
,
1095 .of_parse
= l2x0_of_parse
,
1096 .enable
= l2c_enable
,
1099 .inv_range
= l2c210_inv_range
,
1100 .clean_range
= l2c210_clean_range
,
1101 .flush_range
= l2c210_flush_range
,
1102 .flush_all
= l2c210_flush_all
,
1103 .disable
= l2c_disable
,
1104 .sync
= l2c210_sync
,
1105 .resume
= l2c210_resume
,
1109 static const struct l2c_init_data of_l2c220_data __initconst
= {
1111 .way_size_0
= SZ_8K
,
1113 .of_parse
= l2x0_of_parse
,
1114 .enable
= l2c220_enable
,
1117 .inv_range
= l2c220_inv_range
,
1118 .clean_range
= l2c220_clean_range
,
1119 .flush_range
= l2c220_flush_range
,
1120 .flush_all
= l2c220_flush_all
,
1121 .disable
= l2c_disable
,
1122 .sync
= l2c220_sync
,
1123 .resume
= l2c210_resume
,
1127 static void __init
l2c310_of_parse(const struct device_node
*np
,
1128 u32
*aux_val
, u32
*aux_mask
)
1130 u32 data
[3] = { 0, 0, 0 };
1131 u32 tag
[3] = { 0, 0, 0 };
1132 u32 filter
[2] = { 0, 0 };
1136 of_property_read_u32_array(np
, "arm,tag-latency", tag
, ARRAY_SIZE(tag
));
1137 if (tag
[0] && tag
[1] && tag
[2])
1139 L310_LATENCY_CTRL_RD(tag
[0] - 1) |
1140 L310_LATENCY_CTRL_WR(tag
[1] - 1) |
1141 L310_LATENCY_CTRL_SETUP(tag
[2] - 1),
1142 l2x0_base
+ L310_TAG_LATENCY_CTRL
);
1144 of_property_read_u32_array(np
, "arm,data-latency",
1145 data
, ARRAY_SIZE(data
));
1146 if (data
[0] && data
[1] && data
[2])
1148 L310_LATENCY_CTRL_RD(data
[0] - 1) |
1149 L310_LATENCY_CTRL_WR(data
[1] - 1) |
1150 L310_LATENCY_CTRL_SETUP(data
[2] - 1),
1151 l2x0_base
+ L310_DATA_LATENCY_CTRL
);
1153 of_property_read_u32_array(np
, "arm,filter-ranges",
1154 filter
, ARRAY_SIZE(filter
));
1156 writel_relaxed(ALIGN(filter
[0] + filter
[1], SZ_1M
),
1157 l2x0_base
+ L310_ADDR_FILTER_END
);
1158 writel_relaxed((filter
[0] & ~(SZ_1M
- 1)) | L310_ADDR_FILTER_EN
,
1159 l2x0_base
+ L310_ADDR_FILTER_START
);
1162 ret
= l2x0_cache_size_of_parse(np
, aux_val
, aux_mask
, &assoc
, SZ_512K
);
1168 *aux_val
&= ~L2X0_AUX_CTRL_ASSOC_MASK
;
1169 *aux_val
|= L310_AUX_CTRL_ASSOCIATIVITY_16
;
1170 *aux_mask
&= ~L2X0_AUX_CTRL_ASSOC_MASK
;
1173 *aux_val
&= ~L2X0_AUX_CTRL_ASSOC_MASK
;
1174 *aux_mask
&= ~L2X0_AUX_CTRL_ASSOC_MASK
;
1177 pr_err("L2C-310 OF cache associativity %d invalid, only 8 or 16 permitted\n",
1183 static const struct l2c_init_data of_l2c310_data __initconst
= {
1185 .way_size_0
= SZ_8K
,
1187 .of_parse
= l2c310_of_parse
,
1188 .enable
= l2c310_enable
,
1189 .fixup
= l2c310_fixup
,
1190 .save
= l2c310_save
,
1192 .inv_range
= l2c210_inv_range
,
1193 .clean_range
= l2c210_clean_range
,
1194 .flush_range
= l2c210_flush_range
,
1195 .flush_all
= l2c210_flush_all
,
1196 .disable
= l2c310_disable
,
1197 .sync
= l2c210_sync
,
1198 .resume
= l2c310_resume
,
1203 * This is a variant of the of_l2c310_data with .sync set to
1204 * NULL. Outer sync operations are not needed when the system is I/O
1205 * coherent, and potentially harmful in certain situations (PCIe/PL310
1206 * deadlock on Armada 375/38x due to hardware I/O coherency). The
1207 * other operations are kept because they are infrequent (therefore do
1208 * not cause the deadlock in practice) and needed for secondary CPU
1209 * boot and other power management activities.
1211 static const struct l2c_init_data of_l2c310_coherent_data __initconst
= {
1212 .type
= "L2C-310 Coherent",
1213 .way_size_0
= SZ_8K
,
1215 .of_parse
= l2c310_of_parse
,
1216 .enable
= l2c310_enable
,
1217 .fixup
= l2c310_fixup
,
1218 .save
= l2c310_save
,
1220 .inv_range
= l2c210_inv_range
,
1221 .clean_range
= l2c210_clean_range
,
1222 .flush_range
= l2c210_flush_range
,
1223 .flush_all
= l2c210_flush_all
,
1224 .disable
= l2c310_disable
,
1225 .resume
= l2c310_resume
,
1230 * Note that the end addresses passed to Linux primitives are
1231 * noninclusive, while the hardware cache range operations use
1232 * inclusive start and end addresses.
1234 static unsigned long calc_range_end(unsigned long start
, unsigned long end
)
1237 * Limit the number of cache lines processed at once,
1238 * since cache range operations stall the CPU pipeline
1241 if (end
> start
+ MAX_RANGE_SIZE
)
1242 end
= start
+ MAX_RANGE_SIZE
;
1245 * Cache range operations can't straddle a page boundary.
1247 if (end
> PAGE_ALIGN(start
+1))
1248 end
= PAGE_ALIGN(start
+1);
1254 * Make sure 'start' and 'end' reference the same page, as L2 is PIPT
1255 * and range operations only do a TLB lookup on the start address.
1257 static void aurora_pa_range(unsigned long start
, unsigned long end
,
1258 unsigned long offset
)
1260 unsigned long flags
;
1262 raw_spin_lock_irqsave(&l2x0_lock
, flags
);
1263 writel_relaxed(start
, l2x0_base
+ AURORA_RANGE_BASE_ADDR_REG
);
1264 writel_relaxed(end
, l2x0_base
+ offset
);
1265 raw_spin_unlock_irqrestore(&l2x0_lock
, flags
);
1270 static void aurora_inv_range(unsigned long start
, unsigned long end
)
1273 * round start and end adresses up to cache line size
1275 start
&= ~(CACHE_LINE_SIZE
- 1);
1276 end
= ALIGN(end
, CACHE_LINE_SIZE
);
1279 * Invalidate all full cache lines between 'start' and 'end'.
1281 while (start
< end
) {
1282 unsigned long range_end
= calc_range_end(start
, end
);
1283 aurora_pa_range(start
, range_end
- CACHE_LINE_SIZE
,
1284 AURORA_INVAL_RANGE_REG
);
1289 static void aurora_clean_range(unsigned long start
, unsigned long end
)
1292 * If L2 is forced to WT, the L2 will always be clean and we
1293 * don't need to do anything here.
1295 if (!l2_wt_override
) {
1296 start
&= ~(CACHE_LINE_SIZE
- 1);
1297 end
= ALIGN(end
, CACHE_LINE_SIZE
);
1298 while (start
!= end
) {
1299 unsigned long range_end
= calc_range_end(start
, end
);
1300 aurora_pa_range(start
, range_end
- CACHE_LINE_SIZE
,
1301 AURORA_CLEAN_RANGE_REG
);
1307 static void aurora_flush_range(unsigned long start
, unsigned long end
)
1309 start
&= ~(CACHE_LINE_SIZE
- 1);
1310 end
= ALIGN(end
, CACHE_LINE_SIZE
);
1311 while (start
!= end
) {
1312 unsigned long range_end
= calc_range_end(start
, end
);
1314 * If L2 is forced to WT, the L2 will always be clean and we
1315 * just need to invalidate.
1318 aurora_pa_range(start
, range_end
- CACHE_LINE_SIZE
,
1319 AURORA_INVAL_RANGE_REG
);
1321 aurora_pa_range(start
, range_end
- CACHE_LINE_SIZE
,
1322 AURORA_FLUSH_RANGE_REG
);
1327 static void aurora_save(void __iomem
*base
)
1329 l2x0_saved_regs
.ctrl
= readl_relaxed(base
+ L2X0_CTRL
);
1330 l2x0_saved_regs
.aux_ctrl
= readl_relaxed(base
+ L2X0_AUX_CTRL
);
1333 static void aurora_resume(void)
1335 void __iomem
*base
= l2x0_base
;
1337 if (!(readl(base
+ L2X0_CTRL
) & L2X0_CTRL_EN
)) {
1338 writel_relaxed(l2x0_saved_regs
.aux_ctrl
, base
+ L2X0_AUX_CTRL
);
1339 writel_relaxed(l2x0_saved_regs
.ctrl
, base
+ L2X0_CTRL
);
1344 * For Aurora cache in no outer mode, enable via the CP15 coprocessor
1345 * broadcasting of cache commands to L2.
1347 static void __init
aurora_enable_no_outer(void __iomem
*base
, u32 aux
,
1352 asm volatile("mrc p15, 1, %0, c15, c2, 0" : "=r" (u
));
1353 u
|= AURORA_CTRL_FW
; /* Set the FW bit */
1354 asm volatile("mcr p15, 1, %0, c15, c2, 0" : : "r" (u
));
1358 l2c_enable(base
, aux
, num_lock
);
1361 static void __init
aurora_fixup(void __iomem
*base
, u32 cache_id
,
1362 struct outer_cache_fns
*fns
)
1364 sync_reg_offset
= AURORA_SYNC_REG
;
1367 static void __init
aurora_of_parse(const struct device_node
*np
,
1368 u32
*aux_val
, u32
*aux_mask
)
1370 u32 val
= AURORA_ACR_REPLACEMENT_TYPE_SEMIPLRU
;
1371 u32 mask
= AURORA_ACR_REPLACEMENT_MASK
;
1373 of_property_read_u32(np
, "cache-id-part",
1374 &cache_id_part_number_from_dt
);
1376 /* Determine and save the write policy */
1377 l2_wt_override
= of_property_read_bool(np
, "wt-override");
1379 if (l2_wt_override
) {
1380 val
|= AURORA_ACR_FORCE_WRITE_THRO_POLICY
;
1381 mask
|= AURORA_ACR_FORCE_WRITE_POLICY_MASK
;
1389 static const struct l2c_init_data of_aurora_with_outer_data __initconst
= {
1391 .way_size_0
= SZ_4K
,
1393 .of_parse
= aurora_of_parse
,
1394 .enable
= l2c_enable
,
1395 .fixup
= aurora_fixup
,
1396 .save
= aurora_save
,
1398 .inv_range
= aurora_inv_range
,
1399 .clean_range
= aurora_clean_range
,
1400 .flush_range
= aurora_flush_range
,
1401 .flush_all
= l2x0_flush_all
,
1402 .disable
= l2x0_disable
,
1403 .sync
= l2x0_cache_sync
,
1404 .resume
= aurora_resume
,
1408 static const struct l2c_init_data of_aurora_no_outer_data __initconst
= {
1410 .way_size_0
= SZ_4K
,
1412 .of_parse
= aurora_of_parse
,
1413 .enable
= aurora_enable_no_outer
,
1414 .fixup
= aurora_fixup
,
1415 .save
= aurora_save
,
1417 .resume
= aurora_resume
,
1422 * For certain Broadcom SoCs, depending on the address range, different offsets
1423 * need to be added to the address before passing it to L2 for
1424 * invalidation/clean/flush
1426 * Section Address Range Offset EMI
1427 * 1 0x00000000 - 0x3FFFFFFF 0x80000000 VC
1428 * 2 0x40000000 - 0xBFFFFFFF 0x40000000 SYS
1429 * 3 0xC0000000 - 0xFFFFFFFF 0x80000000 VC
1431 * When the start and end addresses have crossed two different sections, we
1432 * need to break the L2 operation into two, each within its own section.
1433 * For example, if we need to invalidate addresses starts at 0xBFFF0000 and
1434 * ends at 0xC0001000, we need do invalidate 1) 0xBFFF0000 - 0xBFFFFFFF and 2)
1435 * 0xC0000000 - 0xC0001000
1438 * By breaking a single L2 operation into two, we may potentially suffer some
1439 * performance hit, but keep in mind the cross section case is very rare
1442 * We do not need to handle the case when the start address is in
1443 * Section 1 and the end address is in Section 3, since it is not a valid use
1447 * Section 1 in practical terms can no longer be used on rev A2. Because of
1448 * that the code does not need to handle section 1 at all.
1451 #define BCM_SYS_EMI_START_ADDR 0x40000000UL
1452 #define BCM_VC_EMI_SEC3_START_ADDR 0xC0000000UL
1454 #define BCM_SYS_EMI_OFFSET 0x40000000UL
1455 #define BCM_VC_EMI_OFFSET 0x80000000UL
1457 static inline int bcm_addr_is_sys_emi(unsigned long addr
)
1459 return (addr
>= BCM_SYS_EMI_START_ADDR
) &&
1460 (addr
< BCM_VC_EMI_SEC3_START_ADDR
);
1463 static inline unsigned long bcm_l2_phys_addr(unsigned long addr
)
1465 if (bcm_addr_is_sys_emi(addr
))
1466 return addr
+ BCM_SYS_EMI_OFFSET
;
1468 return addr
+ BCM_VC_EMI_OFFSET
;
1471 static void bcm_inv_range(unsigned long start
, unsigned long end
)
1473 unsigned long new_start
, new_end
;
1475 BUG_ON(start
< BCM_SYS_EMI_START_ADDR
);
1477 if (unlikely(end
<= start
))
1480 new_start
= bcm_l2_phys_addr(start
);
1481 new_end
= bcm_l2_phys_addr(end
);
1483 /* normal case, no cross section between start and end */
1484 if (likely(bcm_addr_is_sys_emi(end
) || !bcm_addr_is_sys_emi(start
))) {
1485 l2c210_inv_range(new_start
, new_end
);
1489 /* They cross sections, so it can only be a cross from section
1492 l2c210_inv_range(new_start
,
1493 bcm_l2_phys_addr(BCM_VC_EMI_SEC3_START_ADDR
-1));
1494 l2c210_inv_range(bcm_l2_phys_addr(BCM_VC_EMI_SEC3_START_ADDR
),
1498 static void bcm_clean_range(unsigned long start
, unsigned long end
)
1500 unsigned long new_start
, new_end
;
1502 BUG_ON(start
< BCM_SYS_EMI_START_ADDR
);
1504 if (unlikely(end
<= start
))
1507 new_start
= bcm_l2_phys_addr(start
);
1508 new_end
= bcm_l2_phys_addr(end
);
1510 /* normal case, no cross section between start and end */
1511 if (likely(bcm_addr_is_sys_emi(end
) || !bcm_addr_is_sys_emi(start
))) {
1512 l2c210_clean_range(new_start
, new_end
);
1516 /* They cross sections, so it can only be a cross from section
1519 l2c210_clean_range(new_start
,
1520 bcm_l2_phys_addr(BCM_VC_EMI_SEC3_START_ADDR
-1));
1521 l2c210_clean_range(bcm_l2_phys_addr(BCM_VC_EMI_SEC3_START_ADDR
),
1525 static void bcm_flush_range(unsigned long start
, unsigned long end
)
1527 unsigned long new_start
, new_end
;
1529 BUG_ON(start
< BCM_SYS_EMI_START_ADDR
);
1531 if (unlikely(end
<= start
))
1534 if ((end
- start
) >= l2x0_size
) {
1535 outer_cache
.flush_all();
1539 new_start
= bcm_l2_phys_addr(start
);
1540 new_end
= bcm_l2_phys_addr(end
);
1542 /* normal case, no cross section between start and end */
1543 if (likely(bcm_addr_is_sys_emi(end
) || !bcm_addr_is_sys_emi(start
))) {
1544 l2c210_flush_range(new_start
, new_end
);
1548 /* They cross sections, so it can only be a cross from section
1551 l2c210_flush_range(new_start
,
1552 bcm_l2_phys_addr(BCM_VC_EMI_SEC3_START_ADDR
-1));
1553 l2c210_flush_range(bcm_l2_phys_addr(BCM_VC_EMI_SEC3_START_ADDR
),
1557 /* Broadcom L2C-310 start from ARMs R3P2 or later, and require no fixups */
1558 static const struct l2c_init_data of_bcm_l2x0_data __initconst
= {
1559 .type
= "BCM-L2C-310",
1560 .way_size_0
= SZ_8K
,
1562 .of_parse
= l2c310_of_parse
,
1563 .enable
= l2c310_enable
,
1564 .save
= l2c310_save
,
1566 .inv_range
= bcm_inv_range
,
1567 .clean_range
= bcm_clean_range
,
1568 .flush_range
= bcm_flush_range
,
1569 .flush_all
= l2c210_flush_all
,
1570 .disable
= l2c310_disable
,
1571 .sync
= l2c210_sync
,
1572 .resume
= l2c310_resume
,
1576 static void __init
tauros3_save(void __iomem
*base
)
1580 l2x0_saved_regs
.aux2_ctrl
=
1581 readl_relaxed(base
+ TAUROS3_AUX2_CTRL
);
1582 l2x0_saved_regs
.prefetch_ctrl
=
1583 readl_relaxed(base
+ L310_PREFETCH_CTRL
);
1586 static void tauros3_resume(void)
1588 void __iomem
*base
= l2x0_base
;
1590 if (!(readl_relaxed(base
+ L2X0_CTRL
) & L2X0_CTRL_EN
)) {
1591 writel_relaxed(l2x0_saved_regs
.aux2_ctrl
,
1592 base
+ TAUROS3_AUX2_CTRL
);
1593 writel_relaxed(l2x0_saved_regs
.prefetch_ctrl
,
1594 base
+ L310_PREFETCH_CTRL
);
1596 l2c_enable(base
, l2x0_saved_regs
.aux_ctrl
, 8);
1600 static const struct l2c_init_data of_tauros3_data __initconst
= {
1602 .way_size_0
= SZ_8K
,
1604 .enable
= l2c_enable
,
1605 .save
= tauros3_save
,
1606 /* Tauros3 broadcasts L1 cache operations to L2 */
1608 .resume
= tauros3_resume
,
1612 #define L2C_ID(name, fns) { .compatible = name, .data = (void *)&fns }
1613 static const struct of_device_id l2x0_ids
[] __initconst
= {
1614 L2C_ID("arm,l210-cache", of_l2c210_data
),
1615 L2C_ID("arm,l220-cache", of_l2c220_data
),
1616 L2C_ID("arm,pl310-cache", of_l2c310_data
),
1617 L2C_ID("brcm,bcm11351-a2-pl310-cache", of_bcm_l2x0_data
),
1618 L2C_ID("marvell,aurora-outer-cache", of_aurora_with_outer_data
),
1619 L2C_ID("marvell,aurora-system-cache", of_aurora_no_outer_data
),
1620 L2C_ID("marvell,tauros3-cache", of_tauros3_data
),
1621 /* Deprecated IDs */
1622 L2C_ID("bcm,bcm11351-a2-pl310-cache", of_bcm_l2x0_data
),
1626 int __init
l2x0_of_init(u32 aux_val
, u32 aux_mask
)
1628 const struct l2c_init_data
*data
;
1629 struct device_node
*np
;
1630 struct resource res
;
1631 u32 cache_id
, old_aux
;
1633 np
= of_find_matching_node(NULL
, l2x0_ids
);
1637 if (of_address_to_resource(np
, 0, &res
))
1640 l2x0_base
= ioremap(res
.start
, resource_size(&res
));
1644 l2x0_saved_regs
.phy_base
= res
.start
;
1646 data
= of_match_node(l2x0_ids
, np
)->data
;
1648 if (of_device_is_compatible(np
, "arm,pl310-cache") &&
1649 of_property_read_bool(np
, "arm,io-coherent"))
1650 data
= &of_l2c310_coherent_data
;
1652 old_aux
= readl_relaxed(l2x0_base
+ L2X0_AUX_CTRL
);
1653 if (old_aux
!= ((old_aux
& aux_mask
) | aux_val
)) {
1654 pr_warn("L2C: platform modifies aux control register: 0x%08x -> 0x%08x\n",
1655 old_aux
, (old_aux
& aux_mask
) | aux_val
);
1656 } else if (aux_mask
!= ~0U && aux_val
!= 0) {
1657 pr_alert("L2C: platform provided aux values match the hardware, so have no effect. Please remove them.\n");
1660 /* All L2 caches are unified, so this property should be specified */
1661 if (!of_property_read_bool(np
, "cache-unified"))
1662 pr_err("L2C: device tree omits to specify unified cache\n");
1664 /* L2 configuration can only be changed if the cache is disabled */
1665 if (!(readl_relaxed(l2x0_base
+ L2X0_CTRL
) & L2X0_CTRL_EN
))
1667 data
->of_parse(np
, &aux_val
, &aux_mask
);
1669 if (cache_id_part_number_from_dt
)
1670 cache_id
= cache_id_part_number_from_dt
;
1672 cache_id
= readl_relaxed(l2x0_base
+ L2X0_CACHE_ID
);
1674 __l2c_init(data
, aux_val
, aux_mask
, cache_id
);