Commit | Line | Data |
---|---|---|
f1a0c4aa CM |
1 | /* |
2 | * Cache maintenance | |
3 | * | |
4 | * Copyright (C) 2001 Deep Blue Solutions Ltd. | |
5 | * Copyright (C) 2012 ARM Ltd. | |
6 | * | |
7 | * This program is free software; you can redistribute it and/or modify | |
8 | * it under the terms of the GNU General Public License version 2 as | |
9 | * published by the Free Software Foundation. | |
10 | * | |
11 | * This program is distributed in the hope that it will be useful, | |
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
14 | * GNU General Public License for more details. | |
15 | * | |
16 | * You should have received a copy of the GNU General Public License | |
17 | * along with this program. If not, see <http://www.gnu.org/licenses/>. | |
18 | */ | |
19 | ||
a2d25a53 | 20 | #include <linux/errno.h> |
f1a0c4aa CM |
21 | #include <linux/linkage.h> |
22 | #include <linux/init.h> | |
23 | #include <asm/assembler.h> | |
301bcfac AP |
24 | #include <asm/cpufeature.h> |
25 | #include <asm/alternative-asm.h> | |
f1a0c4aa CM |
26 | |
27 | #include "proc-macros.S" | |
28 | ||
29 | /* | |
30 | * __flush_dcache_all() | |
31 | * | |
32 | * Flush the whole D-cache. | |
33 | * | |
34 | * Corrupted registers: x0-x7, x9-x11 | |
35 | */ | |
bff70595 | 36 | __flush_dcache_all: |
dc60b777 | 37 | dmb sy // ensure ordering with previous memory accesses |
f1a0c4aa CM |
38 | mrs x0, clidr_el1 // read clidr |
39 | and x3, x0, #0x7000000 // extract loc from clidr | |
40 | lsr x3, x3, #23 // left align loc bit field | |
41 | cbz x3, finished // if loc is 0, then no need to clean | |
42 | mov x10, #0 // start clean at cache level 0 | |
43 | loop1: | |
44 | add x2, x10, x10, lsr #1 // work out 3x current cache level | |
45 | lsr x1, x0, x2 // extract cache type bits from clidr | |
46 | and x1, x1, #7 // mask of the bits for current cache only | |
47 | cmp x1, #2 // see what cache we have at this level | |
48 | b.lt skip // skip if no cache, or just i-cache | |
49 | save_and_disable_irqs x9 // make CSSELR and CCSIDR access atomic | |
50 | msr csselr_el1, x10 // select current cache level in csselr | |
51 | isb // isb to sych the new cssr&csidr | |
52 | mrs x1, ccsidr_el1 // read the new ccsidr | |
53 | restore_irqs x9 | |
54 | and x2, x1, #7 // extract the length of the cache lines | |
55 | add x2, x2, #4 // add 4 (line length offset) | |
56 | mov x4, #0x3ff | |
57 | and x4, x4, x1, lsr #3 // find maximum number on the way size | |
b4fed079 | 58 | clz w5, w4 // find bit position of way size increment |
f1a0c4aa CM |
59 | mov x7, #0x7fff |
60 | and x7, x7, x1, lsr #13 // extract max number of the index size | |
61 | loop2: | |
62 | mov x9, x4 // create working copy of max way size | |
63 | loop3: | |
64 | lsl x6, x9, x5 | |
65 | orr x11, x10, x6 // factor way and cache number into x11 | |
66 | lsl x6, x7, x2 | |
67 | orr x11, x11, x6 // factor index number into x11 | |
68 | dc cisw, x11 // clean & invalidate by set/way | |
69 | subs x9, x9, #1 // decrement the way | |
70 | b.ge loop3 | |
71 | subs x7, x7, #1 // decrement the index | |
72 | b.ge loop2 | |
73 | skip: | |
74 | add x10, x10, #2 // increment cache number | |
75 | cmp x3, x10 | |
76 | b.gt loop1 | |
77 | finished: | |
78 | mov x10, #0 // swith back to cache level 0 | |
79 | msr csselr_el1, x10 // select current cache level in csselr | |
80 | dsb sy | |
81 | isb | |
82 | ret | |
83 | ENDPROC(__flush_dcache_all) | |
84 | ||
85 | /* | |
86 | * flush_cache_all() | |
87 | * | |
88 | * Flush the entire cache system. The data cache flush is now achieved | |
89 | * using atomic clean / invalidates working outwards from L1 cache. This | |
90 | * is done using Set/Way based cache maintainance instructions. The | |
91 | * instruction cache can still be invalidated back to the point of | |
92 | * unification in a single instruction. | |
93 | */ | |
94 | ENTRY(flush_cache_all) | |
95 | mov x12, lr | |
96 | bl __flush_dcache_all | |
97 | mov x0, #0 | |
98 | ic ialluis // I+BTB cache invalidate | |
99 | ret x12 | |
100 | ENDPROC(flush_cache_all) | |
101 | ||
102 | /* | |
103 | * flush_icache_range(start,end) | |
104 | * | |
105 | * Ensure that the I and D caches are coherent within specified region. | |
106 | * This is typically used when code has been written to a memory region, | |
107 | * and will be executed. | |
108 | * | |
109 | * - start - virtual start address of region | |
110 | * - end - virtual end address of region | |
111 | */ | |
112 | ENTRY(flush_icache_range) | |
113 | /* FALLTHROUGH */ | |
114 | ||
115 | /* | |
116 | * __flush_cache_user_range(start,end) | |
117 | * | |
118 | * Ensure that the I and D caches are coherent within specified region. | |
119 | * This is typically used when code has been written to a memory region, | |
120 | * and will be executed. | |
121 | * | |
122 | * - start - virtual start address of region | |
123 | * - end - virtual end address of region | |
124 | */ | |
125 | ENTRY(__flush_cache_user_range) | |
126 | dcache_line_size x2, x3 | |
127 | sub x3, x2, #1 | |
128 | bic x4, x0, x3 | |
129 | 1: | |
130 | USER(9f, dc cvau, x4 ) // clean D line to PoU | |
131 | add x4, x4, x2 | |
132 | cmp x4, x1 | |
133 | b.lo 1b | |
dc60b777 | 134 | dsb ish |
f1a0c4aa CM |
135 | |
136 | icache_line_size x2, x3 | |
137 | sub x3, x2, #1 | |
138 | bic x4, x0, x3 | |
139 | 1: | |
140 | USER(9f, ic ivau, x4 ) // invalidate I line PoU | |
141 | add x4, x4, x2 | |
142 | cmp x4, x1 | |
143 | b.lo 1b | |
dc60b777 | 144 | dsb ish |
f1a0c4aa | 145 | isb |
a2d25a53 VM |
146 | mov x0, #0 |
147 | ret | |
148 | 9: | |
149 | mov x0, #-EFAULT | |
f1a0c4aa CM |
150 | ret |
151 | ENDPROC(flush_icache_range) | |
152 | ENDPROC(__flush_cache_user_range) | |
153 | ||
154 | /* | |
03324e6e | 155 | * __flush_dcache_area(kaddr, size) |
f1a0c4aa CM |
156 | * |
157 | * Ensure that the data held in the page kaddr is written back to the | |
158 | * page in question. | |
159 | * | |
160 | * - kaddr - kernel address | |
161 | * - size - size in question | |
162 | */ | |
163 | ENTRY(__flush_dcache_area) | |
164 | dcache_line_size x2, x3 | |
165 | add x1, x0, x1 | |
166 | sub x3, x2, #1 | |
167 | bic x0, x0, x3 | |
168 | 1: dc civac, x0 // clean & invalidate D line / unified line | |
169 | add x0, x0, x2 | |
170 | cmp x0, x1 | |
171 | b.lo 1b | |
172 | dsb sy | |
173 | ret | |
174 | ENDPROC(__flush_dcache_area) | |
7363590d | 175 | |
c218bca7 CM |
176 | /* |
177 | * __inval_cache_range(start, end) | |
178 | * - start - start address of region | |
179 | * - end - end address of region | |
180 | */ | |
181 | ENTRY(__inval_cache_range) | |
182 | /* FALLTHROUGH */ | |
183 | ||
7363590d CM |
184 | /* |
185 | * __dma_inv_range(start, end) | |
186 | * - start - virtual start address of region | |
187 | * - end - virtual end address of region | |
188 | */ | |
189 | __dma_inv_range: | |
190 | dcache_line_size x2, x3 | |
191 | sub x3, x2, #1 | |
ebf81a93 | 192 | tst x1, x3 // end cache line aligned? |
7363590d | 193 | bic x1, x1, x3 |
ebf81a93 CM |
194 | b.eq 1f |
195 | dc civac, x1 // clean & invalidate D / U line | |
196 | 1: tst x0, x3 // start cache line aligned? | |
197 | bic x0, x0, x3 | |
198 | b.eq 2f | |
199 | dc civac, x0 // clean & invalidate D / U line | |
200 | b 3f | |
201 | 2: dc ivac, x0 // invalidate D / U line | |
202 | 3: add x0, x0, x2 | |
7363590d | 203 | cmp x0, x1 |
ebf81a93 | 204 | b.lo 2b |
7363590d CM |
205 | dsb sy |
206 | ret | |
c218bca7 | 207 | ENDPROC(__inval_cache_range) |
7363590d CM |
208 | ENDPROC(__dma_inv_range) |
209 | ||
210 | /* | |
211 | * __dma_clean_range(start, end) | |
212 | * - start - virtual start address of region | |
213 | * - end - virtual end address of region | |
214 | */ | |
215 | __dma_clean_range: | |
216 | dcache_line_size x2, x3 | |
217 | sub x3, x2, #1 | |
218 | bic x0, x0, x3 | |
301bcfac | 219 | 1: alternative_insn "dc cvac, x0", "dc civac, x0", ARM64_WORKAROUND_CLEAN_CACHE |
7363590d CM |
220 | add x0, x0, x2 |
221 | cmp x0, x1 | |
222 | b.lo 1b | |
223 | dsb sy | |
224 | ret | |
225 | ENDPROC(__dma_clean_range) | |
226 | ||
227 | /* | |
228 | * __dma_flush_range(start, end) | |
229 | * - start - virtual start address of region | |
230 | * - end - virtual end address of region | |
231 | */ | |
232 | ENTRY(__dma_flush_range) | |
233 | dcache_line_size x2, x3 | |
234 | sub x3, x2, #1 | |
235 | bic x0, x0, x3 | |
236 | 1: dc civac, x0 // clean & invalidate D / U line | |
237 | add x0, x0, x2 | |
238 | cmp x0, x1 | |
239 | b.lo 1b | |
240 | dsb sy | |
241 | ret | |
242 | ENDPROC(__dma_flush_range) | |
243 | ||
244 | /* | |
245 | * __dma_map_area(start, size, dir) | |
246 | * - start - kernel virtual start address | |
247 | * - size - size of region | |
248 | * - dir - DMA direction | |
249 | */ | |
250 | ENTRY(__dma_map_area) | |
251 | add x1, x1, x0 | |
252 | cmp w2, #DMA_FROM_DEVICE | |
253 | b.eq __dma_inv_range | |
254 | b __dma_clean_range | |
255 | ENDPROC(__dma_map_area) | |
256 | ||
257 | /* | |
258 | * __dma_unmap_area(start, size, dir) | |
259 | * - start - kernel virtual start address | |
260 | * - size - size of region | |
261 | * - dir - DMA direction | |
262 | */ | |
263 | ENTRY(__dma_unmap_area) | |
264 | add x1, x1, x0 | |
265 | cmp w2, #DMA_TO_DEVICE | |
266 | b.ne __dma_inv_range | |
267 | ret | |
268 | ENDPROC(__dma_unmap_area) |