Commit | Line | Data |
---|---|---|
58f07778 DD |
1 | /***********************license start*************** |
2 | * Author: Cavium Networks | |
3 | * | |
4 | * Contact: support@caviumnetworks.com | |
5 | * This file is part of the OCTEON SDK | |
6 | * | |
b8db85b5 | 7 | * Copyright (c) 2003-2010 Cavium Networks |
58f07778 DD |
8 | * |
9 | * This file is free software; you can redistribute it and/or modify | |
10 | * it under the terms of the GNU General Public License, Version 2, as | |
11 | * published by the Free Software Foundation. | |
12 | * | |
13 | * This file is distributed in the hope that it will be useful, but | |
14 | * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty | |
15 | * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or | |
16 | * NONINFRINGEMENT. See the GNU General Public License for more | |
17 | * details. | |
18 | * | |
19 | * You should have received a copy of the GNU General Public License | |
20 | * along with this file; if not, write to the Free Software | |
21 | * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA | |
22 | * or visit http://www.gnu.org/licenses/. | |
23 | * | |
24 | * This file may also be available under a different license from Cavium. | |
25 | * Contact Cavium Networks for more information | |
26 | ***********************license end**************************************/ | |
27 | ||
28 | /* | |
b8db85b5 DD |
29 | * Implementation of the Level 2 Cache (L2C) control, |
30 | * measurement, and debugging facilities. | |
58f07778 DD |
31 | */ |
32 | ||
92d11594 | 33 | #include <linux/irqflags.h> |
58f07778 DD |
34 | #include <asm/octeon/cvmx.h> |
35 | #include <asm/octeon/cvmx-l2c.h> | |
36 | #include <asm/octeon/cvmx-spinlock.h> | |
37 | ||
38 | /* | |
39 | * This spinlock is used internally to ensure that only one core is | |
40 | * performing certain L2 operations at a time. | |
41 | * | |
42 | * NOTE: This only protects calls from within a single application - | |
43 | * if multiple applications or operating systems are running, then it | |
44 | * is up to the user program to coordinate between them. | |
45 | */ | |
b8db85b5 | 46 | cvmx_spinlock_t cvmx_l2c_spinlock; |
58f07778 DD |
47 | |
48 | int cvmx_l2c_get_core_way_partition(uint32_t core) | |
49 | { | |
50 | uint32_t field; | |
51 | ||
52 | /* Validate the core number */ | |
53 | if (core >= cvmx_octeon_num_cores()) | |
54 | return -1; | |
55 | ||
b8db85b5 DD |
56 | if (OCTEON_IS_MODEL(OCTEON_CN63XX)) |
57 | return cvmx_read_csr(CVMX_L2C_WPAR_PPX(core)) & 0xffff; | |
58 | ||
58f07778 DD |
59 | /* |
60 | * Use the lower two bits of the coreNumber to determine the | |
61 | * bit offset of the UMSK[] field in the L2C_SPAR register. | |
62 | */ | |
63 | field = (core & 0x3) * 8; | |
64 | ||
65 | /* | |
66 | * Return the UMSK[] field from the appropriate L2C_SPAR | |
67 | * register based on the coreNumber. | |
68 | */ | |
69 | ||
70 | switch (core & 0xC) { | |
71 | case 0x0: | |
b8db85b5 | 72 | return (cvmx_read_csr(CVMX_L2C_SPAR0) & (0xFF << field)) >> field; |
58f07778 | 73 | case 0x4: |
b8db85b5 | 74 | return (cvmx_read_csr(CVMX_L2C_SPAR1) & (0xFF << field)) >> field; |
58f07778 | 75 | case 0x8: |
b8db85b5 | 76 | return (cvmx_read_csr(CVMX_L2C_SPAR2) & (0xFF << field)) >> field; |
58f07778 | 77 | case 0xC: |
b8db85b5 | 78 | return (cvmx_read_csr(CVMX_L2C_SPAR3) & (0xFF << field)) >> field; |
58f07778 DD |
79 | } |
80 | return 0; | |
81 | } | |
82 | ||
83 | int cvmx_l2c_set_core_way_partition(uint32_t core, uint32_t mask) | |
84 | { | |
85 | uint32_t field; | |
86 | uint32_t valid_mask; | |
87 | ||
88 | valid_mask = (0x1 << cvmx_l2c_get_num_assoc()) - 1; | |
89 | ||
90 | mask &= valid_mask; | |
91 | ||
b8db85b5 DD |
92 | /* A UMSK setting which blocks all L2C Ways is an error on some chips */ |
93 | if (mask == valid_mask && !OCTEON_IS_MODEL(OCTEON_CN63XX)) | |
58f07778 DD |
94 | return -1; |
95 | ||
96 | /* Validate the core number */ | |
97 | if (core >= cvmx_octeon_num_cores()) | |
98 | return -1; | |
99 | ||
b8db85b5 DD |
100 | if (OCTEON_IS_MODEL(OCTEON_CN63XX)) { |
101 | cvmx_write_csr(CVMX_L2C_WPAR_PPX(core), mask); | |
102 | return 0; | |
103 | } | |
58f07778 | 104 | |
b8db85b5 DD |
105 | /* |
106 | * Use the lower two bits of core to determine the bit offset of the | |
58f07778 DD |
107 | * UMSK[] field in the L2C_SPAR register. |
108 | */ | |
109 | field = (core & 0x3) * 8; | |
110 | ||
b8db85b5 DD |
111 | /* |
112 | * Assign the new mask setting to the UMSK[] field in the appropriate | |
58f07778 DD |
113 | * L2C_SPAR register based on the core_num. |
114 | * | |
115 | */ | |
116 | switch (core & 0xC) { | |
117 | case 0x0: | |
118 | cvmx_write_csr(CVMX_L2C_SPAR0, | |
b8db85b5 DD |
119 | (cvmx_read_csr(CVMX_L2C_SPAR0) & ~(0xFF << field)) | |
120 | mask << field); | |
58f07778 DD |
121 | break; |
122 | case 0x4: | |
123 | cvmx_write_csr(CVMX_L2C_SPAR1, | |
b8db85b5 DD |
124 | (cvmx_read_csr(CVMX_L2C_SPAR1) & ~(0xFF << field)) | |
125 | mask << field); | |
58f07778 DD |
126 | break; |
127 | case 0x8: | |
128 | cvmx_write_csr(CVMX_L2C_SPAR2, | |
b8db85b5 DD |
129 | (cvmx_read_csr(CVMX_L2C_SPAR2) & ~(0xFF << field)) | |
130 | mask << field); | |
58f07778 DD |
131 | break; |
132 | case 0xC: | |
133 | cvmx_write_csr(CVMX_L2C_SPAR3, | |
b8db85b5 DD |
134 | (cvmx_read_csr(CVMX_L2C_SPAR3) & ~(0xFF << field)) | |
135 | mask << field); | |
58f07778 DD |
136 | break; |
137 | } | |
138 | return 0; | |
139 | } | |
140 | ||
141 | int cvmx_l2c_set_hw_way_partition(uint32_t mask) | |
142 | { | |
143 | uint32_t valid_mask; | |
144 | ||
b8db85b5 | 145 | valid_mask = (0x1 << cvmx_l2c_get_num_assoc()) - 1; |
58f07778 DD |
146 | mask &= valid_mask; |
147 | ||
b8db85b5 DD |
148 | /* A UMSK setting which blocks all L2C Ways is an error on some chips */ |
149 | if (mask == valid_mask && !OCTEON_IS_MODEL(OCTEON_CN63XX)) | |
58f07778 DD |
150 | return -1; |
151 | ||
b8db85b5 DD |
152 | if (OCTEON_IS_MODEL(OCTEON_CN63XX)) |
153 | cvmx_write_csr(CVMX_L2C_WPAR_IOBX(0), mask); | |
154 | else | |
155 | cvmx_write_csr(CVMX_L2C_SPAR4, | |
156 | (cvmx_read_csr(CVMX_L2C_SPAR4) & ~0xFF) | mask); | |
58f07778 DD |
157 | return 0; |
158 | } | |
159 | ||
160 | int cvmx_l2c_get_hw_way_partition(void) | |
161 | { | |
b8db85b5 DD |
162 | if (OCTEON_IS_MODEL(OCTEON_CN63XX)) |
163 | return cvmx_read_csr(CVMX_L2C_WPAR_IOBX(0)) & 0xffff; | |
164 | else | |
165 | return cvmx_read_csr(CVMX_L2C_SPAR4) & (0xFF); | |
58f07778 DD |
166 | } |
167 | ||
168 | void cvmx_l2c_config_perf(uint32_t counter, enum cvmx_l2c_event event, | |
169 | uint32_t clear_on_read) | |
170 | { | |
b8db85b5 DD |
171 | if (OCTEON_IS_MODEL(OCTEON_CN5XXX) || OCTEON_IS_MODEL(OCTEON_CN3XXX)) { |
172 | union cvmx_l2c_pfctl pfctl; | |
58f07778 | 173 | |
b8db85b5 | 174 | pfctl.u64 = cvmx_read_csr(CVMX_L2C_PFCTL); |
58f07778 | 175 | |
b8db85b5 DD |
176 | switch (counter) { |
177 | case 0: | |
178 | pfctl.s.cnt0sel = event; | |
179 | pfctl.s.cnt0ena = 1; | |
58f07778 | 180 | pfctl.s.cnt0rdclr = clear_on_read; |
b8db85b5 DD |
181 | break; |
182 | case 1: | |
183 | pfctl.s.cnt1sel = event; | |
184 | pfctl.s.cnt1ena = 1; | |
58f07778 | 185 | pfctl.s.cnt1rdclr = clear_on_read; |
b8db85b5 DD |
186 | break; |
187 | case 2: | |
188 | pfctl.s.cnt2sel = event; | |
189 | pfctl.s.cnt2ena = 1; | |
58f07778 | 190 | pfctl.s.cnt2rdclr = clear_on_read; |
b8db85b5 DD |
191 | break; |
192 | case 3: | |
193 | default: | |
194 | pfctl.s.cnt3sel = event; | |
195 | pfctl.s.cnt3ena = 1; | |
58f07778 | 196 | pfctl.s.cnt3rdclr = clear_on_read; |
b8db85b5 DD |
197 | break; |
198 | } | |
58f07778 | 199 | |
b8db85b5 DD |
200 | cvmx_write_csr(CVMX_L2C_PFCTL, pfctl.u64); |
201 | } else { | |
202 | union cvmx_l2c_tadx_prf l2c_tadx_prf; | |
203 | int tad; | |
204 | ||
205 | cvmx_dprintf("L2C performance counter events are different for this chip, mapping 'event' to cvmx_l2c_tad_event_t\n"); | |
206 | if (clear_on_read) | |
207 | cvmx_dprintf("L2C counters don't support clear on read for this chip\n"); | |
208 | ||
209 | l2c_tadx_prf.u64 = cvmx_read_csr(CVMX_L2C_TADX_PRF(0)); | |
210 | ||
211 | switch (counter) { | |
212 | case 0: | |
213 | l2c_tadx_prf.s.cnt0sel = event; | |
214 | break; | |
215 | case 1: | |
216 | l2c_tadx_prf.s.cnt1sel = event; | |
217 | break; | |
218 | case 2: | |
219 | l2c_tadx_prf.s.cnt2sel = event; | |
220 | break; | |
221 | default: | |
222 | case 3: | |
223 | l2c_tadx_prf.s.cnt3sel = event; | |
224 | break; | |
225 | } | |
226 | for (tad = 0; tad < CVMX_L2C_TADS; tad++) | |
227 | cvmx_write_csr(CVMX_L2C_TADX_PRF(tad), | |
228 | l2c_tadx_prf.u64); | |
229 | } | |
58f07778 DD |
230 | } |
231 | ||
232 | uint64_t cvmx_l2c_read_perf(uint32_t counter) | |
233 | { | |
234 | switch (counter) { | |
235 | case 0: | |
b8db85b5 DD |
236 | if (OCTEON_IS_MODEL(OCTEON_CN5XXX) || OCTEON_IS_MODEL(OCTEON_CN3XXX)) |
237 | return cvmx_read_csr(CVMX_L2C_PFC0); | |
238 | else { | |
239 | uint64_t counter = 0; | |
240 | int tad; | |
241 | for (tad = 0; tad < CVMX_L2C_TADS; tad++) | |
242 | counter += cvmx_read_csr(CVMX_L2C_TADX_PFC0(tad)); | |
243 | return counter; | |
244 | } | |
58f07778 | 245 | case 1: |
b8db85b5 DD |
246 | if (OCTEON_IS_MODEL(OCTEON_CN5XXX) || OCTEON_IS_MODEL(OCTEON_CN3XXX)) |
247 | return cvmx_read_csr(CVMX_L2C_PFC1); | |
248 | else { | |
249 | uint64_t counter = 0; | |
250 | int tad; | |
251 | for (tad = 0; tad < CVMX_L2C_TADS; tad++) | |
252 | counter += cvmx_read_csr(CVMX_L2C_TADX_PFC1(tad)); | |
253 | return counter; | |
254 | } | |
58f07778 | 255 | case 2: |
b8db85b5 DD |
256 | if (OCTEON_IS_MODEL(OCTEON_CN5XXX) || OCTEON_IS_MODEL(OCTEON_CN3XXX)) |
257 | return cvmx_read_csr(CVMX_L2C_PFC2); | |
258 | else { | |
259 | uint64_t counter = 0; | |
260 | int tad; | |
261 | for (tad = 0; tad < CVMX_L2C_TADS; tad++) | |
262 | counter += cvmx_read_csr(CVMX_L2C_TADX_PFC2(tad)); | |
263 | return counter; | |
264 | } | |
58f07778 DD |
265 | case 3: |
266 | default: | |
b8db85b5 DD |
267 | if (OCTEON_IS_MODEL(OCTEON_CN5XXX) || OCTEON_IS_MODEL(OCTEON_CN3XXX)) |
268 | return cvmx_read_csr(CVMX_L2C_PFC3); | |
269 | else { | |
270 | uint64_t counter = 0; | |
271 | int tad; | |
272 | for (tad = 0; tad < CVMX_L2C_TADS; tad++) | |
273 | counter += cvmx_read_csr(CVMX_L2C_TADX_PFC3(tad)); | |
274 | return counter; | |
275 | } | |
58f07778 DD |
276 | } |
277 | } | |
278 | ||
279 | /** | |
280 | * @INTERNAL | |
281 | * Helper function use to fault in cache lines for L2 cache locking | |
282 | * | |
283 | * @addr: Address of base of memory region to read into L2 cache | |
284 | * @len: Length (in bytes) of region to fault in | |
285 | */ | |
286 | static void fault_in(uint64_t addr, int len) | |
287 | { | |
288 | volatile char *ptr; | |
289 | volatile char dummy; | |
290 | /* | |
291 | * Adjust addr and length so we get all cache lines even for | |
b8db85b5 | 292 | * small ranges spanning two cache lines. |
58f07778 DD |
293 | */ |
294 | len += addr & CVMX_CACHE_LINE_MASK; | |
295 | addr &= ~CVMX_CACHE_LINE_MASK; | |
296 | ptr = (volatile char *)cvmx_phys_to_ptr(addr); | |
297 | /* | |
298 | * Invalidate L1 cache to make sure all loads result in data | |
299 | * being in L2. | |
300 | */ | |
301 | CVMX_DCACHE_INVALIDATE; | |
302 | while (len > 0) { | |
303 | dummy += *ptr; | |
304 | len -= CVMX_CACHE_LINE_SIZE; | |
305 | ptr += CVMX_CACHE_LINE_SIZE; | |
306 | } | |
307 | } | |
308 | ||
309 | int cvmx_l2c_lock_line(uint64_t addr) | |
310 | { | |
b8db85b5 DD |
311 | if (OCTEON_IS_MODEL(OCTEON_CN63XX)) { |
312 | int shift = CVMX_L2C_TAG_ADDR_ALIAS_SHIFT; | |
313 | uint64_t assoc = cvmx_l2c_get_num_assoc(); | |
314 | uint64_t tag = addr >> shift; | |
315 | uint64_t index = CVMX_ADD_SEG(CVMX_MIPS_SPACE_XKPHYS, cvmx_l2c_address_to_index(addr) << CVMX_L2C_IDX_ADDR_SHIFT); | |
316 | uint64_t way; | |
317 | union cvmx_l2c_tadx_tag l2c_tadx_tag; | |
318 | ||
319 | CVMX_CACHE_LCKL2(CVMX_ADD_SEG(CVMX_MIPS_SPACE_XKPHYS, addr), 0); | |
320 | ||
321 | /* Make sure we were able to lock the line */ | |
322 | for (way = 0; way < assoc; way++) { | |
323 | CVMX_CACHE_LTGL2I(index | (way << shift), 0); | |
324 | /* make sure CVMX_L2C_TADX_TAG is updated */ | |
325 | CVMX_SYNC; | |
326 | l2c_tadx_tag.u64 = cvmx_read_csr(CVMX_L2C_TADX_TAG(0)); | |
327 | if (l2c_tadx_tag.s.valid && l2c_tadx_tag.s.tag == tag) | |
328 | break; | |
329 | } | |
58f07778 | 330 | |
b8db85b5 DD |
331 | /* Check if a valid line is found */ |
332 | if (way >= assoc) { | |
333 | /* cvmx_dprintf("ERROR: cvmx_l2c_lock_line: line not found for locking at 0x%llx address\n", (unsigned long long)addr); */ | |
334 | return -1; | |
335 | } | |
58f07778 | 336 | |
b8db85b5 DD |
337 | /* Check if lock bit is not set */ |
338 | if (!l2c_tadx_tag.s.lock) { | |
339 | /* cvmx_dprintf("ERROR: cvmx_l2c_lock_line: Not able to lock at 0x%llx address\n", (unsigned long long)addr); */ | |
340 | return -1; | |
341 | } | |
342 | return way; | |
58f07778 | 343 | } else { |
b8db85b5 DD |
344 | int retval = 0; |
345 | union cvmx_l2c_dbg l2cdbg; | |
346 | union cvmx_l2c_lckbase lckbase; | |
347 | union cvmx_l2c_lckoff lckoff; | |
348 | union cvmx_l2t_err l2t_err; | |
58f07778 | 349 | |
b8db85b5 | 350 | cvmx_spinlock_lock(&cvmx_l2c_spinlock); |
58f07778 | 351 | |
b8db85b5 DD |
352 | l2cdbg.u64 = 0; |
353 | lckbase.u64 = 0; | |
354 | lckoff.u64 = 0; | |
58f07778 | 355 | |
b8db85b5 DD |
356 | /* Clear l2t error bits if set */ |
357 | l2t_err.u64 = cvmx_read_csr(CVMX_L2T_ERR); | |
358 | l2t_err.s.lckerr = 1; | |
359 | l2t_err.s.lckerr2 = 1; | |
360 | cvmx_write_csr(CVMX_L2T_ERR, l2t_err.u64); | |
58f07778 | 361 | |
b8db85b5 | 362 | addr &= ~CVMX_CACHE_LINE_MASK; |
58f07778 | 363 | |
b8db85b5 DD |
364 | /* Set this core as debug core */ |
365 | l2cdbg.s.ppnum = cvmx_get_core_num(); | |
366 | CVMX_SYNC; | |
367 | cvmx_write_csr(CVMX_L2C_DBG, l2cdbg.u64); | |
368 | cvmx_read_csr(CVMX_L2C_DBG); | |
369 | ||
370 | lckoff.s.lck_offset = 0; /* Only lock 1 line at a time */ | |
371 | cvmx_write_csr(CVMX_L2C_LCKOFF, lckoff.u64); | |
372 | cvmx_read_csr(CVMX_L2C_LCKOFF); | |
373 | ||
374 | if (((union cvmx_l2c_cfg)(cvmx_read_csr(CVMX_L2C_CFG))).s.idxalias) { | |
375 | int alias_shift = CVMX_L2C_IDX_ADDR_SHIFT + 2 * CVMX_L2_SET_BITS - 1; | |
376 | uint64_t addr_tmp = addr ^ (addr & ((1 << alias_shift) - 1)) >> CVMX_L2_SET_BITS; | |
377 | lckbase.s.lck_base = addr_tmp >> 7; | |
378 | } else { | |
379 | lckbase.s.lck_base = addr >> 7; | |
380 | } | |
58f07778 | 381 | |
b8db85b5 DD |
382 | lckbase.s.lck_ena = 1; |
383 | cvmx_write_csr(CVMX_L2C_LCKBASE, lckbase.u64); | |
384 | /* Make sure it gets there */ | |
385 | cvmx_read_csr(CVMX_L2C_LCKBASE); | |
58f07778 | 386 | |
b8db85b5 DD |
387 | fault_in(addr, CVMX_CACHE_LINE_SIZE); |
388 | ||
389 | lckbase.s.lck_ena = 0; | |
390 | cvmx_write_csr(CVMX_L2C_LCKBASE, lckbase.u64); | |
391 | /* Make sure it gets there */ | |
392 | cvmx_read_csr(CVMX_L2C_LCKBASE); | |
393 | ||
394 | /* Stop being debug core */ | |
395 | cvmx_write_csr(CVMX_L2C_DBG, 0); | |
396 | cvmx_read_csr(CVMX_L2C_DBG); | |
397 | ||
398 | l2t_err.u64 = cvmx_read_csr(CVMX_L2T_ERR); | |
399 | if (l2t_err.s.lckerr || l2t_err.s.lckerr2) | |
400 | retval = 1; /* We were unable to lock the line */ | |
401 | ||
402 | cvmx_spinlock_unlock(&cvmx_l2c_spinlock); | |
403 | return retval; | |
404 | } | |
58f07778 DD |
405 | } |
406 | ||
407 | int cvmx_l2c_lock_mem_region(uint64_t start, uint64_t len) | |
408 | { | |
409 | int retval = 0; | |
410 | ||
411 | /* Round start/end to cache line boundaries */ | |
412 | len += start & CVMX_CACHE_LINE_MASK; | |
413 | start &= ~CVMX_CACHE_LINE_MASK; | |
414 | len = (len + CVMX_CACHE_LINE_MASK) & ~CVMX_CACHE_LINE_MASK; | |
415 | ||
416 | while (len) { | |
417 | retval += cvmx_l2c_lock_line(start); | |
418 | start += CVMX_CACHE_LINE_SIZE; | |
419 | len -= CVMX_CACHE_LINE_SIZE; | |
420 | } | |
58f07778 DD |
421 | return retval; |
422 | } | |
423 | ||
424 | void cvmx_l2c_flush(void) | |
425 | { | |
426 | uint64_t assoc, set; | |
427 | uint64_t n_assoc, n_set; | |
58f07778 | 428 | |
b8db85b5 DD |
429 | n_set = cvmx_l2c_get_num_sets(); |
430 | n_assoc = cvmx_l2c_get_num_assoc(); | |
431 | ||
432 | if (OCTEON_IS_MODEL(OCTEON_CN6XXX)) { | |
433 | uint64_t address; | |
434 | /* These may look like constants, but they aren't... */ | |
435 | int assoc_shift = CVMX_L2C_TAG_ADDR_ALIAS_SHIFT; | |
436 | int set_shift = CVMX_L2C_IDX_ADDR_SHIFT; | |
437 | for (set = 0; set < n_set; set++) { | |
438 | for (assoc = 0; assoc < n_assoc; assoc++) { | |
439 | address = CVMX_ADD_SEG(CVMX_MIPS_SPACE_XKPHYS, | |
440 | (assoc << assoc_shift) | (set << set_shift)); | |
441 | CVMX_CACHE_WBIL2I(address, 0); | |
442 | } | |
58f07778 | 443 | } |
b8db85b5 DD |
444 | } else { |
445 | for (set = 0; set < n_set; set++) | |
446 | for (assoc = 0; assoc < n_assoc; assoc++) | |
447 | cvmx_l2c_flush_line(assoc, set); | |
58f07778 | 448 | } |
58f07778 DD |
449 | } |
450 | ||
b8db85b5 | 451 | |
58f07778 DD |
452 | int cvmx_l2c_unlock_line(uint64_t address) |
453 | { | |
58f07778 | 454 | |
b8db85b5 DD |
455 | if (OCTEON_IS_MODEL(OCTEON_CN63XX)) { |
456 | int assoc; | |
457 | union cvmx_l2c_tag tag; | |
458 | uint32_t tag_addr; | |
459 | uint32_t index = cvmx_l2c_address_to_index(address); | |
460 | ||
461 | tag_addr = ((address >> CVMX_L2C_TAG_ADDR_ALIAS_SHIFT) & ((1 << CVMX_L2C_TAG_ADDR_ALIAS_SHIFT) - 1)); | |
462 | ||
463 | /* | |
464 | * For 63XX, we can flush a line by using the physical | |
465 | * address directly, so finding the cache line used by | |
466 | * the address is only required to provide the proper | |
467 | * return value for the function. | |
468 | */ | |
469 | for (assoc = 0; assoc < CVMX_L2_ASSOC; assoc++) { | |
470 | tag = cvmx_l2c_get_tag(assoc, index); | |
471 | ||
472 | if (tag.s.V && (tag.s.addr == tag_addr)) { | |
473 | CVMX_CACHE_WBIL2(CVMX_ADD_SEG(CVMX_MIPS_SPACE_XKPHYS, address), 0); | |
474 | return tag.s.L; | |
475 | } | |
476 | } | |
477 | } else { | |
478 | int assoc; | |
479 | union cvmx_l2c_tag tag; | |
480 | uint32_t tag_addr; | |
58f07778 | 481 | |
b8db85b5 | 482 | uint32_t index = cvmx_l2c_address_to_index(address); |
58f07778 | 483 | |
b8db85b5 DD |
484 | /* Compute portion of address that is stored in tag */ |
485 | tag_addr = ((address >> CVMX_L2C_TAG_ADDR_ALIAS_SHIFT) & ((1 << CVMX_L2C_TAG_ADDR_ALIAS_SHIFT) - 1)); | |
486 | for (assoc = 0; assoc < CVMX_L2_ASSOC; assoc++) { | |
487 | tag = cvmx_l2c_get_tag(assoc, index); | |
58f07778 | 488 | |
b8db85b5 DD |
489 | if (tag.s.V && (tag.s.addr == tag_addr)) { |
490 | cvmx_l2c_flush_line(assoc, index); | |
491 | return tag.s.L; | |
492 | } | |
58f07778 DD |
493 | } |
494 | } | |
58f07778 DD |
495 | return 0; |
496 | } | |
497 | ||
498 | int cvmx_l2c_unlock_mem_region(uint64_t start, uint64_t len) | |
499 | { | |
500 | int num_unlocked = 0; | |
501 | /* Round start/end to cache line boundaries */ | |
502 | len += start & CVMX_CACHE_LINE_MASK; | |
503 | start &= ~CVMX_CACHE_LINE_MASK; | |
504 | len = (len + CVMX_CACHE_LINE_MASK) & ~CVMX_CACHE_LINE_MASK; | |
505 | while (len > 0) { | |
506 | num_unlocked += cvmx_l2c_unlock_line(start); | |
507 | start += CVMX_CACHE_LINE_SIZE; | |
508 | len -= CVMX_CACHE_LINE_SIZE; | |
509 | } | |
510 | ||
511 | return num_unlocked; | |
512 | } | |
513 | ||
514 | /* | |
515 | * Internal l2c tag types. These are converted to a generic structure | |
516 | * that can be used on all chips. | |
517 | */ | |
518 | union __cvmx_l2c_tag { | |
519 | uint64_t u64; | |
520 | struct cvmx_l2c_tag_cn50xx { | |
521 | uint64_t reserved:40; | |
b8db85b5 DD |
522 | uint64_t V:1; /* Line valid */ |
523 | uint64_t D:1; /* Line dirty */ | |
524 | uint64_t L:1; /* Line locked */ | |
525 | uint64_t U:1; /* Use, LRU eviction */ | |
58f07778 DD |
526 | uint64_t addr:20; /* Phys mem addr (33..14) */ |
527 | } cn50xx; | |
528 | struct cvmx_l2c_tag_cn30xx { | |
529 | uint64_t reserved:41; | |
b8db85b5 DD |
530 | uint64_t V:1; /* Line valid */ |
531 | uint64_t D:1; /* Line dirty */ | |
532 | uint64_t L:1; /* Line locked */ | |
533 | uint64_t U:1; /* Use, LRU eviction */ | |
58f07778 DD |
534 | uint64_t addr:19; /* Phys mem addr (33..15) */ |
535 | } cn30xx; | |
536 | struct cvmx_l2c_tag_cn31xx { | |
537 | uint64_t reserved:42; | |
b8db85b5 DD |
538 | uint64_t V:1; /* Line valid */ |
539 | uint64_t D:1; /* Line dirty */ | |
540 | uint64_t L:1; /* Line locked */ | |
541 | uint64_t U:1; /* Use, LRU eviction */ | |
58f07778 DD |
542 | uint64_t addr:18; /* Phys mem addr (33..16) */ |
543 | } cn31xx; | |
544 | struct cvmx_l2c_tag_cn38xx { | |
545 | uint64_t reserved:43; | |
b8db85b5 DD |
546 | uint64_t V:1; /* Line valid */ |
547 | uint64_t D:1; /* Line dirty */ | |
548 | uint64_t L:1; /* Line locked */ | |
549 | uint64_t U:1; /* Use, LRU eviction */ | |
58f07778 DD |
550 | uint64_t addr:17; /* Phys mem addr (33..17) */ |
551 | } cn38xx; | |
552 | struct cvmx_l2c_tag_cn58xx { | |
553 | uint64_t reserved:44; | |
b8db85b5 DD |
554 | uint64_t V:1; /* Line valid */ |
555 | uint64_t D:1; /* Line dirty */ | |
556 | uint64_t L:1; /* Line locked */ | |
557 | uint64_t U:1; /* Use, LRU eviction */ | |
58f07778 DD |
558 | uint64_t addr:16; /* Phys mem addr (33..18) */ |
559 | } cn58xx; | |
560 | struct cvmx_l2c_tag_cn58xx cn56xx; /* 2048 sets */ | |
561 | struct cvmx_l2c_tag_cn31xx cn52xx; /* 512 sets */ | |
562 | }; | |
563 | ||
b8db85b5 | 564 | |
58f07778 DD |
565 | /** |
566 | * @INTERNAL | |
567 | * Function to read a L2C tag. This code make the current core | |
568 | * the 'debug core' for the L2. This code must only be executed by | |
569 | * 1 core at a time. | |
570 | * | |
571 | * @assoc: Association (way) of the tag to dump | |
572 | * @index: Index of the cacheline | |
573 | * | |
574 | * Returns The Octeon model specific tag structure. This is | |
575 | * translated by a wrapper function to a generic form that is | |
576 | * easier for applications to use. | |
577 | */ | |
578 | static union __cvmx_l2c_tag __read_l2_tag(uint64_t assoc, uint64_t index) | |
579 | { | |
580 | ||
b8db85b5 | 581 | uint64_t debug_tag_addr = CVMX_ADD_SEG(CVMX_MIPS_SPACE_XKPHYS, (index << 7) + 96); |
58f07778 DD |
582 | uint64_t core = cvmx_get_core_num(); |
583 | union __cvmx_l2c_tag tag_val; | |
584 | uint64_t dbg_addr = CVMX_L2C_DBG; | |
585 | unsigned long flags; | |
586 | ||
587 | union cvmx_l2c_dbg debug_val; | |
588 | debug_val.u64 = 0; | |
589 | /* | |
b8db85b5 DD |
590 | * For low core count parts, the core number is always small |
591 | * enough to stay in the correct field and not set any | |
592 | * reserved bits. | |
58f07778 DD |
593 | */ |
594 | debug_val.s.ppnum = core; | |
595 | debug_val.s.l2t = 1; | |
596 | debug_val.s.set = assoc; | |
b8db85b5 DD |
597 | |
598 | local_irq_save(flags); | |
58f07778 DD |
599 | /* |
600 | * Make sure core is quiet (no prefetches, etc.) before | |
601 | * entering debug mode. | |
602 | */ | |
603 | CVMX_SYNC; | |
604 | /* Flush L1 to make sure debug load misses L1 */ | |
605 | CVMX_DCACHE_INVALIDATE; | |
606 | ||
58f07778 DD |
607 | /* |
608 | * The following must be done in assembly as when in debug | |
609 | * mode all data loads from L2 return special debug data, not | |
b8db85b5 DD |
610 | * normal memory contents. Also, interrupts must be disabled, |
611 | * since if an interrupt occurs while in debug mode the ISR | |
612 | * will get debug data from all its memory * reads instead of | |
613 | * the contents of memory. | |
58f07778 DD |
614 | */ |
615 | ||
b8db85b5 DD |
616 | asm volatile ( |
617 | ".set push\n\t" | |
618 | ".set mips64\n\t" | |
619 | ".set noreorder\n\t" | |
620 | "sd %[dbg_val], 0(%[dbg_addr])\n\t" /* Enter debug mode, wait for store */ | |
621 | "ld $0, 0(%[dbg_addr])\n\t" | |
622 | "ld %[tag_val], 0(%[tag_addr])\n\t" /* Read L2C tag data */ | |
623 | "sd $0, 0(%[dbg_addr])\n\t" /* Exit debug mode, wait for store */ | |
624 | "ld $0, 0(%[dbg_addr])\n\t" | |
625 | "cache 9, 0($0)\n\t" /* Invalidate dcache to discard debug data */ | |
626 | ".set pop" | |
627 | : [tag_val] "=r" (tag_val) | |
628 | : [dbg_addr] "r" (dbg_addr), [dbg_val] "r" (debug_val), [tag_addr] "r" (debug_tag_addr) | |
629 | : "memory"); | |
58f07778 DD |
630 | |
631 | local_irq_restore(flags); | |
58f07778 | 632 | |
b8db85b5 | 633 | return tag_val; |
58f07778 DD |
634 | } |
635 | ||
b8db85b5 | 636 | |
58f07778 DD |
637 | union cvmx_l2c_tag cvmx_l2c_get_tag(uint32_t association, uint32_t index) |
638 | { | |
58f07778 DD |
639 | union cvmx_l2c_tag tag; |
640 | tag.u64 = 0; | |
641 | ||
642 | if ((int)association >= cvmx_l2c_get_num_assoc()) { | |
b8db85b5 | 643 | cvmx_dprintf("ERROR: cvmx_l2c_get_tag association out of range\n"); |
58f07778 DD |
644 | return tag; |
645 | } | |
646 | if ((int)index >= cvmx_l2c_get_num_sets()) { | |
b8db85b5 DD |
647 | cvmx_dprintf("ERROR: cvmx_l2c_get_tag index out of range (arg: %d, max: %d)\n", |
648 | (int)index, cvmx_l2c_get_num_sets()); | |
58f07778 DD |
649 | return tag; |
650 | } | |
b8db85b5 DD |
651 | if (OCTEON_IS_MODEL(OCTEON_CN63XX)) { |
652 | union cvmx_l2c_tadx_tag l2c_tadx_tag; | |
653 | uint64_t address = CVMX_ADD_SEG(CVMX_MIPS_SPACE_XKPHYS, | |
654 | (association << CVMX_L2C_TAG_ADDR_ALIAS_SHIFT) | | |
655 | (index << CVMX_L2C_IDX_ADDR_SHIFT)); | |
656 | /* | |
657 | * Use L2 cache Index load tag cache instruction, as | |
658 | * hardware loads the virtual tag for the L2 cache | |
659 | * block with the contents of L2C_TAD0_TAG | |
660 | * register. | |
661 | */ | |
662 | CVMX_CACHE_LTGL2I(address, 0); | |
663 | CVMX_SYNC; /* make sure CVMX_L2C_TADX_TAG is updated */ | |
664 | l2c_tadx_tag.u64 = cvmx_read_csr(CVMX_L2C_TADX_TAG(0)); | |
665 | ||
666 | tag.s.V = l2c_tadx_tag.s.valid; | |
667 | tag.s.D = l2c_tadx_tag.s.dirty; | |
668 | tag.s.L = l2c_tadx_tag.s.lock; | |
669 | tag.s.U = l2c_tadx_tag.s.use; | |
670 | tag.s.addr = l2c_tadx_tag.s.tag; | |
58f07778 | 671 | } else { |
b8db85b5 DD |
672 | union __cvmx_l2c_tag tmp_tag; |
673 | /* __read_l2_tag is intended for internal use only */ | |
674 | tmp_tag = __read_l2_tag(association, index); | |
675 | ||
676 | /* | |
677 | * Convert all tag structure types to generic version, | |
678 | * as it can represent all models. | |
679 | */ | |
680 | if (OCTEON_IS_MODEL(OCTEON_CN58XX) || OCTEON_IS_MODEL(OCTEON_CN56XX)) { | |
681 | tag.s.V = tmp_tag.cn58xx.V; | |
682 | tag.s.D = tmp_tag.cn58xx.D; | |
683 | tag.s.L = tmp_tag.cn58xx.L; | |
684 | tag.s.U = tmp_tag.cn58xx.U; | |
685 | tag.s.addr = tmp_tag.cn58xx.addr; | |
686 | } else if (OCTEON_IS_MODEL(OCTEON_CN38XX)) { | |
687 | tag.s.V = tmp_tag.cn38xx.V; | |
688 | tag.s.D = tmp_tag.cn38xx.D; | |
689 | tag.s.L = tmp_tag.cn38xx.L; | |
690 | tag.s.U = tmp_tag.cn38xx.U; | |
691 | tag.s.addr = tmp_tag.cn38xx.addr; | |
692 | } else if (OCTEON_IS_MODEL(OCTEON_CN31XX) || OCTEON_IS_MODEL(OCTEON_CN52XX)) { | |
693 | tag.s.V = tmp_tag.cn31xx.V; | |
694 | tag.s.D = tmp_tag.cn31xx.D; | |
695 | tag.s.L = tmp_tag.cn31xx.L; | |
696 | tag.s.U = tmp_tag.cn31xx.U; | |
697 | tag.s.addr = tmp_tag.cn31xx.addr; | |
698 | } else if (OCTEON_IS_MODEL(OCTEON_CN30XX)) { | |
699 | tag.s.V = tmp_tag.cn30xx.V; | |
700 | tag.s.D = tmp_tag.cn30xx.D; | |
701 | tag.s.L = tmp_tag.cn30xx.L; | |
702 | tag.s.U = tmp_tag.cn30xx.U; | |
703 | tag.s.addr = tmp_tag.cn30xx.addr; | |
704 | } else if (OCTEON_IS_MODEL(OCTEON_CN50XX)) { | |
705 | tag.s.V = tmp_tag.cn50xx.V; | |
706 | tag.s.D = tmp_tag.cn50xx.D; | |
707 | tag.s.L = tmp_tag.cn50xx.L; | |
708 | tag.s.U = tmp_tag.cn50xx.U; | |
709 | tag.s.addr = tmp_tag.cn50xx.addr; | |
710 | } else { | |
711 | cvmx_dprintf("Unsupported OCTEON Model in %s\n", __func__); | |
712 | } | |
58f07778 | 713 | } |
58f07778 DD |
714 | return tag; |
715 | } | |
716 | ||
717 | uint32_t cvmx_l2c_address_to_index(uint64_t addr) | |
718 | { | |
719 | uint64_t idx = addr >> CVMX_L2C_IDX_ADDR_SHIFT; | |
b8db85b5 | 720 | int indxalias = 0; |
58f07778 | 721 | |
b8db85b5 DD |
722 | if (OCTEON_IS_MODEL(OCTEON_CN6XXX)) { |
723 | union cvmx_l2c_ctl l2c_ctl; | |
724 | l2c_ctl.u64 = cvmx_read_csr(CVMX_L2C_CTL); | |
725 | indxalias = !l2c_ctl.s.disidxalias; | |
726 | } else { | |
727 | union cvmx_l2c_cfg l2c_cfg; | |
728 | l2c_cfg.u64 = cvmx_read_csr(CVMX_L2C_CFG); | |
729 | indxalias = l2c_cfg.s.idxalias; | |
730 | } | |
731 | ||
732 | if (indxalias) { | |
733 | if (OCTEON_IS_MODEL(OCTEON_CN63XX)) { | |
734 | uint32_t a_14_12 = (idx / (CVMX_L2C_MEMBANK_SELECT_SIZE/(1<<CVMX_L2C_IDX_ADDR_SHIFT))) & 0x7; | |
735 | idx ^= idx / cvmx_l2c_get_num_sets(); | |
736 | idx ^= a_14_12; | |
737 | } else { | |
738 | idx ^= ((addr & CVMX_L2C_ALIAS_MASK) >> CVMX_L2C_TAG_ADDR_ALIAS_SHIFT); | |
739 | } | |
58f07778 DD |
740 | } |
741 | idx &= CVMX_L2C_IDX_MASK; | |
742 | return idx; | |
743 | } | |
744 | ||
745 | int cvmx_l2c_get_cache_size_bytes(void) | |
746 | { | |
747 | return cvmx_l2c_get_num_sets() * cvmx_l2c_get_num_assoc() * | |
748 | CVMX_CACHE_LINE_SIZE; | |
749 | } | |
750 | ||
751 | /** | |
752 | * Return log base 2 of the number of sets in the L2 cache | |
753 | * Returns | |
754 | */ | |
755 | int cvmx_l2c_get_set_bits(void) | |
756 | { | |
757 | int l2_set_bits; | |
758 | if (OCTEON_IS_MODEL(OCTEON_CN56XX) || OCTEON_IS_MODEL(OCTEON_CN58XX)) | |
759 | l2_set_bits = 11; /* 2048 sets */ | |
b8db85b5 | 760 | else if (OCTEON_IS_MODEL(OCTEON_CN38XX) || OCTEON_IS_MODEL(OCTEON_CN63XX)) |
58f07778 | 761 | l2_set_bits = 10; /* 1024 sets */ |
b8db85b5 | 762 | else if (OCTEON_IS_MODEL(OCTEON_CN31XX) || OCTEON_IS_MODEL(OCTEON_CN52XX)) |
58f07778 DD |
763 | l2_set_bits = 9; /* 512 sets */ |
764 | else if (OCTEON_IS_MODEL(OCTEON_CN30XX)) | |
765 | l2_set_bits = 8; /* 256 sets */ | |
766 | else if (OCTEON_IS_MODEL(OCTEON_CN50XX)) | |
767 | l2_set_bits = 7; /* 128 sets */ | |
768 | else { | |
769 | cvmx_dprintf("Unsupported OCTEON Model in %s\n", __func__); | |
770 | l2_set_bits = 11; /* 2048 sets */ | |
771 | } | |
772 | return l2_set_bits; | |
58f07778 DD |
773 | } |
774 | ||
775 | /* Return the number of sets in the L2 Cache */ | |
776 | int cvmx_l2c_get_num_sets(void) | |
777 | { | |
778 | return 1 << cvmx_l2c_get_set_bits(); | |
779 | } | |
780 | ||
781 | /* Return the number of associations in the L2 Cache */ | |
782 | int cvmx_l2c_get_num_assoc(void) | |
783 | { | |
784 | int l2_assoc; | |
785 | if (OCTEON_IS_MODEL(OCTEON_CN56XX) || | |
786 | OCTEON_IS_MODEL(OCTEON_CN52XX) || | |
787 | OCTEON_IS_MODEL(OCTEON_CN58XX) || | |
b8db85b5 DD |
788 | OCTEON_IS_MODEL(OCTEON_CN50XX) || |
789 | OCTEON_IS_MODEL(OCTEON_CN38XX)) | |
58f07778 | 790 | l2_assoc = 8; |
b8db85b5 DD |
791 | else if (OCTEON_IS_MODEL(OCTEON_CN63XX)) |
792 | l2_assoc = 16; | |
58f07778 DD |
793 | else if (OCTEON_IS_MODEL(OCTEON_CN31XX) || |
794 | OCTEON_IS_MODEL(OCTEON_CN30XX)) | |
795 | l2_assoc = 4; | |
796 | else { | |
797 | cvmx_dprintf("Unsupported OCTEON Model in %s\n", __func__); | |
798 | l2_assoc = 8; | |
799 | } | |
800 | ||
801 | /* Check to see if part of the cache is disabled */ | |
b8db85b5 DD |
802 | if (OCTEON_IS_MODEL(OCTEON_CN63XX)) { |
803 | union cvmx_mio_fus_dat3 mio_fus_dat3; | |
804 | ||
805 | mio_fus_dat3.u64 = cvmx_read_csr(CVMX_MIO_FUS_DAT3); | |
806 | /* | |
807 | * cvmx_mio_fus_dat3.s.l2c_crip fuses map as follows | |
808 | * <2> will be not used for 63xx | |
809 | * <1> disables 1/2 ways | |
810 | * <0> disables 1/4 ways | |
811 | * They are cumulative, so for 63xx: | |
812 | * <1> <0> | |
813 | * 0 0 16-way 2MB cache | |
814 | * 0 1 12-way 1.5MB cache | |
815 | * 1 0 8-way 1MB cache | |
816 | * 1 1 4-way 512KB cache | |
817 | */ | |
818 | ||
819 | if (mio_fus_dat3.s.l2c_crip == 3) | |
820 | l2_assoc = 4; | |
821 | else if (mio_fus_dat3.s.l2c_crip == 2) | |
822 | l2_assoc = 8; | |
823 | else if (mio_fus_dat3.s.l2c_crip == 1) | |
824 | l2_assoc = 12; | |
825 | } else { | |
826 | union cvmx_l2d_fus3 val; | |
827 | val.u64 = cvmx_read_csr(CVMX_L2D_FUS3); | |
828 | /* | |
829 | * Using shifts here, as bit position names are | |
830 | * different for each model but they all mean the | |
831 | * same. | |
832 | */ | |
833 | if ((val.u64 >> 35) & 0x1) | |
834 | l2_assoc = l2_assoc >> 2; | |
835 | else if ((val.u64 >> 34) & 0x1) | |
836 | l2_assoc = l2_assoc >> 1; | |
837 | } | |
58f07778 DD |
838 | return l2_assoc; |
839 | } | |
840 | ||
841 | /** | |
842 | * Flush a line from the L2 cache | |
843 | * This should only be called from one core at a time, as this routine | |
844 | * sets the core to the 'debug' core in order to flush the line. | |
845 | * | |
846 | * @assoc: Association (or way) to flush | |
847 | * @index: Index to flush | |
848 | */ | |
849 | void cvmx_l2c_flush_line(uint32_t assoc, uint32_t index) | |
850 | { | |
b8db85b5 DD |
851 | /* Check the range of the index. */ |
852 | if (index > (uint32_t)cvmx_l2c_get_num_sets()) { | |
853 | cvmx_dprintf("ERROR: cvmx_l2c_flush_line index out of range.\n"); | |
854 | return; | |
855 | } | |
58f07778 | 856 | |
b8db85b5 DD |
857 | /* Check the range of association. */ |
858 | if (assoc > (uint32_t)cvmx_l2c_get_num_assoc()) { | |
859 | cvmx_dprintf("ERROR: cvmx_l2c_flush_line association out of range.\n"); | |
860 | return; | |
861 | } | |
58f07778 | 862 | |
b8db85b5 DD |
863 | if (OCTEON_IS_MODEL(OCTEON_CN63XX)) { |
864 | uint64_t address; | |
865 | /* Create the address based on index and association. | |
866 | * Bits<20:17> select the way of the cache block involved in | |
867 | * the operation | |
868 | * Bits<16:7> of the effect address select the index | |
869 | */ | |
870 | address = CVMX_ADD_SEG(CVMX_MIPS_SPACE_XKPHYS, | |
871 | (assoc << CVMX_L2C_TAG_ADDR_ALIAS_SHIFT) | | |
872 | (index << CVMX_L2C_IDX_ADDR_SHIFT)); | |
873 | CVMX_CACHE_WBIL2I(address, 0); | |
874 | } else { | |
875 | union cvmx_l2c_dbg l2cdbg; | |
876 | ||
877 | l2cdbg.u64 = 0; | |
878 | if (!OCTEON_IS_MODEL(OCTEON_CN30XX)) | |
879 | l2cdbg.s.ppnum = cvmx_get_core_num(); | |
880 | l2cdbg.s.finv = 1; | |
881 | ||
882 | l2cdbg.s.set = assoc; | |
883 | cvmx_spinlock_lock(&cvmx_l2c_spinlock); | |
884 | /* | |
885 | * Enter debug mode, and make sure all other writes | |
886 | * complete before we enter debug mode | |
887 | */ | |
888 | CVMX_SYNC; | |
889 | cvmx_write_csr(CVMX_L2C_DBG, l2cdbg.u64); | |
890 | cvmx_read_csr(CVMX_L2C_DBG); | |
891 | ||
892 | CVMX_PREPARE_FOR_STORE(CVMX_ADD_SEG(CVMX_MIPS_SPACE_XKPHYS, | |
893 | index * CVMX_CACHE_LINE_SIZE), | |
894 | 0); | |
895 | /* Exit debug mode */ | |
896 | CVMX_SYNC; | |
897 | cvmx_write_csr(CVMX_L2C_DBG, 0); | |
898 | cvmx_read_csr(CVMX_L2C_DBG); | |
899 | cvmx_spinlock_unlock(&cvmx_l2c_spinlock); | |
900 | } | |
58f07778 | 901 | } |