Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | /* |
2 | * This file is subject to the terms and conditions of the GNU General Public | |
3 | * License. See the file "COPYING" in the main directory of this archive | |
4 | * for more details. | |
5 | * | |
6 | * Synthesize TLB refill handlers at runtime. | |
7 | * | |
e30ec452 | 8 | * Copyright (C) 2004, 2005, 2006, 2008 Thiemo Seufer |
619b6e18 | 9 | * Copyright (C) 2005, 2007 Maciej W. Rozycki |
41c594ab RB |
10 | * Copyright (C) 2006 Ralf Baechle (ralf@linux-mips.org) |
11 | * | |
12 | * ... and the days got worse and worse and now you see | |
13 | * I've gone completly out of my mind. | |
14 | * | |
15 | * They're coming to take me a away haha | |
16 | * they're coming to take me a away hoho hihi haha | |
17 | * to the funny farm where code is beautiful all the time ... | |
18 | * | |
19 | * (Condolences to Napoleon XIV) | |
1da177e4 LT |
20 | */ |
21 | ||
1da177e4 LT |
22 | #include <linux/kernel.h> |
23 | #include <linux/types.h> | |
24 | #include <linux/string.h> | |
25 | #include <linux/init.h> | |
26 | ||
1da177e4 | 27 | #include <asm/mmu_context.h> |
1da177e4 LT |
28 | #include <asm/war.h> |
29 | ||
e30ec452 TS |
30 | #include "uasm.h" |
31 | ||
aeffdbba | 32 | static inline int r45k_bvahwbug(void) |
1da177e4 LT |
33 | { |
34 | /* XXX: We should probe for the presence of this bug, but we don't. */ | |
35 | return 0; | |
36 | } | |
37 | ||
aeffdbba | 38 | static inline int r4k_250MHZhwbug(void) |
1da177e4 LT |
39 | { |
40 | /* XXX: We should probe for the presence of this bug, but we don't. */ | |
41 | return 0; | |
42 | } | |
43 | ||
aeffdbba | 44 | static inline int __maybe_unused bcm1250_m3_war(void) |
1da177e4 LT |
45 | { |
46 | return BCM1250_M3_WAR; | |
47 | } | |
48 | ||
aeffdbba | 49 | static inline int __maybe_unused r10000_llsc_war(void) |
1da177e4 LT |
50 | { |
51 | return R10000_LLSC_WAR; | |
52 | } | |
53 | ||
8df5beac MR |
54 | /* |
55 | * Found by experiment: At least some revisions of the 4kc throw under | |
56 | * some circumstances a machine check exception, triggered by invalid | |
57 | * values in the index register. Delaying the tlbp instruction until | |
58 | * after the next branch, plus adding an additional nop in front of | |
59 | * tlbwi/tlbwr avoids the invalid index register values. Nobody knows | |
60 | * why; it's not an issue caused by the core RTL. | |
61 | * | |
62 | */ | |
a9af6041 | 63 | static int __init m4kc_tlbp_war(void) |
8df5beac MR |
64 | { |
65 | return (current_cpu_data.processor_id & 0xffff00) == | |
66 | (PRID_COMP_MIPS | PRID_IMP_4KC); | |
67 | } | |
68 | ||
e30ec452 | 69 | /* Handle labels (which must be positive integers). */ |
1da177e4 | 70 | enum label_id { |
e30ec452 | 71 | label_second_part = 1, |
1da177e4 | 72 | label_leave, |
656be92f AN |
73 | #ifdef MODULE_START |
74 | label_module_alloc, | |
75 | #endif | |
1da177e4 LT |
76 | label_vmalloc, |
77 | label_vmalloc_done, | |
78 | label_tlbw_hazard, | |
79 | label_split, | |
80 | label_nopage_tlbl, | |
81 | label_nopage_tlbs, | |
82 | label_nopage_tlbm, | |
83 | label_smp_pgtable_change, | |
84 | label_r3000_write_probe_fail, | |
1da177e4 LT |
85 | }; |
86 | ||
e30ec452 TS |
87 | UASM_L_LA(_second_part) |
88 | UASM_L_LA(_leave) | |
656be92f | 89 | #ifdef MODULE_START |
e30ec452 | 90 | UASM_L_LA(_module_alloc) |
619b6e18 | 91 | #endif |
e30ec452 TS |
92 | UASM_L_LA(_vmalloc) |
93 | UASM_L_LA(_vmalloc_done) | |
94 | UASM_L_LA(_tlbw_hazard) | |
95 | UASM_L_LA(_split) | |
96 | UASM_L_LA(_nopage_tlbl) | |
97 | UASM_L_LA(_nopage_tlbs) | |
98 | UASM_L_LA(_nopage_tlbm) | |
99 | UASM_L_LA(_smp_pgtable_change) | |
100 | UASM_L_LA(_r3000_write_probe_fail) | |
656be92f | 101 | |
92b1e6a6 FBH |
102 | /* |
103 | * For debug purposes. | |
104 | */ | |
105 | static inline void dump_handler(const u32 *handler, int count) | |
106 | { | |
107 | int i; | |
108 | ||
109 | pr_debug("\t.set push\n"); | |
110 | pr_debug("\t.set noreorder\n"); | |
111 | ||
112 | for (i = 0; i < count; i++) | |
113 | pr_debug("\t%p\t.word 0x%08x\n", &handler[i], handler[i]); | |
114 | ||
115 | pr_debug("\t.set pop\n"); | |
116 | } | |
117 | ||
1da177e4 LT |
118 | /* The only general purpose registers allowed in TLB handlers. */ |
119 | #define K0 26 | |
120 | #define K1 27 | |
121 | ||
122 | /* Some CP0 registers */ | |
41c594ab RB |
123 | #define C0_INDEX 0, 0 |
124 | #define C0_ENTRYLO0 2, 0 | |
125 | #define C0_TCBIND 2, 2 | |
126 | #define C0_ENTRYLO1 3, 0 | |
127 | #define C0_CONTEXT 4, 0 | |
128 | #define C0_BADVADDR 8, 0 | |
129 | #define C0_ENTRYHI 10, 0 | |
130 | #define C0_EPC 14, 0 | |
131 | #define C0_XCONTEXT 20, 0 | |
1da177e4 | 132 | |
875d43e7 | 133 | #ifdef CONFIG_64BIT |
e30ec452 | 134 | # define GET_CONTEXT(buf, reg) UASM_i_MFC0(buf, reg, C0_XCONTEXT) |
1da177e4 | 135 | #else |
e30ec452 | 136 | # define GET_CONTEXT(buf, reg) UASM_i_MFC0(buf, reg, C0_CONTEXT) |
1da177e4 LT |
137 | #endif |
138 | ||
139 | /* The worst case length of the handler is around 18 instructions for | |
140 | * R3000-style TLBs and up to 63 instructions for R4000-style TLBs. | |
141 | * Maximum space available is 32 instructions for R3000 and 64 | |
142 | * instructions for R4000. | |
143 | * | |
144 | * We deliberately chose a buffer size of 128, so we won't scribble | |
145 | * over anything important on overflow before we panic. | |
146 | */ | |
a9af6041 | 147 | static u32 tlb_handler[128] __initdata; |
1da177e4 LT |
148 | |
149 | /* simply assume worst case size for labels and relocs */ | |
e30ec452 TS |
150 | static struct uasm_label labels[128] __initdata; |
151 | static struct uasm_reloc relocs[128] __initdata; | |
1da177e4 LT |
152 | |
153 | /* | |
154 | * The R3000 TLB handler is simple. | |
155 | */ | |
156 | static void __init build_r3000_tlb_refill_handler(void) | |
157 | { | |
158 | long pgdc = (long)pgd_current; | |
159 | u32 *p; | |
160 | ||
161 | memset(tlb_handler, 0, sizeof(tlb_handler)); | |
162 | p = tlb_handler; | |
163 | ||
e30ec452 TS |
164 | uasm_i_mfc0(&p, K0, C0_BADVADDR); |
165 | uasm_i_lui(&p, K1, uasm_rel_hi(pgdc)); /* cp0 delay */ | |
166 | uasm_i_lw(&p, K1, uasm_rel_lo(pgdc), K1); | |
167 | uasm_i_srl(&p, K0, K0, 22); /* load delay */ | |
168 | uasm_i_sll(&p, K0, K0, 2); | |
169 | uasm_i_addu(&p, K1, K1, K0); | |
170 | uasm_i_mfc0(&p, K0, C0_CONTEXT); | |
171 | uasm_i_lw(&p, K1, 0, K1); /* cp0 delay */ | |
172 | uasm_i_andi(&p, K0, K0, 0xffc); /* load delay */ | |
173 | uasm_i_addu(&p, K1, K1, K0); | |
174 | uasm_i_lw(&p, K0, 0, K1); | |
175 | uasm_i_nop(&p); /* load delay */ | |
176 | uasm_i_mtc0(&p, K0, C0_ENTRYLO0); | |
177 | uasm_i_mfc0(&p, K1, C0_EPC); /* cp0 delay */ | |
178 | uasm_i_tlbwr(&p); /* cp0 delay */ | |
179 | uasm_i_jr(&p, K1); | |
180 | uasm_i_rfe(&p); /* branch delay */ | |
1da177e4 LT |
181 | |
182 | if (p > tlb_handler + 32) | |
183 | panic("TLB refill handler space exceeded"); | |
184 | ||
e30ec452 TS |
185 | pr_debug("Wrote TLB refill handler (%u instructions).\n", |
186 | (unsigned int)(p - tlb_handler)); | |
1da177e4 | 187 | |
91b05e67 | 188 | memcpy((void *)ebase, tlb_handler, 0x80); |
92b1e6a6 FBH |
189 | |
190 | dump_handler((u32 *)ebase, 32); | |
1da177e4 LT |
191 | } |
192 | ||
193 | /* | |
194 | * The R4000 TLB handler is much more complicated. We have two | |
195 | * consecutive handler areas with 32 instructions space each. | |
196 | * Since they aren't used at the same time, we can overflow in the | |
197 | * other one.To keep things simple, we first assume linear space, | |
198 | * then we relocate it to the final handler layout as needed. | |
199 | */ | |
a9af6041 | 200 | static u32 final_handler[64] __initdata; |
1da177e4 LT |
201 | |
202 | /* | |
203 | * Hazards | |
204 | * | |
205 | * From the IDT errata for the QED RM5230 (Nevada), processor revision 1.0: | |
206 | * 2. A timing hazard exists for the TLBP instruction. | |
207 | * | |
208 | * stalling_instruction | |
209 | * TLBP | |
210 | * | |
211 | * The JTLB is being read for the TLBP throughout the stall generated by the | |
212 | * previous instruction. This is not really correct as the stalling instruction | |
213 | * can modify the address used to access the JTLB. The failure symptom is that | |
214 | * the TLBP instruction will use an address created for the stalling instruction | |
215 | * and not the address held in C0_ENHI and thus report the wrong results. | |
216 | * | |
217 | * The software work-around is to not allow the instruction preceding the TLBP | |
218 | * to stall - make it an NOP or some other instruction guaranteed not to stall. | |
219 | * | |
220 | * Errata 2 will not be fixed. This errata is also on the R5000. | |
221 | * | |
222 | * As if we MIPS hackers wouldn't know how to nop pipelines happy ... | |
223 | */ | |
a9af6041 | 224 | static void __init __maybe_unused build_tlb_probe_entry(u32 **p) |
1da177e4 | 225 | { |
10cc3529 | 226 | switch (current_cpu_type()) { |
f5b4d956 TS |
227 | /* Found by experiment: R4600 v2.0 needs this, too. */ |
228 | case CPU_R4600: | |
1da177e4 LT |
229 | case CPU_R5000: |
230 | case CPU_R5000A: | |
231 | case CPU_NEVADA: | |
e30ec452 TS |
232 | uasm_i_nop(p); |
233 | uasm_i_tlbp(p); | |
1da177e4 LT |
234 | break; |
235 | ||
236 | default: | |
e30ec452 | 237 | uasm_i_tlbp(p); |
1da177e4 LT |
238 | break; |
239 | } | |
240 | } | |
241 | ||
242 | /* | |
243 | * Write random or indexed TLB entry, and care about the hazards from | |
244 | * the preceeding mtc0 and for the following eret. | |
245 | */ | |
246 | enum tlb_write_entry { tlb_random, tlb_indexed }; | |
247 | ||
e30ec452 TS |
248 | static void __init build_tlb_write_entry(u32 **p, struct uasm_label **l, |
249 | struct uasm_reloc **r, | |
1da177e4 LT |
250 | enum tlb_write_entry wmode) |
251 | { | |
252 | void(*tlbw)(u32 **) = NULL; | |
253 | ||
254 | switch (wmode) { | |
e30ec452 TS |
255 | case tlb_random: tlbw = uasm_i_tlbwr; break; |
256 | case tlb_indexed: tlbw = uasm_i_tlbwi; break; | |
1da177e4 LT |
257 | } |
258 | ||
161548bf | 259 | if (cpu_has_mips_r2) { |
e30ec452 | 260 | uasm_i_ehb(p); |
161548bf RB |
261 | tlbw(p); |
262 | return; | |
263 | } | |
264 | ||
10cc3529 | 265 | switch (current_cpu_type()) { |
1da177e4 LT |
266 | case CPU_R4000PC: |
267 | case CPU_R4000SC: | |
268 | case CPU_R4000MC: | |
269 | case CPU_R4400PC: | |
270 | case CPU_R4400SC: | |
271 | case CPU_R4400MC: | |
272 | /* | |
273 | * This branch uses up a mtc0 hazard nop slot and saves | |
274 | * two nops after the tlbw instruction. | |
275 | */ | |
e30ec452 | 276 | uasm_il_bgezl(p, r, 0, label_tlbw_hazard); |
1da177e4 | 277 | tlbw(p); |
e30ec452 TS |
278 | uasm_l_tlbw_hazard(l, *p); |
279 | uasm_i_nop(p); | |
1da177e4 LT |
280 | break; |
281 | ||
282 | case CPU_R4600: | |
283 | case CPU_R4700: | |
284 | case CPU_R5000: | |
285 | case CPU_R5000A: | |
e30ec452 | 286 | uasm_i_nop(p); |
2c93e12c | 287 | tlbw(p); |
e30ec452 | 288 | uasm_i_nop(p); |
2c93e12c MR |
289 | break; |
290 | ||
291 | case CPU_R4300: | |
1da177e4 LT |
292 | case CPU_5KC: |
293 | case CPU_TX49XX: | |
294 | case CPU_AU1000: | |
295 | case CPU_AU1100: | |
296 | case CPU_AU1500: | |
297 | case CPU_AU1550: | |
e3ad1c23 | 298 | case CPU_AU1200: |
237cfee1 ML |
299 | case CPU_AU1210: |
300 | case CPU_AU1250: | |
bdf21b18 | 301 | case CPU_PR4450: |
e30ec452 | 302 | uasm_i_nop(p); |
1da177e4 LT |
303 | tlbw(p); |
304 | break; | |
305 | ||
306 | case CPU_R10000: | |
307 | case CPU_R12000: | |
44d921b2 | 308 | case CPU_R14000: |
1da177e4 LT |
309 | case CPU_4KC: |
310 | case CPU_SB1: | |
93ce2f52 | 311 | case CPU_SB1A: |
1da177e4 LT |
312 | case CPU_4KSC: |
313 | case CPU_20KC: | |
314 | case CPU_25KF: | |
1c0c13eb AJ |
315 | case CPU_BCM3302: |
316 | case CPU_BCM4710: | |
2a21c730 | 317 | case CPU_LOONGSON2: |
8df5beac | 318 | if (m4kc_tlbp_war()) |
e30ec452 | 319 | uasm_i_nop(p); |
1da177e4 LT |
320 | tlbw(p); |
321 | break; | |
322 | ||
323 | case CPU_NEVADA: | |
e30ec452 | 324 | uasm_i_nop(p); /* QED specifies 2 nops hazard */ |
1da177e4 LT |
325 | /* |
326 | * This branch uses up a mtc0 hazard nop slot and saves | |
327 | * a nop after the tlbw instruction. | |
328 | */ | |
e30ec452 | 329 | uasm_il_bgezl(p, r, 0, label_tlbw_hazard); |
1da177e4 | 330 | tlbw(p); |
e30ec452 | 331 | uasm_l_tlbw_hazard(l, *p); |
1da177e4 LT |
332 | break; |
333 | ||
334 | case CPU_RM7000: | |
e30ec452 TS |
335 | uasm_i_nop(p); |
336 | uasm_i_nop(p); | |
337 | uasm_i_nop(p); | |
338 | uasm_i_nop(p); | |
1da177e4 LT |
339 | tlbw(p); |
340 | break; | |
341 | ||
1da177e4 LT |
342 | case CPU_RM9000: |
343 | /* | |
344 | * When the JTLB is updated by tlbwi or tlbwr, a subsequent | |
345 | * use of the JTLB for instructions should not occur for 4 | |
346 | * cpu cycles and use for data translations should not occur | |
347 | * for 3 cpu cycles. | |
348 | */ | |
e30ec452 TS |
349 | uasm_i_ssnop(p); |
350 | uasm_i_ssnop(p); | |
351 | uasm_i_ssnop(p); | |
352 | uasm_i_ssnop(p); | |
1da177e4 | 353 | tlbw(p); |
e30ec452 TS |
354 | uasm_i_ssnop(p); |
355 | uasm_i_ssnop(p); | |
356 | uasm_i_ssnop(p); | |
357 | uasm_i_ssnop(p); | |
1da177e4 LT |
358 | break; |
359 | ||
360 | case CPU_VR4111: | |
361 | case CPU_VR4121: | |
362 | case CPU_VR4122: | |
363 | case CPU_VR4181: | |
364 | case CPU_VR4181A: | |
e30ec452 TS |
365 | uasm_i_nop(p); |
366 | uasm_i_nop(p); | |
1da177e4 | 367 | tlbw(p); |
e30ec452 TS |
368 | uasm_i_nop(p); |
369 | uasm_i_nop(p); | |
1da177e4 LT |
370 | break; |
371 | ||
372 | case CPU_VR4131: | |
373 | case CPU_VR4133: | |
7623debf | 374 | case CPU_R5432: |
e30ec452 TS |
375 | uasm_i_nop(p); |
376 | uasm_i_nop(p); | |
1da177e4 LT |
377 | tlbw(p); |
378 | break; | |
379 | ||
380 | default: | |
381 | panic("No TLB refill handler yet (CPU type: %d)", | |
382 | current_cpu_data.cputype); | |
383 | break; | |
384 | } | |
385 | } | |
386 | ||
875d43e7 | 387 | #ifdef CONFIG_64BIT |
1da177e4 LT |
388 | /* |
389 | * TMP and PTR are scratch. | |
390 | * TMP will be clobbered, PTR will hold the pmd entry. | |
391 | */ | |
a9af6041 | 392 | static void __init |
e30ec452 | 393 | build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r, |
1da177e4 LT |
394 | unsigned int tmp, unsigned int ptr) |
395 | { | |
396 | long pgdc = (long)pgd_current; | |
397 | ||
398 | /* | |
399 | * The vmalloc handling is not in the hotpath. | |
400 | */ | |
e30ec452 | 401 | uasm_i_dmfc0(p, tmp, C0_BADVADDR); |
656be92f | 402 | #ifdef MODULE_START |
e30ec452 | 403 | uasm_il_bltz(p, r, tmp, label_module_alloc); |
656be92f | 404 | #else |
e30ec452 | 405 | uasm_il_bltz(p, r, tmp, label_vmalloc); |
656be92f | 406 | #endif |
e30ec452 | 407 | /* No uasm_i_nop needed here, since the next insn doesn't touch TMP. */ |
1da177e4 LT |
408 | |
409 | #ifdef CONFIG_SMP | |
41c594ab RB |
410 | # ifdef CONFIG_MIPS_MT_SMTC |
411 | /* | |
412 | * SMTC uses TCBind value as "CPU" index | |
413 | */ | |
e30ec452 TS |
414 | uasm_i_mfc0(p, ptr, C0_TCBIND); |
415 | uasm_i_dsrl(p, ptr, ptr, 19); | |
41c594ab | 416 | # else |
1da177e4 | 417 | /* |
1b3a6e97 | 418 | * 64 bit SMP running in XKPHYS has smp_processor_id() << 3 |
1da177e4 LT |
419 | * stored in CONTEXT. |
420 | */ | |
e30ec452 TS |
421 | uasm_i_dmfc0(p, ptr, C0_CONTEXT); |
422 | uasm_i_dsrl(p, ptr, ptr, 23); | |
41c594ab | 423 | #endif |
e30ec452 TS |
424 | UASM_i_LA_mostly(p, tmp, pgdc); |
425 | uasm_i_daddu(p, ptr, ptr, tmp); | |
426 | uasm_i_dmfc0(p, tmp, C0_BADVADDR); | |
427 | uasm_i_ld(p, ptr, uasm_rel_lo(pgdc), ptr); | |
1da177e4 | 428 | #else |
e30ec452 TS |
429 | UASM_i_LA_mostly(p, ptr, pgdc); |
430 | uasm_i_ld(p, ptr, uasm_rel_lo(pgdc), ptr); | |
1da177e4 LT |
431 | #endif |
432 | ||
e30ec452 | 433 | uasm_l_vmalloc_done(l, *p); |
242954b5 RB |
434 | |
435 | if (PGDIR_SHIFT - 3 < 32) /* get pgd offset in bytes */ | |
e30ec452 | 436 | uasm_i_dsrl(p, tmp, tmp, PGDIR_SHIFT-3); |
242954b5 | 437 | else |
e30ec452 TS |
438 | uasm_i_dsrl32(p, tmp, tmp, PGDIR_SHIFT - 3 - 32); |
439 | ||
440 | uasm_i_andi(p, tmp, tmp, (PTRS_PER_PGD - 1)<<3); | |
441 | uasm_i_daddu(p, ptr, ptr, tmp); /* add in pgd offset */ | |
442 | uasm_i_dmfc0(p, tmp, C0_BADVADDR); /* get faulting address */ | |
443 | uasm_i_ld(p, ptr, 0, ptr); /* get pmd pointer */ | |
444 | uasm_i_dsrl(p, tmp, tmp, PMD_SHIFT-3); /* get pmd offset in bytes */ | |
445 | uasm_i_andi(p, tmp, tmp, (PTRS_PER_PMD - 1)<<3); | |
446 | uasm_i_daddu(p, ptr, ptr, tmp); /* add in pmd offset */ | |
1da177e4 LT |
447 | } |
448 | ||
449 | /* | |
450 | * BVADDR is the faulting address, PTR is scratch. | |
451 | * PTR will hold the pgd for vmalloc. | |
452 | */ | |
a9af6041 | 453 | static void __init |
e30ec452 | 454 | build_get_pgd_vmalloc64(u32 **p, struct uasm_label **l, struct uasm_reloc **r, |
1da177e4 LT |
455 | unsigned int bvaddr, unsigned int ptr) |
456 | { | |
457 | long swpd = (long)swapper_pg_dir; | |
458 | ||
656be92f AN |
459 | #ifdef MODULE_START |
460 | long modd = (long)module_pg_dir; | |
461 | ||
e30ec452 | 462 | uasm_l_module_alloc(l, *p); |
656be92f AN |
463 | /* |
464 | * Assumption: | |
465 | * VMALLOC_START >= 0xc000000000000000UL | |
466 | * MODULE_START >= 0xe000000000000000UL | |
467 | */ | |
e30ec452 TS |
468 | UASM_i_SLL(p, ptr, bvaddr, 2); |
469 | uasm_il_bgez(p, r, ptr, label_vmalloc); | |
656be92f | 470 | |
e30ec452 TS |
471 | if (uasm_in_compat_space_p(MODULE_START) && |
472 | !uasm_rel_lo(MODULE_START)) { | |
473 | uasm_i_lui(p, ptr, uasm_rel_hi(MODULE_START)); /* delay slot */ | |
656be92f AN |
474 | } else { |
475 | /* unlikely configuration */ | |
e30ec452 TS |
476 | uasm_i_nop(p); /* delay slot */ |
477 | UASM_i_LA(p, ptr, MODULE_START); | |
656be92f | 478 | } |
e30ec452 | 479 | uasm_i_dsubu(p, bvaddr, bvaddr, ptr); |
656be92f | 480 | |
e30ec452 TS |
481 | if (uasm_in_compat_space_p(modd) && !uasm_rel_lo(modd)) { |
482 | uasm_il_b(p, r, label_vmalloc_done); | |
483 | uasm_i_lui(p, ptr, uasm_rel_hi(modd)); | |
656be92f | 484 | } else { |
e30ec452 TS |
485 | UASM_i_LA_mostly(p, ptr, modd); |
486 | uasm_il_b(p, r, label_vmalloc_done); | |
487 | if (uasm_in_compat_space_p(modd)) | |
488 | uasm_i_addiu(p, ptr, ptr, uasm_rel_lo(modd)); | |
619b6e18 | 489 | else |
e30ec452 | 490 | uasm_i_daddiu(p, ptr, ptr, uasm_rel_lo(modd)); |
656be92f AN |
491 | } |
492 | ||
e30ec452 TS |
493 | uasm_l_vmalloc(l, *p); |
494 | if (uasm_in_compat_space_p(MODULE_START) && | |
495 | !uasm_rel_lo(MODULE_START) && | |
656be92f | 496 | MODULE_START << 32 == VMALLOC_START) |
e30ec452 | 497 | uasm_i_dsll32(p, ptr, ptr, 0); /* typical case */ |
656be92f | 498 | else |
e30ec452 | 499 | UASM_i_LA(p, ptr, VMALLOC_START); |
656be92f | 500 | #else |
e30ec452 TS |
501 | uasm_l_vmalloc(l, *p); |
502 | UASM_i_LA(p, ptr, VMALLOC_START); | |
656be92f | 503 | #endif |
e30ec452 | 504 | uasm_i_dsubu(p, bvaddr, bvaddr, ptr); |
1da177e4 | 505 | |
e30ec452 TS |
506 | if (uasm_in_compat_space_p(swpd) && !uasm_rel_lo(swpd)) { |
507 | uasm_il_b(p, r, label_vmalloc_done); | |
508 | uasm_i_lui(p, ptr, uasm_rel_hi(swpd)); | |
1da177e4 | 509 | } else { |
e30ec452 TS |
510 | UASM_i_LA_mostly(p, ptr, swpd); |
511 | uasm_il_b(p, r, label_vmalloc_done); | |
512 | if (uasm_in_compat_space_p(swpd)) | |
513 | uasm_i_addiu(p, ptr, ptr, uasm_rel_lo(swpd)); | |
619b6e18 | 514 | else |
e30ec452 | 515 | uasm_i_daddiu(p, ptr, ptr, uasm_rel_lo(swpd)); |
1da177e4 LT |
516 | } |
517 | } | |
518 | ||
875d43e7 | 519 | #else /* !CONFIG_64BIT */ |
1da177e4 LT |
520 | |
521 | /* | |
522 | * TMP and PTR are scratch. | |
523 | * TMP will be clobbered, PTR will hold the pgd entry. | |
524 | */ | |
a9af6041 | 525 | static void __init __maybe_unused |
1da177e4 LT |
526 | build_get_pgde32(u32 **p, unsigned int tmp, unsigned int ptr) |
527 | { | |
528 | long pgdc = (long)pgd_current; | |
529 | ||
530 | /* 32 bit SMP has smp_processor_id() stored in CONTEXT. */ | |
531 | #ifdef CONFIG_SMP | |
41c594ab RB |
532 | #ifdef CONFIG_MIPS_MT_SMTC |
533 | /* | |
534 | * SMTC uses TCBind value as "CPU" index | |
535 | */ | |
e30ec452 TS |
536 | uasm_i_mfc0(p, ptr, C0_TCBIND); |
537 | UASM_i_LA_mostly(p, tmp, pgdc); | |
538 | uasm_i_srl(p, ptr, ptr, 19); | |
41c594ab RB |
539 | #else |
540 | /* | |
541 | * smp_processor_id() << 3 is stored in CONTEXT. | |
542 | */ | |
e30ec452 TS |
543 | uasm_i_mfc0(p, ptr, C0_CONTEXT); |
544 | UASM_i_LA_mostly(p, tmp, pgdc); | |
545 | uasm_i_srl(p, ptr, ptr, 23); | |
41c594ab | 546 | #endif |
e30ec452 | 547 | uasm_i_addu(p, ptr, tmp, ptr); |
1da177e4 | 548 | #else |
e30ec452 | 549 | UASM_i_LA_mostly(p, ptr, pgdc); |
1da177e4 | 550 | #endif |
e30ec452 TS |
551 | uasm_i_mfc0(p, tmp, C0_BADVADDR); /* get faulting address */ |
552 | uasm_i_lw(p, ptr, uasm_rel_lo(pgdc), ptr); | |
553 | uasm_i_srl(p, tmp, tmp, PGDIR_SHIFT); /* get pgd only bits */ | |
554 | uasm_i_sll(p, tmp, tmp, PGD_T_LOG2); | |
555 | uasm_i_addu(p, ptr, ptr, tmp); /* add in pgd offset */ | |
1da177e4 LT |
556 | } |
557 | ||
875d43e7 | 558 | #endif /* !CONFIG_64BIT */ |
1da177e4 | 559 | |
a9af6041 | 560 | static void __init build_adjust_context(u32 **p, unsigned int ctx) |
1da177e4 | 561 | { |
242954b5 | 562 | unsigned int shift = 4 - (PTE_T_LOG2 + 1) + PAGE_SHIFT - 12; |
1da177e4 LT |
563 | unsigned int mask = (PTRS_PER_PTE / 2 - 1) << (PTE_T_LOG2 + 1); |
564 | ||
10cc3529 | 565 | switch (current_cpu_type()) { |
1da177e4 LT |
566 | case CPU_VR41XX: |
567 | case CPU_VR4111: | |
568 | case CPU_VR4121: | |
569 | case CPU_VR4122: | |
570 | case CPU_VR4131: | |
571 | case CPU_VR4181: | |
572 | case CPU_VR4181A: | |
573 | case CPU_VR4133: | |
574 | shift += 2; | |
575 | break; | |
576 | ||
577 | default: | |
578 | break; | |
579 | } | |
580 | ||
581 | if (shift) | |
e30ec452 TS |
582 | UASM_i_SRL(p, ctx, ctx, shift); |
583 | uasm_i_andi(p, ctx, ctx, mask); | |
1da177e4 LT |
584 | } |
585 | ||
a9af6041 | 586 | static void __init build_get_ptep(u32 **p, unsigned int tmp, unsigned int ptr) |
1da177e4 LT |
587 | { |
588 | /* | |
589 | * Bug workaround for the Nevada. It seems as if under certain | |
590 | * circumstances the move from cp0_context might produce a | |
591 | * bogus result when the mfc0 instruction and its consumer are | |
592 | * in a different cacheline or a load instruction, probably any | |
593 | * memory reference, is between them. | |
594 | */ | |
10cc3529 | 595 | switch (current_cpu_type()) { |
1da177e4 | 596 | case CPU_NEVADA: |
e30ec452 | 597 | UASM_i_LW(p, ptr, 0, ptr); |
1da177e4 LT |
598 | GET_CONTEXT(p, tmp); /* get context reg */ |
599 | break; | |
600 | ||
601 | default: | |
602 | GET_CONTEXT(p, tmp); /* get context reg */ | |
e30ec452 | 603 | UASM_i_LW(p, ptr, 0, ptr); |
1da177e4 LT |
604 | break; |
605 | } | |
606 | ||
607 | build_adjust_context(p, tmp); | |
e30ec452 | 608 | UASM_i_ADDU(p, ptr, ptr, tmp); /* add in offset */ |
1da177e4 LT |
609 | } |
610 | ||
a9af6041 | 611 | static void __init build_update_entries(u32 **p, unsigned int tmp, |
1da177e4 LT |
612 | unsigned int ptep) |
613 | { | |
614 | /* | |
615 | * 64bit address support (36bit on a 32bit CPU) in a 32bit | |
616 | * Kernel is a special case. Only a few CPUs use it. | |
617 | */ | |
618 | #ifdef CONFIG_64BIT_PHYS_ADDR | |
619 | if (cpu_has_64bits) { | |
e30ec452 TS |
620 | uasm_i_ld(p, tmp, 0, ptep); /* get even pte */ |
621 | uasm_i_ld(p, ptep, sizeof(pte_t), ptep); /* get odd pte */ | |
622 | uasm_i_dsrl(p, tmp, tmp, 6); /* convert to entrylo0 */ | |
623 | uasm_i_mtc0(p, tmp, C0_ENTRYLO0); /* load it */ | |
624 | uasm_i_dsrl(p, ptep, ptep, 6); /* convert to entrylo1 */ | |
625 | uasm_i_mtc0(p, ptep, C0_ENTRYLO1); /* load it */ | |
1da177e4 LT |
626 | } else { |
627 | int pte_off_even = sizeof(pte_t) / 2; | |
628 | int pte_off_odd = pte_off_even + sizeof(pte_t); | |
629 | ||
630 | /* The pte entries are pre-shifted */ | |
e30ec452 TS |
631 | uasm_i_lw(p, tmp, pte_off_even, ptep); /* get even pte */ |
632 | uasm_i_mtc0(p, tmp, C0_ENTRYLO0); /* load it */ | |
633 | uasm_i_lw(p, ptep, pte_off_odd, ptep); /* get odd pte */ | |
634 | uasm_i_mtc0(p, ptep, C0_ENTRYLO1); /* load it */ | |
1da177e4 LT |
635 | } |
636 | #else | |
e30ec452 TS |
637 | UASM_i_LW(p, tmp, 0, ptep); /* get even pte */ |
638 | UASM_i_LW(p, ptep, sizeof(pte_t), ptep); /* get odd pte */ | |
1da177e4 LT |
639 | if (r45k_bvahwbug()) |
640 | build_tlb_probe_entry(p); | |
e30ec452 | 641 | UASM_i_SRL(p, tmp, tmp, 6); /* convert to entrylo0 */ |
1da177e4 | 642 | if (r4k_250MHZhwbug()) |
e30ec452 TS |
643 | uasm_i_mtc0(p, 0, C0_ENTRYLO0); |
644 | uasm_i_mtc0(p, tmp, C0_ENTRYLO0); /* load it */ | |
645 | UASM_i_SRL(p, ptep, ptep, 6); /* convert to entrylo1 */ | |
1da177e4 | 646 | if (r45k_bvahwbug()) |
e30ec452 | 647 | uasm_i_mfc0(p, tmp, C0_INDEX); |
1da177e4 | 648 | if (r4k_250MHZhwbug()) |
e30ec452 TS |
649 | uasm_i_mtc0(p, 0, C0_ENTRYLO1); |
650 | uasm_i_mtc0(p, ptep, C0_ENTRYLO1); /* load it */ | |
1da177e4 LT |
651 | #endif |
652 | } | |
653 | ||
654 | static void __init build_r4000_tlb_refill_handler(void) | |
655 | { | |
656 | u32 *p = tlb_handler; | |
e30ec452 TS |
657 | struct uasm_label *l = labels; |
658 | struct uasm_reloc *r = relocs; | |
1da177e4 LT |
659 | u32 *f; |
660 | unsigned int final_len; | |
661 | ||
662 | memset(tlb_handler, 0, sizeof(tlb_handler)); | |
663 | memset(labels, 0, sizeof(labels)); | |
664 | memset(relocs, 0, sizeof(relocs)); | |
665 | memset(final_handler, 0, sizeof(final_handler)); | |
666 | ||
667 | /* | |
668 | * create the plain linear handler | |
669 | */ | |
670 | if (bcm1250_m3_war()) { | |
e30ec452 TS |
671 | UASM_i_MFC0(&p, K0, C0_BADVADDR); |
672 | UASM_i_MFC0(&p, K1, C0_ENTRYHI); | |
673 | uasm_i_xor(&p, K0, K0, K1); | |
674 | UASM_i_SRL(&p, K0, K0, PAGE_SHIFT + 1); | |
675 | uasm_il_bnez(&p, &r, K0, label_leave); | |
676 | /* No need for uasm_i_nop */ | |
1da177e4 LT |
677 | } |
678 | ||
875d43e7 | 679 | #ifdef CONFIG_64BIT |
1da177e4 LT |
680 | build_get_pmde64(&p, &l, &r, K0, K1); /* get pmd in K1 */ |
681 | #else | |
682 | build_get_pgde32(&p, K0, K1); /* get pgd in K1 */ | |
683 | #endif | |
684 | ||
685 | build_get_ptep(&p, K0, K1); | |
686 | build_update_entries(&p, K0, K1); | |
687 | build_tlb_write_entry(&p, &l, &r, tlb_random); | |
e30ec452 TS |
688 | uasm_l_leave(&l, p); |
689 | uasm_i_eret(&p); /* return from trap */ | |
1da177e4 | 690 | |
875d43e7 | 691 | #ifdef CONFIG_64BIT |
1da177e4 LT |
692 | build_get_pgd_vmalloc64(&p, &l, &r, K0, K1); |
693 | #endif | |
694 | ||
695 | /* | |
696 | * Overflow check: For the 64bit handler, we need at least one | |
697 | * free instruction slot for the wrap-around branch. In worst | |
698 | * case, if the intended insertion point is a delay slot, we | |
4b3f686d | 699 | * need three, with the second nop'ed and the third being |
1da177e4 LT |
700 | * unused. |
701 | */ | |
2a21c730 FZ |
702 | /* Loongson2 ebase is different than r4k, we have more space */ |
703 | #if defined(CONFIG_32BIT) || defined(CONFIG_CPU_LOONGSON2) | |
1da177e4 LT |
704 | if ((p - tlb_handler) > 64) |
705 | panic("TLB refill handler space exceeded"); | |
706 | #else | |
707 | if (((p - tlb_handler) > 63) | |
708 | || (((p - tlb_handler) > 61) | |
e30ec452 | 709 | && uasm_insn_has_bdelay(relocs, tlb_handler + 29))) |
1da177e4 LT |
710 | panic("TLB refill handler space exceeded"); |
711 | #endif | |
712 | ||
713 | /* | |
714 | * Now fold the handler in the TLB refill handler space. | |
715 | */ | |
2a21c730 | 716 | #if defined(CONFIG_32BIT) || defined(CONFIG_CPU_LOONGSON2) |
1da177e4 LT |
717 | f = final_handler; |
718 | /* Simplest case, just copy the handler. */ | |
e30ec452 | 719 | uasm_copy_handler(relocs, labels, tlb_handler, p, f); |
1da177e4 | 720 | final_len = p - tlb_handler; |
875d43e7 | 721 | #else /* CONFIG_64BIT */ |
1da177e4 LT |
722 | f = final_handler + 32; |
723 | if ((p - tlb_handler) <= 32) { | |
724 | /* Just copy the handler. */ | |
e30ec452 | 725 | uasm_copy_handler(relocs, labels, tlb_handler, p, f); |
1da177e4 LT |
726 | final_len = p - tlb_handler; |
727 | } else { | |
728 | u32 *split = tlb_handler + 30; | |
729 | ||
730 | /* | |
731 | * Find the split point. | |
732 | */ | |
e30ec452 | 733 | if (uasm_insn_has_bdelay(relocs, split - 1)) |
1da177e4 LT |
734 | split--; |
735 | ||
736 | /* Copy first part of the handler. */ | |
e30ec452 | 737 | uasm_copy_handler(relocs, labels, tlb_handler, split, f); |
1da177e4 LT |
738 | f += split - tlb_handler; |
739 | ||
740 | /* Insert branch. */ | |
e30ec452 TS |
741 | uasm_l_split(&l, final_handler); |
742 | uasm_il_b(&f, &r, label_split); | |
743 | if (uasm_insn_has_bdelay(relocs, split)) | |
744 | uasm_i_nop(&f); | |
1da177e4 | 745 | else { |
e30ec452 TS |
746 | uasm_copy_handler(relocs, labels, split, split + 1, f); |
747 | uasm_move_labels(labels, f, f + 1, -1); | |
1da177e4 LT |
748 | f++; |
749 | split++; | |
750 | } | |
751 | ||
752 | /* Copy the rest of the handler. */ | |
e30ec452 | 753 | uasm_copy_handler(relocs, labels, split, p, final_handler); |
1da177e4 LT |
754 | final_len = (f - (final_handler + 32)) + (p - split); |
755 | } | |
875d43e7 | 756 | #endif /* CONFIG_64BIT */ |
1da177e4 | 757 | |
e30ec452 TS |
758 | uasm_resolve_relocs(relocs, labels); |
759 | pr_debug("Wrote TLB refill handler (%u instructions).\n", | |
760 | final_len); | |
1da177e4 | 761 | |
91b05e67 | 762 | memcpy((void *)ebase, final_handler, 0x100); |
92b1e6a6 FBH |
763 | |
764 | dump_handler((u32 *)ebase, 64); | |
1da177e4 LT |
765 | } |
766 | ||
767 | /* | |
768 | * TLB load/store/modify handlers. | |
769 | * | |
770 | * Only the fastpath gets synthesized at runtime, the slowpath for | |
771 | * do_page_fault remains normal asm. | |
772 | */ | |
773 | extern void tlb_do_page_fault_0(void); | |
774 | extern void tlb_do_page_fault_1(void); | |
775 | ||
1da177e4 LT |
776 | /* |
777 | * 128 instructions for the fastpath handler is generous and should | |
778 | * never be exceeded. | |
779 | */ | |
780 | #define FASTPATH_SIZE 128 | |
781 | ||
cbdbe07f FBH |
782 | u32 handle_tlbl[FASTPATH_SIZE] __cacheline_aligned; |
783 | u32 handle_tlbs[FASTPATH_SIZE] __cacheline_aligned; | |
784 | u32 handle_tlbm[FASTPATH_SIZE] __cacheline_aligned; | |
1da177e4 LT |
785 | |
786 | static void __init | |
e30ec452 | 787 | iPTE_LW(u32 **p, struct uasm_label **l, unsigned int pte, unsigned int ptr) |
1da177e4 LT |
788 | { |
789 | #ifdef CONFIG_SMP | |
790 | # ifdef CONFIG_64BIT_PHYS_ADDR | |
791 | if (cpu_has_64bits) | |
e30ec452 | 792 | uasm_i_lld(p, pte, 0, ptr); |
1da177e4 LT |
793 | else |
794 | # endif | |
e30ec452 | 795 | UASM_i_LL(p, pte, 0, ptr); |
1da177e4 LT |
796 | #else |
797 | # ifdef CONFIG_64BIT_PHYS_ADDR | |
798 | if (cpu_has_64bits) | |
e30ec452 | 799 | uasm_i_ld(p, pte, 0, ptr); |
1da177e4 LT |
800 | else |
801 | # endif | |
e30ec452 | 802 | UASM_i_LW(p, pte, 0, ptr); |
1da177e4 LT |
803 | #endif |
804 | } | |
805 | ||
806 | static void __init | |
e30ec452 | 807 | iPTE_SW(u32 **p, struct uasm_reloc **r, unsigned int pte, unsigned int ptr, |
63b2d2f4 | 808 | unsigned int mode) |
1da177e4 | 809 | { |
63b2d2f4 TS |
810 | #ifdef CONFIG_64BIT_PHYS_ADDR |
811 | unsigned int hwmode = mode & (_PAGE_VALID | _PAGE_DIRTY); | |
812 | #endif | |
813 | ||
e30ec452 | 814 | uasm_i_ori(p, pte, pte, mode); |
1da177e4 LT |
815 | #ifdef CONFIG_SMP |
816 | # ifdef CONFIG_64BIT_PHYS_ADDR | |
817 | if (cpu_has_64bits) | |
e30ec452 | 818 | uasm_i_scd(p, pte, 0, ptr); |
1da177e4 LT |
819 | else |
820 | # endif | |
e30ec452 | 821 | UASM_i_SC(p, pte, 0, ptr); |
1da177e4 LT |
822 | |
823 | if (r10000_llsc_war()) | |
e30ec452 | 824 | uasm_il_beqzl(p, r, pte, label_smp_pgtable_change); |
1da177e4 | 825 | else |
e30ec452 | 826 | uasm_il_beqz(p, r, pte, label_smp_pgtable_change); |
1da177e4 LT |
827 | |
828 | # ifdef CONFIG_64BIT_PHYS_ADDR | |
829 | if (!cpu_has_64bits) { | |
e30ec452 TS |
830 | /* no uasm_i_nop needed */ |
831 | uasm_i_ll(p, pte, sizeof(pte_t) / 2, ptr); | |
832 | uasm_i_ori(p, pte, pte, hwmode); | |
833 | uasm_i_sc(p, pte, sizeof(pte_t) / 2, ptr); | |
834 | uasm_il_beqz(p, r, pte, label_smp_pgtable_change); | |
835 | /* no uasm_i_nop needed */ | |
836 | uasm_i_lw(p, pte, 0, ptr); | |
1da177e4 | 837 | } else |
e30ec452 | 838 | uasm_i_nop(p); |
1da177e4 | 839 | # else |
e30ec452 | 840 | uasm_i_nop(p); |
1da177e4 LT |
841 | # endif |
842 | #else | |
843 | # ifdef CONFIG_64BIT_PHYS_ADDR | |
844 | if (cpu_has_64bits) | |
e30ec452 | 845 | uasm_i_sd(p, pte, 0, ptr); |
1da177e4 LT |
846 | else |
847 | # endif | |
e30ec452 | 848 | UASM_i_SW(p, pte, 0, ptr); |
1da177e4 LT |
849 | |
850 | # ifdef CONFIG_64BIT_PHYS_ADDR | |
851 | if (!cpu_has_64bits) { | |
e30ec452 TS |
852 | uasm_i_lw(p, pte, sizeof(pte_t) / 2, ptr); |
853 | uasm_i_ori(p, pte, pte, hwmode); | |
854 | uasm_i_sw(p, pte, sizeof(pte_t) / 2, ptr); | |
855 | uasm_i_lw(p, pte, 0, ptr); | |
1da177e4 LT |
856 | } |
857 | # endif | |
858 | #endif | |
859 | } | |
860 | ||
861 | /* | |
862 | * Check if PTE is present, if not then jump to LABEL. PTR points to | |
863 | * the page table where this PTE is located, PTE will be re-loaded | |
864 | * with it's original value. | |
865 | */ | |
866 | static void __init | |
e30ec452 | 867 | build_pte_present(u32 **p, struct uasm_label **l, struct uasm_reloc **r, |
1da177e4 LT |
868 | unsigned int pte, unsigned int ptr, enum label_id lid) |
869 | { | |
e30ec452 TS |
870 | uasm_i_andi(p, pte, pte, _PAGE_PRESENT | _PAGE_READ); |
871 | uasm_i_xori(p, pte, pte, _PAGE_PRESENT | _PAGE_READ); | |
872 | uasm_il_bnez(p, r, pte, lid); | |
63b2d2f4 | 873 | iPTE_LW(p, l, pte, ptr); |
1da177e4 LT |
874 | } |
875 | ||
876 | /* Make PTE valid, store result in PTR. */ | |
877 | static void __init | |
e30ec452 | 878 | build_make_valid(u32 **p, struct uasm_reloc **r, unsigned int pte, |
1da177e4 LT |
879 | unsigned int ptr) |
880 | { | |
63b2d2f4 TS |
881 | unsigned int mode = _PAGE_VALID | _PAGE_ACCESSED; |
882 | ||
883 | iPTE_SW(p, r, pte, ptr, mode); | |
1da177e4 LT |
884 | } |
885 | ||
886 | /* | |
887 | * Check if PTE can be written to, if not branch to LABEL. Regardless | |
888 | * restore PTE with value from PTR when done. | |
889 | */ | |
890 | static void __init | |
e30ec452 | 891 | build_pte_writable(u32 **p, struct uasm_label **l, struct uasm_reloc **r, |
1da177e4 LT |
892 | unsigned int pte, unsigned int ptr, enum label_id lid) |
893 | { | |
e30ec452 TS |
894 | uasm_i_andi(p, pte, pte, _PAGE_PRESENT | _PAGE_WRITE); |
895 | uasm_i_xori(p, pte, pte, _PAGE_PRESENT | _PAGE_WRITE); | |
896 | uasm_il_bnez(p, r, pte, lid); | |
63b2d2f4 | 897 | iPTE_LW(p, l, pte, ptr); |
1da177e4 LT |
898 | } |
899 | ||
900 | /* Make PTE writable, update software status bits as well, then store | |
901 | * at PTR. | |
902 | */ | |
903 | static void __init | |
e30ec452 | 904 | build_make_write(u32 **p, struct uasm_reloc **r, unsigned int pte, |
1da177e4 LT |
905 | unsigned int ptr) |
906 | { | |
63b2d2f4 TS |
907 | unsigned int mode = (_PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID |
908 | | _PAGE_DIRTY); | |
909 | ||
910 | iPTE_SW(p, r, pte, ptr, mode); | |
1da177e4 LT |
911 | } |
912 | ||
913 | /* | |
914 | * Check if PTE can be modified, if not branch to LABEL. Regardless | |
915 | * restore PTE with value from PTR when done. | |
916 | */ | |
917 | static void __init | |
e30ec452 | 918 | build_pte_modifiable(u32 **p, struct uasm_label **l, struct uasm_reloc **r, |
1da177e4 LT |
919 | unsigned int pte, unsigned int ptr, enum label_id lid) |
920 | { | |
e30ec452 TS |
921 | uasm_i_andi(p, pte, pte, _PAGE_WRITE); |
922 | uasm_il_beqz(p, r, pte, lid); | |
63b2d2f4 | 923 | iPTE_LW(p, l, pte, ptr); |
1da177e4 LT |
924 | } |
925 | ||
926 | /* | |
927 | * R3000 style TLB load/store/modify handlers. | |
928 | */ | |
929 | ||
fded2e50 MR |
930 | /* |
931 | * This places the pte into ENTRYLO0 and writes it with tlbwi. | |
932 | * Then it returns. | |
933 | */ | |
1da177e4 | 934 | static void __init |
fded2e50 | 935 | build_r3000_pte_reload_tlbwi(u32 **p, unsigned int pte, unsigned int tmp) |
1da177e4 | 936 | { |
e30ec452 TS |
937 | uasm_i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */ |
938 | uasm_i_mfc0(p, tmp, C0_EPC); /* cp0 delay */ | |
939 | uasm_i_tlbwi(p); | |
940 | uasm_i_jr(p, tmp); | |
941 | uasm_i_rfe(p); /* branch delay */ | |
1da177e4 LT |
942 | } |
943 | ||
944 | /* | |
fded2e50 MR |
945 | * This places the pte into ENTRYLO0 and writes it with tlbwi |
946 | * or tlbwr as appropriate. This is because the index register | |
947 | * may have the probe fail bit set as a result of a trap on a | |
948 | * kseg2 access, i.e. without refill. Then it returns. | |
1da177e4 LT |
949 | */ |
950 | static void __init | |
e30ec452 TS |
951 | build_r3000_tlb_reload_write(u32 **p, struct uasm_label **l, |
952 | struct uasm_reloc **r, unsigned int pte, | |
953 | unsigned int tmp) | |
954 | { | |
955 | uasm_i_mfc0(p, tmp, C0_INDEX); | |
956 | uasm_i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */ | |
957 | uasm_il_bltz(p, r, tmp, label_r3000_write_probe_fail); /* cp0 delay */ | |
958 | uasm_i_mfc0(p, tmp, C0_EPC); /* branch delay */ | |
959 | uasm_i_tlbwi(p); /* cp0 delay */ | |
960 | uasm_i_jr(p, tmp); | |
961 | uasm_i_rfe(p); /* branch delay */ | |
962 | uasm_l_r3000_write_probe_fail(l, *p); | |
963 | uasm_i_tlbwr(p); /* cp0 delay */ | |
964 | uasm_i_jr(p, tmp); | |
965 | uasm_i_rfe(p); /* branch delay */ | |
1da177e4 LT |
966 | } |
967 | ||
968 | static void __init | |
969 | build_r3000_tlbchange_handler_head(u32 **p, unsigned int pte, | |
970 | unsigned int ptr) | |
971 | { | |
972 | long pgdc = (long)pgd_current; | |
973 | ||
e30ec452 TS |
974 | uasm_i_mfc0(p, pte, C0_BADVADDR); |
975 | uasm_i_lui(p, ptr, uasm_rel_hi(pgdc)); /* cp0 delay */ | |
976 | uasm_i_lw(p, ptr, uasm_rel_lo(pgdc), ptr); | |
977 | uasm_i_srl(p, pte, pte, 22); /* load delay */ | |
978 | uasm_i_sll(p, pte, pte, 2); | |
979 | uasm_i_addu(p, ptr, ptr, pte); | |
980 | uasm_i_mfc0(p, pte, C0_CONTEXT); | |
981 | uasm_i_lw(p, ptr, 0, ptr); /* cp0 delay */ | |
982 | uasm_i_andi(p, pte, pte, 0xffc); /* load delay */ | |
983 | uasm_i_addu(p, ptr, ptr, pte); | |
984 | uasm_i_lw(p, pte, 0, ptr); | |
985 | uasm_i_tlbp(p); /* load delay */ | |
1da177e4 LT |
986 | } |
987 | ||
988 | static void __init build_r3000_tlb_load_handler(void) | |
989 | { | |
990 | u32 *p = handle_tlbl; | |
e30ec452 TS |
991 | struct uasm_label *l = labels; |
992 | struct uasm_reloc *r = relocs; | |
1da177e4 LT |
993 | |
994 | memset(handle_tlbl, 0, sizeof(handle_tlbl)); | |
995 | memset(labels, 0, sizeof(labels)); | |
996 | memset(relocs, 0, sizeof(relocs)); | |
997 | ||
998 | build_r3000_tlbchange_handler_head(&p, K0, K1); | |
999 | build_pte_present(&p, &l, &r, K0, K1, label_nopage_tlbl); | |
e30ec452 | 1000 | uasm_i_nop(&p); /* load delay */ |
1da177e4 | 1001 | build_make_valid(&p, &r, K0, K1); |
fded2e50 | 1002 | build_r3000_tlb_reload_write(&p, &l, &r, K0, K1); |
1da177e4 | 1003 | |
e30ec452 TS |
1004 | uasm_l_nopage_tlbl(&l, p); |
1005 | uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff); | |
1006 | uasm_i_nop(&p); | |
1da177e4 LT |
1007 | |
1008 | if ((p - handle_tlbl) > FASTPATH_SIZE) | |
1009 | panic("TLB load handler fastpath space exceeded"); | |
1010 | ||
e30ec452 TS |
1011 | uasm_resolve_relocs(relocs, labels); |
1012 | pr_debug("Wrote TLB load handler fastpath (%u instructions).\n", | |
1013 | (unsigned int)(p - handle_tlbl)); | |
1da177e4 | 1014 | |
92b1e6a6 | 1015 | dump_handler(handle_tlbl, ARRAY_SIZE(handle_tlbl)); |
1da177e4 LT |
1016 | } |
1017 | ||
1018 | static void __init build_r3000_tlb_store_handler(void) | |
1019 | { | |
1020 | u32 *p = handle_tlbs; | |
e30ec452 TS |
1021 | struct uasm_label *l = labels; |
1022 | struct uasm_reloc *r = relocs; | |
1da177e4 LT |
1023 | |
1024 | memset(handle_tlbs, 0, sizeof(handle_tlbs)); | |
1025 | memset(labels, 0, sizeof(labels)); | |
1026 | memset(relocs, 0, sizeof(relocs)); | |
1027 | ||
1028 | build_r3000_tlbchange_handler_head(&p, K0, K1); | |
1029 | build_pte_writable(&p, &l, &r, K0, K1, label_nopage_tlbs); | |
e30ec452 | 1030 | uasm_i_nop(&p); /* load delay */ |
1da177e4 | 1031 | build_make_write(&p, &r, K0, K1); |
fded2e50 | 1032 | build_r3000_tlb_reload_write(&p, &l, &r, K0, K1); |
1da177e4 | 1033 | |
e30ec452 TS |
1034 | uasm_l_nopage_tlbs(&l, p); |
1035 | uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); | |
1036 | uasm_i_nop(&p); | |
1da177e4 LT |
1037 | |
1038 | if ((p - handle_tlbs) > FASTPATH_SIZE) | |
1039 | panic("TLB store handler fastpath space exceeded"); | |
1040 | ||
e30ec452 TS |
1041 | uasm_resolve_relocs(relocs, labels); |
1042 | pr_debug("Wrote TLB store handler fastpath (%u instructions).\n", | |
1043 | (unsigned int)(p - handle_tlbs)); | |
1da177e4 | 1044 | |
92b1e6a6 | 1045 | dump_handler(handle_tlbs, ARRAY_SIZE(handle_tlbs)); |
1da177e4 LT |
1046 | } |
1047 | ||
1048 | static void __init build_r3000_tlb_modify_handler(void) | |
1049 | { | |
1050 | u32 *p = handle_tlbm; | |
e30ec452 TS |
1051 | struct uasm_label *l = labels; |
1052 | struct uasm_reloc *r = relocs; | |
1da177e4 LT |
1053 | |
1054 | memset(handle_tlbm, 0, sizeof(handle_tlbm)); | |
1055 | memset(labels, 0, sizeof(labels)); | |
1056 | memset(relocs, 0, sizeof(relocs)); | |
1057 | ||
1058 | build_r3000_tlbchange_handler_head(&p, K0, K1); | |
1059 | build_pte_modifiable(&p, &l, &r, K0, K1, label_nopage_tlbm); | |
e30ec452 | 1060 | uasm_i_nop(&p); /* load delay */ |
1da177e4 | 1061 | build_make_write(&p, &r, K0, K1); |
fded2e50 | 1062 | build_r3000_pte_reload_tlbwi(&p, K0, K1); |
1da177e4 | 1063 | |
e30ec452 TS |
1064 | uasm_l_nopage_tlbm(&l, p); |
1065 | uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); | |
1066 | uasm_i_nop(&p); | |
1da177e4 LT |
1067 | |
1068 | if ((p - handle_tlbm) > FASTPATH_SIZE) | |
1069 | panic("TLB modify handler fastpath space exceeded"); | |
1070 | ||
e30ec452 TS |
1071 | uasm_resolve_relocs(relocs, labels); |
1072 | pr_debug("Wrote TLB modify handler fastpath (%u instructions).\n", | |
1073 | (unsigned int)(p - handle_tlbm)); | |
1da177e4 | 1074 | |
92b1e6a6 | 1075 | dump_handler(handle_tlbm, ARRAY_SIZE(handle_tlbm)); |
1da177e4 LT |
1076 | } |
1077 | ||
1078 | /* | |
1079 | * R4000 style TLB load/store/modify handlers. | |
1080 | */ | |
1081 | static void __init | |
e30ec452 TS |
1082 | build_r4000_tlbchange_handler_head(u32 **p, struct uasm_label **l, |
1083 | struct uasm_reloc **r, unsigned int pte, | |
1da177e4 LT |
1084 | unsigned int ptr) |
1085 | { | |
875d43e7 | 1086 | #ifdef CONFIG_64BIT |
1da177e4 LT |
1087 | build_get_pmde64(p, l, r, pte, ptr); /* get pmd in ptr */ |
1088 | #else | |
1089 | build_get_pgde32(p, pte, ptr); /* get pgd in ptr */ | |
1090 | #endif | |
1091 | ||
e30ec452 TS |
1092 | UASM_i_MFC0(p, pte, C0_BADVADDR); |
1093 | UASM_i_LW(p, ptr, 0, ptr); | |
1094 | UASM_i_SRL(p, pte, pte, PAGE_SHIFT + PTE_ORDER - PTE_T_LOG2); | |
1095 | uasm_i_andi(p, pte, pte, (PTRS_PER_PTE - 1) << PTE_T_LOG2); | |
1096 | UASM_i_ADDU(p, ptr, ptr, pte); | |
1da177e4 LT |
1097 | |
1098 | #ifdef CONFIG_SMP | |
e30ec452 TS |
1099 | uasm_l_smp_pgtable_change(l, *p); |
1100 | #endif | |
63b2d2f4 | 1101 | iPTE_LW(p, l, pte, ptr); /* get even pte */ |
8df5beac MR |
1102 | if (!m4kc_tlbp_war()) |
1103 | build_tlb_probe_entry(p); | |
1da177e4 LT |
1104 | } |
1105 | ||
1106 | static void __init | |
e30ec452 TS |
1107 | build_r4000_tlbchange_handler_tail(u32 **p, struct uasm_label **l, |
1108 | struct uasm_reloc **r, unsigned int tmp, | |
1da177e4 LT |
1109 | unsigned int ptr) |
1110 | { | |
e30ec452 TS |
1111 | uasm_i_ori(p, ptr, ptr, sizeof(pte_t)); |
1112 | uasm_i_xori(p, ptr, ptr, sizeof(pte_t)); | |
1da177e4 LT |
1113 | build_update_entries(p, tmp, ptr); |
1114 | build_tlb_write_entry(p, l, r, tlb_indexed); | |
e30ec452 TS |
1115 | uasm_l_leave(l, *p); |
1116 | uasm_i_eret(p); /* return from trap */ | |
1da177e4 | 1117 | |
875d43e7 | 1118 | #ifdef CONFIG_64BIT |
1da177e4 LT |
1119 | build_get_pgd_vmalloc64(p, l, r, tmp, ptr); |
1120 | #endif | |
1121 | } | |
1122 | ||
1123 | static void __init build_r4000_tlb_load_handler(void) | |
1124 | { | |
1125 | u32 *p = handle_tlbl; | |
e30ec452 TS |
1126 | struct uasm_label *l = labels; |
1127 | struct uasm_reloc *r = relocs; | |
1da177e4 LT |
1128 | |
1129 | memset(handle_tlbl, 0, sizeof(handle_tlbl)); | |
1130 | memset(labels, 0, sizeof(labels)); | |
1131 | memset(relocs, 0, sizeof(relocs)); | |
1132 | ||
1133 | if (bcm1250_m3_war()) { | |
e30ec452 TS |
1134 | UASM_i_MFC0(&p, K0, C0_BADVADDR); |
1135 | UASM_i_MFC0(&p, K1, C0_ENTRYHI); | |
1136 | uasm_i_xor(&p, K0, K0, K1); | |
1137 | UASM_i_SRL(&p, K0, K0, PAGE_SHIFT + 1); | |
1138 | uasm_il_bnez(&p, &r, K0, label_leave); | |
1139 | /* No need for uasm_i_nop */ | |
1da177e4 LT |
1140 | } |
1141 | ||
1142 | build_r4000_tlbchange_handler_head(&p, &l, &r, K0, K1); | |
1143 | build_pte_present(&p, &l, &r, K0, K1, label_nopage_tlbl); | |
8df5beac MR |
1144 | if (m4kc_tlbp_war()) |
1145 | build_tlb_probe_entry(&p); | |
1da177e4 LT |
1146 | build_make_valid(&p, &r, K0, K1); |
1147 | build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1); | |
1148 | ||
e30ec452 TS |
1149 | uasm_l_nopage_tlbl(&l, p); |
1150 | uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff); | |
1151 | uasm_i_nop(&p); | |
1da177e4 LT |
1152 | |
1153 | if ((p - handle_tlbl) > FASTPATH_SIZE) | |
1154 | panic("TLB load handler fastpath space exceeded"); | |
1155 | ||
e30ec452 TS |
1156 | uasm_resolve_relocs(relocs, labels); |
1157 | pr_debug("Wrote TLB load handler fastpath (%u instructions).\n", | |
1158 | (unsigned int)(p - handle_tlbl)); | |
1da177e4 | 1159 | |
92b1e6a6 | 1160 | dump_handler(handle_tlbl, ARRAY_SIZE(handle_tlbl)); |
1da177e4 LT |
1161 | } |
1162 | ||
1163 | static void __init build_r4000_tlb_store_handler(void) | |
1164 | { | |
1165 | u32 *p = handle_tlbs; | |
e30ec452 TS |
1166 | struct uasm_label *l = labels; |
1167 | struct uasm_reloc *r = relocs; | |
1da177e4 LT |
1168 | |
1169 | memset(handle_tlbs, 0, sizeof(handle_tlbs)); | |
1170 | memset(labels, 0, sizeof(labels)); | |
1171 | memset(relocs, 0, sizeof(relocs)); | |
1172 | ||
1173 | build_r4000_tlbchange_handler_head(&p, &l, &r, K0, K1); | |
1174 | build_pte_writable(&p, &l, &r, K0, K1, label_nopage_tlbs); | |
8df5beac MR |
1175 | if (m4kc_tlbp_war()) |
1176 | build_tlb_probe_entry(&p); | |
1da177e4 LT |
1177 | build_make_write(&p, &r, K0, K1); |
1178 | build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1); | |
1179 | ||
e30ec452 TS |
1180 | uasm_l_nopage_tlbs(&l, p); |
1181 | uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); | |
1182 | uasm_i_nop(&p); | |
1da177e4 LT |
1183 | |
1184 | if ((p - handle_tlbs) > FASTPATH_SIZE) | |
1185 | panic("TLB store handler fastpath space exceeded"); | |
1186 | ||
e30ec452 TS |
1187 | uasm_resolve_relocs(relocs, labels); |
1188 | pr_debug("Wrote TLB store handler fastpath (%u instructions).\n", | |
1189 | (unsigned int)(p - handle_tlbs)); | |
1da177e4 | 1190 | |
92b1e6a6 | 1191 | dump_handler(handle_tlbs, ARRAY_SIZE(handle_tlbs)); |
1da177e4 LT |
1192 | } |
1193 | ||
1194 | static void __init build_r4000_tlb_modify_handler(void) | |
1195 | { | |
1196 | u32 *p = handle_tlbm; | |
e30ec452 TS |
1197 | struct uasm_label *l = labels; |
1198 | struct uasm_reloc *r = relocs; | |
1da177e4 LT |
1199 | |
1200 | memset(handle_tlbm, 0, sizeof(handle_tlbm)); | |
1201 | memset(labels, 0, sizeof(labels)); | |
1202 | memset(relocs, 0, sizeof(relocs)); | |
1203 | ||
1204 | build_r4000_tlbchange_handler_head(&p, &l, &r, K0, K1); | |
1205 | build_pte_modifiable(&p, &l, &r, K0, K1, label_nopage_tlbm); | |
8df5beac MR |
1206 | if (m4kc_tlbp_war()) |
1207 | build_tlb_probe_entry(&p); | |
1da177e4 LT |
1208 | /* Present and writable bits set, set accessed and dirty bits. */ |
1209 | build_make_write(&p, &r, K0, K1); | |
1210 | build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1); | |
1211 | ||
e30ec452 TS |
1212 | uasm_l_nopage_tlbm(&l, p); |
1213 | uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); | |
1214 | uasm_i_nop(&p); | |
1da177e4 LT |
1215 | |
1216 | if ((p - handle_tlbm) > FASTPATH_SIZE) | |
1217 | panic("TLB modify handler fastpath space exceeded"); | |
1218 | ||
e30ec452 TS |
1219 | uasm_resolve_relocs(relocs, labels); |
1220 | pr_debug("Wrote TLB modify handler fastpath (%u instructions).\n", | |
1221 | (unsigned int)(p - handle_tlbm)); | |
115f2a44 | 1222 | |
92b1e6a6 | 1223 | dump_handler(handle_tlbm, ARRAY_SIZE(handle_tlbm)); |
1da177e4 LT |
1224 | } |
1225 | ||
1226 | void __init build_tlb_refill_handler(void) | |
1227 | { | |
1228 | /* | |
1229 | * The refill handler is generated per-CPU, multi-node systems | |
1230 | * may have local storage for it. The other handlers are only | |
1231 | * needed once. | |
1232 | */ | |
1233 | static int run_once = 0; | |
1234 | ||
10cc3529 | 1235 | switch (current_cpu_type()) { |
1da177e4 LT |
1236 | case CPU_R2000: |
1237 | case CPU_R3000: | |
1238 | case CPU_R3000A: | |
1239 | case CPU_R3081E: | |
1240 | case CPU_TX3912: | |
1241 | case CPU_TX3922: | |
1242 | case CPU_TX3927: | |
1243 | build_r3000_tlb_refill_handler(); | |
1244 | if (!run_once) { | |
1245 | build_r3000_tlb_load_handler(); | |
1246 | build_r3000_tlb_store_handler(); | |
1247 | build_r3000_tlb_modify_handler(); | |
1248 | run_once++; | |
1249 | } | |
1250 | break; | |
1251 | ||
1252 | case CPU_R6000: | |
1253 | case CPU_R6000A: | |
1254 | panic("No R6000 TLB refill handler yet"); | |
1255 | break; | |
1256 | ||
1257 | case CPU_R8000: | |
1258 | panic("No R8000 TLB refill handler yet"); | |
1259 | break; | |
1260 | ||
1261 | default: | |
1262 | build_r4000_tlb_refill_handler(); | |
1263 | if (!run_once) { | |
1264 | build_r4000_tlb_load_handler(); | |
1265 | build_r4000_tlb_store_handler(); | |
1266 | build_r4000_tlb_modify_handler(); | |
1267 | run_once++; | |
1268 | } | |
1269 | } | |
1270 | } | |
1d40cfcd RB |
1271 | |
1272 | void __init flush_tlb_handlers(void) | |
1273 | { | |
1274 | flush_icache_range((unsigned long)handle_tlbl, | |
1275 | (unsigned long)handle_tlbl + sizeof(handle_tlbl)); | |
1276 | flush_icache_range((unsigned long)handle_tlbs, | |
1277 | (unsigned long)handle_tlbs + sizeof(handle_tlbs)); | |
1278 | flush_icache_range((unsigned long)handle_tlbm, | |
1279 | (unsigned long)handle_tlbm + sizeof(handle_tlbm)); | |
1280 | } |