Commit | Line | Data |
---|---|---|
d257d5da DM |
1 | /* sun4v_tlb_miss.S: Sun4v TLB miss handlers. |
2 | * | |
3 | * Copyright (C) 2006 <davem@davemloft.net> | |
4 | */ | |
5 | ||
6 | .text | |
7 | .align 32 | |
8 | ||
9 | sun4v_itlb_miss: | |
12eaa328 DM |
10 | /* Load MMU Miss base into %g2. */ |
11 | ldxa [%g0] ASI_SCRATCHPAD, %g3 | |
d257d5da DM |
12 | |
13 | /* Load UTSB reg into %g1. */ | |
12eaa328 DM |
14 | mov SCRATCHPAD_UTSBREG1, %g1 |
15 | ldxa [%g1] ASI_SCRATCHPAD, %g1 | |
d257d5da DM |
16 | |
17 | /* Create a TAG TARGET, "(vaddr>>22) | (ctx << 48)", in %g6. | |
18 | * Branch if kernel TLB miss. The kernel TSB and user TSB miss | |
19 | * code wants the missing virtual address in %g4, so that value | |
20 | * cannot be modified through the entirety of this handler. | |
21 | */ | |
12eaa328 DM |
22 | ldx [%g2 + HV_FAULT_I_ADDR_OFFSET], %g4 |
23 | ldx [%g2 + HV_FAULT_I_CTX_OFFSET], %g5 | |
d257d5da DM |
24 | srlx %g4, 22, %g3 |
25 | sllx %g5, 48, %g6 | |
26 | or %g6, %g3, %g6 | |
27 | brz,pn %g5, kvmap_itlb_4v | |
28 | nop | |
29 | ||
30 | /* Create TSB pointer. This is something like: | |
31 | * | |
32 | * index_mask = (512 << (tsb_reg & 0x7UL)) - 1UL; | |
33 | * tsb_base = tsb_reg & ~0x7UL; | |
34 | */ | |
35 | and %g1, 0x7, %g3 | |
36 | andn %g1, 0x7, %g1 | |
37 | mov 512, %g7 | |
38 | sllx %g7, %g3, %g7 | |
39 | sub %g7, 1, %g7 | |
40 | ||
41 | /* TSB index mask is in %g7, tsb base is in %g1. Compute | |
42 | * the TSB entry pointer into %g1: | |
43 | * | |
44 | * tsb_index = ((vaddr >> PAGE_SHIFT) & tsb_mask); | |
45 | * tsb_ptr = tsb_base + (tsb_index * 16); | |
46 | */ | |
47 | srlx %g4, PAGE_SHIFT, %g3 | |
48 | and %g3, %g7, %g3 | |
49 | sllx %g3, 4, %g3 | |
50 | add %g1, %g3, %g1 | |
51 | ||
52 | /* Load TSB tag/pte into %g2/%g3 and compare the tag. */ | |
53 | ldda [%g1] ASI_QUAD_LDD_PHYS, %g2 | |
54 | cmp %g2, %g6 | |
55 | sethi %hi(_PAGE_EXEC), %g7 | |
56 | bne,a,pn %xcc, tsb_miss_page_table_walk | |
57 | mov FAULT_CODE_ITLB, %g3 | |
58 | andcc %g3, %g7, %g0 | |
59 | be,a,pn %xcc, tsb_do_fault | |
60 | mov FAULT_CODE_ITLB, %g3 | |
61 | ||
62 | /* We have a valid entry, make hypervisor call to load | |
63 | * I-TLB and return from trap. | |
64 | * | |
65 | * %g3: PTE | |
66 | * %g4: vaddr | |
67 | * %g6: TAG TARGET (only "CTX << 48" part matters) | |
68 | */ | |
69 | sun4v_itlb_load: | |
70 | mov %o0, %g1 ! save %o0 | |
71 | mov %o1, %g2 ! save %o1 | |
72 | mov %o2, %g5 ! save %o2 | |
73 | mov %o3, %g7 ! save %o3 | |
74 | mov %g4, %o0 ! vaddr | |
75 | srlx %g6, 48, %o1 ! ctx | |
76 | mov %g3, %o2 ! PTE | |
77 | mov HV_MMU_IMMU, %o3 ! flags | |
78 | ta HV_MMU_MAP_ADDR_TRAP | |
79 | mov %g1, %o0 ! restore %o0 | |
80 | mov %g2, %o1 ! restore %o1 | |
81 | mov %g5, %o2 ! restore %o2 | |
82 | mov %g7, %o3 ! restore %o3 | |
83 | ||
84 | retry | |
85 | ||
86 | sun4v_dtlb_miss: | |
12eaa328 DM |
87 | /* Load MMU Miss base into %g2. */ |
88 | ldxa [%g0] ASI_SCRATCHPAD, %g2 | |
d257d5da DM |
89 | |
90 | /* Load UTSB reg into %g1. */ | |
12eaa328 | 91 | mov SCRATCHPAD_UTSBREG1, %g1 |
d257d5da DM |
92 | ldxa [%g1 + %g1] ASI_SCRATCHPAD, %g1 |
93 | ||
d257d5da DM |
94 | /* Create a TAG TARGET, "(vaddr>>22) | (ctx << 48)", in %g6. |
95 | * Branch if kernel TLB miss. The kernel TSB and user TSB miss | |
96 | * code wants the missing virtual address in %g4, so that value | |
97 | * cannot be modified through the entirety of this handler. | |
98 | */ | |
12eaa328 DM |
99 | ldx [%g2 + HV_FAULT_D_ADDR_OFFSET], %g4 |
100 | ldx [%g2 + HV_FAULT_D_CTX_OFFSET], %g5 | |
d257d5da DM |
101 | srlx %g4, 22, %g3 |
102 | sllx %g5, 48, %g6 | |
103 | or %g6, %g3, %g6 | |
104 | brz,pn %g5, kvmap_dtlb_4v | |
105 | nop | |
106 | ||
107 | /* Create TSB pointer. This is something like: | |
108 | * | |
109 | * index_mask = (512 << (tsb_reg & 0x7UL)) - 1UL; | |
110 | * tsb_base = tsb_reg & ~0x7UL; | |
111 | */ | |
112 | and %g1, 0x7, %g3 | |
113 | andn %g1, 0x7, %g1 | |
114 | mov 512, %g7 | |
115 | sllx %g7, %g3, %g7 | |
116 | sub %g7, 1, %g7 | |
117 | ||
118 | /* TSB index mask is in %g7, tsb base is in %g1. Compute | |
119 | * the TSB entry pointer into %g1: | |
120 | * | |
121 | * tsb_index = ((vaddr >> PAGE_SHIFT) & tsb_mask); | |
122 | * tsb_ptr = tsb_base + (tsb_index * 16); | |
123 | */ | |
124 | srlx %g4, PAGE_SHIFT, %g3 | |
125 | and %g3, %g7, %g3 | |
126 | sllx %g3, 4, %g3 | |
127 | add %g1, %g3, %g1 | |
128 | ||
129 | /* Load TSB tag/pte into %g2/%g3 and compare the tag. */ | |
130 | ldda [%g1] ASI_QUAD_LDD_PHYS, %g2 | |
131 | cmp %g2, %g6 | |
132 | bne,a,pn %xcc, tsb_miss_page_table_walk | |
133 | mov FAULT_CODE_ITLB, %g3 | |
134 | ||
135 | /* We have a valid entry, make hypervisor call to load | |
136 | * D-TLB and return from trap. | |
137 | * | |
138 | * %g3: PTE | |
139 | * %g4: vaddr | |
140 | * %g6: TAG TARGET (only "CTX << 48" part matters) | |
141 | */ | |
142 | sun4v_dtlb_load: | |
143 | mov %o0, %g1 ! save %o0 | |
144 | mov %o1, %g2 ! save %o1 | |
145 | mov %o2, %g5 ! save %o2 | |
146 | mov %o3, %g7 ! save %o3 | |
147 | mov %g4, %o0 ! vaddr | |
148 | srlx %g6, 48, %o1 ! ctx | |
149 | mov %g3, %o2 ! PTE | |
150 | mov HV_MMU_DMMU, %o3 ! flags | |
151 | ta HV_MMU_MAP_ADDR_TRAP | |
152 | mov %g1, %o0 ! restore %o0 | |
153 | mov %g2, %o1 ! restore %o1 | |
154 | mov %g5, %o2 ! restore %o2 | |
155 | mov %g7, %o3 ! restore %o3 | |
156 | ||
157 | retry | |
158 | ||
159 | sun4v_dtlb_prot: | |
12eaa328 DM |
160 | /* Load MMU Miss base into %g2. */ |
161 | ldxa [%g0] ASI_SCRATCHPAD, %g2 | |
d257d5da | 162 | |
12eaa328 | 163 | ldx [%g2 + HV_FAULT_D_ADDR_OFFSET], %g5 |
d257d5da DM |
164 | rdpr %tl, %g1 |
165 | cmp %g1, 1 | |
166 | bgu,pn %xcc, winfix_trampoline | |
167 | nop | |
168 | ba,pt %xcc, sparc64_realfault_common | |
169 | mov FAULT_CODE_DTLB | FAULT_CODE_WRITE, %g4 | |
170 | ||
12eaa328 DM |
171 | /* Called from trap table with TAG TARGET placed into |
172 | * %g6 and SCRATCHPAD_UTSBREG1 contents in %g1. | |
aa9143b9 DM |
173 | */ |
174 | sun4v_itsb_miss: | |
aa9143b9 DM |
175 | ba,pt %xcc, sun4v_tsb_miss_common |
176 | mov FAULT_CODE_ITLB, %g3 | |
177 | ||
12eaa328 DM |
178 | /* Called from trap table with TAG TARGET placed into |
179 | * %g6 and SCRATCHPAD_UTSBREG1 contents in %g1. | |
aa9143b9 DM |
180 | */ |
181 | sun4v_dtsb_miss: | |
aa9143b9 DM |
182 | mov FAULT_CODE_DTLB, %g3 |
183 | ||
184 | /* Create TSB pointer into %g1. This is something like: | |
185 | * | |
186 | * index_mask = (512 << (tsb_reg & 0x7UL)) - 1UL; | |
187 | * tsb_base = tsb_reg & ~0x7UL; | |
188 | * tsb_index = ((vaddr >> PAGE_SHIFT) & tsb_mask); | |
189 | * tsb_ptr = tsb_base + (tsb_index * 16); | |
190 | */ | |
191 | sun4v_tsb_miss_common: | |
192 | and %g1, 0x7, %g2 | |
193 | andn %g1, 0x7, %g1 | |
194 | mov 512, %g7 | |
195 | sllx %g7, %g2, %g7 | |
196 | sub %g7, 1, %g7 | |
197 | srlx %g4, PAGE_SHIFT, %g2 | |
198 | and %g2, %g7, %g2 | |
199 | sllx %g2, 4, %g2 | |
200 | ba,pt %xcc, tsb_miss_page_table_walk | |
201 | add %g1, %g2, %g1 | |
202 | ||
ed6b0b45 DM |
203 | /* Instruction Access Exception, tl0. */ |
204 | sun4v_iacc: | |
12eaa328 DM |
205 | ldxa [%g0] ASI_SCRATCHPAD, %g2 |
206 | ldx [%g2 + HV_FAULT_I_TYPE_OFFSET], %g3 | |
207 | ldx [%g2 + HV_FAULT_I_ADDR_OFFSET], %g4 | |
208 | ldx [%g2 + HV_FAULT_I_CTX_OFFSET], %g5 | |
ed6b0b45 DM |
209 | sllx %g3, 16, %g3 |
210 | or %g5, %g3, %g5 | |
211 | ba,pt %xcc, etrap | |
212 | rd %pc, %g7 | |
213 | mov %l4, %o1 | |
214 | mov %l5, %o2 | |
215 | call sun4v_insn_access_exception | |
216 | add %sp, PTREGS_OFF, %o0 | |
217 | ba,a,pt %xcc, rtrap_clr_l6 | |
218 | ||
219 | /* Instruction Access Exception, tl1. */ | |
220 | sun4v_iacc_tl1: | |
12eaa328 DM |
221 | ldxa [%g0] ASI_SCRATCHPAD, %g2 |
222 | ldx [%g2 + HV_FAULT_I_TYPE_OFFSET], %g3 | |
223 | ldx [%g2 + HV_FAULT_I_ADDR_OFFSET], %g4 | |
224 | ldx [%g2 + HV_FAULT_I_CTX_OFFSET], %g5 | |
ed6b0b45 DM |
225 | sllx %g3, 16, %g3 |
226 | or %g5, %g3, %g5 | |
227 | ba,pt %xcc, etraptl1 | |
228 | rd %pc, %g7 | |
229 | mov %l4, %o1 | |
230 | mov %l5, %o2 | |
231 | call sun4v_insn_access_exception_tl1 | |
232 | add %sp, PTREGS_OFF, %o0 | |
233 | ba,a,pt %xcc, rtrap_clr_l6 | |
234 | ||
235 | /* Data Access Exception, tl0. */ | |
236 | sun4v_dacc: | |
12eaa328 DM |
237 | ldxa [%g0] ASI_SCRATCHPAD, %g2 |
238 | ldx [%g2 + HV_FAULT_D_TYPE_OFFSET], %g3 | |
239 | ldx [%g2 + HV_FAULT_D_ADDR_OFFSET], %g4 | |
240 | ldx [%g2 + HV_FAULT_D_CTX_OFFSET], %g5 | |
ed6b0b45 DM |
241 | sllx %g3, 16, %g3 |
242 | or %g5, %g3, %g5 | |
243 | ba,pt %xcc, etrap | |
244 | rd %pc, %g7 | |
245 | mov %l4, %o1 | |
246 | mov %l5, %o2 | |
247 | call sun4v_data_access_exception | |
248 | add %sp, PTREGS_OFF, %o0 | |
249 | ba,a,pt %xcc, rtrap_clr_l6 | |
250 | ||
251 | /* Data Access Exception, tl1. */ | |
252 | sun4v_dacc_tl1: | |
12eaa328 DM |
253 | ldxa [%g0] ASI_SCRATCHPAD, %g2 |
254 | ldx [%g2 + HV_FAULT_D_TYPE_OFFSET], %g3 | |
255 | ldx [%g2 + HV_FAULT_D_ADDR_OFFSET], %g4 | |
256 | ldx [%g2 + HV_FAULT_D_CTX_OFFSET], %g5 | |
ed6b0b45 DM |
257 | sllx %g3, 16, %g3 |
258 | or %g5, %g3, %g5 | |
259 | ba,pt %xcc, etraptl1 | |
260 | rd %pc, %g7 | |
261 | mov %l4, %o1 | |
262 | mov %l5, %o2 | |
263 | call sun4v_data_access_exception_tl1 | |
264 | add %sp, PTREGS_OFF, %o0 | |
265 | ba,a,pt %xcc, rtrap_clr_l6 | |
266 | ||
267 | /* Memory Address Unaligned. */ | |
268 | sun4v_mna: | |
12eaa328 | 269 | ldxa [%g0] ASI_SCRATCHPAD, %g2 |
ed6b0b45 | 270 | mov HV_FAULT_TYPE_UNALIGNED, %g3 |
12eaa328 DM |
271 | ldx [%g2 + HV_FAULT_D_ADDR_OFFSET], %g4 |
272 | ldx [%g2 + HV_FAULT_D_CTX_OFFSET], %g5 | |
ed6b0b45 DM |
273 | sllx %g3, 16, %g3 |
274 | or %g5, %g3, %g5 | |
275 | ||
276 | /* Window fixup? */ | |
277 | rdpr %tl, %g2 | |
278 | cmp %g2, 1 | |
279 | bgu,pn %icc, winfix_mna | |
280 | rdpr %tpc, %g3 | |
281 | ||
282 | ba,pt %xcc, etrap | |
283 | rd %pc, %g7 | |
284 | mov %l4, %o1 | |
285 | mov %l5, %o2 | |
286 | call sun4v_mna | |
287 | add %sp, PTREGS_OFF, %o0 | |
288 | ba,a,pt %xcc, rtrap_clr_l6 | |
289 | ||
290 | /* Privileged Action. */ | |
291 | sun4v_privact: | |
292 | ba,pt %xcc, etrap | |
293 | rd %pc, %g7 | |
294 | call do_privact | |
295 | add %sp, PTREGS_OFF, %o0 | |
296 | ba,a,pt %xcc, rtrap_clr_l6 | |
297 | ||
298 | /* Unaligned ldd float, tl0. */ | |
299 | sun4v_lddfmna: | |
12eaa328 DM |
300 | ldxa [%g0] ASI_SCRATCHPAD, %g2 |
301 | ldx [%g2 + HV_FAULT_D_TYPE_OFFSET], %g3 | |
302 | ldx [%g2 + HV_FAULT_D_ADDR_OFFSET], %g4 | |
303 | ldx [%g2 + HV_FAULT_D_CTX_OFFSET], %g5 | |
ed6b0b45 DM |
304 | sllx %g3, 16, %g3 |
305 | or %g5, %g3, %g5 | |
306 | ba,pt %xcc, etrap | |
307 | rd %pc, %g7 | |
308 | mov %l4, %o1 | |
309 | mov %l5, %o2 | |
310 | call handle_lddfmna | |
311 | add %sp, PTREGS_OFF, %o0 | |
312 | ba,a,pt %xcc, rtrap_clr_l6 | |
313 | ||
314 | /* Unaligned std float, tl0. */ | |
315 | sun4v_stdfmna: | |
12eaa328 DM |
316 | ldxa [%g0] ASI_SCRATCHPAD, %g2 |
317 | ldx [%g2 + HV_FAULT_D_TYPE_OFFSET], %g3 | |
318 | ldx [%g2 + HV_FAULT_D_ADDR_OFFSET], %g4 | |
319 | ldx [%g2 + HV_FAULT_D_CTX_OFFSET], %g5 | |
ed6b0b45 DM |
320 | sllx %g3, 16, %g3 |
321 | or %g5, %g3, %g5 | |
322 | ba,pt %xcc, etrap | |
323 | rd %pc, %g7 | |
324 | mov %l4, %o1 | |
325 | mov %l5, %o2 | |
326 | call handle_stdfmna | |
327 | add %sp, PTREGS_OFF, %o0 | |
328 | ba,a,pt %xcc, rtrap_clr_l6 | |
aa9143b9 | 329 | |
d257d5da DM |
330 | #define BRANCH_ALWAYS 0x10680000 |
331 | #define NOP 0x01000000 | |
332 | #define SUN4V_DO_PATCH(OLD, NEW) \ | |
333 | sethi %hi(NEW), %g1; \ | |
334 | or %g1, %lo(NEW), %g1; \ | |
335 | sethi %hi(OLD), %g2; \ | |
336 | or %g2, %lo(OLD), %g2; \ | |
337 | sub %g1, %g2, %g1; \ | |
338 | sethi %hi(BRANCH_ALWAYS), %g3; \ | |
339 | srl %g1, 2, %g1; \ | |
340 | or %g3, %lo(BRANCH_ALWAYS), %g3; \ | |
341 | or %g3, %g1, %g3; \ | |
342 | stw %g3, [%g2]; \ | |
343 | sethi %hi(NOP), %g3; \ | |
344 | or %g3, %lo(NOP), %g3; \ | |
345 | stw %g3, [%g2 + 0x4]; \ | |
346 | flush %g2; | |
347 | ||
348 | .globl sun4v_patch_tlb_handlers | |
349 | .type sun4v_patch_tlb_handlers,#function | |
350 | sun4v_patch_tlb_handlers: | |
351 | SUN4V_DO_PATCH(tl0_iamiss, sun4v_itlb_miss) | |
352 | SUN4V_DO_PATCH(tl1_iamiss, sun4v_itlb_miss) | |
353 | SUN4V_DO_PATCH(tl0_damiss, sun4v_dtlb_miss) | |
354 | SUN4V_DO_PATCH(tl1_damiss, sun4v_dtlb_miss) | |
355 | SUN4V_DO_PATCH(tl0_daprot, sun4v_dtlb_prot) | |
356 | SUN4V_DO_PATCH(tl1_daprot, sun4v_dtlb_prot) | |
ed6b0b45 DM |
357 | SUN4V_DO_PATCH(tl0_iax, sun4v_iacc) |
358 | SUN4V_DO_PATCH(tl1_iax, sun4v_iacc_tl1) | |
359 | SUN4V_DO_PATCH(tl0_dax, sun4v_dacc) | |
360 | SUN4V_DO_PATCH(tl1_dax, sun4v_dacc_tl1) | |
361 | SUN4V_DO_PATCH(tl0_mna, sun4v_mna) | |
362 | SUN4V_DO_PATCH(tl1_mna, sun4v_mna) | |
363 | SUN4V_DO_PATCH(tl0_lddfmna, sun4v_lddfmna) | |
364 | SUN4V_DO_PATCH(tl0_stdfmna, sun4v_stdfmna) | |
365 | SUN4V_DO_PATCH(tl0_privact, sun4v_privact) | |
d257d5da DM |
366 | retl |
367 | nop | |
368 | .size sun4v_patch_tlb_handlers,.-sun4v_patch_tlb_handlers |