Commit | Line | Data |
---|---|---|
5a0015d6 CZ |
1 | /* |
2 | * arch/xtensa/kernel/align.S | |
3 | * | |
4 | * Handle unalignment exceptions in kernel space. | |
5 | * | |
6 | * This file is subject to the terms and conditions of the GNU General | |
7 | * Public License. See the file "COPYING" in the main directory of | |
8 | * this archive for more details. | |
9 | * | |
10 | * Copyright (C) 2001 - 2005 Tensilica, Inc. | |
c3ef1f4d | 11 | * Copyright (C) 2014 Cadence Design Systems Inc. |
5a0015d6 CZ |
12 | * |
13 | * Rewritten by Chris Zankel <chris@zankel.net> | |
14 | * | |
15 | * Based on work from Joe Taylor <joe@tensilica.com, joetylr@yahoo.com> | |
16 | * and Marc Gauthier <marc@tensilica.com, marc@alimni.uwaterloo.ca> | |
17 | */ | |
18 | ||
19 | #include <linux/linkage.h> | |
5a0015d6 | 20 | #include <asm/current.h> |
0013a854 | 21 | #include <asm/asm-offsets.h> |
5a0015d6 | 22 | #include <asm/processor.h> |
5a0015d6 CZ |
23 | |
24 | #if XCHAL_UNALIGNED_LOAD_EXCEPTION || XCHAL_UNALIGNED_STORE_EXCEPTION | |
25 | ||
26 | /* First-level exception handler for unaligned exceptions. | |
27 | * | |
28 | * Note: This handler works only for kernel exceptions. Unaligned user | |
29 | * access should get a seg fault. | |
30 | */ | |
31 | ||
32 | /* Big and little endian 16-bit values are located in | |
33 | * different halves of a register. HWORD_START helps to | |
34 | * abstract the notion of extracting a 16-bit value from a | |
35 | * register. | |
36 | * We also have to define new shifting instructions because | |
37 | * lsb and msb are on 'opposite' ends in a register for | |
38 | * different endian machines. | |
39 | * | |
40 | * Assume a memory region in ascending address: | |
41 | * 0 1 2 3|4 5 6 7 | |
42 | * | |
43 | * When loading one word into a register, the content of that register is: | |
44 | * LE 3 2 1 0, 7 6 5 4 | |
45 | * BE 0 1 2 3, 4 5 6 7 | |
46 | * | |
47 | * Masking the bits of the higher/lower address means: | |
48 | * LE X X 0 0, 0 0 X X | |
49 | * BE 0 0 X X, X X 0 0 | |
50 | * | |
51 | * Shifting to higher/lower addresses, means: | |
52 | * LE shift left / shift right | |
53 | * BE shift right / shift left | |
54 | * | |
55 | * Extracting 16 bits from a 32 bit reg. value to higher/lower address means: | |
56 | * LE mask 0 0 X X / shift left | |
57 | * BE shift left / mask 0 0 X X | |
58 | */ | |
59 | ||
60 | #define UNALIGNED_USER_EXCEPTION | |
61 | ||
62 | #if XCHAL_HAVE_BE | |
63 | ||
64 | #define HWORD_START 16 | |
65 | #define INSN_OP0 28 | |
66 | #define INSN_T 24 | |
67 | #define INSN_OP1 16 | |
68 | ||
69 | .macro __src_b r, w0, w1; src \r, \w0, \w1; .endm | |
70 | .macro __ssa8 r; ssa8b \r; .endm | |
71 | .macro __ssa8r r; ssa8l \r; .endm | |
72 | .macro __sh r, s; srl \r, \s; .endm | |
73 | .macro __sl r, s; sll \r, \s; .endm | |
74 | .macro __exth r, s; extui \r, \s, 0, 16; .endm | |
75 | .macro __extl r, s; slli \r, \s, 16; .endm | |
76 | ||
77 | #else | |
78 | ||
79 | #define HWORD_START 0 | |
80 | #define INSN_OP0 0 | |
81 | #define INSN_T 4 | |
82 | #define INSN_OP1 12 | |
83 | ||
84 | .macro __src_b r, w0, w1; src \r, \w1, \w0; .endm | |
85 | .macro __ssa8 r; ssa8l \r; .endm | |
86 | .macro __ssa8r r; ssa8b \r; .endm | |
87 | .macro __sh r, s; sll \r, \s; .endm | |
88 | .macro __sl r, s; srl \r, \s; .endm | |
89 | .macro __exth r, s; slli \r, \s, 16; .endm | |
90 | .macro __extl r, s; extui \r, \s, 0, 16; .endm | |
91 | ||
92 | #endif | |
93 | ||
94 | /* | |
95 | * xxxx xxxx = imm8 field | |
96 | * yyyy = imm4 field | |
97 | * ssss = s field | |
98 | * tttt = t field | |
99 | * | |
100 | * 16 0 | |
101 | * ------------------- | |
102 | * L32I.N yyyy ssss tttt 1000 | |
103 | * S32I.N yyyy ssss tttt 1001 | |
104 | * | |
105 | * 23 0 | |
106 | * ----------------------------- | |
107 | * res 0000 0010 | |
108 | * L16UI xxxx xxxx 0001 ssss tttt 0010 | |
109 | * L32I xxxx xxxx 0010 ssss tttt 0010 | |
110 | * XXX 0011 ssss tttt 0010 | |
111 | * XXX 0100 ssss tttt 0010 | |
112 | * S16I xxxx xxxx 0101 ssss tttt 0010 | |
113 | * S32I xxxx xxxx 0110 ssss tttt 0010 | |
114 | * XXX 0111 ssss tttt 0010 | |
115 | * XXX 1000 ssss tttt 0010 | |
116 | * L16SI xxxx xxxx 1001 ssss tttt 0010 | |
117 | * XXX 1010 0010 | |
118 | * **L32AI xxxx xxxx 1011 ssss tttt 0010 unsupported | |
119 | * XXX 1100 0010 | |
120 | * XXX 1101 0010 | |
121 | * XXX 1110 0010 | |
122 | * **S32RI xxxx xxxx 1111 ssss tttt 0010 unsupported | |
123 | * ----------------------------- | |
124 | * ^ ^ ^ | |
125 | * sub-opcode (NIBBLE_R) -+ | | | |
126 | * t field (NIBBLE_T) -----------+ | | |
127 | * major opcode (NIBBLE_OP0) --------------+ | |
128 | */ | |
129 | ||
130 | #define OP0_L32I_N 0x8 /* load immediate narrow */ | |
131 | #define OP0_S32I_N 0x9 /* store immediate narrow */ | |
132 | #define OP1_SI_MASK 0x4 /* OP1 bit set for stores */ | |
133 | #define OP1_SI_BIT 2 /* OP1 bit number for stores */ | |
134 | ||
135 | #define OP1_L32I 0x2 | |
136 | #define OP1_L16UI 0x1 | |
137 | #define OP1_L16SI 0x9 | |
138 | #define OP1_L32AI 0xb | |
139 | ||
140 | #define OP1_S32I 0x6 | |
141 | #define OP1_S16I 0x5 | |
142 | #define OP1_S32RI 0xf | |
143 | ||
144 | /* | |
145 | * Entry condition: | |
146 | * | |
147 | * a0: trashed, original value saved on stack (PT_AREG0) | |
148 | * a1: a1 | |
149 | * a2: new stack pointer, original in DEPC | |
99d5040e | 150 | * a3: a3 |
5a0015d6 | 151 | * depc: a2, original value saved on stack (PT_DEPC) |
99d5040e | 152 | * excsave_1: dispatch table |
5a0015d6 CZ |
153 | * |
154 | * PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC | |
155 | * < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception | |
156 | */ | |
157 | ||
158 | ||
159 | ENTRY(fast_unaligned) | |
160 | ||
161 | /* Note: We don't expect the address to be aligned on a word | |
162 | * boundary. After all, the processor generated that exception | |
163 | * and it would be a hardware fault. | |
164 | */ | |
165 | ||
166 | /* Save some working register */ | |
167 | ||
168 | s32i a4, a2, PT_AREG4 | |
169 | s32i a5, a2, PT_AREG5 | |
170 | s32i a6, a2, PT_AREG6 | |
171 | s32i a7, a2, PT_AREG7 | |
172 | s32i a8, a2, PT_AREG8 | |
173 | ||
bc5378fc | 174 | rsr a0, depc |
5a0015d6 CZ |
175 | s32i a0, a2, PT_AREG2 |
176 | s32i a3, a2, PT_AREG3 | |
177 | ||
c3ef1f4d MF |
178 | rsr a3, excsave1 |
179 | movi a4, fast_unaligned_fixup | |
180 | s32i a4, a3, EXC_TABLE_FIXUP | |
181 | ||
5a0015d6 CZ |
182 | /* Keep value of SAR in a0 */ |
183 | ||
bc5378fc MF |
184 | rsr a0, sar |
185 | rsr a8, excvaddr # load unaligned memory address | |
5a0015d6 CZ |
186 | |
187 | /* Now, identify one of the following load/store instructions. | |
188 | * | |
189 | * The only possible danger of a double exception on the | |
190 | * following l32i instructions is kernel code in vmalloc | |
191 | * memory. The processor was just executing at the EPC_1 | |
192 | * address, and indeed, already fetched the instruction. That | |
193 | * guarantees a TLB mapping, which hasn't been replaced by | |
194 | * this unaligned exception handler that uses only static TLB | |
195 | * mappings. However, high-level interrupt handlers might | |
196 | * modify TLB entries, so for the generic case, we register a | |
197 | * TABLE_FIXUP handler here, too. | |
198 | */ | |
199 | ||
200 | /* a3...a6 saved on stack, a2 = SP */ | |
201 | ||
202 | /* Extract the instruction that caused the unaligned access. */ | |
203 | ||
bc5378fc | 204 | rsr a7, epc1 # load exception address |
5a0015d6 CZ |
205 | movi a3, ~3 |
206 | and a3, a3, a7 # mask lower bits | |
207 | ||
208 | l32i a4, a3, 0 # load 2 words | |
209 | l32i a5, a3, 4 | |
210 | ||
211 | __ssa8 a7 | |
212 | __src_b a4, a4, a5 # a4 has the instruction | |
213 | ||
214 | /* Analyze the instruction (load or store?). */ | |
215 | ||
216 | extui a5, a4, INSN_OP0, 4 # get insn.op0 nibble | |
217 | ||
173d6681 | 218 | #if XCHAL_HAVE_DENSITY |
5a0015d6 CZ |
219 | _beqi a5, OP0_L32I_N, .Lload # L32I.N, jump |
220 | addi a6, a5, -OP0_S32I_N | |
221 | _beqz a6, .Lstore # S32I.N, do a store | |
222 | #endif | |
223 | /* 'store indicator bit' not set, jump */ | |
224 | _bbci.l a4, OP1_SI_BIT + INSN_OP1, .Lload | |
225 | ||
226 | /* Store: Jump to table entry to get the value in the source register.*/ | |
227 | ||
228 | .Lstore:movi a5, .Lstore_table # table | |
229 | extui a6, a4, INSN_T, 4 # get source register | |
230 | addx8 a5, a6, a5 | |
231 | jx a5 # jump into table | |
232 | ||
5a0015d6 CZ |
233 | /* Load: Load memory address. */ |
234 | ||
235 | .Lload: movi a3, ~3 | |
236 | and a3, a3, a8 # align memory address | |
237 | ||
238 | __ssa8 a8 | |
239 | #ifdef UNALIGNED_USER_EXCEPTION | |
240 | addi a3, a3, 8 | |
241 | l32e a5, a3, -8 | |
242 | l32e a6, a3, -4 | |
243 | #else | |
244 | l32i a5, a3, 0 | |
245 | l32i a6, a3, 4 | |
246 | #endif | |
247 | __src_b a3, a5, a6 # a3 has the data word | |
248 | ||
173d6681 | 249 | #if XCHAL_HAVE_DENSITY |
5a0015d6 CZ |
250 | addi a7, a7, 2 # increment PC (assume 16-bit insn) |
251 | ||
252 | extui a5, a4, INSN_OP0, 4 | |
253 | _beqi a5, OP0_L32I_N, 1f # l32i.n: jump | |
254 | ||
255 | addi a7, a7, 1 | |
256 | #else | |
257 | addi a7, a7, 3 | |
258 | #endif | |
259 | ||
260 | extui a5, a4, INSN_OP1, 4 | |
261 | _beqi a5, OP1_L32I, 1f # l32i: jump | |
262 | ||
263 | extui a3, a3, 0, 16 # extract lower 16 bits | |
264 | _beqi a5, OP1_L16UI, 1f | |
265 | addi a5, a5, -OP1_L16SI | |
266 | _bnez a5, .Linvalid_instruction_load | |
267 | ||
268 | /* sign extend value */ | |
269 | ||
270 | slli a3, a3, 16 | |
271 | srai a3, a3, 16 | |
272 | ||
273 | /* Set target register. */ | |
274 | ||
275 | 1: | |
5a0015d6 CZ |
276 | extui a4, a4, INSN_T, 4 # extract target register |
277 | movi a5, .Lload_table | |
278 | addx8 a4, a4, a5 | |
279 | jx a4 # jump to entry for target register | |
280 | ||
281 | .align 8 | |
282 | .Lload_table: | |
283 | s32i a3, a2, PT_AREG0; _j .Lexit; .align 8 | |
284 | mov a1, a3; _j .Lexit; .align 8 # fishy?? | |
285 | s32i a3, a2, PT_AREG2; _j .Lexit; .align 8 | |
286 | s32i a3, a2, PT_AREG3; _j .Lexit; .align 8 | |
287 | s32i a3, a2, PT_AREG4; _j .Lexit; .align 8 | |
288 | s32i a3, a2, PT_AREG5; _j .Lexit; .align 8 | |
289 | s32i a3, a2, PT_AREG6; _j .Lexit; .align 8 | |
290 | s32i a3, a2, PT_AREG7; _j .Lexit; .align 8 | |
291 | s32i a3, a2, PT_AREG8; _j .Lexit; .align 8 | |
292 | mov a9, a3 ; _j .Lexit; .align 8 | |
293 | mov a10, a3 ; _j .Lexit; .align 8 | |
294 | mov a11, a3 ; _j .Lexit; .align 8 | |
295 | mov a12, a3 ; _j .Lexit; .align 8 | |
296 | mov a13, a3 ; _j .Lexit; .align 8 | |
297 | mov a14, a3 ; _j .Lexit; .align 8 | |
298 | mov a15, a3 ; _j .Lexit; .align 8 | |
299 | ||
300 | .Lstore_table: | |
301 | l32i a3, a2, PT_AREG0; _j 1f; .align 8 | |
302 | mov a3, a1; _j 1f; .align 8 # fishy?? | |
303 | l32i a3, a2, PT_AREG2; _j 1f; .align 8 | |
304 | l32i a3, a2, PT_AREG3; _j 1f; .align 8 | |
305 | l32i a3, a2, PT_AREG4; _j 1f; .align 8 | |
306 | l32i a3, a2, PT_AREG5; _j 1f; .align 8 | |
307 | l32i a3, a2, PT_AREG6; _j 1f; .align 8 | |
308 | l32i a3, a2, PT_AREG7; _j 1f; .align 8 | |
309 | l32i a3, a2, PT_AREG8; _j 1f; .align 8 | |
310 | mov a3, a9 ; _j 1f; .align 8 | |
311 | mov a3, a10 ; _j 1f; .align 8 | |
312 | mov a3, a11 ; _j 1f; .align 8 | |
313 | mov a3, a12 ; _j 1f; .align 8 | |
314 | mov a3, a13 ; _j 1f; .align 8 | |
315 | mov a3, a14 ; _j 1f; .align 8 | |
316 | mov a3, a15 ; _j 1f; .align 8 | |
317 | ||
21570465 MF |
318 | /* We cannot handle this exception. */ |
319 | ||
320 | .extern _kernel_exception | |
321 | .Linvalid_instruction_load: | |
322 | .Linvalid_instruction_store: | |
323 | ||
324 | movi a4, 0 | |
325 | rsr a3, excsave1 | |
326 | s32i a4, a3, EXC_TABLE_FIXUP | |
327 | ||
328 | /* Restore a4...a8 and SAR, set SP, and jump to default exception. */ | |
329 | ||
330 | l32i a8, a2, PT_AREG8 | |
331 | l32i a7, a2, PT_AREG7 | |
332 | l32i a6, a2, PT_AREG6 | |
333 | l32i a5, a2, PT_AREG5 | |
334 | l32i a4, a2, PT_AREG4 | |
335 | wsr a0, sar | |
336 | mov a1, a2 | |
337 | ||
338 | rsr a0, ps | |
339 | bbsi.l a0, PS_UM_BIT, 2f # jump if user mode | |
340 | ||
341 | movi a0, _kernel_exception | |
342 | jx a0 | |
343 | ||
344 | 2: movi a0, _user_exception | |
345 | jx a0 | |
346 | ||
5a0015d6 CZ |
347 | 1: # a7: instruction pointer, a4: instruction, a3: value |
348 | ||
349 | movi a6, 0 # mask: ffffffff:00000000 | |
350 | ||
173d6681 | 351 | #if XCHAL_HAVE_DENSITY |
5a0015d6 CZ |
352 | addi a7, a7, 2 # incr. PC,assume 16-bit instruction |
353 | ||
354 | extui a5, a4, INSN_OP0, 4 # extract OP0 | |
355 | addi a5, a5, -OP0_S32I_N | |
356 | _beqz a5, 1f # s32i.n: jump | |
357 | ||
358 | addi a7, a7, 1 # increment PC, 32-bit instruction | |
359 | #else | |
360 | addi a7, a7, 3 # increment PC, 32-bit instruction | |
361 | #endif | |
362 | ||
363 | extui a5, a4, INSN_OP1, 4 # extract OP1 | |
364 | _beqi a5, OP1_S32I, 1f # jump if 32 bit store | |
365 | _bnei a5, OP1_S16I, .Linvalid_instruction_store | |
366 | ||
367 | movi a5, -1 | |
368 | __extl a3, a3 # get 16-bit value | |
369 | __exth a6, a5 # get 16-bit mask ffffffff:ffff0000 | |
370 | ||
371 | /* Get memory address */ | |
372 | ||
373 | 1: | |
5a0015d6 CZ |
374 | movi a4, ~3 |
375 | and a4, a4, a8 # align memory address | |
376 | ||
377 | /* Insert value into memory */ | |
378 | ||
379 | movi a5, -1 # mask: ffffffff:XXXX0000 | |
380 | #ifdef UNALIGNED_USER_EXCEPTION | |
381 | addi a4, a4, 8 | |
382 | #endif | |
383 | ||
384 | __ssa8r a8 | |
e9500dd8 | 385 | __src_b a8, a5, a6 # lo-mask F..F0..0 (BE) 0..0F..F (LE) |
5a0015d6 CZ |
386 | __src_b a6, a6, a5 # hi-mask 0..0F..F (BE) F..F0..0 (LE) |
387 | #ifdef UNALIGNED_USER_EXCEPTION | |
388 | l32e a5, a4, -8 | |
389 | #else | |
390 | l32i a5, a4, 0 # load lower address word | |
391 | #endif | |
e9500dd8 MF |
392 | and a5, a5, a8 # mask |
393 | __sh a8, a3 # shift value | |
394 | or a5, a5, a8 # or with original value | |
5a0015d6 CZ |
395 | #ifdef UNALIGNED_USER_EXCEPTION |
396 | s32e a5, a4, -8 | |
e9500dd8 | 397 | l32e a8, a4, -4 |
5a0015d6 CZ |
398 | #else |
399 | s32i a5, a4, 0 # store | |
e9500dd8 | 400 | l32i a8, a4, 4 # same for upper address word |
5a0015d6 CZ |
401 | #endif |
402 | __sl a5, a3 | |
e9500dd8 | 403 | and a6, a8, a6 |
5a0015d6 CZ |
404 | or a6, a6, a5 |
405 | #ifdef UNALIGNED_USER_EXCEPTION | |
406 | s32e a6, a4, -4 | |
407 | #else | |
408 | s32i a6, a4, 4 | |
409 | #endif | |
410 | ||
5a0015d6 | 411 | .Lexit: |
e9500dd8 MF |
412 | #if XCHAL_HAVE_LOOPS |
413 | rsr a4, lend # check if we reached LEND | |
414 | bne a7, a4, 1f | |
415 | rsr a4, lcount # and LCOUNT != 0 | |
416 | beqz a4, 1f | |
417 | addi a4, a4, -1 # decrement LCOUNT and set | |
418 | rsr a7, lbeg # set PC to LBEGIN | |
419 | wsr a4, lcount | |
420 | #endif | |
421 | ||
422 | 1: wsr a7, epc1 # skip emulated instruction | |
423 | ||
a83b02e9 MF |
424 | /* Update icount if we're single-stepping in userspace. */ |
425 | rsr a4, icountlevel | |
426 | beqz a4, 1f | |
427 | bgeui a4, LOCKLEVEL + 1, 1f | |
428 | rsr a4, icount | |
429 | addi a4, a4, 1 | |
430 | wsr a4, icount | |
431 | 1: | |
5a0015d6 | 432 | movi a4, 0 |
bc5378fc | 433 | rsr a3, excsave1 |
5a0015d6 CZ |
434 | s32i a4, a3, EXC_TABLE_FIXUP |
435 | ||
436 | /* Restore working register */ | |
437 | ||
173d6681 | 438 | l32i a8, a2, PT_AREG8 |
5a0015d6 CZ |
439 | l32i a7, a2, PT_AREG7 |
440 | l32i a6, a2, PT_AREG6 | |
441 | l32i a5, a2, PT_AREG5 | |
442 | l32i a4, a2, PT_AREG4 | |
443 | l32i a3, a2, PT_AREG3 | |
444 | ||
445 | /* restore SAR and return */ | |
446 | ||
bc5378fc | 447 | wsr a0, sar |
5a0015d6 CZ |
448 | l32i a0, a2, PT_AREG0 |
449 | l32i a2, a2, PT_AREG2 | |
450 | rfe | |
451 | ||
d1538c46 | 452 | ENDPROC(fast_unaligned) |
5a0015d6 | 453 | |
c3ef1f4d MF |
454 | ENTRY(fast_unaligned_fixup) |
455 | ||
456 | l32i a2, a3, EXC_TABLE_DOUBLE_SAVE | |
457 | wsr a3, excsave1 | |
458 | ||
459 | l32i a8, a2, PT_AREG8 | |
460 | l32i a7, a2, PT_AREG7 | |
461 | l32i a6, a2, PT_AREG6 | |
462 | l32i a5, a2, PT_AREG5 | |
463 | l32i a4, a2, PT_AREG4 | |
464 | l32i a0, a2, PT_AREG2 | |
465 | xsr a0, depc # restore depc and a0 | |
466 | wsr a0, sar | |
467 | ||
468 | rsr a0, exccause | |
469 | s32i a0, a2, PT_DEPC # mark as a regular exception | |
470 | ||
471 | rsr a0, ps | |
472 | bbsi.l a0, PS_UM_BIT, 1f # jump if user mode | |
473 | ||
474 | rsr a0, exccause | |
475 | addx4 a0, a0, a3 # find entry in table | |
476 | l32i a0, a0, EXC_TABLE_FAST_KERNEL # load handler | |
477 | l32i a3, a2, PT_AREG3 | |
478 | jx a0 | |
479 | 1: | |
480 | rsr a0, exccause | |
481 | addx4 a0, a0, a3 # find entry in table | |
482 | l32i a0, a0, EXC_TABLE_FAST_USER # load handler | |
483 | l32i a3, a2, PT_AREG3 | |
484 | jx a0 | |
485 | ||
486 | ENDPROC(fast_unaligned_fixup) | |
487 | ||
5a0015d6 | 488 | #endif /* XCHAL_UNALIGNED_LOAD_EXCEPTION || XCHAL_UNALIGNED_STORE_EXCEPTION */ |