Commit | Line | Data |
---|---|---|
080fc66f RK |
1 | /* |
2 | * linux/arch/arm/lib/uaccess.S | |
3 | * | |
4 | * Copyright (C) 1995, 1996,1997,1998 Russell King | |
5 | * | |
6 | * This program is free software; you can redistribute it and/or modify | |
7 | * it under the terms of the GNU General Public License version 2 as | |
8 | * published by the Free Software Foundation. | |
9 | * | |
10 | * Routines to block copy data to/from user memory | |
11 | * These are highly optimised both for the 4k page size | |
12 | * and for various alignments. | |
13 | */ | |
14 | #include <linux/linkage.h> | |
15 | #include <asm/assembler.h> | |
16 | #include <asm/errno.h> | |
17 | #include <asm/domain.h> | |
18 | ||
19 | .text | |
20 | ||
21 | #define PAGE_SHIFT 12 | |
22 | ||
23 | /* Prototype: int __copy_to_user(void *to, const char *from, size_t n) | |
24 | * Purpose : copy a block to user memory from kernel memory | |
25 | * Params : to - user memory | |
26 | * : from - kernel memory | |
27 | * : n - number of bytes to copy | |
28 | * Returns : Number of bytes NOT copied. | |
29 | */ | |
30 | ||
31 | .Lc2u_dest_not_aligned: | |
32 | rsb ip, ip, #4 | |
33 | cmp ip, #2 | |
34 | ldrb r3, [r1], #1 | |
35 | USER( TUSER( strb) r3, [r0], #1) @ May fault | |
36 | ldrgeb r3, [r1], #1 | |
37 | USER( TUSER( strgeb) r3, [r0], #1) @ May fault | |
38 | ldrgtb r3, [r1], #1 | |
39 | USER( TUSER( strgtb) r3, [r0], #1) @ May fault | |
40 | sub r2, r2, ip | |
41 | b .Lc2u_dest_aligned | |
42 | ||
43 | ENTRY(__copy_to_user) | |
44 | stmfd sp!, {r2, r4 - r7, lr} | |
45 | cmp r2, #4 | |
46 | blt .Lc2u_not_enough | |
47 | ands ip, r0, #3 | |
48 | bne .Lc2u_dest_not_aligned | |
49 | .Lc2u_dest_aligned: | |
50 | ||
51 | ands ip, r1, #3 | |
52 | bne .Lc2u_src_not_aligned | |
53 | /* | |
54 | * Seeing as there has to be at least 8 bytes to copy, we can | |
55 | * copy one word, and force a user-mode page fault... | |
56 | */ | |
57 | ||
58 | .Lc2u_0fupi: subs r2, r2, #4 | |
59 | addmi ip, r2, #4 | |
60 | bmi .Lc2u_0nowords | |
61 | ldr r3, [r1], #4 | |
62 | USER( TUSER( str) r3, [r0], #4) @ May fault | |
63 | mov ip, r0, lsl #32 - PAGE_SHIFT @ On each page, use a ld/st??t instruction | |
64 | rsb ip, ip, #0 | |
65 | movs ip, ip, lsr #32 - PAGE_SHIFT | |
66 | beq .Lc2u_0fupi | |
67 | /* | |
68 | * ip = max no. of bytes to copy before needing another "strt" insn | |
69 | */ | |
70 | cmp r2, ip | |
71 | movlt ip, r2 | |
72 | sub r2, r2, ip | |
73 | subs ip, ip, #32 | |
74 | blt .Lc2u_0rem8lp | |
75 | ||
76 | .Lc2u_0cpy8lp: ldmia r1!, {r3 - r6} | |
77 | stmia r0!, {r3 - r6} @ Shouldnt fault | |
78 | ldmia r1!, {r3 - r6} | |
79 | subs ip, ip, #32 | |
80 | stmia r0!, {r3 - r6} @ Shouldnt fault | |
81 | bpl .Lc2u_0cpy8lp | |
82 | ||
83 | .Lc2u_0rem8lp: cmn ip, #16 | |
84 | ldmgeia r1!, {r3 - r6} | |
85 | stmgeia r0!, {r3 - r6} @ Shouldnt fault | |
86 | tst ip, #8 | |
87 | ldmneia r1!, {r3 - r4} | |
88 | stmneia r0!, {r3 - r4} @ Shouldnt fault | |
89 | tst ip, #4 | |
90 | ldrne r3, [r1], #4 | |
91 | TUSER( strne) r3, [r0], #4 @ Shouldnt fault | |
92 | ands ip, ip, #3 | |
93 | beq .Lc2u_0fupi | |
94 | .Lc2u_0nowords: teq ip, #0 | |
95 | beq .Lc2u_finished | |
96 | .Lc2u_nowords: cmp ip, #2 | |
97 | ldrb r3, [r1], #1 | |
98 | USER( TUSER( strb) r3, [r0], #1) @ May fault | |
99 | ldrgeb r3, [r1], #1 | |
100 | USER( TUSER( strgeb) r3, [r0], #1) @ May fault | |
101 | ldrgtb r3, [r1], #1 | |
102 | USER( TUSER( strgtb) r3, [r0], #1) @ May fault | |
103 | b .Lc2u_finished | |
104 | ||
105 | .Lc2u_not_enough: | |
106 | movs ip, r2 | |
107 | bne .Lc2u_nowords | |
108 | .Lc2u_finished: mov r0, #0 | |
109 | ldmfd sp!, {r2, r4 - r7, pc} | |
110 | ||
111 | .Lc2u_src_not_aligned: | |
112 | bic r1, r1, #3 | |
113 | ldr r7, [r1], #4 | |
114 | cmp ip, #2 | |
115 | bgt .Lc2u_3fupi | |
116 | beq .Lc2u_2fupi | |
117 | .Lc2u_1fupi: subs r2, r2, #4 | |
118 | addmi ip, r2, #4 | |
119 | bmi .Lc2u_1nowords | |
d98b90ea | 120 | mov r3, r7, lspull #8 |
080fc66f | 121 | ldr r7, [r1], #4 |
d98b90ea | 122 | orr r3, r3, r7, lspush #24 |
080fc66f RK |
123 | USER( TUSER( str) r3, [r0], #4) @ May fault |
124 | mov ip, r0, lsl #32 - PAGE_SHIFT | |
125 | rsb ip, ip, #0 | |
126 | movs ip, ip, lsr #32 - PAGE_SHIFT | |
127 | beq .Lc2u_1fupi | |
128 | cmp r2, ip | |
129 | movlt ip, r2 | |
130 | sub r2, r2, ip | |
131 | subs ip, ip, #16 | |
132 | blt .Lc2u_1rem8lp | |
133 | ||
d98b90ea | 134 | .Lc2u_1cpy8lp: mov r3, r7, lspull #8 |
080fc66f RK |
135 | ldmia r1!, {r4 - r7} |
136 | subs ip, ip, #16 | |
d98b90ea VK |
137 | orr r3, r3, r4, lspush #24 |
138 | mov r4, r4, lspull #8 | |
139 | orr r4, r4, r5, lspush #24 | |
140 | mov r5, r5, lspull #8 | |
141 | orr r5, r5, r6, lspush #24 | |
142 | mov r6, r6, lspull #8 | |
143 | orr r6, r6, r7, lspush #24 | |
080fc66f RK |
144 | stmia r0!, {r3 - r6} @ Shouldnt fault |
145 | bpl .Lc2u_1cpy8lp | |
146 | ||
147 | .Lc2u_1rem8lp: tst ip, #8 | |
d98b90ea | 148 | movne r3, r7, lspull #8 |
080fc66f | 149 | ldmneia r1!, {r4, r7} |
d98b90ea VK |
150 | orrne r3, r3, r4, lspush #24 |
151 | movne r4, r4, lspull #8 | |
152 | orrne r4, r4, r7, lspush #24 | |
080fc66f RK |
153 | stmneia r0!, {r3 - r4} @ Shouldnt fault |
154 | tst ip, #4 | |
d98b90ea | 155 | movne r3, r7, lspull #8 |
080fc66f | 156 | ldrne r7, [r1], #4 |
d98b90ea | 157 | orrne r3, r3, r7, lspush #24 |
080fc66f RK |
158 | TUSER( strne) r3, [r0], #4 @ Shouldnt fault |
159 | ands ip, ip, #3 | |
160 | beq .Lc2u_1fupi | |
161 | .Lc2u_1nowords: mov r3, r7, get_byte_1 | |
162 | teq ip, #0 | |
163 | beq .Lc2u_finished | |
164 | cmp ip, #2 | |
165 | USER( TUSER( strb) r3, [r0], #1) @ May fault | |
166 | movge r3, r7, get_byte_2 | |
167 | USER( TUSER( strgeb) r3, [r0], #1) @ May fault | |
168 | movgt r3, r7, get_byte_3 | |
169 | USER( TUSER( strgtb) r3, [r0], #1) @ May fault | |
170 | b .Lc2u_finished | |
171 | ||
172 | .Lc2u_2fupi: subs r2, r2, #4 | |
173 | addmi ip, r2, #4 | |
174 | bmi .Lc2u_2nowords | |
d98b90ea | 175 | mov r3, r7, lspull #16 |
080fc66f | 176 | ldr r7, [r1], #4 |
d98b90ea | 177 | orr r3, r3, r7, lspush #16 |
080fc66f RK |
178 | USER( TUSER( str) r3, [r0], #4) @ May fault |
179 | mov ip, r0, lsl #32 - PAGE_SHIFT | |
180 | rsb ip, ip, #0 | |
181 | movs ip, ip, lsr #32 - PAGE_SHIFT | |
182 | beq .Lc2u_2fupi | |
183 | cmp r2, ip | |
184 | movlt ip, r2 | |
185 | sub r2, r2, ip | |
186 | subs ip, ip, #16 | |
187 | blt .Lc2u_2rem8lp | |
188 | ||
d98b90ea | 189 | .Lc2u_2cpy8lp: mov r3, r7, lspull #16 |
080fc66f RK |
190 | ldmia r1!, {r4 - r7} |
191 | subs ip, ip, #16 | |
d98b90ea VK |
192 | orr r3, r3, r4, lspush #16 |
193 | mov r4, r4, lspull #16 | |
194 | orr r4, r4, r5, lspush #16 | |
195 | mov r5, r5, lspull #16 | |
196 | orr r5, r5, r6, lspush #16 | |
197 | mov r6, r6, lspull #16 | |
198 | orr r6, r6, r7, lspush #16 | |
080fc66f RK |
199 | stmia r0!, {r3 - r6} @ Shouldnt fault |
200 | bpl .Lc2u_2cpy8lp | |
201 | ||
202 | .Lc2u_2rem8lp: tst ip, #8 | |
d98b90ea | 203 | movne r3, r7, lspull #16 |
080fc66f | 204 | ldmneia r1!, {r4, r7} |
d98b90ea VK |
205 | orrne r3, r3, r4, lspush #16 |
206 | movne r4, r4, lspull #16 | |
207 | orrne r4, r4, r7, lspush #16 | |
080fc66f RK |
208 | stmneia r0!, {r3 - r4} @ Shouldnt fault |
209 | tst ip, #4 | |
d98b90ea | 210 | movne r3, r7, lspull #16 |
080fc66f | 211 | ldrne r7, [r1], #4 |
d98b90ea | 212 | orrne r3, r3, r7, lspush #16 |
080fc66f RK |
213 | TUSER( strne) r3, [r0], #4 @ Shouldnt fault |
214 | ands ip, ip, #3 | |
215 | beq .Lc2u_2fupi | |
216 | .Lc2u_2nowords: mov r3, r7, get_byte_2 | |
217 | teq ip, #0 | |
218 | beq .Lc2u_finished | |
219 | cmp ip, #2 | |
220 | USER( TUSER( strb) r3, [r0], #1) @ May fault | |
221 | movge r3, r7, get_byte_3 | |
222 | USER( TUSER( strgeb) r3, [r0], #1) @ May fault | |
223 | ldrgtb r3, [r1], #0 | |
224 | USER( TUSER( strgtb) r3, [r0], #1) @ May fault | |
225 | b .Lc2u_finished | |
226 | ||
227 | .Lc2u_3fupi: subs r2, r2, #4 | |
228 | addmi ip, r2, #4 | |
229 | bmi .Lc2u_3nowords | |
d98b90ea | 230 | mov r3, r7, lspull #24 |
080fc66f | 231 | ldr r7, [r1], #4 |
d98b90ea | 232 | orr r3, r3, r7, lspush #8 |
080fc66f RK |
233 | USER( TUSER( str) r3, [r0], #4) @ May fault |
234 | mov ip, r0, lsl #32 - PAGE_SHIFT | |
235 | rsb ip, ip, #0 | |
236 | movs ip, ip, lsr #32 - PAGE_SHIFT | |
237 | beq .Lc2u_3fupi | |
238 | cmp r2, ip | |
239 | movlt ip, r2 | |
240 | sub r2, r2, ip | |
241 | subs ip, ip, #16 | |
242 | blt .Lc2u_3rem8lp | |
243 | ||
d98b90ea | 244 | .Lc2u_3cpy8lp: mov r3, r7, lspull #24 |
080fc66f RK |
245 | ldmia r1!, {r4 - r7} |
246 | subs ip, ip, #16 | |
d98b90ea VK |
247 | orr r3, r3, r4, lspush #8 |
248 | mov r4, r4, lspull #24 | |
249 | orr r4, r4, r5, lspush #8 | |
250 | mov r5, r5, lspull #24 | |
251 | orr r5, r5, r6, lspush #8 | |
252 | mov r6, r6, lspull #24 | |
253 | orr r6, r6, r7, lspush #8 | |
080fc66f RK |
254 | stmia r0!, {r3 - r6} @ Shouldnt fault |
255 | bpl .Lc2u_3cpy8lp | |
256 | ||
257 | .Lc2u_3rem8lp: tst ip, #8 | |
d98b90ea | 258 | movne r3, r7, lspull #24 |
080fc66f | 259 | ldmneia r1!, {r4, r7} |
d98b90ea VK |
260 | orrne r3, r3, r4, lspush #8 |
261 | movne r4, r4, lspull #24 | |
262 | orrne r4, r4, r7, lspush #8 | |
080fc66f RK |
263 | stmneia r0!, {r3 - r4} @ Shouldnt fault |
264 | tst ip, #4 | |
d98b90ea | 265 | movne r3, r7, lspull #24 |
080fc66f | 266 | ldrne r7, [r1], #4 |
d98b90ea | 267 | orrne r3, r3, r7, lspush #8 |
080fc66f RK |
268 | TUSER( strne) r3, [r0], #4 @ Shouldnt fault |
269 | ands ip, ip, #3 | |
270 | beq .Lc2u_3fupi | |
271 | .Lc2u_3nowords: mov r3, r7, get_byte_3 | |
272 | teq ip, #0 | |
273 | beq .Lc2u_finished | |
274 | cmp ip, #2 | |
275 | USER( TUSER( strb) r3, [r0], #1) @ May fault | |
276 | ldrgeb r3, [r1], #1 | |
277 | USER( TUSER( strgeb) r3, [r0], #1) @ May fault | |
278 | ldrgtb r3, [r1], #0 | |
279 | USER( TUSER( strgtb) r3, [r0], #1) @ May fault | |
280 | b .Lc2u_finished | |
281 | ENDPROC(__copy_to_user) | |
282 | ||
283 | .pushsection .fixup,"ax" | |
284 | .align 0 | |
285 | 9001: ldmfd sp!, {r0, r4 - r7, pc} | |
286 | .popsection | |
287 | ||
288 | /* Prototype: unsigned long __copy_from_user(void *to,const void *from,unsigned long n); | |
289 | * Purpose : copy a block from user memory to kernel memory | |
290 | * Params : to - kernel memory | |
291 | * : from - user memory | |
292 | * : n - number of bytes to copy | |
293 | * Returns : Number of bytes NOT copied. | |
294 | */ | |
295 | .Lcfu_dest_not_aligned: | |
296 | rsb ip, ip, #4 | |
297 | cmp ip, #2 | |
298 | USER( TUSER( ldrb) r3, [r1], #1) @ May fault | |
299 | strb r3, [r0], #1 | |
300 | USER( TUSER( ldrgeb) r3, [r1], #1) @ May fault | |
301 | strgeb r3, [r0], #1 | |
302 | USER( TUSER( ldrgtb) r3, [r1], #1) @ May fault | |
303 | strgtb r3, [r0], #1 | |
304 | sub r2, r2, ip | |
305 | b .Lcfu_dest_aligned | |
306 | ||
307 | ENTRY(__copy_from_user) | |
308 | stmfd sp!, {r0, r2, r4 - r7, lr} | |
309 | cmp r2, #4 | |
310 | blt .Lcfu_not_enough | |
311 | ands ip, r0, #3 | |
312 | bne .Lcfu_dest_not_aligned | |
313 | .Lcfu_dest_aligned: | |
314 | ands ip, r1, #3 | |
315 | bne .Lcfu_src_not_aligned | |
316 | ||
317 | /* | |
318 | * Seeing as there has to be at least 8 bytes to copy, we can | |
319 | * copy one word, and force a user-mode page fault... | |
320 | */ | |
321 | ||
322 | .Lcfu_0fupi: subs r2, r2, #4 | |
323 | addmi ip, r2, #4 | |
324 | bmi .Lcfu_0nowords | |
325 | USER( TUSER( ldr) r3, [r1], #4) | |
326 | str r3, [r0], #4 | |
327 | mov ip, r1, lsl #32 - PAGE_SHIFT @ On each page, use a ld/st??t instruction | |
328 | rsb ip, ip, #0 | |
329 | movs ip, ip, lsr #32 - PAGE_SHIFT | |
330 | beq .Lcfu_0fupi | |
331 | /* | |
332 | * ip = max no. of bytes to copy before needing another "strt" insn | |
333 | */ | |
334 | cmp r2, ip | |
335 | movlt ip, r2 | |
336 | sub r2, r2, ip | |
337 | subs ip, ip, #32 | |
338 | blt .Lcfu_0rem8lp | |
339 | ||
340 | .Lcfu_0cpy8lp: ldmia r1!, {r3 - r6} @ Shouldnt fault | |
341 | stmia r0!, {r3 - r6} | |
342 | ldmia r1!, {r3 - r6} @ Shouldnt fault | |
343 | subs ip, ip, #32 | |
344 | stmia r0!, {r3 - r6} | |
345 | bpl .Lcfu_0cpy8lp | |
346 | ||
347 | .Lcfu_0rem8lp: cmn ip, #16 | |
348 | ldmgeia r1!, {r3 - r6} @ Shouldnt fault | |
349 | stmgeia r0!, {r3 - r6} | |
350 | tst ip, #8 | |
351 | ldmneia r1!, {r3 - r4} @ Shouldnt fault | |
352 | stmneia r0!, {r3 - r4} | |
353 | tst ip, #4 | |
354 | TUSER( ldrne) r3, [r1], #4 @ Shouldnt fault | |
355 | strne r3, [r0], #4 | |
356 | ands ip, ip, #3 | |
357 | beq .Lcfu_0fupi | |
358 | .Lcfu_0nowords: teq ip, #0 | |
359 | beq .Lcfu_finished | |
360 | .Lcfu_nowords: cmp ip, #2 | |
361 | USER( TUSER( ldrb) r3, [r1], #1) @ May fault | |
362 | strb r3, [r0], #1 | |
363 | USER( TUSER( ldrgeb) r3, [r1], #1) @ May fault | |
364 | strgeb r3, [r0], #1 | |
365 | USER( TUSER( ldrgtb) r3, [r1], #1) @ May fault | |
366 | strgtb r3, [r0], #1 | |
367 | b .Lcfu_finished | |
368 | ||
369 | .Lcfu_not_enough: | |
370 | movs ip, r2 | |
371 | bne .Lcfu_nowords | |
372 | .Lcfu_finished: mov r0, #0 | |
373 | add sp, sp, #8 | |
374 | ldmfd sp!, {r4 - r7, pc} | |
375 | ||
376 | .Lcfu_src_not_aligned: | |
377 | bic r1, r1, #3 | |
378 | USER( TUSER( ldr) r7, [r1], #4) @ May fault | |
379 | cmp ip, #2 | |
380 | bgt .Lcfu_3fupi | |
381 | beq .Lcfu_2fupi | |
382 | .Lcfu_1fupi: subs r2, r2, #4 | |
383 | addmi ip, r2, #4 | |
384 | bmi .Lcfu_1nowords | |
d98b90ea | 385 | mov r3, r7, lspull #8 |
080fc66f | 386 | USER( TUSER( ldr) r7, [r1], #4) @ May fault |
d98b90ea | 387 | orr r3, r3, r7, lspush #24 |
080fc66f RK |
388 | str r3, [r0], #4 |
389 | mov ip, r1, lsl #32 - PAGE_SHIFT | |
390 | rsb ip, ip, #0 | |
391 | movs ip, ip, lsr #32 - PAGE_SHIFT | |
392 | beq .Lcfu_1fupi | |
393 | cmp r2, ip | |
394 | movlt ip, r2 | |
395 | sub r2, r2, ip | |
396 | subs ip, ip, #16 | |
397 | blt .Lcfu_1rem8lp | |
398 | ||
d98b90ea | 399 | .Lcfu_1cpy8lp: mov r3, r7, lspull #8 |
080fc66f RK |
400 | ldmia r1!, {r4 - r7} @ Shouldnt fault |
401 | subs ip, ip, #16 | |
d98b90ea VK |
402 | orr r3, r3, r4, lspush #24 |
403 | mov r4, r4, lspull #8 | |
404 | orr r4, r4, r5, lspush #24 | |
405 | mov r5, r5, lspull #8 | |
406 | orr r5, r5, r6, lspush #24 | |
407 | mov r6, r6, lspull #8 | |
408 | orr r6, r6, r7, lspush #24 | |
080fc66f RK |
409 | stmia r0!, {r3 - r6} |
410 | bpl .Lcfu_1cpy8lp | |
411 | ||
412 | .Lcfu_1rem8lp: tst ip, #8 | |
d98b90ea | 413 | movne r3, r7, lspull #8 |
080fc66f | 414 | ldmneia r1!, {r4, r7} @ Shouldnt fault |
d98b90ea VK |
415 | orrne r3, r3, r4, lspush #24 |
416 | movne r4, r4, lspull #8 | |
417 | orrne r4, r4, r7, lspush #24 | |
080fc66f RK |
418 | stmneia r0!, {r3 - r4} |
419 | tst ip, #4 | |
d98b90ea | 420 | movne r3, r7, lspull #8 |
080fc66f | 421 | USER( TUSER( ldrne) r7, [r1], #4) @ May fault |
d98b90ea | 422 | orrne r3, r3, r7, lspush #24 |
080fc66f RK |
423 | strne r3, [r0], #4 |
424 | ands ip, ip, #3 | |
425 | beq .Lcfu_1fupi | |
426 | .Lcfu_1nowords: mov r3, r7, get_byte_1 | |
427 | teq ip, #0 | |
428 | beq .Lcfu_finished | |
429 | cmp ip, #2 | |
430 | strb r3, [r0], #1 | |
431 | movge r3, r7, get_byte_2 | |
432 | strgeb r3, [r0], #1 | |
433 | movgt r3, r7, get_byte_3 | |
434 | strgtb r3, [r0], #1 | |
435 | b .Lcfu_finished | |
436 | ||
437 | .Lcfu_2fupi: subs r2, r2, #4 | |
438 | addmi ip, r2, #4 | |
439 | bmi .Lcfu_2nowords | |
d98b90ea | 440 | mov r3, r7, lspull #16 |
080fc66f | 441 | USER( TUSER( ldr) r7, [r1], #4) @ May fault |
d98b90ea | 442 | orr r3, r3, r7, lspush #16 |
080fc66f RK |
443 | str r3, [r0], #4 |
444 | mov ip, r1, lsl #32 - PAGE_SHIFT | |
445 | rsb ip, ip, #0 | |
446 | movs ip, ip, lsr #32 - PAGE_SHIFT | |
447 | beq .Lcfu_2fupi | |
448 | cmp r2, ip | |
449 | movlt ip, r2 | |
450 | sub r2, r2, ip | |
451 | subs ip, ip, #16 | |
452 | blt .Lcfu_2rem8lp | |
453 | ||
454 | ||
d98b90ea | 455 | .Lcfu_2cpy8lp: mov r3, r7, lspull #16 |
080fc66f RK |
456 | ldmia r1!, {r4 - r7} @ Shouldnt fault |
457 | subs ip, ip, #16 | |
d98b90ea VK |
458 | orr r3, r3, r4, lspush #16 |
459 | mov r4, r4, lspull #16 | |
460 | orr r4, r4, r5, lspush #16 | |
461 | mov r5, r5, lspull #16 | |
462 | orr r5, r5, r6, lspush #16 | |
463 | mov r6, r6, lspull #16 | |
464 | orr r6, r6, r7, lspush #16 | |
080fc66f RK |
465 | stmia r0!, {r3 - r6} |
466 | bpl .Lcfu_2cpy8lp | |
467 | ||
468 | .Lcfu_2rem8lp: tst ip, #8 | |
d98b90ea | 469 | movne r3, r7, lspull #16 |
080fc66f | 470 | ldmneia r1!, {r4, r7} @ Shouldnt fault |
d98b90ea VK |
471 | orrne r3, r3, r4, lspush #16 |
472 | movne r4, r4, lspull #16 | |
473 | orrne r4, r4, r7, lspush #16 | |
080fc66f RK |
474 | stmneia r0!, {r3 - r4} |
475 | tst ip, #4 | |
d98b90ea | 476 | movne r3, r7, lspull #16 |
080fc66f | 477 | USER( TUSER( ldrne) r7, [r1], #4) @ May fault |
d98b90ea | 478 | orrne r3, r3, r7, lspush #16 |
080fc66f RK |
479 | strne r3, [r0], #4 |
480 | ands ip, ip, #3 | |
481 | beq .Lcfu_2fupi | |
482 | .Lcfu_2nowords: mov r3, r7, get_byte_2 | |
483 | teq ip, #0 | |
484 | beq .Lcfu_finished | |
485 | cmp ip, #2 | |
486 | strb r3, [r0], #1 | |
487 | movge r3, r7, get_byte_3 | |
488 | strgeb r3, [r0], #1 | |
489 | USER( TUSER( ldrgtb) r3, [r1], #0) @ May fault | |
490 | strgtb r3, [r0], #1 | |
491 | b .Lcfu_finished | |
492 | ||
493 | .Lcfu_3fupi: subs r2, r2, #4 | |
494 | addmi ip, r2, #4 | |
495 | bmi .Lcfu_3nowords | |
d98b90ea | 496 | mov r3, r7, lspull #24 |
080fc66f | 497 | USER( TUSER( ldr) r7, [r1], #4) @ May fault |
d98b90ea | 498 | orr r3, r3, r7, lspush #8 |
080fc66f RK |
499 | str r3, [r0], #4 |
500 | mov ip, r1, lsl #32 - PAGE_SHIFT | |
501 | rsb ip, ip, #0 | |
502 | movs ip, ip, lsr #32 - PAGE_SHIFT | |
503 | beq .Lcfu_3fupi | |
504 | cmp r2, ip | |
505 | movlt ip, r2 | |
506 | sub r2, r2, ip | |
507 | subs ip, ip, #16 | |
508 | blt .Lcfu_3rem8lp | |
509 | ||
d98b90ea | 510 | .Lcfu_3cpy8lp: mov r3, r7, lspull #24 |
080fc66f | 511 | ldmia r1!, {r4 - r7} @ Shouldnt fault |
d98b90ea VK |
512 | orr r3, r3, r4, lspush #8 |
513 | mov r4, r4, lspull #24 | |
514 | orr r4, r4, r5, lspush #8 | |
515 | mov r5, r5, lspull #24 | |
516 | orr r5, r5, r6, lspush #8 | |
517 | mov r6, r6, lspull #24 | |
518 | orr r6, r6, r7, lspush #8 | |
080fc66f RK |
519 | stmia r0!, {r3 - r6} |
520 | subs ip, ip, #16 | |
521 | bpl .Lcfu_3cpy8lp | |
522 | ||
523 | .Lcfu_3rem8lp: tst ip, #8 | |
d98b90ea | 524 | movne r3, r7, lspull #24 |
080fc66f | 525 | ldmneia r1!, {r4, r7} @ Shouldnt fault |
d98b90ea VK |
526 | orrne r3, r3, r4, lspush #8 |
527 | movne r4, r4, lspull #24 | |
528 | orrne r4, r4, r7, lspush #8 | |
080fc66f RK |
529 | stmneia r0!, {r3 - r4} |
530 | tst ip, #4 | |
d98b90ea | 531 | movne r3, r7, lspull #24 |
080fc66f | 532 | USER( TUSER( ldrne) r7, [r1], #4) @ May fault |
d98b90ea | 533 | orrne r3, r3, r7, lspush #8 |
080fc66f RK |
534 | strne r3, [r0], #4 |
535 | ands ip, ip, #3 | |
536 | beq .Lcfu_3fupi | |
537 | .Lcfu_3nowords: mov r3, r7, get_byte_3 | |
538 | teq ip, #0 | |
539 | beq .Lcfu_finished | |
540 | cmp ip, #2 | |
541 | strb r3, [r0], #1 | |
542 | USER( TUSER( ldrgeb) r3, [r1], #1) @ May fault | |
543 | strgeb r3, [r0], #1 | |
544 | USER( TUSER( ldrgtb) r3, [r1], #1) @ May fault | |
545 | strgtb r3, [r0], #1 | |
546 | b .Lcfu_finished | |
547 | ENDPROC(__copy_from_user) | |
548 | ||
549 | .pushsection .fixup,"ax" | |
550 | .align 0 | |
551 | /* | |
552 | * We took an exception. r0 contains a pointer to | |
553 | * the byte not copied. | |
554 | */ | |
555 | 9001: ldr r2, [sp], #4 @ void *to | |
556 | sub r2, r0, r2 @ bytes copied | |
557 | ldr r1, [sp], #4 @ unsigned long count | |
558 | subs r4, r1, r2 @ bytes left to copy | |
559 | movne r1, r4 | |
560 | blne __memzero | |
561 | mov r0, r4 | |
562 | ldmfd sp!, {r4 - r7, pc} | |
563 | .popsection | |
564 |