Commit | Line | Data |
---|---|---|
14cf11af | 1 | /* |
14cf11af PM |
2 | * Copyright (C) 2002 Paul Mackerras, IBM Corp. |
3 | * | |
4 | * This program is free software; you can redistribute it and/or | |
5 | * modify it under the terms of the GNU General Public License | |
6 | * as published by the Free Software Foundation; either version | |
7 | * 2 of the License, or (at your option) any later version. | |
8 | */ | |
9 | #include <asm/processor.h> | |
10 | #include <asm/ppc_asm.h> | |
11 | ||
12 | .align 7 | |
13 | _GLOBAL(__copy_tofrom_user) | |
14 | /* first check for a whole page copy on a page boundary */ | |
15 | cmpldi cr1,r5,16 | |
16 | cmpdi cr6,r5,4096 | |
17 | or r0,r3,r4 | |
18 | neg r6,r3 /* LS 3 bits = # bytes to 8-byte dest bdry */ | |
19 | andi. r0,r0,4095 | |
20 | std r3,-24(r1) | |
21 | crand cr0*4+2,cr0*4+2,cr6*4+2 | |
22 | std r4,-16(r1) | |
23 | std r5,-8(r1) | |
24 | dcbt 0,r4 | |
3c726f8d | 25 | beq .Lcopy_page_4K |
14cf11af PM |
26 | andi. r6,r6,7 |
27 | mtcrf 0x01,r5 | |
28 | blt cr1,.Lshort_copy | |
29 | bne .Ldst_unaligned | |
30 | .Ldst_aligned: | |
31 | andi. r0,r4,7 | |
32 | addi r3,r3,-16 | |
33 | bne .Lsrc_unaligned | |
34 | srdi r7,r5,4 | |
35 | 20: ld r9,0(r4) | |
36 | addi r4,r4,-8 | |
37 | mtctr r7 | |
38 | andi. r5,r5,7 | |
39 | bf cr7*4+0,22f | |
40 | addi r3,r3,8 | |
41 | addi r4,r4,8 | |
42 | mr r8,r9 | |
43 | blt cr1,72f | |
44 | 21: ld r9,8(r4) | |
45 | 70: std r8,8(r3) | |
46 | 22: ldu r8,16(r4) | |
47 | 71: stdu r9,16(r3) | |
48 | bdnz 21b | |
49 | 72: std r8,8(r3) | |
50 | beq+ 3f | |
51 | addi r3,r3,16 | |
52 | 23: ld r9,8(r4) | |
53 | .Ldo_tail: | |
54 | bf cr7*4+1,1f | |
55 | rotldi r9,r9,32 | |
56 | 73: stw r9,0(r3) | |
57 | addi r3,r3,4 | |
58 | 1: bf cr7*4+2,2f | |
59 | rotldi r9,r9,16 | |
60 | 74: sth r9,0(r3) | |
61 | addi r3,r3,2 | |
62 | 2: bf cr7*4+3,3f | |
63 | rotldi r9,r9,8 | |
64 | 75: stb r9,0(r3) | |
65 | 3: li r3,0 | |
66 | blr | |
67 | ||
68 | .Lsrc_unaligned: | |
69 | srdi r6,r5,3 | |
70 | addi r5,r5,-16 | |
71 | subf r4,r0,r4 | |
72 | srdi r7,r5,4 | |
73 | sldi r10,r0,3 | |
74 | cmpldi cr6,r6,3 | |
75 | andi. r5,r5,7 | |
76 | mtctr r7 | |
77 | subfic r11,r10,64 | |
78 | add r5,r5,r0 | |
79 | bt cr7*4+0,28f | |
80 | ||
81 | 24: ld r9,0(r4) /* 3+2n loads, 2+2n stores */ | |
82 | 25: ld r0,8(r4) | |
83 | sld r6,r9,r10 | |
84 | 26: ldu r9,16(r4) | |
85 | srd r7,r0,r11 | |
86 | sld r8,r0,r10 | |
87 | or r7,r7,r6 | |
88 | blt cr6,79f | |
89 | 27: ld r0,8(r4) | |
90 | b 2f | |
91 | ||
92 | 28: ld r0,0(r4) /* 4+2n loads, 3+2n stores */ | |
93 | 29: ldu r9,8(r4) | |
94 | sld r8,r0,r10 | |
95 | addi r3,r3,-8 | |
96 | blt cr6,5f | |
97 | 30: ld r0,8(r4) | |
98 | srd r12,r9,r11 | |
99 | sld r6,r9,r10 | |
100 | 31: ldu r9,16(r4) | |
101 | or r12,r8,r12 | |
102 | srd r7,r0,r11 | |
103 | sld r8,r0,r10 | |
104 | addi r3,r3,16 | |
105 | beq cr6,78f | |
106 | ||
107 | 1: or r7,r7,r6 | |
108 | 32: ld r0,8(r4) | |
109 | 76: std r12,8(r3) | |
110 | 2: srd r12,r9,r11 | |
111 | sld r6,r9,r10 | |
112 | 33: ldu r9,16(r4) | |
113 | or r12,r8,r12 | |
114 | 77: stdu r7,16(r3) | |
115 | srd r7,r0,r11 | |
116 | sld r8,r0,r10 | |
117 | bdnz 1b | |
118 | ||
119 | 78: std r12,8(r3) | |
120 | or r7,r7,r6 | |
121 | 79: std r7,16(r3) | |
122 | 5: srd r12,r9,r11 | |
123 | or r12,r8,r12 | |
124 | 80: std r12,24(r3) | |
125 | bne 6f | |
126 | li r3,0 | |
127 | blr | |
128 | 6: cmpwi cr1,r5,8 | |
129 | addi r3,r3,32 | |
130 | sld r9,r9,r10 | |
131 | ble cr1,.Ldo_tail | |
132 | 34: ld r0,8(r4) | |
133 | srd r7,r0,r11 | |
134 | or r9,r7,r9 | |
135 | b .Ldo_tail | |
136 | ||
137 | .Ldst_unaligned: | |
138 | mtcrf 0x01,r6 /* put #bytes to 8B bdry into cr7 */ | |
139 | subf r5,r6,r5 | |
140 | li r7,0 | |
141 | cmpldi r1,r5,16 | |
142 | bf cr7*4+3,1f | |
143 | 35: lbz r0,0(r4) | |
144 | 81: stb r0,0(r3) | |
145 | addi r7,r7,1 | |
146 | 1: bf cr7*4+2,2f | |
147 | 36: lhzx r0,r7,r4 | |
148 | 82: sthx r0,r7,r3 | |
149 | addi r7,r7,2 | |
150 | 2: bf cr7*4+1,3f | |
151 | 37: lwzx r0,r7,r4 | |
152 | 83: stwx r0,r7,r3 | |
153 | 3: mtcrf 0x01,r5 | |
154 | add r4,r6,r4 | |
155 | add r3,r6,r3 | |
156 | b .Ldst_aligned | |
157 | ||
158 | .Lshort_copy: | |
159 | bf cr7*4+0,1f | |
160 | 38: lwz r0,0(r4) | |
161 | 39: lwz r9,4(r4) | |
162 | addi r4,r4,8 | |
163 | 84: stw r0,0(r3) | |
164 | 85: stw r9,4(r3) | |
165 | addi r3,r3,8 | |
166 | 1: bf cr7*4+1,2f | |
167 | 40: lwz r0,0(r4) | |
168 | addi r4,r4,4 | |
169 | 86: stw r0,0(r3) | |
170 | addi r3,r3,4 | |
171 | 2: bf cr7*4+2,3f | |
172 | 41: lhz r0,0(r4) | |
173 | addi r4,r4,2 | |
174 | 87: sth r0,0(r3) | |
175 | addi r3,r3,2 | |
176 | 3: bf cr7*4+3,4f | |
177 | 42: lbz r0,0(r4) | |
178 | 88: stb r0,0(r3) | |
179 | 4: li r3,0 | |
180 | blr | |
181 | ||
182 | /* | |
183 | * exception handlers follow | |
184 | * we have to return the number of bytes not copied | |
185 | * for an exception on a load, we set the rest of the destination to 0 | |
186 | */ | |
187 | ||
188 | 136: | |
189 | 137: | |
190 | add r3,r3,r7 | |
191 | b 1f | |
192 | 130: | |
193 | 131: | |
194 | addi r3,r3,8 | |
195 | 120: | |
196 | 122: | |
197 | 124: | |
198 | 125: | |
199 | 126: | |
200 | 127: | |
201 | 128: | |
202 | 129: | |
203 | 133: | |
204 | addi r3,r3,8 | |
205 | 121: | |
206 | 132: | |
207 | addi r3,r3,8 | |
208 | 123: | |
209 | 134: | |
210 | 135: | |
211 | 138: | |
212 | 139: | |
213 | 140: | |
214 | 141: | |
215 | 142: | |
216 | ||
217 | /* | |
218 | * here we have had a fault on a load and r3 points to the first | |
219 | * unmodified byte of the destination | |
220 | */ | |
221 | 1: ld r6,-24(r1) | |
222 | ld r4,-16(r1) | |
223 | ld r5,-8(r1) | |
224 | subf r6,r6,r3 | |
225 | add r4,r4,r6 | |
226 | subf r5,r6,r5 /* #bytes left to go */ | |
227 | ||
228 | /* | |
229 | * first see if we can copy any more bytes before hitting another exception | |
230 | */ | |
231 | mtctr r5 | |
232 | 43: lbz r0,0(r4) | |
233 | addi r4,r4,1 | |
234 | 89: stb r0,0(r3) | |
235 | addi r3,r3,1 | |
236 | bdnz 43b | |
237 | li r3,0 /* huh? all copied successfully this time? */ | |
238 | blr | |
239 | ||
240 | /* | |
241 | * here we have trapped again, need to clear ctr bytes starting at r3 | |
242 | */ | |
243 | 143: mfctr r5 | |
244 | li r0,0 | |
245 | mr r4,r3 | |
246 | mr r3,r5 /* return the number of bytes not copied */ | |
247 | 1: andi. r9,r4,7 | |
248 | beq 3f | |
249 | 90: stb r0,0(r4) | |
250 | addic. r5,r5,-1 | |
251 | addi r4,r4,1 | |
252 | bne 1b | |
253 | blr | |
254 | 3: cmpldi cr1,r5,8 | |
255 | srdi r9,r5,3 | |
256 | andi. r5,r5,7 | |
257 | blt cr1,93f | |
258 | mtctr r9 | |
259 | 91: std r0,0(r4) | |
260 | addi r4,r4,8 | |
261 | bdnz 91b | |
262 | 93: beqlr | |
263 | mtctr r5 | |
264 | 92: stb r0,0(r4) | |
265 | addi r4,r4,1 | |
266 | bdnz 92b | |
267 | blr | |
268 | ||
269 | /* | |
270 | * exception handlers for stores: we just need to work | |
271 | * out how many bytes weren't copied | |
272 | */ | |
273 | 182: | |
274 | 183: | |
275 | add r3,r3,r7 | |
276 | b 1f | |
277 | 180: | |
278 | addi r3,r3,8 | |
279 | 171: | |
280 | 177: | |
281 | addi r3,r3,8 | |
282 | 170: | |
283 | 172: | |
284 | 176: | |
285 | 178: | |
286 | addi r3,r3,4 | |
287 | 185: | |
288 | addi r3,r3,4 | |
289 | 173: | |
290 | 174: | |
291 | 175: | |
292 | 179: | |
293 | 181: | |
294 | 184: | |
295 | 186: | |
296 | 187: | |
297 | 188: | |
298 | 189: | |
299 | 1: | |
300 | ld r6,-24(r1) | |
301 | ld r5,-8(r1) | |
302 | add r6,r6,r5 | |
303 | subf r3,r3,r6 /* #bytes not copied */ | |
304 | 190: | |
305 | 191: | |
306 | 192: | |
307 | blr /* #bytes not copied in r3 */ | |
308 | ||
309 | .section __ex_table,"a" | |
310 | .align 3 | |
311 | .llong 20b,120b | |
312 | .llong 21b,121b | |
313 | .llong 70b,170b | |
314 | .llong 22b,122b | |
315 | .llong 71b,171b | |
316 | .llong 72b,172b | |
317 | .llong 23b,123b | |
318 | .llong 73b,173b | |
319 | .llong 74b,174b | |
320 | .llong 75b,175b | |
321 | .llong 24b,124b | |
322 | .llong 25b,125b | |
323 | .llong 26b,126b | |
324 | .llong 27b,127b | |
325 | .llong 28b,128b | |
326 | .llong 29b,129b | |
327 | .llong 30b,130b | |
328 | .llong 31b,131b | |
329 | .llong 32b,132b | |
330 | .llong 76b,176b | |
331 | .llong 33b,133b | |
332 | .llong 77b,177b | |
333 | .llong 78b,178b | |
334 | .llong 79b,179b | |
335 | .llong 80b,180b | |
336 | .llong 34b,134b | |
337 | .llong 35b,135b | |
338 | .llong 81b,181b | |
339 | .llong 36b,136b | |
340 | .llong 82b,182b | |
341 | .llong 37b,137b | |
342 | .llong 83b,183b | |
343 | .llong 38b,138b | |
344 | .llong 39b,139b | |
345 | .llong 84b,184b | |
346 | .llong 85b,185b | |
347 | .llong 40b,140b | |
348 | .llong 86b,186b | |
349 | .llong 41b,141b | |
350 | .llong 87b,187b | |
351 | .llong 42b,142b | |
352 | .llong 88b,188b | |
353 | .llong 43b,143b | |
354 | .llong 89b,189b | |
355 | .llong 90b,190b | |
356 | .llong 91b,191b | |
357 | .llong 92b,192b | |
358 | ||
359 | .text | |
360 | ||
361 | /* | |
362 | * Routine to copy a whole page of data, optimized for POWER4. | |
363 | * On POWER4 it is more than 50% faster than the simple loop | |
364 | * above (following the .Ldst_aligned label) but it runs slightly | |
365 | * slower on POWER3. | |
366 | */ | |
3c726f8d | 367 | .Lcopy_page_4K: |
14cf11af PM |
368 | std r31,-32(1) |
369 | std r30,-40(1) | |
370 | std r29,-48(1) | |
371 | std r28,-56(1) | |
372 | std r27,-64(1) | |
373 | std r26,-72(1) | |
374 | std r25,-80(1) | |
375 | std r24,-88(1) | |
376 | std r23,-96(1) | |
377 | std r22,-104(1) | |
378 | std r21,-112(1) | |
379 | std r20,-120(1) | |
380 | li r5,4096/32 - 1 | |
381 | addi r3,r3,-8 | |
382 | li r0,5 | |
383 | 0: addi r5,r5,-24 | |
384 | mtctr r0 | |
385 | 20: ld r22,640(4) | |
386 | 21: ld r21,512(4) | |
387 | 22: ld r20,384(4) | |
388 | 23: ld r11,256(4) | |
389 | 24: ld r9,128(4) | |
390 | 25: ld r7,0(4) | |
391 | 26: ld r25,648(4) | |
392 | 27: ld r24,520(4) | |
393 | 28: ld r23,392(4) | |
394 | 29: ld r10,264(4) | |
395 | 30: ld r8,136(4) | |
396 | 31: ldu r6,8(4) | |
397 | cmpwi r5,24 | |
398 | 1: | |
399 | 32: std r22,648(3) | |
400 | 33: std r21,520(3) | |
401 | 34: std r20,392(3) | |
402 | 35: std r11,264(3) | |
403 | 36: std r9,136(3) | |
404 | 37: std r7,8(3) | |
405 | 38: ld r28,648(4) | |
406 | 39: ld r27,520(4) | |
407 | 40: ld r26,392(4) | |
408 | 41: ld r31,264(4) | |
409 | 42: ld r30,136(4) | |
410 | 43: ld r29,8(4) | |
411 | 44: std r25,656(3) | |
412 | 45: std r24,528(3) | |
413 | 46: std r23,400(3) | |
414 | 47: std r10,272(3) | |
415 | 48: std r8,144(3) | |
416 | 49: std r6,16(3) | |
417 | 50: ld r22,656(4) | |
418 | 51: ld r21,528(4) | |
419 | 52: ld r20,400(4) | |
420 | 53: ld r11,272(4) | |
421 | 54: ld r9,144(4) | |
422 | 55: ld r7,16(4) | |
423 | 56: std r28,664(3) | |
424 | 57: std r27,536(3) | |
425 | 58: std r26,408(3) | |
426 | 59: std r31,280(3) | |
427 | 60: std r30,152(3) | |
428 | 61: stdu r29,24(3) | |
429 | 62: ld r25,664(4) | |
430 | 63: ld r24,536(4) | |
431 | 64: ld r23,408(4) | |
432 | 65: ld r10,280(4) | |
433 | 66: ld r8,152(4) | |
434 | 67: ldu r6,24(4) | |
435 | bdnz 1b | |
436 | 68: std r22,648(3) | |
437 | 69: std r21,520(3) | |
438 | 70: std r20,392(3) | |
439 | 71: std r11,264(3) | |
440 | 72: std r9,136(3) | |
441 | 73: std r7,8(3) | |
442 | 74: addi r4,r4,640 | |
443 | 75: addi r3,r3,648 | |
444 | bge 0b | |
445 | mtctr r5 | |
446 | 76: ld r7,0(4) | |
447 | 77: ld r8,8(4) | |
448 | 78: ldu r9,16(4) | |
449 | 3: | |
450 | 79: ld r10,8(4) | |
451 | 80: std r7,8(3) | |
452 | 81: ld r7,16(4) | |
453 | 82: std r8,16(3) | |
454 | 83: ld r8,24(4) | |
455 | 84: std r9,24(3) | |
456 | 85: ldu r9,32(4) | |
457 | 86: stdu r10,32(3) | |
458 | bdnz 3b | |
459 | 4: | |
460 | 87: ld r10,8(4) | |
461 | 88: std r7,8(3) | |
462 | 89: std r8,16(3) | |
463 | 90: std r9,24(3) | |
464 | 91: std r10,32(3) | |
465 | 9: ld r20,-120(1) | |
466 | ld r21,-112(1) | |
467 | ld r22,-104(1) | |
468 | ld r23,-96(1) | |
469 | ld r24,-88(1) | |
470 | ld r25,-80(1) | |
471 | ld r26,-72(1) | |
472 | ld r27,-64(1) | |
473 | ld r28,-56(1) | |
474 | ld r29,-48(1) | |
475 | ld r30,-40(1) | |
476 | ld r31,-32(1) | |
477 | li r3,0 | |
478 | blr | |
479 | ||
480 | /* | |
481 | * on an exception, reset to the beginning and jump back into the | |
482 | * standard __copy_tofrom_user | |
483 | */ | |
484 | 100: ld r20,-120(1) | |
485 | ld r21,-112(1) | |
486 | ld r22,-104(1) | |
487 | ld r23,-96(1) | |
488 | ld r24,-88(1) | |
489 | ld r25,-80(1) | |
490 | ld r26,-72(1) | |
491 | ld r27,-64(1) | |
492 | ld r28,-56(1) | |
493 | ld r29,-48(1) | |
494 | ld r30,-40(1) | |
495 | ld r31,-32(1) | |
496 | ld r3,-24(r1) | |
497 | ld r4,-16(r1) | |
498 | li r5,4096 | |
499 | b .Ldst_aligned | |
500 | ||
501 | .section __ex_table,"a" | |
502 | .align 3 | |
503 | .llong 20b,100b | |
504 | .llong 21b,100b | |
505 | .llong 22b,100b | |
506 | .llong 23b,100b | |
507 | .llong 24b,100b | |
508 | .llong 25b,100b | |
509 | .llong 26b,100b | |
510 | .llong 27b,100b | |
511 | .llong 28b,100b | |
512 | .llong 29b,100b | |
513 | .llong 30b,100b | |
514 | .llong 31b,100b | |
515 | .llong 32b,100b | |
516 | .llong 33b,100b | |
517 | .llong 34b,100b | |
518 | .llong 35b,100b | |
519 | .llong 36b,100b | |
520 | .llong 37b,100b | |
521 | .llong 38b,100b | |
522 | .llong 39b,100b | |
523 | .llong 40b,100b | |
524 | .llong 41b,100b | |
525 | .llong 42b,100b | |
526 | .llong 43b,100b | |
527 | .llong 44b,100b | |
528 | .llong 45b,100b | |
529 | .llong 46b,100b | |
530 | .llong 47b,100b | |
531 | .llong 48b,100b | |
532 | .llong 49b,100b | |
533 | .llong 50b,100b | |
534 | .llong 51b,100b | |
535 | .llong 52b,100b | |
536 | .llong 53b,100b | |
537 | .llong 54b,100b | |
538 | .llong 55b,100b | |
539 | .llong 56b,100b | |
540 | .llong 57b,100b | |
541 | .llong 58b,100b | |
542 | .llong 59b,100b | |
543 | .llong 60b,100b | |
544 | .llong 61b,100b | |
545 | .llong 62b,100b | |
546 | .llong 63b,100b | |
547 | .llong 64b,100b | |
548 | .llong 65b,100b | |
549 | .llong 66b,100b | |
550 | .llong 67b,100b | |
551 | .llong 68b,100b | |
552 | .llong 69b,100b | |
553 | .llong 70b,100b | |
554 | .llong 71b,100b | |
555 | .llong 72b,100b | |
556 | .llong 73b,100b | |
557 | .llong 74b,100b | |
558 | .llong 75b,100b | |
559 | .llong 76b,100b | |
560 | .llong 77b,100b | |
561 | .llong 78b,100b | |
562 | .llong 79b,100b | |
563 | .llong 80b,100b | |
564 | .llong 81b,100b | |
565 | .llong 82b,100b | |
566 | .llong 83b,100b | |
567 | .llong 84b,100b | |
568 | .llong 85b,100b | |
569 | .llong 86b,100b | |
570 | .llong 87b,100b | |
571 | .llong 88b,100b | |
572 | .llong 89b,100b | |
573 | .llong 90b,100b | |
574 | .llong 91b,100b |