Commit | Line | Data |
---|---|---|
14cf11af | 1 | /* |
14cf11af PM |
2 | * Copyright (C) 2002 Paul Mackerras, IBM Corp. |
3 | * | |
4 | * This program is free software; you can redistribute it and/or | |
5 | * modify it under the terms of the GNU General Public License | |
6 | * as published by the Free Software Foundation; either version | |
7 | * 2 of the License, or (at your option) any later version. | |
8 | */ | |
9 | #include <asm/processor.h> | |
10 | #include <asm/ppc_asm.h> | |
11 | ||
12 | .align 7 | |
169c7cee | 13 | _GLOBAL_TOC(memcpy) |
b3f271e8 | 14 | BEGIN_FTR_SECTION |
00f554fa PB |
15 | #ifdef __LITTLE_ENDIAN__ |
16 | cmpdi cr7,r5,0 | |
17 | #else | |
752a6422 | 18 | std r3,-STACKFRAMESIZE+STK_REG(R31)(r1) /* save destination pointer for return value */ |
00f554fa | 19 | #endif |
b3f271e8 | 20 | FTR_SECTION_ELSE |
22d651dc | 21 | #ifndef SELFTEST |
b3f271e8 | 22 | b memcpy_power7 |
22d651dc | 23 | #endif |
b3f271e8 | 24 | ALT_FTR_SECTION_END_IFCLR(CPU_FTR_VMX_COPY) |
00f554fa PB |
25 | #ifdef __LITTLE_ENDIAN__ |
26 | /* dumb little-endian memcpy that will get replaced at runtime */ | |
27 | addi r9,r3,-1 | |
28 | addi r4,r4,-1 | |
29 | beqlr cr7 | |
30 | mtctr r5 | |
31 | 1: lbzu r10,1(r4) | |
32 | stbu r10,1(r9) | |
33 | bdnz 1b | |
34 | blr | |
35 | #else | |
86e32fdc | 36 | PPC_MTOCRF(0x01,r5) |
14cf11af PM |
37 | cmpldi cr1,r5,16 |
38 | neg r6,r3 # LS 3 bits = # bytes to 8-byte dest bdry | |
39 | andi. r6,r6,7 | |
40 | dcbt 0,r4 | |
41 | blt cr1,.Lshort_copy | |
25d6e2d7 MN |
42 | /* Below we want to nop out the bne if we're on a CPU that has the |
43 | CPU_FTR_UNALIGNED_LD_STD bit set and the CPU_FTR_CP_USE_DCBTZ bit | |
44 | cleared. | |
45 | At the time of writing the only CPU that has this combination of bits | |
46 | set is Power6. */ | |
47 | BEGIN_FTR_SECTION | |
48 | nop | |
49 | FTR_SECTION_ELSE | |
14cf11af | 50 | bne .Ldst_unaligned |
25d6e2d7 MN |
51 | ALT_FTR_SECTION_END(CPU_FTR_UNALIGNED_LD_STD | CPU_FTR_CP_USE_DCBTZ, \ |
52 | CPU_FTR_UNALIGNED_LD_STD) | |
14cf11af | 53 | .Ldst_aligned: |
14cf11af | 54 | addi r3,r3,-16 |
25d6e2d7 MN |
55 | BEGIN_FTR_SECTION |
56 | andi. r0,r4,7 | |
14cf11af | 57 | bne .Lsrc_unaligned |
25d6e2d7 | 58 | END_FTR_SECTION_IFCLR(CPU_FTR_UNALIGNED_LD_STD) |
14cf11af PM |
59 | srdi r7,r5,4 |
60 | ld r9,0(r4) | |
61 | addi r4,r4,-8 | |
62 | mtctr r7 | |
63 | andi. r5,r5,7 | |
64 | bf cr7*4+0,2f | |
65 | addi r3,r3,8 | |
66 | addi r4,r4,8 | |
67 | mr r8,r9 | |
68 | blt cr1,3f | |
69 | 1: ld r9,8(r4) | |
70 | std r8,8(r3) | |
71 | 2: ldu r8,16(r4) | |
72 | stdu r9,16(r3) | |
73 | bdnz 1b | |
74 | 3: std r8,8(r3) | |
d0027bf0 | 75 | beq 3f |
14cf11af | 76 | addi r3,r3,16 |
14cf11af PM |
77 | .Ldo_tail: |
78 | bf cr7*4+1,1f | |
e423b9ec MN |
79 | lwz r9,8(r4) |
80 | addi r4,r4,4 | |
14cf11af PM |
81 | stw r9,0(r3) |
82 | addi r3,r3,4 | |
83 | 1: bf cr7*4+2,2f | |
e423b9ec MN |
84 | lhz r9,8(r4) |
85 | addi r4,r4,2 | |
14cf11af PM |
86 | sth r9,0(r3) |
87 | addi r3,r3,2 | |
88 | 2: bf cr7*4+3,3f | |
e423b9ec | 89 | lbz r9,8(r4) |
14cf11af | 90 | stb r9,0(r3) |
752a6422 | 91 | 3: ld r3,-STACKFRAMESIZE+STK_REG(R31)(r1) /* return dest pointer */ |
d0027bf0 | 92 | blr |
14cf11af PM |
93 | |
94 | .Lsrc_unaligned: | |
95 | srdi r6,r5,3 | |
96 | addi r5,r5,-16 | |
97 | subf r4,r0,r4 | |
98 | srdi r7,r5,4 | |
99 | sldi r10,r0,3 | |
100 | cmpdi cr6,r6,3 | |
101 | andi. r5,r5,7 | |
102 | mtctr r7 | |
103 | subfic r11,r10,64 | |
104 | add r5,r5,r0 | |
105 | ||
106 | bt cr7*4+0,0f | |
107 | ||
108 | ld r9,0(r4) # 3+2n loads, 2+2n stores | |
109 | ld r0,8(r4) | |
110 | sld r6,r9,r10 | |
111 | ldu r9,16(r4) | |
112 | srd r7,r0,r11 | |
113 | sld r8,r0,r10 | |
114 | or r7,r7,r6 | |
115 | blt cr6,4f | |
116 | ld r0,8(r4) | |
117 | # s1<< in r8, d0=(s0<<|s1>>) in r7, s3 in r0, s2 in r9, nix in r6 & r12 | |
118 | b 2f | |
119 | ||
120 | 0: ld r0,0(r4) # 4+2n loads, 3+2n stores | |
121 | ldu r9,8(r4) | |
122 | sld r8,r0,r10 | |
123 | addi r3,r3,-8 | |
124 | blt cr6,5f | |
125 | ld r0,8(r4) | |
126 | srd r12,r9,r11 | |
127 | sld r6,r9,r10 | |
128 | ldu r9,16(r4) | |
129 | or r12,r8,r12 | |
130 | srd r7,r0,r11 | |
131 | sld r8,r0,r10 | |
132 | addi r3,r3,16 | |
133 | beq cr6,3f | |
134 | ||
135 | # d0=(s0<<|s1>>) in r12, s1<< in r6, s2>> in r7, s2<< in r8, s3 in r9 | |
136 | 1: or r7,r7,r6 | |
137 | ld r0,8(r4) | |
138 | std r12,8(r3) | |
139 | 2: srd r12,r9,r11 | |
140 | sld r6,r9,r10 | |
141 | ldu r9,16(r4) | |
142 | or r12,r8,r12 | |
143 | stdu r7,16(r3) | |
144 | srd r7,r0,r11 | |
145 | sld r8,r0,r10 | |
146 | bdnz 1b | |
147 | ||
148 | 3: std r12,8(r3) | |
149 | or r7,r7,r6 | |
150 | 4: std r7,16(r3) | |
151 | 5: srd r12,r9,r11 | |
152 | or r12,r8,r12 | |
153 | std r12,24(r3) | |
d0027bf0 | 154 | beq 4f |
14cf11af PM |
155 | cmpwi cr1,r5,8 |
156 | addi r3,r3,32 | |
157 | sld r9,r9,r10 | |
e423b9ec | 158 | ble cr1,6f |
14cf11af PM |
159 | ld r0,8(r4) |
160 | srd r7,r0,r11 | |
161 | or r9,r7,r9 | |
e423b9ec MN |
162 | 6: |
163 | bf cr7*4+1,1f | |
164 | rotldi r9,r9,32 | |
165 | stw r9,0(r3) | |
166 | addi r3,r3,4 | |
167 | 1: bf cr7*4+2,2f | |
168 | rotldi r9,r9,16 | |
169 | sth r9,0(r3) | |
170 | addi r3,r3,2 | |
171 | 2: bf cr7*4+3,3f | |
172 | rotldi r9,r9,8 | |
173 | stb r9,0(r3) | |
752a6422 | 174 | 3: ld r3,-STACKFRAMESIZE+STK_REG(R31)(r1) /* return dest pointer */ |
e423b9ec | 175 | blr |
14cf11af PM |
176 | |
177 | .Ldst_unaligned: | |
86e32fdc | 178 | PPC_MTOCRF(0x01,r6) # put #bytes to 8B bdry into cr7 |
14cf11af PM |
179 | subf r5,r6,r5 |
180 | li r7,0 | |
25d6e2d7 | 181 | cmpldi cr1,r5,16 |
14cf11af PM |
182 | bf cr7*4+3,1f |
183 | lbz r0,0(r4) | |
184 | stb r0,0(r3) | |
185 | addi r7,r7,1 | |
186 | 1: bf cr7*4+2,2f | |
187 | lhzx r0,r7,r4 | |
188 | sthx r0,r7,r3 | |
189 | addi r7,r7,2 | |
190 | 2: bf cr7*4+1,3f | |
191 | lwzx r0,r7,r4 | |
192 | stwx r0,r7,r3 | |
86e32fdc | 193 | 3: PPC_MTOCRF(0x01,r5) |
14cf11af PM |
194 | add r4,r6,r4 |
195 | add r3,r6,r3 | |
196 | b .Ldst_aligned | |
197 | ||
198 | .Lshort_copy: | |
199 | bf cr7*4+0,1f | |
200 | lwz r0,0(r4) | |
201 | lwz r9,4(r4) | |
202 | addi r4,r4,8 | |
203 | stw r0,0(r3) | |
204 | stw r9,4(r3) | |
205 | addi r3,r3,8 | |
206 | 1: bf cr7*4+1,2f | |
207 | lwz r0,0(r4) | |
208 | addi r4,r4,4 | |
209 | stw r0,0(r3) | |
210 | addi r3,r3,4 | |
211 | 2: bf cr7*4+2,3f | |
212 | lhz r0,0(r4) | |
213 | addi r4,r4,2 | |
214 | sth r0,0(r3) | |
215 | addi r3,r3,2 | |
216 | 3: bf cr7*4+3,4f | |
217 | lbz r0,0(r4) | |
218 | stb r0,0(r3) | |
752a6422 | 219 | 4: ld r3,-STACKFRAMESIZE+STK_REG(R31)(r1) /* return dest pointer */ |
d0027bf0 | 220 | blr |
00f554fa | 221 | #endif |