Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | /* clear_page.S: UltraSparc optimized copy page. |
2 | * | |
3 | * Copyright (C) 1996, 1998, 1999, 2000, 2004 David S. Miller (davem@redhat.com) | |
4 | * Copyright (C) 1997 Jakub Jelinek (jakub@redhat.com) | |
5 | */ | |
6 | ||
7 | #include <asm/visasm.h> | |
8 | #include <asm/thread_info.h> | |
9 | #include <asm/page.h> | |
10 | #include <asm/pgtable.h> | |
11 | #include <asm/spitfire.h> | |
12 | #include <asm/head.h> | |
d3867f04 | 13 | #include <asm/export.h> |
1da177e4 LT |
14 | |
15 | /* What we used to do was lock a TLB entry into a specific | |
16 | * TLB slot, clear the page with interrupts disabled, then | |
17 | * restore the original TLB entry. This was great for | |
18 | * disturbing the TLB as little as possible, but it meant | |
19 | * we had to keep interrupts disabled for a long time. | |
20 | * | |
21 | * Now, we simply use the normal TLB loading mechanism, | |
22 | * and this makes the cpu choose a slot all by itself. | |
23 | * Then we do a normal TLB flush on exit. We need only | |
24 | * disable preemption during the clear. | |
25 | */ | |
26 | ||
1da177e4 LT |
27 | #define DCACHE_SIZE (PAGE_SIZE * 2) |
28 | ||
f7fe9334 | 29 | #if (PAGE_SHIFT == 13) |
1da177e4 | 30 | #define PAGE_SIZE_REM 0x80 |
f7fe9334 | 31 | #elif (PAGE_SHIFT == 16) |
1da177e4 LT |
32 | #define PAGE_SIZE_REM 0x100 |
33 | #else | |
34 | #error Wrong PAGE_SHIFT specified | |
35 | #endif | |
36 | ||
37 | #define TOUCH(reg0, reg1, reg2, reg3, reg4, reg5, reg6, reg7) \ | |
6f1d827f DM |
38 | fsrc2 %reg0, %f48; fsrc2 %reg1, %f50; \ |
39 | fsrc2 %reg2, %f52; fsrc2 %reg3, %f54; \ | |
40 | fsrc2 %reg4, %f56; fsrc2 %reg5, %f58; \ | |
41 | fsrc2 %reg6, %f60; fsrc2 %reg7, %f62; | |
1da177e4 LT |
42 | |
43 | .text | |
44 | ||
45 | .align 32 | |
46 | .globl copy_user_page | |
47 | .type copy_user_page,#function | |
d3867f04 | 48 | EXPORT_SYMBOL(copy_user_page) |
1da177e4 LT |
49 | copy_user_page: /* %o0=dest, %o1=src, %o2=vaddr */ |
50 | lduw [%g6 + TI_PRE_COUNT], %o4 | |
b2d43834 | 51 | sethi %hi(PAGE_OFFSET), %g2 |
1da177e4 LT |
52 | sethi %hi(PAGE_SIZE), %o3 |
53 | ||
b2d43834 | 54 | ldx [%g2 + %lo(PAGE_OFFSET)], %g2 |
c4bce90e | 55 | sethi %hi(PAGE_KERNEL_LOCKED), %g3 |
1da177e4 | 56 | |
c4bce90e | 57 | ldx [%g3 + %lo(PAGE_KERNEL_LOCKED)], %g3 |
1da177e4 LT |
58 | sub %o0, %g2, %g1 ! dest paddr |
59 | ||
60 | sub %o1, %g2, %g2 ! src paddr | |
1da177e4 LT |
61 | |
62 | and %o2, %o3, %o0 ! vaddr D-cache alias bit | |
63 | or %g1, %g3, %g1 ! dest TTE data | |
64 | ||
65 | or %g2, %g3, %g2 ! src TTE data | |
66 | sethi %hi(TLBTEMP_BASE), %o3 | |
67 | ||
68 | sethi %hi(DCACHE_SIZE), %o1 | |
69 | add %o0, %o3, %o0 ! dest TTE vaddr | |
70 | ||
71 | add %o4, 1, %o2 | |
72 | add %o0, %o1, %o1 ! src TTE vaddr | |
73 | ||
74 | /* Disable preemption. */ | |
75 | mov TLB_TAG_ACCESS, %g3 | |
76 | stw %o2, [%g6 + TI_PRE_COUNT] | |
77 | ||
78 | /* Load TLB entries. */ | |
79 | rdpr %pstate, %o2 | |
80 | wrpr %o2, PSTATE_IE, %pstate | |
81 | stxa %o0, [%g3] ASI_DMMU | |
82 | stxa %g1, [%g0] ASI_DTLB_DATA_IN | |
83 | membar #Sync | |
84 | stxa %o1, [%g3] ASI_DMMU | |
85 | stxa %g2, [%g0] ASI_DTLB_DATA_IN | |
86 | membar #Sync | |
87 | wrpr %o2, 0x0, %pstate | |
88 | ||
dbd2fdf5 | 89 | cheetah_copy_page_insn: |
1da177e4 LT |
90 | ba,pt %xcc, 9f |
91 | nop | |
92 | ||
93 | 1: | |
94 | VISEntryHalf | |
95 | membar #StoreLoad | #StoreStore | #LoadStore | |
96 | sethi %hi((PAGE_SIZE/64)-2), %o2 | |
97 | mov %o0, %g1 | |
98 | prefetch [%o1 + 0x000], #one_read | |
99 | or %o2, %lo((PAGE_SIZE/64)-2), %o2 | |
100 | prefetch [%o1 + 0x040], #one_read | |
101 | prefetch [%o1 + 0x080], #one_read | |
102 | prefetch [%o1 + 0x0c0], #one_read | |
103 | ldd [%o1 + 0x000], %f0 | |
104 | prefetch [%o1 + 0x100], #one_read | |
105 | ldd [%o1 + 0x008], %f2 | |
106 | prefetch [%o1 + 0x140], #one_read | |
107 | ldd [%o1 + 0x010], %f4 | |
108 | prefetch [%o1 + 0x180], #one_read | |
6f1d827f | 109 | fsrc2 %f0, %f16 |
1da177e4 | 110 | ldd [%o1 + 0x018], %f6 |
6f1d827f | 111 | fsrc2 %f2, %f18 |
1da177e4 | 112 | ldd [%o1 + 0x020], %f8 |
6f1d827f | 113 | fsrc2 %f4, %f20 |
1da177e4 | 114 | ldd [%o1 + 0x028], %f10 |
6f1d827f | 115 | fsrc2 %f6, %f22 |
1da177e4 | 116 | ldd [%o1 + 0x030], %f12 |
6f1d827f | 117 | fsrc2 %f8, %f24 |
1da177e4 | 118 | ldd [%o1 + 0x038], %f14 |
6f1d827f | 119 | fsrc2 %f10, %f26 |
1da177e4 LT |
120 | ldd [%o1 + 0x040], %f0 |
121 | 1: ldd [%o1 + 0x048], %f2 | |
6f1d827f | 122 | fsrc2 %f12, %f28 |
1da177e4 | 123 | ldd [%o1 + 0x050], %f4 |
6f1d827f | 124 | fsrc2 %f14, %f30 |
1da177e4 LT |
125 | stda %f16, [%o0] ASI_BLK_P |
126 | ldd [%o1 + 0x058], %f6 | |
6f1d827f | 127 | fsrc2 %f0, %f16 |
1da177e4 | 128 | ldd [%o1 + 0x060], %f8 |
6f1d827f | 129 | fsrc2 %f2, %f18 |
1da177e4 | 130 | ldd [%o1 + 0x068], %f10 |
6f1d827f | 131 | fsrc2 %f4, %f20 |
1da177e4 | 132 | ldd [%o1 + 0x070], %f12 |
6f1d827f | 133 | fsrc2 %f6, %f22 |
1da177e4 | 134 | ldd [%o1 + 0x078], %f14 |
6f1d827f | 135 | fsrc2 %f8, %f24 |
1da177e4 LT |
136 | ldd [%o1 + 0x080], %f0 |
137 | prefetch [%o1 + 0x180], #one_read | |
6f1d827f | 138 | fsrc2 %f10, %f26 |
1da177e4 LT |
139 | subcc %o2, 1, %o2 |
140 | add %o0, 0x40, %o0 | |
141 | bne,pt %xcc, 1b | |
142 | add %o1, 0x40, %o1 | |
143 | ||
144 | ldd [%o1 + 0x048], %f2 | |
6f1d827f | 145 | fsrc2 %f12, %f28 |
1da177e4 | 146 | ldd [%o1 + 0x050], %f4 |
6f1d827f | 147 | fsrc2 %f14, %f30 |
1da177e4 LT |
148 | stda %f16, [%o0] ASI_BLK_P |
149 | ldd [%o1 + 0x058], %f6 | |
6f1d827f | 150 | fsrc2 %f0, %f16 |
1da177e4 | 151 | ldd [%o1 + 0x060], %f8 |
6f1d827f | 152 | fsrc2 %f2, %f18 |
1da177e4 | 153 | ldd [%o1 + 0x068], %f10 |
6f1d827f | 154 | fsrc2 %f4, %f20 |
1da177e4 | 155 | ldd [%o1 + 0x070], %f12 |
6f1d827f | 156 | fsrc2 %f6, %f22 |
1da177e4 LT |
157 | add %o0, 0x40, %o0 |
158 | ldd [%o1 + 0x078], %f14 | |
6f1d827f DM |
159 | fsrc2 %f8, %f24 |
160 | fsrc2 %f10, %f26 | |
161 | fsrc2 %f12, %f28 | |
162 | fsrc2 %f14, %f30 | |
1da177e4 LT |
163 | stda %f16, [%o0] ASI_BLK_P |
164 | membar #Sync | |
165 | VISExitHalf | |
166 | ba,pt %xcc, 5f | |
167 | nop | |
168 | ||
169 | 9: | |
170 | VISEntry | |
171 | ldub [%g6 + TI_FAULT_CODE], %g3 | |
172 | mov %o0, %g1 | |
173 | cmp %g3, 0 | |
174 | rd %asi, %g3 | |
175 | be,a,pt %icc, 1f | |
176 | wr %g0, ASI_BLK_P, %asi | |
177 | wr %g0, ASI_BLK_COMMIT_P, %asi | |
178 | 1: ldda [%o1] ASI_BLK_P, %f0 | |
179 | add %o1, 0x40, %o1 | |
180 | ldda [%o1] ASI_BLK_P, %f16 | |
181 | add %o1, 0x40, %o1 | |
182 | sethi %hi(PAGE_SIZE), %o2 | |
183 | 1: TOUCH(f0, f2, f4, f6, f8, f10, f12, f14) | |
184 | ldda [%o1] ASI_BLK_P, %f32 | |
185 | stda %f48, [%o0] %asi | |
186 | add %o1, 0x40, %o1 | |
187 | sub %o2, 0x40, %o2 | |
188 | add %o0, 0x40, %o0 | |
189 | TOUCH(f16, f18, f20, f22, f24, f26, f28, f30) | |
190 | ldda [%o1] ASI_BLK_P, %f0 | |
191 | stda %f48, [%o0] %asi | |
192 | add %o1, 0x40, %o1 | |
193 | sub %o2, 0x40, %o2 | |
194 | add %o0, 0x40, %o0 | |
195 | TOUCH(f32, f34, f36, f38, f40, f42, f44, f46) | |
196 | ldda [%o1] ASI_BLK_P, %f16 | |
197 | stda %f48, [%o0] %asi | |
198 | sub %o2, 0x40, %o2 | |
199 | add %o1, 0x40, %o1 | |
200 | cmp %o2, PAGE_SIZE_REM | |
201 | bne,pt %xcc, 1b | |
202 | add %o0, 0x40, %o0 | |
f7fe9334 | 203 | #if (PAGE_SHIFT == 16) |
1da177e4 LT |
204 | TOUCH(f0, f2, f4, f6, f8, f10, f12, f14) |
205 | ldda [%o1] ASI_BLK_P, %f32 | |
206 | stda %f48, [%o0] %asi | |
207 | add %o1, 0x40, %o1 | |
208 | sub %o2, 0x40, %o2 | |
209 | add %o0, 0x40, %o0 | |
210 | TOUCH(f16, f18, f20, f22, f24, f26, f28, f30) | |
211 | ldda [%o1] ASI_BLK_P, %f0 | |
212 | stda %f48, [%o0] %asi | |
213 | add %o1, 0x40, %o1 | |
214 | sub %o2, 0x40, %o2 | |
215 | add %o0, 0x40, %o0 | |
216 | membar #Sync | |
217 | stda %f32, [%o0] %asi | |
218 | add %o0, 0x40, %o0 | |
219 | stda %f0, [%o0] %asi | |
220 | #else | |
221 | membar #Sync | |
222 | stda %f0, [%o0] %asi | |
223 | add %o0, 0x40, %o0 | |
224 | stda %f16, [%o0] %asi | |
225 | #endif | |
226 | membar #Sync | |
227 | wr %g3, 0x0, %asi | |
228 | VISExit | |
229 | ||
230 | 5: | |
231 | stxa %g0, [%g1] ASI_DMMU_DEMAP | |
232 | membar #Sync | |
233 | ||
234 | sethi %hi(DCACHE_SIZE), %g2 | |
235 | stxa %g0, [%g1 + %g2] ASI_DMMU_DEMAP | |
236 | membar #Sync | |
237 | ||
238 | retl | |
239 | stw %o4, [%g6 + TI_PRE_COUNT] | |
240 | ||
241 | .size copy_user_page, .-copy_user_page | |
dbd2fdf5 DM |
242 | |
243 | .globl cheetah_patch_copy_page | |
244 | cheetah_patch_copy_page: | |
245 | sethi %hi(0x01000000), %o1 ! NOP | |
246 | sethi %hi(cheetah_copy_page_insn), %o0 | |
247 | or %o0, %lo(cheetah_copy_page_insn), %o0 | |
248 | stw %o1, [%o0] | |
249 | membar #StoreStore | |
250 | flush %o0 | |
251 | retl | |
252 | nop |