Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | /* |
2 | * ppc64 MMU hashtable management routines | |
3 | * | |
3c726f8d | 4 | * (c) Copyright IBM Corp. 2003, 2005 |
1da177e4 LT |
5 | * |
6 | * Maintained by: Benjamin Herrenschmidt | |
7 | * <benh@kernel.crashing.org> | |
8 | * | |
9 | * This file is covered by the GNU Public Licence v2 as | |
10 | * described in the kernel's COPYING file. | |
11 | */ | |
12 | ||
ab1f9dac | 13 | #include <asm/reg.h> |
1da177e4 LT |
14 | #include <asm/pgtable.h> |
15 | #include <asm/mmu.h> | |
16 | #include <asm/page.h> | |
17 | #include <asm/types.h> | |
18 | #include <asm/ppc_asm.h> | |
0013a854 | 19 | #include <asm/asm-offsets.h> |
1da177e4 LT |
20 | #include <asm/cputable.h> |
21 | ||
22 | .text | |
23 | ||
24 | /* | |
25 | * Stackframe: | |
26 | * | |
27 | * +-> Back chain (SP + 256) | |
28 | * | General register save area (SP + 112) | |
29 | * | Parameter save area (SP + 48) | |
30 | * | TOC save area (SP + 40) | |
31 | * | link editor doubleword (SP + 32) | |
32 | * | compiler doubleword (SP + 24) | |
33 | * | LR save area (SP + 16) | |
34 | * | CR save area (SP + 8) | |
35 | * SP ---> +-- Back chain (SP + 0) | |
36 | */ | |
37 | #define STACKFRAMESIZE 256 | |
38 | ||
39 | /* Save parameters offsets */ | |
40 | #define STK_PARM(i) (STACKFRAMESIZE + 48 + ((i)-3)*8) | |
41 | ||
42 | /* Save non-volatile offsets */ | |
43 | #define STK_REG(i) (112 + ((i)-14)*8) | |
44 | ||
3c726f8d BH |
45 | |
46 | #ifndef CONFIG_PPC_64K_PAGES | |
47 | ||
48 | /***************************************************************************** | |
49 | * * | |
50 | * 4K SW & 4K HW pages implementation * | |
51 | * * | |
52 | *****************************************************************************/ | |
53 | ||
54 | ||
1da177e4 | 55 | /* |
3c726f8d | 56 | * _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid, |
1189be65 | 57 | * pte_t *ptep, unsigned long trap, int local, int ssize) |
1da177e4 | 58 | * |
3c726f8d | 59 | * Adds a 4K page to the hash table in a segment of 4K pages only |
1da177e4 LT |
60 | */ |
61 | ||
3c726f8d | 62 | _GLOBAL(__hash_page_4K) |
1da177e4 LT |
63 | mflr r0 |
64 | std r0,16(r1) | |
65 | stdu r1,-STACKFRAMESIZE(r1) | |
66 | /* Save all params that we need after a function call */ | |
67 | std r6,STK_PARM(r6)(r1) | |
68 | std r8,STK_PARM(r8)(r1) | |
1189be65 | 69 | std r9,STK_PARM(r9)(r1) |
1da177e4 LT |
70 | |
71 | /* Add _PAGE_PRESENT to access */ | |
72 | ori r4,r4,_PAGE_PRESENT | |
73 | ||
74 | /* Save non-volatile registers. | |
75 | * r31 will hold "old PTE" | |
76 | * r30 is "new PTE" | |
77 | * r29 is "va" | |
78 | * r28 is a hash value | |
79 | * r27 is hashtab mask (maybe dynamic patched instead ?) | |
80 | */ | |
81 | std r27,STK_REG(r27)(r1) | |
82 | std r28,STK_REG(r28)(r1) | |
83 | std r29,STK_REG(r29)(r1) | |
84 | std r30,STK_REG(r30)(r1) | |
85 | std r31,STK_REG(r31)(r1) | |
86 | ||
87 | /* Step 1: | |
88 | * | |
89 | * Check permissions, atomically mark the linux PTE busy | |
90 | * and hashed. | |
91 | */ | |
92 | 1: | |
93 | ldarx r31,0,r6 | |
94 | /* Check access rights (access & ~(pte_val(*ptep))) */ | |
95 | andc. r0,r4,r31 | |
96 | bne- htab_wrong_access | |
97 | /* Check if PTE is busy */ | |
98 | andi. r0,r31,_PAGE_BUSY | |
d03853d5 OJ |
99 | /* If so, just bail out and refault if needed. Someone else |
100 | * is changing this PTE anyway and might hash it. | |
101 | */ | |
3c726f8d BH |
102 | bne- htab_bail_ok |
103 | ||
1da177e4 LT |
104 | /* Prepare new PTE value (turn access RW into DIRTY, then |
105 | * add BUSY,HASHPTE and ACCESSED) | |
106 | */ | |
107 | rlwinm r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */ | |
108 | or r30,r30,r31 | |
109 | ori r30,r30,_PAGE_BUSY | _PAGE_ACCESSED | _PAGE_HASHPTE | |
110 | /* Write the linux PTE atomically (setting busy) */ | |
111 | stdcx. r30,0,r6 | |
112 | bne- 1b | |
113 | isync | |
114 | ||
115 | /* Step 2: | |
116 | * | |
117 | * Insert/Update the HPTE in the hash table. At this point, | |
118 | * r4 (access) is re-useable, we use it for the new HPTE flags | |
119 | */ | |
120 | ||
1189be65 PM |
121 | BEGIN_FTR_SECTION |
122 | cmpdi r9,0 /* check segment size */ | |
123 | bne 3f | |
124 | END_FTR_SECTION_IFSET(CPU_FTR_1T_SEGMENT) | |
1da177e4 LT |
125 | /* Calc va and put it in r29 */ |
126 | rldicr r29,r5,28,63-28 | |
127 | rldicl r3,r3,0,36 | |
128 | or r29,r3,r29 | |
129 | ||
130 | /* Calculate hash value for primary slot and store it in r28 */ | |
131 | rldicl r5,r5,0,25 /* vsid & 0x0000007fffffffff */ | |
132 | rldicl r0,r3,64-12,48 /* (ea >> 12) & 0xffff */ | |
133 | xor r28,r5,r0 | |
1189be65 PM |
134 | b 4f |
135 | ||
136 | 3: /* Calc VA and hash in r29 and r28 for 1T segment */ | |
137 | sldi r29,r5,40 /* vsid << 40 */ | |
138 | clrldi r3,r3,24 /* ea & 0xffffffffff */ | |
139 | rldic r28,r5,25,25 /* (vsid << 25) & 0x7fffffffff */ | |
140 | clrldi r5,r5,40 /* vsid & 0xffffff */ | |
141 | rldicl r0,r3,64-12,36 /* (ea >> 12) & 0xfffffff */ | |
142 | xor r28,r28,r5 | |
143 | or r29,r3,r29 /* VA */ | |
144 | xor r28,r28,r0 /* hash */ | |
1da177e4 LT |
145 | |
146 | /* Convert linux PTE bits into HW equivalents */ | |
1189be65 | 147 | 4: andi. r3,r30,0x1fe /* Get basic set of flags */ |
3c726f8d | 148 | xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */ |
1da177e4 LT |
149 | rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */ |
150 | rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */ | |
3c726f8d | 151 | and r0,r0,r4 /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/ |
1da177e4 LT |
152 | andc r0,r30,r0 /* r0 = pte & ~r0 */ |
153 | rlwimi r3,r0,32-1,31,31 /* Insert result into PP lsb */ | |
c5cf0e30 | 154 | ori r3,r3,HPTE_R_C /* Always add "C" bit for perf. */ |
1da177e4 LT |
155 | |
156 | /* We eventually do the icache sync here (maybe inline that | |
157 | * code rather than call a C function...) | |
158 | */ | |
1da177e4 LT |
159 | BEGIN_FTR_SECTION |
160 | mr r4,r30 | |
161 | mr r5,r7 | |
162 | bl .hash_page_do_lazy_icache | |
8913ca1c | 163 | END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE) |
1da177e4 LT |
164 | |
165 | /* At this point, r3 contains new PP bits, save them in | |
166 | * place of "access" in the param area (sic) | |
167 | */ | |
168 | std r3,STK_PARM(r4)(r1) | |
169 | ||
170 | /* Get htab_hash_mask */ | |
171 | ld r4,htab_hash_mask@got(2) | |
172 | ld r27,0(r4) /* htab_hash_mask -> r27 */ | |
173 | ||
174 | /* Check if we may already be in the hashtable, in this case, we | |
175 | * go to out-of-line code to try to modify the HPTE | |
176 | */ | |
177 | andi. r0,r31,_PAGE_HASHPTE | |
178 | bne htab_modify_pte | |
179 | ||
180 | htab_insert_pte: | |
181 | /* Clear hpte bits in new pte (we also clear BUSY btw) and | |
182 | * add _PAGE_HASHPTE | |
183 | */ | |
184 | lis r0,_PAGE_HPTEFLAGS@h | |
185 | ori r0,r0,_PAGE_HPTEFLAGS@l | |
186 | andc r30,r30,r0 | |
187 | ori r30,r30,_PAGE_HASHPTE | |
188 | ||
3c726f8d BH |
189 | /* physical address r5 */ |
190 | rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT | |
191 | sldi r5,r5,PAGE_SHIFT | |
1da177e4 LT |
192 | |
193 | /* Calculate primary group hash */ | |
194 | and r0,r28,r27 | |
3c726f8d | 195 | rldicr r3,r0,3,63-3 /* r3 = (hash & mask) << 3 */ |
1da177e4 LT |
196 | |
197 | /* Call ppc_md.hpte_insert */ | |
3c726f8d | 198 | ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */ |
1da177e4 | 199 | mr r4,r29 /* Retreive va */ |
3c726f8d BH |
200 | li r7,0 /* !bolted, !secondary */ |
201 | li r8,MMU_PAGE_4K /* page size */ | |
1189be65 | 202 | ld r9,STK_PARM(r9)(r1) /* segment size */ |
1da177e4 | 203 | _GLOBAL(htab_call_hpte_insert1) |
3c726f8d | 204 | bl . /* Patched by htab_finish_init() */ |
1da177e4 LT |
205 | cmpdi 0,r3,0 |
206 | bge htab_pte_insert_ok /* Insertion successful */ | |
207 | cmpdi 0,r3,-2 /* Critical failure */ | |
208 | beq- htab_pte_insert_failure | |
209 | ||
210 | /* Now try secondary slot */ | |
211 | ||
3c726f8d BH |
212 | /* physical address r5 */ |
213 | rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT | |
214 | sldi r5,r5,PAGE_SHIFT | |
1da177e4 LT |
215 | |
216 | /* Calculate secondary group hash */ | |
217 | andc r0,r27,r28 | |
218 | rldicr r3,r0,3,63-3 /* r0 = (~hash & mask) << 3 */ | |
219 | ||
220 | /* Call ppc_md.hpte_insert */ | |
3c726f8d | 221 | ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */ |
1da177e4 | 222 | mr r4,r29 /* Retreive va */ |
3c726f8d BH |
223 | li r7,HPTE_V_SECONDARY /* !bolted, secondary */ |
224 | li r8,MMU_PAGE_4K /* page size */ | |
1189be65 | 225 | ld r9,STK_PARM(r9)(r1) /* segment size */ |
1da177e4 | 226 | _GLOBAL(htab_call_hpte_insert2) |
3c726f8d | 227 | bl . /* Patched by htab_finish_init() */ |
1da177e4 LT |
228 | cmpdi 0,r3,0 |
229 | bge+ htab_pte_insert_ok /* Insertion successful */ | |
230 | cmpdi 0,r3,-2 /* Critical failure */ | |
231 | beq- htab_pte_insert_failure | |
232 | ||
233 | /* Both are full, we need to evict something */ | |
234 | mftb r0 | |
235 | /* Pick a random group based on TB */ | |
236 | andi. r0,r0,1 | |
237 | mr r5,r28 | |
238 | bne 2f | |
239 | not r5,r5 | |
240 | 2: and r0,r5,r27 | |
241 | rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */ | |
242 | /* Call ppc_md.hpte_remove */ | |
243 | _GLOBAL(htab_call_hpte_remove) | |
3c726f8d | 244 | bl . /* Patched by htab_finish_init() */ |
1da177e4 LT |
245 | |
246 | /* Try all again */ | |
247 | b htab_insert_pte | |
248 | ||
3c726f8d | 249 | htab_bail_ok: |
d03853d5 | 250 | li r3,0 |
3c726f8d | 251 | b htab_bail |
d03853d5 | 252 | |
1da177e4 LT |
253 | htab_pte_insert_ok: |
254 | /* Insert slot number & secondary bit in PTE */ | |
255 | rldimi r30,r3,12,63-15 | |
256 | ||
257 | /* Write out the PTE with a normal write | |
258 | * (maybe add eieio may be good still ?) | |
259 | */ | |
260 | htab_write_out_pte: | |
261 | ld r6,STK_PARM(r6)(r1) | |
262 | std r30,0(r6) | |
263 | li r3, 0 | |
3c726f8d | 264 | htab_bail: |
1da177e4 LT |
265 | ld r27,STK_REG(r27)(r1) |
266 | ld r28,STK_REG(r28)(r1) | |
267 | ld r29,STK_REG(r29)(r1) | |
268 | ld r30,STK_REG(r30)(r1) | |
269 | ld r31,STK_REG(r31)(r1) | |
270 | addi r1,r1,STACKFRAMESIZE | |
271 | ld r0,16(r1) | |
272 | mtlr r0 | |
273 | blr | |
274 | ||
275 | htab_modify_pte: | |
276 | /* Keep PP bits in r4 and slot idx from the PTE around in r3 */ | |
277 | mr r4,r3 | |
278 | rlwinm r3,r31,32-12,29,31 | |
279 | ||
280 | /* Secondary group ? if yes, get a inverted hash value */ | |
281 | mr r5,r28 | |
282 | andi. r0,r31,_PAGE_SECONDARY | |
283 | beq 1f | |
284 | not r5,r5 | |
285 | 1: | |
286 | /* Calculate proper slot value for ppc_md.hpte_updatepp */ | |
287 | and r0,r5,r27 | |
288 | rldicr r0,r0,3,63-3 /* r0 = (hash & mask) << 3 */ | |
289 | add r3,r0,r3 /* add slot idx */ | |
290 | ||
291 | /* Call ppc_md.hpte_updatepp */ | |
292 | mr r5,r29 /* va */ | |
3c726f8d | 293 | li r6,MMU_PAGE_4K /* page size */ |
1189be65 PM |
294 | ld r7,STK_PARM(r9)(r1) /* segment size */ |
295 | ld r8,STK_PARM(r8)(r1) /* get "local" param */ | |
1da177e4 | 296 | _GLOBAL(htab_call_hpte_updatepp) |
3c726f8d | 297 | bl . /* Patched by htab_finish_init() */ |
1da177e4 LT |
298 | |
299 | /* if we failed because typically the HPTE wasn't really here | |
300 | * we try an insertion. | |
301 | */ | |
302 | cmpdi 0,r3,-1 | |
303 | beq- htab_insert_pte | |
304 | ||
305 | /* Clear the BUSY bit and Write out the PTE */ | |
306 | li r0,_PAGE_BUSY | |
307 | andc r30,r30,r0 | |
308 | b htab_write_out_pte | |
309 | ||
310 | htab_wrong_access: | |
311 | /* Bail out clearing reservation */ | |
312 | stdcx. r31,0,r6 | |
313 | li r3,1 | |
3c726f8d BH |
314 | b htab_bail |
315 | ||
316 | htab_pte_insert_failure: | |
317 | /* Bail out restoring old PTE */ | |
318 | ld r6,STK_PARM(r6)(r1) | |
319 | std r31,0(r6) | |
320 | li r3,-1 | |
321 | b htab_bail | |
322 | ||
323 | ||
324 | #else /* CONFIG_PPC_64K_PAGES */ | |
325 | ||
326 | ||
327 | /***************************************************************************** | |
328 | * * | |
329 | * 64K SW & 4K or 64K HW in a 4K segment pages implementation * | |
330 | * * | |
331 | *****************************************************************************/ | |
332 | ||
333 | /* _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid, | |
fa28237c PM |
334 | * pte_t *ptep, unsigned long trap, int local, int ssize, |
335 | * int subpg_prot) | |
3c726f8d BH |
336 | */ |
337 | ||
338 | /* | |
339 | * For now, we do NOT implement Admixed pages | |
340 | */ | |
341 | _GLOBAL(__hash_page_4K) | |
342 | mflr r0 | |
343 | std r0,16(r1) | |
344 | stdu r1,-STACKFRAMESIZE(r1) | |
345 | /* Save all params that we need after a function call */ | |
346 | std r6,STK_PARM(r6)(r1) | |
347 | std r8,STK_PARM(r8)(r1) | |
1189be65 | 348 | std r9,STK_PARM(r9)(r1) |
3c726f8d BH |
349 | |
350 | /* Add _PAGE_PRESENT to access */ | |
351 | ori r4,r4,_PAGE_PRESENT | |
352 | ||
353 | /* Save non-volatile registers. | |
354 | * r31 will hold "old PTE" | |
355 | * r30 is "new PTE" | |
356 | * r29 is "va" | |
357 | * r28 is a hash value | |
358 | * r27 is hashtab mask (maybe dynamic patched instead ?) | |
359 | * r26 is the hidx mask | |
360 | * r25 is the index in combo page | |
361 | */ | |
362 | std r25,STK_REG(r25)(r1) | |
363 | std r26,STK_REG(r26)(r1) | |
364 | std r27,STK_REG(r27)(r1) | |
365 | std r28,STK_REG(r28)(r1) | |
366 | std r29,STK_REG(r29)(r1) | |
367 | std r30,STK_REG(r30)(r1) | |
368 | std r31,STK_REG(r31)(r1) | |
369 | ||
370 | /* Step 1: | |
371 | * | |
372 | * Check permissions, atomically mark the linux PTE busy | |
373 | * and hashed. | |
374 | */ | |
375 | 1: | |
376 | ldarx r31,0,r6 | |
377 | /* Check access rights (access & ~(pte_val(*ptep))) */ | |
378 | andc. r0,r4,r31 | |
379 | bne- htab_wrong_access | |
380 | /* Check if PTE is busy */ | |
381 | andi. r0,r31,_PAGE_BUSY | |
382 | /* If so, just bail out and refault if needed. Someone else | |
383 | * is changing this PTE anyway and might hash it. | |
384 | */ | |
385 | bne- htab_bail_ok | |
386 | /* Prepare new PTE value (turn access RW into DIRTY, then | |
387 | * add BUSY and ACCESSED) | |
388 | */ | |
389 | rlwinm r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */ | |
390 | or r30,r30,r31 | |
41743a4e | 391 | ori r30,r30,_PAGE_BUSY | _PAGE_ACCESSED |
bf72aeba | 392 | oris r30,r30,_PAGE_COMBO@h |
3c726f8d BH |
393 | /* Write the linux PTE atomically (setting busy) */ |
394 | stdcx. r30,0,r6 | |
395 | bne- 1b | |
396 | isync | |
397 | ||
398 | /* Step 2: | |
399 | * | |
400 | * Insert/Update the HPTE in the hash table. At this point, | |
401 | * r4 (access) is re-useable, we use it for the new HPTE flags | |
402 | */ | |
403 | ||
404 | /* Load the hidx index */ | |
405 | rldicl r25,r3,64-12,60 | |
406 | ||
1189be65 PM |
407 | BEGIN_FTR_SECTION |
408 | cmpdi r9,0 /* check segment size */ | |
409 | bne 3f | |
410 | END_FTR_SECTION_IFSET(CPU_FTR_1T_SEGMENT) | |
3c726f8d BH |
411 | /* Calc va and put it in r29 */ |
412 | rldicr r29,r5,28,63-28 /* r29 = (vsid << 28) */ | |
413 | rldicl r3,r3,0,36 /* r3 = (ea & 0x0fffffff) */ | |
1189be65 | 414 | or r29,r3,r29 /* r29 = va */ |
3c726f8d BH |
415 | |
416 | /* Calculate hash value for primary slot and store it in r28 */ | |
417 | rldicl r5,r5,0,25 /* vsid & 0x0000007fffffffff */ | |
418 | rldicl r0,r3,64-12,48 /* (ea >> 12) & 0xffff */ | |
419 | xor r28,r5,r0 | |
1189be65 PM |
420 | b 4f |
421 | ||
422 | 3: /* Calc VA and hash in r29 and r28 for 1T segment */ | |
423 | sldi r29,r5,40 /* vsid << 40 */ | |
424 | clrldi r3,r3,24 /* ea & 0xffffffffff */ | |
425 | rldic r28,r5,25,25 /* (vsid << 25) & 0x7fffffffff */ | |
426 | clrldi r5,r5,40 /* vsid & 0xffffff */ | |
427 | rldicl r0,r3,64-12,36 /* (ea >> 12) & 0xfffffff */ | |
428 | xor r28,r28,r5 | |
429 | or r29,r3,r29 /* VA */ | |
430 | xor r28,r28,r0 /* hash */ | |
3c726f8d BH |
431 | |
432 | /* Convert linux PTE bits into HW equivalents */ | |
fa28237c PM |
433 | 4: |
434 | #ifdef CONFIG_PPC_SUBPAGE_PROT | |
435 | andc r10,r30,r10 | |
436 | andi. r3,r10,0x1fe /* Get basic set of flags */ | |
437 | rlwinm r0,r10,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */ | |
438 | #else | |
439 | andi. r3,r30,0x1fe /* Get basic set of flags */ | |
3c726f8d | 440 | rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */ |
fa28237c PM |
441 | #endif |
442 | xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */ | |
3c726f8d BH |
443 | rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */ |
444 | and r0,r0,r4 /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/ | |
fa28237c | 445 | andc r0,r3,r0 /* r0 = pte & ~r0 */ |
3c726f8d | 446 | rlwimi r3,r0,32-1,31,31 /* Insert result into PP lsb */ |
c5cf0e30 | 447 | ori r3,r3,HPTE_R_C /* Always add "C" bit for perf. */ |
3c726f8d BH |
448 | |
449 | /* We eventually do the icache sync here (maybe inline that | |
450 | * code rather than call a C function...) | |
451 | */ | |
452 | BEGIN_FTR_SECTION | |
453 | mr r4,r30 | |
454 | mr r5,r7 | |
455 | bl .hash_page_do_lazy_icache | |
456 | END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE) | |
457 | ||
458 | /* At this point, r3 contains new PP bits, save them in | |
459 | * place of "access" in the param area (sic) | |
460 | */ | |
461 | std r3,STK_PARM(r4)(r1) | |
462 | ||
463 | /* Get htab_hash_mask */ | |
464 | ld r4,htab_hash_mask@got(2) | |
465 | ld r27,0(r4) /* htab_hash_mask -> r27 */ | |
466 | ||
467 | /* Check if we may already be in the hashtable, in this case, we | |
468 | * go to out-of-line code to try to modify the HPTE. We look for | |
469 | * the bit at (1 >> (index + 32)) | |
470 | */ | |
41743a4e | 471 | rldicl. r0,r31,64-12,48 |
3c726f8d BH |
472 | li r26,0 /* Default hidx */ |
473 | beq htab_insert_pte | |
bf72aeba PM |
474 | |
475 | /* | |
476 | * Check if the pte was already inserted into the hash table | |
477 | * as a 64k HW page, and invalidate the 64k HPTE if so. | |
478 | */ | |
479 | andis. r0,r31,_PAGE_COMBO@h | |
480 | beq htab_inval_old_hpte | |
481 | ||
3c726f8d BH |
482 | ld r6,STK_PARM(r6)(r1) |
483 | ori r26,r6,0x8000 /* Load the hidx mask */ | |
484 | ld r26,0(r26) | |
485 | addi r5,r25,36 /* Check actual HPTE_SUB bit, this */ | |
486 | rldcr. r0,r31,r5,0 /* must match pgtable.h definition */ | |
487 | bne htab_modify_pte | |
488 | ||
489 | htab_insert_pte: | |
490 | /* real page number in r5, PTE RPN value + index */ | |
721151d0 PM |
491 | andis. r0,r31,_PAGE_4K_PFN@h |
492 | srdi r5,r31,PTE_RPN_SHIFT | |
493 | bne- htab_special_pfn | |
3c726f8d BH |
494 | sldi r5,r5,PAGE_SHIFT-HW_PAGE_SHIFT |
495 | add r5,r5,r25 | |
721151d0 | 496 | htab_special_pfn: |
3c726f8d BH |
497 | sldi r5,r5,HW_PAGE_SHIFT |
498 | ||
499 | /* Calculate primary group hash */ | |
500 | and r0,r28,r27 | |
501 | rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */ | |
502 | ||
503 | /* Call ppc_md.hpte_insert */ | |
504 | ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */ | |
505 | mr r4,r29 /* Retreive va */ | |
506 | li r7,0 /* !bolted, !secondary */ | |
507 | li r8,MMU_PAGE_4K /* page size */ | |
1189be65 | 508 | ld r9,STK_PARM(r9)(r1) /* segment size */ |
3c726f8d BH |
509 | _GLOBAL(htab_call_hpte_insert1) |
510 | bl . /* patched by htab_finish_init() */ | |
511 | cmpdi 0,r3,0 | |
512 | bge htab_pte_insert_ok /* Insertion successful */ | |
513 | cmpdi 0,r3,-2 /* Critical failure */ | |
514 | beq- htab_pte_insert_failure | |
515 | ||
516 | /* Now try secondary slot */ | |
517 | ||
518 | /* real page number in r5, PTE RPN value + index */ | |
430404ed PM |
519 | andis. r0,r31,_PAGE_4K_PFN@h |
520 | srdi r5,r31,PTE_RPN_SHIFT | |
521 | bne- 3f | |
3c726f8d BH |
522 | sldi r5,r5,PAGE_SHIFT-HW_PAGE_SHIFT |
523 | add r5,r5,r25 | |
430404ed | 524 | 3: sldi r5,r5,HW_PAGE_SHIFT |
3c726f8d BH |
525 | |
526 | /* Calculate secondary group hash */ | |
527 | andc r0,r27,r28 | |
528 | rldicr r3,r0,3,63-3 /* r0 = (~hash & mask) << 3 */ | |
529 | ||
530 | /* Call ppc_md.hpte_insert */ | |
531 | ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */ | |
532 | mr r4,r29 /* Retreive va */ | |
533 | li r7,HPTE_V_SECONDARY /* !bolted, secondary */ | |
534 | li r8,MMU_PAGE_4K /* page size */ | |
1189be65 | 535 | ld r9,STK_PARM(r9)(r1) /* segment size */ |
3c726f8d BH |
536 | _GLOBAL(htab_call_hpte_insert2) |
537 | bl . /* patched by htab_finish_init() */ | |
538 | cmpdi 0,r3,0 | |
539 | bge+ htab_pte_insert_ok /* Insertion successful */ | |
540 | cmpdi 0,r3,-2 /* Critical failure */ | |
541 | beq- htab_pte_insert_failure | |
542 | ||
543 | /* Both are full, we need to evict something */ | |
544 | mftb r0 | |
545 | /* Pick a random group based on TB */ | |
546 | andi. r0,r0,1 | |
547 | mr r5,r28 | |
548 | bne 2f | |
549 | not r5,r5 | |
550 | 2: and r0,r5,r27 | |
551 | rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */ | |
552 | /* Call ppc_md.hpte_remove */ | |
553 | _GLOBAL(htab_call_hpte_remove) | |
554 | bl . /* patched by htab_finish_init() */ | |
555 | ||
556 | /* Try all again */ | |
557 | b htab_insert_pte | |
558 | ||
bf72aeba PM |
559 | /* |
560 | * Call out to C code to invalidate an 64k HW HPTE that is | |
561 | * useless now that the segment has been switched to 4k pages. | |
562 | */ | |
563 | htab_inval_old_hpte: | |
564 | mr r3,r29 /* virtual addr */ | |
565 | mr r4,r31 /* PTE.pte */ | |
566 | li r5,0 /* PTE.hidx */ | |
567 | li r6,MMU_PAGE_64K /* psize */ | |
f6ab0b92 BH |
568 | ld r7,STK_PARM(r9)(r1) /* ssize */ |
569 | ld r8,STK_PARM(r8)(r1) /* local */ | |
bf72aeba | 570 | bl .flush_hash_page |
65ba6cdc PM |
571 | /* Clear out _PAGE_HPTE_SUB bits in the new linux PTE */ |
572 | lis r0,_PAGE_HPTE_SUB@h | |
573 | ori r0,r0,_PAGE_HPTE_SUB@l | |
574 | andc r30,r30,r0 | |
bf72aeba PM |
575 | b htab_insert_pte |
576 | ||
3c726f8d BH |
577 | htab_bail_ok: |
578 | li r3,0 | |
579 | b htab_bail | |
580 | ||
581 | htab_pte_insert_ok: | |
582 | /* Insert slot number & secondary bit in PTE second half, | |
583 | * clear _PAGE_BUSY and set approriate HPTE slot bit | |
584 | */ | |
585 | ld r6,STK_PARM(r6)(r1) | |
586 | li r0,_PAGE_BUSY | |
587 | andc r30,r30,r0 | |
588 | /* HPTE SUB bit */ | |
589 | li r0,1 | |
590 | subfic r5,r25,27 /* Must match bit position in */ | |
591 | sld r0,r0,r5 /* pgtable.h */ | |
592 | or r30,r30,r0 | |
593 | /* hindx */ | |
594 | sldi r5,r25,2 | |
595 | sld r3,r3,r5 | |
596 | li r4,0xf | |
597 | sld r4,r4,r5 | |
598 | andc r26,r26,r4 | |
599 | or r26,r26,r3 | |
600 | ori r5,r6,0x8000 | |
601 | std r26,0(r5) | |
602 | lwsync | |
603 | std r30,0(r6) | |
604 | li r3, 0 | |
605 | htab_bail: | |
606 | ld r25,STK_REG(r25)(r1) | |
607 | ld r26,STK_REG(r26)(r1) | |
608 | ld r27,STK_REG(r27)(r1) | |
609 | ld r28,STK_REG(r28)(r1) | |
610 | ld r29,STK_REG(r29)(r1) | |
611 | ld r30,STK_REG(r30)(r1) | |
612 | ld r31,STK_REG(r31)(r1) | |
613 | addi r1,r1,STACKFRAMESIZE | |
614 | ld r0,16(r1) | |
615 | mtlr r0 | |
616 | blr | |
617 | ||
618 | htab_modify_pte: | |
619 | /* Keep PP bits in r4 and slot idx from the PTE around in r3 */ | |
620 | mr r4,r3 | |
621 | sldi r5,r25,2 | |
622 | srd r3,r26,r5 | |
623 | ||
624 | /* Secondary group ? if yes, get a inverted hash value */ | |
625 | mr r5,r28 | |
626 | andi. r0,r3,0x8 /* page secondary ? */ | |
627 | beq 1f | |
628 | not r5,r5 | |
629 | 1: andi. r3,r3,0x7 /* extract idx alone */ | |
630 | ||
631 | /* Calculate proper slot value for ppc_md.hpte_updatepp */ | |
632 | and r0,r5,r27 | |
633 | rldicr r0,r0,3,63-3 /* r0 = (hash & mask) << 3 */ | |
634 | add r3,r0,r3 /* add slot idx */ | |
635 | ||
636 | /* Call ppc_md.hpte_updatepp */ | |
637 | mr r5,r29 /* va */ | |
638 | li r6,MMU_PAGE_4K /* page size */ | |
1189be65 PM |
639 | ld r7,STK_PARM(r9)(r1) /* segment size */ |
640 | ld r8,STK_PARM(r8)(r1) /* get "local" param */ | |
3c726f8d BH |
641 | _GLOBAL(htab_call_hpte_updatepp) |
642 | bl . /* patched by htab_finish_init() */ | |
643 | ||
644 | /* if we failed because typically the HPTE wasn't really here | |
645 | * we try an insertion. | |
646 | */ | |
647 | cmpdi 0,r3,-1 | |
648 | beq- htab_insert_pte | |
649 | ||
650 | /* Clear the BUSY bit and Write out the PTE */ | |
651 | li r0,_PAGE_BUSY | |
652 | andc r30,r30,r0 | |
653 | ld r6,STK_PARM(r6)(r1) | |
654 | std r30,0(r6) | |
655 | li r3,0 | |
656 | b htab_bail | |
657 | ||
658 | htab_wrong_access: | |
659 | /* Bail out clearing reservation */ | |
660 | stdcx. r31,0,r6 | |
661 | li r3,1 | |
662 | b htab_bail | |
1da177e4 LT |
663 | |
664 | htab_pte_insert_failure: | |
665 | /* Bail out restoring old PTE */ | |
666 | ld r6,STK_PARM(r6)(r1) | |
667 | std r31,0(r6) | |
668 | li r3,-1 | |
3c726f8d BH |
669 | b htab_bail |
670 | ||
16c2d476 BH |
671 | #endif /* CONFIG_PPC_64K_PAGES */ |
672 | ||
673 | #ifdef CONFIG_PPC_HAS_HASH_64K | |
3c726f8d BH |
674 | |
675 | /***************************************************************************** | |
676 | * * | |
677 | * 64K SW & 64K HW in a 64K segment pages implementation * | |
678 | * * | |
679 | *****************************************************************************/ | |
680 | ||
681 | _GLOBAL(__hash_page_64K) | |
682 | mflr r0 | |
683 | std r0,16(r1) | |
684 | stdu r1,-STACKFRAMESIZE(r1) | |
685 | /* Save all params that we need after a function call */ | |
686 | std r6,STK_PARM(r6)(r1) | |
687 | std r8,STK_PARM(r8)(r1) | |
1189be65 | 688 | std r9,STK_PARM(r9)(r1) |
3c726f8d BH |
689 | |
690 | /* Add _PAGE_PRESENT to access */ | |
691 | ori r4,r4,_PAGE_PRESENT | |
692 | ||
693 | /* Save non-volatile registers. | |
694 | * r31 will hold "old PTE" | |
695 | * r30 is "new PTE" | |
696 | * r29 is "va" | |
697 | * r28 is a hash value | |
698 | * r27 is hashtab mask (maybe dynamic patched instead ?) | |
699 | */ | |
700 | std r27,STK_REG(r27)(r1) | |
701 | std r28,STK_REG(r28)(r1) | |
702 | std r29,STK_REG(r29)(r1) | |
703 | std r30,STK_REG(r30)(r1) | |
704 | std r31,STK_REG(r31)(r1) | |
705 | ||
706 | /* Step 1: | |
707 | * | |
708 | * Check permissions, atomically mark the linux PTE busy | |
709 | * and hashed. | |
710 | */ | |
711 | 1: | |
712 | ldarx r31,0,r6 | |
713 | /* Check access rights (access & ~(pte_val(*ptep))) */ | |
714 | andc. r0,r4,r31 | |
715 | bne- ht64_wrong_access | |
716 | /* Check if PTE is busy */ | |
717 | andi. r0,r31,_PAGE_BUSY | |
718 | /* If so, just bail out and refault if needed. Someone else | |
719 | * is changing this PTE anyway and might hash it. | |
720 | */ | |
721 | bne- ht64_bail_ok | |
bf72aeba PM |
722 | BEGIN_FTR_SECTION |
723 | /* Check if PTE has the cache-inhibit bit set */ | |
724 | andi. r0,r31,_PAGE_NO_CACHE | |
725 | /* If so, bail out and refault as a 4k page */ | |
726 | bne- ht64_bail_ok | |
727 | END_FTR_SECTION_IFCLR(CPU_FTR_CI_LARGE_PAGE) | |
3c726f8d | 728 | /* Prepare new PTE value (turn access RW into DIRTY, then |
41743a4e | 729 | * add BUSY and ACCESSED) |
3c726f8d BH |
730 | */ |
731 | rlwinm r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */ | |
732 | or r30,r30,r31 | |
41743a4e | 733 | ori r30,r30,_PAGE_BUSY | _PAGE_ACCESSED |
3c726f8d BH |
734 | /* Write the linux PTE atomically (setting busy) */ |
735 | stdcx. r30,0,r6 | |
736 | bne- 1b | |
737 | isync | |
738 | ||
739 | /* Step 2: | |
740 | * | |
741 | * Insert/Update the HPTE in the hash table. At this point, | |
742 | * r4 (access) is re-useable, we use it for the new HPTE flags | |
743 | */ | |
744 | ||
1189be65 PM |
745 | BEGIN_FTR_SECTION |
746 | cmpdi r9,0 /* check segment size */ | |
747 | bne 3f | |
748 | END_FTR_SECTION_IFSET(CPU_FTR_1T_SEGMENT) | |
3c726f8d BH |
749 | /* Calc va and put it in r29 */ |
750 | rldicr r29,r5,28,63-28 | |
751 | rldicl r3,r3,0,36 | |
752 | or r29,r3,r29 | |
753 | ||
754 | /* Calculate hash value for primary slot and store it in r28 */ | |
755 | rldicl r5,r5,0,25 /* vsid & 0x0000007fffffffff */ | |
756 | rldicl r0,r3,64-16,52 /* (ea >> 16) & 0xfff */ | |
757 | xor r28,r5,r0 | |
1189be65 PM |
758 | b 4f |
759 | ||
760 | 3: /* Calc VA and hash in r29 and r28 for 1T segment */ | |
761 | sldi r29,r5,40 /* vsid << 40 */ | |
762 | clrldi r3,r3,24 /* ea & 0xffffffffff */ | |
763 | rldic r28,r5,25,25 /* (vsid << 25) & 0x7fffffffff */ | |
764 | clrldi r5,r5,40 /* vsid & 0xffffff */ | |
765 | rldicl r0,r3,64-16,40 /* (ea >> 16) & 0xffffff */ | |
766 | xor r28,r28,r5 | |
767 | or r29,r3,r29 /* VA */ | |
768 | xor r28,r28,r0 /* hash */ | |
3c726f8d BH |
769 | |
770 | /* Convert linux PTE bits into HW equivalents */ | |
1189be65 | 771 | 4: andi. r3,r30,0x1fe /* Get basic set of flags */ |
3c726f8d BH |
772 | xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */ |
773 | rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */ | |
774 | rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */ | |
775 | and r0,r0,r4 /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/ | |
776 | andc r0,r30,r0 /* r0 = pte & ~r0 */ | |
777 | rlwimi r3,r0,32-1,31,31 /* Insert result into PP lsb */ | |
c5cf0e30 | 778 | ori r3,r3,HPTE_R_C /* Always add "C" bit for perf. */ |
3c726f8d BH |
779 | |
780 | /* We eventually do the icache sync here (maybe inline that | |
781 | * code rather than call a C function...) | |
782 | */ | |
783 | BEGIN_FTR_SECTION | |
784 | mr r4,r30 | |
785 | mr r5,r7 | |
786 | bl .hash_page_do_lazy_icache | |
787 | END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE) | |
788 | ||
789 | /* At this point, r3 contains new PP bits, save them in | |
790 | * place of "access" in the param area (sic) | |
791 | */ | |
792 | std r3,STK_PARM(r4)(r1) | |
793 | ||
794 | /* Get htab_hash_mask */ | |
795 | ld r4,htab_hash_mask@got(2) | |
796 | ld r27,0(r4) /* htab_hash_mask -> r27 */ | |
797 | ||
798 | /* Check if we may already be in the hashtable, in this case, we | |
799 | * go to out-of-line code to try to modify the HPTE | |
800 | */ | |
41743a4e | 801 | rldicl. r0,r31,64-12,48 |
3c726f8d BH |
802 | bne ht64_modify_pte |
803 | ||
804 | ht64_insert_pte: | |
805 | /* Clear hpte bits in new pte (we also clear BUSY btw) and | |
41743a4e | 806 | * add _PAGE_HPTE_SUB0 |
3c726f8d BH |
807 | */ |
808 | lis r0,_PAGE_HPTEFLAGS@h | |
809 | ori r0,r0,_PAGE_HPTEFLAGS@l | |
810 | andc r30,r30,r0 | |
41743a4e BH |
811 | #ifdef CONFIG_PPC_64K_PAGES |
812 | oris r30,r30,_PAGE_HPTE_SUB0@h | |
813 | #else | |
3c726f8d | 814 | ori r30,r30,_PAGE_HASHPTE |
41743a4e | 815 | #endif |
3c726f8d BH |
816 | /* Phyical address in r5 */ |
817 | rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT | |
818 | sldi r5,r5,PAGE_SHIFT | |
819 | ||
820 | /* Calculate primary group hash */ | |
821 | and r0,r28,r27 | |
822 | rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */ | |
823 | ||
824 | /* Call ppc_md.hpte_insert */ | |
825 | ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */ | |
826 | mr r4,r29 /* Retreive va */ | |
827 | li r7,0 /* !bolted, !secondary */ | |
828 | li r8,MMU_PAGE_64K | |
1189be65 | 829 | ld r9,STK_PARM(r9)(r1) /* segment size */ |
3c726f8d BH |
830 | _GLOBAL(ht64_call_hpte_insert1) |
831 | bl . /* patched by htab_finish_init() */ | |
832 | cmpdi 0,r3,0 | |
833 | bge ht64_pte_insert_ok /* Insertion successful */ | |
834 | cmpdi 0,r3,-2 /* Critical failure */ | |
835 | beq- ht64_pte_insert_failure | |
836 | ||
837 | /* Now try secondary slot */ | |
838 | ||
839 | /* Phyical address in r5 */ | |
840 | rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT | |
841 | sldi r5,r5,PAGE_SHIFT | |
842 | ||
843 | /* Calculate secondary group hash */ | |
844 | andc r0,r27,r28 | |
845 | rldicr r3,r0,3,63-3 /* r0 = (~hash & mask) << 3 */ | |
846 | ||
847 | /* Call ppc_md.hpte_insert */ | |
848 | ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */ | |
849 | mr r4,r29 /* Retreive va */ | |
850 | li r7,HPTE_V_SECONDARY /* !bolted, secondary */ | |
851 | li r8,MMU_PAGE_64K | |
1189be65 | 852 | ld r9,STK_PARM(r9)(r1) /* segment size */ |
3c726f8d BH |
853 | _GLOBAL(ht64_call_hpte_insert2) |
854 | bl . /* patched by htab_finish_init() */ | |
855 | cmpdi 0,r3,0 | |
856 | bge+ ht64_pte_insert_ok /* Insertion successful */ | |
857 | cmpdi 0,r3,-2 /* Critical failure */ | |
858 | beq- ht64_pte_insert_failure | |
859 | ||
860 | /* Both are full, we need to evict something */ | |
861 | mftb r0 | |
862 | /* Pick a random group based on TB */ | |
863 | andi. r0,r0,1 | |
864 | mr r5,r28 | |
865 | bne 2f | |
866 | not r5,r5 | |
867 | 2: and r0,r5,r27 | |
868 | rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */ | |
869 | /* Call ppc_md.hpte_remove */ | |
870 | _GLOBAL(ht64_call_hpte_remove) | |
871 | bl . /* patched by htab_finish_init() */ | |
872 | ||
873 | /* Try all again */ | |
874 | b ht64_insert_pte | |
875 | ||
876 | ht64_bail_ok: | |
877 | li r3,0 | |
878 | b ht64_bail | |
879 | ||
880 | ht64_pte_insert_ok: | |
881 | /* Insert slot number & secondary bit in PTE */ | |
882 | rldimi r30,r3,12,63-15 | |
883 | ||
884 | /* Write out the PTE with a normal write | |
885 | * (maybe add eieio may be good still ?) | |
886 | */ | |
887 | ht64_write_out_pte: | |
888 | ld r6,STK_PARM(r6)(r1) | |
889 | std r30,0(r6) | |
890 | li r3, 0 | |
891 | ht64_bail: | |
892 | ld r27,STK_REG(r27)(r1) | |
893 | ld r28,STK_REG(r28)(r1) | |
894 | ld r29,STK_REG(r29)(r1) | |
895 | ld r30,STK_REG(r30)(r1) | |
896 | ld r31,STK_REG(r31)(r1) | |
897 | addi r1,r1,STACKFRAMESIZE | |
898 | ld r0,16(r1) | |
899 | mtlr r0 | |
900 | blr | |
901 | ||
902 | ht64_modify_pte: | |
903 | /* Keep PP bits in r4 and slot idx from the PTE around in r3 */ | |
904 | mr r4,r3 | |
905 | rlwinm r3,r31,32-12,29,31 | |
906 | ||
907 | /* Secondary group ? if yes, get a inverted hash value */ | |
908 | mr r5,r28 | |
909 | andi. r0,r31,_PAGE_F_SECOND | |
910 | beq 1f | |
911 | not r5,r5 | |
912 | 1: | |
913 | /* Calculate proper slot value for ppc_md.hpte_updatepp */ | |
914 | and r0,r5,r27 | |
915 | rldicr r0,r0,3,63-3 /* r0 = (hash & mask) << 3 */ | |
916 | add r3,r0,r3 /* add slot idx */ | |
917 | ||
918 | /* Call ppc_md.hpte_updatepp */ | |
919 | mr r5,r29 /* va */ | |
920 | li r6,MMU_PAGE_64K | |
1189be65 PM |
921 | ld r7,STK_PARM(r9)(r1) /* segment size */ |
922 | ld r8,STK_PARM(r8)(r1) /* get "local" param */ | |
3c726f8d BH |
923 | _GLOBAL(ht64_call_hpte_updatepp) |
924 | bl . /* patched by htab_finish_init() */ | |
925 | ||
926 | /* if we failed because typically the HPTE wasn't really here | |
927 | * we try an insertion. | |
928 | */ | |
929 | cmpdi 0,r3,-1 | |
930 | beq- ht64_insert_pte | |
931 | ||
932 | /* Clear the BUSY bit and Write out the PTE */ | |
933 | li r0,_PAGE_BUSY | |
934 | andc r30,r30,r0 | |
935 | b ht64_write_out_pte | |
936 | ||
937 | ht64_wrong_access: | |
938 | /* Bail out clearing reservation */ | |
939 | stdcx. r31,0,r6 | |
940 | li r3,1 | |
941 | b ht64_bail | |
942 | ||
943 | ht64_pte_insert_failure: | |
944 | /* Bail out restoring old PTE */ | |
945 | ld r6,STK_PARM(r6)(r1) | |
946 | std r31,0(r6) | |
947 | li r3,-1 | |
948 | b ht64_bail | |
949 | ||
950 | ||
16c2d476 | 951 | #endif /* CONFIG_PPC_HAS_HASH_64K */ |
1da177e4 LT |
952 | |
953 | ||
3c726f8d BH |
954 | /***************************************************************************** |
955 | * * | |
956 | * Huge pages implementation is in hugetlbpage.c * | |
957 | * * | |
958 | *****************************************************************************/ |