powerpc/pseries: Initialise nvram_pstore_info's buf_lock
[deliverable/linux.git] / arch / powerpc / mm / hugetlbpage.c
CommitLineData
1da177e4 1/*
41151e77 2 * PPC Huge TLB Page Support for Kernel.
1da177e4
LT
3 *
4 * Copyright (C) 2003 David Gibson, IBM Corporation.
41151e77 5 * Copyright (C) 2011 Becky Bruce, Freescale Semiconductor
1da177e4
LT
6 *
7 * Based on the IA-32 version:
8 * Copyright (C) 2002, Rohit Seth <rohit.seth@intel.com>
9 */
10
1da177e4 11#include <linux/mm.h>
883a3e52 12#include <linux/io.h>
5a0e3ad6 13#include <linux/slab.h>
1da177e4 14#include <linux/hugetlb.h>
342d3db7 15#include <linux/export.h>
41151e77
BB
16#include <linux/of_fdt.h>
17#include <linux/memblock.h>
18#include <linux/bootmem.h>
13020be8 19#include <linux/moduleparam.h>
883a3e52 20#include <asm/pgtable.h>
1da177e4
LT
21#include <asm/pgalloc.h>
22#include <asm/tlb.h>
41151e77 23#include <asm/setup.h>
29409997
AK
24#include <asm/hugetlb.h>
25
26#ifdef CONFIG_HUGETLB_PAGE
1da177e4 27
91224346
JT
28#define PAGE_SHIFT_64K 16
29#define PAGE_SHIFT_16M 24
30#define PAGE_SHIFT_16G 34
4ec161cf 31
41151e77 32unsigned int HPAGE_SHIFT;
ec4b2c0c 33
41151e77
BB
34/*
35 * Tracks gpages after the device tree is scanned and before the
a6146888
BB
36 * huge_boot_pages list is ready. On non-Freescale implementations, this is
37 * just used to track 16G pages and so is a single array. FSL-based
38 * implementations may have more than one gpage size, so we need multiple
39 * arrays
41151e77 40 */
881fde1d 41#ifdef CONFIG_PPC_FSL_BOOK3E
41151e77
BB
42#define MAX_NUMBER_GPAGES 128
43struct psize_gpages {
44 u64 gpage_list[MAX_NUMBER_GPAGES];
45 unsigned int nr_gpages;
46};
47static struct psize_gpages gpage_freearray[MMU_PAGE_COUNT];
881fde1d
BB
48#else
49#define MAX_NUMBER_GPAGES 1024
50static u64 gpage_freearray[MAX_NUMBER_GPAGES];
51static unsigned nr_gpages;
41151e77 52#endif
f10a04c0 53
a4fe3ce7
DG
54#define hugepd_none(hpd) ((hpd).pd == 0)
55
e2b3d202
AK
56#ifdef CONFIG_PPC_BOOK3S_64
57/*
58 * At this point we do the placement change only for BOOK3S 64. This would
59 * possibly work on other subarchs.
60 */
61
62/*
63 * We have PGD_INDEX_SIZ = 12 and PTE_INDEX_SIZE = 8, so that we can have
64 * 16GB hugepage pte in PGD and 16MB hugepage pte at PMD;
06743521
AK
65 *
66 * Defined in such a way that we can optimize away code block at build time
67 * if CONFIG_HUGETLB_PAGE=n.
e2b3d202
AK
68 */
69int pmd_huge(pmd_t pmd)
70{
71 /*
72 * leaf pte for huge page, bottom two bits != 00
73 */
74 return ((pmd_val(pmd) & 0x3) != 0x0);
75}
76
77int pud_huge(pud_t pud)
78{
79 /*
80 * leaf pte for huge page, bottom two bits != 00
81 */
82 return ((pud_val(pud) & 0x3) != 0x0);
83}
84
85int pgd_huge(pgd_t pgd)
86{
87 /*
88 * leaf pte for huge page, bottom two bits != 00
89 */
90 return ((pgd_val(pgd) & 0x3) != 0x0);
91}
92#else
93int pmd_huge(pmd_t pmd)
94{
95 return 0;
96}
97
98int pud_huge(pud_t pud)
99{
100 return 0;
101}
102
103int pgd_huge(pgd_t pgd)
104{
105 return 0;
106}
107#endif
108
a4fe3ce7
DG
109pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)
110{
12bc9f6f 111 /* Only called for hugetlbfs pages, hence can ignore THP */
a4fe3ce7
DG
112 return find_linux_pte_or_hugepte(mm->pgd, addr, NULL);
113}
114
f10a04c0 115static int __hugepte_alloc(struct mm_struct *mm, hugepd_t *hpdp,
a4fe3ce7 116 unsigned long address, unsigned pdshift, unsigned pshift)
f10a04c0 117{
41151e77
BB
118 struct kmem_cache *cachep;
119 pte_t *new;
120
881fde1d 121#ifdef CONFIG_PPC_FSL_BOOK3E
41151e77
BB
122 int i;
123 int num_hugepd = 1 << (pshift - pdshift);
124 cachep = hugepte_cache;
881fde1d
BB
125#else
126 cachep = PGT_CACHE(pdshift - pshift);
41151e77
BB
127#endif
128
129 new = kmem_cache_zalloc(cachep, GFP_KERNEL|__GFP_REPEAT);
f10a04c0 130
a4fe3ce7
DG
131 BUG_ON(pshift > HUGEPD_SHIFT_MASK);
132 BUG_ON((unsigned long)new & HUGEPD_SHIFT_MASK);
133
f10a04c0
DG
134 if (! new)
135 return -ENOMEM;
136
137 spin_lock(&mm->page_table_lock);
881fde1d 138#ifdef CONFIG_PPC_FSL_BOOK3E
41151e77
BB
139 /*
140 * We have multiple higher-level entries that point to the same
141 * actual pte location. Fill in each as we go and backtrack on error.
142 * We need all of these so the DTLB pgtable walk code can find the
143 * right higher-level entry without knowing if it's a hugepage or not.
144 */
145 for (i = 0; i < num_hugepd; i++, hpdp++) {
146 if (unlikely(!hugepd_none(*hpdp)))
147 break;
148 else
cf9427b8 149 /* We use the old format for PPC_FSL_BOOK3E */
41151e77
BB
150 hpdp->pd = ((unsigned long)new & ~PD_HUGE) | pshift;
151 }
152 /* If we bailed from the for loop early, an error occurred, clean up */
153 if (i < num_hugepd) {
154 for (i = i - 1 ; i >= 0; i--, hpdp--)
155 hpdp->pd = 0;
156 kmem_cache_free(cachep, new);
157 }
a1cd5419
BB
158#else
159 if (!hugepd_none(*hpdp))
160 kmem_cache_free(cachep, new);
cf9427b8
AK
161 else {
162#ifdef CONFIG_PPC_BOOK3S_64
163 hpdp->pd = (unsigned long)new |
164 (shift_to_mmu_psize(pshift) << 2);
165#else
a1cd5419 166 hpdp->pd = ((unsigned long)new & ~PD_HUGE) | pshift;
cf9427b8
AK
167#endif
168 }
41151e77 169#endif
f10a04c0
DG
170 spin_unlock(&mm->page_table_lock);
171 return 0;
172}
173
a1cd5419
BB
174/*
175 * These macros define how to determine which level of the page table holds
176 * the hpdp.
177 */
178#ifdef CONFIG_PPC_FSL_BOOK3E
179#define HUGEPD_PGD_SHIFT PGDIR_SHIFT
180#define HUGEPD_PUD_SHIFT PUD_SHIFT
181#else
182#define HUGEPD_PGD_SHIFT PUD_SHIFT
183#define HUGEPD_PUD_SHIFT PMD_SHIFT
184#endif
185
e2b3d202
AK
186#ifdef CONFIG_PPC_BOOK3S_64
187/*
188 * At this point we do the placement change only for BOOK3S 64. This would
189 * possibly work on other subarchs.
190 */
191pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr, unsigned long sz)
192{
193 pgd_t *pg;
194 pud_t *pu;
195 pmd_t *pm;
196 hugepd_t *hpdp = NULL;
197 unsigned pshift = __ffs(sz);
198 unsigned pdshift = PGDIR_SHIFT;
199
200 addr &= ~(sz-1);
201 pg = pgd_offset(mm, addr);
202
203 if (pshift == PGDIR_SHIFT)
204 /* 16GB huge page */
205 return (pte_t *) pg;
206 else if (pshift > PUD_SHIFT)
207 /*
208 * We need to use hugepd table
209 */
210 hpdp = (hugepd_t *)pg;
211 else {
212 pdshift = PUD_SHIFT;
213 pu = pud_alloc(mm, pg, addr);
214 if (pshift == PUD_SHIFT)
215 return (pte_t *)pu;
216 else if (pshift > PMD_SHIFT)
217 hpdp = (hugepd_t *)pu;
218 else {
219 pdshift = PMD_SHIFT;
220 pm = pmd_alloc(mm, pu, addr);
221 if (pshift == PMD_SHIFT)
222 /* 16MB hugepage */
223 return (pte_t *)pm;
224 else
225 hpdp = (hugepd_t *)pm;
226 }
227 }
228 if (!hpdp)
229 return NULL;
230
231 BUG_ON(!hugepd_none(*hpdp) && !hugepd_ok(*hpdp));
232
233 if (hugepd_none(*hpdp) && __hugepte_alloc(mm, hpdp, addr, pdshift, pshift))
234 return NULL;
235
b30e7590 236 return hugepte_offset(*hpdp, addr, pdshift);
e2b3d202
AK
237}
238
239#else
240
a4fe3ce7 241pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr, unsigned long sz)
0b26425c 242{
a4fe3ce7
DG
243 pgd_t *pg;
244 pud_t *pu;
245 pmd_t *pm;
246 hugepd_t *hpdp = NULL;
247 unsigned pshift = __ffs(sz);
248 unsigned pdshift = PGDIR_SHIFT;
249
250 addr &= ~(sz-1);
251
252 pg = pgd_offset(mm, addr);
a1cd5419
BB
253
254 if (pshift >= HUGEPD_PGD_SHIFT) {
a4fe3ce7
DG
255 hpdp = (hugepd_t *)pg;
256 } else {
257 pdshift = PUD_SHIFT;
258 pu = pud_alloc(mm, pg, addr);
a1cd5419 259 if (pshift >= HUGEPD_PUD_SHIFT) {
a4fe3ce7
DG
260 hpdp = (hugepd_t *)pu;
261 } else {
262 pdshift = PMD_SHIFT;
263 pm = pmd_alloc(mm, pu, addr);
264 hpdp = (hugepd_t *)pm;
265 }
266 }
267
268 if (!hpdp)
269 return NULL;
270
271 BUG_ON(!hugepd_none(*hpdp) && !hugepd_ok(*hpdp));
272
273 if (hugepd_none(*hpdp) && __hugepte_alloc(mm, hpdp, addr, pdshift, pshift))
274 return NULL;
275
b30e7590 276 return hugepte_offset(*hpdp, addr, pdshift);
4ec161cf 277}
e2b3d202 278#endif
4ec161cf 279
881fde1d 280#ifdef CONFIG_PPC_FSL_BOOK3E
658013e9 281/* Build list of addresses of gigantic pages. This function is used in early
14ed7409 282 * boot before the buddy allocator is setup.
658013e9 283 */
41151e77
BB
284void add_gpage(u64 addr, u64 page_size, unsigned long number_of_pages)
285{
286 unsigned int idx = shift_to_mmu_psize(__ffs(page_size));
287 int i;
288
289 if (addr == 0)
290 return;
291
292 gpage_freearray[idx].nr_gpages = number_of_pages;
293
294 for (i = 0; i < number_of_pages; i++) {
295 gpage_freearray[idx].gpage_list[i] = addr;
296 addr += page_size;
297 }
298}
299
300/*
301 * Moves the gigantic page addresses from the temporary list to the
302 * huge_boot_pages list.
303 */
304int alloc_bootmem_huge_page(struct hstate *hstate)
305{
306 struct huge_bootmem_page *m;
2415cf12 307 int idx = shift_to_mmu_psize(huge_page_shift(hstate));
41151e77
BB
308 int nr_gpages = gpage_freearray[idx].nr_gpages;
309
310 if (nr_gpages == 0)
311 return 0;
312
313#ifdef CONFIG_HIGHMEM
314 /*
315 * If gpages can be in highmem we can't use the trick of storing the
316 * data structure in the page; allocate space for this
317 */
318 m = alloc_bootmem(sizeof(struct huge_bootmem_page));
319 m->phys = gpage_freearray[idx].gpage_list[--nr_gpages];
320#else
321 m = phys_to_virt(gpage_freearray[idx].gpage_list[--nr_gpages]);
322#endif
323
324 list_add(&m->list, &huge_boot_pages);
325 gpage_freearray[idx].nr_gpages = nr_gpages;
326 gpage_freearray[idx].gpage_list[nr_gpages] = 0;
327 m->hstate = hstate;
328
329 return 1;
330}
331/*
332 * Scan the command line hugepagesz= options for gigantic pages; store those in
333 * a list that we use to allocate the memory once all options are parsed.
334 */
335
336unsigned long gpage_npages[MMU_PAGE_COUNT];
337
89528127
PG
338static int __init do_gpage_early_setup(char *param, char *val,
339 const char *unused)
41151e77
BB
340{
341 static phys_addr_t size;
342 unsigned long npages;
343
344 /*
345 * The hugepagesz and hugepages cmdline options are interleaved. We
346 * use the size variable to keep track of whether or not this was done
347 * properly and skip over instances where it is incorrect. Other
348 * command-line parsing code will issue warnings, so we don't need to.
349 *
350 */
351 if ((strcmp(param, "default_hugepagesz") == 0) ||
352 (strcmp(param, "hugepagesz") == 0)) {
353 size = memparse(val, NULL);
354 } else if (strcmp(param, "hugepages") == 0) {
355 if (size != 0) {
356 if (sscanf(val, "%lu", &npages) <= 0)
357 npages = 0;
358 gpage_npages[shift_to_mmu_psize(__ffs(size))] = npages;
359 size = 0;
360 }
361 }
362 return 0;
363}
364
365
366/*
367 * This function allocates physical space for pages that are larger than the
368 * buddy allocator can handle. We want to allocate these in highmem because
369 * the amount of lowmem is limited. This means that this function MUST be
370 * called before lowmem_end_addr is set up in MMU_init() in order for the lmb
371 * allocate to grab highmem.
372 */
373void __init reserve_hugetlb_gpages(void)
374{
375 static __initdata char cmdline[COMMAND_LINE_SIZE];
376 phys_addr_t size, base;
377 int i;
378
379 strlcpy(cmdline, boot_command_line, COMMAND_LINE_SIZE);
026cee00
PM
380 parse_args("hugetlb gpages", cmdline, NULL, 0, 0, 0,
381 &do_gpage_early_setup);
41151e77
BB
382
383 /*
384 * Walk gpage list in reverse, allocating larger page sizes first.
385 * Skip over unsupported sizes, or sizes that have 0 gpages allocated.
386 * When we reach the point in the list where pages are no longer
387 * considered gpages, we're done.
388 */
389 for (i = MMU_PAGE_COUNT-1; i >= 0; i--) {
390 if (mmu_psize_defs[i].shift == 0 || gpage_npages[i] == 0)
391 continue;
392 else if (mmu_psize_to_shift(i) < (MAX_ORDER + PAGE_SHIFT))
393 break;
394
395 size = (phys_addr_t)(1ULL << mmu_psize_to_shift(i));
396 base = memblock_alloc_base(size * gpage_npages[i], size,
397 MEMBLOCK_ALLOC_ANYWHERE);
398 add_gpage(base, size, gpage_npages[i]);
399 }
400}
401
881fde1d 402#else /* !PPC_FSL_BOOK3E */
41151e77
BB
403
404/* Build list of addresses of gigantic pages. This function is used in early
14ed7409 405 * boot before the buddy allocator is setup.
41151e77
BB
406 */
407void add_gpage(u64 addr, u64 page_size, unsigned long number_of_pages)
658013e9
JT
408{
409 if (!addr)
410 return;
411 while (number_of_pages > 0) {
412 gpage_freearray[nr_gpages] = addr;
413 nr_gpages++;
414 number_of_pages--;
415 addr += page_size;
416 }
417}
418
ec4b2c0c 419/* Moves the gigantic page addresses from the temporary list to the
0d9ea754
JT
420 * huge_boot_pages list.
421 */
422int alloc_bootmem_huge_page(struct hstate *hstate)
ec4b2c0c
JT
423{
424 struct huge_bootmem_page *m;
425 if (nr_gpages == 0)
426 return 0;
427 m = phys_to_virt(gpage_freearray[--nr_gpages]);
428 gpage_freearray[nr_gpages] = 0;
429 list_add(&m->list, &huge_boot_pages);
0d9ea754 430 m->hstate = hstate;
ec4b2c0c
JT
431 return 1;
432}
41151e77 433#endif
ec4b2c0c 434
39dde65c
CK
435int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep)
436{
437 return 0;
438}
439
881fde1d 440#ifdef CONFIG_PPC_FSL_BOOK3E
41151e77
BB
441#define HUGEPD_FREELIST_SIZE \
442 ((PAGE_SIZE - sizeof(struct hugepd_freelist)) / sizeof(pte_t))
443
444struct hugepd_freelist {
445 struct rcu_head rcu;
446 unsigned int index;
447 void *ptes[0];
448};
449
450static DEFINE_PER_CPU(struct hugepd_freelist *, hugepd_freelist_cur);
451
452static void hugepd_free_rcu_callback(struct rcu_head *head)
453{
454 struct hugepd_freelist *batch =
455 container_of(head, struct hugepd_freelist, rcu);
456 unsigned int i;
457
458 for (i = 0; i < batch->index; i++)
459 kmem_cache_free(hugepte_cache, batch->ptes[i]);
460
461 free_page((unsigned long)batch);
462}
463
464static void hugepd_free(struct mmu_gather *tlb, void *hugepte)
465{
466 struct hugepd_freelist **batchp;
467
69111bac 468 batchp = this_cpu_ptr(&hugepd_freelist_cur);
41151e77
BB
469
470 if (atomic_read(&tlb->mm->mm_users) < 2 ||
471 cpumask_equal(mm_cpumask(tlb->mm),
472 cpumask_of(smp_processor_id()))) {
473 kmem_cache_free(hugepte_cache, hugepte);
94b09d75 474 put_cpu_var(hugepd_freelist_cur);
41151e77
BB
475 return;
476 }
477
478 if (*batchp == NULL) {
479 *batchp = (struct hugepd_freelist *)__get_free_page(GFP_ATOMIC);
480 (*batchp)->index = 0;
481 }
482
483 (*batchp)->ptes[(*batchp)->index++] = hugepte;
484 if ((*batchp)->index == HUGEPD_FREELIST_SIZE) {
485 call_rcu_sched(&(*batchp)->rcu, hugepd_free_rcu_callback);
486 *batchp = NULL;
487 }
94b09d75 488 put_cpu_var(hugepd_freelist_cur);
41151e77
BB
489}
490#endif
491
a4fe3ce7
DG
492static void free_hugepd_range(struct mmu_gather *tlb, hugepd_t *hpdp, int pdshift,
493 unsigned long start, unsigned long end,
494 unsigned long floor, unsigned long ceiling)
f10a04c0
DG
495{
496 pte_t *hugepte = hugepd_page(*hpdp);
41151e77
BB
497 int i;
498
a4fe3ce7 499 unsigned long pdmask = ~((1UL << pdshift) - 1);
41151e77
BB
500 unsigned int num_hugepd = 1;
501
881fde1d
BB
502#ifdef CONFIG_PPC_FSL_BOOK3E
503 /* Note: On fsl the hpdp may be the first of several */
41151e77 504 num_hugepd = (1 << (hugepd_shift(*hpdp) - pdshift));
881fde1d
BB
505#else
506 unsigned int shift = hugepd_shift(*hpdp);
41151e77 507#endif
a4fe3ce7
DG
508
509 start &= pdmask;
510 if (start < floor)
511 return;
512 if (ceiling) {
513 ceiling &= pdmask;
514 if (! ceiling)
515 return;
516 }
517 if (end - 1 > ceiling - 1)
518 return;
f10a04c0 519
41151e77
BB
520 for (i = 0; i < num_hugepd; i++, hpdp++)
521 hpdp->pd = 0;
522
f10a04c0 523 tlb->need_flush = 1;
881fde1d
BB
524
525#ifdef CONFIG_PPC_FSL_BOOK3E
41151e77 526 hugepd_free(tlb, hugepte);
881fde1d
BB
527#else
528 pgtable_free_tlb(tlb, hugepte, pdshift - shift);
41151e77 529#endif
f10a04c0
DG
530}
531
f10a04c0
DG
532static void hugetlb_free_pmd_range(struct mmu_gather *tlb, pud_t *pud,
533 unsigned long addr, unsigned long end,
a4fe3ce7 534 unsigned long floor, unsigned long ceiling)
f10a04c0
DG
535{
536 pmd_t *pmd;
537 unsigned long next;
538 unsigned long start;
539
540 start = addr;
f10a04c0 541 do {
a1cd5419 542 pmd = pmd_offset(pud, addr);
f10a04c0 543 next = pmd_addr_end(addr, end);
b30e7590 544 if (!is_hugepd(__hugepd(pmd_val(*pmd)))) {
8bbd9f04
AK
545 /*
546 * if it is not hugepd pointer, we should already find
547 * it cleared.
548 */
549 WARN_ON(!pmd_none_or_clear_bad(pmd));
f10a04c0 550 continue;
8bbd9f04 551 }
a1cd5419
BB
552#ifdef CONFIG_PPC_FSL_BOOK3E
553 /*
554 * Increment next by the size of the huge mapping since
555 * there may be more than one entry at this level for a
556 * single hugepage, but all of them point to
557 * the same kmem cache that holds the hugepte.
558 */
559 next = addr + (1 << hugepd_shift(*(hugepd_t *)pmd));
560#endif
a4fe3ce7
DG
561 free_hugepd_range(tlb, (hugepd_t *)pmd, PMD_SHIFT,
562 addr, next, floor, ceiling);
a1cd5419 563 } while (addr = next, addr != end);
f10a04c0
DG
564
565 start &= PUD_MASK;
566 if (start < floor)
567 return;
568 if (ceiling) {
569 ceiling &= PUD_MASK;
570 if (!ceiling)
571 return;
1da177e4 572 }
f10a04c0
DG
573 if (end - 1 > ceiling - 1)
574 return;
1da177e4 575
f10a04c0
DG
576 pmd = pmd_offset(pud, start);
577 pud_clear(pud);
9e1b32ca 578 pmd_free_tlb(tlb, pmd, start);
f10a04c0 579}
f10a04c0
DG
580
581static void hugetlb_free_pud_range(struct mmu_gather *tlb, pgd_t *pgd,
582 unsigned long addr, unsigned long end,
583 unsigned long floor, unsigned long ceiling)
584{
585 pud_t *pud;
586 unsigned long next;
587 unsigned long start;
588
589 start = addr;
f10a04c0 590 do {
a1cd5419 591 pud = pud_offset(pgd, addr);
f10a04c0 592 next = pud_addr_end(addr, end);
b30e7590 593 if (!is_hugepd(__hugepd(pud_val(*pud)))) {
4ec161cf
JT
594 if (pud_none_or_clear_bad(pud))
595 continue;
0d9ea754 596 hugetlb_free_pmd_range(tlb, pud, addr, next, floor,
a4fe3ce7 597 ceiling);
4ec161cf 598 } else {
a1cd5419
BB
599#ifdef CONFIG_PPC_FSL_BOOK3E
600 /*
601 * Increment next by the size of the huge mapping since
602 * there may be more than one entry at this level for a
603 * single hugepage, but all of them point to
604 * the same kmem cache that holds the hugepte.
605 */
606 next = addr + (1 << hugepd_shift(*(hugepd_t *)pud));
607#endif
a4fe3ce7
DG
608 free_hugepd_range(tlb, (hugepd_t *)pud, PUD_SHIFT,
609 addr, next, floor, ceiling);
4ec161cf 610 }
a1cd5419 611 } while (addr = next, addr != end);
f10a04c0
DG
612
613 start &= PGDIR_MASK;
614 if (start < floor)
615 return;
616 if (ceiling) {
617 ceiling &= PGDIR_MASK;
618 if (!ceiling)
619 return;
620 }
621 if (end - 1 > ceiling - 1)
622 return;
623
624 pud = pud_offset(pgd, start);
625 pgd_clear(pgd);
9e1b32ca 626 pud_free_tlb(tlb, pud, start);
f10a04c0
DG
627}
628
629/*
630 * This function frees user-level page tables of a process.
f10a04c0 631 */
42b77728 632void hugetlb_free_pgd_range(struct mmu_gather *tlb,
f10a04c0
DG
633 unsigned long addr, unsigned long end,
634 unsigned long floor, unsigned long ceiling)
635{
636 pgd_t *pgd;
637 unsigned long next;
f10a04c0
DG
638
639 /*
a4fe3ce7
DG
640 * Because there are a number of different possible pagetable
641 * layouts for hugepage ranges, we limit knowledge of how
642 * things should be laid out to the allocation path
643 * (huge_pte_alloc(), above). Everything else works out the
644 * structure as it goes from information in the hugepd
645 * pointers. That means that we can't here use the
646 * optimization used in the normal page free_pgd_range(), of
647 * checking whether we're actually covering a large enough
648 * range to have to do anything at the top level of the walk
649 * instead of at the bottom.
f10a04c0 650 *
a4fe3ce7
DG
651 * To make sense of this, you should probably go read the big
652 * block comment at the top of the normal free_pgd_range(),
653 * too.
f10a04c0 654 */
f10a04c0 655
f10a04c0 656 do {
f10a04c0 657 next = pgd_addr_end(addr, end);
41151e77 658 pgd = pgd_offset(tlb->mm, addr);
b30e7590 659 if (!is_hugepd(__hugepd(pgd_val(*pgd)))) {
0b26425c
DG
660 if (pgd_none_or_clear_bad(pgd))
661 continue;
662 hugetlb_free_pud_range(tlb, pgd, addr, next, floor, ceiling);
663 } else {
881fde1d 664#ifdef CONFIG_PPC_FSL_BOOK3E
41151e77
BB
665 /*
666 * Increment next by the size of the huge mapping since
881fde1d
BB
667 * there may be more than one entry at the pgd level
668 * for a single hugepage, but all of them point to the
669 * same kmem cache that holds the hugepte.
41151e77
BB
670 */
671 next = addr + (1 << hugepd_shift(*(hugepd_t *)pgd));
672#endif
a4fe3ce7
DG
673 free_hugepd_range(tlb, (hugepd_t *)pgd, PGDIR_SHIFT,
674 addr, next, floor, ceiling);
0b26425c 675 }
41151e77 676 } while (addr = next, addr != end);
1da177e4
LT
677}
678
1da177e4
LT
679struct page *
680follow_huge_addr(struct mm_struct *mm, unsigned long address, int write)
681{
682 pte_t *ptep;
683 struct page *page;
a4fe3ce7
DG
684 unsigned shift;
685 unsigned long mask;
12bc9f6f
AK
686 /*
687 * Transparent hugepages are handled by generic code. We can skip them
688 * here.
689 */
a4fe3ce7 690 ptep = find_linux_pte_or_hugepte(mm->pgd, address, &shift);
1da177e4 691
0d9ea754 692 /* Verify it is a huge page else bail. */
12bc9f6f 693 if (!ptep || !shift || pmd_trans_huge(*(pmd_t *)ptep))
1da177e4
LT
694 return ERR_PTR(-EINVAL);
695
a4fe3ce7 696 mask = (1UL << shift) - 1;
1da177e4 697 page = pte_page(*ptep);
a4fe3ce7
DG
698 if (page)
699 page += (address & mask) / PAGE_SIZE;
1da177e4
LT
700
701 return page;
702}
703
1da177e4
LT
704struct page *
705follow_huge_pmd(struct mm_struct *mm, unsigned long address,
706 pmd_t *pmd, int write)
707{
708 BUG();
709 return NULL;
710}
711
39adfa54
DG
712static unsigned long hugepte_addr_end(unsigned long addr, unsigned long end,
713 unsigned long sz)
714{
715 unsigned long __boundary = (addr + sz) & ~(sz-1);
716 return (__boundary - 1 < end - 1) ? __boundary : end;
717}
718
b30e7590
AK
719int gup_huge_pd(hugepd_t hugepd, unsigned long addr, unsigned pdshift,
720 unsigned long end, int write, struct page **pages, int *nr)
a4fe3ce7
DG
721{
722 pte_t *ptep;
b30e7590 723 unsigned long sz = 1UL << hugepd_shift(hugepd);
39adfa54 724 unsigned long next;
a4fe3ce7
DG
725
726 ptep = hugepte_offset(hugepd, addr, pdshift);
727 do {
39adfa54 728 next = hugepte_addr_end(addr, end, sz);
a4fe3ce7
DG
729 if (!gup_hugepte(ptep, sz, addr, end, write, pages, nr))
730 return 0;
39adfa54 731 } while (ptep++, addr = next, addr != end);
a4fe3ce7
DG
732
733 return 1;
734}
1da177e4 735
76512959 736#ifdef CONFIG_PPC_MM_SLICES
1da177e4
LT
737unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
738 unsigned long len, unsigned long pgoff,
739 unsigned long flags)
740{
0d9ea754
JT
741 struct hstate *hstate = hstate_file(file);
742 int mmu_psize = shift_to_mmu_psize(huge_page_shift(hstate));
48f797de 743
34d07177 744 return slice_get_unmapped_area(addr, len, flags, mmu_psize, 1);
1da177e4 745}
76512959 746#endif
1da177e4 747
3340289d
MG
748unsigned long vma_mmu_pagesize(struct vm_area_struct *vma)
749{
25c29f9e 750#ifdef CONFIG_PPC_MM_SLICES
3340289d
MG
751 unsigned int psize = get_slice_psize(vma->vm_mm, vma->vm_start);
752
753 return 1UL << mmu_psize_to_shift(psize);
41151e77
BB
754#else
755 if (!is_vm_hugetlb_page(vma))
756 return PAGE_SIZE;
757
758 return huge_page_size(hstate_vma(vma));
759#endif
760}
761
762static inline bool is_power_of_4(unsigned long x)
763{
764 if (is_power_of_2(x))
765 return (__ilog2(x) % 2) ? false : true;
766 return false;
3340289d
MG
767}
768
d1837cba 769static int __init add_huge_page_size(unsigned long long size)
4ec161cf 770{
d1837cba
DG
771 int shift = __ffs(size);
772 int mmu_psize;
a4fe3ce7 773
4ec161cf 774 /* Check that it is a page size supported by the hardware and
d1837cba 775 * that it fits within pagetable and slice limits. */
41151e77
BB
776#ifdef CONFIG_PPC_FSL_BOOK3E
777 if ((size < PAGE_SIZE) || !is_power_of_4(size))
778 return -EINVAL;
779#else
d1837cba
DG
780 if (!is_power_of_2(size)
781 || (shift > SLICE_HIGH_SHIFT) || (shift <= PAGE_SHIFT))
782 return -EINVAL;
41151e77 783#endif
91224346 784
d1837cba
DG
785 if ((mmu_psize = shift_to_mmu_psize(shift)) < 0)
786 return -EINVAL;
787
788#ifdef CONFIG_SPU_FS_64K_LS
789 /* Disable support for 64K huge pages when 64K SPU local store
790 * support is enabled as the current implementation conflicts.
791 */
792 if (shift == PAGE_SHIFT_64K)
793 return -EINVAL;
794#endif /* CONFIG_SPU_FS_64K_LS */
795
796 BUG_ON(mmu_psize_defs[mmu_psize].shift != shift);
797
798 /* Return if huge page size has already been setup */
799 if (size_to_hstate(size))
800 return 0;
801
802 hugetlb_add_hstate(shift - PAGE_SHIFT);
803
804 return 0;
4ec161cf
JT
805}
806
807static int __init hugepage_setup_sz(char *str)
808{
809 unsigned long long size;
4ec161cf
JT
810
811 size = memparse(str, &str);
812
d1837cba 813 if (add_huge_page_size(size) != 0)
4ec161cf
JT
814 printk(KERN_WARNING "Invalid huge page size specified(%llu)\n", size);
815
816 return 1;
817}
818__setup("hugepagesz=", hugepage_setup_sz);
819
881fde1d 820#ifdef CONFIG_PPC_FSL_BOOK3E
41151e77
BB
821struct kmem_cache *hugepte_cache;
822static int __init hugetlbpage_init(void)
823{
824 int psize;
825
826 for (psize = 0; psize < MMU_PAGE_COUNT; ++psize) {
827 unsigned shift;
828
829 if (!mmu_psize_defs[psize].shift)
830 continue;
831
832 shift = mmu_psize_to_shift(psize);
833
834 /* Don't treat normal page sizes as huge... */
835 if (shift != PAGE_SHIFT)
836 if (add_huge_page_size(1ULL << shift) < 0)
837 continue;
838 }
839
840 /*
841 * Create a kmem cache for hugeptes. The bottom bits in the pte have
842 * size information encoded in them, so align them to allow this
843 */
844 hugepte_cache = kmem_cache_create("hugepte-cache", sizeof(pte_t),
845 HUGEPD_SHIFT_MASK + 1, 0, NULL);
846 if (hugepte_cache == NULL)
847 panic("%s: Unable to create kmem cache for hugeptes\n",
848 __func__);
849
850 /* Default hpage size = 4M */
851 if (mmu_psize_defs[MMU_PAGE_4M].shift)
852 HPAGE_SHIFT = mmu_psize_defs[MMU_PAGE_4M].shift;
853 else
854 panic("%s: Unable to set default huge page size\n", __func__);
855
856
857 return 0;
858}
859#else
f10a04c0
DG
860static int __init hugetlbpage_init(void)
861{
a4fe3ce7 862 int psize;
0d9ea754 863
44ae3ab3 864 if (!mmu_has_feature(MMU_FTR_16M_PAGE))
f10a04c0 865 return -ENODEV;
00df438e 866
d1837cba
DG
867 for (psize = 0; psize < MMU_PAGE_COUNT; ++psize) {
868 unsigned shift;
869 unsigned pdshift;
0d9ea754 870
d1837cba
DG
871 if (!mmu_psize_defs[psize].shift)
872 continue;
00df438e 873
d1837cba
DG
874 shift = mmu_psize_to_shift(psize);
875
876 if (add_huge_page_size(1ULL << shift) < 0)
877 continue;
878
879 if (shift < PMD_SHIFT)
880 pdshift = PMD_SHIFT;
881 else if (shift < PUD_SHIFT)
882 pdshift = PUD_SHIFT;
883 else
884 pdshift = PGDIR_SHIFT;
e2b3d202
AK
885 /*
886 * if we have pdshift and shift value same, we don't
887 * use pgt cache for hugepd.
888 */
889 if (pdshift != shift) {
890 pgtable_cache_add(pdshift - shift, NULL);
891 if (!PGT_CACHE(pdshift - shift))
892 panic("hugetlbpage_init(): could not create "
893 "pgtable cache for %d bit pagesize\n", shift);
894 }
0d9ea754 895 }
f10a04c0 896
d1837cba
DG
897 /* Set default large page size. Currently, we pick 16M or 1M
898 * depending on what is available
899 */
900 if (mmu_psize_defs[MMU_PAGE_16M].shift)
901 HPAGE_SHIFT = mmu_psize_defs[MMU_PAGE_16M].shift;
902 else if (mmu_psize_defs[MMU_PAGE_1M].shift)
903 HPAGE_SHIFT = mmu_psize_defs[MMU_PAGE_1M].shift;
904
f10a04c0
DG
905 return 0;
906}
41151e77 907#endif
f10a04c0 908module_init(hugetlbpage_init);
0895ecda
DG
909
910void flush_dcache_icache_hugepage(struct page *page)
911{
912 int i;
41151e77 913 void *start;
0895ecda
DG
914
915 BUG_ON(!PageCompound(page));
916
41151e77
BB
917 for (i = 0; i < (1UL << compound_order(page)); i++) {
918 if (!PageHighMem(page)) {
919 __flush_dcache_icache(page_address(page+i));
920 } else {
2480b208 921 start = kmap_atomic(page+i);
41151e77 922 __flush_dcache_icache(start);
2480b208 923 kunmap_atomic(start);
41151e77
BB
924 }
925 }
0895ecda 926}
29409997
AK
927
928#endif /* CONFIG_HUGETLB_PAGE */
929
930/*
931 * We have 4 cases for pgds and pmds:
932 * (1) invalid (all zeroes)
933 * (2) pointer to next table, as normal; bottom 6 bits == 0
934 * (3) leaf pte for huge page, bottom two bits != 00
935 * (4) hugepd pointer, bottom two bits == 00, next 4 bits indicate size of table
0ac52dd7
AK
936 *
937 * So long as we atomically load page table pointers we are safe against teardown,
938 * we can follow the address down to the the page and take a ref on it.
29409997 939 */
0ac52dd7 940
29409997
AK
941pte_t *find_linux_pte_or_hugepte(pgd_t *pgdir, unsigned long ea, unsigned *shift)
942{
0ac52dd7
AK
943 pgd_t pgd, *pgdp;
944 pud_t pud, *pudp;
945 pmd_t pmd, *pmdp;
29409997
AK
946 pte_t *ret_pte;
947 hugepd_t *hpdp = NULL;
948 unsigned pdshift = PGDIR_SHIFT;
949
950 if (shift)
951 *shift = 0;
952
0ac52dd7
AK
953 pgdp = pgdir + pgd_index(ea);
954 pgd = ACCESS_ONCE(*pgdp);
ac52ae47 955 /*
0ac52dd7
AK
956 * Always operate on the local stack value. This make sure the
957 * value don't get updated by a parallel THP split/collapse,
958 * page fault or a page unmap. The return pte_t * is still not
959 * stable. So should be checked there for above conditions.
ac52ae47 960 */
0ac52dd7 961 if (pgd_none(pgd))
ac52ae47 962 return NULL;
0ac52dd7
AK
963 else if (pgd_huge(pgd)) {
964 ret_pte = (pte_t *) pgdp;
29409997 965 goto out;
b30e7590 966 } else if (is_hugepd(__hugepd(pgd_val(pgd))))
0ac52dd7 967 hpdp = (hugepd_t *)&pgd;
ac52ae47 968 else {
0ac52dd7
AK
969 /*
970 * Even if we end up with an unmap, the pgtable will not
971 * be freed, because we do an rcu free and here we are
972 * irq disabled
973 */
29409997 974 pdshift = PUD_SHIFT;
0ac52dd7
AK
975 pudp = pud_offset(&pgd, ea);
976 pud = ACCESS_ONCE(*pudp);
29409997 977
0ac52dd7 978 if (pud_none(pud))
ac52ae47 979 return NULL;
0ac52dd7
AK
980 else if (pud_huge(pud)) {
981 ret_pte = (pte_t *) pudp;
29409997 982 goto out;
b30e7590 983 } else if (is_hugepd(__hugepd(pud_val(pud))))
0ac52dd7 984 hpdp = (hugepd_t *)&pud;
ac52ae47 985 else {
29409997 986 pdshift = PMD_SHIFT;
0ac52dd7
AK
987 pmdp = pmd_offset(&pud, ea);
988 pmd = ACCESS_ONCE(*pmdp);
ac52ae47
AK
989 /*
990 * A hugepage collapse is captured by pmd_none, because
991 * it mark the pmd none and do a hpte invalidate.
992 *
993 * A hugepage split is captured by pmd_trans_splitting
994 * because we mark the pmd trans splitting and do a
995 * hpte invalidate
996 *
997 */
0ac52dd7 998 if (pmd_none(pmd) || pmd_trans_splitting(pmd))
ac52ae47 999 return NULL;
29409997 1000
0ac52dd7
AK
1001 if (pmd_huge(pmd) || pmd_large(pmd)) {
1002 ret_pte = (pte_t *) pmdp;
29409997 1003 goto out;
b30e7590 1004 } else if (is_hugepd(__hugepd(pmd_val(pmd))))
0ac52dd7 1005 hpdp = (hugepd_t *)&pmd;
ac52ae47 1006 else
0ac52dd7 1007 return pte_offset_kernel(&pmd, ea);
29409997
AK
1008 }
1009 }
1010 if (!hpdp)
1011 return NULL;
1012
b30e7590 1013 ret_pte = hugepte_offset(*hpdp, ea, pdshift);
29409997
AK
1014 pdshift = hugepd_shift(*hpdp);
1015out:
1016 if (shift)
1017 *shift = pdshift;
1018 return ret_pte;
1019}
1020EXPORT_SYMBOL_GPL(find_linux_pte_or_hugepte);
1021
1022int gup_hugepte(pte_t *ptep, unsigned long sz, unsigned long addr,
1023 unsigned long end, int write, struct page **pages, int *nr)
1024{
1025 unsigned long mask;
1026 unsigned long pte_end;
1027 struct page *head, *page, *tail;
1028 pte_t pte;
1029 int refs;
1030
1031 pte_end = (addr + sz) & ~(sz-1);
1032 if (pte_end < end)
1033 end = pte_end;
1034
7888b4dd 1035 pte = ACCESS_ONCE(*ptep);
29409997
AK
1036 mask = _PAGE_PRESENT | _PAGE_USER;
1037 if (write)
1038 mask |= _PAGE_RW;
1039
1040 if ((pte_val(pte) & mask) != mask)
1041 return 0;
1042
1043 /* hugepages are never "special" */
1044 VM_BUG_ON(!pfn_valid(pte_pfn(pte)));
1045
1046 refs = 0;
1047 head = pte_page(pte);
1048
1049 page = head + ((addr & (sz-1)) >> PAGE_SHIFT);
1050 tail = page;
1051 do {
1052 VM_BUG_ON(compound_head(page) != head);
1053 pages[*nr] = page;
1054 (*nr)++;
1055 page++;
1056 refs++;
1057 } while (addr += PAGE_SIZE, addr != end);
1058
1059 if (!page_cache_add_speculative(head, refs)) {
1060 *nr -= refs;
1061 return 0;
1062 }
1063
1064 if (unlikely(pte_val(pte) != pte_val(*ptep))) {
1065 /* Could be optimized better */
1066 *nr -= refs;
1067 while (refs--)
1068 put_page(head);
1069 return 0;
1070 }
1071
1072 /*
1073 * Any tail page need their mapcount reference taken before we
1074 * return.
1075 */
1076 while (refs--) {
1077 if (PageTail(tail))
1078 get_huge_page_tail(tail);
1079 tail++;
1080 }
1081
1082 return 1;
1083}
This page took 0.843449 seconds and 5 git commands to generate.