2 * linux/arch/x86_64/mm/init.c
4 * Copyright (C) 1995 Linus Torvalds
5 * Copyright (C) 2000 Pavel Machek <pavel@suse.cz>
6 * Copyright (C) 2002,2003 Andi Kleen <ak@suse.de>
9 #include <linux/signal.h>
10 #include <linux/sched.h>
11 #include <linux/kernel.h>
12 #include <linux/errno.h>
13 #include <linux/string.h>
14 #include <linux/types.h>
15 #include <linux/ptrace.h>
16 #include <linux/mman.h>
18 #include <linux/swap.h>
19 #include <linux/smp.h>
20 #include <linux/init.h>
21 #include <linux/pagemap.h>
22 #include <linux/bootmem.h>
23 #include <linux/proc_fs.h>
24 #include <linux/pci.h>
25 #include <linux/pfn.h>
26 #include <linux/poison.h>
27 #include <linux/dma-mapping.h>
28 #include <linux/module.h>
29 #include <linux/memory_hotplug.h>
30 #include <linux/nmi.h>
32 #include <asm/processor.h>
33 #include <asm/system.h>
34 #include <asm/uaccess.h>
35 #include <asm/pgtable.h>
36 #include <asm/pgalloc.h>
38 #include <asm/fixmap.h>
42 #include <asm/mmu_context.h>
43 #include <asm/proto.h>
45 #include <asm/sections.h>
46 #include <asm/kdebug.h>
48 #include <asm/cacheflush.h>
50 static unsigned long dma_reserve __initdata
;
52 DEFINE_PER_CPU(struct mmu_gather
, mmu_gathers
);
54 int direct_gbpages __meminitdata
55 #ifdef CONFIG_DIRECT_GBPAGES
60 static int __init
parse_direct_gbpages_off(char *arg
)
65 early_param("nogbpages", parse_direct_gbpages_off
);
67 static int __init
parse_direct_gbpages_on(char *arg
)
72 early_param("gbpages", parse_direct_gbpages_on
);
75 * NOTE: pagetable_init alloc all the fixmap pagetables contiguous on the
76 * physical space so we can cache the place of the first one and move
77 * around without checking the pgd every time.
82 long i
, total
= 0, reserved
= 0;
83 long shared
= 0, cached
= 0;
87 printk(KERN_INFO
"Mem-info:\n");
89 for_each_online_pgdat(pgdat
) {
90 for (i
= 0; i
< pgdat
->node_spanned_pages
; ++i
) {
92 * This loop can take a while with 256 GB and
93 * 4k pages so defer the NMI watchdog:
95 if (unlikely(i
% MAX_ORDER_NR_PAGES
== 0))
98 if (!pfn_valid(pgdat
->node_start_pfn
+ i
))
101 page
= pfn_to_page(pgdat
->node_start_pfn
+ i
);
103 if (PageReserved(page
))
105 else if (PageSwapCache(page
))
107 else if (page_count(page
))
108 shared
+= page_count(page
) - 1;
111 printk(KERN_INFO
"%lu pages of RAM\n", total
);
112 printk(KERN_INFO
"%lu reserved pages\n", reserved
);
113 printk(KERN_INFO
"%lu pages shared\n", shared
);
114 printk(KERN_INFO
"%lu pages swap cached\n", cached
);
119 static __init
void *spp_getpage(void)
124 ptr
= (void *) get_zeroed_page(GFP_ATOMIC
);
126 ptr
= alloc_bootmem_pages(PAGE_SIZE
);
128 if (!ptr
|| ((unsigned long)ptr
& ~PAGE_MASK
)) {
129 panic("set_pte_phys: cannot allocate page data %s\n",
130 after_bootmem
? "after bootmem" : "");
133 pr_debug("spp_getpage %p\n", ptr
);
139 set_pte_phys(unsigned long vaddr
, unsigned long phys
, pgprot_t prot
)
146 pr_debug("set_pte_phys %lx to %lx\n", vaddr
, phys
);
148 pgd
= pgd_offset_k(vaddr
);
149 if (pgd_none(*pgd
)) {
151 "PGD FIXMAP MISSING, it should be setup in head.S!\n");
154 pud
= pud_offset(pgd
, vaddr
);
155 if (pud_none(*pud
)) {
156 pmd
= (pmd_t
*) spp_getpage();
157 set_pud(pud
, __pud(__pa(pmd
) | _KERNPG_TABLE
| _PAGE_USER
));
158 if (pmd
!= pmd_offset(pud
, 0)) {
159 printk(KERN_ERR
"PAGETABLE BUG #01! %p <-> %p\n",
160 pmd
, pmd_offset(pud
, 0));
164 pmd
= pmd_offset(pud
, vaddr
);
165 if (pmd_none(*pmd
)) {
166 pte
= (pte_t
*) spp_getpage();
167 set_pmd(pmd
, __pmd(__pa(pte
) | _KERNPG_TABLE
| _PAGE_USER
));
168 if (pte
!= pte_offset_kernel(pmd
, 0)) {
169 printk(KERN_ERR
"PAGETABLE BUG #02!\n");
173 new_pte
= pfn_pte(phys
>> PAGE_SHIFT
, prot
);
175 pte
= pte_offset_kernel(pmd
, vaddr
);
176 if (!pte_none(*pte
) && pte_val(new_pte
) &&
177 pte_val(*pte
) != (pte_val(new_pte
) & __supported_pte_mask
))
179 set_pte(pte
, new_pte
);
182 * It's enough to flush this one mapping.
183 * (PGE mappings get flushed as well)
185 __flush_tlb_one(vaddr
);
189 * The head.S code sets up the kernel high mapping:
191 * from __START_KERNEL_map to __START_KERNEL_map + size (== _end-_text)
193 * phys_addr holds the negative offset to the kernel, which is added
194 * to the compile time generated pmds. This results in invalid pmds up
195 * to the point where we hit the physaddr 0 mapping.
197 * We limit the mappings to the region from _text to _end. _end is
198 * rounded up to the 2MB boundary. This catches the invalid pmds as
199 * well, as they are located before _text:
201 void __init
cleanup_highmap(void)
203 unsigned long vaddr
= __START_KERNEL_map
;
204 unsigned long end
= round_up((unsigned long)_end
, PMD_SIZE
) - 1;
205 pmd_t
*pmd
= level2_kernel_pgt
;
206 pmd_t
*last_pmd
= pmd
+ PTRS_PER_PMD
;
208 for (; pmd
< last_pmd
; pmd
++, vaddr
+= PMD_SIZE
) {
211 if (vaddr
< (unsigned long) _text
|| vaddr
> end
)
212 set_pmd(pmd
, __pmd(0));
216 static unsigned long __initdata table_start
;
217 static unsigned long __meminitdata table_end
;
219 static __meminit
void *alloc_low_page(unsigned long *phys
)
221 unsigned long pfn
= table_end
++;
225 adr
= (void *)get_zeroed_page(GFP_ATOMIC
);
232 panic("alloc_low_page: ran out of memory");
234 adr
= early_ioremap(pfn
* PAGE_SIZE
, PAGE_SIZE
);
235 memset(adr
, 0, PAGE_SIZE
);
236 *phys
= pfn
* PAGE_SIZE
;
240 static __meminit
void unmap_low_page(void *adr
)
245 early_iounmap(adr
, PAGE_SIZE
);
248 /* Must run before zap_low_mappings */
249 __meminit
void *early_ioremap(unsigned long addr
, unsigned long size
)
251 pmd_t
*pmd
, *last_pmd
;
255 pmds
= ((addr
& ~PMD_MASK
) + size
+ ~PMD_MASK
) / PMD_SIZE
;
256 vaddr
= __START_KERNEL_map
;
257 pmd
= level2_kernel_pgt
;
258 last_pmd
= level2_kernel_pgt
+ PTRS_PER_PMD
- 1;
260 for (; pmd
<= last_pmd
; pmd
++, vaddr
+= PMD_SIZE
) {
261 for (i
= 0; i
< pmds
; i
++) {
262 if (pmd_present(pmd
[i
]))
263 goto continue_outer_loop
;
265 vaddr
+= addr
& ~PMD_MASK
;
268 for (i
= 0; i
< pmds
; i
++, addr
+= PMD_SIZE
)
269 set_pmd(pmd
+i
, __pmd(addr
| __PAGE_KERNEL_LARGE_EXEC
));
272 return (void *)vaddr
;
276 printk(KERN_ERR
"early_ioremap(0x%lx, %lu) failed\n", addr
, size
);
282 * To avoid virtual aliases later:
284 __meminit
void early_iounmap(void *addr
, unsigned long size
)
290 vaddr
= (unsigned long)addr
;
291 pmds
= ((vaddr
& ~PMD_MASK
) + size
+ ~PMD_MASK
) / PMD_SIZE
;
292 pmd
= level2_kernel_pgt
+ pmd_index(vaddr
);
294 for (i
= 0; i
< pmds
; i
++)
300 static unsigned long __meminit
301 phys_pmd_init(pmd_t
*pmd_page
, unsigned long address
, unsigned long end
)
303 int i
= pmd_index(address
);
305 for (; i
< PTRS_PER_PMD
; i
++, address
+= PMD_SIZE
) {
306 pmd_t
*pmd
= pmd_page
+ pmd_index(address
);
308 if (address
>= end
) {
309 if (!after_bootmem
) {
310 for (; i
< PTRS_PER_PMD
; i
++, pmd
++)
311 set_pmd(pmd
, __pmd(0));
319 set_pte((pte_t
*)pmd
,
320 pfn_pte(address
>> PAGE_SHIFT
, PAGE_KERNEL_LARGE
));
325 static unsigned long __meminit
326 phys_pmd_update(pud_t
*pud
, unsigned long address
, unsigned long end
)
328 pmd_t
*pmd
= pmd_offset(pud
, 0);
329 unsigned long last_map_addr
;
331 spin_lock(&init_mm
.page_table_lock
);
332 last_map_addr
= phys_pmd_init(pmd
, address
, end
);
333 spin_unlock(&init_mm
.page_table_lock
);
335 return last_map_addr
;
338 static unsigned long __meminit
339 phys_pud_init(pud_t
*pud_page
, unsigned long addr
, unsigned long end
)
341 unsigned long last_map_addr
= end
;
342 int i
= pud_index(addr
);
344 for (; i
< PTRS_PER_PUD
; i
++, addr
= (addr
& PUD_MASK
) + PUD_SIZE
) {
345 unsigned long pmd_phys
;
346 pud_t
*pud
= pud_page
+ pud_index(addr
);
352 if (!after_bootmem
&&
353 !e820_any_mapped(addr
, addr
+PUD_SIZE
, 0)) {
354 set_pud(pud
, __pud(0));
359 if (!pud_large(*pud
))
360 last_map_addr
= phys_pmd_update(pud
, addr
, end
);
364 if (direct_gbpages
) {
365 set_pte((pte_t
*)pud
,
366 pfn_pte(addr
>> PAGE_SHIFT
, PAGE_KERNEL_LARGE
));
367 last_map_addr
= (addr
& PUD_MASK
) + PUD_SIZE
;
371 pmd
= alloc_low_page(&pmd_phys
);
373 spin_lock(&init_mm
.page_table_lock
);
374 set_pud(pud
, __pud(pmd_phys
| _KERNPG_TABLE
));
375 last_map_addr
= phys_pmd_init(pmd
, addr
, end
);
376 spin_unlock(&init_mm
.page_table_lock
);
382 return last_map_addr
>> PAGE_SHIFT
;
385 static void __init
find_early_table_space(unsigned long end
)
387 unsigned long puds
, pmds
, tables
, start
;
389 puds
= (end
+ PUD_SIZE
- 1) >> PUD_SHIFT
;
390 tables
= round_up(puds
* sizeof(pud_t
), PAGE_SIZE
);
391 if (!direct_gbpages
) {
392 pmds
= (end
+ PMD_SIZE
- 1) >> PMD_SHIFT
;
393 tables
+= round_up(pmds
* sizeof(pmd_t
), PAGE_SIZE
);
397 * RED-PEN putting page tables only on node 0 could
398 * cause a hotspot and fill up ZONE_DMA. The page tables
399 * need roughly 0.5KB per GB.
402 table_start
= find_e820_area(start
, end
, tables
, PAGE_SIZE
);
403 if (table_start
== -1UL)
404 panic("Cannot find space for the kernel page tables");
406 table_start
>>= PAGE_SHIFT
;
407 table_end
= table_start
;
409 early_printk("kernel direct mapping tables up to %lx @ %lx-%lx\n",
410 end
, table_start
<< PAGE_SHIFT
,
411 (table_start
<< PAGE_SHIFT
) + tables
);
414 static void __init
init_gbpages(void)
416 if (direct_gbpages
&& cpu_has_gbpages
)
417 printk(KERN_INFO
"Using GB pages for direct mapping\n");
422 #ifdef CONFIG_MEMTEST_BOOTPARAM
424 static void __init
memtest(unsigned long start_phys
, unsigned long size
,
428 unsigned long *start
;
429 unsigned long start_bad
;
430 unsigned long last_bad
;
432 unsigned long start_phys_aligned
;
444 val
= 0x5555555555555555UL
;
447 val
= 0xaaaaaaaaaaaaaaaaUL
;
453 incr
= sizeof(unsigned long);
454 start_phys_aligned
= ALIGN(start_phys
, incr
);
455 count
= (size
- (start_phys_aligned
- start_phys
))/incr
;
456 start
= __va(start_phys_aligned
);
460 for (i
= 0; i
< count
; i
++)
462 for (i
= 0; i
< count
; i
++, start
++, start_phys_aligned
+= incr
) {
464 if (start_phys_aligned
== last_bad
+ incr
) {
468 printk(KERN_CONT
"\n %016lx bad mem addr %016lx - %016lx reserved",
469 val
, start_bad
, last_bad
+ incr
);
470 reserve_early(start_bad
, last_bad
- start_bad
, "BAD RAM");
472 start_bad
= last_bad
= start_phys_aligned
;
477 printk(KERN_CONT
"\n %016lx bad mem addr %016lx - %016lx reserved",
478 val
, start_bad
, last_bad
+ incr
);
479 reserve_early(start_bad
, last_bad
- start_bad
, "BAD RAM");
484 static int memtest_pattern __initdata
= CONFIG_MEMTEST_BOOTPARAM_VALUE
;
486 static int __init
parse_memtest(char *arg
)
489 memtest_pattern
= simple_strtoul(arg
, NULL
, 0);
493 early_param("memtest", parse_memtest
);
495 static void __init
early_memtest(unsigned long start
, unsigned long end
)
500 if (!memtest_pattern
)
503 printk(KERN_INFO
"early_memtest: pattern num %d", memtest_pattern
);
504 for (pattern
= 0; pattern
< memtest_pattern
; pattern
++) {
507 while (t_start
< end
) {
508 t_start
= find_e820_area_size(t_start
, &t_size
, 1);
513 if (t_start
+ t_size
> end
)
514 t_size
= end
- t_start
;
516 printk(KERN_CONT
"\n %016llx - %016llx pattern %d",
517 t_start
, t_start
+ t_size
, pattern
);
519 memtest(t_start
, t_size
, pattern
);
524 printk(KERN_CONT
"\n");
527 static void __init
early_memtest(unsigned long start
, unsigned long end
)
533 * Setup the direct mapping of the physical memory at PAGE_OFFSET.
534 * This runs before bootmem is initialized and gets pages directly from
535 * the physical memory. To access them they are temporarily mapped.
537 unsigned long __init_refok
init_memory_mapping(unsigned long start
, unsigned long end
)
539 unsigned long next
, last_map_addr
= end
;
540 unsigned long start_phys
= start
, end_phys
= end
;
542 printk(KERN_INFO
"init_memory_mapping\n");
545 * Find space for the kernel direct mapping tables.
547 * Later we should allocate these tables in the local node of the
548 * memory mapped. Unfortunately this is done currently before the
549 * nodes are discovered.
551 if (!after_bootmem
) {
553 find_early_table_space(end
);
556 start
= (unsigned long)__va(start
);
557 end
= (unsigned long)__va(end
);
559 for (; start
< end
; start
= next
) {
560 pgd_t
*pgd
= pgd_offset_k(start
);
561 unsigned long pud_phys
;
565 pud
= pud_offset(pgd
, start
& PGDIR_MASK
);
567 pud
= alloc_low_page(&pud_phys
);
569 next
= start
+ PGDIR_SIZE
;
572 last_map_addr
= phys_pud_init(pud
, __pa(start
), __pa(next
));
574 set_pgd(pgd_offset_k(start
), mk_kernel_pgd(pud_phys
));
579 mmu_cr4_features
= read_cr4();
583 reserve_early(table_start
<< PAGE_SHIFT
,
584 table_end
<< PAGE_SHIFT
, "PGTABLE");
587 early_memtest(start_phys
, end_phys
);
589 return last_map_addr
;
593 void __init
paging_init(void)
595 unsigned long max_zone_pfns
[MAX_NR_ZONES
];
597 memset(max_zone_pfns
, 0, sizeof(max_zone_pfns
));
598 max_zone_pfns
[ZONE_DMA
] = MAX_DMA_PFN
;
599 max_zone_pfns
[ZONE_DMA32
] = MAX_DMA32_PFN
;
600 max_zone_pfns
[ZONE_NORMAL
] = end_pfn
;
602 memory_present(0, 0, end_pfn
);
604 free_area_init_nodes(max_zone_pfns
);
609 * Memory hotplug specific functions
611 #ifdef CONFIG_MEMORY_HOTPLUG
613 * Memory is added always to NORMAL zone. This means you will never get
614 * additional DMA/DMA32 memory.
616 int arch_add_memory(int nid
, u64 start
, u64 size
)
618 struct pglist_data
*pgdat
= NODE_DATA(nid
);
619 struct zone
*zone
= pgdat
->node_zones
+ ZONE_NORMAL
;
620 unsigned long last_mapped_pfn
, start_pfn
= start
>> PAGE_SHIFT
;
621 unsigned long nr_pages
= size
>> PAGE_SHIFT
;
624 last_mapped_pfn
= init_memory_mapping(start
, start
+ size
-1);
625 if (last_mapped_pfn
> max_pfn_mapped
)
626 max_pfn_mapped
= last_mapped_pfn
;
628 ret
= __add_pages(zone
, start_pfn
, nr_pages
);
633 EXPORT_SYMBOL_GPL(arch_add_memory
);
635 #if !defined(CONFIG_ACPI_NUMA) && defined(CONFIG_NUMA)
636 int memory_add_physaddr_to_nid(u64 start
)
640 EXPORT_SYMBOL_GPL(memory_add_physaddr_to_nid
);
643 #endif /* CONFIG_MEMORY_HOTPLUG */
646 * devmem_is_allowed() checks to see if /dev/mem access to a certain address
647 * is valid. The argument is a physical page number.
650 * On x86, access has to be given to the first megabyte of ram because that area
651 * contains bios code and data regions used by X and dosemu and similar apps.
652 * Access has to be given to non-kernel-ram areas as well, these contain the PCI
653 * mmio resources as well as potential bios/acpi data regions.
655 int devmem_is_allowed(unsigned long pagenr
)
659 if (!page_is_ram(pagenr
))
665 static struct kcore_list kcore_mem
, kcore_vmalloc
, kcore_kernel
,
666 kcore_modules
, kcore_vsyscall
;
668 void __init
mem_init(void)
670 long codesize
, reservedpages
, datasize
, initsize
;
674 /* clear_bss() already clear the empty_zero_page */
678 /* this will put all low memory onto the freelists */
680 totalram_pages
= numa_free_all_bootmem();
682 totalram_pages
= free_all_bootmem();
684 reservedpages
= end_pfn
- totalram_pages
-
685 absent_pages_in_range(0, end_pfn
);
688 codesize
= (unsigned long) &_etext
- (unsigned long) &_text
;
689 datasize
= (unsigned long) &_edata
- (unsigned long) &_etext
;
690 initsize
= (unsigned long) &__init_end
- (unsigned long) &__init_begin
;
692 /* Register memory areas for /proc/kcore */
693 kclist_add(&kcore_mem
, __va(0), max_low_pfn
<< PAGE_SHIFT
);
694 kclist_add(&kcore_vmalloc
, (void *)VMALLOC_START
,
695 VMALLOC_END
-VMALLOC_START
);
696 kclist_add(&kcore_kernel
, &_stext
, _end
- _stext
);
697 kclist_add(&kcore_modules
, (void *)MODULES_VADDR
, MODULES_LEN
);
698 kclist_add(&kcore_vsyscall
, (void *)VSYSCALL_START
,
699 VSYSCALL_END
- VSYSCALL_START
);
701 printk(KERN_INFO
"Memory: %luk/%luk available (%ldk kernel code, "
702 "%ldk reserved, %ldk data, %ldk init)\n",
703 (unsigned long) nr_free_pages() << (PAGE_SHIFT
-10),
704 end_pfn
<< (PAGE_SHIFT
-10),
706 reservedpages
<< (PAGE_SHIFT
-10),
713 void free_init_pages(char *what
, unsigned long begin
, unsigned long end
)
715 unsigned long addr
= begin
;
721 * If debugging page accesses then do not free this memory but
722 * mark them not present - any buggy init-section access will
723 * create a kernel page fault:
725 #ifdef CONFIG_DEBUG_PAGEALLOC
726 printk(KERN_INFO
"debug: unmapping init memory %08lx..%08lx\n",
727 begin
, PAGE_ALIGN(end
));
728 set_memory_np(begin
, (end
- begin
) >> PAGE_SHIFT
);
730 printk(KERN_INFO
"Freeing %s: %luk freed\n", what
, (end
- begin
) >> 10);
732 for (; addr
< end
; addr
+= PAGE_SIZE
) {
733 ClearPageReserved(virt_to_page(addr
));
734 init_page_count(virt_to_page(addr
));
735 memset((void *)(addr
& ~(PAGE_SIZE
-1)),
736 POISON_FREE_INITMEM
, PAGE_SIZE
);
743 void free_initmem(void)
745 free_init_pages("unused kernel memory",
746 (unsigned long)(&__init_begin
),
747 (unsigned long)(&__init_end
));
750 #ifdef CONFIG_DEBUG_RODATA
751 const int rodata_test_data
= 0xC3;
752 EXPORT_SYMBOL_GPL(rodata_test_data
);
754 void mark_rodata_ro(void)
756 unsigned long start
= PFN_ALIGN(_stext
), end
= PFN_ALIGN(__end_rodata
);
758 printk(KERN_INFO
"Write protecting the kernel read-only data: %luk\n",
759 (end
- start
) >> 10);
760 set_memory_ro(start
, (end
- start
) >> PAGE_SHIFT
);
763 * The rodata section (but not the kernel text!) should also be
766 start
= ((unsigned long)__start_rodata
+ PAGE_SIZE
- 1) & PAGE_MASK
;
767 set_memory_nx(start
, (end
- start
) >> PAGE_SHIFT
);
771 #ifdef CONFIG_CPA_DEBUG
772 printk(KERN_INFO
"Testing CPA: undo %lx-%lx\n", start
, end
);
773 set_memory_rw(start
, (end
-start
) >> PAGE_SHIFT
);
775 printk(KERN_INFO
"Testing CPA: again\n");
776 set_memory_ro(start
, (end
-start
) >> PAGE_SHIFT
);
782 #ifdef CONFIG_BLK_DEV_INITRD
783 void free_initrd_mem(unsigned long start
, unsigned long end
)
785 free_init_pages("initrd memory", start
, end
);
789 void __init
reserve_bootmem_generic(unsigned long phys
, unsigned len
)
794 unsigned long pfn
= phys
>> PAGE_SHIFT
;
796 if (pfn
>= end_pfn
) {
798 * This can happen with kdump kernels when accessing
801 if (pfn
< max_pfn_mapped
)
804 printk(KERN_ERR
"reserve_bootmem: illegal reserve %lx %u\n",
809 /* Should check here against the e820 map to avoid double free */
811 nid
= phys_to_nid(phys
);
812 next_nid
= phys_to_nid(phys
+ len
- 1);
814 reserve_bootmem_node(NODE_DATA(nid
), phys
, len
, BOOTMEM_DEFAULT
);
816 reserve_bootmem(phys
, len
, BOOTMEM_DEFAULT
);
818 reserve_bootmem(phys
, len
, BOOTMEM_DEFAULT
);
821 if (phys
+len
<= MAX_DMA_PFN
*PAGE_SIZE
) {
822 dma_reserve
+= len
/ PAGE_SIZE
;
823 set_dma_reserve(dma_reserve
);
827 int kern_addr_valid(unsigned long addr
)
829 unsigned long above
= ((long)addr
) >> __VIRTUAL_MASK_SHIFT
;
835 if (above
!= 0 && above
!= -1UL)
838 pgd
= pgd_offset_k(addr
);
842 pud
= pud_offset(pgd
, addr
);
846 pmd
= pmd_offset(pud
, addr
);
851 return pfn_valid(pmd_pfn(*pmd
));
853 pte
= pte_offset_kernel(pmd
, addr
);
857 return pfn_valid(pte_pfn(*pte
));
861 * A pseudo VMA to allow ptrace access for the vsyscall page. This only
862 * covers the 64bit vsyscall page now. 32bit has a real VMA now and does
863 * not need special handling anymore:
865 static struct vm_area_struct gate_vma
= {
866 .vm_start
= VSYSCALL_START
,
867 .vm_end
= VSYSCALL_START
+ (VSYSCALL_MAPPED_PAGES
* PAGE_SIZE
),
868 .vm_page_prot
= PAGE_READONLY_EXEC
,
869 .vm_flags
= VM_READ
| VM_EXEC
872 struct vm_area_struct
*get_gate_vma(struct task_struct
*tsk
)
874 #ifdef CONFIG_IA32_EMULATION
875 if (test_tsk_thread_flag(tsk
, TIF_IA32
))
881 int in_gate_area(struct task_struct
*task
, unsigned long addr
)
883 struct vm_area_struct
*vma
= get_gate_vma(task
);
888 return (addr
>= vma
->vm_start
) && (addr
< vma
->vm_end
);
892 * Use this when you have no reliable task/vma, typically from interrupt
893 * context. It is less reliable than using the task's vma and may give
896 int in_gate_area_no_task(unsigned long addr
)
898 return (addr
>= VSYSCALL_START
) && (addr
< VSYSCALL_END
);
901 const char *arch_vma_name(struct vm_area_struct
*vma
)
903 if (vma
->vm_mm
&& vma
->vm_start
== (long)vma
->vm_mm
->context
.vdso
)
905 if (vma
== &gate_vma
)
910 #ifdef CONFIG_SPARSEMEM_VMEMMAP
912 * Initialise the sparsemem vmemmap using huge-pages at the PMD level.
914 static long __meminitdata addr_start
, addr_end
;
915 static void __meminitdata
*p_start
, *p_end
;
916 static int __meminitdata node_start
;
919 vmemmap_populate(struct page
*start_page
, unsigned long size
, int node
)
921 unsigned long addr
= (unsigned long)start_page
;
922 unsigned long end
= (unsigned long)(start_page
+ size
);
928 for (; addr
< end
; addr
= next
) {
929 next
= pmd_addr_end(addr
, end
);
931 pgd
= vmemmap_pgd_populate(addr
, node
);
935 pud
= vmemmap_pud_populate(pgd
, addr
, node
);
939 pmd
= pmd_offset(pud
, addr
);
940 if (pmd_none(*pmd
)) {
944 p
= vmemmap_alloc_block(PMD_SIZE
, node
);
948 entry
= pfn_pte(__pa(p
) >> PAGE_SHIFT
,
950 set_pmd(pmd
, __pmd(pte_val(entry
)));
952 /* check to see if we have contiguous blocks */
953 if (p_end
!= p
|| node_start
!= node
) {
955 printk(KERN_DEBUG
" [%lx-%lx] PMD -> [%p-%p] on node %d\n",
956 addr_start
, addr_end
-1, p_start
, p_end
-1, node_start
);
961 addr_end
= addr
+ PMD_SIZE
;
962 p_end
= p
+ PMD_SIZE
;
964 vmemmap_verify((pte_t
*)pmd
, node
, addr
, next
);
970 void __meminit
vmemmap_populate_print_last(void)
973 printk(KERN_DEBUG
" [%lx-%lx] PMD -> [%p-%p] on node %d\n",
974 addr_start
, addr_end
-1, p_start
, p_end
-1, node_start
);