projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
[PATCH] i386: PARAVIRT: Add pagetable accessors to pack and unpack pagetable entries
[deliverable/linux.git] / arch / i386 / mm / init.c
CommitLineData
1da177e4
LT		 1/*
2 * linux/arch/i386/mm/init.c
3 *
4 * Copyright (C) 1995 Linus Torvalds
5 *
6 * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
7 */
8
1da177e4
LT		 9#include <linux/module.h>
10#include <linux/signal.h>
11#include <linux/sched.h>
12#include <linux/kernel.h>
13#include <linux/errno.h>
14#include <linux/string.h>
15#include <linux/types.h>
16#include <linux/ptrace.h>
17#include <linux/mman.h>
18#include <linux/mm.h>
19#include <linux/hugetlb.h>
20#include <linux/swap.h>
21#include <linux/smp.h>
22#include <linux/init.h>
23#include <linux/highmem.h>
24#include <linux/pagemap.h>
6fb14755 25#include <linux/pfn.h>
c9cf5528 26#include <linux/poison.h>
1da177e4
LT		 27#include <linux/bootmem.h>
28#include <linux/slab.h>
29#include <linux/proc_fs.h>
30#include <linux/efi.h>
05039b92 31#include <linux/memory_hotplug.h>
27d99f7e 32#include <linux/initrd.h>
55b2355e 33#include <linux/cpumask.h>
1da177e4
LT		 34
35#include <asm/processor.h>
36#include <asm/system.h>
37#include <asm/uaccess.h>
38#include <asm/pgtable.h>
39#include <asm/dma.h>
40#include <asm/fixmap.h>
41#include <asm/e820.h>
42#include <asm/apic.h>
43#include <asm/tlb.h>
44#include <asm/tlbflush.h>
45#include <asm/sections.h>
46
47unsigned int __VMALLOC_RESERVE = 128 << 20;
48
49DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);
50unsigned long highstart_pfn, highend_pfn;
51
52static int noinline do_test_wp_bit(void);
53
54/*
55 * Creates a middle page table and puts a pointer to it in the
56 * given global directory entry. This only returns the gd entry
57 * in non-PAE compilation mode, since the middle layer is folded.
58 */
59static pmd_t * __init one_md_table_init(pgd_t *pgd)
60{
61 pud_t *pud;
62 pmd_t *pmd_table;
63
64#ifdef CONFIG_X86_PAE
65 pmd_table = (pmd_t *) alloc_bootmem_low_pages(PAGE_SIZE);
c119ecce 66 paravirt_alloc_pd(__pa(pmd_table) >> PAGE_SHIFT);
1da177e4
LT		 67 set_pgd(pgd, __pgd(__pa(pmd_table) | _PAGE_PRESENT));
68 pud = pud_offset(pgd, 0);
69 if (pmd_table != pmd_offset(pud, 0))
70 BUG();
71#else
72 pud = pud_offset(pgd, 0);
73 pmd_table = pmd_offset(pud, 0);
74#endif
75
76 return pmd_table;
77}
78
79/*
80 * Create a page table and place a pointer to it in a middle page
81 * directory entry.
82 */
83static pte_t * __init one_page_table_init(pmd_t *pmd)
84{
85 if (pmd_none(*pmd)) {
86 pte_t *page_table = (pte_t *) alloc_bootmem_low_pages(PAGE_SIZE);
c119ecce 87 paravirt_alloc_pt(__pa(page_table) >> PAGE_SHIFT);
1da177e4
LT		 88 set_pmd(pmd, __pmd(__pa(page_table) | _PAGE_TABLE));
89 if (page_table != pte_offset_kernel(pmd, 0))
90 BUG();
91
92 return page_table;
93 }
94
95 return pte_offset_kernel(pmd, 0);
96}
97
98/*
99 * This function initializes a certain range of kernel virtual memory
100 * with new bootmem page tables, everywhere page tables are missing in
101 * the given range.
102 */
103
104/*
105 * NOTE: The pagetables are allocated contiguous on the physical space
106 * so we can cache the place of the first one and move around without
107 * checking the pgd every time.
108 */
109static void __init page_table_range_init (unsigned long start, unsigned long end, pgd_t *pgd_base)
110{
111 pgd_t *pgd;
112 pud_t *pud;
113 pmd_t *pmd;
114 int pgd_idx, pmd_idx;
115 unsigned long vaddr;
116
117 vaddr = start;
118 pgd_idx = pgd_index(vaddr);
119 pmd_idx = pmd_index(vaddr);
120 pgd = pgd_base + pgd_idx;
121
122 for ( ; (pgd_idx < PTRS_PER_PGD) && (vaddr != end); pgd++, pgd_idx++) {
123 if (pgd_none(*pgd))
124 one_md_table_init(pgd);
125 pud = pud_offset(pgd, vaddr);
126 pmd = pmd_offset(pud, vaddr);
127 for (; (pmd_idx < PTRS_PER_PMD) && (vaddr != end); pmd++, pmd_idx++) {
128 if (pmd_none(*pmd))
129 one_page_table_init(pmd);
130
131 vaddr += PMD_SIZE;
132 }
133 pmd_idx = 0;
134 }
135}
136
137static inline int is_kernel_text(unsigned long addr)
138{
139 if (addr >= PAGE_OFFSET && addr <= (unsigned long)__init_end)
140 return 1;
141 return 0;
142}
143
144/*
145 * This maps the physical memory to kernel virtual address space, a total
146 * of max_low_pfn pages, by creating page tables starting from address
147 * PAGE_OFFSET.
148 */
149static void __init kernel_physical_mapping_init(pgd_t *pgd_base)
150{
151 unsigned long pfn;
152 pgd_t *pgd;
153 pmd_t *pmd;
154 pte_t *pte;
155 int pgd_idx, pmd_idx, pte_ofs;
156
157 pgd_idx = pgd_index(PAGE_OFFSET);
158 pgd = pgd_base + pgd_idx;
159 pfn = 0;
160
161 for (; pgd_idx < PTRS_PER_PGD; pgd++, pgd_idx++) {
162 pmd = one_md_table_init(pgd);
163 if (pfn >= max_low_pfn)
164 continue;
165 for (pmd_idx = 0; pmd_idx < PTRS_PER_PMD && pfn < max_low_pfn; pmd++, pmd_idx++) {
166 unsigned int address = pfn * PAGE_SIZE + PAGE_OFFSET;
167
168 /* Map with big pages if possible, otherwise create normal page tables. */
169 if (cpu_has_pse) {
170 unsigned int address2 = (pfn + PTRS_PER_PTE - 1) * PAGE_SIZE + PAGE_OFFSET + PAGE_SIZE-1;
171
172 if (is_kernel_text(address) || is_kernel_text(address2))
173 set_pmd(pmd, pfn_pmd(pfn, PAGE_KERNEL_LARGE_EXEC));
174 else
175 set_pmd(pmd, pfn_pmd(pfn, PAGE_KERNEL_LARGE));
176 pfn += PTRS_PER_PTE;
177 } else {
178 pte = one_page_table_init(pmd);
179
180 for (pte_ofs = 0; pte_ofs < PTRS_PER_PTE && pfn < max_low_pfn; pte++, pfn++, pte_ofs++) {
181 if (is_kernel_text(address))
182 set_pte(pte, pfn_pte(pfn, PAGE_KERNEL_EXEC));
183 else
184 set_pte(pte, pfn_pte(pfn, PAGE_KERNEL));
185 }
186 }
187 }
188 }
189}
190
191static inline int page_kills_ppro(unsigned long pagenr)
192{
193 if (pagenr >= 0x70000 && pagenr <= 0x7003F)
194 return 1;
195 return 0;
196}
197
5b505b90 198int page_is_ram(unsigned long pagenr)
1da177e4
LT		 199{
200 int i;
201 unsigned long addr, end;
202
203 if (efi_enabled) {
204 efi_memory_desc_t *md;
7ae65fd3 205 void *p;
1da177e4 206
7ae65fd3
MT		 207 for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) {
208 md = p;
1da177e4
LT		 209 if (!is_available_memory(md))
210 continue;
211 addr = (md->phys_addr+PAGE_SIZE-1) >> PAGE_SHIFT;
212 end = (md->phys_addr + (md->num_pages << EFI_PAGE_SHIFT)) >> PAGE_SHIFT;
213
214 if ((pagenr >= addr) && (pagenr < end))
215 return 1;
216 }
217 return 0;
218 }
219
220 for (i = 0; i < e820.nr_map; i++) {
221
222 if (e820.map[i].type != E820_RAM) /* not usable memory */
223 continue;
224 /*
225 * !!!FIXME!!! Some BIOSen report areas as RAM that
226 * are not. Notably the 640->1Mb area. We need a sanity
227 * check here.
228 */
229 addr = (e820.map[i].addr+PAGE_SIZE-1) >> PAGE_SHIFT;
230 end = (e820.map[i].addr+e820.map[i].size) >> PAGE_SHIFT;
231 if ((pagenr >= addr) && (pagenr < end))
232 return 1;
233 }
234 return 0;
235}
236
237#ifdef CONFIG_HIGHMEM
238pte_t *kmap_pte;
239pgprot_t kmap_prot;
240
241#define kmap_get_fixmap_pte(vaddr) \
242 pte_offset_kernel(pmd_offset(pud_offset(pgd_offset_k(vaddr), vaddr), (vaddr)), (vaddr))
243
244static void __init kmap_init(void)
245{
246 unsigned long kmap_vstart;
247
248 /* cache the first kmap pte */
249 kmap_vstart = __fix_to_virt(FIX_KMAP_BEGIN);
250 kmap_pte = kmap_get_fixmap_pte(kmap_vstart);
251
252 kmap_prot = PAGE_KERNEL;
253}
254
255static void __init permanent_kmaps_init(pgd_t *pgd_base)
256{
257 pgd_t *pgd;
258 pud_t *pud;
259 pmd_t *pmd;
260 pte_t *pte;
261 unsigned long vaddr;
262
263 vaddr = PKMAP_BASE;
264 page_table_range_init(vaddr, vaddr + PAGE_SIZE*LAST_PKMAP, pgd_base);
265
266 pgd = swapper_pg_dir + pgd_index(vaddr);
267 pud = pud_offset(pgd, vaddr);
268 pmd = pmd_offset(pud, vaddr);
269 pte = pte_offset_kernel(pmd, vaddr);
270 pkmap_page_table = pte;
271}
272
c09b4240 273static void __meminit free_new_highpage(struct page *page)
05039b92 274{
7835e98b 275 init_page_count(page);
05039b92
DH		 276 __free_page(page);
277 totalhigh_pages++;
278}
279
280void __init add_one_highpage_init(struct page *page, int pfn, int bad_ppro)
1da177e4
LT		 281{
282 if (page_is_ram(pfn) && !(bad_ppro && page_kills_ppro(pfn))) {
283 ClearPageReserved(page);
05039b92 284 free_new_highpage(page);
1da177e4
LT		 285 } else
286 SetPageReserved(page);
287}
288
0e0be25d 289static int __meminit add_one_highpage_hotplug(struct page *page, unsigned long pfn)
05039b92
DH		 290{
291 free_new_highpage(page);
292 totalram_pages++;
293#ifdef CONFIG_FLATMEM
294 max_mapnr = max(pfn, max_mapnr);
295#endif
296 num_physpages++;
297 return 0;
298}
299
300/*
301 * Not currently handling the NUMA case.
302 * Assuming single node and all memory that
303 * has been added dynamically that would be
304 * onlined here is in HIGHMEM
305 */
0e0be25d 306void __meminit online_page(struct page *page)
05039b92
DH		 307{
308 ClearPageReserved(page);
309 add_one_highpage_hotplug(page, page_to_pfn(page));
310}
311
312
05b79bdc
AW		 313#ifdef CONFIG_NUMA
314extern void set_highmem_pages_init(int);
315#else
1da177e4
LT		 316static void __init set_highmem_pages_init(int bad_ppro)
317{
318 int pfn;
319 for (pfn = highstart_pfn; pfn < highend_pfn; pfn++)
05039b92 320 add_one_highpage_init(pfn_to_page(pfn), pfn, bad_ppro);
1da177e4
LT		 321 totalram_pages += totalhigh_pages;
322}
05b79bdc 323#endif /* CONFIG_FLATMEM */
1da177e4
LT		 324
325#else
326#define kmap_init() do { } while (0)
327#define permanent_kmaps_init(pgd_base) do { } while (0)
328#define set_highmem_pages_init(bad_ppro) do { } while (0)
329#endif /* CONFIG_HIGHMEM */
330
331unsigned long long __PAGE_KERNEL = _PAGE_KERNEL;
129f6946 332EXPORT_SYMBOL(__PAGE_KERNEL);
1da177e4
LT		 333unsigned long long __PAGE_KERNEL_EXEC = _PAGE_KERNEL_EXEC;
334
05b79bdc 335#ifdef CONFIG_NUMA
1da177e4 336extern void __init remap_numa_kva(void);
05b79bdc
AW		 337#else
338#define remap_numa_kva() do {} while (0)
1da177e4
LT		 339#endif
340
341static void __init pagetable_init (void)
342{
343 unsigned long vaddr;
344 pgd_t *pgd_base = swapper_pg_dir;
345
346#ifdef CONFIG_X86_PAE
347 int i;
348 /* Init entries of the first-level page table to the zero page */
349 for (i = 0; i < PTRS_PER_PGD; i++)
350 set_pgd(pgd_base + i, __pgd(__pa(empty_zero_page) | _PAGE_PRESENT));
c119ecce
ZA		 351#else
352 paravirt_alloc_pd(__pa(swapper_pg_dir) >> PAGE_SHIFT);
1da177e4
LT		 353#endif
354
355 /* Enable PSE if available */
356 if (cpu_has_pse) {
357 set_in_cr4(X86_CR4_PSE);
358 }
359
360 /* Enable PGE if available */
361 if (cpu_has_pge) {
362 set_in_cr4(X86_CR4_PGE);
363 __PAGE_KERNEL |= _PAGE_GLOBAL;
364 __PAGE_KERNEL_EXEC |= _PAGE_GLOBAL;
365 }
366
367 kernel_physical_mapping_init(pgd_base);
368 remap_numa_kva();
369
370 /*
371 * Fixed mappings, only the page table structure has to be
372 * created - mappings will be set by set_fixmap():
373 */
374 vaddr = __fix_to_virt(__end_of_fixed_addresses - 1) & PMD_MASK;
375 page_table_range_init(vaddr, 0, pgd_base);
376
377 permanent_kmaps_init(pgd_base);
378
379#ifdef CONFIG_X86_PAE
380 /*
381 * Add low memory identity-mappings - SMP needs it when
382 * starting up on an AP from real-mode. In the non-PAE
383 * case we already have these mappings through head.S.
384 * All user-space mappings are explicitly cleared after
385 * SMP startup.
386 */
c9b02a24 387 set_pgd(&pgd_base[0], pgd_base[USER_PTRS_PER_PGD]);
1da177e4
LT		 388#endif
389}
390
55b2355e 391#if defined(CONFIG_SOFTWARE_SUSPEND) || defined(CONFIG_ACPI_SLEEP)
1da177e4
LT		 392/*
393 * Swap suspend & friends need this for resume because things like the intel-agp
394 * driver might have split up a kernel 4MB mapping.
395 */
396char __nosavedata swsusp_pg_dir[PAGE_SIZE]
397 __attribute__ ((aligned (PAGE_SIZE)));
398
399static inline void save_pg_dir(void)
400{
401 memcpy(swsusp_pg_dir, swapper_pg_dir, PAGE_SIZE);
402}
403#else
404static inline void save_pg_dir(void)
405{
406}
407#endif
408
409void zap_low_mappings (void)
410{
411 int i;
412
413 save_pg_dir();
414
415 /*
416 * Zap initial low-memory mappings.
417 *
418 * Note that "pgd_clear()" doesn't do it for
419 * us, because pgd_clear() is a no-op on i386.
420 */
421 for (i = 0; i < USER_PTRS_PER_PGD; i++)
422#ifdef CONFIG_X86_PAE
423 set_pgd(swapper_pg_dir+i, __pgd(1 + __pa(empty_zero_page)));
424#else
425 set_pgd(swapper_pg_dir+i, __pgd(0));
426#endif
427 flush_tlb_all();
428}
429
430static int disable_nx __initdata = 0;
6c231b7b 431u64 __supported_pte_mask __read_mostly = ~_PAGE_NX;
1da177e4
LT		 432
433/*
434 * noexec = on|off
435 *
436 * Control non executable mappings.
437 *
438 * on Enable
439 * off Disable
440 */
1a3f239d 441static int __init noexec_setup(char *str)
1da177e4 442{
1a3f239d
RR		 443 if (!str || !strcmp(str, "on")) {
444 if (cpu_has_nx) {
445 __supported_pte_mask |= _PAGE_NX;
446 disable_nx = 0;
447 }
448 } else if (!strcmp(str,"off")) {
1da177e4
LT		 449 disable_nx = 1;
450 __supported_pte_mask &= ~_PAGE_NX;
1a3f239d
RR		 451 } else
452 return -EINVAL;
453
454 return 0;
1da177e4 455}
1a3f239d 456early_param("noexec", noexec_setup);
1da177e4
LT		 457
458int nx_enabled = 0;
459#ifdef CONFIG_X86_PAE
460
461static void __init set_nx(void)
462{
463 unsigned int v[4], l, h;
464
465 if (cpu_has_pae && (cpuid_eax(0x80000000) > 0x80000001)) {
466 cpuid(0x80000001, &v[0], &v[1], &v[2], &v[3]);
467 if ((v[3] & (1 << 20)) && !disable_nx) {
468 rdmsr(MSR_EFER, l, h);
469 l |= EFER_NX;
470 wrmsr(MSR_EFER, l, h);
471 nx_enabled = 1;
472 __supported_pte_mask |= _PAGE_NX;
473 }
474 }
475}
476
477/*
478 * Enables/disables executability of a given kernel page and
479 * returns the previous setting.
480 */
481int __init set_kernel_exec(unsigned long vaddr, int enable)
482{
483 pte_t *pte;
484 int ret = 1;
485
486 if (!nx_enabled)
487 goto out;
488
489 pte = lookup_address(vaddr);
490 BUG_ON(!pte);
491
492 if (!pte_exec_kernel(*pte))
493 ret = 0;
494
495 if (enable)
496 pte->pte_high &= ~(1 << (_PAGE_BIT_NX - 32));
497 else
498 pte->pte_high |= 1 << (_PAGE_BIT_NX - 32);
789e6ac0 499 pte_update_defer(&init_mm, vaddr, pte);
1da177e4
LT		 500 __flush_tlb_all();
501out:
502 return ret;
503}
504
505#endif
506
507/*
508 * paging_init() sets up the page tables - note that the first 8MB are
509 * already mapped by head.S.
510 *
511 * This routines also unmaps the page at virtual kernel address 0, so
512 * that we can trap those pesky NULL-reference errors in the kernel.
513 */
514void __init paging_init(void)
515{
516#ifdef CONFIG_X86_PAE
517 set_nx();
518 if (nx_enabled)
519 printk("NX (Execute Disable) protection: active\n");
520#endif
521
522 pagetable_init();
523
524 load_cr3(swapper_pg_dir);
525
526#ifdef CONFIG_X86_PAE
527 /*
528 * We will bail out later - printk doesn't work right now so
529 * the user would just see a hanging kernel.
530 */
531 if (cpu_has_pae)
532 set_in_cr4(X86_CR4_PAE);
533#endif
534 __flush_tlb_all();
535
536 kmap_init();
537}
538
539/*
540 * Test if the WP bit works in supervisor mode. It isn't supported on 386's
541 * and also on some strange 486's (NexGen etc.). All 586+'s are OK. This
542 * used to involve black magic jumps to work around some nasty CPU bugs,
543 * but fortunately the switch to using exceptions got rid of all that.
544 */
545
546static void __init test_wp_bit(void)
547{
548 printk("Checking if this processor honours the WP bit even in supervisor mode... ");
549
550 /* Any page-aligned address will do, the test is non-destructive */
551 __set_fixmap(FIX_WP_TEST, __pa(&swapper_pg_dir), PAGE_READONLY);
552 boot_cpu_data.wp_works_ok = do_test_wp_bit();
553 clear_fixmap(FIX_WP_TEST);
554
555 if (!boot_cpu_data.wp_works_ok) {
556 printk("No.\n");
557#ifdef CONFIG_X86_WP_WORKS_OK
558 panic("This kernel doesn't support CPU's with broken WP. Recompile it for a 386!");
559#endif
560 } else {
561 printk("Ok.\n");
562 }
563}
564
1da177e4
LT		 565static struct kcore_list kcore_mem, kcore_vmalloc;
566
567void __init mem_init(void)
568{
569 extern int ppro_with_ram_bug(void);
570 int codesize, reservedpages, datasize, initsize;
571 int tmp;
572 int bad_ppro;
573
05b79bdc 574#ifdef CONFIG_FLATMEM
8d8f3cbe 575 BUG_ON(!mem_map);
1da177e4
LT		 576#endif
577
578 bad_ppro = ppro_with_ram_bug();
579
580#ifdef CONFIG_HIGHMEM
581 /* check that fixmap and pkmap do not overlap */
582 if (PKMAP_BASE+LAST_PKMAP*PAGE_SIZE >= FIXADDR_START) {
583 printk(KERN_ERR "fixmap and kmap areas overlap - this will crash\n");
584 printk(KERN_ERR "pkstart: %lxh pkend: %lxh fixstart %lxh\n",
585 PKMAP_BASE, PKMAP_BASE+LAST_PKMAP*PAGE_SIZE, FIXADDR_START);
586 BUG();
587 }
588#endif
589
1da177e4
LT		 590 /* this will put all low memory onto the freelists */
591 totalram_pages += free_all_bootmem();
592
593 reservedpages = 0;
594 for (tmp = 0; tmp < max_low_pfn; tmp++)
595 /*
596 * Only count reserved RAM pages
597 */
598 if (page_is_ram(tmp) && PageReserved(pfn_to_page(tmp)))
599 reservedpages++;
600
601 set_highmem_pages_init(bad_ppro);
602
603 codesize = (unsigned long) &_etext - (unsigned long) &_text;
604 datasize = (unsigned long) &_edata - (unsigned long) &_etext;
605 initsize = (unsigned long) &__init_end - (unsigned long) &__init_begin;
606
607 kclist_add(&kcore_mem, __va(0), max_low_pfn << PAGE_SHIFT);
608 kclist_add(&kcore_vmalloc, (void *)VMALLOC_START,
609 VMALLOC_END-VMALLOC_START);
610
611 printk(KERN_INFO "Memory: %luk/%luk available (%dk kernel code, %dk reserved, %dk data, %dk init, %ldk highmem)\n",
612 (unsigned long) nr_free_pages() << (PAGE_SHIFT-10),
613 num_physpages << (PAGE_SHIFT-10),
614 codesize >> 10,
615 reservedpages << (PAGE_SHIFT-10),
616 datasize >> 10,
617 initsize >> 10,
618 (unsigned long) (totalhigh_pages << (PAGE_SHIFT-10))
619 );
620
052e7994
JF		 621#if 1 /* double-sanity-check paranoia */
622 printk("virtual kernel memory layout:\n"
623 " fixmap : 0x%08lx - 0x%08lx (%4ld kB)\n"
624#ifdef CONFIG_HIGHMEM
625 " pkmap : 0x%08lx - 0x%08lx (%4ld kB)\n"
626#endif
627 " vmalloc : 0x%08lx - 0x%08lx (%4ld MB)\n"
628 " lowmem : 0x%08lx - 0x%08lx (%4ld MB)\n"
629 " .init : 0x%08lx - 0x%08lx (%4ld kB)\n"
630 " .data : 0x%08lx - 0x%08lx (%4ld kB)\n"
631 " .text : 0x%08lx - 0x%08lx (%4ld kB)\n",
632 FIXADDR_START, FIXADDR_TOP,
633 (FIXADDR_TOP - FIXADDR_START) >> 10,
634
635#ifdef CONFIG_HIGHMEM
636 PKMAP_BASE, PKMAP_BASE+LAST_PKMAP*PAGE_SIZE,
637 (LAST_PKMAP*PAGE_SIZE) >> 10,
638#endif
639
640 VMALLOC_START, VMALLOC_END,
641 (VMALLOC_END - VMALLOC_START) >> 20,
642
643 (unsigned long)__va(0), (unsigned long)high_memory,
644 ((unsigned long)high_memory - (unsigned long)__va(0)) >> 20,
645
646 (unsigned long)&__init_begin, (unsigned long)&__init_end,
647 ((unsigned long)&__init_end - (unsigned long)&__init_begin) >> 10,
648
649 (unsigned long)&_etext, (unsigned long)&_edata,
650 ((unsigned long)&_edata - (unsigned long)&_etext) >> 10,
651
652 (unsigned long)&_text, (unsigned long)&_etext,
653 ((unsigned long)&_etext - (unsigned long)&_text) >> 10);
654
655#ifdef CONFIG_HIGHMEM
656 BUG_ON(PKMAP_BASE+LAST_PKMAP*PAGE_SIZE > FIXADDR_START);
657 BUG_ON(VMALLOC_END > PKMAP_BASE);
658#endif
659 BUG_ON(VMALLOC_START > VMALLOC_END);
660 BUG_ON((unsigned long)high_memory > VMALLOC_START);
661#endif /* double-sanity-check paranoia */
662
1da177e4
LT		 663#ifdef CONFIG_X86_PAE
664 if (!cpu_has_pae)
665 panic("cannot execute a PAE-enabled kernel on a PAE-less CPU!");
666#endif
667 if (boot_cpu_data.wp_works_ok < 0)
668 test_wp_bit();
669
670 /*
671 * Subtle. SMP is doing it's boot stuff late (because it has to
672 * fork idle threads) - but it also needs low mappings for the
673 * protected-mode entry to work. We zap these entries only after
674 * the WP-bit has been tested.
675 */
676#ifndef CONFIG_SMP
677 zap_low_mappings();
678#endif
679}
680
ad8f5797 681#ifdef CONFIG_MEMORY_HOTPLUG
bc02af93 682int arch_add_memory(int nid, u64 start, u64 size)
05039b92 683{
7c7e9425 684 struct pglist_data *pgdata = NODE_DATA(nid);
776ed98b 685 struct zone *zone = pgdata->node_zones + ZONE_HIGHMEM;
05039b92
DH		 686 unsigned long start_pfn = start >> PAGE_SHIFT;
687 unsigned long nr_pages = size >> PAGE_SHIFT;
688
689 return __add_pages(zone, start_pfn, nr_pages);
690}
691
692int remove_memory(u64 start, u64 size)
693{
694 return -EINVAL;
695}
7c7e9425 696EXPORT_SYMBOL_GPL(remove_memory);
9d99aaa3 697#endif
05039b92 698
e18b890b
CL		 699struct kmem_cache *pgd_cache;
700struct kmem_cache *pmd_cache;
1da177e4
LT		 701
702void __init pgtable_cache_init(void)
703{
704 if (PTRS_PER_PMD > 1) {
705 pmd_cache = kmem_cache_create("pmd",
706 PTRS_PER_PMD*sizeof(pmd_t),
707 PTRS_PER_PMD*sizeof(pmd_t),
708 0,
709 pmd_ctor,
710 NULL);
711 if (!pmd_cache)
712 panic("pgtable_cache_init(): cannot create pmd cache");
713 }
714 pgd_cache = kmem_cache_create("pgd",
715 PTRS_PER_PGD*sizeof(pgd_t),
716 PTRS_PER_PGD*sizeof(pgd_t),
717 0,
718 pgd_ctor,
719 PTRS_PER_PMD == 1 ? pgd_dtor : NULL);
720 if (!pgd_cache)
721 panic("pgtable_cache_init(): Cannot create pgd cache");
722}
723
724/*
725 * This function cannot be __init, since exceptions don't work in that
726 * section. Put this after the callers, so that it cannot be inlined.
727 */
728static int noinline do_test_wp_bit(void)
729{
730 char tmp_reg;
731 int flag;
732
733 __asm__ __volatile__(
734 " movb %0,%1 \n"
735 "1: movb %1,%0 \n"
736 " xorl %2,%2 \n"
737 "2: \n"
738 ".section __ex_table,\"a\"\n"
739 " .align 4 \n"
740 " .long 1b,2b \n"
741 ".previous \n"
742 :"=m" (*(char *)fix_to_virt(FIX_WP_TEST)),
743 "=q" (tmp_reg),
744 "=r" (flag)
745 :"2" (1)
746 :"memory");
747
748 return flag;
749}
750
63aaf308
AV		 751#ifdef CONFIG_DEBUG_RODATA
752
63aaf308
AV		 753void mark_rodata_ro(void)
754{
6fb14755
JB		 755 unsigned long start = PFN_ALIGN(_text);
756 unsigned long size = PFN_ALIGN(_etext) - start;
63aaf308 757
6fb14755
JB		 758#ifdef CONFIG_HOTPLUG_CPU
759 /* It must still be possible to apply SMP alternatives. */
760 if (num_possible_cpus() <= 1)
761#endif
762 {
763 change_page_attr(virt_to_page(start),
764 size >> PAGE_SHIFT, PAGE_KERNEL_RX);
765 printk("Write protecting the kernel text: %luk\n", size >> 10);
766 }
63aaf308 767
6fb14755
JB		 768 start += size;
769 size = (unsigned long)__end_rodata - start;
770 change_page_attr(virt_to_page(start),
771 size >> PAGE_SHIFT, PAGE_KERNEL_RO);
772 printk("Write protecting the kernel read-only data: %luk\n",
773 size >> 10);
63aaf308
AV		 774
775 /*
776 * change_page_attr() requires a global_flush_tlb() call after it.
777 * We do this after the printk so that if something went wrong in the
778 * change, the printk gets out at least to give a better debug hint
779 * of who is the culprit.
780 */
781 global_flush_tlb();
782}
783#endif
784
9a0b5817
GH		 785void free_init_pages(char *what, unsigned long begin, unsigned long end)
786{
787 unsigned long addr;
788
789 for (addr = begin; addr < end; addr += PAGE_SIZE) {
0dbf7028
VG		 790 struct page *page = pfn_to_page(addr >> PAGE_SHIFT);
791 ClearPageReserved(page);
792 init_page_count(page);
793 memset(page_address(page), POISON_FREE_INITMEM, PAGE_SIZE);
794 __free_page(page);
9a0b5817
GH		 795 totalram_pages++;
796 }
6fb14755 797 printk(KERN_INFO "Freeing %s: %luk freed\n", what, (end - begin) >> 10);
9a0b5817
GH		 798}
799
800void free_initmem(void)
801{
802 free_init_pages("unused kernel memory",
0dbf7028
VG		 803 __pa_symbol(&__init_begin),
804 __pa_symbol(&__init_end));
9a0b5817 805}
63aaf308 806
1da177e4
LT		 807#ifdef CONFIG_BLK_DEV_INITRD
808void free_initrd_mem(unsigned long start, unsigned long end)
809{
0dbf7028 810 free_init_pages("initrd memory", __pa(start), __pa(end));
1da177e4
LT		 811}
812#endif
9a0b5817 813
Linux kernel - Deliverables
This page took 0.280648 seconds and 5 git commands to generate.