2 * Copyright IBM Corp. 2006
3 * Author(s): Heiko Carstens <heiko.carstens@de.ibm.com>
6 #include <linux/bootmem.h>
9 #include <linux/module.h>
10 #include <linux/list.h>
11 #include <linux/hugetlb.h>
12 #include <linux/slab.h>
13 #include <asm/pgalloc.h>
14 #include <asm/pgtable.h>
15 #include <asm/setup.h>
16 #include <asm/tlbflush.h>
17 #include <asm/sections.h>
19 static DEFINE_MUTEX(vmem_mutex
);
21 struct memory_segment
{
22 struct list_head list
;
27 static LIST_HEAD(mem_segs
);
29 static void __ref
*vmem_alloc_pages(unsigned int order
)
31 if (slab_is_available())
32 return (void *)__get_free_pages(GFP_KERNEL
, order
);
33 return alloc_bootmem_pages((1 << order
) * PAGE_SIZE
);
36 static inline pud_t
*vmem_pud_alloc(void)
41 pud
= vmem_alloc_pages(2);
44 clear_table((unsigned long *) pud
, _REGION3_ENTRY_EMPTY
, PAGE_SIZE
* 4);
49 static inline pmd_t
*vmem_pmd_alloc(void)
54 pmd
= vmem_alloc_pages(2);
57 clear_table((unsigned long *) pmd
, _SEGMENT_ENTRY_EMPTY
, PAGE_SIZE
* 4);
62 static pte_t __ref
*vmem_pte_alloc(unsigned long address
)
66 if (slab_is_available())
67 pte
= (pte_t
*) page_table_alloc(&init_mm
, address
);
69 pte
= alloc_bootmem(PTRS_PER_PTE
* sizeof(pte_t
));
72 clear_table((unsigned long *) pte
, _PAGE_TYPE_EMPTY
,
73 PTRS_PER_PTE
* sizeof(pte_t
));
78 * Add a physical memory range to the 1:1 mapping.
80 static int vmem_add_mem(unsigned long start
, unsigned long size
, int ro
)
82 unsigned long end
= start
+ size
;
83 unsigned long address
= start
;
90 while (address
< end
) {
91 pg_dir
= pgd_offset_k(address
);
92 if (pgd_none(*pg_dir
)) {
93 pu_dir
= vmem_pud_alloc();
96 pgd_populate(&init_mm
, pg_dir
, pu_dir
);
98 pu_dir
= pud_offset(pg_dir
, address
);
99 #if defined(CONFIG_64BIT) && !defined(CONFIG_DEBUG_PAGEALLOC)
100 if (MACHINE_HAS_EDAT2
&& pud_none(*pu_dir
) && address
&&
101 !(address
& ~PUD_MASK
) && (address
+ PUD_SIZE
<= end
)) {
102 pud_val(*pu_dir
) = __pa(address
) |
103 _REGION_ENTRY_TYPE_R3
| _REGION3_ENTRY_LARGE
|
104 (ro
? _REGION_ENTRY_RO
: 0);
109 if (pud_none(*pu_dir
)) {
110 pm_dir
= vmem_pmd_alloc();
113 pud_populate(&init_mm
, pu_dir
, pm_dir
);
115 pm_dir
= pmd_offset(pu_dir
, address
);
116 #if defined(CONFIG_64BIT) && !defined(CONFIG_DEBUG_PAGEALLOC)
117 if (MACHINE_HAS_EDAT1
&& pmd_none(*pm_dir
) && address
&&
118 !(address
& ~PMD_MASK
) && (address
+ PMD_SIZE
<= end
)) {
119 pmd_val(*pm_dir
) = __pa(address
) |
120 _SEGMENT_ENTRY
| _SEGMENT_ENTRY_LARGE
|
121 (ro
? _SEGMENT_ENTRY_RO
: 0);
126 if (pmd_none(*pm_dir
)) {
127 pt_dir
= vmem_pte_alloc(address
);
130 pmd_populate(&init_mm
, pm_dir
, pt_dir
);
133 pt_dir
= pte_offset_kernel(pm_dir
, address
);
134 pte_val(*pt_dir
) = __pa(address
) | (ro
? _PAGE_RO
: 0);
135 address
+= PAGE_SIZE
;
139 flush_tlb_kernel_range(start
, end
);
144 * Remove a physical memory range from the 1:1 mapping.
145 * Currently only invalidates page table entries.
147 static void vmem_remove_range(unsigned long start
, unsigned long size
)
149 unsigned long end
= start
+ size
;
150 unsigned long address
= start
;
157 pte_val(pte
) = _PAGE_TYPE_EMPTY
;
158 while (address
< end
) {
159 pg_dir
= pgd_offset_k(address
);
160 if (pgd_none(*pg_dir
)) {
161 address
+= PGDIR_SIZE
;
164 pu_dir
= pud_offset(pg_dir
, address
);
165 if (pud_none(*pu_dir
)) {
169 if (pud_large(*pu_dir
)) {
174 pm_dir
= pmd_offset(pu_dir
, address
);
175 if (pmd_none(*pm_dir
)) {
179 if (pmd_large(*pm_dir
)) {
184 pt_dir
= pte_offset_kernel(pm_dir
, address
);
186 address
+= PAGE_SIZE
;
188 flush_tlb_kernel_range(start
, end
);
192 * Add a backed mem_map array to the virtual mem_map array.
194 int __meminit
vmemmap_populate(struct page
*start
, unsigned long nr
, int node
)
196 unsigned long address
, start_addr
, end_addr
;
203 start_addr
= (unsigned long) start
;
204 end_addr
= (unsigned long) (start
+ nr
);
206 for (address
= start_addr
; address
< end_addr
;) {
207 pg_dir
= pgd_offset_k(address
);
208 if (pgd_none(*pg_dir
)) {
209 pu_dir
= vmem_pud_alloc();
212 pgd_populate(&init_mm
, pg_dir
, pu_dir
);
215 pu_dir
= pud_offset(pg_dir
, address
);
216 if (pud_none(*pu_dir
)) {
217 pm_dir
= vmem_pmd_alloc();
220 pud_populate(&init_mm
, pu_dir
, pm_dir
);
223 pm_dir
= pmd_offset(pu_dir
, address
);
224 if (pmd_none(*pm_dir
)) {
226 /* Use 1MB frames for vmemmap if available. We always
227 * use large frames even if they are only partially
229 * Otherwise we would have also page tables since
230 * vmemmap_populate gets called for each section
232 if (MACHINE_HAS_EDAT1
) {
235 new_page
= vmemmap_alloc_block(PMD_SIZE
, node
);
238 pmd_val(*pm_dir
) = __pa(new_page
) |
239 _SEGMENT_ENTRY
| _SEGMENT_ENTRY_LARGE
;
240 address
= (address
+ PMD_SIZE
) & PMD_MASK
;
244 pt_dir
= vmem_pte_alloc(address
);
247 pmd_populate(&init_mm
, pm_dir
, pt_dir
);
248 } else if (pmd_large(*pm_dir
)) {
249 address
= (address
+ PMD_SIZE
) & PMD_MASK
;
253 pt_dir
= pte_offset_kernel(pm_dir
, address
);
254 if (pte_none(*pt_dir
)) {
255 unsigned long new_page
;
257 new_page
=__pa(vmem_alloc_pages(0));
260 pte_val(*pt_dir
) = __pa(new_page
);
262 address
+= PAGE_SIZE
;
264 memset(start
, 0, nr
* sizeof(struct page
));
267 flush_tlb_kernel_range(start_addr
, end_addr
);
272 * Add memory segment to the segment list if it doesn't overlap with
273 * an already present segment.
275 static int insert_memory_segment(struct memory_segment
*seg
)
277 struct memory_segment
*tmp
;
279 if (seg
->start
+ seg
->size
> VMEM_MAX_PHYS
||
280 seg
->start
+ seg
->size
< seg
->start
)
283 list_for_each_entry(tmp
, &mem_segs
, list
) {
284 if (seg
->start
>= tmp
->start
+ tmp
->size
)
286 if (seg
->start
+ seg
->size
<= tmp
->start
)
290 list_add(&seg
->list
, &mem_segs
);
295 * Remove memory segment from the segment list.
297 static void remove_memory_segment(struct memory_segment
*seg
)
299 list_del(&seg
->list
);
302 static void __remove_shared_memory(struct memory_segment
*seg
)
304 remove_memory_segment(seg
);
305 vmem_remove_range(seg
->start
, seg
->size
);
308 int vmem_remove_mapping(unsigned long start
, unsigned long size
)
310 struct memory_segment
*seg
;
313 mutex_lock(&vmem_mutex
);
316 list_for_each_entry(seg
, &mem_segs
, list
) {
317 if (seg
->start
== start
&& seg
->size
== size
)
321 if (seg
->start
!= start
|| seg
->size
!= size
)
325 __remove_shared_memory(seg
);
328 mutex_unlock(&vmem_mutex
);
332 int vmem_add_mapping(unsigned long start
, unsigned long size
)
334 struct memory_segment
*seg
;
337 mutex_lock(&vmem_mutex
);
339 seg
= kzalloc(sizeof(*seg
), GFP_KERNEL
);
345 ret
= insert_memory_segment(seg
);
349 ret
= vmem_add_mem(start
, size
, 0);
355 __remove_shared_memory(seg
);
359 mutex_unlock(&vmem_mutex
);
364 * map whole physical memory to virtual memory (identity mapping)
365 * we reserve enough space in the vmalloc area for vmemmap to hotplug
366 * additional memory segments.
368 void __init
vmem_map_init(void)
370 unsigned long ro_start
, ro_end
;
371 unsigned long start
, end
;
374 ro_start
= PFN_ALIGN((unsigned long)&_stext
);
375 ro_end
= (unsigned long)&_eshared
& PAGE_MASK
;
376 for (i
= 0; i
< MEMORY_CHUNKS
&& memory_chunk
[i
].size
> 0; i
++) {
377 if (memory_chunk
[i
].type
== CHUNK_CRASHK
||
378 memory_chunk
[i
].type
== CHUNK_OLDMEM
)
380 start
= memory_chunk
[i
].addr
;
381 end
= memory_chunk
[i
].addr
+ memory_chunk
[i
].size
;
382 if (start
>= ro_end
|| end
<= ro_start
)
383 vmem_add_mem(start
, end
- start
, 0);
384 else if (start
>= ro_start
&& end
<= ro_end
)
385 vmem_add_mem(start
, end
- start
, 1);
386 else if (start
>= ro_start
) {
387 vmem_add_mem(start
, ro_end
- start
, 1);
388 vmem_add_mem(ro_end
, end
- ro_end
, 0);
389 } else if (end
< ro_end
) {
390 vmem_add_mem(start
, ro_start
- start
, 0);
391 vmem_add_mem(ro_start
, end
- ro_start
, 1);
393 vmem_add_mem(start
, ro_start
- start
, 0);
394 vmem_add_mem(ro_start
, ro_end
- ro_start
, 1);
395 vmem_add_mem(ro_end
, end
- ro_end
, 0);
401 * Convert memory chunk array to a memory segment list so there is a single
402 * list that contains both r/w memory and shared memory segments.
404 static int __init
vmem_convert_memory_chunk(void)
406 struct memory_segment
*seg
;
409 mutex_lock(&vmem_mutex
);
410 for (i
= 0; i
< MEMORY_CHUNKS
; i
++) {
411 if (!memory_chunk
[i
].size
)
413 if (memory_chunk
[i
].type
== CHUNK_CRASHK
||
414 memory_chunk
[i
].type
== CHUNK_OLDMEM
)
416 seg
= kzalloc(sizeof(*seg
), GFP_KERNEL
);
418 panic("Out of memory...\n");
419 seg
->start
= memory_chunk
[i
].addr
;
420 seg
->size
= memory_chunk
[i
].size
;
421 insert_memory_segment(seg
);
423 mutex_unlock(&vmem_mutex
);
427 core_initcall(vmem_convert_memory_chunk
);