Commit | Line | Data |
---|---|---|
f4eb07c1 | 1 | /* |
f4eb07c1 HC |
2 | * Copyright IBM Corp. 2006 |
3 | * Author(s): Heiko Carstens <heiko.carstens@de.ibm.com> | |
4 | */ | |
5 | ||
6 | #include <linux/bootmem.h> | |
7 | #include <linux/pfn.h> | |
8 | #include <linux/mm.h> | |
9 | #include <linux/module.h> | |
10 | #include <linux/list.h> | |
53492b1d | 11 | #include <linux/hugetlb.h> |
5a0e3ad6 | 12 | #include <linux/slab.h> |
f4eb07c1 HC |
13 | #include <asm/pgalloc.h> |
14 | #include <asm/pgtable.h> | |
15 | #include <asm/setup.h> | |
16 | #include <asm/tlbflush.h> | |
53492b1d | 17 | #include <asm/sections.h> |
f4eb07c1 | 18 | |
f4eb07c1 HC |
19 | static DEFINE_MUTEX(vmem_mutex); |
20 | ||
21 | struct memory_segment { | |
22 | struct list_head list; | |
23 | unsigned long start; | |
24 | unsigned long size; | |
25 | }; | |
26 | ||
27 | static LIST_HEAD(mem_segs); | |
28 | ||
67060d9c HC |
29 | static void __ref *vmem_alloc_pages(unsigned int order) |
30 | { | |
31 | if (slab_is_available()) | |
32 | return (void *)__get_free_pages(GFP_KERNEL, order); | |
33 | return alloc_bootmem_pages((1 << order) * PAGE_SIZE); | |
34 | } | |
35 | ||
36 | static inline pud_t *vmem_pud_alloc(void) | |
5a216a20 MS |
37 | { |
38 | pud_t *pud = NULL; | |
39 | ||
40 | #ifdef CONFIG_64BIT | |
67060d9c | 41 | pud = vmem_alloc_pages(2); |
5a216a20 MS |
42 | if (!pud) |
43 | return NULL; | |
8fc63658 | 44 | clear_table((unsigned long *) pud, _REGION3_ENTRY_EMPTY, PAGE_SIZE * 4); |
5a216a20 MS |
45 | #endif |
46 | return pud; | |
47 | } | |
190a1d72 | 48 | |
67060d9c | 49 | static inline pmd_t *vmem_pmd_alloc(void) |
f4eb07c1 | 50 | { |
3610cce8 | 51 | pmd_t *pmd = NULL; |
f4eb07c1 | 52 | |
3610cce8 | 53 | #ifdef CONFIG_64BIT |
67060d9c | 54 | pmd = vmem_alloc_pages(2); |
f4eb07c1 HC |
55 | if (!pmd) |
56 | return NULL; | |
8fc63658 | 57 | clear_table((unsigned long *) pmd, _SEGMENT_ENTRY_EMPTY, PAGE_SIZE * 4); |
3610cce8 | 58 | #endif |
f4eb07c1 HC |
59 | return pmd; |
60 | } | |
61 | ||
e5992f2e | 62 | static pte_t __ref *vmem_pte_alloc(unsigned long address) |
f4eb07c1 | 63 | { |
146e4b3c | 64 | pte_t *pte; |
f4eb07c1 | 65 | |
146e4b3c | 66 | if (slab_is_available()) |
e5992f2e | 67 | pte = (pte_t *) page_table_alloc(&init_mm, address); |
146e4b3c MS |
68 | else |
69 | pte = alloc_bootmem(PTRS_PER_PTE * sizeof(pte_t)); | |
f4eb07c1 HC |
70 | if (!pte) |
71 | return NULL; | |
6af7eea2 CB |
72 | clear_table((unsigned long *) pte, _PAGE_TYPE_EMPTY, |
73 | PTRS_PER_PTE * sizeof(pte_t)); | |
f4eb07c1 HC |
74 | return pte; |
75 | } | |
76 | ||
77 | /* | |
78 | * Add a physical memory range to the 1:1 mapping. | |
79 | */ | |
17f34580 | 80 | static int vmem_add_mem(unsigned long start, unsigned long size, int ro) |
f4eb07c1 | 81 | { |
378b1e7a HC |
82 | unsigned long end = start + size; |
83 | unsigned long address = start; | |
f4eb07c1 | 84 | pgd_t *pg_dir; |
190a1d72 | 85 | pud_t *pu_dir; |
f4eb07c1 HC |
86 | pmd_t *pm_dir; |
87 | pte_t *pt_dir; | |
88 | pte_t pte; | |
89 | int ret = -ENOMEM; | |
90 | ||
378b1e7a | 91 | while (address < end) { |
f4eb07c1 HC |
92 | pg_dir = pgd_offset_k(address); |
93 | if (pgd_none(*pg_dir)) { | |
190a1d72 MS |
94 | pu_dir = vmem_pud_alloc(); |
95 | if (!pu_dir) | |
96 | goto out; | |
b2fa47e6 | 97 | pgd_populate(&init_mm, pg_dir, pu_dir); |
190a1d72 MS |
98 | } |
99 | ||
100 | pu_dir = pud_offset(pg_dir, address); | |
101 | if (pud_none(*pu_dir)) { | |
f4eb07c1 HC |
102 | pm_dir = vmem_pmd_alloc(); |
103 | if (!pm_dir) | |
104 | goto out; | |
b2fa47e6 | 105 | pud_populate(&init_mm, pu_dir, pm_dir); |
f4eb07c1 HC |
106 | } |
107 | ||
53492b1d | 108 | pte = mk_pte_phys(address, __pgprot(ro ? _PAGE_RO : 0)); |
190a1d72 | 109 | pm_dir = pmd_offset(pu_dir, address); |
53492b1d | 110 | |
648609e3 | 111 | #if defined(CONFIG_64BIT) && !defined(CONFIG_DEBUG_PAGEALLOC) |
378b1e7a HC |
112 | if (MACHINE_HAS_EDAT1 && address && !(address & ~PMD_MASK) && |
113 | (address + PMD_SIZE <= end)) { | |
6af7eea2 | 114 | pte_val(pte) |= _SEGMENT_ENTRY_LARGE; |
53492b1d | 115 | pmd_val(*pm_dir) = pte_val(pte); |
378b1e7a | 116 | address += PMD_SIZE; |
53492b1d GS |
117 | continue; |
118 | } | |
119 | #endif | |
f4eb07c1 | 120 | if (pmd_none(*pm_dir)) { |
e5992f2e | 121 | pt_dir = vmem_pte_alloc(address); |
f4eb07c1 HC |
122 | if (!pt_dir) |
123 | goto out; | |
b2fa47e6 | 124 | pmd_populate(&init_mm, pm_dir, pt_dir); |
f4eb07c1 HC |
125 | } |
126 | ||
127 | pt_dir = pte_offset_kernel(pm_dir, address); | |
c1821c2e | 128 | *pt_dir = pte; |
378b1e7a | 129 | address += PAGE_SIZE; |
f4eb07c1 HC |
130 | } |
131 | ret = 0; | |
132 | out: | |
378b1e7a | 133 | flush_tlb_kernel_range(start, end); |
f4eb07c1 HC |
134 | return ret; |
135 | } | |
136 | ||
137 | /* | |
138 | * Remove a physical memory range from the 1:1 mapping. | |
139 | * Currently only invalidates page table entries. | |
140 | */ | |
141 | static void vmem_remove_range(unsigned long start, unsigned long size) | |
142 | { | |
378b1e7a HC |
143 | unsigned long end = start + size; |
144 | unsigned long address = start; | |
f4eb07c1 | 145 | pgd_t *pg_dir; |
190a1d72 | 146 | pud_t *pu_dir; |
f4eb07c1 HC |
147 | pmd_t *pm_dir; |
148 | pte_t *pt_dir; | |
149 | pte_t pte; | |
150 | ||
151 | pte_val(pte) = _PAGE_TYPE_EMPTY; | |
378b1e7a | 152 | while (address < end) { |
f4eb07c1 | 153 | pg_dir = pgd_offset_k(address); |
190a1d72 MS |
154 | pu_dir = pud_offset(pg_dir, address); |
155 | if (pud_none(*pu_dir)) | |
f4eb07c1 | 156 | continue; |
190a1d72 | 157 | pm_dir = pmd_offset(pu_dir, address); |
f4eb07c1 HC |
158 | if (pmd_none(*pm_dir)) |
159 | continue; | |
378b1e7a | 160 | if (pmd_large(*pm_dir)) { |
b2fa47e6 | 161 | pmd_clear(pm_dir); |
378b1e7a | 162 | address += PMD_SIZE; |
53492b1d GS |
163 | continue; |
164 | } | |
f4eb07c1 | 165 | pt_dir = pte_offset_kernel(pm_dir, address); |
c1821c2e | 166 | *pt_dir = pte; |
378b1e7a | 167 | address += PAGE_SIZE; |
f4eb07c1 | 168 | } |
378b1e7a | 169 | flush_tlb_kernel_range(start, end); |
f4eb07c1 HC |
170 | } |
171 | ||
172 | /* | |
173 | * Add a backed mem_map array to the virtual mem_map array. | |
174 | */ | |
17f34580 | 175 | int __meminit vmemmap_populate(struct page *start, unsigned long nr, int node) |
f4eb07c1 HC |
176 | { |
177 | unsigned long address, start_addr, end_addr; | |
f4eb07c1 | 178 | pgd_t *pg_dir; |
190a1d72 | 179 | pud_t *pu_dir; |
f4eb07c1 HC |
180 | pmd_t *pm_dir; |
181 | pte_t *pt_dir; | |
182 | pte_t pte; | |
183 | int ret = -ENOMEM; | |
184 | ||
17f34580 HC |
185 | start_addr = (unsigned long) start; |
186 | end_addr = (unsigned long) (start + nr); | |
f4eb07c1 HC |
187 | |
188 | for (address = start_addr; address < end_addr; address += PAGE_SIZE) { | |
189 | pg_dir = pgd_offset_k(address); | |
190 | if (pgd_none(*pg_dir)) { | |
190a1d72 MS |
191 | pu_dir = vmem_pud_alloc(); |
192 | if (!pu_dir) | |
193 | goto out; | |
b2fa47e6 | 194 | pgd_populate(&init_mm, pg_dir, pu_dir); |
190a1d72 MS |
195 | } |
196 | ||
197 | pu_dir = pud_offset(pg_dir, address); | |
198 | if (pud_none(*pu_dir)) { | |
f4eb07c1 HC |
199 | pm_dir = vmem_pmd_alloc(); |
200 | if (!pm_dir) | |
201 | goto out; | |
b2fa47e6 | 202 | pud_populate(&init_mm, pu_dir, pm_dir); |
f4eb07c1 HC |
203 | } |
204 | ||
190a1d72 | 205 | pm_dir = pmd_offset(pu_dir, address); |
f4eb07c1 | 206 | if (pmd_none(*pm_dir)) { |
e5992f2e | 207 | pt_dir = vmem_pte_alloc(address); |
f4eb07c1 HC |
208 | if (!pt_dir) |
209 | goto out; | |
b2fa47e6 | 210 | pmd_populate(&init_mm, pm_dir, pt_dir); |
f4eb07c1 HC |
211 | } |
212 | ||
213 | pt_dir = pte_offset_kernel(pm_dir, address); | |
214 | if (pte_none(*pt_dir)) { | |
215 | unsigned long new_page; | |
216 | ||
67060d9c | 217 | new_page =__pa(vmem_alloc_pages(0)); |
f4eb07c1 HC |
218 | if (!new_page) |
219 | goto out; | |
220 | pte = pfn_pte(new_page >> PAGE_SHIFT, PAGE_KERNEL); | |
c1821c2e | 221 | *pt_dir = pte; |
f4eb07c1 HC |
222 | } |
223 | } | |
67060d9c | 224 | memset(start, 0, nr * sizeof(struct page)); |
f4eb07c1 HC |
225 | ret = 0; |
226 | out: | |
227 | flush_tlb_kernel_range(start_addr, end_addr); | |
228 | return ret; | |
229 | } | |
230 | ||
f4eb07c1 HC |
231 | /* |
232 | * Add memory segment to the segment list if it doesn't overlap with | |
233 | * an already present segment. | |
234 | */ | |
235 | static int insert_memory_segment(struct memory_segment *seg) | |
236 | { | |
237 | struct memory_segment *tmp; | |
238 | ||
ee0ddadd | 239 | if (seg->start + seg->size > VMEM_MAX_PHYS || |
f4eb07c1 HC |
240 | seg->start + seg->size < seg->start) |
241 | return -ERANGE; | |
242 | ||
243 | list_for_each_entry(tmp, &mem_segs, list) { | |
244 | if (seg->start >= tmp->start + tmp->size) | |
245 | continue; | |
246 | if (seg->start + seg->size <= tmp->start) | |
247 | continue; | |
248 | return -ENOSPC; | |
249 | } | |
250 | list_add(&seg->list, &mem_segs); | |
251 | return 0; | |
252 | } | |
253 | ||
254 | /* | |
255 | * Remove memory segment from the segment list. | |
256 | */ | |
257 | static void remove_memory_segment(struct memory_segment *seg) | |
258 | { | |
259 | list_del(&seg->list); | |
260 | } | |
261 | ||
262 | static void __remove_shared_memory(struct memory_segment *seg) | |
263 | { | |
264 | remove_memory_segment(seg); | |
265 | vmem_remove_range(seg->start, seg->size); | |
266 | } | |
267 | ||
17f34580 | 268 | int vmem_remove_mapping(unsigned long start, unsigned long size) |
f4eb07c1 HC |
269 | { |
270 | struct memory_segment *seg; | |
271 | int ret; | |
272 | ||
273 | mutex_lock(&vmem_mutex); | |
274 | ||
275 | ret = -ENOENT; | |
276 | list_for_each_entry(seg, &mem_segs, list) { | |
277 | if (seg->start == start && seg->size == size) | |
278 | break; | |
279 | } | |
280 | ||
281 | if (seg->start != start || seg->size != size) | |
282 | goto out; | |
283 | ||
284 | ret = 0; | |
285 | __remove_shared_memory(seg); | |
286 | kfree(seg); | |
287 | out: | |
288 | mutex_unlock(&vmem_mutex); | |
289 | return ret; | |
290 | } | |
291 | ||
17f34580 | 292 | int vmem_add_mapping(unsigned long start, unsigned long size) |
f4eb07c1 HC |
293 | { |
294 | struct memory_segment *seg; | |
f4eb07c1 HC |
295 | int ret; |
296 | ||
297 | mutex_lock(&vmem_mutex); | |
298 | ret = -ENOMEM; | |
299 | seg = kzalloc(sizeof(*seg), GFP_KERNEL); | |
300 | if (!seg) | |
301 | goto out; | |
302 | seg->start = start; | |
303 | seg->size = size; | |
304 | ||
305 | ret = insert_memory_segment(seg); | |
306 | if (ret) | |
307 | goto out_free; | |
308 | ||
53492b1d | 309 | ret = vmem_add_mem(start, size, 0); |
f4eb07c1 HC |
310 | if (ret) |
311 | goto out_remove; | |
f4eb07c1 HC |
312 | goto out; |
313 | ||
314 | out_remove: | |
315 | __remove_shared_memory(seg); | |
316 | out_free: | |
317 | kfree(seg); | |
318 | out: | |
319 | mutex_unlock(&vmem_mutex); | |
320 | return ret; | |
321 | } | |
322 | ||
323 | /* | |
324 | * map whole physical memory to virtual memory (identity mapping) | |
5fd9c6e2 CB |
325 | * we reserve enough space in the vmalloc area for vmemmap to hotplug |
326 | * additional memory segments. | |
f4eb07c1 HC |
327 | */ |
328 | void __init vmem_map_init(void) | |
329 | { | |
53492b1d GS |
330 | unsigned long ro_start, ro_end; |
331 | unsigned long start, end; | |
f4eb07c1 HC |
332 | int i; |
333 | ||
8fe234d3 HC |
334 | ro_start = PFN_ALIGN((unsigned long)&_stext); |
335 | ro_end = (unsigned long)&_eshared & PAGE_MASK; | |
53492b1d | 336 | for (i = 0; i < MEMORY_CHUNKS && memory_chunk[i].size > 0; i++) { |
60a0c68d MH |
337 | if (memory_chunk[i].type == CHUNK_CRASHK || |
338 | memory_chunk[i].type == CHUNK_OLDMEM) | |
339 | continue; | |
53492b1d GS |
340 | start = memory_chunk[i].addr; |
341 | end = memory_chunk[i].addr + memory_chunk[i].size; | |
342 | if (start >= ro_end || end <= ro_start) | |
343 | vmem_add_mem(start, end - start, 0); | |
344 | else if (start >= ro_start && end <= ro_end) | |
345 | vmem_add_mem(start, end - start, 1); | |
346 | else if (start >= ro_start) { | |
347 | vmem_add_mem(start, ro_end - start, 1); | |
348 | vmem_add_mem(ro_end, end - ro_end, 0); | |
349 | } else if (end < ro_end) { | |
350 | vmem_add_mem(start, ro_start - start, 0); | |
351 | vmem_add_mem(ro_start, end - ro_start, 1); | |
352 | } else { | |
353 | vmem_add_mem(start, ro_start - start, 0); | |
354 | vmem_add_mem(ro_start, ro_end - ro_start, 1); | |
355 | vmem_add_mem(ro_end, end - ro_end, 0); | |
356 | } | |
357 | } | |
f4eb07c1 HC |
358 | } |
359 | ||
360 | /* | |
361 | * Convert memory chunk array to a memory segment list so there is a single | |
362 | * list that contains both r/w memory and shared memory segments. | |
363 | */ | |
364 | static int __init vmem_convert_memory_chunk(void) | |
365 | { | |
366 | struct memory_segment *seg; | |
367 | int i; | |
368 | ||
369 | mutex_lock(&vmem_mutex); | |
9f4b0ba8 | 370 | for (i = 0; i < MEMORY_CHUNKS; i++) { |
f4eb07c1 HC |
371 | if (!memory_chunk[i].size) |
372 | continue; | |
60a0c68d MH |
373 | if (memory_chunk[i].type == CHUNK_CRASHK || |
374 | memory_chunk[i].type == CHUNK_OLDMEM) | |
375 | continue; | |
f4eb07c1 HC |
376 | seg = kzalloc(sizeof(*seg), GFP_KERNEL); |
377 | if (!seg) | |
378 | panic("Out of memory...\n"); | |
379 | seg->start = memory_chunk[i].addr; | |
380 | seg->size = memory_chunk[i].size; | |
381 | insert_memory_segment(seg); | |
382 | } | |
383 | mutex_unlock(&vmem_mutex); | |
384 | return 0; | |
385 | } | |
386 | ||
387 | core_initcall(vmem_convert_memory_chunk); |