Commit | Line | Data |
---|---|---|
1da177e4 | 1 | /* |
57cfc29e | 2 | * bootmem - A boot-time physical memory allocator and configurator |
1da177e4 LT |
3 | * |
4 | * Copyright (C) 1999 Ingo Molnar | |
57cfc29e JW |
5 | * 1999 Kanoj Sarcar, SGI |
6 | * 2008 Johannes Weiner | |
1da177e4 | 7 | * |
57cfc29e JW |
8 | * Access to this subsystem has to be serialized externally (which is true |
9 | * for the boot process anyway). | |
1da177e4 | 10 | */ |
1da177e4 | 11 | #include <linux/init.h> |
bbc7b92e | 12 | #include <linux/pfn.h> |
1da177e4 | 13 | #include <linux/bootmem.h> |
1da177e4 | 14 | #include <linux/module.h> |
e786e86a FBH |
15 | |
16 | #include <asm/bug.h> | |
1da177e4 | 17 | #include <asm/io.h> |
dfd54cbc | 18 | #include <asm/processor.h> |
e786e86a | 19 | |
1da177e4 LT |
20 | #include "internal.h" |
21 | ||
1da177e4 LT |
22 | unsigned long max_low_pfn; |
23 | unsigned long min_low_pfn; | |
24 | unsigned long max_pfn; | |
25 | ||
679bc9fb | 26 | static LIST_HEAD(bdata_list); |
92aa63a5 VG |
27 | #ifdef CONFIG_CRASH_DUMP |
28 | /* | |
29 | * If we have booted due to a crash, max_pfn will be a very low value. We need | |
30 | * to know the amount of memory that the previous kernel used. | |
31 | */ | |
32 | unsigned long saved_max_pfn; | |
33 | #endif | |
34 | ||
b61bfa3c JW |
35 | bootmem_data_t bootmem_node_data[MAX_NUMNODES] __initdata; |
36 | ||
223e8dc9 JW |
37 | /* |
38 | * Given an initialised bdata, it returns the size of the boot bitmap | |
39 | */ | |
40 | static unsigned long __init get_mapsize(bootmem_data_t *bdata) | |
41 | { | |
42 | unsigned long mapsize; | |
43 | unsigned long start = PFN_DOWN(bdata->node_boot_start); | |
44 | unsigned long end = bdata->node_low_pfn; | |
45 | ||
46 | mapsize = ((end - start) + 7) / 8; | |
47 | return ALIGN(mapsize, sizeof(long)); | |
48 | } | |
49 | ||
a66fd7da JW |
50 | /** |
51 | * bootmem_bootmap_pages - calculate bitmap size in pages | |
52 | * @pages: number of pages the bitmap has to represent | |
53 | */ | |
f71bf0ca | 54 | unsigned long __init bootmem_bootmap_pages(unsigned long pages) |
1da177e4 LT |
55 | { |
56 | unsigned long mapsize; | |
57 | ||
58 | mapsize = (pages+7)/8; | |
59 | mapsize = (mapsize + ~PAGE_MASK) & PAGE_MASK; | |
60 | mapsize >>= PAGE_SHIFT; | |
61 | ||
62 | return mapsize; | |
63 | } | |
f71bf0ca | 64 | |
679bc9fb KH |
65 | /* |
66 | * link bdata in order | |
67 | */ | |
69d49e68 | 68 | static void __init link_bootmem(bootmem_data_t *bdata) |
679bc9fb KH |
69 | { |
70 | bootmem_data_t *ent; | |
f71bf0ca | 71 | |
679bc9fb KH |
72 | if (list_empty(&bdata_list)) { |
73 | list_add(&bdata->list, &bdata_list); | |
74 | return; | |
75 | } | |
76 | /* insert in order */ | |
77 | list_for_each_entry(ent, &bdata_list, list) { | |
78 | if (bdata->node_boot_start < ent->node_boot_start) { | |
79 | list_add_tail(&bdata->list, &ent->list); | |
80 | return; | |
81 | } | |
82 | } | |
83 | list_add_tail(&bdata->list, &bdata_list); | |
679bc9fb KH |
84 | } |
85 | ||
1da177e4 LT |
86 | /* |
87 | * Called once to set up the allocator itself. | |
88 | */ | |
8ae04463 | 89 | static unsigned long __init init_bootmem_core(bootmem_data_t *bdata, |
1da177e4 LT |
90 | unsigned long mapstart, unsigned long start, unsigned long end) |
91 | { | |
bbc7b92e | 92 | unsigned long mapsize; |
1da177e4 | 93 | |
2dbb51c4 | 94 | mminit_validate_memmodel_limits(&start, &end); |
bbc7b92e FBH |
95 | bdata->node_bootmem_map = phys_to_virt(PFN_PHYS(mapstart)); |
96 | bdata->node_boot_start = PFN_PHYS(start); | |
1da177e4 | 97 | bdata->node_low_pfn = end; |
679bc9fb | 98 | link_bootmem(bdata); |
1da177e4 LT |
99 | |
100 | /* | |
101 | * Initially all pages are reserved - setup_arch() has to | |
102 | * register free RAM areas explicitly. | |
103 | */ | |
bbc7b92e | 104 | mapsize = get_mapsize(bdata); |
1da177e4 LT |
105 | memset(bdata->node_bootmem_map, 0xff, mapsize); |
106 | ||
107 | return mapsize; | |
108 | } | |
109 | ||
a66fd7da JW |
110 | /** |
111 | * init_bootmem_node - register a node as boot memory | |
112 | * @pgdat: node to register | |
113 | * @freepfn: pfn where the bitmap for this node is to be placed | |
114 | * @startpfn: first pfn on the node | |
115 | * @endpfn: first pfn after the node | |
116 | * | |
117 | * Returns the number of bytes needed to hold the bitmap for this node. | |
118 | */ | |
223e8dc9 JW |
119 | unsigned long __init init_bootmem_node(pg_data_t *pgdat, unsigned long freepfn, |
120 | unsigned long startpfn, unsigned long endpfn) | |
121 | { | |
122 | return init_bootmem_core(pgdat->bdata, freepfn, startpfn, endpfn); | |
123 | } | |
124 | ||
a66fd7da JW |
125 | /** |
126 | * init_bootmem - register boot memory | |
127 | * @start: pfn where the bitmap is to be placed | |
128 | * @pages: number of available physical pages | |
129 | * | |
130 | * Returns the number of bytes needed to hold the bitmap. | |
131 | */ | |
223e8dc9 JW |
132 | unsigned long __init init_bootmem(unsigned long start, unsigned long pages) |
133 | { | |
134 | max_low_pfn = pages; | |
135 | min_low_pfn = start; | |
136 | return init_bootmem_core(NODE_DATA(0)->bdata, start, 0, pages); | |
137 | } | |
138 | ||
139 | static unsigned long __init free_all_bootmem_core(bootmem_data_t *bdata) | |
140 | { | |
141 | struct page *page; | |
142 | unsigned long pfn; | |
143 | unsigned long i, count; | |
144 | unsigned long idx; | |
145 | unsigned long *map; | |
146 | int gofast = 0; | |
147 | ||
148 | BUG_ON(!bdata->node_bootmem_map); | |
149 | ||
150 | count = 0; | |
151 | /* first extant page of the node */ | |
152 | pfn = PFN_DOWN(bdata->node_boot_start); | |
153 | idx = bdata->node_low_pfn - pfn; | |
154 | map = bdata->node_bootmem_map; | |
155 | /* | |
156 | * Check if we are aligned to BITS_PER_LONG pages. If so, we might | |
157 | * be able to free page orders of that size at once. | |
158 | */ | |
159 | if (!(pfn & (BITS_PER_LONG-1))) | |
160 | gofast = 1; | |
161 | ||
162 | for (i = 0; i < idx; ) { | |
163 | unsigned long v = ~map[i / BITS_PER_LONG]; | |
164 | ||
165 | if (gofast && v == ~0UL) { | |
166 | int order; | |
167 | ||
168 | page = pfn_to_page(pfn); | |
169 | count += BITS_PER_LONG; | |
170 | order = ffs(BITS_PER_LONG) - 1; | |
171 | __free_pages_bootmem(page, order); | |
172 | i += BITS_PER_LONG; | |
173 | page += BITS_PER_LONG; | |
174 | } else if (v) { | |
175 | unsigned long m; | |
176 | ||
177 | page = pfn_to_page(pfn); | |
178 | for (m = 1; m && i < idx; m<<=1, page++, i++) { | |
179 | if (v & m) { | |
180 | count++; | |
181 | __free_pages_bootmem(page, 0); | |
182 | } | |
183 | } | |
184 | } else { | |
185 | i += BITS_PER_LONG; | |
186 | } | |
187 | pfn += BITS_PER_LONG; | |
188 | } | |
189 | ||
190 | /* | |
191 | * Now free the allocator bitmap itself, it's not | |
192 | * needed anymore: | |
193 | */ | |
194 | page = virt_to_page(bdata->node_bootmem_map); | |
195 | idx = (get_mapsize(bdata) + PAGE_SIZE-1) >> PAGE_SHIFT; | |
196 | for (i = 0; i < idx; i++, page++) | |
197 | __free_pages_bootmem(page, 0); | |
198 | count += i; | |
199 | bdata->node_bootmem_map = NULL; | |
200 | ||
201 | return count; | |
202 | } | |
203 | ||
a66fd7da JW |
204 | /** |
205 | * free_all_bootmem_node - release a node's free pages to the buddy allocator | |
206 | * @pgdat: node to be released | |
207 | * | |
208 | * Returns the number of pages actually released. | |
209 | */ | |
223e8dc9 JW |
210 | unsigned long __init free_all_bootmem_node(pg_data_t *pgdat) |
211 | { | |
212 | register_page_bootmem_info_node(pgdat); | |
213 | return free_all_bootmem_core(pgdat->bdata); | |
214 | } | |
215 | ||
a66fd7da JW |
216 | /** |
217 | * free_all_bootmem - release free pages to the buddy allocator | |
218 | * | |
219 | * Returns the number of pages actually released. | |
220 | */ | |
223e8dc9 JW |
221 | unsigned long __init free_all_bootmem(void) |
222 | { | |
223 | return free_all_bootmem_core(NODE_DATA(0)->bdata); | |
224 | } | |
225 | ||
226 | static void __init free_bootmem_core(bootmem_data_t *bdata, unsigned long addr, | |
227 | unsigned long size) | |
228 | { | |
229 | unsigned long sidx, eidx; | |
230 | unsigned long i; | |
231 | ||
232 | BUG_ON(!size); | |
233 | ||
234 | /* out range */ | |
235 | if (addr + size < bdata->node_boot_start || | |
236 | PFN_DOWN(addr) > bdata->node_low_pfn) | |
237 | return; | |
238 | /* | |
239 | * round down end of usable mem, partially free pages are | |
240 | * considered reserved. | |
241 | */ | |
242 | ||
243 | if (addr >= bdata->node_boot_start && addr < bdata->last_success) | |
244 | bdata->last_success = addr; | |
245 | ||
246 | /* | |
247 | * Round up to index to the range. | |
248 | */ | |
249 | if (PFN_UP(addr) > PFN_DOWN(bdata->node_boot_start)) | |
250 | sidx = PFN_UP(addr) - PFN_DOWN(bdata->node_boot_start); | |
251 | else | |
252 | sidx = 0; | |
253 | ||
254 | eidx = PFN_DOWN(addr + size - bdata->node_boot_start); | |
255 | if (eidx > bdata->node_low_pfn - PFN_DOWN(bdata->node_boot_start)) | |
256 | eidx = bdata->node_low_pfn - PFN_DOWN(bdata->node_boot_start); | |
257 | ||
258 | for (i = sidx; i < eidx; i++) { | |
259 | if (unlikely(!test_and_clear_bit(i, bdata->node_bootmem_map))) | |
260 | BUG(); | |
261 | } | |
262 | } | |
263 | ||
a66fd7da JW |
264 | /** |
265 | * free_bootmem_node - mark a page range as usable | |
266 | * @pgdat: node the range resides on | |
267 | * @physaddr: starting address of the range | |
268 | * @size: size of the range in bytes | |
269 | * | |
270 | * Partial pages will be considered reserved and left as they are. | |
271 | * | |
272 | * Only physical pages that actually reside on @pgdat are marked. | |
273 | */ | |
223e8dc9 JW |
274 | void __init free_bootmem_node(pg_data_t *pgdat, unsigned long physaddr, |
275 | unsigned long size) | |
276 | { | |
277 | free_bootmem_core(pgdat->bdata, physaddr, size); | |
278 | } | |
279 | ||
a66fd7da JW |
280 | /** |
281 | * free_bootmem - mark a page range as usable | |
282 | * @addr: starting address of the range | |
283 | * @size: size of the range in bytes | |
284 | * | |
285 | * Partial pages will be considered reserved and left as they are. | |
286 | * | |
287 | * All physical pages within the range are marked, no matter what | |
288 | * node they reside on. | |
289 | */ | |
223e8dc9 JW |
290 | void __init free_bootmem(unsigned long addr, unsigned long size) |
291 | { | |
292 | bootmem_data_t *bdata; | |
293 | list_for_each_entry(bdata, &bdata_list, list) | |
294 | free_bootmem_core(bdata, addr, size); | |
295 | } | |
296 | ||
1da177e4 LT |
297 | /* |
298 | * Marks a particular physical memory range as unallocatable. Usable RAM | |
299 | * might be used for boot-time allocations - or it might get added | |
300 | * to the free page pool later on. | |
301 | */ | |
a5645a61 | 302 | static int __init can_reserve_bootmem_core(bootmem_data_t *bdata, |
72a7fe39 | 303 | unsigned long addr, unsigned long size, int flags) |
1da177e4 | 304 | { |
bbc7b92e | 305 | unsigned long sidx, eidx; |
1da177e4 | 306 | unsigned long i; |
a5645a61 YL |
307 | |
308 | BUG_ON(!size); | |
309 | ||
310 | /* out of range, don't hold other */ | |
311 | if (addr + size < bdata->node_boot_start || | |
312 | PFN_DOWN(addr) > bdata->node_low_pfn) | |
313 | return 0; | |
bbc7b92e | 314 | |
1da177e4 | 315 | /* |
a5645a61 | 316 | * Round up to index to the range. |
1da177e4 | 317 | */ |
a5645a61 YL |
318 | if (addr > bdata->node_boot_start) |
319 | sidx= PFN_DOWN(addr - bdata->node_boot_start); | |
320 | else | |
321 | sidx = 0; | |
322 | ||
323 | eidx = PFN_UP(addr + size - bdata->node_boot_start); | |
324 | if (eidx > bdata->node_low_pfn - PFN_DOWN(bdata->node_boot_start)) | |
325 | eidx = bdata->node_low_pfn - PFN_DOWN(bdata->node_boot_start); | |
326 | ||
327 | for (i = sidx; i < eidx; i++) { | |
328 | if (test_bit(i, bdata->node_bootmem_map)) { | |
329 | if (flags & BOOTMEM_EXCLUSIVE) | |
330 | return -EBUSY; | |
331 | } | |
332 | } | |
333 | ||
334 | return 0; | |
335 | ||
336 | } | |
337 | ||
338 | static void __init reserve_bootmem_core(bootmem_data_t *bdata, | |
339 | unsigned long addr, unsigned long size, int flags) | |
340 | { | |
341 | unsigned long sidx, eidx; | |
342 | unsigned long i; | |
343 | ||
1da177e4 | 344 | BUG_ON(!size); |
bbc7b92e | 345 | |
a5645a61 YL |
346 | /* out of range */ |
347 | if (addr + size < bdata->node_boot_start || | |
348 | PFN_DOWN(addr) > bdata->node_low_pfn) | |
349 | return; | |
350 | ||
351 | /* | |
352 | * Round up to index to the range. | |
353 | */ | |
354 | if (addr > bdata->node_boot_start) | |
355 | sidx= PFN_DOWN(addr - bdata->node_boot_start); | |
356 | else | |
357 | sidx = 0; | |
358 | ||
bbc7b92e | 359 | eidx = PFN_UP(addr + size - bdata->node_boot_start); |
a5645a61 YL |
360 | if (eidx > bdata->node_low_pfn - PFN_DOWN(bdata->node_boot_start)) |
361 | eidx = bdata->node_low_pfn - PFN_DOWN(bdata->node_boot_start); | |
1da177e4 | 362 | |
a5645a61 | 363 | for (i = sidx; i < eidx; i++) { |
1da177e4 LT |
364 | if (test_and_set_bit(i, bdata->node_bootmem_map)) { |
365 | #ifdef CONFIG_DEBUG_BOOTMEM | |
366 | printk("hm, page %08lx reserved twice.\n", i*PAGE_SIZE); | |
367 | #endif | |
368 | } | |
a5645a61 | 369 | } |
1da177e4 LT |
370 | } |
371 | ||
a66fd7da JW |
372 | /** |
373 | * reserve_bootmem_node - mark a page range as reserved | |
374 | * @pgdat: node the range resides on | |
375 | * @physaddr: starting address of the range | |
376 | * @size: size of the range in bytes | |
377 | * @flags: reservation flags (see linux/bootmem.h) | |
378 | * | |
379 | * Partial pages will be reserved. | |
380 | * | |
381 | * Only physical pages that actually reside on @pgdat are marked. | |
382 | */ | |
223e8dc9 JW |
383 | int __init reserve_bootmem_node(pg_data_t *pgdat, unsigned long physaddr, |
384 | unsigned long size, int flags) | |
1da177e4 | 385 | { |
223e8dc9 | 386 | int ret; |
1da177e4 | 387 | |
223e8dc9 JW |
388 | ret = can_reserve_bootmem_core(pgdat->bdata, physaddr, size, flags); |
389 | if (ret < 0) | |
390 | return -ENOMEM; | |
391 | reserve_bootmem_core(pgdat->bdata, physaddr, size, flags); | |
392 | return 0; | |
393 | } | |
5a982cbc | 394 | |
223e8dc9 | 395 | #ifndef CONFIG_HAVE_ARCH_BOOTMEM_NODE |
a66fd7da JW |
396 | /** |
397 | * reserve_bootmem - mark a page range as usable | |
398 | * @addr: starting address of the range | |
399 | * @size: size of the range in bytes | |
400 | * @flags: reservation flags (see linux/bootmem.h) | |
401 | * | |
402 | * Partial pages will be reserved. | |
403 | * | |
404 | * All physical pages within the range are marked, no matter what | |
405 | * node they reside on. | |
406 | */ | |
223e8dc9 JW |
407 | int __init reserve_bootmem(unsigned long addr, unsigned long size, |
408 | int flags) | |
409 | { | |
410 | bootmem_data_t *bdata; | |
411 | int ret; | |
1da177e4 | 412 | |
223e8dc9 JW |
413 | list_for_each_entry(bdata, &bdata_list, list) { |
414 | ret = can_reserve_bootmem_core(bdata, addr, size, flags); | |
415 | if (ret < 0) | |
416 | return ret; | |
1da177e4 | 417 | } |
223e8dc9 JW |
418 | list_for_each_entry(bdata, &bdata_list, list) |
419 | reserve_bootmem_core(bdata, addr, size, flags); | |
420 | ||
421 | return 0; | |
1da177e4 | 422 | } |
223e8dc9 | 423 | #endif /* !CONFIG_HAVE_ARCH_BOOTMEM_NODE */ |
1da177e4 LT |
424 | |
425 | /* | |
426 | * We 'merge' subsequent allocations to save space. We might 'lose' | |
427 | * some fraction of a page if allocations cannot be satisfied due to | |
428 | * size constraints on boxes where there is physical RAM space | |
429 | * fragmentation - in these cases (mostly large memory boxes) this | |
430 | * is not a problem. | |
431 | * | |
432 | * On low memory boxes we get it right in 100% of the cases. | |
433 | * | |
434 | * alignment has to be a power of 2 value. | |
435 | * | |
436 | * NOTE: This function is _not_ reentrant. | |
437 | */ | |
ffc6421f JW |
438 | static void * __init |
439 | alloc_bootmem_core(struct bootmem_data *bdata, unsigned long size, | |
440 | unsigned long align, unsigned long goal, unsigned long limit) | |
1da177e4 | 441 | { |
9a2dc04c | 442 | unsigned long areasize, preferred; |
bbc7b92e | 443 | unsigned long i, start = 0, incr, eidx, end_pfn; |
1da177e4 | 444 | void *ret; |
9a2dc04c YL |
445 | unsigned long node_boot_start; |
446 | void *node_bootmem_map; | |
1da177e4 | 447 | |
f71bf0ca | 448 | if (!size) { |
ffc6421f | 449 | printk("alloc_bootmem_core(): zero-sized request\n"); |
1da177e4 LT |
450 | BUG(); |
451 | } | |
452 | BUG_ON(align & (align-1)); | |
453 | ||
7c309a64 CK |
454 | /* on nodes without memory - bootmem_map is NULL */ |
455 | if (!bdata->node_bootmem_map) | |
456 | return NULL; | |
457 | ||
9a2dc04c YL |
458 | /* bdata->node_boot_start is supposed to be (12+6)bits alignment on x86_64 ? */ |
459 | node_boot_start = bdata->node_boot_start; | |
460 | node_bootmem_map = bdata->node_bootmem_map; | |
461 | if (align) { | |
462 | node_boot_start = ALIGN(bdata->node_boot_start, align); | |
463 | if (node_boot_start > bdata->node_boot_start) | |
464 | node_bootmem_map = (unsigned long *)bdata->node_bootmem_map + | |
465 | PFN_DOWN(node_boot_start - bdata->node_boot_start)/BITS_PER_LONG; | |
466 | } | |
467 | ||
468 | if (limit && node_boot_start >= limit) | |
469 | return NULL; | |
470 | ||
bbc7b92e FBH |
471 | end_pfn = bdata->node_low_pfn; |
472 | limit = PFN_DOWN(limit); | |
281dd25c YG |
473 | if (limit && end_pfn > limit) |
474 | end_pfn = limit; | |
475 | ||
9a2dc04c | 476 | eidx = end_pfn - PFN_DOWN(node_boot_start); |
1da177e4 LT |
477 | |
478 | /* | |
479 | * We try to allocate bootmem pages above 'goal' | |
480 | * first, then we try to allocate lower pages. | |
481 | */ | |
ad09315c YL |
482 | preferred = 0; |
483 | if (goal && PFN_DOWN(goal) < end_pfn) { | |
9a2dc04c YL |
484 | if (goal > node_boot_start) |
485 | preferred = goal - node_boot_start; | |
1da177e4 | 486 | |
9a2dc04c YL |
487 | if (bdata->last_success > node_boot_start && |
488 | bdata->last_success - node_boot_start >= preferred) | |
281dd25c | 489 | if (!limit || (limit && limit > bdata->last_success)) |
9a2dc04c | 490 | preferred = bdata->last_success - node_boot_start; |
ad09315c | 491 | } |
1da177e4 | 492 | |
9a2dc04c | 493 | preferred = PFN_DOWN(ALIGN(preferred, align)); |
bbc7b92e | 494 | areasize = (size + PAGE_SIZE-1) / PAGE_SIZE; |
1da177e4 LT |
495 | incr = align >> PAGE_SHIFT ? : 1; |
496 | ||
497 | restart_scan: | |
ad09315c | 498 | for (i = preferred; i < eidx;) { |
1da177e4 | 499 | unsigned long j; |
ad09315c | 500 | |
9a2dc04c | 501 | i = find_next_zero_bit(node_bootmem_map, eidx, i); |
1da177e4 | 502 | i = ALIGN(i, incr); |
66d43e98 HM |
503 | if (i >= eidx) |
504 | break; | |
9a2dc04c | 505 | if (test_bit(i, node_bootmem_map)) { |
ad09315c | 506 | i += incr; |
1da177e4 | 507 | continue; |
ad09315c | 508 | } |
1da177e4 LT |
509 | for (j = i + 1; j < i + areasize; ++j) { |
510 | if (j >= eidx) | |
511 | goto fail_block; | |
9a2dc04c | 512 | if (test_bit(j, node_bootmem_map)) |
1da177e4 LT |
513 | goto fail_block; |
514 | } | |
515 | start = i; | |
516 | goto found; | |
517 | fail_block: | |
518 | i = ALIGN(j, incr); | |
ad09315c YL |
519 | if (i == j) |
520 | i += incr; | |
1da177e4 LT |
521 | } |
522 | ||
9a2dc04c YL |
523 | if (preferred > 0) { |
524 | preferred = 0; | |
1da177e4 LT |
525 | goto restart_scan; |
526 | } | |
527 | return NULL; | |
528 | ||
529 | found: | |
9a2dc04c | 530 | bdata->last_success = PFN_PHYS(start) + node_boot_start; |
1da177e4 LT |
531 | BUG_ON(start >= eidx); |
532 | ||
533 | /* | |
534 | * Is the next page of the previous allocation-end the start | |
535 | * of this allocation's buffer? If yes then we can 'merge' | |
536 | * the previous partial page with this allocation. | |
537 | */ | |
538 | if (align < PAGE_SIZE && | |
539 | bdata->last_offset && bdata->last_pos+1 == start) { | |
9a2dc04c | 540 | unsigned long offset, remaining_size; |
8c0e33c1 | 541 | offset = ALIGN(bdata->last_offset, align); |
1da177e4 | 542 | BUG_ON(offset > PAGE_SIZE); |
f71bf0ca | 543 | remaining_size = PAGE_SIZE - offset; |
1da177e4 LT |
544 | if (size < remaining_size) { |
545 | areasize = 0; | |
546 | /* last_pos unchanged */ | |
f71bf0ca FBH |
547 | bdata->last_offset = offset + size; |
548 | ret = phys_to_virt(bdata->last_pos * PAGE_SIZE + | |
9a2dc04c | 549 | offset + node_boot_start); |
1da177e4 LT |
550 | } else { |
551 | remaining_size = size - remaining_size; | |
f71bf0ca FBH |
552 | areasize = (remaining_size + PAGE_SIZE-1) / PAGE_SIZE; |
553 | ret = phys_to_virt(bdata->last_pos * PAGE_SIZE + | |
9a2dc04c | 554 | offset + node_boot_start); |
f71bf0ca | 555 | bdata->last_pos = start + areasize - 1; |
1da177e4 LT |
556 | bdata->last_offset = remaining_size; |
557 | } | |
558 | bdata->last_offset &= ~PAGE_MASK; | |
559 | } else { | |
560 | bdata->last_pos = start + areasize - 1; | |
561 | bdata->last_offset = size & ~PAGE_MASK; | |
9a2dc04c | 562 | ret = phys_to_virt(start * PAGE_SIZE + node_boot_start); |
1da177e4 LT |
563 | } |
564 | ||
565 | /* | |
566 | * Reserve the area now: | |
567 | */ | |
f71bf0ca | 568 | for (i = start; i < start + areasize; i++) |
9a2dc04c | 569 | if (unlikely(test_and_set_bit(i, node_bootmem_map))) |
1da177e4 LT |
570 | BUG(); |
571 | memset(ret, 0, size); | |
572 | return ret; | |
573 | } | |
574 | ||
a66fd7da JW |
575 | /** |
576 | * __alloc_bootmem_nopanic - allocate boot memory without panicking | |
577 | * @size: size of the request in bytes | |
578 | * @align: alignment of the region | |
579 | * @goal: preferred starting address of the region | |
580 | * | |
581 | * The goal is dropped if it can not be satisfied and the allocation will | |
582 | * fall back to memory below @goal. | |
583 | * | |
584 | * Allocation may happen on any node in the system. | |
585 | * | |
586 | * Returns NULL on failure. | |
587 | */ | |
bb0923a6 FBH |
588 | void * __init __alloc_bootmem_nopanic(unsigned long size, unsigned long align, |
589 | unsigned long goal) | |
1da177e4 | 590 | { |
679bc9fb | 591 | bootmem_data_t *bdata; |
1da177e4 LT |
592 | void *ptr; |
593 | ||
f71bf0ca | 594 | list_for_each_entry(bdata, &bdata_list, list) { |
ffc6421f | 595 | ptr = alloc_bootmem_core(bdata, size, align, goal, 0); |
f71bf0ca FBH |
596 | if (ptr) |
597 | return ptr; | |
598 | } | |
a8062231 AK |
599 | return NULL; |
600 | } | |
1da177e4 | 601 | |
a66fd7da JW |
602 | /** |
603 | * __alloc_bootmem - allocate boot memory | |
604 | * @size: size of the request in bytes | |
605 | * @align: alignment of the region | |
606 | * @goal: preferred starting address of the region | |
607 | * | |
608 | * The goal is dropped if it can not be satisfied and the allocation will | |
609 | * fall back to memory below @goal. | |
610 | * | |
611 | * Allocation may happen on any node in the system. | |
612 | * | |
613 | * The function panics if the request can not be satisfied. | |
614 | */ | |
bb0923a6 FBH |
615 | void * __init __alloc_bootmem(unsigned long size, unsigned long align, |
616 | unsigned long goal) | |
a8062231 AK |
617 | { |
618 | void *mem = __alloc_bootmem_nopanic(size,align,goal); | |
f71bf0ca | 619 | |
a8062231 AK |
620 | if (mem) |
621 | return mem; | |
1da177e4 LT |
622 | /* |
623 | * Whoops, we cannot satisfy the allocation request. | |
624 | */ | |
625 | printk(KERN_ALERT "bootmem alloc of %lu bytes failed!\n", size); | |
626 | panic("Out of memory"); | |
627 | return NULL; | |
628 | } | |
629 | ||
a66fd7da JW |
630 | /** |
631 | * __alloc_bootmem_node - allocate boot memory from a specific node | |
632 | * @pgdat: node to allocate from | |
633 | * @size: size of the request in bytes | |
634 | * @align: alignment of the region | |
635 | * @goal: preferred starting address of the region | |
636 | * | |
637 | * The goal is dropped if it can not be satisfied and the allocation will | |
638 | * fall back to memory below @goal. | |
639 | * | |
640 | * Allocation may fall back to any node in the system if the specified node | |
641 | * can not hold the requested memory. | |
642 | * | |
643 | * The function panics if the request can not be satisfied. | |
644 | */ | |
bb0923a6 FBH |
645 | void * __init __alloc_bootmem_node(pg_data_t *pgdat, unsigned long size, |
646 | unsigned long align, unsigned long goal) | |
1da177e4 LT |
647 | { |
648 | void *ptr; | |
649 | ||
ffc6421f | 650 | ptr = alloc_bootmem_core(pgdat->bdata, size, align, goal, 0); |
1da177e4 | 651 | if (ptr) |
f71bf0ca | 652 | return ptr; |
1da177e4 | 653 | |
008857c1 | 654 | return __alloc_bootmem(size, align, goal); |
1da177e4 LT |
655 | } |
656 | ||
e70260aa | 657 | #ifdef CONFIG_SPARSEMEM |
a66fd7da JW |
658 | /** |
659 | * alloc_bootmem_section - allocate boot memory from a specific section | |
660 | * @size: size of the request in bytes | |
661 | * @section_nr: sparse map section to allocate from | |
662 | * | |
663 | * Return NULL on failure. | |
664 | */ | |
e70260aa YG |
665 | void * __init alloc_bootmem_section(unsigned long size, |
666 | unsigned long section_nr) | |
667 | { | |
668 | void *ptr; | |
669 | unsigned long limit, goal, start_nr, end_nr, pfn; | |
670 | struct pglist_data *pgdat; | |
671 | ||
672 | pfn = section_nr_to_pfn(section_nr); | |
673 | goal = PFN_PHYS(pfn); | |
674 | limit = PFN_PHYS(section_nr_to_pfn(section_nr + 1)) - 1; | |
675 | pgdat = NODE_DATA(early_pfn_to_nid(pfn)); | |
ffc6421f JW |
676 | ptr = alloc_bootmem_core(pgdat->bdata, size, SMP_CACHE_BYTES, goal, |
677 | limit); | |
e70260aa YG |
678 | |
679 | if (!ptr) | |
680 | return NULL; | |
681 | ||
682 | start_nr = pfn_to_section_nr(PFN_DOWN(__pa(ptr))); | |
683 | end_nr = pfn_to_section_nr(PFN_DOWN(__pa(ptr) + size)); | |
684 | if (start_nr != section_nr || end_nr != section_nr) { | |
685 | printk(KERN_WARNING "alloc_bootmem failed on section %ld.\n", | |
686 | section_nr); | |
687 | free_bootmem_core(pgdat->bdata, __pa(ptr), size); | |
688 | ptr = NULL; | |
689 | } | |
690 | ||
691 | return ptr; | |
692 | } | |
693 | #endif | |
694 | ||
b54bbf7b AK |
695 | void * __init __alloc_bootmem_node_nopanic(pg_data_t *pgdat, unsigned long size, |
696 | unsigned long align, unsigned long goal) | |
697 | { | |
698 | void *ptr; | |
699 | ||
700 | ptr = alloc_bootmem_core(pgdat->bdata, size, align, goal, 0); | |
701 | if (ptr) | |
702 | return ptr; | |
703 | ||
704 | return __alloc_bootmem_nopanic(size, align, goal); | |
705 | } | |
706 | ||
dfd54cbc HC |
707 | #ifndef ARCH_LOW_ADDRESS_LIMIT |
708 | #define ARCH_LOW_ADDRESS_LIMIT 0xffffffffUL | |
709 | #endif | |
008857c1 | 710 | |
a66fd7da JW |
711 | /** |
712 | * __alloc_bootmem_low - allocate low boot memory | |
713 | * @size: size of the request in bytes | |
714 | * @align: alignment of the region | |
715 | * @goal: preferred starting address of the region | |
716 | * | |
717 | * The goal is dropped if it can not be satisfied and the allocation will | |
718 | * fall back to memory below @goal. | |
719 | * | |
720 | * Allocation may happen on any node in the system. | |
721 | * | |
722 | * The function panics if the request can not be satisfied. | |
723 | */ | |
bb0923a6 FBH |
724 | void * __init __alloc_bootmem_low(unsigned long size, unsigned long align, |
725 | unsigned long goal) | |
008857c1 | 726 | { |
679bc9fb | 727 | bootmem_data_t *bdata; |
008857c1 RT |
728 | void *ptr; |
729 | ||
f71bf0ca | 730 | list_for_each_entry(bdata, &bdata_list, list) { |
ffc6421f JW |
731 | ptr = alloc_bootmem_core(bdata, size, align, goal, |
732 | ARCH_LOW_ADDRESS_LIMIT); | |
f71bf0ca FBH |
733 | if (ptr) |
734 | return ptr; | |
735 | } | |
008857c1 RT |
736 | |
737 | /* | |
738 | * Whoops, we cannot satisfy the allocation request. | |
739 | */ | |
740 | printk(KERN_ALERT "low bootmem alloc of %lu bytes failed!\n", size); | |
741 | panic("Out of low memory"); | |
742 | return NULL; | |
743 | } | |
744 | ||
a66fd7da JW |
745 | /** |
746 | * __alloc_bootmem_low_node - allocate low boot memory from a specific node | |
747 | * @pgdat: node to allocate from | |
748 | * @size: size of the request in bytes | |
749 | * @align: alignment of the region | |
750 | * @goal: preferred starting address of the region | |
751 | * | |
752 | * The goal is dropped if it can not be satisfied and the allocation will | |
753 | * fall back to memory below @goal. | |
754 | * | |
755 | * Allocation may fall back to any node in the system if the specified node | |
756 | * can not hold the requested memory. | |
757 | * | |
758 | * The function panics if the request can not be satisfied. | |
759 | */ | |
008857c1 RT |
760 | void * __init __alloc_bootmem_low_node(pg_data_t *pgdat, unsigned long size, |
761 | unsigned long align, unsigned long goal) | |
762 | { | |
ffc6421f JW |
763 | return alloc_bootmem_core(pgdat->bdata, size, align, goal, |
764 | ARCH_LOW_ADDRESS_LIMIT); | |
008857c1 | 765 | } |