Commit | Line | Data |
---|---|---|
52d4b9ac KH |
1 | #include <linux/mm.h> |
2 | #include <linux/mmzone.h> | |
3 | #include <linux/bootmem.h> | |
4 | #include <linux/bit_spinlock.h> | |
5 | #include <linux/page_cgroup.h> | |
6 | #include <linux/hash.h> | |
94b6da5a | 7 | #include <linux/slab.h> |
52d4b9ac | 8 | #include <linux/memory.h> |
4c821042 | 9 | #include <linux/vmalloc.h> |
94b6da5a | 10 | #include <linux/cgroup.h> |
52d4b9ac KH |
11 | |
12 | static void __meminit | |
13 | __init_page_cgroup(struct page_cgroup *pc, unsigned long pfn) | |
14 | { | |
15 | pc->flags = 0; | |
16 | pc->mem_cgroup = NULL; | |
17 | pc->page = pfn_to_page(pfn); | |
18 | } | |
19 | static unsigned long total_usage; | |
20 | ||
21 | #if !defined(CONFIG_SPARSEMEM) | |
22 | ||
23 | ||
31168481 | 24 | void __meminit pgdat_page_cgroup_init(struct pglist_data *pgdat) |
52d4b9ac KH |
25 | { |
26 | pgdat->node_page_cgroup = NULL; | |
27 | } | |
28 | ||
29 | struct page_cgroup *lookup_page_cgroup(struct page *page) | |
30 | { | |
31 | unsigned long pfn = page_to_pfn(page); | |
32 | unsigned long offset; | |
33 | struct page_cgroup *base; | |
34 | ||
35 | base = NODE_DATA(page_to_nid(page))->node_page_cgroup; | |
36 | if (unlikely(!base)) | |
37 | return NULL; | |
38 | ||
39 | offset = pfn - NODE_DATA(page_to_nid(page))->node_start_pfn; | |
40 | return base + offset; | |
41 | } | |
42 | ||
43 | static int __init alloc_node_page_cgroup(int nid) | |
44 | { | |
45 | struct page_cgroup *base, *pc; | |
46 | unsigned long table_size; | |
47 | unsigned long start_pfn, nr_pages, index; | |
48 | ||
49 | start_pfn = NODE_DATA(nid)->node_start_pfn; | |
50 | nr_pages = NODE_DATA(nid)->node_spanned_pages; | |
51 | ||
52 | table_size = sizeof(struct page_cgroup) * nr_pages; | |
53 | ||
54 | base = __alloc_bootmem_node_nopanic(NODE_DATA(nid), | |
55 | table_size, PAGE_SIZE, __pa(MAX_DMA_ADDRESS)); | |
56 | if (!base) | |
57 | return -ENOMEM; | |
58 | for (index = 0; index < nr_pages; index++) { | |
59 | pc = base + index; | |
60 | __init_page_cgroup(pc, start_pfn + index); | |
61 | } | |
62 | NODE_DATA(nid)->node_page_cgroup = base; | |
63 | total_usage += table_size; | |
64 | return 0; | |
65 | } | |
66 | ||
67 | void __init page_cgroup_init(void) | |
68 | { | |
69 | ||
70 | int nid, fail; | |
71 | ||
94b6da5a KH |
72 | if (mem_cgroup_subsys.disabled) |
73 | return; | |
74 | ||
52d4b9ac KH |
75 | for_each_online_node(nid) { |
76 | fail = alloc_node_page_cgroup(nid); | |
77 | if (fail) | |
78 | goto fail; | |
79 | } | |
80 | printk(KERN_INFO "allocated %ld bytes of page_cgroup\n", total_usage); | |
81 | printk(KERN_INFO "please try cgroup_disable=memory option if you" | |
82 | " don't want\n"); | |
83 | return; | |
84 | fail: | |
85 | printk(KERN_CRIT "allocation of page_cgroup was failed.\n"); | |
86 | printk(KERN_CRIT "please try cgroup_disable=memory boot option\n"); | |
87 | panic("Out of memory"); | |
88 | } | |
89 | ||
90 | #else /* CONFIG_FLAT_NODE_MEM_MAP */ | |
91 | ||
92 | struct page_cgroup *lookup_page_cgroup(struct page *page) | |
93 | { | |
94 | unsigned long pfn = page_to_pfn(page); | |
95 | struct mem_section *section = __pfn_to_section(pfn); | |
96 | ||
97 | return section->page_cgroup + pfn; | |
98 | } | |
99 | ||
31168481 AV |
100 | /* __alloc_bootmem...() is protected by !slab_available() */ |
101 | int __init_refok init_section_page_cgroup(unsigned long pfn) | |
52d4b9ac KH |
102 | { |
103 | struct mem_section *section; | |
104 | struct page_cgroup *base, *pc; | |
105 | unsigned long table_size; | |
106 | int nid, index; | |
107 | ||
108 | section = __pfn_to_section(pfn); | |
109 | ||
dc19f9db KH |
110 | if (!section->page_cgroup) { |
111 | nid = page_to_nid(pfn_to_page(pfn)); | |
112 | table_size = sizeof(struct page_cgroup) * PAGES_PER_SECTION; | |
113 | if (slab_is_available()) { | |
114 | base = kmalloc_node(table_size, GFP_KERNEL, nid); | |
115 | if (!base) | |
116 | base = vmalloc_node(table_size, nid); | |
117 | } else { | |
118 | base = __alloc_bootmem_node_nopanic(NODE_DATA(nid), | |
119 | table_size, | |
94b6da5a | 120 | PAGE_SIZE, __pa(MAX_DMA_ADDRESS)); |
dc19f9db KH |
121 | } |
122 | } else { | |
123 | /* | |
124 | * We don't have to allocate page_cgroup again, but | |
125 | * address of memmap may be changed. So, we have to initialize | |
126 | * again. | |
127 | */ | |
128 | base = section->page_cgroup + pfn; | |
129 | table_size = 0; | |
130 | /* check address of memmap is changed or not. */ | |
131 | if (base->page == pfn_to_page(pfn)) | |
132 | return 0; | |
94b6da5a | 133 | } |
52d4b9ac KH |
134 | |
135 | if (!base) { | |
136 | printk(KERN_ERR "page cgroup allocation failure\n"); | |
137 | return -ENOMEM; | |
138 | } | |
139 | ||
140 | for (index = 0; index < PAGES_PER_SECTION; index++) { | |
141 | pc = base + index; | |
142 | __init_page_cgroup(pc, pfn + index); | |
143 | } | |
144 | ||
145 | section = __pfn_to_section(pfn); | |
146 | section->page_cgroup = base - pfn; | |
147 | total_usage += table_size; | |
148 | return 0; | |
149 | } | |
150 | #ifdef CONFIG_MEMORY_HOTPLUG | |
151 | void __free_page_cgroup(unsigned long pfn) | |
152 | { | |
153 | struct mem_section *ms; | |
154 | struct page_cgroup *base; | |
155 | ||
156 | ms = __pfn_to_section(pfn); | |
157 | if (!ms || !ms->page_cgroup) | |
158 | return; | |
159 | base = ms->page_cgroup + pfn; | |
94b6da5a | 160 | if (is_vmalloc_addr(base)) { |
52d4b9ac | 161 | vfree(base); |
94b6da5a KH |
162 | ms->page_cgroup = NULL; |
163 | } else { | |
164 | struct page *page = virt_to_page(base); | |
165 | if (!PageReserved(page)) { /* Is bootmem ? */ | |
166 | kfree(base); | |
167 | ms->page_cgroup = NULL; | |
168 | } | |
169 | } | |
52d4b9ac KH |
170 | } |
171 | ||
31168481 | 172 | int __meminit online_page_cgroup(unsigned long start_pfn, |
52d4b9ac KH |
173 | unsigned long nr_pages, |
174 | int nid) | |
175 | { | |
176 | unsigned long start, end, pfn; | |
177 | int fail = 0; | |
178 | ||
33c5d3d6 | 179 | start = start_pfn & ~(PAGES_PER_SECTION - 1); |
52d4b9ac KH |
180 | end = ALIGN(start_pfn + nr_pages, PAGES_PER_SECTION); |
181 | ||
182 | for (pfn = start; !fail && pfn < end; pfn += PAGES_PER_SECTION) { | |
183 | if (!pfn_present(pfn)) | |
184 | continue; | |
185 | fail = init_section_page_cgroup(pfn); | |
186 | } | |
187 | if (!fail) | |
188 | return 0; | |
189 | ||
190 | /* rollback */ | |
191 | for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION) | |
192 | __free_page_cgroup(pfn); | |
193 | ||
194 | return -ENOMEM; | |
195 | } | |
196 | ||
31168481 | 197 | int __meminit offline_page_cgroup(unsigned long start_pfn, |
52d4b9ac KH |
198 | unsigned long nr_pages, int nid) |
199 | { | |
200 | unsigned long start, end, pfn; | |
201 | ||
33c5d3d6 | 202 | start = start_pfn & ~(PAGES_PER_SECTION - 1); |
52d4b9ac KH |
203 | end = ALIGN(start_pfn + nr_pages, PAGES_PER_SECTION); |
204 | ||
205 | for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION) | |
206 | __free_page_cgroup(pfn); | |
207 | return 0; | |
208 | ||
209 | } | |
210 | ||
31168481 | 211 | static int __meminit page_cgroup_callback(struct notifier_block *self, |
52d4b9ac KH |
212 | unsigned long action, void *arg) |
213 | { | |
214 | struct memory_notify *mn = arg; | |
215 | int ret = 0; | |
216 | switch (action) { | |
217 | case MEM_GOING_ONLINE: | |
218 | ret = online_page_cgroup(mn->start_pfn, | |
219 | mn->nr_pages, mn->status_change_nid); | |
220 | break; | |
52d4b9ac KH |
221 | case MEM_OFFLINE: |
222 | offline_page_cgroup(mn->start_pfn, | |
223 | mn->nr_pages, mn->status_change_nid); | |
224 | break; | |
dc19f9db | 225 | case MEM_CANCEL_ONLINE: |
52d4b9ac KH |
226 | case MEM_GOING_OFFLINE: |
227 | break; | |
228 | case MEM_ONLINE: | |
229 | case MEM_CANCEL_OFFLINE: | |
230 | break; | |
231 | } | |
dc19f9db KH |
232 | |
233 | if (ret) | |
234 | ret = notifier_from_errno(ret); | |
235 | else | |
236 | ret = NOTIFY_OK; | |
237 | ||
52d4b9ac KH |
238 | return ret; |
239 | } | |
240 | ||
241 | #endif | |
242 | ||
243 | void __init page_cgroup_init(void) | |
244 | { | |
245 | unsigned long pfn; | |
246 | int fail = 0; | |
247 | ||
94b6da5a KH |
248 | if (mem_cgroup_subsys.disabled) |
249 | return; | |
250 | ||
52d4b9ac KH |
251 | for (pfn = 0; !fail && pfn < max_pfn; pfn += PAGES_PER_SECTION) { |
252 | if (!pfn_present(pfn)) | |
253 | continue; | |
254 | fail = init_section_page_cgroup(pfn); | |
255 | } | |
256 | if (fail) { | |
257 | printk(KERN_CRIT "try cgroup_disable=memory boot option\n"); | |
258 | panic("Out of memory"); | |
259 | } else { | |
260 | hotplug_memory_notifier(page_cgroup_callback, 0); | |
261 | } | |
262 | printk(KERN_INFO "allocated %ld bytes of page_cgroup\n", total_usage); | |
263 | printk(KERN_INFO "please try cgroup_disable=memory option if you don't" | |
264 | " want\n"); | |
265 | } | |
266 | ||
31168481 | 267 | void __meminit pgdat_page_cgroup_init(struct pglist_data *pgdat) |
52d4b9ac KH |
268 | { |
269 | return; | |
270 | } | |
271 | ||
272 | #endif |