Commit | Line | Data |
---|---|---|
52d4b9ac KH |
1 | #include <linux/mm.h> |
2 | #include <linux/mmzone.h> | |
3 | #include <linux/bootmem.h> | |
4 | #include <linux/bit_spinlock.h> | |
5 | #include <linux/page_cgroup.h> | |
6 | #include <linux/hash.h> | |
94b6da5a | 7 | #include <linux/slab.h> |
52d4b9ac | 8 | #include <linux/memory.h> |
4c821042 | 9 | #include <linux/vmalloc.h> |
94b6da5a | 10 | #include <linux/cgroup.h> |
52d4b9ac KH |
11 | |
12 | static void __meminit | |
13 | __init_page_cgroup(struct page_cgroup *pc, unsigned long pfn) | |
14 | { | |
15 | pc->flags = 0; | |
16 | pc->mem_cgroup = NULL; | |
17 | pc->page = pfn_to_page(pfn); | |
18 | } | |
19 | static unsigned long total_usage; | |
20 | ||
21 | #if !defined(CONFIG_SPARSEMEM) | |
22 | ||
23 | ||
24 | void __init pgdat_page_cgroup_init(struct pglist_data *pgdat) | |
25 | { | |
26 | pgdat->node_page_cgroup = NULL; | |
27 | } | |
28 | ||
29 | struct page_cgroup *lookup_page_cgroup(struct page *page) | |
30 | { | |
31 | unsigned long pfn = page_to_pfn(page); | |
32 | unsigned long offset; | |
33 | struct page_cgroup *base; | |
34 | ||
35 | base = NODE_DATA(page_to_nid(page))->node_page_cgroup; | |
36 | if (unlikely(!base)) | |
37 | return NULL; | |
38 | ||
39 | offset = pfn - NODE_DATA(page_to_nid(page))->node_start_pfn; | |
40 | return base + offset; | |
41 | } | |
42 | ||
43 | static int __init alloc_node_page_cgroup(int nid) | |
44 | { | |
45 | struct page_cgroup *base, *pc; | |
46 | unsigned long table_size; | |
47 | unsigned long start_pfn, nr_pages, index; | |
48 | ||
49 | start_pfn = NODE_DATA(nid)->node_start_pfn; | |
50 | nr_pages = NODE_DATA(nid)->node_spanned_pages; | |
51 | ||
52 | table_size = sizeof(struct page_cgroup) * nr_pages; | |
53 | ||
54 | base = __alloc_bootmem_node_nopanic(NODE_DATA(nid), | |
55 | table_size, PAGE_SIZE, __pa(MAX_DMA_ADDRESS)); | |
56 | if (!base) | |
57 | return -ENOMEM; | |
58 | for (index = 0; index < nr_pages; index++) { | |
59 | pc = base + index; | |
60 | __init_page_cgroup(pc, start_pfn + index); | |
61 | } | |
62 | NODE_DATA(nid)->node_page_cgroup = base; | |
63 | total_usage += table_size; | |
64 | return 0; | |
65 | } | |
66 | ||
67 | void __init page_cgroup_init(void) | |
68 | { | |
69 | ||
70 | int nid, fail; | |
71 | ||
94b6da5a KH |
72 | if (mem_cgroup_subsys.disabled) |
73 | return; | |
74 | ||
52d4b9ac KH |
75 | for_each_online_node(nid) { |
76 | fail = alloc_node_page_cgroup(nid); | |
77 | if (fail) | |
78 | goto fail; | |
79 | } | |
80 | printk(KERN_INFO "allocated %ld bytes of page_cgroup\n", total_usage); | |
81 | printk(KERN_INFO "please try cgroup_disable=memory option if you" | |
82 | " don't want\n"); | |
83 | return; | |
84 | fail: | |
85 | printk(KERN_CRIT "allocation of page_cgroup was failed.\n"); | |
86 | printk(KERN_CRIT "please try cgroup_disable=memory boot option\n"); | |
87 | panic("Out of memory"); | |
88 | } | |
89 | ||
90 | #else /* CONFIG_FLAT_NODE_MEM_MAP */ | |
91 | ||
92 | struct page_cgroup *lookup_page_cgroup(struct page *page) | |
93 | { | |
94 | unsigned long pfn = page_to_pfn(page); | |
95 | struct mem_section *section = __pfn_to_section(pfn); | |
96 | ||
97 | return section->page_cgroup + pfn; | |
98 | } | |
99 | ||
100 | int __meminit init_section_page_cgroup(unsigned long pfn) | |
101 | { | |
102 | struct mem_section *section; | |
103 | struct page_cgroup *base, *pc; | |
104 | unsigned long table_size; | |
105 | int nid, index; | |
106 | ||
107 | section = __pfn_to_section(pfn); | |
108 | ||
109 | if (section->page_cgroup) | |
110 | return 0; | |
111 | ||
112 | nid = page_to_nid(pfn_to_page(pfn)); | |
113 | ||
114 | table_size = sizeof(struct page_cgroup) * PAGES_PER_SECTION; | |
94b6da5a KH |
115 | if (slab_is_available()) { |
116 | base = kmalloc_node(table_size, GFP_KERNEL, nid); | |
117 | if (!base) | |
118 | base = vmalloc_node(table_size, nid); | |
119 | } else { | |
120 | base = __alloc_bootmem_node_nopanic(NODE_DATA(nid), table_size, | |
121 | PAGE_SIZE, __pa(MAX_DMA_ADDRESS)); | |
122 | } | |
52d4b9ac KH |
123 | |
124 | if (!base) { | |
125 | printk(KERN_ERR "page cgroup allocation failure\n"); | |
126 | return -ENOMEM; | |
127 | } | |
128 | ||
129 | for (index = 0; index < PAGES_PER_SECTION; index++) { | |
130 | pc = base + index; | |
131 | __init_page_cgroup(pc, pfn + index); | |
132 | } | |
133 | ||
134 | section = __pfn_to_section(pfn); | |
135 | section->page_cgroup = base - pfn; | |
136 | total_usage += table_size; | |
137 | return 0; | |
138 | } | |
139 | #ifdef CONFIG_MEMORY_HOTPLUG | |
140 | void __free_page_cgroup(unsigned long pfn) | |
141 | { | |
142 | struct mem_section *ms; | |
143 | struct page_cgroup *base; | |
144 | ||
145 | ms = __pfn_to_section(pfn); | |
146 | if (!ms || !ms->page_cgroup) | |
147 | return; | |
148 | base = ms->page_cgroup + pfn; | |
94b6da5a | 149 | if (is_vmalloc_addr(base)) { |
52d4b9ac | 150 | vfree(base); |
94b6da5a KH |
151 | ms->page_cgroup = NULL; |
152 | } else { | |
153 | struct page *page = virt_to_page(base); | |
154 | if (!PageReserved(page)) { /* Is bootmem ? */ | |
155 | kfree(base); | |
156 | ms->page_cgroup = NULL; | |
157 | } | |
158 | } | |
52d4b9ac KH |
159 | } |
160 | ||
161 | int online_page_cgroup(unsigned long start_pfn, | |
162 | unsigned long nr_pages, | |
163 | int nid) | |
164 | { | |
165 | unsigned long start, end, pfn; | |
166 | int fail = 0; | |
167 | ||
168 | start = start_pfn & (PAGES_PER_SECTION - 1); | |
169 | end = ALIGN(start_pfn + nr_pages, PAGES_PER_SECTION); | |
170 | ||
171 | for (pfn = start; !fail && pfn < end; pfn += PAGES_PER_SECTION) { | |
172 | if (!pfn_present(pfn)) | |
173 | continue; | |
174 | fail = init_section_page_cgroup(pfn); | |
175 | } | |
176 | if (!fail) | |
177 | return 0; | |
178 | ||
179 | /* rollback */ | |
180 | for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION) | |
181 | __free_page_cgroup(pfn); | |
182 | ||
183 | return -ENOMEM; | |
184 | } | |
185 | ||
186 | int offline_page_cgroup(unsigned long start_pfn, | |
187 | unsigned long nr_pages, int nid) | |
188 | { | |
189 | unsigned long start, end, pfn; | |
190 | ||
191 | start = start_pfn & (PAGES_PER_SECTION - 1); | |
192 | end = ALIGN(start_pfn + nr_pages, PAGES_PER_SECTION); | |
193 | ||
194 | for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION) | |
195 | __free_page_cgroup(pfn); | |
196 | return 0; | |
197 | ||
198 | } | |
199 | ||
200 | static int page_cgroup_callback(struct notifier_block *self, | |
201 | unsigned long action, void *arg) | |
202 | { | |
203 | struct memory_notify *mn = arg; | |
204 | int ret = 0; | |
205 | switch (action) { | |
206 | case MEM_GOING_ONLINE: | |
207 | ret = online_page_cgroup(mn->start_pfn, | |
208 | mn->nr_pages, mn->status_change_nid); | |
209 | break; | |
210 | case MEM_CANCEL_ONLINE: | |
211 | case MEM_OFFLINE: | |
212 | offline_page_cgroup(mn->start_pfn, | |
213 | mn->nr_pages, mn->status_change_nid); | |
214 | break; | |
215 | case MEM_GOING_OFFLINE: | |
216 | break; | |
217 | case MEM_ONLINE: | |
218 | case MEM_CANCEL_OFFLINE: | |
219 | break; | |
220 | } | |
221 | ret = notifier_from_errno(ret); | |
222 | return ret; | |
223 | } | |
224 | ||
225 | #endif | |
226 | ||
227 | void __init page_cgroup_init(void) | |
228 | { | |
229 | unsigned long pfn; | |
230 | int fail = 0; | |
231 | ||
94b6da5a KH |
232 | if (mem_cgroup_subsys.disabled) |
233 | return; | |
234 | ||
52d4b9ac KH |
235 | for (pfn = 0; !fail && pfn < max_pfn; pfn += PAGES_PER_SECTION) { |
236 | if (!pfn_present(pfn)) | |
237 | continue; | |
238 | fail = init_section_page_cgroup(pfn); | |
239 | } | |
240 | if (fail) { | |
241 | printk(KERN_CRIT "try cgroup_disable=memory boot option\n"); | |
242 | panic("Out of memory"); | |
243 | } else { | |
244 | hotplug_memory_notifier(page_cgroup_callback, 0); | |
245 | } | |
246 | printk(KERN_INFO "allocated %ld bytes of page_cgroup\n", total_usage); | |
247 | printk(KERN_INFO "please try cgroup_disable=memory option if you don't" | |
248 | " want\n"); | |
249 | } | |
250 | ||
251 | void __init pgdat_page_cgroup_init(struct pglist_data *pgdat) | |
252 | { | |
253 | return; | |
254 | } | |
255 | ||
256 | #endif |