| 1 | #include <linux/mm.h> |
| 2 | #include <linux/mmzone.h> |
| 3 | #include <linux/bootmem.h> |
| 4 | #include <linux/bit_spinlock.h> |
| 5 | #include <linux/page_cgroup.h> |
| 6 | #include <linux/hash.h> |
| 7 | #include <linux/slab.h> |
| 8 | #include <linux/memory.h> |
| 9 | #include <linux/vmalloc.h> |
| 10 | #include <linux/cgroup.h> |
| 11 | |
| 12 | static void __meminit |
| 13 | __init_page_cgroup(struct page_cgroup *pc, unsigned long pfn) |
| 14 | { |
| 15 | pc->flags = 0; |
| 16 | pc->mem_cgroup = NULL; |
| 17 | pc->page = pfn_to_page(pfn); |
| 18 | } |
| 19 | static unsigned long total_usage; |
| 20 | |
| 21 | #if !defined(CONFIG_SPARSEMEM) |
| 22 | |
| 23 | |
| 24 | void __init pgdat_page_cgroup_init(struct pglist_data *pgdat) |
| 25 | { |
| 26 | pgdat->node_page_cgroup = NULL; |
| 27 | } |
| 28 | |
| 29 | struct page_cgroup *lookup_page_cgroup(struct page *page) |
| 30 | { |
| 31 | unsigned long pfn = page_to_pfn(page); |
| 32 | unsigned long offset; |
| 33 | struct page_cgroup *base; |
| 34 | |
| 35 | base = NODE_DATA(page_to_nid(page))->node_page_cgroup; |
| 36 | if (unlikely(!base)) |
| 37 | return NULL; |
| 38 | |
| 39 | offset = pfn - NODE_DATA(page_to_nid(page))->node_start_pfn; |
| 40 | return base + offset; |
| 41 | } |
| 42 | |
| 43 | static int __init alloc_node_page_cgroup(int nid) |
| 44 | { |
| 45 | struct page_cgroup *base, *pc; |
| 46 | unsigned long table_size; |
| 47 | unsigned long start_pfn, nr_pages, index; |
| 48 | |
| 49 | start_pfn = NODE_DATA(nid)->node_start_pfn; |
| 50 | nr_pages = NODE_DATA(nid)->node_spanned_pages; |
| 51 | |
| 52 | table_size = sizeof(struct page_cgroup) * nr_pages; |
| 53 | |
| 54 | base = __alloc_bootmem_node_nopanic(NODE_DATA(nid), |
| 55 | table_size, PAGE_SIZE, __pa(MAX_DMA_ADDRESS)); |
| 56 | if (!base) |
| 57 | return -ENOMEM; |
| 58 | for (index = 0; index < nr_pages; index++) { |
| 59 | pc = base + index; |
| 60 | __init_page_cgroup(pc, start_pfn + index); |
| 61 | } |
| 62 | NODE_DATA(nid)->node_page_cgroup = base; |
| 63 | total_usage += table_size; |
| 64 | return 0; |
| 65 | } |
| 66 | |
| 67 | void __init page_cgroup_init(void) |
| 68 | { |
| 69 | |
| 70 | int nid, fail; |
| 71 | |
| 72 | if (mem_cgroup_subsys.disabled) |
| 73 | return; |
| 74 | |
| 75 | for_each_online_node(nid) { |
| 76 | fail = alloc_node_page_cgroup(nid); |
| 77 | if (fail) |
| 78 | goto fail; |
| 79 | } |
| 80 | printk(KERN_INFO "allocated %ld bytes of page_cgroup\n", total_usage); |
| 81 | printk(KERN_INFO "please try cgroup_disable=memory option if you" |
| 82 | " don't want\n"); |
| 83 | return; |
| 84 | fail: |
| 85 | printk(KERN_CRIT "allocation of page_cgroup was failed.\n"); |
| 86 | printk(KERN_CRIT "please try cgroup_disable=memory boot option\n"); |
| 87 | panic("Out of memory"); |
| 88 | } |
| 89 | |
| 90 | #else /* CONFIG_FLAT_NODE_MEM_MAP */ |
| 91 | |
| 92 | struct page_cgroup *lookup_page_cgroup(struct page *page) |
| 93 | { |
| 94 | unsigned long pfn = page_to_pfn(page); |
| 95 | struct mem_section *section = __pfn_to_section(pfn); |
| 96 | |
| 97 | return section->page_cgroup + pfn; |
| 98 | } |
| 99 | |
| 100 | int __meminit init_section_page_cgroup(unsigned long pfn) |
| 101 | { |
| 102 | struct mem_section *section; |
| 103 | struct page_cgroup *base, *pc; |
| 104 | unsigned long table_size; |
| 105 | int nid, index; |
| 106 | |
| 107 | section = __pfn_to_section(pfn); |
| 108 | |
| 109 | if (section->page_cgroup) |
| 110 | return 0; |
| 111 | |
| 112 | nid = page_to_nid(pfn_to_page(pfn)); |
| 113 | |
| 114 | table_size = sizeof(struct page_cgroup) * PAGES_PER_SECTION; |
| 115 | if (slab_is_available()) { |
| 116 | base = kmalloc_node(table_size, GFP_KERNEL, nid); |
| 117 | if (!base) |
| 118 | base = vmalloc_node(table_size, nid); |
| 119 | } else { |
| 120 | base = __alloc_bootmem_node_nopanic(NODE_DATA(nid), table_size, |
| 121 | PAGE_SIZE, __pa(MAX_DMA_ADDRESS)); |
| 122 | } |
| 123 | |
| 124 | if (!base) { |
| 125 | printk(KERN_ERR "page cgroup allocation failure\n"); |
| 126 | return -ENOMEM; |
| 127 | } |
| 128 | |
| 129 | for (index = 0; index < PAGES_PER_SECTION; index++) { |
| 130 | pc = base + index; |
| 131 | __init_page_cgroup(pc, pfn + index); |
| 132 | } |
| 133 | |
| 134 | section = __pfn_to_section(pfn); |
| 135 | section->page_cgroup = base - pfn; |
| 136 | total_usage += table_size; |
| 137 | return 0; |
| 138 | } |
| 139 | #ifdef CONFIG_MEMORY_HOTPLUG |
| 140 | void __free_page_cgroup(unsigned long pfn) |
| 141 | { |
| 142 | struct mem_section *ms; |
| 143 | struct page_cgroup *base; |
| 144 | |
| 145 | ms = __pfn_to_section(pfn); |
| 146 | if (!ms || !ms->page_cgroup) |
| 147 | return; |
| 148 | base = ms->page_cgroup + pfn; |
| 149 | if (is_vmalloc_addr(base)) { |
| 150 | vfree(base); |
| 151 | ms->page_cgroup = NULL; |
| 152 | } else { |
| 153 | struct page *page = virt_to_page(base); |
| 154 | if (!PageReserved(page)) { /* Is bootmem ? */ |
| 155 | kfree(base); |
| 156 | ms->page_cgroup = NULL; |
| 157 | } |
| 158 | } |
| 159 | } |
| 160 | |
| 161 | int online_page_cgroup(unsigned long start_pfn, |
| 162 | unsigned long nr_pages, |
| 163 | int nid) |
| 164 | { |
| 165 | unsigned long start, end, pfn; |
| 166 | int fail = 0; |
| 167 | |
| 168 | start = start_pfn & (PAGES_PER_SECTION - 1); |
| 169 | end = ALIGN(start_pfn + nr_pages, PAGES_PER_SECTION); |
| 170 | |
| 171 | for (pfn = start; !fail && pfn < end; pfn += PAGES_PER_SECTION) { |
| 172 | if (!pfn_present(pfn)) |
| 173 | continue; |
| 174 | fail = init_section_page_cgroup(pfn); |
| 175 | } |
| 176 | if (!fail) |
| 177 | return 0; |
| 178 | |
| 179 | /* rollback */ |
| 180 | for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION) |
| 181 | __free_page_cgroup(pfn); |
| 182 | |
| 183 | return -ENOMEM; |
| 184 | } |
| 185 | |
| 186 | int offline_page_cgroup(unsigned long start_pfn, |
| 187 | unsigned long nr_pages, int nid) |
| 188 | { |
| 189 | unsigned long start, end, pfn; |
| 190 | |
| 191 | start = start_pfn & (PAGES_PER_SECTION - 1); |
| 192 | end = ALIGN(start_pfn + nr_pages, PAGES_PER_SECTION); |
| 193 | |
| 194 | for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION) |
| 195 | __free_page_cgroup(pfn); |
| 196 | return 0; |
| 197 | |
| 198 | } |
| 199 | |
| 200 | static int page_cgroup_callback(struct notifier_block *self, |
| 201 | unsigned long action, void *arg) |
| 202 | { |
| 203 | struct memory_notify *mn = arg; |
| 204 | int ret = 0; |
| 205 | switch (action) { |
| 206 | case MEM_GOING_ONLINE: |
| 207 | ret = online_page_cgroup(mn->start_pfn, |
| 208 | mn->nr_pages, mn->status_change_nid); |
| 209 | break; |
| 210 | case MEM_CANCEL_ONLINE: |
| 211 | case MEM_OFFLINE: |
| 212 | offline_page_cgroup(mn->start_pfn, |
| 213 | mn->nr_pages, mn->status_change_nid); |
| 214 | break; |
| 215 | case MEM_GOING_OFFLINE: |
| 216 | break; |
| 217 | case MEM_ONLINE: |
| 218 | case MEM_CANCEL_OFFLINE: |
| 219 | break; |
| 220 | } |
| 221 | ret = notifier_from_errno(ret); |
| 222 | return ret; |
| 223 | } |
| 224 | |
| 225 | #endif |
| 226 | |
| 227 | void __init page_cgroup_init(void) |
| 228 | { |
| 229 | unsigned long pfn; |
| 230 | int fail = 0; |
| 231 | |
| 232 | if (mem_cgroup_subsys.disabled) |
| 233 | return; |
| 234 | |
| 235 | for (pfn = 0; !fail && pfn < max_pfn; pfn += PAGES_PER_SECTION) { |
| 236 | if (!pfn_present(pfn)) |
| 237 | continue; |
| 238 | fail = init_section_page_cgroup(pfn); |
| 239 | } |
| 240 | if (fail) { |
| 241 | printk(KERN_CRIT "try cgroup_disable=memory boot option\n"); |
| 242 | panic("Out of memory"); |
| 243 | } else { |
| 244 | hotplug_memory_notifier(page_cgroup_callback, 0); |
| 245 | } |
| 246 | printk(KERN_INFO "allocated %ld bytes of page_cgroup\n", total_usage); |
| 247 | printk(KERN_INFO "please try cgroup_disable=memory option if you don't" |
| 248 | " want\n"); |
| 249 | } |
| 250 | |
| 251 | void __init pgdat_page_cgroup_init(struct pglist_data *pgdat) |
| 252 | { |
| 253 | return; |
| 254 | } |
| 255 | |
| 256 | #endif |