Commit | Line | Data |
---|---|---|
d00bcc98 CL |
1 | /* |
2 | * linux/mm/allocpercpu.c | |
3 | * | |
4 | * Separated from slab.c August 11, 2006 Christoph Lameter <clameter@sgi.com> | |
5 | */ | |
6 | #include <linux/mm.h> | |
7 | #include <linux/module.h> | |
8 | ||
be852795 ED |
9 | #ifndef cache_line_size |
10 | #define cache_line_size() L1_CACHE_BYTES | |
11 | #endif | |
12 | ||
d00bcc98 CL |
13 | /** |
14 | * percpu_depopulate - depopulate per-cpu data for given cpu | |
15 | * @__pdata: per-cpu data to depopulate | |
16 | * @cpu: depopulate per-cpu data for this cpu | |
17 | * | |
18 | * Depopulating per-cpu data for a cpu going offline would be a typical | |
19 | * use case. You need to register a cpu hotplug handler for that purpose. | |
20 | */ | |
21 | void percpu_depopulate(void *__pdata, int cpu) | |
22 | { | |
23 | struct percpu_data *pdata = __percpu_disguise(__pdata); | |
a1205868 AS |
24 | |
25 | kfree(pdata->ptrs[cpu]); | |
26 | pdata->ptrs[cpu] = NULL; | |
d00bcc98 CL |
27 | } |
28 | EXPORT_SYMBOL_GPL(percpu_depopulate); | |
29 | ||
30 | /** | |
31 | * percpu_depopulate_mask - depopulate per-cpu data for some cpu's | |
32 | * @__pdata: per-cpu data to depopulate | |
33 | * @mask: depopulate per-cpu data for cpu's selected through mask bits | |
34 | */ | |
35 | void __percpu_depopulate_mask(void *__pdata, cpumask_t *mask) | |
36 | { | |
37 | int cpu; | |
38 | for_each_cpu_mask(cpu, *mask) | |
39 | percpu_depopulate(__pdata, cpu); | |
40 | } | |
41 | EXPORT_SYMBOL_GPL(__percpu_depopulate_mask); | |
42 | ||
43 | /** | |
44 | * percpu_populate - populate per-cpu data for given cpu | |
45 | * @__pdata: per-cpu data to populate further | |
46 | * @size: size of per-cpu object | |
47 | * @gfp: may sleep or not etc. | |
48 | * @cpu: populate per-data for this cpu | |
49 | * | |
50 | * Populating per-cpu data for a cpu coming online would be a typical | |
51 | * use case. You need to register a cpu hotplug handler for that purpose. | |
52 | * Per-cpu object is populated with zeroed buffer. | |
53 | */ | |
54 | void *percpu_populate(void *__pdata, size_t size, gfp_t gfp, int cpu) | |
55 | { | |
56 | struct percpu_data *pdata = __percpu_disguise(__pdata); | |
57 | int node = cpu_to_node(cpu); | |
58 | ||
be852795 ED |
59 | /* |
60 | * We should make sure each CPU gets private memory. | |
61 | */ | |
62 | size = roundup(size, cache_line_size()); | |
63 | ||
d00bcc98 | 64 | BUG_ON(pdata->ptrs[cpu]); |
94f6030c CL |
65 | if (node_online(node)) |
66 | pdata->ptrs[cpu] = kmalloc_node(size, gfp|__GFP_ZERO, node); | |
67 | else | |
d00bcc98 CL |
68 | pdata->ptrs[cpu] = kzalloc(size, gfp); |
69 | return pdata->ptrs[cpu]; | |
70 | } | |
71 | EXPORT_SYMBOL_GPL(percpu_populate); | |
72 | ||
73 | /** | |
74 | * percpu_populate_mask - populate per-cpu data for more cpu's | |
75 | * @__pdata: per-cpu data to populate further | |
76 | * @size: size of per-cpu object | |
77 | * @gfp: may sleep or not etc. | |
78 | * @mask: populate per-cpu data for cpu's selected through mask bits | |
79 | * | |
80 | * Per-cpu objects are populated with zeroed buffers. | |
81 | */ | |
82 | int __percpu_populate_mask(void *__pdata, size_t size, gfp_t gfp, | |
83 | cpumask_t *mask) | |
84 | { | |
d366f8cb | 85 | cpumask_t populated; |
d00bcc98 CL |
86 | int cpu; |
87 | ||
d366f8cb | 88 | cpus_clear(populated); |
d00bcc98 CL |
89 | for_each_cpu_mask(cpu, *mask) |
90 | if (unlikely(!percpu_populate(__pdata, size, gfp, cpu))) { | |
91 | __percpu_depopulate_mask(__pdata, &populated); | |
92 | return -ENOMEM; | |
93 | } else | |
94 | cpu_set(cpu, populated); | |
95 | return 0; | |
96 | } | |
97 | EXPORT_SYMBOL_GPL(__percpu_populate_mask); | |
98 | ||
99 | /** | |
100 | * percpu_alloc_mask - initial setup of per-cpu data | |
101 | * @size: size of per-cpu object | |
102 | * @gfp: may sleep or not etc. | |
103 | * @mask: populate per-data for cpu's selected through mask bits | |
104 | * | |
105 | * Populating per-cpu data for all online cpu's would be a typical use case, | |
106 | * which is simplified by the percpu_alloc() wrapper. | |
107 | * Per-cpu objects are populated with zeroed buffers. | |
108 | */ | |
109 | void *__percpu_alloc_mask(size_t size, gfp_t gfp, cpumask_t *mask) | |
110 | { | |
be852795 ED |
111 | /* |
112 | * We allocate whole cache lines to avoid false sharing | |
113 | */ | |
114 | size_t sz = roundup(nr_cpu_ids * sizeof(void *), cache_line_size()); | |
115 | void *pdata = kzalloc(sz, gfp); | |
d00bcc98 CL |
116 | void *__pdata = __percpu_disguise(pdata); |
117 | ||
118 | if (unlikely(!pdata)) | |
119 | return NULL; | |
120 | if (likely(!__percpu_populate_mask(__pdata, size, gfp, mask))) | |
121 | return __pdata; | |
122 | kfree(pdata); | |
123 | return NULL; | |
124 | } | |
125 | EXPORT_SYMBOL_GPL(__percpu_alloc_mask); | |
126 | ||
127 | /** | |
128 | * percpu_free - final cleanup of per-cpu data | |
129 | * @__pdata: object to clean up | |
130 | * | |
131 | * We simply clean up any per-cpu object left. No need for the client to | |
132 | * track and specify through a bis mask which per-cpu objects are to free. | |
133 | */ | |
134 | void percpu_free(void *__pdata) | |
135 | { | |
a1205868 AS |
136 | if (unlikely(!__pdata)) |
137 | return; | |
d00bcc98 CL |
138 | __percpu_depopulate_mask(__pdata, &cpu_possible_map); |
139 | kfree(__percpu_disguise(__pdata)); | |
140 | } | |
141 | EXPORT_SYMBOL_GPL(percpu_free); |