Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | #ifndef _LINUX_PERCPU_COUNTER_H |
2 | #define _LINUX_PERCPU_COUNTER_H | |
3 | /* | |
4 | * A simple "approximate counter" for use in ext2 and ext3 superblocks. | |
5 | * | |
6 | * WARNING: these things are HUGE. 4 kbytes per counter on 32-way P4. | |
7 | */ | |
8 | ||
1da177e4 LT |
9 | #include <linux/spinlock.h> |
10 | #include <linux/smp.h> | |
c67ad917 | 11 | #include <linux/list.h> |
1da177e4 LT |
12 | #include <linux/threads.h> |
13 | #include <linux/percpu.h> | |
0216bfcf | 14 | #include <linux/types.h> |
908c7f19 | 15 | #include <linux/gfp.h> |
1da177e4 LT |
16 | |
17 | #ifdef CONFIG_SMP | |
18 | ||
19 | struct percpu_counter { | |
f032a450 | 20 | raw_spinlock_t lock; |
0216bfcf | 21 | s64 count; |
c67ad917 AM |
22 | #ifdef CONFIG_HOTPLUG_CPU |
23 | struct list_head list; /* All percpu_counters are on a list */ | |
24 | #endif | |
43cf38eb | 25 | s32 __percpu *counters; |
1da177e4 LT |
26 | }; |
27 | ||
179f7ebf | 28 | extern int percpu_counter_batch; |
1da177e4 | 29 | |
908c7f19 | 30 | int __percpu_counter_init(struct percpu_counter *fbc, s64 amount, gfp_t gfp, |
ea319518 PZ |
31 | struct lock_class_key *key); |
32 | ||
908c7f19 | 33 | #define percpu_counter_init(fbc, value, gfp) \ |
ea319518 PZ |
34 | ({ \ |
35 | static struct lock_class_key __key; \ | |
36 | \ | |
908c7f19 | 37 | __percpu_counter_init(fbc, value, gfp, &__key); \ |
ea319518 PZ |
38 | }) |
39 | ||
c67ad917 | 40 | void percpu_counter_destroy(struct percpu_counter *fbc); |
3a587f47 | 41 | void percpu_counter_set(struct percpu_counter *fbc, s64 amount); |
20e89767 | 42 | void __percpu_counter_add(struct percpu_counter *fbc, s64 amount, s32 batch); |
02d21168 | 43 | s64 __percpu_counter_sum(struct percpu_counter *fbc); |
80188b0d DC |
44 | int __percpu_counter_compare(struct percpu_counter *fbc, s64 rhs, s32 batch); |
45 | ||
46 | static inline int percpu_counter_compare(struct percpu_counter *fbc, s64 rhs) | |
47 | { | |
48 | return __percpu_counter_compare(fbc, rhs, percpu_counter_batch); | |
49 | } | |
1da177e4 | 50 | |
20e89767 | 51 | static inline void percpu_counter_add(struct percpu_counter *fbc, s64 amount) |
252e0ba6 | 52 | { |
179f7ebf | 53 | __percpu_counter_add(fbc, amount, percpu_counter_batch); |
252e0ba6 PZ |
54 | } |
55 | ||
bf1d89c8 PZ |
56 | static inline s64 percpu_counter_sum_positive(struct percpu_counter *fbc) |
57 | { | |
02d21168 | 58 | s64 ret = __percpu_counter_sum(fbc); |
bf1d89c8 PZ |
59 | return ret < 0 ? 0 : ret; |
60 | } | |
61 | ||
62 | static inline s64 percpu_counter_sum(struct percpu_counter *fbc) | |
63 | { | |
02d21168 | 64 | return __percpu_counter_sum(fbc); |
bf1d89c8 PZ |
65 | } |
66 | ||
0216bfcf | 67 | static inline s64 percpu_counter_read(struct percpu_counter *fbc) |
1da177e4 LT |
68 | { |
69 | return fbc->count; | |
70 | } | |
71 | ||
72 | /* | |
73 | * It is possible for the percpu_counter_read() to return a small negative | |
74 | * number for some counter which should never be negative. | |
0216bfcf | 75 | * |
1da177e4 | 76 | */ |
0216bfcf | 77 | static inline s64 percpu_counter_read_positive(struct percpu_counter *fbc) |
1da177e4 | 78 | { |
0216bfcf | 79 | s64 ret = fbc->count; |
1da177e4 LT |
80 | |
81 | barrier(); /* Prevent reloads of fbc->count */ | |
0216bfcf | 82 | if (ret >= 0) |
1da177e4 | 83 | return ret; |
c84598bb | 84 | return 0; |
1da177e4 LT |
85 | } |
86 | ||
7f93cff9 TT |
87 | static inline int percpu_counter_initialized(struct percpu_counter *fbc) |
88 | { | |
89 | return (fbc->counters != NULL); | |
90 | } | |
91 | ||
7fa4cf92 | 92 | #else /* !CONFIG_SMP */ |
1da177e4 LT |
93 | |
94 | struct percpu_counter { | |
0216bfcf | 95 | s64 count; |
1da177e4 LT |
96 | }; |
97 | ||
908c7f19 TH |
98 | static inline int percpu_counter_init(struct percpu_counter *fbc, s64 amount, |
99 | gfp_t gfp) | |
1da177e4 | 100 | { |
0216bfcf | 101 | fbc->count = amount; |
833f4077 | 102 | return 0; |
1da177e4 LT |
103 | } |
104 | ||
105 | static inline void percpu_counter_destroy(struct percpu_counter *fbc) | |
106 | { | |
107 | } | |
108 | ||
3a587f47 PZ |
109 | static inline void percpu_counter_set(struct percpu_counter *fbc, s64 amount) |
110 | { | |
111 | fbc->count = amount; | |
112 | } | |
113 | ||
27f5e0f6 TC |
114 | static inline int percpu_counter_compare(struct percpu_counter *fbc, s64 rhs) |
115 | { | |
116 | if (fbc->count > rhs) | |
117 | return 1; | |
118 | else if (fbc->count < rhs) | |
119 | return -1; | |
120 | else | |
121 | return 0; | |
122 | } | |
123 | ||
80188b0d DC |
124 | static inline int |
125 | __percpu_counter_compare(struct percpu_counter *fbc, s64 rhs, s32 batch) | |
126 | { | |
127 | return percpu_counter_compare(fbc, rhs); | |
128 | } | |
129 | ||
1da177e4 | 130 | static inline void |
20e89767 | 131 | percpu_counter_add(struct percpu_counter *fbc, s64 amount) |
1da177e4 LT |
132 | { |
133 | preempt_disable(); | |
134 | fbc->count += amount; | |
135 | preempt_enable(); | |
136 | } | |
137 | ||
0c9cf2ef AB |
138 | static inline void |
139 | __percpu_counter_add(struct percpu_counter *fbc, s64 amount, s32 batch) | |
140 | { | |
141 | percpu_counter_add(fbc, amount); | |
142 | } | |
143 | ||
0216bfcf | 144 | static inline s64 percpu_counter_read(struct percpu_counter *fbc) |
1da177e4 LT |
145 | { |
146 | return fbc->count; | |
147 | } | |
148 | ||
c84598bb SL |
149 | /* |
150 | * percpu_counter is intended to track positive numbers. In the UP case the | |
151 | * number should never be negative. | |
152 | */ | |
0216bfcf | 153 | static inline s64 percpu_counter_read_positive(struct percpu_counter *fbc) |
1da177e4 LT |
154 | { |
155 | return fbc->count; | |
156 | } | |
157 | ||
52d9f3b4 | 158 | static inline s64 percpu_counter_sum_positive(struct percpu_counter *fbc) |
e2bab3d9 AM |
159 | { |
160 | return percpu_counter_read_positive(fbc); | |
161 | } | |
162 | ||
bf1d89c8 PZ |
163 | static inline s64 percpu_counter_sum(struct percpu_counter *fbc) |
164 | { | |
165 | return percpu_counter_read(fbc); | |
166 | } | |
167 | ||
7f93cff9 TT |
168 | static inline int percpu_counter_initialized(struct percpu_counter *fbc) |
169 | { | |
170 | return 1; | |
171 | } | |
172 | ||
1da177e4 LT |
173 | #endif /* CONFIG_SMP */ |
174 | ||
175 | static inline void percpu_counter_inc(struct percpu_counter *fbc) | |
176 | { | |
aa0dff2d | 177 | percpu_counter_add(fbc, 1); |
1da177e4 LT |
178 | } |
179 | ||
180 | static inline void percpu_counter_dec(struct percpu_counter *fbc) | |
181 | { | |
aa0dff2d | 182 | percpu_counter_add(fbc, -1); |
1da177e4 LT |
183 | } |
184 | ||
3cb4f9fa PZ |
185 | static inline void percpu_counter_sub(struct percpu_counter *fbc, s64 amount) |
186 | { | |
187 | percpu_counter_add(fbc, -amount); | |
188 | } | |
189 | ||
1da177e4 | 190 | #endif /* _LINUX_PERCPU_COUNTER_H */ |