2 * Fast batching percpu counters.
5 #include <linux/percpu_counter.h>
6 #include <linux/notifier.h>
7 #include <linux/mutex.h>
8 #include <linux/init.h>
10 #include <linux/module.h>
12 #ifdef CONFIG_HOTPLUG_CPU
13 static LIST_HEAD(percpu_counters
);
14 static DEFINE_MUTEX(percpu_counters_lock
);
17 void percpu_counter_set(struct percpu_counter
*fbc
, s64 amount
)
21 spin_lock(&fbc
->lock
);
22 for_each_possible_cpu(cpu
) {
23 s32
*pcount
= per_cpu_ptr(fbc
->counters
, cpu
);
27 spin_unlock(&fbc
->lock
);
29 EXPORT_SYMBOL(percpu_counter_set
);
31 void __percpu_counter_add(struct percpu_counter
*fbc
, s64 amount
, s32 batch
)
37 pcount
= per_cpu_ptr(fbc
->counters
, cpu
);
38 count
= *pcount
+ amount
;
39 if (count
>= batch
|| count
<= -batch
) {
40 spin_lock(&fbc
->lock
);
43 spin_unlock(&fbc
->lock
);
49 EXPORT_SYMBOL(__percpu_counter_add
);
52 * Add up all the per-cpu counts, return the result. This is a more accurate
53 * but much slower version of percpu_counter_read_positive()
55 s64
__percpu_counter_sum(struct percpu_counter
*fbc
)
60 spin_lock(&fbc
->lock
);
62 for_each_online_cpu(cpu
) {
63 s32
*pcount
= per_cpu_ptr(fbc
->counters
, cpu
);
66 spin_unlock(&fbc
->lock
);
69 EXPORT_SYMBOL(__percpu_counter_sum
);
71 void percpu_counter_init(struct percpu_counter
*fbc
, s64 amount
)
73 spin_lock_init(&fbc
->lock
);
75 fbc
->counters
= alloc_percpu(s32
);
76 #ifdef CONFIG_HOTPLUG_CPU
77 mutex_lock(&percpu_counters_lock
);
78 list_add(&fbc
->list
, &percpu_counters
);
79 mutex_unlock(&percpu_counters_lock
);
82 EXPORT_SYMBOL(percpu_counter_init
);
84 void percpu_counter_destroy(struct percpu_counter
*fbc
)
86 free_percpu(fbc
->counters
);
87 #ifdef CONFIG_HOTPLUG_CPU
88 mutex_lock(&percpu_counters_lock
);
90 mutex_unlock(&percpu_counters_lock
);
93 EXPORT_SYMBOL(percpu_counter_destroy
);
95 #ifdef CONFIG_HOTPLUG_CPU
96 static int __cpuinit
percpu_counter_hotcpu_callback(struct notifier_block
*nb
,
97 unsigned long action
, void *hcpu
)
100 struct percpu_counter
*fbc
;
102 if (action
!= CPU_DEAD
)
105 cpu
= (unsigned long)hcpu
;
106 mutex_lock(&percpu_counters_lock
);
107 list_for_each_entry(fbc
, &percpu_counters
, list
) {
110 spin_lock(&fbc
->lock
);
111 pcount
= per_cpu_ptr(fbc
->counters
, cpu
);
112 fbc
->count
+= *pcount
;
114 spin_unlock(&fbc
->lock
);
116 mutex_unlock(&percpu_counters_lock
);
120 static int __init
percpu_counter_startup(void)
122 hotcpu_notifier(percpu_counter_hotcpu_callback
, 0);
125 module_init(percpu_counter_startup
);