Commit | Line | Data |
---|---|---|
2dc91abe NP |
1 | /* |
2 | * Specialised local-global spinlock. Can only be declared as global variables | |
3 | * to avoid overhead and keep things simple (and we don't want to start using | |
4 | * these inside dynamically allocated structures). | |
5 | * | |
6 | * "local/global locks" (lglocks) can be used to: | |
7 | * | |
8 | * - Provide fast exclusive access to per-CPU data, with exclusive access to | |
9 | * another CPU's data allowed but possibly subject to contention, and to | |
10 | * provide very slow exclusive access to all per-CPU data. | |
11 | * - Or to provide very fast and scalable read serialisation, and to provide | |
12 | * very slow exclusive serialisation of data (not necessarily per-CPU data). | |
13 | * | |
14 | * Brlocks are also implemented as a short-hand notation for the latter use | |
15 | * case. | |
16 | * | |
17 | * Copyright 2009, 2010, Nick Piggin, Novell Inc. | |
18 | */ | |
19 | #ifndef __LINUX_LGLOCK_H | |
20 | #define __LINUX_LGLOCK_H | |
21 | ||
22 | #include <linux/spinlock.h> | |
23 | #include <linux/lockdep.h> | |
24 | #include <linux/percpu.h> | |
e30e2fdf | 25 | #include <linux/cpu.h> |
2dc91abe NP |
26 | |
27 | /* can make br locks by using local lock for read side, global lock for write */ | |
28 | #define br_lock_init(name) name##_lock_init() | |
29 | #define br_read_lock(name) name##_local_lock() | |
30 | #define br_read_unlock(name) name##_local_unlock() | |
31 | #define br_write_lock(name) name##_global_lock_online() | |
32 | #define br_write_unlock(name) name##_global_unlock_online() | |
33 | ||
34 | #define DECLARE_BRLOCK(name) DECLARE_LGLOCK(name) | |
35 | #define DEFINE_BRLOCK(name) DEFINE_LGLOCK(name) | |
36 | ||
37 | ||
38 | #define lg_lock_init(name) name##_lock_init() | |
39 | #define lg_local_lock(name) name##_local_lock() | |
40 | #define lg_local_unlock(name) name##_local_unlock() | |
41 | #define lg_local_lock_cpu(name, cpu) name##_local_lock_cpu(cpu) | |
42 | #define lg_local_unlock_cpu(name, cpu) name##_local_unlock_cpu(cpu) | |
43 | #define lg_global_lock(name) name##_global_lock() | |
44 | #define lg_global_unlock(name) name##_global_unlock() | |
45 | #define lg_global_lock_online(name) name##_global_lock_online() | |
46 | #define lg_global_unlock_online(name) name##_global_unlock_online() | |
47 | ||
48 | #ifdef CONFIG_DEBUG_LOCK_ALLOC | |
49 | #define LOCKDEP_INIT_MAP lockdep_init_map | |
50 | ||
51 | #define DEFINE_LGLOCK_LOCKDEP(name) \ | |
52 | struct lock_class_key name##_lock_key; \ | |
53 | struct lockdep_map name##_lock_dep_map; \ | |
54 | EXPORT_SYMBOL(name##_lock_dep_map) | |
55 | ||
56 | #else | |
57 | #define LOCKDEP_INIT_MAP(a, b, c, d) | |
58 | ||
59 | #define DEFINE_LGLOCK_LOCKDEP(name) | |
60 | #endif | |
61 | ||
62 | ||
63 | #define DECLARE_LGLOCK(name) \ | |
64 | extern void name##_lock_init(void); \ | |
65 | extern void name##_local_lock(void); \ | |
66 | extern void name##_local_unlock(void); \ | |
67 | extern void name##_local_lock_cpu(int cpu); \ | |
68 | extern void name##_local_unlock_cpu(int cpu); \ | |
69 | extern void name##_global_lock(void); \ | |
70 | extern void name##_global_unlock(void); \ | |
71 | extern void name##_global_lock_online(void); \ | |
72 | extern void name##_global_unlock_online(void); \ | |
73 | ||
74 | #define DEFINE_LGLOCK(name) \ | |
75 | \ | |
e30e2fdf SB |
76 | DEFINE_SPINLOCK(name##_cpu_lock); \ |
77 | cpumask_t name##_cpus __read_mostly; \ | |
2dc91abe NP |
78 | DEFINE_PER_CPU(arch_spinlock_t, name##_lock); \ |
79 | DEFINE_LGLOCK_LOCKDEP(name); \ | |
80 | \ | |
e30e2fdf SB |
81 | static int \ |
82 | name##_lg_cpu_callback(struct notifier_block *nb, \ | |
83 | unsigned long action, void *hcpu) \ | |
84 | { \ | |
85 | switch (action & ~CPU_TASKS_FROZEN) { \ | |
86 | case CPU_UP_PREPARE: \ | |
87 | spin_lock(&name##_cpu_lock); \ | |
88 | cpu_set((unsigned long)hcpu, name##_cpus); \ | |
89 | spin_unlock(&name##_cpu_lock); \ | |
90 | break; \ | |
91 | case CPU_UP_CANCELED: case CPU_DEAD: \ | |
92 | spin_lock(&name##_cpu_lock); \ | |
93 | cpu_clear((unsigned long)hcpu, name##_cpus); \ | |
94 | spin_unlock(&name##_cpu_lock); \ | |
95 | } \ | |
96 | return NOTIFY_OK; \ | |
97 | } \ | |
98 | static struct notifier_block name##_lg_cpu_notifier = { \ | |
99 | .notifier_call = name##_lg_cpu_callback, \ | |
100 | }; \ | |
2dc91abe NP |
101 | void name##_lock_init(void) { \ |
102 | int i; \ | |
103 | LOCKDEP_INIT_MAP(&name##_lock_dep_map, #name, &name##_lock_key, 0); \ | |
104 | for_each_possible_cpu(i) { \ | |
105 | arch_spinlock_t *lock; \ | |
106 | lock = &per_cpu(name##_lock, i); \ | |
107 | *lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED; \ | |
108 | } \ | |
e30e2fdf SB |
109 | register_hotcpu_notifier(&name##_lg_cpu_notifier); \ |
110 | get_online_cpus(); \ | |
111 | for_each_online_cpu(i) \ | |
112 | cpu_set(i, name##_cpus); \ | |
113 | put_online_cpus(); \ | |
2dc91abe NP |
114 | } \ |
115 | EXPORT_SYMBOL(name##_lock_init); \ | |
116 | \ | |
117 | void name##_local_lock(void) { \ | |
118 | arch_spinlock_t *lock; \ | |
119 | preempt_disable(); \ | |
120 | rwlock_acquire_read(&name##_lock_dep_map, 0, 0, _THIS_IP_); \ | |
121 | lock = &__get_cpu_var(name##_lock); \ | |
122 | arch_spin_lock(lock); \ | |
123 | } \ | |
124 | EXPORT_SYMBOL(name##_local_lock); \ | |
125 | \ | |
126 | void name##_local_unlock(void) { \ | |
127 | arch_spinlock_t *lock; \ | |
128 | rwlock_release(&name##_lock_dep_map, 1, _THIS_IP_); \ | |
129 | lock = &__get_cpu_var(name##_lock); \ | |
130 | arch_spin_unlock(lock); \ | |
131 | preempt_enable(); \ | |
132 | } \ | |
133 | EXPORT_SYMBOL(name##_local_unlock); \ | |
134 | \ | |
135 | void name##_local_lock_cpu(int cpu) { \ | |
136 | arch_spinlock_t *lock; \ | |
137 | preempt_disable(); \ | |
138 | rwlock_acquire_read(&name##_lock_dep_map, 0, 0, _THIS_IP_); \ | |
139 | lock = &per_cpu(name##_lock, cpu); \ | |
140 | arch_spin_lock(lock); \ | |
141 | } \ | |
142 | EXPORT_SYMBOL(name##_local_lock_cpu); \ | |
143 | \ | |
144 | void name##_local_unlock_cpu(int cpu) { \ | |
145 | arch_spinlock_t *lock; \ | |
146 | rwlock_release(&name##_lock_dep_map, 1, _THIS_IP_); \ | |
147 | lock = &per_cpu(name##_lock, cpu); \ | |
148 | arch_spin_unlock(lock); \ | |
149 | preempt_enable(); \ | |
150 | } \ | |
151 | EXPORT_SYMBOL(name##_local_unlock_cpu); \ | |
152 | \ | |
153 | void name##_global_lock_online(void) { \ | |
154 | int i; \ | |
e30e2fdf | 155 | spin_lock(&name##_cpu_lock); \ |
2dc91abe | 156 | rwlock_acquire(&name##_lock_dep_map, 0, 0, _RET_IP_); \ |
e30e2fdf | 157 | for_each_cpu(i, &name##_cpus) { \ |
2dc91abe NP |
158 | arch_spinlock_t *lock; \ |
159 | lock = &per_cpu(name##_lock, i); \ | |
160 | arch_spin_lock(lock); \ | |
161 | } \ | |
162 | } \ | |
163 | EXPORT_SYMBOL(name##_global_lock_online); \ | |
164 | \ | |
165 | void name##_global_unlock_online(void) { \ | |
166 | int i; \ | |
167 | rwlock_release(&name##_lock_dep_map, 1, _RET_IP_); \ | |
e30e2fdf | 168 | for_each_cpu(i, &name##_cpus) { \ |
2dc91abe NP |
169 | arch_spinlock_t *lock; \ |
170 | lock = &per_cpu(name##_lock, i); \ | |
171 | arch_spin_unlock(lock); \ | |
172 | } \ | |
e30e2fdf | 173 | spin_unlock(&name##_cpu_lock); \ |
2dc91abe NP |
174 | } \ |
175 | EXPORT_SYMBOL(name##_global_unlock_online); \ | |
176 | \ | |
177 | void name##_global_lock(void) { \ | |
178 | int i; \ | |
179 | preempt_disable(); \ | |
180 | rwlock_acquire(&name##_lock_dep_map, 0, 0, _RET_IP_); \ | |
a73f8844 | 181 | for_each_possible_cpu(i) { \ |
2dc91abe NP |
182 | arch_spinlock_t *lock; \ |
183 | lock = &per_cpu(name##_lock, i); \ | |
184 | arch_spin_lock(lock); \ | |
185 | } \ | |
186 | } \ | |
187 | EXPORT_SYMBOL(name##_global_lock); \ | |
188 | \ | |
189 | void name##_global_unlock(void) { \ | |
190 | int i; \ | |
191 | rwlock_release(&name##_lock_dep_map, 1, _RET_IP_); \ | |
a73f8844 | 192 | for_each_possible_cpu(i) { \ |
2dc91abe NP |
193 | arch_spinlock_t *lock; \ |
194 | lock = &per_cpu(name##_lock, i); \ | |
195 | arch_spin_unlock(lock); \ | |
196 | } \ | |
197 | preempt_enable(); \ | |
198 | } \ | |
199 | EXPORT_SYMBOL(name##_global_unlock); | |
200 | #endif |