Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | #ifndef __ASM_SPINLOCK_H |
2 | #define __ASM_SPINLOCK_H | |
3 | ||
4 | #include <asm/atomic.h> | |
5 | #include <asm/rwlock.h> | |
6 | #include <asm/page.h> | |
fb2e2848 | 7 | #include <asm/processor.h> |
1da177e4 LT |
8 | #include <linux/compiler.h> |
9 | ||
d3561b7f RR |
10 | #ifdef CONFIG_PARAVIRT |
11 | #include <asm/paravirt.h> | |
12 | #else | |
0da5db31 RR |
13 | #define CLI_STRING "cli" |
14 | #define STI_STRING "sti" | |
139ec7c4 RR |
15 | #define CLI_STI_CLOBBERS |
16 | #define CLI_STI_INPUT_ARGS | |
d3561b7f | 17 | #endif /* CONFIG_PARAVIRT */ |
0da5db31 | 18 | |
1da177e4 LT |
19 | /* |
20 | * Your basic SMP spinlocks, allowing only a single CPU anywhere | |
fb1c8f93 | 21 | * |
1da177e4 LT |
22 | * Simple spin lock operations. There are two variants, one clears IRQ's |
23 | * on the local processor, one does not. | |
24 | * | |
25 | * We make no fairness assumptions. They have a cost. | |
fb1c8f93 IM |
26 | * |
27 | * (the type definitions are in asm/spinlock_types.h) | |
1da177e4 LT |
28 | */ |
29 | ||
fb2e2848 AK |
30 | static inline int __raw_spin_is_locked(raw_spinlock_t *x) |
31 | { | |
32 | return *(volatile signed char *)(&(x)->slock) <= 0; | |
33 | } | |
1da177e4 | 34 | |
fb1c8f93 IM |
35 | static inline void __raw_spin_lock(raw_spinlock_t *lock) |
36 | { | |
fb2e2848 AK |
37 | asm volatile("\n1:\t" |
38 | LOCK_PREFIX " ; decb %0\n\t" | |
39 | "jns 3f\n" | |
40 | "2:\t" | |
41 | "rep;nop\n\t" | |
42 | "cmpb $0,%0\n\t" | |
43 | "jle 2b\n\t" | |
44 | "jmp 1b\n" | |
45 | "3:\n\t" | |
46 | : "+m" (lock->slock) : : "memory"); | |
fb1c8f93 IM |
47 | } |
48 | ||
8a25d5de IM |
49 | /* |
50 | * It is easier for the lock validator if interrupts are not re-enabled | |
51 | * in the middle of a lock-acquire. This is a performance feature anyway | |
52 | * so we turn it off: | |
fb2e2848 AK |
53 | * |
54 | * NOTE: there's an irqs-on section here, which normally would have to be | |
55 | * irq-traced, but on CONFIG_TRACE_IRQFLAGS we never use this variant. | |
8a25d5de IM |
56 | */ |
57 | #ifndef CONFIG_PROVE_LOCKING | |
fb1c8f93 IM |
58 | static inline void __raw_spin_lock_flags(raw_spinlock_t *lock, unsigned long flags) |
59 | { | |
fb2e2848 AK |
60 | asm volatile( |
61 | "\n1:\t" | |
139ec7c4 | 62 | LOCK_PREFIX " ; decb %[slock]\n\t" |
fb2e2848 AK |
63 | "jns 5f\n" |
64 | "2:\t" | |
139ec7c4 | 65 | "testl $0x200, %[flags]\n\t" |
fb2e2848 | 66 | "jz 4f\n\t" |
0da5db31 | 67 | STI_STRING "\n" |
fb2e2848 AK |
68 | "3:\t" |
69 | "rep;nop\n\t" | |
139ec7c4 | 70 | "cmpb $0, %[slock]\n\t" |
fb2e2848 | 71 | "jle 3b\n\t" |
0da5db31 | 72 | CLI_STRING "\n\t" |
fb2e2848 AK |
73 | "jmp 1b\n" |
74 | "4:\t" | |
75 | "rep;nop\n\t" | |
139ec7c4 | 76 | "cmpb $0, %[slock]\n\t" |
fb2e2848 AK |
77 | "jg 1b\n\t" |
78 | "jmp 4b\n" | |
79 | "5:\n\t" | |
139ec7c4 RR |
80 | : [slock] "+m" (lock->slock) |
81 | : [flags] "r" (flags) | |
82 | CLI_STI_INPUT_ARGS | |
83 | : "memory" CLI_STI_CLOBBERS); | |
fb1c8f93 | 84 | } |
8a25d5de | 85 | #endif |
fb1c8f93 IM |
86 | |
87 | static inline int __raw_spin_trylock(raw_spinlock_t *lock) | |
88 | { | |
89 | char oldval; | |
fb2e2848 | 90 | asm volatile( |
fb1c8f93 | 91 | "xchgb %b0,%1" |
b862f3b0 | 92 | :"=q" (oldval), "+m" (lock->slock) |
fb1c8f93 IM |
93 | :"0" (0) : "memory"); |
94 | return oldval > 0; | |
95 | } | |
96 | ||
1da177e4 | 97 | /* |
fb1c8f93 IM |
98 | * __raw_spin_unlock based on writing $1 to the low byte. |
99 | * This method works. Despite all the confusion. | |
100 | * (except on PPro SMP or if we are using OOSTORE, so we use xchgb there) | |
1da177e4 LT |
101 | * (PPro errata 66, 92) |
102 | */ | |
103 | ||
104 | #if !defined(CONFIG_X86_OOSTORE) && !defined(CONFIG_X86_PPRO_FENCE) | |
105 | ||
fb1c8f93 | 106 | static inline void __raw_spin_unlock(raw_spinlock_t *lock) |
1da177e4 | 107 | { |
fb2e2848 | 108 | asm volatile("movb $1,%0" : "+m" (lock->slock) :: "memory"); |
1da177e4 LT |
109 | } |
110 | ||
111 | #else | |
112 | ||
fb1c8f93 | 113 | static inline void __raw_spin_unlock(raw_spinlock_t *lock) |
1da177e4 LT |
114 | { |
115 | char oldval = 1; | |
1da177e4 | 116 | |
fb2e2848 AK |
117 | asm volatile("xchgb %b0, %1" |
118 | : "=q" (oldval), "+m" (lock->slock) | |
119 | : "0" (oldval) : "memory"); | |
1da177e4 LT |
120 | } |
121 | ||
1da177e4 | 122 | #endif |
1da177e4 | 123 | |
fb2e2848 AK |
124 | static inline void __raw_spin_unlock_wait(raw_spinlock_t *lock) |
125 | { | |
126 | while (__raw_spin_is_locked(lock)) | |
127 | cpu_relax(); | |
128 | } | |
1da177e4 LT |
129 | |
130 | /* | |
131 | * Read-write spinlocks, allowing multiple readers | |
132 | * but only one writer. | |
133 | * | |
134 | * NOTE! it is quite common to have readers in interrupts | |
135 | * but no interrupt writers. For those circumstances we | |
136 | * can "mix" irq-safe locks - any writer needs to get a | |
137 | * irq-safe write-lock, but readers can get non-irqsafe | |
138 | * read-locks. | |
fb1c8f93 IM |
139 | * |
140 | * On x86, we implement read-write locks as a 32-bit counter | |
141 | * with the high bit (sign) being the "contended" bit. | |
142 | * | |
143 | * The inline assembly is non-obvious. Think about it. | |
144 | * | |
145 | * Changed to use the same technique as rw semaphores. See | |
146 | * semaphore.h for details. -ben | |
147 | * | |
148 | * the helpers are in arch/i386/kernel/semaphore.c | |
1da177e4 | 149 | */ |
1da177e4 LT |
150 | |
151 | /** | |
152 | * read_can_lock - would read_trylock() succeed? | |
153 | * @lock: the rwlock in question. | |
154 | */ | |
fb2e2848 AK |
155 | static inline int __raw_read_can_lock(raw_rwlock_t *x) |
156 | { | |
157 | return (int)(x)->lock > 0; | |
158 | } | |
1da177e4 LT |
159 | |
160 | /** | |
161 | * write_can_lock - would write_trylock() succeed? | |
162 | * @lock: the rwlock in question. | |
163 | */ | |
fb2e2848 AK |
164 | static inline int __raw_write_can_lock(raw_rwlock_t *x) |
165 | { | |
166 | return (x)->lock == RW_LOCK_BIAS; | |
167 | } | |
1da177e4 | 168 | |
fb1c8f93 | 169 | static inline void __raw_read_lock(raw_rwlock_t *rw) |
1da177e4 | 170 | { |
fb2e2848 AK |
171 | asm volatile(LOCK_PREFIX " subl $1,(%0)\n\t" |
172 | "jns 1f\n" | |
173 | "call __read_lock_failed\n\t" | |
174 | "1:\n" | |
175 | ::"a" (rw) : "memory"); | |
1da177e4 LT |
176 | } |
177 | ||
fb1c8f93 | 178 | static inline void __raw_write_lock(raw_rwlock_t *rw) |
1da177e4 | 179 | { |
fb2e2848 AK |
180 | asm volatile(LOCK_PREFIX " subl $" RW_LOCK_BIAS_STR ",(%0)\n\t" |
181 | "jz 1f\n" | |
182 | "call __write_lock_failed\n\t" | |
183 | "1:\n" | |
184 | ::"a" (rw) : "memory"); | |
1da177e4 LT |
185 | } |
186 | ||
fb1c8f93 | 187 | static inline int __raw_read_trylock(raw_rwlock_t *lock) |
1da177e4 LT |
188 | { |
189 | atomic_t *count = (atomic_t *)lock; | |
190 | atomic_dec(count); | |
191 | if (atomic_read(count) >= 0) | |
192 | return 1; | |
193 | atomic_inc(count); | |
194 | return 0; | |
195 | } | |
196 | ||
fb1c8f93 | 197 | static inline int __raw_write_trylock(raw_rwlock_t *lock) |
1da177e4 LT |
198 | { |
199 | atomic_t *count = (atomic_t *)lock; | |
200 | if (atomic_sub_and_test(RW_LOCK_BIAS, count)) | |
201 | return 1; | |
202 | atomic_add(RW_LOCK_BIAS, count); | |
203 | return 0; | |
204 | } | |
205 | ||
fb1c8f93 IM |
206 | static inline void __raw_read_unlock(raw_rwlock_t *rw) |
207 | { | |
b862f3b0 | 208 | asm volatile(LOCK_PREFIX "incl %0" :"+m" (rw->lock) : : "memory"); |
fb1c8f93 IM |
209 | } |
210 | ||
211 | static inline void __raw_write_unlock(raw_rwlock_t *rw) | |
212 | { | |
9a0b5817 | 213 | asm volatile(LOCK_PREFIX "addl $" RW_LOCK_BIAS_STR ", %0" |
b862f3b0 | 214 | : "+m" (rw->lock) : : "memory"); |
fb1c8f93 IM |
215 | } |
216 | ||
ef6edc97 MS |
217 | #define _raw_spin_relax(lock) cpu_relax() |
218 | #define _raw_read_relax(lock) cpu_relax() | |
219 | #define _raw_write_relax(lock) cpu_relax() | |
220 | ||
1da177e4 | 221 | #endif /* __ASM_SPINLOCK_H */ |