Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | #ifndef __ASM_SPINLOCK_H |
2 | #define __ASM_SPINLOCK_H | |
3 | ||
4 | #include <asm/atomic.h> | |
5 | #include <asm/rwlock.h> | |
6 | #include <asm/page.h> | |
fb2e2848 | 7 | #include <asm/processor.h> |
0da5db31 | 8 | |
1da177e4 LT |
9 | /* |
10 | * Your basic SMP spinlocks, allowing only a single CPU anywhere | |
fb1c8f93 | 11 | * |
1da177e4 LT |
12 | * Simple spin lock operations. There are two variants, one clears IRQ's |
13 | * on the local processor, one does not. | |
14 | * | |
15 | * We make no fairness assumptions. They have a cost. | |
fb1c8f93 IM |
16 | * |
17 | * (the type definitions are in asm/spinlock_types.h) | |
1da177e4 LT |
18 | */ |
19 | ||
2fed0c50 | 20 | static inline int __raw_spin_is_locked(raw_spinlock_t *lock) |
fb2e2848 | 21 | { |
2fed0c50 | 22 | return *(volatile signed char *)(&(lock)->slock) <= 0; |
fb2e2848 | 23 | } |
1da177e4 | 24 | |
fb1c8f93 IM |
25 | static inline void __raw_spin_lock(raw_spinlock_t *lock) |
26 | { | |
2fed0c50 GOC |
27 | asm volatile( |
28 | "\n1:\t" | |
29 | LOCK_PREFIX " ; decb %0\n\t" | |
30 | "jns 3f\n" | |
31 | "2:\t" | |
32 | "rep;nop\n\t" | |
33 | "cmpb $0,%0\n\t" | |
34 | "jle 2b\n\t" | |
35 | "jmp 1b\n" | |
36 | "3:\n\t" | |
37 | : "+m" (lock->slock) : : "memory"); | |
fb1c8f93 IM |
38 | } |
39 | ||
8a25d5de IM |
40 | /* |
41 | * It is easier for the lock validator if interrupts are not re-enabled | |
42 | * in the middle of a lock-acquire. This is a performance feature anyway | |
43 | * so we turn it off: | |
fb2e2848 AK |
44 | * |
45 | * NOTE: there's an irqs-on section here, which normally would have to be | |
46 | * irq-traced, but on CONFIG_TRACE_IRQFLAGS we never use this variant. | |
8a25d5de IM |
47 | */ |
48 | #ifndef CONFIG_PROVE_LOCKING | |
2fed0c50 GOC |
49 | static inline void __raw_spin_lock_flags(raw_spinlock_t *lock, |
50 | unsigned long flags) | |
fb1c8f93 | 51 | { |
fb2e2848 AK |
52 | asm volatile( |
53 | "\n1:\t" | |
139ec7c4 | 54 | LOCK_PREFIX " ; decb %[slock]\n\t" |
fb2e2848 AK |
55 | "jns 5f\n" |
56 | "2:\t" | |
139ec7c4 | 57 | "testl $0x200, %[flags]\n\t" |
fb2e2848 | 58 | "jz 4f\n\t" |
0da5db31 | 59 | STI_STRING "\n" |
fb2e2848 AK |
60 | "3:\t" |
61 | "rep;nop\n\t" | |
139ec7c4 | 62 | "cmpb $0, %[slock]\n\t" |
fb2e2848 | 63 | "jle 3b\n\t" |
0da5db31 | 64 | CLI_STRING "\n\t" |
fb2e2848 AK |
65 | "jmp 1b\n" |
66 | "4:\t" | |
67 | "rep;nop\n\t" | |
139ec7c4 | 68 | "cmpb $0, %[slock]\n\t" |
fb2e2848 AK |
69 | "jg 1b\n\t" |
70 | "jmp 4b\n" | |
71 | "5:\n\t" | |
139ec7c4 RR |
72 | : [slock] "+m" (lock->slock) |
73 | : [flags] "r" (flags) | |
2fed0c50 | 74 | CLI_STI_INPUT_ARGS |
139ec7c4 | 75 | : "memory" CLI_STI_CLOBBERS); |
fb1c8f93 | 76 | } |
8a25d5de | 77 | #endif |
fb1c8f93 IM |
78 | |
79 | static inline int __raw_spin_trylock(raw_spinlock_t *lock) | |
80 | { | |
2fed0c50 GOC |
81 | signed char oldval; |
82 | ||
fb2e2848 | 83 | asm volatile( |
fb1c8f93 | 84 | "xchgb %b0,%1" |
b862f3b0 | 85 | :"=q" (oldval), "+m" (lock->slock) |
fb1c8f93 | 86 | :"0" (0) : "memory"); |
2fed0c50 | 87 | |
fb1c8f93 IM |
88 | return oldval > 0; |
89 | } | |
90 | ||
1da177e4 | 91 | /* |
fb1c8f93 IM |
92 | * __raw_spin_unlock based on writing $1 to the low byte. |
93 | * This method works. Despite all the confusion. | |
94 | * (except on PPro SMP or if we are using OOSTORE, so we use xchgb there) | |
1da177e4 LT |
95 | * (PPro errata 66, 92) |
96 | */ | |
97 | ||
98 | #if !defined(CONFIG_X86_OOSTORE) && !defined(CONFIG_X86_PPRO_FENCE) | |
99 | ||
fb1c8f93 | 100 | static inline void __raw_spin_unlock(raw_spinlock_t *lock) |
1da177e4 | 101 | { |
fb2e2848 | 102 | asm volatile("movb $1,%0" : "+m" (lock->slock) :: "memory"); |
1da177e4 LT |
103 | } |
104 | ||
105 | #else | |
106 | ||
fb1c8f93 | 107 | static inline void __raw_spin_unlock(raw_spinlock_t *lock) |
1da177e4 | 108 | { |
2fed0c50 | 109 | unsigned char oldval = 1; |
1da177e4 | 110 | |
fb2e2848 AK |
111 | asm volatile("xchgb %b0, %1" |
112 | : "=q" (oldval), "+m" (lock->slock) | |
113 | : "0" (oldval) : "memory"); | |
1da177e4 LT |
114 | } |
115 | ||
1da177e4 | 116 | #endif |
1da177e4 | 117 | |
fb2e2848 AK |
118 | static inline void __raw_spin_unlock_wait(raw_spinlock_t *lock) |
119 | { | |
120 | while (__raw_spin_is_locked(lock)) | |
121 | cpu_relax(); | |
122 | } | |
1da177e4 LT |
123 | |
124 | /* | |
125 | * Read-write spinlocks, allowing multiple readers | |
126 | * but only one writer. | |
127 | * | |
128 | * NOTE! it is quite common to have readers in interrupts | |
129 | * but no interrupt writers. For those circumstances we | |
130 | * can "mix" irq-safe locks - any writer needs to get a | |
131 | * irq-safe write-lock, but readers can get non-irqsafe | |
132 | * read-locks. | |
fb1c8f93 IM |
133 | * |
134 | * On x86, we implement read-write locks as a 32-bit counter | |
135 | * with the high bit (sign) being the "contended" bit. | |
1da177e4 | 136 | */ |
1da177e4 | 137 | |
2fed0c50 | 138 | static inline int __raw_read_can_lock(raw_rwlock_t *lock) |
fb2e2848 | 139 | { |
2fed0c50 | 140 | return (int)(lock)->lock > 0; |
fb2e2848 | 141 | } |
1da177e4 | 142 | |
2fed0c50 | 143 | static inline int __raw_write_can_lock(raw_rwlock_t *lock) |
fb2e2848 | 144 | { |
2fed0c50 | 145 | return (lock)->lock == RW_LOCK_BIAS; |
fb2e2848 | 146 | } |
1da177e4 | 147 | |
fb1c8f93 | 148 | static inline void __raw_read_lock(raw_rwlock_t *rw) |
1da177e4 | 149 | { |
fb2e2848 AK |
150 | asm volatile(LOCK_PREFIX " subl $1,(%0)\n\t" |
151 | "jns 1f\n" | |
152 | "call __read_lock_failed\n\t" | |
153 | "1:\n" | |
154 | ::"a" (rw) : "memory"); | |
1da177e4 LT |
155 | } |
156 | ||
fb1c8f93 | 157 | static inline void __raw_write_lock(raw_rwlock_t *rw) |
1da177e4 | 158 | { |
fb2e2848 AK |
159 | asm volatile(LOCK_PREFIX " subl $" RW_LOCK_BIAS_STR ",(%0)\n\t" |
160 | "jz 1f\n" | |
161 | "call __write_lock_failed\n\t" | |
162 | "1:\n" | |
163 | ::"a" (rw) : "memory"); | |
1da177e4 LT |
164 | } |
165 | ||
fb1c8f93 | 166 | static inline int __raw_read_trylock(raw_rwlock_t *lock) |
1da177e4 LT |
167 | { |
168 | atomic_t *count = (atomic_t *)lock; | |
2fed0c50 | 169 | |
1da177e4 LT |
170 | atomic_dec(count); |
171 | if (atomic_read(count) >= 0) | |
172 | return 1; | |
173 | atomic_inc(count); | |
174 | return 0; | |
175 | } | |
176 | ||
fb1c8f93 | 177 | static inline int __raw_write_trylock(raw_rwlock_t *lock) |
1da177e4 LT |
178 | { |
179 | atomic_t *count = (atomic_t *)lock; | |
2fed0c50 | 180 | |
1da177e4 LT |
181 | if (atomic_sub_and_test(RW_LOCK_BIAS, count)) |
182 | return 1; | |
183 | atomic_add(RW_LOCK_BIAS, count); | |
184 | return 0; | |
185 | } | |
186 | ||
fb1c8f93 IM |
187 | static inline void __raw_read_unlock(raw_rwlock_t *rw) |
188 | { | |
b862f3b0 | 189 | asm volatile(LOCK_PREFIX "incl %0" :"+m" (rw->lock) : : "memory"); |
fb1c8f93 IM |
190 | } |
191 | ||
192 | static inline void __raw_write_unlock(raw_rwlock_t *rw) | |
193 | { | |
9a0b5817 | 194 | asm volatile(LOCK_PREFIX "addl $" RW_LOCK_BIAS_STR ", %0" |
b862f3b0 | 195 | : "+m" (rw->lock) : : "memory"); |
fb1c8f93 IM |
196 | } |
197 | ||
ef6edc97 MS |
198 | #define _raw_spin_relax(lock) cpu_relax() |
199 | #define _raw_read_relax(lock) cpu_relax() | |
200 | #define _raw_write_relax(lock) cpu_relax() | |
201 | ||
1da177e4 | 202 | #endif /* __ASM_SPINLOCK_H */ |