Commit | Line | Data |
---|---|---|
2af7f59e AV |
1 | /* |
2 | * Assembly implementation of the mutex fastpath, based on atomic | |
3 | * decrement/increment. | |
4 | * | |
5 | * started by Ingo Molnar: | |
6 | * | |
7 | * Copyright (C) 2004, 2005, 2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com> | |
8 | */ | |
9 | #ifndef _ASM_MUTEX_H | |
10 | #define _ASM_MUTEX_H | |
11 | ||
9a0b5817 GH |
12 | #include "asm/alternative.h" |
13 | ||
2af7f59e AV |
14 | /** |
15 | * __mutex_fastpath_lock - try to take the lock by moving the count | |
16 | * from 1 to a 0 value | |
17 | * @count: pointer of type atomic_t | |
18 | * @fn: function to call if the original value was not 1 | |
19 | * | |
20 | * Change the count from 1 to a value lower than 1, and call <fn> if it | |
21 | * wasn't 1 originally. This function MUST leave the value lower than 1 | |
22 | * even when the "1" assertion wasn't true. | |
23 | */ | |
24 | #define __mutex_fastpath_lock(count, fail_fn) \ | |
25 | do { \ | |
26 | unsigned int dummy; \ | |
27 | \ | |
28 | typecheck(atomic_t *, count); \ | |
29 | typecheck_fn(fastcall void (*)(atomic_t *), fail_fn); \ | |
30 | \ | |
31 | __asm__ __volatile__( \ | |
9a0b5817 | 32 | LOCK_PREFIX " decl (%%eax) \n" \ |
01215ad8 AK |
33 | " jns 1f \n" \ |
34 | " call "#fail_fn" \n" \ | |
73165b88 IM |
35 | "1: \n" \ |
36 | \ | |
2af7f59e AV |
37 | :"=a" (dummy) \ |
38 | : "a" (count) \ | |
39 | : "memory", "ecx", "edx"); \ | |
40 | } while (0) | |
41 | ||
42 | ||
43 | /** | |
44 | * __mutex_fastpath_lock_retval - try to take the lock by moving the count | |
45 | * from 1 to a 0 value | |
46 | * @count: pointer of type atomic_t | |
47 | * @fail_fn: function to call if the original value was not 1 | |
48 | * | |
49 | * Change the count from 1 to a value lower than 1, and call <fail_fn> if it | |
50 | * wasn't 1 originally. This function returns 0 if the fastpath succeeds, | |
51 | * or anything the slow path function returns | |
52 | */ | |
53 | static inline int | |
54 | __mutex_fastpath_lock_retval(atomic_t *count, | |
55 | int fastcall (*fail_fn)(atomic_t *)) | |
56 | { | |
57 | if (unlikely(atomic_dec_return(count) < 0)) | |
58 | return fail_fn(count); | |
59 | else | |
60 | return 0; | |
61 | } | |
62 | ||
63 | /** | |
64 | * __mutex_fastpath_unlock - try to promote the mutex from 0 to 1 | |
65 | * @count: pointer of type atomic_t | |
66 | * @fail_fn: function to call if the original value was not 0 | |
67 | * | |
68 | * try to promote the mutex from 0 to 1. if it wasn't 0, call <fail_fn>. | |
69 | * In the failure case, this function is allowed to either set the value | |
70 | * to 1, or to set it to a value lower than 1. | |
71 | * | |
72 | * If the implementation sets it to a value of lower than 1, the | |
73 | * __mutex_slowpath_needs_to_unlock() macro needs to return 1, it needs | |
74 | * to return 0 otherwise. | |
75 | */ | |
76 | #define __mutex_fastpath_unlock(count, fail_fn) \ | |
77 | do { \ | |
78 | unsigned int dummy; \ | |
79 | \ | |
80 | typecheck(atomic_t *, count); \ | |
81 | typecheck_fn(fastcall void (*)(atomic_t *), fail_fn); \ | |
82 | \ | |
83 | __asm__ __volatile__( \ | |
9a0b5817 | 84 | LOCK_PREFIX " incl (%%eax) \n" \ |
01215ad8 AK |
85 | " jg 1f \n" \ |
86 | " call "#fail_fn" \n" \ | |
73165b88 IM |
87 | "1: \n" \ |
88 | \ | |
2af7f59e AV |
89 | :"=a" (dummy) \ |
90 | : "a" (count) \ | |
91 | : "memory", "ecx", "edx"); \ | |
92 | } while (0) | |
93 | ||
94 | #define __mutex_slowpath_needs_to_unlock() 1 | |
95 | ||
96 | /** | |
97 | * __mutex_fastpath_trylock - try to acquire the mutex, without waiting | |
98 | * | |
99 | * @count: pointer of type atomic_t | |
100 | * @fail_fn: fallback function | |
101 | * | |
102 | * Change the count from 1 to a value lower than 1, and return 0 (failure) | |
103 | * if it wasn't 1 originally, or return 1 (success) otherwise. This function | |
104 | * MUST leave the value lower than 1 even when the "1" assertion wasn't true. | |
105 | * Additionally, if the value was < 0 originally, this function must not leave | |
106 | * it to 0 on failure. | |
107 | */ | |
108 | static inline int | |
109 | __mutex_fastpath_trylock(atomic_t *count, int (*fail_fn)(atomic_t *)) | |
110 | { | |
111 | /* | |
112 | * We have two variants here. The cmpxchg based one is the best one | |
113 | * because it never induce a false contention state. It is included | |
114 | * here because architectures using the inc/dec algorithms over the | |
115 | * xchg ones are much more likely to support cmpxchg natively. | |
116 | * | |
117 | * If not we fall back to the spinlock based variant - that is | |
118 | * just as efficient (and simpler) as a 'destructive' probing of | |
119 | * the mutex state would be. | |
120 | */ | |
121 | #ifdef __HAVE_ARCH_CMPXCHG | |
4cec8736 | 122 | if (likely(atomic_cmpxchg(count, 1, 0) == 1)) |
2af7f59e AV |
123 | return 1; |
124 | return 0; | |
125 | #else | |
126 | return fail_fn(count); | |
127 | #endif | |
128 | } | |
129 | ||
130 | #endif |