4 * (C) Copyright 2016 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
27 #define smp_mb() __asm__ __volatile__ ("mfence" : : : "memory")
28 #define smp_rmb() barrier()
29 #define smp_wmb() barrier()
31 #define smp_load_acquire(p) \
33 __typeof(*p) ____p1 = READ_ONCE(*p); \
38 #define smp_acquire__after_ctrl_dep() smp_rmb()
40 #define smp_store_release(p, v) \
46 #define has_fast_acquire_release() 1
47 #define has_single_copy_load_64() 1
50 * The __rseq_table section can be used by debuggers to better handle
51 * single-stepping through the restartable critical sections.
53 #define RSEQ_FINISH_ASM(_target_final, _to_write_final, _start_value, \
54 _failure, _spec_store, _spec_input, \
55 _final_store, _final_input, _extra_clobber, \
56 _setup, _teardown, _scratch) \
59 __asm__ __volatile__ goto ( \
60 ".pushsection __rseq_table, \"aw\"\n\t" \
63 ".quad 1f, 2f, 4f, 0x0\n\t" \
68 "leaq 3b(%%rip), %%rax\n\t" \
69 "movq %%rax, %[rseq_cs]\n\t" \
71 "cmpl %[start_event_counter], %[current_event_counter]\n\t" \
78 "movq $0, %[rseq_cs]\n\t" \
80 ".pushsection __rseq_failure, \"a\"\n\t" \
82 "movq $0, %[rseq_cs]\n\t" \
84 "jmp %l[failure]\n\t" \
86 : /* gcc asm goto does not allow outputs */ \
87 : [start_event_counter]"r"((_start_value).event_counter), \
88 [current_event_counter]"m"((_start_value).rseqp->u.e.event_counter), \
89 [rseq_cs]"m"((_start_value).rseqp->rseq_cs) \
93 : "memory", "cc", "rax" \
100 #define RSEQ_FINISH_FINAL_STORE_ASM() \
101 "movq %[to_write_final], %[target_final]\n\t"
104 #define RSEQ_FINISH_FINAL_STORE_RELEASE_ASM() \
105 RSEQ_FINISH_FINAL_STORE_ASM()
107 #define RSEQ_FINISH_FINAL_STORE_INPUT(_target_final, _to_write_final) \
108 , [to_write_final]"r"(_to_write_final), \
109 [target_final]"m"(*(_target_final))
111 #define RSEQ_FINISH_SPECULATIVE_STORE_ASM() \
112 "movq %[to_write_spec], %[target_spec]\n\t" \
115 #define RSEQ_FINISH_SPECULATIVE_STORE_INPUT(_target_spec, _to_write_spec) \
116 , [to_write_spec]"r"(_to_write_spec), \
117 [target_spec]"m"(*(_target_spec))
119 /* TODO: implement a faster memcpy. */
120 #define RSEQ_FINISH_MEMCPY_STORE_ASM() \
121 "test %[len_memcpy], %[len_memcpy]\n\t" \
124 "movb (%[to_write_memcpy]), %%al\n\t" \
125 "movb %%al, (%[target_memcpy])\n\t" \
126 "inc %[to_write_memcpy]\n\t" \
127 "inc %[target_memcpy]\n\t" \
128 "dec %[len_memcpy]\n\t" \
133 #define RSEQ_FINISH_MEMCPY_STORE_INPUT(_target_memcpy, _to_write_memcpy, _len_memcpy) \
134 , [to_write_memcpy]"r"(_to_write_memcpy), \
135 [target_memcpy]"r"(_target_memcpy), \
136 [len_memcpy]"r"(_len_memcpy), \
137 [rseq_scratch0]"m"(rseq_scratch[0]), \
138 [rseq_scratch1]"m"(rseq_scratch[1]), \
139 [rseq_scratch2]"m"(rseq_scratch[2])
141 #define RSEQ_FINISH_MEMCPY_CLOBBER() \
144 #define RSEQ_FINISH_MEMCPY_SCRATCH() \
145 uint64_t rseq_scratch[3];
148 * We need to save and restore those input registers so they can be
149 * modified within the assembly.
151 #define RSEQ_FINISH_MEMCPY_SETUP() \
152 "movq %[to_write_memcpy], %[rseq_scratch0]\n\t" \
153 "movq %[target_memcpy], %[rseq_scratch1]\n\t" \
154 "movq %[len_memcpy], %[rseq_scratch2]\n\t"
156 #define RSEQ_FINISH_MEMCPY_TEARDOWN() \
157 "movq %[rseq_scratch2], %[len_memcpy]\n\t" \
158 "movq %[rseq_scratch1], %[target_memcpy]\n\t" \
159 "movq %[rseq_scratch0], %[to_write_memcpy]\n\t"
164 * Support older 32-bit architectures that do not implement fence
168 __asm__ __volatile__ ("lock; addl $0,0(%%esp)" : : : "memory")
170 __asm__ __volatile__ ("lock; addl $0,0(%%esp)" : : : "memory")
172 __asm__ __volatile__ ("lock; addl $0,0(%%esp)" : : : "memory")
174 #define smp_load_acquire(p) \
176 __typeof(*p) ____p1 = READ_ONCE(*p); \
181 #define smp_acquire__after_ctrl_dep() smp_rmb()
183 #define smp_store_release(p, v) \
189 #define has_fast_acquire_release() 0
190 #define has_single_copy_load_64() 0
193 * Use eax as scratch register and take memory operands as input to
194 * lessen register pressure. Especially needed when compiling
195 * do_rseq_memcpy() in O0.
197 #define RSEQ_FINISH_ASM(_target_final, _to_write_final, _start_value, \
198 _failure, _spec_store, _spec_input, \
199 _final_store, _final_input, _extra_clobber, \
200 _setup, _teardown, _scratch) \
203 __asm__ __volatile__ goto ( \
204 ".pushsection __rseq_table, \"aw\"\n\t" \
207 ".long 1f, 0x0, 2f, 0x0, 4f, 0x0, 0x0, 0x0\n\t" \
212 "movl $3b, %[rseq_cs]\n\t" \
214 "movl %[start_event_counter], %%eax\n\t" \
215 "cmpl %%eax, %[current_event_counter]\n\t" \
222 "movl $0, %[rseq_cs]\n\t" \
224 ".pushsection __rseq_failure, \"a\"\n\t" \
226 "movl $0, %[rseq_cs]\n\t" \
228 "jmp %l[failure]\n\t" \
230 : /* gcc asm goto does not allow outputs */ \
231 : [start_event_counter]"m"((_start_value).event_counter), \
232 [current_event_counter]"m"((_start_value).rseqp->u.e.event_counter), \
233 [rseq_cs]"m"((_start_value).rseqp->rseq_cs) \
237 : "memory", "cc", "eax" \
239 RSEQ_INJECT_CLOBBER \
244 #define RSEQ_FINISH_FINAL_STORE_ASM() \
245 "movl %[to_write_final], %%eax\n\t" \
246 "movl %%eax, %[target_final]\n\t"
248 #define RSEQ_FINISH_FINAL_STORE_RELEASE_ASM() \
249 "lock; addl $0,0(%%esp)\n\t" \
250 RSEQ_FINISH_FINAL_STORE_ASM()
252 #define RSEQ_FINISH_FINAL_STORE_INPUT(_target_final, _to_write_final) \
253 , [to_write_final]"m"(_to_write_final), \
254 [target_final]"m"(*(_target_final))
256 #define RSEQ_FINISH_SPECULATIVE_STORE_ASM() \
257 "movl %[to_write_spec], %%eax\n\t" \
258 "movl %%eax, %[target_spec]\n\t" \
261 #define RSEQ_FINISH_SPECULATIVE_STORE_INPUT(_target_spec, _to_write_spec) \
262 , [to_write_spec]"m"(_to_write_spec), \
263 [target_spec]"m"(*(_target_spec))
265 /* TODO: implement a faster memcpy. */
266 #define RSEQ_FINISH_MEMCPY_STORE_ASM() \
267 "movl %[len_memcpy], %%eax\n\t" \
268 "test %%eax, %%eax\n\t" \
271 "movb (%[to_write_memcpy]), %%al\n\t" \
272 "movb %%al, (%[target_memcpy])\n\t" \
273 "inc %[to_write_memcpy]\n\t" \
274 "inc %[target_memcpy]\n\t" \
275 "decl %[rseq_scratch2]\n\t" \
280 #define RSEQ_FINISH_MEMCPY_STORE_INPUT(_target_memcpy, _to_write_memcpy, _len_memcpy) \
281 , [to_write_memcpy]"r"(_to_write_memcpy), \
282 [target_memcpy]"r"(_target_memcpy), \
283 [len_memcpy]"m"(_len_memcpy), \
284 [rseq_scratch0]"m"(rseq_scratch[0]), \
285 [rseq_scratch1]"m"(rseq_scratch[1]), \
286 [rseq_scratch2]"m"(rseq_scratch[2])
288 #define RSEQ_FINISH_MEMCPY_CLOBBER()
290 #define RSEQ_FINISH_MEMCPY_SCRATCH() \
291 uint32_t rseq_scratch[3];
294 * We need to save and restore those input registers so they can be
295 * modified within the assembly.
297 #define RSEQ_FINISH_MEMCPY_SETUP() \
298 "movl %[to_write_memcpy], %[rseq_scratch0]\n\t" \
299 "movl %[target_memcpy], %[rseq_scratch1]\n\t" \
300 "movl %[len_memcpy], %%eax\n\t" \
301 "movl %%eax, %[rseq_scratch2]\n\t"
303 #define RSEQ_FINISH_MEMCPY_TEARDOWN() \
304 "movl %[rseq_scratch1], %[target_memcpy]\n\t" \
305 "movl %[rseq_scratch0], %[to_write_memcpy]\n\t"
This page took 0.037621 seconds and 5 git commands to generate.