4 * (C) Copyright 2016 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
27 #define smp_mb() __asm__ __volatile__ ("mfence" : : : "memory")
28 #define smp_rmb() barrier()
29 #define smp_wmb() barrier()
31 #define smp_load_acquire(p) \
33 __typeof(*p) ____p1 = READ_ONCE(*p); \
38 #define smp_acquire__after_ctrl_dep() smp_rmb()
40 #define smp_store_release(p, v) \
46 #define has_fast_acquire_release() 1
47 #define has_single_copy_load_64() 1
50 * The __rseq_table section can be used by debuggers to better handle
51 * single-stepping through the restartable critical sections.
53 #define RSEQ_FINISH_ASM(_target_final, _to_write_final, _start_value, \
54 _failure, _spec_store, _spec_input, \
55 _final_store, _final_input, _extra_clobber, \
56 _setup, _teardown, _scratch) \
59 __asm__ __volatile__ goto ( \
60 ".pushsection __rseq_table, \"aw\"\n\t" \
63 ".quad 1f, 2f, 4f, 0x0\n\t" \
68 "leaq 3b(%%rip), %%rax\n\t" \
69 "movq %%rax, %[rseq_cs]\n\t" \
71 "cmpl %[start_event_counter], %[current_event_counter]\n\t" \
79 ".pushsection __rseq_failure, \"a\"\n\t" \
82 "jmp %l[failure]\n\t" \
84 : /* gcc asm goto does not allow outputs */ \
85 : [start_event_counter]"r"((_start_value).event_counter), \
86 [current_event_counter]"m"((_start_value).rseqp->u.e.event_counter), \
87 [rseq_cs]"m"((_start_value).rseqp->rseq_cs) \
91 : "memory", "cc", "rax" \
98 #define RSEQ_FINISH_FINAL_STORE_ASM() \
99 "movq %[to_write_final], %[target_final]\n\t"
102 #define RSEQ_FINISH_FINAL_STORE_RELEASE_ASM() \
103 RSEQ_FINISH_FINAL_STORE_ASM()
105 #define RSEQ_FINISH_FINAL_STORE_INPUT(_target_final, _to_write_final) \
106 , [to_write_final]"r"(_to_write_final), \
107 [target_final]"m"(*(_target_final))
109 #define RSEQ_FINISH_SPECULATIVE_STORE_ASM() \
110 "movq %[to_write_spec], %[target_spec]\n\t" \
113 #define RSEQ_FINISH_SPECULATIVE_STORE_INPUT(_target_spec, _to_write_spec) \
114 , [to_write_spec]"r"(_to_write_spec), \
115 [target_spec]"m"(*(_target_spec))
117 /* TODO: implement a faster memcpy. */
118 #define RSEQ_FINISH_MEMCPY_STORE_ASM() \
119 "test %[len_memcpy], %[len_memcpy]\n\t" \
122 "movb (%[to_write_memcpy]), %%al\n\t" \
123 "movb %%al, (%[target_memcpy])\n\t" \
124 "inc %[to_write_memcpy]\n\t" \
125 "inc %[target_memcpy]\n\t" \
126 "dec %[len_memcpy]\n\t" \
131 #define RSEQ_FINISH_MEMCPY_STORE_INPUT(_target_memcpy, _to_write_memcpy, _len_memcpy) \
132 , [to_write_memcpy]"r"(_to_write_memcpy), \
133 [target_memcpy]"r"(_target_memcpy), \
134 [len_memcpy]"r"(_len_memcpy), \
135 [rseq_scratch0]"m"(rseq_scratch[0]), \
136 [rseq_scratch1]"m"(rseq_scratch[1]), \
137 [rseq_scratch2]"m"(rseq_scratch[2])
139 #define RSEQ_FINISH_MEMCPY_CLOBBER() \
142 #define RSEQ_FINISH_MEMCPY_SCRATCH() \
143 uint64_t rseq_scratch[3];
146 * We need to save and restore those input registers so they can be
147 * modified within the assembly.
149 #define RSEQ_FINISH_MEMCPY_SETUP() \
150 "movq %[to_write_memcpy], %[rseq_scratch0]\n\t" \
151 "movq %[target_memcpy], %[rseq_scratch1]\n\t" \
152 "movq %[len_memcpy], %[rseq_scratch2]\n\t"
154 #define RSEQ_FINISH_MEMCPY_TEARDOWN() \
155 "movq %[rseq_scratch2], %[len_memcpy]\n\t" \
156 "movq %[rseq_scratch1], %[target_memcpy]\n\t" \
157 "movq %[rseq_scratch0], %[to_write_memcpy]\n\t"
162 * Support older 32-bit architectures that do not implement fence
166 __asm__ __volatile__ ("lock; addl $0,0(%%esp)" : : : "memory")
168 __asm__ __volatile__ ("lock; addl $0,0(%%esp)" : : : "memory")
170 __asm__ __volatile__ ("lock; addl $0,0(%%esp)" : : : "memory")
172 #define smp_load_acquire(p) \
174 __typeof(*p) ____p1 = READ_ONCE(*p); \
179 #define smp_acquire__after_ctrl_dep() smp_rmb()
181 #define smp_store_release(p, v) \
187 #define has_fast_acquire_release() 0
188 #define has_single_copy_load_64() 0
191 * Use eax as scratch register and take memory operands as input to
192 * lessen register pressure. Especially needed when compiling
193 * do_rseq_memcpy() in O0.
195 #define RSEQ_FINISH_ASM(_target_final, _to_write_final, _start_value, \
196 _failure, _spec_store, _spec_input, \
197 _final_store, _final_input, _extra_clobber, \
198 _setup, _teardown, _scratch) \
201 __asm__ __volatile__ goto ( \
202 ".pushsection __rseq_table, \"aw\"\n\t" \
205 ".long 1f, 0x0, 2f, 0x0, 4f, 0x0, 0x0, 0x0\n\t" \
210 "movl $3b, %[rseq_cs]\n\t" \
212 "movl %[start_event_counter], %%eax\n\t" \
213 "cmpl %%eax, %[current_event_counter]\n\t" \
221 ".pushsection __rseq_failure, \"a\"\n\t" \
224 "jmp %l[failure]\n\t" \
226 : /* gcc asm goto does not allow outputs */ \
227 : [start_event_counter]"m"((_start_value).event_counter), \
228 [current_event_counter]"m"((_start_value).rseqp->u.e.event_counter), \
229 [rseq_cs]"m"((_start_value).rseqp->rseq_cs) \
233 : "memory", "cc", "eax" \
235 RSEQ_INJECT_CLOBBER \
240 #define RSEQ_FINISH_FINAL_STORE_ASM() \
241 "movl %[to_write_final], %%eax\n\t" \
242 "movl %%eax, %[target_final]\n\t"
244 #define RSEQ_FINISH_FINAL_STORE_RELEASE_ASM() \
245 "lock; addl $0,0(%%esp)\n\t" \
246 RSEQ_FINISH_FINAL_STORE_ASM()
248 #define RSEQ_FINISH_FINAL_STORE_INPUT(_target_final, _to_write_final) \
249 , [to_write_final]"m"(_to_write_final), \
250 [target_final]"m"(*(_target_final))
252 #define RSEQ_FINISH_SPECULATIVE_STORE_ASM() \
253 "movl %[to_write_spec], %%eax\n\t" \
254 "movl %%eax, %[target_spec]\n\t" \
257 #define RSEQ_FINISH_SPECULATIVE_STORE_INPUT(_target_spec, _to_write_spec) \
258 , [to_write_spec]"m"(_to_write_spec), \
259 [target_spec]"m"(*(_target_spec))
261 /* TODO: implement a faster memcpy. */
262 #define RSEQ_FINISH_MEMCPY_STORE_ASM() \
263 "movl %[len_memcpy], %%eax\n\t" \
264 "test %%eax, %%eax\n\t" \
267 "movb (%[to_write_memcpy]), %%al\n\t" \
268 "movb %%al, (%[target_memcpy])\n\t" \
269 "inc %[to_write_memcpy]\n\t" \
270 "inc %[target_memcpy]\n\t" \
271 "decl %[rseq_scratch2]\n\t" \
276 #define RSEQ_FINISH_MEMCPY_STORE_INPUT(_target_memcpy, _to_write_memcpy, _len_memcpy) \
277 , [to_write_memcpy]"r"(_to_write_memcpy), \
278 [target_memcpy]"r"(_target_memcpy), \
279 [len_memcpy]"m"(_len_memcpy), \
280 [rseq_scratch0]"m"(rseq_scratch[0]), \
281 [rseq_scratch1]"m"(rseq_scratch[1]), \
282 [rseq_scratch2]"m"(rseq_scratch[2])
284 #define RSEQ_FINISH_MEMCPY_CLOBBER()
286 #define RSEQ_FINISH_MEMCPY_SCRATCH() \
287 uint32_t rseq_scratch[3];
290 * We need to save and restore those input registers so they can be
291 * modified within the assembly.
293 #define RSEQ_FINISH_MEMCPY_SETUP() \
294 "movl %[to_write_memcpy], %[rseq_scratch0]\n\t" \
295 "movl %[target_memcpy], %[rseq_scratch1]\n\t" \
296 "movl %[len_memcpy], %%eax\n\t" \
297 "movl %%eax, %[rseq_scratch2]\n\t"
299 #define RSEQ_FINISH_MEMCPY_TEARDOWN() \
300 "movl %[rseq_scratch1], %[target_memcpy]\n\t" \
301 "movl %[rseq_scratch0], %[to_write_memcpy]\n\t"
This page took 0.043828 seconds and 6 git commands to generate.