556f4544b31c2d61cedcc17d90239193442e1090
4 * (C) Copyright 2016 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
27 #define has_fast_acquire_release() 1
28 #define has_single_copy_load_64() 1
31 * The __rseq_table section can be used by debuggers to better handle
32 * single-stepping through the restartable critical sections.
34 #define RSEQ_FINISH_ASM(_target_final, _to_write_final, _start_value, \
35 _failure, _spec_store, _spec_input, \
36 _final_store, _final_input, _extra_clobber, \
37 _setup, _teardown, _scratch) \
40 __asm__ __volatile__ goto ( \
41 ".pushsection __rseq_table, \"aw\"\n\t" \
44 ".quad 1f, 2f, 4f, 0x0\n\t" \
49 "leaq 3b(%%rip), %%rax\n\t" \
50 "movq %%rax, %[rseq_cs]\n\t" \
52 "cmpl %[start_event_counter], %[current_event_counter]\n\t" \
59 "movq $0, %[rseq_cs]\n\t" \
61 ".pushsection __rseq_failure, \"a\"\n\t" \
63 "movq $0, %[rseq_cs]\n\t" \
65 "jmp %l[failure]\n\t" \
67 : /* gcc asm goto does not allow outputs */ \
68 : [start_event_counter]"r"((_start_value).event_counter), \
69 [current_event_counter]"m"((_start_value).rseqp->u.e.event_counter), \
70 [rseq_cs]"m"((_start_value).rseqp->rseq_cs) \
74 : "memory", "cc", "rax" \
81 #define RSEQ_FINISH_FINAL_STORE_ASM() \
82 "movq %[to_write_final], %[target_final]\n\t"
85 #define RSEQ_FINISH_FINAL_STORE_RELEASE_ASM() \
86 RSEQ_FINISH_FINAL_STORE_ASM()
88 #define RSEQ_FINISH_FINAL_STORE_INPUT(_target_final, _to_write_final) \
89 , [to_write_final]"r"(_to_write_final), \
90 [target_final]"m"(*(_target_final))
92 #define RSEQ_FINISH_SPECULATIVE_STORE_ASM() \
93 "movq %[to_write_spec], %[target_spec]\n\t" \
96 #define RSEQ_FINISH_SPECULATIVE_STORE_INPUT(_target_spec, _to_write_spec) \
97 , [to_write_spec]"r"(_to_write_spec), \
98 [target_spec]"m"(*(_target_spec))
100 /* TODO: implement a faster memcpy. */
101 #define RSEQ_FINISH_MEMCPY_STORE_ASM() \
102 "test %[len_memcpy], %[len_memcpy]\n\t" \
105 "movb (%[to_write_memcpy]), %%al\n\t" \
106 "movb %%al, (%[target_memcpy])\n\t" \
107 "inc %[to_write_memcpy]\n\t" \
108 "inc %[target_memcpy]\n\t" \
109 "dec %[len_memcpy]\n\t" \
114 #define RSEQ_FINISH_MEMCPY_STORE_INPUT(_target_memcpy, _to_write_memcpy, _len_memcpy) \
115 , [to_write_memcpy]"r"(_to_write_memcpy), \
116 [target_memcpy]"r"(_target_memcpy), \
117 [len_memcpy]"r"(_len_memcpy), \
118 [rseq_scratch0]"m"(rseq_scratch[0]), \
119 [rseq_scratch1]"m"(rseq_scratch[1]), \
120 [rseq_scratch2]"m"(rseq_scratch[2])
122 #define RSEQ_FINISH_MEMCPY_CLOBBER() \
125 #define RSEQ_FINISH_MEMCPY_SCRATCH() \
126 uint64_t rseq_scratch[3];
129 * We need to save and restore those input registers so they can be
130 * modified within the assembly.
132 #define RSEQ_FINISH_MEMCPY_SETUP() \
133 "movq %[to_write_memcpy], %[rseq_scratch0]\n\t" \
134 "movq %[target_memcpy], %[rseq_scratch1]\n\t" \
135 "movq %[len_memcpy], %[rseq_scratch2]\n\t"
137 #define RSEQ_FINISH_MEMCPY_TEARDOWN() \
138 "movq %[rseq_scratch2], %[len_memcpy]\n\t" \
139 "movq %[rseq_scratch1], %[target_memcpy]\n\t" \
140 "movq %[rseq_scratch0], %[to_write_memcpy]\n\t"
144 #define has_fast_acquire_release() 0
145 #define has_single_copy_load_64() 0
148 * Use eax as scratch register and take memory operands as input to
149 * lessen register pressure. Especially needed when compiling
150 * do_rseq_memcpy() in O0.
152 #define RSEQ_FINISH_ASM(_target_final, _to_write_final, _start_value, \
153 _failure, _spec_store, _spec_input, \
154 _final_store, _final_input, _extra_clobber, \
155 _setup, _teardown, _scratch) \
158 __asm__ __volatile__ goto ( \
159 ".pushsection __rseq_table, \"aw\"\n\t" \
162 ".long 1f, 0x0, 2f, 0x0, 4f, 0x0, 0x0, 0x0\n\t" \
167 "movl $3b, %[rseq_cs]\n\t" \
169 "movl %[start_event_counter], %%eax\n\t" \
170 "cmpl %%eax, %[current_event_counter]\n\t" \
177 "movl $0, %[rseq_cs]\n\t" \
179 ".pushsection __rseq_failure, \"a\"\n\t" \
181 "movl $0, %[rseq_cs]\n\t" \
183 "jmp %l[failure]\n\t" \
185 : /* gcc asm goto does not allow outputs */ \
186 : [start_event_counter]"m"((_start_value).event_counter), \
187 [current_event_counter]"m"((_start_value).rseqp->u.e.event_counter), \
188 [rseq_cs]"m"((_start_value).rseqp->rseq_cs) \
192 : "memory", "cc", "eax" \
194 RSEQ_INJECT_CLOBBER \
199 #define RSEQ_FINISH_FINAL_STORE_ASM() \
200 "movl %[to_write_final], %%eax\n\t" \
201 "movl %%eax, %[target_final]\n\t"
203 #define RSEQ_FINISH_FINAL_STORE_RELEASE_ASM() \
204 "lock; addl $0,0(%%esp)\n\t" \
205 RSEQ_FINISH_FINAL_STORE_ASM()
207 #define RSEQ_FINISH_FINAL_STORE_INPUT(_target_final, _to_write_final) \
208 , [to_write_final]"m"(_to_write_final), \
209 [target_final]"m"(*(_target_final))
211 #define RSEQ_FINISH_SPECULATIVE_STORE_ASM() \
212 "movl %[to_write_spec], %%eax\n\t" \
213 "movl %%eax, %[target_spec]\n\t" \
216 #define RSEQ_FINISH_SPECULATIVE_STORE_INPUT(_target_spec, _to_write_spec) \
217 , [to_write_spec]"m"(_to_write_spec), \
218 [target_spec]"m"(*(_target_spec))
220 /* TODO: implement a faster memcpy. */
221 #define RSEQ_FINISH_MEMCPY_STORE_ASM() \
222 "movl %[len_memcpy], %%eax\n\t" \
223 "test %%eax, %%eax\n\t" \
226 "movb (%[to_write_memcpy]), %%al\n\t" \
227 "movb %%al, (%[target_memcpy])\n\t" \
228 "inc %[to_write_memcpy]\n\t" \
229 "inc %[target_memcpy]\n\t" \
230 "decl %[rseq_scratch2]\n\t" \
235 #define RSEQ_FINISH_MEMCPY_STORE_INPUT(_target_memcpy, _to_write_memcpy, _len_memcpy) \
236 , [to_write_memcpy]"r"(_to_write_memcpy), \
237 [target_memcpy]"r"(_target_memcpy), \
238 [len_memcpy]"m"(_len_memcpy), \
239 [rseq_scratch0]"m"(rseq_scratch[0]), \
240 [rseq_scratch1]"m"(rseq_scratch[1]), \
241 [rseq_scratch2]"m"(rseq_scratch[2])
243 #define RSEQ_FINISH_MEMCPY_CLOBBER()
245 #define RSEQ_FINISH_MEMCPY_SCRATCH() \
246 uint32_t rseq_scratch[3];
249 * We need to save and restore those input registers so they can be
250 * modified within the assembly.
252 #define RSEQ_FINISH_MEMCPY_SETUP() \
253 "movl %[to_write_memcpy], %[rseq_scratch0]\n\t" \
254 "movl %[target_memcpy], %[rseq_scratch1]\n\t" \
255 "movl %[len_memcpy], %%eax\n\t" \
256 "movl %%eax, %[rseq_scratch2]\n\t"
258 #define RSEQ_FINISH_MEMCPY_TEARDOWN() \
259 "movl %[rseq_scratch1], %[target_memcpy]\n\t" \
260 "movl %[rseq_scratch0], %[to_write_memcpy]\n\t"
This page took 0.035236 seconds and 4 git commands to generate.