projects / librseq.git / blob
? search:


2bb493654f0a91dc5c0310f76aa95942dd0d0798
[librseq.git] / include / rseq / arch / x86 / bits.h
1 /* SPDX-License-Identifier: MIT */
2 /* SPDX-FileCopyrightText: 2016-2018 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */
3
4 /*
5 * rseq/arch/x86/bits.h
6 */
7
8 #include "rseq/arch/templates/bits.h"
9
10 /*
11 * Refer to rseq/pseudocode.h for pseudo-code of the rseq critical
12 * section helpers.
13 */
14 #include "rseq/pseudocode.h"
15
16 #ifdef __x86_64__
17
18 #if defined(RSEQ_TEMPLATE_MO_RELAXED) && \
19 (defined(RSEQ_TEMPLATE_INDEX_CPU_ID) || defined(RSEQ_TEMPLATE_INDEX_MM_CID))
20
21 static inline __attribute__((always_inline))
22 int RSEQ_TEMPLATE_IDENTIFIER(rseq_load_cbne_store__ptr)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
23 {
24 RSEQ_INJECT_C(9)
25
26 __asm__ __volatile__ goto (
27 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
28 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[ne])
29 #ifdef RSEQ_COMPARE_TWICE
30 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
31 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
32 #endif
33 /* Start rseq by storing table entry pointer into rseq_cs. */
34 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_ASM_CS_OFFSET(%[rseq_offset]))
35 RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_INDEX_CPU_ID_OFFSET(%[rseq_offset]), 4f)
36 RSEQ_INJECT_ASM(3)
37 "cmpq %[v], %[expect]\n\t"
38 "jne %l[ne]\n\t"
39 RSEQ_INJECT_ASM(4)
40 #ifdef RSEQ_COMPARE_TWICE
41 RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_INDEX_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
42 "cmpq %[v], %[expect]\n\t"
43 "jne %l[error2]\n\t"
44 #endif
45 /* final store */
46 "movq %[newv], %[v]\n\t"
47 "2:\n\t"
48 RSEQ_INJECT_ASM(5)
49 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
50 : /* gcc asm goto does not allow outputs */
51 : [cpu_id] "r" (cpu),
52 [rseq_offset] "r" (rseq_offset),
53 [v] "m" (*v),
54 [expect] "r" (expect),
55 [newv] "r" (newv)
56 : "memory", "cc", "rax"
57 RSEQ_INJECT_CLOBBER
58 : abort, ne
59 #ifdef RSEQ_COMPARE_TWICE
60 , error1, error2
61 #endif
62 );
63 rseq_after_asm_goto();
64 return 0;
65 abort:
66 rseq_after_asm_goto();
67 RSEQ_INJECT_FAILED
68 return -1;
69 ne:
70 rseq_after_asm_goto();
71 return 1;
72 #ifdef RSEQ_COMPARE_TWICE
73 error1:
74 rseq_after_asm_goto();
75 rseq_bug("cpu_id comparison failed");
76 error2:
77 rseq_after_asm_goto();
78 rseq_bug("expected value comparison failed");
79 #endif
80 }
81
82 static inline __attribute__((always_inline))
83 int RSEQ_TEMPLATE_IDENTIFIER(rseq_load_cbeq_store_add_load_store__ptr)(intptr_t *v, intptr_t expectnot,
84 long voffp, intptr_t *load, int cpu)
85 {
86 RSEQ_INJECT_C(9)
87
88 __asm__ __volatile__ goto (
89 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
90 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[eq])
91 #ifdef RSEQ_COMPARE_TWICE
92 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
93 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
94 #endif
95 /* Start rseq by storing table entry pointer into rseq_cs. */
96 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_ASM_CS_OFFSET(%[rseq_offset]))
97 RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_INDEX_CPU_ID_OFFSET(%[rseq_offset]), 4f)
98 RSEQ_INJECT_ASM(3)
99 "movq %[v], %%rbx\n\t"
100 "cmpq %%rbx, %[expectnot]\n\t"
101 "je %l[eq]\n\t"
102 RSEQ_INJECT_ASM(4)
103 #ifdef RSEQ_COMPARE_TWICE
104 RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_INDEX_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
105 "movq %[v], %%rbx\n\t"
106 "cmpq %%rbx, %[expectnot]\n\t"
107 "je %l[error2]\n\t"
108 #endif
109 "movq %%rbx, %[load]\n\t"
110 "addq %[voffp], %%rbx\n\t"
111 "movq (%%rbx), %%rbx\n\t"
112 /* final store */
113 "movq %%rbx, %[v]\n\t"
114 "2:\n\t"
115 RSEQ_INJECT_ASM(5)
116 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
117 : /* gcc asm goto does not allow outputs */
118 : [cpu_id] "r" (cpu),
119 [rseq_offset] "r" (rseq_offset),
120 /* final store input */
121 [v] "m" (*v),
122 [expectnot] "r" (expectnot),
123 [voffp] "er" (voffp),
124 [load] "m" (*load)
125 : "memory", "cc", "rax", "rbx"
126 RSEQ_INJECT_CLOBBER
127 : abort, eq
128 #ifdef RSEQ_COMPARE_TWICE
129 , error1, error2
130 #endif
131 );
132 rseq_after_asm_goto();
133 return 0;
134 abort:
135 rseq_after_asm_goto();
136 RSEQ_INJECT_FAILED
137 return -1;
138 eq:
139 rseq_after_asm_goto();
140 return 1;
141 #ifdef RSEQ_COMPARE_TWICE
142 error1:
143 rseq_after_asm_goto();
144 rseq_bug("cpu_id comparison failed");
145 error2:
146 rseq_after_asm_goto();
147 rseq_bug("expected value comparison failed");
148 #endif
149 }
150
151 static inline __attribute__((always_inline))
152 int RSEQ_TEMPLATE_IDENTIFIER(rseq_load_add_store__ptr)(intptr_t *v, intptr_t count, int cpu)
153 {
154 RSEQ_INJECT_C(9)
155
156 __asm__ __volatile__ goto (
157 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
158 #ifdef RSEQ_COMPARE_TWICE
159 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
160 #endif
161 /* Start rseq by storing table entry pointer into rseq_cs. */
162 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_ASM_CS_OFFSET(%[rseq_offset]))
163 RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_INDEX_CPU_ID_OFFSET(%[rseq_offset]), 4f)
164 RSEQ_INJECT_ASM(3)
165 #ifdef RSEQ_COMPARE_TWICE
166 RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_INDEX_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
167 #endif
168 /* final store */
169 "addq %[count], %[v]\n\t"
170 "2:\n\t"
171 RSEQ_INJECT_ASM(4)
172 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
173 : /* gcc asm goto does not allow outputs */
174 : [cpu_id] "r" (cpu),
175 [rseq_offset] "r" (rseq_offset),
176 /* final store input */
177 [v] "m" (*v),
178 [count] "er" (count)
179 : "memory", "cc", "rax"
180 RSEQ_INJECT_CLOBBER
181 : abort
182 #ifdef RSEQ_COMPARE_TWICE
183 , error1
184 #endif
185 );
186 rseq_after_asm_goto();
187 return 0;
188 abort:
189 rseq_after_asm_goto();
190 RSEQ_INJECT_FAILED
191 return -1;
192 #ifdef RSEQ_COMPARE_TWICE
193 error1:
194 rseq_after_asm_goto();
195 rseq_bug("cpu_id comparison failed");
196 #endif
197 }
198
199 #define rseq_arch_has_load_cbne_load_add_load_add_store
200
201 static inline __attribute__((always_inline))
202 int
203 RSEQ_TEMPLATE_IDENTIFIER(rseq_load_cbne_load_add_load_add_store__ptr)(intptr_t *ptr,
204 intptr_t expect, intptr_t *ptr2, ptrdiff_t offset,
205 intptr_t inc, int cpu)
206 {
207 RSEQ_INJECT_C(9)
208
209 __asm__ __volatile__ goto (
210 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
211 #ifdef RSEQ_COMPARE_TWICE
212 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
213 #endif
214 /* Start rseq by storing table entry pointer into rseq_cs. */
215 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_ASM_CS_OFFSET(%[rseq_offset]))
216 RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_INDEX_CPU_ID_OFFSET(%[rseq_offset]), 4f)
217 RSEQ_INJECT_ASM(3)
218 #ifdef RSEQ_COMPARE_TWICE
219 RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_INDEX_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
220 #endif
221 "movq %[ptr], %%rbx\n\t"
222 "cmpq %%rbx, %[expect]\n\t"
223 "jne %l[ne]\n\t"
224 RSEQ_INJECT_ASM(4)
225 "movq %[ptr2], %%rbx\n\t"
226 "addq %[offset], %%rbx\n\t"
227 "addq %[inc], (%%rbx)\n\t"
228 "2:\n\t"
229 RSEQ_INJECT_ASM(5)
230 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
231 : /* gcc asm goto does not allow outputs */
232 : [cpu_id] "r" (cpu),
233 [rseq_offset] "r" (rseq_offset),
234 /* final store input */
235 [ptr] "m" (*ptr),
236 [expect] "r" (expect),
237 [ptr2] "m" (*ptr2),
238 [offset] "er" (offset),
239 [inc] "er" (inc)
240 : "memory", "cc", "rax", "rbx"
241 RSEQ_INJECT_CLOBBER
242 : abort, ne
243 #ifdef RSEQ_COMPARE_TWICE
244 , error1
245 #endif
246 );
247 rseq_after_asm_goto();
248 return 0;
249 abort:
250 rseq_after_asm_goto();
251 RSEQ_INJECT_FAILED
252 return -1;
253 ne:
254 rseq_after_asm_goto();
255 return 1;
256 #ifdef RSEQ_COMPARE_TWICE
257 error1:
258 rseq_after_asm_goto();
259 rseq_bug("cpu_id comparison failed");
260 #endif
261 }
262
263 static inline __attribute__((always_inline))
264 int RSEQ_TEMPLATE_IDENTIFIER(rseq_load_cbne_load_cbne_store__ptr)(intptr_t *v, intptr_t expect,
265 intptr_t *v2, intptr_t expect2,
266 intptr_t newv, int cpu)
267 {
268 RSEQ_INJECT_C(9)
269
270 __asm__ __volatile__ goto (
271 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
272 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[ne])
273 #ifdef RSEQ_COMPARE_TWICE
274 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
275 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
276 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
277 #endif
278 /* Start rseq by storing table entry pointer into rseq_cs. */
279 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_ASM_CS_OFFSET(%[rseq_offset]))
280 RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_INDEX_CPU_ID_OFFSET(%[rseq_offset]), 4f)
281 RSEQ_INJECT_ASM(3)
282 "cmpq %[v], %[expect]\n\t"
283 "jne %l[ne]\n\t"
284 RSEQ_INJECT_ASM(4)
285 "cmpq %[v2], %[expect2]\n\t"
286 "jne %l[ne]\n\t"
287 RSEQ_INJECT_ASM(5)
288 #ifdef RSEQ_COMPARE_TWICE
289 RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_INDEX_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
290 "cmpq %[v], %[expect]\n\t"
291 "jne %l[error2]\n\t"
292 "cmpq %[v2], %[expect2]\n\t"
293 "jne %l[error3]\n\t"
294 #endif
295 /* final store */
296 "movq %[newv], %[v]\n\t"
297 "2:\n\t"
298 RSEQ_INJECT_ASM(6)
299 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
300 : /* gcc asm goto does not allow outputs */
301 : [cpu_id] "r" (cpu),
302 [rseq_offset] "r" (rseq_offset),
303 /* cmp2 input */
304 [v2] "m" (*v2),
305 [expect2] "r" (expect2),
306 /* final store input */
307 [v] "m" (*v),
308 [expect] "r" (expect),
309 [newv] "r" (newv)
310 : "memory", "cc", "rax"
311 RSEQ_INJECT_CLOBBER
312 : abort, ne
313 #ifdef RSEQ_COMPARE_TWICE
314 , error1, error2, error3
315 #endif
316 );
317 rseq_after_asm_goto();
318 return 0;
319 abort:
320 rseq_after_asm_goto();
321 RSEQ_INJECT_FAILED
322 return -1;
323 ne:
324 rseq_after_asm_goto();
325 return 1;
326 #ifdef RSEQ_COMPARE_TWICE
327 error1:
328 rseq_after_asm_goto();
329 rseq_bug("cpu_id comparison failed");
330 error2:
331 rseq_after_asm_goto();
332 rseq_bug("1st expected value comparison failed");
333 error3:
334 rseq_after_asm_goto();
335 rseq_bug("2nd expected value comparison failed");
336 #endif
337 }
338
339 #endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) &&
340 (defined(RSEQ_TEMPLATE_INDEX_CPU_ID) || defined(RSEQ_TEMPLATE_INDEX_MM_CID)) */
341
342 #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \
343 (defined(RSEQ_TEMPLATE_INDEX_CPU_ID) || defined(RSEQ_TEMPLATE_INDEX_MM_CID))
344
345 static inline __attribute__((always_inline))
346 int RSEQ_TEMPLATE_IDENTIFIER(rseq_load_cbne_store_store__ptr)(intptr_t *v, intptr_t expect,
347 intptr_t *v2, intptr_t newv2,
348 intptr_t newv, int cpu)
349 {
350 RSEQ_INJECT_C(9)
351
352 __asm__ __volatile__ goto (
353 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
354 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[ne])
355 #ifdef RSEQ_COMPARE_TWICE
356 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
357 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
358 #endif
359 /* Start rseq by storing table entry pointer into rseq_cs. */
360 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_ASM_CS_OFFSET(%[rseq_offset]))
361 RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_INDEX_CPU_ID_OFFSET(%[rseq_offset]), 4f)
362 RSEQ_INJECT_ASM(3)
363 "cmpq %[v], %[expect]\n\t"
364 "jne %l[ne]\n\t"
365 RSEQ_INJECT_ASM(4)
366 #ifdef RSEQ_COMPARE_TWICE
367 RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_INDEX_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
368 "cmpq %[v], %[expect]\n\t"
369 "jne %l[error2]\n\t"
370 #endif
371 /* try store */
372 "movq %[newv2], %[v2]\n\t"
373 RSEQ_INJECT_ASM(5)
374 /* final store */
375 "movq %[newv], %[v]\n\t"
376 "2:\n\t"
377 RSEQ_INJECT_ASM(6)
378 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
379 : /* gcc asm goto does not allow outputs */
380 : [cpu_id] "r" (cpu),
381 [rseq_offset] "r" (rseq_offset),
382 /* try store input */
383 [v2] "m" (*v2),
384 [newv2] "r" (newv2),
385 /* final store input */
386 [v] "m" (*v),
387 [expect] "r" (expect),
388 [newv] "r" (newv)
389 : "memory", "cc", "rax"
390 RSEQ_INJECT_CLOBBER
391 : abort, ne
392 #ifdef RSEQ_COMPARE_TWICE
393 , error1, error2
394 #endif
395 );
396 rseq_after_asm_goto();
397 return 0;
398 abort:
399 rseq_after_asm_goto();
400 RSEQ_INJECT_FAILED
401 return -1;
402 ne:
403 rseq_after_asm_goto();
404 return 1;
405 #ifdef RSEQ_COMPARE_TWICE
406 error1:
407 rseq_after_asm_goto();
408 rseq_bug("cpu_id comparison failed");
409 error2:
410 rseq_after_asm_goto();
411 rseq_bug("expected value comparison failed");
412 #endif
413 }
414
415 static inline __attribute__((always_inline))
416 int RSEQ_TEMPLATE_IDENTIFIER(rseq_load_cbne_memcpy_store__ptr)(intptr_t *v, intptr_t expect,
417 void *dst, void *src, size_t len,
418 intptr_t newv, int cpu)
419 {
420 uint64_t rseq_scratch[3];
421
422 RSEQ_INJECT_C(9)
423
424 __asm__ __volatile__ goto (
425 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
426 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[ne])
427 #ifdef RSEQ_COMPARE_TWICE
428 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
429 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
430 #endif
431 "movq %[src], %[rseq_scratch0]\n\t"
432 "movq %[dst], %[rseq_scratch1]\n\t"
433 "movq %[len], %[rseq_scratch2]\n\t"
434 /* Start rseq by storing table entry pointer into rseq_cs. */
435 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_ASM_CS_OFFSET(%[rseq_offset]))
436 RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_INDEX_CPU_ID_OFFSET(%[rseq_offset]), 4f)
437 RSEQ_INJECT_ASM(3)
438 "cmpq %[v], %[expect]\n\t"
439 "jne 5f\n\t"
440 RSEQ_INJECT_ASM(4)
441 #ifdef RSEQ_COMPARE_TWICE
442 RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_INDEX_CPU_ID_OFFSET(%[rseq_offset]), 6f)
443 "cmpq %[v], %[expect]\n\t"
444 "jne 7f\n\t"
445 #endif
446 /* try memcpy */
447 "test %[len], %[len]\n\t" \
448 "je 333f\n\t" \
449 "222:\n\t" \
450 "movb (%[src]), %%al\n\t" \
451 "movb %%al, (%[dst])\n\t" \
452 "inc %[src]\n\t" \
453 "inc %[dst]\n\t" \
454 "dec %[len]\n\t" \
455 "jnz 222b\n\t" \
456 "333:\n\t" \
457 RSEQ_INJECT_ASM(5)
458 /* final store */
459 "movq %[newv], %[v]\n\t"
460 "2:\n\t"
461 RSEQ_INJECT_ASM(6)
462 /* teardown */
463 "movq %[rseq_scratch2], %[len]\n\t"
464 "movq %[rseq_scratch1], %[dst]\n\t"
465 "movq %[rseq_scratch0], %[src]\n\t"
466 RSEQ_ASM_DEFINE_ABORT(4,
467 "movq %[rseq_scratch2], %[len]\n\t"
468 "movq %[rseq_scratch1], %[dst]\n\t"
469 "movq %[rseq_scratch0], %[src]\n\t",
470 abort)
471 RSEQ_ASM_DEFINE_TEARDOWN(5,
472 "movq %[rseq_scratch2], %[len]\n\t"
473 "movq %[rseq_scratch1], %[dst]\n\t"
474 "movq %[rseq_scratch0], %[src]\n\t",
475 ne)
476 #ifdef RSEQ_COMPARE_TWICE
477 RSEQ_ASM_DEFINE_TEARDOWN(6,
478 "movq %[rseq_scratch2], %[len]\n\t"
479 "movq %[rseq_scratch1], %[dst]\n\t"
480 "movq %[rseq_scratch0], %[src]\n\t",
481 error1)
482 RSEQ_ASM_DEFINE_TEARDOWN(7,
483 "movq %[rseq_scratch2], %[len]\n\t"
484 "movq %[rseq_scratch1], %[dst]\n\t"
485 "movq %[rseq_scratch0], %[src]\n\t",
486 error2)
487 #endif
488 : /* gcc asm goto does not allow outputs */
489 : [cpu_id] "r" (cpu),
490 [rseq_offset] "r" (rseq_offset),
491 /* final store input */
492 [v] "m" (*v),
493 [expect] "r" (expect),
494 [newv] "r" (newv),
495 /* try memcpy input */
496 [dst] "r" (dst),
497 [src] "r" (src),
498 [len] "r" (len),
499 [rseq_scratch0] "m" (rseq_scratch[0]),
500 [rseq_scratch1] "m" (rseq_scratch[1]),
501 [rseq_scratch2] "m" (rseq_scratch[2])
502 : "memory", "cc", "rax"
503 RSEQ_INJECT_CLOBBER
504 : abort, ne
505 #ifdef RSEQ_COMPARE_TWICE
506 , error1, error2
507 #endif
508 );
509 rseq_after_asm_goto();
510 return 0;
511 abort:
512 rseq_after_asm_goto();
513 RSEQ_INJECT_FAILED
514 return -1;
515 ne:
516 rseq_after_asm_goto();
517 return 1;
518 #ifdef RSEQ_COMPARE_TWICE
519 error1:
520 rseq_after_asm_goto();
521 rseq_bug("cpu_id comparison failed");
522 error2:
523 rseq_after_asm_goto();
524 rseq_bug("expected value comparison failed");
525 #endif
526 }
527
528 #endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) &&
529 (defined(RSEQ_TEMPLATE_INDEX_CPU_ID) || defined(RSEQ_TEMPLATE_INDEX_MM_CID)) */
530
531 #elif defined(__i386__)
532
533 /*
534 * On x86-32, use eax as scratch register and take memory operands as
535 * input to lessen register pressure. Especially needed when compiling
536 * in O0.
537 */
538
539 #if defined(RSEQ_TEMPLATE_MO_RELAXED) && \
540 (defined(RSEQ_TEMPLATE_INDEX_CPU_ID) || defined(RSEQ_TEMPLATE_INDEX_MM_CID))
541
542 static inline __attribute__((always_inline))
543 int RSEQ_TEMPLATE_IDENTIFIER(rseq_load_cbne_store__ptr)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
544 {
545 /*
546 * ref_ip is used to store a reference instruction pointer
547 * for ip-relative addressing.
548 */
549 struct rseq_local {
550 uint32_t ref_ip;
551 } rseq_local;
552
553 RSEQ_INJECT_C(9)
554
555 __asm__ __volatile__ goto (
556 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
557 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[ne])
558 #ifdef RSEQ_COMPARE_TWICE
559 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
560 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
561 #endif
562 /* Start rseq by storing table entry pointer into rseq_cs. */
563 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_ASM_CS_OFFSET(%[rseq_offset]), %[ref_ip], RSEQ_ASM_REF_LABEL)
564 RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_INDEX_CPU_ID_OFFSET(%[rseq_offset]), 4f)
565 RSEQ_INJECT_ASM(3)
566 "cmpl %[v], %[expect]\n\t"
567 "jne %l[ne]\n\t"
568 RSEQ_INJECT_ASM(4)
569 #ifdef RSEQ_COMPARE_TWICE
570 RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_INDEX_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
571 "cmpl %[v], %[expect]\n\t"
572 "jne %l[error2]\n\t"
573 #endif
574 /* final store */
575 "movl %[newv], %[v]\n\t"
576 "2:\n\t"
577 RSEQ_INJECT_ASM(5)
578 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
579 : /* gcc asm goto does not allow outputs */
580 : [cpu_id] "r" (cpu),
581 [rseq_offset] "r" (rseq_offset),
582 [v] "m" (*v),
583 [expect] "r" (expect),
584 [newv] "r" (newv),
585 [ref_ip] "m" (rseq_local.ref_ip)
586 : "memory", "cc", "eax"
587 RSEQ_INJECT_CLOBBER
588 : abort, ne
589 #ifdef RSEQ_COMPARE_TWICE
590 , error1, error2
591 #endif
592 );
593 rseq_after_asm_goto();
594 return 0;
595 abort:
596 rseq_after_asm_goto();
597 RSEQ_INJECT_FAILED
598 return -1;
599 ne:
600 rseq_after_asm_goto();
601 return 1;
602 #ifdef RSEQ_COMPARE_TWICE
603 error1:
604 rseq_after_asm_goto();
605 rseq_bug("cpu_id comparison failed");
606 error2:
607 rseq_after_asm_goto();
608 rseq_bug("expected value comparison failed");
609 #endif
610 }
611
612 static inline __attribute__((always_inline))
613 int RSEQ_TEMPLATE_IDENTIFIER(rseq_load_cbeq_store_add_load_store__ptr)(intptr_t *v, intptr_t expectnot,
614 long voffp, intptr_t *load, int cpu)
615 {
616 /*
617 * ref_ip is used to store a reference instruction pointer
618 * for ip-relative addressing.
619 */
620 struct rseq_local {
621 uint32_t ref_ip;
622 } rseq_local;
623
624 RSEQ_INJECT_C(9)
625
626 __asm__ __volatile__ goto (
627 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
628 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[eq])
629 #ifdef RSEQ_COMPARE_TWICE
630 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
631 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
632 #endif
633 /* Start rseq by storing table entry pointer into rseq_cs. */
634 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_ASM_CS_OFFSET(%[rseq_offset]), %[ref_ip], RSEQ_ASM_REF_LABEL)
635 RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_INDEX_CPU_ID_OFFSET(%[rseq_offset]), 4f)
636 RSEQ_INJECT_ASM(3)
637 "movl %[v], %%ebx\n\t"
638 "cmpl %%ebx, %[expectnot]\n\t"
639 "je %l[eq]\n\t"
640 RSEQ_INJECT_ASM(4)
641 #ifdef RSEQ_COMPARE_TWICE
642 RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_INDEX_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
643 "movl %[v], %%ebx\n\t"
644 "cmpl %%ebx, %[expectnot]\n\t"
645 "je %l[error2]\n\t"
646 #endif
647 "movl %%ebx, %[load]\n\t"
648 "addl %[voffp], %%ebx\n\t"
649 "movl (%%ebx), %%ebx\n\t"
650 /* final store */
651 "movl %%ebx, %[v]\n\t"
652 "2:\n\t"
653 RSEQ_INJECT_ASM(5)
654 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
655 : /* gcc asm goto does not allow outputs */
656 : [cpu_id] "r" (cpu),
657 [rseq_offset] "r" (rseq_offset),
658 /* final store input */
659 [v] "m" (*v),
660 [expectnot] "r" (expectnot),
661 [voffp] "ir" (voffp),
662 [load] "m" (*load),
663 [ref_ip] "m" (rseq_local.ref_ip)
664 : "memory", "cc", "eax", "ebx"
665 RSEQ_INJECT_CLOBBER
666 : abort, eq
667 #ifdef RSEQ_COMPARE_TWICE
668 , error1, error2
669 #endif
670 );
671 rseq_after_asm_goto();
672 return 0;
673 abort:
674 rseq_after_asm_goto();
675 RSEQ_INJECT_FAILED
676 return -1;
677 eq:
678 rseq_after_asm_goto();
679 return 1;
680 #ifdef RSEQ_COMPARE_TWICE
681 error1:
682 rseq_after_asm_goto();
683 rseq_bug("cpu_id comparison failed");
684 error2:
685 rseq_after_asm_goto();
686 rseq_bug("expected value comparison failed");
687 #endif
688 }
689
690 static inline __attribute__((always_inline))
691 int RSEQ_TEMPLATE_IDENTIFIER(rseq_load_add_store__ptr)(intptr_t *v, intptr_t count, int cpu)
692 {
693 /*
694 * ref_ip is used to store a reference instruction pointer
695 * for ip-relative addressing.
696 */
697 struct rseq_local {
698 uint32_t ref_ip;
699 } rseq_local;
700
701 RSEQ_INJECT_C(9)
702
703 __asm__ __volatile__ goto (
704 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
705 #ifdef RSEQ_COMPARE_TWICE
706 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
707 #endif
708 /* Start rseq by storing table entry pointer into rseq_cs. */
709 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_ASM_CS_OFFSET(%[rseq_offset]), %[ref_ip], RSEQ_ASM_REF_LABEL)
710 RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_INDEX_CPU_ID_OFFSET(%[rseq_offset]), 4f)
711 RSEQ_INJECT_ASM(3)
712 #ifdef RSEQ_COMPARE_TWICE
713 RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_INDEX_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
714 #endif
715 /* final store */
716 "addl %[count], %[v]\n\t"
717 "2:\n\t"
718 RSEQ_INJECT_ASM(4)
719 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
720 : /* gcc asm goto does not allow outputs */
721 : [cpu_id] "r" (cpu),
722 [rseq_offset] "r" (rseq_offset),
723 /* final store input */
724 [v] "m" (*v),
725 [count] "ir" (count),
726 [ref_ip] "m" (rseq_local.ref_ip)
727 : "memory", "cc", "eax"
728 RSEQ_INJECT_CLOBBER
729 : abort
730 #ifdef RSEQ_COMPARE_TWICE
731 , error1
732 #endif
733 );
734 rseq_after_asm_goto();
735 return 0;
736 abort:
737 rseq_after_asm_goto();
738 RSEQ_INJECT_FAILED
739 return -1;
740 #ifdef RSEQ_COMPARE_TWICE
741 error1:
742 rseq_after_asm_goto();
743 rseq_bug("cpu_id comparison failed");
744 #endif
745 }
746
747 static inline __attribute__((always_inline))
748 int RSEQ_TEMPLATE_IDENTIFIER(rseq_load_cbne_load_cbne_store__ptr)(intptr_t *v, intptr_t expect,
749 intptr_t *v2, intptr_t expect2,
750 intptr_t newv, int cpu)
751 {
752 /*
753 * ref_ip is used to store a reference instruction pointer
754 * for ip-relative addressing.
755 */
756 struct rseq_local {
757 uint32_t ref_ip;
758 } rseq_local;
759
760 RSEQ_INJECT_C(9)
761
762 __asm__ __volatile__ goto (
763 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
764 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[ne])
765 #ifdef RSEQ_COMPARE_TWICE
766 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
767 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
768 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
769 #endif
770 /* Start rseq by storing table entry pointer into rseq_cs. */
771 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_ASM_CS_OFFSET(%[rseq_offset]), %[ref_ip], RSEQ_ASM_REF_LABEL)
772 RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_INDEX_CPU_ID_OFFSET(%[rseq_offset]), 4f)
773 RSEQ_INJECT_ASM(3)
774 "cmpl %[v], %[expect]\n\t"
775 "jne %l[ne]\n\t"
776 RSEQ_INJECT_ASM(4)
777 "cmpl %[expect2], %[v2]\n\t"
778 "jne %l[ne]\n\t"
779 RSEQ_INJECT_ASM(5)
780 #ifdef RSEQ_COMPARE_TWICE
781 RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_INDEX_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
782 "cmpl %[v], %[expect]\n\t"
783 "jne %l[error2]\n\t"
784 "cmpl %[expect2], %[v2]\n\t"
785 "jne %l[error3]\n\t"
786 #endif
787 "movl %[newv], %%eax\n\t"
788 /* final store */
789 "movl %%eax, %[v]\n\t"
790 "2:\n\t"
791 RSEQ_INJECT_ASM(6)
792 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
793 : /* gcc asm goto does not allow outputs */
794 : [cpu_id] "r" (cpu),
795 [rseq_offset] "r" (rseq_offset),
796 /* cmp2 input */
797 [v2] "m" (*v2),
798 [expect2] "r" (expect2),
799 /* final store input */
800 [v] "m" (*v),
801 [expect] "r" (expect),
802 [newv] "m" (newv),
803 [ref_ip] "m" (rseq_local.ref_ip)
804 : "memory", "cc", "eax"
805 RSEQ_INJECT_CLOBBER
806 : abort, ne
807 #ifdef RSEQ_COMPARE_TWICE
808 , error1, error2, error3
809 #endif
810 );
811 rseq_after_asm_goto();
812 return 0;
813 abort:
814 rseq_after_asm_goto();
815 RSEQ_INJECT_FAILED
816 return -1;
817 ne:
818 rseq_after_asm_goto();
819 return 1;
820 #ifdef RSEQ_COMPARE_TWICE
821 error1:
822 rseq_after_asm_goto();
823 rseq_bug("cpu_id comparison failed");
824 error2:
825 rseq_after_asm_goto();
826 rseq_bug("1st expected value comparison failed");
827 error3:
828 rseq_after_asm_goto();
829 rseq_bug("2nd expected value comparison failed");
830 #endif
831 }
832
833 #endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) &&
834 (defined(RSEQ_TEMPLATE_INDEX_CPU_ID) || defined(RSEQ_TEMPLATE_INDEX_MM_CID)) */
835
836 #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \
837 (defined(RSEQ_TEMPLATE_INDEX_CPU_ID) || defined(RSEQ_TEMPLATE_INDEX_MM_CID))
838
839 static inline __attribute__((always_inline))
840 int RSEQ_TEMPLATE_IDENTIFIER(rseq_load_cbne_store_store__ptr)(intptr_t *v, intptr_t expect,
841 intptr_t *v2, intptr_t newv2,
842 intptr_t newv, int cpu)
843 {
844 /*
845 * ref_ip is used to store a reference instruction pointer
846 * for ip-relative addressing.
847 */
848 struct rseq_local {
849 uint32_t ref_ip;
850 } rseq_local;
851
852 RSEQ_INJECT_C(9)
853
854 __asm__ __volatile__ goto (
855 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
856 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[ne])
857 #ifdef RSEQ_COMPARE_TWICE
858 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
859 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
860 #endif
861 /* Start rseq by storing table entry pointer into rseq_cs. */
862 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_ASM_CS_OFFSET(%[rseq_offset]), %[ref_ip], RSEQ_ASM_REF_LABEL)
863 RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_INDEX_CPU_ID_OFFSET(%[rseq_offset]), 4f)
864 RSEQ_INJECT_ASM(3)
865 "movl %[expect], %%eax\n\t"
866 "cmpl %[v], %%eax\n\t"
867 "jne %l[ne]\n\t"
868 RSEQ_INJECT_ASM(4)
869 #ifdef RSEQ_COMPARE_TWICE
870 RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_INDEX_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
871 "movl %[expect], %%eax\n\t"
872 "cmpl %[v], %%eax\n\t"
873 "jne %l[error2]\n\t"
874 #endif
875 /* try store */
876 "movl %[newv2], %[v2]\n\t"
877 RSEQ_INJECT_ASM(5)
878 #ifdef RSEQ_TEMPLATE_MO_RELEASE
879 "lock; addl $0,-128(%%esp)\n\t"
880 #endif
881 /* final store */
882 "movl %[newv], %[v]\n\t"
883 "2:\n\t"
884 RSEQ_INJECT_ASM(6)
885 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
886 : /* gcc asm goto does not allow outputs */
887 : [cpu_id] "r" (cpu),
888 [rseq_offset] "r" (rseq_offset),
889 /* try store input */
890 [v2] "m" (*v2),
891 [newv2] "r" (newv2),
892 /* final store input */
893 [v] "m" (*v),
894 [expect] "m" (expect),
895 [newv] "r" (newv),
896 [ref_ip] "m" (rseq_local.ref_ip)
897 : "memory", "cc", "eax"
898 RSEQ_INJECT_CLOBBER
899 : abort, ne
900 #ifdef RSEQ_COMPARE_TWICE
901 , error1, error2
902 #endif
903 );
904 rseq_after_asm_goto();
905 return 0;
906 abort:
907 rseq_after_asm_goto();
908 RSEQ_INJECT_FAILED
909 return -1;
910 ne:
911 rseq_after_asm_goto();
912 return 1;
913 #ifdef RSEQ_COMPARE_TWICE
914 error1:
915 rseq_after_asm_goto();
916 rseq_bug("cpu_id comparison failed");
917 error2:
918 rseq_after_asm_goto();
919 rseq_bug("expected value comparison failed");
920 #endif
921
922 }
923
924 /* TODO: implement a faster memcpy. */
925 static inline __attribute__((always_inline))
926 int RSEQ_TEMPLATE_IDENTIFIER(rseq_load_cbne_memcpy_store__ptr)(intptr_t *v, intptr_t expect,
927 void *dst, void *src, size_t len,
928 intptr_t newv, int cpu)
929 {
930 /*
931 * Work-around register pressure limitations.
932 * Old gcc does not support output operands for asm goto, so
933 * input registers cannot simply be re-used as output registers.
934 * This is why clobbered registers are used.
935 * ref_ip is used to store a reference instruction pointer
936 * for ip-relative addressing.
937 */
938 struct rseq_local {
939 uint32_t expect, dst, src, len, newv, ref_ip;
940 } rseq_local = {
941 .expect = (uint32_t) expect,
942 .dst = (uint32_t) dst,
943 .src = (uint32_t) src,
944 .len = (uint32_t) len,
945 .newv = (uint32_t) newv,
946 .ref_ip = 0,
947 };
948
949 RSEQ_INJECT_C(9)
950
951 __asm__ __volatile__ goto (
952 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
953 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[ne])
954 #ifdef RSEQ_COMPARE_TWICE
955 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
956 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
957 #endif
958 /* Start rseq by storing table entry pointer into rseq_cs. */
959 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_ASM_CS_OFFSET(%[rseq_offset]), %[ref_ip], RSEQ_ASM_REF_LABEL)
960 RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_INDEX_CPU_ID_OFFSET(%[rseq_offset]), 4f)
961 RSEQ_INJECT_ASM(3)
962 /* load expect into ebx */
963 "movl %[expect], %%ebx\n\t"
964 "cmpl %%ebx, %[v]\n\t"
965 "jne %l[ne]\n\t"
966 RSEQ_INJECT_ASM(4)
967 #ifdef RSEQ_COMPARE_TWICE
968 RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_INDEX_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
969 "cmpl %%ebx, %[v]\n\t"
970 "jne %l[error2]\n\t"
971 #endif
972 /* try memcpy */
973 /* load dst into ebx */
974 "movl %[dst], %%ebx\n\t"
975 /* load src into ecx */
976 "movl %[src], %%ecx\n\t"
977 /* load len into edx */
978 "movl %[len], %%edx\n\t"
979 "test %%edx, %%edx\n\t"
980 "je 333f\n\t"
981 "222:\n\t"
982 "movb (%%ecx), %%al\n\t"
983 "movb %%al, (%%ebx)\n\t"
984 "inc %%ecx\n\t"
985 "inc %%ebx\n\t"
986 "dec %%edx\n\t"
987 "jnz 222b\n\t"
988 "333:\n\t"
989 RSEQ_INJECT_ASM(5)
990 #ifdef RSEQ_TEMPLATE_MO_RELEASE
991 "lock; addl $0,-128(%%esp)\n\t"
992 #endif
993 /* load newv into ebx */
994 "movl %[newv], %%ebx\n\t"
995 /* final store */
996 "movl %%ebx, %[v]\n\t"
997 "2:\n\t"
998 RSEQ_INJECT_ASM(6)
999 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
1000 : /* gcc asm goto does not allow outputs */
1001 : [cpu_id] "r" (cpu),
1002 [rseq_offset] "r" (rseq_offset),
1003 /* final store input */
1004 [v] "m" (*v),
1005 /* try memcpy input */
1006 [expect] "m" (rseq_local.expect), /* ebx */
1007 [dst] "m" (rseq_local.dst), /* ebx */
1008 [src] "m" (rseq_local.src), /* ecx */
1009 [len] "m" (rseq_local.len), /* edx */
1010 [newv] "m" (rseq_local.newv), /* ebx */
1011 [ref_ip] "m" (rseq_local.ref_ip)
1012 : "memory", "cc", "eax", "ebx", "ecx", "edx"
1013 RSEQ_INJECT_CLOBBER
1014 : abort, ne
1015 #ifdef RSEQ_COMPARE_TWICE
1016 , error1, error2
1017 #endif
1018 );
1019 rseq_after_asm_goto();
1020 return 0;
1021 abort:
1022 rseq_after_asm_goto();
1023 RSEQ_INJECT_FAILED
1024 return -1;
1025 ne:
1026 rseq_after_asm_goto();
1027 return 1;
1028 #ifdef RSEQ_COMPARE_TWICE
1029 error1:
1030 rseq_after_asm_goto();
1031 rseq_bug("cpu_id comparison failed");
1032 error2:
1033 rseq_after_asm_goto();
1034 rseq_bug("expected value comparison failed");
1035 #endif
1036 }
1037
1038 #endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) &&
1039 (defined(RSEQ_TEMPLATE_INDEX_CPU_ID) || defined(RSEQ_TEMPLATE_INDEX_MM_CID)) */
1040
1041 #endif
1042
1043 #include "rseq/arch/templates/bits-reset.h"
Librseq
This page took 0.051891 seconds and 3 git commands to generate.