4 * (C) Copyright 2016 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
37 #include <linux/rseq.h>
40 * Empty code injection macros, override when testing.
41 * It is important to consider that the ASM injection macros need to be
42 * fully reentrant (e.g. do not modify the stack).
44 #ifndef RSEQ_INJECT_ASM
45 #define RSEQ_INJECT_ASM(n)
49 #define RSEQ_INJECT_C(n)
52 #ifndef RSEQ_INJECT_INPUT
53 #define RSEQ_INJECT_INPUT
56 #ifndef RSEQ_INJECT_CLOBBER
57 #define RSEQ_INJECT_CLOBBER
60 #ifndef RSEQ_INJECT_FAILED
61 #define RSEQ_INJECT_FAILED
64 #ifndef RSEQ_FALLBACK_CNT
65 #define RSEQ_FALLBACK_CNT 3
68 uint32_t rseq_get_fallback_wait_cnt(void);
69 uint32_t rseq_get_fallback_cnt(void);
71 extern __thread
volatile struct rseq __rseq_abi
;
72 extern int rseq_has_sys_membarrier
;
74 #define likely(x) __builtin_expect(!!(x), 1)
75 #define unlikely(x) __builtin_expect(!!(x), 0)
76 #define barrier() __asm__ __volatile__("" : : : "memory")
78 #define ACCESS_ONCE(x) (*(__volatile__ __typeof__(x) *)&(x))
79 #define WRITE_ONCE(x, v) __extension__ ({ ACCESS_ONCE(x) = (v); })
80 #define READ_ONCE(x) ACCESS_ONCE(x)
82 #if defined(__x86_64__) || defined(__i386__)
84 #elif defined(__ARMEL__)
86 #elif defined(__PPC__)
89 #error unsupported target
92 enum rseq_lock_state
{
93 RSEQ_LOCK_STATE_RESTART
= 0,
94 RSEQ_LOCK_STATE_LOCK
= 1,
95 RSEQ_LOCK_STATE_FAIL
= 2,
100 int32_t state
; /* enum rseq_lock_state */
103 /* State returned by rseq_start, passed as argument to rseq_finish. */
105 volatile struct rseq
*rseqp
;
106 int32_t cpu_id
; /* cpu_id at start. */
107 uint32_t event_counter
; /* event_counter at start. */
108 int32_t lock_state
; /* Lock state at start. */
112 * Register rseq for the current thread. This needs to be called once
113 * by any thread which uses restartable sequences, before they start
114 * using restartable sequences. If initialization is not invoked, or if
115 * it fails, the restartable critical sections will fall-back on locking
118 int rseq_register_current_thread(void);
121 * Unregister rseq for current thread.
123 int rseq_unregister_current_thread(void);
126 * The fallback lock should be initialized before being used by any
127 * thread, and destroyed after all threads are done using it. This lock
128 * should be used by all rseq calls associated with shared data, either
129 * between threads, or between processes in a shared memory.
131 * There may be many rseq_lock per process, e.g. one per protected data
134 int rseq_init_lock(struct rseq_lock
*rlock
);
135 int rseq_destroy_lock(struct rseq_lock
*rlock
);
138 * Restartable sequence fallback prototypes. Fallback on locking when
139 * rseq is not initialized, not available on the system, or during
140 * single-stepping to ensure forward progress.
142 int rseq_fallback_begin(struct rseq_lock
*rlock
);
143 void rseq_fallback_end(struct rseq_lock
*rlock
, int cpu
);
144 void rseq_fallback_wait(struct rseq_lock
*rlock
);
145 void rseq_fallback_noinit(struct rseq_state
*rseq_state
);
148 * Restartable sequence fallback for reading the current CPU number.
150 int rseq_fallback_current_cpu(void);
152 static inline int32_t rseq_cpu_at_start(struct rseq_state start_value
)
154 return start_value
.cpu_id
;
157 static inline int32_t rseq_current_cpu_raw(void)
159 return ACCESS_ONCE(__rseq_abi
.u
.e
.cpu_id
);
162 static inline int32_t rseq_current_cpu(void)
166 cpu
= rseq_current_cpu_raw();
167 if (unlikely(cpu
< 0))
168 cpu
= rseq_fallback_current_cpu();
172 static inline __attribute__((always_inline
))
173 struct rseq_state
rseq_start(void)
175 struct rseq_state result
;
177 result
.rseqp
= &__rseq_abi
;
178 if (has_single_copy_load_64()) {
179 union rseq_cpu_event u
;
181 u
.v
= ACCESS_ONCE(result
.rseqp
->u
.v
);
182 result
.event_counter
= u
.e
.event_counter
;
183 result
.cpu_id
= u
.e
.cpu_id
;
185 result
.event_counter
=
186 ACCESS_ONCE(result
.rseqp
->u
.e
.event_counter
);
187 /* load event_counter before cpu_id. */
189 result
.cpu_id
= ACCESS_ONCE(result
.rseqp
->u
.e
.cpu_id
);
193 * Ensure the compiler does not re-order loads of protected
194 * values before we load the event counter.
200 static inline __attribute__((always_inline
))
201 struct rseq_state
rseq_start_rlock(struct rseq_lock
*rlock
)
203 struct rseq_state result
;
205 result
= rseq_start();
207 * Read event counter before lock state and cpu_id. This ensures
208 * that when the state changes from RESTART to LOCK, if we have
209 * some threads that have already seen the RESTART still in
210 * flight, they will necessarily be preempted/signalled before a
211 * thread can see the LOCK state for that same CPU. That
212 * preemption/signalling will cause them to restart, so they
213 * don't interfere with the lock.
216 if (!has_fast_acquire_release() && likely(rseq_has_sys_membarrier
)) {
217 result
.lock_state
= ACCESS_ONCE(rlock
->state
);
221 * Load lock state with acquire semantic. Matches
222 * smp_store_release() in rseq_fallback_end().
224 result
.lock_state
= smp_load_acquire(&rlock
->state
);
226 if (unlikely(result
.cpu_id
< 0))
227 rseq_fallback_noinit(&result
);
231 enum rseq_finish_type
{
238 * p_spec and to_write_spec are used for a speculative write attempted
239 * near the end of the restartable sequence. A rseq_finish2 may fail
240 * even after this write takes place.
242 * p_final and to_write_final are used for the final write. If this
243 * write takes place, the rseq_finish2 is guaranteed to succeed.
245 static inline __attribute__((always_inline
))
246 bool __rseq_finish(intptr_t *p_spec
, intptr_t to_write_spec
,
247 void *p_memcpy
, void *to_write_memcpy
, size_t len_memcpy
,
248 intptr_t *p_final
, intptr_t to_write_final
,
249 struct rseq_state start_value
,
250 enum rseq_finish_type type
, bool release
)
255 case RSEQ_FINISH_SINGLE
:
256 RSEQ_FINISH_ASM(p_final
, to_write_final
, start_value
, failure
,
257 /* no speculative write */, /* no speculative write */,
258 RSEQ_FINISH_FINAL_STORE_ASM(),
259 RSEQ_FINISH_FINAL_STORE_INPUT(p_final
, to_write_final
),
260 /* no extra clobber */, /* no arg */, /* no arg */,
264 case RSEQ_FINISH_TWO
:
266 RSEQ_FINISH_ASM(p_final
, to_write_final
, start_value
, failure
,
267 RSEQ_FINISH_SPECULATIVE_STORE_ASM(),
268 RSEQ_FINISH_SPECULATIVE_STORE_INPUT(p_spec
, to_write_spec
),
269 RSEQ_FINISH_FINAL_STORE_RELEASE_ASM(),
270 RSEQ_FINISH_FINAL_STORE_INPUT(p_final
, to_write_final
),
271 /* no extra clobber */, /* no arg */, /* no arg */,
275 RSEQ_FINISH_ASM(p_final
, to_write_final
, start_value
, failure
,
276 RSEQ_FINISH_SPECULATIVE_STORE_ASM(),
277 RSEQ_FINISH_SPECULATIVE_STORE_INPUT(p_spec
, to_write_spec
),
278 RSEQ_FINISH_FINAL_STORE_ASM(),
279 RSEQ_FINISH_FINAL_STORE_INPUT(p_final
, to_write_final
),
280 /* no extra clobber */, /* no arg */, /* no arg */,
285 case RSEQ_FINISH_MEMCPY
:
287 RSEQ_FINISH_ASM(p_final
, to_write_final
, start_value
, failure
,
288 RSEQ_FINISH_MEMCPY_STORE_ASM(),
289 RSEQ_FINISH_MEMCPY_STORE_INPUT(p_memcpy
, to_write_memcpy
, len_memcpy
),
290 RSEQ_FINISH_FINAL_STORE_RELEASE_ASM(),
291 RSEQ_FINISH_FINAL_STORE_INPUT(p_final
, to_write_final
),
292 RSEQ_FINISH_MEMCPY_CLOBBER(),
293 RSEQ_FINISH_MEMCPY_SETUP(),
294 RSEQ_FINISH_MEMCPY_TEARDOWN(),
295 RSEQ_FINISH_MEMCPY_SCRATCH()
298 RSEQ_FINISH_ASM(p_final
, to_write_final
, start_value
, failure
,
299 RSEQ_FINISH_MEMCPY_STORE_ASM(),
300 RSEQ_FINISH_MEMCPY_STORE_INPUT(p_memcpy
, to_write_memcpy
, len_memcpy
),
301 RSEQ_FINISH_FINAL_STORE_ASM(),
302 RSEQ_FINISH_FINAL_STORE_INPUT(p_final
, to_write_final
),
303 RSEQ_FINISH_MEMCPY_CLOBBER(),
304 RSEQ_FINISH_MEMCPY_SETUP(),
305 RSEQ_FINISH_MEMCPY_TEARDOWN(),
306 RSEQ_FINISH_MEMCPY_SCRATCH()
317 static inline __attribute__((always_inline
))
318 bool rseq_finish_rlock(struct rseq_lock
*rlock
,
319 intptr_t *p_spec
, intptr_t to_write_spec
,
320 void *p_memcpy
, void *to_write_memcpy
, size_t len_memcpy
,
321 intptr_t *p_final
, intptr_t to_write_final
,
322 struct rseq_state start_value
,
323 enum rseq_finish_type type
, bool release
)
325 if (unlikely(start_value
.lock_state
!= RSEQ_LOCK_STATE_RESTART
)) {
326 if (start_value
.lock_state
== RSEQ_LOCK_STATE_LOCK
)
327 rseq_fallback_wait(rlock
);
330 return __rseq_finish(p_spec
, to_write_spec
, p_memcpy
,
331 to_write_memcpy
, len_memcpy
,
332 p_final
, to_write_final
,
333 start_value
, type
, release
);
336 static inline __attribute__((always_inline
))
337 bool rseq_finish(intptr_t *p
, intptr_t to_write
,
338 struct rseq_state start_value
)
340 return __rseq_finish(NULL
, 0,
342 p
, to_write
, start_value
,
343 RSEQ_FINISH_SINGLE
, false);
346 static inline __attribute__((always_inline
))
347 bool rseq_finish2(intptr_t *p_spec
, intptr_t to_write_spec
,
348 intptr_t *p_final
, intptr_t to_write_final
,
349 struct rseq_state start_value
)
351 return __rseq_finish(p_spec
, to_write_spec
,
353 p_final
, to_write_final
, start_value
,
354 RSEQ_FINISH_TWO
, false);
357 static inline __attribute__((always_inline
))
358 bool rseq_finish2_release(intptr_t *p_spec
, intptr_t to_write_spec
,
359 intptr_t *p_final
, intptr_t to_write_final
,
360 struct rseq_state start_value
)
362 return __rseq_finish(p_spec
, to_write_spec
,
364 p_final
, to_write_final
, start_value
,
365 RSEQ_FINISH_TWO
, true);
368 static inline __attribute__((always_inline
))
369 bool rseq_finish_memcpy(void *p_memcpy
, void *to_write_memcpy
,
370 size_t len_memcpy
, intptr_t *p_final
, intptr_t to_write_final
,
371 struct rseq_state start_value
)
373 return __rseq_finish(NULL
, 0,
374 p_memcpy
, to_write_memcpy
, len_memcpy
,
375 p_final
, to_write_final
, start_value
,
376 RSEQ_FINISH_MEMCPY
, false);
379 static inline __attribute__((always_inline
))
380 bool rseq_finish_memcpy_release(void *p_memcpy
, void *to_write_memcpy
,
381 size_t len_memcpy
, intptr_t *p_final
, intptr_t to_write_final
,
382 struct rseq_state start_value
)
384 return __rseq_finish(NULL
, 0,
385 p_memcpy
, to_write_memcpy
, len_memcpy
,
386 p_final
, to_write_final
, start_value
,
387 RSEQ_FINISH_MEMCPY
, true);
390 #define __rseq_store_RSEQ_FINISH_SINGLE(_targetptr_spec, _newval_spec, \
391 _dest_memcpy, _src_memcpy, _len_memcpy, \
392 _targetptr_final, _newval_final) \
394 *(_targetptr_final) = (_newval_final); \
397 #define __rseq_store_RSEQ_FINISH_TWO(_targetptr_spec, _newval_spec, \
398 _dest_memcpy, _src_memcpy, _len_memcpy, \
399 _targetptr_final, _newval_final) \
401 *(_targetptr_spec) = (_newval_spec); \
402 *(_targetptr_final) = (_newval_final); \
405 #define __rseq_store_RSEQ_FINISH_MEMCPY(_targetptr_spec, \
406 _newval_spec, _dest_memcpy, _src_memcpy, _len_memcpy, \
407 _targetptr_final, _newval_final) \
409 memcpy(_dest_memcpy, _src_memcpy, _len_memcpy); \
410 *(_targetptr_final) = (_newval_final); \
414 * Helper macro doing two restartable critical section attempts, and if
415 * they fail, fallback on locking.
417 #define __do_rseq(_type, _lock, _rseq_state, _cpu, _result, \
418 _targetptr_spec, _newval_spec, \
419 _dest_memcpy, _src_memcpy, _len_memcpy, \
420 _targetptr_final, _newval_final, _code, _release) \
422 _rseq_state = rseq_start_rlock(_lock); \
423 _cpu = rseq_cpu_at_start(_rseq_state); \
426 if (unlikely(!_result)) \
428 if (likely(rseq_finish_rlock(_lock, \
429 _targetptr_spec, _newval_spec, \
430 _dest_memcpy, _src_memcpy, _len_memcpy, \
431 _targetptr_final, _newval_final, \
432 _rseq_state, _type, _release))) \
434 _rseq_state = rseq_start_rlock(_lock); \
435 _cpu = rseq_cpu_at_start(_rseq_state); \
438 if (unlikely(!_result)) \
440 if (likely(rseq_finish_rlock(_lock, \
441 _targetptr_spec, _newval_spec, \
442 _dest_memcpy, _src_memcpy, _len_memcpy, \
443 _targetptr_final, _newval_final, \
444 _rseq_state, _type, _release))) \
446 _cpu = rseq_fallback_begin(_lock); \
449 if (likely(_result)) \
450 __rseq_store_##_type(_targetptr_spec, \
451 _newval_spec, _dest_memcpy, \
452 _src_memcpy, _len_memcpy, \
453 _targetptr_final, _newval_final); \
454 rseq_fallback_end(_lock, _cpu); \
457 #define do_rseq(_lock, _rseq_state, _cpu, _result, _targetptr, _newval, \
459 __do_rseq(RSEQ_FINISH_SINGLE, _lock, _rseq_state, _cpu, _result,\
460 NULL, 0, NULL, NULL, 0, _targetptr, _newval, _code, false)
462 #define do_rseq2(_lock, _rseq_state, _cpu, _result, \
463 _targetptr_spec, _newval_spec, \
464 _targetptr_final, _newval_final, _code) \
465 __do_rseq(RSEQ_FINISH_TWO, _lock, _rseq_state, _cpu, _result, \
466 _targetptr_spec, _newval_spec, \
468 _targetptr_final, _newval_final, _code, false)
470 #define do_rseq2_release(_lock, _rseq_state, _cpu, _result, \
471 _targetptr_spec, _newval_spec, \
472 _targetptr_final, _newval_final, _code) \
473 __do_rseq(RSEQ_FINISH_TWO, _lock, _rseq_state, _cpu, _result, \
474 _targetptr_spec, _newval_spec, \
476 _targetptr_final, _newval_final, _code, true)
478 #define do_rseq_memcpy(_lock, _rseq_state, _cpu, _result, \
479 _dest_memcpy, _src_memcpy, _len_memcpy, \
480 _targetptr_final, _newval_final, _code) \
481 __do_rseq(RSEQ_FINISH_MEMCPY, _lock, _rseq_state, _cpu, _result,\
483 _dest_memcpy, _src_memcpy, _len_memcpy, \
484 _targetptr_final, _newval_final, _code, false)
486 #define do_rseq_memcpy_release(_lock, _rseq_state, _cpu, _result, \
487 _dest_memcpy, _src_memcpy, _len_memcpy, \
488 _targetptr_final, _newval_final, _code) \
489 __do_rseq(RSEQ_FINISH_MEMCPY, _lock, _rseq_state, _cpu, _result,\
491 _dest_memcpy, _src_memcpy, _len_memcpy, \
492 _targetptr_final, _newval_final, _code, true)
This page took 0.041163 seconds and 5 git commands to generate.