Commit | Line | Data |
---|---|---|
90702366 | 1 | /* SPDX-License-Identifier: MIT */ |
f2d7b530 MJ |
2 | /* SPDX-FileCopyrightText: 2016-2022 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */ |
3 | ||
784b0012 | 4 | /* |
44ec21eb | 5 | * rseq/rseq.h |
784b0012 MD |
6 | */ |
7 | ||
44ec21eb MJ |
8 | #ifndef _RSEQ_RSEQ_H |
9 | #define _RSEQ_RSEQ_H | |
784b0012 MD |
10 | |
11 | #include <stdint.h> | |
12 | #include <stdbool.h> | |
13 | #include <pthread.h> | |
14 | #include <signal.h> | |
15 | #include <sched.h> | |
16 | #include <errno.h> | |
17 | #include <stdio.h> | |
18 | #include <stdlib.h> | |
170f840b | 19 | #include <stddef.h> |
d2fa6d30 | 20 | #include <assert.h> |
784b0012 | 21 | |
44ec21eb MJ |
22 | #include <rseq/abi.h> |
23 | #include <rseq/compiler.h> | |
24 | #include <rseq/inject.h> | |
25 | #include <rseq/thread-pointer.h> | |
26 | #include <rseq/utils.h> | |
784b0012 | 27 | |
44ec21eb MJ |
28 | enum rseq_mo { |
29 | RSEQ_MO_RELAXED = 0, | |
30 | RSEQ_MO_CONSUME = 1, /* Unused */ | |
31 | RSEQ_MO_ACQUIRE = 2, /* Unused */ | |
32 | RSEQ_MO_RELEASE = 3, | |
33 | RSEQ_MO_ACQ_REL = 4, /* Unused */ | |
34 | RSEQ_MO_SEQ_CST = 5, /* Unused */ | |
35 | }; | |
784b0012 | 36 | |
44ec21eb MJ |
37 | enum rseq_percpu_mode { |
38 | RSEQ_PERCPU_CPU_ID = 0, | |
39 | RSEQ_PERCPU_MM_CID = 1, | |
40 | }; | |
784b0012 | 41 | |
44ec21eb MJ |
42 | enum rseq_available_query { |
43 | RSEQ_AVAILABLE_QUERY_KERNEL = 0, | |
44 | RSEQ_AVAILABLE_QUERY_LIBC = 1, | |
45 | }; | |
784b0012 | 46 | |
96b6ce39 MD |
47 | /* |
48 | * User code can define RSEQ_GET_ABI_OVERRIDE to override the | |
49 | * rseq_get_abi() implementation, for instance to use glibc's symbols | |
50 | * directly. | |
51 | */ | |
52 | #ifndef RSEQ_GET_ABI_OVERRIDE | |
53 | ||
96b6ce39 | 54 | # ifdef __cplusplus |
60a27517 | 55 | extern "C" { |
96b6ce39 | 56 | # endif |
60a27517 | 57 | |
baa98a34 | 58 | /* Offset from the thread pointer to the rseq area. */ |
170f840b | 59 | extern ptrdiff_t rseq_offset; |
baa98a34 MD |
60 | |
61 | /* | |
62 | * Size of the registered rseq area. 0 if the registration was | |
63 | * unsuccessful. | |
64 | */ | |
9698c399 | 65 | extern unsigned int rseq_size; |
baa98a34 MD |
66 | |
67 | /* Flags used during rseq registration. */ | |
9698c399 | 68 | extern unsigned int rseq_flags; |
784b0012 | 69 | |
baa98a34 MD |
70 | /* |
71 | * rseq feature size supported by the kernel. 0 if the registration was | |
72 | * unsuccessful. | |
73 | */ | |
74 | extern unsigned int rseq_feature_size; | |
75 | ||
44ec21eb MJ |
76 | /* |
77 | * Returns a pointer to the rseq area. | |
78 | */ | |
79 | static inline __attribute__((always_inline)) | |
80 | struct rseq_abi *rseq_get_abi(void) | |
96b6ce39 | 81 | { |
2d533093 | 82 | return (struct rseq_abi *) ((uintptr_t) rseq_thread_pointer() + rseq_offset); |
60a27517 | 83 | } |
96b6ce39 MD |
84 | |
85 | # ifdef __cplusplus | |
86 | } | |
87 | # endif | |
88 | ||
89 | #endif /* RSEQ_GET_ABI_OVERRIDE */ | |
60a27517 | 90 | |
809f5ee3 | 91 | |
44ec21eb MJ |
92 | /* |
93 | * Architecture specific. | |
94 | */ | |
95 | #include <rseq/arch.h> | |
809f5ee3 | 96 | |
784b0012 | 97 | |
15260018 | 98 | #ifdef __cplusplus |
60a27517 MG |
99 | extern "C" { |
100 | #endif | |
101 | ||
784b0012 MD |
102 | /* |
103 | * Register rseq for the current thread. This needs to be called once | |
104 | * by any thread which uses restartable sequences, before they start | |
105 | * using restartable sequences, to ensure restartable sequences | |
106 | * succeed. A restartable sequence executed from a non-registered | |
107 | * thread will always fail. | |
108 | */ | |
109 | int rseq_register_current_thread(void); | |
110 | ||
111 | /* | |
112 | * Unregister rseq for current thread. | |
113 | */ | |
114 | int rseq_unregister_current_thread(void); | |
115 | ||
116 | /* | |
117 | * Restartable sequence fallback for reading the current CPU number. | |
118 | */ | |
119 | int32_t rseq_fallback_current_cpu(void); | |
120 | ||
baa98a34 MD |
121 | /* |
122 | * Restartable sequence fallback for reading the current node number. | |
123 | */ | |
124 | int32_t rseq_fallback_current_node(void); | |
125 | ||
8b34114a MD |
126 | /* |
127 | * Returns true if rseq is supported. | |
128 | */ | |
129 | bool rseq_available(unsigned int query); | |
52e82b87 | 130 | |
784b0012 MD |
131 | /* |
132 | * Values returned can be either the current CPU number, -1 (rseq is | |
133 | * uninitialized), or -2 (rseq initialization has failed). | |
134 | */ | |
44ec21eb MJ |
135 | static inline __attribute__((always_inline)) |
136 | int32_t rseq_current_cpu_raw(void) | |
784b0012 | 137 | { |
9698c399 | 138 | return RSEQ_READ_ONCE(rseq_get_abi()->cpu_id); |
784b0012 MD |
139 | } |
140 | ||
141 | /* | |
142 | * Returns a possible CPU number, which is typically the current CPU. | |
143 | * The returned CPU number can be used to prepare for an rseq critical | |
144 | * section, which will confirm whether the cpu number is indeed the | |
145 | * current one, and whether rseq is initialized. | |
146 | * | |
147 | * The CPU number returned by rseq_cpu_start should always be validated | |
148 | * by passing it to a rseq asm sequence, or by comparing it to the | |
149 | * return value of rseq_current_cpu_raw() if the rseq asm sequence | |
150 | * does not need to be invoked. | |
151 | */ | |
44ec21eb MJ |
152 | static inline __attribute__((always_inline)) |
153 | uint32_t rseq_cpu_start(void) | |
784b0012 | 154 | { |
9698c399 | 155 | return RSEQ_READ_ONCE(rseq_get_abi()->cpu_id_start); |
784b0012 MD |
156 | } |
157 | ||
44ec21eb MJ |
158 | static inline __attribute__((always_inline)) |
159 | uint32_t rseq_current_cpu(void) | |
784b0012 MD |
160 | { |
161 | int32_t cpu; | |
162 | ||
163 | cpu = rseq_current_cpu_raw(); | |
164 | if (rseq_unlikely(cpu < 0)) | |
165 | cpu = rseq_fallback_current_cpu(); | |
166 | return cpu; | |
167 | } | |
168 | ||
44ec21eb MJ |
169 | static inline __attribute__((always_inline)) |
170 | bool rseq_node_id_available(void) | |
d2fa6d30 MD |
171 | { |
172 | return (int) rseq_feature_size >= (int) rseq_offsetofend(struct rseq_abi, node_id); | |
173 | } | |
174 | ||
175 | /* | |
176 | * Current NUMA node number. | |
177 | */ | |
44ec21eb MJ |
178 | static inline __attribute__((always_inline)) |
179 | uint32_t rseq_current_node_id(void) | |
d2fa6d30 MD |
180 | { |
181 | assert(rseq_node_id_available()); | |
182 | return RSEQ_READ_ONCE(rseq_get_abi()->node_id); | |
183 | } | |
184 | ||
44ec21eb MJ |
185 | static inline __attribute__((always_inline)) |
186 | bool rseq_mm_cid_available(void) | |
d2fa6d30 MD |
187 | { |
188 | return (int) rseq_feature_size >= (int) rseq_offsetofend(struct rseq_abi, mm_cid); | |
189 | } | |
190 | ||
44ec21eb MJ |
191 | static inline __attribute__((always_inline)) |
192 | uint32_t rseq_current_mm_cid(void) | |
d2fa6d30 MD |
193 | { |
194 | return RSEQ_READ_ONCE(rseq_get_abi()->mm_cid); | |
195 | } | |
196 | ||
44ec21eb MJ |
197 | static inline __attribute__((always_inline)) |
198 | void rseq_clear_rseq_cs(void) | |
784b0012 | 199 | { |
5b40603c | 200 | RSEQ_WRITE_ONCE(rseq_get_abi()->rseq_cs.arch.ptr, 0); |
784b0012 MD |
201 | } |
202 | ||
203 | /* | |
204 | * rseq_prepare_unload() should be invoked by each thread executing a rseq | |
205 | * critical section at least once between their last critical section and | |
d2fa6d30 MD |
206 | * library unload of the library defining the rseq critical section (struct |
207 | * rseq_cs) or the code referred to by the struct rseq_cs start_ip and | |
208 | * post_commit_offset fields. This also applies to use of rseq in code | |
209 | * generated by JIT: rseq_prepare_unload() should be invoked at least once by | |
210 | * each thread executing a rseq critical section before reclaim of the memory | |
211 | * holding the struct rseq_cs or reclaim of the code pointed to by struct | |
212 | * rseq_cs start_ip and post_commit_offset fields. | |
784b0012 | 213 | */ |
44ec21eb MJ |
214 | static inline __attribute__((always_inline)) |
215 | void rseq_prepare_unload(void) | |
784b0012 MD |
216 | { |
217 | rseq_clear_rseq_cs(); | |
218 | } | |
219 | ||
201c1a2a | 220 | /* |
44ec21eb | 221 | * Refer to rseq/pseudocode.h for documentation and pseudo-code of the |
201c1a2a MD |
222 | * rseq critical section helpers. |
223 | */ | |
44ec21eb | 224 | #include "rseq/pseudocode.h" |
201c1a2a | 225 | |
809f5ee3 | 226 | static inline __attribute__((always_inline)) |
41149e28 | 227 | int rseq_load_cbne_store__ptr(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode, |
809f5ee3 MD |
228 | intptr_t *v, intptr_t expect, |
229 | intptr_t newv, int cpu) | |
230 | { | |
231 | if (rseq_mo != RSEQ_MO_RELAXED) | |
232 | return -1; | |
233 | switch (percpu_mode) { | |
234 | case RSEQ_PERCPU_CPU_ID: | |
41149e28 | 235 | return rseq_load_cbne_store__ptr_relaxed_cpu_id(v, expect, newv, cpu); |
809f5ee3 | 236 | case RSEQ_PERCPU_MM_CID: |
41149e28 | 237 | return rseq_load_cbne_store__ptr_relaxed_mm_cid(v, expect, newv, cpu); |
95dbaeba GK |
238 | default: |
239 | return -1; | |
809f5ee3 | 240 | } |
809f5ee3 MD |
241 | } |
242 | ||
809f5ee3 | 243 | static inline __attribute__((always_inline)) |
41149e28 | 244 | int rseq_load_cbeq_store_add_load_store__ptr(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode, |
809f5ee3 MD |
245 | intptr_t *v, intptr_t expectnot, long voffp, intptr_t *load, |
246 | int cpu) | |
247 | { | |
248 | if (rseq_mo != RSEQ_MO_RELAXED) | |
249 | return -1; | |
250 | switch (percpu_mode) { | |
251 | case RSEQ_PERCPU_CPU_ID: | |
41149e28 | 252 | return rseq_load_cbeq_store_add_load_store__ptr_relaxed_cpu_id(v, expectnot, voffp, load, cpu); |
809f5ee3 | 253 | case RSEQ_PERCPU_MM_CID: |
41149e28 | 254 | return rseq_load_cbeq_store_add_load_store__ptr_relaxed_mm_cid(v, expectnot, voffp, load, cpu); |
95dbaeba GK |
255 | default: |
256 | return -1; | |
809f5ee3 | 257 | } |
809f5ee3 MD |
258 | } |
259 | ||
260 | static inline __attribute__((always_inline)) | |
41149e28 | 261 | int rseq_load_add_store__ptr(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode, |
809f5ee3 MD |
262 | intptr_t *v, intptr_t count, int cpu) |
263 | { | |
264 | if (rseq_mo != RSEQ_MO_RELAXED) | |
265 | return -1; | |
266 | switch (percpu_mode) { | |
267 | case RSEQ_PERCPU_CPU_ID: | |
41149e28 | 268 | return rseq_load_add_store__ptr_relaxed_cpu_id(v, count, cpu); |
809f5ee3 | 269 | case RSEQ_PERCPU_MM_CID: |
41149e28 | 270 | return rseq_load_add_store__ptr_relaxed_mm_cid(v, count, cpu); |
95dbaeba GK |
271 | default: |
272 | return -1; | |
809f5ee3 | 273 | } |
809f5ee3 MD |
274 | } |
275 | ||
aa021469 | 276 | #ifdef rseq_arch_has_load_add_load_load_add_store |
809f5ee3 | 277 | static inline __attribute__((always_inline)) |
aa021469 MD |
278 | int rseq_load_add_load_load_add_store__ptr(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode, |
279 | intptr_t *ptr, long off, intptr_t inc, int cpu) | |
809f5ee3 MD |
280 | { |
281 | if (rseq_mo != RSEQ_MO_RELAXED) | |
282 | return -1; | |
283 | switch (percpu_mode) { | |
284 | case RSEQ_PERCPU_CPU_ID: | |
aa021469 | 285 | return rseq_load_add_load_load_add_store__ptr_relaxed_cpu_id(ptr, off, inc, cpu); |
809f5ee3 | 286 | case RSEQ_PERCPU_MM_CID: |
aa021469 | 287 | return rseq_load_add_load_load_add_store__ptr_relaxed_mm_cid(ptr, off, inc, cpu); |
95dbaeba GK |
288 | default: |
289 | return -1; | |
809f5ee3 | 290 | } |
809f5ee3 MD |
291 | } |
292 | #endif | |
293 | ||
294 | static inline __attribute__((always_inline)) | |
41149e28 | 295 | int rseq_load_cbne_store_store__ptr(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode, |
809f5ee3 MD |
296 | intptr_t *v, intptr_t expect, |
297 | intptr_t *v2, intptr_t newv2, | |
298 | intptr_t newv, int cpu) | |
299 | { | |
300 | switch (rseq_mo) { | |
301 | case RSEQ_MO_RELAXED: | |
302 | switch (percpu_mode) { | |
303 | case RSEQ_PERCPU_CPU_ID: | |
41149e28 | 304 | return rseq_load_cbne_store_store__ptr_relaxed_cpu_id(v, expect, v2, newv2, newv, cpu); |
809f5ee3 | 305 | case RSEQ_PERCPU_MM_CID: |
41149e28 | 306 | return rseq_load_cbne_store_store__ptr_relaxed_mm_cid(v, expect, v2, newv2, newv, cpu); |
95dbaeba GK |
307 | default: |
308 | return -1; | |
809f5ee3 | 309 | } |
809f5ee3 MD |
310 | case RSEQ_MO_RELEASE: |
311 | switch (percpu_mode) { | |
312 | case RSEQ_PERCPU_CPU_ID: | |
41149e28 | 313 | return rseq_load_cbne_store_store__ptr_release_cpu_id(v, expect, v2, newv2, newv, cpu); |
809f5ee3 | 314 | case RSEQ_PERCPU_MM_CID: |
41149e28 | 315 | return rseq_load_cbne_store_store__ptr_release_mm_cid(v, expect, v2, newv2, newv, cpu); |
95dbaeba GK |
316 | default: |
317 | return -1; | |
809f5ee3 | 318 | } |
8dd73cf9 GK |
319 | case RSEQ_MO_ACQUIRE: /* Fallthrough */ |
320 | case RSEQ_MO_ACQ_REL: /* Fallthrough */ | |
321 | case RSEQ_MO_CONSUME: /* Fallthrough */ | |
322 | case RSEQ_MO_SEQ_CST: /* Fallthrough */ | |
809f5ee3 MD |
323 | default: |
324 | return -1; | |
325 | } | |
326 | } | |
327 | ||
328 | static inline __attribute__((always_inline)) | |
41149e28 | 329 | int rseq_load_cbne_load_cbne_store__ptr(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode, |
809f5ee3 MD |
330 | intptr_t *v, intptr_t expect, |
331 | intptr_t *v2, intptr_t expect2, | |
332 | intptr_t newv, int cpu) | |
333 | { | |
334 | if (rseq_mo != RSEQ_MO_RELAXED) | |
335 | return -1; | |
336 | switch (percpu_mode) { | |
337 | case RSEQ_PERCPU_CPU_ID: | |
41149e28 | 338 | return rseq_load_cbne_load_cbne_store__ptr_relaxed_cpu_id(v, expect, v2, expect2, newv, cpu); |
809f5ee3 | 339 | case RSEQ_PERCPU_MM_CID: |
41149e28 | 340 | return rseq_load_cbne_load_cbne_store__ptr_relaxed_mm_cid(v, expect, v2, expect2, newv, cpu); |
95dbaeba GK |
341 | default: |
342 | return -1; | |
809f5ee3 | 343 | } |
809f5ee3 MD |
344 | } |
345 | ||
346 | static inline __attribute__((always_inline)) | |
41149e28 | 347 | int rseq_load_cbne_memcpy_store__ptr(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode, |
809f5ee3 MD |
348 | intptr_t *v, intptr_t expect, |
349 | void *dst, void *src, size_t len, | |
350 | intptr_t newv, int cpu) | |
351 | { | |
352 | switch (rseq_mo) { | |
353 | case RSEQ_MO_RELAXED: | |
354 | switch (percpu_mode) { | |
355 | case RSEQ_PERCPU_CPU_ID: | |
41149e28 | 356 | return rseq_load_cbne_memcpy_store__ptr_relaxed_cpu_id(v, expect, dst, src, len, newv, cpu); |
809f5ee3 | 357 | case RSEQ_PERCPU_MM_CID: |
41149e28 | 358 | return rseq_load_cbne_memcpy_store__ptr_relaxed_mm_cid(v, expect, dst, src, len, newv, cpu); |
95dbaeba GK |
359 | default: |
360 | return -1; | |
809f5ee3 | 361 | } |
809f5ee3 MD |
362 | case RSEQ_MO_RELEASE: |
363 | switch (percpu_mode) { | |
364 | case RSEQ_PERCPU_CPU_ID: | |
41149e28 | 365 | return rseq_load_cbne_memcpy_store__ptr_release_cpu_id(v, expect, dst, src, len, newv, cpu); |
809f5ee3 | 366 | case RSEQ_PERCPU_MM_CID: |
41149e28 | 367 | return rseq_load_cbne_memcpy_store__ptr_release_mm_cid(v, expect, dst, src, len, newv, cpu); |
95dbaeba GK |
368 | default: |
369 | return -1; | |
809f5ee3 | 370 | } |
8dd73cf9 GK |
371 | case RSEQ_MO_ACQUIRE: /* Fallthrough */ |
372 | case RSEQ_MO_ACQ_REL: /* Fallthrough */ | |
373 | case RSEQ_MO_CONSUME: /* Fallthrough */ | |
374 | case RSEQ_MO_SEQ_CST: /* Fallthrough */ | |
809f5ee3 MD |
375 | default: |
376 | return -1; | |
377 | } | |
378 | } | |
379 | ||
15260018 | 380 | #ifdef __cplusplus |
60a27517 MG |
381 | } |
382 | #endif | |
383 | ||
44ec21eb | 384 | #endif /* _RSEQ_RSEQ_H */ |