Commit | Line | Data |
---|---|---|
85b765b8 MD |
1 | // SPDX-License-Identifier: MIT |
2 | /* | |
3 | * Copyright 2022 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> | |
4 | */ | |
5 | ||
d04d4903 MD |
6 | #ifndef _TGIF_RCU_H |
7 | #define _TGIF_RCU_H | |
0a65ebfb | 8 | |
85b765b8 MD |
9 | #include <sched.h> |
10 | #include <stdint.h> | |
11 | #include <pthread.h> | |
bcdea8ac | 12 | #include <stdbool.h> |
85b765b8 | 13 | #include <poll.h> |
7fb53c62 | 14 | #include <rseq/rseq.h> |
6635dd92 MD |
15 | #include <linux/futex.h> |
16 | #include <sys/time.h> | |
17 | #include <unistd.h> | |
18 | #include <sys/syscall.h> | |
d04d4903 | 19 | #include <tgif/macros.h> |
85b765b8 | 20 | |
d04d4903 | 21 | #define TGIF_CACHE_LINE_SIZE 256 |
85b765b8 | 22 | |
d04d4903 | 23 | struct tgif_rcu_percpu_count { |
85b765b8 | 24 | uintptr_t begin; |
7fb53c62 | 25 | uintptr_t rseq_begin; |
85b765b8 | 26 | uintptr_t end; |
7fb53c62 | 27 | uintptr_t rseq_end; |
f90ef04f | 28 | }; |
85b765b8 | 29 | |
d04d4903 MD |
30 | struct tgif_rcu_cpu_gp_state { |
31 | struct tgif_rcu_percpu_count count[2]; | |
32 | } __attribute__((__aligned__(TGIF_CACHE_LINE_SIZE))); | |
85b765b8 | 33 | |
d04d4903 MD |
34 | struct tgif_rcu_gp_state { |
35 | struct tgif_rcu_cpu_gp_state *percpu_state; | |
85b765b8 | 36 | int nr_cpus; |
6635dd92 | 37 | int32_t futex; |
85b765b8 MD |
38 | unsigned int period; |
39 | pthread_mutex_t gp_lock; | |
40 | }; | |
41 | ||
d04d4903 MD |
42 | struct tgif_rcu_read_state { |
43 | struct tgif_rcu_percpu_count *percpu_count; | |
cd50bc06 MD |
44 | int cpu; |
45 | }; | |
46 | ||
d04d4903 | 47 | extern unsigned int tgif_rcu_rseq_membarrier_available __attribute__((visibility("hidden"))); |
bddcdc92 | 48 | |
6635dd92 MD |
49 | static inline |
50 | int futex(int32_t *uaddr, int op, int32_t val, | |
51 | const struct timespec *timeout, int32_t *uaddr2, int32_t val3) | |
52 | { | |
53 | return syscall(__NR_futex, uaddr, op, val, timeout, uaddr2, val3); | |
54 | } | |
55 | ||
56 | /* | |
d04d4903 | 57 | * Wake-up tgif_rcu_wait_grace_period. Called concurrently from many |
6635dd92 MD |
58 | * threads. |
59 | */ | |
60 | static inline | |
d04d4903 | 61 | void tgif_rcu_wake_up_gp(struct tgif_rcu_gp_state *gp_state) |
6635dd92 | 62 | { |
d04d4903 | 63 | if (tgif_unlikely(__atomic_load_n(&gp_state->futex, __ATOMIC_RELAXED) == -1)) { |
6635dd92 MD |
64 | __atomic_store_n(&gp_state->futex, 0, __ATOMIC_RELAXED); |
65 | /* TODO: handle futex return values. */ | |
66 | (void) futex(&gp_state->futex, FUTEX_WAKE, 1, NULL, NULL, 0); | |
67 | } | |
68 | } | |
69 | ||
85b765b8 | 70 | static inline |
d04d4903 | 71 | void tgif_rcu_read_begin(struct tgif_rcu_gp_state *gp_state, struct tgif_rcu_read_state *read_state) |
85b765b8 | 72 | { |
d04d4903 MD |
73 | struct tgif_rcu_percpu_count *begin_cpu_count; |
74 | struct tgif_rcu_cpu_gp_state *cpu_gp_state; | |
609cb7ea | 75 | unsigned int period; |
7fb53c62 | 76 | int cpu; |
85b765b8 | 77 | |
609cb7ea MD |
78 | cpu = rseq_cpu_start(); |
79 | period = __atomic_load_n(&gp_state->period, __ATOMIC_RELAXED); | |
80 | cpu_gp_state = &gp_state->percpu_state[cpu]; | |
81 | read_state->percpu_count = begin_cpu_count = &cpu_gp_state->count[period]; | |
82 | read_state->cpu = cpu; | |
d04d4903 | 83 | if (tgif_likely(tgif_rcu_rseq_membarrier_available && |
609cb7ea MD |
84 | !rseq_addv((intptr_t *)&begin_cpu_count->rseq_begin, 1, cpu))) { |
85 | /* | |
86 | * This compiler barrier (A) is paired with membarrier() at (C), | |
87 | * (D), (E). It effectively upgrades this compiler barrier to a | |
88 | * SEQ_CST fence with respect to the paired barriers. | |
89 | * | |
d04d4903 | 90 | * This barrier (A) ensures that the contents of the read-tgif |
609cb7ea MD |
91 | * critical section does not leak before the "begin" counter |
92 | * increment. It pairs with memory barriers (D) and (E). | |
93 | * | |
94 | * This barrier (A) also ensures that the "begin" increment is | |
95 | * before the "end" increment. It pairs with memory barrier (C). | |
96 | * It is redundant with barrier (B) for that purpose. | |
97 | */ | |
98 | rseq_barrier(); | |
99 | return; | |
7fb53c62 | 100 | } |
bddcdc92 | 101 | /* Fallback to atomic increment and SEQ_CST. */ |
7fb53c62 | 102 | cpu = sched_getcpu(); |
d04d4903 | 103 | if (tgif_unlikely(cpu < 0)) |
85b765b8 | 104 | cpu = 0; |
cd50bc06 | 105 | read_state->cpu = cpu; |
7fb53c62 | 106 | cpu_gp_state = &gp_state->percpu_state[cpu]; |
cd50bc06 MD |
107 | read_state->percpu_count = begin_cpu_count = &cpu_gp_state->count[period]; |
108 | (void) __atomic_add_fetch(&begin_cpu_count->begin, 1, __ATOMIC_SEQ_CST); | |
85b765b8 MD |
109 | } |
110 | ||
111 | static inline | |
d04d4903 | 112 | void tgif_rcu_read_end(struct tgif_rcu_gp_state *gp_state, struct tgif_rcu_read_state *read_state) |
85b765b8 | 113 | { |
d04d4903 | 114 | struct tgif_rcu_percpu_count *begin_cpu_count = read_state->percpu_count; |
cd50bc06 | 115 | int cpu = read_state->cpu; |
85b765b8 | 116 | |
609cb7ea MD |
117 | /* |
118 | * This compiler barrier (B) is paired with membarrier() at (C), | |
119 | * (D), (E). It effectively upgrades this compiler barrier to a | |
120 | * SEQ_CST fence with respect to the paired barriers. | |
121 | * | |
d04d4903 | 122 | * This barrier (B) ensures that the contents of the read-tgif |
609cb7ea MD |
123 | * critical section does not leak after the "end" counter |
124 | * increment. It pairs with memory barriers (D) and (E). | |
125 | * | |
126 | * This barrier (B) also ensures that the "begin" increment is | |
127 | * before the "end" increment. It pairs with memory barrier (C). | |
128 | * It is redundant with barrier (A) for that purpose. | |
129 | */ | |
130 | rseq_barrier(); | |
d04d4903 | 131 | if (tgif_likely(tgif_rcu_rseq_membarrier_available && |
609cb7ea | 132 | !rseq_addv((intptr_t *)&begin_cpu_count->rseq_end, 1, cpu))) { |
bddcdc92 | 133 | /* |
609cb7ea MD |
134 | * This barrier (F) is paired with membarrier() |
135 | * at (G). It orders increment of the begin/end | |
136 | * counters before load/store to the futex. | |
bddcdc92 MD |
137 | */ |
138 | rseq_barrier(); | |
609cb7ea | 139 | goto end; |
7fb53c62 | 140 | } |
bddcdc92 | 141 | /* Fallback to atomic increment and SEQ_CST. */ |
cd50bc06 | 142 | (void) __atomic_add_fetch(&begin_cpu_count->end, 1, __ATOMIC_SEQ_CST); |
6635dd92 MD |
143 | /* |
144 | * This barrier (F) is paired with SEQ_CST barrier or | |
145 | * membarrier() at (G). It orders increment of the begin/end | |
146 | * counters before load/store to the futex. | |
147 | */ | |
148 | __atomic_thread_fence(__ATOMIC_SEQ_CST); | |
149 | end: | |
d04d4903 | 150 | tgif_rcu_wake_up_gp(gp_state); |
85b765b8 MD |
151 | } |
152 | ||
d04d4903 | 153 | #define tgif_rcu_dereference(p) \ |
11f8a2ce MD |
154 | __extension__ \ |
155 | ({ \ | |
d04d4903 MD |
156 | __typeof__(p) _____tgif_v = __atomic_load_n(&(p), __ATOMIC_CONSUME); \ |
157 | (_____tgif_v); \ | |
11f8a2ce MD |
158 | }) |
159 | ||
d04d4903 | 160 | #define tgif_rcu_assign_pointer(p, v) __atomic_store_n(&(p), v, __ATOMIC_RELEASE); \ |
11f8a2ce | 161 | |
d04d4903 MD |
162 | void tgif_rcu_wait_grace_period(struct tgif_rcu_gp_state *gp_state) __attribute__((visibility("hidden"))); |
163 | void tgif_rcu_gp_init(struct tgif_rcu_gp_state *rcu_gp) __attribute__((visibility("hidden"))); | |
164 | void tgif_rcu_gp_exit(struct tgif_rcu_gp_state *rcu_gp) __attribute__((visibility("hidden"))); | |
0a65ebfb | 165 | |
d04d4903 | 166 | #endif /* _TGIF_RCU_H */ |