1 // SPDX-License-Identifier: MIT
3 * Copyright 2022 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
11 #define SIDE_CACHE_LINE_SIZE 256
12 #define SIDE_RCU_PERCPU_ARRAY_SIZE 2
14 struct side_rcu_percpu_count
{
17 } __attribute__((__aligned__(SIDE_CACHE_LINE_SIZE
)));
19 struct side_rcu_cpu_gp_state
{
20 struct side_rcu_percpu_count count
[SIDE_RCU_PERCPU_ARRAY_SIZE
];
23 struct side_rcu_gp_state
{
24 struct side_rcu_cpu_gp_state
*percpu_state
;
27 pthread_mutex_t gp_lock
;
30 //TODO: replace atomics by rseq (when available)
31 //TODO: replace acquire/release by membarrier+compiler barrier (when available)
32 //TODO: implement wait/wakeup for grace period using sys_futex
34 unsigned int side_rcu_read_begin(struct side_rcu_gp_state
*gp_state
)
36 int cpu
= sched_getcpu();
37 unsigned int period
= __atomic_load_n(&gp_state
->period
, __ATOMIC_RELAXED
);
42 * This acquire MO pairs with the release fence at the end of
43 * side_rcu_wait_grace_period().
45 (void) __atomic_add_fetch(&gp_state
->percpu_state
[cpu
].count
[period
].begin
, 1, __ATOMIC_ACQUIRE
);
50 void side_rcu_read_end(struct side_rcu_gp_state
*gp_state
, unsigned int period
)
52 int cpu
= sched_getcpu();
57 * This release MO pairs with the acquire fence at the beginning
58 * of side_rcu_wait_grace_period().
60 (void) __atomic_add_fetch(&gp_state
->percpu_state
[cpu
].count
[period
].end
, 1, __ATOMIC_RELEASE
);
64 void wait_for_cpus(struct side_rcu_gp_state
*gp_state
)
66 unsigned int prev_period
= 1 - gp_state
->period
;
69 * Wait for the sum of CPU begin/end counts to match for the
73 uintptr_t sum
= 0; /* begin - end */
76 for (i
= 0; i
< gp_state
->nr_cpus
; i
++) {
77 struct side_rcu_cpu_gp_state
*cpu_state
= &gp_state
->percpu_state
[i
];
79 sum
-= __atomic_load_n(&cpu_state
->count
[prev_period
].end
, __ATOMIC_RELAXED
);
83 * Read end counts before begin counts. Reading end
84 * before begin count ensures we never see an end
85 * without having seen its associated begin, in case of
86 * a thread migration during the traversal over each
89 __atomic_thread_fence(__ATOMIC_SEQ_CST
);
91 for (i
= 0; i
< gp_state
->nr_cpus
; i
++) {
92 struct side_rcu_cpu_gp_state
*cpu_state
= &gp_state
->percpu_state
[i
];
94 sum
+= __atomic_load_n(&cpu_state
->count
[prev_period
].begin
, __ATOMIC_RELAXED
);
99 /* Retry after 10ms. */
106 void side_rcu_wait_grace_period(struct side_rcu_gp_state
*gp_state
)
109 * This fence pairs with the acquire MO __atomic_add_fetch in
110 * side_rcu_read_begin().
112 __atomic_thread_fence(__ATOMIC_SEQ_CST
);
114 pthread_mutex_lock(&gp_state
->gp_lock
);
116 wait_for_cpus(gp_state
);
118 /* Flip period: 0 -> 1, 1 -> 0. */
119 (void) __atomic_xor_fetch(&gp_state
->period
, 1, __ATOMIC_SEQ_CST
);
121 wait_for_cpus(gp_state
);
123 pthread_mutex_unlock(&gp_state
->gp_lock
);
126 * This fence pairs with the release MO __atomic_add_fetch in
127 * side_rcu_read_end().
129 __atomic_thread_fence(__ATOMIC_SEQ_CST
);