src/rcu.h

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright 2022 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   4  */
   5
   6 #ifndef _SIDE_RCU_H
   7 #define _SIDE_RCU_H
   8
   9 #include <sched.h>
  10 #include <stdint.h>
  11 #include <pthread.h>
  12 #include <stdbool.h>
  13 #include <poll.h>
  14 #include <side/trace.h>
  15 #include <rseq/rseq.h>
  16
  17 #define SIDE_CACHE_LINE_SIZE            256
  18
  19 struct side_rcu_percpu_count {
  20         uintptr_t begin;
  21         uintptr_t rseq_begin;
  22         uintptr_t end;
  23         uintptr_t rseq_end;
  24 }  __attribute__((__aligned__(SIDE_CACHE_LINE_SIZE)));
  25
  26 struct side_rcu_cpu_gp_state {
  27         struct side_rcu_percpu_count count[2];
  28 };
  29
  30 struct side_rcu_gp_state {
  31         struct side_rcu_cpu_gp_state *percpu_state;
  32         int nr_cpus;
  33         unsigned int period;
  34         pthread_mutex_t gp_lock;
  35 };
  36
  37 //TODO: replace acquire/release by membarrier+compiler barrier (when available)
  38 //TODO: implement wait/wakeup for grace period using sys_futex
  39 static inline
  40 unsigned int side_rcu_read_begin(struct side_rcu_gp_state *gp_state)
  41 {
  42         unsigned int period = __atomic_load_n(&gp_state->period, __ATOMIC_RELAXED);
  43         struct side_rcu_cpu_gp_state *cpu_gp_state;
  44         int cpu;
  45
  46         if (side_likely(rseq_offset > 0)) {
  47                 cpu = rseq_cpu_start();
  48                 cpu_gp_state = &gp_state->percpu_state[cpu];
  49                 if (!rseq_addv((intptr_t *)&cpu_gp_state->count[period].rseq_begin, 1, cpu))
  50                         goto fence;
  51         }
  52         cpu = sched_getcpu();
  53         if (side_unlikely(cpu < 0))
  54                 cpu = 0;
  55         cpu_gp_state = &gp_state->percpu_state[cpu];
  56         (void) __atomic_add_fetch(&cpu_gp_state->count[period].begin, 1, __ATOMIC_RELAXED);
  57 fence:
  58         /*
  59          * This memory barrier (A) ensures that the contents of the
  60          * read-side critical section does not leak before the "begin"
  61          * counter increment. It pairs with memory barriers (D) and (E).
  62          *
  63          * This memory barrier (A) also ensures that the "begin"
  64          * increment is before the "end" increment. It pairs with memory
  65          * barrier (C). It is redundant with memory barrier (B) for that
  66          * purpose.
  67          */
  68         __atomic_thread_fence(__ATOMIC_SEQ_CST);
  69         return period;
  70 }
  71
  72 static inline
  73 void side_rcu_read_end(struct side_rcu_gp_state *gp_state, unsigned int period)
  74 {
  75         struct side_rcu_cpu_gp_state *cpu_gp_state;
  76         int cpu;
  77
  78         /*
  79          * This memory barrier (B) ensures that the contents of the
  80          * read-side critical section does not leak after the "end"
  81          * counter increment. It pairs with memory barriers (D) and (E).
  82          *
  83          * This memory barrier (B) also ensures that the "begin"
  84          * increment is before the "end" increment. It pairs with memory
  85          * barrier (C). It is redundant with memory barrier (A) for that
  86          * purpose.
  87          */
  88         __atomic_thread_fence(__ATOMIC_SEQ_CST);
  89
  90         if (side_likely(rseq_offset > 0)) {
  91                 cpu = rseq_cpu_start();
  92                 cpu_gp_state = &gp_state->percpu_state[cpu];
  93                 if (!rseq_addv((intptr_t *)&cpu_gp_state->count[period].rseq_end, 1, cpu))
  94                         return;
  95         }
  96         cpu = sched_getcpu();
  97         if (side_unlikely(cpu < 0))
  98                 cpu = 0;
  99         cpu_gp_state = &gp_state->percpu_state[cpu];
 100         (void) __atomic_add_fetch(&cpu_gp_state->count[period].end, 1, __ATOMIC_RELAXED);
 101 }
 102
 103 #define side_rcu_dereference(p) \
 104         __extension__ \
 105         ({ \
 106                 __typeof__(p) _____side_v = __atomic_load_n(&(p), __ATOMIC_CONSUME); \
 107                 (_____side_v); \
 108         })
 109
 110 #define side_rcu_assign_pointer(p, v)   __atomic_store_n(&(p), v, __ATOMIC_RELEASE); \
 111
 112 void side_rcu_wait_grace_period(struct side_rcu_gp_state *gp_state) __attribute__((visibility("hidden")));
 113 void side_rcu_gp_init(struct side_rcu_gp_state *rcu_gp) __attribute__((visibility("hidden")));
 114 void side_rcu_gp_exit(struct side_rcu_gp_state *rcu_gp) __attribute__((visibility("hidden")));
 115
 116 #endif /* _SIDE_RCU_H */