src/rcu.h

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright 2022 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   4  */
   5
   6 #ifndef _SIDE_RCU_H
   7 #define _SIDE_RCU_H
   8
   9 #include <sched.h>
  10 #include <stdint.h>
  11 #include <pthread.h>
  12 #include <stdbool.h>
  13 #include <poll.h>
  14 #include <side/trace.h>
  15 #include <rseq/rseq.h>
  16
  17 #define SIDE_CACHE_LINE_SIZE            256
  18
  19 struct side_rcu_percpu_count {
  20         uintptr_t begin;
  21         uintptr_t rseq_begin;
  22         uintptr_t end;
  23         uintptr_t rseq_end;
  24 }  __attribute__((__aligned__(SIDE_CACHE_LINE_SIZE)));
  25
  26 struct side_rcu_cpu_gp_state {
  27         struct side_rcu_percpu_count count[2];
  28 };
  29
  30 struct side_rcu_gp_state {
  31         struct side_rcu_cpu_gp_state *percpu_state;
  32         int nr_cpus;
  33         unsigned int period;
  34         pthread_mutex_t gp_lock;
  35 };
  36
  37 //TODO: implement wait/wakeup for grace period using sys_futex
  38 static inline
  39 unsigned int side_rcu_read_begin(struct side_rcu_gp_state *gp_state)
  40 {
  41         unsigned int period = __atomic_load_n(&gp_state->period, __ATOMIC_RELAXED);
  42         struct side_rcu_cpu_gp_state *cpu_gp_state;
  43         int cpu;
  44
  45         if (side_likely(rseq_offset > 0)) {
  46                 cpu = rseq_cpu_start();
  47                 cpu_gp_state = &gp_state->percpu_state[cpu];
  48                 if (!rseq_addv((intptr_t *)&cpu_gp_state->count[period].rseq_begin, 1, cpu))
  49                         goto fence;
  50         }
  51         cpu = sched_getcpu();
  52         if (side_unlikely(cpu < 0))
  53                 cpu = 0;
  54         cpu_gp_state = &gp_state->percpu_state[cpu];
  55         (void) __atomic_add_fetch(&cpu_gp_state->count[period].begin, 1, __ATOMIC_RELAXED);
  56 fence:
  57         /*
  58          * This memory barrier (A) ensures that the contents of the
  59          * read-side critical section does not leak before the "begin"
  60          * counter increment. It pairs with memory barriers (D) and (E).
  61          *
  62          * This memory barrier (A) also ensures that the "begin"
  63          * increment is before the "end" increment. It pairs with memory
  64          * barrier (C). It is redundant with memory barrier (B) for that
  65          * purpose.
  66          */
  67         rseq_barrier();
  68         return period;
  69 }
  70
  71 static inline
  72 void side_rcu_read_end(struct side_rcu_gp_state *gp_state, unsigned int period)
  73 {
  74         struct side_rcu_cpu_gp_state *cpu_gp_state;
  75         int cpu;
  76
  77         /*
  78          * This memory barrier (B) ensures that the contents of the
  79          * read-side critical section does not leak after the "end"
  80          * counter increment. It pairs with memory barriers (D) and (E).
  81          *
  82          * This memory barrier (B) also ensures that the "begin"
  83          * increment is before the "end" increment. It pairs with memory
  84          * barrier (C). It is redundant with memory barrier (A) for that
  85          * purpose.
  86          */
  87         rseq_barrier();
  88
  89         if (side_likely(rseq_offset > 0)) {
  90                 cpu = rseq_cpu_start();
  91                 cpu_gp_state = &gp_state->percpu_state[cpu];
  92                 if (!rseq_addv((intptr_t *)&cpu_gp_state->count[period].rseq_end, 1, cpu))
  93                         return;
  94         }
  95         cpu = sched_getcpu();
  96         if (side_unlikely(cpu < 0))
  97                 cpu = 0;
  98         cpu_gp_state = &gp_state->percpu_state[cpu];
  99         (void) __atomic_add_fetch(&cpu_gp_state->count[period].end, 1, __ATOMIC_RELAXED);
 100 }
 101
 102 #define side_rcu_dereference(p) \
 103         __extension__ \
 104         ({ \
 105                 __typeof__(p) _____side_v = __atomic_load_n(&(p), __ATOMIC_CONSUME); \
 106                 (_____side_v); \
 107         })
 108
 109 #define side_rcu_assign_pointer(p, v)   __atomic_store_n(&(p), v, __ATOMIC_RELEASE); \
 110
 111 void side_rcu_wait_grace_period(struct side_rcu_gp_state *gp_state) __attribute__((visibility("hidden")));
 112 void side_rcu_gp_init(struct side_rcu_gp_state *rcu_gp) __attribute__((visibility("hidden")));
 113 void side_rcu_gp_exit(struct side_rcu_gp_state *rcu_gp) __attribute__((visibility("hidden")));
 114
 115 #endif /* _SIDE_RCU_H */