rcu: introduce read state

Keep read state from begin, pass it to "end", so the per-cpu data
indexing does not have to be evaluated twice.

Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>

diff --git a/src/rcu.h b/src/rcu.h
index c3c458212458271e95f78461031fcc7e5690b1cc..f2221478c9ec484094c1bad7839cb39b7116100d 100644 (file)
--- a/src/rcu.h
+++ b/src/rcu.h
@@ -39,6 +39,11 @@ struct side_rcu_gp_state {
        pthread_mutex_t gp_lock;
 };
 
+struct side_rcu_read_state {
+       struct side_rcu_percpu_count *percpu_count;
+       int cpu;
+};
+
 extern unsigned int side_rcu_rseq_membarrier_available __attribute__((visibility("hidden")));
 
 static inline
@@ -63,16 +68,19 @@ void side_rcu_wake_up_gp(struct side_rcu_gp_state *gp_state)
 }
 
 static inline
-unsigned int side_rcu_read_begin(struct side_rcu_gp_state *gp_state)
+void side_rcu_read_begin(struct side_rcu_gp_state *gp_state, struct side_rcu_read_state *read_state)
 {
        unsigned int period = __atomic_load_n(&gp_state->period, __ATOMIC_RELAXED);
+       struct side_rcu_percpu_count *begin_cpu_count;
        struct side_rcu_cpu_gp_state *cpu_gp_state;
        int cpu;
 
        if (side_likely(side_rcu_rseq_membarrier_available)) {
                cpu = rseq_cpu_start();
                cpu_gp_state = &gp_state->percpu_state[cpu];
-               if (side_likely(!rseq_addv((intptr_t *)&cpu_gp_state->count[period].rseq_begin, 1, cpu))) {
+               read_state->percpu_count = begin_cpu_count = &cpu_gp_state->count[period];
+               read_state->cpu = cpu;
+               if (side_likely(!rseq_addv((intptr_t *)&begin_cpu_count->rseq_begin, 1, cpu))) {
                        /*
                         * This compiler barrier (A) is paired with membarrier() at (C),
                         * (D), (E). It effectively upgrades this compiler barrier to a
@@ -87,23 +95,24 @@ unsigned int side_rcu_read_begin(struct side_rcu_gp_state *gp_state)
                         * It is redundant with barrier (B) for that purpose.
                         */
                        rseq_barrier();
-                       return period;
+                       return;
                }
        }
        /* Fallback to atomic increment and SEQ_CST. */
        cpu = sched_getcpu();
        if (side_unlikely(cpu < 0))
                cpu = 0;
+       read_state->cpu = cpu;
        cpu_gp_state = &gp_state->percpu_state[cpu];
-       (void) __atomic_add_fetch(&cpu_gp_state->count[period].begin, 1, __ATOMIC_SEQ_CST);
-       return period;
+       read_state->percpu_count = begin_cpu_count = &cpu_gp_state->count[period];
+       (void) __atomic_add_fetch(&begin_cpu_count->begin, 1, __ATOMIC_SEQ_CST);
 }
 
 static inline
-void side_rcu_read_end(struct side_rcu_gp_state *gp_state, unsigned int period)
+void side_rcu_read_end(struct side_rcu_gp_state *gp_state, struct side_rcu_read_state *read_state)
 {
-       struct side_rcu_cpu_gp_state *cpu_gp_state;
-       int cpu;
+       struct side_rcu_percpu_count *begin_cpu_count = read_state->percpu_count;
+       int cpu = read_state->cpu;
 
        if (side_likely(side_rcu_rseq_membarrier_available)) {
                /*
@@ -120,9 +129,7 @@ void side_rcu_read_end(struct side_rcu_gp_state *gp_state, unsigned int period)
                 * It is redundant with barrier (A) for that purpose.
                 */
                rseq_barrier();
-               cpu = rseq_cpu_start();
-               cpu_gp_state = &gp_state->percpu_state[cpu];
-               if (side_likely(!rseq_addv((intptr_t *)&cpu_gp_state->count[period].rseq_end, 1, cpu))) {
+               if (side_likely(!rseq_addv((intptr_t *)&begin_cpu_count->rseq_end, 1, cpu))) {
                        /*
                         * This barrier (F) is paired with membarrier()
                         * at (G). It orders increment of the begin/end
@@ -133,11 +140,7 @@ void side_rcu_read_end(struct side_rcu_gp_state *gp_state, unsigned int period)
                }
        }
        /* Fallback to atomic increment and SEQ_CST. */
-       cpu = sched_getcpu();
-       if (side_unlikely(cpu < 0))
-               cpu = 0;
-       cpu_gp_state = &gp_state->percpu_state[cpu];
-       (void) __atomic_add_fetch(&cpu_gp_state->count[period].end, 1, __ATOMIC_SEQ_CST);
+       (void) __atomic_add_fetch(&begin_cpu_count->end, 1, __ATOMIC_SEQ_CST);
        /*
         * This barrier (F) is paired with SEQ_CST barrier or
         * membarrier() at (G). It orders increment of the begin/end

diff --git a/src/side.c b/src/side.c
index c1641b950b65a09f57e11b619f53fae5e44f35b5..e212d8e4de4c5404921fe7d873adf187b52b882a 100644 (file)
--- a/src/side.c
+++ b/src/side.c
@@ -68,8 +68,8 @@ void side_exit(void) __attribute__((destructor));
 
 void side_call(const struct side_event_description *desc, const struct side_arg_vec *side_arg_vec)
 {
+       struct side_rcu_read_state rcu_read_state;
        const struct side_callback *side_cb;
-       unsigned int rcu_period;
        uintptr_t enabled;
 
        if (side_unlikely(finalized))
@@ -84,18 +84,18 @@ void side_call(const struct side_event_description *desc, const struct side_arg_
        if (side_unlikely(enabled & SIDE_EVENT_ENABLED_KERNEL_USER_EVENT_MASK)) {
                // TODO: call kernel write.
        }
-       rcu_period = side_rcu_read_begin(&rcu_gp);
+       side_rcu_read_begin(&rcu_gp, &rcu_read_state);
        for (side_cb = side_rcu_dereference(desc->callbacks); side_cb->u.call != NULL; side_cb++)
                side_cb->u.call(desc, side_arg_vec, side_cb->priv);
-       side_rcu_read_end(&rcu_gp, rcu_period);
+       side_rcu_read_end(&rcu_gp, &rcu_read_state);
 }
 
 void side_call_variadic(const struct side_event_description *desc,
        const struct side_arg_vec *side_arg_vec,
        const struct side_arg_dynamic_struct *var_struct)
 {
+       struct side_rcu_read_state rcu_read_state;
        const struct side_callback *side_cb;
-       unsigned int rcu_period;
        uintptr_t enabled;
 
        if (side_unlikely(finalized))
@@ -110,10 +110,10 @@ void side_call_variadic(const struct side_event_description *desc,
        if (side_unlikely(enabled & SIDE_EVENT_ENABLED_KERNEL_USER_EVENT_MASK)) {
                // TODO: call kernel write.
        }
-       rcu_period = side_rcu_read_begin(&rcu_gp);
+       side_rcu_read_begin(&rcu_gp, &rcu_read_state);
        for (side_cb = side_rcu_dereference(desc->callbacks); side_cb->u.call_variadic != NULL; side_cb++)
                side_cb->u.call_variadic(desc, side_arg_vec, var_struct, side_cb->priv);
-       side_rcu_read_end(&rcu_gp, rcu_period);
+       side_rcu_read_end(&rcu_gp, &rcu_read_state);
 }
 
 static

diff --git a/tests/regression/side-rcu-test.c b/tests/regression/side-rcu-test.c
index 21971d79b2906190836d8518383ed904eab84563..c8f477f8b80fa7fd37095e56b0faec1c3abe76c5 100644 (file)
--- a/tests/regression/side-rcu-test.c
+++ b/tests/regression/side-rcu-test.c
@@ -41,10 +41,11 @@ void *test_reader_thread(void *arg)
        while (!start_test) { }
 
        while (!stop_test) {
+               struct side_rcu_read_state rcu_read_state;
                struct test_data *p;
-               int v, period;
+               int v;
 
-               period = side_rcu_read_begin(&test_rcu_gp);
+               side_rcu_read_begin(&test_rcu_gp, &rcu_read_state);
                p = side_rcu_dereference(rcu_p);
                if (p) {
                        v = p->v;
@@ -53,7 +54,7 @@ void *test_reader_thread(void *arg)
                                abort();
                        }
                }
-               side_rcu_read_end(&test_rcu_gp, period);
+               side_rcu_read_end(&test_rcu_gp, &rcu_read_state);
                count++;
        }
        thread_ctx->count = count;