Use rseq for commit counter

Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>

diff --git a/include/lttng/ringbuffer-config.h b/include/lttng/ringbuffer-config.h
index ef1526212dcfa530143ff853d1d4474e728af213..2d9f73c697647341c478d32981d8b1ad27336161 100644 (file)
--- a/include/lttng/ringbuffer-config.h
+++ b/include/lttng/ringbuffer-config.h
@@ -279,6 +279,7 @@ struct lttng_ust_lib_ring_buffer_ctx {
         * following fields may be used.
         */
        struct lttng_ust_lib_ring_buffer_backend_pages *backend_pages;
+       struct lttng_rseq_state rseq_state;
 };
 
 /**
@@ -314,6 +315,9 @@ void lib_ring_buffer_ctx_init(struct lttng_ust_lib_ring_buffer_ctx *ctx,
        ctx->ip = 0;
        ctx->priv2 = priv2;
        memset(ctx->padding2, 0, LTTNG_UST_RING_BUFFER_CTX_PADDING);
+       ctx->rseq_state.rseqp = NULL;
+       ctx->rseq_state.cpu_id = -1;
+       ctx->rseq_state.event_counter = 0;
 }
 
 /*

diff --git a/liblttng-ust/lttng-ring-buffer-client.h b/liblttng-ust/lttng-ring-buffer-client.h
index b397a5e06b176b9e3027ab8095d2ddec54a09f63..5fde855cea6947bc1f24f0a2d75388d777930298 100644 (file)
--- a/liblttng-ust/lttng-ring-buffer-client.h
+++ b/liblttng-ust/lttng-ring-buffer-client.h
@@ -743,6 +743,11 @@ retry:
                }
        }
        lttng_write_event_header(&client_config, ctx, event_id);
+
+       if (caa_likely(ctx->ctx_len
+                       >= sizeof(struct lttng_ust_lib_ring_buffer_ctx)))
+               ctx->rseq_state = rseq_state;
+
        return 0;
 end:
        lib_ring_buffer_end(&client_config);

diff --git a/libringbuffer/frontend_api.h b/libringbuffer/frontend_api.h
index 19fc09e9ae5f29d779a78c737e08e8530c795ae4..6ff98da0d1ed22a60800cb2dc7008e4f49e46ff0 100644 (file)
--- a/libringbuffer/frontend_api.h
+++ b/libringbuffer/frontend_api.h
@@ -33,6 +33,7 @@
 #include "frontend.h"
 #include <urcu-bp.h>
 #include <urcu/compiler.h>
+#include "rseq.h"
 
 static inline
 int lib_ring_buffer_get_cpu(const struct lttng_ust_lib_ring_buffer_config *config)
@@ -259,6 +260,14 @@ void lib_ring_buffer_commit(const struct lttng_ust_lib_ring_buffer_config *confi
        unsigned long commit_count;
        struct commit_counters_hot *cc_hot = shmp_index(handle,
                                                buf->commit_hot, endidx);
+       struct lttng_rseq_state rseq_state;
+
+       if (caa_likely(ctx->ctx_len
+                       >= sizeof(struct lttng_ust_lib_ring_buffer_ctx))) {
+               rseq_state = ctx->rseq_state;
+       } else {
+               rseq_state.cpu_id = -2;
+       }
 
        if (caa_unlikely(!cc_hot))
                return;
@@ -274,7 +283,18 @@ void lib_ring_buffer_commit(const struct lttng_ust_lib_ring_buffer_config *confi
         */
        cmm_smp_wmb();
 
+       if (caa_likely(rseq_state.cpu_id >= 0)) {
+               unsigned long newv;
+
+               newv = cc_hot->cc_rseq + ctx->slot_size;
+               if (caa_likely(__rseq_finish(NULL, 0, NULL, NULL, 0,
+                               (intptr_t *)&cc_hot->cc_rseq,
+                               (intptr_t) newv,
+                               rseq_state, RSEQ_FINISH_SINGLE, false)))
+                       goto add_done;
+       }
        v_add(config, ctx->slot_size, &cc_hot->cc);
+add_done:
 
        /*
         * commit count read can race with concurrent OOO commit count updates.
@@ -295,6 +315,7 @@ void lib_ring_buffer_commit(const struct lttng_ust_lib_ring_buffer_config *confi
         *   which is completely independent of the order.
         */
        commit_count = v_read(config, &cc_hot->cc);
+       commit_count += cc_hot->cc_rseq;
 
        lib_ring_buffer_check_deliver(config, buf, chan, offset_end - 1,
                                      commit_count, endidx, handle, ctx->tsc);

diff --git a/libringbuffer/frontend_internal.h b/libringbuffer/frontend_internal.h
index 68aa70223ea5529acd3ecdf67a3c9b4057fcdb49..2bb023f8331b93140c97424f1c0f256f39b0cad2 100644 (file)
--- a/libringbuffer/frontend_internal.h
+++ b/libringbuffer/frontend_internal.h
@@ -253,6 +253,7 @@ int lib_ring_buffer_reserve_committed(const struct lttng_ust_lib_ring_buffer_con
                offset = v_read(config, &buf->offset);
                idx = subbuf_index(offset, chan);
                commit_count = v_read(config, &cc_hot->cc);
+               commit_count += cc_hot->cc_rseq;
        } while (offset != v_read(config, &buf->offset));
 
        return ((buf_trunc(offset, chan) >> chan->backend.num_subbuf_order)

diff --git a/libringbuffer/frontend_types.h b/libringbuffer/frontend_types.h
index bf4b4185e2368945494dd39e3df86efed5ff1aa8..c5c716f753cdd0774319b45fc462ca5f86ff0a60 100644 (file)
--- a/libringbuffer/frontend_types.h
+++ b/libringbuffer/frontend_types.h
@@ -82,6 +82,7 @@ struct channel {
 #define RB_COMMIT_COUNT_HOT_PADDING    16
 struct commit_counters_hot {
        union v_atomic cc;              /* Commit counter */
+       unsigned long cc_rseq;          /* Commit counter for rseq */
        union v_atomic seq;             /* Consecutive commits */
        char padding[RB_COMMIT_COUNT_HOT_PADDING];
 } __attribute__((aligned(CAA_CACHE_LINE_SIZE)));

diff --git a/libringbuffer/ring_buffer_frontend.c b/libringbuffer/ring_buffer_frontend.c
index be20d69b9864d014c9271807254aef282c8b59c8..46751a2d8f58269a940c6efd3fa42f9a64a335c4 100644 (file)
--- a/libringbuffer/ring_buffer_frontend.c
+++ b/libringbuffer/ring_buffer_frontend.c
@@ -185,6 +185,7 @@ void lib_ring_buffer_reset(struct lttng_ust_lib_ring_buffer *buf,
                if (!cc_cold)
                        return;
                v_set(config, &cc_hot->cc, 0);
+               cc_hot->cc_rseq = 0;
                v_set(config, &cc_hot->seq, 0);
                v_set(config, &cc_cold->cc_sb, 0);
        }
@@ -1585,6 +1586,7 @@ void lib_ring_buffer_print_subbuffer_errors(struct lttng_ust_lib_ring_buffer *bu
        if (!cc_cold)
                return;
        commit_count = v_read(config, &cc_hot->cc);
+       commit_count += cc_hot->cc_rseq;
        commit_count_sb = v_read(config, &cc_cold->cc_sb);
 
        if (subbuf_offset(commit_count, chan) != 0)
@@ -1697,6 +1699,7 @@ void lib_ring_buffer_switch_old_start(struct lttng_ust_lib_ring_buffer *buf,
        v_add(config, config->cb.subbuffer_header_size(),
              &cc_hot->cc);
        commit_count = v_read(config, &cc_hot->cc);
+       commit_count += cc_hot->cc_rseq;
        /* Check if the written buffer has to be delivered */
        lib_ring_buffer_check_deliver(config, buf, chan, offsets->old,
                                      commit_count, oldidx, handle, tsc);
@@ -1740,6 +1743,7 @@ void lib_ring_buffer_switch_old_end(struct lttng_ust_lib_ring_buffer *buf,
                return;
        v_add(config, padding_size, &cc_hot->cc);
        commit_count = v_read(config, &cc_hot->cc);
+       commit_count += cc_hot->cc_rseq;
        lib_ring_buffer_check_deliver(config, buf, chan, offsets->old - 1,
                                      commit_count, oldidx, handle, tsc);
        lib_ring_buffer_write_commit_counter(config, buf, chan,
@@ -1778,6 +1782,7 @@ void lib_ring_buffer_switch_new_start(struct lttng_ust_lib_ring_buffer *buf,
                return;
        v_add(config, config->cb.subbuffer_header_size(), &cc_hot->cc);
        commit_count = v_read(config, &cc_hot->cc);
+       commit_count += cc_hot->cc_rseq;
        /* Check if the written buffer has to be delivered */
        lib_ring_buffer_check_deliver(config, buf, chan, offsets->begin,
                                      commit_count, beginidx, handle, tsc);