From 3ea36dea02d1996a43a2ee833ccbd7929570bd58 Mon Sep 17 00:00:00 2001 From: Mathieu Desnoyers Date: Sat, 24 Sep 2016 10:26:09 -0400 Subject: [PATCH] Use rseq for commit counter Signed-off-by: Mathieu Desnoyers --- include/lttng/ringbuffer-config.h | 4 ++++ liblttng-ust/lttng-ring-buffer-client.h | 5 +++++ libringbuffer/frontend_api.h | 21 +++++++++++++++++++++ libringbuffer/frontend_internal.h | 1 + libringbuffer/frontend_types.h | 1 + libringbuffer/ring_buffer_frontend.c | 5 +++++ 6 files changed, 37 insertions(+) diff --git a/include/lttng/ringbuffer-config.h b/include/lttng/ringbuffer-config.h index ef152621..2d9f73c6 100644 --- a/include/lttng/ringbuffer-config.h +++ b/include/lttng/ringbuffer-config.h @@ -279,6 +279,7 @@ struct lttng_ust_lib_ring_buffer_ctx { * following fields may be used. */ struct lttng_ust_lib_ring_buffer_backend_pages *backend_pages; + struct lttng_rseq_state rseq_state; }; /** @@ -314,6 +315,9 @@ void lib_ring_buffer_ctx_init(struct lttng_ust_lib_ring_buffer_ctx *ctx, ctx->ip = 0; ctx->priv2 = priv2; memset(ctx->padding2, 0, LTTNG_UST_RING_BUFFER_CTX_PADDING); + ctx->rseq_state.rseqp = NULL; + ctx->rseq_state.cpu_id = -1; + ctx->rseq_state.event_counter = 0; } /* diff --git a/liblttng-ust/lttng-ring-buffer-client.h b/liblttng-ust/lttng-ring-buffer-client.h index b397a5e0..5fde855c 100644 --- a/liblttng-ust/lttng-ring-buffer-client.h +++ b/liblttng-ust/lttng-ring-buffer-client.h @@ -743,6 +743,11 @@ retry: } } lttng_write_event_header(&client_config, ctx, event_id); + + if (caa_likely(ctx->ctx_len + >= sizeof(struct lttng_ust_lib_ring_buffer_ctx))) + ctx->rseq_state = rseq_state; + return 0; end: lib_ring_buffer_end(&client_config); diff --git a/libringbuffer/frontend_api.h b/libringbuffer/frontend_api.h index 19fc09e9..6ff98da0 100644 --- a/libringbuffer/frontend_api.h +++ b/libringbuffer/frontend_api.h @@ -33,6 +33,7 @@ #include "frontend.h" #include #include +#include "rseq.h" static inline int lib_ring_buffer_get_cpu(const struct lttng_ust_lib_ring_buffer_config *config) @@ -259,6 +260,14 @@ void lib_ring_buffer_commit(const struct lttng_ust_lib_ring_buffer_config *confi unsigned long commit_count; struct commit_counters_hot *cc_hot = shmp_index(handle, buf->commit_hot, endidx); + struct lttng_rseq_state rseq_state; + + if (caa_likely(ctx->ctx_len + >= sizeof(struct lttng_ust_lib_ring_buffer_ctx))) { + rseq_state = ctx->rseq_state; + } else { + rseq_state.cpu_id = -2; + } if (caa_unlikely(!cc_hot)) return; @@ -274,7 +283,18 @@ void lib_ring_buffer_commit(const struct lttng_ust_lib_ring_buffer_config *confi */ cmm_smp_wmb(); + if (caa_likely(rseq_state.cpu_id >= 0)) { + unsigned long newv; + + newv = cc_hot->cc_rseq + ctx->slot_size; + if (caa_likely(__rseq_finish(NULL, 0, NULL, NULL, 0, + (intptr_t *)&cc_hot->cc_rseq, + (intptr_t) newv, + rseq_state, RSEQ_FINISH_SINGLE, false))) + goto add_done; + } v_add(config, ctx->slot_size, &cc_hot->cc); +add_done: /* * commit count read can race with concurrent OOO commit count updates. @@ -295,6 +315,7 @@ void lib_ring_buffer_commit(const struct lttng_ust_lib_ring_buffer_config *confi * which is completely independent of the order. */ commit_count = v_read(config, &cc_hot->cc); + commit_count += cc_hot->cc_rseq; lib_ring_buffer_check_deliver(config, buf, chan, offset_end - 1, commit_count, endidx, handle, ctx->tsc); diff --git a/libringbuffer/frontend_internal.h b/libringbuffer/frontend_internal.h index 68aa7022..2bb023f8 100644 --- a/libringbuffer/frontend_internal.h +++ b/libringbuffer/frontend_internal.h @@ -253,6 +253,7 @@ int lib_ring_buffer_reserve_committed(const struct lttng_ust_lib_ring_buffer_con offset = v_read(config, &buf->offset); idx = subbuf_index(offset, chan); commit_count = v_read(config, &cc_hot->cc); + commit_count += cc_hot->cc_rseq; } while (offset != v_read(config, &buf->offset)); return ((buf_trunc(offset, chan) >> chan->backend.num_subbuf_order) diff --git a/libringbuffer/frontend_types.h b/libringbuffer/frontend_types.h index bf4b4185..c5c716f7 100644 --- a/libringbuffer/frontend_types.h +++ b/libringbuffer/frontend_types.h @@ -82,6 +82,7 @@ struct channel { #define RB_COMMIT_COUNT_HOT_PADDING 16 struct commit_counters_hot { union v_atomic cc; /* Commit counter */ + unsigned long cc_rseq; /* Commit counter for rseq */ union v_atomic seq; /* Consecutive commits */ char padding[RB_COMMIT_COUNT_HOT_PADDING]; } __attribute__((aligned(CAA_CACHE_LINE_SIZE))); diff --git a/libringbuffer/ring_buffer_frontend.c b/libringbuffer/ring_buffer_frontend.c index be20d69b..46751a2d 100644 --- a/libringbuffer/ring_buffer_frontend.c +++ b/libringbuffer/ring_buffer_frontend.c @@ -185,6 +185,7 @@ void lib_ring_buffer_reset(struct lttng_ust_lib_ring_buffer *buf, if (!cc_cold) return; v_set(config, &cc_hot->cc, 0); + cc_hot->cc_rseq = 0; v_set(config, &cc_hot->seq, 0); v_set(config, &cc_cold->cc_sb, 0); } @@ -1585,6 +1586,7 @@ void lib_ring_buffer_print_subbuffer_errors(struct lttng_ust_lib_ring_buffer *bu if (!cc_cold) return; commit_count = v_read(config, &cc_hot->cc); + commit_count += cc_hot->cc_rseq; commit_count_sb = v_read(config, &cc_cold->cc_sb); if (subbuf_offset(commit_count, chan) != 0) @@ -1697,6 +1699,7 @@ void lib_ring_buffer_switch_old_start(struct lttng_ust_lib_ring_buffer *buf, v_add(config, config->cb.subbuffer_header_size(), &cc_hot->cc); commit_count = v_read(config, &cc_hot->cc); + commit_count += cc_hot->cc_rseq; /* Check if the written buffer has to be delivered */ lib_ring_buffer_check_deliver(config, buf, chan, offsets->old, commit_count, oldidx, handle, tsc); @@ -1740,6 +1743,7 @@ void lib_ring_buffer_switch_old_end(struct lttng_ust_lib_ring_buffer *buf, return; v_add(config, padding_size, &cc_hot->cc); commit_count = v_read(config, &cc_hot->cc); + commit_count += cc_hot->cc_rseq; lib_ring_buffer_check_deliver(config, buf, chan, offsets->old - 1, commit_count, oldidx, handle, tsc); lib_ring_buffer_write_commit_counter(config, buf, chan, @@ -1778,6 +1782,7 @@ void lib_ring_buffer_switch_new_start(struct lttng_ust_lib_ring_buffer *buf, return; v_add(config, config->cb.subbuffer_header_size(), &cc_hot->cc); commit_count = v_read(config, &cc_hot->cc); + commit_count += cc_hot->cc_rseq; /* Check if the written buffer has to be delivered */ lib_ring_buffer_check_deliver(config, buf, chan, offsets->begin, commit_count, beginidx, handle, tsc); -- 2.34.1