rseq fallback: use saturated reference counter

[lttng-ust.git] / liblttng-ust / lttng-ring-buffer-client.h

diff --git a/liblttng-ust/lttng-ring-buffer-client.h b/liblttng-ust/lttng-ring-buffer-client.h
index b397a5e06b176b9e3027ab8095d2ddec54a09f63..8a2155ef3e8501309948c49966fd37b9f41b8b52 100644 (file)
--- a/liblttng-ust/lttng-ring-buffer-client.h
+++ b/liblttng-ust/lttng-ring-buffer-client.h
@@ -637,7 +637,7 @@ static const struct lttng_ust_lib_ring_buffer_config client_config = {
 
        .tsc_bits = LTTNG_COMPACT_TSC_BITS,
        .alloc = RING_BUFFER_ALLOC_PER_CPU,
-       .sync = RING_BUFFER_SYNC_GLOBAL,
+       .sync = RING_BUFFER_SYNC_PER_CPU,
        .mode = RING_BUFFER_MODE_TEMPLATE,
        .backend = RING_BUFFER_PAGE,
        .output = RING_BUFFER_MMAP,
@@ -690,6 +690,28 @@ void lttng_channel_destroy(struct lttng_channel *chan)
        channel_destroy(chan->chan, chan->handle, 1);
 }
 
+static
+bool refcount_get_saturate(long *ref)
+{
+       long old, _new, res;
+
+       old = uatomic_read(ref);
+       for (;;) {
+               if (old == LONG_MAX) {
+                       return false;   /* Saturated. */
+               }
+               _new = old + 1;
+               res = uatomic_cmpxchg(ref, old, _new);
+               if (res == old) {
+                       if (_new == LONG_MAX) {
+                               return false; /* Saturation. */
+                       }
+                       return true;    /* Success. */
+               }
+               old = res;
+       }
+}
+
 static
 int lttng_event_reserve(struct lttng_ust_lib_ring_buffer_ctx *ctx,
                      uint32_t event_id)
@@ -697,6 +719,7 @@ int lttng_event_reserve(struct lttng_ust_lib_ring_buffer_ctx *ctx,
        struct lttng_channel *lttng_chan = channel_get_private(ctx->chan);
        struct lttng_rseq_state rseq_state;
        int ret, cpu;
+       bool put_fallback_ref = false;
 
        if (lib_ring_buffer_begin(&client_config))
                return -EPERM;
@@ -716,6 +739,7 @@ retry:
        } else {
                cpu = rseq_cpu_at_start(rseq_state);
        }
+fallback:
        ctx->cpu = cpu;
 
        switch (lttng_chan->header_type) {
@@ -731,9 +755,25 @@ retry:
                WARN_ON_ONCE(1);
        }
 
+       if (caa_likely(ctx->ctx_len
+                       >= sizeof(struct lttng_ust_lib_ring_buffer_ctx)))
+               ctx->rseq_state = rseq_state;
+
        ret = lib_ring_buffer_reserve(&client_config, ctx);
-       if (caa_unlikely(ret))
-               goto put;
+       if (caa_unlikely(ret)) {
+               if (ret == -EAGAIN) {
+                       assert(!put_fallback_ref);
+                       put_fallback_ref = refcount_get_saturate(
+                               &lttng_chan->chan->u.reserve_fallback_ref);
+                       cpu = lib_ring_buffer_get_cpu(&client_config);
+                       if (caa_unlikely(cpu < 0)) {
+                               ret = -EPERM;
+                               goto end;
+                       }
+                       goto fallback;
+               }
+               goto end;
+       }
        if (caa_likely(ctx->ctx_len
                        >= sizeof(struct lttng_ust_lib_ring_buffer_ctx))) {
                if (lib_ring_buffer_backend_get_pages(&client_config, ctx,
@@ -743,9 +783,15 @@ retry:
                }
        }
        lttng_write_event_header(&client_config, ctx, event_id);
+
+       if (caa_unlikely(put_fallback_ref))
+               uatomic_dec(&lttng_chan->chan->u.reserve_fallback_ref);
+
        return 0;
 end:
        lib_ring_buffer_end(&client_config);
+       if (put_fallback_ref)
+               uatomic_dec(&lttng_chan->chan->u.reserve_fallback_ref);
        return ret;
 }