perf/core: Collapse common IPI pattern

Various functions implement the same pattern to send IPIs to an
event's CPU. Collapse the easy ones in a common helper function to
reduce duplication.

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Arnaldo Carvalho de Melo <acme@redhat.com>
Cc: Jiri Olsa <jolsa@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Stephane Eranian <eranian@google.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Vince Weaver <vincent.weaver@maine.edu>
Signed-off-by: Ingo Molnar <mingo@kernel.org>

diff --git a/kernel/events/core.c b/kernel/events/core.c
index 39cf4a40aa4c0a70d72c541cfc2825ba71d41034..c3d61b92d8056b078261e9e857a47d0256553ac9 100644 (file)
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -126,6 +126,37 @@ static int cpu_function_call(int cpu, remote_function_f func, void *info)
        return data.ret;
 }
 
+static void event_function_call(struct perf_event *event,
+                               int (*active)(void *),
+                               void (*inactive)(void *),
+                               void *data)
+{
+       struct perf_event_context *ctx = event->ctx;
+       struct task_struct *task = ctx->task;
+
+       if (!task) {
+               cpu_function_call(event->cpu, active, data);
+               return;
+       }
+
+again:
+       if (!task_function_call(task, active, data))
+               return;
+
+       raw_spin_lock_irq(&ctx->lock);
+       if (ctx->is_active) {
+               /*
+                * Reload the task pointer, it might have been changed by
+                * a concurrent perf_event_context_sched_out().
+                */
+               task = ctx->task;
+               raw_spin_unlock_irq(&ctx->lock);
+               goto again;
+       }
+       inactive(data);
+       raw_spin_unlock_irq(&ctx->lock);
+}
+
 #define EVENT_OWNER_KERNEL ((void *) -1)
 
 static bool is_kernel_event(struct perf_event *event)
@@ -1629,6 +1660,17 @@ struct remove_event {
        bool detach_group;
 };
 
+static void ___perf_remove_from_context(void *info)
+{
+       struct remove_event *re = info;
+       struct perf_event *event = re->event;
+       struct perf_event_context *ctx = event->ctx;
+
+       if (re->detach_group)
+               perf_group_detach(event);
+       list_del_event(event, ctx);
+}
+
 /*
  * Cross CPU call to remove a performance event
  *
@@ -1656,7 +1698,6 @@ static int __perf_remove_from_context(void *info)
        return 0;
 }
 
-
 /*
  * Remove the event from a task's (or a CPU's) list of events.
  *
@@ -1673,7 +1714,6 @@ static int __perf_remove_from_context(void *info)
 static void perf_remove_from_context(struct perf_event *event, bool detach_group)
 {
        struct perf_event_context *ctx = event->ctx;
-       struct task_struct *task = ctx->task;
        struct remove_event re = {
                .event = event,
                .detach_group = detach_group,
@@ -1681,44 +1721,8 @@ static void perf_remove_from_context(struct perf_event *event, bool detach_group
 
        lockdep_assert_held(&ctx->mutex);
 
-       if (!task) {
-               /*
-                * Per cpu events are removed via an smp call. The removal can
-                * fail if the CPU is currently offline, but in that case we
-                * already called __perf_remove_from_context from
-                * perf_event_exit_cpu.
-                */
-               cpu_function_call(event->cpu, __perf_remove_from_context, &re);
-               return;
-       }
-
-retry:
-       if (!task_function_call(task, __perf_remove_from_context, &re))
-               return;
-
-       raw_spin_lock_irq(&ctx->lock);
-       /*
-        * If we failed to find a running task, but find the context active now
-        * that we've acquired the ctx->lock, retry.
-        */
-       if (ctx->is_active) {
-               raw_spin_unlock_irq(&ctx->lock);
-               /*
-                * Reload the task pointer, it might have been changed by
-                * a concurrent perf_event_context_sched_out().
-                */
-               task = ctx->task;
-               goto retry;
-       }
-
-       /*
-        * Since the task isn't running, its safe to remove the event, us
-        * holding the ctx->lock ensures the task won't get scheduled in.
-        */
-       if (detach_group)
-               perf_group_detach(event);
-       list_del_event(event, ctx);
-       raw_spin_unlock_irq(&ctx->lock);
+       event_function_call(event, __perf_remove_from_context,
+                           ___perf_remove_from_context, &re);
 }
 
 /*
@@ -2067,6 +2071,18 @@ static void perf_event_sched_in(struct perf_cpu_context *cpuctx,
                ctx_sched_in(ctx, cpuctx, EVENT_FLEXIBLE, task);
 }
 
+static void ___perf_install_in_context(void *info)
+{
+       struct perf_event *event = info;
+       struct perf_event_context *ctx = event->ctx;
+
+       /*
+        * Since the task isn't running, its safe to add the event, us holding
+        * the ctx->lock ensures the task won't get scheduled in.
+        */
+       add_event_to_ctx(event, ctx);
+}
+
 /*
  * Cross CPU call to install and enable a performance event
  *
@@ -2143,48 +2159,14 @@ perf_install_in_context(struct perf_event_context *ctx,
                        struct perf_event *event,
                        int cpu)
 {
-       struct task_struct *task = ctx->task;
-
        lockdep_assert_held(&ctx->mutex);
 
        event->ctx = ctx;
        if (event->cpu != -1)
                event->cpu = cpu;
 
-       if (!task) {
-               /*
-                * Per cpu events are installed via an smp call and
-                * the install is always successful.
-                */
-               cpu_function_call(cpu, __perf_install_in_context, event);
-               return;
-       }
-
-retry:
-       if (!task_function_call(task, __perf_install_in_context, event))
-               return;
-
-       raw_spin_lock_irq(&ctx->lock);
-       /*
-        * If we failed to find a running task, but find the context active now
-        * that we've acquired the ctx->lock, retry.
-        */
-       if (ctx->is_active) {
-               raw_spin_unlock_irq(&ctx->lock);
-               /*
-                * Reload the task pointer, it might have been changed by
-                * a concurrent perf_event_context_sched_out().
-                */
-               task = ctx->task;
-               goto retry;
-       }
-
-       /*
-        * Since the task isn't running, its safe to add the event, us holding
-        * the ctx->lock ensures the task won't get scheduled in.
-        */
-       add_event_to_ctx(event, ctx);
-       raw_spin_unlock_irq(&ctx->lock);
+       event_function_call(event, __perf_install_in_context,
+                           ___perf_install_in_context, event);
 }
 
 /*
@@ -4154,6 +4136,22 @@ struct period_event {
        u64 value;
 };
 
+static void ___perf_event_period(void *info)
+{
+       struct period_event *pe = info;
+       struct perf_event *event = pe->event;
+       u64 value = pe->value;
+
+       if (event->attr.freq) {
+               event->attr.sample_freq = value;
+       } else {
+               event->attr.sample_period = value;
+               event->hw.sample_period = value;
+       }
+
+       local64_set(&event->hw.period_left, 0);
+}
+
 static int __perf_event_period(void *info)
 {
        struct period_event *pe = info;
@@ -4190,8 +4188,6 @@ static int __perf_event_period(void *info)
 static int perf_event_period(struct perf_event *event, u64 __user *arg)
 {
        struct period_event pe = { .event = event, };
-       struct perf_event_context *ctx = event->ctx;
-       struct task_struct *task;
        u64 value;
 
        if (!is_sampling_event(event))
@@ -4206,34 +4202,10 @@ static int perf_event_period(struct perf_event *event, u64 __user *arg)
        if (event->attr.freq && value > sysctl_perf_event_sample_rate)
                return -EINVAL;
 
-       task = ctx->task;
        pe.value = value;
 
-       if (!task) {
-               cpu_function_call(event->cpu, __perf_event_period, &pe);
-               return 0;
-       }
-
-retry:
-       if (!task_function_call(task, __perf_event_period, &pe))
-               return 0;
-
-       raw_spin_lock_irq(&ctx->lock);
-       if (ctx->is_active) {
-               raw_spin_unlock_irq(&ctx->lock);
-               task = ctx->task;
-               goto retry;
-       }
-
-       if (event->attr.freq) {
-               event->attr.sample_freq = value;
-       } else {
-               event->attr.sample_period = value;
-               event->hw.sample_period = value;
-       }
-
-       local64_set(&event->hw.period_left, 0);
-       raw_spin_unlock_irq(&ctx->lock);
+       event_function_call(event, __perf_event_period,
+                           ___perf_event_period, &pe);
 
        return 0;
 }