perf counters: consolidate hw_perf save/restore APIs

[deliverable/linux.git] / kernel / perf_counter.c

diff --git a/kernel/perf_counter.c b/kernel/perf_counter.c
index 278209c547a870dd12a43997dc3c5915ca084665..0e93fea17120c0a2b98da8dca8be4eba9c9ee7dc 100644 (file)
--- a/kernel/perf_counter.c
+++ b/kernel/perf_counter.c
@@ -37,18 +37,15 @@ static DEFINE_MUTEX(perf_resource_mutex);
 /*
  * Architecture provided APIs - weak aliases:
  */
-
-int __weak hw_perf_counter_init(struct perf_counter *counter)
+extern __weak const struct hw_perf_counter_ops *
+hw_perf_counter_init(struct perf_counter *counter)
 {
-       return -EINVAL;
+       return ERR_PTR(-EINVAL);
 }
 
-void __weak hw_perf_counter_enable(struct perf_counter *counter)        { }
-void __weak hw_perf_counter_disable(struct perf_counter *counter)       { }
-void __weak hw_perf_counter_read(struct perf_counter *counter)          { }
-void __weak hw_perf_disable_all(void) { }
-void __weak hw_perf_enable_all(void) { }
-void __weak hw_perf_counter_setup(void) { }
+u64 __weak hw_perf_save_disable(void)          { return 0; }
+void __weak hw_perf_restore(u64 ctrl)  { }
+void __weak hw_perf_counter_setup(void)                { }
 
 #if BITS_PER_LONG == 64
 
@@ -61,6 +58,16 @@ static inline u64 perf_counter_read_safe(struct perf_counter *counter)
        return (u64) atomic64_read(&counter->count);
 }
 
+void atomic64_counter_set(struct perf_counter *counter, u64 val)
+{
+       atomic64_set(&counter->count, val);
+}
+
+u64 atomic64_counter_read(struct perf_counter *counter)
+{
+       return atomic64_read(&counter->count);
+}
+
 #else
 
 /*
@@ -82,6 +89,20 @@ static u64 perf_counter_read_safe(struct perf_counter *counter)
        return cntl | ((u64) cnth) << 32;
 }
 
+void atomic64_counter_set(struct perf_counter *counter, u64 val64)
+{
+       u32 *val32 = (void *)&val64;
+
+       atomic_set(counter->count32 + 0, *(val32 + 0));
+       atomic_set(counter->count32 + 1, *(val32 + 1));
+}
+
+u64 atomic64_counter_read(struct perf_counter *counter)
+{
+       return atomic_read(counter->count32 + 0) |
+               (u64) atomic_read(counter->count32 + 1) << 32;
+}
+
 #endif
 
 static void
@@ -134,6 +155,7 @@ static void __perf_counter_remove_from_context(void *info)
        struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
        struct perf_counter *counter = info;
        struct perf_counter_context *ctx = counter->ctx;
+       u64 perf_flags;
 
        /*
         * If this is a task context, we need to check whether it is
@@ -146,7 +168,7 @@ static void __perf_counter_remove_from_context(void *info)
        spin_lock(&ctx->lock);
 
        if (counter->active) {
-               hw_perf_counter_disable(counter);
+               counter->hw_ops->hw_perf_counter_disable(counter);
                counter->active = 0;
                ctx->nr_active--;
                cpuctx->active_oncpu--;
@@ -158,9 +180,9 @@ static void __perf_counter_remove_from_context(void *info)
         * Protect the list operation against NMI by disabling the
         * counters on a global level. NOP for non NMI based counters.
         */
-       hw_perf_disable_all();
+       perf_flags = hw_perf_save_disable();
        list_del_counter(counter, ctx);
-       hw_perf_enable_all();
+       hw_perf_restore(perf_flags);
 
        if (!ctx->task) {
                /*
@@ -235,6 +257,7 @@ static void __perf_install_in_context(void *info)
        struct perf_counter *counter = info;
        struct perf_counter_context *ctx = counter->ctx;
        int cpu = smp_processor_id();
+       u64 perf_flags;
 
        /*
         * If this is a task context, we need to check whether it is
@@ -250,14 +273,14 @@ static void __perf_install_in_context(void *info)
         * Protect the list operation against NMI by disabling the
         * counters on a global level. NOP for non NMI based counters.
         */
-       hw_perf_disable_all();
+       perf_flags = hw_perf_save_disable();
        list_add_counter(counter, ctx);
-       hw_perf_enable_all();
+       hw_perf_restore(perf_flags);
 
        ctx->nr_counters++;
 
        if (cpuctx->active_oncpu < perf_max_counters) {
-               hw_perf_counter_enable(counter);
+               counter->hw_ops->hw_perf_counter_enable(counter);
                counter->active = 1;
                counter->oncpu = cpu;
                ctx->nr_active++;
@@ -333,7 +356,7 @@ counter_sched_out(struct perf_counter *counter,
        if (!counter->active)
                return;
 
-       hw_perf_counter_disable(counter);
+       counter->hw_ops->hw_perf_counter_disable(counter);
        counter->active =  0;
        counter->oncpu  = -1;
 
@@ -392,7 +415,7 @@ counter_sched_in(struct perf_counter *counter,
                 struct perf_counter_context *ctx,
                 int cpu)
 {
-       hw_perf_counter_enable(counter);
+       counter->hw_ops->hw_perf_counter_enable(counter);
        counter->active = 1;
        counter->oncpu = cpu;   /* TODO: put 'cpu' into cpuctx->cpu */
 
@@ -460,6 +483,7 @@ void perf_counter_task_tick(struct task_struct *curr, int cpu)
 {
        struct perf_counter_context *ctx = &curr->perf_counter_ctx;
        struct perf_counter *counter;
+       u64 perf_flags;
 
        if (likely(!ctx->nr_counters))
                return;
@@ -471,13 +495,13 @@ void perf_counter_task_tick(struct task_struct *curr, int cpu)
        /*
         * Rotate the first entry last (works just fine for group counters too):
         */
-       hw_perf_disable_all();
+       perf_flags = hw_perf_save_disable();
        list_for_each_entry(counter, &ctx->counter_list, list_entry) {
                list_del(&counter->list_entry);
                list_add_tail(&counter->list_entry, &ctx->counter_list);
                break;
        }
-       hw_perf_enable_all();
+       hw_perf_restore(perf_flags);
 
        spin_unlock(&ctx->lock);
 
@@ -509,7 +533,9 @@ void perf_counter_init_task(struct task_struct *task)
  */
 static void __hw_perf_counter_read(void *info)
 {
-       hw_perf_counter_read(info);
+       struct perf_counter *counter = info;
+
+       counter->hw_ops->hw_perf_counter_read(counter);
 }
 
 static u64 perf_counter_read(struct perf_counter *counter)
@@ -808,6 +834,42 @@ static const struct file_operations perf_fops = {
        .poll                   = perf_poll,
 };
 
+static void cpu_clock_perf_counter_enable(struct perf_counter *counter)
+{
+}
+
+static void cpu_clock_perf_counter_disable(struct perf_counter *counter)
+{
+}
+
+static void cpu_clock_perf_counter_read(struct perf_counter *counter)
+{
+       int cpu = raw_smp_processor_id();
+
+       atomic64_counter_set(counter, cpu_clock(cpu));
+}
+
+static const struct hw_perf_counter_ops perf_ops_cpu_clock = {
+       .hw_perf_counter_enable         = cpu_clock_perf_counter_enable,
+       .hw_perf_counter_disable        = cpu_clock_perf_counter_disable,
+       .hw_perf_counter_read           = cpu_clock_perf_counter_read,
+};
+
+static const struct hw_perf_counter_ops *
+sw_perf_counter_init(struct perf_counter *counter)
+{
+       const struct hw_perf_counter_ops *hw_ops = NULL;
+
+       switch (counter->hw_event.type) {
+       case PERF_COUNT_CPU_CLOCK:
+               hw_ops = &perf_ops_cpu_clock;
+               break;
+       default:
+               break;
+       }
+       return hw_ops;
+}
+
 /*
  * Allocate and initialize a counter structure
  */
@@ -816,8 +878,10 @@ perf_counter_alloc(struct perf_counter_hw_event *hw_event,
                   int cpu,
                   struct perf_counter *group_leader)
 {
-       struct perf_counter *counter = kzalloc(sizeof(*counter), GFP_KERNEL);
+       const struct hw_perf_counter_ops *hw_ops;
+       struct perf_counter *counter;
 
+       counter = kzalloc(sizeof(*counter), GFP_KERNEL);
        if (!counter)
                return NULL;
 
@@ -839,6 +903,20 @@ perf_counter_alloc(struct perf_counter_hw_event *hw_event,
        counter->hw_event               = *hw_event;
        counter->wakeup_pending         = 0;
        counter->group_leader           = group_leader;
+       counter->hw_ops                 = NULL;
+
+       hw_ops = NULL;
+       if (!hw_event->raw && hw_event->type < 0)
+               hw_ops = sw_perf_counter_init(counter);
+       if (!hw_ops) {
+               hw_ops = hw_perf_counter_init(counter);
+       }
+
+       if (!hw_ops) {
+               kfree(counter);
+               return NULL;
+       }
+       counter->hw_ops = hw_ops;
 
        return counter;
 }
@@ -903,15 +981,11 @@ asmlinkage int sys_perf_counter_open(
                        goto err_put_context;
        }
 
-       ret = -ENOMEM;
+       ret = -EINVAL;
        counter = perf_counter_alloc(&hw_event, cpu, group_leader);
        if (!counter)
                goto err_put_context;
 
-       ret = hw_perf_counter_init(counter);
-       if (ret)
-               goto err_free_put_context;
-
        perf_install_in_context(ctx, counter, cpu);
 
        ret = anon_inode_getfd("[perf_counter]", &perf_fops, counter, 0);
@@ -927,8 +1001,6 @@ err_remove_free_put_context:
        mutex_lock(&counter->mutex);
        perf_counter_remove_from_context(counter);
        mutex_unlock(&counter->mutex);
-
-err_free_put_context:
        kfree(counter);
 
 err_put_context: