Merge tag 'asoc-v4.7-2' of git://git.kernel.org/pub/scm/linux/kernel/git/broonie...
[deliverable/linux.git] / arch / x86 / events / core.c
CommitLineData
241771ef 1/*
cdd6c482 2 * Performance events x86 architecture code
241771ef 3 *
98144511
IM
4 * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
5 * Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
6 * Copyright (C) 2009 Jaswinder Singh Rajput
7 * Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter
90eec103 8 * Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra
30dd568c 9 * Copyright (C) 2009 Intel Corporation, <markus.t.metzger@intel.com>
1da53e02 10 * Copyright (C) 2009 Google, Inc., Stephane Eranian
241771ef
IM
11 *
12 * For licencing details see kernel-base/COPYING
13 */
14
cdd6c482 15#include <linux/perf_event.h>
241771ef
IM
16#include <linux/capability.h>
17#include <linux/notifier.h>
18#include <linux/hardirq.h>
19#include <linux/kprobes.h>
4ac13294 20#include <linux/module.h>
241771ef
IM
21#include <linux/kdebug.h>
22#include <linux/sched.h>
d7d59fb3 23#include <linux/uaccess.h>
5a0e3ad6 24#include <linux/slab.h>
30dd568c 25#include <linux/cpu.h>
272d30be 26#include <linux/bitops.h>
0c9d42ed 27#include <linux/device.h>
241771ef 28
241771ef 29#include <asm/apic.h>
d7d59fb3 30#include <asm/stacktrace.h>
4e935e47 31#include <asm/nmi.h>
69092624 32#include <asm/smp.h>
c8e5910e 33#include <asm/alternative.h>
7911d3f7 34#include <asm/mmu_context.h>
375074cc 35#include <asm/tlbflush.h>
e3f3541c 36#include <asm/timer.h>
d07bdfd3
PZ
37#include <asm/desc.h>
38#include <asm/ldt.h>
241771ef 39
27f6d22b 40#include "perf_event.h"
de0428a7 41
de0428a7 42struct x86_pmu x86_pmu __read_mostly;
efc9f05d 43
de0428a7 44DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = {
b0f3f28e
PZ
45 .enabled = 1,
46};
241771ef 47
a6673429
AL
48struct static_key rdpmc_always_available = STATIC_KEY_INIT_FALSE;
49
de0428a7 50u64 __read_mostly hw_cache_event_ids
8326f44d
IM
51 [PERF_COUNT_HW_CACHE_MAX]
52 [PERF_COUNT_HW_CACHE_OP_MAX]
53 [PERF_COUNT_HW_CACHE_RESULT_MAX];
de0428a7 54u64 __read_mostly hw_cache_extra_regs
e994d7d2
AK
55 [PERF_COUNT_HW_CACHE_MAX]
56 [PERF_COUNT_HW_CACHE_OP_MAX]
57 [PERF_COUNT_HW_CACHE_RESULT_MAX];
8326f44d 58
ee06094f 59/*
cdd6c482
IM
60 * Propagate event elapsed time into the generic event.
61 * Can only be executed on the CPU where the event is active.
ee06094f
IM
62 * Returns the delta events processed.
63 */
de0428a7 64u64 x86_perf_event_update(struct perf_event *event)
ee06094f 65{
cc2ad4ba 66 struct hw_perf_event *hwc = &event->hw;
948b1bb8 67 int shift = 64 - x86_pmu.cntval_bits;
ec3232bd 68 u64 prev_raw_count, new_raw_count;
cc2ad4ba 69 int idx = hwc->idx;
ec3232bd 70 s64 delta;
ee06094f 71
15c7ad51 72 if (idx == INTEL_PMC_IDX_FIXED_BTS)
30dd568c
MM
73 return 0;
74
ee06094f 75 /*
cdd6c482 76 * Careful: an NMI might modify the previous event value.
ee06094f
IM
77 *
78 * Our tactic to handle this is to first atomically read and
79 * exchange a new raw count - then add that new-prev delta
cdd6c482 80 * count to the generic event atomically:
ee06094f
IM
81 */
82again:
e7850595 83 prev_raw_count = local64_read(&hwc->prev_count);
c48b6053 84 rdpmcl(hwc->event_base_rdpmc, new_raw_count);
ee06094f 85
e7850595 86 if (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
ee06094f
IM
87 new_raw_count) != prev_raw_count)
88 goto again;
89
90 /*
91 * Now we have the new raw value and have updated the prev
92 * timestamp already. We can now calculate the elapsed delta
cdd6c482 93 * (event-)time and add that to the generic event.
ee06094f
IM
94 *
95 * Careful, not all hw sign-extends above the physical width
ec3232bd 96 * of the count.
ee06094f 97 */
ec3232bd
PZ
98 delta = (new_raw_count << shift) - (prev_raw_count << shift);
99 delta >>= shift;
ee06094f 100
e7850595
PZ
101 local64_add(delta, &event->count);
102 local64_sub(delta, &hwc->period_left);
4b7bfd0d
RR
103
104 return new_raw_count;
ee06094f
IM
105}
106
a7e3ed1e
AK
107/*
108 * Find and validate any extra registers to set up.
109 */
110static int x86_pmu_extra_regs(u64 config, struct perf_event *event)
111{
efc9f05d 112 struct hw_perf_event_extra *reg;
a7e3ed1e
AK
113 struct extra_reg *er;
114
efc9f05d 115 reg = &event->hw.extra_reg;
a7e3ed1e
AK
116
117 if (!x86_pmu.extra_regs)
118 return 0;
119
120 for (er = x86_pmu.extra_regs; er->msr; er++) {
121 if (er->event != (config & er->config_mask))
122 continue;
123 if (event->attr.config1 & ~er->valid_mask)
124 return -EINVAL;
338b522c
KL
125 /* Check if the extra msrs can be safely accessed*/
126 if (!er->extra_msr_access)
127 return -ENXIO;
efc9f05d
SE
128
129 reg->idx = er->idx;
130 reg->config = event->attr.config1;
131 reg->reg = er->msr;
a7e3ed1e
AK
132 break;
133 }
134 return 0;
135}
136
cdd6c482 137static atomic_t active_events;
1b7b938f 138static atomic_t pmc_refcount;
4e935e47
PZ
139static DEFINE_MUTEX(pmc_reserve_mutex);
140
b27ea29c
RR
141#ifdef CONFIG_X86_LOCAL_APIC
142
4e935e47
PZ
143static bool reserve_pmc_hardware(void)
144{
145 int i;
146
948b1bb8 147 for (i = 0; i < x86_pmu.num_counters; i++) {
41bf4989 148 if (!reserve_perfctr_nmi(x86_pmu_event_addr(i)))
4e935e47
PZ
149 goto perfctr_fail;
150 }
151
948b1bb8 152 for (i = 0; i < x86_pmu.num_counters; i++) {
41bf4989 153 if (!reserve_evntsel_nmi(x86_pmu_config_addr(i)))
4e935e47
PZ
154 goto eventsel_fail;
155 }
156
157 return true;
158
159eventsel_fail:
160 for (i--; i >= 0; i--)
41bf4989 161 release_evntsel_nmi(x86_pmu_config_addr(i));
4e935e47 162
948b1bb8 163 i = x86_pmu.num_counters;
4e935e47
PZ
164
165perfctr_fail:
166 for (i--; i >= 0; i--)
41bf4989 167 release_perfctr_nmi(x86_pmu_event_addr(i));
4e935e47 168
4e935e47
PZ
169 return false;
170}
171
172static void release_pmc_hardware(void)
173{
174 int i;
175
948b1bb8 176 for (i = 0; i < x86_pmu.num_counters; i++) {
41bf4989
RR
177 release_perfctr_nmi(x86_pmu_event_addr(i));
178 release_evntsel_nmi(x86_pmu_config_addr(i));
4e935e47 179 }
4e935e47
PZ
180}
181
b27ea29c
RR
182#else
183
184static bool reserve_pmc_hardware(void) { return true; }
185static void release_pmc_hardware(void) {}
186
187#endif
188
33c6d6a7
DZ
189static bool check_hw_exists(void)
190{
a5ebe0ba
GD
191 u64 val, val_fail, val_new= ~0;
192 int i, reg, reg_fail, ret = 0;
193 int bios_fail = 0;
68ab7476 194 int reg_safe = -1;
33c6d6a7 195
4407204c
PZ
196 /*
197 * Check to see if the BIOS enabled any of the counters, if so
198 * complain and bail.
199 */
200 for (i = 0; i < x86_pmu.num_counters; i++) {
41bf4989 201 reg = x86_pmu_config_addr(i);
4407204c
PZ
202 ret = rdmsrl_safe(reg, &val);
203 if (ret)
204 goto msr_fail;
a5ebe0ba
GD
205 if (val & ARCH_PERFMON_EVENTSEL_ENABLE) {
206 bios_fail = 1;
207 val_fail = val;
208 reg_fail = reg;
68ab7476
DZ
209 } else {
210 reg_safe = i;
a5ebe0ba 211 }
4407204c
PZ
212 }
213
214 if (x86_pmu.num_counters_fixed) {
215 reg = MSR_ARCH_PERFMON_FIXED_CTR_CTRL;
216 ret = rdmsrl_safe(reg, &val);
217 if (ret)
218 goto msr_fail;
219 for (i = 0; i < x86_pmu.num_counters_fixed; i++) {
a5ebe0ba
GD
220 if (val & (0x03 << i*4)) {
221 bios_fail = 1;
222 val_fail = val;
223 reg_fail = reg;
224 }
4407204c
PZ
225 }
226 }
227
68ab7476
DZ
228 /*
229 * If all the counters are enabled, the below test will always
230 * fail. The tools will also become useless in this scenario.
231 * Just fail and disable the hardware counters.
232 */
233
234 if (reg_safe == -1) {
235 reg = reg_safe;
236 goto msr_fail;
237 }
238
4407204c 239 /*
bffd5fc2
AP
240 * Read the current value, change it and read it back to see if it
241 * matches, this is needed to detect certain hardware emulators
242 * (qemu/kvm) that don't trap on the MSR access and always return 0s.
4407204c 243 */
68ab7476 244 reg = x86_pmu_event_addr(reg_safe);
bffd5fc2
AP
245 if (rdmsrl_safe(reg, &val))
246 goto msr_fail;
247 val ^= 0xffffUL;
f285f92f
RR
248 ret = wrmsrl_safe(reg, val);
249 ret |= rdmsrl_safe(reg, &val_new);
33c6d6a7 250 if (ret || val != val_new)
4407204c 251 goto msr_fail;
33c6d6a7 252
45daae57
IM
253 /*
254 * We still allow the PMU driver to operate:
255 */
a5ebe0ba 256 if (bios_fail) {
1b74dde7
CY
257 pr_cont("Broken BIOS detected, complain to your hardware vendor.\n");
258 pr_err(FW_BUG "the BIOS has corrupted hw-PMU resources (MSR %x is %Lx)\n",
259 reg_fail, val_fail);
a5ebe0ba 260 }
45daae57
IM
261
262 return true;
4407204c
PZ
263
264msr_fail:
1b74dde7
CY
265 pr_cont("Broken PMU hardware detected, using software events only.\n");
266 pr_info("%sFailed to access perfctr msr (MSR %x is %Lx)\n",
65d71fe1
PZI
267 boot_cpu_has(X86_FEATURE_HYPERVISOR) ? KERN_INFO : KERN_ERR,
268 reg, val_new);
45daae57 269
4407204c 270 return false;
33c6d6a7
DZ
271}
272
cdd6c482 273static void hw_perf_event_destroy(struct perf_event *event)
4e935e47 274{
6b099d9b 275 x86_release_hardware();
1b7b938f 276 atomic_dec(&active_events);
4e935e47
PZ
277}
278
48070342
AS
279void hw_perf_lbr_event_destroy(struct perf_event *event)
280{
281 hw_perf_event_destroy(event);
282
283 /* undo the lbr/bts event accounting */
284 x86_del_exclusive(x86_lbr_exclusive_lbr);
285}
286
85cf9dba
RR
287static inline int x86_pmu_initialized(void)
288{
289 return x86_pmu.handle_irq != NULL;
290}
291
8326f44d 292static inline int
e994d7d2 293set_ext_hw_attr(struct hw_perf_event *hwc, struct perf_event *event)
8326f44d 294{
e994d7d2 295 struct perf_event_attr *attr = &event->attr;
8326f44d
IM
296 unsigned int cache_type, cache_op, cache_result;
297 u64 config, val;
298
299 config = attr->config;
300
301 cache_type = (config >> 0) & 0xff;
302 if (cache_type >= PERF_COUNT_HW_CACHE_MAX)
303 return -EINVAL;
304
305 cache_op = (config >> 8) & 0xff;
306 if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX)
307 return -EINVAL;
308
309 cache_result = (config >> 16) & 0xff;
310 if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
311 return -EINVAL;
312
313 val = hw_cache_event_ids[cache_type][cache_op][cache_result];
314
315 if (val == 0)
316 return -ENOENT;
317
318 if (val == -1)
319 return -EINVAL;
320
321 hwc->config |= val;
e994d7d2
AK
322 attr->config1 = hw_cache_extra_regs[cache_type][cache_op][cache_result];
323 return x86_pmu_extra_regs(val, event);
8326f44d
IM
324}
325
6b099d9b
AS
326int x86_reserve_hardware(void)
327{
328 int err = 0;
329
1b7b938f 330 if (!atomic_inc_not_zero(&pmc_refcount)) {
6b099d9b 331 mutex_lock(&pmc_reserve_mutex);
1b7b938f 332 if (atomic_read(&pmc_refcount) == 0) {
6b099d9b
AS
333 if (!reserve_pmc_hardware())
334 err = -EBUSY;
335 else
336 reserve_ds_buffers();
337 }
338 if (!err)
1b7b938f 339 atomic_inc(&pmc_refcount);
6b099d9b
AS
340 mutex_unlock(&pmc_reserve_mutex);
341 }
342
343 return err;
344}
345
346void x86_release_hardware(void)
347{
1b7b938f 348 if (atomic_dec_and_mutex_lock(&pmc_refcount, &pmc_reserve_mutex)) {
6b099d9b
AS
349 release_pmc_hardware();
350 release_ds_buffers();
351 mutex_unlock(&pmc_reserve_mutex);
352 }
353}
354
48070342
AS
355/*
356 * Check if we can create event of a certain type (that no conflicting events
357 * are present).
358 */
359int x86_add_exclusive(unsigned int what)
360{
93472aff 361 int i;
48070342 362
93472aff
PZ
363 if (!atomic_inc_not_zero(&x86_pmu.lbr_exclusive[what])) {
364 mutex_lock(&pmc_reserve_mutex);
365 for (i = 0; i < ARRAY_SIZE(x86_pmu.lbr_exclusive); i++) {
366 if (i != what && atomic_read(&x86_pmu.lbr_exclusive[i]))
367 goto fail_unlock;
368 }
369 atomic_inc(&x86_pmu.lbr_exclusive[what]);
370 mutex_unlock(&pmc_reserve_mutex);
6b099d9b 371 }
48070342 372
93472aff
PZ
373 atomic_inc(&active_events);
374 return 0;
48070342 375
93472aff 376fail_unlock:
48070342 377 mutex_unlock(&pmc_reserve_mutex);
93472aff 378 return -EBUSY;
48070342
AS
379}
380
381void x86_del_exclusive(unsigned int what)
382{
383 atomic_dec(&x86_pmu.lbr_exclusive[what]);
1b7b938f 384 atomic_dec(&active_events);
48070342
AS
385}
386
de0428a7 387int x86_setup_perfctr(struct perf_event *event)
c1726f34
RR
388{
389 struct perf_event_attr *attr = &event->attr;
390 struct hw_perf_event *hwc = &event->hw;
391 u64 config;
392
6c7e550f 393 if (!is_sampling_event(event)) {
c1726f34
RR
394 hwc->sample_period = x86_pmu.max_period;
395 hwc->last_period = hwc->sample_period;
e7850595 396 local64_set(&hwc->period_left, hwc->sample_period);
c1726f34
RR
397 }
398
399 if (attr->type == PERF_TYPE_RAW)
ed13ec58 400 return x86_pmu_extra_regs(event->attr.config, event);
c1726f34
RR
401
402 if (attr->type == PERF_TYPE_HW_CACHE)
e994d7d2 403 return set_ext_hw_attr(hwc, event);
c1726f34
RR
404
405 if (attr->config >= x86_pmu.max_events)
406 return -EINVAL;
407
408 /*
409 * The generic map:
410 */
411 config = x86_pmu.event_map(attr->config);
412
413 if (config == 0)
414 return -ENOENT;
415
416 if (config == -1LL)
417 return -EINVAL;
418
419 /*
420 * Branch tracing:
421 */
18a073a3
PZ
422 if (attr->config == PERF_COUNT_HW_BRANCH_INSTRUCTIONS &&
423 !attr->freq && hwc->sample_period == 1) {
c1726f34 424 /* BTS is not supported by this architecture. */
6809b6ea 425 if (!x86_pmu.bts_active)
c1726f34
RR
426 return -EOPNOTSUPP;
427
428 /* BTS is currently only allowed for user-mode. */
429 if (!attr->exclude_kernel)
430 return -EOPNOTSUPP;
48070342
AS
431
432 /* disallow bts if conflicting events are present */
433 if (x86_add_exclusive(x86_lbr_exclusive_lbr))
434 return -EBUSY;
435
436 event->destroy = hw_perf_lbr_event_destroy;
c1726f34
RR
437 }
438
439 hwc->config |= config;
440
441 return 0;
442}
4261e0e0 443
ff3fb511
SE
444/*
445 * check that branch_sample_type is compatible with
446 * settings needed for precise_ip > 1 which implies
447 * using the LBR to capture ALL taken branches at the
448 * priv levels of the measurement
449 */
450static inline int precise_br_compat(struct perf_event *event)
451{
452 u64 m = event->attr.branch_sample_type;
453 u64 b = 0;
454
455 /* must capture all branches */
456 if (!(m & PERF_SAMPLE_BRANCH_ANY))
457 return 0;
458
459 m &= PERF_SAMPLE_BRANCH_KERNEL | PERF_SAMPLE_BRANCH_USER;
460
461 if (!event->attr.exclude_user)
462 b |= PERF_SAMPLE_BRANCH_USER;
463
464 if (!event->attr.exclude_kernel)
465 b |= PERF_SAMPLE_BRANCH_KERNEL;
466
467 /*
468 * ignore PERF_SAMPLE_BRANCH_HV, not supported on x86
469 */
470
471 return m == b;
472}
473
de0428a7 474int x86_pmu_hw_config(struct perf_event *event)
a072738e 475{
ab608344
PZ
476 if (event->attr.precise_ip) {
477 int precise = 0;
478
479 /* Support for constant skid */
c93dc84c 480 if (x86_pmu.pebs_active && !x86_pmu.pebs_broken) {
ab608344
PZ
481 precise++;
482
5553be26 483 /* Support for IP fixup */
03de874a 484 if (x86_pmu.lbr_nr || x86_pmu.intel_cap.pebs_format >= 2)
5553be26 485 precise++;
72469764
AK
486
487 if (x86_pmu.pebs_prec_dist)
488 precise++;
5553be26 489 }
ab608344
PZ
490
491 if (event->attr.precise_ip > precise)
492 return -EOPNOTSUPP;
4b854900
YZ
493 }
494 /*
495 * check that PEBS LBR correction does not conflict with
496 * whatever the user is asking with attr->branch_sample_type
497 */
498 if (event->attr.precise_ip > 1 && x86_pmu.intel_cap.pebs_format < 2) {
499 u64 *br_type = &event->attr.branch_sample_type;
500
501 if (has_branch_stack(event)) {
502 if (!precise_br_compat(event))
503 return -EOPNOTSUPP;
504
505 /* branch_sample_type is compatible */
506
507 } else {
508 /*
509 * user did not specify branch_sample_type
510 *
511 * For PEBS fixups, we capture all
512 * the branches at the priv level of the
513 * event.
514 */
515 *br_type = PERF_SAMPLE_BRANCH_ANY;
516
517 if (!event->attr.exclude_user)
518 *br_type |= PERF_SAMPLE_BRANCH_USER;
519
520 if (!event->attr.exclude_kernel)
521 *br_type |= PERF_SAMPLE_BRANCH_KERNEL;
ff3fb511 522 }
ab608344
PZ
523 }
524
e18bf526
YZ
525 if (event->attr.branch_sample_type & PERF_SAMPLE_BRANCH_CALL_STACK)
526 event->attach_state |= PERF_ATTACH_TASK_DATA;
527
a072738e
CG
528 /*
529 * Generate PMC IRQs:
530 * (keep 'enabled' bit clear for now)
531 */
b4cdc5c2 532 event->hw.config = ARCH_PERFMON_EVENTSEL_INT;
a072738e
CG
533
534 /*
535 * Count user and OS events unless requested not to
536 */
b4cdc5c2
PZ
537 if (!event->attr.exclude_user)
538 event->hw.config |= ARCH_PERFMON_EVENTSEL_USR;
539 if (!event->attr.exclude_kernel)
540 event->hw.config |= ARCH_PERFMON_EVENTSEL_OS;
a072738e 541
b4cdc5c2
PZ
542 if (event->attr.type == PERF_TYPE_RAW)
543 event->hw.config |= event->attr.config & X86_RAW_EVENT_MASK;
a072738e 544
294fe0f5
AK
545 if (event->attr.sample_period && x86_pmu.limit_period) {
546 if (x86_pmu.limit_period(event, event->attr.sample_period) >
547 event->attr.sample_period)
548 return -EINVAL;
549 }
550
9d0fcba6 551 return x86_setup_perfctr(event);
a098f448
RR
552}
553
241771ef 554/*
0d48696f 555 * Setup the hardware configuration for a given attr_type
241771ef 556 */
b0a873eb 557static int __x86_pmu_event_init(struct perf_event *event)
241771ef 558{
4e935e47 559 int err;
241771ef 560
85cf9dba
RR
561 if (!x86_pmu_initialized())
562 return -ENODEV;
241771ef 563
6b099d9b 564 err = x86_reserve_hardware();
4e935e47
PZ
565 if (err)
566 return err;
567
1b7b938f 568 atomic_inc(&active_events);
cdd6c482 569 event->destroy = hw_perf_event_destroy;
a1792cda 570
4261e0e0
RR
571 event->hw.idx = -1;
572 event->hw.last_cpu = -1;
573 event->hw.last_tag = ~0ULL;
b690081d 574
efc9f05d
SE
575 /* mark unused */
576 event->hw.extra_reg.idx = EXTRA_REG_NONE;
b36817e8
SE
577 event->hw.branch_reg.idx = EXTRA_REG_NONE;
578
9d0fcba6 579 return x86_pmu.hw_config(event);
4261e0e0
RR
580}
581
de0428a7 582void x86_pmu_disable_all(void)
f87ad35d 583{
89cbc767 584 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
9e35ad38
PZ
585 int idx;
586
948b1bb8 587 for (idx = 0; idx < x86_pmu.num_counters; idx++) {
b0f3f28e
PZ
588 u64 val;
589
43f6201a 590 if (!test_bit(idx, cpuc->active_mask))
4295ee62 591 continue;
41bf4989 592 rdmsrl(x86_pmu_config_addr(idx), val);
bb1165d6 593 if (!(val & ARCH_PERFMON_EVENTSEL_ENABLE))
4295ee62 594 continue;
bb1165d6 595 val &= ~ARCH_PERFMON_EVENTSEL_ENABLE;
41bf4989 596 wrmsrl(x86_pmu_config_addr(idx), val);
f87ad35d 597 }
f87ad35d
JSR
598}
599
c3d266c8
KL
600/*
601 * There may be PMI landing after enabled=0. The PMI hitting could be before or
602 * after disable_all.
603 *
604 * If PMI hits before disable_all, the PMU will be disabled in the NMI handler.
605 * It will not be re-enabled in the NMI handler again, because enabled=0. After
606 * handling the NMI, disable_all will be called, which will not change the
607 * state either. If PMI hits after disable_all, the PMU is already disabled
608 * before entering NMI handler. The NMI handler will not change the state
609 * either.
610 *
611 * So either situation is harmless.
612 */
a4eaf7f1 613static void x86_pmu_disable(struct pmu *pmu)
b56a3802 614{
89cbc767 615 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1da53e02 616
85cf9dba 617 if (!x86_pmu_initialized())
9e35ad38 618 return;
1da53e02 619
1a6e21f7
PZ
620 if (!cpuc->enabled)
621 return;
622
623 cpuc->n_added = 0;
624 cpuc->enabled = 0;
625 barrier();
1da53e02
SE
626
627 x86_pmu.disable_all();
b56a3802 628}
241771ef 629
de0428a7 630void x86_pmu_enable_all(int added)
f87ad35d 631{
89cbc767 632 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
f87ad35d
JSR
633 int idx;
634
948b1bb8 635 for (idx = 0; idx < x86_pmu.num_counters; idx++) {
d45dd923 636 struct hw_perf_event *hwc = &cpuc->events[idx]->hw;
b0f3f28e 637
43f6201a 638 if (!test_bit(idx, cpuc->active_mask))
4295ee62 639 continue;
984b838c 640
d45dd923 641 __x86_pmu_enable_event(hwc, ARCH_PERFMON_EVENTSEL_ENABLE);
f87ad35d
JSR
642 }
643}
644
51b0fe39 645static struct pmu pmu;
1da53e02
SE
646
647static inline int is_x86_event(struct perf_event *event)
648{
649 return event->pmu == &pmu;
650}
651
1e2ad28f
RR
652/*
653 * Event scheduler state:
654 *
655 * Assign events iterating over all events and counters, beginning
656 * with events with least weights first. Keep the current iterator
657 * state in struct sched_state.
658 */
659struct sched_state {
660 int weight;
661 int event; /* event index */
662 int counter; /* counter index */
663 int unassigned; /* number of events to be assigned left */
cc1790cf 664 int nr_gp; /* number of GP counters used */
1e2ad28f
RR
665 unsigned long used[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
666};
667
bc1738f6
RR
668/* Total max is X86_PMC_IDX_MAX, but we are O(n!) limited */
669#define SCHED_STATES_MAX 2
670
1e2ad28f
RR
671struct perf_sched {
672 int max_weight;
673 int max_events;
cc1790cf
PZ
674 int max_gp;
675 int saved_states;
b371b594 676 struct event_constraint **constraints;
1e2ad28f 677 struct sched_state state;
bc1738f6 678 struct sched_state saved[SCHED_STATES_MAX];
1e2ad28f
RR
679};
680
681/*
682 * Initialize interator that runs through all events and counters.
683 */
b371b594 684static void perf_sched_init(struct perf_sched *sched, struct event_constraint **constraints,
cc1790cf 685 int num, int wmin, int wmax, int gpmax)
1e2ad28f
RR
686{
687 int idx;
688
689 memset(sched, 0, sizeof(*sched));
690 sched->max_events = num;
691 sched->max_weight = wmax;
cc1790cf 692 sched->max_gp = gpmax;
b371b594 693 sched->constraints = constraints;
1e2ad28f
RR
694
695 for (idx = 0; idx < num; idx++) {
b371b594 696 if (constraints[idx]->weight == wmin)
1e2ad28f
RR
697 break;
698 }
699
700 sched->state.event = idx; /* start with min weight */
701 sched->state.weight = wmin;
702 sched->state.unassigned = num;
703}
704
bc1738f6
RR
705static void perf_sched_save_state(struct perf_sched *sched)
706{
707 if (WARN_ON_ONCE(sched->saved_states >= SCHED_STATES_MAX))
708 return;
709
710 sched->saved[sched->saved_states] = sched->state;
711 sched->saved_states++;
712}
713
714static bool perf_sched_restore_state(struct perf_sched *sched)
715{
716 if (!sched->saved_states)
717 return false;
718
719 sched->saved_states--;
720 sched->state = sched->saved[sched->saved_states];
721
722 /* continue with next counter: */
723 clear_bit(sched->state.counter++, sched->state.used);
724
725 return true;
726}
727
1e2ad28f
RR
728/*
729 * Select a counter for the current event to schedule. Return true on
730 * success.
731 */
bc1738f6 732static bool __perf_sched_find_counter(struct perf_sched *sched)
1e2ad28f
RR
733{
734 struct event_constraint *c;
735 int idx;
736
737 if (!sched->state.unassigned)
738 return false;
739
740 if (sched->state.event >= sched->max_events)
741 return false;
742
b371b594 743 c = sched->constraints[sched->state.event];
4defea85 744 /* Prefer fixed purpose counters */
15c7ad51
RR
745 if (c->idxmsk64 & (~0ULL << INTEL_PMC_IDX_FIXED)) {
746 idx = INTEL_PMC_IDX_FIXED;
307b1cd7 747 for_each_set_bit_from(idx, c->idxmsk, X86_PMC_IDX_MAX) {
4defea85
PZ
748 if (!__test_and_set_bit(idx, sched->state.used))
749 goto done;
750 }
751 }
cc1790cf 752
1e2ad28f
RR
753 /* Grab the first unused counter starting with idx */
754 idx = sched->state.counter;
15c7ad51 755 for_each_set_bit_from(idx, c->idxmsk, INTEL_PMC_IDX_FIXED) {
cc1790cf
PZ
756 if (!__test_and_set_bit(idx, sched->state.used)) {
757 if (sched->state.nr_gp++ >= sched->max_gp)
758 return false;
759
4defea85 760 goto done;
cc1790cf 761 }
1e2ad28f 762 }
1e2ad28f 763
4defea85
PZ
764 return false;
765
766done:
767 sched->state.counter = idx;
1e2ad28f 768
bc1738f6
RR
769 if (c->overlap)
770 perf_sched_save_state(sched);
771
772 return true;
773}
774
775static bool perf_sched_find_counter(struct perf_sched *sched)
776{
777 while (!__perf_sched_find_counter(sched)) {
778 if (!perf_sched_restore_state(sched))
779 return false;
780 }
781
1e2ad28f
RR
782 return true;
783}
784
785/*
786 * Go through all unassigned events and find the next one to schedule.
787 * Take events with the least weight first. Return true on success.
788 */
789static bool perf_sched_next_event(struct perf_sched *sched)
790{
791 struct event_constraint *c;
792
793 if (!sched->state.unassigned || !--sched->state.unassigned)
794 return false;
795
796 do {
797 /* next event */
798 sched->state.event++;
799 if (sched->state.event >= sched->max_events) {
800 /* next weight */
801 sched->state.event = 0;
802 sched->state.weight++;
803 if (sched->state.weight > sched->max_weight)
804 return false;
805 }
b371b594 806 c = sched->constraints[sched->state.event];
1e2ad28f
RR
807 } while (c->weight != sched->state.weight);
808
809 sched->state.counter = 0; /* start with first counter */
810
811 return true;
812}
813
814/*
815 * Assign a counter for each event.
816 */
b371b594 817int perf_assign_events(struct event_constraint **constraints, int n,
cc1790cf 818 int wmin, int wmax, int gpmax, int *assign)
1e2ad28f
RR
819{
820 struct perf_sched sched;
821
cc1790cf 822 perf_sched_init(&sched, constraints, n, wmin, wmax, gpmax);
1e2ad28f
RR
823
824 do {
825 if (!perf_sched_find_counter(&sched))
826 break; /* failed */
827 if (assign)
828 assign[sched.state.event] = sched.state.counter;
829 } while (perf_sched_next_event(&sched));
830
831 return sched.state.unassigned;
832}
4a3dc121 833EXPORT_SYMBOL_GPL(perf_assign_events);
1e2ad28f 834
de0428a7 835int x86_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign)
1da53e02 836{
43b45780 837 struct event_constraint *c;
1da53e02 838 unsigned long used_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
2f7f73a5 839 struct perf_event *e;
e979121b 840 int i, wmin, wmax, unsched = 0;
1da53e02
SE
841 struct hw_perf_event *hwc;
842
843 bitmap_zero(used_mask, X86_PMC_IDX_MAX);
844
c5362c0c
MD
845 if (x86_pmu.start_scheduling)
846 x86_pmu.start_scheduling(cpuc);
847
1e2ad28f 848 for (i = 0, wmin = X86_PMC_IDX_MAX, wmax = 0; i < n; i++) {
b371b594 849 cpuc->event_constraint[i] = NULL;
79cba822 850 c = x86_pmu.get_event_constraints(cpuc, i, cpuc->event_list[i]);
b371b594 851 cpuc->event_constraint[i] = c;
43b45780 852
1e2ad28f
RR
853 wmin = min(wmin, c->weight);
854 wmax = max(wmax, c->weight);
1da53e02
SE
855 }
856
8113070d
SE
857 /*
858 * fastpath, try to reuse previous register
859 */
c933c1a6 860 for (i = 0; i < n; i++) {
8113070d 861 hwc = &cpuc->event_list[i]->hw;
b371b594 862 c = cpuc->event_constraint[i];
8113070d
SE
863
864 /* never assigned */
865 if (hwc->idx == -1)
866 break;
867
868 /* constraint still honored */
63b14649 869 if (!test_bit(hwc->idx, c->idxmsk))
8113070d
SE
870 break;
871
872 /* not already used */
873 if (test_bit(hwc->idx, used_mask))
874 break;
875
34538ee7 876 __set_bit(hwc->idx, used_mask);
8113070d
SE
877 if (assign)
878 assign[i] = hwc->idx;
879 }
8113070d 880
1e2ad28f 881 /* slow path */
b371b594 882 if (i != n) {
cc1790cf
PZ
883 int gpmax = x86_pmu.num_counters;
884
885 /*
886 * Do not allow scheduling of more than half the available
887 * generic counters.
888 *
889 * This helps avoid counter starvation of sibling thread by
890 * ensuring at most half the counters cannot be in exclusive
891 * mode. There is no designated counters for the limits. Any
892 * N/2 counters can be used. This helps with events with
893 * specific counter constraints.
894 */
895 if (is_ht_workaround_enabled() && !cpuc->is_fake &&
896 READ_ONCE(cpuc->excl_cntrs->exclusive_present))
897 gpmax /= 2;
898
b371b594 899 unsched = perf_assign_events(cpuc->event_constraint, n, wmin,
cc1790cf 900 wmax, gpmax, assign);
b371b594 901 }
8113070d 902
2f7f73a5 903 /*
e979121b
MD
904 * In case of success (unsched = 0), mark events as committed,
905 * so we do not put_constraint() in case new events are added
906 * and fail to be scheduled
907 *
908 * We invoke the lower level commit callback to lock the resource
909 *
910 * We do not need to do all of this in case we are called to
911 * validate an event group (assign == NULL)
2f7f73a5 912 */
e979121b 913 if (!unsched && assign) {
2f7f73a5
SE
914 for (i = 0; i < n; i++) {
915 e = cpuc->event_list[i];
916 e->hw.flags |= PERF_X86_EVENT_COMMITTED;
c5362c0c 917 if (x86_pmu.commit_scheduling)
b371b594 918 x86_pmu.commit_scheduling(cpuc, i, assign[i]);
2f7f73a5 919 }
8736e548 920 } else {
1da53e02 921 for (i = 0; i < n; i++) {
2f7f73a5
SE
922 e = cpuc->event_list[i];
923 /*
924 * do not put_constraint() on comitted events,
925 * because they are good to go
926 */
927 if ((e->hw.flags & PERF_X86_EVENT_COMMITTED))
928 continue;
929
e979121b
MD
930 /*
931 * release events that failed scheduling
932 */
1da53e02 933 if (x86_pmu.put_event_constraints)
2f7f73a5 934 x86_pmu.put_event_constraints(cpuc, e);
1da53e02
SE
935 }
936 }
c5362c0c
MD
937
938 if (x86_pmu.stop_scheduling)
939 x86_pmu.stop_scheduling(cpuc);
940
e979121b 941 return unsched ? -EINVAL : 0;
1da53e02
SE
942}
943
944/*
945 * dogrp: true if must collect siblings events (group)
946 * returns total number of events and error code
947 */
948static int collect_events(struct cpu_hw_events *cpuc, struct perf_event *leader, bool dogrp)
949{
950 struct perf_event *event;
951 int n, max_count;
952
948b1bb8 953 max_count = x86_pmu.num_counters + x86_pmu.num_counters_fixed;
1da53e02
SE
954
955 /* current number of events already accepted */
956 n = cpuc->n_events;
957
958 if (is_x86_event(leader)) {
959 if (n >= max_count)
aa2bc1ad 960 return -EINVAL;
1da53e02
SE
961 cpuc->event_list[n] = leader;
962 n++;
963 }
964 if (!dogrp)
965 return n;
966
967 list_for_each_entry(event, &leader->sibling_list, group_entry) {
968 if (!is_x86_event(event) ||
8113070d 969 event->state <= PERF_EVENT_STATE_OFF)
1da53e02
SE
970 continue;
971
972 if (n >= max_count)
aa2bc1ad 973 return -EINVAL;
1da53e02
SE
974
975 cpuc->event_list[n] = event;
976 n++;
977 }
978 return n;
979}
980
1da53e02 981static inline void x86_assign_hw_event(struct perf_event *event,
447a194b 982 struct cpu_hw_events *cpuc, int i)
1da53e02 983{
447a194b
SE
984 struct hw_perf_event *hwc = &event->hw;
985
986 hwc->idx = cpuc->assign[i];
987 hwc->last_cpu = smp_processor_id();
988 hwc->last_tag = ++cpuc->tags[i];
1da53e02 989
15c7ad51 990 if (hwc->idx == INTEL_PMC_IDX_FIXED_BTS) {
1da53e02
SE
991 hwc->config_base = 0;
992 hwc->event_base = 0;
15c7ad51 993 } else if (hwc->idx >= INTEL_PMC_IDX_FIXED) {
1da53e02 994 hwc->config_base = MSR_ARCH_PERFMON_FIXED_CTR_CTRL;
15c7ad51
RR
995 hwc->event_base = MSR_ARCH_PERFMON_FIXED_CTR0 + (hwc->idx - INTEL_PMC_IDX_FIXED);
996 hwc->event_base_rdpmc = (hwc->idx - INTEL_PMC_IDX_FIXED) | 1<<30;
1da53e02 997 } else {
73d6e522
RR
998 hwc->config_base = x86_pmu_config_addr(hwc->idx);
999 hwc->event_base = x86_pmu_event_addr(hwc->idx);
0fbdad07 1000 hwc->event_base_rdpmc = x86_pmu_rdpmc_index(hwc->idx);
1da53e02
SE
1001 }
1002}
1003
447a194b
SE
1004static inline int match_prev_assignment(struct hw_perf_event *hwc,
1005 struct cpu_hw_events *cpuc,
1006 int i)
1007{
1008 return hwc->idx == cpuc->assign[i] &&
1009 hwc->last_cpu == smp_processor_id() &&
1010 hwc->last_tag == cpuc->tags[i];
1011}
1012
a4eaf7f1 1013static void x86_pmu_start(struct perf_event *event, int flags);
2e841873 1014
a4eaf7f1 1015static void x86_pmu_enable(struct pmu *pmu)
ee06094f 1016{
89cbc767 1017 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1da53e02
SE
1018 struct perf_event *event;
1019 struct hw_perf_event *hwc;
11164cd4 1020 int i, added = cpuc->n_added;
1da53e02 1021
85cf9dba 1022 if (!x86_pmu_initialized())
2b9ff0db 1023 return;
1a6e21f7
PZ
1024
1025 if (cpuc->enabled)
1026 return;
1027
1da53e02 1028 if (cpuc->n_added) {
19925ce7 1029 int n_running = cpuc->n_events - cpuc->n_added;
1da53e02
SE
1030 /*
1031 * apply assignment obtained either from
1032 * hw_perf_group_sched_in() or x86_pmu_enable()
1033 *
1034 * step1: save events moving to new counters
1da53e02 1035 */
19925ce7 1036 for (i = 0; i < n_running; i++) {
1da53e02
SE
1037 event = cpuc->event_list[i];
1038 hwc = &event->hw;
1039
447a194b
SE
1040 /*
1041 * we can avoid reprogramming counter if:
1042 * - assigned same counter as last time
1043 * - running on same CPU as last time
1044 * - no other event has used the counter since
1045 */
1046 if (hwc->idx == -1 ||
1047 match_prev_assignment(hwc, cpuc, i))
1da53e02
SE
1048 continue;
1049
a4eaf7f1
PZ
1050 /*
1051 * Ensure we don't accidentally enable a stopped
1052 * counter simply because we rescheduled.
1053 */
1054 if (hwc->state & PERF_HES_STOPPED)
1055 hwc->state |= PERF_HES_ARCH;
1056
1057 x86_pmu_stop(event, PERF_EF_UPDATE);
1da53e02
SE
1058 }
1059
c347a2f1
PZ
1060 /*
1061 * step2: reprogram moved events into new counters
1062 */
1da53e02 1063 for (i = 0; i < cpuc->n_events; i++) {
1da53e02
SE
1064 event = cpuc->event_list[i];
1065 hwc = &event->hw;
1066
45e16a68 1067 if (!match_prev_assignment(hwc, cpuc, i))
447a194b 1068 x86_assign_hw_event(event, cpuc, i);
45e16a68
PZ
1069 else if (i < n_running)
1070 continue;
1da53e02 1071
a4eaf7f1
PZ
1072 if (hwc->state & PERF_HES_ARCH)
1073 continue;
1074
1075 x86_pmu_start(event, PERF_EF_RELOAD);
1da53e02
SE
1076 }
1077 cpuc->n_added = 0;
1078 perf_events_lapic_init();
1079 }
1a6e21f7
PZ
1080
1081 cpuc->enabled = 1;
1082 barrier();
1083
11164cd4 1084 x86_pmu.enable_all(added);
ee06094f 1085}
ee06094f 1086
245b2e70 1087static DEFINE_PER_CPU(u64 [X86_PMC_IDX_MAX], pmc_prev_left);
241771ef 1088
ee06094f
IM
1089/*
1090 * Set the next IRQ period, based on the hwc->period_left value.
cdd6c482 1091 * To be called with the event disabled in hw:
ee06094f 1092 */
de0428a7 1093int x86_perf_event_set_period(struct perf_event *event)
241771ef 1094{
07088edb 1095 struct hw_perf_event *hwc = &event->hw;
e7850595 1096 s64 left = local64_read(&hwc->period_left);
e4abb5d4 1097 s64 period = hwc->sample_period;
7645a24c 1098 int ret = 0, idx = hwc->idx;
ee06094f 1099
15c7ad51 1100 if (idx == INTEL_PMC_IDX_FIXED_BTS)
30dd568c
MM
1101 return 0;
1102
ee06094f 1103 /*
af901ca1 1104 * If we are way outside a reasonable range then just skip forward:
ee06094f
IM
1105 */
1106 if (unlikely(left <= -period)) {
1107 left = period;
e7850595 1108 local64_set(&hwc->period_left, left);
9e350de3 1109 hwc->last_period = period;
e4abb5d4 1110 ret = 1;
ee06094f
IM
1111 }
1112
1113 if (unlikely(left <= 0)) {
1114 left += period;
e7850595 1115 local64_set(&hwc->period_left, left);
9e350de3 1116 hwc->last_period = period;
e4abb5d4 1117 ret = 1;
ee06094f 1118 }
1c80f4b5 1119 /*
dfc65094 1120 * Quirk: certain CPUs dont like it if just 1 hw_event is left:
1c80f4b5
IM
1121 */
1122 if (unlikely(left < 2))
1123 left = 2;
241771ef 1124
e4abb5d4
PZ
1125 if (left > x86_pmu.max_period)
1126 left = x86_pmu.max_period;
1127
294fe0f5
AK
1128 if (x86_pmu.limit_period)
1129 left = x86_pmu.limit_period(event, left);
1130
245b2e70 1131 per_cpu(pmc_prev_left[idx], smp_processor_id()) = left;
ee06094f 1132
851559e3
YZ
1133 if (!(hwc->flags & PERF_X86_EVENT_AUTO_RELOAD) ||
1134 local64_read(&hwc->prev_count) != (u64)-left) {
1135 /*
1136 * The hw event starts counting from this event offset,
1137 * mark it to be able to extra future deltas:
1138 */
1139 local64_set(&hwc->prev_count, (u64)-left);
ee06094f 1140
851559e3
YZ
1141 wrmsrl(hwc->event_base, (u64)(-left) & x86_pmu.cntval_mask);
1142 }
68aa00ac
CG
1143
1144 /*
1145 * Due to erratum on certan cpu we need
1146 * a second write to be sure the register
1147 * is updated properly
1148 */
1149 if (x86_pmu.perfctr_second_write) {
73d6e522 1150 wrmsrl(hwc->event_base,
948b1bb8 1151 (u64)(-left) & x86_pmu.cntval_mask);
68aa00ac 1152 }
e4abb5d4 1153
cdd6c482 1154 perf_event_update_userpage(event);
194002b2 1155
e4abb5d4 1156 return ret;
2f18d1e8
IM
1157}
1158
de0428a7 1159void x86_pmu_enable_event(struct perf_event *event)
7c90cc45 1160{
0a3aee0d 1161 if (__this_cpu_read(cpu_hw_events.enabled))
31fa58af
RR
1162 __x86_pmu_enable_event(&event->hw,
1163 ARCH_PERFMON_EVENTSEL_ENABLE);
241771ef
IM
1164}
1165
b690081d 1166/*
a4eaf7f1 1167 * Add a single event to the PMU.
1da53e02
SE
1168 *
1169 * The event is added to the group of enabled events
1170 * but only if it can be scehduled with existing events.
fe9081cc 1171 */
a4eaf7f1 1172static int x86_pmu_add(struct perf_event *event, int flags)
fe9081cc 1173{
89cbc767 1174 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1da53e02
SE
1175 struct hw_perf_event *hwc;
1176 int assign[X86_PMC_IDX_MAX];
1177 int n, n0, ret;
fe9081cc 1178
1da53e02 1179 hwc = &event->hw;
fe9081cc 1180
1da53e02 1181 n0 = cpuc->n_events;
24cd7f54
PZ
1182 ret = n = collect_events(cpuc, event, false);
1183 if (ret < 0)
1184 goto out;
53b441a5 1185
a4eaf7f1
PZ
1186 hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
1187 if (!(flags & PERF_EF_START))
1188 hwc->state |= PERF_HES_ARCH;
1189
4d1c52b0
LM
1190 /*
1191 * If group events scheduling transaction was started,
0d2eb44f 1192 * skip the schedulability test here, it will be performed
c347a2f1 1193 * at commit time (->commit_txn) as a whole.
4d1c52b0 1194 */
8f3e5684 1195 if (cpuc->txn_flags & PERF_PMU_TXN_ADD)
24cd7f54 1196 goto done_collect;
4d1c52b0 1197
a072738e 1198 ret = x86_pmu.schedule_events(cpuc, n, assign);
1da53e02 1199 if (ret)
24cd7f54 1200 goto out;
1da53e02
SE
1201 /*
1202 * copy new assignment, now we know it is possible
1203 * will be used by hw_perf_enable()
1204 */
1205 memcpy(cpuc->assign, assign, n*sizeof(int));
7e2ae347 1206
24cd7f54 1207done_collect:
c347a2f1
PZ
1208 /*
1209 * Commit the collect_events() state. See x86_pmu_del() and
1210 * x86_pmu_*_txn().
1211 */
1da53e02 1212 cpuc->n_events = n;
356e1f2e 1213 cpuc->n_added += n - n0;
90151c35 1214 cpuc->n_txn += n - n0;
95cdd2e7 1215
24cd7f54
PZ
1216 ret = 0;
1217out:
24cd7f54 1218 return ret;
241771ef
IM
1219}
1220
a4eaf7f1 1221static void x86_pmu_start(struct perf_event *event, int flags)
d76a0812 1222{
89cbc767 1223 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
c08053e6
PZ
1224 int idx = event->hw.idx;
1225
a4eaf7f1
PZ
1226 if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED)))
1227 return;
1228
1229 if (WARN_ON_ONCE(idx == -1))
1230 return;
1231
1232 if (flags & PERF_EF_RELOAD) {
1233 WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE));
1234 x86_perf_event_set_period(event);
1235 }
1236
1237 event->hw.state = 0;
d76a0812 1238
c08053e6
PZ
1239 cpuc->events[idx] = event;
1240 __set_bit(idx, cpuc->active_mask);
63e6be6d 1241 __set_bit(idx, cpuc->running);
aff3d91a 1242 x86_pmu.enable(event);
c08053e6 1243 perf_event_update_userpage(event);
a78ac325
PZ
1244}
1245
cdd6c482 1246void perf_event_print_debug(void)
241771ef 1247{
2f18d1e8 1248 u64 ctrl, status, overflow, pmc_ctrl, pmc_count, prev_left, fixed;
da3e606d 1249 u64 pebs, debugctl;
cdd6c482 1250 struct cpu_hw_events *cpuc;
5bb9efe3 1251 unsigned long flags;
1e125676
IM
1252 int cpu, idx;
1253
948b1bb8 1254 if (!x86_pmu.num_counters)
1e125676 1255 return;
241771ef 1256
5bb9efe3 1257 local_irq_save(flags);
241771ef
IM
1258
1259 cpu = smp_processor_id();
cdd6c482 1260 cpuc = &per_cpu(cpu_hw_events, cpu);
241771ef 1261
faa28ae0 1262 if (x86_pmu.version >= 2) {
a1ef58f4
JSR
1263 rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
1264 rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
1265 rdmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, overflow);
1266 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR_CTRL, fixed);
1267
1268 pr_info("\n");
1269 pr_info("CPU#%d: ctrl: %016llx\n", cpu, ctrl);
1270 pr_info("CPU#%d: status: %016llx\n", cpu, status);
1271 pr_info("CPU#%d: overflow: %016llx\n", cpu, overflow);
1272 pr_info("CPU#%d: fixed: %016llx\n", cpu, fixed);
15fde110
AK
1273 if (x86_pmu.pebs_constraints) {
1274 rdmsrl(MSR_IA32_PEBS_ENABLE, pebs);
1275 pr_info("CPU#%d: pebs: %016llx\n", cpu, pebs);
1276 }
da3e606d
AK
1277 if (x86_pmu.lbr_nr) {
1278 rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
1279 pr_info("CPU#%d: debugctl: %016llx\n", cpu, debugctl);
1280 }
f87ad35d 1281 }
7645a24c 1282 pr_info("CPU#%d: active: %016llx\n", cpu, *(u64 *)cpuc->active_mask);
241771ef 1283
948b1bb8 1284 for (idx = 0; idx < x86_pmu.num_counters; idx++) {
41bf4989
RR
1285 rdmsrl(x86_pmu_config_addr(idx), pmc_ctrl);
1286 rdmsrl(x86_pmu_event_addr(idx), pmc_count);
241771ef 1287
245b2e70 1288 prev_left = per_cpu(pmc_prev_left[idx], cpu);
241771ef 1289
a1ef58f4 1290 pr_info("CPU#%d: gen-PMC%d ctrl: %016llx\n",
241771ef 1291 cpu, idx, pmc_ctrl);
a1ef58f4 1292 pr_info("CPU#%d: gen-PMC%d count: %016llx\n",
241771ef 1293 cpu, idx, pmc_count);
a1ef58f4 1294 pr_info("CPU#%d: gen-PMC%d left: %016llx\n",
ee06094f 1295 cpu, idx, prev_left);
241771ef 1296 }
948b1bb8 1297 for (idx = 0; idx < x86_pmu.num_counters_fixed; idx++) {
2f18d1e8
IM
1298 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, pmc_count);
1299
a1ef58f4 1300 pr_info("CPU#%d: fixed-PMC%d count: %016llx\n",
2f18d1e8
IM
1301 cpu, idx, pmc_count);
1302 }
5bb9efe3 1303 local_irq_restore(flags);
241771ef
IM
1304}
1305
de0428a7 1306void x86_pmu_stop(struct perf_event *event, int flags)
241771ef 1307{
89cbc767 1308 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
cdd6c482 1309 struct hw_perf_event *hwc = &event->hw;
241771ef 1310
a4eaf7f1
PZ
1311 if (__test_and_clear_bit(hwc->idx, cpuc->active_mask)) {
1312 x86_pmu.disable(event);
1313 cpuc->events[hwc->idx] = NULL;
1314 WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);
1315 hwc->state |= PERF_HES_STOPPED;
1316 }
30dd568c 1317
a4eaf7f1
PZ
1318 if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) {
1319 /*
1320 * Drain the remaining delta count out of a event
1321 * that we are disabling:
1322 */
1323 x86_perf_event_update(event);
1324 hwc->state |= PERF_HES_UPTODATE;
1325 }
2e841873
PZ
1326}
1327
a4eaf7f1 1328static void x86_pmu_del(struct perf_event *event, int flags)
2e841873 1329{
89cbc767 1330 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
2e841873
PZ
1331 int i;
1332
2f7f73a5
SE
1333 /*
1334 * event is descheduled
1335 */
1336 event->hw.flags &= ~PERF_X86_EVENT_COMMITTED;
1337
90151c35
SE
1338 /*
1339 * If we're called during a txn, we don't need to do anything.
1340 * The events never got scheduled and ->cancel_txn will truncate
1341 * the event_list.
c347a2f1
PZ
1342 *
1343 * XXX assumes any ->del() called during a TXN will only be on
1344 * an event added during that same TXN.
90151c35 1345 */
8f3e5684 1346 if (cpuc->txn_flags & PERF_PMU_TXN_ADD)
90151c35
SE
1347 return;
1348
c347a2f1
PZ
1349 /*
1350 * Not a TXN, therefore cleanup properly.
1351 */
a4eaf7f1 1352 x86_pmu_stop(event, PERF_EF_UPDATE);
194002b2 1353
1da53e02 1354 for (i = 0; i < cpuc->n_events; i++) {
c347a2f1
PZ
1355 if (event == cpuc->event_list[i])
1356 break;
1357 }
1da53e02 1358
c347a2f1
PZ
1359 if (WARN_ON_ONCE(i == cpuc->n_events)) /* called ->del() without ->add() ? */
1360 return;
26e61e89 1361
c347a2f1
PZ
1362 /* If we have a newly added event; make sure to decrease n_added. */
1363 if (i >= cpuc->n_events - cpuc->n_added)
1364 --cpuc->n_added;
1da53e02 1365
c347a2f1
PZ
1366 if (x86_pmu.put_event_constraints)
1367 x86_pmu.put_event_constraints(cpuc, event);
1368
1369 /* Delete the array entry. */
b371b594 1370 while (++i < cpuc->n_events) {
c347a2f1 1371 cpuc->event_list[i-1] = cpuc->event_list[i];
b371b594
PZ
1372 cpuc->event_constraint[i-1] = cpuc->event_constraint[i];
1373 }
c347a2f1 1374 --cpuc->n_events;
1da53e02 1375
cdd6c482 1376 perf_event_update_userpage(event);
241771ef
IM
1377}
1378
de0428a7 1379int x86_pmu_handle_irq(struct pt_regs *regs)
a29aa8a7 1380{
df1a132b 1381 struct perf_sample_data data;
cdd6c482
IM
1382 struct cpu_hw_events *cpuc;
1383 struct perf_event *event;
11d1578f 1384 int idx, handled = 0;
9029a5e3
IM
1385 u64 val;
1386
89cbc767 1387 cpuc = this_cpu_ptr(&cpu_hw_events);
962bf7a6 1388
2bce5dac
DZ
1389 /*
1390 * Some chipsets need to unmask the LVTPC in a particular spot
1391 * inside the nmi handler. As a result, the unmasking was pushed
1392 * into all the nmi handlers.
1393 *
1394 * This generic handler doesn't seem to have any issues where the
1395 * unmasking occurs so it was left at the top.
1396 */
1397 apic_write(APIC_LVTPC, APIC_DM_NMI);
1398
948b1bb8 1399 for (idx = 0; idx < x86_pmu.num_counters; idx++) {
63e6be6d
RR
1400 if (!test_bit(idx, cpuc->active_mask)) {
1401 /*
1402 * Though we deactivated the counter some cpus
1403 * might still deliver spurious interrupts still
1404 * in flight. Catch them:
1405 */
1406 if (__test_and_clear_bit(idx, cpuc->running))
1407 handled++;
a29aa8a7 1408 continue;
63e6be6d 1409 }
962bf7a6 1410
cdd6c482 1411 event = cpuc->events[idx];
a4016a79 1412
cc2ad4ba 1413 val = x86_perf_event_update(event);
948b1bb8 1414 if (val & (1ULL << (x86_pmu.cntval_bits - 1)))
48e22d56 1415 continue;
962bf7a6 1416
9e350de3 1417 /*
cdd6c482 1418 * event overflow
9e350de3 1419 */
4177c42a 1420 handled++;
fd0d000b 1421 perf_sample_data_init(&data, 0, event->hw.last_period);
9e350de3 1422
07088edb 1423 if (!x86_perf_event_set_period(event))
e4abb5d4
PZ
1424 continue;
1425
a8b0ca17 1426 if (perf_event_overflow(event, &data, regs))
a4eaf7f1 1427 x86_pmu_stop(event, 0);
a29aa8a7 1428 }
962bf7a6 1429
9e350de3
PZ
1430 if (handled)
1431 inc_irq_stat(apic_perf_irqs);
1432
a29aa8a7
RR
1433 return handled;
1434}
39d81eab 1435
cdd6c482 1436void perf_events_lapic_init(void)
241771ef 1437{
04da8a43 1438 if (!x86_pmu.apic || !x86_pmu_initialized())
241771ef 1439 return;
85cf9dba 1440
241771ef 1441 /*
c323d95f 1442 * Always use NMI for PMU
241771ef 1443 */
c323d95f 1444 apic_write(APIC_LVTPC, APIC_DM_NMI);
241771ef
IM
1445}
1446
9326638c 1447static int
9c48f1c6 1448perf_event_nmi_handler(unsigned int cmd, struct pt_regs *regs)
241771ef 1449{
14c63f17
DH
1450 u64 start_clock;
1451 u64 finish_clock;
e8a923cc 1452 int ret;
14c63f17 1453
1b7b938f
AS
1454 /*
1455 * All PMUs/events that share this PMI handler should make sure to
1456 * increment active_events for their events.
1457 */
cdd6c482 1458 if (!atomic_read(&active_events))
9c48f1c6 1459 return NMI_DONE;
4177c42a 1460
e8a923cc 1461 start_clock = sched_clock();
14c63f17 1462 ret = x86_pmu.handle_irq(regs);
e8a923cc 1463 finish_clock = sched_clock();
14c63f17
DH
1464
1465 perf_sample_event_took(finish_clock - start_clock);
1466
1467 return ret;
241771ef 1468}
9326638c 1469NOKPROBE_SYMBOL(perf_event_nmi_handler);
241771ef 1470
de0428a7
KW
1471struct event_constraint emptyconstraint;
1472struct event_constraint unconstrained;
f87ad35d 1473
148f9bb8 1474static int
3f6da390
PZ
1475x86_pmu_notifier(struct notifier_block *self, unsigned long action, void *hcpu)
1476{
1477 unsigned int cpu = (long)hcpu;
7fdba1ca 1478 struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);
90413464 1479 int i, ret = NOTIFY_OK;
3f6da390
PZ
1480
1481 switch (action & ~CPU_TASKS_FROZEN) {
1482 case CPU_UP_PREPARE:
90413464
SE
1483 for (i = 0 ; i < X86_PERF_KFREE_MAX; i++)
1484 cpuc->kfree_on_online[i] = NULL;
3f6da390 1485 if (x86_pmu.cpu_prepare)
b38b24ea 1486 ret = x86_pmu.cpu_prepare(cpu);
3f6da390
PZ
1487 break;
1488
1489 case CPU_STARTING:
1490 if (x86_pmu.cpu_starting)
1491 x86_pmu.cpu_starting(cpu);
1492 break;
1493
7fdba1ca 1494 case CPU_ONLINE:
90413464
SE
1495 for (i = 0 ; i < X86_PERF_KFREE_MAX; i++) {
1496 kfree(cpuc->kfree_on_online[i]);
1497 cpuc->kfree_on_online[i] = NULL;
1498 }
7fdba1ca
PZ
1499 break;
1500
3f6da390
PZ
1501 case CPU_DYING:
1502 if (x86_pmu.cpu_dying)
1503 x86_pmu.cpu_dying(cpu);
1504 break;
1505
b38b24ea 1506 case CPU_UP_CANCELED:
3f6da390
PZ
1507 case CPU_DEAD:
1508 if (x86_pmu.cpu_dead)
1509 x86_pmu.cpu_dead(cpu);
1510 break;
1511
1512 default:
1513 break;
1514 }
1515
b38b24ea 1516 return ret;
3f6da390
PZ
1517}
1518
12558038
CG
1519static void __init pmu_check_apic(void)
1520{
1521 if (cpu_has_apic)
1522 return;
1523
1524 x86_pmu.apic = 0;
1525 pr_info("no APIC, boot with the \"lapic\" boot parameter to force-enable it.\n");
1526 pr_info("no hardware sampling interrupt available.\n");
c184c980
VW
1527
1528 /*
1529 * If we have a PMU initialized but no APIC
1530 * interrupts, we cannot sample hardware
1531 * events (user-space has to fall back and
1532 * sample via a hrtimer based software event):
1533 */
1534 pmu.capabilities |= PERF_PMU_CAP_NO_INTERRUPT;
1535
12558038
CG
1536}
1537
641cc938
JO
1538static struct attribute_group x86_pmu_format_group = {
1539 .name = "format",
1540 .attrs = NULL,
1541};
1542
8300daa2
JO
1543/*
1544 * Remove all undefined events (x86_pmu.event_map(id) == 0)
1545 * out of events_attr attributes.
1546 */
1547static void __init filter_events(struct attribute **attrs)
1548{
3a54aaa0
SE
1549 struct device_attribute *d;
1550 struct perf_pmu_events_attr *pmu_attr;
61b87cae 1551 int offset = 0;
8300daa2
JO
1552 int i, j;
1553
1554 for (i = 0; attrs[i]; i++) {
3a54aaa0
SE
1555 d = (struct device_attribute *)attrs[i];
1556 pmu_attr = container_of(d, struct perf_pmu_events_attr, attr);
1557 /* str trumps id */
1558 if (pmu_attr->event_str)
1559 continue;
61b87cae 1560 if (x86_pmu.event_map(i + offset))
8300daa2
JO
1561 continue;
1562
1563 for (j = i; attrs[j]; j++)
1564 attrs[j] = attrs[j + 1];
1565
1566 /* Check the shifted attr. */
1567 i--;
61b87cae
SE
1568
1569 /*
1570 * event_map() is index based, the attrs array is organized
1571 * by increasing event index. If we shift the events, then
1572 * we need to compensate for the event_map(), otherwise
1573 * we are looking up the wrong event in the map
1574 */
1575 offset++;
8300daa2
JO
1576 }
1577}
1578
1a6461b1 1579/* Merge two pointer arrays */
47732d88 1580__init struct attribute **merge_attr(struct attribute **a, struct attribute **b)
1a6461b1
AK
1581{
1582 struct attribute **new;
1583 int j, i;
1584
1585 for (j = 0; a[j]; j++)
1586 ;
1587 for (i = 0; b[i]; i++)
1588 j++;
1589 j++;
1590
1591 new = kmalloc(sizeof(struct attribute *) * j, GFP_KERNEL);
1592 if (!new)
1593 return NULL;
1594
1595 j = 0;
1596 for (i = 0; a[i]; i++)
1597 new[j++] = a[i];
1598 for (i = 0; b[i]; i++)
1599 new[j++] = b[i];
1600 new[j] = NULL;
1601
1602 return new;
1603}
1604
c7ab62bf 1605ssize_t events_sysfs_show(struct device *dev, struct device_attribute *attr, char *page)
a4747393
JO
1606{
1607 struct perf_pmu_events_attr *pmu_attr = \
1608 container_of(attr, struct perf_pmu_events_attr, attr);
a4747393 1609 u64 config = x86_pmu.event_map(pmu_attr->id);
a4747393 1610
3a54aaa0
SE
1611 /* string trumps id */
1612 if (pmu_attr->event_str)
1613 return sprintf(page, "%s", pmu_attr->event_str);
a4747393 1614
3a54aaa0
SE
1615 return x86_pmu.events_sysfs_show(page, config);
1616}
c7ab62bf 1617EXPORT_SYMBOL_GPL(events_sysfs_show);
a4747393
JO
1618
1619EVENT_ATTR(cpu-cycles, CPU_CYCLES );
1620EVENT_ATTR(instructions, INSTRUCTIONS );
1621EVENT_ATTR(cache-references, CACHE_REFERENCES );
1622EVENT_ATTR(cache-misses, CACHE_MISSES );
1623EVENT_ATTR(branch-instructions, BRANCH_INSTRUCTIONS );
1624EVENT_ATTR(branch-misses, BRANCH_MISSES );
1625EVENT_ATTR(bus-cycles, BUS_CYCLES );
1626EVENT_ATTR(stalled-cycles-frontend, STALLED_CYCLES_FRONTEND );
1627EVENT_ATTR(stalled-cycles-backend, STALLED_CYCLES_BACKEND );
1628EVENT_ATTR(ref-cycles, REF_CPU_CYCLES );
1629
1630static struct attribute *empty_attrs;
1631
95d18aa2 1632static struct attribute *events_attr[] = {
a4747393
JO
1633 EVENT_PTR(CPU_CYCLES),
1634 EVENT_PTR(INSTRUCTIONS),
1635 EVENT_PTR(CACHE_REFERENCES),
1636 EVENT_PTR(CACHE_MISSES),
1637 EVENT_PTR(BRANCH_INSTRUCTIONS),
1638 EVENT_PTR(BRANCH_MISSES),
1639 EVENT_PTR(BUS_CYCLES),
1640 EVENT_PTR(STALLED_CYCLES_FRONTEND),
1641 EVENT_PTR(STALLED_CYCLES_BACKEND),
1642 EVENT_PTR(REF_CPU_CYCLES),
1643 NULL,
1644};
1645
1646static struct attribute_group x86_pmu_events_group = {
1647 .name = "events",
1648 .attrs = events_attr,
1649};
1650
0bf79d44 1651ssize_t x86_event_sysfs_show(char *page, u64 config, u64 event)
43c032fe 1652{
43c032fe
JO
1653 u64 umask = (config & ARCH_PERFMON_EVENTSEL_UMASK) >> 8;
1654 u64 cmask = (config & ARCH_PERFMON_EVENTSEL_CMASK) >> 24;
1655 bool edge = (config & ARCH_PERFMON_EVENTSEL_EDGE);
1656 bool pc = (config & ARCH_PERFMON_EVENTSEL_PIN_CONTROL);
1657 bool any = (config & ARCH_PERFMON_EVENTSEL_ANY);
1658 bool inv = (config & ARCH_PERFMON_EVENTSEL_INV);
1659 ssize_t ret;
1660
1661 /*
1662 * We have whole page size to spend and just little data
1663 * to write, so we can safely use sprintf.
1664 */
1665 ret = sprintf(page, "event=0x%02llx", event);
1666
1667 if (umask)
1668 ret += sprintf(page + ret, ",umask=0x%02llx", umask);
1669
1670 if (edge)
1671 ret += sprintf(page + ret, ",edge");
1672
1673 if (pc)
1674 ret += sprintf(page + ret, ",pc");
1675
1676 if (any)
1677 ret += sprintf(page + ret, ",any");
1678
1679 if (inv)
1680 ret += sprintf(page + ret, ",inv");
1681
1682 if (cmask)
1683 ret += sprintf(page + ret, ",cmask=0x%02llx", cmask);
1684
1685 ret += sprintf(page + ret, "\n");
1686
1687 return ret;
1688}
1689
dda99116 1690static int __init init_hw_perf_events(void)
b56a3802 1691{
c1d6f42f 1692 struct x86_pmu_quirk *quirk;
72eae04d
RR
1693 int err;
1694
cdd6c482 1695 pr_info("Performance Events: ");
1123e3ad 1696
b56a3802
JSR
1697 switch (boot_cpu_data.x86_vendor) {
1698 case X86_VENDOR_INTEL:
72eae04d 1699 err = intel_pmu_init();
b56a3802 1700 break;
f87ad35d 1701 case X86_VENDOR_AMD:
72eae04d 1702 err = amd_pmu_init();
f87ad35d 1703 break;
4138960a 1704 default:
8a3da6c7 1705 err = -ENOTSUPP;
b56a3802 1706 }
1123e3ad 1707 if (err != 0) {
cdd6c482 1708 pr_cont("no PMU driver, software events only.\n");
004417a6 1709 return 0;
1123e3ad 1710 }
b56a3802 1711
12558038
CG
1712 pmu_check_apic();
1713
33c6d6a7 1714 /* sanity check that the hardware exists or is emulated */
4407204c 1715 if (!check_hw_exists())
004417a6 1716 return 0;
33c6d6a7 1717
1123e3ad 1718 pr_cont("%s PMU driver.\n", x86_pmu.name);
faa28ae0 1719
e97df763
PZ
1720 x86_pmu.attr_rdpmc = 1; /* enable userspace RDPMC usage by default */
1721
c1d6f42f
PZ
1722 for (quirk = x86_pmu.quirks; quirk; quirk = quirk->next)
1723 quirk->func();
3c44780b 1724
a1eac7ac
RR
1725 if (!x86_pmu.intel_ctrl)
1726 x86_pmu.intel_ctrl = (1 << x86_pmu.num_counters) - 1;
241771ef 1727
cdd6c482 1728 perf_events_lapic_init();
9c48f1c6 1729 register_nmi_handler(NMI_LOCAL, perf_event_nmi_handler, 0, "PMI");
1123e3ad 1730
63b14649 1731 unconstrained = (struct event_constraint)
948b1bb8 1732 __EVENT_CONSTRAINT(0, (1ULL << x86_pmu.num_counters) - 1,
9fac2cf3 1733 0, x86_pmu.num_counters, 0, 0);
63b14649 1734
641cc938 1735 x86_pmu_format_group.attrs = x86_pmu.format_attrs;
0c9d42ed 1736
f20093ee
SE
1737 if (x86_pmu.event_attrs)
1738 x86_pmu_events_group.attrs = x86_pmu.event_attrs;
1739
a4747393
JO
1740 if (!x86_pmu.events_sysfs_show)
1741 x86_pmu_events_group.attrs = &empty_attrs;
8300daa2
JO
1742 else
1743 filter_events(x86_pmu_events_group.attrs);
a4747393 1744
1a6461b1
AK
1745 if (x86_pmu.cpu_events) {
1746 struct attribute **tmp;
1747
1748 tmp = merge_attr(x86_pmu_events_group.attrs, x86_pmu.cpu_events);
1749 if (!WARN_ON(!tmp))
1750 x86_pmu_events_group.attrs = tmp;
1751 }
1752
57c0c15b 1753 pr_info("... version: %d\n", x86_pmu.version);
948b1bb8
RR
1754 pr_info("... bit width: %d\n", x86_pmu.cntval_bits);
1755 pr_info("... generic registers: %d\n", x86_pmu.num_counters);
1756 pr_info("... value mask: %016Lx\n", x86_pmu.cntval_mask);
57c0c15b 1757 pr_info("... max period: %016Lx\n", x86_pmu.max_period);
948b1bb8 1758 pr_info("... fixed-purpose events: %d\n", x86_pmu.num_counters_fixed);
d6dc0b4e 1759 pr_info("... event mask: %016Lx\n", x86_pmu.intel_ctrl);
3f6da390 1760
2e80a82a 1761 perf_pmu_register(&pmu, "cpu", PERF_TYPE_RAW);
3f6da390 1762 perf_cpu_notifier(x86_pmu_notifier);
004417a6
PZ
1763
1764 return 0;
241771ef 1765}
004417a6 1766early_initcall(init_hw_perf_events);
621a01ea 1767
cdd6c482 1768static inline void x86_pmu_read(struct perf_event *event)
ee06094f 1769{
cc2ad4ba 1770 x86_perf_event_update(event);
ee06094f
IM
1771}
1772
4d1c52b0
LM
1773/*
1774 * Start group events scheduling transaction
1775 * Set the flag to make pmu::enable() not perform the
1776 * schedulability test, it will be performed at commit time
fbbe0701
SB
1777 *
1778 * We only support PERF_PMU_TXN_ADD transactions. Save the
1779 * transaction flags but otherwise ignore non-PERF_PMU_TXN_ADD
1780 * transactions.
4d1c52b0 1781 */
fbbe0701 1782static void x86_pmu_start_txn(struct pmu *pmu, unsigned int txn_flags)
4d1c52b0 1783{
fbbe0701
SB
1784 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1785
1786 WARN_ON_ONCE(cpuc->txn_flags); /* txn already in flight */
1787
1788 cpuc->txn_flags = txn_flags;
1789 if (txn_flags & ~PERF_PMU_TXN_ADD)
1790 return;
1791
33696fc0 1792 perf_pmu_disable(pmu);
0a3aee0d 1793 __this_cpu_write(cpu_hw_events.n_txn, 0);
4d1c52b0
LM
1794}
1795
1796/*
1797 * Stop group events scheduling transaction
1798 * Clear the flag and pmu::enable() will perform the
1799 * schedulability test.
1800 */
51b0fe39 1801static void x86_pmu_cancel_txn(struct pmu *pmu)
4d1c52b0 1802{
fbbe0701
SB
1803 unsigned int txn_flags;
1804 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1805
1806 WARN_ON_ONCE(!cpuc->txn_flags); /* no txn in flight */
1807
1808 txn_flags = cpuc->txn_flags;
1809 cpuc->txn_flags = 0;
1810 if (txn_flags & ~PERF_PMU_TXN_ADD)
1811 return;
1812
90151c35 1813 /*
c347a2f1
PZ
1814 * Truncate collected array by the number of events added in this
1815 * transaction. See x86_pmu_add() and x86_pmu_*_txn().
90151c35 1816 */
0a3aee0d
TH
1817 __this_cpu_sub(cpu_hw_events.n_added, __this_cpu_read(cpu_hw_events.n_txn));
1818 __this_cpu_sub(cpu_hw_events.n_events, __this_cpu_read(cpu_hw_events.n_txn));
33696fc0 1819 perf_pmu_enable(pmu);
4d1c52b0
LM
1820}
1821
1822/*
1823 * Commit group events scheduling transaction
1824 * Perform the group schedulability test as a whole
1825 * Return 0 if success
c347a2f1
PZ
1826 *
1827 * Does not cancel the transaction on failure; expects the caller to do this.
4d1c52b0 1828 */
51b0fe39 1829static int x86_pmu_commit_txn(struct pmu *pmu)
4d1c52b0 1830{
89cbc767 1831 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
4d1c52b0
LM
1832 int assign[X86_PMC_IDX_MAX];
1833 int n, ret;
1834
fbbe0701
SB
1835 WARN_ON_ONCE(!cpuc->txn_flags); /* no txn in flight */
1836
1837 if (cpuc->txn_flags & ~PERF_PMU_TXN_ADD) {
1838 cpuc->txn_flags = 0;
1839 return 0;
1840 }
1841
4d1c52b0
LM
1842 n = cpuc->n_events;
1843
1844 if (!x86_pmu_initialized())
1845 return -EAGAIN;
1846
1847 ret = x86_pmu.schedule_events(cpuc, n, assign);
1848 if (ret)
1849 return ret;
1850
1851 /*
1852 * copy new assignment, now we know it is possible
1853 * will be used by hw_perf_enable()
1854 */
1855 memcpy(cpuc->assign, assign, n*sizeof(int));
1856
fbbe0701 1857 cpuc->txn_flags = 0;
33696fc0 1858 perf_pmu_enable(pmu);
4d1c52b0
LM
1859 return 0;
1860}
cd8a38d3
SE
1861/*
1862 * a fake_cpuc is used to validate event groups. Due to
1863 * the extra reg logic, we need to also allocate a fake
1864 * per_core and per_cpu structure. Otherwise, group events
1865 * using extra reg may conflict without the kernel being
1866 * able to catch this when the last event gets added to
1867 * the group.
1868 */
1869static void free_fake_cpuc(struct cpu_hw_events *cpuc)
1870{
1871 kfree(cpuc->shared_regs);
1872 kfree(cpuc);
1873}
1874
1875static struct cpu_hw_events *allocate_fake_cpuc(void)
1876{
1877 struct cpu_hw_events *cpuc;
1878 int cpu = raw_smp_processor_id();
1879
1880 cpuc = kzalloc(sizeof(*cpuc), GFP_KERNEL);
1881 if (!cpuc)
1882 return ERR_PTR(-ENOMEM);
1883
1884 /* only needed, if we have extra_regs */
1885 if (x86_pmu.extra_regs) {
1886 cpuc->shared_regs = allocate_shared_regs(cpu);
1887 if (!cpuc->shared_regs)
1888 goto error;
1889 }
b430f7c4 1890 cpuc->is_fake = 1;
cd8a38d3
SE
1891 return cpuc;
1892error:
1893 free_fake_cpuc(cpuc);
1894 return ERR_PTR(-ENOMEM);
1895}
4d1c52b0 1896
ca037701
PZ
1897/*
1898 * validate that we can schedule this event
1899 */
1900static int validate_event(struct perf_event *event)
1901{
1902 struct cpu_hw_events *fake_cpuc;
1903 struct event_constraint *c;
1904 int ret = 0;
1905
cd8a38d3
SE
1906 fake_cpuc = allocate_fake_cpuc();
1907 if (IS_ERR(fake_cpuc))
1908 return PTR_ERR(fake_cpuc);
ca037701 1909
79cba822 1910 c = x86_pmu.get_event_constraints(fake_cpuc, -1, event);
ca037701
PZ
1911
1912 if (!c || !c->weight)
aa2bc1ad 1913 ret = -EINVAL;
ca037701
PZ
1914
1915 if (x86_pmu.put_event_constraints)
1916 x86_pmu.put_event_constraints(fake_cpuc, event);
1917
cd8a38d3 1918 free_fake_cpuc(fake_cpuc);
ca037701
PZ
1919
1920 return ret;
1921}
1922
1da53e02
SE
1923/*
1924 * validate a single event group
1925 *
1926 * validation include:
184f412c
IM
1927 * - check events are compatible which each other
1928 * - events do not compete for the same counter
1929 * - number of events <= number of counters
1da53e02
SE
1930 *
1931 * validation ensures the group can be loaded onto the
1932 * PMU if it was the only group available.
1933 */
fe9081cc
PZ
1934static int validate_group(struct perf_event *event)
1935{
1da53e02 1936 struct perf_event *leader = event->group_leader;
502568d5 1937 struct cpu_hw_events *fake_cpuc;
aa2bc1ad 1938 int ret = -EINVAL, n;
fe9081cc 1939
cd8a38d3
SE
1940 fake_cpuc = allocate_fake_cpuc();
1941 if (IS_ERR(fake_cpuc))
1942 return PTR_ERR(fake_cpuc);
1da53e02
SE
1943 /*
1944 * the event is not yet connected with its
1945 * siblings therefore we must first collect
1946 * existing siblings, then add the new event
1947 * before we can simulate the scheduling
1948 */
502568d5 1949 n = collect_events(fake_cpuc, leader, true);
1da53e02 1950 if (n < 0)
cd8a38d3 1951 goto out;
fe9081cc 1952
502568d5
PZ
1953 fake_cpuc->n_events = n;
1954 n = collect_events(fake_cpuc, event, false);
1da53e02 1955 if (n < 0)
cd8a38d3 1956 goto out;
fe9081cc 1957
502568d5 1958 fake_cpuc->n_events = n;
1da53e02 1959
a072738e 1960 ret = x86_pmu.schedule_events(fake_cpuc, n, NULL);
502568d5 1961
502568d5 1962out:
cd8a38d3 1963 free_fake_cpuc(fake_cpuc);
502568d5 1964 return ret;
fe9081cc
PZ
1965}
1966
dda99116 1967static int x86_pmu_event_init(struct perf_event *event)
621a01ea 1968{
51b0fe39 1969 struct pmu *tmp;
621a01ea
IM
1970 int err;
1971
b0a873eb
PZ
1972 switch (event->attr.type) {
1973 case PERF_TYPE_RAW:
1974 case PERF_TYPE_HARDWARE:
1975 case PERF_TYPE_HW_CACHE:
1976 break;
1977
1978 default:
1979 return -ENOENT;
1980 }
1981
1982 err = __x86_pmu_event_init(event);
fe9081cc 1983 if (!err) {
8113070d
SE
1984 /*
1985 * we temporarily connect event to its pmu
1986 * such that validate_group() can classify
1987 * it as an x86 event using is_x86_event()
1988 */
1989 tmp = event->pmu;
1990 event->pmu = &pmu;
1991
fe9081cc
PZ
1992 if (event->group_leader != event)
1993 err = validate_group(event);
ca037701
PZ
1994 else
1995 err = validate_event(event);
8113070d
SE
1996
1997 event->pmu = tmp;
fe9081cc 1998 }
a1792cda 1999 if (err) {
cdd6c482
IM
2000 if (event->destroy)
2001 event->destroy(event);
a1792cda 2002 }
621a01ea 2003
7911d3f7
AL
2004 if (ACCESS_ONCE(x86_pmu.attr_rdpmc))
2005 event->hw.flags |= PERF_X86_EVENT_RDPMC_ALLOWED;
2006
b0a873eb 2007 return err;
621a01ea 2008}
d7d59fb3 2009
7911d3f7
AL
2010static void refresh_pce(void *ignored)
2011{
2012 if (current->mm)
2013 load_mm_cr4(current->mm);
2014}
2015
2016static void x86_pmu_event_mapped(struct perf_event *event)
2017{
2018 if (!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED))
2019 return;
2020
2021 if (atomic_inc_return(&current->mm->context.perf_rdpmc_allowed) == 1)
2022 on_each_cpu_mask(mm_cpumask(current->mm), refresh_pce, NULL, 1);
2023}
2024
2025static void x86_pmu_event_unmapped(struct perf_event *event)
2026{
2027 if (!current->mm)
2028 return;
2029
2030 if (!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED))
2031 return;
2032
2033 if (atomic_dec_and_test(&current->mm->context.perf_rdpmc_allowed))
2034 on_each_cpu_mask(mm_cpumask(current->mm), refresh_pce, NULL, 1);
2035}
2036
fe4a3308
PZ
2037static int x86_pmu_event_idx(struct perf_event *event)
2038{
2039 int idx = event->hw.idx;
2040
7911d3f7 2041 if (!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED))
c7206205
PZ
2042 return 0;
2043
15c7ad51
RR
2044 if (x86_pmu.num_counters_fixed && idx >= INTEL_PMC_IDX_FIXED) {
2045 idx -= INTEL_PMC_IDX_FIXED;
fe4a3308
PZ
2046 idx |= 1 << 30;
2047 }
2048
2049 return idx + 1;
2050}
2051
0c9d42ed
PZ
2052static ssize_t get_attr_rdpmc(struct device *cdev,
2053 struct device_attribute *attr,
2054 char *buf)
2055{
2056 return snprintf(buf, 40, "%d\n", x86_pmu.attr_rdpmc);
2057}
2058
0c9d42ed
PZ
2059static ssize_t set_attr_rdpmc(struct device *cdev,
2060 struct device_attribute *attr,
2061 const char *buf, size_t count)
2062{
e2b297fc
SK
2063 unsigned long val;
2064 ssize_t ret;
2065
2066 ret = kstrtoul(buf, 0, &val);
2067 if (ret)
2068 return ret;
e97df763 2069
a6673429
AL
2070 if (val > 2)
2071 return -EINVAL;
2072
e97df763
PZ
2073 if (x86_pmu.attr_rdpmc_broken)
2074 return -ENOTSUPP;
0c9d42ed 2075
a6673429
AL
2076 if ((val == 2) != (x86_pmu.attr_rdpmc == 2)) {
2077 /*
2078 * Changing into or out of always available, aka
2079 * perf-event-bypassing mode. This path is extremely slow,
2080 * but only root can trigger it, so it's okay.
2081 */
2082 if (val == 2)
2083 static_key_slow_inc(&rdpmc_always_available);
2084 else
2085 static_key_slow_dec(&rdpmc_always_available);
2086 on_each_cpu(refresh_pce, NULL, 1);
2087 }
2088
2089 x86_pmu.attr_rdpmc = val;
2090
0c9d42ed
PZ
2091 return count;
2092}
2093
2094static DEVICE_ATTR(rdpmc, S_IRUSR | S_IWUSR, get_attr_rdpmc, set_attr_rdpmc);
2095
2096static struct attribute *x86_pmu_attrs[] = {
2097 &dev_attr_rdpmc.attr,
2098 NULL,
2099};
2100
2101static struct attribute_group x86_pmu_attr_group = {
2102 .attrs = x86_pmu_attrs,
2103};
2104
2105static const struct attribute_group *x86_pmu_attr_groups[] = {
2106 &x86_pmu_attr_group,
641cc938 2107 &x86_pmu_format_group,
a4747393 2108 &x86_pmu_events_group,
0c9d42ed
PZ
2109 NULL,
2110};
2111
ba532500 2112static void x86_pmu_sched_task(struct perf_event_context *ctx, bool sched_in)
d010b332 2113{
ba532500
YZ
2114 if (x86_pmu.sched_task)
2115 x86_pmu.sched_task(ctx, sched_in);
d010b332
SE
2116}
2117
c93dc84c
PZ
2118void perf_check_microcode(void)
2119{
2120 if (x86_pmu.check_microcode)
2121 x86_pmu.check_microcode();
2122}
2123EXPORT_SYMBOL_GPL(perf_check_microcode);
2124
b0a873eb 2125static struct pmu pmu = {
d010b332
SE
2126 .pmu_enable = x86_pmu_enable,
2127 .pmu_disable = x86_pmu_disable,
a4eaf7f1 2128
c93dc84c 2129 .attr_groups = x86_pmu_attr_groups,
0c9d42ed 2130
c93dc84c 2131 .event_init = x86_pmu_event_init,
a4eaf7f1 2132
7911d3f7
AL
2133 .event_mapped = x86_pmu_event_mapped,
2134 .event_unmapped = x86_pmu_event_unmapped,
2135
d010b332
SE
2136 .add = x86_pmu_add,
2137 .del = x86_pmu_del,
2138 .start = x86_pmu_start,
2139 .stop = x86_pmu_stop,
2140 .read = x86_pmu_read,
a4eaf7f1 2141
c93dc84c
PZ
2142 .start_txn = x86_pmu_start_txn,
2143 .cancel_txn = x86_pmu_cancel_txn,
2144 .commit_txn = x86_pmu_commit_txn,
fe4a3308 2145
c93dc84c 2146 .event_idx = x86_pmu_event_idx,
ba532500 2147 .sched_task = x86_pmu_sched_task,
e18bf526 2148 .task_ctx_size = sizeof(struct x86_perf_task_context),
b0a873eb
PZ
2149};
2150
c1317ec2
AL
2151void arch_perf_update_userpage(struct perf_event *event,
2152 struct perf_event_mmap_page *userpg, u64 now)
e3f3541c 2153{
20d1c86a
PZ
2154 struct cyc2ns_data *data;
2155
fa731587
PZ
2156 userpg->cap_user_time = 0;
2157 userpg->cap_user_time_zero = 0;
7911d3f7
AL
2158 userpg->cap_user_rdpmc =
2159 !!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED);
c7206205
PZ
2160 userpg->pmc_width = x86_pmu.cntval_bits;
2161
35af99e6 2162 if (!sched_clock_stable())
e3f3541c
PZ
2163 return;
2164
20d1c86a
PZ
2165 data = cyc2ns_read_begin();
2166
34f43927
PZ
2167 /*
2168 * Internal timekeeping for enabled/running/stopped times
2169 * is always in the local_clock domain.
2170 */
fa731587 2171 userpg->cap_user_time = 1;
20d1c86a
PZ
2172 userpg->time_mult = data->cyc2ns_mul;
2173 userpg->time_shift = data->cyc2ns_shift;
2174 userpg->time_offset = data->cyc2ns_offset - now;
c73deb6a 2175
34f43927
PZ
2176 /*
2177 * cap_user_time_zero doesn't make sense when we're using a different
2178 * time base for the records.
2179 */
2180 if (event->clock == &local_clock) {
2181 userpg->cap_user_time_zero = 1;
2182 userpg->time_zero = data->cyc2ns_offset;
2183 }
20d1c86a
PZ
2184
2185 cyc2ns_read_end(data);
e3f3541c
PZ
2186}
2187
d7d59fb3
PZ
2188/*
2189 * callchain support
2190 */
2191
d7d59fb3
PZ
2192static int backtrace_stack(void *data, char *name)
2193{
038e836e 2194 return 0;
d7d59fb3
PZ
2195}
2196
568b329a 2197static int backtrace_address(void *data, unsigned long addr, int reliable)
d7d59fb3
PZ
2198{
2199 struct perf_callchain_entry *entry = data;
2200
568b329a 2201 return perf_callchain_store(entry, addr);
d7d59fb3
PZ
2202}
2203
2204static const struct stacktrace_ops backtrace_ops = {
d7d59fb3
PZ
2205 .stack = backtrace_stack,
2206 .address = backtrace_address,
06d65bda 2207 .walk_stack = print_context_stack_bp,
d7d59fb3
PZ
2208};
2209
56962b44
FW
2210void
2211perf_callchain_kernel(struct perf_callchain_entry *entry, struct pt_regs *regs)
d7d59fb3 2212{
927c7a9e
FW
2213 if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
2214 /* TODO: We don't support guest os callchain now */
ed805261 2215 return;
927c7a9e
FW
2216 }
2217
70791ce9 2218 perf_callchain_store(entry, regs->ip);
d7d59fb3 2219
e8e999cf 2220 dump_trace(NULL, regs, NULL, 0, &backtrace_ops, entry);
d7d59fb3
PZ
2221}
2222
bc6ca7b3
AS
2223static inline int
2224valid_user_frame(const void __user *fp, unsigned long size)
2225{
2226 return (__range_not_ok(fp, size, TASK_SIZE) == 0);
2227}
2228
d07bdfd3
PZ
2229static unsigned long get_segment_base(unsigned int segment)
2230{
2231 struct desc_struct *desc;
2232 int idx = segment >> 3;
2233
2234 if ((segment & SEGMENT_TI_MASK) == SEGMENT_LDT) {
a5b9e5a2 2235#ifdef CONFIG_MODIFY_LDT_SYSCALL
37868fe1
AL
2236 struct ldt_struct *ldt;
2237
d07bdfd3
PZ
2238 if (idx > LDT_ENTRIES)
2239 return 0;
2240
37868fe1
AL
2241 /* IRQs are off, so this synchronizes with smp_store_release */
2242 ldt = lockless_dereference(current->active_mm->context.ldt);
2243 if (!ldt || idx > ldt->size)
d07bdfd3
PZ
2244 return 0;
2245
37868fe1 2246 desc = &ldt->entries[idx];
a5b9e5a2
AL
2247#else
2248 return 0;
2249#endif
d07bdfd3
PZ
2250 } else {
2251 if (idx > GDT_ENTRIES)
2252 return 0;
2253
37868fe1 2254 desc = raw_cpu_ptr(gdt_page.gdt) + idx;
d07bdfd3
PZ
2255 }
2256
37868fe1 2257 return get_desc_base(desc);
d07bdfd3
PZ
2258}
2259
10ed3493 2260#ifdef CONFIG_IA32_EMULATION
d1a797f3
PA
2261
2262#include <asm/compat.h>
2263
257ef9d2
TE
2264static inline int
2265perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry *entry)
74193ef0 2266{
257ef9d2 2267 /* 32-bit process in 64-bit kernel. */
d07bdfd3 2268 unsigned long ss_base, cs_base;
257ef9d2
TE
2269 struct stack_frame_ia32 frame;
2270 const void __user *fp;
74193ef0 2271
257ef9d2
TE
2272 if (!test_thread_flag(TIF_IA32))
2273 return 0;
2274
d07bdfd3
PZ
2275 cs_base = get_segment_base(regs->cs);
2276 ss_base = get_segment_base(regs->ss);
2277
2278 fp = compat_ptr(ss_base + regs->bp);
75925e1a 2279 pagefault_disable();
257ef9d2
TE
2280 while (entry->nr < PERF_MAX_STACK_DEPTH) {
2281 unsigned long bytes;
2282 frame.next_frame = 0;
2283 frame.return_address = 0;
2284
75925e1a
AK
2285 if (!access_ok(VERIFY_READ, fp, 8))
2286 break;
2287
2288 bytes = __copy_from_user_nmi(&frame.next_frame, fp, 4);
2289 if (bytes != 0)
2290 break;
2291 bytes = __copy_from_user_nmi(&frame.return_address, fp+4, 4);
0a196848 2292 if (bytes != 0)
257ef9d2 2293 break;
74193ef0 2294
bc6ca7b3
AS
2295 if (!valid_user_frame(fp, sizeof(frame)))
2296 break;
2297
d07bdfd3
PZ
2298 perf_callchain_store(entry, cs_base + frame.return_address);
2299 fp = compat_ptr(ss_base + frame.next_frame);
257ef9d2 2300 }
75925e1a 2301 pagefault_enable();
257ef9d2 2302 return 1;
d7d59fb3 2303}
257ef9d2
TE
2304#else
2305static inline int
2306perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry *entry)
2307{
2308 return 0;
2309}
2310#endif
d7d59fb3 2311
56962b44
FW
2312void
2313perf_callchain_user(struct perf_callchain_entry *entry, struct pt_regs *regs)
d7d59fb3
PZ
2314{
2315 struct stack_frame frame;
2316 const void __user *fp;
2317
927c7a9e
FW
2318 if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
2319 /* TODO: We don't support guest os callchain now */
ed805261 2320 return;
927c7a9e 2321 }
5a6cec3a 2322
d07bdfd3
PZ
2323 /*
2324 * We don't know what to do with VM86 stacks.. ignore them for now.
2325 */
2326 if (regs->flags & (X86_VM_MASK | PERF_EFLAGS_VM))
2327 return;
2328
74193ef0 2329 fp = (void __user *)regs->bp;
d7d59fb3 2330
70791ce9 2331 perf_callchain_store(entry, regs->ip);
d7d59fb3 2332
20afc60f
AV
2333 if (!current->mm)
2334 return;
2335
257ef9d2
TE
2336 if (perf_callchain_user32(regs, entry))
2337 return;
2338
75925e1a 2339 pagefault_disable();
f9188e02 2340 while (entry->nr < PERF_MAX_STACK_DEPTH) {
257ef9d2 2341 unsigned long bytes;
038e836e 2342 frame.next_frame = NULL;
d7d59fb3
PZ
2343 frame.return_address = 0;
2344
75925e1a
AK
2345 if (!access_ok(VERIFY_READ, fp, 16))
2346 break;
2347
2348 bytes = __copy_from_user_nmi(&frame.next_frame, fp, 8);
2349 if (bytes != 0)
2350 break;
2351 bytes = __copy_from_user_nmi(&frame.return_address, fp+8, 8);
0a196848 2352 if (bytes != 0)
d7d59fb3
PZ
2353 break;
2354
bc6ca7b3
AS
2355 if (!valid_user_frame(fp, sizeof(frame)))
2356 break;
2357
70791ce9 2358 perf_callchain_store(entry, frame.return_address);
75925e1a 2359 fp = (void __user *)frame.next_frame;
d7d59fb3 2360 }
75925e1a 2361 pagefault_enable();
d7d59fb3
PZ
2362}
2363
d07bdfd3
PZ
2364/*
2365 * Deal with code segment offsets for the various execution modes:
2366 *
2367 * VM86 - the good olde 16 bit days, where the linear address is
2368 * 20 bits and we use regs->ip + 0x10 * regs->cs.
2369 *
2370 * IA32 - Where we need to look at GDT/LDT segment descriptor tables
2371 * to figure out what the 32bit base address is.
2372 *
2373 * X32 - has TIF_X32 set, but is running in x86_64
2374 *
2375 * X86_64 - CS,DS,SS,ES are all zero based.
2376 */
2377static unsigned long code_segment_base(struct pt_regs *regs)
39447b38 2378{
383f3af3
AL
2379 /*
2380 * For IA32 we look at the GDT/LDT segment base to convert the
2381 * effective IP to a linear address.
2382 */
2383
2384#ifdef CONFIG_X86_32
d07bdfd3
PZ
2385 /*
2386 * If we are in VM86 mode, add the segment offset to convert to a
2387 * linear address.
2388 */
2389 if (regs->flags & X86_VM_MASK)
2390 return 0x10 * regs->cs;
2391
55474c48 2392 if (user_mode(regs) && regs->cs != __USER_CS)
d07bdfd3
PZ
2393 return get_segment_base(regs->cs);
2394#else
c56716af
AL
2395 if (user_mode(regs) && !user_64bit_mode(regs) &&
2396 regs->cs != __USER32_CS)
2397 return get_segment_base(regs->cs);
d07bdfd3
PZ
2398#endif
2399 return 0;
2400}
dcf46b94 2401
d07bdfd3
PZ
2402unsigned long perf_instruction_pointer(struct pt_regs *regs)
2403{
39447b38 2404 if (perf_guest_cbs && perf_guest_cbs->is_in_guest())
d07bdfd3 2405 return perf_guest_cbs->get_guest_ip();
dcf46b94 2406
d07bdfd3 2407 return regs->ip + code_segment_base(regs);
39447b38
ZY
2408}
2409
2410unsigned long perf_misc_flags(struct pt_regs *regs)
2411{
2412 int misc = 0;
dcf46b94 2413
39447b38 2414 if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
dcf46b94
ZY
2415 if (perf_guest_cbs->is_user_mode())
2416 misc |= PERF_RECORD_MISC_GUEST_USER;
2417 else
2418 misc |= PERF_RECORD_MISC_GUEST_KERNEL;
2419 } else {
d07bdfd3 2420 if (user_mode(regs))
dcf46b94
ZY
2421 misc |= PERF_RECORD_MISC_USER;
2422 else
2423 misc |= PERF_RECORD_MISC_KERNEL;
2424 }
2425
39447b38 2426 if (regs->flags & PERF_EFLAGS_EXACT)
ab608344 2427 misc |= PERF_RECORD_MISC_EXACT_IP;
39447b38
ZY
2428
2429 return misc;
2430}
b3d9468a
GN
2431
2432void perf_get_x86_pmu_capability(struct x86_pmu_capability *cap)
2433{
2434 cap->version = x86_pmu.version;
2435 cap->num_counters_gp = x86_pmu.num_counters;
2436 cap->num_counters_fixed = x86_pmu.num_counters_fixed;
2437 cap->bit_width_gp = x86_pmu.cntval_bits;
2438 cap->bit_width_fixed = x86_pmu.cntval_bits;
2439 cap->events_mask = (unsigned int)x86_pmu.events_maskl;
2440 cap->events_mask_len = x86_pmu.events_mask_len;
2441}
2442EXPORT_SYMBOL_GPL(perf_get_x86_pmu_capability);
This page took 0.615317 seconds and 5 git commands to generate.