2 * BTS PMU driver for perf
3 * Copyright (c) 2013-2014, Intel Corporation.
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
9 * This program is distributed in the hope it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
17 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
19 #include <linux/bitops.h>
20 #include <linux/types.h>
21 #include <linux/slab.h>
22 #include <linux/debugfs.h>
23 #include <linux/device.h>
24 #include <linux/coredump.h>
26 #include <asm-generic/sizes.h>
27 #include <asm/perf_event.h>
29 #include "../perf_event.h"
32 struct perf_output_handle handle
;
33 struct debug_store ds_back
;
37 static DEFINE_PER_CPU(struct bts_ctx
, bts_ctx
);
39 #define BTS_RECORD_SIZE 24
40 #define BTS_SAFETY_MARGIN 4080
46 unsigned long displacement
;
50 size_t real_size
; /* multiple of BTS_RECORD_SIZE */
51 unsigned int nr_pages
;
60 struct bts_phys buf
[0];
65 static size_t buf_size(struct page
*page
)
67 return 1 << (PAGE_SHIFT
+ page_private(page
));
71 bts_buffer_setup_aux(int cpu
, void **pages
, int nr_pages
, bool overwrite
)
73 struct bts_buffer
*buf
;
75 int node
= (cpu
== -1) ? cpu
: cpu_to_node(cpu
);
77 size_t size
= nr_pages
<< PAGE_SHIFT
;
80 /* count all the high order buffers */
81 for (pg
= 0, nbuf
= 0; pg
< nr_pages
;) {
82 page
= virt_to_page(pages
[pg
]);
83 if (WARN_ON_ONCE(!PagePrivate(page
) && nr_pages
> 1))
85 pg
+= 1 << page_private(page
);
90 * to avoid interrupts in overwrite mode, only allow one physical
92 if (overwrite
&& nbuf
> 1)
95 buf
= kzalloc_node(offsetof(struct bts_buffer
, buf
[nbuf
]), GFP_KERNEL
, node
);
99 buf
->nr_pages
= nr_pages
;
101 buf
->snapshot
= overwrite
;
102 buf
->data_pages
= pages
;
103 buf
->real_size
= size
- size
% BTS_RECORD_SIZE
;
105 for (pg
= 0, nbuf
= 0, offset
= 0, pad
= 0; nbuf
< buf
->nr_bufs
; nbuf
++) {
106 unsigned int __nr_pages
;
108 page
= virt_to_page(pages
[pg
]);
109 __nr_pages
= PagePrivate(page
) ? 1 << page_private(page
) : 1;
110 buf
->buf
[nbuf
].page
= page
;
111 buf
->buf
[nbuf
].offset
= offset
;
112 buf
->buf
[nbuf
].displacement
= (pad
? BTS_RECORD_SIZE
- pad
: 0);
113 buf
->buf
[nbuf
].size
= buf_size(page
) - buf
->buf
[nbuf
].displacement
;
114 pad
= buf
->buf
[nbuf
].size
% BTS_RECORD_SIZE
;
115 buf
->buf
[nbuf
].size
-= pad
;
118 offset
+= __nr_pages
<< PAGE_SHIFT
;
124 static void bts_buffer_free_aux(void *data
)
129 static unsigned long bts_buffer_offset(struct bts_buffer
*buf
, unsigned int idx
)
131 return buf
->buf
[idx
].offset
+ buf
->buf
[idx
].displacement
;
135 bts_config_buffer(struct bts_buffer
*buf
)
137 int cpu
= raw_smp_processor_id();
138 struct debug_store
*ds
= per_cpu(cpu_hw_events
, cpu
).ds
;
139 struct bts_phys
*phys
= &buf
->buf
[buf
->cur_buf
];
140 unsigned long index
, thresh
= 0, end
= phys
->size
;
141 struct page
*page
= phys
->page
;
143 index
= local_read(&buf
->head
);
145 if (!buf
->snapshot
) {
146 if (buf
->end
< phys
->offset
+ buf_size(page
))
147 end
= buf
->end
- phys
->offset
- phys
->displacement
;
149 index
-= phys
->offset
+ phys
->displacement
;
151 if (end
- index
> BTS_SAFETY_MARGIN
)
152 thresh
= end
- BTS_SAFETY_MARGIN
;
153 else if (end
- index
> BTS_RECORD_SIZE
)
154 thresh
= end
- BTS_RECORD_SIZE
;
159 ds
->bts_buffer_base
= (u64
)(long)page_address(page
) + phys
->displacement
;
160 ds
->bts_index
= ds
->bts_buffer_base
+ index
;
161 ds
->bts_absolute_maximum
= ds
->bts_buffer_base
+ end
;
162 ds
->bts_interrupt_threshold
= !buf
->snapshot
163 ? ds
->bts_buffer_base
+ thresh
164 : ds
->bts_absolute_maximum
+ BTS_RECORD_SIZE
;
167 static void bts_buffer_pad_out(struct bts_phys
*phys
, unsigned long head
)
169 unsigned long index
= head
- phys
->offset
;
171 memset(page_address(phys
->page
) + index
, 0, phys
->size
- index
);
174 static void bts_update(struct bts_ctx
*bts
)
176 int cpu
= raw_smp_processor_id();
177 struct debug_store
*ds
= per_cpu(cpu_hw_events
, cpu
).ds
;
178 struct bts_buffer
*buf
= perf_get_aux(&bts
->handle
);
179 unsigned long index
= ds
->bts_index
- ds
->bts_buffer_base
, old
, head
;
184 head
= index
+ bts_buffer_offset(buf
, buf
->cur_buf
);
185 old
= local_xchg(&buf
->head
, head
);
187 if (!buf
->snapshot
) {
191 if (ds
->bts_index
>= ds
->bts_absolute_maximum
)
192 local_inc(&buf
->lost
);
195 * old and head are always in the same physical buffer, so we
196 * can subtract them to get the data size.
198 local_add(head
- old
, &buf
->data_size
);
200 local_set(&buf
->data_size
, head
);
205 bts_buffer_reset(struct bts_buffer
*buf
, struct perf_output_handle
*handle
);
207 static void __bts_event_start(struct perf_event
*event
)
209 struct bts_ctx
*bts
= this_cpu_ptr(&bts_ctx
);
210 struct bts_buffer
*buf
= perf_get_aux(&bts
->handle
);
214 config
|= ARCH_PERFMON_EVENTSEL_INT
;
215 if (!event
->attr
.exclude_kernel
)
216 config
|= ARCH_PERFMON_EVENTSEL_OS
;
217 if (!event
->attr
.exclude_user
)
218 config
|= ARCH_PERFMON_EVENTSEL_USR
;
220 bts_config_buffer(buf
);
223 * local barrier to make sure that ds configuration made it
224 * before we enable BTS
228 intel_pmu_enable_bts(config
);
232 static void bts_event_start(struct perf_event
*event
, int flags
)
234 struct cpu_hw_events
*cpuc
= this_cpu_ptr(&cpu_hw_events
);
235 struct bts_ctx
*bts
= this_cpu_ptr(&bts_ctx
);
236 struct bts_buffer
*buf
;
238 buf
= perf_aux_output_begin(&bts
->handle
, event
);
242 if (bts_buffer_reset(buf
, &bts
->handle
))
245 bts
->ds_back
.bts_buffer_base
= cpuc
->ds
->bts_buffer_base
;
246 bts
->ds_back
.bts_absolute_maximum
= cpuc
->ds
->bts_absolute_maximum
;
247 bts
->ds_back
.bts_interrupt_threshold
= cpuc
->ds
->bts_interrupt_threshold
;
249 event
->hw
.itrace_started
= 1;
252 __bts_event_start(event
);
254 /* PMI handler: this counter is running and likely generating PMIs */
255 ACCESS_ONCE(bts
->started
) = 1;
260 perf_aux_output_end(&bts
->handle
, 0, false);
263 event
->hw
.state
= PERF_HES_STOPPED
;
266 static void __bts_event_stop(struct perf_event
*event
)
269 * No extra synchronization is mandated by the documentation to have
270 * BTS data stores globally visible.
272 intel_pmu_disable_bts();
274 if (event
->hw
.state
& PERF_HES_STOPPED
)
277 ACCESS_ONCE(event
->hw
.state
) |= PERF_HES_STOPPED
;
280 static void bts_event_stop(struct perf_event
*event
, int flags
)
282 struct cpu_hw_events
*cpuc
= this_cpu_ptr(&cpu_hw_events
);
283 struct bts_ctx
*bts
= this_cpu_ptr(&bts_ctx
);
284 struct bts_buffer
*buf
= perf_get_aux(&bts
->handle
);
286 /* PMI handler: don't restart this counter */
287 ACCESS_ONCE(bts
->started
) = 0;
289 __bts_event_stop(event
);
291 if (flags
& PERF_EF_UPDATE
) {
297 local_xchg(&buf
->data_size
,
298 buf
->nr_pages
<< PAGE_SHIFT
);
299 perf_aux_output_end(&bts
->handle
, local_xchg(&buf
->data_size
, 0),
300 !!local_xchg(&buf
->lost
, 0));
303 cpuc
->ds
->bts_index
= bts
->ds_back
.bts_buffer_base
;
304 cpuc
->ds
->bts_buffer_base
= bts
->ds_back
.bts_buffer_base
;
305 cpuc
->ds
->bts_absolute_maximum
= bts
->ds_back
.bts_absolute_maximum
;
306 cpuc
->ds
->bts_interrupt_threshold
= bts
->ds_back
.bts_interrupt_threshold
;
310 void intel_bts_enable_local(void)
312 struct bts_ctx
*bts
= this_cpu_ptr(&bts_ctx
);
314 if (bts
->handle
.event
&& bts
->started
)
315 __bts_event_start(bts
->handle
.event
);
318 void intel_bts_disable_local(void)
320 struct bts_ctx
*bts
= this_cpu_ptr(&bts_ctx
);
322 if (bts
->handle
.event
)
323 __bts_event_stop(bts
->handle
.event
);
327 bts_buffer_reset(struct bts_buffer
*buf
, struct perf_output_handle
*handle
)
329 unsigned long head
, space
, next_space
, pad
, gap
, skip
, wakeup
;
330 unsigned int next_buf
;
331 struct bts_phys
*phys
, *next_phys
;
337 head
= handle
->head
& ((buf
->nr_pages
<< PAGE_SHIFT
) - 1);
338 if (WARN_ON_ONCE(head
!= local_read(&buf
->head
)))
341 phys
= &buf
->buf
[buf
->cur_buf
];
342 space
= phys
->offset
+ phys
->displacement
+ phys
->size
- head
;
344 if (space
> handle
->size
) {
345 space
= handle
->size
;
346 space
-= space
% BTS_RECORD_SIZE
;
348 if (space
<= BTS_SAFETY_MARGIN
) {
349 /* See if next phys buffer has more space */
350 next_buf
= buf
->cur_buf
+ 1;
351 if (next_buf
>= buf
->nr_bufs
)
353 next_phys
= &buf
->buf
[next_buf
];
354 gap
= buf_size(phys
->page
) - phys
->displacement
- phys
->size
+
355 next_phys
->displacement
;
357 if (handle
->size
>= skip
) {
358 next_space
= next_phys
->size
;
359 if (next_space
+ skip
> handle
->size
) {
360 next_space
= handle
->size
- skip
;
361 next_space
-= next_space
% BTS_RECORD_SIZE
;
363 if (next_space
> space
|| !space
) {
365 bts_buffer_pad_out(phys
, head
);
366 ret
= perf_aux_output_skip(handle
, skip
);
369 /* Advance to next phys buffer */
372 head
= phys
->offset
+ phys
->displacement
;
374 * After this, cur_buf and head won't match ds
375 * anymore, so we must not be racing with
378 buf
->cur_buf
= next_buf
;
379 local_set(&buf
->head
, head
);
384 /* Don't go far beyond wakeup watermark */
385 wakeup
= BTS_SAFETY_MARGIN
+ BTS_RECORD_SIZE
+ handle
->wakeup
-
387 if (space
> wakeup
) {
389 space
-= space
% BTS_RECORD_SIZE
;
392 buf
->end
= head
+ space
;
395 * If we have no space, the lost notification would have been sent when
396 * we hit absolute_maximum - see bts_update()
404 int intel_bts_interrupt(void)
406 struct bts_ctx
*bts
= this_cpu_ptr(&bts_ctx
);
407 struct perf_event
*event
= bts
->handle
.event
;
408 struct bts_buffer
*buf
;
412 if (!event
|| !bts
->started
)
415 buf
= perf_get_aux(&bts
->handle
);
417 * Skip snapshot counters: they don't use the interrupt, but
418 * there's no other way of telling, because the pointer will
421 if (!buf
|| buf
->snapshot
)
424 old_head
= local_read(&buf
->head
);
428 if (old_head
== local_read(&buf
->head
))
431 perf_aux_output_end(&bts
->handle
, local_xchg(&buf
->data_size
, 0),
432 !!local_xchg(&buf
->lost
, 0));
434 buf
= perf_aux_output_begin(&bts
->handle
, event
);
438 err
= bts_buffer_reset(buf
, &bts
->handle
);
440 perf_aux_output_end(&bts
->handle
, 0, false);
445 static void bts_event_del(struct perf_event
*event
, int mode
)
447 bts_event_stop(event
, PERF_EF_UPDATE
);
450 static int bts_event_add(struct perf_event
*event
, int mode
)
452 struct bts_ctx
*bts
= this_cpu_ptr(&bts_ctx
);
453 struct cpu_hw_events
*cpuc
= this_cpu_ptr(&cpu_hw_events
);
454 struct hw_perf_event
*hwc
= &event
->hw
;
456 event
->hw
.state
= PERF_HES_STOPPED
;
458 if (test_bit(INTEL_PMC_IDX_FIXED_BTS
, cpuc
->active_mask
))
461 if (bts
->handle
.event
)
464 if (mode
& PERF_EF_START
) {
465 bts_event_start(event
, 0);
466 if (hwc
->state
& PERF_HES_STOPPED
)
473 static void bts_event_destroy(struct perf_event
*event
)
475 x86_release_hardware();
476 x86_del_exclusive(x86_lbr_exclusive_bts
);
479 static int bts_event_init(struct perf_event
*event
)
483 if (event
->attr
.type
!= bts_pmu
.type
)
486 if (x86_add_exclusive(x86_lbr_exclusive_bts
))
490 * BTS leaks kernel addresses even when CPL0 tracing is
491 * disabled, so disallow intel_bts driver for unprivileged
492 * users on paranoid systems since it provides trace data
493 * to the user in a zero-copy fashion.
495 * Note that the default paranoia setting permits unprivileged
496 * users to profile the kernel.
498 if (event
->attr
.exclude_kernel
&& perf_paranoid_kernel() &&
499 !capable(CAP_SYS_ADMIN
))
502 ret
= x86_reserve_hardware();
504 x86_del_exclusive(x86_lbr_exclusive_bts
);
508 event
->destroy
= bts_event_destroy
;
513 static void bts_event_read(struct perf_event
*event
)
517 static __init
int bts_init(void)
519 if (!boot_cpu_has(X86_FEATURE_DTES64
) || !x86_pmu
.bts
)
522 bts_pmu
.capabilities
= PERF_PMU_CAP_AUX_NO_SG
| PERF_PMU_CAP_ITRACE
;
523 bts_pmu
.task_ctx_nr
= perf_sw_context
;
524 bts_pmu
.event_init
= bts_event_init
;
525 bts_pmu
.add
= bts_event_add
;
526 bts_pmu
.del
= bts_event_del
;
527 bts_pmu
.start
= bts_event_start
;
528 bts_pmu
.stop
= bts_event_stop
;
529 bts_pmu
.read
= bts_event_read
;
530 bts_pmu
.setup_aux
= bts_buffer_setup_aux
;
531 bts_pmu
.free_aux
= bts_buffer_free_aux
;
533 return perf_pmu_register(&bts_pmu
, "intel_bts", -1);
535 arch_initcall(bts_init
);