2 * Performance events ring-buffer code:
4 * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
5 * Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar
6 * Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
7 * Copyright © 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
9 * For licensing details see kernel-base/COPYING
12 #include <linux/perf_event.h>
13 #include <linux/vmalloc.h>
14 #include <linux/slab.h>
15 #include <linux/circ_buf.h>
16 #include <linux/poll.h>
20 static void perf_output_wakeup(struct perf_output_handle
*handle
)
22 atomic_set(&handle
->rb
->poll
, POLLIN
);
24 handle
->event
->pending_wakeup
= 1;
25 irq_work_queue(&handle
->event
->pending
);
29 * We need to ensure a later event_id doesn't publish a head when a former
30 * event isn't done writing. However since we need to deal with NMIs we
31 * cannot fully serialize things.
33 * We only publish the head (and generate a wakeup) when the outer-most
36 static void perf_output_get_handle(struct perf_output_handle
*handle
)
38 struct ring_buffer
*rb
= handle
->rb
;
42 handle
->wakeup
= local_read(&rb
->wakeup
);
45 static void perf_output_put_handle(struct perf_output_handle
*handle
)
47 struct ring_buffer
*rb
= handle
->rb
;
51 head
= local_read(&rb
->head
);
54 * IRQ/NMI can happen here, which means we can miss a head update.
57 if (!local_dec_and_test(&rb
->nest
))
61 * Since the mmap() consumer (userspace) can run on a different CPU:
65 * if (LOAD ->data_tail) { LOAD ->data_head
67 * STORE $data LOAD $data
68 * smp_wmb() (B) smp_mb() (D)
69 * STORE ->data_head STORE ->data_tail
72 * Where A pairs with D, and B pairs with C.
74 * In our case (A) is a control dependency that separates the load of
75 * the ->data_tail and the stores of $data. In case ->data_tail
76 * indicates there is no room in the buffer to store $data we do not.
78 * D needs to be a full barrier since it separates the data READ
79 * from the tail WRITE.
81 * For B a WMB is sufficient since it separates two WRITEs, and for C
82 * an RMB is sufficient since it separates two READs.
84 * See perf_output_begin().
86 smp_wmb(); /* B, matches C */
87 rb
->user_page
->data_head
= head
;
90 * Now check if we missed an update -- rely on previous implied
91 * compiler barriers to force a re-read.
93 if (unlikely(head
!= local_read(&rb
->head
))) {
98 if (handle
->wakeup
!= local_read(&rb
->wakeup
))
99 perf_output_wakeup(handle
);
105 int perf_output_begin(struct perf_output_handle
*handle
,
106 struct perf_event
*event
, unsigned int size
)
108 struct ring_buffer
*rb
;
109 unsigned long tail
, offset
, head
;
110 int have_lost
, page_shift
;
112 struct perf_event_header header
;
119 * For inherited events we send all the output towards the parent.
122 event
= event
->parent
;
124 rb
= rcu_dereference(event
->rb
);
128 if (unlikely(!rb
->nr_pages
))
132 handle
->event
= event
;
134 have_lost
= local_read(&rb
->lost
);
135 if (unlikely(have_lost
)) {
136 size
+= sizeof(lost_event
);
137 if (event
->attr
.sample_id_all
)
138 size
+= event
->id_header_size
;
141 perf_output_get_handle(handle
);
144 tail
= ACCESS_ONCE(rb
->user_page
->data_tail
);
145 offset
= head
= local_read(&rb
->head
);
146 if (!rb
->overwrite
&&
147 unlikely(CIRC_SPACE(head
, tail
, perf_data_size(rb
)) < size
))
151 * The above forms a control dependency barrier separating the
152 * @tail load above from the data stores below. Since the @tail
153 * load is required to compute the branch to fail below.
155 * A, matches D; the full memory barrier userspace SHOULD issue
156 * after reading the data and before storing the new tail
159 * See perf_output_put_handle().
163 } while (local_cmpxchg(&rb
->head
, offset
, head
) != offset
);
166 * We rely on the implied barrier() by local_cmpxchg() to ensure
167 * none of the data stores below can be lifted up by the compiler.
170 if (unlikely(head
- local_read(&rb
->wakeup
) > rb
->watermark
))
171 local_add(rb
->watermark
, &rb
->wakeup
);
173 page_shift
= PAGE_SHIFT
+ page_order(rb
);
175 handle
->page
= (offset
>> page_shift
) & (rb
->nr_pages
- 1);
176 offset
&= (1UL << page_shift
) - 1;
177 handle
->addr
= rb
->data_pages
[handle
->page
] + offset
;
178 handle
->size
= (1UL << page_shift
) - offset
;
180 if (unlikely(have_lost
)) {
181 struct perf_sample_data sample_data
;
183 lost_event
.header
.size
= sizeof(lost_event
);
184 lost_event
.header
.type
= PERF_RECORD_LOST
;
185 lost_event
.header
.misc
= 0;
186 lost_event
.id
= event
->id
;
187 lost_event
.lost
= local_xchg(&rb
->lost
, 0);
189 perf_event_header__init_id(&lost_event
.header
,
190 &sample_data
, event
);
191 perf_output_put(handle
, lost_event
);
192 perf_event__output_id_sample(event
, handle
, &sample_data
);
198 local_inc(&rb
->lost
);
199 perf_output_put_handle(handle
);
206 unsigned int perf_output_copy(struct perf_output_handle
*handle
,
207 const void *buf
, unsigned int len
)
209 return __output_copy(handle
, buf
, len
);
212 unsigned int perf_output_skip(struct perf_output_handle
*handle
,
215 return __output_skip(handle
, NULL
, len
);
218 void perf_output_end(struct perf_output_handle
*handle
)
220 perf_output_put_handle(handle
);
225 ring_buffer_init(struct ring_buffer
*rb
, long watermark
, int flags
)
227 long max_size
= perf_data_size(rb
);
230 rb
->watermark
= min(max_size
, watermark
);
233 rb
->watermark
= max_size
/ 2;
235 if (flags
& RING_BUFFER_WRITABLE
)
240 atomic_set(&rb
->refcount
, 1);
242 INIT_LIST_HEAD(&rb
->event_list
);
243 spin_lock_init(&rb
->event_lock
);
247 * This is called before hardware starts writing to the AUX area to
248 * obtain an output handle and make sure there's room in the buffer.
249 * When the capture completes, call perf_aux_output_end() to commit
250 * the recorded data to the buffer.
252 * The ordering is similar to that of perf_output_{begin,end}, with
253 * the exception of (B), which should be taken care of by the pmu
254 * driver, since ordering rules will differ depending on hardware.
256 void *perf_aux_output_begin(struct perf_output_handle
*handle
,
257 struct perf_event
*event
)
259 struct perf_event
*output_event
= event
;
260 unsigned long aux_head
, aux_tail
;
261 struct ring_buffer
*rb
;
263 if (output_event
->parent
)
264 output_event
= output_event
->parent
;
267 * Since this will typically be open across pmu::add/pmu::del, we
268 * grab ring_buffer's refcount instead of holding rcu read lock
269 * to make sure it doesn't disappear under us.
271 rb
= ring_buffer_get(output_event
);
275 if (!rb_has_aux(rb
) || !atomic_inc_not_zero(&rb
->aux_refcount
))
279 * Nesting is not supported for AUX area, make sure nested
280 * writers are caught early
282 if (WARN_ON_ONCE(local_xchg(&rb
->aux_nest
, 1)))
285 aux_head
= local_read(&rb
->aux_head
);
288 handle
->event
= event
;
289 handle
->head
= aux_head
;
293 * In overwrite mode, AUX data stores do not depend on aux_tail,
294 * therefore (A) control dependency barrier does not exist. The
295 * (B) <-> (C) ordering is still observed by the pmu driver.
297 if (!rb
->aux_overwrite
) {
298 aux_tail
= ACCESS_ONCE(rb
->user_page
->aux_tail
);
299 handle
->wakeup
= local_read(&rb
->aux_wakeup
) + rb
->aux_watermark
;
300 if (aux_head
- aux_tail
< perf_aux_size(rb
))
301 handle
->size
= CIRC_SPACE(aux_head
, aux_tail
, perf_aux_size(rb
));
304 * handle->size computation depends on aux_tail load; this forms a
305 * control dependency barrier separating aux_tail load from aux data
306 * store that will be enabled on successful return
308 if (!handle
->size
) { /* A, matches D */
309 event
->pending_disable
= 1;
310 perf_output_wakeup(handle
);
311 local_set(&rb
->aux_nest
, 0);
316 return handle
->rb
->aux_priv
;
323 handle
->event
= NULL
;
329 * Commit the data written by hardware into the ring buffer by adjusting
330 * aux_head and posting a PERF_RECORD_AUX into the perf buffer. It is the
331 * pmu driver's responsibility to observe ordering rules of the hardware,
332 * so that all the data is externally visible before this is called.
334 void perf_aux_output_end(struct perf_output_handle
*handle
, unsigned long size
,
337 struct ring_buffer
*rb
= handle
->rb
;
338 unsigned long aux_head
;
342 flags
|= PERF_AUX_FLAG_TRUNCATED
;
344 /* in overwrite mode, driver provides aux_head via handle */
345 if (rb
->aux_overwrite
) {
346 flags
|= PERF_AUX_FLAG_OVERWRITE
;
348 aux_head
= handle
->head
;
349 local_set(&rb
->aux_head
, aux_head
);
351 aux_head
= local_read(&rb
->aux_head
);
352 local_add(size
, &rb
->aux_head
);
357 * Only send RECORD_AUX if we have something useful to communicate
360 perf_event_aux_event(handle
->event
, aux_head
, size
, flags
);
363 aux_head
= rb
->user_page
->aux_head
= local_read(&rb
->aux_head
);
365 if (aux_head
- local_read(&rb
->aux_wakeup
) >= rb
->aux_watermark
) {
366 perf_output_wakeup(handle
);
367 local_add(rb
->aux_watermark
, &rb
->aux_wakeup
);
369 handle
->event
= NULL
;
371 local_set(&rb
->aux_nest
, 0);
377 * Skip over a given number of bytes in the AUX buffer, due to, for example,
378 * hardware's alignment constraints.
380 int perf_aux_output_skip(struct perf_output_handle
*handle
, unsigned long size
)
382 struct ring_buffer
*rb
= handle
->rb
;
383 unsigned long aux_head
;
385 if (size
> handle
->size
)
388 local_add(size
, &rb
->aux_head
);
390 aux_head
= rb
->user_page
->aux_head
= local_read(&rb
->aux_head
);
391 if (aux_head
- local_read(&rb
->aux_wakeup
) >= rb
->aux_watermark
) {
392 perf_output_wakeup(handle
);
393 local_add(rb
->aux_watermark
, &rb
->aux_wakeup
);
394 handle
->wakeup
= local_read(&rb
->aux_wakeup
) +
398 handle
->head
= aux_head
;
399 handle
->size
-= size
;
404 void *perf_get_aux(struct perf_output_handle
*handle
)
406 /* this is only valid between perf_aux_output_begin and *_end */
410 return handle
->rb
->aux_priv
;
413 #define PERF_AUX_GFP (GFP_KERNEL | __GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY)
415 static struct page
*rb_alloc_aux_page(int node
, int order
)
419 if (order
> MAX_ORDER
)
423 page
= alloc_pages_node(node
, PERF_AUX_GFP
, order
);
424 } while (!page
&& order
--);
428 * Communicate the allocation size to the driver
430 split_page(page
, order
);
431 SetPagePrivate(page
);
432 set_page_private(page
, order
);
438 static void rb_free_aux_page(struct ring_buffer
*rb
, int idx
)
440 struct page
*page
= virt_to_page(rb
->aux_pages
[idx
]);
442 ClearPagePrivate(page
);
443 page
->mapping
= NULL
;
447 int rb_alloc_aux(struct ring_buffer
*rb
, struct perf_event
*event
,
448 pgoff_t pgoff
, int nr_pages
, long watermark
, int flags
)
450 bool overwrite
= !(flags
& RING_BUFFER_WRITABLE
);
451 int node
= (event
->cpu
== -1) ? -1 : cpu_to_node(event
->cpu
);
452 int ret
= -ENOMEM
, max_order
= 0;
457 if (event
->pmu
->capabilities
& PERF_PMU_CAP_AUX_NO_SG
) {
459 * We need to start with the max_order that fits in nr_pages,
460 * not the other way around, hence ilog2() and not get_order.
462 max_order
= ilog2(nr_pages
);
465 * PMU requests more than one contiguous chunks of memory
466 * for SW double buffering
468 if ((event
->pmu
->capabilities
& PERF_PMU_CAP_AUX_SW_DOUBLEBUF
) &&
477 rb
->aux_pages
= kzalloc_node(nr_pages
* sizeof(void *), GFP_KERNEL
, node
);
481 rb
->free_aux
= event
->pmu
->free_aux
;
482 for (rb
->aux_nr_pages
= 0; rb
->aux_nr_pages
< nr_pages
;) {
486 order
= min(max_order
, ilog2(nr_pages
- rb
->aux_nr_pages
));
487 page
= rb_alloc_aux_page(node
, order
);
491 for (last
= rb
->aux_nr_pages
+ (1 << page_private(page
));
492 last
> rb
->aux_nr_pages
; rb
->aux_nr_pages
++)
493 rb
->aux_pages
[rb
->aux_nr_pages
] = page_address(page
++);
496 rb
->aux_priv
= event
->pmu
->setup_aux(event
->cpu
, rb
->aux_pages
, nr_pages
,
504 * aux_pages (and pmu driver's private data, aux_priv) will be
505 * referenced in both producer's and consumer's contexts, thus
506 * we keep a refcount here to make sure either of the two can
507 * reference them safely.
509 atomic_set(&rb
->aux_refcount
, 1);
511 rb
->aux_overwrite
= overwrite
;
512 rb
->aux_watermark
= watermark
;
514 if (!rb
->aux_watermark
&& !rb
->aux_overwrite
)
515 rb
->aux_watermark
= nr_pages
<< (PAGE_SHIFT
- 1);
519 rb
->aux_pgoff
= pgoff
;
526 static void __rb_free_aux(struct ring_buffer
*rb
)
531 rb
->free_aux(rb
->aux_priv
);
536 for (pg
= 0; pg
< rb
->aux_nr_pages
; pg
++)
537 rb_free_aux_page(rb
, pg
);
539 kfree(rb
->aux_pages
);
540 rb
->aux_nr_pages
= 0;
543 void rb_free_aux(struct ring_buffer
*rb
)
545 if (atomic_dec_and_test(&rb
->aux_refcount
))
549 #ifndef CONFIG_PERF_USE_VMALLOC
552 * Back perf_mmap() with regular GFP_KERNEL-0 pages.
556 __perf_mmap_to_page(struct ring_buffer
*rb
, unsigned long pgoff
)
558 if (pgoff
> rb
->nr_pages
)
562 return virt_to_page(rb
->user_page
);
564 return virt_to_page(rb
->data_pages
[pgoff
- 1]);
567 static void *perf_mmap_alloc_page(int cpu
)
572 node
= (cpu
== -1) ? cpu
: cpu_to_node(cpu
);
573 page
= alloc_pages_node(node
, GFP_KERNEL
| __GFP_ZERO
, 0);
577 return page_address(page
);
580 struct ring_buffer
*rb_alloc(int nr_pages
, long watermark
, int cpu
, int flags
)
582 struct ring_buffer
*rb
;
586 size
= sizeof(struct ring_buffer
);
587 size
+= nr_pages
* sizeof(void *);
589 rb
= kzalloc(size
, GFP_KERNEL
);
593 rb
->user_page
= perf_mmap_alloc_page(cpu
);
597 for (i
= 0; i
< nr_pages
; i
++) {
598 rb
->data_pages
[i
] = perf_mmap_alloc_page(cpu
);
599 if (!rb
->data_pages
[i
])
600 goto fail_data_pages
;
603 rb
->nr_pages
= nr_pages
;
605 ring_buffer_init(rb
, watermark
, flags
);
610 for (i
--; i
>= 0; i
--)
611 free_page((unsigned long)rb
->data_pages
[i
]);
613 free_page((unsigned long)rb
->user_page
);
622 static void perf_mmap_free_page(unsigned long addr
)
624 struct page
*page
= virt_to_page((void *)addr
);
626 page
->mapping
= NULL
;
630 void rb_free(struct ring_buffer
*rb
)
634 perf_mmap_free_page((unsigned long)rb
->user_page
);
635 for (i
= 0; i
< rb
->nr_pages
; i
++)
636 perf_mmap_free_page((unsigned long)rb
->data_pages
[i
]);
641 static int data_page_nr(struct ring_buffer
*rb
)
643 return rb
->nr_pages
<< page_order(rb
);
647 __perf_mmap_to_page(struct ring_buffer
*rb
, unsigned long pgoff
)
649 /* The '>' counts in the user page. */
650 if (pgoff
> data_page_nr(rb
))
653 return vmalloc_to_page((void *)rb
->user_page
+ pgoff
* PAGE_SIZE
);
656 static void perf_mmap_unmark_page(void *addr
)
658 struct page
*page
= vmalloc_to_page(addr
);
660 page
->mapping
= NULL
;
663 static void rb_free_work(struct work_struct
*work
)
665 struct ring_buffer
*rb
;
669 rb
= container_of(work
, struct ring_buffer
, work
);
670 nr
= data_page_nr(rb
);
672 base
= rb
->user_page
;
673 /* The '<=' counts in the user page. */
674 for (i
= 0; i
<= nr
; i
++)
675 perf_mmap_unmark_page(base
+ (i
* PAGE_SIZE
));
681 void rb_free(struct ring_buffer
*rb
)
683 schedule_work(&rb
->work
);
686 struct ring_buffer
*rb_alloc(int nr_pages
, long watermark
, int cpu
, int flags
)
688 struct ring_buffer
*rb
;
692 size
= sizeof(struct ring_buffer
);
693 size
+= sizeof(void *);
695 rb
= kzalloc(size
, GFP_KERNEL
);
699 INIT_WORK(&rb
->work
, rb_free_work
);
701 all_buf
= vmalloc_user((nr_pages
+ 1) * PAGE_SIZE
);
705 rb
->user_page
= all_buf
;
706 rb
->data_pages
[0] = all_buf
+ PAGE_SIZE
;
707 rb
->page_order
= ilog2(nr_pages
);
708 rb
->nr_pages
= !!nr_pages
;
710 ring_buffer_init(rb
, watermark
, flags
);
724 perf_mmap_to_page(struct ring_buffer
*rb
, unsigned long pgoff
)
726 if (rb
->aux_nr_pages
) {
727 /* above AUX space */
728 if (pgoff
> rb
->aux_pgoff
+ rb
->aux_nr_pages
)
732 if (pgoff
>= rb
->aux_pgoff
)
733 return virt_to_page(rb
->aux_pages
[pgoff
- rb
->aux_pgoff
]);
736 return __perf_mmap_to_page(rb
, pgoff
);