2 * Performance events ring-buffer code:
4 * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
5 * Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar
6 * Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
7 * Copyright © 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
9 * For licensing details see kernel-base/COPYING
12 #include <linux/perf_event.h>
13 #include <linux/vmalloc.h>
14 #include <linux/slab.h>
15 #include <linux/circ_buf.h>
19 static void perf_output_wakeup(struct perf_output_handle
*handle
)
21 atomic_set(&handle
->rb
->poll
, POLL_IN
);
23 handle
->event
->pending_wakeup
= 1;
24 irq_work_queue(&handle
->event
->pending
);
28 * We need to ensure a later event_id doesn't publish a head when a former
29 * event isn't done writing. However since we need to deal with NMIs we
30 * cannot fully serialize things.
32 * We only publish the head (and generate a wakeup) when the outer-most
35 static void perf_output_get_handle(struct perf_output_handle
*handle
)
37 struct ring_buffer
*rb
= handle
->rb
;
41 handle
->wakeup
= local_read(&rb
->wakeup
);
44 static void perf_output_put_handle(struct perf_output_handle
*handle
)
46 struct ring_buffer
*rb
= handle
->rb
;
50 head
= local_read(&rb
->head
);
53 * IRQ/NMI can happen here, which means we can miss a head update.
56 if (!local_dec_and_test(&rb
->nest
))
60 * Since the mmap() consumer (userspace) can run on a different CPU:
64 * READ ->data_tail READ ->data_head
65 * smp_mb() (A) smp_rmb() (C)
66 * WRITE $data READ $data
67 * smp_wmb() (B) smp_mb() (D)
68 * STORE ->data_head WRITE ->data_tail
70 * Where A pairs with D, and B pairs with C.
72 * I don't think A needs to be a full barrier because we won't in fact
73 * write data until we see the store from userspace. So we simply don't
74 * issue the data WRITE until we observe it. Be conservative for now.
76 * OTOH, D needs to be a full barrier since it separates the data READ
77 * from the tail WRITE.
79 * For B a WMB is sufficient since it separates two WRITEs, and for C
80 * an RMB is sufficient since it separates two READs.
82 * See perf_output_begin().
85 rb
->user_page
->data_head
= head
;
88 * Now check if we missed an update -- rely on previous implied
89 * compiler barriers to force a re-read.
91 if (unlikely(head
!= local_read(&rb
->head
))) {
96 if (handle
->wakeup
!= local_read(&rb
->wakeup
))
97 perf_output_wakeup(handle
);
103 int perf_output_begin(struct perf_output_handle
*handle
,
104 struct perf_event
*event
, unsigned int size
)
106 struct ring_buffer
*rb
;
107 unsigned long tail
, offset
, head
;
108 int have_lost
, page_shift
;
110 struct perf_event_header header
;
117 * For inherited events we send all the output towards the parent.
120 event
= event
->parent
;
122 rb
= rcu_dereference(event
->rb
);
126 if (unlikely(!rb
->nr_pages
))
130 handle
->event
= event
;
132 have_lost
= local_read(&rb
->lost
);
133 if (unlikely(have_lost
)) {
134 size
+= sizeof(lost_event
);
135 if (event
->attr
.sample_id_all
)
136 size
+= event
->id_header_size
;
139 perf_output_get_handle(handle
);
142 tail
= ACCESS_ONCE(rb
->user_page
->data_tail
);
143 offset
= head
= local_read(&rb
->head
);
144 if (!rb
->overwrite
&&
145 unlikely(CIRC_SPACE(head
, tail
, perf_data_size(rb
)) < size
))
148 } while (local_cmpxchg(&rb
->head
, offset
, head
) != offset
);
151 * Separate the userpage->tail read from the data stores below.
152 * Matches the MB userspace SHOULD issue after reading the data
153 * and before storing the new tail position.
155 * See perf_output_put_handle().
159 if (unlikely(head
- local_read(&rb
->wakeup
) > rb
->watermark
))
160 local_add(rb
->watermark
, &rb
->wakeup
);
162 page_shift
= PAGE_SHIFT
+ page_order(rb
);
164 handle
->page
= (offset
>> page_shift
) & (rb
->nr_pages
- 1);
165 offset
&= (1UL << page_shift
) - 1;
166 handle
->addr
= rb
->data_pages
[handle
->page
] + offset
;
167 handle
->size
= (1UL << page_shift
) - offset
;
169 if (unlikely(have_lost
)) {
170 struct perf_sample_data sample_data
;
172 lost_event
.header
.size
= sizeof(lost_event
);
173 lost_event
.header
.type
= PERF_RECORD_LOST
;
174 lost_event
.header
.misc
= 0;
175 lost_event
.id
= event
->id
;
176 lost_event
.lost
= local_xchg(&rb
->lost
, 0);
178 perf_event_header__init_id(&lost_event
.header
,
179 &sample_data
, event
);
180 perf_output_put(handle
, lost_event
);
181 perf_event__output_id_sample(event
, handle
, &sample_data
);
187 local_inc(&rb
->lost
);
188 perf_output_put_handle(handle
);
195 unsigned int perf_output_copy(struct perf_output_handle
*handle
,
196 const void *buf
, unsigned int len
)
198 return __output_copy(handle
, buf
, len
);
201 unsigned int perf_output_skip(struct perf_output_handle
*handle
,
204 return __output_skip(handle
, NULL
, len
);
207 void perf_output_end(struct perf_output_handle
*handle
)
209 perf_output_put_handle(handle
);
214 ring_buffer_init(struct ring_buffer
*rb
, long watermark
, int flags
)
216 long max_size
= perf_data_size(rb
);
219 rb
->watermark
= min(max_size
, watermark
);
222 rb
->watermark
= max_size
/ 2;
224 if (flags
& RING_BUFFER_WRITABLE
)
229 atomic_set(&rb
->refcount
, 1);
231 INIT_LIST_HEAD(&rb
->event_list
);
232 spin_lock_init(&rb
->event_lock
);
235 #ifndef CONFIG_PERF_USE_VMALLOC
238 * Back perf_mmap() with regular GFP_KERNEL-0 pages.
242 perf_mmap_to_page(struct ring_buffer
*rb
, unsigned long pgoff
)
244 if (pgoff
> rb
->nr_pages
)
248 return virt_to_page(rb
->user_page
);
250 return virt_to_page(rb
->data_pages
[pgoff
- 1]);
253 static void *perf_mmap_alloc_page(int cpu
)
258 node
= (cpu
== -1) ? cpu
: cpu_to_node(cpu
);
259 page
= alloc_pages_node(node
, GFP_KERNEL
| __GFP_ZERO
, 0);
263 return page_address(page
);
266 struct ring_buffer
*rb_alloc(int nr_pages
, long watermark
, int cpu
, int flags
)
268 struct ring_buffer
*rb
;
272 size
= sizeof(struct ring_buffer
);
273 size
+= nr_pages
* sizeof(void *);
275 rb
= kzalloc(size
, GFP_KERNEL
);
279 rb
->user_page
= perf_mmap_alloc_page(cpu
);
283 for (i
= 0; i
< nr_pages
; i
++) {
284 rb
->data_pages
[i
] = perf_mmap_alloc_page(cpu
);
285 if (!rb
->data_pages
[i
])
286 goto fail_data_pages
;
289 rb
->nr_pages
= nr_pages
;
291 ring_buffer_init(rb
, watermark
, flags
);
296 for (i
--; i
>= 0; i
--)
297 free_page((unsigned long)rb
->data_pages
[i
]);
299 free_page((unsigned long)rb
->user_page
);
308 static void perf_mmap_free_page(unsigned long addr
)
310 struct page
*page
= virt_to_page((void *)addr
);
312 page
->mapping
= NULL
;
316 void rb_free(struct ring_buffer
*rb
)
320 perf_mmap_free_page((unsigned long)rb
->user_page
);
321 for (i
= 0; i
< rb
->nr_pages
; i
++)
322 perf_mmap_free_page((unsigned long)rb
->data_pages
[i
]);
327 static int data_page_nr(struct ring_buffer
*rb
)
329 return rb
->nr_pages
<< page_order(rb
);
333 perf_mmap_to_page(struct ring_buffer
*rb
, unsigned long pgoff
)
335 /* The '>' counts in the user page. */
336 if (pgoff
> data_page_nr(rb
))
339 return vmalloc_to_page((void *)rb
->user_page
+ pgoff
* PAGE_SIZE
);
342 static void perf_mmap_unmark_page(void *addr
)
344 struct page
*page
= vmalloc_to_page(addr
);
346 page
->mapping
= NULL
;
349 static void rb_free_work(struct work_struct
*work
)
351 struct ring_buffer
*rb
;
355 rb
= container_of(work
, struct ring_buffer
, work
);
356 nr
= data_page_nr(rb
);
358 base
= rb
->user_page
;
359 /* The '<=' counts in the user page. */
360 for (i
= 0; i
<= nr
; i
++)
361 perf_mmap_unmark_page(base
+ (i
* PAGE_SIZE
));
367 void rb_free(struct ring_buffer
*rb
)
369 schedule_work(&rb
->work
);
372 struct ring_buffer
*rb_alloc(int nr_pages
, long watermark
, int cpu
, int flags
)
374 struct ring_buffer
*rb
;
378 size
= sizeof(struct ring_buffer
);
379 size
+= sizeof(void *);
381 rb
= kzalloc(size
, GFP_KERNEL
);
385 INIT_WORK(&rb
->work
, rb_free_work
);
387 all_buf
= vmalloc_user((nr_pages
+ 1) * PAGE_SIZE
);
391 rb
->user_page
= all_buf
;
392 rb
->data_pages
[0] = all_buf
+ PAGE_SIZE
;
393 rb
->page_order
= ilog2(nr_pages
);
394 rb
->nr_pages
= !!nr_pages
;
396 ring_buffer_init(rb
, watermark
, flags
);