Commit | Line | Data |
---|---|---|
14f70012 DCZ |
1 | /* |
2 | * Linux performance counter support for MIPS. | |
3 | * | |
4 | * Copyright (C) 2010 MIPS Technologies, Inc. | |
5 | * Author: Deng-Cheng Zhu | |
6 | * | |
7 | * This code is based on the implementation for ARM, which is in turn | |
8 | * based on the sparc64 perf event code and the x86 code. Performance | |
7e788d96 DCZ |
9 | * counter access is based on the MIPS Oprofile code. And the callchain |
10 | * support references the code of MIPS stacktrace.c. | |
14f70012 DCZ |
11 | * |
12 | * This program is free software; you can redistribute it and/or modify | |
13 | * it under the terms of the GNU General Public License version 2 as | |
14 | * published by the Free Software Foundation. | |
15 | */ | |
16 | ||
17 | #include <linux/cpumask.h> | |
18 | #include <linux/interrupt.h> | |
19 | #include <linux/smp.h> | |
20 | #include <linux/kernel.h> | |
21 | #include <linux/perf_event.h> | |
22 | #include <linux/uaccess.h> | |
23 | ||
24 | #include <asm/irq.h> | |
25 | #include <asm/irq_regs.h> | |
26 | #include <asm/stacktrace.h> | |
27 | #include <asm/time.h> /* For perf_irq */ | |
28 | ||
29 | /* These are for 32bit counters. For 64bit ones, define them accordingly. */ | |
30 | #define MAX_PERIOD ((1ULL << 32) - 1) | |
31 | #define VALID_COUNT 0x7fffffff | |
32 | #define TOTAL_BITS 32 | |
33 | #define HIGHEST_BIT 31 | |
34 | ||
35 | #define MIPS_MAX_HWEVENTS 4 | |
36 | ||
37 | struct cpu_hw_events { | |
38 | /* Array of events on this cpu. */ | |
39 | struct perf_event *events[MIPS_MAX_HWEVENTS]; | |
40 | ||
41 | /* | |
42 | * Set the bit (indexed by the counter number) when the counter | |
43 | * is used for an event. | |
44 | */ | |
45 | unsigned long used_mask[BITS_TO_LONGS(MIPS_MAX_HWEVENTS)]; | |
46 | ||
47 | /* | |
48 | * The borrowed MSB for the performance counter. A MIPS performance | |
49 | * counter uses its bit 31 (for 32bit counters) or bit 63 (for 64bit | |
50 | * counters) as a factor of determining whether a counter overflow | |
51 | * should be signaled. So here we use a separate MSB for each | |
52 | * counter to make things easy. | |
53 | */ | |
54 | unsigned long msbs[BITS_TO_LONGS(MIPS_MAX_HWEVENTS)]; | |
55 | ||
56 | /* | |
57 | * Software copy of the control register for each performance counter. | |
58 | * MIPS CPUs vary in performance counters. They use this differently, | |
59 | * and even may not use it. | |
60 | */ | |
61 | unsigned int saved_ctrl[MIPS_MAX_HWEVENTS]; | |
62 | }; | |
63 | DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = { | |
64 | .saved_ctrl = {0}, | |
65 | }; | |
66 | ||
67 | /* The description of MIPS performance events. */ | |
68 | struct mips_perf_event { | |
69 | unsigned int event_id; | |
70 | /* | |
71 | * MIPS performance counters are indexed starting from 0. | |
72 | * CNTR_EVEN indicates the indexes of the counters to be used are | |
73 | * even numbers. | |
74 | */ | |
75 | unsigned int cntr_mask; | |
76 | #define CNTR_EVEN 0x55555555 | |
77 | #define CNTR_ODD 0xaaaaaaaa | |
78 | #ifdef CONFIG_MIPS_MT_SMP | |
79 | enum { | |
80 | T = 0, | |
81 | V = 1, | |
82 | P = 2, | |
83 | } range; | |
84 | #else | |
85 | #define T | |
86 | #define V | |
87 | #define P | |
88 | #endif | |
89 | }; | |
90 | ||
3a9ab99e DCZ |
91 | static struct mips_perf_event raw_event; |
92 | static DEFINE_MUTEX(raw_event_mutex); | |
93 | ||
14f70012 DCZ |
94 | #define UNSUPPORTED_PERF_EVENT_ID 0xffffffff |
95 | #define C(x) PERF_COUNT_HW_CACHE_##x | |
96 | ||
97 | struct mips_pmu { | |
98 | const char *name; | |
99 | int irq; | |
100 | irqreturn_t (*handle_irq)(int irq, void *dev); | |
101 | int (*handle_shared_irq)(void); | |
102 | void (*start)(void); | |
103 | void (*stop)(void); | |
104 | int (*alloc_counter)(struct cpu_hw_events *cpuc, | |
105 | struct hw_perf_event *hwc); | |
106 | u64 (*read_counter)(unsigned int idx); | |
107 | void (*write_counter)(unsigned int idx, u64 val); | |
108 | void (*enable_event)(struct hw_perf_event *evt, int idx); | |
109 | void (*disable_event)(int idx); | |
3a9ab99e | 110 | const struct mips_perf_event *(*map_raw_event)(u64 config); |
14f70012 DCZ |
111 | const struct mips_perf_event (*general_event_map)[PERF_COUNT_HW_MAX]; |
112 | const struct mips_perf_event (*cache_event_map) | |
113 | [PERF_COUNT_HW_CACHE_MAX] | |
114 | [PERF_COUNT_HW_CACHE_OP_MAX] | |
115 | [PERF_COUNT_HW_CACHE_RESULT_MAX]; | |
116 | unsigned int num_counters; | |
117 | }; | |
118 | ||
119 | static const struct mips_pmu *mipspmu; | |
120 | ||
4409af37 DD |
121 | static int mipspmu_event_set_period(struct perf_event *event, |
122 | struct hw_perf_event *hwc, | |
123 | int idx) | |
14f70012 DCZ |
124 | { |
125 | struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); | |
126 | s64 left = local64_read(&hwc->period_left); | |
127 | s64 period = hwc->sample_period; | |
128 | int ret = 0; | |
129 | u64 uleft; | |
130 | unsigned long flags; | |
131 | ||
132 | if (unlikely(left <= -period)) { | |
133 | left = period; | |
134 | local64_set(&hwc->period_left, left); | |
135 | hwc->last_period = period; | |
136 | ret = 1; | |
137 | } | |
138 | ||
139 | if (unlikely(left <= 0)) { | |
140 | left += period; | |
141 | local64_set(&hwc->period_left, left); | |
142 | hwc->last_period = period; | |
143 | ret = 1; | |
144 | } | |
145 | ||
146 | if (left > (s64)MAX_PERIOD) | |
147 | left = MAX_PERIOD; | |
148 | ||
149 | local64_set(&hwc->prev_count, (u64)-left); | |
150 | ||
151 | local_irq_save(flags); | |
152 | uleft = (u64)(-left) & MAX_PERIOD; | |
153 | uleft > VALID_COUNT ? | |
154 | set_bit(idx, cpuc->msbs) : clear_bit(idx, cpuc->msbs); | |
155 | mipspmu->write_counter(idx, (u64)(-left) & VALID_COUNT); | |
156 | local_irq_restore(flags); | |
157 | ||
158 | perf_event_update_userpage(event); | |
159 | ||
160 | return ret; | |
161 | } | |
162 | ||
14f70012 | 163 | static void mipspmu_event_update(struct perf_event *event, |
4409af37 DD |
164 | struct hw_perf_event *hwc, |
165 | int idx) | |
14f70012 DCZ |
166 | { |
167 | struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); | |
168 | unsigned long flags; | |
169 | int shift = 64 - TOTAL_BITS; | |
170 | s64 prev_raw_count, new_raw_count; | |
ba9786f3 | 171 | u64 delta; |
14f70012 DCZ |
172 | |
173 | again: | |
174 | prev_raw_count = local64_read(&hwc->prev_count); | |
175 | local_irq_save(flags); | |
176 | /* Make the counter value be a "real" one. */ | |
177 | new_raw_count = mipspmu->read_counter(idx); | |
178 | if (new_raw_count & (test_bit(idx, cpuc->msbs) << HIGHEST_BIT)) { | |
179 | new_raw_count &= VALID_COUNT; | |
180 | clear_bit(idx, cpuc->msbs); | |
181 | } else | |
182 | new_raw_count |= (test_bit(idx, cpuc->msbs) << HIGHEST_BIT); | |
183 | local_irq_restore(flags); | |
184 | ||
185 | if (local64_cmpxchg(&hwc->prev_count, prev_raw_count, | |
186 | new_raw_count) != prev_raw_count) | |
187 | goto again; | |
188 | ||
189 | delta = (new_raw_count << shift) - (prev_raw_count << shift); | |
190 | delta >>= shift; | |
191 | ||
192 | local64_add(delta, &event->count); | |
193 | local64_sub(delta, &hwc->period_left); | |
14f70012 DCZ |
194 | } |
195 | ||
404ff638 DCZ |
196 | static void mipspmu_start(struct perf_event *event, int flags) |
197 | { | |
198 | struct hw_perf_event *hwc = &event->hw; | |
199 | ||
200 | if (!mipspmu) | |
201 | return; | |
202 | ||
203 | if (flags & PERF_EF_RELOAD) | |
204 | WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE)); | |
205 | ||
206 | hwc->state = 0; | |
207 | ||
208 | /* Set the period for the event. */ | |
209 | mipspmu_event_set_period(event, hwc, hwc->idx); | |
210 | ||
211 | /* Enable the event. */ | |
212 | mipspmu->enable_event(hwc, hwc->idx); | |
213 | } | |
214 | ||
215 | static void mipspmu_stop(struct perf_event *event, int flags) | |
216 | { | |
217 | struct hw_perf_event *hwc = &event->hw; | |
218 | ||
219 | if (!mipspmu) | |
220 | return; | |
221 | ||
222 | if (!(hwc->state & PERF_HES_STOPPED)) { | |
223 | /* We are working on a local event. */ | |
224 | mipspmu->disable_event(hwc->idx); | |
225 | barrier(); | |
226 | mipspmu_event_update(event, hwc, hwc->idx); | |
227 | hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE; | |
228 | } | |
229 | } | |
230 | ||
231 | static int mipspmu_add(struct perf_event *event, int flags) | |
14f70012 DCZ |
232 | { |
233 | struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); | |
234 | struct hw_perf_event *hwc = &event->hw; | |
404ff638 DCZ |
235 | int idx; |
236 | int err = 0; | |
14f70012 | 237 | |
404ff638 | 238 | perf_pmu_disable(event->pmu); |
14f70012 | 239 | |
404ff638 DCZ |
240 | /* To look for a free counter for this event. */ |
241 | idx = mipspmu->alloc_counter(cpuc, hwc); | |
242 | if (idx < 0) { | |
243 | err = idx; | |
244 | goto out; | |
245 | } | |
14f70012 | 246 | |
404ff638 DCZ |
247 | /* |
248 | * If there is an event in the counter we are going to use then | |
249 | * make sure it is disabled. | |
250 | */ | |
251 | event->hw.idx = idx; | |
14f70012 | 252 | mipspmu->disable_event(idx); |
404ff638 | 253 | cpuc->events[idx] = event; |
14f70012 | 254 | |
404ff638 DCZ |
255 | hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE; |
256 | if (flags & PERF_EF_START) | |
257 | mipspmu_start(event, PERF_EF_RELOAD); | |
14f70012 | 258 | |
404ff638 | 259 | /* Propagate our changes to the userspace mapping. */ |
14f70012 | 260 | perf_event_update_userpage(event); |
404ff638 DCZ |
261 | |
262 | out: | |
263 | perf_pmu_enable(event->pmu); | |
264 | return err; | |
14f70012 DCZ |
265 | } |
266 | ||
404ff638 | 267 | static void mipspmu_del(struct perf_event *event, int flags) |
14f70012 | 268 | { |
404ff638 | 269 | struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); |
14f70012 | 270 | struct hw_perf_event *hwc = &event->hw; |
404ff638 | 271 | int idx = hwc->idx; |
14f70012 | 272 | |
404ff638 DCZ |
273 | WARN_ON(idx < 0 || idx >= mipspmu->num_counters); |
274 | ||
275 | mipspmu_stop(event, PERF_EF_UPDATE); | |
276 | cpuc->events[idx] = NULL; | |
277 | clear_bit(idx, cpuc->used_mask); | |
278 | ||
279 | perf_event_update_userpage(event); | |
14f70012 DCZ |
280 | } |
281 | ||
282 | static void mipspmu_read(struct perf_event *event) | |
283 | { | |
284 | struct hw_perf_event *hwc = &event->hw; | |
285 | ||
286 | /* Don't read disabled counters! */ | |
287 | if (hwc->idx < 0) | |
288 | return; | |
289 | ||
290 | mipspmu_event_update(event, hwc, hwc->idx); | |
291 | } | |
292 | ||
404ff638 DCZ |
293 | static void mipspmu_enable(struct pmu *pmu) |
294 | { | |
295 | if (mipspmu) | |
296 | mipspmu->start(); | |
297 | } | |
298 | ||
299 | static void mipspmu_disable(struct pmu *pmu) | |
300 | { | |
301 | if (mipspmu) | |
302 | mipspmu->stop(); | |
303 | } | |
14f70012 DCZ |
304 | |
305 | static atomic_t active_events = ATOMIC_INIT(0); | |
306 | static DEFINE_MUTEX(pmu_reserve_mutex); | |
307 | static int (*save_perf_irq)(void); | |
308 | ||
309 | static int mipspmu_get_irq(void) | |
310 | { | |
311 | int err; | |
312 | ||
313 | if (mipspmu->irq >= 0) { | |
314 | /* Request my own irq handler. */ | |
315 | err = request_irq(mipspmu->irq, mipspmu->handle_irq, | |
316 | IRQF_DISABLED | IRQF_NOBALANCING, | |
317 | "mips_perf_pmu", NULL); | |
318 | if (err) { | |
319 | pr_warning("Unable to request IRQ%d for MIPS " | |
320 | "performance counters!\n", mipspmu->irq); | |
321 | } | |
322 | } else if (cp0_perfcount_irq < 0) { | |
323 | /* | |
324 | * We are sharing the irq number with the timer interrupt. | |
325 | */ | |
326 | save_perf_irq = perf_irq; | |
327 | perf_irq = mipspmu->handle_shared_irq; | |
328 | err = 0; | |
329 | } else { | |
330 | pr_warning("The platform hasn't properly defined its " | |
331 | "interrupt controller.\n"); | |
332 | err = -ENOENT; | |
333 | } | |
334 | ||
335 | return err; | |
336 | } | |
337 | ||
338 | static void mipspmu_free_irq(void) | |
339 | { | |
340 | if (mipspmu->irq >= 0) | |
341 | free_irq(mipspmu->irq, NULL); | |
342 | else if (cp0_perfcount_irq < 0) | |
343 | perf_irq = save_perf_irq; | |
344 | } | |
345 | ||
404ff638 DCZ |
346 | /* |
347 | * mipsxx/rm9000/loongson2 have different performance counters, they have | |
348 | * specific low-level init routines. | |
349 | */ | |
350 | static void reset_counters(void *arg); | |
351 | static int __hw_perf_event_init(struct perf_event *event); | |
352 | ||
353 | static void hw_perf_event_destroy(struct perf_event *event) | |
354 | { | |
355 | if (atomic_dec_and_mutex_lock(&active_events, | |
356 | &pmu_reserve_mutex)) { | |
357 | /* | |
358 | * We must not call the destroy function with interrupts | |
359 | * disabled. | |
360 | */ | |
361 | on_each_cpu(reset_counters, | |
362 | (void *)(long)mipspmu->num_counters, 1); | |
363 | mipspmu_free_irq(); | |
364 | mutex_unlock(&pmu_reserve_mutex); | |
365 | } | |
366 | } | |
367 | ||
368 | static int mipspmu_event_init(struct perf_event *event) | |
369 | { | |
370 | int err = 0; | |
371 | ||
372 | switch (event->attr.type) { | |
373 | case PERF_TYPE_RAW: | |
374 | case PERF_TYPE_HARDWARE: | |
375 | case PERF_TYPE_HW_CACHE: | |
376 | break; | |
377 | ||
378 | default: | |
379 | return -ENOENT; | |
380 | } | |
381 | ||
382 | if (!mipspmu || event->cpu >= nr_cpumask_bits || | |
383 | (event->cpu >= 0 && !cpu_online(event->cpu))) | |
384 | return -ENODEV; | |
385 | ||
386 | if (!atomic_inc_not_zero(&active_events)) { | |
387 | if (atomic_read(&active_events) > MIPS_MAX_HWEVENTS) { | |
388 | atomic_dec(&active_events); | |
389 | return -ENOSPC; | |
390 | } | |
391 | ||
392 | mutex_lock(&pmu_reserve_mutex); | |
393 | if (atomic_read(&active_events) == 0) | |
394 | err = mipspmu_get_irq(); | |
395 | ||
396 | if (!err) | |
397 | atomic_inc(&active_events); | |
398 | mutex_unlock(&pmu_reserve_mutex); | |
399 | } | |
400 | ||
401 | if (err) | |
402 | return err; | |
403 | ||
404 | err = __hw_perf_event_init(event); | |
405 | if (err) | |
406 | hw_perf_event_destroy(event); | |
407 | ||
408 | return err; | |
409 | } | |
410 | ||
411 | static struct pmu pmu = { | |
412 | .pmu_enable = mipspmu_enable, | |
413 | .pmu_disable = mipspmu_disable, | |
414 | .event_init = mipspmu_event_init, | |
415 | .add = mipspmu_add, | |
416 | .del = mipspmu_del, | |
417 | .start = mipspmu_start, | |
418 | .stop = mipspmu_stop, | |
419 | .read = mipspmu_read, | |
420 | }; | |
421 | ||
4409af37 | 422 | static unsigned int mipspmu_perf_event_encode(const struct mips_perf_event *pev) |
14f70012 DCZ |
423 | { |
424 | /* | |
425 | * Top 8 bits for range, next 16 bits for cntr_mask, lowest 8 bits for | |
426 | * event_id. | |
427 | */ | |
428 | #ifdef CONFIG_MIPS_MT_SMP | |
429 | return ((unsigned int)pev->range << 24) | | |
430 | (pev->cntr_mask & 0xffff00) | | |
431 | (pev->event_id & 0xff); | |
432 | #else | |
433 | return (pev->cntr_mask & 0xffff00) | | |
434 | (pev->event_id & 0xff); | |
435 | #endif | |
436 | } | |
437 | ||
4409af37 | 438 | static const struct mips_perf_event *mipspmu_map_general_event(int idx) |
14f70012 DCZ |
439 | { |
440 | const struct mips_perf_event *pev; | |
441 | ||
442 | pev = ((*mipspmu->general_event_map)[idx].event_id == | |
443 | UNSUPPORTED_PERF_EVENT_ID ? ERR_PTR(-EOPNOTSUPP) : | |
444 | &(*mipspmu->general_event_map)[idx]); | |
445 | ||
446 | return pev; | |
447 | } | |
448 | ||
4409af37 | 449 | static const struct mips_perf_event *mipspmu_map_cache_event(u64 config) |
14f70012 DCZ |
450 | { |
451 | unsigned int cache_type, cache_op, cache_result; | |
452 | const struct mips_perf_event *pev; | |
453 | ||
454 | cache_type = (config >> 0) & 0xff; | |
455 | if (cache_type >= PERF_COUNT_HW_CACHE_MAX) | |
456 | return ERR_PTR(-EINVAL); | |
457 | ||
458 | cache_op = (config >> 8) & 0xff; | |
459 | if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX) | |
460 | return ERR_PTR(-EINVAL); | |
461 | ||
462 | cache_result = (config >> 16) & 0xff; | |
463 | if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX) | |
464 | return ERR_PTR(-EINVAL); | |
465 | ||
466 | pev = &((*mipspmu->cache_event_map) | |
467 | [cache_type] | |
468 | [cache_op] | |
469 | [cache_result]); | |
470 | ||
471 | if (pev->event_id == UNSUPPORTED_PERF_EVENT_ID) | |
472 | return ERR_PTR(-EOPNOTSUPP); | |
473 | ||
474 | return pev; | |
475 | ||
476 | } | |
477 | ||
478 | static int validate_event(struct cpu_hw_events *cpuc, | |
479 | struct perf_event *event) | |
480 | { | |
481 | struct hw_perf_event fake_hwc = event->hw; | |
482 | ||
c049b6a5 DCZ |
483 | /* Allow mixed event group. So return 1 to pass validation. */ |
484 | if (event->pmu != &pmu || event->state <= PERF_EVENT_STATE_OFF) | |
485 | return 1; | |
14f70012 DCZ |
486 | |
487 | return mipspmu->alloc_counter(cpuc, &fake_hwc) >= 0; | |
488 | } | |
489 | ||
490 | static int validate_group(struct perf_event *event) | |
491 | { | |
492 | struct perf_event *sibling, *leader = event->group_leader; | |
493 | struct cpu_hw_events fake_cpuc; | |
494 | ||
495 | memset(&fake_cpuc, 0, sizeof(fake_cpuc)); | |
496 | ||
497 | if (!validate_event(&fake_cpuc, leader)) | |
498 | return -ENOSPC; | |
499 | ||
500 | list_for_each_entry(sibling, &leader->sibling_list, group_entry) { | |
501 | if (!validate_event(&fake_cpuc, sibling)) | |
502 | return -ENOSPC; | |
503 | } | |
504 | ||
505 | if (!validate_event(&fake_cpuc, event)) | |
506 | return -ENOSPC; | |
507 | ||
508 | return 0; | |
509 | } | |
510 | ||
14f70012 | 511 | /* This is needed by specific irq handlers in perf_event_*.c */ |
4409af37 DD |
512 | static void handle_associated_event(struct cpu_hw_events *cpuc, |
513 | int idx, struct perf_sample_data *data, | |
514 | struct pt_regs *regs) | |
14f70012 DCZ |
515 | { |
516 | struct perf_event *event = cpuc->events[idx]; | |
517 | struct hw_perf_event *hwc = &event->hw; | |
518 | ||
519 | mipspmu_event_update(event, hwc, idx); | |
520 | data->period = event->hw.last_period; | |
521 | if (!mipspmu_event_set_period(event, hwc, idx)) | |
522 | return; | |
523 | ||
a8b0ca17 | 524 | if (perf_event_overflow(event, data, regs)) |
14f70012 DCZ |
525 | mipspmu->disable_event(idx); |
526 | } | |
7e788d96 | 527 | |
3a9ab99e DCZ |
528 | #include "perf_event_mipsxx.c" |
529 | ||
7e788d96 | 530 | /* Callchain handling code. */ |
7e788d96 DCZ |
531 | |
532 | /* | |
533 | * Leave userspace callchain empty for now. When we find a way to trace | |
534 | * the user stack callchains, we add here. | |
535 | */ | |
98f92f2f DCZ |
536 | void perf_callchain_user(struct perf_callchain_entry *entry, |
537 | struct pt_regs *regs) | |
7e788d96 DCZ |
538 | { |
539 | } | |
540 | ||
541 | static void save_raw_perf_callchain(struct perf_callchain_entry *entry, | |
542 | unsigned long reg29) | |
543 | { | |
544 | unsigned long *sp = (unsigned long *)reg29; | |
545 | unsigned long addr; | |
546 | ||
547 | while (!kstack_end(sp)) { | |
548 | addr = *sp++; | |
549 | if (__kernel_text_address(addr)) { | |
98f92f2f | 550 | perf_callchain_store(entry, addr); |
7e788d96 DCZ |
551 | if (entry->nr >= PERF_MAX_STACK_DEPTH) |
552 | break; | |
553 | } | |
554 | } | |
555 | } | |
556 | ||
98f92f2f DCZ |
557 | void perf_callchain_kernel(struct perf_callchain_entry *entry, |
558 | struct pt_regs *regs) | |
7e788d96 DCZ |
559 | { |
560 | unsigned long sp = regs->regs[29]; | |
561 | #ifdef CONFIG_KALLSYMS | |
562 | unsigned long ra = regs->regs[31]; | |
563 | unsigned long pc = regs->cp0_epc; | |
564 | ||
7e788d96 DCZ |
565 | if (raw_show_trace || !__kernel_text_address(pc)) { |
566 | unsigned long stack_page = | |
567 | (unsigned long)task_stack_page(current); | |
568 | if (stack_page && sp >= stack_page && | |
569 | sp <= stack_page + THREAD_SIZE - 32) | |
570 | save_raw_perf_callchain(entry, sp); | |
571 | return; | |
572 | } | |
573 | do { | |
98f92f2f | 574 | perf_callchain_store(entry, pc); |
7e788d96 DCZ |
575 | if (entry->nr >= PERF_MAX_STACK_DEPTH) |
576 | break; | |
577 | pc = unwind_stack(current, &sp, pc, &ra); | |
578 | } while (pc); | |
579 | #else | |
7e788d96 DCZ |
580 | save_raw_perf_callchain(entry, sp); |
581 | #endif | |
582 | } |