projects / deliverable / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge remote-tracking branch 'omap_dss2/for-next'
[deliverable/linux.git] / tools / perf / util / intel-pt-decoder / intel-pt-decoder.c
1 /*
2 * intel_pt_decoder.c: Intel Processor Trace support
3 * Copyright (c) 2013-2014, Intel Corporation.
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12 * more details.
13 *
14 */
15
16 #ifndef _GNU_SOURCE
17 #define _GNU_SOURCE
18 #endif
19 #include <stdlib.h>
20 #include <stdbool.h>
21 #include <string.h>
22 #include <errno.h>
23 #include <stdint.h>
24 #include <inttypes.h>
25
26 #include "../cache.h"
27 #include "../util.h"
28
29 #include "intel-pt-insn-decoder.h"
30 #include "intel-pt-pkt-decoder.h"
31 #include "intel-pt-decoder.h"
32 #include "intel-pt-log.h"
33
34 #define INTEL_PT_BLK_SIZE 1024
35
36 #define BIT63 (((uint64_t)1 << 63))
37
38 #define INTEL_PT_RETURN 1
39
40 /* Maximum number of loops with no packets consumed i.e. stuck in a loop */
41 #define INTEL_PT_MAX_LOOPS 10000
42
43 struct intel_pt_blk {
44 struct intel_pt_blk *prev;
45 uint64_t ip[INTEL_PT_BLK_SIZE];
46 };
47
48 struct intel_pt_stack {
49 struct intel_pt_blk *blk;
50 struct intel_pt_blk *spare;
51 int pos;
52 };
53
54 enum intel_pt_pkt_state {
55 INTEL_PT_STATE_NO_PSB,
56 INTEL_PT_STATE_NO_IP,
57 INTEL_PT_STATE_ERR_RESYNC,
58 INTEL_PT_STATE_IN_SYNC,
59 INTEL_PT_STATE_TNT,
60 INTEL_PT_STATE_TIP,
61 INTEL_PT_STATE_TIP_PGD,
62 INTEL_PT_STATE_FUP,
63 INTEL_PT_STATE_FUP_NO_TIP,
64 };
65
66 #ifdef INTEL_PT_STRICT
67 #define INTEL_PT_STATE_ERR1 INTEL_PT_STATE_NO_PSB
68 #define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_PSB
69 #define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_NO_PSB
70 #define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_NO_PSB
71 #else
72 #define INTEL_PT_STATE_ERR1 (decoder->pkt_state)
73 #define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_IP
74 #define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_ERR_RESYNC
75 #define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_IN_SYNC
76 #endif
77
78 struct intel_pt_decoder {
79 int (*get_trace)(struct intel_pt_buffer *buffer, void *data);
80 int (*walk_insn)(struct intel_pt_insn *intel_pt_insn,
81 uint64_t *insn_cnt_ptr, uint64_t *ip, uint64_t to_ip,
82 uint64_t max_insn_cnt, void *data);
83 void *data;
84 struct intel_pt_state state;
85 const unsigned char *buf;
86 size_t len;
87 bool return_compression;
88 bool mtc_insn;
89 bool pge;
90 bool have_tma;
91 bool have_cyc;
92 uint64_t pos;
93 uint64_t last_ip;
94 uint64_t ip;
95 uint64_t cr3;
96 uint64_t timestamp;
97 uint64_t tsc_timestamp;
98 uint64_t ref_timestamp;
99 uint64_t ret_addr;
100 uint64_t ctc_timestamp;
101 uint64_t ctc_delta;
102 uint64_t cycle_cnt;
103 uint64_t cyc_ref_timestamp;
104 uint32_t last_mtc;
105 uint32_t tsc_ctc_ratio_n;
106 uint32_t tsc_ctc_ratio_d;
107 uint32_t tsc_ctc_mult;
108 uint32_t tsc_slip;
109 uint32_t ctc_rem_mask;
110 int mtc_shift;
111 struct intel_pt_stack stack;
112 enum intel_pt_pkt_state pkt_state;
113 struct intel_pt_pkt packet;
114 struct intel_pt_pkt tnt;
115 int pkt_step;
116 int pkt_len;
117 int last_packet_type;
118 unsigned int cbr;
119 unsigned int max_non_turbo_ratio;
120 double max_non_turbo_ratio_fp;
121 double cbr_cyc_to_tsc;
122 double calc_cyc_to_tsc;
123 bool have_calc_cyc_to_tsc;
124 int exec_mode;
125 unsigned int insn_bytes;
126 uint64_t period;
127 enum intel_pt_period_type period_type;
128 uint64_t tot_insn_cnt;
129 uint64_t period_insn_cnt;
130 uint64_t period_mask;
131 uint64_t period_ticks;
132 uint64_t last_masked_timestamp;
133 bool continuous_period;
134 bool overflow;
135 bool set_fup_tx_flags;
136 unsigned int fup_tx_flags;
137 unsigned int tx_flags;
138 uint64_t timestamp_insn_cnt;
139 uint64_t stuck_ip;
140 int no_progress;
141 int stuck_ip_prd;
142 int stuck_ip_cnt;
143 const unsigned char *next_buf;
144 size_t next_len;
145 unsigned char temp_buf[INTEL_PT_PKT_MAX_SZ];
146 };
147
148 static uint64_t intel_pt_lower_power_of_2(uint64_t x)
149 {
150 int i;
151
152 for (i = 0; x != 1; i++)
153 x >>= 1;
154
155 return x << i;
156 }
157
158 static void intel_pt_setup_period(struct intel_pt_decoder *decoder)
159 {
160 if (decoder->period_type == INTEL_PT_PERIOD_TICKS) {
161 uint64_t period;
162
163 period = intel_pt_lower_power_of_2(decoder->period);
164 decoder->period_mask = ~(period - 1);
165 decoder->period_ticks = period;
166 }
167 }
168
169 static uint64_t multdiv(uint64_t t, uint32_t n, uint32_t d)
170 {
171 if (!d)
172 return 0;
173 return (t / d) * n + ((t % d) * n) / d;
174 }
175
176 struct intel_pt_decoder *intel_pt_decoder_new(struct intel_pt_params *params)
177 {
178 struct intel_pt_decoder *decoder;
179
180 if (!params->get_trace || !params->walk_insn)
181 return NULL;
182
183 decoder = zalloc(sizeof(struct intel_pt_decoder));
184 if (!decoder)
185 return NULL;
186
187 decoder->get_trace = params->get_trace;
188 decoder->walk_insn = params->walk_insn;
189 decoder->data = params->data;
190 decoder->return_compression = params->return_compression;
191
192 decoder->period = params->period;
193 decoder->period_type = params->period_type;
194
195 decoder->max_non_turbo_ratio = params->max_non_turbo_ratio;
196 decoder->max_non_turbo_ratio_fp = params->max_non_turbo_ratio;
197
198 intel_pt_setup_period(decoder);
199
200 decoder->mtc_shift = params->mtc_period;
201 decoder->ctc_rem_mask = (1 << decoder->mtc_shift) - 1;
202
203 decoder->tsc_ctc_ratio_n = params->tsc_ctc_ratio_n;
204 decoder->tsc_ctc_ratio_d = params->tsc_ctc_ratio_d;
205
206 if (!decoder->tsc_ctc_ratio_n)
207 decoder->tsc_ctc_ratio_d = 0;
208
209 if (decoder->tsc_ctc_ratio_d) {
210 if (!(decoder->tsc_ctc_ratio_n % decoder->tsc_ctc_ratio_d))
211 decoder->tsc_ctc_mult = decoder->tsc_ctc_ratio_n /
212 decoder->tsc_ctc_ratio_d;
213
214 /*
215 * Allow for timestamps appearing to backwards because a TSC
216 * packet has slipped past a MTC packet, so allow 2 MTC ticks
217 * or ...
218 */
219 decoder->tsc_slip = multdiv(2 << decoder->mtc_shift,
220 decoder->tsc_ctc_ratio_n,
221 decoder->tsc_ctc_ratio_d);
222 }
223 /* ... or 0x100 paranoia */
224 if (decoder->tsc_slip < 0x100)
225 decoder->tsc_slip = 0x100;
226
227 intel_pt_log("timestamp: mtc_shift %u\n", decoder->mtc_shift);
228 intel_pt_log("timestamp: tsc_ctc_ratio_n %u\n", decoder->tsc_ctc_ratio_n);
229 intel_pt_log("timestamp: tsc_ctc_ratio_d %u\n", decoder->tsc_ctc_ratio_d);
230 intel_pt_log("timestamp: tsc_ctc_mult %u\n", decoder->tsc_ctc_mult);
231 intel_pt_log("timestamp: tsc_slip %#x\n", decoder->tsc_slip);
232
233 return decoder;
234 }
235
236 static void intel_pt_pop_blk(struct intel_pt_stack *stack)
237 {
238 struct intel_pt_blk *blk = stack->blk;
239
240 stack->blk = blk->prev;
241 if (!stack->spare)
242 stack->spare = blk;
243 else
244 free(blk);
245 }
246
247 static uint64_t intel_pt_pop(struct intel_pt_stack *stack)
248 {
249 if (!stack->pos) {
250 if (!stack->blk)
251 return 0;
252 intel_pt_pop_blk(stack);
253 if (!stack->blk)
254 return 0;
255 stack->pos = INTEL_PT_BLK_SIZE;
256 }
257 return stack->blk->ip[--stack->pos];
258 }
259
260 static int intel_pt_alloc_blk(struct intel_pt_stack *stack)
261 {
262 struct intel_pt_blk *blk;
263
264 if (stack->spare) {
265 blk = stack->spare;
266 stack->spare = NULL;
267 } else {
268 blk = malloc(sizeof(struct intel_pt_blk));
269 if (!blk)
270 return -ENOMEM;
271 }
272
273 blk->prev = stack->blk;
274 stack->blk = blk;
275 stack->pos = 0;
276 return 0;
277 }
278
279 static int intel_pt_push(struct intel_pt_stack *stack, uint64_t ip)
280 {
281 int err;
282
283 if (!stack->blk || stack->pos == INTEL_PT_BLK_SIZE) {
284 err = intel_pt_alloc_blk(stack);
285 if (err)
286 return err;
287 }
288
289 stack->blk->ip[stack->pos++] = ip;
290 return 0;
291 }
292
293 static void intel_pt_clear_stack(struct intel_pt_stack *stack)
294 {
295 while (stack->blk)
296 intel_pt_pop_blk(stack);
297 stack->pos = 0;
298 }
299
300 static void intel_pt_free_stack(struct intel_pt_stack *stack)
301 {
302 intel_pt_clear_stack(stack);
303 zfree(&stack->blk);
304 zfree(&stack->spare);
305 }
306
307 void intel_pt_decoder_free(struct intel_pt_decoder *decoder)
308 {
309 intel_pt_free_stack(&decoder->stack);
310 free(decoder);
311 }
312
313 static int intel_pt_ext_err(int code)
314 {
315 switch (code) {
316 case -ENOMEM:
317 return INTEL_PT_ERR_NOMEM;
318 case -ENOSYS:
319 return INTEL_PT_ERR_INTERN;
320 case -EBADMSG:
321 return INTEL_PT_ERR_BADPKT;
322 case -ENODATA:
323 return INTEL_PT_ERR_NODATA;
324 case -EILSEQ:
325 return INTEL_PT_ERR_NOINSN;
326 case -ENOENT:
327 return INTEL_PT_ERR_MISMAT;
328 case -EOVERFLOW:
329 return INTEL_PT_ERR_OVR;
330 case -ENOSPC:
331 return INTEL_PT_ERR_LOST;
332 case -ELOOP:
333 return INTEL_PT_ERR_NELOOP;
334 default:
335 return INTEL_PT_ERR_UNK;
336 }
337 }
338
339 static const char *intel_pt_err_msgs[] = {
340 [INTEL_PT_ERR_NOMEM] = "Memory allocation failed",
341 [INTEL_PT_ERR_INTERN] = "Internal error",
342 [INTEL_PT_ERR_BADPKT] = "Bad packet",
343 [INTEL_PT_ERR_NODATA] = "No more data",
344 [INTEL_PT_ERR_NOINSN] = "Failed to get instruction",
345 [INTEL_PT_ERR_MISMAT] = "Trace doesn't match instruction",
346 [INTEL_PT_ERR_OVR] = "Overflow packet",
347 [INTEL_PT_ERR_LOST] = "Lost trace data",
348 [INTEL_PT_ERR_UNK] = "Unknown error!",
349 [INTEL_PT_ERR_NELOOP] = "Never-ending loop",
350 };
351
352 int intel_pt__strerror(int code, char *buf, size_t buflen)
353 {
354 if (code < 1 || code >= INTEL_PT_ERR_MAX)
355 code = INTEL_PT_ERR_UNK;
356 strlcpy(buf, intel_pt_err_msgs[code], buflen);
357 return 0;
358 }
359
360 static uint64_t intel_pt_calc_ip(const struct intel_pt_pkt *packet,
361 uint64_t last_ip)
362 {
363 uint64_t ip;
364
365 switch (packet->count) {
366 case 1:
367 ip = (last_ip & (uint64_t)0xffffffffffff0000ULL) |
368 packet->payload;
369 break;
370 case 2:
371 ip = (last_ip & (uint64_t)0xffffffff00000000ULL) |
372 packet->payload;
373 break;
374 case 3:
375 ip = packet->payload;
376 /* Sign-extend 6-byte ip */
377 if (ip & (uint64_t)0x800000000000ULL)
378 ip |= (uint64_t)0xffff000000000000ULL;
379 break;
380 case 4:
381 ip = (last_ip & (uint64_t)0xffff000000000000ULL) |
382 packet->payload;
383 break;
384 case 6:
385 ip = packet->payload;
386 break;
387 default:
388 return 0;
389 }
390
391 return ip;
392 }
393
394 static inline void intel_pt_set_last_ip(struct intel_pt_decoder *decoder)
395 {
396 decoder->last_ip = intel_pt_calc_ip(&decoder->packet, decoder->last_ip);
397 }
398
399 static inline void intel_pt_set_ip(struct intel_pt_decoder *decoder)
400 {
401 intel_pt_set_last_ip(decoder);
402 decoder->ip = decoder->last_ip;
403 }
404
405 static void intel_pt_decoder_log_packet(struct intel_pt_decoder *decoder)
406 {
407 intel_pt_log_packet(&decoder->packet, decoder->pkt_len, decoder->pos,
408 decoder->buf);
409 }
410
411 static int intel_pt_bug(struct intel_pt_decoder *decoder)
412 {
413 intel_pt_log("ERROR: Internal error\n");
414 decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
415 return -ENOSYS;
416 }
417
418 static inline void intel_pt_clear_tx_flags(struct intel_pt_decoder *decoder)
419 {
420 decoder->tx_flags = 0;
421 }
422
423 static inline void intel_pt_update_in_tx(struct intel_pt_decoder *decoder)
424 {
425 decoder->tx_flags = decoder->packet.payload & INTEL_PT_IN_TX;
426 }
427
428 static int intel_pt_bad_packet(struct intel_pt_decoder *decoder)
429 {
430 intel_pt_clear_tx_flags(decoder);
431 decoder->have_tma = false;
432 decoder->pkt_len = 1;
433 decoder->pkt_step = 1;
434 intel_pt_decoder_log_packet(decoder);
435 if (decoder->pkt_state != INTEL_PT_STATE_NO_PSB) {
436 intel_pt_log("ERROR: Bad packet\n");
437 decoder->pkt_state = INTEL_PT_STATE_ERR1;
438 }
439 return -EBADMSG;
440 }
441
442 static int intel_pt_get_data(struct intel_pt_decoder *decoder)
443 {
444 struct intel_pt_buffer buffer = { .buf = 0, };
445 int ret;
446
447 decoder->pkt_step = 0;
448
449 intel_pt_log("Getting more data\n");
450 ret = decoder->get_trace(&buffer, decoder->data);
451 if (ret)
452 return ret;
453 decoder->buf = buffer.buf;
454 decoder->len = buffer.len;
455 if (!decoder->len) {
456 intel_pt_log("No more data\n");
457 return -ENODATA;
458 }
459 if (!buffer.consecutive) {
460 decoder->ip = 0;
461 decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
462 decoder->ref_timestamp = buffer.ref_timestamp;
463 decoder->timestamp = 0;
464 decoder->have_tma = false;
465 decoder->state.trace_nr = buffer.trace_nr;
466 intel_pt_log("Reference timestamp 0x%" PRIx64 "\n",
467 decoder->ref_timestamp);
468 return -ENOLINK;
469 }
470
471 return 0;
472 }
473
474 static int intel_pt_get_next_data(struct intel_pt_decoder *decoder)
475 {
476 if (!decoder->next_buf)
477 return intel_pt_get_data(decoder);
478
479 decoder->buf = decoder->next_buf;
480 decoder->len = decoder->next_len;
481 decoder->next_buf = 0;
482 decoder->next_len = 0;
483 return 0;
484 }
485
486 static int intel_pt_get_split_packet(struct intel_pt_decoder *decoder)
487 {
488 unsigned char *buf = decoder->temp_buf;
489 size_t old_len, len, n;
490 int ret;
491
492 old_len = decoder->len;
493 len = decoder->len;
494 memcpy(buf, decoder->buf, len);
495
496 ret = intel_pt_get_data(decoder);
497 if (ret) {
498 decoder->pos += old_len;
499 return ret < 0 ? ret : -EINVAL;
500 }
501
502 n = INTEL_PT_PKT_MAX_SZ - len;
503 if (n > decoder->len)
504 n = decoder->len;
505 memcpy(buf + len, decoder->buf, n);
506 len += n;
507
508 ret = intel_pt_get_packet(buf, len, &decoder->packet);
509 if (ret < (int)old_len) {
510 decoder->next_buf = decoder->buf;
511 decoder->next_len = decoder->len;
512 decoder->buf = buf;
513 decoder->len = old_len;
514 return intel_pt_bad_packet(decoder);
515 }
516
517 decoder->next_buf = decoder->buf + (ret - old_len);
518 decoder->next_len = decoder->len - (ret - old_len);
519
520 decoder->buf = buf;
521 decoder->len = ret;
522
523 return ret;
524 }
525
526 struct intel_pt_pkt_info {
527 struct intel_pt_decoder *decoder;
528 struct intel_pt_pkt packet;
529 uint64_t pos;
530 int pkt_len;
531 int last_packet_type;
532 void *data;
533 };
534
535 typedef int (*intel_pt_pkt_cb_t)(struct intel_pt_pkt_info *pkt_info);
536
537 /* Lookahead packets in current buffer */
538 static int intel_pt_pkt_lookahead(struct intel_pt_decoder *decoder,
539 intel_pt_pkt_cb_t cb, void *data)
540 {
541 struct intel_pt_pkt_info pkt_info;
542 const unsigned char *buf = decoder->buf;
543 size_t len = decoder->len;
544 int ret;
545
546 pkt_info.decoder = decoder;
547 pkt_info.pos = decoder->pos;
548 pkt_info.pkt_len = decoder->pkt_step;
549 pkt_info.last_packet_type = decoder->last_packet_type;
550 pkt_info.data = data;
551
552 while (1) {
553 do {
554 pkt_info.pos += pkt_info.pkt_len;
555 buf += pkt_info.pkt_len;
556 len -= pkt_info.pkt_len;
557
558 if (!len)
559 return INTEL_PT_NEED_MORE_BYTES;
560
561 ret = intel_pt_get_packet(buf, len, &pkt_info.packet);
562 if (!ret)
563 return INTEL_PT_NEED_MORE_BYTES;
564 if (ret < 0)
565 return ret;
566
567 pkt_info.pkt_len = ret;
568 } while (pkt_info.packet.type == INTEL_PT_PAD);
569
570 ret = cb(&pkt_info);
571 if (ret)
572 return 0;
573
574 pkt_info.last_packet_type = pkt_info.packet.type;
575 }
576 }
577
578 struct intel_pt_calc_cyc_to_tsc_info {
579 uint64_t cycle_cnt;
580 unsigned int cbr;
581 uint32_t last_mtc;
582 uint64_t ctc_timestamp;
583 uint64_t ctc_delta;
584 uint64_t tsc_timestamp;
585 uint64_t timestamp;
586 bool have_tma;
587 bool from_mtc;
588 double cbr_cyc_to_tsc;
589 };
590
591 static int intel_pt_calc_cyc_cb(struct intel_pt_pkt_info *pkt_info)
592 {
593 struct intel_pt_decoder *decoder = pkt_info->decoder;
594 struct intel_pt_calc_cyc_to_tsc_info *data = pkt_info->data;
595 uint64_t timestamp;
596 double cyc_to_tsc;
597 unsigned int cbr;
598 uint32_t mtc, mtc_delta, ctc, fc, ctc_rem;
599
600 switch (pkt_info->packet.type) {
601 case INTEL_PT_TNT:
602 case INTEL_PT_TIP_PGE:
603 case INTEL_PT_TIP:
604 case INTEL_PT_FUP:
605 case INTEL_PT_PSB:
606 case INTEL_PT_PIP:
607 case INTEL_PT_MODE_EXEC:
608 case INTEL_PT_MODE_TSX:
609 case INTEL_PT_PSBEND:
610 case INTEL_PT_PAD:
611 case INTEL_PT_VMCS:
612 case INTEL_PT_MNT:
613 return 0;
614
615 case INTEL_PT_MTC:
616 if (!data->have_tma)
617 return 0;
618
619 mtc = pkt_info->packet.payload;
620 if (mtc > data->last_mtc)
621 mtc_delta = mtc - data->last_mtc;
622 else
623 mtc_delta = mtc + 256 - data->last_mtc;
624 data->ctc_delta += mtc_delta << decoder->mtc_shift;
625 data->last_mtc = mtc;
626
627 if (decoder->tsc_ctc_mult) {
628 timestamp = data->ctc_timestamp +
629 data->ctc_delta * decoder->tsc_ctc_mult;
630 } else {
631 timestamp = data->ctc_timestamp +
632 multdiv(data->ctc_delta,
633 decoder->tsc_ctc_ratio_n,
634 decoder->tsc_ctc_ratio_d);
635 }
636
637 if (timestamp < data->timestamp)
638 return 1;
639
640 if (pkt_info->last_packet_type != INTEL_PT_CYC) {
641 data->timestamp = timestamp;
642 return 0;
643 }
644
645 break;
646
647 case INTEL_PT_TSC:
648 timestamp = pkt_info->packet.payload |
649 (data->timestamp & (0xffULL << 56));
650 if (data->from_mtc && timestamp < data->timestamp &&
651 data->timestamp - timestamp < decoder->tsc_slip)
652 return 1;
653 if (timestamp < data->timestamp)
654 timestamp += (1ULL << 56);
655 if (pkt_info->last_packet_type != INTEL_PT_CYC) {
656 if (data->from_mtc)
657 return 1;
658 data->tsc_timestamp = timestamp;
659 data->timestamp = timestamp;
660 return 0;
661 }
662 break;
663
664 case INTEL_PT_TMA:
665 if (data->from_mtc)
666 return 1;
667
668 if (!decoder->tsc_ctc_ratio_d)
669 return 0;
670
671 ctc = pkt_info->packet.payload;
672 fc = pkt_info->packet.count;
673 ctc_rem = ctc & decoder->ctc_rem_mask;
674
675 data->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
676
677 data->ctc_timestamp = data->tsc_timestamp - fc;
678 if (decoder->tsc_ctc_mult) {
679 data->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;
680 } else {
681 data->ctc_timestamp -=
682 multdiv(ctc_rem, decoder->tsc_ctc_ratio_n,
683 decoder->tsc_ctc_ratio_d);
684 }
685
686 data->ctc_delta = 0;
687 data->have_tma = true;
688
689 return 0;
690
691 case INTEL_PT_CYC:
692 data->cycle_cnt += pkt_info->packet.payload;
693 return 0;
694
695 case INTEL_PT_CBR:
696 cbr = pkt_info->packet.payload;
697 if (data->cbr && data->cbr != cbr)
698 return 1;
699 data->cbr = cbr;
700 data->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;
701 return 0;
702
703 case INTEL_PT_TIP_PGD:
704 case INTEL_PT_TRACESTOP:
705 case INTEL_PT_OVF:
706 case INTEL_PT_BAD: /* Does not happen */
707 default:
708 return 1;
709 }
710
711 if (!data->cbr && decoder->cbr) {
712 data->cbr = decoder->cbr;
713 data->cbr_cyc_to_tsc = decoder->cbr_cyc_to_tsc;
714 }
715
716 if (!data->cycle_cnt)
717 return 1;
718
719 cyc_to_tsc = (double)(timestamp - decoder->timestamp) / data->cycle_cnt;
720
721 if (data->cbr && cyc_to_tsc > data->cbr_cyc_to_tsc &&
722 cyc_to_tsc / data->cbr_cyc_to_tsc > 1.25) {
723 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle too big (c.f. CBR-based value %g), pos " x64_fmt "\n",
724 cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos);
725 return 1;
726 }
727
728 decoder->calc_cyc_to_tsc = cyc_to_tsc;
729 decoder->have_calc_cyc_to_tsc = true;
730
731 if (data->cbr) {
732 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. CBR-based value %g, pos " x64_fmt "\n",
733 cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos);
734 } else {
735 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. unknown CBR-based value, pos " x64_fmt "\n",
736 cyc_to_tsc, pkt_info->pos);
737 }
738
739 return 1;
740 }
741
742 static void intel_pt_calc_cyc_to_tsc(struct intel_pt_decoder *decoder,
743 bool from_mtc)
744 {
745 struct intel_pt_calc_cyc_to_tsc_info data = {
746 .cycle_cnt = 0,
747 .cbr = 0,
748 .last_mtc = decoder->last_mtc,
749 .ctc_timestamp = decoder->ctc_timestamp,
750 .ctc_delta = decoder->ctc_delta,
751 .tsc_timestamp = decoder->tsc_timestamp,
752 .timestamp = decoder->timestamp,
753 .have_tma = decoder->have_tma,
754 .from_mtc = from_mtc,
755 .cbr_cyc_to_tsc = 0,
756 };
757
758 intel_pt_pkt_lookahead(decoder, intel_pt_calc_cyc_cb, &data);
759 }
760
761 static int intel_pt_get_next_packet(struct intel_pt_decoder *decoder)
762 {
763 int ret;
764
765 decoder->last_packet_type = decoder->packet.type;
766
767 do {
768 decoder->pos += decoder->pkt_step;
769 decoder->buf += decoder->pkt_step;
770 decoder->len -= decoder->pkt_step;
771
772 if (!decoder->len) {
773 ret = intel_pt_get_next_data(decoder);
774 if (ret)
775 return ret;
776 }
777
778 ret = intel_pt_get_packet(decoder->buf, decoder->len,
779 &decoder->packet);
780 if (ret == INTEL_PT_NEED_MORE_BYTES &&
781 decoder->len < INTEL_PT_PKT_MAX_SZ && !decoder->next_buf) {
782 ret = intel_pt_get_split_packet(decoder);
783 if (ret < 0)
784 return ret;
785 }
786 if (ret <= 0)
787 return intel_pt_bad_packet(decoder);
788
789 decoder->pkt_len = ret;
790 decoder->pkt_step = ret;
791 intel_pt_decoder_log_packet(decoder);
792 } while (decoder->packet.type == INTEL_PT_PAD);
793
794 return 0;
795 }
796
797 static uint64_t intel_pt_next_period(struct intel_pt_decoder *decoder)
798 {
799 uint64_t timestamp, masked_timestamp;
800
801 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;
802 masked_timestamp = timestamp & decoder->period_mask;
803 if (decoder->continuous_period) {
804 if (masked_timestamp != decoder->last_masked_timestamp)
805 return 1;
806 } else {
807 timestamp += 1;
808 masked_timestamp = timestamp & decoder->period_mask;
809 if (masked_timestamp != decoder->last_masked_timestamp) {
810 decoder->last_masked_timestamp = masked_timestamp;
811 decoder->continuous_period = true;
812 }
813 }
814 return decoder->period_ticks - (timestamp - masked_timestamp);
815 }
816
817 static uint64_t intel_pt_next_sample(struct intel_pt_decoder *decoder)
818 {
819 switch (decoder->period_type) {
820 case INTEL_PT_PERIOD_INSTRUCTIONS:
821 return decoder->period - decoder->period_insn_cnt;
822 case INTEL_PT_PERIOD_TICKS:
823 return intel_pt_next_period(decoder);
824 case INTEL_PT_PERIOD_NONE:
825 case INTEL_PT_PERIOD_MTC:
826 default:
827 return 0;
828 }
829 }
830
831 static void intel_pt_sample_insn(struct intel_pt_decoder *decoder)
832 {
833 uint64_t timestamp, masked_timestamp;
834
835 switch (decoder->period_type) {
836 case INTEL_PT_PERIOD_INSTRUCTIONS:
837 decoder->period_insn_cnt = 0;
838 break;
839 case INTEL_PT_PERIOD_TICKS:
840 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;
841 masked_timestamp = timestamp & decoder->period_mask;
842 decoder->last_masked_timestamp = masked_timestamp;
843 break;
844 case INTEL_PT_PERIOD_NONE:
845 case INTEL_PT_PERIOD_MTC:
846 default:
847 break;
848 }
849
850 decoder->state.type |= INTEL_PT_INSTRUCTION;
851 }
852
853 static int intel_pt_walk_insn(struct intel_pt_decoder *decoder,
854 struct intel_pt_insn *intel_pt_insn, uint64_t ip)
855 {
856 uint64_t max_insn_cnt, insn_cnt = 0;
857 int err;
858
859 if (!decoder->mtc_insn)
860 decoder->mtc_insn = true;
861
862 max_insn_cnt = intel_pt_next_sample(decoder);
863
864 err = decoder->walk_insn(intel_pt_insn, &insn_cnt, &decoder->ip, ip,
865 max_insn_cnt, decoder->data);
866
867 decoder->tot_insn_cnt += insn_cnt;
868 decoder->timestamp_insn_cnt += insn_cnt;
869 decoder->period_insn_cnt += insn_cnt;
870
871 if (err) {
872 decoder->no_progress = 0;
873 decoder->pkt_state = INTEL_PT_STATE_ERR2;
874 intel_pt_log_at("ERROR: Failed to get instruction",
875 decoder->ip);
876 if (err == -ENOENT)
877 return -ENOLINK;
878 return -EILSEQ;
879 }
880
881 if (ip && decoder->ip == ip) {
882 err = -EAGAIN;
883 goto out;
884 }
885
886 if (max_insn_cnt && insn_cnt >= max_insn_cnt)
887 intel_pt_sample_insn(decoder);
888
889 if (intel_pt_insn->branch == INTEL_PT_BR_NO_BRANCH) {
890 decoder->state.type = INTEL_PT_INSTRUCTION;
891 decoder->state.from_ip = decoder->ip;
892 decoder->state.to_ip = 0;
893 decoder->ip += intel_pt_insn->length;
894 err = INTEL_PT_RETURN;
895 goto out;
896 }
897
898 if (intel_pt_insn->op == INTEL_PT_OP_CALL) {
899 /* Zero-length calls are excluded */
900 if (intel_pt_insn->branch != INTEL_PT_BR_UNCONDITIONAL ||
901 intel_pt_insn->rel) {
902 err = intel_pt_push(&decoder->stack, decoder->ip +
903 intel_pt_insn->length);
904 if (err)
905 goto out;
906 }
907 } else if (intel_pt_insn->op == INTEL_PT_OP_RET) {
908 decoder->ret_addr = intel_pt_pop(&decoder->stack);
909 }
910
911 if (intel_pt_insn->branch == INTEL_PT_BR_UNCONDITIONAL) {
912 int cnt = decoder->no_progress++;
913
914 decoder->state.from_ip = decoder->ip;
915 decoder->ip += intel_pt_insn->length +
916 intel_pt_insn->rel;
917 decoder->state.to_ip = decoder->ip;
918 err = INTEL_PT_RETURN;
919
920 /*
921 * Check for being stuck in a loop. This can happen if a
922 * decoder error results in the decoder erroneously setting the
923 * ip to an address that is itself in an infinite loop that
924 * consumes no packets. When that happens, there must be an
925 * unconditional branch.
926 */
927 if (cnt) {
928 if (cnt == 1) {
929 decoder->stuck_ip = decoder->state.to_ip;
930 decoder->stuck_ip_prd = 1;
931 decoder->stuck_ip_cnt = 1;
932 } else if (cnt > INTEL_PT_MAX_LOOPS ||
933 decoder->state.to_ip == decoder->stuck_ip) {
934 intel_pt_log_at("ERROR: Never-ending loop",
935 decoder->state.to_ip);
936 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
937 err = -ELOOP;
938 goto out;
939 } else if (!--decoder->stuck_ip_cnt) {
940 decoder->stuck_ip_prd += 1;
941 decoder->stuck_ip_cnt = decoder->stuck_ip_prd;
942 decoder->stuck_ip = decoder->state.to_ip;
943 }
944 }
945 goto out_no_progress;
946 }
947 out:
948 decoder->no_progress = 0;
949 out_no_progress:
950 decoder->state.insn_op = intel_pt_insn->op;
951 decoder->state.insn_len = intel_pt_insn->length;
952
953 if (decoder->tx_flags & INTEL_PT_IN_TX)
954 decoder->state.flags |= INTEL_PT_IN_TX;
955
956 return err;
957 }
958
959 static int intel_pt_walk_fup(struct intel_pt_decoder *decoder)
960 {
961 struct intel_pt_insn intel_pt_insn;
962 uint64_t ip;
963 int err;
964
965 ip = decoder->last_ip;
966
967 while (1) {
968 err = intel_pt_walk_insn(decoder, &intel_pt_insn, ip);
969 if (err == INTEL_PT_RETURN)
970 return 0;
971 if (err == -EAGAIN) {
972 if (decoder->set_fup_tx_flags) {
973 decoder->set_fup_tx_flags = false;
974 decoder->tx_flags = decoder->fup_tx_flags;
975 decoder->state.type = INTEL_PT_TRANSACTION;
976 decoder->state.from_ip = decoder->ip;
977 decoder->state.to_ip = 0;
978 decoder->state.flags = decoder->fup_tx_flags;
979 return 0;
980 }
981 return err;
982 }
983 decoder->set_fup_tx_flags = false;
984 if (err)
985 return err;
986
987 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
988 intel_pt_log_at("ERROR: Unexpected indirect branch",
989 decoder->ip);
990 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
991 return -ENOENT;
992 }
993
994 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
995 intel_pt_log_at("ERROR: Unexpected conditional branch",
996 decoder->ip);
997 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
998 return -ENOENT;
999 }
1000
1001 intel_pt_bug(decoder);
1002 }
1003 }
1004
1005 static int intel_pt_walk_tip(struct intel_pt_decoder *decoder)
1006 {
1007 struct intel_pt_insn intel_pt_insn;
1008 int err;
1009
1010 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);
1011 if (err == INTEL_PT_RETURN)
1012 return 0;
1013 if (err)
1014 return err;
1015
1016 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1017 if (decoder->pkt_state == INTEL_PT_STATE_TIP_PGD) {
1018 decoder->pge = false;
1019 decoder->continuous_period = false;
1020 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1021 decoder->state.from_ip = decoder->ip;
1022 decoder->state.to_ip = 0;
1023 if (decoder->packet.count != 0)
1024 decoder->ip = decoder->last_ip;
1025 } else {
1026 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1027 decoder->state.from_ip = decoder->ip;
1028 if (decoder->packet.count == 0) {
1029 decoder->state.to_ip = 0;
1030 } else {
1031 decoder->state.to_ip = decoder->last_ip;
1032 decoder->ip = decoder->last_ip;
1033 }
1034 }
1035 return 0;
1036 }
1037
1038 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1039 intel_pt_log_at("ERROR: Conditional branch when expecting indirect branch",
1040 decoder->ip);
1041 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1042 return -ENOENT;
1043 }
1044
1045 return intel_pt_bug(decoder);
1046 }
1047
1048 static int intel_pt_walk_tnt(struct intel_pt_decoder *decoder)
1049 {
1050 struct intel_pt_insn intel_pt_insn;
1051 int err;
1052
1053 while (1) {
1054 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);
1055 if (err == INTEL_PT_RETURN)
1056 return 0;
1057 if (err)
1058 return err;
1059
1060 if (intel_pt_insn.op == INTEL_PT_OP_RET) {
1061 if (!decoder->return_compression) {
1062 intel_pt_log_at("ERROR: RET when expecting conditional branch",
1063 decoder->ip);
1064 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1065 return -ENOENT;
1066 }
1067 if (!decoder->ret_addr) {
1068 intel_pt_log_at("ERROR: Bad RET compression (stack empty)",
1069 decoder->ip);
1070 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1071 return -ENOENT;
1072 }
1073 if (!(decoder->tnt.payload & BIT63)) {
1074 intel_pt_log_at("ERROR: Bad RET compression (TNT=N)",
1075 decoder->ip);
1076 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1077 return -ENOENT;
1078 }
1079 decoder->tnt.count -= 1;
1080 if (!decoder->tnt.count)
1081 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1082 decoder->tnt.payload <<= 1;
1083 decoder->state.from_ip = decoder->ip;
1084 decoder->ip = decoder->ret_addr;
1085 decoder->state.to_ip = decoder->ip;
1086 return 0;
1087 }
1088
1089 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1090 /* Handle deferred TIPs */
1091 err = intel_pt_get_next_packet(decoder);
1092 if (err)
1093 return err;
1094 if (decoder->packet.type != INTEL_PT_TIP ||
1095 decoder->packet.count == 0) {
1096 intel_pt_log_at("ERROR: Missing deferred TIP for indirect branch",
1097 decoder->ip);
1098 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1099 decoder->pkt_step = 0;
1100 return -ENOENT;
1101 }
1102 intel_pt_set_last_ip(decoder);
1103 decoder->state.from_ip = decoder->ip;
1104 decoder->state.to_ip = decoder->last_ip;
1105 decoder->ip = decoder->last_ip;
1106 return 0;
1107 }
1108
1109 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1110 decoder->tnt.count -= 1;
1111 if (!decoder->tnt.count)
1112 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1113 if (decoder->tnt.payload & BIT63) {
1114 decoder->tnt.payload <<= 1;
1115 decoder->state.from_ip = decoder->ip;
1116 decoder->ip += intel_pt_insn.length +
1117 intel_pt_insn.rel;
1118 decoder->state.to_ip = decoder->ip;
1119 return 0;
1120 }
1121 /* Instruction sample for a non-taken branch */
1122 if (decoder->state.type & INTEL_PT_INSTRUCTION) {
1123 decoder->tnt.payload <<= 1;
1124 decoder->state.type = INTEL_PT_INSTRUCTION;
1125 decoder->state.from_ip = decoder->ip;
1126 decoder->state.to_ip = 0;
1127 decoder->ip += intel_pt_insn.length;
1128 return 0;
1129 }
1130 decoder->ip += intel_pt_insn.length;
1131 if (!decoder->tnt.count)
1132 return -EAGAIN;
1133 decoder->tnt.payload <<= 1;
1134 continue;
1135 }
1136
1137 return intel_pt_bug(decoder);
1138 }
1139 }
1140
1141 static int intel_pt_mode_tsx(struct intel_pt_decoder *decoder, bool *no_tip)
1142 {
1143 unsigned int fup_tx_flags;
1144 int err;
1145
1146 fup_tx_flags = decoder->packet.payload &
1147 (INTEL_PT_IN_TX | INTEL_PT_ABORT_TX);
1148 err = intel_pt_get_next_packet(decoder);
1149 if (err)
1150 return err;
1151 if (decoder->packet.type == INTEL_PT_FUP) {
1152 decoder->fup_tx_flags = fup_tx_flags;
1153 decoder->set_fup_tx_flags = true;
1154 if (!(decoder->fup_tx_flags & INTEL_PT_ABORT_TX))
1155 *no_tip = true;
1156 } else {
1157 intel_pt_log_at("ERROR: Missing FUP after MODE.TSX",
1158 decoder->pos);
1159 intel_pt_update_in_tx(decoder);
1160 }
1161 return 0;
1162 }
1163
1164 static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder *decoder)
1165 {
1166 uint64_t timestamp;
1167
1168 decoder->have_tma = false;
1169
1170 if (decoder->ref_timestamp) {
1171 timestamp = decoder->packet.payload |
1172 (decoder->ref_timestamp & (0xffULL << 56));
1173 if (timestamp < decoder->ref_timestamp) {
1174 if (decoder->ref_timestamp - timestamp > (1ULL << 55))
1175 timestamp += (1ULL << 56);
1176 } else {
1177 if (timestamp - decoder->ref_timestamp > (1ULL << 55))
1178 timestamp -= (1ULL << 56);
1179 }
1180 decoder->tsc_timestamp = timestamp;
1181 decoder->timestamp = timestamp;
1182 decoder->ref_timestamp = 0;
1183 decoder->timestamp_insn_cnt = 0;
1184 } else if (decoder->timestamp) {
1185 timestamp = decoder->packet.payload |
1186 (decoder->timestamp & (0xffULL << 56));
1187 decoder->tsc_timestamp = timestamp;
1188 if (timestamp < decoder->timestamp &&
1189 decoder->timestamp - timestamp < decoder->tsc_slip) {
1190 intel_pt_log_to("Suppressing backwards timestamp",
1191 timestamp);
1192 timestamp = decoder->timestamp;
1193 }
1194 if (timestamp < decoder->timestamp) {
1195 intel_pt_log_to("Wraparound timestamp", timestamp);
1196 timestamp += (1ULL << 56);
1197 decoder->tsc_timestamp = timestamp;
1198 }
1199 decoder->timestamp = timestamp;
1200 decoder->timestamp_insn_cnt = 0;
1201 }
1202
1203 if (decoder->last_packet_type == INTEL_PT_CYC) {
1204 decoder->cyc_ref_timestamp = decoder->timestamp;
1205 decoder->cycle_cnt = 0;
1206 decoder->have_calc_cyc_to_tsc = false;
1207 intel_pt_calc_cyc_to_tsc(decoder, false);
1208 }
1209
1210 intel_pt_log_to("Setting timestamp", decoder->timestamp);
1211 }
1212
1213 static int intel_pt_overflow(struct intel_pt_decoder *decoder)
1214 {
1215 intel_pt_log("ERROR: Buffer overflow\n");
1216 intel_pt_clear_tx_flags(decoder);
1217 decoder->have_tma = false;
1218 decoder->cbr = 0;
1219 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1220 decoder->overflow = true;
1221 return -EOVERFLOW;
1222 }
1223
1224 static void intel_pt_calc_tma(struct intel_pt_decoder *decoder)
1225 {
1226 uint32_t ctc = decoder->packet.payload;
1227 uint32_t fc = decoder->packet.count;
1228 uint32_t ctc_rem = ctc & decoder->ctc_rem_mask;
1229
1230 if (!decoder->tsc_ctc_ratio_d)
1231 return;
1232
1233 decoder->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
1234 decoder->ctc_timestamp = decoder->tsc_timestamp - fc;
1235 if (decoder->tsc_ctc_mult) {
1236 decoder->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;
1237 } else {
1238 decoder->ctc_timestamp -= multdiv(ctc_rem,
1239 decoder->tsc_ctc_ratio_n,
1240 decoder->tsc_ctc_ratio_d);
1241 }
1242 decoder->ctc_delta = 0;
1243 decoder->have_tma = true;
1244 intel_pt_log("CTC timestamp " x64_fmt " last MTC %#x CTC rem %#x\n",
1245 decoder->ctc_timestamp, decoder->last_mtc, ctc_rem);
1246 }
1247
1248 static void intel_pt_calc_mtc_timestamp(struct intel_pt_decoder *decoder)
1249 {
1250 uint64_t timestamp;
1251 uint32_t mtc, mtc_delta;
1252
1253 if (!decoder->have_tma)
1254 return;
1255
1256 mtc = decoder->packet.payload;
1257
1258 if (mtc > decoder->last_mtc)
1259 mtc_delta = mtc - decoder->last_mtc;
1260 else
1261 mtc_delta = mtc + 256 - decoder->last_mtc;
1262
1263 decoder->ctc_delta += mtc_delta << decoder->mtc_shift;
1264
1265 if (decoder->tsc_ctc_mult) {
1266 timestamp = decoder->ctc_timestamp +
1267 decoder->ctc_delta * decoder->tsc_ctc_mult;
1268 } else {
1269 timestamp = decoder->ctc_timestamp +
1270 multdiv(decoder->ctc_delta,
1271 decoder->tsc_ctc_ratio_n,
1272 decoder->tsc_ctc_ratio_d);
1273 }
1274
1275 if (timestamp < decoder->timestamp)
1276 intel_pt_log("Suppressing MTC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n",
1277 timestamp, decoder->timestamp);
1278 else
1279 decoder->timestamp = timestamp;
1280
1281 decoder->timestamp_insn_cnt = 0;
1282 decoder->last_mtc = mtc;
1283
1284 if (decoder->last_packet_type == INTEL_PT_CYC) {
1285 decoder->cyc_ref_timestamp = decoder->timestamp;
1286 decoder->cycle_cnt = 0;
1287 decoder->have_calc_cyc_to_tsc = false;
1288 intel_pt_calc_cyc_to_tsc(decoder, true);
1289 }
1290 }
1291
1292 static void intel_pt_calc_cbr(struct intel_pt_decoder *decoder)
1293 {
1294 unsigned int cbr = decoder->packet.payload;
1295
1296 if (decoder->cbr == cbr)
1297 return;
1298
1299 decoder->cbr = cbr;
1300 decoder->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;
1301 }
1302
1303 static void intel_pt_calc_cyc_timestamp(struct intel_pt_decoder *decoder)
1304 {
1305 uint64_t timestamp = decoder->cyc_ref_timestamp;
1306
1307 decoder->have_cyc = true;
1308
1309 decoder->cycle_cnt += decoder->packet.payload;
1310
1311 if (!decoder->cyc_ref_timestamp)
1312 return;
1313
1314 if (decoder->have_calc_cyc_to_tsc)
1315 timestamp += decoder->cycle_cnt * decoder->calc_cyc_to_tsc;
1316 else if (decoder->cbr)
1317 timestamp += decoder->cycle_cnt * decoder->cbr_cyc_to_tsc;
1318 else
1319 return;
1320
1321 if (timestamp < decoder->timestamp)
1322 intel_pt_log("Suppressing CYC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n",
1323 timestamp, decoder->timestamp);
1324 else
1325 decoder->timestamp = timestamp;
1326 }
1327
1328 /* Walk PSB+ packets when already in sync. */
1329 static int intel_pt_walk_psbend(struct intel_pt_decoder *decoder)
1330 {
1331 int err;
1332
1333 while (1) {
1334 err = intel_pt_get_next_packet(decoder);
1335 if (err)
1336 return err;
1337
1338 switch (decoder->packet.type) {
1339 case INTEL_PT_PSBEND:
1340 return 0;
1341
1342 case INTEL_PT_TIP_PGD:
1343 case INTEL_PT_TIP_PGE:
1344 case INTEL_PT_TIP:
1345 case INTEL_PT_TNT:
1346 case INTEL_PT_TRACESTOP:
1347 case INTEL_PT_BAD:
1348 case INTEL_PT_PSB:
1349 decoder->have_tma = false;
1350 intel_pt_log("ERROR: Unexpected packet\n");
1351 return -EAGAIN;
1352
1353 case INTEL_PT_OVF:
1354 return intel_pt_overflow(decoder);
1355
1356 case INTEL_PT_TSC:
1357 intel_pt_calc_tsc_timestamp(decoder);
1358 break;
1359
1360 case INTEL_PT_TMA:
1361 intel_pt_calc_tma(decoder);
1362 break;
1363
1364 case INTEL_PT_CBR:
1365 intel_pt_calc_cbr(decoder);
1366 break;
1367
1368 case INTEL_PT_MODE_EXEC:
1369 decoder->exec_mode = decoder->packet.payload;
1370 break;
1371
1372 case INTEL_PT_PIP:
1373 decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1374 break;
1375
1376 case INTEL_PT_FUP:
1377 decoder->pge = true;
1378 intel_pt_set_last_ip(decoder);
1379 break;
1380
1381 case INTEL_PT_MODE_TSX:
1382 intel_pt_update_in_tx(decoder);
1383 break;
1384
1385 case INTEL_PT_MTC:
1386 intel_pt_calc_mtc_timestamp(decoder);
1387 if (decoder->period_type == INTEL_PT_PERIOD_MTC)
1388 decoder->state.type |= INTEL_PT_INSTRUCTION;
1389 break;
1390
1391 case INTEL_PT_CYC:
1392 case INTEL_PT_VMCS:
1393 case INTEL_PT_MNT:
1394 case INTEL_PT_PAD:
1395 default:
1396 break;
1397 }
1398 }
1399 }
1400
1401 static int intel_pt_walk_fup_tip(struct intel_pt_decoder *decoder)
1402 {
1403 int err;
1404
1405 if (decoder->tx_flags & INTEL_PT_ABORT_TX) {
1406 decoder->tx_flags = 0;
1407 decoder->state.flags &= ~INTEL_PT_IN_TX;
1408 decoder->state.flags |= INTEL_PT_ABORT_TX;
1409 } else {
1410 decoder->state.flags |= INTEL_PT_ASYNC;
1411 }
1412
1413 while (1) {
1414 err = intel_pt_get_next_packet(decoder);
1415 if (err)
1416 return err;
1417
1418 switch (decoder->packet.type) {
1419 case INTEL_PT_TNT:
1420 case INTEL_PT_FUP:
1421 case INTEL_PT_TRACESTOP:
1422 case INTEL_PT_PSB:
1423 case INTEL_PT_TSC:
1424 case INTEL_PT_TMA:
1425 case INTEL_PT_CBR:
1426 case INTEL_PT_MODE_TSX:
1427 case INTEL_PT_BAD:
1428 case INTEL_PT_PSBEND:
1429 intel_pt_log("ERROR: Missing TIP after FUP\n");
1430 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1431 return -ENOENT;
1432
1433 case INTEL_PT_OVF:
1434 return intel_pt_overflow(decoder);
1435
1436 case INTEL_PT_TIP_PGD:
1437 decoder->state.from_ip = decoder->ip;
1438 decoder->state.to_ip = 0;
1439 if (decoder->packet.count != 0) {
1440 intel_pt_set_ip(decoder);
1441 intel_pt_log("Omitting PGD ip " x64_fmt "\n",
1442 decoder->ip);
1443 }
1444 decoder->pge = false;
1445 decoder->continuous_period = false;
1446 return 0;
1447
1448 case INTEL_PT_TIP_PGE:
1449 decoder->pge = true;
1450 intel_pt_log("Omitting PGE ip " x64_fmt "\n",
1451 decoder->ip);
1452 decoder->state.from_ip = 0;
1453 if (decoder->packet.count == 0) {
1454 decoder->state.to_ip = 0;
1455 } else {
1456 intel_pt_set_ip(decoder);
1457 decoder->state.to_ip = decoder->ip;
1458 }
1459 return 0;
1460
1461 case INTEL_PT_TIP:
1462 decoder->state.from_ip = decoder->ip;
1463 if (decoder->packet.count == 0) {
1464 decoder->state.to_ip = 0;
1465 } else {
1466 intel_pt_set_ip(decoder);
1467 decoder->state.to_ip = decoder->ip;
1468 }
1469 return 0;
1470
1471 case INTEL_PT_PIP:
1472 decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1473 break;
1474
1475 case INTEL_PT_MTC:
1476 intel_pt_calc_mtc_timestamp(decoder);
1477 if (decoder->period_type == INTEL_PT_PERIOD_MTC)
1478 decoder->state.type |= INTEL_PT_INSTRUCTION;
1479 break;
1480
1481 case INTEL_PT_CYC:
1482 intel_pt_calc_cyc_timestamp(decoder);
1483 break;
1484
1485 case INTEL_PT_MODE_EXEC:
1486 decoder->exec_mode = decoder->packet.payload;
1487 break;
1488
1489 case INTEL_PT_VMCS:
1490 case INTEL_PT_MNT:
1491 case INTEL_PT_PAD:
1492 break;
1493
1494 default:
1495 return intel_pt_bug(decoder);
1496 }
1497 }
1498 }
1499
1500 static int intel_pt_walk_trace(struct intel_pt_decoder *decoder)
1501 {
1502 bool no_tip = false;
1503 int err;
1504
1505 while (1) {
1506 err = intel_pt_get_next_packet(decoder);
1507 if (err)
1508 return err;
1509 next:
1510 switch (decoder->packet.type) {
1511 case INTEL_PT_TNT:
1512 if (!decoder->packet.count)
1513 break;
1514 decoder->tnt = decoder->packet;
1515 decoder->pkt_state = INTEL_PT_STATE_TNT;
1516 err = intel_pt_walk_tnt(decoder);
1517 if (err == -EAGAIN)
1518 break;
1519 return err;
1520
1521 case INTEL_PT_TIP_PGD:
1522 if (decoder->packet.count != 0)
1523 intel_pt_set_last_ip(decoder);
1524 decoder->pkt_state = INTEL_PT_STATE_TIP_PGD;
1525 return intel_pt_walk_tip(decoder);
1526
1527 case INTEL_PT_TIP_PGE: {
1528 decoder->pge = true;
1529 if (decoder->packet.count == 0) {
1530 intel_pt_log_at("Skipping zero TIP.PGE",
1531 decoder->pos);
1532 break;
1533 }
1534 intel_pt_set_ip(decoder);
1535 decoder->state.from_ip = 0;
1536 decoder->state.to_ip = decoder->ip;
1537 return 0;
1538 }
1539
1540 case INTEL_PT_OVF:
1541 return intel_pt_overflow(decoder);
1542
1543 case INTEL_PT_TIP:
1544 if (decoder->packet.count != 0)
1545 intel_pt_set_last_ip(decoder);
1546 decoder->pkt_state = INTEL_PT_STATE_TIP;
1547 return intel_pt_walk_tip(decoder);
1548
1549 case INTEL_PT_FUP:
1550 if (decoder->packet.count == 0) {
1551 intel_pt_log_at("Skipping zero FUP",
1552 decoder->pos);
1553 no_tip = false;
1554 break;
1555 }
1556 intel_pt_set_last_ip(decoder);
1557 err = intel_pt_walk_fup(decoder);
1558 if (err != -EAGAIN) {
1559 if (err)
1560 return err;
1561 if (no_tip)
1562 decoder->pkt_state =
1563 INTEL_PT_STATE_FUP_NO_TIP;
1564 else
1565 decoder->pkt_state = INTEL_PT_STATE_FUP;
1566 return 0;
1567 }
1568 if (no_tip) {
1569 no_tip = false;
1570 break;
1571 }
1572 return intel_pt_walk_fup_tip(decoder);
1573
1574 case INTEL_PT_TRACESTOP:
1575 decoder->pge = false;
1576 decoder->continuous_period = false;
1577 intel_pt_clear_tx_flags(decoder);
1578 decoder->have_tma = false;
1579 break;
1580
1581 case INTEL_PT_PSB:
1582 intel_pt_clear_stack(&decoder->stack);
1583 err = intel_pt_walk_psbend(decoder);
1584 if (err == -EAGAIN)
1585 goto next;
1586 if (err)
1587 return err;
1588 break;
1589
1590 case INTEL_PT_PIP:
1591 decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1592 break;
1593
1594 case INTEL_PT_MTC:
1595 intel_pt_calc_mtc_timestamp(decoder);
1596 if (decoder->period_type != INTEL_PT_PERIOD_MTC)
1597 break;
1598 /*
1599 * Ensure that there has been an instruction since the
1600 * last MTC.
1601 */
1602 if (!decoder->mtc_insn)
1603 break;
1604 decoder->mtc_insn = false;
1605 /* Ensure that there is a timestamp */
1606 if (!decoder->timestamp)
1607 break;
1608 decoder->state.type = INTEL_PT_INSTRUCTION;
1609 decoder->state.from_ip = decoder->ip;
1610 decoder->state.to_ip = 0;
1611 decoder->mtc_insn = false;
1612 return 0;
1613
1614 case INTEL_PT_TSC:
1615 intel_pt_calc_tsc_timestamp(decoder);
1616 break;
1617
1618 case INTEL_PT_TMA:
1619 intel_pt_calc_tma(decoder);
1620 break;
1621
1622 case INTEL_PT_CYC:
1623 intel_pt_calc_cyc_timestamp(decoder);
1624 break;
1625
1626 case INTEL_PT_CBR:
1627 intel_pt_calc_cbr(decoder);
1628 break;
1629
1630 case INTEL_PT_MODE_EXEC:
1631 decoder->exec_mode = decoder->packet.payload;
1632 break;
1633
1634 case INTEL_PT_MODE_TSX:
1635 /* MODE_TSX need not be followed by FUP */
1636 if (!decoder->pge) {
1637 intel_pt_update_in_tx(decoder);
1638 break;
1639 }
1640 err = intel_pt_mode_tsx(decoder, &no_tip);
1641 if (err)
1642 return err;
1643 goto next;
1644
1645 case INTEL_PT_BAD: /* Does not happen */
1646 return intel_pt_bug(decoder);
1647
1648 case INTEL_PT_PSBEND:
1649 case INTEL_PT_VMCS:
1650 case INTEL_PT_MNT:
1651 case INTEL_PT_PAD:
1652 break;
1653
1654 default:
1655 return intel_pt_bug(decoder);
1656 }
1657 }
1658 }
1659
1660 static inline bool intel_pt_have_ip(struct intel_pt_decoder *decoder)
1661 {
1662 return decoder->last_ip || decoder->packet.count == 0 ||
1663 decoder->packet.count == 3 || decoder->packet.count == 6;
1664 }
1665
1666 /* Walk PSB+ packets to get in sync. */
1667 static int intel_pt_walk_psb(struct intel_pt_decoder *decoder)
1668 {
1669 int err;
1670
1671 while (1) {
1672 err = intel_pt_get_next_packet(decoder);
1673 if (err)
1674 return err;
1675
1676 switch (decoder->packet.type) {
1677 case INTEL_PT_TIP_PGD:
1678 decoder->continuous_period = false;
1679 case INTEL_PT_TIP_PGE:
1680 case INTEL_PT_TIP:
1681 intel_pt_log("ERROR: Unexpected packet\n");
1682 return -ENOENT;
1683
1684 case INTEL_PT_FUP:
1685 decoder->pge = true;
1686 if (intel_pt_have_ip(decoder)) {
1687 uint64_t current_ip = decoder->ip;
1688
1689 intel_pt_set_ip(decoder);
1690 if (current_ip)
1691 intel_pt_log_to("Setting IP",
1692 decoder->ip);
1693 }
1694 break;
1695
1696 case INTEL_PT_MTC:
1697 intel_pt_calc_mtc_timestamp(decoder);
1698 break;
1699
1700 case INTEL_PT_TSC:
1701 intel_pt_calc_tsc_timestamp(decoder);
1702 break;
1703
1704 case INTEL_PT_TMA:
1705 intel_pt_calc_tma(decoder);
1706 break;
1707
1708 case INTEL_PT_CYC:
1709 intel_pt_calc_cyc_timestamp(decoder);
1710 break;
1711
1712 case INTEL_PT_CBR:
1713 intel_pt_calc_cbr(decoder);
1714 break;
1715
1716 case INTEL_PT_PIP:
1717 decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1718 break;
1719
1720 case INTEL_PT_MODE_EXEC:
1721 decoder->exec_mode = decoder->packet.payload;
1722 break;
1723
1724 case INTEL_PT_MODE_TSX:
1725 intel_pt_update_in_tx(decoder);
1726 break;
1727
1728 case INTEL_PT_TRACESTOP:
1729 decoder->pge = false;
1730 decoder->continuous_period = false;
1731 intel_pt_clear_tx_flags(decoder);
1732 case INTEL_PT_TNT:
1733 decoder->have_tma = false;
1734 intel_pt_log("ERROR: Unexpected packet\n");
1735 if (decoder->ip)
1736 decoder->pkt_state = INTEL_PT_STATE_ERR4;
1737 else
1738 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1739 return -ENOENT;
1740
1741 case INTEL_PT_BAD: /* Does not happen */
1742 return intel_pt_bug(decoder);
1743
1744 case INTEL_PT_OVF:
1745 return intel_pt_overflow(decoder);
1746
1747 case INTEL_PT_PSBEND:
1748 return 0;
1749
1750 case INTEL_PT_PSB:
1751 case INTEL_PT_VMCS:
1752 case INTEL_PT_MNT:
1753 case INTEL_PT_PAD:
1754 default:
1755 break;
1756 }
1757 }
1758 }
1759
1760 static int intel_pt_walk_to_ip(struct intel_pt_decoder *decoder)
1761 {
1762 int err;
1763
1764 while (1) {
1765 err = intel_pt_get_next_packet(decoder);
1766 if (err)
1767 return err;
1768
1769 switch (decoder->packet.type) {
1770 case INTEL_PT_TIP_PGD:
1771 decoder->continuous_period = false;
1772 case INTEL_PT_TIP_PGE:
1773 case INTEL_PT_TIP:
1774 decoder->pge = decoder->packet.type != INTEL_PT_TIP_PGD;
1775 if (intel_pt_have_ip(decoder))
1776 intel_pt_set_ip(decoder);
1777 if (decoder->ip)
1778 return 0;
1779 break;
1780
1781 case INTEL_PT_FUP:
1782 if (decoder->overflow) {
1783 if (intel_pt_have_ip(decoder))
1784 intel_pt_set_ip(decoder);
1785 if (decoder->ip)
1786 return 0;
1787 }
1788 if (decoder->packet.count)
1789 intel_pt_set_last_ip(decoder);
1790 break;
1791
1792 case INTEL_PT_MTC:
1793 intel_pt_calc_mtc_timestamp(decoder);
1794 break;
1795
1796 case INTEL_PT_TSC:
1797 intel_pt_calc_tsc_timestamp(decoder);
1798 break;
1799
1800 case INTEL_PT_TMA:
1801 intel_pt_calc_tma(decoder);
1802 break;
1803
1804 case INTEL_PT_CYC:
1805 intel_pt_calc_cyc_timestamp(decoder);
1806 break;
1807
1808 case INTEL_PT_CBR:
1809 intel_pt_calc_cbr(decoder);
1810 break;
1811
1812 case INTEL_PT_PIP:
1813 decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1814 break;
1815
1816 case INTEL_PT_MODE_EXEC:
1817 decoder->exec_mode = decoder->packet.payload;
1818 break;
1819
1820 case INTEL_PT_MODE_TSX:
1821 intel_pt_update_in_tx(decoder);
1822 break;
1823
1824 case INTEL_PT_OVF:
1825 return intel_pt_overflow(decoder);
1826
1827 case INTEL_PT_BAD: /* Does not happen */
1828 return intel_pt_bug(decoder);
1829
1830 case INTEL_PT_TRACESTOP:
1831 decoder->pge = false;
1832 decoder->continuous_period = false;
1833 intel_pt_clear_tx_flags(decoder);
1834 decoder->have_tma = false;
1835 break;
1836
1837 case INTEL_PT_PSB:
1838 err = intel_pt_walk_psb(decoder);
1839 if (err)
1840 return err;
1841 if (decoder->ip) {
1842 /* Do not have a sample */
1843 decoder->state.type = 0;
1844 return 0;
1845 }
1846 break;
1847
1848 case INTEL_PT_TNT:
1849 case INTEL_PT_PSBEND:
1850 case INTEL_PT_VMCS:
1851 case INTEL_PT_MNT:
1852 case INTEL_PT_PAD:
1853 default:
1854 break;
1855 }
1856 }
1857 }
1858
1859 static int intel_pt_sync_ip(struct intel_pt_decoder *decoder)
1860 {
1861 int err;
1862
1863 intel_pt_log("Scanning for full IP\n");
1864 err = intel_pt_walk_to_ip(decoder);
1865 if (err)
1866 return err;
1867
1868 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1869 decoder->overflow = false;
1870
1871 decoder->state.from_ip = 0;
1872 decoder->state.to_ip = decoder->ip;
1873 intel_pt_log_to("Setting IP", decoder->ip);
1874
1875 return 0;
1876 }
1877
1878 static int intel_pt_part_psb(struct intel_pt_decoder *decoder)
1879 {
1880 const unsigned char *end = decoder->buf + decoder->len;
1881 size_t i;
1882
1883 for (i = INTEL_PT_PSB_LEN - 1; i; i--) {
1884 if (i > decoder->len)
1885 continue;
1886 if (!memcmp(end - i, INTEL_PT_PSB_STR, i))
1887 return i;
1888 }
1889 return 0;
1890 }
1891
1892 static int intel_pt_rest_psb(struct intel_pt_decoder *decoder, int part_psb)
1893 {
1894 size_t rest_psb = INTEL_PT_PSB_LEN - part_psb;
1895 const char *psb = INTEL_PT_PSB_STR;
1896
1897 if (rest_psb > decoder->len ||
1898 memcmp(decoder->buf, psb + part_psb, rest_psb))
1899 return 0;
1900
1901 return rest_psb;
1902 }
1903
1904 static int intel_pt_get_split_psb(struct intel_pt_decoder *decoder,
1905 int part_psb)
1906 {
1907 int rest_psb, ret;
1908
1909 decoder->pos += decoder->len;
1910 decoder->len = 0;
1911
1912 ret = intel_pt_get_next_data(decoder);
1913 if (ret)
1914 return ret;
1915
1916 rest_psb = intel_pt_rest_psb(decoder, part_psb);
1917 if (!rest_psb)
1918 return 0;
1919
1920 decoder->pos -= part_psb;
1921 decoder->next_buf = decoder->buf + rest_psb;
1922 decoder->next_len = decoder->len - rest_psb;
1923 memcpy(decoder->temp_buf, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
1924 decoder->buf = decoder->temp_buf;
1925 decoder->len = INTEL_PT_PSB_LEN;
1926
1927 return 0;
1928 }
1929
1930 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder)
1931 {
1932 unsigned char *next;
1933 int ret;
1934
1935 intel_pt_log("Scanning for PSB\n");
1936 while (1) {
1937 if (!decoder->len) {
1938 ret = intel_pt_get_next_data(decoder);
1939 if (ret)
1940 return ret;
1941 }
1942
1943 next = memmem(decoder->buf, decoder->len, INTEL_PT_PSB_STR,
1944 INTEL_PT_PSB_LEN);
1945 if (!next) {
1946 int part_psb;
1947
1948 part_psb = intel_pt_part_psb(decoder);
1949 if (part_psb) {
1950 ret = intel_pt_get_split_psb(decoder, part_psb);
1951 if (ret)
1952 return ret;
1953 } else {
1954 decoder->pos += decoder->len;
1955 decoder->len = 0;
1956 }
1957 continue;
1958 }
1959
1960 decoder->pkt_step = next - decoder->buf;
1961 return intel_pt_get_next_packet(decoder);
1962 }
1963 }
1964
1965 static int intel_pt_sync(struct intel_pt_decoder *decoder)
1966 {
1967 int err;
1968
1969 decoder->pge = false;
1970 decoder->continuous_period = false;
1971 decoder->last_ip = 0;
1972 decoder->ip = 0;
1973 intel_pt_clear_stack(&decoder->stack);
1974
1975 err = intel_pt_scan_for_psb(decoder);
1976 if (err)
1977 return err;
1978
1979 decoder->pkt_state = INTEL_PT_STATE_NO_IP;
1980
1981 err = intel_pt_walk_psb(decoder);
1982 if (err)
1983 return err;
1984
1985 if (decoder->ip) {
1986 decoder->state.type = 0; /* Do not have a sample */
1987 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1988 } else {
1989 return intel_pt_sync_ip(decoder);
1990 }
1991
1992 return 0;
1993 }
1994
1995 static uint64_t intel_pt_est_timestamp(struct intel_pt_decoder *decoder)
1996 {
1997 uint64_t est = decoder->timestamp_insn_cnt << 1;
1998
1999 if (!decoder->cbr || !decoder->max_non_turbo_ratio)
2000 goto out;
2001
2002 est *= decoder->max_non_turbo_ratio;
2003 est /= decoder->cbr;
2004 out:
2005 return decoder->timestamp + est;
2006 }
2007
2008 const struct intel_pt_state *intel_pt_decode(struct intel_pt_decoder *decoder)
2009 {
2010 int err;
2011
2012 do {
2013 decoder->state.type = INTEL_PT_BRANCH;
2014 decoder->state.flags = 0;
2015
2016 switch (decoder->pkt_state) {
2017 case INTEL_PT_STATE_NO_PSB:
2018 err = intel_pt_sync(decoder);
2019 break;
2020 case INTEL_PT_STATE_NO_IP:
2021 decoder->last_ip = 0;
2022 /* Fall through */
2023 case INTEL_PT_STATE_ERR_RESYNC:
2024 err = intel_pt_sync_ip(decoder);
2025 break;
2026 case INTEL_PT_STATE_IN_SYNC:
2027 err = intel_pt_walk_trace(decoder);
2028 break;
2029 case INTEL_PT_STATE_TNT:
2030 err = intel_pt_walk_tnt(decoder);
2031 if (err == -EAGAIN)
2032 err = intel_pt_walk_trace(decoder);
2033 break;
2034 case INTEL_PT_STATE_TIP:
2035 case INTEL_PT_STATE_TIP_PGD:
2036 err = intel_pt_walk_tip(decoder);
2037 break;
2038 case INTEL_PT_STATE_FUP:
2039 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2040 err = intel_pt_walk_fup(decoder);
2041 if (err == -EAGAIN)
2042 err = intel_pt_walk_fup_tip(decoder);
2043 else if (!err)
2044 decoder->pkt_state = INTEL_PT_STATE_FUP;
2045 break;
2046 case INTEL_PT_STATE_FUP_NO_TIP:
2047 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2048 err = intel_pt_walk_fup(decoder);
2049 if (err == -EAGAIN)
2050 err = intel_pt_walk_trace(decoder);
2051 break;
2052 default:
2053 err = intel_pt_bug(decoder);
2054 break;
2055 }
2056 } while (err == -ENOLINK);
2057
2058 decoder->state.err = err ? intel_pt_ext_err(err) : 0;
2059 decoder->state.timestamp = decoder->timestamp;
2060 decoder->state.est_timestamp = intel_pt_est_timestamp(decoder);
2061 decoder->state.cr3 = decoder->cr3;
2062 decoder->state.tot_insn_cnt = decoder->tot_insn_cnt;
2063
2064 if (err)
2065 decoder->state.from_ip = decoder->ip;
2066
2067 return &decoder->state;
2068 }
2069
2070 static bool intel_pt_at_psb(unsigned char *buf, size_t len)
2071 {
2072 if (len < INTEL_PT_PSB_LEN)
2073 return false;
2074 return memmem(buf, INTEL_PT_PSB_LEN, INTEL_PT_PSB_STR,
2075 INTEL_PT_PSB_LEN);
2076 }
2077
2078 /**
2079 * intel_pt_next_psb - move buffer pointer to the start of the next PSB packet.
2080 * @buf: pointer to buffer pointer
2081 * @len: size of buffer
2082 *
2083 * Updates the buffer pointer to point to the start of the next PSB packet if
2084 * there is one, otherwise the buffer pointer is unchanged. If @buf is updated,
2085 * @len is adjusted accordingly.
2086 *
2087 * Return: %true if a PSB packet is found, %false otherwise.
2088 */
2089 static bool intel_pt_next_psb(unsigned char **buf, size_t *len)
2090 {
2091 unsigned char *next;
2092
2093 next = memmem(*buf, *len, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
2094 if (next) {
2095 *len -= next - *buf;
2096 *buf = next;
2097 return true;
2098 }
2099 return false;
2100 }
2101
2102 /**
2103 * intel_pt_step_psb - move buffer pointer to the start of the following PSB
2104 * packet.
2105 * @buf: pointer to buffer pointer
2106 * @len: size of buffer
2107 *
2108 * Updates the buffer pointer to point to the start of the following PSB packet
2109 * (skipping the PSB at @buf itself) if there is one, otherwise the buffer
2110 * pointer is unchanged. If @buf is updated, @len is adjusted accordingly.
2111 *
2112 * Return: %true if a PSB packet is found, %false otherwise.
2113 */
2114 static bool intel_pt_step_psb(unsigned char **buf, size_t *len)
2115 {
2116 unsigned char *next;
2117
2118 if (!*len)
2119 return false;
2120
2121 next = memmem(*buf + 1, *len - 1, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
2122 if (next) {
2123 *len -= next - *buf;
2124 *buf = next;
2125 return true;
2126 }
2127 return false;
2128 }
2129
2130 /**
2131 * intel_pt_last_psb - find the last PSB packet in a buffer.
2132 * @buf: buffer
2133 * @len: size of buffer
2134 *
2135 * This function finds the last PSB in a buffer.
2136 *
2137 * Return: A pointer to the last PSB in @buf if found, %NULL otherwise.
2138 */
2139 static unsigned char *intel_pt_last_psb(unsigned char *buf, size_t len)
2140 {
2141 const char *n = INTEL_PT_PSB_STR;
2142 unsigned char *p;
2143 size_t k;
2144
2145 if (len < INTEL_PT_PSB_LEN)
2146 return NULL;
2147
2148 k = len - INTEL_PT_PSB_LEN + 1;
2149 while (1) {
2150 p = memrchr(buf, n[0], k);
2151 if (!p)
2152 return NULL;
2153 if (!memcmp(p + 1, n + 1, INTEL_PT_PSB_LEN - 1))
2154 return p;
2155 k = p - buf;
2156 if (!k)
2157 return NULL;
2158 }
2159 }
2160
2161 /**
2162 * intel_pt_next_tsc - find and return next TSC.
2163 * @buf: buffer
2164 * @len: size of buffer
2165 * @tsc: TSC value returned
2166 *
2167 * Find a TSC packet in @buf and return the TSC value. This function assumes
2168 * that @buf starts at a PSB and that PSB+ will contain TSC and so stops if a
2169 * PSBEND packet is found.
2170 *
2171 * Return: %true if TSC is found, false otherwise.
2172 */
2173 static bool intel_pt_next_tsc(unsigned char *buf, size_t len, uint64_t *tsc)
2174 {
2175 struct intel_pt_pkt packet;
2176 int ret;
2177
2178 while (len) {
2179 ret = intel_pt_get_packet(buf, len, &packet);
2180 if (ret <= 0)
2181 return false;
2182 if (packet.type == INTEL_PT_TSC) {
2183 *tsc = packet.payload;
2184 return true;
2185 }
2186 if (packet.type == INTEL_PT_PSBEND)
2187 return false;
2188 buf += ret;
2189 len -= ret;
2190 }
2191 return false;
2192 }
2193
2194 /**
2195 * intel_pt_tsc_cmp - compare 7-byte TSCs.
2196 * @tsc1: first TSC to compare
2197 * @tsc2: second TSC to compare
2198 *
2199 * This function compares 7-byte TSC values allowing for the possibility that
2200 * TSC wrapped around. Generally it is not possible to know if TSC has wrapped
2201 * around so for that purpose this function assumes the absolute difference is
2202 * less than half the maximum difference.
2203 *
2204 * Return: %-1 if @tsc1 is before @tsc2, %0 if @tsc1 == @tsc2, %1 if @tsc1 is
2205 * after @tsc2.
2206 */
2207 static int intel_pt_tsc_cmp(uint64_t tsc1, uint64_t tsc2)
2208 {
2209 const uint64_t halfway = (1ULL << 55);
2210
2211 if (tsc1 == tsc2)
2212 return 0;
2213
2214 if (tsc1 < tsc2) {
2215 if (tsc2 - tsc1 < halfway)
2216 return -1;
2217 else
2218 return 1;
2219 } else {
2220 if (tsc1 - tsc2 < halfway)
2221 return 1;
2222 else
2223 return -1;
2224 }
2225 }
2226
2227 /**
2228 * intel_pt_find_overlap_tsc - determine start of non-overlapped trace data
2229 * using TSC.
2230 * @buf_a: first buffer
2231 * @len_a: size of first buffer
2232 * @buf_b: second buffer
2233 * @len_b: size of second buffer
2234 *
2235 * If the trace contains TSC we can look at the last TSC of @buf_a and the
2236 * first TSC of @buf_b in order to determine if the buffers overlap, and then
2237 * walk forward in @buf_b until a later TSC is found. A precondition is that
2238 * @buf_a and @buf_b are positioned at a PSB.
2239 *
2240 * Return: A pointer into @buf_b from where non-overlapped data starts, or
2241 * @buf_b + @len_b if there is no non-overlapped data.
2242 */
2243 static unsigned char *intel_pt_find_overlap_tsc(unsigned char *buf_a,
2244 size_t len_a,
2245 unsigned char *buf_b,
2246 size_t len_b)
2247 {
2248 uint64_t tsc_a, tsc_b;
2249 unsigned char *p;
2250 size_t len;
2251
2252 p = intel_pt_last_psb(buf_a, len_a);
2253 if (!p)
2254 return buf_b; /* No PSB in buf_a => no overlap */
2255
2256 len = len_a - (p - buf_a);
2257 if (!intel_pt_next_tsc(p, len, &tsc_a)) {
2258 /* The last PSB+ in buf_a is incomplete, so go back one more */
2259 len_a -= len;
2260 p = intel_pt_last_psb(buf_a, len_a);
2261 if (!p)
2262 return buf_b; /* No full PSB+ => assume no overlap */
2263 len = len_a - (p - buf_a);
2264 if (!intel_pt_next_tsc(p, len, &tsc_a))
2265 return buf_b; /* No TSC in buf_a => assume no overlap */
2266 }
2267
2268 while (1) {
2269 /* Ignore PSB+ with no TSC */
2270 if (intel_pt_next_tsc(buf_b, len_b, &tsc_b) &&
2271 intel_pt_tsc_cmp(tsc_a, tsc_b) < 0)
2272 return buf_b; /* tsc_a < tsc_b => no overlap */
2273
2274 if (!intel_pt_step_psb(&buf_b, &len_b))
2275 return buf_b + len_b; /* No PSB in buf_b => no data */
2276 }
2277 }
2278
2279 /**
2280 * intel_pt_find_overlap - determine start of non-overlapped trace data.
2281 * @buf_a: first buffer
2282 * @len_a: size of first buffer
2283 * @buf_b: second buffer
2284 * @len_b: size of second buffer
2285 * @have_tsc: can use TSC packets to detect overlap
2286 *
2287 * When trace samples or snapshots are recorded there is the possibility that
2288 * the data overlaps. Note that, for the purposes of decoding, data is only
2289 * useful if it begins with a PSB packet.
2290 *
2291 * Return: A pointer into @buf_b from where non-overlapped data starts, or
2292 * @buf_b + @len_b if there is no non-overlapped data.
2293 */
2294 unsigned char *intel_pt_find_overlap(unsigned char *buf_a, size_t len_a,
2295 unsigned char *buf_b, size_t len_b,
2296 bool have_tsc)
2297 {
2298 unsigned char *found;
2299
2300 /* Buffer 'b' must start at PSB so throw away everything before that */
2301 if (!intel_pt_next_psb(&buf_b, &len_b))
2302 return buf_b + len_b; /* No PSB */
2303
2304 if (!intel_pt_next_psb(&buf_a, &len_a))
2305 return buf_b; /* No overlap */
2306
2307 if (have_tsc) {
2308 found = intel_pt_find_overlap_tsc(buf_a, len_a, buf_b, len_b);
2309 if (found)
2310 return found;
2311 }
2312
2313 /*
2314 * Buffer 'b' cannot end within buffer 'a' so, for comparison purposes,
2315 * we can ignore the first part of buffer 'a'.
2316 */
2317 while (len_b < len_a) {
2318 if (!intel_pt_step_psb(&buf_a, &len_a))
2319 return buf_b; /* No overlap */
2320 }
2321
2322 /* Now len_b >= len_a */
2323 if (len_b > len_a) {
2324 /* The leftover buffer 'b' must start at a PSB */
2325 while (!intel_pt_at_psb(buf_b + len_a, len_b - len_a)) {
2326 if (!intel_pt_step_psb(&buf_a, &len_a))
2327 return buf_b; /* No overlap */
2328 }
2329 }
2330
2331 while (1) {
2332 /* Potential overlap so check the bytes */
2333 found = memmem(buf_a, len_a, buf_b, len_a);
2334 if (found)
2335 return buf_b + len_a;
2336
2337 /* Try again at next PSB in buffer 'a' */
2338 if (!intel_pt_step_psb(&buf_a, &len_a))
2339 return buf_b; /* No overlap */
2340
2341 /* The leftover buffer 'b' must start at a PSB */
2342 while (!intel_pt_at_psb(buf_b + len_a, len_b - len_a)) {
2343 if (!intel_pt_step_psb(&buf_a, &len_a))
2344 return buf_b; /* No overlap */
2345 }
2346 }
2347 }
Linux kernel - Deliverables
This page took 0.086624 seconds and 5 git commands to generate.