linux - tools/perf/util/intel-pt-decoder/intel-pt-decoder.c

4191 lines · 103.2 KiB

1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* intel_pt_decoder.c: Intel Processor Trace support
4	* Copyright (c) 2013-2014, Intel Corporation.
5	*/
6
7	#ifndef _GNU_SOURCE
8	#define _GNU_SOURCE
9	#endif
10	#include <stdlib.h>
11	#include <stdbool.h>
12	#include <string.h>
13	#include <errno.h>
14	#include <stdint.h>
15	#include <inttypes.h>
16	#include <linux/compiler.h>
17	#include <linux/string.h>
18	#include <linux/zalloc.h>
19
20	#include "../auxtrace.h"
21
22	#include "intel-pt-insn-decoder.h"
23	#include "intel-pt-pkt-decoder.h"
24	#include "intel-pt-decoder.h"
25	#include "intel-pt-log.h"
26
27	#define BITULL(x) (1ULL << (x))
28
29	/* IA32_RTIT_CTL MSR bits */
30	#define INTEL_PT_CYC_ENABLE BITULL(1)
31	#define INTEL_PT_CYC_THRESHOLD (BITULL(22) \| BITULL(21) \| BITULL(20) \| BITULL(19))
32	#define INTEL_PT_CYC_THRESHOLD_SHIFT 19
33
34	#define INTEL_PT_BLK_SIZE 1024
35
36	#define BIT63 (((uint64_t)1 << 63))
37
38	#define SEVEN_BYTES 0xffffffffffffffULL
39
40	#define NO_VMCS 0xffffffffffULL
41
42	#define INTEL_PT_RETURN 1
43
44	/*
45	* Default maximum number of loops with no packets consumed i.e. stuck in a
46	* loop.
47	*/
48	#define INTEL_PT_MAX_LOOPS 100000
49
50	struct intel_pt_blk {
51	struct intel_pt_blk *prev;
52	uint64_t ip[INTEL_PT_BLK_SIZE];
53	};
54
55	struct intel_pt_stack {
56	struct intel_pt_blk *blk;
57	struct intel_pt_blk *spare;
58	int pos;
59	};
60
61	enum intel_pt_p_once {
62	INTEL_PT_PRT_ONCE_UNK_VMCS,
63	INTEL_PT_PRT_ONCE_ERANGE,
64	};
65
66	enum intel_pt_pkt_state {
67	INTEL_PT_STATE_NO_PSB,
68	INTEL_PT_STATE_NO_IP,
69	INTEL_PT_STATE_ERR_RESYNC,
70	INTEL_PT_STATE_IN_SYNC,
71	INTEL_PT_STATE_TNT_CONT,
72	INTEL_PT_STATE_TNT,
73	INTEL_PT_STATE_TIP,
74	INTEL_PT_STATE_TIP_PGD,
75	INTEL_PT_STATE_FUP,
76	INTEL_PT_STATE_FUP_NO_TIP,
77	INTEL_PT_STATE_FUP_IN_PSB,
78	INTEL_PT_STATE_RESAMPLE,
79	INTEL_PT_STATE_VM_TIME_CORRELATION,
80	};
81
82	static inline bool intel_pt_sample_time(enum intel_pt_pkt_state pkt_state)
83	{
84	switch (pkt_state) {
85	case INTEL_PT_STATE_NO_PSB:
86	case INTEL_PT_STATE_NO_IP:
87	case INTEL_PT_STATE_ERR_RESYNC:
88	case INTEL_PT_STATE_IN_SYNC:
89	case INTEL_PT_STATE_TNT_CONT:
90	case INTEL_PT_STATE_RESAMPLE:
91	case INTEL_PT_STATE_VM_TIME_CORRELATION:
92	return true;
93	case INTEL_PT_STATE_TNT:
94	case INTEL_PT_STATE_TIP:
95	case INTEL_PT_STATE_TIP_PGD:
96	case INTEL_PT_STATE_FUP:
97	case INTEL_PT_STATE_FUP_NO_TIP:
98	case INTEL_PT_STATE_FUP_IN_PSB:
99	return false;
100	default:
101	return true;
102	};
103	}
104
105	#ifdef INTEL_PT_STRICT
106	#define INTEL_PT_STATE_ERR1 INTEL_PT_STATE_NO_PSB
107	#define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_PSB
108	#define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_NO_PSB
109	#define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_NO_PSB
110	#else
111	#define INTEL_PT_STATE_ERR1 (decoder->pkt_state)
112	#define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_IP
113	#define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_ERR_RESYNC
114	#define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_IN_SYNC
115	#endif
116
117	struct intel_pt_decoder {
118	int (get_trace)(struct intel_pt_buffer buffer, void *data);
119	int (walk_insn)(struct intel_pt_insn intel_pt_insn,
120	uint64_t insn_cnt_ptr, uint64_t ip, uint64_t to_ip,
121	uint64_t max_insn_cnt, void *data);
122	bool (pgd_ip)(uint64_t ip, void data);
123	int (lookahead)(void data, intel_pt_lookahead_cb_t cb, void *cb_data);
124	struct intel_pt_vmcs_info (findnew_vmcs_info)(void *data, uint64_t vmcs);
125	void *data;
126	struct intel_pt_state state;
127	const unsigned char *buf;
128	size_t len;
129	bool return_compression;
130	bool branch_enable;
131	bool mtc_insn;
132	bool pge;
133	bool have_tma;
134	bool have_cyc;
135	bool fixup_last_mtc;
136	bool have_last_ip;
137	bool in_psb;
138	bool hop;
139	bool leap;
140	bool vm_time_correlation;
141	bool vm_tm_corr_dry_run;
142	bool vm_tm_corr_reliable;
143	bool vm_tm_corr_same_buf;
144	bool vm_tm_corr_continuous;
145	bool nr;
146	bool next_nr;
147	enum intel_pt_param_flags flags;
148	uint64_t pos;
149	uint64_t last_ip;
150	uint64_t ip;
151	uint64_t pip_payload;
152	uint64_t timestamp;
153	uint64_t tsc_timestamp;
154	uint64_t ref_timestamp;
155	uint64_t buf_timestamp;
156	uint64_t sample_timestamp;
157	uint64_t ret_addr;
158	uint64_t ctc_timestamp;
159	uint64_t ctc_delta;
160	uint64_t cycle_cnt;
161	uint64_t cyc_ref_timestamp;
162	uint64_t first_timestamp;
163	uint64_t last_reliable_timestamp;
164	uint64_t vmcs;
165	uint64_t print_once;
166	uint64_t last_ctc;
167	uint32_t last_mtc;
168	uint32_t tsc_ctc_ratio_n;
169	uint32_t tsc_ctc_ratio_d;
170	uint32_t tsc_ctc_mult;
171	uint32_t tsc_slip;
172	uint32_t ctc_rem_mask;
173	int mtc_shift;
174	struct intel_pt_stack stack;
175	enum intel_pt_pkt_state pkt_state;
176	enum intel_pt_pkt_ctx pkt_ctx;
177	enum intel_pt_pkt_ctx prev_pkt_ctx;
178	enum intel_pt_blk_type blk_type;
179	int blk_type_pos;
180	struct intel_pt_pkt packet;
181	struct intel_pt_pkt tnt;
182	int pkt_step;
183	int pkt_len;
184	int last_packet_type;
185	unsigned int cbr;
186	unsigned int cbr_seen;
187	unsigned int max_non_turbo_ratio;
188	double max_non_turbo_ratio_fp;
189	double cbr_cyc_to_tsc;
190	double calc_cyc_to_tsc;
191	bool have_calc_cyc_to_tsc;
192	int exec_mode;
193	unsigned int insn_bytes;
194	uint64_t period;
195	enum intel_pt_period_type period_type;
196	uint64_t tot_insn_cnt;
197	uint64_t period_insn_cnt;
198	uint64_t period_mask;
199	uint64_t period_ticks;
200	uint64_t last_masked_timestamp;
201	uint64_t tot_cyc_cnt;
202	uint64_t sample_tot_cyc_cnt;
203	uint64_t base_cyc_cnt;
204	uint64_t cyc_cnt_timestamp;
205	uint64_t ctl;
206	uint64_t cyc_threshold;
207	double tsc_to_cyc;
208	bool continuous_period;
209	bool overflow;
210	bool set_fup_tx_flags;
211	bool set_fup_ptw;
212	bool set_fup_mwait;
213	bool set_fup_pwre;
214	bool set_fup_exstop;
215	bool set_fup_bep;
216	bool sample_cyc;
217	unsigned int fup_tx_flags;
218	unsigned int tx_flags;
219	uint64_t fup_ptw_payload;
220	uint64_t fup_mwait_payload;
221	uint64_t fup_pwre_payload;
222	uint64_t cbr_payload;
223	uint64_t timestamp_insn_cnt;
224	uint64_t sample_insn_cnt;
225	uint64_t stuck_ip;
226	int max_loops;
227	int no_progress;
228	int stuck_ip_prd;
229	int stuck_ip_cnt;
230	uint64_t psb_ip;
231	const unsigned char *next_buf;
232	size_t next_len;
233	unsigned char temp_buf[INTEL_PT_PKT_MAX_SZ];
234	};
235
236	static uint64_t intel_pt_lower_power_of_2(uint64_t x)
237	{
238	int i;
239
240	for (i = 0; x != 1; i++)
241	x >>= 1;
242
243	return x << i;
244	}
245
246	__printf(1, 2)
247	static void p_log(const char *fmt, ...)
248	{
249	char buf[512];
250	va_list args;
251
252	va_start(args, fmt);
253	vsnprintf(buf, sizeof(buf), fmt, args);
254	va_end(args);
255
256	fprintf(stderr, "%s\n", buf);
257	intel_pt_log("%s\n", buf);
258	}
259
260	static bool intel_pt_print_once(struct intel_pt_decoder *decoder,
261	enum intel_pt_p_once id)
262	{
263	uint64_t bit = 1ULL << id;
264
265	if (decoder->print_once & bit)
266	return false;
267	decoder->print_once \|= bit;
268	return true;
269	}
270
271	static uint64_t intel_pt_cyc_threshold(uint64_t ctl)
272	{
273	if (!(ctl & INTEL_PT_CYC_ENABLE))
274	return 0;
275
276	return (ctl & INTEL_PT_CYC_THRESHOLD) >> INTEL_PT_CYC_THRESHOLD_SHIFT;
277	}
278
279	static void intel_pt_setup_period(struct intel_pt_decoder *decoder)
280	{
281	if (decoder->period_type == INTEL_PT_PERIOD_TICKS) {
282	uint64_t period;
283
284	period = intel_pt_lower_power_of_2(decoder->period);
285	decoder->period_mask = ~(period - 1);
286	decoder->period_ticks = period;
287	}
288	}
289
290	static uint64_t multdiv(uint64_t t, uint32_t n, uint32_t d)
291	{
292	if (!d)
293	return 0;
294	return (t / d) * n + ((t % d) * n) / d;
295	}
296
297	struct intel_pt_decoder intel_pt_decoder_new(struct intel_pt_params params)
298	{
299	struct intel_pt_decoder *decoder;
300
301	if (!params->get_trace \|\| !params->walk_insn)
302	return NULL;
303
304	decoder = zalloc(sizeof(struct intel_pt_decoder));
305	if (!decoder)
306	return NULL;
307
308	decoder->get_trace = params->get_trace;
309	decoder->walk_insn = params->walk_insn;
310	decoder->pgd_ip = params->pgd_ip;
311	decoder->lookahead = params->lookahead;
312	decoder->findnew_vmcs_info = params->findnew_vmcs_info;
313	decoder->data = params->data;
314	decoder->return_compression = params->return_compression;
315	decoder->branch_enable = params->branch_enable;
316	decoder->hop = params->quick >= 1;
317	decoder->leap = params->quick >= 2;
318	decoder->vm_time_correlation = params->vm_time_correlation;
319	decoder->vm_tm_corr_dry_run = params->vm_tm_corr_dry_run;
320	decoder->first_timestamp = params->first_timestamp;
321	decoder->last_reliable_timestamp = params->first_timestamp;
322	decoder->max_loops = params->max_loops ? params->max_loops : INTEL_PT_MAX_LOOPS;
323
324	decoder->flags = params->flags;
325
326	decoder->ctl = params->ctl;
327	decoder->period = params->period;
328	decoder->period_type = params->period_type;
329
330	decoder->max_non_turbo_ratio = params->max_non_turbo_ratio;
331	decoder->max_non_turbo_ratio_fp = params->max_non_turbo_ratio;
332
333	decoder->cyc_threshold = intel_pt_cyc_threshold(decoder->ctl);
334
335	intel_pt_setup_period(decoder);
336
337	decoder->mtc_shift = params->mtc_period;
338	decoder->ctc_rem_mask = (1 << decoder->mtc_shift) - 1;
339
340	decoder->tsc_ctc_ratio_n = params->tsc_ctc_ratio_n;
341	decoder->tsc_ctc_ratio_d = params->tsc_ctc_ratio_d;
342
343	if (!decoder->tsc_ctc_ratio_n)
344	decoder->tsc_ctc_ratio_d = 0;
345
346	if (decoder->tsc_ctc_ratio_d) {
347	if (!(decoder->tsc_ctc_ratio_n % decoder->tsc_ctc_ratio_d))
348	decoder->tsc_ctc_mult = decoder->tsc_ctc_ratio_n /
349	decoder->tsc_ctc_ratio_d;
350	}
351
352	/*
353	* A TSC packet can slip past MTC packets so that the timestamp appears
354	* to go backwards. One estimate is that can be up to about 40 CPU
355	* cycles, which is certainly less than 0x1000 TSC ticks, but accept
356	* slippage an order of magnitude more to be on the safe side.
357	*/
358	decoder->tsc_slip = 0x10000;
359
360	intel_pt_log("timestamp: mtc_shift %u\n", decoder->mtc_shift);
361	intel_pt_log("timestamp: tsc_ctc_ratio_n %u\n", decoder->tsc_ctc_ratio_n);
362	intel_pt_log("timestamp: tsc_ctc_ratio_d %u\n", decoder->tsc_ctc_ratio_d);
363	intel_pt_log("timestamp: tsc_ctc_mult %u\n", decoder->tsc_ctc_mult);
364	intel_pt_log("timestamp: tsc_slip %#x\n", decoder->tsc_slip);
365
366	if (decoder->hop)
367	intel_pt_log("Hop mode: decoding FUP and TIPs, but not TNT\n");
368
369	return decoder;
370	}
371
372	void intel_pt_set_first_timestamp(struct intel_pt_decoder *decoder,
373	uint64_t first_timestamp)
374	{
375	decoder->first_timestamp = first_timestamp;
376	}
377
378	static void intel_pt_pop_blk(struct intel_pt_stack *stack)
379	{
380	struct intel_pt_blk *blk = stack->blk;
381
382	stack->blk = blk->prev;
383	if (!stack->spare)
384	stack->spare = blk;
385	else
386	free(blk);
387	}
388
389	static uint64_t intel_pt_pop(struct intel_pt_stack *stack)
390	{
391	if (!stack->pos) {
392	if (!stack->blk)
393	return 0;
394	intel_pt_pop_blk(stack);
395	if (!stack->blk)
396	return 0;
397	stack->pos = INTEL_PT_BLK_SIZE;
398	}
399	return stack->blk->ip[--stack->pos];
400	}
401
402	static int intel_pt_alloc_blk(struct intel_pt_stack *stack)
403	{
404	struct intel_pt_blk *blk;
405
406	if (stack->spare) {
407	blk = stack->spare;
408	stack->spare = NULL;
409	} else {
410	blk = malloc(sizeof(struct intel_pt_blk));
411	if (!blk)
412	return -ENOMEM;
413	}
414
415	blk->prev = stack->blk;
416	stack->blk = blk;
417	stack->pos = 0;
418	return 0;
419	}
420
421	static int intel_pt_push(struct intel_pt_stack *stack, uint64_t ip)
422	{
423	int err;
424
425	if (!stack->blk \|\| stack->pos == INTEL_PT_BLK_SIZE) {
426	err = intel_pt_alloc_blk(stack);
427	if (err)
428	return err;
429	}
430
431	stack->blk->ip[stack->pos++] = ip;
432	return 0;
433	}
434
435	static void intel_pt_clear_stack(struct intel_pt_stack *stack)
436	{
437	while (stack->blk)
438	intel_pt_pop_blk(stack);
439	stack->pos = 0;
440	}
441
442	static void intel_pt_free_stack(struct intel_pt_stack *stack)
443	{
444	intel_pt_clear_stack(stack);
445	zfree(&stack->blk);
446	zfree(&stack->spare);
447	}
448
449	void intel_pt_decoder_free(struct intel_pt_decoder *decoder)
450	{
451	intel_pt_free_stack(&decoder->stack);
452	free(decoder);
453	}
454
455	static int intel_pt_ext_err(int code)
456	{
457	switch (code) {
458	case -ENOMEM:
459	return INTEL_PT_ERR_NOMEM;
460	case -ENOSYS:
461	return INTEL_PT_ERR_INTERN;
462	case -EBADMSG:
463	return INTEL_PT_ERR_BADPKT;
464	case -ENODATA:
465	return INTEL_PT_ERR_NODATA;
466	case -EILSEQ:
467	return INTEL_PT_ERR_NOINSN;
468	case -ENOENT:
469	return INTEL_PT_ERR_MISMAT;
470	case -EOVERFLOW:
471	return INTEL_PT_ERR_OVR;
472	case -ENOSPC:
473	return INTEL_PT_ERR_LOST;
474	case -ELOOP:
475	return INTEL_PT_ERR_NELOOP;
476	default:
477	return INTEL_PT_ERR_UNK;
478	}
479	}
480
481	static const char *intel_pt_err_msgs[] = {
482	[INTEL_PT_ERR_NOMEM] = "Memory allocation failed",
483	[INTEL_PT_ERR_INTERN] = "Internal error",
484	[INTEL_PT_ERR_BADPKT] = "Bad packet",
485	[INTEL_PT_ERR_NODATA] = "No more data",
486	[INTEL_PT_ERR_NOINSN] = "Failed to get instruction",
487	[INTEL_PT_ERR_MISMAT] = "Trace doesn't match instruction",
488	[INTEL_PT_ERR_OVR] = "Overflow packet",
489	[INTEL_PT_ERR_LOST] = "Lost trace data",
490	[INTEL_PT_ERR_UNK] = "Unknown error!",
491	[INTEL_PT_ERR_NELOOP] = "Never-ending loop (refer perf config intel-pt.max-loops)",
492	};
493
494	int intel_pt__strerror(int code, char *buf, size_t buflen)
495	{
496	if (code < 1 \|\| code >= INTEL_PT_ERR_MAX)
497	code = INTEL_PT_ERR_UNK;
498	strlcpy(buf, intel_pt_err_msgs[code], buflen);
499	return 0;
500	}
501
502	static uint64_t intel_pt_calc_ip(const struct intel_pt_pkt *packet,
503	uint64_t last_ip)
504	{
505	uint64_t ip;
506
507	switch (packet->count) {
508	case 1:
509	ip = (last_ip & (uint64_t)0xffffffffffff0000ULL) \|
510	packet->payload;
511	break;
512	case 2:
513	ip = (last_ip & (uint64_t)0xffffffff00000000ULL) \|
514	packet->payload;
515	break;
516	case 3:
517	ip = packet->payload;
518	/* Sign-extend 6-byte ip */
519	if (ip & (uint64_t)0x800000000000ULL)
520	ip \|= (uint64_t)0xffff000000000000ULL;
521	break;
522	case 4:
523	ip = (last_ip & (uint64_t)0xffff000000000000ULL) \|
524	packet->payload;
525	break;
526	case 6:
527	ip = packet->payload;
528	break;
529	default:
530	return 0;
531	}
532
533	return ip;
534	}
535
536	static inline void intel_pt_set_last_ip(struct intel_pt_decoder *decoder)
537	{
538	decoder->last_ip = intel_pt_calc_ip(&decoder->packet, decoder->last_ip);
539	decoder->have_last_ip = true;
540	}
541
542	static inline void intel_pt_set_ip(struct intel_pt_decoder *decoder)
543	{
544	intel_pt_set_last_ip(decoder);
545	decoder->ip = decoder->last_ip;
546	}
547
548	static void intel_pt_decoder_log_packet(struct intel_pt_decoder *decoder)
549	{
550	intel_pt_log_packet(&decoder->packet, decoder->pkt_len, decoder->pos,
551	decoder->buf);
552	}
553
554	static int intel_pt_bug(struct intel_pt_decoder *decoder)
555	{
556	intel_pt_log("ERROR: Internal error\n");
557	decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
558	return -ENOSYS;
559	}
560
561	static inline void intel_pt_clear_tx_flags(struct intel_pt_decoder *decoder)
562	{
563	decoder->tx_flags = 0;
564	}
565
566	static inline void intel_pt_update_in_tx(struct intel_pt_decoder *decoder)
567	{
568	decoder->tx_flags = decoder->packet.payload & INTEL_PT_IN_TX;
569	}
570
571	static inline void intel_pt_update_pip(struct intel_pt_decoder *decoder)
572	{
573	decoder->pip_payload = decoder->packet.payload;
574	}
575
576	static inline void intel_pt_update_nr(struct intel_pt_decoder *decoder)
577	{
578	decoder->next_nr = decoder->pip_payload & 1;
579	}
580
581	static inline void intel_pt_set_nr(struct intel_pt_decoder *decoder)
582	{
583	decoder->nr = decoder->pip_payload & 1;
584	decoder->next_nr = decoder->nr;
585	}
586
587	static inline void intel_pt_set_pip(struct intel_pt_decoder *decoder)
588	{
589	intel_pt_update_pip(decoder);
590	intel_pt_set_nr(decoder);
591	}
592
593	static int intel_pt_bad_packet(struct intel_pt_decoder *decoder)
594	{
595	intel_pt_clear_tx_flags(decoder);
596	decoder->have_tma = false;
597	decoder->pkt_len = 1;
598	decoder->pkt_step = 1;
599	intel_pt_decoder_log_packet(decoder);
600	if (decoder->pkt_state != INTEL_PT_STATE_NO_PSB) {
601	intel_pt_log("ERROR: Bad packet\n");
602	decoder->pkt_state = INTEL_PT_STATE_ERR1;
603	}
604	return -EBADMSG;
605	}
606
607	static inline void intel_pt_update_sample_time(struct intel_pt_decoder *decoder)
608	{
609	decoder->sample_timestamp = decoder->timestamp;
610	decoder->sample_insn_cnt = decoder->timestamp_insn_cnt;
611	}
612
613	static void intel_pt_reposition(struct intel_pt_decoder *decoder)
614	{
615	decoder->ip = 0;
616	decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
617	decoder->timestamp = 0;
618	decoder->have_tma = false;
619	}
620
621	static int intel_pt_get_data(struct intel_pt_decoder *decoder, bool reposition)
622	{
623	struct intel_pt_buffer buffer = { .buf = 0, };
624	int ret;
625
626	decoder->pkt_step = 0;
627
628	intel_pt_log("Getting more data\n");
629	ret = decoder->get_trace(&buffer, decoder->data);
630	if (ret)
631	return ret;
632	decoder->buf = buffer.buf;
633	decoder->len = buffer.len;
634	if (!decoder->len) {
635	intel_pt_log("No more data\n");
636	return -ENODATA;
637	}
638	decoder->buf_timestamp = buffer.ref_timestamp;
639	if (!buffer.consecutive \|\| reposition) {
640	intel_pt_reposition(decoder);
641	decoder->ref_timestamp = buffer.ref_timestamp;
642	decoder->state.trace_nr = buffer.trace_nr;
643	decoder->vm_tm_corr_same_buf = false;
644	intel_pt_log("Reference timestamp 0x%" PRIx64 "\n",
645	decoder->ref_timestamp);
646	return -ENOLINK;
647	}
648
649	return 0;
650	}
651
652	static int intel_pt_get_next_data(struct intel_pt_decoder *decoder,
653	bool reposition)
654	{
655	if (!decoder->next_buf)
656	return intel_pt_get_data(decoder, reposition);
657
658	decoder->buf = decoder->next_buf;
659	decoder->len = decoder->next_len;
660	decoder->next_buf = 0;
661	decoder->next_len = 0;
662	return 0;
663	}
664
665	static int intel_pt_get_split_packet(struct intel_pt_decoder *decoder)
666	{
667	unsigned char *buf = decoder->temp_buf;
668	size_t old_len, len, n;
669	int ret;
670
671	old_len = decoder->len;
672	len = decoder->len;
673	memcpy(buf, decoder->buf, len);
674
675	ret = intel_pt_get_data(decoder, false);
676	if (ret) {
677	decoder->pos += old_len;
678	return ret < 0 ? ret : -EINVAL;
679	}
680
681	n = INTEL_PT_PKT_MAX_SZ - len;
682	if (n > decoder->len)
683	n = decoder->len;
684	memcpy(buf + len, decoder->buf, n);
685	len += n;
686
687	decoder->prev_pkt_ctx = decoder->pkt_ctx;
688	ret = intel_pt_get_packet(buf, len, &decoder->packet, &decoder->pkt_ctx);
689	if (ret < (int)old_len) {
690	decoder->next_buf = decoder->buf;
691	decoder->next_len = decoder->len;
692	decoder->buf = buf;
693	decoder->len = old_len;
694	return intel_pt_bad_packet(decoder);
695	}
696
697	decoder->next_buf = decoder->buf + (ret - old_len);
698	decoder->next_len = decoder->len - (ret - old_len);
699
700	decoder->buf = buf;
701	decoder->len = ret;
702
703	return ret;
704	}
705
706	struct intel_pt_pkt_info {
707	struct intel_pt_decoder *decoder;
708	struct intel_pt_pkt packet;
709	uint64_t pos;
710	int pkt_len;
711	int last_packet_type;
712	void *data;
713	};
714
715	typedef int (intel_pt_pkt_cb_t)(struct intel_pt_pkt_info pkt_info);
716
717	/* Lookahead packets in current buffer */
718	static int intel_pt_pkt_lookahead(struct intel_pt_decoder *decoder,
719	intel_pt_pkt_cb_t cb, void *data)
720	{
721	struct intel_pt_pkt_info pkt_info;
722	const unsigned char *buf = decoder->buf;
723	enum intel_pt_pkt_ctx pkt_ctx = decoder->pkt_ctx;
724	size_t len = decoder->len;
725	int ret;
726
727	pkt_info.decoder = decoder;
728	pkt_info.pos = decoder->pos;
729	pkt_info.pkt_len = decoder->pkt_step;
730	pkt_info.last_packet_type = decoder->last_packet_type;
731	pkt_info.data = data;
732
733	while (1) {
734	do {
735	pkt_info.pos += pkt_info.pkt_len;
736	buf += pkt_info.pkt_len;
737	len -= pkt_info.pkt_len;
738
739	if (!len)
740	return INTEL_PT_NEED_MORE_BYTES;
741
742	ret = intel_pt_get_packet(buf, len, &pkt_info.packet,
743	&pkt_ctx);
744	if (!ret)
745	return INTEL_PT_NEED_MORE_BYTES;
746	if (ret < 0)
747	return ret;
748
749	pkt_info.pkt_len = ret;
750	} while (pkt_info.packet.type == INTEL_PT_PAD);
751
752	ret = cb(&pkt_info);
753	if (ret)
754	return 0;
755
756	pkt_info.last_packet_type = pkt_info.packet.type;
757	}
758	}
759
760	struct intel_pt_calc_cyc_to_tsc_info {
761	uint64_t cycle_cnt;
762	unsigned int cbr;
763	uint32_t last_mtc;
764	uint64_t ctc_timestamp;
765	uint64_t ctc_delta;
766	uint64_t tsc_timestamp;
767	uint64_t timestamp;
768	bool have_tma;
769	bool fixup_last_mtc;
770	bool from_mtc;
771	double cbr_cyc_to_tsc;
772	};
773
774	/*
775	* MTC provides a 8-bit slice of CTC but the TMA packet only provides the lower
776	* 16 bits of CTC. If mtc_shift > 8 then some of the MTC bits are not in the CTC
777	* provided by the TMA packet. Fix-up the last_mtc calculated from the TMA
778	* packet by copying the missing bits from the current MTC assuming the least
779	* difference between the two, and that the current MTC comes after last_mtc.
780	*/
781	static void intel_pt_fixup_last_mtc(uint32_t mtc, int mtc_shift,
782	uint32_t *last_mtc)
783	{
784	uint32_t first_missing_bit = 1U << (16 - mtc_shift);
785	uint32_t mask = ~(first_missing_bit - 1);
786
787	*last_mtc \|= mtc & mask;
788	if (*last_mtc >= mtc) {
789	*last_mtc -= first_missing_bit;
790	*last_mtc &= 0xff;
791	}
792	}
793
794	static int intel_pt_calc_cyc_cb(struct intel_pt_pkt_info *pkt_info)
795	{
796	struct intel_pt_decoder *decoder = pkt_info->decoder;
797	struct intel_pt_calc_cyc_to_tsc_info *data = pkt_info->data;
798	uint64_t timestamp;
799	double cyc_to_tsc;
800	unsigned int cbr;
801	uint32_t mtc, mtc_delta, ctc, fc, ctc_rem;
802
803	switch (pkt_info->packet.type) {
804	case INTEL_PT_TNT:
805	case INTEL_PT_TIP_PGE:
806	case INTEL_PT_TIP:
807	case INTEL_PT_FUP:
808	case INTEL_PT_PSB:
809	case INTEL_PT_PIP:
810	case INTEL_PT_MODE_EXEC:
811	case INTEL_PT_MODE_TSX:
812	case INTEL_PT_PSBEND:
813	case INTEL_PT_PAD:
814	case INTEL_PT_VMCS:
815	case INTEL_PT_MNT:
816	case INTEL_PT_PTWRITE:
817	case INTEL_PT_PTWRITE_IP:
818	case INTEL_PT_BBP:
819	case INTEL_PT_BIP:
820	case INTEL_PT_BEP:
821	case INTEL_PT_BEP_IP:
822	return 0;
823
824	case INTEL_PT_MTC:
825	if (!data->have_tma)
826	return 0;
827
828	mtc = pkt_info->packet.payload;
829	if (decoder->mtc_shift > 8 && data->fixup_last_mtc) {
830	data->fixup_last_mtc = false;
831	intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,
832	&data->last_mtc);
833	}
834	if (mtc > data->last_mtc)
835	mtc_delta = mtc - data->last_mtc;
836	else
837	mtc_delta = mtc + 256 - data->last_mtc;
838	data->ctc_delta += mtc_delta << decoder->mtc_shift;
839	data->last_mtc = mtc;
840
841	if (decoder->tsc_ctc_mult) {
842	timestamp = data->ctc_timestamp +
843	data->ctc_delta * decoder->tsc_ctc_mult;
844	} else {
845	timestamp = data->ctc_timestamp +
846	multdiv(data->ctc_delta,
847	decoder->tsc_ctc_ratio_n,
848	decoder->tsc_ctc_ratio_d);
849	}
850
851	if (timestamp < data->timestamp)
852	return 1;
853
854	if (pkt_info->last_packet_type != INTEL_PT_CYC) {
855	data->timestamp = timestamp;
856	return 0;
857	}
858
859	break;
860
861	case INTEL_PT_TSC:
862	/*
863	* For now, do not support using TSC packets - refer
864	* intel_pt_calc_cyc_to_tsc().
865	*/
866	if (data->from_mtc)
867	return 1;
868	timestamp = pkt_info->packet.payload \|
869	(data->timestamp & (0xffULL << 56));
870	if (data->from_mtc && timestamp < data->timestamp &&
871	data->timestamp - timestamp < decoder->tsc_slip)
872	return 1;
873	if (timestamp < data->timestamp)
874	timestamp += (1ULL << 56);
875	if (pkt_info->last_packet_type != INTEL_PT_CYC) {
876	if (data->from_mtc)
877	return 1;
878	data->tsc_timestamp = timestamp;
879	data->timestamp = timestamp;
880	return 0;
881	}
882	break;
883
884	case INTEL_PT_TMA:
885	if (data->from_mtc)
886	return 1;
887
888	if (!decoder->tsc_ctc_ratio_d)
889	return 0;
890
891	ctc = pkt_info->packet.payload;
892	fc = pkt_info->packet.count;
893	ctc_rem = ctc & decoder->ctc_rem_mask;
894
895	data->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
896
897	data->ctc_timestamp = data->tsc_timestamp - fc;
898	if (decoder->tsc_ctc_mult) {
899	data->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;
900	} else {
901	data->ctc_timestamp -=
902	multdiv(ctc_rem, decoder->tsc_ctc_ratio_n,
903	decoder->tsc_ctc_ratio_d);
904	}
905
906	data->ctc_delta = 0;
907	data->have_tma = true;
908	data->fixup_last_mtc = true;
909
910	return 0;
911
912	case INTEL_PT_CYC:
913	data->cycle_cnt += pkt_info->packet.payload;
914	return 0;
915
916	case INTEL_PT_CBR:
917	cbr = pkt_info->packet.payload;
918	if (data->cbr && data->cbr != cbr)
919	return 1;
920	data->cbr = cbr;
921	data->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;
922	return 0;
923
924	case INTEL_PT_TIP_PGD:
925	case INTEL_PT_TRACESTOP:
926	case INTEL_PT_EXSTOP:
927	case INTEL_PT_EXSTOP_IP:
928	case INTEL_PT_MWAIT:
929	case INTEL_PT_PWRE:
930	case INTEL_PT_PWRX:
931	case INTEL_PT_OVF:
932	case INTEL_PT_BAD: /* Does not happen */
933	default:
934	return 1;
935	}
936
937	if (!data->cbr && decoder->cbr) {
938	data->cbr = decoder->cbr;
939	data->cbr_cyc_to_tsc = decoder->cbr_cyc_to_tsc;
940	}
941
942	if (!data->cycle_cnt)
943	return 1;
944
945	cyc_to_tsc = (double)(timestamp - decoder->timestamp) / data->cycle_cnt;
946
947	if (data->cbr && cyc_to_tsc > data->cbr_cyc_to_tsc &&
948	cyc_to_tsc / data->cbr_cyc_to_tsc > 1.25) {
949	intel_pt_log("Timestamp: calculated %g TSC ticks per cycle too big (c.f. CBR-based value %g), pos " x64_fmt "\n",
950	cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos);
951	return 1;
952	}
953
954	decoder->calc_cyc_to_tsc = cyc_to_tsc;
955	decoder->have_calc_cyc_to_tsc = true;
956
957	if (data->cbr) {
958	intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. CBR-based value %g, pos " x64_fmt "\n",
959	cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos);
960	} else {
961	intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. unknown CBR-based value, pos " x64_fmt "\n",
962	cyc_to_tsc, pkt_info->pos);
963	}
964
965	return 1;
966	}
967
968	static void intel_pt_calc_cyc_to_tsc(struct intel_pt_decoder *decoder,
969	bool from_mtc)
970	{
971	struct intel_pt_calc_cyc_to_tsc_info data = {
972	.cycle_cnt = 0,
973	.cbr = 0,
974	.last_mtc = decoder->last_mtc,
975	.ctc_timestamp = decoder->ctc_timestamp,
976	.ctc_delta = decoder->ctc_delta,
977	.tsc_timestamp = decoder->tsc_timestamp,
978	.timestamp = decoder->timestamp,
979	.have_tma = decoder->have_tma,
980	.fixup_last_mtc = decoder->fixup_last_mtc,
981	.from_mtc = from_mtc,
982	.cbr_cyc_to_tsc = 0,
983	};
984
985	/*
986	* For now, do not support using TSC packets for at least the reasons:
987	* 1) timing might have stopped
988	* 2) TSC packets within PSB+ can slip against CYC packets
989	*/
990	if (!from_mtc)
991	return;
992
993	intel_pt_pkt_lookahead(decoder, intel_pt_calc_cyc_cb, &data);
994	}
995
996	static int intel_pt_get_next_packet(struct intel_pt_decoder *decoder)
997	{
998	int ret;
999
1000	decoder->last_packet_type = decoder->packet.type;
1001
1002	do {
1003	decoder->pos += decoder->pkt_step;
1004	decoder->buf += decoder->pkt_step;
1005	decoder->len -= decoder->pkt_step;
1006
1007	if (!decoder->len) {
1008	ret = intel_pt_get_next_data(decoder, false);
1009	if (ret)
1010	return ret;
1011	}
1012
1013	decoder->prev_pkt_ctx = decoder->pkt_ctx;
1014	ret = intel_pt_get_packet(decoder->buf, decoder->len,
1015	&decoder->packet, &decoder->pkt_ctx);
1016	if (ret == INTEL_PT_NEED_MORE_BYTES && BITS_PER_LONG == 32 &&
1017	decoder->len < INTEL_PT_PKT_MAX_SZ && !decoder->next_buf) {
1018	ret = intel_pt_get_split_packet(decoder);
1019	if (ret < 0)
1020	return ret;
1021	}
1022	if (ret <= 0)
1023	return intel_pt_bad_packet(decoder);
1024
1025	decoder->pkt_len = ret;
1026	decoder->pkt_step = ret;
1027	intel_pt_decoder_log_packet(decoder);
1028	} while (decoder->packet.type == INTEL_PT_PAD);
1029
1030	return 0;
1031	}
1032
1033	static uint64_t intel_pt_next_period(struct intel_pt_decoder *decoder)
1034	{
1035	uint64_t timestamp, masked_timestamp;
1036
1037	timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;
1038	masked_timestamp = timestamp & decoder->period_mask;
1039	if (decoder->continuous_period) {
1040	if (masked_timestamp > decoder->last_masked_timestamp)
1041	return 1;
1042	} else {
1043	timestamp += 1;
1044	masked_timestamp = timestamp & decoder->period_mask;
1045	if (masked_timestamp > decoder->last_masked_timestamp) {
1046	decoder->last_masked_timestamp = masked_timestamp;
1047	decoder->continuous_period = true;
1048	}
1049	}
1050
1051	if (masked_timestamp < decoder->last_masked_timestamp)
1052	return decoder->period_ticks;
1053
1054	return decoder->period_ticks - (timestamp - masked_timestamp);
1055	}
1056
1057	static uint64_t intel_pt_next_sample(struct intel_pt_decoder *decoder)
1058	{
1059	switch (decoder->period_type) {
1060	case INTEL_PT_PERIOD_INSTRUCTIONS:
1061	return decoder->period - decoder->period_insn_cnt;
1062	case INTEL_PT_PERIOD_TICKS:
1063	return intel_pt_next_period(decoder);
1064	case INTEL_PT_PERIOD_NONE:
1065	case INTEL_PT_PERIOD_MTC:
1066	default:
1067	return 0;
1068	}
1069	}
1070
1071	static void intel_pt_sample_insn(struct intel_pt_decoder *decoder)
1072	{
1073	uint64_t timestamp, masked_timestamp;
1074
1075	switch (decoder->period_type) {
1076	case INTEL_PT_PERIOD_INSTRUCTIONS:
1077	decoder->period_insn_cnt = 0;
1078	break;
1079	case INTEL_PT_PERIOD_TICKS:
1080	timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;
1081	masked_timestamp = timestamp & decoder->period_mask;
1082	if (masked_timestamp > decoder->last_masked_timestamp)
1083	decoder->last_masked_timestamp = masked_timestamp;
1084	else
1085	decoder->last_masked_timestamp += decoder->period_ticks;
1086	break;
1087	case INTEL_PT_PERIOD_NONE:
1088	case INTEL_PT_PERIOD_MTC:
1089	default:
1090	break;
1091	}
1092
1093	decoder->state.type \|= INTEL_PT_INSTRUCTION;
1094	}
1095
1096	static int intel_pt_walk_insn(struct intel_pt_decoder *decoder,
1097	struct intel_pt_insn *intel_pt_insn, uint64_t ip)
1098	{
1099	uint64_t max_insn_cnt, insn_cnt = 0;
1100	int err;
1101
1102	if (!decoder->mtc_insn)
1103	decoder->mtc_insn = true;
1104
1105	max_insn_cnt = intel_pt_next_sample(decoder);
1106
1107	err = decoder->walk_insn(intel_pt_insn, &insn_cnt, &decoder->ip, ip,
1108	max_insn_cnt, decoder->data);
1109
1110	decoder->tot_insn_cnt += insn_cnt;
1111	decoder->timestamp_insn_cnt += insn_cnt;
1112	decoder->sample_insn_cnt += insn_cnt;
1113	decoder->period_insn_cnt += insn_cnt;
1114
1115	if (err) {
1116	decoder->no_progress = 0;
1117	decoder->pkt_state = INTEL_PT_STATE_ERR2;
1118	intel_pt_log_at("ERROR: Failed to get instruction",
1119	decoder->ip);
1120	if (err == -ENOENT)
1121	return -ENOLINK;
1122	return -EILSEQ;
1123	}
1124
1125	if (ip && decoder->ip == ip) {
1126	err = -EAGAIN;
1127	goto out;
1128	}
1129
1130	if (max_insn_cnt && insn_cnt >= max_insn_cnt)
1131	intel_pt_sample_insn(decoder);
1132
1133	if (intel_pt_insn->branch == INTEL_PT_BR_NO_BRANCH) {
1134	decoder->state.type = INTEL_PT_INSTRUCTION;
1135	decoder->state.from_ip = decoder->ip;
1136	decoder->state.to_ip = 0;
1137	decoder->ip += intel_pt_insn->length;
1138	err = INTEL_PT_RETURN;
1139	goto out;
1140	}
1141
1142	if (intel_pt_insn->op == INTEL_PT_OP_CALL) {
1143	/* Zero-length calls are excluded */
1144	if (intel_pt_insn->branch != INTEL_PT_BR_UNCONDITIONAL \|\|
1145	intel_pt_insn->rel) {
1146	err = intel_pt_push(&decoder->stack, decoder->ip +
1147	intel_pt_insn->length);
1148	if (err)
1149	goto out;
1150	}
1151	} else if (intel_pt_insn->op == INTEL_PT_OP_RET) {
1152	decoder->ret_addr = intel_pt_pop(&decoder->stack);
1153	}
1154
1155	if (intel_pt_insn->branch == INTEL_PT_BR_UNCONDITIONAL) {
1156	int cnt = decoder->no_progress++;
1157
1158	decoder->state.from_ip = decoder->ip;
1159	decoder->ip += intel_pt_insn->length +
1160	intel_pt_insn->rel;
1161	decoder->state.to_ip = decoder->ip;
1162	err = INTEL_PT_RETURN;
1163
1164	/*
1165	* Check for being stuck in a loop. This can happen if a
1166	* decoder error results in the decoder erroneously setting the
1167	* ip to an address that is itself in an infinite loop that
1168	* consumes no packets. When that happens, there must be an
1169	* unconditional branch.
1170	*/
1171	if (cnt) {
1172	if (cnt == 1) {
1173	decoder->stuck_ip = decoder->state.to_ip;
1174	decoder->stuck_ip_prd = 1;
1175	decoder->stuck_ip_cnt = 1;
1176	} else if (cnt > decoder->max_loops \|\|
1177	decoder->state.to_ip == decoder->stuck_ip) {
1178	intel_pt_log_at("ERROR: Never-ending loop",
1179	decoder->state.to_ip);
1180	decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1181	err = -ELOOP;
1182	goto out;
1183	} else if (!--decoder->stuck_ip_cnt) {
1184	decoder->stuck_ip_prd += 1;
1185	decoder->stuck_ip_cnt = decoder->stuck_ip_prd;
1186	decoder->stuck_ip = decoder->state.to_ip;
1187	}
1188	}
1189	goto out_no_progress;
1190	}
1191	out:
1192	decoder->no_progress = 0;
1193	out_no_progress:
1194	decoder->state.insn_op = intel_pt_insn->op;
1195	decoder->state.insn_len = intel_pt_insn->length;
1196	memcpy(decoder->state.insn, intel_pt_insn->buf,
1197	INTEL_PT_INSN_BUF_SZ);
1198
1199	if (decoder->tx_flags & INTEL_PT_IN_TX)
1200	decoder->state.flags \|= INTEL_PT_IN_TX;
1201
1202	return err;
1203	}
1204
1205	static bool intel_pt_fup_event(struct intel_pt_decoder *decoder)
1206	{
1207	bool ret = false;
1208
1209	if (decoder->set_fup_tx_flags) {
1210	decoder->set_fup_tx_flags = false;
1211	decoder->tx_flags = decoder->fup_tx_flags;
1212	decoder->state.type = INTEL_PT_TRANSACTION;
1213	if (decoder->fup_tx_flags & INTEL_PT_ABORT_TX)
1214	decoder->state.type \|= INTEL_PT_BRANCH;
1215	decoder->state.from_ip = decoder->ip;
1216	decoder->state.to_ip = 0;
1217	decoder->state.flags = decoder->fup_tx_flags;
1218	return true;
1219	}
1220	if (decoder->set_fup_ptw) {
1221	decoder->set_fup_ptw = false;
1222	decoder->state.type = INTEL_PT_PTW;
1223	decoder->state.flags \|= INTEL_PT_FUP_IP;
1224	decoder->state.from_ip = decoder->ip;
1225	decoder->state.to_ip = 0;
1226	decoder->state.ptw_payload = decoder->fup_ptw_payload;
1227	return true;
1228	}
1229	if (decoder->set_fup_mwait) {
1230	decoder->set_fup_mwait = false;
1231	decoder->state.type = INTEL_PT_MWAIT_OP;
1232	decoder->state.from_ip = decoder->ip;
1233	decoder->state.to_ip = 0;
1234	decoder->state.mwait_payload = decoder->fup_mwait_payload;
1235	ret = true;
1236	}
1237	if (decoder->set_fup_pwre) {
1238	decoder->set_fup_pwre = false;
1239	decoder->state.type \|= INTEL_PT_PWR_ENTRY;
1240	decoder->state.type &= ~INTEL_PT_BRANCH;
1241	decoder->state.from_ip = decoder->ip;
1242	decoder->state.to_ip = 0;
1243	decoder->state.pwre_payload = decoder->fup_pwre_payload;
1244	ret = true;
1245	}
1246	if (decoder->set_fup_exstop) {
1247	decoder->set_fup_exstop = false;
1248	decoder->state.type \|= INTEL_PT_EX_STOP;
1249	decoder->state.type &= ~INTEL_PT_BRANCH;
1250	decoder->state.flags \|= INTEL_PT_FUP_IP;
1251	decoder->state.from_ip = decoder->ip;
1252	decoder->state.to_ip = 0;
1253	ret = true;
1254	}
1255	if (decoder->set_fup_bep) {
1256	decoder->set_fup_bep = false;
1257	decoder->state.type \|= INTEL_PT_BLK_ITEMS;
1258	decoder->state.type &= ~INTEL_PT_BRANCH;
1259	decoder->state.from_ip = decoder->ip;
1260	decoder->state.to_ip = 0;
1261	ret = true;
1262	}
1263	return ret;
1264	}
1265
1266	static inline bool intel_pt_fup_with_nlip(struct intel_pt_decoder *decoder,
1267	struct intel_pt_insn *intel_pt_insn,
1268	uint64_t ip, int err)
1269	{
1270	return decoder->flags & INTEL_PT_FUP_WITH_NLIP && !err &&
1271	intel_pt_insn->branch == INTEL_PT_BR_INDIRECT &&
1272	ip == decoder->ip + intel_pt_insn->length;
1273	}
1274
1275	static int intel_pt_walk_fup(struct intel_pt_decoder *decoder)
1276	{
1277	struct intel_pt_insn intel_pt_insn;
1278	uint64_t ip;
1279	int err;
1280
1281	ip = decoder->last_ip;
1282
1283	while (1) {
1284	err = intel_pt_walk_insn(decoder, &intel_pt_insn, ip);
1285	if (err == INTEL_PT_RETURN)
1286	return 0;
1287	if (err == -EAGAIN \|\|
1288	intel_pt_fup_with_nlip(decoder, &intel_pt_insn, ip, err)) {
1289	bool no_tip = decoder->pkt_state != INTEL_PT_STATE_FUP;
1290
1291	decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1292	if (intel_pt_fup_event(decoder) && no_tip)
1293	return 0;
1294	return -EAGAIN;
1295	}
1296	decoder->set_fup_tx_flags = false;
1297	if (err)
1298	return err;
1299
1300	if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1301	intel_pt_log_at("ERROR: Unexpected indirect branch",
1302	decoder->ip);
1303	decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1304	return -ENOENT;
1305	}
1306
1307	if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1308	intel_pt_log_at("ERROR: Unexpected conditional branch",
1309	decoder->ip);
1310	decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1311	return -ENOENT;
1312	}
1313
1314	intel_pt_bug(decoder);
1315	}
1316	}
1317
1318	static int intel_pt_walk_tip(struct intel_pt_decoder *decoder)
1319	{
1320	struct intel_pt_insn intel_pt_insn;
1321	int err;
1322
1323	err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);
1324	if (err == INTEL_PT_RETURN &&
1325	decoder->pgd_ip &&
1326	decoder->pkt_state == INTEL_PT_STATE_TIP_PGD &&
1327	(decoder->state.type & INTEL_PT_BRANCH) &&
1328	decoder->pgd_ip(decoder->state.to_ip, decoder->data)) {
1329	/* Unconditional branch leaving filter region */
1330	decoder->no_progress = 0;
1331	decoder->pge = false;
1332	decoder->continuous_period = false;
1333	decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1334	decoder->state.type \|= INTEL_PT_TRACE_END;
1335	intel_pt_update_nr(decoder);
1336	return 0;
1337	}
1338	if (err == INTEL_PT_RETURN)
1339	return 0;
1340	if (err)
1341	return err;
1342
1343	intel_pt_update_nr(decoder);
1344
1345	if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1346	if (decoder->pkt_state == INTEL_PT_STATE_TIP_PGD) {
1347	decoder->pge = false;
1348	decoder->continuous_period = false;
1349	decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1350	decoder->state.from_ip = decoder->ip;
1351	if (decoder->packet.count == 0) {
1352	decoder->state.to_ip = 0;
1353	} else {
1354	decoder->state.to_ip = decoder->last_ip;
1355	decoder->ip = decoder->last_ip;
1356	}
1357	decoder->state.type \|= INTEL_PT_TRACE_END;
1358	} else {
1359	decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1360	decoder->state.from_ip = decoder->ip;
1361	if (decoder->packet.count == 0) {
1362	decoder->state.to_ip = 0;
1363	} else {
1364	decoder->state.to_ip = decoder->last_ip;
1365	decoder->ip = decoder->last_ip;
1366	}
1367	}
1368	return 0;
1369	}
1370
1371	if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1372	uint64_t to_ip = decoder->ip + intel_pt_insn.length +
1373	intel_pt_insn.rel;
1374
1375	if (decoder->pgd_ip &&
1376	decoder->pkt_state == INTEL_PT_STATE_TIP_PGD &&
1377	decoder->pgd_ip(to_ip, decoder->data)) {
1378	/* Conditional branch leaving filter region */
1379	decoder->pge = false;
1380	decoder->continuous_period = false;
1381	decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1382	decoder->ip = to_ip;
1383	decoder->state.from_ip = decoder->ip;
1384	decoder->state.to_ip = to_ip;
1385	decoder->state.type \|= INTEL_PT_TRACE_END;
1386	return 0;
1387	}
1388	intel_pt_log_at("ERROR: Conditional branch when expecting indirect branch",
1389	decoder->ip);
1390	decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1391	return -ENOENT;
1392	}
1393
1394	return intel_pt_bug(decoder);
1395	}
1396
1397	static int intel_pt_walk_tnt(struct intel_pt_decoder *decoder)
1398	{
1399	struct intel_pt_insn intel_pt_insn;
1400	int err;
1401
1402	while (1) {
1403	err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);
1404	if (err == INTEL_PT_RETURN)
1405	return 0;
1406	if (err)
1407	return err;
1408
1409	if (intel_pt_insn.op == INTEL_PT_OP_RET) {
1410	if (!decoder->return_compression) {
1411	intel_pt_log_at("ERROR: RET when expecting conditional branch",
1412	decoder->ip);
1413	decoder->pkt_state = INTEL_PT_STATE_ERR3;
1414	return -ENOENT;
1415	}
1416	if (!decoder->ret_addr) {
1417	intel_pt_log_at("ERROR: Bad RET compression (stack empty)",
1418	decoder->ip);
1419	decoder->pkt_state = INTEL_PT_STATE_ERR3;
1420	return -ENOENT;
1421	}
1422	if (!(decoder->tnt.payload & BIT63)) {
1423	intel_pt_log_at("ERROR: Bad RET compression (TNT=N)",
1424	decoder->ip);
1425	decoder->pkt_state = INTEL_PT_STATE_ERR3;
1426	return -ENOENT;
1427	}
1428	decoder->tnt.count -= 1;
1429	if (decoder->tnt.count)
1430	decoder->pkt_state = INTEL_PT_STATE_TNT_CONT;
1431	else
1432	decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1433	decoder->tnt.payload <<= 1;
1434	decoder->state.from_ip = decoder->ip;
1435	decoder->ip = decoder->ret_addr;
1436	decoder->state.to_ip = decoder->ip;
1437	return 0;
1438	}
1439
1440	if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1441	/* Handle deferred TIPs */
1442	err = intel_pt_get_next_packet(decoder);
1443	if (err)
1444	return err;
1445	if (decoder->packet.type != INTEL_PT_TIP \|\|
1446	decoder->packet.count == 0) {
1447	intel_pt_log_at("ERROR: Missing deferred TIP for indirect branch",
1448	decoder->ip);
1449	decoder->pkt_state = INTEL_PT_STATE_ERR3;
1450	decoder->pkt_step = 0;
1451	return -ENOENT;
1452	}
1453	intel_pt_set_last_ip(decoder);
1454	decoder->state.from_ip = decoder->ip;
1455	decoder->state.to_ip = decoder->last_ip;
1456	decoder->ip = decoder->last_ip;
1457	intel_pt_update_nr(decoder);
1458	return 0;
1459	}
1460
1461	if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1462	decoder->tnt.count -= 1;
1463	if (decoder->tnt.count)
1464	decoder->pkt_state = INTEL_PT_STATE_TNT_CONT;
1465	else
1466	decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1467	if (decoder->tnt.payload & BIT63) {
1468	decoder->tnt.payload <<= 1;
1469	decoder->state.from_ip = decoder->ip;
1470	decoder->ip += intel_pt_insn.length +
1471	intel_pt_insn.rel;
1472	decoder->state.to_ip = decoder->ip;
1473	return 0;
1474	}
1475	/* Instruction sample for a non-taken branch */
1476	if (decoder->state.type & INTEL_PT_INSTRUCTION) {
1477	decoder->tnt.payload <<= 1;
1478	decoder->state.type = INTEL_PT_INSTRUCTION;
1479	decoder->state.from_ip = decoder->ip;
1480	decoder->state.to_ip = 0;
1481	decoder->ip += intel_pt_insn.length;
1482	return 0;
1483	}
1484	decoder->sample_cyc = false;
1485	decoder->ip += intel_pt_insn.length;
1486	if (!decoder->tnt.count) {
1487	intel_pt_update_sample_time(decoder);
1488	return -EAGAIN;
1489	}
1490	decoder->tnt.payload <<= 1;
1491	continue;
1492	}
1493
1494	return intel_pt_bug(decoder);
1495	}
1496	}
1497
1498	static int intel_pt_mode_tsx(struct intel_pt_decoder decoder, bool no_tip)
1499	{
1500	unsigned int fup_tx_flags;
1501	int err;
1502
1503	fup_tx_flags = decoder->packet.payload &
1504	(INTEL_PT_IN_TX \| INTEL_PT_ABORT_TX);
1505	err = intel_pt_get_next_packet(decoder);
1506	if (err)
1507	return err;
1508	if (decoder->packet.type == INTEL_PT_FUP) {
1509	decoder->fup_tx_flags = fup_tx_flags;
1510	decoder->set_fup_tx_flags = true;
1511	if (!(decoder->fup_tx_flags & INTEL_PT_ABORT_TX))
1512	*no_tip = true;
1513	} else {
1514	intel_pt_log_at("ERROR: Missing FUP after MODE.TSX",
1515	decoder->pos);
1516	intel_pt_update_in_tx(decoder);
1517	}
1518	return 0;
1519	}
1520
1521	static uint64_t intel_pt_8b_tsc(uint64_t timestamp, uint64_t ref_timestamp)
1522	{
1523	timestamp \|= (ref_timestamp & (0xffULL << 56));
1524
1525	if (timestamp < ref_timestamp) {
1526	if (ref_timestamp - timestamp > (1ULL << 55))
1527	timestamp += (1ULL << 56);
1528	} else {
1529	if (timestamp - ref_timestamp > (1ULL << 55))
1530	timestamp -= (1ULL << 56);
1531	}
1532
1533	return timestamp;
1534	}
1535
1536	/* For use only when decoder->vm_time_correlation is true */
1537	static bool intel_pt_time_in_range(struct intel_pt_decoder *decoder,
1538	uint64_t timestamp)
1539	{
1540	uint64_t max_timestamp = decoder->buf_timestamp;
1541
1542	if (!max_timestamp) {
1543	max_timestamp = decoder->last_reliable_timestamp +
1544	0x400000000ULL;
1545	}
1546	return timestamp >= decoder->last_reliable_timestamp &&
1547	timestamp < decoder->buf_timestamp;
1548	}
1549
1550	static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder *decoder)
1551	{
1552	uint64_t timestamp;
1553	bool bad = false;
1554
1555	decoder->have_tma = false;
1556
1557	if (decoder->ref_timestamp) {
1558	timestamp = intel_pt_8b_tsc(decoder->packet.payload,
1559	decoder->ref_timestamp);
1560	decoder->tsc_timestamp = timestamp;
1561	decoder->timestamp = timestamp;
1562	decoder->ref_timestamp = 0;
1563	decoder->timestamp_insn_cnt = 0;
1564	} else if (decoder->timestamp) {
1565	timestamp = decoder->packet.payload \|
1566	(decoder->timestamp & (0xffULL << 56));
1567	decoder->tsc_timestamp = timestamp;
1568	if (timestamp < decoder->timestamp &&
1569	decoder->timestamp - timestamp < decoder->tsc_slip) {
1570	intel_pt_log_to("Suppressing backwards timestamp",
1571	timestamp);
1572	timestamp = decoder->timestamp;
1573	}
1574	if (timestamp < decoder->timestamp) {
1575	if (!decoder->buf_timestamp \|\|
1576	(timestamp + (1ULL << 56) < decoder->buf_timestamp)) {
1577	intel_pt_log_to("Wraparound timestamp", timestamp);
1578	timestamp += (1ULL << 56);
1579	decoder->tsc_timestamp = timestamp;
1580	} else {
1581	intel_pt_log_to("Suppressing bad timestamp", timestamp);
1582	timestamp = decoder->timestamp;
1583	bad = true;
1584	}
1585	}
1586	if (decoder->vm_time_correlation &&
1587	(bad \|\| !intel_pt_time_in_range(decoder, timestamp)) &&
1588	intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_ERANGE))
1589	p_log("Timestamp out of range");
1590	decoder->timestamp = timestamp;
1591	decoder->timestamp_insn_cnt = 0;
1592	}
1593
1594	if (decoder->last_packet_type == INTEL_PT_CYC) {
1595	decoder->cyc_ref_timestamp = decoder->timestamp;
1596	decoder->cycle_cnt = 0;
1597	decoder->have_calc_cyc_to_tsc = false;
1598	intel_pt_calc_cyc_to_tsc(decoder, false);
1599	}
1600
1601	intel_pt_log_to("Setting timestamp", decoder->timestamp);
1602	}
1603
1604	static int intel_pt_overflow(struct intel_pt_decoder *decoder)
1605	{
1606	intel_pt_log("ERROR: Buffer overflow\n");
1607	intel_pt_clear_tx_flags(decoder);
1608	intel_pt_set_nr(decoder);
1609	decoder->timestamp_insn_cnt = 0;
1610	decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1611	decoder->overflow = true;
1612	return -EOVERFLOW;
1613	}
1614
1615	static inline void intel_pt_mtc_cyc_cnt_pge(struct intel_pt_decoder *decoder)
1616	{
1617	if (decoder->have_cyc)
1618	return;
1619
1620	decoder->cyc_cnt_timestamp = decoder->timestamp;
1621	decoder->base_cyc_cnt = decoder->tot_cyc_cnt;
1622	}
1623
1624	static inline void intel_pt_mtc_cyc_cnt_cbr(struct intel_pt_decoder *decoder)
1625	{
1626	decoder->tsc_to_cyc = decoder->cbr / decoder->max_non_turbo_ratio_fp;
1627
1628	if (decoder->pge)
1629	intel_pt_mtc_cyc_cnt_pge(decoder);
1630	}
1631
1632	static inline void intel_pt_mtc_cyc_cnt_upd(struct intel_pt_decoder *decoder)
1633	{
1634	uint64_t tot_cyc_cnt, tsc_delta;
1635
1636	if (decoder->have_cyc)
1637	return;
1638
1639	decoder->sample_cyc = true;
1640
1641	if (!decoder->pge \|\| decoder->timestamp <= decoder->cyc_cnt_timestamp)
1642	return;
1643
1644	tsc_delta = decoder->timestamp - decoder->cyc_cnt_timestamp;
1645	tot_cyc_cnt = tsc_delta * decoder->tsc_to_cyc + decoder->base_cyc_cnt;
1646
1647	if (tot_cyc_cnt > decoder->tot_cyc_cnt)
1648	decoder->tot_cyc_cnt = tot_cyc_cnt;
1649	}
1650
1651	static void intel_pt_calc_tma(struct intel_pt_decoder *decoder)
1652	{
1653	uint32_t ctc = decoder->packet.payload;
1654	uint32_t fc = decoder->packet.count;
1655	uint32_t ctc_rem = ctc & decoder->ctc_rem_mask;
1656
1657	if (!decoder->tsc_ctc_ratio_d)
1658	return;
1659
1660	if (decoder->pge && !decoder->in_psb)
1661	intel_pt_mtc_cyc_cnt_pge(decoder);
1662	else
1663	intel_pt_mtc_cyc_cnt_upd(decoder);
1664
1665	decoder->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
1666	decoder->last_ctc = ctc - ctc_rem;
1667	decoder->ctc_timestamp = decoder->tsc_timestamp - fc;
1668	if (decoder->tsc_ctc_mult) {
1669	decoder->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;
1670	} else {
1671	decoder->ctc_timestamp -= multdiv(ctc_rem,
1672	decoder->tsc_ctc_ratio_n,
1673	decoder->tsc_ctc_ratio_d);
1674	}
1675	decoder->ctc_delta = 0;
1676	decoder->have_tma = true;
1677	decoder->fixup_last_mtc = true;
1678	intel_pt_log("CTC timestamp " x64_fmt " last MTC %#x CTC rem %#x\n",
1679	decoder->ctc_timestamp, decoder->last_mtc, ctc_rem);
1680	}
1681
1682	static void intel_pt_calc_mtc_timestamp(struct intel_pt_decoder *decoder)
1683	{
1684	uint64_t timestamp;
1685	uint32_t mtc, mtc_delta;
1686
1687	if (!decoder->have_tma)
1688	return;
1689
1690	mtc = decoder->packet.payload;
1691
1692	if (decoder->mtc_shift > 8 && decoder->fixup_last_mtc) {
1693	decoder->fixup_last_mtc = false;
1694	intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,
1695	&decoder->last_mtc);
1696	}
1697
1698	if (mtc > decoder->last_mtc)
1699	mtc_delta = mtc - decoder->last_mtc;
1700	else
1701	mtc_delta = mtc + 256 - decoder->last_mtc;
1702
1703	decoder->ctc_delta += mtc_delta << decoder->mtc_shift;
1704
1705	if (decoder->tsc_ctc_mult) {
1706	timestamp = decoder->ctc_timestamp +
1707	decoder->ctc_delta * decoder->tsc_ctc_mult;
1708	} else {
1709	timestamp = decoder->ctc_timestamp +
1710	multdiv(decoder->ctc_delta,
1711	decoder->tsc_ctc_ratio_n,
1712	decoder->tsc_ctc_ratio_d);
1713	}
1714
1715	if (timestamp < decoder->timestamp)
1716	intel_pt_log("Suppressing MTC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n",
1717	timestamp, decoder->timestamp);
1718	else
1719	decoder->timestamp = timestamp;
1720
1721	intel_pt_mtc_cyc_cnt_upd(decoder);
1722
1723	decoder->timestamp_insn_cnt = 0;
1724	decoder->last_mtc = mtc;
1725
1726	if (decoder->last_packet_type == INTEL_PT_CYC) {
1727	decoder->cyc_ref_timestamp = decoder->timestamp;
1728	decoder->cycle_cnt = 0;
1729	decoder->have_calc_cyc_to_tsc = false;
1730	intel_pt_calc_cyc_to_tsc(decoder, true);
1731	}
1732
1733	intel_pt_log_to("Setting timestamp", decoder->timestamp);
1734	}
1735
1736	static void intel_pt_calc_cbr(struct intel_pt_decoder *decoder)
1737	{
1738	unsigned int cbr = decoder->packet.payload & 0xff;
1739
1740	decoder->cbr_payload = decoder->packet.payload;
1741
1742	if (decoder->cbr == cbr)
1743	return;
1744
1745	decoder->cbr = cbr;
1746	decoder->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;
1747
1748	intel_pt_mtc_cyc_cnt_cbr(decoder);
1749	}
1750
1751	static void intel_pt_calc_cyc_timestamp(struct intel_pt_decoder *decoder)
1752	{
1753	uint64_t timestamp = decoder->cyc_ref_timestamp;
1754
1755	decoder->have_cyc = true;
1756
1757	decoder->cycle_cnt += decoder->packet.payload;
1758	if (decoder->pge)
1759	decoder->tot_cyc_cnt += decoder->packet.payload;
1760	decoder->sample_cyc = true;
1761
1762	if (!decoder->cyc_ref_timestamp)
1763	return;
1764
1765	if (decoder->have_calc_cyc_to_tsc)
1766	timestamp += decoder->cycle_cnt * decoder->calc_cyc_to_tsc;
1767	else if (decoder->cbr)
1768	timestamp += decoder->cycle_cnt * decoder->cbr_cyc_to_tsc;
1769	else
1770	return;
1771
1772	if (timestamp < decoder->timestamp)
1773	intel_pt_log("Suppressing CYC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n",
1774	timestamp, decoder->timestamp);
1775	else
1776	decoder->timestamp = timestamp;
1777
1778	decoder->timestamp_insn_cnt = 0;
1779
1780	intel_pt_log_to("Setting timestamp", decoder->timestamp);
1781	}
1782
1783	static void intel_pt_bbp(struct intel_pt_decoder *decoder)
1784	{
1785	if (decoder->prev_pkt_ctx == INTEL_PT_NO_CTX) {
1786	memset(decoder->state.items.mask, 0, sizeof(decoder->state.items.mask));
1787	decoder->state.items.is_32_bit = false;
1788	}
1789	decoder->blk_type = decoder->packet.payload;
1790	decoder->blk_type_pos = intel_pt_blk_type_pos(decoder->blk_type);
1791	if (decoder->blk_type == INTEL_PT_GP_REGS)
1792	decoder->state.items.is_32_bit = decoder->packet.count;
1793	if (decoder->blk_type_pos < 0) {
1794	intel_pt_log("WARNING: Unknown block type %u\n",
1795	decoder->blk_type);
1796	} else if (decoder->state.items.mask[decoder->blk_type_pos]) {
1797	intel_pt_log("WARNING: Duplicate block type %u\n",
1798	decoder->blk_type);
1799	}
1800	}
1801
1802	static void intel_pt_bip(struct intel_pt_decoder *decoder)
1803	{
1804	uint32_t id = decoder->packet.count;
1805	uint32_t bit = 1 << id;
1806	int pos = decoder->blk_type_pos;
1807
1808	if (pos < 0 \|\| id >= INTEL_PT_BLK_ITEM_ID_CNT) {
1809	intel_pt_log("WARNING: Unknown block item %u type %d\n",
1810	id, decoder->blk_type);
1811	return;
1812	}
1813
1814	if (decoder->state.items.mask[pos] & bit) {
1815	intel_pt_log("WARNING: Duplicate block item %u type %d\n",
1816	id, decoder->blk_type);
1817	}
1818
1819	decoder->state.items.mask[pos] \|= bit;
1820	decoder->state.items.val[pos][id] = decoder->packet.payload;
1821	}
1822
1823	/* Walk PSB+ packets when already in sync. */
1824	static int intel_pt_walk_psbend(struct intel_pt_decoder *decoder)
1825	{
1826	int err;
1827
1828	decoder->in_psb = true;
1829
1830	while (1) {
1831	err = intel_pt_get_next_packet(decoder);
1832	if (err)
1833	goto out;
1834
1835	switch (decoder->packet.type) {
1836	case INTEL_PT_PSBEND:
1837	err = 0;
1838	goto out;
1839
1840	case INTEL_PT_TIP_PGD:
1841	case INTEL_PT_TIP_PGE:
1842	case INTEL_PT_TIP:
1843	case INTEL_PT_TNT:
1844	case INTEL_PT_TRACESTOP:
1845	case INTEL_PT_BAD:
1846	case INTEL_PT_PSB:
1847	case INTEL_PT_PTWRITE:
1848	case INTEL_PT_PTWRITE_IP:
1849	case INTEL_PT_EXSTOP:
1850	case INTEL_PT_EXSTOP_IP:
1851	case INTEL_PT_MWAIT:
1852	case INTEL_PT_PWRE:
1853	case INTEL_PT_PWRX:
1854	case INTEL_PT_BBP:
1855	case INTEL_PT_BIP:
1856	case INTEL_PT_BEP:
1857	case INTEL_PT_BEP_IP:
1858	decoder->have_tma = false;
1859	intel_pt_log("ERROR: Unexpected packet\n");
1860	err = -EAGAIN;
1861	goto out;
1862
1863	case INTEL_PT_OVF:
1864	err = intel_pt_overflow(decoder);
1865	goto out;
1866
1867	case INTEL_PT_TSC:
1868	intel_pt_calc_tsc_timestamp(decoder);
1869	break;
1870
1871	case INTEL_PT_TMA:
1872	intel_pt_calc_tma(decoder);
1873	break;
1874
1875	case INTEL_PT_CBR:
1876	intel_pt_calc_cbr(decoder);
1877	break;
1878
1879	case INTEL_PT_MODE_EXEC:
1880	decoder->exec_mode = decoder->packet.payload;
1881	break;
1882
1883	case INTEL_PT_PIP:
1884	intel_pt_set_pip(decoder);
1885	break;
1886
1887	case INTEL_PT_FUP:
1888	decoder->pge = true;
1889	if (decoder->packet.count) {
1890	intel_pt_set_last_ip(decoder);
1891	decoder->psb_ip = decoder->last_ip;
1892	}
1893	break;
1894
1895	case INTEL_PT_MODE_TSX:
1896	intel_pt_update_in_tx(decoder);
1897	break;
1898
1899	case INTEL_PT_MTC:
1900	intel_pt_calc_mtc_timestamp(decoder);
1901	if (decoder->period_type == INTEL_PT_PERIOD_MTC)
1902	decoder->state.type \|= INTEL_PT_INSTRUCTION;
1903	break;
1904
1905	case INTEL_PT_CYC:
1906	intel_pt_calc_cyc_timestamp(decoder);
1907	break;
1908
1909	case INTEL_PT_VMCS:
1910	case INTEL_PT_MNT:
1911	case INTEL_PT_PAD:
1912	default:
1913	break;
1914	}
1915	}
1916	out:
1917	decoder->in_psb = false;
1918
1919	return err;
1920	}
1921
1922	static int intel_pt_walk_fup_tip(struct intel_pt_decoder *decoder)
1923	{
1924	int err;
1925
1926	if (decoder->tx_flags & INTEL_PT_ABORT_TX) {
1927	decoder->tx_flags = 0;
1928	decoder->state.flags &= ~INTEL_PT_IN_TX;
1929	decoder->state.flags \|= INTEL_PT_ABORT_TX;
1930	} else {
1931	decoder->state.flags \|= INTEL_PT_ASYNC;
1932	}
1933
1934	while (1) {
1935	err = intel_pt_get_next_packet(decoder);
1936	if (err)
1937	return err;
1938
1939	switch (decoder->packet.type) {
1940	case INTEL_PT_TNT:
1941	case INTEL_PT_FUP:
1942	case INTEL_PT_TRACESTOP:
1943	case INTEL_PT_PSB:
1944	case INTEL_PT_TSC:
1945	case INTEL_PT_TMA:
1946	case INTEL_PT_MODE_TSX:
1947	case INTEL_PT_BAD:
1948	case INTEL_PT_PSBEND:
1949	case INTEL_PT_PTWRITE:
1950	case INTEL_PT_PTWRITE_IP:
1951	case INTEL_PT_EXSTOP:
1952	case INTEL_PT_EXSTOP_IP:
1953	case INTEL_PT_MWAIT:
1954	case INTEL_PT_PWRE:
1955	case INTEL_PT_PWRX:
1956	case INTEL_PT_BBP:
1957	case INTEL_PT_BIP:
1958	case INTEL_PT_BEP:
1959	case INTEL_PT_BEP_IP:
1960	intel_pt_log("ERROR: Missing TIP after FUP\n");
1961	decoder->pkt_state = INTEL_PT_STATE_ERR3;
1962	decoder->pkt_step = 0;
1963	return -ENOENT;
1964
1965	case INTEL_PT_CBR:
1966	intel_pt_calc_cbr(decoder);
1967	break;
1968
1969	case INTEL_PT_OVF:
1970	return intel_pt_overflow(decoder);
1971
1972	case INTEL_PT_TIP_PGD:
1973	decoder->state.from_ip = decoder->ip;
1974	if (decoder->packet.count == 0) {
1975	decoder->state.to_ip = 0;
1976	} else {
1977	intel_pt_set_ip(decoder);
1978	decoder->state.to_ip = decoder->ip;
1979	}
1980	decoder->pge = false;
1981	decoder->continuous_period = false;
1982	decoder->state.type \|= INTEL_PT_TRACE_END;
1983	intel_pt_update_nr(decoder);
1984	return 0;
1985
1986	case INTEL_PT_TIP_PGE:
1987	decoder->pge = true;
1988	intel_pt_log("Omitting PGE ip " x64_fmt "\n",
1989	decoder->ip);
1990	decoder->state.from_ip = 0;
1991	if (decoder->packet.count == 0) {
1992	decoder->state.to_ip = 0;
1993	} else {
1994	intel_pt_set_ip(decoder);
1995	decoder->state.to_ip = decoder->ip;
1996	}
1997	decoder->state.type \|= INTEL_PT_TRACE_BEGIN;
1998	intel_pt_mtc_cyc_cnt_pge(decoder);
1999	intel_pt_set_nr(decoder);
2000	return 0;
2001
2002	case INTEL_PT_TIP:
2003	decoder->state.from_ip = decoder->ip;
2004	if (decoder->packet.count == 0) {
2005	decoder->state.to_ip = 0;
2006	} else {
2007	intel_pt_set_ip(decoder);
2008	decoder->state.to_ip = decoder->ip;
2009	}
2010	intel_pt_update_nr(decoder);
2011	return 0;
2012
2013	case INTEL_PT_PIP:
2014	intel_pt_update_pip(decoder);
2015	break;
2016
2017	case INTEL_PT_MTC:
2018	intel_pt_calc_mtc_timestamp(decoder);
2019	if (decoder->period_type == INTEL_PT_PERIOD_MTC)
2020	decoder->state.type \|= INTEL_PT_INSTRUCTION;
2021	break;
2022
2023	case INTEL_PT_CYC:
2024	intel_pt_calc_cyc_timestamp(decoder);
2025	break;
2026
2027	case INTEL_PT_MODE_EXEC:
2028	decoder->exec_mode = decoder->packet.payload;
2029	break;
2030
2031	case INTEL_PT_VMCS:
2032	case INTEL_PT_MNT:
2033	case INTEL_PT_PAD:
2034	break;
2035
2036	default:
2037	return intel_pt_bug(decoder);
2038	}
2039	}
2040	}
2041
2042	static int intel_pt_resample(struct intel_pt_decoder *decoder)
2043	{
2044	decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2045	decoder->state.type = INTEL_PT_INSTRUCTION;
2046	decoder->state.from_ip = decoder->ip;
2047	decoder->state.to_ip = 0;
2048	return 0;
2049	}
2050
2051	struct intel_pt_vm_tsc_info {
2052	struct intel_pt_pkt pip_packet;
2053	struct intel_pt_pkt vmcs_packet;
2054	struct intel_pt_pkt tma_packet;
2055	bool tsc, pip, vmcs, tma, psbend;
2056	uint64_t ctc_delta;
2057	uint64_t last_ctc;
2058	int max_lookahead;
2059	};
2060
2061	/* Lookahead and get the PIP, VMCS and TMA packets from PSB+ */
2062	static int intel_pt_vm_psb_lookahead_cb(struct intel_pt_pkt_info *pkt_info)
2063	{
2064	struct intel_pt_vm_tsc_info *data = pkt_info->data;
2065
2066	switch (pkt_info->packet.type) {
2067	case INTEL_PT_PAD:
2068	case INTEL_PT_MNT:
2069	case INTEL_PT_MODE_EXEC:
2070	case INTEL_PT_MODE_TSX:
2071	case INTEL_PT_MTC:
2072	case INTEL_PT_FUP:
2073	case INTEL_PT_CYC:
2074	case INTEL_PT_CBR:
2075	break;
2076
2077	case INTEL_PT_TSC:
2078	data->tsc = true;
2079	break;
2080
2081	case INTEL_PT_TMA:
2082	data->tma_packet = pkt_info->packet;
2083	data->tma = true;
2084	break;
2085
2086	case INTEL_PT_PIP:
2087	data->pip_packet = pkt_info->packet;
2088	data->pip = true;
2089	break;
2090
2091	case INTEL_PT_VMCS:
2092	data->vmcs_packet = pkt_info->packet;
2093	data->vmcs = true;
2094	break;
2095
2096	case INTEL_PT_PSBEND:
2097	data->psbend = true;
2098	return 1;
2099
2100	case INTEL_PT_TIP_PGE:
2101	case INTEL_PT_PTWRITE:
2102	case INTEL_PT_PTWRITE_IP:
2103	case INTEL_PT_EXSTOP:
2104	case INTEL_PT_EXSTOP_IP:
2105	case INTEL_PT_MWAIT:
2106	case INTEL_PT_PWRE:
2107	case INTEL_PT_PWRX:
2108	case INTEL_PT_BBP:
2109	case INTEL_PT_BIP:
2110	case INTEL_PT_BEP:
2111	case INTEL_PT_BEP_IP:
2112	case INTEL_PT_OVF:
2113	case INTEL_PT_BAD:
2114	case INTEL_PT_TNT:
2115	case INTEL_PT_TIP_PGD:
2116	case INTEL_PT_TIP:
2117	case INTEL_PT_PSB:
2118	case INTEL_PT_TRACESTOP:
2119	default:
2120	return 1;
2121	}
2122
2123	return 0;
2124	}
2125
2126	struct intel_pt_ovf_fup_info {
2127	int max_lookahead;
2128	bool found;
2129	};
2130
2131	/* Lookahead to detect a FUP packet after OVF */
2132	static int intel_pt_ovf_fup_lookahead_cb(struct intel_pt_pkt_info *pkt_info)
2133	{
2134	struct intel_pt_ovf_fup_info *data = pkt_info->data;
2135
2136	if (pkt_info->packet.type == INTEL_PT_CYC \|\|
2137	pkt_info->packet.type == INTEL_PT_MTC \|\|
2138	pkt_info->packet.type == INTEL_PT_TSC)
2139	return !--(data->max_lookahead);
2140	data->found = pkt_info->packet.type == INTEL_PT_FUP;
2141	return 1;
2142	}
2143
2144	static bool intel_pt_ovf_fup_lookahead(struct intel_pt_decoder *decoder)
2145	{
2146	struct intel_pt_ovf_fup_info data = {
2147	.max_lookahead = 16,
2148	.found = false,
2149	};
2150
2151	intel_pt_pkt_lookahead(decoder, intel_pt_ovf_fup_lookahead_cb, &data);
2152	return data.found;
2153	}
2154
2155	/* Lookahead and get the TMA packet after TSC */
2156	static int intel_pt_tma_lookahead_cb(struct intel_pt_pkt_info *pkt_info)
2157	{
2158	struct intel_pt_vm_tsc_info *data = pkt_info->data;
2159
2160	if (pkt_info->packet.type == INTEL_PT_CYC \|\|
2161	pkt_info->packet.type == INTEL_PT_MTC)
2162	return !--(data->max_lookahead);
2163
2164	if (pkt_info->packet.type == INTEL_PT_TMA) {
2165	data->tma_packet = pkt_info->packet;
2166	data->tma = true;
2167	}
2168	return 1;
2169	}
2170
2171	static uint64_t intel_pt_ctc_to_tsc(struct intel_pt_decoder *decoder, uint64_t ctc)
2172	{
2173	if (decoder->tsc_ctc_mult)
2174	return ctc * decoder->tsc_ctc_mult;
2175	else
2176	return multdiv(ctc, decoder->tsc_ctc_ratio_n, decoder->tsc_ctc_ratio_d);
2177	}
2178
2179	static uint64_t intel_pt_calc_expected_tsc(struct intel_pt_decoder *decoder,
2180	uint32_t ctc,
2181	uint32_t fc,
2182	uint64_t last_ctc_timestamp,
2183	uint64_t ctc_delta,
2184	uint32_t last_ctc)
2185	{
2186	/* Number of CTC ticks from last_ctc_timestamp to last_mtc */
2187	uint64_t last_mtc_ctc = last_ctc + ctc_delta;
2188	/*
2189	* Number of CTC ticks from there until current TMA packet. We would
2190	* expect last_mtc_ctc to be before ctc, but the TSC packet can slip
2191	* past an MTC, so a sign-extended value is used.
2192	*/
2193	uint64_t delta = (int16_t)((uint16_t)ctc - (uint16_t)last_mtc_ctc);
2194	/* Total CTC ticks from last_ctc_timestamp to current TMA packet */
2195	uint64_t new_ctc_delta = ctc_delta + delta;
2196	uint64_t expected_tsc;
2197
2198	/*
2199	* Convert CTC ticks to TSC ticks, add the starting point
2200	* (last_ctc_timestamp) and the fast counter from the TMA packet.
2201	*/
2202	expected_tsc = last_ctc_timestamp + intel_pt_ctc_to_tsc(decoder, new_ctc_delta) + fc;
2203
2204	if (intel_pt_enable_logging) {
2205	intel_pt_log_x64(last_mtc_ctc);
2206	intel_pt_log_x32(last_ctc);
2207	intel_pt_log_x64(ctc_delta);
2208	intel_pt_log_x64(delta);
2209	intel_pt_log_x32(ctc);
2210	intel_pt_log_x64(new_ctc_delta);
2211	intel_pt_log_x64(last_ctc_timestamp);
2212	intel_pt_log_x32(fc);
2213	intel_pt_log_x64(intel_pt_ctc_to_tsc(decoder, new_ctc_delta));
2214	intel_pt_log_x64(expected_tsc);
2215	}
2216
2217	return expected_tsc;
2218	}
2219
2220	static uint64_t intel_pt_expected_tsc(struct intel_pt_decoder *decoder,
2221	struct intel_pt_vm_tsc_info *data)
2222	{
2223	uint32_t ctc = data->tma_packet.payload;
2224	uint32_t fc = data->tma_packet.count;
2225
2226	return intel_pt_calc_expected_tsc(decoder, ctc, fc,
2227	decoder->ctc_timestamp,
2228	data->ctc_delta, data->last_ctc);
2229	}
2230
2231	static void intel_pt_translate_vm_tsc(struct intel_pt_decoder *decoder,
2232	struct intel_pt_vmcs_info *vmcs_info)
2233	{
2234	uint64_t payload = decoder->packet.payload;
2235
2236	/* VMX adds the TSC Offset, so subtract to get host TSC */
2237	decoder->packet.payload -= vmcs_info->tsc_offset;
2238	/* TSC packet has only 7 bytes */
2239	decoder->packet.payload &= SEVEN_BYTES;
2240
2241	/*
2242	* The buffer is mmapped from the data file, so this also updates the
2243	* data file.
2244	*/
2245	if (!decoder->vm_tm_corr_dry_run)
2246	memcpy((void *)decoder->buf + 1, &decoder->packet.payload, 7);
2247
2248	intel_pt_log("Translated VM TSC %#" PRIx64 " -> %#" PRIx64
2249	" VMCS %#" PRIx64 " TSC Offset %#" PRIx64 "\n",
2250	payload, decoder->packet.payload, vmcs_info->vmcs,
2251	vmcs_info->tsc_offset);
2252	}
2253
2254	static void intel_pt_translate_vm_tsc_offset(struct intel_pt_decoder *decoder,
2255	uint64_t tsc_offset)
2256	{
2257	struct intel_pt_vmcs_info vmcs_info = {
2258	.vmcs = NO_VMCS,
2259	.tsc_offset = tsc_offset
2260	};
2261
2262	intel_pt_translate_vm_tsc(decoder, &vmcs_info);
2263	}
2264
2265	static inline bool in_vm(uint64_t pip_payload)
2266	{
2267	return pip_payload & 1;
2268	}
2269
2270	static inline bool pip_in_vm(struct intel_pt_pkt *pip_packet)
2271	{
2272	return pip_packet->payload & 1;
2273	}
2274
2275	static void intel_pt_print_vmcs_info(struct intel_pt_vmcs_info *vmcs_info)
2276	{
2277	p_log("VMCS: %#" PRIx64 " TSC Offset %#" PRIx64,
2278	vmcs_info->vmcs, vmcs_info->tsc_offset);
2279	}
2280
2281	static void intel_pt_vm_tm_corr_psb(struct intel_pt_decoder *decoder,
2282	struct intel_pt_vm_tsc_info *data)
2283	{
2284	memset(data, 0, sizeof(*data));
2285	data->ctc_delta = decoder->ctc_delta;
2286	data->last_ctc = decoder->last_ctc;
2287	intel_pt_pkt_lookahead(decoder, intel_pt_vm_psb_lookahead_cb, data);
2288	if (data->tsc && !data->psbend)
2289	p_log("ERROR: PSB without PSBEND");
2290	decoder->in_psb = data->psbend;
2291	}
2292
2293	static void intel_pt_vm_tm_corr_first_tsc(struct intel_pt_decoder *decoder,
2294	struct intel_pt_vm_tsc_info *data,
2295	struct intel_pt_vmcs_info *vmcs_info,
2296	uint64_t host_tsc)
2297	{
2298	if (!decoder->in_psb) {
2299	/* Can't happen */
2300	p_log("ERROR: First TSC is not in PSB+");
2301	}
2302
2303	if (data->pip) {
2304	if (pip_in_vm(&data->pip_packet)) { /* Guest */
2305	if (vmcs_info && vmcs_info->tsc_offset) {
2306	intel_pt_translate_vm_tsc(decoder, vmcs_info);
2307	decoder->vm_tm_corr_reliable = true;
2308	} else {
2309	p_log("ERROR: First TSC, unknown TSC Offset");
2310	}
2311	} else { /* Host */
2312	decoder->vm_tm_corr_reliable = true;
2313	}
2314	} else { /* Host or Guest */
2315	decoder->vm_tm_corr_reliable = false;
2316	if (intel_pt_time_in_range(decoder, host_tsc)) {
2317	/* Assume Host */
2318	} else {
2319	/* Assume Guest */
2320	if (vmcs_info && vmcs_info->tsc_offset)
2321	intel_pt_translate_vm_tsc(decoder, vmcs_info);
2322	else
2323	p_log("ERROR: First TSC, no PIP, unknown TSC Offset");
2324	}
2325	}
2326	}
2327
2328	static void intel_pt_vm_tm_corr_tsc(struct intel_pt_decoder *decoder,
2329	struct intel_pt_vm_tsc_info *data)
2330	{
2331	struct intel_pt_vmcs_info *vmcs_info;
2332	uint64_t tsc_offset = 0;
2333	uint64_t vmcs;
2334	bool reliable = true;
2335	uint64_t expected_tsc;
2336	uint64_t host_tsc;
2337	uint64_t ref_timestamp;
2338
2339	bool assign = false;
2340	bool assign_reliable = false;
2341
2342	/* Already have 'data' for the in_psb case */
2343	if (!decoder->in_psb) {
2344	memset(data, 0, sizeof(*data));
2345	data->ctc_delta = decoder->ctc_delta;
2346	data->last_ctc = decoder->last_ctc;
2347	data->max_lookahead = 16;
2348	intel_pt_pkt_lookahead(decoder, intel_pt_tma_lookahead_cb, data);
2349	if (decoder->pge) {
2350	data->pip = true;
2351	data->pip_packet.payload = decoder->pip_payload;
2352	}
2353	}
2354
2355	/* Calculations depend on having TMA packets */
2356	if (!data->tma) {
2357	p_log("ERROR: TSC without TMA");
2358	return;
2359	}
2360
2361	vmcs = data->vmcs ? data->vmcs_packet.payload : decoder->vmcs;
2362	if (vmcs == NO_VMCS)
2363	vmcs = 0;
2364
2365	vmcs_info = decoder->findnew_vmcs_info(decoder->data, vmcs);
2366
2367	ref_timestamp = decoder->timestamp ? decoder->timestamp : decoder->buf_timestamp;
2368	host_tsc = intel_pt_8b_tsc(decoder->packet.payload, ref_timestamp);
2369
2370	if (!decoder->ctc_timestamp) {
2371	intel_pt_vm_tm_corr_first_tsc(decoder, data, vmcs_info, host_tsc);
2372	return;
2373	}
2374
2375	expected_tsc = intel_pt_expected_tsc(decoder, data);
2376
2377	tsc_offset = host_tsc - expected_tsc;
2378
2379	/* Determine if TSC is from Host or Guest */
2380	if (data->pip) {
2381	if (pip_in_vm(&data->pip_packet)) { /* Guest */
2382	if (!vmcs_info) {
2383	/* PIP NR=1 without VMCS cannot happen */
2384	p_log("ERROR: Missing VMCS");
2385	intel_pt_translate_vm_tsc_offset(decoder, tsc_offset);
2386	decoder->vm_tm_corr_reliable = false;
2387	return;
2388	}
2389	} else { /* Host */
2390	decoder->last_reliable_timestamp = host_tsc;
2391	decoder->vm_tm_corr_reliable = true;
2392	return;
2393	}
2394	} else { /* Host or Guest */
2395	reliable = false; /* Host/Guest is a guess, so not reliable */
2396	if (decoder->in_psb) {
2397	if (!tsc_offset)
2398	return; /* Zero TSC Offset, assume Host */
2399	/*
2400	* TSC packet has only 7 bytes of TSC. We have no
2401	* information about the Guest's 8th byte, but it
2402	* doesn't matter because we only need 7 bytes.
2403	* Here, since the 8th byte is unreliable and
2404	* irrelevant, compare only 7 byes.
2405	*/
2406	if (vmcs_info &&
2407	(tsc_offset & SEVEN_BYTES) ==
2408	(vmcs_info->tsc_offset & SEVEN_BYTES)) {
2409	/* Same TSC Offset as last VMCS, assume Guest */
2410	goto guest;
2411	}
2412	}
2413	/*
2414	* Check if the host_tsc is within the expected range.
2415	* Note, we could narrow the range more by looking ahead for
2416	* the next host TSC in the same buffer, but we don't bother to
2417	* do that because this is probably good enough.
2418	*/
2419	if (host_tsc >= expected_tsc && intel_pt_time_in_range(decoder, host_tsc)) {
2420	/* Within expected range for Host TSC, assume Host */
2421	decoder->vm_tm_corr_reliable = false;
2422	return;
2423	}
2424	}
2425
2426	guest: /* Assuming Guest */
2427
2428	/* Determine whether to assign TSC Offset */
2429	if (vmcs_info && vmcs_info->vmcs) {
2430	if (vmcs_info->tsc_offset && vmcs_info->reliable) {
2431	assign = false;
2432	} else if (decoder->in_psb && data->pip && decoder->vm_tm_corr_reliable &&
2433	decoder->vm_tm_corr_continuous && decoder->vm_tm_corr_same_buf) {
2434	/* Continuous tracing, TSC in a PSB is not a time loss */
2435	assign = true;
2436	assign_reliable = true;
2437	} else if (decoder->in_psb && data->pip && decoder->vm_tm_corr_same_buf) {
2438	/*
2439	* Unlikely to be a time loss TSC in a PSB which is not
2440	* at the start of a buffer.
2441	*/
2442	assign = true;
2443	assign_reliable = false;
2444	}
2445	}
2446
2447	/* Record VMCS TSC Offset */
2448	if (assign && (vmcs_info->tsc_offset != tsc_offset \|\|
2449	vmcs_info->reliable != assign_reliable)) {
2450	bool print = vmcs_info->tsc_offset != tsc_offset;
2451
2452	vmcs_info->tsc_offset = tsc_offset;
2453	vmcs_info->reliable = assign_reliable;
2454	if (print)
2455	intel_pt_print_vmcs_info(vmcs_info);
2456	}
2457
2458	/* Determine what TSC Offset to use */
2459	if (vmcs_info && vmcs_info->tsc_offset) {
2460	if (!vmcs_info->reliable)
2461	reliable = false;
2462	intel_pt_translate_vm_tsc(decoder, vmcs_info);
2463	} else {
2464	reliable = false;
2465	if (vmcs_info) {
2466	if (!vmcs_info->error_printed) {
2467	p_log("ERROR: Unknown TSC Offset for VMCS %#" PRIx64,
2468	vmcs_info->vmcs);
2469	vmcs_info->error_printed = true;
2470	}
2471	} else {
2472	if (intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_UNK_VMCS))
2473	p_log("ERROR: Unknown VMCS");
2474	}
2475	intel_pt_translate_vm_tsc_offset(decoder, tsc_offset);
2476	}
2477
2478	decoder->vm_tm_corr_reliable = reliable;
2479	}
2480
2481	static void intel_pt_vm_tm_corr_pebs_tsc(struct intel_pt_decoder *decoder)
2482	{
2483	uint64_t host_tsc = decoder->packet.payload;
2484	uint64_t guest_tsc = decoder->packet.payload;
2485	struct intel_pt_vmcs_info *vmcs_info;
2486	uint64_t vmcs;
2487
2488	vmcs = decoder->vmcs;
2489	if (vmcs == NO_VMCS)
2490	vmcs = 0;
2491
2492	vmcs_info = decoder->findnew_vmcs_info(decoder->data, vmcs);
2493
2494	if (decoder->pge) {
2495	if (in_vm(decoder->pip_payload)) { /* Guest */
2496	if (!vmcs_info) {
2497	/* PIP NR=1 without VMCS cannot happen */
2498	p_log("ERROR: Missing VMCS");
2499	}
2500	} else { /* Host */
2501	return;
2502	}
2503	} else { /* Host or Guest */
2504	if (intel_pt_time_in_range(decoder, host_tsc)) {
2505	/* Within expected range for Host TSC, assume Host */
2506	return;
2507	}
2508	}
2509
2510	if (vmcs_info) {
2511	/* Translate Guest TSC to Host TSC */
2512	host_tsc = ((guest_tsc & SEVEN_BYTES) - vmcs_info->tsc_offset) & SEVEN_BYTES;
2513	host_tsc = intel_pt_8b_tsc(host_tsc, decoder->timestamp);
2514	intel_pt_log("Translated VM TSC %#" PRIx64 " -> %#" PRIx64
2515	" VMCS %#" PRIx64 " TSC Offset %#" PRIx64 "\n",
2516	guest_tsc, host_tsc, vmcs_info->vmcs,
2517	vmcs_info->tsc_offset);
2518	if (!intel_pt_time_in_range(decoder, host_tsc) &&
2519	intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_ERANGE))
2520	p_log("Timestamp out of range");
2521	} else {
2522	if (intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_UNK_VMCS))
2523	p_log("ERROR: Unknown VMCS");
2524	host_tsc = decoder->timestamp;
2525	}
2526
2527	decoder->packet.payload = host_tsc;
2528
2529	if (!decoder->vm_tm_corr_dry_run)
2530	memcpy((void *)decoder->buf + 1, &host_tsc, 8);
2531	}
2532
2533	static int intel_pt_vm_time_correlation(struct intel_pt_decoder *decoder)
2534	{
2535	struct intel_pt_vm_tsc_info data = { .psbend = false };
2536	bool pge;
2537	int err;
2538
2539	if (decoder->in_psb)
2540	intel_pt_vm_tm_corr_psb(decoder, &data);
2541
2542	while (1) {
2543	err = intel_pt_get_next_packet(decoder);
2544	if (err == -ENOLINK)
2545	continue;
2546	if (err)
2547	break;
2548
2549	switch (decoder->packet.type) {
2550	case INTEL_PT_TIP_PGD:
2551	decoder->pge = false;
2552	decoder->vm_tm_corr_continuous = false;
2553	break;
2554
2555	case INTEL_PT_TNT:
2556	case INTEL_PT_TIP:
2557	case INTEL_PT_TIP_PGE:
2558	decoder->pge = true;
2559	break;
2560
2561	case INTEL_PT_OVF:
2562	decoder->in_psb = false;
2563	pge = decoder->pge;
2564	decoder->pge = intel_pt_ovf_fup_lookahead(decoder);
2565	if (pge != decoder->pge)
2566	intel_pt_log("Surprising PGE change in OVF!");
2567	if (!decoder->pge)
2568	decoder->vm_tm_corr_continuous = false;
2569	break;
2570
2571	case INTEL_PT_FUP:
2572	if (decoder->in_psb)
2573	decoder->pge = true;
2574	break;
2575
2576	case INTEL_PT_TRACESTOP:
2577	decoder->pge = false;
2578	decoder->vm_tm_corr_continuous = false;
2579	decoder->have_tma = false;
2580	break;
2581
2582	case INTEL_PT_PSB:
2583	intel_pt_vm_tm_corr_psb(decoder, &data);
2584	break;
2585
2586	case INTEL_PT_PIP:
2587	decoder->pip_payload = decoder->packet.payload;
2588	break;
2589
2590	case INTEL_PT_MTC:
2591	intel_pt_calc_mtc_timestamp(decoder);
2592	break;
2593
2594	case INTEL_PT_TSC:
2595	intel_pt_vm_tm_corr_tsc(decoder, &data);
2596	intel_pt_calc_tsc_timestamp(decoder);
2597	decoder->vm_tm_corr_same_buf = true;
2598	decoder->vm_tm_corr_continuous = decoder->pge;
2599	break;
2600
2601	case INTEL_PT_TMA:
2602	intel_pt_calc_tma(decoder);
2603	break;
2604
2605	case INTEL_PT_CYC:
2606	intel_pt_calc_cyc_timestamp(decoder);
2607	break;
2608
2609	case INTEL_PT_CBR:
2610	intel_pt_calc_cbr(decoder);
2611	break;
2612
2613	case INTEL_PT_PSBEND:
2614	decoder->in_psb = false;
2615	data.psbend = false;
2616	break;
2617
2618	case INTEL_PT_VMCS:
2619	if (decoder->packet.payload != NO_VMCS)
2620	decoder->vmcs = decoder->packet.payload;
2621	break;
2622
2623	case INTEL_PT_BBP:
2624	decoder->blk_type = decoder->packet.payload;
2625	break;
2626
2627	case INTEL_PT_BIP:
2628	if (decoder->blk_type == INTEL_PT_PEBS_BASIC &&
2629	decoder->packet.count == 2)
2630	intel_pt_vm_tm_corr_pebs_tsc(decoder);
2631	break;
2632
2633	case INTEL_PT_BEP:
2634	case INTEL_PT_BEP_IP:
2635	decoder->blk_type = 0;
2636	break;
2637
2638	case INTEL_PT_MODE_EXEC:
2639	case INTEL_PT_MODE_TSX:
2640	case INTEL_PT_MNT:
2641	case INTEL_PT_PAD:
2642	case INTEL_PT_PTWRITE_IP:
2643	case INTEL_PT_PTWRITE:
2644	case INTEL_PT_MWAIT:
2645	case INTEL_PT_PWRE:
2646	case INTEL_PT_EXSTOP_IP:
2647	case INTEL_PT_EXSTOP:
2648	case INTEL_PT_PWRX:
2649	case INTEL_PT_BAD: /* Does not happen */
2650	default:
2651	break;
2652	}
2653	}
2654
2655	return err;
2656	}
2657
2658	#define HOP_PROCESS 0
2659	#define HOP_IGNORE 1
2660	#define HOP_RETURN 2
2661	#define HOP_AGAIN 3
2662
2663	static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder);
2664
2665	/* Hop mode: Ignore TNT, do not walk code, but get ip from FUPs and TIPs */
2666	static int intel_pt_hop_trace(struct intel_pt_decoder decoder, bool no_tip, int *err)
2667	{
2668	/* Leap from PSB to PSB, getting ip from FUP within PSB+ */
2669	if (decoder->leap && !decoder->in_psb && decoder->packet.type != INTEL_PT_PSB) {
2670	*err = intel_pt_scan_for_psb(decoder);
2671	if (*err)
2672	return HOP_RETURN;
2673	}
2674
2675	switch (decoder->packet.type) {
2676	case INTEL_PT_TNT:
2677	return HOP_IGNORE;
2678
2679	case INTEL_PT_TIP_PGD:
2680	if (!decoder->packet.count) {
2681	intel_pt_set_nr(decoder);
2682	return HOP_IGNORE;
2683	}
2684	intel_pt_set_ip(decoder);
2685	decoder->state.type \|= INTEL_PT_TRACE_END;
2686	decoder->state.from_ip = 0;
2687	decoder->state.to_ip = decoder->ip;
2688	intel_pt_update_nr(decoder);
2689	return HOP_RETURN;
2690
2691	case INTEL_PT_TIP:
2692	if (!decoder->packet.count) {
2693	intel_pt_set_nr(decoder);
2694	return HOP_IGNORE;
2695	}
2696	intel_pt_set_ip(decoder);
2697	decoder->state.type = INTEL_PT_INSTRUCTION;
2698	decoder->state.from_ip = decoder->ip;
2699	decoder->state.to_ip = 0;
2700	intel_pt_update_nr(decoder);
2701	return HOP_RETURN;
2702
2703	case INTEL_PT_FUP:
2704	if (!decoder->packet.count)
2705	return HOP_IGNORE;
2706	intel_pt_set_ip(decoder);
2707	if (intel_pt_fup_event(decoder))
2708	return HOP_RETURN;
2709	if (!decoder->branch_enable)
2710	*no_tip = true;
2711	if (*no_tip) {
2712	decoder->state.type = INTEL_PT_INSTRUCTION;
2713	decoder->state.from_ip = decoder->ip;
2714	decoder->state.to_ip = 0;
2715	return HOP_RETURN;
2716	}
2717	*err = intel_pt_walk_fup_tip(decoder);
2718	if (!*err)
2719	decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;
2720	return HOP_RETURN;
2721
2722	case INTEL_PT_PSB:
2723	decoder->state.psb_offset = decoder->pos;
2724	decoder->psb_ip = 0;
2725	decoder->last_ip = 0;
2726	decoder->have_last_ip = true;
2727	*err = intel_pt_walk_psbend(decoder);
2728	if (*err == -EAGAIN)
2729	return HOP_AGAIN;
2730	if (*err)
2731	return HOP_RETURN;
2732	decoder->state.type = INTEL_PT_PSB_EVT;
2733	if (decoder->psb_ip) {
2734	decoder->state.type \|= INTEL_PT_INSTRUCTION;
2735	decoder->ip = decoder->psb_ip;
2736	}
2737	decoder->state.from_ip = decoder->psb_ip;
2738	decoder->state.to_ip = 0;
2739	return HOP_RETURN;
2740
2741	case INTEL_PT_BAD:
2742	case INTEL_PT_PAD:
2743	case INTEL_PT_TIP_PGE:
2744	case INTEL_PT_TSC:
2745	case INTEL_PT_TMA:
2746	case INTEL_PT_MODE_EXEC:
2747	case INTEL_PT_MODE_TSX:
2748	case INTEL_PT_MTC:
2749	case INTEL_PT_CYC:
2750	case INTEL_PT_VMCS:
2751	case INTEL_PT_PSBEND:
2752	case INTEL_PT_CBR:
2753	case INTEL_PT_TRACESTOP:
2754	case INTEL_PT_PIP:
2755	case INTEL_PT_OVF:
2756	case INTEL_PT_MNT:
2757	case INTEL_PT_PTWRITE:
2758	case INTEL_PT_PTWRITE_IP:
2759	case INTEL_PT_EXSTOP:
2760	case INTEL_PT_EXSTOP_IP:
2761	case INTEL_PT_MWAIT:
2762	case INTEL_PT_PWRE:
2763	case INTEL_PT_PWRX:
2764	case INTEL_PT_BBP:
2765	case INTEL_PT_BIP:
2766	case INTEL_PT_BEP:
2767	case INTEL_PT_BEP_IP:
2768	default:
2769	return HOP_PROCESS;
2770	}
2771	}
2772
2773	struct intel_pt_psb_info {
2774	struct intel_pt_pkt fup_packet;
2775	bool fup;
2776	int after_psbend;
2777	};
2778
2779	/* Lookahead and get the FUP packet from PSB+ */
2780	static int intel_pt_psb_lookahead_cb(struct intel_pt_pkt_info *pkt_info)
2781	{
2782	struct intel_pt_psb_info *data = pkt_info->data;
2783
2784	switch (pkt_info->packet.type) {
2785	case INTEL_PT_PAD:
2786	case INTEL_PT_MNT:
2787	case INTEL_PT_TSC:
2788	case INTEL_PT_TMA:
2789	case INTEL_PT_MODE_EXEC:
2790	case INTEL_PT_MODE_TSX:
2791	case INTEL_PT_MTC:
2792	case INTEL_PT_CYC:
2793	case INTEL_PT_VMCS:
2794	case INTEL_PT_CBR:
2795	case INTEL_PT_PIP:
2796	if (data->after_psbend) {
2797	data->after_psbend -= 1;
2798	if (!data->after_psbend)
2799	return 1;
2800	}
2801	break;
2802
2803	case INTEL_PT_FUP:
2804	if (data->after_psbend)
2805	return 1;
2806	if (data->fup \|\| pkt_info->packet.count == 0)
2807	return 1;
2808	data->fup_packet = pkt_info->packet;
2809	data->fup = true;
2810	break;
2811
2812	case INTEL_PT_PSBEND:
2813	if (!data->fup)
2814	return 1;
2815	/* Keep going to check for a TIP.PGE */
2816	data->after_psbend = 6;
2817	break;
2818
2819	case INTEL_PT_TIP_PGE:
2820	/* Ignore FUP in PSB+ if followed by TIP.PGE */
2821	if (data->after_psbend)
2822	data->fup = false;
2823	return 1;
2824
2825	case INTEL_PT_PTWRITE:
2826	case INTEL_PT_PTWRITE_IP:
2827	case INTEL_PT_EXSTOP:
2828	case INTEL_PT_EXSTOP_IP:
2829	case INTEL_PT_MWAIT:
2830	case INTEL_PT_PWRE:
2831	case INTEL_PT_PWRX:
2832	case INTEL_PT_BBP:
2833	case INTEL_PT_BIP:
2834	case INTEL_PT_BEP:
2835	case INTEL_PT_BEP_IP:
2836	if (data->after_psbend) {
2837	data->after_psbend -= 1;
2838	if (!data->after_psbend)
2839	return 1;
2840	break;
2841	}
2842	return 1;
2843
2844	case INTEL_PT_OVF:
2845	case INTEL_PT_BAD:
2846	case INTEL_PT_TNT:
2847	case INTEL_PT_TIP_PGD:
2848	case INTEL_PT_TIP:
2849	case INTEL_PT_PSB:
2850	case INTEL_PT_TRACESTOP:
2851	default:
2852	return 1;
2853	}
2854
2855	return 0;
2856	}
2857
2858	static int intel_pt_psb(struct intel_pt_decoder *decoder)
2859	{
2860	int err;
2861
2862	decoder->last_ip = 0;
2863	decoder->psb_ip = 0;
2864	decoder->have_last_ip = true;
2865	intel_pt_clear_stack(&decoder->stack);
2866	err = intel_pt_walk_psbend(decoder);
2867	if (err)
2868	return err;
2869	decoder->state.type = INTEL_PT_PSB_EVT;
2870	decoder->state.from_ip = decoder->psb_ip;
2871	decoder->state.to_ip = 0;
2872	return 0;
2873	}
2874
2875	static int intel_pt_fup_in_psb(struct intel_pt_decoder *decoder)
2876	{
2877	int err;
2878
2879	if (decoder->ip != decoder->last_ip) {
2880	err = intel_pt_walk_fup(decoder);
2881	if (!err \|\| err != -EAGAIN)
2882	return err;
2883	}
2884
2885	decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2886	err = intel_pt_psb(decoder);
2887	if (err) {
2888	decoder->pkt_state = INTEL_PT_STATE_ERR3;
2889	return -ENOENT;
2890	}
2891
2892	return 0;
2893	}
2894
2895	static bool intel_pt_psb_with_fup(struct intel_pt_decoder decoder, int err)
2896	{
2897	struct intel_pt_psb_info data = { .fup = false };
2898
2899	if (!decoder->branch_enable \|\| !decoder->pge)
2900	return false;
2901
2902	intel_pt_pkt_lookahead(decoder, intel_pt_psb_lookahead_cb, &data);
2903	if (!data.fup)
2904	return false;
2905
2906	decoder->packet = data.fup_packet;
2907	intel_pt_set_last_ip(decoder);
2908	decoder->pkt_state = INTEL_PT_STATE_FUP_IN_PSB;
2909
2910	*err = intel_pt_fup_in_psb(decoder);
2911
2912	return true;
2913	}
2914
2915	static int intel_pt_walk_trace(struct intel_pt_decoder *decoder)
2916	{
2917	int last_packet_type = INTEL_PT_PAD;
2918	bool no_tip = false;
2919	int err;
2920
2921	while (1) {
2922	err = intel_pt_get_next_packet(decoder);
2923	if (err)
2924	return err;
2925	next:
2926	if (decoder->cyc_threshold) {
2927	if (decoder->sample_cyc && last_packet_type != INTEL_PT_CYC)
2928	decoder->sample_cyc = false;
2929	last_packet_type = decoder->packet.type;
2930	}
2931
2932	if (decoder->hop) {
2933	switch (intel_pt_hop_trace(decoder, &no_tip, &err)) {
2934	case HOP_IGNORE:
2935	continue;
2936	case HOP_RETURN:
2937	return err;
2938	case HOP_AGAIN:
2939	goto next;
2940	default:
2941	break;
2942	}
2943	}
2944
2945	switch (decoder->packet.type) {
2946	case INTEL_PT_TNT:
2947	if (!decoder->packet.count)
2948	break;
2949	decoder->tnt = decoder->packet;
2950	decoder->pkt_state = INTEL_PT_STATE_TNT;
2951	err = intel_pt_walk_tnt(decoder);
2952	if (err == -EAGAIN)
2953	break;
2954	return err;
2955
2956	case INTEL_PT_TIP_PGD:
2957	if (decoder->packet.count != 0)
2958	intel_pt_set_last_ip(decoder);
2959	decoder->pkt_state = INTEL_PT_STATE_TIP_PGD;
2960	return intel_pt_walk_tip(decoder);
2961
2962	case INTEL_PT_TIP_PGE: {
2963	decoder->pge = true;
2964	intel_pt_mtc_cyc_cnt_pge(decoder);
2965	intel_pt_set_nr(decoder);
2966	if (decoder->packet.count == 0) {
2967	intel_pt_log_at("Skipping zero TIP.PGE",
2968	decoder->pos);
2969	break;
2970	}
2971	intel_pt_set_ip(decoder);
2972	decoder->state.from_ip = 0;
2973	decoder->state.to_ip = decoder->ip;
2974	decoder->state.type \|= INTEL_PT_TRACE_BEGIN;
2975	/*
2976	* In hop mode, resample to get the to_ip as an
2977	* "instruction" sample.
2978	*/
2979	if (decoder->hop)
2980	decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;
2981	return 0;
2982	}
2983
2984	case INTEL_PT_OVF:
2985	return intel_pt_overflow(decoder);
2986
2987	case INTEL_PT_TIP:
2988	if (decoder->packet.count != 0)
2989	intel_pt_set_last_ip(decoder);
2990	decoder->pkt_state = INTEL_PT_STATE_TIP;
2991	return intel_pt_walk_tip(decoder);
2992
2993	case INTEL_PT_FUP:
2994	if (decoder->packet.count == 0) {
2995	intel_pt_log_at("Skipping zero FUP",
2996	decoder->pos);
2997	no_tip = false;
2998	break;
2999	}
3000	intel_pt_set_last_ip(decoder);
3001	if (!decoder->branch_enable) {
3002	decoder->ip = decoder->last_ip;
3003	if (intel_pt_fup_event(decoder))
3004	return 0;
3005	no_tip = false;
3006	break;
3007	}
3008	if (decoder->set_fup_mwait)
3009	no_tip = true;
3010	if (no_tip)
3011	decoder->pkt_state = INTEL_PT_STATE_FUP_NO_TIP;
3012	else
3013	decoder->pkt_state = INTEL_PT_STATE_FUP;
3014	err = intel_pt_walk_fup(decoder);
3015	if (err != -EAGAIN)
3016	return err;
3017	if (no_tip) {
3018	no_tip = false;
3019	break;
3020	}
3021	return intel_pt_walk_fup_tip(decoder);
3022
3023	case INTEL_PT_TRACESTOP:
3024	decoder->pge = false;
3025	decoder->continuous_period = false;
3026	intel_pt_clear_tx_flags(decoder);
3027	decoder->have_tma = false;
3028	break;
3029
3030	case INTEL_PT_PSB:
3031	decoder->state.psb_offset = decoder->pos;
3032	decoder->psb_ip = 0;
3033	if (intel_pt_psb_with_fup(decoder, &err))
3034	return err;
3035	err = intel_pt_psb(decoder);
3036	if (err == -EAGAIN)
3037	goto next;
3038	return err;
3039
3040	case INTEL_PT_PIP:
3041	intel_pt_update_pip(decoder);
3042	break;
3043
3044	case INTEL_PT_MTC:
3045	intel_pt_calc_mtc_timestamp(decoder);
3046	if (decoder->period_type != INTEL_PT_PERIOD_MTC)
3047	break;
3048	/*
3049	* Ensure that there has been an instruction since the
3050	* last MTC.
3051	*/
3052	if (!decoder->mtc_insn)
3053	break;
3054	decoder->mtc_insn = false;
3055	/* Ensure that there is a timestamp */
3056	if (!decoder->timestamp)
3057	break;
3058	decoder->state.type = INTEL_PT_INSTRUCTION;
3059	decoder->state.from_ip = decoder->ip;
3060	decoder->state.to_ip = 0;
3061	decoder->mtc_insn = false;
3062	return 0;
3063
3064	case INTEL_PT_TSC:
3065	intel_pt_calc_tsc_timestamp(decoder);
3066	break;
3067
3068	case INTEL_PT_TMA:
3069	intel_pt_calc_tma(decoder);
3070	break;
3071
3072	case INTEL_PT_CYC:
3073	intel_pt_calc_cyc_timestamp(decoder);
3074	break;
3075
3076	case INTEL_PT_CBR:
3077	intel_pt_calc_cbr(decoder);
3078	if (decoder->cbr != decoder->cbr_seen) {
3079	decoder->state.type = 0;
3080	return 0;
3081	}
3082	break;
3083
3084	case INTEL_PT_MODE_EXEC:
3085	decoder->exec_mode = decoder->packet.payload;
3086	break;
3087
3088	case INTEL_PT_MODE_TSX:
3089	/* MODE_TSX need not be followed by FUP */
3090	if (!decoder->pge \|\| decoder->in_psb) {
3091	intel_pt_update_in_tx(decoder);
3092	break;
3093	}
3094	err = intel_pt_mode_tsx(decoder, &no_tip);
3095	if (err)
3096	return err;
3097	goto next;
3098
3099	case INTEL_PT_BAD: /* Does not happen */
3100	return intel_pt_bug(decoder);
3101
3102	case INTEL_PT_PSBEND:
3103	case INTEL_PT_VMCS:
3104	case INTEL_PT_MNT:
3105	case INTEL_PT_PAD:
3106	break;
3107
3108	case INTEL_PT_PTWRITE_IP:
3109	decoder->fup_ptw_payload = decoder->packet.payload;
3110	err = intel_pt_get_next_packet(decoder);
3111	if (err)
3112	return err;
3113	if (decoder->packet.type == INTEL_PT_FUP) {
3114	decoder->set_fup_ptw = true;
3115	no_tip = true;
3116	} else {
3117	intel_pt_log_at("ERROR: Missing FUP after PTWRITE",
3118	decoder->pos);
3119	}
3120	goto next;
3121
3122	case INTEL_PT_PTWRITE:
3123	decoder->state.type = INTEL_PT_PTW;
3124	decoder->state.from_ip = decoder->ip;
3125	decoder->state.to_ip = 0;
3126	decoder->state.ptw_payload = decoder->packet.payload;
3127	return 0;
3128
3129	case INTEL_PT_MWAIT:
3130	decoder->fup_mwait_payload = decoder->packet.payload;
3131	decoder->set_fup_mwait = true;
3132	break;
3133
3134	case INTEL_PT_PWRE:
3135	if (decoder->set_fup_mwait) {
3136	decoder->fup_pwre_payload =
3137	decoder->packet.payload;
3138	decoder->set_fup_pwre = true;
3139	break;
3140	}
3141	decoder->state.type = INTEL_PT_PWR_ENTRY;
3142	decoder->state.from_ip = decoder->ip;
3143	decoder->state.to_ip = 0;
3144	decoder->state.pwrx_payload = decoder->packet.payload;
3145	return 0;
3146
3147	case INTEL_PT_EXSTOP_IP:
3148	err = intel_pt_get_next_packet(decoder);
3149	if (err)
3150	return err;
3151	if (decoder->packet.type == INTEL_PT_FUP) {
3152	decoder->set_fup_exstop = true;
3153	no_tip = true;
3154	} else {
3155	intel_pt_log_at("ERROR: Missing FUP after EXSTOP",
3156	decoder->pos);
3157	}
3158	goto next;
3159
3160	case INTEL_PT_EXSTOP:
3161	decoder->state.type = INTEL_PT_EX_STOP;
3162	decoder->state.from_ip = decoder->ip;
3163	decoder->state.to_ip = 0;
3164	return 0;
3165
3166	case INTEL_PT_PWRX:
3167	decoder->state.type = INTEL_PT_PWR_EXIT;
3168	decoder->state.from_ip = decoder->ip;
3169	decoder->state.to_ip = 0;
3170	decoder->state.pwrx_payload = decoder->packet.payload;
3171	return 0;
3172
3173	case INTEL_PT_BBP:
3174	intel_pt_bbp(decoder);
3175	break;
3176
3177	case INTEL_PT_BIP:
3178	intel_pt_bip(decoder);
3179	break;
3180
3181	case INTEL_PT_BEP:
3182	decoder->state.type = INTEL_PT_BLK_ITEMS;
3183	decoder->state.from_ip = decoder->ip;
3184	decoder->state.to_ip = 0;
3185	return 0;
3186
3187	case INTEL_PT_BEP_IP:
3188	err = intel_pt_get_next_packet(decoder);
3189	if (err)
3190	return err;
3191	if (decoder->packet.type == INTEL_PT_FUP) {
3192	decoder->set_fup_bep = true;
3193	no_tip = true;
3194	} else {
3195	intel_pt_log_at("ERROR: Missing FUP after BEP",
3196	decoder->pos);
3197	}
3198	goto next;
3199
3200	default:
3201	return intel_pt_bug(decoder);
3202	}
3203	}
3204	}
3205
3206	static inline bool intel_pt_have_ip(struct intel_pt_decoder *decoder)
3207	{
3208	return decoder->packet.count &&
3209	(decoder->have_last_ip \|\| decoder->packet.count == 3 \|\|
3210	decoder->packet.count == 6);
3211	}
3212
3213	/* Walk PSB+ packets to get in sync. */
3214	static int intel_pt_walk_psb(struct intel_pt_decoder *decoder)
3215	{
3216	int err;
3217
3218	decoder->in_psb = true;
3219
3220	while (1) {
3221	err = intel_pt_get_next_packet(decoder);
3222	if (err)
3223	goto out;
3224
3225	switch (decoder->packet.type) {
3226	case INTEL_PT_TIP_PGD:
3227	decoder->continuous_period = false;
3228	__fallthrough;
3229	case INTEL_PT_TIP_PGE:
3230	case INTEL_PT_TIP:
3231	case INTEL_PT_PTWRITE:
3232	case INTEL_PT_PTWRITE_IP:
3233	case INTEL_PT_EXSTOP:
3234	case INTEL_PT_EXSTOP_IP:
3235	case INTEL_PT_MWAIT:
3236	case INTEL_PT_PWRE:
3237	case INTEL_PT_PWRX:
3238	case INTEL_PT_BBP:
3239	case INTEL_PT_BIP:
3240	case INTEL_PT_BEP:
3241	case INTEL_PT_BEP_IP:
3242	intel_pt_log("ERROR: Unexpected packet\n");
3243	err = -ENOENT;
3244	goto out;
3245
3246	case INTEL_PT_FUP:
3247	decoder->pge = true;
3248	if (intel_pt_have_ip(decoder)) {
3249	uint64_t current_ip = decoder->ip;
3250
3251	intel_pt_set_ip(decoder);
3252	decoder->psb_ip = decoder->ip;
3253	if (current_ip)
3254	intel_pt_log_to("Setting IP",
3255	decoder->ip);
3256	}
3257	break;
3258
3259	case INTEL_PT_MTC:
3260	intel_pt_calc_mtc_timestamp(decoder);
3261	break;
3262
3263	case INTEL_PT_TSC:
3264	intel_pt_calc_tsc_timestamp(decoder);
3265	break;
3266
3267	case INTEL_PT_TMA:
3268	intel_pt_calc_tma(decoder);
3269	break;
3270
3271	case INTEL_PT_CYC:
3272	intel_pt_calc_cyc_timestamp(decoder);
3273	break;
3274
3275	case INTEL_PT_CBR:
3276	intel_pt_calc_cbr(decoder);
3277	break;
3278
3279	case INTEL_PT_PIP:
3280	intel_pt_set_pip(decoder);
3281	break;
3282
3283	case INTEL_PT_MODE_EXEC:
3284	decoder->exec_mode = decoder->packet.payload;
3285	break;
3286
3287	case INTEL_PT_MODE_TSX:
3288	intel_pt_update_in_tx(decoder);
3289	break;
3290
3291	case INTEL_PT_TRACESTOP:
3292	decoder->pge = false;
3293	decoder->continuous_period = false;
3294	intel_pt_clear_tx_flags(decoder);
3295	__fallthrough;
3296
3297	case INTEL_PT_TNT:
3298	decoder->have_tma = false;
3299	intel_pt_log("ERROR: Unexpected packet\n");
3300	if (decoder->ip)
3301	decoder->pkt_state = INTEL_PT_STATE_ERR4;
3302	else
3303	decoder->pkt_state = INTEL_PT_STATE_ERR3;
3304	err = -ENOENT;
3305	goto out;
3306
3307	case INTEL_PT_BAD: /* Does not happen */
3308	err = intel_pt_bug(decoder);
3309	goto out;
3310
3311	case INTEL_PT_OVF:
3312	err = intel_pt_overflow(decoder);
3313	goto out;
3314
3315	case INTEL_PT_PSBEND:
3316	err = 0;
3317	goto out;
3318
3319	case INTEL_PT_PSB:
3320	case INTEL_PT_VMCS:
3321	case INTEL_PT_MNT:
3322	case INTEL_PT_PAD:
3323	default:
3324	break;
3325	}
3326	}
3327	out:
3328	decoder->in_psb = false;
3329
3330	return err;
3331	}
3332
3333	static int intel_pt_walk_to_ip(struct intel_pt_decoder *decoder)
3334	{
3335	int err;
3336
3337	while (1) {
3338	err = intel_pt_get_next_packet(decoder);
3339	if (err)
3340	return err;
3341
3342	switch (decoder->packet.type) {
3343	case INTEL_PT_TIP_PGD:
3344	decoder->continuous_period = false;
3345	decoder->pge = false;
3346	if (intel_pt_have_ip(decoder))
3347	intel_pt_set_ip(decoder);
3348	if (!decoder->ip)
3349	break;
3350	decoder->state.type \|= INTEL_PT_TRACE_END;
3351	return 0;
3352
3353	case INTEL_PT_TIP_PGE:
3354	decoder->pge = true;
3355	intel_pt_mtc_cyc_cnt_pge(decoder);
3356	if (intel_pt_have_ip(decoder))
3357	intel_pt_set_ip(decoder);
3358	if (!decoder->ip)
3359	break;
3360	decoder->state.type \|= INTEL_PT_TRACE_BEGIN;
3361	return 0;
3362
3363	case INTEL_PT_TIP:
3364	decoder->pge = true;
3365	if (intel_pt_have_ip(decoder))
3366	intel_pt_set_ip(decoder);
3367	if (!decoder->ip)
3368	break;
3369	return 0;
3370
3371	case INTEL_PT_FUP:
3372	if (intel_pt_have_ip(decoder))
3373	intel_pt_set_ip(decoder);
3374	if (decoder->ip)
3375	return 0;
3376	break;
3377
3378	case INTEL_PT_MTC:
3379	intel_pt_calc_mtc_timestamp(decoder);
3380	break;
3381
3382	case INTEL_PT_TSC:
3383	intel_pt_calc_tsc_timestamp(decoder);
3384	break;
3385
3386	case INTEL_PT_TMA:
3387	intel_pt_calc_tma(decoder);
3388	break;
3389
3390	case INTEL_PT_CYC:
3391	intel_pt_calc_cyc_timestamp(decoder);
3392	break;
3393
3394	case INTEL_PT_CBR:
3395	intel_pt_calc_cbr(decoder);
3396	break;
3397
3398	case INTEL_PT_PIP:
3399	intel_pt_set_pip(decoder);
3400	break;
3401
3402	case INTEL_PT_MODE_EXEC:
3403	decoder->exec_mode = decoder->packet.payload;
3404	break;
3405
3406	case INTEL_PT_MODE_TSX:
3407	intel_pt_update_in_tx(decoder);
3408	break;
3409
3410	case INTEL_PT_OVF:
3411	return intel_pt_overflow(decoder);
3412
3413	case INTEL_PT_BAD: /* Does not happen */
3414	return intel_pt_bug(decoder);
3415
3416	case INTEL_PT_TRACESTOP:
3417	decoder->pge = false;
3418	decoder->continuous_period = false;
3419	intel_pt_clear_tx_flags(decoder);
3420	decoder->have_tma = false;
3421	break;
3422
3423	case INTEL_PT_PSB:
3424	decoder->state.psb_offset = decoder->pos;
3425	decoder->psb_ip = 0;
3426	decoder->last_ip = 0;
3427	decoder->have_last_ip = true;
3428	intel_pt_clear_stack(&decoder->stack);
3429	err = intel_pt_walk_psb(decoder);
3430	if (err)
3431	return err;
3432	decoder->state.type = INTEL_PT_PSB_EVT;
3433	decoder->state.from_ip = decoder->psb_ip;
3434	decoder->state.to_ip = 0;
3435	return 0;
3436
3437	case INTEL_PT_TNT:
3438	case INTEL_PT_PSBEND:
3439	case INTEL_PT_VMCS:
3440	case INTEL_PT_MNT:
3441	case INTEL_PT_PAD:
3442	case INTEL_PT_PTWRITE:
3443	case INTEL_PT_PTWRITE_IP:
3444	case INTEL_PT_EXSTOP:
3445	case INTEL_PT_EXSTOP_IP:
3446	case INTEL_PT_MWAIT:
3447	case INTEL_PT_PWRE:
3448	case INTEL_PT_PWRX:
3449	case INTEL_PT_BBP:
3450	case INTEL_PT_BIP:
3451	case INTEL_PT_BEP:
3452	case INTEL_PT_BEP_IP:
3453	default:
3454	break;
3455	}
3456	}
3457	}
3458
3459	static int intel_pt_sync_ip(struct intel_pt_decoder *decoder)
3460	{
3461	int err;
3462
3463	decoder->set_fup_tx_flags = false;
3464	decoder->set_fup_ptw = false;
3465	decoder->set_fup_mwait = false;
3466	decoder->set_fup_pwre = false;
3467	decoder->set_fup_exstop = false;
3468	decoder->set_fup_bep = false;
3469
3470	if (!decoder->branch_enable) {
3471	decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
3472	decoder->overflow = false;
3473	decoder->state.type = 0; /* Do not have a sample */
3474	return 0;
3475	}
3476
3477	intel_pt_log("Scanning for full IP\n");
3478	err = intel_pt_walk_to_ip(decoder);
3479	if (err \|\| ((decoder->state.type & INTEL_PT_PSB_EVT) && !decoder->ip))
3480	return err;
3481
3482	/* In hop mode, resample to get the to_ip as an "instruction" sample */
3483	if (decoder->hop)
3484	decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;
3485	else
3486	decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
3487	decoder->overflow = false;
3488
3489	decoder->state.from_ip = 0;
3490	decoder->state.to_ip = decoder->ip;
3491	intel_pt_log_to("Setting IP", decoder->ip);
3492
3493	return 0;
3494	}
3495
3496	static int intel_pt_part_psb(struct intel_pt_decoder *decoder)
3497	{
3498	const unsigned char *end = decoder->buf + decoder->len;
3499	size_t i;
3500
3501	for (i = INTEL_PT_PSB_LEN - 1; i; i--) {
3502	if (i > decoder->len)
3503	continue;
3504	if (!memcmp(end - i, INTEL_PT_PSB_STR, i))
3505	return i;
3506	}
3507	return 0;
3508	}
3509
3510	static int intel_pt_rest_psb(struct intel_pt_decoder *decoder, int part_psb)
3511	{
3512	size_t rest_psb = INTEL_PT_PSB_LEN - part_psb;
3513	const char *psb = INTEL_PT_PSB_STR;
3514
3515	if (rest_psb > decoder->len \|\|
3516	memcmp(decoder->buf, psb + part_psb, rest_psb))
3517	return 0;
3518
3519	return rest_psb;
3520	}
3521
3522	static int intel_pt_get_split_psb(struct intel_pt_decoder *decoder,
3523	int part_psb)
3524	{
3525	int rest_psb, ret;
3526
3527	decoder->pos += decoder->len;
3528	decoder->len = 0;
3529
3530	ret = intel_pt_get_next_data(decoder, false);
3531	if (ret)
3532	return ret;
3533
3534	rest_psb = intel_pt_rest_psb(decoder, part_psb);
3535	if (!rest_psb)
3536	return 0;
3537
3538	decoder->pos -= part_psb;
3539	decoder->next_buf = decoder->buf + rest_psb;
3540	decoder->next_len = decoder->len - rest_psb;
3541	memcpy(decoder->temp_buf, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
3542	decoder->buf = decoder->temp_buf;
3543	decoder->len = INTEL_PT_PSB_LEN;
3544
3545	return 0;
3546	}
3547
3548	static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder)
3549	{
3550	unsigned char *next;
3551	int ret;
3552
3553	intel_pt_log("Scanning for PSB\n");
3554	while (1) {
3555	if (!decoder->len) {
3556	ret = intel_pt_get_next_data(decoder, false);
3557	if (ret)
3558	return ret;
3559	}
3560
3561	next = memmem(decoder->buf, decoder->len, INTEL_PT_PSB_STR,
3562	INTEL_PT_PSB_LEN);
3563	if (!next) {
3564	int part_psb;
3565
3566	part_psb = intel_pt_part_psb(decoder);
3567	if (part_psb) {
3568	ret = intel_pt_get_split_psb(decoder, part_psb);
3569	if (ret)
3570	return ret;
3571	} else {
3572	decoder->pos += decoder->len;
3573	decoder->len = 0;
3574	}
3575	continue;
3576	}
3577
3578	decoder->pkt_step = next - decoder->buf;
3579	return intel_pt_get_next_packet(decoder);
3580	}
3581	}
3582
3583	static int intel_pt_sync(struct intel_pt_decoder *decoder)
3584	{
3585	int err;
3586
3587	decoder->pge = false;
3588	decoder->continuous_period = false;
3589	decoder->have_last_ip = false;
3590	decoder->last_ip = 0;
3591	decoder->psb_ip = 0;
3592	decoder->ip = 0;
3593	intel_pt_clear_stack(&decoder->stack);
3594
3595	err = intel_pt_scan_for_psb(decoder);
3596	if (err)
3597	return err;
3598
3599	if (decoder->vm_time_correlation) {
3600	decoder->in_psb = true;
3601	if (!decoder->timestamp)
3602	decoder->timestamp = 1;
3603	decoder->state.type = 0;
3604	decoder->pkt_state = INTEL_PT_STATE_VM_TIME_CORRELATION;
3605	return 0;
3606	}
3607
3608	decoder->have_last_ip = true;
3609	decoder->pkt_state = INTEL_PT_STATE_NO_IP;
3610
3611	err = intel_pt_walk_psb(decoder);
3612	if (err)
3613	return err;
3614
3615	decoder->state.type = INTEL_PT_PSB_EVT; /* Only PSB sample */
3616	decoder->state.from_ip = decoder->psb_ip;
3617	decoder->state.to_ip = 0;
3618
3619	if (decoder->ip) {
3620	/*
3621	* In hop mode, resample to get the PSB FUP ip as an
3622	* "instruction" sample.
3623	*/
3624	if (decoder->hop)
3625	decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;
3626	else
3627	decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
3628	}
3629
3630	return 0;
3631	}
3632
3633	static uint64_t intel_pt_est_timestamp(struct intel_pt_decoder *decoder)
3634	{
3635	uint64_t est = decoder->sample_insn_cnt << 1;
3636
3637	if (!decoder->cbr \|\| !decoder->max_non_turbo_ratio)
3638	goto out;
3639
3640	est *= decoder->max_non_turbo_ratio;
3641	est /= decoder->cbr;
3642	out:
3643	return decoder->sample_timestamp + est;
3644	}
3645
3646	const struct intel_pt_state intel_pt_decode(struct intel_pt_decoder decoder)
3647	{
3648	int err;
3649
3650	do {
3651	decoder->state.type = INTEL_PT_BRANCH;
3652	decoder->state.flags = 0;
3653
3654	switch (decoder->pkt_state) {
3655	case INTEL_PT_STATE_NO_PSB:
3656	err = intel_pt_sync(decoder);
3657	break;
3658	case INTEL_PT_STATE_NO_IP:
3659	decoder->have_last_ip = false;
3660	decoder->last_ip = 0;
3661	decoder->ip = 0;
3662	__fallthrough;
3663	case INTEL_PT_STATE_ERR_RESYNC:
3664	err = intel_pt_sync_ip(decoder);
3665	break;
3666	case INTEL_PT_STATE_IN_SYNC:
3667	err = intel_pt_walk_trace(decoder);
3668	break;
3669	case INTEL_PT_STATE_TNT:
3670	case INTEL_PT_STATE_TNT_CONT:
3671	err = intel_pt_walk_tnt(decoder);
3672	if (err == -EAGAIN)
3673	err = intel_pt_walk_trace(decoder);
3674	break;
3675	case INTEL_PT_STATE_TIP:
3676	case INTEL_PT_STATE_TIP_PGD:
3677	err = intel_pt_walk_tip(decoder);
3678	break;
3679	case INTEL_PT_STATE_FUP:
3680	err = intel_pt_walk_fup(decoder);
3681	if (err == -EAGAIN)
3682	err = intel_pt_walk_fup_tip(decoder);
3683	break;
3684	case INTEL_PT_STATE_FUP_NO_TIP:
3685	err = intel_pt_walk_fup(decoder);
3686	if (err == -EAGAIN)
3687	err = intel_pt_walk_trace(decoder);
3688	break;
3689	case INTEL_PT_STATE_FUP_IN_PSB:
3690	err = intel_pt_fup_in_psb(decoder);
3691	break;
3692	case INTEL_PT_STATE_RESAMPLE:
3693	err = intel_pt_resample(decoder);
3694	break;
3695	case INTEL_PT_STATE_VM_TIME_CORRELATION:
3696	err = intel_pt_vm_time_correlation(decoder);
3697	break;
3698	default:
3699	err = intel_pt_bug(decoder);
3700	break;
3701	}
3702	} while (err == -ENOLINK);
3703
3704	if (err) {
3705	decoder->state.err = intel_pt_ext_err(err);
3706	decoder->state.from_ip = decoder->ip;
3707	intel_pt_update_sample_time(decoder);
3708	decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt;
3709	intel_pt_set_nr(decoder);
3710	} else {
3711	decoder->state.err = 0;
3712	if (decoder->cbr != decoder->cbr_seen) {
3713	decoder->cbr_seen = decoder->cbr;
3714	if (!decoder->state.type) {
3715	decoder->state.from_ip = decoder->ip;
3716	decoder->state.to_ip = 0;
3717	}
3718	decoder->state.type \|= INTEL_PT_CBR_CHG;
3719	decoder->state.cbr_payload = decoder->cbr_payload;
3720	decoder->state.cbr = decoder->cbr;
3721	}
3722	if (intel_pt_sample_time(decoder->pkt_state)) {
3723	intel_pt_update_sample_time(decoder);
3724	if (decoder->sample_cyc) {
3725	decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt;
3726	decoder->state.flags \|= INTEL_PT_SAMPLE_IPC;
3727	decoder->sample_cyc = false;
3728	}
3729	}
3730	/*
3731	* When using only TSC/MTC to compute cycles, IPC can be
3732	* sampled as soon as the cycle count changes.
3733	*/
3734	if (!decoder->have_cyc)
3735	decoder->state.flags \|= INTEL_PT_SAMPLE_IPC;
3736	}
3737
3738	/* Let PSB event always have TSC timestamp */
3739	if ((decoder->state.type & INTEL_PT_PSB_EVT) && decoder->tsc_timestamp)
3740	decoder->sample_timestamp = decoder->tsc_timestamp;
3741
3742	decoder->state.from_nr = decoder->nr;
3743	decoder->state.to_nr = decoder->next_nr;
3744	decoder->nr = decoder->next_nr;
3745
3746	decoder->state.timestamp = decoder->sample_timestamp;
3747	decoder->state.est_timestamp = intel_pt_est_timestamp(decoder);
3748	decoder->state.tot_insn_cnt = decoder->tot_insn_cnt;
3749	decoder->state.tot_cyc_cnt = decoder->sample_tot_cyc_cnt;
3750
3751	return &decoder->state;
3752	}
3753
3754	/**
3755	* intel_pt_next_psb - move buffer pointer to the start of the next PSB packet.
3756	* @buf: pointer to buffer pointer
3757	* @len: size of buffer
3758	*
3759	* Updates the buffer pointer to point to the start of the next PSB packet if
3760	* there is one, otherwise the buffer pointer is unchanged. If @buf is updated,
3761	* @len is adjusted accordingly.
3762	*
3763	* Return: %true if a PSB packet is found, %false otherwise.
3764	*/
3765	static bool intel_pt_next_psb(unsigned char *buf, size_t len)
3766	{
3767	unsigned char *next;
3768
3769	next = memmem(buf, len, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
3770	if (next) {
3771	len -= next - buf;
3772	*buf = next;
3773	return true;
3774	}
3775	return false;
3776	}
3777
3778	/**
3779	* intel_pt_step_psb - move buffer pointer to the start of the following PSB
3780	* packet.
3781	* @buf: pointer to buffer pointer
3782	* @len: size of buffer
3783	*
3784	* Updates the buffer pointer to point to the start of the following PSB packet
3785	* (skipping the PSB at @buf itself) if there is one, otherwise the buffer
3786	* pointer is unchanged. If @buf is updated, @len is adjusted accordingly.
3787	*
3788	* Return: %true if a PSB packet is found, %false otherwise.
3789	*/
3790	static bool intel_pt_step_psb(unsigned char *buf, size_t len)
3791	{
3792	unsigned char *next;
3793
3794	if (!*len)
3795	return false;
3796
3797	next = memmem(buf + 1, len - 1, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
3798	if (next) {
3799	len -= next - buf;
3800	*buf = next;
3801	return true;
3802	}
3803	return false;
3804	}
3805
3806	/**
3807	* intel_pt_last_psb - find the last PSB packet in a buffer.
3808	* @buf: buffer
3809	* @len: size of buffer
3810	*
3811	* This function finds the last PSB in a buffer.
3812	*
3813	* Return: A pointer to the last PSB in @buf if found, %NULL otherwise.
3814	*/
3815	static unsigned char intel_pt_last_psb(unsigned char buf, size_t len)
3816	{
3817	const char *n = INTEL_PT_PSB_STR;
3818	unsigned char *p;
3819	size_t k;
3820
3821	if (len < INTEL_PT_PSB_LEN)
3822	return NULL;
3823
3824	k = len - INTEL_PT_PSB_LEN + 1;
3825	while (1) {
3826	p = memrchr(buf, n[0], k);
3827	if (!p)
3828	return NULL;
3829	if (!memcmp(p + 1, n + 1, INTEL_PT_PSB_LEN - 1))
3830	return p;
3831	k = p - buf;
3832	if (!k)
3833	return NULL;
3834	}
3835	}
3836
3837	/**
3838	* intel_pt_next_tsc - find and return next TSC.
3839	* @buf: buffer
3840	* @len: size of buffer
3841	* @tsc: TSC value returned
3842	* @rem: returns remaining size when TSC is found
3843	*
3844	* Find a TSC packet in @buf and return the TSC value. This function assumes
3845	* that @buf starts at a PSB and that PSB+ will contain TSC and so stops if a
3846	* PSBEND packet is found.
3847	*
3848	* Return: %true if TSC is found, false otherwise.
3849	*/
3850	static bool intel_pt_next_tsc(unsigned char buf, size_t len, uint64_t tsc,
3851	size_t *rem)
3852	{
3853	enum intel_pt_pkt_ctx ctx = INTEL_PT_NO_CTX;
3854	struct intel_pt_pkt packet;
3855	int ret;
3856
3857	while (len) {
3858	ret = intel_pt_get_packet(buf, len, &packet, &ctx);
3859	if (ret <= 0)
3860	return false;
3861	if (packet.type == INTEL_PT_TSC) {
3862	*tsc = packet.payload;
3863	*rem = len;
3864	return true;
3865	}
3866	if (packet.type == INTEL_PT_PSBEND)
3867	return false;
3868	buf += ret;
3869	len -= ret;
3870	}
3871	return false;
3872	}
3873
3874	/**
3875	* intel_pt_tsc_cmp - compare 7-byte TSCs.
3876	* @tsc1: first TSC to compare
3877	* @tsc2: second TSC to compare
3878	*
3879	* This function compares 7-byte TSC values allowing for the possibility that
3880	* TSC wrapped around. Generally it is not possible to know if TSC has wrapped
3881	* around so for that purpose this function assumes the absolute difference is
3882	* less than half the maximum difference.
3883	*
3884	* Return: %-1 if @tsc1 is before @tsc2, %0 if @tsc1 == @tsc2, %1 if @tsc1 is
3885	* after @tsc2.
3886	*/
3887	static int intel_pt_tsc_cmp(uint64_t tsc1, uint64_t tsc2)
3888	{
3889	const uint64_t halfway = (1ULL << 55);
3890
3891	if (tsc1 == tsc2)
3892	return 0;
3893
3894	if (tsc1 < tsc2) {
3895	if (tsc2 - tsc1 < halfway)
3896	return -1;
3897	else
3898	return 1;
3899	} else {
3900	if (tsc1 - tsc2 < halfway)
3901	return 1;
3902	else
3903	return -1;
3904	}
3905	}
3906
3907	#define MAX_PADDING (PERF_AUXTRACE_RECORD_ALIGNMENT - 1)
3908
3909	/**
3910	* adj_for_padding - adjust overlap to account for padding.
3911	* @buf_b: second buffer
3912	* @buf_a: first buffer
3913	* @len_a: size of first buffer
3914	*
3915	* @buf_a might have up to 7 bytes of padding appended. Adjust the overlap
3916	* accordingly.
3917	*
3918	* Return: A pointer into @buf_b from where non-overlapped data starts
3919	*/
3920	static unsigned char adj_for_padding(unsigned char buf_b,
3921	unsigned char *buf_a, size_t len_a)
3922	{
3923	unsigned char *p = buf_b - MAX_PADDING;
3924	unsigned char *q = buf_a + len_a - MAX_PADDING;
3925	int i;
3926
3927	for (i = MAX_PADDING; i; i--, p++, q++) {
3928	if (p != q)
3929	break;
3930	}
3931
3932	return p;
3933	}
3934
3935	/**
3936	* intel_pt_find_overlap_tsc - determine start of non-overlapped trace data
3937	* using TSC.
3938	* @buf_a: first buffer
3939	* @len_a: size of first buffer
3940	* @buf_b: second buffer
3941	* @len_b: size of second buffer
3942	* @consecutive: returns true if there is data in buf_b that is consecutive
3943	* to buf_a
3944	* @ooo_tsc: out-of-order TSC due to VM TSC offset / scaling
3945	*
3946	* If the trace contains TSC we can look at the last TSC of @buf_a and the
3947	* first TSC of @buf_b in order to determine if the buffers overlap, and then
3948	* walk forward in @buf_b until a later TSC is found. A precondition is that
3949	* @buf_a and @buf_b are positioned at a PSB.
3950	*
3951	* Return: A pointer into @buf_b from where non-overlapped data starts, or
3952	* @buf_b + @len_b if there is no non-overlapped data.
3953	*/
3954	static unsigned char intel_pt_find_overlap_tsc(unsigned char buf_a,
3955	size_t len_a,
3956	unsigned char *buf_b,
3957	size_t len_b, bool *consecutive,
3958	bool ooo_tsc)
3959	{
3960	uint64_t tsc_a, tsc_b;
3961	unsigned char *p;
3962	size_t len, rem_a, rem_b;
3963
3964	p = intel_pt_last_psb(buf_a, len_a);
3965	if (!p)
3966	return buf_b; /* No PSB in buf_a => no overlap */
3967
3968	len = len_a - (p - buf_a);
3969	if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a)) {
3970	/* The last PSB+ in buf_a is incomplete, so go back one more */
3971	len_a -= len;
3972	p = intel_pt_last_psb(buf_a, len_a);
3973	if (!p)
3974	return buf_b; /* No full PSB+ => assume no overlap */
3975	len = len_a - (p - buf_a);
3976	if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a))
3977	return buf_b; /* No TSC in buf_a => assume no overlap */
3978	}
3979
3980	while (1) {
3981	/* Ignore PSB+ with no TSC */
3982	if (intel_pt_next_tsc(buf_b, len_b, &tsc_b, &rem_b)) {
3983	int cmp = intel_pt_tsc_cmp(tsc_a, tsc_b);
3984
3985	/* Same TSC, so buffers are consecutive */
3986	if (!cmp && rem_b >= rem_a) {
3987	unsigned char *start;
3988
3989	*consecutive = true;
3990	start = buf_b + len_b - (rem_b - rem_a);
3991	return adj_for_padding(start, buf_a, len_a);
3992	}
3993	if (cmp < 0 && !ooo_tsc)
3994	return buf_b; /* tsc_a < tsc_b => no overlap */
3995	}
3996
3997	if (!intel_pt_step_psb(&buf_b, &len_b))
3998	return buf_b + len_b; /* No PSB in buf_b => no data */
3999	}
4000	}
4001
4002	/**
4003	* intel_pt_find_overlap - determine start of non-overlapped trace data.
4004	* @buf_a: first buffer
4005	* @len_a: size of first buffer
4006	* @buf_b: second buffer
4007	* @len_b: size of second buffer
4008	* @have_tsc: can use TSC packets to detect overlap
4009	* @consecutive: returns true if there is data in buf_b that is consecutive
4010	* to buf_a
4011	* @ooo_tsc: out-of-order TSC due to VM TSC offset / scaling
4012	*
4013	* When trace samples or snapshots are recorded there is the possibility that
4014	* the data overlaps. Note that, for the purposes of decoding, data is only
4015	* useful if it begins with a PSB packet.
4016	*
4017	* Return: A pointer into @buf_b from where non-overlapped data starts, or
4018	* @buf_b + @len_b if there is no non-overlapped data.
4019	*/
4020	unsigned char intel_pt_find_overlap(unsigned char buf_a, size_t len_a,
4021	unsigned char *buf_b, size_t len_b,
4022	bool have_tsc, bool *consecutive,
4023	bool ooo_tsc)
4024	{
4025	unsigned char *found;
4026
4027	/* Buffer 'b' must start at PSB so throw away everything before that */
4028	if (!intel_pt_next_psb(&buf_b, &len_b))
4029	return buf_b + len_b; /* No PSB */
4030
4031	if (!intel_pt_next_psb(&buf_a, &len_a))
4032	return buf_b; /* No overlap */
4033
4034	if (have_tsc) {
4035	found = intel_pt_find_overlap_tsc(buf_a, len_a, buf_b, len_b,
4036	consecutive, ooo_tsc);
4037	if (found)
4038	return found;
4039	}
4040
4041	/*
4042	* Buffer 'b' cannot end within buffer 'a' so, for comparison purposes,
4043	* we can ignore the first part of buffer 'a'.
4044	*/
4045	while (len_b < len_a) {
4046	if (!intel_pt_step_psb(&buf_a, &len_a))
4047	return buf_b; /* No overlap */
4048	}
4049
4050	/* Now len_b >= len_a */
4051	while (1) {
4052	/* Potential overlap so check the bytes */
4053	found = memmem(buf_a, len_a, buf_b, len_a);
4054	if (found) {
4055	*consecutive = true;
4056	return adj_for_padding(buf_b + len_a, buf_a, len_a);
4057	}
4058
4059	/* Try again at next PSB in buffer 'a' */
4060	if (!intel_pt_step_psb(&buf_a, &len_a))
4061	return buf_b; /* No overlap */
4062	}
4063	}
4064
4065	/**
4066	* struct fast_forward_data - data used by intel_pt_ff_cb().
4067	* @timestamp: timestamp to fast forward towards
4068	* @buf_timestamp: buffer timestamp of last buffer with trace data earlier than
4069	* the fast forward timestamp.
4070	*/
4071	struct fast_forward_data {
4072	uint64_t timestamp;
4073	uint64_t buf_timestamp;
4074	};
4075
4076	/**
4077	* intel_pt_ff_cb - fast forward lookahead callback.
4078	* @buffer: Intel PT trace buffer
4079	* @data: opaque pointer to fast forward data (struct fast_forward_data)
4080	*
4081	* Determine if @buffer trace is past the fast forward timestamp.
4082	*
4083	* Return: 1 (stop lookahead) if @buffer trace is past the fast forward
4084	* timestamp, and 0 otherwise.
4085	*/
4086	static int intel_pt_ff_cb(struct intel_pt_buffer buffer, void data)
4087	{
4088	struct fast_forward_data *d = data;
4089	unsigned char *buf;
4090	uint64_t tsc;
4091	size_t rem;
4092	size_t len;
4093
4094	buf = (unsigned char *)buffer->buf;
4095	len = buffer->len;
4096
4097	if (!intel_pt_next_psb(&buf, &len) \|\|
4098	!intel_pt_next_tsc(buf, len, &tsc, &rem))
4099	return 0;
4100
4101	tsc = intel_pt_8b_tsc(tsc, buffer->ref_timestamp);
4102
4103	intel_pt_log("Buffer 1st timestamp " x64_fmt " ref timestamp " x64_fmt "\n",
4104	tsc, buffer->ref_timestamp);
4105
4106	/*
4107	* If the buffer contains a timestamp earlier that the fast forward
4108	* timestamp, then record it, else stop.
4109	*/
4110	if (tsc < d->timestamp)
4111	d->buf_timestamp = buffer->ref_timestamp;
4112	else
4113	return 1;
4114
4115	return 0;
4116	}
4117
4118	/**
4119	* intel_pt_fast_forward - reposition decoder forwards.
4120	* @decoder: Intel PT decoder
4121	* @timestamp: timestamp to fast forward towards
4122	*
4123	* Reposition decoder at the last PSB with a timestamp earlier than @timestamp.
4124	*
4125	* Return: 0 on success or negative error code on failure.
4126	*/
4127	int intel_pt_fast_forward(struct intel_pt_decoder *decoder, uint64_t timestamp)
4128	{
4129	struct fast_forward_data d = { .timestamp = timestamp };
4130	unsigned char *buf;
4131	size_t len;
4132	int err;
4133
4134	intel_pt_log("Fast forward towards timestamp " x64_fmt "\n", timestamp);
4135
4136	/* Find buffer timestamp of buffer to fast forward to */
4137	err = decoder->lookahead(decoder->data, intel_pt_ff_cb, &d);
4138	if (err < 0)
4139	return err;
4140
4141	/* Walk to buffer with same buffer timestamp */
4142	if (d.buf_timestamp) {
4143	do {
4144	decoder->pos += decoder->len;
4145	decoder->len = 0;
4146	err = intel_pt_get_next_data(decoder, true);
4147	/* -ENOLINK means non-consecutive trace */
4148	if (err && err != -ENOLINK)
4149	return err;
4150	} while (decoder->buf_timestamp != d.buf_timestamp);
4151	}
4152
4153	if (!decoder->buf)
4154	return 0;
4155
4156	buf = (unsigned char *)decoder->buf;
4157	len = decoder->len;
4158
4159	if (!intel_pt_next_psb(&buf, &len))
4160	return 0;
4161
4162	/*
4163	* Walk PSBs while the PSB timestamp is less than the fast forward
4164	* timestamp.
4165	*/
4166	do {
4167	uint64_t tsc;
4168	size_t rem;
4169
4170	if (!intel_pt_next_tsc(buf, len, &tsc, &rem))
4171	break;
4172	tsc = intel_pt_8b_tsc(tsc, decoder->buf_timestamp);
4173	/*
4174	* A TSC packet can slip past MTC packets but, after fast
4175	* forward, decoding starts at the TSC timestamp. That means
4176	* the timestamps may not be exactly the same as the timestamps
4177	* that would have been decoded without fast forward.
4178	*/
4179	if (tsc < timestamp) {
4180	intel_pt_log("Fast forward to next PSB timestamp " x64_fmt "\n", tsc);
4181	decoder->pos += decoder->len - len;
4182	decoder->buf = buf;
4183	decoder->len = len;
4184	intel_pt_reposition(decoder);
4185	} else {
4186	break;
4187	}
4188	} while (intel_pt_step_psb(&buf, &len));
4189
4190	return 0;
4191	}