linux - tools/perf/util/intel-pt-decoder/intel-pt-decoder.c

4515 lines · 111.2 KiB

1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* intel_pt_decoder.c: Intel Processor Trace support
4	* Copyright (c) 2013-2014, Intel Corporation.
5	*/
6
7	#ifndef _GNU_SOURCE
8	#define _GNU_SOURCE
9	#endif
10	#include <stdlib.h>
11	#include <stdbool.h>
12	#include <string.h>
13	#include <errno.h>
14	#include <stdint.h>
15	#include <inttypes.h>
16	#include <linux/compiler.h>
17	#include <linux/string.h>
18	#include <linux/zalloc.h>
19
20	#include "../auxtrace.h"
21
22	#include "intel-pt-insn-decoder.h"
23	#include "intel-pt-pkt-decoder.h"
24	#include "intel-pt-decoder.h"
25	#include "intel-pt-log.h"
26
27	#define BITULL(x) (1ULL << (x))
28
29	/* IA32_RTIT_CTL MSR bits */
30	#define INTEL_PT_CYC_ENABLE BITULL(1)
31	#define INTEL_PT_CYC_THRESHOLD (BITULL(22) \| BITULL(21) \| BITULL(20) \| BITULL(19))
32	#define INTEL_PT_CYC_THRESHOLD_SHIFT 19
33
34	#define INTEL_PT_BLK_SIZE 1024
35
36	#define BIT63 (((uint64_t)1 << 63))
37
38	#define SEVEN_BYTES 0xffffffffffffffULL
39
40	#define NO_VMCS 0xffffffffffULL
41
42	#define INTEL_PT_RETURN 1
43
44	/*
45	* Default maximum number of loops with no packets consumed i.e. stuck in a
46	* loop.
47	*/
48	#define INTEL_PT_MAX_LOOPS 100000
49
50	struct intel_pt_blk {
51	struct intel_pt_blk *prev;
52	uint64_t ip[INTEL_PT_BLK_SIZE];
53	};
54
55	struct intel_pt_stack {
56	struct intel_pt_blk *blk;
57	struct intel_pt_blk *spare;
58	int pos;
59	};
60
61	enum intel_pt_p_once {
62	INTEL_PT_PRT_ONCE_UNK_VMCS,
63	INTEL_PT_PRT_ONCE_ERANGE,
64	};
65
66	enum intel_pt_pkt_state {
67	INTEL_PT_STATE_NO_PSB,
68	INTEL_PT_STATE_NO_IP,
69	INTEL_PT_STATE_ERR_RESYNC,
70	INTEL_PT_STATE_IN_SYNC,
71	INTEL_PT_STATE_TNT_CONT,
72	INTEL_PT_STATE_TNT,
73	INTEL_PT_STATE_TIP,
74	INTEL_PT_STATE_TIP_PGD,
75	INTEL_PT_STATE_FUP,
76	INTEL_PT_STATE_FUP_NO_TIP,
77	INTEL_PT_STATE_FUP_IN_PSB,
78	INTEL_PT_STATE_RESAMPLE,
79	INTEL_PT_STATE_VM_TIME_CORRELATION,
80	};
81
82	static inline bool intel_pt_sample_time(enum intel_pt_pkt_state pkt_state)
83	{
84	switch (pkt_state) {
85	case INTEL_PT_STATE_NO_PSB:
86	case INTEL_PT_STATE_NO_IP:
87	case INTEL_PT_STATE_ERR_RESYNC:
88	case INTEL_PT_STATE_IN_SYNC:
89	case INTEL_PT_STATE_TNT_CONT:
90	case INTEL_PT_STATE_RESAMPLE:
91	case INTEL_PT_STATE_VM_TIME_CORRELATION:
92	return true;
93	case INTEL_PT_STATE_TNT:
94	case INTEL_PT_STATE_TIP:
95	case INTEL_PT_STATE_TIP_PGD:
96	case INTEL_PT_STATE_FUP:
97	case INTEL_PT_STATE_FUP_NO_TIP:
98	case INTEL_PT_STATE_FUP_IN_PSB:
99	return false;
100	default:
101	return true;
102	};
103	}
104
105	#ifdef INTEL_PT_STRICT
106	#define INTEL_PT_STATE_ERR1 INTEL_PT_STATE_NO_PSB
107	#define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_PSB
108	#define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_NO_PSB
109	#define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_NO_PSB
110	#else
111	#define INTEL_PT_STATE_ERR1 (decoder->pkt_state)
112	#define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_IP
113	#define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_ERR_RESYNC
114	#define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_IN_SYNC
115	#endif
116
117	struct intel_pt_decoder {
118	int (get_trace)(struct intel_pt_buffer buffer, void *data);
119	int (walk_insn)(struct intel_pt_insn intel_pt_insn,
120	uint64_t insn_cnt_ptr, uint64_t ip, uint64_t to_ip,
121	uint64_t max_insn_cnt, void *data);
122	bool (pgd_ip)(uint64_t ip, void data);
123	int (lookahead)(void data, intel_pt_lookahead_cb_t cb, void *cb_data);
124	struct intel_pt_vmcs_info (findnew_vmcs_info)(void *data, uint64_t vmcs);
125	void *data;
126	struct intel_pt_state state;
127	const unsigned char *buf;
128	size_t len;
129	bool return_compression;
130	bool branch_enable;
131	bool mtc_insn;
132	bool pge;
133	bool have_tma;
134	bool have_cyc;
135	bool fixup_last_mtc;
136	bool have_last_ip;
137	bool in_psb;
138	bool hop;
139	bool leap;
140	bool emulated_ptwrite;
141	bool vm_time_correlation;
142	bool vm_tm_corr_dry_run;
143	bool vm_tm_corr_reliable;
144	bool vm_tm_corr_same_buf;
145	bool vm_tm_corr_continuous;
146	bool nr;
147	bool next_nr;
148	bool iflag;
149	bool next_iflag;
150	enum intel_pt_param_flags flags;
151	uint64_t pos;
152	uint64_t last_ip;
153	uint64_t ip;
154	uint64_t pip_payload;
155	uint64_t timestamp;
156	uint64_t tsc_timestamp;
157	uint64_t ref_timestamp;
158	uint64_t buf_timestamp;
159	uint64_t sample_timestamp;
160	uint64_t ret_addr;
161	uint64_t ctc_timestamp;
162	uint64_t ctc_delta;
163	uint64_t cycle_cnt;
164	uint64_t cyc_ref_timestamp;
165	uint64_t first_timestamp;
166	uint64_t last_reliable_timestamp;
167	uint64_t vmcs;
168	uint64_t print_once;
169	uint64_t last_ctc;
170	uint32_t last_mtc;
171	uint32_t tsc_ctc_ratio_n;
172	uint32_t tsc_ctc_ratio_d;
173	uint32_t tsc_ctc_mult;
174	uint32_t tsc_slip;
175	uint32_t ctc_rem_mask;
176	int mtc_shift;
177	struct intel_pt_stack stack;
178	enum intel_pt_pkt_state pkt_state;
179	enum intel_pt_pkt_ctx pkt_ctx;
180	enum intel_pt_pkt_ctx prev_pkt_ctx;
181	enum intel_pt_blk_type blk_type;
182	int blk_type_pos;
183	struct intel_pt_pkt packet;
184	struct intel_pt_pkt tnt;
185	int pkt_step;
186	int pkt_len;
187	int last_packet_type;
188	unsigned int cbr;
189	unsigned int cbr_seen;
190	unsigned int max_non_turbo_ratio;
191	double max_non_turbo_ratio_fp;
192	double cbr_cyc_to_tsc;
193	double calc_cyc_to_tsc;
194	bool have_calc_cyc_to_tsc;
195	int exec_mode;
196	unsigned int insn_bytes;
197	uint64_t period;
198	enum intel_pt_period_type period_type;
199	uint64_t tot_insn_cnt;
200	uint64_t period_insn_cnt;
201	uint64_t period_mask;
202	uint64_t period_ticks;
203	uint64_t last_masked_timestamp;
204	uint64_t tot_cyc_cnt;
205	uint64_t sample_tot_cyc_cnt;
206	uint64_t base_cyc_cnt;
207	uint64_t cyc_cnt_timestamp;
208	uint64_t ctl;
209	uint64_t cyc_threshold;
210	double tsc_to_cyc;
211	bool continuous_period;
212	bool overflow;
213	bool set_fup_tx_flags;
214	bool set_fup_ptw;
215	bool set_fup_mwait;
216	bool set_fup_pwre;
217	bool set_fup_exstop;
218	bool set_fup_bep;
219	bool set_fup_cfe_ip;
220	bool set_fup_cfe;
221	bool set_fup_mode_exec;
222	bool sample_cyc;
223	unsigned int fup_tx_flags;
224	unsigned int tx_flags;
225	uint64_t fup_ptw_payload;
226	uint64_t fup_mwait_payload;
227	uint64_t fup_pwre_payload;
228	uint64_t cbr_payload;
229	uint64_t timestamp_insn_cnt;
230	uint64_t sample_insn_cnt;
231	uint64_t stuck_ip;
232	struct intel_pt_pkt fup_cfe_pkt;
233	int max_loops;
234	int no_progress;
235	int stuck_ip_prd;
236	int stuck_ip_cnt;
237	uint64_t psb_ip;
238	const unsigned char *next_buf;
239	size_t next_len;
240	unsigned char temp_buf[INTEL_PT_PKT_MAX_SZ];
241	int evd_cnt;
242	struct intel_pt_evd evd[INTEL_PT_MAX_EVDS];
243	};
244
245	static uint64_t intel_pt_lower_power_of_2(uint64_t x)
246	{
247	int i;
248
249	for (i = 0; x != 1; i++)
250	x >>= 1;
251
252	return x << i;
253	}
254
255	__printf(1, 2)
256	static void p_log(const char *fmt, ...)
257	{
258	char buf[512];
259	va_list args;
260
261	va_start(args, fmt);
262	vsnprintf(buf, sizeof(buf), fmt, args);
263	va_end(args);
264
265	fprintf(stderr, "%s\n", buf);
266	intel_pt_log("%s\n", buf);
267	}
268
269	static bool intel_pt_print_once(struct intel_pt_decoder *decoder,
270	enum intel_pt_p_once id)
271	{
272	uint64_t bit = 1ULL << id;
273
274	if (decoder->print_once & bit)
275	return false;
276	decoder->print_once \|= bit;
277	return true;
278	}
279
280	static uint64_t intel_pt_cyc_threshold(uint64_t ctl)
281	{
282	if (!(ctl & INTEL_PT_CYC_ENABLE))
283	return 0;
284
285	return (ctl & INTEL_PT_CYC_THRESHOLD) >> INTEL_PT_CYC_THRESHOLD_SHIFT;
286	}
287
288	static void intel_pt_setup_period(struct intel_pt_decoder *decoder)
289	{
290	if (decoder->period_type == INTEL_PT_PERIOD_TICKS) {
291	uint64_t period;
292
293	period = intel_pt_lower_power_of_2(decoder->period);
294	decoder->period_mask = ~(period - 1);
295	decoder->period_ticks = period;
296	}
297	}
298
299	static uint64_t multdiv(uint64_t t, uint32_t n, uint32_t d)
300	{
301	if (!d)
302	return 0;
303	return (t / d) * n + ((t % d) * n) / d;
304	}
305
306	struct intel_pt_decoder intel_pt_decoder_new(struct intel_pt_params params)
307	{
308	struct intel_pt_decoder *decoder;
309
310	if (!params->get_trace \|\| !params->walk_insn)
311	return NULL;
312
313	decoder = zalloc(sizeof(struct intel_pt_decoder));
314	if (!decoder)
315	return NULL;
316
317	decoder->get_trace = params->get_trace;
318	decoder->walk_insn = params->walk_insn;
319	decoder->pgd_ip = params->pgd_ip;
320	decoder->lookahead = params->lookahead;
321	decoder->findnew_vmcs_info = params->findnew_vmcs_info;
322	decoder->data = params->data;
323	decoder->return_compression = params->return_compression;
324	decoder->branch_enable = params->branch_enable;
325	decoder->hop = params->quick >= 1;
326	decoder->leap = params->quick >= 2;
327	decoder->vm_time_correlation = params->vm_time_correlation;
328	decoder->vm_tm_corr_dry_run = params->vm_tm_corr_dry_run;
329	decoder->first_timestamp = params->first_timestamp;
330	decoder->last_reliable_timestamp = params->first_timestamp;
331	decoder->max_loops = params->max_loops ? params->max_loops : INTEL_PT_MAX_LOOPS;
332
333	decoder->flags = params->flags;
334
335	decoder->ctl = params->ctl;
336	decoder->period = params->period;
337	decoder->period_type = params->period_type;
338
339	decoder->max_non_turbo_ratio = params->max_non_turbo_ratio;
340	decoder->max_non_turbo_ratio_fp = params->max_non_turbo_ratio;
341
342	decoder->cyc_threshold = intel_pt_cyc_threshold(decoder->ctl);
343
344	intel_pt_setup_period(decoder);
345
346	decoder->mtc_shift = params->mtc_period;
347	decoder->ctc_rem_mask = (1 << decoder->mtc_shift) - 1;
348
349	decoder->tsc_ctc_ratio_n = params->tsc_ctc_ratio_n;
350	decoder->tsc_ctc_ratio_d = params->tsc_ctc_ratio_d;
351
352	if (!decoder->tsc_ctc_ratio_n)
353	decoder->tsc_ctc_ratio_d = 0;
354
355	if (decoder->tsc_ctc_ratio_d) {
356	if (!(decoder->tsc_ctc_ratio_n % decoder->tsc_ctc_ratio_d))
357	decoder->tsc_ctc_mult = decoder->tsc_ctc_ratio_n /
358	decoder->tsc_ctc_ratio_d;
359	}
360
361	/*
362	* A TSC packet can slip past MTC packets so that the timestamp appears
363	* to go backwards. One estimate is that can be up to about 40 CPU
364	* cycles, which is certainly less than 0x1000 TSC ticks, but accept
365	* slippage an order of magnitude more to be on the safe side.
366	*/
367	decoder->tsc_slip = 0x10000;
368
369	intel_pt_log("timestamp: mtc_shift %u\n", decoder->mtc_shift);
370	intel_pt_log("timestamp: tsc_ctc_ratio_n %u\n", decoder->tsc_ctc_ratio_n);
371	intel_pt_log("timestamp: tsc_ctc_ratio_d %u\n", decoder->tsc_ctc_ratio_d);
372	intel_pt_log("timestamp: tsc_ctc_mult %u\n", decoder->tsc_ctc_mult);
373	intel_pt_log("timestamp: tsc_slip %#x\n", decoder->tsc_slip);
374
375	if (decoder->hop)
376	intel_pt_log("Hop mode: decoding FUP and TIPs, but not TNT\n");
377
378	return decoder;
379	}
380
381	void intel_pt_set_first_timestamp(struct intel_pt_decoder *decoder,
382	uint64_t first_timestamp)
383	{
384	decoder->first_timestamp = first_timestamp;
385	}
386
387	static void intel_pt_pop_blk(struct intel_pt_stack *stack)
388	{
389	struct intel_pt_blk *blk = stack->blk;
390
391	stack->blk = blk->prev;
392	if (!stack->spare)
393	stack->spare = blk;
394	else
395	free(blk);
396	}
397
398	static uint64_t intel_pt_pop(struct intel_pt_stack *stack)
399	{
400	if (!stack->pos) {
401	if (!stack->blk)
402	return 0;
403	intel_pt_pop_blk(stack);
404	if (!stack->blk)
405	return 0;
406	stack->pos = INTEL_PT_BLK_SIZE;
407	}
408	return stack->blk->ip[--stack->pos];
409	}
410
411	static int intel_pt_alloc_blk(struct intel_pt_stack *stack)
412	{
413	struct intel_pt_blk *blk;
414
415	if (stack->spare) {
416	blk = stack->spare;
417	stack->spare = NULL;
418	} else {
419	blk = malloc(sizeof(struct intel_pt_blk));
420	if (!blk)
421	return -ENOMEM;
422	}
423
424	blk->prev = stack->blk;
425	stack->blk = blk;
426	stack->pos = 0;
427	return 0;
428	}
429
430	static int intel_pt_push(struct intel_pt_stack *stack, uint64_t ip)
431	{
432	int err;
433
434	if (!stack->blk \|\| stack->pos == INTEL_PT_BLK_SIZE) {
435	err = intel_pt_alloc_blk(stack);
436	if (err)
437	return err;
438	}
439
440	stack->blk->ip[stack->pos++] = ip;
441	return 0;
442	}
443
444	static void intel_pt_clear_stack(struct intel_pt_stack *stack)
445	{
446	while (stack->blk)
447	intel_pt_pop_blk(stack);
448	stack->pos = 0;
449	}
450
451	static void intel_pt_free_stack(struct intel_pt_stack *stack)
452	{
453	intel_pt_clear_stack(stack);
454	zfree(&stack->blk);
455	zfree(&stack->spare);
456	}
457
458	void intel_pt_decoder_free(struct intel_pt_decoder *decoder)
459	{
460	intel_pt_free_stack(&decoder->stack);
461	free(decoder);
462	}
463
464	static int intel_pt_ext_err(int code)
465	{
466	switch (code) {
467	case -ENOMEM:
468	return INTEL_PT_ERR_NOMEM;
469	case -ENOSYS:
470	return INTEL_PT_ERR_INTERN;
471	case -EBADMSG:
472	return INTEL_PT_ERR_BADPKT;
473	case -ENODATA:
474	return INTEL_PT_ERR_NODATA;
475	case -EILSEQ:
476	return INTEL_PT_ERR_NOINSN;
477	case -ENOENT:
478	return INTEL_PT_ERR_MISMAT;
479	case -EOVERFLOW:
480	return INTEL_PT_ERR_OVR;
481	case -ENOSPC:
482	return INTEL_PT_ERR_LOST;
483	case -ELOOP:
484	return INTEL_PT_ERR_NELOOP;
485	case -ECONNRESET:
486	return INTEL_PT_ERR_EPTW;
487	default:
488	return INTEL_PT_ERR_UNK;
489	}
490	}
491
492	static const char *intel_pt_err_msgs[] = {
493	[INTEL_PT_ERR_NOMEM] = "Memory allocation failed",
494	[INTEL_PT_ERR_INTERN] = "Internal error",
495	[INTEL_PT_ERR_BADPKT] = "Bad packet",
496	[INTEL_PT_ERR_NODATA] = "No more data",
497	[INTEL_PT_ERR_NOINSN] = "Failed to get instruction",
498	[INTEL_PT_ERR_MISMAT] = "Trace doesn't match instruction",
499	[INTEL_PT_ERR_OVR] = "Overflow packet",
500	[INTEL_PT_ERR_LOST] = "Lost trace data",
501	[INTEL_PT_ERR_UNK] = "Unknown error!",
502	[INTEL_PT_ERR_NELOOP] = "Never-ending loop (refer perf config intel-pt.max-loops)",
503	[INTEL_PT_ERR_EPTW] = "Broken emulated ptwrite",
504	};
505
506	int intel_pt__strerror(int code, char *buf, size_t buflen)
507	{
508	if (code < 1 \|\| code >= INTEL_PT_ERR_MAX)
509	code = INTEL_PT_ERR_UNK;
510	strlcpy(buf, intel_pt_err_msgs[code], buflen);
511	return 0;
512	}
513
514	static uint64_t intel_pt_calc_ip(const struct intel_pt_pkt *packet,
515	uint64_t last_ip)
516	{
517	uint64_t ip;
518
519	switch (packet->count) {
520	case 1:
521	ip = (last_ip & (uint64_t)0xffffffffffff0000ULL) \|
522	packet->payload;
523	break;
524	case 2:
525	ip = (last_ip & (uint64_t)0xffffffff00000000ULL) \|
526	packet->payload;
527	break;
528	case 3:
529	ip = packet->payload;
530	/* Sign-extend 6-byte ip */
531	if (ip & (uint64_t)0x800000000000ULL)
532	ip \|= (uint64_t)0xffff000000000000ULL;
533	break;
534	case 4:
535	ip = (last_ip & (uint64_t)0xffff000000000000ULL) \|
536	packet->payload;
537	break;
538	case 6:
539	ip = packet->payload;
540	break;
541	default:
542	return 0;
543	}
544
545	return ip;
546	}
547
548	static inline void intel_pt_set_last_ip(struct intel_pt_decoder *decoder)
549	{
550	decoder->last_ip = intel_pt_calc_ip(&decoder->packet, decoder->last_ip);
551	decoder->have_last_ip = true;
552	}
553
554	static inline void intel_pt_set_ip(struct intel_pt_decoder *decoder)
555	{
556	intel_pt_set_last_ip(decoder);
557	decoder->ip = decoder->last_ip;
558	}
559
560	static void intel_pt_decoder_log_packet(struct intel_pt_decoder *decoder)
561	{
562	intel_pt_log_packet(&decoder->packet, decoder->pkt_len, decoder->pos,
563	decoder->buf);
564	}
565
566	static int intel_pt_bug(struct intel_pt_decoder *decoder)
567	{
568	intel_pt_log("ERROR: Internal error\n");
569	decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
570	return -ENOSYS;
571	}
572
573	static inline void intel_pt_clear_tx_flags(struct intel_pt_decoder *decoder)
574	{
575	decoder->tx_flags = 0;
576	}
577
578	static inline void intel_pt_update_in_tx(struct intel_pt_decoder *decoder)
579	{
580	decoder->tx_flags = decoder->packet.payload & INTEL_PT_IN_TX;
581	}
582
583	static inline void intel_pt_update_pip(struct intel_pt_decoder *decoder)
584	{
585	decoder->pip_payload = decoder->packet.payload;
586	}
587
588	static inline void intel_pt_update_nr(struct intel_pt_decoder *decoder)
589	{
590	decoder->next_nr = decoder->pip_payload & 1;
591	}
592
593	static inline void intel_pt_set_nr(struct intel_pt_decoder *decoder)
594	{
595	decoder->nr = decoder->pip_payload & 1;
596	decoder->next_nr = decoder->nr;
597	}
598
599	static inline void intel_pt_set_pip(struct intel_pt_decoder *decoder)
600	{
601	intel_pt_update_pip(decoder);
602	intel_pt_set_nr(decoder);
603	}
604
605	static int intel_pt_bad_packet(struct intel_pt_decoder *decoder)
606	{
607	intel_pt_clear_tx_flags(decoder);
608	decoder->have_tma = false;
609	decoder->pkt_len = 1;
610	decoder->pkt_step = 1;
611	intel_pt_decoder_log_packet(decoder);
612	if (decoder->pkt_state != INTEL_PT_STATE_NO_PSB) {
613	intel_pt_log("ERROR: Bad packet\n");
614	decoder->pkt_state = INTEL_PT_STATE_ERR1;
615	}
616	return -EBADMSG;
617	}
618
619	static inline void intel_pt_update_sample_time(struct intel_pt_decoder *decoder)
620	{
621	decoder->sample_timestamp = decoder->timestamp;
622	decoder->sample_insn_cnt = decoder->timestamp_insn_cnt;
623	decoder->state.cycles = decoder->tot_cyc_cnt;
624	}
625
626	static void intel_pt_reposition(struct intel_pt_decoder *decoder)
627	{
628	decoder->ip = 0;
629	decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
630	decoder->timestamp = 0;
631	decoder->have_tma = false;
632	}
633
634	static int intel_pt_get_data(struct intel_pt_decoder *decoder, bool reposition)
635	{
636	struct intel_pt_buffer buffer = { .buf = 0, };
637	int ret;
638
639	decoder->pkt_step = 0;
640
641	intel_pt_log("Getting more data\n");
642	ret = decoder->get_trace(&buffer, decoder->data);
643	if (ret)
644	return ret;
645	decoder->buf = buffer.buf;
646	decoder->len = buffer.len;
647	if (!decoder->len) {
648	intel_pt_log("No more data\n");
649	return -ENODATA;
650	}
651	decoder->buf_timestamp = buffer.ref_timestamp;
652	if (!buffer.consecutive \|\| reposition) {
653	intel_pt_reposition(decoder);
654	decoder->ref_timestamp = buffer.ref_timestamp;
655	decoder->state.trace_nr = buffer.trace_nr;
656	decoder->vm_tm_corr_same_buf = false;
657	intel_pt_log("Reference timestamp 0x%" PRIx64 "\n",
658	decoder->ref_timestamp);
659	return -ENOLINK;
660	}
661
662	return 0;
663	}
664
665	static int intel_pt_get_next_data(struct intel_pt_decoder *decoder,
666	bool reposition)
667	{
668	if (!decoder->next_buf)
669	return intel_pt_get_data(decoder, reposition);
670
671	decoder->buf = decoder->next_buf;
672	decoder->len = decoder->next_len;
673	decoder->next_buf = 0;
674	decoder->next_len = 0;
675	return 0;
676	}
677
678	static int intel_pt_get_split_packet(struct intel_pt_decoder *decoder)
679	{
680	unsigned char *buf = decoder->temp_buf;
681	size_t old_len, len, n;
682	int ret;
683
684	old_len = decoder->len;
685	len = decoder->len;
686	memcpy(buf, decoder->buf, len);
687
688	ret = intel_pt_get_data(decoder, false);
689	if (ret) {
690	decoder->pos += old_len;
691	return ret < 0 ? ret : -EINVAL;
692	}
693
694	n = INTEL_PT_PKT_MAX_SZ - len;
695	if (n > decoder->len)
696	n = decoder->len;
697	memcpy(buf + len, decoder->buf, n);
698	len += n;
699
700	decoder->prev_pkt_ctx = decoder->pkt_ctx;
701	ret = intel_pt_get_packet(buf, len, &decoder->packet, &decoder->pkt_ctx);
702	if (ret < (int)old_len) {
703	decoder->next_buf = decoder->buf;
704	decoder->next_len = decoder->len;
705	decoder->buf = buf;
706	decoder->len = old_len;
707	return intel_pt_bad_packet(decoder);
708	}
709
710	decoder->next_buf = decoder->buf + (ret - old_len);
711	decoder->next_len = decoder->len - (ret - old_len);
712
713	decoder->buf = buf;
714	decoder->len = ret;
715
716	return ret;
717	}
718
719	struct intel_pt_pkt_info {
720	struct intel_pt_decoder *decoder;
721	struct intel_pt_pkt packet;
722	uint64_t pos;
723	int pkt_len;
724	int last_packet_type;
725	void *data;
726	};
727
728	typedef int (intel_pt_pkt_cb_t)(struct intel_pt_pkt_info pkt_info);
729
730	/* Lookahead packets in current buffer */
731	static int intel_pt_pkt_lookahead(struct intel_pt_decoder *decoder,
732	intel_pt_pkt_cb_t cb, void *data)
733	{
734	struct intel_pt_pkt_info pkt_info;
735	const unsigned char *buf = decoder->buf;
736	enum intel_pt_pkt_ctx pkt_ctx = decoder->pkt_ctx;
737	size_t len = decoder->len;
738	int ret;
739
740	pkt_info.decoder = decoder;
741	pkt_info.pos = decoder->pos;
742	pkt_info.pkt_len = decoder->pkt_step;
743	pkt_info.last_packet_type = decoder->last_packet_type;
744	pkt_info.data = data;
745
746	while (1) {
747	do {
748	pkt_info.pos += pkt_info.pkt_len;
749	buf += pkt_info.pkt_len;
750	len -= pkt_info.pkt_len;
751
752	if (!len)
753	return INTEL_PT_NEED_MORE_BYTES;
754
755	ret = intel_pt_get_packet(buf, len, &pkt_info.packet,
756	&pkt_ctx);
757	if (!ret)
758	return INTEL_PT_NEED_MORE_BYTES;
759	if (ret < 0)
760	return ret;
761
762	pkt_info.pkt_len = ret;
763	} while (pkt_info.packet.type == INTEL_PT_PAD);
764
765	ret = cb(&pkt_info);
766	if (ret)
767	return 0;
768
769	pkt_info.last_packet_type = pkt_info.packet.type;
770	}
771	}
772
773	struct intel_pt_calc_cyc_to_tsc_info {
774	uint64_t cycle_cnt;
775	unsigned int cbr;
776	uint32_t last_mtc;
777	uint64_t ctc_timestamp;
778	uint64_t ctc_delta;
779	uint64_t tsc_timestamp;
780	uint64_t timestamp;
781	bool have_tma;
782	bool fixup_last_mtc;
783	bool from_mtc;
784	double cbr_cyc_to_tsc;
785	};
786
787	/*
788	* MTC provides a 8-bit slice of CTC but the TMA packet only provides the lower
789	* 16 bits of CTC. If mtc_shift > 8 then some of the MTC bits are not in the CTC
790	* provided by the TMA packet. Fix-up the last_mtc calculated from the TMA
791	* packet by copying the missing bits from the current MTC assuming the least
792	* difference between the two, and that the current MTC comes after last_mtc.
793	*/
794	static void intel_pt_fixup_last_mtc(uint32_t mtc, int mtc_shift,
795	uint32_t *last_mtc)
796	{
797	uint32_t first_missing_bit = 1U << (16 - mtc_shift);
798	uint32_t mask = ~(first_missing_bit - 1);
799
800	*last_mtc \|= mtc & mask;
801	if (*last_mtc >= mtc) {
802	*last_mtc -= first_missing_bit;
803	*last_mtc &= 0xff;
804	}
805	}
806
807	static int intel_pt_calc_cyc_cb(struct intel_pt_pkt_info *pkt_info)
808	{
809	struct intel_pt_decoder *decoder = pkt_info->decoder;
810	struct intel_pt_calc_cyc_to_tsc_info *data = pkt_info->data;
811	uint64_t timestamp;
812	double cyc_to_tsc;
813	unsigned int cbr;
814	uint32_t mtc, mtc_delta, ctc, fc, ctc_rem;
815
816	switch (pkt_info->packet.type) {
817	case INTEL_PT_TNT:
818	case INTEL_PT_TIP_PGE:
819	case INTEL_PT_TIP:
820	case INTEL_PT_FUP:
821	case INTEL_PT_PSB:
822	case INTEL_PT_PIP:
823	case INTEL_PT_MODE_EXEC:
824	case INTEL_PT_MODE_TSX:
825	case INTEL_PT_PSBEND:
826	case INTEL_PT_PAD:
827	case INTEL_PT_VMCS:
828	case INTEL_PT_MNT:
829	case INTEL_PT_PTWRITE:
830	case INTEL_PT_PTWRITE_IP:
831	case INTEL_PT_BBP:
832	case INTEL_PT_BIP:
833	case INTEL_PT_BEP:
834	case INTEL_PT_BEP_IP:
835	case INTEL_PT_CFE:
836	case INTEL_PT_CFE_IP:
837	case INTEL_PT_EVD:
838	return 0;
839
840	case INTEL_PT_MTC:
841	if (!data->have_tma)
842	return 0;
843
844	mtc = pkt_info->packet.payload;
845	if (decoder->mtc_shift > 8 && data->fixup_last_mtc) {
846	data->fixup_last_mtc = false;
847	intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,
848	&data->last_mtc);
849	}
850	if (mtc > data->last_mtc)
851	mtc_delta = mtc - data->last_mtc;
852	else
853	mtc_delta = mtc + 256 - data->last_mtc;
854	data->ctc_delta += mtc_delta << decoder->mtc_shift;
855	data->last_mtc = mtc;
856
857	if (decoder->tsc_ctc_mult) {
858	timestamp = data->ctc_timestamp +
859	data->ctc_delta * decoder->tsc_ctc_mult;
860	} else {
861	timestamp = data->ctc_timestamp +
862	multdiv(data->ctc_delta,
863	decoder->tsc_ctc_ratio_n,
864	decoder->tsc_ctc_ratio_d);
865	}
866
867	if (timestamp < data->timestamp)
868	return 1;
869
870	if (pkt_info->last_packet_type != INTEL_PT_CYC) {
871	data->timestamp = timestamp;
872	return 0;
873	}
874
875	break;
876
877	case INTEL_PT_TSC:
878	/*
879	* For now, do not support using TSC packets - refer
880	* intel_pt_calc_cyc_to_tsc().
881	*/
882	if (data->from_mtc)
883	return 1;
884	timestamp = pkt_info->packet.payload \|
885	(data->timestamp & (0xffULL << 56));
886	if (data->from_mtc && timestamp < data->timestamp &&
887	data->timestamp - timestamp < decoder->tsc_slip)
888	return 1;
889	if (timestamp < data->timestamp)
890	timestamp += (1ULL << 56);
891	if (pkt_info->last_packet_type != INTEL_PT_CYC) {
892	if (data->from_mtc)
893	return 1;
894	data->tsc_timestamp = timestamp;
895	data->timestamp = timestamp;
896	return 0;
897	}
898	break;
899
900	case INTEL_PT_TMA:
901	if (data->from_mtc)
902	return 1;
903
904	if (!decoder->tsc_ctc_ratio_d)
905	return 0;
906
907	ctc = pkt_info->packet.payload;
908	fc = pkt_info->packet.count;
909	ctc_rem = ctc & decoder->ctc_rem_mask;
910
911	data->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
912
913	data->ctc_timestamp = data->tsc_timestamp - fc;
914	if (decoder->tsc_ctc_mult) {
915	data->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;
916	} else {
917	data->ctc_timestamp -=
918	multdiv(ctc_rem, decoder->tsc_ctc_ratio_n,
919	decoder->tsc_ctc_ratio_d);
920	}
921
922	data->ctc_delta = 0;
923	data->have_tma = true;
924	data->fixup_last_mtc = true;
925
926	return 0;
927
928	case INTEL_PT_CYC:
929	data->cycle_cnt += pkt_info->packet.payload;
930	return 0;
931
932	case INTEL_PT_CBR:
933	cbr = pkt_info->packet.payload;
934	if (data->cbr && data->cbr != cbr)
935	return 1;
936	data->cbr = cbr;
937	data->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;
938	return 0;
939
940	case INTEL_PT_TIP_PGD:
941	case INTEL_PT_TRACESTOP:
942	case INTEL_PT_EXSTOP:
943	case INTEL_PT_EXSTOP_IP:
944	case INTEL_PT_MWAIT:
945	case INTEL_PT_PWRE:
946	case INTEL_PT_PWRX:
947	case INTEL_PT_OVF:
948	case INTEL_PT_BAD: /* Does not happen */
949	default:
950	return 1;
951	}
952
953	if (!data->cbr && decoder->cbr) {
954	data->cbr = decoder->cbr;
955	data->cbr_cyc_to_tsc = decoder->cbr_cyc_to_tsc;
956	}
957
958	if (!data->cycle_cnt)
959	return 1;
960
961	cyc_to_tsc = (double)(timestamp - decoder->timestamp) / data->cycle_cnt;
962
963	if (data->cbr && cyc_to_tsc > data->cbr_cyc_to_tsc &&
964	cyc_to_tsc / data->cbr_cyc_to_tsc > 1.25) {
965	intel_pt_log("Timestamp: calculated %g TSC ticks per cycle too big (c.f. CBR-based value %g), pos " x64_fmt "\n",
966	cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos);
967	return 1;
968	}
969
970	decoder->calc_cyc_to_tsc = cyc_to_tsc;
971	decoder->have_calc_cyc_to_tsc = true;
972
973	if (data->cbr) {
974	intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. CBR-based value %g, pos " x64_fmt "\n",
975	cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos);
976	} else {
977	intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. unknown CBR-based value, pos " x64_fmt "\n",
978	cyc_to_tsc, pkt_info->pos);
979	}
980
981	return 1;
982	}
983
984	static void intel_pt_calc_cyc_to_tsc(struct intel_pt_decoder *decoder,
985	bool from_mtc)
986	{
987	struct intel_pt_calc_cyc_to_tsc_info data = {
988	.cycle_cnt = 0,
989	.cbr = 0,
990	.last_mtc = decoder->last_mtc,
991	.ctc_timestamp = decoder->ctc_timestamp,
992	.ctc_delta = decoder->ctc_delta,
993	.tsc_timestamp = decoder->tsc_timestamp,
994	.timestamp = decoder->timestamp,
995	.have_tma = decoder->have_tma,
996	.fixup_last_mtc = decoder->fixup_last_mtc,
997	.from_mtc = from_mtc,
998	.cbr_cyc_to_tsc = 0,
999	};
1000
1001	/*
1002	* For now, do not support using TSC packets for at least the reasons:
1003	* 1) timing might have stopped
1004	* 2) TSC packets within PSB+ can slip against CYC packets
1005	*/
1006	if (!from_mtc)
1007	return;
1008
1009	intel_pt_pkt_lookahead(decoder, intel_pt_calc_cyc_cb, &data);
1010	}
1011
1012	static int intel_pt_get_next_packet(struct intel_pt_decoder *decoder)
1013	{
1014	int ret;
1015
1016	decoder->last_packet_type = decoder->packet.type;
1017
1018	do {
1019	decoder->pos += decoder->pkt_step;
1020	decoder->buf += decoder->pkt_step;
1021	decoder->len -= decoder->pkt_step;
1022
1023	if (!decoder->len) {
1024	ret = intel_pt_get_next_data(decoder, false);
1025	if (ret)
1026	return ret;
1027	}
1028
1029	decoder->prev_pkt_ctx = decoder->pkt_ctx;
1030	ret = intel_pt_get_packet(decoder->buf, decoder->len,
1031	&decoder->packet, &decoder->pkt_ctx);
1032	if (ret == INTEL_PT_NEED_MORE_BYTES && BITS_PER_LONG == 32 &&
1033	decoder->len < INTEL_PT_PKT_MAX_SZ && !decoder->next_buf) {
1034	ret = intel_pt_get_split_packet(decoder);
1035	if (ret < 0)
1036	return ret;
1037	}
1038	if (ret <= 0)
1039	return intel_pt_bad_packet(decoder);
1040
1041	decoder->pkt_len = ret;
1042	decoder->pkt_step = ret;
1043	intel_pt_decoder_log_packet(decoder);
1044	} while (decoder->packet.type == INTEL_PT_PAD);
1045
1046	return 0;
1047	}
1048
1049	static uint64_t intel_pt_next_period(struct intel_pt_decoder *decoder)
1050	{
1051	uint64_t timestamp, masked_timestamp;
1052
1053	timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;
1054	masked_timestamp = timestamp & decoder->period_mask;
1055	if (decoder->continuous_period) {
1056	if (masked_timestamp > decoder->last_masked_timestamp)
1057	return 1;
1058	} else {
1059	timestamp += 1;
1060	masked_timestamp = timestamp & decoder->period_mask;
1061	if (masked_timestamp > decoder->last_masked_timestamp) {
1062	decoder->last_masked_timestamp = masked_timestamp;
1063	decoder->continuous_period = true;
1064	}
1065	}
1066
1067	if (masked_timestamp < decoder->last_masked_timestamp)
1068	return decoder->period_ticks;
1069
1070	return decoder->period_ticks - (timestamp - masked_timestamp);
1071	}
1072
1073	static uint64_t intel_pt_next_sample(struct intel_pt_decoder *decoder)
1074	{
1075	switch (decoder->period_type) {
1076	case INTEL_PT_PERIOD_INSTRUCTIONS:
1077	return decoder->period - decoder->period_insn_cnt;
1078	case INTEL_PT_PERIOD_TICKS:
1079	return intel_pt_next_period(decoder);
1080	case INTEL_PT_PERIOD_NONE:
1081	case INTEL_PT_PERIOD_MTC:
1082	default:
1083	return 0;
1084	}
1085	}
1086
1087	static void intel_pt_sample_insn(struct intel_pt_decoder *decoder)
1088	{
1089	uint64_t timestamp, masked_timestamp;
1090
1091	switch (decoder->period_type) {
1092	case INTEL_PT_PERIOD_INSTRUCTIONS:
1093	decoder->period_insn_cnt = 0;
1094	break;
1095	case INTEL_PT_PERIOD_TICKS:
1096	timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;
1097	masked_timestamp = timestamp & decoder->period_mask;
1098	if (masked_timestamp > decoder->last_masked_timestamp)
1099	decoder->last_masked_timestamp = masked_timestamp;
1100	else
1101	decoder->last_masked_timestamp += decoder->period_ticks;
1102	break;
1103	case INTEL_PT_PERIOD_NONE:
1104	case INTEL_PT_PERIOD_MTC:
1105	default:
1106	break;
1107	}
1108
1109	decoder->state.type \|= INTEL_PT_INSTRUCTION;
1110	}
1111
1112	/*
1113	* Sample FUP instruction at the same time as reporting the FUP event, so the
1114	* instruction sample gets the same flags as the FUP event.
1115	*/
1116	static void intel_pt_sample_fup_insn(struct intel_pt_decoder *decoder)
1117	{
1118	struct intel_pt_insn intel_pt_insn;
1119	uint64_t max_insn_cnt, insn_cnt = 0;
1120	int err;
1121
1122	decoder->state.insn_op = INTEL_PT_OP_OTHER;
1123	decoder->state.insn_len = 0;
1124
1125	if (!decoder->branch_enable \|\| !decoder->pge \|\| decoder->hop \|\|
1126	decoder->ip != decoder->last_ip)
1127	return;
1128
1129	if (!decoder->mtc_insn)
1130	decoder->mtc_insn = true;
1131
1132	max_insn_cnt = intel_pt_next_sample(decoder);
1133	if (max_insn_cnt != 1)
1134	return;
1135
1136	err = decoder->walk_insn(&intel_pt_insn, &insn_cnt, &decoder->ip,
1137	0, max_insn_cnt, decoder->data);
1138	/* Ignore error, it will be reported next walk anyway */
1139	if (err)
1140	return;
1141
1142	if (intel_pt_insn.branch != INTEL_PT_BR_NO_BRANCH) {
1143	intel_pt_log_at("ERROR: Unexpected branch at FUP instruction", decoder->ip);
1144	return;
1145	}
1146
1147	decoder->tot_insn_cnt += insn_cnt;
1148	decoder->timestamp_insn_cnt += insn_cnt;
1149	decoder->sample_insn_cnt += insn_cnt;
1150	decoder->period_insn_cnt += insn_cnt;
1151
1152	intel_pt_sample_insn(decoder);
1153
1154	decoder->state.type \|= INTEL_PT_INSTRUCTION;
1155	decoder->ip += intel_pt_insn.length;
1156	}
1157
1158	static int intel_pt_walk_insn(struct intel_pt_decoder *decoder,
1159	struct intel_pt_insn *intel_pt_insn, uint64_t ip)
1160	{
1161	uint64_t max_insn_cnt, insn_cnt = 0;
1162	int err;
1163
1164	if (!decoder->mtc_insn)
1165	decoder->mtc_insn = true;
1166
1167	max_insn_cnt = intel_pt_next_sample(decoder);
1168
1169	err = decoder->walk_insn(intel_pt_insn, &insn_cnt, &decoder->ip, ip,
1170	max_insn_cnt, decoder->data);
1171
1172	decoder->tot_insn_cnt += insn_cnt;
1173	decoder->timestamp_insn_cnt += insn_cnt;
1174	decoder->sample_insn_cnt += insn_cnt;
1175	decoder->period_insn_cnt += insn_cnt;
1176
1177	if (err) {
1178	decoder->no_progress = 0;
1179	decoder->pkt_state = INTEL_PT_STATE_ERR2;
1180	intel_pt_log_at("ERROR: Failed to get instruction",
1181	decoder->ip);
1182	if (err == -ENOENT)
1183	return -ENOLINK;
1184	return -EILSEQ;
1185	}
1186
1187	if (ip && decoder->ip == ip) {
1188	err = -EAGAIN;
1189	goto out;
1190	}
1191
1192	if (max_insn_cnt && insn_cnt >= max_insn_cnt)
1193	intel_pt_sample_insn(decoder);
1194
1195	if (intel_pt_insn->branch == INTEL_PT_BR_NO_BRANCH) {
1196	decoder->state.type = INTEL_PT_INSTRUCTION;
1197	decoder->state.from_ip = decoder->ip;
1198	decoder->state.to_ip = 0;
1199	decoder->ip += intel_pt_insn->length;
1200	err = INTEL_PT_RETURN;
1201	goto out;
1202	}
1203
1204	if (intel_pt_insn->op == INTEL_PT_OP_CALL) {
1205	/* Zero-length calls are excluded */
1206	if (intel_pt_insn->branch != INTEL_PT_BR_UNCONDITIONAL \|\|
1207	intel_pt_insn->rel) {
1208	err = intel_pt_push(&decoder->stack, decoder->ip +
1209	intel_pt_insn->length);
1210	if (err)
1211	goto out;
1212	}
1213	} else if (intel_pt_insn->op == INTEL_PT_OP_RET) {
1214	decoder->ret_addr = intel_pt_pop(&decoder->stack);
1215	}
1216
1217	if (intel_pt_insn->branch == INTEL_PT_BR_UNCONDITIONAL) {
1218	int cnt = decoder->no_progress++;
1219
1220	decoder->state.from_ip = decoder->ip;
1221	decoder->ip += intel_pt_insn->length +
1222	intel_pt_insn->rel;
1223	decoder->state.to_ip = decoder->ip;
1224	err = INTEL_PT_RETURN;
1225
1226	/*
1227	* Check for being stuck in a loop. This can happen if a
1228	* decoder error results in the decoder erroneously setting the
1229	* ip to an address that is itself in an infinite loop that
1230	* consumes no packets. When that happens, there must be an
1231	* unconditional branch.
1232	*/
1233	if (cnt) {
1234	if (cnt == 1) {
1235	decoder->stuck_ip = decoder->state.to_ip;
1236	decoder->stuck_ip_prd = 1;
1237	decoder->stuck_ip_cnt = 1;
1238	} else if (cnt > decoder->max_loops \|\|
1239	decoder->state.to_ip == decoder->stuck_ip) {
1240	intel_pt_log_at("ERROR: Never-ending loop",
1241	decoder->state.to_ip);
1242	decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1243	err = -ELOOP;
1244	goto out;
1245	} else if (!--decoder->stuck_ip_cnt) {
1246	decoder->stuck_ip_prd += 1;
1247	decoder->stuck_ip_cnt = decoder->stuck_ip_prd;
1248	decoder->stuck_ip = decoder->state.to_ip;
1249	}
1250	}
1251	goto out_no_progress;
1252	}
1253	out:
1254	decoder->no_progress = 0;
1255	out_no_progress:
1256	decoder->state.insn_op = intel_pt_insn->op;
1257	decoder->state.insn_len = intel_pt_insn->length;
1258	memcpy(decoder->state.insn, intel_pt_insn->buf,
1259	INTEL_PT_INSN_BUF_SZ);
1260
1261	if (decoder->tx_flags & INTEL_PT_IN_TX)
1262	decoder->state.flags \|= INTEL_PT_IN_TX;
1263
1264	return err;
1265	}
1266
1267	static void intel_pt_mode_exec_status(struct intel_pt_decoder *decoder)
1268	{
1269	bool iflag = decoder->packet.count & INTEL_PT_IFLAG;
1270
1271	decoder->exec_mode = decoder->packet.payload;
1272	decoder->iflag = iflag;
1273	decoder->next_iflag = iflag;
1274	decoder->state.from_iflag = iflag;
1275	decoder->state.to_iflag = iflag;
1276	}
1277
1278	static void intel_pt_mode_exec(struct intel_pt_decoder *decoder)
1279	{
1280	bool iflag = decoder->packet.count & INTEL_PT_IFLAG;
1281
1282	decoder->exec_mode = decoder->packet.payload;
1283	decoder->next_iflag = iflag;
1284	}
1285
1286	static void intel_pt_sample_iflag(struct intel_pt_decoder *decoder)
1287	{
1288	decoder->state.type \|= INTEL_PT_IFLAG_CHG;
1289	decoder->state.from_iflag = decoder->iflag;
1290	decoder->state.to_iflag = decoder->next_iflag;
1291	decoder->iflag = decoder->next_iflag;
1292	}
1293
1294	static void intel_pt_sample_iflag_chg(struct intel_pt_decoder *decoder)
1295	{
1296	if (decoder->iflag != decoder->next_iflag)
1297	intel_pt_sample_iflag(decoder);
1298	}
1299
1300	static void intel_pt_clear_fup_event(struct intel_pt_decoder *decoder)
1301	{
1302	decoder->set_fup_tx_flags = false;
1303	decoder->set_fup_ptw = false;
1304	decoder->set_fup_mwait = false;
1305	decoder->set_fup_pwre = false;
1306	decoder->set_fup_exstop = false;
1307	decoder->set_fup_bep = false;
1308	decoder->set_fup_cfe_ip = false;
1309	decoder->set_fup_cfe = false;
1310	decoder->evd_cnt = 0;
1311	decoder->set_fup_mode_exec = false;
1312	decoder->iflag = decoder->next_iflag;
1313	}
1314
1315	static bool intel_pt_fup_event(struct intel_pt_decoder *decoder, bool no_tip)
1316	{
1317	enum intel_pt_sample_type type = decoder->state.type;
1318	bool sample_fup_insn = false;
1319	bool ret = false;
1320
1321	decoder->state.type &= ~INTEL_PT_BRANCH;
1322	decoder->state.insn_op = INTEL_PT_OP_OTHER;
1323	decoder->state.insn_len = 0;
1324
1325	if (decoder->set_fup_cfe_ip \|\| decoder->set_fup_cfe) {
1326	bool ip = decoder->set_fup_cfe_ip;
1327
1328	decoder->set_fup_cfe_ip = false;
1329	decoder->set_fup_cfe = false;
1330	decoder->state.type \|= INTEL_PT_EVT;
1331	if (!ip && decoder->pge)
1332	decoder->state.type \|= INTEL_PT_BRANCH;
1333	decoder->state.cfe_type = decoder->fup_cfe_pkt.count;
1334	decoder->state.cfe_vector = decoder->fup_cfe_pkt.payload;
1335	decoder->state.evd_cnt = decoder->evd_cnt;
1336	decoder->state.evd = decoder->evd;
1337	decoder->evd_cnt = 0;
1338	if (ip \|\| decoder->pge)
1339	decoder->state.flags \|= INTEL_PT_FUP_IP;
1340	ret = true;
1341	}
1342	if (decoder->set_fup_mode_exec) {
1343	decoder->set_fup_mode_exec = false;
1344	intel_pt_sample_iflag(decoder);
1345	sample_fup_insn = no_tip;
1346	ret = true;
1347	}
1348	if (decoder->set_fup_tx_flags) {
1349	decoder->set_fup_tx_flags = false;
1350	decoder->tx_flags = decoder->fup_tx_flags;
1351	decoder->state.type \|= INTEL_PT_TRANSACTION;
1352	if (decoder->fup_tx_flags & INTEL_PT_ABORT_TX)
1353	decoder->state.type \|= INTEL_PT_BRANCH;
1354	decoder->state.flags = decoder->fup_tx_flags;
1355	ret = true;
1356	}
1357	if (decoder->set_fup_ptw) {
1358	decoder->set_fup_ptw = false;
1359	decoder->state.type \|= INTEL_PT_PTW;
1360	decoder->state.flags \|= INTEL_PT_FUP_IP;
1361	decoder->state.ptw_payload = decoder->fup_ptw_payload;
1362	ret = true;
1363	}
1364	if (decoder->set_fup_mwait) {
1365	decoder->set_fup_mwait = false;
1366	decoder->state.type \|= INTEL_PT_MWAIT_OP;
1367	decoder->state.mwait_payload = decoder->fup_mwait_payload;
1368	ret = true;
1369	}
1370	if (decoder->set_fup_pwre) {
1371	decoder->set_fup_pwre = false;
1372	decoder->state.type \|= INTEL_PT_PWR_ENTRY;
1373	decoder->state.pwre_payload = decoder->fup_pwre_payload;
1374	ret = true;
1375	}
1376	if (decoder->set_fup_exstop) {
1377	decoder->set_fup_exstop = false;
1378	decoder->state.type \|= INTEL_PT_EX_STOP;
1379	decoder->state.flags \|= INTEL_PT_FUP_IP;
1380	ret = true;
1381	}
1382	if (decoder->set_fup_bep) {
1383	decoder->set_fup_bep = false;
1384	decoder->state.type \|= INTEL_PT_BLK_ITEMS;
1385	ret = true;
1386	}
1387	if (decoder->overflow) {
1388	decoder->overflow = false;
1389	if (!ret && !decoder->pge) {
1390	if (decoder->hop) {
1391	decoder->state.type = 0;
1392	decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;
1393	}
1394	decoder->pge = true;
1395	decoder->state.type \|= INTEL_PT_BRANCH \| INTEL_PT_TRACE_BEGIN;
1396	decoder->state.from_ip = 0;
1397	decoder->state.to_ip = decoder->ip;
1398	return true;
1399	}
1400	}
1401	if (ret) {
1402	decoder->state.from_ip = decoder->ip;
1403	decoder->state.to_ip = 0;
1404	if (sample_fup_insn)
1405	intel_pt_sample_fup_insn(decoder);
1406	} else {
1407	decoder->state.type = type;
1408	}
1409	return ret;
1410	}
1411
1412	static inline bool intel_pt_fup_with_nlip(struct intel_pt_decoder *decoder,
1413	struct intel_pt_insn *intel_pt_insn,
1414	uint64_t ip, int err)
1415	{
1416	return decoder->flags & INTEL_PT_FUP_WITH_NLIP && !err &&
1417	intel_pt_insn->branch == INTEL_PT_BR_INDIRECT &&
1418	ip == decoder->ip + intel_pt_insn->length;
1419	}
1420
1421	static int intel_pt_walk_fup(struct intel_pt_decoder *decoder)
1422	{
1423	struct intel_pt_insn intel_pt_insn;
1424	uint64_t ip;
1425	int err;
1426
1427	ip = decoder->last_ip;
1428
1429	while (1) {
1430	err = intel_pt_walk_insn(decoder, &intel_pt_insn, ip);
1431	if (err == INTEL_PT_RETURN)
1432	return 0;
1433	if (err == -EAGAIN \|\|
1434	intel_pt_fup_with_nlip(decoder, &intel_pt_insn, ip, err)) {
1435	bool no_tip = decoder->pkt_state != INTEL_PT_STATE_FUP;
1436
1437	decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1438	if (intel_pt_fup_event(decoder, no_tip) && no_tip)
1439	return 0;
1440	return -EAGAIN;
1441	}
1442	decoder->set_fup_tx_flags = false;
1443	if (err)
1444	return err;
1445
1446	if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1447	intel_pt_log_at("ERROR: Unexpected indirect branch",
1448	decoder->ip);
1449	decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1450	return -ENOENT;
1451	}
1452
1453	if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1454	intel_pt_log_at("ERROR: Unexpected conditional branch",
1455	decoder->ip);
1456	decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1457	return -ENOENT;
1458	}
1459
1460	intel_pt_bug(decoder);
1461	}
1462	}
1463
1464	static int intel_pt_walk_tip(struct intel_pt_decoder *decoder)
1465	{
1466	struct intel_pt_insn intel_pt_insn;
1467	int err;
1468
1469	err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);
1470	if (err == INTEL_PT_RETURN &&
1471	decoder->pgd_ip &&
1472	decoder->pkt_state == INTEL_PT_STATE_TIP_PGD &&
1473	(decoder->state.type & INTEL_PT_BRANCH) &&
1474	decoder->pgd_ip(decoder->state.to_ip, decoder->data)) {
1475	/* Unconditional branch leaving filter region */
1476	decoder->no_progress = 0;
1477	decoder->pge = false;
1478	decoder->continuous_period = false;
1479	decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1480	decoder->state.type \|= INTEL_PT_TRACE_END;
1481	intel_pt_update_nr(decoder);
1482	return 0;
1483	}
1484	if (err == INTEL_PT_RETURN)
1485	return 0;
1486	if (err)
1487	return err;
1488
1489	intel_pt_update_nr(decoder);
1490	intel_pt_sample_iflag_chg(decoder);
1491
1492	if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1493	if (decoder->pkt_state == INTEL_PT_STATE_TIP_PGD) {
1494	decoder->pge = false;
1495	decoder->continuous_period = false;
1496	decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1497	decoder->state.from_ip = decoder->ip;
1498	if (decoder->packet.count == 0) {
1499	decoder->state.to_ip = 0;
1500	} else {
1501	decoder->state.to_ip = decoder->last_ip;
1502	decoder->ip = decoder->last_ip;
1503	}
1504	decoder->state.type \|= INTEL_PT_TRACE_END;
1505	} else {
1506	decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1507	decoder->state.from_ip = decoder->ip;
1508	if (decoder->packet.count == 0) {
1509	decoder->state.to_ip = 0;
1510	} else {
1511	decoder->state.to_ip = decoder->last_ip;
1512	decoder->ip = decoder->last_ip;
1513	}
1514	}
1515	return 0;
1516	}
1517
1518	if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1519	uint64_t to_ip = decoder->ip + intel_pt_insn.length +
1520	intel_pt_insn.rel;
1521
1522	if (decoder->pgd_ip &&
1523	decoder->pkt_state == INTEL_PT_STATE_TIP_PGD &&
1524	decoder->pgd_ip(to_ip, decoder->data)) {
1525	/* Conditional branch leaving filter region */
1526	decoder->pge = false;
1527	decoder->continuous_period = false;
1528	decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1529	decoder->ip = to_ip;
1530	decoder->state.from_ip = decoder->ip;
1531	decoder->state.to_ip = to_ip;
1532	decoder->state.type \|= INTEL_PT_TRACE_END;
1533	return 0;
1534	}
1535	intel_pt_log_at("ERROR: Conditional branch when expecting indirect branch",
1536	decoder->ip);
1537	decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1538	return -ENOENT;
1539	}
1540
1541	return intel_pt_bug(decoder);
1542	}
1543
1544	struct eptw_data {
1545	int bit_countdown;
1546	uint64_t payload;
1547	};
1548
1549	static int intel_pt_eptw_lookahead_cb(struct intel_pt_pkt_info *pkt_info)
1550	{
1551	struct eptw_data *data = pkt_info->data;
1552	int nr_bits;
1553
1554	switch (pkt_info->packet.type) {
1555	case INTEL_PT_PAD:
1556	case INTEL_PT_MNT:
1557	case INTEL_PT_MODE_EXEC:
1558	case INTEL_PT_MODE_TSX:
1559	case INTEL_PT_MTC:
1560	case INTEL_PT_FUP:
1561	case INTEL_PT_CYC:
1562	case INTEL_PT_CBR:
1563	case INTEL_PT_TSC:
1564	case INTEL_PT_TMA:
1565	case INTEL_PT_PIP:
1566	case INTEL_PT_VMCS:
1567	case INTEL_PT_PSB:
1568	case INTEL_PT_PSBEND:
1569	case INTEL_PT_PTWRITE:
1570	case INTEL_PT_PTWRITE_IP:
1571	case INTEL_PT_EXSTOP:
1572	case INTEL_PT_EXSTOP_IP:
1573	case INTEL_PT_MWAIT:
1574	case INTEL_PT_PWRE:
1575	case INTEL_PT_PWRX:
1576	case INTEL_PT_BBP:
1577	case INTEL_PT_BIP:
1578	case INTEL_PT_BEP:
1579	case INTEL_PT_BEP_IP:
1580	case INTEL_PT_CFE:
1581	case INTEL_PT_CFE_IP:
1582	case INTEL_PT_EVD:
1583	break;
1584
1585	case INTEL_PT_TNT:
1586	nr_bits = data->bit_countdown;
1587	if (nr_bits > pkt_info->packet.count)
1588	nr_bits = pkt_info->packet.count;
1589	data->payload <<= nr_bits;
1590	data->payload \|= pkt_info->packet.payload >> (64 - nr_bits);
1591	data->bit_countdown -= nr_bits;
1592	return !data->bit_countdown;
1593
1594	case INTEL_PT_TIP_PGE:
1595	case INTEL_PT_TIP_PGD:
1596	case INTEL_PT_TIP:
1597	case INTEL_PT_BAD:
1598	case INTEL_PT_OVF:
1599	case INTEL_PT_TRACESTOP:
1600	default:
1601	return 1;
1602	}
1603
1604	return 0;
1605	}
1606
1607	static int intel_pt_emulated_ptwrite(struct intel_pt_decoder *decoder)
1608	{
1609	int n = 64 - decoder->tnt.count;
1610	struct eptw_data data = {
1611	.bit_countdown = n,
1612	.payload = decoder->tnt.payload >> n,
1613	};
1614
1615	decoder->emulated_ptwrite = false;
1616	intel_pt_log("Emulated ptwrite detected\n");
1617
1618	intel_pt_pkt_lookahead(decoder, intel_pt_eptw_lookahead_cb, &data);
1619	if (data.bit_countdown)
1620	return -ECONNRESET;
1621
1622	decoder->state.type = INTEL_PT_PTW;
1623	decoder->state.from_ip = decoder->ip;
1624	decoder->state.to_ip = 0;
1625	decoder->state.ptw_payload = data.payload;
1626	return 0;
1627	}
1628
1629	static int intel_pt_walk_tnt(struct intel_pt_decoder *decoder)
1630	{
1631	struct intel_pt_insn intel_pt_insn;
1632	int err;
1633
1634	while (1) {
1635	if (decoder->emulated_ptwrite)
1636	return intel_pt_emulated_ptwrite(decoder);
1637	err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);
1638	if (err == INTEL_PT_RETURN) {
1639	decoder->emulated_ptwrite = intel_pt_insn.emulated_ptwrite;
1640	return 0;
1641	}
1642	if (err) {
1643	decoder->emulated_ptwrite = false;
1644	return err;
1645	}
1646
1647	if (intel_pt_insn.op == INTEL_PT_OP_RET) {
1648	if (!decoder->return_compression) {
1649	intel_pt_log_at("ERROR: RET when expecting conditional branch",
1650	decoder->ip);
1651	decoder->pkt_state = INTEL_PT_STATE_ERR3;
1652	return -ENOENT;
1653	}
1654	if (!decoder->ret_addr) {
1655	intel_pt_log_at("ERROR: Bad RET compression (stack empty)",
1656	decoder->ip);
1657	decoder->pkt_state = INTEL_PT_STATE_ERR3;
1658	return -ENOENT;
1659	}
1660	if (!(decoder->tnt.payload & BIT63)) {
1661	intel_pt_log_at("ERROR: Bad RET compression (TNT=N)",
1662	decoder->ip);
1663	decoder->pkt_state = INTEL_PT_STATE_ERR3;
1664	return -ENOENT;
1665	}
1666	decoder->tnt.count -= 1;
1667	if (decoder->tnt.count)
1668	decoder->pkt_state = INTEL_PT_STATE_TNT_CONT;
1669	else
1670	decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1671	decoder->tnt.payload <<= 1;
1672	decoder->state.from_ip = decoder->ip;
1673	decoder->ip = decoder->ret_addr;
1674	decoder->state.to_ip = decoder->ip;
1675	return 0;
1676	}
1677
1678	if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1679	/* Handle deferred TIPs */
1680	err = intel_pt_get_next_packet(decoder);
1681	if (err)
1682	return err;
1683	if (decoder->packet.type != INTEL_PT_TIP \|\|
1684	decoder->packet.count == 0) {
1685	intel_pt_log_at("ERROR: Missing deferred TIP for indirect branch",
1686	decoder->ip);
1687	decoder->pkt_state = INTEL_PT_STATE_ERR3;
1688	decoder->pkt_step = 0;
1689	return -ENOENT;
1690	}
1691	intel_pt_set_last_ip(decoder);
1692	decoder->state.from_ip = decoder->ip;
1693	decoder->state.to_ip = decoder->last_ip;
1694	decoder->ip = decoder->last_ip;
1695	intel_pt_update_nr(decoder);
1696	intel_pt_sample_iflag_chg(decoder);
1697	return 0;
1698	}
1699
1700	if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1701	decoder->tnt.count -= 1;
1702	if (decoder->tnt.count)
1703	decoder->pkt_state = INTEL_PT_STATE_TNT_CONT;
1704	else
1705	decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1706	if (decoder->tnt.payload & BIT63) {
1707	decoder->tnt.payload <<= 1;
1708	decoder->state.from_ip = decoder->ip;
1709	decoder->ip += intel_pt_insn.length +
1710	intel_pt_insn.rel;
1711	decoder->state.to_ip = decoder->ip;
1712	return 0;
1713	}
1714	/* Instruction sample for a non-taken branch */
1715	if (decoder->state.type & INTEL_PT_INSTRUCTION) {
1716	decoder->tnt.payload <<= 1;
1717	decoder->state.type = INTEL_PT_INSTRUCTION;
1718	decoder->state.from_ip = decoder->ip;
1719	decoder->state.to_ip = 0;
1720	decoder->ip += intel_pt_insn.length;
1721	return 0;
1722	}
1723	decoder->sample_cyc = false;
1724	decoder->ip += intel_pt_insn.length;
1725	if (!decoder->tnt.count) {
1726	intel_pt_update_sample_time(decoder);
1727	return -EAGAIN;
1728	}
1729	decoder->tnt.payload <<= 1;
1730	continue;
1731	}
1732
1733	return intel_pt_bug(decoder);
1734	}
1735	}
1736
1737	static int intel_pt_mode_tsx(struct intel_pt_decoder decoder, bool no_tip)
1738	{
1739	unsigned int fup_tx_flags;
1740	int err;
1741
1742	fup_tx_flags = decoder->packet.payload &
1743	(INTEL_PT_IN_TX \| INTEL_PT_ABORT_TX);
1744	err = intel_pt_get_next_packet(decoder);
1745	if (err)
1746	return err;
1747	if (decoder->packet.type == INTEL_PT_FUP) {
1748	decoder->fup_tx_flags = fup_tx_flags;
1749	decoder->set_fup_tx_flags = true;
1750	if (!(decoder->fup_tx_flags & INTEL_PT_ABORT_TX))
1751	*no_tip = true;
1752	} else {
1753	intel_pt_log_at("ERROR: Missing FUP after MODE.TSX",
1754	decoder->pos);
1755	intel_pt_update_in_tx(decoder);
1756	}
1757	return 0;
1758	}
1759
1760	static int intel_pt_evd(struct intel_pt_decoder *decoder)
1761	{
1762	if (decoder->evd_cnt >= INTEL_PT_MAX_EVDS) {
1763	intel_pt_log_at("ERROR: Too many EVD packets", decoder->pos);
1764	return -ENOSYS;
1765	}
1766	decoder->evd[decoder->evd_cnt++] = (struct intel_pt_evd){
1767	.type = decoder->packet.count,
1768	.payload = decoder->packet.payload,
1769	};
1770	return 0;
1771	}
1772
1773	static uint64_t intel_pt_8b_tsc(uint64_t timestamp, uint64_t ref_timestamp)
1774	{
1775	timestamp \|= (ref_timestamp & (0xffULL << 56));
1776
1777	if (timestamp < ref_timestamp) {
1778	if (ref_timestamp - timestamp > (1ULL << 55))
1779	timestamp += (1ULL << 56);
1780	} else {
1781	if (timestamp - ref_timestamp > (1ULL << 55))
1782	timestamp -= (1ULL << 56);
1783	}
1784
1785	return timestamp;
1786	}
1787
1788	/* For use only when decoder->vm_time_correlation is true */
1789	static bool intel_pt_time_in_range(struct intel_pt_decoder *decoder,
1790	uint64_t timestamp)
1791	{
1792	uint64_t max_timestamp = decoder->buf_timestamp;
1793
1794	if (!max_timestamp) {
1795	max_timestamp = decoder->last_reliable_timestamp +
1796	0x400000000ULL;
1797	}
1798	return timestamp >= decoder->last_reliable_timestamp &&
1799	timestamp < decoder->buf_timestamp;
1800	}
1801
1802	static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder *decoder)
1803	{
1804	uint64_t timestamp;
1805	bool bad = false;
1806
1807	decoder->have_tma = false;
1808
1809	if (decoder->ref_timestamp) {
1810	timestamp = intel_pt_8b_tsc(decoder->packet.payload,
1811	decoder->ref_timestamp);
1812	decoder->tsc_timestamp = timestamp;
1813	decoder->timestamp = timestamp;
1814	decoder->ref_timestamp = 0;
1815	decoder->timestamp_insn_cnt = 0;
1816	} else if (decoder->timestamp) {
1817	timestamp = decoder->packet.payload \|
1818	(decoder->timestamp & (0xffULL << 56));
1819	decoder->tsc_timestamp = timestamp;
1820	if (timestamp < decoder->timestamp &&
1821	decoder->timestamp - timestamp < decoder->tsc_slip) {
1822	intel_pt_log_to("Suppressing backwards timestamp",
1823	timestamp);
1824	timestamp = decoder->timestamp;
1825	}
1826	if (timestamp < decoder->timestamp) {
1827	if (!decoder->buf_timestamp \|\|
1828	(timestamp + (1ULL << 56) < decoder->buf_timestamp)) {
1829	intel_pt_log_to("Wraparound timestamp", timestamp);
1830	timestamp += (1ULL << 56);
1831	decoder->tsc_timestamp = timestamp;
1832	} else {
1833	intel_pt_log_to("Suppressing bad timestamp", timestamp);
1834	timestamp = decoder->timestamp;
1835	bad = true;
1836	}
1837	}
1838	if (decoder->vm_time_correlation &&
1839	(bad \|\| !intel_pt_time_in_range(decoder, timestamp)) &&
1840	intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_ERANGE))
1841	p_log("Timestamp out of range");
1842	decoder->timestamp = timestamp;
1843	decoder->timestamp_insn_cnt = 0;
1844	}
1845
1846	if (decoder->last_packet_type == INTEL_PT_CYC) {
1847	decoder->cyc_ref_timestamp = decoder->timestamp;
1848	decoder->cycle_cnt = 0;
1849	decoder->have_calc_cyc_to_tsc = false;
1850	intel_pt_calc_cyc_to_tsc(decoder, false);
1851	}
1852
1853	intel_pt_log_to("Setting timestamp", decoder->timestamp);
1854	}
1855
1856	static int intel_pt_overflow(struct intel_pt_decoder *decoder)
1857	{
1858	intel_pt_log("ERROR: Buffer overflow\n");
1859	intel_pt_clear_tx_flags(decoder);
1860	intel_pt_set_nr(decoder);
1861	decoder->timestamp_insn_cnt = 0;
1862	decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1863	decoder->state.from_ip = decoder->ip;
1864	decoder->ip = 0;
1865	decoder->pge = false;
1866	intel_pt_clear_fup_event(decoder);
1867	decoder->overflow = true;
1868	return -EOVERFLOW;
1869	}
1870
1871	static inline void intel_pt_mtc_cyc_cnt_pge(struct intel_pt_decoder *decoder)
1872	{
1873	if (decoder->have_cyc)
1874	return;
1875
1876	decoder->cyc_cnt_timestamp = decoder->timestamp;
1877	decoder->base_cyc_cnt = decoder->tot_cyc_cnt;
1878	}
1879
1880	static inline void intel_pt_mtc_cyc_cnt_cbr(struct intel_pt_decoder *decoder)
1881	{
1882	decoder->tsc_to_cyc = decoder->cbr / decoder->max_non_turbo_ratio_fp;
1883
1884	if (decoder->pge)
1885	intel_pt_mtc_cyc_cnt_pge(decoder);
1886	}
1887
1888	static inline void intel_pt_mtc_cyc_cnt_upd(struct intel_pt_decoder *decoder)
1889	{
1890	uint64_t tot_cyc_cnt, tsc_delta;
1891
1892	if (decoder->have_cyc)
1893	return;
1894
1895	decoder->sample_cyc = true;
1896
1897	if (!decoder->pge \|\| decoder->timestamp <= decoder->cyc_cnt_timestamp)
1898	return;
1899
1900	tsc_delta = decoder->timestamp - decoder->cyc_cnt_timestamp;
1901	tot_cyc_cnt = tsc_delta * decoder->tsc_to_cyc + decoder->base_cyc_cnt;
1902
1903	if (tot_cyc_cnt > decoder->tot_cyc_cnt)
1904	decoder->tot_cyc_cnt = tot_cyc_cnt;
1905	}
1906
1907	static void intel_pt_calc_tma(struct intel_pt_decoder *decoder)
1908	{
1909	uint32_t ctc = decoder->packet.payload;
1910	uint32_t fc = decoder->packet.count;
1911	uint32_t ctc_rem = ctc & decoder->ctc_rem_mask;
1912
1913	if (!decoder->tsc_ctc_ratio_d)
1914	return;
1915
1916	if (decoder->pge && !decoder->in_psb)
1917	intel_pt_mtc_cyc_cnt_pge(decoder);
1918	else
1919	intel_pt_mtc_cyc_cnt_upd(decoder);
1920
1921	decoder->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
1922	decoder->last_ctc = ctc - ctc_rem;
1923	decoder->ctc_timestamp = decoder->tsc_timestamp - fc;
1924	if (decoder->tsc_ctc_mult) {
1925	decoder->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;
1926	} else {
1927	decoder->ctc_timestamp -= multdiv(ctc_rem,
1928	decoder->tsc_ctc_ratio_n,
1929	decoder->tsc_ctc_ratio_d);
1930	}
1931	decoder->ctc_delta = 0;
1932	decoder->have_tma = true;
1933	decoder->fixup_last_mtc = true;
1934	intel_pt_log("CTC timestamp " x64_fmt " last MTC %#x CTC rem %#x\n",
1935	decoder->ctc_timestamp, decoder->last_mtc, ctc_rem);
1936	}
1937
1938	static void intel_pt_calc_mtc_timestamp(struct intel_pt_decoder *decoder)
1939	{
1940	uint64_t timestamp;
1941	uint32_t mtc, mtc_delta;
1942
1943	if (!decoder->have_tma)
1944	return;
1945
1946	mtc = decoder->packet.payload;
1947
1948	if (decoder->mtc_shift > 8 && decoder->fixup_last_mtc) {
1949	decoder->fixup_last_mtc = false;
1950	intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,
1951	&decoder->last_mtc);
1952	}
1953
1954	if (mtc > decoder->last_mtc)
1955	mtc_delta = mtc - decoder->last_mtc;
1956	else
1957	mtc_delta = mtc + 256 - decoder->last_mtc;
1958
1959	decoder->ctc_delta += mtc_delta << decoder->mtc_shift;
1960
1961	if (decoder->tsc_ctc_mult) {
1962	timestamp = decoder->ctc_timestamp +
1963	decoder->ctc_delta * decoder->tsc_ctc_mult;
1964	} else {
1965	timestamp = decoder->ctc_timestamp +
1966	multdiv(decoder->ctc_delta,
1967	decoder->tsc_ctc_ratio_n,
1968	decoder->tsc_ctc_ratio_d);
1969	}
1970
1971	if (timestamp < decoder->timestamp)
1972	intel_pt_log("Suppressing MTC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n",
1973	timestamp, decoder->timestamp);
1974	else
1975	decoder->timestamp = timestamp;
1976
1977	intel_pt_mtc_cyc_cnt_upd(decoder);
1978
1979	decoder->timestamp_insn_cnt = 0;
1980	decoder->last_mtc = mtc;
1981
1982	if (decoder->last_packet_type == INTEL_PT_CYC) {
1983	decoder->cyc_ref_timestamp = decoder->timestamp;
1984	decoder->cycle_cnt = 0;
1985	decoder->have_calc_cyc_to_tsc = false;
1986	intel_pt_calc_cyc_to_tsc(decoder, true);
1987	}
1988
1989	intel_pt_log_to("Setting timestamp", decoder->timestamp);
1990	}
1991
1992	static void intel_pt_calc_cbr(struct intel_pt_decoder *decoder)
1993	{
1994	unsigned int cbr = decoder->packet.payload & 0xff;
1995
1996	decoder->cbr_payload = decoder->packet.payload;
1997
1998	if (decoder->cbr == cbr)
1999	return;
2000
2001	decoder->cbr = cbr;
2002	decoder->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;
2003	decoder->cyc_ref_timestamp = decoder->timestamp;
2004	decoder->cycle_cnt = 0;
2005
2006	intel_pt_mtc_cyc_cnt_cbr(decoder);
2007	}
2008
2009	static void intel_pt_calc_cyc_timestamp(struct intel_pt_decoder *decoder)
2010	{
2011	uint64_t timestamp = decoder->cyc_ref_timestamp;
2012
2013	decoder->have_cyc = true;
2014
2015	decoder->cycle_cnt += decoder->packet.payload;
2016	if (decoder->pge)
2017	decoder->tot_cyc_cnt += decoder->packet.payload;
2018	decoder->sample_cyc = true;
2019
2020	if (!decoder->cyc_ref_timestamp)
2021	return;
2022
2023	if (decoder->have_calc_cyc_to_tsc)
2024	timestamp += decoder->cycle_cnt * decoder->calc_cyc_to_tsc;
2025	else if (decoder->cbr)
2026	timestamp += decoder->cycle_cnt * decoder->cbr_cyc_to_tsc;
2027	else
2028	return;
2029
2030	if (timestamp < decoder->timestamp)
2031	intel_pt_log("Suppressing CYC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n",
2032	timestamp, decoder->timestamp);
2033	else
2034	decoder->timestamp = timestamp;
2035
2036	decoder->timestamp_insn_cnt = 0;
2037
2038	intel_pt_log_to("Setting timestamp", decoder->timestamp);
2039	}
2040
2041	static void intel_pt_bbp(struct intel_pt_decoder *decoder)
2042	{
2043	if (decoder->prev_pkt_ctx == INTEL_PT_NO_CTX) {
2044	memset(decoder->state.items.mask, 0, sizeof(decoder->state.items.mask));
2045	decoder->state.items.is_32_bit = false;
2046	}
2047	decoder->blk_type = decoder->packet.payload;
2048	decoder->blk_type_pos = intel_pt_blk_type_pos(decoder->blk_type);
2049	if (decoder->blk_type == INTEL_PT_GP_REGS)
2050	decoder->state.items.is_32_bit = decoder->packet.count;
2051	if (decoder->blk_type_pos < 0) {
2052	intel_pt_log("WARNING: Unknown block type %u\n",
2053	decoder->blk_type);
2054	} else if (decoder->state.items.mask[decoder->blk_type_pos]) {
2055	intel_pt_log("WARNING: Duplicate block type %u\n",
2056	decoder->blk_type);
2057	}
2058	}
2059
2060	static void intel_pt_bip(struct intel_pt_decoder *decoder)
2061	{
2062	uint32_t id = decoder->packet.count;
2063	uint32_t bit = 1 << id;
2064	int pos = decoder->blk_type_pos;
2065
2066	if (pos < 0 \|\| id >= INTEL_PT_BLK_ITEM_ID_CNT) {
2067	intel_pt_log("WARNING: Unknown block item %u type %d\n",
2068	id, decoder->blk_type);
2069	return;
2070	}
2071
2072	if (decoder->state.items.mask[pos] & bit) {
2073	intel_pt_log("WARNING: Duplicate block item %u type %d\n",
2074	id, decoder->blk_type);
2075	}
2076
2077	decoder->state.items.mask[pos] \|= bit;
2078	decoder->state.items.val[pos][id] = decoder->packet.payload;
2079	}
2080
2081	/* Walk PSB+ packets when already in sync. */
2082	static int intel_pt_walk_psbend(struct intel_pt_decoder *decoder)
2083	{
2084	int err;
2085
2086	decoder->in_psb = true;
2087
2088	while (1) {
2089	err = intel_pt_get_next_packet(decoder);
2090	if (err)
2091	goto out;
2092
2093	switch (decoder->packet.type) {
2094	case INTEL_PT_PSBEND:
2095	err = 0;
2096	goto out;
2097
2098	case INTEL_PT_TIP_PGD:
2099	case INTEL_PT_TIP_PGE:
2100	case INTEL_PT_TIP:
2101	case INTEL_PT_TNT:
2102	case INTEL_PT_TRACESTOP:
2103	case INTEL_PT_BAD:
2104	case INTEL_PT_PSB:
2105	case INTEL_PT_PTWRITE:
2106	case INTEL_PT_PTWRITE_IP:
2107	case INTEL_PT_EXSTOP:
2108	case INTEL_PT_EXSTOP_IP:
2109	case INTEL_PT_MWAIT:
2110	case INTEL_PT_PWRE:
2111	case INTEL_PT_PWRX:
2112	case INTEL_PT_BBP:
2113	case INTEL_PT_BIP:
2114	case INTEL_PT_BEP:
2115	case INTEL_PT_BEP_IP:
2116	case INTEL_PT_CFE:
2117	case INTEL_PT_CFE_IP:
2118	case INTEL_PT_EVD:
2119	decoder->have_tma = false;
2120	intel_pt_log("ERROR: Unexpected packet\n");
2121	err = -EAGAIN;
2122	goto out;
2123
2124	case INTEL_PT_OVF:
2125	err = intel_pt_overflow(decoder);
2126	goto out;
2127
2128	case INTEL_PT_TSC:
2129	intel_pt_calc_tsc_timestamp(decoder);
2130	break;
2131
2132	case INTEL_PT_TMA:
2133	intel_pt_calc_tma(decoder);
2134	break;
2135
2136	case INTEL_PT_CBR:
2137	intel_pt_calc_cbr(decoder);
2138	break;
2139
2140	case INTEL_PT_MODE_EXEC:
2141	intel_pt_mode_exec_status(decoder);
2142	break;
2143
2144	case INTEL_PT_PIP:
2145	intel_pt_set_pip(decoder);
2146	break;
2147
2148	case INTEL_PT_FUP:
2149	decoder->pge = true;
2150	if (decoder->packet.count) {
2151	intel_pt_set_last_ip(decoder);
2152	decoder->psb_ip = decoder->last_ip;
2153	}
2154	break;
2155
2156	case INTEL_PT_MODE_TSX:
2157	intel_pt_update_in_tx(decoder);
2158	break;
2159
2160	case INTEL_PT_MTC:
2161	intel_pt_calc_mtc_timestamp(decoder);
2162	if (decoder->period_type == INTEL_PT_PERIOD_MTC)
2163	decoder->state.type \|= INTEL_PT_INSTRUCTION;
2164	break;
2165
2166	case INTEL_PT_CYC:
2167	intel_pt_calc_cyc_timestamp(decoder);
2168	break;
2169
2170	case INTEL_PT_VMCS:
2171	case INTEL_PT_MNT:
2172	case INTEL_PT_PAD:
2173	default:
2174	break;
2175	}
2176	}
2177	out:
2178	decoder->in_psb = false;
2179
2180	return err;
2181	}
2182
2183	static int intel_pt_walk_fup_tip(struct intel_pt_decoder *decoder)
2184	{
2185	int err;
2186
2187	if (decoder->tx_flags & INTEL_PT_ABORT_TX) {
2188	decoder->tx_flags = 0;
2189	decoder->state.flags &= ~INTEL_PT_IN_TX;
2190	decoder->state.flags \|= INTEL_PT_ABORT_TX;
2191	} else {
2192	decoder->state.flags \|= INTEL_PT_ASYNC;
2193	}
2194
2195	while (1) {
2196	err = intel_pt_get_next_packet(decoder);
2197	if (err)
2198	return err;
2199
2200	switch (decoder->packet.type) {
2201	case INTEL_PT_TNT:
2202	case INTEL_PT_FUP:
2203	case INTEL_PT_TRACESTOP:
2204	case INTEL_PT_PSB:
2205	case INTEL_PT_TSC:
2206	case INTEL_PT_TMA:
2207	case INTEL_PT_MODE_TSX:
2208	case INTEL_PT_BAD:
2209	case INTEL_PT_PSBEND:
2210	case INTEL_PT_PTWRITE:
2211	case INTEL_PT_PTWRITE_IP:
2212	case INTEL_PT_EXSTOP:
2213	case INTEL_PT_EXSTOP_IP:
2214	case INTEL_PT_MWAIT:
2215	case INTEL_PT_PWRE:
2216	case INTEL_PT_PWRX:
2217	case INTEL_PT_BBP:
2218	case INTEL_PT_BIP:
2219	case INTEL_PT_BEP:
2220	case INTEL_PT_BEP_IP:
2221	case INTEL_PT_CFE:
2222	case INTEL_PT_CFE_IP:
2223	case INTEL_PT_EVD:
2224	intel_pt_log("ERROR: Missing TIP after FUP\n");
2225	decoder->pkt_state = INTEL_PT_STATE_ERR3;
2226	decoder->pkt_step = 0;
2227	return -ENOENT;
2228
2229	case INTEL_PT_CBR:
2230	intel_pt_calc_cbr(decoder);
2231	break;
2232
2233	case INTEL_PT_OVF:
2234	return intel_pt_overflow(decoder);
2235
2236	case INTEL_PT_TIP_PGD:
2237	decoder->state.from_ip = decoder->ip;
2238	if (decoder->packet.count == 0) {
2239	decoder->state.to_ip = 0;
2240	} else {
2241	intel_pt_set_ip(decoder);
2242	decoder->state.to_ip = decoder->ip;
2243	}
2244	decoder->pge = false;
2245	decoder->continuous_period = false;
2246	decoder->state.type \|= INTEL_PT_TRACE_END;
2247	intel_pt_update_nr(decoder);
2248	return 0;
2249
2250	case INTEL_PT_TIP_PGE:
2251	decoder->pge = true;
2252	intel_pt_log("Omitting PGE ip " x64_fmt "\n",
2253	decoder->ip);
2254	decoder->state.from_ip = 0;
2255	if (decoder->packet.count == 0) {
2256	decoder->state.to_ip = 0;
2257	} else {
2258	intel_pt_set_ip(decoder);
2259	decoder->state.to_ip = decoder->ip;
2260	}
2261	decoder->state.type \|= INTEL_PT_TRACE_BEGIN;
2262	intel_pt_mtc_cyc_cnt_pge(decoder);
2263	intel_pt_set_nr(decoder);
2264	return 0;
2265
2266	case INTEL_PT_TIP:
2267	decoder->state.from_ip = decoder->ip;
2268	if (decoder->packet.count == 0) {
2269	decoder->state.to_ip = 0;
2270	} else {
2271	intel_pt_set_ip(decoder);
2272	decoder->state.to_ip = decoder->ip;
2273	}
2274	intel_pt_update_nr(decoder);
2275	intel_pt_sample_iflag_chg(decoder);
2276	return 0;
2277
2278	case INTEL_PT_PIP:
2279	intel_pt_update_pip(decoder);
2280	break;
2281
2282	case INTEL_PT_MTC:
2283	intel_pt_calc_mtc_timestamp(decoder);
2284	if (decoder->period_type == INTEL_PT_PERIOD_MTC)
2285	decoder->state.type \|= INTEL_PT_INSTRUCTION;
2286	break;
2287
2288	case INTEL_PT_CYC:
2289	intel_pt_calc_cyc_timestamp(decoder);
2290	break;
2291
2292	case INTEL_PT_MODE_EXEC:
2293	intel_pt_mode_exec(decoder);
2294	break;
2295
2296	case INTEL_PT_VMCS:
2297	case INTEL_PT_MNT:
2298	case INTEL_PT_PAD:
2299	break;
2300
2301	default:
2302	return intel_pt_bug(decoder);
2303	}
2304	}
2305	}
2306
2307	static int intel_pt_resample(struct intel_pt_decoder *decoder)
2308	{
2309	decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2310	decoder->state.type = INTEL_PT_INSTRUCTION;
2311	decoder->state.from_ip = decoder->ip;
2312	decoder->state.to_ip = 0;
2313	return 0;
2314	}
2315
2316	struct intel_pt_vm_tsc_info {
2317	struct intel_pt_pkt pip_packet;
2318	struct intel_pt_pkt vmcs_packet;
2319	struct intel_pt_pkt tma_packet;
2320	bool tsc, pip, vmcs, tma, psbend;
2321	uint64_t ctc_delta;
2322	uint64_t last_ctc;
2323	int max_lookahead;
2324	};
2325
2326	/* Lookahead and get the PIP, VMCS and TMA packets from PSB+ */
2327	static int intel_pt_vm_psb_lookahead_cb(struct intel_pt_pkt_info *pkt_info)
2328	{
2329	struct intel_pt_vm_tsc_info *data = pkt_info->data;
2330
2331	switch (pkt_info->packet.type) {
2332	case INTEL_PT_PAD:
2333	case INTEL_PT_MNT:
2334	case INTEL_PT_MODE_EXEC:
2335	case INTEL_PT_MODE_TSX:
2336	case INTEL_PT_MTC:
2337	case INTEL_PT_FUP:
2338	case INTEL_PT_CYC:
2339	case INTEL_PT_CBR:
2340	break;
2341
2342	case INTEL_PT_TSC:
2343	data->tsc = true;
2344	break;
2345
2346	case INTEL_PT_TMA:
2347	data->tma_packet = pkt_info->packet;
2348	data->tma = true;
2349	break;
2350
2351	case INTEL_PT_PIP:
2352	data->pip_packet = pkt_info->packet;
2353	data->pip = true;
2354	break;
2355
2356	case INTEL_PT_VMCS:
2357	data->vmcs_packet = pkt_info->packet;
2358	data->vmcs = true;
2359	break;
2360
2361	case INTEL_PT_PSBEND:
2362	data->psbend = true;
2363	return 1;
2364
2365	case INTEL_PT_TIP_PGE:
2366	case INTEL_PT_PTWRITE:
2367	case INTEL_PT_PTWRITE_IP:
2368	case INTEL_PT_EXSTOP:
2369	case INTEL_PT_EXSTOP_IP:
2370	case INTEL_PT_MWAIT:
2371	case INTEL_PT_PWRE:
2372	case INTEL_PT_PWRX:
2373	case INTEL_PT_BBP:
2374	case INTEL_PT_BIP:
2375	case INTEL_PT_BEP:
2376	case INTEL_PT_BEP_IP:
2377	case INTEL_PT_OVF:
2378	case INTEL_PT_BAD:
2379	case INTEL_PT_TNT:
2380	case INTEL_PT_TIP_PGD:
2381	case INTEL_PT_TIP:
2382	case INTEL_PT_PSB:
2383	case INTEL_PT_TRACESTOP:
2384	case INTEL_PT_CFE:
2385	case INTEL_PT_CFE_IP:
2386	case INTEL_PT_EVD:
2387	default:
2388	return 1;
2389	}
2390
2391	return 0;
2392	}
2393
2394	struct intel_pt_ovf_fup_info {
2395	int max_lookahead;
2396	bool found;
2397	};
2398
2399	/* Lookahead to detect a FUP packet after OVF */
2400	static int intel_pt_ovf_fup_lookahead_cb(struct intel_pt_pkt_info *pkt_info)
2401	{
2402	struct intel_pt_ovf_fup_info *data = pkt_info->data;
2403
2404	if (pkt_info->packet.type == INTEL_PT_CYC \|\|
2405	pkt_info->packet.type == INTEL_PT_MTC \|\|
2406	pkt_info->packet.type == INTEL_PT_TSC)
2407	return !--(data->max_lookahead);
2408	data->found = pkt_info->packet.type == INTEL_PT_FUP;
2409	return 1;
2410	}
2411
2412	static bool intel_pt_ovf_fup_lookahead(struct intel_pt_decoder *decoder)
2413	{
2414	struct intel_pt_ovf_fup_info data = {
2415	.max_lookahead = 16,
2416	.found = false,
2417	};
2418
2419	intel_pt_pkt_lookahead(decoder, intel_pt_ovf_fup_lookahead_cb, &data);
2420	return data.found;
2421	}
2422
2423	/* Lookahead and get the TMA packet after TSC */
2424	static int intel_pt_tma_lookahead_cb(struct intel_pt_pkt_info *pkt_info)
2425	{
2426	struct intel_pt_vm_tsc_info *data = pkt_info->data;
2427
2428	if (pkt_info->packet.type == INTEL_PT_CYC \|\|
2429	pkt_info->packet.type == INTEL_PT_MTC)
2430	return !--(data->max_lookahead);
2431
2432	if (pkt_info->packet.type == INTEL_PT_TMA) {
2433	data->tma_packet = pkt_info->packet;
2434	data->tma = true;
2435	}
2436	return 1;
2437	}
2438
2439	static uint64_t intel_pt_ctc_to_tsc(struct intel_pt_decoder *decoder, uint64_t ctc)
2440	{
2441	if (decoder->tsc_ctc_mult)
2442	return ctc * decoder->tsc_ctc_mult;
2443	else
2444	return multdiv(ctc, decoder->tsc_ctc_ratio_n, decoder->tsc_ctc_ratio_d);
2445	}
2446
2447	static uint64_t intel_pt_calc_expected_tsc(struct intel_pt_decoder *decoder,
2448	uint32_t ctc,
2449	uint32_t fc,
2450	uint64_t last_ctc_timestamp,
2451	uint64_t ctc_delta,
2452	uint32_t last_ctc)
2453	{
2454	/* Number of CTC ticks from last_ctc_timestamp to last_mtc */
2455	uint64_t last_mtc_ctc = last_ctc + ctc_delta;
2456	/*
2457	* Number of CTC ticks from there until current TMA packet. We would
2458	* expect last_mtc_ctc to be before ctc, but the TSC packet can slip
2459	* past an MTC, so a sign-extended value is used.
2460	*/
2461	uint64_t delta = (int16_t)((uint16_t)ctc - (uint16_t)last_mtc_ctc);
2462	/* Total CTC ticks from last_ctc_timestamp to current TMA packet */
2463	uint64_t new_ctc_delta = ctc_delta + delta;
2464	uint64_t expected_tsc;
2465
2466	/*
2467	* Convert CTC ticks to TSC ticks, add the starting point
2468	* (last_ctc_timestamp) and the fast counter from the TMA packet.
2469	*/
2470	expected_tsc = last_ctc_timestamp + intel_pt_ctc_to_tsc(decoder, new_ctc_delta) + fc;
2471
2472	if (intel_pt_enable_logging) {
2473	intel_pt_log_x64(last_mtc_ctc);
2474	intel_pt_log_x32(last_ctc);
2475	intel_pt_log_x64(ctc_delta);
2476	intel_pt_log_x64(delta);
2477	intel_pt_log_x32(ctc);
2478	intel_pt_log_x64(new_ctc_delta);
2479	intel_pt_log_x64(last_ctc_timestamp);
2480	intel_pt_log_x32(fc);
2481	intel_pt_log_x64(intel_pt_ctc_to_tsc(decoder, new_ctc_delta));
2482	intel_pt_log_x64(expected_tsc);
2483	}
2484
2485	return expected_tsc;
2486	}
2487
2488	static uint64_t intel_pt_expected_tsc(struct intel_pt_decoder *decoder,
2489	struct intel_pt_vm_tsc_info *data)
2490	{
2491	uint32_t ctc = data->tma_packet.payload;
2492	uint32_t fc = data->tma_packet.count;
2493
2494	return intel_pt_calc_expected_tsc(decoder, ctc, fc,
2495	decoder->ctc_timestamp,
2496	data->ctc_delta, data->last_ctc);
2497	}
2498
2499	static void intel_pt_translate_vm_tsc(struct intel_pt_decoder *decoder,
2500	struct intel_pt_vmcs_info *vmcs_info)
2501	{
2502	uint64_t payload = decoder->packet.payload;
2503
2504	/* VMX adds the TSC Offset, so subtract to get host TSC */
2505	decoder->packet.payload -= vmcs_info->tsc_offset;
2506	/* TSC packet has only 7 bytes */
2507	decoder->packet.payload &= SEVEN_BYTES;
2508
2509	/*
2510	* The buffer is mmapped from the data file, so this also updates the
2511	* data file.
2512	*/
2513	if (!decoder->vm_tm_corr_dry_run)
2514	memcpy((void *)decoder->buf + 1, &decoder->packet.payload, 7);
2515
2516	intel_pt_log("Translated VM TSC %#" PRIx64 " -> %#" PRIx64
2517	" VMCS %#" PRIx64 " TSC Offset %#" PRIx64 "\n",
2518	payload, decoder->packet.payload, vmcs_info->vmcs,
2519	vmcs_info->tsc_offset);
2520	}
2521
2522	static void intel_pt_translate_vm_tsc_offset(struct intel_pt_decoder *decoder,
2523	uint64_t tsc_offset)
2524	{
2525	struct intel_pt_vmcs_info vmcs_info = {
2526	.vmcs = NO_VMCS,
2527	.tsc_offset = tsc_offset
2528	};
2529
2530	intel_pt_translate_vm_tsc(decoder, &vmcs_info);
2531	}
2532
2533	static inline bool in_vm(uint64_t pip_payload)
2534	{
2535	return pip_payload & 1;
2536	}
2537
2538	static inline bool pip_in_vm(struct intel_pt_pkt *pip_packet)
2539	{
2540	return pip_packet->payload & 1;
2541	}
2542
2543	static void intel_pt_print_vmcs_info(struct intel_pt_vmcs_info *vmcs_info)
2544	{
2545	p_log("VMCS: %#" PRIx64 " TSC Offset %#" PRIx64,
2546	vmcs_info->vmcs, vmcs_info->tsc_offset);
2547	}
2548
2549	static void intel_pt_vm_tm_corr_psb(struct intel_pt_decoder *decoder,
2550	struct intel_pt_vm_tsc_info *data)
2551	{
2552	memset(data, 0, sizeof(*data));
2553	data->ctc_delta = decoder->ctc_delta;
2554	data->last_ctc = decoder->last_ctc;
2555	intel_pt_pkt_lookahead(decoder, intel_pt_vm_psb_lookahead_cb, data);
2556	if (data->tsc && !data->psbend)
2557	p_log("ERROR: PSB without PSBEND");
2558	decoder->in_psb = data->psbend;
2559	}
2560
2561	static void intel_pt_vm_tm_corr_first_tsc(struct intel_pt_decoder *decoder,
2562	struct intel_pt_vm_tsc_info *data,
2563	struct intel_pt_vmcs_info *vmcs_info,
2564	uint64_t host_tsc)
2565	{
2566	if (!decoder->in_psb) {
2567	/* Can't happen */
2568	p_log("ERROR: First TSC is not in PSB+");
2569	}
2570
2571	if (data->pip) {
2572	if (pip_in_vm(&data->pip_packet)) { /* Guest */
2573	if (vmcs_info && vmcs_info->tsc_offset) {
2574	intel_pt_translate_vm_tsc(decoder, vmcs_info);
2575	decoder->vm_tm_corr_reliable = true;
2576	} else {
2577	p_log("ERROR: First TSC, unknown TSC Offset");
2578	}
2579	} else { /* Host */
2580	decoder->vm_tm_corr_reliable = true;
2581	}
2582	} else { /* Host or Guest */
2583	decoder->vm_tm_corr_reliable = false;
2584	if (intel_pt_time_in_range(decoder, host_tsc)) {
2585	/* Assume Host */
2586	} else {
2587	/* Assume Guest */
2588	if (vmcs_info && vmcs_info->tsc_offset)
2589	intel_pt_translate_vm_tsc(decoder, vmcs_info);
2590	else
2591	p_log("ERROR: First TSC, no PIP, unknown TSC Offset");
2592	}
2593	}
2594	}
2595
2596	static void intel_pt_vm_tm_corr_tsc(struct intel_pt_decoder *decoder,
2597	struct intel_pt_vm_tsc_info *data)
2598	{
2599	struct intel_pt_vmcs_info *vmcs_info;
2600	uint64_t tsc_offset = 0;
2601	uint64_t vmcs;
2602	bool reliable = true;
2603	uint64_t expected_tsc;
2604	uint64_t host_tsc;
2605	uint64_t ref_timestamp;
2606
2607	bool assign = false;
2608	bool assign_reliable = false;
2609
2610	/* Already have 'data' for the in_psb case */
2611	if (!decoder->in_psb) {
2612	memset(data, 0, sizeof(*data));
2613	data->ctc_delta = decoder->ctc_delta;
2614	data->last_ctc = decoder->last_ctc;
2615	data->max_lookahead = 16;
2616	intel_pt_pkt_lookahead(decoder, intel_pt_tma_lookahead_cb, data);
2617	if (decoder->pge) {
2618	data->pip = true;
2619	data->pip_packet.payload = decoder->pip_payload;
2620	}
2621	}
2622
2623	/* Calculations depend on having TMA packets */
2624	if (!data->tma) {
2625	p_log("ERROR: TSC without TMA");
2626	return;
2627	}
2628
2629	vmcs = data->vmcs ? data->vmcs_packet.payload : decoder->vmcs;
2630	if (vmcs == NO_VMCS)
2631	vmcs = 0;
2632
2633	vmcs_info = decoder->findnew_vmcs_info(decoder->data, vmcs);
2634
2635	ref_timestamp = decoder->timestamp ? decoder->timestamp : decoder->buf_timestamp;
2636	host_tsc = intel_pt_8b_tsc(decoder->packet.payload, ref_timestamp);
2637
2638	if (!decoder->ctc_timestamp) {
2639	intel_pt_vm_tm_corr_first_tsc(decoder, data, vmcs_info, host_tsc);
2640	return;
2641	}
2642
2643	expected_tsc = intel_pt_expected_tsc(decoder, data);
2644
2645	tsc_offset = host_tsc - expected_tsc;
2646
2647	/* Determine if TSC is from Host or Guest */
2648	if (data->pip) {
2649	if (pip_in_vm(&data->pip_packet)) { /* Guest */
2650	if (!vmcs_info) {
2651	/* PIP NR=1 without VMCS cannot happen */
2652	p_log("ERROR: Missing VMCS");
2653	intel_pt_translate_vm_tsc_offset(decoder, tsc_offset);
2654	decoder->vm_tm_corr_reliable = false;
2655	return;
2656	}
2657	} else { /* Host */
2658	decoder->last_reliable_timestamp = host_tsc;
2659	decoder->vm_tm_corr_reliable = true;
2660	return;
2661	}
2662	} else { /* Host or Guest */
2663	reliable = false; /* Host/Guest is a guess, so not reliable */
2664	if (decoder->in_psb) {
2665	if (!tsc_offset)
2666	return; /* Zero TSC Offset, assume Host */
2667	/*
2668	* TSC packet has only 7 bytes of TSC. We have no
2669	* information about the Guest's 8th byte, but it
2670	* doesn't matter because we only need 7 bytes.
2671	* Here, since the 8th byte is unreliable and
2672	* irrelevant, compare only 7 byes.
2673	*/
2674	if (vmcs_info &&
2675	(tsc_offset & SEVEN_BYTES) ==
2676	(vmcs_info->tsc_offset & SEVEN_BYTES)) {
2677	/* Same TSC Offset as last VMCS, assume Guest */
2678	goto guest;
2679	}
2680	}
2681	/*
2682	* Check if the host_tsc is within the expected range.
2683	* Note, we could narrow the range more by looking ahead for
2684	* the next host TSC in the same buffer, but we don't bother to
2685	* do that because this is probably good enough.
2686	*/
2687	if (host_tsc >= expected_tsc && intel_pt_time_in_range(decoder, host_tsc)) {
2688	/* Within expected range for Host TSC, assume Host */
2689	decoder->vm_tm_corr_reliable = false;
2690	return;
2691	}
2692	}
2693
2694	guest: /* Assuming Guest */
2695
2696	/* Determine whether to assign TSC Offset */
2697	if (vmcs_info && vmcs_info->vmcs) {
2698	if (vmcs_info->tsc_offset && vmcs_info->reliable) {
2699	assign = false;
2700	} else if (decoder->in_psb && data->pip && decoder->vm_tm_corr_reliable &&
2701	decoder->vm_tm_corr_continuous && decoder->vm_tm_corr_same_buf) {
2702	/* Continuous tracing, TSC in a PSB is not a time loss */
2703	assign = true;
2704	assign_reliable = true;
2705	} else if (decoder->in_psb && data->pip && decoder->vm_tm_corr_same_buf) {
2706	/*
2707	* Unlikely to be a time loss TSC in a PSB which is not
2708	* at the start of a buffer.
2709	*/
2710	assign = true;
2711	assign_reliable = false;
2712	}
2713	}
2714
2715	/* Record VMCS TSC Offset */
2716	if (assign && (vmcs_info->tsc_offset != tsc_offset \|\|
2717	vmcs_info->reliable != assign_reliable)) {
2718	bool print = vmcs_info->tsc_offset != tsc_offset;
2719
2720	vmcs_info->tsc_offset = tsc_offset;
2721	vmcs_info->reliable = assign_reliable;
2722	if (print)
2723	intel_pt_print_vmcs_info(vmcs_info);
2724	}
2725
2726	/* Determine what TSC Offset to use */
2727	if (vmcs_info && vmcs_info->tsc_offset) {
2728	if (!vmcs_info->reliable)
2729	reliable = false;
2730	intel_pt_translate_vm_tsc(decoder, vmcs_info);
2731	} else {
2732	reliable = false;
2733	if (vmcs_info) {
2734	if (!vmcs_info->error_printed) {
2735	p_log("ERROR: Unknown TSC Offset for VMCS %#" PRIx64,
2736	vmcs_info->vmcs);
2737	vmcs_info->error_printed = true;
2738	}
2739	} else {
2740	if (intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_UNK_VMCS))
2741	p_log("ERROR: Unknown VMCS");
2742	}
2743	intel_pt_translate_vm_tsc_offset(decoder, tsc_offset);
2744	}
2745
2746	decoder->vm_tm_corr_reliable = reliable;
2747	}
2748
2749	static void intel_pt_vm_tm_corr_pebs_tsc(struct intel_pt_decoder *decoder)
2750	{
2751	uint64_t host_tsc = decoder->packet.payload;
2752	uint64_t guest_tsc = decoder->packet.payload;
2753	struct intel_pt_vmcs_info *vmcs_info;
2754	uint64_t vmcs;
2755
2756	vmcs = decoder->vmcs;
2757	if (vmcs == NO_VMCS)
2758	vmcs = 0;
2759
2760	vmcs_info = decoder->findnew_vmcs_info(decoder->data, vmcs);
2761
2762	if (decoder->pge) {
2763	if (in_vm(decoder->pip_payload)) { /* Guest */
2764	if (!vmcs_info) {
2765	/* PIP NR=1 without VMCS cannot happen */
2766	p_log("ERROR: Missing VMCS");
2767	}
2768	} else { /* Host */
2769	return;
2770	}
2771	} else { /* Host or Guest */
2772	if (intel_pt_time_in_range(decoder, host_tsc)) {
2773	/* Within expected range for Host TSC, assume Host */
2774	return;
2775	}
2776	}
2777
2778	if (vmcs_info) {
2779	/* Translate Guest TSC to Host TSC */
2780	host_tsc = ((guest_tsc & SEVEN_BYTES) - vmcs_info->tsc_offset) & SEVEN_BYTES;
2781	host_tsc = intel_pt_8b_tsc(host_tsc, decoder->timestamp);
2782	intel_pt_log("Translated VM TSC %#" PRIx64 " -> %#" PRIx64
2783	" VMCS %#" PRIx64 " TSC Offset %#" PRIx64 "\n",
2784	guest_tsc, host_tsc, vmcs_info->vmcs,
2785	vmcs_info->tsc_offset);
2786	if (!intel_pt_time_in_range(decoder, host_tsc) &&
2787	intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_ERANGE))
2788	p_log("Timestamp out of range");
2789	} else {
2790	if (intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_UNK_VMCS))
2791	p_log("ERROR: Unknown VMCS");
2792	host_tsc = decoder->timestamp;
2793	}
2794
2795	decoder->packet.payload = host_tsc;
2796
2797	if (!decoder->vm_tm_corr_dry_run)
2798	memcpy((void *)decoder->buf + 1, &host_tsc, 8);
2799	}
2800
2801	static int intel_pt_vm_time_correlation(struct intel_pt_decoder *decoder)
2802	{
2803	struct intel_pt_vm_tsc_info data = { .psbend = false };
2804	bool pge;
2805	int err;
2806
2807	if (decoder->in_psb)
2808	intel_pt_vm_tm_corr_psb(decoder, &data);
2809
2810	while (1) {
2811	err = intel_pt_get_next_packet(decoder);
2812	if (err == -ENOLINK)
2813	continue;
2814	if (err)
2815	break;
2816
2817	switch (decoder->packet.type) {
2818	case INTEL_PT_TIP_PGD:
2819	decoder->pge = false;
2820	decoder->vm_tm_corr_continuous = false;
2821	break;
2822
2823	case INTEL_PT_TNT:
2824	case INTEL_PT_TIP:
2825	case INTEL_PT_TIP_PGE:
2826	decoder->pge = true;
2827	break;
2828
2829	case INTEL_PT_OVF:
2830	decoder->in_psb = false;
2831	pge = decoder->pge;
2832	decoder->pge = intel_pt_ovf_fup_lookahead(decoder);
2833	if (pge != decoder->pge)
2834	intel_pt_log("Surprising PGE change in OVF!");
2835	if (!decoder->pge)
2836	decoder->vm_tm_corr_continuous = false;
2837	break;
2838
2839	case INTEL_PT_FUP:
2840	if (decoder->in_psb)
2841	decoder->pge = true;
2842	break;
2843
2844	case INTEL_PT_TRACESTOP:
2845	decoder->pge = false;
2846	decoder->vm_tm_corr_continuous = false;
2847	decoder->have_tma = false;
2848	break;
2849
2850	case INTEL_PT_PSB:
2851	intel_pt_vm_tm_corr_psb(decoder, &data);
2852	break;
2853
2854	case INTEL_PT_PIP:
2855	decoder->pip_payload = decoder->packet.payload;
2856	break;
2857
2858	case INTEL_PT_MTC:
2859	intel_pt_calc_mtc_timestamp(decoder);
2860	break;
2861
2862	case INTEL_PT_TSC:
2863	intel_pt_vm_tm_corr_tsc(decoder, &data);
2864	intel_pt_calc_tsc_timestamp(decoder);
2865	decoder->vm_tm_corr_same_buf = true;
2866	decoder->vm_tm_corr_continuous = decoder->pge;
2867	break;
2868
2869	case INTEL_PT_TMA:
2870	intel_pt_calc_tma(decoder);
2871	break;
2872
2873	case INTEL_PT_CYC:
2874	intel_pt_calc_cyc_timestamp(decoder);
2875	break;
2876
2877	case INTEL_PT_CBR:
2878	intel_pt_calc_cbr(decoder);
2879	break;
2880
2881	case INTEL_PT_PSBEND:
2882	decoder->in_psb = false;
2883	data.psbend = false;
2884	break;
2885
2886	case INTEL_PT_VMCS:
2887	if (decoder->packet.payload != NO_VMCS)
2888	decoder->vmcs = decoder->packet.payload;
2889	break;
2890
2891	case INTEL_PT_BBP:
2892	decoder->blk_type = decoder->packet.payload;
2893	break;
2894
2895	case INTEL_PT_BIP:
2896	if (decoder->blk_type == INTEL_PT_PEBS_BASIC &&
2897	decoder->packet.count == 2)
2898	intel_pt_vm_tm_corr_pebs_tsc(decoder);
2899	break;
2900
2901	case INTEL_PT_BEP:
2902	case INTEL_PT_BEP_IP:
2903	decoder->blk_type = 0;
2904	break;
2905
2906	case INTEL_PT_CFE:
2907	case INTEL_PT_CFE_IP:
2908	case INTEL_PT_EVD:
2909	case INTEL_PT_MODE_EXEC:
2910	case INTEL_PT_MODE_TSX:
2911	case INTEL_PT_MNT:
2912	case INTEL_PT_PAD:
2913	case INTEL_PT_PTWRITE_IP:
2914	case INTEL_PT_PTWRITE:
2915	case INTEL_PT_MWAIT:
2916	case INTEL_PT_PWRE:
2917	case INTEL_PT_EXSTOP_IP:
2918	case INTEL_PT_EXSTOP:
2919	case INTEL_PT_PWRX:
2920	case INTEL_PT_BAD: /* Does not happen */
2921	default:
2922	break;
2923	}
2924	}
2925
2926	return err;
2927	}
2928
2929	#define HOP_PROCESS 0
2930	#define HOP_IGNORE 1
2931	#define HOP_RETURN 2
2932	#define HOP_AGAIN 3
2933
2934	static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder);
2935
2936	/* Hop mode: Ignore TNT, do not walk code, but get ip from FUPs and TIPs */
2937	static int intel_pt_hop_trace(struct intel_pt_decoder decoder, bool no_tip, int *err)
2938	{
2939	*err = 0;
2940
2941	/* Leap from PSB to PSB, getting ip from FUP within PSB+ */
2942	if (decoder->leap && !decoder->in_psb && decoder->packet.type != INTEL_PT_PSB) {
2943	*err = intel_pt_scan_for_psb(decoder);
2944	if (*err)
2945	return HOP_RETURN;
2946	}
2947
2948	switch (decoder->packet.type) {
2949	case INTEL_PT_TNT:
2950	return HOP_IGNORE;
2951
2952	case INTEL_PT_TIP_PGD:
2953	decoder->pge = false;
2954	if (!decoder->packet.count) {
2955	intel_pt_set_nr(decoder);
2956	return HOP_IGNORE;
2957	}
2958	intel_pt_set_ip(decoder);
2959	decoder->state.type \|= INTEL_PT_TRACE_END;
2960	decoder->state.from_ip = 0;
2961	decoder->state.to_ip = decoder->ip;
2962	intel_pt_update_nr(decoder);
2963	return HOP_RETURN;
2964
2965	case INTEL_PT_TIP:
2966	if (!decoder->packet.count) {
2967	intel_pt_set_nr(decoder);
2968	return HOP_IGNORE;
2969	}
2970	intel_pt_set_ip(decoder);
2971	decoder->state.type = INTEL_PT_INSTRUCTION;
2972	decoder->state.from_ip = decoder->ip;
2973	decoder->state.to_ip = 0;
2974	intel_pt_update_nr(decoder);
2975	intel_pt_sample_iflag_chg(decoder);
2976	return HOP_RETURN;
2977
2978	case INTEL_PT_FUP:
2979	if (!decoder->packet.count)
2980	return HOP_IGNORE;
2981	intel_pt_set_ip(decoder);
2982	if (decoder->set_fup_mwait \|\| decoder->set_fup_pwre)
2983	*no_tip = true;
2984	if (!decoder->branch_enable \|\| !decoder->pge)
2985	*no_tip = true;
2986	if (*no_tip) {
2987	decoder->state.type = INTEL_PT_INSTRUCTION;
2988	decoder->state.from_ip = decoder->ip;
2989	decoder->state.to_ip = 0;
2990	intel_pt_fup_event(decoder, *no_tip);
2991	return HOP_RETURN;
2992	}
2993	intel_pt_fup_event(decoder, *no_tip);
2994	decoder->state.type \|= INTEL_PT_INSTRUCTION \| INTEL_PT_BRANCH;
2995	*err = intel_pt_walk_fup_tip(decoder);
2996	if (!*err && decoder->state.to_ip)
2997	decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;
2998	return HOP_RETURN;
2999
3000	case INTEL_PT_PSB:
3001	decoder->state.psb_offset = decoder->pos;
3002	decoder->psb_ip = 0;
3003	decoder->last_ip = 0;
3004	decoder->have_last_ip = true;
3005	*err = intel_pt_walk_psbend(decoder);
3006	if (*err == -EAGAIN)
3007	return HOP_AGAIN;
3008	if (*err)
3009	return HOP_RETURN;
3010	decoder->state.type = INTEL_PT_PSB_EVT;
3011	if (decoder->psb_ip) {
3012	decoder->state.type \|= INTEL_PT_INSTRUCTION;
3013	decoder->ip = decoder->psb_ip;
3014	}
3015	decoder->state.from_ip = decoder->psb_ip;
3016	decoder->state.to_ip = 0;
3017	return HOP_RETURN;
3018
3019	case INTEL_PT_BAD:
3020	case INTEL_PT_PAD:
3021	case INTEL_PT_TIP_PGE:
3022	case INTEL_PT_TSC:
3023	case INTEL_PT_TMA:
3024	case INTEL_PT_MODE_EXEC:
3025	case INTEL_PT_MODE_TSX:
3026	case INTEL_PT_MTC:
3027	case INTEL_PT_CYC:
3028	case INTEL_PT_VMCS:
3029	case INTEL_PT_PSBEND:
3030	case INTEL_PT_CBR:
3031	case INTEL_PT_TRACESTOP:
3032	case INTEL_PT_PIP:
3033	case INTEL_PT_OVF:
3034	case INTEL_PT_MNT:
3035	case INTEL_PT_PTWRITE:
3036	case INTEL_PT_PTWRITE_IP:
3037	case INTEL_PT_EXSTOP:
3038	case INTEL_PT_EXSTOP_IP:
3039	case INTEL_PT_MWAIT:
3040	case INTEL_PT_PWRE:
3041	case INTEL_PT_PWRX:
3042	case INTEL_PT_BBP:
3043	case INTEL_PT_BIP:
3044	case INTEL_PT_BEP:
3045	case INTEL_PT_BEP_IP:
3046	case INTEL_PT_CFE:
3047	case INTEL_PT_CFE_IP:
3048	case INTEL_PT_EVD:
3049	default:
3050	return HOP_PROCESS;
3051	}
3052	}
3053
3054	struct intel_pt_psb_info {
3055	struct intel_pt_pkt fup_packet;
3056	bool fup;
3057	int after_psbend;
3058	};
3059
3060	/* Lookahead and get the FUP packet from PSB+ */
3061	static int intel_pt_psb_lookahead_cb(struct intel_pt_pkt_info *pkt_info)
3062	{
3063	struct intel_pt_psb_info *data = pkt_info->data;
3064
3065	switch (pkt_info->packet.type) {
3066	case INTEL_PT_PAD:
3067	case INTEL_PT_MNT:
3068	case INTEL_PT_TSC:
3069	case INTEL_PT_TMA:
3070	case INTEL_PT_MODE_EXEC:
3071	case INTEL_PT_MODE_TSX:
3072	case INTEL_PT_MTC:
3073	case INTEL_PT_CYC:
3074	case INTEL_PT_VMCS:
3075	case INTEL_PT_CBR:
3076	case INTEL_PT_PIP:
3077	if (data->after_psbend) {
3078	data->after_psbend -= 1;
3079	if (!data->after_psbend)
3080	return 1;
3081	}
3082	break;
3083
3084	case INTEL_PT_FUP:
3085	if (data->after_psbend)
3086	return 1;
3087	if (data->fup \|\| pkt_info->packet.count == 0)
3088	return 1;
3089	data->fup_packet = pkt_info->packet;
3090	data->fup = true;
3091	break;
3092
3093	case INTEL_PT_PSBEND:
3094	if (!data->fup)
3095	return 1;
3096	/* Keep going to check for a TIP.PGE */
3097	data->after_psbend = 6;
3098	break;
3099
3100	case INTEL_PT_TIP_PGE:
3101	/* Ignore FUP in PSB+ if followed by TIP.PGE */
3102	if (data->after_psbend)
3103	data->fup = false;
3104	return 1;
3105
3106	case INTEL_PT_PTWRITE:
3107	case INTEL_PT_PTWRITE_IP:
3108	case INTEL_PT_EXSTOP:
3109	case INTEL_PT_EXSTOP_IP:
3110	case INTEL_PT_MWAIT:
3111	case INTEL_PT_PWRE:
3112	case INTEL_PT_PWRX:
3113	case INTEL_PT_BBP:
3114	case INTEL_PT_BIP:
3115	case INTEL_PT_BEP:
3116	case INTEL_PT_BEP_IP:
3117	case INTEL_PT_CFE:
3118	case INTEL_PT_CFE_IP:
3119	case INTEL_PT_EVD:
3120	if (data->after_psbend) {
3121	data->after_psbend -= 1;
3122	if (!data->after_psbend)
3123	return 1;
3124	break;
3125	}
3126	return 1;
3127
3128	case INTEL_PT_OVF:
3129	case INTEL_PT_BAD:
3130	case INTEL_PT_TNT:
3131	case INTEL_PT_TIP_PGD:
3132	case INTEL_PT_TIP:
3133	case INTEL_PT_PSB:
3134	case INTEL_PT_TRACESTOP:
3135	default:
3136	return 1;
3137	}
3138
3139	return 0;
3140	}
3141
3142	static int intel_pt_psb(struct intel_pt_decoder *decoder)
3143	{
3144	int err;
3145
3146	decoder->last_ip = 0;
3147	decoder->psb_ip = 0;
3148	decoder->have_last_ip = true;
3149	intel_pt_clear_stack(&decoder->stack);
3150	err = intel_pt_walk_psbend(decoder);
3151	if (err)
3152	return err;
3153	decoder->state.type = INTEL_PT_PSB_EVT;
3154	decoder->state.from_ip = decoder->psb_ip;
3155	decoder->state.to_ip = 0;
3156	return 0;
3157	}
3158
3159	static int intel_pt_fup_in_psb(struct intel_pt_decoder *decoder)
3160	{
3161	int err;
3162
3163	if (decoder->ip != decoder->last_ip) {
3164	err = intel_pt_walk_fup(decoder);
3165	if (!err \|\| err != -EAGAIN)
3166	return err;
3167	}
3168
3169	decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
3170	err = intel_pt_psb(decoder);
3171	if (err) {
3172	decoder->pkt_state = INTEL_PT_STATE_ERR3;
3173	return -ENOENT;
3174	}
3175
3176	return 0;
3177	}
3178
3179	static bool intel_pt_psb_with_fup(struct intel_pt_decoder decoder, int err)
3180	{
3181	struct intel_pt_psb_info data = { .fup = false };
3182
3183	if (!decoder->branch_enable)
3184	return false;
3185
3186	intel_pt_pkt_lookahead(decoder, intel_pt_psb_lookahead_cb, &data);
3187	if (!data.fup)
3188	return false;
3189
3190	decoder->packet = data.fup_packet;
3191	intel_pt_set_last_ip(decoder);
3192	decoder->pkt_state = INTEL_PT_STATE_FUP_IN_PSB;
3193
3194	*err = intel_pt_fup_in_psb(decoder);
3195
3196	return true;
3197	}
3198
3199	static int intel_pt_walk_trace(struct intel_pt_decoder *decoder)
3200	{
3201	int last_packet_type = INTEL_PT_PAD;
3202	bool no_tip = false;
3203	int err;
3204
3205	while (1) {
3206	err = intel_pt_get_next_packet(decoder);
3207	if (err)
3208	return err;
3209	next:
3210	err = 0;
3211	if (decoder->cyc_threshold) {
3212	if (decoder->sample_cyc && last_packet_type != INTEL_PT_CYC)
3213	decoder->sample_cyc = false;
3214	last_packet_type = decoder->packet.type;
3215	}
3216
3217	if (decoder->hop) {
3218	switch (intel_pt_hop_trace(decoder, &no_tip, &err)) {
3219	case HOP_IGNORE:
3220	continue;
3221	case HOP_RETURN:
3222	return err;
3223	case HOP_AGAIN:
3224	goto next;
3225	default:
3226	break;
3227	}
3228	}
3229
3230	switch (decoder->packet.type) {
3231	case INTEL_PT_TNT:
3232	if (!decoder->packet.count)
3233	break;
3234	decoder->tnt = decoder->packet;
3235	decoder->pkt_state = INTEL_PT_STATE_TNT;
3236	err = intel_pt_walk_tnt(decoder);
3237	if (err == -EAGAIN)
3238	break;
3239	return err;
3240
3241	case INTEL_PT_TIP_PGD:
3242	if (decoder->packet.count != 0)
3243	intel_pt_set_last_ip(decoder);
3244	decoder->pkt_state = INTEL_PT_STATE_TIP_PGD;
3245	return intel_pt_walk_tip(decoder);
3246
3247	case INTEL_PT_TIP_PGE: {
3248	decoder->pge = true;
3249	decoder->overflow = false;
3250	intel_pt_mtc_cyc_cnt_pge(decoder);
3251	intel_pt_set_nr(decoder);
3252	if (decoder->packet.count == 0) {
3253	intel_pt_log_at("Skipping zero TIP.PGE",
3254	decoder->pos);
3255	break;
3256	}
3257	intel_pt_sample_iflag_chg(decoder);
3258	intel_pt_set_ip(decoder);
3259	decoder->state.from_ip = 0;
3260	decoder->state.to_ip = decoder->ip;
3261	decoder->state.type \|= INTEL_PT_TRACE_BEGIN;
3262	/*
3263	* In hop mode, resample to get the to_ip as an
3264	* "instruction" sample.
3265	*/
3266	if (decoder->hop)
3267	decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;
3268	return 0;
3269	}
3270
3271	case INTEL_PT_OVF:
3272	return intel_pt_overflow(decoder);
3273
3274	case INTEL_PT_TIP:
3275	if (decoder->packet.count != 0)
3276	intel_pt_set_last_ip(decoder);
3277	decoder->pkt_state = INTEL_PT_STATE_TIP;
3278	return intel_pt_walk_tip(decoder);
3279
3280	case INTEL_PT_FUP:
3281	if (decoder->packet.count == 0) {
3282	intel_pt_log_at("Skipping zero FUP",
3283	decoder->pos);
3284	no_tip = false;
3285	break;
3286	}
3287	intel_pt_set_last_ip(decoder);
3288	if (!decoder->branch_enable \|\| !decoder->pge) {
3289	decoder->ip = decoder->last_ip;
3290	if (intel_pt_fup_event(decoder, no_tip))
3291	return 0;
3292	no_tip = false;
3293	break;
3294	}
3295	if (decoder->set_fup_mwait)
3296	no_tip = true;
3297	if (no_tip)
3298	decoder->pkt_state = INTEL_PT_STATE_FUP_NO_TIP;
3299	else
3300	decoder->pkt_state = INTEL_PT_STATE_FUP;
3301	err = intel_pt_walk_fup(decoder);
3302	if (err != -EAGAIN)
3303	return err;
3304	if (no_tip) {
3305	no_tip = false;
3306	break;
3307	}
3308	return intel_pt_walk_fup_tip(decoder);
3309
3310	case INTEL_PT_TRACESTOP:
3311	decoder->pge = false;
3312	decoder->continuous_period = false;
3313	intel_pt_clear_tx_flags(decoder);
3314	decoder->have_tma = false;
3315	break;
3316
3317	case INTEL_PT_PSB:
3318	decoder->state.psb_offset = decoder->pos;
3319	decoder->psb_ip = 0;
3320	if (intel_pt_psb_with_fup(decoder, &err))
3321	return err;
3322	err = intel_pt_psb(decoder);
3323	if (err == -EAGAIN)
3324	goto next;
3325	return err;
3326
3327	case INTEL_PT_PIP:
3328	intel_pt_update_pip(decoder);
3329	break;
3330
3331	case INTEL_PT_MTC:
3332	intel_pt_calc_mtc_timestamp(decoder);
3333	if (decoder->period_type != INTEL_PT_PERIOD_MTC)
3334	break;
3335	/*
3336	* Ensure that there has been an instruction since the
3337	* last MTC.
3338	*/
3339	if (!decoder->mtc_insn)
3340	break;
3341	decoder->mtc_insn = false;
3342	/* Ensure that there is a timestamp */
3343	if (!decoder->timestamp)
3344	break;
3345	decoder->state.type = INTEL_PT_INSTRUCTION;
3346	decoder->state.from_ip = decoder->ip;
3347	decoder->state.to_ip = 0;
3348	decoder->mtc_insn = false;
3349	return 0;
3350
3351	case INTEL_PT_TSC:
3352	intel_pt_calc_tsc_timestamp(decoder);
3353	break;
3354
3355	case INTEL_PT_TMA:
3356	intel_pt_calc_tma(decoder);
3357	break;
3358
3359	case INTEL_PT_CYC:
3360	intel_pt_calc_cyc_timestamp(decoder);
3361	break;
3362
3363	case INTEL_PT_CBR:
3364	intel_pt_calc_cbr(decoder);
3365	if (decoder->cbr != decoder->cbr_seen) {
3366	decoder->state.type = 0;
3367	return 0;
3368	}
3369	break;
3370
3371	case INTEL_PT_MODE_EXEC:
3372	intel_pt_mode_exec(decoder);
3373	err = intel_pt_get_next_packet(decoder);
3374	if (err)
3375	return err;
3376	if (decoder->packet.type == INTEL_PT_FUP) {
3377	decoder->set_fup_mode_exec = true;
3378	no_tip = true;
3379	}
3380	goto next;
3381
3382	case INTEL_PT_MODE_TSX:
3383	/* MODE_TSX need not be followed by FUP */
3384	if (!decoder->pge \|\| decoder->in_psb) {
3385	intel_pt_update_in_tx(decoder);
3386	break;
3387	}
3388	err = intel_pt_mode_tsx(decoder, &no_tip);
3389	if (err)
3390	return err;
3391	goto next;
3392
3393	case INTEL_PT_BAD: /* Does not happen */
3394	return intel_pt_bug(decoder);
3395
3396	case INTEL_PT_PSBEND:
3397	case INTEL_PT_VMCS:
3398	case INTEL_PT_MNT:
3399	case INTEL_PT_PAD:
3400	break;
3401
3402	case INTEL_PT_PTWRITE_IP:
3403	decoder->fup_ptw_payload = decoder->packet.payload;
3404	err = intel_pt_get_next_packet(decoder);
3405	if (err)
3406	return err;
3407	if (decoder->packet.type == INTEL_PT_FUP) {
3408	decoder->set_fup_ptw = true;
3409	no_tip = true;
3410	} else {
3411	intel_pt_log_at("ERROR: Missing FUP after PTWRITE",
3412	decoder->pos);
3413	}
3414	goto next;
3415
3416	case INTEL_PT_PTWRITE:
3417	decoder->state.type = INTEL_PT_PTW;
3418	decoder->state.from_ip = decoder->ip;
3419	decoder->state.to_ip = 0;
3420	decoder->state.ptw_payload = decoder->packet.payload;
3421	return 0;
3422
3423	case INTEL_PT_MWAIT:
3424	decoder->fup_mwait_payload = decoder->packet.payload;
3425	decoder->set_fup_mwait = true;
3426	break;
3427
3428	case INTEL_PT_PWRE:
3429	if (decoder->set_fup_mwait) {
3430	decoder->fup_pwre_payload =
3431	decoder->packet.payload;
3432	decoder->set_fup_pwre = true;
3433	break;
3434	}
3435	decoder->state.type = INTEL_PT_PWR_ENTRY;
3436	decoder->state.from_ip = decoder->ip;
3437	decoder->state.to_ip = 0;
3438	decoder->state.pwrx_payload = decoder->packet.payload;
3439	return 0;
3440
3441	case INTEL_PT_EXSTOP_IP:
3442	err = intel_pt_get_next_packet(decoder);
3443	if (err)
3444	return err;
3445	if (decoder->packet.type == INTEL_PT_FUP) {
3446	decoder->set_fup_exstop = true;
3447	no_tip = true;
3448	} else {
3449	intel_pt_log_at("ERROR: Missing FUP after EXSTOP",
3450	decoder->pos);
3451	}
3452	goto next;
3453
3454	case INTEL_PT_EXSTOP:
3455	decoder->state.type = INTEL_PT_EX_STOP;
3456	decoder->state.from_ip = decoder->ip;
3457	decoder->state.to_ip = 0;
3458	return 0;
3459
3460	case INTEL_PT_PWRX:
3461	decoder->state.type = INTEL_PT_PWR_EXIT;
3462	decoder->state.from_ip = decoder->ip;
3463	decoder->state.to_ip = 0;
3464	decoder->state.pwrx_payload = decoder->packet.payload;
3465	return 0;
3466
3467	case INTEL_PT_BBP:
3468	intel_pt_bbp(decoder);
3469	break;
3470
3471	case INTEL_PT_BIP:
3472	intel_pt_bip(decoder);
3473	break;
3474
3475	case INTEL_PT_BEP:
3476	decoder->state.type = INTEL_PT_BLK_ITEMS;
3477	decoder->state.from_ip = decoder->ip;
3478	decoder->state.to_ip = 0;
3479	return 0;
3480
3481	case INTEL_PT_BEP_IP:
3482	err = intel_pt_get_next_packet(decoder);
3483	if (err)
3484	return err;
3485	if (decoder->packet.type == INTEL_PT_FUP) {
3486	decoder->set_fup_bep = true;
3487	no_tip = true;
3488	} else {
3489	intel_pt_log_at("ERROR: Missing FUP after BEP",
3490	decoder->pos);
3491	}
3492	goto next;
3493
3494	case INTEL_PT_CFE:
3495	decoder->fup_cfe_pkt = decoder->packet;
3496	decoder->set_fup_cfe = true;
3497	if (!decoder->pge) {
3498	intel_pt_fup_event(decoder, true);
3499	return 0;
3500	}
3501	break;
3502
3503	case INTEL_PT_CFE_IP:
3504	decoder->fup_cfe_pkt = decoder->packet;
3505	err = intel_pt_get_next_packet(decoder);
3506	if (err)
3507	return err;
3508	if (decoder->packet.type == INTEL_PT_FUP) {
3509	decoder->set_fup_cfe_ip = true;
3510	no_tip = true;
3511	} else {
3512	intel_pt_log_at("ERROR: Missing FUP after CFE",
3513	decoder->pos);
3514	}
3515	goto next;
3516
3517	case INTEL_PT_EVD:
3518	err = intel_pt_evd(decoder);
3519	if (err)
3520	return err;
3521	break;
3522
3523	default:
3524	return intel_pt_bug(decoder);
3525	}
3526	}
3527	}
3528
3529	static inline bool intel_pt_have_ip(struct intel_pt_decoder *decoder)
3530	{
3531	return decoder->packet.count &&
3532	(decoder->have_last_ip \|\| decoder->packet.count == 3 \|\|
3533	decoder->packet.count == 6);
3534	}
3535
3536	/* Walk PSB+ packets to get in sync. */
3537	static int intel_pt_walk_psb(struct intel_pt_decoder *decoder)
3538	{
3539	int err;
3540
3541	decoder->in_psb = true;
3542
3543	while (1) {
3544	err = intel_pt_get_next_packet(decoder);
3545	if (err)
3546	goto out;
3547
3548	switch (decoder->packet.type) {
3549	case INTEL_PT_TIP_PGD:
3550	decoder->continuous_period = false;
3551	fallthrough;
3552	case INTEL_PT_TIP_PGE:
3553	case INTEL_PT_TIP:
3554	case INTEL_PT_PTWRITE:
3555	case INTEL_PT_PTWRITE_IP:
3556	case INTEL_PT_EXSTOP:
3557	case INTEL_PT_EXSTOP_IP:
3558	case INTEL_PT_MWAIT:
3559	case INTEL_PT_PWRE:
3560	case INTEL_PT_PWRX:
3561	case INTEL_PT_BBP:
3562	case INTEL_PT_BIP:
3563	case INTEL_PT_BEP:
3564	case INTEL_PT_BEP_IP:
3565	case INTEL_PT_CFE:
3566	case INTEL_PT_CFE_IP:
3567	case INTEL_PT_EVD:
3568	intel_pt_log("ERROR: Unexpected packet\n");
3569	err = -ENOENT;
3570	goto out;
3571
3572	case INTEL_PT_FUP:
3573	decoder->pge = true;
3574	if (intel_pt_have_ip(decoder)) {
3575	uint64_t current_ip = decoder->ip;
3576
3577	intel_pt_set_ip(decoder);
3578	decoder->psb_ip = decoder->ip;
3579	if (current_ip)
3580	intel_pt_log_to("Setting IP",
3581	decoder->ip);
3582	}
3583	break;
3584
3585	case INTEL_PT_MTC:
3586	intel_pt_calc_mtc_timestamp(decoder);
3587	break;
3588
3589	case INTEL_PT_TSC:
3590	intel_pt_calc_tsc_timestamp(decoder);
3591	break;
3592
3593	case INTEL_PT_TMA:
3594	intel_pt_calc_tma(decoder);
3595	break;
3596
3597	case INTEL_PT_CYC:
3598	intel_pt_calc_cyc_timestamp(decoder);
3599	break;
3600
3601	case INTEL_PT_CBR:
3602	intel_pt_calc_cbr(decoder);
3603	break;
3604
3605	case INTEL_PT_PIP:
3606	intel_pt_set_pip(decoder);
3607	break;
3608
3609	case INTEL_PT_MODE_EXEC:
3610	intel_pt_mode_exec_status(decoder);
3611	break;
3612
3613	case INTEL_PT_MODE_TSX:
3614	intel_pt_update_in_tx(decoder);
3615	break;
3616
3617	case INTEL_PT_TRACESTOP:
3618	decoder->pge = false;
3619	decoder->continuous_period = false;
3620	intel_pt_clear_tx_flags(decoder);
3621	fallthrough;
3622
3623	case INTEL_PT_TNT:
3624	decoder->have_tma = false;
3625	intel_pt_log("ERROR: Unexpected packet\n");
3626	if (decoder->ip)
3627	decoder->pkt_state = INTEL_PT_STATE_ERR4;
3628	else
3629	decoder->pkt_state = INTEL_PT_STATE_ERR3;
3630	err = -ENOENT;
3631	goto out;
3632
3633	case INTEL_PT_BAD: /* Does not happen */
3634	err = intel_pt_bug(decoder);
3635	goto out;
3636
3637	case INTEL_PT_OVF:
3638	err = intel_pt_overflow(decoder);
3639	goto out;
3640
3641	case INTEL_PT_PSBEND:
3642	err = 0;
3643	goto out;
3644
3645	case INTEL_PT_PSB:
3646	case INTEL_PT_VMCS:
3647	case INTEL_PT_MNT:
3648	case INTEL_PT_PAD:
3649	default:
3650	break;
3651	}
3652	}
3653	out:
3654	decoder->in_psb = false;
3655
3656	return err;
3657	}
3658
3659	static int intel_pt_walk_to_ip(struct intel_pt_decoder *decoder)
3660	{
3661	int err;
3662
3663	while (1) {
3664	err = intel_pt_get_next_packet(decoder);
3665	if (err)
3666	return err;
3667
3668	switch (decoder->packet.type) {
3669	case INTEL_PT_TIP_PGD:
3670	decoder->continuous_period = false;
3671	decoder->pge = false;
3672	if (intel_pt_have_ip(decoder))
3673	intel_pt_set_ip(decoder);
3674	if (!decoder->ip)
3675	break;
3676	decoder->state.type \|= INTEL_PT_TRACE_END;
3677	return 0;
3678
3679	case INTEL_PT_TIP_PGE:
3680	decoder->pge = true;
3681	intel_pt_mtc_cyc_cnt_pge(decoder);
3682	if (intel_pt_have_ip(decoder))
3683	intel_pt_set_ip(decoder);
3684	if (!decoder->ip)
3685	break;
3686	decoder->state.type \|= INTEL_PT_TRACE_BEGIN;
3687	return 0;
3688
3689	case INTEL_PT_TIP:
3690	decoder->pge = true;
3691	if (intel_pt_have_ip(decoder))
3692	intel_pt_set_ip(decoder);
3693	if (!decoder->ip)
3694	break;
3695	return 0;
3696
3697	case INTEL_PT_FUP:
3698	if (intel_pt_have_ip(decoder))
3699	intel_pt_set_ip(decoder);
3700	if (decoder->ip)
3701	return 0;
3702	break;
3703
3704	case INTEL_PT_MTC:
3705	intel_pt_calc_mtc_timestamp(decoder);
3706	break;
3707
3708	case INTEL_PT_TSC:
3709	intel_pt_calc_tsc_timestamp(decoder);
3710	break;
3711
3712	case INTEL_PT_TMA:
3713	intel_pt_calc_tma(decoder);
3714	break;
3715
3716	case INTEL_PT_CYC:
3717	intel_pt_calc_cyc_timestamp(decoder);
3718	break;
3719
3720	case INTEL_PT_CBR:
3721	intel_pt_calc_cbr(decoder);
3722	break;
3723
3724	case INTEL_PT_PIP:
3725	intel_pt_set_pip(decoder);
3726	break;
3727
3728	case INTEL_PT_MODE_EXEC:
3729	intel_pt_mode_exec_status(decoder);
3730	break;
3731
3732	case INTEL_PT_MODE_TSX:
3733	intel_pt_update_in_tx(decoder);
3734	break;
3735
3736	case INTEL_PT_OVF:
3737	return intel_pt_overflow(decoder);
3738
3739	case INTEL_PT_BAD: /* Does not happen */
3740	return intel_pt_bug(decoder);
3741
3742	case INTEL_PT_TRACESTOP:
3743	decoder->pge = false;
3744	decoder->continuous_period = false;
3745	intel_pt_clear_tx_flags(decoder);
3746	decoder->have_tma = false;
3747	break;
3748
3749	case INTEL_PT_PSB:
3750	decoder->state.psb_offset = decoder->pos;
3751	decoder->psb_ip = 0;
3752	decoder->last_ip = 0;
3753	decoder->have_last_ip = true;
3754	intel_pt_clear_stack(&decoder->stack);
3755	err = intel_pt_walk_psb(decoder);
3756	if (err)
3757	return err;
3758	decoder->state.type = INTEL_PT_PSB_EVT;
3759	decoder->state.from_ip = decoder->psb_ip;
3760	decoder->state.to_ip = 0;
3761	return 0;
3762
3763	case INTEL_PT_TNT:
3764	case INTEL_PT_PSBEND:
3765	case INTEL_PT_VMCS:
3766	case INTEL_PT_MNT:
3767	case INTEL_PT_PAD:
3768	case INTEL_PT_PTWRITE:
3769	case INTEL_PT_PTWRITE_IP:
3770	case INTEL_PT_EXSTOP:
3771	case INTEL_PT_EXSTOP_IP:
3772	case INTEL_PT_MWAIT:
3773	case INTEL_PT_PWRE:
3774	case INTEL_PT_PWRX:
3775	case INTEL_PT_BBP:
3776	case INTEL_PT_BIP:
3777	case INTEL_PT_BEP:
3778	case INTEL_PT_BEP_IP:
3779	case INTEL_PT_CFE:
3780	case INTEL_PT_CFE_IP:
3781	case INTEL_PT_EVD:
3782	default:
3783	break;
3784	}
3785	}
3786	}
3787
3788	static int intel_pt_sync_ip(struct intel_pt_decoder *decoder)
3789	{
3790	int err;
3791
3792	intel_pt_clear_fup_event(decoder);
3793	decoder->overflow = false;
3794
3795	if (!decoder->branch_enable) {
3796	decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
3797	decoder->state.type = 0; /* Do not have a sample */
3798	return 0;
3799	}
3800
3801	intel_pt_log("Scanning for full IP\n");
3802	err = intel_pt_walk_to_ip(decoder);
3803	if (err \|\| ((decoder->state.type & INTEL_PT_PSB_EVT) && !decoder->ip))
3804	return err;
3805
3806	/* In hop mode, resample to get the to_ip as an "instruction" sample */
3807	if (decoder->hop)
3808	decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;
3809	else
3810	decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
3811
3812	decoder->state.from_ip = 0;
3813	decoder->state.to_ip = decoder->ip;
3814	intel_pt_log_to("Setting IP", decoder->ip);
3815
3816	return 0;
3817	}
3818
3819	static int intel_pt_part_psb(struct intel_pt_decoder *decoder)
3820	{
3821	const unsigned char *end = decoder->buf + decoder->len;
3822	size_t i;
3823
3824	for (i = INTEL_PT_PSB_LEN - 1; i; i--) {
3825	if (i > decoder->len)
3826	continue;
3827	if (!memcmp(end - i, INTEL_PT_PSB_STR, i))
3828	return i;
3829	}
3830	return 0;
3831	}
3832
3833	static int intel_pt_rest_psb(struct intel_pt_decoder *decoder, int part_psb)
3834	{
3835	size_t rest_psb = INTEL_PT_PSB_LEN - part_psb;
3836	const char *psb = INTEL_PT_PSB_STR;
3837
3838	if (rest_psb > decoder->len \|\|
3839	memcmp(decoder->buf, psb + part_psb, rest_psb))
3840	return 0;
3841
3842	return rest_psb;
3843	}
3844
3845	static int intel_pt_get_split_psb(struct intel_pt_decoder *decoder,
3846	int part_psb)
3847	{
3848	int rest_psb, ret;
3849
3850	decoder->pos += decoder->len;
3851	decoder->len = 0;
3852
3853	ret = intel_pt_get_next_data(decoder, false);
3854	if (ret)
3855	return ret;
3856
3857	rest_psb = intel_pt_rest_psb(decoder, part_psb);
3858	if (!rest_psb)
3859	return 0;
3860
3861	decoder->pos -= part_psb;
3862	decoder->next_buf = decoder->buf + rest_psb;
3863	decoder->next_len = decoder->len - rest_psb;
3864	memcpy(decoder->temp_buf, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
3865	decoder->buf = decoder->temp_buf;
3866	decoder->len = INTEL_PT_PSB_LEN;
3867
3868	return 0;
3869	}
3870
3871	static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder)
3872	{
3873	unsigned char *next;
3874	int ret;
3875
3876	intel_pt_log("Scanning for PSB\n");
3877	while (1) {
3878	if (!decoder->len) {
3879	ret = intel_pt_get_next_data(decoder, false);
3880	if (ret)
3881	return ret;
3882	}
3883
3884	next = memmem(decoder->buf, decoder->len, INTEL_PT_PSB_STR,
3885	INTEL_PT_PSB_LEN);
3886	if (!next) {
3887	int part_psb;
3888
3889	part_psb = intel_pt_part_psb(decoder);
3890	if (part_psb) {
3891	ret = intel_pt_get_split_psb(decoder, part_psb);
3892	if (ret)
3893	return ret;
3894	} else {
3895	decoder->pos += decoder->len;
3896	decoder->len = 0;
3897	}
3898	continue;
3899	}
3900
3901	decoder->pkt_step = next - decoder->buf;
3902	return intel_pt_get_next_packet(decoder);
3903	}
3904	}
3905
3906	static int intel_pt_sync(struct intel_pt_decoder *decoder)
3907	{
3908	int err;
3909
3910	decoder->pge = false;
3911	decoder->continuous_period = false;
3912	decoder->have_last_ip = false;
3913	decoder->last_ip = 0;
3914	decoder->psb_ip = 0;
3915	decoder->ip = 0;
3916	intel_pt_clear_stack(&decoder->stack);
3917
3918	err = intel_pt_scan_for_psb(decoder);
3919	if (err)
3920	return err;
3921
3922	if (decoder->vm_time_correlation) {
3923	decoder->in_psb = true;
3924	if (!decoder->timestamp)
3925	decoder->timestamp = 1;
3926	decoder->state.type = 0;
3927	decoder->pkt_state = INTEL_PT_STATE_VM_TIME_CORRELATION;
3928	return 0;
3929	}
3930
3931	decoder->have_last_ip = true;
3932	decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
3933
3934	err = intel_pt_walk_psb(decoder);
3935	if (err)
3936	return err;
3937
3938	decoder->state.type = INTEL_PT_PSB_EVT; /* Only PSB sample */
3939	decoder->state.from_ip = decoder->psb_ip;
3940	decoder->state.to_ip = 0;
3941
3942	if (decoder->ip) {
3943	/*
3944	* In hop mode, resample to get the PSB FUP ip as an
3945	* "instruction" sample.
3946	*/
3947	if (decoder->hop)
3948	decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;
3949	else
3950	decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
3951	}
3952
3953	return 0;
3954	}
3955
3956	static uint64_t intel_pt_est_timestamp(struct intel_pt_decoder *decoder)
3957	{
3958	uint64_t est = decoder->sample_insn_cnt << 1;
3959
3960	if (!decoder->cbr \|\| !decoder->max_non_turbo_ratio)
3961	goto out;
3962
3963	est *= decoder->max_non_turbo_ratio;
3964	est /= decoder->cbr;
3965	out:
3966	return decoder->sample_timestamp + est;
3967	}
3968
3969	const struct intel_pt_state intel_pt_decode(struct intel_pt_decoder decoder)
3970	{
3971	int err;
3972
3973	do {
3974	decoder->state.type = INTEL_PT_BRANCH;
3975	decoder->state.flags = 0;
3976
3977	switch (decoder->pkt_state) {
3978	case INTEL_PT_STATE_NO_PSB:
3979	err = intel_pt_sync(decoder);
3980	break;
3981	case INTEL_PT_STATE_NO_IP:
3982	decoder->have_last_ip = false;
3983	decoder->last_ip = 0;
3984	decoder->ip = 0;
3985	fallthrough;
3986	case INTEL_PT_STATE_ERR_RESYNC:
3987	err = intel_pt_sync_ip(decoder);
3988	break;
3989	case INTEL_PT_STATE_IN_SYNC:
3990	err = intel_pt_walk_trace(decoder);
3991	break;
3992	case INTEL_PT_STATE_TNT:
3993	case INTEL_PT_STATE_TNT_CONT:
3994	err = intel_pt_walk_tnt(decoder);
3995	if (err == -EAGAIN)
3996	err = intel_pt_walk_trace(decoder);
3997	break;
3998	case INTEL_PT_STATE_TIP:
3999	case INTEL_PT_STATE_TIP_PGD:
4000	err = intel_pt_walk_tip(decoder);
4001	break;
4002	case INTEL_PT_STATE_FUP:
4003	err = intel_pt_walk_fup(decoder);
4004	if (err == -EAGAIN)
4005	err = intel_pt_walk_fup_tip(decoder);
4006	break;
4007	case INTEL_PT_STATE_FUP_NO_TIP:
4008	err = intel_pt_walk_fup(decoder);
4009	if (err == -EAGAIN)
4010	err = intel_pt_walk_trace(decoder);
4011	break;
4012	case INTEL_PT_STATE_FUP_IN_PSB:
4013	err = intel_pt_fup_in_psb(decoder);
4014	break;
4015	case INTEL_PT_STATE_RESAMPLE:
4016	err = intel_pt_resample(decoder);
4017	break;
4018	case INTEL_PT_STATE_VM_TIME_CORRELATION:
4019	err = intel_pt_vm_time_correlation(decoder);
4020	break;
4021	default:
4022	err = intel_pt_bug(decoder);
4023	break;
4024	}
4025	} while (err == -ENOLINK);
4026
4027	if (err) {
4028	decoder->state.err = intel_pt_ext_err(err);
4029	if (err != -EOVERFLOW)
4030	decoder->state.from_ip = decoder->ip;
4031	intel_pt_update_sample_time(decoder);
4032	decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt;
4033	intel_pt_set_nr(decoder);
4034	} else {
4035	decoder->state.err = 0;
4036	if (decoder->cbr != decoder->cbr_seen) {
4037	decoder->cbr_seen = decoder->cbr;
4038	if (!decoder->state.type) {
4039	decoder->state.from_ip = decoder->ip;
4040	decoder->state.to_ip = 0;
4041	}
4042	decoder->state.type \|= INTEL_PT_CBR_CHG;
4043	decoder->state.cbr_payload = decoder->cbr_payload;
4044	decoder->state.cbr = decoder->cbr;
4045	}
4046	if (intel_pt_sample_time(decoder->pkt_state)) {
4047	intel_pt_update_sample_time(decoder);
4048	if (decoder->sample_cyc) {
4049	decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt;
4050	decoder->state.flags \|= INTEL_PT_SAMPLE_IPC;
4051	decoder->sample_cyc = false;
4052	}
4053	}
4054	/*
4055	* When using only TSC/MTC to compute cycles, IPC can be
4056	* sampled as soon as the cycle count changes.
4057	*/
4058	if (!decoder->have_cyc)
4059	decoder->state.flags \|= INTEL_PT_SAMPLE_IPC;
4060	}
4061
4062	/* Let PSB event always have TSC timestamp */
4063	if ((decoder->state.type & INTEL_PT_PSB_EVT) && decoder->tsc_timestamp)
4064	decoder->sample_timestamp = decoder->tsc_timestamp;
4065
4066	decoder->state.from_nr = decoder->nr;
4067	decoder->state.to_nr = decoder->next_nr;
4068	decoder->nr = decoder->next_nr;
4069
4070	decoder->state.timestamp = decoder->sample_timestamp;
4071	decoder->state.est_timestamp = intel_pt_est_timestamp(decoder);
4072	decoder->state.tot_insn_cnt = decoder->tot_insn_cnt;
4073	decoder->state.tot_cyc_cnt = decoder->sample_tot_cyc_cnt;
4074
4075	return &decoder->state;
4076	}
4077
4078	/**
4079	* intel_pt_next_psb - move buffer pointer to the start of the next PSB packet.
4080	* @buf: pointer to buffer pointer
4081	* @len: size of buffer
4082	*
4083	* Updates the buffer pointer to point to the start of the next PSB packet if
4084	* there is one, otherwise the buffer pointer is unchanged. If @buf is updated,
4085	* @len is adjusted accordingly.
4086	*
4087	* Return: %true if a PSB packet is found, %false otherwise.
4088	*/
4089	static bool intel_pt_next_psb(unsigned char *buf, size_t len)
4090	{
4091	unsigned char *next;
4092
4093	next = memmem(buf, len, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
4094	if (next) {
4095	len -= next - buf;
4096	*buf = next;
4097	return true;
4098	}
4099	return false;
4100	}
4101
4102	/**
4103	* intel_pt_step_psb - move buffer pointer to the start of the following PSB
4104	* packet.
4105	* @buf: pointer to buffer pointer
4106	* @len: size of buffer
4107	*
4108	* Updates the buffer pointer to point to the start of the following PSB packet
4109	* (skipping the PSB at @buf itself) if there is one, otherwise the buffer
4110	* pointer is unchanged. If @buf is updated, @len is adjusted accordingly.
4111	*
4112	* Return: %true if a PSB packet is found, %false otherwise.
4113	*/
4114	static bool intel_pt_step_psb(unsigned char *buf, size_t len)
4115	{
4116	unsigned char *next;
4117
4118	if (!*len)
4119	return false;
4120
4121	next = memmem(buf + 1, len - 1, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
4122	if (next) {
4123	len -= next - buf;
4124	*buf = next;
4125	return true;
4126	}
4127	return false;
4128	}
4129
4130	/**
4131	* intel_pt_last_psb - find the last PSB packet in a buffer.
4132	* @buf: buffer
4133	* @len: size of buffer
4134	*
4135	* This function finds the last PSB in a buffer.
4136	*
4137	* Return: A pointer to the last PSB in @buf if found, %NULL otherwise.
4138	*/
4139	static unsigned char intel_pt_last_psb(unsigned char buf, size_t len)
4140	{
4141	const char *n = INTEL_PT_PSB_STR;
4142	unsigned char *p;
4143	size_t k;
4144
4145	if (len < INTEL_PT_PSB_LEN)
4146	return NULL;
4147
4148	k = len - INTEL_PT_PSB_LEN + 1;
4149	while (1) {
4150	p = memrchr(buf, n[0], k);
4151	if (!p)
4152	return NULL;
4153	if (!memcmp(p + 1, n + 1, INTEL_PT_PSB_LEN - 1))
4154	return p;
4155	k = p - buf;
4156	if (!k)
4157	return NULL;
4158	}
4159	}
4160
4161	/**
4162	* intel_pt_next_tsc - find and return next TSC.
4163	* @buf: buffer
4164	* @len: size of buffer
4165	* @tsc: TSC value returned
4166	* @rem: returns remaining size when TSC is found
4167	*
4168	* Find a TSC packet in @buf and return the TSC value. This function assumes
4169	* that @buf starts at a PSB and that PSB+ will contain TSC and so stops if a
4170	* PSBEND packet is found.
4171	*
4172	* Return: %true if TSC is found, false otherwise.
4173	*/
4174	static bool intel_pt_next_tsc(unsigned char buf, size_t len, uint64_t tsc,
4175	size_t *rem)
4176	{
4177	enum intel_pt_pkt_ctx ctx = INTEL_PT_NO_CTX;
4178	struct intel_pt_pkt packet;
4179	int ret;
4180
4181	while (len) {
4182	ret = intel_pt_get_packet(buf, len, &packet, &ctx);
4183	if (ret <= 0)
4184	return false;
4185	if (packet.type == INTEL_PT_TSC) {
4186	*tsc = packet.payload;
4187	*rem = len;
4188	return true;
4189	}
4190	if (packet.type == INTEL_PT_PSBEND)
4191	return false;
4192	buf += ret;
4193	len -= ret;
4194	}
4195	return false;
4196	}
4197
4198	/**
4199	* intel_pt_tsc_cmp - compare 7-byte TSCs.
4200	* @tsc1: first TSC to compare
4201	* @tsc2: second TSC to compare
4202	*
4203	* This function compares 7-byte TSC values allowing for the possibility that
4204	* TSC wrapped around. Generally it is not possible to know if TSC has wrapped
4205	* around so for that purpose this function assumes the absolute difference is
4206	* less than half the maximum difference.
4207	*
4208	* Return: %-1 if @tsc1 is before @tsc2, %0 if @tsc1 == @tsc2, %1 if @tsc1 is
4209	* after @tsc2.
4210	*/
4211	static int intel_pt_tsc_cmp(uint64_t tsc1, uint64_t tsc2)
4212	{
4213	const uint64_t halfway = (1ULL << 55);
4214
4215	if (tsc1 == tsc2)
4216	return 0;
4217
4218	if (tsc1 < tsc2) {
4219	if (tsc2 - tsc1 < halfway)
4220	return -1;
4221	else
4222	return 1;
4223	} else {
4224	if (tsc1 - tsc2 < halfway)
4225	return 1;
4226	else
4227	return -1;
4228	}
4229	}
4230
4231	#define MAX_PADDING (PERF_AUXTRACE_RECORD_ALIGNMENT - 1)
4232
4233	/**
4234	* adj_for_padding - adjust overlap to account for padding.
4235	* @buf_b: second buffer
4236	* @buf_a: first buffer
4237	* @len_a: size of first buffer
4238	*
4239	* @buf_a might have up to 7 bytes of padding appended. Adjust the overlap
4240	* accordingly.
4241	*
4242	* Return: A pointer into @buf_b from where non-overlapped data starts
4243	*/
4244	static unsigned char adj_for_padding(unsigned char buf_b,
4245	unsigned char *buf_a, size_t len_a)
4246	{
4247	unsigned char *p = buf_b - MAX_PADDING;
4248	unsigned char *q = buf_a + len_a - MAX_PADDING;
4249	int i;
4250
4251	for (i = MAX_PADDING; i; i--, p++, q++) {
4252	if (p != q)
4253	break;
4254	}
4255
4256	return p;
4257	}
4258
4259	/**
4260	* intel_pt_find_overlap_tsc - determine start of non-overlapped trace data
4261	* using TSC.
4262	* @buf_a: first buffer
4263	* @len_a: size of first buffer
4264	* @buf_b: second buffer
4265	* @len_b: size of second buffer
4266	* @consecutive: returns true if there is data in buf_b that is consecutive
4267	* to buf_a
4268	* @ooo_tsc: out-of-order TSC due to VM TSC offset / scaling
4269	*
4270	* If the trace contains TSC we can look at the last TSC of @buf_a and the
4271	* first TSC of @buf_b in order to determine if the buffers overlap, and then
4272	* walk forward in @buf_b until a later TSC is found. A precondition is that
4273	* @buf_a and @buf_b are positioned at a PSB.
4274	*
4275	* Return: A pointer into @buf_b from where non-overlapped data starts, or
4276	* @buf_b + @len_b if there is no non-overlapped data.
4277	*/
4278	static unsigned char intel_pt_find_overlap_tsc(unsigned char buf_a,
4279	size_t len_a,
4280	unsigned char *buf_b,
4281	size_t len_b, bool *consecutive,
4282	bool ooo_tsc)
4283	{
4284	uint64_t tsc_a, tsc_b;
4285	unsigned char *p;
4286	size_t len, rem_a, rem_b;
4287
4288	p = intel_pt_last_psb(buf_a, len_a);
4289	if (!p)
4290	return buf_b; /* No PSB in buf_a => no overlap */
4291
4292	len = len_a - (p - buf_a);
4293	if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a)) {
4294	/* The last PSB+ in buf_a is incomplete, so go back one more */
4295	len_a -= len;
4296	p = intel_pt_last_psb(buf_a, len_a);
4297	if (!p)
4298	return buf_b; /* No full PSB+ => assume no overlap */
4299	len = len_a - (p - buf_a);
4300	if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a))
4301	return buf_b; /* No TSC in buf_a => assume no overlap */
4302	}
4303
4304	while (1) {
4305	/* Ignore PSB+ with no TSC */
4306	if (intel_pt_next_tsc(buf_b, len_b, &tsc_b, &rem_b)) {
4307	int cmp = intel_pt_tsc_cmp(tsc_a, tsc_b);
4308
4309	/* Same TSC, so buffers are consecutive */
4310	if (!cmp && rem_b >= rem_a) {
4311	unsigned char *start;
4312
4313	*consecutive = true;
4314	start = buf_b + len_b - (rem_b - rem_a);
4315	return adj_for_padding(start, buf_a, len_a);
4316	}
4317	if (cmp < 0 && !ooo_tsc)
4318	return buf_b; /* tsc_a < tsc_b => no overlap */
4319	}
4320
4321	if (!intel_pt_step_psb(&buf_b, &len_b))
4322	return buf_b + len_b; /* No PSB in buf_b => no data */
4323	}
4324	}
4325
4326	/**
4327	* intel_pt_find_overlap - determine start of non-overlapped trace data.
4328	* @buf_a: first buffer
4329	* @len_a: size of first buffer
4330	* @buf_b: second buffer
4331	* @len_b: size of second buffer
4332	* @have_tsc: can use TSC packets to detect overlap
4333	* @consecutive: returns true if there is data in buf_b that is consecutive
4334	* to buf_a
4335	* @ooo_tsc: out-of-order TSC due to VM TSC offset / scaling
4336	*
4337	* When trace samples or snapshots are recorded there is the possibility that
4338	* the data overlaps. Note that, for the purposes of decoding, data is only
4339	* useful if it begins with a PSB packet.
4340	*
4341	* Return: A pointer into @buf_b from where non-overlapped data starts, or
4342	* @buf_b + @len_b if there is no non-overlapped data.
4343	*/
4344	unsigned char intel_pt_find_overlap(unsigned char buf_a, size_t len_a,
4345	unsigned char *buf_b, size_t len_b,
4346	bool have_tsc, bool *consecutive,
4347	bool ooo_tsc)
4348	{
4349	unsigned char *found;
4350
4351	/* Buffer 'b' must start at PSB so throw away everything before that */
4352	if (!intel_pt_next_psb(&buf_b, &len_b))
4353	return buf_b + len_b; /* No PSB */
4354
4355	if (!intel_pt_next_psb(&buf_a, &len_a))
4356	return buf_b; /* No overlap */
4357
4358	if (have_tsc) {
4359	found = intel_pt_find_overlap_tsc(buf_a, len_a, buf_b, len_b,
4360	consecutive, ooo_tsc);
4361	if (found)
4362	return found;
4363	}
4364
4365	/*
4366	* Buffer 'b' cannot end within buffer 'a' so, for comparison purposes,
4367	* we can ignore the first part of buffer 'a'.
4368	*/
4369	while (len_b < len_a) {
4370	if (!intel_pt_step_psb(&buf_a, &len_a))
4371	return buf_b; /* No overlap */
4372	}
4373
4374	/* Now len_b >= len_a */
4375	while (1) {
4376	/* Potential overlap so check the bytes */
4377	found = memmem(buf_a, len_a, buf_b, len_a);
4378	if (found) {
4379	*consecutive = true;
4380	return adj_for_padding(buf_b + len_a, buf_a, len_a);
4381	}
4382
4383	/* Try again at next PSB in buffer 'a' */
4384	if (!intel_pt_step_psb(&buf_a, &len_a))
4385	return buf_b; /* No overlap */
4386	}
4387	}
4388
4389	/**
4390	* struct fast_forward_data - data used by intel_pt_ff_cb().
4391	* @timestamp: timestamp to fast forward towards
4392	* @buf_timestamp: buffer timestamp of last buffer with trace data earlier than
4393	* the fast forward timestamp.
4394	*/
4395	struct fast_forward_data {
4396	uint64_t timestamp;
4397	uint64_t buf_timestamp;
4398	};
4399
4400	/**
4401	* intel_pt_ff_cb - fast forward lookahead callback.
4402	* @buffer: Intel PT trace buffer
4403	* @data: opaque pointer to fast forward data (struct fast_forward_data)
4404	*
4405	* Determine if @buffer trace is past the fast forward timestamp.
4406	*
4407	* Return: 1 (stop lookahead) if @buffer trace is past the fast forward
4408	* timestamp, and 0 otherwise.
4409	*/
4410	static int intel_pt_ff_cb(struct intel_pt_buffer buffer, void data)
4411	{
4412	struct fast_forward_data *d = data;
4413	unsigned char *buf;
4414	uint64_t tsc;
4415	size_t rem;
4416	size_t len;
4417
4418	buf = (unsigned char *)buffer->buf;
4419	len = buffer->len;
4420
4421	if (!intel_pt_next_psb(&buf, &len) \|\|
4422	!intel_pt_next_tsc(buf, len, &tsc, &rem))
4423	return 0;
4424
4425	tsc = intel_pt_8b_tsc(tsc, buffer->ref_timestamp);
4426
4427	intel_pt_log("Buffer 1st timestamp " x64_fmt " ref timestamp " x64_fmt "\n",
4428	tsc, buffer->ref_timestamp);
4429
4430	/*
4431	* If the buffer contains a timestamp earlier that the fast forward
4432	* timestamp, then record it, else stop.
4433	*/
4434	if (tsc < d->timestamp)
4435	d->buf_timestamp = buffer->ref_timestamp;
4436	else
4437	return 1;
4438
4439	return 0;
4440	}
4441
4442	/**
4443	* intel_pt_fast_forward - reposition decoder forwards.
4444	* @decoder: Intel PT decoder
4445	* @timestamp: timestamp to fast forward towards
4446	*
4447	* Reposition decoder at the last PSB with a timestamp earlier than @timestamp.
4448	*
4449	* Return: 0 on success or negative error code on failure.
4450	*/
4451	int intel_pt_fast_forward(struct intel_pt_decoder *decoder, uint64_t timestamp)
4452	{
4453	struct fast_forward_data d = { .timestamp = timestamp };
4454	unsigned char *buf;
4455	size_t len;
4456	int err;
4457
4458	intel_pt_log("Fast forward towards timestamp " x64_fmt "\n", timestamp);
4459
4460	/* Find buffer timestamp of buffer to fast forward to */
4461	err = decoder->lookahead(decoder->data, intel_pt_ff_cb, &d);
4462	if (err < 0)
4463	return err;
4464
4465	/* Walk to buffer with same buffer timestamp */
4466	if (d.buf_timestamp) {
4467	do {
4468	decoder->pos += decoder->len;
4469	decoder->len = 0;
4470	err = intel_pt_get_next_data(decoder, true);
4471	/* -ENOLINK means non-consecutive trace */
4472	if (err && err != -ENOLINK)
4473	return err;
4474	} while (decoder->buf_timestamp != d.buf_timestamp);
4475	}
4476
4477	if (!decoder->buf)
4478	return 0;
4479
4480	buf = (unsigned char *)decoder->buf;
4481	len = decoder->len;
4482
4483	if (!intel_pt_next_psb(&buf, &len))
4484	return 0;
4485
4486	/*
4487	* Walk PSBs while the PSB timestamp is less than the fast forward
4488	* timestamp.
4489	*/
4490	do {
4491	uint64_t tsc;
4492	size_t rem;
4493
4494	if (!intel_pt_next_tsc(buf, len, &tsc, &rem))
4495	break;
4496	tsc = intel_pt_8b_tsc(tsc, decoder->buf_timestamp);
4497	/*
4498	* A TSC packet can slip past MTC packets but, after fast
4499	* forward, decoding starts at the TSC timestamp. That means
4500	* the timestamps may not be exactly the same as the timestamps
4501	* that would have been decoded without fast forward.
4502	*/
4503	if (tsc < timestamp) {
4504	intel_pt_log("Fast forward to next PSB timestamp " x64_fmt "\n", tsc);
4505	decoder->pos += decoder->len - len;
4506	decoder->buf = buf;
4507	decoder->len = len;
4508	intel_pt_reposition(decoder);
4509	} else {
4510	break;
4511	}
4512	} while (intel_pt_step_psb(&buf, &len));
4513
4514	return 0;
4515	}