linux - tools/perf/util/intel-pt.c

4748 lines · 119.0 KiB

1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* intel_pt.c: Intel Processor Trace support
4	* Copyright (c) 2013-2015, Intel Corporation.
5	*/
6
7	#include <inttypes.h>
8	#include <linux/perf_event.h>
9	#include <stdio.h>
10	#include <stdbool.h>
11	#include <errno.h>
12	#include <linux/kernel.h>
13	#include <linux/string.h>
14	#include <linux/types.h>
15	#include <linux/zalloc.h>
16
17	#include "session.h"
18	#include "machine.h"
19	#include "memswap.h"
20	#include "sort.h"
21	#include "tool.h"
22	#include "event.h"
23	#include "evlist.h"
24	#include "evsel.h"
25	#include "map.h"
26	#include "color.h"
27	#include "thread.h"
28	#include "thread-stack.h"
29	#include "symbol.h"
30	#include "callchain.h"
31	#include "dso.h"
32	#include "debug.h"
33	#include "auxtrace.h"
34	#include "tsc.h"
35	#include "intel-pt.h"
36	#include "config.h"
37	#include "util/perf_api_probe.h"
38	#include "util/synthetic-events.h"
39	#include "time-utils.h"
40
41	#include "../arch/x86/include/uapi/asm/perf_regs.h"
42
43	#include "intel-pt-decoder/intel-pt-log.h"
44	#include "intel-pt-decoder/intel-pt-decoder.h"
45	#include "intel-pt-decoder/intel-pt-insn-decoder.h"
46	#include "intel-pt-decoder/intel-pt-pkt-decoder.h"
47
48	#define MAX_TIMESTAMP (~0ULL)
49
50	#define INTEL_PT_CFG_PASS_THRU BIT_ULL(0)
51	#define INTEL_PT_CFG_PWR_EVT_EN BIT_ULL(4)
52	#define INTEL_PT_CFG_BRANCH_EN BIT_ULL(13)
53	#define INTEL_PT_CFG_EVT_EN BIT_ULL(31)
54	#define INTEL_PT_CFG_TNT_DIS BIT_ULL(55)
55
56	struct range {
57	u64 start;
58	u64 end;
59	};
60
61	struct intel_pt {
62	struct auxtrace auxtrace;
63	struct auxtrace_queues queues;
64	struct auxtrace_heap heap;
65	u32 auxtrace_type;
66	struct perf_session *session;
67	struct machine *machine;
68	struct evsel *switch_evsel;
69	struct thread *unknown_thread;
70	bool timeless_decoding;
71	bool sampling_mode;
72	bool snapshot_mode;
73	bool per_cpu_mmaps;
74	bool have_tsc;
75	bool data_queued;
76	bool est_tsc;
77	bool sync_switch;
78	bool sync_switch_not_supported;
79	bool mispred_all;
80	bool use_thread_stack;
81	bool callstack;
82	bool cap_event_trace;
83	bool have_guest_sideband;
84	unsigned int br_stack_sz;
85	unsigned int br_stack_sz_plus;
86	int have_sched_switch;
87	u32 pmu_type;
88	u64 kernel_start;
89	u64 switch_ip;
90	u64 ptss_ip;
91	u64 first_timestamp;
92
93	struct perf_tsc_conversion tc;
94	bool cap_user_time_zero;
95
96	struct itrace_synth_opts synth_opts;
97
98	bool sample_instructions;
99	u64 instructions_sample_type;
100	u64 instructions_id;
101
102	bool sample_cycles;
103	u64 cycles_sample_type;
104	u64 cycles_id;
105
106	bool sample_branches;
107	u32 branches_filter;
108	u64 branches_sample_type;
109	u64 branches_id;
110
111	bool sample_transactions;
112	u64 transactions_sample_type;
113	u64 transactions_id;
114
115	bool sample_ptwrites;
116	u64 ptwrites_sample_type;
117	u64 ptwrites_id;
118
119	bool sample_pwr_events;
120	u64 pwr_events_sample_type;
121	u64 mwait_id;
122	u64 pwre_id;
123	u64 exstop_id;
124	u64 pwrx_id;
125	u64 cbr_id;
126	u64 psb_id;
127
128	bool single_pebs;
129	bool sample_pebs;
130	int pebs_data_src_fmt;
131	struct evsel *pebs_evsel;
132
133	u64 evt_sample_type;
134	u64 evt_id;
135
136	u64 iflag_chg_sample_type;
137	u64 iflag_chg_id;
138
139	u64 tsc_bit;
140	u64 mtc_bit;
141	u64 mtc_freq_bits;
142	u32 tsc_ctc_ratio_n;
143	u32 tsc_ctc_ratio_d;
144	u64 cyc_bit;
145	u64 noretcomp_bit;
146	unsigned max_non_turbo_ratio;
147	unsigned cbr2khz;
148	int max_loops;
149
150	unsigned long num_events;
151
152	char *filter;
153	struct addr_filters filts;
154
155	struct range *time_ranges;
156	unsigned int range_cnt;
157
158	struct ip_callchain *chain;
159	struct branch_stack *br_stack;
160
161	u64 dflt_tsc_offset;
162	struct rb_root vmcs_info;
163	};
164
165	enum switch_state {
166	INTEL_PT_SS_NOT_TRACING,
167	INTEL_PT_SS_UNKNOWN,
168	INTEL_PT_SS_TRACING,
169	INTEL_PT_SS_EXPECTING_SWITCH_EVENT,
170	INTEL_PT_SS_EXPECTING_SWITCH_IP,
171	};
172
173	/* applicable_counters is 64-bits */
174	#define INTEL_PT_MAX_PEBS 64
175
176	struct intel_pt_pebs_event {
177	struct evsel *evsel;
178	u64 id;
179	int data_src_fmt;
180	};
181
182	struct intel_pt_queue {
183	struct intel_pt *pt;
184	unsigned int queue_nr;
185	struct auxtrace_buffer *buffer;
186	struct auxtrace_buffer *old_buffer;
187	void *decoder;
188	const struct intel_pt_state *state;
189	struct ip_callchain *chain;
190	struct branch_stack *last_branch;
191	union perf_event *event_buf;
192	bool on_heap;
193	bool stop;
194	bool step_through_buffers;
195	bool use_buffer_pid_tid;
196	bool sync_switch;
197	bool sample_ipc;
198	pid_t pid, tid;
199	int cpu;
200	int switch_state;
201	pid_t next_tid;
202	struct thread *thread;
203	struct machine *guest_machine;
204	struct thread *guest_thread;
205	struct thread *unknown_guest_thread;
206	pid_t guest_machine_pid;
207	pid_t guest_pid;
208	pid_t guest_tid;
209	int vcpu;
210	bool exclude_kernel;
211	bool have_sample;
212	u64 time;
213	u64 timestamp;
214	u64 sel_timestamp;
215	bool sel_start;
216	unsigned int sel_idx;
217	u32 flags;
218	u16 insn_len;
219	u64 last_insn_cnt;
220	u64 ipc_insn_cnt;
221	u64 ipc_cyc_cnt;
222	u64 last_in_insn_cnt;
223	u64 last_in_cyc_cnt;
224	u64 last_cy_insn_cnt;
225	u64 last_cy_cyc_cnt;
226	u64 last_br_insn_cnt;
227	u64 last_br_cyc_cnt;
228	unsigned int cbr_seen;
229	char insn[INTEL_PT_INSN_BUF_SZ];
230	struct intel_pt_pebs_event pebs[INTEL_PT_MAX_PEBS];
231	};
232
233	static void intel_pt_dump(struct intel_pt *pt __maybe_unused,
234	unsigned char *buf, size_t len)
235	{
236	struct intel_pt_pkt packet;
237	size_t pos = 0;
238	int ret, pkt_len, i;
239	char desc[INTEL_PT_PKT_DESC_MAX];
240	const char *color = PERF_COLOR_BLUE;
241	enum intel_pt_pkt_ctx ctx = INTEL_PT_NO_CTX;
242
243	color_fprintf(stdout, color,
244	". ... Intel Processor Trace data: size %zu bytes\n",
245	len);
246
247	while (len) {
248	ret = intel_pt_get_packet(buf, len, &packet, &ctx);
249	if (ret > 0)
250	pkt_len = ret;
251	else
252	pkt_len = 1;
253	printf(".");
254	color_fprintf(stdout, color, " %08zx: ", pos);
255	for (i = 0; i < pkt_len; i++)
256	color_fprintf(stdout, color, " %02x", buf[i]);
257	for (; i < 16; i++)
258	color_fprintf(stdout, color, " ");
259	if (ret > 0) {
260	ret = intel_pt_pkt_desc(&packet, desc,
261	INTEL_PT_PKT_DESC_MAX);
262	if (ret > 0)
263	color_fprintf(stdout, color, " %s\n", desc);
264	} else {
265	color_fprintf(stdout, color, " Bad packet!\n");
266	}
267	pos += pkt_len;
268	buf += pkt_len;
269	len -= pkt_len;
270	}
271	}
272
273	static void intel_pt_dump_event(struct intel_pt pt, unsigned char buf,
274	size_t len)
275	{
276	printf(".\n");
277	intel_pt_dump(pt, buf, len);
278	}
279
280	static void intel_pt_log_event(union perf_event *event)
281	{
282	FILE *f = intel_pt_log_fp();
283
284	if (!intel_pt_enable_logging \|\| !f)
285	return;
286
287	perf_event__fprintf(event, NULL, f);
288	}
289
290	static void intel_pt_dump_sample(struct perf_session *session,
291	struct perf_sample *sample)
292	{
293	struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
294	auxtrace);
295
296	printf("\n");
297	intel_pt_dump(pt, sample->aux_sample.data, sample->aux_sample.size);
298	}
299
300	static bool intel_pt_log_events(struct intel_pt *pt, u64 tm)
301	{
302	struct perf_time_interval *range = pt->synth_opts.ptime_range;
303	int n = pt->synth_opts.range_num;
304
305	if (pt->synth_opts.log_plus_flags & AUXTRACE_LOG_FLG_ALL_PERF_EVTS)
306	return true;
307
308	if (pt->synth_opts.log_minus_flags & AUXTRACE_LOG_FLG_ALL_PERF_EVTS)
309	return false;
310
311	/* perf_time__ranges_skip_sample does not work if time is zero */
312	if (!tm)
313	tm = 1;
314
315	return !n \|\| !perf_time__ranges_skip_sample(range, n, tm);
316	}
317
318	static struct intel_pt_vmcs_info intel_pt_findnew_vmcs(struct rb_root rb_root,
319	u64 vmcs,
320	u64 dflt_tsc_offset)
321	{
322	struct rb_node **p = &rb_root->rb_node;
323	struct rb_node *parent = NULL;
324	struct intel_pt_vmcs_info *v;
325
326	while (*p) {
327	parent = *p;
328	v = rb_entry(parent, struct intel_pt_vmcs_info, rb_node);
329
330	if (v->vmcs == vmcs)
331	return v;
332
333	if (vmcs < v->vmcs)
334	p = &(*p)->rb_left;
335	else
336	p = &(*p)->rb_right;
337	}
338
339	v = zalloc(sizeof(*v));
340	if (v) {
341	v->vmcs = vmcs;
342	v->tsc_offset = dflt_tsc_offset;
343	v->reliable = dflt_tsc_offset;
344
345	rb_link_node(&v->rb_node, parent, p);
346	rb_insert_color(&v->rb_node, rb_root);
347	}
348
349	return v;
350	}
351
352	static struct intel_pt_vmcs_info intel_pt_findnew_vmcs_info(void data, uint64_t vmcs)
353	{
354	struct intel_pt_queue *ptq = data;
355	struct intel_pt *pt = ptq->pt;
356
357	if (!vmcs && !pt->dflt_tsc_offset)
358	return NULL;
359
360	return intel_pt_findnew_vmcs(&pt->vmcs_info, vmcs, pt->dflt_tsc_offset);
361	}
362
363	static void intel_pt_free_vmcs_info(struct intel_pt *pt)
364	{
365	struct intel_pt_vmcs_info *v;
366	struct rb_node *n;
367
368	n = rb_first(&pt->vmcs_info);
369	while (n) {
370	v = rb_entry(n, struct intel_pt_vmcs_info, rb_node);
371	n = rb_next(n);
372	rb_erase(&v->rb_node, &pt->vmcs_info);
373	free(v);
374	}
375	}
376
377	static int intel_pt_do_fix_overlap(struct intel_pt pt, struct auxtrace_buffer a,
378	struct auxtrace_buffer *b)
379	{
380	bool consecutive = false;
381	void *start;
382
383	start = intel_pt_find_overlap(a->data, a->size, b->data, b->size,
384	pt->have_tsc, &consecutive,
385	pt->synth_opts.vm_time_correlation);
386	if (!start)
387	return -EINVAL;
388	/*
389	* In the case of vm_time_correlation, the overlap might contain TSC
390	* packets that will not be fixed, and that will then no longer work for
391	* overlap detection. Avoid that by zeroing out the overlap.
392	*/
393	if (pt->synth_opts.vm_time_correlation)
394	memset(b->data, 0, start - b->data);
395	b->use_size = b->data + b->size - start;
396	b->use_data = start;
397	if (b->use_size && consecutive)
398	b->consecutive = true;
399	return 0;
400	}
401
402	static int intel_pt_get_buffer(struct intel_pt_queue *ptq,
403	struct auxtrace_buffer *buffer,
404	struct auxtrace_buffer *old_buffer,
405	struct intel_pt_buffer *b)
406	{
407	bool might_overlap;
408
409	if (!buffer->data) {
410	int fd = perf_data__fd(ptq->pt->session->data);
411
412	buffer->data = auxtrace_buffer__get_data(buffer, fd);
413	if (!buffer->data)
414	return -ENOMEM;
415	}
416
417	might_overlap = ptq->pt->snapshot_mode \|\| ptq->pt->sampling_mode;
418	if (might_overlap && !buffer->consecutive && old_buffer &&
419	intel_pt_do_fix_overlap(ptq->pt, old_buffer, buffer))
420	return -ENOMEM;
421
422	if (buffer->use_data) {
423	b->len = buffer->use_size;
424	b->buf = buffer->use_data;
425	} else {
426	b->len = buffer->size;
427	b->buf = buffer->data;
428	}
429	b->ref_timestamp = buffer->reference;
430
431	if (!old_buffer \|\| (might_overlap && !buffer->consecutive)) {
432	b->consecutive = false;
433	b->trace_nr = buffer->buffer_nr + 1;
434	} else {
435	b->consecutive = true;
436	}
437
438	return 0;
439	}
440
441	/* Do not drop buffers with references - refer intel_pt_get_trace() */
442	static void intel_pt_lookahead_drop_buffer(struct intel_pt_queue *ptq,
443	struct auxtrace_buffer *buffer)
444	{
445	if (!buffer \|\| buffer == ptq->buffer \|\| buffer == ptq->old_buffer)
446	return;
447
448	auxtrace_buffer__drop_data(buffer);
449	}
450
451	/* Must be serialized with respect to intel_pt_get_trace() */
452	static int intel_pt_lookahead(void *data, intel_pt_lookahead_cb_t cb,
453	void *cb_data)
454	{
455	struct intel_pt_queue *ptq = data;
456	struct auxtrace_buffer *buffer = ptq->buffer;
457	struct auxtrace_buffer *old_buffer = ptq->old_buffer;
458	struct auxtrace_queue *queue;
459	int err = 0;
460
461	queue = &ptq->pt->queues.queue_array[ptq->queue_nr];
462
463	while (1) {
464	struct intel_pt_buffer b = { .len = 0 };
465
466	buffer = auxtrace_buffer__next(queue, buffer);
467	if (!buffer)
468	break;
469
470	err = intel_pt_get_buffer(ptq, buffer, old_buffer, &b);
471	if (err)
472	break;
473
474	if (b.len) {
475	intel_pt_lookahead_drop_buffer(ptq, old_buffer);
476	old_buffer = buffer;
477	} else {
478	intel_pt_lookahead_drop_buffer(ptq, buffer);
479	continue;
480	}
481
482	err = cb(&b, cb_data);
483	if (err)
484	break;
485	}
486
487	if (buffer != old_buffer)
488	intel_pt_lookahead_drop_buffer(ptq, buffer);
489	intel_pt_lookahead_drop_buffer(ptq, old_buffer);
490
491	return err;
492	}
493
494	/*
495	* This function assumes data is processed sequentially only.
496	* Must be serialized with respect to intel_pt_lookahead()
497	*/
498	static int intel_pt_get_trace(struct intel_pt_buffer b, void data)
499	{
500	struct intel_pt_queue *ptq = data;
501	struct auxtrace_buffer *buffer = ptq->buffer;
502	struct auxtrace_buffer *old_buffer = ptq->old_buffer;
503	struct auxtrace_queue *queue;
504	int err;
505
506	if (ptq->stop) {
507	b->len = 0;
508	return 0;
509	}
510
511	queue = &ptq->pt->queues.queue_array[ptq->queue_nr];
512
513	buffer = auxtrace_buffer__next(queue, buffer);
514	if (!buffer) {
515	if (old_buffer)
516	auxtrace_buffer__drop_data(old_buffer);
517	b->len = 0;
518	return 0;
519	}
520
521	ptq->buffer = buffer;
522
523	err = intel_pt_get_buffer(ptq, buffer, old_buffer, b);
524	if (err)
525	return err;
526
527	if (ptq->step_through_buffers)
528	ptq->stop = true;
529
530	if (b->len) {
531	if (old_buffer)
532	auxtrace_buffer__drop_data(old_buffer);
533	ptq->old_buffer = buffer;
534	} else {
535	auxtrace_buffer__drop_data(buffer);
536	return intel_pt_get_trace(b, data);
537	}
538
539	return 0;
540	}
541
542	struct intel_pt_cache_entry {
543	struct auxtrace_cache_entry entry;
544	u64 insn_cnt;
545	u64 byte_cnt;
546	enum intel_pt_insn_op op;
547	enum intel_pt_insn_branch branch;
548	bool emulated_ptwrite;
549	int length;
550	int32_t rel;
551	char insn[INTEL_PT_INSN_BUF_SZ];
552	};
553
554	static int intel_pt_config_div(const char var, const char value, void *data)
555	{
556	int *d = data;
557	long val;
558
559	if (!strcmp(var, "intel-pt.cache-divisor")) {
560	val = strtol(value, NULL, 0);
561	if (val > 0 && val <= INT_MAX)
562	*d = val;
563	}
564
565	return 0;
566	}
567
568	static int intel_pt_cache_divisor(void)
569	{
570	static int d;
571
572	if (d)
573	return d;
574
575	perf_config(intel_pt_config_div, &d);
576
577	if (!d)
578	d = 64;
579
580	return d;
581	}
582
583	static unsigned int intel_pt_cache_size(struct dso *dso,
584	struct machine *machine)
585	{
586	off_t size;
587
588	size = dso__data_size(dso, machine);
589	size /= intel_pt_cache_divisor();
590	if (size < 1000)
591	return 10;
592	if (size > (1 << 21))
593	return 21;
594	return 32 - __builtin_clz(size);
595	}
596
597	static struct auxtrace_cache intel_pt_cache(struct dso dso,
598	struct machine *machine)
599	{
600	struct auxtrace_cache *c;
601	unsigned int bits;
602
603	if (dso__auxtrace_cache(dso))
604	return dso__auxtrace_cache(dso);
605
606	bits = intel_pt_cache_size(dso, machine);
607
608	/* Ignoring cache creation failure */
609	c = auxtrace_cache__new(bits, sizeof(struct intel_pt_cache_entry), 200);
610
611	dso__set_auxtrace_cache(dso, c);
612
613	return c;
614	}
615
616	static int intel_pt_cache_add(struct dso dso, struct machine machine,
617	u64 offset, u64 insn_cnt, u64 byte_cnt,
618	struct intel_pt_insn *intel_pt_insn)
619	{
620	struct auxtrace_cache *c = intel_pt_cache(dso, machine);
621	struct intel_pt_cache_entry *e;
622	int err;
623
624	if (!c)
625	return -ENOMEM;
626
627	e = auxtrace_cache__alloc_entry(c);
628	if (!e)
629	return -ENOMEM;
630
631	e->insn_cnt = insn_cnt;
632	e->byte_cnt = byte_cnt;
633	e->op = intel_pt_insn->op;
634	e->branch = intel_pt_insn->branch;
635	e->emulated_ptwrite = intel_pt_insn->emulated_ptwrite;
636	e->length = intel_pt_insn->length;
637	e->rel = intel_pt_insn->rel;
638	memcpy(e->insn, intel_pt_insn->buf, INTEL_PT_INSN_BUF_SZ);
639
640	err = auxtrace_cache__add(c, offset, &e->entry);
641	if (err)
642	auxtrace_cache__free_entry(c, e);
643
644	return err;
645	}
646
647	static struct intel_pt_cache_entry *
648	intel_pt_cache_lookup(struct dso dso, struct machine machine, u64 offset)
649	{
650	struct auxtrace_cache *c = intel_pt_cache(dso, machine);
651
652	if (!c)
653	return NULL;
654
655	return auxtrace_cache__lookup(dso__auxtrace_cache(dso), offset);
656	}
657
658	static void intel_pt_cache_invalidate(struct dso dso, struct machine machine,
659	u64 offset)
660	{
661	struct auxtrace_cache *c = intel_pt_cache(dso, machine);
662
663	if (!c)
664	return;
665
666	auxtrace_cache__remove(dso__auxtrace_cache(dso), offset);
667	}
668
669	static inline bool intel_pt_guest_kernel_ip(uint64_t ip)
670	{
671	/* Assumes 64-bit kernel */
672	return ip & (1ULL << 63);
673	}
674
675	static inline u8 intel_pt_nr_cpumode(struct intel_pt_queue *ptq, uint64_t ip, bool nr)
676	{
677	if (nr) {
678	return intel_pt_guest_kernel_ip(ip) ?
679	PERF_RECORD_MISC_GUEST_KERNEL :
680	PERF_RECORD_MISC_GUEST_USER;
681	}
682
683	return ip >= ptq->pt->kernel_start ?
684	PERF_RECORD_MISC_KERNEL :
685	PERF_RECORD_MISC_USER;
686	}
687
688	static inline u8 intel_pt_cpumode(struct intel_pt_queue *ptq, uint64_t from_ip, uint64_t to_ip)
689	{
690	/* No support for non-zero CS base */
691	if (from_ip)
692	return intel_pt_nr_cpumode(ptq, from_ip, ptq->state->from_nr);
693	return intel_pt_nr_cpumode(ptq, to_ip, ptq->state->to_nr);
694	}
695
696	static int intel_pt_get_guest(struct intel_pt_queue *ptq)
697	{
698	struct machines *machines = &ptq->pt->session->machines;
699	struct machine *machine;
700	pid_t pid = ptq->pid <= 0 ? DEFAULT_GUEST_KERNEL_ID : ptq->pid;
701
702	if (ptq->guest_machine && pid == ptq->guest_machine->pid)
703	return 0;
704
705	ptq->guest_machine = NULL;
706	thread__zput(ptq->unknown_guest_thread);
707
708	if (symbol_conf.guest_code) {
709	thread__zput(ptq->guest_thread);
710	ptq->guest_thread = machines__findnew_guest_code(machines, pid);
711	}
712
713	machine = machines__find_guest(machines, pid);
714	if (!machine)
715	return -1;
716
717	ptq->unknown_guest_thread = machine__idle_thread(machine);
718	if (!ptq->unknown_guest_thread)
719	return -1;
720
721	ptq->guest_machine = machine;
722
723	return 0;
724	}
725
726	static inline bool intel_pt_jmp_16(struct intel_pt_insn *intel_pt_insn)
727	{
728	return intel_pt_insn->rel == 16 && intel_pt_insn->branch == INTEL_PT_BR_UNCONDITIONAL;
729	}
730
731	#define PTWRITE_MAGIC "\x0f\x0bperf,ptwrite "
732	#define PTWRITE_MAGIC_LEN 16
733
734	static bool intel_pt_emulated_ptwrite(struct dso dso, struct machine machine, u64 offset)
735	{
736	unsigned char buf[PTWRITE_MAGIC_LEN];
737	ssize_t len;
738
739	len = dso__data_read_offset(dso, machine, offset, buf, PTWRITE_MAGIC_LEN);
740	if (len == PTWRITE_MAGIC_LEN && !memcmp(buf, PTWRITE_MAGIC, PTWRITE_MAGIC_LEN)) {
741	intel_pt_log("Emulated ptwrite signature found\n");
742	return true;
743	}
744	intel_pt_log("Emulated ptwrite signature not found\n");
745	return false;
746	}
747
748	static int intel_pt_walk_next_insn(struct intel_pt_insn *intel_pt_insn,
749	uint64_t insn_cnt_ptr, uint64_t ip,
750	uint64_t to_ip, uint64_t max_insn_cnt,
751	void *data)
752	{
753	struct intel_pt_queue *ptq = data;
754	struct machine *machine = ptq->pt->machine;
755	struct thread *thread;
756	struct addr_location al;
757	unsigned char buf[INTEL_PT_INSN_BUF_SZ];
758	ssize_t len;
759	int x86_64, ret = 0;
760	u8 cpumode;
761	u64 offset, start_offset, start_ip;
762	u64 insn_cnt = 0;
763	bool one_map = true;
764	bool nr;
765
766
767	addr_location__init(&al);
768	intel_pt_insn->length = 0;
769	intel_pt_insn->op = INTEL_PT_OP_OTHER;
770
771	if (to_ip && *ip == to_ip)
772	goto out_no_cache;
773
774	nr = ptq->state->to_nr;
775	cpumode = intel_pt_nr_cpumode(ptq, *ip, nr);
776
777	if (nr) {
778	if (ptq->pt->have_guest_sideband) {
779	if (!ptq->guest_machine \|\| ptq->guest_machine_pid != ptq->pid) {
780	intel_pt_log("ERROR: guest sideband but no guest machine\n");
781	ret = -EINVAL;
782	goto out_ret;
783	}
784	} else if ((!symbol_conf.guest_code && cpumode != PERF_RECORD_MISC_GUEST_KERNEL) \|\|
785	intel_pt_get_guest(ptq)) {
786	intel_pt_log("ERROR: no guest machine\n");
787	ret = -EINVAL;
788	goto out_ret;
789	}
790	machine = ptq->guest_machine;
791	thread = ptq->guest_thread;
792	if (!thread) {
793	if (cpumode != PERF_RECORD_MISC_GUEST_KERNEL) {
794	intel_pt_log("ERROR: no guest thread\n");
795	ret = -EINVAL;
796	goto out_ret;
797	}
798	thread = ptq->unknown_guest_thread;
799	}
800	} else {
801	thread = ptq->thread;
802	if (!thread) {
803	if (cpumode != PERF_RECORD_MISC_KERNEL) {
804	intel_pt_log("ERROR: no thread\n");
805	ret = -EINVAL;
806	goto out_ret;
807	}
808	thread = ptq->pt->unknown_thread;
809	}
810	}
811
812	while (1) {
813	struct dso *dso;
814
815	if (!thread__find_map(thread, cpumode, *ip, &al) \|\| !map__dso(al.map)) {
816	if (al.map)
817	intel_pt_log("ERROR: thread has no dso for %#" PRIx64 "\n", *ip);
818	else
819	intel_pt_log("ERROR: thread has no map for %#" PRIx64 "\n", *ip);
820	addr_location__exit(&al);
821	ret = -EINVAL;
822	goto out_ret;
823	}
824	dso = map__dso(al.map);
825
826	if (dso__data(dso)->status == DSO_DATA_STATUS_ERROR &&
827	dso__data_status_seen(dso, DSO_DATA_STATUS_SEEN_ITRACE)) {
828	ret = -ENOENT;
829	goto out_ret;
830	}
831
832	offset = map__map_ip(al.map, *ip);
833
834	if (!to_ip && one_map) {
835	struct intel_pt_cache_entry *e;
836
837	e = intel_pt_cache_lookup(dso, machine, offset);
838	if (e &&
839	(!max_insn_cnt \|\| e->insn_cnt <= max_insn_cnt)) {
840	*insn_cnt_ptr = e->insn_cnt;
841	*ip += e->byte_cnt;
842	intel_pt_insn->op = e->op;
843	intel_pt_insn->branch = e->branch;
844	intel_pt_insn->emulated_ptwrite = e->emulated_ptwrite;
845	intel_pt_insn->length = e->length;
846	intel_pt_insn->rel = e->rel;
847	memcpy(intel_pt_insn->buf, e->insn, INTEL_PT_INSN_BUF_SZ);
848	intel_pt_log_insn_no_data(intel_pt_insn, *ip);
849	ret = 0;
850	goto out_ret;
851	}
852	}
853
854	start_offset = offset;
855	start_ip = *ip;
856
857	/* Load maps to ensure dso->is_64_bit has been updated */
858	map__load(al.map);
859
860	x86_64 = dso__is_64_bit(dso);
861
862	while (1) {
863	len = dso__data_read_offset(dso, machine,
864	offset, buf,
865	INTEL_PT_INSN_BUF_SZ);
866	if (len <= 0) {
867	intel_pt_log("ERROR: failed to read at offset %#" PRIx64 " ",
868	offset);
869	if (intel_pt_enable_logging)
870	dso__fprintf(dso, intel_pt_log_fp());
871	ret = -EINVAL;
872	goto out_ret;
873	}
874
875	if (intel_pt_get_insn(buf, len, x86_64, intel_pt_insn)) {
876	ret = -EINVAL;
877	goto out_ret;
878	}
879
880	intel_pt_log_insn(intel_pt_insn, *ip);
881
882	insn_cnt += 1;
883
884	if (intel_pt_insn->branch != INTEL_PT_BR_NO_BRANCH) {
885	bool eptw;
886	u64 offs;
887
888	if (!intel_pt_jmp_16(intel_pt_insn))
889	goto out;
890	/* Check for emulated ptwrite */
891	offs = offset + intel_pt_insn->length;
892	eptw = intel_pt_emulated_ptwrite(dso, machine, offs);
893	intel_pt_insn->emulated_ptwrite = eptw;
894	goto out;
895	}
896
897	if (max_insn_cnt && insn_cnt >= max_insn_cnt)
898	goto out_no_cache;
899
900	*ip += intel_pt_insn->length;
901
902	if (to_ip && *ip == to_ip) {
903	intel_pt_insn->length = 0;
904	intel_pt_insn->op = INTEL_PT_OP_OTHER;
905	goto out_no_cache;
906	}
907
908	if (*ip >= map__end(al.map))
909	break;
910
911	offset += intel_pt_insn->length;
912	}
913	one_map = false;
914	}
915	out:
916	*insn_cnt_ptr = insn_cnt;
917
918	if (!one_map)
919	goto out_no_cache;
920
921	/*
922	* Didn't lookup in the 'to_ip' case, so do it now to prevent duplicate
923	* entries.
924	*/
925	if (to_ip) {
926	struct intel_pt_cache_entry *e;
927
928	e = intel_pt_cache_lookup(map__dso(al.map), machine, start_offset);
929	if (e)
930	goto out_ret;
931	}
932
933	/* Ignore cache errors */
934	intel_pt_cache_add(map__dso(al.map), machine, start_offset, insn_cnt,
935	*ip - start_ip, intel_pt_insn);
936
937	out_ret:
938	addr_location__exit(&al);
939	return ret;
940
941	out_no_cache:
942	*insn_cnt_ptr = insn_cnt;
943	addr_location__exit(&al);
944	return 0;
945	}
946
947	static bool intel_pt_match_pgd_ip(struct intel_pt *pt, uint64_t ip,
948	uint64_t offset, const char *filename)
949	{
950	struct addr_filter *filt;
951	bool have_filter = false;
952	bool hit_tracestop = false;
953	bool hit_filter = false;
954
955	list_for_each_entry(filt, &pt->filts.head, list) {
956	if (filt->start)
957	have_filter = true;
958
959	if ((filename && !filt->filename) \|\|
960	(!filename && filt->filename) \|\|
961	(filename && strcmp(filename, filt->filename)))
962	continue;
963
964	if (!(offset >= filt->addr && offset < filt->addr + filt->size))
965	continue;
966
967	intel_pt_log("TIP.PGD ip %#"PRIx64" offset %#"PRIx64" in %s hit filter: %s offset %#"PRIx64" size %#"PRIx64"\n",
968	ip, offset, filename ? filename : "[kernel]",
969	filt->start ? "filter" : "stop",
970	filt->addr, filt->size);
971
972	if (filt->start)
973	hit_filter = true;
974	else
975	hit_tracestop = true;
976	}
977
978	if (!hit_tracestop && !hit_filter)
979	intel_pt_log("TIP.PGD ip %#"PRIx64" offset %#"PRIx64" in %s is not in a filter region\n",
980	ip, offset, filename ? filename : "[kernel]");
981
982	return hit_tracestop \|\| (have_filter && !hit_filter);
983	}
984
985	static int __intel_pt_pgd_ip(uint64_t ip, void *data)
986	{
987	struct intel_pt_queue *ptq = data;
988	struct thread *thread;
989	struct addr_location al;
990	u8 cpumode;
991	u64 offset;
992	int res;
993
994	if (ptq->state->to_nr) {
995	if (intel_pt_guest_kernel_ip(ip))
996	return intel_pt_match_pgd_ip(ptq->pt, ip, ip, NULL);
997	/* No support for decoding guest user space */
998	return -EINVAL;
999	} else if (ip >= ptq->pt->kernel_start) {
1000	return intel_pt_match_pgd_ip(ptq->pt, ip, ip, NULL);
1001	}
1002
1003	cpumode = PERF_RECORD_MISC_USER;
1004
1005	thread = ptq->thread;
1006	if (!thread)
1007	return -EINVAL;
1008
1009	addr_location__init(&al);
1010	if (!thread__find_map(thread, cpumode, ip, &al) \|\| !map__dso(al.map))
1011	return -EINVAL;
1012
1013	offset = map__map_ip(al.map, ip);
1014
1015	res = intel_pt_match_pgd_ip(ptq->pt, ip, offset, dso__long_name(map__dso(al.map)));
1016	addr_location__exit(&al);
1017	return res;
1018	}
1019
1020	static bool intel_pt_pgd_ip(uint64_t ip, void *data)
1021	{
1022	return __intel_pt_pgd_ip(ip, data) > 0;
1023	}
1024
1025	static bool intel_pt_get_config(struct intel_pt *pt,
1026	struct perf_event_attr attr, u64 config)
1027	{
1028	if (attr->type == pt->pmu_type) {
1029	if (config)
1030	*config = attr->config;
1031	return true;
1032	}
1033
1034	return false;
1035	}
1036
1037	static bool intel_pt_exclude_kernel(struct intel_pt *pt)
1038	{
1039	struct evsel *evsel;
1040
1041	evlist__for_each_entry(pt->session->evlist, evsel) {
1042	if (intel_pt_get_config(pt, &evsel->core.attr, NULL) &&
1043	!evsel->core.attr.exclude_kernel)
1044	return false;
1045	}
1046	return true;
1047	}
1048
1049	static bool intel_pt_return_compression(struct intel_pt *pt)
1050	{
1051	struct evsel *evsel;
1052	u64 config;
1053
1054	if (!pt->noretcomp_bit)
1055	return true;
1056
1057	evlist__for_each_entry(pt->session->evlist, evsel) {
1058	if (intel_pt_get_config(pt, &evsel->core.attr, &config) &&
1059	(config & pt->noretcomp_bit))
1060	return false;
1061	}
1062	return true;
1063	}
1064
1065	static bool intel_pt_branch_enable(struct intel_pt *pt)
1066	{
1067	struct evsel *evsel;
1068	u64 config;
1069
1070	evlist__for_each_entry(pt->session->evlist, evsel) {
1071	if (intel_pt_get_config(pt, &evsel->core.attr, &config) &&
1072	(config & INTEL_PT_CFG_PASS_THRU) &&
1073	!(config & INTEL_PT_CFG_BRANCH_EN))
1074	return false;
1075	}
1076	return true;
1077	}
1078
1079	static bool intel_pt_disabled_tnt(struct intel_pt *pt)
1080	{
1081	struct evsel *evsel;
1082	u64 config;
1083
1084	evlist__for_each_entry(pt->session->evlist, evsel) {
1085	if (intel_pt_get_config(pt, &evsel->core.attr, &config) &&
1086	config & INTEL_PT_CFG_TNT_DIS)
1087	return true;
1088	}
1089	return false;
1090	}
1091
1092	static unsigned int intel_pt_mtc_period(struct intel_pt *pt)
1093	{
1094	struct evsel *evsel;
1095	unsigned int shift;
1096	u64 config;
1097
1098	if (!pt->mtc_freq_bits)
1099	return 0;
1100
1101	for (shift = 0, config = pt->mtc_freq_bits; !(config & 1); shift++)
1102	config >>= 1;
1103
1104	evlist__for_each_entry(pt->session->evlist, evsel) {
1105	if (intel_pt_get_config(pt, &evsel->core.attr, &config))
1106	return (config & pt->mtc_freq_bits) >> shift;
1107	}
1108	return 0;
1109	}
1110
1111	static bool intel_pt_timeless_decoding(struct intel_pt *pt)
1112	{
1113	struct evsel *evsel;
1114	bool timeless_decoding = true;
1115	u64 config;
1116
1117	if (!pt->tsc_bit \|\| !pt->cap_user_time_zero \|\| pt->synth_opts.timeless_decoding)
1118	return true;
1119
1120	evlist__for_each_entry(pt->session->evlist, evsel) {
1121	if (!(evsel->core.attr.sample_type & PERF_SAMPLE_TIME))
1122	return true;
1123	if (intel_pt_get_config(pt, &evsel->core.attr, &config)) {
1124	if (config & pt->tsc_bit)
1125	timeless_decoding = false;
1126	else
1127	return true;
1128	}
1129	}
1130	return timeless_decoding;
1131	}
1132
1133	static bool intel_pt_tracing_kernel(struct intel_pt *pt)
1134	{
1135	struct evsel *evsel;
1136
1137	evlist__for_each_entry(pt->session->evlist, evsel) {
1138	if (intel_pt_get_config(pt, &evsel->core.attr, NULL) &&
1139	!evsel->core.attr.exclude_kernel)
1140	return true;
1141	}
1142	return false;
1143	}
1144
1145	static bool intel_pt_have_tsc(struct intel_pt *pt)
1146	{
1147	struct evsel *evsel;
1148	bool have_tsc = false;
1149	u64 config;
1150
1151	if (!pt->tsc_bit)
1152	return false;
1153
1154	evlist__for_each_entry(pt->session->evlist, evsel) {
1155	if (intel_pt_get_config(pt, &evsel->core.attr, &config)) {
1156	if (config & pt->tsc_bit)
1157	have_tsc = true;
1158	else
1159	return false;
1160	}
1161	}
1162	return have_tsc;
1163	}
1164
1165	static bool intel_pt_have_mtc(struct intel_pt *pt)
1166	{
1167	struct evsel *evsel;
1168	u64 config;
1169
1170	evlist__for_each_entry(pt->session->evlist, evsel) {
1171	if (intel_pt_get_config(pt, &evsel->core.attr, &config) &&
1172	(config & pt->mtc_bit))
1173	return true;
1174	}
1175	return false;
1176	}
1177
1178	static bool intel_pt_sampling_mode(struct intel_pt *pt)
1179	{
1180	struct evsel *evsel;
1181
1182	evlist__for_each_entry(pt->session->evlist, evsel) {
1183	if ((evsel->core.attr.sample_type & PERF_SAMPLE_AUX) &&
1184	evsel->core.attr.aux_sample_size)
1185	return true;
1186	}
1187	return false;
1188	}
1189
1190	static u64 intel_pt_ctl(struct intel_pt *pt)
1191	{
1192	struct evsel *evsel;
1193	u64 config;
1194
1195	evlist__for_each_entry(pt->session->evlist, evsel) {
1196	if (intel_pt_get_config(pt, &evsel->core.attr, &config))
1197	return config;
1198	}
1199	return 0;
1200	}
1201
1202	static u64 intel_pt_ns_to_ticks(const struct intel_pt *pt, u64 ns)
1203	{
1204	u64 quot, rem;
1205
1206	quot = ns / pt->tc.time_mult;
1207	rem = ns % pt->tc.time_mult;
1208	return (quot << pt->tc.time_shift) + (rem << pt->tc.time_shift) /
1209	pt->tc.time_mult;
1210	}
1211
1212	static struct ip_callchain intel_pt_alloc_chain(struct intel_pt pt)
1213	{
1214	size_t sz = sizeof(struct ip_callchain);
1215
1216	/* Add 1 to callchain_sz for callchain context */
1217	sz += (pt->synth_opts.callchain_sz + 1) * sizeof(u64);
1218	return zalloc(sz);
1219	}
1220
1221	static int intel_pt_callchain_init(struct intel_pt *pt)
1222	{
1223	struct evsel *evsel;
1224
1225	evlist__for_each_entry(pt->session->evlist, evsel) {
1226	if (!(evsel->core.attr.sample_type & PERF_SAMPLE_CALLCHAIN))
1227	evsel->synth_sample_type \|= PERF_SAMPLE_CALLCHAIN;
1228	}
1229
1230	pt->chain = intel_pt_alloc_chain(pt);
1231	if (!pt->chain)
1232	return -ENOMEM;
1233
1234	return 0;
1235	}
1236
1237	static void intel_pt_add_callchain(struct intel_pt *pt,
1238	struct perf_sample *sample)
1239	{
1240	struct thread *thread = machine__findnew_thread(pt->machine,
1241	sample->pid,
1242	sample->tid);
1243
1244	thread_stack__sample_late(thread, sample->cpu, pt->chain,
1245	pt->synth_opts.callchain_sz + 1, sample->ip,
1246	pt->kernel_start);
1247
1248	sample->callchain = pt->chain;
1249	}
1250
1251	static struct branch_stack *intel_pt_alloc_br_stack(unsigned int entry_cnt)
1252	{
1253	size_t sz = sizeof(struct branch_stack);
1254
1255	sz += entry_cnt * sizeof(struct branch_entry);
1256	return zalloc(sz);
1257	}
1258
1259	static int intel_pt_br_stack_init(struct intel_pt *pt)
1260	{
1261	struct evsel *evsel;
1262
1263	evlist__for_each_entry(pt->session->evlist, evsel) {
1264	if (!(evsel->core.attr.sample_type & PERF_SAMPLE_BRANCH_STACK))
1265	evsel->synth_sample_type \|= PERF_SAMPLE_BRANCH_STACK;
1266	}
1267
1268	pt->br_stack = intel_pt_alloc_br_stack(pt->br_stack_sz);
1269	if (!pt->br_stack)
1270	return -ENOMEM;
1271
1272	return 0;
1273	}
1274
1275	static void intel_pt_add_br_stack(struct intel_pt *pt,
1276	struct perf_sample *sample)
1277	{
1278	struct thread *thread = machine__findnew_thread(pt->machine,
1279	sample->pid,
1280	sample->tid);
1281
1282	thread_stack__br_sample_late(thread, sample->cpu, pt->br_stack,
1283	pt->br_stack_sz, sample->ip,
1284	pt->kernel_start);
1285
1286	sample->branch_stack = pt->br_stack;
1287	thread__put(thread);
1288	}
1289
1290	/* INTEL_PT_LBR_0, INTEL_PT_LBR_1 and INTEL_PT_LBR_2 */
1291	#define LBRS_MAX (INTEL_PT_BLK_ITEM_ID_CNT * 3U)
1292
1293	static struct intel_pt_queue intel_pt_alloc_queue(struct intel_pt pt,
1294	unsigned int queue_nr)
1295	{
1296	struct intel_pt_params params = { .get_trace = 0, };
1297	struct perf_env *env = pt->machine->env;
1298	struct intel_pt_queue *ptq;
1299
1300	ptq = zalloc(sizeof(struct intel_pt_queue));
1301	if (!ptq)
1302	return NULL;
1303
1304	if (pt->synth_opts.callchain) {
1305	ptq->chain = intel_pt_alloc_chain(pt);
1306	if (!ptq->chain)
1307	goto out_free;
1308	}
1309
1310	if (pt->synth_opts.last_branch \|\| pt->synth_opts.add_last_branch \|\|
1311	pt->synth_opts.other_events) {
1312	unsigned int entry_cnt = max(LBRS_MAX, pt->br_stack_sz);
1313
1314	ptq->last_branch = intel_pt_alloc_br_stack(entry_cnt);
1315	if (!ptq->last_branch)
1316	goto out_free;
1317	}
1318
1319	ptq->event_buf = malloc(PERF_SAMPLE_MAX_SIZE);
1320	if (!ptq->event_buf)
1321	goto out_free;
1322
1323	ptq->pt = pt;
1324	ptq->queue_nr = queue_nr;
1325	ptq->exclude_kernel = intel_pt_exclude_kernel(pt);
1326	ptq->pid = -1;
1327	ptq->tid = -1;
1328	ptq->cpu = -1;
1329	ptq->next_tid = -1;
1330
1331	params.get_trace = intel_pt_get_trace;
1332	params.walk_insn = intel_pt_walk_next_insn;
1333	params.lookahead = intel_pt_lookahead;
1334	params.findnew_vmcs_info = intel_pt_findnew_vmcs_info;
1335	params.data = ptq;
1336	params.return_compression = intel_pt_return_compression(pt);
1337	params.branch_enable = intel_pt_branch_enable(pt);
1338	params.ctl = intel_pt_ctl(pt);
1339	params.max_non_turbo_ratio = pt->max_non_turbo_ratio;
1340	params.mtc_period = intel_pt_mtc_period(pt);
1341	params.tsc_ctc_ratio_n = pt->tsc_ctc_ratio_n;
1342	params.tsc_ctc_ratio_d = pt->tsc_ctc_ratio_d;
1343	params.quick = pt->synth_opts.quick;
1344	params.vm_time_correlation = pt->synth_opts.vm_time_correlation;
1345	params.vm_tm_corr_dry_run = pt->synth_opts.vm_tm_corr_dry_run;
1346	params.first_timestamp = pt->first_timestamp;
1347	params.max_loops = pt->max_loops;
1348
1349	/* Cannot walk code without TNT, so force 'quick' mode */
1350	if (params.branch_enable && intel_pt_disabled_tnt(pt) && !params.quick)
1351	params.quick = 1;
1352
1353	if (pt->filts.cnt > 0)
1354	params.pgd_ip = intel_pt_pgd_ip;
1355
1356	if (pt->synth_opts.instructions \|\| pt->synth_opts.cycles) {
1357	if (pt->synth_opts.period) {
1358	switch (pt->synth_opts.period_type) {
1359	case PERF_ITRACE_PERIOD_INSTRUCTIONS:
1360	params.period_type =
1361	INTEL_PT_PERIOD_INSTRUCTIONS;
1362	params.period = pt->synth_opts.period;
1363	break;
1364	case PERF_ITRACE_PERIOD_TICKS:
1365	params.period_type = INTEL_PT_PERIOD_TICKS;
1366	params.period = pt->synth_opts.period;
1367	break;
1368	case PERF_ITRACE_PERIOD_NANOSECS:
1369	params.period_type = INTEL_PT_PERIOD_TICKS;
1370	params.period = intel_pt_ns_to_ticks(pt,
1371	pt->synth_opts.period);
1372	break;
1373	default:
1374	break;
1375	}
1376	}
1377
1378	if (!params.period) {
1379	params.period_type = INTEL_PT_PERIOD_INSTRUCTIONS;
1380	params.period = 1;
1381	}
1382	}
1383
1384	if (env->cpuid && !strncmp(env->cpuid, "GenuineIntel,6,92,", 18))
1385	params.flags \|= INTEL_PT_FUP_WITH_NLIP;
1386
1387	ptq->decoder = intel_pt_decoder_new(&params);
1388	if (!ptq->decoder)
1389	goto out_free;
1390
1391	return ptq;
1392
1393	out_free:
1394	zfree(&ptq->event_buf);
1395	zfree(&ptq->last_branch);
1396	zfree(&ptq->chain);
1397	free(ptq);
1398	return NULL;
1399	}
1400
1401	static void intel_pt_free_queue(void *priv)
1402	{
1403	struct intel_pt_queue *ptq = priv;
1404
1405	if (!ptq)
1406	return;
1407	thread__zput(ptq->thread);
1408	thread__zput(ptq->guest_thread);
1409	thread__zput(ptq->unknown_guest_thread);
1410	intel_pt_decoder_free(ptq->decoder);
1411	zfree(&ptq->event_buf);
1412	zfree(&ptq->last_branch);
1413	zfree(&ptq->chain);
1414	free(ptq);
1415	}
1416
1417	static void intel_pt_first_timestamp(struct intel_pt *pt, u64 timestamp)
1418	{
1419	unsigned int i;
1420
1421	pt->first_timestamp = timestamp;
1422
1423	for (i = 0; i < pt->queues.nr_queues; i++) {
1424	struct auxtrace_queue *queue = &pt->queues.queue_array[i];
1425	struct intel_pt_queue *ptq = queue->priv;
1426
1427	if (ptq && ptq->decoder)
1428	intel_pt_set_first_timestamp(ptq->decoder, timestamp);
1429	}
1430	}
1431
1432	static int intel_pt_get_guest_from_sideband(struct intel_pt_queue *ptq)
1433	{
1434	struct machines *machines = &ptq->pt->session->machines;
1435	struct machine *machine;
1436	pid_t machine_pid = ptq->pid;
1437	pid_t tid;
1438	int vcpu;
1439
1440	if (machine_pid <= 0)
1441	return 0; /* Not a guest machine */
1442
1443	machine = machines__find(machines, machine_pid);
1444	if (!machine)
1445	return 0; /* Not a guest machine */
1446
1447	if (ptq->guest_machine != machine) {
1448	ptq->guest_machine = NULL;
1449	thread__zput(ptq->guest_thread);
1450	thread__zput(ptq->unknown_guest_thread);
1451
1452	ptq->unknown_guest_thread = machine__find_thread(machine, 0, 0);
1453	if (!ptq->unknown_guest_thread)
1454	return -1;
1455	ptq->guest_machine = machine;
1456	}
1457
1458	vcpu = ptq->thread ? thread__guest_cpu(ptq->thread) : -1;
1459	if (vcpu < 0)
1460	return -1;
1461
1462	tid = machine__get_current_tid(machine, vcpu);
1463
1464	if (ptq->guest_thread && thread__tid(ptq->guest_thread) != tid)
1465	thread__zput(ptq->guest_thread);
1466
1467	if (!ptq->guest_thread) {
1468	ptq->guest_thread = machine__find_thread(machine, -1, tid);
1469	if (!ptq->guest_thread)
1470	return -1;
1471	}
1472
1473	ptq->guest_machine_pid = machine_pid;
1474	ptq->guest_pid = thread__pid(ptq->guest_thread);
1475	ptq->guest_tid = tid;
1476	ptq->vcpu = vcpu;
1477
1478	return 0;
1479	}
1480
1481	static void intel_pt_set_pid_tid_cpu(struct intel_pt *pt,
1482	struct auxtrace_queue *queue)
1483	{
1484	struct intel_pt_queue *ptq = queue->priv;
1485
1486	if (queue->tid == -1 \|\| pt->have_sched_switch) {
1487	ptq->tid = machine__get_current_tid(pt->machine, ptq->cpu);
1488	if (ptq->tid == -1)
1489	ptq->pid = -1;
1490	thread__zput(ptq->thread);
1491	}
1492
1493	if (!ptq->thread && ptq->tid != -1)
1494	ptq->thread = machine__find_thread(pt->machine, -1, ptq->tid);
1495
1496	if (ptq->thread) {
1497	ptq->pid = thread__pid(ptq->thread);
1498	if (queue->cpu == -1)
1499	ptq->cpu = thread__cpu(ptq->thread);
1500	}
1501
1502	if (pt->have_guest_sideband && intel_pt_get_guest_from_sideband(ptq)) {
1503	ptq->guest_machine_pid = 0;
1504	ptq->guest_pid = -1;
1505	ptq->guest_tid = -1;
1506	ptq->vcpu = -1;
1507	}
1508	}
1509
1510	static void intel_pt_sample_flags(struct intel_pt_queue *ptq)
1511	{
1512	struct intel_pt *pt = ptq->pt;
1513
1514	ptq->insn_len = 0;
1515	if (ptq->state->flags & INTEL_PT_ABORT_TX) {
1516	ptq->flags = PERF_IP_FLAG_BRANCH \| PERF_IP_FLAG_TX_ABORT;
1517	} else if (ptq->state->flags & INTEL_PT_ASYNC) {
1518	if (!ptq->state->to_ip)
1519	ptq->flags = PERF_IP_FLAG_BRANCH \|
1520	PERF_IP_FLAG_ASYNC \|
1521	PERF_IP_FLAG_TRACE_END;
1522	else if (ptq->state->from_nr && !ptq->state->to_nr)
1523	ptq->flags = PERF_IP_FLAG_BRANCH \| PERF_IP_FLAG_CALL \|
1524	PERF_IP_FLAG_ASYNC \|
1525	PERF_IP_FLAG_VMEXIT;
1526	else
1527	ptq->flags = PERF_IP_FLAG_BRANCH \| PERF_IP_FLAG_CALL \|
1528	PERF_IP_FLAG_ASYNC \|
1529	PERF_IP_FLAG_INTERRUPT;
1530	} else {
1531	if (ptq->state->from_ip)
1532	ptq->flags = intel_pt_insn_type(ptq->state->insn_op);
1533	else
1534	ptq->flags = PERF_IP_FLAG_BRANCH \|
1535	PERF_IP_FLAG_TRACE_BEGIN;
1536	if (ptq->state->flags & INTEL_PT_IN_TX)
1537	ptq->flags \|= PERF_IP_FLAG_IN_TX;
1538	ptq->insn_len = ptq->state->insn_len;
1539	memcpy(ptq->insn, ptq->state->insn, INTEL_PT_INSN_BUF_SZ);
1540	}
1541
1542	if (ptq->state->type & INTEL_PT_TRACE_BEGIN)
1543	ptq->flags \|= PERF_IP_FLAG_TRACE_BEGIN;
1544	if (ptq->state->type & INTEL_PT_TRACE_END)
1545	ptq->flags \|= PERF_IP_FLAG_TRACE_END;
1546
1547	if (pt->cap_event_trace) {
1548	if (ptq->state->type & INTEL_PT_IFLAG_CHG) {
1549	if (!ptq->state->from_iflag)
1550	ptq->flags \|= PERF_IP_FLAG_INTR_DISABLE;
1551	if (ptq->state->from_iflag != ptq->state->to_iflag)
1552	ptq->flags \|= PERF_IP_FLAG_INTR_TOGGLE;
1553	} else if (!ptq->state->to_iflag) {
1554	ptq->flags \|= PERF_IP_FLAG_INTR_DISABLE;
1555	}
1556	}
1557	}
1558
1559	static void intel_pt_setup_time_range(struct intel_pt *pt,
1560	struct intel_pt_queue *ptq)
1561	{
1562	if (!pt->range_cnt)
1563	return;
1564
1565	ptq->sel_timestamp = pt->time_ranges[0].start;
1566	ptq->sel_idx = 0;
1567
1568	if (ptq->sel_timestamp) {
1569	ptq->sel_start = true;
1570	} else {
1571	ptq->sel_timestamp = pt->time_ranges[0].end;
1572	ptq->sel_start = false;
1573	}
1574	}
1575
1576	static int intel_pt_setup_queue(struct intel_pt *pt,
1577	struct auxtrace_queue *queue,
1578	unsigned int queue_nr)
1579	{
1580	struct intel_pt_queue *ptq = queue->priv;
1581
1582	if (list_empty(&queue->head))
1583	return 0;
1584
1585	if (!ptq) {
1586	ptq = intel_pt_alloc_queue(pt, queue_nr);
1587	if (!ptq)
1588	return -ENOMEM;
1589	queue->priv = ptq;
1590
1591	if (queue->cpu != -1)
1592	ptq->cpu = queue->cpu;
1593	ptq->tid = queue->tid;
1594
1595	ptq->cbr_seen = UINT_MAX;
1596
1597	if (pt->sampling_mode && !pt->snapshot_mode &&
1598	pt->timeless_decoding)
1599	ptq->step_through_buffers = true;
1600
1601	ptq->sync_switch = pt->sync_switch;
1602
1603	intel_pt_setup_time_range(pt, ptq);
1604	}
1605
1606	if (!ptq->on_heap &&
1607	(!ptq->sync_switch \|\|
1608	ptq->switch_state != INTEL_PT_SS_EXPECTING_SWITCH_EVENT)) {
1609	const struct intel_pt_state *state;
1610	int ret;
1611
1612	if (pt->timeless_decoding)
1613	return 0;
1614
1615	intel_pt_log("queue %u getting timestamp\n", queue_nr);
1616	intel_pt_log("queue %u decoding cpu %d pid %d tid %d\n",
1617	queue_nr, ptq->cpu, ptq->pid, ptq->tid);
1618
1619	if (ptq->sel_start && ptq->sel_timestamp) {
1620	ret = intel_pt_fast_forward(ptq->decoder,
1621	ptq->sel_timestamp);
1622	if (ret)
1623	return ret;
1624	}
1625
1626	while (1) {
1627	state = intel_pt_decode(ptq->decoder);
1628	if (state->err) {
1629	if (state->err == INTEL_PT_ERR_NODATA) {
1630	intel_pt_log("queue %u has no timestamp\n",
1631	queue_nr);
1632	return 0;
1633	}
1634	continue;
1635	}
1636	if (state->timestamp)
1637	break;
1638	}
1639
1640	ptq->timestamp = state->timestamp;
1641	intel_pt_log("queue %u timestamp 0x%" PRIx64 "\n",
1642	queue_nr, ptq->timestamp);
1643	ptq->state = state;
1644	ptq->have_sample = true;
1645	if (ptq->sel_start && ptq->sel_timestamp &&
1646	ptq->timestamp < ptq->sel_timestamp)
1647	ptq->have_sample = false;
1648	intel_pt_sample_flags(ptq);
1649	ret = auxtrace_heap__add(&pt->heap, queue_nr, ptq->timestamp);
1650	if (ret)
1651	return ret;
1652	ptq->on_heap = true;
1653	}
1654
1655	return 0;
1656	}
1657
1658	static int intel_pt_setup_queues(struct intel_pt *pt)
1659	{
1660	unsigned int i;
1661	int ret;
1662
1663	for (i = 0; i < pt->queues.nr_queues; i++) {
1664	ret = intel_pt_setup_queue(pt, &pt->queues.queue_array[i], i);
1665	if (ret)
1666	return ret;
1667	}
1668	return 0;
1669	}
1670
1671	static inline bool intel_pt_skip_event(struct intel_pt *pt)
1672	{
1673	return pt->synth_opts.initial_skip &&
1674	pt->num_events++ < pt->synth_opts.initial_skip;
1675	}
1676
1677	/*
1678	* Cannot count CBR as skipped because it won't go away until cbr == cbr_seen.
1679	* Also ensure CBR is first non-skipped event by allowing for 4 more samples
1680	* from this decoder state.
1681	*/
1682	static inline bool intel_pt_skip_cbr_event(struct intel_pt *pt)
1683	{
1684	return pt->synth_opts.initial_skip &&
1685	pt->num_events + 4 < pt->synth_opts.initial_skip;
1686	}
1687
1688	static void intel_pt_prep_a_sample(struct intel_pt_queue *ptq,
1689	union perf_event *event,
1690	struct perf_sample *sample)
1691	{
1692	event->sample.header.type = PERF_RECORD_SAMPLE;
1693	event->sample.header.size = sizeof(struct perf_event_header);
1694
1695	sample->pid = ptq->pid;
1696	sample->tid = ptq->tid;
1697
1698	if (ptq->pt->have_guest_sideband) {
1699	if ((ptq->state->from_ip && ptq->state->from_nr) \|\|
1700	(ptq->state->to_ip && ptq->state->to_nr)) {
1701	sample->pid = ptq->guest_pid;
1702	sample->tid = ptq->guest_tid;
1703	sample->machine_pid = ptq->guest_machine_pid;
1704	sample->vcpu = ptq->vcpu;
1705	}
1706	}
1707
1708	sample->cpu = ptq->cpu;
1709	sample->insn_len = ptq->insn_len;
1710	memcpy(sample->insn, ptq->insn, INTEL_PT_INSN_BUF_SZ);
1711	}
1712
1713	static void intel_pt_prep_b_sample(struct intel_pt *pt,
1714	struct intel_pt_queue *ptq,
1715	union perf_event *event,
1716	struct perf_sample *sample)
1717	{
1718	intel_pt_prep_a_sample(ptq, event, sample);
1719
1720	if (!pt->timeless_decoding)
1721	sample->time = tsc_to_perf_time(ptq->timestamp, &pt->tc);
1722
1723	sample->ip = ptq->state->from_ip;
1724	sample->addr = ptq->state->to_ip;
1725	sample->cpumode = intel_pt_cpumode(ptq, sample->ip, sample->addr);
1726	sample->period = 1;
1727	sample->flags = ptq->flags;
1728
1729	event->sample.header.misc = sample->cpumode;
1730	}
1731
1732	static int intel_pt_inject_event(struct intel_pt pt, union perf_event event,
1733	struct perf_sample *sample, u64 type)
1734	{
1735	struct evsel *evsel = sample->evsel;
1736	u64 branch_sample_type = 0;
1737	size_t sz;
1738
1739	if (!evsel && pt->session && pt->session->evlist)
1740	evsel = evlist__id2evsel(pt->session->evlist, sample->id);
1741
1742	if (evsel)
1743	branch_sample_type = evsel->core.attr.branch_sample_type;
1744
1745	event->header.type = PERF_RECORD_SAMPLE;
1746	sz = perf_event__sample_event_size(sample, type, /read_format=/0,
1747	branch_sample_type);
1748	if (sz >= PERF_SAMPLE_MAX_SIZE) {
1749	pr_err("Sample size %zu exceeds max size %d\n", sz, PERF_SAMPLE_MAX_SIZE);
1750	return -EFAULT;
1751	}
1752	event->header.size = sz;
1753
1754	return perf_event__synthesize_sample(event, type, /read_format=/0,
1755	branch_sample_type, sample);
1756	}
1757
1758	static inline int intel_pt_opt_inject(struct intel_pt *pt,
1759	union perf_event *event,
1760	struct perf_sample *sample, u64 type)
1761	{
1762	if (!pt->synth_opts.inject)
1763	return 0;
1764
1765	return intel_pt_inject_event(pt, event, sample, type);
1766	}
1767
1768	static int intel_pt_deliver_synth_event(struct intel_pt *pt,
1769	union perf_event *event,
1770	struct perf_sample *sample, u64 type)
1771	{
1772	int ret;
1773
1774	ret = intel_pt_opt_inject(pt, event, sample, type);
1775	if (ret)
1776	return ret;
1777
1778	ret = perf_session__deliver_synth_event(pt->session, event, sample);
1779	if (ret)
1780	pr_err("Intel PT: failed to deliver event, error %d\n", ret);
1781
1782	return ret;
1783	}
1784
1785	static int intel_pt_synth_branch_sample(struct intel_pt_queue *ptq)
1786	{
1787	struct intel_pt *pt = ptq->pt;
1788	union perf_event *event = ptq->event_buf;
1789	struct perf_sample sample;
1790	struct dummy_branch_stack {
1791	u64 nr;
1792	u64 hw_idx;
1793	struct branch_entry entries;
1794	} dummy_bs;
1795	int ret;
1796
1797	if (pt->branches_filter && !(pt->branches_filter & ptq->flags))
1798	return 0;
1799
1800	if (intel_pt_skip_event(pt))
1801	return 0;
1802
1803	perf_sample__init(&sample, /all=/true);
1804	intel_pt_prep_b_sample(pt, ptq, event, &sample);
1805
1806	sample.id = ptq->pt->branches_id;
1807	sample.stream_id = ptq->pt->branches_id;
1808
1809	/*
1810	* perf report cannot handle events without a branch stack when using
1811	* SORT_MODE__BRANCH so make a dummy one.
1812	*/
1813	if (pt->synth_opts.last_branch && sort__mode == SORT_MODE__BRANCH) {
1814	dummy_bs = (struct dummy_branch_stack){
1815	.nr = 1,
1816	.hw_idx = -1ULL,
1817	.entries = {
1818	.from = sample.ip,
1819	.to = sample.addr,
1820	},
1821	};
1822	sample.branch_stack = (struct branch_stack *)&dummy_bs;
1823	}
1824
1825	if (ptq->sample_ipc)
1826	sample.cyc_cnt = ptq->ipc_cyc_cnt - ptq->last_br_cyc_cnt;
1827	if (sample.cyc_cnt) {
1828	sample.insn_cnt = ptq->ipc_insn_cnt - ptq->last_br_insn_cnt;
1829	ptq->last_br_insn_cnt = ptq->ipc_insn_cnt;
1830	ptq->last_br_cyc_cnt = ptq->ipc_cyc_cnt;
1831	}
1832
1833	ret = intel_pt_deliver_synth_event(pt, event, &sample,
1834	pt->branches_sample_type);
1835	perf_sample__exit(&sample);
1836	return ret;
1837	}
1838
1839	static void intel_pt_prep_sample(struct intel_pt *pt,
1840	struct intel_pt_queue *ptq,
1841	union perf_event *event,
1842	struct perf_sample *sample)
1843	{
1844	intel_pt_prep_b_sample(pt, ptq, event, sample);
1845
1846	if (pt->synth_opts.callchain) {
1847	thread_stack__sample(ptq->thread, ptq->cpu, ptq->chain,
1848	pt->synth_opts.callchain_sz + 1,
1849	sample->ip, pt->kernel_start);
1850	sample->callchain = ptq->chain;
1851	}
1852
1853	if (pt->synth_opts.last_branch) {
1854	thread_stack__br_sample(ptq->thread, ptq->cpu, ptq->last_branch,
1855	pt->br_stack_sz);
1856	sample->branch_stack = ptq->last_branch;
1857	}
1858	}
1859
1860	static int intel_pt_synth_instruction_sample(struct intel_pt_queue *ptq)
1861	{
1862	struct intel_pt *pt = ptq->pt;
1863	union perf_event *event = ptq->event_buf;
1864	struct perf_sample sample;
1865	int ret;
1866
1867	if (intel_pt_skip_event(pt))
1868	return 0;
1869
1870	perf_sample__init(&sample, /all=/true);
1871	intel_pt_prep_sample(pt, ptq, event, &sample);
1872
1873	sample.id = ptq->pt->instructions_id;
1874	sample.stream_id = ptq->pt->instructions_id;
1875	if (pt->synth_opts.quick)
1876	sample.period = 1;
1877	else
1878	sample.period = ptq->state->tot_insn_cnt - ptq->last_insn_cnt;
1879
1880	if (ptq->sample_ipc)
1881	sample.cyc_cnt = ptq->ipc_cyc_cnt - ptq->last_in_cyc_cnt;
1882	if (sample.cyc_cnt) {
1883	sample.insn_cnt = ptq->ipc_insn_cnt - ptq->last_in_insn_cnt;
1884	ptq->last_in_insn_cnt = ptq->ipc_insn_cnt;
1885	ptq->last_in_cyc_cnt = ptq->ipc_cyc_cnt;
1886	}
1887
1888	ptq->last_insn_cnt = ptq->state->tot_insn_cnt;
1889
1890	ret = intel_pt_deliver_synth_event(pt, event, &sample,
1891	pt->instructions_sample_type);
1892	perf_sample__exit(&sample);
1893	return ret;
1894	}
1895
1896	static int intel_pt_synth_cycle_sample(struct intel_pt_queue *ptq)
1897	{
1898	struct intel_pt *pt = ptq->pt;
1899	union perf_event *event = ptq->event_buf;
1900	struct perf_sample sample;
1901	u64 period = 0;
1902	int ret;
1903
1904	if (ptq->sample_ipc)
1905	period = ptq->ipc_cyc_cnt - ptq->last_cy_cyc_cnt;
1906
1907	if (!period \|\| intel_pt_skip_event(pt))
1908	return 0;
1909
1910	perf_sample__init(&sample, /all=/true);
1911	intel_pt_prep_sample(pt, ptq, event, &sample);
1912
1913	sample.id = ptq->pt->cycles_id;
1914	sample.stream_id = ptq->pt->cycles_id;
1915	sample.period = period;
1916
1917	sample.cyc_cnt = period;
1918	sample.insn_cnt = ptq->ipc_insn_cnt - ptq->last_cy_insn_cnt;
1919	ptq->last_cy_insn_cnt = ptq->ipc_insn_cnt;
1920	ptq->last_cy_cyc_cnt = ptq->ipc_cyc_cnt;
1921
1922	ret = intel_pt_deliver_synth_event(pt, event, &sample, pt->cycles_sample_type);
1923	perf_sample__exit(&sample);
1924	return ret;
1925	}
1926
1927	static int intel_pt_synth_transaction_sample(struct intel_pt_queue *ptq)
1928	{
1929	struct intel_pt *pt = ptq->pt;
1930	union perf_event *event = ptq->event_buf;
1931	struct perf_sample sample;
1932	int ret;
1933
1934	if (intel_pt_skip_event(pt))
1935	return 0;
1936
1937	perf_sample__init(&sample, /all=/true);
1938	intel_pt_prep_sample(pt, ptq, event, &sample);
1939
1940	sample.id = ptq->pt->transactions_id;
1941	sample.stream_id = ptq->pt->transactions_id;
1942
1943	ret = intel_pt_deliver_synth_event(pt, event, &sample,
1944	pt->transactions_sample_type);
1945	perf_sample__exit(&sample);
1946	return ret;
1947	}
1948
1949	static void intel_pt_prep_p_sample(struct intel_pt *pt,
1950	struct intel_pt_queue *ptq,
1951	union perf_event *event,
1952	struct perf_sample *sample)
1953	{
1954	intel_pt_prep_sample(pt, ptq, event, sample);
1955
1956	/*
1957	* Zero IP is used to mean "trace start" but that is not the case for
1958	* power or PTWRITE events with no IP, so clear the flags.
1959	*/
1960	if (!sample->ip)
1961	sample->flags = 0;
1962	}
1963
1964	static int intel_pt_synth_ptwrite_sample(struct intel_pt_queue *ptq)
1965	{
1966	struct intel_pt *pt = ptq->pt;
1967	union perf_event *event = ptq->event_buf;
1968	struct perf_sample sample = { .ip = 0, };
1969	struct perf_synth_intel_ptwrite raw;
1970	int ret;
1971
1972	if (intel_pt_skip_event(pt))
1973	return 0;
1974
1975	intel_pt_prep_p_sample(pt, ptq, event, &sample);
1976
1977	sample.id = ptq->pt->ptwrites_id;
1978	sample.stream_id = ptq->pt->ptwrites_id;
1979
1980	raw.flags = 0;
1981	raw.ip = !!(ptq->state->flags & INTEL_PT_FUP_IP);
1982	raw.payload = cpu_to_le64(ptq->state->ptw_payload);
1983
1984	sample.raw_size = perf_synth__raw_size(raw);
1985	sample.raw_data = perf_synth__raw_data(&raw);
1986
1987	ret = intel_pt_deliver_synth_event(pt, event, &sample, pt->ptwrites_sample_type);
1988	perf_sample__exit(&sample);
1989	return ret;
1990	}
1991
1992	static int intel_pt_synth_cbr_sample(struct intel_pt_queue *ptq)
1993	{
1994	struct intel_pt *pt = ptq->pt;
1995	union perf_event *event = ptq->event_buf;
1996	struct perf_sample sample;
1997	struct perf_synth_intel_cbr raw;
1998	u32 flags;
1999	int ret;
2000
2001	if (intel_pt_skip_cbr_event(pt))
2002	return 0;
2003
2004	ptq->cbr_seen = ptq->state->cbr;
2005
2006	perf_sample__init(&sample, /all=/true);
2007	intel_pt_prep_p_sample(pt, ptq, event, &sample);
2008
2009	sample.id = ptq->pt->cbr_id;
2010	sample.stream_id = ptq->pt->cbr_id;
2011
2012	flags = (u16)ptq->state->cbr_payload \| (pt->max_non_turbo_ratio << 16);
2013	raw.flags = cpu_to_le32(flags);
2014	raw.freq = cpu_to_le32(raw.cbr * pt->cbr2khz);
2015	raw.reserved3 = 0;
2016
2017	sample.raw_size = perf_synth__raw_size(raw);
2018	sample.raw_data = perf_synth__raw_data(&raw);
2019
2020	ret = intel_pt_deliver_synth_event(pt, event, &sample,
2021	pt->pwr_events_sample_type);
2022	perf_sample__exit(&sample);
2023	return ret;
2024	}
2025
2026	static int intel_pt_synth_psb_sample(struct intel_pt_queue *ptq)
2027	{
2028	struct intel_pt *pt = ptq->pt;
2029	union perf_event *event = ptq->event_buf;
2030	struct perf_sample sample;
2031	struct perf_synth_intel_psb raw;
2032	int ret;
2033
2034	if (intel_pt_skip_event(pt))
2035	return 0;
2036
2037	perf_sample__init(&sample, /all=/true);
2038	intel_pt_prep_p_sample(pt, ptq, event, &sample);
2039
2040	sample.id = ptq->pt->psb_id;
2041	sample.stream_id = ptq->pt->psb_id;
2042	sample.flags = 0;
2043
2044	raw.reserved = 0;
2045	raw.offset = ptq->state->psb_offset;
2046
2047	sample.raw_size = perf_synth__raw_size(raw);
2048	sample.raw_data = perf_synth__raw_data(&raw);
2049
2050	ret = intel_pt_deliver_synth_event(pt, event, &sample,
2051	pt->pwr_events_sample_type);
2052	perf_sample__exit(&sample);
2053	return ret;
2054	}
2055
2056	static int intel_pt_synth_mwait_sample(struct intel_pt_queue *ptq)
2057	{
2058	struct intel_pt *pt = ptq->pt;
2059	union perf_event *event = ptq->event_buf;
2060	struct perf_sample sample;
2061	struct perf_synth_intel_mwait raw;
2062	int ret;
2063
2064	if (intel_pt_skip_event(pt))
2065	return 0;
2066
2067	perf_sample__init(&sample, /all=/true);
2068	intel_pt_prep_p_sample(pt, ptq, event, &sample);
2069
2070	sample.id = ptq->pt->mwait_id;
2071	sample.stream_id = ptq->pt->mwait_id;
2072
2073	raw.reserved = 0;
2074	raw.payload = cpu_to_le64(ptq->state->mwait_payload);
2075
2076	sample.raw_size = perf_synth__raw_size(raw);
2077	sample.raw_data = perf_synth__raw_data(&raw);
2078
2079	ret = intel_pt_deliver_synth_event(pt, event, &sample,
2080	pt->pwr_events_sample_type);
2081	perf_sample__exit(&sample);
2082	return ret;
2083	}
2084
2085	static int intel_pt_synth_pwre_sample(struct intel_pt_queue *ptq)
2086	{
2087	struct intel_pt *pt = ptq->pt;
2088	union perf_event *event = ptq->event_buf;
2089	struct perf_sample sample;
2090	struct perf_synth_intel_pwre raw;
2091	int ret;
2092
2093	if (intel_pt_skip_event(pt))
2094	return 0;
2095
2096	perf_sample__init(&sample, /all=/true);
2097	intel_pt_prep_p_sample(pt, ptq, event, &sample);
2098
2099	sample.id = ptq->pt->pwre_id;
2100	sample.stream_id = ptq->pt->pwre_id;
2101
2102	raw.reserved = 0;
2103	raw.payload = cpu_to_le64(ptq->state->pwre_payload);
2104
2105	sample.raw_size = perf_synth__raw_size(raw);
2106	sample.raw_data = perf_synth__raw_data(&raw);
2107
2108	ret = intel_pt_deliver_synth_event(pt, event, &sample,
2109	pt->pwr_events_sample_type);
2110	perf_sample__exit(&sample);
2111	return ret;
2112	}
2113
2114	static int intel_pt_synth_exstop_sample(struct intel_pt_queue *ptq)
2115	{
2116	struct intel_pt *pt = ptq->pt;
2117	union perf_event *event = ptq->event_buf;
2118	struct perf_sample sample;
2119	struct perf_synth_intel_exstop raw;
2120	int ret;
2121
2122	if (intel_pt_skip_event(pt))
2123	return 0;
2124
2125	perf_sample__init(&sample, /all=/true);
2126	intel_pt_prep_p_sample(pt, ptq, event, &sample);
2127
2128	sample.id = ptq->pt->exstop_id;
2129	sample.stream_id = ptq->pt->exstop_id;
2130
2131	raw.flags = 0;
2132	raw.ip = !!(ptq->state->flags & INTEL_PT_FUP_IP);
2133
2134	sample.raw_size = perf_synth__raw_size(raw);
2135	sample.raw_data = perf_synth__raw_data(&raw);
2136
2137	ret = intel_pt_deliver_synth_event(pt, event, &sample,
2138	pt->pwr_events_sample_type);
2139	perf_sample__exit(&sample);
2140	return ret;
2141	}
2142
2143	static int intel_pt_synth_pwrx_sample(struct intel_pt_queue *ptq)
2144	{
2145	struct intel_pt *pt = ptq->pt;
2146	union perf_event *event = ptq->event_buf;
2147	struct perf_sample sample;
2148	struct perf_synth_intel_pwrx raw;
2149	int ret;
2150
2151	if (intel_pt_skip_event(pt))
2152	return 0;
2153
2154	perf_sample__init(&sample, /all=/true);
2155	intel_pt_prep_p_sample(pt, ptq, event, &sample);
2156
2157	sample.id = ptq->pt->pwrx_id;
2158	sample.stream_id = ptq->pt->pwrx_id;
2159
2160	raw.reserved = 0;
2161	raw.payload = cpu_to_le64(ptq->state->pwrx_payload);
2162
2163	sample.raw_size = perf_synth__raw_size(raw);
2164	sample.raw_data = perf_synth__raw_data(&raw);
2165
2166	ret = intel_pt_deliver_synth_event(pt, event, &sample,
2167	pt->pwr_events_sample_type);
2168	perf_sample__exit(&sample);
2169	return ret;
2170	}
2171
2172	/*
2173	* PEBS gp_regs array indexes plus 1 so that 0 means not present. Refer
2174	* intel_pt_add_gp_regs().
2175	*/
2176	static const int pebs_gp_regs[] = {
2177	[PERF_REG_X86_FLAGS] = 1,
2178	[PERF_REG_X86_IP] = 2,
2179	[PERF_REG_X86_AX] = 3,
2180	[PERF_REG_X86_CX] = 4,
2181	[PERF_REG_X86_DX] = 5,
2182	[PERF_REG_X86_BX] = 6,
2183	[PERF_REG_X86_SP] = 7,
2184	[PERF_REG_X86_BP] = 8,
2185	[PERF_REG_X86_SI] = 9,
2186	[PERF_REG_X86_DI] = 10,
2187	[PERF_REG_X86_R8] = 11,
2188	[PERF_REG_X86_R9] = 12,
2189	[PERF_REG_X86_R10] = 13,
2190	[PERF_REG_X86_R11] = 14,
2191	[PERF_REG_X86_R12] = 15,
2192	[PERF_REG_X86_R13] = 16,
2193	[PERF_REG_X86_R14] = 17,
2194	[PERF_REG_X86_R15] = 18,
2195	};
2196
2197	static u64 intel_pt_add_gp_regs(struct regs_dump intr_regs, u64 *pos,
2198	const struct intel_pt_blk_items *items,
2199	u64 regs_mask)
2200	{
2201	const u64 *gp_regs = items->val[INTEL_PT_GP_REGS_POS];
2202	u32 mask = items->mask[INTEL_PT_GP_REGS_POS];
2203	u32 bit;
2204	int i;
2205
2206	for (i = 0, bit = 1; i < PERF_REG_X86_64_MAX; i++, bit <<= 1) {
2207	/* Get the PEBS gp_regs array index */
2208	int n = pebs_gp_regs[i] - 1;
2209
2210	if (n < 0)
2211	continue;
2212	/*
2213	* Add only registers that were requested (i.e. 'regs_mask') and
2214	* that were provided (i.e. 'mask'), and update the resulting
2215	* mask (i.e. 'intr_regs->mask') accordingly.
2216	*/
2217	if (mask & 1 << n && regs_mask & bit) {
2218	intr_regs->mask \|= bit;
2219	*pos++ = gp_regs[n];
2220	}
2221	}
2222
2223	return pos;
2224	}
2225
2226	#ifndef PERF_REG_X86_XMM0
2227	#define PERF_REG_X86_XMM0 32
2228	#endif
2229
2230	static void intel_pt_add_xmm(struct regs_dump intr_regs, u64 pos,
2231	const struct intel_pt_blk_items *items,
2232	u64 regs_mask)
2233	{
2234	u32 mask = items->has_xmm & (regs_mask >> PERF_REG_X86_XMM0);
2235	const u64 *xmm = items->xmm;
2236
2237	/*
2238	* If there are any XMM registers, then there should be all of them.
2239	* Nevertheless, follow the logic to add only registers that were
2240	* requested (i.e. 'regs_mask') and that were provided (i.e. 'mask'),
2241	* and update the resulting mask (i.e. 'intr_regs->mask') accordingly.
2242	*/
2243	intr_regs->mask \|= (u64)mask << PERF_REG_X86_XMM0;
2244
2245	for (; mask; mask >>= 1, xmm++) {
2246	if (mask & 1)
2247	pos++ = xmm;
2248	}
2249	}
2250
2251	#define LBR_INFO_MISPRED (1ULL << 63)
2252	#define LBR_INFO_IN_TX (1ULL << 62)
2253	#define LBR_INFO_ABORT (1ULL << 61)
2254	#define LBR_INFO_CYCLES 0xffff
2255
2256	/* Refer kernel's intel_pmu_store_pebs_lbrs() */
2257	static u64 intel_pt_lbr_flags(u64 info)
2258	{
2259	union {
2260	struct branch_flags flags;
2261	u64 result;
2262	} u;
2263
2264	u.result = 0;
2265	u.flags.mispred = !!(info & LBR_INFO_MISPRED);
2266	u.flags.predicted = !(info & LBR_INFO_MISPRED);
2267	u.flags.in_tx = !!(info & LBR_INFO_IN_TX);
2268	u.flags.abort = !!(info & LBR_INFO_ABORT);
2269	u.flags.cycles = info & LBR_INFO_CYCLES;
2270
2271	return u.result;
2272	}
2273
2274	static void intel_pt_add_lbrs(struct branch_stack *br_stack,
2275	const struct intel_pt_blk_items *items)
2276	{
2277	u64 *to;
2278	int i;
2279
2280	br_stack->nr = 0;
2281
2282	to = &br_stack->entries[0].from;
2283
2284	for (i = INTEL_PT_LBR_0_POS; i <= INTEL_PT_LBR_2_POS; i++) {
2285	u32 mask = items->mask[i];
2286	const u64 *from = items->val[i];
2287
2288	for (; mask; mask >>= 3, from += 3) {
2289	if ((mask & 7) == 7) {
2290	*to++ = from[0];
2291	*to++ = from[1];
2292	*to++ = intel_pt_lbr_flags(from[2]);
2293	br_stack->nr += 1;
2294	}
2295	}
2296	}
2297	}
2298
2299	#define P(a, b) PERF_MEM_S(a, b)
2300	#define OP_LH (P(OP, LOAD) \| P(LVL, HIT))
2301	#define LEVEL(x) P(LVLNUM, x)
2302	#define REM P(REMOTE, REMOTE)
2303	#define SNOOP_NONE_MISS (P(SNOOP, NONE) \| P(SNOOP, MISS))
2304
2305	#define PERF_PEBS_DATA_SOURCE_GRT_MAX 0x10
2306	#define PERF_PEBS_DATA_SOURCE_GRT_MASK (PERF_PEBS_DATA_SOURCE_GRT_MAX - 1)
2307
2308	/* Based on kernel __intel_pmu_pebs_data_source_grt() and pebs_data_source */
2309	static const u64 pebs_data_source_grt[PERF_PEBS_DATA_SOURCE_GRT_MAX] = {
2310	P(OP, LOAD) \| P(LVL, MISS) \| LEVEL(L3) \| P(SNOOP, NA), /* L3 miss\|SNP N/A */
2311	OP_LH \| P(LVL, L1) \| LEVEL(L1) \| P(SNOOP, NONE), /* L1 hit\|SNP None */
2312	OP_LH \| P(LVL, LFB) \| LEVEL(LFB) \| P(SNOOP, NONE), /* LFB/MAB hit\|SNP None */
2313	OP_LH \| P(LVL, L2) \| LEVEL(L2) \| P(SNOOP, NONE), /* L2 hit\|SNP None */
2314	OP_LH \| P(LVL, L3) \| LEVEL(L3) \| P(SNOOP, NONE), /* L3 hit\|SNP None */
2315	OP_LH \| P(LVL, L3) \| LEVEL(L3) \| P(SNOOP, HIT), /* L3 hit\|SNP Hit */
2316	OP_LH \| P(LVL, L3) \| LEVEL(L3) \| P(SNOOP, HITM), /* L3 hit\|SNP HitM */
2317	OP_LH \| P(LVL, L3) \| LEVEL(L3) \| P(SNOOP, HITM), /* L3 hit\|SNP HitM */
2318	OP_LH \| P(LVL, L3) \| LEVEL(L3) \| P(SNOOPX, FWD), /* L3 hit\|SNP Fwd */
2319	OP_LH \| P(LVL, REM_CCE1) \| REM \| LEVEL(L3) \| P(SNOOP, HITM), /* Remote L3 hit\|SNP HitM */
2320	OP_LH \| P(LVL, LOC_RAM) \| LEVEL(RAM) \| P(SNOOP, HIT), /* RAM hit\|SNP Hit */
2321	OP_LH \| P(LVL, REM_RAM1) \| REM \| LEVEL(L3) \| P(SNOOP, HIT), /* Remote L3 hit\|SNP Hit */
2322	OP_LH \| P(LVL, LOC_RAM) \| LEVEL(RAM) \| SNOOP_NONE_MISS, /* RAM hit\|SNP None or Miss */
2323	OP_LH \| P(LVL, REM_RAM1) \| LEVEL(RAM) \| REM \| SNOOP_NONE_MISS, /* Remote RAM hit\|SNP None or Miss */
2324	OP_LH \| P(LVL, IO) \| LEVEL(NA) \| P(SNOOP, NONE), /* I/O hit\|SNP None */
2325	OP_LH \| P(LVL, UNC) \| LEVEL(NA) \| P(SNOOP, NONE), /* Uncached hit\|SNP None */
2326	};
2327
2328	/* Based on kernel __intel_pmu_pebs_data_source_cmt() and pebs_data_source */
2329	static const u64 pebs_data_source_cmt[PERF_PEBS_DATA_SOURCE_GRT_MAX] = {
2330	P(OP, LOAD) \| P(LVL, MISS) \| LEVEL(L3) \| P(SNOOP, NA), /* L3 miss\|SNP N/A */
2331	OP_LH \| P(LVL, L1) \| LEVEL(L1) \| P(SNOOP, NONE), /* L1 hit\|SNP None */
2332	OP_LH \| P(LVL, LFB) \| LEVEL(LFB) \| P(SNOOP, NONE), /* LFB/MAB hit\|SNP None */
2333	OP_LH \| P(LVL, L2) \| LEVEL(L2) \| P(SNOOP, NONE), /* L2 hit\|SNP None */
2334	OP_LH \| P(LVL, L3) \| LEVEL(L3) \| P(SNOOP, NONE), /* L3 hit\|SNP None */
2335	OP_LH \| P(LVL, L3) \| LEVEL(L3) \| P(SNOOP, MISS), /* L3 hit\|SNP Hit */
2336	OP_LH \| P(LVL, L3) \| LEVEL(L3) \| P(SNOOP, HIT), /* L3 hit\|SNP HitM */
2337	OP_LH \| P(LVL, L3) \| LEVEL(L3) \| P(SNOOPX, FWD), /* L3 hit\|SNP HitM */
2338	OP_LH \| P(LVL, L3) \| LEVEL(L3) \| P(SNOOP, HITM), /* L3 hit\|SNP Fwd */
2339	OP_LH \| P(LVL, REM_CCE1) \| REM \| LEVEL(L3) \| P(SNOOP, HITM), /* Remote L3 hit\|SNP HitM */
2340	OP_LH \| P(LVL, LOC_RAM) \| LEVEL(RAM) \| P(SNOOP, NONE), /* RAM hit\|SNP Hit */
2341	OP_LH \| LEVEL(RAM) \| REM \| P(SNOOP, NONE), /* Remote L3 hit\|SNP Hit */
2342	OP_LH \| LEVEL(RAM) \| REM \| P(SNOOPX, FWD), /* RAM hit\|SNP None or Miss */
2343	OP_LH \| LEVEL(RAM) \| REM \| P(SNOOP, HITM), /* Remote RAM hit\|SNP None or Miss */
2344	OP_LH \| P(LVL, IO) \| LEVEL(NA) \| P(SNOOP, NONE), /* I/O hit\|SNP None */
2345	OP_LH \| P(LVL, UNC) \| LEVEL(NA) \| P(SNOOP, NONE), /* Uncached hit\|SNP None */
2346	};
2347
2348	/* Based on kernel pebs_set_tlb_lock() */
2349	static inline void pebs_set_tlb_lock(u64 *val, bool tlb, bool lock)
2350	{
2351	/*
2352	* TLB access
2353	* 0 = did not miss 2nd level TLB
2354	* 1 = missed 2nd level TLB
2355	*/
2356	if (tlb)
2357	*val \|= P(TLB, MISS) \| P(TLB, L2);
2358	else
2359	*val \|= P(TLB, HIT) \| P(TLB, L1) \| P(TLB, L2);
2360
2361	/* locked prefix */
2362	if (lock)
2363	*val \|= P(LOCK, LOCKED);
2364	}
2365
2366	/* Based on kernel __grt_latency_data() */
2367	static u64 intel_pt_grt_latency_data(u8 dse, bool tlb, bool lock, bool blk,
2368	const u64 *pebs_data_source)
2369	{
2370	u64 val;
2371
2372	dse &= PERF_PEBS_DATA_SOURCE_GRT_MASK;
2373	val = pebs_data_source[dse];
2374
2375	pebs_set_tlb_lock(&val, tlb, lock);
2376
2377	if (blk)
2378	val \|= P(BLK, DATA);
2379	else
2380	val \|= P(BLK, NA);
2381
2382	return val;
2383	}
2384
2385	/* Default value for data source */
2386	#define PERF_MEM_NA (PERF_MEM_S(OP, NA) \|\
2387	PERF_MEM_S(LVL, NA) \|\
2388	PERF_MEM_S(SNOOP, NA) \|\
2389	PERF_MEM_S(LOCK, NA) \|\
2390	PERF_MEM_S(TLB, NA) \|\
2391	PERF_MEM_S(LVLNUM, NA))
2392
2393	enum DATA_SRC_FORMAT {
2394	DATA_SRC_FORMAT_ERR = -1,
2395	DATA_SRC_FORMAT_NA = 0,
2396	DATA_SRC_FORMAT_GRT = 1,
2397	DATA_SRC_FORMAT_CMT = 2,
2398	};
2399
2400	/* Based on kernel grt_latency_data() and cmt_latency_data */
2401	static u64 intel_pt_get_data_src(u64 mem_aux_info, int data_src_fmt)
2402	{
2403	switch (data_src_fmt) {
2404	case DATA_SRC_FORMAT_GRT: {
2405	union {
2406	u64 val;
2407	struct {
2408	unsigned int dse:4;
2409	unsigned int locked:1;
2410	unsigned int stlb_miss:1;
2411	unsigned int fwd_blk:1;
2412	unsigned int reserved:25;
2413	};
2414	} x = {.val = mem_aux_info};
2415	return intel_pt_grt_latency_data(x.dse, x.stlb_miss, x.locked, x.fwd_blk,
2416	pebs_data_source_grt);
2417	}
2418	case DATA_SRC_FORMAT_CMT: {
2419	union {
2420	u64 val;
2421	struct {
2422	unsigned int dse:5;
2423	unsigned int locked:1;
2424	unsigned int stlb_miss:1;
2425	unsigned int fwd_blk:1;
2426	unsigned int reserved:24;
2427	};
2428	} x = {.val = mem_aux_info};
2429	return intel_pt_grt_latency_data(x.dse, x.stlb_miss, x.locked, x.fwd_blk,
2430	pebs_data_source_cmt);
2431	}
2432	default:
2433	return PERF_MEM_NA;
2434	}
2435	}
2436
2437	static int intel_pt_do_synth_pebs_sample(struct intel_pt_queue ptq, struct evsel evsel,
2438	u64 id, int data_src_fmt)
2439	{
2440	const struct intel_pt_blk_items *items = &ptq->state->items;
2441	struct perf_sample sample;
2442	union perf_event *event = ptq->event_buf;
2443	struct intel_pt *pt = ptq->pt;
2444	u64 sample_type = evsel->core.attr.sample_type;
2445	u8 cpumode;
2446	u64 regs[8 * sizeof(sample.intr_regs->mask)];
2447	int ret;
2448
2449	if (intel_pt_skip_event(pt))
2450	return 0;
2451
2452	perf_sample__init(&sample, /all=/true);
2453	intel_pt_prep_a_sample(ptq, event, &sample);
2454
2455	sample.id = id;
2456	sample.stream_id = id;
2457
2458	if (!evsel->core.attr.freq)
2459	sample.period = evsel->core.attr.sample_period;
2460
2461	/* No support for non-zero CS base */
2462	if (items->has_ip)
2463	sample.ip = items->ip;
2464	else if (items->has_rip)
2465	sample.ip = items->rip;
2466	else
2467	sample.ip = ptq->state->from_ip;
2468
2469	cpumode = intel_pt_cpumode(ptq, sample.ip, 0);
2470
2471	event->sample.header.misc = cpumode \| PERF_RECORD_MISC_EXACT_IP;
2472
2473	sample.cpumode = cpumode;
2474
2475	if (sample_type & PERF_SAMPLE_TIME) {
2476	u64 timestamp = 0;
2477
2478	if (items->has_timestamp)
2479	timestamp = items->timestamp;
2480	else if (!pt->timeless_decoding)
2481	timestamp = ptq->timestamp;
2482	if (timestamp)
2483	sample.time = tsc_to_perf_time(timestamp, &pt->tc);
2484	}
2485
2486	if (sample_type & PERF_SAMPLE_CALLCHAIN &&
2487	pt->synth_opts.callchain) {
2488	thread_stack__sample(ptq->thread, ptq->cpu, ptq->chain,
2489	pt->synth_opts.callchain_sz, sample.ip,
2490	pt->kernel_start);
2491	sample.callchain = ptq->chain;
2492	}
2493
2494	if (sample_type & PERF_SAMPLE_REGS_INTR &&
2495	(items->mask[INTEL_PT_GP_REGS_POS] \|\|
2496	items->mask[INTEL_PT_XMM_POS])) {
2497	u64 regs_mask = evsel->core.attr.sample_regs_intr;
2498	u64 *pos;
2499	struct regs_dump *intr_regs = perf_sample__intr_regs(&sample);
2500
2501	intr_regs->abi = items->is_32_bit ?
2502	PERF_SAMPLE_REGS_ABI_32 :
2503	PERF_SAMPLE_REGS_ABI_64;
2504	intr_regs->regs = regs;
2505
2506	pos = intel_pt_add_gp_regs(intr_regs, regs, items, regs_mask);
2507
2508	intel_pt_add_xmm(intr_regs, pos, items, regs_mask);
2509	}
2510
2511	if ((sample_type \| evsel->synth_sample_type) & PERF_SAMPLE_BRANCH_STACK) {
2512	if (items->mask[INTEL_PT_LBR_0_POS] \|\|
2513	items->mask[INTEL_PT_LBR_1_POS] \|\|
2514	items->mask[INTEL_PT_LBR_2_POS]) {
2515	intel_pt_add_lbrs(ptq->last_branch, items);
2516	} else if (pt->synth_opts.last_branch) {
2517	thread_stack__br_sample(ptq->thread, ptq->cpu,
2518	ptq->last_branch,
2519	pt->br_stack_sz);
2520	} else {
2521	ptq->last_branch->nr = 0;
2522	}
2523	sample.branch_stack = ptq->last_branch;
2524	}
2525
2526	if (sample_type & PERF_SAMPLE_ADDR && items->has_mem_access_address)
2527	sample.addr = items->mem_access_address;
2528
2529	if (sample_type & PERF_SAMPLE_WEIGHT_TYPE) {
2530	/*
2531	* Refer kernel's setup_pebs_adaptive_sample_data() and
2532	* intel_hsw_weight().
2533	*/
2534	if (items->has_mem_access_latency) {
2535	u64 weight = items->mem_access_latency >> 32;
2536
2537	/*
2538	* Starts from SPR, the mem access latency field
2539	* contains both cache latency [47:32] and instruction
2540	* latency [15:0]. The cache latency is the same as the
2541	* mem access latency on previous platforms.
2542	*
2543	* In practice, no memory access could last than 4G
2544	* cycles. Use latency >> 32 to distinguish the
2545	* different format of the mem access latency field.
2546	*/
2547	if (weight > 0) {
2548	sample.weight = weight & 0xffff;
2549	sample.ins_lat = items->mem_access_latency & 0xffff;
2550	} else
2551	sample.weight = items->mem_access_latency;
2552	}
2553	if (!sample.weight && items->has_tsx_aux_info) {
2554	/* Cycles last block */
2555	sample.weight = (u32)items->tsx_aux_info;
2556	}
2557	}
2558
2559	if (sample_type & PERF_SAMPLE_DATA_SRC) {
2560	if (items->has_mem_aux_info && data_src_fmt) {
2561	if (data_src_fmt < 0) {
2562	pr_err("Intel PT missing data_src info\n");
2563	return -1;
2564	}
2565	sample.data_src = intel_pt_get_data_src(items->mem_aux_info, data_src_fmt);
2566	} else {
2567	sample.data_src = PERF_MEM_NA;
2568	}
2569	}
2570
2571	if (sample_type & PERF_SAMPLE_TRANSACTION && items->has_tsx_aux_info) {
2572	u64 ax = items->has_rax ? items->rax : 0;
2573	/* Refer kernel's intel_hsw_transaction() */
2574	u64 txn = (u8)(items->tsx_aux_info >> 32);
2575
2576	/* For RTM XABORTs also log the abort code from AX */
2577	if (txn & PERF_TXN_TRANSACTION && ax & 1)
2578	txn \|= ((ax >> 24) & 0xff) << PERF_TXN_ABORT_SHIFT;
2579	sample.transaction = txn;
2580	}
2581
2582	ret = intel_pt_deliver_synth_event(pt, event, &sample,
2583	sample_type \| evsel->synth_sample_type);
2584	perf_sample__exit(&sample);
2585	return ret;
2586	}
2587
2588	static int intel_pt_synth_single_pebs_sample(struct intel_pt_queue *ptq)
2589	{
2590	struct intel_pt *pt = ptq->pt;
2591	struct evsel *evsel = pt->pebs_evsel;
2592	int data_src_fmt = pt->pebs_data_src_fmt;
2593	u64 id = evsel->core.id[0];
2594
2595	return intel_pt_do_synth_pebs_sample(ptq, evsel, id, data_src_fmt);
2596	}
2597
2598	static int intel_pt_synth_pebs_sample(struct intel_pt_queue *ptq)
2599	{
2600	const struct intel_pt_blk_items *items = &ptq->state->items;
2601	struct intel_pt_pebs_event *pe;
2602	struct intel_pt *pt = ptq->pt;
2603	int err = -EINVAL;
2604	int hw_id;
2605
2606	if (!items->has_applicable_counters \|\| !items->applicable_counters) {
2607	if (!pt->single_pebs)
2608	pr_err("PEBS-via-PT record with no applicable_counters\n");
2609	return intel_pt_synth_single_pebs_sample(ptq);
2610	}
2611
2612	for_each_set_bit(hw_id, (unsigned long *)&items->applicable_counters, INTEL_PT_MAX_PEBS) {
2613	pe = &ptq->pebs[hw_id];
2614	if (!pe->evsel) {
2615	if (!pt->single_pebs)
2616	pr_err("PEBS-via-PT record with no matching event, hw_id %d\n",
2617	hw_id);
2618	return intel_pt_synth_single_pebs_sample(ptq);
2619	}
2620	err = intel_pt_do_synth_pebs_sample(ptq, pe->evsel, pe->id, pe->data_src_fmt);
2621	if (err)
2622	return err;
2623	}
2624
2625	return err;
2626	}
2627
2628	static int intel_pt_synth_events_sample(struct intel_pt_queue *ptq)
2629	{
2630	struct intel_pt *pt = ptq->pt;
2631	union perf_event *event = ptq->event_buf;
2632	struct perf_sample sample;
2633	struct {
2634	struct perf_synth_intel_evt cfe;
2635	struct perf_synth_intel_evd evd[INTEL_PT_MAX_EVDS];
2636	} raw;
2637	int i, ret;
2638
2639	if (intel_pt_skip_event(pt))
2640	return 0;
2641
2642	perf_sample__init(&sample, /all=/true);
2643	intel_pt_prep_p_sample(pt, ptq, event, &sample);
2644
2645	sample.id = ptq->pt->evt_id;
2646	sample.stream_id = ptq->pt->evt_id;
2647
2648	raw.cfe.type = ptq->state->cfe_type;
2649	raw.cfe.reserved = 0;
2650	raw.cfe.ip = !!(ptq->state->flags & INTEL_PT_FUP_IP);
2651	raw.cfe.vector = ptq->state->cfe_vector;
2652	raw.cfe.evd_cnt = ptq->state->evd_cnt;
2653
2654	for (i = 0; i < ptq->state->evd_cnt; i++) {
2655	raw.evd[i].et = 0;
2656	raw.evd[i].evd_type = ptq->state->evd[i].type;
2657	raw.evd[i].payload = ptq->state->evd[i].payload;
2658	}
2659
2660	sample.raw_size = perf_synth__raw_size(raw) +
2661	ptq->state->evd_cnt * sizeof(struct perf_synth_intel_evd);
2662	sample.raw_data = perf_synth__raw_data(&raw);
2663
2664	ret = intel_pt_deliver_synth_event(pt, event, &sample,
2665	pt->evt_sample_type);
2666	perf_sample__exit(&sample);
2667	return ret;
2668	}
2669
2670	static int intel_pt_synth_iflag_chg_sample(struct intel_pt_queue *ptq)
2671	{
2672	struct intel_pt *pt = ptq->pt;
2673	union perf_event *event = ptq->event_buf;
2674	struct perf_sample sample;
2675	struct perf_synth_intel_iflag_chg raw;
2676	int ret;
2677
2678	if (intel_pt_skip_event(pt))
2679	return 0;
2680
2681	perf_sample__init(&sample, /all=/true);
2682	intel_pt_prep_p_sample(pt, ptq, event, &sample);
2683
2684	sample.id = ptq->pt->iflag_chg_id;
2685	sample.stream_id = ptq->pt->iflag_chg_id;
2686
2687	raw.flags = 0;
2688	raw.iflag = ptq->state->to_iflag;
2689
2690	if (ptq->state->type & INTEL_PT_BRANCH) {
2691	raw.via_branch = 1;
2692	raw.branch_ip = ptq->state->to_ip;
2693	} else {
2694	sample.addr = 0;
2695	}
2696	sample.flags = ptq->flags;
2697
2698	sample.raw_size = perf_synth__raw_size(raw);
2699	sample.raw_data = perf_synth__raw_data(&raw);
2700
2701	ret = intel_pt_deliver_synth_event(pt, event, &sample,
2702	pt->iflag_chg_sample_type);
2703	perf_sample__exit(&sample);
2704	return ret;
2705	}
2706
2707	static int intel_pt_synth_error(struct intel_pt *pt, int code, int cpu,
2708	pid_t pid, pid_t tid, u64 ip, u64 timestamp,
2709	pid_t machine_pid, int vcpu)
2710	{
2711	bool dump_log_on_error = pt->synth_opts.log_plus_flags & AUXTRACE_LOG_FLG_ON_ERROR;
2712	bool log_on_stdout = pt->synth_opts.log_plus_flags & AUXTRACE_LOG_FLG_USE_STDOUT;
2713	union perf_event event;
2714	char msg[MAX_AUXTRACE_ERROR_MSG];
2715	int err;
2716
2717	if (pt->synth_opts.error_minus_flags) {
2718	if (code == INTEL_PT_ERR_OVR &&
2719	pt->synth_opts.error_minus_flags & AUXTRACE_ERR_FLG_OVERFLOW)
2720	return 0;
2721	if (code == INTEL_PT_ERR_LOST &&
2722	pt->synth_opts.error_minus_flags & AUXTRACE_ERR_FLG_DATA_LOST)
2723	return 0;
2724	}
2725
2726	intel_pt__strerror(code, msg, MAX_AUXTRACE_ERROR_MSG);
2727
2728	auxtrace_synth_guest_error(&event.auxtrace_error, PERF_AUXTRACE_ERROR_ITRACE,
2729	code, cpu, pid, tid, ip, msg, timestamp,
2730	machine_pid, vcpu);
2731
2732	if (intel_pt_enable_logging && !log_on_stdout) {
2733	FILE *fp = intel_pt_log_fp();
2734
2735	if (fp)
2736	perf_event__fprintf_auxtrace_error(&event, fp);
2737	}
2738
2739	if (code != INTEL_PT_ERR_LOST && dump_log_on_error)
2740	intel_pt_log_dump_buf();
2741
2742	err = perf_session__deliver_synth_event(pt->session, &event, NULL);
2743	if (err)
2744	pr_err("Intel Processor Trace: failed to deliver error event, error %d\n",
2745	err);
2746
2747	return err;
2748	}
2749
2750	static int intel_ptq_synth_error(struct intel_pt_queue *ptq,
2751	const struct intel_pt_state *state)
2752	{
2753	struct intel_pt *pt = ptq->pt;
2754	u64 tm = ptq->timestamp;
2755	pid_t machine_pid = 0;
2756	pid_t pid = ptq->pid;
2757	pid_t tid = ptq->tid;
2758	int vcpu = -1;
2759
2760	tm = pt->timeless_decoding ? 0 : tsc_to_perf_time(tm, &pt->tc);
2761
2762	if (pt->have_guest_sideband && state->from_nr) {
2763	machine_pid = ptq->guest_machine_pid;
2764	vcpu = ptq->vcpu;
2765	pid = ptq->guest_pid;
2766	tid = ptq->guest_tid;
2767	}
2768
2769	return intel_pt_synth_error(pt, state->err, ptq->cpu, pid, tid,
2770	state->from_ip, tm, machine_pid, vcpu);
2771	}
2772
2773	static int intel_pt_next_tid(struct intel_pt pt, struct intel_pt_queue ptq)
2774	{
2775	struct auxtrace_queue *queue;
2776	pid_t tid = ptq->next_tid;
2777	int err;
2778
2779	if (tid == -1)
2780	return 0;
2781
2782	intel_pt_log("switch: cpu %d tid %d\n", ptq->cpu, tid);
2783
2784	err = machine__set_current_tid(pt->machine, ptq->cpu, -1, tid);
2785
2786	queue = &pt->queues.queue_array[ptq->queue_nr];
2787	intel_pt_set_pid_tid_cpu(pt, queue);
2788
2789	ptq->next_tid = -1;
2790
2791	return err;
2792	}
2793
2794	static inline bool intel_pt_is_switch_ip(struct intel_pt_queue *ptq, u64 ip)
2795	{
2796	struct intel_pt *pt = ptq->pt;
2797
2798	return ip == pt->switch_ip &&
2799	(ptq->flags & PERF_IP_FLAG_BRANCH) &&
2800	!(ptq->flags & (PERF_IP_FLAG_CONDITIONAL \| PERF_IP_FLAG_ASYNC \|
2801	PERF_IP_FLAG_INTERRUPT \| PERF_IP_FLAG_TX_ABORT));
2802	}
2803
2804	#define INTEL_PT_PWR_EVT (INTEL_PT_MWAIT_OP \| INTEL_PT_PWR_ENTRY \| \
2805	INTEL_PT_EX_STOP \| INTEL_PT_PWR_EXIT)
2806
2807	static int intel_pt_sample(struct intel_pt_queue *ptq)
2808	{
2809	const struct intel_pt_state *state = ptq->state;
2810	struct intel_pt *pt = ptq->pt;
2811	int err;
2812
2813	if (!ptq->have_sample)
2814	return 0;
2815
2816	ptq->have_sample = false;
2817
2818	if (pt->synth_opts.approx_ipc) {
2819	ptq->ipc_insn_cnt = ptq->state->tot_insn_cnt;
2820	ptq->ipc_cyc_cnt = ptq->state->cycles;
2821	ptq->sample_ipc = true;
2822	} else {
2823	ptq->ipc_insn_cnt = ptq->state->tot_insn_cnt;
2824	ptq->ipc_cyc_cnt = ptq->state->tot_cyc_cnt;
2825	ptq->sample_ipc = ptq->state->flags & INTEL_PT_SAMPLE_IPC;
2826	}
2827
2828	/* Ensure guest code maps are set up */
2829	if (symbol_conf.guest_code && (state->from_nr \|\| state->to_nr))
2830	intel_pt_get_guest(ptq);
2831
2832	/*
2833	* Do PEBS first to allow for the possibility that the PEBS timestamp
2834	* precedes the current timestamp.
2835	*/
2836	if (pt->sample_pebs && state->type & INTEL_PT_BLK_ITEMS) {
2837	err = intel_pt_synth_pebs_sample(ptq);
2838	if (err)
2839	return err;
2840	}
2841
2842	if (pt->synth_opts.intr_events) {
2843	if (state->type & INTEL_PT_EVT) {
2844	err = intel_pt_synth_events_sample(ptq);
2845	if (err)
2846	return err;
2847	}
2848	if (state->type & INTEL_PT_IFLAG_CHG) {
2849	err = intel_pt_synth_iflag_chg_sample(ptq);
2850	if (err)
2851	return err;
2852	}
2853	}
2854
2855	if (pt->sample_pwr_events) {
2856	if (state->type & INTEL_PT_PSB_EVT) {
2857	err = intel_pt_synth_psb_sample(ptq);
2858	if (err)
2859	return err;
2860	}
2861	if (ptq->state->cbr != ptq->cbr_seen) {
2862	err = intel_pt_synth_cbr_sample(ptq);
2863	if (err)
2864	return err;
2865	}
2866	if (state->type & INTEL_PT_PWR_EVT) {
2867	if (state->type & INTEL_PT_MWAIT_OP) {
2868	err = intel_pt_synth_mwait_sample(ptq);
2869	if (err)
2870	return err;
2871	}
2872	if (state->type & INTEL_PT_PWR_ENTRY) {
2873	err = intel_pt_synth_pwre_sample(ptq);
2874	if (err)
2875	return err;
2876	}
2877	if (state->type & INTEL_PT_EX_STOP) {
2878	err = intel_pt_synth_exstop_sample(ptq);
2879	if (err)
2880	return err;
2881	}
2882	if (state->type & INTEL_PT_PWR_EXIT) {
2883	err = intel_pt_synth_pwrx_sample(ptq);
2884	if (err)
2885	return err;
2886	}
2887	}
2888	}
2889
2890	if (state->type & INTEL_PT_INSTRUCTION) {
2891	if (pt->sample_instructions) {
2892	err = intel_pt_synth_instruction_sample(ptq);
2893	if (err)
2894	return err;
2895	}
2896	if (pt->sample_cycles) {
2897	err = intel_pt_synth_cycle_sample(ptq);
2898	if (err)
2899	return err;
2900	}
2901	}
2902
2903	if (pt->sample_transactions && (state->type & INTEL_PT_TRANSACTION)) {
2904	err = intel_pt_synth_transaction_sample(ptq);
2905	if (err)
2906	return err;
2907	}
2908
2909	if (pt->sample_ptwrites && (state->type & INTEL_PT_PTW)) {
2910	err = intel_pt_synth_ptwrite_sample(ptq);
2911	if (err)
2912	return err;
2913	}
2914
2915	if (!(state->type & INTEL_PT_BRANCH))
2916	return 0;
2917
2918	if (pt->use_thread_stack) {
2919	thread_stack__event(ptq->thread, ptq->cpu, ptq->flags,
2920	state->from_ip, state->to_ip, ptq->insn_len,
2921	state->trace_nr, pt->callstack,
2922	pt->br_stack_sz_plus,
2923	pt->mispred_all);
2924	} else {
2925	thread_stack__set_trace_nr(ptq->thread, ptq->cpu, state->trace_nr);
2926	}
2927
2928	if (pt->sample_branches) {
2929	if (state->from_nr != state->to_nr &&
2930	state->from_ip && state->to_ip) {
2931	struct intel_pt_state st = (struct intel_pt_state )state;
2932	u64 to_ip = st->to_ip;
2933	u64 from_ip = st->from_ip;
2934
2935	/*
2936	* perf cannot handle having different machines for ip
2937	* and addr, so create 2 branches.
2938	*/
2939	st->to_ip = 0;
2940	err = intel_pt_synth_branch_sample(ptq);
2941	if (err)
2942	return err;
2943	st->from_ip = 0;
2944	st->to_ip = to_ip;
2945	err = intel_pt_synth_branch_sample(ptq);
2946	st->from_ip = from_ip;
2947	} else {
2948	err = intel_pt_synth_branch_sample(ptq);
2949	}
2950	if (err)
2951	return err;
2952	}
2953
2954	if (!ptq->sync_switch)
2955	return 0;
2956
2957	if (intel_pt_is_switch_ip(ptq, state->to_ip)) {
2958	switch (ptq->switch_state) {
2959	case INTEL_PT_SS_NOT_TRACING:
2960	case INTEL_PT_SS_UNKNOWN:
2961	case INTEL_PT_SS_EXPECTING_SWITCH_IP:
2962	err = intel_pt_next_tid(pt, ptq);
2963	if (err)
2964	return err;
2965	ptq->switch_state = INTEL_PT_SS_TRACING;
2966	break;
2967	default:
2968	ptq->switch_state = INTEL_PT_SS_EXPECTING_SWITCH_EVENT;
2969	return 1;
2970	}
2971	} else if (!state->to_ip) {
2972	ptq->switch_state = INTEL_PT_SS_NOT_TRACING;
2973	} else if (ptq->switch_state == INTEL_PT_SS_NOT_TRACING) {
2974	ptq->switch_state = INTEL_PT_SS_UNKNOWN;
2975	} else if (ptq->switch_state == INTEL_PT_SS_UNKNOWN &&
2976	state->to_ip == pt->ptss_ip &&
2977	(ptq->flags & PERF_IP_FLAG_CALL)) {
2978	ptq->switch_state = INTEL_PT_SS_TRACING;
2979	}
2980
2981	return 0;
2982	}
2983
2984	static u64 intel_pt_switch_ip(struct intel_pt pt, u64 ptss_ip)
2985	{
2986	struct machine *machine = pt->machine;
2987	struct map *map;
2988	struct symbol sym, start;
2989	u64 ip, switch_ip = 0;
2990	const char *ptss;
2991
2992	if (ptss_ip)
2993	*ptss_ip = 0;
2994
2995	map = machine__kernel_map(machine);
2996	if (!map)
2997	return 0;
2998
2999	if (map__load(map))
3000	return 0;
3001
3002	start = dso__first_symbol(map__dso(map));
3003
3004	for (sym = start; sym; sym = dso__next_symbol(sym)) {
3005	if (symbol__binding(sym) == STB_GLOBAL &&
3006	!strcmp(sym->name, "__switch_to")) {
3007	ip = map__unmap_ip(map, sym->start);
3008	if (ip >= map__start(map) && ip < map__end(map)) {
3009	switch_ip = ip;
3010	break;
3011	}
3012	}
3013	}
3014
3015	if (!switch_ip \|\| !ptss_ip)
3016	return 0;
3017
3018	if (pt->have_sched_switch == 1)
3019	ptss = "perf_trace_sched_switch";
3020	else
3021	ptss = "__perf_event_task_sched_out";
3022
3023	for (sym = start; sym; sym = dso__next_symbol(sym)) {
3024	if (!strcmp(sym->name, ptss)) {
3025	ip = map__unmap_ip(map, sym->start);
3026	if (ip >= map__start(map) && ip < map__end(map)) {
3027	*ptss_ip = ip;
3028	break;
3029	}
3030	}
3031	}
3032
3033	return switch_ip;
3034	}
3035
3036	static void intel_pt_enable_sync_switch(struct intel_pt *pt)
3037	{
3038	unsigned int i;
3039
3040	if (pt->sync_switch_not_supported)
3041	return;
3042
3043	pt->sync_switch = true;
3044
3045	for (i = 0; i < pt->queues.nr_queues; i++) {
3046	struct auxtrace_queue *queue = &pt->queues.queue_array[i];
3047	struct intel_pt_queue *ptq = queue->priv;
3048
3049	if (ptq)
3050	ptq->sync_switch = true;
3051	}
3052	}
3053
3054	static void intel_pt_disable_sync_switch(struct intel_pt *pt)
3055	{
3056	unsigned int i;
3057
3058	pt->sync_switch = false;
3059
3060	for (i = 0; i < pt->queues.nr_queues; i++) {
3061	struct auxtrace_queue *queue = &pt->queues.queue_array[i];
3062	struct intel_pt_queue *ptq = queue->priv;
3063
3064	if (ptq) {
3065	ptq->sync_switch = false;
3066	intel_pt_next_tid(pt, ptq);
3067	}
3068	}
3069	}
3070
3071	/*
3072	* To filter against time ranges, it is only necessary to look at the next start
3073	* or end time.
3074	*/
3075	static bool intel_pt_next_time(struct intel_pt_queue *ptq)
3076	{
3077	struct intel_pt *pt = ptq->pt;
3078
3079	if (ptq->sel_start) {
3080	/* Next time is an end time */
3081	ptq->sel_start = false;
3082	ptq->sel_timestamp = pt->time_ranges[ptq->sel_idx].end;
3083	return true;
3084	} else if (ptq->sel_idx + 1 < pt->range_cnt) {
3085	/* Next time is a start time */
3086	ptq->sel_start = true;
3087	ptq->sel_idx += 1;
3088	ptq->sel_timestamp = pt->time_ranges[ptq->sel_idx].start;
3089	return true;
3090	}
3091
3092	/* No next time */
3093	return false;
3094	}
3095
3096	static int intel_pt_time_filter(struct intel_pt_queue ptq, u64 ff_timestamp)
3097	{
3098	int err;
3099
3100	while (1) {
3101	if (ptq->sel_start) {
3102	if (ptq->timestamp >= ptq->sel_timestamp) {
3103	/* After start time, so consider next time */
3104	intel_pt_next_time(ptq);
3105	if (!ptq->sel_timestamp) {
3106	/* No end time */
3107	return 0;
3108	}
3109	/* Check against end time */
3110	continue;
3111	}
3112	/* Before start time, so fast forward */
3113	ptq->have_sample = false;
3114	if (ptq->sel_timestamp > *ff_timestamp) {
3115	if (ptq->sync_switch) {
3116	intel_pt_next_tid(ptq->pt, ptq);
3117	ptq->switch_state = INTEL_PT_SS_UNKNOWN;
3118	}
3119	*ff_timestamp = ptq->sel_timestamp;
3120	err = intel_pt_fast_forward(ptq->decoder,
3121	ptq->sel_timestamp);
3122	if (err)
3123	return err;
3124	}
3125	return 0;
3126	} else if (ptq->timestamp > ptq->sel_timestamp) {
3127	/* After end time, so consider next time */
3128	if (!intel_pt_next_time(ptq)) {
3129	/* No next time range, so stop decoding */
3130	ptq->have_sample = false;
3131	ptq->switch_state = INTEL_PT_SS_NOT_TRACING;
3132	return 1;
3133	}
3134	/* Check against next start time */
3135	continue;
3136	} else {
3137	/* Before end time */
3138	return 0;
3139	}
3140	}
3141	}
3142
3143	static int intel_pt_run_decoder(struct intel_pt_queue ptq, u64 timestamp)
3144	{
3145	const struct intel_pt_state *state = ptq->state;
3146	struct intel_pt *pt = ptq->pt;
3147	u64 ff_timestamp = 0;
3148	int err;
3149
3150	if (!pt->kernel_start) {
3151	pt->kernel_start = machine__kernel_start(pt->machine);
3152	if (pt->per_cpu_mmaps &&
3153	(pt->have_sched_switch == 1 \|\| pt->have_sched_switch == 3) &&
3154	!pt->timeless_decoding && intel_pt_tracing_kernel(pt) &&
3155	!pt->sampling_mode && !pt->synth_opts.vm_time_correlation) {
3156	pt->switch_ip = intel_pt_switch_ip(pt, &pt->ptss_ip);
3157	if (pt->switch_ip) {
3158	intel_pt_log("switch_ip: %"PRIx64" ptss_ip: %"PRIx64"\n",
3159	pt->switch_ip, pt->ptss_ip);
3160	intel_pt_enable_sync_switch(pt);
3161	}
3162	}
3163	}
3164
3165	intel_pt_log("queue %u decoding cpu %d pid %d tid %d\n",
3166	ptq->queue_nr, ptq->cpu, ptq->pid, ptq->tid);
3167	while (1) {
3168	err = intel_pt_sample(ptq);
3169	if (err)
3170	return err;
3171
3172	state = intel_pt_decode(ptq->decoder);
3173	if (state->err) {
3174	if (state->err == INTEL_PT_ERR_NODATA)
3175	return 1;
3176	if (ptq->sync_switch &&
3177	state->from_ip >= pt->kernel_start) {
3178	ptq->sync_switch = false;
3179	intel_pt_next_tid(pt, ptq);
3180	}
3181	ptq->timestamp = state->est_timestamp;
3182	if (pt->synth_opts.errors) {
3183	err = intel_ptq_synth_error(ptq, state);
3184	if (err)
3185	return err;
3186	}
3187	continue;
3188	}
3189
3190	ptq->state = state;
3191	ptq->have_sample = true;
3192	intel_pt_sample_flags(ptq);
3193
3194	/* Use estimated TSC upon return to user space */
3195	if (pt->est_tsc &&
3196	(state->from_ip >= pt->kernel_start \|\| !state->from_ip) &&
3197	state->to_ip && state->to_ip < pt->kernel_start) {
3198	intel_pt_log("TSC %"PRIx64" est. TSC %"PRIx64"\n",
3199	state->timestamp, state->est_timestamp);
3200	ptq->timestamp = state->est_timestamp;
3201	/* Use estimated TSC in unknown switch state */
3202	} else if (ptq->sync_switch &&
3203	ptq->switch_state == INTEL_PT_SS_UNKNOWN &&
3204	intel_pt_is_switch_ip(ptq, state->to_ip) &&
3205	ptq->next_tid == -1) {
3206	intel_pt_log("TSC %"PRIx64" est. TSC %"PRIx64"\n",
3207	state->timestamp, state->est_timestamp);
3208	ptq->timestamp = state->est_timestamp;
3209	} else if (state->timestamp > ptq->timestamp) {
3210	ptq->timestamp = state->timestamp;
3211	}
3212
3213	if (ptq->sel_timestamp) {
3214	err = intel_pt_time_filter(ptq, &ff_timestamp);
3215	if (err)
3216	return err;
3217	}
3218
3219	if (!pt->timeless_decoding && ptq->timestamp >= *timestamp) {
3220	*timestamp = ptq->timestamp;
3221	return 0;
3222	}
3223	}
3224	return 0;
3225	}
3226
3227	static inline int intel_pt_update_queues(struct intel_pt *pt)
3228	{
3229	if (pt->queues.new_data) {
3230	pt->queues.new_data = false;
3231	return intel_pt_setup_queues(pt);
3232	}
3233	return 0;
3234	}
3235
3236	static int intel_pt_process_queues(struct intel_pt *pt, u64 timestamp)
3237	{
3238	unsigned int queue_nr;
3239	u64 ts;
3240	int ret;
3241
3242	while (1) {
3243	struct auxtrace_queue *queue;
3244	struct intel_pt_queue *ptq;
3245
3246	if (!pt->heap.heap_cnt)
3247	return 0;
3248
3249	if (pt->heap.heap_array[0].ordinal >= timestamp)
3250	return 0;
3251
3252	queue_nr = pt->heap.heap_array[0].queue_nr;
3253	queue = &pt->queues.queue_array[queue_nr];
3254	ptq = queue->priv;
3255
3256	intel_pt_log("queue %u processing 0x%" PRIx64 " to 0x%" PRIx64 "\n",
3257	queue_nr, pt->heap.heap_array[0].ordinal,
3258	timestamp);
3259
3260	auxtrace_heap__pop(&pt->heap);
3261
3262	if (pt->heap.heap_cnt) {
3263	ts = pt->heap.heap_array[0].ordinal + 1;
3264	if (ts > timestamp)
3265	ts = timestamp;
3266	} else {
3267	ts = timestamp;
3268	}
3269
3270	intel_pt_set_pid_tid_cpu(pt, queue);
3271
3272	ret = intel_pt_run_decoder(ptq, &ts);
3273
3274	if (ret < 0) {
3275	auxtrace_heap__add(&pt->heap, queue_nr, ts);
3276	return ret;
3277	}
3278
3279	if (!ret) {
3280	ret = auxtrace_heap__add(&pt->heap, queue_nr, ts);
3281	if (ret < 0)
3282	return ret;
3283	} else {
3284	ptq->on_heap = false;
3285	}
3286	}
3287
3288	return 0;
3289	}
3290
3291	static int intel_pt_process_timeless_queues(struct intel_pt *pt, pid_t tid,
3292	u64 time_)
3293	{
3294	struct auxtrace_queues *queues = &pt->queues;
3295	unsigned int i;
3296	u64 ts = 0;
3297
3298	for (i = 0; i < queues->nr_queues; i++) {
3299	struct auxtrace_queue *queue = &pt->queues.queue_array[i];
3300	struct intel_pt_queue *ptq = queue->priv;
3301
3302	if (ptq && (tid == -1 \|\| ptq->tid == tid)) {
3303	ptq->time = time_;
3304	intel_pt_set_pid_tid_cpu(pt, queue);
3305	intel_pt_run_decoder(ptq, &ts);
3306	}
3307	}
3308	return 0;
3309	}
3310
3311	static void intel_pt_sample_set_pid_tid_cpu(struct intel_pt_queue *ptq,
3312	struct auxtrace_queue *queue,
3313	struct perf_sample *sample)
3314	{
3315	struct machine *m = ptq->pt->machine;
3316
3317	ptq->pid = sample->pid;
3318	ptq->tid = sample->tid;
3319	ptq->cpu = queue->cpu;
3320
3321	intel_pt_log("queue %u cpu %d pid %d tid %d\n",
3322	ptq->queue_nr, ptq->cpu, ptq->pid, ptq->tid);
3323
3324	thread__zput(ptq->thread);
3325
3326	if (ptq->tid == -1)
3327	return;
3328
3329	if (ptq->pid == -1) {
3330	ptq->thread = machine__find_thread(m, -1, ptq->tid);
3331	if (ptq->thread)
3332	ptq->pid = thread__pid(ptq->thread);
3333	return;
3334	}
3335
3336	ptq->thread = machine__findnew_thread(m, ptq->pid, ptq->tid);
3337	}
3338
3339	static int intel_pt_process_timeless_sample(struct intel_pt *pt,
3340	struct perf_sample *sample)
3341	{
3342	struct auxtrace_queue *queue;
3343	struct intel_pt_queue *ptq;
3344	u64 ts = 0;
3345
3346	queue = auxtrace_queues__sample_queue(&pt->queues, sample, pt->session);
3347	if (!queue)
3348	return -EINVAL;
3349
3350	ptq = queue->priv;
3351	if (!ptq)
3352	return 0;
3353
3354	ptq->stop = false;
3355	ptq->time = sample->time;
3356	intel_pt_sample_set_pid_tid_cpu(ptq, queue, sample);
3357	intel_pt_run_decoder(ptq, &ts);
3358	return 0;
3359	}
3360
3361	static int intel_pt_lost(struct intel_pt pt, struct perf_sample sample)
3362	{
3363	return intel_pt_synth_error(pt, INTEL_PT_ERR_LOST, sample->cpu,
3364	sample->pid, sample->tid, 0, sample->time,
3365	sample->machine_pid, sample->vcpu);
3366	}
3367
3368	static struct intel_pt_queue intel_pt_cpu_to_ptq(struct intel_pt pt, int cpu)
3369	{
3370	unsigned i, j;
3371
3372	if (cpu < 0 \|\| !pt->queues.nr_queues)
3373	return NULL;
3374
3375	if ((unsigned)cpu >= pt->queues.nr_queues)
3376	i = pt->queues.nr_queues - 1;
3377	else
3378	i = cpu;
3379
3380	if (pt->queues.queue_array[i].cpu == cpu)
3381	return pt->queues.queue_array[i].priv;
3382
3383	for (j = 0; i > 0; j++) {
3384	if (pt->queues.queue_array[--i].cpu == cpu)
3385	return pt->queues.queue_array[i].priv;
3386	}
3387
3388	for (; j < pt->queues.nr_queues; j++) {
3389	if (pt->queues.queue_array[j].cpu == cpu)
3390	return pt->queues.queue_array[j].priv;
3391	}
3392
3393	return NULL;
3394	}
3395
3396	static int intel_pt_sync_switch(struct intel_pt *pt, int cpu, pid_t tid,
3397	u64 timestamp)
3398	{
3399	struct intel_pt_queue *ptq;
3400	int err;
3401
3402	if (!pt->sync_switch)
3403	return 1;
3404
3405	ptq = intel_pt_cpu_to_ptq(pt, cpu);
3406	if (!ptq \|\| !ptq->sync_switch)
3407	return 1;
3408
3409	switch (ptq->switch_state) {
3410	case INTEL_PT_SS_NOT_TRACING:
3411	break;
3412	case INTEL_PT_SS_UNKNOWN:
3413	case INTEL_PT_SS_TRACING:
3414	ptq->next_tid = tid;
3415	ptq->switch_state = INTEL_PT_SS_EXPECTING_SWITCH_IP;
3416	return 0;
3417	case INTEL_PT_SS_EXPECTING_SWITCH_EVENT:
3418	if (!ptq->on_heap) {
3419	ptq->timestamp = perf_time_to_tsc(timestamp,
3420	&pt->tc);
3421	err = auxtrace_heap__add(&pt->heap, ptq->queue_nr,
3422	ptq->timestamp);
3423	if (err)
3424	return err;
3425	ptq->on_heap = true;
3426	}
3427	ptq->switch_state = INTEL_PT_SS_TRACING;
3428	break;
3429	case INTEL_PT_SS_EXPECTING_SWITCH_IP:
3430	intel_pt_log("ERROR: cpu %d expecting switch ip\n", cpu);
3431	break;
3432	default:
3433	break;
3434	}
3435
3436	ptq->next_tid = -1;
3437
3438	return 1;
3439	}
3440
3441	#ifdef HAVE_LIBTRACEEVENT
3442	static int intel_pt_process_switch(struct intel_pt *pt,
3443	struct perf_sample *sample)
3444	{
3445	pid_t tid;
3446	int cpu, ret;
3447	struct evsel *evsel = evlist__id2evsel(pt->session->evlist, sample->id);
3448
3449	if (evsel != pt->switch_evsel)
3450	return 0;
3451
3452	tid = perf_sample__intval(sample, "next_pid");
3453	cpu = sample->cpu;
3454
3455	intel_pt_log("sched_switch: cpu %d tid %d time %"PRIu64" tsc %#"PRIx64"\n",
3456	cpu, tid, sample->time, perf_time_to_tsc(sample->time,
3457	&pt->tc));
3458
3459	ret = intel_pt_sync_switch(pt, cpu, tid, sample->time);
3460	if (ret <= 0)
3461	return ret;
3462
3463	return machine__set_current_tid(pt->machine, cpu, -1, tid);
3464	}
3465	#endif /* HAVE_LIBTRACEEVENT */
3466
3467	static int intel_pt_context_switch_in(struct intel_pt *pt,
3468	struct perf_sample *sample)
3469	{
3470	pid_t pid = sample->pid;
3471	pid_t tid = sample->tid;
3472	int cpu = sample->cpu;
3473
3474	if (pt->sync_switch) {
3475	struct intel_pt_queue *ptq;
3476
3477	ptq = intel_pt_cpu_to_ptq(pt, cpu);
3478	if (ptq && ptq->sync_switch) {
3479	ptq->next_tid = -1;
3480	switch (ptq->switch_state) {
3481	case INTEL_PT_SS_NOT_TRACING:
3482	case INTEL_PT_SS_UNKNOWN:
3483	case INTEL_PT_SS_TRACING:
3484	break;
3485	case INTEL_PT_SS_EXPECTING_SWITCH_EVENT:
3486	case INTEL_PT_SS_EXPECTING_SWITCH_IP:
3487	ptq->switch_state = INTEL_PT_SS_TRACING;
3488	break;
3489	default:
3490	break;
3491	}
3492	}
3493	}
3494
3495	/*
3496	* If the current tid has not been updated yet, ensure it is now that
3497	* a "switch in" event has occurred.
3498	*/
3499	if (machine__get_current_tid(pt->machine, cpu) == tid)
3500	return 0;
3501
3502	return machine__set_current_tid(pt->machine, cpu, pid, tid);
3503	}
3504
3505	static int intel_pt_guest_context_switch(struct intel_pt *pt,
3506	union perf_event *event,
3507	struct perf_sample *sample)
3508	{
3509	bool out = event->header.misc & PERF_RECORD_MISC_SWITCH_OUT;
3510	struct machines *machines = &pt->session->machines;
3511	struct machine *machine = machines__find(machines, sample->machine_pid);
3512
3513	pt->have_guest_sideband = true;
3514
3515	/*
3516	* sync_switch cannot handle guest machines at present, so just disable
3517	* it.
3518	*/
3519	pt->sync_switch_not_supported = true;
3520	if (pt->sync_switch)
3521	intel_pt_disable_sync_switch(pt);
3522
3523	if (out)
3524	return 0;
3525
3526	if (!machine)
3527	return -EINVAL;
3528
3529	return machine__set_current_tid(machine, sample->vcpu, sample->pid, sample->tid);
3530	}
3531
3532	static int intel_pt_context_switch(struct intel_pt pt, union perf_event event,
3533	struct perf_sample *sample)
3534	{
3535	bool out = event->header.misc & PERF_RECORD_MISC_SWITCH_OUT;
3536	pid_t pid, tid;
3537	int cpu, ret;
3538
3539	if (perf_event__is_guest(event))
3540	return intel_pt_guest_context_switch(pt, event, sample);
3541
3542	cpu = sample->cpu;
3543
3544	if (pt->have_sched_switch == 3) {
3545	if (!out)
3546	return intel_pt_context_switch_in(pt, sample);
3547	if (event->header.type != PERF_RECORD_SWITCH_CPU_WIDE) {
3548	pr_err("Expecting CPU-wide context switch event\n");
3549	return -EINVAL;
3550	}
3551	pid = event->context_switch.next_prev_pid;
3552	tid = event->context_switch.next_prev_tid;
3553	} else {
3554	if (out)
3555	return 0;
3556	pid = sample->pid;
3557	tid = sample->tid;
3558	}
3559
3560	if (tid == -1)
3561	intel_pt_log("context_switch event has no tid\n");
3562
3563	ret = intel_pt_sync_switch(pt, cpu, tid, sample->time);
3564	if (ret <= 0)
3565	return ret;
3566
3567	return machine__set_current_tid(pt->machine, cpu, pid, tid);
3568	}
3569
3570	static int intel_pt_process_itrace_start(struct intel_pt *pt,
3571	union perf_event *event,
3572	struct perf_sample *sample)
3573	{
3574	if (!pt->per_cpu_mmaps)
3575	return 0;
3576
3577	intel_pt_log("itrace_start: cpu %d pid %d tid %d time %"PRIu64" tsc %#"PRIx64"\n",
3578	sample->cpu, event->itrace_start.pid,
3579	event->itrace_start.tid, sample->time,
3580	perf_time_to_tsc(sample->time, &pt->tc));
3581
3582	return machine__set_current_tid(pt->machine, sample->cpu,
3583	event->itrace_start.pid,
3584	event->itrace_start.tid);
3585	}
3586
3587	/*
3588	* Events with data_src are identified by L1_Hit_Indication
3589	* refer https://github.com/intel/perfmon
3590	*/
3591	static int intel_pt_data_src_fmt(struct intel_pt pt, struct evsel evsel)
3592	{
3593	struct perf_env *env = pt->machine->env;
3594	int fmt = DATA_SRC_FORMAT_NA;
3595
3596	if (!env->cpuid)
3597	return DATA_SRC_FORMAT_ERR;
3598
3599	/*
3600	* PEBS-via-PT is only supported on E-core non-hybrid. Of those only
3601	* Gracemont and Crestmont have data_src. Check for:
3602	* Alderlake N (Gracemont)
3603	* Sierra Forest (Crestmont)
3604	* Grand Ridge (Crestmont)
3605	*/
3606
3607	if (!strncmp(env->cpuid, "GenuineIntel,6,190,", 19))
3608	fmt = DATA_SRC_FORMAT_GRT;
3609
3610	if (!strncmp(env->cpuid, "GenuineIntel,6,175,", 19) \|\|
3611	!strncmp(env->cpuid, "GenuineIntel,6,182,", 19))
3612	fmt = DATA_SRC_FORMAT_CMT;
3613
3614	if (fmt == DATA_SRC_FORMAT_NA)
3615	return fmt;
3616
3617	/*
3618	* Only data_src events are:
3619	* mem-loads event=0xd0,umask=0x5
3620	* mem-stores event=0xd0,umask=0x6
3621	*/
3622	if (evsel->core.attr.type == PERF_TYPE_RAW &&
3623	((evsel->core.attr.config & 0xffff) == 0x5d0 \|\|
3624	(evsel->core.attr.config & 0xffff) == 0x6d0))
3625	return fmt;
3626
3627	return DATA_SRC_FORMAT_NA;
3628	}
3629
3630	static int intel_pt_process_aux_output_hw_id(struct intel_pt *pt,
3631	union perf_event *event,
3632	struct perf_sample *sample)
3633	{
3634	u64 hw_id = event->aux_output_hw_id.hw_id;
3635	struct auxtrace_queue *queue;
3636	struct intel_pt_queue *ptq;
3637	struct evsel *evsel;
3638
3639	queue = auxtrace_queues__sample_queue(&pt->queues, sample, pt->session);
3640	evsel = evlist__id2evsel_strict(pt->session->evlist, sample->id);
3641	if (!queue \|\| !queue->priv \|\| !evsel \|\| hw_id > INTEL_PT_MAX_PEBS) {
3642	pr_err("Bad AUX output hardware ID\n");
3643	return -EINVAL;
3644	}
3645
3646	ptq = queue->priv;
3647
3648	ptq->pebs[hw_id].evsel = evsel;
3649	ptq->pebs[hw_id].id = sample->id;
3650	ptq->pebs[hw_id].data_src_fmt = intel_pt_data_src_fmt(pt, evsel);
3651
3652	return 0;
3653	}
3654
3655	static int intel_pt_find_map(struct thread *thread, u8 cpumode, u64 addr,
3656	struct addr_location *al)
3657	{
3658	if (!al->map \|\| addr < map__start(al->map) \|\| addr >= map__end(al->map)) {
3659	if (!thread__find_map(thread, cpumode, addr, al))
3660	return -1;
3661	}
3662
3663	return 0;
3664	}
3665
3666	/* Invalidate all instruction cache entries that overlap the text poke */
3667	static int intel_pt_text_poke(struct intel_pt pt, union perf_event event)
3668	{
3669	u8 cpumode = event->header.misc & PERF_RECORD_MISC_CPUMODE_MASK;
3670	u64 addr = event->text_poke.addr + event->text_poke.new_len - 1;
3671	/* Assume text poke begins in a basic block no more than 4096 bytes */
3672	int cnt = 4096 + event->text_poke.new_len;
3673	struct thread *thread = pt->unknown_thread;
3674	struct addr_location al;
3675	struct machine *machine = pt->machine;
3676	struct intel_pt_cache_entry *e;
3677	u64 offset;
3678	int ret = 0;
3679
3680	addr_location__init(&al);
3681	if (!event->text_poke.new_len)
3682	goto out;
3683
3684	for (; cnt; cnt--, addr--) {
3685	struct dso *dso;
3686
3687	if (intel_pt_find_map(thread, cpumode, addr, &al)) {
3688	if (addr < event->text_poke.addr)
3689	goto out;
3690	continue;
3691	}
3692
3693	dso = map__dso(al.map);
3694	if (!dso \|\| !dso__auxtrace_cache(dso))
3695	continue;
3696
3697	offset = map__map_ip(al.map, addr);
3698
3699	e = intel_pt_cache_lookup(dso, machine, offset);
3700	if (!e)
3701	continue;
3702
3703	if (addr + e->byte_cnt + e->length <= event->text_poke.addr) {
3704	/*
3705	* No overlap. Working backwards there cannot be another
3706	* basic block that overlaps the text poke if there is a
3707	* branch instruction before the text poke address.
3708	*/
3709	if (e->branch != INTEL_PT_BR_NO_BRANCH)
3710	goto out;
3711	} else {
3712	intel_pt_cache_invalidate(dso, machine, offset);
3713	intel_pt_log("Invalidated instruction cache for %s at %#"PRIx64"\n",
3714	dso__long_name(dso), addr);
3715	}
3716	}
3717	out:
3718	addr_location__exit(&al);
3719	return ret;
3720	}
3721
3722	static int intel_pt_process_event(struct perf_session *session,
3723	union perf_event *event,
3724	struct perf_sample *sample,
3725	const struct perf_tool *tool)
3726	{
3727	struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
3728	auxtrace);
3729	u64 timestamp;
3730	int err = 0;
3731
3732	if (dump_trace)
3733	return 0;
3734
3735	if (!tool->ordered_events) {
3736	pr_err("Intel Processor Trace requires ordered events\n");
3737	return -EINVAL;
3738	}
3739
3740	if (sample->time && sample->time != (u64)-1)
3741	timestamp = perf_time_to_tsc(sample->time, &pt->tc);
3742	else
3743	timestamp = 0;
3744
3745	if (timestamp \|\| pt->timeless_decoding) {
3746	err = intel_pt_update_queues(pt);
3747	if (err)
3748	return err;
3749	}
3750
3751	if (pt->timeless_decoding) {
3752	if (pt->sampling_mode) {
3753	if (sample->aux_sample.size)
3754	err = intel_pt_process_timeless_sample(pt,
3755	sample);
3756	} else if (event->header.type == PERF_RECORD_EXIT) {
3757	err = intel_pt_process_timeless_queues(pt,
3758	event->fork.tid,
3759	sample->time);
3760	}
3761	} else if (timestamp) {
3762	if (!pt->first_timestamp)
3763	intel_pt_first_timestamp(pt, timestamp);
3764	err = intel_pt_process_queues(pt, timestamp);
3765	}
3766	if (err)
3767	return err;
3768
3769	if (event->header.type == PERF_RECORD_SAMPLE) {
3770	if (pt->synth_opts.add_callchain && !sample->callchain)
3771	intel_pt_add_callchain(pt, sample);
3772	if (pt->synth_opts.add_last_branch && !sample->branch_stack)
3773	intel_pt_add_br_stack(pt, sample);
3774	}
3775
3776	if (event->header.type == PERF_RECORD_AUX &&
3777	(event->aux.flags & PERF_AUX_FLAG_TRUNCATED) &&
3778	pt->synth_opts.errors) {
3779	err = intel_pt_lost(pt, sample);
3780	if (err)
3781	return err;
3782	}
3783
3784	#ifdef HAVE_LIBTRACEEVENT
3785	if (pt->switch_evsel && event->header.type == PERF_RECORD_SAMPLE)
3786	err = intel_pt_process_switch(pt, sample);
3787	else
3788	#endif
3789	if (event->header.type == PERF_RECORD_ITRACE_START)
3790	err = intel_pt_process_itrace_start(pt, event, sample);
3791	else if (event->header.type == PERF_RECORD_AUX_OUTPUT_HW_ID)
3792	err = intel_pt_process_aux_output_hw_id(pt, event, sample);
3793	else if (event->header.type == PERF_RECORD_SWITCH \|\|
3794	event->header.type == PERF_RECORD_SWITCH_CPU_WIDE)
3795	err = intel_pt_context_switch(pt, event, sample);
3796
3797	if (!err && event->header.type == PERF_RECORD_TEXT_POKE)
3798	err = intel_pt_text_poke(pt, event);
3799
3800	if (intel_pt_enable_logging && intel_pt_log_events(pt, sample->time)) {
3801	intel_pt_log("event %u: cpu %d time %"PRIu64" tsc %#"PRIx64" ",
3802	event->header.type, sample->cpu, sample->time, timestamp);
3803	intel_pt_log_event(event);
3804	}
3805
3806	return err;
3807	}
3808
3809	static int intel_pt_flush(struct perf_session session, const struct perf_tool tool)
3810	{
3811	struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
3812	auxtrace);
3813	int ret;
3814
3815	if (dump_trace)
3816	return 0;
3817
3818	if (!tool->ordered_events)
3819	return -EINVAL;
3820
3821	ret = intel_pt_update_queues(pt);
3822	if (ret < 0)
3823	return ret;
3824
3825	if (pt->timeless_decoding)
3826	return intel_pt_process_timeless_queues(pt, -1,
3827	MAX_TIMESTAMP - 1);
3828
3829	return intel_pt_process_queues(pt, MAX_TIMESTAMP);
3830	}
3831
3832	static void intel_pt_free_events(struct perf_session *session)
3833	{
3834	struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
3835	auxtrace);
3836	struct auxtrace_queues *queues = &pt->queues;
3837	unsigned int i;
3838
3839	for (i = 0; i < queues->nr_queues; i++) {
3840	intel_pt_free_queue(queues->queue_array[i].priv);
3841	queues->queue_array[i].priv = NULL;
3842	}
3843	intel_pt_log_disable();
3844	auxtrace_queues__free(queues);
3845	}
3846
3847	static void intel_pt_free(struct perf_session *session)
3848	{
3849	struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
3850	auxtrace);
3851
3852	auxtrace_heap__free(&pt->heap);
3853	intel_pt_free_events(session);
3854	session->auxtrace = NULL;
3855	intel_pt_free_vmcs_info(pt);
3856	thread__put(pt->unknown_thread);
3857	addr_filters__exit(&pt->filts);
3858	zfree(&pt->chain);
3859	zfree(&pt->filter);
3860	zfree(&pt->time_ranges);
3861	zfree(&pt->br_stack);
3862	free(pt);
3863	}
3864
3865	static bool intel_pt_evsel_is_auxtrace(struct perf_session *session,
3866	struct evsel *evsel)
3867	{
3868	struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
3869	auxtrace);
3870
3871	return evsel->core.attr.type == pt->pmu_type;
3872	}
3873
3874	static int intel_pt_process_auxtrace_event(struct perf_session *session,
3875	union perf_event *event,
3876	const struct perf_tool *tool __maybe_unused)
3877	{
3878	struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
3879	auxtrace);
3880
3881	if (!pt->data_queued) {
3882	struct auxtrace_buffer *buffer;
3883	off_t data_offset;
3884	int fd = perf_data__fd(session->data);
3885	int err;
3886
3887	if (perf_data__is_pipe(session->data)) {
3888	data_offset = 0;
3889	} else {
3890	data_offset = lseek(fd, 0, SEEK_CUR);
3891	if (data_offset == -1)
3892	return -errno;
3893	}
3894
3895	err = auxtrace_queues__add_event(&pt->queues, session, event,
3896	data_offset, &buffer);
3897	if (err)
3898	return err;
3899
3900	/* Dump here now we have copied a piped trace out of the pipe */
3901	if (dump_trace) {
3902	if (auxtrace_buffer__get_data(buffer, fd)) {
3903	intel_pt_dump_event(pt, buffer->data,
3904	buffer->size);
3905	auxtrace_buffer__put_data(buffer);
3906	}
3907	}
3908	}
3909
3910	return 0;
3911	}
3912
3913	static int intel_pt_queue_data(struct perf_session *session,
3914	struct perf_sample *sample,
3915	union perf_event *event, u64 data_offset)
3916	{
3917	struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
3918	auxtrace);
3919	u64 timestamp;
3920
3921	if (event) {
3922	return auxtrace_queues__add_event(&pt->queues, session, event,
3923	data_offset, NULL);
3924	}
3925
3926	if (sample->time && sample->time != (u64)-1)
3927	timestamp = perf_time_to_tsc(sample->time, &pt->tc);
3928	else
3929	timestamp = 0;
3930
3931	return auxtrace_queues__add_sample(&pt->queues, session, sample,
3932	data_offset, timestamp);
3933	}
3934
3935	static int intel_pt_synth_event(struct perf_session session, const char name,
3936	struct perf_event_attr *attr, u64 id)
3937	{
3938	int err;
3939
3940	pr_debug("Synthesizing '%s' event with id %" PRIu64 " sample type %#" PRIx64 "\n",
3941	name, id, (u64)attr->sample_type);
3942
3943	err = perf_session__deliver_synth_attr_event(session, attr, id);
3944	if (err)
3945	pr_err("%s: failed to synthesize '%s' event type\n",
3946	__func__, name);
3947
3948	return err;
3949	}
3950
3951	static void intel_pt_set_event_name(struct evlist *evlist, u64 id,
3952	const char *name)
3953	{
3954	struct evsel *evsel;
3955
3956	evlist__for_each_entry(evlist, evsel) {
3957	if (evsel->core.id && evsel->core.id[0] == id) {
3958	if (evsel->name)
3959	zfree(&evsel->name);
3960	evsel->name = strdup(name);
3961	break;
3962	}
3963	}
3964	}
3965
3966	static struct evsel intel_pt_evsel(struct intel_pt pt,
3967	struct evlist *evlist)
3968	{
3969	struct evsel *evsel;
3970
3971	evlist__for_each_entry(evlist, evsel) {
3972	if (evsel->core.attr.type == pt->pmu_type && evsel->core.ids)
3973	return evsel;
3974	}
3975
3976	return NULL;
3977	}
3978
3979	static int intel_pt_synth_events(struct intel_pt *pt,
3980	struct perf_session *session)
3981	{
3982	struct evlist *evlist = session->evlist;
3983	struct evsel *evsel = intel_pt_evsel(pt, evlist);
3984	struct perf_event_attr attr;
3985	u64 id;
3986	int err;
3987
3988	if (!evsel) {
3989	pr_debug("There are no selected events with Intel Processor Trace data\n");
3990	return 0;
3991	}
3992
3993	memset(&attr, 0, sizeof(struct perf_event_attr));
3994	attr.size = sizeof(struct perf_event_attr);
3995	attr.type = PERF_TYPE_HARDWARE;
3996	attr.sample_type = evsel->core.attr.sample_type & PERF_SAMPLE_MASK;
3997	attr.sample_type \|= PERF_SAMPLE_IP \| PERF_SAMPLE_TID \|
3998	PERF_SAMPLE_PERIOD;
3999	if (pt->timeless_decoding)
4000	attr.sample_type &= ~(u64)PERF_SAMPLE_TIME;
4001	else
4002	attr.sample_type \|= PERF_SAMPLE_TIME;
4003	if (!pt->per_cpu_mmaps)
4004	attr.sample_type &= ~(u64)PERF_SAMPLE_CPU;
4005	attr.exclude_user = evsel->core.attr.exclude_user;
4006	attr.exclude_kernel = evsel->core.attr.exclude_kernel;
4007	attr.exclude_hv = evsel->core.attr.exclude_hv;
4008	attr.exclude_host = evsel->core.attr.exclude_host;
4009	attr.exclude_guest = evsel->core.attr.exclude_guest;
4010	attr.sample_id_all = evsel->core.attr.sample_id_all;
4011	attr.read_format = evsel->core.attr.read_format;
4012
4013	id = auxtrace_synth_id_range_start(evsel);
4014
4015	if (pt->synth_opts.branches) {
4016	attr.config = PERF_COUNT_HW_BRANCH_INSTRUCTIONS;
4017	attr.sample_period = 1;
4018	attr.sample_type \|= PERF_SAMPLE_ADDR;
4019	err = intel_pt_synth_event(session, "branches", &attr, id);
4020	if (err)
4021	return err;
4022	pt->sample_branches = true;
4023	pt->branches_sample_type = attr.sample_type;
4024	pt->branches_id = id;
4025	id += 1;
4026	attr.sample_type &= ~(u64)PERF_SAMPLE_ADDR;
4027	}
4028
4029	if (pt->synth_opts.callchain)
4030	attr.sample_type \|= PERF_SAMPLE_CALLCHAIN;
4031	if (pt->synth_opts.last_branch) {
4032	attr.sample_type \|= PERF_SAMPLE_BRANCH_STACK;
4033	/*
4034	* We don't use the hardware index, but the sample generation
4035	* code uses the new format branch_stack with this field,
4036	* so the event attributes must indicate that it's present.
4037	*/
4038	attr.branch_sample_type \|= PERF_SAMPLE_BRANCH_HW_INDEX;
4039	}
4040
4041	if (pt->synth_opts.instructions) {
4042	attr.config = PERF_COUNT_HW_INSTRUCTIONS;
4043	if (pt->synth_opts.period_type == PERF_ITRACE_PERIOD_NANOSECS)
4044	attr.sample_period =
4045	intel_pt_ns_to_ticks(pt, pt->synth_opts.period);
4046	else
4047	attr.sample_period = pt->synth_opts.period;
4048	err = intel_pt_synth_event(session, "instructions", &attr, id);
4049	if (err)
4050	return err;
4051	pt->sample_instructions = true;
4052	pt->instructions_sample_type = attr.sample_type;
4053	pt->instructions_id = id;
4054	id += 1;
4055	}
4056
4057	if (pt->synth_opts.cycles) {
4058	attr.config = PERF_COUNT_HW_CPU_CYCLES;
4059	if (pt->synth_opts.period_type == PERF_ITRACE_PERIOD_NANOSECS)
4060	attr.sample_period =
4061	intel_pt_ns_to_ticks(pt, pt->synth_opts.period);
4062	else
4063	attr.sample_period = pt->synth_opts.period;
4064	err = intel_pt_synth_event(session, "cycles", &attr, id);
4065	if (err)
4066	return err;
4067	pt->sample_cycles = true;
4068	pt->cycles_sample_type = attr.sample_type;
4069	pt->cycles_id = id;
4070	id += 1;
4071	}
4072
4073	attr.sample_type &= ~(u64)PERF_SAMPLE_PERIOD;
4074	attr.sample_period = 1;
4075
4076	if (pt->synth_opts.transactions) {
4077	attr.config = PERF_COUNT_HW_INSTRUCTIONS;
4078	err = intel_pt_synth_event(session, "transactions", &attr, id);
4079	if (err)
4080	return err;
4081	pt->sample_transactions = true;
4082	pt->transactions_sample_type = attr.sample_type;
4083	pt->transactions_id = id;
4084	intel_pt_set_event_name(evlist, id, "transactions");
4085	id += 1;
4086	}
4087
4088	attr.type = PERF_TYPE_SYNTH;
4089	attr.sample_type \|= PERF_SAMPLE_RAW;
4090
4091	if (pt->synth_opts.ptwrites) {
4092	attr.config = PERF_SYNTH_INTEL_PTWRITE;
4093	err = intel_pt_synth_event(session, "ptwrite", &attr, id);
4094	if (err)
4095	return err;
4096	pt->sample_ptwrites = true;
4097	pt->ptwrites_sample_type = attr.sample_type;
4098	pt->ptwrites_id = id;
4099	intel_pt_set_event_name(evlist, id, "ptwrite");
4100	id += 1;
4101	}
4102
4103	if (pt->synth_opts.pwr_events) {
4104	pt->sample_pwr_events = true;
4105	pt->pwr_events_sample_type = attr.sample_type;
4106
4107	attr.config = PERF_SYNTH_INTEL_CBR;
4108	err = intel_pt_synth_event(session, "cbr", &attr, id);
4109	if (err)
4110	return err;
4111	pt->cbr_id = id;
4112	intel_pt_set_event_name(evlist, id, "cbr");
4113	id += 1;
4114
4115	attr.config = PERF_SYNTH_INTEL_PSB;
4116	err = intel_pt_synth_event(session, "psb", &attr, id);
4117	if (err)
4118	return err;
4119	pt->psb_id = id;
4120	intel_pt_set_event_name(evlist, id, "psb");
4121	id += 1;
4122	}
4123
4124	if (pt->synth_opts.pwr_events && (evsel->core.attr.config & INTEL_PT_CFG_PWR_EVT_EN)) {
4125	attr.config = PERF_SYNTH_INTEL_MWAIT;
4126	err = intel_pt_synth_event(session, "mwait", &attr, id);
4127	if (err)
4128	return err;
4129	pt->mwait_id = id;
4130	intel_pt_set_event_name(evlist, id, "mwait");
4131	id += 1;
4132
4133	attr.config = PERF_SYNTH_INTEL_PWRE;
4134	err = intel_pt_synth_event(session, "pwre", &attr, id);
4135	if (err)
4136	return err;
4137	pt->pwre_id = id;
4138	intel_pt_set_event_name(evlist, id, "pwre");
4139	id += 1;
4140
4141	attr.config = PERF_SYNTH_INTEL_EXSTOP;
4142	err = intel_pt_synth_event(session, "exstop", &attr, id);
4143	if (err)
4144	return err;
4145	pt->exstop_id = id;
4146	intel_pt_set_event_name(evlist, id, "exstop");
4147	id += 1;
4148
4149	attr.config = PERF_SYNTH_INTEL_PWRX;
4150	err = intel_pt_synth_event(session, "pwrx", &attr, id);
4151	if (err)
4152	return err;
4153	pt->pwrx_id = id;
4154	intel_pt_set_event_name(evlist, id, "pwrx");
4155	id += 1;
4156	}
4157
4158	if (pt->synth_opts.intr_events && (evsel->core.attr.config & INTEL_PT_CFG_EVT_EN)) {
4159	attr.config = PERF_SYNTH_INTEL_EVT;
4160	err = intel_pt_synth_event(session, "evt", &attr, id);
4161	if (err)
4162	return err;
4163	pt->evt_sample_type = attr.sample_type;
4164	pt->evt_id = id;
4165	intel_pt_set_event_name(evlist, id, "evt");
4166	id += 1;
4167	}
4168
4169	if (pt->synth_opts.intr_events && pt->cap_event_trace) {
4170	attr.config = PERF_SYNTH_INTEL_IFLAG_CHG;
4171	err = intel_pt_synth_event(session, "iflag", &attr, id);
4172	if (err)
4173	return err;
4174	pt->iflag_chg_sample_type = attr.sample_type;
4175	pt->iflag_chg_id = id;
4176	intel_pt_set_event_name(evlist, id, "iflag");
4177	id += 1;
4178	}
4179
4180	return 0;
4181	}
4182
4183	static void intel_pt_setup_pebs_events(struct intel_pt *pt)
4184	{
4185	struct evsel *evsel;
4186
4187	if (!pt->synth_opts.other_events)
4188	return;
4189
4190	evlist__for_each_entry(pt->session->evlist, evsel) {
4191	if (evsel->core.attr.aux_output && evsel->core.id) {
4192	if (pt->single_pebs) {
4193	pt->single_pebs = false;
4194	return;
4195	}
4196	pt->single_pebs = true;
4197	pt->sample_pebs = true;
4198	pt->pebs_data_src_fmt = intel_pt_data_src_fmt(pt, evsel);
4199	pt->pebs_evsel = evsel;
4200	}
4201	}
4202	}
4203
4204	static struct evsel intel_pt_find_sched_switch(struct evlist evlist)
4205	{
4206	struct evsel *evsel;
4207
4208	evlist__for_each_entry_reverse(evlist, evsel) {
4209	const char *name = evsel__name(evsel);
4210
4211	if (!strcmp(name, "sched:sched_switch"))
4212	return evsel;
4213	}
4214
4215	return NULL;
4216	}
4217
4218	static bool intel_pt_find_switch(struct evlist *evlist)
4219	{
4220	struct evsel *evsel;
4221
4222	evlist__for_each_entry(evlist, evsel) {
4223	if (evsel->core.attr.context_switch)
4224	return true;
4225	}
4226
4227	return false;
4228	}
4229
4230	static int intel_pt_perf_config(const char var, const char value, void *data)
4231	{
4232	struct intel_pt *pt = data;
4233
4234	if (!strcmp(var, "intel-pt.mispred-all"))
4235	pt->mispred_all = perf_config_bool(var, value);
4236
4237	if (!strcmp(var, "intel-pt.max-loops"))
4238	perf_config_int(&pt->max_loops, var, value);
4239
4240	return 0;
4241	}
4242
4243	/* Find least TSC which converts to ns or later */
4244	static u64 intel_pt_tsc_start(u64 ns, struct intel_pt *pt)
4245	{
4246	u64 tsc, tm;
4247
4248	tsc = perf_time_to_tsc(ns, &pt->tc);
4249
4250	while (1) {
4251	tm = tsc_to_perf_time(tsc, &pt->tc);
4252	if (tm < ns)
4253	break;
4254	tsc -= 1;
4255	}
4256
4257	while (tm < ns)
4258	tm = tsc_to_perf_time(++tsc, &pt->tc);
4259
4260	return tsc;
4261	}
4262
4263	/* Find greatest TSC which converts to ns or earlier */
4264	static u64 intel_pt_tsc_end(u64 ns, struct intel_pt *pt)
4265	{
4266	u64 tsc, tm;
4267
4268	tsc = perf_time_to_tsc(ns, &pt->tc);
4269
4270	while (1) {
4271	tm = tsc_to_perf_time(tsc, &pt->tc);
4272	if (tm > ns)
4273	break;
4274	tsc += 1;
4275	}
4276
4277	while (tm > ns)
4278	tm = tsc_to_perf_time(--tsc, &pt->tc);
4279
4280	return tsc;
4281	}
4282
4283	static int intel_pt_setup_time_ranges(struct intel_pt *pt,
4284	struct itrace_synth_opts *opts)
4285	{
4286	struct perf_time_interval *p = opts->ptime_range;
4287	int n = opts->range_num;
4288	int i;
4289
4290	if (!n \|\| !p \|\| pt->timeless_decoding)
4291	return 0;
4292
4293	pt->time_ranges = calloc(n, sizeof(struct range));
4294	if (!pt->time_ranges)
4295	return -ENOMEM;
4296
4297	pt->range_cnt = n;
4298
4299	intel_pt_log("%s: %u range(s)\n", __func__, n);
4300
4301	for (i = 0; i < n; i++) {
4302	struct range *r = &pt->time_ranges[i];
4303	u64 ts = p[i].start;
4304	u64 te = p[i].end;
4305
4306	/*
4307	* Take care to ensure the TSC range matches the perf-time range
4308	* when converted back to perf-time.
4309	*/
4310	r->start = ts ? intel_pt_tsc_start(ts, pt) : 0;
4311	r->end = te ? intel_pt_tsc_end(te, pt) : 0;
4312
4313	intel_pt_log("range %d: perf time interval: %"PRIu64" to %"PRIu64"\n",
4314	i, ts, te);
4315	intel_pt_log("range %d: TSC time interval: %#"PRIx64" to %#"PRIx64"\n",
4316	i, r->start, r->end);
4317	}
4318
4319	return 0;
4320	}
4321
4322	static int intel_pt_parse_vm_tm_corr_arg(struct intel_pt pt, char *args)
4323	{
4324	struct intel_pt_vmcs_info *vmcs_info;
4325	u64 tsc_offset, vmcs;
4326	char p = args;
4327
4328	errno = 0;
4329
4330	p = skip_spaces(p);
4331	if (!*p)
4332	return 1;
4333
4334	tsc_offset = strtoull(p, &p, 0);
4335	if (errno)
4336	return -errno;
4337	p = skip_spaces(p);
4338	if (*p != ':') {
4339	pt->dflt_tsc_offset = tsc_offset;
4340	*args = p;
4341	return 0;
4342	}
4343	p += 1;
4344	while (1) {
4345	vmcs = strtoull(p, &p, 0);
4346	if (errno)
4347	return -errno;
4348	if (!vmcs)
4349	return -EINVAL;
4350	vmcs_info = intel_pt_findnew_vmcs(&pt->vmcs_info, vmcs, tsc_offset);
4351	if (!vmcs_info)
4352	return -ENOMEM;
4353	p = skip_spaces(p);
4354	if (*p != ',')
4355	break;
4356	p += 1;
4357	}
4358	*args = p;
4359	return 0;
4360	}
4361
4362	static int intel_pt_parse_vm_tm_corr_args(struct intel_pt *pt)
4363	{
4364	char *args = pt->synth_opts.vm_tm_corr_args;
4365	int ret;
4366
4367	if (!args)
4368	return 0;
4369
4370	do {
4371	ret = intel_pt_parse_vm_tm_corr_arg(pt, &args);
4372	} while (!ret);
4373
4374	if (ret < 0) {
4375	pr_err("Failed to parse VM Time Correlation options\n");
4376	return ret;
4377	}
4378
4379	return 0;
4380	}
4381
4382	static const char * const intel_pt_info_fmts[] = {
4383	[INTEL_PT_PMU_TYPE] = " PMU Type %"PRId64"\n",
4384	[INTEL_PT_TIME_SHIFT] = " Time Shift %"PRIu64"\n",
4385	[INTEL_PT_TIME_MULT] = " Time Multiplier %"PRIu64"\n",
4386	[INTEL_PT_TIME_ZERO] = " Time Zero %"PRIu64"\n",
4387	[INTEL_PT_CAP_USER_TIME_ZERO] = " Cap Time Zero %"PRId64"\n",
4388	[INTEL_PT_TSC_BIT] = " TSC bit %#"PRIx64"\n",
4389	[INTEL_PT_NORETCOMP_BIT] = " NoRETComp bit %#"PRIx64"\n",
4390	[INTEL_PT_HAVE_SCHED_SWITCH] = " Have sched_switch %"PRId64"\n",
4391	[INTEL_PT_SNAPSHOT_MODE] = " Snapshot mode %"PRId64"\n",
4392	[INTEL_PT_PER_CPU_MMAPS] = " Per-cpu maps %"PRId64"\n",
4393	[INTEL_PT_MTC_BIT] = " MTC bit %#"PRIx64"\n",
4394	[INTEL_PT_MTC_FREQ_BITS] = " MTC freq bits %#"PRIx64"\n",
4395	[INTEL_PT_TSC_CTC_N] = " TSC:CTC numerator %"PRIu64"\n",
4396	[INTEL_PT_TSC_CTC_D] = " TSC:CTC denominator %"PRIu64"\n",
4397	[INTEL_PT_CYC_BIT] = " CYC bit %#"PRIx64"\n",
4398	[INTEL_PT_MAX_NONTURBO_RATIO] = " Max non-turbo ratio %"PRIu64"\n",
4399	[INTEL_PT_FILTER_STR_LEN] = " Filter string len. %"PRIu64"\n",
4400	};
4401
4402	static void intel_pt_print_info(__u64 *arr, int start, int finish)
4403	{
4404	int i;
4405
4406	if (!dump_trace)
4407	return;
4408
4409	for (i = start; i <= finish; i++) {
4410	const char *fmt = intel_pt_info_fmts[i];
4411
4412	if (fmt)
4413	fprintf(stdout, fmt, arr[i]);
4414	}
4415	}
4416
4417	static void intel_pt_print_info_str(const char name, const char str)
4418	{
4419	if (!dump_trace)
4420	return;
4421
4422	fprintf(stdout, " %-20s%s\n", name, str ? str : "");
4423	}
4424
4425	static bool intel_pt_has(struct perf_record_auxtrace_info *auxtrace_info, int pos)
4426	{
4427	return auxtrace_info->header.size >=
4428	sizeof(struct perf_record_auxtrace_info) + (sizeof(u64) * (pos + 1));
4429	}
4430
4431	int intel_pt_process_auxtrace_info(union perf_event *event,
4432	struct perf_session *session)
4433	{
4434	struct perf_record_auxtrace_info *auxtrace_info = &event->auxtrace_info;
4435	size_t min_sz = sizeof(u64) * (INTEL_PT_PER_CPU_MMAPS + 1);
4436	struct intel_pt *pt;
4437	void *info_end;
4438	__u64 *info;
4439	int err;
4440
4441	if (auxtrace_info->header.size < sizeof(struct perf_record_auxtrace_info) +
4442	min_sz)
4443	return -EINVAL;
4444
4445	pt = zalloc(sizeof(struct intel_pt));
4446	if (!pt)
4447	return -ENOMEM;
4448
4449	pt->vmcs_info = RB_ROOT;
4450
4451	addr_filters__init(&pt->filts);
4452
4453	err = perf_config(intel_pt_perf_config, pt);
4454	if (err)
4455	goto err_free;
4456
4457	err = auxtrace_queues__init(&pt->queues);
4458	if (err)
4459	goto err_free;
4460
4461	if (session->itrace_synth_opts->set) {
4462	pt->synth_opts = *session->itrace_synth_opts;
4463	} else {
4464	struct itrace_synth_opts *opts = session->itrace_synth_opts;
4465
4466	itrace_synth_opts__set_default(&pt->synth_opts, opts->default_no_sample);
4467	if (!opts->default_no_sample && !opts->inject) {
4468	pt->synth_opts.branches = false;
4469	pt->synth_opts.callchain = true;
4470	pt->synth_opts.add_callchain = true;
4471	}
4472	pt->synth_opts.thread_stack = opts->thread_stack;
4473	}
4474
4475	if (!(pt->synth_opts.log_plus_flags & AUXTRACE_LOG_FLG_USE_STDOUT))
4476	intel_pt_log_set_name(INTEL_PT_PMU_NAME);
4477
4478	pt->session = session;
4479	pt->machine = &session->machines.host; /* No kvm support */
4480	pt->auxtrace_type = auxtrace_info->type;
4481	pt->pmu_type = auxtrace_info->priv[INTEL_PT_PMU_TYPE];
4482	pt->tc.time_shift = auxtrace_info->priv[INTEL_PT_TIME_SHIFT];
4483	pt->tc.time_mult = auxtrace_info->priv[INTEL_PT_TIME_MULT];
4484	pt->tc.time_zero = auxtrace_info->priv[INTEL_PT_TIME_ZERO];
4485	pt->cap_user_time_zero = auxtrace_info->priv[INTEL_PT_CAP_USER_TIME_ZERO];
4486	pt->tsc_bit = auxtrace_info->priv[INTEL_PT_TSC_BIT];
4487	pt->noretcomp_bit = auxtrace_info->priv[INTEL_PT_NORETCOMP_BIT];
4488	pt->have_sched_switch = auxtrace_info->priv[INTEL_PT_HAVE_SCHED_SWITCH];
4489	pt->snapshot_mode = auxtrace_info->priv[INTEL_PT_SNAPSHOT_MODE];
4490	pt->per_cpu_mmaps = auxtrace_info->priv[INTEL_PT_PER_CPU_MMAPS];
4491	intel_pt_print_info(&auxtrace_info->priv[0], INTEL_PT_PMU_TYPE,
4492	INTEL_PT_PER_CPU_MMAPS);
4493
4494	if (intel_pt_has(auxtrace_info, INTEL_PT_CYC_BIT)) {
4495	pt->mtc_bit = auxtrace_info->priv[INTEL_PT_MTC_BIT];
4496	pt->mtc_freq_bits = auxtrace_info->priv[INTEL_PT_MTC_FREQ_BITS];
4497	pt->tsc_ctc_ratio_n = auxtrace_info->priv[INTEL_PT_TSC_CTC_N];
4498	pt->tsc_ctc_ratio_d = auxtrace_info->priv[INTEL_PT_TSC_CTC_D];
4499	pt->cyc_bit = auxtrace_info->priv[INTEL_PT_CYC_BIT];
4500	intel_pt_print_info(&auxtrace_info->priv[0], INTEL_PT_MTC_BIT,
4501	INTEL_PT_CYC_BIT);
4502	}
4503
4504	if (intel_pt_has(auxtrace_info, INTEL_PT_MAX_NONTURBO_RATIO)) {
4505	pt->max_non_turbo_ratio =
4506	auxtrace_info->priv[INTEL_PT_MAX_NONTURBO_RATIO];
4507	intel_pt_print_info(&auxtrace_info->priv[0],
4508	INTEL_PT_MAX_NONTURBO_RATIO,
4509	INTEL_PT_MAX_NONTURBO_RATIO);
4510	}
4511
4512	info = &auxtrace_info->priv[INTEL_PT_FILTER_STR_LEN] + 1;
4513	info_end = (void *)auxtrace_info + auxtrace_info->header.size;
4514
4515	if (intel_pt_has(auxtrace_info, INTEL_PT_FILTER_STR_LEN)) {
4516	size_t len;
4517
4518	len = auxtrace_info->priv[INTEL_PT_FILTER_STR_LEN];
4519	intel_pt_print_info(&auxtrace_info->priv[0],
4520	INTEL_PT_FILTER_STR_LEN,
4521	INTEL_PT_FILTER_STR_LEN);
4522	if (len) {
4523	const char filter = (const char )info;
4524
4525	len = roundup(len + 1, 8);
4526	info += len >> 3;
4527	if ((void *)info > info_end) {
4528	pr_err("%s: bad filter string length\n", __func__);
4529	err = -EINVAL;
4530	goto err_free_queues;
4531	}
4532	pt->filter = memdup(filter, len);
4533	if (!pt->filter) {
4534	err = -ENOMEM;
4535	goto err_free_queues;
4536	}
4537	if (session->header.needs_swap)
4538	mem_bswap_64(pt->filter, len);
4539	if (pt->filter[len - 1]) {
4540	pr_err("%s: filter string not null terminated\n", __func__);
4541	err = -EINVAL;
4542	goto err_free_queues;
4543	}
4544	err = addr_filters__parse_bare_filter(&pt->filts,
4545	filter);
4546	if (err)
4547	goto err_free_queues;
4548	}
4549	intel_pt_print_info_str("Filter string", pt->filter);
4550	}
4551
4552	if ((void *)info < info_end) {
4553	pt->cap_event_trace = *info++;
4554	if (dump_trace)
4555	fprintf(stdout, " Cap Event Trace %d\n",
4556	pt->cap_event_trace);
4557	}
4558
4559	pt->timeless_decoding = intel_pt_timeless_decoding(pt);
4560	if (pt->timeless_decoding && !pt->tc.time_mult)
4561	pt->tc.time_mult = 1;
4562	pt->have_tsc = intel_pt_have_tsc(pt);
4563	pt->sampling_mode = intel_pt_sampling_mode(pt);
4564	pt->est_tsc = !pt->timeless_decoding;
4565
4566	if (pt->synth_opts.vm_time_correlation) {
4567	if (pt->timeless_decoding) {
4568	pr_err("Intel PT has no time information for VM Time Correlation\n");
4569	err = -EINVAL;
4570	goto err_free_queues;
4571	}
4572	if (session->itrace_synth_opts->ptime_range) {
4573	pr_err("Time ranges cannot be specified with VM Time Correlation\n");
4574	err = -EINVAL;
4575	goto err_free_queues;
4576	}
4577	/* Currently TSC Offset is calculated using MTC packets */
4578	if (!intel_pt_have_mtc(pt)) {
4579	pr_err("MTC packets must have been enabled for VM Time Correlation\n");
4580	err = -EINVAL;
4581	goto err_free_queues;
4582	}
4583	err = intel_pt_parse_vm_tm_corr_args(pt);
4584	if (err)
4585	goto err_free_queues;
4586	}
4587
4588	pt->unknown_thread = thread__new(999999999, 999999999);
4589	if (!pt->unknown_thread) {
4590	err = -ENOMEM;
4591	goto err_free_queues;
4592	}
4593
4594	err = thread__set_comm(pt->unknown_thread, "unknown", 0);
4595	if (err)
4596	goto err_delete_thread;
4597	if (thread__init_maps(pt->unknown_thread, pt->machine)) {
4598	err = -ENOMEM;
4599	goto err_delete_thread;
4600	}
4601
4602	pt->auxtrace.process_event = intel_pt_process_event;
4603	pt->auxtrace.process_auxtrace_event = intel_pt_process_auxtrace_event;
4604	pt->auxtrace.queue_data = intel_pt_queue_data;
4605	pt->auxtrace.dump_auxtrace_sample = intel_pt_dump_sample;
4606	pt->auxtrace.flush_events = intel_pt_flush;
4607	pt->auxtrace.free_events = intel_pt_free_events;
4608	pt->auxtrace.free = intel_pt_free;
4609	pt->auxtrace.evsel_is_auxtrace = intel_pt_evsel_is_auxtrace;
4610	session->auxtrace = &pt->auxtrace;
4611
4612	if (dump_trace)
4613	return 0;
4614
4615	if (pt->have_sched_switch == 1) {
4616	pt->switch_evsel = intel_pt_find_sched_switch(session->evlist);
4617	if (!pt->switch_evsel) {
4618	pr_err("%s: missing sched_switch event\n", __func__);
4619	err = -EINVAL;
4620	goto err_delete_thread;
4621	}
4622	} else if (pt->have_sched_switch == 2 &&
4623	!intel_pt_find_switch(session->evlist)) {
4624	pr_err("%s: missing context_switch attribute flag\n", __func__);
4625	err = -EINVAL;
4626	goto err_delete_thread;
4627	}
4628
4629	if (pt->synth_opts.log) {
4630	bool log_on_error = pt->synth_opts.log_plus_flags & AUXTRACE_LOG_FLG_ON_ERROR;
4631	unsigned int log_on_error_size = pt->synth_opts.log_on_error_size;
4632
4633	intel_pt_log_enable(log_on_error, log_on_error_size);
4634	}
4635
4636	/* Maximum non-turbo ratio is TSC freq / 100 MHz */
4637	if (pt->tc.time_mult) {
4638	u64 tsc_freq = intel_pt_ns_to_ticks(pt, 1000000000);
4639
4640	if (!pt->max_non_turbo_ratio)
4641	pt->max_non_turbo_ratio =
4642	(tsc_freq + 50000000) / 100000000;
4643	intel_pt_log("TSC frequency %"PRIu64"\n", tsc_freq);
4644	intel_pt_log("Maximum non-turbo ratio %u\n",
4645	pt->max_non_turbo_ratio);
4646	pt->cbr2khz = tsc_freq / pt->max_non_turbo_ratio / 1000;
4647	}
4648
4649	err = intel_pt_setup_time_ranges(pt, session->itrace_synth_opts);
4650	if (err)
4651	goto err_delete_thread;
4652
4653	if (pt->synth_opts.calls)
4654	pt->branches_filter \|= PERF_IP_FLAG_CALL \| PERF_IP_FLAG_ASYNC \|
4655	PERF_IP_FLAG_TRACE_END;
4656	if (pt->synth_opts.returns)
4657	pt->branches_filter \|= PERF_IP_FLAG_RETURN \|
4658	PERF_IP_FLAG_TRACE_BEGIN;
4659
4660	if ((pt->synth_opts.callchain \|\| pt->synth_opts.add_callchain) &&
4661	!symbol_conf.use_callchain) {
4662	symbol_conf.use_callchain = true;
4663	if (callchain_register_param(&callchain_param) < 0) {
4664	symbol_conf.use_callchain = false;
4665	pt->synth_opts.callchain = false;
4666	pt->synth_opts.add_callchain = false;
4667	}
4668	}
4669
4670	if (pt->synth_opts.add_callchain) {
4671	err = intel_pt_callchain_init(pt);
4672	if (err)
4673	goto err_delete_thread;
4674	}
4675
4676	if (pt->synth_opts.last_branch \|\| pt->synth_opts.add_last_branch) {
4677	pt->br_stack_sz = pt->synth_opts.last_branch_sz;
4678	pt->br_stack_sz_plus = pt->br_stack_sz;
4679	}
4680
4681	if (pt->synth_opts.add_last_branch) {
4682	err = intel_pt_br_stack_init(pt);
4683	if (err)
4684	goto err_delete_thread;
4685	/*
4686	* Additional branch stack size to cater for tracing from the
4687	* actual sample ip to where the sample time is recorded.
4688	* Measured at about 200 branches, but generously set to 1024.
4689	* If kernel space is not being traced, then add just 1 for the
4690	* branch to kernel space.
4691	*/
4692	if (intel_pt_tracing_kernel(pt))
4693	pt->br_stack_sz_plus += 1024;
4694	else
4695	pt->br_stack_sz_plus += 1;
4696	}
4697
4698	pt->use_thread_stack = pt->synth_opts.callchain \|\|
4699	pt->synth_opts.add_callchain \|\|
4700	pt->synth_opts.thread_stack \|\|
4701	pt->synth_opts.last_branch \|\|
4702	pt->synth_opts.add_last_branch;
4703
4704	pt->callstack = pt->synth_opts.callchain \|\|
4705	pt->synth_opts.add_callchain \|\|
4706	pt->synth_opts.thread_stack;
4707
4708	err = intel_pt_synth_events(pt, session);
4709	if (err)
4710	goto err_delete_thread;
4711
4712	intel_pt_setup_pebs_events(pt);
4713
4714	if (perf_data__is_pipe(session->data)) {
4715	pr_warning("WARNING: Intel PT with pipe mode is not recommended.\n"
4716	" The output cannot relied upon. In particular,\n"
4717	" timestamps and the order of events may be incorrect.\n");
4718	}
4719
4720	if (pt->sampling_mode \|\| list_empty(&session->auxtrace_index))
4721	err = auxtrace_queue_data(session, true, true);
4722	else
4723	err = auxtrace_queues__process_index(&pt->queues, session);
4724	if (err)
4725	goto err_delete_thread;
4726
4727	if (pt->queues.populated)
4728	pt->data_queued = true;
4729
4730	if (pt->timeless_decoding)
4731	pr_debug2("Intel PT decoding without timestamps\n");
4732
4733	return 0;
4734
4735	err_delete_thread:
4736	zfree(&pt->chain);
4737	thread__zput(pt->unknown_thread);
4738	err_free_queues:
4739	intel_pt_log_disable();
4740	auxtrace_queues__free(&pt->queues);
4741	session->auxtrace = NULL;
4742	err_free:
4743	addr_filters__exit(&pt->filts);
4744	zfree(&pt->filter);
4745	zfree(&pt->time_ranges);
4746	free(pt);
4747	return err;
4748	}