projects/linux·blob master · refs

4147 lines · 120.6 KiB

1// SPDX-License-Identifier: GPL-2.0
2#include <errno.h>
3#include <signal.h>
4#include <inttypes.h>
5#include <linux/err.h>
6#include <linux/kernel.h>
7#include <linux/zalloc.h>
8#include <api/fs/fs.h>
9
10#include <byteswap.h>
11#include <unistd.h>
12#include <sys/types.h>
13#include <sys/mman.h>
14#include <perf/cpumap.h>
15#include <perf/event.h>
16
17#include "map_symbol.h"
18#include "branch.h"
19#include "debug.h"
20#include "dwarf-regs.h"
21#include "env.h"
22#include "evlist.h"
23#include "evsel.h"
24#include "memswap.h"
25#include "map.h"
26#include "symbol.h"
27#include "session.h"
28#include "tool.h"
29#include "perf_regs.h"
30#include "asm/bug.h"
31#include "auxtrace.h"
32#include "thread.h"
33#include "thread-stack.h"
34#include "sample-raw.h"
35#include "stat.h"
36#include "tsc.h"
37#include "ui/progress.h"
38#include "util.h"
39#include "arch/common.h"
40#include "units.h"
41#include "annotate.h"
42#include "perf.h"
43#include <internal/lib.h>
44
45static int perf_session__deliver_event(struct perf_session *session,
46 union perf_event *event,
47 const struct perf_tool *tool,
48 u64 file_offset,
49 const char *file_path);
50
51static int perf_session__open(struct perf_session *session)
52{
53 struct perf_data *data = session->data;
54
55 if (perf_session__read_header(session) < 0) {
56 pr_err("incompatible file format (rerun with -v to learn more)\n");
57 return -1;
58 }
59
60 if (perf_header__has_feat(&session->header, HEADER_AUXTRACE)) {
61 /* Auxiliary events may reference exited threads, hold onto dead ones. */
62 symbol_conf.keep_exited_threads = true;
63 }
64
65 if (perf_data__is_pipe(data))
66 return 0;
67
68 if (perf_header__has_feat(&session->header, HEADER_STAT))
69 return 0;
70
71 if (!evlist__valid_sample_type(session->evlist)) {
72 pr_err("non matching sample_type\n");
73 return -1;
74 }
75
76 if (!evlist__valid_sample_id_all(session->evlist)) {
77 pr_err("non matching sample_id_all\n");
78 return -1;
79 }
80
81 if (!evlist__valid_read_format(session->evlist)) {
82 pr_err("non matching read_format\n");
83 return -1;
84 }
85
86 return 0;
87}
88
89void perf_session__set_id_hdr_size(struct perf_session *session)
90{
91 u16 id_hdr_size = evlist__id_hdr_size(session->evlist);
92
93 machines__set_id_hdr_size(&session->machines, id_hdr_size);
94}
95
96int perf_session__create_kernel_maps(struct perf_session *session)
97{
98 int ret = machine__create_kernel_maps(&session->machines.host);
99
100 if (ret >= 0)
101 ret = machines__create_guest_kernel_maps(&session->machines);
102 return ret;
103}
104
105static void perf_session__destroy_kernel_maps(struct perf_session *session)
106{
107 machines__destroy_kernel_maps(&session->machines);
108}
109
110static bool perf_session__has_comm_exec(struct perf_session *session)
111{
112 struct evsel *evsel;
113
114 evlist__for_each_entry(session->evlist, evsel) {
115 if (evsel->core.attr.comm_exec)
116 return true;
117 }
118
119 return false;
120}
121
122static void perf_session__set_comm_exec(struct perf_session *session)
123{
124 bool comm_exec = perf_session__has_comm_exec(session);
125
126 machines__set_comm_exec(&session->machines, comm_exec);
127}
128
129static int ordered_events__deliver_event(struct ordered_events *oe,
130 struct ordered_event *event)
131{
132 struct perf_session *session = container_of(oe, struct perf_session,
133 ordered_events);
134 int ret = perf_session__deliver_event(session, event->event,
135 session->tool, event->file_offset,
136 event->file_path);
137
138 if (ret) {
139 pr_err("%#" PRIx64 " [%#x]: ordered event processing failed (%d) for event of type: %s (%d)\n",
140 event->file_offset, event->event->header.size, ret,
141 perf_event__name(event->event->header.type),
142 event->event->header.type);
143 }
144 return ret;
145}
146
147struct perf_session *__perf_session__new(struct perf_data *data,
148 struct perf_tool *tool,
149 bool trace_event_repipe,
150 struct perf_env *host_env)
151{
152 int ret = -ENOMEM;
153 struct perf_session *session = zalloc(sizeof(*session));
154
155 if (!session)
156 goto out;
157
158 session->trace_event_repipe = trace_event_repipe;
159 session->tool = tool;
160 session->decomp_data.zstd_decomp = &session->zstd_data;
161 session->active_decomp = &session->decomp_data;
162 INIT_LIST_HEAD(&session->auxtrace_index);
163 perf_env__init(&session->header.env);
164 if (machines__init(&session->machines))
165 goto out_delete;
166
167 ordered_events__init(&session->ordered_events,
168 ordered_events__deliver_event, NULL);
169 if (data) {
170 ret = perf_data__open(data);
171 if (ret < 0)
172 goto out_delete;
173
174 session->data = data;
175
176 if (perf_data__is_read(data)) {
177 ret = perf_session__open(session);
178 if (ret < 0)
179 goto out_delete;
180
181 /*
182 * set session attributes that are present in perf.data
183 * but not in pipe-mode.
184 */
185 if (!data->is_pipe) {
186 perf_session__set_id_hdr_size(session);
187 perf_session__set_comm_exec(session);
188 }
189
190 evlist__init_trace_event_sample_raw(session->evlist, &session->header.env);
191
192 /* Open the directory data. */
193 if (data->is_dir) {
194 ret = perf_data__open_dir(data);
195 if (ret)
196 goto out_delete;
197 }
198
199 if (!symbol_conf.kallsyms_name &&
200 !symbol_conf.vmlinux_name)
201 symbol_conf.kallsyms_name = perf_data__kallsyms_name(data);
202 }
203 } else {
204 assert(host_env != NULL);
205 session->machines.host.env = host_env;
206 }
207 if (session->evlist)
208 evlist__set_session(session->evlist, session);
209
210 session->machines.host.single_address_space =
211 perf_env__single_address_space(session->machines.host.env);
212
213 if (!data || perf_data__is_write(data)) {
214 /*
215 * In O_RDONLY mode this will be performed when reading the
216 * kernel MMAP event, in perf_event__process_mmap().
217 */
218 if (perf_session__create_kernel_maps(session) < 0)
219 pr_warning("Cannot read kernel map\n");
220 }
221
222 /*
223 * In pipe-mode, evlist is empty until PERF_RECORD_HEADER_ATTR is
224 * processed, so evlist__sample_id_all is not meaningful here.
225 */
226 if ((!data || !data->is_pipe) && tool && tool->ordering_requires_timestamps &&
227 tool->ordered_events && !evlist__sample_id_all(session->evlist)) {
228 dump_printf("WARNING: No sample_id_all support, falling back to unordered processing\n");
229 tool->ordered_events = false;
230 }
231
232 return session;
233
234 out_delete:
235 perf_session__delete(session);
236 out:
237 return ERR_PTR(ret);
238}
239
240static void perf_decomp__release_events(struct decomp *next)
241{
242 struct decomp *decomp;
243 size_t mmap_len;
244
245 do {
246 decomp = next;
247 if (decomp == NULL)
248 break;
249 next = decomp->next;
250 mmap_len = decomp->mmap_len;
251 munmap(decomp, mmap_len);
252 } while (1);
253}
254
255void perf_session__delete(struct perf_session *session)
256{
257 if (session == NULL)
258 return;
259 auxtrace__free(session);
260 auxtrace_index__free(&session->auxtrace_index);
261 debuginfo_cache__delete();
262 perf_session__destroy_kernel_maps(session);
263 perf_decomp__release_events(session->decomp_data.decomp);
264 perf_env__exit(&session->header.env);
265 machines__exit(&session->machines);
266 if (session->data) {
267 if (perf_data__is_read(session->data))
268 evlist__put(session->evlist);
269 perf_data__close(session->data);
270 }
271#ifdef HAVE_LIBTRACEEVENT
272 trace_event__cleanup(&session->tevent);
273#endif
274 free(session);
275}
276
277static void swap_sample_id_all(union perf_event *event, void *data)
278{
279 void *end = (void *) event + event->header.size;
280 int size;
281
282 if (data >= end)
283 return;
284
285 size = end - data;
286 if (size % sizeof(u64)) {
287 pr_warning("swap_sample_id_all: unaligned sample_id_all remainder (%d), skipping swap\n", size);
288 return;
289 }
290 if (size > 0)
291 mem_bswap_64(data, size);
292}
293
294static int perf_event__all64_swap(union perf_event *event,
295 bool sample_id_all __maybe_unused)
296{
297 struct perf_event_header *hdr = &event->header;
298 size_t size = event->header.size - sizeof(*hdr);
299
300 /* mem_bswap_64 rounds up to 8-byte chunks — unaligned size overruns the buffer */
301 if (size % sizeof(u64))
302 return -1;
303 mem_bswap_64(hdr + 1, size);
304 return 0;
305}
306
307static int perf_event__comm_swap(union perf_event *event, bool sample_id_all)
308{
309 event->comm.pid = bswap_32(event->comm.pid);
310 event->comm.tid = bswap_32(event->comm.tid);
311
312 if (sample_id_all) {
313 void *data = &event->comm.comm;
314 void *end = (void *)event + event->header.size;
315 size_t len = strnlen(data, end - data);
316
317 /*
318 * No NUL within the event boundary — can't locate where
319 * sample_id_all starts. Reject so the event is skipped
320 * rather than swapping garbage.
321 */
322 if (len == (size_t)(end - data))
323 return -1;
324 data += PERF_ALIGN(len + 1, sizeof(u64));
325 swap_sample_id_all(event, data);
326 }
327 return 0;
328}
329
330static int perf_event__mmap_swap(union perf_event *event,
331 bool sample_id_all)
332{
333 event->mmap.pid = bswap_32(event->mmap.pid);
334 event->mmap.tid = bswap_32(event->mmap.tid);
335 event->mmap.start = bswap_64(event->mmap.start);
336 event->mmap.len = bswap_64(event->mmap.len);
337 event->mmap.pgoff = bswap_64(event->mmap.pgoff);
338
339 if (sample_id_all) {
340 void *data = &event->mmap.filename;
341 void *end = (void *)event + event->header.size;
342 size_t len = strnlen(data, end - data);
343
344 /* See comment in perf_event__comm_swap() */
345 if (len == (size_t)(end - data))
346 return -1;
347 data += PERF_ALIGN(len + 1, sizeof(u64));
348 swap_sample_id_all(event, data);
349 }
350 return 0;
351}
352
353static int perf_event__mmap2_swap(union perf_event *event,
354 bool sample_id_all)
355{
356 event->mmap2.pid = bswap_32(event->mmap2.pid);
357 event->mmap2.tid = bswap_32(event->mmap2.tid);
358 event->mmap2.start = bswap_64(event->mmap2.start);
359 event->mmap2.len = bswap_64(event->mmap2.len);
360 event->mmap2.pgoff = bswap_64(event->mmap2.pgoff);
361
362 if (!(event->header.misc & PERF_RECORD_MISC_MMAP_BUILD_ID)) {
363 event->mmap2.maj = bswap_32(event->mmap2.maj);
364 event->mmap2.min = bswap_32(event->mmap2.min);
365 event->mmap2.ino = bswap_64(event->mmap2.ino);
366 event->mmap2.ino_generation = bswap_64(event->mmap2.ino_generation);
367 }
368
369 if (sample_id_all) {
370 void *data = &event->mmap2.filename;
371 void *end = (void *)event + event->header.size;
372 size_t len = strnlen(data, end - data);
373
374 /* See comment in perf_event__comm_swap() */
375 if (len == (size_t)(end - data))
376 return -1;
377 data += PERF_ALIGN(len + 1, sizeof(u64));
378 swap_sample_id_all(event, data);
379 }
380 return 0;
381}
382
383static int perf_event__task_swap(union perf_event *event, bool sample_id_all)
384{
385 event->fork.pid = bswap_32(event->fork.pid);
386 event->fork.tid = bswap_32(event->fork.tid);
387 event->fork.ppid = bswap_32(event->fork.ppid);
388 event->fork.ptid = bswap_32(event->fork.ptid);
389 event->fork.time = bswap_64(event->fork.time);
390
391 if (sample_id_all)
392 swap_sample_id_all(event, &event->fork + 1);
393 return 0;
394}
395
396static int perf_event__read_swap(union perf_event *event,
397 bool sample_id_all __maybe_unused)
398{
399 size_t tail;
400
401 event->read.pid = bswap_32(event->read.pid);
402 event->read.tid = bswap_32(event->read.tid);
403 /*
404 * Everything after pid/tid is u64: the read values (variable
405 * set determined by attr.read_format, which we don't have
406 * here) optionally followed by sample_id_all fields.
407 * Since all are u64, swap the entire remaining tail at once.
408 */
409 tail = event->header.size - offsetof(struct perf_record_read, value);
410 /* mem_bswap_64 rounds up to 8-byte chunks — unaligned tail overruns the buffer */
411 if (tail % sizeof(u64))
412 return -1;
413 mem_bswap_64(&event->read.value, tail);
414 return 0;
415}
416
417static int perf_event__aux_swap(union perf_event *event, bool sample_id_all)
418{
419 event->aux.aux_offset = bswap_64(event->aux.aux_offset);
420 event->aux.aux_size = bswap_64(event->aux.aux_size);
421 event->aux.flags = bswap_64(event->aux.flags);
422
423 if (sample_id_all)
424 swap_sample_id_all(event, &event->aux + 1);
425 return 0;
426}
427
428static int perf_event__itrace_start_swap(union perf_event *event,
429 bool sample_id_all)
430{
431 event->itrace_start.pid = bswap_32(event->itrace_start.pid);
432 event->itrace_start.tid = bswap_32(event->itrace_start.tid);
433
434 if (sample_id_all)
435 swap_sample_id_all(event, &event->itrace_start + 1);
436 return 0;
437}
438
439static int perf_event__switch_swap(union perf_event *event, bool sample_id_all)
440{
441 if (event->header.type == PERF_RECORD_SWITCH_CPU_WIDE) {
442 event->context_switch.next_prev_pid =
443 bswap_32(event->context_switch.next_prev_pid);
444 event->context_switch.next_prev_tid =
445 bswap_32(event->context_switch.next_prev_tid);
446 }
447
448 if (sample_id_all) {
449 /*
450 * PERF_RECORD_SWITCH has no fields beyond the header;
451 * SWITCH_CPU_WIDE adds pid/tid. Use the right offset
452 * so sample_id starts at the correct position.
453 */
454 if (event->header.type == PERF_RECORD_SWITCH)
455 swap_sample_id_all(event, (void *)event + sizeof(event->header));
456 else
457 swap_sample_id_all(event, &event->context_switch + 1);
458 }
459 return 0;
460}
461
462static int perf_event__text_poke_swap(union perf_event *event, bool sample_id_all)
463{
464 event->text_poke.addr = bswap_64(event->text_poke.addr);
465 event->text_poke.old_len = bswap_16(event->text_poke.old_len);
466 event->text_poke.new_len = bswap_16(event->text_poke.new_len);
467
468 if (sample_id_all) {
469 void *data = &event->text_poke.old_len;
470 void *end = (void *)event + event->header.size;
471 size_t len = sizeof(event->text_poke.old_len) +
472 sizeof(event->text_poke.new_len) +
473 event->text_poke.old_len +
474 event->text_poke.new_len;
475
476 /* old_len + new_len exceeds event — can't find sample_id_all */
477 if (data + len > end)
478 return -1;
479 data += PERF_ALIGN(len, sizeof(u64));
480 swap_sample_id_all(event, data);
481 }
482 return 0;
483}
484
485static int perf_event__throttle_swap(union perf_event *event,
486 bool sample_id_all)
487{
488 event->throttle.time = bswap_64(event->throttle.time);
489 event->throttle.id = bswap_64(event->throttle.id);
490 event->throttle.stream_id = bswap_64(event->throttle.stream_id);
491
492 if (sample_id_all)
493 swap_sample_id_all(event, &event->throttle + 1);
494 return 0;
495}
496
497static int perf_event__namespaces_swap(union perf_event *event,
498 bool sample_id_all)
499{
500 u64 i, nr, max_nr;
501
502 event->namespaces.pid = bswap_32(event->namespaces.pid);
503 event->namespaces.tid = bswap_32(event->namespaces.tid);
504 event->namespaces.nr_namespaces = bswap_64(event->namespaces.nr_namespaces);
505
506 nr = event->namespaces.nr_namespaces;
507 /*
508 * Cannot underflow: perf_event__min_size[] guarantees header.size >= sizeof.
509 * When sample_id_all is present max_nr slightly overestimates the
510 * array space because header.size includes the trailing sample_id.
511 * Harmless: both the per-element bswap_64 loop and swap_sample_id_all()
512 * perform the same u64 byte swap, so the result is correct regardless
513 * of where the boundary between array and sample_id falls.
514 */
515 max_nr = (event->header.size - sizeof(event->namespaces)) /
516 sizeof(event->namespaces.link_info[0]);
517 /*
518 * Safe to clamp: each namespace entry is indexed by type;
519 * missing entries just won't be resolved.
520 */
521 if (nr > max_nr) {
522 pr_warning("WARNING: PERF_RECORD_NAMESPACES: nr_namespaces %" PRIu64 " exceeds payload (max %" PRIu64 "), clamping\n",
523 nr, max_nr);
524 nr = max_nr;
525 event->namespaces.nr_namespaces = nr;
526 }
527
528 for (i = 0; i < nr; i++) {
529 struct perf_ns_link_info *ns = &event->namespaces.link_info[i];
530
531 ns->dev = bswap_64(ns->dev);
532 ns->ino = bswap_64(ns->ino);
533 }
534
535 if (sample_id_all)
536 swap_sample_id_all(event, &event->namespaces.link_info[i]);
537 return 0;
538}
539
540static int perf_event__cgroup_swap(union perf_event *event, bool sample_id_all)
541{
542 event->cgroup.id = bswap_64(event->cgroup.id);
543
544 if (sample_id_all) {
545 void *data = &event->cgroup.path;
546 void *end = (void *)event + event->header.size;
547 size_t len = strnlen(data, end - data);
548
549 /* See comment in perf_event__comm_swap() */
550 if (len == (size_t)(end - data))
551 return -1;
552 data += PERF_ALIGN(len + 1, sizeof(u64));
553 swap_sample_id_all(event, data);
554 }
555 return 0;
556}
557
558static u8 revbyte(u8 b)
559{
560 int rev = (b >> 4) | ((b & 0xf) << 4);
561 rev = ((rev & 0xcc) >> 2) | ((rev & 0x33) << 2);
562 rev = ((rev & 0xaa) >> 1) | ((rev & 0x55) << 1);
563 return (u8) rev;
564}
565
566/*
567 * XXX this is hack in attempt to carry flags bitfield
568 * through endian village. ABI says:
569 *
570 * Bit-fields are allocated from right to left (least to most significant)
571 * on little-endian implementations and from left to right (most to least
572 * significant) on big-endian implementations.
573 *
574 * The above seems to be byte specific, so we need to reverse each
575 * byte of the bitfield. 'Internet' also says this might be implementation
576 * specific and we probably need proper fix and carry perf_event_attr
577 * bitfield flags in separate data file FEAT_ section. Thought this seems
578 * to work for now.
579 */
580static void swap_bitfield(u8 *p, unsigned len)
581{
582 unsigned i;
583
584 for (i = 0; i < len; i++) {
585 *p = revbyte(*p);
586 p++;
587 }
588}
589
590/* exported for swapping attributes in file header */
591void perf_event__attr_swap(struct perf_event_attr *attr)
592{
593 attr->type = bswap_32(attr->type);
594 attr->size = bswap_32(attr->size);
595
596 /*
597 * ABI0: size == 0 means the producer didn't set it.
598 * Assume PERF_ATTR_SIZE_VER0 so bswap_safe() below
599 * correctly swaps the VER0 fields instead of skipping
600 * everything. Same convention as read_attr().
601 */
602 if (!attr->size)
603 attr->size = PERF_ATTR_SIZE_VER0;
604
605/* Verify the full field extent fits, not just its start offset */
606#define bswap_safe(f, n) \
607 (attr->size >= (offsetof(struct perf_event_attr, f) + \
608 sizeof(attr->f) * ((n) + 1)))
609#define bswap_field(f, sz) \
610do { \
611 if (bswap_safe(f, 0)) \
612 attr->f = bswap_##sz(attr->f); \
613} while(0)
614#define bswap_field_16(f) bswap_field(f, 16)
615#define bswap_field_32(f) bswap_field(f, 32)
616#define bswap_field_64(f) bswap_field(f, 64)
617
618 bswap_field_64(config);
619 bswap_field_64(sample_period);
620 bswap_field_64(sample_type);
621 bswap_field_64(read_format);
622 bswap_field_32(wakeup_events);
623 bswap_field_32(bp_type);
624 bswap_field_64(bp_addr);
625 bswap_field_64(bp_len);
626 bswap_field_64(branch_sample_type);
627 bswap_field_64(sample_regs_user);
628 bswap_field_32(sample_stack_user);
629 bswap_field_32(aux_watermark);
630 bswap_field_16(sample_max_stack);
631 bswap_field_32(aux_sample_size);
632
633 /*
634 * After read_format are bitfields. Check read_format because
635 * we are unable to use offsetof on bitfield.
636 */
637 if (bswap_safe(read_format, 1))
638 swap_bitfield((u8 *) (&attr->read_format + 1),
639 sizeof(u64));
640#undef bswap_field_64
641#undef bswap_field_32
642#undef bswap_field
643#undef bswap_safe
644}
645
646static int perf_event__hdr_attr_swap(union perf_event *event,
647 bool sample_id_all __maybe_unused)
648{
649 u32 attr_size, payload_size;
650 size_t size;
651
652 /*
653 * Validate attr.size (still foreign-endian) before calling
654 * perf_event__attr_swap(), which uses it via bswap_safe()
655 * to decide which fields to swap. A crafted attr.size
656 * larger than the event payload would swap past the event
657 * boundary and corrupt adjacent memory.
658 *
659 * header.size alignment is already validated by
660 * perf_session__process_event(). The min_size table
661 * guarantees header.size >= sizeof(header) +
662 * PERF_ATTR_SIZE_VER0, so attr.size is safe to access.
663 */
664 attr_size = bswap_32(event->attr.attr.size);
665 /*
666 * ABI0: size field not set. This only happens in pipe/inject
667 * mode where HEADER_ATTR events carry their own attr. For
668 * regular perf.data files, read_attr() uses f_header.attr_size
669 * from the file header instead. Assume PERF_ATTR_SIZE_VER0.
670 */
671 if (!attr_size)
672 attr_size = PERF_ATTR_SIZE_VER0;
673 payload_size = event->header.size - sizeof(event->header);
674
675 if (attr_size < PERF_ATTR_SIZE_VER0 || attr_size % sizeof(u64) ||
676 attr_size > payload_size) {
677 pr_err("PERF_RECORD_HEADER_ATTR: invalid attr.size %u (min: %d, max: %u, 8-byte aligned)\n",
678 attr_size, PERF_ATTR_SIZE_VER0, payload_size);
679 return -1;
680 }
681
682 perf_event__attr_swap(&event->attr.attr);
683
684 size = event->header.size;
685 size -= perf_record_header_attr_id(event) - (void *)event;
686 mem_bswap_64(perf_record_header_attr_id(event), size);
687 return 0;
688}
689
690static int perf_event__build_id_swap(union perf_event *event,
691 bool sample_id_all)
692{
693 event->build_id.pid = bswap_32(event->build_id.pid);
694
695 if (sample_id_all) {
696 void *data = &event->build_id.filename;
697 void *end = (void *)event + event->header.size;
698 size_t len = strnlen(data, end - data);
699
700 /* See comment in perf_event__comm_swap() */
701 if (len == (size_t)(end - data))
702 return -1;
703 data += PERF_ALIGN(len + 1, sizeof(u64));
704 swap_sample_id_all(event, data);
705 }
706 return 0;
707}
708
709static int perf_event__event_update_swap(union perf_event *event,
710 bool sample_id_all __maybe_unused)
711{
712 struct perf_record_event_update *ev = &event->event_update;
713
714 ev->type = bswap_64(ev->type);
715 ev->id = bswap_64(ev->id);
716
717 /*
718 * Swap variant-specific fields so the processing path
719 * sees native byte order.
720 */
721 if (ev->type == PERF_EVENT_UPDATE__SCALE) {
722 if (event->header.size < offsetof(struct perf_record_event_update, scale) +
723 sizeof(ev->scale))
724 return -1;
725 mem_bswap_64(&ev->scale.scale, sizeof(ev->scale.scale));
726 } else if (ev->type == PERF_EVENT_UPDATE__CPUS) {
727 u32 cpus_payload;
728 struct perf_record_cpu_map_data *data = &ev->cpus.cpus;
729
730 /* CPUS fields start at the same offset as scale (union) */
731 if (event->header.size < offsetof(struct perf_record_event_update, cpus) +
732 sizeof(__u16) + sizeof(struct perf_record_range_cpu_map))
733 return -1;
734 cpus_payload = event->header.size - offsetof(struct perf_record_event_update, cpus);
735 data->type = bswap_16(data->type);
736 /*
737 * Full swap including array elements — same logic as
738 * perf_event__cpu_map_swap() but scoped to the
739 * embedded cpu_map_data within EVENT_UPDATE.
740 */
741 switch (data->type) {
742 case PERF_CPU_MAP__CPUS: {
743 u16 nr, max_nr;
744
745 data->cpus_data.nr = bswap_16(data->cpus_data.nr);
746 nr = data->cpus_data.nr;
747 max_nr = (cpus_payload - offsetof(struct perf_record_cpu_map_data,
748 cpus_data.cpu)) /
749 sizeof(data->cpus_data.cpu[0]);
750 if (nr > max_nr) {
751 nr = max_nr;
752 data->cpus_data.nr = nr;
753 }
754 for (unsigned int i = 0; i < nr; i++)
755 data->cpus_data.cpu[i] = bswap_16(data->cpus_data.cpu[i]);
756 break;
757 }
758 case PERF_CPU_MAP__MASK:
759 data->mask32_data.long_size = bswap_16(data->mask32_data.long_size);
760 switch (data->mask32_data.long_size) {
761 case 4: {
762 u16 nr, max_nr;
763
764 data->mask32_data.nr = bswap_16(data->mask32_data.nr);
765 nr = data->mask32_data.nr;
766 max_nr = (cpus_payload - offsetof(struct perf_record_cpu_map_data,
767 mask32_data.mask)) /
768 sizeof(data->mask32_data.mask[0]);
769 if (nr > max_nr) {
770 nr = max_nr;
771 data->mask32_data.nr = nr;
772 }
773 for (unsigned int i = 0; i < nr; i++)
774 data->mask32_data.mask[i] = bswap_32(data->mask32_data.mask[i]);
775 break;
776 }
777 case 8: {
778 u16 nr, max_nr;
779
780 data->mask64_data.nr = bswap_16(data->mask64_data.nr);
781 nr = data->mask64_data.nr;
782 if (cpus_payload < offsetof(struct perf_record_cpu_map_data, mask64_data.mask)) {
783 data->mask64_data.nr = 0;
784 break;
785 }
786 max_nr = (cpus_payload - offsetof(struct perf_record_cpu_map_data,
787 mask64_data.mask)) /
788 sizeof(data->mask64_data.mask[0]);
789 if (nr > max_nr) {
790 nr = max_nr;
791 data->mask64_data.nr = nr;
792 }
793 for (unsigned int i = 0; i < nr; i++)
794 data->mask64_data.mask[i] = bswap_64(data->mask64_data.mask[i]);
795 break;
796 }
797 default:
798 break;
799 }
800 break;
801 case PERF_CPU_MAP__RANGE_CPUS:
802 data->range_cpu_data.start_cpu = bswap_16(data->range_cpu_data.start_cpu);
803 data->range_cpu_data.end_cpu = bswap_16(data->range_cpu_data.end_cpu);
804 break;
805 default:
806 break;
807 }
808 }
809 return 0;
810}
811
812static int perf_event__event_type_swap(union perf_event *event,
813 bool sample_id_all __maybe_unused)
814{
815 event->event_type.event_type.event_id =
816 bswap_64(event->event_type.event_type.event_id);
817 return 0;
818}
819
820static int perf_event__tracing_data_swap(union perf_event *event,
821 bool sample_id_all __maybe_unused)
822{
823 event->tracing_data.size = bswap_32(event->tracing_data.size);
824 return 0;
825}
826
827static int perf_event__auxtrace_info_swap(union perf_event *event,
828 bool sample_id_all __maybe_unused)
829{
830 size_t size;
831
832 event->auxtrace_info.type = bswap_32(event->auxtrace_info.type);
833
834 size = event->header.size;
835 size -= (void *)&event->auxtrace_info.priv - (void *)event;
836 mem_bswap_64(event->auxtrace_info.priv, size);
837 return 0;
838}
839
840static int perf_event__auxtrace_swap(union perf_event *event,
841 bool sample_id_all __maybe_unused)
842{
843 event->auxtrace.size = bswap_64(event->auxtrace.size);
844 event->auxtrace.offset = bswap_64(event->auxtrace.offset);
845 event->auxtrace.reference = bswap_64(event->auxtrace.reference);
846 event->auxtrace.idx = bswap_32(event->auxtrace.idx);
847 event->auxtrace.tid = bswap_32(event->auxtrace.tid);
848 event->auxtrace.cpu = bswap_32(event->auxtrace.cpu);
849 return 0;
850}
851
852static int perf_event__auxtrace_error_swap(union perf_event *event,
853 bool sample_id_all __maybe_unused)
854{
855 event->auxtrace_error.type = bswap_32(event->auxtrace_error.type);
856 event->auxtrace_error.code = bswap_32(event->auxtrace_error.code);
857 event->auxtrace_error.cpu = bswap_32(event->auxtrace_error.cpu);
858 event->auxtrace_error.pid = bswap_32(event->auxtrace_error.pid);
859 event->auxtrace_error.tid = bswap_32(event->auxtrace_error.tid);
860 event->auxtrace_error.fmt = bswap_32(event->auxtrace_error.fmt);
861 event->auxtrace_error.ip = bswap_64(event->auxtrace_error.ip);
862 if (event->auxtrace_error.fmt)
863 event->auxtrace_error.time = bswap_64(event->auxtrace_error.time);
864 if (event->auxtrace_error.fmt >= 2) {
865 /*
866 * fmt >= 2 adds machine_pid and vcpu after msg[64].
867 * Older files may have fmt >= 2 but an event size
868 * that doesn't include these fields — downgrade to
869 * avoid swapping out of bounds.
870 */
871 if (event->header.size < offsetof(typeof(event->auxtrace_error), vcpu) +
872 sizeof(event->auxtrace_error.vcpu)) {
873 pr_warning("WARNING: PERF_RECORD_AUXTRACE_ERROR: fmt %u but event too small for machine_pid/vcpu (%u bytes), downgrading fmt\n",
874 event->auxtrace_error.fmt,
875 event->header.size);
876 event->auxtrace_error.fmt = 1;
877 } else {
878 event->auxtrace_error.machine_pid = bswap_32(event->auxtrace_error.machine_pid);
879 event->auxtrace_error.vcpu = bswap_32(event->auxtrace_error.vcpu);
880 }
881 }
882 return 0;
883}
884
885static int perf_event__thread_map_swap(union perf_event *event,
886 bool sample_id_all __maybe_unused)
887{
888 unsigned int i;
889 u64 nr;
890
891 event->thread_map.nr = bswap_64(event->thread_map.nr);
892
893 /*
894 * Reject rather than clamp: unlike namespaces (indexed by type)
895 * or stat_config (self-describing tags), a truncated thread map
896 * is structurally broken — downstream would get a wrong map.
897 */
898 /* Cannot underflow: perf_event__min_size[] guarantees header.size >= sizeof */
899 nr = event->thread_map.nr;
900 if (nr > (event->header.size - sizeof(event->thread_map)) /
901 sizeof(event->thread_map.entries[0]))
902 return -1;
903
904 for (i = 0; i < nr; i++)
905 event->thread_map.entries[i].pid = bswap_64(event->thread_map.entries[i].pid);
906 return 0;
907}
908
909static int perf_event__cpu_map_swap(union perf_event *event,
910 bool sample_id_all __maybe_unused)
911{
912 struct perf_record_cpu_map_data *data = &event->cpu_map.data;
913 u32 payload = event->header.size - sizeof(event->header);
914
915 data->type = bswap_16(data->type);
916
917 /*
918 * Safe to clamp: a shorter CPU map just means some CPUs
919 * are absent; tools process the CPUs that are present.
920 */
921 switch (data->type) {
922 case PERF_CPU_MAP__CPUS: {
923 u16 nr, max_nr;
924
925 data->cpus_data.nr = bswap_16(data->cpus_data.nr);
926 nr = data->cpus_data.nr;
927 max_nr = (payload - offsetof(struct perf_record_cpu_map_data,
928 cpus_data.cpu)) /
929 sizeof(data->cpus_data.cpu[0]);
930 if (nr > max_nr) {
931 pr_warning("WARNING: PERF_RECORD_CPU_MAP: nr %u exceeds payload (max %u), clamping\n",
932 nr, max_nr);
933 nr = max_nr;
934 data->cpus_data.nr = nr;
935 }
936 for (unsigned int i = 0; i < nr; i++)
937 data->cpus_data.cpu[i] = bswap_16(data->cpus_data.cpu[i]);
938 break;
939 }
940 case PERF_CPU_MAP__MASK:
941 data->mask32_data.long_size = bswap_16(data->mask32_data.long_size);
942
943 switch (data->mask32_data.long_size) {
944 case 4: {
945 u16 nr, max_nr;
946
947 data->mask32_data.nr = bswap_16(data->mask32_data.nr);
948 nr = data->mask32_data.nr;
949 max_nr = (payload - offsetof(struct perf_record_cpu_map_data,
950 mask32_data.mask)) /
951 sizeof(data->mask32_data.mask[0]);
952 if (nr > max_nr) {
953 pr_warning("WARNING: PERF_RECORD_CPU_MAP mask32: nr %u exceeds payload (max %u), clamping\n",
954 nr, max_nr);
955 nr = max_nr;
956 data->mask32_data.nr = nr;
957 }
958 for (unsigned int i = 0; i < nr; i++)
959 data->mask32_data.mask[i] = bswap_32(data->mask32_data.mask[i]);
960 break;
961 }
962 case 8: {
963 u16 nr, max_nr;
964
965 data->mask64_data.nr = bswap_16(data->mask64_data.nr);
966 nr = data->mask64_data.nr;
967 if (payload < offsetof(struct perf_record_cpu_map_data, mask64_data.mask)) {
968 data->mask64_data.nr = 0;
969 break;
970 }
971 max_nr = (payload - offsetof(struct perf_record_cpu_map_data,
972 mask64_data.mask)) /
973 sizeof(data->mask64_data.mask[0]);
974 if (nr > max_nr) {
975 pr_warning("WARNING: PERF_RECORD_CPU_MAP mask64: nr %u exceeds payload (max %u), clamping\n",
976 nr, max_nr);
977 nr = max_nr;
978 data->mask64_data.nr = nr;
979 }
980 for (unsigned int i = 0; i < nr; i++)
981 data->mask64_data.mask[i] = bswap_64(data->mask64_data.mask[i]);
982 break;
983 }
984 default:
985 pr_err("cpu_map swap: unsupported long size %u\n",
986 data->mask32_data.long_size);
987 }
988 break;
989 case PERF_CPU_MAP__RANGE_CPUS:
990 data->range_cpu_data.start_cpu = bswap_16(data->range_cpu_data.start_cpu);
991 data->range_cpu_data.end_cpu = bswap_16(data->range_cpu_data.end_cpu);
992 break;
993 default:
994 break;
995 }
996 return 0;
997}
998
999static int perf_event__stat_config_swap(union perf_event *event,
1000 bool sample_id_all __maybe_unused)
1001{
1002 u64 nr, max_nr, size;
1003
1004 nr = bswap_64(event->stat_config.nr);
1005 /* Cannot underflow: perf_event__min_size[] guarantees header.size >= sizeof */
1006 max_nr = (event->header.size - sizeof(event->stat_config)) /
1007 sizeof(event->stat_config.data[0]);
1008 /*
1009 * Safe to clamp: each config entry is self-describing
1010 * via its tag; missing entries keep their defaults.
1011 */
1012 if (nr > max_nr) {
1013 pr_warning("WARNING: PERF_RECORD_STAT_CONFIG: nr %" PRIu64 " exceeds payload (max %" PRIu64 "), clamping\n",
1014 nr, max_nr);
1015 nr = max_nr;
1016 }
1017 size = nr * sizeof(event->stat_config.data[0]);
1018 /* The swap starts at &nr, so add its size to cover the full range */
1019 size += sizeof(event->stat_config.nr);
1020 mem_bswap_64(&event->stat_config.nr, size);
1021 /* Persist the clamped value in native byte order */
1022 event->stat_config.nr = nr;
1023 return 0;
1024}
1025
1026static int perf_event__stat_swap(union perf_event *event,
1027 bool sample_id_all __maybe_unused)
1028{
1029 event->stat.id = bswap_64(event->stat.id);
1030 event->stat.thread = bswap_32(event->stat.thread);
1031 event->stat.cpu = bswap_32(event->stat.cpu);
1032 event->stat.val = bswap_64(event->stat.val);
1033 event->stat.ena = bswap_64(event->stat.ena);
1034 event->stat.run = bswap_64(event->stat.run);
1035 return 0;
1036}
1037
1038static int perf_event__stat_round_swap(union perf_event *event,
1039 bool sample_id_all __maybe_unused)
1040{
1041 event->stat_round.type = bswap_64(event->stat_round.type);
1042 event->stat_round.time = bswap_64(event->stat_round.time);
1043 return 0;
1044}
1045
1046static int perf_event__time_conv_swap(union perf_event *event,
1047 bool sample_id_all __maybe_unused)
1048{
1049 event->time_conv.time_shift = bswap_64(event->time_conv.time_shift);
1050 event->time_conv.time_mult = bswap_64(event->time_conv.time_mult);
1051 event->time_conv.time_zero = bswap_64(event->time_conv.time_zero);
1052
1053 if (event_contains(event->time_conv, time_cycles))
1054 event->time_conv.time_cycles = bswap_64(event->time_conv.time_cycles);
1055 if (event_contains(event->time_conv, time_mask))
1056 event->time_conv.time_mask = bswap_64(event->time_conv.time_mask);
1057 return 0;
1058}
1059
1060static int perf_event__compressed2_swap(union perf_event *event,
1061 bool sample_id_all __maybe_unused)
1062{
1063 /* Only data_size needs swapping — compressed payload is a raw byte stream */
1064 event->pack2.data_size = bswap_64(event->pack2.data_size);
1065 return 0;
1066}
1067
1068static int perf_event__bpf_metadata_swap(union perf_event *event,
1069 bool sample_id_all __maybe_unused)
1070{
1071 u64 i, nr, max_nr;
1072
1073 /* Fixed header must fit before accessing nr_entries or prog_name */
1074 if (event->header.size < sizeof(event->bpf_metadata))
1075 return -1;
1076
1077 event->bpf_metadata.nr_entries = bswap_64(event->bpf_metadata.nr_entries);
1078
1079 /*
1080 * Ensure NUL-termination on the cross-endian path where the
1081 * mapping is writable (MAP_PRIVATE + PROT_WRITE). Fixing
1082 * the string in place is preferred over rejecting because it
1083 * preserves the event for downstream processing — only the
1084 * last byte is lost.
1085 *
1086 * The native-endian path (MAP_SHARED + PROT_READ) cannot
1087 * write, so it validates and skips unterminated events in
1088 * perf_session__process_user_event() instead. The two
1089 * strategies produce different outcomes for the same
1090 * malformed input (fix vs skip), which is inherent in the
1091 * writable-vs-read-only mapping model.
1092 */
1093 event->bpf_metadata.prog_name[BPF_PROG_NAME_LEN - 1] = '\0';
1094
1095 nr = event->bpf_metadata.nr_entries;
1096 max_nr = (event->header.size - sizeof(event->bpf_metadata)) /
1097 sizeof(event->bpf_metadata.entries[0]);
1098 if (nr > max_nr) {
1099 /* Persist clamped value so the native path processes entries, not skips */
1100 nr = max_nr;
1101 event->bpf_metadata.nr_entries = nr;
1102 }
1103
1104 for (i = 0; i < nr; i++) {
1105 event->bpf_metadata.entries[i].key[BPF_METADATA_KEY_LEN - 1] = '\0';
1106 event->bpf_metadata.entries[i].value[BPF_METADATA_VALUE_LEN - 1] = '\0';
1107 }
1108 return 0;
1109}
1110static int
1111perf_event__schedstat_cpu_swap(union perf_event *event __maybe_unused,
1112 bool sample_id_all __maybe_unused)
1113{
1114 /* FIXME */
1115 return 0;
1116}
1117
1118static int
1119perf_event__schedstat_domain_swap(union perf_event *event __maybe_unused,
1120 bool sample_id_all __maybe_unused)
1121{
1122 /* FIXME */
1123 return 0;
1124}
1125
1126static int perf_event__ksymbol_swap(union perf_event *event,
1127 bool sample_id_all)
1128{
1129 event->ksymbol.addr = bswap_64(event->ksymbol.addr);
1130 event->ksymbol.len = bswap_32(event->ksymbol.len);
1131 event->ksymbol.ksym_type = bswap_16(event->ksymbol.ksym_type);
1132 event->ksymbol.flags = bswap_16(event->ksymbol.flags);
1133
1134 if (sample_id_all) {
1135 void *data = &event->ksymbol.name;
1136 void *end = (void *)event + event->header.size;
1137 size_t len = strnlen(data, end - data);
1138
1139 /* See comment in perf_event__comm_swap() */
1140 if (len == (size_t)(end - data))
1141 return -1;
1142 data += PERF_ALIGN(len + 1, sizeof(u64));
1143 swap_sample_id_all(event, data);
1144 }
1145 return 0;
1146}
1147
1148static int perf_event__bpf_event_swap(union perf_event *event,
1149 bool sample_id_all)
1150{
1151 event->bpf.type = bswap_16(event->bpf.type);
1152 event->bpf.flags = bswap_16(event->bpf.flags);
1153 event->bpf.id = bswap_32(event->bpf.id);
1154
1155 if (sample_id_all)
1156 swap_sample_id_all(event, &event->bpf + 1);
1157 return 0;
1158}
1159
1160static int perf_event__header_feature_swap(union perf_event *event,
1161 bool sample_id_all __maybe_unused)
1162{
1163 event->feat.feat_id = bswap_64(event->feat.feat_id);
1164 return 0;
1165}
1166
1167typedef int (*perf_event__swap_op)(union perf_event *event,
1168 bool sample_id_all);
1169
1170static perf_event__swap_op perf_event__swap_ops[] = {
1171 [PERF_RECORD_MMAP] = perf_event__mmap_swap,
1172 [PERF_RECORD_MMAP2] = perf_event__mmap2_swap,
1173 [PERF_RECORD_COMM] = perf_event__comm_swap,
1174 [PERF_RECORD_FORK] = perf_event__task_swap,
1175 [PERF_RECORD_EXIT] = perf_event__task_swap,
1176 [PERF_RECORD_LOST] = perf_event__all64_swap,
1177 [PERF_RECORD_READ] = perf_event__read_swap,
1178 [PERF_RECORD_THROTTLE] = perf_event__throttle_swap,
1179 [PERF_RECORD_UNTHROTTLE] = perf_event__throttle_swap,
1180 [PERF_RECORD_SAMPLE] = perf_event__all64_swap,
1181 [PERF_RECORD_AUX] = perf_event__aux_swap,
1182 [PERF_RECORD_ITRACE_START] = perf_event__itrace_start_swap,
1183 [PERF_RECORD_LOST_SAMPLES] = perf_event__all64_swap,
1184 [PERF_RECORD_SWITCH] = perf_event__switch_swap,
1185 [PERF_RECORD_SWITCH_CPU_WIDE] = perf_event__switch_swap,
1186 [PERF_RECORD_NAMESPACES] = perf_event__namespaces_swap,
1187 [PERF_RECORD_CGROUP] = perf_event__cgroup_swap,
1188 [PERF_RECORD_KSYMBOL] = perf_event__ksymbol_swap,
1189 [PERF_RECORD_BPF_EVENT] = perf_event__bpf_event_swap,
1190 [PERF_RECORD_TEXT_POKE] = perf_event__text_poke_swap,
1191 [PERF_RECORD_AUX_OUTPUT_HW_ID] = perf_event__all64_swap,
1192 [PERF_RECORD_CALLCHAIN_DEFERRED] = perf_event__all64_swap,
1193 [PERF_RECORD_HEADER_ATTR] = perf_event__hdr_attr_swap,
1194 [PERF_RECORD_HEADER_EVENT_TYPE] = perf_event__event_type_swap,
1195 [PERF_RECORD_HEADER_TRACING_DATA] = perf_event__tracing_data_swap,
1196 [PERF_RECORD_HEADER_BUILD_ID] = perf_event__build_id_swap,
1197 [PERF_RECORD_HEADER_FEATURE] = perf_event__header_feature_swap,
1198 [PERF_RECORD_ID_INDEX] = perf_event__all64_swap,
1199 [PERF_RECORD_AUXTRACE_INFO] = perf_event__auxtrace_info_swap,
1200 [PERF_RECORD_AUXTRACE] = perf_event__auxtrace_swap,
1201 [PERF_RECORD_AUXTRACE_ERROR] = perf_event__auxtrace_error_swap,
1202 [PERF_RECORD_THREAD_MAP] = perf_event__thread_map_swap,
1203 [PERF_RECORD_CPU_MAP] = perf_event__cpu_map_swap,
1204 [PERF_RECORD_STAT_CONFIG] = perf_event__stat_config_swap,
1205 [PERF_RECORD_STAT] = perf_event__stat_swap,
1206 [PERF_RECORD_STAT_ROUND] = perf_event__stat_round_swap,
1207 [PERF_RECORD_EVENT_UPDATE] = perf_event__event_update_swap,
1208 [PERF_RECORD_TIME_CONV] = perf_event__time_conv_swap,
1209 [PERF_RECORD_COMPRESSED2] = perf_event__compressed2_swap,
1210 [PERF_RECORD_BPF_METADATA] = perf_event__bpf_metadata_swap,
1211 [PERF_RECORD_SCHEDSTAT_CPU] = perf_event__schedstat_cpu_swap,
1212 [PERF_RECORD_SCHEDSTAT_DOMAIN] = perf_event__schedstat_domain_swap,
1213 [PERF_RECORD_HEADER_MAX] = NULL,
1214};
1215
1216/*
1217 * When perf record finishes a pass on every buffers, it records this pseudo
1218 * event.
1219 * We record the max timestamp t found in the pass n.
1220 * Assuming these timestamps are monotonic across cpus, we know that if
1221 * a buffer still has events with timestamps below t, they will be all
1222 * available and then read in the pass n + 1.
1223 * Hence when we start to read the pass n + 2, we can safely flush every
1224 * events with timestamps below t.
1225 *
1226 * ============ PASS n =================
1227 * CPU 0 | CPU 1
1228 * |
1229 * cnt1 timestamps | cnt2 timestamps
1230 * 1 | 2
1231 * 2 | 3
1232 * - | 4 <--- max recorded
1233 *
1234 * ============ PASS n + 1 ==============
1235 * CPU 0 | CPU 1
1236 * |
1237 * cnt1 timestamps | cnt2 timestamps
1238 * 3 | 5
1239 * 4 | 6
1240 * 5 | 7 <---- max recorded
1241 *
1242 * Flush every events below timestamp 4
1243 *
1244 * ============ PASS n + 2 ==============
1245 * CPU 0 | CPU 1
1246 * |
1247 * cnt1 timestamps | cnt2 timestamps
1248 * 6 | 8
1249 * 7 | 9
1250 * - | 10
1251 *
1252 * Flush every events below timestamp 7
1253 * etc...
1254 */
1255int perf_event__process_finished_round(const struct perf_tool *tool __maybe_unused,
1256 union perf_event *event __maybe_unused,
1257 struct ordered_events *oe)
1258{
1259 if (dump_trace)
1260 fprintf(stdout, "\n");
1261 return ordered_events__flush(oe, OE_FLUSH__ROUND);
1262}
1263
1264int perf_session__queue_event(struct perf_session *s, union perf_event *event,
1265 u64 timestamp, u64 file_offset, const char *file_path)
1266{
1267 return ordered_events__queue(&s->ordered_events, event, timestamp, file_offset, file_path);
1268}
1269
1270static void callchain__lbr_callstack_printf(struct perf_sample *sample)
1271{
1272 struct ip_callchain *callchain = sample->callchain;
1273 struct branch_stack *lbr_stack = sample->branch_stack;
1274 struct branch_entry *entries = perf_sample__branch_entries(sample);
1275 u64 kernel_callchain_nr = callchain->nr;
1276 unsigned int i;
1277
1278 for (i = 0; i < kernel_callchain_nr; i++) {
1279 if (callchain->ips[i] == PERF_CONTEXT_USER)
1280 break;
1281 }
1282
1283 if ((i != kernel_callchain_nr) && lbr_stack->nr) {
1284 u64 total_nr;
1285 /*
1286 * LBR callstack can only get user call chain,
1287 * i is kernel call chain number,
1288 * 1 is PERF_CONTEXT_USER.
1289 *
1290 * The user call chain is stored in LBR registers.
1291 * LBR are pair registers. The caller is stored
1292 * in "from" register, while the callee is stored
1293 * in "to" register.
1294 * For example, there is a call stack
1295 * "A"->"B"->"C"->"D".
1296 * The LBR registers will be recorded like
1297 * "C"->"D", "B"->"C", "A"->"B".
1298 * So only the first "to" register and all "from"
1299 * registers are needed to construct the whole stack.
1300 */
1301 total_nr = i + 1 + lbr_stack->nr + 1;
1302 kernel_callchain_nr = i + 1;
1303
1304 printf("... LBR call chain: nr:%" PRIu64 "\n", total_nr);
1305
1306 for (i = 0; i < kernel_callchain_nr; i++)
1307 printf("..... %2d: %016" PRIx64 "\n",
1308 i, callchain->ips[i]);
1309
1310 printf("..... %2d: %016" PRIx64 "\n",
1311 (int)(kernel_callchain_nr), entries[0].to);
1312 for (i = 0; i < lbr_stack->nr; i++)
1313 printf("..... %2d: %016" PRIx64 "\n",
1314 (int)(i + kernel_callchain_nr + 1), entries[i].from);
1315 }
1316}
1317
1318static const char *callchain_context_str(u64 ip)
1319{
1320 switch (ip) {
1321 case PERF_CONTEXT_HV:
1322 return " (PERF_CONTEXT_HV)";
1323 case PERF_CONTEXT_KERNEL:
1324 return " (PERF_CONTEXT_KERNEL)";
1325 case PERF_CONTEXT_USER:
1326 return " (PERF_CONTEXT_USER)";
1327 case PERF_CONTEXT_GUEST:
1328 return " (PERF_CONTEXT_GUEST)";
1329 case PERF_CONTEXT_GUEST_KERNEL:
1330 return " (PERF_CONTEXT_GUEST_KERNEL)";
1331 case PERF_CONTEXT_GUEST_USER:
1332 return " (PERF_CONTEXT_GUEST_USER)";
1333 case PERF_CONTEXT_USER_DEFERRED:
1334 return " (PERF_CONTEXT_USER_DEFERRED)";
1335 default:
1336 return "";
1337 }
1338}
1339
1340static void callchain__printf(struct evsel *evsel,
1341 struct perf_sample *sample)
1342{
1343 unsigned int i;
1344 struct ip_callchain *callchain = sample->callchain;
1345
1346 if (evsel__has_branch_callstack(evsel))
1347 callchain__lbr_callstack_printf(sample);
1348
1349 printf("... FP chain: nr:%" PRIu64 "\n", callchain->nr);
1350
1351 for (i = 0; i < callchain->nr; i++)
1352 printf("..... %2d: %016" PRIx64 "%s\n",
1353 i, callchain->ips[i],
1354 callchain_context_str(callchain->ips[i]));
1355
1356 if (sample->deferred_callchain)
1357 printf("...... (deferred)\n");
1358}
1359
1360static void branch_stack__printf(struct perf_sample *sample,
1361 struct evsel *evsel)
1362{
1363 struct branch_entry *entries = perf_sample__branch_entries(sample);
1364 bool callstack = evsel__has_branch_callstack(evsel);
1365 u64 *branch_stack_cntr = sample->branch_stack_cntr;
1366 uint64_t i;
1367
1368 if (!callstack) {
1369 printf("%s: nr:%" PRIu64 "\n", "... branch stack", sample->branch_stack->nr);
1370 } else {
1371 /* the reason of adding 1 to nr is because after expanding
1372 * branch stack it generates nr + 1 callstack records. e.g.,
1373 * B()->C()
1374 * A()->B()
1375 * the final callstack should be:
1376 * C()
1377 * B()
1378 * A()
1379 */
1380 printf("%s: nr:%" PRIu64 "\n", "... branch callstack", sample->branch_stack->nr+1);
1381 }
1382
1383 for (i = 0; i < sample->branch_stack->nr; i++) {
1384 struct branch_entry *e = &entries[i];
1385
1386 if (!callstack) {
1387 printf("..... %2"PRIu64": %016" PRIx64 " -> %016" PRIx64 " %hu cycles %s%s%s%s %x %s %s\n",
1388 i, e->from, e->to,
1389 (unsigned short)e->flags.cycles,
1390 e->flags.mispred ? "M" : " ",
1391 e->flags.predicted ? "P" : " ",
1392 e->flags.abort ? "A" : " ",
1393 e->flags.in_tx ? "T" : " ",
1394 (unsigned)e->flags.reserved,
1395 get_branch_type(e),
1396 e->flags.spec ? branch_spec_desc(e->flags.spec) : "");
1397 } else {
1398 if (i == 0) {
1399 printf("..... %2"PRIu64": %016" PRIx64 "\n"
1400 "..... %2"PRIu64": %016" PRIx64 "\n",
1401 i, e->to, i+1, e->from);
1402 } else {
1403 printf("..... %2"PRIu64": %016" PRIx64 "\n", i+1, e->from);
1404 }
1405 }
1406 }
1407
1408 if (branch_stack_cntr) {
1409 unsigned int br_cntr_width, br_cntr_nr;
1410
1411 perf_env__find_br_cntr_info(evsel__env(evsel), &br_cntr_nr, &br_cntr_width);
1412 printf("... branch stack counters: nr:%" PRIu64 " (counter width: %u max counter nr:%u)\n",
1413 sample->branch_stack->nr, br_cntr_width, br_cntr_nr);
1414 for (i = 0; i < sample->branch_stack->nr; i++)
1415 printf("..... %2"PRIu64": %016" PRIx64 "\n", i, branch_stack_cntr[i]);
1416 }
1417}
1418
1419static void regs_dump__printf(u64 mask, u64 *regs, uint16_t e_machine, uint32_t e_flags)
1420{
1421 unsigned rid, i = 0;
1422
1423 for_each_set_bit(rid, (unsigned long *) &mask, sizeof(mask) * 8) {
1424 u64 val = regs[i++];
1425
1426 printf(".... %-5s 0x%016" PRIx64 "\n",
1427 perf_reg_name(rid, e_machine, e_flags), val);
1428 }
1429}
1430
1431static const char *regs_abi[] = {
1432 [PERF_SAMPLE_REGS_ABI_NONE] = "none",
1433 [PERF_SAMPLE_REGS_ABI_32] = "32-bit",
1434 [PERF_SAMPLE_REGS_ABI_64] = "64-bit",
1435};
1436
1437static inline const char *regs_dump_abi(struct regs_dump *d)
1438{
1439 if (d->abi > PERF_SAMPLE_REGS_ABI_64)
1440 return "unknown";
1441
1442 return regs_abi[d->abi];
1443}
1444
1445static void regs__printf(const char *type, struct regs_dump *regs,
1446 uint16_t e_machine, uint32_t e_flags)
1447{
1448 u64 mask = regs->mask;
1449
1450 printf("... %s regs: mask 0x%" PRIx64 " ABI %s\n",
1451 type,
1452 mask,
1453 regs_dump_abi(regs));
1454
1455 regs_dump__printf(mask, regs->regs, e_machine, e_flags);
1456}
1457
1458static void regs_user__printf(struct perf_sample *sample, uint16_t e_machine, uint32_t e_flags)
1459{
1460 struct regs_dump *user_regs;
1461
1462 if (!sample->user_regs)
1463 return;
1464
1465 user_regs = perf_sample__user_regs(sample);
1466
1467 if (user_regs->regs)
1468 regs__printf("user", user_regs, e_machine, e_flags);
1469}
1470
1471static void regs_intr__printf(struct perf_sample *sample, uint16_t e_machine, uint32_t e_flags)
1472{
1473 struct regs_dump *intr_regs;
1474
1475 if (!sample->intr_regs)
1476 return;
1477
1478 intr_regs = perf_sample__intr_regs(sample);
1479
1480 if (intr_regs->regs)
1481 regs__printf("intr", intr_regs, e_machine, e_flags);
1482}
1483
1484static void stack_user__printf(struct stack_dump *dump)
1485{
1486 printf("... ustack: size %" PRIu64 ", offset 0x%x\n",
1487 dump->size, dump->offset);
1488}
1489
1490static void evlist__print_tstamp(struct evlist *evlist, union perf_event *event, struct perf_sample *sample)
1491{
1492 u64 sample_type = __evlist__combined_sample_type(evlist);
1493
1494 if (event->header.type != PERF_RECORD_SAMPLE &&
1495 !evlist__sample_id_all(evlist)) {
1496 fputs("-1 -1 ", stdout);
1497 return;
1498 }
1499
1500 if ((sample_type & PERF_SAMPLE_CPU))
1501 printf("%u ", sample->cpu);
1502
1503 if (sample_type & PERF_SAMPLE_TIME)
1504 printf("%" PRIu64 " ", sample->time);
1505}
1506
1507static void sample_read__printf(struct perf_sample *sample, u64 read_format)
1508{
1509 printf("... sample_read:\n");
1510
1511 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
1512 printf("...... time enabled %016" PRIx64 "\n",
1513 sample->read.time_enabled);
1514
1515 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
1516 printf("...... time running %016" PRIx64 "\n",
1517 sample->read.time_running);
1518
1519 if (read_format & PERF_FORMAT_GROUP) {
1520 struct sample_read_value *value = sample->read.group.values;
1521
1522 printf(".... group nr %" PRIu64 "\n", sample->read.group.nr);
1523
1524 sample_read_group__for_each(value, sample->read.group.nr, read_format) {
1525 printf("..... id %016" PRIx64
1526 ", value %016" PRIx64,
1527 value->id, value->value);
1528 if (read_format & PERF_FORMAT_LOST)
1529 printf(", lost %" PRIu64, value->lost);
1530 printf("\n");
1531 }
1532 } else {
1533 printf("..... id %016" PRIx64 ", value %016" PRIx64,
1534 sample->read.one.id, sample->read.one.value);
1535 if (read_format & PERF_FORMAT_LOST)
1536 printf(", lost %" PRIu64, sample->read.one.lost);
1537 printf("\n");
1538 }
1539}
1540
1541static void dump_event(struct evlist *evlist, union perf_event *event,
1542 u64 file_offset, struct perf_sample *sample,
1543 const char *file_path)
1544{
1545 if (!dump_trace)
1546 return;
1547
1548 printf("\n%#" PRIx64 "@%s [%#x]: event: %d\n",
1549 file_offset, file_path, event->header.size, event->header.type);
1550
1551 trace_event(event);
1552 if (event->header.type == PERF_RECORD_SAMPLE && evlist__trace_event_sample_raw(evlist))
1553 evlist__trace_event_sample_raw(evlist)(evlist, event, sample);
1554
1555 if (sample)
1556 evlist__print_tstamp(evlist, event, sample);
1557
1558 printf("%#" PRIx64 " [%#x]: PERF_RECORD_%s", file_offset,
1559 event->header.size, perf_event__name(event->header.type));
1560}
1561
1562char *get_page_size_name(u64 size, char *str)
1563{
1564 if (!size || !unit_number__scnprintf(str, PAGE_SIZE_NAME_LEN, size))
1565 snprintf(str, PAGE_SIZE_NAME_LEN, "%s", "N/A");
1566
1567 return str;
1568}
1569
1570static void dump_sample(struct machine *machine, union perf_event *event,
1571 struct perf_sample *sample)
1572{
1573 struct evsel *evsel = sample->evsel;
1574 u64 sample_type;
1575 char str[PAGE_SIZE_NAME_LEN];
1576 uint16_t e_machine = EM_NONE;
1577 uint32_t e_flags = 0;
1578
1579 if (!dump_trace)
1580 return;
1581
1582 sample_type = evsel->core.attr.sample_type;
1583
1584 if (sample_type & (PERF_SAMPLE_REGS_USER | PERF_SAMPLE_REGS_INTR)) {
1585 struct thread *thread = machine__find_thread(machine, sample->pid, sample->pid);
1586
1587 e_machine = thread__e_machine(thread, machine, &e_flags);
1588 }
1589
1590 printf("(IP, 0x%x): %d/%d: %#" PRIx64 " period: %" PRIu64 " addr: %#" PRIx64 "\n",
1591 event->header.misc, sample->pid, sample->tid, sample->ip,
1592 sample->period, sample->addr);
1593
1594 if (evsel__has_callchain(evsel))
1595 callchain__printf(evsel, sample);
1596
1597 if (evsel__has_br_stack(evsel))
1598 branch_stack__printf(sample, evsel);
1599
1600 if (sample_type & PERF_SAMPLE_REGS_USER)
1601 regs_user__printf(sample, e_machine, e_flags);
1602
1603 if (sample_type & PERF_SAMPLE_REGS_INTR)
1604 regs_intr__printf(sample, e_machine, e_flags);
1605
1606 if (sample_type & PERF_SAMPLE_STACK_USER)
1607 stack_user__printf(&sample->user_stack);
1608
1609 if (sample_type & PERF_SAMPLE_WEIGHT_TYPE) {
1610 printf("... weight: %" PRIu64 "", sample->weight);
1611 if (sample_type & PERF_SAMPLE_WEIGHT_STRUCT) {
1612 printf(",0x%"PRIx16"", sample->ins_lat);
1613 printf(",0x%"PRIx16"", sample->weight3);
1614 }
1615 printf("\n");
1616 }
1617
1618 if (sample_type & PERF_SAMPLE_DATA_SRC)
1619 printf(" . data_src: 0x%"PRIx64"\n", sample->data_src);
1620
1621 if (sample_type & PERF_SAMPLE_PHYS_ADDR)
1622 printf(" .. phys_addr: 0x%"PRIx64"\n", sample->phys_addr);
1623
1624 if (sample_type & PERF_SAMPLE_DATA_PAGE_SIZE)
1625 printf(" .. data page size: %s\n", get_page_size_name(sample->data_page_size, str));
1626
1627 if (sample_type & PERF_SAMPLE_CODE_PAGE_SIZE)
1628 printf(" .. code page size: %s\n", get_page_size_name(sample->code_page_size, str));
1629
1630 if (sample_type & PERF_SAMPLE_TRANSACTION)
1631 printf("... transaction: %" PRIx64 "\n", sample->transaction);
1632
1633 if (sample_type & PERF_SAMPLE_READ)
1634 sample_read__printf(sample, evsel->core.attr.read_format);
1635}
1636
1637static void dump_deferred_callchain(union perf_event *event, struct perf_sample *sample)
1638{
1639 struct evsel *evsel = sample->evsel;
1640
1641 if (!dump_trace)
1642 return;
1643
1644 printf("(IP, 0x%x): %d/%d: %#" PRIx64 "\n",
1645 event->header.misc, sample->pid, sample->tid, sample->deferred_cookie);
1646
1647 if (evsel__has_callchain(evsel))
1648 callchain__printf(evsel, sample);
1649}
1650
1651static void dump_read(struct evsel *evsel, union perf_event *event)
1652{
1653 u64 read_format;
1654 __u64 *array;
1655 void *end;
1656
1657 if (!dump_trace)
1658 return;
1659
1660 printf(": %d %d %s %" PRI_lu64 "\n", event->read.pid, event->read.tid,
1661 evsel__name(evsel), event->read.value);
1662
1663 if (!evsel)
1664 return;
1665
1666 read_format = evsel->core.attr.read_format;
1667 /*
1668 * The kernel packs only the enabled read_format fields
1669 * after value, with no gaps. Walk the packed array
1670 * instead of using fixed struct offsets.
1671 */
1672 array = &event->read.value + 1;
1673 end = (void *)event + event->header.size;
1674
1675 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
1676 if ((void *)(array + 1) > end)
1677 return;
1678 printf("... time enabled : %" PRI_lu64 "\n", *array++);
1679 }
1680
1681 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
1682 if ((void *)(array + 1) > end)
1683 return;
1684 printf("... time running : %" PRI_lu64 "\n", *array++);
1685 }
1686
1687 if (read_format & PERF_FORMAT_ID) {
1688 if ((void *)(array + 1) > end)
1689 return;
1690 printf("... id : %" PRI_lu64 "\n", *array++);
1691 }
1692
1693 if (read_format & PERF_FORMAT_LOST) {
1694 if ((void *)(array + 1) > end)
1695 return;
1696 printf("... lost : %" PRI_lu64 "\n", *array++);
1697 }
1698}
1699
1700static struct machine *machines__find_for_cpumode(struct machines *machines,
1701 union perf_event *event,
1702 struct perf_sample *sample)
1703{
1704 if (perf_guest &&
1705 ((sample->cpumode == PERF_RECORD_MISC_GUEST_KERNEL) ||
1706 (sample->cpumode == PERF_RECORD_MISC_GUEST_USER))) {
1707 u32 pid;
1708
1709 if (sample->machine_pid)
1710 pid = sample->machine_pid;
1711 else if (event->header.type == PERF_RECORD_MMAP
1712 || event->header.type == PERF_RECORD_MMAP2)
1713 pid = event->mmap.pid;
1714 else
1715 pid = sample->pid;
1716
1717 /*
1718 * Guest code machine is created as needed and does not use
1719 * DEFAULT_GUEST_KERNEL_ID.
1720 */
1721 if (symbol_conf.guest_code)
1722 return machines__findnew(machines, pid);
1723
1724 return machines__find_guest(machines, pid);
1725 }
1726
1727 return &machines->host;
1728}
1729
1730static int deliver_sample_value(struct evlist *evlist,
1731 const struct perf_tool *tool,
1732 union perf_event *event,
1733 struct perf_sample *sample,
1734 struct sample_read_value *v,
1735 struct machine *machine,
1736 bool per_thread)
1737{
1738 struct perf_sample_id *sid = evlist__id2sid(evlist, v->id);
1739 struct evsel *saved_evsel = sample->evsel;
1740 u64 *storage = NULL;
1741 int ret;
1742
1743 if (sid) {
1744 storage = perf_sample_id__get_period_storage(sid, sample->tid, per_thread);
1745 }
1746
1747 if (storage) {
1748 sample->id = v->id;
1749 sample->period = v->value - *storage;
1750 *storage = v->value;
1751 }
1752
1753 if (!storage || sid->evsel == NULL) {
1754 ++evlist__stats(evlist)->nr_unknown_id;
1755 return 0;
1756 }
1757
1758 /*
1759 * There's no reason to deliver sample
1760 * for zero period, bail out.
1761 */
1762 if (!sample->period)
1763 return 0;
1764
1765 sample->evsel = container_of(sid->evsel, struct evsel, core);
1766 ret = tool->sample(tool, event, sample, machine);
1767 sample->evsel = saved_evsel;
1768 return ret;
1769}
1770
1771static int deliver_sample_group(struct evlist *evlist,
1772 const struct perf_tool *tool,
1773 union perf_event *event,
1774 struct perf_sample *sample,
1775 struct machine *machine,
1776 u64 read_format,
1777 bool per_thread)
1778{
1779 int ret = -EINVAL;
1780 struct sample_read_value *v = sample->read.group.values;
1781
1782 if (tool->dont_split_sample_group)
1783 return deliver_sample_value(evlist, tool, event, sample, v, machine,
1784 per_thread);
1785
1786 sample_read_group__for_each(v, sample->read.group.nr, read_format) {
1787 ret = deliver_sample_value(evlist, tool, event, sample, v,
1788 machine, per_thread);
1789 if (ret)
1790 break;
1791 }
1792
1793 return ret;
1794}
1795
1796static int evlist__deliver_sample(struct evlist *evlist, const struct perf_tool *tool,
1797 union perf_event *event, struct perf_sample *sample,
1798 struct machine *machine)
1799{
1800 struct evsel *evsel = sample->evsel;
1801 /* We know evsel != NULL. */
1802 u64 sample_type = evsel->core.attr.sample_type;
1803 u64 read_format = evsel->core.attr.read_format;
1804 bool per_thread = perf_evsel__attr_has_per_thread_sample_period(&evsel->core);
1805
1806 /* Standard sample delivery. */
1807 if (!(sample_type & PERF_SAMPLE_READ))
1808 return tool->sample(tool, event, sample, machine);
1809
1810 /* For PERF_SAMPLE_READ we have either single or group mode. */
1811 if (read_format & PERF_FORMAT_GROUP)
1812 return deliver_sample_group(evlist, tool, event, sample,
1813 machine, read_format, per_thread);
1814 else
1815 return deliver_sample_value(evlist, tool, event, sample,
1816 &sample->read.one, machine,
1817 per_thread);
1818}
1819
1820/*
1821 * Samples with deferred callchains should wait for the next matching
1822 * PERF_RECORD_CALLCHAIN_RECORD entries. Keep the events in a list and
1823 * deliver them once it finds the callchains.
1824 */
1825struct deferred_event {
1826 struct list_head list;
1827 union perf_event *event;
1828 u64 file_offset;
1829};
1830
1831/*
1832 * This is called when a deferred callchain record comes up. Find all matching
1833 * samples, merge the callchains and process them.
1834 */
1835static int evlist__deliver_deferred_callchain(struct evlist *evlist,
1836 const struct perf_tool *tool,
1837 union perf_event *event,
1838 struct perf_sample *sample,
1839 struct machine *machine)
1840{
1841 struct deferred_event *de, *tmp;
1842 int ret = 0;
1843
1844 if (!tool->merge_deferred_callchains) {
1845 struct evsel *saved_evsel = sample->evsel;
1846
1847 sample->evsel = evlist__id2evsel(evlist, sample->id);
1848 if (sample->evsel)
1849 sample->evsel = evsel__get(sample->evsel);
1850 ret = tool->callchain_deferred(tool, event, sample, machine);
1851 evsel__put(sample->evsel);
1852 sample->evsel = saved_evsel;
1853 return ret;
1854 }
1855
1856 list_for_each_entry_safe(de, tmp, evlist__deferred_samples(evlist), list) {
1857 struct perf_sample orig_sample;
1858 struct evsel *new_evsel;
1859
1860 perf_sample__init(&orig_sample, /*all=*/false);
1861 ret = evlist__parse_sample(evlist, de->event, &orig_sample);
1862 if (ret < 0) {
1863 pr_err("failed to parse original sample\n");
1864 perf_sample__exit(&orig_sample);
1865 break;
1866 }
1867 orig_sample.file_offset = de->file_offset;
1868
1869 if (sample->tid != orig_sample.tid) {
1870 perf_sample__exit(&orig_sample);
1871 continue;
1872 }
1873
1874 if (event->callchain_deferred.cookie == orig_sample.deferred_cookie)
1875 sample__merge_deferred_callchain(&orig_sample, sample);
1876 else
1877 orig_sample.deferred_callchain = false;
1878
1879 new_evsel = evlist__id2evsel(evlist, orig_sample.id);
1880 if (new_evsel != orig_sample.evsel) {
1881 evsel__put(orig_sample.evsel);
1882 orig_sample.evsel = evsel__get(new_evsel);
1883 }
1884 ret = evlist__deliver_sample(evlist, tool, de->event,
1885 &orig_sample, machine);
1886
1887 perf_sample__exit(&orig_sample);
1888 list_del(&de->list);
1889 free(de->event);
1890 free(de);
1891
1892 if (ret)
1893 break;
1894 }
1895 return ret;
1896}
1897
1898/*
1899 * This is called at the end of the data processing for the session. Flush the
1900 * remaining samples as there's no hope for matching deferred callchains.
1901 */
1902static int session__flush_deferred_samples(struct perf_session *session,
1903 const struct perf_tool *tool)
1904{
1905 struct evlist *evlist = session->evlist;
1906 struct machine *machine = &session->machines.host;
1907 struct deferred_event *de, *tmp;
1908 int ret = 0;
1909
1910 list_for_each_entry_safe(de, tmp, evlist__deferred_samples(evlist), list) {
1911 struct perf_sample sample;
1912 struct evsel *new_evsel;
1913
1914 perf_sample__init(&sample, /*all=*/false);
1915 ret = evlist__parse_sample(evlist, de->event, &sample);
1916 if (ret < 0) {
1917 pr_err("failed to parse original sample\n");
1918 perf_sample__exit(&sample);
1919 break;
1920 }
1921 sample.file_offset = de->file_offset;
1922
1923 new_evsel = evlist__id2evsel(evlist, sample.id);
1924 if (new_evsel != sample.evsel) {
1925 evsel__put(sample.evsel);
1926 sample.evsel = evsel__get(new_evsel);
1927 }
1928 ret = evlist__deliver_sample(evlist, tool, de->event,
1929 &sample, machine);
1930
1931 perf_sample__exit(&sample);
1932 list_del(&de->list);
1933 free(de->event);
1934 free(de);
1935
1936 if (ret)
1937 break;
1938 }
1939 return ret;
1940}
1941
1942/*
1943 * Return true if the string field is properly null-terminated
1944 * within the event boundary. Native-endian files are mapped
1945 * read-only (MAP_SHARED + PROT_READ) so we cannot write a
1946 * null byte in place; skip the event instead.
1947 */
1948static bool perf_event__check_nul(const char *str, const void *end,
1949 const char *event_name, u64 file_offset)
1950{
1951 size_t max_len = (const char *)end - str;
1952
1953 if (max_len == 0 || strnlen(str, max_len) == max_len) {
1954 pr_warning("WARNING: at offset %#" PRIx64 ": PERF_RECORD_%s: string not null-terminated, skipping event\n",
1955 file_offset, event_name);
1956 return false;
1957 }
1958
1959 return true;
1960}
1961
1962static int machines__deliver_event(struct machines *machines,
1963 struct evlist *evlist,
1964 union perf_event *event,
1965 struct perf_sample *sample,
1966 const struct perf_tool *tool, u64 file_offset,
1967 const char *file_path)
1968{
1969 struct machine *machine;
1970
1971 dump_event(evlist, event, file_offset, sample, file_path);
1972
1973 if (!sample->evsel) {
1974 sample->evsel = evlist__id2evsel(evlist, sample->id);
1975 if (sample->evsel)
1976 sample->evsel = evsel__get(sample->evsel);
1977 }
1978 else
1979 assert(sample->evsel == evlist__id2evsel(evlist, sample->id));
1980 machine = machines__find_for_cpumode(machines, event, sample);
1981
1982 switch (event->header.type) {
1983 case PERF_RECORD_SAMPLE:
1984 if (sample->evsel == NULL) {
1985 ++evlist__stats(evlist)->nr_unknown_id;
1986 return 0;
1987 }
1988 if (machine == NULL) {
1989 ++evlist__stats(evlist)->nr_unprocessable_samples;
1990 dump_sample(machine, event, sample);
1991 return 0;
1992 }
1993 dump_sample(machine, event, sample);
1994 if (sample->deferred_callchain && tool->merge_deferred_callchains) {
1995 struct deferred_event *de = malloc(sizeof(*de));
1996 size_t sz = event->header.size;
1997
1998 if (de == NULL)
1999 return -ENOMEM;
2000
2001 de->event = malloc(sz);
2002 if (de->event == NULL) {
2003 free(de);
2004 return -ENOMEM;
2005 }
2006 memcpy(de->event, event, sz);
2007 de->file_offset = sample->file_offset;
2008 list_add_tail(&de->list, evlist__deferred_samples(evlist));
2009 return 0;
2010 }
2011 return evlist__deliver_sample(evlist, tool, event, sample, machine);
2012 case PERF_RECORD_MMAP:
2013 if (!perf_event__check_nul(event->mmap.filename,
2014 (void *)event + event->header.size,
2015 "MMAP", file_offset))
2016 return 0;
2017 return tool->mmap(tool, event, sample, machine);
2018 case PERF_RECORD_MMAP2:
2019 if (event->header.misc & PERF_RECORD_MISC_PROC_MAP_PARSE_TIMEOUT)
2020 ++evlist__stats(evlist)->nr_proc_map_timeout;
2021 if (!perf_event__check_nul(event->mmap2.filename,
2022 (void *)event + event->header.size,
2023 "MMAP2", file_offset))
2024 return 0;
2025 return tool->mmap2(tool, event, sample, machine);
2026 case PERF_RECORD_COMM:
2027 if (!perf_event__check_nul(event->comm.comm,
2028 (void *)event + event->header.size,
2029 "COMM", file_offset))
2030 return 0;
2031 return tool->comm(tool, event, sample, machine);
2032 case PERF_RECORD_NAMESPACES: {
2033 /*
2034 * Cannot underflow: perf_event__min_size[] guarantees header.size >= sizeof.
2035 * Includes trailing sample_id space when present, but prevents OOB.
2036 */
2037 u64 max_nr = (event->header.size - sizeof(event->namespaces)) /
2038 sizeof(event->namespaces.link_info[0]);
2039
2040 /*
2041 * Native-endian events are mmap'd read-only, so we
2042 * cannot clamp nr in place. Skip the event instead.
2043 * The swap handler already clamps on the writable
2044 * cross-endian path.
2045 */
2046 if (event->namespaces.nr_namespaces > max_nr) {
2047 pr_warning("WARNING: at offset %#" PRIx64 ": PERF_RECORD_NAMESPACES: nr_namespaces %" PRIu64 " exceeds payload (max %" PRIu64 "), skipping\n",
2048 file_offset, (u64)event->namespaces.nr_namespaces, max_nr);
2049 return 0;
2050 }
2051 return tool->namespaces(tool, event, sample, machine);
2052 }
2053 case PERF_RECORD_CGROUP:
2054 if (!perf_event__check_nul(event->cgroup.path,
2055 (void *)event + event->header.size,
2056 "CGROUP", file_offset))
2057 return 0;
2058 return tool->cgroup(tool, event, sample, machine);
2059 case PERF_RECORD_FORK:
2060 return tool->fork(tool, event, sample, machine);
2061 case PERF_RECORD_EXIT:
2062 return tool->exit(tool, event, sample, machine);
2063 case PERF_RECORD_LOST:
2064 if (tool->lost == perf_event__process_lost)
2065 evlist__stats(evlist)->total_lost += event->lost.lost;
2066 return tool->lost(tool, event, sample, machine);
2067 case PERF_RECORD_LOST_SAMPLES:
2068 if (event->header.misc & PERF_RECORD_MISC_LOST_SAMPLES_BPF)
2069 evlist__stats(evlist)->total_dropped_samples += event->lost_samples.lost;
2070 else if (tool->lost_samples == perf_event__process_lost_samples)
2071 evlist__stats(evlist)->total_lost_samples += event->lost_samples.lost;
2072 return tool->lost_samples(tool, event, sample, machine);
2073 case PERF_RECORD_READ:
2074 dump_read(sample->evsel, event);
2075 return tool->read(tool, event, sample, machine);
2076 case PERF_RECORD_THROTTLE:
2077 return tool->throttle(tool, event, sample, machine);
2078 case PERF_RECORD_UNTHROTTLE:
2079 return tool->unthrottle(tool, event, sample, machine);
2080 case PERF_RECORD_AUX:
2081 if (tool->aux == perf_event__process_aux) {
2082 if (event->aux.flags & PERF_AUX_FLAG_TRUNCATED)
2083 evlist__stats(evlist)->total_aux_lost += 1;
2084 if (event->aux.flags & PERF_AUX_FLAG_PARTIAL)
2085 evlist__stats(evlist)->total_aux_partial += 1;
2086 if (event->aux.flags & PERF_AUX_FLAG_COLLISION)
2087 evlist__stats(evlist)->total_aux_collision += 1;
2088 }
2089 return tool->aux(tool, event, sample, machine);
2090 case PERF_RECORD_ITRACE_START:
2091 return tool->itrace_start(tool, event, sample, machine);
2092 case PERF_RECORD_SWITCH:
2093 case PERF_RECORD_SWITCH_CPU_WIDE:
2094 return tool->context_switch(tool, event, sample, machine);
2095 case PERF_RECORD_KSYMBOL:
2096 if (!perf_event__check_nul(event->ksymbol.name,
2097 (void *)event + event->header.size,
2098 "KSYMBOL", file_offset))
2099 return 0;
2100 return tool->ksymbol(tool, event, sample, machine);
2101 case PERF_RECORD_BPF_EVENT:
2102 return tool->bpf(tool, event, sample, machine);
2103 case PERF_RECORD_TEXT_POKE: {
2104 /* offsetof(bytes), not sizeof — sizeof includes padding past the flexible array */
2105 size_t text_poke_len = offsetof(struct perf_record_text_poke_event, bytes) +
2106 event->text_poke.old_len +
2107 event->text_poke.new_len;
2108
2109 if (event->header.size < text_poke_len) {
2110 pr_warning("WARNING: at offset %#" PRIx64 ": PERF_RECORD_TEXT_POKE: old_len+new_len exceeds event, skipping\n",
2111 file_offset);
2112 return 0;
2113 }
2114 return tool->text_poke(tool, event, sample, machine);
2115 }
2116 case PERF_RECORD_AUX_OUTPUT_HW_ID:
2117 return tool->aux_output_hw_id(tool, event, sample, machine);
2118 case PERF_RECORD_CALLCHAIN_DEFERRED:
2119 dump_deferred_callchain(event, sample);
2120 return evlist__deliver_deferred_callchain(evlist, tool, event,
2121 sample, machine);
2122 default:
2123 ++evlist__stats(evlist)->nr_unknown_events;
2124 return -1;
2125 }
2126}
2127
2128static int perf_session__deliver_event(struct perf_session *session,
2129 union perf_event *event,
2130 const struct perf_tool *tool,
2131 u64 file_offset,
2132 const char *file_path)
2133{
2134 struct perf_sample sample;
2135 struct evsel *evsel;
2136 int ret;
2137
2138 perf_sample__init(&sample, /*all=*/false);
2139 evsel = evlist__event2evsel(session->evlist, event);
2140 if (!evsel) {
2141 pr_err("ERROR: at offset %#" PRIx64 ": no evsel found for %s (%u) event\n",
2142 file_offset, perf_event__name(event->header.type),
2143 event->header.type);
2144 ret = -EFAULT;
2145 goto out;
2146 }
2147 ret = evsel__parse_sample(evsel, event, &sample);
2148 if (ret) {
2149 pr_err("ERROR: at offset %#" PRIx64 ": can't parse %s (%u) sample, err = %d\n",
2150 file_offset, perf_event__name(event->header.type),
2151 event->header.type, ret);
2152 goto out;
2153 }
2154 sample.file_offset = file_offset;
2155 /*
2156 * evsel__parse_sample() doesn't populate machine_pid/vcpu,
2157 * which are needed by machines__find_for_cpumode() to
2158 * attribute samples to guest VMs. The SID table maps
2159 * sample IDs to the guest that owns the event.
2160 */
2161 if (perf_guest && sample.id) {
2162 struct perf_sample_id *sid = evlist__id2sid(session->evlist, sample.id);
2163
2164 if (sid) {
2165 sample.machine_pid = sid->machine_pid;
2166 sample.vcpu = sid->vcpu.cpu;
2167 }
2168 }
2169
2170 /*
2171 * Validate sample.cpu before any callback can use it as an
2172 * array index (kwork cpus_runtime, timechart cpus_cstate_*,
2173 * sched cpu_last_switched).
2174 *
2175 * When PERF_SAMPLE_CPU is absent, evsel__parse_sample() leaves
2176 * sample.cpu as (u32)-1 — a sentinel that downstream tools
2177 * (script, inject) check to identify events without CPU info.
2178 * Only check when sample.cpu was actually populated from event
2179 * data: PERF_RECORD_SAMPLE always has it when PERF_SAMPLE_CPU
2180 * is set; non-sample events only have it when sample_id_all is
2181 * enabled. Otherwise sample.cpu is the (u32)-1 sentinel from
2182 * evsel__parse_sample() and must not be validated or clamped.
2183 */
2184 if ((evsel->core.attr.sample_type & PERF_SAMPLE_CPU) &&
2185 (event->header.type == PERF_RECORD_SAMPLE ||
2186 evsel->core.attr.sample_id_all)) {
2187 int nr_cpus_avail = perf_session__env(session)->nr_cpus_avail;
2188
2189 /*
2190 * For perf.data files the MAX_NR_CPUS fallback in
2191 * perf_session__read_header() guarantees this is set.
2192 * For pipe mode, HEADER_NRCPUS may arrive late or not
2193 * at all (pre-2017 perf, third-party tools). Fall
2194 * back to MAX_NR_CPUS so the bounds check still works
2195 * against fixed-size downstream arrays.
2196 *
2197 * Do NOT write back to env: this function runs during
2198 * recording (synthesized events) when nr_cpus_avail is
2199 * legitimately 0. Writing MAX_NR_CPUS would cause
2200 * write_cpu_topology() to emit 4096 core_id/socket_id
2201 * pairs instead of the real CPU count, corrupting the
2202 * topology section in the generated perf.data.
2203 */
2204 if (nr_cpus_avail <= 0)
2205 nr_cpus_avail = MAX_NR_CPUS;
2206 /*
2207 * Cap at MAX_NR_CPUS for the bounds check — downstream
2208 * consumers use fixed-size arrays of that size. Keep
2209 * the true nr_cpus_avail in env for header parsing
2210 * (e.g. process_cpu_topology) which needs the real count.
2211 */
2212 if (nr_cpus_avail > MAX_NR_CPUS)
2213 nr_cpus_avail = MAX_NR_CPUS;
2214 if (sample.cpu >= (u32)nr_cpus_avail &&
2215 sample.cpu != (u32)-1) {
2216 /*
2217 * Warn rather than abort: synthesized events
2218 * (MMAP, COMM) lack sample_id_all data, so
2219 * parse_id_sample reads garbage from the event
2220 * payload. Clamping to 0 protects downstream
2221 * array indexing while keeping the session alive.
2222 *
2223 * Preserve (u32)-1: perf script and perf inject
2224 * use it as a sentinel for "CPU not applicable."
2225 * Downstream array users (timechart, kwork) have
2226 * their own per-callback bounds checks.
2227 */
2228 pr_warning_once("WARNING: at offset %#" PRIx64 ": sample CPU %u >= nr_cpus_avail %u, clamping to 0\n",
2229 file_offset, sample.cpu, nr_cpus_avail);
2230 sample.cpu = 0;
2231 }
2232 }
2233
2234 ret = auxtrace__process_event(session, event, &sample, tool);
2235 if (ret < 0)
2236 goto out;
2237 if (ret > 0) {
2238 ret = 0;
2239 goto out;
2240 }
2241
2242 ret = machines__deliver_event(&session->machines, session->evlist,
2243 event, &sample, tool, file_offset, file_path);
2244
2245 if (dump_trace && sample.aux_sample.size)
2246 auxtrace__dump_auxtrace_sample(session, &sample);
2247out:
2248 perf_sample__exit(&sample);
2249 return ret;
2250}
2251
2252static s64 perf_session__process_user_event(struct perf_session *session,
2253 union perf_event *event,
2254 u64 file_offset,
2255 const char *file_path)
2256{
2257 struct ordered_events *oe = &session->ordered_events;
2258 const struct perf_tool *tool = session->tool;
2259 const u32 event_size = READ_ONCE(event->header.size);
2260 struct perf_sample sample;
2261 int fd = perf_data__fd(session->data);
2262 s64 err;
2263
2264 perf_sample__init(&sample, /*all=*/true);
2265 if ((event->header.type != PERF_RECORD_COMPRESSED &&
2266 event->header.type != PERF_RECORD_COMPRESSED2) ||
2267 perf_tool__compressed_is_stub(tool))
2268 dump_event(session->evlist, event, file_offset, &sample, file_path);
2269
2270 /* These events are processed right away */
2271 switch (event->header.type) {
2272 case PERF_RECORD_HEADER_ATTR:
2273 err = tool->attr(tool, event, &session->evlist);
2274 if (err == 0) {
2275 perf_session__set_id_hdr_size(session);
2276 perf_session__set_comm_exec(session);
2277 }
2278 break;
2279 case PERF_RECORD_EVENT_UPDATE:
2280 err = tool->event_update(tool, event, &session->evlist);
2281 break;
2282 case PERF_RECORD_HEADER_EVENT_TYPE:
2283 /*
2284 * Deprecated, but we need to handle it for sake
2285 * of old data files create in pipe mode.
2286 */
2287 err = 0;
2288 break;
2289 case PERF_RECORD_HEADER_TRACING_DATA:
2290 /*
2291 * Setup for reading amidst mmap, but only when we
2292 * are in 'file' mode. The 'pipe' fd is in proper
2293 * place already.
2294 */
2295 if (!perf_data__is_pipe(session->data))
2296 lseek(fd, file_offset, SEEK_SET);
2297 err = tool->tracing_data(tool, session, event);
2298 break;
2299 case PERF_RECORD_HEADER_BUILD_ID:
2300 if (!perf_event__check_nul(event->build_id.filename,
2301 (void *)event + event_size,
2302 "HEADER_BUILD_ID", file_offset)) {
2303 err = 0;
2304 break;
2305 }
2306 err = tool->build_id(tool, session, event);
2307 break;
2308 case PERF_RECORD_FINISHED_ROUND:
2309 err = tool->finished_round(tool, event, oe);
2310 break;
2311 case PERF_RECORD_ID_INDEX:
2312 err = tool->id_index(tool, session, event);
2313 break;
2314 case PERF_RECORD_AUXTRACE_INFO:
2315 err = tool->auxtrace_info(tool, session, event);
2316 break;
2317 case PERF_RECORD_AUXTRACE:
2318 /*
2319 * Setup for reading amidst mmap, but only when we
2320 * are in 'file' mode. The 'pipe' fd is in proper
2321 * place already.
2322 */
2323 if (!perf_data__is_pipe(session->data))
2324 lseek(fd, file_offset + event_size, SEEK_SET);
2325 err = tool->auxtrace(tool, session, event);
2326 break;
2327 case PERF_RECORD_AUXTRACE_ERROR:
2328 perf_session__auxtrace_error_inc(session, event);
2329 err = tool->auxtrace_error(tool, session, event);
2330 break;
2331 case PERF_RECORD_THREAD_MAP: {
2332 u64 max_nr;
2333
2334 if (event_size < sizeof(event->thread_map)) {
2335 pr_err("ERROR: at offset %#" PRIx64 ": PERF_RECORD_THREAD_MAP: header.size (%u) too small\n",
2336 file_offset, event_size);
2337 err = -EINVAL;
2338 break;
2339 }
2340
2341 max_nr = (event_size - sizeof(event->thread_map)) /
2342 sizeof(event->thread_map.entries[0]);
2343 if (event->thread_map.nr > max_nr) {
2344 pr_err("ERROR: at offset %#" PRIx64 ": PERF_RECORD_THREAD_MAP: nr %" PRIu64 " exceeds max %" PRIu64 "\n",
2345 file_offset, (u64)event->thread_map.nr, max_nr);
2346 err = -EINVAL;
2347 break;
2348 }
2349
2350 err = tool->thread_map(tool, session, event);
2351 break;
2352 }
2353 case PERF_RECORD_CPU_MAP: {
2354 struct perf_record_cpu_map_data *data = &event->cpu_map.data;
2355 u32 payload = event_size - sizeof(event->header);
2356
2357 /*
2358 * Native-endian events are mmap'd read-only, so we
2359 * cannot clamp nr fields in place. Skip the event
2360 * if any variant overflows.
2361 */
2362 switch (data->type) {
2363 case PERF_CPU_MAP__CPUS: {
2364 u16 max_nr = (payload - offsetof(struct perf_record_cpu_map_data,
2365 cpus_data.cpu)) /
2366 sizeof(data->cpus_data.cpu[0]);
2367
2368 if (data->cpus_data.nr > max_nr) {
2369 pr_warning("WARNING: at offset %#" PRIx64 ": PERF_RECORD_CPU_MAP: nr %u exceeds payload (max %u), skipping\n",
2370 file_offset, data->cpus_data.nr, max_nr);
2371 err = 0;
2372 goto out;
2373 }
2374 break;
2375 }
2376 case PERF_CPU_MAP__MASK:
2377 if (data->mask32_data.long_size == 4) {
2378 u16 max_nr = (payload - offsetof(struct perf_record_cpu_map_data,
2379 mask32_data.mask)) /
2380 sizeof(data->mask32_data.mask[0]);
2381
2382 if (data->mask32_data.nr > max_nr) {
2383 pr_warning("WARNING: at offset %#" PRIx64 ": PERF_RECORD_CPU_MAP mask32: nr %u exceeds payload (max %u), skipping\n",
2384 file_offset, data->mask32_data.nr, max_nr);
2385 err = 0;
2386 goto out;
2387 }
2388 } else if (data->mask64_data.long_size == 8) {
2389 u16 max_nr;
2390
2391 if (payload < offsetof(struct perf_record_cpu_map_data, mask64_data.mask)) {
2392 err = 0;
2393 goto out;
2394 }
2395 max_nr = (payload - offsetof(struct perf_record_cpu_map_data,
2396 mask64_data.mask)) /
2397 sizeof(data->mask64_data.mask[0]);
2398 if (data->mask64_data.nr > max_nr) {
2399 pr_warning("WARNING: at offset %#" PRIx64 ": PERF_RECORD_CPU_MAP mask64: nr %u exceeds payload (max %u), skipping\n",
2400 file_offset, data->mask64_data.nr, max_nr);
2401 err = 0;
2402 goto out;
2403 }
2404 } else {
2405 pr_warning("WARNING: at offset %#" PRIx64 ": PERF_RECORD_CPU_MAP: unsupported long_size %u, skipping\n",
2406 file_offset, data->mask32_data.long_size);
2407 err = 0;
2408 goto out;
2409 }
2410 break;
2411 default:
2412 break;
2413 }
2414
2415 err = tool->cpu_map(tool, session, event);
2416 break;
2417 }
2418 case PERF_RECORD_STAT_CONFIG: {
2419 /* Cannot underflow: perf_event__min_size[] guarantees event_size >= sizeof */
2420 u64 max_nr = (event_size - sizeof(event->stat_config)) /
2421 sizeof(event->stat_config.data[0]);
2422
2423 /*
2424 * Native-endian events are mmap'd read-only, so we
2425 * cannot clamp nr in place. Skip the event instead.
2426 */
2427 if (event->stat_config.nr > max_nr) {
2428 pr_warning("WARNING: at offset %#" PRIx64 ": PERF_RECORD_STAT_CONFIG: nr %" PRIu64 " exceeds payload (max %" PRIu64 "), skipping\n",
2429 file_offset, (u64)event->stat_config.nr, max_nr);
2430 err = 0;
2431 goto out;
2432 }
2433
2434 err = tool->stat_config(tool, session, event);
2435 break;
2436 }
2437 case PERF_RECORD_STAT:
2438 err = tool->stat(tool, session, event);
2439 break;
2440 case PERF_RECORD_STAT_ROUND:
2441 err = tool->stat_round(tool, session, event);
2442 break;
2443 case PERF_RECORD_TIME_CONV:
2444 /*
2445 * Bounded copy: older kernels emit a shorter struct
2446 * without time_cycles/time_mask/cap_user_time_*.
2447 * Zero the rest so extended fields default to off.
2448 */
2449 memset(&session->time_conv, 0, sizeof(session->time_conv));
2450 memcpy(&session->time_conv, &event->time_conv,
2451 min((size_t)event_size, sizeof(session->time_conv)));
2452 err = tool->time_conv(tool, session, event);
2453 break;
2454 case PERF_RECORD_HEADER_FEATURE:
2455 err = tool->feature(tool, session, event);
2456 break;
2457 case PERF_RECORD_COMPRESSED:
2458 case PERF_RECORD_COMPRESSED2:
2459 err = tool->compressed(tool, session, event, file_offset, file_path);
2460 if (err)
2461 dump_event(session->evlist, event, file_offset, &sample, file_path);
2462 break;
2463 case PERF_RECORD_FINISHED_INIT:
2464 err = tool->finished_init(tool, session, event);
2465 break;
2466 case PERF_RECORD_BPF_METADATA: {
2467 u64 nr_entries, max_entries;
2468
2469 if (event_size < sizeof(event->bpf_metadata)) {
2470 pr_warning("WARNING: at offset %#" PRIx64 ": PERF_RECORD_BPF_METADATA: header.size (%u) too small, skipping\n",
2471 file_offset, event_size);
2472 err = 0;
2473 break;
2474 }
2475
2476 /*
2477 * Native-endian files are mmap'd read-only — validate
2478 * NUL-termination instead of writing.
2479 */
2480 if (strnlen(event->bpf_metadata.prog_name,
2481 BPF_PROG_NAME_LEN) == BPF_PROG_NAME_LEN) {
2482 pr_warning("WARNING: at offset %#" PRIx64 ": PERF_RECORD_BPF_METADATA: prog_name not null-terminated, skipping\n",
2483 file_offset);
2484 err = 0;
2485 break;
2486 }
2487
2488 nr_entries = READ_ONCE(event->bpf_metadata.nr_entries);
2489 max_entries = (event_size - sizeof(event->bpf_metadata)) /
2490 sizeof(event->bpf_metadata.entries[0]);
2491 if (nr_entries > max_entries) {
2492 pr_warning("WARNING: at offset %#" PRIx64 ": PERF_RECORD_BPF_METADATA: nr_entries %" PRIu64 " exceeds max %" PRIu64 ", skipping\n",
2493 file_offset, nr_entries, max_entries);
2494 err = 0;
2495 break;
2496 }
2497
2498 for (u64 i = 0; i < nr_entries; i++) {
2499 if (strnlen(event->bpf_metadata.entries[i].key,
2500 BPF_METADATA_KEY_LEN) == BPF_METADATA_KEY_LEN ||
2501 strnlen(event->bpf_metadata.entries[i].value,
2502 BPF_METADATA_VALUE_LEN) == BPF_METADATA_VALUE_LEN) {
2503 pr_warning("WARNING: at offset %#" PRIx64 ": PERF_RECORD_BPF_METADATA: entry %" PRIu64 " key/value not null-terminated, skipping\n",
2504 file_offset, i);
2505 err = 0;
2506 goto out;
2507 }
2508 }
2509
2510 err = tool->bpf_metadata(tool, session, event);
2511 break;
2512 }
2513 case PERF_RECORD_SCHEDSTAT_CPU:
2514 err = tool->schedstat_cpu(tool, session, event);
2515 break;
2516 case PERF_RECORD_SCHEDSTAT_DOMAIN:
2517 err = tool->schedstat_domain(tool, session, event);
2518 break;
2519 default:
2520 err = -EINVAL;
2521 break;
2522 }
2523out:
2524 perf_sample__exit(&sample);
2525 return err;
2526}
2527
2528int perf_session__deliver_synth_event(struct perf_session *session,
2529 union perf_event *event,
2530 struct perf_sample *sample)
2531{
2532 struct evlist *evlist = session->evlist;
2533 const struct perf_tool *tool = session->tool;
2534
2535 events_stats__inc(evlist__stats(evlist), event->header.type);
2536
2537 if (event->header.type >= PERF_RECORD_USER_TYPE_START)
2538 return perf_session__process_user_event(session, event, 0, NULL);
2539
2540 return machines__deliver_event(&session->machines, evlist, event, sample, tool, 0, NULL);
2541}
2542
2543int perf_session__deliver_synth_attr_event(struct perf_session *session,
2544 const struct perf_event_attr *attr,
2545 u64 id)
2546{
2547 union {
2548 struct {
2549 struct perf_record_header_attr attr;
2550 u64 ids[1];
2551 } attr_id;
2552 union perf_event ev;
2553 } ev = {
2554 .attr_id.attr.header.type = PERF_RECORD_HEADER_ATTR,
2555 .attr_id.attr.header.size = sizeof(ev.attr_id),
2556 .attr_id.ids[0] = id,
2557 };
2558
2559 if (attr->size != sizeof(ev.attr_id.attr.attr)) {
2560 pr_debug("Unexpected perf_event_attr size\n");
2561 return -EINVAL;
2562 }
2563 ev.attr_id.attr.attr = *attr;
2564 return perf_session__deliver_synth_event(session, &ev.ev, NULL);
2565}
2566
2567/* Caller must ensure event->header.type < PERF_RECORD_HEADER_MAX */
2568static int event_swap(union perf_event *event, bool sample_id_all)
2569{
2570 perf_event__swap_op swap = perf_event__swap_ops[event->header.type];
2571
2572 if (swap)
2573 return swap(event, sample_id_all);
2574 return 0;
2575}
2576
2577/*
2578 * Minimum event sizes indexed by type. Checked before swap and
2579 * processing so that both cross-endian and native-endian paths
2580 * are protected from accessing fields past the event boundary.
2581 * Zero means no minimum beyond the 8-byte header (already
2582 * enforced by the reader).
2583 *
2584 * These values represent the smallest event the kernel has ever
2585 * emitted for each type, so they do not reject legitimate legacy
2586 * perf.data files from older kernels. Variable-length events
2587 * use offsetof() to the first variable field; the variable
2588 * content is validated separately (e.g., perf_event__check_nul).
2589 */
2590static const u32 perf_event__min_size[PERF_RECORD_HEADER_MAX] = {
2591 /*
2592 * offsetof() + 1 for types with a trailing variable-length
2593 * string (filename, comm, path, name, msg): the +1 ensures
2594 * room for at least a null terminator. Full null-termination
2595 * within the event boundary is checked separately.
2596 *
2597 * PERF_RECORD_SAMPLE is omitted: all64_swap is bounded by
2598 * header.size, and the internal layout varies by sample_type
2599 * so a fixed minimum is not meaningful.
2600 */
2601 [PERF_RECORD_MMAP] = offsetof(struct perf_record_mmap, filename) + 1,
2602 [PERF_RECORD_LOST] = sizeof(struct perf_record_lost),
2603 [PERF_RECORD_COMM] = offsetof(struct perf_record_comm, comm) + 1,
2604 [PERF_RECORD_EXIT] = sizeof(struct perf_record_fork),
2605 [PERF_RECORD_THROTTLE] = sizeof(struct perf_record_throttle),
2606 [PERF_RECORD_UNTHROTTLE] = sizeof(struct perf_record_throttle),
2607 [PERF_RECORD_FORK] = sizeof(struct perf_record_fork),
2608 /*
2609 * The kernel dynamically sizes PERF_RECORD_READ based on
2610 * attr.read_format — only the enabled fields are emitted,
2611 * packed with no gaps. The minimum valid event has just
2612 * pid + tid + one u64 value (no optional fields).
2613 */
2614 [PERF_RECORD_READ] = offsetof(struct perf_record_read, time_enabled),
2615 [PERF_RECORD_MMAP2] = offsetof(struct perf_record_mmap2, filename) + 1,
2616 [PERF_RECORD_LOST_SAMPLES] = sizeof(struct perf_record_lost_samples),
2617 [PERF_RECORD_AUX] = sizeof(struct perf_record_aux),
2618 [PERF_RECORD_ITRACE_START] = sizeof(struct perf_record_itrace_start),
2619 [PERF_RECORD_SWITCH] = sizeof(struct perf_event_header),
2620 [PERF_RECORD_SWITCH_CPU_WIDE] = sizeof(struct perf_record_switch),
2621 [PERF_RECORD_NAMESPACES] = sizeof(struct perf_record_namespaces),
2622 [PERF_RECORD_CGROUP] = offsetof(struct perf_record_cgroup, path) + 1,
2623 [PERF_RECORD_TEXT_POKE] = sizeof(struct perf_record_text_poke_event),
2624 [PERF_RECORD_KSYMBOL] = offsetof(struct perf_record_ksymbol, name) + 1,
2625 [PERF_RECORD_BPF_EVENT] = sizeof(struct perf_record_bpf_event),
2626 [PERF_RECORD_HEADER_ATTR] = sizeof(struct perf_event_header) + PERF_ATTR_SIZE_VER0,
2627 [PERF_RECORD_HEADER_EVENT_TYPE] = sizeof(struct perf_record_header_event_type),
2628 /* Legacy events predate the __u32 pad field, accept 12-byte records */
2629 [PERF_RECORD_HEADER_TRACING_DATA] = offsetof(struct perf_record_header_tracing_data, pad),
2630 [PERF_RECORD_AUX_OUTPUT_HW_ID] = sizeof(struct perf_record_aux_output_hw_id),
2631 [PERF_RECORD_AUXTRACE_INFO] = sizeof(struct perf_record_auxtrace_info),
2632 [PERF_RECORD_AUXTRACE] = sizeof(struct perf_record_auxtrace),
2633 [PERF_RECORD_AUXTRACE_ERROR] = offsetof(struct perf_record_auxtrace_error, msg) + 1,
2634 [PERF_RECORD_THREAD_MAP] = sizeof(struct perf_record_thread_map),
2635 /*
2636 * sizeof(perf_record_cpu_map) is 20 because the outer struct
2637 * isn't packed and GCC adds 2 bytes of trailing padding.
2638 * The smallest valid variant (RANGE_CPUS) is only 16 bytes:
2639 * header(8) + type(2) + range_cpu_data(6). Per-variant
2640 * bounds are checked in the swap handler via payload.
2641 */
2642 [PERF_RECORD_CPU_MAP] = sizeof(struct perf_event_header) +
2643 sizeof(__u16) +
2644 sizeof(struct perf_record_range_cpu_map),
2645 [PERF_RECORD_STAT_CONFIG] = sizeof(struct perf_record_stat_config),
2646 [PERF_RECORD_STAT] = sizeof(struct perf_record_stat),
2647 [PERF_RECORD_STAT_ROUND] = sizeof(struct perf_record_stat_round),
2648 /*
2649 * EVENT_UPDATE has a union whose largest member (cpus)
2650 * inflates sizeof to 40, but SCALE events are only 32
2651 * and UNIT/NAME events can be even smaller. Use the
2652 * fixed header fields (header + type + id) as minimum.
2653 */
2654 [PERF_RECORD_EVENT_UPDATE] = offsetof(struct perf_record_event_update, scale),
2655 [PERF_RECORD_TIME_CONV] = offsetof(struct perf_record_time_conv, time_cycles),
2656 [PERF_RECORD_ID_INDEX] = sizeof(struct perf_record_id_index),
2657 [PERF_RECORD_HEADER_BUILD_ID] = sizeof(struct perf_record_header_build_id),
2658 [PERF_RECORD_HEADER_FEATURE] = sizeof(struct perf_record_header_feature),
2659 [PERF_RECORD_COMPRESSED2] = sizeof(struct perf_record_compressed2),
2660 [PERF_RECORD_BPF_METADATA] = sizeof(struct perf_record_bpf_metadata),
2661 [PERF_RECORD_CALLCHAIN_DEFERRED] = sizeof(struct perf_event_header) + sizeof(__u64),
2662 /*
2663 * SCHEDSTAT events have a version-dependent union after the
2664 * fixed header fields; the minimum is the base (pre-union)
2665 * portion so old and new versions both pass.
2666 */
2667 [PERF_RECORD_SCHEDSTAT_CPU] = offsetof(struct perf_record_schedstat_cpu, v15),
2668 [PERF_RECORD_SCHEDSTAT_DOMAIN] = offsetof(struct perf_record_schedstat_domain, v15),
2669};
2670
2671/*
2672 * Return true if the event is too small for its declared type.
2673 * Caller must ensure event->header.type < PERF_RECORD_HEADER_MAX.
2674 * If min is non-NULL, stores the required minimum on failure.
2675 */
2676bool perf_event__too_small(const union perf_event *event, u32 *min)
2677{
2678 u32 min_sz = perf_event__min_size[event->header.type];
2679
2680 if (min_sz && event->header.size < min_sz) {
2681 if (min)
2682 *min = min_sz;
2683 return true;
2684 }
2685
2686 return false;
2687}
2688
2689/*
2690 * Read and validate the event at @file_offset.
2691 *
2692 * Returns:
2693 * 0 — success: *event_ptr is set and safe to access.
2694 * -1 — error; check *event_ptr to decide whether to advance or abort:
2695 * *event_ptr set — event header was read but the event is
2696 * malformed (too small for its type, or byte-swap
2697 * failed). header.size is still valid, so the
2698 * caller can advance past the event.
2699 * *event_ptr NULL — fatal: couldn't read the header at all
2700 * (I/O error, offset out of range, pipe mode).
2701 * Caller must abort.
2702 */
2703int perf_session__peek_event(struct perf_session *session, off_t file_offset,
2704 void *buf, size_t buf_sz,
2705 union perf_event **event_ptr,
2706 struct perf_sample *sample)
2707{
2708 union perf_event *event;
2709 size_t hdr_sz, rest;
2710 u32 min_sz;
2711 int fd;
2712
2713 *event_ptr = NULL;
2714
2715 if (session->one_mmap && !session->header.needs_swap) {
2716 u64 offset_in_mmap;
2717
2718 /* Validate offset with integer arithmetic to avoid pointer UB */
2719 if ((u64)file_offset < session->one_mmap_offset)
2720 return -1;
2721
2722 offset_in_mmap = (u64)file_offset - session->one_mmap_offset;
2723
2724 /* Use subtraction to avoid addition overflow */
2725 if (offset_in_mmap >= session->one_mmap_size ||
2726 session->one_mmap_size - offset_in_mmap < sizeof(struct perf_event_header))
2727 return -1;
2728
2729 event = session->one_mmap_addr + offset_in_mmap;
2730
2731 if (event->header.size < sizeof(struct perf_event_header))
2732 return -1;
2733
2734 /* Ensure full event is within the mmap region */
2735 if (session->one_mmap_size - offset_in_mmap < event->header.size)
2736 return -1;
2737 } else {
2738 if (perf_data__is_pipe(session->data))
2739 return -1;
2740
2741 fd = perf_data__fd(session->data);
2742 hdr_sz = sizeof(struct perf_event_header);
2743
2744 if (buf_sz < hdr_sz)
2745 return -1;
2746
2747 if (lseek(fd, file_offset, SEEK_SET) == (off_t)-1 ||
2748 readn(fd, buf, hdr_sz) != (ssize_t)hdr_sz)
2749 return -1;
2750
2751 event = (union perf_event *)buf;
2752
2753 if (session->header.needs_swap)
2754 perf_event_header__bswap(&event->header);
2755
2756 if (event->header.size < hdr_sz || event->header.size > buf_sz)
2757 return -1;
2758
2759 buf += hdr_sz;
2760 rest = event->header.size - hdr_sz;
2761
2762 if (readn(fd, buf, rest) != (ssize_t)rest)
2763 return -1;
2764 }
2765
2766 /* Event data is fully loaded — expose so callers can advance */
2767 *event_ptr = event;
2768
2769 /*
2770 * Check alignment before type: an unaligned size misaligns the
2771 * stream for all subsequent reads regardless of event type.
2772 * Three legacy user events predate the 8-byte rule — exempt them.
2773 */
2774 if (event->header.size % sizeof(u64) &&
2775 event->header.type != PERF_RECORD_HEADER_TRACING_DATA &&
2776 event->header.type != PERF_RECORD_COMPRESSED &&
2777 event->header.type != PERF_RECORD_HEADER_FEATURE) {
2778 pr_warning("WARNING: at offset %#" PRIx64 ": %s (%u) event size %u not aligned to %zu\n",
2779 (u64)file_offset, perf_event__name(event->header.type),
2780 event->header.type, event->header.size, sizeof(u64));
2781 return -1;
2782 }
2783
2784 if (event->header.type >= PERF_RECORD_HEADER_MAX) {
2785 pr_warning("WARNING: at offset %#" PRIx64 ": unsupported event type %u, skipping\n",
2786 (u64)file_offset, event->header.type);
2787 return 0;
2788 }
2789
2790 if (perf_event__too_small(event, &min_sz)) {
2791 pr_warning("WARNING: at offset %#" PRIx64 ": %s (%u) event size %u too small (min %u)\n",
2792 (u64)file_offset, perf_event__name(event->header.type),
2793 event->header.type, event->header.size, min_sz);
2794 return -1;
2795 }
2796
2797 if (session->header.needs_swap &&
2798 event_swap(event, evlist__sample_id_all(session->evlist))) {
2799 /*
2800 * The header was already swapped so header.size is
2801 * valid — expose the event so callers can advance
2802 * past this malformed entry instead of aborting.
2803 */
2804 *event_ptr = event;
2805 return -1;
2806 }
2807
2808 if (sample && event->header.type < PERF_RECORD_USER_TYPE_START &&
2809 evlist__parse_sample(session->evlist, event, sample))
2810 return -1;
2811
2812 return 0;
2813}
2814
2815int perf_session__peek_events(struct perf_session *session, u64 offset,
2816 u64 size, peek_events_cb_t cb, void *data)
2817{
2818 u64 max_offset = offset + size;
2819 char buf[PERF_SAMPLE_MAX_SIZE];
2820 union perf_event *event;
2821 int err;
2822
2823 do {
2824 event = NULL;
2825 err = perf_session__peek_event(session, offset, buf,
2826 PERF_SAMPLE_MAX_SIZE, &event,
2827 NULL);
2828 if (err) {
2829 /*
2830 * Recoverable error: peek_event returns -1 but
2831 * sets event_ptr when the header was read
2832 * successfully but the event is malformed (too
2833 * small or swap failed). Skip past it using
2834 * header.size — don't invoke the callback since
2835 * type-specific fields may be truncated.
2836 *
2837 * Must abort if: event_ptr is NULL (I/O error),
2838 * size is 0 (can't advance), type is AUXTRACE
2839 * (payload extends beyond header.size), or size
2840 * is unaligned (would misalign all subsequent reads).
2841 *
2842 * Direct callers (auxtrace, cs-etm) treat any
2843 * non-zero return as fatal — only this loop skips.
2844 */
2845 if (event && event->header.size &&
2846 event->header.type != PERF_RECORD_AUXTRACE &&
2847 event->header.size % sizeof(u64) == 0) {
2848 offset += event->header.size;
2849 err = 0;
2850 } else {
2851 return err;
2852 }
2853 continue;
2854 }
2855
2856 err = cb(session, event, offset, data);
2857 if (err)
2858 return err;
2859
2860 offset += event->header.size;
2861 if (event->header.type == PERF_RECORD_AUXTRACE)
2862 offset += event->auxtrace.size;
2863
2864 } while (offset < max_offset);
2865
2866 return err;
2867}
2868
2869static s64 perf_session__process_event(struct perf_session *session,
2870 union perf_event *event, u64 file_offset,
2871 const char *file_path)
2872{
2873 struct evlist *evlist = session->evlist;
2874 const struct perf_tool *tool = session->tool;
2875 u32 min_sz;
2876 int ret;
2877
2878 /*
2879 * The kernel aligns all event sizes to sizeof(u64) — see
2880 * perf_event_comm_event() (ALIGN), perf_event_mmap_event(),
2881 * perf_event_cgroup(), perf_event_ksymbol() (IS_ALIGNED loops),
2882 * and perf_event_text_poke() (ALIGN) in kernel/events/core.c.
2883 *
2884 * An unaligned size means the file is corrupted or crafted.
2885 * Abort: there is no point continuing to read unaligned records
2886 * because the caller advances rd->head by event->header.size,
2887 * so every subsequent read would start at a misaligned offset,
2888 * producing garbage headers for the rest of the file.
2889 *
2890 * Exempt three legacy user events that predate the alignment rule:
2891 *
2892 * TRACING_DATA (66): struct tracing_data_event was 12 bytes before
2893 * b39c915a4f36 ("libperf event: Ensure tracing data is multiple
2894 * of 8 sized") added __u32 pad; old perf.data files still contain
2895 * 12-byte records.
2896 * TODO: introduce HEADER_TRACING_DATA2 with guaranteed alignment.
2897 *
2898 * COMPRESSED (81): raw ZSTD output, arbitrary length. Already
2899 * superseded by COMPRESSED2 (83) with PERF_ALIGN.
2900 *
2901 * HEADER_FEATURE (80): do_write_string() uses a 4-byte length
2902 * prefix with no padding to 8-byte total.
2903 * TODO: introduce HEADER_FEATURE2 with guaranteed alignment.
2904 */
2905 if (event->header.size % sizeof(u64) &&
2906 event->header.type != PERF_RECORD_HEADER_TRACING_DATA &&
2907 event->header.type != PERF_RECORD_COMPRESSED &&
2908 event->header.type != PERF_RECORD_HEADER_FEATURE) {
2909 pr_err("ERROR: at offset %#" PRIx64 ": %s (%u) event size %u is not 8-byte aligned, aborting\n",
2910 file_offset, perf_event__name(event->header.type),
2911 event->header.type, event->header.size);
2912 return -EINVAL;
2913 }
2914
2915 if (event->header.type >= PERF_RECORD_HEADER_MAX) {
2916 /* This perf is outdated and does not support the latest event type. */
2917 ui__warning("Unsupported header type %u, please consider updating perf.\n",
2918 event->header.type);
2919 /*
2920 * Return 0 to skip: the caller (reader__read_event)
2921 * already advances by event->header.size.
2922 */
2923 return 0;
2924 }
2925
2926 /*
2927 * Skip rather than abort: a too-small-but-aligned event
2928 * can be safely stepped over without misaligning the stream.
2929 */
2930 if (perf_event__too_small(event, &min_sz)) {
2931 pr_warning("WARNING: at offset %#" PRIx64 ": %s (%u) event size %u too small (min %u), skipping\n",
2932 file_offset, perf_event__name(event->header.type),
2933 event->header.type, event->header.size, min_sz);
2934 return 0;
2935 }
2936
2937 if (session->header.needs_swap &&
2938 event_swap(event, evlist__sample_id_all(evlist))) {
2939 pr_warning("WARNING: at offset %#" PRIx64 ": swap failed for %s (%u) event, skipping\n",
2940 file_offset, perf_event__name(event->header.type),
2941 event->header.type);
2942 return 0;
2943 }
2944
2945 events_stats__inc(evlist__stats(evlist), event->header.type);
2946
2947 if (event->header.type >= PERF_RECORD_USER_TYPE_START)
2948 return perf_session__process_user_event(session, event, file_offset, file_path);
2949
2950 if (tool->ordered_events) {
2951 u64 timestamp = -1ULL;
2952
2953 ret = evlist__parse_sample_timestamp(evlist, event, &timestamp);
2954 if (ret && ret != -1)
2955 return ret;
2956
2957 ret = perf_session__queue_event(session, event, timestamp, file_offset, file_path);
2958 if (ret != -ETIME)
2959 return ret;
2960 }
2961
2962 return perf_session__deliver_event(session, event, tool, file_offset, file_path);
2963}
2964
2965void perf_event_header__bswap(struct perf_event_header *hdr)
2966{
2967 hdr->type = bswap_32(hdr->type);
2968 hdr->misc = bswap_16(hdr->misc);
2969 hdr->size = bswap_16(hdr->size);
2970}
2971
2972struct thread *perf_session__findnew(struct perf_session *session, pid_t pid)
2973{
2974 return machine__findnew_thread(&session->machines.host, -1, pid);
2975}
2976
2977int perf_session__register_idle_thread(struct perf_session *session)
2978{
2979 struct thread *thread = machine__idle_thread(&session->machines.host);
2980
2981 /* machine__idle_thread() got the thread, so put it */
2982 thread__put(thread);
2983 return thread ? 0 : -1;
2984}
2985
2986static void
2987perf_session__warn_order(const struct perf_session *session)
2988{
2989 const struct ordered_events *oe = &session->ordered_events;
2990 struct evsel *evsel;
2991 bool should_warn = true;
2992
2993 evlist__for_each_entry(session->evlist, evsel) {
2994 if (evsel->core.attr.write_backward)
2995 should_warn = false;
2996 }
2997
2998 if (!should_warn)
2999 return;
3000 if (oe->nr_unordered_events != 0)
3001 ui__warning("%u out of order events recorded.\n", oe->nr_unordered_events);
3002}
3003
3004static void perf_session__warn_about_errors(const struct perf_session *session)
3005{
3006 const struct events_stats *stats = evlist__stats(session->evlist);
3007
3008 if (session->tool->lost == perf_event__process_lost &&
3009 stats->nr_events[PERF_RECORD_LOST] != 0) {
3010 ui__warning("Processed %d events and lost %d chunks!\n\n"
3011 "Check IO/CPU overload!\n\n",
3012 stats->nr_events[0],
3013 stats->nr_events[PERF_RECORD_LOST]);
3014 }
3015
3016 if (session->tool->lost_samples == perf_event__process_lost_samples) {
3017 double drop_rate;
3018
3019 drop_rate = (double)stats->total_lost_samples /
3020 (double) (stats->nr_events[PERF_RECORD_SAMPLE] + stats->total_lost_samples);
3021 if (drop_rate > 0.05) {
3022 ui__warning("Processed %" PRIu64 " samples and lost %3.2f%%!\n\n",
3023 stats->nr_events[PERF_RECORD_SAMPLE] + stats->total_lost_samples,
3024 drop_rate * 100.0);
3025 }
3026 }
3027
3028 if (session->tool->aux == perf_event__process_aux &&
3029 stats->total_aux_lost != 0) {
3030 ui__warning("AUX data lost %" PRIu64 " times out of %u!\n\n",
3031 stats->total_aux_lost,
3032 stats->nr_events[PERF_RECORD_AUX]);
3033 }
3034
3035 if (session->tool->aux == perf_event__process_aux &&
3036 stats->total_aux_partial != 0) {
3037 bool vmm_exclusive = false;
3038
3039 (void)sysfs__read_bool("module/kvm_intel/parameters/vmm_exclusive",
3040 &vmm_exclusive);
3041
3042 ui__warning("AUX data had gaps in it %" PRIu64 " times out of %u!\n\n"
3043 "Are you running a KVM guest in the background?%s\n\n",
3044 stats->total_aux_partial,
3045 stats->nr_events[PERF_RECORD_AUX],
3046 vmm_exclusive ?
3047 "\nReloading kvm_intel module with vmm_exclusive=0\n"
3048 "will reduce the gaps to only guest's timeslices." :
3049 "");
3050 }
3051
3052 if (session->tool->aux == perf_event__process_aux &&
3053 stats->total_aux_collision != 0) {
3054 ui__warning("AUX data detected collision %" PRIu64 " times out of %u!\n\n",
3055 stats->total_aux_collision,
3056 stats->nr_events[PERF_RECORD_AUX]);
3057 }
3058
3059 if (stats->nr_unknown_events != 0) {
3060 ui__warning("Found %u unknown events!\n\n"
3061 "Is this an older tool processing a perf.data "
3062 "file generated by a more recent tool?\n\n"
3063 "If that is not the case, consider "
3064 "reporting to [email protected].\n\n",
3065 stats->nr_unknown_events);
3066 }
3067
3068 if (stats->nr_unknown_id != 0) {
3069 ui__warning("%u samples with id not present in the header\n",
3070 stats->nr_unknown_id);
3071 }
3072
3073 if (stats->nr_invalid_chains != 0) {
3074 ui__warning("Found invalid callchains!\n\n"
3075 "%u out of %u events were discarded for this reason.\n\n"
3076 "Consider reporting to [email protected].\n\n",
3077 stats->nr_invalid_chains,
3078 stats->nr_events[PERF_RECORD_SAMPLE]);
3079 }
3080
3081 if (stats->nr_unprocessable_samples != 0) {
3082 ui__warning("%u unprocessable samples recorded.\n"
3083 "Do you have a KVM guest running and not using 'perf kvm'?\n",
3084 stats->nr_unprocessable_samples);
3085 }
3086
3087 perf_session__warn_order(session);
3088
3089 events_stats__auxtrace_error_warn(stats);
3090
3091 if (stats->nr_proc_map_timeout != 0) {
3092 ui__warning("%d map information files for pre-existing threads were\n"
3093 "not processed, if there are samples for addresses they\n"
3094 "will not be resolved, you may find out which are these\n"
3095 "threads by running with -v and redirecting the output\n"
3096 "to a file.\n"
3097 "The time limit to process proc map is too short?\n"
3098 "Increase it by --proc-map-timeout\n",
3099 stats->nr_proc_map_timeout);
3100 }
3101}
3102
3103static int perf_session__flush_thread_stack(struct thread *thread,
3104 void *p __maybe_unused)
3105{
3106 return thread_stack__flush(thread);
3107}
3108
3109static int perf_session__flush_thread_stacks(struct perf_session *session)
3110{
3111 return machines__for_each_thread(&session->machines,
3112 perf_session__flush_thread_stack,
3113 NULL);
3114}
3115
3116volatile sig_atomic_t session_done;
3117
3118static int __perf_session__process_decomp_events(struct perf_session *session);
3119
3120static int __perf_session__process_pipe_events(struct perf_session *session)
3121{
3122 struct ordered_events *oe = &session->ordered_events;
3123 const struct perf_tool *tool = session->tool;
3124 struct ui_progress prog;
3125 union perf_event *event;
3126 uint32_t size, cur_size = 0;
3127 void *buf = NULL;
3128 s64 skip = 0;
3129 u64 head;
3130 ssize_t err;
3131 void *p;
3132 bool update_prog = false;
3133
3134 /*
3135 * If it's from a file saving pipe data (by redirection), it would have
3136 * a file name other than "-". Then we can get the total size and show
3137 * the progress.
3138 */
3139 if (strcmp(session->data->path, "-") && session->data->file.size) {
3140 ui_progress__init_size(&prog, session->data->file.size,
3141 "Processing events...");
3142 update_prog = true;
3143 }
3144
3145 head = 0;
3146 cur_size = sizeof(union perf_event);
3147
3148 buf = malloc(cur_size);
3149 if (!buf)
3150 return -errno;
3151 ordered_events__set_copy_on_queue(oe, true);
3152more:
3153 event = buf;
3154 err = perf_data__read(session->data, event,
3155 sizeof(struct perf_event_header));
3156 if (err <= 0) {
3157 if (err == 0)
3158 goto done;
3159
3160 pr_err("failed to read event header\n");
3161 goto out_err;
3162 }
3163
3164 if (session->header.needs_swap)
3165 perf_event_header__bswap(&event->header);
3166
3167 size = event->header.size;
3168 if (size < sizeof(struct perf_event_header)) {
3169 pr_err("bad event header size\n");
3170 goto out_err;
3171 }
3172
3173 if (size > cur_size) {
3174 void *new = realloc(buf, size);
3175 if (!new) {
3176 pr_err("failed to allocate memory to read event\n");
3177 goto out_err;
3178 }
3179 buf = new;
3180 cur_size = size;
3181 event = buf;
3182 }
3183 p = event;
3184 p += sizeof(struct perf_event_header);
3185
3186 if (size - sizeof(struct perf_event_header)) {
3187 err = perf_data__read(session->data, p,
3188 size - sizeof(struct perf_event_header));
3189 if (err <= 0) {
3190 if (err == 0) {
3191 pr_err("unexpected end of event stream\n");
3192 goto done;
3193 }
3194
3195 pr_err("failed to read event data\n");
3196 goto out_err;
3197 }
3198 }
3199
3200 if ((skip = perf_session__process_event(session, event, head, "pipe")) < 0) {
3201 pr_err("%#" PRIx64 " [%#x]: piped event processing failed for event of type: %s (%d)\n",
3202 head, event->header.size,
3203 perf_event__name(event->header.type),
3204 event->header.type);
3205 err = -EINVAL;
3206 goto out_err;
3207 }
3208
3209 head += size;
3210
3211 if (skip > 0)
3212 head += skip;
3213
3214 err = __perf_session__process_decomp_events(session);
3215 if (err)
3216 goto out_err;
3217
3218 if (update_prog)
3219 ui_progress__update(&prog, size);
3220
3221 if (!session_done())
3222 goto more;
3223done:
3224 /* do the final flush for ordered samples */
3225 err = ordered_events__flush(oe, OE_FLUSH__FINAL);
3226 if (err)
3227 goto out_err;
3228 err = session__flush_deferred_samples(session, tool);
3229 if (err)
3230 goto out_err;
3231 err = auxtrace__flush_events(session, tool);
3232 if (err)
3233 goto out_err;
3234 err = perf_session__flush_thread_stacks(session);
3235out_err:
3236 free(buf);
3237 if (update_prog)
3238 ui_progress__finish();
3239 if (!tool->no_warn)
3240 perf_session__warn_about_errors(session);
3241 ordered_events__free(&session->ordered_events);
3242 auxtrace__free_events(session);
3243 return err;
3244}
3245
3246static union perf_event *
3247prefetch_event(char *buf, u64 head, size_t mmap_size,
3248 bool needs_swap, union perf_event *error)
3249{
3250 union perf_event *event;
3251 u16 event_size;
3252
3253 /*
3254 * Ensure we have enough space remaining to read
3255 * the size of the event in the headers.
3256 */
3257 if (head + sizeof(event->header) > mmap_size)
3258 return NULL;
3259
3260 event = (union perf_event *)(buf + head);
3261 if (needs_swap)
3262 perf_event_header__bswap(&event->header);
3263
3264 event_size = event->header.size;
3265 if (head + event_size <= mmap_size)
3266 return event;
3267
3268 /* We're not fetching the event so swap back again */
3269 if (needs_swap)
3270 perf_event_header__bswap(&event->header);
3271
3272 /* Check if the event fits into the next mmapped buf. */
3273 if (event_size <= mmap_size - head % page_size) {
3274 /* Remap buf and fetch again. */
3275 return NULL;
3276 }
3277
3278 /* Invalid input. Event size should never exceed mmap_size. */
3279 pr_debug("%s: head=%#" PRIx64 " event->header.size=%#x, mmap_size=%#zx:"
3280 " fuzzed or compressed perf.data?\n", __func__, head, event_size, mmap_size);
3281
3282 return error;
3283}
3284
3285static union perf_event *
3286fetch_mmaped_event(u64 head, size_t mmap_size, char *buf, bool needs_swap)
3287{
3288 return prefetch_event(buf, head, mmap_size, needs_swap, ERR_PTR(-EINVAL));
3289}
3290
3291static union perf_event *
3292fetch_decomp_event(u64 head, size_t mmap_size, char *buf, bool needs_swap)
3293{
3294 return prefetch_event(buf, head, mmap_size, needs_swap, NULL);
3295}
3296
3297static int __perf_session__process_decomp_events(struct perf_session *session)
3298{
3299 s64 skip;
3300 u64 size;
3301 struct decomp *decomp = session->active_decomp->decomp_last;
3302
3303 if (!decomp)
3304 return 0;
3305
3306 while (decomp->head < decomp->size && !session_done()) {
3307 union perf_event *event = fetch_decomp_event(decomp->head, decomp->size, decomp->data,
3308 session->header.needs_swap);
3309
3310 if (!event)
3311 break;
3312
3313 size = event->header.size;
3314
3315 if (size < sizeof(struct perf_event_header) ||
3316 (skip = perf_session__process_event(session, event, decomp->file_pos,
3317 decomp->file_path)) < 0) {
3318 pr_err("%#" PRIx64 " [%#x]: decompress event processing failed for event of type: %s (%d)\n",
3319 decomp->file_pos + decomp->head, event->header.size,
3320 perf_event__name(event->header.type),
3321 event->header.type);
3322 return -EINVAL;
3323 }
3324
3325 if (skip)
3326 size += skip;
3327
3328 decomp->head += size;
3329 }
3330
3331 return 0;
3332}
3333
3334/*
3335 * On 64bit we can mmap the data file in one go. No need for tiny mmap
3336 * slices. On 32bit we use 32MB.
3337 */
3338#if BITS_PER_LONG == 64
3339#define MMAP_SIZE ULLONG_MAX
3340#define NUM_MMAPS 1
3341#else
3342#define MMAP_SIZE (32 * 1024 * 1024ULL)
3343#define NUM_MMAPS 128
3344#endif
3345
3346struct reader;
3347
3348typedef s64 (*reader_cb_t)(struct perf_session *session,
3349 union perf_event *event,
3350 u64 file_offset,
3351 const char *file_path);
3352
3353struct reader {
3354 int fd;
3355 const char *path;
3356 u64 data_size;
3357 u64 data_offset;
3358 reader_cb_t process;
3359 bool in_place_update;
3360 char *mmaps[NUM_MMAPS];
3361 size_t mmap_size;
3362 int mmap_idx;
3363 char *mmap_cur;
3364 u64 file_pos;
3365 u64 file_offset;
3366 u64 head;
3367 u64 size;
3368 bool done;
3369 struct zstd_data zstd_data;
3370 struct decomp_data decomp_data;
3371};
3372
3373static int
3374reader__init(struct reader *rd, bool *one_mmap)
3375{
3376 u64 data_size = rd->data_size;
3377 char **mmaps = rd->mmaps;
3378
3379 rd->head = rd->data_offset;
3380 data_size += rd->data_offset;
3381
3382 rd->mmap_size = MMAP_SIZE;
3383 if (rd->mmap_size > data_size) {
3384 rd->mmap_size = data_size;
3385 if (one_mmap)
3386 *one_mmap = true;
3387 }
3388
3389 memset(mmaps, 0, sizeof(rd->mmaps));
3390
3391 if (zstd_init(&rd->zstd_data, 0))
3392 return -1;
3393 rd->decomp_data.zstd_decomp = &rd->zstd_data;
3394
3395 return 0;
3396}
3397
3398static void
3399reader__release_decomp(struct reader *rd)
3400{
3401 perf_decomp__release_events(rd->decomp_data.decomp);
3402 zstd_fini(&rd->zstd_data);
3403}
3404
3405static int
3406reader__mmap(struct reader *rd, struct perf_session *session)
3407{
3408 int mmap_prot, mmap_flags;
3409 char *buf, **mmaps = rd->mmaps;
3410 u64 page_offset;
3411
3412 /*
3413 * Native-endian: MAP_SHARED + PROT_READ — the kernel
3414 * guarantees page-level coherence but a concurrent writer
3415 * could modify the file between validation and use. This
3416 * is a theoretical TOCTOU that affects the entire perf.data
3417 * processing pipeline; fixing it would require copying each
3418 * event to a private buffer before processing.
3419 *
3420 * Cross-endian: MAP_PRIVATE + PROT_WRITE — swap handlers
3421 * get a copy-on-write snapshot immune to concurrent writes.
3422 */
3423 mmap_prot = PROT_READ;
3424 mmap_flags = MAP_SHARED;
3425
3426 if (rd->in_place_update) {
3427 mmap_prot |= PROT_WRITE;
3428 } else if (session->header.needs_swap) {
3429 mmap_prot |= PROT_WRITE;
3430 mmap_flags = MAP_PRIVATE;
3431 }
3432
3433 if (mmaps[rd->mmap_idx]) {
3434 munmap(mmaps[rd->mmap_idx], rd->mmap_size);
3435 mmaps[rd->mmap_idx] = NULL;
3436 }
3437
3438 page_offset = page_size * (rd->head / page_size);
3439 rd->file_offset += page_offset;
3440 rd->head -= page_offset;
3441
3442 buf = mmap(NULL, rd->mmap_size, mmap_prot, mmap_flags, rd->fd,
3443 rd->file_offset);
3444 if (buf == MAP_FAILED) {
3445 pr_err("failed to mmap file\n");
3446 return -errno;
3447 }
3448 mmaps[rd->mmap_idx] = rd->mmap_cur = buf;
3449 rd->mmap_idx = (rd->mmap_idx + 1) & (ARRAY_SIZE(rd->mmaps) - 1);
3450 rd->file_pos = rd->file_offset + rd->head;
3451 if (session->one_mmap) {
3452 session->one_mmap_addr = buf;
3453 session->one_mmap_offset = rd->file_offset;
3454 /*
3455 * mmap_size was set to the full file extent (data_offset +
3456 * data_size) but file_offset was shifted forward by
3457 * page_offset for page alignment. Reduce by page_offset
3458 * so the bounds check reflects the file-backed portion
3459 * of the mapping — pages beyond the file cause SIGBUS.
3460 */
3461 session->one_mmap_size = rd->mmap_size - page_offset;
3462 }
3463
3464 return 0;
3465}
3466
3467enum {
3468 READER_OK,
3469 READER_NODATA,
3470};
3471
3472static int
3473reader__read_event(struct reader *rd, struct perf_session *session,
3474 struct ui_progress *prog)
3475{
3476 u64 size;
3477 int err = READER_OK;
3478 union perf_event *event;
3479 s64 skip;
3480
3481 event = fetch_mmaped_event(rd->head, rd->mmap_size, rd->mmap_cur,
3482 session->header.needs_swap);
3483 if (IS_ERR(event))
3484 return PTR_ERR(event);
3485
3486 if (!event)
3487 return READER_NODATA;
3488
3489 size = event->header.size;
3490
3491 skip = -EINVAL;
3492
3493 if (size < sizeof(struct perf_event_header) ||
3494 (skip = rd->process(session, event, rd->file_pos, rd->path)) < 0) {
3495 errno = -skip;
3496 pr_err("%#" PRIx64 " [%#x]: processing failed for event of type: %s (%d) [%m]\n",
3497 rd->file_offset + rd->head, event->header.size,
3498 perf_event__name(event->header.type),
3499 event->header.type);
3500 err = skip;
3501 goto out;
3502 }
3503
3504 if (skip)
3505 size += skip;
3506
3507 rd->size += size;
3508 rd->head += size;
3509 rd->file_pos += size;
3510
3511 err = __perf_session__process_decomp_events(session);
3512 if (err)
3513 goto out;
3514
3515 ui_progress__update(prog, size);
3516
3517out:
3518 return err;
3519}
3520
3521static inline bool
3522reader__eof(struct reader *rd)
3523{
3524 return (rd->file_pos >= rd->data_size + rd->data_offset);
3525}
3526
3527static int
3528reader__process_events(struct reader *rd, struct perf_session *session,
3529 struct ui_progress *prog)
3530{
3531 int err;
3532
3533 err = reader__init(rd, &session->one_mmap);
3534 if (err)
3535 goto out;
3536
3537 session->active_decomp = &rd->decomp_data;
3538
3539remap:
3540 err = reader__mmap(rd, session);
3541 if (err)
3542 goto out;
3543
3544more:
3545 err = reader__read_event(rd, session, prog);
3546 if (err < 0)
3547 goto out;
3548 else if (err == READER_NODATA)
3549 goto remap;
3550
3551 if (session_done())
3552 goto out;
3553
3554 if (!reader__eof(rd))
3555 goto more;
3556
3557out:
3558 session->active_decomp = &session->decomp_data;
3559 return err;
3560}
3561
3562static s64 process_simple(struct perf_session *session,
3563 union perf_event *event,
3564 u64 file_offset,
3565 const char *file_path)
3566{
3567 return perf_session__process_event(session, event, file_offset, file_path);
3568}
3569
3570static int __perf_session__process_events(struct perf_session *session)
3571{
3572 struct reader rd = {
3573 .fd = perf_data__fd(session->data),
3574 .path = session->data->file.path,
3575 .data_size = session->header.data_size,
3576 .data_offset = session->header.data_offset,
3577 .process = process_simple,
3578 .in_place_update = session->data->in_place_update,
3579 };
3580 struct ordered_events *oe = &session->ordered_events;
3581 const struct perf_tool *tool = session->tool;
3582 struct ui_progress prog;
3583 int err;
3584
3585 if (rd.data_size == 0)
3586 return -1;
3587
3588 ui_progress__init_size(&prog, rd.data_size, "Processing events...");
3589
3590 err = reader__process_events(&rd, session, &prog);
3591 if (err)
3592 goto out_err;
3593 /* do the final flush for ordered samples */
3594 err = ordered_events__flush(oe, OE_FLUSH__FINAL);
3595 if (err)
3596 goto out_err;
3597 err = auxtrace__flush_events(session, tool);
3598 if (err)
3599 goto out_err;
3600 err = session__flush_deferred_samples(session, tool);
3601 if (err)
3602 goto out_err;
3603 err = perf_session__flush_thread_stacks(session);
3604out_err:
3605 ui_progress__finish();
3606 if (!tool->no_warn)
3607 perf_session__warn_about_errors(session);
3608 /*
3609 * We may switching perf.data output, make ordered_events
3610 * reusable.
3611 */
3612 ordered_events__reinit(&session->ordered_events);
3613 auxtrace__free_events(session);
3614 reader__release_decomp(&rd);
3615 session->one_mmap = false;
3616 return err;
3617}
3618
3619/*
3620 * Processing 2 MB of data from each reader in sequence,
3621 * because that's the way the ordered events sorting works
3622 * most efficiently.
3623 */
3624#define READER_MAX_SIZE (2 * 1024 * 1024)
3625
3626/*
3627 * This function reads, merge and process directory data.
3628 * It assumens the version 1 of directory data, where each
3629 * data file holds per-cpu data, already sorted by kernel.
3630 */
3631static int __perf_session__process_dir_events(struct perf_session *session)
3632{
3633 struct perf_data *data = session->data;
3634 const struct perf_tool *tool = session->tool;
3635 int i, ret, readers, nr_readers;
3636 struct ui_progress prog;
3637 u64 total_size = perf_data__size(session->data);
3638 struct reader *rd;
3639
3640 ui_progress__init_size(&prog, total_size, "Processing events...");
3641
3642 nr_readers = 1;
3643 for (i = 0; i < data->dir.nr; i++) {
3644 if (data->dir.files[i].size)
3645 nr_readers++;
3646 }
3647
3648 rd = calloc(nr_readers, sizeof(struct reader));
3649 if (!rd)
3650 return -ENOMEM;
3651
3652 rd[0] = (struct reader) {
3653 .fd = perf_data__fd(session->data),
3654 .path = session->data->file.path,
3655 .data_size = session->header.data_size,
3656 .data_offset = session->header.data_offset,
3657 .process = process_simple,
3658 .in_place_update = session->data->in_place_update,
3659 };
3660 ret = reader__init(&rd[0], NULL);
3661 if (ret)
3662 goto out_err;
3663 ret = reader__mmap(&rd[0], session);
3664 if (ret)
3665 goto out_err;
3666 readers = 1;
3667
3668 for (i = 0; i < data->dir.nr; i++) {
3669 if (!data->dir.files[i].size)
3670 continue;
3671 rd[readers] = (struct reader) {
3672 .fd = perf_data_file__fd(&data->dir.files[i]),
3673 .path = data->dir.files[i].path,
3674 .data_size = data->dir.files[i].size,
3675 .data_offset = 0,
3676 .process = process_simple,
3677 .in_place_update = session->data->in_place_update,
3678 };
3679 ret = reader__init(&rd[readers], NULL);
3680 if (ret)
3681 goto out_err;
3682 ret = reader__mmap(&rd[readers], session);
3683 if (ret)
3684 goto out_err;
3685 readers++;
3686 }
3687
3688 i = 0;
3689 while (readers) {
3690 if (session_done())
3691 break;
3692
3693 if (rd[i].done) {
3694 i = (i + 1) % nr_readers;
3695 continue;
3696 }
3697 if (reader__eof(&rd[i])) {
3698 rd[i].done = true;
3699 readers--;
3700 continue;
3701 }
3702
3703 session->active_decomp = &rd[i].decomp_data;
3704 ret = reader__read_event(&rd[i], session, &prog);
3705 if (ret < 0) {
3706 goto out_err;
3707 } else if (ret == READER_NODATA) {
3708 ret = reader__mmap(&rd[i], session);
3709 if (ret)
3710 goto out_err;
3711 }
3712
3713 if (rd[i].size >= READER_MAX_SIZE) {
3714 rd[i].size = 0;
3715 i = (i + 1) % nr_readers;
3716 }
3717 }
3718
3719 ret = ordered_events__flush(&session->ordered_events, OE_FLUSH__FINAL);
3720 if (ret)
3721 goto out_err;
3722
3723 ret = session__flush_deferred_samples(session, tool);
3724 if (ret)
3725 goto out_err;
3726
3727 ret = perf_session__flush_thread_stacks(session);
3728out_err:
3729 ui_progress__finish();
3730
3731 if (!tool->no_warn)
3732 perf_session__warn_about_errors(session);
3733
3734 /*
3735 * We may switching perf.data output, make ordered_events
3736 * reusable.
3737 */
3738 ordered_events__reinit(&session->ordered_events);
3739
3740 session->one_mmap = false;
3741
3742 session->active_decomp = &session->decomp_data;
3743 for (i = 0; i < nr_readers; i++)
3744 reader__release_decomp(&rd[i]);
3745 zfree(&rd);
3746
3747 return ret;
3748}
3749
3750int perf_session__process_events(struct perf_session *session)
3751{
3752 if (perf_session__register_idle_thread(session) < 0)
3753 return -ENOMEM;
3754
3755 if (perf_data__is_pipe(session->data))
3756 return __perf_session__process_pipe_events(session);
3757
3758 if (perf_data__is_dir(session->data) && session->data->dir.nr)
3759 return __perf_session__process_dir_events(session);
3760
3761 return __perf_session__process_events(session);
3762}
3763
3764bool perf_session__has_traces(struct perf_session *session, const char *msg)
3765{
3766 struct evsel *evsel;
3767
3768 evlist__for_each_entry(session->evlist, evsel) {
3769 if (evsel->core.attr.type == PERF_TYPE_TRACEPOINT)
3770 return true;
3771 }
3772
3773 pr_err("No trace sample to read. Did you call 'perf %s'?\n", msg);
3774 return false;
3775}
3776
3777bool perf_session__has_switch_events(struct perf_session *session)
3778{
3779 struct evsel *evsel;
3780
3781 evlist__for_each_entry(session->evlist, evsel) {
3782 if (evsel->core.attr.context_switch)
3783 return true;
3784 }
3785
3786 return false;
3787}
3788
3789int map__set_kallsyms_ref_reloc_sym(struct map *map, const char *symbol_name, u64 addr)
3790{
3791 char *bracket, *name;
3792 struct ref_reloc_sym *ref;
3793 struct kmap *kmap;
3794
3795 ref = zalloc(sizeof(struct ref_reloc_sym));
3796 if (ref == NULL)
3797 return -ENOMEM;
3798
3799 ref->name = name = strdup(symbol_name);
3800 if (ref->name == NULL) {
3801 free(ref);
3802 return -ENOMEM;
3803 }
3804
3805 bracket = strchr(name, ']');
3806 if (bracket)
3807 *bracket = '\0';
3808
3809 ref->addr = addr;
3810
3811 kmap = map__kmap(map);
3812 if (kmap)
3813 kmap->ref_reloc_sym = ref;
3814
3815 return 0;
3816}
3817
3818size_t perf_session__fprintf_dsos(struct perf_session *session, FILE *fp)
3819{
3820 return machines__fprintf_dsos(&session->machines, fp);
3821}
3822
3823size_t perf_session__fprintf_dsos_buildid(struct perf_session *session, FILE *fp,
3824 bool (skip)(struct dso *dso, int parm), int parm)
3825{
3826 return machines__fprintf_dsos_buildid(&session->machines, fp, skip, parm);
3827}
3828
3829size_t perf_session__fprintf_nr_events(struct perf_session *session, FILE *fp)
3830{
3831 size_t ret;
3832 const char *msg = "";
3833
3834 if (perf_header__has_feat(&session->header, HEADER_AUXTRACE))
3835 msg = " (excludes AUX area (e.g. instruction trace) decoded / synthesized events)";
3836
3837 ret = fprintf(fp, "\nAggregated stats:%s\n", msg);
3838
3839 ret += events_stats__fprintf(evlist__stats(session->evlist), fp);
3840 return ret;
3841}
3842
3843size_t perf_session__fprintf(struct perf_session *session, FILE *fp)
3844{
3845 size_t ret = machine__fprintf(&session->machines.host, fp);
3846
3847 for (struct rb_node *nd = rb_first_cached(&session->machines.guests); nd; nd = rb_next(nd)) {
3848 struct machine *pos = rb_entry(nd, struct machine, rb_node);
3849
3850 ret += machine__fprintf(pos, fp);
3851 }
3852 return ret;
3853}
3854
3855void perf_session__dump_kmaps(struct perf_session *session)
3856{
3857 int save_verbose = verbose;
3858
3859 fflush(stdout);
3860 fprintf(stderr, "Kernel and module maps:\n");
3861 verbose = 0; /* Suppress verbose to print a summary only */
3862 maps__fprintf(machine__kernel_maps(&session->machines.host), stderr);
3863 verbose = save_verbose;
3864}
3865
3866struct evsel *perf_session__find_first_evtype(struct perf_session *session,
3867 unsigned int type)
3868{
3869 struct evsel *pos;
3870
3871 evlist__for_each_entry(session->evlist, pos) {
3872 if (pos->core.attr.type == type)
3873 return pos;
3874 }
3875 return NULL;
3876}
3877
3878int perf_session__cpu_bitmap(struct perf_session *session,
3879 const char *cpu_list, unsigned long *cpu_bitmap)
3880{
3881 unsigned int i;
3882 int err = -1;
3883 struct perf_cpu_map *map;
3884 int nr_cpus = min(perf_session__env(session)->nr_cpus_avail, MAX_NR_CPUS);
3885 struct perf_cpu cpu;
3886
3887 for (i = 0; i < PERF_TYPE_MAX; ++i) {
3888 struct evsel *evsel;
3889
3890 evsel = perf_session__find_first_evtype(session, i);
3891 if (!evsel)
3892 continue;
3893
3894 if (!(evsel->core.attr.sample_type & PERF_SAMPLE_CPU)) {
3895 pr_err("File does not contain CPU events. "
3896 "Remove -C option to proceed.\n");
3897 return -1;
3898 }
3899 }
3900
3901 map = perf_cpu_map__new(cpu_list);
3902 if (map == NULL) {
3903 pr_err("Invalid cpu_list\n");
3904 return -1;
3905 }
3906
3907 perf_cpu_map__for_each_cpu(cpu, i, map) {
3908 if (cpu.cpu >= nr_cpus) {
3909 pr_err("Requested CPU %d too large. "
3910 "Consider raising MAX_NR_CPUS\n", cpu.cpu);
3911 goto out_delete_map;
3912 }
3913
3914 __set_bit(cpu.cpu, cpu_bitmap);
3915 }
3916
3917 err = 0;
3918
3919out_delete_map:
3920 perf_cpu_map__put(map);
3921 return err;
3922}
3923
3924void perf_session__fprintf_info(struct perf_session *session, FILE *fp,
3925 bool full)
3926{
3927 if (session == NULL || fp == NULL)
3928 return;
3929
3930 fprintf(fp, "# ========\n");
3931 perf_header__fprintf_info(session, fp, full);
3932 fprintf(fp, "# ========\n#\n");
3933}
3934
3935static int perf_session__register_guest(struct perf_session *session, pid_t machine_pid)
3936{
3937 struct machine *machine = machines__findnew(&session->machines, machine_pid);
3938 struct thread *thread;
3939
3940 if (!machine)
3941 return -ENOMEM;
3942
3943 machine->single_address_space = session->machines.host.single_address_space;
3944
3945 thread = machine__idle_thread(machine);
3946 if (!thread)
3947 return -ENOMEM;
3948 thread__put(thread);
3949
3950 machine->kallsyms_filename = perf_data__guest_kallsyms_name(session->data, machine_pid);
3951
3952 return 0;
3953}
3954
3955static int perf_session__set_guest_cpu(struct perf_session *session, pid_t pid,
3956 pid_t tid, int guest_cpu)
3957{
3958 struct machine *machine = &session->machines.host;
3959 struct thread *thread = machine__findnew_thread(machine, pid, tid);
3960
3961 if (!thread)
3962 return -ENOMEM;
3963 thread__set_guest_cpu(thread, guest_cpu);
3964 thread__put(thread);
3965
3966 return 0;
3967}
3968
3969int perf_event__process_id_index(const struct perf_tool *tool __maybe_unused,
3970 struct perf_session *session,
3971 union perf_event *event)
3972{
3973 struct evlist *evlist = session->evlist;
3974 struct perf_record_id_index *ie = &event->id_index;
3975 size_t sz = ie->header.size - sizeof(*ie);
3976 size_t i, nr, max_nr;
3977 size_t e1_sz = sizeof(struct id_index_entry);
3978 size_t e2_sz = sizeof(struct id_index_entry_2);
3979 size_t etot_sz = e1_sz + e2_sz;
3980 struct id_index_entry_2 *e2;
3981 pid_t last_pid = 0;
3982
3983 max_nr = sz / e1_sz;
3984 nr = ie->nr;
3985 if (nr > max_nr) {
3986 printf("Too big: nr %zu max_nr %zu\n", nr, max_nr);
3987 return -EINVAL;
3988 }
3989
3990 if (sz >= nr * etot_sz) {
3991 max_nr = sz / etot_sz;
3992 if (nr > max_nr) {
3993 printf("Too big2: nr %zu max_nr %zu\n", nr, max_nr);
3994 return -EINVAL;
3995 }
3996 e2 = (void *)ie + sizeof(*ie) + nr * e1_sz;
3997 } else {
3998 e2 = NULL;
3999 }
4000
4001 if (dump_trace)
4002 fprintf(stdout, " nr: %zu\n", nr);
4003
4004 for (i = 0; i < nr; i++, (e2 ? e2++ : 0)) {
4005 struct id_index_entry *e = &ie->entries[i];
4006 struct perf_sample_id *sid;
4007 int ret;
4008
4009 if (dump_trace) {
4010 fprintf(stdout, " ... id: %"PRI_lu64, e->id);
4011 fprintf(stdout, " idx: %"PRI_lu64, e->idx);
4012 fprintf(stdout, " cpu: %"PRI_ld64, e->cpu);
4013 fprintf(stdout, " tid: %"PRI_ld64, e->tid);
4014 if (e2) {
4015 fprintf(stdout, " machine_pid: %"PRI_ld64, e2->machine_pid);
4016 fprintf(stdout, " vcpu: %"PRI_lu64"\n", e2->vcpu);
4017 } else {
4018 fprintf(stdout, "\n");
4019 }
4020 }
4021
4022 sid = evlist__id2sid(evlist, e->id);
4023 if (!sid)
4024 return -ENOENT;
4025
4026 sid->idx = e->idx;
4027 sid->cpu.cpu = e->cpu;
4028 sid->tid = e->tid;
4029
4030 if (!e2)
4031 continue;
4032
4033 sid->machine_pid = e2->machine_pid;
4034 sid->vcpu.cpu = e2->vcpu;
4035
4036 if (!sid->machine_pid)
4037 continue;
4038
4039 if (sid->machine_pid != last_pid) {
4040 ret = perf_session__register_guest(session, sid->machine_pid);
4041 if (ret)
4042 return ret;
4043 last_pid = sid->machine_pid;
4044 perf_guest = true;
4045 }
4046
4047 ret = perf_session__set_guest_cpu(session, sid->machine_pid, e->tid, e2->vcpu);
4048 if (ret)
4049 return ret;
4050 }
4051 return 0;
4052}
4053
4054int perf_session__dsos_hit_all(struct perf_session *session)
4055{
4056 struct rb_node *nd;
4057 int err;
4058
4059 err = machine__hit_all_dsos(&session->machines.host);
4060 if (err)
4061 return err;
4062
4063 for (nd = rb_first_cached(&session->machines.guests); nd;
4064 nd = rb_next(nd)) {
4065 struct machine *pos = rb_entry(nd, struct machine, rb_node);
4066
4067 err = machine__hit_all_dsos(pos);
4068 if (err)
4069 return err;
4070 }
4071
4072 return 0;
4073}
4074
4075struct perf_env *perf_session__env(struct perf_session *session)
4076{
4077 return &session->header.env;
4078}
4079
4080struct perf_session__e_machine_cb_args {
4081 uint32_t e_flags;
4082 uint16_t e_machine;
4083};
4084
4085static int perf_session__e_machine_cb(struct thread *thread, void *_args)
4086{
4087 struct perf_session__e_machine_cb_args *args = _args;
4088
4089 args->e_machine = thread__e_machine(thread, /*machine=*/NULL, &args->e_flags);
4090 return args->e_machine != EM_NONE ? 1 : 0;
4091}
4092
4093/*
4094 * Note, a machine may have mixed 32-bit and 64-bit processes and so mixed
4095 * e_machines. Use thread__e_machine when this matters.
4096 */
4097uint16_t perf_session__e_machine(struct perf_session *session, uint32_t *e_flags)
4098{
4099 struct perf_session__e_machine_cb_args args = {
4100 .e_machine = EM_NONE,
4101 };
4102 struct perf_env *env;
4103
4104 if (!session) {
4105 /* Default to assuming a host machine. */
4106 if (e_flags)
4107 *e_flags = EF_HOST;
4108
4109 return EM_HOST;
4110 }
4111
4112 /*
4113 * Is the env caching an e_machine? If not we want to compute from the
4114 * more accurate threads.
4115 */
4116 env = perf_session__env(session);
4117 if (env && env->e_machine != EM_NONE)
4118 return perf_env__e_machine(env, e_flags);
4119
4120 /*
4121 * Compute from threads, note this is more accurate than
4122 * perf_env__e_machine that falls back on EM_HOST and doesn't consider
4123 * mixed 32-bit and 64-bit threads.
4124 */
4125 machines__for_each_thread(&session->machines,
4126 perf_session__e_machine_cb,
4127 &args);
4128
4129 if (args.e_machine != EM_NONE) {
4130 if (env) {
4131 env->e_machine = args.e_machine;
4132 env->e_flags = args.e_flags;
4133 }
4134 if (e_flags)
4135 *e_flags = args.e_flags;
4136
4137 return args.e_machine;
4138 }
4139
4140 /*
4141 * Couldn't determine from the perf_env or current set of
4142 * threads. Potentially use logic that uses the arch string otherwise
4143 * default to the host. Don't cache in the perf_env in case later
4144 * threads indicate a better ELF machine type.
4145 */
4146 return perf_env__e_machine_nocache(env, e_flags);
4147}